Cache is used to store data temporarily. No more tempfile. Solves issue #282 gae
authorTarashish Mishra <sunu0000@gmail.com>
Sun, 16 Dec 2012 00:09:30 +0530
branchgae
changeset 18757946089b94f4
parent 1874 7c794f545aaf
child 1876 19305b7bca4d
Cache is used to store data temporarily. No more tempfile. Solves issue #282
MoinMoin/items/content.py
moin.py
wikiconfig.py
wikiconfig_gae.py
     1.1 --- a/MoinMoin/items/content.py	Fri Dec 14 23:29:15 2012 +0100
     1.2 +++ b/MoinMoin/items/content.py	Sun Dec 16 00:09:30 2012 +0530
     1.3 @@ -5,6 +5,7 @@
     1.4  # Copyright: 2008,2009 MoinMoin:BastianBlank
     1.5  # Copyright: 2010 MoinMoin:ValentinJaniaut
     1.6  # Copyright: 2010 MoinMoin:DiogenesAugusto
     1.7 +# Copyright: 2012 MoinMoin:TarashishMishra
     1.8  # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
     1.9  
    1.10  """
    1.11 @@ -20,7 +21,6 @@
    1.12  import os, re, base64
    1.13  import tarfile
    1.14  import zipfile
    1.15 -import tempfile
    1.16  from StringIO import StringIO
    1.17  from array import array
    1.18  from collections import namedtuple
    1.19 @@ -426,34 +426,41 @@
    1.20              raise StorageError("tried to add unexpected member {0!r} to container item {1!r}".format(name, self.name))
    1.21          if isinstance(name, unicode):
    1.22              name = name.encode('utf-8')
    1.23 -        temp_fname = os.path.join(tempfile.gettempdir(), 'TarContainer_' +
    1.24 -                                  cache_key(usage='TarContainer', name=self.name))
    1.25 -        tf = tarfile.TarFile(temp_fname, mode='a')
    1.26 -        ti = tarfile.TarInfo(name)
    1.27 -        if isinstance(content, str):
    1.28 -            if content_length is None:
    1.29 -                content_length = len(content)
    1.30 -            content = StringIO(content) # we need a file obj
    1.31 -        elif not hasattr(content, 'read'):
    1.32 +        # cache is used to store data temporarily without touching filesystem.
    1.33 +        # tempfile stdlib module can not be used on GAE due to that restriction.
    1.34 +        tar_storage_key = cache_key(usage='tar_storage', name=self.name)
    1.35 +        cache_contents = app.cache.get(tar_storage_key)
    1.36 +        if cache_contents is None:
    1.37 +            cache_contents = {}
    1.38 +        if hasattr(content, 'read'):
    1.39 +            content.seek(0)
    1.40 +            content = content.read()
    1.41 +        elif not isinstance(content, str):
    1.42              logging.error("unsupported content object: {0!r}".format(content))
    1.43              raise StorageError("unsupported content object: {0!r}".format(content))
    1.44 -        assert content_length >= 0  # we don't want -1 interpreted as 4G-1
    1.45 -        ti.size = content_length
    1.46 -        tf.addfile(ti, content)
    1.47 -        tf_members = set(tf.getnames())
    1.48 -        tf.close()
    1.49 -        if tf_members - expected_members:
    1.50 +        cache_contents[name] = content
    1.51 +        if set(cache_contents) - expected_members:
    1.52              msg = "found unexpected members in container item {0!r}".format(self.name)
    1.53              logging.error(msg)
    1.54 -            os.remove(temp_fname)
    1.55              raise StorageError(msg)
    1.56 -        if tf_members == expected_members:
    1.57 -            # everything we expected has been added to the tar file, save the container as revision
    1.58 +        if set(cache_contents) == expected_members:
    1.59 +            # everything we expected has been added to cache_contents, save the container as revision
    1.60              meta = {CONTENTTYPE: self.contenttype}
    1.61 -            data = open(temp_fname, 'rb')
    1.62 -            self.item._save(meta, data, name=self.name, action=u'SAVE', comment='')
    1.63 -            data.close()
    1.64 -            os.remove(temp_fname)
    1.65 +            tarbuffer = StringIO()
    1.66 +            tf = tarfile.TarFile(fileobj=tarbuffer, mode='w')
    1.67 +            for name in cache_contents:
    1.68 +                content = cache_contents[name]
    1.69 +                content_length = len(content)
    1.70 +                assert content_length >= 0  # we don't want -1 interpreted as 4G-1
    1.71 +                ti = tarfile.TarInfo(name)
    1.72 +                ti.size = content_length
    1.73 +                tf.addfile(ti, StringIO(content))
    1.74 +            tf.close()
    1.75 +            self.item._save(meta, tarbuffer, name=self.name, action=u'SAVE', comment='')
    1.76 +            tarbuffer.close()
    1.77 +            app.cache.delete(tar_storage_key)
    1.78 +        else:
    1.79 +            app.cache.set(tar_storage_key, cache_contents)
    1.80  
    1.81  
    1.82  @register
     2.1 --- a/moin.py	Fri Dec 14 23:29:15 2012 +0100
     2.2 +++ b/moin.py	Sun Dec 16 00:09:30 2012 +0530
     2.3 @@ -26,7 +26,6 @@
     2.4      application = create_app(create_index=create_index)
     2.5      application.on_gae = True  # GAE specific code can check this
     2.6  
     2.7 -
     2.8  elif __name__ == '__main__':
     2.9      from MoinMoin.script import main
    2.10      main()
     3.1 --- a/wikiconfig.py	Fri Dec 14 23:29:15 2012 +0100
     3.2 +++ b/wikiconfig.py	Sun Dec 16 00:09:30 2012 +0530
     3.3 @@ -85,7 +85,7 @@
     3.4  #USE_X_SENDFILE = False
     3.5  #LOGGER_NAME = 'MoinMoin'
     3.6  #config for flask-cache:
     3.7 -#CACHE_TYPE = 'filesystem'
     3.8 +CACHE_TYPE = 'simple'
     3.9  #CACHE_DIR = '/path/to/flask-cache-dir'
    3.10  
    3.11  # DEVELOPERS! Do not add your configuration items here - you could accidentally
     4.1 --- a/wikiconfig_gae.py	Fri Dec 14 23:29:15 2012 +0100
     4.2 +++ b/wikiconfig_gae.py	Sun Dec 16 00:09:30 2012 +0530
     4.3 @@ -84,7 +84,7 @@
     4.4  #USE_X_SENDFILE = False
     4.5  #LOGGER_NAME = 'MoinMoin'
     4.6  #config for flask-cache:
     4.7 -#CACHE_TYPE = 'filesystem'
     4.8 +CACHE_TYPE = 'gaememcached'
     4.9  #CACHE_DIR = '/path/to/flask-cache-dir'
    4.10  
    4.11  # DEVELOPERS! Do not add your configuration items here - you could accidentally