Mercurial > moin > 1.9
changeset 80:99f0d19d0285
Integrated MoinMoin:PackageInstaller and zip support.
imported from: moin--main--1.5--patch-82
| author | Alexander Schremmer <alex@alexanderweb.de.tla> |
|---|---|
| date | Thu, 06 Oct 2005 16:00:49 +0000 |
| parents | e51cb51522d1 |
| children | dcbfffac3f9c |
| files | MoinMoin/_tests/test_packages.py MoinMoin/action/AttachFile.py MoinMoin/packages.py |
| diffstat | 3 files changed, 628 insertions(+), 3 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MoinMoin/_tests/test_packages.py Thu Oct 06 16:00:49 2005 +0000 @@ -0,0 +1,66 @@ +# -*- coding: iso-8859-1 -*- +""" +MoinMoin - MoinMoin.packages tests + +@copyright: 2005 MoinMoin:AlexanderSchremmer +@license: GNU GPL, see COPYING for details. +""" + +import unittest +from MoinMoin.Page import Page +from MoinMoin._tests import TestConfig +from MoinMoin._tests import TestSkiped as TestSkipped +from MoinMoin.packages import Package, ScriptEngine, MOIN_PACKAGE_FILE, packLine, unpackLine + +class DebugPackage(Package, ScriptEngine): + """ Used for debugging, does not need a real .zip file. """ + def __init__(self, request, filename): + Package.__init__(self, request) + ScriptEngine.__init__(self) + self.filename = filename + + def extract_file(self, filename): + if filename == MOIN_PACKAGE_FILE: + return u"""moinmoinpackage|1 +print|foo +ReplaceUnderlay|testdatei|TestSeite2 +DeletePage|TestSeite2|Test ... +IgnoreExceptions|True +DeletePage|TestSeiteDoesNotExist|Test ... +IgnoreExceptions|False +AddRevision|foofile|FooPage +DeletePage|FooPage|Test ... +setthemename|foo +#foobar +installplugin|foo|local|parser|testy +""".encode("utf-8") + else: + return "Hello world, I am the file " + filename.encode("utf-8") + + def filelist(self): + return [MOIN_PACKAGE_FILE, "foo"] + + def isPackage(self): + return True + +class PackagesTests(unittest.TestCase): + """ Tests various things in the packages package. Note that this package does + not care to clean up and needs to run in a test wiki because of that. """ + + def setUp(self): + if not getattr(self.request.cfg, 'is_test_wiki', False): + raise TestSkipped('This test needs to be run using the test wiki.') + + def testBasicPackageThings(self): + myPackage = DebugPackage(self.request, 'test') + myPackage.installPackage() + self.assertEqual(myPackage.msg, "foo\n") + testseite2 = Page(self.request, 'TestSeite2') + self.assertEqual(testseite2.getPageText(), "Hello world, I am the file testdatei") + self.assert_(testseite2.isUnderlayPage()) + self.assert_(not Page(self.request, 'FooPage').exists()) + +class QuotingTests(unittest.TestCase): + def testQuoting(self): + for line in ([':foo', 'is\\', 'ja|', u't|ü', u'baAzß'], [], ['', '']): + self.assertEqual(line, unpackLine(packLine(line)))
--- a/MoinMoin/action/AttachFile.py Thu Oct 06 15:56:14 2005 +0000 +++ b/MoinMoin/action/AttachFile.py Thu Oct 06 16:00:49 2005 +0000 @@ -16,15 +16,17 @@ to view the content of the file To insert an attachment into the page, use the "attachment:" pseudo - schema. + schema. @copyright: 2001 by Ken Sugino (sugino@mediaone.net) @copyright: 2001-2004 by Jürgen Hermann <jh@web.de> + @copyright: 2005 R. Bauer + @copyright: 2005 MoinMoin:AlexanderSchremmer @license: GNU GPL, see COPYING for details. """ -import os, mimetypes, time, urllib -from MoinMoin import config, user, util, wikiutil +import os, mimetypes, time, urllib, zipfile +from MoinMoin import config, user, util, wikiutil, packages from MoinMoin.Page import Page from MoinMoin.util import MoinMoinNoFooter, filesys, web @@ -203,6 +205,8 @@ label_get = _("get") label_edit = _("edit") label_view = _("view") + label_unzip = _("unzip") + label_install = _("install") for file in files: fsize = float(os.stat(os.path.join(attach_dir,file).encode(config.charset))[6]) # in byte @@ -218,6 +222,8 @@ 'urlfile': urlfile, 'label_del': label_del, 'base': base, 'label_edit': label_edit, 'label_view': label_view, + 'label_unzip': label_unzip, + 'label_install': label_install, 'get_url': get_url, 'label_get': label_get, 'file': wikiutil.escape(file), 'fsize': fsize, 'pagename': pagename} @@ -232,6 +238,15 @@ else: viewlink = '<a href="%(baseurl)s/%(urlpagename)s?action=%(action)s&do=view&target=%(urlfile)s">%(label_view)s</a>' % parmdict + if (packages.ZipPackage(request, os.path.join(attach_dir, file).encode(config.charset)).isPackage() and + request.user.name in request.cfg.superuser): + viewlink += ' | <a href="%(baseurl)s/%(urlpagename)s?action=%(action)s&do=install&target=%(urlfile)s">%(label_install)s</a>' % parmdict + elif (zipfile.is_zipfile(os.path.join(attach_dir,file).encode(config.charset)) and + request.user.may.read(pagename) and request.user.may.delete(pagename) + and request.user.may.write(pagename)): + viewlink += ' | <a href="%(baseurl)s/%(urlpagename)s?action=%(action)s&do=unzip&target=%(urlfile)s">%(label_unzip)s</a>' % parmdict + + parmdict['viewlink'] = viewlink parmdict['del_link'] = del_link str = str + ('<li>[%(del_link)s' @@ -419,6 +434,16 @@ get_file(pagename, request) else: msg = _('You are not allowed to get attachments from this page.') + elif request.form['do'][0] == 'unzip': + if request.user.may.delete(pagename) and request.user.may.read(pagename) and request.user.may.write(pagename): + unzip_file(pagename, request) + else: + msg = _('You are not allowed to unzip attachments of this page.') + elif request.form['do'][0] == 'install': + if request.user.name in request.cfg.superuser: + install_package(pagename, request) + else: + msg = _('You are not allowed to install files.') elif request.form['do'][0] == 'view': if request.user.may.read(pagename): view_file(pagename, request) @@ -587,6 +612,91 @@ raise MoinMoinNoFooter +def install_package(pagename, request): + _ = request.getText + + target, targetpath = _access_file(pagename, request) + if not target: + return + + package = packages.ZipPackage(request, targetpath) + + if package.isPackage(): + if package.installPackage(): + msg=_("Attachment '%(filename)s' installed.") % {'filename': wikiutil.escape(target)} + else: + msg=_("Installation of '%(filename)s' failed.") % {'filename': wikiutil.escape(target)} + if package.msg != "": + msg += "<br><pre>" + wikiutil.escape(package.msg) + "</pre>" + else: + msg = _('The file %s is not a MoinMoin package file.' % wikiutil.escape(target)) + + upload_form(pagename, request, msg=msg) + +def unzip_file(pagename, request): + _ = request.getText + valid_pathname = lambda name: (name.find('/') == -1) and (name.find('\\') == -1) + + filename, fpath = _access_file(pagename, request) + if not filename: return # error msg already sent in _access_file + + attachment_path = getAttachDir(request, pagename) + single_file_size = 2.0 * 1000**2 + attachments_file_space = 200.0 * 1000**2 + + files = _get_files(request, pagename) + + msg = "" + if files: + fsize = 0.0 + for file in files: + fsize += float(os.stat(getFilename(request, pagename, file))[6]) # in byte + + available_attachments_file_space = attachments_file_space - fsize + + if zipfile.is_zipfile(fpath): + zf = zipfile.ZipFile(fpath) + sum_size_over_all_valid_files = 0.0 + for name in zf.namelist(): + if valid_pathname(name): + sum_size_over_all_valid_files += zf.getinfo(name).file_size + + if sum_size_over_all_valid_files < available_attachments_file_space: + valid_name = False + for name in zf.namelist(): + if valid_pathname(name): + zi = zf.getinfo(name) + if zi.file_size < single_file_size: + new_file = getFilename(request, pagename, name) + if not os.path.exists(new_file): + outfile = open(new_file, 'wb') + outfile.write(zf.read(name)) + outfile.close() + # it's not allowed to zip a zip file so it is dropped + if zipfile.is_zipfile(new_file): + os.unlink(new_file) + else: + valid_name = True + os.chmod(new_file, 0666 & config.umask) + _addLogEntry(request, 'ATTNEW', pagename, new_file) + + if valid_name: + msg=_("Attachment '%(filename)s' unzipped.") % {'filename': filename} + else: + msg=_("Attachment '%(filename)s' not unzipped because the " + "files are too big, .zip files only, exist already or " + "reside in folders.") % {'filename': filename} + else: + msg=_("Attachment '%(filename)s' could not be unzipped because" + " the resulting files would be too large (%(space)d kB" + " missing).") % {'filename': filename, + 'space': (sum_size_over_all_valid_files - + available_attachments_file_space) / 1000} + else: + msg = _('The file %(target) is not a .zip file.' % target) + + upload_form(pagename, request, msg=wikiutil.escape(msg)) + def send_viewfile(pagename, request): _ = request.getText @@ -614,6 +724,21 @@ request.write("</pre>") return + package = packages.ZipPackage(request, fpath) + if package.isPackage(): + request.write("<pre><b>%s</b>\n%s</pre>" % (_("Package script:"),wikiutil.escape(package.getScript()))) + return + + import zipfile + if zipfile.is_zipfile(fpath): + zf = zipfile.ZipFile(fpath, mode='r') + request.write("<pre>%-46s %19s %12s\n" % (_("File Name"), _("Modified")+" "*5, _("Size"))) + for zinfo in zf.filelist: + date = "%d-%02d-%02d %02d:%02d:%02d" % zinfo.date_time + request.write(wikiutil.escape("%-46s %s %12d\n" % (zinfo.filename, date, zinfo.file_size))) + request.write("</pre>") + return + request.write('<p>' + _("Unknown file type, cannot display this attachment inline.") + '</p>') request.write('<a href="%s">%s</a>' % ( getAttachUrl(pagename, filename, request, escaped=1), wikiutil.escape(filename)))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MoinMoin/packages.py Thu Oct 06 16:00:49 2005 +0000 @@ -0,0 +1,434 @@ +# -*- coding: iso-8859-1 -*- +""" + MoinMoin - Package Installer + + @copyright: 2005 by MoinMoin:AlexanderSchremmer + @license: GNU GPL, see COPYING for details. +""" + +import os +import sys +import zipfile + +from MoinMoin import config, wikiutil, caching +from MoinMoin.Page import Page +from MoinMoin.PageEditor import PageEditor + +MOIN_PACKAGE_FILE = 'MOIN_PACKAGE' +MAX_VERSION = 1 + +# Exceptions +class PackageException(Exception): + """ Raised if the package is broken. """ + pass + +class ScriptException(Exception): + """ Raised when there is a problem in the script. """ + + def __unicode__(self): + """ Return unicode error message """ + if isinstance(self.args[0], str): + return unicode(self.args[0], config.charset) + else: + return unicode(self.args[0]) + +class RuntimeScriptException(ScriptException): + """ Raised when the script problem occurs at runtime. """ + +class ScriptExit(Exception): + """ Raised by the script commands when the script should quit. """ + +# Parsing and (un)quoting for script files +def packLine(list): + return '|'.join([x.replace('\\', '\\\\').replace('|', r'\|') for x in list]) + +def unpackLine(string): + result = [] + token = None + escaped = False + for x in string: + if token is None: + token = "" + if escaped and x in ('\\', '|'): + token += x + escaped = False + continue + escaped = (x == '\\') + if escaped: + continue + if x == '|': + result.append(token) + token = "" + else: + token += x + if token is not None: + result.append(token) + return result + +class ScriptEngine: + """ + The script engine supplies the needed commands to execute the installation + script. + """ + + def _toBoolean(string): + """ + Converts the parameter to a boolean value by recognising different + truth literals. + """ + return (string.lower() in ('yes', 'true', '1')) + _toBoolean = staticmethod(_toBoolean) + + def _extractToFile(self, source, target): + """ Extracts source and writes the contents into target. """ + # TODO, add file dates + f = open(target, "wb") + f.write(self.extract_file(source)) + f.close() + + def __init__(self): + self.themename = None + self.ignoreExceptions = False + self.goto = 0 + + def do_print(self, *param): + """ Prints the parameters into output of the script. """ + self.msg += '; '.join(param) + "\n" + + def do_exit(self): + """ Exits the script. """ + raise ScriptExit + + def do_ignoreexceptions(self, boolean): + """ Sets the ignore exceptions setting. If exceptions are ignored, the + script does not stop if one is encountered. """ + self.ignoreExceptions = self._toBoolean(boolean) + + def do_ensureversion(self, version, lines=0): + """ Ensures that the version of MoinMoin is greater or equal than + version. If lines is unspecified, the script aborts. Otherwise, + the next lines (amount specified by lines) are not executed. + + @param version: required version of MoinMoin (e.g. "1.3.4") + @param lines: lines to ignore + """ + from MoinMoin.version import release + version_int = [int(x) for x in version.split(".")] + release = [int(x) for x in release.split(".")] + if version_int > release: + if lines > 0: + self.goto = lines + else: + raise RuntimeScriptException(_("The package needs a newer version" + " of MoinMoin (at least %s).") % + version) + + def do_setthemename(self, themename): + """ Sets the name of the theme which will be altered next. """ + self.themename = wikiutil.taintfilename(str(themename)) + + def do_copythemefile(self, filename, type, target): + """ Copies a theme-related file (CSS, PNG, etc.) into a directory of the + current theme. + + @param filename: name of the file in this package + @param type: the subdirectory of the theme directory, e.g. "css" + @param target: filename, e.g. "screen.css" + """ + _ = self.request.getText + if self.themename is None: + raise RuntimeScriptException(_("The theme name is not set.")) + sa = getattr(self.request, "sareq", None) + if sa is None: + raise RuntimeScriptException(_("Installing theme files is only supported " + "for standalone type servers.")) + htdocs_dir = sa.server.htdocs + theme_file = os.path.join(htdocs_dir, self.themename, + wikiutil.taintfilename(type), + wikiutil.taintfilename(target)) + theme_dir = os.path.dirname(theme_file) + if not os.path.exists(theme_dir): + os.makedirs(theme_dir, 0777 & config.umask) + self._extractToFile(filename, theme_file) + + def do_installplugin(self, filename, visibility, ptype, target): + """ + Installs a python code file into the appropriate directory. + + @param filename: name of the file in this package + @param visibility: 'local' will copy it into the plugin folder of the + current wiki. 'global' will use the folder of the MoinMoin python + package. + @param ptype: the type of the plugin, e.g. "parser" + @param target: the filename of the plugin, e.g. wiki.py + """ + visibility = visibility.lower() + ptype = wikiutil.taintfilename(ptype.lower()) + + if visibility == 'global': + basedir = os.path.dirname(__import__("MoinMoin").__file__) + elif visibility == 'local': + basedir = self.request.cfg.plugin_dir + + target = os.path.join(basedir, ptype, wikiutil.taintfilename(target)) + + self._extractToFile(filename, target) + wikiutil._wiki_plugins = {} + + def do_installpackage(self, pagename, filename): + """ + Installs a package. + + @param pagename: Page where the file is attached. Or in 2.0, the file itself. + @param filename: Filename of the attachment (just applicable for MoinMoin < 2.0) + """ + _ = self.request.getText + + attachments = Page(self.request, pagename).getPagePath("attachments", check_create=0) + package = ZipPackage(self.request, os.path.join(attachments, wikiutil.taintfilename(filename))) + + if package.isPackage(): + if not package.installPackage(): + raise RuntimeScriptException(_("Installation of '%(filename)s' failed.") % { + 'filename': filename} + "\n" + package.msg) + else: + raise RuntimeScriptException(_('The file %s is not a MoinMoin package file.' % filename)) + + self.msg += package.msg + + def do_addrevision(self, filename, pagename, author=u"Scripting Subsystem", comment=u"", trivial = u"No"): + """ Adds a revision to a page. + + @param filename: name of the file in this package + @param pagename: name of the target page + @param author: user name of the editor (optional) + @param comment: comment related to this revision (optional) + @param trivial: boolean, if it is a trivial edit + """ + _ = self.request.getText + trivial = self._toBoolean(trivial) + + page = PageEditor(self.request, pagename, do_editor_backup=0, uid_override=author) + page.saveText(self.extract_file(filename), 0, trivial=trivial, comment=comment) + + page.clean_acl_cache() + + def do_deletepage(self, pagename, comment="Deleted by the scripting subsystem."): + """ Marks a page as deleted (like the DeletePage action). + + @param pagename: page to delete + @param comment: the related comment (optional) + """ + _ = self.request.getText + page = PageEditor(self.request, pagename, do_editor_backup=0) + if not page.exists(): + raise RuntimeScriptException(_("The page %s does not exist.") % pagename) + + page.deletePage(comment) + + def do_replaceunderlay(self, filename, pagename): + """ Overwrites underlay pages. Implementational detail: This needs to be + kept in sync with the page class. + + @param filename: name of the file in the package + @param pagename: page to be overwritten + """ + page = Page(self.request, pagename) + + pagedir = page.getPagePath(use_underlay=1, check_create=1) + + revdir = os.path.join(pagedir, 'revisions') + cfn = os.path.join(pagedir,'current') + + revstr = '%08d' % 1 + if not os.path.exists(revdir): + os.mkdir(revdir) + os.chmod(revdir, 0777 & config.umask) + + f = open(cfn, 'w') + f.write(revstr + "\n") + f.close() + os.chmod(cfn, 0666 & config.umask) + + pagefile = os.path.join(revdir, revstr) + self._extractToFile(filename, pagefile) + os.chmod(pagefile, 0666 & config.umask) + + # Clear caches + try: + del self.request.cfg.DICTS_DATA + except AttributeError: + pass + self.request.pages = {} + caching.CacheEntry(self.request, 'wikidicts', 'dicts_groups').remove() + page.clean_acl_cache() + + def runScript(self, commands): + """ Runs the commands. + + @param commands: list of strings which contain a command each + @return True on success + """ + _ = self.request.getText + + headerline = unpackLine(commands[0]) + + if headerline[0].lower() != "MoinMoinPackage".lower(): + raise PackageException(_("Invalid package file header.")) + + self.revision = int(headerline[1]) + if self.revision > MAX_VERSION: + raise PackageException(_("Package file format unsupported.")) + + lineno = 1 + success = True + + for line in commands[1:]: + lineno += 1 + if self.goto > 0: + self.goto -= 1 + continue + + if line.startswith("#"): + continue + elements = unpackLine(line) + fnname = elements[0].strip().lower() + if fnname == '': + continue + try: + fn = getattr(self, "do_" + fnname) + except AttributeError: + self.msg += u"Exception RuntimeScriptException (line %i): %s\n" % ( + lineno, _("Unknown function %s in line %i.") % (elements[0], lineno)) + success = False + break + + try: + fn(*elements[1:]) + except ScriptExit: + break + except TypeError, e: + self.msg += u"Exception %s (line %i): %s\n" % (e.__class__.__name__, lineno, unicode(e)) + success = False + break + except RuntimeScriptException, e: + if not self.ignoreExceptions: + self.msg += u"Exception %s (line %i): %s\n" % (e.__class__.__name__, lineno, unicode(e)) + success = False + break + + return success + +class Package: + """ A package consists of a bunch of files which can be installed. """ + def __init__(self, request): + self.request = request + self.msg = "" + + def installPackage(self): + """ Opens the package and executes the script. """ + + _ = self.request.getText + + if not self.isPackage(): + raise PackageException(_("The file %s was not found in the package.") % MOIN_PACKAGE_FILE) + + commands = self.getScript().splitlines() + + return self.runScript(commands) + + def getScript(self): + """ Returns the script. """ + return self.extract_file(MOIN_PACKAGE_FILE).decode("utf-8").replace(u"\ufeff", "") + + def extract_file(self, filename): + """ Returns the contents of a file in the package. """ + raise NotImplementedException + + def filelist(self): + """ Returns a list of all files. """ + raise NotImplementedException + + def isPackage(self): + """ Returns true if this package is recognised. """ + raise NotImplementedException + +class ZipPackage(Package, ScriptEngine): + """ A package that reads its files from a .zip file. """ + def __init__(self, request, filename): + """ Initialise the package. + + @param request RequestBase instance + @param filename filename of the .zip file + """ + + Package.__init__(self, request) + ScriptEngine.__init__(self) + self.filename = filename + + self._isZipfile = zipfile.is_zipfile(filename) + if self._isZipfile: + self.zipfile = zipfile.ZipFile(filename) + # self.zipfile.getinfo(name) + + def extract_file(self, filename): + """ Returns the contents of a file in the package. """ + _ = self.request.getText + try: + return self.zipfile.read(filename.encode("cp437")) + except KeyError: + raise RuntimeScriptException(_( + "The file %s was not found in the package.") % filename) + + def filelist(self): + """ Returns a list of all files. """ + return self.zipfile.namelist() + + def isPackage(self): + """ Returns true if this package is recognised. """ + return self._isZipfile and MOIN_PACKAGE_FILE in self.zipfile.namelist() + +if __name__ == '__main__': + args = sys.argv + if len(args)-1 not in (2, 3) or args[1] not in ('l', 'i'): + print >>sys.stderr, """MoinMoin Package Installer v%(version)i + +%(myname)s action packagefile [request URL] + +action - Either "l" for listing the script or "i" for installing. +packagefile - The path to the file containing the MoinMoin installer package +request URL - Just needed if you are running a wiki farm, used to differentiate + the correct wiki. + +Example: + +%(myname)s i ../package.zip + +""" % {"version": MAX_VERSION, "myname": os.path.basename(args[0])} + raise SystemExit + + packagefile = args[2] + if len(args) > 3: + request_url = args[3] + else: + request_url = "localhost/" + + # Setup MoinMoin environment + from MoinMoin.request import RequestCLI + request = RequestCLI(url = 'localhost/') + request.form = request.args = request.setup_args() + + package = ZipPackage(request, packagefile) + if not package.isPackage(): + print "The specified file %s is not a package." % packagefile + raise SystemExit + + if args[1] == 'l': + print package.getScript() + elif args[1] == 'i': + if package.installPackage(): + print "Installation was successful!" + else: + print "Installation failed." + if package.msg: + print package.msg