changeset 1261:ae9eb32b6899

Refactored code, cleaned up some parts of the code, moved some classes to wikisync
author Alexander Schremmer <alex AT alexanderweb DOT de>
date Sat, 12 Aug 2006 22:30:49 +0200
parents 5d555ec6b40a
children 16bed977b054
files MoinMoin/action/SyncPages.py MoinMoin/wikisync.py docs/CHANGES.aschremmer
diffstat 3 files changed, 278 insertions(+), 273 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/action/SyncPages.py	Fri Aug 11 22:28:14 2006 +0200
+++ b/MoinMoin/action/SyncPages.py	Sat Aug 12 22:30:49 2006 +0200
@@ -10,9 +10,7 @@
 
 import os
 import re
-import zipfile
 import xmlrpclib
-from datetime import datetime
 
 # Compatiblity to Python 2.3
 try:
@@ -26,7 +24,7 @@
 from MoinMoin.PageEditor import PageEditor, conflict_markers
 from MoinMoin.Page import Page
 from MoinMoin.wikidicts import Dict, Group
-from MoinMoin.wikisync import TagStore
+from MoinMoin.wikisync import TagStore, UnsupportedWikiException, SyncPage, MoinLocalWiki, MoinRemoteWiki
 from MoinMoin.util.bdiff import decompress, patch, compress, textdiff
 from MoinMoin.util import diff3
 
@@ -35,260 +33,8 @@
 directions_map = {"up": UP, "down": DOWN, "both": BOTH}
 
 
-def normalise_pagename(page_name, prefix):
-    """ Checks if the page_name starts with the prefix.
-        Returns None if it does not, otherwise strips the prefix.
-    """
-    if prefix:
-        if not page_name.startswith(prefix):
-            return None
-        else:
-            return page_name[len(prefix):]
-    else:
-        return page_name
-
-
 class ActionStatus(Exception): pass
 
-class UnsupportedWikiException(Exception): pass
-
-# XXX Move these classes to MoinMoin.wikisync
-class SyncPage(object):
-    """ This class represents a page in one or two wiki(s). """
-    def __init__(self, name, local_rev=None, remote_rev=None, local_name=None, remote_name=None):
-        """ Creates a SyncPage instance.
-            @param name: The canonical name of the page, without prefixes.
-            @param local_rev: The revision of the page in the local wiki.
-            @param remote_rev: The revision of the page in the remote wiki.
-            @param local_name: The page name of the page in the local wiki.
-            @param remote_name: The page name of the page in the remote wiki.
-        """
-        self.name = name
-        self.local_rev = local_rev
-        self.remote_rev = remote_rev
-        self.local_name = local_name
-        self.remote_name = remote_name
-        assert local_rev or remote_rev
-        assert local_name or remote_name
-
-    def __repr__(self):
-        return repr("<Remote Page %r>" % unicode(self))
-
-    def __unicode__(self):
-        return u"%s[%s|%s]<%r:%r>" % (self.name, self.local_name, self.remote_name, self.local_rev, self.remote_rev)
-
-    def __lt__(self, other):
-        return self.name < other.name
-
-    def __hash__(self):
-        """ Ensures that the hash value of this page only depends on the canonical name. """
-        return hash(self.name)
-
-    def __eq__(self, other):
-        if not isinstance(other, SyncPage):
-            return false
-        return self.name == other.name
-
-    def add_missing_pagename(self, local, remote):
-        """ Checks if the particular concrete page names are unknown and fills
-            them in.
-        """
-        if self.local_name is None:
-            n_name = normalise_pagename(self.remote_name, remote.prefix)
-            assert n_name is not None
-            self.local_name = (local.prefix or "") + n_name
-        elif self.remote_name is None:
-            n_name = normalise_pagename(self.local_name, local.prefix)
-            assert n_name is not None
-            self.remote_name = (local.prefix or "") + n_name
-
-        return self # makes using list comps easier
-
-    def filter(cls, sp_list, func):
-        """ Returns all pages in sp_list that let func return True
-            for the canonical page name.
-        """
-        return [x for x in sp_list if func(x.name)]
-    filter = classmethod(filter)
-
-    def merge(cls, local_list, remote_list):
-        """ Merges two lists of SyncPages into one, migrating attributes like the names. """
-        # map page names to SyncPage objects :-)
-        d = dict(zip(local_list, local_list))
-        for sp in remote_list:
-            if sp in d:
-                d[sp].remote_rev = sp.remote_rev
-                d[sp].remote_name = sp.remote_name
-            else:
-                d[sp] = sp
-        return d.keys()
-    merge = classmethod(merge)
-
-    def is_only_local(self):
-        """ Is true if the page is only in the local wiki. """
-        return not self.remote_rev
-
-    def is_only_remote(self):
-        """ Is true if the page is only in the remote wiki. """
-        return not self.local_rev
-
-    def is_local_and_remote(self):
-        """ Is true if the page is in both wikis. """
-        return self.local_rev and self.remote_rev
-
-    def iter_local_only(cls, sp_list):
-        """ Iterates over all pages that are local only. """
-        for x in sp_list:
-            if x.is_only_local():
-                yield x
-    iter_local_only = classmethod(iter_local_only)
-
-    def iter_remote_only(cls, sp_list):
-        """ Iterates over all pages that are remote only. """
-        for x in sp_list:
-            if x.is_only_remote():
-                yield x
-    iter_remote_only = classmethod(iter_remote_only)
-
-    def iter_local_and_remote(cls, sp_list):
-        """ Iterates over all pages that are local and remote. """
-        for x in sp_list:
-            if x.is_local_and_remote():
-                yield x
-    iter_local_and_remote = classmethod(iter_local_and_remote)
-
-class RemoteWiki(object):
-    """ This class should be the base for all implementations of remote wiki
-        classes. """
-
-    def __repr__(self):
-        """ Returns a representation of the instance for debugging purposes. """
-        return NotImplemented
-
-    def get_interwiki_name(self):
-        """ Returns the interwiki name of the other wiki. """
-        return NotImplemented
-
-    def get_iwid(self):
-        """ Returns the InterWiki ID. """
-        return NotImplemented
-
-    def get_pages(self, **kwargs):
-        """ Returns a list of SyncPage instances. """
-        return NotImplemented
-
-
-class MoinRemoteWiki(RemoteWiki):
-    """ Used for MoinMoin wikis reachable via XMLRPC. """
-    def __init__(self, request, interwikiname, prefix):
-        self.request = request
-        self.prefix = prefix
-        _ = self.request.getText
-
-        wikitag, wikiurl, wikitail, wikitag_bad = wikiutil.resolve_wiki(self.request, '%s:""' % (interwikiname, ))
-        self.wiki_url = wikiutil.mapURL(self.request, wikiurl)
-        self.valid = not wikitag_bad
-        self.xmlrpc_url = self.wiki_url + "?action=xmlrpc2"
-        if not self.valid:
-            self.connection = None
-            return
-
-        self.connection = self.createConnection()
-
-        iw_list = self.connection.interwikiName()
-
-        #version = self.connection.getMoinVersion()
-        #if not isinstance(version, (tuple, list)):
-        #    raise UnsupportedWikiException(_("The remote version of MoinMoin is too old, the version 1.6 is required at least."))
-
-        self.remote_interwikiname = remote_interwikiname = iw_list[0]
-        self.remote_iwid = remote_iwid = iw_list[1]
-        self.is_anonymous = remote_interwikiname is None
-        if not self.is_anonymous and interwikiname != remote_interwikiname:
-            raise UnsupportedWikiException(_("The remote wiki uses a different InterWiki name (%(remotename)s)"
-                                             " internally than you specified (%(localname)s).") % {
-                "remotename": wikiutil.escape(remote_interwikiname), "localname": wikiutil.escape(interwikiname)})
-
-        if self.is_anonymous:
-            self.iwid_full = packLine([remote_iwid])
-        else:
-            self.iwid_full = packLine([remote_iwid, interwikiname])
-
-    def createConnection(self):
-        return xmlrpclib.ServerProxy(self.xmlrpc_url, allow_none=True, verbose=True)
-
-    # Public methods
-    def get_diff(self, pagename, from_rev, to_rev):
-        """ Returns the binary diff of the remote page named pagename, given
-            from_rev and to_rev. """
-        result = self.connection.getDiff(pagename, from_rev, to_rev)
-        result["diff"] = str(result["diff"]) # unmarshal Binary object
-        return result
-
-    def merge_diff(self, pagename, diff, local_rev, delta_remote_rev, last_remote_rev, interwiki_name):
-        """ Merges the diff into the page on the remote side. """
-        result = self.connection.mergeDiff(pagename, xmlrpclib.Binary(diff), local_rev, delta_remote_rev, last_remote_rev, interwiki_name)
-        return result
-
-    # Methods implementing the RemoteWiki interface
-    def get_interwiki_name(self):
-        return self.remote_interwikiname
-
-    def get_iwid(self):
-        return self.remote_iwid
-
-    def get_pages(self, **kwargs):
-        options = {"include_revno": True,
-                   "include_deleted": True,
-                   "exclude_non_writable": kwargs["exclude_non_writable"]}
-        pages = self.connection.getAllPagesEx(options)
-        rpages = []
-        for name, revno in pages:
-            normalised_name = normalise_pagename(name, self.prefix)
-            if normalised_name is None:
-                continue
-            rpages.append(SyncPage(normalised_name, remote_rev=revno, remote_name=name))
-        return rpages
-
-    def __repr__(self):
-        return "<MoinRemoteWiki wiki_url=%r valid=%r>" % (self.wiki_url, self.valid)
-
-
-class MoinLocalWiki(RemoteWiki):
-    """ Used for the current MoinMoin wiki. """
-    def __init__(self, request, prefix):
-        self.request = request
-        self.prefix = prefix
-
-    def getGroupItems(self, group_list):
-        """ Returns all page names that are listed on the page group_list. """
-        pages = []
-        for group_pagename in group_list:
-            pages.extend(Group(self.request, group_pagename).members())
-        return [self.createSyncPage(x) for x in pages]
-
-    def createSyncPage(self, page_name):
-        normalised_name = normalise_pagename(page_name, self.prefix)
-        if normalised_name is None:
-            return None
-        return SyncPage(normalised_name, local_rev=Page(self.request, page_name).get_real_rev(), local_name=page_name)
-
-    # Public methods:
-
-    # Methods implementing the RemoteWiki interface
-    def get_interwiki_name(self):
-        return self.request.cfg.interwikiname
-
-    def get_iwid(self):
-        return self.request.cfg.iwid
-
-    def get_pages(self, **kwargs):
-        assert not kwargs
-        return [x for x in [self.createSyncPage(x) for x in self.request.rootpage.getPageList(exists=1)] if x]
-
-    def __repr__(self):
-        return "<MoinLocalWiki>"
-
 
 class ActionClass:
     INFO, WARN, ERROR = range(3) # used for logging
@@ -485,13 +231,14 @@
             except PageEditor.Unchanged:
                 pass
             except PageEditor.EditConflict:
-                # XXX remote rollback needed
                 assert False, "You stumbled on a problem with the current storage system - I cannot lock pages"
+
             new_local_rev = current_page.get_real_rev()
+
             try:
                 very_current_remote_rev = remote.merge_diff(rp.remote_name, compress(diff), new_local_rev, current_remote_rev, current_remote_rev, local_full_iwid)
             except Exception, e:
-                raise # XXX rollback
+                raise # XXX rollback locally
             else:
                 tags.add(remote_wiki=remote_full_iwid, remote_rev=very_current_remote_rev, current_rev=new_local_rev)
 
--- a/MoinMoin/wikisync.py	Fri Aug 11 22:28:14 2006 +0200
+++ b/MoinMoin/wikisync.py	Sat Aug 12 22:30:49 2006 +0200
@@ -7,14 +7,270 @@
 """
 
 import os
+import xmlrpclib
 
 try:
     import cPickle as pickle
 except ImportError:
     import pickle
 
+
+from MoinMoin import wikiutil
 from MoinMoin.util import lock
-from MoinMoin.packages import unpackLine
+from MoinMoin.Page import Page
+from MoinMoin.packages import unpackLine, packLine
+
+
+def normalise_pagename(page_name, prefix):
+    """ Checks if the page_name starts with the prefix.
+        Returns None if it does not, otherwise strips the prefix.
+    """
+    if prefix:
+        if not page_name.startswith(prefix):
+            return None
+        else:
+            return page_name[len(prefix):]
+    else:
+        return page_name
+
+
+class UnsupportedWikiException(Exception): pass
+
+
+class SyncPage(object):
+    """ This class represents a page in one or two wiki(s). """
+    def __init__(self, name, local_rev=None, remote_rev=None, local_name=None, remote_name=None):
+        """ Creates a SyncPage instance.
+            @param name: The canonical name of the page, without prefixes.
+            @param local_rev: The revision of the page in the local wiki.
+            @param remote_rev: The revision of the page in the remote wiki.
+            @param local_name: The page name of the page in the local wiki.
+            @param remote_name: The page name of the page in the remote wiki.
+        """
+        self.name = name
+        self.local_rev = local_rev
+        self.remote_rev = remote_rev
+        self.local_name = local_name
+        self.remote_name = remote_name
+        assert local_rev or remote_rev
+        assert local_name or remote_name
+
+    def __repr__(self):
+        return repr("<Remote Page %r>" % unicode(self))
+
+    def __unicode__(self):
+        return u"%s[%s|%s]<%r:%r>" % (self.name, self.local_name, self.remote_name, self.local_rev, self.remote_rev)
+
+    def __lt__(self, other):
+        return self.name < other.name
+
+    def __hash__(self):
+        """ Ensures that the hash value of this page only depends on the canonical name. """
+        return hash(self.name)
+
+    def __eq__(self, other):
+        if not isinstance(other, SyncPage):
+            return false
+        return self.name == other.name
+
+    def add_missing_pagename(self, local, remote):
+        """ Checks if the particular concrete page names are unknown and fills
+            them in.
+        """
+        if self.local_name is None:
+            n_name = normalise_pagename(self.remote_name, remote.prefix)
+            assert n_name is not None
+            self.local_name = (local.prefix or "") + n_name
+        elif self.remote_name is None:
+            n_name = normalise_pagename(self.local_name, local.prefix)
+            assert n_name is not None
+            self.remote_name = (local.prefix or "") + n_name
+
+        return self # makes using list comps easier
+
+    def filter(cls, sp_list, func):
+        """ Returns all pages in sp_list that let func return True
+            for the canonical page name.
+        """
+        return [x for x in sp_list if func(x.name)]
+    filter = classmethod(filter)
+
+    def merge(cls, local_list, remote_list):
+        """ Merges two lists of SyncPages into one, migrating attributes like the names. """
+        # map page names to SyncPage objects :-)
+        d = dict(zip(local_list, local_list))
+        for sp in remote_list:
+            if sp in d:
+                d[sp].remote_rev = sp.remote_rev
+                d[sp].remote_name = sp.remote_name
+            else:
+                d[sp] = sp
+        return d.keys()
+    merge = classmethod(merge)
+
+    def is_only_local(self):
+        """ Is true if the page is only in the local wiki. """
+        return not self.remote_rev
+
+    def is_only_remote(self):
+        """ Is true if the page is only in the remote wiki. """
+        return not self.local_rev
+
+    def is_local_and_remote(self):
+        """ Is true if the page is in both wikis. """
+        return self.local_rev and self.remote_rev
+
+    def iter_local_only(cls, sp_list):
+        """ Iterates over all pages that are local only. """
+        for x in sp_list:
+            if x.is_only_local():
+                yield x
+    iter_local_only = classmethod(iter_local_only)
+
+    def iter_remote_only(cls, sp_list):
+        """ Iterates over all pages that are remote only. """
+        for x in sp_list:
+            if x.is_only_remote():
+                yield x
+    iter_remote_only = classmethod(iter_remote_only)
+
+    def iter_local_and_remote(cls, sp_list):
+        """ Iterates over all pages that are local and remote. """
+        for x in sp_list:
+            if x.is_local_and_remote():
+                yield x
+    iter_local_and_remote = classmethod(iter_local_and_remote)
+
+class RemoteWiki(object):
+    """ This class should be the base for all implementations of remote wiki
+        classes. """
+
+    def __repr__(self):
+        """ Returns a representation of the instance for debugging purposes. """
+        return NotImplemented
+
+    def get_interwiki_name(self):
+        """ Returns the interwiki name of the other wiki. """
+        return NotImplemented
+
+    def get_iwid(self):
+        """ Returns the InterWiki ID. """
+        return NotImplemented
+
+    def get_pages(self, **kwargs):
+        """ Returns a list of SyncPage instances. """
+        return NotImplemented
+
+
+class MoinRemoteWiki(RemoteWiki):
+    """ Used for MoinMoin wikis reachable via XMLRPC. """
+    def __init__(self, request, interwikiname, prefix):
+        self.request = request
+        self.prefix = prefix
+        _ = self.request.getText
+
+        wikitag, wikiurl, wikitail, wikitag_bad = wikiutil.resolve_wiki(self.request, '%s:""' % (interwikiname, ))
+        self.wiki_url = wikiutil.mapURL(self.request, wikiurl)
+        self.valid = not wikitag_bad
+        self.xmlrpc_url = self.wiki_url + "?action=xmlrpc2"
+        if not self.valid:
+            self.connection = None
+            return
+
+        self.connection = self.createConnection()
+
+        try:
+            iw_list = self.connection.interwikiName()
+        except xmlrpclib.Fault, e:
+            raise UnsupportedWikiException(_("The remote version of MoinMoin is too old, the version 1.6 is required at least."))
+
+        self.remote_interwikiname = remote_interwikiname = iw_list[0]
+        self.remote_iwid = remote_iwid = iw_list[1]
+        self.is_anonymous = remote_interwikiname is None
+        if not self.is_anonymous and interwikiname != remote_interwikiname:
+            raise UnsupportedWikiException(_("The remote wiki uses a different InterWiki name (%(remotename)s)"
+                                             " internally than you specified (%(localname)s).") % {
+                "remotename": wikiutil.escape(remote_interwikiname), "localname": wikiutil.escape(interwikiname)})
+
+        if self.is_anonymous:
+            self.iwid_full = packLine([remote_iwid])
+        else:
+            self.iwid_full = packLine([remote_iwid, interwikiname])
+
+    def createConnection(self):
+        return xmlrpclib.ServerProxy(self.xmlrpc_url, allow_none=True, verbose=True)
+
+    # Public methods
+    def get_diff(self, pagename, from_rev, to_rev):
+        """ Returns the binary diff of the remote page named pagename, given
+            from_rev and to_rev. """
+        result = self.connection.getDiff(pagename, from_rev, to_rev)
+        result["diff"] = str(result["diff"]) # unmarshal Binary object
+        return result
+
+    def merge_diff(self, pagename, diff, local_rev, delta_remote_rev, last_remote_rev, interwiki_name):
+        """ Merges the diff into the page on the remote side. """
+        result = self.connection.mergeDiff(pagename, xmlrpclib.Binary(diff), local_rev, delta_remote_rev, last_remote_rev, interwiki_name)
+        return result
+
+    # Methods implementing the RemoteWiki interface
+    def get_interwiki_name(self):
+        return self.remote_interwikiname
+
+    def get_iwid(self):
+        return self.remote_iwid
+
+    def get_pages(self, **kwargs):
+        options = {"include_revno": True,
+                   "include_deleted": True,
+                   "exclude_non_writable": kwargs["exclude_non_writable"]}
+        pages = self.connection.getAllPagesEx(options)
+        rpages = []
+        for name, revno in pages:
+            normalised_name = normalise_pagename(name, self.prefix)
+            if normalised_name is None:
+                continue
+            rpages.append(SyncPage(normalised_name, remote_rev=revno, remote_name=name))
+        return rpages
+
+    def __repr__(self):
+        return "<MoinRemoteWiki wiki_url=%r valid=%r>" % (getattr(self, "wiki_url", Ellipsis), getattr(self, "valid", Ellipsis))
+
+
+class MoinLocalWiki(RemoteWiki):
+    """ Used for the current MoinMoin wiki. """
+    def __init__(self, request, prefix):
+        self.request = request
+        self.prefix = prefix
+
+    def getGroupItems(self, group_list):
+        """ Returns all page names that are listed on the page group_list. """
+        pages = []
+        for group_pagename in group_list:
+            pages.extend(Group(self.request, group_pagename).members())
+        return [self.createSyncPage(x) for x in pages]
+
+    def createSyncPage(self, page_name):
+        normalised_name = normalise_pagename(page_name, self.prefix)
+        if normalised_name is None:
+            return None
+        return SyncPage(normalised_name, local_rev=Page(self.request, page_name).get_real_rev(), local_name=page_name)
+
+    # Public methods:
+
+    # Methods implementing the RemoteWiki interface
+    def get_interwiki_name(self):
+        return self.request.cfg.interwikiname
+
+    def get_iwid(self):
+        return self.request.cfg.iwid
+
+    def get_pages(self, **kwargs):
+        assert not kwargs
+        return [x for x in [self.createSyncPage(x) for x in self.request.rootpage.getPageList(exists=1)] if x]
+
+    def __repr__(self):
+        return "<MoinLocalWiki>"
 
 
 class Tag(object):
@@ -126,17 +382,17 @@
         self.tags = []
         self.commit()
 
-    def fetch(self, iwid_full=None, iw_name=None):
-        assert bool(iwid_full) ^ bool(iw_name)
-        if iwid_full:
-            iwid_full = unpackLine(iwid_full)
-            if len(iwid_full) == 1:
-                assert False, "This case is not supported yet" # XXX
-            iw_name = iwid_full[1]
-
-        return [t for t in self.tags if unpackLine(t.remote_wiki)[1] == iw_name]
+    def fetch(self, iwid_full):
+        iwid_full = unpackLine(iwid_full)
+        matching_tags = []
+        for t in self.tags:
+            t_iwid_full = unpackLine(t.remote_wiki)
+            if ((t_iwid_full[0] == iwid_full[0]) # either match IWID or IW name
+                or (len(t_iwid_full) == 2 and len(iwid_full) == 2 and t_iwid_full[1] == iwid_full[1])):
+                matching_tags.append(t)
+        return matching_tags
 
 
 # currently we just have one implementation, so we do not need
 # a factory method
-TagStore = PickleTagStore
\ No newline at end of file
+TagStore = PickleTagStore
--- a/docs/CHANGES.aschremmer	Fri Aug 11 22:28:14 2006 +0200
+++ b/docs/CHANGES.aschremmer	Sat Aug 12 22:30:49 2006 +0200
@@ -2,20 +2,21 @@
 ===============================
 
   Known main issues:
+    * How to handle renames/deletes?
     * How will we store tags? (Metadata support would be handy)
-    * How to handle renames/deletes?
-    * How to handle colliding/empty interwiki names?
+      (currently done in Pickle files)
 
   ToDo:
-    * Implement actual syncronisation.
-      * Add correct IWID_full handling.
+    * Implement all syncronisation cases (all directions, all 3 page sets).
     * Reduce round-trip times by caching queries and using MultiCall objects.
+    * Attach the status information to the job page.
     * Implement a cross-site authentication system, i.e. mainly an
       identity storage.
     * Clean up trailing whitespace.
     * Add page locking, i.e. use the one in the new storage layer.
     * Check what needs to be documented on MoinMaster.
     * Search for XXX
+    * Put author names into the comment field, transmit mimetypes.
 
   New Features:
     * XMLRPC method to return the Moin version
@@ -94,7 +95,8 @@
          detection in SyncPages. Added logging support to SyncPages. Refactored conflict flag
          detection from the edit action into the PageEditor class. Enhanced XMLRPC server in Moin to allow
          XMLRPC functions to return Fault instances. Introduced a new diff3 mode that should reduce the
-         conflicts. Fixed hard to track down bugs in SyncPages.
+         conflicts. Fixed hard to track down bugs in SyncPages. Store full IWIDs and match by
+         either of both components when searching for tags.
 
 2006-07-18: the requested daily entry is missing here, see http://moinmoin.wikiwikiweb.de/GoogleSoc2006/BetterProgress
 2006-07-19: the requested daily entry is missing here, see http://moinmoin.wikiwikiweb.de/GoogleSoc2006/BetterProgress