changeset 3994:bdb71fda36eb

merge with moin-1.8 main branch
author Byeongweon [tasyblue@gmail.com]
date Sun, 03 Aug 2008 05:45:48 +0900
parents e0301e5bf019 (current diff) b4bdfaaa79b0 (diff)
children 529b71b82fdc
files
diffstat 7 files changed, 213 insertions(+), 175 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/_tests/maketestwiki.py	Sun Aug 03 03:56:42 2008 +0900
+++ b/MoinMoin/_tests/maketestwiki.py	Sun Aug 03 05:45:48 2008 +0900
@@ -14,7 +14,6 @@
 
 filename = globals().get("__file__") or sys.argv[0]
 moinpath = os.path.abspath(os.path.join(os.path.dirname(filename), os.pardir, os.pardir))
-sys.path.insert(0, moinpath)
 
 from MoinMoin.support import tarfile
 
@@ -62,5 +61,6 @@
     untarUnderlay()
 
 if __name__ == '__main__':
+    sys.path.insert(0, moinpath)
     run()
 
--- a/MoinMoin/conftest.py	Sun Aug 03 03:56:42 2008 +0900
+++ b/MoinMoin/conftest.py	Sun Aug 03 05:45:48 2008 +0900
@@ -33,7 +33,7 @@
 from MoinMoin._tests import maketestwiki, compat
 modules["unittest"] = compat # evil hack
 
-sys.path.insert(0, str(moindir.join("tests")))
+wikiconfig_dir = str(moindir.join("tests"))
 
 from MoinMoin.support.python_compatibility import set
 
@@ -77,6 +77,9 @@
     if not static_state[0]:
         maketestwiki.run(True)
         static_state[0] = True
+    if sys.path[0] != wikiconfig_dir:
+        sys.path.insert(0, wikiconfig_dir) # this is a race with py.test's collectors
+                                           # because they modify sys.path as well
     request = request_cli.Request()
     request.form = request.args = request.setup_args()
     request.user = User(request)
--- a/MoinMoin/parser/_ParserBase.py	Sun Aug 03 03:56:42 2008 +0900
+++ b/MoinMoin/parser/_ParserBase.py	Sun Aug 03 05:45:48 2008 +0900
@@ -2,10 +2,12 @@
 """
     MoinMoin - Base Source Parser
 
-    @copyright: 2002 by Taesu Pyo <bigflood@hitel.net>
+    @copyright: 2002 by Taesu Pyo <bigflood@hitel.net>,
+                2005 by Oliver Graf <ograf@bitart.de>,
+                2005-2008 MoinMoin:ThomasWaldmann
+
     @license: GNU GPL, see COPYING for details.
 
-    Docstrings and some refactoring by Oliver Graf <ograf@bitart.de>
 
 basic css:
 
@@ -23,8 +25,95 @@
 """
 
 import re, sha
+
 from MoinMoin import config, wikiutil
 
+class FormatTextBase:
+    pass
+
+class FormatBeginLine(FormatTextBase):
+    def formatString(self, formatter, word):
+        return formatter.code_line(1)
+
+class FormatEndLine(FormatTextBase):
+    def formatString(self, formatter, word):
+        return formatter.code_line(0)
+
+class FormatText(FormatTextBase):
+
+    def __init__(self, fmt):
+        self.fmt = fmt
+
+    def formatString(self, formatter, word):
+        return (formatter.code_token(1, self.fmt) +
+                formatter.text(word) +
+                formatter.code_token(0, self.fmt))
+
+class FormatTextID(FormatTextBase):
+
+    def __init__(self, fmt, icase=False):
+        if not isinstance(fmt, FormatText):
+            fmt = FormatText(fmt)
+        self.setDefaultFormat(fmt)
+        self._ignore_case = icase
+        self.fmt = {}
+
+    def setDefaultFormat(self, fmt):
+        self._def_fmt = fmt
+
+    def addFormat(self, word, fmt):
+        if self._ignore_case:
+            word = word.lower()
+        self.fmt[word] = fmt
+
+    def formatString(self, formatter, word):
+        if self._ignore_case:
+            sword = word.lower()
+        else:
+            sword = word
+        return self.fmt.get(sword, self._def_fmt).formatString(formatter, word)
+
+
+class FormattingRuleSingle:
+
+    def __init__(self, name, str_re, icase=False):
+        self.name = name
+        self.str_re = str_re
+
+    def getStartRe(self):
+        return self.str_re
+
+    def getText(self, parser, hit):
+        return hit
+
+
+class FormattingRulePair:
+
+    def __init__(self, name, str_begin, str_end, icase=False):
+        self.name = name
+        self.str_begin = str_begin
+        self.str_end = str_end
+        re_flags = re.M
+        if icase:
+            re_flags |= re.I
+        self.end_re = re.compile(str_end, re_flags)
+
+    def getStartRe(self):
+        return self.str_begin
+
+    def getText(self, parser, hit):
+        match = self.end_re.search(parser.text, parser.lastpos)
+        if not match:
+            next_lastpos = parser.text_len
+        else:
+            next_lastpos = match.end() + (match.end() == parser.lastpos)
+        r = parser.text[parser.lastpos:next_lastpos]
+        parser.lastpos = next_lastpos
+        return hit + r
+
+
+# ------------------------------------------------------------------------
+
 def parse_start_step(request, args):
     """
     Parses common Colorizer parameters start, step, numbers.
@@ -60,101 +149,39 @@
             nums = -1
     return nums, start, step, attrs
 
-class FormatTextBase:
-    pass
-
-class FormatText(FormatTextBase):
-
-    def __init__(self, fmt):
-        self.fmt = fmt
-
-    def formatString(self, formatter, word):
-        return (formatter.code_token(1, self.fmt) +
-                formatter.text(word) +
-                formatter.code_token(0, self.fmt))
-
-class FormatTextID(FormatTextBase):
-
-    def __init__(self, fmt, icase=0):
-        if not isinstance(fmt, FormatText):
-            self.def_fmt = FormatText(fmt)
-        else:
-            self.def_fmt = fmt
-        self._ignore_case = icase
-        self.fmt = {}
-
-    def addFormat(self, word, fmt):
-        if self._ignore_case:
-            word = word.lower()
-        self.fmt[word] = fmt
-
-    def setDefaultFormat(self, fmt):
-        self.def_fmt = fmt
-
-    def formatString(self, formatter, word):
-        if self._ignore_case:
-            sword = word.lower()
-        else:
-            sword = word
-        return self.fmt.get(sword, self.def_fmt).formatString(formatter, word)
-
-class FormattingRuleSingle:
-
-    def __init__(self, name, str_re, icase=0):
-        self.name = name
-        self.str_re = str_re
-
-    def getStartRe(self):
-        return self.str_re
-
-    def getText(self, parser, hit):
-        return hit
-
-class FormattingRulePair:
-
-    def __init__(self, name, str_begin, str_end, icase=0):
-        self.name = name
-        self.str_begin = str_begin
-        self.str_end = str_end
-        if icase:
-            self.end_re = re.compile(str_end, re.M|re.I)
-        else:
-            self.end_re = re.compile(str_end, re.M)
-
-    def getStartRe(self):
-        return self.str_begin
-
-    def getText(self, parser, hit):
-        match = self.end_re.search(parser.line, parser.lastpos)
-        if not match:
-            next_lastpos = len(parser.line)
-        else:
-            next_lastpos = match.end() + (match.end() == parser.lastpos)
-        r = parser.line[parser.lastpos:next_lastpos]
-        parser.lastpos = next_lastpos
-        return hit + r
-
-
-# ------------------------------------------------------------------------
 
 class ParserBase:
 
     parsername = 'ParserBase'
+    tabwidth = 4
+
+    # for dirty tricks, see comment in format():
+    STARTL, STARTL_RE = u"^\n", ur"\^\n"
+    ENDL, ENDL_RE = u"\n$", ur"\n\$"
+    LINESEP = ENDL + STARTL
 
     def __init__(self, raw, request, **kw):
         self.raw = raw
         self.request = request
         self.show_nums, self.num_start, self.num_step, attrs = parse_start_step(request, kw.get('format_args', ''))
 
-        self._ignore_case = 0
+        self._ignore_case = False
         self._formatting_rules = []
         self._formatting_rules_n2r = {}
         self._formatting_rule_index = 0
         self.rule_fmt = {}
-        self.line_count = len(raw.split('\n'))+1
+        #self.line_count = len(raw.split('\n')) + 1
 
     def setupRules(self):
+        self.addRuleFormat("BEGINLINE", FormatBeginLine())
+        self.addRuleFormat("ENDLINE", FormatEndLine())
+        # we need a little dirty trick here, see comment in format():
+        self.addRule("BEGINLINE", self.STARTL_RE)
+        self.addRule("ENDLINE", self.ENDL_RE)
+
         self.def_format = FormatText('Default')
+        self.reserved_word_format = FormatText('ResWord')
+        self.constant_word_format = FormatText('ConsWord')
         self.ID_format = FormatTextID('ID', self._ignore_case)
         self.addRuleFormat("ID", self.ID_format)
         self.addRuleFormat("Operator")
@@ -169,22 +196,18 @@
         self.addRuleFormat("Special")
         self.addRuleFormat("Preprc")
         self.addRuleFormat("Error")
-        self.reserved_word_format = FormatText('ResWord')
-        self.constant_word_format = FormatText('ConsWord')
+
+    def _addRule(self, name, fmt):
+        self._formatting_rule_index += 1
+        name = "%s_%s" % (name, self._formatting_rule_index) # create unique name
+        self._formatting_rules.append((name, fmt))
+        self._formatting_rules_n2r[name] = fmt
 
     def addRule(self, name, str_re):
-        self._formatting_rule_index += 1
-        n = "%s_%s" % (name, self._formatting_rule_index)
-        f = FormattingRuleSingle(name, str_re, self._ignore_case)
-        self._formatting_rules.append((n, f))
-        self._formatting_rules_n2r[n] = f
+        self._addRule(name, FormattingRuleSingle(name, str_re, self._ignore_case))
 
     def addRulePair(self, name, start_re, end_re):
-        self._formatting_rule_index += 1
-        n = "%s_%s" % (name, self._formatting_rule_index)
-        f = FormattingRulePair(name, start_re, end_re, self._ignore_case)
-        self._formatting_rules.append((n, f))
-        self._formatting_rules_n2r[n] = f
+        self._addRule(name, FormattingRulePair(name, start_re, end_re, self._ignore_case))
 
     def addWords(self, words, fmt):
         if not isinstance(fmt, FormatTextBase):
@@ -209,64 +232,69 @@
 
         self.setupRules()
 
-        l = []
-        for n, f in self._formatting_rules:
-            l.append("(?P<%s>%s)" % (n, f.getStartRe()))
-
+        formatting_regexes = ["(?P<%s>%s)" % (n, f.getStartRe())
+                              for n, f in self._formatting_rules]
+        re_flags = re.M
         if self._ignore_case:
-            scan_re = re.compile("|".join(l), re.M|re.I)
-        else:
-            scan_re = re.compile("|".join(l), re.M)
+            re_flags |= re.I
+        scan_re = re.compile("|".join(formatting_regexes), re_flags)
 
-        self.lastpos = 0
-        self.line = self.raw
+        self.text = self.raw
+
+        # dirty little trick to work around re lib's limitations (it can't have
+        # zero length matches at line beginning for ^ and at the same time match
+        # something else at the beginning of the line):
+        self.text = self.LINESEP.join([line.replace('\r', '') for line in self.text.splitlines()])
+        self.text = self.STARTL + self.text + self.ENDL
+        self.text_len = len(self.text)
+
+        result = [] # collects output
 
         self._code_id = sha.new(self.raw.encode(config.charset)).hexdigest()
-        self.request.write(formatter.code_area(1, self._code_id, self.parsername, self.show_nums, self.num_start, self.num_step))
-
-        self.request.write(formatter.code_line(1))
-            #formatter, len('%d' % (self.line_count,)))
+        result.append(formatter.code_area(1, self._code_id, self.parsername, self.show_nums, self.num_start, self.num_step))
 
-        match = scan_re.search(self.line)
-
-        while match and self.lastpos < len(self.line):
-            # add the match we found
-            self.write_normal_text(formatter,
-                                   self.line[self.lastpos:match.start()])
+        self.lastpos = 0
+        match = scan_re.search(self.text)
+        while match and self.lastpos < self.text_len:
+            # add the rendering of the text left of the match we found
+            text = self.text[self.lastpos:match.start()]
+            if text:
+                result.extend(self.format_normal_text(formatter, text))
             self.lastpos = match.end() + (match.end() == self.lastpos)
 
-            self.write_match(formatter, match)
+            # add the rendering of the match we found
+            result.extend(self.format_match(formatter, match))
 
             # search for the next one
-            match = scan_re.search(self.line, self.lastpos)
-
-        self.write_normal_text(formatter, self.line[self.lastpos:])
-
-        self.request.write(formatter.code_area(0, self._code_id))
-
+            match = scan_re.search(self.text, self.lastpos)
 
-    def write_normal_text(self, formatter, text):
-        first = 1
-        for line in text.expandtabs(4).split('\n'):
-            if not first:
-                self.request.write(formatter.code_line(1))
-            else:
-                first = 0
-            self.request.write(formatter.text(line))
+        # add the rendering of the text right of the last match we found
+        text = self.text[self.lastpos:]
+        if text:
+            result.extend(self.format_normal_text(formatter, text))
 
-    def write_match(self, formatter, match):
+        result.append(formatter.code_area(0, self._code_id))
+        self.request.write(''.join(result))
+
+    def format_normal_text(self, formatter, text):
+        return [formatter.text(text.expandtabs(self.tabwidth))]
+
+    def format_match(self, formatter, match):
+        result = []
         for n, hit in match.groupdict().items():
-            if not hit:
+            if hit is None:
                 continue
             r = self._formatting_rules_n2r[n]
             s = r.getText(self, hit)
             c = self.rule_fmt.get(r.name, None)
             if not c:
                 c = self.def_format
-            first = 1
-            for line in s.expandtabs(4).split('\n'):
-                if not first:
-                    self.request.write(formatter.code_line(1))
-                else:
-                    first = 0
-                self.request.write(c.formatString(formatter, line))
+            if s:
+                lines = s.expandtabs(self.tabwidth).split(self.LINESEP)
+                for line in lines[:-1]:
+                    result.append(c.formatString(formatter, line))
+                    result.append(FormatEndLine().formatString(formatter, ''))
+                    result.append(FormatBeginLine().formatString(formatter, ''))
+                result.append(c.formatString(formatter, lines[-1]))
+        return result
+
--- a/MoinMoin/parser/text_cplusplus.py	Sun Aug 03 03:56:42 2008 +0900
+++ b/MoinMoin/parser/text_cplusplus.py	Sun Aug 03 05:45:48 2008 +0900
@@ -36,13 +36,13 @@
     def setupRules(self):
         ParserBase.setupRules(self)
 
-        self.addRulePair("Comment", "/[*]", "[*]/")
-        self.addRule("Comment", "//.*$")
-        self.addRulePair("String", 'L?"', r'$|[^\\](\\\\)*"')
+        self.addRulePair("Comment", r"/[*]", r"[*]/")
+        self.addRule("Comment", r"//.*$")
+        self.addRulePair("String", r'L?"', r'$|[^\\](\\\\)*"')
         self.addRule("Char", r"'\\.'|'[^\\]'")
         self.addRule("Number", r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?")
         self.addRule("Preprc", r"^\s*#(.*\\\n)*(.*(?!\\))$")
-        self.addRule("ID", "[a-zA-Z_][0-9a-zA-Z_]*")
+        self.addRule("ID", r"[a-zA-Z_][0-9a-zA-Z_]*")
         self.addRule("SPChar", r"[~!%^&*()+=|\[\]:;,.<>/?{}-]")
 
         reserved_words = ['struct', 'class', 'union', 'enum',
--- a/MoinMoin/parser/text_java.py	Sun Aug 03 03:56:42 2008 +0900
+++ b/MoinMoin/parser/text_java.py	Sun Aug 03 05:45:48 2008 +0900
@@ -20,12 +20,12 @@
     def setupRules(self):
         ParserBase.setupRules(self)
 
-        self.addRulePair("Comment", "/[*]", "[*]/")
-        self.addRule("Comment", "//.*$")
-        self.addRulePair("String", '"', r'$|[^\\](\\\\)*"')
+        self.addRulePair("Comment", r"/[*]", r"[*]/")
+        self.addRule("Comment", r"//.*$")
+        self.addRulePair("String", r'"', r'$|[^\\](\\\\)*"')
         self.addRule("Char", r"'\\.'|'[^\\]'")
         self.addRule("Number", r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?")
-        self.addRule("ID", "[a-zA-Z_][0-9a-zA-Z_]*")
+        self.addRule("ID", r"[a-zA-Z_][0-9a-zA-Z_]*")
         self.addRule("SPChar", r"[~!%^&*()+=|\[\]:;,.<>/?{}-]")
 
         reserved_words = ['class', 'interface', 'enum', 'import', 'package',
@@ -40,3 +40,4 @@
         constant_words = ['true', 'false', 'null']
 
         self.addConstant(constant_words)
+
--- a/MoinMoin/parser/text_pascal.py	Sun Aug 03 03:56:42 2008 +0900
+++ b/MoinMoin/parser/text_pascal.py	Sun Aug 03 05:45:48 2008 +0900
@@ -18,18 +18,18 @@
 
     def __init__(self, raw, request, **kw):
         ParserBase.__init__(self, raw, request, **kw)
-        self._ignore_case = 1
+        self._ignore_case = True
 
     def setupRules(self):
         ParserBase.setupRules(self)
 
-        self.addRulePair("Comment", "\(\*", "\*\)")
-        self.addRulePair("Comment", "\{", "\}")
-        self.addRule("Comment", "//.*$")
-        self.addRulePair("String", '\'', '\'')
+        self.addRulePair("Comment", r"\(\*", r"\*\)")
+        self.addRulePair("Comment", r"\{", r"\}")
+        self.addRule("Comment", r"//.*$")
+        self.addRulePair("String", r"'", r"'")
         self.addRule("Char", r"'\\.'|#[a-f0-9][a-f0-9]")
         self.addRule("Number", r"[0-9](\.[0-9]*)?(eE[+-][0-9])?|\$[0-9a-fA-F]+")
-        self.addRule("ID", "[a-zA-Z_][0-9a-zA-Z_]*")
+        self.addRule("ID", r"[a-zA-Z_][0-9a-zA-Z_]*")
         self.addRule("SPChar", r"[~!%^&*()+=|\[\]:;,.<>/?{}-]")
 
         reserved_words = ['class', 'interface', 'set', 'uses', 'unit',
@@ -48,3 +48,4 @@
         constant_words = ['true', 'false', 'nil']
 
         self.addConstant(constant_words)
+
--- a/MoinMoin/parser/text_python.py	Sun Aug 03 03:56:42 2008 +0900
+++ b/MoinMoin/parser/text_python.py	Sun Aug 03 05:45:48 2008 +0900
@@ -2,27 +2,29 @@
 """
     MoinMoin - highlighting Python Source Parser
 
-    @copyright: 2001 Juergen Hermann <jh@web.de>
+    @copyright: 2001 Juergen Hermann <jh@web.de>,
+                2006-2008 MoinMoin:ThomasWaldmann
     @license: GNU GPL, see COPYING for details.
 """
 
 import StringIO
 import keyword, token, tokenize, sha
+
 from MoinMoin import config, wikiutil
 from MoinMoin.parser._ParserBase import parse_start_step
 
 _KEYWORD = token.NT_OFFSET + 1
-_TEXT    = token.NT_OFFSET + 2
+_TEXT = token.NT_OFFSET + 2
 
 _tokens = {
-    token.NUMBER:       'Number',
-    token.OP:           'Operator',
-    token.STRING:       'String',
-    tokenize.COMMENT:   'Comment',
-    token.NAME:         'ID',
-    token.ERRORTOKEN:   'Error',
-    _KEYWORD:           'ResWord',
-    _TEXT:              'Text',
+    token.NUMBER: 'Number',
+    token.OP: 'Operator',
+    token.STRING: 'String',
+    tokenize.COMMENT: 'Comment',
+    token.NAME: 'ID',
+    token.ERRORTOKEN: 'Error',
+    _KEYWORD: 'ResWord',
+    _TEXT: 'Text',
 }
 
 Dependencies = ['user'] # the "Toggle line numbers link" depends on user's language
@@ -58,10 +60,12 @@
             self.lines.append(pos)
         self.lines.append(len(self.raw))
 
+        self.result = [] # collects output
+
         self._code_id = sha.new(self.raw.encode(config.charset)).hexdigest()
-        self.request.write(formatter.code_area(1, self._code_id, 'ColorizedPython', self.show_num, self.num_start, self.num_step))
+        self.result.append(formatter.code_area(1, self._code_id, 'ColorizedPython', self.show_num, self.num_start, self.num_step))
         self.formatter = formatter
-        self.request.write(formatter.code_line(1))
+        self.result.append(formatter.code_line(1))
         #len('%d' % (len(self.lines)-1, )))
 
         # parse the source and write it
@@ -74,7 +78,7 @@
             errmsg = (self.formatter.linebreak() +
                       self.formatter.strong(1) + "ERROR: %s" % msg + self.formatter.strong(0) +
                       self.formatter.linebreak())
-            self.request.write(errmsg)
+            self.result.append(errmsg)
         except tokenize.TokenError, ex:
             msg = ex[0]
             line = ex[1][0]
@@ -82,9 +86,10 @@
                       self.formatter.strong(1) + "ERROR: %s" % msg + self.formatter.strong(0) +
                       self.formatter.linebreak() +
                       wikiutil.escape(self.raw[self.lines[line]:]))
-            self.request.write(errmsg)
-        self.request.write(self.formatter.code_line(0))
-        self.request.write(formatter.code_area(0, self._code_id))
+            self.result.append(errmsg)
+        self.result.append(self.formatter.code_line(0))
+        self.result.append(formatter.code_area(0, self._code_id))
+        self.request.write(''.join(self.result))
 
     def __call__(self, toktype, toktext, (srow, scol), (erow, ecol), line):
         """ Token handler.
@@ -96,13 +101,13 @@
 
         # handle newlines
         if toktype in [token.NEWLINE, tokenize.NL]:
-            self.request.write(self.formatter.code_line(0))
-            self.request.write(self.formatter.code_line(1))
+            self.result.append(self.formatter.code_line(0))
+            self.result.append(self.formatter.code_line(1))
             return
 
         # send the original whitespace, if needed
         if newpos > oldpos:
-            self.request.write(self.formatter.text(self.raw[oldpos:newpos]))
+            self.result.append(self.formatter.text(self.raw[oldpos:newpos]))
 
         # skip indenting tokens
         if toktype in [token.INDENT, token.DEDENT]:
@@ -117,14 +122,14 @@
         tokid = _tokens.get(toktype, _tokens[_TEXT])
 
         # send text
-        first = 1
+        first = True
         for part in toktext.split('\n'):
             if not first:
-                self.request.write(self.formatter.code_line(0))
-                self.request.write(self.formatter.code_line(1))
+                self.result.append(self.formatter.code_line(0))
+                self.result.append(self.formatter.code_line(1))
             else:
-                first = 0
-            self.request.write(self.formatter.code_token(1, tokid) +
+                first = False
+            self.result.append(self.formatter.code_token(1, tokid) +
                                self.formatter.text(part) +
                                self.formatter.code_token(0, tokid))