changeset 4382:5eb02932f387

replace the python/pascal/java/cpp parser by a thin wrapper around the highlight parser (to keep syntax like {{{#python ... working)
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Sat, 11 Oct 2008 23:54:29 +0200
parents 67f263c94bcb
children e25b00b3a308
files MoinMoin/parser/highlight.py MoinMoin/parser/text_cplusplus.py MoinMoin/parser/text_java.py MoinMoin/parser/text_pascal.py MoinMoin/parser/text_python.py
diffstat 5 files changed, 86 insertions(+), 276 deletions(-) [+]
line wrap: on
line diff
--- a/MoinMoin/parser/highlight.py	Sat Oct 11 23:12:42 2008 +0200
+++ b/MoinMoin/parser/highlight.py	Sat Oct 11 23:54:29 2008 +0200
@@ -96,23 +96,29 @@
 
 
 class Parser:
-    parsername = "highlight"
+    parsername = "highlight"  # compatibility wrappers override this with the pygments lexer name
     Dependencies = Dependencies
     extensions = extensions
 
-    def __init__(self, raw, request, filename=None, **kw):
+    def __init__(self, raw, request, filename=None, format_args='', **kw):
         self.request = request
         self.raw = raw.strip('\n')
         self.filename = filename
-        parts = kw.get('format_args', '').split(None)
-        if parts:
-            self.syntax = parts[0]
+        if self.parsername == 'highlight':
+            # user is directly using the highlight parser
+            parts = format_args.split(None)
+            if parts:
+                self.syntax = parts[0]
+            else:
+                self.syntax = ''
+            if len(parts) > 1:
+                params = ' '.join(parts[1:])
+            else:
+                params = ''
         else:
-            self.syntax = ''
-        if len(parts) > 1:
-            params = ' '.join(parts[1:])
-        else:
-            params = ''
+            # a compatibility wrapper inherited from this class
+            self.syntax = self.parsername
+            params = format_args
         self.show_nums, self.num_start, self.num_step, attrs = parse_start_step(request, params)
 
     def format(self, formatter):
--- a/MoinMoin/parser/text_cplusplus.py	Sat Oct 11 23:12:42 2008 +0200
+++ b/MoinMoin/parser/text_cplusplus.py	Sat Oct 11 23:54:29 2008 +0200
@@ -1,70 +1,26 @@
 # -*- coding: iso-8859-1 -*-
 """
-    MoinMoin - C++ Source Parser
-
-    @copyright: 2002 Taesu Pyo <bigflood@hitel.net>
-    @license: GNU GPL, see COPYING for details.
-
-css:
-
-pre.cpparea     { font-style: sans-serif; color: #000000; }
+    MoinMoin - C/C++ Source Parser
+    
+    DEPRECATED compatibility wrapper calling the highlight parser.
+    
+    This is to support (deprecated) existing syntax like:
+    {{{#!cplusplus ...
+    ...
+    }}}
 
-pre.cpparea span.ID       { color: #000000; }
-pre.cpparea span.Char     { color: #004080; }
-pre.cpparea span.Comment  { color: #808080; }
-pre.cpparea span.Number   { color: #008080; font-weight: bold; }
-pre.cpparea span.String   { color: #004080; }
-pre.cpparea span.SPChar   { color: #0000C0; }
-pre.cpparea span.ResWord  { color: #4040ff; font-weight: bold; }
-pre.cpparea span.ConsWord { color: #008080; font-weight: bold; }
-pre.cpparea span.ResWord2 { color: #0080ff; font-weight: bold; }
-pre.cpparea span.Special  { color: #0000ff; }
-pre.cpparea span.Preprc   { color: #804000; }
+    It is equivalent to the new way to highlight code:
+    {{{#!highlight cpp ...
+    ...
+    }}}
 
+    @copyright: 2008 MoinMoin:ThomasWaldmann
+    @license: GNU GPL, see COPYING for details.
 """
 
-from MoinMoin.parser._ParserBase import ParserBase
-
-Dependencies = ['user'] # the "Toggle line numbers link" depends on user's language
-
-class Parser(ParserBase):
-
-    parsername = "ColorizedCPlusPlus"
-    extensions = ['.c', '.h', '.cpp', '.c++']
-    Dependencies = Dependencies
-
-    def setupRules(self):
-        ParserBase.setupRules(self)
-
-        self.addRulePair("Comment", r"/[*]", r"[*]/")
-        self.addRule("Comment", r"//.*$")
-        self.addRulePair("String", r'L?"', r'$|[^\\](\\\\)*"')
-        self.addRule("Char", r"'\\.'|'[^\\]'")
-        self.addRule("Number", r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?")
-        self.addRule("Preprc", r"^\s*#(.*\\\n)*(.*(?!\\))$")
-        self.addRule("ID", r"[a-zA-Z_][0-9a-zA-Z_]*")
-        self.addRule("SPChar", r"[~!%^&*()+=|\[\]:;,.<>/?{}-]")
+from MoinMoin.parser.highlight import Parser as HighlightParser
+from MoinMoin.parser.highlight import Dependencies
 
-        reserved_words = ['struct', 'class', 'union', 'enum',
-        'int', 'float', 'double', 'signed', 'unsigned', 'char', 'short', 'void', 'bool',
-        'long', 'register', 'auto', 'operator',
-        'static', 'const', 'private', 'public', 'protected', 'virtual', 'explicit',
-        'new', 'delete', 'this',
-        'if', 'else', 'while', 'for', 'do', 'switch', 'case', 'default', 'sizeof',
-        'dynamic_cast', 'static_cast', 'const_cast', 'reinterpret_cast', 'typeid',
-        'try', 'catch', 'throw', 'throws', 'return', 'continue', 'break', 'goto']
+class Parser(HighlightParser):
+    parsername = 'cpp'  # Lexer name pygments recognizes
 
-        reserved_words2 = ['extern', 'volatile', 'typedef', 'friend',
-                           '__declspec', 'inline', '__asm', 'thread', 'naked',
-                           'dllimport', 'dllexport', 'namespace', 'using',
-                           'template', 'typename', 'goto']
-
-        special_words = ['std', 'string', 'vector', 'map', 'set', 'cout', 'cin', 'cerr', 'endl']
-        constant_words = ['true', 'false', 'NULL']
-
-        self.addReserved(reserved_words)
-        self.addConstant(constant_words)
-
-        self.addWords(reserved_words2, 'ResWord2')
-        self.addWords(special_words, 'Special')
-
--- a/MoinMoin/parser/text_java.py	Sat Oct 11 23:12:42 2008 +0200
+++ b/MoinMoin/parser/text_java.py	Sat Oct 11 23:54:29 2008 +0200
@@ -1,43 +1,26 @@
 # -*- coding: iso-8859-1 -*-
 """
     MoinMoin - Java Source Parser
+    
+    DEPRECATED compatibility wrapper calling the highlight parser.
+    
+    This is to support (deprecated) existing syntax like:
+    {{{#!java ...
+    ...
+    }}}
 
-    @copyright: 2002 Taesu Pyo <bigflood@hitel.net>
+    It is equivalent to the new way to highlight code:
+    {{{#!highlight java ...
+    ...
+    }}}
+
+    @copyright: 2008 MoinMoin:ThomasWaldmann
     @license: GNU GPL, see COPYING for details.
-
 """
 
-from MoinMoin.parser._ParserBase import ParserBase
-
-Dependencies = ['user'] # the "Toggle line numbers link" depends on user's language
-
-class Parser(ParserBase):
-
-    parsername = "ColorizedJava"
-    extensions = ['.java']
-    Dependencies = Dependencies
-
-    def setupRules(self):
-        ParserBase.setupRules(self)
+from MoinMoin.parser.highlight import Parser as HighlightParser
+from MoinMoin.parser.highlight import Dependencies
 
-        self.addRulePair("Comment", r"/[*]", r"[*]/")
-        self.addRule("Comment", r"//.*$")
-        self.addRulePair("String", r'"', r'$|[^\\](\\\\)*"')
-        self.addRule("Char", r"'\\.'|'[^\\]'")
-        self.addRule("Number", r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?")
-        self.addRule("ID", r"[a-zA-Z_][0-9a-zA-Z_]*")
-        self.addRule("SPChar", r"[~!%^&*()+=|\[\]:;,.<>/?{}-]")
+class Parser(HighlightParser):
+    parsername = 'java'  # Lexer name pygments recognizes
 
-        reserved_words = ['class', 'interface', 'enum', 'import', 'package',
-        'byte', 'int', 'long', 'float', 'double', 'char', 'short', 'void', 'boolean',
-        'static', 'final', 'const', 'private', 'public', 'protected',
-        'new', 'this', 'super', 'abstract', 'native', 'synchronized', 'transient', 'volatile', 'strictfp',
-        'extends', 'implements', 'if', 'else', 'while', 'for', 'do', 'switch', 'case', 'default', 'instanceof',
-        'try', 'catch', 'finally', 'throw', 'throws', 'return', 'continue', 'break']
-
-        self.addReserved(reserved_words)
-
-        constant_words = ['true', 'false', 'null']
-
-        self.addConstant(constant_words)
-
--- a/MoinMoin/parser/text_pascal.py	Sat Oct 11 23:12:42 2008 +0200
+++ b/MoinMoin/parser/text_pascal.py	Sat Oct 11 23:54:29 2008 +0200
@@ -1,51 +1,26 @@
 # -*- coding: iso-8859-1 -*-
 """
     MoinMoin - Pascal Source Parser
+    
+    DEPRECATED compatibility wrapper calling the highlight parser.
+    
+    This is to support (deprecated) existing syntax like:
+    {{{#!pascal ...
+    ...
+    }}}
 
-    @copyright: 2004-2005 Johannes Berg <johannes@sipsolutions.net>
+    It is equivalent to the new way to highlight code:
+    {{{#!highlight pascal ...
+    ...
+    }}}
+
+    @copyright: 2008 MoinMoin:ThomasWaldmann
     @license: GNU GPL, see COPYING for details.
 """
 
-from MoinMoin.parser._ParserBase import ParserBase
-
-Dependencies = ['user'] # the "Toggle line numbers link" depends on user's language
-
-class Parser(ParserBase):
-
-    parsername = 'ColorizedPascal'
-    extensions = ['.pas']
-    Dependencies = Dependencies
-
-    def __init__(self, raw, request, **kw):
-        ParserBase.__init__(self, raw, request, **kw)
-        self._ignore_case = True
-
-    def setupRules(self):
-        ParserBase.setupRules(self)
+from MoinMoin.parser.highlight import Parser as HighlightParser
+from MoinMoin.parser.highlight import Dependencies
 
-        self.addRulePair("Comment", r"\(\*", r"\*\)")
-        self.addRulePair("Comment", r"\{", r"\}")
-        self.addRule("Comment", r"//.*$")
-        self.addRulePair("String", r"'", r"'")
-        self.addRule("Char", r"'\\.'|#[a-f0-9][a-f0-9]")
-        self.addRule("Number", r"[0-9](\.[0-9]*)?(eE[+-][0-9])?|\$[0-9a-fA-F]+")
-        self.addRule("ID", r"[a-zA-Z_][0-9a-zA-Z_]*")
-        self.addRule("SPChar", r"[~!%^&*()+=|\[\]:;,.<>/?{}-]")
+class Parser(HighlightParser):
+    parsername = 'pascal'  # Lexer name pygments recognizes
 
-        reserved_words = ['class', 'interface', 'set', 'uses', 'unit',
-                          'byte', 'integer', 'longint', 'float', 'double',
-                          'extended', 'char', 'shortint', 'boolean',
-                          'var', 'const', 'private', 'public', 'protected',
-                          'new', 'this', 'super', 'abstract', 'native',
-                          'synchronized', 'transient', 'volatile', 'strictfp',
-                          'if', 'else', 'while', 'for', 'do', 'case', 'default',
-                          'try', 'except', 'finally', 'raise', 'continue', 'break',
-                          'begin', 'end', 'type', 'class', 'implementation',
-                          'procedure', 'function', 'constructor', 'destructor', 'program']
-
-        self.addReserved(reserved_words)
-
-        constant_words = ['true', 'false', 'nil']
-
-        self.addConstant(constant_words)
-
--- a/MoinMoin/parser/text_python.py	Sat Oct 11 23:12:42 2008 +0200
+++ b/MoinMoin/parser/text_python.py	Sat Oct 11 23:54:29 2008 +0200
@@ -1,136 +1,26 @@
 # -*- coding: iso-8859-1 -*-
 """
-    MoinMoin - highlighting Python Source Parser
+    MoinMoin - Python Source Parser
+    
+    DEPRECATED compatibility wrapper calling the highlight parser.
+    
+    This is to support (deprecated) existing syntax like:
+    {{{#!python ...
+    ...
+    }}}
 
-    @copyright: 2001 Juergen Hermann <jh@web.de>,
-                2006-2008 MoinMoin:ThomasWaldmann
+    It is equivalent to the new way to highlight code:
+    {{{#!highlight python ...
+    ...
+    }}}
+
+    @copyright: 2008 MoinMoin:ThomasWaldmann
     @license: GNU GPL, see COPYING for details.
 """
 
-import StringIO
-import keyword, token, tokenize
-
-from MoinMoin import config, wikiutil
-from MoinMoin.parser._ParserBase import parse_start_step
-from MoinMoin.support.python_compatibility import hash_new
-
-_KEYWORD = token.NT_OFFSET + 1
-_TEXT = token.NT_OFFSET + 2
-
-_tokens = {
-    token.NUMBER: 'Number',
-    token.OP: 'Operator',
-    token.STRING: 'String',
-    tokenize.COMMENT: 'Comment',
-    token.NAME: 'ID',
-    token.ERRORTOKEN: 'Error',
-    _KEYWORD: 'ResWord',
-    _TEXT: 'Text',
-}
-
-Dependencies = ['user'] # the "Toggle line numbers link" depends on user's language
-
-class Parser:
-    """ Send colored python source.
-    """
-
-    extensions = ['.py']
-    Dependencies = Dependencies
-
-    def __init__(self, raw, request, **kw):
-        """ Store the source text.
-        """
-        self.raw = raw.expandtabs().rstrip()
-        self.request = request
-        self.form = request.form
-        self._ = request.getText
-
-        self.show_num, self.num_start, self.num_step, attrs = parse_start_step(request, kw.get('format_args', ''))
-
-    def format(self, formatter):
-        """ Parse and send the colored source.
-        """
-        # store line offsets in self.lines
-        self.lines = [0, 0]
-        pos = 0
-        while 1:
-            try:
-                pos = self.raw.index('\n', pos) + 1
-            except ValueError:
-                break
-            self.lines.append(pos)
-        self.lines.append(len(self.raw))
-
-        self.result = [] # collects output
-
-        self._code_id = hash_new('sha1', self.raw.encode(config.charset)).hexdigest()
-        self.result.append(formatter.code_area(1, self._code_id, 'ColorizedPython', self.show_num, self.num_start, self.num_step))
-        self.formatter = formatter
-        self.result.append(formatter.code_line(1))
-        #len('%d' % (len(self.lines)-1, )))
+from MoinMoin.parser.highlight import Parser as HighlightParser
+from MoinMoin.parser.highlight import Dependencies
 
-        # parse the source and write it
-        self.pos = 0
-        text = StringIO.StringIO(self.raw)
-        try:
-            tokenize.tokenize(text.readline, self)
-        except IndentationError, ex:
-            msg = ex[0]
-            errmsg = (self.formatter.linebreak() +
-                      self.formatter.strong(1) + "ERROR: %s" % msg + self.formatter.strong(0) +
-                      self.formatter.linebreak())
-            self.result.append(errmsg)
-        except tokenize.TokenError, ex:
-            msg = ex[0]
-            line = ex[1][0]
-            errmsg = (self.formatter.linebreak() +
-                      self.formatter.strong(1) + "ERROR: %s" % msg + self.formatter.strong(0) +
-                      self.formatter.linebreak() +
-                      wikiutil.escape(self.raw[self.lines[line]:]))
-            self.result.append(errmsg)
-        self.result.append(self.formatter.code_line(0))
-        self.result.append(formatter.code_area(0, self._code_id))
-        self.request.write(''.join(self.result))
-
-    def __call__(self, toktype, toktext, (srow, scol), (erow, ecol), line):
-        """ Token handler.
-        """
-        # calculate new positions
-        oldpos = self.pos
-        newpos = self.lines[srow] + scol
-        self.pos = newpos + len(toktext)
+class Parser(HighlightParser):
+    parsername = 'python'  # Lexer name pygments recognizes
 
-        # handle newlines
-        if toktype in [token.NEWLINE, tokenize.NL]:
-            self.result.append(self.formatter.code_line(0))
-            self.result.append(self.formatter.code_line(1))
-            return
-
-        # send the original whitespace, if needed
-        if newpos > oldpos:
-            self.result.append(self.formatter.text(self.raw[oldpos:newpos]))
-
-        # skip indenting tokens
-        if toktype in [token.INDENT, token.DEDENT]:
-            self.pos = newpos
-            return
-
-        # map token type to a color group
-        if token.LPAR <= toktype and toktype <= token.OP:
-            toktype = token.OP
-        elif toktype == token.NAME and keyword.iskeyword(toktext):
-            toktype = _KEYWORD
-        tokid = _tokens.get(toktype, _tokens[_TEXT])
-
-        # send text
-        first = True
-        for part in toktext.split('\n'):
-            if not first:
-                self.result.append(self.formatter.code_line(0))
-                self.result.append(self.formatter.code_line(1))
-            else:
-                first = False
-            self.result.append(self.formatter.code_token(1, tokid) +
-                               self.formatter.text(part) +
-                               self.formatter.code_token(0, tokid))
-