changeset 52:672b181a8ce0

HTML Support - Change module name, cleanup contents
author Bastian Blank <bblank@thinkmo.de>
date Mon, 20 Jul 2009 11:07:06 +0200
parents 847897e5fab8
children d071598a93ef
files emeraldtree/HTMLTreeBuilder.py emeraldtree/html.py emeraldtree/tests/test_HTMLTreeBuilder.py emeraldtree/tests/test_html.py
diffstat 4 files changed, 251 insertions(+), 274 deletions(-) [+]
line wrap: on
line diff
--- a/emeraldtree/HTMLTreeBuilder.py	Sun Sep 21 11:01:19 2008 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,225 +0,0 @@
-#
-# ElementTree
-# $Id: HTMLTreeBuilder.py 3265 2007-09-06 20:42:00Z fredrik $
-#
-# a simple tree builder, for HTML input
-#
-# history:
-# 2002-04-06 fl   created
-# 2002-04-07 fl   ignore IMG and HR end tags
-# 2002-04-07 fl   added support for 1.5.2 and later
-# 2003-04-13 fl   added HTMLTreeBuilder alias
-# 2004-12-02 fl   don't feed non-ASCII charrefs/entities as 8-bit strings
-# 2004-12-05 fl   don't feed non-ASCII CDATA as 8-bit strings
-#
-# Copyright (c) 1999-2004 by Fredrik Lundh.  All rights reserved.
-#
-# fredrik@pythonware.com
-# http://www.pythonware.com
-#
-# --------------------------------------------------------------------
-# The ElementTree toolkit is
-#
-# Copyright (c) 1999-2007 by Fredrik Lundh
-#
-# By obtaining, using, and/or copying this software and/or its
-# associated documentation, you agree that you have read, understood,
-# and will comply with the following terms and conditions:
-#
-# Permission to use, copy, modify, and distribute this software and
-# its associated documentation for any purpose and without fee is
-# hereby granted, provided that the above copyright notice appears in
-# all copies, and that both that copyright notice and this permission
-# notice appear in supporting documentation, and that the name of
-# Secret Labs AB or the author not be used in advertising or publicity
-# pertaining to distribution of the software without specific, written
-# prior permission.
-#
-# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
-# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
-# ABILITY AND FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
-# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
-# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
-# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
-# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
-# OF THIS SOFTWARE.
-# --------------------------------------------------------------------
-
-##
-# Tools to build element trees from HTML files.
-##
-
-import htmlentitydefs
-import re
-import mimetools, StringIO
-from HTMLParser import HTMLParser as HTMLParserBase
-
-import ElementTree
-
-
-##
-# ElementTree builder for HTML source code.  This builder converts an
-# HTML document or fragment to an ElementTree.
-# <p>
-# The parser is relatively picky, and requires balanced tags for most
-# elements.  However, elements belonging to the following group are
-# automatically closed: P, LI, TR, TH, and TD.  In addition, the
-# parser automatically inserts end tags immediately after the start
-# tag, and ignores any end tags for the following group: IMG, HR,
-# META, and LINK.
-#
-# @keyparam builder Optional builder object.  If omitted, the parser
-#     uses the standard <b>elementtree</b> builder.
-# @keyparam encoding Optional character encoding, if known.  If omitted,
-#     the parser looks for META tags inside the document.  If no tags
-#     are found, the parser defaults to ISO-8859-1.  Note that if your
-#     document uses a non-ASCII compatible encoding, you must decode
-#     the document before parsing.
-#
-# @see elementtree.ElementTree
-
-class HTMLParser(HTMLParserBase):
-    AUTOCLOSE = "p", "li", "tr", "th", "td", "head", "body"
-    IGNOREEND = "img", "hr", "meta", "link", "br", "input", "col"
-
-    namespace = "http://www.w3.org/1999/xhtml"
-
-    def __init__(self, builder=None, encoding=None):
-        self.__stack = []
-        if builder is None:
-            builder = ElementTree.TreeBuilder()
-        self.__builder = builder
-        self.encoding = encoding or "iso-8859-1"
-        HTMLParserBase.__init__(self)
-
-    ##
-    # Flushes parser buffers, and return the root element.
-    #
-    # @return An Element instance.
-
-    def close(self):
-        HTMLParserBase.close(self)
-        return self.__builder.close()
-
-    ##
-    # (Internal) Handles start tags.
-
-    def handle_starttag(self, tag, attrs):
-        tag = ElementTree.QName(tag.lower(), self.namespace)
-        if tag.name == "meta":
-            # look for encoding directives
-            http_equiv = content = None
-            for k, v in attrs:
-                if k == "http-equiv":
-                    http_equiv = v.lower()
-                elif k == "content":
-                    content = v
-            if http_equiv == "content-type" and content:
-                # use mimetools to parse the http header
-                header = mimetools.Message(
-                    StringIO.StringIO("%s: %s\n\n" % (http_equiv, content))
-                    )
-                encoding = header.getparam("charset")
-                if encoding:
-                    self.encoding = encoding
-        if tag.name in self.AUTOCLOSE:
-            if self.__stack and self.__stack[-1] == tag:
-                self.handle_endtag(tag)
-        self.__stack.append(tag)
-        attrib = {}
-        if attrs:
-            for key, value in attrs:
-                # Handle short attributes
-                if value is None:
-                    value = key
-                key = ElementTree.QName(key.lower(), self.namespace)
-                attrib[key] = value
-        self.__builder.start(tag, attrib)
-        if tag.name in self.IGNOREEND:
-            self.__stack.pop()
-            self.__builder.end(tag)
-
-    ##
-    # (Internal) Handles end tags.
-
-    def handle_endtag(self, tag):
-        if not isinstance(tag, ElementTree.QName):
-            tag = ElementTree.QName(tag.lower(), self.namespace)
-        if tag.name in self.IGNOREEND:
-            return
-        lasttag = self.__stack.pop()
-        if tag != lasttag and lasttag.name in self.AUTOCLOSE:
-            self.handle_endtag(lasttag)
-        self.__builder.end(tag)
-
-    ##
-    # (Internal) Handles character references.
-
-    def handle_charref(self, char):
-        if char[:1] == "x":
-            char = int(char[1:], 16)
-        else:
-            char = int(char)
-        if 0 <= char < 128:
-            self.__builder.data(chr(char))
-        else:
-            self.__builder.data(unichr(char))
-
-    ##
-    # (Internal) Handles entity references.
-
-    def handle_entityref(self, name):
-        entity = htmlentitydefs.entitydefs.get(name)
-        if entity:
-            if len(entity) == 1:
-                entity = ord(entity)
-            else:
-                entity = int(entity[2:-1])
-            if 0 <= entity < 128:
-                self.__builder.data(chr(entity))
-            else:
-                self.__builder.data(unichr(entity))
-        else:
-            self.unknown_entityref(name)
-
-    ##
-    # (Internal) Handles character data.
-
-    def handle_data(self, data):
-        if isinstance(data, str):
-            # convert to unicode, but only if necessary
-            data = unicode(data, self.encoding, "ignore")
-        self.__builder.data(data)
-
-    ##
-    # (Hook) Handles unknown entity references.  The default action
-    # is to ignore unknown entities.
-
-    def unknown_entityref(self, name):
-        pass # ignore by default; override if necessary
-
-##
-# An alias for the <b>HTMLParser</b> class.
-
-TreeBuilder = HTMLTreeBuilder = HTMLParser
-
-##
-# Parse an HTML document or document fragment.
-#
-# @param source A filename or file object containing HTML data.
-# @param encoding Optional character encoding, if known.  If omitted,
-#     the parser looks for META tags inside the document.  If no tags
-#     are found, the parser defaults to ISO-8859-1.
-# @return An ElementTree instance
-
-def parse(source, encoding=None):
-    return ElementTree.parse(source, HTMLTreeBuilder(encoding=encoding))
-
-def HTML(text):
-    parser = HTMLParser()
-    parser.feed(text)
-    return parser.close()
-
-if __name__ == "__main__":
-    import sys
-    ElementTree.dump(parse(open(sys.argv[1])))
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/emeraldtree/html.py	Mon Jul 20 11:07:06 2009 +0200
@@ -0,0 +1,202 @@
+#
+# The ElementTree toolkit is
+#
+# Copyright (c) 1999-2007 by Fredrik Lundh
+#               2008-2009 Bastian Blank <bblank@thinkmo.de>
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Secret Labs AB or the author not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+# ABILITY AND FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+# OF THIS SOFTWARE.
+# --------------------------------------------------------------------
+
+##
+# Tools to build element trees from HTML files.
+##
+
+import htmlentitydefs
+import re
+import mimetools, StringIO
+from HTMLParser import HTMLParser as HTMLParserBase
+
+import ElementTree
+
+
+##
+# ElementTree builder for HTML source code.  This builder converts an
+# HTML document or fragment to an ElementTree.
+# <p>
+# The parser is relatively picky, and requires balanced tags for most
+# elements.  However, elements belonging to the following group are
+# automatically closed: P, LI, TR, TH, and TD.  In addition, the
+# parser automatically inserts end tags immediately after the start
+# tag, and ignores any end tags for the following group: IMG, HR,
+# META, and LINK.
+#
+# @keyparam builder Optional builder object.  If omitted, the parser
+#     uses the standard <b>elementtree</b> builder.
+# @keyparam encoding Optional character encoding, if known.  If omitted,
+#     the parser looks for META tags inside the document.  If no tags
+#     are found, the parser defaults to ISO-8859-1.  Note that if your
+#     document uses a non-ASCII compatible encoding, you must decode
+#     the document before parsing.
+#
+# @see elementtree.ElementTree
+
+class HTMLParser(HTMLParserBase):
+    AUTOCLOSE = "p", "li", "tr", "th", "td", "head", "body"
+    IGNOREEND = "img", "hr", "meta", "link", "br", "input", "col"
+
+    namespace = "http://www.w3.org/1999/xhtml"
+
+    def __init__(self, encoding=None, builder=None):
+        self.__stack = []
+        if builder is None:
+            builder = ElementTree.TreeBuilder()
+        self.__builder = builder
+        self.encoding = encoding or "iso-8859-1"
+        HTMLParserBase.__init__(self)
+
+    ##
+    # Flushes parser buffers, and return the root element.
+    #
+    # @return An Element instance.
+
+    def close(self):
+        HTMLParserBase.close(self)
+        return self.__builder.close()
+
+    ##
+    # (Internal) Handles start tags.
+
+    def handle_starttag(self, tag, attrs):
+        tag = ElementTree.QName(tag.lower(), self.namespace)
+        if tag.name == "meta":
+            # look for encoding directives
+            http_equiv = content = None
+            for k, v in attrs:
+                if k == "http-equiv":
+                    http_equiv = v.lower()
+                elif k == "content":
+                    content = v
+            if http_equiv == "content-type" and content:
+                # use mimetools to parse the http header
+                header = mimetools.Message(
+                    StringIO.StringIO("%s: %s\n\n" % (http_equiv, content))
+                    )
+                encoding = header.getparam("charset")
+                if encoding:
+                    self.encoding = encoding
+        if tag.name in self.AUTOCLOSE:
+            if self.__stack and self.__stack[-1] == tag:
+                self.handle_endtag(tag)
+        self.__stack.append(tag)
+        attrib = {}
+        if attrs:
+            for key, value in attrs:
+                # Handle short attributes
+                if value is None:
+                    value = key
+                key = ElementTree.QName(key.lower(), self.namespace)
+                attrib[key] = value
+        self.__builder.start(tag, attrib)
+        if tag.name in self.IGNOREEND:
+            self.__stack.pop()
+            self.__builder.end(tag)
+
+    ##
+    # (Internal) Handles end tags.
+
+    def handle_endtag(self, tag):
+        if not isinstance(tag, ElementTree.QName):
+            tag = ElementTree.QName(tag.lower(), self.namespace)
+        if tag.name in self.IGNOREEND:
+            return
+        lasttag = self.__stack.pop()
+        if tag != lasttag and lasttag.name in self.AUTOCLOSE:
+            self.handle_endtag(lasttag)
+        self.__builder.end(tag)
+
+    ##
+    # (Internal) Handles character references.
+
+    def handle_charref(self, char):
+        if char[:1] == "x":
+            char = int(char[1:], 16)
+        else:
+            char = int(char)
+        if 0 <= char < 128:
+            self.__builder.data(chr(char))
+        else:
+            self.__builder.data(unichr(char))
+
+    ##
+    # (Internal) Handles entity references.
+
+    def handle_entityref(self, name):
+        entity = htmlentitydefs.entitydefs.get(name)
+        if entity:
+            if len(entity) == 1:
+                entity = ord(entity)
+            else:
+                entity = int(entity[2:-1])
+            if 0 <= entity < 128:
+                self.__builder.data(chr(entity))
+            else:
+                self.__builder.data(unichr(entity))
+        else:
+            self.unknown_entityref(name)
+
+    ##
+    # (Internal) Handles character data.
+
+    def handle_data(self, data):
+        if isinstance(data, str):
+            # convert to unicode, but only if necessary
+            data = unicode(data, self.encoding, "ignore")
+        self.__builder.data(data)
+
+    ##
+    # (Hook) Handles unknown entity references.  The default action
+    # is to ignore unknown entities.
+
+    def unknown_entityref(self, name):
+        pass # ignore by default; override if necessary
+
+##
+# Parse an HTML document or document fragment.
+#
+# @param source A filename or file object containing HTML data.
+# @param encoding Optional character encoding, if known.  If omitted,
+#     the parser looks for META tags inside the document.  If no tags
+#     are found, the parser defaults to ISO-8859-1.
+# @return An ElementTree instance
+
+def parse(source, encoding=None):
+    return ElementTree.parse(source, HTMLTreeBuilder(encoding=encoding))
+
+def HTML(text):
+    parser = HTMLParser()
+    parser.feed(text)
+    return parser.close()
+
+if __name__ == "__main__":
+    import sys
+    ElementTree.dump(parse(open(sys.argv[1])))
--- a/emeraldtree/tests/test_HTMLTreeBuilder.py	Sun Sep 21 11:01:19 2008 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,49 +0,0 @@
-import py.test
-from emeraldtree.ElementTree import *
-from emeraldtree.HTMLTreeBuilder import *
-
-def serialize(elem, **options):
-    from cStringIO import StringIO
-    file = StringIO()
-    tree = ElementTree(elem)
-    tree.write(file, **options)
-    return file.getvalue()
-
-def test_HTMLParser_simple1():
-    elem = HTML('<a></a>')
-    assert elem.tag.name == 'a'
-    assert len(elem) == 0
-
-def test_XMLParser_simple2():
-    elem = HTML('<a><b /></a>')
-    assert elem.tag.name == 'a'
-    assert len(elem) == 1
-    assert elem[0].tag.name == 'b'
-    assert len(elem[0]) == 0
-
-def test_XMLParser_text1():
-    elem = HTML('<a>b</a>')
-    assert elem.tag.name == 'a'
-    assert isinstance(elem, Element)
-    assert len(elem) == 1
-    assert elem[0] == 'b'
-    assert isinstance(elem[0], unicode)
-
-def test_XMLParser_text2():
-    elem = HTML('<a>b<c>d</c>d</a>')
-    assert elem.tag.name == 'a'
-    assert len(elem) == 3
-    assert elem[0] == 'b'
-    assert elem[1].tag.name == 'c'
-    assert elem[2] == 'd'
-
-def test_HTMLParser_ignoreend():
-    elem = HTML('<br>')
-    assert elem.tag.name == 'br'
-    assert len(elem) == 0
-
-    elem = HTML('<br></br>')
-    assert elem.tag.name == 'br'
-    assert len(elem) == 0
-
-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/emeraldtree/tests/test_html.py	Mon Jul 20 11:07:06 2009 +0200
@@ -0,0 +1,49 @@
+import py.test
+from emeraldtree.ElementTree import *
+from emeraldtree.html import *
+
+def serialize(elem, **options):
+    from cStringIO import StringIO
+    file = StringIO()
+    tree = ElementTree(elem)
+    tree.write(file, **options)
+    return file.getvalue()
+
+def test_HTMLParser_simple1():
+    elem = HTML('<a></a>')
+    assert elem.tag.name == 'a'
+    assert len(elem) == 0
+
+def test_HTMLParser_simple2():
+    elem = HTML('<a><b /></a>')
+    assert elem.tag.name == 'a'
+    assert len(elem) == 1
+    assert elem[0].tag.name == 'b'
+    assert len(elem[0]) == 0
+
+def test_HTMLParser_text1():
+    elem = HTML('<a>b</a>')
+    assert elem.tag.name == 'a'
+    assert isinstance(elem, Element)
+    assert len(elem) == 1
+    assert elem[0] == 'b'
+    assert isinstance(elem[0], unicode)
+
+def test_HTMLParser_text2():
+    elem = HTML('<a>b<c>d</c>d</a>')
+    assert elem.tag.name == 'a'
+    assert len(elem) == 3
+    assert elem[0] == 'b'
+    assert elem[1].tag.name == 'c'
+    assert elem[2] == 'd'
+
+def test_HTMLParser_ignoreend():
+    elem = HTML('<br>')
+    assert elem.tag.name == 'br'
+    assert len(elem) == 0
+
+    elem = HTML('<br></br>')
+    assert elem.tag.name == 'br'
+    assert len(elem) == 0
+
+