HTMLTreeBuilder.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230
  1. #
  2. # ElementTree
  3. # $Id: HTMLTreeBuilder.py 2325 2005-03-16 15:50:43Z fredrik $
  4. #
  5. # a simple tree builder, for HTML input
  6. #
  7. # history:
  8. # 2002-04-06 fl created
  9. # 2002-04-07 fl ignore IMG and HR end tags
  10. # 2002-04-07 fl added support for 1.5.2 and later
  11. # 2003-04-13 fl added HTMLTreeBuilder alias
  12. # 2004-12-02 fl don't feed non-ASCII charrefs/entities as 8-bit strings
  13. # 2004-12-05 fl don't feed non-ASCII CDATA as 8-bit strings
  14. #
  15. # Copyright (c) 1999-2004 by Fredrik Lundh. All rights reserved.
  16. #
  17. # fredrik@pythonware.com
  18. # http://www.pythonware.com
  19. #
  20. # --------------------------------------------------------------------
  21. # The ElementTree toolkit is
  22. #
  23. # Copyright (c) 1999-2004 by Fredrik Lundh
  24. #
  25. # By obtaining, using, and/or copying this software and/or its
  26. # associated documentation, you agree that you have read, understood,
  27. # and will comply with the following terms and conditions:
  28. #
  29. # Permission to use, copy, modify, and distribute this software and
  30. # its associated documentation for any purpose and without fee is
  31. # hereby granted, provided that the above copyright notice appears in
  32. # all copies, and that both that copyright notice and this permission
  33. # notice appear in supporting documentation, and that the name of
  34. # Secret Labs AB or the author not be used in advertising or publicity
  35. # pertaining to distribution of the software without specific, written
  36. # prior permission.
  37. #
  38. # SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
  39. # TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
  40. # ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
  41. # BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
  42. # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  43. # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  44. # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  45. # OF THIS SOFTWARE.
  46. # --------------------------------------------------------------------
  47. ##
  48. # Tools to build element trees from HTML files.
  49. ##
  50. import htmlentitydefs
  51. import re, string, sys
  52. import mimetools, StringIO
  53. import ElementTree
  54. AUTOCLOSE = "p", "li", "tr", "th", "td", "head", "body"
  55. IGNOREEND = "img", "hr", "meta", "link", "br"
  56. if sys.version[:3] == "1.5":
  57. is_not_ascii = re.compile(r"[\x80-\xff]").search # 1.5.2
  58. else:
  59. is_not_ascii = re.compile(eval(r'u"[\u0080-\uffff]"')).search
  60. try:
  61. from HTMLParser import HTMLParser
  62. except ImportError:
  63. from sgmllib import SGMLParser
  64. # hack to use sgmllib's SGMLParser to emulate 2.2's HTMLParser
  65. class HTMLParser(SGMLParser):
  66. # the following only works as long as this class doesn't
  67. # provide any do, start, or end handlers
  68. def unknown_starttag(self, tag, attrs):
  69. self.handle_starttag(tag, attrs)
  70. def unknown_endtag(self, tag):
  71. self.handle_endtag(tag)
  72. ##
  73. # ElementTree builder for HTML source code. This builder converts an
  74. # HTML document or fragment to an ElementTree.
  75. # <p>
  76. # The parser is relatively picky, and requires balanced tags for most
  77. # elements. However, elements belonging to the following group are
  78. # automatically closed: P, LI, TR, TH, and TD. In addition, the
  79. # parser automatically inserts end tags immediately after the start
  80. # tag, and ignores any end tags for the following group: IMG, HR,
  81. # META, and LINK.
  82. #
  83. # @keyparam builder Optional builder object. If omitted, the parser
  84. # uses the standard <b>elementtree</b> builder.
  85. # @keyparam encoding Optional character encoding, if known. If omitted,
  86. # the parser looks for META tags inside the document. If no tags
  87. # are found, the parser defaults to ISO-8859-1. Note that if your
  88. # document uses a non-ASCII compatible encoding, you must decode
  89. # the document before parsing.
  90. #
  91. # @see elementtree.ElementTree
  92. class HTMLTreeBuilder(HTMLParser):
  93. # FIXME: shouldn't this class be named Parser, not Builder?
  94. def __init__(self, builder=None, encoding=None):
  95. self.__stack = []
  96. if builder is None:
  97. builder = ElementTree.TreeBuilder()
  98. self.__builder = builder
  99. self.encoding = encoding or "iso-8859-1"
  100. HTMLParser.__init__(self)
  101. ##
  102. # Flushes parser buffers, and return the root element.
  103. #
  104. # @return An Element instance.
  105. def close(self):
  106. HTMLParser.close(self)
  107. return self.__builder.close()
  108. ##
  109. # (Internal) Handles start tags.
  110. def handle_starttag(self, tag, attrs):
  111. if tag == "meta":
  112. # look for encoding directives
  113. http_equiv = content = None
  114. for k, v in attrs:
  115. if k == "http-equiv":
  116. http_equiv = string.lower(v)
  117. elif k == "content":
  118. content = v
  119. if http_equiv == "content-type" and content:
  120. # use mimetools to parse the http header
  121. header = mimetools.Message(
  122. StringIO.StringIO("%s: %s\n\n" % (http_equiv, content))
  123. )
  124. encoding = header.getparam("charset")
  125. if encoding:
  126. self.encoding = encoding
  127. if tag in AUTOCLOSE:
  128. if self.__stack and self.__stack[-1] == tag:
  129. self.handle_endtag(tag)
  130. self.__stack.append(tag)
  131. attrib = {}
  132. if attrs:
  133. for k, v in attrs:
  134. attrib[string.lower(k)] = v
  135. self.__builder.start(tag, attrib)
  136. if tag in IGNOREEND:
  137. self.__stack.pop()
  138. self.__builder.end(tag)
  139. ##
  140. # (Internal) Handles end tags.
  141. def handle_endtag(self, tag):
  142. if tag in IGNOREEND:
  143. return
  144. lasttag = self.__stack.pop()
  145. if tag != lasttag and lasttag in AUTOCLOSE:
  146. self.handle_endtag(lasttag)
  147. self.__builder.end(tag)
  148. ##
  149. # (Internal) Handles character references.
  150. def handle_charref(self, char):
  151. if char[:1] == "x":
  152. char = int(char[1:], 16)
  153. else:
  154. char = int(char)
  155. if 0 <= char < 128:
  156. self.__builder.data(chr(char))
  157. else:
  158. self.__builder.data(unichr(char))
  159. ##
  160. # (Internal) Handles entity references.
  161. def handle_entityref(self, name):
  162. entity = htmlentitydefs.entitydefs.get(name)
  163. if entity:
  164. if len(entity) == 1:
  165. entity = ord(entity)
  166. else:
  167. entity = int(entity[2:-1])
  168. if 0 <= entity < 128:
  169. self.__builder.data(chr(entity))
  170. else:
  171. self.__builder.data(unichr(entity))
  172. else:
  173. self.unknown_entityref(name)
  174. ##
  175. # (Internal) Handles character data.
  176. def handle_data(self, data):
  177. if isinstance(data, type('')) and is_not_ascii(data):
  178. # convert to unicode, but only if necessary
  179. data = unicode(data, self.encoding, "ignore")
  180. self.__builder.data(data)
  181. ##
  182. # (Hook) Handles unknown entity references. The default action
  183. # is to ignore unknown entities.
  184. def unknown_entityref(self, name):
  185. pass # ignore by default; override if necessary
  186. ##
  187. # An alias for the <b>HTMLTreeBuilder</b> class.
  188. TreeBuilder = HTMLTreeBuilder
  189. ##
  190. # Parse an HTML document or document fragment.
  191. #
  192. # @param source A filename or file object containing HTML data.
  193. # @param encoding Optional character encoding, if known. If omitted,
  194. # the parser looks for META tags inside the document. If no tags
  195. # are found, the parser defaults to ISO-8859-1.
  196. # @return An ElementTree instance
  197. def parse(source, encoding=None):
  198. return ElementTree.parse(source, HTMLTreeBuilder(encoding=encoding))
  199. if __name__ == "__main__":
  200. import sys
  201. ElementTree.dump(parse(open(sys.argv[1])))