tidy.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. # Copyright 2009-2015 Jason Stitt
  2. #
  3. # Permission is hereby granted, free of charge, to any person obtaining a copy
  4. # of this software and associated documentation files (the "Software"), to deal
  5. # in the Software without restriction, including without limitation the rights
  6. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  7. # copies of the Software, and to permit persons to whom the Software is
  8. # furnished to do so, subject to the following conditions:
  9. #
  10. # The above copyright notice and this permission notice shall be included in
  11. # all copies or substantial portions of the Software.
  12. #
  13. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  16. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  17. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  18. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  19. # THE SOFTWARE.
  20. import ctypes
  21. import ctypes.util
  22. import threading
  23. import platform
  24. import warnings
  25. from contextlib import contextmanager
  26. from .sink import create_sink, destroy_sink
  27. __all__ = ['Tidy', 'PersistentTidy']
  28. # Default search order for library names if nothing is passed in
  29. LIB_NAMES = ['libtidy', 'libtidy.so', 'libtidy-0.99.so.0', 'cygtidy-0-99-0',
  30. 'tidylib', 'libtidy.dylib', 'tidy']
  31. # Error code from library
  32. ENOMEM = -12
  33. # Default options; can be overriden with argument to Tidy()
  34. BASE_OPTIONS = {
  35. "indent": 1, # Pretty; not too much of a performance hit
  36. "tidy-mark": 0, # No tidy meta tag in output
  37. "wrap": 0, # No wrapping
  38. "alt-text": "", # Help ensure validation
  39. "doctype": 'strict', # Little sense in transitional for tool-generated markup...
  40. "force-output": 1, # May not get what you expect but you will get something
  41. }
  42. KEEP_DOC_WARNING = "keep_doc and release_tidy_doc are no longer used. Create a PersistentTidy object instead."
  43. # Fix for Windows b/c tidy uses stdcall on Windows
  44. if "Windows" == platform.system():
  45. load_library = ctypes.windll.LoadLibrary
  46. else:
  47. load_library = ctypes.cdll.LoadLibrary
  48. # -------------------------------------------------------------------------- #
  49. # 3.x/2.x cross-compatibility
  50. try:
  51. unicode # 2.x
  52. def is_unicode(obj):
  53. return isinstance(obj, unicode)
  54. def encode_key_value(k, v):
  55. return unicode(k).encode('utf-8'), unicode(v).encode('utf-8')
  56. except NameError:
  57. # 3.x
  58. def is_unicode(obj):
  59. return isinstance(obj, str)
  60. def encode_key_value(k, v):
  61. return str(k).encode('utf-8'), str(v).encode('utf-8')
  62. # -------------------------------------------------------------------------- #
  63. # The main python interface
  64. class Tidy(object):
  65. """ Wrapper around the HTML Tidy library for cleaning up possibly invalid
  66. HTML and XHTML. """
  67. def __init__(self, lib_names=None):
  68. self._tidy = None
  69. if lib_names is None:
  70. lib_names = ctypes.util.find_library('tidy') or LIB_NAMES
  71. if isinstance(lib_names, str):
  72. lib_names = [lib_names]
  73. for name in lib_names:
  74. try:
  75. self._tidy = load_library(name)
  76. break
  77. except OSError:
  78. continue
  79. if self._tidy is None:
  80. raise OSError(
  81. "Could not load libtidy using any of these names: "
  82. + ",".join(lib_names))
  83. self._tidy.tidyCreate.restype = ctypes.POINTER(ctypes.c_void_p) # Fix for 64-bit systems
  84. @contextmanager
  85. def _doc_and_sink(self):
  86. " Create and cleanup a Tidy document and error sink "
  87. doc = self._tidy.tidyCreate()
  88. sink = create_sink()
  89. self._tidy.tidySetErrorSink(doc, sink)
  90. yield (doc, sink)
  91. destroy_sink(sink)
  92. self._tidy.tidyRelease(doc)
  93. def tidy_document(self, text, options=None):
  94. """ Run a string with markup through HTML Tidy; return the corrected one
  95. and any error output.
  96. text: The markup, which may be anything from an empty string to a complete
  97. (X)HTML document. If you pass in a unicode type (py3 str, py2 unicode) you
  98. get one back out, and tidy will have some options set that may affect
  99. behavior (e.g. named entities converted to plain unicode characters). If
  100. you pass in a bytes type (py3 bytes, py2 str) you will get one of those
  101. back.
  102. options (dict): Options passed directly to HTML Tidy; see the HTML Tidy docs
  103. (http://tidy.sourceforge.net/docs/quickref.html) or run tidy -help-config
  104. from the command line.
  105. returns (str, str): The tidied markup and unparsed warning/error messages.
  106. Warnings and errors are returned just as tidylib returns them.
  107. """
  108. # Unicode approach is to encode as string, then decode libtidy output
  109. use_unicode = False
  110. if is_unicode(text):
  111. use_unicode = True
  112. text = text.encode('utf-8')
  113. with self._doc_and_sink() as (doc, sink):
  114. tidy_options = dict(BASE_OPTIONS)
  115. if options:
  116. tidy_options.update(options)
  117. if use_unicode:
  118. tidy_options['input-encoding'] = 'utf8'
  119. tidy_options['output-encoding'] = 'utf8'
  120. for key in tidy_options:
  121. value = tidy_options[key]
  122. key = key.replace('_', '-')
  123. if value is None:
  124. value = ''
  125. key, value = encode_key_value(key, value)
  126. self._tidy.tidyOptParseValue(doc, key, value)
  127. error = str(sink)
  128. if error:
  129. raise ValueError("(tidylib) " + error)
  130. self._tidy.tidyParseString(doc, text)
  131. self._tidy.tidyCleanAndRepair(doc)
  132. # Guess at buffer size; tidy returns ENOMEM if the buffer is too
  133. # small and puts the required size into out_length
  134. out_length = ctypes.c_int(8192)
  135. out = ctypes.c_buffer(out_length.value)
  136. while ENOMEM == self._tidy.tidySaveString(doc, out, ctypes.byref(out_length)):
  137. out = ctypes.c_buffer(out_length.value)
  138. document = out.value
  139. if use_unicode:
  140. document = document.decode('utf-8')
  141. errors = str(sink)
  142. return (document, errors)
  143. def tidy_fragment(self, text, options=None):
  144. """ Tidy a string with markup and return only the <body> contents.
  145. HTML Tidy normally returns a full (X)HTML document; this function returns only
  146. the contents of the <body> element and is meant to be used for snippets.
  147. Calling tidy_fragment on elements that don't go in the <body>, like <title>,
  148. will produce incorrect behavior.
  149. Arguments and return value are the same as tidy_document. Note that HTML
  150. Tidy will always complain about the lack of a doctype and <title> element
  151. in fragments, and these errors are not stripped out for you. """
  152. options = dict(options) if options else dict()
  153. options["show-body-only"] = 1
  154. document, errors = self.tidy_document(text, options)
  155. document = document.strip()
  156. return document, errors
  157. class PersistentTidy(Tidy):
  158. """ Functions the same as the Tidy class but keeps a persistent reference
  159. to one Tidy document object. This increases performance slightly when
  160. tidying many documents in a row. It also persists all options (not just
  161. the base options) between runs, which could lead to unexpected behavior.
  162. If you plan to use different options on each run with PersistentTidy, set
  163. all options that could change on every call. Note that passing in unicode
  164. text will result in the input-encoding and output-encoding options being
  165. automatically set. Thread-local storage is used for the document object
  166. (one document per thread). """
  167. def __init__(self, lib_names=None):
  168. Tidy.__init__(self, lib_names)
  169. self._local = threading.local()
  170. self._local.doc = self._tidy.tidyCreate()
  171. def __del__(self):
  172. self._tidy.tidyRelease(self._local.doc)
  173. @contextmanager
  174. def _doc_and_sink(self):
  175. " Create and cleanup an error sink but use the persistent doc object "
  176. sink = create_sink()
  177. self._tidy.tidySetErrorSink(self._local.doc, sink)
  178. yield (self._local.doc, sink)
  179. destroy_sink(sink)
  180. def tidy_document(text, options=None, keep_doc=False):
  181. if keep_doc:
  182. warnings.warn(KEEP_DOC_WARNING, DeprecationWarning, stacklevel=2)
  183. return get_module_tidy().tidy_document(text, options)
  184. def tidy_fragment(text, options=None, keep_doc=False):
  185. if keep_doc:
  186. warnings.warn(KEEP_DOC_WARNING, DeprecationWarning, stacklevel=2)
  187. return get_module_tidy().tidy_fragment(text, options)
  188. def get_module_tidy():
  189. global _tidy
  190. if '_tidy' not in globals():
  191. _tidy = Tidy()
  192. return _tidy
  193. def release_tidy_doc():
  194. warnings.warn(KEEP_DOC_WARNING, DeprecationWarning, stacklevel=2)