htmlStripper.py 1.1 KB

1234567891011121314151617181920212223242526272829303132
  1. #
  2. # htmlStripper.py
  3. #
  4. # Sample code for stripping HTML markup tags and scripts from
  5. # HTML source files.
  6. #
  7. # Copyright (c) 2006, 2016, Paul McGuire
  8. #
  9. from contextlib import closing
  10. import urllib.request, urllib.parse, urllib.error
  11. from pyparsing import (makeHTMLTags, commonHTMLEntity, replaceHTMLEntity,
  12. htmlComment, anyOpenTag, anyCloseTag, LineEnd, OneOrMore, replaceWith)
  13. scriptOpen, scriptClose = makeHTMLTags("script")
  14. scriptBody = scriptOpen + scriptOpen.tag_body + scriptClose
  15. commonHTMLEntity.setParseAction(replaceHTMLEntity)
  16. # get some HTML
  17. targetURL = "https://wiki.python.org/moin/PythonDecoratorLibrary"
  18. with closing(urllib.request.urlopen( targetURL )) as targetPage:
  19. targetHTML = targetPage.read().decode("UTF-8")
  20. # first pass, strip out tags and translate entities
  21. firstPass = (htmlComment | scriptBody | commonHTMLEntity |
  22. anyOpenTag | anyCloseTag ).suppress().transformString(targetHTML)
  23. # first pass leaves many blank lines, collapse these down
  24. repeatedNewlines = LineEnd()*(2,)
  25. repeatedNewlines.setParseAction(replaceWith("\n\n"))
  26. secondPass = repeatedNewlines.transformString(firstPass)
  27. print(secondPass)