htmlTableParser.py 2.1 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061
  1. #
  2. # htmlTableParser.py
  3. #
  4. # Example of parsing a simple HTML table into a list of rows, and optionally into a little database
  5. #
  6. # Copyright 2019, Paul McGuire
  7. #
  8. import pyparsing as pp
  9. import urllib.request
  10. # define basic HTML tags, and compose into a Table
  11. table, table_end = pp.makeHTMLTags('table')
  12. thead, thead_end = pp.makeHTMLTags('thead')
  13. tbody, tbody_end = pp.makeHTMLTags('tbody')
  14. tr, tr_end = pp.makeHTMLTags('tr')
  15. th, th_end = pp.makeHTMLTags('th')
  16. td, td_end = pp.makeHTMLTags('td')
  17. a, a_end = pp.makeHTMLTags('a')
  18. # method to strip HTML tags from a string - will be used to clean up content of table cells
  19. strip_html = (pp.anyOpenTag | pp.anyCloseTag).suppress().transformString
  20. # expression for parsing <a href="url">text</a> links, returning a (text, url) tuple
  21. link = pp.Group(a + a.tag_body('text') + a_end.suppress())
  22. link.addParseAction(lambda t: (t[0].text, t[0].href))
  23. # method to create table rows of header and data tags
  24. def table_row(start_tag, end_tag):
  25. body = start_tag.tag_body
  26. body.addParseAction(pp.tokenMap(str.strip),
  27. pp.tokenMap(strip_html))
  28. row = pp.Group(tr.suppress()
  29. + pp.ZeroOrMore(start_tag.suppress()
  30. + body
  31. + end_tag.suppress())
  32. + tr_end.suppress())
  33. return row
  34. th_row = table_row(th, th_end)
  35. td_row = table_row(td, td_end)
  36. # define expression for overall table - may vary slightly for different pages
  37. html_table = table + tbody + pp.Optional(th_row('headers')) + pp.ZeroOrMore(td_row)('rows') + tbody_end + table_end
  38. # read in a web page containing an interesting HTML table
  39. with urllib.request.urlopen("https://en.wikipedia.org/wiki/List_of_tz_database_time_zones") as page:
  40. page_html = page.read().decode()
  41. tz_table = html_table.searchString(page_html)[0]
  42. # convert rows to dicts
  43. rows = [dict(zip(tz_table.headers, row)) for row in tz_table.rows]
  44. # make a dict keyed by TZ database name
  45. tz_db = {row['TZ database name']: row for row in rows}
  46. from pprint import pprint
  47. pprint(tz_db['America/Chicago'])