mkhtml.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252
  1. from docstructure import SITE_STRUCTURE, HREF_MAP, BASENAME_MAP
  2. from lxml.etree import (parse, fromstring, ElementTree,
  3. Element, SubElement, XPath, XML)
  4. import os
  5. import re
  6. import sys
  7. import copy
  8. import shutil
  9. import subprocess
  10. try:
  11. from io import open as open_file
  12. except ImportError:
  13. from codecs import open as open_file
  14. RST2HTML_OPTIONS = " ".join([
  15. '--no-toc-backlinks',
  16. '--strip-comments',
  17. '--language en',
  18. '--date',
  19. ])
  20. XHTML_NS = 'http://www.w3.org/1999/xhtml'
  21. htmlnsmap = {"h" : XHTML_NS}
  22. find_title = XPath("/h:html/h:head/h:title/text()", namespaces=htmlnsmap)
  23. find_title_tag = XPath("/h:html/h:head/h:title", namespaces=htmlnsmap)
  24. find_headings = XPath("//h:h1[not(@class)]//text()", namespaces=htmlnsmap)
  25. find_heading_tag = XPath("//h:h1[@class = 'title'][1]", namespaces=htmlnsmap)
  26. find_menu = XPath("//h:ul[@id=$name]", namespaces=htmlnsmap)
  27. find_page_end = XPath("/h:html/h:body/h:div[last()]", namespaces=htmlnsmap)
  28. find_words = re.compile('(\w+)').findall
  29. replace_invalid = re.compile(r'[-_/.\s\\]').sub
  30. def make_menu_section_head(section, menuroot):
  31. section_id = section + '-section'
  32. section_head = menuroot.xpath("//ul[@id=$section]/li", section=section_id)
  33. if not section_head:
  34. ul = SubElement(menuroot, "ul", id=section_id)
  35. section_head = SubElement(ul, "li")
  36. title = SubElement(section_head, "span", {"class":"section title"})
  37. title.text = section
  38. else:
  39. section_head = section_head[0]
  40. return section_head
  41. def build_menu(tree, basename, section_head):
  42. page_title = find_title(tree)
  43. if page_title:
  44. page_title = page_title[0]
  45. else:
  46. page_title = replace_invalid('', basename.capitalize())
  47. build_menu_entry(page_title, basename+".html", section_head,
  48. headings=find_headings(tree))
  49. def build_menu_entry(page_title, url, section_head, headings=None):
  50. page_id = replace_invalid(' ', os.path.splitext(url)[0]) + '-menu'
  51. ul = SubElement(section_head, "ul", {"class":"menu foreign", "id":page_id})
  52. title = SubElement(ul, "li", {"class":"menu title"})
  53. a = SubElement(title, "a", href=url)
  54. a.text = page_title
  55. if headings:
  56. subul = SubElement(title, "ul", {"class":"submenu"})
  57. for heading in headings:
  58. li = SubElement(subul, "li", {"class":"menu item"})
  59. try:
  60. ref = heading.getparent().getparent().get('id')
  61. except AttributeError:
  62. ref = None
  63. if ref is None:
  64. ref = '-'.join(find_words(replace_invalid(' ', heading.lower())))
  65. a = SubElement(li, "a", href=url+'#'+ref)
  66. a.text = heading
  67. def merge_menu(tree, menu, name):
  68. menu_root = copy.deepcopy(menu)
  69. tree.getroot()[1][0].insert(0, menu_root) # html->body->div[class=document]
  70. for el in menu_root.iter():
  71. tag = el.tag
  72. if tag[0] != '{':
  73. el.tag = "{http://www.w3.org/1999/xhtml}" + tag
  74. current_menu = find_menu(
  75. menu_root, name=replace_invalid(' ', name) + '-menu')
  76. if not current_menu:
  77. current_menu = find_menu(
  78. menu_root, name=replace_invalid('-', name) + '-menu')
  79. if current_menu:
  80. for submenu in current_menu:
  81. submenu.set("class", submenu.get("class", "").
  82. replace("foreign", "current"))
  83. return tree
  84. def inject_flatter_button(tree):
  85. head = tree.xpath('h:head[1]', namespaces=htmlnsmap)[0]
  86. script = SubElement(head, '{%s}script' % XHTML_NS, type='text/javascript')
  87. script.text = """
  88. (function() {
  89. var s = document.createElement('script');
  90. var t = document.getElementsByTagName('script')[0];
  91. s.type = 'text/javascript';
  92. s.async = true;
  93. s.src = 'http://api.flattr.com/js/0.6/load.js?mode=auto';
  94. t.parentNode.insertBefore(s, t);
  95. })();
  96. """
  97. script.tail = '\n'
  98. intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]', namespaces=htmlnsmap)[0]
  99. intro_div.insert(-1, XML(
  100. '<p style="text-align: center;">Like working with lxml? '
  101. 'Happy about the time that it just saved you? <br />'
  102. 'Show your appreciation with <a href="http://flattr.com/thing/268156/lxml-The-Python-XML-Toolkit">Flattr</a>.<br />'
  103. '<a class="FlattrButton" style="display:none;" rev="flattr;button:compact;" href="http://lxml.de/"></a>'
  104. '</p>'
  105. ))
  106. def inject_donate_buttons(lxml_path, rst2html_script, tree):
  107. command = ([sys.executable, rst2html_script]
  108. + RST2HTML_OPTIONS.split() + [os.path.join(lxml_path, 'README.rst')])
  109. rst2html = subprocess.Popen(command, stdout=subprocess.PIPE)
  110. stdout, _ = rst2html.communicate()
  111. readme = fromstring(stdout)
  112. intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]',
  113. namespaces=htmlnsmap)[0]
  114. support_div = readme.xpath('h:body//h:div[@id = "support-the-project"][1]',
  115. namespaces=htmlnsmap)[0]
  116. intro_div.append(support_div)
  117. legal = readme.xpath('h:body//h:div[@id = "legal-notice-for-donations"][1]',
  118. namespaces=htmlnsmap)[0]
  119. last_div = tree.xpath('h:body//h:div//h:div', namespaces=htmlnsmap)[-1]
  120. last_div.addnext(legal)
  121. def rest2html(script, source_path, dest_path, stylesheet_url):
  122. command = ('%s %s %s --stylesheet=%s --link-stylesheet %s > %s' %
  123. (sys.executable, script, RST2HTML_OPTIONS,
  124. stylesheet_url, source_path, dest_path))
  125. subprocess.call(command, shell=True)
  126. def convert_changelog(lxml_path, changelog_file_path, rst2html_script, stylesheet_url):
  127. f = open_file(os.path.join(lxml_path, 'CHANGES.txt'), 'r', encoding='utf-8')
  128. try:
  129. content = f.read()
  130. finally:
  131. f.close()
  132. links = dict(LP='`%s <https://bugs.launchpad.net/lxml/+bug/%s>`_',
  133. GH='`%s <https://github.com/lxml/lxml/issues/%s>`_')
  134. replace_tracker_links = re.compile('((LP|GH)#([0-9]+))').sub
  135. def insert_link(match):
  136. text, ref_type, ref_id = match.groups()
  137. return links[ref_type] % (text, ref_id)
  138. content = replace_tracker_links(insert_link, content)
  139. command = [sys.executable, rst2html_script] + RST2HTML_OPTIONS.split() + [
  140. '--link-stylesheet', '--stylesheet', stylesheet_url ]
  141. out_file = open(changelog_file_path, 'wb')
  142. try:
  143. rst2html = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=out_file)
  144. rst2html.communicate(content.encode('utf8'))
  145. finally:
  146. out_file.close()
  147. def publish(dirname, lxml_path, release):
  148. if not os.path.exists(dirname):
  149. os.mkdir(dirname)
  150. doc_dir = os.path.join(lxml_path, 'doc')
  151. script = os.path.join(doc_dir, 'rest2html.py')
  152. pubkey = os.path.join(doc_dir, 'pubkey.asc')
  153. stylesheet_url = 'style.css'
  154. shutil.copy(pubkey, dirname)
  155. href_map = HREF_MAP.copy()
  156. changelog_basename = 'changes-%s' % release
  157. href_map['Release Changelog'] = changelog_basename + '.html'
  158. trees = {}
  159. menu = Element("div", {"class":"sidemenu"})
  160. # build HTML pages and parse them back
  161. for section, text_files in SITE_STRUCTURE:
  162. section_head = make_menu_section_head(section, menu)
  163. for filename in text_files:
  164. if filename.startswith('@'):
  165. # special menu entry
  166. page_title = filename[1:]
  167. url = href_map[page_title]
  168. build_menu_entry(page_title, url, section_head)
  169. else:
  170. path = os.path.join(doc_dir, filename)
  171. basename = os.path.splitext(os.path.basename(filename))[0]
  172. basename = BASENAME_MAP.get(basename, basename)
  173. outname = basename + '.html'
  174. outpath = os.path.join(dirname, outname)
  175. rest2html(script, path, outpath, stylesheet_url)
  176. tree = parse(outpath)
  177. if filename == 'main.txt':
  178. # inject donation buttons
  179. #inject_flatter_button(tree)
  180. inject_donate_buttons(lxml_path, script, tree)
  181. trees[filename] = (tree, basename, outpath)
  182. build_menu(tree, basename, section_head)
  183. # also convert CHANGES.txt
  184. convert_changelog(lxml_path, os.path.join(dirname, 'changes-%s.html' % release),
  185. script, stylesheet_url)
  186. # generate sitemap from menu
  187. sitemap = XML('''\
  188. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  189. <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  190. <head>
  191. <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  192. <title>Sitemap of lxml.de - Processing XML and HTML with Python</title>
  193. <meta content="lxml - the most feature-rich and easy-to-use library for processing XML and HTML in the Python language"
  194. name="description" />
  195. <meta content="Python XML, XML, XML processing, HTML, lxml, simple XML, ElementTree, etree, lxml.etree, objectify, XML parsing, XML validation, XPath, XSLT"
  196. name="keywords" />
  197. </head>
  198. <body>
  199. <h1>Sitemap of lxml.de - Processing XML and HTML with Python</h1>
  200. </body>
  201. </html>
  202. '''.replace(' ', ' '))
  203. sitemap_menu = copy.deepcopy(menu)
  204. SubElement(SubElement(sitemap_menu[-1], 'li'), 'a', href='http://lxml.de/files/').text = 'Download files'
  205. sitemap[-1].append(sitemap_menu) # append to body
  206. ElementTree(sitemap).write(os.path.join(dirname, 'sitemap.html'))
  207. # integrate sitemap into the menu
  208. SubElement(SubElement(menu[-1], 'li'), 'a', href='http://lxml.de/sitemap.html').text = 'Sitemap'
  209. # integrate menu into web pages
  210. for tree, basename, outpath in trees.itervalues():
  211. new_tree = merge_menu(tree, menu, basename)
  212. title = find_title_tag(new_tree)
  213. if title and title[0].text == 'lxml':
  214. title[0].text = "lxml - Processing XML and HTML with Python"
  215. heading = find_heading_tag(new_tree)
  216. if heading:
  217. heading[0].text = "lxml - XML and HTML with Python"
  218. new_tree.write(outpath)
  219. if __name__ == '__main__':
  220. publish(sys.argv[1], sys.argv[2], sys.argv[3])