mkhtml.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332
  1. from __future__ import absolute_import
  2. from docstructure import SITE_STRUCTURE, HREF_MAP, BASENAME_MAP
  3. from lxml.etree import (parse, fromstring, ElementTree,
  4. Element, SubElement, XPath, XML)
  5. import glob
  6. import hashlib
  7. import os
  8. import re
  9. import sys
  10. import copy
  11. import shutil
  12. import textwrap
  13. import subprocess
  14. from io import open as open_file
  15. RST2HTML_OPTIONS = " ".join([
  16. '--no-toc-backlinks',
  17. '--strip-comments',
  18. '--language en',
  19. '--date',
  20. ])
  21. XHTML_NS = 'http://www.w3.org/1999/xhtml'
  22. htmlnsmap = {"h": XHTML_NS}
  23. find_head = XPath("/h:html/h:head[1]", namespaces=htmlnsmap)
  24. find_body = XPath("/h:html/h:body[1]", namespaces=htmlnsmap)
  25. find_title = XPath("/h:html/h:head/h:title/text()", namespaces=htmlnsmap)
  26. find_title_tag = XPath("/h:html/h:head/h:title", namespaces=htmlnsmap)
  27. find_headings = XPath("//h:h1[not(@class)]//text()", namespaces=htmlnsmap)
  28. find_heading_tag = XPath("//h:h1[@class = 'title'][1]", namespaces=htmlnsmap)
  29. find_menu = XPath("//h:ul[@id=$name]", namespaces=htmlnsmap)
  30. find_page_end = XPath("/h:html/h:body/h:div[last()]", namespaces=htmlnsmap)
  31. find_words = re.compile(r'(\w+)').findall
  32. replace_invalid = re.compile(r'[-_/.\s\\]').sub
  33. def make_menu_section_head(section, menuroot):
  34. section_id = section + '-section'
  35. section_head = menuroot.xpath("//ul[@id=$section]/li", section=section_id)
  36. if not section_head:
  37. ul = SubElement(menuroot, "ul", id=section_id)
  38. section_head = SubElement(ul, "li")
  39. title = SubElement(section_head, "span", {"class":"section title"})
  40. title.text = section
  41. else:
  42. section_head = section_head[0]
  43. return section_head
  44. def build_menu(tree, basename, section_head):
  45. page_title = find_title(tree)
  46. if page_title:
  47. page_title = page_title[0]
  48. else:
  49. page_title = replace_invalid('', basename.capitalize())
  50. build_menu_entry(page_title, basename+".html", section_head,
  51. headings=find_headings(tree))
  52. def build_menu_entry(page_title, url, section_head, headings=None):
  53. page_id = replace_invalid(' ', os.path.splitext(url)[0]) + '-menu'
  54. ul = SubElement(section_head, "ul", {"class":"menu foreign", "id":page_id})
  55. title = SubElement(ul, "li", {"class":"menu title"})
  56. a = SubElement(title, "a", href=url)
  57. a.text = page_title
  58. if headings:
  59. subul = SubElement(title, "ul", {"class":"submenu"})
  60. for heading in headings:
  61. li = SubElement(subul, "li", {"class":"menu item"})
  62. try:
  63. ref = heading.getparent().getparent().get('id')
  64. except AttributeError:
  65. ref = None
  66. if ref is None:
  67. ref = '-'.join(find_words(replace_invalid(' ', heading.lower())))
  68. a = SubElement(li, "a", href=url+'#'+ref)
  69. a.text = heading
  70. def merge_menu(tree, menu, name):
  71. menu_root = copy.deepcopy(menu)
  72. tree.getroot()[1][0].insert(0, menu_root) # html->body->div[class=document]
  73. for el in menu_root.iter():
  74. tag = el.tag
  75. if tag[0] != '{':
  76. el.tag = "{http://www.w3.org/1999/xhtml}" + tag
  77. current_menu = find_menu(
  78. menu_root, name=replace_invalid(' ', name) + '-menu')
  79. if not current_menu:
  80. current_menu = find_menu(
  81. menu_root, name=replace_invalid('-', name) + '-menu')
  82. if current_menu:
  83. for submenu in current_menu:
  84. submenu.set("class", submenu.get("class", "").
  85. replace("foreign", "current"))
  86. return tree
  87. def inject_flatter_button(tree):
  88. head = tree.xpath('h:head[1]', namespaces=htmlnsmap)[0]
  89. script = SubElement(head, '{%s}script' % XHTML_NS, type='text/javascript')
  90. script.text = """
  91. (function() {
  92. var s = document.createElement('script');
  93. var t = document.getElementsByTagName('script')[0];
  94. s.type = 'text/javascript';
  95. s.async = true;
  96. s.src = 'http://api.flattr.com/js/0.6/load.js?mode=auto';
  97. t.parentNode.insertBefore(s, t);
  98. })();
  99. """
  100. script.tail = '\n'
  101. intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]', namespaces=htmlnsmap)[0]
  102. intro_div.insert(-1, XML(
  103. '<p style="text-align: center;">Like working with lxml? '
  104. 'Happy about the time that it just saved you? <br />'
  105. 'Show your appreciation with <a href="http://flattr.com/thing/268156/lxml-The-Python-XML-Toolkit">Flattr</a>.<br />'
  106. '<a class="FlattrButton" style="display:none;" rev="flattr;button:compact;" href="https://lxml.de/"></a>'
  107. '</p>'
  108. ))
  109. def inject_donate_buttons(lxml_path, rst2html_script, tree):
  110. command = ([sys.executable, rst2html_script]
  111. + RST2HTML_OPTIONS.split() + [os.path.join(lxml_path, 'README.rst')])
  112. rst2html = subprocess.Popen(command, stdout=subprocess.PIPE)
  113. stdout, _ = rst2html.communicate()
  114. readme = fromstring(stdout)
  115. intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]',
  116. namespaces=htmlnsmap)[0]
  117. support_div = readme.xpath('h:body//h:div[@id = "support-the-project"][1]',
  118. namespaces=htmlnsmap)[0]
  119. intro_div.append(support_div)
  120. finance_div = readme.xpath('h:body//h:div[@id = "project-income-report"][1]',
  121. namespaces=htmlnsmap)[0]
  122. legal = readme.xpath('h:body//h:div[@id = "legal-notice-for-donations"][1]',
  123. namespaces=htmlnsmap)[0]
  124. last_div = tree.xpath('h:body//h:div//h:div', namespaces=htmlnsmap)[-1]
  125. last_div.addnext(finance_div)
  126. finance_div.addnext(legal)
  127. def inject_banner(parent):
  128. banner = parent.makeelement('div', {'class': 'banner'})
  129. parent.insert(0, banner)
  130. banner_image = SubElement(banner, 'div', {'class': "banner_image"})
  131. SubElement(banner_image, 'img', src="python-xml-title.png")
  132. banner_text = SubElement(banner, 'div', {'class': "banner_link"})
  133. banner_link = SubElement(banner_text, 'a', href="index.html#support-the-project")
  134. banner_link.text = "Like the tool? "
  135. SubElement(banner_link, 'br', {'class': "first"}).tail = "Help making it better! "
  136. SubElement(banner_link, 'br', {'class': "second"}).tail = "Your donation helps!"
  137. def rest2html(script, source_path, dest_path, stylesheet_url):
  138. command = ('%s %s %s --stylesheet=%s --link-stylesheet %s > %s' %
  139. (sys.executable, script, RST2HTML_OPTIONS,
  140. stylesheet_url, source_path, dest_path))
  141. subprocess.call(command, shell=True)
  142. def convert_changelog(lxml_path, changelog_file_path, rst2html_script, stylesheet_url):
  143. f = open_file(os.path.join(lxml_path, 'CHANGES.txt'), 'r', encoding='utf-8')
  144. try:
  145. content = f.read()
  146. finally:
  147. f.close()
  148. links = dict(LP='`%s <https://bugs.launchpad.net/lxml/+bug/%s>`_',
  149. GH='`%s <https://github.com/lxml/lxml/issues/%s>`_')
  150. replace_tracker_links = re.compile('((LP|GH)#([0-9]+))').sub
  151. def insert_link(match):
  152. text, ref_type, ref_id = match.groups()
  153. return links[ref_type] % (text, ref_id)
  154. content = replace_tracker_links(insert_link, content)
  155. command = [sys.executable, rst2html_script] + RST2HTML_OPTIONS.split() + [
  156. '--link-stylesheet', '--stylesheet', stylesheet_url ]
  157. out_file = open(changelog_file_path, 'wb')
  158. try:
  159. rst2html = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=out_file)
  160. rst2html.communicate(content.encode('utf8'))
  161. finally:
  162. out_file.close()
  163. def publish(dirname, lxml_path, release, with_donations=True):
  164. if not os.path.exists(dirname):
  165. os.mkdir(dirname)
  166. doc_dir = os.path.join(lxml_path, 'doc')
  167. script = os.path.join(doc_dir, 'rest2html.py')
  168. pubkey = os.path.join(doc_dir, 'pubkey.asc')
  169. stylesheet_file = 'style.css'
  170. shutil.copy(pubkey, dirname)
  171. # FIXME: find a way to make hashed filenames work both locally and in the versioned directories.
  172. stylesheet_url = stylesheet_file
  173. """
  174. style_file_pattern = "style_%s.css"
  175. for old_stylesheet in glob.iglob(os.path.join(dirname, style_file_pattern % "*")):
  176. os.unlink(old_stylesheet)
  177. with open(os.path.join(dirname, stylesheet_file), 'rb') as f:
  178. css = f.read()
  179. checksum = hashlib.sha256(css).hexdigest()[:32]
  180. stylesheet_url = style_file_pattern % checksum
  181. with open(os.path.join(dirname, stylesheet_url), 'wb') as out:
  182. out.write(css)
  183. """
  184. href_map = HREF_MAP.copy()
  185. changelog_basename = 'changes-%s' % release
  186. href_map['Release Changelog'] = changelog_basename + '.html'
  187. menu_js = textwrap.dedent('''
  188. function trigger_menu(event) {
  189. var sidemenu = document.getElementById("sidemenu");
  190. var classes = sidemenu.getAttribute("class");
  191. classes = (classes.indexOf(" visible") === -1) ? classes + " visible" : classes.replace(" visible", "");
  192. sidemenu.setAttribute("class", classes);
  193. event.preventDefault();
  194. event.stopPropagation();
  195. }
  196. function hide_menu() {
  197. var sidemenu = document.getElementById("sidemenu");
  198. var classes = sidemenu.getAttribute("class");
  199. if (classes.indexOf(" visible") !== -1) {
  200. sidemenu.setAttribute("class", classes.replace(" visible", ""));
  201. }
  202. }
  203. ''')
  204. trees = {}
  205. menu = Element("div", {'class': 'sidemenu', 'id': 'sidemenu'})
  206. SubElement(menu, 'div', {'class': 'menutrigger', 'onclick': 'trigger_menu(event)'}).text = "Menu"
  207. menu_div = SubElement(menu, 'div', {'class': 'menu'})
  208. if with_donations:
  209. inject_banner(menu_div)
  210. # build HTML pages and parse them back
  211. for section, text_files in SITE_STRUCTURE:
  212. section_head = make_menu_section_head(section, menu_div)
  213. for filename in text_files:
  214. if filename.startswith('@'):
  215. # special menu entry
  216. page_title = filename[1:]
  217. url = href_map[page_title]
  218. build_menu_entry(page_title, url, section_head)
  219. else:
  220. path = os.path.join(doc_dir, filename)
  221. basename = os.path.splitext(os.path.basename(filename))[0]
  222. basename = BASENAME_MAP.get(basename, basename)
  223. outname = basename + '.html'
  224. outpath = os.path.join(dirname, outname)
  225. rest2html(script, path, outpath, stylesheet_url)
  226. tree = parse(outpath)
  227. if with_donations:
  228. page_div = tree.getroot()[1][0] # html->body->div[class=document]
  229. inject_banner(page_div)
  230. if filename == 'main.txt':
  231. # inject donation buttons
  232. #inject_flatter_button(tree)
  233. inject_donate_buttons(lxml_path, script, tree)
  234. trees[filename] = (tree, basename, outpath)
  235. build_menu(tree, basename, section_head)
  236. # also convert CHANGES.txt
  237. convert_changelog(lxml_path, os.path.join(dirname, 'changes-%s.html' % release),
  238. script, stylesheet_url)
  239. # generate sitemap from menu
  240. sitemap = XML(textwrap.dedent('''\
  241. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  242. <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  243. <head>
  244. <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  245. <title>Sitemap of lxml.de - Processing XML and HTML with Python</title>
  246. <meta content="lxml - the most feature-rich and easy-to-use library for processing XML and HTML in the Python language"
  247. name="description" />
  248. <meta content="Python XML, XML, XML processing, HTML, lxml, simple XML, ElementTree, etree, lxml.etree, objectify, XML parsing, XML validation, XPath, XSLT"
  249. name="keywords" />
  250. </head>
  251. <body>
  252. <h1>Sitemap of lxml.de - Processing XML and HTML with Python</h1>
  253. </body>
  254. </html>
  255. '''))
  256. sitemap_menu = copy.deepcopy(menu)
  257. SubElement(SubElement(sitemap_menu[-1], 'li'), 'a', href='https://lxml.de/files/').text = 'Download files'
  258. sitemap[-1].append(sitemap_menu) # append to body
  259. ElementTree(sitemap).write(os.path.join(dirname, 'sitemap.html'))
  260. # integrate sitemap into the menu
  261. SubElement(SubElement(menu_div[-1], 'li'), 'a', href='/sitemap.html').text = 'Sitemap'
  262. # integrate menu into web pages
  263. for tree, basename, outpath in trees.values():
  264. head = find_head(tree)[0]
  265. SubElement(head, 'script', type='text/javascript').text = menu_js
  266. SubElement(head, 'meta', name='viewport', content="width=device-width, initial-scale=1")
  267. find_body(tree)[0].set('onclick', 'hide_menu()')
  268. new_tree = merge_menu(tree, menu, basename)
  269. title = find_title_tag(new_tree)
  270. if title and title[0].text == 'lxml':
  271. title[0].text = "lxml - Processing XML and HTML with Python"
  272. heading = find_heading_tag(new_tree)
  273. if heading:
  274. heading[0].text = "lxml - XML and HTML with Python"
  275. new_tree.write(outpath)
  276. if __name__ == '__main__':
  277. no_donations = '--no-donations' in sys.argv[1:]
  278. if no_donations:
  279. sys.argv.remove('--no-donations')
  280. publish(sys.argv[1], sys.argv[2], sys.argv[3], with_donations=not no_donations)