generate.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. """
  4. Generate Pygments Documentation
  5. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  6. Generates a bunch of html files containing the documentation.
  7. :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
  8. :license: BSD, see LICENSE for details.
  9. """
  10. import os
  11. import sys
  12. from datetime import datetime
  13. from cgi import escape
  14. from docutils import nodes
  15. from docutils.parsers.rst import directives
  16. from docutils.core import publish_parts
  17. from docutils.writers import html4css1
  18. from jinja2 import Template
  19. # try to use the right Pygments to build the docs
  20. sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
  21. from pygments import highlight, __version__
  22. from pygments.lexers import get_lexer_by_name
  23. from pygments.formatters import HtmlFormatter
  24. LEXERDOC = '''
  25. `%s`
  26. %s
  27. :Short names: %s
  28. :Filename patterns: %s
  29. :Mimetypes: %s
  30. '''
  31. def generate_lexer_docs():
  32. from pygments.lexers import LEXERS
  33. out = []
  34. modules = {}
  35. moduledocstrings = {}
  36. for classname, data in sorted(LEXERS.iteritems(), key=lambda x: x[0]):
  37. module = data[0]
  38. mod = __import__(module, None, None, [classname])
  39. cls = getattr(mod, classname)
  40. if not cls.__doc__:
  41. print "Warning: %s does not have a docstring." % classname
  42. modules.setdefault(module, []).append((
  43. classname,
  44. cls.__doc__,
  45. ', '.join(data[2]) or 'None',
  46. ', '.join(data[3]).replace('*', '\\*') or 'None',
  47. ', '.join(data[4]) or 'None'))
  48. if module not in moduledocstrings:
  49. moduledocstrings[module] = mod.__doc__
  50. for module, lexers in sorted(modules.iteritems(), key=lambda x: x[0]):
  51. heading = moduledocstrings[module].splitlines()[4].strip().rstrip('.')
  52. out.append('\n' + heading + '\n' + '-'*len(heading) + '\n')
  53. for data in lexers:
  54. out.append(LEXERDOC % data)
  55. return ''.join(out).decode('utf-8')
  56. def generate_formatter_docs():
  57. from pygments.formatters import FORMATTERS
  58. out = []
  59. for cls, data in sorted(FORMATTERS.iteritems(),
  60. key=lambda x: x[0].__name__):
  61. heading = cls.__name__
  62. out.append('`' + heading + '`\n' + '-'*(2+len(heading)) + '\n')
  63. out.append(cls.__doc__)
  64. out.append('''
  65. :Short names: %s
  66. :Filename patterns: %s
  67. ''' % (', '.join(data[1]) or 'None', ', '.join(data[2]).replace('*', '\\*') or 'None'))
  68. return ''.join(out).decode('utf-8')
  69. def generate_filter_docs():
  70. from pygments.filters import FILTERS
  71. out = []
  72. for name, cls in FILTERS.iteritems():
  73. out.append('''
  74. `%s`
  75. %s
  76. :Name: %s
  77. ''' % (cls.__name__, cls.__doc__, name))
  78. return ''.join(out).decode('utf-8')
  79. def generate_changelog():
  80. fn = os.path.abspath(os.path.join(os.path.dirname(__file__), '..',
  81. 'CHANGES'))
  82. f = file(fn)
  83. result = []
  84. in_header = False
  85. header = True
  86. for line in f:
  87. if header:
  88. if not in_header and line.strip():
  89. in_header = True
  90. elif in_header and not line.strip():
  91. header = False
  92. else:
  93. result.append(line.rstrip())
  94. f.close()
  95. return '\n'.join(result).decode('utf-8')
  96. def generate_authors():
  97. fn = os.path.abspath(os.path.join(os.path.dirname(__file__), '..',
  98. 'AUTHORS'))
  99. f = file(fn)
  100. r = f.read().rstrip().decode('utf-8')
  101. f.close()
  102. return r
  103. LEXERDOCS = generate_lexer_docs()
  104. FORMATTERDOCS = generate_formatter_docs()
  105. FILTERDOCS = generate_filter_docs()
  106. CHANGELOG = generate_changelog()
  107. AUTHORS = generate_authors()
  108. PYGMENTS_FORMATTER = HtmlFormatter(style='pastie', cssclass='syntax')
  109. USAGE = '''\
  110. Usage: %s <mode> <destination> [<source.txt> ...]
  111. Generate either python or html files out of the documentation.
  112. Mode can either be python or html.\
  113. ''' % sys.argv[0]
  114. TEMPLATE = '''\
  115. <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
  116. "http://www.w3.org/TR/html4/strict.dtd">
  117. <html>
  118. <head>
  119. <title>{{ title }} &mdash; Pygments</title>
  120. <meta http-equiv="content-type" content="text/html; charset=utf-8">
  121. <style type="text/css">
  122. {{ style }}
  123. </style>
  124. </head>
  125. <body>
  126. <div id="content">
  127. <h1 class="heading">Pygments</h1>
  128. <h2 class="subheading">{{ title }}</h2>
  129. {% if file_id != "index" %}
  130. <a id="backlink" href="index.html">&laquo; Back To Index</a>
  131. {% endif %}
  132. {% if toc %}
  133. <div class="toc">
  134. <h2>Contents</h2>
  135. <ul class="contents">
  136. {% for key, value in toc %}
  137. <li><a href="{{ key }}">{{ value }}</a></li>
  138. {% endfor %}
  139. </ul>
  140. </div>
  141. {% endif %}
  142. {{ body }}
  143. </div>
  144. </body>
  145. <!-- generated on: {{ generation_date }}
  146. file id: {{ file_id }} -->
  147. </html>\
  148. '''
  149. STYLESHEET = '''\
  150. body {
  151. background-color: #f2f2f2;
  152. margin: 0;
  153. padding: 0;
  154. font-family: 'Georgia', serif;
  155. color: #111;
  156. }
  157. #content {
  158. background-color: white;
  159. padding: 20px;
  160. margin: 20px auto 20px auto;
  161. max-width: 800px;
  162. border: 4px solid #ddd;
  163. }
  164. h1 {
  165. font-weight: normal;
  166. font-size: 40px;
  167. color: #09839A;
  168. }
  169. h2 {
  170. font-weight: normal;
  171. font-size: 30px;
  172. color: #C73F00;
  173. }
  174. h1.heading {
  175. margin: 0 0 30px 0;
  176. }
  177. h2.subheading {
  178. margin: -30px 0 0 45px;
  179. }
  180. h3 {
  181. margin-top: 30px;
  182. }
  183. table.docutils {
  184. border-collapse: collapse;
  185. border: 2px solid #aaa;
  186. margin: 0.5em 1.5em 0.5em 1.5em;
  187. }
  188. table.docutils td {
  189. padding: 2px;
  190. border: 1px solid #ddd;
  191. }
  192. p, li, dd, dt, blockquote {
  193. font-size: 15px;
  194. color: #333;
  195. }
  196. p {
  197. line-height: 150%;
  198. margin-bottom: 0;
  199. margin-top: 10px;
  200. }
  201. hr {
  202. border-top: 1px solid #ccc;
  203. border-bottom: 0;
  204. border-right: 0;
  205. border-left: 0;
  206. margin-bottom: 10px;
  207. margin-top: 20px;
  208. }
  209. dl {
  210. margin-left: 10px;
  211. }
  212. li, dt {
  213. margin-top: 5px;
  214. }
  215. dt {
  216. font-weight: bold;
  217. }
  218. th {
  219. text-align: left;
  220. }
  221. a {
  222. color: #990000;
  223. }
  224. a:hover {
  225. color: #c73f00;
  226. }
  227. pre {
  228. background-color: #f9f9f9;
  229. border-top: 1px solid #ccc;
  230. border-bottom: 1px solid #ccc;
  231. padding: 5px;
  232. font-size: 13px;
  233. font-family: Bitstream Vera Sans Mono,monospace;
  234. }
  235. tt {
  236. font-size: 13px;
  237. font-family: Bitstream Vera Sans Mono,monospace;
  238. color: black;
  239. padding: 1px 2px 1px 2px;
  240. background-color: #f0f0f0;
  241. }
  242. cite {
  243. /* abusing <cite>, it's generated by ReST for `x` */
  244. font-size: 13px;
  245. font-family: Bitstream Vera Sans Mono,monospace;
  246. font-weight: bold;
  247. font-style: normal;
  248. }
  249. #backlink {
  250. float: right;
  251. font-size: 11px;
  252. color: #888;
  253. }
  254. div.toc {
  255. margin: 0 0 10px 0;
  256. }
  257. div.toc h2 {
  258. font-size: 20px;
  259. }
  260. ''' #'
  261. def pygments_directive(name, arguments, options, content, lineno,
  262. content_offset, block_text, state, state_machine):
  263. try:
  264. lexer = get_lexer_by_name(arguments[0])
  265. except ValueError:
  266. # no lexer found
  267. lexer = get_lexer_by_name('text')
  268. parsed = highlight(u'\n'.join(content), lexer, PYGMENTS_FORMATTER)
  269. return [nodes.raw('', parsed, format="html")]
  270. pygments_directive.arguments = (1, 0, 1)
  271. pygments_directive.content = 1
  272. directives.register_directive('sourcecode', pygments_directive)
  273. def create_translator(link_style):
  274. class Translator(html4css1.HTMLTranslator):
  275. def visit_reference(self, node):
  276. refuri = node.get('refuri')
  277. if refuri is not None and '/' not in refuri and refuri.endswith('.txt'):
  278. node['refuri'] = link_style(refuri[:-4])
  279. html4css1.HTMLTranslator.visit_reference(self, node)
  280. return Translator
  281. class DocumentationWriter(html4css1.Writer):
  282. def __init__(self, link_style):
  283. html4css1.Writer.__init__(self)
  284. self.translator_class = create_translator(link_style)
  285. def translate(self):
  286. html4css1.Writer.translate(self)
  287. # generate table of contents
  288. contents = self.build_contents(self.document)
  289. contents_doc = self.document.copy()
  290. contents_doc.children = contents
  291. contents_visitor = self.translator_class(contents_doc)
  292. contents_doc.walkabout(contents_visitor)
  293. self.parts['toc'] = self._generated_toc
  294. def build_contents(self, node, level=0):
  295. sections = []
  296. i = len(node) - 1
  297. while i >= 0 and isinstance(node[i], nodes.section):
  298. sections.append(node[i])
  299. i -= 1
  300. sections.reverse()
  301. toc = []
  302. for section in sections:
  303. try:
  304. reference = nodes.reference('', '', refid=section['ids'][0], *section[0])
  305. except IndexError:
  306. continue
  307. ref_id = reference['refid']
  308. text = escape(reference.astext())
  309. toc.append((ref_id, text))
  310. self._generated_toc = [('#%s' % href, caption) for href, caption in toc]
  311. # no further processing
  312. return []
  313. def generate_documentation(data, link_style):
  314. writer = DocumentationWriter(link_style)
  315. data = data.replace('[builtin_lexer_docs]', LEXERDOCS).\
  316. replace('[builtin_formatter_docs]', FORMATTERDOCS).\
  317. replace('[builtin_filter_docs]', FILTERDOCS).\
  318. replace('[changelog]', CHANGELOG).\
  319. replace('[authors]', AUTHORS)
  320. parts = publish_parts(
  321. data,
  322. writer=writer,
  323. settings_overrides={
  324. 'initial_header_level': 3,
  325. 'field_name_limit': 50,
  326. }
  327. )
  328. return {
  329. 'title': parts['title'],
  330. 'body': parts['body'],
  331. 'toc': parts['toc']
  332. }
  333. def handle_python(filename, fp, dst):
  334. now = datetime.now()
  335. title = os.path.basename(filename)[:-4]
  336. content = fp.read()
  337. def urlize(href):
  338. # create links for the pygments webpage
  339. if href == 'index.txt':
  340. return '/docs/'
  341. else:
  342. return '/docs/%s/' % href
  343. parts = generate_documentation(content, urlize)
  344. result = file(os.path.join(dst, title + '.py'), 'w')
  345. result.write('# -*- coding: utf-8 -*-\n')
  346. result.write('"""\n Pygments Documentation - %s\n' % title)
  347. result.write(' %s\n\n' % ('~' * (24 + len(title))))
  348. result.write(' Generated on: %s\n"""\n\n' % now)
  349. result.write('import datetime\n')
  350. result.write('DATE = %r\n' % now)
  351. result.write('TITLE = %r\n' % parts['title'])
  352. result.write('TOC = %r\n' % parts['toc'])
  353. result.write('BODY = %r\n' % parts['body'])
  354. result.close()
  355. def handle_html(filename, fp, dst):
  356. now = datetime.now()
  357. title = os.path.basename(filename)[:-4]
  358. content = fp.read().decode('utf-8')
  359. c = generate_documentation(content, (lambda x: './%s.html' % x))
  360. result = file(os.path.join(dst, title + '.html'), 'w')
  361. c['style'] = STYLESHEET + PYGMENTS_FORMATTER.get_style_defs('.syntax')
  362. c['generation_date'] = now
  363. c['file_id'] = title
  364. t = Template(TEMPLATE)
  365. result.write(t.render(c).encode('utf-8'))
  366. result.close()
  367. def run(handle_file, dst, sources=()):
  368. path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'src'))
  369. if not sources:
  370. sources = [os.path.join(path, fn) for fn in os.listdir(path)]
  371. if not os.path.isdir(dst):
  372. os.makedirs(dst)
  373. print 'Making docs for Pygments %s in %s' % (__version__, dst)
  374. for fn in sources:
  375. if not os.path.isfile(fn):
  376. continue
  377. print 'Processing %s' % fn
  378. f = open(fn)
  379. try:
  380. handle_file(fn, f, dst)
  381. finally:
  382. f.close()
  383. def main(mode, dst='build/', *sources):
  384. try:
  385. handler = {
  386. 'html': handle_html,
  387. 'python': handle_python
  388. }[mode]
  389. except KeyError:
  390. print 'Error: unknown mode "%s"' % mode
  391. sys.exit(1)
  392. run(handler, os.path.realpath(dst), sources)
  393. if __name__ == '__main__':
  394. if len(sys.argv) == 1:
  395. print USAGE
  396. else:
  397. main(*sys.argv[1:])