test-markdown.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347
  1. #!/usr/bin/env python
  2. import os, difflib, time, gc, codecs, platform, sys
  3. from pprint import pprint
  4. import textwrap
  5. # Setup a logger manually for compatibility with Python 2.3
  6. import logging
  7. logging.getLogger('MARKDOWN').addHandler(logging.StreamHandler())
  8. import markdown
  9. TEST_DIR = "tests"
  10. TMP_DIR = "./tmp/"
  11. WRITE_BENCHMARK = True
  12. WRITE_BENCHMARK = False
  13. ACTUALLY_MEASURE_MEMORY = True
  14. ######################################################################
  15. if platform.system().lower() == "darwin": # Darwin
  16. _proc_status = '/proc/%d/stat' % os.getpid()
  17. else: # Linux
  18. _proc_status = '/proc/%d/status' % os.getpid()
  19. _scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
  20. 'KB': 1024.0, 'MB': 1024.0*1024.0}
  21. def _VmB(VmKey):
  22. '''Private.
  23. '''
  24. global _proc_status, _scale
  25. # get pseudo file /proc/<pid>/status
  26. try:
  27. t = open(_proc_status)
  28. v = t.read()
  29. t.close()
  30. except:
  31. return 0.0 # non-Linux?
  32. # get VmKey line e.g. 'VmRSS: 9999 kB\n ...'
  33. i = v.index(VmKey)
  34. v = v[i:].split(None, 3) # whitespace
  35. if len(v) < 3:
  36. return 0.0 # invalid format?
  37. # convert Vm value to bytes
  38. return float(v[1]) * _scale[v[2]]
  39. def memory(since=0.0):
  40. '''Return memory usage in bytes.
  41. '''
  42. if ACTUALLY_MEASURE_MEMORY :
  43. return _VmB('VmSize:') - since
  44. def resident(since=0.0):
  45. '''Return resident memory usage in bytes.
  46. '''
  47. return _VmB('VmRSS:') - since
  48. def stacksize(since=0.0):
  49. '''Return stack size in bytes.
  50. '''
  51. return _VmB('VmStk:') - since
  52. ############################################################
  53. DIFF_FILE_TEMPLATE = """
  54. <html>
  55. <head>
  56. <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  57. <style>
  58. td {
  59. padding-left: 10px;
  60. padding-right: 10px;
  61. }
  62. colgroup {
  63. margin: 10px;
  64. }
  65. .diff_header {
  66. color: gray;
  67. }
  68. .ok {
  69. color: green;
  70. }
  71. .gray {
  72. color: gray;
  73. }
  74. .failed a {
  75. color: red;
  76. }
  77. .failed {
  78. color: red;
  79. }
  80. </style>
  81. </head>
  82. <body>
  83. <h1>Results Summary</h1>
  84. <table rules="groups" >
  85. <colgroup></colgroup>
  86. <colgroup></colgroup>
  87. <colgroup></colgroup>
  88. <colgroup></colgroup>
  89. <colgroup></colgroup>
  90. <th>
  91. <td></td>
  92. <td>Seconds</td>
  93. <td></td>
  94. <td>Memory</td>
  95. </th>
  96. <tbody>
  97. """
  98. FOOTER = """
  99. </body>
  100. </html>
  101. """
  102. DIFF_TABLE_TEMPLATE = """
  103. <table class="diff" rules="groups" >
  104. <colgroup></colgroup>
  105. <colgroup></colgroup>
  106. <colgroup></colgroup>
  107. <colgroup></colgroup>
  108. <colgroup></colgroup>
  109. <colgroup></colgroup>
  110. <th>
  111. <td></td>
  112. <td>Expected</td>
  113. <td></td>
  114. <td></td>
  115. <td>Actual</td>
  116. </th>
  117. <tbody>
  118. %s
  119. </tbody>
  120. </table>
  121. """
  122. def smart_split(text) :
  123. result = []
  124. for x in text.splitlines() :
  125. for y in textwrap.wrap(textwrap.dedent(x), 40):
  126. result.append(y)
  127. return result
  128. differ = difflib.Differ()
  129. try :
  130. htmldiff = difflib.HtmlDiff()
  131. except:
  132. htmldiff = None
  133. class TestRunner :
  134. def __init__ (self) :
  135. self.failedTests = []
  136. if not os.path.exists(TMP_DIR):
  137. os.mkdir(TMP_DIR)
  138. def test_directory(self, dir, measure_time=False, safe_mode=False, encoding="utf8", output_format='xhtml1') :
  139. self.encoding = encoding
  140. benchmark_file_name = os.path.join(dir, "benchmark.dat")
  141. self.saved_benchmarks = {}
  142. if measure_time :
  143. if os.path.exists(benchmark_file_name) :
  144. file = open(benchmark_file_name)
  145. for line in file.readlines() :
  146. test, str_time, str_mem = line.strip().split(":")
  147. self.saved_benchmarks[test] = (float(str_time), float(str_mem))
  148. repeat = range(10)
  149. else :
  150. repeat = (0,)
  151. # First, determine from the name of the directory if any extensions
  152. # need to be loaded.
  153. parts = os.path.split(dir)[-1].split("-x-")
  154. if len(parts) > 1 :
  155. extensions = parts[1].split("-")
  156. print extensions
  157. else :
  158. extensions = []
  159. mem = memory()
  160. start = time.clock()
  161. self.md = markdown.Markdown(extensions=extensions, safe_mode = safe_mode, output_format=output_format)
  162. construction_time = time.clock() - start
  163. construction_mem = memory(mem)
  164. self.benchmark_buffer = "construction:%f:%f\n" % (construction_time,
  165. construction_mem)
  166. html_diff_file_path = os.path.join(TMP_DIR, os.path.split(dir)[-1]) + ".html"
  167. self.html_diff_file = codecs.open(html_diff_file_path, "w", encoding=encoding)
  168. self.html_diff_file.write(DIFF_FILE_TEMPLATE)
  169. self.diffs_buffer = ""
  170. tests = [x.replace(".txt", "")
  171. for x in os.listdir(dir) if x.endswith(".txt")]
  172. tests.sort()
  173. for test in tests :
  174. self.run_test(dir, test, repeat)
  175. self.html_diff_file.write("</table>")
  176. if sys.version < "3.0":
  177. self.html_diff_file.write(self.diffs_buffer.decode("utf8"))
  178. self.html_diff_file.write(FOOTER)
  179. self.html_diff_file.close()
  180. print "Diff written to %s" % html_diff_file_path
  181. benchmark_output_file_name = benchmark_file_name
  182. if not WRITE_BENCHMARK:
  183. benchmark_output_file_name += ".tmp"
  184. self.benchmark_file = open(benchmark_output_file_name, "w")
  185. self.benchmark_file.write(self.benchmark_buffer)
  186. self.benchmark_file.close()
  187. ####################
  188. def run_test(self, dir, test, repeat):
  189. print "--- %s ---" % test
  190. self.html_diff_file.write("<tr><td>%s</td>" % test)
  191. input_file = os.path.join(dir, test + ".txt")
  192. output_file = os.path.join(dir, test + ".html")
  193. expected_output = codecs.open(output_file, encoding=self.encoding).read()
  194. input = codecs.open(input_file, encoding=self.encoding).read()
  195. actual_output = ""
  196. actual_lines = []
  197. self.md.source = ""
  198. gc.collect()
  199. mem = memory()
  200. start = time.clock()
  201. for x in repeat:
  202. actual_output = self.md.convert(input)
  203. conversion_time = time.clock() - start
  204. conversion_mem = memory(mem)
  205. self.md.reset()
  206. expected_lines = [x.encode("utf8") for x in smart_split(expected_output)]
  207. actual_lines = [x.encode("utf8") for x in smart_split(actual_output)]
  208. #diff = difflib.ndiff(expected_output.split("\n"),
  209. # actual_output.split("\n"))
  210. diff = [x for x in differ.compare(expected_lines,
  211. actual_lines)
  212. if not x.startswith(" ")]
  213. if not diff:
  214. self.html_diff_file.write("<td class='ok'>OK</td>")
  215. else :
  216. self.failedTests.append(test)
  217. self.html_diff_file.write("<td class='failed'>" +
  218. "<a href='#diff-%s'>FAILED</a></td>" % test)
  219. print "MISMATCH on %s/%s.txt" % (dir, test)
  220. print
  221. for line in diff :
  222. print line
  223. if htmldiff!=None :
  224. htmlDiff = htmldiff.make_table(expected_lines, actual_lines,
  225. context=True)
  226. htmlDiff = "\n".join( [x for x in htmlDiff.splitlines()
  227. if x.strip().startswith("<tr>")] )
  228. self.diffs_buffer += "<a name='diff-%s'/><h2>%s</h2>" % (test, test)
  229. self.diffs_buffer += DIFF_TABLE_TEMPLATE % htmlDiff
  230. expected_time, expected_mem = self.saved_benchmarks.get(test, ("na", "na"))
  231. self.html_diff_file.write(get_benchmark_html(conversion_time, expected_time))
  232. self.html_diff_file.write(get_benchmark_html(conversion_mem, expected_mem))
  233. self.html_diff_file.write("</tr>\n")
  234. self.benchmark_buffer += "%s:%f:%f\n" % (test,
  235. conversion_time, conversion_mem)
  236. def get_benchmark_html (actual, expected) :
  237. buffer = ""
  238. if not expected == "na":
  239. if actual > expected * 1.5:
  240. tdiff = "failed"
  241. elif actual * 1.5 < expected :
  242. tdiff = "ok"
  243. else :
  244. tdiff = "same"
  245. if ( (actual <= 0 and expected < 0.015) or
  246. (expected <= 0 and actual < 0.015)) :
  247. tdiff = "same"
  248. else :
  249. tdiff = "same"
  250. buffer += "<td class='%s'>%.2f</td>" % (tdiff, actual)
  251. if not expected == "na":
  252. buffer += "<td class='gray'>%.2f</td>" % (expected)
  253. return buffer
  254. def run_tests() :
  255. tester = TestRunner()
  256. #test.test_directory("tests/basic")
  257. tester.test_directory("tests/markdown-test", measure_time=True)
  258. tester.test_directory("tests/misc", measure_time=True)
  259. tester.test_directory("tests/extensions-x-tables")
  260. tester.test_directory("tests/extensions-x-footnotes")
  261. #tester.test_directory("tests/extensions-x-ext1-ext2")
  262. tester.test_directory("tests/safe_mode", measure_time=True, safe_mode="escape")
  263. tester.test_directory("tests/extensions-x-wikilinks")
  264. tester.test_directory("tests/extensions-x-toc")
  265. tester.test_directory("tests/extensions-x-def_list")
  266. tester.test_directory("tests/extensions-x-abbr")
  267. tester.test_directory("tests/html4", output_format='html4')
  268. try:
  269. import pygments
  270. except ImportError:
  271. # Dependancy not avalable - skip test
  272. pass
  273. else:
  274. tester.test_directory("tests/extensions-x-codehilite")
  275. print "\n### Final result ###"
  276. if len(tester.failedTests):
  277. print "%d failed tests: %s" % (len(tester.failedTests), str(tester.failedTests))
  278. else:
  279. print "All tests passed, no errors!"
  280. run_tests()