benchbase.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546
  1. import sys, re, string, time, copy, gc
  2. from itertools import *
  3. import time
  4. try:
  5. from io import BytesIO
  6. except ImportError:
  7. from StringIO import StringIO as BytesIO
  8. try:
  9. izip
  10. except NameError:
  11. izip = zip # Py3
  12. def exec_(code, glob):
  13. if sys.version_info[0] >= 3:
  14. exec(code, glob)
  15. else:
  16. exec("exec code in glob")
  17. TREE_FACTOR = 1 # increase tree size with '-l / '-L' cmd option
  18. _TEXT = "some ASCII text" * TREE_FACTOR
  19. _UTEXT = u"some klingon: \F8D2" * TREE_FACTOR
  20. _ATTRIBUTES = {
  21. '{attr}test1' : _TEXT,
  22. '{attr}test2' : _TEXT,
  23. 'bla1' : _TEXT,
  24. 'bla2' : _TEXT,
  25. 'bla3' : _TEXT
  26. }
  27. def initArgs(argv):
  28. global TREE_FACTOR
  29. try:
  30. argv.remove('-l')
  31. # use large trees
  32. TREE_FACTOR *= 2
  33. except ValueError:
  34. pass
  35. try:
  36. argv.remove('-L')
  37. # use LARGE trees
  38. TREE_FACTOR *= 2
  39. except ValueError:
  40. pass
  41. ############################################################
  42. # benchmark decorators
  43. ############################################################
  44. def with_attributes(*use_attributes):
  45. "Decorator for benchmarks that use attributes"
  46. vmap = {False : 0, True : 1}
  47. values = [ vmap[bool(v)] for v in use_attributes ]
  48. def set_value(function):
  49. try:
  50. function.ATTRIBUTES.update(values)
  51. except AttributeError:
  52. function.ATTRIBUTES = set(values)
  53. return function
  54. return set_value
  55. def with_text(no_text=False, text=False, utext=False):
  56. "Decorator for benchmarks that use text"
  57. values = []
  58. if no_text:
  59. values.append(0)
  60. if text:
  61. values.append(1)
  62. if utext:
  63. values.append(2)
  64. def set_value(function):
  65. try:
  66. function.TEXT.add(values)
  67. except AttributeError:
  68. function.TEXT = set(values)
  69. return function
  70. return set_value
  71. def onlylib(*libs):
  72. "Decorator to restrict benchmarks to specific libraries"
  73. def set_libs(function):
  74. if libs:
  75. function.LIBS = libs
  76. return function
  77. return set_libs
  78. def serialized(function):
  79. "Decorator for benchmarks that require serialized XML data"
  80. function.STRING = True
  81. return function
  82. def children(function):
  83. "Decorator for benchmarks that require a list of root children"
  84. function.CHILDREN = True
  85. return function
  86. def nochange(function):
  87. "Decorator for benchmarks that do not change the XML tree"
  88. function.NO_CHANGE = True
  89. return function
  90. ############################################################
  91. # benchmark baseclass
  92. ############################################################
  93. class SkippedTest(Exception):
  94. pass
  95. class TreeBenchMark(object):
  96. atoz = string.ascii_lowercase
  97. repeat100 = range(100)
  98. repeat500 = range(500)
  99. repeat1000 = range(1000)
  100. _LIB_NAME_MAP = {
  101. 'etree' : 'lxe',
  102. 'ElementTree' : 'ET',
  103. 'cElementTree' : 'cET'
  104. }
  105. SEARCH_TAG = "{cdefg}a00001"
  106. def __init__(self, etree, etree_parser=None):
  107. self.etree = etree
  108. libname = etree.__name__.split('.')[-1]
  109. self.lib_name = self._LIB_NAME_MAP.get(libname, libname)
  110. if libname == 'etree':
  111. deepcopy = copy.deepcopy
  112. def set_property(root, fname):
  113. xml = self._serialize_tree(root)
  114. if etree_parser is not None:
  115. setattr(self, fname, lambda : etree.XML(xml, etree_parser))
  116. else:
  117. setattr(self, fname, lambda : deepcopy(root))
  118. setattr(self, fname + '_xml', lambda : xml)
  119. setattr(self, fname + '_children', lambda : root[:])
  120. else:
  121. def set_property(root, fname):
  122. setattr(self, fname, self.et_make_clone_factory(root))
  123. xml = self._serialize_tree(root)
  124. setattr(self, fname + '_xml', lambda : xml)
  125. setattr(self, fname + '_children', lambda : root[:])
  126. attribute_list = list(enumerate( [{}, _ATTRIBUTES] ))
  127. text_list = list(enumerate( [None, _TEXT, _UTEXT] ))
  128. build_name = self._tree_builder_name
  129. self.setup_times = []
  130. for tree in self._all_trees():
  131. times = []
  132. self.setup_times.append(times)
  133. setup = getattr(self, '_setup_tree%d' % tree)
  134. for an, attributes in attribute_list:
  135. for tn, text in text_list:
  136. root, t = setup(text, attributes)
  137. times.append(t)
  138. set_property(root, build_name(tree, tn, an))
  139. def _tree_builder_name(self, tree, tn, an):
  140. return '_root%d_T%d_A%d' % (tree, tn, an)
  141. def tree_builder(self, tree, tn, an, serial, children):
  142. name = self._tree_builder_name(tree, tn, an)
  143. if serial:
  144. name += '_xml'
  145. elif children:
  146. name += '_children'
  147. return getattr(self, name)
  148. def _serialize_tree(self, root):
  149. return self.etree.tostring(root, encoding='UTF-8')
  150. def et_make_clone_factory(self, elem):
  151. def generate_elem(append, elem, level):
  152. var = "e" + str(level)
  153. arg = repr(elem.tag)
  154. if elem.attrib:
  155. arg += ", **%r" % elem.attrib
  156. if level == 1:
  157. append(" e1 = Element(%s)" % arg)
  158. else:
  159. append(" %s = SubElement(e%d, %s)" % (var, level-1, arg))
  160. if elem.text:
  161. append(" %s.text = %r" % (var, elem.text))
  162. if elem.tail:
  163. append(" %s.tail = %r" % (var, elem.tail))
  164. for e in elem:
  165. generate_elem(append, e, level+1)
  166. # generate code for a function that creates a tree
  167. output = ["def element_factory():"]
  168. generate_elem(output.append, elem, 1)
  169. output.append(" return e1")
  170. # setup global function namespace
  171. namespace = {
  172. "Element" : self.etree.Element,
  173. "SubElement" : self.etree.SubElement
  174. }
  175. # create function object
  176. exec_("\n".join(output), namespace)
  177. return namespace["element_factory"]
  178. def _all_trees(self):
  179. all_trees = []
  180. for name in dir(self):
  181. if name.startswith('_setup_tree'):
  182. all_trees.append(int(name[11:]))
  183. return all_trees
  184. def _setup_tree1(self, text, attributes):
  185. "tree with 26 2nd level and 520 * TREE_FACTOR 3rd level children"
  186. atoz = self.atoz
  187. SubElement = self.etree.SubElement
  188. current_time = time.time
  189. t = current_time()
  190. root = self.etree.Element('{abc}rootnode')
  191. for ch1 in atoz:
  192. el = SubElement(root, "{abc}"+ch1*5, attributes)
  193. el.text = text
  194. for ch2 in atoz:
  195. tag = "{cdefg}%s00001" % ch2
  196. for i in range(20 * TREE_FACTOR):
  197. SubElement(el, tag).tail = text
  198. t = current_time() - t
  199. return (root, t)
  200. def _setup_tree2(self, text, attributes):
  201. "tree with 520 * TREE_FACTOR 2nd level and 26 3rd level children"
  202. atoz = self.atoz
  203. SubElement = self.etree.SubElement
  204. current_time = time.time
  205. t = current_time()
  206. root = self.etree.Element('{abc}rootnode')
  207. for ch1 in atoz:
  208. for i in range(20 * TREE_FACTOR):
  209. el = SubElement(root, "{abc}"+ch1*5, attributes)
  210. el.text = text
  211. for ch2 in atoz:
  212. SubElement(el, "{cdefg}%s00001" % ch2).tail = text
  213. t = current_time() - t
  214. return (root, t)
  215. def _setup_tree3(self, text, attributes):
  216. "tree of depth 8 + TREE_FACTOR with 3 children per node"
  217. SubElement = self.etree.SubElement
  218. current_time = time.time
  219. t = current_time()
  220. root = self.etree.Element('{abc}rootnode')
  221. children = [root]
  222. for i in range(6 + TREE_FACTOR):
  223. children = [ SubElement(c, "{cdefg}a%05d" % (i%8), attributes)
  224. for i,c in enumerate(chain(children, children, children)) ]
  225. for child in children:
  226. child.text = text
  227. child.tail = text
  228. t = current_time() - t
  229. return (root, t)
  230. def _setup_tree4(self, text, attributes):
  231. "small tree with 26 2nd level and 2 3rd level children"
  232. SubElement = self.etree.SubElement
  233. current_time = time.time
  234. t = current_time()
  235. root = self.etree.Element('{abc}rootnode')
  236. for ch1 in self.atoz:
  237. el = SubElement(root, "{abc}"+ch1*5, attributes)
  238. el.text = text
  239. SubElement(el, "{cdefg}a00001", attributes).tail = text
  240. SubElement(el, "{cdefg}z00000", attributes).tail = text
  241. t = current_time() - t
  242. return (root, t)
  243. def benchmarks(self):
  244. """Returns a list of all benchmarks.
  245. A benchmark is a tuple containing a method name and a list of tree
  246. numbers. Trees are prepared by the setup function.
  247. """
  248. all_trees = self._all_trees()
  249. benchmarks = []
  250. for name in dir(self):
  251. if not name.startswith('bench_'):
  252. continue
  253. method = getattr(self, name)
  254. if hasattr(method, 'LIBS') and self.lib_name not in method.LIBS:
  255. method_call = None
  256. else:
  257. method_call = method
  258. if method.__doc__:
  259. tree_sets = method.__doc__.split()
  260. else:
  261. tree_sets = ()
  262. if tree_sets:
  263. tree_tuples = [list(map(int, tree_set.split(',')))
  264. for tree_set in tree_sets]
  265. else:
  266. try:
  267. arg_count = method.func_code.co_argcount - 1
  268. except AttributeError:
  269. try:
  270. arg_count = method.__code__.co_argcount - 1
  271. except AttributeError:
  272. arg_count = 1
  273. tree_tuples = self._permutations(all_trees, arg_count)
  274. serialized = getattr(method, 'STRING', False)
  275. children = getattr(method, 'CHILDREN', False)
  276. no_change = getattr(method, 'NO_CHANGE', False)
  277. for tree_tuple in tree_tuples:
  278. for tn in sorted(getattr(method, 'TEXT', (0,))):
  279. for an in sorted(getattr(method, 'ATTRIBUTES', (0,))):
  280. benchmarks.append((name, method_call, tree_tuple,
  281. tn, an, serialized, children,
  282. no_change))
  283. return benchmarks
  284. def _permutations(self, seq, count):
  285. def _permutations(prefix, remainder, count):
  286. if count == 0:
  287. return [ prefix[:] ]
  288. count -= 1
  289. perms = []
  290. prefix.append(None)
  291. for pos, el in enumerate(remainder):
  292. new_remainder = remainder[:pos] + remainder[pos+1:]
  293. prefix[-1] = el
  294. perms.extend( _permutations(prefix, new_remainder, count) )
  295. prefix.pop()
  296. return perms
  297. return _permutations([], seq, count)
  298. ############################################################
  299. # Prepare and run benchmark suites
  300. ############################################################
  301. def buildSuites(benchmark_class, etrees, selected):
  302. benchmark_suites = list(map(benchmark_class, etrees))
  303. # sorted by name and tree tuple
  304. benchmarks = [ sorted(b.benchmarks()) for b in benchmark_suites ]
  305. selected = [ re.compile(r).search for r in selected ]
  306. if selected:
  307. benchmarks = [ [ b for b in bs
  308. if [ match for match in selected
  309. if match(b[0]) ] ]
  310. for bs in benchmarks ]
  311. return (benchmark_suites, benchmarks)
  312. def build_treeset_name(trees, tn, an, serialized, children):
  313. text = {0:'-', 1:'S', 2:'U'}[tn]
  314. attr = {0:'-', 1:'A'}[an]
  315. ser = {True:'X', False:'T'}[serialized]
  316. chd = {True:'C', False:'R'}[children]
  317. return "%s%s%s%s T%s" % (text, attr, ser, chd, ',T'.join(map(str, trees))[:6])
  318. def printSetupTimes(benchmark_suites):
  319. print("Setup times for trees in seconds:")
  320. for b in benchmark_suites:
  321. sys.stdout.write("%-3s: " % b.lib_name)
  322. for an in (0,1):
  323. for tn in (0,1,2):
  324. sys.stdout.write(' %s ' %
  325. build_treeset_name((), tn, an, False, False)[:2])
  326. print('')
  327. for i, tree_times in enumerate(b.setup_times):
  328. print(" T%d: %s" % (i+1, ' '.join("%6.4f" % t for t in tree_times)))
  329. print('')
  330. def runBench(suite, method_name, method_call, tree_set, tn, an,
  331. serial, children, no_change):
  332. if method_call is None:
  333. raise SkippedTest
  334. current_time = time.time
  335. call_repeat = range(10)
  336. tree_builders = [ suite.tree_builder(tree, tn, an, serial, children)
  337. for tree in tree_set ]
  338. rebuild_trees = not no_change and not serial
  339. args = tuple([ build() for build in tree_builders ])
  340. method_call(*args) # run once to skip setup overhead
  341. times = []
  342. for i in range(3):
  343. gc.collect()
  344. gc.disable()
  345. t = -1
  346. for i in call_repeat:
  347. if rebuild_trees:
  348. args = [ build() for build in tree_builders ]
  349. t_one_call = current_time()
  350. method_call(*args)
  351. t_one_call = current_time() - t_one_call
  352. if t < 0:
  353. t = t_one_call
  354. else:
  355. t = min(t, t_one_call)
  356. times.append(1000.0 * t)
  357. gc.enable()
  358. if rebuild_trees:
  359. args = ()
  360. args = ()
  361. gc.collect()
  362. return times
  363. def runBenchmarks(benchmark_suites, benchmarks):
  364. for bench_calls in izip(*benchmarks):
  365. for lib, (bench, benchmark_setup) in enumerate(izip(benchmark_suites, bench_calls)):
  366. bench_name = benchmark_setup[0]
  367. tree_set_name = build_treeset_name(*benchmark_setup[-6:-1])
  368. sys.stdout.write("%-3s: %-28s (%-10s) " % (
  369. bench.lib_name, bench_name[6:34], tree_set_name))
  370. sys.stdout.flush()
  371. try:
  372. result = runBench(bench, *benchmark_setup)
  373. except SkippedTest:
  374. print("skipped")
  375. except KeyboardInterrupt:
  376. print("interrupted by user")
  377. sys.exit(1)
  378. except Exception:
  379. exc_type, exc_value = sys.exc_info()[:2]
  380. print("failed: %s: %s" % (exc_type.__name__, exc_value))
  381. exc_type = exc_value = None
  382. else:
  383. print("%9.4f msec/pass, best of (%s)" % (
  384. min(result), ' '.join("%9.4f" % t for t in result)))
  385. if len(benchmark_suites) > 1:
  386. print('') # empty line between different benchmarks
  387. ############################################################
  388. # Main program
  389. ############################################################
  390. def main(benchmark_class):
  391. import_lxml = True
  392. callgrind_zero = False
  393. if len(sys.argv) > 1:
  394. try:
  395. sys.argv.remove('-i')
  396. # run benchmark 'inplace'
  397. sys.path.insert(0, 'src')
  398. except ValueError:
  399. pass
  400. try:
  401. sys.argv.remove('-nolxml')
  402. # run without lxml
  403. import_lxml = False
  404. except ValueError:
  405. pass
  406. try:
  407. sys.argv.remove('-z')
  408. # reset callgrind after tree setup
  409. callgrind_zero = True
  410. except ValueError:
  411. pass
  412. initArgs(sys.argv)
  413. _etrees = []
  414. if import_lxml:
  415. from lxml import etree
  416. _etrees.append(etree)
  417. try:
  418. sys.argv.remove('-fel')
  419. except ValueError:
  420. pass
  421. else:
  422. # use fast element creation in lxml.etree
  423. etree.set_element_class_lookup(
  424. etree.ElementDefaultClassLookup())
  425. if len(sys.argv) > 1:
  426. if '-a' in sys.argv or '-c' in sys.argv:
  427. # 'all' or 'C-implementations' ?
  428. try:
  429. sys.argv.remove('-c')
  430. except ValueError:
  431. pass
  432. try:
  433. import cElementTree as cET
  434. _etrees.append(cET)
  435. except ImportError:
  436. try:
  437. import xml.etree.cElementTree as cET
  438. _etrees.append(cET)
  439. except ImportError:
  440. pass
  441. try:
  442. # 'all' ?
  443. sys.argv.remove('-a')
  444. except ValueError:
  445. pass
  446. else:
  447. try:
  448. from elementtree import ElementTree as ET
  449. _etrees.append(ET)
  450. except ImportError:
  451. try:
  452. from xml.etree import ElementTree as ET
  453. _etrees.append(ET)
  454. except ImportError:
  455. pass
  456. if not _etrees:
  457. print("No library to test. Exiting.")
  458. sys.exit(1)
  459. print("Preparing test suites and trees ...")
  460. selected = set( sys.argv[1:] )
  461. benchmark_suites, benchmarks = \
  462. buildSuites(benchmark_class, _etrees, selected)
  463. print("Running benchmark on", ', '.join(b.lib_name
  464. for b in benchmark_suites))
  465. print('')
  466. printSetupTimes(benchmark_suites)
  467. if callgrind_zero:
  468. cmd = open("callgrind.cmd", 'w')
  469. cmd.write('+Instrumentation\n')
  470. cmd.write('Zero\n')
  471. cmd.close()
  472. runBenchmarks(benchmark_suites, benchmarks)