benchbase.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545
  1. import sys, re, string, copy, gc
  2. from itertools import *
  3. import time
  4. try:
  5. izip
  6. except NameError:
  7. izip = zip # Py3
  8. def exec_(code, glob):
  9. if sys.version_info[0] >= 3:
  10. exec(code, glob)
  11. else:
  12. exec("exec code in glob")
  13. TREE_FACTOR = 1 # increase tree size with '-l / '-L' cmd option
  14. _TEXT = "some ASCII text" * TREE_FACTOR
  15. _UTEXT = u"some klingon: \F8D2" * TREE_FACTOR
  16. _ATTRIBUTES = {
  17. '{attr}test1' : _TEXT,
  18. '{attr}test2' : _TEXT,
  19. 'bla1' : _TEXT,
  20. 'bla2' : _TEXT,
  21. 'bla3' : _TEXT
  22. }
  23. def initArgs(argv):
  24. global TREE_FACTOR
  25. try:
  26. argv.remove('-l')
  27. # use large trees
  28. TREE_FACTOR *= 2
  29. except ValueError:
  30. pass
  31. try:
  32. argv.remove('-L')
  33. # use LARGE trees
  34. TREE_FACTOR *= 2
  35. except ValueError:
  36. pass
  37. ############################################################
  38. # benchmark decorators
  39. ############################################################
  40. def with_attributes(*use_attributes):
  41. "Decorator for benchmarks that use attributes"
  42. vmap = {False : 0, True : 1}
  43. values = [ vmap[bool(v)] for v in use_attributes ]
  44. def set_value(function):
  45. try:
  46. function.ATTRIBUTES.update(values)
  47. except AttributeError:
  48. function.ATTRIBUTES = set(values)
  49. return function
  50. return set_value
  51. def with_text(no_text=False, text=False, utext=False):
  52. "Decorator for benchmarks that use text"
  53. values = []
  54. if no_text:
  55. values.append(0)
  56. if text:
  57. values.append(1)
  58. if utext:
  59. values.append(2)
  60. def set_value(function):
  61. try:
  62. function.TEXT.add(values)
  63. except AttributeError:
  64. function.TEXT = set(values)
  65. return function
  66. return set_value
  67. def onlylib(*libs):
  68. "Decorator to restrict benchmarks to specific libraries"
  69. def set_libs(function):
  70. if libs:
  71. function.LIBS = libs
  72. return function
  73. return set_libs
  74. def serialized(function):
  75. "Decorator for benchmarks that require serialized XML data"
  76. function.STRING = True
  77. return function
  78. def children(function):
  79. "Decorator for benchmarks that require a list of root children"
  80. function.CHILDREN = True
  81. return function
  82. def nochange(function):
  83. "Decorator for benchmarks that do not change the XML tree"
  84. function.NO_CHANGE = True
  85. return function
  86. ############################################################
  87. # benchmark baseclass
  88. ############################################################
  89. class SkippedTest(Exception):
  90. pass
  91. class TreeBenchMark(object):
  92. atoz = string.ascii_lowercase
  93. repeat100 = range(100)
  94. repeat500 = range(500)
  95. repeat1000 = range(1000)
  96. _LIB_NAME_MAP = {
  97. 'etree' : 'lxe',
  98. 'ElementTree' : 'ET',
  99. 'cElementTree' : 'cET'
  100. }
  101. SEARCH_TAG = "{cdefg}a00001"
  102. def __init__(self, etree, etree_parser=None):
  103. self.etree = etree
  104. libname = etree.__name__.split('.')[-1]
  105. self.lib_name = self._LIB_NAME_MAP.get(libname, libname)
  106. if libname == 'etree':
  107. deepcopy = copy.deepcopy
  108. def set_property(root, fname):
  109. xml = self._serialize_tree(root)
  110. if etree_parser is not None:
  111. setattr(self, fname, lambda : etree.XML(xml, etree_parser))
  112. else:
  113. setattr(self, fname, lambda : deepcopy(root))
  114. setattr(self, fname + '_xml', lambda : xml)
  115. setattr(self, fname + '_children', lambda : root[:])
  116. else:
  117. def set_property(root, fname):
  118. setattr(self, fname, self.et_make_clone_factory(root))
  119. xml = self._serialize_tree(root)
  120. setattr(self, fname + '_xml', lambda : xml)
  121. setattr(self, fname + '_children', lambda : root[:])
  122. attribute_list = list(enumerate( [{}, _ATTRIBUTES] ))
  123. text_list = list(enumerate( [None, _TEXT, _UTEXT] ))
  124. build_name = self._tree_builder_name
  125. self.setup_times = []
  126. for tree in self._all_trees():
  127. times = []
  128. self.setup_times.append(times)
  129. setup = getattr(self, '_setup_tree%d' % tree)
  130. for an, attributes in attribute_list:
  131. for tn, text in text_list:
  132. root, t = setup(text, attributes)
  133. times.append(t)
  134. set_property(root, build_name(tree, tn, an))
  135. def _tree_builder_name(self, tree, tn, an):
  136. return '_root%d_T%d_A%d' % (tree, tn, an)
  137. def tree_builder(self, tree, tn, an, serial, children):
  138. name = self._tree_builder_name(tree, tn, an)
  139. if serial:
  140. name += '_xml'
  141. elif children:
  142. name += '_children'
  143. return getattr(self, name)
  144. def _serialize_tree(self, root):
  145. return self.etree.tostring(root, encoding='UTF-8')
  146. def et_make_clone_factory(self, elem):
  147. def generate_elem(append, elem, level):
  148. var = "e" + str(level)
  149. arg = repr(elem.tag)
  150. if elem.attrib:
  151. arg += ", **%r" % elem.attrib
  152. if level == 1:
  153. append(" e1 = Element(%s)" % arg)
  154. else:
  155. append(" %s = SubElement(e%d, %s)" % (var, level-1, arg))
  156. if elem.text:
  157. append(" %s.text = %r" % (var, elem.text))
  158. if elem.tail:
  159. append(" %s.tail = %r" % (var, elem.tail))
  160. for e in elem:
  161. generate_elem(append, e, level+1)
  162. # generate code for a function that creates a tree
  163. output = ["def element_factory():"]
  164. generate_elem(output.append, elem, 1)
  165. output.append(" return e1")
  166. # setup global function namespace
  167. namespace = {
  168. "Element" : self.etree.Element,
  169. "SubElement" : self.etree.SubElement
  170. }
  171. # create function object
  172. exec_("\n".join(output), namespace)
  173. return namespace["element_factory"]
  174. def _all_trees(self):
  175. all_trees = []
  176. for name in dir(self):
  177. if name.startswith('_setup_tree'):
  178. all_trees.append(int(name[11:]))
  179. return all_trees
  180. def _setup_tree1(self, text, attributes):
  181. "tree with 26 2nd level and 520 * TREE_FACTOR 3rd level children"
  182. atoz = self.atoz
  183. SubElement = self.etree.SubElement
  184. current_time = time.time
  185. t = current_time()
  186. root = self.etree.Element('{abc}rootnode')
  187. for ch1 in atoz:
  188. el = SubElement(root, "{abc}"+ch1*5, attributes)
  189. el.text = text
  190. for ch2 in atoz:
  191. tag = "{cdefg}%s00001" % ch2
  192. for i in range(20 * TREE_FACTOR):
  193. SubElement(el, tag).tail = text
  194. t = current_time() - t
  195. return root, t
  196. def _setup_tree2(self, text, attributes):
  197. "tree with 520 * TREE_FACTOR 2nd level and 26 3rd level children"
  198. atoz = self.atoz
  199. SubElement = self.etree.SubElement
  200. current_time = time.time
  201. t = current_time()
  202. root = self.etree.Element('{abc}rootnode')
  203. for ch1 in atoz:
  204. for i in range(20 * TREE_FACTOR):
  205. el = SubElement(root, "{abc}"+ch1*5, attributes)
  206. el.text = text
  207. for ch2 in atoz:
  208. SubElement(el, "{cdefg}%s00001" % ch2).tail = text
  209. t = current_time() - t
  210. return root, t
  211. def _setup_tree3(self, text, attributes):
  212. "tree of depth 8 + TREE_FACTOR with 3 children per node"
  213. SubElement = self.etree.SubElement
  214. current_time = time.time
  215. t = current_time()
  216. root = self.etree.Element('{abc}rootnode')
  217. children = [root]
  218. for i in range(6 + TREE_FACTOR):
  219. children = [ SubElement(c, "{cdefg}a%05d" % (i%8), attributes)
  220. for i,c in enumerate(chain(children, children, children)) ]
  221. for child in children:
  222. child.text = text
  223. child.tail = text
  224. t = current_time() - t
  225. return root, t
  226. def _setup_tree4(self, text, attributes):
  227. "small tree with 26 2nd level and 2 3rd level children"
  228. SubElement = self.etree.SubElement
  229. current_time = time.time
  230. t = current_time()
  231. root = self.etree.Element('{abc}rootnode')
  232. for ch1 in self.atoz:
  233. el = SubElement(root, "{abc}"+ch1*5, attributes)
  234. el.text = text
  235. SubElement(el, "{cdefg}a00001", attributes).tail = text
  236. SubElement(el, "{cdefg}z00000", attributes).tail = text
  237. t = current_time() - t
  238. return root, t
  239. def benchmarks(self):
  240. """Returns a list of all benchmarks.
  241. A benchmark is a tuple containing a method name and a list of tree
  242. numbers. Trees are prepared by the setup function.
  243. """
  244. all_trees = self._all_trees()
  245. benchmarks = []
  246. for name in dir(self):
  247. if not name.startswith('bench_'):
  248. continue
  249. method = getattr(self, name)
  250. if hasattr(method, 'LIBS') and self.lib_name not in method.LIBS:
  251. method_call = None
  252. else:
  253. method_call = method
  254. if method.__doc__:
  255. tree_sets = method.__doc__.split()
  256. else:
  257. tree_sets = ()
  258. if tree_sets:
  259. tree_tuples = [list(map(int, tree_set.split(',')))
  260. for tree_set in tree_sets]
  261. else:
  262. try:
  263. arg_count = method.func_code.co_argcount - 1
  264. except AttributeError:
  265. try:
  266. arg_count = method.__code__.co_argcount - 1
  267. except AttributeError:
  268. arg_count = 1
  269. tree_tuples = self._permutations(all_trees, arg_count)
  270. serialized = getattr(method, 'STRING', False)
  271. children = getattr(method, 'CHILDREN', False)
  272. no_change = getattr(method, 'NO_CHANGE', False)
  273. for tree_tuple in tree_tuples:
  274. for tn in sorted(getattr(method, 'TEXT', (0,))):
  275. for an in sorted(getattr(method, 'ATTRIBUTES', (0,))):
  276. benchmarks.append((name, method_call, tree_tuple,
  277. tn, an, serialized, children,
  278. no_change))
  279. return benchmarks
  280. def _permutations(self, seq, count):
  281. def _permutations(prefix, remainder, count):
  282. if count == 0:
  283. return [ prefix[:] ]
  284. count -= 1
  285. perms = []
  286. prefix.append(None)
  287. for pos, el in enumerate(remainder):
  288. new_remainder = remainder[:pos] + remainder[pos+1:]
  289. prefix[-1] = el
  290. perms.extend( _permutations(prefix, new_remainder, count) )
  291. prefix.pop()
  292. return perms
  293. return _permutations([], seq, count)
  294. ############################################################
  295. # Prepare and run benchmark suites
  296. ############################################################
  297. def buildSuites(benchmark_class, etrees, selected):
  298. benchmark_suites = list(map(benchmark_class, etrees))
  299. # sorted by name and tree tuple
  300. benchmarks = [ sorted(b.benchmarks()) for b in benchmark_suites ]
  301. selected = [ re.compile(r).search for r in selected ]
  302. if selected:
  303. benchmarks = [ [ b for b in bs
  304. if [ match for match in selected
  305. if match(b[0]) ] ]
  306. for bs in benchmarks ]
  307. return benchmark_suites, benchmarks
  308. def build_treeset_name(trees, tn, an, serialized, children):
  309. text = {0:'-', 1:'S', 2:'U'}[tn]
  310. attr = {0:'-', 1:'A'}[an]
  311. ser = {True:'X', False:'T'}[serialized]
  312. chd = {True:'C', False:'R'}[children]
  313. return "%s%s%s%s T%s" % (text, attr, ser, chd, ',T'.join(map(str, trees))[:6])
  314. def printSetupTimes(benchmark_suites):
  315. print("Setup times for trees in seconds:")
  316. for b in benchmark_suites:
  317. sys.stdout.write("%-3s: " % b.lib_name)
  318. for an in (0,1):
  319. for tn in (0,1,2):
  320. sys.stdout.write(' %s ' %
  321. build_treeset_name((), tn, an, False, False)[:2])
  322. print('')
  323. for i, tree_times in enumerate(b.setup_times):
  324. print(" T%d: %s" % (i+1, ' '.join("%6.4f" % t for t in tree_times)))
  325. print('')
  326. def runBench(suite, method_name, method_call, tree_set, tn, an,
  327. serial, children, no_change):
  328. if method_call is None:
  329. raise SkippedTest
  330. current_time = time.time
  331. call_repeat = range(10)
  332. tree_builders = [ suite.tree_builder(tree, tn, an, serial, children)
  333. for tree in tree_set ]
  334. rebuild_trees = not no_change and not serial
  335. args = tuple([ build() for build in tree_builders ])
  336. method_call(*args) # run once to skip setup overhead
  337. times = []
  338. for i in range(3):
  339. gc.collect()
  340. gc.disable()
  341. t = -1
  342. for i in call_repeat:
  343. if rebuild_trees:
  344. args = [ build() for build in tree_builders ]
  345. t_one_call = current_time()
  346. method_call(*args)
  347. t_one_call = current_time() - t_one_call
  348. if t < 0:
  349. t = t_one_call
  350. else:
  351. t = min(t, t_one_call)
  352. times.append(1000.0 * t)
  353. gc.enable()
  354. if rebuild_trees:
  355. args = ()
  356. args = ()
  357. gc.collect()
  358. return times
  359. def runBenchmarks(benchmark_suites, benchmarks):
  360. for bench_calls in izip(*benchmarks):
  361. for lib, (bench, benchmark_setup) in enumerate(izip(benchmark_suites, bench_calls)):
  362. bench_name = benchmark_setup[0]
  363. tree_set_name = build_treeset_name(*benchmark_setup[-6:-1])
  364. sys.stdout.write("%-3s: %-28s (%-10s) " % (
  365. bench.lib_name, bench_name[6:34], tree_set_name))
  366. sys.stdout.flush()
  367. try:
  368. result = runBench(bench, *benchmark_setup)
  369. except SkippedTest:
  370. print("skipped")
  371. except KeyboardInterrupt:
  372. print("interrupted by user")
  373. sys.exit(1)
  374. except Exception:
  375. exc_type, exc_value = sys.exc_info()[:2]
  376. print("failed: %s: %s" % (exc_type.__name__, exc_value))
  377. exc_type = exc_value = None
  378. else:
  379. print("%9.4f msec/pass, best of (%s)" % (
  380. min(result), ' '.join("%9.4f" % t for t in result)))
  381. if len(benchmark_suites) > 1:
  382. print('') # empty line between different benchmarks
  383. ############################################################
  384. # Main program
  385. ############################################################
  386. def main(benchmark_class):
  387. import_lxml = True
  388. callgrind_zero = False
  389. if len(sys.argv) > 1:
  390. try:
  391. sys.argv.remove('-i')
  392. # run benchmark 'inplace'
  393. sys.path.insert(0, 'src')
  394. except ValueError:
  395. pass
  396. try:
  397. sys.argv.remove('-nolxml')
  398. # run without lxml
  399. import_lxml = False
  400. except ValueError:
  401. pass
  402. try:
  403. sys.argv.remove('-z')
  404. # reset callgrind after tree setup
  405. callgrind_zero = True
  406. except ValueError:
  407. pass
  408. initArgs(sys.argv)
  409. _etrees = []
  410. if import_lxml:
  411. from lxml import etree
  412. _etrees.append(etree)
  413. print("Using lxml %s (with libxml2 %s)" % (
  414. etree.__version__, '.'.join(map(str, etree.LIBXML_VERSION))))
  415. try:
  416. sys.argv.remove('-fel')
  417. except ValueError:
  418. pass
  419. else:
  420. # use fast element creation in lxml.etree
  421. etree.set_element_class_lookup(
  422. etree.ElementDefaultClassLookup())
  423. if len(sys.argv) > 1:
  424. if '-a' in sys.argv or '-c' in sys.argv:
  425. # 'all' or 'C-implementations' ?
  426. try:
  427. sys.argv.remove('-c')
  428. except ValueError:
  429. pass
  430. try:
  431. import cElementTree as cET
  432. _etrees.append(cET)
  433. except ImportError:
  434. try:
  435. import xml.etree.cElementTree as cET
  436. _etrees.append(cET)
  437. except ImportError:
  438. pass
  439. try:
  440. # 'all' ?
  441. sys.argv.remove('-a')
  442. except ValueError:
  443. pass
  444. else:
  445. try:
  446. from elementtree import ElementTree as ET
  447. _etrees.append(ET)
  448. except ImportError:
  449. try:
  450. from xml.etree import ElementTree as ET
  451. _etrees.append(ET)
  452. except ImportError:
  453. pass
  454. if not _etrees:
  455. print("No library to test. Exiting.")
  456. sys.exit(1)
  457. print("Running benchmarks in Python %s" % (sys.version_info,))
  458. print("Preparing test suites and trees ...")
  459. selected = set( sys.argv[1:] )
  460. benchmark_suites, benchmarks = \
  461. buildSuites(benchmark_class, _etrees, selected)
  462. print("Running benchmark on", ', '.join(b.lib_name
  463. for b in benchmark_suites))
  464. print('')
  465. printSetupTimes(benchmark_suites)
  466. if callgrind_zero:
  467. cmd = open("callgrind.cmd", 'w')
  468. cmd.write('+Instrumentation\n')
  469. cmd.write('Zero\n')
  470. cmd.close()
  471. runBenchmarks(benchmark_suites, benchmarks)