TAP.py 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. #
  2. # TAP.py - TAP parser
  3. #
  4. # A pyparsing parser to process the output of the Perl
  5. # "Test Anything Protocol"
  6. # (https://metacpan.org/pod/release/PETDANCE/TAP-1.00/TAP.pm)
  7. #
  8. # TAP output lines are preceded or followed by a test number range:
  9. # 1..n
  10. # with 'n' TAP output lines.
  11. #
  12. # The general format of a TAP output line is:
  13. # ok/not ok (required)
  14. # Test number (recommended)
  15. # Description (recommended)
  16. # Directive (only when necessary)
  17. #
  18. # A TAP output line may also indicate abort of the test suit with the line:
  19. # Bail out!
  20. # optionally followed by a reason for bailing
  21. #
  22. # Copyright 2008, by Paul McGuire
  23. #
  24. from pyparsing import ParserElement,LineEnd,Optional,Word,nums,Regex,\
  25. Literal,CaselessLiteral,Group,OneOrMore,Suppress,restOfLine,\
  26. FollowedBy,empty
  27. __all__ = ['tapOutputParser', 'TAPTest', 'TAPSummary']
  28. # newlines are significant whitespace, so set default skippable
  29. # whitespace to just spaces and tabs
  30. ParserElement.setDefaultWhitespaceChars(" \t")
  31. NL = LineEnd().suppress()
  32. integer = Word(nums)
  33. plan = '1..' + integer("ubound")
  34. OK,NOT_OK = map(Literal,['ok','not ok'])
  35. testStatus = (OK | NOT_OK)
  36. description = Regex("[^#\n]+")
  37. description.setParseAction(lambda t:t[0].lstrip('- '))
  38. TODO,SKIP = map(CaselessLiteral,'TODO SKIP'.split())
  39. directive = Group(Suppress('#') + (TODO + restOfLine |
  40. FollowedBy(SKIP) +
  41. restOfLine.copy().setParseAction(lambda t:['SKIP',t[0]]) ))
  42. commentLine = Suppress("#") + empty + restOfLine
  43. testLine = Group(
  44. Optional(OneOrMore(commentLine + NL))("comments") +
  45. testStatus("passed") +
  46. Optional(integer)("testNumber") +
  47. Optional(description)("description") +
  48. Optional(directive)("directive")
  49. )
  50. bailLine = Group(Literal("Bail out!")("BAIL") +
  51. empty + Optional(restOfLine)("reason"))
  52. tapOutputParser = Optional(Group(plan)("plan") + NL) & \
  53. Group(OneOrMore((testLine|bailLine) + NL))("tests")
  54. class TAPTest(object):
  55. def __init__(self,results):
  56. self.num = results.testNumber
  57. self.passed = (results.passed=="ok")
  58. self.skipped = self.todo = False
  59. if results.directive:
  60. self.skipped = (results.directive[0][0]=='SKIP')
  61. self.todo = (results.directive[0][0]=='TODO')
  62. @classmethod
  63. def bailedTest(cls,num):
  64. ret = TAPTest(empty.parseString(""))
  65. ret.num = num
  66. ret.skipped = True
  67. return ret
  68. class TAPSummary(object):
  69. def __init__(self,results):
  70. self.passedTests = []
  71. self.failedTests = []
  72. self.skippedTests = []
  73. self.todoTests = []
  74. self.bonusTests = []
  75. self.bail = False
  76. if results.plan:
  77. expected = list(range(1, int(results.plan.ubound)+1))
  78. else:
  79. expected = list(range(1,len(results.tests)+1))
  80. for i,res in enumerate(results.tests):
  81. # test for bail out
  82. if res.BAIL:
  83. #~ print "Test suite aborted: " + res.reason
  84. #~ self.failedTests += expected[i:]
  85. self.bail = True
  86. self.skippedTests += [ TAPTest.bailedTest(ii) for ii in expected[i:] ]
  87. self.bailReason = res.reason
  88. break
  89. #~ print res.dump()
  90. testnum = i+1
  91. if res.testNumber != "":
  92. if testnum != int(res.testNumber):
  93. print("ERROR! test %(testNumber)s out of sequence" % res)
  94. testnum = int(res.testNumber)
  95. res["testNumber"] = testnum
  96. test = TAPTest(res)
  97. if test.passed:
  98. self.passedTests.append(test)
  99. else:
  100. self.failedTests.append(test)
  101. if test.skipped: self.skippedTests.append(test)
  102. if test.todo: self.todoTests.append(test)
  103. if test.todo and test.passed: self.bonusTests.append(test)
  104. self.passedSuite = not self.bail and (set(self.failedTests)-set(self.todoTests) == set())
  105. def summary(self, showPassed=False, showAll=False):
  106. testListStr = lambda tl : "[" + ",".join(str(t.num) for t in tl) + "]"
  107. summaryText = []
  108. if showPassed or showAll:
  109. summaryText.append( "PASSED: %s" % testListStr(self.passedTests) )
  110. if self.failedTests or showAll:
  111. summaryText.append( "FAILED: %s" % testListStr(self.failedTests) )
  112. if self.skippedTests or showAll:
  113. summaryText.append( "SKIPPED: %s" % testListStr(self.skippedTests) )
  114. if self.todoTests or showAll:
  115. summaryText.append( "TODO: %s" % testListStr(self.todoTests) )
  116. if self.bonusTests or showAll:
  117. summaryText.append( "BONUS: %s" % testListStr(self.bonusTests) )
  118. if self.passedSuite:
  119. summaryText.append( "PASSED" )
  120. else:
  121. summaryText.append( "FAILED" )
  122. return "\n".join(summaryText)
  123. # create TAPSummary objects from tapOutput parsed results, by setting
  124. # class as parse action
  125. tapOutputParser.setParseAction(TAPSummary)
  126. if __name__ == "__main__":
  127. test1 = """\
  128. 1..4
  129. ok 1 - Input file opened
  130. not ok 2 - First line of the input valid
  131. ok 3 - Read the rest of the file
  132. not ok 4 - Summarized correctly # TODO Not written yet
  133. """
  134. test2 = """\
  135. ok 1
  136. not ok 2 some description # TODO with a directive
  137. ok 3 a description only, no directive
  138. ok 4 # TODO directive only
  139. ok a description only, no directive
  140. ok # Skipped only a directive, no description
  141. ok
  142. """
  143. test3 = """\
  144. ok - created Board
  145. ok
  146. ok
  147. not ok
  148. ok
  149. ok
  150. ok
  151. ok
  152. # +------+------+------+------+
  153. # | |16G | |05C |
  154. # | |G N C | |C C G |
  155. # | | G | | C +|
  156. # +------+------+------+------+
  157. # |10C |01G | |03C |
  158. # |R N G |G A G | |C C C |
  159. # | R | G | | C +|
  160. # +------+------+------+------+
  161. # | |01G |17C |00C |
  162. # | |G A G |G N R |R N R |
  163. # | | G | R | G |
  164. # +------+------+------+------+
  165. ok - board has 7 tiles + starter tile
  166. 1..9
  167. """
  168. test4 = """\
  169. 1..4
  170. ok 1 - Creating test program
  171. ok 2 - Test program runs, no error
  172. not ok 3 - infinite loop # TODO halting problem unsolved
  173. not ok 4 - infinite loop 2 # TODO halting problem unsolved
  174. """
  175. test5 = """\
  176. 1..20
  177. ok - database handle
  178. not ok - failed database login
  179. Bail out! Couldn't connect to database.
  180. """
  181. test6 = """\
  182. ok 1 - retrieving servers from the database
  183. # need to ping 6 servers
  184. ok 2 - pinged diamond
  185. ok 3 - pinged ruby
  186. not ok 4 - pinged sapphire
  187. ok 5 - pinged onyx
  188. not ok 6 - pinged quartz
  189. ok 7 - pinged gold
  190. 1..7
  191. """
  192. for test in (test1,test2,test3,test4,test5,test6):
  193. print(test)
  194. tapResult = tapOutputParser.parseString(test)[0]
  195. print(tapResult.summary(showAll=True))
  196. print()