sexpParser.py 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157
  1. # sexpParser.py
  2. #
  3. # Demonstration of the pyparsing module, implementing a simple S-expression
  4. # parser.
  5. #
  6. # Updates:
  7. # November, 2011 - fixed errors in precedence of alternatives in simpleString;
  8. # fixed exception raised in verifyLen to properly signal the input string
  9. # and exception location so that markInputline works correctly; fixed
  10. # definition of decimal to accept a single '0' and optional leading '-'
  11. # sign; updated tests to improve parser coverage
  12. #
  13. # Copyright 2007-2011, by Paul McGuire
  14. #
  15. """
  16. BNF reference: http://theory.lcs.mit.edu/~rivest/sexp.txt
  17. <sexp> :: <string> | <list>
  18. <string> :: <display>? <simple-string> ;
  19. <simple-string> :: <raw> | <token> | <base-64> | <hexadecimal> |
  20. <quoted-string> ;
  21. <display> :: "[" <simple-string> "]" ;
  22. <raw> :: <decimal> ":" <bytes> ;
  23. <decimal> :: <decimal-digit>+ ;
  24. -- decimal numbers should have no unnecessary leading zeros
  25. <bytes> -- any string of bytes, of the indicated length
  26. <token> :: <tokenchar>+ ;
  27. <base-64> :: <decimal>? "|" ( <base-64-char> | <whitespace> )* "|" ;
  28. <hexadecimal> :: "#" ( <hex-digit> | <white-space> )* "#" ;
  29. <quoted-string> :: <decimal>? <quoted-string-body>
  30. <quoted-string-body> :: "\"" <bytes> "\""
  31. <list> :: "(" ( <sexp> | <whitespace> )* ")" ;
  32. <whitespace> :: <whitespace-char>* ;
  33. <token-char> :: <alpha> | <decimal-digit> | <simple-punc> ;
  34. <alpha> :: <upper-case> | <lower-case> | <digit> ;
  35. <lower-case> :: "a" | ... | "z" ;
  36. <upper-case> :: "A" | ... | "Z" ;
  37. <decimal-digit> :: "0" | ... | "9" ;
  38. <hex-digit> :: <decimal-digit> | "A" | ... | "F" | "a" | ... | "f" ;
  39. <simple-punc> :: "-" | "." | "/" | "_" | ":" | "*" | "+" | "=" ;
  40. <whitespace-char> :: " " | "\t" | "\r" | "\n" ;
  41. <base-64-char> :: <alpha> | <decimal-digit> | "+" | "/" | "=" ;
  42. <null> :: "" ;
  43. """
  44. import pyparsing as pp
  45. from base64 import b64decode
  46. import pprint
  47. def verify_length(s, l, t):
  48. t = t[0]
  49. if t.len is not None:
  50. t1len = len(t[1])
  51. if t1len != t.len:
  52. raise pp.ParseFatalException(s, l, "invalid data of length {0}, expected {1}".format(t1len, t.len))
  53. return t[1]
  54. # define punctuation literals
  55. LPAR, RPAR, LBRK, RBRK, LBRC, RBRC, VBAR, COLON = (pp.Suppress(c).setName(c) for c in "()[]{}|:")
  56. decimal = pp.Regex(r'-?0|[1-9]\d*').setParseAction(lambda t: int(t[0]))
  57. hexadecimal = ("#" + pp.Word(pp.hexnums)[1, ...] + "#").setParseAction(lambda t: int("".join(t[1:-1]), 16))
  58. bytes = pp.Word(pp.printables)
  59. raw = pp.Group(decimal("len") + COLON + bytes).setParseAction(verify_length)
  60. base64_ = pp.Group(pp.Optional(decimal | hexadecimal, default=None)("len")
  61. + VBAR
  62. + pp.Word(pp.alphanums + "+/=")[1, ...].setParseAction(lambda t: b64decode("".join(t)))
  63. + VBAR
  64. ).setParseAction(verify_length)
  65. real = pp.Regex(r"[+-]?\d+\.\d*([eE][+-]?\d+)?").setParseAction(lambda tokens: float(tokens[0]))
  66. token = pp.Word(pp.alphanums + "-./_:*+=!<>")
  67. qString = pp.Group(pp.Optional(decimal, default=None)("len")
  68. + pp.dblQuotedString.setParseAction(pp.removeQuotes)
  69. ).setParseAction(verify_length)
  70. simpleString = real | base64_ | raw | decimal | token | hexadecimal | qString
  71. display = LBRK + simpleString + RBRK
  72. string_ = pp.Optional(display) + simpleString
  73. sexp = pp.Forward()
  74. sexpList = pp.Group(LPAR + sexp[...] + RPAR)
  75. sexp <<= string_ | sexpList
  76. # Test data
  77. test00 = """(snicker "abc" (#03# |YWJj|))"""
  78. test01 = """(certificate
  79. (issuer
  80. (name
  81. (public-key
  82. rsa-with-md5
  83. (e 15 |NFGq/E3wh9f4rJIQVXhS|)
  84. (n |d738/4ghP9rFZ0gAIYZ5q9y6iskDJwASi5rEQpEQq8ZyMZeIZzIAR2I5iGE=|))
  85. aid-committee))
  86. (subject
  87. (ref
  88. (public-key
  89. rsa-with-md5
  90. (e |NFGq/E3wh9f4rJIQVXhS|)
  91. (n |d738/4ghP9rFZ0gAIYZ5q9y6iskDJwASi5rEQpEQq8ZyMZeIZzIAR2I5iGE=|))
  92. tom
  93. mother))
  94. (not-before "1997-01-01_09:00:00")
  95. (not-after "1998-01-01_09:00:00")
  96. (tag
  97. (spend (account "12345678") (* numeric range "1" "1000"))))
  98. """
  99. test02 = """(lambda (x) (* x x))"""
  100. test03 = """(def length
  101. (lambda (x)
  102. (cond
  103. ((not x) 0)
  104. ( t (+ 1 (length (cdr x))))
  105. )
  106. )
  107. )
  108. """
  109. test04 = """(2:XX "abc" (#03# |YWJj|))"""
  110. test05 = """(if (is (window_name) "XMMS") (set_workspace 2))"""
  111. test06 = """(if
  112. (and
  113. (is (application_name) "Firefox")
  114. (or
  115. (contains (window_name) "Enter name of file to save to")
  116. (contains (window_name) "Save As")
  117. (contains (window_name) "Save Image")
  118. ()
  119. )
  120. )
  121. (geometry "+140+122")
  122. )
  123. """
  124. test07 = """(defun factorial (x)
  125. (if (zerop x) 1
  126. (* x (factorial (- x 1)))))
  127. """
  128. test51 = """(2:XX "abc" (#03# |YWJj|))"""
  129. test51error = """(3:XX "abc" (#03# |YWJj|))"""
  130. test52 = """
  131. (and
  132. (or (> uid 1000)
  133. (!= gid 20)
  134. )
  135. (> quota 5.0e+03)
  136. )
  137. """
  138. # Run tests
  139. alltests = [globals()[testname] for testname in sorted(locals()) if testname.startswith("test")]
  140. sexp.runTests(alltests, fullDump=False)