httpServerLogParser.py 3.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172
  1. # httpServerLogParser.py
  2. #
  3. # Copyright (c) 2016, Paul McGuire
  4. #
  5. """
  6. Parser for HTTP server log output, of the form:
  7. 195.146.134.15 - - [20/Jan/2003:08:55:36 -0800]
  8. "GET /path/to/page.html HTTP/1.0" 200 4649 "http://www.somedomain.com/020602/page.html"
  9. "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"
  10. 127.0.0.1 - u.surname@domain.com [12/Sep/2006:14:13:53 +0300]
  11. "GET /skins/monobook/external.png HTTP/1.0" 304 - "http://wiki.mysite.com/skins/monobook/main.css"
  12. "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.6) Gecko/20060728 Firefox/1.5.0.6"
  13. You can then break it up as follows:
  14. IP ADDRESS - -
  15. Server Date / Time [SPACE]
  16. "GET /path/to/page
  17. HTTP/Type Request"
  18. Success Code
  19. Bytes Sent To Client
  20. Referer
  21. Client Software
  22. """
  23. from pyparsing import alphas,nums, dblQuotedString, Combine, Word, Group, delimitedList, Suppress, removeQuotes
  24. import string
  25. def getCmdFields( s, l, t ):
  26. t["method"],t["requestURI"],t["protocolVersion"] = t[0].strip('"').split()
  27. logLineBNF = None
  28. def getLogLineBNF():
  29. global logLineBNF
  30. if logLineBNF is None:
  31. integer = Word( nums )
  32. ipAddress = delimitedList( integer, ".", combine=True )
  33. timeZoneOffset = Word("+-",nums)
  34. month = Word(string.ascii_uppercase, string.ascii_lowercase, exact=3)
  35. serverDateTime = Group( Suppress("[") +
  36. Combine( integer + "/" + month + "/" + integer +
  37. ":" + integer + ":" + integer + ":" + integer ) +
  38. timeZoneOffset +
  39. Suppress("]") )
  40. logLineBNF = ( ipAddress.setResultsName("ipAddr") +
  41. Suppress("-") +
  42. ("-" | Word( alphas+nums+"@._" )).setResultsName("auth") +
  43. serverDateTime.setResultsName("timestamp") +
  44. dblQuotedString.setResultsName("cmd").setParseAction(getCmdFields) +
  45. (integer | "-").setResultsName("statusCode") +
  46. (integer | "-").setResultsName("numBytesSent") +
  47. dblQuotedString.setResultsName("referrer").setParseAction(removeQuotes) +
  48. dblQuotedString.setResultsName("clientSfw").setParseAction(removeQuotes) )
  49. return logLineBNF
  50. testdata = """
  51. 195.146.134.15 - - [20/Jan/2003:08:55:36 -0800] "GET /path/to/page.html HTTP/1.0" 200 4649 "http://www.somedomain.com/020602/page.html" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"
  52. 111.111.111.11 - - [16/Feb/2004:04:09:49 -0800] "GET /ads/redirectads/336x280redirect.htm HTTP/1.1" 304 - "http://www.foobarp.org/theme_detail.php?type=vs&cat=0&mid=27512" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"
  53. 11.111.11.111 - - [16/Feb/2004:10:35:12 -0800] "GET /ads/redirectads/468x60redirect.htm HTTP/1.1" 200 541 "http://11.11.111.11/adframe.php?n=ad1f311a&what=zone:56" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.20 [ru\"]"
  54. 127.0.0.1 - u.surname@domain.com [12/Sep/2006:14:13:53 +0300] "GET /skins/monobook/external.png HTTP/1.0" 304 - "http://wiki.mysite.com/skins/monobook/main.css" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.6) Gecko/20060728 Firefox/1.5.0.6"
  55. """
  56. for line in testdata.split("\n"):
  57. if not line: continue
  58. fields = getLogLineBNF().parseString(line)
  59. print(fields.dump())
  60. #~ print repr(fields)
  61. #~ for k in fields.keys():
  62. #~ print "fields." + k + " =", fields[k]
  63. print()