sparser.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363
  1. #!/usr/bin/env python
  2. """
  3. NAME:
  4. sparser.py
  5. SYNOPSIS:
  6. sparser.py [options] filename
  7. DESCRIPTION:
  8. The sparser.py script is a Specified PARSER. It is unique (as far as I can
  9. tell) because it doesn't care about the delimiter(s). The user specifies
  10. what is expected, and the order, for each line of text. All of the heavy
  11. lifting is handled by pyparsing (http://pyparsing.sf.net).
  12. OPTIONS:
  13. -h,--help this message
  14. -v,--version version
  15. -d,--debug turn on debug messages
  16. EXAMPLES:
  17. 1. As standalone
  18. sparser.py myfile
  19. 2. As library
  20. import sparser
  21. ...
  22. #Copyright (C) 2006 Tim Cera timcera@earthlink.net
  23. #
  24. #
  25. # This program is free software; you can redistribute it and/or modify it
  26. # under the terms of the GNU General Public License as published by the Free
  27. # Software Foundation; either version 2 of the License, or (at your option)
  28. # any later version.
  29. #
  30. # This program is distributed in the hope that it will be useful, but
  31. # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  32. # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
  33. # for more details.
  34. #
  35. # You should have received a copy of the GNU General Public License along
  36. # with this program; if not, write to the Free Software Foundation, Inc.,
  37. # 675 Mass Ave, Cambridge, MA 02139, USA.
  38. """
  39. #===imports======================
  40. import sys
  41. import os
  42. import getopt
  43. from pyparsing import *
  44. #===globals======================
  45. modname = "sparser"
  46. __version__ = "0.1"
  47. #--option args--
  48. debug_p = 0
  49. #opt_b=None #string arg, default is undefined
  50. #---positional args, default is empty---
  51. pargs = []
  52. #---other---
  53. #===utilities====================
  54. def msg(txt):
  55. """Send message to stdout."""
  56. sys.stdout.write(txt)
  57. sys.stdout.flush()
  58. def debug(ftn, txt):
  59. """Used for debugging."""
  60. if debug_p:
  61. sys.stdout.write("{0}.{1}:{2}\n".format(modname, ftn, txt))
  62. sys.stdout.flush()
  63. def fatal(ftn, txt):
  64. """If can't continue."""
  65. msg = "{0}.{1}:FATAL:{2}\n".format(modname, ftn, txt)
  66. raise SystemExit(msg)
  67. def usage():
  68. """Prints the docstring."""
  69. print(__doc__)
  70. #====================================
  71. class ToInteger(TokenConverter):
  72. """Converter to make token into an integer."""
  73. def postParse( self, instring, loc, tokenlist ):
  74. return int(tokenlist[0])
  75. class ToFloat(TokenConverter):
  76. """Converter to make token into a float."""
  77. def postParse( self, instring, loc, tokenlist ):
  78. return float(tokenlist[0])
  79. class ParseFileLineByLine:
  80. """
  81. Bring data from text files into a program, optionally parsing each line
  82. according to specifications in a parse definition file.
  83. ParseFileLineByLine instances can be used like normal file objects (i.e. by
  84. calling readline(), readlines(), and write()), but can also be used as
  85. sequences of lines in for-loops.
  86. ParseFileLineByLine objects also handle compression transparently. i.e. it
  87. is possible to read lines from a compressed text file as if it were not
  88. compressed. Compression is deduced from the file name suffixes '.Z'
  89. (compress/uncompress), '.gz' (gzip/gunzip), and '.bz2' (bzip2).
  90. The parse definition fi le name is developed based on the input file name.
  91. If the input file name is 'basename.ext', then the definition file is
  92. 'basename_def.ext'. If a definition file specific to the input file is not
  93. found, then the program searches for the file 'sparse.def' which would be
  94. the definition file for all files in that directory without a file specific
  95. definition file.
  96. Finally, ParseFileLineByLine objects accept file names that start with '~'
  97. or '~user' to indicate a home directory, as well as URLs (for reading
  98. only).
  99. Constructor:
  100. ParseFileLineByLine(|filename|, |mode|='"r"'), where |filename| is the name
  101. of the file (or a URL) and |mode| is one of '"r"' (read), '"w"' (write) or
  102. '"a"' (append, not supported for .Z files).
  103. """
  104. def __init__(self, filename, mode = 'r'):
  105. """Opens input file, and if available the definition file. If the
  106. definition file is available __init__ will then create some pyparsing
  107. helper variables. """
  108. if mode not in ['r', 'w', 'a']:
  109. raise IOError(0, 'Illegal mode: ' + repr(mode))
  110. if string.find(filename, ':/') > 1: # URL
  111. if mode == 'w':
  112. raise IOError("can't write to a URL")
  113. import urllib.request, urllib.parse, urllib.error
  114. self.file = urllib.request.urlopen(filename)
  115. else:
  116. filename = os.path.expanduser(filename)
  117. if mode == 'r' or mode == 'a':
  118. if not os.path.exists(filename):
  119. raise IOError(2, 'No such file or directory: ' + filename)
  120. filen, file_extension = os.path.splitext(filename)
  121. command_dict = {
  122. ('.Z', 'r'):
  123. "self.file = os.popen('uncompress -c ' + filename, mode)",
  124. ('.gz', 'r'):
  125. "self.file = gzip.GzipFile(filename, 'rb')",
  126. ('.bz2', 'r'):
  127. "self.file = os.popen('bzip2 -dc ' + filename, mode)",
  128. ('.Z', 'w'):
  129. "self.file = os.popen('compress > ' + filename, mode)",
  130. ('.gz', 'w'):
  131. "self.file = gzip.GzipFile(filename, 'wb')",
  132. ('.bz2', 'w'):
  133. "self.file = os.popen('bzip2 > ' + filename, mode)",
  134. ('.Z', 'a'):
  135. "raise IOError, (0, 'Can\'t append to .Z files')",
  136. ('.gz', 'a'):
  137. "self.file = gzip.GzipFile(filename, 'ab')",
  138. ('.bz2', 'a'):
  139. "raise IOError, (0, 'Can\'t append to .bz2 files')",
  140. }
  141. exec(command_dict.get((file_extension, mode),
  142. 'self.file = open(filename, mode)'))
  143. self.grammar = None
  144. # Try to find a parse ('*_def.ext') definition file. First try to find
  145. # a file specific parse definition file, then look for 'sparse.def'
  146. # that would be the definition file for all files within the directory.
  147. # The definition file is pure Python. The one variable that needs to
  148. # be specified is 'parse'. The 'parse' variable is a list of tuples
  149. # defining the name, type, and because it is a list, the order of
  150. # variables on each line in the data file. The variable name is a
  151. # string, the type variable is defined as integer, real, and qString.
  152. # parse = [
  153. # ('year', integer),
  154. # ('month', integer),
  155. # ('day', integer),
  156. # ('value', real),
  157. # ]
  158. definition_file_one = filen + "_def" + file_extension
  159. definition_file_two = os.path.dirname(filen) + os.sep + "sparse.def"
  160. if os.path.exists(definition_file_one):
  161. self.parsedef = definition_file_one
  162. elif os.path.exists(definition_file_two):
  163. self.parsedef = definition_file_two
  164. else:
  165. self.parsedef = None
  166. return None
  167. # Create some handy pyparsing constructs. I kept 'decimal_sep' so that
  168. # could easily change to parse if the decimal separator is a ",".
  169. decimal_sep = "."
  170. sign = oneOf("+ -")
  171. # part of printables without decimal_sep, +, -
  172. special_chars = string.replace('!"#$%&\'()*,./:;<=>?@[\\]^_`{|}~',
  173. decimal_sep, "")
  174. integer = ToInteger(
  175. Combine(Optional(sign) +
  176. Word(nums))).setName("integer")
  177. positive_integer = ToInteger(
  178. Combine(Optional("+") +
  179. Word(nums))).setName("integer")
  180. negative_integer = ToInteger(
  181. Combine("-" +
  182. Word(nums))).setName("integer")
  183. real = ToFloat(
  184. Combine(Optional(sign) +
  185. Word(nums) +
  186. decimal_sep +
  187. Optional(Word(nums)) +
  188. Optional(oneOf("E e") +
  189. Word(nums)))).setName("real")
  190. positive_real = ToFloat(
  191. Combine(Optional("+") +
  192. Word(nums) +
  193. decimal_sep +
  194. Optional(Word(nums)) +
  195. Optional(oneOf("E e") +
  196. Word(nums)))).setName("real")
  197. negative_real = ToFloat(
  198. Combine("-" +
  199. Word(nums) +
  200. decimal_sep +
  201. Optional(Word(nums)) +
  202. Optional(oneOf("E e") +
  203. Word(nums)))).setName("real")
  204. qString = ( sglQuotedString | dblQuotedString ).setName("qString")
  205. # add other characters we should skip over between interesting fields
  206. integer_junk = Optional(
  207. Suppress(
  208. Word(alphas +
  209. special_chars +
  210. decimal_sep))).setName("integer_junk")
  211. real_junk = Optional(
  212. Suppress(
  213. Word(alphas +
  214. special_chars))).setName("real_junk")
  215. qString_junk = SkipTo(qString).setName("qString_junk")
  216. # Now that 'integer', 'real', and 'qString' have been assigned I can
  217. # execute the definition file.
  218. exec(compile(open(self.parsedef).read(), self.parsedef, 'exec'))
  219. # Build the grammar, combination of the 'integer', 'real, 'qString',
  220. # and '*_junk' variables assigned above in the order specified in the
  221. # definition file.
  222. grammar = []
  223. for nam, expr in parse:
  224. grammar.append( eval(expr.name + "_junk"))
  225. grammar.append( expr.setResultsName(nam) )
  226. self.grammar = And( grammar[1:] + [restOfLine] )
  227. def __del__(self):
  228. """Delete (close) the file wrapper."""
  229. self.close()
  230. def __getitem__(self, item):
  231. """Used in 'for line in fp:' idiom."""
  232. line = self.readline()
  233. if not line:
  234. raise IndexError
  235. return line
  236. def readline(self):
  237. """Reads (and optionally parses) a single line."""
  238. line = self.file.readline()
  239. if self.grammar and line:
  240. try:
  241. return self.grammar.parseString(line).asDict()
  242. except ParseException:
  243. return self.readline()
  244. else:
  245. return line
  246. def readlines(self):
  247. """Returns a list of all lines (optionally parsed) in the file."""
  248. if self.grammar:
  249. tot = []
  250. # Used this way instead of a 'for' loop against
  251. # self.file.readlines() so that there wasn't two copies of the file
  252. # in memory.
  253. while 1:
  254. line = self.file.readline()
  255. if not line:
  256. break
  257. tot.append(line)
  258. return tot
  259. return self.file.readlines()
  260. def write(self, data):
  261. """Write to a file."""
  262. self.file.write(data)
  263. def writelines(self, list):
  264. """Write a list to a file. Each item in the list is a line in the
  265. file.
  266. """
  267. for line in list:
  268. self.file.write(line)
  269. def close(self):
  270. """Close the file."""
  271. self.file.close()
  272. def flush(self):
  273. """Flush in memory contents to file."""
  274. self.file.flush()
  275. #=============================
  276. def main(pargs):
  277. """This should only be used for testing. The primary mode of operation is
  278. as an imported library.
  279. """
  280. input_file = sys.argv[1]
  281. fp = ParseFileLineByLine(input_file)
  282. for i in fp:
  283. print(i)
  284. #-------------------------
  285. if __name__ == '__main__':
  286. ftn = "main"
  287. opts, pargs = getopt.getopt(sys.argv[1:], 'hvd',
  288. ['help', 'version', 'debug', 'bb='])
  289. for opt in opts:
  290. if opt[0] == '-h' or opt[0] == '--help':
  291. print(modname+": version="+__version__)
  292. usage()
  293. sys.exit(0)
  294. elif opt[0] == '-v' or opt[0] == '--version':
  295. print(modname+": version="+__version__)
  296. sys.exit(0)
  297. elif opt[0] == '-d' or opt[0] == '--debug':
  298. debug_p = 1
  299. elif opt[0] == '--bb':
  300. opt_b = opt[1]
  301. #---make the object and run it---
  302. main(pargs)
  303. #===Revision Log===
  304. #Created by mkpythonproj:
  305. #2006-02-06 Tim Cera
  306. #