xpathgrep.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. #!/usr/bin/env python
  2. import sys
  3. import os.path
  4. def error(message, *args):
  5. if args:
  6. message = message % args
  7. sys.stderr.write('ERROR: %s\n' % message)
  8. try:
  9. import lxml.etree as et
  10. except ImportError:
  11. error(sys.exc_info()[1])
  12. sys.exit(5)
  13. try:
  14. basestring
  15. except NameError:
  16. basestring = (str, bytes)
  17. try:
  18. unicode
  19. except NameError:
  20. unicode = str
  21. SHORT_DESCRIPTION = "An XPath file finder for XML files."
  22. __doc__ = SHORT_DESCRIPTION + '''
  23. Evaluates an XPath expression against a series of files and prints the
  24. matching subtrees to stdout.
  25. Examples::
  26. $ cat test.xml
  27. <root>
  28. <a num="1234" notnum="1234abc"/>
  29. <b text="abc"/>
  30. <c text="aBc"/>
  31. <d xmlns="http://www.example.org/ns/example" num="2"/>
  32. <d xmlns="http://www.example.org/ns/example" num="4"/>
  33. </root>
  34. # find all leaf elements:
  35. $ SCRIPT '//*[not(*)]' test.xml
  36. <a num="1234" notnum="1234abc"/>
  37. <b text="abc"/>
  38. <c text="aBc"/>
  39. # find all elements with attribute values containing "abc" ignoring case:
  40. $ SCRIPT '//*[@*[contains(py:lower(.), "abc")]]' test.xml
  41. <a num="1234" notnum="1234abc"/>
  42. <b text="abc"/>
  43. <c text="aBc"/>
  44. # find all numeric attribute values:
  45. $ SCRIPT '//@*[re:match(., "^[0-9]+$")]' test.xml
  46. 1234
  47. * find all elements with numeric attribute values:
  48. $ SCRIPT '//*[@*[re:match(., "^[0-9]+$")]]' test.xml
  49. <a num="1234" notnum="1234abc"/>
  50. * find all elements with numeric attribute values in more than one file:
  51. $ SCRIPT '//*[@*[re:match(., "^[0-9]+$")]]' test.xml test.xml test.xml
  52. >> test.xml
  53. <a num="1234" notnum="1234abc"/>
  54. >> test.xml
  55. <a num="1234" notnum="1234abc"/>
  56. >> test.xml
  57. <a num="1234" notnum="1234abc"/>
  58. * find XML files that have non-empty root nodes:
  59. $ SCRIPT -q '*' test.xml test.xml test.xml
  60. >> test.xml
  61. >> test.xml
  62. >> test.xml
  63. * find out if an XML file has at most depth three:
  64. $ SCRIPT 'not(/*/*/*)' test.xml
  65. True
  66. * find all elements that belong to a specific namespace and have @num=2
  67. $ SCRIPT --ns e=http://www.example.org/ns/example '//e:*[@num="2"]' test.xml
  68. <d xmlns="http://www.example.org/ns/example" num="2"/>
  69. By default, all Python builtins and string methods are available as
  70. XPath functions through the ``py`` prefix. There is also a string
  71. comparison function ``py:within(x, a, b)`` that tests the string x for
  72. being lexicographically within the interval ``a <= x <= b``.
  73. '''.replace('SCRIPT', os.path.basename(sys.argv[0]))
  74. REGEXP_NS = "http://exslt.org/regular-expressions"
  75. PYTHON_BUILTINS_NS = "PYTHON-BUILTINS"
  76. def make_parser(remove_blank_text=True, **kwargs):
  77. return et.XMLParser(remove_blank_text=remove_blank_text, **kwargs)
  78. def print_result(result, pretty_print, encoding=None, _is_py3=sys.version_info[0] >= 3):
  79. stdout = sys.stdout
  80. if not stdout.isatty() and not encoding:
  81. encoding = 'utf8'
  82. if et.iselement(result):
  83. result = et.tostring(result, xml_declaration=False, with_tail=False,
  84. pretty_print=pretty_print, encoding=encoding)
  85. if not pretty_print:
  86. # pretty printing appends newline, otherwise we do it
  87. if isinstance(result, unicode):
  88. result += '\n'
  89. else:
  90. result += '\n'.encode('ascii')
  91. elif isinstance(result, basestring):
  92. result += '\n'
  93. else:
  94. result = '%r\n' % result # '%r' for better number formatting
  95. if encoding and encoding != 'unicode' and isinstance(result, unicode):
  96. result = result.encode(encoding)
  97. if _is_py3 and not isinstance(result, unicode):
  98. stdout.buffer.write(result)
  99. else:
  100. stdout.write(result)
  101. def print_results(results, pretty_print):
  102. if isinstance(results, list):
  103. for result in results:
  104. print_result(result, pretty_print)
  105. else:
  106. print_result(results, pretty_print)
  107. def iter_input(input, filename, parser, line_by_line):
  108. if isinstance(input, basestring):
  109. with open(input, 'rb') as f:
  110. for tree in iter_input(f, filename, parser, line_by_line):
  111. yield tree
  112. else:
  113. try:
  114. if line_by_line:
  115. for line in input:
  116. if line:
  117. yield et.ElementTree(et.fromstring(line, parser))
  118. else:
  119. yield et.parse(input, parser)
  120. except IOError:
  121. e = sys.exc_info()[1]
  122. error("parsing %r failed: %s: %s",
  123. filename, e.__class__.__name__, e)
  124. def find_in_file(f, xpath, print_name=True, xinclude=False, pretty_print=True, line_by_line=False,
  125. encoding=None, verbose=True):
  126. try:
  127. filename = f.name
  128. except AttributeError:
  129. filename = f
  130. xml_parser = et.XMLParser(encoding=encoding)
  131. try:
  132. if not callable(xpath):
  133. xpath = et.XPath(xpath)
  134. found = False
  135. for tree in iter_input(f, filename, xml_parser, line_by_line):
  136. try:
  137. if xinclude:
  138. tree.xinclude()
  139. except IOError:
  140. e = sys.exc_info()[1]
  141. error("XInclude for %r failed: %s: %s",
  142. filename, e.__class__.__name__, e)
  143. results = xpath(tree)
  144. if results is not None and results != []:
  145. found = True
  146. if verbose:
  147. print_results(results, pretty_print)
  148. if not found:
  149. return False
  150. if not verbose and print_name:
  151. print(filename)
  152. return True
  153. except Exception:
  154. e = sys.exc_info()[1]
  155. error("%r: %s: %s",
  156. filename, e.__class__.__name__, e)
  157. return False
  158. def register_builtins():
  159. ns = et.FunctionNamespace(PYTHON_BUILTINS_NS)
  160. tostring = et.tostring
  161. def make_string(s):
  162. if isinstance(s, list):
  163. if not s:
  164. return ''
  165. s = s[0]
  166. if not isinstance(s, unicode):
  167. if et.iselement(s):
  168. s = tostring(s, method="text", encoding='unicode')
  169. else:
  170. s = unicode(s)
  171. return s
  172. def wrap_builtin(b):
  173. def wrapped_builtin(_, *args):
  174. return b(*args)
  175. return wrapped_builtin
  176. for (name, builtin) in vars(__builtins__).items():
  177. if callable(builtin):
  178. if not name.startswith('_') and name == name.lower():
  179. ns[name] = wrap_builtin(builtin)
  180. def wrap_str_method(b):
  181. def wrapped_method(_, *args):
  182. args = tuple(map(make_string, args))
  183. return b(*args)
  184. return wrapped_method
  185. for (name, method) in vars(unicode).items():
  186. if callable(method):
  187. if not name.startswith('_'):
  188. ns[name] = wrap_str_method(method)
  189. def within(_, s, a, b):
  190. return make_string(a) <= make_string(s) <= make_string(b)
  191. ns["within"] = within
  192. def parse_options():
  193. from optparse import OptionParser
  194. usage = "usage: %prog [options] XPATH [FILE ...]"
  195. parser = OptionParser(
  196. usage = usage,
  197. version = "%prog using lxml.etree " + et.__version__,
  198. description = SHORT_DESCRIPTION)
  199. parser.add_option("-H", "--long-help",
  200. action="store_true", dest="long_help", default=False,
  201. help="a longer help text including usage examples")
  202. parser.add_option("-i", "--xinclude",
  203. action="store_true", dest="xinclude", default=False,
  204. help="run XInclude on the file before XPath")
  205. parser.add_option("--no-python",
  206. action="store_false", dest="python", default=True,
  207. help="disable Python builtins and functions (prefix 'py')")
  208. parser.add_option("--no-regexp",
  209. action="store_false", dest="regexp", default=True,
  210. help="disable regular expressions (prefix 're')")
  211. parser.add_option("-q", "--quiet",
  212. action="store_false", dest="verbose", default=True,
  213. help="don't print status messages to stdout")
  214. parser.add_option("-t", "--root-tag",
  215. dest="root_tag", metavar="TAG",
  216. help="surround output with <TAG>...</TAG> to produce a well-formed XML document")
  217. parser.add_option("-p", "--plain",
  218. action="store_false", dest="pretty_print", default=True,
  219. help="do not pretty-print the output")
  220. parser.add_option("-l", "--lines",
  221. action="store_true", dest="line_by_line", default=False,
  222. help="parse each line of input separately (e.g. grep output)")
  223. parser.add_option("-e", "--encoding",
  224. dest="encoding",
  225. help="use a specific encoding for parsing (may be required with --lines)")
  226. parser.add_option("-N", "--ns", metavar="PREFIX=NS",
  227. action="append", dest="namespaces", default=[],
  228. help="add a namespace declaration")
  229. options, args = parser.parse_args()
  230. if options.long_help:
  231. parser.print_help()
  232. print(__doc__[__doc__.find('\n\n')+1:])
  233. sys.exit(0)
  234. if len(args) < 1:
  235. parser.error("first argument must be an XPath expression")
  236. return options, args
  237. def main(options, args):
  238. namespaces = {}
  239. if options.regexp:
  240. namespaces["re"] = REGEXP_NS
  241. if options.python:
  242. register_builtins()
  243. namespaces["py"] = PYTHON_BUILTINS_NS
  244. for ns in options.namespaces:
  245. prefix, NS = ns.split("=", 1)
  246. namespaces[prefix.strip()] = NS.strip()
  247. xpath = et.XPath(args[0], namespaces=namespaces)
  248. files = args[1:] or [sys.stdin]
  249. if options.root_tag and options.verbose:
  250. print('<%s>' % options.root_tag)
  251. found = False
  252. print_name = len(files) > 1 and not options.root_tag
  253. for input in files:
  254. found |= find_in_file(
  255. input, xpath,
  256. print_name=print_name,
  257. xinclude=options.xinclude,
  258. pretty_print=options.pretty_print,
  259. line_by_line=options.line_by_line,
  260. encoding=options.encoding,
  261. verbose=options.verbose,
  262. )
  263. if options.root_tag and options.verbose:
  264. print('</%s>' % options.root_tag)
  265. return found
  266. if __name__ == "__main__":
  267. try:
  268. options, args = parse_options()
  269. found = main(options, args)
  270. if found:
  271. sys.exit(0)
  272. else:
  273. sys.exit(1)
  274. except et.XPathSyntaxError:
  275. error(sys.exc_info()[1])
  276. sys.exit(4)
  277. except KeyboardInterrupt:
  278. pass