util.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306
  1. # util.py
  2. # Copyright (C) 2006, 2007, 2008, 2009, 2010 Michael Bayer mike_mp@zzzcomputing.com
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. import sys
  7. py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0)
  8. py24 = sys.version_info >= (2, 4) and sys.version_info < (2, 5)
  9. jython = sys.platform.startswith('java')
  10. win32 = sys.platform.startswith('win')
  11. if py3k:
  12. from io import StringIO
  13. else:
  14. try:
  15. from cStringIO import StringIO
  16. except:
  17. from StringIO import StringIO
  18. import codecs, re, weakref, os, time, operator
  19. try:
  20. import threading
  21. import thread
  22. except ImportError:
  23. import dummy_threading as threading
  24. import dummy_thread as thread
  25. if win32 or jython:
  26. time_func = time.clock
  27. else:
  28. time_func = time.time
  29. def function_named(fn, name):
  30. """Return a function with a given __name__.
  31. Will assign to __name__ and return the original function if possible on
  32. the Python implementation, otherwise a new function will be constructed.
  33. """
  34. fn.__name__ = name
  35. return fn
  36. if py24:
  37. def exception_name(exc):
  38. try:
  39. return exc.__class__.__name__
  40. except AttributeError:
  41. return exc.__name__
  42. else:
  43. def exception_name(exc):
  44. return exc.__class__.__name__
  45. def verify_directory(dir):
  46. """create and/or verify a filesystem directory."""
  47. tries = 0
  48. while not os.path.exists(dir):
  49. try:
  50. tries += 1
  51. os.makedirs(dir, 0775)
  52. except:
  53. if tries > 5:
  54. raise
  55. def to_list(x, default=None):
  56. if x is None:
  57. return default
  58. if not isinstance(x, (list, tuple)):
  59. return [x]
  60. else:
  61. return x
  62. class SetLikeDict(dict):
  63. """a dictionary that has some setlike methods on it"""
  64. def union(self, other):
  65. """produce a 'union' of this dict and another (at the key level).
  66. values in the second dict take precedence over that of the first"""
  67. x = SetLikeDict(**self)
  68. x.update(other)
  69. return x
  70. class FastEncodingBuffer(object):
  71. """a very rudimentary buffer that is faster than StringIO,
  72. but doesnt crash on unicode data like cStringIO."""
  73. def __init__(self, encoding=None, errors='strict', unicode=False):
  74. self.data = []
  75. self.encoding = encoding
  76. if unicode:
  77. self.delim = u''
  78. else:
  79. self.delim = ''
  80. self.unicode = unicode
  81. self.errors = errors
  82. self.write = self.data.append
  83. def truncate(self):
  84. self.data =[]
  85. def getvalue(self):
  86. if self.encoding:
  87. return self.delim.join(self.data).encode(self.encoding, self.errors)
  88. else:
  89. return self.delim.join(self.data)
  90. class LRUCache(dict):
  91. """A dictionary-like object that stores a limited number of items, discarding
  92. lesser used items periodically.
  93. this is a rewrite of LRUCache from Myghty to use a periodic timestamp-based
  94. paradigm so that synchronization is not really needed. the size management
  95. is inexact.
  96. """
  97. class _Item(object):
  98. def __init__(self, key, value):
  99. self.key = key
  100. self.value = value
  101. self.timestamp = time_func()
  102. def __repr__(self):
  103. return repr(self.value)
  104. def __init__(self, capacity, threshold=.5):
  105. self.capacity = capacity
  106. self.threshold = threshold
  107. def __getitem__(self, key):
  108. item = dict.__getitem__(self, key)
  109. item.timestamp = time_func()
  110. return item.value
  111. def values(self):
  112. return [i.value for i in dict.values(self)]
  113. def setdefault(self, key, value):
  114. if key in self:
  115. return self[key]
  116. else:
  117. self[key] = value
  118. return value
  119. def __setitem__(self, key, value):
  120. item = dict.get(self, key)
  121. if item is None:
  122. item = self._Item(key, value)
  123. dict.__setitem__(self, key, item)
  124. else:
  125. item.value = value
  126. self._manage_size()
  127. def _manage_size(self):
  128. while len(self) > self.capacity + self.capacity * self.threshold:
  129. bytime = sorted(dict.values(self),
  130. key=operator.attrgetter('timestamp'), reverse=True)
  131. for item in bytime[self.capacity:]:
  132. try:
  133. del self[item.key]
  134. except KeyError:
  135. # if we couldnt find a key, most likely some other thread broke in
  136. # on us. loop around and try again
  137. break
  138. # Regexp to match python magic encoding line
  139. _PYTHON_MAGIC_COMMENT_re = re.compile(
  140. r'[ \t\f]* \# .* coding[=:][ \t]*([-\w.]+)',
  141. re.VERBOSE)
  142. def parse_encoding(fp):
  143. """Deduce the encoding of a Python source file (binary mode) from magic comment.
  144. It does this in the same way as the `Python interpreter`__
  145. .. __: http://docs.python.org/ref/encodings.html
  146. The ``fp`` argument should be a seekable file object in binary mode.
  147. """
  148. pos = fp.tell()
  149. fp.seek(0)
  150. try:
  151. line1 = fp.readline()
  152. has_bom = line1.startswith(codecs.BOM_UTF8)
  153. if has_bom:
  154. line1 = line1[len(codecs.BOM_UTF8):]
  155. m = _PYTHON_MAGIC_COMMENT_re.match(line1.decode('ascii', 'ignore'))
  156. if not m:
  157. try:
  158. import parser
  159. parser.suite(line1.decode('ascii', 'ignore'))
  160. except (ImportError, SyntaxError):
  161. # Either it's a real syntax error, in which case the source
  162. # is not valid python source, or line2 is a continuation of
  163. # line1, in which case we don't want to scan line2 for a magic
  164. # comment.
  165. pass
  166. else:
  167. line2 = fp.readline()
  168. m = _PYTHON_MAGIC_COMMENT_re.match(line2.decode('ascii', 'ignore'))
  169. if has_bom:
  170. if m:
  171. raise SyntaxError, \
  172. "python refuses to compile code with both a UTF8" \
  173. " byte-order-mark and a magic encoding comment"
  174. return 'utf_8'
  175. elif m:
  176. return m.group(1)
  177. else:
  178. return None
  179. finally:
  180. fp.seek(pos)
  181. def sorted_dict_repr(d):
  182. """repr() a dictionary with the keys in order.
  183. Used by the lexer unit test to compare parse trees based on strings.
  184. """
  185. keys = d.keys()
  186. keys.sort()
  187. return "{" + ", ".join(["%r: %r" % (k, d[k]) for k in keys]) + "}"
  188. def restore__ast(_ast):
  189. """Attempt to restore the required classes to the _ast module if it
  190. appears to be missing them
  191. """
  192. if hasattr(_ast, 'AST'):
  193. return
  194. _ast.PyCF_ONLY_AST = 2 << 9
  195. m = compile("""\
  196. def foo(): pass
  197. class Bar(object): pass
  198. if False: pass
  199. baz = 'mako'
  200. 1 + 2 - 3 * 4 / 5
  201. 6 // 7 % 8 << 9 >> 10
  202. 11 & 12 ^ 13 | 14
  203. 15 and 16 or 17
  204. -baz + (not +18) - ~17
  205. baz and 'foo' or 'bar'
  206. (mako is baz == baz) is not baz != mako
  207. mako > baz < mako >= baz <= mako
  208. mako in baz not in mako""", '<unknown>', 'exec', _ast.PyCF_ONLY_AST)
  209. _ast.Module = type(m)
  210. for cls in _ast.Module.__mro__:
  211. if cls.__name__ == 'mod':
  212. _ast.mod = cls
  213. elif cls.__name__ == 'AST':
  214. _ast.AST = cls
  215. _ast.FunctionDef = type(m.body[0])
  216. _ast.ClassDef = type(m.body[1])
  217. _ast.If = type(m.body[2])
  218. _ast.Name = type(m.body[3].targets[0])
  219. _ast.Store = type(m.body[3].targets[0].ctx)
  220. _ast.Str = type(m.body[3].value)
  221. _ast.Sub = type(m.body[4].value.op)
  222. _ast.Add = type(m.body[4].value.left.op)
  223. _ast.Div = type(m.body[4].value.right.op)
  224. _ast.Mult = type(m.body[4].value.right.left.op)
  225. _ast.RShift = type(m.body[5].value.op)
  226. _ast.LShift = type(m.body[5].value.left.op)
  227. _ast.Mod = type(m.body[5].value.left.left.op)
  228. _ast.FloorDiv = type(m.body[5].value.left.left.left.op)
  229. _ast.BitOr = type(m.body[6].value.op)
  230. _ast.BitXor = type(m.body[6].value.left.op)
  231. _ast.BitAnd = type(m.body[6].value.left.left.op)
  232. _ast.Or = type(m.body[7].value.op)
  233. _ast.And = type(m.body[7].value.values[0].op)
  234. _ast.Invert = type(m.body[8].value.right.op)
  235. _ast.Not = type(m.body[8].value.left.right.op)
  236. _ast.UAdd = type(m.body[8].value.left.right.operand.op)
  237. _ast.USub = type(m.body[8].value.left.left.op)
  238. _ast.Or = type(m.body[9].value.op)
  239. _ast.And = type(m.body[9].value.values[0].op)
  240. _ast.IsNot = type(m.body[10].value.ops[0])
  241. _ast.NotEq = type(m.body[10].value.ops[1])
  242. _ast.Is = type(m.body[10].value.left.ops[0])
  243. _ast.Eq = type(m.body[10].value.left.ops[1])
  244. _ast.Gt = type(m.body[11].value.ops[0])
  245. _ast.Lt = type(m.body[11].value.ops[1])
  246. _ast.GtE = type(m.body[11].value.ops[2])
  247. _ast.LtE = type(m.body[11].value.ops[3])
  248. _ast.In = type(m.body[12].value.ops[0])
  249. _ast.NotIn = type(m.body[12].value.ops[1])