util.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365
  1. # mako/util.py
  2. # Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. import re
  7. import collections
  8. import codecs
  9. import os
  10. from mako import compat
  11. import operator
  12. def function_named(fn, name):
  13. """Return a function with a given __name__.
  14. Will assign to __name__ and return the original function if possible on
  15. the Python implementation, otherwise a new function will be constructed.
  16. """
  17. fn.__name__ = name
  18. return fn
  19. class PluginLoader(object):
  20. def __init__(self, group):
  21. self.group = group
  22. self.impls = {}
  23. def load(self, name):
  24. if name in self.impls:
  25. return self.impls[name]()
  26. else:
  27. import pkg_resources
  28. for impl in pkg_resources.iter_entry_points(
  29. self.group,
  30. name):
  31. self.impls[name] = impl.load
  32. return impl.load()
  33. else:
  34. from mako import exceptions
  35. raise exceptions.RuntimeException(
  36. "Can't load plugin %s %s" %
  37. (self.group, name))
  38. def register(self, name, modulepath, objname):
  39. def load():
  40. mod = __import__(modulepath)
  41. for token in modulepath.split(".")[1:]:
  42. mod = getattr(mod, token)
  43. return getattr(mod, objname)
  44. self.impls[name] = load
  45. def verify_directory(dir):
  46. """create and/or verify a filesystem directory."""
  47. tries = 0
  48. while not os.path.exists(dir):
  49. try:
  50. tries += 1
  51. os.makedirs(dir, compat.octal("0775"))
  52. except:
  53. if tries > 5:
  54. raise
  55. def to_list(x, default=None):
  56. if x is None:
  57. return default
  58. if not isinstance(x, (list, tuple)):
  59. return [x]
  60. else:
  61. return x
  62. class memoized_property(object):
  63. """A read-only @property that is only evaluated once."""
  64. def __init__(self, fget, doc=None):
  65. self.fget = fget
  66. self.__doc__ = doc or fget.__doc__
  67. self.__name__ = fget.__name__
  68. def __get__(self, obj, cls):
  69. if obj is None:
  70. return self
  71. obj.__dict__[self.__name__] = result = self.fget(obj)
  72. return result
  73. class memoized_instancemethod(object):
  74. """Decorate a method memoize its return value.
  75. Best applied to no-arg methods: memoization is not sensitive to
  76. argument values, and will always return the same value even when
  77. called with different arguments.
  78. """
  79. def __init__(self, fget, doc=None):
  80. self.fget = fget
  81. self.__doc__ = doc or fget.__doc__
  82. self.__name__ = fget.__name__
  83. def __get__(self, obj, cls):
  84. if obj is None:
  85. return self
  86. def oneshot(*args, **kw):
  87. result = self.fget(obj, *args, **kw)
  88. memo = lambda *a, **kw: result
  89. memo.__name__ = self.__name__
  90. memo.__doc__ = self.__doc__
  91. obj.__dict__[self.__name__] = memo
  92. return result
  93. oneshot.__name__ = self.__name__
  94. oneshot.__doc__ = self.__doc__
  95. return oneshot
  96. class SetLikeDict(dict):
  97. """a dictionary that has some setlike methods on it"""
  98. def union(self, other):
  99. """produce a 'union' of this dict and another (at the key level).
  100. values in the second dict take precedence over that of the first"""
  101. x = SetLikeDict(**self)
  102. x.update(other)
  103. return x
  104. class FastEncodingBuffer(object):
  105. """a very rudimentary buffer that is faster than StringIO,
  106. but doesn't crash on unicode data like cStringIO."""
  107. def __init__(self, encoding=None, errors='strict', as_unicode=False):
  108. self.data = collections.deque()
  109. self.encoding = encoding
  110. if as_unicode:
  111. self.delim = compat.u('')
  112. else:
  113. self.delim = ''
  114. self.as_unicode = as_unicode
  115. self.errors = errors
  116. self.write = self.data.append
  117. def truncate(self):
  118. self.data = collections.deque()
  119. self.write = self.data.append
  120. def getvalue(self):
  121. if self.encoding:
  122. return self.delim.join(self.data).encode(self.encoding,
  123. self.errors)
  124. else:
  125. return self.delim.join(self.data)
  126. class LRUCache(dict):
  127. """A dictionary-like object that stores a limited number of items,
  128. discarding lesser used items periodically.
  129. this is a rewrite of LRUCache from Myghty to use a periodic timestamp-based
  130. paradigm so that synchronization is not really needed. the size management
  131. is inexact.
  132. """
  133. class _Item(object):
  134. def __init__(self, key, value):
  135. self.key = key
  136. self.value = value
  137. self.timestamp = compat.time_func()
  138. def __repr__(self):
  139. return repr(self.value)
  140. def __init__(self, capacity, threshold=.5):
  141. self.capacity = capacity
  142. self.threshold = threshold
  143. def __getitem__(self, key):
  144. item = dict.__getitem__(self, key)
  145. item.timestamp = compat.time_func()
  146. return item.value
  147. def values(self):
  148. return [i.value for i in dict.values(self)]
  149. def setdefault(self, key, value):
  150. if key in self:
  151. return self[key]
  152. else:
  153. self[key] = value
  154. return value
  155. def __setitem__(self, key, value):
  156. item = dict.get(self, key)
  157. if item is None:
  158. item = self._Item(key, value)
  159. dict.__setitem__(self, key, item)
  160. else:
  161. item.value = value
  162. self._manage_size()
  163. def _manage_size(self):
  164. while len(self) > self.capacity + self.capacity * self.threshold:
  165. bytime = sorted(dict.values(self),
  166. key=operator.attrgetter('timestamp'), reverse=True)
  167. for item in bytime[self.capacity:]:
  168. try:
  169. del self[item.key]
  170. except KeyError:
  171. # if we couldn't find a key, most likely some other thread
  172. # broke in on us. loop around and try again
  173. break
  174. # Regexp to match python magic encoding line
  175. _PYTHON_MAGIC_COMMENT_re = re.compile(
  176. r'[ \t\f]* \# .* coding[=:][ \t]*([-\w.]+)',
  177. re.VERBOSE)
  178. def parse_encoding(fp):
  179. """Deduce the encoding of a Python source file (binary mode) from magic
  180. comment.
  181. It does this in the same way as the `Python interpreter`__
  182. .. __: http://docs.python.org/ref/encodings.html
  183. The ``fp`` argument should be a seekable file object in binary mode.
  184. """
  185. pos = fp.tell()
  186. fp.seek(0)
  187. try:
  188. line1 = fp.readline()
  189. has_bom = line1.startswith(codecs.BOM_UTF8)
  190. if has_bom:
  191. line1 = line1[len(codecs.BOM_UTF8):]
  192. m = _PYTHON_MAGIC_COMMENT_re.match(line1.decode('ascii', 'ignore'))
  193. if not m:
  194. try:
  195. import parser
  196. parser.suite(line1.decode('ascii', 'ignore'))
  197. except (ImportError, SyntaxError):
  198. # Either it's a real syntax error, in which case the source
  199. # is not valid python source, or line2 is a continuation of
  200. # line1, in which case we don't want to scan line2 for a magic
  201. # comment.
  202. pass
  203. else:
  204. line2 = fp.readline()
  205. m = _PYTHON_MAGIC_COMMENT_re.match(
  206. line2.decode('ascii', 'ignore'))
  207. if has_bom:
  208. if m:
  209. raise SyntaxError("python refuses to compile code with both a UTF8" \
  210. " byte-order-mark and a magic encoding comment")
  211. return 'utf_8'
  212. elif m:
  213. return m.group(1)
  214. else:
  215. return None
  216. finally:
  217. fp.seek(pos)
  218. def sorted_dict_repr(d):
  219. """repr() a dictionary with the keys in order.
  220. Used by the lexer unit test to compare parse trees based on strings.
  221. """
  222. keys = list(d.keys())
  223. keys.sort()
  224. return "{" + ", ".join(["%r: %r" % (k, d[k]) for k in keys]) + "}"
  225. def restore__ast(_ast):
  226. """Attempt to restore the required classes to the _ast module if it
  227. appears to be missing them
  228. """
  229. if hasattr(_ast, 'AST'):
  230. return
  231. _ast.PyCF_ONLY_AST = 2 << 9
  232. m = compile("""\
  233. def foo(): pass
  234. class Bar(object): pass
  235. if False: pass
  236. baz = 'mako'
  237. 1 + 2 - 3 * 4 / 5
  238. 6 // 7 % 8 << 9 >> 10
  239. 11 & 12 ^ 13 | 14
  240. 15 and 16 or 17
  241. -baz + (not +18) - ~17
  242. baz and 'foo' or 'bar'
  243. (mako is baz == baz) is not baz != mako
  244. mako > baz < mako >= baz <= mako
  245. mako in baz not in mako""", '<unknown>', 'exec', _ast.PyCF_ONLY_AST)
  246. _ast.Module = type(m)
  247. for cls in _ast.Module.__mro__:
  248. if cls.__name__ == 'mod':
  249. _ast.mod = cls
  250. elif cls.__name__ == 'AST':
  251. _ast.AST = cls
  252. _ast.FunctionDef = type(m.body[0])
  253. _ast.ClassDef = type(m.body[1])
  254. _ast.If = type(m.body[2])
  255. _ast.Name = type(m.body[3].targets[0])
  256. _ast.Store = type(m.body[3].targets[0].ctx)
  257. _ast.Str = type(m.body[3].value)
  258. _ast.Sub = type(m.body[4].value.op)
  259. _ast.Add = type(m.body[4].value.left.op)
  260. _ast.Div = type(m.body[4].value.right.op)
  261. _ast.Mult = type(m.body[4].value.right.left.op)
  262. _ast.RShift = type(m.body[5].value.op)
  263. _ast.LShift = type(m.body[5].value.left.op)
  264. _ast.Mod = type(m.body[5].value.left.left.op)
  265. _ast.FloorDiv = type(m.body[5].value.left.left.left.op)
  266. _ast.BitOr = type(m.body[6].value.op)
  267. _ast.BitXor = type(m.body[6].value.left.op)
  268. _ast.BitAnd = type(m.body[6].value.left.left.op)
  269. _ast.Or = type(m.body[7].value.op)
  270. _ast.And = type(m.body[7].value.values[0].op)
  271. _ast.Invert = type(m.body[8].value.right.op)
  272. _ast.Not = type(m.body[8].value.left.right.op)
  273. _ast.UAdd = type(m.body[8].value.left.right.operand.op)
  274. _ast.USub = type(m.body[8].value.left.left.op)
  275. _ast.Or = type(m.body[9].value.op)
  276. _ast.And = type(m.body[9].value.values[0].op)
  277. _ast.IsNot = type(m.body[10].value.ops[0])
  278. _ast.NotEq = type(m.body[10].value.ops[1])
  279. _ast.Is = type(m.body[10].value.left.ops[0])
  280. _ast.Eq = type(m.body[10].value.left.ops[1])
  281. _ast.Gt = type(m.body[11].value.ops[0])
  282. _ast.Lt = type(m.body[11].value.ops[1])
  283. _ast.GtE = type(m.body[11].value.ops[2])
  284. _ast.LtE = type(m.body[11].value.ops[3])
  285. _ast.In = type(m.body[12].value.ops[0])
  286. _ast.NotIn = type(m.body[12].value.ops[1])
  287. def read_file(path, mode='rb'):
  288. fp = open(path, mode)
  289. try:
  290. data = fp.read()
  291. return data
  292. finally:
  293. fp.close()
  294. def read_python_file(path):
  295. fp = open(path, "rb")
  296. try:
  297. encoding = parse_encoding(fp)
  298. data = fp.read()
  299. if encoding:
  300. data = data.decode(encoding)
  301. return data
  302. finally:
  303. fp.close()