util.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436
  1. # mako/util.py
  2. # Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. import imp
  7. import sys
  8. py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0)
  9. py26 = sys.version_info >= (2, 6)
  10. py24 = sys.version_info >= (2, 4) and sys.version_info < (2, 5)
  11. jython = sys.platform.startswith('java')
  12. win32 = sys.platform.startswith('win')
  13. if py3k:
  14. from io import StringIO
  15. else:
  16. try:
  17. from cStringIO import StringIO
  18. except:
  19. from StringIO import StringIO
  20. import codecs, re, weakref, os, time, operator
  21. import collections
  22. try:
  23. import threading
  24. import thread
  25. except ImportError:
  26. import dummy_threading as threading
  27. import dummy_thread as thread
  28. if win32 or jython:
  29. time_func = time.clock
  30. else:
  31. time_func = time.time
  32. def function_named(fn, name):
  33. """Return a function with a given __name__.
  34. Will assign to __name__ and return the original function if possible on
  35. the Python implementation, otherwise a new function will be constructed.
  36. """
  37. fn.__name__ = name
  38. return fn
  39. try:
  40. from functools import partial
  41. except:
  42. def partial(func, *args, **keywords):
  43. def newfunc(*fargs, **fkeywords):
  44. newkeywords = keywords.copy()
  45. newkeywords.update(fkeywords)
  46. return func(*(args + fargs), **newkeywords)
  47. return newfunc
  48. if py24:
  49. def all(iterable):
  50. for i in iterable:
  51. if not i:
  52. return False
  53. return True
  54. def exception_name(exc):
  55. try:
  56. return exc.__class__.__name__
  57. except AttributeError:
  58. return exc.__name__
  59. else:
  60. all = all
  61. def exception_name(exc):
  62. return exc.__class__.__name__
  63. class PluginLoader(object):
  64. def __init__(self, group):
  65. self.group = group
  66. self.impls = {}
  67. def load(self, name):
  68. if name in self.impls:
  69. return self.impls[name]()
  70. else:
  71. import pkg_resources
  72. for impl in pkg_resources.iter_entry_points(
  73. self.group,
  74. name):
  75. self.impls[name] = impl.load
  76. return impl.load()
  77. else:
  78. raise exceptions.RuntimeException(
  79. "Can't load plugin %s %s" %
  80. (self.group, name))
  81. def register(self, name, modulepath, objname):
  82. def load():
  83. mod = __import__(modulepath)
  84. for token in modulepath.split(".")[1:]:
  85. mod = getattr(mod, token)
  86. return getattr(mod, objname)
  87. self.impls[name] = load
  88. def verify_directory(dir):
  89. """create and/or verify a filesystem directory."""
  90. tries = 0
  91. while not os.path.exists(dir):
  92. try:
  93. tries += 1
  94. os.makedirs(dir, 0775)
  95. except:
  96. if tries > 5:
  97. raise
  98. def to_list(x, default=None):
  99. if x is None:
  100. return default
  101. if not isinstance(x, (list, tuple)):
  102. return [x]
  103. else:
  104. return x
  105. class memoized_property(object):
  106. """A read-only @property that is only evaluated once."""
  107. def __init__(self, fget, doc=None):
  108. self.fget = fget
  109. self.__doc__ = doc or fget.__doc__
  110. self.__name__ = fget.__name__
  111. def __get__(self, obj, cls):
  112. if obj is None:
  113. return self
  114. obj.__dict__[self.__name__] = result = self.fget(obj)
  115. return result
  116. class memoized_instancemethod(object):
  117. """Decorate a method memoize its return value.
  118. Best applied to no-arg methods: memoization is not sensitive to
  119. argument values, and will always return the same value even when
  120. called with different arguments.
  121. """
  122. def __init__(self, fget, doc=None):
  123. self.fget = fget
  124. self.__doc__ = doc or fget.__doc__
  125. self.__name__ = fget.__name__
  126. def __get__(self, obj, cls):
  127. if obj is None:
  128. return self
  129. def oneshot(*args, **kw):
  130. result = self.fget(obj, *args, **kw)
  131. memo = lambda *a, **kw: result
  132. memo.__name__ = self.__name__
  133. memo.__doc__ = self.__doc__
  134. obj.__dict__[self.__name__] = memo
  135. return result
  136. oneshot.__name__ = self.__name__
  137. oneshot.__doc__ = self.__doc__
  138. return oneshot
  139. class SetLikeDict(dict):
  140. """a dictionary that has some setlike methods on it"""
  141. def union(self, other):
  142. """produce a 'union' of this dict and another (at the key level).
  143. values in the second dict take precedence over that of the first"""
  144. x = SetLikeDict(**self)
  145. x.update(other)
  146. return x
  147. class FastEncodingBuffer(object):
  148. """a very rudimentary buffer that is faster than StringIO,
  149. but doesn't crash on unicode data like cStringIO."""
  150. def __init__(self, encoding=None, errors='strict', unicode=False):
  151. self.data = collections.deque()
  152. self.encoding = encoding
  153. if unicode:
  154. self.delim = u''
  155. else:
  156. self.delim = ''
  157. self.unicode = unicode
  158. self.errors = errors
  159. self.write = self.data.append
  160. def truncate(self):
  161. self.data = collections.deque()
  162. self.write = self.data.append
  163. def getvalue(self):
  164. if self.encoding:
  165. return self.delim.join(self.data).encode(self.encoding,
  166. self.errors)
  167. else:
  168. return self.delim.join(self.data)
  169. class LRUCache(dict):
  170. """A dictionary-like object that stores a limited number of items,
  171. discarding lesser used items periodically.
  172. this is a rewrite of LRUCache from Myghty to use a periodic timestamp-based
  173. paradigm so that synchronization is not really needed. the size management
  174. is inexact.
  175. """
  176. class _Item(object):
  177. def __init__(self, key, value):
  178. self.key = key
  179. self.value = value
  180. self.timestamp = time_func()
  181. def __repr__(self):
  182. return repr(self.value)
  183. def __init__(self, capacity, threshold=.5):
  184. self.capacity = capacity
  185. self.threshold = threshold
  186. def __getitem__(self, key):
  187. item = dict.__getitem__(self, key)
  188. item.timestamp = time_func()
  189. return item.value
  190. def values(self):
  191. return [i.value for i in dict.values(self)]
  192. def setdefault(self, key, value):
  193. if key in self:
  194. return self[key]
  195. else:
  196. self[key] = value
  197. return value
  198. def __setitem__(self, key, value):
  199. item = dict.get(self, key)
  200. if item is None:
  201. item = self._Item(key, value)
  202. dict.__setitem__(self, key, item)
  203. else:
  204. item.value = value
  205. self._manage_size()
  206. def _manage_size(self):
  207. while len(self) > self.capacity + self.capacity * self.threshold:
  208. bytime = sorted(dict.values(self),
  209. key=operator.attrgetter('timestamp'), reverse=True)
  210. for item in bytime[self.capacity:]:
  211. try:
  212. del self[item.key]
  213. except KeyError:
  214. # if we couldn't find a key, most likely some other thread
  215. # broke in on us. loop around and try again
  216. break
  217. # Regexp to match python magic encoding line
  218. _PYTHON_MAGIC_COMMENT_re = re.compile(
  219. r'[ \t\f]* \# .* coding[=:][ \t]*([-\w.]+)',
  220. re.VERBOSE)
  221. def parse_encoding(fp):
  222. """Deduce the encoding of a Python source file (binary mode) from magic
  223. comment.
  224. It does this in the same way as the `Python interpreter`__
  225. .. __: http://docs.python.org/ref/encodings.html
  226. The ``fp`` argument should be a seekable file object in binary mode.
  227. """
  228. pos = fp.tell()
  229. fp.seek(0)
  230. try:
  231. line1 = fp.readline()
  232. has_bom = line1.startswith(codecs.BOM_UTF8)
  233. if has_bom:
  234. line1 = line1[len(codecs.BOM_UTF8):]
  235. m = _PYTHON_MAGIC_COMMENT_re.match(line1.decode('ascii', 'ignore'))
  236. if not m:
  237. try:
  238. import parser
  239. parser.suite(line1.decode('ascii', 'ignore'))
  240. except (ImportError, SyntaxError):
  241. # Either it's a real syntax error, in which case the source
  242. # is not valid python source, or line2 is a continuation of
  243. # line1, in which case we don't want to scan line2 for a magic
  244. # comment.
  245. pass
  246. else:
  247. line2 = fp.readline()
  248. m = _PYTHON_MAGIC_COMMENT_re.match(
  249. line2.decode('ascii', 'ignore'))
  250. if has_bom:
  251. if m:
  252. raise SyntaxError, \
  253. "python refuses to compile code with both a UTF8" \
  254. " byte-order-mark and a magic encoding comment"
  255. return 'utf_8'
  256. elif m:
  257. return m.group(1)
  258. else:
  259. return None
  260. finally:
  261. fp.seek(pos)
  262. def sorted_dict_repr(d):
  263. """repr() a dictionary with the keys in order.
  264. Used by the lexer unit test to compare parse trees based on strings.
  265. """
  266. keys = d.keys()
  267. keys.sort()
  268. return "{" + ", ".join(["%r: %r" % (k, d[k]) for k in keys]) + "}"
  269. def restore__ast(_ast):
  270. """Attempt to restore the required classes to the _ast module if it
  271. appears to be missing them
  272. """
  273. if hasattr(_ast, 'AST'):
  274. return
  275. _ast.PyCF_ONLY_AST = 2 << 9
  276. m = compile("""\
  277. def foo(): pass
  278. class Bar(object): pass
  279. if False: pass
  280. baz = 'mako'
  281. 1 + 2 - 3 * 4 / 5
  282. 6 // 7 % 8 << 9 >> 10
  283. 11 & 12 ^ 13 | 14
  284. 15 and 16 or 17
  285. -baz + (not +18) - ~17
  286. baz and 'foo' or 'bar'
  287. (mako is baz == baz) is not baz != mako
  288. mako > baz < mako >= baz <= mako
  289. mako in baz not in mako""", '<unknown>', 'exec', _ast.PyCF_ONLY_AST)
  290. _ast.Module = type(m)
  291. for cls in _ast.Module.__mro__:
  292. if cls.__name__ == 'mod':
  293. _ast.mod = cls
  294. elif cls.__name__ == 'AST':
  295. _ast.AST = cls
  296. _ast.FunctionDef = type(m.body[0])
  297. _ast.ClassDef = type(m.body[1])
  298. _ast.If = type(m.body[2])
  299. _ast.Name = type(m.body[3].targets[0])
  300. _ast.Store = type(m.body[3].targets[0].ctx)
  301. _ast.Str = type(m.body[3].value)
  302. _ast.Sub = type(m.body[4].value.op)
  303. _ast.Add = type(m.body[4].value.left.op)
  304. _ast.Div = type(m.body[4].value.right.op)
  305. _ast.Mult = type(m.body[4].value.right.left.op)
  306. _ast.RShift = type(m.body[5].value.op)
  307. _ast.LShift = type(m.body[5].value.left.op)
  308. _ast.Mod = type(m.body[5].value.left.left.op)
  309. _ast.FloorDiv = type(m.body[5].value.left.left.left.op)
  310. _ast.BitOr = type(m.body[6].value.op)
  311. _ast.BitXor = type(m.body[6].value.left.op)
  312. _ast.BitAnd = type(m.body[6].value.left.left.op)
  313. _ast.Or = type(m.body[7].value.op)
  314. _ast.And = type(m.body[7].value.values[0].op)
  315. _ast.Invert = type(m.body[8].value.right.op)
  316. _ast.Not = type(m.body[8].value.left.right.op)
  317. _ast.UAdd = type(m.body[8].value.left.right.operand.op)
  318. _ast.USub = type(m.body[8].value.left.left.op)
  319. _ast.Or = type(m.body[9].value.op)
  320. _ast.And = type(m.body[9].value.values[0].op)
  321. _ast.IsNot = type(m.body[10].value.ops[0])
  322. _ast.NotEq = type(m.body[10].value.ops[1])
  323. _ast.Is = type(m.body[10].value.left.ops[0])
  324. _ast.Eq = type(m.body[10].value.left.ops[1])
  325. _ast.Gt = type(m.body[11].value.ops[0])
  326. _ast.Lt = type(m.body[11].value.ops[1])
  327. _ast.GtE = type(m.body[11].value.ops[2])
  328. _ast.LtE = type(m.body[11].value.ops[3])
  329. _ast.In = type(m.body[12].value.ops[0])
  330. _ast.NotIn = type(m.body[12].value.ops[1])
  331. try:
  332. from inspect import CO_VARKEYWORDS, CO_VARARGS
  333. def inspect_func_args(fn):
  334. co = fn.func_code
  335. nargs = co.co_argcount
  336. names = co.co_varnames
  337. args = list(names[:nargs])
  338. varargs = None
  339. if co.co_flags & CO_VARARGS:
  340. varargs = co.co_varnames[nargs]
  341. nargs = nargs + 1
  342. varkw = None
  343. if co.co_flags & CO_VARKEYWORDS:
  344. varkw = co.co_varnames[nargs]
  345. return args, varargs, varkw, fn.func_defaults
  346. except ImportError:
  347. import inspect
  348. def inspect_func_args(fn):
  349. return inspect.getargspec(fn)
  350. def read_file(path, mode='rb'):
  351. fp = open(path, mode)
  352. try:
  353. data = fp.read()
  354. return data
  355. finally:
  356. fp.close()
  357. def load_module(module_id, path):
  358. fp = open(path, 'rb')
  359. try:
  360. return imp.load_source(module_id, path, fp)
  361. finally:
  362. fp.close()