util.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364
  1. """Beaker utilities"""
  2. try:
  3. import thread as _thread
  4. import threading as _threading
  5. except ImportError:
  6. import dummy_thread as _thread
  7. import dummy_threading as _threading
  8. from datetime import datetime, timedelta
  9. import os
  10. import string
  11. import types
  12. import weakref
  13. import warnings
  14. try:
  15. Set = set
  16. except NameError:
  17. from sets import Set
  18. try:
  19. from hashlib import sha1
  20. except ImportError:
  21. from sha import sha as sha1
  22. from beaker.converters import asbool
  23. try:
  24. from base64 import b64encode, b64decode
  25. except ImportError:
  26. import binascii
  27. _translation = [chr(_x) for _x in range(256)]
  28. # From Python 2.5 base64.py
  29. def _translate(s, altchars):
  30. translation = _translation[:]
  31. for k, v in altchars.items():
  32. translation[ord(k)] = v
  33. return s.translate(''.join(translation))
  34. def b64encode(s, altchars=None):
  35. """Encode a string using Base64.
  36. s is the string to encode. Optional altchars must be a string of at least
  37. length 2 (additional characters are ignored) which specifies an
  38. alternative alphabet for the '+' and '/' characters. This allows an
  39. application to e.g. generate url or filesystem safe Base64 strings.
  40. The encoded string is returned.
  41. """
  42. # Strip off the trailing newline
  43. encoded = binascii.b2a_base64(s)[:-1]
  44. if altchars is not None:
  45. return _translate(encoded, {'+': altchars[0], '/': altchars[1]})
  46. return encoded
  47. def b64decode(s, altchars=None):
  48. """Decode a Base64 encoded string.
  49. s is the string to decode. Optional altchars must be a string of at least
  50. length 2 (additional characters are ignored) which specifies the
  51. alternative alphabet used instead of the '+' and '/' characters.
  52. The decoded string is returned. A TypeError is raised if s were
  53. incorrectly padded or if there are non-alphabet characters present in the
  54. string.
  55. """
  56. if altchars is not None:
  57. s = _translate(s, {altchars[0]: '+', altchars[1]: '/'})
  58. try:
  59. return binascii.a2b_base64(s)
  60. except binascii.Error, msg:
  61. # Transform this exception for consistency
  62. raise TypeError(msg)
  63. try:
  64. from threading import local as _tlocal
  65. except ImportError:
  66. try:
  67. from dummy_threading import local as _tlocal
  68. except ImportError:
  69. class _tlocal(object):
  70. def __init__(self):
  71. self.__dict__['_tdict'] = {}
  72. def __delattr__(self, key):
  73. try:
  74. del self._tdict[(thread.get_ident(), key)]
  75. except KeyError:
  76. raise AttributeError(key)
  77. def __getattr__(self, key):
  78. try:
  79. return self._tdict[(thread.get_ident(), key)]
  80. except KeyError:
  81. raise AttributeError(key)
  82. def __setattr__(self, key, value):
  83. self._tdict[(thread.get_ident(), key)] = value
  84. __all__ = ["ThreadLocal", "Registry", "WeakValuedRegistry", "SyncDict",
  85. "encoded_path", "verify_directory"]
  86. def verify_directory(dir):
  87. """verifies and creates a directory. tries to
  88. ignore collisions with other threads and processes."""
  89. tries = 0
  90. while not os.access(dir, os.F_OK):
  91. try:
  92. tries += 1
  93. os.makedirs(dir)
  94. except:
  95. if tries > 5:
  96. raise
  97. def deprecated(func, message):
  98. def deprecated_method(*args, **kargs):
  99. warnings.warn(message, DeprecationWarning, 2)
  100. return func(*args, **kargs)
  101. try:
  102. deprecated_method.__name__ = func.__name__
  103. except TypeError: # Python < 2.4
  104. pass
  105. deprecated_method.__doc__ = "%s\n\n%s" % (message, func.__doc__)
  106. return deprecated_method
  107. class ThreadLocal(object):
  108. """stores a value on a per-thread basis"""
  109. __slots__ = 'tlocal'
  110. def __init__(self):
  111. self.tlocal = _tlocal()
  112. def put(self, value):
  113. self.tlocal.value = value
  114. def has(self):
  115. return hasattr(self.tlocal, 'value')
  116. def get(self, default=None):
  117. return getattr(self.tlocal, 'value', default)
  118. def remove(self):
  119. del self.tlocal.value
  120. class SyncDict(object):
  121. """
  122. An efficient/threadsafe singleton map algorithm, a.k.a.
  123. "get a value based on this key, and create if not found or not
  124. valid" paradigm:
  125. exists && isvalid ? get : create
  126. Works with weakref dictionaries and the LRUCache to handle items
  127. asynchronously disappearing from the dictionary.
  128. Use python 2.3.3 or greater ! a major bug was just fixed in Nov.
  129. 2003 that was driving me nuts with garbage collection/weakrefs in
  130. this section.
  131. """
  132. def __init__(self):
  133. self.mutex = _thread.allocate_lock()
  134. self.dict = {}
  135. def get(self, key, createfunc, *args, **kwargs):
  136. try:
  137. if self.has_key(key):
  138. return self.dict[key]
  139. else:
  140. return self.sync_get(key, createfunc, *args, **kwargs)
  141. except KeyError:
  142. return self.sync_get(key, createfunc, *args, **kwargs)
  143. def sync_get(self, key, createfunc, *args, **kwargs):
  144. self.mutex.acquire()
  145. try:
  146. try:
  147. if self.has_key(key):
  148. return self.dict[key]
  149. else:
  150. return self._create(key, createfunc, *args, **kwargs)
  151. except KeyError:
  152. return self._create(key, createfunc, *args, **kwargs)
  153. finally:
  154. self.mutex.release()
  155. def _create(self, key, createfunc, *args, **kwargs):
  156. self[key] = obj = createfunc(*args, **kwargs)
  157. return obj
  158. def has_key(self, key):
  159. return self.dict.has_key(key)
  160. def __contains__(self, key):
  161. return self.dict.__contains__(key)
  162. def __getitem__(self, key):
  163. return self.dict.__getitem__(key)
  164. def __setitem__(self, key, value):
  165. self.dict.__setitem__(key, value)
  166. def __delitem__(self, key):
  167. return self.dict.__delitem__(key)
  168. def clear(self):
  169. self.dict.clear()
  170. class WeakValuedRegistry(SyncDict):
  171. def __init__(self):
  172. self.mutex = _threading.RLock()
  173. self.dict = weakref.WeakValueDictionary()
  174. def encoded_path(root, identifiers, extension = ".enc", depth = 3,
  175. digest_filenames=True):
  176. """Generate a unique file-accessible path from the given list of
  177. identifiers starting at the given root directory."""
  178. ident = string.join(identifiers, "_")
  179. if digest_filenames:
  180. ident = sha1(ident).hexdigest()
  181. ident = os.path.basename(ident)
  182. tokens = []
  183. for d in range(1, depth):
  184. tokens.append(ident[0:d])
  185. dir = os.path.join(root, *tokens)
  186. verify_directory(dir)
  187. return os.path.join(dir, ident + extension)
  188. def verify_options(opt, types, error):
  189. if not isinstance(opt, types):
  190. if not isinstance(types, tuple):
  191. types = (types,)
  192. coerced = False
  193. for typ in types:
  194. try:
  195. if typ in (list, tuple):
  196. opt = [x.strip() for x in opt.split(',')]
  197. else:
  198. if typ == bool:
  199. typ = asbool
  200. opt = typ(opt)
  201. coerced = True
  202. except:
  203. pass
  204. if coerced:
  205. break
  206. if not coerced:
  207. raise Exception(error)
  208. elif isinstance(opt, str) and not opt.strip():
  209. raise Exception("Empty strings are invalid for: %s" % error)
  210. return opt
  211. def verify_rules(params, ruleset):
  212. for key, types, message in ruleset:
  213. if key in params:
  214. params[key] = verify_options(params[key], types, message)
  215. return params
  216. def coerce_session_params(params):
  217. rules = [
  218. ('data_dir', (str, types.NoneType), "data_dir must be a string "
  219. "referring to a directory."),
  220. ('lock_dir', (str,), "lock_dir must be a string referring to a "
  221. "directory."),
  222. ('type', (str, types.NoneType), "Session type must be a string."),
  223. ('cookie_expires', (bool, datetime, timedelta), "Cookie expires was "
  224. "not a boolean, datetime, or timedelta instance."),
  225. ('cookie_domain', (str, types.NoneType), "Cookie domain must be a "
  226. "string."),
  227. ('id', (str,), "Session id must be a string."),
  228. ('key', (str,), "Session key must be a string."),
  229. ('secret', (str, types.NoneType), "Session secret must be a string."),
  230. ('validate_key', (str, types.NoneType), "Session encrypt_key must be "
  231. "a string."),
  232. ('encrypt_key', (str, types.NoneType), "Session validate_key must be "
  233. "a string."),
  234. ('secure', (bool, types.NoneType), "Session secure must be a boolean."),
  235. ('timeout', (int, types.NoneType), "Session timeout must be an "
  236. "integer."),
  237. ('auto', (bool, types.NoneType), "Session is created if accessed."),
  238. ]
  239. return verify_rules(params, rules)
  240. def coerce_cache_params(params):
  241. rules = [
  242. ('data_dir', (str, types.NoneType), "data_dir must be a string "
  243. "referring to a directory."),
  244. ('lock_dir', (str,), "lock_dir must be a string referring to a "
  245. "directory."),
  246. ('type', (str,), "Cache type must be a string."),
  247. ('enabled', (bool, types.NoneType), "enabled must be true/false "
  248. "if present."),
  249. ('expire', (int, types.NoneType), "expire must be an integer representing "
  250. "how many seconds the cache is valid for"),
  251. ('regions', (list, tuple, types.NoneType), "Regions must be a "
  252. "comma seperated list of valid regions")
  253. ]
  254. return verify_rules(params, rules)
  255. def parse_cache_config_options(config, include_defaults=True):
  256. """Parse configuration options and validate for use with the
  257. CacheManager"""
  258. # Load default cache options
  259. if include_defaults:
  260. options= dict(type='memory', data_dir=None, expire=None,
  261. log_file=None)
  262. else:
  263. options = {}
  264. for key, val in config.iteritems():
  265. if key.startswith('beaker.cache.'):
  266. options[key[13:]] = val
  267. if key.startswith('cache.'):
  268. options[key[6:]] = val
  269. coerce_cache_params(options)
  270. # Set cache to enabled if not turned off
  271. if 'enabled' not in options:
  272. options['enabled'] = True
  273. # Configure region dict if regions are available
  274. regions = options.pop('regions', None)
  275. if regions:
  276. region_configs = {}
  277. for region in regions:
  278. # Setup the default cache options
  279. region_options = dict(data_dir=options.get('data_dir'),
  280. lock_dir=options.get('lock_dir'),
  281. type=options.get('type'),
  282. enabled=options['enabled'],
  283. expire=options.get('expire'))
  284. region_len = len(region) + 1
  285. for key in options.keys():
  286. if key.startswith('%s.' % region):
  287. region_options[key[region_len:]] = options.pop(key)
  288. coerce_cache_params(region_options)
  289. region_configs[region] = region_options
  290. options['cache_regions'] = region_configs
  291. return options
  292. def func_namespace(func):
  293. """Generates a unique namespace for a function"""
  294. kls = None
  295. if hasattr(func, 'im_func'):
  296. kls = func.im_class
  297. func = func.im_func
  298. if kls:
  299. return '%s.%s' % (kls.__module__, kls.__name__)
  300. else:
  301. return func.__module__