cache.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385
  1. """Cache object
  2. The Cache object is used to manage a set of cache files and their
  3. associated backend. The backends can be rotated on the fly by
  4. specifying an alternate type when used.
  5. Advanced users can add new backends in beaker.backends
  6. """
  7. import pkg_resources
  8. import warnings
  9. import beaker.container as container
  10. import beaker.util as util
  11. from beaker.exceptions import BeakerException, InvalidCacheBackendError
  12. # Initialize the basic available backends
  13. clsmap = {
  14. 'memory':container.MemoryNamespaceManager,
  15. 'dbm':container.DBMNamespaceManager,
  16. 'file':container.FileNamespaceManager,
  17. }
  18. # Load up the additional entry point defined backends
  19. for entry_point in pkg_resources.iter_entry_points('beaker.backends'):
  20. try:
  21. NamespaceManager = entry_point.load()
  22. name = entry_point.name
  23. if name in clsmap:
  24. raise BeakerException("NamespaceManager name conflict,'%s' "
  25. "already loaded" % name)
  26. clsmap[name] = NamespaceManager
  27. except (InvalidCacheBackendError, SyntaxError):
  28. # Ignore invalid backends
  29. pass
  30. except:
  31. import sys
  32. from pkg_resources import DistributionNotFound
  33. # Warn when there's a problem loading a NamespaceManager
  34. if not isinstance(sys.exc_info()[1], DistributionNotFound):
  35. import traceback
  36. from StringIO import StringIO
  37. tb = StringIO()
  38. traceback.print_exc(file=tb)
  39. warnings.warn("Unable to load NamespaceManager entry point: '%s': "
  40. "%s" % (entry_point, tb.getvalue()), RuntimeWarning,
  41. 2)
  42. # Load legacy-style backends
  43. try:
  44. import beaker.ext.memcached as memcached
  45. clsmap['ext:memcached'] = memcached.MemcachedNamespaceManager
  46. except InvalidCacheBackendError, e:
  47. clsmap['ext:memcached'] = e
  48. try:
  49. import beaker.ext.database as database
  50. clsmap['ext:database'] = database.DatabaseNamespaceManager
  51. except InvalidCacheBackendError, e:
  52. clsmap['ext:database'] = e
  53. try:
  54. import beaker.ext.sqla as sqla
  55. clsmap['ext:sqla'] = sqla.SqlaNamespaceManager
  56. except InvalidCacheBackendError, e:
  57. clsmap['ext:sqla'] = e
  58. try:
  59. import beaker.ext.google as google
  60. clsmap['ext:google'] = google.GoogleNamespaceManager
  61. except (InvalidCacheBackendError, SyntaxError), e:
  62. clsmap['ext:google'] = e
  63. class Cache(object):
  64. """Front-end to the containment API implementing a data cache.
  65. ``namespace``
  66. the namespace of this Cache
  67. ``type``
  68. type of cache to use
  69. ``expire``
  70. seconds to keep cached data
  71. ``expiretime``
  72. seconds to keep cached data (legacy support)
  73. ``starttime``
  74. time when cache was cache was
  75. """
  76. def __init__(self, namespace, type='memory', expiretime=None,
  77. starttime=None, expire=None, **nsargs):
  78. try:
  79. cls = clsmap[type]
  80. if isinstance(cls, InvalidCacheBackendError):
  81. raise cls
  82. except KeyError:
  83. raise TypeError("Unknown cache implementation %r" % type)
  84. self.namespace = cls(namespace, **nsargs)
  85. self.expiretime = expiretime or expire
  86. self.starttime = starttime
  87. self.nsargs = nsargs
  88. def put(self, key, value, **kw):
  89. self._get_value(key, **kw).set_value(value)
  90. set_value = put
  91. def get(self, key, **kw):
  92. """Retrieve a cached value from the container"""
  93. return self._get_value(key, **kw).get_value()
  94. get_value = get
  95. def remove_value(self, key, **kw):
  96. mycontainer = self._get_value(key, **kw)
  97. if mycontainer.has_current_value():
  98. mycontainer.clear_value()
  99. remove = remove_value
  100. def _get_value(self, key, **kw):
  101. if isinstance(key, unicode):
  102. key = key.encode('ascii', 'backslashreplace')
  103. if 'type' in kw:
  104. return self._legacy_get_value(key, **kw)
  105. kw.setdefault('expiretime', self.expiretime)
  106. kw.setdefault('starttime', self.starttime)
  107. return container.Value(key, self.namespace, **kw)
  108. def _legacy_get_value(self, key, type, **kw):
  109. expiretime = kw.pop('expiretime', self.expiretime)
  110. starttime = kw.pop('starttime', None)
  111. createfunc = kw.pop('createfunc', None)
  112. kwargs = self.nsargs.copy()
  113. kwargs.update(kw)
  114. c = Cache(self.namespace.namespace, type=type, **kwargs)
  115. return c._get_value(key, expiretime=expiretime, createfunc=createfunc,
  116. starttime=starttime)
  117. _legacy_get_value = util.deprecated(_legacy_get_value, "Specifying a "
  118. "'type' and other namespace configuration with cache.get()/put()/etc. "
  119. "is deprecated. Specify 'type' and other namespace configuration to "
  120. "cache_manager.get_cache() and/or the Cache constructor instead.")
  121. def clear(self):
  122. """Clear all the values from the namespace"""
  123. self.namespace.remove()
  124. # dict interface
  125. def __getitem__(self, key):
  126. return self.get(key)
  127. def __contains__(self, key):
  128. return self._get_value(key).has_current_value()
  129. def has_key(self, key):
  130. return key in self
  131. def __delitem__(self, key):
  132. self.remove_value(key)
  133. def __setitem__(self, key, value):
  134. self.put(key, value)
  135. class CacheManager(object):
  136. def __init__(self, **kwargs):
  137. """Initialize a CacheManager object with a set of options
  138. Options should be parsed with the
  139. :func:`~beaker.util.parse_cache_config_options` function to
  140. ensure only valid options are used.
  141. """
  142. self.kwargs = kwargs
  143. self.caches = {}
  144. self.regions = kwargs.pop('cache_regions', {})
  145. def get_cache(self, name, **kwargs):
  146. kw = self.kwargs.copy()
  147. kw.update(kwargs)
  148. return self.caches.setdefault(name + str(kw), Cache(name, **kw))
  149. def get_cache_region(self, name, region):
  150. if region not in self.regions:
  151. raise BeakerException('Cache region not configured: %s' % region)
  152. kw = self.regions[region]
  153. return self.caches.setdefault(name + str(kw), Cache(name, **kw))
  154. def region(self, region, *args):
  155. """Decorate a function to cache itself using a cache region
  156. The region decorator requires arguments if there are more than
  157. 2 of the same named function, in the same module. This is
  158. because the namespace used for the functions cache is based on
  159. the functions name and the module.
  160. Example::
  161. # Assuming a cache object is available like:
  162. cache = CacheManager(dict_of_config_options)
  163. def populate_things():
  164. @cache.region('short_term', 'some_data')
  165. def load(search_term, limit, offset):
  166. return load_the_data(search_term, limit, offset)
  167. return load('rabbits', 20, 0)
  168. .. note::
  169. The function being decorated must only be called with
  170. positional arguments.
  171. """
  172. cache = [None]
  173. key = " ".join(str(x) for x in args)
  174. def decorate(func):
  175. namespace = util.func_namespace(func)
  176. def cached(*args):
  177. reg = self.regions[region]
  178. if not reg.get('enabled', True):
  179. return func(*args)
  180. if not cache[0]:
  181. cache[0] = self.get_cache_region(namespace, region)
  182. cache_key = key + " " + " ".join(str(x) for x in args)
  183. def go():
  184. return func(*args)
  185. return cache[0].get_value(cache_key, createfunc=go)
  186. cached._arg_namespace = namespace
  187. cached._arg_region = region
  188. return cached
  189. return decorate
  190. def region_invalidate(self, namespace, region, *args):
  191. """Invalidate a cache region namespace or decorated function
  192. This function only invalidates cache spaces created with the
  193. cache_region decorator.
  194. namespace
  195. Either the namespace of the result to invalidate, or the
  196. name of the cached function
  197. region
  198. The region the function was cached to. If the function was
  199. cached to a single region then this argument can be None
  200. args
  201. Arguments that were used to differentiate the cached
  202. function as well as the arguments passed to the decorated
  203. function
  204. Example::
  205. # Assuming a cache object is available like:
  206. cache = CacheManager(dict_of_config_options)
  207. def populate_things(invalidate=False):
  208. @cache.region('short_term', 'some_data')
  209. def load(search_term, limit, offset):
  210. return load_the_data(search_term, limit, offset)
  211. # If the results should be invalidated first
  212. if invalidate:
  213. cache.region_invalidate(load, None, 'some_data',
  214. 'rabbits', 20, 0)
  215. return load('rabbits', 20, 0)
  216. """
  217. if callable(namespace):
  218. if not region:
  219. region = namespace._arg_region
  220. namespace = namespace._arg_namespace
  221. if not region:
  222. raise BeakerException("Region or callable function namespace is required")
  223. else:
  224. region = self.regions[region]
  225. cache = self.get_cache(namespace, **region)
  226. cache_key = " ".join(str(x) for x in args)
  227. cache.remove_value(cache_key)
  228. def cache(self, *args, **kwargs):
  229. """Decorate a function to cache itself with supplied parameters
  230. args
  231. Used to make the key unique for this function, as in region()
  232. above.
  233. kwargs
  234. Parameters to be passed to get_cache(), will override defaults
  235. Example::
  236. # Assuming a cache object is available like:
  237. cache = CacheManager(dict_of_config_options)
  238. def populate_things():
  239. @cache.cache('mycache', expire=15)
  240. def load(search_term, limit, offset):
  241. return load_the_data(search_term, limit, offset)
  242. return load('rabbits', 20, 0)
  243. .. note::
  244. The function being decorated must only be called with
  245. positional arguments.
  246. """
  247. cache = [None]
  248. key = " ".join(str(x) for x in args)
  249. def decorate(func):
  250. namespace = util.func_namespace(func)
  251. def cached(*args):
  252. if not cache[0]:
  253. cache[0] = self.get_cache(namespace, **kwargs)
  254. cache_key = key + " " + " ".join(str(x) for x in args)
  255. def go():
  256. return func(*args)
  257. return cache[0].get_value(cache_key, createfunc=go)
  258. cached._arg_namespace = namespace
  259. return cached
  260. return decorate
  261. def invalidate(self, func, *args, **kwargs):
  262. """Invalidate a cache decorated function
  263. This function only invalidates cache spaces created with the
  264. cache decorator.
  265. func
  266. Decorated function to invalidate
  267. args
  268. Used to make the key unique for this function, as in region()
  269. above.
  270. kwargs
  271. Parameters that were passed for use by get_cache(), note that
  272. this is only required if a ``type`` was specified for the
  273. function
  274. Example::
  275. # Assuming a cache object is available like:
  276. cache = CacheManager(dict_of_config_options)
  277. def populate_things(invalidate=False):
  278. @cache.cache('mycache', type="file", expire=15)
  279. def load(search_term, limit, offset):
  280. return load_the_data(search_term, limit, offset)
  281. # If the results should be invalidated first
  282. if invalidate:
  283. cache.invalidate(load, 'mycache', 'rabbits', 20, 0, type="file")
  284. return load('rabbits', 20, 0)
  285. """
  286. namespace = func._arg_namespace
  287. cache = self.get_cache(namespace, **kwargs)
  288. cache_key = " ".join(str(x) for x in args)
  289. cache.remove_value(cache_key)