container.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600
  1. """Container and Namespace classes"""
  2. import anydbm
  3. import cPickle
  4. import logging
  5. import os.path
  6. import time
  7. import beaker.util as util
  8. from beaker.exceptions import CreationAbortedError, MissingCacheParameter
  9. from beaker.synchronization import _threading, file_synchronizer, \
  10. mutex_synchronizer, NameLock, null_synchronizer
  11. __all__ = ['Value', 'Container', 'ContainerContext',
  12. 'MemoryContainer', 'DBMContainer', 'NamespaceManager',
  13. 'MemoryNamespaceManager', 'DBMNamespaceManager', 'FileContainer',
  14. 'OpenResourceNamespaceManager',
  15. 'FileNamespaceManager', 'CreationAbortedError']
  16. logger = logging.getLogger('beaker.container')
  17. if logger.isEnabledFor(logging.DEBUG):
  18. debug = logger.debug
  19. else:
  20. def debug(message, *args):
  21. pass
  22. class NamespaceManager(object):
  23. """Handles dictionary operations and locking for a namespace of
  24. values.
  25. The implementation for setting and retrieving the namespace data is
  26. handled by subclasses.
  27. NamespaceManager may be used alone, or may be privately accessed by
  28. one or more Container objects. Container objects provide per-key
  29. services like expiration times and automatic recreation of values.
  30. Multiple NamespaceManagers created with a particular name will all
  31. share access to the same underlying datasource and will attempt to
  32. synchronize against a common mutex object. The scope of this
  33. sharing may be within a single process or across multiple
  34. processes, depending on the type of NamespaceManager used.
  35. The NamespaceManager itself is generally threadsafe, except in the
  36. case of the DBMNamespaceManager in conjunction with the gdbm dbm
  37. implementation.
  38. """
  39. def __init__(self, namespace):
  40. self.namespace = namespace
  41. def get_creation_lock(self, key):
  42. raise NotImplementedError()
  43. def do_remove(self):
  44. raise NotImplementedError()
  45. def acquire_read_lock(self):
  46. pass
  47. def release_read_lock(self):
  48. pass
  49. def acquire_write_lock(self, wait=True):
  50. return True
  51. def release_write_lock(self):
  52. pass
  53. def has_key(self, key):
  54. return self.__contains__(key)
  55. def __getitem__(self, key):
  56. raise NotImplementedError()
  57. def __setitem__(self, key, value):
  58. raise NotImplementedError()
  59. def set_value(self, key, value, expiretime=None):
  60. """Optional set_value() method called by Value.
  61. Allows an expiretime to be passed, for namespace
  62. implementations which can prune their collections
  63. using expiretime.
  64. """
  65. self[key] = value
  66. def __contains__(self, key):
  67. raise NotImplementedError()
  68. def __delitem__(self, key):
  69. raise NotImplementedError()
  70. def keys(self):
  71. raise NotImplementedError()
  72. def remove(self):
  73. self.do_remove()
  74. class OpenResourceNamespaceManager(NamespaceManager):
  75. """A NamespaceManager where read/write operations require opening/
  76. closing of a resource which is possibly mutexed.
  77. """
  78. def __init__(self, namespace):
  79. NamespaceManager.__init__(self, namespace)
  80. self.access_lock = self.get_access_lock()
  81. self.openers = 0
  82. self.mutex = _threading.Lock()
  83. def get_access_lock(self):
  84. raise NotImplementedError()
  85. def do_open(self, flags):
  86. raise NotImplementedError()
  87. def do_close(self):
  88. raise NotImplementedError()
  89. def acquire_read_lock(self):
  90. self.access_lock.acquire_read_lock()
  91. try:
  92. self.open('r', checkcount = True)
  93. except:
  94. self.access_lock.release_read_lock()
  95. raise
  96. def release_read_lock(self):
  97. try:
  98. self.close(checkcount = True)
  99. finally:
  100. self.access_lock.release_read_lock()
  101. def acquire_write_lock(self, wait=True):
  102. r = self.access_lock.acquire_write_lock(wait)
  103. try:
  104. if (wait or r):
  105. self.open('c', checkcount = True)
  106. return r
  107. except:
  108. self.access_lock.release_write_lock()
  109. raise
  110. def release_write_lock(self):
  111. try:
  112. self.close(checkcount=True)
  113. finally:
  114. self.access_lock.release_write_lock()
  115. def open(self, flags, checkcount=False):
  116. self.mutex.acquire()
  117. try:
  118. if checkcount:
  119. if self.openers == 0:
  120. self.do_open(flags)
  121. self.openers += 1
  122. else:
  123. self.do_open(flags)
  124. self.openers = 1
  125. finally:
  126. self.mutex.release()
  127. def close(self, checkcount=False):
  128. self.mutex.acquire()
  129. try:
  130. if checkcount:
  131. self.openers -= 1
  132. if self.openers == 0:
  133. self.do_close()
  134. else:
  135. if self.openers > 0:
  136. self.do_close()
  137. self.openers = 0
  138. finally:
  139. self.mutex.release()
  140. def remove(self):
  141. self.access_lock.acquire_write_lock()
  142. try:
  143. self.close(checkcount=False)
  144. self.do_remove()
  145. finally:
  146. self.access_lock.release_write_lock()
  147. class Value(object):
  148. __slots__ = 'key', 'createfunc', 'expiretime', 'expire_argument', 'starttime', 'storedtime',\
  149. 'namespace'
  150. def __init__(self, key, namespace, createfunc=None, expiretime=None, starttime=None):
  151. self.key = key
  152. self.createfunc = createfunc
  153. self.expire_argument = expiretime
  154. self.starttime = starttime
  155. self.storedtime = -1
  156. self.namespace = namespace
  157. def has_value(self):
  158. """return true if the container has a value stored.
  159. This is regardless of it being expired or not.
  160. """
  161. self.namespace.acquire_read_lock()
  162. try:
  163. return self.namespace.has_key(self.key)
  164. finally:
  165. self.namespace.release_read_lock()
  166. def can_have_value(self):
  167. return self.has_current_value() or self.createfunc is not None
  168. def has_current_value(self):
  169. self.namespace.acquire_read_lock()
  170. try:
  171. has_value = self.namespace.has_key(self.key)
  172. if has_value:
  173. value = self.__get_value()
  174. return not self._is_expired()
  175. else:
  176. return False
  177. finally:
  178. self.namespace.release_read_lock()
  179. def _is_expired(self):
  180. """Return true if this container's value is expired.
  181. Note that this method is only correct if has_current_value()
  182. or get_value() have been called already.
  183. """
  184. return (
  185. (
  186. self.starttime is not None and
  187. self.storedtime < self.starttime
  188. )
  189. or
  190. (
  191. self.expiretime is not None and
  192. time.time() >= self.expiretime + self.storedtime
  193. )
  194. )
  195. def get_value(self):
  196. self.namespace.acquire_read_lock()
  197. try:
  198. has_value = self.has_value()
  199. if has_value:
  200. try:
  201. value = self.__get_value()
  202. if not self._is_expired():
  203. return value
  204. except KeyError:
  205. # guard against un-mutexed backends raising KeyError
  206. pass
  207. if not self.createfunc:
  208. raise KeyError(self.key)
  209. finally:
  210. self.namespace.release_read_lock()
  211. has_createlock = False
  212. creation_lock = self.namespace.get_creation_lock(self.key)
  213. if has_value:
  214. if not creation_lock.acquire(wait=False):
  215. debug("get_value returning old value while new one is created")
  216. return value
  217. else:
  218. debug("lock_creatfunc (didnt wait)")
  219. has_createlock = True
  220. if not has_createlock:
  221. debug("lock_createfunc (waiting)")
  222. creation_lock.acquire()
  223. debug("lock_createfunc (waited)")
  224. try:
  225. # see if someone created the value already
  226. self.namespace.acquire_read_lock()
  227. try:
  228. if self.has_value():
  229. try:
  230. value = self.__get_value()
  231. if not self._is_expired():
  232. return value
  233. except KeyError:
  234. # guard against un-mutexed backends raising KeyError
  235. pass
  236. finally:
  237. self.namespace.release_read_lock()
  238. debug("get_value creating new value")
  239. v = self.createfunc()
  240. self.set_value(v)
  241. return v
  242. finally:
  243. creation_lock.release()
  244. debug("released create lock")
  245. def __get_value(self):
  246. value = self.namespace[self.key]
  247. try:
  248. self.storedtime, self.expiretime, value = value
  249. except ValueError:
  250. if not len(value) == 2:
  251. raise
  252. # Old format: upgrade
  253. self.storedtime, value = value
  254. self.expiretime = self.expire_argument = None
  255. debug("get_value upgrading time %r expire time %r", self.storedtime, self.expire_argument)
  256. self.namespace.release_read_lock()
  257. self.set_value(value)
  258. self.namespace.acquire_read_lock()
  259. except TypeError:
  260. # occurs when the value is None. memcached
  261. # may yank the rug from under us in which case
  262. # that's the result
  263. raise KeyError(self.key)
  264. return value
  265. def set_value(self, value):
  266. self.namespace.acquire_write_lock()
  267. try:
  268. self.storedtime = time.time()
  269. debug("set_value stored time %r expire time %r", self.storedtime, self.expire_argument)
  270. self.namespace.set_value(self.key, (self.storedtime, self.expire_argument, value))
  271. finally:
  272. self.namespace.release_write_lock()
  273. def clear_value(self):
  274. self.namespace.acquire_write_lock()
  275. try:
  276. debug("clear_value")
  277. if self.namespace.has_key(self.key):
  278. try:
  279. del self.namespace[self.key]
  280. except KeyError:
  281. # guard against un-mutexed backends raising KeyError
  282. pass
  283. self.storedtime = -1
  284. finally:
  285. self.namespace.release_write_lock()
  286. class MemoryNamespaceManager(NamespaceManager):
  287. namespaces = util.SyncDict()
  288. def __init__(self, namespace, **kwargs):
  289. NamespaceManager.__init__(self, namespace)
  290. self.dictionary = MemoryNamespaceManager.namespaces.get(self.namespace,
  291. dict)
  292. def get_creation_lock(self, key):
  293. return NameLock(
  294. identifier="memorycontainer/funclock/%s/%s" % (self.namespace, key),
  295. reentrant=True
  296. )
  297. def __getitem__(self, key):
  298. return self.dictionary[key]
  299. def __contains__(self, key):
  300. return self.dictionary.__contains__(key)
  301. def has_key(self, key):
  302. return self.dictionary.__contains__(key)
  303. def __setitem__(self, key, value):
  304. self.dictionary[key] = value
  305. def __delitem__(self, key):
  306. del self.dictionary[key]
  307. def do_remove(self):
  308. self.dictionary.clear()
  309. def keys(self):
  310. return self.dictionary.keys()
  311. class DBMNamespaceManager(OpenResourceNamespaceManager):
  312. def __init__(self, namespace, dbmmodule=None, data_dir=None,
  313. dbm_dir=None, lock_dir=None, digest_filenames=True, **kwargs):
  314. self.digest_filenames = digest_filenames
  315. if not dbm_dir and not data_dir:
  316. raise MissingCacheParameter("data_dir or dbm_dir is required")
  317. elif dbm_dir:
  318. self.dbm_dir = dbm_dir
  319. else:
  320. self.dbm_dir = data_dir + "/container_dbm"
  321. util.verify_directory(self.dbm_dir)
  322. if not lock_dir and not data_dir:
  323. raise MissingCacheParameter("data_dir or lock_dir is required")
  324. elif lock_dir:
  325. self.lock_dir = lock_dir
  326. else:
  327. self.lock_dir = data_dir + "/container_dbm_lock"
  328. util.verify_directory(self.lock_dir)
  329. self.dbmmodule = dbmmodule or anydbm
  330. self.dbm = None
  331. OpenResourceNamespaceManager.__init__(self, namespace)
  332. self.file = util.encoded_path(root= self.dbm_dir,
  333. identifiers=[self.namespace],
  334. extension='.dbm',
  335. digest_filenames=self.digest_filenames)
  336. debug("data file %s", self.file)
  337. self._checkfile()
  338. def get_access_lock(self):
  339. return file_synchronizer(identifier=self.namespace,
  340. lock_dir=self.lock_dir)
  341. def get_creation_lock(self, key):
  342. return file_synchronizer(
  343. identifier = "dbmcontainer/funclock/%s" % self.namespace,
  344. lock_dir=self.lock_dir
  345. )
  346. def file_exists(self, file):
  347. if os.access(file, os.F_OK):
  348. return True
  349. else:
  350. for ext in ('db', 'dat', 'pag', 'dir'):
  351. if os.access(file + os.extsep + ext, os.F_OK):
  352. return True
  353. return False
  354. def _checkfile(self):
  355. if not self.file_exists(self.file):
  356. g = self.dbmmodule.open(self.file, 'c')
  357. g.close()
  358. def get_filenames(self):
  359. list = []
  360. if os.access(self.file, os.F_OK):
  361. list.append(self.file)
  362. for ext in ('pag', 'dir', 'db', 'dat'):
  363. if os.access(self.file + os.extsep + ext, os.F_OK):
  364. list.append(self.file + os.extsep + ext)
  365. return list
  366. def do_open(self, flags):
  367. debug("opening dbm file %s", self.file)
  368. try:
  369. self.dbm = self.dbmmodule.open(self.file, flags)
  370. except:
  371. self._checkfile()
  372. self.dbm = self.dbmmodule.open(self.file, flags)
  373. def do_close(self):
  374. if self.dbm is not None:
  375. debug("closing dbm file %s", self.file)
  376. self.dbm.close()
  377. def do_remove(self):
  378. for f in self.get_filenames():
  379. os.remove(f)
  380. def __getitem__(self, key):
  381. return cPickle.loads(self.dbm[key])
  382. def __contains__(self, key):
  383. return self.dbm.has_key(key)
  384. def __setitem__(self, key, value):
  385. self.dbm[key] = cPickle.dumps(value)
  386. def __delitem__(self, key):
  387. del self.dbm[key]
  388. def keys(self):
  389. return self.dbm.keys()
  390. class FileNamespaceManager(OpenResourceNamespaceManager):
  391. def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None,
  392. digest_filenames=True, **kwargs):
  393. self.digest_filenames = digest_filenames
  394. if not file_dir and not data_dir:
  395. raise MissingCacheParameter("data_dir or file_dir is required")
  396. elif file_dir:
  397. self.file_dir = file_dir
  398. else:
  399. self.file_dir = data_dir + "/container_file"
  400. util.verify_directory(self.file_dir)
  401. if not lock_dir and not data_dir:
  402. raise MissingCacheParameter("data_dir or lock_dir is required")
  403. elif lock_dir:
  404. self.lock_dir = lock_dir
  405. else:
  406. self.lock_dir = data_dir + "/container_file_lock"
  407. util.verify_directory(self.lock_dir)
  408. OpenResourceNamespaceManager.__init__(self, namespace)
  409. self.file = util.encoded_path(root=self.file_dir,
  410. identifiers=[self.namespace],
  411. extension='.cache',
  412. digest_filenames=self.digest_filenames)
  413. self.hash = {}
  414. debug("data file %s", self.file)
  415. def get_access_lock(self):
  416. return file_synchronizer(identifier=self.namespace,
  417. lock_dir=self.lock_dir)
  418. def get_creation_lock(self, key):
  419. return file_synchronizer(
  420. identifier = "filecontainer/funclock/%s" % self.namespace,
  421. lock_dir = self.lock_dir
  422. )
  423. def file_exists(self, file):
  424. return os.access(file, os.F_OK)
  425. def do_open(self, flags):
  426. if self.file_exists(self.file):
  427. fh = open(self.file, 'rb')
  428. try:
  429. self.hash = cPickle.load(fh)
  430. except (IOError, OSError, EOFError, cPickle.PickleError, ValueError):
  431. pass
  432. fh.close()
  433. self.flags = flags
  434. def do_close(self):
  435. if self.flags == 'c' or self.flags == 'w':
  436. fh = open(self.file, 'wb')
  437. cPickle.dump(self.hash, fh)
  438. fh.close()
  439. self.hash = {}
  440. self.flags = None
  441. def do_remove(self):
  442. os.remove(self.file)
  443. self.hash = {}
  444. def __getitem__(self, key):
  445. return self.hash[key]
  446. def __contains__(self, key):
  447. return self.hash.has_key(key)
  448. def __setitem__(self, key, value):
  449. self.hash[key] = value
  450. def __delitem__(self, key):
  451. del self.hash[key]
  452. def keys(self):
  453. return self.hash.keys()
  454. #### legacy stuff to support the old "Container" class interface
  455. namespace_classes = {}
  456. ContainerContext = dict
  457. class ContainerMeta(type):
  458. def __init__(cls, classname, bases, dict_):
  459. namespace_classes[cls] = cls.namespace_class
  460. return type.__init__(cls, classname, bases, dict_)
  461. def __call__(self, key, context, namespace, createfunc=None,
  462. expiretime=None, starttime=None, **kwargs):
  463. if namespace in context:
  464. ns = context[namespace]
  465. else:
  466. nscls = namespace_classes[self]
  467. context[namespace] = ns = nscls(namespace, **kwargs)
  468. return Value(key, ns, createfunc=createfunc,
  469. expiretime=expiretime, starttime=starttime)
  470. class Container(object):
  471. __metaclass__ = ContainerMeta
  472. namespace_class = NamespaceManager
  473. class FileContainer(Container):
  474. namespace_class = FileNamespaceManager
  475. class MemoryContainer(Container):
  476. namespace_class = MemoryNamespaceManager
  477. class DBMContainer(Container):
  478. namespace_class = DBMNamespaceManager
  479. DbmContainer = DBMContainer