lockfile.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498
  1. """
  2. lockfile.py - Platform-independent advisory file locks.
  3. Requires Python 2.5 unless you apply 2.4.diff
  4. Locking is done on a per-thread basis instead of a per-process basis.
  5. Usage:
  6. >>> lock = FileLock('somefile')
  7. >>> try:
  8. ... lock.acquire()
  9. ... except AlreadyLocked:
  10. ... print 'somefile', 'is locked already.'
  11. ... except LockFailed:
  12. ... print 'somefile', 'can\\'t be locked.'
  13. ... else:
  14. ... print 'got lock'
  15. got lock
  16. >>> print lock.is_locked()
  17. True
  18. >>> lock.release()
  19. >>> lock = FileLock('somefile')
  20. >>> print lock.is_locked()
  21. False
  22. >>> with lock:
  23. ... print lock.is_locked()
  24. True
  25. >>> print lock.is_locked()
  26. False
  27. >>> # It is okay to lock twice from the same thread...
  28. >>> with lock:
  29. ... lock.acquire()
  30. ...
  31. >>> # Though no counter is kept, so you can't unlock multiple times...
  32. >>> print lock.is_locked()
  33. False
  34. Exceptions:
  35. Error - base class for other exceptions
  36. LockError - base class for all locking exceptions
  37. AlreadyLocked - Another thread or process already holds the lock
  38. LockFailed - Lock failed for some other reason
  39. UnlockError - base class for all unlocking exceptions
  40. AlreadyUnlocked - File was not locked.
  41. NotMyLock - File was locked but not by the current thread/process
  42. """
  43. from __future__ import division
  44. import sys
  45. import socket
  46. import os
  47. import thread
  48. import threading
  49. import time
  50. import errno
  51. import urllib
  52. # Work with PEP8 and non-PEP8 versions of threading module.
  53. if not hasattr(threading, "current_thread"):
  54. threading.current_thread = threading.currentThread
  55. if not hasattr(threading.Thread, "get_name"):
  56. threading.Thread.get_name = threading.Thread.getName
  57. __all__ = ['Error', 'LockError', 'LockTimeout', 'AlreadyLocked',
  58. 'LockFailed', 'UnlockError', 'NotLocked', 'NotMyLock',
  59. 'LinkFileLock', 'MkdirFileLock', 'SQLiteFileLock']
  60. class Error(Exception):
  61. """
  62. Base class for other exceptions.
  63. >>> try:
  64. ... raise Error
  65. ... except Exception:
  66. ... pass
  67. """
  68. pass
  69. class LockError(Error):
  70. """
  71. Base class for error arising from attempts to acquire the lock.
  72. >>> try:
  73. ... raise LockError
  74. ... except Error:
  75. ... pass
  76. """
  77. pass
  78. class LockTimeout(LockError):
  79. """Raised when lock creation fails within a user-defined period of time.
  80. >>> try:
  81. ... raise LockTimeout
  82. ... except LockError:
  83. ... pass
  84. """
  85. pass
  86. class AlreadyLocked(LockError):
  87. """Some other thread/process is locking the file.
  88. >>> try:
  89. ... raise AlreadyLocked
  90. ... except LockError:
  91. ... pass
  92. """
  93. pass
  94. class LockFailed(LockError):
  95. """Lock file creation failed for some other reason.
  96. >>> try:
  97. ... raise LockFailed
  98. ... except LockError:
  99. ... pass
  100. """
  101. pass
  102. class UnlockError(Error):
  103. """
  104. Base class for errors arising from attempts to release the lock.
  105. >>> try:
  106. ... raise UnlockError
  107. ... except Error:
  108. ... pass
  109. """
  110. pass
  111. class NotLocked(UnlockError):
  112. """Raised when an attempt is made to unlock an unlocked file.
  113. >>> try:
  114. ... raise NotLocked
  115. ... except UnlockError:
  116. ... pass
  117. """
  118. pass
  119. class NotMyLock(UnlockError):
  120. """Raised when an attempt is made to unlock a file someone else locked.
  121. >>> try:
  122. ... raise NotMyLock
  123. ... except UnlockError:
  124. ... pass
  125. """
  126. pass
  127. class LockBase:
  128. """Base class for platform-specific lock classes."""
  129. def __init__(self, path, threaded=True):
  130. """
  131. >>> lock = LockBase('somefile')
  132. >>> lock = LockBase('somefile', threaded=False)
  133. """
  134. self.path = path
  135. self.lock_file = os.path.abspath(path) + ".lock"
  136. self.hostname = socket.gethostname()
  137. self.pid = os.getpid()
  138. if threaded:
  139. name = threading.current_thread().get_name()
  140. tname = "%s-" % urllib.quote(name, safe="")
  141. else:
  142. tname = ""
  143. dirname = os.path.dirname(self.lock_file)
  144. self.unique_name = os.path.join(dirname,
  145. "%s.%s%s" % (self.hostname,
  146. tname,
  147. self.pid))
  148. def acquire(self, timeout=None):
  149. """
  150. Acquire the lock.
  151. * If timeout is omitted (or None), wait forever trying to lock the
  152. file.
  153. * If timeout > 0, try to acquire the lock for that many seconds. If
  154. the lock period expires and the file is still locked, raise
  155. LockTimeout.
  156. * If timeout <= 0, raise AlreadyLocked immediately if the file is
  157. already locked.
  158. """
  159. raise NotImplemented("implement in subclass")
  160. def release(self):
  161. """
  162. Release the lock.
  163. If the file is not locked, raise NotLocked.
  164. """
  165. raise NotImplemented("implement in subclass")
  166. def is_locked(self):
  167. """
  168. Tell whether or not the file is locked.
  169. """
  170. raise NotImplemented("implement in subclass")
  171. def i_am_locking(self):
  172. """
  173. Return True if this object is locking the file.
  174. """
  175. raise NotImplemented("implement in subclass")
  176. def break_lock(self):
  177. """
  178. Remove a lock. Useful if a locking thread failed to unlock.
  179. """
  180. raise NotImplemented("implement in subclass")
  181. def __enter__(self):
  182. """
  183. Context manager support.
  184. """
  185. self.acquire()
  186. return self
  187. def __exit__(self, *_exc):
  188. """
  189. Context manager support.
  190. """
  191. self.release()
  192. class LinkFileLock(LockBase):
  193. """Lock access to a file using atomic property of link(2)."""
  194. def acquire(self, timeout=None):
  195. try:
  196. open(self.unique_name, "wb").close()
  197. except IOError:
  198. raise LockFailed("failed to create %s" % self.unique_name)
  199. end_time = time.time()
  200. if timeout is not None and timeout > 0:
  201. end_time += timeout
  202. while True:
  203. # Try and create a hard link to it.
  204. try:
  205. os.link(self.unique_name, self.lock_file)
  206. except OSError:
  207. # Link creation failed. Maybe we've double-locked?
  208. nlinks = os.stat(self.unique_name).st_nlink
  209. if nlinks == 2:
  210. # The original link plus the one I created == 2. We're
  211. # good to go.
  212. return
  213. else:
  214. # Otherwise the lock creation failed.
  215. if timeout is not None and time.time() > end_time:
  216. os.unlink(self.unique_name)
  217. if timeout > 0:
  218. raise LockTimeout
  219. else:
  220. raise AlreadyLocked
  221. time.sleep(timeout is not None and timeout/10 or 0.1)
  222. else:
  223. # Link creation succeeded. We're good to go.
  224. return
  225. def release(self):
  226. if not self.is_locked():
  227. raise NotLocked
  228. elif not os.path.exists(self.unique_name):
  229. raise NotMyLock
  230. os.unlink(self.unique_name)
  231. os.unlink(self.lock_file)
  232. def is_locked(self):
  233. return os.path.exists(self.lock_file)
  234. def i_am_locking(self):
  235. return (self.is_locked() and
  236. os.path.exists(self.unique_name) and
  237. os.stat(self.unique_name).st_nlink == 2)
  238. def break_lock(self):
  239. if os.path.exists(self.lock_file):
  240. os.unlink(self.lock_file)
  241. class MkdirFileLock(LockBase):
  242. """Lock file by creating a directory."""
  243. def __init__(self, path, threaded=True):
  244. """
  245. >>> lock = MkdirFileLock('somefile')
  246. >>> lock = MkdirFileLock('somefile', threaded=False)
  247. """
  248. LockBase.__init__(self, path, threaded)
  249. if threaded:
  250. tname = "%x-" % thread.get_ident()
  251. else:
  252. tname = ""
  253. # Lock file itself is a directory. Place the unique file name into
  254. # it.
  255. self.unique_name = os.path.join(self.lock_file,
  256. "%s.%s%s" % (self.hostname,
  257. tname,
  258. self.pid))
  259. def acquire(self, timeout=None):
  260. end_time = time.time()
  261. if timeout is not None and timeout > 0:
  262. end_time += timeout
  263. if timeout is None:
  264. wait = 0.1
  265. else:
  266. wait = max(0, timeout / 10)
  267. while True:
  268. try:
  269. os.mkdir(self.lock_file)
  270. except OSError:
  271. err = sys.exc_info()[1]
  272. if err.errno == errno.EEXIST:
  273. # Already locked.
  274. if os.path.exists(self.unique_name):
  275. # Already locked by me.
  276. return
  277. if timeout is not None and time.time() > end_time:
  278. if timeout > 0:
  279. raise LockTimeout
  280. else:
  281. # Someone else has the lock.
  282. raise AlreadyLocked
  283. time.sleep(wait)
  284. else:
  285. # Couldn't create the lock for some other reason
  286. raise LockFailed("failed to create %s" % self.lock_file)
  287. else:
  288. open(self.unique_name, "wb").close()
  289. return
  290. def release(self):
  291. if not self.is_locked():
  292. raise NotLocked
  293. elif not os.path.exists(self.unique_name):
  294. raise NotMyLock
  295. os.unlink(self.unique_name)
  296. os.rmdir(self.lock_file)
  297. def is_locked(self):
  298. return os.path.exists(self.lock_file)
  299. def i_am_locking(self):
  300. return (self.is_locked() and
  301. os.path.exists(self.unique_name))
  302. def break_lock(self):
  303. if os.path.exists(self.lock_file):
  304. for name in os.listdir(self.lock_file):
  305. os.unlink(os.path.join(self.lock_file, name))
  306. os.rmdir(self.lock_file)
  307. class SQLiteFileLock(LockBase):
  308. "Demonstration of using same SQL-based locking."
  309. import tempfile
  310. _fd, testdb = tempfile.mkstemp()
  311. os.close(_fd)
  312. os.unlink(testdb)
  313. del _fd, tempfile
  314. def __init__(self, path, threaded=True):
  315. LockBase.__init__(self, path, threaded)
  316. self.lock_file = unicode(self.lock_file)
  317. self.unique_name = unicode(self.unique_name)
  318. import sqlite3
  319. self.connection = sqlite3.connect(SQLiteFileLock.testdb)
  320. c = self.connection.cursor()
  321. try:
  322. c.execute("create table locks"
  323. "("
  324. " lock_file varchar(32),"
  325. " unique_name varchar(32)"
  326. ")")
  327. except sqlite3.OperationalError:
  328. pass
  329. else:
  330. self.connection.commit()
  331. import atexit
  332. atexit.register(os.unlink, SQLiteFileLock.testdb)
  333. def acquire(self, timeout=None):
  334. end_time = time.time()
  335. if timeout is not None and timeout > 0:
  336. end_time += timeout
  337. if timeout is None:
  338. wait = 0.1
  339. elif timeout <= 0:
  340. wait = 0
  341. else:
  342. wait = timeout / 10
  343. cursor = self.connection.cursor()
  344. while True:
  345. if not self.is_locked():
  346. # Not locked. Try to lock it.
  347. cursor.execute("insert into locks"
  348. " (lock_file, unique_name)"
  349. " values"
  350. " (?, ?)",
  351. (self.lock_file, self.unique_name))
  352. self.connection.commit()
  353. # Check to see if we are the only lock holder.
  354. cursor.execute("select * from locks"
  355. " where unique_name = ?",
  356. (self.unique_name,))
  357. rows = cursor.fetchall()
  358. if len(rows) > 1:
  359. # Nope. Someone else got there. Remove our lock.
  360. cursor.execute("delete from locks"
  361. " where unique_name = ?",
  362. (self.unique_name,))
  363. self.connection.commit()
  364. else:
  365. # Yup. We're done, so go home.
  366. return
  367. else:
  368. # Check to see if we are the only lock holder.
  369. cursor.execute("select * from locks"
  370. " where unique_name = ?",
  371. (self.unique_name,))
  372. rows = cursor.fetchall()
  373. if len(rows) == 1:
  374. # We're the locker, so go home.
  375. return
  376. # Maybe we should wait a bit longer.
  377. if timeout is not None and time.time() > end_time:
  378. if timeout > 0:
  379. # No more waiting.
  380. raise LockTimeout
  381. else:
  382. # Someone else has the lock and we are impatient..
  383. raise AlreadyLocked
  384. # Well, okay. We'll give it a bit longer.
  385. time.sleep(wait)
  386. def release(self):
  387. if not self.is_locked():
  388. raise NotLocked
  389. if not self.i_am_locking():
  390. raise NotMyLock((self._who_is_locking(), self.unique_name))
  391. cursor = self.connection.cursor()
  392. cursor.execute("delete from locks"
  393. " where unique_name = ?",
  394. (self.unique_name,))
  395. self.connection.commit()
  396. def _who_is_locking(self):
  397. cursor = self.connection.cursor()
  398. cursor.execute("select unique_name from locks"
  399. " where lock_file = ?",
  400. (self.lock_file,))
  401. return cursor.fetchone()[0]
  402. def is_locked(self):
  403. cursor = self.connection.cursor()
  404. cursor.execute("select * from locks"
  405. " where lock_file = ?",
  406. (self.lock_file,))
  407. rows = cursor.fetchall()
  408. return not not rows
  409. def i_am_locking(self):
  410. cursor = self.connection.cursor()
  411. cursor.execute("select * from locks"
  412. " where lock_file = ?"
  413. " and unique_name = ?",
  414. (self.lock_file, self.unique_name))
  415. return not not cursor.fetchall()
  416. def break_lock(self):
  417. cursor = self.connection.cursor()
  418. cursor.execute("delete from locks"
  419. " where lock_file = ?",
  420. (self.lock_file,))
  421. self.connection.commit()
  422. if hasattr(os, "link"):
  423. FileLock = LinkFileLock
  424. else:
  425. FileLock = MkdirFileLock