diff --git a/docs/changelog.rst b/docs/changelog.rst index d73107af..e3985532 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,5 +1,11 @@ Changelog ========= +v3.12.0 (2023-04-18) +-------------------- +- Make the thread local behaviour something the caller can enable/disable via a flag during the lock creation, it's on + by default. +- Better error handling on Windows. + v3.11.0 (2023-04-06) -------------------- - Make the lock thread local. diff --git a/docs/index.rst b/docs/index.rst index c37c923f..5fb5fb65 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -143,6 +143,29 @@ Asyncio support This library currently does not support asyncio. We'd recommend adding an asyncio variant though if someone can make a pull request for it, `see here `_. +FileLocks and threads +--------------------- + +By default the :class:`FileLock ` internally uses :class:`threading.local ` +to ensure that the lock is thread-local. If you have a use case where you'd like an instance of ``FileLock`` to be shared +across threads, you can set the ``thread_local`` parameter to ``False`` when creating a lock. For example: + +.. code-block:: python + + lock = FileLock("test.lock", thread_local=False) + # lock will be re-entrant across threads + + # The same behavior would also work with other instances of BaseFileLock like SoftFileLock: + soft_lock = SoftFileLock("soft_test.lock", thread_local=False) + # soft_lock will be re-entrant across threads. + + +Behavior where :class:`FileLock ` is thread-local started in version 3.11.0. Previous versions, +were not thread-local by default. + +Note: If disabling thread-local, be sure to remember that locks are re-entrant: You will be able to +:meth:`acquire ` the same lock multiple times across multiple threads. + Contributions and issues ------------------------ diff --git a/src/filelock/__init__.py b/src/filelock/__init__.py index 31d2bce1..c7492ba5 100644 --- a/src/filelock/__init__.py +++ b/src/filelock/__init__.py @@ -32,11 +32,10 @@ if warnings is not None: warnings.warn("only soft file lock is available", stacklevel=2) -#: Alias for the lock, which should be used for the current platform. On Windows, this is an alias for -# :class:`WindowsFileLock`, on Unix for :class:`UnixFileLock` and otherwise for :class:`SoftFileLock`. if TYPE_CHECKING: FileLock = SoftFileLock else: + #: Alias for the lock, which should be used for the current platform. FileLock = _FileLock diff --git a/src/filelock/_api.py b/src/filelock/_api.py index 958369a6..66710cc5 100644 --- a/src/filelock/_api.py +++ b/src/filelock/_api.py @@ -6,6 +6,7 @@ import time import warnings from abc import ABC, abstractmethod +from dataclasses import dataclass from threading import local from types import TracebackType from typing import Any @@ -36,7 +37,38 @@ def __exit__( self.lock.release() -class BaseFileLock(ABC, contextlib.ContextDecorator, local): +@dataclass +class FileLockContext: + """ + A dataclass which holds the context for a ``BaseFileLock`` object. + """ + + # The context is held in a separate class to allow optional use of thread local storage via the + # ThreadLocalFileContext class. + + #: The path to the lock file. + lock_file: str + + #: The default timeout value. + timeout: float + + #: The mode for the lock files + mode: int + + #: The file descriptor for the *_lock_file* as it is returned by the os.open() function, not None when lock held + lock_file_fd: int | None = None + + #: The lock counter is used for implementing the nested locking mechanism. + lock_counter: int = 0 # When the lock is acquired is increased and the lock is only released, when this value is 0 + + +class ThreadLocalFileContext(FileLockContext, local): + """ + A thread local version of the ``FileLockContext`` class. + """ + + +class BaseFileLock(ABC, contextlib.ContextDecorator): """Abstract base class for a file lock object.""" def __init__( @@ -44,6 +76,7 @@ def __init__( lock_file: str | os.PathLike[Any], timeout: float = -1, mode: int = 0o644, + thread_local: bool = True, ) -> None: """ Create a new lock object. @@ -52,29 +85,29 @@ def __init__( :param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a negative value. A timeout of 0 means, that there is exactly one attempt to acquire the file lock. - : param mode: file permissions for the lockfile. + :param mode: file permissions for the lockfile. + :param thread_local: Whether this object's internal context should be thread local or not. + If this is set to ``False`` then the lock will be reentrant across threads. """ - # The path to the lock file. - self._lock_file: str = os.fspath(lock_file) - - # The file descriptor for the *_lock_file* as it is returned by the os.open() function. - # This file lock is only NOT None, if the object currently holds the lock. - self._lock_file_fd: int | None = None - - # The default timeout value. - self._timeout: float = timeout + self._is_thread_local = thread_local - # The mode for the lock files - self._mode: int = mode + # Create the context. Note that external code should not work with the context directly and should instead use + # properties of this class. + kwargs: dict[str, Any] = { + "lock_file": os.fspath(lock_file), + "timeout": timeout, + "mode": mode, + } + self._context: FileLockContext = (ThreadLocalFileContext if thread_local else FileLockContext)(**kwargs) - # The lock counter is used for implementing the nested locking mechanism. Whenever the lock is acquired, the - # counter is increased and the lock is only released, when this value is 0 again. - self._lock_counter: int = 0 + def is_thread_local(self) -> bool: + """:return: a flag indicating if this lock is thread local or not""" + return self._is_thread_local @property def lock_file(self) -> str: """:return: path to the lock file""" - return self._lock_file + return self._context.lock_file @property def timeout(self) -> float: @@ -83,7 +116,7 @@ def timeout(self) -> float: .. versionadded:: 2.0.0 """ - return self._timeout + return self._context.timeout @timeout.setter def timeout(self, value: float | str) -> None: @@ -92,16 +125,16 @@ def timeout(self, value: float | str) -> None: :param value: the new value, in seconds """ - self._timeout = float(value) + self._context.timeout = float(value) @abstractmethod def _acquire(self) -> None: - """If the file lock could be acquired, self._lock_file_fd holds the file descriptor of the lock file.""" + """If the file lock could be acquired, self._context.lock_file_fd holds the file descriptor of the lock file.""" raise NotImplementedError @abstractmethod def _release(self) -> None: - """Releases the lock and sets self._lock_file_fd to None.""" + """Releases the lock and sets self._context.lock_file_fd to None.""" raise NotImplementedError @property @@ -114,7 +147,14 @@ def is_locked(self) -> bool: This was previously a method and is now a property. """ - return self._lock_file_fd is not None + return self._context.lock_file_fd is not None + + @property + def lock_counter(self) -> int: + """ + :return: The number of times this lock has been acquired (but not yet released). + """ + return self._context.lock_counter def acquire( self, @@ -132,7 +172,7 @@ def acquire( :param poll_interval: interval of trying to acquire the lock file :param poll_intervall: deprecated, kept for backwards compatibility, use ``poll_interval`` instead :param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the - first attempt. Otherwise this method will block until the timeout expires or the lock is acquired. + first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired. :raises Timeout: if fails to acquire lock within the timeout period :return: a context object that will unlock the file when the context is exited @@ -157,7 +197,7 @@ def acquire( """ # Use the default timeout, if no timeout is provided. if timeout is None: - timeout = self.timeout + timeout = self._context.timeout if poll_intervall is not None: msg = "use poll_interval instead of poll_intervall" @@ -165,10 +205,10 @@ def acquire( poll_interval = poll_intervall # Increment the number right at the beginning. We can still undo it, if something fails. - self._lock_counter += 1 + self._context.lock_counter += 1 lock_id = id(self) - lock_filename = self._lock_file + lock_filename = self.lock_file start_time = time.perf_counter() try: while True: @@ -180,16 +220,16 @@ def acquire( break elif blocking is False: _LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename) - raise Timeout(self._lock_file) + raise Timeout(lock_filename) elif 0 <= timeout < time.perf_counter() - start_time: _LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename) - raise Timeout(self._lock_file) + raise Timeout(lock_filename) else: msg = "Lock %s not acquired on %s, waiting %s seconds ..." _LOGGER.debug(msg, lock_id, lock_filename, poll_interval) time.sleep(poll_interval) except BaseException: # Something did go wrong, so decrement the counter. - self._lock_counter = max(0, self._lock_counter - 1) + self._context.lock_counter = max(0, self._context.lock_counter - 1) raise return AcquireReturnProxy(lock=self) @@ -201,14 +241,14 @@ def release(self, force: bool = False) -> None: :param force: If true, the lock counter is ignored and the lock is released in every case/ """ if self.is_locked: - self._lock_counter -= 1 + self._context.lock_counter -= 1 - if self._lock_counter == 0 or force: - lock_id, lock_filename = id(self), self._lock_file + if self._context.lock_counter == 0 or force: + lock_id, lock_filename = id(self), self.lock_file _LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename) self._release() - self._lock_counter = 0 + self._context.lock_counter = 0 _LOGGER.debug("Lock %s released on %s", lock_id, lock_filename) def __enter__(self) -> BaseFileLock: diff --git a/src/filelock/_soft.py b/src/filelock/_soft.py index 1a14df71..57a6c04d 100644 --- a/src/filelock/_soft.py +++ b/src/filelock/_soft.py @@ -12,7 +12,7 @@ class SoftFileLock(BaseFileLock): """Simply watches the existence of the lock file.""" def _acquire(self) -> None: - raise_on_not_writable_file(self._lock_file) + raise_on_not_writable_file(self.lock_file) # first check for exists and read-only mode as the open will mask this case as EEXIST flags = ( os.O_WRONLY # open for writing only @@ -21,7 +21,7 @@ def _acquire(self) -> None: | os.O_TRUNC # truncate the file to zero byte ) try: - file_handler = os.open(self._lock_file, flags, self._mode) + file_handler = os.open(self.lock_file, flags, self._context.mode) except OSError as exception: # re-raise unless expected exception if not ( exception.errno == EEXIST # lock already exist @@ -29,13 +29,13 @@ def _acquire(self) -> None: ): # pragma: win32 no cover raise else: - self._lock_file_fd = file_handler + self._context.lock_file_fd = file_handler def _release(self) -> None: - os.close(self._lock_file_fd) # type: ignore # the lock file is definitely not None - self._lock_file_fd = None + os.close(self._context.lock_file_fd) # type: ignore # the lock file is definitely not None + self._context.lock_file_fd = None try: - os.remove(self._lock_file) + os.remove(self.lock_file) except OSError: # the file is already deleted and that's what we want pass diff --git a/src/filelock/_unix.py b/src/filelock/_unix.py index 4c0a2603..641bd8d9 100644 --- a/src/filelock/_unix.py +++ b/src/filelock/_unix.py @@ -33,9 +33,9 @@ class UnixFileLock(BaseFileLock): def _acquire(self) -> None: open_flags = os.O_RDWR | os.O_CREAT | os.O_TRUNC - fd = os.open(self._lock_file, open_flags, self._mode) + fd = os.open(self.lock_file, open_flags, self._context.mode) try: - os.fchmod(fd, self._mode) + os.fchmod(fd, self._context.mode) except PermissionError: pass # This locked is not owned by this UID try: @@ -45,14 +45,14 @@ def _acquire(self) -> None: if exception.errno == ENOSYS: # NotImplemented error raise NotImplementedError("FileSystem does not appear to support flock; user SoftFileLock instead") else: - self._lock_file_fd = fd + self._context.lock_file_fd = fd def _release(self) -> None: # Do not remove the lockfile: # https://github.com/tox-dev/py-filelock/issues/31 # https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition - fd = cast(int, self._lock_file_fd) - self._lock_file_fd = None + fd = cast(int, self._context.lock_file_fd) + self._context.lock_file_fd = None fcntl.flock(fd, fcntl.LOCK_UN) os.close(fd) diff --git a/src/filelock/_windows.py b/src/filelock/_windows.py index 67dd194d..799644c8 100644 --- a/src/filelock/_windows.py +++ b/src/filelock/_windows.py @@ -12,17 +12,17 @@ import msvcrt class WindowsFileLock(BaseFileLock): - """Uses the :func:`msvcrt.locking` function to hard lock the lock file on windows systems.""" + """Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems.""" def _acquire(self) -> None: - raise_on_not_writable_file(self._lock_file) + raise_on_not_writable_file(self.lock_file) flags = ( os.O_RDWR # open for read and write | os.O_CREAT # create file if not exists | os.O_TRUNC # truncate file if not empty ) try: - fd = os.open(self._lock_file, flags, self._mode) + fd = os.open(self.lock_file, flags, self._context.mode) except OSError as exception: if exception.errno != EACCES: # has no access to this lock raise @@ -34,16 +34,16 @@ def _acquire(self) -> None: if exception.errno != EACCES: # file is already locked raise else: - self._lock_file_fd = fd + self._context.lock_file_fd = fd def _release(self) -> None: - fd = cast(int, self._lock_file_fd) - self._lock_file_fd = None + fd = cast(int, self._context.lock_file_fd) + self._context.lock_file_fd = None msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) os.close(fd) try: - os.remove(self._lock_file) + os.remove(self.lock_file) # Probably another instance of the application hat acquired the file lock. except OSError: pass @@ -51,7 +51,7 @@ def _release(self) -> None: else: # pragma: win32 no cover class WindowsFileLock(BaseFileLock): - """Uses the :func:`msvcrt.locking` function to hard lock the lock file on windows systems.""" + """Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems.""" def _acquire(self) -> None: raise NotImplementedError diff --git a/tests/test_filelock.py b/tests/test_filelock.py index 9b9b164d..4ed4c3ca 100644 --- a/tests/test_filelock.py +++ b/tests/test_filelock.py @@ -612,3 +612,20 @@ def mess_with_file() -> None: results.append(executor.submit(mess_with_file)) assert all(r.result() is None for r in results) + + +@pytest.mark.parametrize("lock_type", [FileLock, SoftFileLock]) +def test_lock_can_be_non_thread_local( + tmp_path: Path, + lock_type: type[BaseFileLock], +) -> None: + lock = lock_type(tmp_path / "test.lock", thread_local=False) + + for _ in range(2): + thread = threading.Thread(target=lock.acquire, kwargs={"timeout": 2}) + thread.start() + thread.join() + + assert lock.lock_counter == 2 + + lock.release(force=True)