Conditionally disable/enable thread-local lock behavior. (#232)

Co-authored-by: Bernát Gábor <[email protected]>

Author: csm10495

docs/changelog.rst

@@ -1,5 +1,11 @@
 Changelog
 =========
+v3.12.0 (2023-04-18)
+--------------------
+- Make the thread local behaviour something the caller can enable/disable via a flag during the lock creation, it's on
+  by default.
+- Better error handling on Windows.
+
 v3.11.0 (2023-04-06)
 --------------------
 - Make the lock thread local.

docs/index.rst

@@ -143,6 +143,29 @@ Asyncio support
 This library currently does not support asyncio. We'd recommend adding an asyncio variant though if someone can make a
 pull request for it, `see here <https://github.com/tox-dev/py-filelock/issues/99>`_.
 
+FileLocks and threads
+---------------------
+
+By default the :class:`FileLock <filelock.FileLock>` internally uses :class:`threading.local <threading.local>`
+to ensure that the lock is thread-local. If you have a use case where you'd like an instance of ``FileLock`` to be shared
+across threads, you can set the ``thread_local`` parameter to ``False`` when creating a lock. For example:
+
+.. code-block:: python
+
+    lock = FileLock("test.lock", thread_local=False)
+    # lock will be re-entrant across threads
+
+    # The same behavior would also work with other instances of BaseFileLock like SoftFileLock:
+    soft_lock = SoftFileLock("soft_test.lock", thread_local=False)
+    # soft_lock will be re-entrant across threads.
+
+
+Behavior where :class:`FileLock <filelock.FileLock>` is thread-local started in version 3.11.0. Previous versions,
+were not thread-local by default.
+
+Note: If disabling thread-local, be sure to remember that locks are re-entrant: You will be able to
+:meth:`acquire <filelock.BaseFileLock.acquire>` the same lock multiple times across multiple threads.
+
 Contributions and issues
 ------------------------
 

src/filelock/__init__.py

@@ -32,11 +32,10 @@
         if warnings is not None:
             warnings.warn("only soft file lock is available", stacklevel=2)
 
-#: Alias for the lock, which should be used for the current platform. On Windows, this is an alias for
-# :class:`WindowsFileLock`, on Unix for :class:`UnixFileLock` and otherwise for :class:`SoftFileLock`.
 if TYPE_CHECKING:
     FileLock = SoftFileLock
 else:
+    #: Alias for the lock, which should be used for the current platform.
     FileLock = _FileLock
 
 

src/filelock/_api.py

@@ -6,6 +6,7 @@
 import time
 import warnings
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from threading import local
 from types import TracebackType
 from typing import Any
@@ -36,14 +37,46 @@ def __exit__(
         self.lock.release()
 
 
-class BaseFileLock(ABC, contextlib.ContextDecorator, local):
+@dataclass
+class FileLockContext:
+    """
+    A dataclass which holds the context for a ``BaseFileLock`` object.
+    """
+
+    # The context is held in a separate class to allow optional use of thread local storage via the
+    # ThreadLocalFileContext class.
+
+    #: The path to the lock file.
+    lock_file: str
+
+    #: The default timeout value.
+    timeout: float
+
+    #: The mode for the lock files
+    mode: int
+
+    #: The file descriptor for the *_lock_file* as it is returned by the os.open() function, not None when lock held
+    lock_file_fd: int | None = None
+
+    #: The lock counter is used for implementing the nested locking mechanism.
+    lock_counter: int = 0  # When the lock is acquired is increased and the lock is only released, when this value is 0
+
+
+class ThreadLocalFileContext(FileLockContext, local):
+    """
+    A thread local version of the ``FileLockContext`` class.
+    """
+
+
+class BaseFileLock(ABC, contextlib.ContextDecorator):
     """Abstract base class for a file lock object."""
 
     def __init__(
         self,
         lock_file: str | os.PathLike[Any],
         timeout: float = -1,
         mode: int = 0o644,
+        thread_local: bool = True,
     ) -> None:
         """
         Create a new lock object.
@@ -52,29 +85,29 @@ def __init__(
         :param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in
         the acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it
         to a negative value. A timeout of 0 means, that there is exactly one attempt to acquire the file lock.
-        : param mode: file permissions for the lockfile.
+        :param mode: file permissions for the lockfile.
+        :param thread_local: Whether this object's internal context should be thread local or not.
+        If this is set to ``False`` then the lock will be reentrant across threads.
         """
-        # The path to the lock file.
-        self._lock_file: str = os.fspath(lock_file)
-
-        # The file descriptor for the *_lock_file* as it is returned by the os.open() function.
-        # This file lock is only NOT None, if the object currently holds the lock.
-        self._lock_file_fd: int | None = None
-
-        # The default timeout value.
-        self._timeout: float = timeout
+        self._is_thread_local = thread_local
 
-        # The mode for the lock files
-        self._mode: int = mode
+        # Create the context. Note that external code should not work with the context directly  and should instead use
+        # properties of this class.
+        kwargs: dict[str, Any] = {
+            "lock_file": os.fspath(lock_file),
+            "timeout": timeout,
+            "mode": mode,
+        }
+        self._context: FileLockContext = (ThreadLocalFileContext if thread_local else FileLockContext)(**kwargs)
 
-        # The lock counter is used for implementing the nested locking mechanism. Whenever the lock is acquired, the
-        # counter is increased and the lock is only released, when this value is 0 again.
-        self._lock_counter: int = 0
+    def is_thread_local(self) -> bool:
+        """:return: a flag indicating if this lock is thread local or not"""
+        return self._is_thread_local
 
     @property
     def lock_file(self) -> str:
         """:return: path to the lock file"""
-        return self._lock_file
+        return self._context.lock_file
 
     @property
     def timeout(self) -> float:
@@ -83,7 +116,7 @@ def timeout(self) -> float:
 
         .. versionadded:: 2.0.0
         """
-        return self._timeout
+        return self._context.timeout
 
     @timeout.setter
     def timeout(self, value: float | str) -> None:
@@ -92,16 +125,16 @@ def timeout(self, value: float | str) -> None:
 
         :param value: the new value, in seconds
         """
-        self._timeout = float(value)
+        self._context.timeout = float(value)
 
     @abstractmethod
     def _acquire(self) -> None:
-        """If the file lock could be acquired, self._lock_file_fd holds the file descriptor of the lock file."""
+        """If the file lock could be acquired, self._context.lock_file_fd holds the file descriptor of the lock file."""
         raise NotImplementedError
 
     @abstractmethod
     def _release(self) -> None:
-        """Releases the lock and sets self._lock_file_fd to None."""
+        """Releases the lock and sets self._context.lock_file_fd to None."""
         raise NotImplementedError
 
     @property
@@ -114,7 +147,14 @@ def is_locked(self) -> bool:
 
             This was previously a method and is now a property.
         """
-        return self._lock_file_fd is not None
+        return self._context.lock_file_fd is not None
+
+    @property
+    def lock_counter(self) -> int:
+        """
+        :return: The number of times this lock has been acquired (but not yet released).
+        """
+        return self._context.lock_counter
 
     def acquire(
         self,
@@ -132,7 +172,7 @@ def acquire(
         :param poll_interval: interval of trying to acquire the lock file
         :param poll_intervall: deprecated, kept for backwards compatibility, use ``poll_interval`` instead
         :param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the
-         first attempt. Otherwise this method will block until the timeout expires or the lock is acquired.
+         first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired.
         :raises Timeout: if fails to acquire lock within the timeout period
         :return: a context object that will unlock the file when the context is exited
 
@@ -157,18 +197,18 @@ def acquire(
         """
         # Use the default timeout, if no timeout is provided.
         if timeout is None:
-            timeout = self.timeout
+            timeout = self._context.timeout
 
         if poll_intervall is not None:
             msg = "use poll_interval instead of poll_intervall"
             warnings.warn(msg, DeprecationWarning, stacklevel=2)
             poll_interval = poll_intervall
 
         # Increment the number right at the beginning. We can still undo it, if something fails.
-        self._lock_counter += 1
+        self._context.lock_counter += 1
 
         lock_id = id(self)
-        lock_filename = self._lock_file
+        lock_filename = self.lock_file
         start_time = time.perf_counter()
         try:
             while True:
@@ -180,16 +220,16 @@ def acquire(
                     break
                 elif blocking is False:
                     _LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename)
-                    raise Timeout(self._lock_file)
+                    raise Timeout(lock_filename)
                 elif 0 <= timeout < time.perf_counter() - start_time:
                     _LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename)
-                    raise Timeout(self._lock_file)
+                    raise Timeout(lock_filename)
                 else:
                     msg = "Lock %s not acquired on %s, waiting %s seconds ..."
                     _LOGGER.debug(msg, lock_id, lock_filename, poll_interval)
                     time.sleep(poll_interval)
         except BaseException:  # Something did go wrong, so decrement the counter.
-            self._lock_counter = max(0, self._lock_counter - 1)
+            self._context.lock_counter = max(0, self._context.lock_counter - 1)
             raise
         return AcquireReturnProxy(lock=self)
 
@@ -201,14 +241,14 @@ def release(self, force: bool = False) -> None:
         :param force: If true, the lock counter is ignored and the lock is released in every case/
         """
         if self.is_locked:
-            self._lock_counter -= 1
+            self._context.lock_counter -= 1
 
-            if self._lock_counter == 0 or force:
-                lock_id, lock_filename = id(self), self._lock_file
+            if self._context.lock_counter == 0 or force:
+                lock_id, lock_filename = id(self), self.lock_file
 
                 _LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename)
                 self._release()
-                self._lock_counter = 0
+                self._context.lock_counter = 0
                 _LOGGER.debug("Lock %s released on %s", lock_id, lock_filename)
 
     def __enter__(self) -> BaseFileLock:

src/filelock/_soft.py

@@ -12,7 +12,7 @@ class SoftFileLock(BaseFileLock):
     """Simply watches the existence of the lock file."""
 
     def _acquire(self) -> None:
-        raise_on_not_writable_file(self._lock_file)
+        raise_on_not_writable_file(self.lock_file)
         # first check for exists and read-only mode as the open will mask this case as EEXIST
         flags = (
             os.O_WRONLY  # open for writing only
@@ -21,21 +21,21 @@ def _acquire(self) -> None:
             | os.O_TRUNC  # truncate the file to zero byte
         )
         try:
-            file_handler = os.open(self._lock_file, flags, self._mode)
+            file_handler = os.open(self.lock_file, flags, self._context.mode)
         except OSError as exception:  # re-raise unless expected exception
             if not (
                 exception.errno == EEXIST  # lock already exist
                 or (exception.errno == EACCES and sys.platform == "win32")  # has no access to this lock
             ):  # pragma: win32 no cover
                 raise
         else:
-            self._lock_file_fd = file_handler
+            self._context.lock_file_fd = file_handler
 
     def _release(self) -> None:
-        os.close(self._lock_file_fd)  # type: ignore # the lock file is definitely not None
-        self._lock_file_fd = None
+        os.close(self._context.lock_file_fd)  # type: ignore # the lock file is definitely not None
+        self._context.lock_file_fd = None
         try:
-            os.remove(self._lock_file)
+            os.remove(self.lock_file)
         except OSError:  # the file is already deleted and that's what we want
             pass
 

src/filelock/_unix.py

@@ -33,9 +33,9 @@ class UnixFileLock(BaseFileLock):
 
         def _acquire(self) -> None:
             open_flags = os.O_RDWR | os.O_CREAT | os.O_TRUNC
-            fd = os.open(self._lock_file, open_flags, self._mode)
+            fd = os.open(self.lock_file, open_flags, self._context.mode)
             try:
-                os.fchmod(fd, self._mode)
+                os.fchmod(fd, self._context.mode)
             except PermissionError:
                 pass  # This locked is not owned by this UID
             try:
@@ -45,14 +45,14 @@ def _acquire(self) -> None:
                 if exception.errno == ENOSYS:  # NotImplemented error
                     raise NotImplementedError("FileSystem does not appear to support flock; user SoftFileLock instead")
             else:
-                self._lock_file_fd = fd
+                self._context.lock_file_fd = fd
 
         def _release(self) -> None:
             # Do not remove the lockfile:
             #   https://github.com/tox-dev/py-filelock/issues/31
             #   https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
-            fd = cast(int, self._lock_file_fd)
-            self._lock_file_fd = None
+            fd = cast(int, self._context.lock_file_fd)
+            self._context.lock_file_fd = None
             fcntl.flock(fd, fcntl.LOCK_UN)
             os.close(fd)
 

src/filelock/_windows.py

@@ -12,17 +12,17 @@
     import msvcrt
 
     class WindowsFileLock(BaseFileLock):
-        """Uses the :func:`msvcrt.locking` function to hard lock the lock file on windows systems."""
+        """Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems."""
 
         def _acquire(self) -> None:
-            raise_on_not_writable_file(self._lock_file)
+            raise_on_not_writable_file(self.lock_file)
             flags = (
                 os.O_RDWR  # open for read and write
                 | os.O_CREAT  # create file if not exists
                 | os.O_TRUNC  # truncate file if not empty
             )
             try:
-                fd = os.open(self._lock_file, flags, self._mode)
+                fd = os.open(self.lock_file, flags, self._context.mode)
             except OSError as exception:
                 if exception.errno != EACCES:  # has no access to this lock
                     raise
@@ -34,24 +34,24 @@ def _acquire(self) -> None:
                     if exception.errno != EACCES:  # file is already locked
                         raise
                 else:
-                    self._lock_file_fd = fd
+                    self._context.lock_file_fd = fd
 
         def _release(self) -> None:
-            fd = cast(int, self._lock_file_fd)
-            self._lock_file_fd = None
+            fd = cast(int, self._context.lock_file_fd)
+            self._context.lock_file_fd = None
             msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
             os.close(fd)
 
             try:
-                os.remove(self._lock_file)
+                os.remove(self.lock_file)
             # Probably another instance of the application hat acquired the file lock.
             except OSError:
                 pass
 
 else:  # pragma: win32 no cover
 
     class WindowsFileLock(BaseFileLock):
-        """Uses the :func:`msvcrt.locking` function to hard lock the lock file on windows systems."""
+        """Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems."""
 
         def _acquire(self) -> None:
             raise NotImplementedError