Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/filelock/_util.py: 24%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3__lazy_modules__ = ["pathlib", "platform", "secrets"]
5import errno
6import os
7import platform
8import secrets
9import stat
10import sys
11from contextlib import contextmanager, suppress
12from errno import EACCES, EINVAL, EIO, EISDIR, ENOSYS, EXDEV
13from functools import cache
14from pathlib import Path
15from typing import TYPE_CHECKING, Final, cast
17if TYPE_CHECKING:
18 import ctypes
19 from collections.abc import Callable, Generator
20 from contextlib import AbstractContextManager
23def write_all(fd: int, data: bytes) -> None:
24 """
25 Write the whole buffer to *fd*, looping over the short writes POSIX lets ``os.write`` make.
27 A caller that stopped after one short write would leave a truncated marker behind. A peer that reads between two
28 writes sees a prefix of the record; each caller has to tolerate that. Lock records need not survive a power loss, so
29 we skip ``fsync``.
31 :param fd: file descriptor open for writing.
32 :param data: bytes to write in full.
34 :raises OSError: if a write reports zero progress before the record is complete.
36 """
37 remaining = memoryview(data)
38 while remaining:
39 if (written := os.write(fd, remaining)) == 0:
40 raise OSError(EIO, "os.write wrote 0 bytes before the record was complete")
41 remaining = remaining[written:]
44def raise_on_not_writable_file(filename: str) -> None:
45 """
46 Raise an exception if attempting to open the file for writing would fail.
48 Separates files that can never be written from files that are writable but currently locked.
50 :param filename: file to check
52 :raises OSError: as if the file was opened for writing.
54 """
55 try:
56 # lstat, not stat: settles exists-and-writable in one syscall, and a hostile symlink at the lock path would
57 # make stat inspect the link target, letting an attacker turn a contended acquire into a misleading
58 # PermissionError / IsADirectoryError and probe that target's attributes. The real open passes O_NOFOLLOW and
59 # refuses the symlink anyway.
60 file_stat = os.lstat(filename)
61 except OSError:
62 return # does not exist, or an error the caller cannot act on
64 if stat.S_ISDIR(file_stat.st_mode):
65 if sys.platform == "win32": # pragma: win32 cover
66 raise PermissionError(EACCES, "Permission denied", filename)
67 raise IsADirectoryError(EISDIR, "Is a directory", filename) # pragma: win32 no cover
69 # No mtime guard: the old `if st_mtime != 0` skip covered NFS/Linux quirks where os.lstat returned an all-zero
70 # struct, which it no longer does. Skipping on mtime 0 let a read-only file or a directory at the lock path pass
71 # as missing, so acquire() blocked forever on an open that cannot succeed.
72 # Match open() credentials where supported; group permissions and ACLs can grant access without S_IWUSR. Do not
73 # follow a symlink, as the lstat above does not: one swapped in since would be judged by its target.
74 if not (file_stat.st_mode & stat.S_IWUSR) and not os.access(
75 filename,
76 os.W_OK,
77 effective_ids=os.access in os.supports_effective_ids,
78 follow_symlinks=os.access not in os.supports_follow_symlinks,
79 ):
80 try:
81 os.lstat(filename)
82 except FileNotFoundError:
83 # A holder may unlink its marker between lstat() and access(); let the caller attempt creation.
84 return
85 raise PermissionError(EACCES, "Permission denied", filename)
88def ensure_directory_exists(filename: Path | str) -> None:
89 """
90 Ensure the directory containing the file exists (create it if necessary).
92 :param filename: file.
94 """
95 Path(filename).parent.mkdir(parents=True, exist_ok=True)
98@contextmanager
99def pin_inode(lock_file: str) -> Generator[int | None]:
100 """
101 Keep the file at *lock_file* allocated for the length of the block and yield its inode number.
103 An inode is the kernel's identity for a file, independent of its name, and its number names a file only while the
104 inode stays allocated. ext4 hands a freed number to the next file created. Once a stale marker is unlinked, the next
105 holder's marker can therefore carry the number a contender read a moment ago. A caller that reads the marker inside
106 the block, gives up unless it read the yielded inode, and breaks the marker before leaving can rely on the number.
108 :param lock_file: path to the lock file about to be read and perhaps broken.
110 :returns: the pinned inode number, or ``None`` where nothing can be pinned. Windows cannot rename a file this
111 process holds open, so it needs no pin. Without ``O_PATH`` a symlink or a socket cannot be opened.
113 :raises FileNotFoundError: if the path is gone.
115 """
116 if sys.platform == "win32": # pragma: win32 cover
117 yield None
118 else: # pragma: win32 no cover
119 # O_PATH pins any node, a symlink or socket included, without read permission and without the filesystem seeing
120 # an open; elsewhere a nonblocking read-only open pins what it can.
121 flags: Final = getattr(os, "O_PATH", os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) | getattr(os, "O_NOFOLLOW", 0)
122 try:
123 fd = os.open(lock_file, flags)
124 except FileNotFoundError:
125 raise
126 except OSError:
127 yield None
128 return
129 try:
130 yield os.fstat(fd).st_ino
131 finally:
132 os.close(fd)
135def remove_own_file(lock_file: str, fd: int) -> None: # pragma: win32 no cover
136 """
137 Remove *lock_file* while it still names the file open as *fd*; the caller closes the descriptor afterwards.
139 Where the platform has an atomic exchange, a stale break takes the file through it, so a peer's marker found at the
140 path goes back. Otherwise the path is unlinked by name if it still names this file. The open descriptor keeps the
141 inode allocated, so a matching number cannot belong to a successor's marker.
143 :raises OSError: if the descriptor cannot be read or the unlink fails.
145 """
146 own: Final = os.fstat(fd)
147 with suppress(FileNotFoundError):
148 break_lock_file(lock_file, own.st_mtime, own.st_ino)
149 if os.path.samestat(os.lstat(lock_file), own):
150 Path(lock_file).unlink()
153def foreign_directory_owner(directory: os.stat_result, parent: os.stat_result) -> int | None:
154 """
155 Return the uid that owns *directory* when another account controls it under a sticky parent.
157 A lock that keeps its ownership records in a directory relies on them staying in place. The sticky bit (a directory
158 flag that lets only a file's owner remove it) on a parent such as ``/tmp`` stops other accounts from renaming the
159 directory away. A directory that this account or the parent's owner owns is therefore safe there. One that another
160 account owns is outside the lock's control. Without the sticky bit, any process that can create the directory can
161 also rename this account's own one away, so a refusal there would change nothing. Linux applies the same rule for
162 ``fs.protected_regular``.
164 :param directory: ``lstat`` of the directory found in place.
165 :param parent: ``stat`` of the directory that holds it.
167 :returns: the owning uid when another account controls the directory, otherwise ``None``.
169 """
170 if sys.platform != "win32" and parent.st_mode & stat.S_ISVTX: # pragma: needs foreign-sticky-tmp
171 return None if directory.st_uid in {os.geteuid(), parent.st_uid} else directory.st_uid
172 return None
175def break_lock_file(lock_file: str, mtime_before: float, ino_before: int) -> None:
176 """
177 Remove the lock file a caller found stale with modification time *mtime_before* and inode *ino_before*.
179 The caller keeps the judged file open until we return, through :func:`pin_inode` or its own descriptor, so no
180 other file can carry *ino_before* meanwhile. Right before breaking we ``lstat`` the lock again and leave it alone
181 if a peer recreated it since the caller judged it stale, as Mercurial re-reads a lock under its break lock.
183 On Linux (``renameat2``) and macOS (``renamex_np``) the platform can exchange two names in one step. There we swap
184 the marker with a placeholder of our own, so the lock path never stands empty, and a contender that polls meanwhile
185 finds a file it cannot parse and waits. If the file we took is not the one judged stale, a peer recreated the lock,
186 and a second exchange puts it back. Otherwise we remove it and then our placeholder. Of several processes racing to
187 break the same lock, only one takes the stale file.
189 Windows cannot exchange names, but it refuses to rename a marker its live holder keeps open. There we rename the
190 marker aside and, if it was recreated after all, link it back. ``link`` never replaces a name, so a marker a third
191 process created meanwhile keeps the path. Elsewhere without an exchange (NFS, SMB and other filesystems that refuse
192 it, platforms without the call) a rename would leave the path empty beside a possibly live holder. We then do not
193 move the marker, and the caller waits for ``timeout`` or ``break_lock()``.
195 We compare inodes as well as modification times, because NFS and FAT store coarse modification times. The times are
196 compared for equality, since a file's owner can set any time. We ``lstat`` so a symlink a peer swaps in is never
197 followed. The break name carries a random token so no other process can guess it.
199 :param lock_file: path to the lock file to break.
200 :param mtime_before: modification time the caller saw when it judged the lock stale.
201 :param ino_before: inode number the caller saw when it judged the lock stale.
203 :raises OSError: if taking the marker fails, for example because the file vanished, another user owns it in a
204 sticky directory, or on Windows its holder keeps it open.
206 """
207 if (current := os.lstat(lock_file)).st_mtime != mtime_before or current.st_ino != ino_before:
208 return
209 break_path: Final[str] = f"{lock_file}.break.{os.getpid()}.{secrets.token_hex(16)}"
210 if exchange_supported(): # pragma: win32 no cover
211 _swap_out(lock_file, break_path, mtime_before, ino_before)
212 elif _MOVES_WITHOUT_EXCHANGE: # pragma: win32 cover
213 _break_by_rename(lock_file, break_path, mtime_before, ino_before)
216def _swap_out(lock_file: str, break_path: str, mtime_before: float, ino_before: int) -> None: # pragma: win32 no cover
217 # An atomic exchange with a placeholder takes the marker without the lock path ever being empty. No contender can
218 # then create a marker while this one is judged, and a live marker goes back the same way.
219 fd: Final = os.open(break_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600)
220 try:
221 try:
222 # Not a record any lock writes, so a contender that reads it waits, and one left by a crash ages out. The
223 # PID tells the contender that evicts it whose break names to search for a marker this process left.
224 write_all(fd, f"filelock/break\n{os.getpid()}\n".encode())
225 exchanged = exchange_names(break_path, lock_file)
226 except OSError:
227 exchanged = False
228 if not exchanged:
229 Path(break_path).unlink()
230 return
231 if (grabbed := os.lstat(break_path)).st_mtime == mtime_before and grabbed.st_ino == ino_before:
232 Path(break_path).unlink()
233 # Our descriptor keeps the placeholder's inode allocated, so a matching number at the path is still ours.
234 if os.path.samestat(os.lstat(lock_file), os.fstat(fd)):
235 Path(lock_file).unlink()
236 else:
237 exchange_names(break_path, lock_file)
238 Path(break_path).unlink()
239 finally:
240 os.close(fd)
243def _break_by_rename(
244 lock_file: str, break_path: str, mtime_before: float, ino_before: int
245) -> None: # pragma: win32 cover
246 # Renaming the marker aside empties the path until a recreated live marker is put back. link never replaces a name,
247 # so a marker a third process created meanwhile keeps the path, and the live one waits under the break name.
248 # Without os.link, or where the filesystem refuses hard links, the live one waits there too.
249 with _pinned_for_rename(lock_file):
250 Path(lock_file).rename(break_path)
251 if (st_after := os.lstat(break_path)).st_mtime == mtime_before and st_after.st_ino == ino_before:
252 unlink_path(break_path)
253 elif stat.S_ISREG(st_after.st_mode) and hasattr(os, "link"): # pragma: needs hard-link
254 with suppress(OSError):
255 os.link(break_path, lock_file, follow_symlinks=False)
256 unlink_path(break_path)
259if sys.platform == "win32": # pragma: win32 cover
261 def _pinned_for_rename(path: str) -> AbstractContextManager[object]:
262 # A rename by path waits on a Level 1 or batch oplock holder until it answers or closes. Opening first with
263 # FILE_COMPLETE_IF_OPLOCKED turns that wait into BlockingIOError, and the open handle shares delete, so the
264 # rename then proceeds. Imported here because _windows imports this module.
265 from filelock._windows import open_for_delete_without_oplock_wait # ruff:ignore[import-outside-top-level]
267 return os.fdopen(open_for_delete_without_oplock_wait(path), "rb")
269 def unlink_path(path: str) -> None:
270 from filelock._windows import unlink_without_oplock_wait # ruff:ignore[import-outside-top-level]
272 unlink_without_oplock_wait(path)
274else: # pragma: win32 no cover
276 def unlink_path(path: str) -> None:
277 Path(path).unlink()
280def exchange_supported() -> bool:
281 """Whether this platform has a call that swaps two names in one step; a filesystem may still refuse it."""
282 return _rename_call() is not None
285def exchange_names(source: str, target: str) -> bool: # pragma: win32 no cover
286 """
287 Swap the names *source* and *target* in one step, so neither ever stands empty, where :func:`exchange_supported`.
289 :returns: ``False`` where the filesystem cannot, so the caller falls back
291 :raises OSError: for any other failure, such as a missing name or a denied rename
293 """
294 renamer, exchange_flag = cast("tuple[Callable[[bytes, bytes, int], int], int]", _rename_call())
295 if not (error := renamer(os.fsencode(source), os.fsencode(target), exchange_flag)):
296 return True
297 # EXDEV: overlayfs and some FUSE filesystems refuse an exchange between layers.
298 if error in {EINVAL, ENOSYS, EXDEV, getattr(errno, "ENOTSUP", EINVAL), getattr(errno, "EOPNOTSUPP", EINVAL)}:
299 return False
300 raise OSError(error, os.strerror(error), source, None, target)
303@cache
304def _rename_call() -> tuple[Callable[[bytes, bytes, int], int], int] | None:
305 # The call returns 0 or the errno. Looked up on first use, since most processes never break a lock and ctypes is
306 # slow to import. None where ctypes is missing (GraalPy) or the platform has no exchange.
307 try:
308 import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it
309 except ImportError: # pragma: no cover # GraalPy ships without ctypes
310 return None
311 if sys.platform in {"linux", "android"}: # pragma: linux cover
312 if (renameat2 := _renameat2(ctypes.CDLL(None, use_errno=True))) is None:
313 return None # pragma: no cover # an architecture the CI matrix does not run
314 return renameat2, 2 # RENAME_EXCHANGE in linux/fs.h
315 if sys.platform == "darwin": # pragma: darwin cover
316 renamex_np: Final = ctypes.CDLL(None, use_errno=True).renamex_np
317 renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint]
318 renamex_np.restype = ctypes.c_int
320 def call_renamex_np(source: bytes, target: bytes, flags: int) -> int:
321 return 0 if renamex_np(source, target, flags) == 0 else ctypes.get_errno()
323 return call_renamex_np, 0x2 # RENAME_SWAP in sys/stdio.h
324 return None # pragma: linux no cover # pragma: android no cover # pragma: darwin no cover
327def _renameat2(libc: ctypes.CDLL) -> Callable[[bytes, bytes, int], int] | None: # pragma: linux cover
328 # Reached through syscall(2) rather than the C library's wrapper. glibc 2.28, musl 1.2.2 and bionic API 30 added
329 # the wrapper, and Termux ships an older bionic; the kernel has had the call since 3.15. The number is the
330 # architecture's own, so an architecture outside the table gets no exchange rather than another call.
331 import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it
333 numbers: Final = {
334 "x86_64": 316,
335 "i386": 353,
336 "i686": 353,
337 "armv7l": 382,
338 "armv8l": 382,
339 "aarch64": 276,
340 "riscv64": 276,
341 "loongarch64": 276,
342 "ppc64le": 357,
343 "s390x": 347,
344 }
345 if (number := numbers.get(platform.machine())) is None:
346 return None # pragma: no cover # an architecture the CI matrix does not run
347 syscall: Final = libc.syscall
348 syscall.restype = ctypes.c_long
349 at_fdcwd: Final = -100
351 def call_renameat2(source: bytes, target: bytes, flags: int) -> int:
352 arguments: Final = (
353 ctypes.c_long(number),
354 ctypes.c_int(at_fdcwd),
355 ctypes.c_char_p(source),
356 ctypes.c_int(at_fdcwd),
357 ctypes.c_char_p(target),
358 ctypes.c_uint(flags),
359 )
360 return 0 if syscall(*arguments) == 0 else ctypes.get_errno()
362 return call_renameat2
365def touch(name: str, *, fd: int) -> None:
366 # Prefer the already-open, already-verified fd so a peer that swaps a symlink or a different file in at the
367 # path after our O_NOFOLLOW read cannot redirect the touch: utime then targets the inode behind the fd.
368 # Where the platform cannot utime an fd, fall back to a path-based touch that still refuses to follow a
369 # symlink where supported, matching the O_NOFOLLOW reads used elsewhere here.
370 if _SUPPORTS_UTIME_FD: # pragma: needs utime-fd
371 os.utime(fd, None)
372 return
373 os.utime(name, None, follow_symlinks=not _SUPPORTS_UTIME_NOFOLLOW) # pragma: lacks utime-fd
376# Windows refuses to rename a marker its live holder keeps open, so renaming aside only moves a marker nobody holds.
377# Elsewhere without an exchange a contender could take the emptied path beside a live holder, so the marker stays and
378# the lock counts as held.
379_MOVES_WITHOUT_EXCHANGE: Final[bool] = sys.platform == "win32"
380# Retargeting os.utime to an open fd lets a heartbeat refresh the exact inode it verified instead of whatever the
381# pathname now names.
382_SUPPORTS_UTIME_FD: Final[bool] = sys.platform != "win32" and os.utime in os.supports_fd
383# os.utime follows symlinks unless told not to; not every platform can refuse the follow, so probe support.
384_SUPPORTS_UTIME_NOFOLLOW: Final[bool] = os.utime in os.supports_follow_symlinks
387__all__ = [
388 "break_lock_file",
389 "ensure_directory_exists",
390 "exchange_names",
391 "exchange_supported",
392 "foreign_directory_owner",
393 "pin_inode",
394 "raise_on_not_writable_file",
395 "remove_own_file",
396 "touch",
397 "unlink_path",
398 "write_all",
399]