Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/filelock/_util.py: 22%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3__lazy_modules__ = ["pathlib", "platform", "secrets"]
5import errno
6import os
7import platform
8import secrets
9import stat
10import sys
11import time
12from contextlib import contextmanager, suppress
13from errno import EACCES, EINVAL, EIO, EISDIR, ENOSYS, EXDEV
14from functools import cache
15from pathlib import Path
16from typing import TYPE_CHECKING, Final, cast
18if TYPE_CHECKING:
19 import ctypes
20 from collections.abc import Callable, Generator
21 from contextlib import AbstractContextManager
22 from io import BufferedReader
25def raise_on_not_writable_file(filename: str) -> None:
26 # Raise an exception if attempting to open the file for writing would fail.
27 #
28 # Separates files that can never be written from files that are writable but currently locked.
29 #
30 # :param filename: file to check
31 #
32 # :raises OSError: as if the file was opened for writing.
33 try:
34 # lstat, not stat: settles exists-and-writable in one syscall, and a hostile symlink at the lock path would
35 # make stat inspect the link target, letting an attacker turn a contended acquire into a misleading
36 # PermissionError / IsADirectoryError and probe that target's attributes. The real open passes O_NOFOLLOW and
37 # refuses the symlink anyway.
38 file_stat = os.lstat(filename)
39 except OSError:
40 return # does not exist, or an error the caller cannot act on
42 if stat.S_ISDIR(file_stat.st_mode):
43 if sys.platform == "win32": # pragma: win32 cover
44 raise PermissionError(EACCES, "Permission denied", filename)
45 raise IsADirectoryError(EISDIR, "Is a directory", filename) # pragma: win32 no cover
47 # No mtime guard: the old `if st_mtime != 0` skip covered NFS/Linux quirks where os.lstat returned an all-zero
48 # struct, which it no longer does. Skipping on mtime 0 let a read-only file or a directory at the lock path pass
49 # as missing, so acquire() blocked forever on an open that cannot succeed.
50 # Match open() credentials where supported; group permissions and ACLs can grant access without S_IWUSR. Do not
51 # follow a symlink, as the lstat above does not: one swapped in since would be judged by its target.
52 if not (file_stat.st_mode & stat.S_IWUSR) and not os.access(
53 filename,
54 os.W_OK,
55 effective_ids=os.access in os.supports_effective_ids,
56 follow_symlinks=os.access not in os.supports_follow_symlinks,
57 ):
58 try:
59 os.lstat(filename)
60 except FileNotFoundError:
61 # A holder may unlink its marker between lstat() and access(); let the caller attempt creation.
62 return
63 raise PermissionError(EACCES, "Permission denied", filename)
66def ensure_directory_exists(filename: Path | str) -> None:
67 # Ensure the directory containing the file exists (create it if necessary).
68 #
69 # :param filename: file.
70 Path(filename).parent.mkdir(parents=True, exist_ok=True)
73@contextmanager
74def pin_inode(lock_file: str) -> Generator[int | None]:
75 # Keep the file at *lock_file* allocated for the length of the block and yield its inode number.
76 #
77 # An inode is the kernel's identity for a file, independent of its name, and its number names a file only while the
78 # inode stays allocated. ext4 hands a freed number to the next file created. Once a stale marker is unlinked, the
79 # next holder's marker can therefore carry the number a contender read a moment ago. A caller that reads the marker
80 # inside the block, gives up unless it read the yielded inode, and breaks the marker before leaving can rely on the
81 # number.
82 #
83 # :param lock_file: path to the lock file about to be read and perhaps broken.
84 #
85 # :returns: the pinned inode number, or ``None`` where nothing can be pinned. Windows cannot rename a file this
86 # process holds open, so it needs no pin. Without ``O_PATH`` a symlink or a socket cannot be opened.
87 #
88 # :raises FileNotFoundError: if the path is gone.
89 if sys.platform == "win32": # pragma: win32 cover
90 yield None
91 else: # pragma: win32 no cover
92 # O_PATH pins any node, a symlink or socket included, without read permission and without the filesystem seeing
93 # an open; elsewhere a nonblocking read-only open pins what it can.
94 flags: Final = getattr(os, "O_PATH", os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) | getattr(os, "O_NOFOLLOW", 0)
95 try:
96 fd = os.open(lock_file, flags)
97 except FileNotFoundError:
98 raise
99 except OSError:
100 yield None
101 return
102 try:
103 yield os.fstat(fd).st_ino
104 finally:
105 os.close(fd)
108def remove_own_file(lock_file: str, fd: int) -> None: # pragma: win32 no cover
109 # Remove *lock_file* while it still names the file open as *fd*; the caller closes the descriptor afterwards.
110 #
111 # Where the platform has an atomic exchange, a stale break takes the file through it, so a peer's marker found at
112 # the path goes back. Otherwise the path is unlinked by name if it still names this file. The open descriptor keeps
113 # the inode allocated, so a matching number cannot belong to a successor's marker.
114 #
115 # :raises OSError: if the descriptor cannot be read or the unlink fails.
116 own: Final = os.fstat(fd)
117 with suppress(FileNotFoundError):
118 break_lock_file(lock_file, own.st_mtime, own.st_ino)
119 if os.path.samestat(os.lstat(lock_file), own):
120 Path(lock_file).unlink()
123def break_lock_file(lock_file: str, mtime_before: float, ino_before: int) -> None:
124 # Remove the lock file a caller found stale with modification time *mtime_before* and inode *ino_before*.
125 #
126 # The caller keeps the judged file open until we return, through :func:`pin_inode` or its own descriptor, so no
127 # other file can carry *ino_before* meanwhile. Right before breaking we ``lstat`` the lock again and leave it alone
128 # if a peer recreated it since the caller judged it stale, as Mercurial re-reads a lock under its break lock.
129 #
130 # On Linux (``renameat2``) and macOS (``renamex_np``) the platform can exchange two names in one step. There we swap
131 # the marker with a placeholder of our own, so the lock path never stands empty, and a contender that polls
132 # meanwhile finds a file it cannot parse and waits. If the file we took is not the one judged stale, a peer
133 # recreated the lock, and a second exchange puts it back. Otherwise we remove it and then our placeholder. Of
134 # several processes racing to break the same lock, only one takes the stale file.
135 #
136 # Windows cannot exchange names, but it refuses to rename a marker its live holder keeps open. There we rename the
137 # marker aside and, if it was recreated after all, link it back. ``link`` never replaces a name, so a marker a third
138 # process created meanwhile keeps the path. Elsewhere without an exchange (NFS, SMB and other filesystems that
139 # refuse it, platforms without the call) a rename would leave the path empty beside a possibly live holder. We then
140 # do not move the marker, and the caller waits for ``timeout`` or ``break_lock()``.
141 #
142 # We compare inodes as well as modification times, because NFS and FAT store coarse modification times. The times
143 # are compared for equality, since a file's owner can set any time. We ``lstat`` so a symlink a peer swaps in is
144 # never followed. The break name carries a random token so no other process can guess it.
145 #
146 # :param lock_file: path to the lock file to break.
147 # :param mtime_before: modification time the caller saw when it judged the lock stale.
148 # :param ino_before: inode number the caller saw when it judged the lock stale.
149 #
150 # :raises OSError: if taking the marker fails, for example because the file vanished, another user owns it in a
151 # sticky directory, or on Windows its holder keeps it open.
152 if (current := os.lstat(lock_file)).st_mtime != mtime_before or current.st_ino != ino_before:
153 return
154 break_path: Final[str] = f"{lock_file}.break.{os.getpid()}.{secrets.token_hex(16)}"
155 if exchange_supported(): # pragma: win32 no cover
156 # An atomic exchange with a placeholder takes the marker without the lock path ever being empty. No contender
157 # can then create a marker while this one is judged, and a live marker goes back the same way.
158 fd: Final = os.open(break_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600)
159 try:
160 try:
161 # Not a record any lock writes, so a contender that reads it waits, and one left by a crash ages out.
162 # The PID tells the contender that evicts it whose break names to search for a marker this process left.
163 write_all(fd, f"filelock/break\n{os.getpid()}\n".encode())
164 exchanged = exchange_names(break_path, lock_file)
165 except OSError:
166 exchanged = False
167 if not exchanged:
168 Path(break_path).unlink()
169 return
170 if (grabbed := os.lstat(break_path)).st_mtime == mtime_before and grabbed.st_ino == ino_before:
171 Path(break_path).unlink()
172 # Our descriptor keeps the placeholder's inode allocated, so a matching number at the path is still
173 # ours.
174 if os.path.samestat(os.lstat(lock_file), os.fstat(fd)):
175 Path(lock_file).unlink()
176 else:
177 exchange_back(lock_file, break_path, fd)
178 finally:
179 os.close(fd)
180 elif _MOVES_WITHOUT_EXCHANGE: # pragma: win32 cover
181 # Renaming the marker aside empties the path until a recreated live marker is put back. link never replaces a
182 # name, so a marker a third process created meanwhile keeps the path, and the live one waits under the break
183 # name. Without os.link, or where the filesystem refuses hard links, the live one waits there too.
184 with _pinned_for_rename(lock_file):
185 Path(lock_file).rename(break_path)
186 if (st_after := os.lstat(break_path)).st_mtime == mtime_before and st_after.st_ino == ino_before:
187 unlink_path(break_path)
188 elif stat.S_ISREG(st_after.st_mode) and hasattr(os, "link"): # pragma: needs hard-link
189 with suppress(OSError):
190 os.link(break_path, lock_file, follow_symlinks=False)
191 unlink_path(break_path)
194def exchange_supported() -> bool:
195 # Whether this platform has a call that swaps two names in one step; a filesystem may still refuse it.
196 return _rename_call() is not None
199@cache
200def _rename_call() -> tuple[Callable[[bytes, bytes, int], int], int] | None:
201 # The call returns 0 or the errno. Looked up on first use, since most processes never break a lock and ctypes is
202 # slow to import. None where ctypes is missing (GraalPy) or the platform has no exchange.
203 try:
204 import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it
205 except ImportError: # pragma: no cover # GraalPy ships without ctypes
206 return None
207 if sys.platform in {"linux", "android"}: # pragma: linux cover
208 if (renameat2 := _renameat2(ctypes.CDLL(None, use_errno=True))) is None:
209 return None # pragma: no cover # an architecture the CI matrix does not run
210 return renameat2, 2 # RENAME_EXCHANGE in linux/fs.h
211 if sys.platform == "darwin": # pragma: darwin cover
212 renamex_np: Final = ctypes.CDLL(None, use_errno=True).renamex_np
213 renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint]
214 renamex_np.restype = ctypes.c_int
216 def call_renamex_np(source: bytes, target: bytes, flags: int) -> int:
217 return 0 if renamex_np(source, target, flags) == 0 else ctypes.get_errno()
219 return call_renamex_np, 0x2 # RENAME_SWAP in sys/stdio.h
220 return None # pragma: linux no cover # pragma: android no cover # pragma: darwin no cover
223def _renameat2(libc: ctypes.CDLL) -> Callable[[bytes, bytes, int], int] | None: # pragma: linux cover
224 # Reached through syscall(2) rather than the C library's wrapper. glibc 2.28, musl 1.2.2 and bionic API 30 added
225 # the wrapper, and Termux ships an older bionic; the kernel has had the call since 3.15. The number is the
226 # architecture's own, so an architecture outside the table gets no exchange rather than another call.
227 import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it
229 numbers: Final = {
230 "x86_64": 316,
231 "i386": 353,
232 "i686": 353,
233 "armv7l": 382,
234 "armv8l": 382,
235 "aarch64": 276,
236 "riscv64": 276,
237 "loongarch64": 276,
238 "ppc64le": 357,
239 "s390x": 347,
240 }
241 if (number := numbers.get(platform.machine())) is None:
242 return None # pragma: no cover # an architecture the CI matrix does not run
243 syscall: Final = libc.syscall
244 syscall.restype = ctypes.c_long
245 at_fdcwd: Final = -100
247 def call_renameat2(source: bytes, target: bytes, flags: int) -> int:
248 arguments: Final = (
249 ctypes.c_long(number),
250 ctypes.c_int(at_fdcwd),
251 ctypes.c_char_p(source),
252 ctypes.c_int(at_fdcwd),
253 ctypes.c_char_p(target),
254 ctypes.c_uint(flags),
255 )
256 return 0 if syscall(*arguments) == 0 else ctypes.get_errno()
258 return call_renameat2
261def write_all(fd: int, data: bytes) -> None:
262 # Write the whole buffer to *fd*, looping over the short writes POSIX lets ``os.write`` make.
263 #
264 # A caller that stopped after one short write would leave a truncated marker behind. A peer that reads between two
265 # writes sees a prefix of the record; each caller has to tolerate that. Lock records need not survive a power loss,
266 # so we skip ``fsync``.
267 #
268 # :param fd: file descriptor open for writing.
269 # :param data: bytes to write in full.
270 #
271 # :raises OSError: if a write reports zero progress before the record is complete.
272 remaining = memoryview(data)
273 while remaining:
274 if (written := os.write(fd, remaining)) == 0:
275 raise OSError(EIO, "os.write wrote 0 bytes before the record was complete")
276 remaining = remaining[written:]
279def exchange_names(source: str, target: str) -> bool: # pragma: win32 no cover
280 # Swap the names *source* and *target* in one step, so neither ever stands empty, where :func:`exchange_supported`.
281 #
282 # :returns: ``False`` where the filesystem cannot, so the caller falls back
283 #
284 # :raises OSError: for any other failure, such as a missing name or a denied rename
285 renamer, exchange_flag = cast("tuple[Callable[[bytes, bytes, int], int], int]", _rename_call())
286 if not (error := renamer(os.fsencode(source), os.fsencode(target), exchange_flag)):
287 return True
288 # EXDEV: overlayfs and some FUSE filesystems refuse an exchange between layers.
289 if error in {EINVAL, ENOSYS, EXDEV, getattr(errno, "ENOTSUP", EINVAL), getattr(errno, "EOPNOTSUPP", EINVAL)}:
290 return False
291 raise OSError(error, os.strerror(error), source, None, target)
294def exchange_back(lock_file: str, aside: str, fd: int) -> bool: # pragma: win32 no cover
295 # Raced breakers nest: each aside holds the prior marker. Returning out of order can strand a live holder's marker.
296 # A peer may move the path between our check and exchange, so stalled breakers exchange to open any resulting ring.
297 try:
298 settled = _exchange_until_aside(lock_file, aside, os.fstat(fd))
299 except FileNotFoundError:
300 settled = True
301 if settled:
302 Path(aside).unlink(missing_ok=True)
303 return settled
306def _exchange_until_aside(lock_file: str, aside: str, own: os.stat_result) -> bool: # pragma: win32 no cover
307 # Peers need a few system calls to unwind; a paused or dead peer must not block the caller indefinitely.
308 deadline: Final = time.monotonic() + 2.0
309 stall: Final = 0.01
310 feed_at = time.monotonic() + stall
311 while not os.path.samestat(os.lstat(aside), own):
312 now = time.monotonic()
313 ours = os.path.samestat(os.lstat(lock_file), own)
314 if not ours and now >= deadline:
315 return False
316 if ours or now >= feed_at:
317 if not exchange_names(aside, lock_file):
318 return False
319 feed_at = now + stall
320 else:
321 time.sleep(0.0005)
322 return True
325def foreign_directory_owner(directory: os.stat_result, parent: os.stat_result) -> int | None:
326 # Return the uid that owns *directory* when another account controls it under a sticky parent.
327 #
328 # A lock that keeps its ownership records in a directory relies on them staying in place. The sticky bit (a
329 # directory flag that lets only a file's owner remove it) on a parent such as ``/tmp`` stops other accounts from
330 # renaming the directory away. A directory that this account or the parent's owner owns is therefore safe there.
331 #
332 # One that another account owns is outside the lock's control. Without the sticky bit, any process that can create
333 # the directory can also rename this account's own one away, so a refusal there would change nothing.
334 #
335 # Linux applies the same rule for ``fs.protected_regular``.
336 #
337 # :param directory: ``lstat`` of the directory found in place.
338 # :param parent: ``stat`` of the directory that holds it.
339 #
340 # :returns: the owning uid when another account controls the directory, otherwise ``None``.
341 if sys.platform != "win32" and parent.st_mode & stat.S_ISVTX: # pragma: needs foreign-sticky-tmp
342 return None if directory.st_uid in {os.geteuid(), parent.st_uid} else directory.st_uid
343 return None
346if sys.platform == "win32": # pragma: win32 cover
348 def _pinned_for_rename(path: str) -> AbstractContextManager[BufferedReader]:
349 # A rename by path waits on a Level 1 or batch oplock holder until it answers or closes. Opening first with
350 # FILE_COMPLETE_IF_OPLOCKED turns that wait into BlockingIOError, and the open handle shares delete, so the
351 # rename then proceeds. Imported here because _windows imports this module.
352 from filelock._windows import open_for_delete_without_oplock_wait # ruff:ignore[import-outside-top-level]
354 return os.fdopen(open_for_delete_without_oplock_wait(path), "rb")
356 def unlink_path(path: str) -> None:
357 from filelock._windows import unlink_without_oplock_wait # ruff:ignore[import-outside-top-level] # _windows binds Windows-only APIs on import
359 unlink_without_oplock_wait(path)
361else: # pragma: win32 no cover
363 def unlink_path(path: str) -> None:
364 Path(path).unlink()
367def touch(name: str, *, fd: int) -> None:
368 # Prefer the already-open, already-verified fd so a peer that swaps a symlink or a different file in at the
369 # path after our O_NOFOLLOW read cannot redirect the touch: utime then targets the inode behind the fd.
370 # Where the platform cannot utime an fd, fall back to a path-based touch that still refuses to follow a
371 # symlink where supported, matching the O_NOFOLLOW reads used elsewhere here.
372 if _SUPPORTS_UTIME_FD: # pragma: needs utime-fd
373 os.utime(fd, None)
374 return
375 os.utime(name, None, follow_symlinks=not _SUPPORTS_UTIME_NOFOLLOW) # pragma: lacks utime-fd
378# Windows refuses to rename a marker its live holder keeps open, so renaming aside only moves a marker nobody holds.
379# Elsewhere without an exchange a contender could take the emptied path beside a live holder, so the marker stays and
380# the lock counts as held.
381_MOVES_WITHOUT_EXCHANGE: Final[bool] = sys.platform == "win32"
382# Retargeting os.utime to an open fd lets a heartbeat refresh the exact inode it verified instead of whatever the
383# pathname now names.
384_SUPPORTS_UTIME_FD: Final[bool] = sys.platform != "win32" and os.utime in os.supports_fd
385# os.utime follows symlinks unless told not to; not every platform can refuse the follow, so probe support.
386_SUPPORTS_UTIME_NOFOLLOW: Final[bool] = os.utime in os.supports_follow_symlinks
389__all__ = [
390 "break_lock_file",
391 "ensure_directory_exists",
392 "exchange_back",
393 "exchange_names",
394 "exchange_supported",
395 "foreign_directory_owner",
396 "pin_inode",
397 "raise_on_not_writable_file",
398 "remove_own_file",
399 "touch",
400 "unlink_path",
401 "write_all",
402]