Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/filelock/_identity.py: 23%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3__lazy_modules__ = ["hashlib", "pathlib", "socket", "typing"]
5import hashlib
6import os
7import socket
8import sys
9from errno import EPERM, ESRCH
10from pathlib import Path
11from typing import Final
13#: Every marker format in this package caps the hostname at the RFC 1123 limit and reads a longer one as malformed.
14_HOST_NAME_LIMIT: Final[int] = 253
15#: The bytes those formats carry verbatim: printable non-space ASCII, less the ``?`` reserved for the escape.
16_HOST_NAME_VERBATIM: Final[frozenset[int]] = frozenset(range(0x21, 0x7F)) - {ord("?")}
17#: The characters at the end of an over-long name that carry a digest of the part cut off.
18_HOST_NAME_DIGEST_LEN: Final[int] = 8
19#: The kernel ABI fixes the initial PID namespace's inode (``PID_NS_INIT_INO`` in ``linux/nsfs.h``); every namespace
20#: created later draws one from ``0xF0000000`` up, so this value names the host's own namespace.
21_INITIAL_PID_NAMESPACE: Final[int] = 0xEFFFFFFC
22_NAMESPACE_MARK: Final[str] = "?pidns-"
25def host_name() -> str:
26 """
27 The hostname recorded alongside an owner, so a marker written on another machine is never probed here.
29 Marker formats here hold the field to printable non-space ASCII, while ``socket.gethostname()`` reports whatever
30 the kernel stores: a space, a newline that forges an extra marker line, or a byte no codec encodes, which Python
31 hands back as a surrogate. A holder that publishes one raw writes a marker it cannot read back, and loses the lock
32 to the first peer that ages it out as malformed. ``?`` is illegal in a hostname, so escaping it along with every
33 out-of-grammar byte as ``?<hex>`` leaves a real name untouched and still keeps two hosts apart, which
34 :func:`owner_is_stale` relies on to refuse to probe a foreign PID.
36 Linux appends the PID namespace, as Mercurial's lock prefix does. Two containers in one pod share the hostname and
37 the boot id but not the PID namespace, so a PID recorded in one names another process, or none, in the other. The
38 suffix starts with a ``?`` no escape can produce, so ``a`` in a namespace never reads as a host literally named
39 ``a?pidns-...``. The initial namespace adds nothing, which keeps a host's name what older releases recorded.
41 An over-long name ends in a digest of the part cut to fit the limit. A bare cut gives every name that shares the
42 kept prefix one identity, and :func:`owner_is_stale` would then probe a local PID for another host's holder.
43 """
44 name = (
45 "".join(
46 chr(byte) if byte in _HOST_NAME_VERBATIM else f"?{byte:02x}"
47 for byte in socket.gethostname().encode("utf-8", "surrogateescape")
48 )
49 + _pid_namespace_suffix()
50 )
51 if len(name) > _HOST_NAME_LIMIT:
52 keep = _HOST_NAME_LIMIT - _HOST_NAME_DIGEST_LEN - 1
53 name = f"{name[:keep]}-{hashlib.sha256(name[keep:].encode()).hexdigest()[:_HOST_NAME_DIGEST_LEN]}"
54 # An escape is three characters and a kept byte is never '?', so a bare '?' can only mean an empty hostname.
55 return name or "?"
58def _pid_namespace_suffix() -> str:
59 try:
60 namespace: Final = Path("/proc/self/ns/pid").stat().st_ino
61 except OSError: # only Linux 3.8+ has the file, and a sandbox may hide it; nothing else tells namespaces apart
62 return ""
63 return "" if namespace == _INITIAL_PID_NAMESPACE else f"{_NAMESPACE_MARK}{namespace:x}"
66def owner_is_stale(pid: int, hostname: str, start_token: int | None) -> bool:
67 """
68 Whether the recorded owner is provably gone, so reclaiming its marker cannot detach a live holder.
70 Fail closed: return ``True`` only when this process can prove the exact recorded owner is dead. A marker from
71 another host cannot be probed; a live PID whose start token still matches, or whose token cannot be read, is the
72 holder or is indistinguishable from it; a live PID whose start token differs is a recycled PID, so the process that
73 wrote the marker is gone. PostgreSQL, Qt ``QLockFile`` and Mercurial all break a stale lock only on proof of death
74 and treat an unreadable or foreign owner as still holding.
75 """
76 if hostname != host_name():
77 return False
78 if process_alive(pid) and (
79 start_token is None or (current := process_start_token(pid)) is None or _same_start(current, start_token)
80 ):
81 return False
82 # Only a bare hostname can be an older release's, written from a container that shares this host's name.
83 return _NAMESPACE_MARK in hostname or not _owner_runs_elsewhere(pid, start_token)
86def owner_is_current_process(pid: int, hostname: str, start_token: int | None) -> bool:
87 """
88 Whether the recorded owner is this process, so a marker a recycled PID left behind never reads as ours.
90 Fail closed: a recorded start token must match this process's. One that differs was written by an earlier process
91 that reused this PID, and when this process cannot read its own token nothing proves the marker is ours. A marker
92 without a token falls back to the PID and hostname.
93 """
94 if pid != os.getpid() or hostname != host_name():
95 return False
96 return start_token is None or start_token == process_start_token(pid)
99def _same_start(current: int, recorded: int) -> bool:
100 # A Linux token carries the boot id in its high 64 bits, or 0 there when the process could not read the boot id.
101 # systemd's ProcSubset=pid hides /proc/sys from one unit, so one process may lack the boot id while its peer has
102 # it. A missing boot id then proves nothing, and only starttime is compared. Other platforms' tokens fit in the
103 # low bits.
104 return current == recorded or (0 in {current >> 64, recorded >> 64} and current % 2**64 == recorded % 2**64)
107if sys.platform == "win32": # pragma: win32 cover
108 import ctypes
109 from ctypes import wintypes
111 _KERNEL32: Final[ctypes.WinDLL] = ctypes.WinDLL("kernel32", use_last_error=True)
112 _KERNEL32.CloseHandle.argtypes = [wintypes.HANDLE]
113 _KERNEL32.CloseHandle.restype = wintypes.BOOL
114 _KERNEL32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
115 _KERNEL32.OpenProcess.restype = wintypes.HANDLE
116 _KERNEL32.GetProcessTimes.argtypes = [
117 wintypes.HANDLE,
118 ctypes.POINTER(wintypes.FILETIME),
119 ctypes.POINTER(wintypes.FILETIME),
120 ctypes.POINTER(wintypes.FILETIME),
121 ctypes.POINTER(wintypes.FILETIME),
122 ]
123 _KERNEL32.GetProcessTimes.restype = wintypes.BOOL
124 _KERNEL32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD]
125 _KERNEL32.WaitForSingleObject.restype = wintypes.DWORD
127 _WIN_SYNCHRONIZE: Final[int] = 0x100000
128 _WIN_PROCESS_QUERY_LIMITED_INFORMATION: Final[int] = 0x1000
129 _WIN_ERROR_INVALID_PARAMETER: Final[int] = 87
130 _WIN_INHERIT_HANDLE: Final[bool] = False
131 _WIN_WAIT_OBJECT_0: Final[int] = 0
133 def process_alive(pid: int) -> bool:
134 """Whether a process with this PID still runs, treating an access denial as proof it does."""
135 handle = _KERNEL32.OpenProcess(_WIN_SYNCHRONIZE, _WIN_INHERIT_HANDLE, pid)
136 if handle:
137 try:
138 # Any open handle keeps an exited process openable, but its process object is signaled once it exits.
139 return _KERNEL32.WaitForSingleObject(handle, 0) != _WIN_WAIT_OBJECT_0
140 finally:
141 _KERNEL32.CloseHandle(handle)
142 return ctypes.get_last_error() != _WIN_ERROR_INVALID_PARAMETER
144 def process_start_token(pid: int) -> int | None:
145 """The process creation FILETIME as a 100ns tick count, or ``None`` when it cannot be read."""
146 handle = _KERNEL32.OpenProcess(_WIN_PROCESS_QUERY_LIMITED_INFORMATION, _WIN_INHERIT_HANDLE, pid)
147 if not handle:
148 return None
149 creation, exit_time, kernel_time, user_time = (wintypes.FILETIME() for _ in range(4))
150 try:
151 if not _KERNEL32.GetProcessTimes(
152 handle,
153 ctypes.byref(creation),
154 ctypes.byref(exit_time),
155 ctypes.byref(kernel_time),
156 ctypes.byref(user_time),
157 ):
158 return None # pragma: no cover # win32 GetProcessTimes failure path; not reproducible on a live handle
159 finally:
160 _KERNEL32.CloseHandle(handle)
161 return (creation.dwHighDateTime << 32) | creation.dwLowDateTime
163else: # pragma: win32 no cover
165 def process_alive(pid: int) -> bool:
166 """Whether a process with this PID exists, treating an access denial (``EPERM``) as proof it does."""
167 try:
168 os.kill(pid, 0)
169 except OSError as error:
170 if error.errno == ESRCH:
171 return False
172 if error.errno == EPERM:
173 return True
174 raise
175 return True
177 if sys.platform in {"linux", "android"}: # pragma: linux cover
178 # Termux/Android reports sys.platform == "android" but runs the Linux kernel, so /proc/<pid>/stat and the boot
179 # id are the same reliable start-time source; treat it exactly like Linux rather than the tokenless fallback.
180 # comm (field 2) is wrapped in parentheses and may itself contain spaces or a ')', so the fixed fields start
181 # after the final ')'. starttime is field 22 overall, the twentieth of those trailing fields (index 19).
182 _STARTTIME_INDEX: Final[int] = 19
184 def _read_boot_id() -> int:
185 # starttime is measured in clock ticks since boot, so on its own it repeats across a reboot. Folding the
186 # boot id into the high bits makes the Linux token reboot-safe like the absolute clocks macOS and Windows
187 # expose, while staying a single integer so a 3.29 reader still parses the third marker line. 0 when this
188 # process cannot read the boot id leaves the bare starttime, which _same_start compares alone.
189 try:
190 boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii")
191 return int(boot_id.strip().replace("-", ""), 16)
192 except (OSError, ValueError):
193 return 0
195 _BOOT_ID: Final[int] = _read_boot_id()
196 #: ``TIME_NS_INIT_INO`` in ``linux/proc_ns.h``, fixed by the kernel ABI like the initial PID namespace's inode.
197 _INITIAL_TIME_NAMESPACE: Final[int] = 0xEFFFFFFA
199 def process_start_token(pid: int) -> int | None:
200 """
201 The ``/proc/<pid>/stat`` ``starttime`` folded with the boot id.
203 ``None`` when the process is gone, or when this process runs in a time namespace with a boottime offset.
204 Since Linux 5.6 the kernel shifts ``starttime`` by the reader's offset, so a value read there differs
205 from what a peer outside reads.
206 """
207 return _read_start_token(pid) if _starttime_unshifted() else None
209 def _starttime_unshifted() -> bool:
210 # setns can move this process into another time namespace, so ask on every read.
211 try:
212 namespace: Final = Path("/proc/self/ns/time").stat().st_ino
213 except FileNotFoundError: # no entry means no time namespaces on this kernel
214 return True
215 except OSError: # any other failure leaves the namespace unknown, so starttime is not comparable
216 return False
217 if namespace == _INITIAL_TIME_NAMESPACE:
218 return True
219 # A container runtime may give each container a time namespace without offsets. timens_offsets describes
220 # the namespace this process's children get, its own unless it unshared one since.
221 try:
222 if Path("/proc/self/ns/time_for_children").stat().st_ino != namespace:
223 return False
224 offsets: Final = Path("/proc/self/timens_offsets").read_text(encoding="ascii")
225 except OSError:
226 return False
227 # The clock goes by name since Linux 5.8 and by clock id before.
228 return any(line.split() in (["boottime", "0", "0"], ["7", "0", "0"]) for line in offsets.splitlines())
230 def _read_start_token(pid: int) -> int | None:
231 try:
232 data = Path(f"/proc/{pid}/stat").read_bytes()
233 except OSError:
234 return None
235 # psutil identifies a process by the same (pid, starttime) pair; the boot id extends that across reboots.
236 fields = data[data.rfind(b")") + 1 :].split()
237 if len(fields) <= _STARTTIME_INDEX: # pragma: no cover # a truncated /proc read, never seen in practice
238 return None
239 try:
240 starttime = int(fields[_STARTTIME_INDEX])
241 except ValueError: # pragma: no cover # /proc always renders starttime as an integer
242 return None
243 return (_BOOT_ID << 64) | starttime
245 def _owner_runs_elsewhere(pid: int, start_token: int | None) -> bool:
246 """
247 Whether the owner runs in another PID namespace as *pid*.
249 An older release in a container sharing this hostname recorded its in-container PID, which means nothing
250 here. Readers without a boottime offset read the same starttime and boot id. A task with the recorded
251 start token whose innermost PID is *pid* is therefore the owner; one that started in the same tick is not.
252 A reader with an offset cannot compare start tokens, so the innermost PID alone keeps the marker.
253 """
254 if start_token is None or (_BOOT_ID and start_token >> 64 not in {0, _BOOT_ID}):
255 return False
256 unshifted: Final = _starttime_unshifted()
257 try:
258 with os.scandir("/proc") as entries:
259 return any(
260 entry.name.isdigit()
261 and (task := int(entry.name)) != pid
262 and (
263 not unshifted
264 or ((token := _read_start_token(task)) is not None and _same_start(token, start_token))
265 )
266 and _innermost_pid(task) == pid
267 for entry in entries
268 )
269 except OSError:
270 # A /proc this process cannot list proves nothing either way, which leaves 4.0.12's verdict.
271 return False
273 def _innermost_pid(task: int) -> int | None:
274 try:
275 status: Final = Path(f"/proc/{task}/status").read_bytes()
276 except OSError: # pragma: no cover # the task exited between the stat read and this one
277 return None
278 # The kernel ends a line only with "\n". A task names itself, and its Name line keeps "\r", "\v" and the
279 # other separators that a text-mode read and str.splitlines() split at. A name could then forge a line.
280 return next(
281 (int(line.split()[-1]) for line in status.split(b"\n") if line.startswith(b"NSpid:")),
282 None, # kernels before 4.1 record no namespace PIDs
283 )
285 elif sys.platform == "darwin": # pragma: darwin cover
286 import ctypes
287 import struct
289 _LIBC: Final[ctypes.CDLL] = ctypes.CDLL(None, use_errno=True)
290 _CTL_KERN: Final[int] = 1
291 _KERN_PROC: Final[int] = 14
292 _KERN_PROC_PID: Final[int] = 1
293 # kinfo_proc opens with kp_proc.p_starttime (a struct timeval) at offset 0: int64 seconds, int32 microseconds.
294 # The offset is fixed by the struct chain kinfo_proc -> extern_proc -> p_un, so a read at 0 is not a guess.
295 _TIMEVAL_AT_ZERO: Final[str] = "<qi"
296 _TIMEVAL_SIZE: Final[int] = struct.calcsize(_TIMEVAL_AT_ZERO)
298 def process_start_token(pid: int) -> int | None:
299 """The process start time in microseconds from ``sysctl(KERN_PROC_PID)``, or ``None`` when it is gone."""
300 mib = (ctypes.c_int * 4)(_CTL_KERN, _KERN_PROC, _KERN_PROC_PID, pid)
301 length = ctypes.c_size_t(0)
302 # The size probe reports the kinfo_proc size for any PID, so the read below, not the probe, tells a live
303 # process from a gone one: a gone PID leaves the fetch a zero-length success, so re-check the length after.
304 if _LIBC.sysctl(mib, 4, None, ctypes.byref(length), None, 0) != 0:
305 return None
306 buffer = (ctypes.c_char * length.value)()
307 if _LIBC.sysctl(mib, 4, buffer, ctypes.byref(length), None, 0) != 0 or length.value < _TIMEVAL_SIZE:
308 return None
309 seconds, microseconds = struct.unpack_from(_TIMEVAL_AT_ZERO, buffer.raw, 0)
310 return seconds * 1_000_000 + microseconds
312 else: # pragma: no cover # a POSIX platform without a proven start-time source falls back to fail-closed liveness
314 def process_start_token(pid: int) -> int | None:
315 """No proven start-time source, so the owner carries no token and liveness rests on the PID alone."""
316 del pid
317 return None
320if sys.platform not in {"linux", "android"}: # pragma: linux no cover # pragma: android no cover
322 def _owner_runs_elsewhere(pid: int, start_token: int | None) -> bool:
323 """Without Linux PID namespaces, a PID that is gone or recycled leaves its owner nowhere else to run."""
324 del pid, start_token
325 return False
328__all__ = [
329 "host_name",
330 "owner_is_current_process",
331 "owner_is_stale",
332 "process_alive",
333 "process_start_token",
334]