/src/qemu/util/userfaultfd.c
Line | Count | Source |
1 | | /* |
2 | | * Linux UFFD-WP support |
3 | | * |
4 | | * Copyright Virtuozzo GmbH, 2020 |
5 | | * |
6 | | * Authors: |
7 | | * Andrey Gruzdev <andrey.gruzdev@virtuozzo.com> |
8 | | * |
9 | | * This work is licensed under the terms of the GNU GPL, version 2 or |
10 | | * later. See the COPYING file in the top-level directory. |
11 | | */ |
12 | | |
13 | | #include "qemu/osdep.h" |
14 | | #include "qemu/bitops.h" |
15 | | #include "qemu/error-report.h" |
16 | | #include "qemu/userfaultfd.h" |
17 | | #include "trace.h" |
18 | | #include <poll.h> |
19 | | #include <sys/syscall.h> |
20 | | #include <sys/ioctl.h> |
21 | | |
22 | | typedef enum { |
23 | | UFFD_UNINITIALIZED = 0, |
24 | | UFFD_USE_DEV_PATH, |
25 | | UFFD_USE_SYSCALL, |
26 | | } uffd_open_mode; |
27 | | |
28 | | int uffd_open(int flags) |
29 | 0 | { |
30 | 0 | #if defined(__NR_userfaultfd) |
31 | 0 | static uffd_open_mode open_mode; |
32 | 0 | static int uffd_dev; |
33 | | |
34 | | /* Detect how to generate uffd desc when run the 1st time */ |
35 | 0 | if (open_mode == UFFD_UNINITIALIZED) { |
36 | | /* |
37 | | * Make /dev/userfaultfd the default approach because it has better |
38 | | * permission controls, meanwhile allows kernel faults without any |
39 | | * privilege requirement (e.g. SYS_CAP_PTRACE). |
40 | | */ |
41 | 0 | uffd_dev = open("/dev/userfaultfd", O_RDWR | O_CLOEXEC); |
42 | 0 | if (uffd_dev >= 0) { |
43 | 0 | open_mode = UFFD_USE_DEV_PATH; |
44 | 0 | } else { |
45 | | /* Fallback to the system call */ |
46 | 0 | open_mode = UFFD_USE_SYSCALL; |
47 | 0 | } |
48 | 0 | trace_uffd_detect_open_mode(open_mode); |
49 | 0 | } |
50 | |
|
51 | 0 | if (open_mode == UFFD_USE_DEV_PATH) { |
52 | 0 | assert(uffd_dev >= 0); |
53 | 0 | return ioctl(uffd_dev, USERFAULTFD_IOC_NEW, flags); |
54 | 0 | } |
55 | | |
56 | 0 | return syscall(__NR_userfaultfd, flags); |
57 | | #else |
58 | | return -EINVAL; |
59 | | #endif |
60 | 0 | } |
61 | | |
62 | | /** |
63 | | * uffd_query_features: query UFFD features |
64 | | * |
65 | | * Returns: 0 on success, negative value in case of an error |
66 | | * |
67 | | * @features: parameter to receive 'uffdio_api.features' |
68 | | */ |
69 | | int uffd_query_features(uint64_t *features) |
70 | 0 | { |
71 | 0 | int uffd_fd; |
72 | 0 | struct uffdio_api api_struct = { 0 }; |
73 | 0 | int ret = -1; |
74 | |
|
75 | 0 | uffd_fd = uffd_open(O_CLOEXEC); |
76 | 0 | if (uffd_fd < 0) { |
77 | 0 | trace_uffd_query_features_nosys(errno); |
78 | 0 | return -1; |
79 | 0 | } |
80 | | |
81 | 0 | api_struct.api = UFFD_API; |
82 | 0 | api_struct.features = 0; |
83 | |
|
84 | 0 | if (ioctl(uffd_fd, UFFDIO_API, &api_struct)) { |
85 | 0 | trace_uffd_query_features_api_failed(errno); |
86 | 0 | goto out; |
87 | 0 | } |
88 | 0 | *features = api_struct.features; |
89 | 0 | ret = 0; |
90 | |
|
91 | 0 | out: |
92 | 0 | close(uffd_fd); |
93 | 0 | return ret; |
94 | 0 | } |
95 | | |
96 | | /** |
97 | | * uffd_create_fd: create UFFD file descriptor |
98 | | * |
99 | | * Returns non-negative file descriptor or negative value in case of an error |
100 | | * |
101 | | * @features: UFFD features to request |
102 | | * @non_blocking: create UFFD file descriptor for non-blocking operation |
103 | | */ |
104 | | int uffd_create_fd(uint64_t features, bool non_blocking) |
105 | 0 | { |
106 | 0 | int uffd_fd; |
107 | 0 | int flags; |
108 | 0 | struct uffdio_api api_struct = { 0 }; |
109 | 0 | uint64_t ioctl_mask = BIT(_UFFDIO_REGISTER) | BIT(_UFFDIO_UNREGISTER); |
110 | |
|
111 | 0 | flags = O_CLOEXEC | (non_blocking ? O_NONBLOCK : 0); |
112 | 0 | uffd_fd = uffd_open(flags); |
113 | 0 | if (uffd_fd < 0) { |
114 | 0 | trace_uffd_create_fd_nosys(errno); |
115 | 0 | return -1; |
116 | 0 | } |
117 | | |
118 | 0 | api_struct.api = UFFD_API; |
119 | 0 | api_struct.features = features; |
120 | 0 | if (ioctl(uffd_fd, UFFDIO_API, &api_struct)) { |
121 | 0 | trace_uffd_create_fd_api_failed(errno); |
122 | 0 | goto fail; |
123 | 0 | } |
124 | 0 | if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) { |
125 | 0 | trace_uffd_create_fd_api_noioctl(ioctl_mask, api_struct.ioctls); |
126 | 0 | goto fail; |
127 | 0 | } |
128 | | |
129 | 0 | return uffd_fd; |
130 | | |
131 | 0 | fail: |
132 | 0 | close(uffd_fd); |
133 | 0 | return -1; |
134 | 0 | } |
135 | | |
136 | | /** |
137 | | * uffd_close_fd: close UFFD file descriptor |
138 | | * |
139 | | * @uffd_fd: UFFD file descriptor |
140 | | */ |
141 | | void uffd_close_fd(int uffd_fd) |
142 | 0 | { |
143 | 0 | assert(uffd_fd >= 0); |
144 | 0 | close(uffd_fd); |
145 | 0 | } |
146 | | |
147 | | /** |
148 | | * uffd_register_memory: register memory range via UFFD-IO |
149 | | * |
150 | | * Returns 0 in case of success, negative value in case of an error |
151 | | * |
152 | | * @uffd_fd: UFFD file descriptor |
153 | | * @addr: base address of memory range |
154 | | * @length: length of memory range |
155 | | * @mode: UFFD register mode (UFFDIO_REGISTER_MODE_MISSING, ...) |
156 | | * @ioctls: optional pointer to receive supported IOCTL mask |
157 | | */ |
158 | | int uffd_register_memory(int uffd_fd, void *addr, uint64_t length, |
159 | | uint64_t mode, uint64_t *ioctls) |
160 | 0 | { |
161 | 0 | struct uffdio_register uffd_register; |
162 | |
|
163 | 0 | uffd_register.range.start = (uintptr_t) addr; |
164 | 0 | uffd_register.range.len = length; |
165 | 0 | uffd_register.mode = mode; |
166 | |
|
167 | 0 | if (ioctl(uffd_fd, UFFDIO_REGISTER, &uffd_register)) { |
168 | 0 | trace_uffd_register_memory_failed(addr, length, mode, errno); |
169 | 0 | return -1; |
170 | 0 | } |
171 | 0 | if (ioctls) { |
172 | 0 | *ioctls = uffd_register.ioctls; |
173 | 0 | } |
174 | |
|
175 | 0 | return 0; |
176 | 0 | } |
177 | | |
178 | | /** |
179 | | * uffd_unregister_memory: un-register memory range with UFFD-IO |
180 | | * |
181 | | * Returns 0 in case of success, negative value in case of an error |
182 | | * |
183 | | * @uffd_fd: UFFD file descriptor |
184 | | * @addr: base address of memory range |
185 | | * @length: length of memory range |
186 | | */ |
187 | | int uffd_unregister_memory(int uffd_fd, void *addr, uint64_t length) |
188 | 0 | { |
189 | 0 | struct uffdio_range uffd_range; |
190 | |
|
191 | 0 | uffd_range.start = (uintptr_t) addr; |
192 | 0 | uffd_range.len = length; |
193 | |
|
194 | 0 | if (ioctl(uffd_fd, UFFDIO_UNREGISTER, &uffd_range)) { |
195 | 0 | trace_uffd_unregister_memory_failed(addr, length, errno); |
196 | 0 | return -1; |
197 | 0 | } |
198 | | |
199 | 0 | return 0; |
200 | 0 | } |
201 | | |
202 | | /** |
203 | | * uffd_change_protection: protect/un-protect memory range for writes via UFFD-IO |
204 | | * |
205 | | * Returns 0 on success, negative value in case of error |
206 | | * |
207 | | * @uffd_fd: UFFD file descriptor |
208 | | * @addr: base address of memory range |
209 | | * @length: length of memory range |
210 | | * @wp: write-protect/unprotect |
211 | | * @dont_wake: do not wake threads waiting on wr-protected page |
212 | | */ |
213 | | int uffd_change_protection(int uffd_fd, void *addr, uint64_t length, |
214 | | bool wp, bool dont_wake) |
215 | 0 | { |
216 | 0 | struct uffdio_writeprotect uffd_writeprotect; |
217 | |
|
218 | 0 | uffd_writeprotect.range.start = (uintptr_t) addr; |
219 | 0 | uffd_writeprotect.range.len = length; |
220 | 0 | if (!wp && dont_wake) { |
221 | | /* DONTWAKE is meaningful only on protection release */ |
222 | 0 | uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_DONTWAKE; |
223 | 0 | } else { |
224 | 0 | uffd_writeprotect.mode = (wp ? UFFDIO_WRITEPROTECT_MODE_WP : 0); |
225 | 0 | } |
226 | |
|
227 | 0 | if (ioctl(uffd_fd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) { |
228 | 0 | error_report("uffd_change_protection() failed: addr=%p len=%" PRIu64 |
229 | 0 | " mode=%" PRIx64 " errno=%i", addr, length, |
230 | 0 | (uint64_t) uffd_writeprotect.mode, errno); |
231 | 0 | return -1; |
232 | 0 | } |
233 | | |
234 | 0 | return 0; |
235 | 0 | } |
236 | | |
237 | | /** |
238 | | * uffd_copy_page: copy range of pages to destination via UFFD-IO |
239 | | * |
240 | | * Copy range of source pages to the destination to resolve |
241 | | * missing page fault somewhere in the destination range. |
242 | | * |
243 | | * Returns 0 on success, -errno in case of an error |
244 | | * |
245 | | * @uffd_fd: UFFD file descriptor |
246 | | * @dst_addr: destination base address |
247 | | * @src_addr: source base address |
248 | | * @length: length of the range to copy |
249 | | * @dont_wake: do not wake threads waiting on missing page |
250 | | */ |
251 | | int uffd_copy_page(int uffd_fd, void *dst_addr, void *src_addr, |
252 | | uint64_t length, bool dont_wake) |
253 | 0 | { |
254 | 0 | struct uffdio_copy uffd_copy; |
255 | |
|
256 | 0 | uffd_copy.dst = (uintptr_t) dst_addr; |
257 | 0 | uffd_copy.src = (uintptr_t) src_addr; |
258 | 0 | uffd_copy.len = length; |
259 | 0 | uffd_copy.mode = dont_wake ? UFFDIO_COPY_MODE_DONTWAKE : 0; |
260 | |
|
261 | 0 | if (ioctl(uffd_fd, UFFDIO_COPY, &uffd_copy)) { |
262 | 0 | int e = errno; |
263 | 0 | error_report("uffd_copy_page() failed: dst_addr=%p src_addr=%p length=%" PRIu64 |
264 | 0 | " mode=%" PRIx64 " errno=%i", dst_addr, src_addr, |
265 | 0 | length, (uint64_t) uffd_copy.mode, e); |
266 | 0 | return -e; |
267 | 0 | } |
268 | | |
269 | 0 | return 0; |
270 | 0 | } |
271 | | |
272 | | /** |
273 | | * uffd_zero_page: fill range of pages with zeroes via UFFD-IO |
274 | | * |
275 | | * Fill range pages with zeroes to resolve missing page fault within the range. |
276 | | * |
277 | | * Returns 0 on success, -errno in case of an error |
278 | | * |
279 | | * @uffd_fd: UFFD file descriptor |
280 | | * @addr: base address |
281 | | * @length: length of the range to fill with zeroes |
282 | | * @dont_wake: do not wake threads waiting on missing page |
283 | | */ |
284 | | int uffd_zero_page(int uffd_fd, void *addr, uint64_t length, bool dont_wake) |
285 | 0 | { |
286 | 0 | struct uffdio_zeropage uffd_zeropage; |
287 | |
|
288 | 0 | uffd_zeropage.range.start = (uintptr_t) addr; |
289 | 0 | uffd_zeropage.range.len = length; |
290 | 0 | uffd_zeropage.mode = dont_wake ? UFFDIO_ZEROPAGE_MODE_DONTWAKE : 0; |
291 | |
|
292 | 0 | if (ioctl(uffd_fd, UFFDIO_ZEROPAGE, &uffd_zeropage)) { |
293 | 0 | int e = errno; |
294 | 0 | error_report("uffd_zero_page() failed: addr=%p length=%" PRIu64 |
295 | 0 | " mode=%" PRIx64 " errno=%i", addr, length, |
296 | 0 | (uint64_t) uffd_zeropage.mode, e); |
297 | 0 | return -e; |
298 | 0 | } |
299 | | |
300 | 0 | return 0; |
301 | 0 | } |
302 | | |
303 | | /** |
304 | | * uffd_wakeup: wake up threads waiting on page UFFD-managed page fault resolution |
305 | | * |
306 | | * Wake up threads waiting on any page/pages from the designated range. |
307 | | * The main use case is when during some period, page faults are resolved |
308 | | * via UFFD-IO IOCTLs with MODE_DONTWAKE flag set, then after that all waits |
309 | | * for the whole memory range are satisfied in a single call to uffd_wakeup(). |
310 | | * |
311 | | * Returns 0 on success, -errno in case of an error |
312 | | * |
313 | | * @uffd_fd: UFFD file descriptor |
314 | | * @addr: base address |
315 | | * @length: length of the range |
316 | | */ |
317 | | int uffd_wakeup(int uffd_fd, void *addr, uint64_t length) |
318 | 0 | { |
319 | 0 | struct uffdio_range uffd_range; |
320 | |
|
321 | 0 | uffd_range.start = (uintptr_t) addr; |
322 | 0 | uffd_range.len = length; |
323 | |
|
324 | 0 | if (ioctl(uffd_fd, UFFDIO_WAKE, &uffd_range)) { |
325 | 0 | int e = errno; |
326 | 0 | error_report("uffd_wakeup() failed: addr=%p length=%" PRIu64 " errno=%i", |
327 | 0 | addr, length, e); |
328 | 0 | return -e; |
329 | 0 | } |
330 | | |
331 | 0 | return 0; |
332 | 0 | } |
333 | | |
334 | | /** |
335 | | * uffd_read_events: read pending UFFD events |
336 | | * |
337 | | * Returns number of fetched messages, 0 if non is available or |
338 | | * negative value in case of an error |
339 | | * |
340 | | * @uffd_fd: UFFD file descriptor |
341 | | * @msgs: pointer to message buffer |
342 | | * @count: number of messages that can fit in the buffer |
343 | | */ |
344 | | int uffd_read_events(int uffd_fd, struct uffd_msg *msgs, int count) |
345 | 0 | { |
346 | 0 | ssize_t res; |
347 | 0 | do { |
348 | 0 | res = read(uffd_fd, msgs, count * sizeof(struct uffd_msg)); |
349 | 0 | } while (res < 0 && errno == EINTR); |
350 | |
|
351 | 0 | if ((res < 0 && errno == EAGAIN)) { |
352 | 0 | return 0; |
353 | 0 | } |
354 | 0 | if (res < 0) { |
355 | 0 | error_report("uffd_read_events() failed: errno=%i", errno); |
356 | 0 | return -1; |
357 | 0 | } |
358 | | |
359 | 0 | return (int) (res / sizeof(struct uffd_msg)); |
360 | 0 | } |