Coverage Report

Created: 2026-10-31 08:12

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qemu/util/userfaultfd.c
Line
Count
Source
1
/*
2
 * Linux UFFD-WP support
3
 *
4
 * Copyright Virtuozzo GmbH, 2020
5
 *
6
 * Authors:
7
 *  Andrey Gruzdev   <andrey.gruzdev@virtuozzo.com>
8
 *
9
 * This work is licensed under the terms of the GNU GPL, version 2 or
10
 * later.  See the COPYING file in the top-level directory.
11
 */
12
13
#include "qemu/osdep.h"
14
#include "qemu/bitops.h"
15
#include "qemu/error-report.h"
16
#include "qemu/userfaultfd.h"
17
#include "trace.h"
18
#include <poll.h>
19
#include <sys/syscall.h>
20
#include <sys/ioctl.h>
21
22
typedef enum {
23
    UFFD_UNINITIALIZED = 0,
24
    UFFD_USE_DEV_PATH,
25
    UFFD_USE_SYSCALL,
26
} uffd_open_mode;
27
28
int uffd_open(int flags)
29
0
{
30
0
#if defined(__NR_userfaultfd)
31
0
    static uffd_open_mode open_mode;
32
0
    static int uffd_dev;
33
34
    /* Detect how to generate uffd desc when run the 1st time */
35
0
    if (open_mode == UFFD_UNINITIALIZED) {
36
        /*
37
         * Make /dev/userfaultfd the default approach because it has better
38
         * permission controls, meanwhile allows kernel faults without any
39
         * privilege requirement (e.g. SYS_CAP_PTRACE).
40
         */
41
0
        uffd_dev = open("/dev/userfaultfd", O_RDWR | O_CLOEXEC);
42
0
        if (uffd_dev >= 0) {
43
0
            open_mode = UFFD_USE_DEV_PATH;
44
0
        } else {
45
            /* Fallback to the system call */
46
0
            open_mode = UFFD_USE_SYSCALL;
47
0
        }
48
0
        trace_uffd_detect_open_mode(open_mode);
49
0
    }
50
51
0
    if (open_mode == UFFD_USE_DEV_PATH) {
52
0
        assert(uffd_dev >= 0);
53
0
        return ioctl(uffd_dev, USERFAULTFD_IOC_NEW, flags);
54
0
    }
55
56
0
    return syscall(__NR_userfaultfd, flags);
57
#else
58
    return -EINVAL;
59
#endif
60
0
}
61
62
/**
63
 * uffd_query_features: query UFFD features
64
 *
65
 * Returns: 0 on success, negative value in case of an error
66
 *
67
 * @features: parameter to receive 'uffdio_api.features'
68
 */
69
int uffd_query_features(uint64_t *features)
70
0
{
71
0
    int uffd_fd;
72
0
    struct uffdio_api api_struct = { 0 };
73
0
    int ret = -1;
74
75
0
    uffd_fd = uffd_open(O_CLOEXEC);
76
0
    if (uffd_fd < 0) {
77
0
        trace_uffd_query_features_nosys(errno);
78
0
        return -1;
79
0
    }
80
81
0
    api_struct.api = UFFD_API;
82
0
    api_struct.features = 0;
83
84
0
    if (ioctl(uffd_fd, UFFDIO_API, &api_struct)) {
85
0
        trace_uffd_query_features_api_failed(errno);
86
0
        goto out;
87
0
    }
88
0
    *features = api_struct.features;
89
0
    ret = 0;
90
91
0
out:
92
0
    close(uffd_fd);
93
0
    return ret;
94
0
}
95
96
/**
97
 * uffd_create_fd: create UFFD file descriptor
98
 *
99
 * Returns non-negative file descriptor or negative value in case of an error
100
 *
101
 * @features: UFFD features to request
102
 * @non_blocking: create UFFD file descriptor for non-blocking operation
103
 */
104
int uffd_create_fd(uint64_t features, bool non_blocking)
105
0
{
106
0
    int uffd_fd;
107
0
    int flags;
108
0
    struct uffdio_api api_struct = { 0 };
109
0
    uint64_t ioctl_mask = BIT(_UFFDIO_REGISTER) | BIT(_UFFDIO_UNREGISTER);
110
111
0
    flags = O_CLOEXEC | (non_blocking ? O_NONBLOCK : 0);
112
0
    uffd_fd = uffd_open(flags);
113
0
    if (uffd_fd < 0) {
114
0
        trace_uffd_create_fd_nosys(errno);
115
0
        return -1;
116
0
    }
117
118
0
    api_struct.api = UFFD_API;
119
0
    api_struct.features = features;
120
0
    if (ioctl(uffd_fd, UFFDIO_API, &api_struct)) {
121
0
        trace_uffd_create_fd_api_failed(errno);
122
0
        goto fail;
123
0
    }
124
0
    if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) {
125
0
        trace_uffd_create_fd_api_noioctl(ioctl_mask, api_struct.ioctls);
126
0
        goto fail;
127
0
    }
128
129
0
    return uffd_fd;
130
131
0
fail:
132
0
    close(uffd_fd);
133
0
    return -1;
134
0
}
135
136
/**
137
 * uffd_close_fd: close UFFD file descriptor
138
 *
139
 * @uffd_fd: UFFD file descriptor
140
 */
141
void uffd_close_fd(int uffd_fd)
142
0
{
143
0
    assert(uffd_fd >= 0);
144
0
    close(uffd_fd);
145
0
}
146
147
/**
148
 * uffd_register_memory: register memory range via UFFD-IO
149
 *
150
 * Returns 0 in case of success, negative value in case of an error
151
 *
152
 * @uffd_fd: UFFD file descriptor
153
 * @addr: base address of memory range
154
 * @length: length of memory range
155
 * @mode: UFFD register mode (UFFDIO_REGISTER_MODE_MISSING, ...)
156
 * @ioctls: optional pointer to receive supported IOCTL mask
157
 */
158
int uffd_register_memory(int uffd_fd, void *addr, uint64_t length,
159
        uint64_t mode, uint64_t *ioctls)
160
0
{
161
0
    struct uffdio_register uffd_register;
162
163
0
    uffd_register.range.start = (uintptr_t) addr;
164
0
    uffd_register.range.len = length;
165
0
    uffd_register.mode = mode;
166
167
0
    if (ioctl(uffd_fd, UFFDIO_REGISTER, &uffd_register)) {
168
0
        trace_uffd_register_memory_failed(addr, length, mode, errno);
169
0
        return -1;
170
0
    }
171
0
    if (ioctls) {
172
0
        *ioctls = uffd_register.ioctls;
173
0
    }
174
175
0
    return 0;
176
0
}
177
178
/**
179
 * uffd_unregister_memory: un-register memory range with UFFD-IO
180
 *
181
 * Returns 0 in case of success, negative value in case of an error
182
 *
183
 * @uffd_fd: UFFD file descriptor
184
 * @addr: base address of memory range
185
 * @length: length of memory range
186
 */
187
int uffd_unregister_memory(int uffd_fd, void *addr, uint64_t length)
188
0
{
189
0
    struct uffdio_range uffd_range;
190
191
0
    uffd_range.start = (uintptr_t) addr;
192
0
    uffd_range.len = length;
193
194
0
    if (ioctl(uffd_fd, UFFDIO_UNREGISTER, &uffd_range)) {
195
0
        trace_uffd_unregister_memory_failed(addr, length, errno);
196
0
        return -1;
197
0
    }
198
199
0
    return 0;
200
0
}
201
202
/**
203
 * uffd_change_protection: protect/un-protect memory range for writes via UFFD-IO
204
 *
205
 * Returns 0 on success, negative value in case of error
206
 *
207
 * @uffd_fd: UFFD file descriptor
208
 * @addr: base address of memory range
209
 * @length: length of memory range
210
 * @wp: write-protect/unprotect
211
 * @dont_wake: do not wake threads waiting on wr-protected page
212
 */
213
int uffd_change_protection(int uffd_fd, void *addr, uint64_t length,
214
        bool wp, bool dont_wake)
215
0
{
216
0
    struct uffdio_writeprotect uffd_writeprotect;
217
218
0
    uffd_writeprotect.range.start = (uintptr_t) addr;
219
0
    uffd_writeprotect.range.len = length;
220
0
    if (!wp && dont_wake) {
221
        /* DONTWAKE is meaningful only on protection release */
222
0
        uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_DONTWAKE;
223
0
    } else {
224
0
        uffd_writeprotect.mode = (wp ? UFFDIO_WRITEPROTECT_MODE_WP : 0);
225
0
    }
226
227
0
    if (ioctl(uffd_fd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) {
228
0
        error_report("uffd_change_protection() failed: addr=%p len=%" PRIu64
229
0
                " mode=%" PRIx64 " errno=%i", addr, length,
230
0
                (uint64_t) uffd_writeprotect.mode, errno);
231
0
        return -1;
232
0
    }
233
234
0
    return 0;
235
0
}
236
237
/**
238
 * uffd_copy_page: copy range of pages to destination via UFFD-IO
239
 *
240
 * Copy range of source pages to the destination to resolve
241
 * missing page fault somewhere in the destination range.
242
 *
243
 * Returns 0 on success, -errno in case of an error
244
 *
245
 * @uffd_fd: UFFD file descriptor
246
 * @dst_addr: destination base address
247
 * @src_addr: source base address
248
 * @length: length of the range to copy
249
 * @dont_wake: do not wake threads waiting on missing page
250
 */
251
int uffd_copy_page(int uffd_fd, void *dst_addr, void *src_addr,
252
        uint64_t length, bool dont_wake)
253
0
{
254
0
    struct uffdio_copy uffd_copy;
255
256
0
    uffd_copy.dst = (uintptr_t) dst_addr;
257
0
    uffd_copy.src = (uintptr_t) src_addr;
258
0
    uffd_copy.len = length;
259
0
    uffd_copy.mode = dont_wake ? UFFDIO_COPY_MODE_DONTWAKE : 0;
260
261
0
    if (ioctl(uffd_fd, UFFDIO_COPY, &uffd_copy)) {
262
0
        int e = errno;
263
0
        error_report("uffd_copy_page() failed: dst_addr=%p src_addr=%p length=%" PRIu64
264
0
                " mode=%" PRIx64 " errno=%i", dst_addr, src_addr,
265
0
                length, (uint64_t) uffd_copy.mode, e);
266
0
        return -e;
267
0
    }
268
269
0
    return 0;
270
0
}
271
272
/**
273
 * uffd_zero_page: fill range of pages with zeroes via UFFD-IO
274
 *
275
 * Fill range pages with zeroes to resolve missing page fault within the range.
276
 *
277
 * Returns 0 on success, -errno in case of an error
278
 *
279
 * @uffd_fd: UFFD file descriptor
280
 * @addr: base address
281
 * @length: length of the range to fill with zeroes
282
 * @dont_wake: do not wake threads waiting on missing page
283
 */
284
int uffd_zero_page(int uffd_fd, void *addr, uint64_t length, bool dont_wake)
285
0
{
286
0
    struct uffdio_zeropage uffd_zeropage;
287
288
0
    uffd_zeropage.range.start = (uintptr_t) addr;
289
0
    uffd_zeropage.range.len = length;
290
0
    uffd_zeropage.mode = dont_wake ? UFFDIO_ZEROPAGE_MODE_DONTWAKE : 0;
291
292
0
    if (ioctl(uffd_fd, UFFDIO_ZEROPAGE, &uffd_zeropage)) {
293
0
        int e = errno;
294
0
        error_report("uffd_zero_page() failed: addr=%p length=%" PRIu64
295
0
                " mode=%" PRIx64 " errno=%i", addr, length,
296
0
                (uint64_t) uffd_zeropage.mode, e);
297
0
        return -e;
298
0
    }
299
300
0
    return 0;
301
0
}
302
303
/**
304
 * uffd_wakeup: wake up threads waiting on page UFFD-managed page fault resolution
305
 *
306
 * Wake up threads waiting on any page/pages from the designated range.
307
 * The main use case is when during some period, page faults are resolved
308
 * via UFFD-IO IOCTLs with MODE_DONTWAKE flag set, then after that all waits
309
 * for the whole memory range are satisfied in a single call to uffd_wakeup().
310
 *
311
 * Returns 0 on success, -errno in case of an error
312
 *
313
 * @uffd_fd: UFFD file descriptor
314
 * @addr: base address
315
 * @length: length of the range
316
 */
317
int uffd_wakeup(int uffd_fd, void *addr, uint64_t length)
318
0
{
319
0
    struct uffdio_range uffd_range;
320
321
0
    uffd_range.start = (uintptr_t) addr;
322
0
    uffd_range.len = length;
323
324
0
    if (ioctl(uffd_fd, UFFDIO_WAKE, &uffd_range)) {
325
0
        int e = errno;
326
0
        error_report("uffd_wakeup() failed: addr=%p length=%" PRIu64 " errno=%i",
327
0
                addr, length, e);
328
0
        return -e;
329
0
    }
330
331
0
    return 0;
332
0
}
333
334
/**
335
 * uffd_read_events: read pending UFFD events
336
 *
337
 * Returns number of fetched messages, 0 if non is available or
338
 * negative value in case of an error
339
 *
340
 * @uffd_fd: UFFD file descriptor
341
 * @msgs: pointer to message buffer
342
 * @count: number of messages that can fit in the buffer
343
 */
344
int uffd_read_events(int uffd_fd, struct uffd_msg *msgs, int count)
345
0
{
346
0
    ssize_t res;
347
0
    do {
348
0
        res = read(uffd_fd, msgs, count * sizeof(struct uffd_msg));
349
0
    } while (res < 0 && errno == EINTR);
350
351
0
    if ((res < 0 && errno == EAGAIN)) {
352
0
        return 0;
353
0
    }
354
0
    if (res < 0) {
355
0
        error_report("uffd_read_events() failed: errno=%i", errno);
356
0
        return -1;
357
0
    }
358
359
0
    return (int) (res / sizeof(struct uffd_msg));
360
0
}