Coverage Report

Created: 2026-11-20 08:13

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qemu/accel/kvm/kvm-all.c
Line
Count
Source
1
/*
2
 * QEMU KVM support
3
 *
4
 * Copyright IBM, Corp. 2008
5
 *           Red Hat, Inc. 2008
6
 *
7
 * Authors:
8
 *  Anthony Liguori   <aliguori@us.ibm.com>
9
 *  Glauber Costa     <gcosta@redhat.com>
10
 *
11
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12
 * See the COPYING file in the top-level directory.
13
 *
14
 */
15
16
#include "qemu/osdep.h"
17
#include <sys/ioctl.h>
18
#include <poll.h>
19
20
#include <linux/kvm.h>
21
22
#include "qemu/atomic.h"
23
#include "qemu/option.h"
24
#include "qemu/config-file.h"
25
#include "qemu/error-report.h"
26
#include "qapi/error.h"
27
#include "qapi/qapi-builtin-type-infos.h"
28
#include "hw/pci/msi.h"
29
#include "hw/pci/msix.h"
30
#include "hw/s390x/adapter.h"
31
#include "gdbstub/enums.h"
32
#include "system/kvm_int.h"
33
#include "system/runstate.h"
34
#include "system/cpus.h"
35
#include "system/accel-blocker.h"
36
#include "system/physmem.h"
37
#include "system/ramblock.h"
38
#include "accel/accel-ops.h"
39
#include "qemu/bswap.h"
40
#include "exec/tswap.h"
41
#include "exec/target_page.h"
42
#include "system/memory.h"
43
#include "qemu/event_notifier.h"
44
#include "qemu/main-loop.h"
45
#include "trace.h"
46
#include "hw/core/irq.h"
47
#include "qapi/visitor.h"
48
#include "qapi/qapi-types-common.h"
49
#include "qapi/qapi-visit-common.h"
50
#include "qapi/qapi-type-infos-common.h"
51
#include "system/reset.h"
52
#include "qemu/guest-random.h"
53
#include "system/hw_accel.h"
54
#include "kvm-cpus.h"
55
#include "system/dirtylimit.h"
56
#include "qemu/range.h"
57
58
#include "hw/core/boards.h"
59
#include "system/stats.h"
60
61
/* This check must be after config-host.h is included */
62
#ifdef CONFIG_EVENTFD
63
#include <sys/eventfd.h>
64
#endif
65
66
#if defined(__x86_64__) || defined(__aarch64__)
67
# define KVM_HAVE_MCE_INJECTION 1
68
#endif
69
70
71
/* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We
72
 * need to use the real host PAGE_SIZE, as that's what KVM will use.
73
 */
74
#ifdef PAGE_SIZE
75
#undef PAGE_SIZE
76
#endif
77
0
#define PAGE_SIZE qemu_real_host_page_size()
78
79
#ifndef KVM_GUESTDBG_BLOCKIRQ
80
#define KVM_GUESTDBG_BLOCKIRQ 0
81
#endif
82
83
/* Default num of memslots to be allocated when VM starts */
84
0
#define  KVM_MEMSLOTS_NR_ALLOC_DEFAULT                      16
85
/* Default max allowed memslots if kernel reported nothing */
86
0
#define  KVM_MEMSLOTS_NR_MAX_DEFAULT                        32
87
88
struct KVMParkedVcpu {
89
    unsigned long vcpu_id;
90
    int kvm_fd;
91
    QLIST_ENTRY(KVMParkedVcpu) node;
92
};
93
94
KVMState *kvm_state;
95
VmfdChangeNotifier vmfd_notifier;
96
bool kvm_kernel_irqchip;
97
bool kvm_split_irqchip;
98
bool kvm_async_interrupts_allowed;
99
bool kvm_halt_in_kernel_allowed;
100
bool kvm_resamplefds_allowed;
101
bool kvm_msi_via_irqfd_allowed;
102
bool kvm_gsi_routing_allowed;
103
bool kvm_gsi_direct_mapping;
104
bool kvm_allowed;
105
bool kvm_readonly_mem_allowed;
106
bool kvm_vm_attributes_allowed;
107
bool kvm_msi_use_devid;
108
bool kvm_pre_fault_memory_supported;
109
static bool kvm_immediate_exit;
110
static uint64_t kvm_supported_memory_attributes;
111
static bool kvm_guest_memfd_supported;
112
static uint64_t kvm_guest_memfd_flags_supported;
113
static hwaddr kvm_max_slot_size = ~0;
114
115
static const KVMCapabilityInfo kvm_required_capabilities[] = {
116
    KVM_CAP_INFO(USER_MEMORY),
117
    KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
118
    KVM_CAP_INFO(JOIN_MEMORY_REGIONS_WORKS),
119
    KVM_CAP_INFO(INTERNAL_ERROR_DATA),
120
    KVM_CAP_INFO(IOEVENTFD),
121
    KVM_CAP_INFO(IOEVENTFD_ANY_LENGTH),
122
    KVM_CAP_LAST_INFO
123
};
124
125
static NotifierList kvm_irqchip_change_notifiers =
126
    NOTIFIER_LIST_INITIALIZER(kvm_irqchip_change_notifiers);
127
128
static NotifierWithReturnList register_vmfd_changed_notifiers =
129
    NOTIFIER_WITH_RETURN_LIST_INITIALIZER(register_vmfd_changed_notifiers);
130
131
static NotifierWithReturnList register_vcpufd_changed_notifiers =
132
    NOTIFIER_WITH_RETURN_LIST_INITIALIZER(register_vcpufd_changed_notifiers);
133
134
static int map_kvm_run(KVMState *s, CPUState *cpu, Error **errp);
135
static int map_kvm_dirty_gfns(KVMState *s, CPUState *cpu, Error **errp);
136
static int vcpu_unmap_regions(KVMState *s, CPUState *cpu);
137
138
struct KVMResampleFd {
139
    int gsi;
140
    EventNotifier *resample_event;
141
    QLIST_ENTRY(KVMResampleFd) node;
142
};
143
typedef struct KVMResampleFd KVMResampleFd;
144
145
/*
146
 * Only used with split irqchip where we need to do the resample fd
147
 * kick for the kernel from userspace.
148
 */
149
static QLIST_HEAD(, KVMResampleFd) kvm_resample_fd_list =
150
    QLIST_HEAD_INITIALIZER(kvm_resample_fd_list);
151
152
static QemuMutex kvm_resample_fd_list_lock;
153
static QemuMutex kml_slots_lock;
154
155
0
#define kvm_slots_lock()    qemu_mutex_lock(&kml_slots_lock)
156
0
#define kvm_slots_unlock()  qemu_mutex_unlock(&kml_slots_lock)
157
158
static void kvm_slot_init_dirty_bitmap(KVMSlot *mem);
159
160
static inline void kvm_resample_fd_remove(int gsi)
161
0
{
162
0
    KVMResampleFd *rfd;
163
164
0
    qemu_mutex_lock(&kvm_resample_fd_list_lock);
165
166
0
    QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) {
167
0
        if (rfd->gsi == gsi) {
168
0
            QLIST_REMOVE(rfd, node);
169
0
            g_free(rfd);
170
0
            break;
171
0
        }
172
0
    }
173
174
0
    qemu_mutex_unlock(&kvm_resample_fd_list_lock);
175
0
}
176
177
static inline void kvm_resample_fd_insert(int gsi, EventNotifier *event)
178
0
{
179
0
    KVMResampleFd *rfd = g_new0(KVMResampleFd, 1);
180
181
0
    rfd->gsi = gsi;
182
0
    rfd->resample_event = event;
183
184
0
    qemu_mutex_lock(&kvm_resample_fd_list_lock);
185
0
    QLIST_INSERT_HEAD(&kvm_resample_fd_list, rfd, node);
186
0
    qemu_mutex_unlock(&kvm_resample_fd_list_lock);
187
0
}
188
189
void kvm_resample_fd_notify(int gsi)
190
0
{
191
0
    KVMResampleFd *rfd;
192
193
0
    qemu_mutex_lock(&kvm_resample_fd_list_lock);
194
195
0
    QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) {
196
0
        if (rfd->gsi == gsi) {
197
0
            event_notifier_set(rfd->resample_event);
198
0
            trace_kvm_resample_fd_notify(gsi);
199
0
            break;
200
0
        }
201
0
    }
202
203
0
    qemu_mutex_unlock(&kvm_resample_fd_list_lock);
204
0
}
205
206
/**
207
 * kvm_slots_grow(): Grow the slots[] array in the KVMMemoryListener
208
 *
209
 * @kml: The KVMMemoryListener* to grow the slots[] array
210
 * @nr_slots_new: The new size of slots[] array
211
 *
212
 * Returns: True if the array grows larger, false otherwise.
213
 */
214
static bool kvm_slots_grow(KVMMemoryListener *kml, unsigned int nr_slots_new)
215
0
{
216
0
    unsigned int i, cur = kml->nr_slots_allocated;
217
0
    KVMSlot *slots;
218
219
0
    if (nr_slots_new > kvm_state->nr_slots_max) {
220
0
        nr_slots_new = kvm_state->nr_slots_max;
221
0
    }
222
223
0
    if (cur >= nr_slots_new) {
224
        /* Big enough, no need to grow, or we reached max */
225
0
        return false;
226
0
    }
227
228
0
    if (cur == 0) {
229
0
        slots = g_new0(KVMSlot, nr_slots_new);
230
0
    } else {
231
0
        assert(kml->slots);
232
0
        slots = g_renew(KVMSlot, kml->slots, nr_slots_new);
233
        /*
234
         * g_renew() doesn't initialize extended buffers, however kvm
235
         * memslots require fields to be zero-initialized. E.g. pointers,
236
         * memory_size field, etc.
237
         */
238
0
        memset(&slots[cur], 0x0, sizeof(slots[0]) * (nr_slots_new - cur));
239
0
    }
240
241
0
    for (i = cur; i < nr_slots_new; i++) {
242
0
        slots[i].slot = i;
243
0
    }
244
245
0
    kml->slots = slots;
246
0
    kml->nr_slots_allocated = nr_slots_new;
247
0
    trace_kvm_slots_grow(cur, nr_slots_new);
248
249
0
    return true;
250
0
}
251
252
static bool kvm_slots_double(KVMMemoryListener *kml)
253
0
{
254
0
    return kvm_slots_grow(kml, kml->nr_slots_allocated * 2);
255
0
}
256
257
unsigned int kvm_get_max_memslots(void)
258
0
{
259
0
    KVMState *s = KVM_STATE(current_accel());
260
261
0
    return s->nr_slots_max;
262
0
}
263
264
unsigned int kvm_get_free_memslots(void)
265
0
{
266
0
    unsigned int used_slots = 0;
267
0
    KVMState *s = kvm_state;
268
0
    int i;
269
270
0
    kvm_slots_lock();
271
0
    for (i = 0; i < s->nr_as; i++) {
272
0
        if (!s->as[i].ml) {
273
0
            continue;
274
0
        }
275
0
        used_slots = MAX(used_slots, s->as[i].ml->nr_slots_used);
276
0
    }
277
0
    kvm_slots_unlock();
278
279
0
    return s->nr_slots_max - used_slots;
280
0
}
281
282
/* Called with KVMMemoryListener.slots_lock held */
283
static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
284
0
{
285
0
    unsigned int n;
286
0
    int i;
287
288
0
    for (i = 0; i < kml->nr_slots_allocated; i++) {
289
0
        if (kml->slots[i].memory_size == 0) {
290
0
            return &kml->slots[i];
291
0
        }
292
0
    }
293
294
    /*
295
     * If no free slots, try to grow first by doubling.  Cache the old size
296
     * here to avoid another round of search: if the grow succeeded, it
297
     * means slots[] now must have the existing "n" slots occupied,
298
     * followed by one or more free slots starting from slots[n].
299
     */
300
0
    n = kml->nr_slots_allocated;
301
0
    if (kvm_slots_double(kml)) {
302
0
        return &kml->slots[n];
303
0
    }
304
305
0
    return NULL;
306
0
}
307
308
/* Called with KVMMemoryListener.slots_lock held */
309
static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml)
310
0
{
311
0
    KVMSlot *slot = kvm_get_free_slot(kml);
312
313
0
    if (slot) {
314
0
        return slot;
315
0
    }
316
317
0
    fprintf(stderr, "%s: no free slot available\n", __func__);
318
0
    abort();
319
0
}
320
321
static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml,
322
                                         hwaddr start_addr,
323
                                         hwaddr size)
324
0
{
325
0
    int i;
326
327
0
    for (i = 0; i < kml->nr_slots_allocated; i++) {
328
0
        KVMSlot *mem = &kml->slots[i];
329
330
0
        if (start_addr == mem->start_addr && size == mem->memory_size) {
331
0
            return mem;
332
0
        }
333
0
    }
334
335
0
    return NULL;
336
0
}
337
338
/*
339
 * Calculate and align the start address and the size of the section.
340
 * Return the size. If the size is 0, the aligned section is empty.
341
 */
342
static hwaddr kvm_align_section(MemoryRegionSection *section,
343
                                hwaddr *start)
344
0
{
345
0
    hwaddr size = int128_get64(section->size);
346
0
    hwaddr delta, aligned;
347
348
    /* kvm works in page size chunks, but the function may be called
349
       with sub-page size and unaligned start address. Pad the start
350
       address to next and truncate size to previous page boundary. */
351
0
    aligned = ROUND_UP(section->offset_within_address_space,
352
0
                       qemu_real_host_page_size());
353
0
    delta = aligned - section->offset_within_address_space;
354
0
    *start = aligned;
355
0
    if (delta > size) {
356
0
        return 0;
357
0
    }
358
359
0
    return (size - delta) & qemu_real_host_page_mask();
360
0
}
361
362
int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
363
                                       hwaddr *phys_addr)
364
0
{
365
0
    KVMMemoryListener *kml = &s->memory_listener;
366
0
    int i, ret = 0;
367
368
0
    kvm_slots_lock();
369
0
    for (i = 0; i < kml->nr_slots_allocated; i++) {
370
0
        KVMSlot *mem = &kml->slots[i];
371
372
0
        if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
373
0
            *phys_addr = mem->start_addr + (ram - mem->ram);
374
0
            ret = 1;
375
0
            break;
376
0
        }
377
0
    }
378
0
    kvm_slots_unlock();
379
380
0
    return ret;
381
0
}
382
383
static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot, bool new)
384
0
{
385
0
    KVMState *s = kvm_state;
386
0
    struct kvm_userspace_memory_region2 mem = {};
387
0
    int ret;
388
389
0
    mem.slot = slot->slot | (kml->as_id << 16);
390
0
    mem.guest_phys_addr = slot->start_addr;
391
0
    mem.userspace_addr = (unsigned long)slot->ram;
392
0
    mem.flags = slot->flags;
393
0
    mem.guest_memfd = slot->guest_memfd;
394
0
    mem.guest_memfd_offset = slot->guest_memfd_offset;
395
396
0
    if (slot->memory_size && !new && (mem.flags ^ slot->old_flags) & KVM_MEM_READONLY) {
397
        /* Set the slot size to 0 before setting the slot to the desired
398
         * value. This is needed based on KVM commit 75d61fbc. */
399
0
        mem.memory_size = 0;
400
401
0
        if (kvm_guest_memfd_supported) {
402
0
            ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION2, &mem);
403
0
        } else {
404
0
            ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
405
0
        }
406
0
        if (ret < 0) {
407
0
            goto err;
408
0
        }
409
0
    }
410
0
    mem.memory_size = slot->memory_size;
411
0
    if (kvm_guest_memfd_supported) {
412
0
        ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION2, &mem);
413
0
    } else {
414
0
        ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
415
0
    }
416
0
    slot->old_flags = mem.flags;
417
0
err:
418
0
    trace_kvm_set_user_memory(mem.slot >> 16, (uint16_t)mem.slot, mem.flags,
419
0
                              mem.guest_phys_addr, mem.memory_size,
420
0
                              mem.userspace_addr, mem.guest_memfd,
421
0
                              mem.guest_memfd_offset, ret);
422
0
    if (ret < 0) {
423
0
        if (kvm_guest_memfd_supported) {
424
0
                error_report("%s: KVM_SET_USER_MEMORY_REGION2 failed, slot=%d,"
425
0
                        " start=0x%" PRIx64 ", size=0x%" PRIx64 ","
426
0
                        " flags=0x%" PRIx32 ", guest_memfd=%" PRId32 ","
427
0
                        " guest_memfd_offset=0x%" PRIx64 ": %s",
428
0
                        __func__, mem.slot, slot->start_addr,
429
0
                        (uint64_t)mem.memory_size, mem.flags,
430
0
                        mem.guest_memfd, (uint64_t)mem.guest_memfd_offset,
431
0
                        strerror(errno));
432
0
        } else {
433
0
                error_report("%s: KVM_SET_USER_MEMORY_REGION failed, slot=%d,"
434
0
                            " start=0x%" PRIx64 ", size=0x%" PRIx64 ": %s",
435
0
                            __func__, mem.slot, slot->start_addr,
436
0
                            (uint64_t)mem.memory_size, strerror(errno));
437
0
        }
438
0
    }
439
0
    return ret;
440
0
}
441
442
static void kvm_create_vcpu_internal(CPUState *cpu, KVMState *s, int kvm_fd)
443
0
{
444
0
    cpu->kvm_fd = kvm_fd;
445
0
    cpu->kvm_state = s;
446
0
    if (!s->guest_state_protected) {
447
0
        cpu->vcpu_dirty = true;
448
0
    }
449
0
    cpu->dirty_pages = 0;
450
0
    cpu->throttle_us_per_full = 0;
451
452
0
    return;
453
0
}
454
455
static int kvm_rebind_vcpus(Error **errp)
456
0
{
457
0
    CPUState *cpu;
458
0
    unsigned long vcpu_id;
459
0
    KVMState *s = kvm_state;
460
0
    int kvm_fd, ret = 0;
461
462
0
    CPU_FOREACH(cpu) {
463
0
        vcpu_id = kvm_arch_vcpu_id(cpu);
464
465
0
        if (cpu->kvm_fd) {
466
0
            close(cpu->kvm_fd);
467
0
        }
468
469
0
        ret = kvm_arch_destroy_vcpu(cpu);
470
0
        if (ret < 0) {
471
0
            goto err;
472
0
        }
473
474
0
        if (s->coalesced_mmio_ring == (void *)cpu->kvm_run + PAGE_SIZE) {
475
0
            s->coalesced_mmio_ring = NULL;
476
0
        }
477
478
0
        ret = vcpu_unmap_regions(s, cpu);
479
0
        if (ret < 0) {
480
0
            goto err;
481
0
        }
482
483
0
        ret = kvm_arch_pre_create_vcpu(cpu, errp);
484
0
        if (ret < 0) {
485
0
            goto err;
486
0
        }
487
488
0
        kvm_fd = kvm_vm_ioctl(s, KVM_CREATE_VCPU, vcpu_id);
489
0
        if (kvm_fd < 0) {
490
0
            error_report("KVM_CREATE_VCPU IOCTL failed for vCPU %lu (%s)",
491
0
                         vcpu_id, strerror(kvm_fd));
492
0
            return kvm_fd;
493
0
        }
494
495
0
        kvm_create_vcpu_internal(cpu, s, kvm_fd);
496
497
0
        ret = map_kvm_run(s, cpu, errp);
498
0
        if (ret < 0) {
499
0
            goto err;
500
0
        }
501
502
0
        if (s->kvm_dirty_ring_size) {
503
0
            ret = map_kvm_dirty_gfns(s, cpu, errp);
504
0
            if (ret < 0) {
505
0
                goto err;
506
0
            }
507
0
        }
508
509
0
        ret = kvm_arch_init_vcpu(cpu, errp);
510
0
        if (ret < 0) {
511
0
            error_prepend(errp,
512
0
                          "kvm_init_vcpu: kvm_arch_init_vcpu failed (%lu)",
513
0
                          vcpu_id);
514
0
        }
515
516
0
        close(cpu->kvm_vcpu_stats_fd);
517
0
        cpu->kvm_vcpu_stats_fd = kvm_vcpu_ioctl(cpu, KVM_GET_STATS_FD, NULL);
518
0
        kvm_init_cpu_signals(cpu);
519
0
    }
520
0
    trace_kvm_rebind_vcpus();
521
522
0
 err:
523
0
    return ret;
524
0
}
525
526
static void kvm_park_vcpu(CPUState *cpu)
527
0
{
528
0
    struct KVMParkedVcpu *vcpu;
529
530
0
    trace_kvm_park_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
531
532
0
    vcpu = g_malloc0(sizeof(*vcpu));
533
0
    vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
534
0
    vcpu->kvm_fd = cpu->kvm_fd;
535
0
    QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
536
0
}
537
538
static int kvm_unpark_vcpu(KVMState *s, unsigned long vcpu_id)
539
0
{
540
0
    struct KVMParkedVcpu *cpu;
541
0
    int kvm_fd = -ENOENT;
542
543
0
    QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
544
0
        if (cpu->vcpu_id == vcpu_id) {
545
0
            QLIST_REMOVE(cpu, node);
546
0
            kvm_fd = cpu->kvm_fd;
547
0
            g_free(cpu);
548
0
            break;
549
0
        }
550
0
    }
551
552
0
    trace_kvm_unpark_vcpu(vcpu_id, kvm_fd > 0 ? "unparked" : "!found parked");
553
554
0
    return kvm_fd;
555
0
}
556
557
static void kvm_reset_parked_vcpus(KVMState *s)
558
0
{
559
0
    struct KVMParkedVcpu *cpu;
560
561
0
    QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
562
0
        kvm_arch_reset_parked_vcpu(cpu->vcpu_id, cpu->kvm_fd);
563
0
    }
564
0
}
565
566
/**
567
 * kvm_create_vcpu - Gets a parked KVM vCPU or creates a KVM vCPU
568
 * @cpu: QOM CPUState object for which KVM vCPU has to be fetched/created.
569
 *
570
 * @returns: 0 when success, errno (<0) when failed.
571
 */
572
static int kvm_create_vcpu(CPUState *cpu)
573
0
{
574
0
    unsigned long vcpu_id = kvm_arch_vcpu_id(cpu);
575
0
    KVMState *s = kvm_state;
576
0
    int kvm_fd;
577
578
    /* check if the KVM vCPU already exist but is parked */
579
0
    kvm_fd = kvm_unpark_vcpu(s, vcpu_id);
580
0
    if (kvm_fd < 0) {
581
        /* vCPU not parked: create a new KVM vCPU */
582
0
        kvm_fd = kvm_vm_ioctl(s, KVM_CREATE_VCPU, vcpu_id);
583
0
        if (kvm_fd < 0) {
584
0
            error_report("KVM_CREATE_VCPU IOCTL failed for vCPU %lu", vcpu_id);
585
0
            return kvm_fd;
586
0
        }
587
0
    }
588
589
0
    kvm_create_vcpu_internal(cpu, s, kvm_fd);
590
591
0
    trace_kvm_create_vcpu(cpu->cpu_index, vcpu_id, kvm_fd);
592
593
0
    return 0;
594
0
}
595
596
int kvm_create_and_park_vcpu(CPUState *cpu)
597
0
{
598
0
    int ret = 0;
599
600
0
    ret = kvm_create_vcpu(cpu);
601
0
    if (!ret) {
602
0
        kvm_park_vcpu(cpu);
603
0
    }
604
605
0
    return ret;
606
0
}
607
608
static int vcpu_unmap_regions(KVMState *s, CPUState *cpu)
609
0
{
610
0
    int mmap_size;
611
0
    int ret = 0;
612
613
0
    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
614
0
    if (mmap_size < 0) {
615
0
        ret = mmap_size;
616
0
        trace_kvm_failed_get_vcpu_mmap_size();
617
0
        goto err;
618
0
    }
619
620
    /* If I am the CPU that created coalesced_mmio_ring, then discard it */
621
0
    if (s->coalesced_mmio_ring ==
622
0
           (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE) {
623
0
        s->coalesced_mmio_ring = NULL;
624
0
    }
625
626
0
    ret = munmap(cpu->kvm_run, mmap_size);
627
0
    if (ret < 0) {
628
0
        goto err;
629
0
    }
630
0
    cpu->kvm_run = NULL;
631
632
0
    if (cpu->kvm_dirty_gfns) {
633
0
        ret = munmap(cpu->kvm_dirty_gfns, s->kvm_dirty_ring_bytes);
634
0
        if (ret < 0) {
635
0
            goto err;
636
0
        }
637
0
        cpu->kvm_dirty_gfns = NULL;
638
0
    }
639
640
0
 err:
641
0
    return ret;
642
0
}
643
644
static int do_kvm_destroy_vcpu(CPUState *cpu)
645
0
{
646
0
    KVMState *s = kvm_state;
647
0
    int ret = 0;
648
649
0
    trace_kvm_destroy_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
650
651
0
    ret = kvm_arch_destroy_vcpu(cpu);
652
0
    if (ret < 0) {
653
0
        goto err;
654
0
    }
655
656
    /* If I am the CPU that created coalesced_mmio_ring, then discard it */
657
0
    if (s->coalesced_mmio_ring == (void *)cpu->kvm_run + PAGE_SIZE) {
658
0
        s->coalesced_mmio_ring = NULL;
659
0
    }
660
661
0
    ret = vcpu_unmap_regions(s, cpu);
662
0
    if (ret < 0) {
663
0
        goto err;
664
0
    }
665
0
    kvm_park_vcpu(cpu);
666
0
err:
667
0
    return ret;
668
0
}
669
670
void kvm_destroy_vcpu(CPUState *cpu)
671
0
{
672
0
    if (do_kvm_destroy_vcpu(cpu) < 0) {
673
0
        error_report("kvm_destroy_vcpu failed");
674
0
        exit(EXIT_FAILURE);
675
0
    }
676
0
}
677
678
static int map_kvm_run(KVMState *s, CPUState *cpu, Error **errp)
679
0
{
680
0
    int mmap_size, ret = 0;
681
682
0
    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
683
0
    if (mmap_size < 0) {
684
0
        ret = mmap_size;
685
0
        error_setg_errno(errp, -mmap_size,
686
0
                         "kvm_init_vcpu: KVM_GET_VCPU_MMAP_SIZE failed");
687
0
        goto err;
688
0
    }
689
690
0
    cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
691
0
                        cpu->kvm_fd, 0);
692
0
    if (cpu->kvm_run == MAP_FAILED) {
693
0
        ret = -errno;
694
0
        error_setg_errno(errp, ret,
695
0
                         "kvm_init_vcpu: mmap'ing vcpu state failed (%lu)",
696
0
                         kvm_arch_vcpu_id(cpu));
697
0
        goto err;
698
0
    }
699
700
0
    if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
701
0
        s->coalesced_mmio_ring =
702
0
            (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
703
0
    }
704
705
0
 err:
706
0
    return ret;
707
0
}
708
709
static int map_kvm_dirty_gfns(KVMState *s, CPUState *cpu, Error **errp)
710
0
{
711
0
    int ret = 0;
712
    /* Use MAP_SHARED to share pages with the kernel */
713
0
    cpu->kvm_dirty_gfns = mmap(NULL, s->kvm_dirty_ring_bytes,
714
0
                               PROT_READ | PROT_WRITE, MAP_SHARED,
715
0
                               cpu->kvm_fd,
716
0
                               PAGE_SIZE * KVM_DIRTY_LOG_PAGE_OFFSET);
717
0
    if (cpu->kvm_dirty_gfns == MAP_FAILED) {
718
0
        ret = -errno;
719
0
    }
720
721
0
    return ret;
722
0
}
723
724
int kvm_init_vcpu(CPUState *cpu, Error **errp)
725
0
{
726
0
    KVMState *s = kvm_state;
727
0
    int ret;
728
729
0
    trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
730
731
0
    ret = kvm_arch_pre_create_vcpu(cpu, errp);
732
0
    if (ret < 0) {
733
0
        goto err;
734
0
    }
735
736
0
    ret = kvm_create_vcpu(cpu);
737
0
    if (ret < 0) {
738
0
        error_setg_errno(errp, -ret,
739
0
                         "kvm_init_vcpu: kvm_create_vcpu failed (%lu)",
740
0
                         kvm_arch_vcpu_id(cpu));
741
0
        goto err;
742
0
    }
743
744
0
    ret = map_kvm_run(s, cpu, errp);
745
0
    if (ret < 0) {
746
0
        goto err;
747
0
    }
748
749
0
    if (s->kvm_dirty_ring_size) {
750
0
        ret = map_kvm_dirty_gfns(s, cpu, errp);
751
0
        if (ret < 0) {
752
0
            goto err;
753
0
        }
754
0
    }
755
756
0
    ret = kvm_arch_init_vcpu(cpu, errp);
757
0
    if (ret < 0) {
758
0
        error_prepend(errp, "kvm_init_vcpu: kvm_arch_init_vcpu failed (%lu)",
759
0
                      kvm_arch_vcpu_id(cpu));
760
0
    }
761
0
    cpu->kvm_vcpu_stats_fd = kvm_vcpu_ioctl(cpu, KVM_GET_STATS_FD, NULL);
762
763
0
err:
764
0
    return ret;
765
0
}
766
767
void kvm_close(void)
768
0
{
769
0
    CPUState *cpu;
770
771
0
    if (!kvm_state || kvm_state->fd == -1) {
772
0
        return;
773
0
    }
774
775
0
    CPU_FOREACH(cpu) {
776
0
        cpu_remove_sync(cpu);
777
0
        close(cpu->kvm_fd);
778
0
        cpu->kvm_fd = -1;
779
0
        close(cpu->kvm_vcpu_stats_fd);
780
0
        cpu->kvm_vcpu_stats_fd = -1;
781
0
    }
782
783
0
    if (kvm_state && kvm_state->fd != -1) {
784
0
        close(kvm_state->vmfd);
785
0
        kvm_state->vmfd = -1;
786
0
        close(kvm_state->fd);
787
0
        kvm_state->fd = -1;
788
0
    }
789
0
    kvm_state = NULL;
790
0
}
791
792
/*
793
 * dirty pages logging control
794
 */
795
796
static int kvm_mem_flags(MemoryRegion *mr)
797
0
{
798
0
    bool readonly = mr->readonly || memory_region_is_romd(mr);
799
0
    int flags = 0;
800
801
0
    if (memory_region_get_dirty_log_mask(mr) != 0) {
802
0
        flags |= KVM_MEM_LOG_DIRTY_PAGES;
803
0
    }
804
0
    if (readonly && kvm_readonly_mem_allowed) {
805
0
        flags |= KVM_MEM_READONLY;
806
0
    }
807
0
    if (memory_region_has_guest_memfd_private(mr)) {
808
0
        assert(kvm_guest_memfd_supported);
809
0
        flags |= KVM_MEM_GUEST_MEMFD;
810
0
    }
811
0
    return flags;
812
0
}
813
814
/* Called with KVMMemoryListener.slots_lock held */
815
static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
816
                                 MemoryRegion *mr)
817
0
{
818
0
    mem->flags = kvm_mem_flags(mr);
819
820
    /* If nothing changed effectively, no need to issue ioctl */
821
0
    if (mem->flags == mem->old_flags) {
822
0
        return 0;
823
0
    }
824
825
0
    kvm_slot_init_dirty_bitmap(mem);
826
0
    return kvm_set_user_memory_region(kml, mem, false);
827
0
}
828
829
static int kvm_section_update_flags(KVMMemoryListener *kml,
830
                                    MemoryRegionSection *section)
831
0
{
832
0
    hwaddr start_addr, size, slot_size;
833
0
    KVMSlot *mem;
834
0
    int ret = 0;
835
836
0
    size = kvm_align_section(section, &start_addr);
837
0
    if (!size) {
838
0
        return 0;
839
0
    }
840
841
0
    kvm_slots_lock();
842
843
0
    while (size && !ret) {
844
0
        slot_size = MIN(kvm_max_slot_size, size);
845
0
        mem = kvm_lookup_matching_slot(kml, start_addr, slot_size);
846
0
        if (!mem) {
847
            /* We don't have a slot if we want to trap every access. */
848
0
            goto out;
849
0
        }
850
851
0
        ret = kvm_slot_update_flags(kml, mem, section->mr);
852
0
        start_addr += slot_size;
853
0
        size -= slot_size;
854
0
    }
855
856
0
out:
857
0
    kvm_slots_unlock();
858
0
    return ret;
859
0
}
860
861
static void kvm_log_start(MemoryListener *listener,
862
                          MemoryRegionSection *section,
863
                          int old, int new)
864
0
{
865
0
    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
866
0
    int r;
867
868
0
    if (old != 0) {
869
0
        return;
870
0
    }
871
872
0
    r = kvm_section_update_flags(kml, section);
873
0
    if (r < 0) {
874
0
        abort();
875
0
    }
876
0
}
877
878
static void kvm_log_stop(MemoryListener *listener,
879
                          MemoryRegionSection *section,
880
                          int old, int new)
881
0
{
882
0
    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
883
0
    int r;
884
885
0
    if (new != 0) {
886
0
        return;
887
0
    }
888
889
0
    r = kvm_section_update_flags(kml, section);
890
0
    if (r < 0) {
891
0
        abort();
892
0
    }
893
0
}
894
895
/* get kvm's dirty pages bitmap and update qemu's */
896
static void kvm_slot_sync_dirty_pages(KVMSlot *slot)
897
0
{
898
0
    ram_addr_t start = slot->ram_start_offset;
899
0
    ram_addr_t pages = slot->memory_size / qemu_real_host_page_size();
900
901
0
    physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages);
902
0
}
903
904
static void kvm_slot_reset_dirty_pages(KVMSlot *slot)
905
0
{
906
0
    memset(slot->dirty_bmap, 0, slot->dirty_bmap_size);
907
0
}
908
909
0
#define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
910
911
/* Allocate the dirty bitmap for a slot  */
912
static void kvm_slot_init_dirty_bitmap(KVMSlot *mem)
913
0
{
914
0
    if (!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) || mem->dirty_bmap) {
915
0
        return;
916
0
    }
917
918
    /*
919
     * XXX bad kernel interface alert
920
     * For dirty bitmap, kernel allocates array of size aligned to
921
     * bits-per-long.  But for case when the kernel is 64bits and
922
     * the userspace is 32bits, userspace can't align to the same
923
     * bits-per-long, since sizeof(long) is different between kernel
924
     * and user space.  This way, userspace will provide buffer which
925
     * may be 4 bytes less than the kernel will use, resulting in
926
     * userspace memory corruption (which is not detectable by valgrind
927
     * too, in most cases).
928
     * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
929
     * a hope that sizeof(long) won't become >8 any time soon.
930
     *
931
     * Note: the granule of kvm dirty log is qemu_real_host_page_size.
932
     * And mem->memory_size is aligned to it (otherwise this mem can't
933
     * be registered to KVM).
934
     */
935
0
    hwaddr bitmap_size = ALIGN(mem->memory_size / qemu_real_host_page_size(),
936
0
                                        /*HOST_LONG_BITS*/ 64) / 8;
937
0
    mem->dirty_bmap = g_malloc0(bitmap_size);
938
0
    mem->dirty_bmap_size = bitmap_size;
939
0
}
940
941
/*
942
 * Sync dirty bitmap from kernel to KVMSlot.dirty_bmap, return true if
943
 * succeeded, false otherwise
944
 */
945
static bool kvm_slot_get_dirty_log(KVMState *s, KVMSlot *slot)
946
0
{
947
0
    struct kvm_dirty_log d = {};
948
0
    int ret;
949
950
0
    d.dirty_bitmap = slot->dirty_bmap;
951
0
    d.slot = slot->slot | (slot->as_id << 16);
952
0
    ret = kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d);
953
954
0
    if (ret == -ENOENT) {
955
        /* kernel does not have dirty bitmap in this slot */
956
0
        ret = 0;
957
0
    }
958
0
    if (ret) {
959
0
        error_report_once("%s: KVM_GET_DIRTY_LOG failed with %d",
960
0
                          __func__, ret);
961
0
    }
962
0
    return ret == 0;
963
0
}
964
965
/* Should be with all slots_lock held for the address spaces. */
966
static void kvm_dirty_ring_mark_page(KVMState *s, uint32_t as_id,
967
                                     uint32_t slot_id, uint64_t offset)
968
0
{
969
0
    KVMMemoryListener *kml;
970
0
    KVMSlot *mem;
971
972
0
    if (as_id >= s->nr_as) {
973
0
        return;
974
0
    }
975
976
0
    kml = s->as[as_id].ml;
977
0
    mem = &kml->slots[slot_id];
978
979
0
    if (!mem->memory_size || offset >=
980
0
        (mem->memory_size / qemu_real_host_page_size())) {
981
0
        return;
982
0
    }
983
984
0
    set_bit(offset, mem->dirty_bmap);
985
0
}
986
987
static bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn)
988
0
{
989
    /*
990
     * Read the flags before the value.  Pairs with barrier in
991
     * KVM's kvm_dirty_ring_push() function.
992
     */
993
0
    return qatomic_load_acquire(&gfn->flags) == KVM_DIRTY_GFN_F_DIRTY;
994
0
}
995
996
static void dirty_gfn_set_collected(struct kvm_dirty_gfn *gfn)
997
0
{
998
    /*
999
     * Use a store-release so that the CPU that executes KVM_RESET_DIRTY_RINGS
1000
     * sees the full content of the ring:
1001
     *
1002
     * CPU0                     CPU1                         CPU2
1003
     * ------------------------------------------------------------------------------
1004
     *                                                       fill gfn0
1005
     *                                                       store-rel flags for gfn0
1006
     * load-acq flags for gfn0
1007
     * store-rel RESET for gfn0
1008
     *                          ioctl(RESET_RINGS)
1009
     *                            load-acq flags for gfn0
1010
     *                            check if flags have RESET
1011
     *
1012
     * The synchronization goes from CPU2 to CPU0 to CPU1.
1013
     */
1014
0
    qatomic_store_release(&gfn->flags, KVM_DIRTY_GFN_F_RESET);
1015
0
}
1016
1017
/*
1018
 * Should be with all slots_lock held for the address spaces.  It returns the
1019
 * dirty page we've collected on this dirty ring.
1020
 */
1021
static uint32_t kvm_dirty_ring_reap_one(KVMState *s, CPUState *cpu)
1022
0
{
1023
0
    struct kvm_dirty_gfn *dirty_gfns = cpu->kvm_dirty_gfns, *cur;
1024
0
    uint32_t ring_size = s->kvm_dirty_ring_size;
1025
0
    uint32_t count = 0, fetch = cpu->kvm_fetch_index;
1026
1027
    /*
1028
     * It's possible that we race with vcpu creation code where the vcpu is
1029
     * put onto the vcpus list but not yet initialized the dirty ring
1030
     * structures.  If so, skip it.
1031
     */
1032
0
    if (!cpu->created) {
1033
0
        return 0;
1034
0
    }
1035
1036
0
    assert(dirty_gfns && ring_size);
1037
0
    trace_kvm_dirty_ring_reap_vcpu(cpu->cpu_index);
1038
1039
0
    while (true) {
1040
0
        cur = &dirty_gfns[fetch % ring_size];
1041
0
        if (!dirty_gfn_is_dirtied(cur)) {
1042
0
            break;
1043
0
        }
1044
0
        kvm_dirty_ring_mark_page(s, cur->slot >> 16, cur->slot & 0xffff,
1045
0
                                 cur->offset);
1046
0
        dirty_gfn_set_collected(cur);
1047
0
        trace_kvm_dirty_ring_page(cpu->cpu_index, fetch, cur->offset);
1048
0
        fetch++;
1049
0
        count++;
1050
0
    }
1051
0
    cpu->kvm_fetch_index = fetch;
1052
0
    cpu->dirty_pages += count;
1053
1054
0
    return count;
1055
0
}
1056
1057
/* Must be with slots_lock held */
1058
static uint64_t kvm_dirty_ring_reap_locked(KVMState *s, CPUState* cpu)
1059
0
{
1060
0
    int ret;
1061
0
    uint64_t total = 0;
1062
0
    int64_t stamp;
1063
1064
0
    stamp = get_clock();
1065
1066
0
    if (cpu) {
1067
0
        total = kvm_dirty_ring_reap_one(s, cpu);
1068
0
    } else {
1069
0
        CPU_FOREACH(cpu) {
1070
0
            total += kvm_dirty_ring_reap_one(s, cpu);
1071
0
        }
1072
0
    }
1073
1074
0
    if (total) {
1075
0
        ret = kvm_vm_ioctl(s, KVM_RESET_DIRTY_RINGS);
1076
0
        assert(ret == total);
1077
0
    }
1078
1079
0
    stamp = get_clock() - stamp;
1080
1081
0
    if (total) {
1082
0
        trace_kvm_dirty_ring_reap(total, stamp / 1000);
1083
0
    }
1084
1085
0
    return total;
1086
0
}
1087
1088
/*
1089
 * Currently for simplicity, we must hold BQL before calling this.  We can
1090
 * consider to drop the BQL if we're clear with all the race conditions.
1091
 */
1092
static uint64_t kvm_dirty_ring_reap(KVMState *s, CPUState *cpu)
1093
0
{
1094
0
    uint64_t total;
1095
1096
    /*
1097
     * We need to lock all kvm slots for all address spaces here,
1098
     * because:
1099
     *
1100
     * (1) We need to mark dirty for dirty bitmaps in multiple slots
1101
     *     and for tons of pages, so it's better to take the lock here
1102
     *     once rather than once per page.  And more importantly,
1103
     *
1104
     * (2) We must _NOT_ publish dirty bits to the other threads
1105
     *     (e.g., the migration thread) via the kvm memory slot dirty
1106
     *     bitmaps before correctly re-protect those dirtied pages.
1107
     *     Otherwise we can have potential risk of data corruption if
1108
     *     the page data is read in the other thread before we do
1109
     *     reset below.
1110
     */
1111
0
    kvm_slots_lock();
1112
0
    total = kvm_dirty_ring_reap_locked(s, cpu);
1113
0
    kvm_slots_unlock();
1114
1115
0
    return total;
1116
0
}
1117
1118
static void do_kvm_cpu_synchronize_kick(CPUState *cpu, run_on_cpu_data arg)
1119
0
{
1120
    /* No need to do anything */
1121
0
}
1122
1123
/*
1124
 * Kick all vcpus out in a synchronized way.  When returned, we
1125
 * guarantee that every vcpu has been kicked and at least returned to
1126
 * userspace once.
1127
 */
1128
static void kvm_cpu_synchronize_kick_all(void)
1129
0
{
1130
0
    CPUState *cpu;
1131
1132
0
    CPU_FOREACH(cpu) {
1133
0
        run_on_cpu(cpu, do_kvm_cpu_synchronize_kick, RUN_ON_CPU_NULL);
1134
0
    }
1135
0
}
1136
1137
/*
1138
 * Flush all the existing dirty pages to the KVM slot buffers.  When
1139
 * this call returns, we guarantee that all the touched dirty pages
1140
 * before calling this function have been put into the per-kvmslot
1141
 * dirty bitmap.
1142
 *
1143
 * This function must be called with BQL held.
1144
 */
1145
static void kvm_dirty_ring_flush(void)
1146
0
{
1147
0
    trace_kvm_dirty_ring_flush(0);
1148
    /*
1149
     * The function needs to be serialized.  Since this function
1150
     * should always be with BQL held, serialization is guaranteed.
1151
     * However, let's be sure of it.
1152
     */
1153
0
    assert(bql_locked());
1154
    /*
1155
     * First make sure to flush the hardware buffers by kicking all
1156
     * vcpus out in a synchronous way.
1157
     */
1158
0
    kvm_cpu_synchronize_kick_all();
1159
0
    kvm_dirty_ring_reap(kvm_state, NULL);
1160
0
    trace_kvm_dirty_ring_flush(1);
1161
0
}
1162
1163
/**
1164
 * kvm_physical_sync_dirty_bitmap - Sync dirty bitmap from kernel space
1165
 *
1166
 * This function will first try to fetch dirty bitmap from the kernel,
1167
 * and then updates qemu's dirty bitmap.
1168
 *
1169
 * NOTE: caller must be with kml->slots_lock held.
1170
 *
1171
 * @kml: the KVM memory listener object
1172
 * @section: the memory section to sync the dirty bitmap with
1173
 */
1174
static void kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
1175
                                           MemoryRegionSection *section)
1176
0
{
1177
0
    KVMState *s = kvm_state;
1178
0
    KVMSlot *mem;
1179
0
    hwaddr start_addr, size;
1180
0
    hwaddr slot_size;
1181
1182
0
    size = kvm_align_section(section, &start_addr);
1183
0
    while (size) {
1184
0
        slot_size = MIN(kvm_max_slot_size, size);
1185
0
        mem = kvm_lookup_matching_slot(kml, start_addr, slot_size);
1186
0
        if (!mem) {
1187
            /* We don't have a slot if we want to trap every access. */
1188
0
            return;
1189
0
        }
1190
0
        if (kvm_slot_get_dirty_log(s, mem)) {
1191
0
            kvm_slot_sync_dirty_pages(mem);
1192
0
        }
1193
0
        start_addr += slot_size;
1194
0
        size -= slot_size;
1195
0
    }
1196
0
}
1197
1198
/* Alignment requirement for KVM_CLEAR_DIRTY_LOG - 64 pages */
1199
0
#define KVM_CLEAR_LOG_SHIFT  6
1200
0
#define KVM_CLEAR_LOG_ALIGN  (qemu_real_host_page_size() << KVM_CLEAR_LOG_SHIFT)
1201
0
#define KVM_CLEAR_LOG_MASK   (-KVM_CLEAR_LOG_ALIGN)
1202
1203
static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start,
1204
                                  uint64_t size)
1205
0
{
1206
0
    KVMState *s = kvm_state;
1207
0
    uint64_t end, bmap_start, start_delta, bmap_npages;
1208
0
    struct kvm_clear_dirty_log d;
1209
0
    unsigned long *bmap_clear = NULL, psize = qemu_real_host_page_size();
1210
0
    int ret;
1211
1212
    /*
1213
     * We need to extend either the start or the size or both to
1214
     * satisfy the KVM interface requirement.  Firstly, do the start
1215
     * page alignment on 64 host pages
1216
     */
1217
0
    bmap_start = start & KVM_CLEAR_LOG_MASK;
1218
0
    start_delta = start - bmap_start;
1219
0
    bmap_start /= psize;
1220
1221
    /*
1222
     * The kernel interface has restriction on the size too, that either:
1223
     *
1224
     * (1) the size is 64 host pages aligned (just like the start), or
1225
     * (2) the size fills up until the end of the KVM memslot.
1226
     */
1227
0
    bmap_npages = DIV_ROUND_UP(size + start_delta, KVM_CLEAR_LOG_ALIGN)
1228
0
        << KVM_CLEAR_LOG_SHIFT;
1229
0
    end = mem->memory_size / psize;
1230
0
    if (bmap_npages > end - bmap_start) {
1231
0
        bmap_npages = end - bmap_start;
1232
0
    }
1233
0
    start_delta /= psize;
1234
1235
    /*
1236
     * Prepare the bitmap to clear dirty bits.  Here we must guarantee
1237
     * that we won't clear any unknown dirty bits otherwise we might
1238
     * accidentally clear some set bits which are not yet synced from
1239
     * the kernel into QEMU's bitmap, then we'll lose track of the
1240
     * guest modifications upon those pages (which can directly lead
1241
     * to guest data loss or panic after migration).
1242
     *
1243
     * Layout of the KVMSlot.dirty_bmap:
1244
     *
1245
     *                   |<-------- bmap_npages -----------..>|
1246
     *                                                     [1]
1247
     *                     start_delta         size
1248
     *  |----------------|-------------|------------------|------------|
1249
     *  ^                ^             ^                               ^
1250
     *  |                |             |                               |
1251
     * start          bmap_start     (start)                         end
1252
     * of memslot                                             of memslot
1253
     *
1254
     * [1] bmap_npages can be aligned to either 64 pages or the end of slot
1255
     */
1256
1257
0
    assert(bmap_start % BITS_PER_LONG == 0);
1258
    /* We should never do log_clear before log_sync */
1259
0
    assert(mem->dirty_bmap);
1260
0
    if (start_delta || bmap_npages - size / psize) {
1261
        /* Slow path - we need to manipulate a temp bitmap */
1262
0
        bmap_clear = bitmap_new(bmap_npages);
1263
0
        bitmap_copy_with_src_offset(bmap_clear, mem->dirty_bmap,
1264
0
                                    bmap_start, start_delta + size / psize);
1265
        /*
1266
         * We need to fill the holes at start because that was not
1267
         * specified by the caller and we extended the bitmap only for
1268
         * 64 pages alignment
1269
         */
1270
0
        bitmap_clear(bmap_clear, 0, start_delta);
1271
0
        d.dirty_bitmap = bmap_clear;
1272
0
    } else {
1273
        /*
1274
         * Fast path - both start and size align well with BITS_PER_LONG
1275
         * (or the end of memory slot)
1276
         */
1277
0
        d.dirty_bitmap = mem->dirty_bmap + BIT_WORD(bmap_start);
1278
0
    }
1279
1280
0
    d.first_page = bmap_start;
1281
    /* It should never overflow.  If it happens, say something */
1282
0
    assert(bmap_npages <= UINT32_MAX);
1283
0
    d.num_pages = bmap_npages;
1284
0
    d.slot = mem->slot | (as_id << 16);
1285
1286
0
    ret = kvm_vm_ioctl(s, KVM_CLEAR_DIRTY_LOG, &d);
1287
0
    if (ret < 0 && ret != -ENOENT) {
1288
0
        error_report("%s: KVM_CLEAR_DIRTY_LOG failed, slot=%d, "
1289
0
                     "start=0x%"PRIx64", size=0x%"PRIx32", errno=%d",
1290
0
                     __func__, d.slot, (uint64_t)d.first_page,
1291
0
                     (uint32_t)d.num_pages, ret);
1292
0
    } else {
1293
0
        ret = 0;
1294
0
        trace_kvm_clear_dirty_log(d.slot, d.first_page, d.num_pages);
1295
0
    }
1296
1297
    /*
1298
     * After we have updated the remote dirty bitmap, we update the
1299
     * cached bitmap as well for the memslot, then if another user
1300
     * clears the same region we know we shouldn't clear it again on
1301
     * the remote otherwise it's data loss as well.
1302
     */
1303
0
    bitmap_clear(mem->dirty_bmap, bmap_start + start_delta,
1304
0
                 size / psize);
1305
    /* This handles the NULL case well */
1306
0
    g_free(bmap_clear);
1307
0
    return ret;
1308
0
}
1309
1310
1311
/**
1312
 * kvm_physical_log_clear - Clear the kernel's dirty bitmap for range
1313
 *
1314
 * NOTE: this will be a no-op if we haven't enabled manual dirty log
1315
 * protection in the host kernel because in that case this operation
1316
 * will be done within log_sync().
1317
 *
1318
 * @kml:     the kvm memory listener
1319
 * @section: the memory range to clear dirty bitmap
1320
 */
1321
static int kvm_physical_log_clear(KVMMemoryListener *kml,
1322
                                  MemoryRegionSection *section)
1323
0
{
1324
0
    KVMState *s = kvm_state;
1325
0
    uint64_t start, size, offset, count;
1326
0
    KVMSlot *mem;
1327
0
    int ret = 0, i;
1328
1329
0
    if (!s->manual_dirty_log_protect) {
1330
        /* No need to do explicit clear */
1331
0
        return ret;
1332
0
    }
1333
1334
0
    start = section->offset_within_address_space;
1335
0
    size = int128_get64(section->size);
1336
1337
0
    if (!size) {
1338
        /* Nothing more we can do... */
1339
0
        return ret;
1340
0
    }
1341
1342
0
    kvm_slots_lock();
1343
1344
0
    for (i = 0; i < kml->nr_slots_allocated; i++) {
1345
0
        mem = &kml->slots[i];
1346
        /* Discard slots that are empty or do not overlap the section */
1347
0
        if (!mem->memory_size ||
1348
0
            mem->start_addr > start + size - 1 ||
1349
0
            start > mem->start_addr + mem->memory_size - 1) {
1350
0
            continue;
1351
0
        }
1352
1353
0
        if (start >= mem->start_addr) {
1354
            /* The slot starts before section or is aligned to it.  */
1355
0
            offset = start - mem->start_addr;
1356
0
            count = MIN(mem->memory_size - offset, size);
1357
0
        } else {
1358
            /* The slot starts after section.  */
1359
0
            offset = 0;
1360
0
            count = MIN(mem->memory_size, size - (mem->start_addr - start));
1361
0
        }
1362
0
        ret = kvm_log_clear_one_slot(mem, kml->as_id, offset, count);
1363
0
        if (ret < 0) {
1364
0
            break;
1365
0
        }
1366
0
    }
1367
1368
0
    kvm_slots_unlock();
1369
1370
0
    return ret;
1371
0
}
1372
1373
static void kvm_coalesce_mmio_region(MemoryListener *listener,
1374
                                     MemoryRegionSection *secion,
1375
                                     hwaddr start, hwaddr size)
1376
0
{
1377
0
    KVMState *s = kvm_state;
1378
1379
0
    if (s->coalesced_mmio) {
1380
0
        struct kvm_coalesced_mmio_zone zone;
1381
1382
0
        zone.addr = start;
1383
0
        zone.size = size;
1384
0
        zone.pad = 0;
1385
1386
0
        (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
1387
0
    }
1388
0
}
1389
1390
static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
1391
                                       MemoryRegionSection *secion,
1392
                                       hwaddr start, hwaddr size)
1393
0
{
1394
0
    KVMState *s = kvm_state;
1395
1396
0
    if (s->coalesced_mmio) {
1397
0
        struct kvm_coalesced_mmio_zone zone;
1398
1399
0
        zone.addr = start;
1400
0
        zone.size = size;
1401
0
        zone.pad = 0;
1402
1403
0
        (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
1404
0
    }
1405
0
}
1406
1407
static void kvm_coalesce_pio_add(MemoryListener *listener,
1408
                                MemoryRegionSection *section,
1409
                                hwaddr start, hwaddr size)
1410
0
{
1411
0
    KVMState *s = kvm_state;
1412
1413
0
    if (s->coalesced_pio) {
1414
0
        struct kvm_coalesced_mmio_zone zone;
1415
1416
0
        zone.addr = start;
1417
0
        zone.size = size;
1418
0
        zone.pio = 1;
1419
1420
0
        (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
1421
0
    }
1422
0
}
1423
1424
static void kvm_coalesce_pio_del(MemoryListener *listener,
1425
                                MemoryRegionSection *section,
1426
                                hwaddr start, hwaddr size)
1427
0
{
1428
0
    KVMState *s = kvm_state;
1429
1430
0
    if (s->coalesced_pio) {
1431
0
        struct kvm_coalesced_mmio_zone zone;
1432
1433
0
        zone.addr = start;
1434
0
        zone.size = size;
1435
0
        zone.pio = 1;
1436
1437
0
        (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
1438
0
     }
1439
0
}
1440
1441
int kvm_check_extension(KVMState *s, unsigned int extension)
1442
0
{
1443
0
    int ret;
1444
1445
0
    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
1446
0
    if (ret < 0) {
1447
0
        ret = 0;
1448
0
    }
1449
1450
0
    return ret;
1451
0
}
1452
1453
int kvm_vm_check_extension(KVMState *s, unsigned int extension)
1454
0
{
1455
0
    int ret;
1456
1457
0
    ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
1458
0
    if (ret < 0) {
1459
        /* VM wide version not implemented, use global one instead */
1460
0
        ret = kvm_check_extension(s, extension);
1461
0
    }
1462
1463
0
    return ret;
1464
0
}
1465
1466
/*
1467
 * We track the poisoned pages to be able to:
1468
 * - replace them on VM reset
1469
 * - block a migration for a VM with a poisoned page
1470
 */
1471
typedef struct HWPoisonPage {
1472
    ram_addr_t ram_addr;
1473
    QLIST_ENTRY(HWPoisonPage) list;
1474
} HWPoisonPage;
1475
1476
static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list =
1477
    QLIST_HEAD_INITIALIZER(hwpoison_page_list);
1478
1479
static void kvm_unpoison_all(void *param)
1480
0
{
1481
0
    HWPoisonPage *page, *next_page;
1482
1483
0
    QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) {
1484
0
        QLIST_REMOVE(page, list);
1485
0
        qemu_ram_remap(page->ram_addr);
1486
0
        g_free(page);
1487
0
    }
1488
0
}
1489
1490
void kvm_hwpoison_page_add(ram_addr_t ram_addr)
1491
0
{
1492
0
    HWPoisonPage *page;
1493
1494
0
    QLIST_FOREACH(page, &hwpoison_page_list, list) {
1495
0
        if (page->ram_addr == ram_addr) {
1496
0
            return;
1497
0
        }
1498
0
    }
1499
0
    page = g_new(HWPoisonPage, 1);
1500
0
    page->ram_addr = ram_addr;
1501
0
    QLIST_INSERT_HEAD(&hwpoison_page_list, page, list);
1502
0
}
1503
1504
bool kvm_hwpoisoned_mem(void)
1505
0
{
1506
0
    return !QLIST_EMPTY(&hwpoison_page_list);
1507
0
}
1508
1509
static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
1510
0
{
1511
0
    if (target_needs_bswap()) {
1512
        /*
1513
         * The kernel expects ioeventfd values in HOST_BIG_ENDIAN
1514
         * endianness, but the memory core hands them in target endianness.
1515
         * For example, PPC is always treated as big-endian even if running
1516
         * on KVM and on PPC64LE.  Correct here, swapping back.
1517
         */
1518
0
        switch (size) {
1519
0
        case 2:
1520
0
            val = bswap16(val);
1521
0
            break;
1522
0
        case 4:
1523
0
            val = bswap32(val);
1524
0
            break;
1525
0
        }
1526
0
    }
1527
0
    return val;
1528
0
}
1529
1530
static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
1531
                                  bool assign, uint32_t size, bool datamatch)
1532
0
{
1533
0
    int ret;
1534
0
    struct kvm_ioeventfd iofd = {
1535
0
        .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
1536
0
        .addr = addr,
1537
0
        .len = size,
1538
0
        .flags = 0,
1539
0
        .fd = fd,
1540
0
    };
1541
1542
0
    trace_kvm_set_ioeventfd_mmio(fd, (uint64_t)addr, val, assign, size,
1543
0
                                 datamatch);
1544
0
    if (!kvm_enabled()) {
1545
0
        return -ENOSYS;
1546
0
    }
1547
1548
0
    if (datamatch) {
1549
0
        iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
1550
0
    }
1551
0
    if (!assign) {
1552
0
        iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1553
0
    }
1554
1555
0
    ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1556
1557
0
    if (ret < 0) {
1558
0
        return -errno;
1559
0
    }
1560
1561
0
    return 0;
1562
0
}
1563
1564
static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
1565
                                 bool assign, uint32_t size, bool datamatch)
1566
0
{
1567
0
    struct kvm_ioeventfd kick = {
1568
0
        .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
1569
0
        .addr = addr,
1570
0
        .flags = KVM_IOEVENTFD_FLAG_PIO,
1571
0
        .len = size,
1572
0
        .fd = fd,
1573
0
    };
1574
0
    int r;
1575
0
    trace_kvm_set_ioeventfd_pio(fd, addr, val, assign, size, datamatch);
1576
0
    if (!kvm_enabled()) {
1577
0
        return -ENOSYS;
1578
0
    }
1579
0
    if (datamatch) {
1580
0
        kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
1581
0
    }
1582
0
    if (!assign) {
1583
0
        kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1584
0
    }
1585
0
    r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1586
0
    if (r < 0) {
1587
0
        return r;
1588
0
    }
1589
0
    return 0;
1590
0
}
1591
1592
1593
static const KVMCapabilityInfo *
1594
kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
1595
0
{
1596
0
    while (list->name) {
1597
0
        if (!kvm_check_extension(s, list->value)) {
1598
0
            return list;
1599
0
        }
1600
0
        list++;
1601
0
    }
1602
0
    return NULL;
1603
0
}
1604
1605
void kvm_set_max_memslot_size(hwaddr max_slot_size)
1606
0
{
1607
0
    g_assert(
1608
0
        ROUND_UP(max_slot_size, qemu_real_host_page_size()) == max_slot_size
1609
0
    );
1610
0
    kvm_max_slot_size = max_slot_size;
1611
0
}
1612
1613
static int kvm_set_memory_attributes(hwaddr start, uint64_t size, uint64_t attr)
1614
0
{
1615
0
    struct kvm_memory_attributes attrs;
1616
0
    int r;
1617
1618
0
    assert((attr & kvm_supported_memory_attributes) == attr);
1619
0
    attrs.attributes = attr;
1620
0
    attrs.address = start;
1621
0
    attrs.size = size;
1622
0
    attrs.flags = 0;
1623
1624
0
    r = kvm_vm_ioctl(kvm_state, KVM_SET_MEMORY_ATTRIBUTES, &attrs);
1625
0
    if (r) {
1626
0
        error_report("failed to set memory (0x%" HWADDR_PRIx "+0x%" PRIx64 ") "
1627
0
                     "with attr 0x%" PRIx64 " error '%s'",
1628
0
                     start, size, attr, strerror(errno));
1629
0
    }
1630
0
    return r;
1631
0
}
1632
1633
int kvm_set_memory_attributes_private(hwaddr start, uint64_t size)
1634
0
{
1635
0
    return kvm_set_memory_attributes(start, size, KVM_MEMORY_ATTRIBUTE_PRIVATE);
1636
0
}
1637
1638
int kvm_set_memory_attributes_shared(hwaddr start, uint64_t size)
1639
0
{
1640
0
    return kvm_set_memory_attributes(start, size, 0);
1641
0
}
1642
1643
bool kvm_private_memory_attribute_supported(void)
1644
0
{
1645
0
    return !!(kvm_supported_memory_attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE);
1646
0
}
1647
1648
/* Called with KVMMemoryListener.slots_lock held */
1649
static void kvm_set_phys_mem(KVMMemoryListener *kml,
1650
                             MemoryRegionSection *section, bool add)
1651
0
{
1652
0
    KVMSlot *mem;
1653
0
    int err;
1654
0
    MemoryRegion *mr = section->mr;
1655
0
    bool writable = !mr->readonly && !mr->rom_device;
1656
0
    hwaddr start_addr, size, slot_size, mr_offset;
1657
0
    ram_addr_t ram_start_offset;
1658
0
    void *ram;
1659
1660
0
    if (!memory_region_is_ram(mr)) {
1661
0
        if (writable || !kvm_readonly_mem_allowed) {
1662
0
            return;
1663
0
        } else if (!mr->romd_mode) {
1664
            /* If the memory device is not in romd_mode, then we actually want
1665
             * to remove the kvm memory slot so all accesses will trap. */
1666
0
            add = false;
1667
0
        }
1668
0
    }
1669
1670
0
    size = kvm_align_section(section, &start_addr);
1671
0
    if (!size) {
1672
0
        return;
1673
0
    }
1674
1675
    /* The offset of the kvmslot within the memory region */
1676
0
    mr_offset = section->offset_within_region + start_addr -
1677
0
        section->offset_within_address_space;
1678
1679
    /* use aligned delta to align the ram address and offset */
1680
0
    ram = memory_region_get_ram_ptr(mr) + mr_offset;
1681
0
    ram_start_offset = memory_region_get_ram_addr(mr) + mr_offset;
1682
1683
0
    if (!add) {
1684
0
        do {
1685
0
            slot_size = MIN(kvm_max_slot_size, size);
1686
0
            mem = kvm_lookup_matching_slot(kml, start_addr, slot_size);
1687
0
            if (!mem) {
1688
0
                return;
1689
0
            }
1690
0
            if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
1691
                /*
1692
                 * NOTE: We should be aware of the fact that here we're only
1693
                 * doing a best effort to sync dirty bits.  No matter whether
1694
                 * we're using dirty log or dirty ring, we ignored two facts:
1695
                 *
1696
                 * (1) dirty bits can reside in hardware buffers (PML)
1697
                 *
1698
                 * (2) after we collected dirty bits here, pages can be dirtied
1699
                 * again before we do the final KVM_SET_USER_MEMORY_REGION to
1700
                 * remove the slot.
1701
                 *
1702
                 * Not easy.  Let's cross the fingers until it's fixed.
1703
                 */
1704
0
                if (kvm_state->kvm_dirty_ring_size) {
1705
0
                    kvm_dirty_ring_reap_locked(kvm_state, NULL);
1706
0
                    if (kvm_state->kvm_dirty_ring_with_bitmap) {
1707
0
                        kvm_slot_sync_dirty_pages(mem);
1708
0
                        kvm_slot_get_dirty_log(kvm_state, mem);
1709
0
                    }
1710
0
                } else {
1711
0
                    kvm_slot_get_dirty_log(kvm_state, mem);
1712
0
                }
1713
0
                kvm_slot_sync_dirty_pages(mem);
1714
0
            }
1715
1716
            /* unregister the slot */
1717
0
            g_free(mem->dirty_bmap);
1718
0
            mem->dirty_bmap = NULL;
1719
0
            mem->memory_size = 0;
1720
0
            mem->flags = 0;
1721
0
            err = kvm_set_user_memory_region(kml, mem, false);
1722
0
            if (err) {
1723
0
                fprintf(stderr, "%s: error unregistering slot: %s\n",
1724
0
                        __func__, strerror(-err));
1725
0
                abort();
1726
0
            }
1727
0
            start_addr += slot_size;
1728
0
            size -= slot_size;
1729
0
            kml->nr_slots_used--;
1730
0
        } while (size);
1731
0
        return;
1732
0
    }
1733
1734
    /* register the new slot */
1735
0
    do {
1736
0
        slot_size = MIN(kvm_max_slot_size, size);
1737
0
        mem = kvm_alloc_slot(kml);
1738
0
        mem->as_id = kml->as_id;
1739
0
        mem->memory_size = slot_size;
1740
0
        mem->start_addr = start_addr;
1741
0
        mem->ram_start_offset = ram_start_offset;
1742
0
        mem->ram = ram;
1743
0
        mem->flags = kvm_mem_flags(mr);
1744
0
        mem->guest_memfd = mr->ram_block->guest_memfd_private;
1745
0
        mem->guest_memfd_offset = mem->guest_memfd >= 0 ?
1746
0
                                  (uint8_t*)ram - mr->ram_block->host : 0;
1747
1748
0
        kvm_slot_init_dirty_bitmap(mem);
1749
0
        err = kvm_set_user_memory_region(kml, mem, true);
1750
0
        if (err) {
1751
0
            fprintf(stderr, "%s: error registering slot: %s\n", __func__,
1752
0
                    strerror(-err));
1753
0
            abort();
1754
0
        }
1755
1756
0
        if (memory_region_has_guest_memfd_private(mr)) {
1757
0
            err = kvm_set_memory_attributes_private(start_addr, slot_size);
1758
0
            if (err) {
1759
0
                error_report("%s: failed to set memory attribute private: %s",
1760
0
                             __func__, strerror(-err));
1761
0
                exit(1);
1762
0
            }
1763
0
        }
1764
1765
0
        start_addr += slot_size;
1766
0
        ram_start_offset += slot_size;
1767
0
        ram += slot_size;
1768
0
        size -= slot_size;
1769
0
        kml->nr_slots_used++;
1770
0
    } while (size);
1771
0
}
1772
1773
static void *kvm_dirty_ring_reaper_thread(void *data)
1774
0
{
1775
0
    KVMState *s = data;
1776
0
    struct KVMDirtyRingReaper *r = &s->reaper;
1777
1778
0
    rcu_register_thread();
1779
1780
0
    trace_kvm_dirty_ring_reaper("init");
1781
1782
0
    while (true) {
1783
0
        r->reaper_state = KVM_DIRTY_RING_REAPER_WAIT;
1784
0
        trace_kvm_dirty_ring_reaper("wait");
1785
        /*
1786
         * TODO: provide a smarter timeout rather than a constant?
1787
         */
1788
0
        sleep(1);
1789
1790
        /* keep sleeping so that dirtylimit not be interfered by reaper */
1791
0
        if (dirtylimit_in_service()) {
1792
0
            continue;
1793
0
        }
1794
1795
0
        trace_kvm_dirty_ring_reaper("wakeup");
1796
0
        r->reaper_state = KVM_DIRTY_RING_REAPER_REAPING;
1797
1798
0
        bql_lock();
1799
0
        kvm_dirty_ring_reap(s, NULL);
1800
0
        bql_unlock();
1801
1802
0
        r->reaper_iteration++;
1803
0
    }
1804
1805
0
    g_assert_not_reached();
1806
0
}
1807
1808
static void kvm_dirty_ring_reaper_init(KVMState *s)
1809
0
{
1810
0
    struct KVMDirtyRingReaper *r = &s->reaper;
1811
1812
0
    qemu_thread_create(&r->reaper_thr, "kvm-reaper",
1813
0
                       kvm_dirty_ring_reaper_thread,
1814
0
                       s, QEMU_THREAD_JOINABLE);
1815
0
}
1816
1817
static int kvm_dirty_ring_init(KVMState *s)
1818
0
{
1819
0
    uint32_t ring_size = s->kvm_dirty_ring_size;
1820
0
    uint64_t ring_bytes = ring_size * sizeof(struct kvm_dirty_gfn);
1821
0
    unsigned int capability = KVM_CAP_DIRTY_LOG_RING;
1822
0
    int ret;
1823
1824
0
    s->kvm_dirty_ring_size = 0;
1825
0
    s->kvm_dirty_ring_bytes = 0;
1826
1827
    /* Bail if the dirty ring size isn't specified */
1828
0
    if (!ring_size) {
1829
0
        return 0;
1830
0
    }
1831
1832
    /*
1833
     * Read the max supported pages. Fall back to dirty logging mode
1834
     * if the dirty ring isn't supported.
1835
     */
1836
0
    ret = kvm_vm_check_extension(s, capability);
1837
0
    if (ret <= 0) {
1838
0
        capability = KVM_CAP_DIRTY_LOG_RING_ACQ_REL;
1839
0
        ret = kvm_vm_check_extension(s, capability);
1840
0
    }
1841
1842
0
    if (ret <= 0) {
1843
0
        warn_report("KVM dirty ring not available, using bitmap method");
1844
0
        return 0;
1845
0
    }
1846
1847
0
    if (ring_bytes > ret) {
1848
0
        error_report("KVM dirty ring size %" PRIu32 " too big "
1849
0
                     "(maximum is %ld).  Please use a smaller value.",
1850
0
                     ring_size, (long)ret / sizeof(struct kvm_dirty_gfn));
1851
0
        return -EINVAL;
1852
0
    }
1853
1854
0
    ret = kvm_vm_enable_cap(s, capability, 0, ring_bytes);
1855
0
    if (ret) {
1856
0
        error_report("Enabling of KVM dirty ring failed: %s. "
1857
0
                     "Suggested minimum value is 1024.", strerror(-ret));
1858
0
        return -EIO;
1859
0
    }
1860
1861
    /* Enable the backup bitmap if it is supported */
1862
0
    ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP);
1863
0
    if (ret > 0) {
1864
0
        ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP, 0);
1865
0
        if (ret) {
1866
0
            error_report("Enabling of KVM dirty ring's backup bitmap failed: "
1867
0
                         "%s. ", strerror(-ret));
1868
0
            return -EIO;
1869
0
        }
1870
1871
0
        s->kvm_dirty_ring_with_bitmap = true;
1872
0
    }
1873
1874
0
    s->kvm_dirty_ring_size = ring_size;
1875
0
    s->kvm_dirty_ring_bytes = ring_bytes;
1876
1877
0
    return 0;
1878
0
}
1879
1880
static void kvm_region_add(MemoryListener *listener,
1881
                           MemoryRegionSection *section)
1882
0
{
1883
0
    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
1884
0
    KVMMemoryUpdate *update;
1885
1886
0
    update = g_new0(KVMMemoryUpdate, 1);
1887
0
    update->section = *section;
1888
1889
0
    QSIMPLEQ_INSERT_TAIL(&kml->transaction_add, update, next);
1890
0
}
1891
1892
static void kvm_region_del(MemoryListener *listener,
1893
                           MemoryRegionSection *section)
1894
0
{
1895
0
    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
1896
0
    KVMMemoryUpdate *update;
1897
1898
0
    update = g_new0(KVMMemoryUpdate, 1);
1899
0
    update->section = *section;
1900
1901
0
    QSIMPLEQ_INSERT_TAIL(&kml->transaction_del, update, next);
1902
0
}
1903
1904
static void kvm_region_commit(MemoryListener *listener)
1905
0
{
1906
0
    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener,
1907
0
                                          listener);
1908
0
    KVMMemoryUpdate *u1, *u2;
1909
0
    bool need_inhibit = false;
1910
1911
0
    if (QSIMPLEQ_EMPTY(&kml->transaction_add) &&
1912
0
        QSIMPLEQ_EMPTY(&kml->transaction_del)) {
1913
0
        return;
1914
0
    }
1915
1916
    /*
1917
     * We have to be careful when regions to add overlap with ranges to remove.
1918
     * We have to simulate atomic KVM memslot updates by making sure no ioctl()
1919
     * is currently active.
1920
     *
1921
     * The lists are order by addresses, so it's easy to find overlaps.
1922
     */
1923
0
    u1 = QSIMPLEQ_FIRST(&kml->transaction_del);
1924
0
    u2 = QSIMPLEQ_FIRST(&kml->transaction_add);
1925
0
    while (u1 && u2) {
1926
0
        Range r1, r2;
1927
1928
0
        range_init_nofail(&r1, u1->section.offset_within_address_space,
1929
0
                          int128_get64(u1->section.size));
1930
0
        range_init_nofail(&r2, u2->section.offset_within_address_space,
1931
0
                          int128_get64(u2->section.size));
1932
1933
0
        if (range_overlaps_range(&r1, &r2)) {
1934
0
            need_inhibit = true;
1935
0
            break;
1936
0
        }
1937
0
        if (range_lob(&r1) < range_lob(&r2)) {
1938
0
            u1 = QSIMPLEQ_NEXT(u1, next);
1939
0
        } else {
1940
0
            u2 = QSIMPLEQ_NEXT(u2, next);
1941
0
        }
1942
0
    }
1943
1944
0
    kvm_slots_lock();
1945
0
    if (need_inhibit) {
1946
0
        accel_ioctl_inhibit_begin();
1947
0
    }
1948
1949
    /* Remove all memslots before adding the new ones. */
1950
0
    while (!QSIMPLEQ_EMPTY(&kml->transaction_del)) {
1951
0
        u1 = QSIMPLEQ_FIRST(&kml->transaction_del);
1952
0
        QSIMPLEQ_REMOVE_HEAD(&kml->transaction_del, next);
1953
1954
0
        kvm_set_phys_mem(kml, &u1->section, false);
1955
0
        memory_region_unref(u1->section.mr);
1956
1957
0
        g_free(u1);
1958
0
    }
1959
0
    while (!QSIMPLEQ_EMPTY(&kml->transaction_add)) {
1960
0
        u1 = QSIMPLEQ_FIRST(&kml->transaction_add);
1961
0
        QSIMPLEQ_REMOVE_HEAD(&kml->transaction_add, next);
1962
1963
0
        memory_region_ref(u1->section.mr);
1964
0
        kvm_set_phys_mem(kml, &u1->section, true);
1965
1966
0
        g_free(u1);
1967
0
    }
1968
1969
0
    if (need_inhibit) {
1970
0
        accel_ioctl_inhibit_end();
1971
0
    }
1972
0
    kvm_slots_unlock();
1973
0
}
1974
1975
static void kvm_log_sync(MemoryListener *listener,
1976
                         MemoryRegionSection *section)
1977
0
{
1978
0
    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
1979
1980
0
    kvm_slots_lock();
1981
0
    kvm_physical_sync_dirty_bitmap(kml, section);
1982
0
    kvm_slots_unlock();
1983
0
}
1984
1985
static void kvm_log_sync_global(MemoryListener *l, bool last_stage)
1986
0
{
1987
0
    KVMMemoryListener *kml = container_of(l, KVMMemoryListener, listener);
1988
0
    KVMState *s = kvm_state;
1989
0
    KVMSlot *mem;
1990
0
    int i;
1991
1992
    /* Flush all kernel dirty addresses into KVMSlot dirty bitmap */
1993
0
    kvm_dirty_ring_flush();
1994
1995
0
    kvm_slots_lock();
1996
0
    for (i = 0; i < kml->nr_slots_allocated; i++) {
1997
0
        mem = &kml->slots[i];
1998
0
        if (mem->memory_size && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
1999
0
            kvm_slot_sync_dirty_pages(mem);
2000
2001
0
            if (s->kvm_dirty_ring_with_bitmap && last_stage &&
2002
0
                kvm_slot_get_dirty_log(s, mem)) {
2003
0
                kvm_slot_sync_dirty_pages(mem);
2004
0
            }
2005
2006
            /*
2007
             * This is not needed by KVM_GET_DIRTY_LOG because the
2008
             * ioctl will unconditionally overwrite the whole region.
2009
             * However kvm dirty ring has no such side effect.
2010
             */
2011
0
            kvm_slot_reset_dirty_pages(mem);
2012
0
        }
2013
0
    }
2014
0
    kvm_slots_unlock();
2015
0
}
2016
2017
static void kvm_log_clear(MemoryListener *listener,
2018
                          MemoryRegionSection *section)
2019
0
{
2020
0
    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
2021
0
    int r;
2022
2023
0
    r = kvm_physical_log_clear(kml, section);
2024
0
    if (r < 0) {
2025
0
        error_report_once("%s: kvm log clear failed: mr=%s "
2026
0
                          "offset=%"HWADDR_PRIx" size=%"PRIx64, __func__,
2027
0
                          section->mr->name, section->offset_within_region,
2028
0
                          int128_get64(section->size));
2029
0
        abort();
2030
0
    }
2031
0
}
2032
2033
static void kvm_mem_ioeventfd_add(MemoryListener *listener,
2034
                                  MemoryRegionSection *section,
2035
                                  bool match_data, uint64_t data,
2036
                                  EventNotifier *e)
2037
0
{
2038
0
    int fd = event_notifier_get_fd(e);
2039
0
    int r;
2040
2041
0
    r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
2042
0
                               data, true, int128_get64(section->size),
2043
0
                               match_data);
2044
0
    if (r < 0) {
2045
0
        fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n",
2046
0
                __func__, strerror(-r), -r);
2047
0
        abort();
2048
0
    }
2049
0
}
2050
2051
static void kvm_mem_ioeventfd_del(MemoryListener *listener,
2052
                                  MemoryRegionSection *section,
2053
                                  bool match_data, uint64_t data,
2054
                                  EventNotifier *e)
2055
0
{
2056
0
    int fd = event_notifier_get_fd(e);
2057
0
    int r;
2058
2059
0
    r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
2060
0
                               data, false, int128_get64(section->size),
2061
0
                               match_data);
2062
0
    if (r < 0) {
2063
0
        fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n",
2064
0
                __func__, strerror(-r), -r);
2065
0
        abort();
2066
0
    }
2067
0
}
2068
2069
static void kvm_io_ioeventfd_add(MemoryListener *listener,
2070
                                 MemoryRegionSection *section,
2071
                                 bool match_data, uint64_t data,
2072
                                 EventNotifier *e)
2073
0
{
2074
0
    int fd = event_notifier_get_fd(e);
2075
0
    int r;
2076
2077
0
    r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
2078
0
                              data, true, int128_get64(section->size),
2079
0
                              match_data);
2080
0
    if (r < 0) {
2081
0
        fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n",
2082
0
                __func__, strerror(-r), -r);
2083
0
        abort();
2084
0
    }
2085
0
}
2086
2087
static void kvm_io_ioeventfd_del(MemoryListener *listener,
2088
                                 MemoryRegionSection *section,
2089
                                 bool match_data, uint64_t data,
2090
                                 EventNotifier *e)
2091
2092
0
{
2093
0
    int fd = event_notifier_get_fd(e);
2094
0
    int r;
2095
2096
0
    r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
2097
0
                              data, false, int128_get64(section->size),
2098
0
                              match_data);
2099
0
    if (r < 0) {
2100
0
        fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n",
2101
0
                __func__, strerror(-r), -r);
2102
0
        abort();
2103
0
    }
2104
0
}
2105
2106
void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml,
2107
                                  AddressSpace *as, int as_id, const char *name)
2108
0
{
2109
0
    int i;
2110
2111
0
    kml->as_id = as_id;
2112
2113
0
    kvm_slots_grow(kml, KVM_MEMSLOTS_NR_ALLOC_DEFAULT);
2114
2115
0
    QSIMPLEQ_INIT(&kml->transaction_add);
2116
0
    QSIMPLEQ_INIT(&kml->transaction_del);
2117
2118
0
    kml->listener.region_add = kvm_region_add;
2119
0
    kml->listener.region_del = kvm_region_del;
2120
0
    kml->listener.commit = kvm_region_commit;
2121
0
    kml->listener.log_start = kvm_log_start;
2122
0
    kml->listener.log_stop = kvm_log_stop;
2123
0
    kml->listener.priority = MEMORY_LISTENER_PRIORITY_ACCEL;
2124
0
    kml->listener.name = name;
2125
2126
0
    if (s->kvm_dirty_ring_size) {
2127
0
        kml->listener.log_sync_global = kvm_log_sync_global;
2128
0
    } else {
2129
0
        kml->listener.log_sync = kvm_log_sync;
2130
0
        kml->listener.log_clear = kvm_log_clear;
2131
0
    }
2132
2133
0
    memory_listener_register(&kml->listener, as);
2134
2135
0
    for (i = 0; i < s->nr_as; ++i) {
2136
0
        if (!s->as[i].as) {
2137
0
            s->as[i].as = as;
2138
0
            s->as[i].ml = kml;
2139
0
            break;
2140
0
        }
2141
0
    }
2142
0
}
2143
2144
static MemoryListener kvm_io_listener = {
2145
    .name = "kvm-io",
2146
    .coalesced_io_add = kvm_coalesce_pio_add,
2147
    .coalesced_io_del = kvm_coalesce_pio_del,
2148
    .eventfd_add = kvm_io_ioeventfd_add,
2149
    .eventfd_del = kvm_io_ioeventfd_del,
2150
    .priority = MEMORY_LISTENER_PRIORITY_DEV_BACKEND,
2151
};
2152
2153
int kvm_set_irq(KVMState *s, int irq, int level)
2154
0
{
2155
0
    struct kvm_irq_level event;
2156
0
    int ret;
2157
2158
0
    assert(kvm_async_interrupts_enabled());
2159
2160
0
    event.level = level;
2161
0
    event.irq = irq;
2162
0
    ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
2163
0
    if (ret < 0) {
2164
0
        perror("kvm_set_irq");
2165
0
        abort();
2166
0
    }
2167
2168
0
    return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
2169
0
}
2170
2171
#ifdef KVM_CAP_IRQ_ROUTING
2172
typedef struct KVMMSIRoute {
2173
    struct kvm_irq_routing_entry kroute;
2174
    QTAILQ_ENTRY(KVMMSIRoute) entry;
2175
} KVMMSIRoute;
2176
2177
static void set_gsi(KVMState *s, unsigned int gsi)
2178
0
{
2179
0
    set_bit(gsi, s->used_gsi_bitmap);
2180
0
}
2181
2182
static void clear_gsi(KVMState *s, unsigned int gsi)
2183
0
{
2184
0
    clear_bit(gsi, s->used_gsi_bitmap);
2185
0
}
2186
2187
void kvm_init_irq_routing(KVMState *s)
2188
0
{
2189
0
    int gsi_count;
2190
2191
0
    gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
2192
0
    if (gsi_count > 0) {
2193
        /* Round up so we can search ints using ffs */
2194
0
        s->used_gsi_bitmap = bitmap_new(gsi_count);
2195
0
        s->gsi_count = gsi_count;
2196
0
    }
2197
2198
0
    s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
2199
0
    s->nr_allocated_irq_routes = 0;
2200
2201
0
    kvm_arch_init_irq_routing(s);
2202
0
}
2203
2204
void kvm_irqchip_commit_routes(KVMState *s)
2205
0
{
2206
0
    int ret;
2207
2208
0
    if (kvm_gsi_direct_mapping()) {
2209
0
        return;
2210
0
    }
2211
2212
0
    if (!kvm_gsi_routing_enabled()) {
2213
0
        return;
2214
0
    }
2215
2216
0
    s->irq_routes->flags = 0;
2217
0
    trace_kvm_irqchip_commit_routes();
2218
0
    ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
2219
0
    assert(ret == 0);
2220
0
}
2221
2222
void kvm_add_routing_entry(KVMState *s,
2223
                           struct kvm_irq_routing_entry *entry)
2224
0
{
2225
0
    struct kvm_irq_routing_entry *new;
2226
0
    int n, size;
2227
2228
0
    if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
2229
0
        n = s->nr_allocated_irq_routes * 2;
2230
0
        if (n < 64) {
2231
0
            n = 64;
2232
0
        }
2233
0
        size = sizeof(struct kvm_irq_routing);
2234
0
        size += n * sizeof(*new);
2235
0
        s->irq_routes = g_realloc(s->irq_routes, size);
2236
0
        s->nr_allocated_irq_routes = n;
2237
0
    }
2238
0
    n = s->irq_routes->nr++;
2239
0
    new = &s->irq_routes->entries[n];
2240
2241
0
    *new = *entry;
2242
2243
0
    set_gsi(s, entry->gsi);
2244
0
}
2245
2246
static int kvm_update_routing_entry(KVMState *s,
2247
                                    struct kvm_irq_routing_entry *new_entry)
2248
0
{
2249
0
    struct kvm_irq_routing_entry *entry;
2250
0
    int n;
2251
2252
0
    for (n = 0; n < s->irq_routes->nr; n++) {
2253
0
        entry = &s->irq_routes->entries[n];
2254
0
        if (entry->gsi != new_entry->gsi) {
2255
0
            continue;
2256
0
        }
2257
2258
0
        if(!memcmp(entry, new_entry, sizeof *entry)) {
2259
0
            return 0;
2260
0
        }
2261
2262
0
        *entry = *new_entry;
2263
2264
0
        return 0;
2265
0
    }
2266
2267
0
    return -ESRCH;
2268
0
}
2269
2270
void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
2271
0
{
2272
0
    struct kvm_irq_routing_entry e = {};
2273
2274
0
    assert(pin < s->gsi_count);
2275
2276
0
    e.gsi = irq;
2277
0
    e.type = KVM_IRQ_ROUTING_IRQCHIP;
2278
0
    e.flags = 0;
2279
0
    e.u.irqchip.irqchip = irqchip;
2280
0
    e.u.irqchip.pin = pin;
2281
0
    kvm_add_routing_entry(s, &e);
2282
0
}
2283
2284
void kvm_irqchip_release_virq(KVMState *s, int virq)
2285
0
{
2286
0
    struct kvm_irq_routing_entry *e;
2287
0
    int i;
2288
2289
0
    if (kvm_gsi_direct_mapping()) {
2290
0
        return;
2291
0
    }
2292
2293
0
    for (i = 0; i < s->irq_routes->nr; i++) {
2294
0
        e = &s->irq_routes->entries[i];
2295
0
        if (e->gsi == virq) {
2296
0
            s->irq_routes->nr--;
2297
0
            *e = s->irq_routes->entries[s->irq_routes->nr];
2298
0
        }
2299
0
    }
2300
0
    clear_gsi(s, virq);
2301
0
    kvm_arch_release_virq_post(virq);
2302
0
    trace_kvm_irqchip_release_virq(virq);
2303
0
}
2304
2305
void kvm_irqchip_add_change_notifier(Notifier *n)
2306
0
{
2307
0
    notifier_list_add(&kvm_irqchip_change_notifiers, n);
2308
0
}
2309
2310
void kvm_irqchip_remove_change_notifier(Notifier *n)
2311
0
{
2312
0
    notifier_remove(n);
2313
0
}
2314
2315
void kvm_irqchip_change_notify(void)
2316
0
{
2317
0
    notifier_list_notify(&kvm_irqchip_change_notifiers, NULL);
2318
0
}
2319
2320
void kvm_vmfd_add_change_notifier(NotifierWithReturn *n)
2321
0
{
2322
0
    notifier_with_return_list_add(&register_vmfd_changed_notifiers, n);
2323
0
}
2324
2325
void kvm_vmfd_remove_change_notifier(NotifierWithReturn *n)
2326
0
{
2327
0
    notifier_with_return_remove(n);
2328
0
}
2329
2330
static int kvm_vmfd_change_notify(Error **errp)
2331
0
{
2332
0
    return notifier_with_return_list_notify(&register_vmfd_changed_notifiers,
2333
0
                                            &vmfd_notifier, errp);
2334
0
}
2335
2336
void kvm_vcpufd_add_change_notifier(NotifierWithReturn *n)
2337
0
{
2338
0
    notifier_with_return_list_add(&register_vcpufd_changed_notifiers, n);
2339
0
}
2340
2341
void kvm_vcpufd_remove_change_notifier(NotifierWithReturn *n)
2342
0
{
2343
0
    notifier_with_return_remove(n);
2344
0
}
2345
2346
static int kvm_vcpufd_change_notify(Error **errp)
2347
0
{
2348
0
    return notifier_with_return_list_notify(&register_vcpufd_changed_notifiers,
2349
0
                                            &vmfd_notifier, errp);
2350
0
}
2351
2352
int kvm_irqchip_get_virq(KVMState *s)
2353
0
{
2354
0
    int next_virq;
2355
2356
    /* Return the lowest unused GSI in the bitmap */
2357
0
    next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count);
2358
0
    if (next_virq >= s->gsi_count) {
2359
0
        return -ENOSPC;
2360
0
    } else {
2361
0
        return next_virq;
2362
0
    }
2363
0
}
2364
2365
int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
2366
0
{
2367
0
    struct kvm_msi msi;
2368
2369
0
    msi.address_lo = (uint32_t)msg.address;
2370
0
    msi.address_hi = msg.address >> 32;
2371
0
    msi.data = le32_to_cpu(msg.data);
2372
0
    msi.flags = 0;
2373
0
    memset(msi.pad, 0, sizeof(msi.pad));
2374
2375
0
    return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
2376
0
}
2377
2378
int kvm_irqchip_add_msi_route(AccelRouteChange *c, int vector, PCIDevice *dev)
2379
0
{
2380
0
    struct kvm_irq_routing_entry kroute = {};
2381
0
    int virq;
2382
0
    KVMState *s = KVM_STATE(c->accel);
2383
0
    MSIMessage msg = {0, 0};
2384
2385
0
    if (pci_available && dev) {
2386
0
        msg = pci_get_msi_message(dev, vector);
2387
0
    }
2388
2389
0
    if (kvm_gsi_direct_mapping()) {
2390
0
        return kvm_arch_msi_data_to_gsi(msg.data);
2391
0
    }
2392
2393
0
    if (!kvm_gsi_routing_enabled()) {
2394
0
        return -ENOSYS;
2395
0
    }
2396
2397
0
    virq = kvm_irqchip_get_virq(s);
2398
0
    if (virq < 0) {
2399
0
        return virq;
2400
0
    }
2401
2402
0
    kroute.gsi = virq;
2403
0
    kroute.type = KVM_IRQ_ROUTING_MSI;
2404
0
    kroute.flags = 0;
2405
0
    kroute.u.msi.address_lo = (uint32_t)msg.address;
2406
0
    kroute.u.msi.address_hi = msg.address >> 32;
2407
0
    kroute.u.msi.data = le32_to_cpu(msg.data);
2408
0
    if (pci_available && kvm_msi_devid_required()) {
2409
0
        kroute.flags = KVM_MSI_VALID_DEVID;
2410
0
        kroute.u.msi.devid = pci_requester_id(dev);
2411
0
    }
2412
0
    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
2413
0
        kvm_irqchip_release_virq(s, virq);
2414
0
        return -EINVAL;
2415
0
    }
2416
2417
0
    if (s->irq_routes->nr < s->gsi_count) {
2418
0
        trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A",
2419
0
                                        vector, virq);
2420
2421
0
        kvm_add_routing_entry(s, &kroute);
2422
0
        kvm_arch_add_msi_route_post(&kroute, vector, dev);
2423
0
        c->changes++;
2424
0
    } else {
2425
0
        kvm_irqchip_release_virq(s, virq);
2426
0
        return -ENOSPC;
2427
0
    }
2428
2429
0
    return virq;
2430
0
}
2431
2432
int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
2433
                                 PCIDevice *dev)
2434
0
{
2435
0
    struct kvm_irq_routing_entry kroute = {};
2436
2437
0
    if (kvm_gsi_direct_mapping()) {
2438
0
        return 0;
2439
0
    }
2440
2441
0
    if (!kvm_irqchip_in_kernel()) {
2442
0
        return -ENOSYS;
2443
0
    }
2444
2445
0
    kroute.gsi = virq;
2446
0
    kroute.type = KVM_IRQ_ROUTING_MSI;
2447
0
    kroute.flags = 0;
2448
0
    kroute.u.msi.address_lo = (uint32_t)msg.address;
2449
0
    kroute.u.msi.address_hi = msg.address >> 32;
2450
0
    kroute.u.msi.data = le32_to_cpu(msg.data);
2451
0
    if (pci_available && kvm_msi_devid_required()) {
2452
0
        kroute.flags = KVM_MSI_VALID_DEVID;
2453
0
        kroute.u.msi.devid = pci_requester_id(dev);
2454
0
    }
2455
0
    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
2456
0
        return -EINVAL;
2457
0
    }
2458
2459
0
    trace_kvm_irqchip_update_msi_route(virq);
2460
2461
0
    return kvm_update_routing_entry(s, &kroute);
2462
0
}
2463
2464
static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event,
2465
                                    EventNotifier *resample, int virq,
2466
                                    bool assign)
2467
0
{
2468
0
    int fd = event_notifier_get_fd(event);
2469
0
    int rfd = resample ? event_notifier_get_fd(resample) : -1;
2470
2471
0
    struct kvm_irqfd irqfd = {
2472
0
        .fd = fd,
2473
0
        .gsi = virq,
2474
0
        .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
2475
0
    };
2476
2477
0
    if (rfd != -1) {
2478
0
        assert(assign);
2479
0
        if (kvm_irqchip_is_split()) {
2480
            /*
2481
             * When the slow irqchip (e.g. IOAPIC) is in the
2482
             * userspace, KVM kernel resamplefd will not work because
2483
             * the EOI of the interrupt will be delivered to userspace
2484
             * instead, so the KVM kernel resamplefd kick will be
2485
             * skipped.  The userspace here mimics what the kernel
2486
             * provides with resamplefd, remember the resamplefd and
2487
             * kick it when we receive EOI of this IRQ.
2488
             *
2489
             * This is hackery because IOAPIC is mostly bypassed
2490
             * (except EOI broadcasts) when irqfd is used.  However
2491
             * this can bring much performance back for split irqchip
2492
             * with INTx IRQs (for VFIO, this gives 93% perf of the
2493
             * full fast path, which is 46% perf boost comparing to
2494
             * the INTx slow path).
2495
             */
2496
0
            kvm_resample_fd_insert(virq, resample);
2497
0
        } else {
2498
0
            irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
2499
0
            irqfd.resamplefd = rfd;
2500
0
        }
2501
0
    } else if (!assign) {
2502
0
        if (kvm_irqchip_is_split()) {
2503
0
            kvm_resample_fd_remove(virq);
2504
0
        }
2505
0
    }
2506
2507
0
    return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
2508
0
}
2509
2510
#else /* !KVM_CAP_IRQ_ROUTING */
2511
2512
void kvm_init_irq_routing(KVMState *s)
2513
{
2514
}
2515
2516
void kvm_irqchip_release_virq(KVMState *s, int virq)
2517
{
2518
}
2519
2520
int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
2521
{
2522
    abort();
2523
}
2524
2525
int kvm_irqchip_add_msi_route(AccelRouteChange *c, int vector, PCIDevice *dev)
2526
{
2527
    return -ENOSYS;
2528
}
2529
2530
int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
2531
{
2532
    return -ENOSYS;
2533
}
2534
2535
int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
2536
{
2537
    return -ENOSYS;
2538
}
2539
2540
static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event,
2541
                                    EventNotifier *resample, int virq,
2542
                                    bool assign)
2543
{
2544
    abort();
2545
}
2546
2547
int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
2548
{
2549
    return -ENOSYS;
2550
}
2551
#endif /* !KVM_CAP_IRQ_ROUTING */
2552
2553
int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
2554
                                       EventNotifier *rn, int virq)
2555
0
{
2556
0
    return kvm_irqchip_assign_irqfd(s, n, rn, virq, true);
2557
0
}
2558
2559
int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
2560
                                          int virq)
2561
0
{
2562
0
    return kvm_irqchip_assign_irqfd(s, n, NULL, virq, false);
2563
0
}
2564
2565
int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
2566
                                   EventNotifier *rn, qemu_irq irq)
2567
0
{
2568
0
    gpointer key, gsi;
2569
0
    gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
2570
2571
0
    if (!found) {
2572
0
        return -ENXIO;
2573
0
    }
2574
0
    return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi));
2575
0
}
2576
2577
int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n,
2578
                                      qemu_irq irq)
2579
0
{
2580
0
    gpointer key, gsi;
2581
0
    gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
2582
2583
0
    if (!found) {
2584
0
        return -ENXIO;
2585
0
    }
2586
0
    return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi));
2587
0
}
2588
2589
void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi)
2590
0
{
2591
0
    g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi));
2592
0
}
2593
2594
static int do_kvm_irqchip_create(KVMState *s)
2595
0
{
2596
0
    int ret;
2597
0
    if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
2598
0
        ;
2599
0
    } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) {
2600
0
        ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0);
2601
0
        if (ret < 0) {
2602
0
            fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret));
2603
0
            exit(1);
2604
0
        }
2605
0
    } else {
2606
0
        return -EOPNOTSUPP;
2607
0
    }
2608
2609
0
    if (kvm_check_extension(s, KVM_CAP_IRQFD) <= 0) {
2610
0
        fprintf(stderr, "kvm: irqfd not implemented\n");
2611
0
        exit(1);
2612
0
    }
2613
2614
    /* First probe and see if there's a arch-specific hook to create the
2615
     * in-kernel irqchip for us */
2616
0
    ret = kvm_arch_irqchip_create(s);
2617
0
    if (ret == 0) {
2618
0
        if (s->kernel_irqchip_split == ON_OFF_AUTO_ON) {
2619
0
            error_report("Split IRQ chip mode not supported.");
2620
0
            exit(1);
2621
0
        } else {
2622
0
            ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
2623
0
        }
2624
0
    }
2625
0
    if (ret < 0) {
2626
0
        fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret));
2627
0
        exit(1);
2628
0
    }
2629
2630
0
    return 0;
2631
0
}
2632
2633
static void kvm_irqchip_create(KVMState *s)
2634
0
{
2635
0
    assert(s->kernel_irqchip_split != ON_OFF_AUTO_AUTO);
2636
2637
0
    if (do_kvm_irqchip_create(s) < 0) {
2638
0
        return;
2639
0
    }
2640
0
    kvm_kernel_irqchip = true;
2641
    /* If we have an in-kernel IRQ chip then we must have asynchronous
2642
     * interrupt delivery (though the reverse is not necessarily true)
2643
     */
2644
0
    kvm_async_interrupts_allowed = true;
2645
0
    kvm_halt_in_kernel_allowed = true;
2646
2647
0
    kvm_init_irq_routing(s);
2648
2649
0
    s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal);
2650
0
}
2651
2652
/* Find number of supported CPUs using the recommended
2653
 * procedure from the kernel API documentation to cope with
2654
 * older kernels that may be missing capabilities.
2655
 */
2656
static int kvm_recommended_vcpus(KVMState *s)
2657
0
{
2658
0
    int ret = kvm_vm_check_extension(s, KVM_CAP_NR_VCPUS);
2659
0
    return (ret) ? ret : 4;
2660
0
}
2661
2662
static int kvm_max_vcpus(KVMState *s)
2663
0
{
2664
0
    int ret = kvm_vm_check_extension(s, KVM_CAP_MAX_VCPUS);
2665
0
    return (ret) ? ret : kvm_recommended_vcpus(s);
2666
0
}
2667
2668
static int kvm_max_vcpu_id(KVMState *s)
2669
0
{
2670
0
    int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID);
2671
0
    return (ret) ? ret : kvm_max_vcpus(s);
2672
0
}
2673
2674
bool kvm_vcpu_id_is_valid(int vcpu_id)
2675
0
{
2676
0
    KVMState *s = KVM_STATE(current_accel());
2677
0
    return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s);
2678
0
}
2679
2680
bool kvm_dirty_ring_enabled(void)
2681
0
{
2682
0
    return kvm_state && kvm_state->kvm_dirty_ring_size;
2683
0
}
2684
2685
static void query_stats_cb(StatsResultList **result, StatsTarget target,
2686
                           strList *names, strList *targets, Error **errp);
2687
static void query_stats_schemas_cb(StatsSchemaList **result, Error **errp);
2688
2689
uint32_t kvm_dirty_ring_size(void)
2690
0
{
2691
0
    return kvm_state->kvm_dirty_ring_size;
2692
0
}
2693
2694
static int do_kvm_create_vm(KVMState *s, int type)
2695
0
{
2696
0
    int ret;
2697
2698
0
    do {
2699
0
        ret = kvm_ioctl(s, KVM_CREATE_VM, type);
2700
0
    } while (ret == -EINTR);
2701
2702
0
    if (ret < 0) {
2703
0
        error_report("ioctl(KVM_CREATE_VM) failed: %s", strerror(-ret));
2704
2705
#ifdef TARGET_S390X
2706
        if (ret == -EINVAL) {
2707
            error_printf("Host kernel setup problem detected."
2708
                         " Please verify:\n");
2709
            error_printf("- for kernels supporting the"
2710
                        " switch_amode or user_mode parameters, whether");
2711
            error_printf(" user space is running in primary address space\n");
2712
            error_printf("- for kernels supporting the vm.allocate_pgste"
2713
                         " sysctl, whether it is enabled\n");
2714
        }
2715
#elif defined(TARGET_PPC)
2716
        if (ret == -EINVAL) {
2717
            error_printf("PPC KVM module is not loaded. Try modprobe kvm_%s.\n",
2718
                         (type == 2) ? "pr" : "hv");
2719
        }
2720
#endif
2721
0
    }
2722
2723
0
    return ret;
2724
0
}
2725
2726
static int find_kvm_machine_type(MachineState *ms)
2727
0
{
2728
0
    MachineClass *mc = MACHINE_GET_CLASS(ms);
2729
0
    int type;
2730
2731
0
    if (object_property_find(OBJECT(current_machine), "kvm-type")) {
2732
0
        g_autofree char *kvm_type;
2733
0
        kvm_type = object_property_get_str(OBJECT(current_machine),
2734
0
                                           "kvm-type",
2735
0
                                           &error_abort);
2736
0
        type = mc->kvm_type(ms, kvm_type);
2737
0
    } else if (mc->kvm_type) {
2738
0
        type = mc->kvm_type(ms, NULL);
2739
0
    } else {
2740
0
        type = kvm_arch_get_default_type(ms);
2741
0
    }
2742
0
    return type;
2743
0
}
2744
2745
static int kvm_setup_dirty_ring(KVMState *s)
2746
0
{
2747
0
    uint64_t dirty_log_manual_caps;
2748
0
    int ret;
2749
2750
    /*
2751
     * Enable KVM dirty ring if supported, otherwise fall back to
2752
     * dirty logging mode
2753
     */
2754
0
    ret = kvm_dirty_ring_init(s);
2755
0
    if (ret < 0) {
2756
0
        return ret;
2757
0
    }
2758
2759
    /*
2760
     * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 is not needed when dirty ring is
2761
     * enabled.  More importantly, KVM_DIRTY_LOG_INITIALLY_SET will assume no
2762
     * page is wr-protected initially, which is against how kvm dirty ring is
2763
     * usage - kvm dirty ring requires all pages are wr-protected at the very
2764
     * beginning.  Enabling this feature for dirty ring causes data corruption.
2765
     *
2766
     * TODO: Without KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and kvm clear dirty log,
2767
     * we may expect a higher stall time when starting the migration.  In the
2768
     * future we can enable KVM_CLEAR_DIRTY_LOG to work with dirty ring too:
2769
     * instead of clearing dirty bit, it can be a way to explicitly wr-protect
2770
     * guest pages.
2771
     */
2772
0
    if (!s->kvm_dirty_ring_size) {
2773
0
        dirty_log_manual_caps =
2774
0
            kvm_check_extension(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2);
2775
0
        dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE |
2776
0
                                  KVM_DIRTY_LOG_INITIALLY_SET);
2777
0
        s->manual_dirty_log_protect = dirty_log_manual_caps;
2778
0
        if (dirty_log_manual_caps) {
2779
0
            ret = kvm_vm_enable_cap(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2, 0,
2780
0
                                    dirty_log_manual_caps);
2781
0
            if (ret) {
2782
0
                warn_report("Trying to enable capability %"PRIu64" of "
2783
0
                            "KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 but failed. "
2784
0
                            "Falling back to the legacy mode. ",
2785
0
                            dirty_log_manual_caps);
2786
0
                s->manual_dirty_log_protect = 0;
2787
0
            }
2788
0
        }
2789
0
    }
2790
2791
0
    return 0;
2792
0
}
2793
2794
static int kvm_reset_vmfd(MachineState *ms)
2795
0
{
2796
0
    KVMState *s;
2797
0
    KVMMemoryListener *kml;
2798
0
    int ret = 0, type;
2799
0
    Error *err = NULL;
2800
2801
    /*
2802
     * bail if the current architecture does not support VM file
2803
     * descriptor change.
2804
     */
2805
0
    if (!kvm_arch_supports_vmfd_change()) {
2806
0
        error_report("This target architecture does not support KVM VM "
2807
0
                     "file descriptor change.");
2808
0
        return -EOPNOTSUPP;
2809
0
    }
2810
2811
0
    s = KVM_STATE(ms->accelerator);
2812
0
    kml = &s->memory_listener;
2813
2814
0
    memory_listener_unregister(&kml->listener);
2815
0
    memory_listener_unregister(&kvm_io_listener);
2816
2817
0
    vmfd_notifier.pre = true;
2818
0
    ret = kvm_vmfd_change_notify(&err);
2819
0
    if (ret < 0) {
2820
0
        return ret;
2821
0
    }
2822
0
    assert(!err);
2823
2824
0
    if (s->vmfd >= 0) {
2825
0
        close(s->vmfd);
2826
0
    }
2827
2828
0
    type = find_kvm_machine_type(ms);
2829
0
    if (type < 0) {
2830
0
        return -EINVAL;
2831
0
    }
2832
2833
0
    ret = do_kvm_create_vm(s, type);
2834
0
    if (ret < 0) {
2835
0
        return ret;
2836
0
    }
2837
2838
0
    s->vmfd = ret;
2839
2840
    /* guest state is now unprotected again */
2841
0
    kvm_state->guest_state_protected = false;
2842
2843
0
    kvm_setup_dirty_ring(s);
2844
2845
    /* rebind memory to new vm fd */
2846
0
    ret = ram_block_rebind(&err);
2847
0
    if (ret < 0) {
2848
0
        return ret;
2849
0
    }
2850
0
    assert(!err);
2851
2852
0
    ret = kvm_arch_on_vmfd_change(ms, s);
2853
0
    if (ret < 0) {
2854
0
        return ret;
2855
0
    }
2856
2857
0
    if (s->kernel_irqchip_allowed) {
2858
        /* ignore return from this function */
2859
0
        do_kvm_irqchip_create(s);
2860
0
    }
2861
2862
    /*
2863
     * notify everyone that vmfd has changed.
2864
     */
2865
0
    vmfd_notifier.vmfd = s->vmfd;
2866
0
    vmfd_notifier.pre = false;
2867
2868
0
    ret = kvm_vmfd_change_notify(&err);
2869
0
    if (ret < 0) {
2870
0
        return ret;
2871
0
    }
2872
0
    assert(!err);
2873
2874
    /*
2875
     * rebind new vcpu fds with the new kvm fds
2876
     * These can only be called after kvm_arch_on_vmfd_change()
2877
     */
2878
0
    ret = kvm_rebind_vcpus(&err);
2879
0
    if (ret < 0) {
2880
0
        return ret;
2881
0
    }
2882
0
    assert(!err);
2883
2884
    /* notify everyone that vcpu fd has changed. */
2885
0
    ret = kvm_vcpufd_change_notify(&err);
2886
0
    if (ret < 0) {
2887
0
        return ret;
2888
0
    }
2889
0
    assert(!err);
2890
2891
    /* these can be only called after ram_block_rebind() */
2892
0
    memory_listener_register(&kml->listener, &address_space_memory);
2893
0
    memory_listener_register(&kvm_io_listener, &address_space_io);
2894
2895
    /*
2896
     * kvm fd has changed. Commit the irq routes to KVM once more.
2897
     */
2898
0
    kvm_irqchip_commit_routes(s);
2899
    /*
2900
     * for confidential guest, this is the last possible place where we
2901
     * can call synchronize_all_post_init() to sync all vcpu states to
2902
     * kvm.
2903
     */
2904
0
    if (ms->cgs) {
2905
0
        cpu_synchronize_all_post_init();
2906
0
    }
2907
0
    trace_kvm_reset_vmfd();
2908
0
    return ret;
2909
0
}
2910
2911
static int kvm_init(AccelState *as, MachineState *ms)
2912
0
{
2913
0
    MachineClass *mc = MACHINE_GET_CLASS(ms);
2914
0
    static const char upgrade_note[] =
2915
0
        "Please upgrade to at least kernel 4.5.\n";
2916
0
    const struct {
2917
0
        const char *name;
2918
0
        int num;
2919
0
    } num_cpus[] = {
2920
0
        { "SMP",          ms->smp.cpus },
2921
0
        { "hotpluggable", ms->smp.max_cpus },
2922
0
        { /* end of list */ }
2923
0
    }, *nc = num_cpus;
2924
0
    int soft_vcpus_limit, hard_vcpus_limit;
2925
0
    KVMState *s = KVM_STATE(as);
2926
0
    const KVMCapabilityInfo *missing_cap;
2927
0
    int ret;
2928
0
    int type;
2929
2930
0
    qemu_mutex_init(&kml_slots_lock);
2931
0
    qemu_mutex_init(&kvm_resample_fd_list_lock);
2932
2933
    /*
2934
     * On systems where the kernel can support different base page
2935
     * sizes, host page size may be different from TARGET_PAGE_SIZE,
2936
     * even with KVM.  TARGET_PAGE_SIZE is assumed to be the minimum
2937
     * page size for the system though.
2938
     */
2939
0
    assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size());
2940
2941
0
    s->sigmask_len = 8;
2942
0
    accel_blocker_init();
2943
2944
0
#ifdef TARGET_KVM_HAVE_GUEST_DEBUG
2945
0
    QTAILQ_INIT(&s->kvm_sw_breakpoints);
2946
0
#endif
2947
0
    QLIST_INIT(&s->kvm_parked_vcpus);
2948
0
    s->fd = qemu_open_old(s->device ?: "/dev/kvm", O_RDWR);
2949
0
    if (s->fd == -1) {
2950
0
        error_report("Could not access KVM kernel module: %m");
2951
0
        ret = -errno;
2952
0
        goto err;
2953
0
    }
2954
2955
0
    ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
2956
0
    if (ret < KVM_API_VERSION) {
2957
0
        if (ret >= 0) {
2958
0
            ret = -EINVAL;
2959
0
        }
2960
0
        error_report("kvm version too old");
2961
0
        goto err;
2962
0
    }
2963
2964
0
    if (ret > KVM_API_VERSION) {
2965
0
        ret = -EINVAL;
2966
0
        error_report("kvm version not supported");
2967
0
        goto err;
2968
0
    }
2969
2970
0
    kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT);
2971
0
    s->nr_slots_max = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
2972
2973
    /* If unspecified, use the default value */
2974
0
    if (!s->nr_slots_max) {
2975
0
        s->nr_slots_max = KVM_MEMSLOTS_NR_MAX_DEFAULT;
2976
0
    }
2977
2978
0
    type = find_kvm_machine_type(ms);
2979
0
    if (type < 0) {
2980
0
        ret = -EINVAL;
2981
0
        goto err;
2982
0
    }
2983
2984
0
    ret = do_kvm_create_vm(s, type);
2985
0
    if (ret < 0) {
2986
0
        goto err;
2987
0
    }
2988
2989
0
    s->vmfd = ret;
2990
2991
0
    s->nr_as = kvm_vm_check_extension(s, KVM_CAP_MULTI_ADDRESS_SPACE);
2992
0
    if (s->nr_as <= 1) {
2993
0
        s->nr_as = 1;
2994
0
    }
2995
0
    s->as = g_new0(struct KVMAs, s->nr_as);
2996
2997
    /* check the vcpu limits */
2998
0
    soft_vcpus_limit = kvm_recommended_vcpus(s);
2999
0
    hard_vcpus_limit = kvm_max_vcpus(s);
3000
3001
0
    while (nc->name) {
3002
0
        if (nc->num > soft_vcpus_limit) {
3003
0
            warn_report("Number of %s cpus requested (%d) exceeds "
3004
0
                        "the recommended cpus supported by KVM (%d)",
3005
0
                        nc->name, nc->num, soft_vcpus_limit);
3006
3007
0
            if (nc->num > hard_vcpus_limit) {
3008
0
                error_report("Number of %s cpus requested (%d) exceeds "
3009
0
                             "the maximum cpus supported by KVM (%d)",
3010
0
                             nc->name, nc->num, hard_vcpus_limit);
3011
0
                exit(1);
3012
0
            }
3013
0
        }
3014
0
        nc++;
3015
0
    }
3016
3017
0
    missing_cap = kvm_check_extension_list(s, kvm_required_capabilities);
3018
0
    if (!missing_cap) {
3019
0
        missing_cap =
3020
0
            kvm_check_extension_list(s, kvm_arch_required_capabilities);
3021
0
    }
3022
0
    if (missing_cap) {
3023
0
        ret = -EINVAL;
3024
0
        error_report("kvm does not support %s", missing_cap->name);
3025
0
        error_printf("%s", upgrade_note);
3026
0
        goto err;
3027
0
    }
3028
3029
0
    s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
3030
0
    s->coalesced_pio = s->coalesced_mmio &&
3031
0
                       kvm_check_extension(s, KVM_CAP_COALESCED_PIO);
3032
3033
0
    ret = kvm_setup_dirty_ring(s);
3034
0
    if (ret < 0) {
3035
0
        goto err;
3036
0
    }
3037
3038
0
#ifdef KVM_CAP_VCPU_EVENTS
3039
0
    s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
3040
0
#endif
3041
0
    s->max_nested_state_len = kvm_check_extension(s, KVM_CAP_NESTED_STATE);
3042
3043
0
    s->irq_set_ioctl = KVM_IRQ_LINE;
3044
0
    if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
3045
0
        s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
3046
0
    }
3047
3048
0
    kvm_readonly_mem_allowed =
3049
0
        (kvm_vm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
3050
3051
0
    kvm_resamplefds_allowed =
3052
0
        (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
3053
3054
0
    kvm_vm_attributes_allowed =
3055
0
        (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
3056
3057
0
#ifdef TARGET_KVM_HAVE_GUEST_DEBUG
3058
0
    if (kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG) > 0) {
3059
0
        as->gdbstub.sstep_flags = SSTEP_ENABLE;
3060
3061
0
        int guest_debug_flags =
3062
0
            kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG2);
3063
3064
0
        if (guest_debug_flags & KVM_GUESTDBG_BLOCKIRQ) {
3065
0
            as->gdbstub.sstep_flags |= SSTEP_NOIRQ;
3066
0
        }
3067
0
    }
3068
0
#endif
3069
3070
0
    kvm_state = s;
3071
3072
0
    ret = kvm_arch_init(ms, s);
3073
0
    if (ret < 0) {
3074
0
        goto err;
3075
0
    }
3076
3077
0
    kvm_supported_memory_attributes = kvm_vm_check_extension(s, KVM_CAP_MEMORY_ATTRIBUTES);
3078
0
    kvm_guest_memfd_supported =
3079
0
        kvm_vm_check_extension(s, KVM_CAP_GUEST_MEMFD) &&
3080
0
        kvm_vm_check_extension(s, KVM_CAP_USER_MEMORY2);
3081
0
    kvm_pre_fault_memory_supported = kvm_vm_check_extension(s, KVM_CAP_PRE_FAULT_MEMORY);
3082
0
    kvm_guest_memfd_flags_supported =
3083
0
        kvm_vm_check_extension(s, KVM_CAP_GUEST_MEMFD_FLAGS);
3084
3085
0
    if (s->kernel_irqchip_split == ON_OFF_AUTO_AUTO) {
3086
0
        s->kernel_irqchip_split = mc->default_kernel_irqchip_split ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
3087
0
    }
3088
3089
0
    qemu_register_reset(kvm_unpoison_all, NULL);
3090
3091
0
    if (s->kernel_irqchip_allowed) {
3092
0
        kvm_irqchip_create(s);
3093
0
    }
3094
3095
0
    s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
3096
0
    s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
3097
0
    s->memory_listener.listener.coalesced_io_add = kvm_coalesce_mmio_region;
3098
0
    s->memory_listener.listener.coalesced_io_del = kvm_uncoalesce_mmio_region;
3099
3100
0
    kvm_memory_listener_register(s, &s->memory_listener,
3101
0
                                 &address_space_memory, 0, "kvm-memory");
3102
0
    memory_listener_register(&kvm_io_listener,
3103
0
                             &address_space_io);
3104
3105
0
    s->sync_mmu = !!kvm_vm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
3106
0
    if (!s->sync_mmu) {
3107
0
        ret = ram_block_discard_disable(true);
3108
0
        assert(!ret);
3109
0
    }
3110
3111
0
    if (s->kvm_dirty_ring_size) {
3112
0
        kvm_dirty_ring_reaper_init(s);
3113
0
    }
3114
3115
0
    if (kvm_check_extension(kvm_state, KVM_CAP_BINARY_STATS_FD)) {
3116
0
        add_stats_callbacks(STATS_PROVIDER_KVM, query_stats_cb,
3117
0
                            query_stats_schemas_cb);
3118
0
    }
3119
3120
0
    return 0;
3121
3122
0
err:
3123
0
    assert(ret < 0);
3124
0
    if (s->vmfd >= 0) {
3125
0
        close(s->vmfd);
3126
0
    }
3127
0
    if (s->fd != -1) {
3128
0
        close(s->fd);
3129
0
    }
3130
0
    g_free(s->as);
3131
0
    g_free(s->memory_listener.slots);
3132
3133
0
    return ret;
3134
0
}
3135
3136
void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len)
3137
0
{
3138
0
    s->sigmask_len = sigmask_len;
3139
0
}
3140
3141
static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction,
3142
                          int size, uint32_t count)
3143
0
{
3144
0
    int i;
3145
0
    uint8_t *ptr = data;
3146
3147
0
    for (i = 0; i < count; i++) {
3148
0
        address_space_rw(&address_space_io, port, attrs,
3149
0
                         ptr, size,
3150
0
                         direction == KVM_EXIT_IO_OUT);
3151
0
        ptr += size;
3152
0
    }
3153
0
}
3154
3155
static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run)
3156
0
{
3157
0
    int i;
3158
3159
0
    fprintf(stderr, "KVM internal error. Suberror: %d\n",
3160
0
            run->internal.suberror);
3161
3162
0
    for (i = 0; i < run->internal.ndata; ++i) {
3163
0
        fprintf(stderr, "extra data[%d]: 0x%016"PRIx64"\n",
3164
0
                i, (uint64_t)run->internal.data[i]);
3165
0
    }
3166
0
    if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
3167
0
        fprintf(stderr, "emulation failure\n");
3168
0
        if (!kvm_arch_stop_on_emulation_error(cpu)) {
3169
0
            cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
3170
0
            return EXCP_INTERRUPT;
3171
0
        }
3172
0
    }
3173
    /* FIXME: Should trigger a qmp message to let management know
3174
     * something went wrong.
3175
     */
3176
0
    return -1;
3177
0
}
3178
3179
void kvm_flush_coalesced_mmio_buffer(void)
3180
0
{
3181
0
    KVMState *s = kvm_state;
3182
3183
0
    if (!s || s->coalesced_flush_in_progress) {
3184
0
        return;
3185
0
    }
3186
3187
0
    s->coalesced_flush_in_progress = true;
3188
3189
0
    if (s->coalesced_mmio_ring) {
3190
0
        struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
3191
0
        while (ring->first != ring->last) {
3192
0
            struct kvm_coalesced_mmio *ent;
3193
0
            const AddressSpace *as;
3194
3195
0
            ent = &ring->coalesced_mmio[ring->first];
3196
0
            if (ent->len > sizeof(ent->data)) {
3197
0
                warn_report("coalesced MMIO entry has invalid len %u",
3198
0
                            ent->len);
3199
0
            } else {
3200
0
                as = ent->pio == 1 ? &address_space_io : &address_space_memory;
3201
0
                address_space_write(as, ent->phys_addr, MEMTXATTRS_UNSPECIFIED,
3202
0
                                    ent->data, ent->len);
3203
0
            }
3204
3205
0
            smp_wmb();
3206
0
            ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
3207
0
        }
3208
0
    }
3209
3210
0
    s->coalesced_flush_in_progress = false;
3211
0
}
3212
3213
static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
3214
0
{
3215
0
    if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) {
3216
0
        Error *err = NULL;
3217
0
        int ret = kvm_arch_get_registers(cpu, &err);
3218
0
        if (ret) {
3219
0
            if (err) {
3220
0
                error_reportf_err(err, "Failed to synchronize CPU state: ");
3221
0
            } else {
3222
0
                error_report("Failed to get registers: %s", strerror(-ret));
3223
0
            }
3224
3225
0
            cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
3226
0
            vm_stop(RUN_STATE_INTERNAL_ERROR);
3227
0
        }
3228
3229
0
        cpu->vcpu_dirty = true;
3230
0
    }
3231
0
}
3232
3233
void kvm_cpu_synchronize_state(CPUState *cpu)
3234
0
{
3235
0
    if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) {
3236
0
        run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL);
3237
0
    }
3238
0
}
3239
3240
static bool kvm_cpu_synchronize_put(CPUState *cpu, KvmPutState state,
3241
                                    const char *desc)
3242
0
{
3243
0
    Error *err = NULL;
3244
0
    int ret = kvm_arch_put_registers(cpu, state, &err);
3245
0
    if (ret) {
3246
0
        if (err) {
3247
0
            error_reportf_err(err, "Restoring resisters %s: ", desc);
3248
0
        } else {
3249
0
            error_report("Failed to put registers %s: %s", desc,
3250
0
                         strerror(-ret));
3251
0
        }
3252
0
        return false;
3253
0
    }
3254
3255
0
    cpu->vcpu_dirty = false;
3256
3257
0
    return true;
3258
0
}
3259
3260
static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
3261
0
{
3262
0
    if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_RESET_STATE, "after reset")) {
3263
0
        cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
3264
0
        vm_stop(RUN_STATE_INTERNAL_ERROR);
3265
0
    }
3266
0
}
3267
3268
void kvm_cpu_synchronize_post_reset(CPUState *cpu)
3269
0
{
3270
0
    run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
3271
3272
0
    if (cpu == first_cpu) {
3273
0
        kvm_reset_parked_vcpus(kvm_state);
3274
0
    }
3275
0
}
3276
3277
static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
3278
0
{
3279
0
    if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_FULL_STATE, "after init")) {
3280
0
        exit(1);
3281
0
    }
3282
0
}
3283
3284
void kvm_cpu_synchronize_post_init(CPUState *cpu)
3285
0
{
3286
0
    if (!kvm_state->guest_state_protected) {
3287
        /*
3288
         * This runs before the machine_init_done notifiers, and is the last
3289
         * opportunity to synchronize the state of confidential guests.
3290
         */
3291
0
        run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
3292
0
    }
3293
0
}
3294
3295
static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg)
3296
0
{
3297
0
    cpu->vcpu_dirty = true;
3298
0
}
3299
3300
void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu)
3301
0
{
3302
0
    run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
3303
0
}
3304
3305
#ifdef KVM_HAVE_MCE_INJECTION
3306
static __thread void *pending_sigbus_addr;
3307
static __thread int pending_sigbus_code;
3308
static __thread bool have_sigbus_pending;
3309
#endif
3310
3311
static void kvm_cpu_kick(CPUState *cpu)
3312
0
{
3313
0
    qatomic_set(&cpu->kvm_run->immediate_exit, 1);
3314
0
}
3315
3316
static void kvm_cpu_kick_self(void)
3317
0
{
3318
0
    if (kvm_immediate_exit) {
3319
0
        kvm_cpu_kick(current_cpu);
3320
0
    } else {
3321
0
        qemu_cpu_kick_self();
3322
0
    }
3323
0
}
3324
3325
static void kvm_eat_signals(CPUState *cpu)
3326
0
{
3327
0
    struct timespec ts = { 0, 0 };
3328
0
    siginfo_t siginfo;
3329
0
    sigset_t waitset;
3330
0
    sigset_t chkset;
3331
0
    int r;
3332
3333
0
    if (kvm_immediate_exit) {
3334
0
        qatomic_set(&cpu->kvm_run->immediate_exit, 0);
3335
0
        return;
3336
0
    }
3337
3338
0
    sigemptyset(&waitset);
3339
0
    sigaddset(&waitset, SIG_IPI);
3340
3341
0
    do {
3342
0
        r = sigtimedwait(&waitset, &siginfo, &ts);
3343
0
        if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
3344
0
            perror("sigtimedwait");
3345
0
            exit(1);
3346
0
        }
3347
3348
0
        r = sigpending(&chkset);
3349
0
        if (r == -1) {
3350
0
            perror("sigpending");
3351
0
            exit(1);
3352
0
        }
3353
0
    } while (sigismember(&chkset, SIG_IPI));
3354
0
}
3355
3356
int kvm_convert_memory(hwaddr start, hwaddr size, bool to_private)
3357
0
{
3358
0
    MemoryRegionSection section;
3359
0
    ram_addr_t offset;
3360
0
    MemoryRegion *mr;
3361
0
    RAMBlock *rb;
3362
0
    void *addr;
3363
0
    int ret = -EINVAL;
3364
3365
0
    trace_kvm_convert_memory(start, size, to_private ? "shared_to_private" : "private_to_shared");
3366
3367
0
    if (!QEMU_PTR_IS_ALIGNED(start, qemu_real_host_page_size()) ||
3368
0
        !QEMU_PTR_IS_ALIGNED(size, qemu_real_host_page_size())) {
3369
0
        return ret;
3370
0
    }
3371
3372
0
    if (!size) {
3373
0
        return ret;
3374
0
    }
3375
3376
0
    section = memory_region_find(get_system_memory(), start, size);
3377
0
    mr = section.mr;
3378
0
    if (!mr) {
3379
        /*
3380
         * Ignore converting non-assigned region to shared.
3381
         *
3382
         * TDX requires vMMIO region to be shared to inject #VE to guest.
3383
         * OVMF issues conservatively MapGPA(shared) on 32bit PCI MMIO region,
3384
         * and vIO-APIC 0xFEC00000 4K page.
3385
         * OVMF assigns 32bit PCI MMIO region to
3386
         * [top of low memory: typically 2GB=0xC000000,  0xFC00000)
3387
         */
3388
0
        if (!to_private) {
3389
0
            return 0;
3390
0
        }
3391
0
        return ret;
3392
0
    }
3393
3394
0
    if (!memory_region_has_guest_memfd_private(mr)) {
3395
        /*
3396
         * Because vMMIO region must be shared, guest TD may convert vMMIO
3397
         * region to shared explicitly.  Don't complain such case.  See
3398
         * memory_region_type() for checking if the region is MMIO region.
3399
         */
3400
0
        if (!to_private &&
3401
0
            !memory_region_is_ram(mr) &&
3402
0
            !memory_region_is_ram_device(mr) &&
3403
0
            !memory_region_is_rom(mr) &&
3404
0
            !memory_region_is_romd(mr)) {
3405
0
            ret = 0;
3406
0
        } else {
3407
0
            error_report("Convert non guest_memfd backed memory region "
3408
0
                        "(0x%"HWADDR_PRIx" ,+ 0x%"HWADDR_PRIx") to %s",
3409
0
                        start, size, to_private ? "private" : "shared");
3410
0
        }
3411
0
        goto out_unref;
3412
0
    }
3413
3414
0
    if (to_private) {
3415
0
        ret = kvm_set_memory_attributes_private(start, size);
3416
0
    } else {
3417
0
        ret = kvm_set_memory_attributes_shared(start, size);
3418
0
    }
3419
0
    if (ret) {
3420
0
        goto out_unref;
3421
0
    }
3422
3423
0
    addr = memory_region_get_ram_ptr(mr) + section.offset_within_region;
3424
0
    rb = qemu_ram_block_from_host(addr, false, &offset);
3425
3426
0
    ret = ram_block_attributes_state_change(rb->attributes,
3427
0
                                            offset, size, to_private);
3428
0
    if (ret) {
3429
0
        error_report("Failed to notify the listener the state change of "
3430
0
                     "(0x%"HWADDR_PRIx" + 0x%"HWADDR_PRIx") to %s",
3431
0
                     start, size, to_private ? "private" : "shared");
3432
0
        goto out_unref;
3433
0
    }
3434
3435
0
    if (to_private) {
3436
0
        if (rb->page_size != qemu_real_host_page_size()) {
3437
            /*
3438
             * shared memory is backed by hugetlb, which is supposed to be
3439
             * pre-allocated and doesn't need to be discarded
3440
             */
3441
0
            goto out_unref;
3442
0
        }
3443
0
        ret = ram_block_discard_shared_range(rb, offset, size);
3444
0
    } else {
3445
0
        ret = ram_block_discard_guest_memfd_range(rb, offset, size);
3446
0
    }
3447
3448
0
out_unref:
3449
0
    memory_region_unref(mr);
3450
0
    return ret;
3451
0
}
3452
3453
int kvm_cpu_exec(CPUState *cpu)
3454
0
{
3455
0
    struct kvm_run *run = cpu->kvm_run;
3456
0
    int ret, run_ret;
3457
3458
0
    trace_kvm_cpu_exec();
3459
3460
0
    if (kvm_arch_process_async_events(cpu)) {
3461
0
        return EXCP_HLT;
3462
0
    }
3463
3464
0
    bql_unlock();
3465
0
    cpu_exec_start(cpu);
3466
3467
0
    do {
3468
0
        MemTxAttrs attrs;
3469
3470
0
        if (cpu->vcpu_dirty) {
3471
0
            if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_RUNTIME_STATE,
3472
0
                                         "at runtime")) {
3473
0
                ret = -1;
3474
0
                break;
3475
0
            }
3476
0
        }
3477
3478
0
        kvm_arch_pre_run(cpu, run);
3479
        /* Corresponding store-release is in cpu_exit. */
3480
0
        if (qatomic_load_acquire(&cpu->exit_request)) {
3481
0
            trace_kvm_interrupt_exit_request();
3482
            /*
3483
             * KVM requires us to reenter the kernel after IO exits to complete
3484
             * instruction emulation. This self-signal will ensure that we
3485
             * leave ASAP again.
3486
             */
3487
0
            kvm_cpu_kick_self();
3488
0
        }
3489
3490
0
        run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
3491
3492
        /*
3493
         * After writing cpu->exit_request, cpu_exit() sends a signal that writes
3494
         * kvm->run->immediate_exit.  The signal is already happening after the
3495
         * write to cpu->exit_request so, if KVM read kvm->run->immediate_exit
3496
         * as true, cpu->exit_request will always read as true.
3497
         */
3498
3499
0
        attrs = kvm_arch_post_run(cpu, run);
3500
3501
0
#ifdef KVM_HAVE_MCE_INJECTION
3502
0
        if (unlikely(have_sigbus_pending)) {
3503
0
            bql_lock();
3504
0
            kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code,
3505
0
                                    pending_sigbus_addr);
3506
0
            have_sigbus_pending = false;
3507
0
            bql_unlock();
3508
0
        }
3509
0
#endif
3510
3511
0
        if (run_ret < 0) {
3512
0
            if (run_ret == -EINTR || run_ret == -EAGAIN) {
3513
0
                trace_kvm_io_window_exit();
3514
0
                kvm_eat_signals(cpu);
3515
0
                ret = EXCP_INTERRUPT;
3516
0
                break;
3517
0
            }
3518
0
            if (!(run_ret == -EFAULT && run->exit_reason == KVM_EXIT_MEMORY_FAULT)) {
3519
0
                fprintf(stderr, "error: kvm run failed %s\n",
3520
0
                        strerror(-run_ret));
3521
#ifdef TARGET_PPC
3522
                if (run_ret == -EBUSY) {
3523
                    fprintf(stderr,
3524
                            "This is probably because your SMT is enabled.\n"
3525
                            "VCPU can only run on primary threads with all "
3526
                            "secondary threads offline.\n");
3527
                }
3528
#endif
3529
0
                ret = -1;
3530
0
                break;
3531
0
            }
3532
0
        }
3533
3534
0
        trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
3535
0
        switch (run->exit_reason) {
3536
0
        case KVM_EXIT_IO:
3537
            /* Called outside BQL */
3538
0
            kvm_handle_io(run->io.port, attrs,
3539
0
                          (uint8_t *)run + run->io.data_offset,
3540
0
                          run->io.direction,
3541
0
                          run->io.size,
3542
0
                          run->io.count);
3543
0
            ret = 0;
3544
0
            break;
3545
0
        case KVM_EXIT_MMIO:
3546
            /* Called outside BQL */
3547
0
            address_space_rw(&address_space_memory,
3548
0
                             run->mmio.phys_addr, attrs,
3549
0
                             run->mmio.data,
3550
0
                             run->mmio.len,
3551
0
                             run->mmio.is_write);
3552
0
            ret = 0;
3553
0
            break;
3554
0
        case KVM_EXIT_IRQ_WINDOW_OPEN:
3555
0
            ret = EXCP_INTERRUPT;
3556
0
            break;
3557
0
        case KVM_EXIT_SHUTDOWN:
3558
0
            qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
3559
0
            ret = EXCP_INTERRUPT;
3560
0
            break;
3561
0
        case KVM_EXIT_UNKNOWN:
3562
0
            fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
3563
0
                    (uint64_t)run->hw.hardware_exit_reason);
3564
0
            ret = -1;
3565
0
            break;
3566
0
        case KVM_EXIT_INTERNAL_ERROR:
3567
0
            ret = kvm_handle_internal_error(cpu, run);
3568
0
            break;
3569
0
        case KVM_EXIT_DIRTY_RING_FULL:
3570
            /*
3571
             * We shouldn't continue if the dirty ring of this vcpu is
3572
             * still full.  Got kicked by KVM_RESET_DIRTY_RINGS.
3573
             */
3574
0
            trace_kvm_dirty_ring_full(cpu->cpu_index);
3575
0
            bql_lock();
3576
            /*
3577
             * We throttle vCPU by making it sleep once it exit from kernel
3578
             * due to dirty ring full. In the dirtylimit scenario, reaping
3579
             * all vCPUs after a single vCPU dirty ring get full result in
3580
             * the miss of sleep, so just reap the ring-fulled vCPU.
3581
             */
3582
0
            if (dirtylimit_in_service()) {
3583
0
                kvm_dirty_ring_reap(kvm_state, cpu);
3584
0
            } else {
3585
0
                kvm_dirty_ring_reap(kvm_state, NULL);
3586
0
            }
3587
0
            bql_unlock();
3588
0
            dirtylimit_vcpu_execute(cpu);
3589
0
            ret = 0;
3590
0
            break;
3591
0
        case KVM_EXIT_SYSTEM_EVENT:
3592
0
            trace_kvm_run_exit_system_event(cpu->cpu_index, run->system_event.type);
3593
0
            switch (run->system_event.type) {
3594
0
            case KVM_SYSTEM_EVENT_SHUTDOWN:
3595
0
                qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
3596
0
                ret = EXCP_INTERRUPT;
3597
0
                break;
3598
0
            case KVM_SYSTEM_EVENT_RESET:
3599
0
                qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
3600
0
                ret = EXCP_INTERRUPT;
3601
0
                break;
3602
0
            case KVM_SYSTEM_EVENT_SEV_TERM:
3603
0
            case KVM_SYSTEM_EVENT_CRASH:
3604
0
                kvm_cpu_synchronize_state(cpu);
3605
0
                bql_lock();
3606
0
                qemu_system_guest_panicked(cpu_get_crash_info(cpu));
3607
0
                bql_unlock();
3608
0
                ret = 0;
3609
0
                break;
3610
0
            default:
3611
0
                ret = kvm_arch_handle_exit(cpu, run);
3612
0
                break;
3613
0
            }
3614
0
            break;
3615
0
        case KVM_EXIT_MEMORY_FAULT:
3616
0
            trace_kvm_memory_fault(run->memory_fault.gpa,
3617
0
                                   run->memory_fault.size,
3618
0
                                   run->memory_fault.flags);
3619
0
            if (run->memory_fault.flags & ~KVM_MEMORY_EXIT_FLAG_PRIVATE) {
3620
0
                error_report("KVM_EXIT_MEMORY_FAULT: Unknown flag 0x%" PRIx64,
3621
0
                             (uint64_t)run->memory_fault.flags);
3622
0
                ret = -1;
3623
0
                break;
3624
0
            }
3625
0
            ret = kvm_convert_memory(run->memory_fault.gpa, run->memory_fault.size,
3626
0
                                     run->memory_fault.flags & KVM_MEMORY_EXIT_FLAG_PRIVATE);
3627
0
            break;
3628
0
        default:
3629
0
            ret = kvm_arch_handle_exit(cpu, run);
3630
0
            break;
3631
0
        }
3632
0
    } while (ret == 0);
3633
3634
0
    cpu_exec_end(cpu);
3635
0
    bql_lock();
3636
3637
0
    if (ret < 0) {
3638
0
        cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
3639
0
        vm_stop(RUN_STATE_INTERNAL_ERROR);
3640
0
    }
3641
3642
0
    return ret;
3643
0
}
3644
3645
int kvm_ioctl(KVMState *s, unsigned long type, ...)
3646
0
{
3647
0
    int ret;
3648
0
    void *arg;
3649
0
    va_list ap;
3650
3651
0
    va_start(ap, type);
3652
0
    arg = va_arg(ap, void *);
3653
0
    va_end(ap);
3654
3655
0
    trace_kvm_ioctl(type, arg);
3656
0
    ret = ioctl(s->fd, type, arg);
3657
0
    if (ret == -1) {
3658
0
        ret = -errno;
3659
0
    }
3660
0
    return ret;
3661
0
}
3662
3663
int kvm_vm_ioctl(KVMState *s, unsigned long type, ...)
3664
0
{
3665
0
    int ret;
3666
0
    void *arg;
3667
0
    va_list ap;
3668
3669
0
    va_start(ap, type);
3670
0
    arg = va_arg(ap, void *);
3671
0
    va_end(ap);
3672
3673
0
    trace_kvm_vm_ioctl(type, arg);
3674
0
    accel_ioctl_begin();
3675
0
    ret = ioctl(s->vmfd, type, arg);
3676
0
    if (ret == -1) {
3677
0
        ret = -errno;
3678
0
    }
3679
0
    accel_ioctl_end();
3680
0
    return ret;
3681
0
}
3682
3683
int kvm_vcpu_ioctl(CPUState *cpu, unsigned long type, ...)
3684
0
{
3685
0
    int ret;
3686
0
    void *arg;
3687
0
    va_list ap;
3688
3689
0
    va_start(ap, type);
3690
0
    arg = va_arg(ap, void *);
3691
0
    va_end(ap);
3692
3693
0
    trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
3694
0
    accel_cpu_ioctl_begin(cpu);
3695
0
    ret = ioctl(cpu->kvm_fd, type, arg);
3696
0
    accel_cpu_ioctl_end(cpu);
3697
0
    if (ret == -1) {
3698
0
        ret = -errno;
3699
0
    }
3700
0
    return ret;
3701
0
}
3702
3703
int kvm_device_ioctl(int fd, unsigned long type, ...)
3704
0
{
3705
0
    int ret;
3706
0
    void *arg;
3707
0
    va_list ap;
3708
3709
0
    va_start(ap, type);
3710
0
    arg = va_arg(ap, void *);
3711
0
    va_end(ap);
3712
3713
0
    trace_kvm_device_ioctl(fd, type, arg);
3714
0
    accel_ioctl_begin();
3715
0
    ret = ioctl(fd, type, arg);
3716
0
    if (ret == -1) {
3717
0
        ret = -errno;
3718
0
    }
3719
0
    accel_ioctl_end();
3720
0
    return ret;
3721
0
}
3722
3723
int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
3724
0
{
3725
0
    int ret;
3726
0
    struct kvm_device_attr attribute = {
3727
0
        .group = group,
3728
0
        .attr = attr,
3729
0
    };
3730
3731
0
    if (!kvm_vm_attributes_allowed) {
3732
0
        return 0;
3733
0
    }
3734
3735
0
    ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
3736
    /* kvm returns 0 on success for HAS_DEVICE_ATTR */
3737
0
    return ret ? 0 : 1;
3738
0
}
3739
3740
int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
3741
0
{
3742
0
    struct kvm_device_attr attribute = {
3743
0
        .group = group,
3744
0
        .attr = attr,
3745
0
        .flags = 0,
3746
0
    };
3747
3748
0
    return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1;
3749
0
}
3750
3751
int kvm_device_access(int fd, int group, uint64_t attr,
3752
                      void *val, bool write, Error **errp)
3753
0
{
3754
0
    struct kvm_device_attr kvmattr;
3755
0
    int err;
3756
3757
0
    kvmattr.flags = 0;
3758
0
    kvmattr.group = group;
3759
0
    kvmattr.attr = attr;
3760
0
    kvmattr.addr = (uintptr_t)val;
3761
3762
0
    err = kvm_device_ioctl(fd,
3763
0
                           write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
3764
0
                           &kvmattr);
3765
0
    if (err < 0) {
3766
0
        error_setg_errno(errp, -err,
3767
0
                         "KVM_%s_DEVICE_ATTR failed: Group %d "
3768
0
                         "attr 0x%016" PRIx64,
3769
0
                         write ? "SET" : "GET", group, attr);
3770
0
    }
3771
0
    return err;
3772
0
}
3773
3774
bool kvm_has_sync_mmu(void)
3775
0
{
3776
0
    return kvm_state->sync_mmu;
3777
0
}
3778
3779
int kvm_has_vcpu_events(void)
3780
0
{
3781
0
    return kvm_state->vcpu_events;
3782
0
}
3783
3784
int kvm_max_nested_state_length(void)
3785
0
{
3786
0
    return kvm_state->max_nested_state_len;
3787
0
}
3788
3789
int kvm_has_gsi_routing(void)
3790
0
{
3791
0
#ifdef KVM_CAP_IRQ_ROUTING
3792
0
    return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
3793
#else
3794
    return false;
3795
#endif
3796
0
}
3797
3798
bool kvm_arm_supports_user_irq(void)
3799
0
{
3800
0
    return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ);
3801
0
}
3802
3803
#ifdef TARGET_KVM_HAVE_GUEST_DEBUG
3804
struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu, vaddr pc)
3805
0
{
3806
0
    struct kvm_sw_breakpoint *bp;
3807
3808
0
    QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
3809
0
        if (bp->pc == pc) {
3810
0
            return bp;
3811
0
        }
3812
0
    }
3813
0
    return NULL;
3814
0
}
3815
3816
int kvm_sw_breakpoints_active(CPUState *cpu)
3817
0
{
3818
0
    return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
3819
0
}
3820
3821
struct kvm_set_guest_debug_data {
3822
    struct kvm_guest_debug dbg;
3823
    int err;
3824
};
3825
3826
static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data)
3827
0
{
3828
0
    struct kvm_set_guest_debug_data *dbg_data =
3829
0
        (struct kvm_set_guest_debug_data *) data.host_ptr;
3830
3831
0
    dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG,
3832
0
                                   &dbg_data->dbg);
3833
0
}
3834
3835
int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
3836
0
{
3837
0
    struct kvm_set_guest_debug_data data;
3838
3839
0
    data.dbg.control = reinject_trap;
3840
3841
0
    if (cpu_single_stepping(cpu)) {
3842
0
        data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
3843
3844
0
        if (cpu->singlestep_flags & SSTEP_NOIRQ) {
3845
0
            data.dbg.control |= KVM_GUESTDBG_BLOCKIRQ;
3846
0
        }
3847
0
    }
3848
0
    kvm_arch_update_guest_debug(cpu, &data.dbg);
3849
3850
0
    run_on_cpu(cpu, kvm_invoke_set_guest_debug,
3851
0
               RUN_ON_CPU_HOST_PTR(&data));
3852
0
    return data.err;
3853
0
}
3854
3855
int kvm_insert_gdbstub_breakpoint(CPUState *cpu, GdbBreakpointType type,
3856
                                  vaddr addr, vaddr len)
3857
0
{
3858
0
    struct kvm_sw_breakpoint *bp;
3859
0
    int err;
3860
3861
0
    if (type == GDB_BREAKPOINT_SW) {
3862
0
        bp = kvm_find_sw_breakpoint(cpu, addr);
3863
0
        if (bp) {
3864
0
            bp->use_count++;
3865
0
            return 0;
3866
0
        }
3867
3868
0
        bp = g_new(struct kvm_sw_breakpoint, 1);
3869
0
        bp->pc = addr;
3870
0
        bp->use_count = 1;
3871
0
        err = kvm_arch_insert_sw_breakpoint(cpu, bp);
3872
0
        if (err) {
3873
0
            g_free(bp);
3874
0
            return err;
3875
0
        }
3876
3877
0
        QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
3878
0
    } else {
3879
0
        err = kvm_arch_insert_gdbstub_hw_breakpoint(addr, len, type);
3880
0
        if (err) {
3881
0
            return err;
3882
0
        }
3883
0
    }
3884
3885
0
    CPU_FOREACH(cpu) {
3886
0
        err = kvm_update_guest_debug(cpu, 0);
3887
0
        if (err) {
3888
0
            return err;
3889
0
        }
3890
0
    }
3891
0
    return 0;
3892
0
}
3893
3894
int kvm_remove_gdbstub_breakpoint(CPUState *cpu, GdbBreakpointType type,
3895
                                  vaddr addr, vaddr len)
3896
0
{
3897
0
    struct kvm_sw_breakpoint *bp;
3898
0
    int err;
3899
3900
0
    if (type == GDB_BREAKPOINT_SW) {
3901
0
        bp = kvm_find_sw_breakpoint(cpu, addr);
3902
0
        if (!bp) {
3903
0
            return -ENOENT;
3904
0
        }
3905
3906
0
        if (bp->use_count > 1) {
3907
0
            bp->use_count--;
3908
0
            return 0;
3909
0
        }
3910
3911
0
        err = kvm_arch_remove_sw_breakpoint(cpu, bp);
3912
0
        if (err) {
3913
0
            return err;
3914
0
        }
3915
3916
0
        QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
3917
0
        g_free(bp);
3918
0
    } else {
3919
0
        err = kvm_arch_remove_gdbstub_hw_breakpoint(addr, len, type);
3920
0
        if (err) {
3921
0
            return err;
3922
0
        }
3923
0
    }
3924
3925
0
    CPU_FOREACH(cpu) {
3926
0
        err = kvm_update_guest_debug(cpu, 0);
3927
0
        if (err) {
3928
0
            return err;
3929
0
        }
3930
0
    }
3931
0
    return 0;
3932
0
}
3933
3934
void kvm_remove_all_gdbstub_breakpoints(CPUState *cpu)
3935
0
{
3936
0
    struct kvm_sw_breakpoint *bp, *next;
3937
0
    KVMState *s = cpu->kvm_state;
3938
0
    CPUState *tmpcpu;
3939
3940
0
    QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
3941
0
        if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) {
3942
            /* Try harder to find a CPU that currently sees the breakpoint. */
3943
0
            CPU_FOREACH(tmpcpu) {
3944
0
                if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) {
3945
0
                    break;
3946
0
                }
3947
0
            }
3948
0
        }
3949
0
        QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
3950
0
        g_free(bp);
3951
0
    }
3952
0
    kvm_arch_remove_all_gdbstub_hw_breakpoints();
3953
3954
0
    CPU_FOREACH(cpu) {
3955
0
        kvm_update_guest_debug(cpu, 0);
3956
0
    }
3957
0
}
3958
3959
#endif /* !TARGET_KVM_HAVE_GUEST_DEBUG */
3960
3961
static int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset)
3962
0
{
3963
0
    KVMState *s = kvm_state;
3964
0
    struct kvm_signal_mask *sigmask;
3965
0
    int r;
3966
3967
0
    sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
3968
3969
0
    sigmask->len = s->sigmask_len;
3970
0
    memcpy(sigmask->sigset, sigset, sizeof(*sigset));
3971
0
    r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
3972
0
    g_free(sigmask);
3973
3974
0
    return r;
3975
0
}
3976
3977
static void kvm_ipi_signal(int sig)
3978
0
{
3979
0
    if (current_cpu) {
3980
0
        assert(kvm_immediate_exit);
3981
0
        kvm_cpu_kick(current_cpu);
3982
0
    }
3983
0
}
3984
3985
void kvm_init_cpu_signals(CPUState *cpu)
3986
0
{
3987
0
    int r;
3988
0
    sigset_t set;
3989
0
    struct sigaction sigact;
3990
3991
0
    memset(&sigact, 0, sizeof(sigact));
3992
0
    sigact.sa_handler = kvm_ipi_signal;
3993
0
    sigaction(SIG_IPI, &sigact, NULL);
3994
3995
0
    pthread_sigmask(SIG_BLOCK, NULL, &set);
3996
0
#if defined KVM_HAVE_MCE_INJECTION
3997
0
    sigdelset(&set, SIGBUS);
3998
0
    pthread_sigmask(SIG_SETMASK, &set, NULL);
3999
0
#endif
4000
0
    sigdelset(&set, SIG_IPI);
4001
0
    if (kvm_immediate_exit) {
4002
0
        r = pthread_sigmask(SIG_SETMASK, &set, NULL);
4003
0
    } else {
4004
0
        r = kvm_set_signal_mask(cpu, &set);
4005
0
    }
4006
0
    if (r) {
4007
0
        fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
4008
0
        exit(1);
4009
0
    }
4010
0
}
4011
4012
/* Called asynchronously in VCPU thread.  */
4013
int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
4014
0
{
4015
0
#ifdef KVM_HAVE_MCE_INJECTION
4016
0
    if (have_sigbus_pending) {
4017
0
        return 1;
4018
0
    }
4019
0
    have_sigbus_pending = true;
4020
0
    pending_sigbus_addr = addr;
4021
0
    pending_sigbus_code = code;
4022
0
    qatomic_set(&cpu->exit_request, true);
4023
0
    return 0;
4024
#else
4025
    return 1;
4026
#endif
4027
0
}
4028
4029
/* Called synchronously (via signalfd) in main thread.  */
4030
int kvm_on_sigbus(int code, void *addr)
4031
0
{
4032
0
#ifdef KVM_HAVE_MCE_INJECTION
4033
    /* Action required MCE kills the process if SIGBUS is blocked.  Because
4034
     * that's what happens in the I/O thread, where we handle MCE via signalfd,
4035
     * we can only get action optional here.
4036
     */
4037
0
    assert(code != BUS_MCEERR_AR);
4038
0
    kvm_arch_on_sigbus_vcpu(first_cpu, code, addr);
4039
0
    return 0;
4040
#else
4041
    return 1;
4042
#endif
4043
0
}
4044
4045
int kvm_create_device(KVMState *s, uint64_t type, bool test)
4046
0
{
4047
0
    int ret;
4048
0
    struct kvm_create_device create_dev;
4049
4050
0
    create_dev.type = type;
4051
0
    create_dev.fd = -1;
4052
0
    create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
4053
4054
0
    if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) {
4055
0
        return -ENOTSUP;
4056
0
    }
4057
4058
0
    ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev);
4059
0
    if (ret) {
4060
0
        return ret;
4061
0
    }
4062
4063
0
    return test ? 0 : create_dev.fd;
4064
0
}
4065
4066
bool kvm_device_supported(int vmfd, uint64_t type)
4067
0
{
4068
0
    struct kvm_create_device create_dev = {
4069
0
        .type = type,
4070
0
        .fd = -1,
4071
0
        .flags = KVM_CREATE_DEVICE_TEST,
4072
0
    };
4073
4074
0
    if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) {
4075
0
        return false;
4076
0
    }
4077
4078
0
    return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0);
4079
0
}
4080
4081
int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
4082
0
{
4083
0
    struct kvm_one_reg reg;
4084
0
    int r;
4085
4086
0
    reg.id = id;
4087
0
    reg.addr = (uintptr_t) source;
4088
0
    r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
4089
0
    if (r) {
4090
0
        trace_kvm_failed_reg_set(id, strerror(-r));
4091
0
    }
4092
0
    return r;
4093
0
}
4094
4095
int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
4096
0
{
4097
0
    struct kvm_one_reg reg;
4098
0
    int r;
4099
4100
0
    reg.id = id;
4101
0
    reg.addr = (uintptr_t) target;
4102
0
    r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
4103
0
    if (r) {
4104
0
        trace_kvm_failed_reg_get(id, strerror(-r));
4105
0
    }
4106
0
    return r;
4107
0
}
4108
4109
static bool kvm_accel_has_memory(AccelState *accel, AddressSpace *as,
4110
                                 hwaddr start_addr, hwaddr size)
4111
0
{
4112
0
    KVMState *kvm = KVM_STATE(accel);
4113
0
    int i;
4114
4115
0
    for (i = 0; i < kvm->nr_as; ++i) {
4116
0
        if (kvm->as[i].as == as && kvm->as[i].ml) {
4117
0
            size = MIN(kvm_max_slot_size, size);
4118
0
            return NULL != kvm_lookup_matching_slot(kvm->as[i].ml,
4119
0
                                                    start_addr, size);
4120
0
        }
4121
0
    }
4122
4123
0
    return false;
4124
0
}
4125
4126
static void kvm_get_kvm_shadow_mem(Object *obj, Visitor *v,
4127
                                   const char *name, void *opaque,
4128
                                   Error **errp)
4129
0
{
4130
0
    KVMState *s = KVM_STATE(obj);
4131
0
    int64_t value = s->kvm_shadow_mem;
4132
4133
0
    visit_type_int(v, name, &value, errp);
4134
0
}
4135
4136
static void kvm_set_kvm_shadow_mem(Object *obj, Visitor *v,
4137
                                   const char *name, void *opaque,
4138
                                   Error **errp)
4139
0
{
4140
0
    KVMState *s = KVM_STATE(obj);
4141
0
    int64_t value;
4142
4143
0
    if (s->fd != -1) {
4144
0
        error_setg(errp, "Cannot set properties after the accelerator has been initialized");
4145
0
        return;
4146
0
    }
4147
4148
0
    if (!visit_type_int(v, name, &value, errp)) {
4149
0
        return;
4150
0
    }
4151
4152
0
    s->kvm_shadow_mem = value;
4153
0
}
4154
4155
static void kvm_set_kernel_irqchip(Object *obj, int value,
4156
                                   Error **errp)
4157
0
{
4158
0
    KVMState *s = KVM_STATE(obj);
4159
4160
0
    if (s->fd != -1) {
4161
0
        error_setg(errp, "Cannot set properties after the accelerator has been initialized");
4162
0
        return;
4163
0
    }
4164
4165
0
    switch (value) {
4166
0
    case ON_OFF_SPLIT_ON:
4167
0
        s->kernel_irqchip_allowed = true;
4168
0
        s->kernel_irqchip_required = true;
4169
0
        s->kernel_irqchip_split = ON_OFF_AUTO_OFF;
4170
0
        break;
4171
0
    case ON_OFF_SPLIT_OFF:
4172
0
        s->kernel_irqchip_allowed = false;
4173
0
        s->kernel_irqchip_required = false;
4174
0
        s->kernel_irqchip_split = ON_OFF_AUTO_OFF;
4175
0
        break;
4176
0
    case ON_OFF_SPLIT_SPLIT:
4177
0
        s->kernel_irqchip_allowed = true;
4178
0
        s->kernel_irqchip_required = true;
4179
0
        s->kernel_irqchip_split = ON_OFF_AUTO_ON;
4180
0
        break;
4181
0
    default:
4182
0
        g_assert_not_reached();
4183
0
    }
4184
0
}
4185
4186
bool kvm_kernel_irqchip_allowed(void)
4187
0
{
4188
0
    return kvm_state->kernel_irqchip_allowed;
4189
0
}
4190
4191
bool kvm_kernel_irqchip_required(void)
4192
0
{
4193
0
    return kvm_state->kernel_irqchip_required;
4194
0
}
4195
4196
bool kvm_kernel_irqchip_split(void)
4197
0
{
4198
0
    return kvm_state->kernel_irqchip_split == ON_OFF_AUTO_ON;
4199
0
}
4200
4201
static void kvm_get_dirty_ring_size(Object *obj, Visitor *v,
4202
                                    const char *name, void *opaque,
4203
                                    Error **errp)
4204
0
{
4205
0
    KVMState *s = KVM_STATE(obj);
4206
0
    uint32_t value = s->kvm_dirty_ring_size;
4207
4208
0
    visit_type_uint32(v, name, &value, errp);
4209
0
}
4210
4211
static void kvm_set_dirty_ring_size(Object *obj, Visitor *v,
4212
                                    const char *name, void *opaque,
4213
                                    Error **errp)
4214
0
{
4215
0
    KVMState *s = KVM_STATE(obj);
4216
0
    uint32_t value;
4217
4218
0
    if (s->fd != -1) {
4219
0
        error_setg(errp, "Cannot set properties after the accelerator has been initialized");
4220
0
        return;
4221
0
    }
4222
4223
0
    if (!visit_type_uint32(v, name, &value, errp)) {
4224
0
        return;
4225
0
    }
4226
0
    if (value & (value - 1)) {
4227
0
        error_setg(errp, "dirty-ring-size must be a power of two.");
4228
0
        return;
4229
0
    }
4230
4231
0
    s->kvm_dirty_ring_size = value;
4232
0
}
4233
4234
static char *kvm_get_device(Object *obj,
4235
                            Error **errp G_GNUC_UNUSED)
4236
0
{
4237
0
    KVMState *s = KVM_STATE(obj);
4238
4239
0
    return g_strdup(s->device);
4240
0
}
4241
4242
static void kvm_set_device(Object *obj,
4243
                           const char *value,
4244
                           Error **errp G_GNUC_UNUSED)
4245
0
{
4246
0
    KVMState *s = KVM_STATE(obj);
4247
4248
0
    g_free(s->device);
4249
0
    s->device = g_strdup(value);
4250
0
}
4251
4252
static void kvm_set_kvm_rapl(Object *obj, bool value, Error **errp)
4253
0
{
4254
0
    KVMState *s = KVM_STATE(obj);
4255
0
    s->msr_energy.enable = value;
4256
0
}
4257
4258
static void kvm_set_kvm_rapl_socket_path(Object *obj,
4259
                                         const char *str,
4260
                                         Error **errp)
4261
0
{
4262
0
    KVMState *s = KVM_STATE(obj);
4263
0
    g_free(s->msr_energy.socket_path);
4264
0
    s->msr_energy.socket_path = g_strdup(str);
4265
0
}
4266
4267
static void kvm_accel_instance_init(Object *obj)
4268
0
{
4269
0
    KVMState *s = KVM_STATE(obj);
4270
4271
0
    s->fd = -1;
4272
0
    s->vmfd = -1;
4273
0
    s->kvm_shadow_mem = -1;
4274
0
    s->kernel_irqchip_allowed = true;
4275
0
    s->kernel_irqchip_split = ON_OFF_AUTO_AUTO;
4276
    /* KVM dirty ring is by default off */
4277
0
    s->kvm_dirty_ring_size = 0;
4278
0
    s->kvm_dirty_ring_with_bitmap = false;
4279
0
    s->kvm_eager_split_size = 0;
4280
0
    s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN;
4281
0
    s->notify_window = 0;
4282
0
    s->xen_version = 0;
4283
0
    s->xen_gnttab_max_frames = 64;
4284
0
    s->xen_evtchn_max_pirq = 256;
4285
0
    s->device = NULL;
4286
0
    s->msr_energy.enable = false;
4287
0
    s->honor_guest_pat = ON_OFF_AUTO_OFF;
4288
0
}
4289
4290
static void kvm_accel_class_init(ObjectClass *oc, const void *data)
4291
63
{
4292
63
    AccelClass *ac = ACCEL_CLASS(oc);
4293
63
    ac->name = "KVM";
4294
63
    ac->init_machine = kvm_init;
4295
63
    ac->rebuild_guest = kvm_reset_vmfd;
4296
63
    ac->has_memory = kvm_accel_has_memory;
4297
63
    ac->allowed = &kvm_allowed;
4298
4299
63
    object_class_property_add_enum(oc, "kernel-irqchip", "on|off|split",
4300
63
                                   &OnOffSplit_type_info, NULL,
4301
63
                                   kvm_set_kernel_irqchip);
4302
63
    object_class_property_set_description(oc, "kernel-irqchip",
4303
63
        "Configure KVM in-kernel irqchip");
4304
4305
63
    object_class_property_add(oc, "kvm-shadow-mem", "int", &int_type_info,
4306
63
                              kvm_get_kvm_shadow_mem, kvm_set_kvm_shadow_mem,
4307
63
                              NULL, NULL);
4308
63
    object_class_property_set_description(oc, "kvm-shadow-mem",
4309
63
        "KVM shadow MMU size");
4310
4311
63
    object_class_property_add(oc, "dirty-ring-size", "uint32",
4312
63
                              &uint32_type_info,
4313
63
                              kvm_get_dirty_ring_size, kvm_set_dirty_ring_size,
4314
63
                              NULL, NULL);
4315
63
    object_class_property_set_description(oc, "dirty-ring-size",
4316
63
        "Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)");
4317
4318
63
    object_class_property_add_str(oc, "device", kvm_get_device, kvm_set_device);
4319
63
    object_class_property_set_description(oc, "device",
4320
63
        "Path to the device node to use (default: /dev/kvm)");
4321
4322
63
    object_class_property_add_bool(oc, "rapl",
4323
63
                                   NULL,
4324
63
                                   kvm_set_kvm_rapl);
4325
63
    object_class_property_set_description(oc, "rapl",
4326
63
        "Allow energy related MSRs for RAPL interface in Guest");
4327
4328
63
    object_class_property_add_str(oc, "rapl-helper-socket", NULL,
4329
63
                                  kvm_set_kvm_rapl_socket_path);
4330
63
    object_class_property_set_description(oc, "rapl-helper-socket",
4331
63
        "Socket Path for comminucating with the Virtual MSR helper daemon");
4332
4333
63
    kvm_arch_accel_class_init(oc);
4334
63
}
4335
4336
static void kvm_accel_finalize(Object *obj)
4337
0
{
4338
0
    KVMState *s = KVM_STATE(obj);
4339
4340
0
    g_free(s->device);
4341
0
}
4342
4343
static const TypeInfo kvm_accel_type = {
4344
    .name = TYPE_KVM_ACCEL,
4345
    .parent = TYPE_ACCEL,
4346
    .instance_init = kvm_accel_instance_init,
4347
    .instance_finalize = kvm_accel_finalize,
4348
    .class_init = kvm_accel_class_init,
4349
    .instance_size = sizeof(KVMState),
4350
    .secure = true,
4351
};
4352
4353
static void kvm_type_init(void)
4354
63
{
4355
63
    type_register_static(&kvm_accel_type);
4356
63
}
4357
4358
type_init(kvm_type_init);
4359
4360
typedef struct StatsArgs {
4361
    union StatsResultsType {
4362
        StatsResultList **stats;
4363
        StatsSchemaList **schema;
4364
    } result;
4365
    strList *names;
4366
    Error **errp;
4367
} StatsArgs;
4368
4369
static StatsList *add_kvmstat_entry(struct kvm_stats_desc *pdesc,
4370
                                    uint64_t *stats_data,
4371
                                    StatsList *stats_list,
4372
                                    Error **errp)
4373
0
{
4374
4375
0
    Stats *stats;
4376
0
    uint64List *val_list = NULL;
4377
4378
    /* Only add stats that we understand.  */
4379
0
    switch (pdesc->flags & KVM_STATS_TYPE_MASK) {
4380
0
    case KVM_STATS_TYPE_CUMULATIVE:
4381
0
    case KVM_STATS_TYPE_INSTANT:
4382
0
    case KVM_STATS_TYPE_PEAK:
4383
0
    case KVM_STATS_TYPE_LINEAR_HIST:
4384
0
    case KVM_STATS_TYPE_LOG_HIST:
4385
0
        break;
4386
0
    default:
4387
0
        return stats_list;
4388
0
    }
4389
4390
0
    switch (pdesc->flags & KVM_STATS_UNIT_MASK) {
4391
0
    case KVM_STATS_UNIT_NONE:
4392
0
    case KVM_STATS_UNIT_BYTES:
4393
0
    case KVM_STATS_UNIT_CYCLES:
4394
0
    case KVM_STATS_UNIT_SECONDS:
4395
0
    case KVM_STATS_UNIT_BOOLEAN:
4396
0
        break;
4397
0
    default:
4398
0
        return stats_list;
4399
0
    }
4400
4401
0
    switch (pdesc->flags & KVM_STATS_BASE_MASK) {
4402
0
    case KVM_STATS_BASE_POW10:
4403
0
    case KVM_STATS_BASE_POW2:
4404
0
        break;
4405
0
    default:
4406
0
        return stats_list;
4407
0
    }
4408
4409
    /* Alloc and populate data list */
4410
0
    stats = g_new0(Stats, 1);
4411
0
    stats->name = g_strdup(pdesc->name);
4412
0
    stats->value = g_new0(StatsValue, 1);
4413
4414
0
    if ((pdesc->flags & KVM_STATS_UNIT_MASK) == KVM_STATS_UNIT_BOOLEAN) {
4415
0
        stats->value->u.boolean = *stats_data;
4416
0
        stats->value->type = QTYPE_QBOOL;
4417
0
    } else if (pdesc->size == 1) {
4418
0
        stats->value->u.scalar = *stats_data;
4419
0
        stats->value->type = QTYPE_QNUM;
4420
0
    } else {
4421
0
        int i;
4422
0
        for (i = 0; i < pdesc->size; i++) {
4423
0
            QAPI_LIST_PREPEND(val_list, stats_data[i]);
4424
0
        }
4425
0
        stats->value->u.list = val_list;
4426
0
        stats->value->type = QTYPE_QLIST;
4427
0
    }
4428
4429
0
    QAPI_LIST_PREPEND(stats_list, stats);
4430
0
    return stats_list;
4431
0
}
4432
4433
static StatsSchemaValueList *add_kvmschema_entry(struct kvm_stats_desc *pdesc,
4434
                                                 StatsSchemaValueList *list,
4435
                                                 Error **errp)
4436
0
{
4437
0
    StatsSchemaValueList *schema_entry = g_new0(StatsSchemaValueList, 1);
4438
0
    schema_entry->value = g_new0(StatsSchemaValue, 1);
4439
4440
0
    switch (pdesc->flags & KVM_STATS_TYPE_MASK) {
4441
0
    case KVM_STATS_TYPE_CUMULATIVE:
4442
0
        schema_entry->value->type = STATS_TYPE_CUMULATIVE;
4443
0
        break;
4444
0
    case KVM_STATS_TYPE_INSTANT:
4445
0
        schema_entry->value->type = STATS_TYPE_INSTANT;
4446
0
        break;
4447
0
    case KVM_STATS_TYPE_PEAK:
4448
0
        schema_entry->value->type = STATS_TYPE_PEAK;
4449
0
        break;
4450
0
    case KVM_STATS_TYPE_LINEAR_HIST:
4451
0
        schema_entry->value->type = STATS_TYPE_LINEAR_HISTOGRAM;
4452
0
        schema_entry->value->bucket_size = pdesc->bucket_size;
4453
0
        schema_entry->value->has_bucket_size = true;
4454
0
        break;
4455
0
    case KVM_STATS_TYPE_LOG_HIST:
4456
0
        schema_entry->value->type = STATS_TYPE_LOG2_HISTOGRAM;
4457
0
        break;
4458
0
    default:
4459
0
        goto exit;
4460
0
    }
4461
4462
0
    switch (pdesc->flags & KVM_STATS_UNIT_MASK) {
4463
0
    case KVM_STATS_UNIT_NONE:
4464
0
        break;
4465
0
    case KVM_STATS_UNIT_BOOLEAN:
4466
0
        schema_entry->value->has_unit = true;
4467
0
        schema_entry->value->unit = STATS_UNIT_BOOLEAN;
4468
0
        break;
4469
0
    case KVM_STATS_UNIT_BYTES:
4470
0
        schema_entry->value->has_unit = true;
4471
0
        schema_entry->value->unit = STATS_UNIT_BYTES;
4472
0
        break;
4473
0
    case KVM_STATS_UNIT_CYCLES:
4474
0
        schema_entry->value->has_unit = true;
4475
0
        schema_entry->value->unit = STATS_UNIT_CYCLES;
4476
0
        break;
4477
0
    case KVM_STATS_UNIT_SECONDS:
4478
0
        schema_entry->value->has_unit = true;
4479
0
        schema_entry->value->unit = STATS_UNIT_SECONDS;
4480
0
        break;
4481
0
    default:
4482
0
        goto exit;
4483
0
    }
4484
4485
0
    schema_entry->value->exponent = pdesc->exponent;
4486
0
    if (pdesc->exponent) {
4487
0
        switch (pdesc->flags & KVM_STATS_BASE_MASK) {
4488
0
        case KVM_STATS_BASE_POW10:
4489
0
            schema_entry->value->has_base = true;
4490
0
            schema_entry->value->base = 10;
4491
0
            break;
4492
0
        case KVM_STATS_BASE_POW2:
4493
0
            schema_entry->value->has_base = true;
4494
0
            schema_entry->value->base = 2;
4495
0
            break;
4496
0
        default:
4497
0
            goto exit;
4498
0
        }
4499
0
    }
4500
4501
0
    schema_entry->value->name = g_strdup(pdesc->name);
4502
0
    schema_entry->next = list;
4503
0
    return schema_entry;
4504
0
exit:
4505
0
    g_free(schema_entry->value);
4506
0
    g_free(schema_entry);
4507
0
    return list;
4508
0
}
4509
4510
/* Cached stats descriptors */
4511
typedef struct StatsDescriptors {
4512
    const char *ident; /* cache key, currently the StatsTarget */
4513
    struct kvm_stats_desc *kvm_stats_desc;
4514
    struct kvm_stats_header kvm_stats_header;
4515
    QTAILQ_ENTRY(StatsDescriptors) next;
4516
} StatsDescriptors;
4517
4518
static QTAILQ_HEAD(, StatsDescriptors) stats_descriptors =
4519
    QTAILQ_HEAD_INITIALIZER(stats_descriptors);
4520
4521
/*
4522
 * Return the descriptors for 'target', that either have already been read
4523
 * or are retrieved from 'stats_fd'.
4524
 */
4525
static StatsDescriptors *find_stats_descriptors(StatsTarget target, int stats_fd,
4526
                                                Error **errp)
4527
0
{
4528
0
    StatsDescriptors *descriptors;
4529
0
    const char *ident;
4530
0
    struct kvm_stats_desc *kvm_stats_desc;
4531
0
    struct kvm_stats_header *kvm_stats_header;
4532
0
    size_t size_desc;
4533
0
    ssize_t ret;
4534
4535
0
    ident = StatsTarget_str(target);
4536
0
    QTAILQ_FOREACH(descriptors, &stats_descriptors, next) {
4537
0
        if (g_str_equal(descriptors->ident, ident)) {
4538
0
            return descriptors;
4539
0
        }
4540
0
    }
4541
4542
0
    descriptors = g_new0(StatsDescriptors, 1);
4543
4544
    /* Read stats header */
4545
0
    kvm_stats_header = &descriptors->kvm_stats_header;
4546
0
    ret = pread(stats_fd, kvm_stats_header, sizeof(*kvm_stats_header), 0);
4547
0
    if (ret != sizeof(*kvm_stats_header)) {
4548
0
        error_setg(errp, "KVM stats: failed to read stats header: "
4549
0
                   "expected %zu actual %zu",
4550
0
                   sizeof(*kvm_stats_header), ret);
4551
0
        g_free(descriptors);
4552
0
        return NULL;
4553
0
    }
4554
0
    size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size;
4555
4556
    /* Read stats descriptors */
4557
0
    kvm_stats_desc = g_malloc0_n(kvm_stats_header->num_desc, size_desc);
4558
0
    ret = pread(stats_fd, kvm_stats_desc,
4559
0
                size_desc * kvm_stats_header->num_desc,
4560
0
                kvm_stats_header->desc_offset);
4561
4562
0
    if (ret != size_desc * kvm_stats_header->num_desc) {
4563
0
        error_setg(errp, "KVM stats: failed to read stats descriptors: "
4564
0
                   "expected %zu actual %zu",
4565
0
                   size_desc * kvm_stats_header->num_desc, ret);
4566
0
        g_free(descriptors);
4567
0
        g_free(kvm_stats_desc);
4568
0
        return NULL;
4569
0
    }
4570
0
    descriptors->kvm_stats_desc = kvm_stats_desc;
4571
0
    descriptors->ident = ident;
4572
0
    QTAILQ_INSERT_TAIL(&stats_descriptors, descriptors, next);
4573
0
    return descriptors;
4574
0
}
4575
4576
static void query_stats(StatsResultList **result, StatsTarget target,
4577
                        strList *names, int stats_fd, CPUState *cpu,
4578
                        Error **errp)
4579
0
{
4580
0
    struct kvm_stats_desc *kvm_stats_desc;
4581
0
    struct kvm_stats_header *kvm_stats_header;
4582
0
    StatsDescriptors *descriptors;
4583
0
    g_autofree uint64_t *stats_data = NULL;
4584
0
    struct kvm_stats_desc *pdesc;
4585
0
    StatsList *stats_list = NULL;
4586
0
    size_t size_desc, size_data = 0;
4587
0
    ssize_t ret;
4588
0
    int i;
4589
4590
0
    descriptors = find_stats_descriptors(target, stats_fd, errp);
4591
0
    if (!descriptors) {
4592
0
        return;
4593
0
    }
4594
4595
0
    kvm_stats_header = &descriptors->kvm_stats_header;
4596
0
    kvm_stats_desc = descriptors->kvm_stats_desc;
4597
0
    size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size;
4598
4599
    /* Tally the total data size; read schema data */
4600
0
    for (i = 0; i < kvm_stats_header->num_desc; ++i) {
4601
0
        pdesc = (void *)kvm_stats_desc + i * size_desc;
4602
0
        size_data += pdesc->size * sizeof(*stats_data);
4603
0
    }
4604
4605
0
    stats_data = g_malloc0(size_data);
4606
0
    ret = pread(stats_fd, stats_data, size_data, kvm_stats_header->data_offset);
4607
4608
0
    if (ret != size_data) {
4609
0
        error_setg(errp, "KVM stats: failed to read data: "
4610
0
                   "expected %zu actual %zu", size_data, ret);
4611
0
        return;
4612
0
    }
4613
4614
0
    for (i = 0; i < kvm_stats_header->num_desc; ++i) {
4615
0
        uint64_t *stats;
4616
0
        pdesc = (void *)kvm_stats_desc + i * size_desc;
4617
4618
        /* Add entry to the list */
4619
0
        stats = (void *)stats_data + pdesc->offset;
4620
0
        if (!apply_str_list_filter(pdesc->name, names)) {
4621
0
            continue;
4622
0
        }
4623
0
        stats_list = add_kvmstat_entry(pdesc, stats, stats_list, errp);
4624
0
    }
4625
4626
0
    if (!stats_list) {
4627
0
        return;
4628
0
    }
4629
4630
0
    switch (target) {
4631
0
    case STATS_TARGET_VM:
4632
0
        add_stats_entry(result, STATS_PROVIDER_KVM, NULL, stats_list);
4633
0
        break;
4634
0
    case STATS_TARGET_VCPU:
4635
0
        add_stats_entry(result, STATS_PROVIDER_KVM,
4636
0
                        cpu->parent_obj.canonical_path,
4637
0
                        stats_list);
4638
0
        break;
4639
0
    default:
4640
0
        g_assert_not_reached();
4641
0
    }
4642
0
}
4643
4644
static void query_stats_schema(StatsSchemaList **result, StatsTarget target,
4645
                               int stats_fd, Error **errp)
4646
0
{
4647
0
    struct kvm_stats_desc *kvm_stats_desc;
4648
0
    struct kvm_stats_header *kvm_stats_header;
4649
0
    StatsDescriptors *descriptors;
4650
0
    struct kvm_stats_desc *pdesc;
4651
0
    StatsSchemaValueList *stats_list = NULL;
4652
0
    size_t size_desc;
4653
0
    int i;
4654
4655
0
    descriptors = find_stats_descriptors(target, stats_fd, errp);
4656
0
    if (!descriptors) {
4657
0
        return;
4658
0
    }
4659
4660
0
    kvm_stats_header = &descriptors->kvm_stats_header;
4661
0
    kvm_stats_desc = descriptors->kvm_stats_desc;
4662
0
    size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size;
4663
4664
    /* Tally the total data size; read schema data */
4665
0
    for (i = 0; i < kvm_stats_header->num_desc; ++i) {
4666
0
        pdesc = (void *)kvm_stats_desc + i * size_desc;
4667
0
        stats_list = add_kvmschema_entry(pdesc, stats_list, errp);
4668
0
    }
4669
4670
0
    add_stats_schema(result, STATS_PROVIDER_KVM, target, stats_list);
4671
0
}
4672
4673
static void query_stats_vcpu(CPUState *cpu, StatsArgs *kvm_stats_args)
4674
0
{
4675
0
    int stats_fd = cpu->kvm_vcpu_stats_fd;
4676
0
    Error *local_err = NULL;
4677
4678
0
    if (stats_fd == -1) {
4679
0
        error_setg_errno(&local_err, errno, "KVM stats: ioctl failed");
4680
0
        error_propagate(kvm_stats_args->errp, local_err);
4681
0
        return;
4682
0
    }
4683
0
    query_stats(kvm_stats_args->result.stats, STATS_TARGET_VCPU,
4684
0
                kvm_stats_args->names, stats_fd, cpu,
4685
0
                kvm_stats_args->errp);
4686
0
}
4687
4688
static void query_stats_schema_vcpu(CPUState *cpu, StatsArgs *kvm_stats_args)
4689
0
{
4690
0
    int stats_fd = cpu->kvm_vcpu_stats_fd;
4691
0
    Error *local_err = NULL;
4692
4693
0
    if (stats_fd == -1) {
4694
0
        error_setg_errno(&local_err, errno, "KVM stats: ioctl failed");
4695
0
        error_propagate(kvm_stats_args->errp, local_err);
4696
0
        return;
4697
0
    }
4698
0
    query_stats_schema(kvm_stats_args->result.schema, STATS_TARGET_VCPU, stats_fd,
4699
0
                       kvm_stats_args->errp);
4700
0
}
4701
4702
static void query_stats_cb(StatsResultList **result, StatsTarget target,
4703
                           strList *names, strList *targets, Error **errp)
4704
0
{
4705
0
    KVMState *s = kvm_state;
4706
0
    CPUState *cpu;
4707
0
    int stats_fd;
4708
4709
0
    switch (target) {
4710
0
    case STATS_TARGET_VM:
4711
0
    {
4712
0
        stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL);
4713
0
        if (stats_fd == -1) {
4714
0
            error_setg_errno(errp, errno, "KVM stats: ioctl failed");
4715
0
            return;
4716
0
        }
4717
0
        query_stats(result, target, names, stats_fd, NULL, errp);
4718
0
        close(stats_fd);
4719
0
        break;
4720
0
    }
4721
0
    case STATS_TARGET_VCPU:
4722
0
    {
4723
0
        StatsArgs stats_args;
4724
0
        stats_args.result.stats = result;
4725
0
        stats_args.names = names;
4726
0
        stats_args.errp = errp;
4727
0
        CPU_FOREACH(cpu) {
4728
0
            if (!apply_str_list_filter(cpu->parent_obj.canonical_path, targets)) {
4729
0
                continue;
4730
0
            }
4731
0
            query_stats_vcpu(cpu, &stats_args);
4732
0
        }
4733
0
        break;
4734
0
    }
4735
0
    default:
4736
0
        break;
4737
0
    }
4738
0
}
4739
4740
void query_stats_schemas_cb(StatsSchemaList **result, Error **errp)
4741
0
{
4742
0
    StatsArgs stats_args;
4743
0
    KVMState *s = kvm_state;
4744
0
    int stats_fd;
4745
4746
0
    stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL);
4747
0
    if (stats_fd == -1) {
4748
0
        error_setg_errno(errp, errno, "KVM stats: ioctl failed");
4749
0
        return;
4750
0
    }
4751
0
    query_stats_schema(result, STATS_TARGET_VM, stats_fd, errp);
4752
0
    close(stats_fd);
4753
4754
0
    if (first_cpu) {
4755
0
        stats_args.result.schema = result;
4756
0
        stats_args.errp = errp;
4757
0
        query_stats_schema_vcpu(first_cpu, &stats_args);
4758
0
    }
4759
0
}
4760
4761
void kvm_mark_guest_state_protected(void)
4762
0
{
4763
0
    kvm_state->guest_state_protected = true;
4764
0
}
4765
4766
int kvm_create_guest_memfd(uint64_t size, uint64_t flags, Error **errp)
4767
0
{
4768
0
    int fd;
4769
0
    struct kvm_create_guest_memfd guest_memfd = {
4770
0
        .size = size,
4771
0
        .flags = flags,
4772
0
    };
4773
4774
    /*
4775
     * kvm_create_guest_memfd() can be invoked from generic hostmem code
4776
     * where KVM might be compiled but not necessarily enabled. Because
4777
     * of this, check for KVM support explicitly before continuing with
4778
     * any KVM-specific checks.
4779
     */
4780
0
    if (!kvm_enabled()) {
4781
0
        error_setg(errp, "guest-memfd requires KVM accelerator");
4782
0
        return -1;
4783
0
    }
4784
4785
0
    if (!kvm_guest_memfd_supported) {
4786
0
        error_setg(errp, "KVM does not support guest_memfd");
4787
0
        return -1;
4788
0
    }
4789
4790
0
    if (flags & ~kvm_guest_memfd_flags_supported) {
4791
0
        error_setg(errp, "Current KVM instance does not support "
4792
0
                   "guest-memfd flag: 0x%"PRIx64,
4793
0
                   flags & ~kvm_guest_memfd_flags_supported);
4794
0
        return -1;
4795
0
    }
4796
4797
0
    fd = kvm_vm_ioctl(kvm_state, KVM_CREATE_GUEST_MEMFD, &guest_memfd);
4798
0
    if (fd < 0) {
4799
0
        error_setg_errno(errp, errno, "Error creating KVM guest_memfd");
4800
0
        return -1;
4801
0
    }
4802
4803
0
    return fd;
4804
0
}