/src/qemu/accel/kvm/kvm-all.c
Line | Count | Source |
1 | | /* |
2 | | * QEMU KVM support |
3 | | * |
4 | | * Copyright IBM, Corp. 2008 |
5 | | * Red Hat, Inc. 2008 |
6 | | * |
7 | | * Authors: |
8 | | * Anthony Liguori <aliguori@us.ibm.com> |
9 | | * Glauber Costa <gcosta@redhat.com> |
10 | | * |
11 | | * This work is licensed under the terms of the GNU GPL, version 2 or later. |
12 | | * See the COPYING file in the top-level directory. |
13 | | * |
14 | | */ |
15 | | |
16 | | #include "qemu/osdep.h" |
17 | | #include <sys/ioctl.h> |
18 | | #include <poll.h> |
19 | | |
20 | | #include <linux/kvm.h> |
21 | | |
22 | | #include "qemu/atomic.h" |
23 | | #include "qemu/option.h" |
24 | | #include "qemu/config-file.h" |
25 | | #include "qemu/error-report.h" |
26 | | #include "qapi/error.h" |
27 | | #include "qapi/qapi-builtin-type-infos.h" |
28 | | #include "hw/pci/msi.h" |
29 | | #include "hw/pci/msix.h" |
30 | | #include "hw/s390x/adapter.h" |
31 | | #include "gdbstub/enums.h" |
32 | | #include "system/kvm_int.h" |
33 | | #include "system/runstate.h" |
34 | | #include "system/cpus.h" |
35 | | #include "system/accel-blocker.h" |
36 | | #include "system/physmem.h" |
37 | | #include "system/ramblock.h" |
38 | | #include "accel/accel-ops.h" |
39 | | #include "qemu/bswap.h" |
40 | | #include "exec/tswap.h" |
41 | | #include "exec/target_page.h" |
42 | | #include "system/memory.h" |
43 | | #include "qemu/event_notifier.h" |
44 | | #include "qemu/main-loop.h" |
45 | | #include "trace.h" |
46 | | #include "hw/core/irq.h" |
47 | | #include "qapi/visitor.h" |
48 | | #include "qapi/qapi-types-common.h" |
49 | | #include "qapi/qapi-visit-common.h" |
50 | | #include "qapi/qapi-type-infos-common.h" |
51 | | #include "system/reset.h" |
52 | | #include "qemu/guest-random.h" |
53 | | #include "system/hw_accel.h" |
54 | | #include "kvm-cpus.h" |
55 | | #include "system/dirtylimit.h" |
56 | | #include "qemu/range.h" |
57 | | |
58 | | #include "hw/core/boards.h" |
59 | | #include "system/stats.h" |
60 | | |
61 | | /* This check must be after config-host.h is included */ |
62 | | #ifdef CONFIG_EVENTFD |
63 | | #include <sys/eventfd.h> |
64 | | #endif |
65 | | |
66 | | #if defined(__x86_64__) || defined(__aarch64__) |
67 | | # define KVM_HAVE_MCE_INJECTION 1 |
68 | | #endif |
69 | | |
70 | | |
71 | | /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We |
72 | | * need to use the real host PAGE_SIZE, as that's what KVM will use. |
73 | | */ |
74 | | #ifdef PAGE_SIZE |
75 | | #undef PAGE_SIZE |
76 | | #endif |
77 | 0 | #define PAGE_SIZE qemu_real_host_page_size() |
78 | | |
79 | | #ifndef KVM_GUESTDBG_BLOCKIRQ |
80 | | #define KVM_GUESTDBG_BLOCKIRQ 0 |
81 | | #endif |
82 | | |
83 | | /* Default num of memslots to be allocated when VM starts */ |
84 | 0 | #define KVM_MEMSLOTS_NR_ALLOC_DEFAULT 16 |
85 | | /* Default max allowed memslots if kernel reported nothing */ |
86 | 0 | #define KVM_MEMSLOTS_NR_MAX_DEFAULT 32 |
87 | | |
88 | | struct KVMParkedVcpu { |
89 | | unsigned long vcpu_id; |
90 | | int kvm_fd; |
91 | | QLIST_ENTRY(KVMParkedVcpu) node; |
92 | | }; |
93 | | |
94 | | KVMState *kvm_state; |
95 | | VmfdChangeNotifier vmfd_notifier; |
96 | | bool kvm_kernel_irqchip; |
97 | | bool kvm_split_irqchip; |
98 | | bool kvm_async_interrupts_allowed; |
99 | | bool kvm_halt_in_kernel_allowed; |
100 | | bool kvm_resamplefds_allowed; |
101 | | bool kvm_msi_via_irqfd_allowed; |
102 | | bool kvm_gsi_routing_allowed; |
103 | | bool kvm_gsi_direct_mapping; |
104 | | bool kvm_allowed; |
105 | | bool kvm_readonly_mem_allowed; |
106 | | bool kvm_vm_attributes_allowed; |
107 | | bool kvm_msi_use_devid; |
108 | | bool kvm_pre_fault_memory_supported; |
109 | | static bool kvm_immediate_exit; |
110 | | static uint64_t kvm_supported_memory_attributes; |
111 | | static bool kvm_guest_memfd_supported; |
112 | | static uint64_t kvm_guest_memfd_flags_supported; |
113 | | static hwaddr kvm_max_slot_size = ~0; |
114 | | |
115 | | static const KVMCapabilityInfo kvm_required_capabilities[] = { |
116 | | KVM_CAP_INFO(USER_MEMORY), |
117 | | KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS), |
118 | | KVM_CAP_INFO(JOIN_MEMORY_REGIONS_WORKS), |
119 | | KVM_CAP_INFO(INTERNAL_ERROR_DATA), |
120 | | KVM_CAP_INFO(IOEVENTFD), |
121 | | KVM_CAP_INFO(IOEVENTFD_ANY_LENGTH), |
122 | | KVM_CAP_LAST_INFO |
123 | | }; |
124 | | |
125 | | static NotifierList kvm_irqchip_change_notifiers = |
126 | | NOTIFIER_LIST_INITIALIZER(kvm_irqchip_change_notifiers); |
127 | | |
128 | | static NotifierWithReturnList register_vmfd_changed_notifiers = |
129 | | NOTIFIER_WITH_RETURN_LIST_INITIALIZER(register_vmfd_changed_notifiers); |
130 | | |
131 | | static NotifierWithReturnList register_vcpufd_changed_notifiers = |
132 | | NOTIFIER_WITH_RETURN_LIST_INITIALIZER(register_vcpufd_changed_notifiers); |
133 | | |
134 | | static int map_kvm_run(KVMState *s, CPUState *cpu, Error **errp); |
135 | | static int map_kvm_dirty_gfns(KVMState *s, CPUState *cpu, Error **errp); |
136 | | static int vcpu_unmap_regions(KVMState *s, CPUState *cpu); |
137 | | |
138 | | struct KVMResampleFd { |
139 | | int gsi; |
140 | | EventNotifier *resample_event; |
141 | | QLIST_ENTRY(KVMResampleFd) node; |
142 | | }; |
143 | | typedef struct KVMResampleFd KVMResampleFd; |
144 | | |
145 | | /* |
146 | | * Only used with split irqchip where we need to do the resample fd |
147 | | * kick for the kernel from userspace. |
148 | | */ |
149 | | static QLIST_HEAD(, KVMResampleFd) kvm_resample_fd_list = |
150 | | QLIST_HEAD_INITIALIZER(kvm_resample_fd_list); |
151 | | |
152 | | static QemuMutex kvm_resample_fd_list_lock; |
153 | | static QemuMutex kml_slots_lock; |
154 | | |
155 | 0 | #define kvm_slots_lock() qemu_mutex_lock(&kml_slots_lock) |
156 | 0 | #define kvm_slots_unlock() qemu_mutex_unlock(&kml_slots_lock) |
157 | | |
158 | | static void kvm_slot_init_dirty_bitmap(KVMSlot *mem); |
159 | | |
160 | | static inline void kvm_resample_fd_remove(int gsi) |
161 | 0 | { |
162 | 0 | KVMResampleFd *rfd; |
163 | |
|
164 | 0 | qemu_mutex_lock(&kvm_resample_fd_list_lock); |
165 | |
|
166 | 0 | QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) { |
167 | 0 | if (rfd->gsi == gsi) { |
168 | 0 | QLIST_REMOVE(rfd, node); |
169 | 0 | g_free(rfd); |
170 | 0 | break; |
171 | 0 | } |
172 | 0 | } |
173 | |
|
174 | 0 | qemu_mutex_unlock(&kvm_resample_fd_list_lock); |
175 | 0 | } |
176 | | |
177 | | static inline void kvm_resample_fd_insert(int gsi, EventNotifier *event) |
178 | 0 | { |
179 | 0 | KVMResampleFd *rfd = g_new0(KVMResampleFd, 1); |
180 | |
|
181 | 0 | rfd->gsi = gsi; |
182 | 0 | rfd->resample_event = event; |
183 | |
|
184 | 0 | qemu_mutex_lock(&kvm_resample_fd_list_lock); |
185 | 0 | QLIST_INSERT_HEAD(&kvm_resample_fd_list, rfd, node); |
186 | 0 | qemu_mutex_unlock(&kvm_resample_fd_list_lock); |
187 | 0 | } |
188 | | |
189 | | void kvm_resample_fd_notify(int gsi) |
190 | 0 | { |
191 | 0 | KVMResampleFd *rfd; |
192 | |
|
193 | 0 | qemu_mutex_lock(&kvm_resample_fd_list_lock); |
194 | |
|
195 | 0 | QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) { |
196 | 0 | if (rfd->gsi == gsi) { |
197 | 0 | event_notifier_set(rfd->resample_event); |
198 | 0 | trace_kvm_resample_fd_notify(gsi); |
199 | 0 | break; |
200 | 0 | } |
201 | 0 | } |
202 | |
|
203 | 0 | qemu_mutex_unlock(&kvm_resample_fd_list_lock); |
204 | 0 | } |
205 | | |
206 | | /** |
207 | | * kvm_slots_grow(): Grow the slots[] array in the KVMMemoryListener |
208 | | * |
209 | | * @kml: The KVMMemoryListener* to grow the slots[] array |
210 | | * @nr_slots_new: The new size of slots[] array |
211 | | * |
212 | | * Returns: True if the array grows larger, false otherwise. |
213 | | */ |
214 | | static bool kvm_slots_grow(KVMMemoryListener *kml, unsigned int nr_slots_new) |
215 | 0 | { |
216 | 0 | unsigned int i, cur = kml->nr_slots_allocated; |
217 | 0 | KVMSlot *slots; |
218 | |
|
219 | 0 | if (nr_slots_new > kvm_state->nr_slots_max) { |
220 | 0 | nr_slots_new = kvm_state->nr_slots_max; |
221 | 0 | } |
222 | |
|
223 | 0 | if (cur >= nr_slots_new) { |
224 | | /* Big enough, no need to grow, or we reached max */ |
225 | 0 | return false; |
226 | 0 | } |
227 | | |
228 | 0 | if (cur == 0) { |
229 | 0 | slots = g_new0(KVMSlot, nr_slots_new); |
230 | 0 | } else { |
231 | 0 | assert(kml->slots); |
232 | 0 | slots = g_renew(KVMSlot, kml->slots, nr_slots_new); |
233 | | /* |
234 | | * g_renew() doesn't initialize extended buffers, however kvm |
235 | | * memslots require fields to be zero-initialized. E.g. pointers, |
236 | | * memory_size field, etc. |
237 | | */ |
238 | 0 | memset(&slots[cur], 0x0, sizeof(slots[0]) * (nr_slots_new - cur)); |
239 | 0 | } |
240 | |
|
241 | 0 | for (i = cur; i < nr_slots_new; i++) { |
242 | 0 | slots[i].slot = i; |
243 | 0 | } |
244 | |
|
245 | 0 | kml->slots = slots; |
246 | 0 | kml->nr_slots_allocated = nr_slots_new; |
247 | 0 | trace_kvm_slots_grow(cur, nr_slots_new); |
248 | |
|
249 | 0 | return true; |
250 | 0 | } |
251 | | |
252 | | static bool kvm_slots_double(KVMMemoryListener *kml) |
253 | 0 | { |
254 | 0 | return kvm_slots_grow(kml, kml->nr_slots_allocated * 2); |
255 | 0 | } |
256 | | |
257 | | unsigned int kvm_get_max_memslots(void) |
258 | 0 | { |
259 | 0 | KVMState *s = KVM_STATE(current_accel()); |
260 | |
|
261 | 0 | return s->nr_slots_max; |
262 | 0 | } |
263 | | |
264 | | unsigned int kvm_get_free_memslots(void) |
265 | 0 | { |
266 | 0 | unsigned int used_slots = 0; |
267 | 0 | KVMState *s = kvm_state; |
268 | 0 | int i; |
269 | |
|
270 | 0 | kvm_slots_lock(); |
271 | 0 | for (i = 0; i < s->nr_as; i++) { |
272 | 0 | if (!s->as[i].ml) { |
273 | 0 | continue; |
274 | 0 | } |
275 | 0 | used_slots = MAX(used_slots, s->as[i].ml->nr_slots_used); |
276 | 0 | } |
277 | 0 | kvm_slots_unlock(); |
278 | |
|
279 | 0 | return s->nr_slots_max - used_slots; |
280 | 0 | } |
281 | | |
282 | | /* Called with KVMMemoryListener.slots_lock held */ |
283 | | static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml) |
284 | 0 | { |
285 | 0 | unsigned int n; |
286 | 0 | int i; |
287 | |
|
288 | 0 | for (i = 0; i < kml->nr_slots_allocated; i++) { |
289 | 0 | if (kml->slots[i].memory_size == 0) { |
290 | 0 | return &kml->slots[i]; |
291 | 0 | } |
292 | 0 | } |
293 | | |
294 | | /* |
295 | | * If no free slots, try to grow first by doubling. Cache the old size |
296 | | * here to avoid another round of search: if the grow succeeded, it |
297 | | * means slots[] now must have the existing "n" slots occupied, |
298 | | * followed by one or more free slots starting from slots[n]. |
299 | | */ |
300 | 0 | n = kml->nr_slots_allocated; |
301 | 0 | if (kvm_slots_double(kml)) { |
302 | 0 | return &kml->slots[n]; |
303 | 0 | } |
304 | | |
305 | 0 | return NULL; |
306 | 0 | } |
307 | | |
308 | | /* Called with KVMMemoryListener.slots_lock held */ |
309 | | static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml) |
310 | 0 | { |
311 | 0 | KVMSlot *slot = kvm_get_free_slot(kml); |
312 | |
|
313 | 0 | if (slot) { |
314 | 0 | return slot; |
315 | 0 | } |
316 | | |
317 | 0 | fprintf(stderr, "%s: no free slot available\n", __func__); |
318 | 0 | abort(); |
319 | 0 | } |
320 | | |
321 | | static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml, |
322 | | hwaddr start_addr, |
323 | | hwaddr size) |
324 | 0 | { |
325 | 0 | int i; |
326 | |
|
327 | 0 | for (i = 0; i < kml->nr_slots_allocated; i++) { |
328 | 0 | KVMSlot *mem = &kml->slots[i]; |
329 | |
|
330 | 0 | if (start_addr == mem->start_addr && size == mem->memory_size) { |
331 | 0 | return mem; |
332 | 0 | } |
333 | 0 | } |
334 | | |
335 | 0 | return NULL; |
336 | 0 | } |
337 | | |
338 | | /* |
339 | | * Calculate and align the start address and the size of the section. |
340 | | * Return the size. If the size is 0, the aligned section is empty. |
341 | | */ |
342 | | static hwaddr kvm_align_section(MemoryRegionSection *section, |
343 | | hwaddr *start) |
344 | 0 | { |
345 | 0 | hwaddr size = int128_get64(section->size); |
346 | 0 | hwaddr delta, aligned; |
347 | | |
348 | | /* kvm works in page size chunks, but the function may be called |
349 | | with sub-page size and unaligned start address. Pad the start |
350 | | address to next and truncate size to previous page boundary. */ |
351 | 0 | aligned = ROUND_UP(section->offset_within_address_space, |
352 | 0 | qemu_real_host_page_size()); |
353 | 0 | delta = aligned - section->offset_within_address_space; |
354 | 0 | *start = aligned; |
355 | 0 | if (delta > size) { |
356 | 0 | return 0; |
357 | 0 | } |
358 | | |
359 | 0 | return (size - delta) & qemu_real_host_page_mask(); |
360 | 0 | } |
361 | | |
362 | | int kvm_physical_memory_addr_from_host(KVMState *s, void *ram, |
363 | | hwaddr *phys_addr) |
364 | 0 | { |
365 | 0 | KVMMemoryListener *kml = &s->memory_listener; |
366 | 0 | int i, ret = 0; |
367 | |
|
368 | 0 | kvm_slots_lock(); |
369 | 0 | for (i = 0; i < kml->nr_slots_allocated; i++) { |
370 | 0 | KVMSlot *mem = &kml->slots[i]; |
371 | |
|
372 | 0 | if (ram >= mem->ram && ram < mem->ram + mem->memory_size) { |
373 | 0 | *phys_addr = mem->start_addr + (ram - mem->ram); |
374 | 0 | ret = 1; |
375 | 0 | break; |
376 | 0 | } |
377 | 0 | } |
378 | 0 | kvm_slots_unlock(); |
379 | |
|
380 | 0 | return ret; |
381 | 0 | } |
382 | | |
383 | | static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot, bool new) |
384 | 0 | { |
385 | 0 | KVMState *s = kvm_state; |
386 | 0 | struct kvm_userspace_memory_region2 mem = {}; |
387 | 0 | int ret; |
388 | |
|
389 | 0 | mem.slot = slot->slot | (kml->as_id << 16); |
390 | 0 | mem.guest_phys_addr = slot->start_addr; |
391 | 0 | mem.userspace_addr = (unsigned long)slot->ram; |
392 | 0 | mem.flags = slot->flags; |
393 | 0 | mem.guest_memfd = slot->guest_memfd; |
394 | 0 | mem.guest_memfd_offset = slot->guest_memfd_offset; |
395 | |
|
396 | 0 | if (slot->memory_size && !new && (mem.flags ^ slot->old_flags) & KVM_MEM_READONLY) { |
397 | | /* Set the slot size to 0 before setting the slot to the desired |
398 | | * value. This is needed based on KVM commit 75d61fbc. */ |
399 | 0 | mem.memory_size = 0; |
400 | |
|
401 | 0 | if (kvm_guest_memfd_supported) { |
402 | 0 | ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION2, &mem); |
403 | 0 | } else { |
404 | 0 | ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); |
405 | 0 | } |
406 | 0 | if (ret < 0) { |
407 | 0 | goto err; |
408 | 0 | } |
409 | 0 | } |
410 | 0 | mem.memory_size = slot->memory_size; |
411 | 0 | if (kvm_guest_memfd_supported) { |
412 | 0 | ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION2, &mem); |
413 | 0 | } else { |
414 | 0 | ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); |
415 | 0 | } |
416 | 0 | slot->old_flags = mem.flags; |
417 | 0 | err: |
418 | 0 | trace_kvm_set_user_memory(mem.slot >> 16, (uint16_t)mem.slot, mem.flags, |
419 | 0 | mem.guest_phys_addr, mem.memory_size, |
420 | 0 | mem.userspace_addr, mem.guest_memfd, |
421 | 0 | mem.guest_memfd_offset, ret); |
422 | 0 | if (ret < 0) { |
423 | 0 | if (kvm_guest_memfd_supported) { |
424 | 0 | error_report("%s: KVM_SET_USER_MEMORY_REGION2 failed, slot=%d," |
425 | 0 | " start=0x%" PRIx64 ", size=0x%" PRIx64 "," |
426 | 0 | " flags=0x%" PRIx32 ", guest_memfd=%" PRId32 "," |
427 | 0 | " guest_memfd_offset=0x%" PRIx64 ": %s", |
428 | 0 | __func__, mem.slot, slot->start_addr, |
429 | 0 | (uint64_t)mem.memory_size, mem.flags, |
430 | 0 | mem.guest_memfd, (uint64_t)mem.guest_memfd_offset, |
431 | 0 | strerror(errno)); |
432 | 0 | } else { |
433 | 0 | error_report("%s: KVM_SET_USER_MEMORY_REGION failed, slot=%d," |
434 | 0 | " start=0x%" PRIx64 ", size=0x%" PRIx64 ": %s", |
435 | 0 | __func__, mem.slot, slot->start_addr, |
436 | 0 | (uint64_t)mem.memory_size, strerror(errno)); |
437 | 0 | } |
438 | 0 | } |
439 | 0 | return ret; |
440 | 0 | } |
441 | | |
442 | | static void kvm_create_vcpu_internal(CPUState *cpu, KVMState *s, int kvm_fd) |
443 | 0 | { |
444 | 0 | cpu->kvm_fd = kvm_fd; |
445 | 0 | cpu->kvm_state = s; |
446 | 0 | if (!s->guest_state_protected) { |
447 | 0 | cpu->vcpu_dirty = true; |
448 | 0 | } |
449 | 0 | cpu->dirty_pages = 0; |
450 | 0 | cpu->throttle_us_per_full = 0; |
451 | |
|
452 | 0 | return; |
453 | 0 | } |
454 | | |
455 | | static int kvm_rebind_vcpus(Error **errp) |
456 | 0 | { |
457 | 0 | CPUState *cpu; |
458 | 0 | unsigned long vcpu_id; |
459 | 0 | KVMState *s = kvm_state; |
460 | 0 | int kvm_fd, ret = 0; |
461 | |
|
462 | 0 | CPU_FOREACH(cpu) { |
463 | 0 | vcpu_id = kvm_arch_vcpu_id(cpu); |
464 | |
|
465 | 0 | if (cpu->kvm_fd) { |
466 | 0 | close(cpu->kvm_fd); |
467 | 0 | } |
468 | |
|
469 | 0 | ret = kvm_arch_destroy_vcpu(cpu); |
470 | 0 | if (ret < 0) { |
471 | 0 | goto err; |
472 | 0 | } |
473 | | |
474 | 0 | if (s->coalesced_mmio_ring == (void *)cpu->kvm_run + PAGE_SIZE) { |
475 | 0 | s->coalesced_mmio_ring = NULL; |
476 | 0 | } |
477 | |
|
478 | 0 | ret = vcpu_unmap_regions(s, cpu); |
479 | 0 | if (ret < 0) { |
480 | 0 | goto err; |
481 | 0 | } |
482 | | |
483 | 0 | ret = kvm_arch_pre_create_vcpu(cpu, errp); |
484 | 0 | if (ret < 0) { |
485 | 0 | goto err; |
486 | 0 | } |
487 | | |
488 | 0 | kvm_fd = kvm_vm_ioctl(s, KVM_CREATE_VCPU, vcpu_id); |
489 | 0 | if (kvm_fd < 0) { |
490 | 0 | error_report("KVM_CREATE_VCPU IOCTL failed for vCPU %lu (%s)", |
491 | 0 | vcpu_id, strerror(kvm_fd)); |
492 | 0 | return kvm_fd; |
493 | 0 | } |
494 | | |
495 | 0 | kvm_create_vcpu_internal(cpu, s, kvm_fd); |
496 | |
|
497 | 0 | ret = map_kvm_run(s, cpu, errp); |
498 | 0 | if (ret < 0) { |
499 | 0 | goto err; |
500 | 0 | } |
501 | | |
502 | 0 | if (s->kvm_dirty_ring_size) { |
503 | 0 | ret = map_kvm_dirty_gfns(s, cpu, errp); |
504 | 0 | if (ret < 0) { |
505 | 0 | goto err; |
506 | 0 | } |
507 | 0 | } |
508 | | |
509 | 0 | ret = kvm_arch_init_vcpu(cpu, errp); |
510 | 0 | if (ret < 0) { |
511 | 0 | error_prepend(errp, |
512 | 0 | "kvm_init_vcpu: kvm_arch_init_vcpu failed (%lu)", |
513 | 0 | vcpu_id); |
514 | 0 | } |
515 | |
|
516 | 0 | close(cpu->kvm_vcpu_stats_fd); |
517 | 0 | cpu->kvm_vcpu_stats_fd = kvm_vcpu_ioctl(cpu, KVM_GET_STATS_FD, NULL); |
518 | 0 | kvm_init_cpu_signals(cpu); |
519 | 0 | } |
520 | 0 | trace_kvm_rebind_vcpus(); |
521 | |
|
522 | 0 | err: |
523 | 0 | return ret; |
524 | 0 | } |
525 | | |
526 | | static void kvm_park_vcpu(CPUState *cpu) |
527 | 0 | { |
528 | 0 | struct KVMParkedVcpu *vcpu; |
529 | |
|
530 | 0 | trace_kvm_park_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu)); |
531 | |
|
532 | 0 | vcpu = g_malloc0(sizeof(*vcpu)); |
533 | 0 | vcpu->vcpu_id = kvm_arch_vcpu_id(cpu); |
534 | 0 | vcpu->kvm_fd = cpu->kvm_fd; |
535 | 0 | QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node); |
536 | 0 | } |
537 | | |
538 | | static int kvm_unpark_vcpu(KVMState *s, unsigned long vcpu_id) |
539 | 0 | { |
540 | 0 | struct KVMParkedVcpu *cpu; |
541 | 0 | int kvm_fd = -ENOENT; |
542 | |
|
543 | 0 | QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) { |
544 | 0 | if (cpu->vcpu_id == vcpu_id) { |
545 | 0 | QLIST_REMOVE(cpu, node); |
546 | 0 | kvm_fd = cpu->kvm_fd; |
547 | 0 | g_free(cpu); |
548 | 0 | break; |
549 | 0 | } |
550 | 0 | } |
551 | |
|
552 | 0 | trace_kvm_unpark_vcpu(vcpu_id, kvm_fd > 0 ? "unparked" : "!found parked"); |
553 | |
|
554 | 0 | return kvm_fd; |
555 | 0 | } |
556 | | |
557 | | static void kvm_reset_parked_vcpus(KVMState *s) |
558 | 0 | { |
559 | 0 | struct KVMParkedVcpu *cpu; |
560 | |
|
561 | 0 | QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) { |
562 | 0 | kvm_arch_reset_parked_vcpu(cpu->vcpu_id, cpu->kvm_fd); |
563 | 0 | } |
564 | 0 | } |
565 | | |
566 | | /** |
567 | | * kvm_create_vcpu - Gets a parked KVM vCPU or creates a KVM vCPU |
568 | | * @cpu: QOM CPUState object for which KVM vCPU has to be fetched/created. |
569 | | * |
570 | | * @returns: 0 when success, errno (<0) when failed. |
571 | | */ |
572 | | static int kvm_create_vcpu(CPUState *cpu) |
573 | 0 | { |
574 | 0 | unsigned long vcpu_id = kvm_arch_vcpu_id(cpu); |
575 | 0 | KVMState *s = kvm_state; |
576 | 0 | int kvm_fd; |
577 | | |
578 | | /* check if the KVM vCPU already exist but is parked */ |
579 | 0 | kvm_fd = kvm_unpark_vcpu(s, vcpu_id); |
580 | 0 | if (kvm_fd < 0) { |
581 | | /* vCPU not parked: create a new KVM vCPU */ |
582 | 0 | kvm_fd = kvm_vm_ioctl(s, KVM_CREATE_VCPU, vcpu_id); |
583 | 0 | if (kvm_fd < 0) { |
584 | 0 | error_report("KVM_CREATE_VCPU IOCTL failed for vCPU %lu", vcpu_id); |
585 | 0 | return kvm_fd; |
586 | 0 | } |
587 | 0 | } |
588 | | |
589 | 0 | kvm_create_vcpu_internal(cpu, s, kvm_fd); |
590 | |
|
591 | 0 | trace_kvm_create_vcpu(cpu->cpu_index, vcpu_id, kvm_fd); |
592 | |
|
593 | 0 | return 0; |
594 | 0 | } |
595 | | |
596 | | int kvm_create_and_park_vcpu(CPUState *cpu) |
597 | 0 | { |
598 | 0 | int ret = 0; |
599 | |
|
600 | 0 | ret = kvm_create_vcpu(cpu); |
601 | 0 | if (!ret) { |
602 | 0 | kvm_park_vcpu(cpu); |
603 | 0 | } |
604 | |
|
605 | 0 | return ret; |
606 | 0 | } |
607 | | |
608 | | static int vcpu_unmap_regions(KVMState *s, CPUState *cpu) |
609 | 0 | { |
610 | 0 | int mmap_size; |
611 | 0 | int ret = 0; |
612 | |
|
613 | 0 | mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); |
614 | 0 | if (mmap_size < 0) { |
615 | 0 | ret = mmap_size; |
616 | 0 | trace_kvm_failed_get_vcpu_mmap_size(); |
617 | 0 | goto err; |
618 | 0 | } |
619 | | |
620 | | /* If I am the CPU that created coalesced_mmio_ring, then discard it */ |
621 | 0 | if (s->coalesced_mmio_ring == |
622 | 0 | (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE) { |
623 | 0 | s->coalesced_mmio_ring = NULL; |
624 | 0 | } |
625 | |
|
626 | 0 | ret = munmap(cpu->kvm_run, mmap_size); |
627 | 0 | if (ret < 0) { |
628 | 0 | goto err; |
629 | 0 | } |
630 | 0 | cpu->kvm_run = NULL; |
631 | |
|
632 | 0 | if (cpu->kvm_dirty_gfns) { |
633 | 0 | ret = munmap(cpu->kvm_dirty_gfns, s->kvm_dirty_ring_bytes); |
634 | 0 | if (ret < 0) { |
635 | 0 | goto err; |
636 | 0 | } |
637 | 0 | cpu->kvm_dirty_gfns = NULL; |
638 | 0 | } |
639 | | |
640 | 0 | err: |
641 | 0 | return ret; |
642 | 0 | } |
643 | | |
644 | | static int do_kvm_destroy_vcpu(CPUState *cpu) |
645 | 0 | { |
646 | 0 | KVMState *s = kvm_state; |
647 | 0 | int ret = 0; |
648 | |
|
649 | 0 | trace_kvm_destroy_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu)); |
650 | |
|
651 | 0 | ret = kvm_arch_destroy_vcpu(cpu); |
652 | 0 | if (ret < 0) { |
653 | 0 | goto err; |
654 | 0 | } |
655 | | |
656 | | /* If I am the CPU that created coalesced_mmio_ring, then discard it */ |
657 | 0 | if (s->coalesced_mmio_ring == (void *)cpu->kvm_run + PAGE_SIZE) { |
658 | 0 | s->coalesced_mmio_ring = NULL; |
659 | 0 | } |
660 | |
|
661 | 0 | ret = vcpu_unmap_regions(s, cpu); |
662 | 0 | if (ret < 0) { |
663 | 0 | goto err; |
664 | 0 | } |
665 | 0 | kvm_park_vcpu(cpu); |
666 | 0 | err: |
667 | 0 | return ret; |
668 | 0 | } |
669 | | |
670 | | void kvm_destroy_vcpu(CPUState *cpu) |
671 | 0 | { |
672 | 0 | if (do_kvm_destroy_vcpu(cpu) < 0) { |
673 | 0 | error_report("kvm_destroy_vcpu failed"); |
674 | 0 | exit(EXIT_FAILURE); |
675 | 0 | } |
676 | 0 | } |
677 | | |
678 | | static int map_kvm_run(KVMState *s, CPUState *cpu, Error **errp) |
679 | 0 | { |
680 | 0 | int mmap_size, ret = 0; |
681 | |
|
682 | 0 | mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); |
683 | 0 | if (mmap_size < 0) { |
684 | 0 | ret = mmap_size; |
685 | 0 | error_setg_errno(errp, -mmap_size, |
686 | 0 | "kvm_init_vcpu: KVM_GET_VCPU_MMAP_SIZE failed"); |
687 | 0 | goto err; |
688 | 0 | } |
689 | | |
690 | 0 | cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, |
691 | 0 | cpu->kvm_fd, 0); |
692 | 0 | if (cpu->kvm_run == MAP_FAILED) { |
693 | 0 | ret = -errno; |
694 | 0 | error_setg_errno(errp, ret, |
695 | 0 | "kvm_init_vcpu: mmap'ing vcpu state failed (%lu)", |
696 | 0 | kvm_arch_vcpu_id(cpu)); |
697 | 0 | goto err; |
698 | 0 | } |
699 | | |
700 | 0 | if (s->coalesced_mmio && !s->coalesced_mmio_ring) { |
701 | 0 | s->coalesced_mmio_ring = |
702 | 0 | (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE; |
703 | 0 | } |
704 | |
|
705 | 0 | err: |
706 | 0 | return ret; |
707 | 0 | } |
708 | | |
709 | | static int map_kvm_dirty_gfns(KVMState *s, CPUState *cpu, Error **errp) |
710 | 0 | { |
711 | 0 | int ret = 0; |
712 | | /* Use MAP_SHARED to share pages with the kernel */ |
713 | 0 | cpu->kvm_dirty_gfns = mmap(NULL, s->kvm_dirty_ring_bytes, |
714 | 0 | PROT_READ | PROT_WRITE, MAP_SHARED, |
715 | 0 | cpu->kvm_fd, |
716 | 0 | PAGE_SIZE * KVM_DIRTY_LOG_PAGE_OFFSET); |
717 | 0 | if (cpu->kvm_dirty_gfns == MAP_FAILED) { |
718 | 0 | ret = -errno; |
719 | 0 | } |
720 | |
|
721 | 0 | return ret; |
722 | 0 | } |
723 | | |
724 | | int kvm_init_vcpu(CPUState *cpu, Error **errp) |
725 | 0 | { |
726 | 0 | KVMState *s = kvm_state; |
727 | 0 | int ret; |
728 | |
|
729 | 0 | trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu)); |
730 | |
|
731 | 0 | ret = kvm_arch_pre_create_vcpu(cpu, errp); |
732 | 0 | if (ret < 0) { |
733 | 0 | goto err; |
734 | 0 | } |
735 | | |
736 | 0 | ret = kvm_create_vcpu(cpu); |
737 | 0 | if (ret < 0) { |
738 | 0 | error_setg_errno(errp, -ret, |
739 | 0 | "kvm_init_vcpu: kvm_create_vcpu failed (%lu)", |
740 | 0 | kvm_arch_vcpu_id(cpu)); |
741 | 0 | goto err; |
742 | 0 | } |
743 | | |
744 | 0 | ret = map_kvm_run(s, cpu, errp); |
745 | 0 | if (ret < 0) { |
746 | 0 | goto err; |
747 | 0 | } |
748 | | |
749 | 0 | if (s->kvm_dirty_ring_size) { |
750 | 0 | ret = map_kvm_dirty_gfns(s, cpu, errp); |
751 | 0 | if (ret < 0) { |
752 | 0 | goto err; |
753 | 0 | } |
754 | 0 | } |
755 | | |
756 | 0 | ret = kvm_arch_init_vcpu(cpu, errp); |
757 | 0 | if (ret < 0) { |
758 | 0 | error_prepend(errp, "kvm_init_vcpu: kvm_arch_init_vcpu failed (%lu)", |
759 | 0 | kvm_arch_vcpu_id(cpu)); |
760 | 0 | } |
761 | 0 | cpu->kvm_vcpu_stats_fd = kvm_vcpu_ioctl(cpu, KVM_GET_STATS_FD, NULL); |
762 | |
|
763 | 0 | err: |
764 | 0 | return ret; |
765 | 0 | } |
766 | | |
767 | | void kvm_close(void) |
768 | 0 | { |
769 | 0 | CPUState *cpu; |
770 | |
|
771 | 0 | if (!kvm_state || kvm_state->fd == -1) { |
772 | 0 | return; |
773 | 0 | } |
774 | | |
775 | 0 | CPU_FOREACH(cpu) { |
776 | 0 | cpu_remove_sync(cpu); |
777 | 0 | close(cpu->kvm_fd); |
778 | 0 | cpu->kvm_fd = -1; |
779 | 0 | close(cpu->kvm_vcpu_stats_fd); |
780 | 0 | cpu->kvm_vcpu_stats_fd = -1; |
781 | 0 | } |
782 | |
|
783 | 0 | if (kvm_state && kvm_state->fd != -1) { |
784 | 0 | close(kvm_state->vmfd); |
785 | 0 | kvm_state->vmfd = -1; |
786 | 0 | close(kvm_state->fd); |
787 | 0 | kvm_state->fd = -1; |
788 | 0 | } |
789 | 0 | kvm_state = NULL; |
790 | 0 | } |
791 | | |
792 | | /* |
793 | | * dirty pages logging control |
794 | | */ |
795 | | |
796 | | static int kvm_mem_flags(MemoryRegion *mr) |
797 | 0 | { |
798 | 0 | bool readonly = mr->readonly || memory_region_is_romd(mr); |
799 | 0 | int flags = 0; |
800 | |
|
801 | 0 | if (memory_region_get_dirty_log_mask(mr) != 0) { |
802 | 0 | flags |= KVM_MEM_LOG_DIRTY_PAGES; |
803 | 0 | } |
804 | 0 | if (readonly && kvm_readonly_mem_allowed) { |
805 | 0 | flags |= KVM_MEM_READONLY; |
806 | 0 | } |
807 | 0 | if (memory_region_has_guest_memfd_private(mr)) { |
808 | 0 | assert(kvm_guest_memfd_supported); |
809 | 0 | flags |= KVM_MEM_GUEST_MEMFD; |
810 | 0 | } |
811 | 0 | return flags; |
812 | 0 | } |
813 | | |
814 | | /* Called with KVMMemoryListener.slots_lock held */ |
815 | | static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem, |
816 | | MemoryRegion *mr) |
817 | 0 | { |
818 | 0 | mem->flags = kvm_mem_flags(mr); |
819 | | |
820 | | /* If nothing changed effectively, no need to issue ioctl */ |
821 | 0 | if (mem->flags == mem->old_flags) { |
822 | 0 | return 0; |
823 | 0 | } |
824 | | |
825 | 0 | kvm_slot_init_dirty_bitmap(mem); |
826 | 0 | return kvm_set_user_memory_region(kml, mem, false); |
827 | 0 | } |
828 | | |
829 | | static int kvm_section_update_flags(KVMMemoryListener *kml, |
830 | | MemoryRegionSection *section) |
831 | 0 | { |
832 | 0 | hwaddr start_addr, size, slot_size; |
833 | 0 | KVMSlot *mem; |
834 | 0 | int ret = 0; |
835 | |
|
836 | 0 | size = kvm_align_section(section, &start_addr); |
837 | 0 | if (!size) { |
838 | 0 | return 0; |
839 | 0 | } |
840 | | |
841 | 0 | kvm_slots_lock(); |
842 | |
|
843 | 0 | while (size && !ret) { |
844 | 0 | slot_size = MIN(kvm_max_slot_size, size); |
845 | 0 | mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); |
846 | 0 | if (!mem) { |
847 | | /* We don't have a slot if we want to trap every access. */ |
848 | 0 | goto out; |
849 | 0 | } |
850 | | |
851 | 0 | ret = kvm_slot_update_flags(kml, mem, section->mr); |
852 | 0 | start_addr += slot_size; |
853 | 0 | size -= slot_size; |
854 | 0 | } |
855 | | |
856 | 0 | out: |
857 | 0 | kvm_slots_unlock(); |
858 | 0 | return ret; |
859 | 0 | } |
860 | | |
861 | | static void kvm_log_start(MemoryListener *listener, |
862 | | MemoryRegionSection *section, |
863 | | int old, int new) |
864 | 0 | { |
865 | 0 | KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); |
866 | 0 | int r; |
867 | |
|
868 | 0 | if (old != 0) { |
869 | 0 | return; |
870 | 0 | } |
871 | | |
872 | 0 | r = kvm_section_update_flags(kml, section); |
873 | 0 | if (r < 0) { |
874 | 0 | abort(); |
875 | 0 | } |
876 | 0 | } |
877 | | |
878 | | static void kvm_log_stop(MemoryListener *listener, |
879 | | MemoryRegionSection *section, |
880 | | int old, int new) |
881 | 0 | { |
882 | 0 | KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); |
883 | 0 | int r; |
884 | |
|
885 | 0 | if (new != 0) { |
886 | 0 | return; |
887 | 0 | } |
888 | | |
889 | 0 | r = kvm_section_update_flags(kml, section); |
890 | 0 | if (r < 0) { |
891 | 0 | abort(); |
892 | 0 | } |
893 | 0 | } |
894 | | |
895 | | /* get kvm's dirty pages bitmap and update qemu's */ |
896 | | static void kvm_slot_sync_dirty_pages(KVMSlot *slot) |
897 | 0 | { |
898 | 0 | ram_addr_t start = slot->ram_start_offset; |
899 | 0 | ram_addr_t pages = slot->memory_size / qemu_real_host_page_size(); |
900 | |
|
901 | 0 | physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages); |
902 | 0 | } |
903 | | |
904 | | static void kvm_slot_reset_dirty_pages(KVMSlot *slot) |
905 | 0 | { |
906 | 0 | memset(slot->dirty_bmap, 0, slot->dirty_bmap_size); |
907 | 0 | } |
908 | | |
909 | 0 | #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1)) |
910 | | |
911 | | /* Allocate the dirty bitmap for a slot */ |
912 | | static void kvm_slot_init_dirty_bitmap(KVMSlot *mem) |
913 | 0 | { |
914 | 0 | if (!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) || mem->dirty_bmap) { |
915 | 0 | return; |
916 | 0 | } |
917 | | |
918 | | /* |
919 | | * XXX bad kernel interface alert |
920 | | * For dirty bitmap, kernel allocates array of size aligned to |
921 | | * bits-per-long. But for case when the kernel is 64bits and |
922 | | * the userspace is 32bits, userspace can't align to the same |
923 | | * bits-per-long, since sizeof(long) is different between kernel |
924 | | * and user space. This way, userspace will provide buffer which |
925 | | * may be 4 bytes less than the kernel will use, resulting in |
926 | | * userspace memory corruption (which is not detectable by valgrind |
927 | | * too, in most cases). |
928 | | * So for now, let's align to 64 instead of HOST_LONG_BITS here, in |
929 | | * a hope that sizeof(long) won't become >8 any time soon. |
930 | | * |
931 | | * Note: the granule of kvm dirty log is qemu_real_host_page_size. |
932 | | * And mem->memory_size is aligned to it (otherwise this mem can't |
933 | | * be registered to KVM). |
934 | | */ |
935 | 0 | hwaddr bitmap_size = ALIGN(mem->memory_size / qemu_real_host_page_size(), |
936 | 0 | /*HOST_LONG_BITS*/ 64) / 8; |
937 | 0 | mem->dirty_bmap = g_malloc0(bitmap_size); |
938 | 0 | mem->dirty_bmap_size = bitmap_size; |
939 | 0 | } |
940 | | |
941 | | /* |
942 | | * Sync dirty bitmap from kernel to KVMSlot.dirty_bmap, return true if |
943 | | * succeeded, false otherwise |
944 | | */ |
945 | | static bool kvm_slot_get_dirty_log(KVMState *s, KVMSlot *slot) |
946 | 0 | { |
947 | 0 | struct kvm_dirty_log d = {}; |
948 | 0 | int ret; |
949 | |
|
950 | 0 | d.dirty_bitmap = slot->dirty_bmap; |
951 | 0 | d.slot = slot->slot | (slot->as_id << 16); |
952 | 0 | ret = kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d); |
953 | |
|
954 | 0 | if (ret == -ENOENT) { |
955 | | /* kernel does not have dirty bitmap in this slot */ |
956 | 0 | ret = 0; |
957 | 0 | } |
958 | 0 | if (ret) { |
959 | 0 | error_report_once("%s: KVM_GET_DIRTY_LOG failed with %d", |
960 | 0 | __func__, ret); |
961 | 0 | } |
962 | 0 | return ret == 0; |
963 | 0 | } |
964 | | |
965 | | /* Should be with all slots_lock held for the address spaces. */ |
966 | | static void kvm_dirty_ring_mark_page(KVMState *s, uint32_t as_id, |
967 | | uint32_t slot_id, uint64_t offset) |
968 | 0 | { |
969 | 0 | KVMMemoryListener *kml; |
970 | 0 | KVMSlot *mem; |
971 | |
|
972 | 0 | if (as_id >= s->nr_as) { |
973 | 0 | return; |
974 | 0 | } |
975 | | |
976 | 0 | kml = s->as[as_id].ml; |
977 | 0 | mem = &kml->slots[slot_id]; |
978 | |
|
979 | 0 | if (!mem->memory_size || offset >= |
980 | 0 | (mem->memory_size / qemu_real_host_page_size())) { |
981 | 0 | return; |
982 | 0 | } |
983 | | |
984 | 0 | set_bit(offset, mem->dirty_bmap); |
985 | 0 | } |
986 | | |
987 | | static bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn) |
988 | 0 | { |
989 | | /* |
990 | | * Read the flags before the value. Pairs with barrier in |
991 | | * KVM's kvm_dirty_ring_push() function. |
992 | | */ |
993 | 0 | return qatomic_load_acquire(&gfn->flags) == KVM_DIRTY_GFN_F_DIRTY; |
994 | 0 | } |
995 | | |
996 | | static void dirty_gfn_set_collected(struct kvm_dirty_gfn *gfn) |
997 | 0 | { |
998 | | /* |
999 | | * Use a store-release so that the CPU that executes KVM_RESET_DIRTY_RINGS |
1000 | | * sees the full content of the ring: |
1001 | | * |
1002 | | * CPU0 CPU1 CPU2 |
1003 | | * ------------------------------------------------------------------------------ |
1004 | | * fill gfn0 |
1005 | | * store-rel flags for gfn0 |
1006 | | * load-acq flags for gfn0 |
1007 | | * store-rel RESET for gfn0 |
1008 | | * ioctl(RESET_RINGS) |
1009 | | * load-acq flags for gfn0 |
1010 | | * check if flags have RESET |
1011 | | * |
1012 | | * The synchronization goes from CPU2 to CPU0 to CPU1. |
1013 | | */ |
1014 | 0 | qatomic_store_release(&gfn->flags, KVM_DIRTY_GFN_F_RESET); |
1015 | 0 | } |
1016 | | |
1017 | | /* |
1018 | | * Should be with all slots_lock held for the address spaces. It returns the |
1019 | | * dirty page we've collected on this dirty ring. |
1020 | | */ |
1021 | | static uint32_t kvm_dirty_ring_reap_one(KVMState *s, CPUState *cpu) |
1022 | 0 | { |
1023 | 0 | struct kvm_dirty_gfn *dirty_gfns = cpu->kvm_dirty_gfns, *cur; |
1024 | 0 | uint32_t ring_size = s->kvm_dirty_ring_size; |
1025 | 0 | uint32_t count = 0, fetch = cpu->kvm_fetch_index; |
1026 | | |
1027 | | /* |
1028 | | * It's possible that we race with vcpu creation code where the vcpu is |
1029 | | * put onto the vcpus list but not yet initialized the dirty ring |
1030 | | * structures. If so, skip it. |
1031 | | */ |
1032 | 0 | if (!cpu->created) { |
1033 | 0 | return 0; |
1034 | 0 | } |
1035 | | |
1036 | 0 | assert(dirty_gfns && ring_size); |
1037 | 0 | trace_kvm_dirty_ring_reap_vcpu(cpu->cpu_index); |
1038 | |
|
1039 | 0 | while (true) { |
1040 | 0 | cur = &dirty_gfns[fetch % ring_size]; |
1041 | 0 | if (!dirty_gfn_is_dirtied(cur)) { |
1042 | 0 | break; |
1043 | 0 | } |
1044 | 0 | kvm_dirty_ring_mark_page(s, cur->slot >> 16, cur->slot & 0xffff, |
1045 | 0 | cur->offset); |
1046 | 0 | dirty_gfn_set_collected(cur); |
1047 | 0 | trace_kvm_dirty_ring_page(cpu->cpu_index, fetch, cur->offset); |
1048 | 0 | fetch++; |
1049 | 0 | count++; |
1050 | 0 | } |
1051 | 0 | cpu->kvm_fetch_index = fetch; |
1052 | 0 | cpu->dirty_pages += count; |
1053 | |
|
1054 | 0 | return count; |
1055 | 0 | } |
1056 | | |
1057 | | /* Must be with slots_lock held */ |
1058 | | static uint64_t kvm_dirty_ring_reap_locked(KVMState *s, CPUState* cpu) |
1059 | 0 | { |
1060 | 0 | int ret; |
1061 | 0 | uint64_t total = 0; |
1062 | 0 | int64_t stamp; |
1063 | |
|
1064 | 0 | stamp = get_clock(); |
1065 | |
|
1066 | 0 | if (cpu) { |
1067 | 0 | total = kvm_dirty_ring_reap_one(s, cpu); |
1068 | 0 | } else { |
1069 | 0 | CPU_FOREACH(cpu) { |
1070 | 0 | total += kvm_dirty_ring_reap_one(s, cpu); |
1071 | 0 | } |
1072 | 0 | } |
1073 | |
|
1074 | 0 | if (total) { |
1075 | 0 | ret = kvm_vm_ioctl(s, KVM_RESET_DIRTY_RINGS); |
1076 | 0 | assert(ret == total); |
1077 | 0 | } |
1078 | |
|
1079 | 0 | stamp = get_clock() - stamp; |
1080 | |
|
1081 | 0 | if (total) { |
1082 | 0 | trace_kvm_dirty_ring_reap(total, stamp / 1000); |
1083 | 0 | } |
1084 | |
|
1085 | 0 | return total; |
1086 | 0 | } |
1087 | | |
1088 | | /* |
1089 | | * Currently for simplicity, we must hold BQL before calling this. We can |
1090 | | * consider to drop the BQL if we're clear with all the race conditions. |
1091 | | */ |
1092 | | static uint64_t kvm_dirty_ring_reap(KVMState *s, CPUState *cpu) |
1093 | 0 | { |
1094 | 0 | uint64_t total; |
1095 | | |
1096 | | /* |
1097 | | * We need to lock all kvm slots for all address spaces here, |
1098 | | * because: |
1099 | | * |
1100 | | * (1) We need to mark dirty for dirty bitmaps in multiple slots |
1101 | | * and for tons of pages, so it's better to take the lock here |
1102 | | * once rather than once per page. And more importantly, |
1103 | | * |
1104 | | * (2) We must _NOT_ publish dirty bits to the other threads |
1105 | | * (e.g., the migration thread) via the kvm memory slot dirty |
1106 | | * bitmaps before correctly re-protect those dirtied pages. |
1107 | | * Otherwise we can have potential risk of data corruption if |
1108 | | * the page data is read in the other thread before we do |
1109 | | * reset below. |
1110 | | */ |
1111 | 0 | kvm_slots_lock(); |
1112 | 0 | total = kvm_dirty_ring_reap_locked(s, cpu); |
1113 | 0 | kvm_slots_unlock(); |
1114 | |
|
1115 | 0 | return total; |
1116 | 0 | } |
1117 | | |
1118 | | static void do_kvm_cpu_synchronize_kick(CPUState *cpu, run_on_cpu_data arg) |
1119 | 0 | { |
1120 | | /* No need to do anything */ |
1121 | 0 | } |
1122 | | |
1123 | | /* |
1124 | | * Kick all vcpus out in a synchronized way. When returned, we |
1125 | | * guarantee that every vcpu has been kicked and at least returned to |
1126 | | * userspace once. |
1127 | | */ |
1128 | | static void kvm_cpu_synchronize_kick_all(void) |
1129 | 0 | { |
1130 | 0 | CPUState *cpu; |
1131 | |
|
1132 | 0 | CPU_FOREACH(cpu) { |
1133 | 0 | run_on_cpu(cpu, do_kvm_cpu_synchronize_kick, RUN_ON_CPU_NULL); |
1134 | 0 | } |
1135 | 0 | } |
1136 | | |
1137 | | /* |
1138 | | * Flush all the existing dirty pages to the KVM slot buffers. When |
1139 | | * this call returns, we guarantee that all the touched dirty pages |
1140 | | * before calling this function have been put into the per-kvmslot |
1141 | | * dirty bitmap. |
1142 | | * |
1143 | | * This function must be called with BQL held. |
1144 | | */ |
1145 | | static void kvm_dirty_ring_flush(void) |
1146 | 0 | { |
1147 | 0 | trace_kvm_dirty_ring_flush(0); |
1148 | | /* |
1149 | | * The function needs to be serialized. Since this function |
1150 | | * should always be with BQL held, serialization is guaranteed. |
1151 | | * However, let's be sure of it. |
1152 | | */ |
1153 | 0 | assert(bql_locked()); |
1154 | | /* |
1155 | | * First make sure to flush the hardware buffers by kicking all |
1156 | | * vcpus out in a synchronous way. |
1157 | | */ |
1158 | 0 | kvm_cpu_synchronize_kick_all(); |
1159 | 0 | kvm_dirty_ring_reap(kvm_state, NULL); |
1160 | 0 | trace_kvm_dirty_ring_flush(1); |
1161 | 0 | } |
1162 | | |
1163 | | /** |
1164 | | * kvm_physical_sync_dirty_bitmap - Sync dirty bitmap from kernel space |
1165 | | * |
1166 | | * This function will first try to fetch dirty bitmap from the kernel, |
1167 | | * and then updates qemu's dirty bitmap. |
1168 | | * |
1169 | | * NOTE: caller must be with kml->slots_lock held. |
1170 | | * |
1171 | | * @kml: the KVM memory listener object |
1172 | | * @section: the memory section to sync the dirty bitmap with |
1173 | | */ |
1174 | | static void kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml, |
1175 | | MemoryRegionSection *section) |
1176 | 0 | { |
1177 | 0 | KVMState *s = kvm_state; |
1178 | 0 | KVMSlot *mem; |
1179 | 0 | hwaddr start_addr, size; |
1180 | 0 | hwaddr slot_size; |
1181 | |
|
1182 | 0 | size = kvm_align_section(section, &start_addr); |
1183 | 0 | while (size) { |
1184 | 0 | slot_size = MIN(kvm_max_slot_size, size); |
1185 | 0 | mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); |
1186 | 0 | if (!mem) { |
1187 | | /* We don't have a slot if we want to trap every access. */ |
1188 | 0 | return; |
1189 | 0 | } |
1190 | 0 | if (kvm_slot_get_dirty_log(s, mem)) { |
1191 | 0 | kvm_slot_sync_dirty_pages(mem); |
1192 | 0 | } |
1193 | 0 | start_addr += slot_size; |
1194 | 0 | size -= slot_size; |
1195 | 0 | } |
1196 | 0 | } |
1197 | | |
1198 | | /* Alignment requirement for KVM_CLEAR_DIRTY_LOG - 64 pages */ |
1199 | 0 | #define KVM_CLEAR_LOG_SHIFT 6 |
1200 | 0 | #define KVM_CLEAR_LOG_ALIGN (qemu_real_host_page_size() << KVM_CLEAR_LOG_SHIFT) |
1201 | 0 | #define KVM_CLEAR_LOG_MASK (-KVM_CLEAR_LOG_ALIGN) |
1202 | | |
1203 | | static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start, |
1204 | | uint64_t size) |
1205 | 0 | { |
1206 | 0 | KVMState *s = kvm_state; |
1207 | 0 | uint64_t end, bmap_start, start_delta, bmap_npages; |
1208 | 0 | struct kvm_clear_dirty_log d; |
1209 | 0 | unsigned long *bmap_clear = NULL, psize = qemu_real_host_page_size(); |
1210 | 0 | int ret; |
1211 | | |
1212 | | /* |
1213 | | * We need to extend either the start or the size or both to |
1214 | | * satisfy the KVM interface requirement. Firstly, do the start |
1215 | | * page alignment on 64 host pages |
1216 | | */ |
1217 | 0 | bmap_start = start & KVM_CLEAR_LOG_MASK; |
1218 | 0 | start_delta = start - bmap_start; |
1219 | 0 | bmap_start /= psize; |
1220 | | |
1221 | | /* |
1222 | | * The kernel interface has restriction on the size too, that either: |
1223 | | * |
1224 | | * (1) the size is 64 host pages aligned (just like the start), or |
1225 | | * (2) the size fills up until the end of the KVM memslot. |
1226 | | */ |
1227 | 0 | bmap_npages = DIV_ROUND_UP(size + start_delta, KVM_CLEAR_LOG_ALIGN) |
1228 | 0 | << KVM_CLEAR_LOG_SHIFT; |
1229 | 0 | end = mem->memory_size / psize; |
1230 | 0 | if (bmap_npages > end - bmap_start) { |
1231 | 0 | bmap_npages = end - bmap_start; |
1232 | 0 | } |
1233 | 0 | start_delta /= psize; |
1234 | | |
1235 | | /* |
1236 | | * Prepare the bitmap to clear dirty bits. Here we must guarantee |
1237 | | * that we won't clear any unknown dirty bits otherwise we might |
1238 | | * accidentally clear some set bits which are not yet synced from |
1239 | | * the kernel into QEMU's bitmap, then we'll lose track of the |
1240 | | * guest modifications upon those pages (which can directly lead |
1241 | | * to guest data loss or panic after migration). |
1242 | | * |
1243 | | * Layout of the KVMSlot.dirty_bmap: |
1244 | | * |
1245 | | * |<-------- bmap_npages -----------..>| |
1246 | | * [1] |
1247 | | * start_delta size |
1248 | | * |----------------|-------------|------------------|------------| |
1249 | | * ^ ^ ^ ^ |
1250 | | * | | | | |
1251 | | * start bmap_start (start) end |
1252 | | * of memslot of memslot |
1253 | | * |
1254 | | * [1] bmap_npages can be aligned to either 64 pages or the end of slot |
1255 | | */ |
1256 | |
|
1257 | 0 | assert(bmap_start % BITS_PER_LONG == 0); |
1258 | | /* We should never do log_clear before log_sync */ |
1259 | 0 | assert(mem->dirty_bmap); |
1260 | 0 | if (start_delta || bmap_npages - size / psize) { |
1261 | | /* Slow path - we need to manipulate a temp bitmap */ |
1262 | 0 | bmap_clear = bitmap_new(bmap_npages); |
1263 | 0 | bitmap_copy_with_src_offset(bmap_clear, mem->dirty_bmap, |
1264 | 0 | bmap_start, start_delta + size / psize); |
1265 | | /* |
1266 | | * We need to fill the holes at start because that was not |
1267 | | * specified by the caller and we extended the bitmap only for |
1268 | | * 64 pages alignment |
1269 | | */ |
1270 | 0 | bitmap_clear(bmap_clear, 0, start_delta); |
1271 | 0 | d.dirty_bitmap = bmap_clear; |
1272 | 0 | } else { |
1273 | | /* |
1274 | | * Fast path - both start and size align well with BITS_PER_LONG |
1275 | | * (or the end of memory slot) |
1276 | | */ |
1277 | 0 | d.dirty_bitmap = mem->dirty_bmap + BIT_WORD(bmap_start); |
1278 | 0 | } |
1279 | |
|
1280 | 0 | d.first_page = bmap_start; |
1281 | | /* It should never overflow. If it happens, say something */ |
1282 | 0 | assert(bmap_npages <= UINT32_MAX); |
1283 | 0 | d.num_pages = bmap_npages; |
1284 | 0 | d.slot = mem->slot | (as_id << 16); |
1285 | |
|
1286 | 0 | ret = kvm_vm_ioctl(s, KVM_CLEAR_DIRTY_LOG, &d); |
1287 | 0 | if (ret < 0 && ret != -ENOENT) { |
1288 | 0 | error_report("%s: KVM_CLEAR_DIRTY_LOG failed, slot=%d, " |
1289 | 0 | "start=0x%"PRIx64", size=0x%"PRIx32", errno=%d", |
1290 | 0 | __func__, d.slot, (uint64_t)d.first_page, |
1291 | 0 | (uint32_t)d.num_pages, ret); |
1292 | 0 | } else { |
1293 | 0 | ret = 0; |
1294 | 0 | trace_kvm_clear_dirty_log(d.slot, d.first_page, d.num_pages); |
1295 | 0 | } |
1296 | | |
1297 | | /* |
1298 | | * After we have updated the remote dirty bitmap, we update the |
1299 | | * cached bitmap as well for the memslot, then if another user |
1300 | | * clears the same region we know we shouldn't clear it again on |
1301 | | * the remote otherwise it's data loss as well. |
1302 | | */ |
1303 | 0 | bitmap_clear(mem->dirty_bmap, bmap_start + start_delta, |
1304 | 0 | size / psize); |
1305 | | /* This handles the NULL case well */ |
1306 | 0 | g_free(bmap_clear); |
1307 | 0 | return ret; |
1308 | 0 | } |
1309 | | |
1310 | | |
1311 | | /** |
1312 | | * kvm_physical_log_clear - Clear the kernel's dirty bitmap for range |
1313 | | * |
1314 | | * NOTE: this will be a no-op if we haven't enabled manual dirty log |
1315 | | * protection in the host kernel because in that case this operation |
1316 | | * will be done within log_sync(). |
1317 | | * |
1318 | | * @kml: the kvm memory listener |
1319 | | * @section: the memory range to clear dirty bitmap |
1320 | | */ |
1321 | | static int kvm_physical_log_clear(KVMMemoryListener *kml, |
1322 | | MemoryRegionSection *section) |
1323 | 0 | { |
1324 | 0 | KVMState *s = kvm_state; |
1325 | 0 | uint64_t start, size, offset, count; |
1326 | 0 | KVMSlot *mem; |
1327 | 0 | int ret = 0, i; |
1328 | |
|
1329 | 0 | if (!s->manual_dirty_log_protect) { |
1330 | | /* No need to do explicit clear */ |
1331 | 0 | return ret; |
1332 | 0 | } |
1333 | | |
1334 | 0 | start = section->offset_within_address_space; |
1335 | 0 | size = int128_get64(section->size); |
1336 | |
|
1337 | 0 | if (!size) { |
1338 | | /* Nothing more we can do... */ |
1339 | 0 | return ret; |
1340 | 0 | } |
1341 | | |
1342 | 0 | kvm_slots_lock(); |
1343 | |
|
1344 | 0 | for (i = 0; i < kml->nr_slots_allocated; i++) { |
1345 | 0 | mem = &kml->slots[i]; |
1346 | | /* Discard slots that are empty or do not overlap the section */ |
1347 | 0 | if (!mem->memory_size || |
1348 | 0 | mem->start_addr > start + size - 1 || |
1349 | 0 | start > mem->start_addr + mem->memory_size - 1) { |
1350 | 0 | continue; |
1351 | 0 | } |
1352 | | |
1353 | 0 | if (start >= mem->start_addr) { |
1354 | | /* The slot starts before section or is aligned to it. */ |
1355 | 0 | offset = start - mem->start_addr; |
1356 | 0 | count = MIN(mem->memory_size - offset, size); |
1357 | 0 | } else { |
1358 | | /* The slot starts after section. */ |
1359 | 0 | offset = 0; |
1360 | 0 | count = MIN(mem->memory_size, size - (mem->start_addr - start)); |
1361 | 0 | } |
1362 | 0 | ret = kvm_log_clear_one_slot(mem, kml->as_id, offset, count); |
1363 | 0 | if (ret < 0) { |
1364 | 0 | break; |
1365 | 0 | } |
1366 | 0 | } |
1367 | |
|
1368 | 0 | kvm_slots_unlock(); |
1369 | |
|
1370 | 0 | return ret; |
1371 | 0 | } |
1372 | | |
1373 | | static void kvm_coalesce_mmio_region(MemoryListener *listener, |
1374 | | MemoryRegionSection *secion, |
1375 | | hwaddr start, hwaddr size) |
1376 | 0 | { |
1377 | 0 | KVMState *s = kvm_state; |
1378 | |
|
1379 | 0 | if (s->coalesced_mmio) { |
1380 | 0 | struct kvm_coalesced_mmio_zone zone; |
1381 | |
|
1382 | 0 | zone.addr = start; |
1383 | 0 | zone.size = size; |
1384 | 0 | zone.pad = 0; |
1385 | |
|
1386 | 0 | (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone); |
1387 | 0 | } |
1388 | 0 | } |
1389 | | |
1390 | | static void kvm_uncoalesce_mmio_region(MemoryListener *listener, |
1391 | | MemoryRegionSection *secion, |
1392 | | hwaddr start, hwaddr size) |
1393 | 0 | { |
1394 | 0 | KVMState *s = kvm_state; |
1395 | |
|
1396 | 0 | if (s->coalesced_mmio) { |
1397 | 0 | struct kvm_coalesced_mmio_zone zone; |
1398 | |
|
1399 | 0 | zone.addr = start; |
1400 | 0 | zone.size = size; |
1401 | 0 | zone.pad = 0; |
1402 | |
|
1403 | 0 | (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone); |
1404 | 0 | } |
1405 | 0 | } |
1406 | | |
1407 | | static void kvm_coalesce_pio_add(MemoryListener *listener, |
1408 | | MemoryRegionSection *section, |
1409 | | hwaddr start, hwaddr size) |
1410 | 0 | { |
1411 | 0 | KVMState *s = kvm_state; |
1412 | |
|
1413 | 0 | if (s->coalesced_pio) { |
1414 | 0 | struct kvm_coalesced_mmio_zone zone; |
1415 | |
|
1416 | 0 | zone.addr = start; |
1417 | 0 | zone.size = size; |
1418 | 0 | zone.pio = 1; |
1419 | |
|
1420 | 0 | (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone); |
1421 | 0 | } |
1422 | 0 | } |
1423 | | |
1424 | | static void kvm_coalesce_pio_del(MemoryListener *listener, |
1425 | | MemoryRegionSection *section, |
1426 | | hwaddr start, hwaddr size) |
1427 | 0 | { |
1428 | 0 | KVMState *s = kvm_state; |
1429 | |
|
1430 | 0 | if (s->coalesced_pio) { |
1431 | 0 | struct kvm_coalesced_mmio_zone zone; |
1432 | |
|
1433 | 0 | zone.addr = start; |
1434 | 0 | zone.size = size; |
1435 | 0 | zone.pio = 1; |
1436 | |
|
1437 | 0 | (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone); |
1438 | 0 | } |
1439 | 0 | } |
1440 | | |
1441 | | int kvm_check_extension(KVMState *s, unsigned int extension) |
1442 | 0 | { |
1443 | 0 | int ret; |
1444 | |
|
1445 | 0 | ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension); |
1446 | 0 | if (ret < 0) { |
1447 | 0 | ret = 0; |
1448 | 0 | } |
1449 | |
|
1450 | 0 | return ret; |
1451 | 0 | } |
1452 | | |
1453 | | int kvm_vm_check_extension(KVMState *s, unsigned int extension) |
1454 | 0 | { |
1455 | 0 | int ret; |
1456 | |
|
1457 | 0 | ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension); |
1458 | 0 | if (ret < 0) { |
1459 | | /* VM wide version not implemented, use global one instead */ |
1460 | 0 | ret = kvm_check_extension(s, extension); |
1461 | 0 | } |
1462 | |
|
1463 | 0 | return ret; |
1464 | 0 | } |
1465 | | |
1466 | | /* |
1467 | | * We track the poisoned pages to be able to: |
1468 | | * - replace them on VM reset |
1469 | | * - block a migration for a VM with a poisoned page |
1470 | | */ |
1471 | | typedef struct HWPoisonPage { |
1472 | | ram_addr_t ram_addr; |
1473 | | QLIST_ENTRY(HWPoisonPage) list; |
1474 | | } HWPoisonPage; |
1475 | | |
1476 | | static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list = |
1477 | | QLIST_HEAD_INITIALIZER(hwpoison_page_list); |
1478 | | |
1479 | | static void kvm_unpoison_all(void *param) |
1480 | 0 | { |
1481 | 0 | HWPoisonPage *page, *next_page; |
1482 | |
|
1483 | 0 | QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) { |
1484 | 0 | QLIST_REMOVE(page, list); |
1485 | 0 | qemu_ram_remap(page->ram_addr); |
1486 | 0 | g_free(page); |
1487 | 0 | } |
1488 | 0 | } |
1489 | | |
1490 | | void kvm_hwpoison_page_add(ram_addr_t ram_addr) |
1491 | 0 | { |
1492 | 0 | HWPoisonPage *page; |
1493 | |
|
1494 | 0 | QLIST_FOREACH(page, &hwpoison_page_list, list) { |
1495 | 0 | if (page->ram_addr == ram_addr) { |
1496 | 0 | return; |
1497 | 0 | } |
1498 | 0 | } |
1499 | 0 | page = g_new(HWPoisonPage, 1); |
1500 | 0 | page->ram_addr = ram_addr; |
1501 | 0 | QLIST_INSERT_HEAD(&hwpoison_page_list, page, list); |
1502 | 0 | } |
1503 | | |
1504 | | bool kvm_hwpoisoned_mem(void) |
1505 | 0 | { |
1506 | 0 | return !QLIST_EMPTY(&hwpoison_page_list); |
1507 | 0 | } |
1508 | | |
1509 | | static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size) |
1510 | 0 | { |
1511 | 0 | if (target_needs_bswap()) { |
1512 | | /* |
1513 | | * The kernel expects ioeventfd values in HOST_BIG_ENDIAN |
1514 | | * endianness, but the memory core hands them in target endianness. |
1515 | | * For example, PPC is always treated as big-endian even if running |
1516 | | * on KVM and on PPC64LE. Correct here, swapping back. |
1517 | | */ |
1518 | 0 | switch (size) { |
1519 | 0 | case 2: |
1520 | 0 | val = bswap16(val); |
1521 | 0 | break; |
1522 | 0 | case 4: |
1523 | 0 | val = bswap32(val); |
1524 | 0 | break; |
1525 | 0 | } |
1526 | 0 | } |
1527 | 0 | return val; |
1528 | 0 | } |
1529 | | |
1530 | | static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val, |
1531 | | bool assign, uint32_t size, bool datamatch) |
1532 | 0 | { |
1533 | 0 | int ret; |
1534 | 0 | struct kvm_ioeventfd iofd = { |
1535 | 0 | .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, |
1536 | 0 | .addr = addr, |
1537 | 0 | .len = size, |
1538 | 0 | .flags = 0, |
1539 | 0 | .fd = fd, |
1540 | 0 | }; |
1541 | |
|
1542 | 0 | trace_kvm_set_ioeventfd_mmio(fd, (uint64_t)addr, val, assign, size, |
1543 | 0 | datamatch); |
1544 | 0 | if (!kvm_enabled()) { |
1545 | 0 | return -ENOSYS; |
1546 | 0 | } |
1547 | | |
1548 | 0 | if (datamatch) { |
1549 | 0 | iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; |
1550 | 0 | } |
1551 | 0 | if (!assign) { |
1552 | 0 | iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; |
1553 | 0 | } |
1554 | |
|
1555 | 0 | ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd); |
1556 | |
|
1557 | 0 | if (ret < 0) { |
1558 | 0 | return -errno; |
1559 | 0 | } |
1560 | | |
1561 | 0 | return 0; |
1562 | 0 | } |
1563 | | |
1564 | | static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val, |
1565 | | bool assign, uint32_t size, bool datamatch) |
1566 | 0 | { |
1567 | 0 | struct kvm_ioeventfd kick = { |
1568 | 0 | .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, |
1569 | 0 | .addr = addr, |
1570 | 0 | .flags = KVM_IOEVENTFD_FLAG_PIO, |
1571 | 0 | .len = size, |
1572 | 0 | .fd = fd, |
1573 | 0 | }; |
1574 | 0 | int r; |
1575 | 0 | trace_kvm_set_ioeventfd_pio(fd, addr, val, assign, size, datamatch); |
1576 | 0 | if (!kvm_enabled()) { |
1577 | 0 | return -ENOSYS; |
1578 | 0 | } |
1579 | 0 | if (datamatch) { |
1580 | 0 | kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; |
1581 | 0 | } |
1582 | 0 | if (!assign) { |
1583 | 0 | kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; |
1584 | 0 | } |
1585 | 0 | r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick); |
1586 | 0 | if (r < 0) { |
1587 | 0 | return r; |
1588 | 0 | } |
1589 | 0 | return 0; |
1590 | 0 | } |
1591 | | |
1592 | | |
1593 | | static const KVMCapabilityInfo * |
1594 | | kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list) |
1595 | 0 | { |
1596 | 0 | while (list->name) { |
1597 | 0 | if (!kvm_check_extension(s, list->value)) { |
1598 | 0 | return list; |
1599 | 0 | } |
1600 | 0 | list++; |
1601 | 0 | } |
1602 | 0 | return NULL; |
1603 | 0 | } |
1604 | | |
1605 | | void kvm_set_max_memslot_size(hwaddr max_slot_size) |
1606 | 0 | { |
1607 | 0 | g_assert( |
1608 | 0 | ROUND_UP(max_slot_size, qemu_real_host_page_size()) == max_slot_size |
1609 | 0 | ); |
1610 | 0 | kvm_max_slot_size = max_slot_size; |
1611 | 0 | } |
1612 | | |
1613 | | static int kvm_set_memory_attributes(hwaddr start, uint64_t size, uint64_t attr) |
1614 | 0 | { |
1615 | 0 | struct kvm_memory_attributes attrs; |
1616 | 0 | int r; |
1617 | |
|
1618 | 0 | assert((attr & kvm_supported_memory_attributes) == attr); |
1619 | 0 | attrs.attributes = attr; |
1620 | 0 | attrs.address = start; |
1621 | 0 | attrs.size = size; |
1622 | 0 | attrs.flags = 0; |
1623 | |
|
1624 | 0 | r = kvm_vm_ioctl(kvm_state, KVM_SET_MEMORY_ATTRIBUTES, &attrs); |
1625 | 0 | if (r) { |
1626 | 0 | error_report("failed to set memory (0x%" HWADDR_PRIx "+0x%" PRIx64 ") " |
1627 | 0 | "with attr 0x%" PRIx64 " error '%s'", |
1628 | 0 | start, size, attr, strerror(errno)); |
1629 | 0 | } |
1630 | 0 | return r; |
1631 | 0 | } |
1632 | | |
1633 | | int kvm_set_memory_attributes_private(hwaddr start, uint64_t size) |
1634 | 0 | { |
1635 | 0 | return kvm_set_memory_attributes(start, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); |
1636 | 0 | } |
1637 | | |
1638 | | int kvm_set_memory_attributes_shared(hwaddr start, uint64_t size) |
1639 | 0 | { |
1640 | 0 | return kvm_set_memory_attributes(start, size, 0); |
1641 | 0 | } |
1642 | | |
1643 | | bool kvm_private_memory_attribute_supported(void) |
1644 | 0 | { |
1645 | 0 | return !!(kvm_supported_memory_attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE); |
1646 | 0 | } |
1647 | | |
1648 | | /* Called with KVMMemoryListener.slots_lock held */ |
1649 | | static void kvm_set_phys_mem(KVMMemoryListener *kml, |
1650 | | MemoryRegionSection *section, bool add) |
1651 | 0 | { |
1652 | 0 | KVMSlot *mem; |
1653 | 0 | int err; |
1654 | 0 | MemoryRegion *mr = section->mr; |
1655 | 0 | bool writable = !mr->readonly && !mr->rom_device; |
1656 | 0 | hwaddr start_addr, size, slot_size, mr_offset; |
1657 | 0 | ram_addr_t ram_start_offset; |
1658 | 0 | void *ram; |
1659 | |
|
1660 | 0 | if (!memory_region_is_ram(mr)) { |
1661 | 0 | if (writable || !kvm_readonly_mem_allowed) { |
1662 | 0 | return; |
1663 | 0 | } else if (!mr->romd_mode) { |
1664 | | /* If the memory device is not in romd_mode, then we actually want |
1665 | | * to remove the kvm memory slot so all accesses will trap. */ |
1666 | 0 | add = false; |
1667 | 0 | } |
1668 | 0 | } |
1669 | | |
1670 | 0 | size = kvm_align_section(section, &start_addr); |
1671 | 0 | if (!size) { |
1672 | 0 | return; |
1673 | 0 | } |
1674 | | |
1675 | | /* The offset of the kvmslot within the memory region */ |
1676 | 0 | mr_offset = section->offset_within_region + start_addr - |
1677 | 0 | section->offset_within_address_space; |
1678 | | |
1679 | | /* use aligned delta to align the ram address and offset */ |
1680 | 0 | ram = memory_region_get_ram_ptr(mr) + mr_offset; |
1681 | 0 | ram_start_offset = memory_region_get_ram_addr(mr) + mr_offset; |
1682 | |
|
1683 | 0 | if (!add) { |
1684 | 0 | do { |
1685 | 0 | slot_size = MIN(kvm_max_slot_size, size); |
1686 | 0 | mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); |
1687 | 0 | if (!mem) { |
1688 | 0 | return; |
1689 | 0 | } |
1690 | 0 | if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { |
1691 | | /* |
1692 | | * NOTE: We should be aware of the fact that here we're only |
1693 | | * doing a best effort to sync dirty bits. No matter whether |
1694 | | * we're using dirty log or dirty ring, we ignored two facts: |
1695 | | * |
1696 | | * (1) dirty bits can reside in hardware buffers (PML) |
1697 | | * |
1698 | | * (2) after we collected dirty bits here, pages can be dirtied |
1699 | | * again before we do the final KVM_SET_USER_MEMORY_REGION to |
1700 | | * remove the slot. |
1701 | | * |
1702 | | * Not easy. Let's cross the fingers until it's fixed. |
1703 | | */ |
1704 | 0 | if (kvm_state->kvm_dirty_ring_size) { |
1705 | 0 | kvm_dirty_ring_reap_locked(kvm_state, NULL); |
1706 | 0 | if (kvm_state->kvm_dirty_ring_with_bitmap) { |
1707 | 0 | kvm_slot_sync_dirty_pages(mem); |
1708 | 0 | kvm_slot_get_dirty_log(kvm_state, mem); |
1709 | 0 | } |
1710 | 0 | } else { |
1711 | 0 | kvm_slot_get_dirty_log(kvm_state, mem); |
1712 | 0 | } |
1713 | 0 | kvm_slot_sync_dirty_pages(mem); |
1714 | 0 | } |
1715 | | |
1716 | | /* unregister the slot */ |
1717 | 0 | g_free(mem->dirty_bmap); |
1718 | 0 | mem->dirty_bmap = NULL; |
1719 | 0 | mem->memory_size = 0; |
1720 | 0 | mem->flags = 0; |
1721 | 0 | err = kvm_set_user_memory_region(kml, mem, false); |
1722 | 0 | if (err) { |
1723 | 0 | fprintf(stderr, "%s: error unregistering slot: %s\n", |
1724 | 0 | __func__, strerror(-err)); |
1725 | 0 | abort(); |
1726 | 0 | } |
1727 | 0 | start_addr += slot_size; |
1728 | 0 | size -= slot_size; |
1729 | 0 | kml->nr_slots_used--; |
1730 | 0 | } while (size); |
1731 | 0 | return; |
1732 | 0 | } |
1733 | | |
1734 | | /* register the new slot */ |
1735 | 0 | do { |
1736 | 0 | slot_size = MIN(kvm_max_slot_size, size); |
1737 | 0 | mem = kvm_alloc_slot(kml); |
1738 | 0 | mem->as_id = kml->as_id; |
1739 | 0 | mem->memory_size = slot_size; |
1740 | 0 | mem->start_addr = start_addr; |
1741 | 0 | mem->ram_start_offset = ram_start_offset; |
1742 | 0 | mem->ram = ram; |
1743 | 0 | mem->flags = kvm_mem_flags(mr); |
1744 | 0 | mem->guest_memfd = mr->ram_block->guest_memfd_private; |
1745 | 0 | mem->guest_memfd_offset = mem->guest_memfd >= 0 ? |
1746 | 0 | (uint8_t*)ram - mr->ram_block->host : 0; |
1747 | |
|
1748 | 0 | kvm_slot_init_dirty_bitmap(mem); |
1749 | 0 | err = kvm_set_user_memory_region(kml, mem, true); |
1750 | 0 | if (err) { |
1751 | 0 | fprintf(stderr, "%s: error registering slot: %s\n", __func__, |
1752 | 0 | strerror(-err)); |
1753 | 0 | abort(); |
1754 | 0 | } |
1755 | | |
1756 | 0 | if (memory_region_has_guest_memfd_private(mr)) { |
1757 | 0 | err = kvm_set_memory_attributes_private(start_addr, slot_size); |
1758 | 0 | if (err) { |
1759 | 0 | error_report("%s: failed to set memory attribute private: %s", |
1760 | 0 | __func__, strerror(-err)); |
1761 | 0 | exit(1); |
1762 | 0 | } |
1763 | 0 | } |
1764 | | |
1765 | 0 | start_addr += slot_size; |
1766 | 0 | ram_start_offset += slot_size; |
1767 | 0 | ram += slot_size; |
1768 | 0 | size -= slot_size; |
1769 | 0 | kml->nr_slots_used++; |
1770 | 0 | } while (size); |
1771 | 0 | } |
1772 | | |
1773 | | static void *kvm_dirty_ring_reaper_thread(void *data) |
1774 | 0 | { |
1775 | 0 | KVMState *s = data; |
1776 | 0 | struct KVMDirtyRingReaper *r = &s->reaper; |
1777 | |
|
1778 | 0 | rcu_register_thread(); |
1779 | |
|
1780 | 0 | trace_kvm_dirty_ring_reaper("init"); |
1781 | |
|
1782 | 0 | while (true) { |
1783 | 0 | r->reaper_state = KVM_DIRTY_RING_REAPER_WAIT; |
1784 | 0 | trace_kvm_dirty_ring_reaper("wait"); |
1785 | | /* |
1786 | | * TODO: provide a smarter timeout rather than a constant? |
1787 | | */ |
1788 | 0 | sleep(1); |
1789 | | |
1790 | | /* keep sleeping so that dirtylimit not be interfered by reaper */ |
1791 | 0 | if (dirtylimit_in_service()) { |
1792 | 0 | continue; |
1793 | 0 | } |
1794 | | |
1795 | 0 | trace_kvm_dirty_ring_reaper("wakeup"); |
1796 | 0 | r->reaper_state = KVM_DIRTY_RING_REAPER_REAPING; |
1797 | |
|
1798 | 0 | bql_lock(); |
1799 | 0 | kvm_dirty_ring_reap(s, NULL); |
1800 | 0 | bql_unlock(); |
1801 | |
|
1802 | 0 | r->reaper_iteration++; |
1803 | 0 | } |
1804 | |
|
1805 | 0 | g_assert_not_reached(); |
1806 | 0 | } |
1807 | | |
1808 | | static void kvm_dirty_ring_reaper_init(KVMState *s) |
1809 | 0 | { |
1810 | 0 | struct KVMDirtyRingReaper *r = &s->reaper; |
1811 | |
|
1812 | 0 | qemu_thread_create(&r->reaper_thr, "kvm-reaper", |
1813 | 0 | kvm_dirty_ring_reaper_thread, |
1814 | 0 | s, QEMU_THREAD_JOINABLE); |
1815 | 0 | } |
1816 | | |
1817 | | static int kvm_dirty_ring_init(KVMState *s) |
1818 | 0 | { |
1819 | 0 | uint32_t ring_size = s->kvm_dirty_ring_size; |
1820 | 0 | uint64_t ring_bytes = ring_size * sizeof(struct kvm_dirty_gfn); |
1821 | 0 | unsigned int capability = KVM_CAP_DIRTY_LOG_RING; |
1822 | 0 | int ret; |
1823 | |
|
1824 | 0 | s->kvm_dirty_ring_size = 0; |
1825 | 0 | s->kvm_dirty_ring_bytes = 0; |
1826 | | |
1827 | | /* Bail if the dirty ring size isn't specified */ |
1828 | 0 | if (!ring_size) { |
1829 | 0 | return 0; |
1830 | 0 | } |
1831 | | |
1832 | | /* |
1833 | | * Read the max supported pages. Fall back to dirty logging mode |
1834 | | * if the dirty ring isn't supported. |
1835 | | */ |
1836 | 0 | ret = kvm_vm_check_extension(s, capability); |
1837 | 0 | if (ret <= 0) { |
1838 | 0 | capability = KVM_CAP_DIRTY_LOG_RING_ACQ_REL; |
1839 | 0 | ret = kvm_vm_check_extension(s, capability); |
1840 | 0 | } |
1841 | |
|
1842 | 0 | if (ret <= 0) { |
1843 | 0 | warn_report("KVM dirty ring not available, using bitmap method"); |
1844 | 0 | return 0; |
1845 | 0 | } |
1846 | | |
1847 | 0 | if (ring_bytes > ret) { |
1848 | 0 | error_report("KVM dirty ring size %" PRIu32 " too big " |
1849 | 0 | "(maximum is %ld). Please use a smaller value.", |
1850 | 0 | ring_size, (long)ret / sizeof(struct kvm_dirty_gfn)); |
1851 | 0 | return -EINVAL; |
1852 | 0 | } |
1853 | | |
1854 | 0 | ret = kvm_vm_enable_cap(s, capability, 0, ring_bytes); |
1855 | 0 | if (ret) { |
1856 | 0 | error_report("Enabling of KVM dirty ring failed: %s. " |
1857 | 0 | "Suggested minimum value is 1024.", strerror(-ret)); |
1858 | 0 | return -EIO; |
1859 | 0 | } |
1860 | | |
1861 | | /* Enable the backup bitmap if it is supported */ |
1862 | 0 | ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP); |
1863 | 0 | if (ret > 0) { |
1864 | 0 | ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP, 0); |
1865 | 0 | if (ret) { |
1866 | 0 | error_report("Enabling of KVM dirty ring's backup bitmap failed: " |
1867 | 0 | "%s. ", strerror(-ret)); |
1868 | 0 | return -EIO; |
1869 | 0 | } |
1870 | | |
1871 | 0 | s->kvm_dirty_ring_with_bitmap = true; |
1872 | 0 | } |
1873 | | |
1874 | 0 | s->kvm_dirty_ring_size = ring_size; |
1875 | 0 | s->kvm_dirty_ring_bytes = ring_bytes; |
1876 | |
|
1877 | 0 | return 0; |
1878 | 0 | } |
1879 | | |
1880 | | static void kvm_region_add(MemoryListener *listener, |
1881 | | MemoryRegionSection *section) |
1882 | 0 | { |
1883 | 0 | KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); |
1884 | 0 | KVMMemoryUpdate *update; |
1885 | |
|
1886 | 0 | update = g_new0(KVMMemoryUpdate, 1); |
1887 | 0 | update->section = *section; |
1888 | |
|
1889 | 0 | QSIMPLEQ_INSERT_TAIL(&kml->transaction_add, update, next); |
1890 | 0 | } |
1891 | | |
1892 | | static void kvm_region_del(MemoryListener *listener, |
1893 | | MemoryRegionSection *section) |
1894 | 0 | { |
1895 | 0 | KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); |
1896 | 0 | KVMMemoryUpdate *update; |
1897 | |
|
1898 | 0 | update = g_new0(KVMMemoryUpdate, 1); |
1899 | 0 | update->section = *section; |
1900 | |
|
1901 | 0 | QSIMPLEQ_INSERT_TAIL(&kml->transaction_del, update, next); |
1902 | 0 | } |
1903 | | |
1904 | | static void kvm_region_commit(MemoryListener *listener) |
1905 | 0 | { |
1906 | 0 | KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, |
1907 | 0 | listener); |
1908 | 0 | KVMMemoryUpdate *u1, *u2; |
1909 | 0 | bool need_inhibit = false; |
1910 | |
|
1911 | 0 | if (QSIMPLEQ_EMPTY(&kml->transaction_add) && |
1912 | 0 | QSIMPLEQ_EMPTY(&kml->transaction_del)) { |
1913 | 0 | return; |
1914 | 0 | } |
1915 | | |
1916 | | /* |
1917 | | * We have to be careful when regions to add overlap with ranges to remove. |
1918 | | * We have to simulate atomic KVM memslot updates by making sure no ioctl() |
1919 | | * is currently active. |
1920 | | * |
1921 | | * The lists are order by addresses, so it's easy to find overlaps. |
1922 | | */ |
1923 | 0 | u1 = QSIMPLEQ_FIRST(&kml->transaction_del); |
1924 | 0 | u2 = QSIMPLEQ_FIRST(&kml->transaction_add); |
1925 | 0 | while (u1 && u2) { |
1926 | 0 | Range r1, r2; |
1927 | |
|
1928 | 0 | range_init_nofail(&r1, u1->section.offset_within_address_space, |
1929 | 0 | int128_get64(u1->section.size)); |
1930 | 0 | range_init_nofail(&r2, u2->section.offset_within_address_space, |
1931 | 0 | int128_get64(u2->section.size)); |
1932 | |
|
1933 | 0 | if (range_overlaps_range(&r1, &r2)) { |
1934 | 0 | need_inhibit = true; |
1935 | 0 | break; |
1936 | 0 | } |
1937 | 0 | if (range_lob(&r1) < range_lob(&r2)) { |
1938 | 0 | u1 = QSIMPLEQ_NEXT(u1, next); |
1939 | 0 | } else { |
1940 | 0 | u2 = QSIMPLEQ_NEXT(u2, next); |
1941 | 0 | } |
1942 | 0 | } |
1943 | |
|
1944 | 0 | kvm_slots_lock(); |
1945 | 0 | if (need_inhibit) { |
1946 | 0 | accel_ioctl_inhibit_begin(); |
1947 | 0 | } |
1948 | | |
1949 | | /* Remove all memslots before adding the new ones. */ |
1950 | 0 | while (!QSIMPLEQ_EMPTY(&kml->transaction_del)) { |
1951 | 0 | u1 = QSIMPLEQ_FIRST(&kml->transaction_del); |
1952 | 0 | QSIMPLEQ_REMOVE_HEAD(&kml->transaction_del, next); |
1953 | |
|
1954 | 0 | kvm_set_phys_mem(kml, &u1->section, false); |
1955 | 0 | memory_region_unref(u1->section.mr); |
1956 | |
|
1957 | 0 | g_free(u1); |
1958 | 0 | } |
1959 | 0 | while (!QSIMPLEQ_EMPTY(&kml->transaction_add)) { |
1960 | 0 | u1 = QSIMPLEQ_FIRST(&kml->transaction_add); |
1961 | 0 | QSIMPLEQ_REMOVE_HEAD(&kml->transaction_add, next); |
1962 | |
|
1963 | 0 | memory_region_ref(u1->section.mr); |
1964 | 0 | kvm_set_phys_mem(kml, &u1->section, true); |
1965 | |
|
1966 | 0 | g_free(u1); |
1967 | 0 | } |
1968 | |
|
1969 | 0 | if (need_inhibit) { |
1970 | 0 | accel_ioctl_inhibit_end(); |
1971 | 0 | } |
1972 | 0 | kvm_slots_unlock(); |
1973 | 0 | } |
1974 | | |
1975 | | static void kvm_log_sync(MemoryListener *listener, |
1976 | | MemoryRegionSection *section) |
1977 | 0 | { |
1978 | 0 | KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); |
1979 | |
|
1980 | 0 | kvm_slots_lock(); |
1981 | 0 | kvm_physical_sync_dirty_bitmap(kml, section); |
1982 | 0 | kvm_slots_unlock(); |
1983 | 0 | } |
1984 | | |
1985 | | static void kvm_log_sync_global(MemoryListener *l, bool last_stage) |
1986 | 0 | { |
1987 | 0 | KVMMemoryListener *kml = container_of(l, KVMMemoryListener, listener); |
1988 | 0 | KVMState *s = kvm_state; |
1989 | 0 | KVMSlot *mem; |
1990 | 0 | int i; |
1991 | | |
1992 | | /* Flush all kernel dirty addresses into KVMSlot dirty bitmap */ |
1993 | 0 | kvm_dirty_ring_flush(); |
1994 | |
|
1995 | 0 | kvm_slots_lock(); |
1996 | 0 | for (i = 0; i < kml->nr_slots_allocated; i++) { |
1997 | 0 | mem = &kml->slots[i]; |
1998 | 0 | if (mem->memory_size && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { |
1999 | 0 | kvm_slot_sync_dirty_pages(mem); |
2000 | |
|
2001 | 0 | if (s->kvm_dirty_ring_with_bitmap && last_stage && |
2002 | 0 | kvm_slot_get_dirty_log(s, mem)) { |
2003 | 0 | kvm_slot_sync_dirty_pages(mem); |
2004 | 0 | } |
2005 | | |
2006 | | /* |
2007 | | * This is not needed by KVM_GET_DIRTY_LOG because the |
2008 | | * ioctl will unconditionally overwrite the whole region. |
2009 | | * However kvm dirty ring has no such side effect. |
2010 | | */ |
2011 | 0 | kvm_slot_reset_dirty_pages(mem); |
2012 | 0 | } |
2013 | 0 | } |
2014 | 0 | kvm_slots_unlock(); |
2015 | 0 | } |
2016 | | |
2017 | | static void kvm_log_clear(MemoryListener *listener, |
2018 | | MemoryRegionSection *section) |
2019 | 0 | { |
2020 | 0 | KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); |
2021 | 0 | int r; |
2022 | |
|
2023 | 0 | r = kvm_physical_log_clear(kml, section); |
2024 | 0 | if (r < 0) { |
2025 | 0 | error_report_once("%s: kvm log clear failed: mr=%s " |
2026 | 0 | "offset=%"HWADDR_PRIx" size=%"PRIx64, __func__, |
2027 | 0 | section->mr->name, section->offset_within_region, |
2028 | 0 | int128_get64(section->size)); |
2029 | 0 | abort(); |
2030 | 0 | } |
2031 | 0 | } |
2032 | | |
2033 | | static void kvm_mem_ioeventfd_add(MemoryListener *listener, |
2034 | | MemoryRegionSection *section, |
2035 | | bool match_data, uint64_t data, |
2036 | | EventNotifier *e) |
2037 | 0 | { |
2038 | 0 | int fd = event_notifier_get_fd(e); |
2039 | 0 | int r; |
2040 | |
|
2041 | 0 | r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, |
2042 | 0 | data, true, int128_get64(section->size), |
2043 | 0 | match_data); |
2044 | 0 | if (r < 0) { |
2045 | 0 | fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n", |
2046 | 0 | __func__, strerror(-r), -r); |
2047 | 0 | abort(); |
2048 | 0 | } |
2049 | 0 | } |
2050 | | |
2051 | | static void kvm_mem_ioeventfd_del(MemoryListener *listener, |
2052 | | MemoryRegionSection *section, |
2053 | | bool match_data, uint64_t data, |
2054 | | EventNotifier *e) |
2055 | 0 | { |
2056 | 0 | int fd = event_notifier_get_fd(e); |
2057 | 0 | int r; |
2058 | |
|
2059 | 0 | r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, |
2060 | 0 | data, false, int128_get64(section->size), |
2061 | 0 | match_data); |
2062 | 0 | if (r < 0) { |
2063 | 0 | fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n", |
2064 | 0 | __func__, strerror(-r), -r); |
2065 | 0 | abort(); |
2066 | 0 | } |
2067 | 0 | } |
2068 | | |
2069 | | static void kvm_io_ioeventfd_add(MemoryListener *listener, |
2070 | | MemoryRegionSection *section, |
2071 | | bool match_data, uint64_t data, |
2072 | | EventNotifier *e) |
2073 | 0 | { |
2074 | 0 | int fd = event_notifier_get_fd(e); |
2075 | 0 | int r; |
2076 | |
|
2077 | 0 | r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, |
2078 | 0 | data, true, int128_get64(section->size), |
2079 | 0 | match_data); |
2080 | 0 | if (r < 0) { |
2081 | 0 | fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n", |
2082 | 0 | __func__, strerror(-r), -r); |
2083 | 0 | abort(); |
2084 | 0 | } |
2085 | 0 | } |
2086 | | |
2087 | | static void kvm_io_ioeventfd_del(MemoryListener *listener, |
2088 | | MemoryRegionSection *section, |
2089 | | bool match_data, uint64_t data, |
2090 | | EventNotifier *e) |
2091 | | |
2092 | 0 | { |
2093 | 0 | int fd = event_notifier_get_fd(e); |
2094 | 0 | int r; |
2095 | |
|
2096 | 0 | r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, |
2097 | 0 | data, false, int128_get64(section->size), |
2098 | 0 | match_data); |
2099 | 0 | if (r < 0) { |
2100 | 0 | fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n", |
2101 | 0 | __func__, strerror(-r), -r); |
2102 | 0 | abort(); |
2103 | 0 | } |
2104 | 0 | } |
2105 | | |
2106 | | void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, |
2107 | | AddressSpace *as, int as_id, const char *name) |
2108 | 0 | { |
2109 | 0 | int i; |
2110 | |
|
2111 | 0 | kml->as_id = as_id; |
2112 | |
|
2113 | 0 | kvm_slots_grow(kml, KVM_MEMSLOTS_NR_ALLOC_DEFAULT); |
2114 | |
|
2115 | 0 | QSIMPLEQ_INIT(&kml->transaction_add); |
2116 | 0 | QSIMPLEQ_INIT(&kml->transaction_del); |
2117 | |
|
2118 | 0 | kml->listener.region_add = kvm_region_add; |
2119 | 0 | kml->listener.region_del = kvm_region_del; |
2120 | 0 | kml->listener.commit = kvm_region_commit; |
2121 | 0 | kml->listener.log_start = kvm_log_start; |
2122 | 0 | kml->listener.log_stop = kvm_log_stop; |
2123 | 0 | kml->listener.priority = MEMORY_LISTENER_PRIORITY_ACCEL; |
2124 | 0 | kml->listener.name = name; |
2125 | |
|
2126 | 0 | if (s->kvm_dirty_ring_size) { |
2127 | 0 | kml->listener.log_sync_global = kvm_log_sync_global; |
2128 | 0 | } else { |
2129 | 0 | kml->listener.log_sync = kvm_log_sync; |
2130 | 0 | kml->listener.log_clear = kvm_log_clear; |
2131 | 0 | } |
2132 | |
|
2133 | 0 | memory_listener_register(&kml->listener, as); |
2134 | |
|
2135 | 0 | for (i = 0; i < s->nr_as; ++i) { |
2136 | 0 | if (!s->as[i].as) { |
2137 | 0 | s->as[i].as = as; |
2138 | 0 | s->as[i].ml = kml; |
2139 | 0 | break; |
2140 | 0 | } |
2141 | 0 | } |
2142 | 0 | } |
2143 | | |
2144 | | static MemoryListener kvm_io_listener = { |
2145 | | .name = "kvm-io", |
2146 | | .coalesced_io_add = kvm_coalesce_pio_add, |
2147 | | .coalesced_io_del = kvm_coalesce_pio_del, |
2148 | | .eventfd_add = kvm_io_ioeventfd_add, |
2149 | | .eventfd_del = kvm_io_ioeventfd_del, |
2150 | | .priority = MEMORY_LISTENER_PRIORITY_DEV_BACKEND, |
2151 | | }; |
2152 | | |
2153 | | int kvm_set_irq(KVMState *s, int irq, int level) |
2154 | 0 | { |
2155 | 0 | struct kvm_irq_level event; |
2156 | 0 | int ret; |
2157 | |
|
2158 | 0 | assert(kvm_async_interrupts_enabled()); |
2159 | |
|
2160 | 0 | event.level = level; |
2161 | 0 | event.irq = irq; |
2162 | 0 | ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event); |
2163 | 0 | if (ret < 0) { |
2164 | 0 | perror("kvm_set_irq"); |
2165 | 0 | abort(); |
2166 | 0 | } |
2167 | | |
2168 | 0 | return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status; |
2169 | 0 | } |
2170 | | |
2171 | | #ifdef KVM_CAP_IRQ_ROUTING |
2172 | | typedef struct KVMMSIRoute { |
2173 | | struct kvm_irq_routing_entry kroute; |
2174 | | QTAILQ_ENTRY(KVMMSIRoute) entry; |
2175 | | } KVMMSIRoute; |
2176 | | |
2177 | | static void set_gsi(KVMState *s, unsigned int gsi) |
2178 | 0 | { |
2179 | 0 | set_bit(gsi, s->used_gsi_bitmap); |
2180 | 0 | } |
2181 | | |
2182 | | static void clear_gsi(KVMState *s, unsigned int gsi) |
2183 | 0 | { |
2184 | 0 | clear_bit(gsi, s->used_gsi_bitmap); |
2185 | 0 | } |
2186 | | |
2187 | | void kvm_init_irq_routing(KVMState *s) |
2188 | 0 | { |
2189 | 0 | int gsi_count; |
2190 | |
|
2191 | 0 | gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1; |
2192 | 0 | if (gsi_count > 0) { |
2193 | | /* Round up so we can search ints using ffs */ |
2194 | 0 | s->used_gsi_bitmap = bitmap_new(gsi_count); |
2195 | 0 | s->gsi_count = gsi_count; |
2196 | 0 | } |
2197 | |
|
2198 | 0 | s->irq_routes = g_malloc0(sizeof(*s->irq_routes)); |
2199 | 0 | s->nr_allocated_irq_routes = 0; |
2200 | |
|
2201 | 0 | kvm_arch_init_irq_routing(s); |
2202 | 0 | } |
2203 | | |
2204 | | void kvm_irqchip_commit_routes(KVMState *s) |
2205 | 0 | { |
2206 | 0 | int ret; |
2207 | |
|
2208 | 0 | if (kvm_gsi_direct_mapping()) { |
2209 | 0 | return; |
2210 | 0 | } |
2211 | | |
2212 | 0 | if (!kvm_gsi_routing_enabled()) { |
2213 | 0 | return; |
2214 | 0 | } |
2215 | | |
2216 | 0 | s->irq_routes->flags = 0; |
2217 | 0 | trace_kvm_irqchip_commit_routes(); |
2218 | 0 | ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes); |
2219 | 0 | assert(ret == 0); |
2220 | 0 | } |
2221 | | |
2222 | | void kvm_add_routing_entry(KVMState *s, |
2223 | | struct kvm_irq_routing_entry *entry) |
2224 | 0 | { |
2225 | 0 | struct kvm_irq_routing_entry *new; |
2226 | 0 | int n, size; |
2227 | |
|
2228 | 0 | if (s->irq_routes->nr == s->nr_allocated_irq_routes) { |
2229 | 0 | n = s->nr_allocated_irq_routes * 2; |
2230 | 0 | if (n < 64) { |
2231 | 0 | n = 64; |
2232 | 0 | } |
2233 | 0 | size = sizeof(struct kvm_irq_routing); |
2234 | 0 | size += n * sizeof(*new); |
2235 | 0 | s->irq_routes = g_realloc(s->irq_routes, size); |
2236 | 0 | s->nr_allocated_irq_routes = n; |
2237 | 0 | } |
2238 | 0 | n = s->irq_routes->nr++; |
2239 | 0 | new = &s->irq_routes->entries[n]; |
2240 | |
|
2241 | 0 | *new = *entry; |
2242 | |
|
2243 | 0 | set_gsi(s, entry->gsi); |
2244 | 0 | } |
2245 | | |
2246 | | static int kvm_update_routing_entry(KVMState *s, |
2247 | | struct kvm_irq_routing_entry *new_entry) |
2248 | 0 | { |
2249 | 0 | struct kvm_irq_routing_entry *entry; |
2250 | 0 | int n; |
2251 | |
|
2252 | 0 | for (n = 0; n < s->irq_routes->nr; n++) { |
2253 | 0 | entry = &s->irq_routes->entries[n]; |
2254 | 0 | if (entry->gsi != new_entry->gsi) { |
2255 | 0 | continue; |
2256 | 0 | } |
2257 | | |
2258 | 0 | if(!memcmp(entry, new_entry, sizeof *entry)) { |
2259 | 0 | return 0; |
2260 | 0 | } |
2261 | | |
2262 | 0 | *entry = *new_entry; |
2263 | |
|
2264 | 0 | return 0; |
2265 | 0 | } |
2266 | | |
2267 | 0 | return -ESRCH; |
2268 | 0 | } |
2269 | | |
2270 | | void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin) |
2271 | 0 | { |
2272 | 0 | struct kvm_irq_routing_entry e = {}; |
2273 | |
|
2274 | 0 | assert(pin < s->gsi_count); |
2275 | |
|
2276 | 0 | e.gsi = irq; |
2277 | 0 | e.type = KVM_IRQ_ROUTING_IRQCHIP; |
2278 | 0 | e.flags = 0; |
2279 | 0 | e.u.irqchip.irqchip = irqchip; |
2280 | 0 | e.u.irqchip.pin = pin; |
2281 | 0 | kvm_add_routing_entry(s, &e); |
2282 | 0 | } |
2283 | | |
2284 | | void kvm_irqchip_release_virq(KVMState *s, int virq) |
2285 | 0 | { |
2286 | 0 | struct kvm_irq_routing_entry *e; |
2287 | 0 | int i; |
2288 | |
|
2289 | 0 | if (kvm_gsi_direct_mapping()) { |
2290 | 0 | return; |
2291 | 0 | } |
2292 | | |
2293 | 0 | for (i = 0; i < s->irq_routes->nr; i++) { |
2294 | 0 | e = &s->irq_routes->entries[i]; |
2295 | 0 | if (e->gsi == virq) { |
2296 | 0 | s->irq_routes->nr--; |
2297 | 0 | *e = s->irq_routes->entries[s->irq_routes->nr]; |
2298 | 0 | } |
2299 | 0 | } |
2300 | 0 | clear_gsi(s, virq); |
2301 | 0 | kvm_arch_release_virq_post(virq); |
2302 | 0 | trace_kvm_irqchip_release_virq(virq); |
2303 | 0 | } |
2304 | | |
2305 | | void kvm_irqchip_add_change_notifier(Notifier *n) |
2306 | 0 | { |
2307 | 0 | notifier_list_add(&kvm_irqchip_change_notifiers, n); |
2308 | 0 | } |
2309 | | |
2310 | | void kvm_irqchip_remove_change_notifier(Notifier *n) |
2311 | 0 | { |
2312 | 0 | notifier_remove(n); |
2313 | 0 | } |
2314 | | |
2315 | | void kvm_irqchip_change_notify(void) |
2316 | 0 | { |
2317 | 0 | notifier_list_notify(&kvm_irqchip_change_notifiers, NULL); |
2318 | 0 | } |
2319 | | |
2320 | | void kvm_vmfd_add_change_notifier(NotifierWithReturn *n) |
2321 | 0 | { |
2322 | 0 | notifier_with_return_list_add(®ister_vmfd_changed_notifiers, n); |
2323 | 0 | } |
2324 | | |
2325 | | void kvm_vmfd_remove_change_notifier(NotifierWithReturn *n) |
2326 | 0 | { |
2327 | 0 | notifier_with_return_remove(n); |
2328 | 0 | } |
2329 | | |
2330 | | static int kvm_vmfd_change_notify(Error **errp) |
2331 | 0 | { |
2332 | 0 | return notifier_with_return_list_notify(®ister_vmfd_changed_notifiers, |
2333 | 0 | &vmfd_notifier, errp); |
2334 | 0 | } |
2335 | | |
2336 | | void kvm_vcpufd_add_change_notifier(NotifierWithReturn *n) |
2337 | 0 | { |
2338 | 0 | notifier_with_return_list_add(®ister_vcpufd_changed_notifiers, n); |
2339 | 0 | } |
2340 | | |
2341 | | void kvm_vcpufd_remove_change_notifier(NotifierWithReturn *n) |
2342 | 0 | { |
2343 | 0 | notifier_with_return_remove(n); |
2344 | 0 | } |
2345 | | |
2346 | | static int kvm_vcpufd_change_notify(Error **errp) |
2347 | 0 | { |
2348 | 0 | return notifier_with_return_list_notify(®ister_vcpufd_changed_notifiers, |
2349 | 0 | &vmfd_notifier, errp); |
2350 | 0 | } |
2351 | | |
2352 | | int kvm_irqchip_get_virq(KVMState *s) |
2353 | 0 | { |
2354 | 0 | int next_virq; |
2355 | | |
2356 | | /* Return the lowest unused GSI in the bitmap */ |
2357 | 0 | next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count); |
2358 | 0 | if (next_virq >= s->gsi_count) { |
2359 | 0 | return -ENOSPC; |
2360 | 0 | } else { |
2361 | 0 | return next_virq; |
2362 | 0 | } |
2363 | 0 | } |
2364 | | |
2365 | | int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) |
2366 | 0 | { |
2367 | 0 | struct kvm_msi msi; |
2368 | |
|
2369 | 0 | msi.address_lo = (uint32_t)msg.address; |
2370 | 0 | msi.address_hi = msg.address >> 32; |
2371 | 0 | msi.data = le32_to_cpu(msg.data); |
2372 | 0 | msi.flags = 0; |
2373 | 0 | memset(msi.pad, 0, sizeof(msi.pad)); |
2374 | |
|
2375 | 0 | return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi); |
2376 | 0 | } |
2377 | | |
2378 | | int kvm_irqchip_add_msi_route(AccelRouteChange *c, int vector, PCIDevice *dev) |
2379 | 0 | { |
2380 | 0 | struct kvm_irq_routing_entry kroute = {}; |
2381 | 0 | int virq; |
2382 | 0 | KVMState *s = KVM_STATE(c->accel); |
2383 | 0 | MSIMessage msg = {0, 0}; |
2384 | |
|
2385 | 0 | if (pci_available && dev) { |
2386 | 0 | msg = pci_get_msi_message(dev, vector); |
2387 | 0 | } |
2388 | |
|
2389 | 0 | if (kvm_gsi_direct_mapping()) { |
2390 | 0 | return kvm_arch_msi_data_to_gsi(msg.data); |
2391 | 0 | } |
2392 | | |
2393 | 0 | if (!kvm_gsi_routing_enabled()) { |
2394 | 0 | return -ENOSYS; |
2395 | 0 | } |
2396 | | |
2397 | 0 | virq = kvm_irqchip_get_virq(s); |
2398 | 0 | if (virq < 0) { |
2399 | 0 | return virq; |
2400 | 0 | } |
2401 | | |
2402 | 0 | kroute.gsi = virq; |
2403 | 0 | kroute.type = KVM_IRQ_ROUTING_MSI; |
2404 | 0 | kroute.flags = 0; |
2405 | 0 | kroute.u.msi.address_lo = (uint32_t)msg.address; |
2406 | 0 | kroute.u.msi.address_hi = msg.address >> 32; |
2407 | 0 | kroute.u.msi.data = le32_to_cpu(msg.data); |
2408 | 0 | if (pci_available && kvm_msi_devid_required()) { |
2409 | 0 | kroute.flags = KVM_MSI_VALID_DEVID; |
2410 | 0 | kroute.u.msi.devid = pci_requester_id(dev); |
2411 | 0 | } |
2412 | 0 | if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { |
2413 | 0 | kvm_irqchip_release_virq(s, virq); |
2414 | 0 | return -EINVAL; |
2415 | 0 | } |
2416 | | |
2417 | 0 | if (s->irq_routes->nr < s->gsi_count) { |
2418 | 0 | trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A", |
2419 | 0 | vector, virq); |
2420 | |
|
2421 | 0 | kvm_add_routing_entry(s, &kroute); |
2422 | 0 | kvm_arch_add_msi_route_post(&kroute, vector, dev); |
2423 | 0 | c->changes++; |
2424 | 0 | } else { |
2425 | 0 | kvm_irqchip_release_virq(s, virq); |
2426 | 0 | return -ENOSPC; |
2427 | 0 | } |
2428 | | |
2429 | 0 | return virq; |
2430 | 0 | } |
2431 | | |
2432 | | int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg, |
2433 | | PCIDevice *dev) |
2434 | 0 | { |
2435 | 0 | struct kvm_irq_routing_entry kroute = {}; |
2436 | |
|
2437 | 0 | if (kvm_gsi_direct_mapping()) { |
2438 | 0 | return 0; |
2439 | 0 | } |
2440 | | |
2441 | 0 | if (!kvm_irqchip_in_kernel()) { |
2442 | 0 | return -ENOSYS; |
2443 | 0 | } |
2444 | | |
2445 | 0 | kroute.gsi = virq; |
2446 | 0 | kroute.type = KVM_IRQ_ROUTING_MSI; |
2447 | 0 | kroute.flags = 0; |
2448 | 0 | kroute.u.msi.address_lo = (uint32_t)msg.address; |
2449 | 0 | kroute.u.msi.address_hi = msg.address >> 32; |
2450 | 0 | kroute.u.msi.data = le32_to_cpu(msg.data); |
2451 | 0 | if (pci_available && kvm_msi_devid_required()) { |
2452 | 0 | kroute.flags = KVM_MSI_VALID_DEVID; |
2453 | 0 | kroute.u.msi.devid = pci_requester_id(dev); |
2454 | 0 | } |
2455 | 0 | if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { |
2456 | 0 | return -EINVAL; |
2457 | 0 | } |
2458 | | |
2459 | 0 | trace_kvm_irqchip_update_msi_route(virq); |
2460 | |
|
2461 | 0 | return kvm_update_routing_entry(s, &kroute); |
2462 | 0 | } |
2463 | | |
2464 | | static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event, |
2465 | | EventNotifier *resample, int virq, |
2466 | | bool assign) |
2467 | 0 | { |
2468 | 0 | int fd = event_notifier_get_fd(event); |
2469 | 0 | int rfd = resample ? event_notifier_get_fd(resample) : -1; |
2470 | |
|
2471 | 0 | struct kvm_irqfd irqfd = { |
2472 | 0 | .fd = fd, |
2473 | 0 | .gsi = virq, |
2474 | 0 | .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN, |
2475 | 0 | }; |
2476 | |
|
2477 | 0 | if (rfd != -1) { |
2478 | 0 | assert(assign); |
2479 | 0 | if (kvm_irqchip_is_split()) { |
2480 | | /* |
2481 | | * When the slow irqchip (e.g. IOAPIC) is in the |
2482 | | * userspace, KVM kernel resamplefd will not work because |
2483 | | * the EOI of the interrupt will be delivered to userspace |
2484 | | * instead, so the KVM kernel resamplefd kick will be |
2485 | | * skipped. The userspace here mimics what the kernel |
2486 | | * provides with resamplefd, remember the resamplefd and |
2487 | | * kick it when we receive EOI of this IRQ. |
2488 | | * |
2489 | | * This is hackery because IOAPIC is mostly bypassed |
2490 | | * (except EOI broadcasts) when irqfd is used. However |
2491 | | * this can bring much performance back for split irqchip |
2492 | | * with INTx IRQs (for VFIO, this gives 93% perf of the |
2493 | | * full fast path, which is 46% perf boost comparing to |
2494 | | * the INTx slow path). |
2495 | | */ |
2496 | 0 | kvm_resample_fd_insert(virq, resample); |
2497 | 0 | } else { |
2498 | 0 | irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE; |
2499 | 0 | irqfd.resamplefd = rfd; |
2500 | 0 | } |
2501 | 0 | } else if (!assign) { |
2502 | 0 | if (kvm_irqchip_is_split()) { |
2503 | 0 | kvm_resample_fd_remove(virq); |
2504 | 0 | } |
2505 | 0 | } |
2506 | |
|
2507 | 0 | return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd); |
2508 | 0 | } |
2509 | | |
2510 | | #else /* !KVM_CAP_IRQ_ROUTING */ |
2511 | | |
2512 | | void kvm_init_irq_routing(KVMState *s) |
2513 | | { |
2514 | | } |
2515 | | |
2516 | | void kvm_irqchip_release_virq(KVMState *s, int virq) |
2517 | | { |
2518 | | } |
2519 | | |
2520 | | int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) |
2521 | | { |
2522 | | abort(); |
2523 | | } |
2524 | | |
2525 | | int kvm_irqchip_add_msi_route(AccelRouteChange *c, int vector, PCIDevice *dev) |
2526 | | { |
2527 | | return -ENOSYS; |
2528 | | } |
2529 | | |
2530 | | int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter) |
2531 | | { |
2532 | | return -ENOSYS; |
2533 | | } |
2534 | | |
2535 | | int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint) |
2536 | | { |
2537 | | return -ENOSYS; |
2538 | | } |
2539 | | |
2540 | | static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event, |
2541 | | EventNotifier *resample, int virq, |
2542 | | bool assign) |
2543 | | { |
2544 | | abort(); |
2545 | | } |
2546 | | |
2547 | | int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg) |
2548 | | { |
2549 | | return -ENOSYS; |
2550 | | } |
2551 | | #endif /* !KVM_CAP_IRQ_ROUTING */ |
2552 | | |
2553 | | int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, |
2554 | | EventNotifier *rn, int virq) |
2555 | 0 | { |
2556 | 0 | return kvm_irqchip_assign_irqfd(s, n, rn, virq, true); |
2557 | 0 | } |
2558 | | |
2559 | | int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, |
2560 | | int virq) |
2561 | 0 | { |
2562 | 0 | return kvm_irqchip_assign_irqfd(s, n, NULL, virq, false); |
2563 | 0 | } |
2564 | | |
2565 | | int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, |
2566 | | EventNotifier *rn, qemu_irq irq) |
2567 | 0 | { |
2568 | 0 | gpointer key, gsi; |
2569 | 0 | gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); |
2570 | |
|
2571 | 0 | if (!found) { |
2572 | 0 | return -ENXIO; |
2573 | 0 | } |
2574 | 0 | return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi)); |
2575 | 0 | } |
2576 | | |
2577 | | int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, |
2578 | | qemu_irq irq) |
2579 | 0 | { |
2580 | 0 | gpointer key, gsi; |
2581 | 0 | gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); |
2582 | |
|
2583 | 0 | if (!found) { |
2584 | 0 | return -ENXIO; |
2585 | 0 | } |
2586 | 0 | return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi)); |
2587 | 0 | } |
2588 | | |
2589 | | void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi) |
2590 | 0 | { |
2591 | 0 | g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi)); |
2592 | 0 | } |
2593 | | |
2594 | | static int do_kvm_irqchip_create(KVMState *s) |
2595 | 0 | { |
2596 | 0 | int ret; |
2597 | 0 | if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) { |
2598 | 0 | ; |
2599 | 0 | } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) { |
2600 | 0 | ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0); |
2601 | 0 | if (ret < 0) { |
2602 | 0 | fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret)); |
2603 | 0 | exit(1); |
2604 | 0 | } |
2605 | 0 | } else { |
2606 | 0 | return -EOPNOTSUPP; |
2607 | 0 | } |
2608 | | |
2609 | 0 | if (kvm_check_extension(s, KVM_CAP_IRQFD) <= 0) { |
2610 | 0 | fprintf(stderr, "kvm: irqfd not implemented\n"); |
2611 | 0 | exit(1); |
2612 | 0 | } |
2613 | | |
2614 | | /* First probe and see if there's a arch-specific hook to create the |
2615 | | * in-kernel irqchip for us */ |
2616 | 0 | ret = kvm_arch_irqchip_create(s); |
2617 | 0 | if (ret == 0) { |
2618 | 0 | if (s->kernel_irqchip_split == ON_OFF_AUTO_ON) { |
2619 | 0 | error_report("Split IRQ chip mode not supported."); |
2620 | 0 | exit(1); |
2621 | 0 | } else { |
2622 | 0 | ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP); |
2623 | 0 | } |
2624 | 0 | } |
2625 | 0 | if (ret < 0) { |
2626 | 0 | fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret)); |
2627 | 0 | exit(1); |
2628 | 0 | } |
2629 | | |
2630 | 0 | return 0; |
2631 | 0 | } |
2632 | | |
2633 | | static void kvm_irqchip_create(KVMState *s) |
2634 | 0 | { |
2635 | 0 | assert(s->kernel_irqchip_split != ON_OFF_AUTO_AUTO); |
2636 | |
|
2637 | 0 | if (do_kvm_irqchip_create(s) < 0) { |
2638 | 0 | return; |
2639 | 0 | } |
2640 | 0 | kvm_kernel_irqchip = true; |
2641 | | /* If we have an in-kernel IRQ chip then we must have asynchronous |
2642 | | * interrupt delivery (though the reverse is not necessarily true) |
2643 | | */ |
2644 | 0 | kvm_async_interrupts_allowed = true; |
2645 | 0 | kvm_halt_in_kernel_allowed = true; |
2646 | |
|
2647 | 0 | kvm_init_irq_routing(s); |
2648 | |
|
2649 | 0 | s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal); |
2650 | 0 | } |
2651 | | |
2652 | | /* Find number of supported CPUs using the recommended |
2653 | | * procedure from the kernel API documentation to cope with |
2654 | | * older kernels that may be missing capabilities. |
2655 | | */ |
2656 | | static int kvm_recommended_vcpus(KVMState *s) |
2657 | 0 | { |
2658 | 0 | int ret = kvm_vm_check_extension(s, KVM_CAP_NR_VCPUS); |
2659 | 0 | return (ret) ? ret : 4; |
2660 | 0 | } |
2661 | | |
2662 | | static int kvm_max_vcpus(KVMState *s) |
2663 | 0 | { |
2664 | 0 | int ret = kvm_vm_check_extension(s, KVM_CAP_MAX_VCPUS); |
2665 | 0 | return (ret) ? ret : kvm_recommended_vcpus(s); |
2666 | 0 | } |
2667 | | |
2668 | | static int kvm_max_vcpu_id(KVMState *s) |
2669 | 0 | { |
2670 | 0 | int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID); |
2671 | 0 | return (ret) ? ret : kvm_max_vcpus(s); |
2672 | 0 | } |
2673 | | |
2674 | | bool kvm_vcpu_id_is_valid(int vcpu_id) |
2675 | 0 | { |
2676 | 0 | KVMState *s = KVM_STATE(current_accel()); |
2677 | 0 | return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s); |
2678 | 0 | } |
2679 | | |
2680 | | bool kvm_dirty_ring_enabled(void) |
2681 | 0 | { |
2682 | 0 | return kvm_state && kvm_state->kvm_dirty_ring_size; |
2683 | 0 | } |
2684 | | |
2685 | | static void query_stats_cb(StatsResultList **result, StatsTarget target, |
2686 | | strList *names, strList *targets, Error **errp); |
2687 | | static void query_stats_schemas_cb(StatsSchemaList **result, Error **errp); |
2688 | | |
2689 | | uint32_t kvm_dirty_ring_size(void) |
2690 | 0 | { |
2691 | 0 | return kvm_state->kvm_dirty_ring_size; |
2692 | 0 | } |
2693 | | |
2694 | | static int do_kvm_create_vm(KVMState *s, int type) |
2695 | 0 | { |
2696 | 0 | int ret; |
2697 | |
|
2698 | 0 | do { |
2699 | 0 | ret = kvm_ioctl(s, KVM_CREATE_VM, type); |
2700 | 0 | } while (ret == -EINTR); |
2701 | |
|
2702 | 0 | if (ret < 0) { |
2703 | 0 | error_report("ioctl(KVM_CREATE_VM) failed: %s", strerror(-ret)); |
2704 | |
|
2705 | | #ifdef TARGET_S390X |
2706 | | if (ret == -EINVAL) { |
2707 | | error_printf("Host kernel setup problem detected." |
2708 | | " Please verify:\n"); |
2709 | | error_printf("- for kernels supporting the" |
2710 | | " switch_amode or user_mode parameters, whether"); |
2711 | | error_printf(" user space is running in primary address space\n"); |
2712 | | error_printf("- for kernels supporting the vm.allocate_pgste" |
2713 | | " sysctl, whether it is enabled\n"); |
2714 | | } |
2715 | | #elif defined(TARGET_PPC) |
2716 | | if (ret == -EINVAL) { |
2717 | | error_printf("PPC KVM module is not loaded. Try modprobe kvm_%s.\n", |
2718 | | (type == 2) ? "pr" : "hv"); |
2719 | | } |
2720 | | #endif |
2721 | 0 | } |
2722 | |
|
2723 | 0 | return ret; |
2724 | 0 | } |
2725 | | |
2726 | | static int find_kvm_machine_type(MachineState *ms) |
2727 | 0 | { |
2728 | 0 | MachineClass *mc = MACHINE_GET_CLASS(ms); |
2729 | 0 | int type; |
2730 | |
|
2731 | 0 | if (object_property_find(OBJECT(current_machine), "kvm-type")) { |
2732 | 0 | g_autofree char *kvm_type; |
2733 | 0 | kvm_type = object_property_get_str(OBJECT(current_machine), |
2734 | 0 | "kvm-type", |
2735 | 0 | &error_abort); |
2736 | 0 | type = mc->kvm_type(ms, kvm_type); |
2737 | 0 | } else if (mc->kvm_type) { |
2738 | 0 | type = mc->kvm_type(ms, NULL); |
2739 | 0 | } else { |
2740 | 0 | type = kvm_arch_get_default_type(ms); |
2741 | 0 | } |
2742 | 0 | return type; |
2743 | 0 | } |
2744 | | |
2745 | | static int kvm_setup_dirty_ring(KVMState *s) |
2746 | 0 | { |
2747 | 0 | uint64_t dirty_log_manual_caps; |
2748 | 0 | int ret; |
2749 | | |
2750 | | /* |
2751 | | * Enable KVM dirty ring if supported, otherwise fall back to |
2752 | | * dirty logging mode |
2753 | | */ |
2754 | 0 | ret = kvm_dirty_ring_init(s); |
2755 | 0 | if (ret < 0) { |
2756 | 0 | return ret; |
2757 | 0 | } |
2758 | | |
2759 | | /* |
2760 | | * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 is not needed when dirty ring is |
2761 | | * enabled. More importantly, KVM_DIRTY_LOG_INITIALLY_SET will assume no |
2762 | | * page is wr-protected initially, which is against how kvm dirty ring is |
2763 | | * usage - kvm dirty ring requires all pages are wr-protected at the very |
2764 | | * beginning. Enabling this feature for dirty ring causes data corruption. |
2765 | | * |
2766 | | * TODO: Without KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and kvm clear dirty log, |
2767 | | * we may expect a higher stall time when starting the migration. In the |
2768 | | * future we can enable KVM_CLEAR_DIRTY_LOG to work with dirty ring too: |
2769 | | * instead of clearing dirty bit, it can be a way to explicitly wr-protect |
2770 | | * guest pages. |
2771 | | */ |
2772 | 0 | if (!s->kvm_dirty_ring_size) { |
2773 | 0 | dirty_log_manual_caps = |
2774 | 0 | kvm_check_extension(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2); |
2775 | 0 | dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | |
2776 | 0 | KVM_DIRTY_LOG_INITIALLY_SET); |
2777 | 0 | s->manual_dirty_log_protect = dirty_log_manual_caps; |
2778 | 0 | if (dirty_log_manual_caps) { |
2779 | 0 | ret = kvm_vm_enable_cap(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2, 0, |
2780 | 0 | dirty_log_manual_caps); |
2781 | 0 | if (ret) { |
2782 | 0 | warn_report("Trying to enable capability %"PRIu64" of " |
2783 | 0 | "KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 but failed. " |
2784 | 0 | "Falling back to the legacy mode. ", |
2785 | 0 | dirty_log_manual_caps); |
2786 | 0 | s->manual_dirty_log_protect = 0; |
2787 | 0 | } |
2788 | 0 | } |
2789 | 0 | } |
2790 | |
|
2791 | 0 | return 0; |
2792 | 0 | } |
2793 | | |
2794 | | static int kvm_reset_vmfd(MachineState *ms) |
2795 | 0 | { |
2796 | 0 | KVMState *s; |
2797 | 0 | KVMMemoryListener *kml; |
2798 | 0 | int ret = 0, type; |
2799 | 0 | Error *err = NULL; |
2800 | | |
2801 | | /* |
2802 | | * bail if the current architecture does not support VM file |
2803 | | * descriptor change. |
2804 | | */ |
2805 | 0 | if (!kvm_arch_supports_vmfd_change()) { |
2806 | 0 | error_report("This target architecture does not support KVM VM " |
2807 | 0 | "file descriptor change."); |
2808 | 0 | return -EOPNOTSUPP; |
2809 | 0 | } |
2810 | | |
2811 | 0 | s = KVM_STATE(ms->accelerator); |
2812 | 0 | kml = &s->memory_listener; |
2813 | |
|
2814 | 0 | memory_listener_unregister(&kml->listener); |
2815 | 0 | memory_listener_unregister(&kvm_io_listener); |
2816 | |
|
2817 | 0 | vmfd_notifier.pre = true; |
2818 | 0 | ret = kvm_vmfd_change_notify(&err); |
2819 | 0 | if (ret < 0) { |
2820 | 0 | return ret; |
2821 | 0 | } |
2822 | 0 | assert(!err); |
2823 | |
|
2824 | 0 | if (s->vmfd >= 0) { |
2825 | 0 | close(s->vmfd); |
2826 | 0 | } |
2827 | |
|
2828 | 0 | type = find_kvm_machine_type(ms); |
2829 | 0 | if (type < 0) { |
2830 | 0 | return -EINVAL; |
2831 | 0 | } |
2832 | | |
2833 | 0 | ret = do_kvm_create_vm(s, type); |
2834 | 0 | if (ret < 0) { |
2835 | 0 | return ret; |
2836 | 0 | } |
2837 | | |
2838 | 0 | s->vmfd = ret; |
2839 | | |
2840 | | /* guest state is now unprotected again */ |
2841 | 0 | kvm_state->guest_state_protected = false; |
2842 | |
|
2843 | 0 | kvm_setup_dirty_ring(s); |
2844 | | |
2845 | | /* rebind memory to new vm fd */ |
2846 | 0 | ret = ram_block_rebind(&err); |
2847 | 0 | if (ret < 0) { |
2848 | 0 | return ret; |
2849 | 0 | } |
2850 | 0 | assert(!err); |
2851 | |
|
2852 | 0 | ret = kvm_arch_on_vmfd_change(ms, s); |
2853 | 0 | if (ret < 0) { |
2854 | 0 | return ret; |
2855 | 0 | } |
2856 | | |
2857 | 0 | if (s->kernel_irqchip_allowed) { |
2858 | | /* ignore return from this function */ |
2859 | 0 | do_kvm_irqchip_create(s); |
2860 | 0 | } |
2861 | | |
2862 | | /* |
2863 | | * notify everyone that vmfd has changed. |
2864 | | */ |
2865 | 0 | vmfd_notifier.vmfd = s->vmfd; |
2866 | 0 | vmfd_notifier.pre = false; |
2867 | |
|
2868 | 0 | ret = kvm_vmfd_change_notify(&err); |
2869 | 0 | if (ret < 0) { |
2870 | 0 | return ret; |
2871 | 0 | } |
2872 | 0 | assert(!err); |
2873 | | |
2874 | | /* |
2875 | | * rebind new vcpu fds with the new kvm fds |
2876 | | * These can only be called after kvm_arch_on_vmfd_change() |
2877 | | */ |
2878 | 0 | ret = kvm_rebind_vcpus(&err); |
2879 | 0 | if (ret < 0) { |
2880 | 0 | return ret; |
2881 | 0 | } |
2882 | 0 | assert(!err); |
2883 | | |
2884 | | /* notify everyone that vcpu fd has changed. */ |
2885 | 0 | ret = kvm_vcpufd_change_notify(&err); |
2886 | 0 | if (ret < 0) { |
2887 | 0 | return ret; |
2888 | 0 | } |
2889 | 0 | assert(!err); |
2890 | | |
2891 | | /* these can be only called after ram_block_rebind() */ |
2892 | 0 | memory_listener_register(&kml->listener, &address_space_memory); |
2893 | 0 | memory_listener_register(&kvm_io_listener, &address_space_io); |
2894 | | |
2895 | | /* |
2896 | | * kvm fd has changed. Commit the irq routes to KVM once more. |
2897 | | */ |
2898 | 0 | kvm_irqchip_commit_routes(s); |
2899 | | /* |
2900 | | * for confidential guest, this is the last possible place where we |
2901 | | * can call synchronize_all_post_init() to sync all vcpu states to |
2902 | | * kvm. |
2903 | | */ |
2904 | 0 | if (ms->cgs) { |
2905 | 0 | cpu_synchronize_all_post_init(); |
2906 | 0 | } |
2907 | 0 | trace_kvm_reset_vmfd(); |
2908 | 0 | return ret; |
2909 | 0 | } |
2910 | | |
2911 | | static int kvm_init(AccelState *as, MachineState *ms) |
2912 | 0 | { |
2913 | 0 | MachineClass *mc = MACHINE_GET_CLASS(ms); |
2914 | 0 | static const char upgrade_note[] = |
2915 | 0 | "Please upgrade to at least kernel 4.5.\n"; |
2916 | 0 | const struct { |
2917 | 0 | const char *name; |
2918 | 0 | int num; |
2919 | 0 | } num_cpus[] = { |
2920 | 0 | { "SMP", ms->smp.cpus }, |
2921 | 0 | { "hotpluggable", ms->smp.max_cpus }, |
2922 | 0 | { /* end of list */ } |
2923 | 0 | }, *nc = num_cpus; |
2924 | 0 | int soft_vcpus_limit, hard_vcpus_limit; |
2925 | 0 | KVMState *s = KVM_STATE(as); |
2926 | 0 | const KVMCapabilityInfo *missing_cap; |
2927 | 0 | int ret; |
2928 | 0 | int type; |
2929 | |
|
2930 | 0 | qemu_mutex_init(&kml_slots_lock); |
2931 | 0 | qemu_mutex_init(&kvm_resample_fd_list_lock); |
2932 | | |
2933 | | /* |
2934 | | * On systems where the kernel can support different base page |
2935 | | * sizes, host page size may be different from TARGET_PAGE_SIZE, |
2936 | | * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum |
2937 | | * page size for the system though. |
2938 | | */ |
2939 | 0 | assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size()); |
2940 | |
|
2941 | 0 | s->sigmask_len = 8; |
2942 | 0 | accel_blocker_init(); |
2943 | |
|
2944 | 0 | #ifdef TARGET_KVM_HAVE_GUEST_DEBUG |
2945 | 0 | QTAILQ_INIT(&s->kvm_sw_breakpoints); |
2946 | 0 | #endif |
2947 | 0 | QLIST_INIT(&s->kvm_parked_vcpus); |
2948 | 0 | s->fd = qemu_open_old(s->device ?: "/dev/kvm", O_RDWR); |
2949 | 0 | if (s->fd == -1) { |
2950 | 0 | error_report("Could not access KVM kernel module: %m"); |
2951 | 0 | ret = -errno; |
2952 | 0 | goto err; |
2953 | 0 | } |
2954 | | |
2955 | 0 | ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0); |
2956 | 0 | if (ret < KVM_API_VERSION) { |
2957 | 0 | if (ret >= 0) { |
2958 | 0 | ret = -EINVAL; |
2959 | 0 | } |
2960 | 0 | error_report("kvm version too old"); |
2961 | 0 | goto err; |
2962 | 0 | } |
2963 | | |
2964 | 0 | if (ret > KVM_API_VERSION) { |
2965 | 0 | ret = -EINVAL; |
2966 | 0 | error_report("kvm version not supported"); |
2967 | 0 | goto err; |
2968 | 0 | } |
2969 | | |
2970 | 0 | kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT); |
2971 | 0 | s->nr_slots_max = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS); |
2972 | | |
2973 | | /* If unspecified, use the default value */ |
2974 | 0 | if (!s->nr_slots_max) { |
2975 | 0 | s->nr_slots_max = KVM_MEMSLOTS_NR_MAX_DEFAULT; |
2976 | 0 | } |
2977 | |
|
2978 | 0 | type = find_kvm_machine_type(ms); |
2979 | 0 | if (type < 0) { |
2980 | 0 | ret = -EINVAL; |
2981 | 0 | goto err; |
2982 | 0 | } |
2983 | | |
2984 | 0 | ret = do_kvm_create_vm(s, type); |
2985 | 0 | if (ret < 0) { |
2986 | 0 | goto err; |
2987 | 0 | } |
2988 | | |
2989 | 0 | s->vmfd = ret; |
2990 | |
|
2991 | 0 | s->nr_as = kvm_vm_check_extension(s, KVM_CAP_MULTI_ADDRESS_SPACE); |
2992 | 0 | if (s->nr_as <= 1) { |
2993 | 0 | s->nr_as = 1; |
2994 | 0 | } |
2995 | 0 | s->as = g_new0(struct KVMAs, s->nr_as); |
2996 | | |
2997 | | /* check the vcpu limits */ |
2998 | 0 | soft_vcpus_limit = kvm_recommended_vcpus(s); |
2999 | 0 | hard_vcpus_limit = kvm_max_vcpus(s); |
3000 | |
|
3001 | 0 | while (nc->name) { |
3002 | 0 | if (nc->num > soft_vcpus_limit) { |
3003 | 0 | warn_report("Number of %s cpus requested (%d) exceeds " |
3004 | 0 | "the recommended cpus supported by KVM (%d)", |
3005 | 0 | nc->name, nc->num, soft_vcpus_limit); |
3006 | |
|
3007 | 0 | if (nc->num > hard_vcpus_limit) { |
3008 | 0 | error_report("Number of %s cpus requested (%d) exceeds " |
3009 | 0 | "the maximum cpus supported by KVM (%d)", |
3010 | 0 | nc->name, nc->num, hard_vcpus_limit); |
3011 | 0 | exit(1); |
3012 | 0 | } |
3013 | 0 | } |
3014 | 0 | nc++; |
3015 | 0 | } |
3016 | | |
3017 | 0 | missing_cap = kvm_check_extension_list(s, kvm_required_capabilities); |
3018 | 0 | if (!missing_cap) { |
3019 | 0 | missing_cap = |
3020 | 0 | kvm_check_extension_list(s, kvm_arch_required_capabilities); |
3021 | 0 | } |
3022 | 0 | if (missing_cap) { |
3023 | 0 | ret = -EINVAL; |
3024 | 0 | error_report("kvm does not support %s", missing_cap->name); |
3025 | 0 | error_printf("%s", upgrade_note); |
3026 | 0 | goto err; |
3027 | 0 | } |
3028 | | |
3029 | 0 | s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO); |
3030 | 0 | s->coalesced_pio = s->coalesced_mmio && |
3031 | 0 | kvm_check_extension(s, KVM_CAP_COALESCED_PIO); |
3032 | |
|
3033 | 0 | ret = kvm_setup_dirty_ring(s); |
3034 | 0 | if (ret < 0) { |
3035 | 0 | goto err; |
3036 | 0 | } |
3037 | | |
3038 | 0 | #ifdef KVM_CAP_VCPU_EVENTS |
3039 | 0 | s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS); |
3040 | 0 | #endif |
3041 | 0 | s->max_nested_state_len = kvm_check_extension(s, KVM_CAP_NESTED_STATE); |
3042 | |
|
3043 | 0 | s->irq_set_ioctl = KVM_IRQ_LINE; |
3044 | 0 | if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) { |
3045 | 0 | s->irq_set_ioctl = KVM_IRQ_LINE_STATUS; |
3046 | 0 | } |
3047 | |
|
3048 | 0 | kvm_readonly_mem_allowed = |
3049 | 0 | (kvm_vm_check_extension(s, KVM_CAP_READONLY_MEM) > 0); |
3050 | |
|
3051 | 0 | kvm_resamplefds_allowed = |
3052 | 0 | (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0); |
3053 | |
|
3054 | 0 | kvm_vm_attributes_allowed = |
3055 | 0 | (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0); |
3056 | |
|
3057 | 0 | #ifdef TARGET_KVM_HAVE_GUEST_DEBUG |
3058 | 0 | if (kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG) > 0) { |
3059 | 0 | as->gdbstub.sstep_flags = SSTEP_ENABLE; |
3060 | |
|
3061 | 0 | int guest_debug_flags = |
3062 | 0 | kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG2); |
3063 | |
|
3064 | 0 | if (guest_debug_flags & KVM_GUESTDBG_BLOCKIRQ) { |
3065 | 0 | as->gdbstub.sstep_flags |= SSTEP_NOIRQ; |
3066 | 0 | } |
3067 | 0 | } |
3068 | 0 | #endif |
3069 | |
|
3070 | 0 | kvm_state = s; |
3071 | |
|
3072 | 0 | ret = kvm_arch_init(ms, s); |
3073 | 0 | if (ret < 0) { |
3074 | 0 | goto err; |
3075 | 0 | } |
3076 | | |
3077 | 0 | kvm_supported_memory_attributes = kvm_vm_check_extension(s, KVM_CAP_MEMORY_ATTRIBUTES); |
3078 | 0 | kvm_guest_memfd_supported = |
3079 | 0 | kvm_vm_check_extension(s, KVM_CAP_GUEST_MEMFD) && |
3080 | 0 | kvm_vm_check_extension(s, KVM_CAP_USER_MEMORY2); |
3081 | 0 | kvm_pre_fault_memory_supported = kvm_vm_check_extension(s, KVM_CAP_PRE_FAULT_MEMORY); |
3082 | 0 | kvm_guest_memfd_flags_supported = |
3083 | 0 | kvm_vm_check_extension(s, KVM_CAP_GUEST_MEMFD_FLAGS); |
3084 | |
|
3085 | 0 | if (s->kernel_irqchip_split == ON_OFF_AUTO_AUTO) { |
3086 | 0 | s->kernel_irqchip_split = mc->default_kernel_irqchip_split ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; |
3087 | 0 | } |
3088 | |
|
3089 | 0 | qemu_register_reset(kvm_unpoison_all, NULL); |
3090 | |
|
3091 | 0 | if (s->kernel_irqchip_allowed) { |
3092 | 0 | kvm_irqchip_create(s); |
3093 | 0 | } |
3094 | |
|
3095 | 0 | s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add; |
3096 | 0 | s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del; |
3097 | 0 | s->memory_listener.listener.coalesced_io_add = kvm_coalesce_mmio_region; |
3098 | 0 | s->memory_listener.listener.coalesced_io_del = kvm_uncoalesce_mmio_region; |
3099 | |
|
3100 | 0 | kvm_memory_listener_register(s, &s->memory_listener, |
3101 | 0 | &address_space_memory, 0, "kvm-memory"); |
3102 | 0 | memory_listener_register(&kvm_io_listener, |
3103 | 0 | &address_space_io); |
3104 | |
|
3105 | 0 | s->sync_mmu = !!kvm_vm_check_extension(kvm_state, KVM_CAP_SYNC_MMU); |
3106 | 0 | if (!s->sync_mmu) { |
3107 | 0 | ret = ram_block_discard_disable(true); |
3108 | 0 | assert(!ret); |
3109 | 0 | } |
3110 | |
|
3111 | 0 | if (s->kvm_dirty_ring_size) { |
3112 | 0 | kvm_dirty_ring_reaper_init(s); |
3113 | 0 | } |
3114 | |
|
3115 | 0 | if (kvm_check_extension(kvm_state, KVM_CAP_BINARY_STATS_FD)) { |
3116 | 0 | add_stats_callbacks(STATS_PROVIDER_KVM, query_stats_cb, |
3117 | 0 | query_stats_schemas_cb); |
3118 | 0 | } |
3119 | |
|
3120 | 0 | return 0; |
3121 | | |
3122 | 0 | err: |
3123 | 0 | assert(ret < 0); |
3124 | 0 | if (s->vmfd >= 0) { |
3125 | 0 | close(s->vmfd); |
3126 | 0 | } |
3127 | 0 | if (s->fd != -1) { |
3128 | 0 | close(s->fd); |
3129 | 0 | } |
3130 | 0 | g_free(s->as); |
3131 | 0 | g_free(s->memory_listener.slots); |
3132 | |
|
3133 | 0 | return ret; |
3134 | 0 | } |
3135 | | |
3136 | | void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len) |
3137 | 0 | { |
3138 | 0 | s->sigmask_len = sigmask_len; |
3139 | 0 | } |
3140 | | |
3141 | | static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction, |
3142 | | int size, uint32_t count) |
3143 | 0 | { |
3144 | 0 | int i; |
3145 | 0 | uint8_t *ptr = data; |
3146 | |
|
3147 | 0 | for (i = 0; i < count; i++) { |
3148 | 0 | address_space_rw(&address_space_io, port, attrs, |
3149 | 0 | ptr, size, |
3150 | 0 | direction == KVM_EXIT_IO_OUT); |
3151 | 0 | ptr += size; |
3152 | 0 | } |
3153 | 0 | } |
3154 | | |
3155 | | static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run) |
3156 | 0 | { |
3157 | 0 | int i; |
3158 | |
|
3159 | 0 | fprintf(stderr, "KVM internal error. Suberror: %d\n", |
3160 | 0 | run->internal.suberror); |
3161 | |
|
3162 | 0 | for (i = 0; i < run->internal.ndata; ++i) { |
3163 | 0 | fprintf(stderr, "extra data[%d]: 0x%016"PRIx64"\n", |
3164 | 0 | i, (uint64_t)run->internal.data[i]); |
3165 | 0 | } |
3166 | 0 | if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) { |
3167 | 0 | fprintf(stderr, "emulation failure\n"); |
3168 | 0 | if (!kvm_arch_stop_on_emulation_error(cpu)) { |
3169 | 0 | cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); |
3170 | 0 | return EXCP_INTERRUPT; |
3171 | 0 | } |
3172 | 0 | } |
3173 | | /* FIXME: Should trigger a qmp message to let management know |
3174 | | * something went wrong. |
3175 | | */ |
3176 | 0 | return -1; |
3177 | 0 | } |
3178 | | |
3179 | | void kvm_flush_coalesced_mmio_buffer(void) |
3180 | 0 | { |
3181 | 0 | KVMState *s = kvm_state; |
3182 | |
|
3183 | 0 | if (!s || s->coalesced_flush_in_progress) { |
3184 | 0 | return; |
3185 | 0 | } |
3186 | | |
3187 | 0 | s->coalesced_flush_in_progress = true; |
3188 | |
|
3189 | 0 | if (s->coalesced_mmio_ring) { |
3190 | 0 | struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring; |
3191 | 0 | while (ring->first != ring->last) { |
3192 | 0 | struct kvm_coalesced_mmio *ent; |
3193 | 0 | const AddressSpace *as; |
3194 | |
|
3195 | 0 | ent = &ring->coalesced_mmio[ring->first]; |
3196 | 0 | if (ent->len > sizeof(ent->data)) { |
3197 | 0 | warn_report("coalesced MMIO entry has invalid len %u", |
3198 | 0 | ent->len); |
3199 | 0 | } else { |
3200 | 0 | as = ent->pio == 1 ? &address_space_io : &address_space_memory; |
3201 | 0 | address_space_write(as, ent->phys_addr, MEMTXATTRS_UNSPECIFIED, |
3202 | 0 | ent->data, ent->len); |
3203 | 0 | } |
3204 | |
|
3205 | 0 | smp_wmb(); |
3206 | 0 | ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX; |
3207 | 0 | } |
3208 | 0 | } |
3209 | |
|
3210 | 0 | s->coalesced_flush_in_progress = false; |
3211 | 0 | } |
3212 | | |
3213 | | static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg) |
3214 | 0 | { |
3215 | 0 | if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) { |
3216 | 0 | Error *err = NULL; |
3217 | 0 | int ret = kvm_arch_get_registers(cpu, &err); |
3218 | 0 | if (ret) { |
3219 | 0 | if (err) { |
3220 | 0 | error_reportf_err(err, "Failed to synchronize CPU state: "); |
3221 | 0 | } else { |
3222 | 0 | error_report("Failed to get registers: %s", strerror(-ret)); |
3223 | 0 | } |
3224 | |
|
3225 | 0 | cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); |
3226 | 0 | vm_stop(RUN_STATE_INTERNAL_ERROR); |
3227 | 0 | } |
3228 | |
|
3229 | 0 | cpu->vcpu_dirty = true; |
3230 | 0 | } |
3231 | 0 | } |
3232 | | |
3233 | | void kvm_cpu_synchronize_state(CPUState *cpu) |
3234 | 0 | { |
3235 | 0 | if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) { |
3236 | 0 | run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL); |
3237 | 0 | } |
3238 | 0 | } |
3239 | | |
3240 | | static bool kvm_cpu_synchronize_put(CPUState *cpu, KvmPutState state, |
3241 | | const char *desc) |
3242 | 0 | { |
3243 | 0 | Error *err = NULL; |
3244 | 0 | int ret = kvm_arch_put_registers(cpu, state, &err); |
3245 | 0 | if (ret) { |
3246 | 0 | if (err) { |
3247 | 0 | error_reportf_err(err, "Restoring resisters %s: ", desc); |
3248 | 0 | } else { |
3249 | 0 | error_report("Failed to put registers %s: %s", desc, |
3250 | 0 | strerror(-ret)); |
3251 | 0 | } |
3252 | 0 | return false; |
3253 | 0 | } |
3254 | | |
3255 | 0 | cpu->vcpu_dirty = false; |
3256 | |
|
3257 | 0 | return true; |
3258 | 0 | } |
3259 | | |
3260 | | static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg) |
3261 | 0 | { |
3262 | 0 | if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_RESET_STATE, "after reset")) { |
3263 | 0 | cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); |
3264 | 0 | vm_stop(RUN_STATE_INTERNAL_ERROR); |
3265 | 0 | } |
3266 | 0 | } |
3267 | | |
3268 | | void kvm_cpu_synchronize_post_reset(CPUState *cpu) |
3269 | 0 | { |
3270 | 0 | run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL); |
3271 | |
|
3272 | 0 | if (cpu == first_cpu) { |
3273 | 0 | kvm_reset_parked_vcpus(kvm_state); |
3274 | 0 | } |
3275 | 0 | } |
3276 | | |
3277 | | static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg) |
3278 | 0 | { |
3279 | 0 | if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_FULL_STATE, "after init")) { |
3280 | 0 | exit(1); |
3281 | 0 | } |
3282 | 0 | } |
3283 | | |
3284 | | void kvm_cpu_synchronize_post_init(CPUState *cpu) |
3285 | 0 | { |
3286 | 0 | if (!kvm_state->guest_state_protected) { |
3287 | | /* |
3288 | | * This runs before the machine_init_done notifiers, and is the last |
3289 | | * opportunity to synchronize the state of confidential guests. |
3290 | | */ |
3291 | 0 | run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL); |
3292 | 0 | } |
3293 | 0 | } |
3294 | | |
3295 | | static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg) |
3296 | 0 | { |
3297 | 0 | cpu->vcpu_dirty = true; |
3298 | 0 | } |
3299 | | |
3300 | | void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu) |
3301 | 0 | { |
3302 | 0 | run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL); |
3303 | 0 | } |
3304 | | |
3305 | | #ifdef KVM_HAVE_MCE_INJECTION |
3306 | | static __thread void *pending_sigbus_addr; |
3307 | | static __thread int pending_sigbus_code; |
3308 | | static __thread bool have_sigbus_pending; |
3309 | | #endif |
3310 | | |
3311 | | static void kvm_cpu_kick(CPUState *cpu) |
3312 | 0 | { |
3313 | 0 | qatomic_set(&cpu->kvm_run->immediate_exit, 1); |
3314 | 0 | } |
3315 | | |
3316 | | static void kvm_cpu_kick_self(void) |
3317 | 0 | { |
3318 | 0 | if (kvm_immediate_exit) { |
3319 | 0 | kvm_cpu_kick(current_cpu); |
3320 | 0 | } else { |
3321 | 0 | qemu_cpu_kick_self(); |
3322 | 0 | } |
3323 | 0 | } |
3324 | | |
3325 | | static void kvm_eat_signals(CPUState *cpu) |
3326 | 0 | { |
3327 | 0 | struct timespec ts = { 0, 0 }; |
3328 | 0 | siginfo_t siginfo; |
3329 | 0 | sigset_t waitset; |
3330 | 0 | sigset_t chkset; |
3331 | 0 | int r; |
3332 | |
|
3333 | 0 | if (kvm_immediate_exit) { |
3334 | 0 | qatomic_set(&cpu->kvm_run->immediate_exit, 0); |
3335 | 0 | return; |
3336 | 0 | } |
3337 | | |
3338 | 0 | sigemptyset(&waitset); |
3339 | 0 | sigaddset(&waitset, SIG_IPI); |
3340 | |
|
3341 | 0 | do { |
3342 | 0 | r = sigtimedwait(&waitset, &siginfo, &ts); |
3343 | 0 | if (r == -1 && !(errno == EAGAIN || errno == EINTR)) { |
3344 | 0 | perror("sigtimedwait"); |
3345 | 0 | exit(1); |
3346 | 0 | } |
3347 | | |
3348 | 0 | r = sigpending(&chkset); |
3349 | 0 | if (r == -1) { |
3350 | 0 | perror("sigpending"); |
3351 | 0 | exit(1); |
3352 | 0 | } |
3353 | 0 | } while (sigismember(&chkset, SIG_IPI)); |
3354 | 0 | } |
3355 | | |
3356 | | int kvm_convert_memory(hwaddr start, hwaddr size, bool to_private) |
3357 | 0 | { |
3358 | 0 | MemoryRegionSection section; |
3359 | 0 | ram_addr_t offset; |
3360 | 0 | MemoryRegion *mr; |
3361 | 0 | RAMBlock *rb; |
3362 | 0 | void *addr; |
3363 | 0 | int ret = -EINVAL; |
3364 | |
|
3365 | 0 | trace_kvm_convert_memory(start, size, to_private ? "shared_to_private" : "private_to_shared"); |
3366 | |
|
3367 | 0 | if (!QEMU_PTR_IS_ALIGNED(start, qemu_real_host_page_size()) || |
3368 | 0 | !QEMU_PTR_IS_ALIGNED(size, qemu_real_host_page_size())) { |
3369 | 0 | return ret; |
3370 | 0 | } |
3371 | | |
3372 | 0 | if (!size) { |
3373 | 0 | return ret; |
3374 | 0 | } |
3375 | | |
3376 | 0 | section = memory_region_find(get_system_memory(), start, size); |
3377 | 0 | mr = section.mr; |
3378 | 0 | if (!mr) { |
3379 | | /* |
3380 | | * Ignore converting non-assigned region to shared. |
3381 | | * |
3382 | | * TDX requires vMMIO region to be shared to inject #VE to guest. |
3383 | | * OVMF issues conservatively MapGPA(shared) on 32bit PCI MMIO region, |
3384 | | * and vIO-APIC 0xFEC00000 4K page. |
3385 | | * OVMF assigns 32bit PCI MMIO region to |
3386 | | * [top of low memory: typically 2GB=0xC000000, 0xFC00000) |
3387 | | */ |
3388 | 0 | if (!to_private) { |
3389 | 0 | return 0; |
3390 | 0 | } |
3391 | 0 | return ret; |
3392 | 0 | } |
3393 | | |
3394 | 0 | if (!memory_region_has_guest_memfd_private(mr)) { |
3395 | | /* |
3396 | | * Because vMMIO region must be shared, guest TD may convert vMMIO |
3397 | | * region to shared explicitly. Don't complain such case. See |
3398 | | * memory_region_type() for checking if the region is MMIO region. |
3399 | | */ |
3400 | 0 | if (!to_private && |
3401 | 0 | !memory_region_is_ram(mr) && |
3402 | 0 | !memory_region_is_ram_device(mr) && |
3403 | 0 | !memory_region_is_rom(mr) && |
3404 | 0 | !memory_region_is_romd(mr)) { |
3405 | 0 | ret = 0; |
3406 | 0 | } else { |
3407 | 0 | error_report("Convert non guest_memfd backed memory region " |
3408 | 0 | "(0x%"HWADDR_PRIx" ,+ 0x%"HWADDR_PRIx") to %s", |
3409 | 0 | start, size, to_private ? "private" : "shared"); |
3410 | 0 | } |
3411 | 0 | goto out_unref; |
3412 | 0 | } |
3413 | | |
3414 | 0 | if (to_private) { |
3415 | 0 | ret = kvm_set_memory_attributes_private(start, size); |
3416 | 0 | } else { |
3417 | 0 | ret = kvm_set_memory_attributes_shared(start, size); |
3418 | 0 | } |
3419 | 0 | if (ret) { |
3420 | 0 | goto out_unref; |
3421 | 0 | } |
3422 | | |
3423 | 0 | addr = memory_region_get_ram_ptr(mr) + section.offset_within_region; |
3424 | 0 | rb = qemu_ram_block_from_host(addr, false, &offset); |
3425 | |
|
3426 | 0 | ret = ram_block_attributes_state_change(rb->attributes, |
3427 | 0 | offset, size, to_private); |
3428 | 0 | if (ret) { |
3429 | 0 | error_report("Failed to notify the listener the state change of " |
3430 | 0 | "(0x%"HWADDR_PRIx" + 0x%"HWADDR_PRIx") to %s", |
3431 | 0 | start, size, to_private ? "private" : "shared"); |
3432 | 0 | goto out_unref; |
3433 | 0 | } |
3434 | | |
3435 | 0 | if (to_private) { |
3436 | 0 | if (rb->page_size != qemu_real_host_page_size()) { |
3437 | | /* |
3438 | | * shared memory is backed by hugetlb, which is supposed to be |
3439 | | * pre-allocated and doesn't need to be discarded |
3440 | | */ |
3441 | 0 | goto out_unref; |
3442 | 0 | } |
3443 | 0 | ret = ram_block_discard_shared_range(rb, offset, size); |
3444 | 0 | } else { |
3445 | 0 | ret = ram_block_discard_guest_memfd_range(rb, offset, size); |
3446 | 0 | } |
3447 | | |
3448 | 0 | out_unref: |
3449 | 0 | memory_region_unref(mr); |
3450 | 0 | return ret; |
3451 | 0 | } |
3452 | | |
3453 | | int kvm_cpu_exec(CPUState *cpu) |
3454 | 0 | { |
3455 | 0 | struct kvm_run *run = cpu->kvm_run; |
3456 | 0 | int ret, run_ret; |
3457 | |
|
3458 | 0 | trace_kvm_cpu_exec(); |
3459 | |
|
3460 | 0 | if (kvm_arch_process_async_events(cpu)) { |
3461 | 0 | return EXCP_HLT; |
3462 | 0 | } |
3463 | | |
3464 | 0 | bql_unlock(); |
3465 | 0 | cpu_exec_start(cpu); |
3466 | |
|
3467 | 0 | do { |
3468 | 0 | MemTxAttrs attrs; |
3469 | |
|
3470 | 0 | if (cpu->vcpu_dirty) { |
3471 | 0 | if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_RUNTIME_STATE, |
3472 | 0 | "at runtime")) { |
3473 | 0 | ret = -1; |
3474 | 0 | break; |
3475 | 0 | } |
3476 | 0 | } |
3477 | | |
3478 | 0 | kvm_arch_pre_run(cpu, run); |
3479 | | /* Corresponding store-release is in cpu_exit. */ |
3480 | 0 | if (qatomic_load_acquire(&cpu->exit_request)) { |
3481 | 0 | trace_kvm_interrupt_exit_request(); |
3482 | | /* |
3483 | | * KVM requires us to reenter the kernel after IO exits to complete |
3484 | | * instruction emulation. This self-signal will ensure that we |
3485 | | * leave ASAP again. |
3486 | | */ |
3487 | 0 | kvm_cpu_kick_self(); |
3488 | 0 | } |
3489 | |
|
3490 | 0 | run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0); |
3491 | | |
3492 | | /* |
3493 | | * After writing cpu->exit_request, cpu_exit() sends a signal that writes |
3494 | | * kvm->run->immediate_exit. The signal is already happening after the |
3495 | | * write to cpu->exit_request so, if KVM read kvm->run->immediate_exit |
3496 | | * as true, cpu->exit_request will always read as true. |
3497 | | */ |
3498 | |
|
3499 | 0 | attrs = kvm_arch_post_run(cpu, run); |
3500 | |
|
3501 | 0 | #ifdef KVM_HAVE_MCE_INJECTION |
3502 | 0 | if (unlikely(have_sigbus_pending)) { |
3503 | 0 | bql_lock(); |
3504 | 0 | kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code, |
3505 | 0 | pending_sigbus_addr); |
3506 | 0 | have_sigbus_pending = false; |
3507 | 0 | bql_unlock(); |
3508 | 0 | } |
3509 | 0 | #endif |
3510 | |
|
3511 | 0 | if (run_ret < 0) { |
3512 | 0 | if (run_ret == -EINTR || run_ret == -EAGAIN) { |
3513 | 0 | trace_kvm_io_window_exit(); |
3514 | 0 | kvm_eat_signals(cpu); |
3515 | 0 | ret = EXCP_INTERRUPT; |
3516 | 0 | break; |
3517 | 0 | } |
3518 | 0 | if (!(run_ret == -EFAULT && run->exit_reason == KVM_EXIT_MEMORY_FAULT)) { |
3519 | 0 | fprintf(stderr, "error: kvm run failed %s\n", |
3520 | 0 | strerror(-run_ret)); |
3521 | | #ifdef TARGET_PPC |
3522 | | if (run_ret == -EBUSY) { |
3523 | | fprintf(stderr, |
3524 | | "This is probably because your SMT is enabled.\n" |
3525 | | "VCPU can only run on primary threads with all " |
3526 | | "secondary threads offline.\n"); |
3527 | | } |
3528 | | #endif |
3529 | 0 | ret = -1; |
3530 | 0 | break; |
3531 | 0 | } |
3532 | 0 | } |
3533 | | |
3534 | 0 | trace_kvm_run_exit(cpu->cpu_index, run->exit_reason); |
3535 | 0 | switch (run->exit_reason) { |
3536 | 0 | case KVM_EXIT_IO: |
3537 | | /* Called outside BQL */ |
3538 | 0 | kvm_handle_io(run->io.port, attrs, |
3539 | 0 | (uint8_t *)run + run->io.data_offset, |
3540 | 0 | run->io.direction, |
3541 | 0 | run->io.size, |
3542 | 0 | run->io.count); |
3543 | 0 | ret = 0; |
3544 | 0 | break; |
3545 | 0 | case KVM_EXIT_MMIO: |
3546 | | /* Called outside BQL */ |
3547 | 0 | address_space_rw(&address_space_memory, |
3548 | 0 | run->mmio.phys_addr, attrs, |
3549 | 0 | run->mmio.data, |
3550 | 0 | run->mmio.len, |
3551 | 0 | run->mmio.is_write); |
3552 | 0 | ret = 0; |
3553 | 0 | break; |
3554 | 0 | case KVM_EXIT_IRQ_WINDOW_OPEN: |
3555 | 0 | ret = EXCP_INTERRUPT; |
3556 | 0 | break; |
3557 | 0 | case KVM_EXIT_SHUTDOWN: |
3558 | 0 | qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); |
3559 | 0 | ret = EXCP_INTERRUPT; |
3560 | 0 | break; |
3561 | 0 | case KVM_EXIT_UNKNOWN: |
3562 | 0 | fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n", |
3563 | 0 | (uint64_t)run->hw.hardware_exit_reason); |
3564 | 0 | ret = -1; |
3565 | 0 | break; |
3566 | 0 | case KVM_EXIT_INTERNAL_ERROR: |
3567 | 0 | ret = kvm_handle_internal_error(cpu, run); |
3568 | 0 | break; |
3569 | 0 | case KVM_EXIT_DIRTY_RING_FULL: |
3570 | | /* |
3571 | | * We shouldn't continue if the dirty ring of this vcpu is |
3572 | | * still full. Got kicked by KVM_RESET_DIRTY_RINGS. |
3573 | | */ |
3574 | 0 | trace_kvm_dirty_ring_full(cpu->cpu_index); |
3575 | 0 | bql_lock(); |
3576 | | /* |
3577 | | * We throttle vCPU by making it sleep once it exit from kernel |
3578 | | * due to dirty ring full. In the dirtylimit scenario, reaping |
3579 | | * all vCPUs after a single vCPU dirty ring get full result in |
3580 | | * the miss of sleep, so just reap the ring-fulled vCPU. |
3581 | | */ |
3582 | 0 | if (dirtylimit_in_service()) { |
3583 | 0 | kvm_dirty_ring_reap(kvm_state, cpu); |
3584 | 0 | } else { |
3585 | 0 | kvm_dirty_ring_reap(kvm_state, NULL); |
3586 | 0 | } |
3587 | 0 | bql_unlock(); |
3588 | 0 | dirtylimit_vcpu_execute(cpu); |
3589 | 0 | ret = 0; |
3590 | 0 | break; |
3591 | 0 | case KVM_EXIT_SYSTEM_EVENT: |
3592 | 0 | trace_kvm_run_exit_system_event(cpu->cpu_index, run->system_event.type); |
3593 | 0 | switch (run->system_event.type) { |
3594 | 0 | case KVM_SYSTEM_EVENT_SHUTDOWN: |
3595 | 0 | qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN); |
3596 | 0 | ret = EXCP_INTERRUPT; |
3597 | 0 | break; |
3598 | 0 | case KVM_SYSTEM_EVENT_RESET: |
3599 | 0 | qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); |
3600 | 0 | ret = EXCP_INTERRUPT; |
3601 | 0 | break; |
3602 | 0 | case KVM_SYSTEM_EVENT_SEV_TERM: |
3603 | 0 | case KVM_SYSTEM_EVENT_CRASH: |
3604 | 0 | kvm_cpu_synchronize_state(cpu); |
3605 | 0 | bql_lock(); |
3606 | 0 | qemu_system_guest_panicked(cpu_get_crash_info(cpu)); |
3607 | 0 | bql_unlock(); |
3608 | 0 | ret = 0; |
3609 | 0 | break; |
3610 | 0 | default: |
3611 | 0 | ret = kvm_arch_handle_exit(cpu, run); |
3612 | 0 | break; |
3613 | 0 | } |
3614 | 0 | break; |
3615 | 0 | case KVM_EXIT_MEMORY_FAULT: |
3616 | 0 | trace_kvm_memory_fault(run->memory_fault.gpa, |
3617 | 0 | run->memory_fault.size, |
3618 | 0 | run->memory_fault.flags); |
3619 | 0 | if (run->memory_fault.flags & ~KVM_MEMORY_EXIT_FLAG_PRIVATE) { |
3620 | 0 | error_report("KVM_EXIT_MEMORY_FAULT: Unknown flag 0x%" PRIx64, |
3621 | 0 | (uint64_t)run->memory_fault.flags); |
3622 | 0 | ret = -1; |
3623 | 0 | break; |
3624 | 0 | } |
3625 | 0 | ret = kvm_convert_memory(run->memory_fault.gpa, run->memory_fault.size, |
3626 | 0 | run->memory_fault.flags & KVM_MEMORY_EXIT_FLAG_PRIVATE); |
3627 | 0 | break; |
3628 | 0 | default: |
3629 | 0 | ret = kvm_arch_handle_exit(cpu, run); |
3630 | 0 | break; |
3631 | 0 | } |
3632 | 0 | } while (ret == 0); |
3633 | | |
3634 | 0 | cpu_exec_end(cpu); |
3635 | 0 | bql_lock(); |
3636 | |
|
3637 | 0 | if (ret < 0) { |
3638 | 0 | cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); |
3639 | 0 | vm_stop(RUN_STATE_INTERNAL_ERROR); |
3640 | 0 | } |
3641 | |
|
3642 | 0 | return ret; |
3643 | 0 | } |
3644 | | |
3645 | | int kvm_ioctl(KVMState *s, unsigned long type, ...) |
3646 | 0 | { |
3647 | 0 | int ret; |
3648 | 0 | void *arg; |
3649 | 0 | va_list ap; |
3650 | |
|
3651 | 0 | va_start(ap, type); |
3652 | 0 | arg = va_arg(ap, void *); |
3653 | 0 | va_end(ap); |
3654 | |
|
3655 | 0 | trace_kvm_ioctl(type, arg); |
3656 | 0 | ret = ioctl(s->fd, type, arg); |
3657 | 0 | if (ret == -1) { |
3658 | 0 | ret = -errno; |
3659 | 0 | } |
3660 | 0 | return ret; |
3661 | 0 | } |
3662 | | |
3663 | | int kvm_vm_ioctl(KVMState *s, unsigned long type, ...) |
3664 | 0 | { |
3665 | 0 | int ret; |
3666 | 0 | void *arg; |
3667 | 0 | va_list ap; |
3668 | |
|
3669 | 0 | va_start(ap, type); |
3670 | 0 | arg = va_arg(ap, void *); |
3671 | 0 | va_end(ap); |
3672 | |
|
3673 | 0 | trace_kvm_vm_ioctl(type, arg); |
3674 | 0 | accel_ioctl_begin(); |
3675 | 0 | ret = ioctl(s->vmfd, type, arg); |
3676 | 0 | if (ret == -1) { |
3677 | 0 | ret = -errno; |
3678 | 0 | } |
3679 | 0 | accel_ioctl_end(); |
3680 | 0 | return ret; |
3681 | 0 | } |
3682 | | |
3683 | | int kvm_vcpu_ioctl(CPUState *cpu, unsigned long type, ...) |
3684 | 0 | { |
3685 | 0 | int ret; |
3686 | 0 | void *arg; |
3687 | 0 | va_list ap; |
3688 | |
|
3689 | 0 | va_start(ap, type); |
3690 | 0 | arg = va_arg(ap, void *); |
3691 | 0 | va_end(ap); |
3692 | |
|
3693 | 0 | trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg); |
3694 | 0 | accel_cpu_ioctl_begin(cpu); |
3695 | 0 | ret = ioctl(cpu->kvm_fd, type, arg); |
3696 | 0 | accel_cpu_ioctl_end(cpu); |
3697 | 0 | if (ret == -1) { |
3698 | 0 | ret = -errno; |
3699 | 0 | } |
3700 | 0 | return ret; |
3701 | 0 | } |
3702 | | |
3703 | | int kvm_device_ioctl(int fd, unsigned long type, ...) |
3704 | 0 | { |
3705 | 0 | int ret; |
3706 | 0 | void *arg; |
3707 | 0 | va_list ap; |
3708 | |
|
3709 | 0 | va_start(ap, type); |
3710 | 0 | arg = va_arg(ap, void *); |
3711 | 0 | va_end(ap); |
3712 | |
|
3713 | 0 | trace_kvm_device_ioctl(fd, type, arg); |
3714 | 0 | accel_ioctl_begin(); |
3715 | 0 | ret = ioctl(fd, type, arg); |
3716 | 0 | if (ret == -1) { |
3717 | 0 | ret = -errno; |
3718 | 0 | } |
3719 | 0 | accel_ioctl_end(); |
3720 | 0 | return ret; |
3721 | 0 | } |
3722 | | |
3723 | | int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr) |
3724 | 0 | { |
3725 | 0 | int ret; |
3726 | 0 | struct kvm_device_attr attribute = { |
3727 | 0 | .group = group, |
3728 | 0 | .attr = attr, |
3729 | 0 | }; |
3730 | |
|
3731 | 0 | if (!kvm_vm_attributes_allowed) { |
3732 | 0 | return 0; |
3733 | 0 | } |
3734 | | |
3735 | 0 | ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute); |
3736 | | /* kvm returns 0 on success for HAS_DEVICE_ATTR */ |
3737 | 0 | return ret ? 0 : 1; |
3738 | 0 | } |
3739 | | |
3740 | | int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr) |
3741 | 0 | { |
3742 | 0 | struct kvm_device_attr attribute = { |
3743 | 0 | .group = group, |
3744 | 0 | .attr = attr, |
3745 | 0 | .flags = 0, |
3746 | 0 | }; |
3747 | |
|
3748 | 0 | return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1; |
3749 | 0 | } |
3750 | | |
3751 | | int kvm_device_access(int fd, int group, uint64_t attr, |
3752 | | void *val, bool write, Error **errp) |
3753 | 0 | { |
3754 | 0 | struct kvm_device_attr kvmattr; |
3755 | 0 | int err; |
3756 | |
|
3757 | 0 | kvmattr.flags = 0; |
3758 | 0 | kvmattr.group = group; |
3759 | 0 | kvmattr.attr = attr; |
3760 | 0 | kvmattr.addr = (uintptr_t)val; |
3761 | |
|
3762 | 0 | err = kvm_device_ioctl(fd, |
3763 | 0 | write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR, |
3764 | 0 | &kvmattr); |
3765 | 0 | if (err < 0) { |
3766 | 0 | error_setg_errno(errp, -err, |
3767 | 0 | "KVM_%s_DEVICE_ATTR failed: Group %d " |
3768 | 0 | "attr 0x%016" PRIx64, |
3769 | 0 | write ? "SET" : "GET", group, attr); |
3770 | 0 | } |
3771 | 0 | return err; |
3772 | 0 | } |
3773 | | |
3774 | | bool kvm_has_sync_mmu(void) |
3775 | 0 | { |
3776 | 0 | return kvm_state->sync_mmu; |
3777 | 0 | } |
3778 | | |
3779 | | int kvm_has_vcpu_events(void) |
3780 | 0 | { |
3781 | 0 | return kvm_state->vcpu_events; |
3782 | 0 | } |
3783 | | |
3784 | | int kvm_max_nested_state_length(void) |
3785 | 0 | { |
3786 | 0 | return kvm_state->max_nested_state_len; |
3787 | 0 | } |
3788 | | |
3789 | | int kvm_has_gsi_routing(void) |
3790 | 0 | { |
3791 | 0 | #ifdef KVM_CAP_IRQ_ROUTING |
3792 | 0 | return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING); |
3793 | | #else |
3794 | | return false; |
3795 | | #endif |
3796 | 0 | } |
3797 | | |
3798 | | bool kvm_arm_supports_user_irq(void) |
3799 | 0 | { |
3800 | 0 | return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ); |
3801 | 0 | } |
3802 | | |
3803 | | #ifdef TARGET_KVM_HAVE_GUEST_DEBUG |
3804 | | struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu, vaddr pc) |
3805 | 0 | { |
3806 | 0 | struct kvm_sw_breakpoint *bp; |
3807 | |
|
3808 | 0 | QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) { |
3809 | 0 | if (bp->pc == pc) { |
3810 | 0 | return bp; |
3811 | 0 | } |
3812 | 0 | } |
3813 | 0 | return NULL; |
3814 | 0 | } |
3815 | | |
3816 | | int kvm_sw_breakpoints_active(CPUState *cpu) |
3817 | 0 | { |
3818 | 0 | return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints); |
3819 | 0 | } |
3820 | | |
3821 | | struct kvm_set_guest_debug_data { |
3822 | | struct kvm_guest_debug dbg; |
3823 | | int err; |
3824 | | }; |
3825 | | |
3826 | | static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data) |
3827 | 0 | { |
3828 | 0 | struct kvm_set_guest_debug_data *dbg_data = |
3829 | 0 | (struct kvm_set_guest_debug_data *) data.host_ptr; |
3830 | |
|
3831 | 0 | dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG, |
3832 | 0 | &dbg_data->dbg); |
3833 | 0 | } |
3834 | | |
3835 | | int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap) |
3836 | 0 | { |
3837 | 0 | struct kvm_set_guest_debug_data data; |
3838 | |
|
3839 | 0 | data.dbg.control = reinject_trap; |
3840 | |
|
3841 | 0 | if (cpu_single_stepping(cpu)) { |
3842 | 0 | data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP; |
3843 | |
|
3844 | 0 | if (cpu->singlestep_flags & SSTEP_NOIRQ) { |
3845 | 0 | data.dbg.control |= KVM_GUESTDBG_BLOCKIRQ; |
3846 | 0 | } |
3847 | 0 | } |
3848 | 0 | kvm_arch_update_guest_debug(cpu, &data.dbg); |
3849 | |
|
3850 | 0 | run_on_cpu(cpu, kvm_invoke_set_guest_debug, |
3851 | 0 | RUN_ON_CPU_HOST_PTR(&data)); |
3852 | 0 | return data.err; |
3853 | 0 | } |
3854 | | |
3855 | | int kvm_insert_gdbstub_breakpoint(CPUState *cpu, GdbBreakpointType type, |
3856 | | vaddr addr, vaddr len) |
3857 | 0 | { |
3858 | 0 | struct kvm_sw_breakpoint *bp; |
3859 | 0 | int err; |
3860 | |
|
3861 | 0 | if (type == GDB_BREAKPOINT_SW) { |
3862 | 0 | bp = kvm_find_sw_breakpoint(cpu, addr); |
3863 | 0 | if (bp) { |
3864 | 0 | bp->use_count++; |
3865 | 0 | return 0; |
3866 | 0 | } |
3867 | | |
3868 | 0 | bp = g_new(struct kvm_sw_breakpoint, 1); |
3869 | 0 | bp->pc = addr; |
3870 | 0 | bp->use_count = 1; |
3871 | 0 | err = kvm_arch_insert_sw_breakpoint(cpu, bp); |
3872 | 0 | if (err) { |
3873 | 0 | g_free(bp); |
3874 | 0 | return err; |
3875 | 0 | } |
3876 | | |
3877 | 0 | QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); |
3878 | 0 | } else { |
3879 | 0 | err = kvm_arch_insert_gdbstub_hw_breakpoint(addr, len, type); |
3880 | 0 | if (err) { |
3881 | 0 | return err; |
3882 | 0 | } |
3883 | 0 | } |
3884 | | |
3885 | 0 | CPU_FOREACH(cpu) { |
3886 | 0 | err = kvm_update_guest_debug(cpu, 0); |
3887 | 0 | if (err) { |
3888 | 0 | return err; |
3889 | 0 | } |
3890 | 0 | } |
3891 | 0 | return 0; |
3892 | 0 | } |
3893 | | |
3894 | | int kvm_remove_gdbstub_breakpoint(CPUState *cpu, GdbBreakpointType type, |
3895 | | vaddr addr, vaddr len) |
3896 | 0 | { |
3897 | 0 | struct kvm_sw_breakpoint *bp; |
3898 | 0 | int err; |
3899 | |
|
3900 | 0 | if (type == GDB_BREAKPOINT_SW) { |
3901 | 0 | bp = kvm_find_sw_breakpoint(cpu, addr); |
3902 | 0 | if (!bp) { |
3903 | 0 | return -ENOENT; |
3904 | 0 | } |
3905 | | |
3906 | 0 | if (bp->use_count > 1) { |
3907 | 0 | bp->use_count--; |
3908 | 0 | return 0; |
3909 | 0 | } |
3910 | | |
3911 | 0 | err = kvm_arch_remove_sw_breakpoint(cpu, bp); |
3912 | 0 | if (err) { |
3913 | 0 | return err; |
3914 | 0 | } |
3915 | | |
3916 | 0 | QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); |
3917 | 0 | g_free(bp); |
3918 | 0 | } else { |
3919 | 0 | err = kvm_arch_remove_gdbstub_hw_breakpoint(addr, len, type); |
3920 | 0 | if (err) { |
3921 | 0 | return err; |
3922 | 0 | } |
3923 | 0 | } |
3924 | | |
3925 | 0 | CPU_FOREACH(cpu) { |
3926 | 0 | err = kvm_update_guest_debug(cpu, 0); |
3927 | 0 | if (err) { |
3928 | 0 | return err; |
3929 | 0 | } |
3930 | 0 | } |
3931 | 0 | return 0; |
3932 | 0 | } |
3933 | | |
3934 | | void kvm_remove_all_gdbstub_breakpoints(CPUState *cpu) |
3935 | 0 | { |
3936 | 0 | struct kvm_sw_breakpoint *bp, *next; |
3937 | 0 | KVMState *s = cpu->kvm_state; |
3938 | 0 | CPUState *tmpcpu; |
3939 | |
|
3940 | 0 | QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) { |
3941 | 0 | if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) { |
3942 | | /* Try harder to find a CPU that currently sees the breakpoint. */ |
3943 | 0 | CPU_FOREACH(tmpcpu) { |
3944 | 0 | if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) { |
3945 | 0 | break; |
3946 | 0 | } |
3947 | 0 | } |
3948 | 0 | } |
3949 | 0 | QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry); |
3950 | 0 | g_free(bp); |
3951 | 0 | } |
3952 | 0 | kvm_arch_remove_all_gdbstub_hw_breakpoints(); |
3953 | |
|
3954 | 0 | CPU_FOREACH(cpu) { |
3955 | 0 | kvm_update_guest_debug(cpu, 0); |
3956 | 0 | } |
3957 | 0 | } |
3958 | | |
3959 | | #endif /* !TARGET_KVM_HAVE_GUEST_DEBUG */ |
3960 | | |
3961 | | static int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset) |
3962 | 0 | { |
3963 | 0 | KVMState *s = kvm_state; |
3964 | 0 | struct kvm_signal_mask *sigmask; |
3965 | 0 | int r; |
3966 | |
|
3967 | 0 | sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset)); |
3968 | |
|
3969 | 0 | sigmask->len = s->sigmask_len; |
3970 | 0 | memcpy(sigmask->sigset, sigset, sizeof(*sigset)); |
3971 | 0 | r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask); |
3972 | 0 | g_free(sigmask); |
3973 | |
|
3974 | 0 | return r; |
3975 | 0 | } |
3976 | | |
3977 | | static void kvm_ipi_signal(int sig) |
3978 | 0 | { |
3979 | 0 | if (current_cpu) { |
3980 | 0 | assert(kvm_immediate_exit); |
3981 | 0 | kvm_cpu_kick(current_cpu); |
3982 | 0 | } |
3983 | 0 | } |
3984 | | |
3985 | | void kvm_init_cpu_signals(CPUState *cpu) |
3986 | 0 | { |
3987 | 0 | int r; |
3988 | 0 | sigset_t set; |
3989 | 0 | struct sigaction sigact; |
3990 | |
|
3991 | 0 | memset(&sigact, 0, sizeof(sigact)); |
3992 | 0 | sigact.sa_handler = kvm_ipi_signal; |
3993 | 0 | sigaction(SIG_IPI, &sigact, NULL); |
3994 | |
|
3995 | 0 | pthread_sigmask(SIG_BLOCK, NULL, &set); |
3996 | 0 | #if defined KVM_HAVE_MCE_INJECTION |
3997 | 0 | sigdelset(&set, SIGBUS); |
3998 | 0 | pthread_sigmask(SIG_SETMASK, &set, NULL); |
3999 | 0 | #endif |
4000 | 0 | sigdelset(&set, SIG_IPI); |
4001 | 0 | if (kvm_immediate_exit) { |
4002 | 0 | r = pthread_sigmask(SIG_SETMASK, &set, NULL); |
4003 | 0 | } else { |
4004 | 0 | r = kvm_set_signal_mask(cpu, &set); |
4005 | 0 | } |
4006 | 0 | if (r) { |
4007 | 0 | fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r)); |
4008 | 0 | exit(1); |
4009 | 0 | } |
4010 | 0 | } |
4011 | | |
4012 | | /* Called asynchronously in VCPU thread. */ |
4013 | | int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr) |
4014 | 0 | { |
4015 | 0 | #ifdef KVM_HAVE_MCE_INJECTION |
4016 | 0 | if (have_sigbus_pending) { |
4017 | 0 | return 1; |
4018 | 0 | } |
4019 | 0 | have_sigbus_pending = true; |
4020 | 0 | pending_sigbus_addr = addr; |
4021 | 0 | pending_sigbus_code = code; |
4022 | 0 | qatomic_set(&cpu->exit_request, true); |
4023 | 0 | return 0; |
4024 | | #else |
4025 | | return 1; |
4026 | | #endif |
4027 | 0 | } |
4028 | | |
4029 | | /* Called synchronously (via signalfd) in main thread. */ |
4030 | | int kvm_on_sigbus(int code, void *addr) |
4031 | 0 | { |
4032 | 0 | #ifdef KVM_HAVE_MCE_INJECTION |
4033 | | /* Action required MCE kills the process if SIGBUS is blocked. Because |
4034 | | * that's what happens in the I/O thread, where we handle MCE via signalfd, |
4035 | | * we can only get action optional here. |
4036 | | */ |
4037 | 0 | assert(code != BUS_MCEERR_AR); |
4038 | 0 | kvm_arch_on_sigbus_vcpu(first_cpu, code, addr); |
4039 | 0 | return 0; |
4040 | | #else |
4041 | | return 1; |
4042 | | #endif |
4043 | 0 | } |
4044 | | |
4045 | | int kvm_create_device(KVMState *s, uint64_t type, bool test) |
4046 | 0 | { |
4047 | 0 | int ret; |
4048 | 0 | struct kvm_create_device create_dev; |
4049 | |
|
4050 | 0 | create_dev.type = type; |
4051 | 0 | create_dev.fd = -1; |
4052 | 0 | create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0; |
4053 | |
|
4054 | 0 | if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) { |
4055 | 0 | return -ENOTSUP; |
4056 | 0 | } |
4057 | | |
4058 | 0 | ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev); |
4059 | 0 | if (ret) { |
4060 | 0 | return ret; |
4061 | 0 | } |
4062 | | |
4063 | 0 | return test ? 0 : create_dev.fd; |
4064 | 0 | } |
4065 | | |
4066 | | bool kvm_device_supported(int vmfd, uint64_t type) |
4067 | 0 | { |
4068 | 0 | struct kvm_create_device create_dev = { |
4069 | 0 | .type = type, |
4070 | 0 | .fd = -1, |
4071 | 0 | .flags = KVM_CREATE_DEVICE_TEST, |
4072 | 0 | }; |
4073 | |
|
4074 | 0 | if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) { |
4075 | 0 | return false; |
4076 | 0 | } |
4077 | | |
4078 | 0 | return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0); |
4079 | 0 | } |
4080 | | |
4081 | | int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source) |
4082 | 0 | { |
4083 | 0 | struct kvm_one_reg reg; |
4084 | 0 | int r; |
4085 | |
|
4086 | 0 | reg.id = id; |
4087 | 0 | reg.addr = (uintptr_t) source; |
4088 | 0 | r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); |
4089 | 0 | if (r) { |
4090 | 0 | trace_kvm_failed_reg_set(id, strerror(-r)); |
4091 | 0 | } |
4092 | 0 | return r; |
4093 | 0 | } |
4094 | | |
4095 | | int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target) |
4096 | 0 | { |
4097 | 0 | struct kvm_one_reg reg; |
4098 | 0 | int r; |
4099 | |
|
4100 | 0 | reg.id = id; |
4101 | 0 | reg.addr = (uintptr_t) target; |
4102 | 0 | r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); |
4103 | 0 | if (r) { |
4104 | 0 | trace_kvm_failed_reg_get(id, strerror(-r)); |
4105 | 0 | } |
4106 | 0 | return r; |
4107 | 0 | } |
4108 | | |
4109 | | static bool kvm_accel_has_memory(AccelState *accel, AddressSpace *as, |
4110 | | hwaddr start_addr, hwaddr size) |
4111 | 0 | { |
4112 | 0 | KVMState *kvm = KVM_STATE(accel); |
4113 | 0 | int i; |
4114 | |
|
4115 | 0 | for (i = 0; i < kvm->nr_as; ++i) { |
4116 | 0 | if (kvm->as[i].as == as && kvm->as[i].ml) { |
4117 | 0 | size = MIN(kvm_max_slot_size, size); |
4118 | 0 | return NULL != kvm_lookup_matching_slot(kvm->as[i].ml, |
4119 | 0 | start_addr, size); |
4120 | 0 | } |
4121 | 0 | } |
4122 | | |
4123 | 0 | return false; |
4124 | 0 | } |
4125 | | |
4126 | | static void kvm_get_kvm_shadow_mem(Object *obj, Visitor *v, |
4127 | | const char *name, void *opaque, |
4128 | | Error **errp) |
4129 | 0 | { |
4130 | 0 | KVMState *s = KVM_STATE(obj); |
4131 | 0 | int64_t value = s->kvm_shadow_mem; |
4132 | |
|
4133 | 0 | visit_type_int(v, name, &value, errp); |
4134 | 0 | } |
4135 | | |
4136 | | static void kvm_set_kvm_shadow_mem(Object *obj, Visitor *v, |
4137 | | const char *name, void *opaque, |
4138 | | Error **errp) |
4139 | 0 | { |
4140 | 0 | KVMState *s = KVM_STATE(obj); |
4141 | 0 | int64_t value; |
4142 | |
|
4143 | 0 | if (s->fd != -1) { |
4144 | 0 | error_setg(errp, "Cannot set properties after the accelerator has been initialized"); |
4145 | 0 | return; |
4146 | 0 | } |
4147 | | |
4148 | 0 | if (!visit_type_int(v, name, &value, errp)) { |
4149 | 0 | return; |
4150 | 0 | } |
4151 | | |
4152 | 0 | s->kvm_shadow_mem = value; |
4153 | 0 | } |
4154 | | |
4155 | | static void kvm_set_kernel_irqchip(Object *obj, int value, |
4156 | | Error **errp) |
4157 | 0 | { |
4158 | 0 | KVMState *s = KVM_STATE(obj); |
4159 | |
|
4160 | 0 | if (s->fd != -1) { |
4161 | 0 | error_setg(errp, "Cannot set properties after the accelerator has been initialized"); |
4162 | 0 | return; |
4163 | 0 | } |
4164 | | |
4165 | 0 | switch (value) { |
4166 | 0 | case ON_OFF_SPLIT_ON: |
4167 | 0 | s->kernel_irqchip_allowed = true; |
4168 | 0 | s->kernel_irqchip_required = true; |
4169 | 0 | s->kernel_irqchip_split = ON_OFF_AUTO_OFF; |
4170 | 0 | break; |
4171 | 0 | case ON_OFF_SPLIT_OFF: |
4172 | 0 | s->kernel_irqchip_allowed = false; |
4173 | 0 | s->kernel_irqchip_required = false; |
4174 | 0 | s->kernel_irqchip_split = ON_OFF_AUTO_OFF; |
4175 | 0 | break; |
4176 | 0 | case ON_OFF_SPLIT_SPLIT: |
4177 | 0 | s->kernel_irqchip_allowed = true; |
4178 | 0 | s->kernel_irqchip_required = true; |
4179 | 0 | s->kernel_irqchip_split = ON_OFF_AUTO_ON; |
4180 | 0 | break; |
4181 | 0 | default: |
4182 | 0 | g_assert_not_reached(); |
4183 | 0 | } |
4184 | 0 | } |
4185 | | |
4186 | | bool kvm_kernel_irqchip_allowed(void) |
4187 | 0 | { |
4188 | 0 | return kvm_state->kernel_irqchip_allowed; |
4189 | 0 | } |
4190 | | |
4191 | | bool kvm_kernel_irqchip_required(void) |
4192 | 0 | { |
4193 | 0 | return kvm_state->kernel_irqchip_required; |
4194 | 0 | } |
4195 | | |
4196 | | bool kvm_kernel_irqchip_split(void) |
4197 | 0 | { |
4198 | 0 | return kvm_state->kernel_irqchip_split == ON_OFF_AUTO_ON; |
4199 | 0 | } |
4200 | | |
4201 | | static void kvm_get_dirty_ring_size(Object *obj, Visitor *v, |
4202 | | const char *name, void *opaque, |
4203 | | Error **errp) |
4204 | 0 | { |
4205 | 0 | KVMState *s = KVM_STATE(obj); |
4206 | 0 | uint32_t value = s->kvm_dirty_ring_size; |
4207 | |
|
4208 | 0 | visit_type_uint32(v, name, &value, errp); |
4209 | 0 | } |
4210 | | |
4211 | | static void kvm_set_dirty_ring_size(Object *obj, Visitor *v, |
4212 | | const char *name, void *opaque, |
4213 | | Error **errp) |
4214 | 0 | { |
4215 | 0 | KVMState *s = KVM_STATE(obj); |
4216 | 0 | uint32_t value; |
4217 | |
|
4218 | 0 | if (s->fd != -1) { |
4219 | 0 | error_setg(errp, "Cannot set properties after the accelerator has been initialized"); |
4220 | 0 | return; |
4221 | 0 | } |
4222 | | |
4223 | 0 | if (!visit_type_uint32(v, name, &value, errp)) { |
4224 | 0 | return; |
4225 | 0 | } |
4226 | 0 | if (value & (value - 1)) { |
4227 | 0 | error_setg(errp, "dirty-ring-size must be a power of two."); |
4228 | 0 | return; |
4229 | 0 | } |
4230 | | |
4231 | 0 | s->kvm_dirty_ring_size = value; |
4232 | 0 | } |
4233 | | |
4234 | | static char *kvm_get_device(Object *obj, |
4235 | | Error **errp G_GNUC_UNUSED) |
4236 | 0 | { |
4237 | 0 | KVMState *s = KVM_STATE(obj); |
4238 | |
|
4239 | 0 | return g_strdup(s->device); |
4240 | 0 | } |
4241 | | |
4242 | | static void kvm_set_device(Object *obj, |
4243 | | const char *value, |
4244 | | Error **errp G_GNUC_UNUSED) |
4245 | 0 | { |
4246 | 0 | KVMState *s = KVM_STATE(obj); |
4247 | |
|
4248 | 0 | g_free(s->device); |
4249 | 0 | s->device = g_strdup(value); |
4250 | 0 | } |
4251 | | |
4252 | | static void kvm_set_kvm_rapl(Object *obj, bool value, Error **errp) |
4253 | 0 | { |
4254 | 0 | KVMState *s = KVM_STATE(obj); |
4255 | 0 | s->msr_energy.enable = value; |
4256 | 0 | } |
4257 | | |
4258 | | static void kvm_set_kvm_rapl_socket_path(Object *obj, |
4259 | | const char *str, |
4260 | | Error **errp) |
4261 | 0 | { |
4262 | 0 | KVMState *s = KVM_STATE(obj); |
4263 | 0 | g_free(s->msr_energy.socket_path); |
4264 | 0 | s->msr_energy.socket_path = g_strdup(str); |
4265 | 0 | } |
4266 | | |
4267 | | static void kvm_accel_instance_init(Object *obj) |
4268 | 0 | { |
4269 | 0 | KVMState *s = KVM_STATE(obj); |
4270 | |
|
4271 | 0 | s->fd = -1; |
4272 | 0 | s->vmfd = -1; |
4273 | 0 | s->kvm_shadow_mem = -1; |
4274 | 0 | s->kernel_irqchip_allowed = true; |
4275 | 0 | s->kernel_irqchip_split = ON_OFF_AUTO_AUTO; |
4276 | | /* KVM dirty ring is by default off */ |
4277 | 0 | s->kvm_dirty_ring_size = 0; |
4278 | 0 | s->kvm_dirty_ring_with_bitmap = false; |
4279 | 0 | s->kvm_eager_split_size = 0; |
4280 | 0 | s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN; |
4281 | 0 | s->notify_window = 0; |
4282 | 0 | s->xen_version = 0; |
4283 | 0 | s->xen_gnttab_max_frames = 64; |
4284 | 0 | s->xen_evtchn_max_pirq = 256; |
4285 | 0 | s->device = NULL; |
4286 | 0 | s->msr_energy.enable = false; |
4287 | 0 | s->honor_guest_pat = ON_OFF_AUTO_OFF; |
4288 | 0 | } |
4289 | | |
4290 | | static void kvm_accel_class_init(ObjectClass *oc, const void *data) |
4291 | 63 | { |
4292 | 63 | AccelClass *ac = ACCEL_CLASS(oc); |
4293 | 63 | ac->name = "KVM"; |
4294 | 63 | ac->init_machine = kvm_init; |
4295 | 63 | ac->rebuild_guest = kvm_reset_vmfd; |
4296 | 63 | ac->has_memory = kvm_accel_has_memory; |
4297 | 63 | ac->allowed = &kvm_allowed; |
4298 | | |
4299 | 63 | object_class_property_add_enum(oc, "kernel-irqchip", "on|off|split", |
4300 | 63 | &OnOffSplit_type_info, NULL, |
4301 | 63 | kvm_set_kernel_irqchip); |
4302 | 63 | object_class_property_set_description(oc, "kernel-irqchip", |
4303 | 63 | "Configure KVM in-kernel irqchip"); |
4304 | | |
4305 | 63 | object_class_property_add(oc, "kvm-shadow-mem", "int", &int_type_info, |
4306 | 63 | kvm_get_kvm_shadow_mem, kvm_set_kvm_shadow_mem, |
4307 | 63 | NULL, NULL); |
4308 | 63 | object_class_property_set_description(oc, "kvm-shadow-mem", |
4309 | 63 | "KVM shadow MMU size"); |
4310 | | |
4311 | 63 | object_class_property_add(oc, "dirty-ring-size", "uint32", |
4312 | 63 | &uint32_type_info, |
4313 | 63 | kvm_get_dirty_ring_size, kvm_set_dirty_ring_size, |
4314 | 63 | NULL, NULL); |
4315 | 63 | object_class_property_set_description(oc, "dirty-ring-size", |
4316 | 63 | "Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)"); |
4317 | | |
4318 | 63 | object_class_property_add_str(oc, "device", kvm_get_device, kvm_set_device); |
4319 | 63 | object_class_property_set_description(oc, "device", |
4320 | 63 | "Path to the device node to use (default: /dev/kvm)"); |
4321 | | |
4322 | 63 | object_class_property_add_bool(oc, "rapl", |
4323 | 63 | NULL, |
4324 | 63 | kvm_set_kvm_rapl); |
4325 | 63 | object_class_property_set_description(oc, "rapl", |
4326 | 63 | "Allow energy related MSRs for RAPL interface in Guest"); |
4327 | | |
4328 | 63 | object_class_property_add_str(oc, "rapl-helper-socket", NULL, |
4329 | 63 | kvm_set_kvm_rapl_socket_path); |
4330 | 63 | object_class_property_set_description(oc, "rapl-helper-socket", |
4331 | 63 | "Socket Path for comminucating with the Virtual MSR helper daemon"); |
4332 | | |
4333 | 63 | kvm_arch_accel_class_init(oc); |
4334 | 63 | } |
4335 | | |
4336 | | static void kvm_accel_finalize(Object *obj) |
4337 | 0 | { |
4338 | 0 | KVMState *s = KVM_STATE(obj); |
4339 | |
|
4340 | 0 | g_free(s->device); |
4341 | 0 | } |
4342 | | |
4343 | | static const TypeInfo kvm_accel_type = { |
4344 | | .name = TYPE_KVM_ACCEL, |
4345 | | .parent = TYPE_ACCEL, |
4346 | | .instance_init = kvm_accel_instance_init, |
4347 | | .instance_finalize = kvm_accel_finalize, |
4348 | | .class_init = kvm_accel_class_init, |
4349 | | .instance_size = sizeof(KVMState), |
4350 | | .secure = true, |
4351 | | }; |
4352 | | |
4353 | | static void kvm_type_init(void) |
4354 | 63 | { |
4355 | 63 | type_register_static(&kvm_accel_type); |
4356 | 63 | } |
4357 | | |
4358 | | type_init(kvm_type_init); |
4359 | | |
4360 | | typedef struct StatsArgs { |
4361 | | union StatsResultsType { |
4362 | | StatsResultList **stats; |
4363 | | StatsSchemaList **schema; |
4364 | | } result; |
4365 | | strList *names; |
4366 | | Error **errp; |
4367 | | } StatsArgs; |
4368 | | |
4369 | | static StatsList *add_kvmstat_entry(struct kvm_stats_desc *pdesc, |
4370 | | uint64_t *stats_data, |
4371 | | StatsList *stats_list, |
4372 | | Error **errp) |
4373 | 0 | { |
4374 | |
|
4375 | 0 | Stats *stats; |
4376 | 0 | uint64List *val_list = NULL; |
4377 | | |
4378 | | /* Only add stats that we understand. */ |
4379 | 0 | switch (pdesc->flags & KVM_STATS_TYPE_MASK) { |
4380 | 0 | case KVM_STATS_TYPE_CUMULATIVE: |
4381 | 0 | case KVM_STATS_TYPE_INSTANT: |
4382 | 0 | case KVM_STATS_TYPE_PEAK: |
4383 | 0 | case KVM_STATS_TYPE_LINEAR_HIST: |
4384 | 0 | case KVM_STATS_TYPE_LOG_HIST: |
4385 | 0 | break; |
4386 | 0 | default: |
4387 | 0 | return stats_list; |
4388 | 0 | } |
4389 | | |
4390 | 0 | switch (pdesc->flags & KVM_STATS_UNIT_MASK) { |
4391 | 0 | case KVM_STATS_UNIT_NONE: |
4392 | 0 | case KVM_STATS_UNIT_BYTES: |
4393 | 0 | case KVM_STATS_UNIT_CYCLES: |
4394 | 0 | case KVM_STATS_UNIT_SECONDS: |
4395 | 0 | case KVM_STATS_UNIT_BOOLEAN: |
4396 | 0 | break; |
4397 | 0 | default: |
4398 | 0 | return stats_list; |
4399 | 0 | } |
4400 | | |
4401 | 0 | switch (pdesc->flags & KVM_STATS_BASE_MASK) { |
4402 | 0 | case KVM_STATS_BASE_POW10: |
4403 | 0 | case KVM_STATS_BASE_POW2: |
4404 | 0 | break; |
4405 | 0 | default: |
4406 | 0 | return stats_list; |
4407 | 0 | } |
4408 | | |
4409 | | /* Alloc and populate data list */ |
4410 | 0 | stats = g_new0(Stats, 1); |
4411 | 0 | stats->name = g_strdup(pdesc->name); |
4412 | 0 | stats->value = g_new0(StatsValue, 1); |
4413 | |
|
4414 | 0 | if ((pdesc->flags & KVM_STATS_UNIT_MASK) == KVM_STATS_UNIT_BOOLEAN) { |
4415 | 0 | stats->value->u.boolean = *stats_data; |
4416 | 0 | stats->value->type = QTYPE_QBOOL; |
4417 | 0 | } else if (pdesc->size == 1) { |
4418 | 0 | stats->value->u.scalar = *stats_data; |
4419 | 0 | stats->value->type = QTYPE_QNUM; |
4420 | 0 | } else { |
4421 | 0 | int i; |
4422 | 0 | for (i = 0; i < pdesc->size; i++) { |
4423 | 0 | QAPI_LIST_PREPEND(val_list, stats_data[i]); |
4424 | 0 | } |
4425 | 0 | stats->value->u.list = val_list; |
4426 | 0 | stats->value->type = QTYPE_QLIST; |
4427 | 0 | } |
4428 | |
|
4429 | 0 | QAPI_LIST_PREPEND(stats_list, stats); |
4430 | 0 | return stats_list; |
4431 | 0 | } |
4432 | | |
4433 | | static StatsSchemaValueList *add_kvmschema_entry(struct kvm_stats_desc *pdesc, |
4434 | | StatsSchemaValueList *list, |
4435 | | Error **errp) |
4436 | 0 | { |
4437 | 0 | StatsSchemaValueList *schema_entry = g_new0(StatsSchemaValueList, 1); |
4438 | 0 | schema_entry->value = g_new0(StatsSchemaValue, 1); |
4439 | |
|
4440 | 0 | switch (pdesc->flags & KVM_STATS_TYPE_MASK) { |
4441 | 0 | case KVM_STATS_TYPE_CUMULATIVE: |
4442 | 0 | schema_entry->value->type = STATS_TYPE_CUMULATIVE; |
4443 | 0 | break; |
4444 | 0 | case KVM_STATS_TYPE_INSTANT: |
4445 | 0 | schema_entry->value->type = STATS_TYPE_INSTANT; |
4446 | 0 | break; |
4447 | 0 | case KVM_STATS_TYPE_PEAK: |
4448 | 0 | schema_entry->value->type = STATS_TYPE_PEAK; |
4449 | 0 | break; |
4450 | 0 | case KVM_STATS_TYPE_LINEAR_HIST: |
4451 | 0 | schema_entry->value->type = STATS_TYPE_LINEAR_HISTOGRAM; |
4452 | 0 | schema_entry->value->bucket_size = pdesc->bucket_size; |
4453 | 0 | schema_entry->value->has_bucket_size = true; |
4454 | 0 | break; |
4455 | 0 | case KVM_STATS_TYPE_LOG_HIST: |
4456 | 0 | schema_entry->value->type = STATS_TYPE_LOG2_HISTOGRAM; |
4457 | 0 | break; |
4458 | 0 | default: |
4459 | 0 | goto exit; |
4460 | 0 | } |
4461 | | |
4462 | 0 | switch (pdesc->flags & KVM_STATS_UNIT_MASK) { |
4463 | 0 | case KVM_STATS_UNIT_NONE: |
4464 | 0 | break; |
4465 | 0 | case KVM_STATS_UNIT_BOOLEAN: |
4466 | 0 | schema_entry->value->has_unit = true; |
4467 | 0 | schema_entry->value->unit = STATS_UNIT_BOOLEAN; |
4468 | 0 | break; |
4469 | 0 | case KVM_STATS_UNIT_BYTES: |
4470 | 0 | schema_entry->value->has_unit = true; |
4471 | 0 | schema_entry->value->unit = STATS_UNIT_BYTES; |
4472 | 0 | break; |
4473 | 0 | case KVM_STATS_UNIT_CYCLES: |
4474 | 0 | schema_entry->value->has_unit = true; |
4475 | 0 | schema_entry->value->unit = STATS_UNIT_CYCLES; |
4476 | 0 | break; |
4477 | 0 | case KVM_STATS_UNIT_SECONDS: |
4478 | 0 | schema_entry->value->has_unit = true; |
4479 | 0 | schema_entry->value->unit = STATS_UNIT_SECONDS; |
4480 | 0 | break; |
4481 | 0 | default: |
4482 | 0 | goto exit; |
4483 | 0 | } |
4484 | | |
4485 | 0 | schema_entry->value->exponent = pdesc->exponent; |
4486 | 0 | if (pdesc->exponent) { |
4487 | 0 | switch (pdesc->flags & KVM_STATS_BASE_MASK) { |
4488 | 0 | case KVM_STATS_BASE_POW10: |
4489 | 0 | schema_entry->value->has_base = true; |
4490 | 0 | schema_entry->value->base = 10; |
4491 | 0 | break; |
4492 | 0 | case KVM_STATS_BASE_POW2: |
4493 | 0 | schema_entry->value->has_base = true; |
4494 | 0 | schema_entry->value->base = 2; |
4495 | 0 | break; |
4496 | 0 | default: |
4497 | 0 | goto exit; |
4498 | 0 | } |
4499 | 0 | } |
4500 | | |
4501 | 0 | schema_entry->value->name = g_strdup(pdesc->name); |
4502 | 0 | schema_entry->next = list; |
4503 | 0 | return schema_entry; |
4504 | 0 | exit: |
4505 | 0 | g_free(schema_entry->value); |
4506 | 0 | g_free(schema_entry); |
4507 | 0 | return list; |
4508 | 0 | } |
4509 | | |
4510 | | /* Cached stats descriptors */ |
4511 | | typedef struct StatsDescriptors { |
4512 | | const char *ident; /* cache key, currently the StatsTarget */ |
4513 | | struct kvm_stats_desc *kvm_stats_desc; |
4514 | | struct kvm_stats_header kvm_stats_header; |
4515 | | QTAILQ_ENTRY(StatsDescriptors) next; |
4516 | | } StatsDescriptors; |
4517 | | |
4518 | | static QTAILQ_HEAD(, StatsDescriptors) stats_descriptors = |
4519 | | QTAILQ_HEAD_INITIALIZER(stats_descriptors); |
4520 | | |
4521 | | /* |
4522 | | * Return the descriptors for 'target', that either have already been read |
4523 | | * or are retrieved from 'stats_fd'. |
4524 | | */ |
4525 | | static StatsDescriptors *find_stats_descriptors(StatsTarget target, int stats_fd, |
4526 | | Error **errp) |
4527 | 0 | { |
4528 | 0 | StatsDescriptors *descriptors; |
4529 | 0 | const char *ident; |
4530 | 0 | struct kvm_stats_desc *kvm_stats_desc; |
4531 | 0 | struct kvm_stats_header *kvm_stats_header; |
4532 | 0 | size_t size_desc; |
4533 | 0 | ssize_t ret; |
4534 | |
|
4535 | 0 | ident = StatsTarget_str(target); |
4536 | 0 | QTAILQ_FOREACH(descriptors, &stats_descriptors, next) { |
4537 | 0 | if (g_str_equal(descriptors->ident, ident)) { |
4538 | 0 | return descriptors; |
4539 | 0 | } |
4540 | 0 | } |
4541 | | |
4542 | 0 | descriptors = g_new0(StatsDescriptors, 1); |
4543 | | |
4544 | | /* Read stats header */ |
4545 | 0 | kvm_stats_header = &descriptors->kvm_stats_header; |
4546 | 0 | ret = pread(stats_fd, kvm_stats_header, sizeof(*kvm_stats_header), 0); |
4547 | 0 | if (ret != sizeof(*kvm_stats_header)) { |
4548 | 0 | error_setg(errp, "KVM stats: failed to read stats header: " |
4549 | 0 | "expected %zu actual %zu", |
4550 | 0 | sizeof(*kvm_stats_header), ret); |
4551 | 0 | g_free(descriptors); |
4552 | 0 | return NULL; |
4553 | 0 | } |
4554 | 0 | size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size; |
4555 | | |
4556 | | /* Read stats descriptors */ |
4557 | 0 | kvm_stats_desc = g_malloc0_n(kvm_stats_header->num_desc, size_desc); |
4558 | 0 | ret = pread(stats_fd, kvm_stats_desc, |
4559 | 0 | size_desc * kvm_stats_header->num_desc, |
4560 | 0 | kvm_stats_header->desc_offset); |
4561 | |
|
4562 | 0 | if (ret != size_desc * kvm_stats_header->num_desc) { |
4563 | 0 | error_setg(errp, "KVM stats: failed to read stats descriptors: " |
4564 | 0 | "expected %zu actual %zu", |
4565 | 0 | size_desc * kvm_stats_header->num_desc, ret); |
4566 | 0 | g_free(descriptors); |
4567 | 0 | g_free(kvm_stats_desc); |
4568 | 0 | return NULL; |
4569 | 0 | } |
4570 | 0 | descriptors->kvm_stats_desc = kvm_stats_desc; |
4571 | 0 | descriptors->ident = ident; |
4572 | 0 | QTAILQ_INSERT_TAIL(&stats_descriptors, descriptors, next); |
4573 | 0 | return descriptors; |
4574 | 0 | } |
4575 | | |
4576 | | static void query_stats(StatsResultList **result, StatsTarget target, |
4577 | | strList *names, int stats_fd, CPUState *cpu, |
4578 | | Error **errp) |
4579 | 0 | { |
4580 | 0 | struct kvm_stats_desc *kvm_stats_desc; |
4581 | 0 | struct kvm_stats_header *kvm_stats_header; |
4582 | 0 | StatsDescriptors *descriptors; |
4583 | 0 | g_autofree uint64_t *stats_data = NULL; |
4584 | 0 | struct kvm_stats_desc *pdesc; |
4585 | 0 | StatsList *stats_list = NULL; |
4586 | 0 | size_t size_desc, size_data = 0; |
4587 | 0 | ssize_t ret; |
4588 | 0 | int i; |
4589 | |
|
4590 | 0 | descriptors = find_stats_descriptors(target, stats_fd, errp); |
4591 | 0 | if (!descriptors) { |
4592 | 0 | return; |
4593 | 0 | } |
4594 | | |
4595 | 0 | kvm_stats_header = &descriptors->kvm_stats_header; |
4596 | 0 | kvm_stats_desc = descriptors->kvm_stats_desc; |
4597 | 0 | size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size; |
4598 | | |
4599 | | /* Tally the total data size; read schema data */ |
4600 | 0 | for (i = 0; i < kvm_stats_header->num_desc; ++i) { |
4601 | 0 | pdesc = (void *)kvm_stats_desc + i * size_desc; |
4602 | 0 | size_data += pdesc->size * sizeof(*stats_data); |
4603 | 0 | } |
4604 | |
|
4605 | 0 | stats_data = g_malloc0(size_data); |
4606 | 0 | ret = pread(stats_fd, stats_data, size_data, kvm_stats_header->data_offset); |
4607 | |
|
4608 | 0 | if (ret != size_data) { |
4609 | 0 | error_setg(errp, "KVM stats: failed to read data: " |
4610 | 0 | "expected %zu actual %zu", size_data, ret); |
4611 | 0 | return; |
4612 | 0 | } |
4613 | | |
4614 | 0 | for (i = 0; i < kvm_stats_header->num_desc; ++i) { |
4615 | 0 | uint64_t *stats; |
4616 | 0 | pdesc = (void *)kvm_stats_desc + i * size_desc; |
4617 | | |
4618 | | /* Add entry to the list */ |
4619 | 0 | stats = (void *)stats_data + pdesc->offset; |
4620 | 0 | if (!apply_str_list_filter(pdesc->name, names)) { |
4621 | 0 | continue; |
4622 | 0 | } |
4623 | 0 | stats_list = add_kvmstat_entry(pdesc, stats, stats_list, errp); |
4624 | 0 | } |
4625 | |
|
4626 | 0 | if (!stats_list) { |
4627 | 0 | return; |
4628 | 0 | } |
4629 | | |
4630 | 0 | switch (target) { |
4631 | 0 | case STATS_TARGET_VM: |
4632 | 0 | add_stats_entry(result, STATS_PROVIDER_KVM, NULL, stats_list); |
4633 | 0 | break; |
4634 | 0 | case STATS_TARGET_VCPU: |
4635 | 0 | add_stats_entry(result, STATS_PROVIDER_KVM, |
4636 | 0 | cpu->parent_obj.canonical_path, |
4637 | 0 | stats_list); |
4638 | 0 | break; |
4639 | 0 | default: |
4640 | 0 | g_assert_not_reached(); |
4641 | 0 | } |
4642 | 0 | } |
4643 | | |
4644 | | static void query_stats_schema(StatsSchemaList **result, StatsTarget target, |
4645 | | int stats_fd, Error **errp) |
4646 | 0 | { |
4647 | 0 | struct kvm_stats_desc *kvm_stats_desc; |
4648 | 0 | struct kvm_stats_header *kvm_stats_header; |
4649 | 0 | StatsDescriptors *descriptors; |
4650 | 0 | struct kvm_stats_desc *pdesc; |
4651 | 0 | StatsSchemaValueList *stats_list = NULL; |
4652 | 0 | size_t size_desc; |
4653 | 0 | int i; |
4654 | |
|
4655 | 0 | descriptors = find_stats_descriptors(target, stats_fd, errp); |
4656 | 0 | if (!descriptors) { |
4657 | 0 | return; |
4658 | 0 | } |
4659 | | |
4660 | 0 | kvm_stats_header = &descriptors->kvm_stats_header; |
4661 | 0 | kvm_stats_desc = descriptors->kvm_stats_desc; |
4662 | 0 | size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size; |
4663 | | |
4664 | | /* Tally the total data size; read schema data */ |
4665 | 0 | for (i = 0; i < kvm_stats_header->num_desc; ++i) { |
4666 | 0 | pdesc = (void *)kvm_stats_desc + i * size_desc; |
4667 | 0 | stats_list = add_kvmschema_entry(pdesc, stats_list, errp); |
4668 | 0 | } |
4669 | |
|
4670 | 0 | add_stats_schema(result, STATS_PROVIDER_KVM, target, stats_list); |
4671 | 0 | } |
4672 | | |
4673 | | static void query_stats_vcpu(CPUState *cpu, StatsArgs *kvm_stats_args) |
4674 | 0 | { |
4675 | 0 | int stats_fd = cpu->kvm_vcpu_stats_fd; |
4676 | 0 | Error *local_err = NULL; |
4677 | |
|
4678 | 0 | if (stats_fd == -1) { |
4679 | 0 | error_setg_errno(&local_err, errno, "KVM stats: ioctl failed"); |
4680 | 0 | error_propagate(kvm_stats_args->errp, local_err); |
4681 | 0 | return; |
4682 | 0 | } |
4683 | 0 | query_stats(kvm_stats_args->result.stats, STATS_TARGET_VCPU, |
4684 | 0 | kvm_stats_args->names, stats_fd, cpu, |
4685 | 0 | kvm_stats_args->errp); |
4686 | 0 | } |
4687 | | |
4688 | | static void query_stats_schema_vcpu(CPUState *cpu, StatsArgs *kvm_stats_args) |
4689 | 0 | { |
4690 | 0 | int stats_fd = cpu->kvm_vcpu_stats_fd; |
4691 | 0 | Error *local_err = NULL; |
4692 | |
|
4693 | 0 | if (stats_fd == -1) { |
4694 | 0 | error_setg_errno(&local_err, errno, "KVM stats: ioctl failed"); |
4695 | 0 | error_propagate(kvm_stats_args->errp, local_err); |
4696 | 0 | return; |
4697 | 0 | } |
4698 | 0 | query_stats_schema(kvm_stats_args->result.schema, STATS_TARGET_VCPU, stats_fd, |
4699 | 0 | kvm_stats_args->errp); |
4700 | 0 | } |
4701 | | |
4702 | | static void query_stats_cb(StatsResultList **result, StatsTarget target, |
4703 | | strList *names, strList *targets, Error **errp) |
4704 | 0 | { |
4705 | 0 | KVMState *s = kvm_state; |
4706 | 0 | CPUState *cpu; |
4707 | 0 | int stats_fd; |
4708 | |
|
4709 | 0 | switch (target) { |
4710 | 0 | case STATS_TARGET_VM: |
4711 | 0 | { |
4712 | 0 | stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL); |
4713 | 0 | if (stats_fd == -1) { |
4714 | 0 | error_setg_errno(errp, errno, "KVM stats: ioctl failed"); |
4715 | 0 | return; |
4716 | 0 | } |
4717 | 0 | query_stats(result, target, names, stats_fd, NULL, errp); |
4718 | 0 | close(stats_fd); |
4719 | 0 | break; |
4720 | 0 | } |
4721 | 0 | case STATS_TARGET_VCPU: |
4722 | 0 | { |
4723 | 0 | StatsArgs stats_args; |
4724 | 0 | stats_args.result.stats = result; |
4725 | 0 | stats_args.names = names; |
4726 | 0 | stats_args.errp = errp; |
4727 | 0 | CPU_FOREACH(cpu) { |
4728 | 0 | if (!apply_str_list_filter(cpu->parent_obj.canonical_path, targets)) { |
4729 | 0 | continue; |
4730 | 0 | } |
4731 | 0 | query_stats_vcpu(cpu, &stats_args); |
4732 | 0 | } |
4733 | 0 | break; |
4734 | 0 | } |
4735 | 0 | default: |
4736 | 0 | break; |
4737 | 0 | } |
4738 | 0 | } |
4739 | | |
4740 | | void query_stats_schemas_cb(StatsSchemaList **result, Error **errp) |
4741 | 0 | { |
4742 | 0 | StatsArgs stats_args; |
4743 | 0 | KVMState *s = kvm_state; |
4744 | 0 | int stats_fd; |
4745 | |
|
4746 | 0 | stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL); |
4747 | 0 | if (stats_fd == -1) { |
4748 | 0 | error_setg_errno(errp, errno, "KVM stats: ioctl failed"); |
4749 | 0 | return; |
4750 | 0 | } |
4751 | 0 | query_stats_schema(result, STATS_TARGET_VM, stats_fd, errp); |
4752 | 0 | close(stats_fd); |
4753 | |
|
4754 | 0 | if (first_cpu) { |
4755 | 0 | stats_args.result.schema = result; |
4756 | 0 | stats_args.errp = errp; |
4757 | 0 | query_stats_schema_vcpu(first_cpu, &stats_args); |
4758 | 0 | } |
4759 | 0 | } |
4760 | | |
4761 | | void kvm_mark_guest_state_protected(void) |
4762 | 0 | { |
4763 | 0 | kvm_state->guest_state_protected = true; |
4764 | 0 | } |
4765 | | |
4766 | | int kvm_create_guest_memfd(uint64_t size, uint64_t flags, Error **errp) |
4767 | 0 | { |
4768 | 0 | int fd; |
4769 | 0 | struct kvm_create_guest_memfd guest_memfd = { |
4770 | 0 | .size = size, |
4771 | 0 | .flags = flags, |
4772 | 0 | }; |
4773 | | |
4774 | | /* |
4775 | | * kvm_create_guest_memfd() can be invoked from generic hostmem code |
4776 | | * where KVM might be compiled but not necessarily enabled. Because |
4777 | | * of this, check for KVM support explicitly before continuing with |
4778 | | * any KVM-specific checks. |
4779 | | */ |
4780 | 0 | if (!kvm_enabled()) { |
4781 | 0 | error_setg(errp, "guest-memfd requires KVM accelerator"); |
4782 | 0 | return -1; |
4783 | 0 | } |
4784 | | |
4785 | 0 | if (!kvm_guest_memfd_supported) { |
4786 | 0 | error_setg(errp, "KVM does not support guest_memfd"); |
4787 | 0 | return -1; |
4788 | 0 | } |
4789 | | |
4790 | 0 | if (flags & ~kvm_guest_memfd_flags_supported) { |
4791 | 0 | error_setg(errp, "Current KVM instance does not support " |
4792 | 0 | "guest-memfd flag: 0x%"PRIx64, |
4793 | 0 | flags & ~kvm_guest_memfd_flags_supported); |
4794 | 0 | return -1; |
4795 | 0 | } |
4796 | | |
4797 | 0 | fd = kvm_vm_ioctl(kvm_state, KVM_CREATE_GUEST_MEMFD, &guest_memfd); |
4798 | 0 | if (fd < 0) { |
4799 | 0 | error_setg_errno(errp, errno, "Error creating KVM guest_memfd"); |
4800 | 0 | return -1; |
4801 | 0 | } |
4802 | | |
4803 | 0 | return fd; |
4804 | 0 | } |