/src/qemu/hw/virtio/virtio.c
Line | Count | Source |
1 | | /* |
2 | | * Virtio Support |
3 | | * |
4 | | * Copyright IBM, Corp. 2007 |
5 | | * |
6 | | * Authors: |
7 | | * Anthony Liguori <aliguori@us.ibm.com> |
8 | | * |
9 | | * This work is licensed under the terms of the GNU GPL, version 2. See |
10 | | * the COPYING file in the top-level directory. |
11 | | * |
12 | | */ |
13 | | |
14 | | #include "qemu/osdep.h" |
15 | | #include "qapi/error.h" |
16 | | #include "qapi/qapi-commands-virtio.h" |
17 | | #include "trace.h" |
18 | | #include "qemu/defer-call.h" |
19 | | #include "qemu/error-report.h" |
20 | | #include "qemu/log.h" |
21 | | #include "qemu/main-loop.h" |
22 | | #include "qemu/module.h" |
23 | | #include "qemu/target-info.h" |
24 | | #include "qom/object_interfaces.h" |
25 | | #include "hw/core/cpu.h" |
26 | | #include "hw/virtio/virtio.h" |
27 | | #include "hw/virtio/vhost.h" |
28 | | #include "migration/qemu-file-types.h" |
29 | | #include "qemu/atomic.h" |
30 | | #include "hw/virtio/virtio-bus.h" |
31 | | #include "hw/core/qdev-properties.h" |
32 | | #include "hw/virtio/virtio-access.h" |
33 | | #include "system/dma.h" |
34 | | #include "system/iothread.h" |
35 | | #include "system/memory.h" |
36 | | #include "system/runstate.h" |
37 | | #include "virtio-qmp.h" |
38 | | |
39 | | #include "standard-headers/linux/virtio_ids.h" |
40 | | #include "standard-headers/linux/vhost_types.h" |
41 | | #include "standard-headers/linux/virtio_blk.h" |
42 | | #include "standard-headers/linux/virtio_console.h" |
43 | | #include "standard-headers/linux/virtio_gpu.h" |
44 | | #include "standard-headers/linux/virtio_net.h" |
45 | | #include "standard-headers/linux/virtio_scsi.h" |
46 | | #include "standard-headers/linux/virtio_i2c.h" |
47 | | #include "standard-headers/linux/virtio_balloon.h" |
48 | | #include "standard-headers/linux/virtio_iommu.h" |
49 | | #include "standard-headers/linux/virtio_mem.h" |
50 | | #include "standard-headers/linux/virtio_vsock.h" |
51 | | #include "standard-headers/linux/virtio_spi.h" |
52 | | |
53 | | /* |
54 | | * Maximum size of virtio device config space |
55 | | */ |
56 | | #define VHOST_USER_MAX_CONFIG_SIZE 256 |
57 | | |
58 | | /* |
59 | | * The alignment to use between consumer and producer parts of vring. |
60 | | * x86 pagesize again. This is the default, used by transports like PCI |
61 | | * which don't provide a means for the guest to tell the host the alignment. |
62 | | */ |
63 | 179 | #define VIRTIO_PCI_VRING_ALIGN 4096 |
64 | | |
65 | | typedef struct VRingDesc |
66 | | { |
67 | | uint64_t addr; |
68 | | uint32_t len; |
69 | | uint16_t flags; |
70 | | uint16_t next; |
71 | | } VRingDesc; |
72 | | |
73 | | typedef struct VRingPackedDesc { |
74 | | uint64_t addr; |
75 | | uint32_t len; |
76 | | uint16_t id; |
77 | | uint16_t flags; |
78 | | } VRingPackedDesc; |
79 | | |
80 | | typedef struct VRingAvail |
81 | | { |
82 | | uint16_t flags; |
83 | | uint16_t idx; |
84 | | uint16_t ring[]; |
85 | | } VRingAvail; |
86 | | |
87 | | typedef struct VRingUsedElem |
88 | | { |
89 | | uint32_t id; |
90 | | uint32_t len; |
91 | | } VRingUsedElem; |
92 | | |
93 | | typedef struct VRingUsed |
94 | | { |
95 | | uint16_t flags; |
96 | | uint16_t idx; |
97 | | VRingUsedElem ring[]; |
98 | | } VRingUsed; |
99 | | |
100 | | typedef struct VRingMemoryRegionCaches { |
101 | | struct rcu_head rcu; |
102 | | MemoryRegionCache desc; |
103 | | MemoryRegionCache avail; |
104 | | MemoryRegionCache used; |
105 | | } VRingMemoryRegionCaches; |
106 | | |
107 | | typedef struct VRing |
108 | | { |
109 | | unsigned int num; |
110 | | unsigned int num_default; |
111 | | unsigned int align; |
112 | | hwaddr desc; |
113 | | hwaddr avail; |
114 | | hwaddr used; |
115 | | VRingMemoryRegionCaches *caches; |
116 | | } VRing; |
117 | | |
118 | | typedef struct VRingPackedDescEvent { |
119 | | uint16_t off_wrap; |
120 | | uint16_t flags; |
121 | | } VRingPackedDescEvent ; |
122 | | |
123 | | struct VirtQueue |
124 | | { |
125 | | VRing vring; |
126 | | VirtQueueElement *used_elems; |
127 | | |
128 | | /* Next head to pop */ |
129 | | uint16_t last_avail_idx; |
130 | | bool last_avail_wrap_counter; |
131 | | |
132 | | /* Last avail_idx read from VQ. */ |
133 | | uint16_t shadow_avail_idx; |
134 | | bool shadow_avail_wrap_counter; |
135 | | |
136 | | uint16_t used_idx; |
137 | | bool used_wrap_counter; |
138 | | |
139 | | /* Last used index value we have signalled on */ |
140 | | uint16_t signalled_used; |
141 | | |
142 | | /* Last used index value we have signalled on */ |
143 | | bool signalled_used_valid; |
144 | | |
145 | | /* Notification enabled? */ |
146 | | bool notification; |
147 | | |
148 | | uint16_t queue_index; |
149 | | |
150 | | unsigned int inuse; |
151 | | |
152 | | uint16_t vector; |
153 | | VirtIOHandleOutput handle_output; |
154 | | VirtIODevice *vdev; |
155 | | EventNotifier guest_notifier; |
156 | | EventNotifier host_notifier; |
157 | | bool host_notifier_enabled; |
158 | | QLIST_ENTRY(VirtQueue) node; |
159 | | }; |
160 | | |
161 | | const char *virtio_device_names[] = { |
162 | | [VIRTIO_ID_NET] = "virtio-net", |
163 | | [VIRTIO_ID_BLOCK] = "virtio-blk", |
164 | | [VIRTIO_ID_CONSOLE] = "virtio-serial", |
165 | | [VIRTIO_ID_RNG] = "virtio-rng", |
166 | | [VIRTIO_ID_BALLOON] = "virtio-balloon", |
167 | | [VIRTIO_ID_IOMEM] = "virtio-iomem", |
168 | | [VIRTIO_ID_RPMSG] = "virtio-rpmsg", |
169 | | [VIRTIO_ID_SCSI] = "virtio-scsi", |
170 | | [VIRTIO_ID_9P] = "virtio-9p", |
171 | | [VIRTIO_ID_MAC80211_WLAN] = "virtio-mac-wlan", |
172 | | [VIRTIO_ID_RPROC_SERIAL] = "virtio-rproc-serial", |
173 | | [VIRTIO_ID_CAIF] = "virtio-caif", |
174 | | [VIRTIO_ID_MEMORY_BALLOON] = "virtio-mem-balloon", |
175 | | [VIRTIO_ID_GPU] = "virtio-gpu", |
176 | | [VIRTIO_ID_CLOCK] = "virtio-clk", |
177 | | [VIRTIO_ID_INPUT] = "virtio-input", |
178 | | [VIRTIO_ID_VSOCK] = "vhost-vsock", |
179 | | [VIRTIO_ID_CRYPTO] = "virtio-crypto", |
180 | | [VIRTIO_ID_SIGNAL_DIST] = "virtio-signal", |
181 | | [VIRTIO_ID_PSTORE] = "virtio-pstore", |
182 | | [VIRTIO_ID_IOMMU] = "virtio-iommu", |
183 | | [VIRTIO_ID_MEM] = "virtio-mem", |
184 | | [VIRTIO_ID_SOUND] = "virtio-sound", |
185 | | [VIRTIO_ID_FS] = "virtio-user-fs", |
186 | | [VIRTIO_ID_PMEM] = "virtio-pmem", |
187 | | [VIRTIO_ID_RPMB] = "virtio-rpmb", |
188 | | [VIRTIO_ID_MAC80211_HWSIM] = "virtio-mac-hwsim", |
189 | | [VIRTIO_ID_VIDEO_ENCODER] = "virtio-vid-encoder", |
190 | | [VIRTIO_ID_VIDEO_DECODER] = "virtio-vid-decoder", |
191 | | [VIRTIO_ID_SCMI] = "virtio-scmi", |
192 | | [VIRTIO_ID_NITRO_SEC_MOD] = "virtio-nitro-sec-mod", |
193 | | [VIRTIO_ID_I2C_ADAPTER] = "vhost-user-i2c", |
194 | | [VIRTIO_ID_WATCHDOG] = "virtio-watchdog", |
195 | | [VIRTIO_ID_CAN] = "virtio-can", |
196 | | [VIRTIO_ID_DMABUF] = "virtio-dmabuf", |
197 | | [VIRTIO_ID_PARAM_SERV] = "virtio-param-serv", |
198 | | [VIRTIO_ID_AUDIO_POLICY] = "virtio-audio-pol", |
199 | | [VIRTIO_ID_BT] = "virtio-bluetooth", |
200 | | [VIRTIO_ID_GPIO] = "virtio-gpio", |
201 | | [VIRTIO_ID_SPI] = "virtio-spi", |
202 | | [VIRTIO_ID_MEDIA] = "virtio-media", |
203 | | }; |
204 | | |
205 | | static const char *virtio_id_to_name(uint16_t device_id) |
206 | 21 | { |
207 | 21 | assert(device_id < G_N_ELEMENTS(virtio_device_names)); |
208 | 21 | const char *name = virtio_device_names[device_id]; |
209 | 21 | assert(name != NULL); |
210 | 21 | return name; |
211 | 21 | } |
212 | | |
213 | | static void virtio_check_indirect_feature(VirtIODevice *vdev) |
214 | 186k | { |
215 | 186k | if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) { |
216 | 178k | qemu_log_mask(LOG_GUEST_ERROR, |
217 | 178k | "Device %s: indirect_desc was not negotiated!\n", |
218 | 178k | vdev->name); |
219 | 178k | } |
220 | 186k | } |
221 | | |
222 | | static inline uint16_t virtio_lduw_phys_cached(VirtIODevice *vdev, |
223 | | MemoryRegionCache *cache, |
224 | | hwaddr pa) |
225 | 14.2M | { |
226 | 14.2M | if (virtio_vdev_is_big_endian(vdev)) { |
227 | 0 | return lduw_be_phys_cached(cache, pa); |
228 | 0 | } |
229 | 14.2M | return lduw_le_phys_cached(cache, pa); |
230 | 14.2M | } |
231 | | |
232 | | static inline void virtio_stw_phys_cached(VirtIODevice *vdev, |
233 | | MemoryRegionCache *cache, |
234 | | hwaddr pa, uint16_t value) |
235 | 4.97M | { |
236 | 4.97M | if (virtio_vdev_is_big_endian(vdev)) { |
237 | 0 | stw_be_phys_cached(cache, pa, value); |
238 | 4.97M | } else { |
239 | 4.97M | stw_le_phys_cached(cache, pa, value); |
240 | 4.97M | } |
241 | 4.97M | } |
242 | | |
243 | | /* Called within call_rcu(). */ |
244 | | static void virtio_free_region_cache(VRingMemoryRegionCaches *caches) |
245 | 1.35M | { |
246 | 1.35M | assert(caches != NULL); |
247 | 1.35M | address_space_cache_destroy(&caches->desc); |
248 | 1.35M | address_space_cache_destroy(&caches->avail); |
249 | 1.35M | address_space_cache_destroy(&caches->used); |
250 | 1.35M | g_free(caches); |
251 | 1.35M | } |
252 | | |
253 | | static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq) |
254 | 385M | { |
255 | 385M | VRingMemoryRegionCaches *caches; |
256 | | |
257 | 385M | caches = qatomic_read(&vq->vring.caches); |
258 | 385M | qatomic_rcu_set(&vq->vring.caches, NULL); |
259 | 385M | if (caches) { |
260 | 341k | call_rcu(caches, virtio_free_region_cache, rcu); |
261 | 341k | } |
262 | 385M | } |
263 | | |
264 | | void virtio_init_region_cache(VirtIODevice *vdev, int n) |
265 | 7.49M | { |
266 | 7.49M | VirtQueue *vq = &vdev->vq[n]; |
267 | 7.49M | VRingMemoryRegionCaches *old = vq->vring.caches; |
268 | 7.49M | VRingMemoryRegionCaches *new = NULL; |
269 | 7.49M | hwaddr addr, size; |
270 | 7.49M | int64_t len; |
271 | 7.49M | bool packed; |
272 | | |
273 | | |
274 | 7.49M | addr = vq->vring.desc; |
275 | 7.49M | if (!addr) { |
276 | 6.05M | goto out_no_cache; |
277 | 6.05M | } |
278 | 1.44M | new = g_new0(VRingMemoryRegionCaches, 1); |
279 | 1.44M | size = virtio_queue_get_desc_size(vdev, n); |
280 | 1.44M | packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? |
281 | 1.44M | true : false; |
282 | 1.44M | len = address_space_cache_init(&new->desc, vdev->dma_as, |
283 | 1.44M | addr, size, packed); |
284 | 1.44M | if (len < size) { |
285 | 68.2k | g_autofree char *devname = qdev_get_human_name(DEVICE(vdev)); |
286 | | |
287 | 68.2k | virtio_error(vdev, |
288 | 68.2k | "Failed to map descriptor ring for device %s: " |
289 | 68.2k | "invalid guest physical address or corrupted queue setup", |
290 | 68.2k | devname); |
291 | 68.2k | goto err_desc; |
292 | 68.2k | } |
293 | | |
294 | 1.37M | size = virtio_queue_get_used_size(vdev, n); |
295 | 1.37M | len = address_space_cache_init(&new->used, vdev->dma_as, |
296 | 1.37M | vq->vring.used, size, true); |
297 | 1.37M | if (len < size) { |
298 | 8.29k | g_autofree char *devname = qdev_get_human_name(DEVICE(vdev)); |
299 | | |
300 | 8.29k | virtio_error(vdev, |
301 | 8.29k | "Failed to map used ring for device %s: " |
302 | 8.29k | "possible guest misconfiguration or insufficient memory", |
303 | 8.29k | devname); |
304 | 8.29k | goto err_used; |
305 | 8.29k | } |
306 | | |
307 | 1.36M | size = virtio_queue_get_avail_size(vdev, n); |
308 | 1.36M | len = address_space_cache_init(&new->avail, vdev->dma_as, |
309 | 1.36M | vq->vring.avail, size, false); |
310 | 1.36M | if (len < size) { |
311 | 13.5k | g_autofree char *devname = qdev_get_human_name(DEVICE(vdev)); |
312 | | |
313 | 13.5k | virtio_error(vdev, |
314 | 13.5k | "Failed to map avalaible ring for device %s: " |
315 | 13.5k | "possible queue misconfiguration or overlapping memory region", |
316 | 13.5k | devname); |
317 | 13.5k | goto err_avail; |
318 | 13.5k | } |
319 | | |
320 | 1.35M | qatomic_rcu_set(&vq->vring.caches, new); |
321 | 1.35M | if (old) { |
322 | 1.01M | call_rcu(old, virtio_free_region_cache, rcu); |
323 | 1.01M | } |
324 | 1.35M | return; |
325 | | |
326 | 13.5k | err_avail: |
327 | 13.5k | address_space_cache_destroy(&new->avail); |
328 | 21.8k | err_used: |
329 | 21.8k | address_space_cache_destroy(&new->used); |
330 | 90.0k | err_desc: |
331 | 90.0k | address_space_cache_destroy(&new->desc); |
332 | 6.14M | out_no_cache: |
333 | 6.14M | g_free(new); |
334 | 6.14M | virtio_virtqueue_reset_region_cache(vq); |
335 | 6.14M | } |
336 | | |
337 | | /* virt queue functions */ |
338 | | void virtio_queue_update_rings(VirtIODevice *vdev, int n) |
339 | 438k | { |
340 | 438k | VRing *vring = &vdev->vq[n].vring; |
341 | | |
342 | 438k | if (!vring->num || !vring->desc || !vring->align) { |
343 | | /* not yet setup -> nothing to do */ |
344 | 0 | return; |
345 | 0 | } |
346 | 438k | vring->avail = vring->desc + vring->num * sizeof(VRingDesc); |
347 | 438k | vring->used = vring_align(vring->avail + |
348 | 438k | offsetof(VRingAvail, ring[vring->num]), |
349 | 438k | vring->align); |
350 | 438k | virtio_init_region_cache(vdev, n); |
351 | 438k | } |
352 | | |
353 | | /* Called within rcu_read_lock(). */ |
354 | | static void vring_split_desc_read(VirtIODevice *vdev, VRingDesc *desc, |
355 | | MemoryRegionCache *cache, int i) |
356 | 92.7M | { |
357 | 92.7M | address_space_read_cached(cache, i * sizeof(VRingDesc), |
358 | 92.7M | desc, sizeof(VRingDesc)); |
359 | 92.7M | virtio_tswap64s(vdev, &desc->addr); |
360 | 92.7M | virtio_tswap32s(vdev, &desc->len); |
361 | 92.7M | virtio_tswap16s(vdev, &desc->flags); |
362 | 92.7M | virtio_tswap16s(vdev, &desc->next); |
363 | 92.7M | } |
364 | | |
365 | | static void vring_packed_event_read(VirtIODevice *vdev, |
366 | | MemoryRegionCache *cache, |
367 | | VRingPackedDescEvent *e) |
368 | 0 | { |
369 | 0 | hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap); |
370 | 0 | hwaddr off_flags = offsetof(VRingPackedDescEvent, flags); |
371 | |
|
372 | 0 | e->flags = virtio_lduw_phys_cached(vdev, cache, off_flags); |
373 | | /* Make sure flags is seen before off_wrap */ |
374 | 0 | smp_rmb(); |
375 | 0 | e->off_wrap = virtio_lduw_phys_cached(vdev, cache, off_off); |
376 | 0 | } |
377 | | |
378 | | static void vring_packed_off_wrap_write(VirtIODevice *vdev, |
379 | | MemoryRegionCache *cache, |
380 | | uint16_t off_wrap) |
381 | 0 | { |
382 | 0 | hwaddr off = offsetof(VRingPackedDescEvent, off_wrap); |
383 | |
|
384 | 0 | virtio_stw_phys_cached(vdev, cache, off, off_wrap); |
385 | 0 | address_space_cache_invalidate(cache, off, sizeof(off_wrap)); |
386 | 0 | } |
387 | | |
388 | | static void vring_packed_flags_write(VirtIODevice *vdev, |
389 | | MemoryRegionCache *cache, uint16_t flags) |
390 | 0 | { |
391 | 0 | hwaddr off = offsetof(VRingPackedDescEvent, flags); |
392 | |
|
393 | 0 | virtio_stw_phys_cached(vdev, cache, off, flags); |
394 | 0 | address_space_cache_invalidate(cache, off, sizeof(flags)); |
395 | 0 | } |
396 | | |
397 | | /* Called within rcu_read_lock(). */ |
398 | | static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq) |
399 | 27.7M | { |
400 | 27.7M | return qatomic_rcu_read(&vq->vring.caches); |
401 | 27.7M | } |
402 | | |
403 | | /* Called within rcu_read_lock(). */ |
404 | | static inline uint16_t vring_avail_flags(VirtQueue *vq) |
405 | 2.91M | { |
406 | 2.91M | VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); |
407 | 2.91M | hwaddr pa = offsetof(VRingAvail, flags); |
408 | | |
409 | 2.91M | if (!caches) { |
410 | 93 | return 0; |
411 | 93 | } |
412 | | |
413 | 2.91M | return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); |
414 | 2.91M | } |
415 | | |
416 | | /* Called within rcu_read_lock(). */ |
417 | | static inline uint16_t vring_avail_idx(VirtQueue *vq) |
418 | 1.20M | { |
419 | 1.20M | VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); |
420 | 1.20M | hwaddr pa = offsetof(VRingAvail, idx); |
421 | | |
422 | 1.20M | if (!caches) { |
423 | 3.01k | return 0; |
424 | 3.01k | } |
425 | | |
426 | 1.20M | vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); |
427 | 1.20M | return vq->shadow_avail_idx; |
428 | 1.20M | } |
429 | | |
430 | | /* Called within rcu_read_lock(). */ |
431 | | static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) |
432 | 9.47M | { |
433 | 9.47M | VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); |
434 | 9.47M | hwaddr pa = offsetof(VRingAvail, ring[i]); |
435 | | |
436 | 9.47M | if (!caches) { |
437 | 258 | return 0; |
438 | 258 | } |
439 | | |
440 | 9.47M | return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); |
441 | 9.47M | } |
442 | | |
443 | | /* Called within rcu_read_lock(). */ |
444 | | static inline uint16_t vring_get_used_event(VirtQueue *vq) |
445 | 543k | { |
446 | 543k | return vring_avail_ring(vq, vq->vring.num); |
447 | 543k | } |
448 | | |
449 | | /* Called within rcu_read_lock(). */ |
450 | | static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem, |
451 | | int i) |
452 | 3.86M | { |
453 | 3.86M | VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); |
454 | 3.86M | hwaddr pa = offsetof(VRingUsed, ring[i]); |
455 | | |
456 | 3.86M | if (!caches) { |
457 | 178 | return; |
458 | 178 | } |
459 | | |
460 | 3.86M | virtio_tswap32s(vq->vdev, &uelem->id); |
461 | 3.86M | virtio_tswap32s(vq->vdev, &uelem->len); |
462 | 3.86M | address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem)); |
463 | 3.86M | address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem)); |
464 | 3.86M | } |
465 | | |
466 | | /* Called within rcu_read_lock(). */ |
467 | | static inline uint16_t vring_used_flags(VirtQueue *vq) |
468 | 0 | { |
469 | 0 | VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); |
470 | 0 | hwaddr pa = offsetof(VRingUsed, flags); |
471 | |
|
472 | 0 | if (!caches) { |
473 | 0 | return 0; |
474 | 0 | } |
475 | | |
476 | 0 | return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); |
477 | 0 | } |
478 | | |
479 | | /* Called within rcu_read_lock(). */ |
480 | | static uint16_t vring_used_idx(VirtQueue *vq) |
481 | 0 | { |
482 | 0 | VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); |
483 | 0 | hwaddr pa = offsetof(VRingUsed, idx); |
484 | |
|
485 | 0 | if (!caches) { |
486 | 0 | return 0; |
487 | 0 | } |
488 | | |
489 | 0 | return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); |
490 | 0 | } |
491 | | |
492 | | /* Called within rcu_read_lock(). */ |
493 | | static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val) |
494 | 3.86M | { |
495 | 3.86M | VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); |
496 | 3.86M | hwaddr pa = offsetof(VRingUsed, idx); |
497 | | |
498 | 3.86M | if (caches) { |
499 | 3.86M | virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); |
500 | 3.86M | address_space_cache_invalidate(&caches->used, pa, sizeof(val)); |
501 | 3.86M | } |
502 | | |
503 | 3.86M | vq->used_idx = val; |
504 | 3.86M | } |
505 | | |
506 | | /* Called within rcu_read_lock(). */ |
507 | | static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) |
508 | 535k | { |
509 | 535k | VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); |
510 | 535k | VirtIODevice *vdev = vq->vdev; |
511 | 535k | hwaddr pa = offsetof(VRingUsed, flags); |
512 | 535k | uint16_t flags; |
513 | | |
514 | 535k | if (!caches) { |
515 | 0 | return; |
516 | 0 | } |
517 | | |
518 | 535k | flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); |
519 | 535k | virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask); |
520 | 535k | address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); |
521 | 535k | } |
522 | | |
523 | | /* Called within rcu_read_lock(). */ |
524 | | static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) |
525 | 88.9k | { |
526 | 88.9k | VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); |
527 | 88.9k | VirtIODevice *vdev = vq->vdev; |
528 | 88.9k | hwaddr pa = offsetof(VRingUsed, flags); |
529 | 88.9k | uint16_t flags; |
530 | | |
531 | 88.9k | if (!caches) { |
532 | 451 | return; |
533 | 451 | } |
534 | | |
535 | 88.4k | flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); |
536 | 88.4k | virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask); |
537 | 88.4k | address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); |
538 | 88.4k | } |
539 | | |
540 | | /* Called within rcu_read_lock(). */ |
541 | | static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val) |
542 | 1.63M | { |
543 | 1.63M | VRingMemoryRegionCaches *caches; |
544 | 1.63M | hwaddr pa; |
545 | 1.63M | if (!vq->notification) { |
546 | 1.14M | return; |
547 | 1.14M | } |
548 | | |
549 | 489k | caches = vring_get_region_caches(vq); |
550 | 489k | if (!caches) { |
551 | 708 | return; |
552 | 708 | } |
553 | | |
554 | 488k | pa = offsetof(VRingUsed, ring[vq->vring.num]); |
555 | 488k | virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); |
556 | 488k | address_space_cache_invalidate(&caches->used, pa, sizeof(val)); |
557 | 488k | } |
558 | | |
559 | | static void virtio_queue_split_set_notification(VirtQueue *vq, int enable) |
560 | 1.36M | { |
561 | 1.36M | RCU_READ_LOCK_GUARD(); |
562 | | |
563 | 1.36M | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { |
564 | 743k | vring_set_avail_event(vq, vring_avail_idx(vq)); |
565 | 743k | } else if (enable) { |
566 | 88.9k | vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); |
567 | 535k | } else { |
568 | 535k | vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); |
569 | 535k | } |
570 | 1.36M | if (enable) { |
571 | | /* Expose avail event/used flags before caller checks the avail idx. */ |
572 | 115k | smp_mb(); |
573 | 115k | } |
574 | 1.36M | } |
575 | | |
576 | | static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable) |
577 | 0 | { |
578 | 0 | uint16_t off_wrap; |
579 | 0 | VRingPackedDescEvent e; |
580 | 0 | VRingMemoryRegionCaches *caches; |
581 | |
|
582 | 0 | RCU_READ_LOCK_GUARD(); |
583 | 0 | caches = vring_get_region_caches(vq); |
584 | 0 | if (!caches) { |
585 | 0 | return; |
586 | 0 | } |
587 | | |
588 | 0 | vring_packed_event_read(vq->vdev, &caches->used, &e); |
589 | |
|
590 | 0 | if (!enable) { |
591 | 0 | e.flags = VRING_PACKED_EVENT_FLAG_DISABLE; |
592 | 0 | } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { |
593 | 0 | off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15; |
594 | 0 | vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap); |
595 | | /* Make sure off_wrap is wrote before flags */ |
596 | 0 | smp_wmb(); |
597 | 0 | e.flags = VRING_PACKED_EVENT_FLAG_DESC; |
598 | 0 | } else { |
599 | 0 | e.flags = VRING_PACKED_EVENT_FLAG_ENABLE; |
600 | 0 | } |
601 | |
|
602 | 0 | vring_packed_flags_write(vq->vdev, &caches->used, e.flags); |
603 | 0 | if (enable) { |
604 | | /* Expose avail event/used flags before caller checks the avail idx. */ |
605 | 0 | smp_mb(); |
606 | 0 | } |
607 | 0 | } |
608 | | |
609 | | bool virtio_queue_get_notification(VirtQueue *vq) |
610 | 556k | { |
611 | 556k | return vq->notification; |
612 | 556k | } |
613 | | |
614 | | void virtio_queue_set_notification(VirtQueue *vq, int enable) |
615 | 1.37M | { |
616 | 1.37M | vq->notification = enable; |
617 | | |
618 | 1.37M | if (!vq->vring.desc) { |
619 | 3.34k | return; |
620 | 3.34k | } |
621 | | |
622 | 1.36M | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
623 | 0 | virtio_queue_packed_set_notification(vq, enable); |
624 | 1.36M | } else { |
625 | 1.36M | virtio_queue_split_set_notification(vq, enable); |
626 | 1.36M | } |
627 | 1.36M | } |
628 | | |
629 | | int virtio_queue_ready(VirtQueue *vq) |
630 | 3.21M | { |
631 | 3.21M | return vq->vring.avail != 0; |
632 | 3.21M | } |
633 | | |
634 | | static void vring_packed_desc_read_flags(VirtIODevice *vdev, |
635 | | uint16_t *flags, |
636 | | MemoryRegionCache *cache, |
637 | | int i) |
638 | 0 | { |
639 | 0 | hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags); |
640 | |
|
641 | 0 | *flags = virtio_lduw_phys_cached(vdev, cache, off); |
642 | 0 | } |
643 | | |
644 | | static void vring_packed_desc_read(VirtIODevice *vdev, |
645 | | VRingPackedDesc *desc, |
646 | | MemoryRegionCache *cache, |
647 | | int i, bool strict_order) |
648 | 0 | { |
649 | 0 | hwaddr off = i * sizeof(VRingPackedDesc); |
650 | |
|
651 | 0 | vring_packed_desc_read_flags(vdev, &desc->flags, cache, i); |
652 | |
|
653 | 0 | if (strict_order) { |
654 | | /* Make sure flags is read before the rest fields. */ |
655 | 0 | smp_rmb(); |
656 | 0 | } |
657 | |
|
658 | 0 | address_space_read_cached(cache, off + offsetof(VRingPackedDesc, addr), |
659 | 0 | &desc->addr, sizeof(desc->addr)); |
660 | 0 | address_space_read_cached(cache, off + offsetof(VRingPackedDesc, id), |
661 | 0 | &desc->id, sizeof(desc->id)); |
662 | 0 | address_space_read_cached(cache, off + offsetof(VRingPackedDesc, len), |
663 | 0 | &desc->len, sizeof(desc->len)); |
664 | 0 | virtio_tswap64s(vdev, &desc->addr); |
665 | 0 | virtio_tswap16s(vdev, &desc->id); |
666 | 0 | virtio_tswap32s(vdev, &desc->len); |
667 | 0 | } |
668 | | |
669 | | static void vring_packed_desc_write_data(VirtIODevice *vdev, |
670 | | VRingPackedDesc *desc, |
671 | | MemoryRegionCache *cache, |
672 | | int i) |
673 | 0 | { |
674 | 0 | hwaddr off_id = i * sizeof(VRingPackedDesc) + |
675 | 0 | offsetof(VRingPackedDesc, id); |
676 | 0 | hwaddr off_len = i * sizeof(VRingPackedDesc) + |
677 | 0 | offsetof(VRingPackedDesc, len); |
678 | |
|
679 | 0 | virtio_tswap32s(vdev, &desc->len); |
680 | 0 | virtio_tswap16s(vdev, &desc->id); |
681 | 0 | address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id)); |
682 | 0 | address_space_cache_invalidate(cache, off_id, sizeof(desc->id)); |
683 | 0 | address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len)); |
684 | 0 | address_space_cache_invalidate(cache, off_len, sizeof(desc->len)); |
685 | 0 | } |
686 | | |
687 | | static void vring_packed_desc_write_flags(VirtIODevice *vdev, |
688 | | VRingPackedDesc *desc, |
689 | | MemoryRegionCache *cache, |
690 | | int i) |
691 | 0 | { |
692 | 0 | hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags); |
693 | |
|
694 | 0 | virtio_stw_phys_cached(vdev, cache, off, desc->flags); |
695 | 0 | address_space_cache_invalidate(cache, off, sizeof(desc->flags)); |
696 | 0 | } |
697 | | |
698 | | static void vring_packed_desc_write(VirtIODevice *vdev, |
699 | | VRingPackedDesc *desc, |
700 | | MemoryRegionCache *cache, |
701 | | int i, bool strict_order) |
702 | 0 | { |
703 | 0 | vring_packed_desc_write_data(vdev, desc, cache, i); |
704 | 0 | if (strict_order) { |
705 | | /* Make sure data is wrote before flags. */ |
706 | 0 | smp_wmb(); |
707 | 0 | } |
708 | 0 | vring_packed_desc_write_flags(vdev, desc, cache, i); |
709 | 0 | } |
710 | | |
711 | | static inline bool is_desc_avail(uint16_t flags, bool wrap_counter) |
712 | 0 | { |
713 | 0 | bool avail, used; |
714 | |
|
715 | 0 | avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL)); |
716 | 0 | used = !!(flags & (1 << VRING_PACKED_DESC_F_USED)); |
717 | 0 | return (avail != used) && (avail == wrap_counter); |
718 | 0 | } |
719 | | |
720 | | static int virtio_queue_split_empty(VirtQueue *vq) |
721 | 1.38M | { |
722 | 1.38M | bool empty; |
723 | | |
724 | 1.38M | if (virtio_device_disabled(vq->vdev)) { |
725 | 1.02M | return 1; |
726 | 1.02M | } |
727 | | |
728 | 356k | if (unlikely(!vq->vring.avail)) { |
729 | 518 | return 1; |
730 | 518 | } |
731 | | |
732 | 356k | if (vq->shadow_avail_idx != vq->last_avail_idx) { |
733 | 300k | return 0; |
734 | 300k | } |
735 | | |
736 | 356k | RCU_READ_LOCK_GUARD(); |
737 | 56.2k | empty = vring_avail_idx(vq) == vq->last_avail_idx; |
738 | 56.2k | return empty; |
739 | 356k | } |
740 | | |
741 | | /* Called within rcu_read_lock(). */ |
742 | | static int virtio_queue_packed_empty_rcu(VirtQueue *vq) |
743 | 0 | { |
744 | 0 | struct VRingPackedDesc desc; |
745 | 0 | VRingMemoryRegionCaches *cache; |
746 | |
|
747 | 0 | if (virtio_device_disabled(vq->vdev)) { |
748 | 0 | return 1; |
749 | 0 | } |
750 | | |
751 | 0 | if (unlikely(!vq->vring.desc)) { |
752 | 0 | return 1; |
753 | 0 | } |
754 | | |
755 | 0 | cache = vring_get_region_caches(vq); |
756 | 0 | if (!cache) { |
757 | 0 | return 1; |
758 | 0 | } |
759 | | |
760 | 0 | vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc, |
761 | 0 | vq->last_avail_idx); |
762 | |
|
763 | 0 | return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter); |
764 | 0 | } |
765 | | |
766 | | static int virtio_queue_packed_empty(VirtQueue *vq) |
767 | 0 | { |
768 | 0 | RCU_READ_LOCK_GUARD(); |
769 | 0 | return virtio_queue_packed_empty_rcu(vq); |
770 | 0 | } |
771 | | |
772 | | int virtio_queue_empty(VirtQueue *vq) |
773 | 1.38M | { |
774 | 1.38M | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
775 | 0 | return virtio_queue_packed_empty(vq); |
776 | 1.38M | } else { |
777 | 1.38M | return virtio_queue_split_empty(vq); |
778 | 1.38M | } |
779 | 1.38M | } |
780 | | |
781 | | static bool virtio_queue_split_poll(VirtQueue *vq, unsigned shadow_idx) |
782 | 26.9k | { |
783 | 26.9k | if (unlikely(!vq->vring.avail)) { |
784 | 0 | return false; |
785 | 0 | } |
786 | | |
787 | 26.9k | return (uint16_t)shadow_idx != vring_avail_idx(vq); |
788 | 26.9k | } |
789 | | |
790 | | static bool virtio_queue_packed_poll(VirtQueue *vq, unsigned shadow_idx) |
791 | 0 | { |
792 | 0 | VRingPackedDesc desc; |
793 | 0 | VRingMemoryRegionCaches *caches; |
794 | |
|
795 | 0 | if (unlikely(!vq->vring.desc)) { |
796 | 0 | return false; |
797 | 0 | } |
798 | | |
799 | 0 | caches = vring_get_region_caches(vq); |
800 | 0 | if (!caches) { |
801 | 0 | return false; |
802 | 0 | } |
803 | | |
804 | 0 | vring_packed_desc_read(vq->vdev, &desc, &caches->desc, |
805 | 0 | shadow_idx, true); |
806 | |
|
807 | 0 | return is_desc_avail(desc.flags, vq->shadow_avail_wrap_counter); |
808 | 0 | } |
809 | | |
810 | | static bool virtio_queue_poll(VirtQueue *vq, unsigned shadow_idx) |
811 | 44.7k | { |
812 | 44.7k | if (virtio_device_disabled(vq->vdev)) { |
813 | 17.7k | return false; |
814 | 17.7k | } |
815 | | |
816 | 26.9k | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
817 | 0 | return virtio_queue_packed_poll(vq, shadow_idx); |
818 | 26.9k | } else { |
819 | 26.9k | return virtio_queue_split_poll(vq, shadow_idx); |
820 | 26.9k | } |
821 | 26.9k | } |
822 | | |
823 | | bool virtio_queue_enable_notification_and_check(VirtQueue *vq, |
824 | | int opaque) |
825 | 48.3k | { |
826 | 48.3k | virtio_queue_set_notification(vq, 1); |
827 | | |
828 | 48.3k | if (opaque >= 0) { |
829 | 44.7k | return virtio_queue_poll(vq, (unsigned)opaque); |
830 | 44.7k | } else { |
831 | 3.67k | return false; |
832 | 3.67k | } |
833 | 48.3k | } |
834 | | |
835 | | static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem, |
836 | | unsigned int len) |
837 | 4.16M | { |
838 | 4.16M | AddressSpace *dma_as = vq->vdev->dma_as; |
839 | 4.16M | unsigned int offset; |
840 | 4.16M | int i; |
841 | | |
842 | 4.16M | offset = 0; |
843 | 5.01M | for (i = 0; i < elem->in_num; i++) { |
844 | 849k | size_t size = MIN(len - offset, elem->in_sg[i].iov_len); |
845 | | |
846 | 849k | dma_memory_unmap(dma_as, elem->in_sg[i].iov_base, |
847 | 849k | elem->in_sg[i].iov_len, |
848 | 849k | DMA_DIRECTION_FROM_DEVICE, size); |
849 | | |
850 | 849k | offset += size; |
851 | 849k | } |
852 | | |
853 | 8.01M | for (i = 0; i < elem->out_num; i++) |
854 | 3.84M | dma_memory_unmap(dma_as, elem->out_sg[i].iov_base, |
855 | 3.84M | elem->out_sg[i].iov_len, |
856 | 3.84M | DMA_DIRECTION_TO_DEVICE, |
857 | 3.84M | elem->out_sg[i].iov_len); |
858 | 4.16M | } |
859 | | |
860 | | /* virtqueue_detach_element: |
861 | | * @vq: The #VirtQueue |
862 | | * @elem: The #VirtQueueElement |
863 | | * @len: number of bytes written |
864 | | * |
865 | | * Detach the element from the virtqueue. This function is suitable for device |
866 | | * reset or other situations where a #VirtQueueElement is simply freed and will |
867 | | * not be pushed or discarded. |
868 | | */ |
869 | | void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem, |
870 | | unsigned int len) |
871 | 178k | { |
872 | 178k | vq->inuse -= elem->ndescs; |
873 | 178k | virtqueue_unmap_sg(vq, elem, len); |
874 | 178k | } |
875 | | |
876 | | static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num) |
877 | 159k | { |
878 | 159k | vq->last_avail_idx -= num; |
879 | 159k | } |
880 | | |
881 | | static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num) |
882 | 0 | { |
883 | 0 | if (vq->last_avail_idx < num) { |
884 | 0 | vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num; |
885 | 0 | vq->last_avail_wrap_counter ^= 1; |
886 | 0 | } else { |
887 | 0 | vq->last_avail_idx -= num; |
888 | 0 | } |
889 | 0 | } |
890 | | |
891 | | /* virtqueue_unpop: |
892 | | * @vq: The #VirtQueue |
893 | | * @elem: The #VirtQueueElement |
894 | | * @len: number of bytes written |
895 | | * |
896 | | * Pretend the most recent element wasn't popped from the virtqueue. The next |
897 | | * call to virtqueue_pop() will refetch the element. |
898 | | */ |
899 | | void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem, |
900 | | unsigned int len) |
901 | 159k | { |
902 | | |
903 | 159k | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
904 | 0 | virtqueue_packed_rewind(vq, 1); |
905 | 159k | } else { |
906 | 159k | virtqueue_split_rewind(vq, 1); |
907 | 159k | } |
908 | | |
909 | 159k | virtqueue_detach_element(vq, elem, len); |
910 | 159k | } |
911 | | |
912 | | /* virtqueue_rewind: |
913 | | * @vq: The #VirtQueue |
914 | | * @num: Number of elements to push back |
915 | | * |
916 | | * Pretend that elements weren't popped from the virtqueue. The next |
917 | | * virtqueue_pop() will refetch the oldest element. |
918 | | * |
919 | | * Use virtqueue_unpop() instead if you have a VirtQueueElement. |
920 | | * |
921 | | * Returns: true on success, false if @num is greater than the number of in use |
922 | | * elements. |
923 | | */ |
924 | | bool virtqueue_rewind(VirtQueue *vq, unsigned int num) |
925 | 27.0k | { |
926 | 27.0k | if (num > vq->inuse) { |
927 | 27.0k | return false; |
928 | 27.0k | } |
929 | | |
930 | 0 | vq->inuse -= num; |
931 | 0 | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
932 | 0 | virtqueue_packed_rewind(vq, num); |
933 | 0 | } else { |
934 | 0 | virtqueue_split_rewind(vq, num); |
935 | 0 | } |
936 | 0 | return true; |
937 | 27.0k | } |
938 | | |
939 | | static void virtqueue_split_fill(VirtQueue *vq, const VirtQueueElement *elem, |
940 | | unsigned int len, unsigned int idx) |
941 | 3.93M | { |
942 | 3.93M | VRingUsedElem uelem; |
943 | | |
944 | 3.93M | if (unlikely(!vq->vring.used)) { |
945 | 61.3k | return; |
946 | 61.3k | } |
947 | | |
948 | 3.86M | idx = (idx + vq->used_idx) % vq->vring.num; |
949 | | |
950 | 3.86M | uelem.id = elem->index; |
951 | 3.86M | uelem.len = len; |
952 | 3.86M | vring_used_write(vq, &uelem, idx); |
953 | 3.86M | } |
954 | | |
955 | | static void virtqueue_packed_fill(VirtQueue *vq, const VirtQueueElement *elem, |
956 | | unsigned int len, unsigned int idx) |
957 | 0 | { |
958 | 0 | vq->used_elems[idx].index = elem->index; |
959 | 0 | vq->used_elems[idx].len = len; |
960 | 0 | vq->used_elems[idx].ndescs = elem->ndescs; |
961 | 0 | } |
962 | | |
963 | | static void virtqueue_ordered_fill(VirtQueue *vq, const VirtQueueElement *elem, |
964 | | unsigned int len) |
965 | 0 | { |
966 | 0 | unsigned int i, steps, max_steps, ndescs; |
967 | |
|
968 | 0 | i = vq->used_idx % vq->vring.num; |
969 | 0 | steps = 0; |
970 | | /* |
971 | | * We shouldn't need to increase 'i' by more than or equal to |
972 | | * the distance between used_idx and last_avail_idx (max_steps). |
973 | | */ |
974 | 0 | max_steps = MIN(vq->last_avail_idx - vq->used_idx, vq->vring.num); |
975 | | |
976 | | /* Search for element in vq->used_elems */ |
977 | 0 | while (steps < max_steps) { |
978 | | /* Found element, set length and mark as filled */ |
979 | 0 | if (vq->used_elems[i].index == elem->index) { |
980 | 0 | vq->used_elems[i].len = len; |
981 | 0 | vq->used_elems[i].in_order_filled = true; |
982 | 0 | break; |
983 | 0 | } |
984 | | |
985 | 0 | ndescs = vq->used_elems[i].ndescs; |
986 | | |
987 | | /* Defensive sanity check */ |
988 | 0 | if (unlikely(ndescs == 0 || ndescs > vq->vring.num)) { |
989 | 0 | qemu_log_mask(LOG_GUEST_ERROR, |
990 | 0 | "%s: %s invalid ndescs %u at position %u\n", |
991 | 0 | __func__, vq->vdev->name, ndescs, i); |
992 | 0 | return; |
993 | 0 | } |
994 | | |
995 | 0 | i += ndescs; |
996 | 0 | steps += ndescs; |
997 | |
|
998 | 0 | if (i >= vq->vring.num) { |
999 | 0 | i -= vq->vring.num; |
1000 | 0 | } |
1001 | 0 | } |
1002 | | |
1003 | | /* |
1004 | | * We should be able to find a matching VirtQueueElement in |
1005 | | * used_elems. If we don't, this is an error. |
1006 | | */ |
1007 | 0 | if (steps >= max_steps) { |
1008 | 0 | qemu_log_mask(LOG_GUEST_ERROR, "%s: %s cannot fill buffer id %u\n", |
1009 | 0 | __func__, vq->vdev->name, elem->index); |
1010 | 0 | } |
1011 | 0 | } |
1012 | | |
1013 | | static void virtqueue_packed_fill_desc(VirtQueue *vq, |
1014 | | const VirtQueueElement *elem, |
1015 | | unsigned int idx, |
1016 | | bool strict_order) |
1017 | 0 | { |
1018 | 0 | uint16_t head; |
1019 | 0 | VRingMemoryRegionCaches *caches; |
1020 | 0 | VRingPackedDesc desc = { |
1021 | 0 | .id = elem->index, |
1022 | 0 | .len = elem->len, |
1023 | 0 | }; |
1024 | 0 | bool wrap_counter = vq->used_wrap_counter; |
1025 | |
|
1026 | 0 | if (unlikely(!vq->vring.desc)) { |
1027 | 0 | return; |
1028 | 0 | } |
1029 | | |
1030 | 0 | head = vq->used_idx + idx; |
1031 | 0 | if (head >= vq->vring.num) { |
1032 | 0 | head -= vq->vring.num; |
1033 | 0 | wrap_counter ^= 1; |
1034 | 0 | } |
1035 | 0 | if (wrap_counter) { |
1036 | 0 | desc.flags |= (1 << VRING_PACKED_DESC_F_AVAIL); |
1037 | 0 | desc.flags |= (1 << VRING_PACKED_DESC_F_USED); |
1038 | 0 | } else { |
1039 | 0 | desc.flags &= ~(1 << VRING_PACKED_DESC_F_AVAIL); |
1040 | 0 | desc.flags &= ~(1 << VRING_PACKED_DESC_F_USED); |
1041 | 0 | } |
1042 | |
|
1043 | 0 | caches = vring_get_region_caches(vq); |
1044 | 0 | if (!caches) { |
1045 | 0 | return; |
1046 | 0 | } |
1047 | | |
1048 | 0 | vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order); |
1049 | 0 | } |
1050 | | |
1051 | | /* Called within rcu_read_lock(). */ |
1052 | | void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, |
1053 | | unsigned int len, unsigned int idx) |
1054 | 3.99M | { |
1055 | 3.99M | trace_virtqueue_fill(vq, elem, len, idx); |
1056 | | |
1057 | 3.99M | virtqueue_unmap_sg(vq, elem, len); |
1058 | | |
1059 | 3.99M | if (virtio_device_disabled(vq->vdev)) { |
1060 | 60.7k | return; |
1061 | 60.7k | } |
1062 | | |
1063 | 3.93M | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) { |
1064 | 0 | virtqueue_ordered_fill(vq, elem, len); |
1065 | 3.93M | } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
1066 | 0 | virtqueue_packed_fill(vq, elem, len, idx); |
1067 | 3.93M | } else { |
1068 | 3.93M | virtqueue_split_fill(vq, elem, len, idx); |
1069 | 3.93M | } |
1070 | 3.93M | } |
1071 | | |
1072 | | /* Called within rcu_read_lock(). */ |
1073 | | static void virtqueue_split_flush(VirtQueue *vq, unsigned int count) |
1074 | 3.92M | { |
1075 | 3.92M | uint16_t old, new; |
1076 | | |
1077 | 3.92M | if (unlikely(!vq->vring.used)) { |
1078 | 61.3k | return; |
1079 | 61.3k | } |
1080 | | |
1081 | | /* Make sure buffer is written before we update index. */ |
1082 | 3.86M | smp_wmb(); |
1083 | 3.86M | trace_virtqueue_flush(vq, count); |
1084 | 3.86M | old = vq->used_idx; |
1085 | 3.86M | new = old + count; |
1086 | 3.86M | vring_used_idx_set(vq, new); |
1087 | 3.86M | vq->inuse -= count; |
1088 | 3.86M | if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) |
1089 | 0 | vq->signalled_used_valid = false; |
1090 | 3.86M | } |
1091 | | |
1092 | | static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count) |
1093 | 0 | { |
1094 | 0 | unsigned int i, ndescs = 0; |
1095 | |
|
1096 | 0 | if (unlikely(!vq->vring.desc)) { |
1097 | 0 | return; |
1098 | 0 | } |
1099 | | |
1100 | | /* |
1101 | | * For indirect element's 'ndescs' is 1. |
1102 | | * For all other elemment's 'ndescs' is the |
1103 | | * number of descriptors chained by NEXT (as set in virtqueue_packed_pop). |
1104 | | * So When the 'elem' be filled into the descriptor ring, |
1105 | | * The 'idx' of this 'elem' shall be |
1106 | | * the value of 'vq->used_idx' plus the 'ndescs'. |
1107 | | */ |
1108 | 0 | ndescs += vq->used_elems[0].ndescs; |
1109 | 0 | for (i = 1; i < count; i++) { |
1110 | 0 | virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false); |
1111 | 0 | ndescs += vq->used_elems[i].ndescs; |
1112 | 0 | } |
1113 | 0 | virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true); |
1114 | |
|
1115 | 0 | vq->inuse -= ndescs; |
1116 | 0 | vq->used_idx += ndescs; |
1117 | 0 | if (vq->used_idx >= vq->vring.num) { |
1118 | 0 | vq->used_idx -= vq->vring.num; |
1119 | 0 | vq->used_wrap_counter ^= 1; |
1120 | 0 | vq->signalled_used_valid = false; |
1121 | 0 | } |
1122 | 0 | } |
1123 | | |
1124 | | static void virtqueue_ordered_flush(VirtQueue *vq) |
1125 | 0 | { |
1126 | 0 | unsigned int i = vq->used_idx % vq->vring.num; |
1127 | 0 | unsigned int ndescs = 0; |
1128 | 0 | uint16_t old = vq->used_idx; |
1129 | 0 | uint16_t new; |
1130 | 0 | bool packed; |
1131 | 0 | VRingUsedElem uelem; |
1132 | |
|
1133 | 0 | packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED); |
1134 | |
|
1135 | 0 | if (packed) { |
1136 | 0 | if (unlikely(!vq->vring.desc)) { |
1137 | 0 | return; |
1138 | 0 | } |
1139 | 0 | } else if (unlikely(!vq->vring.used)) { |
1140 | 0 | return; |
1141 | 0 | } |
1142 | | |
1143 | | /* First expected in-order element isn't ready, nothing to do */ |
1144 | 0 | if (!vq->used_elems[i].in_order_filled) { |
1145 | 0 | return; |
1146 | 0 | } |
1147 | | |
1148 | | /* Search for filled elements in-order */ |
1149 | 0 | while (vq->used_elems[i].in_order_filled) { |
1150 | | /* |
1151 | | * First entry for packed VQs is written last so the guest |
1152 | | * doesn't see invalid descriptors. |
1153 | | */ |
1154 | 0 | if (packed && i != vq->used_idx) { |
1155 | 0 | virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false); |
1156 | 0 | } else if (!packed) { |
1157 | 0 | uelem.id = vq->used_elems[i].index; |
1158 | 0 | uelem.len = vq->used_elems[i].len; |
1159 | 0 | vring_used_write(vq, &uelem, i); |
1160 | 0 | } |
1161 | |
|
1162 | 0 | vq->used_elems[i].in_order_filled = false; |
1163 | 0 | ndescs += vq->used_elems[i].ndescs; |
1164 | 0 | i += vq->used_elems[i].ndescs; |
1165 | 0 | if (i >= vq->vring.num) { |
1166 | 0 | i -= vq->vring.num; |
1167 | 0 | } |
1168 | 0 | } |
1169 | |
|
1170 | 0 | if (packed) { |
1171 | 0 | virtqueue_packed_fill_desc(vq, &vq->used_elems[vq->used_idx], 0, true); |
1172 | 0 | vq->used_idx += ndescs; |
1173 | 0 | if (vq->used_idx >= vq->vring.num) { |
1174 | 0 | vq->used_idx -= vq->vring.num; |
1175 | 0 | vq->used_wrap_counter ^= 1; |
1176 | 0 | vq->signalled_used_valid = false; |
1177 | 0 | } |
1178 | 0 | } else { |
1179 | | /* Make sure buffer is written before we update index. */ |
1180 | 0 | smp_wmb(); |
1181 | 0 | new = old + ndescs; |
1182 | 0 | vring_used_idx_set(vq, new); |
1183 | 0 | if (unlikely((int16_t)(new - vq->signalled_used) < |
1184 | 0 | (uint16_t)(new - old))) { |
1185 | 0 | vq->signalled_used_valid = false; |
1186 | 0 | } |
1187 | 0 | } |
1188 | 0 | vq->inuse -= ndescs; |
1189 | 0 | } |
1190 | | |
1191 | | void virtqueue_flush(VirtQueue *vq, unsigned int count) |
1192 | 3.98M | { |
1193 | 3.98M | if (virtio_device_disabled(vq->vdev)) { |
1194 | 61.0k | vq->inuse -= count; |
1195 | 61.0k | return; |
1196 | 61.0k | } |
1197 | | |
1198 | 3.92M | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) { |
1199 | 0 | virtqueue_ordered_flush(vq); |
1200 | 3.92M | } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
1201 | 0 | virtqueue_packed_flush(vq, count); |
1202 | 3.92M | } else { |
1203 | 3.92M | virtqueue_split_flush(vq, count); |
1204 | 3.92M | } |
1205 | 3.92M | } |
1206 | | |
1207 | | void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, |
1208 | | unsigned int len) |
1209 | 3.94M | { |
1210 | 3.94M | RCU_READ_LOCK_GUARD(); |
1211 | 3.94M | virtqueue_fill(vq, elem, len, 0); |
1212 | 3.94M | virtqueue_flush(vq, 1); |
1213 | 3.94M | } |
1214 | | |
1215 | | /* Called within rcu_read_lock(). */ |
1216 | | static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) |
1217 | 9.11M | { |
1218 | 9.11M | uint16_t avail_idx, num_heads; |
1219 | | |
1220 | | /* Use shadow index whenever possible. */ |
1221 | 9.11M | avail_idx = (vq->shadow_avail_idx != idx) ? vq->shadow_avail_idx |
1222 | 9.11M | : vring_avail_idx(vq); |
1223 | 9.11M | num_heads = avail_idx - idx; |
1224 | | |
1225 | | /* Check it isn't doing very strange things with descriptor numbers. */ |
1226 | 9.11M | if (num_heads > vq->vring.num) { |
1227 | 28.5k | virtio_error(vq->vdev, "Guest moved used index from %u to %u", |
1228 | 28.5k | idx, vq->shadow_avail_idx); |
1229 | 28.5k | return -EINVAL; |
1230 | 28.5k | } |
1231 | | /* |
1232 | | * On success, callers read a descriptor at vq->last_avail_idx. |
1233 | | * Make sure descriptor read does not bypass avail index read. |
1234 | | * |
1235 | | * This is necessary even if we are using a shadow index, since |
1236 | | * the shadow index could have been initialized by calling |
1237 | | * vring_avail_idx() outside of this function, i.e., by a guest |
1238 | | * memory read not accompanied by a barrier. |
1239 | | */ |
1240 | 9.08M | if (num_heads) { |
1241 | 8.93M | smp_rmb(); |
1242 | 8.93M | } |
1243 | | |
1244 | 9.08M | return num_heads; |
1245 | 9.11M | } |
1246 | | |
1247 | | /* Called within rcu_read_lock(). */ |
1248 | | static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx, |
1249 | | unsigned int *head) |
1250 | 8.93M | { |
1251 | | /* Grab the next descriptor number they're advertising, and increment |
1252 | | * the index we've seen. */ |
1253 | 8.93M | *head = vring_avail_ring(vq, idx % vq->vring.num); |
1254 | | |
1255 | | /* If their number is silly, that's a fatal mistake. */ |
1256 | 8.93M | if (*head >= vq->vring.num) { |
1257 | 30.3k | virtio_error(vq->vdev, "Guest says index %u is available", *head); |
1258 | 30.3k | return false; |
1259 | 30.3k | } |
1260 | | |
1261 | 8.89M | return true; |
1262 | 8.93M | } |
1263 | | |
1264 | | enum { |
1265 | | VIRTQUEUE_READ_DESC_ERROR = -1, |
1266 | | VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */ |
1267 | | VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */ |
1268 | | }; |
1269 | | |
1270 | | /* Reads the 'desc->next' descriptor into '*desc'. */ |
1271 | | static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc, |
1272 | | MemoryRegionCache *desc_cache, |
1273 | | unsigned int max) |
1274 | 91.4M | { |
1275 | | /* If this descriptor says it doesn't chain, we're done. */ |
1276 | 91.4M | if (!(desc->flags & VRING_DESC_F_NEXT)) { |
1277 | 7.83M | return VIRTQUEUE_READ_DESC_DONE; |
1278 | 7.83M | } |
1279 | | |
1280 | | /* Check they're not leading us off end of descriptors. */ |
1281 | 83.5M | if (desc->next >= max) { |
1282 | 3.97k | virtio_error(vdev, "Desc next is %u", desc->next); |
1283 | 3.97k | return VIRTQUEUE_READ_DESC_ERROR; |
1284 | 3.97k | } |
1285 | | |
1286 | 83.5M | vring_split_desc_read(vdev, desc, desc_cache, desc->next); |
1287 | 83.5M | return VIRTQUEUE_READ_DESC_MORE; |
1288 | 83.5M | } |
1289 | | |
1290 | | /* Called within rcu_read_lock(). */ |
1291 | | static void virtqueue_split_get_avail_bytes(VirtQueue *vq, |
1292 | | unsigned int *in_bytes, unsigned int *out_bytes, |
1293 | | unsigned max_in_bytes, unsigned max_out_bytes, |
1294 | | VRingMemoryRegionCaches *caches) |
1295 | 1.02M | { |
1296 | 1.02M | VirtIODevice *vdev = vq->vdev; |
1297 | 1.02M | unsigned int idx; |
1298 | 1.02M | unsigned int total_bufs, in_total, out_total; |
1299 | 1.02M | MemoryRegionCache indirect_desc_cache; |
1300 | 1.02M | int64_t len = 0; |
1301 | 1.02M | int rc; |
1302 | | |
1303 | 1.02M | address_space_cache_init_empty(&indirect_desc_cache); |
1304 | | |
1305 | 1.02M | idx = vq->last_avail_idx; |
1306 | 1.02M | total_bufs = in_total = out_total = 0; |
1307 | | |
1308 | 4.68M | while ((rc = virtqueue_num_heads(vq, idx)) > 0) { |
1309 | 4.64M | MemoryRegionCache *desc_cache = &caches->desc; |
1310 | 4.64M | unsigned int num_bufs; |
1311 | 4.64M | VRingDesc desc; |
1312 | 4.64M | unsigned int i; |
1313 | 4.64M | unsigned int max = vq->vring.num; |
1314 | | |
1315 | 4.64M | num_bufs = total_bufs; |
1316 | | |
1317 | 4.64M | if (!virtqueue_get_head(vq, idx++, &i)) { |
1318 | 3.80k | goto err; |
1319 | 3.80k | } |
1320 | | |
1321 | 4.64M | vring_split_desc_read(vdev, &desc, desc_cache, i); |
1322 | | |
1323 | 4.64M | if (desc.flags & VRING_DESC_F_INDIRECT) { |
1324 | 101k | if (!desc.len || (desc.len % sizeof(VRingDesc))) { |
1325 | 1.53k | virtio_error(vdev, "Invalid size for indirect buffer table"); |
1326 | 1.53k | goto err; |
1327 | 1.53k | } |
1328 | | |
1329 | | /* If we've got too many, that implies a descriptor loop. */ |
1330 | 100k | if (num_bufs >= max) { |
1331 | 202 | virtio_error(vdev, "Looped descriptor"); |
1332 | 202 | goto err; |
1333 | 202 | } |
1334 | | |
1335 | | /* loop over the indirect descriptor table */ |
1336 | 99.9k | len = address_space_cache_init(&indirect_desc_cache, |
1337 | 99.9k | vdev->dma_as, |
1338 | 99.9k | desc.addr, desc.len, false); |
1339 | 99.9k | desc_cache = &indirect_desc_cache; |
1340 | 99.9k | if (len < desc.len) { |
1341 | 253 | virtio_error(vdev, "Cannot map indirect buffer"); |
1342 | 253 | goto err; |
1343 | 253 | } |
1344 | | |
1345 | 99.6k | max = desc.len / sizeof(VRingDesc); |
1346 | 99.6k | num_bufs = i = 0; |
1347 | 99.6k | vring_split_desc_read(vdev, &desc, desc_cache, i); |
1348 | 99.6k | } |
1349 | | |
1350 | 82.2M | do { |
1351 | | /* If we've got too many, that implies a descriptor loop. */ |
1352 | 82.2M | if (++num_bufs > max) { |
1353 | 422 | virtio_error(vdev, "Looped descriptor"); |
1354 | 422 | goto err; |
1355 | 422 | } |
1356 | | |
1357 | 82.2M | if (desc.flags & VRING_DESC_F_WRITE) { |
1358 | 3.37M | in_total += desc.len; |
1359 | 78.8M | } else { |
1360 | 78.8M | out_total += desc.len; |
1361 | 78.8M | } |
1362 | 82.2M | if (in_total >= max_in_bytes && out_total >= max_out_bytes) { |
1363 | 977k | goto done; |
1364 | 977k | } |
1365 | | |
1366 | 81.2M | rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max); |
1367 | 81.2M | } while (rc == VIRTQUEUE_READ_DESC_MORE); |
1368 | | |
1369 | 3.66M | if (rc == VIRTQUEUE_READ_DESC_ERROR) { |
1370 | 611 | goto err; |
1371 | 611 | } |
1372 | | |
1373 | 3.66M | if (desc_cache == &indirect_desc_cache) { |
1374 | 70.3k | address_space_cache_destroy(&indirect_desc_cache); |
1375 | 70.3k | total_bufs++; |
1376 | 3.59M | } else { |
1377 | 3.59M | total_bufs = num_bufs; |
1378 | 3.59M | } |
1379 | 3.66M | } |
1380 | | |
1381 | 37.8k | if (rc < 0) { |
1382 | 6.30k | goto err; |
1383 | 6.30k | } |
1384 | | |
1385 | 1.02M | done: |
1386 | 1.02M | address_space_cache_destroy(&indirect_desc_cache); |
1387 | 1.02M | if (in_bytes) { |
1388 | 1.02M | *in_bytes = in_total; |
1389 | 1.02M | } |
1390 | 1.02M | if (out_bytes) { |
1391 | 0 | *out_bytes = out_total; |
1392 | 0 | } |
1393 | 1.02M | return; |
1394 | | |
1395 | 13.1k | err: |
1396 | 13.1k | in_total = out_total = 0; |
1397 | 13.1k | goto done; |
1398 | 37.8k | } |
1399 | | |
1400 | | static int virtqueue_packed_read_next_desc(VirtQueue *vq, |
1401 | | VRingPackedDesc *desc, |
1402 | | MemoryRegionCache |
1403 | | *desc_cache, |
1404 | | unsigned int max, |
1405 | | unsigned int *next, |
1406 | | bool indirect) |
1407 | 0 | { |
1408 | | /* If this descriptor says it doesn't chain, we're done. */ |
1409 | 0 | if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) { |
1410 | 0 | return VIRTQUEUE_READ_DESC_DONE; |
1411 | 0 | } |
1412 | | |
1413 | 0 | ++*next; |
1414 | 0 | if (*next == max) { |
1415 | 0 | if (indirect) { |
1416 | 0 | return VIRTQUEUE_READ_DESC_DONE; |
1417 | 0 | } else { |
1418 | 0 | (*next) -= vq->vring.num; |
1419 | 0 | } |
1420 | 0 | } |
1421 | | |
1422 | 0 | vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false); |
1423 | 0 | return VIRTQUEUE_READ_DESC_MORE; |
1424 | 0 | } |
1425 | | |
1426 | | /* Called within rcu_read_lock(). */ |
1427 | | static void virtqueue_packed_get_avail_bytes(VirtQueue *vq, |
1428 | | unsigned int *in_bytes, |
1429 | | unsigned int *out_bytes, |
1430 | | unsigned max_in_bytes, |
1431 | | unsigned max_out_bytes, |
1432 | | VRingMemoryRegionCaches *caches) |
1433 | 0 | { |
1434 | 0 | VirtIODevice *vdev = vq->vdev; |
1435 | 0 | unsigned int idx; |
1436 | 0 | unsigned int total_bufs, in_total, out_total; |
1437 | 0 | MemoryRegionCache indirect_desc_cache; |
1438 | 0 | MemoryRegionCache *desc_cache; |
1439 | 0 | int64_t len = 0; |
1440 | 0 | VRingPackedDesc desc; |
1441 | 0 | bool wrap_counter; |
1442 | |
|
1443 | 0 | address_space_cache_init_empty(&indirect_desc_cache); |
1444 | |
|
1445 | 0 | idx = vq->last_avail_idx; |
1446 | 0 | wrap_counter = vq->last_avail_wrap_counter; |
1447 | 0 | total_bufs = in_total = out_total = 0; |
1448 | |
|
1449 | 0 | for (;;) { |
1450 | 0 | unsigned int num_bufs = total_bufs; |
1451 | 0 | unsigned int i = idx; |
1452 | 0 | int rc; |
1453 | 0 | unsigned int max = vq->vring.num; |
1454 | |
|
1455 | 0 | desc_cache = &caches->desc; |
1456 | |
|
1457 | 0 | vring_packed_desc_read(vdev, &desc, desc_cache, idx, true); |
1458 | 0 | if (!is_desc_avail(desc.flags, wrap_counter)) { |
1459 | 0 | break; |
1460 | 0 | } |
1461 | | |
1462 | 0 | if (desc.flags & VRING_DESC_F_INDIRECT) { |
1463 | 0 | if (!desc.len || (desc.len % sizeof(VRingPackedDesc))) { |
1464 | 0 | virtio_error(vdev, "Invalid size for indirect buffer table"); |
1465 | 0 | goto err; |
1466 | 0 | } |
1467 | | |
1468 | | /* If we've got too many, that implies a descriptor loop. */ |
1469 | 0 | if (num_bufs >= max) { |
1470 | 0 | virtio_error(vdev, "Looped descriptor"); |
1471 | 0 | goto err; |
1472 | 0 | } |
1473 | | |
1474 | | /* loop over the indirect descriptor table */ |
1475 | 0 | len = address_space_cache_init(&indirect_desc_cache, |
1476 | 0 | vdev->dma_as, |
1477 | 0 | desc.addr, desc.len, false); |
1478 | 0 | desc_cache = &indirect_desc_cache; |
1479 | 0 | if (len < desc.len) { |
1480 | 0 | virtio_error(vdev, "Cannot map indirect buffer"); |
1481 | 0 | goto err; |
1482 | 0 | } |
1483 | | |
1484 | 0 | max = desc.len / sizeof(VRingPackedDesc); |
1485 | 0 | num_bufs = i = 0; |
1486 | 0 | vring_packed_desc_read(vdev, &desc, desc_cache, i, false); |
1487 | 0 | } |
1488 | | |
1489 | 0 | do { |
1490 | | /* If we've got too many, that implies a descriptor loop. */ |
1491 | 0 | if (++num_bufs > max) { |
1492 | 0 | virtio_error(vdev, "Looped descriptor"); |
1493 | 0 | goto err; |
1494 | 0 | } |
1495 | | |
1496 | 0 | if (desc.flags & VRING_DESC_F_WRITE) { |
1497 | 0 | in_total += desc.len; |
1498 | 0 | } else { |
1499 | 0 | out_total += desc.len; |
1500 | 0 | } |
1501 | 0 | if (in_total >= max_in_bytes && out_total >= max_out_bytes) { |
1502 | 0 | goto done; |
1503 | 0 | } |
1504 | | |
1505 | 0 | rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, |
1506 | 0 | &i, desc_cache == |
1507 | 0 | &indirect_desc_cache); |
1508 | 0 | } while (rc == VIRTQUEUE_READ_DESC_MORE); |
1509 | | |
1510 | 0 | if (desc_cache == &indirect_desc_cache) { |
1511 | 0 | address_space_cache_destroy(&indirect_desc_cache); |
1512 | 0 | total_bufs++; |
1513 | 0 | idx++; |
1514 | 0 | } else { |
1515 | 0 | idx += num_bufs - total_bufs; |
1516 | 0 | total_bufs = num_bufs; |
1517 | 0 | } |
1518 | |
|
1519 | 0 | if (idx >= vq->vring.num) { |
1520 | 0 | idx -= vq->vring.num; |
1521 | 0 | wrap_counter ^= 1; |
1522 | 0 | } |
1523 | 0 | } |
1524 | | |
1525 | | /* Record the index and wrap counter for a kick we want */ |
1526 | 0 | vq->shadow_avail_idx = idx; |
1527 | 0 | vq->shadow_avail_wrap_counter = wrap_counter; |
1528 | 0 | done: |
1529 | 0 | address_space_cache_destroy(&indirect_desc_cache); |
1530 | 0 | if (in_bytes) { |
1531 | 0 | *in_bytes = in_total; |
1532 | 0 | } |
1533 | 0 | if (out_bytes) { |
1534 | 0 | *out_bytes = out_total; |
1535 | 0 | } |
1536 | 0 | return; |
1537 | | |
1538 | 0 | err: |
1539 | 0 | in_total = out_total = 0; |
1540 | 0 | goto done; |
1541 | 0 | } |
1542 | | |
1543 | | int virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes, |
1544 | | unsigned int *out_bytes, unsigned max_in_bytes, |
1545 | | unsigned max_out_bytes) |
1546 | 1.02M | { |
1547 | 1.02M | uint16_t desc_size; |
1548 | 1.02M | VRingMemoryRegionCaches *caches; |
1549 | | |
1550 | 1.02M | RCU_READ_LOCK_GUARD(); |
1551 | | |
1552 | 1.02M | if (unlikely(!vq->vring.desc)) { |
1553 | 2.51k | goto err; |
1554 | 2.51k | } |
1555 | | |
1556 | 1.02M | caches = vring_get_region_caches(vq); |
1557 | 1.02M | if (!caches) { |
1558 | 1.15k | goto err; |
1559 | 1.15k | } |
1560 | | |
1561 | 1.02M | desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? |
1562 | 1.02M | sizeof(VRingPackedDesc) : sizeof(VRingDesc); |
1563 | 1.02M | if (caches->desc.len < vq->vring.num * desc_size) { |
1564 | 0 | virtio_error(vq->vdev, "Cannot map descriptor ring"); |
1565 | 0 | goto err; |
1566 | 0 | } |
1567 | | |
1568 | 1.02M | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
1569 | 0 | virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes, |
1570 | 0 | max_in_bytes, max_out_bytes, |
1571 | 0 | caches); |
1572 | 1.02M | } else { |
1573 | 1.02M | virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes, |
1574 | 1.02M | max_in_bytes, max_out_bytes, |
1575 | 1.02M | caches); |
1576 | 1.02M | } |
1577 | | |
1578 | 1.02M | return (int)vq->shadow_avail_idx; |
1579 | 3.67k | err: |
1580 | 3.67k | if (in_bytes) { |
1581 | 3.67k | *in_bytes = 0; |
1582 | 3.67k | } |
1583 | 3.67k | if (out_bytes) { |
1584 | 0 | *out_bytes = 0; |
1585 | 0 | } |
1586 | | |
1587 | 3.67k | return -1; |
1588 | 1.02M | } |
1589 | | |
1590 | | int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes, |
1591 | | unsigned int out_bytes) |
1592 | 0 | { |
1593 | 0 | unsigned int in_total, out_total; |
1594 | |
|
1595 | 0 | virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes); |
1596 | 0 | return in_bytes <= in_total && out_bytes <= out_total; |
1597 | 0 | } |
1598 | | |
1599 | | static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg, |
1600 | | hwaddr *addr, struct iovec *iov, |
1601 | | unsigned int max_num_sg, bool is_write, |
1602 | | hwaddr pa, size_t sz) |
1603 | 10.2M | { |
1604 | 10.2M | bool ok = false; |
1605 | 10.2M | unsigned num_sg = *p_num_sg; |
1606 | 10.2M | assert(num_sg <= max_num_sg); |
1607 | | |
1608 | 10.2M | if (!sz) { |
1609 | 19.9k | virtio_error(vdev, "virtio: zero sized buffers are not allowed"); |
1610 | 19.9k | goto out; |
1611 | 19.9k | } |
1612 | | |
1613 | 23.0M | while (sz) { |
1614 | 12.8M | hwaddr len = sz; |
1615 | | |
1616 | 12.8M | if (num_sg == max_num_sg) { |
1617 | 6.78k | virtio_error(vdev, "virtio: too many write descriptors in " |
1618 | 6.78k | "indirect table"); |
1619 | 6.78k | goto out; |
1620 | 6.78k | } |
1621 | | |
1622 | 12.8M | iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len, |
1623 | 12.8M | is_write ? |
1624 | 2.75M | DMA_DIRECTION_FROM_DEVICE : |
1625 | 12.8M | DMA_DIRECTION_TO_DEVICE, |
1626 | 12.8M | MEMTXATTRS_UNSPECIFIED); |
1627 | 12.8M | if (!iov[num_sg].iov_base) { |
1628 | 23.7k | virtio_error(vdev, "virtio: bogus descriptor or out of resources"); |
1629 | 23.7k | goto out; |
1630 | 23.7k | } |
1631 | | |
1632 | 12.8M | iov[num_sg].iov_len = len; |
1633 | 12.8M | addr[num_sg] = pa; |
1634 | | |
1635 | 12.8M | sz -= len; |
1636 | 12.8M | pa += len; |
1637 | 12.8M | num_sg++; |
1638 | 12.8M | } |
1639 | 10.2M | ok = true; |
1640 | | |
1641 | 10.2M | out: |
1642 | 10.2M | *p_num_sg = num_sg; |
1643 | 10.2M | return ok; |
1644 | 10.2M | } |
1645 | | |
1646 | | /* Only used by error code paths before we have a VirtQueueElement (therefore |
1647 | | * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to |
1648 | | * yet. |
1649 | | */ |
1650 | | static void virtqueue_undo_map_desc(AddressSpace *as, |
1651 | | unsigned int out_num, unsigned int in_num, |
1652 | | struct iovec *iov) |
1653 | 60.8k | { |
1654 | 60.8k | unsigned int i; |
1655 | | |
1656 | 8.21M | for (i = 0; i < out_num + in_num; i++) { |
1657 | 8.15M | int is_write = i >= out_num; |
1658 | | |
1659 | 8.15M | address_space_unmap(as, iov->iov_base, iov->iov_len, is_write, 0); |
1660 | 8.15M | iov++; |
1661 | 8.15M | } |
1662 | 60.8k | } |
1663 | | |
1664 | | static bool virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg, |
1665 | | hwaddr *addr, unsigned int num_sg, |
1666 | | bool is_write) |
1667 | 0 | { |
1668 | 0 | unsigned int i; |
1669 | 0 | hwaddr len; |
1670 | 0 | DMADirection dir = is_write ? DMA_DIRECTION_FROM_DEVICE : |
1671 | 0 | DMA_DIRECTION_TO_DEVICE; |
1672 | |
|
1673 | 0 | for (i = 0; i < num_sg; i++) { |
1674 | 0 | len = sg[i].iov_len; |
1675 | 0 | sg[i].iov_base = dma_memory_map(vdev->dma_as, addr[i], &len, |
1676 | 0 | dir, MEMTXATTRS_UNSPECIFIED); |
1677 | 0 | if (!sg[i].iov_base) { |
1678 | 0 | error_report("virtio: error trying to map MMIO memory"); |
1679 | 0 | goto err_undo_map; |
1680 | 0 | } |
1681 | 0 | if (len != sg[i].iov_len) { |
1682 | 0 | error_report("virtio: unexpected memory split"); |
1683 | 0 | dma_memory_unmap(vdev->dma_as, sg[i].iov_base, len, dir, 0); |
1684 | 0 | goto err_undo_map; |
1685 | 0 | } |
1686 | 0 | } |
1687 | 0 | return true; |
1688 | | |
1689 | 0 | err_undo_map: |
1690 | 0 | while (i-- > 0) { |
1691 | 0 | dma_memory_unmap(vdev->dma_as, sg[i].iov_base, sg[i].iov_len, |
1692 | 0 | dir, 0); |
1693 | 0 | } |
1694 | 0 | return false; |
1695 | 0 | } |
1696 | | |
1697 | | bool virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem) |
1698 | 0 | { |
1699 | 0 | if (!virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, |
1700 | 0 | elem->in_num, true)) { |
1701 | 0 | return false; |
1702 | 0 | } |
1703 | 0 | if (!virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, |
1704 | 0 | elem->out_num, false)) { |
1705 | 0 | unsigned int i; |
1706 | 0 | for (i = 0; i < elem->in_num; i++) { |
1707 | 0 | dma_memory_unmap(vdev->dma_as, elem->in_sg[i].iov_base, |
1708 | 0 | elem->in_sg[i].iov_len, |
1709 | 0 | DMA_DIRECTION_FROM_DEVICE, 0); |
1710 | 0 | } |
1711 | 0 | return false; |
1712 | 0 | } |
1713 | 0 | return true; |
1714 | 0 | } |
1715 | | |
1716 | | static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num) |
1717 | 4.16M | { |
1718 | 4.16M | VirtQueueElement *elem; |
1719 | 4.16M | size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0])); |
1720 | 4.16M | size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]); |
1721 | 4.16M | size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]); |
1722 | 4.16M | size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0])); |
1723 | 4.16M | size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); |
1724 | 4.16M | size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); |
1725 | | |
1726 | 4.16M | assert(sz >= sizeof(VirtQueueElement)); |
1727 | 4.16M | elem = g_malloc(out_sg_end); |
1728 | 4.16M | trace_virtqueue_alloc_element(elem, sz, in_num, out_num); |
1729 | 4.16M | elem->out_num = out_num; |
1730 | 4.16M | elem->in_num = in_num; |
1731 | 4.16M | elem->in_addr = (void *)elem + in_addr_ofs; |
1732 | 4.16M | elem->out_addr = (void *)elem + out_addr_ofs; |
1733 | 4.16M | elem->in_sg = (void *)elem + in_sg_ofs; |
1734 | 4.16M | elem->out_sg = (void *)elem + out_sg_ofs; |
1735 | 4.16M | return elem; |
1736 | 4.16M | } |
1737 | | |
1738 | | static void *virtqueue_split_pop(VirtQueue *vq, size_t sz) |
1739 | 4.43M | { |
1740 | 4.43M | unsigned int i, head, max, idx; |
1741 | 4.43M | VRingMemoryRegionCaches *caches; |
1742 | 4.43M | MemoryRegionCache indirect_desc_cache; |
1743 | 4.43M | MemoryRegionCache *desc_cache; |
1744 | 4.43M | int64_t len; |
1745 | 4.43M | VirtIODevice *vdev = vq->vdev; |
1746 | 4.43M | VirtQueueElement *elem = NULL; |
1747 | 4.43M | unsigned out_num, in_num, elem_entries; |
1748 | 4.43M | hwaddr QEMU_UNINITIALIZED addr[VIRTQUEUE_MAX_SIZE]; |
1749 | 4.43M | struct iovec QEMU_UNINITIALIZED iov[VIRTQUEUE_MAX_SIZE]; |
1750 | 4.43M | VRingDesc desc; |
1751 | 4.43M | int rc; |
1752 | | |
1753 | 4.43M | address_space_cache_init_empty(&indirect_desc_cache); |
1754 | | |
1755 | 4.43M | RCU_READ_LOCK_GUARD(); |
1756 | 4.43M | if (unlikely(!vq->vring.avail)) { |
1757 | 2.98k | goto done; |
1758 | 2.98k | } |
1759 | | |
1760 | 4.43M | rc = virtqueue_num_heads(vq, vq->last_avail_idx); |
1761 | 4.43M | if (rc <= 0) { |
1762 | 148k | goto done; |
1763 | 148k | } |
1764 | | |
1765 | | /* When we start there are none of either input nor output. */ |
1766 | 4.28M | out_num = in_num = elem_entries = 0; |
1767 | | |
1768 | 4.28M | max = vq->vring.num; |
1769 | | |
1770 | 4.28M | if (vq->inuse >= vq->vring.num) { |
1771 | 20 | virtio_error(vdev, "Virtqueue size exceeded"); |
1772 | 20 | goto done; |
1773 | 20 | } |
1774 | | |
1775 | 4.28M | if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) { |
1776 | 26.5k | goto done; |
1777 | 26.5k | } |
1778 | | |
1779 | 4.25M | if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { |
1780 | 888k | vring_set_avail_event(vq, vq->last_avail_idx); |
1781 | 888k | } |
1782 | | |
1783 | 4.25M | i = head; |
1784 | | |
1785 | 4.25M | caches = vring_get_region_caches(vq); |
1786 | 4.25M | if (!caches) { |
1787 | 116 | virtio_error(vdev, "Region caches not initialized"); |
1788 | 116 | goto done; |
1789 | 116 | } |
1790 | | |
1791 | 4.25M | if (caches->desc.len < max * sizeof(VRingDesc)) { |
1792 | 0 | virtio_error(vdev, "Cannot map descriptor ring"); |
1793 | 0 | goto done; |
1794 | 0 | } |
1795 | | |
1796 | 4.25M | desc_cache = &caches->desc; |
1797 | 4.25M | vring_split_desc_read(vdev, &desc, desc_cache, i); |
1798 | 4.25M | if (desc.flags & VRING_DESC_F_INDIRECT) { |
1799 | 207k | if (!desc.len || (desc.len % sizeof(VRingDesc))) { |
1800 | 21.3k | virtio_error(vdev, "Invalid size for indirect buffer table"); |
1801 | 21.3k | goto done; |
1802 | 21.3k | } |
1803 | 186k | virtio_check_indirect_feature(vdev); |
1804 | | |
1805 | | /* loop over the indirect descriptor table */ |
1806 | 186k | len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, |
1807 | 186k | desc.addr, desc.len, false); |
1808 | 186k | desc_cache = &indirect_desc_cache; |
1809 | 186k | if (len < desc.len) { |
1810 | 3.20k | virtio_error(vdev, "Cannot map indirect buffer"); |
1811 | 3.20k | goto done; |
1812 | 3.20k | } |
1813 | | |
1814 | 182k | max = desc.len / sizeof(VRingDesc); |
1815 | 182k | i = 0; |
1816 | 182k | vring_split_desc_read(vdev, &desc, desc_cache, i); |
1817 | 182k | } |
1818 | | |
1819 | | /* Collect all the descriptors */ |
1820 | 10.2M | do { |
1821 | 10.2M | bool map_ok; |
1822 | | |
1823 | 10.2M | if (desc.flags & VRING_DESC_F_WRITE) { |
1824 | 2.74M | map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, |
1825 | 2.74M | iov + out_num, |
1826 | 2.74M | VIRTQUEUE_MAX_SIZE - out_num, true, |
1827 | 2.74M | desc.addr, desc.len); |
1828 | 7.51M | } else { |
1829 | 7.51M | if (in_num) { |
1830 | 2.25k | virtio_error(vdev, "Incorrect order for descriptors"); |
1831 | 2.25k | goto err_undo_map; |
1832 | 2.25k | } |
1833 | 7.51M | map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, |
1834 | 7.51M | VIRTQUEUE_MAX_SIZE, false, |
1835 | 7.51M | desc.addr, desc.len); |
1836 | 7.51M | } |
1837 | 10.2M | if (!map_ok) { |
1838 | 50.5k | goto err_undo_map; |
1839 | 50.5k | } |
1840 | | |
1841 | | /* If we've got too many, that implies a descriptor loop. */ |
1842 | 10.2M | if (++elem_entries > max) { |
1843 | 4.75k | virtio_error(vdev, "Looped descriptor"); |
1844 | 4.75k | goto err_undo_map; |
1845 | 4.75k | } |
1846 | | |
1847 | 10.2M | rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max); |
1848 | 10.2M | } while (rc == VIRTQUEUE_READ_DESC_MORE); |
1849 | | |
1850 | 4.17M | if (rc == VIRTQUEUE_READ_DESC_ERROR) { |
1851 | 3.36k | goto err_undo_map; |
1852 | 3.36k | } |
1853 | | |
1854 | | /* Now copy what we have collected and mapped */ |
1855 | 4.16M | elem = virtqueue_alloc_element(sz, out_num, in_num); |
1856 | 4.16M | elem->index = head; |
1857 | 4.16M | elem->ndescs = 1; |
1858 | 8.01M | for (i = 0; i < out_num; i++) { |
1859 | 3.84M | elem->out_addr[i] = addr[i]; |
1860 | 3.84M | elem->out_sg[i] = iov[i]; |
1861 | 3.84M | } |
1862 | 5.01M | for (i = 0; i < in_num; i++) { |
1863 | 849k | elem->in_addr[i] = addr[out_num + i]; |
1864 | 849k | elem->in_sg[i] = iov[out_num + i]; |
1865 | 849k | } |
1866 | | |
1867 | 4.16M | if (virtio_vdev_has_feature(vdev, VIRTIO_F_IN_ORDER)) { |
1868 | 0 | idx = (vq->last_avail_idx - 1) % vq->vring.num; |
1869 | 0 | vq->used_elems[idx].index = elem->index; |
1870 | 0 | vq->used_elems[idx].len = elem->len; |
1871 | 0 | vq->used_elems[idx].ndescs = elem->ndescs; |
1872 | 0 | } |
1873 | | |
1874 | 4.16M | vq->inuse++; |
1875 | | |
1876 | 4.16M | trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); |
1877 | 4.43M | done: |
1878 | 4.43M | address_space_cache_destroy(&indirect_desc_cache); |
1879 | | |
1880 | 4.43M | return elem; |
1881 | | |
1882 | 60.8k | err_undo_map: |
1883 | 60.8k | virtqueue_undo_map_desc(vdev->dma_as, out_num, in_num, iov); |
1884 | 60.8k | goto done; |
1885 | 4.16M | } |
1886 | | |
1887 | | static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz) |
1888 | 0 | { |
1889 | 0 | unsigned int i, max; |
1890 | 0 | VRingMemoryRegionCaches *caches; |
1891 | 0 | MemoryRegionCache indirect_desc_cache; |
1892 | 0 | MemoryRegionCache *desc_cache; |
1893 | 0 | int64_t len; |
1894 | 0 | VirtIODevice *vdev = vq->vdev; |
1895 | 0 | VirtQueueElement *elem = NULL; |
1896 | 0 | unsigned out_num, in_num, elem_entries; |
1897 | 0 | hwaddr QEMU_UNINITIALIZED addr[VIRTQUEUE_MAX_SIZE]; |
1898 | 0 | struct iovec QEMU_UNINITIALIZED iov[VIRTQUEUE_MAX_SIZE]; |
1899 | 0 | VRingPackedDesc desc; |
1900 | 0 | uint16_t id; |
1901 | 0 | int rc; |
1902 | |
|
1903 | 0 | address_space_cache_init_empty(&indirect_desc_cache); |
1904 | |
|
1905 | 0 | RCU_READ_LOCK_GUARD(); |
1906 | 0 | if (virtio_queue_packed_empty_rcu(vq)) { |
1907 | 0 | goto done; |
1908 | 0 | } |
1909 | | |
1910 | | /* When we start there are none of either input nor output. */ |
1911 | 0 | out_num = in_num = elem_entries = 0; |
1912 | |
|
1913 | 0 | max = vq->vring.num; |
1914 | |
|
1915 | 0 | if (vq->inuse >= vq->vring.num) { |
1916 | 0 | virtio_error(vdev, "Virtqueue size exceeded"); |
1917 | 0 | goto done; |
1918 | 0 | } |
1919 | | |
1920 | 0 | i = vq->last_avail_idx; |
1921 | |
|
1922 | 0 | caches = vring_get_region_caches(vq); |
1923 | 0 | if (!caches) { |
1924 | 0 | virtio_error(vdev, "Region caches not initialized"); |
1925 | 0 | goto done; |
1926 | 0 | } |
1927 | | |
1928 | 0 | if (caches->desc.len < max * sizeof(VRingDesc)) { |
1929 | 0 | virtio_error(vdev, "Cannot map descriptor ring"); |
1930 | 0 | goto done; |
1931 | 0 | } |
1932 | | |
1933 | 0 | desc_cache = &caches->desc; |
1934 | 0 | vring_packed_desc_read(vdev, &desc, desc_cache, i, true); |
1935 | 0 | id = desc.id; |
1936 | 0 | if (desc.flags & VRING_DESC_F_INDIRECT) { |
1937 | 0 | if (!desc.len || (desc.len % sizeof(VRingPackedDesc))) { |
1938 | 0 | virtio_error(vdev, "Invalid size for indirect buffer table"); |
1939 | 0 | goto done; |
1940 | 0 | } |
1941 | 0 | virtio_check_indirect_feature(vdev); |
1942 | | |
1943 | | /* loop over the indirect descriptor table */ |
1944 | 0 | len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, |
1945 | 0 | desc.addr, desc.len, false); |
1946 | 0 | desc_cache = &indirect_desc_cache; |
1947 | 0 | if (len < desc.len) { |
1948 | 0 | virtio_error(vdev, "Cannot map indirect buffer"); |
1949 | 0 | goto done; |
1950 | 0 | } |
1951 | | |
1952 | 0 | max = desc.len / sizeof(VRingPackedDesc); |
1953 | 0 | i = 0; |
1954 | 0 | vring_packed_desc_read(vdev, &desc, desc_cache, i, false); |
1955 | 0 | } |
1956 | | |
1957 | | /* Collect all the descriptors */ |
1958 | 0 | do { |
1959 | 0 | bool map_ok; |
1960 | |
|
1961 | 0 | if (desc.flags & VRING_DESC_F_WRITE) { |
1962 | 0 | map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, |
1963 | 0 | iov + out_num, |
1964 | 0 | VIRTQUEUE_MAX_SIZE - out_num, true, |
1965 | 0 | desc.addr, desc.len); |
1966 | 0 | } else { |
1967 | 0 | if (in_num) { |
1968 | 0 | virtio_error(vdev, "Incorrect order for descriptors"); |
1969 | 0 | goto err_undo_map; |
1970 | 0 | } |
1971 | 0 | map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, |
1972 | 0 | VIRTQUEUE_MAX_SIZE, false, |
1973 | 0 | desc.addr, desc.len); |
1974 | 0 | } |
1975 | 0 | if (!map_ok) { |
1976 | 0 | goto err_undo_map; |
1977 | 0 | } |
1978 | | |
1979 | | /* If we've got too many, that implies a descriptor loop. */ |
1980 | 0 | if (++elem_entries > max) { |
1981 | 0 | virtio_error(vdev, "Looped descriptor"); |
1982 | 0 | goto err_undo_map; |
1983 | 0 | } |
1984 | | |
1985 | 0 | rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, &i, |
1986 | 0 | desc_cache == |
1987 | 0 | &indirect_desc_cache); |
1988 | 0 | } while (rc == VIRTQUEUE_READ_DESC_MORE); |
1989 | | |
1990 | 0 | if (desc_cache != &indirect_desc_cache) { |
1991 | | /* Buffer ID is included in the last descriptor in the list. */ |
1992 | 0 | id = desc.id; |
1993 | 0 | } |
1994 | | |
1995 | | /* Now copy what we have collected and mapped */ |
1996 | 0 | elem = virtqueue_alloc_element(sz, out_num, in_num); |
1997 | 0 | for (i = 0; i < out_num; i++) { |
1998 | 0 | elem->out_addr[i] = addr[i]; |
1999 | 0 | elem->out_sg[i] = iov[i]; |
2000 | 0 | } |
2001 | 0 | for (i = 0; i < in_num; i++) { |
2002 | 0 | elem->in_addr[i] = addr[out_num + i]; |
2003 | 0 | elem->in_sg[i] = iov[out_num + i]; |
2004 | 0 | } |
2005 | |
|
2006 | 0 | elem->index = id; |
2007 | 0 | elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries; |
2008 | |
|
2009 | 0 | if (virtio_vdev_has_feature(vdev, VIRTIO_F_IN_ORDER)) { |
2010 | 0 | vq->used_elems[vq->last_avail_idx].index = elem->index; |
2011 | 0 | vq->used_elems[vq->last_avail_idx].len = elem->len; |
2012 | 0 | vq->used_elems[vq->last_avail_idx].ndescs = elem->ndescs; |
2013 | 0 | } |
2014 | |
|
2015 | 0 | vq->last_avail_idx += elem->ndescs; |
2016 | 0 | vq->inuse += elem->ndescs; |
2017 | |
|
2018 | 0 | if (vq->last_avail_idx >= vq->vring.num) { |
2019 | 0 | vq->last_avail_idx -= vq->vring.num; |
2020 | 0 | vq->last_avail_wrap_counter ^= 1; |
2021 | 0 | } |
2022 | |
|
2023 | 0 | vq->shadow_avail_idx = vq->last_avail_idx; |
2024 | 0 | vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter; |
2025 | |
|
2026 | 0 | trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); |
2027 | 0 | done: |
2028 | 0 | address_space_cache_destroy(&indirect_desc_cache); |
2029 | |
|
2030 | 0 | return elem; |
2031 | | |
2032 | 0 | err_undo_map: |
2033 | 0 | virtqueue_undo_map_desc(vdev->dma_as, out_num, in_num, iov); |
2034 | 0 | goto done; |
2035 | 0 | } |
2036 | | |
2037 | | void *virtqueue_pop(VirtQueue *vq, size_t sz) |
2038 | 5.42M | { |
2039 | 5.42M | if (virtio_device_disabled(vq->vdev)) { |
2040 | 986k | return NULL; |
2041 | 986k | } |
2042 | | |
2043 | 4.43M | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
2044 | 0 | return virtqueue_packed_pop(vq, sz); |
2045 | 4.43M | } else { |
2046 | 4.43M | return virtqueue_split_pop(vq, sz); |
2047 | 4.43M | } |
2048 | 4.43M | } |
2049 | | |
2050 | | static unsigned int virtqueue_packed_drop_all(VirtQueue *vq) |
2051 | 0 | { |
2052 | 0 | VRingMemoryRegionCaches *caches; |
2053 | 0 | MemoryRegionCache *desc_cache; |
2054 | 0 | unsigned int dropped = 0; |
2055 | 0 | VirtQueueElement elem = {}; |
2056 | 0 | VirtIODevice *vdev = vq->vdev; |
2057 | 0 | VRingPackedDesc desc; |
2058 | |
|
2059 | 0 | RCU_READ_LOCK_GUARD(); |
2060 | |
|
2061 | 0 | caches = vring_get_region_caches(vq); |
2062 | 0 | if (!caches) { |
2063 | 0 | return 0; |
2064 | 0 | } |
2065 | | |
2066 | 0 | desc_cache = &caches->desc; |
2067 | |
|
2068 | 0 | virtio_queue_set_notification(vq, 0); |
2069 | |
|
2070 | 0 | while (vq->inuse < vq->vring.num) { |
2071 | 0 | unsigned int idx = vq->last_avail_idx; |
2072 | | /* |
2073 | | * works similar to virtqueue_pop but does not map buffers |
2074 | | * and does not allocate any memory. |
2075 | | */ |
2076 | 0 | vring_packed_desc_read(vdev, &desc, desc_cache, |
2077 | 0 | vq->last_avail_idx , true); |
2078 | 0 | if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) { |
2079 | 0 | break; |
2080 | 0 | } |
2081 | 0 | elem.index = desc.id; |
2082 | 0 | elem.ndescs = 1; |
2083 | 0 | while (virtqueue_packed_read_next_desc(vq, &desc, desc_cache, |
2084 | 0 | vq->vring.num, &idx, false)) { |
2085 | 0 | ++elem.ndescs; |
2086 | 0 | } |
2087 | | /* |
2088 | | * immediately push the element, nothing to unmap |
2089 | | * as both in_num and out_num are set to 0. |
2090 | | */ |
2091 | 0 | virtqueue_push(vq, &elem, 0); |
2092 | 0 | dropped++; |
2093 | 0 | vq->last_avail_idx += elem.ndescs; |
2094 | 0 | if (vq->last_avail_idx >= vq->vring.num) { |
2095 | 0 | vq->last_avail_idx -= vq->vring.num; |
2096 | 0 | vq->last_avail_wrap_counter ^= 1; |
2097 | 0 | } |
2098 | 0 | } |
2099 | |
|
2100 | 0 | return dropped; |
2101 | 0 | } |
2102 | | |
2103 | | static unsigned int virtqueue_split_drop_all(VirtQueue *vq) |
2104 | 0 | { |
2105 | 0 | unsigned int dropped = 0; |
2106 | 0 | VirtQueueElement elem = {}; |
2107 | 0 | VirtIODevice *vdev = vq->vdev; |
2108 | 0 | bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); |
2109 | |
|
2110 | 0 | while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) { |
2111 | | /* works similar to virtqueue_pop but does not map buffers |
2112 | | * and does not allocate any memory */ |
2113 | 0 | smp_rmb(); |
2114 | 0 | if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) { |
2115 | 0 | break; |
2116 | 0 | } |
2117 | 0 | vq->inuse++; |
2118 | 0 | vq->last_avail_idx++; |
2119 | 0 | if (fEventIdx) { |
2120 | 0 | vring_set_avail_event(vq, vq->last_avail_idx); |
2121 | 0 | } |
2122 | | /* immediately push the element, nothing to unmap |
2123 | | * as both in_num and out_num are set to 0 */ |
2124 | 0 | virtqueue_push(vq, &elem, 0); |
2125 | 0 | dropped++; |
2126 | 0 | } |
2127 | |
|
2128 | 0 | return dropped; |
2129 | 0 | } |
2130 | | |
2131 | | /* virtqueue_drop_all: |
2132 | | * @vq: The #VirtQueue |
2133 | | * Drops all queued buffers and indicates them to the guest |
2134 | | * as if they are done. Useful when buffers can not be |
2135 | | * processed but must be returned to the guest. |
2136 | | */ |
2137 | | unsigned int virtqueue_drop_all(VirtQueue *vq) |
2138 | 0 | { |
2139 | 0 | struct VirtIODevice *vdev = vq->vdev; |
2140 | |
|
2141 | 0 | if (virtio_device_disabled(vq->vdev)) { |
2142 | 0 | return 0; |
2143 | 0 | } |
2144 | | |
2145 | 0 | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
2146 | 0 | return virtqueue_packed_drop_all(vq); |
2147 | 0 | } else { |
2148 | 0 | return virtqueue_split_drop_all(vq); |
2149 | 0 | } |
2150 | 0 | } |
2151 | | |
2152 | | /* Reading and writing a structure directly to QEMUFile is *awful*, but |
2153 | | * it is what QEMU has always done by mistake. We can change it sooner |
2154 | | * or later by bumping the version number of the affected vm states. |
2155 | | * In the meanwhile, since the in-memory layout of VirtQueueElement |
2156 | | * has changed, we need to marshal to and from the layout that was |
2157 | | * used before the change. |
2158 | | */ |
2159 | | typedef struct VirtQueueElementOld { |
2160 | | unsigned int index; |
2161 | | unsigned int out_num; |
2162 | | unsigned int in_num; |
2163 | | hwaddr in_addr[VIRTQUEUE_MAX_SIZE]; |
2164 | | hwaddr out_addr[VIRTQUEUE_MAX_SIZE]; |
2165 | | struct iovec in_sg[VIRTQUEUE_MAX_SIZE]; |
2166 | | struct iovec out_sg[VIRTQUEUE_MAX_SIZE]; |
2167 | | } VirtQueueElementOld; |
2168 | | |
2169 | | void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz) |
2170 | 0 | { |
2171 | 0 | VirtQueueElement *elem; |
2172 | 0 | VirtQueueElementOld data; |
2173 | 0 | int i; |
2174 | |
|
2175 | 0 | qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); |
2176 | | |
2177 | | /* TODO: teach all callers that this can fail, and return failure instead |
2178 | | * of asserting here. |
2179 | | * This is just one thing (there are probably more) that must be |
2180 | | * fixed before we can allow NDEBUG compilation. |
2181 | | */ |
2182 | 0 | assert(ARRAY_SIZE(data.in_addr) >= data.in_num); |
2183 | 0 | assert(ARRAY_SIZE(data.out_addr) >= data.out_num); |
2184 | |
|
2185 | 0 | elem = virtqueue_alloc_element(sz, data.out_num, data.in_num); |
2186 | 0 | elem->index = data.index; |
2187 | |
|
2188 | 0 | for (i = 0; i < elem->in_num; i++) { |
2189 | 0 | elem->in_addr[i] = data.in_addr[i]; |
2190 | 0 | } |
2191 | |
|
2192 | 0 | for (i = 0; i < elem->out_num; i++) { |
2193 | 0 | elem->out_addr[i] = data.out_addr[i]; |
2194 | 0 | } |
2195 | |
|
2196 | 0 | for (i = 0; i < elem->in_num; i++) { |
2197 | | /* Base is overwritten by virtqueue_map. */ |
2198 | 0 | elem->in_sg[i].iov_base = 0; |
2199 | 0 | elem->in_sg[i].iov_len = data.in_sg[i].iov_len; |
2200 | 0 | } |
2201 | |
|
2202 | 0 | for (i = 0; i < elem->out_num; i++) { |
2203 | | /* Base is overwritten by virtqueue_map. */ |
2204 | 0 | elem->out_sg[i].iov_base = 0; |
2205 | 0 | elem->out_sg[i].iov_len = data.out_sg[i].iov_len; |
2206 | 0 | } |
2207 | |
|
2208 | 0 | if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
2209 | 0 | qemu_get_be32s(f, &elem->ndescs); |
2210 | 0 | } |
2211 | |
|
2212 | 0 | if (!virtqueue_map(vdev, elem)) { |
2213 | 0 | g_free(elem); |
2214 | 0 | return NULL; |
2215 | 0 | } |
2216 | 0 | return elem; |
2217 | 0 | } |
2218 | | |
2219 | | void qemu_put_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, |
2220 | | VirtQueueElement *elem) |
2221 | 0 | { |
2222 | 0 | VirtQueueElementOld data; |
2223 | 0 | int i; |
2224 | |
|
2225 | 0 | memset(&data, 0, sizeof(data)); |
2226 | 0 | data.index = elem->index; |
2227 | 0 | data.in_num = elem->in_num; |
2228 | 0 | data.out_num = elem->out_num; |
2229 | |
|
2230 | 0 | for (i = 0; i < elem->in_num; i++) { |
2231 | 0 | data.in_addr[i] = elem->in_addr[i]; |
2232 | 0 | } |
2233 | |
|
2234 | 0 | for (i = 0; i < elem->out_num; i++) { |
2235 | 0 | data.out_addr[i] = elem->out_addr[i]; |
2236 | 0 | } |
2237 | |
|
2238 | 0 | for (i = 0; i < elem->in_num; i++) { |
2239 | | /* Base is overwritten by virtqueue_map when loading. Do not |
2240 | | * save it, as it would leak the QEMU address space layout. */ |
2241 | 0 | data.in_sg[i].iov_len = elem->in_sg[i].iov_len; |
2242 | 0 | } |
2243 | |
|
2244 | 0 | for (i = 0; i < elem->out_num; i++) { |
2245 | | /* Do not save iov_base as above. */ |
2246 | 0 | data.out_sg[i].iov_len = elem->out_sg[i].iov_len; |
2247 | 0 | } |
2248 | |
|
2249 | 0 | if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
2250 | 0 | qemu_put_be32s(f, &elem->ndescs); |
2251 | 0 | } |
2252 | |
|
2253 | 0 | qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); |
2254 | 0 | } |
2255 | | |
2256 | | /* virtio device */ |
2257 | | static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) |
2258 | 1.31M | { |
2259 | 1.31M | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
2260 | 1.31M | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
2261 | | |
2262 | 1.31M | if (virtio_device_disabled(vdev)) { |
2263 | 56.6k | return; |
2264 | 56.6k | } |
2265 | | |
2266 | 1.26M | if (k->notify) { |
2267 | 1.26M | k->notify(qbus->parent, vector); |
2268 | 1.26M | } |
2269 | 1.26M | } |
2270 | | |
2271 | | void virtio_update_irq(VirtIODevice *vdev) |
2272 | 0 | { |
2273 | 0 | virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); |
2274 | 0 | } |
2275 | | |
2276 | | static int virtio_validate_features(VirtIODevice *vdev) |
2277 | 16.8k | { |
2278 | 16.8k | VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); |
2279 | | |
2280 | 16.8k | if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) && |
2281 | 0 | !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { |
2282 | 0 | return -EFAULT; |
2283 | 0 | } |
2284 | | |
2285 | 16.8k | if (k->validate_features) { |
2286 | 362 | return k->validate_features(vdev); |
2287 | 16.5k | } else { |
2288 | 16.5k | return 0; |
2289 | 16.5k | } |
2290 | 16.8k | } |
2291 | | |
2292 | | int virtio_set_status(VirtIODevice *vdev, uint8_t val) |
2293 | 1.26M | { |
2294 | 1.26M | VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); |
2295 | 1.26M | trace_virtio_set_status(vdev, val); |
2296 | 1.26M | int ret = 0; |
2297 | | |
2298 | 1.26M | if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { |
2299 | 90.4k | if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && |
2300 | 65.3k | val & VIRTIO_CONFIG_S_FEATURES_OK) { |
2301 | 16.8k | ret = virtio_validate_features(vdev); |
2302 | 16.8k | if (ret) { |
2303 | 148 | return ret; |
2304 | 148 | } |
2305 | 16.8k | } |
2306 | 90.4k | } |
2307 | | |
2308 | 1.26M | if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) != |
2309 | 1.26M | (val & VIRTIO_CONFIG_S_DRIVER_OK)) { |
2310 | 456k | virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK); |
2311 | 456k | } |
2312 | | |
2313 | 1.26M | if (k->set_status) { |
2314 | 596k | ret = k->set_status(vdev, val); |
2315 | 596k | if (ret) { |
2316 | 0 | qemu_log("set %s status to %d failed, old status: %d\n", |
2317 | 0 | vdev->name, val, vdev->status); |
2318 | 0 | } |
2319 | 596k | } |
2320 | 1.26M | vdev->status = val; |
2321 | | |
2322 | 1.26M | return ret; |
2323 | 1.26M | } |
2324 | | |
2325 | | static enum virtio_device_endian virtio_default_endian(void) |
2326 | 370k | { |
2327 | 370k | if (target_big_endian()) { |
2328 | 0 | return VIRTIO_DEVICE_ENDIAN_BIG; |
2329 | 370k | } else { |
2330 | 370k | return VIRTIO_DEVICE_ENDIAN_LITTLE; |
2331 | 370k | } |
2332 | 370k | } |
2333 | | |
2334 | | static enum virtio_device_endian virtio_current_cpu_endian(void) |
2335 | 0 | { |
2336 | 0 | if (cpu_internal_is_big_endian(current_cpu)) { |
2337 | 0 | return VIRTIO_DEVICE_ENDIAN_BIG; |
2338 | 0 | } else { |
2339 | 0 | return VIRTIO_DEVICE_ENDIAN_LITTLE; |
2340 | 0 | } |
2341 | 0 | } |
2342 | | |
2343 | | static void __virtio_queue_reset(VirtIODevice *vdev, uint32_t i) |
2344 | 379M | { |
2345 | 379M | vdev->vq[i].vring.desc = 0; |
2346 | 379M | vdev->vq[i].vring.avail = 0; |
2347 | 379M | vdev->vq[i].vring.used = 0; |
2348 | 379M | vdev->vq[i].last_avail_idx = 0; |
2349 | 379M | vdev->vq[i].shadow_avail_idx = 0; |
2350 | 379M | vdev->vq[i].used_idx = 0; |
2351 | 379M | vdev->vq[i].last_avail_wrap_counter = true; |
2352 | 379M | vdev->vq[i].shadow_avail_wrap_counter = true; |
2353 | 379M | vdev->vq[i].used_wrap_counter = true; |
2354 | 379M | virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR); |
2355 | 379M | vdev->vq[i].signalled_used = 0; |
2356 | 379M | vdev->vq[i].signalled_used_valid = false; |
2357 | 379M | vdev->vq[i].notification = true; |
2358 | 379M | vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; |
2359 | 379M | vdev->vq[i].inuse = 0; |
2360 | 379M | virtio_virtqueue_reset_region_cache(&vdev->vq[i]); |
2361 | 379M | } |
2362 | | |
2363 | | void virtio_queue_reset(VirtIODevice *vdev, uint32_t queue_index) |
2364 | 58.5k | { |
2365 | 58.5k | VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); |
2366 | | |
2367 | 58.5k | if (k->queue_reset) { |
2368 | 3.92k | k->queue_reset(vdev, queue_index); |
2369 | 3.92k | } |
2370 | | |
2371 | 58.5k | __virtio_queue_reset(vdev, queue_index); |
2372 | 58.5k | } |
2373 | | |
2374 | | void virtio_queue_enable(VirtIODevice *vdev, uint32_t queue_index) |
2375 | 101k | { |
2376 | 101k | VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); |
2377 | | |
2378 | | /* |
2379 | | * TODO: Seabios is currently out of spec and triggering this error. |
2380 | | * So this needs to be fixed in Seabios, then this can |
2381 | | * be re-enabled for new machine types only, and also after |
2382 | | * being converted to LOG_GUEST_ERROR. |
2383 | | * |
2384 | | if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { |
2385 | | error_report("queue_enable is only supported in devices of virtio " |
2386 | | "1.0 or later."); |
2387 | | } |
2388 | | */ |
2389 | | |
2390 | 101k | if (k->queue_enable) { |
2391 | 5.12k | k->queue_enable(vdev, queue_index); |
2392 | 5.12k | } |
2393 | 101k | } |
2394 | | |
2395 | | void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr) |
2396 | 460k | { |
2397 | 460k | if (!vdev->vq[n].vring.num) { |
2398 | 22.6k | return; |
2399 | 22.6k | } |
2400 | 438k | vdev->vq[n].vring.desc = addr; |
2401 | 438k | virtio_queue_update_rings(vdev, n); |
2402 | 438k | } |
2403 | | |
2404 | | hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n) |
2405 | 5.41k | { |
2406 | 5.41k | return vdev->vq[n].vring.desc; |
2407 | 5.41k | } |
2408 | | |
2409 | | void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, |
2410 | | hwaddr avail, hwaddr used) |
2411 | 101k | { |
2412 | 101k | if (!vdev->vq[n].vring.num) { |
2413 | 2.81k | return; |
2414 | 2.81k | } |
2415 | 98.1k | vdev->vq[n].vring.desc = desc; |
2416 | 98.1k | vdev->vq[n].vring.avail = avail; |
2417 | 98.1k | vdev->vq[n].vring.used = used; |
2418 | 98.1k | virtio_init_region_cache(vdev, n); |
2419 | 98.1k | } |
2420 | | |
2421 | | void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) |
2422 | 505k | { |
2423 | | /* Don't allow guest to flip queue between existent and |
2424 | | * nonexistent states, or to set it to an invalid size. |
2425 | | */ |
2426 | 505k | if (!!num != !!vdev->vq[n].vring.num || |
2427 | 393k | num > VIRTQUEUE_MAX_SIZE || |
2428 | 403k | num < 0) { |
2429 | 403k | return; |
2430 | 403k | } |
2431 | 101k | if (num > vdev->vq[n].vring.num_default) { |
2432 | 32.3k | virtio_error(vdev, "virtio: queue %d size %d exceeds max size %u", |
2433 | 32.3k | n, num, vdev->vq[n].vring.num_default); |
2434 | 32.3k | return; |
2435 | 32.3k | } |
2436 | 69.1k | vdev->vq[n].vring.num = num; |
2437 | 69.1k | } |
2438 | | |
2439 | | VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector) |
2440 | 0 | { |
2441 | 0 | return QLIST_FIRST(&vdev->vector_queues[vector]); |
2442 | 0 | } |
2443 | | |
2444 | | VirtQueue *virtio_vector_next_queue(VirtQueue *vq) |
2445 | 0 | { |
2446 | 0 | return QLIST_NEXT(vq, node); |
2447 | 0 | } |
2448 | | |
2449 | | int virtio_queue_get_num(VirtIODevice *vdev, int n) |
2450 | 501M | { |
2451 | 501M | return vdev->vq[n].vring.num; |
2452 | 501M | } |
2453 | | |
2454 | | int virtio_queue_get_max_num(VirtIODevice *vdev, int n) |
2455 | 0 | { |
2456 | 0 | return vdev->vq[n].vring.num_default; |
2457 | 0 | } |
2458 | | |
2459 | | int virtio_get_num_queues(VirtIODevice *vdev) |
2460 | 206k | { |
2461 | 206k | int i; |
2462 | | |
2463 | 825k | for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { |
2464 | 825k | if (!virtio_queue_get_num(vdev, i)) { |
2465 | 206k | break; |
2466 | 206k | } |
2467 | 825k | } |
2468 | | |
2469 | 206k | return i; |
2470 | 206k | } |
2471 | | |
2472 | | void virtio_queue_set_align(VirtIODevice *vdev, int n, int align) |
2473 | 0 | { |
2474 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
2475 | 0 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
2476 | | |
2477 | | /* virtio-1 compliant devices cannot change the alignment */ |
2478 | 0 | if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { |
2479 | 0 | error_report("tried to modify queue alignment for virtio-1 device"); |
2480 | 0 | return; |
2481 | 0 | } |
2482 | | /* Check that the transport told us it was going to do this |
2483 | | * (so a buggy transport will immediately assert rather than |
2484 | | * silently failing to migrate this state) |
2485 | | */ |
2486 | 0 | assert(k->has_variable_vring_alignment); |
2487 | |
|
2488 | 0 | if (align) { |
2489 | 0 | vdev->vq[n].vring.align = align; |
2490 | 0 | virtio_queue_update_rings(vdev, n); |
2491 | 0 | } |
2492 | 0 | } |
2493 | | |
2494 | | void virtio_queue_set_shadow_avail_idx(VirtQueue *vq, uint16_t shadow_avail_idx) |
2495 | 0 | { |
2496 | 0 | if (!vq->vring.desc) { |
2497 | 0 | return; |
2498 | 0 | } |
2499 | | |
2500 | | /* |
2501 | | * 16-bit data for packed VQs include 1-bit wrap counter and |
2502 | | * 15-bit shadow_avail_idx. |
2503 | | */ |
2504 | 0 | if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { |
2505 | 0 | vq->shadow_avail_wrap_counter = (shadow_avail_idx >> 15) & 0x1; |
2506 | 0 | vq->shadow_avail_idx = shadow_avail_idx & 0x7FFF; |
2507 | 0 | } else { |
2508 | 0 | vq->shadow_avail_idx = shadow_avail_idx; |
2509 | 0 | } |
2510 | 0 | } |
2511 | | |
2512 | | static void virtio_queue_notify_vq(VirtQueue *vq) |
2513 | 1.03M | { |
2514 | 1.03M | if (vq->vring.desc && vq->handle_output) { |
2515 | 216k | VirtIODevice *vdev = vq->vdev; |
2516 | | |
2517 | 216k | if (unlikely(vdev->broken)) { |
2518 | 54.4k | return; |
2519 | 54.4k | } |
2520 | | |
2521 | 162k | trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); |
2522 | 162k | vq->handle_output(vdev, vq); |
2523 | | |
2524 | 162k | if (unlikely(vdev->start_on_kick)) { |
2525 | 35 | virtio_set_started(vdev, true); |
2526 | 35 | } |
2527 | 162k | } |
2528 | 1.03M | } |
2529 | | |
2530 | | void virtio_queue_notify(VirtIODevice *vdev, int n) |
2531 | 544k | { |
2532 | 544k | VirtQueue *vq = &vdev->vq[n]; |
2533 | | |
2534 | 544k | if (unlikely(!vq->vring.desc || vdev->broken)) { |
2535 | 377k | return; |
2536 | 377k | } |
2537 | | |
2538 | 166k | trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); |
2539 | 166k | if (vq->host_notifier_enabled) { |
2540 | 5.09k | event_notifier_set(&vq->host_notifier); |
2541 | 161k | } else if (vq->handle_output) { |
2542 | 161k | vq->handle_output(vdev, vq); |
2543 | | |
2544 | 161k | if (unlikely(vdev->start_on_kick)) { |
2545 | 3.52k | virtio_set_started(vdev, true); |
2546 | 3.52k | } |
2547 | 161k | } |
2548 | 166k | } |
2549 | | |
2550 | | uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) |
2551 | 240k | { |
2552 | 240k | return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : |
2553 | 240k | VIRTIO_NO_VECTOR; |
2554 | 240k | } |
2555 | | |
2556 | | void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) |
2557 | 379M | { |
2558 | 379M | VirtQueue *vq = &vdev->vq[n]; |
2559 | | |
2560 | 379M | if (n < VIRTIO_QUEUE_MAX) { |
2561 | 379M | if (vdev->vector_queues && |
2562 | 379M | vdev->vq[n].vector != VIRTIO_NO_VECTOR) { |
2563 | 100k | QLIST_REMOVE(vq, node); |
2564 | 100k | } |
2565 | 379M | vdev->vq[n].vector = vector; |
2566 | 379M | if (vdev->vector_queues && |
2567 | 379M | vector != VIRTIO_NO_VECTOR) { |
2568 | 100k | QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); |
2569 | 100k | } |
2570 | 379M | } |
2571 | 379M | } |
2572 | | |
2573 | | VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, |
2574 | | VirtIOHandleOutput handle_output) |
2575 | 179 | { |
2576 | 179 | int i; |
2577 | | |
2578 | 4.31k | for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { |
2579 | 4.31k | if (vdev->vq[i].vring.num == 0) |
2580 | 179 | break; |
2581 | 4.31k | } |
2582 | | |
2583 | 179 | if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) |
2584 | 0 | abort(); |
2585 | | |
2586 | 179 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
2587 | 179 | if (qbus && qbus->parent && |
2588 | 179 | object_property_find(OBJECT(qbus->parent), VIRTIO_QUEUE_SIZE_OVERRIDE)) { |
2589 | 0 | int override = object_property_get_int(OBJECT(qbus->parent), |
2590 | 0 | VIRTIO_QUEUE_SIZE_OVERRIDE, |
2591 | 0 | &error_abort); |
2592 | 0 | if (override) { |
2593 | 0 | queue_size = override; |
2594 | 0 | } |
2595 | 0 | } |
2596 | | |
2597 | 179 | vdev->vq[i].vring.num = queue_size; |
2598 | 179 | vdev->vq[i].vring.num_default = queue_size; |
2599 | 179 | vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; |
2600 | 179 | vdev->vq[i].handle_output = handle_output; |
2601 | 179 | vdev->vq[i].used_elems = g_new0(VirtQueueElement, queue_size); |
2602 | | |
2603 | 179 | return &vdev->vq[i]; |
2604 | 179 | } |
2605 | | |
2606 | | void virtio_delete_queue(VirtQueue *vq) |
2607 | 0 | { |
2608 | 0 | vq->vring.num = 0; |
2609 | 0 | vq->vring.num_default = 0; |
2610 | 0 | vq->handle_output = NULL; |
2611 | 0 | g_free(vq->used_elems); |
2612 | 0 | vq->used_elems = NULL; |
2613 | 0 | virtio_virtqueue_reset_region_cache(vq); |
2614 | 0 | } |
2615 | | |
2616 | | void virtio_del_queue(VirtIODevice *vdev, int n) |
2617 | 0 | { |
2618 | 0 | if (n < 0 || n >= VIRTIO_QUEUE_MAX) { |
2619 | 0 | abort(); |
2620 | 0 | } |
2621 | | |
2622 | 0 | virtio_delete_queue(&vdev->vq[n]); |
2623 | 0 | } |
2624 | | |
2625 | | static void virtio_set_isr(VirtIODevice *vdev, int value) |
2626 | 947k | { |
2627 | 947k | uint8_t old = qatomic_read(&vdev->isr); |
2628 | | |
2629 | | /* Do not write ISR if it does not change, so that its cacheline remains |
2630 | | * shared in the common case where the guest does not read it. |
2631 | | */ |
2632 | 947k | if ((old & value) != value) { |
2633 | 75.5k | qatomic_or(&vdev->isr, value); |
2634 | 75.5k | } |
2635 | 947k | } |
2636 | | |
2637 | | /* Called within rcu_read_lock(). */ |
2638 | | static bool virtio_split_should_notify(VirtIODevice *vdev, VirtQueue *vq) |
2639 | 3.48M | { |
2640 | 3.48M | uint16_t old, new; |
2641 | 3.48M | bool v; |
2642 | | /* We need to expose used array entries before checking used event. */ |
2643 | 3.48M | smp_mb(); |
2644 | | /* Always notify when queue is empty (when feature acknowledge) */ |
2645 | 3.48M | if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) && |
2646 | 348k | !vq->inuse && virtio_queue_empty(vq)) { |
2647 | 16.4k | return true; |
2648 | 16.4k | } |
2649 | | |
2650 | 3.47M | if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { |
2651 | 2.91M | return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); |
2652 | 2.91M | } |
2653 | | |
2654 | 559k | v = vq->signalled_used_valid; |
2655 | 559k | vq->signalled_used_valid = true; |
2656 | 559k | old = vq->signalled_used; |
2657 | 559k | new = vq->signalled_used = vq->used_idx; |
2658 | 559k | return !v || vring_need_event(vring_get_used_event(vq), new, old); |
2659 | 3.47M | } |
2660 | | |
2661 | | static bool vring_packed_need_event(VirtQueue *vq, bool wrap, |
2662 | | uint16_t off_wrap, uint16_t new, |
2663 | | uint16_t old) |
2664 | 0 | { |
2665 | 0 | int off = off_wrap & ~(1 << 15); |
2666 | |
|
2667 | 0 | if (wrap != off_wrap >> 15) { |
2668 | 0 | off -= vq->vring.num; |
2669 | 0 | } |
2670 | |
|
2671 | 0 | return vring_need_event(off, new, old); |
2672 | 0 | } |
2673 | | |
2674 | | /* Called within rcu_read_lock(). */ |
2675 | | static bool virtio_packed_should_notify(VirtIODevice *vdev, VirtQueue *vq) |
2676 | 0 | { |
2677 | 0 | VRingPackedDescEvent e; |
2678 | 0 | uint16_t old, new; |
2679 | 0 | bool v; |
2680 | 0 | VRingMemoryRegionCaches *caches; |
2681 | |
|
2682 | 0 | caches = vring_get_region_caches(vq); |
2683 | 0 | if (!caches) { |
2684 | 0 | return false; |
2685 | 0 | } |
2686 | | |
2687 | 0 | vring_packed_event_read(vdev, &caches->avail, &e); |
2688 | |
|
2689 | 0 | old = vq->signalled_used; |
2690 | 0 | new = vq->signalled_used = vq->used_idx; |
2691 | 0 | v = vq->signalled_used_valid; |
2692 | 0 | vq->signalled_used_valid = true; |
2693 | |
|
2694 | 0 | if (e.flags == VRING_PACKED_EVENT_FLAG_DISABLE) { |
2695 | 0 | return false; |
2696 | 0 | } else if (e.flags == VRING_PACKED_EVENT_FLAG_ENABLE) { |
2697 | 0 | return true; |
2698 | 0 | } |
2699 | | |
2700 | 0 | return !v || vring_packed_need_event(vq, vq->used_wrap_counter, |
2701 | 0 | e.off_wrap, new, old); |
2702 | 0 | } |
2703 | | |
2704 | | /* Called within rcu_read_lock(). */ |
2705 | | static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq) |
2706 | 3.48M | { |
2707 | 3.48M | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
2708 | 0 | return virtio_packed_should_notify(vdev, vq); |
2709 | 3.48M | } else { |
2710 | 3.48M | return virtio_split_should_notify(vdev, vq); |
2711 | 3.48M | } |
2712 | 3.48M | } |
2713 | | |
2714 | | /* Batch irqs while inside a defer_call_begin()/defer_call_end() section */ |
2715 | | static void virtio_notify_irqfd_deferred_fn(void *opaque) |
2716 | 0 | { |
2717 | 0 | EventNotifier *notifier = opaque; |
2718 | 0 | VirtQueue *vq = container_of(notifier, VirtQueue, guest_notifier); |
2719 | |
|
2720 | 0 | trace_virtio_notify_irqfd_deferred_fn(vq->vdev, vq); |
2721 | 0 | event_notifier_set(notifier); |
2722 | 0 | } |
2723 | | |
2724 | | static void virtio_irq(VirtQueue *vq) |
2725 | 884k | { |
2726 | | /* |
2727 | | * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but |
2728 | | * windows drivers included in virtio-win 1.8.0 (circa 2015) are |
2729 | | * incorrectly polling this bit during crashdump and hibernation |
2730 | | * in MSI mode, causing a hang if this bit is never updated. |
2731 | | * Recent releases of Windows do not really shut down, but rather |
2732 | | * log out and hibernate to make the next startup faster. Hence, |
2733 | | * this manifested as a more serious hang during shutdown with |
2734 | | * |
2735 | | * Next driver release from 2016 fixed this problem, so working around it |
2736 | | * is not a must, but it's easy to do so let's do it here. |
2737 | | * |
2738 | | * Note: it's safe to update ISR from any thread as it was switched |
2739 | | * to an atomic operation. |
2740 | | */ |
2741 | 884k | virtio_set_isr(vq->vdev, 0x1); |
2742 | | |
2743 | | /* |
2744 | | * The interrupt code path requires the Big QEMU Lock (BQL), so use the |
2745 | | * notifier instead when in an IOThread. This assumes that device models |
2746 | | * have already called ->set_guest_notifiers() sometime before calling this |
2747 | | * function. |
2748 | | */ |
2749 | 884k | if (qemu_in_iothread()) { |
2750 | 0 | defer_call(virtio_notify_irqfd_deferred_fn, &vq->guest_notifier); |
2751 | 884k | } else { |
2752 | 884k | virtio_notify_vector(vq->vdev, vq->vector); |
2753 | 884k | } |
2754 | 884k | } |
2755 | | |
2756 | | void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) |
2757 | 3.48M | { |
2758 | 3.48M | WITH_RCU_READ_LOCK_GUARD() { |
2759 | 3.48M | if (!virtio_should_notify(vdev, vq)) { |
2760 | 2.60M | return; |
2761 | 2.60M | } |
2762 | 3.48M | } |
2763 | | |
2764 | 884k | trace_virtio_notify(vdev, vq); |
2765 | 884k | virtio_irq(vq); |
2766 | 884k | } |
2767 | | |
2768 | | void virtio_notify_config(VirtIODevice *vdev) |
2769 | 115k | { |
2770 | 115k | if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) |
2771 | 51.4k | return; |
2772 | | |
2773 | 63.7k | virtio_set_isr(vdev, 0x3); |
2774 | 63.7k | vdev->generation++; |
2775 | | |
2776 | 63.7k | if (qemu_in_iothread()) { |
2777 | 0 | defer_call(virtio_notify_irqfd_deferred_fn, &vdev->config_notifier); |
2778 | 63.7k | } else { |
2779 | 63.7k | virtio_notify_vector(vdev, vdev->config_vector); |
2780 | 63.7k | } |
2781 | 63.7k | } |
2782 | | |
2783 | | static bool virtio_device_endian_needed(void *opaque) |
2784 | 0 | { |
2785 | 0 | VirtIODevice *vdev = opaque; |
2786 | |
|
2787 | 0 | assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); |
2788 | 0 | if (virtio_vdev_is_legacy(vdev)) { |
2789 | 0 | return vdev->device_endian != virtio_default_endian(); |
2790 | 0 | } |
2791 | | /* Devices conforming to VIRTIO 1.0 or later are always LE. */ |
2792 | 0 | return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; |
2793 | 0 | } |
2794 | | |
2795 | | static bool virtio_64bit_features_needed(void *opaque) |
2796 | 0 | { |
2797 | 0 | VirtIODevice *vdev = opaque; |
2798 | |
|
2799 | 0 | return (vdev->host_features >> 32) != 0; |
2800 | 0 | } |
2801 | | |
2802 | | static bool virtio_virtqueue_needed(void *opaque) |
2803 | 0 | { |
2804 | 0 | VirtIODevice *vdev = opaque; |
2805 | |
|
2806 | 0 | return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1); |
2807 | 0 | } |
2808 | | |
2809 | | static bool virtio_packed_virtqueue_needed(void *opaque) |
2810 | 0 | { |
2811 | 0 | VirtIODevice *vdev = opaque; |
2812 | |
|
2813 | 0 | return virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED); |
2814 | 0 | } |
2815 | | |
2816 | | static bool virtio_ringsize_needed(void *opaque) |
2817 | 0 | { |
2818 | 0 | return false; |
2819 | 0 | } |
2820 | | |
2821 | | static bool virtio_extra_state_needed(void *opaque) |
2822 | 0 | { |
2823 | 0 | VirtIODevice *vdev = opaque; |
2824 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
2825 | 0 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
2826 | |
|
2827 | 0 | return k->has_extra_state && |
2828 | 0 | k->has_extra_state(qbus->parent); |
2829 | 0 | } |
2830 | | |
2831 | | static bool virtio_broken_needed(void *opaque) |
2832 | 0 | { |
2833 | 0 | VirtIODevice *vdev = opaque; |
2834 | |
|
2835 | 0 | return vdev->broken; |
2836 | 0 | } |
2837 | | |
2838 | | static bool virtio_started_needed(void *opaque) |
2839 | 0 | { |
2840 | 0 | VirtIODevice *vdev = opaque; |
2841 | |
|
2842 | 0 | return vdev->started; |
2843 | 0 | } |
2844 | | |
2845 | | static bool virtio_disabled_needed(void *opaque) |
2846 | 0 | { |
2847 | 0 | VirtIODevice *vdev = opaque; |
2848 | |
|
2849 | 0 | return vdev->disabled; |
2850 | 0 | } |
2851 | | |
2852 | | static const VMStateDescription vmstate_virtqueue = { |
2853 | | .name = "virtqueue_state", |
2854 | | .version_id = 1, |
2855 | | .minimum_version_id = 1, |
2856 | | .fields = (const VMStateField[]) { |
2857 | | VMSTATE_UINT64(vring.avail, struct VirtQueue), |
2858 | | VMSTATE_UINT64(vring.used, struct VirtQueue), |
2859 | | VMSTATE_END_OF_LIST() |
2860 | | } |
2861 | | }; |
2862 | | |
2863 | | static const VMStateDescription vmstate_packed_virtqueue = { |
2864 | | .name = "packed_virtqueue_state", |
2865 | | .version_id = 1, |
2866 | | .minimum_version_id = 1, |
2867 | | .fields = (const VMStateField[]) { |
2868 | | VMSTATE_UINT16(last_avail_idx, struct VirtQueue), |
2869 | | VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue), |
2870 | | VMSTATE_UINT16(used_idx, struct VirtQueue), |
2871 | | VMSTATE_BOOL(used_wrap_counter, struct VirtQueue), |
2872 | | VMSTATE_UINT32(inuse, struct VirtQueue), |
2873 | | VMSTATE_END_OF_LIST() |
2874 | | } |
2875 | | }; |
2876 | | |
2877 | | static const VMStateDescription vmstate_virtio_virtqueues = { |
2878 | | .name = "virtio/virtqueues", |
2879 | | .version_id = 1, |
2880 | | .minimum_version_id = 1, |
2881 | | .needed = &virtio_virtqueue_needed, |
2882 | | .fields = (const VMStateField[]) { |
2883 | | VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, |
2884 | | VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue), |
2885 | | VMSTATE_END_OF_LIST() |
2886 | | } |
2887 | | }; |
2888 | | |
2889 | | static const VMStateDescription vmstate_virtio_packed_virtqueues = { |
2890 | | .name = "virtio/packed_virtqueues", |
2891 | | .version_id = 1, |
2892 | | .minimum_version_id = 1, |
2893 | | .needed = &virtio_packed_virtqueue_needed, |
2894 | | .fields = (const VMStateField[]) { |
2895 | | VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, |
2896 | | VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue), |
2897 | | VMSTATE_END_OF_LIST() |
2898 | | } |
2899 | | }; |
2900 | | |
2901 | | static const VMStateDescription vmstate_ringsize = { |
2902 | | .name = "ringsize_state", |
2903 | | .version_id = 1, |
2904 | | .minimum_version_id = 1, |
2905 | | .fields = (const VMStateField[]) { |
2906 | | VMSTATE_UNUSED(sizeof(uint32_t)), |
2907 | | VMSTATE_END_OF_LIST() |
2908 | | } |
2909 | | }; |
2910 | | |
2911 | | static const VMStateDescription vmstate_virtio_ringsize = { |
2912 | | .name = "virtio/ringsize", |
2913 | | .version_id = 1, |
2914 | | .minimum_version_id = 1, |
2915 | | .needed = &virtio_ringsize_needed, |
2916 | | .fields = (const VMStateField[]) { |
2917 | | VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, |
2918 | | VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue), |
2919 | | VMSTATE_END_OF_LIST() |
2920 | | } |
2921 | | }; |
2922 | | |
2923 | | static int get_extra_state(QEMUFile *f, void *pv, size_t size, |
2924 | | const VMStateField *field) |
2925 | 0 | { |
2926 | 0 | VirtIODevice *vdev = pv; |
2927 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
2928 | 0 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
2929 | |
|
2930 | 0 | if (!k->load_extra_state) { |
2931 | 0 | return -1; |
2932 | 0 | } else { |
2933 | 0 | return k->load_extra_state(qbus->parent, f); |
2934 | 0 | } |
2935 | 0 | } |
2936 | | |
2937 | | static int put_extra_state(QEMUFile *f, void *pv, size_t size, |
2938 | | const VMStateField *field, JSONWriter *vmdesc) |
2939 | 0 | { |
2940 | 0 | VirtIODevice *vdev = pv; |
2941 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
2942 | 0 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
2943 | |
|
2944 | 0 | k->save_extra_state(qbus->parent, f); |
2945 | 0 | return 0; |
2946 | 0 | } |
2947 | | |
2948 | | static const VMStateInfo vmstate_info_extra_state = { |
2949 | | .name = "virtqueue_extra_state", |
2950 | | .get = get_extra_state, |
2951 | | .put = put_extra_state, |
2952 | | }; |
2953 | | |
2954 | | static const VMStateDescription vmstate_virtio_extra_state = { |
2955 | | .name = "virtio/extra_state", |
2956 | | .version_id = 1, |
2957 | | .minimum_version_id = 1, |
2958 | | .needed = &virtio_extra_state_needed, |
2959 | | .fields = (const VMStateField[]) { |
2960 | | { |
2961 | | .name = "extra_state", |
2962 | | .version_id = 0, |
2963 | | .field_exists = NULL, |
2964 | | .size = 0, |
2965 | | .info = &vmstate_info_extra_state, |
2966 | | .flags = VMS_SINGLE, |
2967 | | .offset = 0, |
2968 | | }, |
2969 | | VMSTATE_END_OF_LIST() |
2970 | | } |
2971 | | }; |
2972 | | |
2973 | | static const VMStateDescription vmstate_virtio_device_endian = { |
2974 | | .name = "virtio/device_endian", |
2975 | | .version_id = 1, |
2976 | | .minimum_version_id = 1, |
2977 | | .needed = &virtio_device_endian_needed, |
2978 | | .fields = (const VMStateField[]) { |
2979 | | VMSTATE_UINT8(device_endian, VirtIODevice), |
2980 | | VMSTATE_END_OF_LIST() |
2981 | | } |
2982 | | }; |
2983 | | |
2984 | | static const VMStateDescription vmstate_virtio_64bit_features = { |
2985 | | .name = "virtio/64bit_features", |
2986 | | .version_id = 1, |
2987 | | .minimum_version_id = 1, |
2988 | | .needed = &virtio_64bit_features_needed, |
2989 | | .fields = (const VMStateField[]) { |
2990 | | VMSTATE_UINT64(guest_features, VirtIODevice), |
2991 | | VMSTATE_END_OF_LIST() |
2992 | | } |
2993 | | }; |
2994 | | |
2995 | | static const VMStateDescription vmstate_virtio_broken = { |
2996 | | .name = "virtio/broken", |
2997 | | .version_id = 1, |
2998 | | .minimum_version_id = 1, |
2999 | | .needed = &virtio_broken_needed, |
3000 | | .fields = (const VMStateField[]) { |
3001 | | VMSTATE_BOOL(broken, VirtIODevice), |
3002 | | VMSTATE_END_OF_LIST() |
3003 | | } |
3004 | | }; |
3005 | | |
3006 | | static const VMStateDescription vmstate_virtio_started = { |
3007 | | .name = "virtio/started", |
3008 | | .version_id = 1, |
3009 | | .minimum_version_id = 1, |
3010 | | .needed = &virtio_started_needed, |
3011 | | .fields = (const VMStateField[]) { |
3012 | | VMSTATE_BOOL(started, VirtIODevice), |
3013 | | VMSTATE_END_OF_LIST() |
3014 | | } |
3015 | | }; |
3016 | | |
3017 | | static const VMStateDescription vmstate_virtio_disabled = { |
3018 | | .name = "virtio/disabled", |
3019 | | .version_id = 1, |
3020 | | .minimum_version_id = 1, |
3021 | | .needed = &virtio_disabled_needed, |
3022 | | .fields = (const VMStateField[]) { |
3023 | | VMSTATE_BOOL(disabled, VirtIODevice), |
3024 | | VMSTATE_END_OF_LIST() |
3025 | | } |
3026 | | }; |
3027 | | |
3028 | | static bool virtio_128bit_features_needed(void *opaque) |
3029 | 0 | { |
3030 | 0 | VirtIODevice *vdev = opaque; |
3031 | |
|
3032 | 0 | return virtio_features_use_ex(vdev->host_features_ex); |
3033 | 0 | } |
3034 | | |
3035 | | static const VMStateDescription vmstate_virtio_128bit_features = { |
3036 | | .name = "virtio/128bit_features", |
3037 | | .version_id = 1, |
3038 | | .minimum_version_id = 1, |
3039 | | .needed = &virtio_128bit_features_needed, |
3040 | | .fields = (const VMStateField[]) { |
3041 | | VMSTATE_UINT64(guest_features_ex[1], VirtIODevice), |
3042 | | VMSTATE_END_OF_LIST() |
3043 | | } |
3044 | | }; |
3045 | | |
3046 | | /* |
3047 | | * Avoid silently breaking migration should the feature space increase |
3048 | | * even more in the (far away) future |
3049 | | */ |
3050 | | QEMU_BUILD_BUG_ON(VIRTIO_FEATURES_NU64S != 2); |
3051 | | |
3052 | | static const VMStateDescription vmstate_virtio = { |
3053 | | .name = "virtio", |
3054 | | .version_id = 1, |
3055 | | .minimum_version_id = 1, |
3056 | | .fields = (const VMStateField[]) { |
3057 | | VMSTATE_END_OF_LIST() |
3058 | | }, |
3059 | | .subsections = (const VMStateDescription * const []) { |
3060 | | &vmstate_virtio_device_endian, |
3061 | | &vmstate_virtio_128bit_features, |
3062 | | &vmstate_virtio_64bit_features, |
3063 | | &vmstate_virtio_virtqueues, |
3064 | | &vmstate_virtio_ringsize, |
3065 | | &vmstate_virtio_broken, |
3066 | | &vmstate_virtio_extra_state, |
3067 | | &vmstate_virtio_started, |
3068 | | &vmstate_virtio_packed_virtqueues, |
3069 | | &vmstate_virtio_disabled, |
3070 | | NULL |
3071 | | } |
3072 | | }; |
3073 | | |
3074 | | int virtio_save(VirtIODevice *vdev, QEMUFile *f) |
3075 | 0 | { |
3076 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
3077 | 0 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
3078 | 0 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); |
3079 | 0 | uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); |
3080 | 0 | int i, ret; |
3081 | 0 | Error *local_err = NULL; |
3082 | |
|
3083 | 0 | if (k->save_config) { |
3084 | 0 | k->save_config(qbus->parent, f); |
3085 | 0 | } |
3086 | |
|
3087 | 0 | qemu_put_8s(f, &vdev->status); |
3088 | 0 | qemu_put_8s(f, &vdev->isr); |
3089 | 0 | qemu_put_be16s(f, &vdev->queue_sel); |
3090 | 0 | qemu_put_be32s(f, &guest_features_lo); |
3091 | 0 | qemu_put_be32(f, vdev->config_len); |
3092 | 0 | qemu_put_buffer(f, vdev->config, vdev->config_len); |
3093 | |
|
3094 | 0 | for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { |
3095 | 0 | if (vdev->vq[i].vring.num == 0) |
3096 | 0 | break; |
3097 | 0 | } |
3098 | |
|
3099 | 0 | qemu_put_be32(f, i); |
3100 | |
|
3101 | 0 | for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { |
3102 | 0 | if (vdev->vq[i].vring.num == 0) |
3103 | 0 | break; |
3104 | | |
3105 | 0 | qemu_put_be32(f, vdev->vq[i].vring.num); |
3106 | 0 | if (k->has_variable_vring_alignment) { |
3107 | 0 | qemu_put_be32(f, vdev->vq[i].vring.align); |
3108 | 0 | } |
3109 | | /* |
3110 | | * Save desc now, the rest of the ring addresses are saved in |
3111 | | * subsections for VIRTIO-1 devices. |
3112 | | */ |
3113 | 0 | qemu_put_be64(f, vdev->vq[i].vring.desc); |
3114 | 0 | qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); |
3115 | 0 | if (k->save_queue) { |
3116 | 0 | k->save_queue(qbus->parent, i, f); |
3117 | 0 | } |
3118 | 0 | } |
3119 | |
|
3120 | 0 | if (vdc->save != NULL) { |
3121 | 0 | vdc->save(vdev, f); |
3122 | 0 | } |
3123 | |
|
3124 | 0 | if (vdc->vmsd) { |
3125 | 0 | ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL, &local_err); |
3126 | 0 | if (ret) { |
3127 | 0 | error_report_err(local_err); |
3128 | 0 | return ret; |
3129 | 0 | } |
3130 | 0 | } |
3131 | | |
3132 | | /* Subsections */ |
3133 | 0 | ret = vmstate_save_state(f, &vmstate_virtio, vdev, NULL, &local_err); |
3134 | 0 | if (ret < 0) { |
3135 | 0 | error_report_err(local_err); |
3136 | 0 | } |
3137 | 0 | return ret; |
3138 | 0 | } |
3139 | | |
3140 | | VirtioSharedMemory *virtio_new_shmem_region(VirtIODevice *vdev, uint8_t shmid, uint64_t size) |
3141 | 0 | { |
3142 | 0 | VirtioSharedMemory *elem; |
3143 | 0 | g_autofree char *name = NULL; |
3144 | |
|
3145 | 0 | elem = g_new0(VirtioSharedMemory, 1); |
3146 | 0 | elem->shmid = shmid; |
3147 | | |
3148 | | /* Initialize embedded MemoryRegion as container for shmem mappings */ |
3149 | 0 | name = g_strdup_printf("virtio-shmem-%d", shmid); |
3150 | 0 | memory_region_init(&elem->mr, OBJECT(vdev), name, size); |
3151 | 0 | QTAILQ_INIT(&elem->mmaps); |
3152 | 0 | QSIMPLEQ_INSERT_TAIL(&vdev->shmem_list, elem, entry); |
3153 | 0 | return elem; |
3154 | 0 | } |
3155 | | |
3156 | | VirtioSharedMemory *virtio_find_shmem_region(VirtIODevice *vdev, uint8_t shmid) |
3157 | 0 | { |
3158 | 0 | VirtioSharedMemory *shmem, *next; |
3159 | 0 | QSIMPLEQ_FOREACH_SAFE(shmem, &vdev->shmem_list, entry, next) { |
3160 | 0 | if (shmem->shmid == shmid) { |
3161 | 0 | return shmem; |
3162 | 0 | } |
3163 | 0 | } |
3164 | 0 | return NULL; |
3165 | 0 | } |
3166 | | |
3167 | | static void virtio_shared_memory_mapping_instance_init(Object *obj) |
3168 | 0 | { |
3169 | 0 | VirtioSharedMemoryMapping *mapping = VIRTIO_SHARED_MEMORY_MAPPING(obj); |
3170 | |
|
3171 | 0 | mapping->shmid = 0; |
3172 | 0 | mapping->offset = 0; |
3173 | 0 | mapping->len = 0; |
3174 | 0 | mapping->mr = NULL; |
3175 | 0 | } |
3176 | | |
3177 | | static void virtio_shared_memory_mapping_instance_finalize(Object *obj) |
3178 | 0 | { |
3179 | 0 | VirtioSharedMemoryMapping *mapping = VIRTIO_SHARED_MEMORY_MAPPING(obj); |
3180 | | |
3181 | | /* Clean up MemoryRegion if it exists */ |
3182 | 0 | if (mapping->mr) { |
3183 | | /* Unparent the MemoryRegion to trigger cleanup */ |
3184 | 0 | object_unparent(OBJECT(mapping->mr)); |
3185 | 0 | mapping->mr = NULL; |
3186 | 0 | } |
3187 | 0 | } |
3188 | | |
3189 | | VirtioSharedMemoryMapping *virtio_shared_memory_mapping_new(uint8_t shmid, |
3190 | | int fd, |
3191 | | uint64_t fd_offset, |
3192 | | uint64_t shm_offset, |
3193 | | uint64_t len, |
3194 | | bool allow_write) |
3195 | 0 | { |
3196 | 0 | VirtioSharedMemoryMapping *mapping; |
3197 | 0 | MemoryRegion *mr; |
3198 | 0 | g_autoptr(GString) mr_name = g_string_new(NULL); |
3199 | 0 | uint32_t ram_flags; |
3200 | 0 | Error *local_err = NULL; |
3201 | |
|
3202 | 0 | if (len == 0) { |
3203 | 0 | error_report("Shared memory mapping size cannot be zero"); |
3204 | 0 | return NULL; |
3205 | 0 | } |
3206 | | |
3207 | 0 | fd = dup(fd); |
3208 | 0 | if (fd < 0) { |
3209 | 0 | error_report("Failed to duplicate fd: %s", strerror(errno)); |
3210 | 0 | return NULL; |
3211 | 0 | } |
3212 | | |
3213 | | /* Determine RAM flags */ |
3214 | 0 | ram_flags = RAM_SHARED; |
3215 | 0 | if (!allow_write) { |
3216 | 0 | ram_flags |= RAM_READONLY_FD; |
3217 | 0 | } |
3218 | | |
3219 | | /* Create the VirtioSharedMemoryMapping */ |
3220 | 0 | mapping = VIRTIO_SHARED_MEMORY_MAPPING( |
3221 | 0 | object_new(TYPE_VIRTIO_SHARED_MEMORY_MAPPING)); |
3222 | | |
3223 | | /* Set up object properties */ |
3224 | 0 | mapping->shmid = shmid; |
3225 | 0 | mapping->offset = shm_offset; |
3226 | 0 | mapping->len = len; |
3227 | | |
3228 | | /* Create MemoryRegion as a child of this object */ |
3229 | 0 | mr = g_new0(MemoryRegion, 1); |
3230 | 0 | g_string_printf(mr_name, "virtio-shmem-%d-%" PRIx64, shmid, shm_offset); |
3231 | | |
3232 | | /* Initialize MemoryRegion with file descriptor */ |
3233 | 0 | if (!memory_region_init_ram_from_fd(mr, OBJECT(mapping), mr_name->str, |
3234 | 0 | len, ram_flags, fd, fd_offset, |
3235 | 0 | &local_err)) { |
3236 | 0 | error_report_err(local_err); |
3237 | 0 | g_free(mr); |
3238 | 0 | close(fd); |
3239 | 0 | object_unref(OBJECT(mapping)); |
3240 | 0 | return NULL; |
3241 | 0 | } |
3242 | | |
3243 | 0 | mapping->mr = mr; |
3244 | 0 | return mapping; |
3245 | 0 | } |
3246 | | |
3247 | | int virtio_add_shmem_map(VirtioSharedMemory *shmem, |
3248 | | VirtioSharedMemoryMapping *mapping) |
3249 | 0 | { |
3250 | 0 | if (!mapping) { |
3251 | 0 | error_report("VirtioSharedMemoryMapping cannot be NULL"); |
3252 | 0 | return -1; |
3253 | 0 | } |
3254 | 0 | if (!mapping->mr) { |
3255 | 0 | error_report("VirtioSharedMemoryMapping has no MemoryRegion"); |
3256 | 0 | return -1; |
3257 | 0 | } |
3258 | | |
3259 | | /* Validate boundaries against the VIRTIO shared memory region */ |
3260 | 0 | if (mapping->offset + mapping->len > memory_region_size(&shmem->mr)) { |
3261 | 0 | error_report("Memory exceeds the shared memory boundaries"); |
3262 | 0 | return -1; |
3263 | 0 | } |
3264 | | |
3265 | | /* Add as subregion to the VIRTIO shared memory */ |
3266 | 0 | memory_region_add_subregion(&shmem->mr, mapping->offset, mapping->mr); |
3267 | | |
3268 | | /* Add to the mapped regions list */ |
3269 | 0 | QTAILQ_INSERT_TAIL(&shmem->mmaps, mapping, link); |
3270 | |
|
3271 | 0 | return 0; |
3272 | 0 | } |
3273 | | |
3274 | | VirtioSharedMemoryMapping *virtio_find_shmem_map(VirtioSharedMemory *shmem, |
3275 | | hwaddr offset, uint64_t size) |
3276 | 0 | { |
3277 | 0 | VirtioSharedMemoryMapping *mapping; |
3278 | 0 | QTAILQ_FOREACH(mapping, &shmem->mmaps, link) { |
3279 | 0 | if (mapping->offset == offset && mapping->len == size) { |
3280 | 0 | return mapping; |
3281 | 0 | } |
3282 | 0 | } |
3283 | 0 | return NULL; |
3284 | 0 | } |
3285 | | |
3286 | | void virtio_del_shmem_map(VirtioSharedMemory *shmem, hwaddr offset, |
3287 | | uint64_t size) |
3288 | 0 | { |
3289 | 0 | VirtioSharedMemoryMapping *mapping = virtio_find_shmem_map(shmem, offset, size); |
3290 | 0 | if (mapping == NULL) { |
3291 | 0 | return; |
3292 | 0 | } |
3293 | | |
3294 | | /* |
3295 | | * Remove from memory region first |
3296 | | */ |
3297 | 0 | memory_region_del_subregion(&shmem->mr, mapping->mr); |
3298 | | |
3299 | | /* |
3300 | | * Remove from list and unref the mapping which will trigger automatic cleanup |
3301 | | * when the reference count reaches zero. |
3302 | | */ |
3303 | 0 | QTAILQ_REMOVE(&shmem->mmaps, mapping, link); |
3304 | 0 | object_unref(OBJECT(mapping)); |
3305 | 0 | } |
3306 | | |
3307 | | /* A wrapper for use as a VMState .put function */ |
3308 | | static int virtio_device_put(QEMUFile *f, void *opaque, size_t size, |
3309 | | const VMStateField *field, JSONWriter *vmdesc) |
3310 | 0 | { |
3311 | 0 | return virtio_save(VIRTIO_DEVICE(opaque), f); |
3312 | 0 | } |
3313 | | |
3314 | | /* A wrapper for use as a VMState .get function */ |
3315 | | static int coroutine_mixed_fn |
3316 | | virtio_device_get(QEMUFile *f, void *opaque, size_t size, |
3317 | | const VMStateField *field) |
3318 | 0 | { |
3319 | 0 | VirtIODevice *vdev = VIRTIO_DEVICE(opaque); |
3320 | 0 | DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev)); |
3321 | |
|
3322 | 0 | return virtio_load(vdev, f, dc->vmsd->version_id); |
3323 | 0 | } |
3324 | | |
3325 | | const VMStateInfo virtio_vmstate_info = { |
3326 | | .name = "virtio", |
3327 | | .get = virtio_device_get, |
3328 | | .put = virtio_device_put, |
3329 | | }; |
3330 | | |
3331 | | static int virtio_set_features_nocheck(VirtIODevice *vdev, const uint64_t *val) |
3332 | 775k | { |
3333 | 775k | VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); |
3334 | 775k | uint64_t tmp[VIRTIO_FEATURES_NU64S]; |
3335 | 775k | bool bad; |
3336 | | |
3337 | 775k | bad = virtio_features_andnot(tmp, val, vdev->host_features_ex); |
3338 | 775k | virtio_features_and(tmp, val, vdev->host_features_ex); |
3339 | | |
3340 | 775k | if (k->set_features_ex) { |
3341 | 206k | k->set_features_ex(vdev, tmp); |
3342 | 568k | } else if (k->set_features) { |
3343 | 83.7k | bad = bad || virtio_features_use_ex(tmp); |
3344 | 83.7k | k->set_features(vdev, tmp[0]); |
3345 | 83.7k | } |
3346 | | |
3347 | 775k | virtio_features_copy(vdev->guest_features_ex, tmp); |
3348 | 775k | return bad ? -1 : 0; |
3349 | 775k | } |
3350 | | |
3351 | | typedef struct VirtioSetFeaturesNocheckData { |
3352 | | Coroutine *co; |
3353 | | VirtIODevice *vdev; |
3354 | | uint64_t val[VIRTIO_FEATURES_NU64S]; |
3355 | | int ret; |
3356 | | } VirtioSetFeaturesNocheckData; |
3357 | | |
3358 | | static void virtio_set_features_nocheck_bh(void *opaque) |
3359 | 0 | { |
3360 | 0 | VirtioSetFeaturesNocheckData *data = opaque; |
3361 | |
|
3362 | 0 | data->ret = virtio_set_features_nocheck(data->vdev, data->val); |
3363 | 0 | aio_co_wake(data->co); |
3364 | 0 | } |
3365 | | |
3366 | | static int coroutine_mixed_fn |
3367 | | virtio_set_features_nocheck_maybe_co(VirtIODevice *vdev, |
3368 | | const uint64_t *val) |
3369 | 0 | { |
3370 | 0 | if (qemu_in_coroutine()) { |
3371 | 0 | VirtioSetFeaturesNocheckData data = { |
3372 | 0 | .co = qemu_coroutine_self(), |
3373 | 0 | .vdev = vdev, |
3374 | 0 | }; |
3375 | 0 | virtio_features_copy(data.val, val); |
3376 | 0 | aio_bh_schedule_oneshot(qemu_get_current_aio_context(), |
3377 | 0 | virtio_set_features_nocheck_bh, &data); |
3378 | 0 | qemu_coroutine_yield(); |
3379 | 0 | return data.ret; |
3380 | 0 | } else { |
3381 | 0 | return virtio_set_features_nocheck(vdev, val); |
3382 | 0 | } |
3383 | 0 | } |
3384 | | |
3385 | | int virtio_set_features(VirtIODevice *vdev, uint64_t val) |
3386 | 326k | { |
3387 | 326k | uint64_t features[VIRTIO_FEATURES_NU64S]; |
3388 | | |
3389 | 326k | virtio_features_from_u64(features, val); |
3390 | 326k | return virtio_set_features_ex(vdev, features); |
3391 | 326k | } |
3392 | | |
3393 | | int virtio_set_features_ex(VirtIODevice *vdev, const uint64_t *features) |
3394 | 439k | { |
3395 | 439k | int ret; |
3396 | | /* |
3397 | | * The driver must not attempt to set features after feature negotiation |
3398 | | * has finished. |
3399 | | */ |
3400 | 439k | if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { |
3401 | 34.8k | return -EINVAL; |
3402 | 34.8k | } |
3403 | | |
3404 | 405k | if (features[0] & (1ull << VIRTIO_F_BAD_FEATURE)) { |
3405 | 36.3k | qemu_log_mask(LOG_GUEST_ERROR, |
3406 | 36.3k | "%s: guest driver for %s has enabled UNUSED(30) feature bit!\n", |
3407 | 36.3k | __func__, vdev->name); |
3408 | 36.3k | } |
3409 | | |
3410 | 405k | ret = virtio_set_features_nocheck(vdev, features); |
3411 | 405k | if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { |
3412 | | /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */ |
3413 | 335k | int i; |
3414 | 343M | for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { |
3415 | 343M | if (vdev->vq[i].vring.num != 0) { |
3416 | 1.29M | virtio_init_region_cache(vdev, i); |
3417 | 1.29M | } |
3418 | 343M | } |
3419 | 335k | } |
3420 | 405k | if (!ret) { |
3421 | 30.3k | if (!virtio_device_started(vdev, vdev->status) && |
3422 | 22.6k | !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { |
3423 | 16.2k | vdev->start_on_kick = true; |
3424 | 16.2k | } |
3425 | 30.3k | } |
3426 | 405k | return ret; |
3427 | 439k | } |
3428 | | |
3429 | | void virtio_reset(VirtIODevice *vdev) |
3430 | 370k | { |
3431 | 370k | VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); |
3432 | 370k | VirtioSharedMemory *shmem; |
3433 | 370k | uint64_t features[VIRTIO_FEATURES_NU64S]; |
3434 | 370k | int i; |
3435 | | |
3436 | 370k | virtio_set_status(vdev, 0); |
3437 | 370k | if (current_cpu) { |
3438 | | /* Guest initiated reset */ |
3439 | 0 | vdev->device_endian = virtio_current_cpu_endian(); |
3440 | 370k | } else { |
3441 | | /* System reset */ |
3442 | 370k | vdev->device_endian = virtio_default_endian(); |
3443 | 370k | } |
3444 | | |
3445 | 370k | if (k->get_vhost) { |
3446 | 59.9k | struct vhost_dev *hdev = k->get_vhost(vdev); |
3447 | | /* Only reset when vhost back-end is connected */ |
3448 | 59.9k | if (hdev && hdev->vhost_ops) { |
3449 | 0 | vhost_reset_device(hdev); |
3450 | 0 | } |
3451 | 59.9k | } |
3452 | | |
3453 | 370k | if (k->reset) { |
3454 | 370k | k->reset(vdev); |
3455 | 370k | } |
3456 | | |
3457 | 370k | vdev->start_on_kick = false; |
3458 | 370k | vdev->started = false; |
3459 | 370k | vdev->broken = false; |
3460 | 370k | virtio_features_clear(features); |
3461 | 370k | virtio_set_features_nocheck(vdev, features); |
3462 | 370k | vdev->queue_sel = 0; |
3463 | 370k | vdev->status = 0; |
3464 | 370k | vdev->disabled = false; |
3465 | 370k | qatomic_set(&vdev->isr, 0); |
3466 | 370k | vdev->config_vector = VIRTIO_NO_VECTOR; |
3467 | 370k | virtio_notify_vector(vdev, vdev->config_vector); |
3468 | | |
3469 | 379M | for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { |
3470 | 378M | __virtio_queue_reset(vdev, i); |
3471 | 378M | } |
3472 | | |
3473 | | /* Mappings are removed to prevent stale fds from remaining open. */ |
3474 | 370k | QSIMPLEQ_FOREACH(shmem, &vdev->shmem_list, entry) { |
3475 | 0 | while (!QTAILQ_EMPTY(&shmem->mmaps)) { |
3476 | 0 | VirtioSharedMemoryMapping *mapping = QTAILQ_FIRST(&shmem->mmaps); |
3477 | 0 | virtio_del_shmem_map(shmem, mapping->offset, |
3478 | 0 | memory_region_size(mapping->mr)); |
3479 | 0 | } |
3480 | 0 | } |
3481 | 370k | } |
3482 | | |
3483 | | static void virtio_device_check_notification_compatibility(VirtIODevice *vdev, |
3484 | | Error **errp) |
3485 | 21 | { |
3486 | 21 | VirtioBusState *bus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); |
3487 | 21 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(bus); |
3488 | 21 | DeviceState *proxy = DEVICE(BUS(bus)->parent); |
3489 | | |
3490 | 21 | if (virtio_host_has_feature(vdev, VIRTIO_F_NOTIFICATION_DATA) && |
3491 | 0 | k->ioeventfd_enabled(proxy)) { |
3492 | 0 | error_setg(errp, |
3493 | 0 | "notification_data=on without ioeventfd=off is not supported"); |
3494 | 0 | } |
3495 | 21 | } |
3496 | | |
3497 | | size_t virtio_get_config_size(const VirtIOConfigSizeParams *params, |
3498 | | uint64_t host_features) |
3499 | 4 | { |
3500 | 4 | size_t config_size = params->min_size; |
3501 | 4 | const VirtIOFeature *feature_sizes = params->feature_sizes; |
3502 | 4 | size_t i; |
3503 | | |
3504 | 22 | for (i = 0; feature_sizes[i].flags != 0; i++) { |
3505 | 18 | if (host_features & feature_sizes[i].flags) { |
3506 | 8 | config_size = MAX(feature_sizes[i].end, config_size); |
3507 | 8 | } |
3508 | 18 | } |
3509 | | |
3510 | 4 | assert(config_size <= params->max_size); |
3511 | 4 | return config_size; |
3512 | 4 | } |
3513 | | |
3514 | | int coroutine_mixed_fn |
3515 | | virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id) |
3516 | 0 | { |
3517 | 0 | int i, ret; |
3518 | 0 | uint32_t config_len; |
3519 | 0 | uint32_t num; |
3520 | 0 | uint32_t features; |
3521 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
3522 | 0 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
3523 | 0 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); |
3524 | 0 | Error *local_err = NULL; |
3525 | | |
3526 | | /* |
3527 | | * We poison the endianness to ensure it does not get used before |
3528 | | * subsections have been loaded. |
3529 | | */ |
3530 | 0 | vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; |
3531 | |
|
3532 | 0 | if (k->load_config) { |
3533 | 0 | ret = k->load_config(qbus->parent, f); |
3534 | 0 | if (ret) |
3535 | 0 | return ret; |
3536 | 0 | } |
3537 | | |
3538 | 0 | qemu_get_8s(f, &vdev->status); |
3539 | 0 | qemu_get_8s(f, &vdev->isr); |
3540 | 0 | qemu_get_be16s(f, &vdev->queue_sel); |
3541 | 0 | if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { |
3542 | 0 | return -1; |
3543 | 0 | } |
3544 | 0 | qemu_get_be32s(f, &features); |
3545 | | |
3546 | | /* |
3547 | | * Temporarily set guest_features low bits - needed by |
3548 | | * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS |
3549 | | * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ. |
3550 | | * |
3551 | | * Note: devices should always test host features in future - don't create |
3552 | | * new dependencies like this. |
3553 | | */ |
3554 | 0 | virtio_features_from_u64(vdev->guest_features_ex, features); |
3555 | |
|
3556 | 0 | config_len = qemu_get_be32(f); |
3557 | | |
3558 | | /* |
3559 | | * There are cases where the incoming config can be bigger or smaller |
3560 | | * than what we have; so load what we have space for, and skip |
3561 | | * any excess that's in the stream. |
3562 | | */ |
3563 | 0 | qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); |
3564 | |
|
3565 | 0 | while (config_len > vdev->config_len) { |
3566 | 0 | if (qemu_file_get_error(f)) { |
3567 | 0 | return -1; |
3568 | 0 | } |
3569 | 0 | qemu_get_byte(f); |
3570 | 0 | if (qemu_file_get_error(f)) { |
3571 | 0 | return -1; |
3572 | 0 | } |
3573 | 0 | config_len--; |
3574 | 0 | } |
3575 | | |
3576 | 0 | num = qemu_get_be32(f); |
3577 | |
|
3578 | 0 | if (num > VIRTIO_QUEUE_MAX) { |
3579 | 0 | error_report("Invalid number of virtqueues: 0x%x", num); |
3580 | 0 | return -1; |
3581 | 0 | } |
3582 | | |
3583 | 0 | if (vdc->pre_load_queues) { |
3584 | 0 | ret = vdc->pre_load_queues(vdev, num); |
3585 | 0 | if (ret) { |
3586 | 0 | return ret; |
3587 | 0 | } |
3588 | 0 | } |
3589 | | |
3590 | 0 | for (i = 0; i < num; i++) { |
3591 | 0 | vdev->vq[i].vring.num = qemu_get_be32(f); |
3592 | 0 | if (vdev->vq[i].vring.num > vdev->vq[i].vring.num_default) { |
3593 | 0 | error_report("VQ %d vring.num %u exceeds allocated max %u", |
3594 | 0 | i, vdev->vq[i].vring.num, |
3595 | 0 | vdev->vq[i].vring.num_default); |
3596 | 0 | return -1; |
3597 | 0 | } |
3598 | 0 | if (k->has_variable_vring_alignment) { |
3599 | 0 | vdev->vq[i].vring.align = qemu_get_be32(f); |
3600 | 0 | } |
3601 | 0 | vdev->vq[i].vring.desc = qemu_get_be64(f); |
3602 | 0 | qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); |
3603 | 0 | vdev->vq[i].signalled_used_valid = false; |
3604 | 0 | vdev->vq[i].notification = true; |
3605 | |
|
3606 | 0 | if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) { |
3607 | 0 | error_report("VQ %d address 0x0 " |
3608 | 0 | "inconsistent with Host index 0x%x", |
3609 | 0 | i, vdev->vq[i].last_avail_idx); |
3610 | 0 | return -1; |
3611 | 0 | } |
3612 | 0 | if (k->load_queue) { |
3613 | 0 | ret = k->load_queue(qbus->parent, i, f); |
3614 | 0 | if (ret) |
3615 | 0 | return ret; |
3616 | 0 | } |
3617 | 0 | } |
3618 | | |
3619 | 0 | virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); |
3620 | |
|
3621 | 0 | if (vdc->load != NULL) { |
3622 | 0 | ret = vdc->load(vdev, f, version_id); |
3623 | 0 | if (ret) { |
3624 | 0 | return ret; |
3625 | 0 | } |
3626 | 0 | } |
3627 | | |
3628 | 0 | if (vdc->vmsd) { |
3629 | 0 | ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id, &local_err); |
3630 | 0 | if (ret) { |
3631 | 0 | error_report_err(local_err); |
3632 | 0 | return ret; |
3633 | 0 | } |
3634 | 0 | } |
3635 | | |
3636 | | /* Subsections */ |
3637 | 0 | ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1, &local_err); |
3638 | 0 | if (ret) { |
3639 | 0 | error_report_err(local_err); |
3640 | 0 | return ret; |
3641 | 0 | } |
3642 | | |
3643 | 0 | if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { |
3644 | 0 | vdev->device_endian = virtio_default_endian(); |
3645 | 0 | } |
3646 | | |
3647 | | /* |
3648 | | * guest_features_ex is fully initialized with u32 features and upper |
3649 | | * bits have been filled as needed by the later load. |
3650 | | */ |
3651 | 0 | if (virtio_set_features_nocheck_maybe_co(vdev, |
3652 | 0 | vdev->guest_features_ex) < 0) { |
3653 | 0 | error_report("Features 0x" VIRTIO_FEATURES_FMT " unsupported. " |
3654 | 0 | "Allowed features: 0x" VIRTIO_FEATURES_FMT, |
3655 | 0 | VIRTIO_FEATURES_PR(vdev->guest_features_ex), |
3656 | 0 | VIRTIO_FEATURES_PR(vdev->host_features_ex)); |
3657 | 0 | return -1; |
3658 | 0 | } |
3659 | | |
3660 | 0 | if (!virtio_device_started(vdev, vdev->status) && |
3661 | 0 | !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { |
3662 | 0 | vdev->start_on_kick = true; |
3663 | 0 | } |
3664 | |
|
3665 | 0 | RCU_READ_LOCK_GUARD(); |
3666 | 0 | for (i = 0; i < num; i++) { |
3667 | 0 | if (vdev->vq[i].vring.desc) { |
3668 | 0 | uint16_t nheads; |
3669 | | |
3670 | | /* |
3671 | | * VIRTIO-1 devices migrate desc, used, and avail ring addresses so |
3672 | | * only the region cache needs to be set up. Legacy devices need |
3673 | | * to calculate used and avail ring addresses based on the desc |
3674 | | * address. |
3675 | | */ |
3676 | 0 | if (virtio_vdev_is_legacy(vdev)) { |
3677 | 0 | virtio_queue_update_rings(vdev, i); |
3678 | 0 | } else { |
3679 | 0 | virtio_init_region_cache(vdev, i); |
3680 | 0 | } |
3681 | |
|
3682 | 0 | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
3683 | 0 | vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx; |
3684 | 0 | vdev->vq[i].shadow_avail_wrap_counter = |
3685 | 0 | vdev->vq[i].last_avail_wrap_counter; |
3686 | 0 | continue; |
3687 | 0 | } |
3688 | | |
3689 | 0 | nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; |
3690 | | /* Check it isn't doing strange things with descriptor numbers. */ |
3691 | 0 | if (nheads > vdev->vq[i].vring.num) { |
3692 | 0 | virtio_error(vdev, "VQ %d size 0x%x Guest index 0x%x " |
3693 | 0 | "inconsistent with Host index 0x%x: delta 0x%x", |
3694 | 0 | i, vdev->vq[i].vring.num, |
3695 | 0 | vring_avail_idx(&vdev->vq[i]), |
3696 | 0 | vdev->vq[i].last_avail_idx, nheads); |
3697 | 0 | vdev->vq[i].used_idx = 0; |
3698 | 0 | vdev->vq[i].shadow_avail_idx = 0; |
3699 | 0 | vdev->vq[i].inuse = 0; |
3700 | 0 | continue; |
3701 | 0 | } |
3702 | 0 | vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]); |
3703 | 0 | vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]); |
3704 | | |
3705 | | /* |
3706 | | * Some devices migrate VirtQueueElements that have been popped |
3707 | | * from the avail ring but not yet returned to the used ring. |
3708 | | * Since max ring size < UINT16_MAX it's safe to use modulo |
3709 | | * UINT16_MAX + 1 subtraction. |
3710 | | */ |
3711 | 0 | vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx - |
3712 | 0 | vdev->vq[i].used_idx); |
3713 | 0 | if (vdev->vq[i].inuse > vdev->vq[i].vring.num) { |
3714 | 0 | error_report("VQ %d size 0x%x < last_avail_idx 0x%x - " |
3715 | 0 | "used_idx 0x%x", |
3716 | 0 | i, vdev->vq[i].vring.num, |
3717 | 0 | vdev->vq[i].last_avail_idx, |
3718 | 0 | vdev->vq[i].used_idx); |
3719 | 0 | return -1; |
3720 | 0 | } |
3721 | 0 | } |
3722 | 0 | } |
3723 | | |
3724 | 0 | if (vdc->post_load) { |
3725 | 0 | ret = vdc->post_load(vdev); |
3726 | 0 | if (ret) { |
3727 | 0 | return ret; |
3728 | 0 | } |
3729 | 0 | } |
3730 | | |
3731 | 0 | return 0; |
3732 | 0 | } |
3733 | | |
3734 | | void virtio_cleanup(VirtIODevice *vdev) |
3735 | 0 | { |
3736 | 0 | qemu_del_vm_change_state_handler(vdev->vmstate); |
3737 | 0 | } |
3738 | | |
3739 | | static int virtio_vmstate_change(void *opaque, bool running, RunState state) |
3740 | 21 | { |
3741 | 21 | VirtIODevice *vdev = opaque; |
3742 | 21 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
3743 | 21 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
3744 | 21 | bool backend_run = running && virtio_device_started(vdev, vdev->status); |
3745 | 21 | vdev->vm_running = running; |
3746 | | |
3747 | 21 | if (backend_run) { |
3748 | 0 | virtio_set_status(vdev, vdev->status); |
3749 | 0 | } |
3750 | | |
3751 | 21 | if (k->vmstate_change) { |
3752 | 21 | k->vmstate_change(qbus->parent, backend_run); |
3753 | 21 | } |
3754 | | |
3755 | 21 | if (!backend_run) { |
3756 | 21 | int ret = virtio_set_status(vdev, vdev->status); |
3757 | 21 | if (ret) { |
3758 | 0 | return ret; |
3759 | 0 | } |
3760 | 21 | } |
3761 | 21 | return 0; |
3762 | 21 | } |
3763 | | |
3764 | | void virtio_instance_init_common(Object *proxy_obj, void *data, |
3765 | | size_t vdev_size, const char *vdev_name) |
3766 | 210 | { |
3767 | 210 | DeviceState *vdev = data; |
3768 | | |
3769 | 210 | object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev, |
3770 | 210 | vdev_size, vdev_name, &error_abort, |
3771 | 210 | NULL); |
3772 | 210 | qdev_alias_all_properties(vdev, proxy_obj); |
3773 | 210 | } |
3774 | | |
3775 | | void virtio_init(VirtIODevice *vdev, uint16_t device_id, size_t config_size) |
3776 | 21 | { |
3777 | 21 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
3778 | 21 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
3779 | 21 | int i; |
3780 | 21 | int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; |
3781 | | |
3782 | 21 | if (nvectors) { |
3783 | 21 | vdev->vector_queues = |
3784 | 21 | g_malloc0(sizeof(*vdev->vector_queues) * nvectors); |
3785 | 21 | } |
3786 | | |
3787 | 21 | vdev->start_on_kick = false; |
3788 | 21 | vdev->started = false; |
3789 | 21 | vdev->vhost_started = false; |
3790 | 21 | vdev->device_id = device_id; |
3791 | 21 | vdev->status = 0; |
3792 | 21 | qatomic_set(&vdev->isr, 0); |
3793 | 21 | vdev->queue_sel = 0; |
3794 | 21 | vdev->config_vector = VIRTIO_NO_VECTOR; |
3795 | 21 | vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX); |
3796 | 21 | vdev->vm_running = runstate_is_running(); |
3797 | 21 | vdev->broken = false; |
3798 | 21.5k | for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { |
3799 | 21.5k | vdev->vq[i].vector = VIRTIO_NO_VECTOR; |
3800 | 21.5k | vdev->vq[i].vdev = vdev; |
3801 | 21.5k | vdev->vq[i].queue_index = i; |
3802 | 21.5k | vdev->vq[i].host_notifier_enabled = false; |
3803 | 21.5k | } |
3804 | | |
3805 | 21 | vdev->name = virtio_id_to_name(device_id); |
3806 | 21 | vdev->config_len = config_size; |
3807 | 21 | if (vdev->config_len) { |
3808 | 20 | vdev->config = g_malloc0(config_size); |
3809 | 20 | } else { |
3810 | 1 | vdev->config = NULL; |
3811 | 1 | } |
3812 | 21 | vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev), |
3813 | 21 | NULL, virtio_vmstate_change, vdev); |
3814 | 21 | vdev->device_endian = virtio_default_endian(); |
3815 | 21 | vdev->use_guest_notifier_mask = true; |
3816 | 21 | QSIMPLEQ_INIT(&vdev->shmem_list); |
3817 | 21 | } |
3818 | | |
3819 | | /* |
3820 | | * Only devices that have already been around prior to defining the virtio |
3821 | | * standard support legacy mode; this includes devices not specified in the |
3822 | | * standard. All newer devices conform to the virtio standard only. |
3823 | | */ |
3824 | | bool virtio_legacy_allowed(VirtIODevice *vdev) |
3825 | 15 | { |
3826 | 15 | switch (vdev->device_id) { |
3827 | 2 | case VIRTIO_ID_NET: |
3828 | 4 | case VIRTIO_ID_BLOCK: |
3829 | 6 | case VIRTIO_ID_CONSOLE: |
3830 | 7 | case VIRTIO_ID_RNG: |
3831 | 9 | case VIRTIO_ID_BALLOON: |
3832 | 9 | case VIRTIO_ID_RPMSG: |
3833 | 11 | case VIRTIO_ID_SCSI: |
3834 | 15 | case VIRTIO_ID_9P: |
3835 | 15 | case VIRTIO_ID_RPROC_SERIAL: |
3836 | 15 | case VIRTIO_ID_CAIF: |
3837 | 15 | return true; |
3838 | 0 | default: |
3839 | 0 | return false; |
3840 | 15 | } |
3841 | 15 | } |
3842 | | |
3843 | | bool virtio_legacy_check_disabled(VirtIODevice *vdev) |
3844 | 0 | { |
3845 | 0 | return vdev->disable_legacy_check; |
3846 | 0 | } |
3847 | | |
3848 | | hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) |
3849 | 0 | { |
3850 | 0 | return vdev->vq[n].vring.desc; |
3851 | 0 | } |
3852 | | |
3853 | | bool virtio_queue_enabled_legacy(VirtIODevice *vdev, int n) |
3854 | 0 | { |
3855 | 0 | return virtio_queue_get_desc_addr(vdev, n) != 0; |
3856 | 0 | } |
3857 | | |
3858 | | bool virtio_queue_enabled(VirtIODevice *vdev, int n) |
3859 | 0 | { |
3860 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
3861 | 0 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
3862 | |
|
3863 | 0 | if (k->queue_enabled) { |
3864 | 0 | return k->queue_enabled(qbus->parent, n); |
3865 | 0 | } |
3866 | 0 | return virtio_queue_enabled_legacy(vdev, n); |
3867 | 0 | } |
3868 | | |
3869 | | hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) |
3870 | 0 | { |
3871 | 0 | return vdev->vq[n].vring.avail; |
3872 | 0 | } |
3873 | | |
3874 | | hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n) |
3875 | 0 | { |
3876 | 0 | return vdev->vq[n].vring.used; |
3877 | 0 | } |
3878 | | |
3879 | | hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) |
3880 | 1.44M | { |
3881 | 1.44M | return sizeof(VRingDesc) * vdev->vq[n].vring.num; |
3882 | 1.44M | } |
3883 | | |
3884 | | hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n) |
3885 | 1.36M | { |
3886 | 1.36M | int s; |
3887 | | |
3888 | 1.36M | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
3889 | 0 | return sizeof(struct VRingPackedDescEvent); |
3890 | 0 | } |
3891 | | |
3892 | 1.36M | s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; |
3893 | 1.36M | return offsetof(VRingAvail, ring) + |
3894 | 1.36M | sizeof(uint16_t) * vdev->vq[n].vring.num + s; |
3895 | 1.36M | } |
3896 | | |
3897 | | hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n) |
3898 | 1.37M | { |
3899 | 1.37M | int s; |
3900 | | |
3901 | 1.37M | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
3902 | 0 | return sizeof(struct VRingPackedDescEvent); |
3903 | 0 | } |
3904 | | |
3905 | 1.37M | s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; |
3906 | 1.37M | return offsetof(VRingUsed, ring) + |
3907 | 1.37M | sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s; |
3908 | 1.37M | } |
3909 | | |
3910 | | static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice *vdev, |
3911 | | int n) |
3912 | 0 | { |
3913 | 0 | unsigned int avail, used; |
3914 | |
|
3915 | 0 | avail = vdev->vq[n].last_avail_idx; |
3916 | 0 | avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15; |
3917 | |
|
3918 | 0 | used = vdev->vq[n].used_idx; |
3919 | 0 | used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15; |
3920 | |
|
3921 | 0 | return avail | used << 16; |
3922 | 0 | } |
3923 | | |
3924 | | static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev, |
3925 | | int n) |
3926 | 0 | { |
3927 | 0 | return vdev->vq[n].last_avail_idx; |
3928 | 0 | } |
3929 | | |
3930 | | unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) |
3931 | 0 | { |
3932 | 0 | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
3933 | 0 | return virtio_queue_packed_get_last_avail_idx(vdev, n); |
3934 | 0 | } else { |
3935 | 0 | return virtio_queue_split_get_last_avail_idx(vdev, n); |
3936 | 0 | } |
3937 | 0 | } |
3938 | | |
3939 | | static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev, |
3940 | | int n, unsigned int idx) |
3941 | 0 | { |
3942 | 0 | struct VirtQueue *vq = &vdev->vq[n]; |
3943 | |
|
3944 | 0 | vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff; |
3945 | 0 | vq->last_avail_wrap_counter = |
3946 | 0 | vq->shadow_avail_wrap_counter = !!(idx & 0x8000); |
3947 | 0 | idx >>= 16; |
3948 | 0 | vq->used_idx = idx & 0x7fff; |
3949 | 0 | vq->used_wrap_counter = !!(idx & 0x8000); |
3950 | 0 | } |
3951 | | |
3952 | | static void virtio_queue_split_set_last_avail_idx(VirtIODevice *vdev, |
3953 | | int n, unsigned int idx) |
3954 | 0 | { |
3955 | 0 | vdev->vq[n].last_avail_idx = idx; |
3956 | 0 | vdev->vq[n].shadow_avail_idx = idx; |
3957 | 0 | } |
3958 | | |
3959 | | void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, |
3960 | | unsigned int idx) |
3961 | 0 | { |
3962 | 0 | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
3963 | 0 | virtio_queue_packed_set_last_avail_idx(vdev, n, idx); |
3964 | 0 | } else { |
3965 | 0 | virtio_queue_split_set_last_avail_idx(vdev, n, idx); |
3966 | 0 | } |
3967 | 0 | } |
3968 | | |
3969 | | static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice *vdev, |
3970 | | int n) |
3971 | 0 | { |
3972 | | /* We don't have a reference like avail idx in shared memory */ |
3973 | 0 | } |
3974 | | |
3975 | | static void virtio_queue_split_restore_last_avail_idx(VirtIODevice *vdev, |
3976 | | int n) |
3977 | 0 | { |
3978 | 0 | RCU_READ_LOCK_GUARD(); |
3979 | 0 | if (vdev->vq[n].vring.desc) { |
3980 | 0 | vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]); |
3981 | 0 | vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx; |
3982 | 0 | } |
3983 | 0 | } |
3984 | | |
3985 | | void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n) |
3986 | 0 | { |
3987 | 0 | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
3988 | 0 | virtio_queue_packed_restore_last_avail_idx(vdev, n); |
3989 | 0 | } else { |
3990 | 0 | virtio_queue_split_restore_last_avail_idx(vdev, n); |
3991 | 0 | } |
3992 | 0 | } |
3993 | | |
3994 | | static void virtio_queue_packed_update_used_idx(VirtIODevice *vdev, int n) |
3995 | 0 | { |
3996 | | /* used idx was updated through set_last_avail_idx() */ |
3997 | 0 | } |
3998 | | |
3999 | | static void virtio_queue_split_update_used_idx(VirtIODevice *vdev, int n) |
4000 | 0 | { |
4001 | 0 | RCU_READ_LOCK_GUARD(); |
4002 | 0 | if (vdev->vq[n].vring.desc) { |
4003 | 0 | vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]); |
4004 | 0 | } |
4005 | 0 | } |
4006 | | |
4007 | | void virtio_queue_update_used_idx(VirtIODevice *vdev, int n) |
4008 | 0 | { |
4009 | 0 | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
4010 | 0 | return virtio_queue_packed_update_used_idx(vdev, n); |
4011 | 0 | } else { |
4012 | 0 | return virtio_queue_split_update_used_idx(vdev, n); |
4013 | 0 | } |
4014 | 0 | } |
4015 | | |
4016 | | void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n) |
4017 | 0 | { |
4018 | 0 | vdev->vq[n].signalled_used_valid = false; |
4019 | 0 | } |
4020 | | |
4021 | | VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) |
4022 | 6.53M | { |
4023 | 6.53M | return vdev->vq + n; |
4024 | 6.53M | } |
4025 | | |
4026 | | uint16_t virtio_get_queue_index(VirtQueue *vq) |
4027 | 2.10M | { |
4028 | 2.10M | return vq->queue_index; |
4029 | 2.10M | } |
4030 | | |
4031 | | static void virtio_queue_guest_notifier_read(EventNotifier *n) |
4032 | 510k | { |
4033 | 510k | VirtQueue *vq = container_of(n, VirtQueue, guest_notifier); |
4034 | 510k | if (event_notifier_test_and_clear(n)) { |
4035 | 0 | virtio_irq(vq); |
4036 | 0 | } |
4037 | 510k | } |
4038 | | static void virtio_config_guest_notifier_read(EventNotifier *n) |
4039 | 67.3k | { |
4040 | 67.3k | VirtIODevice *vdev = container_of(n, VirtIODevice, config_notifier); |
4041 | | |
4042 | 67.3k | if (event_notifier_test_and_clear(n)) { |
4043 | 0 | virtio_notify_config(vdev); |
4044 | 0 | } |
4045 | 67.3k | } |
4046 | | |
4047 | | int virtio_set_guest_notifier(VirtIODevice *vdev, int n, bool assign, |
4048 | | bool with_irqfd) |
4049 | 1.15M | { |
4050 | 1.15M | bool is_config = n == VIRTIO_CONFIG_IRQ_IDX; |
4051 | 1.15M | VirtQueue *vq = is_config ? NULL : virtio_get_queue(vdev, n); |
4052 | 1.15M | EventNotifier *notifier = is_config ? |
4053 | 134k | virtio_config_get_guest_notifier(vdev) : |
4054 | 1.15M | virtio_queue_get_guest_notifier(vq); |
4055 | 1.15M | EventNotifierHandler *read_fn = is_config ? |
4056 | 134k | virtio_config_guest_notifier_read : |
4057 | 1.15M | virtio_queue_guest_notifier_read; |
4058 | | |
4059 | 1.15M | if (assign) { |
4060 | 577k | int r = event_notifier_init(notifier, 0); |
4061 | 577k | if (r < 0) { |
4062 | 0 | return r; |
4063 | 0 | } |
4064 | 577k | } |
4065 | | |
4066 | 1.15M | event_notifier_set_handler(notifier, |
4067 | 1.15M | (assign && !with_irqfd) ? read_fn : NULL); |
4068 | | |
4069 | 1.15M | if (!assign) { |
4070 | | /* Test and clear notifier before closing it,*/ |
4071 | | /* in case poll callback didn't have time to run. */ |
4072 | 577k | read_fn(notifier); |
4073 | 577k | event_notifier_cleanup(notifier); |
4074 | 577k | } |
4075 | | |
4076 | 1.15M | return 0; |
4077 | 1.15M | } |
4078 | | |
4079 | | EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) |
4080 | 1.02M | { |
4081 | 1.02M | return &vq->guest_notifier; |
4082 | 1.02M | } |
4083 | | |
4084 | | static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n) |
4085 | 0 | { |
4086 | 0 | VirtQueue *vq = container_of(n, VirtQueue, host_notifier); |
4087 | |
|
4088 | 0 | virtio_queue_set_notification(vq, 0); |
4089 | 0 | } |
4090 | | |
4091 | | static bool virtio_queue_host_notifier_aio_poll(void *opaque) |
4092 | 0 | { |
4093 | 0 | EventNotifier *n = opaque; |
4094 | 0 | VirtQueue *vq = container_of(n, VirtQueue, host_notifier); |
4095 | |
|
4096 | 0 | return vq->vring.desc && !virtio_queue_empty(vq); |
4097 | 0 | } |
4098 | | |
4099 | | static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier *n) |
4100 | 0 | { |
4101 | 0 | VirtQueue *vq = container_of(n, VirtQueue, host_notifier); |
4102 | |
|
4103 | 0 | virtio_queue_notify_vq(vq); |
4104 | 0 | } |
4105 | | |
4106 | | static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n) |
4107 | 0 | { |
4108 | 0 | VirtQueue *vq = container_of(n, VirtQueue, host_notifier); |
4109 | | |
4110 | | /* Caller polls once more after this to catch requests that race with us */ |
4111 | 0 | virtio_queue_set_notification(vq, 1); |
4112 | 0 | } |
4113 | | |
4114 | | void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx) |
4115 | 463k | { |
4116 | | /* |
4117 | | * virtio_queue_aio_detach_host_notifier() can leave notifications disabled. |
4118 | | * Re-enable them. (And if detach has not been used before, notifications |
4119 | | * being enabled is still the default state while a notifier is attached; |
4120 | | * see virtio_queue_host_notifier_aio_poll_end(), which will always leave |
4121 | | * notifications enabled once the polling section is left.) |
4122 | | */ |
4123 | 463k | if (!virtio_queue_get_notification(vq)) { |
4124 | 0 | virtio_queue_set_notification(vq, 1); |
4125 | 0 | } |
4126 | | |
4127 | 463k | aio_set_event_notifier(ctx, &vq->host_notifier, |
4128 | 463k | virtio_queue_host_notifier_read, |
4129 | 463k | virtio_queue_host_notifier_aio_poll, |
4130 | 463k | virtio_queue_host_notifier_aio_poll_ready); |
4131 | 463k | aio_set_event_notifier_poll(ctx, &vq->host_notifier, |
4132 | 463k | virtio_queue_host_notifier_aio_poll_begin, |
4133 | 463k | virtio_queue_host_notifier_aio_poll_end); |
4134 | | |
4135 | | /* |
4136 | | * We will have ignored notifications about new requests from the guest |
4137 | | * while no notifiers were attached, so "kick" the virt queue to process |
4138 | | * those requests now. |
4139 | | */ |
4140 | 463k | event_notifier_set(&vq->host_notifier); |
4141 | 463k | } |
4142 | | |
4143 | | /* |
4144 | | * Same as virtio_queue_aio_attach_host_notifier() but without polling. Use |
4145 | | * this for rx virtqueues and similar cases where the virtqueue handler |
4146 | | * function does not pop all elements. When the virtqueue is left non-empty |
4147 | | * polling consumes CPU cycles and should not be used. |
4148 | | */ |
4149 | | void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue *vq, AioContext *ctx) |
4150 | 49.4k | { |
4151 | | /* See virtio_queue_aio_attach_host_notifier() */ |
4152 | 49.4k | if (!virtio_queue_get_notification(vq)) { |
4153 | 0 | virtio_queue_set_notification(vq, 1); |
4154 | 0 | } |
4155 | | |
4156 | 49.4k | aio_set_event_notifier(ctx, &vq->host_notifier, |
4157 | 49.4k | virtio_queue_host_notifier_read, |
4158 | 49.4k | NULL, NULL); |
4159 | | |
4160 | | /* |
4161 | | * See virtio_queue_aio_attach_host_notifier(). |
4162 | | * Note that this may be unnecessary for the type of virtqueues this |
4163 | | * function is used for. Still, it will not hurt to have a quick look into |
4164 | | * whether we can/should process any of the virtqueue elements. |
4165 | | */ |
4166 | 49.4k | event_notifier_set(&vq->host_notifier); |
4167 | 49.4k | } |
4168 | | |
4169 | | void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx) |
4170 | 512k | { |
4171 | 512k | aio_set_event_notifier(ctx, &vq->host_notifier, NULL, NULL, NULL); |
4172 | | |
4173 | | /* |
4174 | | * aio_set_event_notifier_poll() does not guarantee whether io_poll_end() |
4175 | | * will run after io_poll_begin(), so by removing the notifier, we do not |
4176 | | * know whether virtio_queue_host_notifier_aio_poll_end() has run after a |
4177 | | * previous virtio_queue_host_notifier_aio_poll_begin(), i.e. whether |
4178 | | * notifications are enabled or disabled. It does not really matter anyway; |
4179 | | * we just removed the notifier, so we do not care about notifications until |
4180 | | * we potentially re-attach it. The attach_host_notifier functions will |
4181 | | * ensure that notifications are enabled again when they are needed. |
4182 | | */ |
4183 | 512k | } |
4184 | | |
4185 | | void virtio_queue_host_notifier_read(EventNotifier *n) |
4186 | 2.54M | { |
4187 | 2.54M | VirtQueue *vq = container_of(n, VirtQueue, host_notifier); |
4188 | 2.54M | if (event_notifier_test_and_clear(n)) { |
4189 | 1.03M | virtio_queue_notify_vq(vq); |
4190 | 1.03M | } |
4191 | 2.54M | } |
4192 | | |
4193 | | EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) |
4194 | 3.50M | { |
4195 | 3.50M | return &vq->host_notifier; |
4196 | 3.50M | } |
4197 | | |
4198 | | EventNotifier *virtio_config_get_guest_notifier(VirtIODevice *vdev) |
4199 | 134k | { |
4200 | 134k | return &vdev->config_notifier; |
4201 | 134k | } |
4202 | | |
4203 | | void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled) |
4204 | 1.99M | { |
4205 | 1.99M | vq->host_notifier_enabled = enabled; |
4206 | 1.99M | } |
4207 | | |
4208 | | int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n, |
4209 | | MemoryRegion *mr, bool assign) |
4210 | 0 | { |
4211 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
4212 | 0 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
4213 | |
|
4214 | 0 | if (k->set_host_notifier_mr) { |
4215 | 0 | return k->set_host_notifier_mr(qbus->parent, n, mr, assign); |
4216 | 0 | } |
4217 | | |
4218 | 0 | return -1; |
4219 | 0 | } |
4220 | | |
4221 | | void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name) |
4222 | 0 | { |
4223 | 0 | g_free(vdev->bus_name); |
4224 | 0 | vdev->bus_name = g_strdup(bus_name); |
4225 | 0 | } |
4226 | | |
4227 | | void G_GNUC_PRINTF(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...) |
4228 | 378k | { |
4229 | 378k | va_list ap; |
4230 | | |
4231 | 378k | va_start(ap, fmt); |
4232 | 378k | error_vreport(fmt, ap); |
4233 | 378k | va_end(ap); |
4234 | | |
4235 | 378k | if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { |
4236 | 44.4k | vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET; |
4237 | 44.4k | virtio_notify_config(vdev); |
4238 | 44.4k | } |
4239 | | |
4240 | 378k | vdev->broken = true; |
4241 | 378k | } |
4242 | | |
4243 | | static void virtio_memory_listener_commit(MemoryListener *listener) |
4244 | 693k | { |
4245 | 693k | VirtIODevice *vdev = container_of(listener, VirtIODevice, listener); |
4246 | 693k | int i; |
4247 | | |
4248 | 5.95M | for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { |
4249 | 5.95M | if (vdev->vq[i].vring.num == 0) { |
4250 | 693k | break; |
4251 | 693k | } |
4252 | 5.26M | virtio_init_region_cache(vdev, i); |
4253 | 5.26M | } |
4254 | 693k | } |
4255 | | |
4256 | | static void virtio_device_realize(DeviceState *dev, Error **errp) |
4257 | 21 | { |
4258 | 21 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); |
4259 | 21 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); |
4260 | 21 | Error *err = NULL; |
4261 | | |
4262 | | /* Devices should either use vmsd or the load/save methods */ |
4263 | 21 | assert(!vdc->vmsd || !vdc->load); |
4264 | | |
4265 | 21 | if (vdc->realize != NULL) { |
4266 | 21 | vdc->realize(dev, &err); |
4267 | 21 | if (err != NULL) { |
4268 | 0 | error_propagate(errp, err); |
4269 | 0 | return; |
4270 | 0 | } |
4271 | 21 | } |
4272 | | |
4273 | | /* Devices should not use both ioeventfd and notification data feature */ |
4274 | 21 | virtio_device_check_notification_compatibility(vdev, &err); |
4275 | 21 | if (err != NULL) { |
4276 | 0 | error_propagate(errp, err); |
4277 | 0 | vdc->unrealize(dev); |
4278 | 0 | return; |
4279 | 0 | } |
4280 | | |
4281 | 21 | virtio_bus_device_plugged(vdev, &err); |
4282 | 21 | if (err != NULL) { |
4283 | 0 | error_propagate(errp, err); |
4284 | 0 | vdc->unrealize(dev); |
4285 | 0 | return; |
4286 | 0 | } |
4287 | | |
4288 | 21 | vdev->listener.commit = virtio_memory_listener_commit; |
4289 | 21 | vdev->listener.name = "virtio"; |
4290 | 21 | memory_listener_register(&vdev->listener, vdev->dma_as); |
4291 | 21 | } |
4292 | | |
4293 | | static void virtio_device_unrealize(DeviceState *dev) |
4294 | 0 | { |
4295 | 0 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); |
4296 | 0 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); |
4297 | |
|
4298 | 0 | memory_listener_unregister(&vdev->listener); |
4299 | 0 | virtio_bus_device_unplugged(vdev); |
4300 | |
|
4301 | 0 | if (vdc->unrealize != NULL) { |
4302 | 0 | vdc->unrealize(dev); |
4303 | 0 | } |
4304 | |
|
4305 | 0 | g_free(vdev->bus_name); |
4306 | 0 | vdev->bus_name = NULL; |
4307 | 0 | } |
4308 | | |
4309 | | static void virtio_device_free_virtqueues(VirtIODevice *vdev) |
4310 | 189 | { |
4311 | 189 | int i; |
4312 | 189 | if (!vdev->vq) { |
4313 | 189 | return; |
4314 | 189 | } |
4315 | | |
4316 | 0 | for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { |
4317 | 0 | if (vdev->vq[i].vring.num == 0) { |
4318 | 0 | break; |
4319 | 0 | } |
4320 | 0 | virtio_virtqueue_reset_region_cache(&vdev->vq[i]); |
4321 | 0 | } |
4322 | 0 | g_free(vdev->vq); |
4323 | 0 | } |
4324 | | |
4325 | | static void virtio_device_instance_finalize(Object *obj) |
4326 | 189 | { |
4327 | 189 | VirtIODevice *vdev = VIRTIO_DEVICE(obj); |
4328 | 189 | VirtioSharedMemory *shmem; |
4329 | | |
4330 | 189 | virtio_device_free_virtqueues(vdev); |
4331 | | |
4332 | 189 | g_free(vdev->config); |
4333 | 189 | g_free(vdev->vector_queues); |
4334 | 189 | while (!QSIMPLEQ_EMPTY(&vdev->shmem_list)) { |
4335 | 0 | shmem = QSIMPLEQ_FIRST(&vdev->shmem_list); |
4336 | 0 | while (!QTAILQ_EMPTY(&shmem->mmaps)) { |
4337 | 0 | VirtioSharedMemoryMapping *mapping = QTAILQ_FIRST(&shmem->mmaps); |
4338 | 0 | virtio_del_shmem_map(shmem, mapping->offset, |
4339 | 0 | memory_region_size(mapping->mr)); |
4340 | 0 | } |
4341 | | |
4342 | | /* Clean up the embedded MemoryRegion */ |
4343 | 0 | object_unparent(OBJECT(&shmem->mr)); |
4344 | 0 | QSIMPLEQ_REMOVE_HEAD(&vdev->shmem_list, entry); |
4345 | 0 | g_free(shmem); |
4346 | 0 | } |
4347 | 189 | } |
4348 | | |
4349 | | static const Property virtio_properties[] = { |
4350 | | DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), |
4351 | | DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true), |
4352 | | DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true), |
4353 | | DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice, |
4354 | | disable_legacy_check, false), |
4355 | | }; |
4356 | | |
4357 | | static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev) |
4358 | 143k | { |
4359 | 143k | VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); |
4360 | 143k | int i, n, r, err; |
4361 | | |
4362 | | /* |
4363 | | * Batch all the host notifiers in a single transaction to avoid |
4364 | | * quadratic time complexity in address_space_update_ioeventfds(). |
4365 | | */ |
4366 | 143k | memory_region_transaction_begin(); |
4367 | 147M | for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { |
4368 | 146M | VirtQueue *vq = &vdev->vq[n]; |
4369 | 146M | if (!virtio_queue_get_num(vdev, n)) { |
4370 | 146M | continue; |
4371 | 146M | } |
4372 | 486k | r = virtio_bus_set_host_notifier(qbus, n, true); |
4373 | 486k | if (r < 0) { |
4374 | 0 | err = r; |
4375 | 0 | goto assign_error; |
4376 | 0 | } |
4377 | 486k | event_notifier_set_handler(&vq->host_notifier, |
4378 | 486k | virtio_queue_host_notifier_read); |
4379 | 486k | } |
4380 | | |
4381 | 147M | for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { |
4382 | | /* Kick right away to begin processing requests already in vring */ |
4383 | 146M | VirtQueue *vq = &vdev->vq[n]; |
4384 | 146M | if (!vq->vring.num) { |
4385 | 146M | continue; |
4386 | 146M | } |
4387 | 486k | event_notifier_set(&vq->host_notifier); |
4388 | 486k | } |
4389 | 143k | memory_region_transaction_commit(); |
4390 | 143k | return 0; |
4391 | | |
4392 | 0 | assign_error: |
4393 | 0 | i = n; /* save n for a second iteration after transaction is committed. */ |
4394 | 0 | while (--n >= 0) { |
4395 | 0 | VirtQueue *vq = &vdev->vq[n]; |
4396 | 0 | if (!virtio_queue_get_num(vdev, n)) { |
4397 | 0 | continue; |
4398 | 0 | } |
4399 | | |
4400 | 0 | event_notifier_set_handler(&vq->host_notifier, NULL); |
4401 | 0 | r = virtio_bus_set_host_notifier(qbus, n, false); |
4402 | 0 | assert(r >= 0); |
4403 | 0 | } |
4404 | | /* |
4405 | | * The transaction expects the ioeventfds to be open when it |
4406 | | * commits. Do it now, before the cleanup loop. |
4407 | | */ |
4408 | 0 | memory_region_transaction_commit(); |
4409 | |
|
4410 | 0 | while (--i >= 0) { |
4411 | 0 | if (!virtio_queue_get_num(vdev, i)) { |
4412 | 0 | continue; |
4413 | 0 | } |
4414 | 0 | virtio_bus_cleanup_host_notifier(qbus, i); |
4415 | 0 | } |
4416 | 0 | return err; |
4417 | 0 | } |
4418 | | |
4419 | | int virtio_device_start_ioeventfd(VirtIODevice *vdev) |
4420 | 3.33k | { |
4421 | 3.33k | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
4422 | 3.33k | VirtioBusState *vbus = VIRTIO_BUS(qbus); |
4423 | | |
4424 | 3.33k | return virtio_bus_start_ioeventfd(vbus); |
4425 | 3.33k | } |
4426 | | |
4427 | | static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev) |
4428 | 143k | { |
4429 | 143k | VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); |
4430 | 143k | int n, r; |
4431 | | |
4432 | | /* |
4433 | | * Batch all the host notifiers in a single transaction to avoid |
4434 | | * quadratic time complexity in address_space_update_ioeventfds(). |
4435 | | */ |
4436 | 143k | memory_region_transaction_begin(); |
4437 | 147M | for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { |
4438 | 146M | VirtQueue *vq = &vdev->vq[n]; |
4439 | | |
4440 | 146M | if (!virtio_queue_get_num(vdev, n)) { |
4441 | 146M | continue; |
4442 | 146M | } |
4443 | 486k | event_notifier_set_handler(&vq->host_notifier, NULL); |
4444 | 486k | r = virtio_bus_set_host_notifier(qbus, n, false); |
4445 | 486k | assert(r >= 0); |
4446 | 486k | } |
4447 | | /* |
4448 | | * The transaction expects the ioeventfds to be open when it |
4449 | | * commits. Do it now, before the cleanup loop. |
4450 | | */ |
4451 | 143k | memory_region_transaction_commit(); |
4452 | | |
4453 | 147M | for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { |
4454 | 146M | if (!virtio_queue_get_num(vdev, n)) { |
4455 | 146M | continue; |
4456 | 146M | } |
4457 | 486k | virtio_bus_cleanup_host_notifier(qbus, n); |
4458 | 486k | } |
4459 | 143k | } |
4460 | | |
4461 | | int virtio_device_grab_ioeventfd(VirtIODevice *vdev) |
4462 | 0 | { |
4463 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
4464 | 0 | VirtioBusState *vbus = VIRTIO_BUS(qbus); |
4465 | |
|
4466 | 0 | return virtio_bus_grab_ioeventfd(vbus); |
4467 | 0 | } |
4468 | | |
4469 | | void virtio_device_release_ioeventfd(VirtIODevice *vdev) |
4470 | 0 | { |
4471 | 0 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
4472 | 0 | VirtioBusState *vbus = VIRTIO_BUS(qbus); |
4473 | |
|
4474 | 0 | virtio_bus_release_ioeventfd(vbus); |
4475 | 0 | } |
4476 | | |
4477 | | static void virtio_device_class_init(ObjectClass *klass, const void *data) |
4478 | 63 | { |
4479 | | /* Set the default value here. */ |
4480 | 63 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); |
4481 | 63 | DeviceClass *dc = DEVICE_CLASS(klass); |
4482 | | |
4483 | 63 | dc->realize = virtio_device_realize; |
4484 | 63 | dc->unrealize = virtio_device_unrealize; |
4485 | 63 | dc->bus_type = TYPE_VIRTIO_BUS; |
4486 | 63 | device_class_set_props(dc, virtio_properties); |
4487 | 63 | vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl; |
4488 | 63 | vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl; |
4489 | | |
4490 | 63 | vdc->legacy_features |= VIRTIO_LEGACY_FEATURES; |
4491 | 63 | } |
4492 | | |
4493 | | bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev) |
4494 | 2 | { |
4495 | 2 | BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); |
4496 | 2 | VirtioBusState *vbus = VIRTIO_BUS(qbus); |
4497 | | |
4498 | 2 | return virtio_bus_ioeventfd_enabled(vbus); |
4499 | 2 | } |
4500 | | |
4501 | | VirtQueueStatus *qmp_x_query_virtio_queue_status(const char *path, |
4502 | | uint16_t queue, |
4503 | | Error **errp) |
4504 | 0 | { |
4505 | 0 | VirtIODevice *vdev; |
4506 | 0 | VirtQueueStatus *status; |
4507 | |
|
4508 | 0 | vdev = qmp_find_virtio_device(path); |
4509 | 0 | if (vdev == NULL) { |
4510 | 0 | error_setg(errp, "Path %s is not a VirtIODevice", path); |
4511 | 0 | return NULL; |
4512 | 0 | } |
4513 | | |
4514 | 0 | if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) { |
4515 | 0 | error_setg(errp, "Invalid virtqueue number %d", queue); |
4516 | 0 | return NULL; |
4517 | 0 | } |
4518 | | |
4519 | 0 | status = g_new0(VirtQueueStatus, 1); |
4520 | 0 | status->name = g_strdup(vdev->name); |
4521 | 0 | status->queue_index = vdev->vq[queue].queue_index; |
4522 | 0 | status->inuse = vdev->vq[queue].inuse; |
4523 | 0 | status->vring_num = vdev->vq[queue].vring.num; |
4524 | 0 | status->vring_num_default = vdev->vq[queue].vring.num_default; |
4525 | 0 | status->vring_align = vdev->vq[queue].vring.align; |
4526 | 0 | status->vring_desc = vdev->vq[queue].vring.desc; |
4527 | 0 | status->vring_avail = vdev->vq[queue].vring.avail; |
4528 | 0 | status->vring_used = vdev->vq[queue].vring.used; |
4529 | 0 | status->used_idx = vdev->vq[queue].used_idx; |
4530 | 0 | status->signalled_used = vdev->vq[queue].signalled_used; |
4531 | 0 | status->signalled_used_valid = vdev->vq[queue].signalled_used_valid; |
4532 | |
|
4533 | 0 | if (vdev->vhost_started) { |
4534 | 0 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); |
4535 | 0 | struct vhost_dev *hdev = vdc->get_vhost(vdev); |
4536 | | |
4537 | | /* check if vq index exists for vhost as well */ |
4538 | 0 | if (queue >= hdev->vq_index && queue < hdev->vq_index + hdev->nvqs) { |
4539 | 0 | status->has_last_avail_idx = true; |
4540 | |
|
4541 | 0 | int vhost_vq_index = |
4542 | 0 | hdev->vhost_ops->vhost_get_vq_index(hdev, queue); |
4543 | 0 | struct vhost_vring_state state = { |
4544 | 0 | .index = vhost_vq_index, |
4545 | 0 | }; |
4546 | |
|
4547 | 0 | status->last_avail_idx = |
4548 | 0 | hdev->vhost_ops->vhost_get_vring_base(hdev, &state); |
4549 | 0 | } |
4550 | 0 | } else { |
4551 | 0 | status->has_shadow_avail_idx = true; |
4552 | 0 | status->has_last_avail_idx = true; |
4553 | 0 | status->last_avail_idx = vdev->vq[queue].last_avail_idx; |
4554 | 0 | status->shadow_avail_idx = vdev->vq[queue].shadow_avail_idx; |
4555 | 0 | } |
4556 | |
|
4557 | 0 | return status; |
4558 | 0 | } |
4559 | | |
4560 | | static strList *qmp_decode_vring_desc_flags(uint16_t flags) |
4561 | 0 | { |
4562 | 0 | strList *list = NULL; |
4563 | 0 | strList *node; |
4564 | 0 | int i; |
4565 | |
|
4566 | 0 | struct { |
4567 | 0 | uint16_t flag; |
4568 | 0 | const char *value; |
4569 | 0 | } map[] = { |
4570 | 0 | { VRING_DESC_F_NEXT, "next" }, |
4571 | 0 | { VRING_DESC_F_WRITE, "write" }, |
4572 | 0 | { VRING_DESC_F_INDIRECT, "indirect" }, |
4573 | 0 | { 1 << VRING_PACKED_DESC_F_AVAIL, "avail" }, |
4574 | 0 | { 1 << VRING_PACKED_DESC_F_USED, "used" }, |
4575 | 0 | { 0, "" } |
4576 | 0 | }; |
4577 | |
|
4578 | 0 | for (i = 0; map[i].flag; i++) { |
4579 | 0 | if ((map[i].flag & flags) == 0) { |
4580 | 0 | continue; |
4581 | 0 | } |
4582 | 0 | node = g_malloc0(sizeof(strList)); |
4583 | 0 | node->value = g_strdup(map[i].value); |
4584 | 0 | node->next = list; |
4585 | 0 | list = node; |
4586 | 0 | } |
4587 | |
|
4588 | 0 | return list; |
4589 | 0 | } |
4590 | | |
4591 | | VirtioQueueElement *qmp_x_query_virtio_queue_element(const char *path, |
4592 | | uint16_t queue, |
4593 | | bool has_index, |
4594 | | uint16_t index, |
4595 | | Error **errp) |
4596 | 0 | { |
4597 | 0 | VirtIODevice *vdev; |
4598 | 0 | VirtQueue *vq; |
4599 | 0 | VirtioQueueElement *element = NULL; |
4600 | |
|
4601 | 0 | vdev = qmp_find_virtio_device(path); |
4602 | 0 | if (vdev == NULL) { |
4603 | 0 | error_setg(errp, "Path %s is not a VirtIO device", path); |
4604 | 0 | return NULL; |
4605 | 0 | } |
4606 | | |
4607 | 0 | if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) { |
4608 | 0 | error_setg(errp, "Invalid virtqueue number %d", queue); |
4609 | 0 | return NULL; |
4610 | 0 | } |
4611 | 0 | vq = &vdev->vq[queue]; |
4612 | |
|
4613 | 0 | if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { |
4614 | 0 | error_setg(errp, "Packed ring not supported"); |
4615 | 0 | return NULL; |
4616 | 0 | } else { |
4617 | 0 | unsigned int head, i, max; |
4618 | 0 | VRingMemoryRegionCaches *caches; |
4619 | 0 | MemoryRegionCache indirect_desc_cache; |
4620 | 0 | MemoryRegionCache *desc_cache; |
4621 | 0 | VRingDesc desc; |
4622 | 0 | VirtioRingDescList *list = NULL; |
4623 | 0 | VirtioRingDescList *node; |
4624 | 0 | int rc; int ndescs; |
4625 | |
|
4626 | 0 | address_space_cache_init_empty(&indirect_desc_cache); |
4627 | |
|
4628 | 0 | RCU_READ_LOCK_GUARD(); |
4629 | |
|
4630 | 0 | max = vq->vring.num; |
4631 | |
|
4632 | 0 | if (!has_index) { |
4633 | 0 | head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num); |
4634 | 0 | } else { |
4635 | 0 | head = vring_avail_ring(vq, index % vq->vring.num); |
4636 | 0 | } |
4637 | 0 | i = head; |
4638 | |
|
4639 | 0 | caches = vring_get_region_caches(vq); |
4640 | 0 | if (!caches) { |
4641 | 0 | error_setg(errp, "Region caches not initialized"); |
4642 | 0 | return NULL; |
4643 | 0 | } |
4644 | 0 | if (caches->desc.len < max * sizeof(VRingDesc)) { |
4645 | 0 | error_setg(errp, "Cannot map descriptor ring"); |
4646 | 0 | return NULL; |
4647 | 0 | } |
4648 | | |
4649 | 0 | desc_cache = &caches->desc; |
4650 | 0 | vring_split_desc_read(vdev, &desc, desc_cache, i); |
4651 | 0 | if (desc.flags & VRING_DESC_F_INDIRECT) { |
4652 | 0 | int64_t len; |
4653 | 0 | len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, |
4654 | 0 | desc.addr, desc.len, false); |
4655 | 0 | desc_cache = &indirect_desc_cache; |
4656 | 0 | if (len < desc.len) { |
4657 | 0 | error_setg(errp, "Cannot map indirect buffer"); |
4658 | 0 | goto done; |
4659 | 0 | } |
4660 | | |
4661 | 0 | max = desc.len / sizeof(VRingDesc); |
4662 | 0 | i = 0; |
4663 | 0 | vring_split_desc_read(vdev, &desc, desc_cache, i); |
4664 | 0 | } |
4665 | | |
4666 | 0 | element = g_new0(VirtioQueueElement, 1); |
4667 | 0 | element->avail = g_new0(VirtioRingAvail, 1); |
4668 | 0 | element->used = g_new0(VirtioRingUsed, 1); |
4669 | 0 | element->name = g_strdup(vdev->name); |
4670 | 0 | element->index = head; |
4671 | 0 | element->avail->flags = vring_avail_flags(vq); |
4672 | 0 | element->avail->idx = vring_avail_idx(vq); |
4673 | 0 | element->avail->ring = head; |
4674 | 0 | element->used->flags = vring_used_flags(vq); |
4675 | 0 | element->used->idx = vring_used_idx(vq); |
4676 | 0 | ndescs = 0; |
4677 | |
|
4678 | 0 | do { |
4679 | | /* A buggy driver may produce an infinite loop */ |
4680 | 0 | if (ndescs >= max) { |
4681 | 0 | break; |
4682 | 0 | } |
4683 | 0 | node = g_new0(VirtioRingDescList, 1); |
4684 | 0 | node->value = g_new0(VirtioRingDesc, 1); |
4685 | 0 | node->value->addr = desc.addr; |
4686 | 0 | node->value->len = desc.len; |
4687 | 0 | node->value->flags = qmp_decode_vring_desc_flags(desc.flags); |
4688 | 0 | node->next = list; |
4689 | 0 | list = node; |
4690 | |
|
4691 | 0 | ndescs++; |
4692 | 0 | rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max); |
4693 | 0 | } while (rc == VIRTQUEUE_READ_DESC_MORE); |
4694 | 0 | element->descs = list; |
4695 | 0 | done: |
4696 | 0 | address_space_cache_destroy(&indirect_desc_cache); |
4697 | 0 | } |
4698 | | |
4699 | 0 | return element; |
4700 | 0 | } |
4701 | | |
4702 | | static const TypeInfo virtio_device_info = { |
4703 | | .name = TYPE_VIRTIO_DEVICE, |
4704 | | .parent = TYPE_DEVICE, |
4705 | | .instance_size = sizeof(VirtIODevice), |
4706 | | .class_init = virtio_device_class_init, |
4707 | | .instance_finalize = virtio_device_instance_finalize, |
4708 | | .abstract = true, |
4709 | | .class_size = sizeof(VirtioDeviceClass), |
4710 | | }; |
4711 | | |
4712 | | static const TypeInfo virtio_shared_memory_mapping_info = { |
4713 | | .name = TYPE_VIRTIO_SHARED_MEMORY_MAPPING, |
4714 | | .parent = TYPE_OBJECT, |
4715 | | .instance_size = sizeof(VirtioSharedMemoryMapping), |
4716 | | .instance_init = virtio_shared_memory_mapping_instance_init, |
4717 | | .instance_finalize = virtio_shared_memory_mapping_instance_finalize, |
4718 | | }; |
4719 | | |
4720 | | static void virtio_register_types(void) |
4721 | 63 | { |
4722 | 63 | type_register_static(&virtio_device_info); |
4723 | 63 | type_register_static(&virtio_shared_memory_mapping_info); |
4724 | 63 | } |
4725 | | |
4726 | | type_init(virtio_register_types) |
4727 | | |
4728 | | QEMUBH *virtio_bh_new_guarded_full(DeviceState *dev, |
4729 | | QEMUBHFunc *cb, void *opaque, |
4730 | | const char *name) |
4731 | 2 | { |
4732 | 2 | DeviceState *transport = qdev_get_parent_bus(dev)->parent; |
4733 | | |
4734 | 2 | return qemu_bh_new_full(cb, opaque, name, |
4735 | 2 | &transport->mem_reentrancy_guard); |
4736 | 2 | } |
4737 | | |
4738 | | QEMUBH *virtio_bh_io_new_guarded_full(DeviceState *dev, |
4739 | | QEMUBHFunc *cb, void *opaque, |
4740 | | const char *name) |
4741 | 12 | { |
4742 | 12 | DeviceState *transport = qdev_get_parent_bus(dev)->parent; |
4743 | | |
4744 | 12 | return aio_bh_new_full(iohandler_get_aio_context(), cb, opaque, name, |
4745 | 12 | &transport->mem_reentrancy_guard); |
4746 | 12 | } |