Coverage Report

Created: 2026-11-20 08:13

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qemu/hw/virtio/virtio.c
Line
Count
Source
1
/*
2
 * Virtio Support
3
 *
4
 * Copyright IBM, Corp. 2007
5
 *
6
 * Authors:
7
 *  Anthony Liguori   <aliguori@us.ibm.com>
8
 *
9
 * This work is licensed under the terms of the GNU GPL, version 2.  See
10
 * the COPYING file in the top-level directory.
11
 *
12
 */
13
14
#include "qemu/osdep.h"
15
#include "qapi/error.h"
16
#include "qapi/qapi-commands-virtio.h"
17
#include "trace.h"
18
#include "qemu/defer-call.h"
19
#include "qemu/error-report.h"
20
#include "qemu/log.h"
21
#include "qemu/main-loop.h"
22
#include "qemu/module.h"
23
#include "qemu/target-info.h"
24
#include "qom/object_interfaces.h"
25
#include "hw/core/cpu.h"
26
#include "hw/virtio/virtio.h"
27
#include "hw/virtio/vhost.h"
28
#include "migration/qemu-file-types.h"
29
#include "qemu/atomic.h"
30
#include "hw/virtio/virtio-bus.h"
31
#include "hw/core/qdev-properties.h"
32
#include "hw/virtio/virtio-access.h"
33
#include "system/dma.h"
34
#include "system/iothread.h"
35
#include "system/memory.h"
36
#include "system/runstate.h"
37
#include "virtio-qmp.h"
38
39
#include "standard-headers/linux/virtio_ids.h"
40
#include "standard-headers/linux/vhost_types.h"
41
#include "standard-headers/linux/virtio_blk.h"
42
#include "standard-headers/linux/virtio_console.h"
43
#include "standard-headers/linux/virtio_gpu.h"
44
#include "standard-headers/linux/virtio_net.h"
45
#include "standard-headers/linux/virtio_scsi.h"
46
#include "standard-headers/linux/virtio_i2c.h"
47
#include "standard-headers/linux/virtio_balloon.h"
48
#include "standard-headers/linux/virtio_iommu.h"
49
#include "standard-headers/linux/virtio_mem.h"
50
#include "standard-headers/linux/virtio_vsock.h"
51
#include "standard-headers/linux/virtio_spi.h"
52
53
/*
54
 * Maximum size of virtio device config space
55
 */
56
#define VHOST_USER_MAX_CONFIG_SIZE 256
57
58
/*
59
 * The alignment to use between consumer and producer parts of vring.
60
 * x86 pagesize again. This is the default, used by transports like PCI
61
 * which don't provide a means for the guest to tell the host the alignment.
62
 */
63
179
#define VIRTIO_PCI_VRING_ALIGN         4096
64
65
typedef struct VRingDesc
66
{
67
    uint64_t addr;
68
    uint32_t len;
69
    uint16_t flags;
70
    uint16_t next;
71
} VRingDesc;
72
73
typedef struct VRingPackedDesc {
74
    uint64_t addr;
75
    uint32_t len;
76
    uint16_t id;
77
    uint16_t flags;
78
} VRingPackedDesc;
79
80
typedef struct VRingAvail
81
{
82
    uint16_t flags;
83
    uint16_t idx;
84
    uint16_t ring[];
85
} VRingAvail;
86
87
typedef struct VRingUsedElem
88
{
89
    uint32_t id;
90
    uint32_t len;
91
} VRingUsedElem;
92
93
typedef struct VRingUsed
94
{
95
    uint16_t flags;
96
    uint16_t idx;
97
    VRingUsedElem ring[];
98
} VRingUsed;
99
100
typedef struct VRingMemoryRegionCaches {
101
    struct rcu_head rcu;
102
    MemoryRegionCache desc;
103
    MemoryRegionCache avail;
104
    MemoryRegionCache used;
105
} VRingMemoryRegionCaches;
106
107
typedef struct VRing
108
{
109
    unsigned int num;
110
    unsigned int num_default;
111
    unsigned int align;
112
    hwaddr desc;
113
    hwaddr avail;
114
    hwaddr used;
115
    VRingMemoryRegionCaches *caches;
116
} VRing;
117
118
typedef struct VRingPackedDescEvent {
119
    uint16_t off_wrap;
120
    uint16_t flags;
121
} VRingPackedDescEvent ;
122
123
struct VirtQueue
124
{
125
    VRing vring;
126
    VirtQueueElement *used_elems;
127
128
    /* Next head to pop */
129
    uint16_t last_avail_idx;
130
    bool last_avail_wrap_counter;
131
132
    /* Last avail_idx read from VQ. */
133
    uint16_t shadow_avail_idx;
134
    bool shadow_avail_wrap_counter;
135
136
    uint16_t used_idx;
137
    bool used_wrap_counter;
138
139
    /* Last used index value we have signalled on */
140
    uint16_t signalled_used;
141
142
    /* Last used index value we have signalled on */
143
    bool signalled_used_valid;
144
145
    /* Notification enabled? */
146
    bool notification;
147
148
    uint16_t queue_index;
149
150
    unsigned int inuse;
151
152
    uint16_t vector;
153
    VirtIOHandleOutput handle_output;
154
    VirtIODevice *vdev;
155
    EventNotifier guest_notifier;
156
    EventNotifier host_notifier;
157
    bool host_notifier_enabled;
158
    QLIST_ENTRY(VirtQueue) node;
159
};
160
161
const char *virtio_device_names[] = {
162
    [VIRTIO_ID_NET] = "virtio-net",
163
    [VIRTIO_ID_BLOCK] = "virtio-blk",
164
    [VIRTIO_ID_CONSOLE] = "virtio-serial",
165
    [VIRTIO_ID_RNG] = "virtio-rng",
166
    [VIRTIO_ID_BALLOON] = "virtio-balloon",
167
    [VIRTIO_ID_IOMEM] = "virtio-iomem",
168
    [VIRTIO_ID_RPMSG] = "virtio-rpmsg",
169
    [VIRTIO_ID_SCSI] = "virtio-scsi",
170
    [VIRTIO_ID_9P] = "virtio-9p",
171
    [VIRTIO_ID_MAC80211_WLAN] = "virtio-mac-wlan",
172
    [VIRTIO_ID_RPROC_SERIAL] = "virtio-rproc-serial",
173
    [VIRTIO_ID_CAIF] = "virtio-caif",
174
    [VIRTIO_ID_MEMORY_BALLOON] = "virtio-mem-balloon",
175
    [VIRTIO_ID_GPU] = "virtio-gpu",
176
    [VIRTIO_ID_CLOCK] = "virtio-clk",
177
    [VIRTIO_ID_INPUT] = "virtio-input",
178
    [VIRTIO_ID_VSOCK] = "vhost-vsock",
179
    [VIRTIO_ID_CRYPTO] = "virtio-crypto",
180
    [VIRTIO_ID_SIGNAL_DIST] = "virtio-signal",
181
    [VIRTIO_ID_PSTORE] = "virtio-pstore",
182
    [VIRTIO_ID_IOMMU] = "virtio-iommu",
183
    [VIRTIO_ID_MEM] = "virtio-mem",
184
    [VIRTIO_ID_SOUND] = "virtio-sound",
185
    [VIRTIO_ID_FS] = "virtio-user-fs",
186
    [VIRTIO_ID_PMEM] = "virtio-pmem",
187
    [VIRTIO_ID_RPMB] = "virtio-rpmb",
188
    [VIRTIO_ID_MAC80211_HWSIM] = "virtio-mac-hwsim",
189
    [VIRTIO_ID_VIDEO_ENCODER] = "virtio-vid-encoder",
190
    [VIRTIO_ID_VIDEO_DECODER] = "virtio-vid-decoder",
191
    [VIRTIO_ID_SCMI] = "virtio-scmi",
192
    [VIRTIO_ID_NITRO_SEC_MOD] = "virtio-nitro-sec-mod",
193
    [VIRTIO_ID_I2C_ADAPTER] = "vhost-user-i2c",
194
    [VIRTIO_ID_WATCHDOG] = "virtio-watchdog",
195
    [VIRTIO_ID_CAN] = "virtio-can",
196
    [VIRTIO_ID_DMABUF] = "virtio-dmabuf",
197
    [VIRTIO_ID_PARAM_SERV] = "virtio-param-serv",
198
    [VIRTIO_ID_AUDIO_POLICY] = "virtio-audio-pol",
199
    [VIRTIO_ID_BT] = "virtio-bluetooth",
200
    [VIRTIO_ID_GPIO] = "virtio-gpio",
201
    [VIRTIO_ID_SPI] = "virtio-spi",
202
    [VIRTIO_ID_MEDIA] = "virtio-media",
203
};
204
205
static const char *virtio_id_to_name(uint16_t device_id)
206
21
{
207
21
    assert(device_id < G_N_ELEMENTS(virtio_device_names));
208
21
    const char *name = virtio_device_names[device_id];
209
21
    assert(name != NULL);
210
21
    return name;
211
21
}
212
213
static void virtio_check_indirect_feature(VirtIODevice *vdev)
214
186k
{
215
186k
    if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) {
216
178k
        qemu_log_mask(LOG_GUEST_ERROR,
217
178k
                      "Device %s: indirect_desc was not negotiated!\n",
218
178k
                      vdev->name);
219
178k
    }
220
186k
}
221
222
static inline uint16_t virtio_lduw_phys_cached(VirtIODevice *vdev,
223
                                               MemoryRegionCache *cache,
224
                                               hwaddr pa)
225
14.2M
{
226
14.2M
    if (virtio_vdev_is_big_endian(vdev)) {
227
0
        return lduw_be_phys_cached(cache, pa);
228
0
    }
229
14.2M
    return lduw_le_phys_cached(cache, pa);
230
14.2M
}
231
232
static inline void virtio_stw_phys_cached(VirtIODevice *vdev,
233
                                          MemoryRegionCache *cache,
234
                                          hwaddr pa, uint16_t value)
235
4.97M
{
236
4.97M
    if (virtio_vdev_is_big_endian(vdev)) {
237
0
        stw_be_phys_cached(cache, pa, value);
238
4.97M
    } else {
239
4.97M
        stw_le_phys_cached(cache, pa, value);
240
4.97M
    }
241
4.97M
}
242
243
/* Called within call_rcu().  */
244
static void virtio_free_region_cache(VRingMemoryRegionCaches *caches)
245
1.35M
{
246
1.35M
    assert(caches != NULL);
247
1.35M
    address_space_cache_destroy(&caches->desc);
248
1.35M
    address_space_cache_destroy(&caches->avail);
249
1.35M
    address_space_cache_destroy(&caches->used);
250
1.35M
    g_free(caches);
251
1.35M
}
252
253
static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)
254
385M
{
255
385M
    VRingMemoryRegionCaches *caches;
256
257
385M
    caches = qatomic_read(&vq->vring.caches);
258
385M
    qatomic_rcu_set(&vq->vring.caches, NULL);
259
385M
    if (caches) {
260
341k
        call_rcu(caches, virtio_free_region_cache, rcu);
261
341k
    }
262
385M
}
263
264
void virtio_init_region_cache(VirtIODevice *vdev, int n)
265
7.49M
{
266
7.49M
    VirtQueue *vq = &vdev->vq[n];
267
7.49M
    VRingMemoryRegionCaches *old = vq->vring.caches;
268
7.49M
    VRingMemoryRegionCaches *new = NULL;
269
7.49M
    hwaddr addr, size;
270
7.49M
    int64_t len;
271
7.49M
    bool packed;
272
273
274
7.49M
    addr = vq->vring.desc;
275
7.49M
    if (!addr) {
276
6.05M
        goto out_no_cache;
277
6.05M
    }
278
1.44M
    new = g_new0(VRingMemoryRegionCaches, 1);
279
1.44M
    size = virtio_queue_get_desc_size(vdev, n);
280
1.44M
    packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
281
1.44M
                                   true : false;
282
1.44M
    len = address_space_cache_init(&new->desc, vdev->dma_as,
283
1.44M
                                   addr, size, packed);
284
1.44M
    if (len < size) {
285
68.2k
        g_autofree char *devname = qdev_get_human_name(DEVICE(vdev));
286
287
68.2k
        virtio_error(vdev,
288
68.2k
                "Failed to map descriptor ring for device %s: "
289
68.2k
                "invalid guest physical address or corrupted queue setup",
290
68.2k
                devname);
291
68.2k
        goto err_desc;
292
68.2k
    }
293
294
1.37M
    size = virtio_queue_get_used_size(vdev, n);
295
1.37M
    len = address_space_cache_init(&new->used, vdev->dma_as,
296
1.37M
                                   vq->vring.used, size, true);
297
1.37M
    if (len < size) {
298
8.29k
        g_autofree char *devname = qdev_get_human_name(DEVICE(vdev));
299
300
8.29k
        virtio_error(vdev,
301
8.29k
                "Failed to map used ring for device %s: "
302
8.29k
                "possible guest misconfiguration or insufficient memory",
303
8.29k
                devname);
304
8.29k
        goto err_used;
305
8.29k
    }
306
307
1.36M
    size = virtio_queue_get_avail_size(vdev, n);
308
1.36M
    len = address_space_cache_init(&new->avail, vdev->dma_as,
309
1.36M
                                   vq->vring.avail, size, false);
310
1.36M
    if (len < size) {
311
13.5k
        g_autofree char *devname = qdev_get_human_name(DEVICE(vdev));
312
313
13.5k
        virtio_error(vdev,
314
13.5k
                "Failed to map avalaible ring for device %s: "
315
13.5k
                "possible queue misconfiguration or overlapping memory region",
316
13.5k
                devname);
317
13.5k
        goto err_avail;
318
13.5k
    }
319
320
1.35M
    qatomic_rcu_set(&vq->vring.caches, new);
321
1.35M
    if (old) {
322
1.01M
        call_rcu(old, virtio_free_region_cache, rcu);
323
1.01M
    }
324
1.35M
    return;
325
326
13.5k
err_avail:
327
13.5k
    address_space_cache_destroy(&new->avail);
328
21.8k
err_used:
329
21.8k
    address_space_cache_destroy(&new->used);
330
90.0k
err_desc:
331
90.0k
    address_space_cache_destroy(&new->desc);
332
6.14M
out_no_cache:
333
6.14M
    g_free(new);
334
6.14M
    virtio_virtqueue_reset_region_cache(vq);
335
6.14M
}
336
337
/* virt queue functions */
338
void virtio_queue_update_rings(VirtIODevice *vdev, int n)
339
438k
{
340
438k
    VRing *vring = &vdev->vq[n].vring;
341
342
438k
    if (!vring->num || !vring->desc || !vring->align) {
343
        /* not yet setup -> nothing to do */
344
0
        return;
345
0
    }
346
438k
    vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
347
438k
    vring->used = vring_align(vring->avail +
348
438k
                              offsetof(VRingAvail, ring[vring->num]),
349
438k
                              vring->align);
350
438k
    virtio_init_region_cache(vdev, n);
351
438k
}
352
353
/* Called within rcu_read_lock().  */
354
static void vring_split_desc_read(VirtIODevice *vdev, VRingDesc *desc,
355
                                  MemoryRegionCache *cache, int i)
356
92.7M
{
357
92.7M
    address_space_read_cached(cache, i * sizeof(VRingDesc),
358
92.7M
                              desc, sizeof(VRingDesc));
359
92.7M
    virtio_tswap64s(vdev, &desc->addr);
360
92.7M
    virtio_tswap32s(vdev, &desc->len);
361
92.7M
    virtio_tswap16s(vdev, &desc->flags);
362
92.7M
    virtio_tswap16s(vdev, &desc->next);
363
92.7M
}
364
365
static void vring_packed_event_read(VirtIODevice *vdev,
366
                                    MemoryRegionCache *cache,
367
                                    VRingPackedDescEvent *e)
368
0
{
369
0
    hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap);
370
0
    hwaddr off_flags = offsetof(VRingPackedDescEvent, flags);
371
372
0
    e->flags = virtio_lduw_phys_cached(vdev, cache, off_flags);
373
    /* Make sure flags is seen before off_wrap */
374
0
    smp_rmb();
375
0
    e->off_wrap = virtio_lduw_phys_cached(vdev, cache, off_off);
376
0
}
377
378
static void vring_packed_off_wrap_write(VirtIODevice *vdev,
379
                                        MemoryRegionCache *cache,
380
                                        uint16_t off_wrap)
381
0
{
382
0
    hwaddr off = offsetof(VRingPackedDescEvent, off_wrap);
383
384
0
    virtio_stw_phys_cached(vdev, cache, off, off_wrap);
385
0
    address_space_cache_invalidate(cache, off, sizeof(off_wrap));
386
0
}
387
388
static void vring_packed_flags_write(VirtIODevice *vdev,
389
                                     MemoryRegionCache *cache, uint16_t flags)
390
0
{
391
0
    hwaddr off = offsetof(VRingPackedDescEvent, flags);
392
393
0
    virtio_stw_phys_cached(vdev, cache, off, flags);
394
0
    address_space_cache_invalidate(cache, off, sizeof(flags));
395
0
}
396
397
/* Called within rcu_read_lock().  */
398
static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq)
399
27.7M
{
400
27.7M
    return qatomic_rcu_read(&vq->vring.caches);
401
27.7M
}
402
403
/* Called within rcu_read_lock().  */
404
static inline uint16_t vring_avail_flags(VirtQueue *vq)
405
2.91M
{
406
2.91M
    VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
407
2.91M
    hwaddr pa = offsetof(VRingAvail, flags);
408
409
2.91M
    if (!caches) {
410
93
        return 0;
411
93
    }
412
413
2.91M
    return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
414
2.91M
}
415
416
/* Called within rcu_read_lock().  */
417
static inline uint16_t vring_avail_idx(VirtQueue *vq)
418
1.20M
{
419
1.20M
    VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
420
1.20M
    hwaddr pa = offsetof(VRingAvail, idx);
421
422
1.20M
    if (!caches) {
423
3.01k
        return 0;
424
3.01k
    }
425
426
1.20M
    vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
427
1.20M
    return vq->shadow_avail_idx;
428
1.20M
}
429
430
/* Called within rcu_read_lock().  */
431
static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
432
9.47M
{
433
9.47M
    VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
434
9.47M
    hwaddr pa = offsetof(VRingAvail, ring[i]);
435
436
9.47M
    if (!caches) {
437
258
        return 0;
438
258
    }
439
440
9.47M
    return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
441
9.47M
}
442
443
/* Called within rcu_read_lock().  */
444
static inline uint16_t vring_get_used_event(VirtQueue *vq)
445
543k
{
446
543k
    return vring_avail_ring(vq, vq->vring.num);
447
543k
}
448
449
/* Called within rcu_read_lock().  */
450
static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
451
                                    int i)
452
3.86M
{
453
3.86M
    VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
454
3.86M
    hwaddr pa = offsetof(VRingUsed, ring[i]);
455
456
3.86M
    if (!caches) {
457
178
        return;
458
178
    }
459
460
3.86M
    virtio_tswap32s(vq->vdev, &uelem->id);
461
3.86M
    virtio_tswap32s(vq->vdev, &uelem->len);
462
3.86M
    address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem));
463
3.86M
    address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem));
464
3.86M
}
465
466
/* Called within rcu_read_lock(). */
467
static inline uint16_t vring_used_flags(VirtQueue *vq)
468
0
{
469
0
    VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
470
0
    hwaddr pa = offsetof(VRingUsed, flags);
471
472
0
    if (!caches) {
473
0
        return 0;
474
0
    }
475
476
0
    return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
477
0
}
478
479
/* Called within rcu_read_lock().  */
480
static uint16_t vring_used_idx(VirtQueue *vq)
481
0
{
482
0
    VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
483
0
    hwaddr pa = offsetof(VRingUsed, idx);
484
485
0
    if (!caches) {
486
0
        return 0;
487
0
    }
488
489
0
    return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
490
0
}
491
492
/* Called within rcu_read_lock().  */
493
static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
494
3.86M
{
495
3.86M
    VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
496
3.86M
    hwaddr pa = offsetof(VRingUsed, idx);
497
498
3.86M
    if (caches) {
499
3.86M
        virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
500
3.86M
        address_space_cache_invalidate(&caches->used, pa, sizeof(val));
501
3.86M
    }
502
503
3.86M
    vq->used_idx = val;
504
3.86M
}
505
506
/* Called within rcu_read_lock().  */
507
static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
508
535k
{
509
535k
    VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
510
535k
    VirtIODevice *vdev = vq->vdev;
511
535k
    hwaddr pa = offsetof(VRingUsed, flags);
512
535k
    uint16_t flags;
513
514
535k
    if (!caches) {
515
0
        return;
516
0
    }
517
518
535k
    flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
519
535k
    virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask);
520
535k
    address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
521
535k
}
522
523
/* Called within rcu_read_lock().  */
524
static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
525
88.9k
{
526
88.9k
    VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
527
88.9k
    VirtIODevice *vdev = vq->vdev;
528
88.9k
    hwaddr pa = offsetof(VRingUsed, flags);
529
88.9k
    uint16_t flags;
530
531
88.9k
    if (!caches) {
532
451
        return;
533
451
    }
534
535
88.4k
    flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
536
88.4k
    virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask);
537
88.4k
    address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
538
88.4k
}
539
540
/* Called within rcu_read_lock().  */
541
static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
542
1.63M
{
543
1.63M
    VRingMemoryRegionCaches *caches;
544
1.63M
    hwaddr pa;
545
1.63M
    if (!vq->notification) {
546
1.14M
        return;
547
1.14M
    }
548
549
489k
    caches = vring_get_region_caches(vq);
550
489k
    if (!caches) {
551
708
        return;
552
708
    }
553
554
488k
    pa = offsetof(VRingUsed, ring[vq->vring.num]);
555
488k
    virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
556
488k
    address_space_cache_invalidate(&caches->used, pa, sizeof(val));
557
488k
}
558
559
static void virtio_queue_split_set_notification(VirtQueue *vq, int enable)
560
1.36M
{
561
1.36M
    RCU_READ_LOCK_GUARD();
562
563
1.36M
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
564
743k
        vring_set_avail_event(vq, vring_avail_idx(vq));
565
743k
    } else if (enable) {
566
88.9k
        vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
567
535k
    } else {
568
535k
        vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
569
535k
    }
570
1.36M
    if (enable) {
571
        /* Expose avail event/used flags before caller checks the avail idx. */
572
115k
        smp_mb();
573
115k
    }
574
1.36M
}
575
576
static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable)
577
0
{
578
0
    uint16_t off_wrap;
579
0
    VRingPackedDescEvent e;
580
0
    VRingMemoryRegionCaches *caches;
581
582
0
    RCU_READ_LOCK_GUARD();
583
0
    caches = vring_get_region_caches(vq);
584
0
    if (!caches) {
585
0
        return;
586
0
    }
587
588
0
    vring_packed_event_read(vq->vdev, &caches->used, &e);
589
590
0
    if (!enable) {
591
0
        e.flags = VRING_PACKED_EVENT_FLAG_DISABLE;
592
0
    } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
593
0
        off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15;
594
0
        vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap);
595
        /* Make sure off_wrap is wrote before flags */
596
0
        smp_wmb();
597
0
        e.flags = VRING_PACKED_EVENT_FLAG_DESC;
598
0
    } else {
599
0
        e.flags = VRING_PACKED_EVENT_FLAG_ENABLE;
600
0
    }
601
602
0
    vring_packed_flags_write(vq->vdev, &caches->used, e.flags);
603
0
    if (enable) {
604
        /* Expose avail event/used flags before caller checks the avail idx. */
605
0
        smp_mb();
606
0
    }
607
0
}
608
609
bool virtio_queue_get_notification(VirtQueue *vq)
610
556k
{
611
556k
    return vq->notification;
612
556k
}
613
614
void virtio_queue_set_notification(VirtQueue *vq, int enable)
615
1.37M
{
616
1.37M
    vq->notification = enable;
617
618
1.37M
    if (!vq->vring.desc) {
619
3.34k
        return;
620
3.34k
    }
621
622
1.36M
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
623
0
        virtio_queue_packed_set_notification(vq, enable);
624
1.36M
    } else {
625
1.36M
        virtio_queue_split_set_notification(vq, enable);
626
1.36M
    }
627
1.36M
}
628
629
int virtio_queue_ready(VirtQueue *vq)
630
3.21M
{
631
3.21M
    return vq->vring.avail != 0;
632
3.21M
}
633
634
static void vring_packed_desc_read_flags(VirtIODevice *vdev,
635
                                         uint16_t *flags,
636
                                         MemoryRegionCache *cache,
637
                                         int i)
638
0
{
639
0
    hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags);
640
641
0
    *flags = virtio_lduw_phys_cached(vdev, cache, off);
642
0
}
643
644
static void vring_packed_desc_read(VirtIODevice *vdev,
645
                                   VRingPackedDesc *desc,
646
                                   MemoryRegionCache *cache,
647
                                   int i, bool strict_order)
648
0
{
649
0
    hwaddr off = i * sizeof(VRingPackedDesc);
650
651
0
    vring_packed_desc_read_flags(vdev, &desc->flags, cache, i);
652
653
0
    if (strict_order) {
654
        /* Make sure flags is read before the rest fields. */
655
0
        smp_rmb();
656
0
    }
657
658
0
    address_space_read_cached(cache, off + offsetof(VRingPackedDesc, addr),
659
0
                              &desc->addr, sizeof(desc->addr));
660
0
    address_space_read_cached(cache, off + offsetof(VRingPackedDesc, id),
661
0
                              &desc->id, sizeof(desc->id));
662
0
    address_space_read_cached(cache, off + offsetof(VRingPackedDesc, len),
663
0
                              &desc->len, sizeof(desc->len));
664
0
    virtio_tswap64s(vdev, &desc->addr);
665
0
    virtio_tswap16s(vdev, &desc->id);
666
0
    virtio_tswap32s(vdev, &desc->len);
667
0
}
668
669
static void vring_packed_desc_write_data(VirtIODevice *vdev,
670
                                         VRingPackedDesc *desc,
671
                                         MemoryRegionCache *cache,
672
                                         int i)
673
0
{
674
0
    hwaddr off_id = i * sizeof(VRingPackedDesc) +
675
0
                    offsetof(VRingPackedDesc, id);
676
0
    hwaddr off_len = i * sizeof(VRingPackedDesc) +
677
0
                    offsetof(VRingPackedDesc, len);
678
679
0
    virtio_tswap32s(vdev, &desc->len);
680
0
    virtio_tswap16s(vdev, &desc->id);
681
0
    address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id));
682
0
    address_space_cache_invalidate(cache, off_id, sizeof(desc->id));
683
0
    address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len));
684
0
    address_space_cache_invalidate(cache, off_len, sizeof(desc->len));
685
0
}
686
687
static void vring_packed_desc_write_flags(VirtIODevice *vdev,
688
                                          VRingPackedDesc *desc,
689
                                          MemoryRegionCache *cache,
690
                                          int i)
691
0
{
692
0
    hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags);
693
694
0
    virtio_stw_phys_cached(vdev, cache, off, desc->flags);
695
0
    address_space_cache_invalidate(cache, off, sizeof(desc->flags));
696
0
}
697
698
static void vring_packed_desc_write(VirtIODevice *vdev,
699
                                    VRingPackedDesc *desc,
700
                                    MemoryRegionCache *cache,
701
                                    int i, bool strict_order)
702
0
{
703
0
    vring_packed_desc_write_data(vdev, desc, cache, i);
704
0
    if (strict_order) {
705
        /* Make sure data is wrote before flags. */
706
0
        smp_wmb();
707
0
    }
708
0
    vring_packed_desc_write_flags(vdev, desc, cache, i);
709
0
}
710
711
static inline bool is_desc_avail(uint16_t flags, bool wrap_counter)
712
0
{
713
0
    bool avail, used;
714
715
0
    avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
716
0
    used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
717
0
    return (avail != used) && (avail == wrap_counter);
718
0
}
719
720
static int virtio_queue_split_empty(VirtQueue *vq)
721
1.38M
{
722
1.38M
    bool empty;
723
724
1.38M
    if (virtio_device_disabled(vq->vdev)) {
725
1.02M
        return 1;
726
1.02M
    }
727
728
356k
    if (unlikely(!vq->vring.avail)) {
729
518
        return 1;
730
518
    }
731
732
356k
    if (vq->shadow_avail_idx != vq->last_avail_idx) {
733
300k
        return 0;
734
300k
    }
735
736
356k
    RCU_READ_LOCK_GUARD();
737
56.2k
    empty = vring_avail_idx(vq) == vq->last_avail_idx;
738
56.2k
    return empty;
739
356k
}
740
741
/* Called within rcu_read_lock().  */
742
static int virtio_queue_packed_empty_rcu(VirtQueue *vq)
743
0
{
744
0
    struct VRingPackedDesc desc;
745
0
    VRingMemoryRegionCaches *cache;
746
747
0
    if (virtio_device_disabled(vq->vdev)) {
748
0
        return 1;
749
0
    }
750
751
0
    if (unlikely(!vq->vring.desc)) {
752
0
        return 1;
753
0
    }
754
755
0
    cache = vring_get_region_caches(vq);
756
0
    if (!cache) {
757
0
        return 1;
758
0
    }
759
760
0
    vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc,
761
0
                                 vq->last_avail_idx);
762
763
0
    return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter);
764
0
}
765
766
static int virtio_queue_packed_empty(VirtQueue *vq)
767
0
{
768
0
    RCU_READ_LOCK_GUARD();
769
0
    return virtio_queue_packed_empty_rcu(vq);
770
0
}
771
772
int virtio_queue_empty(VirtQueue *vq)
773
1.38M
{
774
1.38M
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
775
0
        return virtio_queue_packed_empty(vq);
776
1.38M
    } else {
777
1.38M
        return virtio_queue_split_empty(vq);
778
1.38M
    }
779
1.38M
}
780
781
static bool virtio_queue_split_poll(VirtQueue *vq, unsigned shadow_idx)
782
26.9k
{
783
26.9k
    if (unlikely(!vq->vring.avail)) {
784
0
        return false;
785
0
    }
786
787
26.9k
    return (uint16_t)shadow_idx != vring_avail_idx(vq);
788
26.9k
}
789
790
static bool virtio_queue_packed_poll(VirtQueue *vq, unsigned shadow_idx)
791
0
{
792
0
    VRingPackedDesc desc;
793
0
    VRingMemoryRegionCaches *caches;
794
795
0
    if (unlikely(!vq->vring.desc)) {
796
0
        return false;
797
0
    }
798
799
0
    caches = vring_get_region_caches(vq);
800
0
    if (!caches) {
801
0
        return false;
802
0
    }
803
804
0
    vring_packed_desc_read(vq->vdev, &desc, &caches->desc,
805
0
                           shadow_idx, true);
806
807
0
    return is_desc_avail(desc.flags, vq->shadow_avail_wrap_counter);
808
0
}
809
810
static bool virtio_queue_poll(VirtQueue *vq, unsigned shadow_idx)
811
44.7k
{
812
44.7k
    if (virtio_device_disabled(vq->vdev)) {
813
17.7k
        return false;
814
17.7k
    }
815
816
26.9k
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
817
0
        return virtio_queue_packed_poll(vq, shadow_idx);
818
26.9k
    } else {
819
26.9k
        return virtio_queue_split_poll(vq, shadow_idx);
820
26.9k
    }
821
26.9k
}
822
823
bool virtio_queue_enable_notification_and_check(VirtQueue *vq,
824
                                                int opaque)
825
48.3k
{
826
48.3k
    virtio_queue_set_notification(vq, 1);
827
828
48.3k
    if (opaque >= 0) {
829
44.7k
        return virtio_queue_poll(vq, (unsigned)opaque);
830
44.7k
    } else {
831
3.67k
        return false;
832
3.67k
    }
833
48.3k
}
834
835
static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
836
                               unsigned int len)
837
4.16M
{
838
4.16M
    AddressSpace *dma_as = vq->vdev->dma_as;
839
4.16M
    unsigned int offset;
840
4.16M
    int i;
841
842
4.16M
    offset = 0;
843
5.01M
    for (i = 0; i < elem->in_num; i++) {
844
849k
        size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
845
846
849k
        dma_memory_unmap(dma_as, elem->in_sg[i].iov_base,
847
849k
                         elem->in_sg[i].iov_len,
848
849k
                         DMA_DIRECTION_FROM_DEVICE, size);
849
850
849k
        offset += size;
851
849k
    }
852
853
8.01M
    for (i = 0; i < elem->out_num; i++)
854
3.84M
        dma_memory_unmap(dma_as, elem->out_sg[i].iov_base,
855
3.84M
                         elem->out_sg[i].iov_len,
856
3.84M
                         DMA_DIRECTION_TO_DEVICE,
857
3.84M
                         elem->out_sg[i].iov_len);
858
4.16M
}
859
860
/* virtqueue_detach_element:
861
 * @vq: The #VirtQueue
862
 * @elem: The #VirtQueueElement
863
 * @len: number of bytes written
864
 *
865
 * Detach the element from the virtqueue.  This function is suitable for device
866
 * reset or other situations where a #VirtQueueElement is simply freed and will
867
 * not be pushed or discarded.
868
 */
869
void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem,
870
                              unsigned int len)
871
178k
{
872
178k
    vq->inuse -= elem->ndescs;
873
178k
    virtqueue_unmap_sg(vq, elem, len);
874
178k
}
875
876
static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num)
877
159k
{
878
159k
    vq->last_avail_idx -= num;
879
159k
}
880
881
static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num)
882
0
{
883
0
    if (vq->last_avail_idx < num) {
884
0
        vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num;
885
0
        vq->last_avail_wrap_counter ^= 1;
886
0
    } else {
887
0
        vq->last_avail_idx -= num;
888
0
    }
889
0
}
890
891
/* virtqueue_unpop:
892
 * @vq: The #VirtQueue
893
 * @elem: The #VirtQueueElement
894
 * @len: number of bytes written
895
 *
896
 * Pretend the most recent element wasn't popped from the virtqueue.  The next
897
 * call to virtqueue_pop() will refetch the element.
898
 */
899
void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem,
900
                     unsigned int len)
901
159k
{
902
903
159k
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
904
0
        virtqueue_packed_rewind(vq, 1);
905
159k
    } else {
906
159k
        virtqueue_split_rewind(vq, 1);
907
159k
    }
908
909
159k
    virtqueue_detach_element(vq, elem, len);
910
159k
}
911
912
/* virtqueue_rewind:
913
 * @vq: The #VirtQueue
914
 * @num: Number of elements to push back
915
 *
916
 * Pretend that elements weren't popped from the virtqueue.  The next
917
 * virtqueue_pop() will refetch the oldest element.
918
 *
919
 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
920
 *
921
 * Returns: true on success, false if @num is greater than the number of in use
922
 * elements.
923
 */
924
bool virtqueue_rewind(VirtQueue *vq, unsigned int num)
925
27.0k
{
926
27.0k
    if (num > vq->inuse) {
927
27.0k
        return false;
928
27.0k
    }
929
930
0
    vq->inuse -= num;
931
0
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
932
0
        virtqueue_packed_rewind(vq, num);
933
0
    } else {
934
0
        virtqueue_split_rewind(vq, num);
935
0
    }
936
0
    return true;
937
27.0k
}
938
939
static void virtqueue_split_fill(VirtQueue *vq, const VirtQueueElement *elem,
940
                    unsigned int len, unsigned int idx)
941
3.93M
{
942
3.93M
    VRingUsedElem uelem;
943
944
3.93M
    if (unlikely(!vq->vring.used)) {
945
61.3k
        return;
946
61.3k
    }
947
948
3.86M
    idx = (idx + vq->used_idx) % vq->vring.num;
949
950
3.86M
    uelem.id = elem->index;
951
3.86M
    uelem.len = len;
952
3.86M
    vring_used_write(vq, &uelem, idx);
953
3.86M
}
954
955
static void virtqueue_packed_fill(VirtQueue *vq, const VirtQueueElement *elem,
956
                                  unsigned int len, unsigned int idx)
957
0
{
958
0
    vq->used_elems[idx].index = elem->index;
959
0
    vq->used_elems[idx].len = len;
960
0
    vq->used_elems[idx].ndescs = elem->ndescs;
961
0
}
962
963
static void virtqueue_ordered_fill(VirtQueue *vq, const VirtQueueElement *elem,
964
                                   unsigned int len)
965
0
{
966
0
    unsigned int i, steps, max_steps, ndescs;
967
968
0
    i = vq->used_idx % vq->vring.num;
969
0
    steps = 0;
970
    /*
971
     * We shouldn't need to increase 'i' by more than or equal to
972
     * the distance between used_idx and last_avail_idx (max_steps).
973
     */
974
0
    max_steps = MIN(vq->last_avail_idx - vq->used_idx, vq->vring.num);
975
976
    /* Search for element in vq->used_elems */
977
0
    while (steps < max_steps) {
978
        /* Found element, set length and mark as filled */
979
0
        if (vq->used_elems[i].index == elem->index) {
980
0
            vq->used_elems[i].len = len;
981
0
            vq->used_elems[i].in_order_filled = true;
982
0
            break;
983
0
        }
984
985
0
        ndescs = vq->used_elems[i].ndescs;
986
987
        /* Defensive sanity check */
988
0
        if (unlikely(ndescs == 0 || ndescs > vq->vring.num)) {
989
0
            qemu_log_mask(LOG_GUEST_ERROR,
990
0
                          "%s: %s invalid ndescs %u at position %u\n",
991
0
                          __func__, vq->vdev->name, ndescs, i);
992
0
            return;
993
0
        }
994
995
0
        i += ndescs;
996
0
        steps += ndescs;
997
998
0
        if (i >= vq->vring.num) {
999
0
            i -= vq->vring.num;
1000
0
        }
1001
0
    }
1002
1003
    /*
1004
     * We should be able to find a matching VirtQueueElement in
1005
     * used_elems. If we don't, this is an error.
1006
     */
1007
0
    if (steps >= max_steps) {
1008
0
        qemu_log_mask(LOG_GUEST_ERROR, "%s: %s cannot fill buffer id %u\n",
1009
0
                      __func__, vq->vdev->name, elem->index);
1010
0
    }
1011
0
}
1012
1013
static void virtqueue_packed_fill_desc(VirtQueue *vq,
1014
                                       const VirtQueueElement *elem,
1015
                                       unsigned int idx,
1016
                                       bool strict_order)
1017
0
{
1018
0
    uint16_t head;
1019
0
    VRingMemoryRegionCaches *caches;
1020
0
    VRingPackedDesc desc = {
1021
0
        .id = elem->index,
1022
0
        .len = elem->len,
1023
0
    };
1024
0
    bool wrap_counter = vq->used_wrap_counter;
1025
1026
0
    if (unlikely(!vq->vring.desc)) {
1027
0
        return;
1028
0
    }
1029
1030
0
    head = vq->used_idx + idx;
1031
0
    if (head >= vq->vring.num) {
1032
0
        head -= vq->vring.num;
1033
0
        wrap_counter ^= 1;
1034
0
    }
1035
0
    if (wrap_counter) {
1036
0
        desc.flags |= (1 << VRING_PACKED_DESC_F_AVAIL);
1037
0
        desc.flags |= (1 << VRING_PACKED_DESC_F_USED);
1038
0
    } else {
1039
0
        desc.flags &= ~(1 << VRING_PACKED_DESC_F_AVAIL);
1040
0
        desc.flags &= ~(1 << VRING_PACKED_DESC_F_USED);
1041
0
    }
1042
1043
0
    caches = vring_get_region_caches(vq);
1044
0
    if (!caches) {
1045
0
        return;
1046
0
    }
1047
1048
0
    vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order);
1049
0
}
1050
1051
/* Called within rcu_read_lock().  */
1052
void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
1053
                    unsigned int len, unsigned int idx)
1054
3.99M
{
1055
3.99M
    trace_virtqueue_fill(vq, elem, len, idx);
1056
1057
3.99M
    virtqueue_unmap_sg(vq, elem, len);
1058
1059
3.99M
    if (virtio_device_disabled(vq->vdev)) {
1060
60.7k
        return;
1061
60.7k
    }
1062
1063
3.93M
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) {
1064
0
        virtqueue_ordered_fill(vq, elem, len);
1065
3.93M
    } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1066
0
        virtqueue_packed_fill(vq, elem, len, idx);
1067
3.93M
    } else {
1068
3.93M
        virtqueue_split_fill(vq, elem, len, idx);
1069
3.93M
    }
1070
3.93M
}
1071
1072
/* Called within rcu_read_lock().  */
1073
static void virtqueue_split_flush(VirtQueue *vq, unsigned int count)
1074
3.92M
{
1075
3.92M
    uint16_t old, new;
1076
1077
3.92M
    if (unlikely(!vq->vring.used)) {
1078
61.3k
        return;
1079
61.3k
    }
1080
1081
    /* Make sure buffer is written before we update index. */
1082
3.86M
    smp_wmb();
1083
3.86M
    trace_virtqueue_flush(vq, count);
1084
3.86M
    old = vq->used_idx;
1085
3.86M
    new = old + count;
1086
3.86M
    vring_used_idx_set(vq, new);
1087
3.86M
    vq->inuse -= count;
1088
3.86M
    if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
1089
0
        vq->signalled_used_valid = false;
1090
3.86M
}
1091
1092
static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count)
1093
0
{
1094
0
    unsigned int i, ndescs = 0;
1095
1096
0
    if (unlikely(!vq->vring.desc)) {
1097
0
        return;
1098
0
    }
1099
1100
    /*
1101
     * For indirect element's 'ndescs' is 1.
1102
     * For all other elemment's 'ndescs' is the
1103
     * number of descriptors chained by NEXT (as set in virtqueue_packed_pop).
1104
     * So When the 'elem' be filled into the descriptor ring,
1105
     * The 'idx' of this 'elem' shall be
1106
     * the value of 'vq->used_idx' plus the 'ndescs'.
1107
     */
1108
0
    ndescs += vq->used_elems[0].ndescs;
1109
0
    for (i = 1; i < count; i++) {
1110
0
        virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false);
1111
0
        ndescs += vq->used_elems[i].ndescs;
1112
0
    }
1113
0
    virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true);
1114
1115
0
    vq->inuse -= ndescs;
1116
0
    vq->used_idx += ndescs;
1117
0
    if (vq->used_idx >= vq->vring.num) {
1118
0
        vq->used_idx -= vq->vring.num;
1119
0
        vq->used_wrap_counter ^= 1;
1120
0
        vq->signalled_used_valid = false;
1121
0
    }
1122
0
}
1123
1124
static void virtqueue_ordered_flush(VirtQueue *vq)
1125
0
{
1126
0
    unsigned int i = vq->used_idx % vq->vring.num;
1127
0
    unsigned int ndescs = 0;
1128
0
    uint16_t old = vq->used_idx;
1129
0
    uint16_t new;
1130
0
    bool packed;
1131
0
    VRingUsedElem uelem;
1132
1133
0
    packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED);
1134
1135
0
    if (packed) {
1136
0
        if (unlikely(!vq->vring.desc)) {
1137
0
            return;
1138
0
        }
1139
0
    } else if (unlikely(!vq->vring.used)) {
1140
0
        return;
1141
0
    }
1142
1143
    /* First expected in-order element isn't ready, nothing to do */
1144
0
    if (!vq->used_elems[i].in_order_filled) {
1145
0
        return;
1146
0
    }
1147
1148
    /* Search for filled elements in-order */
1149
0
    while (vq->used_elems[i].in_order_filled) {
1150
        /*
1151
         * First entry for packed VQs is written last so the guest
1152
         * doesn't see invalid descriptors.
1153
         */
1154
0
        if (packed && i != vq->used_idx) {
1155
0
            virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false);
1156
0
        } else if (!packed) {
1157
0
            uelem.id = vq->used_elems[i].index;
1158
0
            uelem.len = vq->used_elems[i].len;
1159
0
            vring_used_write(vq, &uelem, i);
1160
0
        }
1161
1162
0
        vq->used_elems[i].in_order_filled = false;
1163
0
        ndescs += vq->used_elems[i].ndescs;
1164
0
        i += vq->used_elems[i].ndescs;
1165
0
        if (i >= vq->vring.num) {
1166
0
            i -= vq->vring.num;
1167
0
        }
1168
0
    }
1169
1170
0
    if (packed) {
1171
0
        virtqueue_packed_fill_desc(vq, &vq->used_elems[vq->used_idx], 0, true);
1172
0
        vq->used_idx += ndescs;
1173
0
        if (vq->used_idx >= vq->vring.num) {
1174
0
            vq->used_idx -= vq->vring.num;
1175
0
            vq->used_wrap_counter ^= 1;
1176
0
            vq->signalled_used_valid = false;
1177
0
        }
1178
0
    } else {
1179
        /* Make sure buffer is written before we update index. */
1180
0
        smp_wmb();
1181
0
        new = old + ndescs;
1182
0
        vring_used_idx_set(vq, new);
1183
0
        if (unlikely((int16_t)(new - vq->signalled_used) <
1184
0
                     (uint16_t)(new - old))) {
1185
0
            vq->signalled_used_valid = false;
1186
0
        }
1187
0
    }
1188
0
    vq->inuse -= ndescs;
1189
0
}
1190
1191
void virtqueue_flush(VirtQueue *vq, unsigned int count)
1192
3.98M
{
1193
3.98M
    if (virtio_device_disabled(vq->vdev)) {
1194
61.0k
        vq->inuse -= count;
1195
61.0k
        return;
1196
61.0k
    }
1197
1198
3.92M
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) {
1199
0
        virtqueue_ordered_flush(vq);
1200
3.92M
    } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1201
0
        virtqueue_packed_flush(vq, count);
1202
3.92M
    } else {
1203
3.92M
        virtqueue_split_flush(vq, count);
1204
3.92M
    }
1205
3.92M
}
1206
1207
void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
1208
                    unsigned int len)
1209
3.94M
{
1210
3.94M
    RCU_READ_LOCK_GUARD();
1211
3.94M
    virtqueue_fill(vq, elem, len, 0);
1212
3.94M
    virtqueue_flush(vq, 1);
1213
3.94M
}
1214
1215
/* Called within rcu_read_lock().  */
1216
static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
1217
9.11M
{
1218
9.11M
    uint16_t avail_idx, num_heads;
1219
1220
    /* Use shadow index whenever possible. */
1221
9.11M
    avail_idx = (vq->shadow_avail_idx != idx) ? vq->shadow_avail_idx
1222
9.11M
                                              : vring_avail_idx(vq);
1223
9.11M
    num_heads = avail_idx - idx;
1224
1225
    /* Check it isn't doing very strange things with descriptor numbers. */
1226
9.11M
    if (num_heads > vq->vring.num) {
1227
28.5k
        virtio_error(vq->vdev, "Guest moved used index from %u to %u",
1228
28.5k
                     idx, vq->shadow_avail_idx);
1229
28.5k
        return -EINVAL;
1230
28.5k
    }
1231
    /*
1232
     * On success, callers read a descriptor at vq->last_avail_idx.
1233
     * Make sure descriptor read does not bypass avail index read.
1234
     *
1235
     * This is necessary even if we are using a shadow index, since
1236
     * the shadow index could have been initialized by calling
1237
     * vring_avail_idx() outside of this function, i.e., by a guest
1238
     * memory read not accompanied by a barrier.
1239
     */
1240
9.08M
    if (num_heads) {
1241
8.93M
        smp_rmb();
1242
8.93M
    }
1243
1244
9.08M
    return num_heads;
1245
9.11M
}
1246
1247
/* Called within rcu_read_lock().  */
1248
static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx,
1249
                               unsigned int *head)
1250
8.93M
{
1251
    /* Grab the next descriptor number they're advertising, and increment
1252
     * the index we've seen. */
1253
8.93M
    *head = vring_avail_ring(vq, idx % vq->vring.num);
1254
1255
    /* If their number is silly, that's a fatal mistake. */
1256
8.93M
    if (*head >= vq->vring.num) {
1257
30.3k
        virtio_error(vq->vdev, "Guest says index %u is available", *head);
1258
30.3k
        return false;
1259
30.3k
    }
1260
1261
8.89M
    return true;
1262
8.93M
}
1263
1264
enum {
1265
    VIRTQUEUE_READ_DESC_ERROR = -1,
1266
    VIRTQUEUE_READ_DESC_DONE = 0,   /* end of chain */
1267
    VIRTQUEUE_READ_DESC_MORE = 1,   /* more buffers in chain */
1268
};
1269
1270
/* Reads the 'desc->next' descriptor into '*desc'. */
1271
static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
1272
                                          MemoryRegionCache *desc_cache,
1273
                                          unsigned int max)
1274
91.4M
{
1275
    /* If this descriptor says it doesn't chain, we're done. */
1276
91.4M
    if (!(desc->flags & VRING_DESC_F_NEXT)) {
1277
7.83M
        return VIRTQUEUE_READ_DESC_DONE;
1278
7.83M
    }
1279
1280
    /* Check they're not leading us off end of descriptors. */
1281
83.5M
    if (desc->next >= max) {
1282
3.97k
        virtio_error(vdev, "Desc next is %u", desc->next);
1283
3.97k
        return VIRTQUEUE_READ_DESC_ERROR;
1284
3.97k
    }
1285
1286
83.5M
    vring_split_desc_read(vdev, desc, desc_cache, desc->next);
1287
83.5M
    return VIRTQUEUE_READ_DESC_MORE;
1288
83.5M
}
1289
1290
/* Called within rcu_read_lock().  */
1291
static void virtqueue_split_get_avail_bytes(VirtQueue *vq,
1292
                            unsigned int *in_bytes, unsigned int *out_bytes,
1293
                            unsigned max_in_bytes, unsigned max_out_bytes,
1294
                            VRingMemoryRegionCaches *caches)
1295
1.02M
{
1296
1.02M
    VirtIODevice *vdev = vq->vdev;
1297
1.02M
    unsigned int idx;
1298
1.02M
    unsigned int total_bufs, in_total, out_total;
1299
1.02M
    MemoryRegionCache indirect_desc_cache;
1300
1.02M
    int64_t len = 0;
1301
1.02M
    int rc;
1302
1303
1.02M
    address_space_cache_init_empty(&indirect_desc_cache);
1304
1305
1.02M
    idx = vq->last_avail_idx;
1306
1.02M
    total_bufs = in_total = out_total = 0;
1307
1308
4.68M
    while ((rc = virtqueue_num_heads(vq, idx)) > 0) {
1309
4.64M
        MemoryRegionCache *desc_cache = &caches->desc;
1310
4.64M
        unsigned int num_bufs;
1311
4.64M
        VRingDesc desc;
1312
4.64M
        unsigned int i;
1313
4.64M
        unsigned int max = vq->vring.num;
1314
1315
4.64M
        num_bufs = total_bufs;
1316
1317
4.64M
        if (!virtqueue_get_head(vq, idx++, &i)) {
1318
3.80k
            goto err;
1319
3.80k
        }
1320
1321
4.64M
        vring_split_desc_read(vdev, &desc, desc_cache, i);
1322
1323
4.64M
        if (desc.flags & VRING_DESC_F_INDIRECT) {
1324
101k
            if (!desc.len || (desc.len % sizeof(VRingDesc))) {
1325
1.53k
                virtio_error(vdev, "Invalid size for indirect buffer table");
1326
1.53k
                goto err;
1327
1.53k
            }
1328
1329
            /* If we've got too many, that implies a descriptor loop. */
1330
100k
            if (num_bufs >= max) {
1331
202
                virtio_error(vdev, "Looped descriptor");
1332
202
                goto err;
1333
202
            }
1334
1335
            /* loop over the indirect descriptor table */
1336
99.9k
            len = address_space_cache_init(&indirect_desc_cache,
1337
99.9k
                                           vdev->dma_as,
1338
99.9k
                                           desc.addr, desc.len, false);
1339
99.9k
            desc_cache = &indirect_desc_cache;
1340
99.9k
            if (len < desc.len) {
1341
253
                virtio_error(vdev, "Cannot map indirect buffer");
1342
253
                goto err;
1343
253
            }
1344
1345
99.6k
            max = desc.len / sizeof(VRingDesc);
1346
99.6k
            num_bufs = i = 0;
1347
99.6k
            vring_split_desc_read(vdev, &desc, desc_cache, i);
1348
99.6k
        }
1349
1350
82.2M
        do {
1351
            /* If we've got too many, that implies a descriptor loop. */
1352
82.2M
            if (++num_bufs > max) {
1353
422
                virtio_error(vdev, "Looped descriptor");
1354
422
                goto err;
1355
422
            }
1356
1357
82.2M
            if (desc.flags & VRING_DESC_F_WRITE) {
1358
3.37M
                in_total += desc.len;
1359
78.8M
            } else {
1360
78.8M
                out_total += desc.len;
1361
78.8M
            }
1362
82.2M
            if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1363
977k
                goto done;
1364
977k
            }
1365
1366
81.2M
            rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max);
1367
81.2M
        } while (rc == VIRTQUEUE_READ_DESC_MORE);
1368
1369
3.66M
        if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1370
611
            goto err;
1371
611
        }
1372
1373
3.66M
        if (desc_cache == &indirect_desc_cache) {
1374
70.3k
            address_space_cache_destroy(&indirect_desc_cache);
1375
70.3k
            total_bufs++;
1376
3.59M
        } else {
1377
3.59M
            total_bufs = num_bufs;
1378
3.59M
        }
1379
3.66M
    }
1380
1381
37.8k
    if (rc < 0) {
1382
6.30k
        goto err;
1383
6.30k
    }
1384
1385
1.02M
done:
1386
1.02M
    address_space_cache_destroy(&indirect_desc_cache);
1387
1.02M
    if (in_bytes) {
1388
1.02M
        *in_bytes = in_total;
1389
1.02M
    }
1390
1.02M
    if (out_bytes) {
1391
0
        *out_bytes = out_total;
1392
0
    }
1393
1.02M
    return;
1394
1395
13.1k
err:
1396
13.1k
    in_total = out_total = 0;
1397
13.1k
    goto done;
1398
37.8k
}
1399
1400
static int virtqueue_packed_read_next_desc(VirtQueue *vq,
1401
                                           VRingPackedDesc *desc,
1402
                                           MemoryRegionCache
1403
                                           *desc_cache,
1404
                                           unsigned int max,
1405
                                           unsigned int *next,
1406
                                           bool indirect)
1407
0
{
1408
    /* If this descriptor says it doesn't chain, we're done. */
1409
0
    if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) {
1410
0
        return VIRTQUEUE_READ_DESC_DONE;
1411
0
    }
1412
1413
0
    ++*next;
1414
0
    if (*next == max) {
1415
0
        if (indirect) {
1416
0
            return VIRTQUEUE_READ_DESC_DONE;
1417
0
        } else {
1418
0
            (*next) -= vq->vring.num;
1419
0
        }
1420
0
    }
1421
1422
0
    vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false);
1423
0
    return VIRTQUEUE_READ_DESC_MORE;
1424
0
}
1425
1426
/* Called within rcu_read_lock().  */
1427
static void virtqueue_packed_get_avail_bytes(VirtQueue *vq,
1428
                                             unsigned int *in_bytes,
1429
                                             unsigned int *out_bytes,
1430
                                             unsigned max_in_bytes,
1431
                                             unsigned max_out_bytes,
1432
                                             VRingMemoryRegionCaches *caches)
1433
0
{
1434
0
    VirtIODevice *vdev = vq->vdev;
1435
0
    unsigned int idx;
1436
0
    unsigned int total_bufs, in_total, out_total;
1437
0
    MemoryRegionCache indirect_desc_cache;
1438
0
    MemoryRegionCache *desc_cache;
1439
0
    int64_t len = 0;
1440
0
    VRingPackedDesc desc;
1441
0
    bool wrap_counter;
1442
1443
0
    address_space_cache_init_empty(&indirect_desc_cache);
1444
1445
0
    idx = vq->last_avail_idx;
1446
0
    wrap_counter = vq->last_avail_wrap_counter;
1447
0
    total_bufs = in_total = out_total = 0;
1448
1449
0
    for (;;) {
1450
0
        unsigned int num_bufs = total_bufs;
1451
0
        unsigned int i = idx;
1452
0
        int rc;
1453
0
        unsigned int max = vq->vring.num;
1454
1455
0
        desc_cache = &caches->desc;
1456
1457
0
        vring_packed_desc_read(vdev, &desc, desc_cache, idx, true);
1458
0
        if (!is_desc_avail(desc.flags, wrap_counter)) {
1459
0
            break;
1460
0
        }
1461
1462
0
        if (desc.flags & VRING_DESC_F_INDIRECT) {
1463
0
            if (!desc.len || (desc.len % sizeof(VRingPackedDesc))) {
1464
0
                virtio_error(vdev, "Invalid size for indirect buffer table");
1465
0
                goto err;
1466
0
            }
1467
1468
            /* If we've got too many, that implies a descriptor loop. */
1469
0
            if (num_bufs >= max) {
1470
0
                virtio_error(vdev, "Looped descriptor");
1471
0
                goto err;
1472
0
            }
1473
1474
            /* loop over the indirect descriptor table */
1475
0
            len = address_space_cache_init(&indirect_desc_cache,
1476
0
                                           vdev->dma_as,
1477
0
                                           desc.addr, desc.len, false);
1478
0
            desc_cache = &indirect_desc_cache;
1479
0
            if (len < desc.len) {
1480
0
                virtio_error(vdev, "Cannot map indirect buffer");
1481
0
                goto err;
1482
0
            }
1483
1484
0
            max = desc.len / sizeof(VRingPackedDesc);
1485
0
            num_bufs = i = 0;
1486
0
            vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1487
0
        }
1488
1489
0
        do {
1490
            /* If we've got too many, that implies a descriptor loop. */
1491
0
            if (++num_bufs > max) {
1492
0
                virtio_error(vdev, "Looped descriptor");
1493
0
                goto err;
1494
0
            }
1495
1496
0
            if (desc.flags & VRING_DESC_F_WRITE) {
1497
0
                in_total += desc.len;
1498
0
            } else {
1499
0
                out_total += desc.len;
1500
0
            }
1501
0
            if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1502
0
                goto done;
1503
0
            }
1504
1505
0
            rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max,
1506
0
                                                 &i, desc_cache ==
1507
0
                                                 &indirect_desc_cache);
1508
0
        } while (rc == VIRTQUEUE_READ_DESC_MORE);
1509
1510
0
        if (desc_cache == &indirect_desc_cache) {
1511
0
            address_space_cache_destroy(&indirect_desc_cache);
1512
0
            total_bufs++;
1513
0
            idx++;
1514
0
        } else {
1515
0
            idx += num_bufs - total_bufs;
1516
0
            total_bufs = num_bufs;
1517
0
        }
1518
1519
0
        if (idx >= vq->vring.num) {
1520
0
            idx -= vq->vring.num;
1521
0
            wrap_counter ^= 1;
1522
0
        }
1523
0
    }
1524
1525
    /* Record the index and wrap counter for a kick we want */
1526
0
    vq->shadow_avail_idx = idx;
1527
0
    vq->shadow_avail_wrap_counter = wrap_counter;
1528
0
done:
1529
0
    address_space_cache_destroy(&indirect_desc_cache);
1530
0
    if (in_bytes) {
1531
0
        *in_bytes = in_total;
1532
0
    }
1533
0
    if (out_bytes) {
1534
0
        *out_bytes = out_total;
1535
0
    }
1536
0
    return;
1537
1538
0
err:
1539
0
    in_total = out_total = 0;
1540
0
    goto done;
1541
0
}
1542
1543
int virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
1544
                              unsigned int *out_bytes, unsigned max_in_bytes,
1545
                              unsigned max_out_bytes)
1546
1.02M
{
1547
1.02M
    uint16_t desc_size;
1548
1.02M
    VRingMemoryRegionCaches *caches;
1549
1550
1.02M
    RCU_READ_LOCK_GUARD();
1551
1552
1.02M
    if (unlikely(!vq->vring.desc)) {
1553
2.51k
        goto err;
1554
2.51k
    }
1555
1556
1.02M
    caches = vring_get_region_caches(vq);
1557
1.02M
    if (!caches) {
1558
1.15k
        goto err;
1559
1.15k
    }
1560
1561
1.02M
    desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
1562
1.02M
                                sizeof(VRingPackedDesc) : sizeof(VRingDesc);
1563
1.02M
    if (caches->desc.len < vq->vring.num * desc_size) {
1564
0
        virtio_error(vq->vdev, "Cannot map descriptor ring");
1565
0
        goto err;
1566
0
    }
1567
1568
1.02M
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1569
0
        virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes,
1570
0
                                         max_in_bytes, max_out_bytes,
1571
0
                                         caches);
1572
1.02M
    } else {
1573
1.02M
        virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes,
1574
1.02M
                                        max_in_bytes, max_out_bytes,
1575
1.02M
                                        caches);
1576
1.02M
    }
1577
1578
1.02M
    return (int)vq->shadow_avail_idx;
1579
3.67k
err:
1580
3.67k
    if (in_bytes) {
1581
3.67k
        *in_bytes = 0;
1582
3.67k
    }
1583
3.67k
    if (out_bytes) {
1584
0
        *out_bytes = 0;
1585
0
    }
1586
1587
3.67k
    return -1;
1588
1.02M
}
1589
1590
int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
1591
                          unsigned int out_bytes)
1592
0
{
1593
0
    unsigned int in_total, out_total;
1594
1595
0
    virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
1596
0
    return in_bytes <= in_total && out_bytes <= out_total;
1597
0
}
1598
1599
static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg,
1600
                               hwaddr *addr, struct iovec *iov,
1601
                               unsigned int max_num_sg, bool is_write,
1602
                               hwaddr pa, size_t sz)
1603
10.2M
{
1604
10.2M
    bool ok = false;
1605
10.2M
    unsigned num_sg = *p_num_sg;
1606
10.2M
    assert(num_sg <= max_num_sg);
1607
1608
10.2M
    if (!sz) {
1609
19.9k
        virtio_error(vdev, "virtio: zero sized buffers are not allowed");
1610
19.9k
        goto out;
1611
19.9k
    }
1612
1613
23.0M
    while (sz) {
1614
12.8M
        hwaddr len = sz;
1615
1616
12.8M
        if (num_sg == max_num_sg) {
1617
6.78k
            virtio_error(vdev, "virtio: too many write descriptors in "
1618
6.78k
                               "indirect table");
1619
6.78k
            goto out;
1620
6.78k
        }
1621
1622
12.8M
        iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len,
1623
12.8M
                                              is_write ?
1624
2.75M
                                              DMA_DIRECTION_FROM_DEVICE :
1625
12.8M
                                              DMA_DIRECTION_TO_DEVICE,
1626
12.8M
                                              MEMTXATTRS_UNSPECIFIED);
1627
12.8M
        if (!iov[num_sg].iov_base) {
1628
23.7k
            virtio_error(vdev, "virtio: bogus descriptor or out of resources");
1629
23.7k
            goto out;
1630
23.7k
        }
1631
1632
12.8M
        iov[num_sg].iov_len = len;
1633
12.8M
        addr[num_sg] = pa;
1634
1635
12.8M
        sz -= len;
1636
12.8M
        pa += len;
1637
12.8M
        num_sg++;
1638
12.8M
    }
1639
10.2M
    ok = true;
1640
1641
10.2M
out:
1642
10.2M
    *p_num_sg = num_sg;
1643
10.2M
    return ok;
1644
10.2M
}
1645
1646
/* Only used by error code paths before we have a VirtQueueElement (therefore
1647
 * virtqueue_unmap_sg() can't be used).  Assumes buffers weren't written to
1648
 * yet.
1649
 */
1650
static void virtqueue_undo_map_desc(AddressSpace *as,
1651
                                    unsigned int out_num, unsigned int in_num,
1652
                                    struct iovec *iov)
1653
60.8k
{
1654
60.8k
    unsigned int i;
1655
1656
8.21M
    for (i = 0; i < out_num + in_num; i++) {
1657
8.15M
        int is_write = i >= out_num;
1658
1659
8.15M
        address_space_unmap(as, iov->iov_base, iov->iov_len, is_write, 0);
1660
8.15M
        iov++;
1661
8.15M
    }
1662
60.8k
}
1663
1664
static bool virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg,
1665
                                hwaddr *addr, unsigned int num_sg,
1666
                                bool is_write)
1667
0
{
1668
0
    unsigned int i;
1669
0
    hwaddr len;
1670
0
    DMADirection dir = is_write ? DMA_DIRECTION_FROM_DEVICE :
1671
0
                                 DMA_DIRECTION_TO_DEVICE;
1672
1673
0
    for (i = 0; i < num_sg; i++) {
1674
0
        len = sg[i].iov_len;
1675
0
        sg[i].iov_base = dma_memory_map(vdev->dma_as, addr[i], &len,
1676
0
                                        dir, MEMTXATTRS_UNSPECIFIED);
1677
0
        if (!sg[i].iov_base) {
1678
0
            error_report("virtio: error trying to map MMIO memory");
1679
0
            goto err_undo_map;
1680
0
        }
1681
0
        if (len != sg[i].iov_len) {
1682
0
            error_report("virtio: unexpected memory split");
1683
0
            dma_memory_unmap(vdev->dma_as, sg[i].iov_base, len, dir, 0);
1684
0
            goto err_undo_map;
1685
0
        }
1686
0
    }
1687
0
    return true;
1688
1689
0
err_undo_map:
1690
0
    while (i-- > 0) {
1691
0
        dma_memory_unmap(vdev->dma_as, sg[i].iov_base, sg[i].iov_len,
1692
0
                         dir, 0);
1693
0
    }
1694
0
    return false;
1695
0
}
1696
1697
bool virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem)
1698
0
{
1699
0
    if (!virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr,
1700
0
                             elem->in_num, true)) {
1701
0
        return false;
1702
0
    }
1703
0
    if (!virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr,
1704
0
                             elem->out_num, false)) {
1705
0
        unsigned int i;
1706
0
        for (i = 0; i < elem->in_num; i++) {
1707
0
            dma_memory_unmap(vdev->dma_as, elem->in_sg[i].iov_base,
1708
0
                             elem->in_sg[i].iov_len,
1709
0
                             DMA_DIRECTION_FROM_DEVICE, 0);
1710
0
        }
1711
0
        return false;
1712
0
    }
1713
0
    return true;
1714
0
}
1715
1716
static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
1717
4.16M
{
1718
4.16M
    VirtQueueElement *elem;
1719
4.16M
    size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
1720
4.16M
    size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
1721
4.16M
    size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
1722
4.16M
    size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
1723
4.16M
    size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
1724
4.16M
    size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
1725
1726
4.16M
    assert(sz >= sizeof(VirtQueueElement));
1727
4.16M
    elem = g_malloc(out_sg_end);
1728
4.16M
    trace_virtqueue_alloc_element(elem, sz, in_num, out_num);
1729
4.16M
    elem->out_num = out_num;
1730
4.16M
    elem->in_num = in_num;
1731
4.16M
    elem->in_addr = (void *)elem + in_addr_ofs;
1732
4.16M
    elem->out_addr = (void *)elem + out_addr_ofs;
1733
4.16M
    elem->in_sg = (void *)elem + in_sg_ofs;
1734
4.16M
    elem->out_sg = (void *)elem + out_sg_ofs;
1735
4.16M
    return elem;
1736
4.16M
}
1737
1738
static void *virtqueue_split_pop(VirtQueue *vq, size_t sz)
1739
4.43M
{
1740
4.43M
    unsigned int i, head, max, idx;
1741
4.43M
    VRingMemoryRegionCaches *caches;
1742
4.43M
    MemoryRegionCache indirect_desc_cache;
1743
4.43M
    MemoryRegionCache *desc_cache;
1744
4.43M
    int64_t len;
1745
4.43M
    VirtIODevice *vdev = vq->vdev;
1746
4.43M
    VirtQueueElement *elem = NULL;
1747
4.43M
    unsigned out_num, in_num, elem_entries;
1748
4.43M
    hwaddr QEMU_UNINITIALIZED addr[VIRTQUEUE_MAX_SIZE];
1749
4.43M
    struct iovec QEMU_UNINITIALIZED iov[VIRTQUEUE_MAX_SIZE];
1750
4.43M
    VRingDesc desc;
1751
4.43M
    int rc;
1752
1753
4.43M
    address_space_cache_init_empty(&indirect_desc_cache);
1754
1755
4.43M
    RCU_READ_LOCK_GUARD();
1756
4.43M
    if (unlikely(!vq->vring.avail)) {
1757
2.98k
        goto done;
1758
2.98k
    }
1759
1760
4.43M
    rc = virtqueue_num_heads(vq, vq->last_avail_idx);
1761
4.43M
    if (rc <= 0) {
1762
148k
        goto done;
1763
148k
    }
1764
1765
    /* When we start there are none of either input nor output. */
1766
4.28M
    out_num = in_num = elem_entries = 0;
1767
1768
4.28M
    max = vq->vring.num;
1769
1770
4.28M
    if (vq->inuse >= vq->vring.num) {
1771
20
        virtio_error(vdev, "Virtqueue size exceeded");
1772
20
        goto done;
1773
20
    }
1774
1775
4.28M
    if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) {
1776
26.5k
        goto done;
1777
26.5k
    }
1778
1779
4.25M
    if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1780
888k
        vring_set_avail_event(vq, vq->last_avail_idx);
1781
888k
    }
1782
1783
4.25M
    i = head;
1784
1785
4.25M
    caches = vring_get_region_caches(vq);
1786
4.25M
    if (!caches) {
1787
116
        virtio_error(vdev, "Region caches not initialized");
1788
116
        goto done;
1789
116
    }
1790
1791
4.25M
    if (caches->desc.len < max * sizeof(VRingDesc)) {
1792
0
        virtio_error(vdev, "Cannot map descriptor ring");
1793
0
        goto done;
1794
0
    }
1795
1796
4.25M
    desc_cache = &caches->desc;
1797
4.25M
    vring_split_desc_read(vdev, &desc, desc_cache, i);
1798
4.25M
    if (desc.flags & VRING_DESC_F_INDIRECT) {
1799
207k
        if (!desc.len || (desc.len % sizeof(VRingDesc))) {
1800
21.3k
            virtio_error(vdev, "Invalid size for indirect buffer table");
1801
21.3k
            goto done;
1802
21.3k
        }
1803
186k
        virtio_check_indirect_feature(vdev);
1804
1805
        /* loop over the indirect descriptor table */
1806
186k
        len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1807
186k
                                       desc.addr, desc.len, false);
1808
186k
        desc_cache = &indirect_desc_cache;
1809
186k
        if (len < desc.len) {
1810
3.20k
            virtio_error(vdev, "Cannot map indirect buffer");
1811
3.20k
            goto done;
1812
3.20k
        }
1813
1814
182k
        max = desc.len / sizeof(VRingDesc);
1815
182k
        i = 0;
1816
182k
        vring_split_desc_read(vdev, &desc, desc_cache, i);
1817
182k
    }
1818
1819
    /* Collect all the descriptors */
1820
10.2M
    do {
1821
10.2M
        bool map_ok;
1822
1823
10.2M
        if (desc.flags & VRING_DESC_F_WRITE) {
1824
2.74M
            map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1825
2.74M
                                        iov + out_num,
1826
2.74M
                                        VIRTQUEUE_MAX_SIZE - out_num, true,
1827
2.74M
                                        desc.addr, desc.len);
1828
7.51M
        } else {
1829
7.51M
            if (in_num) {
1830
2.25k
                virtio_error(vdev, "Incorrect order for descriptors");
1831
2.25k
                goto err_undo_map;
1832
2.25k
            }
1833
7.51M
            map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1834
7.51M
                                        VIRTQUEUE_MAX_SIZE, false,
1835
7.51M
                                        desc.addr, desc.len);
1836
7.51M
        }
1837
10.2M
        if (!map_ok) {
1838
50.5k
            goto err_undo_map;
1839
50.5k
        }
1840
1841
        /* If we've got too many, that implies a descriptor loop. */
1842
10.2M
        if (++elem_entries > max) {
1843
4.75k
            virtio_error(vdev, "Looped descriptor");
1844
4.75k
            goto err_undo_map;
1845
4.75k
        }
1846
1847
10.2M
        rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max);
1848
10.2M
    } while (rc == VIRTQUEUE_READ_DESC_MORE);
1849
1850
4.17M
    if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1851
3.36k
        goto err_undo_map;
1852
3.36k
    }
1853
1854
    /* Now copy what we have collected and mapped */
1855
4.16M
    elem = virtqueue_alloc_element(sz, out_num, in_num);
1856
4.16M
    elem->index = head;
1857
4.16M
    elem->ndescs = 1;
1858
8.01M
    for (i = 0; i < out_num; i++) {
1859
3.84M
        elem->out_addr[i] = addr[i];
1860
3.84M
        elem->out_sg[i] = iov[i];
1861
3.84M
    }
1862
5.01M
    for (i = 0; i < in_num; i++) {
1863
849k
        elem->in_addr[i] = addr[out_num + i];
1864
849k
        elem->in_sg[i] = iov[out_num + i];
1865
849k
    }
1866
1867
4.16M
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_IN_ORDER)) {
1868
0
        idx = (vq->last_avail_idx - 1) % vq->vring.num;
1869
0
        vq->used_elems[idx].index = elem->index;
1870
0
        vq->used_elems[idx].len = elem->len;
1871
0
        vq->used_elems[idx].ndescs = elem->ndescs;
1872
0
    }
1873
1874
4.16M
    vq->inuse++;
1875
1876
4.16M
    trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
1877
4.43M
done:
1878
4.43M
    address_space_cache_destroy(&indirect_desc_cache);
1879
1880
4.43M
    return elem;
1881
1882
60.8k
err_undo_map:
1883
60.8k
    virtqueue_undo_map_desc(vdev->dma_as, out_num, in_num, iov);
1884
60.8k
    goto done;
1885
4.16M
}
1886
1887
static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz)
1888
0
{
1889
0
    unsigned int i, max;
1890
0
    VRingMemoryRegionCaches *caches;
1891
0
    MemoryRegionCache indirect_desc_cache;
1892
0
    MemoryRegionCache *desc_cache;
1893
0
    int64_t len;
1894
0
    VirtIODevice *vdev = vq->vdev;
1895
0
    VirtQueueElement *elem = NULL;
1896
0
    unsigned out_num, in_num, elem_entries;
1897
0
    hwaddr QEMU_UNINITIALIZED addr[VIRTQUEUE_MAX_SIZE];
1898
0
    struct iovec QEMU_UNINITIALIZED iov[VIRTQUEUE_MAX_SIZE];
1899
0
    VRingPackedDesc desc;
1900
0
    uint16_t id;
1901
0
    int rc;
1902
1903
0
    address_space_cache_init_empty(&indirect_desc_cache);
1904
1905
0
    RCU_READ_LOCK_GUARD();
1906
0
    if (virtio_queue_packed_empty_rcu(vq)) {
1907
0
        goto done;
1908
0
    }
1909
1910
    /* When we start there are none of either input nor output. */
1911
0
    out_num = in_num = elem_entries = 0;
1912
1913
0
    max = vq->vring.num;
1914
1915
0
    if (vq->inuse >= vq->vring.num) {
1916
0
        virtio_error(vdev, "Virtqueue size exceeded");
1917
0
        goto done;
1918
0
    }
1919
1920
0
    i = vq->last_avail_idx;
1921
1922
0
    caches = vring_get_region_caches(vq);
1923
0
    if (!caches) {
1924
0
        virtio_error(vdev, "Region caches not initialized");
1925
0
        goto done;
1926
0
    }
1927
1928
0
    if (caches->desc.len < max * sizeof(VRingDesc)) {
1929
0
        virtio_error(vdev, "Cannot map descriptor ring");
1930
0
        goto done;
1931
0
    }
1932
1933
0
    desc_cache = &caches->desc;
1934
0
    vring_packed_desc_read(vdev, &desc, desc_cache, i, true);
1935
0
    id = desc.id;
1936
0
    if (desc.flags & VRING_DESC_F_INDIRECT) {
1937
0
        if (!desc.len || (desc.len % sizeof(VRingPackedDesc))) {
1938
0
            virtio_error(vdev, "Invalid size for indirect buffer table");
1939
0
            goto done;
1940
0
        }
1941
0
        virtio_check_indirect_feature(vdev);
1942
1943
        /* loop over the indirect descriptor table */
1944
0
        len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1945
0
                                       desc.addr, desc.len, false);
1946
0
        desc_cache = &indirect_desc_cache;
1947
0
        if (len < desc.len) {
1948
0
            virtio_error(vdev, "Cannot map indirect buffer");
1949
0
            goto done;
1950
0
        }
1951
1952
0
        max = desc.len / sizeof(VRingPackedDesc);
1953
0
        i = 0;
1954
0
        vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1955
0
    }
1956
1957
    /* Collect all the descriptors */
1958
0
    do {
1959
0
        bool map_ok;
1960
1961
0
        if (desc.flags & VRING_DESC_F_WRITE) {
1962
0
            map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1963
0
                                        iov + out_num,
1964
0
                                        VIRTQUEUE_MAX_SIZE - out_num, true,
1965
0
                                        desc.addr, desc.len);
1966
0
        } else {
1967
0
            if (in_num) {
1968
0
                virtio_error(vdev, "Incorrect order for descriptors");
1969
0
                goto err_undo_map;
1970
0
            }
1971
0
            map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1972
0
                                        VIRTQUEUE_MAX_SIZE, false,
1973
0
                                        desc.addr, desc.len);
1974
0
        }
1975
0
        if (!map_ok) {
1976
0
            goto err_undo_map;
1977
0
        }
1978
1979
        /* If we've got too many, that implies a descriptor loop. */
1980
0
        if (++elem_entries > max) {
1981
0
            virtio_error(vdev, "Looped descriptor");
1982
0
            goto err_undo_map;
1983
0
        }
1984
1985
0
        rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, &i,
1986
0
                                             desc_cache ==
1987
0
                                             &indirect_desc_cache);
1988
0
    } while (rc == VIRTQUEUE_READ_DESC_MORE);
1989
1990
0
    if (desc_cache != &indirect_desc_cache) {
1991
        /* Buffer ID is included in the last descriptor in the list. */
1992
0
        id = desc.id;
1993
0
    }
1994
1995
    /* Now copy what we have collected and mapped */
1996
0
    elem = virtqueue_alloc_element(sz, out_num, in_num);
1997
0
    for (i = 0; i < out_num; i++) {
1998
0
        elem->out_addr[i] = addr[i];
1999
0
        elem->out_sg[i] = iov[i];
2000
0
    }
2001
0
    for (i = 0; i < in_num; i++) {
2002
0
        elem->in_addr[i] = addr[out_num + i];
2003
0
        elem->in_sg[i] = iov[out_num + i];
2004
0
    }
2005
2006
0
    elem->index = id;
2007
0
    elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries;
2008
2009
0
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_IN_ORDER)) {
2010
0
        vq->used_elems[vq->last_avail_idx].index = elem->index;
2011
0
        vq->used_elems[vq->last_avail_idx].len = elem->len;
2012
0
        vq->used_elems[vq->last_avail_idx].ndescs = elem->ndescs;
2013
0
    }
2014
2015
0
    vq->last_avail_idx += elem->ndescs;
2016
0
    vq->inuse += elem->ndescs;
2017
2018
0
    if (vq->last_avail_idx >= vq->vring.num) {
2019
0
        vq->last_avail_idx -= vq->vring.num;
2020
0
        vq->last_avail_wrap_counter ^= 1;
2021
0
    }
2022
2023
0
    vq->shadow_avail_idx = vq->last_avail_idx;
2024
0
    vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter;
2025
2026
0
    trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
2027
0
done:
2028
0
    address_space_cache_destroy(&indirect_desc_cache);
2029
2030
0
    return elem;
2031
2032
0
err_undo_map:
2033
0
    virtqueue_undo_map_desc(vdev->dma_as, out_num, in_num, iov);
2034
0
    goto done;
2035
0
}
2036
2037
void *virtqueue_pop(VirtQueue *vq, size_t sz)
2038
5.42M
{
2039
5.42M
    if (virtio_device_disabled(vq->vdev)) {
2040
986k
        return NULL;
2041
986k
    }
2042
2043
4.43M
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
2044
0
        return virtqueue_packed_pop(vq, sz);
2045
4.43M
    } else {
2046
4.43M
        return virtqueue_split_pop(vq, sz);
2047
4.43M
    }
2048
4.43M
}
2049
2050
static unsigned int virtqueue_packed_drop_all(VirtQueue *vq)
2051
0
{
2052
0
    VRingMemoryRegionCaches *caches;
2053
0
    MemoryRegionCache *desc_cache;
2054
0
    unsigned int dropped = 0;
2055
0
    VirtQueueElement elem = {};
2056
0
    VirtIODevice *vdev = vq->vdev;
2057
0
    VRingPackedDesc desc;
2058
2059
0
    RCU_READ_LOCK_GUARD();
2060
2061
0
    caches = vring_get_region_caches(vq);
2062
0
    if (!caches) {
2063
0
        return 0;
2064
0
    }
2065
2066
0
    desc_cache = &caches->desc;
2067
2068
0
    virtio_queue_set_notification(vq, 0);
2069
2070
0
    while (vq->inuse < vq->vring.num) {
2071
0
        unsigned int idx = vq->last_avail_idx;
2072
        /*
2073
         * works similar to virtqueue_pop but does not map buffers
2074
         * and does not allocate any memory.
2075
         */
2076
0
        vring_packed_desc_read(vdev, &desc, desc_cache,
2077
0
                               vq->last_avail_idx , true);
2078
0
        if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) {
2079
0
            break;
2080
0
        }
2081
0
        elem.index = desc.id;
2082
0
        elem.ndescs = 1;
2083
0
        while (virtqueue_packed_read_next_desc(vq, &desc, desc_cache,
2084
0
                                               vq->vring.num, &idx, false)) {
2085
0
            ++elem.ndescs;
2086
0
        }
2087
        /*
2088
         * immediately push the element, nothing to unmap
2089
         * as both in_num and out_num are set to 0.
2090
         */
2091
0
        virtqueue_push(vq, &elem, 0);
2092
0
        dropped++;
2093
0
        vq->last_avail_idx += elem.ndescs;
2094
0
        if (vq->last_avail_idx >= vq->vring.num) {
2095
0
            vq->last_avail_idx -= vq->vring.num;
2096
0
            vq->last_avail_wrap_counter ^= 1;
2097
0
        }
2098
0
    }
2099
2100
0
    return dropped;
2101
0
}
2102
2103
static unsigned int virtqueue_split_drop_all(VirtQueue *vq)
2104
0
{
2105
0
    unsigned int dropped = 0;
2106
0
    VirtQueueElement elem = {};
2107
0
    VirtIODevice *vdev = vq->vdev;
2108
0
    bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2109
2110
0
    while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) {
2111
        /* works similar to virtqueue_pop but does not map buffers
2112
        * and does not allocate any memory */
2113
0
        smp_rmb();
2114
0
        if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) {
2115
0
            break;
2116
0
        }
2117
0
        vq->inuse++;
2118
0
        vq->last_avail_idx++;
2119
0
        if (fEventIdx) {
2120
0
            vring_set_avail_event(vq, vq->last_avail_idx);
2121
0
        }
2122
        /* immediately push the element, nothing to unmap
2123
         * as both in_num and out_num are set to 0 */
2124
0
        virtqueue_push(vq, &elem, 0);
2125
0
        dropped++;
2126
0
    }
2127
2128
0
    return dropped;
2129
0
}
2130
2131
/* virtqueue_drop_all:
2132
 * @vq: The #VirtQueue
2133
 * Drops all queued buffers and indicates them to the guest
2134
 * as if they are done. Useful when buffers can not be
2135
 * processed but must be returned to the guest.
2136
 */
2137
unsigned int virtqueue_drop_all(VirtQueue *vq)
2138
0
{
2139
0
    struct VirtIODevice *vdev = vq->vdev;
2140
2141
0
    if (virtio_device_disabled(vq->vdev)) {
2142
0
        return 0;
2143
0
    }
2144
2145
0
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2146
0
        return virtqueue_packed_drop_all(vq);
2147
0
    } else {
2148
0
        return virtqueue_split_drop_all(vq);
2149
0
    }
2150
0
}
2151
2152
/* Reading and writing a structure directly to QEMUFile is *awful*, but
2153
 * it is what QEMU has always done by mistake.  We can change it sooner
2154
 * or later by bumping the version number of the affected vm states.
2155
 * In the meanwhile, since the in-memory layout of VirtQueueElement
2156
 * has changed, we need to marshal to and from the layout that was
2157
 * used before the change.
2158
 */
2159
typedef struct VirtQueueElementOld {
2160
    unsigned int index;
2161
    unsigned int out_num;
2162
    unsigned int in_num;
2163
    hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
2164
    hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
2165
    struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
2166
    struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
2167
} VirtQueueElementOld;
2168
2169
void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz)
2170
0
{
2171
0
    VirtQueueElement *elem;
2172
0
    VirtQueueElementOld data;
2173
0
    int i;
2174
2175
0
    qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
2176
2177
    /* TODO: teach all callers that this can fail, and return failure instead
2178
     * of asserting here.
2179
     * This is just one thing (there are probably more) that must be
2180
     * fixed before we can allow NDEBUG compilation.
2181
     */
2182
0
    assert(ARRAY_SIZE(data.in_addr) >= data.in_num);
2183
0
    assert(ARRAY_SIZE(data.out_addr) >= data.out_num);
2184
2185
0
    elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
2186
0
    elem->index = data.index;
2187
2188
0
    for (i = 0; i < elem->in_num; i++) {
2189
0
        elem->in_addr[i] = data.in_addr[i];
2190
0
    }
2191
2192
0
    for (i = 0; i < elem->out_num; i++) {
2193
0
        elem->out_addr[i] = data.out_addr[i];
2194
0
    }
2195
2196
0
    for (i = 0; i < elem->in_num; i++) {
2197
        /* Base is overwritten by virtqueue_map.  */
2198
0
        elem->in_sg[i].iov_base = 0;
2199
0
        elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
2200
0
    }
2201
2202
0
    for (i = 0; i < elem->out_num; i++) {
2203
        /* Base is overwritten by virtqueue_map.  */
2204
0
        elem->out_sg[i].iov_base = 0;
2205
0
        elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
2206
0
    }
2207
2208
0
    if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2209
0
        qemu_get_be32s(f, &elem->ndescs);
2210
0
    }
2211
2212
0
    if (!virtqueue_map(vdev, elem)) {
2213
0
        g_free(elem);
2214
0
        return NULL;
2215
0
    }
2216
0
    return elem;
2217
0
}
2218
2219
void qemu_put_virtqueue_element(VirtIODevice *vdev, QEMUFile *f,
2220
                                VirtQueueElement *elem)
2221
0
{
2222
0
    VirtQueueElementOld data;
2223
0
    int i;
2224
2225
0
    memset(&data, 0, sizeof(data));
2226
0
    data.index = elem->index;
2227
0
    data.in_num = elem->in_num;
2228
0
    data.out_num = elem->out_num;
2229
2230
0
    for (i = 0; i < elem->in_num; i++) {
2231
0
        data.in_addr[i] = elem->in_addr[i];
2232
0
    }
2233
2234
0
    for (i = 0; i < elem->out_num; i++) {
2235
0
        data.out_addr[i] = elem->out_addr[i];
2236
0
    }
2237
2238
0
    for (i = 0; i < elem->in_num; i++) {
2239
        /* Base is overwritten by virtqueue_map when loading.  Do not
2240
         * save it, as it would leak the QEMU address space layout.  */
2241
0
        data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
2242
0
    }
2243
2244
0
    for (i = 0; i < elem->out_num; i++) {
2245
        /* Do not save iov_base as above.  */
2246
0
        data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
2247
0
    }
2248
2249
0
    if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2250
0
        qemu_put_be32s(f, &elem->ndescs);
2251
0
    }
2252
2253
0
    qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
2254
0
}
2255
2256
/* virtio device */
2257
static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
2258
1.31M
{
2259
1.31M
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2260
1.31M
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2261
2262
1.31M
    if (virtio_device_disabled(vdev)) {
2263
56.6k
        return;
2264
56.6k
    }
2265
2266
1.26M
    if (k->notify) {
2267
1.26M
        k->notify(qbus->parent, vector);
2268
1.26M
    }
2269
1.26M
}
2270
2271
void virtio_update_irq(VirtIODevice *vdev)
2272
0
{
2273
0
    virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
2274
0
}
2275
2276
static int virtio_validate_features(VirtIODevice *vdev)
2277
16.8k
{
2278
16.8k
    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2279
2280
16.8k
    if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) &&
2281
0
        !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
2282
0
        return -EFAULT;
2283
0
    }
2284
2285
16.8k
    if (k->validate_features) {
2286
362
        return k->validate_features(vdev);
2287
16.5k
    } else {
2288
16.5k
        return 0;
2289
16.5k
    }
2290
16.8k
}
2291
2292
int virtio_set_status(VirtIODevice *vdev, uint8_t val)
2293
1.26M
{
2294
1.26M
    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2295
1.26M
    trace_virtio_set_status(vdev, val);
2296
1.26M
    int ret = 0;
2297
2298
1.26M
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2299
90.4k
        if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
2300
65.3k
            val & VIRTIO_CONFIG_S_FEATURES_OK) {
2301
16.8k
            ret = virtio_validate_features(vdev);
2302
16.8k
            if (ret) {
2303
148
                return ret;
2304
148
            }
2305
16.8k
        }
2306
90.4k
    }
2307
2308
1.26M
    if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) !=
2309
1.26M
        (val & VIRTIO_CONFIG_S_DRIVER_OK)) {
2310
456k
        virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK);
2311
456k
    }
2312
2313
1.26M
    if (k->set_status) {
2314
596k
        ret = k->set_status(vdev, val);
2315
596k
        if (ret) {
2316
0
            qemu_log("set %s status to %d failed, old status: %d\n",
2317
0
                     vdev->name, val, vdev->status);
2318
0
        }
2319
596k
    }
2320
1.26M
    vdev->status = val;
2321
2322
1.26M
    return ret;
2323
1.26M
}
2324
2325
static enum virtio_device_endian virtio_default_endian(void)
2326
370k
{
2327
370k
    if (target_big_endian()) {
2328
0
        return VIRTIO_DEVICE_ENDIAN_BIG;
2329
370k
    } else {
2330
370k
        return VIRTIO_DEVICE_ENDIAN_LITTLE;
2331
370k
    }
2332
370k
}
2333
2334
static enum virtio_device_endian virtio_current_cpu_endian(void)
2335
0
{
2336
0
    if (cpu_internal_is_big_endian(current_cpu)) {
2337
0
        return VIRTIO_DEVICE_ENDIAN_BIG;
2338
0
    } else {
2339
0
        return VIRTIO_DEVICE_ENDIAN_LITTLE;
2340
0
    }
2341
0
}
2342
2343
static void __virtio_queue_reset(VirtIODevice *vdev, uint32_t i)
2344
379M
{
2345
379M
    vdev->vq[i].vring.desc = 0;
2346
379M
    vdev->vq[i].vring.avail = 0;
2347
379M
    vdev->vq[i].vring.used = 0;
2348
379M
    vdev->vq[i].last_avail_idx = 0;
2349
379M
    vdev->vq[i].shadow_avail_idx = 0;
2350
379M
    vdev->vq[i].used_idx = 0;
2351
379M
    vdev->vq[i].last_avail_wrap_counter = true;
2352
379M
    vdev->vq[i].shadow_avail_wrap_counter = true;
2353
379M
    vdev->vq[i].used_wrap_counter = true;
2354
379M
    virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
2355
379M
    vdev->vq[i].signalled_used = 0;
2356
379M
    vdev->vq[i].signalled_used_valid = false;
2357
379M
    vdev->vq[i].notification = true;
2358
379M
    vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
2359
379M
    vdev->vq[i].inuse = 0;
2360
379M
    virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
2361
379M
}
2362
2363
void virtio_queue_reset(VirtIODevice *vdev, uint32_t queue_index)
2364
58.5k
{
2365
58.5k
    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2366
2367
58.5k
    if (k->queue_reset) {
2368
3.92k
        k->queue_reset(vdev, queue_index);
2369
3.92k
    }
2370
2371
58.5k
    __virtio_queue_reset(vdev, queue_index);
2372
58.5k
}
2373
2374
void virtio_queue_enable(VirtIODevice *vdev, uint32_t queue_index)
2375
101k
{
2376
101k
    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2377
2378
    /*
2379
     * TODO: Seabios is currently out of spec and triggering this error.
2380
     * So this needs to be fixed in Seabios, then this can
2381
     * be re-enabled for new machine types only, and also after
2382
     * being converted to LOG_GUEST_ERROR.
2383
     *
2384
    if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2385
        error_report("queue_enable is only supported in devices of virtio "
2386
                     "1.0 or later.");
2387
    }
2388
    */
2389
2390
101k
    if (k->queue_enable) {
2391
5.12k
        k->queue_enable(vdev, queue_index);
2392
5.12k
    }
2393
101k
}
2394
2395
void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
2396
460k
{
2397
460k
    if (!vdev->vq[n].vring.num) {
2398
22.6k
        return;
2399
22.6k
    }
2400
438k
    vdev->vq[n].vring.desc = addr;
2401
438k
    virtio_queue_update_rings(vdev, n);
2402
438k
}
2403
2404
hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
2405
5.41k
{
2406
5.41k
    return vdev->vq[n].vring.desc;
2407
5.41k
}
2408
2409
void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
2410
                            hwaddr avail, hwaddr used)
2411
101k
{
2412
101k
    if (!vdev->vq[n].vring.num) {
2413
2.81k
        return;
2414
2.81k
    }
2415
98.1k
    vdev->vq[n].vring.desc = desc;
2416
98.1k
    vdev->vq[n].vring.avail = avail;
2417
98.1k
    vdev->vq[n].vring.used = used;
2418
98.1k
    virtio_init_region_cache(vdev, n);
2419
98.1k
}
2420
2421
void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
2422
505k
{
2423
    /* Don't allow guest to flip queue between existent and
2424
     * nonexistent states, or to set it to an invalid size.
2425
     */
2426
505k
    if (!!num != !!vdev->vq[n].vring.num ||
2427
393k
        num > VIRTQUEUE_MAX_SIZE ||
2428
403k
        num < 0) {
2429
403k
        return;
2430
403k
    }
2431
101k
    if (num > vdev->vq[n].vring.num_default) {
2432
32.3k
        virtio_error(vdev, "virtio: queue %d size %d exceeds max size %u",
2433
32.3k
                     n, num, vdev->vq[n].vring.num_default);
2434
32.3k
        return;
2435
32.3k
    }
2436
69.1k
    vdev->vq[n].vring.num = num;
2437
69.1k
}
2438
2439
VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
2440
0
{
2441
0
    return QLIST_FIRST(&vdev->vector_queues[vector]);
2442
0
}
2443
2444
VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
2445
0
{
2446
0
    return QLIST_NEXT(vq, node);
2447
0
}
2448
2449
int virtio_queue_get_num(VirtIODevice *vdev, int n)
2450
501M
{
2451
501M
    return vdev->vq[n].vring.num;
2452
501M
}
2453
2454
int virtio_queue_get_max_num(VirtIODevice *vdev, int n)
2455
0
{
2456
0
    return vdev->vq[n].vring.num_default;
2457
0
}
2458
2459
int virtio_get_num_queues(VirtIODevice *vdev)
2460
206k
{
2461
206k
    int i;
2462
2463
825k
    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2464
825k
        if (!virtio_queue_get_num(vdev, i)) {
2465
206k
            break;
2466
206k
        }
2467
825k
    }
2468
2469
206k
    return i;
2470
206k
}
2471
2472
void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
2473
0
{
2474
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2475
0
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2476
2477
    /* virtio-1 compliant devices cannot change the alignment */
2478
0
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2479
0
        error_report("tried to modify queue alignment for virtio-1 device");
2480
0
        return;
2481
0
    }
2482
    /* Check that the transport told us it was going to do this
2483
     * (so a buggy transport will immediately assert rather than
2484
     * silently failing to migrate this state)
2485
     */
2486
0
    assert(k->has_variable_vring_alignment);
2487
2488
0
    if (align) {
2489
0
        vdev->vq[n].vring.align = align;
2490
0
        virtio_queue_update_rings(vdev, n);
2491
0
    }
2492
0
}
2493
2494
void virtio_queue_set_shadow_avail_idx(VirtQueue *vq, uint16_t shadow_avail_idx)
2495
0
{
2496
0
    if (!vq->vring.desc) {
2497
0
        return;
2498
0
    }
2499
2500
    /*
2501
     * 16-bit data for packed VQs include 1-bit wrap counter and
2502
     * 15-bit shadow_avail_idx.
2503
     */
2504
0
    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
2505
0
        vq->shadow_avail_wrap_counter = (shadow_avail_idx >> 15) & 0x1;
2506
0
        vq->shadow_avail_idx = shadow_avail_idx & 0x7FFF;
2507
0
    } else {
2508
0
        vq->shadow_avail_idx = shadow_avail_idx;
2509
0
    }
2510
0
}
2511
2512
static void virtio_queue_notify_vq(VirtQueue *vq)
2513
1.03M
{
2514
1.03M
    if (vq->vring.desc && vq->handle_output) {
2515
216k
        VirtIODevice *vdev = vq->vdev;
2516
2517
216k
        if (unlikely(vdev->broken)) {
2518
54.4k
            return;
2519
54.4k
        }
2520
2521
162k
        trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2522
162k
        vq->handle_output(vdev, vq);
2523
2524
162k
        if (unlikely(vdev->start_on_kick)) {
2525
35
            virtio_set_started(vdev, true);
2526
35
        }
2527
162k
    }
2528
1.03M
}
2529
2530
void virtio_queue_notify(VirtIODevice *vdev, int n)
2531
544k
{
2532
544k
    VirtQueue *vq = &vdev->vq[n];
2533
2534
544k
    if (unlikely(!vq->vring.desc || vdev->broken)) {
2535
377k
        return;
2536
377k
    }
2537
2538
166k
    trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2539
166k
    if (vq->host_notifier_enabled) {
2540
5.09k
        event_notifier_set(&vq->host_notifier);
2541
161k
    } else if (vq->handle_output) {
2542
161k
        vq->handle_output(vdev, vq);
2543
2544
161k
        if (unlikely(vdev->start_on_kick)) {
2545
3.52k
            virtio_set_started(vdev, true);
2546
3.52k
        }
2547
161k
    }
2548
166k
}
2549
2550
uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
2551
240k
{
2552
240k
    return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
2553
240k
        VIRTIO_NO_VECTOR;
2554
240k
}
2555
2556
void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
2557
379M
{
2558
379M
    VirtQueue *vq = &vdev->vq[n];
2559
2560
379M
    if (n < VIRTIO_QUEUE_MAX) {
2561
379M
        if (vdev->vector_queues &&
2562
379M
            vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
2563
100k
            QLIST_REMOVE(vq, node);
2564
100k
        }
2565
379M
        vdev->vq[n].vector = vector;
2566
379M
        if (vdev->vector_queues &&
2567
379M
            vector != VIRTIO_NO_VECTOR) {
2568
100k
            QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
2569
100k
        }
2570
379M
    }
2571
379M
}
2572
2573
VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
2574
                            VirtIOHandleOutput handle_output)
2575
179
{
2576
179
    int i;
2577
2578
4.31k
    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2579
4.31k
        if (vdev->vq[i].vring.num == 0)
2580
179
            break;
2581
4.31k
    }
2582
2583
179
    if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
2584
0
        abort();
2585
2586
179
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2587
179
    if (qbus && qbus->parent &&
2588
179
        object_property_find(OBJECT(qbus->parent), VIRTIO_QUEUE_SIZE_OVERRIDE)) {
2589
0
        int override = object_property_get_int(OBJECT(qbus->parent),
2590
0
                                               VIRTIO_QUEUE_SIZE_OVERRIDE,
2591
0
                                               &error_abort);
2592
0
        if (override) {
2593
0
            queue_size = override;
2594
0
        }
2595
0
    }
2596
2597
179
    vdev->vq[i].vring.num = queue_size;
2598
179
    vdev->vq[i].vring.num_default = queue_size;
2599
179
    vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
2600
179
    vdev->vq[i].handle_output = handle_output;
2601
179
    vdev->vq[i].used_elems = g_new0(VirtQueueElement, queue_size);
2602
2603
179
    return &vdev->vq[i];
2604
179
}
2605
2606
void virtio_delete_queue(VirtQueue *vq)
2607
0
{
2608
0
    vq->vring.num = 0;
2609
0
    vq->vring.num_default = 0;
2610
0
    vq->handle_output = NULL;
2611
0
    g_free(vq->used_elems);
2612
0
    vq->used_elems = NULL;
2613
0
    virtio_virtqueue_reset_region_cache(vq);
2614
0
}
2615
2616
void virtio_del_queue(VirtIODevice *vdev, int n)
2617
0
{
2618
0
    if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
2619
0
        abort();
2620
0
    }
2621
2622
0
    virtio_delete_queue(&vdev->vq[n]);
2623
0
}
2624
2625
static void virtio_set_isr(VirtIODevice *vdev, int value)
2626
947k
{
2627
947k
    uint8_t old = qatomic_read(&vdev->isr);
2628
2629
    /* Do not write ISR if it does not change, so that its cacheline remains
2630
     * shared in the common case where the guest does not read it.
2631
     */
2632
947k
    if ((old & value) != value) {
2633
75.5k
        qatomic_or(&vdev->isr, value);
2634
75.5k
    }
2635
947k
}
2636
2637
/* Called within rcu_read_lock(). */
2638
static bool virtio_split_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2639
3.48M
{
2640
3.48M
    uint16_t old, new;
2641
3.48M
    bool v;
2642
    /* We need to expose used array entries before checking used event. */
2643
3.48M
    smp_mb();
2644
    /* Always notify when queue is empty (when feature acknowledge) */
2645
3.48M
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
2646
348k
        !vq->inuse && virtio_queue_empty(vq)) {
2647
16.4k
        return true;
2648
16.4k
    }
2649
2650
3.47M
    if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
2651
2.91M
        return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
2652
2.91M
    }
2653
2654
559k
    v = vq->signalled_used_valid;
2655
559k
    vq->signalled_used_valid = true;
2656
559k
    old = vq->signalled_used;
2657
559k
    new = vq->signalled_used = vq->used_idx;
2658
559k
    return !v || vring_need_event(vring_get_used_event(vq), new, old);
2659
3.47M
}
2660
2661
static bool vring_packed_need_event(VirtQueue *vq, bool wrap,
2662
                                    uint16_t off_wrap, uint16_t new,
2663
                                    uint16_t old)
2664
0
{
2665
0
    int off = off_wrap & ~(1 << 15);
2666
2667
0
    if (wrap != off_wrap >> 15) {
2668
0
        off -= vq->vring.num;
2669
0
    }
2670
2671
0
    return vring_need_event(off, new, old);
2672
0
}
2673
2674
/* Called within rcu_read_lock(). */
2675
static bool virtio_packed_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2676
0
{
2677
0
    VRingPackedDescEvent e;
2678
0
    uint16_t old, new;
2679
0
    bool v;
2680
0
    VRingMemoryRegionCaches *caches;
2681
2682
0
    caches = vring_get_region_caches(vq);
2683
0
    if (!caches) {
2684
0
        return false;
2685
0
    }
2686
2687
0
    vring_packed_event_read(vdev, &caches->avail, &e);
2688
2689
0
    old = vq->signalled_used;
2690
0
    new = vq->signalled_used = vq->used_idx;
2691
0
    v = vq->signalled_used_valid;
2692
0
    vq->signalled_used_valid = true;
2693
2694
0
    if (e.flags == VRING_PACKED_EVENT_FLAG_DISABLE) {
2695
0
        return false;
2696
0
    } else if (e.flags == VRING_PACKED_EVENT_FLAG_ENABLE) {
2697
0
        return true;
2698
0
    }
2699
2700
0
    return !v || vring_packed_need_event(vq, vq->used_wrap_counter,
2701
0
                                         e.off_wrap, new, old);
2702
0
}
2703
2704
/* Called within rcu_read_lock().  */
2705
static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2706
3.48M
{
2707
3.48M
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2708
0
        return virtio_packed_should_notify(vdev, vq);
2709
3.48M
    } else {
2710
3.48M
        return virtio_split_should_notify(vdev, vq);
2711
3.48M
    }
2712
3.48M
}
2713
2714
/* Batch irqs while inside a defer_call_begin()/defer_call_end() section */
2715
static void virtio_notify_irqfd_deferred_fn(void *opaque)
2716
0
{
2717
0
    EventNotifier *notifier = opaque;
2718
0
    VirtQueue *vq = container_of(notifier, VirtQueue, guest_notifier);
2719
2720
0
    trace_virtio_notify_irqfd_deferred_fn(vq->vdev, vq);
2721
0
    event_notifier_set(notifier);
2722
0
}
2723
2724
static void virtio_irq(VirtQueue *vq)
2725
884k
{
2726
    /*
2727
     * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
2728
     * windows drivers included in virtio-win 1.8.0 (circa 2015) are
2729
     * incorrectly polling this bit during crashdump and hibernation
2730
     * in MSI mode, causing a hang if this bit is never updated.
2731
     * Recent releases of Windows do not really shut down, but rather
2732
     * log out and hibernate to make the next startup faster.  Hence,
2733
     * this manifested as a more serious hang during shutdown with
2734
     *
2735
     * Next driver release from 2016 fixed this problem, so working around it
2736
     * is not a must, but it's easy to do so let's do it here.
2737
     *
2738
     * Note: it's safe to update ISR from any thread as it was switched
2739
     * to an atomic operation.
2740
     */
2741
884k
    virtio_set_isr(vq->vdev, 0x1);
2742
2743
    /*
2744
     * The interrupt code path requires the Big QEMU Lock (BQL), so use the
2745
     * notifier instead when in an IOThread. This assumes that device models
2746
     * have already called ->set_guest_notifiers() sometime before calling this
2747
     * function.
2748
     */
2749
884k
    if (qemu_in_iothread()) {
2750
0
        defer_call(virtio_notify_irqfd_deferred_fn, &vq->guest_notifier);
2751
884k
    } else {
2752
884k
        virtio_notify_vector(vq->vdev, vq->vector);
2753
884k
    }
2754
884k
}
2755
2756
void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
2757
3.48M
{
2758
3.48M
    WITH_RCU_READ_LOCK_GUARD() {
2759
3.48M
        if (!virtio_should_notify(vdev, vq)) {
2760
2.60M
            return;
2761
2.60M
        }
2762
3.48M
    }
2763
2764
884k
    trace_virtio_notify(vdev, vq);
2765
884k
    virtio_irq(vq);
2766
884k
}
2767
2768
void virtio_notify_config(VirtIODevice *vdev)
2769
115k
{
2770
115k
    if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
2771
51.4k
        return;
2772
2773
63.7k
    virtio_set_isr(vdev, 0x3);
2774
63.7k
    vdev->generation++;
2775
2776
63.7k
    if (qemu_in_iothread()) {
2777
0
        defer_call(virtio_notify_irqfd_deferred_fn, &vdev->config_notifier);
2778
63.7k
    } else {
2779
63.7k
        virtio_notify_vector(vdev, vdev->config_vector);
2780
63.7k
    }
2781
63.7k
}
2782
2783
static bool virtio_device_endian_needed(void *opaque)
2784
0
{
2785
0
    VirtIODevice *vdev = opaque;
2786
2787
0
    assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
2788
0
    if (virtio_vdev_is_legacy(vdev)) {
2789
0
        return vdev->device_endian != virtio_default_endian();
2790
0
    }
2791
    /* Devices conforming to VIRTIO 1.0 or later are always LE. */
2792
0
    return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
2793
0
}
2794
2795
static bool virtio_64bit_features_needed(void *opaque)
2796
0
{
2797
0
    VirtIODevice *vdev = opaque;
2798
2799
0
    return (vdev->host_features >> 32) != 0;
2800
0
}
2801
2802
static bool virtio_virtqueue_needed(void *opaque)
2803
0
{
2804
0
    VirtIODevice *vdev = opaque;
2805
2806
0
    return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
2807
0
}
2808
2809
static bool virtio_packed_virtqueue_needed(void *opaque)
2810
0
{
2811
0
    VirtIODevice *vdev = opaque;
2812
2813
0
    return virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED);
2814
0
}
2815
2816
static bool virtio_ringsize_needed(void *opaque)
2817
0
{
2818
0
    return false;
2819
0
}
2820
2821
static bool virtio_extra_state_needed(void *opaque)
2822
0
{
2823
0
    VirtIODevice *vdev = opaque;
2824
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2825
0
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2826
2827
0
    return k->has_extra_state &&
2828
0
        k->has_extra_state(qbus->parent);
2829
0
}
2830
2831
static bool virtio_broken_needed(void *opaque)
2832
0
{
2833
0
    VirtIODevice *vdev = opaque;
2834
2835
0
    return vdev->broken;
2836
0
}
2837
2838
static bool virtio_started_needed(void *opaque)
2839
0
{
2840
0
    VirtIODevice *vdev = opaque;
2841
2842
0
    return vdev->started;
2843
0
}
2844
2845
static bool virtio_disabled_needed(void *opaque)
2846
0
{
2847
0
    VirtIODevice *vdev = opaque;
2848
2849
0
    return vdev->disabled;
2850
0
}
2851
2852
static const VMStateDescription vmstate_virtqueue = {
2853
    .name = "virtqueue_state",
2854
    .version_id = 1,
2855
    .minimum_version_id = 1,
2856
    .fields = (const VMStateField[]) {
2857
        VMSTATE_UINT64(vring.avail, struct VirtQueue),
2858
        VMSTATE_UINT64(vring.used, struct VirtQueue),
2859
        VMSTATE_END_OF_LIST()
2860
    }
2861
};
2862
2863
static const VMStateDescription vmstate_packed_virtqueue = {
2864
    .name = "packed_virtqueue_state",
2865
    .version_id = 1,
2866
    .minimum_version_id = 1,
2867
    .fields = (const VMStateField[]) {
2868
        VMSTATE_UINT16(last_avail_idx, struct VirtQueue),
2869
        VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue),
2870
        VMSTATE_UINT16(used_idx, struct VirtQueue),
2871
        VMSTATE_BOOL(used_wrap_counter, struct VirtQueue),
2872
        VMSTATE_UINT32(inuse, struct VirtQueue),
2873
        VMSTATE_END_OF_LIST()
2874
    }
2875
};
2876
2877
static const VMStateDescription vmstate_virtio_virtqueues = {
2878
    .name = "virtio/virtqueues",
2879
    .version_id = 1,
2880
    .minimum_version_id = 1,
2881
    .needed = &virtio_virtqueue_needed,
2882
    .fields = (const VMStateField[]) {
2883
        VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2884
                      VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
2885
        VMSTATE_END_OF_LIST()
2886
    }
2887
};
2888
2889
static const VMStateDescription vmstate_virtio_packed_virtqueues = {
2890
    .name = "virtio/packed_virtqueues",
2891
    .version_id = 1,
2892
    .minimum_version_id = 1,
2893
    .needed = &virtio_packed_virtqueue_needed,
2894
    .fields = (const VMStateField[]) {
2895
        VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2896
                      VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue),
2897
        VMSTATE_END_OF_LIST()
2898
    }
2899
};
2900
2901
static const VMStateDescription vmstate_ringsize = {
2902
    .name = "ringsize_state",
2903
    .version_id = 1,
2904
    .minimum_version_id = 1,
2905
    .fields = (const VMStateField[]) {
2906
        VMSTATE_UNUSED(sizeof(uint32_t)),
2907
        VMSTATE_END_OF_LIST()
2908
    }
2909
};
2910
2911
static const VMStateDescription vmstate_virtio_ringsize = {
2912
    .name = "virtio/ringsize",
2913
    .version_id = 1,
2914
    .minimum_version_id = 1,
2915
    .needed = &virtio_ringsize_needed,
2916
    .fields = (const VMStateField[]) {
2917
        VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2918
                      VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
2919
        VMSTATE_END_OF_LIST()
2920
    }
2921
};
2922
2923
static int get_extra_state(QEMUFile *f, void *pv, size_t size,
2924
                           const VMStateField *field)
2925
0
{
2926
0
    VirtIODevice *vdev = pv;
2927
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2928
0
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2929
2930
0
    if (!k->load_extra_state) {
2931
0
        return -1;
2932
0
    } else {
2933
0
        return k->load_extra_state(qbus->parent, f);
2934
0
    }
2935
0
}
2936
2937
static int put_extra_state(QEMUFile *f, void *pv, size_t size,
2938
                           const VMStateField *field, JSONWriter *vmdesc)
2939
0
{
2940
0
    VirtIODevice *vdev = pv;
2941
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2942
0
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2943
2944
0
    k->save_extra_state(qbus->parent, f);
2945
0
    return 0;
2946
0
}
2947
2948
static const VMStateInfo vmstate_info_extra_state = {
2949
    .name = "virtqueue_extra_state",
2950
    .get = get_extra_state,
2951
    .put = put_extra_state,
2952
};
2953
2954
static const VMStateDescription vmstate_virtio_extra_state = {
2955
    .name = "virtio/extra_state",
2956
    .version_id = 1,
2957
    .minimum_version_id = 1,
2958
    .needed = &virtio_extra_state_needed,
2959
    .fields = (const VMStateField[]) {
2960
        {
2961
            .name         = "extra_state",
2962
            .version_id   = 0,
2963
            .field_exists = NULL,
2964
            .size         = 0,
2965
            .info         = &vmstate_info_extra_state,
2966
            .flags        = VMS_SINGLE,
2967
            .offset       = 0,
2968
        },
2969
        VMSTATE_END_OF_LIST()
2970
    }
2971
};
2972
2973
static const VMStateDescription vmstate_virtio_device_endian = {
2974
    .name = "virtio/device_endian",
2975
    .version_id = 1,
2976
    .minimum_version_id = 1,
2977
    .needed = &virtio_device_endian_needed,
2978
    .fields = (const VMStateField[]) {
2979
        VMSTATE_UINT8(device_endian, VirtIODevice),
2980
        VMSTATE_END_OF_LIST()
2981
    }
2982
};
2983
2984
static const VMStateDescription vmstate_virtio_64bit_features = {
2985
    .name = "virtio/64bit_features",
2986
    .version_id = 1,
2987
    .minimum_version_id = 1,
2988
    .needed = &virtio_64bit_features_needed,
2989
    .fields = (const VMStateField[]) {
2990
        VMSTATE_UINT64(guest_features, VirtIODevice),
2991
        VMSTATE_END_OF_LIST()
2992
    }
2993
};
2994
2995
static const VMStateDescription vmstate_virtio_broken = {
2996
    .name = "virtio/broken",
2997
    .version_id = 1,
2998
    .minimum_version_id = 1,
2999
    .needed = &virtio_broken_needed,
3000
    .fields = (const VMStateField[]) {
3001
        VMSTATE_BOOL(broken, VirtIODevice),
3002
        VMSTATE_END_OF_LIST()
3003
    }
3004
};
3005
3006
static const VMStateDescription vmstate_virtio_started = {
3007
    .name = "virtio/started",
3008
    .version_id = 1,
3009
    .minimum_version_id = 1,
3010
    .needed = &virtio_started_needed,
3011
    .fields = (const VMStateField[]) {
3012
        VMSTATE_BOOL(started, VirtIODevice),
3013
        VMSTATE_END_OF_LIST()
3014
    }
3015
};
3016
3017
static const VMStateDescription vmstate_virtio_disabled = {
3018
    .name = "virtio/disabled",
3019
    .version_id = 1,
3020
    .minimum_version_id = 1,
3021
    .needed = &virtio_disabled_needed,
3022
    .fields = (const VMStateField[]) {
3023
        VMSTATE_BOOL(disabled, VirtIODevice),
3024
        VMSTATE_END_OF_LIST()
3025
    }
3026
};
3027
3028
static bool virtio_128bit_features_needed(void *opaque)
3029
0
{
3030
0
    VirtIODevice *vdev = opaque;
3031
3032
0
    return virtio_features_use_ex(vdev->host_features_ex);
3033
0
}
3034
3035
static const VMStateDescription vmstate_virtio_128bit_features = {
3036
    .name = "virtio/128bit_features",
3037
    .version_id = 1,
3038
    .minimum_version_id = 1,
3039
    .needed = &virtio_128bit_features_needed,
3040
    .fields = (const VMStateField[]) {
3041
        VMSTATE_UINT64(guest_features_ex[1], VirtIODevice),
3042
        VMSTATE_END_OF_LIST()
3043
    }
3044
};
3045
3046
/*
3047
 * Avoid silently breaking migration should the feature space increase
3048
 * even more in the (far away) future
3049
 */
3050
QEMU_BUILD_BUG_ON(VIRTIO_FEATURES_NU64S != 2);
3051
3052
static const VMStateDescription vmstate_virtio = {
3053
    .name = "virtio",
3054
    .version_id = 1,
3055
    .minimum_version_id = 1,
3056
    .fields = (const VMStateField[]) {
3057
        VMSTATE_END_OF_LIST()
3058
    },
3059
    .subsections = (const VMStateDescription * const []) {
3060
        &vmstate_virtio_device_endian,
3061
        &vmstate_virtio_128bit_features,
3062
        &vmstate_virtio_64bit_features,
3063
        &vmstate_virtio_virtqueues,
3064
        &vmstate_virtio_ringsize,
3065
        &vmstate_virtio_broken,
3066
        &vmstate_virtio_extra_state,
3067
        &vmstate_virtio_started,
3068
        &vmstate_virtio_packed_virtqueues,
3069
        &vmstate_virtio_disabled,
3070
        NULL
3071
    }
3072
};
3073
3074
int virtio_save(VirtIODevice *vdev, QEMUFile *f)
3075
0
{
3076
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3077
0
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3078
0
    VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
3079
0
    uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
3080
0
    int i, ret;
3081
0
    Error *local_err = NULL;
3082
3083
0
    if (k->save_config) {
3084
0
        k->save_config(qbus->parent, f);
3085
0
    }
3086
3087
0
    qemu_put_8s(f, &vdev->status);
3088
0
    qemu_put_8s(f, &vdev->isr);
3089
0
    qemu_put_be16s(f, &vdev->queue_sel);
3090
0
    qemu_put_be32s(f, &guest_features_lo);
3091
0
    qemu_put_be32(f, vdev->config_len);
3092
0
    qemu_put_buffer(f, vdev->config, vdev->config_len);
3093
3094
0
    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3095
0
        if (vdev->vq[i].vring.num == 0)
3096
0
            break;
3097
0
    }
3098
3099
0
    qemu_put_be32(f, i);
3100
3101
0
    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3102
0
        if (vdev->vq[i].vring.num == 0)
3103
0
            break;
3104
3105
0
        qemu_put_be32(f, vdev->vq[i].vring.num);
3106
0
        if (k->has_variable_vring_alignment) {
3107
0
            qemu_put_be32(f, vdev->vq[i].vring.align);
3108
0
        }
3109
        /*
3110
         * Save desc now, the rest of the ring addresses are saved in
3111
         * subsections for VIRTIO-1 devices.
3112
         */
3113
0
        qemu_put_be64(f, vdev->vq[i].vring.desc);
3114
0
        qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
3115
0
        if (k->save_queue) {
3116
0
            k->save_queue(qbus->parent, i, f);
3117
0
        }
3118
0
    }
3119
3120
0
    if (vdc->save != NULL) {
3121
0
        vdc->save(vdev, f);
3122
0
    }
3123
3124
0
    if (vdc->vmsd) {
3125
0
        ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL, &local_err);
3126
0
        if (ret) {
3127
0
            error_report_err(local_err);
3128
0
            return ret;
3129
0
        }
3130
0
    }
3131
3132
    /* Subsections */
3133
0
    ret = vmstate_save_state(f, &vmstate_virtio, vdev, NULL, &local_err);
3134
0
    if (ret < 0) {
3135
0
        error_report_err(local_err);
3136
0
    }
3137
0
    return ret;
3138
0
}
3139
3140
VirtioSharedMemory *virtio_new_shmem_region(VirtIODevice *vdev, uint8_t shmid, uint64_t size)
3141
0
{
3142
0
    VirtioSharedMemory *elem;
3143
0
    g_autofree char *name = NULL;
3144
3145
0
    elem = g_new0(VirtioSharedMemory, 1);
3146
0
    elem->shmid = shmid;
3147
3148
    /* Initialize embedded MemoryRegion as container for shmem mappings */
3149
0
    name = g_strdup_printf("virtio-shmem-%d", shmid);
3150
0
    memory_region_init(&elem->mr, OBJECT(vdev), name, size);
3151
0
    QTAILQ_INIT(&elem->mmaps);
3152
0
    QSIMPLEQ_INSERT_TAIL(&vdev->shmem_list, elem, entry);
3153
0
    return elem;
3154
0
}
3155
3156
VirtioSharedMemory *virtio_find_shmem_region(VirtIODevice *vdev, uint8_t shmid)
3157
0
{
3158
0
    VirtioSharedMemory *shmem, *next;
3159
0
    QSIMPLEQ_FOREACH_SAFE(shmem, &vdev->shmem_list, entry, next) {
3160
0
        if (shmem->shmid == shmid) {
3161
0
            return shmem;
3162
0
        }
3163
0
    }
3164
0
    return NULL;
3165
0
}
3166
3167
static void virtio_shared_memory_mapping_instance_init(Object *obj)
3168
0
{
3169
0
    VirtioSharedMemoryMapping *mapping = VIRTIO_SHARED_MEMORY_MAPPING(obj);
3170
3171
0
    mapping->shmid = 0;
3172
0
    mapping->offset = 0;
3173
0
    mapping->len = 0;
3174
0
    mapping->mr = NULL;
3175
0
}
3176
3177
static void virtio_shared_memory_mapping_instance_finalize(Object *obj)
3178
0
{
3179
0
    VirtioSharedMemoryMapping *mapping = VIRTIO_SHARED_MEMORY_MAPPING(obj);
3180
3181
    /* Clean up MemoryRegion if it exists */
3182
0
    if (mapping->mr) {
3183
        /* Unparent the MemoryRegion to trigger cleanup */
3184
0
        object_unparent(OBJECT(mapping->mr));
3185
0
        mapping->mr = NULL;
3186
0
    }
3187
0
}
3188
3189
VirtioSharedMemoryMapping *virtio_shared_memory_mapping_new(uint8_t shmid,
3190
                                                            int fd,
3191
                                                            uint64_t fd_offset,
3192
                                                            uint64_t shm_offset,
3193
                                                            uint64_t len,
3194
                                                            bool allow_write)
3195
0
{
3196
0
    VirtioSharedMemoryMapping *mapping;
3197
0
    MemoryRegion *mr;
3198
0
    g_autoptr(GString) mr_name = g_string_new(NULL);
3199
0
    uint32_t ram_flags;
3200
0
    Error *local_err = NULL;
3201
3202
0
    if (len == 0) {
3203
0
        error_report("Shared memory mapping size cannot be zero");
3204
0
        return NULL;
3205
0
    }
3206
3207
0
    fd = dup(fd);
3208
0
    if (fd < 0) {
3209
0
        error_report("Failed to duplicate fd: %s", strerror(errno));
3210
0
        return NULL;
3211
0
    }
3212
3213
    /* Determine RAM flags */
3214
0
    ram_flags = RAM_SHARED;
3215
0
    if (!allow_write) {
3216
0
        ram_flags |= RAM_READONLY_FD;
3217
0
    }
3218
3219
    /* Create the VirtioSharedMemoryMapping */
3220
0
    mapping = VIRTIO_SHARED_MEMORY_MAPPING(
3221
0
        object_new(TYPE_VIRTIO_SHARED_MEMORY_MAPPING));
3222
3223
    /* Set up object properties */
3224
0
    mapping->shmid = shmid;
3225
0
    mapping->offset = shm_offset;
3226
0
    mapping->len = len;
3227
3228
    /* Create MemoryRegion as a child of this object */
3229
0
    mr = g_new0(MemoryRegion, 1);
3230
0
    g_string_printf(mr_name, "virtio-shmem-%d-%" PRIx64, shmid, shm_offset);
3231
3232
    /* Initialize MemoryRegion with file descriptor */
3233
0
    if (!memory_region_init_ram_from_fd(mr, OBJECT(mapping), mr_name->str,
3234
0
                                        len, ram_flags, fd, fd_offset,
3235
0
                                        &local_err)) {
3236
0
        error_report_err(local_err);
3237
0
        g_free(mr);
3238
0
        close(fd);
3239
0
        object_unref(OBJECT(mapping));
3240
0
        return NULL;
3241
0
    }
3242
3243
0
    mapping->mr = mr;
3244
0
    return mapping;
3245
0
}
3246
3247
int virtio_add_shmem_map(VirtioSharedMemory *shmem,
3248
                         VirtioSharedMemoryMapping *mapping)
3249
0
{
3250
0
    if (!mapping) {
3251
0
        error_report("VirtioSharedMemoryMapping cannot be NULL");
3252
0
        return -1;
3253
0
    }
3254
0
    if (!mapping->mr) {
3255
0
        error_report("VirtioSharedMemoryMapping has no MemoryRegion");
3256
0
        return -1;
3257
0
    }
3258
3259
    /* Validate boundaries against the VIRTIO shared memory region */
3260
0
    if (mapping->offset + mapping->len > memory_region_size(&shmem->mr)) {
3261
0
        error_report("Memory exceeds the shared memory boundaries");
3262
0
        return -1;
3263
0
    }
3264
3265
    /* Add as subregion to the VIRTIO shared memory */
3266
0
    memory_region_add_subregion(&shmem->mr, mapping->offset, mapping->mr);
3267
3268
    /* Add to the mapped regions list */
3269
0
    QTAILQ_INSERT_TAIL(&shmem->mmaps, mapping, link);
3270
3271
0
    return 0;
3272
0
}
3273
3274
VirtioSharedMemoryMapping *virtio_find_shmem_map(VirtioSharedMemory *shmem,
3275
                                          hwaddr offset, uint64_t size)
3276
0
{
3277
0
    VirtioSharedMemoryMapping *mapping;
3278
0
    QTAILQ_FOREACH(mapping, &shmem->mmaps, link) {
3279
0
        if (mapping->offset == offset && mapping->len == size) {
3280
0
            return mapping;
3281
0
        }
3282
0
    }
3283
0
    return NULL;
3284
0
}
3285
3286
void virtio_del_shmem_map(VirtioSharedMemory *shmem, hwaddr offset,
3287
                          uint64_t size)
3288
0
{
3289
0
    VirtioSharedMemoryMapping *mapping = virtio_find_shmem_map(shmem, offset, size);
3290
0
    if (mapping == NULL) {
3291
0
        return;
3292
0
    }
3293
3294
    /*
3295
     * Remove from memory region first
3296
     */
3297
0
    memory_region_del_subregion(&shmem->mr, mapping->mr);
3298
3299
    /*
3300
     * Remove from list and unref the mapping which will trigger automatic cleanup
3301
     * when the reference count reaches zero.
3302
     */
3303
0
    QTAILQ_REMOVE(&shmem->mmaps, mapping, link);
3304
0
    object_unref(OBJECT(mapping));
3305
0
}
3306
3307
/* A wrapper for use as a VMState .put function */
3308
static int virtio_device_put(QEMUFile *f, void *opaque, size_t size,
3309
                              const VMStateField *field, JSONWriter *vmdesc)
3310
0
{
3311
0
    return virtio_save(VIRTIO_DEVICE(opaque), f);
3312
0
}
3313
3314
/* A wrapper for use as a VMState .get function */
3315
static int coroutine_mixed_fn
3316
virtio_device_get(QEMUFile *f, void *opaque, size_t size,
3317
                  const VMStateField *field)
3318
0
{
3319
0
    VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
3320
0
    DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
3321
3322
0
    return virtio_load(vdev, f, dc->vmsd->version_id);
3323
0
}
3324
3325
const VMStateInfo  virtio_vmstate_info = {
3326
    .name = "virtio",
3327
    .get = virtio_device_get,
3328
    .put = virtio_device_put,
3329
};
3330
3331
static int virtio_set_features_nocheck(VirtIODevice *vdev, const uint64_t *val)
3332
775k
{
3333
775k
    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
3334
775k
    uint64_t tmp[VIRTIO_FEATURES_NU64S];
3335
775k
    bool bad;
3336
3337
775k
    bad = virtio_features_andnot(tmp, val, vdev->host_features_ex);
3338
775k
    virtio_features_and(tmp, val, vdev->host_features_ex);
3339
3340
775k
    if (k->set_features_ex) {
3341
206k
        k->set_features_ex(vdev, tmp);
3342
568k
    } else if (k->set_features) {
3343
83.7k
        bad = bad || virtio_features_use_ex(tmp);
3344
83.7k
        k->set_features(vdev, tmp[0]);
3345
83.7k
    }
3346
3347
775k
    virtio_features_copy(vdev->guest_features_ex, tmp);
3348
775k
    return bad ? -1 : 0;
3349
775k
}
3350
3351
typedef struct VirtioSetFeaturesNocheckData {
3352
    Coroutine *co;
3353
    VirtIODevice *vdev;
3354
    uint64_t val[VIRTIO_FEATURES_NU64S];
3355
    int ret;
3356
} VirtioSetFeaturesNocheckData;
3357
3358
static void virtio_set_features_nocheck_bh(void *opaque)
3359
0
{
3360
0
    VirtioSetFeaturesNocheckData *data = opaque;
3361
3362
0
    data->ret = virtio_set_features_nocheck(data->vdev, data->val);
3363
0
    aio_co_wake(data->co);
3364
0
}
3365
3366
static int coroutine_mixed_fn
3367
virtio_set_features_nocheck_maybe_co(VirtIODevice *vdev,
3368
                                     const uint64_t *val)
3369
0
{
3370
0
    if (qemu_in_coroutine()) {
3371
0
        VirtioSetFeaturesNocheckData data = {
3372
0
            .co = qemu_coroutine_self(),
3373
0
            .vdev = vdev,
3374
0
        };
3375
0
        virtio_features_copy(data.val, val);
3376
0
        aio_bh_schedule_oneshot(qemu_get_current_aio_context(),
3377
0
                                virtio_set_features_nocheck_bh, &data);
3378
0
        qemu_coroutine_yield();
3379
0
        return data.ret;
3380
0
    } else {
3381
0
        return virtio_set_features_nocheck(vdev, val);
3382
0
    }
3383
0
}
3384
3385
int virtio_set_features(VirtIODevice *vdev, uint64_t val)
3386
326k
{
3387
326k
    uint64_t features[VIRTIO_FEATURES_NU64S];
3388
3389
326k
    virtio_features_from_u64(features, val);
3390
326k
    return virtio_set_features_ex(vdev, features);
3391
326k
}
3392
3393
int virtio_set_features_ex(VirtIODevice *vdev, const uint64_t *features)
3394
439k
{
3395
439k
    int ret;
3396
    /*
3397
     * The driver must not attempt to set features after feature negotiation
3398
     * has finished.
3399
     */
3400
439k
    if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
3401
34.8k
        return -EINVAL;
3402
34.8k
    }
3403
3404
405k
    if (features[0] & (1ull << VIRTIO_F_BAD_FEATURE)) {
3405
36.3k
        qemu_log_mask(LOG_GUEST_ERROR,
3406
36.3k
                      "%s: guest driver for %s has enabled UNUSED(30) feature bit!\n",
3407
36.3k
                      __func__, vdev->name);
3408
36.3k
    }
3409
3410
405k
    ret = virtio_set_features_nocheck(vdev, features);
3411
405k
    if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
3412
        /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches.  */
3413
335k
        int i;
3414
343M
        for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3415
343M
            if (vdev->vq[i].vring.num != 0) {
3416
1.29M
                virtio_init_region_cache(vdev, i);
3417
1.29M
            }
3418
343M
        }
3419
335k
    }
3420
405k
    if (!ret) {
3421
30.3k
        if (!virtio_device_started(vdev, vdev->status) &&
3422
22.6k
            !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3423
16.2k
            vdev->start_on_kick = true;
3424
16.2k
        }
3425
30.3k
    }
3426
405k
    return ret;
3427
439k
}
3428
3429
void virtio_reset(VirtIODevice *vdev)
3430
370k
{
3431
370k
    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
3432
370k
    VirtioSharedMemory *shmem;
3433
370k
    uint64_t features[VIRTIO_FEATURES_NU64S];
3434
370k
    int i;
3435
3436
370k
    virtio_set_status(vdev, 0);
3437
370k
    if (current_cpu) {
3438
        /* Guest initiated reset */
3439
0
        vdev->device_endian = virtio_current_cpu_endian();
3440
370k
    } else {
3441
        /* System reset */
3442
370k
        vdev->device_endian = virtio_default_endian();
3443
370k
    }
3444
3445
370k
    if (k->get_vhost) {
3446
59.9k
        struct vhost_dev *hdev = k->get_vhost(vdev);
3447
        /* Only reset when vhost back-end is connected */
3448
59.9k
        if (hdev && hdev->vhost_ops) {
3449
0
            vhost_reset_device(hdev);
3450
0
        }
3451
59.9k
    }
3452
3453
370k
    if (k->reset) {
3454
370k
        k->reset(vdev);
3455
370k
    }
3456
3457
370k
    vdev->start_on_kick = false;
3458
370k
    vdev->started = false;
3459
370k
    vdev->broken = false;
3460
370k
    virtio_features_clear(features);
3461
370k
    virtio_set_features_nocheck(vdev, features);
3462
370k
    vdev->queue_sel = 0;
3463
370k
    vdev->status = 0;
3464
370k
    vdev->disabled = false;
3465
370k
    qatomic_set(&vdev->isr, 0);
3466
370k
    vdev->config_vector = VIRTIO_NO_VECTOR;
3467
370k
    virtio_notify_vector(vdev, vdev->config_vector);
3468
3469
379M
    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3470
378M
        __virtio_queue_reset(vdev, i);
3471
378M
    }
3472
3473
    /* Mappings are removed to prevent stale fds from remaining open. */
3474
370k
    QSIMPLEQ_FOREACH(shmem, &vdev->shmem_list, entry) {
3475
0
        while (!QTAILQ_EMPTY(&shmem->mmaps)) {
3476
0
            VirtioSharedMemoryMapping *mapping = QTAILQ_FIRST(&shmem->mmaps);
3477
0
            virtio_del_shmem_map(shmem, mapping->offset,
3478
0
                                 memory_region_size(mapping->mr));
3479
0
        }
3480
0
    }
3481
370k
}
3482
3483
static void virtio_device_check_notification_compatibility(VirtIODevice *vdev,
3484
                                                           Error **errp)
3485
21
{
3486
21
    VirtioBusState *bus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
3487
21
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(bus);
3488
21
    DeviceState *proxy = DEVICE(BUS(bus)->parent);
3489
3490
21
    if (virtio_host_has_feature(vdev, VIRTIO_F_NOTIFICATION_DATA) &&
3491
0
        k->ioeventfd_enabled(proxy)) {
3492
0
        error_setg(errp,
3493
0
                   "notification_data=on without ioeventfd=off is not supported");
3494
0
    }
3495
21
}
3496
3497
size_t virtio_get_config_size(const VirtIOConfigSizeParams *params,
3498
                              uint64_t host_features)
3499
4
{
3500
4
    size_t config_size = params->min_size;
3501
4
    const VirtIOFeature *feature_sizes = params->feature_sizes;
3502
4
    size_t i;
3503
3504
22
    for (i = 0; feature_sizes[i].flags != 0; i++) {
3505
18
        if (host_features & feature_sizes[i].flags) {
3506
8
            config_size = MAX(feature_sizes[i].end, config_size);
3507
8
        }
3508
18
    }
3509
3510
4
    assert(config_size <= params->max_size);
3511
4
    return config_size;
3512
4
}
3513
3514
int coroutine_mixed_fn
3515
virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
3516
0
{
3517
0
    int i, ret;
3518
0
    uint32_t config_len;
3519
0
    uint32_t num;
3520
0
    uint32_t features;
3521
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3522
0
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3523
0
    VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
3524
0
    Error *local_err = NULL;
3525
3526
    /*
3527
     * We poison the endianness to ensure it does not get used before
3528
     * subsections have been loaded.
3529
     */
3530
0
    vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
3531
3532
0
    if (k->load_config) {
3533
0
        ret = k->load_config(qbus->parent, f);
3534
0
        if (ret)
3535
0
            return ret;
3536
0
    }
3537
3538
0
    qemu_get_8s(f, &vdev->status);
3539
0
    qemu_get_8s(f, &vdev->isr);
3540
0
    qemu_get_be16s(f, &vdev->queue_sel);
3541
0
    if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
3542
0
        return -1;
3543
0
    }
3544
0
    qemu_get_be32s(f, &features);
3545
3546
    /*
3547
     * Temporarily set guest_features low bits - needed by
3548
     * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
3549
     * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
3550
     *
3551
     * Note: devices should always test host features in future - don't create
3552
     * new dependencies like this.
3553
     */
3554
0
    virtio_features_from_u64(vdev->guest_features_ex, features);
3555
3556
0
    config_len = qemu_get_be32(f);
3557
3558
    /*
3559
     * There are cases where the incoming config can be bigger or smaller
3560
     * than what we have; so load what we have space for, and skip
3561
     * any excess that's in the stream.
3562
     */
3563
0
    qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
3564
3565
0
    while (config_len > vdev->config_len) {
3566
0
        if (qemu_file_get_error(f)) {
3567
0
            return -1;
3568
0
        }
3569
0
        qemu_get_byte(f);
3570
0
        if (qemu_file_get_error(f)) {
3571
0
            return -1;
3572
0
        }
3573
0
        config_len--;
3574
0
    }
3575
3576
0
    num = qemu_get_be32(f);
3577
3578
0
    if (num > VIRTIO_QUEUE_MAX) {
3579
0
        error_report("Invalid number of virtqueues: 0x%x", num);
3580
0
        return -1;
3581
0
    }
3582
3583
0
    if (vdc->pre_load_queues) {
3584
0
        ret = vdc->pre_load_queues(vdev, num);
3585
0
        if (ret) {
3586
0
            return ret;
3587
0
        }
3588
0
    }
3589
3590
0
    for (i = 0; i < num; i++) {
3591
0
        vdev->vq[i].vring.num = qemu_get_be32(f);
3592
0
        if (vdev->vq[i].vring.num > vdev->vq[i].vring.num_default) {
3593
0
            error_report("VQ %d vring.num %u exceeds allocated max %u",
3594
0
                         i, vdev->vq[i].vring.num,
3595
0
                         vdev->vq[i].vring.num_default);
3596
0
            return -1;
3597
0
        }
3598
0
        if (k->has_variable_vring_alignment) {
3599
0
            vdev->vq[i].vring.align = qemu_get_be32(f);
3600
0
        }
3601
0
        vdev->vq[i].vring.desc = qemu_get_be64(f);
3602
0
        qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
3603
0
        vdev->vq[i].signalled_used_valid = false;
3604
0
        vdev->vq[i].notification = true;
3605
3606
0
        if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) {
3607
0
            error_report("VQ %d address 0x0 "
3608
0
                         "inconsistent with Host index 0x%x",
3609
0
                         i, vdev->vq[i].last_avail_idx);
3610
0
            return -1;
3611
0
        }
3612
0
        if (k->load_queue) {
3613
0
            ret = k->load_queue(qbus->parent, i, f);
3614
0
            if (ret)
3615
0
                return ret;
3616
0
        }
3617
0
    }
3618
3619
0
    virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
3620
3621
0
    if (vdc->load != NULL) {
3622
0
        ret = vdc->load(vdev, f, version_id);
3623
0
        if (ret) {
3624
0
            return ret;
3625
0
        }
3626
0
    }
3627
3628
0
    if (vdc->vmsd) {
3629
0
        ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id, &local_err);
3630
0
        if (ret) {
3631
0
            error_report_err(local_err);
3632
0
            return ret;
3633
0
        }
3634
0
    }
3635
3636
    /* Subsections */
3637
0
    ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1, &local_err);
3638
0
    if (ret) {
3639
0
        error_report_err(local_err);
3640
0
        return ret;
3641
0
    }
3642
3643
0
    if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
3644
0
        vdev->device_endian = virtio_default_endian();
3645
0
    }
3646
3647
    /*
3648
     * guest_features_ex is fully initialized with u32 features and upper
3649
     * bits have been filled as needed by the later load.
3650
     */
3651
0
    if (virtio_set_features_nocheck_maybe_co(vdev,
3652
0
                                             vdev->guest_features_ex) < 0) {
3653
0
        error_report("Features 0x" VIRTIO_FEATURES_FMT " unsupported. "
3654
0
                     "Allowed features: 0x" VIRTIO_FEATURES_FMT,
3655
0
                     VIRTIO_FEATURES_PR(vdev->guest_features_ex),
3656
0
                     VIRTIO_FEATURES_PR(vdev->host_features_ex));
3657
0
        return -1;
3658
0
    }
3659
3660
0
    if (!virtio_device_started(vdev, vdev->status) &&
3661
0
        !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3662
0
        vdev->start_on_kick = true;
3663
0
    }
3664
3665
0
    RCU_READ_LOCK_GUARD();
3666
0
    for (i = 0; i < num; i++) {
3667
0
        if (vdev->vq[i].vring.desc) {
3668
0
            uint16_t nheads;
3669
3670
            /*
3671
             * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
3672
             * only the region cache needs to be set up.  Legacy devices need
3673
             * to calculate used and avail ring addresses based on the desc
3674
             * address.
3675
             */
3676
0
            if (virtio_vdev_is_legacy(vdev)) {
3677
0
                virtio_queue_update_rings(vdev, i);
3678
0
            } else {
3679
0
                virtio_init_region_cache(vdev, i);
3680
0
            }
3681
3682
0
            if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3683
0
                vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx;
3684
0
                vdev->vq[i].shadow_avail_wrap_counter =
3685
0
                                        vdev->vq[i].last_avail_wrap_counter;
3686
0
                continue;
3687
0
            }
3688
3689
0
            nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
3690
            /* Check it isn't doing strange things with descriptor numbers. */
3691
0
            if (nheads > vdev->vq[i].vring.num) {
3692
0
                virtio_error(vdev, "VQ %d size 0x%x Guest index 0x%x "
3693
0
                             "inconsistent with Host index 0x%x: delta 0x%x",
3694
0
                             i, vdev->vq[i].vring.num,
3695
0
                             vring_avail_idx(&vdev->vq[i]),
3696
0
                             vdev->vq[i].last_avail_idx, nheads);
3697
0
                vdev->vq[i].used_idx = 0;
3698
0
                vdev->vq[i].shadow_avail_idx = 0;
3699
0
                vdev->vq[i].inuse = 0;
3700
0
                continue;
3701
0
            }
3702
0
            vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
3703
0
            vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
3704
3705
            /*
3706
             * Some devices migrate VirtQueueElements that have been popped
3707
             * from the avail ring but not yet returned to the used ring.
3708
             * Since max ring size < UINT16_MAX it's safe to use modulo
3709
             * UINT16_MAX + 1 subtraction.
3710
             */
3711
0
            vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
3712
0
                                vdev->vq[i].used_idx);
3713
0
            if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
3714
0
                error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
3715
0
                             "used_idx 0x%x",
3716
0
                             i, vdev->vq[i].vring.num,
3717
0
                             vdev->vq[i].last_avail_idx,
3718
0
                             vdev->vq[i].used_idx);
3719
0
                return -1;
3720
0
            }
3721
0
        }
3722
0
    }
3723
3724
0
    if (vdc->post_load) {
3725
0
        ret = vdc->post_load(vdev);
3726
0
        if (ret) {
3727
0
            return ret;
3728
0
        }
3729
0
    }
3730
3731
0
    return 0;
3732
0
}
3733
3734
void virtio_cleanup(VirtIODevice *vdev)
3735
0
{
3736
0
    qemu_del_vm_change_state_handler(vdev->vmstate);
3737
0
}
3738
3739
static int virtio_vmstate_change(void *opaque, bool running, RunState state)
3740
21
{
3741
21
    VirtIODevice *vdev = opaque;
3742
21
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3743
21
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3744
21
    bool backend_run = running && virtio_device_started(vdev, vdev->status);
3745
21
    vdev->vm_running = running;
3746
3747
21
    if (backend_run) {
3748
0
        virtio_set_status(vdev, vdev->status);
3749
0
    }
3750
3751
21
    if (k->vmstate_change) {
3752
21
        k->vmstate_change(qbus->parent, backend_run);
3753
21
    }
3754
3755
21
    if (!backend_run) {
3756
21
        int ret = virtio_set_status(vdev, vdev->status);
3757
21
        if (ret) {
3758
0
            return ret;
3759
0
        }
3760
21
    }
3761
21
    return 0;
3762
21
}
3763
3764
void virtio_instance_init_common(Object *proxy_obj, void *data,
3765
                                 size_t vdev_size, const char *vdev_name)
3766
210
{
3767
210
    DeviceState *vdev = data;
3768
3769
210
    object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev,
3770
210
                                       vdev_size, vdev_name, &error_abort,
3771
210
                                       NULL);
3772
210
    qdev_alias_all_properties(vdev, proxy_obj);
3773
210
}
3774
3775
void virtio_init(VirtIODevice *vdev, uint16_t device_id, size_t config_size)
3776
21
{
3777
21
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3778
21
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3779
21
    int i;
3780
21
    int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
3781
3782
21
    if (nvectors) {
3783
21
        vdev->vector_queues =
3784
21
            g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
3785
21
    }
3786
3787
21
    vdev->start_on_kick = false;
3788
21
    vdev->started = false;
3789
21
    vdev->vhost_started = false;
3790
21
    vdev->device_id = device_id;
3791
21
    vdev->status = 0;
3792
21
    qatomic_set(&vdev->isr, 0);
3793
21
    vdev->queue_sel = 0;
3794
21
    vdev->config_vector = VIRTIO_NO_VECTOR;
3795
21
    vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX);
3796
21
    vdev->vm_running = runstate_is_running();
3797
21
    vdev->broken = false;
3798
21.5k
    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3799
21.5k
        vdev->vq[i].vector = VIRTIO_NO_VECTOR;
3800
21.5k
        vdev->vq[i].vdev = vdev;
3801
21.5k
        vdev->vq[i].queue_index = i;
3802
21.5k
        vdev->vq[i].host_notifier_enabled = false;
3803
21.5k
    }
3804
3805
21
    vdev->name = virtio_id_to_name(device_id);
3806
21
    vdev->config_len = config_size;
3807
21
    if (vdev->config_len) {
3808
20
        vdev->config = g_malloc0(config_size);
3809
20
    } else {
3810
1
        vdev->config = NULL;
3811
1
    }
3812
21
    vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev),
3813
21
            NULL, virtio_vmstate_change, vdev);
3814
21
    vdev->device_endian = virtio_default_endian();
3815
21
    vdev->use_guest_notifier_mask = true;
3816
21
    QSIMPLEQ_INIT(&vdev->shmem_list);
3817
21
}
3818
3819
/*
3820
 * Only devices that have already been around prior to defining the virtio
3821
 * standard support legacy mode; this includes devices not specified in the
3822
 * standard. All newer devices conform to the virtio standard only.
3823
 */
3824
bool virtio_legacy_allowed(VirtIODevice *vdev)
3825
15
{
3826
15
    switch (vdev->device_id) {
3827
2
    case VIRTIO_ID_NET:
3828
4
    case VIRTIO_ID_BLOCK:
3829
6
    case VIRTIO_ID_CONSOLE:
3830
7
    case VIRTIO_ID_RNG:
3831
9
    case VIRTIO_ID_BALLOON:
3832
9
    case VIRTIO_ID_RPMSG:
3833
11
    case VIRTIO_ID_SCSI:
3834
15
    case VIRTIO_ID_9P:
3835
15
    case VIRTIO_ID_RPROC_SERIAL:
3836
15
    case VIRTIO_ID_CAIF:
3837
15
        return true;
3838
0
    default:
3839
0
        return false;
3840
15
    }
3841
15
}
3842
3843
bool virtio_legacy_check_disabled(VirtIODevice *vdev)
3844
0
{
3845
0
    return vdev->disable_legacy_check;
3846
0
}
3847
3848
hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
3849
0
{
3850
0
    return vdev->vq[n].vring.desc;
3851
0
}
3852
3853
bool virtio_queue_enabled_legacy(VirtIODevice *vdev, int n)
3854
0
{
3855
0
    return virtio_queue_get_desc_addr(vdev, n) != 0;
3856
0
}
3857
3858
bool virtio_queue_enabled(VirtIODevice *vdev, int n)
3859
0
{
3860
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3861
0
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3862
3863
0
    if (k->queue_enabled) {
3864
0
        return k->queue_enabled(qbus->parent, n);
3865
0
    }
3866
0
    return virtio_queue_enabled_legacy(vdev, n);
3867
0
}
3868
3869
hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
3870
0
{
3871
0
    return vdev->vq[n].vring.avail;
3872
0
}
3873
3874
hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
3875
0
{
3876
0
    return vdev->vq[n].vring.used;
3877
0
}
3878
3879
hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
3880
1.44M
{
3881
1.44M
    return sizeof(VRingDesc) * vdev->vq[n].vring.num;
3882
1.44M
}
3883
3884
hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
3885
1.36M
{
3886
1.36M
    int s;
3887
3888
1.36M
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3889
0
        return sizeof(struct VRingPackedDescEvent);
3890
0
    }
3891
3892
1.36M
    s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
3893
1.36M
    return offsetof(VRingAvail, ring) +
3894
1.36M
        sizeof(uint16_t) * vdev->vq[n].vring.num + s;
3895
1.36M
}
3896
3897
hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
3898
1.37M
{
3899
1.37M
    int s;
3900
3901
1.37M
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3902
0
        return sizeof(struct VRingPackedDescEvent);
3903
0
    }
3904
3905
1.37M
    s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
3906
1.37M
    return offsetof(VRingUsed, ring) +
3907
1.37M
        sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s;
3908
1.37M
}
3909
3910
static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice *vdev,
3911
                                                           int n)
3912
0
{
3913
0
    unsigned int avail, used;
3914
3915
0
    avail = vdev->vq[n].last_avail_idx;
3916
0
    avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15;
3917
3918
0
    used = vdev->vq[n].used_idx;
3919
0
    used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15;
3920
3921
0
    return avail | used << 16;
3922
0
}
3923
3924
static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev,
3925
                                                      int n)
3926
0
{
3927
0
    return vdev->vq[n].last_avail_idx;
3928
0
}
3929
3930
unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
3931
0
{
3932
0
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3933
0
        return virtio_queue_packed_get_last_avail_idx(vdev, n);
3934
0
    } else {
3935
0
        return virtio_queue_split_get_last_avail_idx(vdev, n);
3936
0
    }
3937
0
}
3938
3939
static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev,
3940
                                                   int n, unsigned int idx)
3941
0
{
3942
0
    struct VirtQueue *vq = &vdev->vq[n];
3943
3944
0
    vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff;
3945
0
    vq->last_avail_wrap_counter =
3946
0
        vq->shadow_avail_wrap_counter = !!(idx & 0x8000);
3947
0
    idx >>= 16;
3948
0
    vq->used_idx = idx & 0x7fff;
3949
0
    vq->used_wrap_counter = !!(idx & 0x8000);
3950
0
}
3951
3952
static void virtio_queue_split_set_last_avail_idx(VirtIODevice *vdev,
3953
                                                  int n, unsigned int idx)
3954
0
{
3955
0
        vdev->vq[n].last_avail_idx = idx;
3956
0
        vdev->vq[n].shadow_avail_idx = idx;
3957
0
}
3958
3959
void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n,
3960
                                     unsigned int idx)
3961
0
{
3962
0
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3963
0
        virtio_queue_packed_set_last_avail_idx(vdev, n, idx);
3964
0
    } else {
3965
0
        virtio_queue_split_set_last_avail_idx(vdev, n, idx);
3966
0
    }
3967
0
}
3968
3969
static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice *vdev,
3970
                                                       int n)
3971
0
{
3972
    /* We don't have a reference like avail idx in shared memory */
3973
0
}
3974
3975
static void virtio_queue_split_restore_last_avail_idx(VirtIODevice *vdev,
3976
                                                      int n)
3977
0
{
3978
0
    RCU_READ_LOCK_GUARD();
3979
0
    if (vdev->vq[n].vring.desc) {
3980
0
        vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]);
3981
0
        vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx;
3982
0
    }
3983
0
}
3984
3985
void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n)
3986
0
{
3987
0
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3988
0
        virtio_queue_packed_restore_last_avail_idx(vdev, n);
3989
0
    } else {
3990
0
        virtio_queue_split_restore_last_avail_idx(vdev, n);
3991
0
    }
3992
0
}
3993
3994
static void virtio_queue_packed_update_used_idx(VirtIODevice *vdev, int n)
3995
0
{
3996
    /* used idx was updated through set_last_avail_idx() */
3997
0
}
3998
3999
static void virtio_queue_split_update_used_idx(VirtIODevice *vdev, int n)
4000
0
{
4001
0
    RCU_READ_LOCK_GUARD();
4002
0
    if (vdev->vq[n].vring.desc) {
4003
0
        vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]);
4004
0
    }
4005
0
}
4006
4007
void virtio_queue_update_used_idx(VirtIODevice *vdev, int n)
4008
0
{
4009
0
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
4010
0
        return virtio_queue_packed_update_used_idx(vdev, n);
4011
0
    } else {
4012
0
        return virtio_queue_split_update_used_idx(vdev, n);
4013
0
    }
4014
0
}
4015
4016
void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
4017
0
{
4018
0
    vdev->vq[n].signalled_used_valid = false;
4019
0
}
4020
4021
VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
4022
6.53M
{
4023
6.53M
    return vdev->vq + n;
4024
6.53M
}
4025
4026
uint16_t virtio_get_queue_index(VirtQueue *vq)
4027
2.10M
{
4028
2.10M
    return vq->queue_index;
4029
2.10M
}
4030
4031
static void virtio_queue_guest_notifier_read(EventNotifier *n)
4032
510k
{
4033
510k
    VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
4034
510k
    if (event_notifier_test_and_clear(n)) {
4035
0
        virtio_irq(vq);
4036
0
    }
4037
510k
}
4038
static void virtio_config_guest_notifier_read(EventNotifier *n)
4039
67.3k
{
4040
67.3k
    VirtIODevice *vdev = container_of(n, VirtIODevice, config_notifier);
4041
4042
67.3k
    if (event_notifier_test_and_clear(n)) {
4043
0
        virtio_notify_config(vdev);
4044
0
    }
4045
67.3k
}
4046
4047
int virtio_set_guest_notifier(VirtIODevice *vdev, int n, bool assign,
4048
                              bool with_irqfd)
4049
1.15M
{
4050
1.15M
    bool is_config = n == VIRTIO_CONFIG_IRQ_IDX;
4051
1.15M
    VirtQueue *vq = is_config ? NULL : virtio_get_queue(vdev, n);
4052
1.15M
    EventNotifier *notifier = is_config ?
4053
134k
        virtio_config_get_guest_notifier(vdev) :
4054
1.15M
        virtio_queue_get_guest_notifier(vq);
4055
1.15M
    EventNotifierHandler *read_fn = is_config ?
4056
134k
        virtio_config_guest_notifier_read :
4057
1.15M
        virtio_queue_guest_notifier_read;
4058
4059
1.15M
    if (assign) {
4060
577k
        int r = event_notifier_init(notifier, 0);
4061
577k
        if (r < 0) {
4062
0
            return r;
4063
0
        }
4064
577k
    }
4065
4066
1.15M
    event_notifier_set_handler(notifier,
4067
1.15M
                               (assign && !with_irqfd) ? read_fn : NULL);
4068
4069
1.15M
    if (!assign) {
4070
        /* Test and clear notifier before closing it,*/
4071
        /* in case poll callback didn't have time to run. */
4072
577k
        read_fn(notifier);
4073
577k
        event_notifier_cleanup(notifier);
4074
577k
    }
4075
4076
1.15M
    return 0;
4077
1.15M
}
4078
4079
EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
4080
1.02M
{
4081
1.02M
    return &vq->guest_notifier;
4082
1.02M
}
4083
4084
static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
4085
0
{
4086
0
    VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4087
4088
0
    virtio_queue_set_notification(vq, 0);
4089
0
}
4090
4091
static bool virtio_queue_host_notifier_aio_poll(void *opaque)
4092
0
{
4093
0
    EventNotifier *n = opaque;
4094
0
    VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4095
4096
0
    return vq->vring.desc && !virtio_queue_empty(vq);
4097
0
}
4098
4099
static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier *n)
4100
0
{
4101
0
    VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4102
4103
0
    virtio_queue_notify_vq(vq);
4104
0
}
4105
4106
static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
4107
0
{
4108
0
    VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4109
4110
    /* Caller polls once more after this to catch requests that race with us */
4111
0
    virtio_queue_set_notification(vq, 1);
4112
0
}
4113
4114
void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx)
4115
463k
{
4116
    /*
4117
     * virtio_queue_aio_detach_host_notifier() can leave notifications disabled.
4118
     * Re-enable them.  (And if detach has not been used before, notifications
4119
     * being enabled is still the default state while a notifier is attached;
4120
     * see virtio_queue_host_notifier_aio_poll_end(), which will always leave
4121
     * notifications enabled once the polling section is left.)
4122
     */
4123
463k
    if (!virtio_queue_get_notification(vq)) {
4124
0
        virtio_queue_set_notification(vq, 1);
4125
0
    }
4126
4127
463k
    aio_set_event_notifier(ctx, &vq->host_notifier,
4128
463k
                           virtio_queue_host_notifier_read,
4129
463k
                           virtio_queue_host_notifier_aio_poll,
4130
463k
                           virtio_queue_host_notifier_aio_poll_ready);
4131
463k
    aio_set_event_notifier_poll(ctx, &vq->host_notifier,
4132
463k
                                virtio_queue_host_notifier_aio_poll_begin,
4133
463k
                                virtio_queue_host_notifier_aio_poll_end);
4134
4135
    /*
4136
     * We will have ignored notifications about new requests from the guest
4137
     * while no notifiers were attached, so "kick" the virt queue to process
4138
     * those requests now.
4139
     */
4140
463k
    event_notifier_set(&vq->host_notifier);
4141
463k
}
4142
4143
/*
4144
 * Same as virtio_queue_aio_attach_host_notifier() but without polling. Use
4145
 * this for rx virtqueues and similar cases where the virtqueue handler
4146
 * function does not pop all elements. When the virtqueue is left non-empty
4147
 * polling consumes CPU cycles and should not be used.
4148
 */
4149
void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue *vq, AioContext *ctx)
4150
49.4k
{
4151
    /* See virtio_queue_aio_attach_host_notifier() */
4152
49.4k
    if (!virtio_queue_get_notification(vq)) {
4153
0
        virtio_queue_set_notification(vq, 1);
4154
0
    }
4155
4156
49.4k
    aio_set_event_notifier(ctx, &vq->host_notifier,
4157
49.4k
                           virtio_queue_host_notifier_read,
4158
49.4k
                           NULL, NULL);
4159
4160
    /*
4161
     * See virtio_queue_aio_attach_host_notifier().
4162
     * Note that this may be unnecessary for the type of virtqueues this
4163
     * function is used for.  Still, it will not hurt to have a quick look into
4164
     * whether we can/should process any of the virtqueue elements.
4165
     */
4166
49.4k
    event_notifier_set(&vq->host_notifier);
4167
49.4k
}
4168
4169
void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx)
4170
512k
{
4171
512k
    aio_set_event_notifier(ctx, &vq->host_notifier, NULL, NULL, NULL);
4172
4173
    /*
4174
     * aio_set_event_notifier_poll() does not guarantee whether io_poll_end()
4175
     * will run after io_poll_begin(), so by removing the notifier, we do not
4176
     * know whether virtio_queue_host_notifier_aio_poll_end() has run after a
4177
     * previous virtio_queue_host_notifier_aio_poll_begin(), i.e. whether
4178
     * notifications are enabled or disabled.  It does not really matter anyway;
4179
     * we just removed the notifier, so we do not care about notifications until
4180
     * we potentially re-attach it.  The attach_host_notifier functions will
4181
     * ensure that notifications are enabled again when they are needed.
4182
     */
4183
512k
}
4184
4185
void virtio_queue_host_notifier_read(EventNotifier *n)
4186
2.54M
{
4187
2.54M
    VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4188
2.54M
    if (event_notifier_test_and_clear(n)) {
4189
1.03M
        virtio_queue_notify_vq(vq);
4190
1.03M
    }
4191
2.54M
}
4192
4193
EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
4194
3.50M
{
4195
3.50M
    return &vq->host_notifier;
4196
3.50M
}
4197
4198
EventNotifier *virtio_config_get_guest_notifier(VirtIODevice *vdev)
4199
134k
{
4200
134k
    return &vdev->config_notifier;
4201
134k
}
4202
4203
void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled)
4204
1.99M
{
4205
1.99M
    vq->host_notifier_enabled = enabled;
4206
1.99M
}
4207
4208
int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n,
4209
                                      MemoryRegion *mr, bool assign)
4210
0
{
4211
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4212
0
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
4213
4214
0
    if (k->set_host_notifier_mr) {
4215
0
        return k->set_host_notifier_mr(qbus->parent, n, mr, assign);
4216
0
    }
4217
4218
0
    return -1;
4219
0
}
4220
4221
void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
4222
0
{
4223
0
    g_free(vdev->bus_name);
4224
0
    vdev->bus_name = g_strdup(bus_name);
4225
0
}
4226
4227
void G_GNUC_PRINTF(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...)
4228
378k
{
4229
378k
    va_list ap;
4230
4231
378k
    va_start(ap, fmt);
4232
378k
    error_vreport(fmt, ap);
4233
378k
    va_end(ap);
4234
4235
378k
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
4236
44.4k
        vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET;
4237
44.4k
        virtio_notify_config(vdev);
4238
44.4k
    }
4239
4240
378k
    vdev->broken = true;
4241
378k
}
4242
4243
static void virtio_memory_listener_commit(MemoryListener *listener)
4244
693k
{
4245
693k
    VirtIODevice *vdev = container_of(listener, VirtIODevice, listener);
4246
693k
    int i;
4247
4248
5.95M
    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
4249
5.95M
        if (vdev->vq[i].vring.num == 0) {
4250
693k
            break;
4251
693k
        }
4252
5.26M
        virtio_init_region_cache(vdev, i);
4253
5.26M
    }
4254
693k
}
4255
4256
static void virtio_device_realize(DeviceState *dev, Error **errp)
4257
21
{
4258
21
    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
4259
21
    VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
4260
21
    Error *err = NULL;
4261
4262
    /* Devices should either use vmsd or the load/save methods */
4263
21
    assert(!vdc->vmsd || !vdc->load);
4264
4265
21
    if (vdc->realize != NULL) {
4266
21
        vdc->realize(dev, &err);
4267
21
        if (err != NULL) {
4268
0
            error_propagate(errp, err);
4269
0
            return;
4270
0
        }
4271
21
    }
4272
4273
    /* Devices should not use both ioeventfd and notification data feature */
4274
21
    virtio_device_check_notification_compatibility(vdev, &err);
4275
21
    if (err != NULL) {
4276
0
        error_propagate(errp, err);
4277
0
        vdc->unrealize(dev);
4278
0
        return;
4279
0
    }
4280
4281
21
    virtio_bus_device_plugged(vdev, &err);
4282
21
    if (err != NULL) {
4283
0
        error_propagate(errp, err);
4284
0
        vdc->unrealize(dev);
4285
0
        return;
4286
0
    }
4287
4288
21
    vdev->listener.commit = virtio_memory_listener_commit;
4289
21
    vdev->listener.name = "virtio";
4290
21
    memory_listener_register(&vdev->listener, vdev->dma_as);
4291
21
}
4292
4293
static void virtio_device_unrealize(DeviceState *dev)
4294
0
{
4295
0
    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
4296
0
    VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
4297
4298
0
    memory_listener_unregister(&vdev->listener);
4299
0
    virtio_bus_device_unplugged(vdev);
4300
4301
0
    if (vdc->unrealize != NULL) {
4302
0
        vdc->unrealize(dev);
4303
0
    }
4304
4305
0
    g_free(vdev->bus_name);
4306
0
    vdev->bus_name = NULL;
4307
0
}
4308
4309
static void virtio_device_free_virtqueues(VirtIODevice *vdev)
4310
189
{
4311
189
    int i;
4312
189
    if (!vdev->vq) {
4313
189
        return;
4314
189
    }
4315
4316
0
    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
4317
0
        if (vdev->vq[i].vring.num == 0) {
4318
0
            break;
4319
0
        }
4320
0
        virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
4321
0
    }
4322
0
    g_free(vdev->vq);
4323
0
}
4324
4325
static void virtio_device_instance_finalize(Object *obj)
4326
189
{
4327
189
    VirtIODevice *vdev = VIRTIO_DEVICE(obj);
4328
189
    VirtioSharedMemory *shmem;
4329
4330
189
    virtio_device_free_virtqueues(vdev);
4331
4332
189
    g_free(vdev->config);
4333
189
    g_free(vdev->vector_queues);
4334
189
    while (!QSIMPLEQ_EMPTY(&vdev->shmem_list)) {
4335
0
        shmem = QSIMPLEQ_FIRST(&vdev->shmem_list);
4336
0
        while (!QTAILQ_EMPTY(&shmem->mmaps)) {
4337
0
            VirtioSharedMemoryMapping *mapping = QTAILQ_FIRST(&shmem->mmaps);
4338
0
            virtio_del_shmem_map(shmem, mapping->offset,
4339
0
                                 memory_region_size(mapping->mr));
4340
0
        }
4341
4342
        /* Clean up the embedded MemoryRegion */
4343
0
        object_unparent(OBJECT(&shmem->mr));
4344
0
        QSIMPLEQ_REMOVE_HEAD(&vdev->shmem_list, entry);
4345
0
        g_free(shmem);
4346
0
    }
4347
189
}
4348
4349
static const Property virtio_properties[] = {
4350
    DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
4351
    DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true),
4352
    DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true),
4353
    DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice,
4354
                     disable_legacy_check, false),
4355
};
4356
4357
static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
4358
143k
{
4359
143k
    VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
4360
143k
    int i, n, r, err;
4361
4362
    /*
4363
     * Batch all the host notifiers in a single transaction to avoid
4364
     * quadratic time complexity in address_space_update_ioeventfds().
4365
     */
4366
143k
    memory_region_transaction_begin();
4367
147M
    for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
4368
146M
        VirtQueue *vq = &vdev->vq[n];
4369
146M
        if (!virtio_queue_get_num(vdev, n)) {
4370
146M
            continue;
4371
146M
        }
4372
486k
        r = virtio_bus_set_host_notifier(qbus, n, true);
4373
486k
        if (r < 0) {
4374
0
            err = r;
4375
0
            goto assign_error;
4376
0
        }
4377
486k
        event_notifier_set_handler(&vq->host_notifier,
4378
486k
                                   virtio_queue_host_notifier_read);
4379
486k
    }
4380
4381
147M
    for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
4382
        /* Kick right away to begin processing requests already in vring */
4383
146M
        VirtQueue *vq = &vdev->vq[n];
4384
146M
        if (!vq->vring.num) {
4385
146M
            continue;
4386
146M
        }
4387
486k
        event_notifier_set(&vq->host_notifier);
4388
486k
    }
4389
143k
    memory_region_transaction_commit();
4390
143k
    return 0;
4391
4392
0
assign_error:
4393
0
    i = n; /* save n for a second iteration after transaction is committed. */
4394
0
    while (--n >= 0) {
4395
0
        VirtQueue *vq = &vdev->vq[n];
4396
0
        if (!virtio_queue_get_num(vdev, n)) {
4397
0
            continue;
4398
0
        }
4399
4400
0
        event_notifier_set_handler(&vq->host_notifier, NULL);
4401
0
        r = virtio_bus_set_host_notifier(qbus, n, false);
4402
0
        assert(r >= 0);
4403
0
    }
4404
    /*
4405
     * The transaction expects the ioeventfds to be open when it
4406
     * commits. Do it now, before the cleanup loop.
4407
     */
4408
0
    memory_region_transaction_commit();
4409
4410
0
    while (--i >= 0) {
4411
0
        if (!virtio_queue_get_num(vdev, i)) {
4412
0
            continue;
4413
0
        }
4414
0
        virtio_bus_cleanup_host_notifier(qbus, i);
4415
0
    }
4416
0
    return err;
4417
0
}
4418
4419
int virtio_device_start_ioeventfd(VirtIODevice *vdev)
4420
3.33k
{
4421
3.33k
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4422
3.33k
    VirtioBusState *vbus = VIRTIO_BUS(qbus);
4423
4424
3.33k
    return virtio_bus_start_ioeventfd(vbus);
4425
3.33k
}
4426
4427
static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
4428
143k
{
4429
143k
    VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
4430
143k
    int n, r;
4431
4432
    /*
4433
     * Batch all the host notifiers in a single transaction to avoid
4434
     * quadratic time complexity in address_space_update_ioeventfds().
4435
     */
4436
143k
    memory_region_transaction_begin();
4437
147M
    for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
4438
146M
        VirtQueue *vq = &vdev->vq[n];
4439
4440
146M
        if (!virtio_queue_get_num(vdev, n)) {
4441
146M
            continue;
4442
146M
        }
4443
486k
        event_notifier_set_handler(&vq->host_notifier, NULL);
4444
486k
        r = virtio_bus_set_host_notifier(qbus, n, false);
4445
486k
        assert(r >= 0);
4446
486k
    }
4447
    /*
4448
     * The transaction expects the ioeventfds to be open when it
4449
     * commits. Do it now, before the cleanup loop.
4450
     */
4451
143k
    memory_region_transaction_commit();
4452
4453
147M
    for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
4454
146M
        if (!virtio_queue_get_num(vdev, n)) {
4455
146M
            continue;
4456
146M
        }
4457
486k
        virtio_bus_cleanup_host_notifier(qbus, n);
4458
486k
    }
4459
143k
}
4460
4461
int virtio_device_grab_ioeventfd(VirtIODevice *vdev)
4462
0
{
4463
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4464
0
    VirtioBusState *vbus = VIRTIO_BUS(qbus);
4465
4466
0
    return virtio_bus_grab_ioeventfd(vbus);
4467
0
}
4468
4469
void virtio_device_release_ioeventfd(VirtIODevice *vdev)
4470
0
{
4471
0
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4472
0
    VirtioBusState *vbus = VIRTIO_BUS(qbus);
4473
4474
0
    virtio_bus_release_ioeventfd(vbus);
4475
0
}
4476
4477
static void virtio_device_class_init(ObjectClass *klass, const void *data)
4478
63
{
4479
    /* Set the default value here. */
4480
63
    VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
4481
63
    DeviceClass *dc = DEVICE_CLASS(klass);
4482
4483
63
    dc->realize = virtio_device_realize;
4484
63
    dc->unrealize = virtio_device_unrealize;
4485
63
    dc->bus_type = TYPE_VIRTIO_BUS;
4486
63
    device_class_set_props(dc, virtio_properties);
4487
63
    vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl;
4488
63
    vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl;
4489
4490
63
    vdc->legacy_features |= VIRTIO_LEGACY_FEATURES;
4491
63
}
4492
4493
bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev)
4494
2
{
4495
2
    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4496
2
    VirtioBusState *vbus = VIRTIO_BUS(qbus);
4497
4498
2
    return virtio_bus_ioeventfd_enabled(vbus);
4499
2
}
4500
4501
VirtQueueStatus *qmp_x_query_virtio_queue_status(const char *path,
4502
                                                 uint16_t queue,
4503
                                                 Error **errp)
4504
0
{
4505
0
    VirtIODevice *vdev;
4506
0
    VirtQueueStatus *status;
4507
4508
0
    vdev = qmp_find_virtio_device(path);
4509
0
    if (vdev == NULL) {
4510
0
        error_setg(errp, "Path %s is not a VirtIODevice", path);
4511
0
        return NULL;
4512
0
    }
4513
4514
0
    if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) {
4515
0
        error_setg(errp, "Invalid virtqueue number %d", queue);
4516
0
        return NULL;
4517
0
    }
4518
4519
0
    status = g_new0(VirtQueueStatus, 1);
4520
0
    status->name = g_strdup(vdev->name);
4521
0
    status->queue_index = vdev->vq[queue].queue_index;
4522
0
    status->inuse = vdev->vq[queue].inuse;
4523
0
    status->vring_num = vdev->vq[queue].vring.num;
4524
0
    status->vring_num_default = vdev->vq[queue].vring.num_default;
4525
0
    status->vring_align = vdev->vq[queue].vring.align;
4526
0
    status->vring_desc = vdev->vq[queue].vring.desc;
4527
0
    status->vring_avail = vdev->vq[queue].vring.avail;
4528
0
    status->vring_used = vdev->vq[queue].vring.used;
4529
0
    status->used_idx = vdev->vq[queue].used_idx;
4530
0
    status->signalled_used = vdev->vq[queue].signalled_used;
4531
0
    status->signalled_used_valid = vdev->vq[queue].signalled_used_valid;
4532
4533
0
    if (vdev->vhost_started) {
4534
0
        VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
4535
0
        struct vhost_dev *hdev = vdc->get_vhost(vdev);
4536
4537
        /* check if vq index exists for vhost as well  */
4538
0
        if (queue >= hdev->vq_index && queue < hdev->vq_index + hdev->nvqs) {
4539
0
            status->has_last_avail_idx = true;
4540
4541
0
            int vhost_vq_index =
4542
0
                hdev->vhost_ops->vhost_get_vq_index(hdev, queue);
4543
0
            struct vhost_vring_state state = {
4544
0
                .index = vhost_vq_index,
4545
0
            };
4546
4547
0
            status->last_avail_idx =
4548
0
                hdev->vhost_ops->vhost_get_vring_base(hdev, &state);
4549
0
        }
4550
0
    } else {
4551
0
        status->has_shadow_avail_idx = true;
4552
0
        status->has_last_avail_idx = true;
4553
0
        status->last_avail_idx = vdev->vq[queue].last_avail_idx;
4554
0
        status->shadow_avail_idx = vdev->vq[queue].shadow_avail_idx;
4555
0
    }
4556
4557
0
    return status;
4558
0
}
4559
4560
static strList *qmp_decode_vring_desc_flags(uint16_t flags)
4561
0
{
4562
0
    strList *list = NULL;
4563
0
    strList *node;
4564
0
    int i;
4565
4566
0
    struct {
4567
0
        uint16_t flag;
4568
0
        const char *value;
4569
0
    } map[] = {
4570
0
        { VRING_DESC_F_NEXT, "next" },
4571
0
        { VRING_DESC_F_WRITE, "write" },
4572
0
        { VRING_DESC_F_INDIRECT, "indirect" },
4573
0
        { 1 << VRING_PACKED_DESC_F_AVAIL, "avail" },
4574
0
        { 1 << VRING_PACKED_DESC_F_USED, "used" },
4575
0
        { 0, "" }
4576
0
    };
4577
4578
0
    for (i = 0; map[i].flag; i++) {
4579
0
        if ((map[i].flag & flags) == 0) {
4580
0
            continue;
4581
0
        }
4582
0
        node = g_malloc0(sizeof(strList));
4583
0
        node->value = g_strdup(map[i].value);
4584
0
        node->next = list;
4585
0
        list = node;
4586
0
    }
4587
4588
0
    return list;
4589
0
}
4590
4591
VirtioQueueElement *qmp_x_query_virtio_queue_element(const char *path,
4592
                                                     uint16_t queue,
4593
                                                     bool has_index,
4594
                                                     uint16_t index,
4595
                                                     Error **errp)
4596
0
{
4597
0
    VirtIODevice *vdev;
4598
0
    VirtQueue *vq;
4599
0
    VirtioQueueElement *element = NULL;
4600
4601
0
    vdev = qmp_find_virtio_device(path);
4602
0
    if (vdev == NULL) {
4603
0
        error_setg(errp, "Path %s is not a VirtIO device", path);
4604
0
        return NULL;
4605
0
    }
4606
4607
0
    if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) {
4608
0
        error_setg(errp, "Invalid virtqueue number %d", queue);
4609
0
        return NULL;
4610
0
    }
4611
0
    vq = &vdev->vq[queue];
4612
4613
0
    if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
4614
0
        error_setg(errp, "Packed ring not supported");
4615
0
        return NULL;
4616
0
    } else {
4617
0
        unsigned int head, i, max;
4618
0
        VRingMemoryRegionCaches *caches;
4619
0
        MemoryRegionCache indirect_desc_cache;
4620
0
        MemoryRegionCache *desc_cache;
4621
0
        VRingDesc desc;
4622
0
        VirtioRingDescList *list = NULL;
4623
0
        VirtioRingDescList *node;
4624
0
        int rc; int ndescs;
4625
4626
0
        address_space_cache_init_empty(&indirect_desc_cache);
4627
4628
0
        RCU_READ_LOCK_GUARD();
4629
4630
0
        max = vq->vring.num;
4631
4632
0
        if (!has_index) {
4633
0
            head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num);
4634
0
        } else {
4635
0
            head = vring_avail_ring(vq, index % vq->vring.num);
4636
0
        }
4637
0
        i = head;
4638
4639
0
        caches = vring_get_region_caches(vq);
4640
0
        if (!caches) {
4641
0
            error_setg(errp, "Region caches not initialized");
4642
0
            return NULL;
4643
0
        }
4644
0
        if (caches->desc.len < max * sizeof(VRingDesc)) {
4645
0
            error_setg(errp, "Cannot map descriptor ring");
4646
0
            return NULL;
4647
0
        }
4648
4649
0
        desc_cache = &caches->desc;
4650
0
        vring_split_desc_read(vdev, &desc, desc_cache, i);
4651
0
        if (desc.flags & VRING_DESC_F_INDIRECT) {
4652
0
            int64_t len;
4653
0
            len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
4654
0
                                           desc.addr, desc.len, false);
4655
0
            desc_cache = &indirect_desc_cache;
4656
0
            if (len < desc.len) {
4657
0
                error_setg(errp, "Cannot map indirect buffer");
4658
0
                goto done;
4659
0
            }
4660
4661
0
            max = desc.len / sizeof(VRingDesc);
4662
0
            i = 0;
4663
0
            vring_split_desc_read(vdev, &desc, desc_cache, i);
4664
0
        }
4665
4666
0
        element = g_new0(VirtioQueueElement, 1);
4667
0
        element->avail = g_new0(VirtioRingAvail, 1);
4668
0
        element->used = g_new0(VirtioRingUsed, 1);
4669
0
        element->name = g_strdup(vdev->name);
4670
0
        element->index = head;
4671
0
        element->avail->flags = vring_avail_flags(vq);
4672
0
        element->avail->idx = vring_avail_idx(vq);
4673
0
        element->avail->ring = head;
4674
0
        element->used->flags = vring_used_flags(vq);
4675
0
        element->used->idx = vring_used_idx(vq);
4676
0
        ndescs = 0;
4677
4678
0
        do {
4679
            /* A buggy driver may produce an infinite loop */
4680
0
            if (ndescs >= max) {
4681
0
                break;
4682
0
            }
4683
0
            node = g_new0(VirtioRingDescList, 1);
4684
0
            node->value = g_new0(VirtioRingDesc, 1);
4685
0
            node->value->addr = desc.addr;
4686
0
            node->value->len = desc.len;
4687
0
            node->value->flags = qmp_decode_vring_desc_flags(desc.flags);
4688
0
            node->next = list;
4689
0
            list = node;
4690
4691
0
            ndescs++;
4692
0
            rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max);
4693
0
        } while (rc == VIRTQUEUE_READ_DESC_MORE);
4694
0
        element->descs = list;
4695
0
done:
4696
0
        address_space_cache_destroy(&indirect_desc_cache);
4697
0
    }
4698
4699
0
    return element;
4700
0
}
4701
4702
static const TypeInfo virtio_device_info = {
4703
    .name = TYPE_VIRTIO_DEVICE,
4704
    .parent = TYPE_DEVICE,
4705
    .instance_size = sizeof(VirtIODevice),
4706
    .class_init = virtio_device_class_init,
4707
    .instance_finalize = virtio_device_instance_finalize,
4708
    .abstract = true,
4709
    .class_size = sizeof(VirtioDeviceClass),
4710
};
4711
4712
static const TypeInfo virtio_shared_memory_mapping_info = {
4713
    .name = TYPE_VIRTIO_SHARED_MEMORY_MAPPING,
4714
    .parent = TYPE_OBJECT,
4715
    .instance_size = sizeof(VirtioSharedMemoryMapping),
4716
    .instance_init = virtio_shared_memory_mapping_instance_init,
4717
    .instance_finalize = virtio_shared_memory_mapping_instance_finalize,
4718
};
4719
4720
static void virtio_register_types(void)
4721
63
{
4722
63
    type_register_static(&virtio_device_info);
4723
63
    type_register_static(&virtio_shared_memory_mapping_info);
4724
63
}
4725
4726
type_init(virtio_register_types)
4727
4728
QEMUBH *virtio_bh_new_guarded_full(DeviceState *dev,
4729
                                   QEMUBHFunc *cb, void *opaque,
4730
                                   const char *name)
4731
2
{
4732
2
    DeviceState *transport = qdev_get_parent_bus(dev)->parent;
4733
4734
2
    return qemu_bh_new_full(cb, opaque, name,
4735
2
                            &transport->mem_reentrancy_guard);
4736
2
}
4737
4738
QEMUBH *virtio_bh_io_new_guarded_full(DeviceState *dev,
4739
                                      QEMUBHFunc *cb, void *opaque,
4740
                                      const char *name)
4741
12
{
4742
12
    DeviceState *transport = qdev_get_parent_bus(dev)->parent;
4743
4744
12
    return aio_bh_new_full(iohandler_get_aio_context(), cb, opaque, name,
4745
12
                           &transport->mem_reentrancy_guard);
4746
12
}