Coverage Report

Created: 2026-10-31 08:12

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qemu/hw/virtio/vhost.c
Line
Count
Source
1
/*
2
 * vhost support
3
 *
4
 * Copyright Red Hat, Inc. 2010
5
 *
6
 * Authors:
7
 *  Michael S. Tsirkin <mst@redhat.com>
8
 *
9
 * This work is licensed under the terms of the GNU GPL, version 2.  See
10
 * the COPYING file in the top-level directory.
11
 *
12
 * Contributions after 2012-01-13 are licensed under the terms of the
13
 * GNU GPL, version 2 or (at your option) any later version.
14
 */
15
16
#include "qemu/osdep.h"
17
#include "qapi/error.h"
18
#include "hw/virtio/vhost.h"
19
#include "qemu/atomic.h"
20
#include "qemu/range.h"
21
#include "qemu/error-report.h"
22
#include "qemu/memfd.h"
23
#include "qemu/log.h"
24
#include "standard-headers/linux/vhost_types.h"
25
#include "hw/virtio/virtio-bus.h"
26
#include "hw/mem/memory-device.h"
27
#include "migration/blocker.h"
28
#include "migration/misc.h"
29
#include "migration/qemu-file-types.h"
30
#include "system/dma.h"
31
#include "system/memory.h"
32
#include "system/ramblock.h"
33
#include "system/system.h"
34
#include "trace.h"
35
36
/* enabled until disconnected backend stabilizes */
37
#define _VHOST_DEBUG 1
38
39
#ifdef _VHOST_DEBUG
40
#define VHOST_OPS_DEBUG(retval, fmt, ...) \
41
0
    do { \
42
0
        error_report(fmt ": %s (%d)", ## __VA_ARGS__, \
43
0
                     strerror(-retval), -retval); \
44
0
    } while (0)
45
#else
46
#define VHOST_OPS_DEBUG(retval, fmt, ...) \
47
    do { } while (0)
48
#endif
49
50
static struct vhost_log *vhost_log[VHOST_BACKEND_TYPE_MAX];
51
static struct vhost_log *vhost_log_shm[VHOST_BACKEND_TYPE_MAX];
52
static QLIST_HEAD(, vhost_dev) vhost_log_devs[VHOST_BACKEND_TYPE_MAX];
53
54
static QLIST_HEAD(, vhost_dev) vhost_devices =
55
    QLIST_HEAD_INITIALIZER(vhost_devices);
56
57
/*
58
 * Release the device ownership if it was handed over to us by another
59
 * process, e.g. the CPR source.
60
 */
61
static void vhost_dev_try_release_owner(struct vhost_dev *hdev,
62
                                        const char *when)
63
0
{
64
0
    int r;
65
66
0
    if (!hdev->owner || !hdev->owner_handed_over) {
67
0
        return;
68
0
    }
69
0
    r = vhost_dev_reset_owner(hdev);
70
0
    if (r < 0) {
71
0
        error_report("vhost: cannot release device ownership on %s: %s",
72
0
                     when, strerror(-r));
73
0
    }
74
0
}
75
76
/*
77
 * A CPR target takes the device ownership in post_load().  If the load
78
 * fails afterwards, release it before the source learns about the failure
79
 * and resumes, so that the source can take the device back.
80
 */
81
static int vhost_migration_notify(NotifierWithReturn *notifier,
82
                                  MigrationEvent *e, Error **errp)
83
0
{
84
0
    struct vhost_dev *hdev = container_of(notifier, struct vhost_dev,
85
0
                                          migration_notifier);
86
87
0
    if (e->type == MIG_EVENT_INCOMING_FAILED) {
88
0
        vhost_dev_try_release_owner(hdev, "incoming migration failure");
89
0
    }
90
0
    return 0;
91
0
}
92
93
/*
94
 * Release the ownership of the vhost devices still held at exit(), so that
95
 * another process sharing the backend FDs (e.g. the CPR source) can take
96
 * them over.
97
 */
98
static void vhost_exit_notify(Notifier *notifier, void *data)
99
0
{
100
0
    struct vhost_dev *hdev = container_of(notifier, struct vhost_dev,
101
0
                                          exit_notifier);
102
103
0
    vhost_dev_try_release_owner(hdev, "exit");
104
0
}
105
106
unsigned int vhost_get_max_memslots(void)
107
0
{
108
0
    unsigned int max = UINT_MAX;
109
0
    struct vhost_dev *hdev;
110
111
0
    QLIST_FOREACH(hdev, &vhost_devices, entry) {
112
0
        max = MIN(max, hdev->vhost_ops->vhost_memslots_limit(hdev));
113
0
    }
114
0
    return max;
115
0
}
116
117
unsigned int vhost_get_free_memslots(void)
118
0
{
119
0
    unsigned int free = UINT_MAX;
120
0
    struct vhost_dev *hdev;
121
122
0
    QLIST_FOREACH(hdev, &vhost_devices, entry) {
123
0
        unsigned int r = hdev->vhost_ops->vhost_memslots_limit(hdev);
124
0
        unsigned int cur_free = r - hdev->mem->nregions;
125
126
0
        if (unlikely(r < hdev->mem->nregions)) {
127
0
            warn_report_once("used (%u) vhost backend memory slots exceed"
128
0
                             " the device limit (%u).", hdev->mem->nregions, r);
129
0
            free = 0;
130
0
        } else {
131
0
            free = MIN(free, cur_free);
132
0
        }
133
0
    }
134
0
    return free;
135
0
}
136
137
static void vhost_dev_sync_region(struct vhost_dev *dev,
138
                                  MemoryRegionSection *section,
139
                                  uint64_t mfirst, uint64_t mlast,
140
                                  uint64_t rfirst, uint64_t rlast)
141
0
{
142
0
    vhost_log_chunk_t *dev_log = dev->log->log;
143
144
0
    uint64_t start = MAX(mfirst, rfirst);
145
0
    uint64_t end = MIN(mlast, rlast);
146
0
    vhost_log_chunk_t *from = dev_log + start / VHOST_LOG_CHUNK;
147
0
    vhost_log_chunk_t *to = dev_log + end / VHOST_LOG_CHUNK + 1;
148
0
    uint64_t addr = QEMU_ALIGN_DOWN(start, VHOST_LOG_CHUNK);
149
150
0
    if (end < start) {
151
0
        return;
152
0
    }
153
0
    assert(end / VHOST_LOG_CHUNK < dev->log_size);
154
0
    assert(start / VHOST_LOG_CHUNK < dev->log_size);
155
156
0
    for (;from < to; ++from) {
157
0
        vhost_log_chunk_t log;
158
        /* We first check with non-atomic: much cheaper,
159
         * and we expect non-dirty to be the common case. */
160
0
        if (!*from) {
161
0
            addr += VHOST_LOG_CHUNK;
162
0
            continue;
163
0
        }
164
        /* Data must be read atomically. We don't really need barrier semantics
165
         * but it's easier to use atomic_* than roll our own. */
166
0
        log = qatomic_xchg(from, 0);
167
0
        while (log) {
168
0
            int bit = ctzl(log);
169
0
            hwaddr page_addr;
170
0
            hwaddr section_offset;
171
0
            hwaddr mr_offset;
172
0
            page_addr = addr + bit * VHOST_LOG_PAGE;
173
0
            section_offset = page_addr - section->offset_within_address_space;
174
0
            mr_offset = section_offset + section->offset_within_region;
175
0
            memory_region_set_dirty(section->mr, mr_offset, VHOST_LOG_PAGE);
176
0
            log &= ~(0x1ull << bit);
177
0
        }
178
0
        addr += VHOST_LOG_CHUNK;
179
0
    }
180
0
}
181
182
bool vhost_dev_has_iommu(struct vhost_dev *dev)
183
0
{
184
0
    VirtIODevice *vdev = dev->vdev;
185
186
    /*
187
     * For vhost, VIRTIO_F_IOMMU_PLATFORM means the backend support
188
     * incremental memory mapping API via IOTLB API. For platform that
189
     * does not have IOMMU, there's no need to enable this feature
190
     * which may cause unnecessary IOTLB miss/update transactions.
191
     */
192
0
    if (vdev) {
193
0
        return virtio_bus_device_iommu_enabled(vdev) &&
194
0
            virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM);
195
0
    } else {
196
0
        return false;
197
0
    }
198
0
}
199
200
static inline bool vhost_dev_should_log(struct vhost_dev *dev)
201
0
{
202
0
    assert(dev->vhost_ops);
203
0
    assert(dev->vhost_ops->backend_type > VHOST_BACKEND_TYPE_NONE);
204
0
    assert(dev->vhost_ops->backend_type < VHOST_BACKEND_TYPE_MAX);
205
206
0
    return dev == QLIST_FIRST(&vhost_log_devs[dev->vhost_ops->backend_type]);
207
0
}
208
209
static inline void vhost_dev_elect_mem_logger(struct vhost_dev *hdev, bool add)
210
0
{
211
0
    VhostBackendType backend_type;
212
213
0
    assert(hdev->vhost_ops);
214
215
0
    backend_type = hdev->vhost_ops->backend_type;
216
0
    assert(backend_type > VHOST_BACKEND_TYPE_NONE);
217
0
    assert(backend_type < VHOST_BACKEND_TYPE_MAX);
218
219
0
    if (add && !QLIST_IS_INSERTED(hdev, logdev_entry)) {
220
0
        if (QLIST_EMPTY(&vhost_log_devs[backend_type])) {
221
0
            QLIST_INSERT_HEAD(&vhost_log_devs[backend_type],
222
0
                              hdev, logdev_entry);
223
0
        } else {
224
            /*
225
             * The first vhost_device in the list is selected as the shared
226
             * logger to scan memory sections. Put new entry next to the head
227
             * to avoid inadvertent change to the underlying logger device.
228
             * This is done in order to get better cache locality and to avoid
229
             * performance churn on the hot path for log scanning. Even when
230
             * new devices come and go quickly, it wouldn't end up changing
231
             * the active leading logger device at all.
232
             */
233
0
            QLIST_INSERT_AFTER(QLIST_FIRST(&vhost_log_devs[backend_type]),
234
0
                               hdev, logdev_entry);
235
0
        }
236
0
    } else if (!add && QLIST_IS_INSERTED(hdev, logdev_entry)) {
237
0
        QLIST_REMOVE(hdev, logdev_entry);
238
0
    }
239
0
}
240
241
static int vhost_sync_dirty_bitmap(struct vhost_dev *dev,
242
                                   MemoryRegionSection *section,
243
                                   hwaddr first,
244
                                   hwaddr last)
245
0
{
246
0
    int i;
247
0
    hwaddr start_addr;
248
0
    hwaddr end_addr;
249
250
0
    if (!dev->log_enabled || !dev->started) {
251
0
        return 0;
252
0
    }
253
0
    start_addr = section->offset_within_address_space;
254
0
    end_addr = range_get_last(start_addr, int128_get64(section->size));
255
0
    start_addr = MAX(first, start_addr);
256
0
    end_addr = MIN(last, end_addr);
257
258
0
    if (vhost_dev_should_log(dev)) {
259
0
        for (i = 0; i < dev->mem->nregions; ++i) {
260
0
            struct vhost_memory_region *reg = dev->mem->regions + i;
261
0
            vhost_dev_sync_region(dev, section, start_addr, end_addr,
262
0
                                  reg->guest_phys_addr,
263
0
                                  range_get_last(reg->guest_phys_addr,
264
0
                                                 reg->memory_size));
265
0
        }
266
0
    }
267
0
    for (i = 0; i < dev->nvqs; ++i) {
268
0
        struct vhost_virtqueue *vq = dev->vqs + i;
269
270
0
        if (!vq->used_phys && !vq->used_size) {
271
0
            continue;
272
0
        }
273
274
0
        if (vhost_dev_has_iommu(dev)) {
275
0
            IOMMUTLBEntry iotlb;
276
0
            hwaddr used_phys = vq->used_phys, used_size = vq->used_size;
277
0
            hwaddr phys, s, offset;
278
279
0
            while (used_size) {
280
0
                rcu_read_lock();
281
0
                iotlb = address_space_get_iotlb_entry(dev->vdev->dma_as,
282
0
                                                      used_phys,
283
0
                                                      true,
284
0
                                                      MEMTXATTRS_UNSPECIFIED);
285
0
                rcu_read_unlock();
286
287
0
                if (!iotlb.target_as) {
288
0
                    qemu_log_mask(LOG_GUEST_ERROR, "translation "
289
0
                                  "failure for used_iova %"PRIx64"\n",
290
0
                                  used_phys);
291
0
                    return -EINVAL;
292
0
                }
293
294
0
                offset = used_phys & iotlb.addr_mask;
295
0
                phys = iotlb.translated_addr + offset;
296
297
                /*
298
                 * Distance from start of used ring until last byte of
299
                 * IOMMU page.
300
                 */
301
0
                s = iotlb.addr_mask - offset;
302
                /*
303
                 * Size of used ring, or of the part of it until end
304
                 * of IOMMU page. To avoid zero result, do the adding
305
                 * outside of MIN().
306
                 */
307
0
                s = MIN(s, used_size - 1) + 1;
308
309
0
                vhost_dev_sync_region(dev, section, start_addr, end_addr, phys,
310
0
                                      range_get_last(phys, s));
311
0
                used_size -= s;
312
0
                used_phys += s;
313
0
            }
314
0
        } else {
315
0
            vhost_dev_sync_region(dev, section, start_addr,
316
0
                                  end_addr, vq->used_phys,
317
0
                                  range_get_last(vq->used_phys, vq->used_size));
318
0
        }
319
0
    }
320
0
    return 0;
321
0
}
322
323
static void vhost_log_sync(MemoryListener *listener,
324
                          MemoryRegionSection *section)
325
0
{
326
0
    struct vhost_dev *dev = container_of(listener, struct vhost_dev,
327
0
                                         memory_listener);
328
0
    vhost_sync_dirty_bitmap(dev, section, 0x0, ~0x0ULL);
329
0
}
330
331
static void vhost_log_sync_range(struct vhost_dev *dev,
332
                                 hwaddr first, hwaddr last)
333
0
{
334
0
    int i;
335
    /* FIXME: this is N^2 in number of sections */
336
0
    for (i = 0; i < dev->n_mem_sections; ++i) {
337
0
        MemoryRegionSection *section = &dev->mem_sections[i];
338
0
        vhost_sync_dirty_bitmap(dev, section, first, last);
339
0
    }
340
0
}
341
342
static uint64_t vhost_get_log_size(struct vhost_dev *dev)
343
0
{
344
0
    uint64_t log_size = 0;
345
0
    int i;
346
0
    for (i = 0; i < dev->mem->nregions; ++i) {
347
0
        struct vhost_memory_region *reg = dev->mem->regions + i;
348
0
        uint64_t last = range_get_last(reg->guest_phys_addr,
349
0
                                       reg->memory_size);
350
0
        log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
351
0
    }
352
0
    return log_size;
353
0
}
354
355
static int vhost_set_backend_type(struct vhost_dev *dev,
356
                                  VhostBackendType backend_type)
357
0
{
358
0
    int r = 0;
359
360
0
    switch (backend_type) {
361
0
#ifdef CONFIG_VHOST_KERNEL
362
0
    case VHOST_BACKEND_TYPE_KERNEL:
363
0
        dev->vhost_ops = &kernel_ops;
364
0
        break;
365
0
#endif
366
0
#ifdef CONFIG_VHOST_USER
367
0
    case VHOST_BACKEND_TYPE_USER:
368
0
        dev->vhost_ops = &user_ops;
369
0
        break;
370
0
#endif
371
0
#ifdef CONFIG_VHOST_VDPA
372
0
    case VHOST_BACKEND_TYPE_VDPA:
373
0
        dev->vhost_ops = &vdpa_ops;
374
0
        break;
375
0
#endif
376
0
    default:
377
0
        error_report("Unknown vhost backend type");
378
0
        r = -1;
379
0
    }
380
381
0
    if (r == 0) {
382
0
        assert(dev->vhost_ops->backend_type == backend_type);
383
0
    }
384
385
0
    return r;
386
0
}
387
388
static struct vhost_log *vhost_log_alloc(uint64_t size, bool share)
389
0
{
390
0
    Error *err = NULL;
391
0
    struct vhost_log *log;
392
0
    uint64_t logsize = size * sizeof(*(log->log));
393
0
    int fd = -1;
394
395
0
    log = g_new0(struct vhost_log, 1);
396
0
    if (share) {
397
0
        log->log = qemu_memfd_alloc("vhost-log", logsize,
398
0
                                    F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
399
0
                                    &fd, &err);
400
0
        if (err) {
401
0
            error_report_err(err);
402
0
            g_free(log);
403
0
            return NULL;
404
0
        }
405
0
        memset(log->log, 0, logsize);
406
0
    } else {
407
0
        log->log = g_malloc0(logsize);
408
0
    }
409
410
0
    log->size = size;
411
0
    log->refcnt = 1;
412
0
    log->fd = fd;
413
414
0
    return log;
415
0
}
416
417
static struct vhost_log *vhost_log_get(VhostBackendType backend_type,
418
                                       uint64_t size, bool share)
419
0
{
420
0
    struct vhost_log *log;
421
422
0
    assert(backend_type > VHOST_BACKEND_TYPE_NONE);
423
0
    assert(backend_type < VHOST_BACKEND_TYPE_MAX);
424
425
0
    log = share ? vhost_log_shm[backend_type] : vhost_log[backend_type];
426
427
0
    if (!log || log->size != size) {
428
0
        log = vhost_log_alloc(size, share);
429
0
        if (share) {
430
0
            vhost_log_shm[backend_type] = log;
431
0
        } else {
432
0
            vhost_log[backend_type] = log;
433
0
        }
434
0
    } else {
435
0
        ++log->refcnt;
436
0
    }
437
438
0
    return log;
439
0
}
440
441
static void vhost_log_put(struct vhost_dev *dev, bool sync)
442
0
{
443
0
    struct vhost_log *log = dev->log;
444
0
    VhostBackendType backend_type;
445
446
0
    if (!log) {
447
0
        return;
448
0
    }
449
450
0
    assert(dev->vhost_ops);
451
0
    backend_type = dev->vhost_ops->backend_type;
452
453
0
    if (backend_type == VHOST_BACKEND_TYPE_NONE ||
454
0
        backend_type >= VHOST_BACKEND_TYPE_MAX) {
455
0
        return;
456
0
    }
457
458
0
    --log->refcnt;
459
0
    if (log->refcnt == 0) {
460
        /* Sync only the range covered by the old log */
461
0
        if (dev->log_size && sync) {
462
0
            vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1);
463
0
        }
464
465
0
        if (vhost_log[backend_type] == log) {
466
0
            g_free(log->log);
467
0
            vhost_log[backend_type] = NULL;
468
0
        } else if (vhost_log_shm[backend_type] == log) {
469
0
            qemu_memfd_free(log->log, log->size * sizeof(*(log->log)),
470
0
                            log->fd);
471
0
            vhost_log_shm[backend_type] = NULL;
472
0
        }
473
474
0
        g_free(log);
475
0
    }
476
477
0
    vhost_dev_elect_mem_logger(dev, false);
478
0
    dev->log = NULL;
479
0
    dev->log_size = 0;
480
0
}
481
482
static bool vhost_dev_log_is_shared(struct vhost_dev *dev)
483
0
{
484
0
    return dev->vhost_ops->vhost_requires_shm_log &&
485
0
           dev->vhost_ops->vhost_requires_shm_log(dev);
486
0
}
487
488
static inline void vhost_dev_log_resize(struct vhost_dev *dev, uint64_t size)
489
0
{
490
0
    struct vhost_log *log = vhost_log_get(dev->vhost_ops->backend_type,
491
0
                                          size, vhost_dev_log_is_shared(dev));
492
0
    uint64_t log_base = (uintptr_t)log->log;
493
0
    int r;
494
495
    /* inform backend of log switching, this must be done before
496
       releasing the current log, to ensure no logging is lost */
497
0
    r = dev->vhost_ops->vhost_set_log_base(dev, log_base, log);
498
0
    if (r < 0) {
499
0
        VHOST_OPS_DEBUG(r, "vhost_set_log_base failed");
500
0
    }
501
502
0
    vhost_log_put(dev, true);
503
0
    dev->log = log;
504
0
    dev->log_size = size;
505
0
}
506
507
static void *vhost_memory_map(struct vhost_dev *dev, hwaddr addr,
508
                              hwaddr len, bool is_write)
509
0
{
510
0
    hwaddr mapped_len = len;
511
0
    void *res = address_space_map(dev->vdev->dma_as, addr, &mapped_len,
512
0
                                  is_write, MEMTXATTRS_UNSPECIFIED);
513
0
    if (!res) {
514
0
        return NULL;
515
0
    }
516
0
    if (len != mapped_len) {
517
0
        address_space_unmap(dev->vdev->dma_as, res, mapped_len, 0, 0);
518
0
        return NULL;
519
0
    }
520
0
    return res;
521
0
}
522
523
static void vhost_memory_unmap(struct vhost_dev *dev, void **buffer,
524
                               hwaddr len, int is_write,
525
                               hwaddr access_len)
526
0
{
527
0
    if (!*buffer) {
528
0
        return;
529
0
    }
530
531
0
    address_space_unmap(dev->vdev->dma_as, *buffer, len, is_write,
532
0
                        access_len);
533
0
    *buffer = NULL;
534
0
}
535
536
static void vhost_vrings_unmap(struct vhost_dev *dev,
537
                               struct vhost_virtqueue *vq, bool touched)
538
0
{
539
0
    if (vhost_dev_has_iommu(dev)) {
540
0
        return;
541
0
    }
542
543
0
    vhost_memory_unmap(dev, &vq->used_user, vq->used_size, touched,
544
0
                       touched ? vq->used_size : 0);
545
0
    vhost_memory_unmap(dev, &vq->avail_user, vq->avail_size, 0,
546
0
                       touched ? vq->avail_size : 0);
547
0
    vhost_memory_unmap(dev, &vq->desc_user, vq->desc_size, 0,
548
0
                       touched ? vq->desc_size : 0);
549
0
}
550
551
static int vhost_vrings_map(struct vhost_dev *dev,
552
                            struct VirtIODevice *vdev,
553
                            struct vhost_virtqueue *vq,
554
                            unsigned idx)
555
0
{
556
0
    vq->desc_size = virtio_queue_get_desc_size(vdev, idx);
557
0
    vq->desc_phys = virtio_queue_get_desc_addr(vdev, idx);
558
0
    vq->desc_user = NULL;
559
0
    vq->avail_size = virtio_queue_get_avail_size(vdev, idx);
560
0
    vq->avail_phys = virtio_queue_get_avail_addr(vdev, idx);
561
0
    vq->avail_user = NULL;
562
0
    vq->used_size = virtio_queue_get_used_size(vdev, idx);
563
0
    vq->used_phys = virtio_queue_get_used_addr(vdev, idx);
564
0
    vq->used_user = NULL;
565
566
0
    if (vq->desc_phys == 0) {
567
        /* Queue might not be ready for start */
568
0
        return 0;
569
0
    }
570
571
0
    if (vhost_dev_has_iommu(dev)) {
572
0
        return 1;
573
0
    }
574
575
0
    vq->desc_user = vhost_memory_map(dev, vq->desc_phys, vq->desc_size, false);
576
0
    if (!vq->desc_user) {
577
0
        goto fail;
578
0
    }
579
0
    vq->avail_user = vhost_memory_map(dev, vq->avail_phys, vq->avail_size,
580
0
                                      false);
581
0
    if (!vq->avail_user) {
582
0
        goto fail;
583
0
    }
584
0
    vq->used_user = vhost_memory_map(dev, vq->used_phys, vq->used_size, true);
585
0
    if (!vq->used_user) {
586
0
        goto fail;
587
0
    }
588
589
0
    return 1;
590
591
0
fail:
592
0
    vhost_vrings_unmap(dev, vq, false);
593
0
    return -ENOMEM;
594
0
}
595
596
static int vhost_verify_ring_part_mapping(void *ring_hva,
597
                                          uint64_t ring_gpa,
598
                                          uint64_t ring_size,
599
                                          void *reg_hva,
600
                                          uint64_t reg_gpa,
601
                                          uint64_t reg_size)
602
0
{
603
0
    uint64_t hva_ring_offset;
604
0
    uint64_t ring_last = range_get_last(ring_gpa, ring_size);
605
0
    uint64_t reg_last = range_get_last(reg_gpa, reg_size);
606
607
0
    if (ring_last < reg_gpa || ring_gpa > reg_last) {
608
0
        return 0;
609
0
    }
610
    /* check that whole ring's is mapped */
611
0
    if (ring_last > reg_last) {
612
0
        return -ENOMEM;
613
0
    }
614
    /* check that ring's MemoryRegion wasn't replaced */
615
0
    hva_ring_offset = ring_gpa - reg_gpa;
616
0
    if (ring_hva != reg_hva + hva_ring_offset) {
617
0
        return -EBUSY;
618
0
    }
619
620
0
    return 0;
621
0
}
622
623
static int vhost_verify_ring_mappings(struct vhost_dev *dev,
624
                                      void *reg_hva,
625
                                      uint64_t reg_gpa,
626
                                      uint64_t reg_size)
627
0
{
628
0
    int i, j;
629
0
    int r = 0;
630
0
    const char *part_name[] = {
631
0
        "descriptor table",
632
0
        "available ring",
633
0
        "used ring"
634
0
    };
635
636
0
    if (vhost_dev_has_iommu(dev)) {
637
0
        return 0;
638
0
    }
639
640
0
    for (i = 0; i < dev->nvqs; ++i) {
641
0
        struct vhost_virtqueue *vq = dev->vqs + i;
642
643
0
        if (vq->desc_phys == 0) {
644
0
            continue;
645
0
        }
646
647
0
        j = 0;
648
0
        r = vhost_verify_ring_part_mapping(
649
0
                vq->desc_user, vq->desc_phys, vq->desc_size,
650
0
                reg_hva, reg_gpa, reg_size);
651
0
        if (r) {
652
0
            break;
653
0
        }
654
655
0
        j++;
656
0
        r = vhost_verify_ring_part_mapping(
657
0
                vq->avail_user, vq->avail_phys, vq->avail_size,
658
0
                reg_hva, reg_gpa, reg_size);
659
0
        if (r) {
660
0
            break;
661
0
        }
662
663
0
        j++;
664
0
        r = vhost_verify_ring_part_mapping(
665
0
                vq->used_user, vq->used_phys, vq->used_size,
666
0
                reg_hva, reg_gpa, reg_size);
667
0
        if (r) {
668
0
            break;
669
0
        }
670
0
    }
671
672
0
    if (r == -ENOMEM) {
673
0
        error_report("Unable to map %s for ring %d", part_name[j], i);
674
0
    } else if (r == -EBUSY) {
675
0
        error_report("%s relocated for ring %d", part_name[j], i);
676
0
    }
677
0
    return r;
678
0
}
679
680
/*
681
 * vhost_section: identify sections needed for vhost access
682
 *
683
 * We only care about RAM sections here (where virtqueue and guest
684
 * internals accessed by virtio might live).
685
 */
686
static bool vhost_section(struct vhost_dev *dev, MemoryRegionSection *section)
687
0
{
688
0
    MemoryRegion *mr = section->mr;
689
690
0
    if (memory_region_is_ram(mr) && !memory_region_is_rom(mr)) {
691
0
        uint8_t dirty_mask = memory_region_get_dirty_log_mask(mr);
692
0
        uint8_t handled_dirty;
693
694
        /*
695
         * Kernel based vhost doesn't handle any block which is doing
696
         * dirty-tracking other than migration for which it has
697
         * specific logging support. However for TCG the kernel never
698
         * gets involved anyway so we can also ignore it's
699
         * self-modiying code detection flags. However a vhost-user
700
         * client could still confuse a TCG guest if it re-writes
701
         * executable memory that has already been translated.
702
         */
703
0
        handled_dirty = (1 << DIRTY_MEMORY_MIGRATION) |
704
0
            (1 << DIRTY_MEMORY_CODE);
705
706
0
        if (dirty_mask & ~handled_dirty) {
707
0
            trace_vhost_reject_section(mr->name, 1);
708
0
            return false;
709
0
        }
710
711
        /*
712
         * Some backends (like vhost-user) can only handle memory regions
713
         * that have an fd (can be mapped into a different process). Filter
714
         * the ones without an fd out, if requested. Also make sure that
715
         * this region is mapped as shared so that the vhost backend can
716
         * observe modifications to this region, otherwise we consider it
717
         * private.
718
         */
719
0
        if ((memory_region_get_fd(section->mr) < 0 ||
720
0
            !qemu_ram_is_shared(section->mr->ram_block)) &&
721
0
            dev->vhost_ops->vhost_no_private_memslots &&
722
0
            dev->vhost_ops->vhost_no_private_memslots(dev)) {
723
0
            trace_vhost_reject_section(mr->name, 2);
724
0
            return false;
725
0
        }
726
727
0
        trace_vhost_section(mr->name);
728
0
        return true;
729
0
    } else {
730
0
        trace_vhost_reject_section(mr->name, 3);
731
0
        return false;
732
0
    }
733
0
}
734
735
static void vhost_begin(MemoryListener *listener)
736
0
{
737
0
    struct vhost_dev *dev = container_of(listener, struct vhost_dev,
738
0
                                         memory_listener);
739
0
    dev->tmp_sections = NULL;
740
0
    dev->n_tmp_sections = 0;
741
0
}
742
743
static void vhost_commit(MemoryListener *listener)
744
0
{
745
0
    struct vhost_dev *dev = container_of(listener, struct vhost_dev,
746
0
                                         memory_listener);
747
0
    MemoryRegionSection *old_sections;
748
0
    int n_old_sections;
749
0
    uint64_t log_size;
750
0
    size_t regions_size;
751
0
    int r;
752
0
    int i;
753
0
    bool changed = false;
754
755
    /* Note we can be called before the device is started, but then
756
     * starting the device calls set_mem_table, so we need to have
757
     * built the data structures.
758
     */
759
0
    old_sections = dev->mem_sections;
760
0
    n_old_sections = dev->n_mem_sections;
761
0
    dev->mem_sections = dev->tmp_sections;
762
0
    dev->n_mem_sections = dev->n_tmp_sections;
763
764
0
    if (dev->n_mem_sections != n_old_sections) {
765
0
        changed = true;
766
0
    } else {
767
        /* Same size, lets check the contents */
768
0
        for (i = 0; i < n_old_sections; i++) {
769
0
            if (!MemoryRegionSection_eq(&old_sections[i],
770
0
                                        &dev->mem_sections[i])) {
771
0
                changed = true;
772
0
                break;
773
0
            }
774
0
        }
775
0
    }
776
777
0
    trace_vhost_commit(dev->started, changed);
778
0
    if (!changed) {
779
0
        goto out;
780
0
    }
781
782
    /* Rebuild the regions list from the new sections list */
783
0
    regions_size = offsetof(struct vhost_memory, regions) +
784
0
                       dev->n_mem_sections * sizeof dev->mem->regions[0];
785
0
    dev->mem = g_realloc(dev->mem, regions_size);
786
0
    dev->mem->nregions = dev->n_mem_sections;
787
788
0
    for (i = 0; i < dev->n_mem_sections; i++) {
789
0
        struct vhost_memory_region *cur_vmr = dev->mem->regions + i;
790
0
        struct MemoryRegionSection *mrs = dev->mem_sections + i;
791
792
0
        cur_vmr->guest_phys_addr = mrs->offset_within_address_space;
793
0
        cur_vmr->memory_size     = int128_get64(mrs->size);
794
0
        cur_vmr->userspace_addr  =
795
0
            (uintptr_t)memory_region_get_ram_ptr(mrs->mr) +
796
0
            mrs->offset_within_region;
797
0
        cur_vmr->flags_padding   = 0;
798
0
    }
799
800
0
    if (!dev->started) {
801
0
        goto out;
802
0
    }
803
804
0
    for (i = 0; i < dev->mem->nregions; i++) {
805
0
        if (vhost_verify_ring_mappings(dev,
806
0
                       (void *)(uintptr_t)dev->mem->regions[i].userspace_addr,
807
0
                       dev->mem->regions[i].guest_phys_addr,
808
0
                       dev->mem->regions[i].memory_size)) {
809
0
            virtio_error(dev->vdev,
810
0
                         "Verify ring failure on region %d", i);
811
0
            goto out;
812
0
        }
813
0
    }
814
815
0
    if (!dev->log_enabled) {
816
0
        r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
817
0
        if (r < 0) {
818
0
            VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed");
819
0
        }
820
0
        goto out;
821
0
    }
822
0
    log_size = vhost_get_log_size(dev);
823
    /* We allocate an extra 4K bytes to log,
824
     * to reduce the * number of reallocations. */
825
0
#define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log)
826
    /* To log more, must increase log size before table update. */
827
0
    if (dev->log_size < log_size) {
828
0
        vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
829
0
    }
830
0
    r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
831
0
    if (r < 0) {
832
0
        VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed");
833
0
    }
834
    /* To log less, can only decrease log size after table update. */
835
0
    if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
836
0
        vhost_dev_log_resize(dev, log_size);
837
0
    }
838
839
0
out:
840
    /* Deref the old list of sections, this must happen _after_ the
841
     * vhost_set_mem_table to ensure the client isn't still using the
842
     * section we're about to unref.
843
     */
844
0
    while (n_old_sections--) {
845
0
        memory_region_unref(old_sections[n_old_sections].mr);
846
0
    }
847
0
    g_free(old_sections);
848
0
}
849
850
/* Adds the section data to the tmp_section structure.
851
 * It relies on the listener calling us in memory address order
852
 * and for each region (via the _add and _nop methods) to
853
 * join neighbours.
854
 */
855
static void vhost_region_add_section(struct vhost_dev *dev,
856
                                     MemoryRegionSection *section)
857
0
{
858
0
    bool need_add = true;
859
0
    uint64_t mrs_size = int128_get64(section->size);
860
0
    uint64_t mrs_gpa = section->offset_within_address_space;
861
0
    uintptr_t mrs_host = (uintptr_t)memory_region_get_ram_ptr(section->mr) +
862
0
                         section->offset_within_region;
863
0
    RAMBlock *mrs_rb = section->mr->ram_block;
864
865
0
    trace_vhost_region_add_section(section->mr->name, mrs_gpa, mrs_size,
866
0
                                   mrs_host);
867
868
0
    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER) {
869
        /* Round the section to it's page size */
870
        /* First align the start down to a page boundary */
871
0
        size_t mrs_page = qemu_ram_pagesize(mrs_rb);
872
0
        uint64_t alignage = mrs_host & (mrs_page - 1);
873
0
        if (alignage) {
874
0
            mrs_host -= alignage;
875
0
            mrs_size += alignage;
876
0
            mrs_gpa  -= alignage;
877
0
        }
878
        /* Now align the size up to a page boundary */
879
0
        alignage = mrs_size & (mrs_page - 1);
880
0
        if (alignage) {
881
0
            mrs_size += mrs_page - alignage;
882
0
        }
883
0
        trace_vhost_region_add_section_aligned(section->mr->name, mrs_gpa,
884
0
                                               mrs_size, mrs_host);
885
0
    }
886
887
0
    if (dev->n_tmp_sections && !section->unmergeable) {
888
        /* Since we already have at least one section, lets see if
889
         * this extends it; since we're scanning in order, we only
890
         * have to look at the last one, and the FlatView that calls
891
         * us shouldn't have overlaps.
892
         */
893
0
        MemoryRegionSection *prev_sec = dev->tmp_sections +
894
0
                                               (dev->n_tmp_sections - 1);
895
0
        uint64_t prev_gpa_start = prev_sec->offset_within_address_space;
896
0
        uint64_t prev_size = int128_get64(prev_sec->size);
897
0
        uint64_t prev_gpa_end   = range_get_last(prev_gpa_start, prev_size);
898
0
        uint64_t prev_host_start =
899
0
                        (uintptr_t)memory_region_get_ram_ptr(prev_sec->mr) +
900
0
                        prev_sec->offset_within_region;
901
0
        uint64_t prev_host_end   = range_get_last(prev_host_start, prev_size);
902
903
0
        if (mrs_gpa <= (prev_gpa_end + 1)) {
904
            /* OK, looks like overlapping/intersecting - it's possible that
905
             * the rounding to page sizes has made them overlap, but they should
906
             * match up in the same RAMBlock if they do.
907
             */
908
0
            if (mrs_gpa < prev_gpa_start) {
909
0
                error_report("%s:Section '%s' rounded to %"PRIx64
910
0
                             " prior to previous '%s' %"PRIx64,
911
0
                             __func__, section->mr->name, mrs_gpa,
912
0
                             prev_sec->mr->name, prev_gpa_start);
913
                /* A way to cleanly fail here would be better */
914
0
                return;
915
0
            }
916
            /* Offset from the start of the previous GPA to this GPA */
917
0
            size_t offset = mrs_gpa - prev_gpa_start;
918
919
0
            if (prev_host_start + offset == mrs_host &&
920
0
                section->mr == prev_sec->mr && !prev_sec->unmergeable) {
921
0
                uint64_t max_end = MAX(prev_host_end, mrs_host + mrs_size);
922
0
                need_add = false;
923
0
                prev_sec->offset_within_address_space =
924
0
                    MIN(prev_gpa_start, mrs_gpa);
925
0
                prev_sec->offset_within_region =
926
0
                    MIN(prev_host_start, mrs_host) -
927
0
                    (uintptr_t)memory_region_get_ram_ptr(prev_sec->mr);
928
0
                prev_sec->size = int128_make64(max_end - MIN(prev_host_start,
929
0
                                               mrs_host));
930
0
                trace_vhost_region_add_section_merge(section->mr->name,
931
0
                                        int128_get64(prev_sec->size),
932
0
                                        prev_sec->offset_within_address_space,
933
0
                                        prev_sec->offset_within_region);
934
0
            } else {
935
                /* adjoining regions are fine, but overlapping ones with
936
                 * different blocks/offsets shouldn't happen
937
                 */
938
0
                if (mrs_gpa != prev_gpa_end + 1) {
939
0
                    error_report("%s: Overlapping but not coherent sections "
940
0
                                 "at %"PRIx64,
941
0
                                 __func__, mrs_gpa);
942
0
                    return;
943
0
                }
944
0
            }
945
0
        }
946
0
    }
947
948
0
    if (need_add) {
949
0
        ++dev->n_tmp_sections;
950
0
        dev->tmp_sections = g_renew(MemoryRegionSection, dev->tmp_sections,
951
0
                                    dev->n_tmp_sections);
952
0
        dev->tmp_sections[dev->n_tmp_sections - 1] = *section;
953
        /* The flatview isn't stable and we don't use it, making it NULL
954
         * means we can memcmp the list.
955
         */
956
0
        dev->tmp_sections[dev->n_tmp_sections - 1].fv = NULL;
957
0
        memory_region_ref(section->mr);
958
0
    }
959
0
}
960
961
/* Used for both add and nop callbacks */
962
static void vhost_region_addnop(MemoryListener *listener,
963
                                MemoryRegionSection *section)
964
0
{
965
0
    struct vhost_dev *dev = container_of(listener, struct vhost_dev,
966
0
                                         memory_listener);
967
968
0
    if (!vhost_section(dev, section)) {
969
0
        return;
970
0
    }
971
0
    vhost_region_add_section(dev, section);
972
0
}
973
974
static int vhost_update_device_iotlb(struct vhost_dev *dev,
975
                                     uint64_t iova,
976
                                     uint64_t paddr,
977
                                     uint64_t uaddr,
978
                                     uint64_t len,
979
                                     IOMMUAccessFlags perm)
980
0
{
981
0
    bool phys = dev->vhost_ops->vhost_phys_iotlb_msg &&
982
0
        dev->vhost_ops->vhost_phys_iotlb_msg(dev);
983
0
    struct vhost_iotlb_msg imsg;
984
985
0
    imsg.iova =  iova;
986
0
    imsg.uaddr = phys ? paddr : uaddr;
987
0
    imsg.size = len;
988
0
    imsg.type = VHOST_IOTLB_UPDATE;
989
990
0
    switch (perm) {
991
0
    case IOMMU_RO:
992
0
        imsg.perm = VHOST_ACCESS_RO;
993
0
        break;
994
0
    case IOMMU_WO:
995
0
        imsg.perm = VHOST_ACCESS_WO;
996
0
        break;
997
0
    case IOMMU_RW:
998
0
        imsg.perm = VHOST_ACCESS_RW;
999
0
        break;
1000
0
    default:
1001
0
        return -EINVAL;
1002
0
    }
1003
1004
0
    if (dev->vhost_ops && dev->vhost_ops->vhost_send_device_iotlb_msg) {
1005
0
        return dev->vhost_ops->vhost_send_device_iotlb_msg(dev, &imsg);
1006
0
    }
1007
1008
0
    return -ENODEV;
1009
0
}
1010
1011
static int vhost_invalidate_device_iotlb(struct vhost_dev *dev,
1012
                                         uint64_t iova, uint64_t len)
1013
0
{
1014
0
    struct vhost_iotlb_msg imsg;
1015
1016
0
    imsg.iova = iova;
1017
0
    imsg.size = len;
1018
0
    imsg.type = VHOST_IOTLB_INVALIDATE;
1019
1020
0
    if (dev->vhost_ops && dev->vhost_ops->vhost_send_device_iotlb_msg) {
1021
0
        return dev->vhost_ops->vhost_send_device_iotlb_msg(dev, &imsg);
1022
0
    }
1023
1024
0
    return -ENODEV;
1025
0
}
1026
1027
int vhost_handle_iotlb_msg(struct vhost_dev *dev, struct vhost_iotlb_msg *imsg)
1028
0
{
1029
0
    int ret = 0;
1030
1031
0
    if (unlikely(!dev->vdev)) {
1032
0
        error_report("Unexpected IOTLB message when virtio device is stopped");
1033
0
        return -EINVAL;
1034
0
    }
1035
1036
0
    switch (imsg->type) {
1037
0
    case VHOST_IOTLB_MISS:
1038
0
        ret = vhost_device_iotlb_miss(dev, imsg->iova,
1039
0
                                      imsg->perm != VHOST_ACCESS_RO);
1040
0
        break;
1041
0
    case VHOST_IOTLB_ACCESS_FAIL:
1042
        /* FIXME: report device iotlb error */
1043
0
        error_report("Access failure IOTLB message type not supported");
1044
0
        ret = -ENOTSUP;
1045
0
        break;
1046
0
    case VHOST_IOTLB_UPDATE:
1047
0
    case VHOST_IOTLB_INVALIDATE:
1048
0
    default:
1049
0
        error_report("Unexpected IOTLB message type");
1050
0
        ret = -EINVAL;
1051
0
        break;
1052
0
    }
1053
1054
0
    return ret;
1055
0
}
1056
1057
static void vhost_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
1058
0
{
1059
0
    struct vhost_iommu *iommu = container_of(n, struct vhost_iommu, n);
1060
0
    struct vhost_dev *hdev = iommu->hdev;
1061
0
    hwaddr iova = iotlb->iova + iommu->iommu_offset;
1062
1063
0
    if (vhost_invalidate_device_iotlb(hdev, iova, iotlb->addr_mask + 1)) {
1064
0
        error_report("Fail to invalidate device iotlb");
1065
0
    }
1066
0
}
1067
1068
static void vhost_iommu_region_add(MemoryListener *listener,
1069
                                   MemoryRegionSection *section)
1070
0
{
1071
0
    struct vhost_dev *dev = container_of(listener, struct vhost_dev,
1072
0
                                         iommu_listener);
1073
0
    struct vhost_iommu *iommu;
1074
0
    Int128 end;
1075
0
    int iommu_idx;
1076
0
    IOMMUMemoryRegion *iommu_mr;
1077
1078
0
    if (!memory_region_is_iommu(section->mr)) {
1079
0
        return;
1080
0
    }
1081
1082
0
    iommu_mr = IOMMU_MEMORY_REGION(section->mr);
1083
1084
0
    iommu = g_malloc0(sizeof(*iommu));
1085
0
    end = int128_add(int128_make64(section->offset_within_region),
1086
0
                     section->size);
1087
0
    end = int128_sub(end, int128_one());
1088
0
    iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
1089
0
                                                   MEMTXATTRS_UNSPECIFIED);
1090
0
    iommu_notifier_init(&iommu->n, vhost_iommu_unmap_notify,
1091
0
                        dev->vdev->device_iotlb_enabled ?
1092
0
                            IOMMU_NOTIFIER_DEVIOTLB_UNMAP :
1093
0
                            IOMMU_NOTIFIER_UNMAP,
1094
0
                        section->offset_within_region,
1095
0
                        int128_get64(end),
1096
0
                        iommu_idx);
1097
0
    iommu->mr = section->mr;
1098
0
    iommu->iommu_offset = section->offset_within_address_space -
1099
0
                          section->offset_within_region;
1100
0
    iommu->hdev = dev;
1101
0
    memory_region_register_iommu_notifier(section->mr, &iommu->n,
1102
0
                                          &error_fatal);
1103
0
    QLIST_INSERT_HEAD(&dev->iommu_list, iommu, iommu_next);
1104
    /* TODO: can replay help performance here? */
1105
0
}
1106
1107
static void vhost_iommu_region_del(MemoryListener *listener,
1108
                                   MemoryRegionSection *section)
1109
0
{
1110
0
    struct vhost_dev *dev = container_of(listener, struct vhost_dev,
1111
0
                                         iommu_listener);
1112
0
    struct vhost_iommu *iommu;
1113
1114
0
    if (!memory_region_is_iommu(section->mr)) {
1115
0
        return;
1116
0
    }
1117
1118
0
    QLIST_FOREACH(iommu, &dev->iommu_list, iommu_next) {
1119
0
        if (iommu->mr == section->mr &&
1120
0
            iommu->n.start == section->offset_within_region) {
1121
0
            memory_region_unregister_iommu_notifier(iommu->mr,
1122
0
                                                    &iommu->n);
1123
0
            QLIST_REMOVE(iommu, iommu_next);
1124
0
            g_free(iommu);
1125
0
            break;
1126
0
        }
1127
0
    }
1128
0
}
1129
1130
void vhost_toggle_device_iotlb(VirtIODevice *vdev)
1131
0
{
1132
0
    VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1133
0
    struct vhost_dev *dev;
1134
0
    struct vhost_iommu *iommu;
1135
1136
0
    if (vdev->vhost_started) {
1137
0
        dev = vdc->get_vhost(vdev);
1138
0
    } else {
1139
0
        return;
1140
0
    }
1141
1142
0
    QLIST_FOREACH(iommu, &dev->iommu_list, iommu_next) {
1143
0
        memory_region_unregister_iommu_notifier(iommu->mr, &iommu->n);
1144
0
        iommu->n.notifier_flags = vdev->device_iotlb_enabled ?
1145
0
                IOMMU_NOTIFIER_DEVIOTLB_UNMAP : IOMMU_NOTIFIER_UNMAP;
1146
0
        memory_region_register_iommu_notifier(iommu->mr, &iommu->n,
1147
0
                                              &error_fatal);
1148
0
    }
1149
0
}
1150
1151
static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
1152
                                    struct vhost_virtqueue *vq,
1153
                                    unsigned idx, bool enable_log)
1154
0
{
1155
0
    bool phys = dev->vhost_ops->vhost_phys_vring_addr &&
1156
0
        dev->vhost_ops->vhost_phys_vring_addr(dev);
1157
0
    struct vhost_vring_addr addr;
1158
0
    int r;
1159
0
    memset(&addr, 0, sizeof(struct vhost_vring_addr));
1160
1161
0
    if (phys || vhost_dev_has_iommu(dev)) {
1162
0
        addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
1163
0
        addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
1164
0
        addr.used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
1165
0
    } else {
1166
0
        addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_user;
1167
0
        addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_user;
1168
0
        addr.used_user_addr = (uint64_t)(unsigned long)vq->used_user;
1169
0
    }
1170
0
    addr.index = idx;
1171
0
    addr.log_guest_addr = vq->used_phys;
1172
0
    addr.flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0;
1173
0
    r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
1174
0
    if (r < 0) {
1175
0
        VHOST_OPS_DEBUG(r, "vhost_set_vring_addr failed");
1176
0
    }
1177
0
    return r;
1178
0
}
1179
1180
static int vhost_dev_set_features(struct vhost_dev *dev,
1181
                                  bool enable_log)
1182
0
{
1183
0
    uint64_t features[VIRTIO_FEATURES_NU64S];
1184
0
    int r;
1185
1186
0
    virtio_features_copy(features, dev->acked_features_ex);
1187
0
    if (enable_log) {
1188
0
        virtio_add_feature_ex(features, VHOST_F_LOG_ALL);
1189
0
    }
1190
0
    if (!vhost_dev_has_iommu(dev)) {
1191
0
        virtio_clear_feature_ex(features, VIRTIO_F_IOMMU_PLATFORM);
1192
0
    }
1193
0
    if (dev->vhost_ops->vhost_force_iommu) {
1194
0
        if (dev->vhost_ops->vhost_force_iommu(dev) == true) {
1195
0
            virtio_add_feature_ex(features, VIRTIO_F_IOMMU_PLATFORM);
1196
0
       }
1197
0
    }
1198
1199
0
    if (virtio_features_use_ex(features) &&
1200
0
        !dev->vhost_ops->vhost_set_features_ex) {
1201
0
        r = -EINVAL;
1202
0
        VHOST_OPS_DEBUG(r, "extended features without device support");
1203
0
        goto out;
1204
0
    }
1205
1206
0
    if (dev->vhost_ops->vhost_set_features_ex) {
1207
0
        r = dev->vhost_ops->vhost_set_features_ex(dev, features);
1208
0
    } else {
1209
0
        r = dev->vhost_ops->vhost_set_features(dev, features[0]);
1210
0
    }
1211
0
    if (r < 0) {
1212
0
        VHOST_OPS_DEBUG(r, "vhost_set_features failed");
1213
0
        goto out;
1214
0
    }
1215
0
    if (dev->vhost_ops->vhost_set_backend_cap) {
1216
0
        r = dev->vhost_ops->vhost_set_backend_cap(dev);
1217
0
        if (r < 0) {
1218
0
            VHOST_OPS_DEBUG(r, "vhost_set_backend_cap failed");
1219
0
            goto out;
1220
0
        }
1221
0
    }
1222
1223
0
out:
1224
0
    return r;
1225
0
}
1226
1227
static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
1228
0
{
1229
0
    int r, i, idx;
1230
0
    hwaddr addr;
1231
1232
0
    r = vhost_dev_set_features(dev, enable_log);
1233
0
    if (r < 0) {
1234
0
        goto err_features;
1235
0
    }
1236
0
    for (i = 0; i < dev->nvqs; ++i) {
1237
0
        idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
1238
0
        addr = virtio_queue_get_desc_addr(dev->vdev, idx);
1239
0
        if (!addr) {
1240
            /*
1241
             * The queue might not be ready for start. If this
1242
             * is the case there is no reason to continue the process.
1243
             * The similar logic is used by the vhost_virtqueue_start()
1244
             * routine.
1245
             */
1246
0
            continue;
1247
0
        }
1248
0
        r = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
1249
0
                                     enable_log);
1250
0
        if (r < 0) {
1251
0
            goto err_vq;
1252
0
        }
1253
0
    }
1254
1255
    /*
1256
     * At log start we select our vhost_device logger that will scan the
1257
     * memory sections and skip for the others. This is possible because
1258
     * the log is shared amongst all vhost devices for a given type of
1259
     * backend.
1260
     */
1261
0
    vhost_dev_elect_mem_logger(dev, enable_log);
1262
1263
0
    return 0;
1264
0
err_vq:
1265
0
    for (; i >= 0; --i) {
1266
0
        idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
1267
0
        addr = virtio_queue_get_desc_addr(dev->vdev, idx);
1268
0
        if (!addr) {
1269
0
            continue;
1270
0
        }
1271
0
        vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
1272
0
                                 dev->log_enabled);
1273
0
    }
1274
0
    vhost_dev_set_features(dev, dev->log_enabled);
1275
0
err_features:
1276
0
    return r;
1277
0
}
1278
1279
static int vhost_migration_log(MemoryListener *listener, bool enable)
1280
0
{
1281
0
    struct vhost_dev *dev = container_of(listener, struct vhost_dev,
1282
0
                                         memory_listener);
1283
0
    int r;
1284
0
    if (enable == dev->log_enabled) {
1285
0
        return 0;
1286
0
    }
1287
0
    if (!dev->started) {
1288
0
        dev->log_enabled = enable;
1289
0
        return 0;
1290
0
    }
1291
1292
0
    r = 0;
1293
0
    if (!enable) {
1294
0
        r = vhost_dev_set_log(dev, false);
1295
0
        if (r < 0) {
1296
0
            goto check_dev_state;
1297
0
        }
1298
0
        vhost_log_put(dev, false);
1299
0
    } else {
1300
0
        vhost_dev_log_resize(dev, vhost_get_log_size(dev));
1301
0
        r = vhost_dev_set_log(dev, true);
1302
0
        if (r < 0) {
1303
0
            goto check_dev_state;
1304
0
        }
1305
0
    }
1306
1307
0
check_dev_state:
1308
0
    dev->log_enabled = enable;
1309
    /*
1310
     * vhost-user-* devices could change their state during log
1311
     * initialization due to disconnect. So check dev state after
1312
     * vhost communication.
1313
     */
1314
0
    if (!dev->started) {
1315
        /*
1316
         * Since device is in the stopped state, it is okay for
1317
         * migration. Return success.
1318
         */
1319
0
        r = 0;
1320
0
    }
1321
0
    if (r) {
1322
        /* An error occurred. */
1323
0
        dev->log_enabled = false;
1324
0
    }
1325
1326
0
    return r;
1327
0
}
1328
1329
static bool vhost_log_global_start(MemoryListener *listener, Error **errp)
1330
0
{
1331
0
    int r;
1332
1333
0
    r = vhost_migration_log(listener, true);
1334
0
    if (r < 0) {
1335
0
        error_setg_errno(errp, -r, "vhost: Failed to start logging");
1336
0
        return false;
1337
0
    }
1338
0
    return true;
1339
0
}
1340
1341
static void vhost_log_global_stop(MemoryListener *listener)
1342
0
{
1343
0
    int r;
1344
1345
0
    r = vhost_migration_log(listener, false);
1346
0
    if (r < 0) {
1347
        /* Not fatal, so report it, but take no further action */
1348
0
        warn_report("vhost: Failed to stop logging");
1349
0
    }
1350
0
}
1351
1352
static void vhost_log_start(MemoryListener *listener,
1353
                            MemoryRegionSection *section,
1354
                            int old, int new)
1355
0
{
1356
    /* FIXME: implement */
1357
0
}
1358
1359
static void vhost_log_stop(MemoryListener *listener,
1360
                           MemoryRegionSection *section,
1361
                           int old, int new)
1362
0
{
1363
    /* FIXME: implement */
1364
0
}
1365
1366
/* The vhost driver natively knows how to handle the vrings of non
1367
 * cross-endian legacy devices and modern devices. Only legacy devices
1368
 * exposed to a bi-endian guest may require the vhost driver to use a
1369
 * specific endianness.
1370
 */
1371
static inline bool vhost_needs_vring_endian(VirtIODevice *vdev)
1372
0
{
1373
0
    if (virtio_vdev_is_legacy(vdev)) {
1374
0
        return vdev->device_endian == (HOST_BIG_ENDIAN
1375
0
                                       ? VIRTIO_DEVICE_ENDIAN_LITTLE
1376
0
                                       : VIRTIO_DEVICE_ENDIAN_BIG);
1377
0
    }
1378
0
    return false;
1379
0
}
1380
1381
static int vhost_virtqueue_set_vring_endian_legacy(struct vhost_dev *dev,
1382
                                                   bool is_big_endian,
1383
                                                   int vhost_vq_index)
1384
0
{
1385
0
    int r;
1386
0
    struct vhost_vring_state s = {
1387
0
        .index = vhost_vq_index,
1388
0
        .num = is_big_endian
1389
0
    };
1390
1391
0
    r = dev->vhost_ops->vhost_set_vring_endian(dev, &s);
1392
0
    if (r < 0) {
1393
0
        VHOST_OPS_DEBUG(r, "vhost_set_vring_endian failed");
1394
0
    }
1395
0
    return r;
1396
0
}
1397
1398
static int vhost_memory_region_lookup(struct vhost_dev *hdev,
1399
                                      uint64_t gpa, uint64_t *uaddr,
1400
                                      uint64_t *len)
1401
0
{
1402
0
    int i;
1403
1404
0
    for (i = 0; i < hdev->mem->nregions; i++) {
1405
0
        struct vhost_memory_region *reg = hdev->mem->regions + i;
1406
1407
0
        if (gpa >= reg->guest_phys_addr &&
1408
0
            reg->guest_phys_addr + reg->memory_size > gpa) {
1409
0
            *uaddr = reg->userspace_addr + gpa - reg->guest_phys_addr;
1410
0
            *len = reg->guest_phys_addr + reg->memory_size - gpa;
1411
0
            return 0;
1412
0
        }
1413
0
    }
1414
1415
0
    return -EFAULT;
1416
0
}
1417
1418
int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int write)
1419
0
{
1420
0
    IOMMUTLBEntry iotlb;
1421
0
    uint64_t uaddr, len;
1422
0
    int ret = -EFAULT;
1423
1424
0
    RCU_READ_LOCK_GUARD();
1425
1426
0
    trace_vhost_iotlb_miss(dev, 1);
1427
1428
0
    iotlb = address_space_get_iotlb_entry(dev->vdev->dma_as,
1429
0
                                          iova, write,
1430
0
                                          MEMTXATTRS_UNSPECIFIED);
1431
0
    if (iotlb.target_as != NULL) {
1432
0
        ret = vhost_memory_region_lookup(dev, iotlb.translated_addr,
1433
0
                                         &uaddr, &len);
1434
0
        if (ret) {
1435
0
            trace_vhost_iotlb_miss(dev, 3);
1436
0
            error_report("Fail to lookup the translated address "
1437
0
                         "%"PRIx64, iotlb.translated_addr);
1438
0
            goto out;
1439
0
        }
1440
1441
0
        len = MIN(iotlb.addr_mask + 1, len);
1442
0
        iova = iova & ~iotlb.addr_mask;
1443
1444
0
        ret = vhost_update_device_iotlb(dev, iova, iotlb.translated_addr, uaddr,
1445
0
                                        len, iotlb.perm);
1446
0
        if (ret) {
1447
0
            trace_vhost_iotlb_miss(dev, 4);
1448
0
            error_report("Fail to update device iotlb");
1449
0
            goto out;
1450
0
        }
1451
0
    }
1452
1453
0
    trace_vhost_iotlb_miss(dev, 2);
1454
1455
0
out:
1456
0
    return ret;
1457
0
}
1458
1459
int vhost_virtqueue_start(struct vhost_dev *dev,
1460
                          struct VirtIODevice *vdev,
1461
                          struct vhost_virtqueue *vq,
1462
                          unsigned idx)
1463
0
{
1464
0
    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
1465
0
    VirtioBusState *vbus = VIRTIO_BUS(qbus);
1466
0
    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
1467
0
    int r;
1468
0
    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
1469
0
    struct vhost_vring_file file = {
1470
0
        .index = vhost_vq_index
1471
0
    };
1472
0
    struct vhost_vring_state state = {
1473
0
        .index = vhost_vq_index
1474
0
    };
1475
0
    struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
1476
1477
0
    trace_vhost_virtqueue_start_in(dev, vdev->name, idx);
1478
1479
0
    r = vhost_vrings_map(dev, vdev, vq, idx);
1480
0
    if (r <= 0) {
1481
0
        return r;
1482
0
    }
1483
1484
0
    vq->num = state.num = virtio_queue_get_num(vdev, idx);
1485
0
    r = dev->vhost_ops->vhost_set_vring_num(dev, &state);
1486
0
    if (r) {
1487
0
        VHOST_OPS_DEBUG(r, "vhost_set_vring_num failed");
1488
0
        goto fail;
1489
0
    }
1490
1491
0
    state.num = virtio_queue_get_last_avail_idx(vdev, idx);
1492
0
    r = dev->vhost_ops->vhost_set_vring_base(dev, &state);
1493
0
    if (r) {
1494
0
        VHOST_OPS_DEBUG(r, "vhost_set_vring_base failed");
1495
0
        goto fail;
1496
0
    }
1497
1498
0
    if (vhost_needs_vring_endian(vdev)) {
1499
0
        r = vhost_virtqueue_set_vring_endian_legacy(
1500
0
            dev, virtio_vdev_is_big_endian(vdev), vhost_vq_index);
1501
0
        if (r) {
1502
0
            goto fail;
1503
0
        }
1504
0
    }
1505
1506
0
    r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled);
1507
0
    if (r < 0) {
1508
0
        goto fail;
1509
0
    }
1510
1511
0
    file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
1512
0
    r = dev->vhost_ops->vhost_set_vring_kick(dev, &file);
1513
0
    if (r) {
1514
0
        VHOST_OPS_DEBUG(r, "vhost_set_vring_kick failed");
1515
0
        goto fail;
1516
0
    }
1517
1518
    /* Clear and discard previous events if any. */
1519
0
    event_notifier_test_and_clear(&vq->masked_notifier);
1520
1521
    /* Init vring in unmasked state, unless guest_notifier_mask
1522
     * will do it later.
1523
     */
1524
0
    if (!vdev->use_guest_notifier_mask) {
1525
0
        r = vhost_virtqueue_mask(dev, vdev, idx, false);
1526
0
        if (r < 0) {
1527
0
            goto fail;
1528
0
        }
1529
0
    }
1530
1531
0
    if (k->query_guest_notifiers &&
1532
0
        k->query_guest_notifiers(qbus->parent) &&
1533
0
        virtio_queue_vector(vdev, idx) == VIRTIO_NO_VECTOR) {
1534
0
        file.fd = -1;
1535
0
        r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
1536
0
        if (r) {
1537
0
            goto fail;
1538
0
        }
1539
0
    }
1540
1541
0
    trace_vhost_virtqueue_start_out(dev, vdev->name, idx);
1542
1543
0
    return 0;
1544
1545
0
fail:
1546
0
    vhost_vrings_unmap(dev, vq, false);
1547
0
    return r;
1548
0
}
1549
1550
static int do_vhost_virtqueue_stop(struct vhost_dev *dev,
1551
                                   struct VirtIODevice *vdev,
1552
                                   struct vhost_virtqueue *vq,
1553
                                   unsigned idx, bool force,
1554
                                   bool skip_drain)
1555
0
{
1556
0
    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
1557
0
    struct vhost_vring_state state = {
1558
0
        .index = vhost_vq_index,
1559
0
    };
1560
0
    int r = 0;
1561
1562
0
    trace_vhost_virtqueue_stop_in(dev, vdev->name, idx);
1563
1564
0
    if (virtio_queue_get_desc_addr(vdev, idx) == 0) {
1565
        /* Don't stop the virtqueue which might have not been started */
1566
0
        return 0;
1567
0
    }
1568
1569
0
    if (!force) {
1570
0
        if (!skip_drain) {
1571
0
            r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
1572
0
        } else {
1573
0
            r = dev->vhost_ops->vhost_get_vring_base_skip_drain(dev, &state);
1574
0
        }
1575
0
        if (r < 0) {
1576
0
            VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d", idx, r);
1577
0
        }
1578
0
    }
1579
1580
0
    if (r < 0 || force) {
1581
        /* Connection to the backend is broken, so let's sync internal
1582
         * last avail idx to the device used idx.
1583
         */
1584
0
        virtio_queue_restore_last_avail_idx(vdev, idx);
1585
0
    } else {
1586
0
        virtio_queue_set_last_avail_idx(vdev, idx, state.num);
1587
0
    }
1588
0
    virtio_queue_invalidate_signalled_used(vdev, idx);
1589
0
    virtio_queue_update_used_idx(vdev, idx);
1590
1591
    /* In the cross-endian case, we need to reset the vring endianness to
1592
     * native as legacy devices expect so by default.
1593
     */
1594
0
    if (vhost_needs_vring_endian(vdev)) {
1595
0
        vhost_virtqueue_set_vring_endian_legacy(dev,
1596
0
                                                !virtio_vdev_is_big_endian(vdev),
1597
0
                                                vhost_vq_index);
1598
0
    }
1599
1600
0
    vhost_vrings_unmap(dev, vq, true);
1601
1602
0
    trace_vhost_virtqueue_stop_out(dev, vdev->name, idx);
1603
0
    return r;
1604
0
}
1605
1606
int vhost_virtqueue_stop(struct vhost_dev *dev,
1607
                         struct VirtIODevice *vdev,
1608
                         struct vhost_virtqueue *vq,
1609
                         unsigned idx,
1610
                         bool skip_drain)
1611
0
{
1612
0
    return do_vhost_virtqueue_stop(dev, vdev, vq, idx, false, skip_drain);
1613
0
}
1614
1615
static int vhost_virtqueue_set_busyloop_timeout(struct vhost_dev *dev,
1616
                                                int n, uint32_t timeout)
1617
0
{
1618
0
    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
1619
0
    struct vhost_vring_state state = {
1620
0
        .index = vhost_vq_index,
1621
0
        .num = timeout,
1622
0
    };
1623
0
    int r;
1624
1625
0
    if (!dev->vhost_ops->vhost_set_vring_busyloop_timeout) {
1626
0
        return -EINVAL;
1627
0
    }
1628
1629
0
    r = dev->vhost_ops->vhost_set_vring_busyloop_timeout(dev, &state);
1630
0
    if (r) {
1631
0
        VHOST_OPS_DEBUG(r, "vhost_set_vring_busyloop_timeout failed");
1632
0
        return r;
1633
0
    }
1634
1635
0
    return 0;
1636
0
}
1637
1638
static void vhost_virtqueue_error_notifier(EventNotifier *n)
1639
0
{
1640
0
    struct vhost_virtqueue *vq = container_of(n, struct vhost_virtqueue,
1641
0
                                              error_notifier);
1642
0
    struct vhost_dev *dev = vq->dev;
1643
0
    int index = vq - dev->vqs;
1644
1645
0
    if (event_notifier_test_and_clear(n) && dev->vdev) {
1646
0
        VHOST_OPS_DEBUG(-EINVAL,  "vhost vring error in virtqueue %d",
1647
0
                        dev->vq_index + index);
1648
0
    }
1649
0
}
1650
1651
static int vhost_virtqueue_init(struct vhost_dev *dev,
1652
                                struct vhost_virtqueue *vq, int n,
1653
                                bool busyloop_timeout)
1654
0
{
1655
0
    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
1656
0
    struct vhost_vring_file file = {
1657
0
        .index = vhost_vq_index,
1658
0
    };
1659
0
    int r = event_notifier_init(&vq->masked_notifier, 0);
1660
0
    if (r < 0) {
1661
0
        return r;
1662
0
    }
1663
1664
0
    file.fd = event_notifier_get_wfd(&vq->masked_notifier);
1665
0
    r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
1666
0
    if (r) {
1667
0
        VHOST_OPS_DEBUG(r, "vhost_set_vring_call failed");
1668
0
        goto fail_call;
1669
0
    }
1670
1671
0
    vq->dev = dev;
1672
1673
0
    if (dev->vhost_ops->vhost_set_vring_err) {
1674
0
        r = event_notifier_init(&vq->error_notifier, 0);
1675
0
        if (r < 0) {
1676
0
            goto fail_call;
1677
0
        }
1678
1679
0
        file.fd = event_notifier_get_fd(&vq->error_notifier);
1680
0
        r = dev->vhost_ops->vhost_set_vring_err(dev, &file);
1681
0
        if (r) {
1682
0
            VHOST_OPS_DEBUG(r, "vhost_set_vring_err failed");
1683
0
            goto fail_err;
1684
0
        }
1685
1686
0
        event_notifier_set_handler(&vq->error_notifier,
1687
0
                                   vhost_virtqueue_error_notifier);
1688
0
    }
1689
1690
0
    if (busyloop_timeout) {
1691
0
        r = vhost_virtqueue_set_busyloop_timeout(dev, n, busyloop_timeout);
1692
0
        if (r < 0) {
1693
0
            VHOST_OPS_DEBUG(r, "Failed to set busyloop timeout");
1694
0
            goto fail_err;
1695
0
        }
1696
0
    }
1697
1698
0
    return 0;
1699
1700
0
fail_err:
1701
0
    event_notifier_cleanup(&vq->error_notifier);
1702
0
fail_call:
1703
0
    event_notifier_cleanup(&vq->masked_notifier);
1704
0
    return r;
1705
0
}
1706
1707
static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq)
1708
0
{
1709
0
    event_notifier_cleanup(&vq->masked_notifier);
1710
0
    if (vq->dev && vq->dev->vhost_ops->vhost_set_vring_err) {
1711
0
        event_notifier_set_handler(&vq->error_notifier, NULL);
1712
0
        event_notifier_cleanup(&vq->error_notifier);
1713
0
    }
1714
0
}
1715
1716
static int vhost_dev_init_features(struct vhost_dev *hdev)
1717
0
{
1718
0
    uint64_t features64;
1719
0
    int r;
1720
1721
0
    if (hdev->vhost_ops->vhost_get_features_ex) {
1722
0
        return hdev->vhost_ops->vhost_get_features_ex(hdev, hdev->_features_ex);
1723
0
    }
1724
1725
0
    r = hdev->vhost_ops->vhost_get_features(hdev, &features64);
1726
0
    virtio_features_from_u64(hdev->_features_ex, features64);
1727
0
    return r;
1728
0
}
1729
1730
int vhost_dev_init_backend(struct vhost_dev *hdev, void *opaque,
1731
                           VhostBackendType backend_type, Error **errp)
1732
0
{
1733
0
    int r;
1734
1735
0
    r = vhost_set_backend_type(hdev, backend_type);
1736
0
    assert(r >= 0);
1737
1738
0
    r = hdev->vhost_ops->vhost_init(hdev, opaque, errp);
1739
0
    if (r < 0) {
1740
0
        return r;
1741
0
    }
1742
1743
0
    r = vhost_dev_init_features(hdev);
1744
0
    if (r < 0) {
1745
0
        error_setg_errno(errp, -r, "vhost_init_features failed");
1746
0
        return r;
1747
0
    }
1748
1749
0
    return 0;
1750
0
}
1751
1752
int vhost_dev_set_owner(struct vhost_dev *hdev)
1753
0
{
1754
0
    int r;
1755
1756
0
    assert(hdev->vhost_ops);
1757
0
    if (!hdev->vhost_ops->vhost_set_owner) {
1758
0
        return -ENOSYS;
1759
0
    }
1760
0
    r = hdev->vhost_ops->vhost_set_owner(hdev);
1761
0
    if (!r) {
1762
0
        hdev->owner = true;
1763
0
    }
1764
0
    return r;
1765
0
}
1766
1767
int vhost_dev_reset_owner(struct vhost_dev *hdev)
1768
0
{
1769
0
    int r;
1770
1771
0
    assert(hdev->vhost_ops);
1772
0
    if (!hdev->vhost_ops->vhost_reset_owner) {
1773
0
        return -ENOSYS;
1774
0
    }
1775
0
    r = hdev->vhost_ops->vhost_reset_owner(hdev);
1776
0
    if (!r) {
1777
0
        hdev->owner = false;
1778
0
    }
1779
0
    return r;
1780
0
}
1781
1782
int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
1783
                   VhostBackendType backend_type, uint32_t busyloop_timeout,
1784
                   Error **errp)
1785
0
{
1786
0
    unsigned int used, reserved, limit;
1787
0
    int i, r, n_initialized_vqs = 0;
1788
1789
0
    trace_vhost_dev_init_in(hdev);
1790
1791
0
    hdev->vdev = NULL;
1792
0
    hdev->migration_blocker = NULL;
1793
1794
0
    r = vhost_dev_init_backend(hdev, opaque, backend_type, errp);
1795
0
    if (r < 0) {
1796
0
        goto fail;
1797
0
    }
1798
1799
0
    r = vhost_dev_set_owner(hdev);
1800
0
    if (r < 0) {
1801
0
        error_setg_errno(errp, -r, "vhost_set_owner failed");
1802
0
        goto fail;
1803
0
    }
1804
1805
0
    limit = hdev->vhost_ops->vhost_memslots_limit(hdev);
1806
0
    if (limit < MEMORY_DEVICES_SAFE_MAX_MEMSLOTS &&
1807
0
        memory_devices_memslot_auto_decision_active()) {
1808
0
        error_setg(errp, "some memory device (like virtio-mem)"
1809
0
            " decided how many memory slots to use based on the overall"
1810
0
            " number of memory slots; this vhost backend would further"
1811
0
            " restricts the overall number of memory slots");
1812
0
        error_append_hint(errp, "Try plugging this vhost backend before"
1813
0
            " plugging such memory devices.\n");
1814
0
        r = -EINVAL;
1815
0
        goto fail;
1816
0
    }
1817
1818
0
    for (i = 0; i < hdev->nvqs; ++i, ++n_initialized_vqs) {
1819
0
        r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i,
1820
0
                                 busyloop_timeout);
1821
0
        if (r < 0) {
1822
0
            error_setg_errno(errp, -r, "Failed to initialize virtqueue %d", i);
1823
0
            goto fail;
1824
0
        }
1825
0
    }
1826
1827
0
    hdev->memory_listener = (MemoryListener) {
1828
0
        .name = "vhost",
1829
0
        .begin = vhost_begin,
1830
0
        .commit = vhost_commit,
1831
0
        .region_add = vhost_region_addnop,
1832
0
        .region_nop = vhost_region_addnop,
1833
0
        .log_start = vhost_log_start,
1834
0
        .log_stop = vhost_log_stop,
1835
0
        .log_sync = vhost_log_sync,
1836
0
        .log_global_start = vhost_log_global_start,
1837
0
        .log_global_stop = vhost_log_global_stop,
1838
0
        .priority = MEMORY_LISTENER_PRIORITY_DEV_BACKEND
1839
0
    };
1840
1841
0
    hdev->iommu_listener = (MemoryListener) {
1842
0
        .name = "vhost-iommu",
1843
0
        .region_add = vhost_iommu_region_add,
1844
0
        .region_del = vhost_iommu_region_del,
1845
0
    };
1846
1847
0
    if (hdev->migration_blocker == NULL) {
1848
0
        if (!vhost_dev_has_feature_ex(hdev, VHOST_F_LOG_ALL)) {
1849
0
            error_setg(&hdev->migration_blocker,
1850
0
                       "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
1851
0
        } else if (vhost_dev_log_is_shared(hdev) && !qemu_memfd_alloc_check()) {
1852
0
            error_setg(&hdev->migration_blocker,
1853
0
                       "Migration disabled: failed to allocate shared memory");
1854
0
        }
1855
0
    }
1856
1857
0
    if (hdev->migration_blocker != NULL) {
1858
0
        r = migrate_add_blocker_normal(&hdev->migration_blocker, errp);
1859
0
        if (r < 0) {
1860
0
            goto fail;
1861
0
        }
1862
0
    }
1863
1864
0
    hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
1865
0
    hdev->n_mem_sections = 0;
1866
0
    hdev->mem_sections = NULL;
1867
0
    hdev->log = NULL;
1868
0
    hdev->log_size = 0;
1869
0
    hdev->log_enabled = false;
1870
0
    hdev->started = false;
1871
0
    memory_listener_register(&hdev->memory_listener, &address_space_memory);
1872
0
    hdev->exit_notifier.notify = vhost_exit_notify;
1873
0
    qemu_add_exit_notifier(&hdev->exit_notifier);
1874
0
    migration_add_notifier_modes(&hdev->migration_notifier,
1875
0
                                 vhost_migration_notify,
1876
0
                                 BIT(MIG_MODE_CPR_TRANSFER) |
1877
0
                                 BIT(MIG_MODE_CPR_EXEC));
1878
0
    QLIST_INSERT_HEAD(&vhost_devices, hdev, entry);
1879
1880
    /*
1881
     * The listener we registered properly setup the number of required
1882
     * memslots in vhost_commit().
1883
     */
1884
0
    used = hdev->mem->nregions;
1885
1886
    /*
1887
     * We assume that all reserved memslots actually require a real memslot
1888
     * in our vhost backend. This might not be true, for example, if the
1889
     * memslot would be ROM. If ever relevant, we can optimize for that --
1890
     * but we'll need additional information about the reservations.
1891
     */
1892
0
    reserved = memory_devices_get_reserved_memslots();
1893
0
    if (used + reserved > limit) {
1894
0
        error_setg(errp, "vhost backend memory slots limit (%d) is less"
1895
0
                   " than current number of used (%d) and reserved (%d)"
1896
0
                   " memory slots for memory devices.", limit, used, reserved);
1897
0
        r = -EINVAL;
1898
0
        goto fail;
1899
0
    }
1900
1901
0
    hdev->initialized = true;
1902
1903
0
    trace_vhost_dev_init_out(hdev);
1904
1905
0
    return 0;
1906
1907
0
fail:
1908
0
    hdev->nvqs = n_initialized_vqs;
1909
0
    vhost_dev_cleanup(hdev);
1910
0
    return r;
1911
0
}
1912
1913
void vhost_dev_cleanup(struct vhost_dev *hdev)
1914
0
{
1915
0
    int i;
1916
1917
0
    trace_vhost_dev_cleanup(hdev);
1918
1919
0
    for (i = 0; i < hdev->nvqs; ++i) {
1920
0
        vhost_virtqueue_cleanup(hdev->vqs + i);
1921
0
    }
1922
0
    if (hdev->mem) {
1923
        /* those are only safe after successful init */
1924
0
        memory_listener_unregister(&hdev->memory_listener);
1925
0
        QLIST_REMOVE(hdev, entry);
1926
0
    }
1927
0
    if (hdev->exit_notifier.notify) {
1928
0
        qemu_remove_exit_notifier(&hdev->exit_notifier);
1929
0
        hdev->exit_notifier.notify = NULL;
1930
0
    }
1931
0
    migration_remove_notifier(&hdev->migration_notifier);
1932
0
    migrate_del_blocker(&hdev->migration_blocker);
1933
0
    g_free(hdev->mem);
1934
0
    g_free(hdev->mem_sections);
1935
0
    if (hdev->vhost_ops) {
1936
        /*
1937
         * Release the device ownership if we hold it.  Needed e.g. for
1938
         * failing CPR target, so the source can re-acquire it.
1939
         */
1940
0
        vhost_dev_try_release_owner(hdev, "cleanup");
1941
0
        hdev->vhost_ops->vhost_cleanup(hdev);
1942
0
    }
1943
0
    assert(!hdev->log);
1944
1945
0
    memset(hdev, 0, sizeof(struct vhost_dev));
1946
0
}
1947
1948
void vhost_dev_disable_notifiers_nvqs(struct vhost_dev *hdev,
1949
                                      VirtIODevice *vdev,
1950
                                      unsigned int nvqs)
1951
0
{
1952
0
    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
1953
0
    int i, r;
1954
1955
    /*
1956
     * Batch all the host notifiers in a single transaction to avoid
1957
     * quadratic time complexity in address_space_update_ioeventfds().
1958
     */
1959
0
    memory_region_transaction_begin();
1960
1961
0
    for (i = 0; i < nvqs; ++i) {
1962
0
        r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
1963
0
                                         false);
1964
0
        if (r < 0) {
1965
0
            error_report("vhost VQ %d notifier cleanup failed: %d", i, -r);
1966
0
        }
1967
0
        assert(r >= 0);
1968
0
    }
1969
1970
    /*
1971
     * The transaction expects the ioeventfds to be open when it
1972
     * commits. Do it now, before the cleanup loop.
1973
     */
1974
0
    memory_region_transaction_commit();
1975
1976
0
    for (i = 0; i < nvqs; ++i) {
1977
0
        virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i);
1978
0
    }
1979
0
    virtio_device_release_ioeventfd(vdev);
1980
0
}
1981
1982
/* Stop processing guest IO notifications in qemu.
1983
 * Start processing them in vhost in kernel.
1984
 */
1985
int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
1986
0
{
1987
0
    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
1988
0
    int i, r;
1989
1990
    /* We will pass the notifiers to the kernel, make sure that QEMU
1991
     * doesn't interfere.
1992
     */
1993
0
    r = virtio_device_grab_ioeventfd(vdev);
1994
0
    if (r < 0) {
1995
0
        error_report("binding does not support host notifiers");
1996
0
        return r;
1997
0
    }
1998
1999
    /*
2000
     * Batch all the host notifiers in a single transaction to avoid
2001
     * quadratic time complexity in address_space_update_ioeventfds().
2002
     */
2003
0
    memory_region_transaction_begin();
2004
2005
0
    for (i = 0; i < hdev->nvqs; ++i) {
2006
0
        r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
2007
0
                                         true);
2008
0
        if (r < 0) {
2009
0
            error_report("vhost VQ %d notifier binding failed: %d", i, -r);
2010
0
            memory_region_transaction_commit();
2011
0
            vhost_dev_disable_notifiers_nvqs(hdev, vdev, i);
2012
0
            return r;
2013
0
        }
2014
0
    }
2015
2016
0
    memory_region_transaction_commit();
2017
2018
0
    return 0;
2019
0
}
2020
2021
/* Stop processing guest IO notifications in vhost.
2022
 * Start processing them in qemu.
2023
 * This might actually run the qemu handlers right away,
2024
 * so virtio in qemu must be completely setup when this is called.
2025
 */
2026
void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
2027
0
{
2028
0
    vhost_dev_disable_notifiers_nvqs(hdev, vdev, hdev->nvqs);
2029
0
}
2030
2031
/* Test and clear event pending status.
2032
 * Should be called after unmask to avoid losing events.
2033
 */
2034
bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n)
2035
0
{
2036
0
    struct vhost_virtqueue *vq = hdev->vqs + n - hdev->vq_index;
2037
0
    assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
2038
0
    return event_notifier_test_and_clear(&vq->masked_notifier);
2039
0
}
2040
2041
/* Mask/unmask events from this vq. */
2042
int vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
2043
                         bool mask)
2044
0
{
2045
0
    struct VirtQueue *vvq = virtio_get_queue(vdev, n);
2046
0
    int r, index = n - hdev->vq_index;
2047
0
    struct vhost_vring_file file;
2048
2049
    /* should only be called after backend is connected */
2050
0
    assert(hdev->vhost_ops);
2051
2052
0
    if (mask) {
2053
0
        assert(vdev->use_guest_notifier_mask);
2054
0
        file.fd = event_notifier_get_wfd(&hdev->vqs[index].masked_notifier);
2055
0
    } else {
2056
0
        file.fd = event_notifier_get_wfd(virtio_queue_get_guest_notifier(vvq));
2057
0
    }
2058
2059
0
    file.index = hdev->vhost_ops->vhost_get_vq_index(hdev, n);
2060
0
    r = hdev->vhost_ops->vhost_set_vring_call(hdev, &file);
2061
0
    if (r < 0) {
2062
0
        error_report("vhost_set_vring_call failed %d", -r);
2063
0
        return r;
2064
0
    }
2065
0
    return 0;
2066
0
}
2067
2068
bool vhost_config_pending(struct vhost_dev *hdev)
2069
0
{
2070
0
    assert(hdev->vhost_ops);
2071
0
    if ((hdev->started == false) ||
2072
0
        (hdev->vhost_ops->vhost_set_config_call == NULL)) {
2073
0
        return false;
2074
0
    }
2075
2076
0
    EventNotifier *notifier =
2077
0
        &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier;
2078
0
    return event_notifier_test_and_clear(notifier);
2079
0
}
2080
2081
void vhost_config_mask(struct vhost_dev *hdev, VirtIODevice *vdev, bool mask)
2082
0
{
2083
0
    int fd;
2084
0
    int r;
2085
0
    EventNotifier *notifier =
2086
0
        &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier;
2087
0
    EventNotifier *config_notifier = virtio_config_get_guest_notifier(vdev);
2088
0
    assert(hdev->vhost_ops);
2089
2090
0
    if ((hdev->started == false) ||
2091
0
        (hdev->vhost_ops->vhost_set_config_call == NULL)) {
2092
0
        return;
2093
0
    }
2094
0
    if (mask) {
2095
0
        assert(vdev->use_guest_notifier_mask);
2096
0
        fd = event_notifier_get_fd(notifier);
2097
0
    } else {
2098
0
        fd = event_notifier_get_fd(config_notifier);
2099
0
    }
2100
0
    r = hdev->vhost_ops->vhost_set_config_call(hdev, fd);
2101
0
    if (r < 0) {
2102
0
        error_report("vhost_set_config_call failed %d", -r);
2103
0
    }
2104
0
}
2105
2106
static void vhost_stop_config_intr(struct vhost_dev *dev)
2107
0
{
2108
0
    int fd = -1;
2109
0
    assert(dev->vhost_ops);
2110
0
    if (dev->vhost_ops->vhost_set_config_call) {
2111
0
        dev->vhost_ops->vhost_set_config_call(dev, fd);
2112
0
    }
2113
0
}
2114
2115
static void vhost_start_config_intr(struct vhost_dev *dev)
2116
0
{
2117
0
    int r;
2118
0
    EventNotifier *config_notifier =
2119
0
        virtio_config_get_guest_notifier(dev->vdev);
2120
2121
0
    assert(dev->vhost_ops);
2122
0
    int fd = event_notifier_get_fd(config_notifier);
2123
0
    if (dev->vhost_ops->vhost_set_config_call) {
2124
0
        r = dev->vhost_ops->vhost_set_config_call(dev, fd);
2125
0
        if (!r) {
2126
0
            event_notifier_set(config_notifier);
2127
0
        }
2128
0
    }
2129
0
}
2130
2131
void vhost_get_features_ex(struct vhost_dev *hdev,
2132
                           const int *feature_bits,
2133
                           uint64_t *features)
2134
0
{
2135
0
    const int *bit = feature_bits;
2136
2137
0
    while (*bit != VHOST_INVALID_FEATURE_BIT) {
2138
0
        if (!vhost_dev_has_feature_ex(hdev, *bit)) {
2139
0
            virtio_clear_feature_ex(features, *bit);
2140
0
        }
2141
0
        bit++;
2142
0
    }
2143
0
}
2144
2145
static bool vhost_inflight_buffer_pre_load(void *opaque, Error **errp)
2146
0
{
2147
0
    struct vhost_inflight *inflight = opaque;
2148
2149
0
    int fd = -1;
2150
0
    void *addr = qemu_memfd_alloc("vhost-inflight", inflight->size,
2151
0
                                  F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
2152
0
                                  &fd, errp);
2153
0
    if (!addr) {
2154
0
        return false;
2155
0
    }
2156
2157
0
    inflight->offset = 0;
2158
0
    inflight->addr = addr;
2159
0
    inflight->fd = fd;
2160
2161
0
    return true;
2162
0
}
2163
2164
const VMStateDescription vmstate_vhost_inflight_region_buffer = {
2165
    .name = "vhost-inflight-region/buffer",
2166
    .pre_load_errp = vhost_inflight_buffer_pre_load,
2167
    .fields = (const VMStateField[]) {
2168
        VMSTATE_VBUFFER_UINT64(addr, struct vhost_inflight, 0, NULL, size),
2169
        VMSTATE_END_OF_LIST()
2170
    }
2171
};
2172
2173
static bool vhost_inflight_region_post_load(void *opaque,
2174
                                           int version_id,
2175
                                           Error **errp)
2176
0
{
2177
0
    struct vhost_inflight *inflight = opaque;
2178
2179
0
    if (inflight->addr == NULL) {
2180
0
        error_setg(errp, "inflight buffer subsection has not been loaded");
2181
0
        return false;
2182
0
    }
2183
2184
0
    return true;
2185
0
}
2186
2187
const VMStateDescription vmstate_vhost_inflight_region = {
2188
    .name = "vhost-inflight-region",
2189
    .post_load_errp = vhost_inflight_region_post_load,
2190
    .fields = (const VMStateField[]) {
2191
        VMSTATE_UINT64(size, struct vhost_inflight),
2192
        VMSTATE_UINT16(queue_size, struct vhost_inflight),
2193
        VMSTATE_END_OF_LIST()
2194
    },
2195
    .subsections = (const VMStateDescription * const []) {
2196
        &vmstate_vhost_inflight_region_buffer,
2197
        NULL
2198
    }
2199
};
2200
2201
void vhost_ack_features_ex(struct vhost_dev *hdev, const int *feature_bits,
2202
                           const uint64_t *features)
2203
0
{
2204
0
    const int *bit = feature_bits;
2205
0
    while (*bit != VHOST_INVALID_FEATURE_BIT) {
2206
0
        if (virtio_has_feature_ex(features, *bit)) {
2207
0
            virtio_add_feature_ex(hdev->acked_features_ex, *bit);
2208
0
        }
2209
0
        bit++;
2210
0
    }
2211
0
}
2212
2213
int vhost_dev_get_config(struct vhost_dev *hdev, uint8_t *config,
2214
                         uint32_t config_len, Error **errp)
2215
0
{
2216
0
    assert(hdev->vhost_ops);
2217
2218
0
    if (hdev->vhost_ops->vhost_get_config) {
2219
0
        return hdev->vhost_ops->vhost_get_config(hdev, config, config_len,
2220
0
                                                 errp);
2221
0
    }
2222
2223
0
    error_setg(errp, "vhost_get_config not implemented");
2224
0
    return -ENOSYS;
2225
0
}
2226
2227
int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *data,
2228
                         uint32_t offset, uint32_t size, uint32_t flags)
2229
0
{
2230
0
    assert(hdev->vhost_ops);
2231
2232
0
    if (hdev->vhost_ops->vhost_set_config) {
2233
0
        return hdev->vhost_ops->vhost_set_config(hdev, data, offset,
2234
0
                                                 size, flags);
2235
0
    }
2236
2237
0
    return -ENOSYS;
2238
0
}
2239
2240
void vhost_dev_set_config_notifier(struct vhost_dev *hdev,
2241
                                   const VhostDevConfigOps *ops)
2242
0
{
2243
0
    hdev->config_ops = ops;
2244
0
}
2245
2246
void vhost_dev_free_inflight(struct vhost_inflight *inflight)
2247
0
{
2248
0
    if (inflight && inflight->addr) {
2249
0
        qemu_memfd_free(inflight->addr, inflight->size, inflight->fd);
2250
0
        inflight->addr = NULL;
2251
0
        inflight->fd = -1;
2252
0
    }
2253
0
}
2254
2255
int vhost_dev_prepare_inflight(struct vhost_dev *hdev, VirtIODevice *vdev)
2256
0
{
2257
0
    int r;
2258
2259
0
    if (hdev->vhost_ops->vhost_get_inflight_fd == NULL ||
2260
0
        hdev->vhost_ops->vhost_set_inflight_fd == NULL) {
2261
0
        return 0;
2262
0
    }
2263
2264
0
    hdev->vdev = vdev;
2265
2266
0
    r = vhost_dev_set_features(hdev, hdev->log_enabled);
2267
0
    if (r < 0) {
2268
0
        VHOST_OPS_DEBUG(r, "vhost_dev_prepare_inflight failed");
2269
0
        return r;
2270
0
    }
2271
2272
0
    return 0;
2273
0
}
2274
2275
int vhost_dev_set_inflight(struct vhost_dev *dev,
2276
                           struct vhost_inflight *inflight)
2277
0
{
2278
0
    int r;
2279
2280
0
    if (dev->vhost_ops->vhost_set_inflight_fd && inflight->addr) {
2281
0
        r = dev->vhost_ops->vhost_set_inflight_fd(dev, inflight);
2282
0
        if (r) {
2283
0
            VHOST_OPS_DEBUG(r, "vhost_set_inflight_fd failed");
2284
0
            return r;
2285
0
        }
2286
0
    }
2287
2288
0
    return 0;
2289
0
}
2290
2291
int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size,
2292
                           struct vhost_inflight *inflight)
2293
0
{
2294
0
    int r;
2295
2296
0
    if (dev->vhost_ops->vhost_get_inflight_fd) {
2297
0
        r = dev->vhost_ops->vhost_get_inflight_fd(dev, queue_size, inflight);
2298
0
        if (r) {
2299
0
            VHOST_OPS_DEBUG(r, "vhost_get_inflight_fd failed");
2300
0
            return r;
2301
0
        }
2302
0
    }
2303
2304
0
    return 0;
2305
0
}
2306
2307
/*
2308
 * Host notifiers must be enabled at this point.
2309
 *
2310
 * If @vrings is true, this function will enable all vrings before starting the
2311
 * device. If it is false, the vring initialization is left to be done by the
2312
 * caller.
2313
 */
2314
int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings)
2315
0
{
2316
0
    int i, r;
2317
2318
    /* should only be called after backend is connected */
2319
0
    assert(hdev->vhost_ops);
2320
2321
0
    trace_vhost_dev_start_in(hdev, vdev->name, vrings);
2322
2323
0
    vdev->vhost_started = true;
2324
0
    hdev->started = true;
2325
0
    hdev->vdev = vdev;
2326
2327
0
    r = vhost_dev_set_features(hdev, hdev->log_enabled);
2328
0
    if (r < 0) {
2329
0
        goto fail_features;
2330
0
    }
2331
2332
0
    if (vhost_dev_has_iommu(hdev)) {
2333
0
        memory_listener_register(&hdev->iommu_listener, vdev->dma_as);
2334
0
    }
2335
2336
0
    r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem);
2337
0
    if (r < 0) {
2338
0
        VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed");
2339
0
        goto fail_mem;
2340
0
    }
2341
0
    for (i = 0; i < hdev->nvqs; ++i) {
2342
0
        r = vhost_virtqueue_start(hdev,
2343
0
                                  vdev,
2344
0
                                  hdev->vqs + i,
2345
0
                                  hdev->vq_index + i);
2346
0
        if (r < 0) {
2347
0
            goto fail_vq;
2348
0
        }
2349
0
    }
2350
2351
0
    r = event_notifier_init(
2352
0
        &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier, 0);
2353
0
    if (r < 0) {
2354
0
        VHOST_OPS_DEBUG(r, "event_notifier_init failed");
2355
0
        goto fail_vq;
2356
0
    }
2357
0
    event_notifier_test_and_clear(
2358
0
        &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier);
2359
0
    if (!vdev->use_guest_notifier_mask) {
2360
0
        vhost_config_mask(hdev, vdev, true);
2361
0
    }
2362
0
    if (hdev->log_enabled) {
2363
0
        uint64_t log_base;
2364
2365
0
        hdev->log_size = vhost_get_log_size(hdev);
2366
0
        hdev->log = vhost_log_get(hdev->vhost_ops->backend_type,
2367
0
                                  hdev->log_size,
2368
0
                                  vhost_dev_log_is_shared(hdev));
2369
0
        log_base = (uintptr_t)hdev->log->log;
2370
0
        r = hdev->vhost_ops->vhost_set_log_base(hdev,
2371
0
                                                hdev->log_size ? log_base : 0,
2372
0
                                                hdev->log);
2373
0
        if (r < 0) {
2374
0
            VHOST_OPS_DEBUG(r, "vhost_set_log_base failed");
2375
0
            goto fail_log;
2376
0
        }
2377
0
        vhost_dev_elect_mem_logger(hdev, true);
2378
0
    }
2379
0
    if (vrings) {
2380
0
        r = vhost_dev_set_vring_enable(hdev, true);
2381
0
        if (r) {
2382
0
            goto fail_log;
2383
0
        }
2384
0
    }
2385
0
    if (hdev->vhost_ops->vhost_dev_start) {
2386
0
        r = hdev->vhost_ops->vhost_dev_start(hdev, true);
2387
0
        if (r) {
2388
0
            goto fail_start;
2389
0
        }
2390
0
    }
2391
0
    if (vhost_dev_has_iommu(hdev) &&
2392
0
        hdev->vhost_ops->vhost_set_iotlb_callback) {
2393
0
            hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true);
2394
2395
        /* Update used ring information for IOTLB to work correctly,
2396
         * vhost-kernel code requires for this.*/
2397
0
        for (i = 0; i < hdev->nvqs; ++i) {
2398
0
            struct vhost_virtqueue *vq = hdev->vqs + i;
2399
0
            r = vhost_device_iotlb_miss(hdev, vq->used_phys, true);
2400
0
            if (r) {
2401
0
                goto fail_iotlb;
2402
0
            }
2403
0
        }
2404
0
    }
2405
0
    vhost_start_config_intr(hdev);
2406
2407
0
    trace_vhost_dev_start_out(hdev, vdev->name);
2408
0
    return 0;
2409
0
fail_iotlb:
2410
0
    if (vhost_dev_has_iommu(hdev) &&
2411
0
        hdev->vhost_ops->vhost_set_iotlb_callback) {
2412
0
        hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
2413
0
    }
2414
0
    if (hdev->vhost_ops->vhost_dev_start) {
2415
0
        hdev->vhost_ops->vhost_dev_start(hdev, false);
2416
0
    }
2417
0
fail_start:
2418
0
    if (vrings) {
2419
0
        vhost_dev_set_vring_enable(hdev, false);
2420
0
    }
2421
0
fail_log:
2422
0
    vhost_log_put(hdev, false);
2423
0
fail_vq:
2424
0
    while (--i >= 0) {
2425
0
        vhost_virtqueue_stop(hdev,
2426
0
                             vdev,
2427
0
                             hdev->vqs + i,
2428
0
                             hdev->vq_index + i,
2429
0
                             false);
2430
0
    }
2431
2432
0
fail_mem:
2433
0
    if (vhost_dev_has_iommu(hdev)) {
2434
0
        memory_listener_unregister(&hdev->iommu_listener);
2435
0
    }
2436
0
fail_features:
2437
0
    vdev->vhost_started = false;
2438
0
    hdev->started = false;
2439
0
    return r;
2440
0
}
2441
2442
/* Host notifiers must be enabled at this point. */
2443
static int do_vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev,
2444
                             bool vrings, bool force, bool skip_drain)
2445
0
{
2446
0
    int i;
2447
0
    int rc = 0;
2448
0
    EventNotifier *config_notifier = virtio_config_get_guest_notifier(vdev);
2449
2450
    /* should only be called after backend is connected */
2451
0
    assert(hdev->vhost_ops);
2452
0
    event_notifier_test_and_clear(
2453
0
        &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier);
2454
0
    event_notifier_test_and_clear(config_notifier);
2455
0
    event_notifier_cleanup(
2456
0
        &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier);
2457
2458
0
    trace_vhost_dev_stop_in(hdev, vdev->name, vrings);
2459
2460
0
    if (hdev->vhost_ops->vhost_dev_start) {
2461
0
        hdev->vhost_ops->vhost_dev_start(hdev, false);
2462
0
    }
2463
0
    if (vrings) {
2464
0
        vhost_dev_set_vring_enable(hdev, false);
2465
0
    }
2466
0
    for (i = 0; i < hdev->nvqs; ++i) {
2467
0
        rc |= do_vhost_virtqueue_stop(hdev,
2468
0
                                      vdev,
2469
0
                                      hdev->vqs + i,
2470
0
                                      hdev->vq_index + i,
2471
0
                                      force,
2472
0
                                      skip_drain);
2473
0
    }
2474
0
    if (hdev->vhost_ops->vhost_reset_status) {
2475
0
        hdev->vhost_ops->vhost_reset_status(hdev);
2476
0
    }
2477
2478
0
    if (vhost_dev_has_iommu(hdev)) {
2479
0
        if (hdev->vhost_ops->vhost_set_iotlb_callback) {
2480
0
            hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
2481
0
        }
2482
0
        memory_listener_unregister(&hdev->iommu_listener);
2483
0
    }
2484
0
    vhost_stop_config_intr(hdev);
2485
0
    vhost_log_put(hdev, true);
2486
0
    hdev->started = false;
2487
0
    vdev->vhost_started = false;
2488
0
    hdev->vdev = NULL;
2489
2490
0
    trace_vhost_dev_stop_out(hdev, vdev->name);
2491
0
    return rc;
2492
0
}
2493
2494
int vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings,
2495
                   bool skip_drain)
2496
0
{
2497
0
    return do_vhost_dev_stop(hdev, vdev, vrings, false, skip_drain);
2498
0
}
2499
2500
int vhost_dev_force_stop(struct vhost_dev *hdev, VirtIODevice *vdev,
2501
                         bool vrings)
2502
0
{
2503
0
    return do_vhost_dev_stop(hdev, vdev, vrings, true, false);
2504
0
}
2505
2506
int vhost_net_set_backend(struct vhost_dev *hdev,
2507
                          struct vhost_vring_file *file)
2508
0
{
2509
0
    if (hdev->vhost_ops->vhost_net_set_backend) {
2510
0
        return hdev->vhost_ops->vhost_net_set_backend(hdev, file);
2511
0
    }
2512
2513
0
    return -ENOSYS;
2514
0
}
2515
2516
int vhost_reset_device(struct vhost_dev *hdev)
2517
0
{
2518
0
    if (hdev->vhost_ops->vhost_reset_device) {
2519
0
        return hdev->vhost_ops->vhost_reset_device(hdev);
2520
0
    }
2521
2522
0
    return -ENOSYS;
2523
0
}
2524
2525
bool vhost_supports_device_state(struct vhost_dev *dev)
2526
0
{
2527
0
    if (dev->vhost_ops->vhost_supports_device_state) {
2528
0
        return dev->vhost_ops->vhost_supports_device_state(dev);
2529
0
    }
2530
2531
0
    return false;
2532
0
}
2533
2534
int vhost_set_device_state_fd(struct vhost_dev *dev,
2535
                              VhostDeviceStateDirection direction,
2536
                              VhostDeviceStatePhase phase,
2537
                              int fd,
2538
                              int *reply_fd,
2539
                              Error **errp)
2540
0
{
2541
0
    if (dev->vhost_ops->vhost_set_device_state_fd) {
2542
0
        return dev->vhost_ops->vhost_set_device_state_fd(dev, direction, phase,
2543
0
                                                         fd, reply_fd, errp);
2544
0
    }
2545
2546
0
    error_setg(errp,
2547
0
               "vhost transport does not support migration state transfer");
2548
0
    return -ENOSYS;
2549
0
}
2550
2551
int vhost_check_device_state(struct vhost_dev *dev, Error **errp)
2552
0
{
2553
0
    if (dev->vhost_ops->vhost_check_device_state) {
2554
0
        return dev->vhost_ops->vhost_check_device_state(dev, errp);
2555
0
    }
2556
2557
0
    error_setg(errp,
2558
0
               "vhost transport does not support migration state transfer");
2559
0
    return -ENOSYS;
2560
0
}
2561
2562
int vhost_save_backend_state(struct vhost_dev *dev, QEMUFile *f, Error **errp)
2563
0
{
2564
0
    ERRP_GUARD();
2565
    /* Maximum chunk size in which to transfer the state */
2566
0
    const size_t chunk_size = 1 * 1024 * 1024;
2567
0
    g_autofree void *transfer_buf = NULL;
2568
0
    g_autoptr(GError) g_err = NULL;
2569
0
    int pipe_fds[2], read_fd = -1, write_fd = -1, reply_fd = -1;
2570
0
    int ret;
2571
2572
    /* [0] for reading (our end), [1] for writing (back-end's end) */
2573
0
    if (!g_unix_open_pipe(pipe_fds, FD_CLOEXEC, &g_err)) {
2574
0
        error_setg(errp, "Failed to set up state transfer pipe: %s",
2575
0
                   g_err->message);
2576
0
        ret = -EINVAL;
2577
0
        goto fail;
2578
0
    }
2579
2580
0
    read_fd = pipe_fds[0];
2581
0
    write_fd = pipe_fds[1];
2582
2583
    /*
2584
     * VHOST_TRANSFER_STATE_PHASE_STOPPED means the device must be stopped.
2585
     * Ideally, it is suspended, but SUSPEND/RESUME currently do not exist for
2586
     * vhost-user, so just check that it is stopped at all.
2587
     */
2588
0
    assert(!dev->started);
2589
2590
    /* Transfer ownership of write_fd to the back-end */
2591
0
    ret = vhost_set_device_state_fd(dev,
2592
0
                                    VHOST_TRANSFER_STATE_DIRECTION_SAVE,
2593
0
                                    VHOST_TRANSFER_STATE_PHASE_STOPPED,
2594
0
                                    write_fd,
2595
0
                                    &reply_fd,
2596
0
                                    errp);
2597
0
    if (ret < 0) {
2598
0
        error_prepend(errp, "Failed to initiate state transfer: ");
2599
0
        goto fail;
2600
0
    }
2601
2602
    /* If the back-end wishes to use a different pipe, switch over */
2603
0
    if (reply_fd >= 0) {
2604
0
        close(read_fd);
2605
0
        read_fd = reply_fd;
2606
0
    }
2607
2608
0
    transfer_buf = g_malloc(chunk_size);
2609
2610
0
    while (true) {
2611
0
        ssize_t read_ret;
2612
2613
0
        read_ret = RETRY_ON_EINTR(read(read_fd, transfer_buf, chunk_size));
2614
0
        if (read_ret < 0) {
2615
0
            ret = -errno;
2616
0
            error_setg_errno(errp, -ret, "Failed to receive state");
2617
0
            goto fail;
2618
0
        }
2619
2620
0
        assert(read_ret <= chunk_size);
2621
0
        qemu_put_be32(f, read_ret);
2622
2623
0
        if (read_ret == 0) {
2624
            /* EOF */
2625
0
            break;
2626
0
        }
2627
2628
0
        qemu_put_buffer(f, transfer_buf, read_ret);
2629
0
    }
2630
2631
    /*
2632
     * Back-end will not really care, but be clean and close our end of the pipe
2633
     * before inquiring the back-end about whether transfer was successful
2634
     */
2635
0
    close(read_fd);
2636
0
    read_fd = -1;
2637
2638
    /* Also, verify that the device is still stopped */
2639
0
    assert(!dev->started);
2640
2641
0
    ret = vhost_check_device_state(dev, errp);
2642
0
    if (ret < 0) {
2643
0
        goto fail;
2644
0
    }
2645
2646
0
    ret = 0;
2647
0
fail:
2648
0
    if (read_fd >= 0) {
2649
0
        close(read_fd);
2650
0
    }
2651
2652
0
    return ret;
2653
0
}
2654
2655
int vhost_load_backend_state(struct vhost_dev *dev, QEMUFile *f, Error **errp)
2656
0
{
2657
0
    ERRP_GUARD();
2658
0
    size_t transfer_buf_size = 0;
2659
0
    g_autofree void *transfer_buf = NULL;
2660
0
    g_autoptr(GError) g_err = NULL;
2661
0
    int pipe_fds[2], read_fd = -1, write_fd = -1, reply_fd = -1;
2662
0
    int ret;
2663
2664
    /* [0] for reading (back-end's end), [1] for writing (our end) */
2665
0
    if (!g_unix_open_pipe(pipe_fds, FD_CLOEXEC, &g_err)) {
2666
0
        error_setg(errp, "Failed to set up state transfer pipe: %s",
2667
0
                   g_err->message);
2668
0
        ret = -EINVAL;
2669
0
        goto fail;
2670
0
    }
2671
2672
0
    read_fd = pipe_fds[0];
2673
0
    write_fd = pipe_fds[1];
2674
2675
    /*
2676
     * VHOST_TRANSFER_STATE_PHASE_STOPPED means the device must be stopped.
2677
     * Ideally, it is suspended, but SUSPEND/RESUME currently do not exist for
2678
     * vhost-user, so just check that it is stopped at all.
2679
     */
2680
0
    assert(!dev->started);
2681
2682
    /* Transfer ownership of read_fd to the back-end */
2683
0
    ret = vhost_set_device_state_fd(dev,
2684
0
                                    VHOST_TRANSFER_STATE_DIRECTION_LOAD,
2685
0
                                    VHOST_TRANSFER_STATE_PHASE_STOPPED,
2686
0
                                    read_fd,
2687
0
                                    &reply_fd,
2688
0
                                    errp);
2689
0
    if (ret < 0) {
2690
0
        error_prepend(errp, "Failed to initiate state transfer: ");
2691
0
        goto fail;
2692
0
    }
2693
2694
    /* If the back-end wishes to use a different pipe, switch over */
2695
0
    if (reply_fd >= 0) {
2696
0
        close(write_fd);
2697
0
        write_fd = reply_fd;
2698
0
    }
2699
2700
0
    while (true) {
2701
0
        size_t this_chunk_size = qemu_get_be32(f);
2702
0
        ssize_t write_ret;
2703
0
        const uint8_t *transfer_pointer;
2704
2705
0
        if (this_chunk_size == 0) {
2706
            /* End of state */
2707
0
            break;
2708
0
        }
2709
2710
0
        if (transfer_buf_size < this_chunk_size) {
2711
0
            transfer_buf = g_realloc(transfer_buf, this_chunk_size);
2712
0
            transfer_buf_size = this_chunk_size;
2713
0
        }
2714
2715
0
        if (qemu_get_buffer(f, transfer_buf, this_chunk_size) <
2716
0
                this_chunk_size)
2717
0
        {
2718
0
            error_setg(errp, "Failed to read state");
2719
0
            ret = -EINVAL;
2720
0
            goto fail;
2721
0
        }
2722
2723
0
        transfer_pointer = transfer_buf;
2724
0
        while (this_chunk_size > 0) {
2725
0
            write_ret = RETRY_ON_EINTR(
2726
0
                write(write_fd, transfer_pointer, this_chunk_size)
2727
0
            );
2728
0
            if (write_ret < 0) {
2729
0
                ret = -errno;
2730
0
                error_setg_errno(errp, -ret, "Failed to send state");
2731
0
                goto fail;
2732
0
            } else if (write_ret == 0) {
2733
0
                error_setg(errp, "Failed to send state: Connection is closed");
2734
0
                ret = -ECONNRESET;
2735
0
                goto fail;
2736
0
            }
2737
2738
0
            assert(write_ret <= this_chunk_size);
2739
0
            this_chunk_size -= write_ret;
2740
0
            transfer_pointer += write_ret;
2741
0
        }
2742
0
    }
2743
2744
    /*
2745
     * Close our end, thus ending transfer, before inquiring the back-end about
2746
     * whether transfer was successful
2747
     */
2748
0
    close(write_fd);
2749
0
    write_fd = -1;
2750
2751
    /* Also, verify that the device is still stopped */
2752
0
    assert(!dev->started);
2753
2754
0
    ret = vhost_check_device_state(dev, errp);
2755
0
    if (ret < 0) {
2756
0
        goto fail;
2757
0
    }
2758
2759
0
    ret = 0;
2760
0
fail:
2761
0
    if (write_fd >= 0) {
2762
0
        close(write_fd);
2763
0
    }
2764
2765
0
    return ret;
2766
0
}