Coverage Report

Created: 2026-10-31 08:12

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qemu/hw/i386/kvm/xen_evtchn.c
Line
Count
Source
1
/*
2
 * QEMU Xen emulation: Event channel support
3
 *
4
 * Copyright © 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
5
 *
6
 * Authors: David Woodhouse <dwmw2@infradead.org>
7
 *
8
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
9
 * See the COPYING file in the top-level directory.
10
 */
11
12
#include "qemu/osdep.h"
13
#include "qemu/host-utils.h"
14
#include "qemu/module.h"
15
#include "qemu/lockable.h"
16
#include "qemu/main-loop.h"
17
#include "qemu/log.h"
18
#include "qemu/error-report.h"
19
#include "monitor/monitor.h"
20
#include "monitor/hmp.h"
21
#include "qapi/error.h"
22
#include "qapi/qapi-commands-misc-i386.h"
23
#include "qobject/qdict.h"
24
#include "qom/object.h"
25
#include "exec/target_page.h"
26
#include "system/address-spaces.h"
27
#include "migration/vmstate.h"
28
#include "trace.h"
29
30
#include "hw/core/sysbus.h"
31
#include "hw/xen/xen.h"
32
#include "hw/i386/x86.h"
33
#include "hw/i386/pc.h"
34
#include "hw/pci/pci.h"
35
#include "hw/pci/msi.h"
36
#include "hw/pci/msix.h"
37
#include "hw/core/irq.h"
38
#include "hw/xen/xen_backend_ops.h"
39
40
#include "xen_evtchn.h"
41
#include "xen_overlay.h"
42
#include "xen_xenstore.h"
43
44
#include "system/kvm.h"
45
#include "system/kvm_xen.h"
46
#include <linux/kvm.h>
47
#include <sys/eventfd.h>
48
49
#include "hw/xen/interface/memory.h"
50
#include "hw/xen/interface/hvm/params.h"
51
52
/* XX: For kvm_update_msi_routes_all() */
53
#include "target/i386/kvm/kvm_i386.h"
54
55
0
#define TYPE_XEN_EVTCHN "xen-evtchn"
56
OBJECT_DECLARE_SIMPLE_TYPE(XenEvtchnState, XEN_EVTCHN)
57
58
typedef struct XenEvtchnPort {
59
    uint32_t vcpu;      /* Xen/ACPI vcpu_id */
60
    uint16_t type;      /* EVTCHNSTAT_xxxx */
61
    union {
62
        uint16_t val;  /* raw value for serialization etc. */
63
        uint16_t pirq;
64
        uint16_t virq;
65
        struct {
66
            uint16_t port:15;
67
            uint16_t to_qemu:1; /* Only two targets; qemu or loopback */
68
        } interdomain;
69
    } u;
70
} XenEvtchnPort;
71
72
/* 32-bit compatibility definitions, also used natively in 32-bit build */
73
struct compat_arch_vcpu_info {
74
    unsigned int cr2;
75
    unsigned int pad[5];
76
};
77
78
struct compat_vcpu_info {
79
    uint8_t evtchn_upcall_pending;
80
    uint8_t evtchn_upcall_mask;
81
    uint16_t pad;
82
    uint32_t evtchn_pending_sel;
83
    struct compat_arch_vcpu_info arch;
84
    struct vcpu_time_info time;
85
}; /* 64 bytes (x86) */
86
87
struct compat_arch_shared_info {
88
    unsigned int max_pfn;
89
    unsigned int pfn_to_mfn_frame_list_list;
90
    unsigned int nmi_reason;
91
    unsigned int p2m_cr3;
92
    unsigned int p2m_vaddr;
93
    unsigned int p2m_generation;
94
    uint32_t wc_sec_hi;
95
};
96
97
struct compat_shared_info {
98
    struct compat_vcpu_info vcpu_info[XEN_LEGACY_MAX_VCPUS];
99
    uint32_t evtchn_pending[32];
100
    uint32_t evtchn_mask[32];
101
    uint32_t wc_version;      /* Version counter: see vcpu_time_info_t. */
102
    uint32_t wc_sec;
103
    uint32_t wc_nsec;
104
    struct compat_arch_shared_info arch;
105
};
106
107
0
#define COMPAT_EVTCHN_2L_NR_CHANNELS            1024
108
109
/* Local private implementation of struct xenevtchn_handle */
110
struct xenevtchn_handle {
111
    evtchn_port_t be_port;
112
    evtchn_port_t guest_port; /* Or zero for unbound */
113
    int fd;
114
};
115
116
/*
117
 * These 'emuirq' values are used by Xen in the LM stream... and yes, I am
118
 * insane enough to think about guest-transparent live migration from actual
119
 * Xen to QEMU, and ensuring that we can convert/consume the stream.
120
 */
121
0
#define IRQ_UNBOUND -1
122
#define IRQ_PT -2
123
0
#define IRQ_MSI_EMU -3
124
125
126
struct pirq_info {
127
    int gsi;
128
    uint16_t port;
129
    PCIDevice *dev;
130
    int vector;
131
    bool is_msix;
132
    bool is_masked;
133
    bool is_translated;
134
};
135
136
struct XenEvtchnState {
137
    /*< private >*/
138
    SysBusDevice busdev;
139
    /*< public >*/
140
141
    uint64_t callback_param;
142
    bool evtchn_in_kernel;
143
    bool setting_callback_gsi;
144
    int extern_gsi_level;
145
    uint32_t callback_gsi;
146
147
    QEMUBH *gsi_bh;
148
149
    QemuMutex port_lock;
150
    uint32_t nr_ports;
151
    XenEvtchnPort port_table[EVTCHN_2L_NR_CHANNELS];
152
153
    /* Connected to the system GSIs for raising callback as GSI / INTx */
154
    unsigned int nr_callback_gsis;
155
    qemu_irq *callback_gsis;
156
157
    struct xenevtchn_handle *be_handles[EVTCHN_2L_NR_CHANNELS];
158
159
    uint32_t nr_pirqs;
160
161
    /* Bitmap of allocated PIRQs (serialized) */
162
    uint16_t nr_pirq_inuse_words;
163
    uint64_t *pirq_inuse_bitmap;
164
165
    /* GSI → PIRQ mapping (serialized) */
166
    uint16_t gsi_pirq[IOAPIC_NUM_PINS];
167
168
    /* Per-GSI assertion state (serialized) */
169
    uint32_t pirq_gsi_set;
170
171
    /* Per-PIRQ information (rebuilt on migration, protected by BQL) */
172
    struct pirq_info *pirq;
173
};
174
175
0
#define pirq_inuse_word(s, pirq) (s->pirq_inuse_bitmap[((pirq) / 64)])
176
0
#define pirq_inuse_bit(pirq) (1ULL << ((pirq) & 63))
177
178
0
#define pirq_inuse(s, pirq) (pirq_inuse_word(s, pirq) & pirq_inuse_bit(pirq))
179
180
struct XenEvtchnState *xen_evtchn_singleton;
181
182
/* Top bits of callback_param are the type (HVM_PARAM_CALLBACK_TYPE_xxx) */
183
0
#define CALLBACK_VIA_TYPE_SHIFT 56
184
185
static void unbind_backend_ports(XenEvtchnState *s);
186
187
static int xen_evtchn_pre_load(void *opaque)
188
0
{
189
0
    XenEvtchnState *s = opaque;
190
191
    /* Unbind all the backend-side ports; they need to rebind */
192
0
    unbind_backend_ports(s);
193
194
    /* It'll be leaked otherwise. */
195
0
    g_free(s->pirq_inuse_bitmap);
196
0
    s->pirq_inuse_bitmap = NULL;
197
198
0
    return 0;
199
0
}
200
201
static int xen_evtchn_post_load(void *opaque, int version_id)
202
0
{
203
0
    XenEvtchnState *s = opaque;
204
0
    uint32_t i;
205
206
0
    if (s->callback_param) {
207
0
        xen_evtchn_set_callback_param(s->callback_param);
208
0
    }
209
210
    /* Rebuild s->pirq[].port mapping */
211
0
    for (i = 0; i < s->nr_ports; i++) {
212
0
        XenEvtchnPort *p = &s->port_table[i];
213
214
0
        if (p->type == EVTCHNSTAT_pirq) {
215
0
            assert(p->u.pirq);
216
0
            assert(p->u.pirq < s->nr_pirqs);
217
218
            /*
219
             * Set the gsi to IRQ_UNBOUND; it may be changed to an actual
220
             * GSI# below, or to IRQ_MSI_EMU when the MSI table snooping
221
             * catches up with it.
222
             */
223
0
            s->pirq[p->u.pirq].gsi = IRQ_UNBOUND;
224
0
            s->pirq[p->u.pirq].port = i;
225
0
        }
226
0
    }
227
    /* Rebuild s->pirq[].gsi mapping */
228
0
    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
229
0
        if (s->gsi_pirq[i]) {
230
0
            s->pirq[s->gsi_pirq[i]].gsi = i;
231
0
        }
232
0
    }
233
0
    return 0;
234
0
}
235
236
static bool xen_evtchn_is_needed(void *opaque)
237
0
{
238
0
    return xen_mode == XEN_EMULATE;
239
0
}
240
241
static const VMStateDescription xen_evtchn_port_vmstate = {
242
    .name = "xen_evtchn_port",
243
    .version_id = 1,
244
    .minimum_version_id = 1,
245
    .fields = (const VMStateField[]) {
246
        VMSTATE_UINT32(vcpu, XenEvtchnPort),
247
        VMSTATE_UINT16(type, XenEvtchnPort),
248
        VMSTATE_UINT16(u.val, XenEvtchnPort),
249
        VMSTATE_END_OF_LIST()
250
    }
251
};
252
253
static const VMStateDescription xen_evtchn_vmstate = {
254
    .name = "xen_evtchn",
255
    .version_id = 1,
256
    .minimum_version_id = 1,
257
    .needed = xen_evtchn_is_needed,
258
    .pre_load = xen_evtchn_pre_load,
259
    .post_load = xen_evtchn_post_load,
260
    .fields = (const VMStateField[]) {
261
        VMSTATE_UINT64(callback_param, XenEvtchnState),
262
        VMSTATE_UINT32(nr_ports, XenEvtchnState),
263
        VMSTATE_STRUCT_VARRAY_UINT32(port_table, XenEvtchnState, nr_ports, 1,
264
                                     xen_evtchn_port_vmstate, XenEvtchnPort),
265
        VMSTATE_UINT16_ARRAY(gsi_pirq, XenEvtchnState, IOAPIC_NUM_PINS),
266
        VMSTATE_VARRAY_UINT16_ALLOC(pirq_inuse_bitmap, XenEvtchnState,
267
                                    nr_pirq_inuse_words, 0,
268
                                    vmstate_info_uint64, uint64_t),
269
        VMSTATE_UINT32(pirq_gsi_set, XenEvtchnState),
270
        VMSTATE_END_OF_LIST()
271
    }
272
};
273
274
static void xen_evtchn_class_init(ObjectClass *klass, const void *data)
275
61
{
276
61
    DeviceClass *dc = DEVICE_CLASS(klass);
277
278
61
    dc->vmsd = &xen_evtchn_vmstate;
279
61
}
280
281
static const TypeInfo xen_evtchn_info = {
282
    .name          = TYPE_XEN_EVTCHN,
283
    .parent        = TYPE_SYS_BUS_DEVICE,
284
    .instance_size = sizeof(XenEvtchnState),
285
    .class_init    = xen_evtchn_class_init,
286
};
287
288
static struct evtchn_backend_ops emu_evtchn_backend_ops = {
289
    .open = xen_be_evtchn_open,
290
    .bind_interdomain = xen_be_evtchn_bind_interdomain,
291
    .unbind = xen_be_evtchn_unbind,
292
    .close = xen_be_evtchn_close,
293
    .get_fd = xen_be_evtchn_fd,
294
    .notify = xen_be_evtchn_notify,
295
    .unmask = xen_be_evtchn_unmask,
296
    .pending = xen_be_evtchn_pending,
297
};
298
299
static void gsi_assert_bh(void *opaque)
300
0
{
301
0
    struct vcpu_info *vi = kvm_xen_get_vcpu_info_hva(0);
302
0
    if (vi) {
303
0
        xen_evtchn_set_callback_level(!!vi->evtchn_upcall_pending);
304
0
    }
305
0
}
306
307
void xen_evtchn_create(unsigned int nr_gsis, qemu_irq *system_gsis)
308
0
{
309
0
    XenEvtchnState *s = XEN_EVTCHN(sysbus_create_simple(TYPE_XEN_EVTCHN,
310
0
                                                        -1, NULL));
311
0
    int i;
312
313
0
    xen_evtchn_singleton = s;
314
315
0
    qemu_mutex_init(&s->port_lock);
316
0
    s->gsi_bh = aio_bh_new(qemu_get_aio_context(), gsi_assert_bh, s);
317
318
    /*
319
     * These are the *output* GSI from event channel support, for
320
     * signalling CPU0's events via GSI or PCI INTx instead of the
321
     * per-CPU vector. We create a *set* of irqs and connect one to
322
     * each of the system GSIs which were passed in from the platform
323
     * code, and then just trigger the right one as appropriate from
324
     * xen_evtchn_set_callback_level().
325
     */
326
0
    s->nr_callback_gsis = nr_gsis;
327
0
    s->callback_gsis = g_new0(qemu_irq, nr_gsis);
328
0
    for (i = 0; i < nr_gsis; i++) {
329
0
        sysbus_init_irq(SYS_BUS_DEVICE(s), &s->callback_gsis[i]);
330
0
        sysbus_connect_irq(SYS_BUS_DEVICE(s), i, system_gsis[i]);
331
0
    }
332
333
    /*
334
     * The Xen scheme for encoding PIRQ# into an MSI message is not
335
     * compatible with 32-bit MSI, as it puts the high bits of the
336
     * PIRQ# into the high bits of the MSI message address, instead of
337
     * using the Extended Destination ID in address bits 4-11 which
338
     * perhaps would have been a better choice.
339
     *
340
     * To keep life simple, kvm_accel_instance_init() initialises the
341
     * default to 256. which conveniently doesn't need to set anything
342
     * outside the low 32 bits of the address. It can be increased by
343
     * setting the xen-evtchn-max-pirq property.
344
     */
345
0
    s->nr_pirqs = kvm_xen_get_evtchn_max_pirq();
346
347
0
    s->nr_pirq_inuse_words = DIV_ROUND_UP(s->nr_pirqs, 64);
348
0
    s->pirq_inuse_bitmap = g_new0(uint64_t, s->nr_pirq_inuse_words);
349
0
    s->pirq = g_new0(struct pirq_info, s->nr_pirqs);
350
351
    /* Set event channel functions for backend drivers to use */
352
0
    xen_evtchn_ops = &emu_evtchn_backend_ops;
353
0
}
354
355
static void xen_evtchn_register_types(void)
356
61
{
357
61
    type_register_static(&xen_evtchn_info);
358
61
}
359
360
type_init(xen_evtchn_register_types)
361
362
static int set_callback_pci_intx(XenEvtchnState *s, uint64_t param)
363
0
{
364
0
    PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
365
0
    uint8_t pin = param & 3;
366
0
    uint8_t devfn = (param >> 8) & 0xff;
367
0
    uint16_t bus = (param >> 16) & 0xffff;
368
0
    uint16_t domain = (param >> 32) & 0xffff;
369
0
    PCIDevice *pdev;
370
0
    PCIINTxRoute r;
371
372
0
    if (domain || !pcms) {
373
0
        return 0;
374
0
    }
375
376
0
    pdev = pci_find_device(pcms->pcibus, bus, devfn);
377
0
    if (!pdev) {
378
0
        return 0;
379
0
    }
380
381
0
    r = pci_device_route_intx_to_irq(pdev, pin);
382
0
    if (r.mode != PCI_INTX_ENABLED) {
383
0
        return 0;
384
0
    }
385
386
    /*
387
     * Hm, can we be notified of INTX routing changes? Not without
388
     * *owning* the device and being allowed to overwrite its own
389
     * ->intx_routing_notifier, AFAICT. So let's not.
390
     */
391
0
    return r.irq;
392
0
}
393
394
void xen_evtchn_set_callback_level(int level)
395
0
{
396
0
    XenEvtchnState *s = xen_evtchn_singleton;
397
0
    if (!s) {
398
0
        return;
399
0
    }
400
401
    /*
402
     * We get to this function in a number of ways:
403
     *
404
     *  • From I/O context, via PV backend drivers sending a notification to
405
     *    the guest.
406
     *
407
     *  • From guest vCPU context, via loopback interdomain event channels
408
     *    (or theoretically even IPIs but guests don't use those with GSI
409
     *    delivery because that's pointless. We don't want a malicious guest
410
     *    to be able to trigger a deadlock though, so we can't rule it out.)
411
     *
412
     *  • From guest vCPU context when the HVM_PARAM_CALLBACK_IRQ is being
413
     *    configured.
414
     *
415
     *  • From guest vCPU context in the KVM exit handler, if the upcall
416
     *    pending flag has been cleared and the GSI needs to be deasserted.
417
     *
418
     *  • Maybe in future, in an interrupt ack/eoi notifier when the GSI has
419
     *    been acked in the irqchip.
420
     *
421
     * Whichever context we come from if we aren't already holding the BQL
422
     * then e can't take it now, as we may already hold s->port_lock. So
423
     * trigger the BH to set the IRQ for us instead of doing it immediately.
424
     *
425
     * In the HVM_PARAM_CALLBACK_IRQ and KVM exit handler cases, the caller
426
     * will deliberately take the BQL because they want the change to take
427
     * effect immediately. That just leaves interdomain loopback as the case
428
     * which uses the BH.
429
     */
430
0
    if (!bql_locked()) {
431
0
        qemu_bh_schedule(s->gsi_bh);
432
0
        return;
433
0
    }
434
435
0
    if (s->callback_gsi && s->callback_gsi < s->nr_callback_gsis) {
436
        /*
437
         * Ugly, but since we hold the BQL we can set this flag so that
438
         * xen_evtchn_set_gsi() can tell the difference between this code
439
         * setting the GSI, and an external device (PCI INTx) doing so.
440
         */
441
0
        s->setting_callback_gsi = true;
442
        /* Do not deassert the line if an external device is asserting it. */
443
0
        qemu_set_irq(s->callback_gsis[s->callback_gsi],
444
0
                     level || s->extern_gsi_level);
445
0
        s->setting_callback_gsi = false;
446
447
        /*
448
         * If the callback GSI is the only one asserted, ensure the status
449
         * is polled for deassertion in kvm_arch_post_run().
450
         */
451
0
        if (level && !s->extern_gsi_level) {
452
0
            kvm_xen_set_callback_asserted();
453
0
        }
454
0
    }
455
0
}
456
457
int xen_evtchn_set_callback_param(uint64_t param)
458
0
{
459
0
    XenEvtchnState *s = xen_evtchn_singleton;
460
0
    struct kvm_xen_hvm_attr xa = {
461
0
        .type = KVM_XEN_ATTR_TYPE_UPCALL_VECTOR,
462
0
        .u.vector = 0,
463
0
    };
464
0
    bool in_kernel = false;
465
0
    uint32_t gsi = 0;
466
0
    int type = param >> CALLBACK_VIA_TYPE_SHIFT;
467
0
    int ret;
468
469
0
    if (!s) {
470
0
        return -ENOTSUP;
471
0
    }
472
473
    /*
474
     * We need the BQL because set_callback_pci_intx() may call into PCI code,
475
     * and because we may need to manipulate the old and new GSI levels.
476
     */
477
0
    assert(bql_locked());
478
0
    qemu_mutex_lock(&s->port_lock);
479
480
0
    switch (type) {
481
0
    case HVM_PARAM_CALLBACK_TYPE_VECTOR: {
482
0
        xa.u.vector = (uint8_t)param,
483
484
0
        ret = kvm_vm_ioctl(kvm_state, KVM_XEN_HVM_SET_ATTR, &xa);
485
0
        if (!ret && kvm_xen_has_cap(EVTCHN_SEND)) {
486
0
            in_kernel = true;
487
0
        }
488
0
        gsi = 0;
489
0
        break;
490
0
    }
491
492
0
    case HVM_PARAM_CALLBACK_TYPE_PCI_INTX:
493
0
        gsi = set_callback_pci_intx(s, param);
494
0
        ret = gsi ? 0 : -EINVAL;
495
0
        break;
496
497
0
    case HVM_PARAM_CALLBACK_TYPE_GSI:
498
0
        gsi = (uint32_t)param;
499
0
        ret = 0;
500
0
        break;
501
502
0
    default:
503
        /* Xen doesn't return error even if you set something bogus */
504
0
        ret = 0;
505
0
        break;
506
0
    }
507
508
    /* If the guest has set a per-vCPU callback vector, prefer that. */
509
0
    if (gsi && kvm_xen_has_vcpu_callback_vector()) {
510
0
        in_kernel = kvm_xen_has_cap(EVTCHN_SEND);
511
0
        gsi = 0;
512
0
    }
513
514
0
    if (!ret) {
515
        /* If vector delivery was turned *off* then tell the kernel */
516
0
        if ((s->callback_param >> CALLBACK_VIA_TYPE_SHIFT) ==
517
0
            HVM_PARAM_CALLBACK_TYPE_VECTOR && !xa.u.vector) {
518
0
            kvm_vm_ioctl(kvm_state, KVM_XEN_HVM_SET_ATTR, &xa);
519
0
        }
520
0
        s->callback_param = param;
521
0
        s->evtchn_in_kernel = in_kernel;
522
523
0
        if (gsi != s->callback_gsi) {
524
0
            struct vcpu_info *vi = kvm_xen_get_vcpu_info_hva(0);
525
526
0
            xen_evtchn_set_callback_level(0);
527
0
            s->callback_gsi = gsi;
528
529
0
            if (gsi && vi && vi->evtchn_upcall_pending) {
530
0
                kvm_xen_inject_vcpu_callback_vector(0, type);
531
0
            }
532
0
        }
533
0
    }
534
535
0
    qemu_mutex_unlock(&s->port_lock);
536
537
0
    return ret;
538
0
}
539
540
static void inject_callback(XenEvtchnState *s, uint32_t vcpu)
541
0
{
542
0
    int type = s->callback_param >> CALLBACK_VIA_TYPE_SHIFT;
543
544
0
    kvm_xen_inject_vcpu_callback_vector(vcpu, type);
545
0
}
546
547
static void deassign_kernel_port(evtchn_port_t port)
548
0
{
549
0
    struct kvm_xen_hvm_attr ha;
550
0
    int ret;
551
552
0
    ha.type = KVM_XEN_ATTR_TYPE_EVTCHN;
553
0
    ha.u.evtchn.send_port = port;
554
0
    ha.u.evtchn.flags = KVM_XEN_EVTCHN_DEASSIGN;
555
556
0
    ret = kvm_vm_ioctl(kvm_state, KVM_XEN_HVM_SET_ATTR, &ha);
557
0
    if (ret) {
558
0
        qemu_log_mask(LOG_GUEST_ERROR, "Failed to unbind kernel port %d: %s\n",
559
0
                      port, strerror(ret));
560
0
    }
561
0
}
562
563
static int assign_kernel_port(uint16_t type, evtchn_port_t port,
564
                              uint32_t vcpu_id)
565
0
{
566
0
    CPUState *cpu = qemu_get_cpu(vcpu_id);
567
0
    struct kvm_xen_hvm_attr ha;
568
569
0
    if (!cpu) {
570
0
        return -ENOENT;
571
0
    }
572
573
0
    ha.type = KVM_XEN_ATTR_TYPE_EVTCHN;
574
0
    ha.u.evtchn.send_port = port;
575
0
    ha.u.evtchn.type = type;
576
0
    ha.u.evtchn.flags = 0;
577
0
    ha.u.evtchn.deliver.port.port = port;
578
0
    ha.u.evtchn.deliver.port.vcpu = kvm_arch_vcpu_id(cpu);
579
0
    ha.u.evtchn.deliver.port.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL;
580
581
0
    return kvm_vm_ioctl(kvm_state, KVM_XEN_HVM_SET_ATTR, &ha);
582
0
}
583
584
static int assign_kernel_eventfd(uint16_t type, evtchn_port_t port, int fd)
585
0
{
586
0
    struct kvm_xen_hvm_attr ha;
587
588
0
    ha.type = KVM_XEN_ATTR_TYPE_EVTCHN;
589
0
    ha.u.evtchn.send_port = port;
590
0
    ha.u.evtchn.type = type;
591
0
    ha.u.evtchn.flags = 0;
592
0
    ha.u.evtchn.deliver.eventfd.port = 0;
593
0
    ha.u.evtchn.deliver.eventfd.fd = fd;
594
595
0
    return kvm_vm_ioctl(kvm_state, KVM_XEN_HVM_SET_ATTR, &ha);
596
0
}
597
598
static bool valid_port(evtchn_port_t port)
599
0
{
600
0
    if (!port) {
601
0
        return false;
602
0
    }
603
604
0
    if (xen_is_long_mode()) {
605
0
        return port < EVTCHN_2L_NR_CHANNELS;
606
0
    } else {
607
0
        return port < COMPAT_EVTCHN_2L_NR_CHANNELS;
608
0
    }
609
0
}
610
611
static bool valid_vcpu(uint32_t vcpu)
612
0
{
613
0
    return !!qemu_get_cpu(vcpu);
614
0
}
615
616
static void unbind_backend_ports(XenEvtchnState *s)
617
0
{
618
0
    XenEvtchnPort *p;
619
0
    int i;
620
621
0
    for (i = 1; i < s->nr_ports; i++) {
622
0
        p = &s->port_table[i];
623
0
        if (p->type == EVTCHNSTAT_interdomain && p->u.interdomain.to_qemu) {
624
0
            evtchn_port_t be_port = p->u.interdomain.port;
625
626
0
            if (s->be_handles[be_port]) {
627
                /* This part will be overwritten on the load anyway. */
628
0
                p->type = EVTCHNSTAT_unbound;
629
0
                p->u.interdomain.port = 0;
630
631
                /* Leave the backend port open and unbound too. */
632
0
                if (kvm_xen_has_cap(EVTCHN_SEND)) {
633
0
                    deassign_kernel_port(i);
634
0
                }
635
0
                s->be_handles[be_port]->guest_port = 0;
636
0
            }
637
0
        }
638
0
    }
639
0
}
640
641
int xen_evtchn_status_op(struct evtchn_status *status)
642
0
{
643
0
    XenEvtchnState *s = xen_evtchn_singleton;
644
0
    XenEvtchnPort *p;
645
646
0
    if (!s) {
647
0
        return -ENOTSUP;
648
0
    }
649
650
0
    if (status->dom != DOMID_SELF && status->dom != xen_domid) {
651
0
        return -ESRCH;
652
0
    }
653
654
0
    if (!valid_port(status->port)) {
655
0
        return -EINVAL;
656
0
    }
657
658
0
    qemu_mutex_lock(&s->port_lock);
659
660
0
    p = &s->port_table[status->port];
661
662
0
    status->status = p->type;
663
0
    status->vcpu = p->vcpu;
664
665
0
    switch (p->type) {
666
0
    case EVTCHNSTAT_unbound:
667
0
        status->u.unbound.dom = p->u.interdomain.to_qemu ? DOMID_QEMU
668
0
                                                         : xen_domid;
669
0
        break;
670
671
0
    case EVTCHNSTAT_interdomain:
672
0
        status->u.interdomain.dom = p->u.interdomain.to_qemu ? DOMID_QEMU
673
0
                                                             : xen_domid;
674
0
        status->u.interdomain.port = p->u.interdomain.port;
675
0
        break;
676
677
0
    case EVTCHNSTAT_pirq:
678
0
        status->u.pirq = p->u.pirq;
679
0
        break;
680
681
0
    case EVTCHNSTAT_virq:
682
0
        status->u.virq = p->u.virq;
683
0
        break;
684
0
    }
685
686
0
    qemu_mutex_unlock(&s->port_lock);
687
0
    return 0;
688
0
}
689
690
/*
691
 * Never thought I'd hear myself say this, but C++ templates would be
692
 * kind of nice here.
693
 *
694
 * template<class T> static int do_unmask_port(T *shinfo, ...);
695
 */
696
static int do_unmask_port_lm(XenEvtchnState *s, evtchn_port_t port,
697
                             bool do_unmask, struct shared_info *shinfo,
698
                             struct vcpu_info *vcpu_info)
699
0
{
700
0
    const int bits_per_word = BITS_PER_BYTE * sizeof(shinfo->evtchn_pending[0]);
701
0
    typeof(shinfo->evtchn_pending[0]) mask;
702
0
    int idx = port / bits_per_word;
703
0
    int offset = port % bits_per_word;
704
705
0
    mask = 1UL << offset;
706
707
0
    if (idx >= bits_per_word) {
708
0
        return -EINVAL;
709
0
    }
710
711
0
    if (do_unmask) {
712
        /*
713
         * If this is a true unmask operation, clear the mask bit. If
714
         * it was already unmasked, we have nothing further to do.
715
         */
716
0
        if (!((qatomic_fetch_and(&shinfo->evtchn_mask[idx], ~mask) & mask))) {
717
0
            return 0;
718
0
        }
719
0
    } else {
720
        /*
721
         * This is a pseudo-unmask for affinity changes. We don't
722
         * change the mask bit, and if it's *masked* we have nothing
723
         * else to do.
724
         */
725
0
        if (qatomic_fetch_or(&shinfo->evtchn_mask[idx], 0) & mask) {
726
0
            return 0;
727
0
        }
728
0
    }
729
730
    /* If the event was not pending, we're done. */
731
0
    if (!(qatomic_fetch_or(&shinfo->evtchn_pending[idx], 0) & mask)) {
732
0
        return 0;
733
0
    }
734
735
    /* Now on to the vcpu_info evtchn_pending_sel index... */
736
0
    mask = 1UL << idx;
737
738
    /* If a port in this word was already pending for this vCPU, all done. */
739
0
    if (qatomic_fetch_or(&vcpu_info->evtchn_pending_sel, mask) & mask) {
740
0
        return 0;
741
0
    }
742
743
    /* Set evtchn_upcall_pending for this vCPU */
744
0
    if (qatomic_fetch_or(&vcpu_info->evtchn_upcall_pending, 1)) {
745
0
        return 0;
746
0
    }
747
748
0
    inject_callback(s, s->port_table[port].vcpu);
749
750
0
    return 0;
751
0
}
752
753
static int do_unmask_port_compat(XenEvtchnState *s, evtchn_port_t port,
754
                                 bool do_unmask,
755
                                 struct compat_shared_info *shinfo,
756
                                 struct compat_vcpu_info *vcpu_info)
757
0
{
758
0
    const int bits_per_word = BITS_PER_BYTE * sizeof(shinfo->evtchn_pending[0]);
759
0
    typeof(shinfo->evtchn_pending[0]) mask;
760
0
    int idx = port / bits_per_word;
761
0
    int offset = port % bits_per_word;
762
763
0
    mask = 1UL << offset;
764
765
0
    if (idx >= bits_per_word) {
766
0
        return -EINVAL;
767
0
    }
768
769
0
    if (do_unmask) {
770
        /*
771
         * If this is a true unmask operation, clear the mask bit. If
772
         * it was already unmasked, we have nothing further to do.
773
         */
774
0
        if (!((qatomic_fetch_and(&shinfo->evtchn_mask[idx], ~mask) & mask))) {
775
0
            return 0;
776
0
        }
777
0
    } else {
778
        /*
779
         * This is a pseudo-unmask for affinity changes. We don't
780
         * change the mask bit, and if it's *masked* we have nothing
781
         * else to do.
782
         */
783
0
        if (qatomic_fetch_or(&shinfo->evtchn_mask[idx], 0) & mask) {
784
0
            return 0;
785
0
        }
786
0
    }
787
788
    /* If the event was not pending, we're done. */
789
0
    if (!(qatomic_fetch_or(&shinfo->evtchn_pending[idx], 0) & mask)) {
790
0
        return 0;
791
0
    }
792
793
    /* Now on to the vcpu_info evtchn_pending_sel index... */
794
0
    mask = 1UL << idx;
795
796
    /* If a port in this word was already pending for this vCPU, all done. */
797
0
    if (qatomic_fetch_or(&vcpu_info->evtchn_pending_sel, mask) & mask) {
798
0
        return 0;
799
0
    }
800
801
    /* Set evtchn_upcall_pending for this vCPU */
802
0
    if (qatomic_fetch_or(&vcpu_info->evtchn_upcall_pending, 1)) {
803
0
        return 0;
804
0
    }
805
806
0
    inject_callback(s, s->port_table[port].vcpu);
807
808
0
    return 0;
809
0
}
810
811
static int unmask_port(XenEvtchnState *s, evtchn_port_t port, bool do_unmask)
812
0
{
813
0
    void *vcpu_info, *shinfo;
814
815
0
    if (s->port_table[port].type == EVTCHNSTAT_closed) {
816
0
        return -EINVAL;
817
0
    }
818
819
0
    shinfo = xen_overlay_get_shinfo_ptr();
820
0
    if (!shinfo) {
821
0
        return -ENOTSUP;
822
0
    }
823
824
0
    vcpu_info = kvm_xen_get_vcpu_info_hva(s->port_table[port].vcpu);
825
0
    if (!vcpu_info) {
826
0
        return -EINVAL;
827
0
    }
828
829
0
    if (xen_is_long_mode()) {
830
0
        return do_unmask_port_lm(s, port, do_unmask, shinfo, vcpu_info);
831
0
    } else {
832
0
        return do_unmask_port_compat(s, port, do_unmask, shinfo, vcpu_info);
833
0
    }
834
0
}
835
836
static int do_set_port_lm(XenEvtchnState *s, evtchn_port_t port,
837
                          struct shared_info *shinfo,
838
                          struct vcpu_info *vcpu_info)
839
0
{
840
0
    const int bits_per_word = BITS_PER_BYTE * sizeof(shinfo->evtchn_pending[0]);
841
0
    typeof(shinfo->evtchn_pending[0]) mask;
842
0
    int idx = port / bits_per_word;
843
0
    int offset = port % bits_per_word;
844
845
0
    mask = 1UL << offset;
846
847
0
    if (idx >= bits_per_word) {
848
0
        return -EINVAL;
849
0
    }
850
851
    /* Update the pending bit itself. If it was already set, we're done. */
852
0
    if (qatomic_fetch_or(&shinfo->evtchn_pending[idx], mask) & mask) {
853
0
        return 0;
854
0
    }
855
856
    /* Check if it's masked. */
857
0
    if (qatomic_fetch_or(&shinfo->evtchn_mask[idx], 0) & mask) {
858
0
        return 0;
859
0
    }
860
861
    /* Now on to the vcpu_info evtchn_pending_sel index... */
862
0
    mask = 1UL << idx;
863
864
    /* If a port in this word was already pending for this vCPU, all done. */
865
0
    if (qatomic_fetch_or(&vcpu_info->evtchn_pending_sel, mask) & mask) {
866
0
        return 0;
867
0
    }
868
869
    /* Set evtchn_upcall_pending for this vCPU */
870
0
    if (qatomic_fetch_or(&vcpu_info->evtchn_upcall_pending, 1)) {
871
0
        return 0;
872
0
    }
873
874
0
    inject_callback(s, s->port_table[port].vcpu);
875
876
0
    return 0;
877
0
}
878
879
static int do_set_port_compat(XenEvtchnState *s, evtchn_port_t port,
880
                              struct compat_shared_info *shinfo,
881
                              struct compat_vcpu_info *vcpu_info)
882
0
{
883
0
    const int bits_per_word = BITS_PER_BYTE * sizeof(shinfo->evtchn_pending[0]);
884
0
    typeof(shinfo->evtchn_pending[0]) mask;
885
0
    int idx = port / bits_per_word;
886
0
    int offset = port % bits_per_word;
887
888
0
    mask = 1UL << offset;
889
890
0
    if (idx >= bits_per_word) {
891
0
        return -EINVAL;
892
0
    }
893
894
    /* Update the pending bit itself. If it was already set, we're done. */
895
0
    if (qatomic_fetch_or(&shinfo->evtchn_pending[idx], mask) & mask) {
896
0
        return 0;
897
0
    }
898
899
    /* Check if it's masked. */
900
0
    if (qatomic_fetch_or(&shinfo->evtchn_mask[idx], 0) & mask) {
901
0
        return 0;
902
0
    }
903
904
    /* Now on to the vcpu_info evtchn_pending_sel index... */
905
0
    mask = 1UL << idx;
906
907
    /* If a port in this word was already pending for this vCPU, all done. */
908
0
    if (qatomic_fetch_or(&vcpu_info->evtchn_pending_sel, mask) & mask) {
909
0
        return 0;
910
0
    }
911
912
    /* Set evtchn_upcall_pending for this vCPU */
913
0
    if (qatomic_fetch_or(&vcpu_info->evtchn_upcall_pending, 1)) {
914
0
        return 0;
915
0
    }
916
917
0
    inject_callback(s, s->port_table[port].vcpu);
918
919
0
    return 0;
920
0
}
921
922
static int set_port_pending(XenEvtchnState *s, evtchn_port_t port)
923
0
{
924
0
    void *vcpu_info, *shinfo;
925
926
0
    if (s->port_table[port].type == EVTCHNSTAT_closed) {
927
0
        return -EINVAL;
928
0
    }
929
930
0
    if (s->evtchn_in_kernel) {
931
0
        XenEvtchnPort *p = &s->port_table[port];
932
0
        CPUState *cpu = qemu_get_cpu(p->vcpu);
933
0
        struct kvm_irq_routing_xen_evtchn evt;
934
935
0
        if (!cpu) {
936
0
            return 0;
937
0
        }
938
939
0
        evt.port = port;
940
0
        evt.vcpu = kvm_arch_vcpu_id(cpu);
941
0
        evt.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL;
942
943
0
        return kvm_vm_ioctl(kvm_state, KVM_XEN_HVM_EVTCHN_SEND, &evt);
944
0
    }
945
946
0
    shinfo = xen_overlay_get_shinfo_ptr();
947
0
    if (!shinfo) {
948
0
        return -ENOTSUP;
949
0
    }
950
951
0
    vcpu_info = kvm_xen_get_vcpu_info_hva(s->port_table[port].vcpu);
952
0
    if (!vcpu_info) {
953
0
        return -EINVAL;
954
0
    }
955
956
0
    if (xen_is_long_mode()) {
957
0
        return do_set_port_lm(s, port, shinfo, vcpu_info);
958
0
    } else {
959
0
        return do_set_port_compat(s, port, shinfo, vcpu_info);
960
0
    }
961
0
}
962
963
static int clear_port_pending(XenEvtchnState *s, evtchn_port_t port)
964
0
{
965
0
    void *p = xen_overlay_get_shinfo_ptr();
966
967
0
    if (!p) {
968
0
        return -ENOTSUP;
969
0
    }
970
971
0
    if (xen_is_long_mode()) {
972
0
        struct shared_info *shinfo = p;
973
0
        const int bits_per_word = BITS_PER_BYTE * sizeof(shinfo->evtchn_pending[0]);
974
0
        typeof(shinfo->evtchn_pending[0]) mask;
975
0
        int idx = port / bits_per_word;
976
0
        int offset = port % bits_per_word;
977
978
0
        mask = 1UL << offset;
979
980
0
        qatomic_fetch_and(&shinfo->evtchn_pending[idx], ~mask);
981
0
    } else {
982
0
        struct compat_shared_info *shinfo = p;
983
0
        const int bits_per_word = BITS_PER_BYTE * sizeof(shinfo->evtchn_pending[0]);
984
0
        typeof(shinfo->evtchn_pending[0]) mask;
985
0
        int idx = port / bits_per_word;
986
0
        int offset = port % bits_per_word;
987
988
0
        mask = 1UL << offset;
989
990
0
        qatomic_fetch_and(&shinfo->evtchn_pending[idx], ~mask);
991
0
    }
992
0
    return 0;
993
0
}
994
995
static void free_port(XenEvtchnState *s, evtchn_port_t port)
996
0
{
997
0
    s->port_table[port].type = EVTCHNSTAT_closed;
998
0
    s->port_table[port].u.val = 0;
999
0
    s->port_table[port].vcpu = 0;
1000
1001
0
    if (s->nr_ports == port + 1) {
1002
0
        do {
1003
0
            s->nr_ports--;
1004
0
        } while (s->nr_ports &&
1005
0
                 s->port_table[s->nr_ports - 1].type == EVTCHNSTAT_closed);
1006
0
    }
1007
1008
    /* Clear pending event to avoid unexpected behavior on re-bind. */
1009
0
    clear_port_pending(s, port);
1010
0
}
1011
1012
static int allocate_port(XenEvtchnState *s, uint32_t vcpu, uint16_t type,
1013
                         uint16_t val, evtchn_port_t *port)
1014
0
{
1015
0
    evtchn_port_t p = 1;
1016
1017
0
    for (p = 1; valid_port(p); p++) {
1018
0
        if (s->port_table[p].type == EVTCHNSTAT_closed) {
1019
0
            s->port_table[p].vcpu = vcpu;
1020
0
            s->port_table[p].type = type;
1021
0
            s->port_table[p].u.val = val;
1022
1023
0
            *port = p;
1024
1025
0
            if (s->nr_ports < p + 1) {
1026
0
                s->nr_ports = p + 1;
1027
0
            }
1028
1029
0
            return 0;
1030
0
        }
1031
0
    }
1032
0
    return -ENOSPC;
1033
0
}
1034
1035
static bool virq_is_global(uint32_t virq)
1036
0
{
1037
0
    switch (virq) {
1038
0
    case VIRQ_TIMER:
1039
0
    case VIRQ_DEBUG:
1040
0
    case VIRQ_XENOPROF:
1041
0
    case VIRQ_XENPMU:
1042
0
        return false;
1043
1044
0
    default:
1045
0
        return true;
1046
0
    }
1047
0
}
1048
1049
static int close_port(XenEvtchnState *s, evtchn_port_t port,
1050
                      bool *flush_kvm_routes)
1051
0
{
1052
0
    XenEvtchnPort *p = &s->port_table[port];
1053
1054
    /* Because it *might* be a PIRQ port */
1055
0
    assert(bql_locked());
1056
1057
0
    switch (p->type) {
1058
0
    case EVTCHNSTAT_closed:
1059
0
        return -ENOENT;
1060
1061
0
    case EVTCHNSTAT_pirq:
1062
0
        s->pirq[p->u.pirq].port = 0;
1063
0
        if (s->pirq[p->u.pirq].is_translated) {
1064
0
            *flush_kvm_routes = true;
1065
0
        }
1066
0
        break;
1067
1068
0
    case EVTCHNSTAT_virq:
1069
0
        kvm_xen_set_vcpu_virq(virq_is_global(p->u.virq) ? 0 : p->vcpu,
1070
0
                              p->u.virq, 0);
1071
0
        break;
1072
1073
0
    case EVTCHNSTAT_ipi:
1074
0
        if (s->evtchn_in_kernel) {
1075
0
            deassign_kernel_port(port);
1076
0
        }
1077
0
        break;
1078
1079
0
    case EVTCHNSTAT_interdomain:
1080
0
        if (p->u.interdomain.to_qemu) {
1081
0
            uint16_t be_port = p->u.interdomain.port;
1082
0
            struct xenevtchn_handle *xc = s->be_handles[be_port];
1083
0
            if (xc) {
1084
0
                if (kvm_xen_has_cap(EVTCHN_SEND)) {
1085
0
                    deassign_kernel_port(port);
1086
0
                }
1087
0
                xc->guest_port = 0;
1088
0
            }
1089
0
        } else {
1090
            /* Loopback interdomain */
1091
0
            XenEvtchnPort *rp = &s->port_table[p->u.interdomain.port];
1092
0
            if (!valid_port(p->u.interdomain.port) ||
1093
0
                rp->u.interdomain.port != port ||
1094
0
                rp->type != EVTCHNSTAT_interdomain) {
1095
0
                error_report("Inconsistent state for interdomain unbind");
1096
0
            } else {
1097
                /* Set the other end back to unbound */
1098
0
                rp->type = EVTCHNSTAT_unbound;
1099
0
                rp->u.interdomain.port = 0;
1100
0
            }
1101
0
        }
1102
0
        break;
1103
1104
0
    default:
1105
0
        break;
1106
0
    }
1107
1108
0
    free_port(s, port);
1109
0
    return 0;
1110
0
}
1111
1112
int xen_evtchn_soft_reset(void)
1113
0
{
1114
0
    XenEvtchnState *s = xen_evtchn_singleton;
1115
0
    bool flush_kvm_routes = false;
1116
0
    int i;
1117
1118
0
    if (!s) {
1119
0
        return -ENOTSUP;
1120
0
    }
1121
1122
0
    assert(bql_locked());
1123
1124
0
    qemu_mutex_lock(&s->port_lock);
1125
1126
0
    for (i = 0; i < s->nr_ports; i++) {
1127
0
        close_port(s, i, &flush_kvm_routes);
1128
0
    }
1129
1130
0
    qemu_mutex_unlock(&s->port_lock);
1131
1132
0
    if (flush_kvm_routes) {
1133
0
        kvm_update_msi_routes_all(NULL, true, 0, 0);
1134
0
    }
1135
1136
0
    return 0;
1137
0
}
1138
1139
int xen_evtchn_reset_op(struct evtchn_reset *reset)
1140
0
{
1141
0
    if (reset->dom != DOMID_SELF && reset->dom != xen_domid) {
1142
0
        return -ESRCH;
1143
0
    }
1144
1145
0
    BQL_LOCK_GUARD();
1146
0
    return xen_evtchn_soft_reset();
1147
0
}
1148
1149
int xen_evtchn_close_op(struct evtchn_close *close)
1150
0
{
1151
0
    XenEvtchnState *s = xen_evtchn_singleton;
1152
0
    bool flush_kvm_routes = false;
1153
0
    int ret;
1154
1155
0
    if (!s) {
1156
0
        return -ENOTSUP;
1157
0
    }
1158
1159
0
    if (!valid_port(close->port)) {
1160
0
        return -EINVAL;
1161
0
    }
1162
1163
0
    BQL_LOCK_GUARD();
1164
0
    qemu_mutex_lock(&s->port_lock);
1165
1166
0
    ret = close_port(s, close->port, &flush_kvm_routes);
1167
1168
0
    qemu_mutex_unlock(&s->port_lock);
1169
1170
0
    if (flush_kvm_routes) {
1171
0
        kvm_update_msi_routes_all(NULL, true, 0, 0);
1172
0
    }
1173
1174
0
    return ret;
1175
0
}
1176
1177
int xen_evtchn_unmask_op(struct evtchn_unmask *unmask)
1178
0
{
1179
0
    XenEvtchnState *s = xen_evtchn_singleton;
1180
0
    int ret;
1181
1182
0
    if (!s) {
1183
0
        return -ENOTSUP;
1184
0
    }
1185
1186
0
    if (!valid_port(unmask->port)) {
1187
0
        return -EINVAL;
1188
0
    }
1189
1190
0
    qemu_mutex_lock(&s->port_lock);
1191
1192
0
    ret = unmask_port(s, unmask->port, true);
1193
1194
0
    qemu_mutex_unlock(&s->port_lock);
1195
1196
0
    return ret;
1197
0
}
1198
1199
int xen_evtchn_bind_vcpu_op(struct evtchn_bind_vcpu *vcpu)
1200
0
{
1201
0
    XenEvtchnState *s = xen_evtchn_singleton;
1202
0
    XenEvtchnPort *p;
1203
0
    int ret = -EINVAL;
1204
1205
0
    if (!s) {
1206
0
        return -ENOTSUP;
1207
0
    }
1208
1209
0
    if (!valid_port(vcpu->port)) {
1210
0
        return -EINVAL;
1211
0
    }
1212
1213
0
    if (!valid_vcpu(vcpu->vcpu)) {
1214
0
        return -ENOENT;
1215
0
    }
1216
1217
0
    qemu_mutex_lock(&s->port_lock);
1218
1219
0
    p = &s->port_table[vcpu->port];
1220
1221
0
    if (p->type == EVTCHNSTAT_interdomain ||
1222
0
        p->type == EVTCHNSTAT_unbound ||
1223
0
        p->type == EVTCHNSTAT_pirq ||
1224
0
        (p->type == EVTCHNSTAT_virq && virq_is_global(p->u.virq))) {
1225
        /*
1226
         * unmask_port() with do_unmask==false will just raise the event
1227
         * on the new vCPU if the port was already pending.
1228
         */
1229
0
        p->vcpu = vcpu->vcpu;
1230
0
        unmask_port(s, vcpu->port, false);
1231
0
        ret = 0;
1232
0
    }
1233
1234
0
    qemu_mutex_unlock(&s->port_lock);
1235
1236
0
    return ret;
1237
0
}
1238
1239
int xen_evtchn_bind_virq_op(struct evtchn_bind_virq *virq)
1240
0
{
1241
0
    XenEvtchnState *s = xen_evtchn_singleton;
1242
0
    int ret;
1243
1244
0
    if (!s) {
1245
0
        return -ENOTSUP;
1246
0
    }
1247
1248
0
    if (virq->virq >= NR_VIRQS) {
1249
0
        return -EINVAL;
1250
0
    }
1251
1252
    /* Global VIRQ must be allocated on vCPU0 first */
1253
0
    if (virq_is_global(virq->virq) && virq->vcpu != 0) {
1254
0
        return -EINVAL;
1255
0
    }
1256
1257
0
    if (!valid_vcpu(virq->vcpu)) {
1258
0
        return -ENOENT;
1259
0
    }
1260
1261
0
    qemu_mutex_lock(&s->port_lock);
1262
1263
0
    ret = allocate_port(s, virq->vcpu, EVTCHNSTAT_virq, virq->virq,
1264
0
                        &virq->port);
1265
0
    if (!ret) {
1266
0
        ret = kvm_xen_set_vcpu_virq(virq->vcpu, virq->virq, virq->port);
1267
0
        if (ret) {
1268
0
            free_port(s, virq->port);
1269
0
        }
1270
0
    }
1271
1272
0
    qemu_mutex_unlock(&s->port_lock);
1273
1274
0
    return ret;
1275
0
}
1276
1277
int xen_evtchn_bind_pirq_op(struct evtchn_bind_pirq *pirq)
1278
0
{
1279
0
    XenEvtchnState *s = xen_evtchn_singleton;
1280
0
    int ret;
1281
1282
0
    if (!s) {
1283
0
        return -ENOTSUP;
1284
0
    }
1285
1286
0
    if (pirq->pirq >= s->nr_pirqs) {
1287
0
        return -EINVAL;
1288
0
    }
1289
1290
0
    BQL_LOCK_GUARD();
1291
1292
0
    if (s->pirq[pirq->pirq].port) {
1293
0
        return -EBUSY;
1294
0
    }
1295
1296
0
    qemu_mutex_lock(&s->port_lock);
1297
1298
0
    ret = allocate_port(s, 0, EVTCHNSTAT_pirq, pirq->pirq,
1299
0
                        &pirq->port);
1300
0
    if (ret) {
1301
0
        qemu_mutex_unlock(&s->port_lock);
1302
0
        return ret;
1303
0
    }
1304
1305
0
    s->pirq[pirq->pirq].port = pirq->port;
1306
0
    trace_kvm_xen_bind_pirq(pirq->pirq, pirq->port);
1307
1308
0
    qemu_mutex_unlock(&s->port_lock);
1309
1310
    /*
1311
     * Need to do the unmask outside port_lock because it may call
1312
     * back into the MSI translate function.
1313
     */
1314
0
    if (s->pirq[pirq->pirq].gsi == IRQ_MSI_EMU) {
1315
0
        if (s->pirq[pirq->pirq].is_masked) {
1316
0
            PCIDevice *dev = s->pirq[pirq->pirq].dev;
1317
0
            int vector = s->pirq[pirq->pirq].vector;
1318
0
            char *dev_path = qdev_get_dev_path(DEVICE(dev));
1319
1320
0
            trace_kvm_xen_unmask_pirq(pirq->pirq, dev_path, vector);
1321
0
            g_free(dev_path);
1322
1323
0
            if (s->pirq[pirq->pirq].is_msix) {
1324
0
                msix_set_mask(dev, vector, false);
1325
0
            } else {
1326
0
                msi_set_mask(dev, vector, false, NULL);
1327
0
            }
1328
0
        } else if (s->pirq[pirq->pirq].is_translated) {
1329
            /*
1330
             * If KVM had attempted to translate this one before, make it try
1331
             * again. If we unmasked, then the notifier on the MSI(-X) vector
1332
             * will already have had the same effect.
1333
             */
1334
0
            kvm_update_msi_routes_all(NULL, true, 0, 0);
1335
0
        }
1336
0
    }
1337
1338
0
    return ret;
1339
0
}
1340
1341
int xen_evtchn_bind_ipi_op(struct evtchn_bind_ipi *ipi)
1342
0
{
1343
0
    XenEvtchnState *s = xen_evtchn_singleton;
1344
0
    int ret;
1345
1346
0
    if (!s) {
1347
0
        return -ENOTSUP;
1348
0
    }
1349
1350
0
    if (!valid_vcpu(ipi->vcpu)) {
1351
0
        return -ENOENT;
1352
0
    }
1353
1354
0
    qemu_mutex_lock(&s->port_lock);
1355
1356
0
    ret = allocate_port(s, ipi->vcpu, EVTCHNSTAT_ipi, 0, &ipi->port);
1357
0
    if (!ret && s->evtchn_in_kernel) {
1358
0
        assign_kernel_port(EVTCHNSTAT_ipi, ipi->port, ipi->vcpu);
1359
0
    }
1360
1361
0
    qemu_mutex_unlock(&s->port_lock);
1362
1363
0
    return ret;
1364
0
}
1365
1366
int xen_evtchn_bind_interdomain_op(struct evtchn_bind_interdomain *interdomain)
1367
0
{
1368
0
    XenEvtchnState *s = xen_evtchn_singleton;
1369
0
    int ret;
1370
1371
0
    if (!s) {
1372
0
        return -ENOTSUP;
1373
0
    }
1374
1375
0
    if (interdomain->remote_dom != DOMID_QEMU &&
1376
0
        interdomain->remote_dom != DOMID_SELF &&
1377
0
        interdomain->remote_dom != xen_domid) {
1378
0
        return -ESRCH;
1379
0
    }
1380
1381
0
    if (!valid_port(interdomain->remote_port)) {
1382
0
        return -EINVAL;
1383
0
    }
1384
1385
0
    qemu_mutex_lock(&s->port_lock);
1386
1387
    /* The newly allocated port starts out as unbound */
1388
0
    ret = allocate_port(s, 0, EVTCHNSTAT_unbound, 0, &interdomain->local_port);
1389
1390
0
    if (ret) {
1391
0
        goto out;
1392
0
    }
1393
1394
0
    if (interdomain->remote_dom == DOMID_QEMU) {
1395
0
        struct xenevtchn_handle *xc = s->be_handles[interdomain->remote_port];
1396
0
        XenEvtchnPort *lp = &s->port_table[interdomain->local_port];
1397
1398
0
        if (!xc) {
1399
0
            ret = -ENOENT;
1400
0
            goto out_free_port;
1401
0
        }
1402
1403
0
        if (xc->guest_port) {
1404
0
            ret = -EBUSY;
1405
0
            goto out_free_port;
1406
0
        }
1407
1408
0
        assert(xc->be_port == interdomain->remote_port);
1409
0
        xc->guest_port = interdomain->local_port;
1410
0
        if (kvm_xen_has_cap(EVTCHN_SEND)) {
1411
0
            assign_kernel_eventfd(lp->type, xc->guest_port, xc->fd);
1412
0
        }
1413
0
        lp->type = EVTCHNSTAT_interdomain;
1414
0
        lp->u.interdomain.to_qemu = 1;
1415
0
        lp->u.interdomain.port = interdomain->remote_port;
1416
0
        ret = 0;
1417
0
    } else {
1418
        /* Loopback */
1419
0
        XenEvtchnPort *rp = &s->port_table[interdomain->remote_port];
1420
0
        XenEvtchnPort *lp = &s->port_table[interdomain->local_port];
1421
1422
        /*
1423
         * The 'remote' port for loopback must be an unbound port allocated
1424
         * for communication with the local domain, and must *not* be the
1425
         * port that was just allocated for the local end.
1426
         */
1427
0
        if (interdomain->local_port != interdomain->remote_port &&
1428
0
            rp->type == EVTCHNSTAT_unbound && !rp->u.interdomain.to_qemu) {
1429
1430
0
            rp->type = EVTCHNSTAT_interdomain;
1431
0
            rp->u.interdomain.port = interdomain->local_port;
1432
1433
0
            lp->type = EVTCHNSTAT_interdomain;
1434
0
            lp->u.interdomain.port = interdomain->remote_port;
1435
0
        } else {
1436
0
            ret = -EINVAL;
1437
0
        }
1438
0
    }
1439
1440
0
 out_free_port:
1441
0
    if (ret) {
1442
0
        free_port(s, interdomain->local_port);
1443
0
    }
1444
0
 out:
1445
0
    qemu_mutex_unlock(&s->port_lock);
1446
1447
0
    return ret;
1448
1449
0
}
1450
int xen_evtchn_alloc_unbound_op(struct evtchn_alloc_unbound *alloc)
1451
0
{
1452
0
    XenEvtchnState *s = xen_evtchn_singleton;
1453
0
    int ret;
1454
1455
0
    if (!s) {
1456
0
        return -ENOTSUP;
1457
0
    }
1458
1459
0
    if (alloc->dom != DOMID_SELF && alloc->dom != xen_domid) {
1460
0
        return -ESRCH;
1461
0
    }
1462
1463
0
    if (alloc->remote_dom != DOMID_QEMU &&
1464
0
        alloc->remote_dom != DOMID_SELF &&
1465
0
        alloc->remote_dom != xen_domid) {
1466
0
        return -EPERM;
1467
0
    }
1468
1469
0
    qemu_mutex_lock(&s->port_lock);
1470
1471
0
    ret = allocate_port(s, 0, EVTCHNSTAT_unbound, 0, &alloc->port);
1472
1473
0
    if (!ret && alloc->remote_dom == DOMID_QEMU) {
1474
0
        XenEvtchnPort *p = &s->port_table[alloc->port];
1475
0
        p->u.interdomain.to_qemu = 1;
1476
0
    }
1477
1478
0
    qemu_mutex_unlock(&s->port_lock);
1479
1480
0
    return ret;
1481
0
}
1482
1483
int xen_evtchn_send_op(struct evtchn_send *send)
1484
0
{
1485
0
    XenEvtchnState *s = xen_evtchn_singleton;
1486
0
    XenEvtchnPort *p;
1487
0
    int ret = 0;
1488
1489
0
    if (!s) {
1490
0
        return -ENOTSUP;
1491
0
    }
1492
1493
0
    if (!valid_port(send->port)) {
1494
0
        return -EINVAL;
1495
0
    }
1496
1497
0
    qemu_mutex_lock(&s->port_lock);
1498
1499
0
    p = &s->port_table[send->port];
1500
1501
0
    switch (p->type) {
1502
0
    case EVTCHNSTAT_interdomain:
1503
0
        if (p->u.interdomain.to_qemu) {
1504
            /*
1505
             * This is an event from the guest to qemu itself, which is
1506
             * serving as the driver domain.
1507
             */
1508
0
            uint16_t be_port = p->u.interdomain.port;
1509
0
            struct xenevtchn_handle *xc = s->be_handles[be_port];
1510
0
            if (xc) {
1511
0
                eventfd_write(xc->fd, 1);
1512
0
                ret = 0;
1513
0
            } else {
1514
0
                ret = -ENOENT;
1515
0
            }
1516
0
        } else {
1517
            /* Loopback interdomain ports; just a complex IPI */
1518
0
            set_port_pending(s, p->u.interdomain.port);
1519
0
        }
1520
0
        break;
1521
1522
0
    case EVTCHNSTAT_ipi:
1523
0
        set_port_pending(s, send->port);
1524
0
        break;
1525
1526
0
    case EVTCHNSTAT_unbound:
1527
        /* Xen will silently drop these */
1528
0
        break;
1529
1530
0
    default:
1531
0
        ret = -EINVAL;
1532
0
        break;
1533
0
    }
1534
1535
0
    qemu_mutex_unlock(&s->port_lock);
1536
1537
0
    return ret;
1538
0
}
1539
1540
int xen_evtchn_set_port(uint16_t port)
1541
0
{
1542
0
    XenEvtchnState *s = xen_evtchn_singleton;
1543
0
    XenEvtchnPort *p;
1544
0
    int ret = -EINVAL;
1545
1546
0
    if (!s) {
1547
0
        return -ENOTSUP;
1548
0
    }
1549
1550
0
    if (!valid_port(port)) {
1551
0
        return -EINVAL;
1552
0
    }
1553
1554
0
    qemu_mutex_lock(&s->port_lock);
1555
1556
0
    p = &s->port_table[port];
1557
1558
    /* QEMU has no business sending to anything but these */
1559
0
    if (p->type == EVTCHNSTAT_virq ||
1560
0
        (p->type == EVTCHNSTAT_interdomain && p->u.interdomain.to_qemu)) {
1561
0
        set_port_pending(s, port);
1562
0
        ret = 0;
1563
0
    }
1564
1565
0
    qemu_mutex_unlock(&s->port_lock);
1566
1567
0
    return ret;
1568
0
}
1569
1570
static int allocate_pirq(XenEvtchnState *s, int type, int gsi)
1571
0
{
1572
0
    uint16_t pirq;
1573
1574
    /*
1575
     * Preserve the allocation strategy that Xen has. It looks like
1576
     * we *never* give out PIRQ 0-15, we give out 16-nr_irqs_gsi only
1577
     * to GSIs (counting up from 16), and then we count backwards from
1578
     * the top for MSIs or when the GSI space is exhausted.
1579
     */
1580
0
    if (type == MAP_PIRQ_TYPE_GSI) {
1581
0
        for (pirq = 16 ; pirq < IOAPIC_NUM_PINS; pirq++) {
1582
0
            if (pirq_inuse(s, pirq)) {
1583
0
                continue;
1584
0
            }
1585
1586
            /* Found it */
1587
0
            goto found;
1588
0
        }
1589
0
    }
1590
0
    for (pirq = s->nr_pirqs - 1; pirq >= IOAPIC_NUM_PINS; pirq--) {
1591
        /* Skip whole words at a time when they're full */
1592
0
        if (pirq_inuse_word(s, pirq) == UINT64_MAX) {
1593
0
            pirq &= ~63ULL;
1594
0
            continue;
1595
0
        }
1596
0
        if (pirq_inuse(s, pirq)) {
1597
0
            continue;
1598
0
        }
1599
1600
0
        goto found;
1601
0
    }
1602
0
    return -ENOSPC;
1603
1604
0
 found:
1605
0
    pirq_inuse_word(s, pirq) |= pirq_inuse_bit(pirq);
1606
0
    if (gsi >= 0) {
1607
0
        assert(gsi < IOAPIC_NUM_PINS);
1608
0
        s->gsi_pirq[gsi] = pirq;
1609
0
    }
1610
0
    s->pirq[pirq].gsi = gsi;
1611
0
    return pirq;
1612
0
}
1613
1614
bool xen_evtchn_set_gsi(int gsi, int *level)
1615
0
{
1616
0
    XenEvtchnState *s = xen_evtchn_singleton;
1617
0
    int pirq;
1618
1619
0
    assert(bql_locked());
1620
1621
0
    if (!s || gsi < 0 || gsi >= IOAPIC_NUM_PINS) {
1622
0
        return false;
1623
0
    }
1624
1625
    /*
1626
     * For the callback_gsi we need to implement a logical OR of the event
1627
     * channel GSI and the external input (e.g. from PCI INTx), because
1628
     * QEMU itself doesn't support shared level interrupts via demux or
1629
     * resamplers.
1630
     */
1631
0
    if (gsi && gsi == s->callback_gsi) {
1632
        /* Remember the external state of the GSI pin (e.g. from PCI INTx) */
1633
0
        if (!s->setting_callback_gsi) {
1634
0
            s->extern_gsi_level = *level;
1635
1636
            /*
1637
             * Don't allow the external device to deassert the line if the
1638
             * eveht channel GSI should still be asserted.
1639
             */
1640
0
            if (!s->extern_gsi_level) {
1641
0
                struct vcpu_info *vi = kvm_xen_get_vcpu_info_hva(0);
1642
0
                if (vi && vi->evtchn_upcall_pending) {
1643
                    /* Need to poll for deassertion */
1644
0
                    kvm_xen_set_callback_asserted();
1645
0
                    *level = 1;
1646
0
                }
1647
0
            }
1648
0
        }
1649
1650
        /*
1651
         * The event channel GSI cannot be routed to PIRQ, as that would make
1652
         * no sense. It could also deadlock on s->port_lock, if we proceed.
1653
         * So bail out now.
1654
         */
1655
0
        return false;
1656
0
    }
1657
1658
0
    QEMU_LOCK_GUARD(&s->port_lock);
1659
1660
0
    pirq = s->gsi_pirq[gsi];
1661
0
    if (!pirq) {
1662
0
        return false;
1663
0
    }
1664
1665
0
    if (*level) {
1666
0
        int port = s->pirq[pirq].port;
1667
1668
0
        s->pirq_gsi_set |= (1U << gsi);
1669
0
        if (port) {
1670
0
            set_port_pending(s, port);
1671
0
        }
1672
0
    } else {
1673
0
        s->pirq_gsi_set &= ~(1U << gsi);
1674
0
    }
1675
0
    return true;
1676
0
}
1677
1678
static uint32_t msi_pirq_target(uint64_t addr, uint32_t data)
1679
0
{
1680
    /* The vector (in low 8 bits of data) must be zero */
1681
0
    if (data & 0xff) {
1682
0
        return 0;
1683
0
    }
1684
1685
0
    uint32_t pirq = (addr & 0xff000) >> 12;
1686
0
    pirq |= (addr >> 32) & 0xffffff00;
1687
1688
0
    return pirq;
1689
0
}
1690
1691
static void do_remove_pci_vector(XenEvtchnState *s, PCIDevice *dev, int vector,
1692
                                 int except_pirq)
1693
0
{
1694
0
    uint32_t pirq;
1695
1696
0
    for (pirq = 0; pirq < s->nr_pirqs; pirq++) {
1697
        /*
1698
         * We could be cleverer here, but it isn't really a fast path, and
1699
         * this trivial optimisation is enough to let us skip the big gap
1700
         * in the middle a bit quicker (in terms of both loop iterations,
1701
         * and cache lines).
1702
         */
1703
0
        if (!(pirq & 63) && !(pirq_inuse_word(s, pirq))) {
1704
0
            pirq += 64;
1705
0
            continue;
1706
0
        }
1707
0
        if (except_pirq && pirq == except_pirq) {
1708
0
            continue;
1709
0
        }
1710
0
        if (s->pirq[pirq].dev != dev) {
1711
0
            continue;
1712
0
        }
1713
0
        if (vector != -1 && s->pirq[pirq].vector != vector) {
1714
0
            continue;
1715
0
        }
1716
1717
        /* It could theoretically be bound to a port already, but that is OK. */
1718
0
        s->pirq[pirq].dev = dev;
1719
0
        s->pirq[pirq].gsi = IRQ_UNBOUND;
1720
0
        s->pirq[pirq].is_msix = false;
1721
0
        s->pirq[pirq].vector = 0;
1722
0
        s->pirq[pirq].is_masked = false;
1723
0
        s->pirq[pirq].is_translated = false;
1724
0
    }
1725
0
}
1726
1727
void xen_evtchn_remove_pci_device(PCIDevice *dev)
1728
0
{
1729
0
    XenEvtchnState *s = xen_evtchn_singleton;
1730
1731
0
    if (!s) {
1732
0
        return;
1733
0
    }
1734
1735
0
    QEMU_LOCK_GUARD(&s->port_lock);
1736
0
    do_remove_pci_vector(s, dev, -1, 0);
1737
0
}
1738
1739
void xen_evtchn_snoop_msi(PCIDevice *dev, bool is_msix, unsigned int vector,
1740
                          uint64_t addr, uint32_t data, bool is_masked)
1741
0
{
1742
0
    XenEvtchnState *s = xen_evtchn_singleton;
1743
0
    uint32_t pirq;
1744
1745
0
    if (!s) {
1746
0
        return;
1747
0
    }
1748
1749
0
    assert(bql_locked());
1750
1751
0
    pirq = msi_pirq_target(addr, data);
1752
1753
    /*
1754
     * The PIRQ# must be sane, and there must be an allocated PIRQ in
1755
     * IRQ_UNBOUND or IRQ_MSI_EMU state to match it.
1756
     */
1757
0
    if (!pirq || pirq >= s->nr_pirqs || !pirq_inuse(s, pirq) ||
1758
0
        (s->pirq[pirq].gsi != IRQ_UNBOUND &&
1759
0
         s->pirq[pirq].gsi != IRQ_MSI_EMU)) {
1760
0
        pirq = 0;
1761
0
    }
1762
1763
0
    if (pirq) {
1764
0
        s->pirq[pirq].dev = dev;
1765
0
        s->pirq[pirq].gsi = IRQ_MSI_EMU;
1766
0
        s->pirq[pirq].is_msix = is_msix;
1767
0
        s->pirq[pirq].vector = vector;
1768
0
        s->pirq[pirq].is_masked = is_masked;
1769
0
    }
1770
1771
    /* Remove any (other) entries for this {device, vector} */
1772
0
    do_remove_pci_vector(s, dev, vector, pirq);
1773
0
}
1774
1775
int xen_evtchn_translate_pirq_msi(struct kvm_irq_routing_entry *route,
1776
                                  uint64_t address, uint32_t data)
1777
0
{
1778
0
    XenEvtchnState *s = xen_evtchn_singleton;
1779
0
    uint32_t pirq, port;
1780
0
    CPUState *cpu;
1781
1782
0
    if (!s) {
1783
0
        return 1; /* Not a PIRQ */
1784
0
    }
1785
1786
0
    assert(bql_locked());
1787
1788
0
    pirq = msi_pirq_target(address, data);
1789
0
    if (!pirq || pirq >= s->nr_pirqs) {
1790
0
        return 1; /* Not a PIRQ */
1791
0
    }
1792
1793
0
    if (!kvm_xen_has_cap(EVTCHN_2LEVEL)) {
1794
0
        return -ENOTSUP;
1795
0
    }
1796
1797
0
    if (s->pirq[pirq].gsi != IRQ_MSI_EMU) {
1798
0
        return -EINVAL;
1799
0
    }
1800
1801
    /* Remember that KVM tried to translate this. It might need to try again. */
1802
0
    s->pirq[pirq].is_translated = true;
1803
1804
0
    QEMU_LOCK_GUARD(&s->port_lock);
1805
1806
0
    port = s->pirq[pirq].port;
1807
0
    if (!valid_port(port)) {
1808
0
        return -EINVAL;
1809
0
    }
1810
1811
0
    cpu = qemu_get_cpu(s->port_table[port].vcpu);
1812
0
    if (!cpu) {
1813
0
        return -EINVAL;
1814
0
    }
1815
1816
0
    route->type = KVM_IRQ_ROUTING_XEN_EVTCHN;
1817
0
    route->u.xen_evtchn.port = port;
1818
0
    route->u.xen_evtchn.vcpu = kvm_arch_vcpu_id(cpu);
1819
0
    route->u.xen_evtchn.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL;
1820
1821
0
    return 0; /* Handled */
1822
0
}
1823
1824
bool xen_evtchn_deliver_pirq_msi(uint64_t address, uint32_t data)
1825
0
{
1826
0
    XenEvtchnState *s = xen_evtchn_singleton;
1827
0
    uint32_t pirq, port;
1828
1829
0
    if (!s) {
1830
0
        return false;
1831
0
    }
1832
1833
0
    assert(bql_locked());
1834
1835
0
    pirq = msi_pirq_target(address, data);
1836
0
    if (!pirq || pirq >= s->nr_pirqs) {
1837
0
        return false;
1838
0
    }
1839
1840
0
    QEMU_LOCK_GUARD(&s->port_lock);
1841
1842
0
    port = s->pirq[pirq].port;
1843
0
    if (!valid_port(port)) {
1844
0
        return false;
1845
0
    }
1846
1847
0
    set_port_pending(s, port);
1848
0
    return true;
1849
0
}
1850
1851
int xen_physdev_map_pirq(struct physdev_map_pirq *map)
1852
0
{
1853
0
    XenEvtchnState *s = xen_evtchn_singleton;
1854
0
    int pirq = map->pirq;
1855
0
    int gsi = map->index;
1856
1857
0
    if (!s) {
1858
0
        return -ENOTSUP;
1859
0
    }
1860
1861
0
    BQL_LOCK_GUARD();
1862
0
    QEMU_LOCK_GUARD(&s->port_lock);
1863
1864
0
    if (map->domid != DOMID_SELF && map->domid != xen_domid) {
1865
0
        return -EPERM;
1866
0
    }
1867
0
    if (map->type != MAP_PIRQ_TYPE_GSI) {
1868
0
        return -EINVAL;
1869
0
    }
1870
0
    if (gsi < 0 || gsi >= IOAPIC_NUM_PINS) {
1871
0
        return -EINVAL;
1872
0
    }
1873
1874
0
    if (pirq < 0) {
1875
0
        pirq = allocate_pirq(s, map->type, gsi);
1876
0
        if (pirq < 0) {
1877
0
            return pirq;
1878
0
        }
1879
0
        map->pirq = pirq;
1880
0
    } else if (pirq >= s->nr_pirqs) {
1881
0
        return -EINVAL;
1882
0
    } else {
1883
        /*
1884
         * User specified a valid-looking PIRQ#. Allow it if it is
1885
         * allocated and not yet bound, or if it is unallocated
1886
         */
1887
0
        if (pirq_inuse(s, pirq)) {
1888
0
            if (s->pirq[pirq].gsi != IRQ_UNBOUND) {
1889
0
                return -EBUSY;
1890
0
            }
1891
0
        } else {
1892
            /* If it was unused, mark it used now. */
1893
0
            pirq_inuse_word(s, pirq) |= pirq_inuse_bit(pirq);
1894
0
        }
1895
        /* Set the mapping in both directions. */
1896
0
        s->pirq[pirq].gsi = gsi;
1897
0
        s->gsi_pirq[gsi] = pirq;
1898
0
    }
1899
1900
0
    trace_kvm_xen_map_pirq(pirq, gsi);
1901
0
    return 0;
1902
0
}
1903
1904
int xen_physdev_unmap_pirq(struct physdev_unmap_pirq *unmap)
1905
0
{
1906
0
    XenEvtchnState *s = xen_evtchn_singleton;
1907
0
    int pirq = unmap->pirq;
1908
0
    int gsi;
1909
1910
0
    if (!s) {
1911
0
        return -ENOTSUP;
1912
0
    }
1913
1914
0
    if (unmap->domid != DOMID_SELF && unmap->domid != xen_domid) {
1915
0
        return -EPERM;
1916
0
    }
1917
0
    if (pirq < 0 || pirq >= s->nr_pirqs) {
1918
0
        return -EINVAL;
1919
0
    }
1920
1921
0
    BQL_LOCK_GUARD();
1922
0
    qemu_mutex_lock(&s->port_lock);
1923
1924
0
    if (!pirq_inuse(s, pirq)) {
1925
0
        qemu_mutex_unlock(&s->port_lock);
1926
0
        return -ENOENT;
1927
0
    }
1928
1929
0
    gsi = s->pirq[pirq].gsi;
1930
1931
    /* We can only unmap GSI PIRQs */
1932
0
    if (gsi < 0) {
1933
0
        qemu_mutex_unlock(&s->port_lock);
1934
0
        return -EINVAL;
1935
0
    }
1936
1937
0
    s->gsi_pirq[gsi] = 0;
1938
0
    s->pirq[pirq].gsi = IRQ_UNBOUND; /* Doesn't actually matter because: */
1939
0
    pirq_inuse_word(s, pirq) &= ~pirq_inuse_bit(pirq);
1940
1941
0
    trace_kvm_xen_unmap_pirq(pirq, gsi);
1942
0
    qemu_mutex_unlock(&s->port_lock);
1943
1944
0
    if (gsi == IRQ_MSI_EMU) {
1945
0
        kvm_update_msi_routes_all(NULL, true, 0, 0);
1946
0
    }
1947
1948
0
    return 0;
1949
0
}
1950
1951
int xen_physdev_eoi_pirq(struct physdev_eoi *eoi)
1952
0
{
1953
0
    XenEvtchnState *s = xen_evtchn_singleton;
1954
0
    int pirq = eoi->irq;
1955
0
    int gsi;
1956
1957
0
    if (!s) {
1958
0
        return -ENOTSUP;
1959
0
    }
1960
1961
0
    BQL_LOCK_GUARD();
1962
0
    QEMU_LOCK_GUARD(&s->port_lock);
1963
1964
0
    if (!pirq_inuse(s, pirq)) {
1965
0
        return -ENOENT;
1966
0
    }
1967
1968
0
    gsi = s->pirq[pirq].gsi;
1969
0
    if (gsi < 0) {
1970
0
        return -EINVAL;
1971
0
    }
1972
1973
    /* Reassert a level IRQ if needed */
1974
0
    if (s->pirq_gsi_set & (1U << gsi)) {
1975
0
        int port = s->pirq[pirq].port;
1976
0
        if (port) {
1977
0
            set_port_pending(s, port);
1978
0
        }
1979
0
    }
1980
1981
0
    return 0;
1982
0
}
1983
1984
int xen_physdev_query_pirq(struct physdev_irq_status_query *query)
1985
0
{
1986
0
    XenEvtchnState *s = xen_evtchn_singleton;
1987
0
    int pirq = query->irq;
1988
1989
0
    if (!s) {
1990
0
        return -ENOTSUP;
1991
0
    }
1992
1993
0
    BQL_LOCK_GUARD();
1994
0
    QEMU_LOCK_GUARD(&s->port_lock);
1995
1996
0
    if (!pirq_inuse(s, pirq)) {
1997
0
        return -ENOENT;
1998
0
    }
1999
2000
0
    if (s->pirq[pirq].gsi >= 0) {
2001
0
        query->flags = XENIRQSTAT_needs_eoi;
2002
0
    } else {
2003
0
        query->flags = 0;
2004
0
    }
2005
2006
0
    return 0;
2007
0
}
2008
2009
int xen_physdev_get_free_pirq(struct physdev_get_free_pirq *get)
2010
0
{
2011
0
    XenEvtchnState *s = xen_evtchn_singleton;
2012
0
    int pirq;
2013
2014
0
    if (!s) {
2015
0
        return -ENOTSUP;
2016
0
    }
2017
2018
0
    QEMU_LOCK_GUARD(&s->port_lock);
2019
2020
0
    pirq = allocate_pirq(s, get->type, IRQ_UNBOUND);
2021
0
    if (pirq < 0) {
2022
0
        return pirq;
2023
0
    }
2024
2025
0
    get->pirq = pirq;
2026
0
    trace_kvm_xen_get_free_pirq(pirq, get->type);
2027
0
    return 0;
2028
0
}
2029
2030
struct xenevtchn_handle *xen_be_evtchn_open(void)
2031
0
{
2032
0
    struct xenevtchn_handle *xc = g_new0(struct xenevtchn_handle, 1);
2033
2034
0
    xc->fd = eventfd(0, EFD_CLOEXEC);
2035
0
    if (xc->fd < 0) {
2036
0
        free(xc);
2037
0
        return NULL;
2038
0
    }
2039
2040
0
    return xc;
2041
0
}
2042
2043
static int find_be_port(XenEvtchnState *s, struct xenevtchn_handle *xc)
2044
0
{
2045
0
    int i;
2046
2047
0
    for (i = 1; i < EVTCHN_2L_NR_CHANNELS; i++) {
2048
0
        if (!s->be_handles[i]) {
2049
0
            s->be_handles[i] = xc;
2050
0
            xc->be_port = i;
2051
0
            return i;
2052
0
        }
2053
0
    }
2054
0
    return 0;
2055
0
}
2056
2057
int xen_be_evtchn_bind_interdomain(struct xenevtchn_handle *xc, uint32_t domid,
2058
                                   evtchn_port_t guest_port)
2059
0
{
2060
0
    XenEvtchnState *s = xen_evtchn_singleton;
2061
0
    XenEvtchnPort *gp;
2062
0
    uint16_t be_port = 0;
2063
0
    int ret;
2064
2065
0
    if (!s) {
2066
0
        return -ENOTSUP;
2067
0
    }
2068
2069
0
    if (!xc) {
2070
0
        return -EFAULT;
2071
0
    }
2072
2073
0
    if (domid != xen_domid) {
2074
0
        return -ESRCH;
2075
0
    }
2076
2077
0
    if (!valid_port(guest_port)) {
2078
0
        return -EINVAL;
2079
0
    }
2080
2081
0
    qemu_mutex_lock(&s->port_lock);
2082
2083
    /* The guest has to have an unbound port waiting for us to bind */
2084
0
    gp = &s->port_table[guest_port];
2085
2086
0
    switch (gp->type) {
2087
0
    case EVTCHNSTAT_interdomain:
2088
        /* Allow rebinding after migration, preserve port # if possible */
2089
0
        be_port = gp->u.interdomain.port;
2090
0
        assert(be_port != 0);
2091
0
        if (!s->be_handles[be_port]) {
2092
0
            s->be_handles[be_port] = xc;
2093
0
            xc->guest_port = guest_port;
2094
0
            ret = xc->be_port = be_port;
2095
0
            if (kvm_xen_has_cap(EVTCHN_SEND)) {
2096
0
                assign_kernel_eventfd(gp->type, guest_port, xc->fd);
2097
0
            }
2098
0
            break;
2099
0
        }
2100
        /* fall through */
2101
2102
0
    case EVTCHNSTAT_unbound:
2103
0
        be_port = find_be_port(s, xc);
2104
0
        if (!be_port) {
2105
0
            ret = -ENOSPC;
2106
0
            goto out;
2107
0
        }
2108
2109
0
        gp->type = EVTCHNSTAT_interdomain;
2110
0
        gp->u.interdomain.to_qemu = 1;
2111
0
        gp->u.interdomain.port = be_port;
2112
0
        xc->guest_port = guest_port;
2113
0
        if (kvm_xen_has_cap(EVTCHN_SEND)) {
2114
0
            assign_kernel_eventfd(gp->type, guest_port, xc->fd);
2115
0
        }
2116
0
        ret = be_port;
2117
0
        break;
2118
2119
0
    default:
2120
0
        ret = -EINVAL;
2121
0
        break;
2122
0
    }
2123
2124
0
 out:
2125
0
    qemu_mutex_unlock(&s->port_lock);
2126
2127
0
    return ret;
2128
0
}
2129
2130
int xen_be_evtchn_unbind(struct xenevtchn_handle *xc, evtchn_port_t port)
2131
0
{
2132
0
    XenEvtchnState *s = xen_evtchn_singleton;
2133
0
    int ret;
2134
2135
0
    if (!s) {
2136
0
        return -ENOTSUP;
2137
0
    }
2138
2139
0
    if (!xc) {
2140
0
        return -EFAULT;
2141
0
    }
2142
2143
0
    qemu_mutex_lock(&s->port_lock);
2144
2145
0
    if (port && port != xc->be_port) {
2146
0
        ret = -EINVAL;
2147
0
        goto out;
2148
0
    }
2149
2150
0
    if (xc->guest_port) {
2151
0
        XenEvtchnPort *gp = &s->port_table[xc->guest_port];
2152
2153
        /* This should never *not* be true */
2154
0
        if (gp->type == EVTCHNSTAT_interdomain) {
2155
0
            gp->type = EVTCHNSTAT_unbound;
2156
0
            gp->u.interdomain.port = 0;
2157
0
        }
2158
2159
0
        if (kvm_xen_has_cap(EVTCHN_SEND)) {
2160
0
            deassign_kernel_port(xc->guest_port);
2161
0
        }
2162
0
        xc->guest_port = 0;
2163
0
    }
2164
2165
0
    s->be_handles[xc->be_port] = NULL;
2166
0
    xc->be_port = 0;
2167
0
    ret = 0;
2168
0
 out:
2169
0
    qemu_mutex_unlock(&s->port_lock);
2170
0
    return ret;
2171
0
}
2172
2173
int xen_be_evtchn_close(struct xenevtchn_handle *xc)
2174
0
{
2175
0
    if (!xc) {
2176
0
        return -EFAULT;
2177
0
    }
2178
2179
0
    xen_be_evtchn_unbind(xc, 0);
2180
2181
0
    close(xc->fd);
2182
0
    free(xc);
2183
0
    return 0;
2184
0
}
2185
2186
int xen_be_evtchn_fd(struct xenevtchn_handle *xc)
2187
0
{
2188
0
    if (!xc) {
2189
0
        return -1;
2190
0
    }
2191
0
    return xc->fd;
2192
0
}
2193
2194
int xen_be_evtchn_notify(struct xenevtchn_handle *xc, evtchn_port_t port)
2195
0
{
2196
0
    XenEvtchnState *s = xen_evtchn_singleton;
2197
0
    int ret;
2198
2199
0
    if (!s) {
2200
0
        return -ENOTSUP;
2201
0
    }
2202
2203
0
    if (!xc) {
2204
0
        return -EFAULT;
2205
0
    }
2206
2207
0
    qemu_mutex_lock(&s->port_lock);
2208
2209
0
    if (xc->guest_port) {
2210
0
        set_port_pending(s, xc->guest_port);
2211
0
        ret = 0;
2212
0
    } else {
2213
0
        ret = -ENOTCONN;
2214
0
    }
2215
2216
0
    qemu_mutex_unlock(&s->port_lock);
2217
2218
0
    return ret;
2219
0
}
2220
2221
int xen_be_evtchn_pending(struct xenevtchn_handle *xc)
2222
0
{
2223
0
    uint64_t val;
2224
2225
0
    if (!xc) {
2226
0
        return -EFAULT;
2227
0
    }
2228
2229
0
    if (!xc->be_port) {
2230
0
        return 0;
2231
0
    }
2232
2233
0
    if (eventfd_read(xc->fd, &val)) {
2234
0
        return -errno;
2235
0
    }
2236
2237
0
    return val ? xc->be_port : 0;
2238
0
}
2239
2240
int xen_be_evtchn_unmask(struct xenevtchn_handle *xc, evtchn_port_t port)
2241
0
{
2242
0
    if (!xc) {
2243
0
        return -EFAULT;
2244
0
    }
2245
2246
0
    if (xc->be_port != port) {
2247
0
        return -EINVAL;
2248
0
    }
2249
2250
    /*
2251
     * We don't actually do anything to unmask it; the event was already
2252
     * consumed in xen_be_evtchn_pending().
2253
     */
2254
0
    return 0;
2255
0
}
2256
2257
int xen_be_evtchn_get_guest_port(struct xenevtchn_handle *xc)
2258
0
{
2259
0
    return xc->guest_port;
2260
0
}
2261
2262
EvtchnInfoList *qmp_xen_event_list(Error **errp)
2263
0
{
2264
0
    XenEvtchnState *s = xen_evtchn_singleton;
2265
0
    EvtchnInfoList *head = NULL, **tail = &head;
2266
0
    void *shinfo, *pending, *mask;
2267
0
    int i;
2268
2269
0
    if (!s) {
2270
0
        error_setg(errp, "Xen event channel emulation not enabled");
2271
0
        return NULL;
2272
0
    }
2273
2274
0
    shinfo = xen_overlay_get_shinfo_ptr();
2275
0
    if (!shinfo) {
2276
0
        error_setg(errp, "Xen shared info page not allocated");
2277
0
        return NULL;
2278
0
    }
2279
2280
0
    if (xen_is_long_mode()) {
2281
0
        pending = shinfo + offsetof(struct shared_info, evtchn_pending);
2282
0
        mask = shinfo + offsetof(struct shared_info, evtchn_mask);
2283
0
    } else {
2284
0
        pending = shinfo + offsetof(struct compat_shared_info, evtchn_pending);
2285
0
        mask = shinfo + offsetof(struct compat_shared_info, evtchn_mask);
2286
0
    }
2287
2288
0
    QEMU_LOCK_GUARD(&s->port_lock);
2289
2290
0
    for (i = 0; i < s->nr_ports; i++) {
2291
0
        XenEvtchnPort *p = &s->port_table[i];
2292
0
        EvtchnInfo *info;
2293
2294
0
        if (p->type == EVTCHNSTAT_closed) {
2295
0
            continue;
2296
0
        }
2297
2298
0
        info = g_new0(EvtchnInfo, 1);
2299
2300
0
        info->port = i;
2301
0
        qemu_build_assert(EVTCHN_PORT_TYPE_CLOSED == EVTCHNSTAT_closed);
2302
0
        qemu_build_assert(EVTCHN_PORT_TYPE_UNBOUND == EVTCHNSTAT_unbound);
2303
0
        qemu_build_assert(EVTCHN_PORT_TYPE_INTERDOMAIN == EVTCHNSTAT_interdomain);
2304
0
        qemu_build_assert(EVTCHN_PORT_TYPE_PIRQ == EVTCHNSTAT_pirq);
2305
0
        qemu_build_assert(EVTCHN_PORT_TYPE_VIRQ == EVTCHNSTAT_virq);
2306
0
        qemu_build_assert(EVTCHN_PORT_TYPE_IPI == EVTCHNSTAT_ipi);
2307
2308
0
        info->type = p->type;
2309
0
        if (p->type == EVTCHNSTAT_interdomain) {
2310
0
            info->remote_domain = g_strdup(p->u.interdomain.to_qemu ?
2311
0
                                           "qemu" : "loopback");
2312
0
            info->target = p->u.interdomain.port;
2313
0
        } else {
2314
0
            info->target = p->u.val; /* pirq# or virq# */
2315
0
        }
2316
0
        info->vcpu = p->vcpu;
2317
0
        info->pending = test_bit(i, pending);
2318
0
        info->masked = test_bit(i, mask);
2319
2320
0
        QAPI_LIST_APPEND(tail, info);
2321
0
    }
2322
2323
0
    return head;
2324
0
}
2325
2326
void qmp_xen_event_inject(uint32_t port, Error **errp)
2327
0
{
2328
0
    XenEvtchnState *s = xen_evtchn_singleton;
2329
2330
0
    if (!s) {
2331
0
        error_setg(errp, "Xen event channel emulation not enabled");
2332
0
        return;
2333
0
    }
2334
2335
0
    if (!valid_port(port)) {
2336
0
        error_setg(errp, "Invalid port %u", port);
2337
0
    }
2338
2339
0
    QEMU_LOCK_GUARD(&s->port_lock);
2340
2341
0
    if (set_port_pending(s, port)) {
2342
0
        error_setg(errp, "Failed to set port %u", port);
2343
0
        return;
2344
0
    }
2345
0
}
2346
2347
#ifdef CONFIG_HMP
2348
void hmp_xen_event_list(MonitorHMP *hmp, const QDict *qdict)
2349
0
{
2350
0
    EvtchnInfoList *iter, *info_list;
2351
0
    Error *err = NULL;
2352
2353
0
    info_list = qmp_xen_event_list(&err);
2354
0
    if (err) {
2355
0
        hmp_handle_error(hmp, err);
2356
0
        return;
2357
0
    }
2358
2359
0
    for (iter = info_list; iter; iter = iter->next) {
2360
0
        EvtchnInfo *info = iter->value;
2361
2362
0
        monitor_hmp_printf(hmp, "port %4u: vcpu: %d %s", info->port, info->vcpu,
2363
0
                           EvtchnPortType_str(info->type));
2364
0
        if (info->type != EVTCHN_PORT_TYPE_IPI) {
2365
0
            monitor_hmp_printf(hmp,  "(");
2366
0
            if (info->remote_domain) {
2367
0
                monitor_hmp_printf(hmp, "%s:", info->remote_domain);
2368
0
            }
2369
0
            monitor_hmp_printf(hmp, "%d)", info->target);
2370
0
        }
2371
0
        if (info->pending) {
2372
0
            monitor_hmp_printf(hmp, " PENDING");
2373
0
        }
2374
0
        if (info->masked) {
2375
0
            monitor_hmp_printf(hmp, " MASKED");
2376
0
        }
2377
0
        monitor_hmp_printf(hmp, "\n");
2378
0
    }
2379
2380
0
    qapi_free_EvtchnInfoList(info_list);
2381
0
}
2382
2383
void hmp_xen_event_inject(MonitorHMP *hmp, const QDict *qdict)
2384
0
{
2385
0
    int port = qdict_get_int(qdict, "port");
2386
0
    Error *err = NULL;
2387
2388
0
    qmp_xen_event_inject(port, &err);
2389
0
    if (err) {
2390
0
        hmp_handle_error(hmp, err);
2391
0
    } else {
2392
0
        monitor_hmp_printf(hmp, "Delivered port %d\n", port);
2393
0
    }
2394
0
}
2395
2396
#endif