Coverage Report

Created: 2026-11-20 08:13

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qemu/hw/nvme/ctrl.c
Line
Count
Source
1
/*
2
 * QEMU NVM Express Controller
3
 *
4
 * Copyright (c) 2012, Intel Corporation
5
 *
6
 * Written by Keith Busch <keith.busch@intel.com>
7
 *
8
 * This code is licensed under the GNU GPL v2 or later.
9
 */
10
11
/**
12
 * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e
13
 *
14
 *  https://nvmexpress.org/developers/nvme-specification/
15
 *
16
 *
17
 * Notes on coding style
18
 * ---------------------
19
 * While QEMU coding style prefers lowercase hexadecimals in constants, the
20
 * NVMe subsystem use this format from the NVMe specifications in the comments
21
 * (i.e. 'h' suffix instead of '0x' prefix).
22
 *
23
 * Usage
24
 * -----
25
 * See docs/system/devices/nvme.rst for extensive documentation.
26
 *
27
 * Add options:
28
 *      -drive file=<file>,if=none,id=<drive_id>
29
 *      -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id>
30
 *      -device nvme,serial=<serial>,id=<bus_name>, \
31
 *              cmb_size_mb=<cmb_size_mb[optional]>, \
32
 *              [pmrdev=<mem_backend_file_id>,] \
33
 *              max_ioqpairs=<N[optional]>, \
34
 *              aerl=<N[optional]>,aer_max_queued=<N[optional]>, \
35
 *              mdts=<N[optional]>,vsl=<N[optional]>, \
36
 *              zoned.zasl=<N[optional]>, \
37
 *              zoned.auto_transition=<on|off[optional]>, \
38
 *              sriov_max_vfs=<N[optional]> \
39
 *              sriov_vq_flexible=<N[optional]> \
40
 *              sriov_vi_flexible=<N[optional]> \
41
 *              sriov_max_vi_per_vf=<N[optional]> \
42
 *              sriov_max_vq_per_vf=<N[optional]> \
43
 *              atomic.dn=<on|off[optional]>, \
44
 *              atomic.awun<N[optional]>, \
45
 *              atomic.awupf<N[optional]>, \
46
 *              subsys=<subsys_id>, \
47
 *              model=<model-str>, \
48
 *              firmware-version=<version-str>
49
 *      -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
50
 *              zoned=<true|false[optional]>, \
51
 *              subsys=<subsys_id>,shared=<true|false[optional]>, \
52
 *              detached=<true|false[optional]>, \
53
 *              zoned.zone_size=<N[optional]>, \
54
 *              zoned.zone_capacity=<N[optional]>, \
55
 *              zoned.descr_ext_size=<N[optional]>, \
56
 *              zoned.max_active=<N[optional]>, \
57
 *              zoned.max_open=<N[optional]>, \
58
 *              zoned.cross_read=<true|false[optional]>
59
 *
60
 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
61
 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
62
 * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to
63
 * always enable the CMBLOC and CMBSZ registers (v1.3 behavior).
64
 *
65
 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
66
 * For example:
67
 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
68
 *  size=<size> .... -device nvme,...,pmrdev=<mem_id>
69
 *
70
 * The PMR will use BAR 4/5 exclusively.
71
 *
72
 * To place controller(s) and namespace(s) to a subsystem, then provide
73
 * nvme-subsys device as above.
74
 *
75
 * nvme subsystem device parameters
76
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
77
 * - `nqn`
78
 *   This parameter provides the `<nqn_id>` part of the string
79
 *   `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field
80
 *   of subsystem controllers. Note that `<nqn_id>` should be unique per
81
 *   subsystem, but this is not enforced by QEMU. If not specified, it will
82
 *   default to the value of the `id` parameter (`<subsys_id>`).
83
 *
84
 * nvme device parameters
85
 * ~~~~~~~~~~~~~~~~~~~~~~
86
 * - `subsys`
87
 *   Specifying this parameter attaches the controller to the subsystem and
88
 *   the SUBNQN field in the controller will report the NQN of the subsystem
89
 *   device. This also enables multi controller capability represented in
90
 *   Identify Controller data structure in CMIC (Controller Multi-path I/O and
91
 *   Namespace Sharing Capabilities).
92
 *
93
 * - `aerl`
94
 *   The Asynchronous Event Request Limit (AERL). Indicates the maximum number
95
 *   of concurrently outstanding Asynchronous Event Request commands support
96
 *   by the controller. This is a 0's based value.
97
 *
98
 * - `aer_max_queued`
99
 *   This is the maximum number of events that the device will enqueue for
100
 *   completion when there are no outstanding AERs. When the maximum number of
101
 *   enqueued events are reached, subsequent events will be dropped.
102
 *
103
 * - `mdts`
104
 *   Indicates the maximum data transfer size for a command that transfers data
105
 *   between host-accessible memory and the controller. The value is specified
106
 *   as a power of two (2^n) and is in units of the minimum memory page size
107
 *   (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB).
108
 *
109
 * - `vsl`
110
 *   Indicates the maximum data size limit for the Verify command. Like `mdts`,
111
 *   this value is specified as a power of two (2^n) and is in units of the
112
 *   minimum memory page size (CAP.MPSMIN). The default value is 7 (i.e. 512
113
 *   KiB).
114
 *
115
 * - `zoned.zasl`
116
 *   Indicates the maximum data transfer size for the Zone Append command. Like
117
 *   `mdts`, the value is specified as a power of two (2^n) and is in units of
118
 *   the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e.
119
 *   defaulting to the value of `mdts`).
120
 *
121
 * - `zoned.auto_transition`
122
 *   Indicates if zones in zone state implicitly opened can be automatically
123
 *   transitioned to zone state closed for resource management purposes.
124
 *   Defaults to 'on'.
125
 *
126
 * - `sriov_max_vfs`
127
 *   Indicates the maximum number of PCIe virtual functions supported
128
 *   by the controller. The default value is 0. Specifying a non-zero value
129
 *   enables reporting of both SR-IOV and ARI capabilities by the NVMe device.
130
 *   Virtual function controllers will not report SR-IOV capability.
131
 *
132
 *   NOTE: Single Root I/O Virtualization support is experimental.
133
 *   All the related parameters may be subject to change.
134
 *
135
 * - `sriov_vq_flexible`
136
 *   Indicates the total number of flexible queue resources assignable to all
137
 *   the secondary controllers. Implicitly sets the number of primary
138
 *   controller's private resources to `(max_ioqpairs - sriov_vq_flexible)`.
139
 *
140
 * - `sriov_vi_flexible`
141
 *   Indicates the total number of flexible interrupt resources assignable to
142
 *   all the secondary controllers. Implicitly sets the number of primary
143
 *   controller's private resources to `(msix_qsize - sriov_vi_flexible)`.
144
 *
145
 * - `sriov_max_vi_per_vf`
146
 *   Indicates the maximum number of virtual interrupt resources assignable
147
 *   to a secondary controller. The default 0 resolves to
148
 *   `(sriov_vi_flexible / sriov_max_vfs)`.
149
 *
150
 * - `sriov_max_vq_per_vf`
151
 *   Indicates the maximum number of virtual queue resources assignable to
152
 *   a secondary controller. The default 0 resolves to
153
 *   `(sriov_vq_flexible / sriov_max_vfs)`.
154
 *
155
 * nvme namespace device parameters
156
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
157
 * - `shared`
158
 *   When the parent nvme device (as defined explicitly by the 'bus' parameter
159
 *   or implicitly by the most recently defined NvmeBus) is linked to an
160
 *   nvme-subsys device, the namespace will be attached to all controllers in
161
 *   the subsystem. If set to 'off' (the default), the namespace will remain a
162
 *   private namespace and may only be attached to a single controller at a
163
 *   time.
164
 *
165
 * - `detached`
166
 *   This parameter is only valid together with the `subsys` parameter. If left
167
 *   at the default value (`false/off`), the namespace will be attached to all
168
 *   controllers in the NVMe subsystem at boot-up. If set to `true/on`, the
169
 *   namespace will be available in the subsystem but not attached to any
170
 *   controllers.
171
 *
172
 * Setting `zoned` to true selects Zoned Command Set at the namespace.
173
 * In this case, the following namespace properties are available to configure
174
 * zoned operation:
175
 *     zoned.zone_size=<zone size in bytes, default: 128MiB>
176
 *         The number may be followed by K, M, G as in kilo-, mega- or giga-.
177
 *
178
 *     zoned.zone_capacity=<zone capacity in bytes, default: zone size>
179
 *         The value 0 (default) forces zone capacity to be the same as zone
180
 *         size. The value of this property may not exceed zone size.
181
 *
182
 *     zoned.descr_ext_size=<zone descriptor extension size, default 0>
183
 *         This value needs to be specified in 64B units. If it is zero,
184
 *         namespace(s) will not support zone descriptor extensions.
185
 *
186
 *     zoned.max_active=<Maximum Active Resources (zones), default: 0>
187
 *         The default value means there is no limit to the number of
188
 *         concurrently active zones.
189
 *
190
 *     zoned.max_open=<Maximum Open Resources (zones), default: 0>
191
 *         The default value means there is no limit to the number of
192
 *         concurrently open zones.
193
 *
194
 *     zoned.cross_read=<enable RAZB, default: false>
195
 *         Setting this property to true enables Read Across Zone Boundaries.
196
 */
197
198
#include "qemu/osdep.h"
199
#include "qemu/bitops.h"
200
#include "qemu/cutils.h"
201
#include "qemu/error-report.h"
202
#include "qemu/log.h"
203
#include "qemu/units.h"
204
#include "qemu/range.h"
205
#include "qapi/error.h"
206
#include "qapi/qapi-builtin-type-infos.h"
207
#include "qapi/visitor.h"
208
#include "system/system.h"
209
#include "system/block-backend.h"
210
#include "system/hostmem.h"
211
#include "hw/pci/msix.h"
212
#include "hw/pci/pcie_sriov.h"
213
#include "system/spdm-socket.h"
214
#include "migration/blocker.h"
215
#include "migration/qemu-file-types.h"
216
#include "migration/vmstate.h"
217
218
#include "nvme.h"
219
#include "dif.h"
220
#include "trace.h"
221
222
0
#define NVME_MAX_IOQPAIRS 0xffff
223
0
#define NVME_DB_SIZE  4
224
0
#define NVME_SPEC_VER 0x00010400
225
0
#define NVME_CMB_BIR 2
226
0
#define NVME_PMR_BIR 4
227
0
#define NVME_TEMPERATURE 0x143
228
0
#define NVME_TEMPERATURE_WARNING 0x157
229
0
#define NVME_TEMPERATURE_CRITICAL 0x175
230
0
#define NVME_NUM_FW_SLOTS 1
231
#define NVME_DEFAULT_MAX_ZA_SIZE (128 * KiB)
232
0
#define NVME_VF_RES_GRANULARITY 1
233
0
#define NVME_VF_OFFSET 0x1
234
0
#define NVME_VF_STRIDE 1
235
236
#define NVME_GUEST_ERR(trace, fmt, ...) \
237
0
    do { \
238
0
        (trace_##trace)(__VA_ARGS__); \
239
0
        qemu_log_mask(LOG_GUEST_ERROR, #trace \
240
0
            " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
241
0
    } while (0)
242
243
static const bool nvme_feature_support[NVME_FID_MAX] = {
244
    [NVME_ARBITRATION]              = true,
245
    [NVME_POWER_MANAGEMENT]         = true,
246
    [NVME_TEMPERATURE_THRESHOLD]    = true,
247
    [NVME_ERROR_RECOVERY]           = true,
248
    [NVME_VOLATILE_WRITE_CACHE]     = true,
249
    [NVME_NUMBER_OF_QUEUES]         = true,
250
    [NVME_INTERRUPT_COALESCING]     = true,
251
    [NVME_INTERRUPT_VECTOR_CONF]    = true,
252
    [NVME_WRITE_ATOMICITY]          = true,
253
    [NVME_ASYNCHRONOUS_EVENT_CONF]  = true,
254
    [NVME_TIMESTAMP]                = true,
255
    [NVME_HOST_BEHAVIOR_SUPPORT]    = true,
256
    [NVME_COMMAND_SET_PROFILE]      = true,
257
    [NVME_FDP_MODE]                 = true,
258
    [NVME_FDP_EVENTS]               = true,
259
    /* if you add something here, please update nvme_set_migration_blockers() */
260
};
261
262
static const uint32_t nvme_feature_cap[NVME_FID_MAX] = {
263
    [NVME_TEMPERATURE_THRESHOLD]    = NVME_FEAT_CAP_CHANGE,
264
    [NVME_ERROR_RECOVERY]           = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
265
    [NVME_VOLATILE_WRITE_CACHE]     = NVME_FEAT_CAP_CHANGE,
266
    [NVME_NUMBER_OF_QUEUES]         = NVME_FEAT_CAP_CHANGE,
267
    [NVME_WRITE_ATOMICITY]          = NVME_FEAT_CAP_CHANGE,
268
    [NVME_ASYNCHRONOUS_EVENT_CONF]  = NVME_FEAT_CAP_CHANGE,
269
    [NVME_TIMESTAMP]                = NVME_FEAT_CAP_CHANGE,
270
    [NVME_HOST_BEHAVIOR_SUPPORT]    = NVME_FEAT_CAP_CHANGE,
271
    [NVME_COMMAND_SET_PROFILE]      = NVME_FEAT_CAP_CHANGE,
272
    [NVME_FDP_MODE]                 = NVME_FEAT_CAP_CHANGE,
273
    [NVME_FDP_EVENTS]               = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
274
};
275
276
static const uint32_t nvme_cse_acs_default[256] = {
277
    [NVME_ADM_CMD_DELETE_SQ]        = NVME_CMD_EFF_CSUPP,
278
    [NVME_ADM_CMD_CREATE_SQ]        = NVME_CMD_EFF_CSUPP,
279
    [NVME_ADM_CMD_GET_LOG_PAGE]     = NVME_CMD_EFF_CSUPP,
280
    [NVME_ADM_CMD_DELETE_CQ]        = NVME_CMD_EFF_CSUPP,
281
    [NVME_ADM_CMD_CREATE_CQ]        = NVME_CMD_EFF_CSUPP,
282
    [NVME_ADM_CMD_IDENTIFY]         = NVME_CMD_EFF_CSUPP,
283
    [NVME_ADM_CMD_ABORT]            = NVME_CMD_EFF_CSUPP,
284
    [NVME_ADM_CMD_SET_FEATURES]     = NVME_CMD_EFF_CSUPP,
285
    [NVME_ADM_CMD_GET_FEATURES]     = NVME_CMD_EFF_CSUPP,
286
    [NVME_ADM_CMD_ASYNC_EV_REQ]     = NVME_CMD_EFF_CSUPP,
287
    [NVME_ADM_CMD_NS_ATTACHMENT]    = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_NIC |
288
                                      NVME_CMD_EFF_CCC,
289
    [NVME_ADM_CMD_FORMAT_NVM]       = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
290
    [NVME_ADM_CMD_DIRECTIVE_RECV]   = NVME_CMD_EFF_CSUPP,
291
    [NVME_ADM_CMD_DIRECTIVE_SEND]   = NVME_CMD_EFF_CSUPP,
292
    [NVME_ADM_CMD_SECURITY_SEND]    = NVME_CMD_EFF_CSUPP,
293
    [NVME_ADM_CMD_SECURITY_RECV]    = NVME_CMD_EFF_CSUPP,
294
};
295
296
static const uint32_t nvme_cse_iocs_nvm_default[256] = {
297
    [NVME_CMD_FLUSH]                = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
298
    [NVME_CMD_WRITE_ZEROES]         = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
299
    [NVME_CMD_WRITE]                = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
300
    [NVME_CMD_READ]                 = NVME_CMD_EFF_CSUPP,
301
    [NVME_CMD_DSM]                  = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
302
    [NVME_CMD_VERIFY]               = NVME_CMD_EFF_CSUPP,
303
    [NVME_CMD_COPY]                 = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
304
    [NVME_CMD_COMPARE]              = NVME_CMD_EFF_CSUPP,
305
    [NVME_CMD_IO_MGMT_RECV]         = NVME_CMD_EFF_CSUPP,
306
    [NVME_CMD_IO_MGMT_SEND]         = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
307
};
308
309
static const uint32_t nvme_cse_iocs_zoned_default[256] = {
310
    [NVME_CMD_FLUSH]                = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
311
    [NVME_CMD_WRITE_ZEROES]         = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
312
    [NVME_CMD_WRITE]                = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
313
    [NVME_CMD_READ]                 = NVME_CMD_EFF_CSUPP,
314
    [NVME_CMD_DSM]                  = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
315
    [NVME_CMD_VERIFY]               = NVME_CMD_EFF_CSUPP,
316
    [NVME_CMD_COPY]                 = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
317
    [NVME_CMD_COMPARE]              = NVME_CMD_EFF_CSUPP,
318
    [NVME_CMD_IO_MGMT_RECV]         = NVME_CMD_EFF_CSUPP,
319
    [NVME_CMD_IO_MGMT_SEND]         = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
320
321
    [NVME_CMD_ZONE_APPEND]          = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
322
    [NVME_CMD_ZONE_MGMT_SEND]       = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
323
    [NVME_CMD_ZONE_MGMT_RECV]       = NVME_CMD_EFF_CSUPP,
324
};
325
326
static void nvme_process_sq(void *opaque);
327
static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst);
328
static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n);
329
330
static uint16_t nvme_sqid(NvmeRequest *req)
331
0
{
332
0
    return le16_to_cpu(req->sq->sqid);
333
0
}
334
335
static inline uint16_t nvme_make_pid(NvmeNamespace *ns, uint16_t rg,
336
                                     uint16_t ph)
337
0
{
338
0
    uint16_t rgif = ns->endgrp->fdp.rgif;
339
340
0
    if (!rgif) {
341
0
        return ph;
342
0
    }
343
344
0
    return (rg << (16 - rgif)) | ph;
345
0
}
346
347
static inline bool nvme_ph_valid(NvmeNamespace *ns, uint16_t ph)
348
0
{
349
0
    return ph < ns->fdp.nphs;
350
0
}
351
352
static inline bool nvme_rg_valid(NvmeEnduranceGroup *endgrp, uint16_t rg)
353
0
{
354
0
    return rg < endgrp->fdp.nrg;
355
0
}
356
357
static inline uint16_t nvme_pid2ph(NvmeNamespace *ns, uint16_t pid)
358
0
{
359
0
    uint16_t rgif = ns->endgrp->fdp.rgif;
360
361
0
    if (!rgif) {
362
0
        return pid;
363
0
    }
364
365
0
    return pid & ((1 << (15 - rgif)) - 1);
366
0
}
367
368
static inline uint16_t nvme_pid2rg(NvmeNamespace *ns, uint16_t pid)
369
0
{
370
0
    uint16_t rgif = ns->endgrp->fdp.rgif;
371
372
0
    if (!rgif) {
373
0
        return 0;
374
0
    }
375
376
0
    return pid >> (16 - rgif);
377
0
}
378
379
static inline bool nvme_parse_pid(NvmeNamespace *ns, uint16_t pid,
380
                                  uint16_t *ph, uint16_t *rg)
381
0
{
382
0
    *rg = nvme_pid2rg(ns, pid);
383
0
    *ph = nvme_pid2ph(ns, pid);
384
385
0
    return nvme_ph_valid(ns, *ph) && nvme_rg_valid(ns->endgrp, *rg);
386
0
}
387
388
static void nvme_assign_zone_state(NvmeNamespace *ns, NvmeZone *zone,
389
                                   NvmeZoneState state)
390
0
{
391
0
    if (QTAILQ_IN_USE(zone, entry)) {
392
0
        switch (nvme_get_zone_state(zone)) {
393
0
        case NVME_ZONE_STATE_EXPLICITLY_OPEN:
394
0
            QTAILQ_REMOVE(&ns->exp_open_zones, zone, entry);
395
0
            break;
396
0
        case NVME_ZONE_STATE_IMPLICITLY_OPEN:
397
0
            QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
398
0
            break;
399
0
        case NVME_ZONE_STATE_CLOSED:
400
0
            QTAILQ_REMOVE(&ns->closed_zones, zone, entry);
401
0
            break;
402
0
        case NVME_ZONE_STATE_FULL:
403
0
            QTAILQ_REMOVE(&ns->full_zones, zone, entry);
404
0
        default:
405
0
            ;
406
0
        }
407
0
    }
408
409
0
    nvme_set_zone_state(zone, state);
410
411
0
    switch (state) {
412
0
    case NVME_ZONE_STATE_EXPLICITLY_OPEN:
413
0
        QTAILQ_INSERT_TAIL(&ns->exp_open_zones, zone, entry);
414
0
        break;
415
0
    case NVME_ZONE_STATE_IMPLICITLY_OPEN:
416
0
        QTAILQ_INSERT_TAIL(&ns->imp_open_zones, zone, entry);
417
0
        break;
418
0
    case NVME_ZONE_STATE_CLOSED:
419
0
        QTAILQ_INSERT_TAIL(&ns->closed_zones, zone, entry);
420
0
        break;
421
0
    case NVME_ZONE_STATE_FULL:
422
0
        QTAILQ_INSERT_TAIL(&ns->full_zones, zone, entry);
423
0
    case NVME_ZONE_STATE_READ_ONLY:
424
0
        break;
425
0
    default:
426
0
        zone->d.za = 0;
427
0
    }
428
0
}
429
430
static uint16_t nvme_zns_check_resources(NvmeNamespace *ns, uint32_t act,
431
                                         uint32_t opn, uint32_t zrwa)
432
0
{
433
0
    if (ns->params.max_active_zones != 0 &&
434
0
        ns->nr_active_zones + act > ns->params.max_active_zones) {
435
0
        trace_pci_nvme_err_insuff_active_res(ns->params.max_active_zones);
436
0
        return NVME_ZONE_TOO_MANY_ACTIVE | NVME_DNR;
437
0
    }
438
439
0
    if (ns->params.max_open_zones != 0 &&
440
0
        ns->nr_open_zones + opn > ns->params.max_open_zones) {
441
0
        trace_pci_nvme_err_insuff_open_res(ns->params.max_open_zones);
442
0
        return NVME_ZONE_TOO_MANY_OPEN | NVME_DNR;
443
0
    }
444
445
0
    if (zrwa > ns->zns.numzrwa) {
446
0
        return NVME_NOZRWA | NVME_DNR;
447
0
    }
448
449
0
    return NVME_SUCCESS;
450
0
}
451
452
/*
453
 * Check if we can open a zone without exceeding open/active limits.
454
 * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5).
455
 */
456
static uint16_t nvme_aor_check(NvmeNamespace *ns, uint32_t act, uint32_t opn)
457
0
{
458
0
    return nvme_zns_check_resources(ns, act, opn, 0);
459
0
}
460
461
static NvmeFdpEvent *nvme_fdp_alloc_event(NvmeCtrl *n, NvmeFdpEventBuffer *ebuf)
462
0
{
463
0
    NvmeFdpEvent *ret = NULL;
464
0
    bool is_full = ebuf->next == ebuf->start && ebuf->nelems;
465
466
0
    ret = &ebuf->events[ebuf->next++];
467
0
    if (unlikely(ebuf->next == NVME_FDP_MAX_EVENTS)) {
468
0
        ebuf->next = 0;
469
0
    }
470
0
    if (is_full) {
471
0
        ebuf->start = ebuf->next;
472
0
    } else {
473
0
        ebuf->nelems++;
474
0
    }
475
476
0
    memset(ret, 0, sizeof(NvmeFdpEvent));
477
0
    ret->timestamp = nvme_get_timestamp(n);
478
479
0
    return ret;
480
0
}
481
482
static inline int log_event(NvmeRuHandle *ruh, uint8_t event_type)
483
0
{
484
0
    return (ruh->event_filter >> nvme_fdp_evf_shifts[event_type]) & 0x1;
485
0
}
486
487
static bool nvme_update_ruh(NvmeCtrl *n, NvmeNamespace *ns, uint16_t pid)
488
0
{
489
0
    NvmeEnduranceGroup *endgrp = ns->endgrp;
490
0
    NvmeRuHandle *ruh;
491
0
    NvmeReclaimUnit *ru;
492
0
    NvmeFdpEvent *e = NULL;
493
0
    uint16_t ph, rg, ruhid;
494
495
0
    if (!nvme_parse_pid(ns, pid, &ph, &rg)) {
496
0
        return false;
497
0
    }
498
499
0
    ruhid = ns->fdp.phs[ph];
500
501
0
    ruh = &endgrp->fdp.ruhs[ruhid];
502
0
    ru = &ruh->rus[rg];
503
504
0
    if (ru->ruamw) {
505
0
        if (log_event(ruh, FDP_EVT_RU_NOT_FULLY_WRITTEN)) {
506
0
            e = nvme_fdp_alloc_event(n, &endgrp->fdp.host_events);
507
0
            e->type = FDP_EVT_RU_NOT_FULLY_WRITTEN;
508
0
            e->flags = FDPEF_PIV | FDPEF_NSIDV | FDPEF_LV;
509
0
            e->pid = cpu_to_le16(pid);
510
0
            e->nsid = cpu_to_le32(ns->params.nsid);
511
0
            e->rgid = cpu_to_le16(rg);
512
0
            e->ruhid = cpu_to_le16(ruhid);
513
0
        }
514
515
        /* log (eventual) GC overhead of prematurely swapping the RU */
516
0
        nvme_fdp_stat_inc(&endgrp->fdp.mbmw, nvme_l2b(ns, ru->ruamw));
517
0
    }
518
519
0
    ru->ruamw = ruh->ruamw;
520
521
0
    return true;
522
0
}
523
524
static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
525
0
{
526
0
    hwaddr hi, lo;
527
528
0
    if (!n->cmb.cmse) {
529
0
        return false;
530
0
    }
531
532
0
    lo = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
533
0
    hi = lo + int128_get64(n->cmb.mem.size);
534
535
0
    return addr >= lo && addr < hi;
536
0
}
537
538
static inline void *nvme_addr_to_cmb(NvmeCtrl *n, hwaddr addr)
539
0
{
540
0
    hwaddr base = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
541
0
    return &n->cmb.buf[addr - base];
542
0
}
543
544
static bool nvme_addr_is_pmr(NvmeCtrl *n, hwaddr addr)
545
0
{
546
0
    hwaddr hi;
547
548
0
    if (!n->pmr.cmse) {
549
0
        return false;
550
0
    }
551
552
0
    hi = n->pmr.cba + int128_get64(n->pmr.dev->mr.size);
553
554
0
    return addr >= n->pmr.cba && addr < hi;
555
0
}
556
557
static inline void *nvme_addr_to_pmr(NvmeCtrl *n, hwaddr addr)
558
0
{
559
0
    return memory_region_get_ram_ptr(&n->pmr.dev->mr) + (addr - n->pmr.cba);
560
0
}
561
562
static inline bool nvme_addr_is_iomem(NvmeCtrl *n, hwaddr addr)
563
0
{
564
0
    hwaddr hi, lo;
565
566
    /*
567
     * The purpose of this check is to guard against invalid "local" access to
568
     * the iomem (i.e. controller registers). Thus, we check against the range
569
     * covered by the 'bar0' MemoryRegion since that is currently composed of
570
     * two subregions (the NVMe "MBAR" and the MSI-X table/pba). Note, however,
571
     * that if the device model is ever changed to allow the CMB to be located
572
     * in BAR0 as well, then this must be changed.
573
     */
574
0
    lo = n->bar0.addr;
575
0
    hi = lo + int128_get64(n->bar0.size);
576
577
0
    return addr >= lo && addr < hi;
578
0
}
579
580
static int nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
581
0
{
582
0
    hwaddr hi = addr + size - 1;
583
0
    if (hi < addr) {
584
0
        return 1;
585
0
    }
586
587
0
    if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
588
0
        memcpy(buf, nvme_addr_to_cmb(n, addr), size);
589
0
        return 0;
590
0
    }
591
592
0
    if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
593
0
        memcpy(buf, nvme_addr_to_pmr(n, addr), size);
594
0
        return 0;
595
0
    }
596
597
0
    return pci_dma_read(PCI_DEVICE(n), addr, buf, size);
598
0
}
599
600
static int nvme_addr_write(NvmeCtrl *n, hwaddr addr, const void *buf, int size)
601
0
{
602
0
    hwaddr hi = addr + size - 1;
603
0
    if (hi < addr) {
604
0
        return 1;
605
0
    }
606
607
0
    if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
608
0
        memcpy(nvme_addr_to_cmb(n, addr), buf, size);
609
0
        return 0;
610
0
    }
611
612
0
    if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
613
0
        memcpy(nvme_addr_to_pmr(n, addr), buf, size);
614
0
        return 0;
615
0
    }
616
617
0
    return pci_dma_write(PCI_DEVICE(n), addr, buf, size);
618
0
}
619
620
static bool nvme_nsid_valid(NvmeCtrl *n, uint32_t nsid)
621
0
{
622
0
    return nsid &&
623
0
        (nsid == NVME_NSID_BROADCAST || nsid <= NVME_MAX_NAMESPACES);
624
0
}
625
626
static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
627
0
{
628
0
    return sqid < n->conf_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1;
629
0
}
630
631
static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
632
0
{
633
0
    return cqid < n->conf_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1;
634
0
}
635
636
static void nvme_inc_cq_tail(NvmeCQueue *cq)
637
0
{
638
0
    cq->tail++;
639
0
    if (cq->tail >= cq->size) {
640
0
        cq->tail = 0;
641
0
        cq->phase = !cq->phase;
642
0
    }
643
0
}
644
645
static void nvme_inc_sq_head(NvmeSQueue *sq)
646
0
{
647
0
    sq->head = (sq->head + 1) % sq->size;
648
0
}
649
650
static uint8_t nvme_cq_full(NvmeCQueue *cq)
651
0
{
652
0
    return (cq->tail + 1) % cq->size == cq->head;
653
0
}
654
655
static uint8_t nvme_sq_empty(NvmeSQueue *sq)
656
0
{
657
0
    return sq->head == sq->tail;
658
0
}
659
660
static void nvme_irq_check(NvmeCtrl *n)
661
0
{
662
0
    PCIDevice *pci = PCI_DEVICE(n);
663
0
    uint32_t intms = ldl_le_p(&n->bar.intms);
664
665
0
    if (msix_enabled(pci)) {
666
0
        return;
667
0
    }
668
669
    /* vfs does not implement intx */
670
0
    if (pci_is_vf(pci)) {
671
0
        return;
672
0
    }
673
674
0
    if (~intms & n->irq_status) {
675
0
        pci_irq_assert(pci);
676
0
    } else {
677
0
        pci_irq_deassert(pci);
678
0
    }
679
0
}
680
681
static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
682
0
{
683
0
    PCIDevice *pci = PCI_DEVICE(n);
684
685
0
    if (cq->irq_enabled) {
686
0
        if (msix_enabled(pci)) {
687
0
            trace_pci_nvme_irq_msix(cq->vector);
688
0
            msix_notify(pci, cq->vector);
689
0
        } else {
690
0
            trace_pci_nvme_irq_pin();
691
0
            assert(cq->vector < 32);
692
0
            n->irq_status |= 1 << cq->vector;
693
0
            nvme_irq_check(n);
694
0
        }
695
0
    } else {
696
0
        trace_pci_nvme_irq_masked();
697
0
    }
698
0
}
699
700
static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
701
0
{
702
0
    if (cq->irq_enabled) {
703
0
        if (msix_enabled(PCI_DEVICE(n))) {
704
0
            return;
705
0
        } else {
706
0
            assert(cq->vector < 32);
707
0
            if (!n->cq_pending) {
708
0
                n->irq_status &= ~(1 << cq->vector);
709
0
            }
710
0
            nvme_irq_check(n);
711
0
        }
712
0
    }
713
0
}
714
715
static void nvme_req_clear(NvmeRequest *req)
716
0
{
717
0
    req->ns = NULL;
718
0
    req->opaque = NULL;
719
0
    req->aiocb = NULL;
720
0
    memset(&req->cqe, 0x0, sizeof(req->cqe));
721
0
    req->status = NVME_SUCCESS;
722
0
}
723
724
static inline void nvme_sg_init(NvmeCtrl *n, NvmeSg *sg, bool dma)
725
0
{
726
0
    if (dma) {
727
0
        pci_dma_sglist_init(&sg->qsg, PCI_DEVICE(n), 0);
728
0
        sg->flags = NVME_SG_DMA;
729
0
    } else {
730
0
        qemu_iovec_init(&sg->iov, 0);
731
0
    }
732
733
0
    sg->flags |= NVME_SG_ALLOC;
734
0
}
735
736
static inline void nvme_sg_unmap(NvmeSg *sg)
737
0
{
738
0
    if (!(sg->flags & NVME_SG_ALLOC)) {
739
0
        return;
740
0
    }
741
742
0
    if (sg->flags & NVME_SG_DMA) {
743
0
        qemu_sglist_destroy(&sg->qsg);
744
0
    } else {
745
0
        qemu_iovec_destroy(&sg->iov);
746
0
    }
747
748
0
    memset(sg, 0x0, sizeof(*sg));
749
0
}
750
751
/*
752
 * When metadata is transferred as extended LBAs, the DPTR mapped into `sg`
753
 * holds both data and metadata. This function splits the data and metadata
754
 * into two separate QSG/IOVs.
755
 */
756
static void nvme_sg_split(NvmeSg *sg, NvmeNamespace *ns, NvmeSg *data,
757
                          NvmeSg *mdata)
758
0
{
759
0
    NvmeSg *dst = data;
760
0
    uint32_t trans_len, count = ns->lbasz;
761
0
    uint64_t offset = 0;
762
0
    bool dma = sg->flags & NVME_SG_DMA;
763
0
    size_t sge_len;
764
0
    size_t sg_len = dma ? sg->qsg.size : sg->iov.size;
765
0
    int sg_idx = 0;
766
767
0
    assert(sg->flags & NVME_SG_ALLOC);
768
769
0
    while (sg_len) {
770
0
        sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
771
772
0
        trans_len = MIN(sg_len, count);
773
0
        trans_len = MIN(trans_len, sge_len - offset);
774
775
0
        if (dst) {
776
0
            if (dma) {
777
0
                qemu_sglist_add(&dst->qsg, sg->qsg.sg[sg_idx].base + offset,
778
0
                                trans_len);
779
0
            } else {
780
0
                qemu_iovec_add(&dst->iov,
781
0
                               sg->iov.iov[sg_idx].iov_base + offset,
782
0
                               trans_len);
783
0
            }
784
0
        }
785
786
0
        sg_len -= trans_len;
787
0
        count -= trans_len;
788
0
        offset += trans_len;
789
790
0
        if (count == 0) {
791
0
            dst = (dst == data) ? mdata : data;
792
0
            count = (dst == data) ? ns->lbasz : ns->lbaf.ms;
793
0
        }
794
795
0
        if (sge_len == offset) {
796
0
            offset = 0;
797
0
            sg_idx++;
798
0
        }
799
0
    }
800
0
}
801
802
static uint16_t nvme_map_addr_cmb(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
803
                                  size_t len)
804
0
{
805
0
    if (!len) {
806
0
        return NVME_SUCCESS;
807
0
    }
808
809
0
    trace_pci_nvme_map_addr_cmb(addr, len);
810
811
0
    if (!nvme_addr_is_cmb(n, addr) || !nvme_addr_is_cmb(n, addr + len - 1)) {
812
0
        return NVME_DATA_TRAS_ERROR;
813
0
    }
814
815
0
    qemu_iovec_add(iov, nvme_addr_to_cmb(n, addr), len);
816
817
0
    return NVME_SUCCESS;
818
0
}
819
820
static uint16_t nvme_map_addr_pmr(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
821
                                  size_t len)
822
0
{
823
0
    if (!len) {
824
0
        return NVME_SUCCESS;
825
0
    }
826
827
0
    if (!nvme_addr_is_pmr(n, addr) || !nvme_addr_is_pmr(n, addr + len - 1)) {
828
0
        return NVME_DATA_TRAS_ERROR;
829
0
    }
830
831
0
    qemu_iovec_add(iov, nvme_addr_to_pmr(n, addr), len);
832
833
0
    return NVME_SUCCESS;
834
0
}
835
836
static uint16_t nvme_map_addr(NvmeCtrl *n, NvmeSg *sg, hwaddr addr, size_t len)
837
0
{
838
0
    bool cmb = false, pmr = false;
839
840
0
    if (!len) {
841
0
        return NVME_SUCCESS;
842
0
    }
843
844
0
    trace_pci_nvme_map_addr(addr, len);
845
846
0
    if (nvme_addr_is_iomem(n, addr)) {
847
0
        return NVME_DATA_TRAS_ERROR;
848
0
    }
849
850
0
    if (nvme_addr_is_cmb(n, addr)) {
851
0
        cmb = true;
852
0
    } else if (nvme_addr_is_pmr(n, addr)) {
853
0
        pmr = true;
854
0
    }
855
856
0
    if (cmb || pmr) {
857
0
        if (sg->flags & NVME_SG_DMA) {
858
0
            return NVME_INVALID_USE_OF_CMB | NVME_DNR;
859
0
        }
860
861
0
        if (sg->iov.niov + 1 > IOV_MAX) {
862
0
            goto max_mappings_exceeded;
863
0
        }
864
865
0
        if (cmb) {
866
0
            return nvme_map_addr_cmb(n, &sg->iov, addr, len);
867
0
        } else {
868
0
            return nvme_map_addr_pmr(n, &sg->iov, addr, len);
869
0
        }
870
0
    }
871
872
0
    if (!(sg->flags & NVME_SG_DMA)) {
873
0
        return NVME_INVALID_USE_OF_CMB | NVME_DNR;
874
0
    }
875
876
0
    if (sg->qsg.nsg + 1 > IOV_MAX) {
877
0
        goto max_mappings_exceeded;
878
0
    }
879
880
0
    qemu_sglist_add(&sg->qsg, addr, len);
881
882
0
    return NVME_SUCCESS;
883
884
0
max_mappings_exceeded:
885
0
    NVME_GUEST_ERR(pci_nvme_ub_too_many_mappings,
886
0
                   "number of mappings exceed 1024");
887
0
    return NVME_INTERNAL_DEV_ERROR | NVME_DNR;
888
0
}
889
890
static inline bool nvme_addr_is_dma(NvmeCtrl *n, hwaddr addr)
891
0
{
892
0
    return !(nvme_addr_is_cmb(n, addr) || nvme_addr_is_pmr(n, addr));
893
0
}
894
895
static uint16_t nvme_map_prp(NvmeCtrl *n, NvmeSg *sg, uint64_t prp1,
896
                             uint64_t prp2, uint32_t len)
897
0
{
898
0
    hwaddr trans_len = n->page_size - (prp1 % n->page_size);
899
0
    trans_len = MIN(len, trans_len);
900
0
    int num_prps = (len >> n->page_bits) + 1;
901
0
    uint16_t status;
902
0
    int ret;
903
904
0
    trace_pci_nvme_map_prp(trans_len, len, prp1, prp2, num_prps);
905
906
0
    nvme_sg_init(n, sg, nvme_addr_is_dma(n, prp1));
907
908
0
    status = nvme_map_addr(n, sg, prp1, trans_len);
909
0
    if (status) {
910
0
        goto unmap;
911
0
    }
912
913
0
    len -= trans_len;
914
0
    if (len) {
915
0
        if (len > n->page_size) {
916
0
            g_autofree uint64_t *prp_list = g_new(uint64_t, n->max_prp_ents);
917
0
            uint32_t nents, prp_trans;
918
0
            int i = 0;
919
920
            /*
921
             * The first PRP list entry, pointed to by PRP2 may contain offset.
922
             * Hence, we need to calculate the number of entries in based on
923
             * that offset.
924
             */
925
0
            nents = (n->page_size - (prp2 & (n->page_size - 1))) >> 3;
926
0
            prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
927
0
            ret = nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
928
0
            if (ret) {
929
0
                trace_pci_nvme_err_addr_read(prp2);
930
0
                status = NVME_DATA_TRAS_ERROR;
931
0
                goto unmap;
932
0
            }
933
0
            while (len != 0) {
934
0
                uint64_t prp_ent = le64_to_cpu(prp_list[i]);
935
936
0
                if (i == nents - 1 && len > n->page_size) {
937
0
                    if (unlikely(prp_ent & (n->page_size - 1))) {
938
0
                        trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
939
0
                        status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
940
0
                        goto unmap;
941
0
                    }
942
943
0
                    i = 0;
944
0
                    nents = (len + n->page_size - 1) >> n->page_bits;
945
0
                    nents = MIN(nents, n->max_prp_ents);
946
0
                    prp_trans = nents * sizeof(uint64_t);
947
0
                    ret = nvme_addr_read(n, prp_ent, (void *)prp_list,
948
0
                                         prp_trans);
949
0
                    if (ret) {
950
0
                        trace_pci_nvme_err_addr_read(prp_ent);
951
0
                        status = NVME_DATA_TRAS_ERROR;
952
0
                        goto unmap;
953
0
                    }
954
0
                    prp_ent = le64_to_cpu(prp_list[i]);
955
0
                }
956
957
0
                if (unlikely(prp_ent & (n->page_size - 1))) {
958
0
                    trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
959
0
                    status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
960
0
                    goto unmap;
961
0
                }
962
963
0
                trans_len = MIN(len, n->page_size);
964
0
                status = nvme_map_addr(n, sg, prp_ent, trans_len);
965
0
                if (status) {
966
0
                    goto unmap;
967
0
                }
968
969
0
                len -= trans_len;
970
0
                i++;
971
0
            }
972
0
        } else {
973
0
            if (unlikely(prp2 & (n->page_size - 1))) {
974
0
                trace_pci_nvme_err_invalid_prp2_align(prp2);
975
0
                status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
976
0
                goto unmap;
977
0
            }
978
0
            status = nvme_map_addr(n, sg, prp2, len);
979
0
            if (status) {
980
0
                goto unmap;
981
0
            }
982
0
        }
983
0
    }
984
985
0
    return NVME_SUCCESS;
986
987
0
unmap:
988
0
    nvme_sg_unmap(sg);
989
0
    return status;
990
0
}
991
992
/*
993
 * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
994
 * number of bytes mapped in len.
995
 */
996
static uint16_t nvme_map_sgl_data(NvmeCtrl *n, NvmeSg *sg,
997
                                  NvmeSglDescriptor *segment, uint64_t nsgld,
998
                                  size_t *len, NvmeCmd *cmd)
999
0
{
1000
0
    dma_addr_t addr, trans_len;
1001
0
    uint32_t dlen;
1002
0
    uint16_t status;
1003
1004
0
    for (int i = 0; i < nsgld; i++) {
1005
0
        uint8_t type = NVME_SGL_TYPE(segment[i].type);
1006
1007
0
        switch (type) {
1008
0
        case NVME_SGL_DESCR_TYPE_DATA_BLOCK:
1009
0
            break;
1010
0
        case NVME_SGL_DESCR_TYPE_SEGMENT:
1011
0
        case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
1012
0
            return NVME_INVALID_NUM_SGL_DESCRS | NVME_DNR;
1013
0
        default:
1014
0
            return NVME_SGL_DESCR_TYPE_INVALID | NVME_DNR;
1015
0
        }
1016
1017
0
        dlen = le32_to_cpu(segment[i].len);
1018
1019
0
        if (!dlen) {
1020
0
            continue;
1021
0
        }
1022
1023
0
        if (*len == 0) {
1024
            /*
1025
             * All data has been mapped, but the SGL contains additional
1026
             * segments and/or descriptors. The controller might accept
1027
             * ignoring the rest of the SGL.
1028
             */
1029
0
            uint32_t sgls = le32_to_cpu(n->id_ctrl.sgls);
1030
0
            if (sgls & NVME_CTRL_SGLS_EXCESS_LENGTH) {
1031
0
                break;
1032
0
            }
1033
1034
0
            trace_pci_nvme_err_invalid_sgl_excess_length(dlen);
1035
0
            return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1036
0
        }
1037
1038
0
        trans_len = MIN(*len, dlen);
1039
1040
0
        addr = le64_to_cpu(segment[i].addr);
1041
1042
0
        if (UINT64_MAX - addr < dlen) {
1043
0
            return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1044
0
        }
1045
1046
0
        status = nvme_map_addr(n, sg, addr, trans_len);
1047
0
        if (status) {
1048
0
            return status;
1049
0
        }
1050
1051
0
        *len -= trans_len;
1052
0
    }
1053
1054
0
    return NVME_SUCCESS;
1055
0
}
1056
1057
static uint16_t nvme_map_sgl(NvmeCtrl *n, NvmeSg *sg, NvmeSglDescriptor sgl,
1058
                             size_t len, NvmeCmd *cmd)
1059
0
{
1060
    /*
1061
     * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
1062
     * dynamically allocating a potentially huge SGL. The spec allows the SGL
1063
     * to be larger (as in number of bytes required to describe the SGL
1064
     * descriptors and segment chain) than the command transfer size, so it is
1065
     * not bounded by MDTS.
1066
     */
1067
0
#define SEG_CHUNK_SIZE 256
1068
1069
0
    QEMU_UNINITIALIZED NvmeSglDescriptor segment[SEG_CHUNK_SIZE];
1070
0
    NvmeSglDescriptor *sgld, *last_sgld;
1071
0
    uint64_t nsgld;
1072
0
    uint32_t seg_len;
1073
0
    uint16_t status;
1074
0
    hwaddr addr;
1075
0
    int ret;
1076
1077
0
    sgld = &sgl;
1078
0
    addr = le64_to_cpu(sgl.addr);
1079
1080
0
    trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl.type), len);
1081
1082
0
    nvme_sg_init(n, sg, nvme_addr_is_dma(n, addr));
1083
1084
    /*
1085
     * If the entire transfer can be described with a single data block it can
1086
     * be mapped directly.
1087
     */
1088
0
    if (NVME_SGL_TYPE(sgl.type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1089
0
        status = nvme_map_sgl_data(n, sg, sgld, 1, &len, cmd);
1090
0
        if (status) {
1091
0
            goto unmap;
1092
0
        }
1093
1094
0
        goto out;
1095
0
    }
1096
1097
0
    for (;;) {
1098
0
        size_t prev_len = len;
1099
1100
0
        switch (NVME_SGL_TYPE(sgld->type)) {
1101
0
        case NVME_SGL_DESCR_TYPE_SEGMENT:
1102
0
        case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
1103
0
            break;
1104
0
        default:
1105
0
            return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1106
0
        }
1107
1108
0
        seg_len = le32_to_cpu(sgld->len);
1109
1110
        /* check the length of the (Last) Segment descriptor */
1111
0
        if (!seg_len || seg_len & 0xf) {
1112
0
            return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1113
0
        }
1114
1115
0
        if (UINT64_MAX - addr < seg_len) {
1116
0
            return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1117
0
        }
1118
1119
0
        nsgld = seg_len / sizeof(NvmeSglDescriptor);
1120
1121
0
        while (nsgld > SEG_CHUNK_SIZE) {
1122
0
            if (nvme_addr_read(n, addr, segment, sizeof(segment))) {
1123
0
                trace_pci_nvme_err_addr_read(addr);
1124
0
                status = NVME_DATA_TRAS_ERROR;
1125
0
                goto unmap;
1126
0
            }
1127
1128
0
            status = nvme_map_sgl_data(n, sg, segment, SEG_CHUNK_SIZE,
1129
0
                                       &len, cmd);
1130
0
            if (status) {
1131
0
                goto unmap;
1132
0
            }
1133
1134
0
            nsgld -= SEG_CHUNK_SIZE;
1135
0
            addr += SEG_CHUNK_SIZE * sizeof(NvmeSglDescriptor);
1136
0
        }
1137
1138
0
        ret = nvme_addr_read(n, addr, segment, nsgld *
1139
0
                             sizeof(NvmeSglDescriptor));
1140
0
        if (ret) {
1141
0
            trace_pci_nvme_err_addr_read(addr);
1142
0
            status = NVME_DATA_TRAS_ERROR;
1143
0
            goto unmap;
1144
0
        }
1145
1146
0
        last_sgld = &segment[nsgld - 1];
1147
1148
        /*
1149
         * If the segment ends with a Data Block, then we are done.
1150
         */
1151
0
        if (NVME_SGL_TYPE(last_sgld->type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1152
0
            status = nvme_map_sgl_data(n, sg, segment, nsgld, &len, cmd);
1153
0
            if (status) {
1154
0
                goto unmap;
1155
0
            }
1156
1157
0
            goto out;
1158
0
        }
1159
1160
        /*
1161
         * If the last descriptor was not a Data Block, then the current
1162
         * segment must not be a Last Segment.
1163
         */
1164
0
        if (NVME_SGL_TYPE(sgld->type) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT) {
1165
0
            status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1166
0
            goto unmap;
1167
0
        }
1168
1169
0
        sgld = last_sgld;
1170
0
        addr = le64_to_cpu(sgld->addr);
1171
1172
        /*
1173
         * Do not map the last descriptor; it will be a Segment or Last Segment
1174
         * descriptor and is handled by the next iteration.
1175
         */
1176
0
        status = nvme_map_sgl_data(n, sg, segment, nsgld - 1, &len, cmd);
1177
0
        if (status) {
1178
0
            goto unmap;
1179
0
        }
1180
1181
        /*
1182
         * Reject if this segment made no forward progress. The host should
1183
         * have skipped linking an empty segment. While not strictly spec
1184
         * compliant, allowing this makes it easy for a pathological host to
1185
         * create an infinite loop.
1186
         */
1187
0
        if (len == prev_len) {
1188
0
            status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1189
0
            goto unmap;
1190
0
        }
1191
0
    }
1192
1193
0
out:
1194
    /* if there is any residual left in len, the SGL was too short */
1195
0
    if (len) {
1196
0
        status = NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1197
0
        goto unmap;
1198
0
    }
1199
1200
0
    return NVME_SUCCESS;
1201
1202
0
unmap:
1203
0
    nvme_sg_unmap(sg);
1204
0
    return status;
1205
0
}
1206
1207
uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1208
                       NvmeCmd *cmd)
1209
0
{
1210
0
    uint64_t prp1, prp2;
1211
1212
0
    switch (NVME_CMD_FLAGS_PSDT(cmd->flags)) {
1213
0
    case NVME_PSDT_PRP:
1214
0
        prp1 = le64_to_cpu(cmd->dptr.prp1);
1215
0
        prp2 = le64_to_cpu(cmd->dptr.prp2);
1216
1217
0
        return nvme_map_prp(n, sg, prp1, prp2, len);
1218
0
    case NVME_PSDT_SGL_MPTR_CONTIGUOUS:
1219
0
    case NVME_PSDT_SGL_MPTR_SGL:
1220
0
        return nvme_map_sgl(n, sg, cmd->dptr.sgl, len, cmd);
1221
0
    default:
1222
0
        return NVME_INVALID_FIELD;
1223
0
    }
1224
0
}
1225
1226
static uint16_t nvme_map_mptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1227
                              NvmeCmd *cmd)
1228
0
{
1229
0
    int psdt = NVME_CMD_FLAGS_PSDT(cmd->flags);
1230
0
    hwaddr mptr = le64_to_cpu(cmd->mptr);
1231
0
    uint16_t status;
1232
1233
0
    if (psdt == NVME_PSDT_SGL_MPTR_SGL) {
1234
0
        NvmeSglDescriptor sgl;
1235
1236
0
        if (nvme_addr_read(n, mptr, &sgl, sizeof(sgl))) {
1237
0
            return NVME_DATA_TRAS_ERROR;
1238
0
        }
1239
1240
0
        status = nvme_map_sgl(n, sg, sgl, len, cmd);
1241
0
        if (status && (status & 0x7ff) == NVME_DATA_SGL_LEN_INVALID) {
1242
0
            status = NVME_MD_SGL_LEN_INVALID | NVME_DNR;
1243
0
        }
1244
1245
0
        return status;
1246
0
    }
1247
1248
0
    nvme_sg_init(n, sg, nvme_addr_is_dma(n, mptr));
1249
0
    status = nvme_map_addr(n, sg, mptr, len);
1250
0
    if (status) {
1251
0
        nvme_sg_unmap(sg);
1252
0
    }
1253
1254
0
    return status;
1255
0
}
1256
1257
static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1258
0
{
1259
0
    NvmeNamespace *ns = req->ns;
1260
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1261
0
    bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1262
0
    bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1263
0
    size_t len = nvme_l2b(ns, nlb);
1264
0
    uint16_t status;
1265
1266
0
    if (nvme_ns_ext(ns) &&
1267
0
        !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1268
0
        NvmeSg sg;
1269
1270
0
        len += nvme_m2b(ns, nlb);
1271
1272
0
        status = nvme_map_dptr(n, &sg, len, &req->cmd);
1273
0
        if (status) {
1274
0
            return status;
1275
0
        }
1276
1277
0
        nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1278
0
        nvme_sg_split(&sg, ns, &req->sg, NULL);
1279
0
        nvme_sg_unmap(&sg);
1280
1281
0
        return NVME_SUCCESS;
1282
0
    }
1283
1284
0
    return nvme_map_dptr(n, &req->sg, len, &req->cmd);
1285
0
}
1286
1287
static uint16_t nvme_map_mdata(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1288
0
{
1289
0
    NvmeNamespace *ns = req->ns;
1290
0
    size_t len = nvme_m2b(ns, nlb);
1291
0
    uint16_t status;
1292
1293
0
    if (nvme_ns_ext(ns)) {
1294
0
        NvmeSg sg;
1295
1296
0
        len += nvme_l2b(ns, nlb);
1297
1298
0
        status = nvme_map_dptr(n, &sg, len, &req->cmd);
1299
0
        if (status) {
1300
0
            return status;
1301
0
        }
1302
1303
0
        nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1304
0
        nvme_sg_split(&sg, ns, NULL, &req->sg);
1305
0
        nvme_sg_unmap(&sg);
1306
1307
0
        return NVME_SUCCESS;
1308
0
    }
1309
1310
0
    return nvme_map_mptr(n, &req->sg, len, &req->cmd);
1311
0
}
1312
1313
static uint16_t nvme_tx_interleaved(NvmeCtrl *n, NvmeSg *sg, uint8_t *ptr,
1314
                                    uint32_t len, uint32_t bytes,
1315
                                    int32_t skip_bytes, int64_t offset,
1316
                                    NvmeTxDirection dir)
1317
0
{
1318
0
    hwaddr addr;
1319
0
    uint32_t trans_len, count = bytes;
1320
0
    bool dma = sg->flags & NVME_SG_DMA;
1321
0
    int64_t sge_len;
1322
0
    int sg_idx = 0;
1323
0
    int ret;
1324
1325
0
    assert(sg->flags & NVME_SG_ALLOC);
1326
1327
0
    while (len) {
1328
0
        sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
1329
1330
0
        if (sge_len - offset < 0) {
1331
0
            offset -= sge_len;
1332
0
            sg_idx++;
1333
0
            continue;
1334
0
        }
1335
1336
0
        if (sge_len == offset) {
1337
0
            offset = 0;
1338
0
            sg_idx++;
1339
0
            continue;
1340
0
        }
1341
1342
0
        trans_len = MIN(len, count);
1343
0
        trans_len = MIN(trans_len, sge_len - offset);
1344
1345
0
        if (dma) {
1346
0
            addr = sg->qsg.sg[sg_idx].base + offset;
1347
0
        } else {
1348
0
            addr = (hwaddr)(uintptr_t)sg->iov.iov[sg_idx].iov_base + offset;
1349
0
        }
1350
1351
0
        if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1352
0
            ret = nvme_addr_read(n, addr, ptr, trans_len);
1353
0
        } else {
1354
0
            ret = nvme_addr_write(n, addr, ptr, trans_len);
1355
0
        }
1356
1357
0
        if (ret) {
1358
0
            return NVME_DATA_TRAS_ERROR;
1359
0
        }
1360
1361
0
        ptr += trans_len;
1362
0
        len -= trans_len;
1363
0
        count -= trans_len;
1364
0
        offset += trans_len;
1365
1366
0
        if (count == 0) {
1367
0
            count = bytes;
1368
0
            offset += skip_bytes;
1369
0
        }
1370
0
    }
1371
1372
0
    return NVME_SUCCESS;
1373
0
}
1374
1375
static uint16_t nvme_tx(NvmeCtrl *n, NvmeSg *sg, void *ptr, uint32_t len,
1376
                        NvmeTxDirection dir)
1377
0
{
1378
0
    assert(sg->flags & NVME_SG_ALLOC);
1379
1380
0
    if (sg->flags & NVME_SG_DMA) {
1381
0
        const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
1382
0
        dma_addr_t residual;
1383
1384
0
        if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1385
0
            dma_buf_write(ptr, len, &residual, &sg->qsg, attrs);
1386
0
        } else {
1387
0
            dma_buf_read(ptr, len, &residual, &sg->qsg, attrs);
1388
0
        }
1389
1390
0
        if (unlikely(residual)) {
1391
0
            trace_pci_nvme_err_invalid_dma();
1392
0
            return NVME_INVALID_FIELD | NVME_DNR;
1393
0
        }
1394
0
    } else {
1395
0
        size_t bytes;
1396
1397
0
        if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1398
0
            bytes = qemu_iovec_to_buf(&sg->iov, 0, ptr, len);
1399
0
        } else {
1400
0
            bytes = qemu_iovec_from_buf(&sg->iov, 0, ptr, len);
1401
0
        }
1402
1403
0
        if (unlikely(bytes != len)) {
1404
0
            trace_pci_nvme_err_invalid_dma();
1405
0
            return NVME_INVALID_FIELD | NVME_DNR;
1406
0
        }
1407
0
    }
1408
1409
0
    return NVME_SUCCESS;
1410
0
}
1411
1412
static inline uint16_t nvme_c2h(NvmeCtrl *n, void *ptr, uint32_t len,
1413
                                NvmeRequest *req)
1414
0
{
1415
0
    uint16_t status;
1416
1417
0
    status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1418
0
    if (status) {
1419
0
        return status;
1420
0
    }
1421
1422
0
    return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_FROM_DEVICE);
1423
0
}
1424
1425
static inline uint16_t nvme_h2c(NvmeCtrl *n, void *ptr, uint32_t len,
1426
                                NvmeRequest *req)
1427
0
{
1428
0
    uint16_t status;
1429
1430
0
    status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1431
0
    if (status) {
1432
0
        return status;
1433
0
    }
1434
1435
0
    return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_TO_DEVICE);
1436
0
}
1437
1438
uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
1439
                          NvmeTxDirection dir, NvmeRequest *req)
1440
0
{
1441
0
    NvmeNamespace *ns = req->ns;
1442
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1443
0
    bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1444
0
    bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1445
1446
0
    if (nvme_ns_ext(ns) &&
1447
0
        !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1448
0
        return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbasz,
1449
0
                                   ns->lbaf.ms, 0, dir);
1450
0
    }
1451
1452
0
    return nvme_tx(n, &req->sg, ptr, len, dir);
1453
0
}
1454
1455
uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
1456
                           NvmeTxDirection dir, NvmeRequest *req)
1457
0
{
1458
0
    NvmeNamespace *ns = req->ns;
1459
0
    uint16_t status;
1460
1461
0
    if (nvme_ns_ext(ns)) {
1462
0
        return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbaf.ms,
1463
0
                                   ns->lbasz, ns->lbasz, dir);
1464
0
    }
1465
1466
0
    nvme_sg_unmap(&req->sg);
1467
1468
0
    status = nvme_map_mptr(n, &req->sg, len, &req->cmd);
1469
0
    if (status) {
1470
0
        return status;
1471
0
    }
1472
1473
0
    return nvme_tx(n, &req->sg, ptr, len, dir);
1474
0
}
1475
1476
static inline void nvme_blk_read(BlockBackend *blk, int64_t offset,
1477
                                 uint32_t align, BlockCompletionFunc *cb,
1478
                                 NvmeRequest *req)
1479
0
{
1480
0
    assert(req->sg.flags & NVME_SG_ALLOC);
1481
1482
0
    if (req->sg.flags & NVME_SG_DMA) {
1483
0
        req->aiocb = dma_blk_read(blk, &req->sg.qsg, offset, align, cb, req);
1484
0
    } else {
1485
0
        req->aiocb = blk_aio_preadv(blk, offset, &req->sg.iov, 0, cb, req);
1486
0
    }
1487
0
}
1488
1489
static inline void nvme_blk_write(BlockBackend *blk, int64_t offset,
1490
                                  uint32_t align, BlockCompletionFunc *cb,
1491
                                  NvmeRequest *req)
1492
0
{
1493
0
    assert(req->sg.flags & NVME_SG_ALLOC);
1494
1495
0
    if (req->sg.flags & NVME_SG_DMA) {
1496
0
        req->aiocb = dma_blk_write(blk, &req->sg.qsg, offset, align, cb, req);
1497
0
    } else {
1498
0
        req->aiocb = blk_aio_pwritev(blk, offset, &req->sg.iov, 0, cb, req);
1499
0
    }
1500
0
}
1501
1502
static void nvme_update_cq_eventidx(const NvmeCQueue *cq)
1503
0
{
1504
0
    trace_pci_nvme_update_cq_eventidx(cq->cqid, cq->head);
1505
1506
0
    stl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->ei_addr, cq->head,
1507
0
                   MEMTXATTRS_UNSPECIFIED);
1508
0
}
1509
1510
static void nvme_update_cq_head(NvmeCQueue *cq)
1511
0
{
1512
0
    ldl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->db_addr, &cq->head,
1513
0
                   MEMTXATTRS_UNSPECIFIED);
1514
1515
0
    trace_pci_nvme_update_cq_head(cq->cqid, cq->head);
1516
0
}
1517
1518
static void nvme_post_cqes(void *opaque)
1519
0
{
1520
0
    NvmeCQueue *cq = opaque;
1521
0
    NvmeCtrl *n = cq->ctrl;
1522
0
    NvmeRequest *req, *next;
1523
0
    bool pending = cq->head != cq->tail;
1524
0
    int ret;
1525
1526
0
    QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
1527
0
        NvmeSQueue *sq;
1528
0
        hwaddr addr;
1529
1530
0
        if (n->dbbuf_enabled) {
1531
0
            nvme_update_cq_eventidx(cq);
1532
0
            nvme_update_cq_head(cq);
1533
0
        }
1534
1535
0
        if (nvme_cq_full(cq)) {
1536
0
            break;
1537
0
        }
1538
1539
        /*
1540
         * Here we take the following fields from NvmeRequest structure
1541
         * and write cqe to the guest RAM based on them:
1542
         * - req->sq
1543
         * - req->status
1544
         * - req->cqe
1545
         *
1546
         * If you change this code and more fields from NvmeRequest are
1547
         * used, please make sure that you have handled this in:
1548
         * nvme_vmstate_request and nvme_ctrl_pre_save().
1549
         */
1550
1551
0
        sq = req->sq;
1552
0
        req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
1553
0
        req->cqe.sq_head = cpu_to_le16(sq->head);
1554
0
        addr = cq->dma_addr + (cq->tail << NVME_CQES);
1555
0
        ret = pci_dma_write(PCI_DEVICE(n), addr, (void *)&req->cqe,
1556
0
                            sizeof(req->cqe));
1557
0
        if (ret) {
1558
0
            trace_pci_nvme_err_addr_write(addr);
1559
0
            trace_pci_nvme_err_cfs();
1560
0
            stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
1561
0
            break;
1562
0
        }
1563
1564
0
        QTAILQ_REMOVE(&cq->req_list, req, entry);
1565
1566
0
        nvme_inc_cq_tail(cq);
1567
1568
0
        if (QTAILQ_EMPTY(&sq->req_list) && !nvme_sq_empty(sq)) {
1569
0
            qemu_bh_schedule(sq->bh);
1570
0
        }
1571
1572
0
        QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
1573
0
    }
1574
0
    if (cq->tail != cq->head) {
1575
0
        if (cq->irq_enabled && !pending) {
1576
0
            n->cq_pending++;
1577
0
        }
1578
1579
0
        nvme_irq_assert(n, cq);
1580
0
    }
1581
0
}
1582
1583
static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
1584
0
{
1585
0
    assert(cq->cqid == req->sq->cqid);
1586
0
    trace_pci_nvme_enqueue_req_completion(nvme_cid(req), cq->cqid,
1587
0
                                          le32_to_cpu(req->cqe.result),
1588
0
                                          le32_to_cpu(req->cqe.dw1),
1589
0
                                          req->status);
1590
1591
0
    if (req->status) {
1592
0
        trace_pci_nvme_err_req_status(nvme_cid(req), nvme_nsid(req->ns),
1593
0
                                      req->status, req->cmd.opcode);
1594
0
    }
1595
1596
0
    nvme_sg_unmap(&req->sg);
1597
1598
0
    QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
1599
0
    QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
1600
1601
0
    qemu_bh_schedule(cq->bh);
1602
0
}
1603
1604
static void nvme_process_aers(void *opaque)
1605
0
{
1606
0
    NvmeCtrl *n = opaque;
1607
0
    NvmeAsyncEvent *event, *next;
1608
1609
0
    trace_pci_nvme_process_aers(n->aer_queued);
1610
1611
0
    QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
1612
0
        NvmeRequest *req;
1613
0
        NvmeAerResult *result;
1614
1615
        /* can't post cqe if there is nothing to complete */
1616
0
        if (!n->outstanding_aers) {
1617
0
            trace_pci_nvme_no_outstanding_aers();
1618
0
            break;
1619
0
        }
1620
1621
        /* ignore if masked (cqe posted, but event not cleared) */
1622
0
        if (n->aer_mask & (1 << event->result.event_type)) {
1623
0
            trace_pci_nvme_aer_masked(event->result.event_type, n->aer_mask);
1624
0
            continue;
1625
0
        }
1626
1627
0
        QTAILQ_REMOVE(&n->aer_queue, event, entry);
1628
0
        n->aer_queued--;
1629
1630
0
        n->aer_mask |= 1 << event->result.event_type;
1631
0
        n->outstanding_aers--;
1632
1633
0
        req = n->aer_reqs[n->outstanding_aers];
1634
1635
0
        result = (NvmeAerResult *) &req->cqe.result;
1636
0
        result->event_type = event->result.event_type;
1637
0
        result->event_info = event->result.event_info;
1638
0
        result->log_page = event->result.log_page;
1639
0
        g_free(event);
1640
1641
0
        trace_pci_nvme_aer_post_cqe(result->event_type, result->event_info,
1642
0
                                    result->log_page);
1643
1644
0
        nvme_enqueue_req_completion(&n->admin_cq, req);
1645
0
    }
1646
0
}
1647
1648
static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type,
1649
                               uint8_t event_info, uint8_t log_page)
1650
0
{
1651
0
    NvmeAsyncEvent *event;
1652
1653
0
    trace_pci_nvme_enqueue_event(event_type, event_info, log_page);
1654
1655
0
    if (n->aer_queued == n->params.aer_max_queued) {
1656
0
        trace_pci_nvme_enqueue_event_noqueue(n->aer_queued);
1657
0
        return;
1658
0
    }
1659
1660
0
    event = g_new(NvmeAsyncEvent, 1);
1661
0
    event->result = (NvmeAerResult) {
1662
0
        .event_type = event_type,
1663
0
        .event_info = event_info,
1664
0
        .log_page   = log_page,
1665
0
    };
1666
1667
0
    QTAILQ_INSERT_TAIL(&n->aer_queue, event, entry);
1668
0
    n->aer_queued++;
1669
1670
0
    nvme_process_aers(n);
1671
0
}
1672
1673
static void nvme_smart_event(NvmeCtrl *n, uint8_t event)
1674
0
{
1675
0
    uint8_t aer_info;
1676
1677
    /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */
1678
0
    if (!(NVME_AEC_SMART(n->features.async_config) & event)) {
1679
0
        return;
1680
0
    }
1681
1682
0
    switch (event) {
1683
0
    case NVME_SMART_SPARE:
1684
0
        aer_info = NVME_AER_INFO_SMART_SPARE_THRESH;
1685
0
        break;
1686
0
    case NVME_SMART_TEMPERATURE:
1687
0
        aer_info = NVME_AER_INFO_SMART_TEMP_THRESH;
1688
0
        break;
1689
0
    case NVME_SMART_RELIABILITY:
1690
0
    case NVME_SMART_MEDIA_READ_ONLY:
1691
0
    case NVME_SMART_FAILED_VOLATILE_MEDIA:
1692
0
    case NVME_SMART_PMR_UNRELIABLE:
1693
0
        aer_info = NVME_AER_INFO_SMART_RELIABILITY;
1694
0
        break;
1695
0
    default:
1696
0
        return;
1697
0
    }
1698
1699
0
    nvme_enqueue_event(n, NVME_AER_TYPE_SMART, aer_info, NVME_LOG_SMART_INFO);
1700
0
}
1701
1702
static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type)
1703
0
{
1704
0
    NvmeAsyncEvent *event, *next;
1705
1706
0
    n->aer_mask &= ~(1 << event_type);
1707
1708
0
    QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
1709
0
        if (event->result.event_type == event_type) {
1710
0
            QTAILQ_REMOVE(&n->aer_queue, event, entry);
1711
0
            n->aer_queued--;
1712
0
            g_free(event);
1713
0
        }
1714
0
    }
1715
0
}
1716
1717
static inline uint16_t nvme_check_mdts(NvmeCtrl *n, size_t len)
1718
0
{
1719
0
    uint8_t mdts = n->params.mdts;
1720
1721
0
    if (mdts && len > n->page_size << mdts) {
1722
0
        trace_pci_nvme_err_mdts(len);
1723
0
        return NVME_INVALID_FIELD | NVME_DNR;
1724
0
    }
1725
1726
0
    return NVME_SUCCESS;
1727
0
}
1728
1729
static inline uint16_t nvme_check_bounds(NvmeNamespace *ns, uint64_t slba,
1730
                                         uint32_t nlb)
1731
0
{
1732
0
    uint64_t nsze = le64_to_cpu(ns->id_ns.nsze);
1733
1734
0
    if (unlikely(UINT64_MAX - slba < nlb || slba + nlb > nsze)) {
1735
0
        trace_pci_nvme_err_invalid_lba_range(slba, nlb, nsze);
1736
0
        return NVME_LBA_RANGE | NVME_DNR;
1737
0
    }
1738
1739
0
    return NVME_SUCCESS;
1740
0
}
1741
1742
static int nvme_block_status_all(NvmeNamespace *ns, uint64_t slba,
1743
                                 uint32_t nlb, int flags)
1744
0
{
1745
0
    BlockDriverState *bs = blk_bs(ns->blkconf.blk);
1746
1747
0
    int64_t pnum = 0, bytes = nvme_l2b(ns, nlb);
1748
0
    int64_t offset = nvme_l2b(ns, slba);
1749
0
    int ret;
1750
1751
    /*
1752
     * `pnum` holds the number of bytes after offset that shares the same
1753
     * allocation status as the byte at offset. If `pnum` is different from
1754
     * `bytes`, we should check the allocation status of the next range and
1755
     * continue this until all bytes have been checked.
1756
     */
1757
0
    do {
1758
0
        bytes -= pnum;
1759
1760
0
        ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL);
1761
0
        if (ret < 0) {
1762
0
            return ret;
1763
0
        }
1764
1765
1766
0
        trace_pci_nvme_block_status(offset, bytes, pnum, ret,
1767
0
                                    !!(ret & BDRV_BLOCK_ZERO));
1768
1769
0
        if (!(ret & flags)) {
1770
0
            return 1;
1771
0
        }
1772
1773
0
        offset += pnum;
1774
0
    } while (pnum != bytes);
1775
1776
0
    return 0;
1777
0
}
1778
1779
static uint16_t nvme_check_dulbe(NvmeNamespace *ns, uint64_t slba,
1780
                                 uint32_t nlb)
1781
0
{
1782
0
    int ret;
1783
0
    Error *err = NULL;
1784
1785
0
    ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_DATA);
1786
0
    if (ret) {
1787
0
        if (ret < 0) {
1788
0
            error_setg_errno(&err, -ret, "unable to get block status");
1789
0
            error_report_err(err);
1790
1791
0
            return NVME_INTERNAL_DEV_ERROR;
1792
0
        }
1793
1794
0
        return NVME_DULB;
1795
0
    }
1796
1797
0
    return NVME_SUCCESS;
1798
0
}
1799
1800
static inline uint32_t nvme_zone_idx(NvmeNamespace *ns, uint64_t slba)
1801
0
{
1802
0
    return ns->zone_size_log2 > 0 ? slba >> ns->zone_size_log2 :
1803
0
                                    slba / ns->zone_size;
1804
0
}
1805
1806
static inline NvmeZone *nvme_get_zone_by_slba(NvmeNamespace *ns, uint64_t slba)
1807
0
{
1808
0
    uint32_t zone_idx = nvme_zone_idx(ns, slba);
1809
1810
0
    if (zone_idx >= ns->num_zones) {
1811
0
        return NULL;
1812
0
    }
1813
1814
0
    return &ns->zone_array[zone_idx];
1815
0
}
1816
1817
static uint16_t nvme_check_zone_state_for_write(NvmeZone *zone)
1818
0
{
1819
0
    uint64_t zslba = zone->d.zslba;
1820
1821
0
    switch (nvme_get_zone_state(zone)) {
1822
0
    case NVME_ZONE_STATE_EMPTY:
1823
0
    case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1824
0
    case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1825
0
    case NVME_ZONE_STATE_CLOSED:
1826
0
        return NVME_SUCCESS;
1827
0
    case NVME_ZONE_STATE_FULL:
1828
0
        trace_pci_nvme_err_zone_is_full(zslba);
1829
0
        return NVME_ZONE_FULL;
1830
0
    case NVME_ZONE_STATE_OFFLINE:
1831
0
        trace_pci_nvme_err_zone_is_offline(zslba);
1832
0
        return NVME_ZONE_OFFLINE;
1833
0
    case NVME_ZONE_STATE_READ_ONLY:
1834
0
        trace_pci_nvme_err_zone_is_read_only(zslba);
1835
0
        return NVME_ZONE_READ_ONLY;
1836
0
    default:
1837
0
        g_assert_not_reached();
1838
0
    }
1839
1840
0
    return NVME_INTERNAL_DEV_ERROR;
1841
0
}
1842
1843
static uint16_t nvme_check_zone_write(NvmeNamespace *ns, NvmeZone *zone,
1844
                                      uint64_t slba, uint32_t nlb)
1845
0
{
1846
0
    uint64_t zcap = nvme_zone_wr_boundary(zone);
1847
0
    uint16_t status;
1848
1849
0
    status = nvme_check_zone_state_for_write(zone);
1850
0
    if (status) {
1851
0
        return status;
1852
0
    }
1853
1854
0
    if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1855
0
        uint64_t ezrwa = zone->w_ptr + 2 * ns->zns.zrwas;
1856
1857
0
        if (slba < zone->w_ptr || slba + nlb > ezrwa) {
1858
0
            trace_pci_nvme_err_zone_invalid_write(slba, zone->w_ptr);
1859
0
            return NVME_ZONE_INVALID_WRITE;
1860
0
        }
1861
0
    } else {
1862
0
        if (unlikely(slba != zone->w_ptr)) {
1863
0
            trace_pci_nvme_err_write_not_at_wp(slba, zone->d.zslba,
1864
0
                                               zone->w_ptr);
1865
0
            return NVME_ZONE_INVALID_WRITE;
1866
0
        }
1867
0
    }
1868
1869
0
    if (unlikely((slba + nlb) > zcap)) {
1870
0
        trace_pci_nvme_err_zone_boundary(slba, nlb, zcap);
1871
0
        return NVME_ZONE_BOUNDARY_ERROR;
1872
0
    }
1873
1874
0
    return NVME_SUCCESS;
1875
0
}
1876
1877
static uint16_t nvme_check_zone_state_for_read(NvmeZone *zone)
1878
0
{
1879
0
    switch (nvme_get_zone_state(zone)) {
1880
0
    case NVME_ZONE_STATE_EMPTY:
1881
0
    case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1882
0
    case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1883
0
    case NVME_ZONE_STATE_FULL:
1884
0
    case NVME_ZONE_STATE_CLOSED:
1885
0
    case NVME_ZONE_STATE_READ_ONLY:
1886
0
        return NVME_SUCCESS;
1887
0
    case NVME_ZONE_STATE_OFFLINE:
1888
0
        trace_pci_nvme_err_zone_is_offline(zone->d.zslba);
1889
0
        return NVME_ZONE_OFFLINE;
1890
0
    default:
1891
0
        g_assert_not_reached();
1892
0
    }
1893
1894
0
    return NVME_INTERNAL_DEV_ERROR;
1895
0
}
1896
1897
static uint16_t nvme_check_zone_read(NvmeNamespace *ns, uint64_t slba,
1898
                                     uint32_t nlb)
1899
0
{
1900
0
    NvmeZone *zone;
1901
0
    uint64_t bndry, end;
1902
0
    uint16_t status;
1903
1904
0
    zone = nvme_get_zone_by_slba(ns, slba);
1905
0
    assert(zone);
1906
1907
0
    bndry = nvme_zone_rd_boundary(ns, zone);
1908
0
    end = slba + nlb;
1909
1910
0
    status = nvme_check_zone_state_for_read(zone);
1911
0
    if (status) {
1912
0
        ;
1913
0
    } else if (unlikely(end > bndry)) {
1914
0
        if (!ns->params.cross_zone_read) {
1915
0
            status = NVME_ZONE_BOUNDARY_ERROR;
1916
0
        } else {
1917
            /*
1918
             * Read across zone boundary - check that all subsequent
1919
             * zones that are being read have an appropriate state.
1920
             */
1921
0
            do {
1922
0
                zone++;
1923
0
                status = nvme_check_zone_state_for_read(zone);
1924
0
                if (status) {
1925
0
                    break;
1926
0
                }
1927
0
            } while (end > nvme_zone_rd_boundary(ns, zone));
1928
0
        }
1929
0
    }
1930
1931
0
    return status;
1932
0
}
1933
1934
static uint16_t nvme_zrm_finish(NvmeNamespace *ns, NvmeZone *zone)
1935
0
{
1936
0
    switch (nvme_get_zone_state(zone)) {
1937
0
    case NVME_ZONE_STATE_FULL:
1938
0
        return NVME_SUCCESS;
1939
1940
0
    case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1941
0
    case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1942
0
        nvme_aor_dec_open(ns);
1943
        /* fallthrough */
1944
0
    case NVME_ZONE_STATE_CLOSED:
1945
0
        nvme_aor_dec_active(ns);
1946
1947
0
        if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1948
0
            zone->d.za &= ~NVME_ZA_ZRWA_VALID;
1949
0
            if (ns->params.numzrwa) {
1950
0
                ns->zns.numzrwa++;
1951
0
            }
1952
0
        }
1953
1954
        /* fallthrough */
1955
0
    case NVME_ZONE_STATE_EMPTY:
1956
0
        nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_FULL);
1957
0
        return NVME_SUCCESS;
1958
1959
0
    default:
1960
0
        return NVME_ZONE_INVAL_TRANSITION;
1961
0
    }
1962
0
}
1963
1964
static uint16_t nvme_zrm_close(NvmeNamespace *ns, NvmeZone *zone)
1965
0
{
1966
0
    switch (nvme_get_zone_state(zone)) {
1967
0
    case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1968
0
    case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1969
0
        nvme_aor_dec_open(ns);
1970
0
        nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
1971
        /* fall through */
1972
0
    case NVME_ZONE_STATE_CLOSED:
1973
0
        return NVME_SUCCESS;
1974
1975
0
    default:
1976
0
        return NVME_ZONE_INVAL_TRANSITION;
1977
0
    }
1978
0
}
1979
1980
static uint16_t nvme_zrm_reset(NvmeNamespace *ns, NvmeZone *zone)
1981
0
{
1982
0
    switch (nvme_get_zone_state(zone)) {
1983
0
    case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1984
0
    case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1985
0
        nvme_aor_dec_open(ns);
1986
        /* fallthrough */
1987
0
    case NVME_ZONE_STATE_CLOSED:
1988
0
        nvme_aor_dec_active(ns);
1989
1990
0
        if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1991
0
            if (ns->params.numzrwa) {
1992
0
                ns->zns.numzrwa++;
1993
0
            }
1994
0
        }
1995
1996
        /* fallthrough */
1997
0
    case NVME_ZONE_STATE_FULL:
1998
0
        zone->w_ptr = zone->d.zslba;
1999
0
        zone->d.wp = zone->w_ptr;
2000
0
        nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EMPTY);
2001
        /* fallthrough */
2002
0
    case NVME_ZONE_STATE_EMPTY:
2003
0
        return NVME_SUCCESS;
2004
2005
0
    default:
2006
0
        return NVME_ZONE_INVAL_TRANSITION;
2007
0
    }
2008
0
}
2009
2010
static void nvme_zrm_auto_transition_zone(NvmeNamespace *ns)
2011
0
{
2012
0
    NvmeZone *zone;
2013
2014
0
    if (ns->params.max_open_zones &&
2015
0
        ns->nr_open_zones == ns->params.max_open_zones) {
2016
0
        zone = QTAILQ_FIRST(&ns->imp_open_zones);
2017
0
        if (zone) {
2018
            /*
2019
             * Automatically close this implicitly open zone.
2020
             */
2021
0
            QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
2022
0
            nvme_zrm_close(ns, zone);
2023
0
        }
2024
0
    }
2025
0
}
2026
2027
enum {
2028
    NVME_ZRM_AUTO = 1 << 0,
2029
    NVME_ZRM_ZRWA = 1 << 1,
2030
};
2031
2032
static uint16_t nvme_zrm_open_flags(NvmeCtrl *n, NvmeNamespace *ns,
2033
                                    NvmeZone *zone, int flags)
2034
0
{
2035
0
    int act = 0;
2036
0
    uint16_t status;
2037
2038
0
    switch (nvme_get_zone_state(zone)) {
2039
0
    case NVME_ZONE_STATE_EMPTY:
2040
0
        act = 1;
2041
2042
        /* fallthrough */
2043
2044
0
    case NVME_ZONE_STATE_CLOSED:
2045
0
        if (n->params.auto_transition_zones) {
2046
0
            nvme_zrm_auto_transition_zone(ns);
2047
0
        }
2048
0
        status = nvme_zns_check_resources(ns, act, 1,
2049
0
                                          (flags & NVME_ZRM_ZRWA) ? 1 : 0);
2050
0
        if (status) {
2051
0
            return status;
2052
0
        }
2053
2054
0
        if (act) {
2055
0
            nvme_aor_inc_active(ns);
2056
0
        }
2057
2058
0
        nvme_aor_inc_open(ns);
2059
2060
0
        if (flags & NVME_ZRM_AUTO) {
2061
0
            nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_IMPLICITLY_OPEN);
2062
0
            return NVME_SUCCESS;
2063
0
        }
2064
2065
        /* fallthrough */
2066
2067
0
    case NVME_ZONE_STATE_IMPLICITLY_OPEN:
2068
0
        if (flags & NVME_ZRM_AUTO) {
2069
0
            return NVME_SUCCESS;
2070
0
        }
2071
2072
0
        nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EXPLICITLY_OPEN);
2073
2074
        /* fallthrough */
2075
2076
0
    case NVME_ZONE_STATE_EXPLICITLY_OPEN:
2077
0
        if (flags & NVME_ZRM_ZRWA) {
2078
0
            ns->zns.numzrwa--;
2079
2080
0
            zone->d.za |= NVME_ZA_ZRWA_VALID;
2081
0
        }
2082
2083
0
        return NVME_SUCCESS;
2084
2085
0
    default:
2086
0
        return NVME_ZONE_INVAL_TRANSITION;
2087
0
    }
2088
0
}
2089
2090
static inline uint16_t nvme_zrm_auto(NvmeCtrl *n, NvmeNamespace *ns,
2091
                                     NvmeZone *zone)
2092
0
{
2093
0
    return nvme_zrm_open_flags(n, ns, zone, NVME_ZRM_AUTO);
2094
0
}
2095
2096
static void nvme_advance_zone_wp(NvmeNamespace *ns, NvmeZone *zone,
2097
                                 uint32_t nlb)
2098
0
{
2099
0
    zone->d.wp += nlb;
2100
2101
0
    if (zone->d.wp == nvme_zone_wr_boundary(zone)) {
2102
0
        nvme_zrm_finish(ns, zone);
2103
0
    }
2104
0
}
2105
2106
static void nvme_zoned_zrwa_implicit_flush(NvmeNamespace *ns, NvmeZone *zone,
2107
                                           uint32_t nlbc)
2108
0
{
2109
0
    uint16_t nzrwafgs = DIV_ROUND_UP(nlbc, ns->zns.zrwafg);
2110
2111
0
    nlbc = nzrwafgs * ns->zns.zrwafg;
2112
2113
0
    trace_pci_nvme_zoned_zrwa_implicit_flush(zone->d.zslba, nlbc);
2114
2115
0
    zone->w_ptr += nlbc;
2116
2117
0
    nvme_advance_zone_wp(ns, zone, nlbc);
2118
0
}
2119
2120
static void nvme_finalize_zoned_write(NvmeNamespace *ns, NvmeRequest *req)
2121
0
{
2122
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2123
0
    NvmeZone *zone;
2124
0
    uint64_t slba;
2125
0
    uint32_t nlb;
2126
2127
0
    slba = le64_to_cpu(rw->slba);
2128
0
    nlb = le16_to_cpu(rw->nlb) + 1;
2129
0
    zone = nvme_get_zone_by_slba(ns, slba);
2130
0
    assert(zone);
2131
2132
0
    if (zone->d.za & NVME_ZA_ZRWA_VALID) {
2133
0
        uint64_t ezrwa = zone->w_ptr + ns->zns.zrwas - 1;
2134
0
        uint64_t elba = slba + nlb - 1;
2135
2136
0
        if (elba > ezrwa) {
2137
0
            nvme_zoned_zrwa_implicit_flush(ns, zone, elba - ezrwa);
2138
0
        }
2139
2140
0
        return;
2141
0
    }
2142
2143
0
    nvme_advance_zone_wp(ns, zone, nlb);
2144
0
}
2145
2146
static inline bool nvme_is_write(NvmeRequest *req)
2147
0
{
2148
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2149
2150
0
    return rw->opcode == NVME_CMD_WRITE ||
2151
0
           rw->opcode == NVME_CMD_ZONE_APPEND ||
2152
0
           rw->opcode == NVME_CMD_WRITE_ZEROES;
2153
0
}
2154
2155
static void nvme_misc_cb(void *opaque, int ret)
2156
0
{
2157
0
    NvmeRequest *req = opaque;
2158
0
    uint16_t cid = nvme_cid(req);
2159
2160
0
    trace_pci_nvme_misc_cb(cid);
2161
2162
0
    if (ret) {
2163
0
        if (!req->status) {
2164
0
            req->status = NVME_INTERNAL_DEV_ERROR;
2165
0
        }
2166
2167
0
        trace_pci_nvme_err_aio(cid, strerror(-ret), req->status);
2168
0
    }
2169
2170
0
    nvme_enqueue_req_completion(nvme_cq(req), req);
2171
0
}
2172
2173
void nvme_rw_complete_cb(void *opaque, int ret)
2174
0
{
2175
0
    NvmeRequest *req = opaque;
2176
0
    NvmeNamespace *ns = req->ns;
2177
0
    BlockBackend *blk = ns->blkconf.blk;
2178
0
    BlockAcctCookie *acct = &req->acct;
2179
0
    BlockAcctStats *stats = blk_get_stats(blk);
2180
2181
0
    trace_pci_nvme_rw_complete_cb(nvme_cid(req), blk_name(blk));
2182
2183
0
    if (ret) {
2184
0
        Error *err = NULL;
2185
2186
0
        block_acct_failed(stats, acct);
2187
2188
0
        switch (req->cmd.opcode) {
2189
0
        case NVME_CMD_READ:
2190
0
            req->status = NVME_UNRECOVERED_READ;
2191
0
            break;
2192
2193
0
        case NVME_CMD_WRITE:
2194
0
        case NVME_CMD_WRITE_ZEROES:
2195
0
        case NVME_CMD_ZONE_APPEND:
2196
0
            req->status = NVME_WRITE_FAULT;
2197
0
            break;
2198
2199
0
        default:
2200
0
            req->status = NVME_INTERNAL_DEV_ERROR;
2201
0
            break;
2202
0
        }
2203
2204
0
        trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status);
2205
2206
0
        error_setg_errno(&err, -ret, "aio failed");
2207
0
        error_report_err(err);
2208
0
    } else {
2209
0
        block_acct_done(stats, acct);
2210
0
    }
2211
2212
0
    if (ns->params.zoned && nvme_is_write(req)) {
2213
0
        nvme_finalize_zoned_write(ns, req);
2214
0
    }
2215
2216
0
    nvme_enqueue_req_completion(nvme_cq(req), req);
2217
0
}
2218
2219
static void nvme_rw_cb(void *opaque, int ret)
2220
0
{
2221
0
    NvmeRequest *req = opaque;
2222
0
    NvmeNamespace *ns = req->ns;
2223
2224
0
    BlockBackend *blk = ns->blkconf.blk;
2225
2226
0
    trace_pci_nvme_rw_cb(nvme_cid(req), blk_name(blk));
2227
2228
0
    if (ret) {
2229
0
        goto out;
2230
0
    }
2231
2232
0
    if (ns->lbaf.ms) {
2233
0
        NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2234
0
        uint64_t slba = le64_to_cpu(rw->slba);
2235
0
        uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
2236
0
        uint64_t offset = nvme_moff(ns, slba);
2237
2238
0
        if (req->cmd.opcode == NVME_CMD_WRITE_ZEROES) {
2239
0
            size_t mlen = nvme_m2b(ns, nlb);
2240
2241
0
            req->aiocb = blk_aio_pwrite_zeroes(blk, offset, mlen,
2242
0
                                               BDRV_REQ_MAY_UNMAP,
2243
0
                                               nvme_rw_complete_cb, req);
2244
0
            return;
2245
0
        }
2246
2247
0
        if (nvme_ns_ext(ns) || req->cmd.mptr) {
2248
0
            uint16_t status;
2249
2250
0
            nvme_sg_unmap(&req->sg);
2251
0
            status = nvme_map_mdata(nvme_ctrl(req), nlb, req);
2252
0
            if (status) {
2253
0
                ret = -EFAULT;
2254
0
                goto out;
2255
0
            }
2256
2257
0
            if (req->cmd.opcode == NVME_CMD_READ) {
2258
0
                return nvme_blk_read(blk, offset, 1, nvme_rw_complete_cb, req);
2259
0
            }
2260
2261
0
            return nvme_blk_write(blk, offset, 1, nvme_rw_complete_cb, req);
2262
0
        }
2263
0
    }
2264
2265
0
out:
2266
0
    nvme_rw_complete_cb(req, ret);
2267
0
}
2268
2269
static void nvme_verify_cb(void *opaque, int ret)
2270
0
{
2271
0
    NvmeBounceContext *ctx = opaque;
2272
0
    NvmeRequest *req = ctx->req;
2273
0
    NvmeNamespace *ns = req->ns;
2274
0
    BlockBackend *blk = ns->blkconf.blk;
2275
0
    BlockAcctCookie *acct = &req->acct;
2276
0
    BlockAcctStats *stats = blk_get_stats(blk);
2277
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2278
0
    uint64_t slba = le64_to_cpu(rw->slba);
2279
0
    uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2280
0
    uint16_t apptag = le16_to_cpu(rw->apptag);
2281
0
    uint16_t appmask = le16_to_cpu(rw->appmask);
2282
0
    uint64_t reftag = le32_to_cpu(rw->reftag);
2283
0
    uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2284
0
    uint16_t status;
2285
2286
0
    reftag |= cdw3 << 32;
2287
2288
0
    trace_pci_nvme_verify_cb(nvme_cid(req), prinfo, apptag, appmask, reftag);
2289
2290
0
    if (ret) {
2291
0
        block_acct_failed(stats, acct);
2292
0
        req->status = NVME_UNRECOVERED_READ;
2293
2294
0
        trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status);
2295
2296
0
        goto out;
2297
0
    }
2298
2299
0
    block_acct_done(stats, acct);
2300
2301
0
    if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2302
0
        status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce,
2303
0
                                       ctx->mdata.iov.size, slba);
2304
0
        if (status) {
2305
0
            req->status = status;
2306
0
            goto out;
2307
0
        }
2308
2309
0
        req->status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2310
0
                                     ctx->mdata.bounce, ctx->mdata.iov.size,
2311
0
                                     prinfo, slba, apptag, appmask, &reftag);
2312
0
    }
2313
2314
0
out:
2315
0
    qemu_iovec_destroy(&ctx->data.iov);
2316
0
    g_free(ctx->data.bounce);
2317
2318
0
    qemu_iovec_destroy(&ctx->mdata.iov);
2319
0
    g_free(ctx->mdata.bounce);
2320
2321
0
    g_free(ctx);
2322
2323
0
    nvme_enqueue_req_completion(nvme_cq(req), req);
2324
0
}
2325
2326
2327
static void nvme_verify_mdata_in_cb(void *opaque, int ret)
2328
0
{
2329
0
    NvmeBounceContext *ctx = opaque;
2330
0
    NvmeRequest *req = ctx->req;
2331
0
    NvmeNamespace *ns = req->ns;
2332
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2333
0
    uint64_t slba = le64_to_cpu(rw->slba);
2334
0
    uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2335
0
    size_t mlen = nvme_m2b(ns, nlb);
2336
0
    uint64_t offset = nvme_moff(ns, slba);
2337
0
    BlockBackend *blk = ns->blkconf.blk;
2338
2339
0
    trace_pci_nvme_verify_mdata_in_cb(nvme_cid(req), blk_name(blk));
2340
2341
0
    if (ret) {
2342
0
        goto out;
2343
0
    }
2344
2345
0
    ctx->mdata.bounce = g_malloc(mlen);
2346
2347
0
    qemu_iovec_reset(&ctx->mdata.iov);
2348
0
    qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2349
2350
0
    req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2351
0
                                nvme_verify_cb, ctx);
2352
0
    return;
2353
2354
0
out:
2355
0
    nvme_verify_cb(ctx, ret);
2356
0
}
2357
2358
struct nvme_compare_ctx {
2359
    struct {
2360
        QEMUIOVector iov;
2361
        uint8_t *bounce;
2362
    } data;
2363
2364
    struct {
2365
        QEMUIOVector iov;
2366
        uint8_t *bounce;
2367
    } mdata;
2368
};
2369
2370
static void nvme_compare_mdata_cb(void *opaque, int ret)
2371
0
{
2372
0
    NvmeRequest *req = opaque;
2373
0
    NvmeNamespace *ns = req->ns;
2374
0
    NvmeCtrl *n = nvme_ctrl(req);
2375
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2376
0
    uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2377
0
    uint16_t apptag = le16_to_cpu(rw->apptag);
2378
0
    uint16_t appmask = le16_to_cpu(rw->appmask);
2379
0
    uint64_t reftag = le32_to_cpu(rw->reftag);
2380
0
    uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2381
0
    struct nvme_compare_ctx *ctx = req->opaque;
2382
0
    g_autofree uint8_t *buf = NULL;
2383
0
    BlockBackend *blk = ns->blkconf.blk;
2384
0
    BlockAcctCookie *acct = &req->acct;
2385
0
    BlockAcctStats *stats = blk_get_stats(blk);
2386
0
    uint16_t status = NVME_SUCCESS;
2387
2388
0
    reftag |= cdw3 << 32;
2389
2390
0
    trace_pci_nvme_compare_mdata_cb(nvme_cid(req));
2391
2392
0
    if (ret) {
2393
0
        req->status = NVME_UNRECOVERED_READ;
2394
2395
0
        trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status);
2396
2397
0
        goto out;
2398
0
    }
2399
2400
0
    buf = g_malloc(ctx->mdata.iov.size);
2401
2402
0
    status = nvme_bounce_mdata(n, buf, ctx->mdata.iov.size,
2403
0
                               NVME_TX_DIRECTION_TO_DEVICE, req);
2404
0
    if (status) {
2405
0
        req->status = status;
2406
0
        goto out;
2407
0
    }
2408
2409
0
    if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2410
0
        uint64_t slba = le64_to_cpu(rw->slba);
2411
0
        uint8_t *bufp;
2412
0
        uint8_t *mbufp = ctx->mdata.bounce;
2413
0
        uint8_t *end = mbufp + ctx->mdata.iov.size;
2414
0
        int16_t pil = 0;
2415
2416
0
        status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2417
0
                                ctx->mdata.bounce, ctx->mdata.iov.size, prinfo,
2418
0
                                slba, apptag, appmask, &reftag);
2419
0
        if (status) {
2420
0
            req->status = status;
2421
0
            goto out;
2422
0
        }
2423
2424
        /*
2425
         * When formatted with protection information, do not compare the DIF
2426
         * tuple.
2427
         */
2428
0
        if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
2429
0
            pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
2430
0
        }
2431
2432
0
        for (bufp = buf; mbufp < end; bufp += ns->lbaf.ms, mbufp += ns->lbaf.ms) {
2433
0
            if (memcmp(bufp + pil, mbufp + pil, ns->lbaf.ms - pil)) {
2434
0
                req->status = NVME_CMP_FAILURE | NVME_DNR;
2435
0
                goto out;
2436
0
            }
2437
0
        }
2438
2439
0
        goto out;
2440
0
    }
2441
2442
0
    if (memcmp(buf, ctx->mdata.bounce, ctx->mdata.iov.size)) {
2443
0
        req->status = NVME_CMP_FAILURE | NVME_DNR;
2444
0
        goto out;
2445
0
    }
2446
2447
2448
0
out:
2449
0
    if (req->status == NVME_SUCCESS) {
2450
0
        block_acct_done(stats, acct);
2451
0
    } else {
2452
0
        block_acct_failed(stats, acct);
2453
0
    }
2454
2455
0
    qemu_iovec_destroy(&ctx->data.iov);
2456
0
    g_free(ctx->data.bounce);
2457
2458
0
    qemu_iovec_destroy(&ctx->mdata.iov);
2459
0
    g_free(ctx->mdata.bounce);
2460
2461
0
    g_free(ctx);
2462
2463
0
    nvme_enqueue_req_completion(nvme_cq(req), req);
2464
0
}
2465
2466
static void nvme_compare_data_cb(void *opaque, int ret)
2467
0
{
2468
0
    NvmeRequest *req = opaque;
2469
0
    NvmeCtrl *n = nvme_ctrl(req);
2470
0
    NvmeNamespace *ns = req->ns;
2471
0
    BlockBackend *blk = ns->blkconf.blk;
2472
0
    BlockAcctCookie *acct = &req->acct;
2473
0
    BlockAcctStats *stats = blk_get_stats(blk);
2474
2475
0
    struct nvme_compare_ctx *ctx = req->opaque;
2476
0
    g_autofree uint8_t *buf = NULL;
2477
0
    uint16_t status;
2478
2479
0
    trace_pci_nvme_compare_data_cb(nvme_cid(req));
2480
2481
0
    if (ret) {
2482
0
        req->status = NVME_UNRECOVERED_READ;
2483
2484
0
        trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status);
2485
2486
0
        goto out;
2487
0
    }
2488
2489
0
    buf = g_malloc(ctx->data.iov.size);
2490
2491
0
    status = nvme_bounce_data(n, buf, ctx->data.iov.size,
2492
0
                              NVME_TX_DIRECTION_TO_DEVICE, req);
2493
0
    if (status) {
2494
0
        req->status = status;
2495
0
        goto out;
2496
0
    }
2497
2498
0
    if (memcmp(buf, ctx->data.bounce, ctx->data.iov.size)) {
2499
0
        req->status = NVME_CMP_FAILURE | NVME_DNR;
2500
0
        goto out;
2501
0
    }
2502
2503
0
    if (ns->lbaf.ms) {
2504
0
        NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2505
0
        uint64_t slba = le64_to_cpu(rw->slba);
2506
0
        uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2507
0
        size_t mlen = nvme_m2b(ns, nlb);
2508
0
        uint64_t offset = nvme_moff(ns, slba);
2509
2510
0
        ctx->mdata.bounce = g_malloc(mlen);
2511
2512
0
        qemu_iovec_init(&ctx->mdata.iov, 1);
2513
0
        qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2514
2515
0
        req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2516
0
                                    nvme_compare_mdata_cb, req);
2517
0
        return;
2518
0
    }
2519
2520
0
out:
2521
0
    if (req->status == NVME_SUCCESS) {
2522
0
        block_acct_done(stats, acct);
2523
0
    } else {
2524
0
        block_acct_failed(stats, acct);
2525
0
    }
2526
2527
0
    qemu_iovec_destroy(&ctx->data.iov);
2528
0
    g_free(ctx->data.bounce);
2529
0
    g_free(ctx);
2530
2531
0
    nvme_enqueue_req_completion(nvme_cq(req), req);
2532
0
}
2533
2534
typedef struct NvmeDSMAIOCB {
2535
    BlockAIOCB common;
2536
    BlockAIOCB *aiocb;
2537
    NvmeRequest *req;
2538
    int ret;
2539
2540
    NvmeDsmRange *range;
2541
    unsigned int nr;
2542
    unsigned int idx;
2543
} NvmeDSMAIOCB;
2544
2545
static void nvme_dsm_cancel(BlockAIOCB *aiocb)
2546
0
{
2547
0
    NvmeDSMAIOCB *iocb = container_of(aiocb, NvmeDSMAIOCB, common);
2548
2549
    /* break nvme_dsm_cb loop */
2550
0
    iocb->idx = iocb->nr;
2551
0
    iocb->ret = -ECANCELED;
2552
2553
0
    if (iocb->aiocb) {
2554
0
        blk_aio_cancel_async(iocb->aiocb);
2555
0
        iocb->aiocb = NULL;
2556
0
    } else {
2557
        /*
2558
         * We only reach this if nvme_dsm_cancel() has already been called or
2559
         * the command ran to completion.
2560
         */
2561
0
        assert(iocb->idx == iocb->nr);
2562
0
    }
2563
0
}
2564
2565
static const AIOCBInfo nvme_dsm_aiocb_info = {
2566
    .aiocb_size   = sizeof(NvmeDSMAIOCB),
2567
    .cancel_async = nvme_dsm_cancel,
2568
};
2569
2570
static void nvme_dsm_cb(void *opaque, int ret);
2571
2572
static void nvme_dsm_md_cb(void *opaque, int ret)
2573
0
{
2574
0
    NvmeDSMAIOCB *iocb = opaque;
2575
0
    NvmeRequest *req = iocb->req;
2576
0
    NvmeNamespace *ns = req->ns;
2577
0
    NvmeDsmRange *range;
2578
0
    uint64_t slba;
2579
0
    uint32_t nlb;
2580
2581
0
    if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2582
0
        goto done;
2583
0
    }
2584
2585
0
    range = &iocb->range[iocb->idx - 1];
2586
0
    slba = le64_to_cpu(range->slba);
2587
0
    nlb = le32_to_cpu(range->nlb);
2588
2589
    /*
2590
     * Check that all block were discarded (zeroed); otherwise we do not zero
2591
     * the metadata.
2592
     */
2593
2594
0
    ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_ZERO);
2595
0
    if (ret) {
2596
0
        if (ret < 0) {
2597
0
            goto done;
2598
0
        }
2599
2600
0
        nvme_dsm_cb(iocb, 0);
2601
0
        return;
2602
0
    }
2603
2604
0
    iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, nvme_moff(ns, slba),
2605
0
                                        nvme_m2b(ns, nlb), BDRV_REQ_MAY_UNMAP,
2606
0
                                        nvme_dsm_cb, iocb);
2607
0
    return;
2608
2609
0
done:
2610
0
    nvme_dsm_cb(iocb, ret);
2611
0
}
2612
2613
static void nvme_dsm_cb(void *opaque, int ret)
2614
0
{
2615
0
    NvmeDSMAIOCB *iocb = opaque;
2616
0
    NvmeRequest *req = iocb->req;
2617
0
    NvmeCtrl *n = nvme_ctrl(req);
2618
0
    NvmeNamespace *ns = req->ns;
2619
0
    NvmeDsmRange *range;
2620
0
    uint64_t slba;
2621
0
    uint32_t nlb;
2622
2623
0
    if (iocb->ret < 0) {
2624
0
        goto done;
2625
0
    } else if (ret < 0) {
2626
0
        iocb->ret = ret;
2627
0
        goto done;
2628
0
    }
2629
2630
0
next:
2631
0
    if (iocb->idx == iocb->nr) {
2632
0
        goto done;
2633
0
    }
2634
2635
0
    range = &iocb->range[iocb->idx++];
2636
0
    slba = le64_to_cpu(range->slba);
2637
0
    nlb = le32_to_cpu(range->nlb);
2638
2639
0
    trace_pci_nvme_dsm_deallocate(slba, nlb);
2640
2641
0
    if (nlb > n->dmrsl) {
2642
0
        trace_pci_nvme_dsm_single_range_limit_exceeded(nlb, n->dmrsl);
2643
0
        goto next;
2644
0
    }
2645
2646
0
    if (nvme_check_bounds(ns, slba, nlb)) {
2647
0
        trace_pci_nvme_err_invalid_lba_range(slba, nlb,
2648
0
                                             ns->id_ns.nsze);
2649
0
        goto next;
2650
0
    }
2651
2652
0
    iocb->aiocb = blk_aio_pdiscard(ns->blkconf.blk, nvme_l2b(ns, slba),
2653
0
                                   nvme_l2b(ns, nlb),
2654
0
                                   nvme_dsm_md_cb, iocb);
2655
0
    return;
2656
2657
0
done:
2658
0
    iocb->aiocb = NULL;
2659
0
    iocb->common.cb(iocb->common.opaque, iocb->ret);
2660
0
    g_free(iocb->range);
2661
0
    qemu_aio_unref(iocb);
2662
0
}
2663
2664
static uint16_t nvme_dsm(NvmeCtrl *n, NvmeRequest *req)
2665
0
{
2666
0
    NvmeNamespace *ns = req->ns;
2667
0
    NvmeDsmCmd *dsm = (NvmeDsmCmd *) &req->cmd;
2668
0
    uint32_t attr = le32_to_cpu(dsm->attributes);
2669
0
    uint32_t nr = (le32_to_cpu(dsm->nr) & 0xff) + 1;
2670
0
    uint16_t status = NVME_SUCCESS;
2671
2672
0
    trace_pci_nvme_dsm(nr, attr);
2673
2674
0
    if (attr & NVME_DSMGMT_AD) {
2675
0
        NvmeDSMAIOCB *iocb = blk_aio_get(&nvme_dsm_aiocb_info, ns->blkconf.blk,
2676
0
                                         nvme_misc_cb, req);
2677
2678
0
        iocb->req = req;
2679
0
        iocb->ret = 0;
2680
0
        iocb->range = g_new(NvmeDsmRange, nr);
2681
0
        iocb->nr = nr;
2682
0
        iocb->idx = 0;
2683
2684
0
        status = nvme_h2c(n, (uint8_t *)iocb->range, sizeof(NvmeDsmRange) * nr,
2685
0
                          req);
2686
0
        if (status) {
2687
0
            g_free(iocb->range);
2688
0
            qemu_aio_unref(iocb);
2689
2690
0
            return status;
2691
0
        }
2692
2693
0
        req->aiocb = &iocb->common;
2694
0
        nvme_dsm_cb(iocb, 0);
2695
2696
0
        return NVME_NO_COMPLETE;
2697
0
    }
2698
2699
0
    return status;
2700
0
}
2701
2702
static uint16_t nvme_verify(NvmeCtrl *n, NvmeRequest *req)
2703
0
{
2704
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2705
0
    NvmeNamespace *ns = req->ns;
2706
0
    BlockBackend *blk = ns->blkconf.blk;
2707
0
    uint64_t slba = le64_to_cpu(rw->slba);
2708
0
    uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2709
0
    size_t len = nvme_l2b(ns, nlb);
2710
0
    size_t data_len = len;
2711
0
    int64_t offset = nvme_l2b(ns, slba);
2712
0
    uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2713
0
    uint32_t reftag = le32_to_cpu(rw->reftag);
2714
0
    NvmeBounceContext *ctx = NULL;
2715
0
    uint16_t status;
2716
2717
0
    trace_pci_nvme_verify(nvme_cid(req), nvme_nsid(ns), slba, nlb);
2718
2719
0
    if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2720
0
        status = nvme_check_prinfo(ns, prinfo, slba, reftag);
2721
0
        if (status) {
2722
0
            return status;
2723
0
        }
2724
2725
0
        if (prinfo & NVME_PRINFO_PRACT) {
2726
0
            return NVME_INVALID_PROT_INFO | NVME_DNR;
2727
0
        }
2728
0
    }
2729
2730
0
    if (nvme_ns_ext(ns) && !(NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt))) {
2731
0
        data_len += nvme_m2b(ns, nlb);
2732
0
    }
2733
2734
0
    if (data_len > (n->page_size << n->params.vsl)) {
2735
0
        return NVME_INVALID_FIELD | NVME_DNR;
2736
0
    }
2737
2738
0
    status = nvme_check_bounds(ns, slba, nlb);
2739
0
    if (status) {
2740
0
        return status;
2741
0
    }
2742
2743
0
    if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
2744
0
        status = nvme_check_dulbe(ns, slba, nlb);
2745
0
        if (status) {
2746
0
            return status;
2747
0
        }
2748
0
    }
2749
2750
0
    ctx = g_new0(NvmeBounceContext, 1);
2751
0
    ctx->req = req;
2752
2753
0
    ctx->data.bounce = g_malloc(len);
2754
2755
0
    qemu_iovec_init(&ctx->data.iov, 1);
2756
0
    qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len);
2757
2758
0
    block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size,
2759
0
                     BLOCK_ACCT_READ);
2760
2761
0
    req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0,
2762
0
                                nvme_verify_mdata_in_cb, ctx);
2763
0
    return NVME_NO_COMPLETE;
2764
0
}
2765
2766
typedef struct NvmeCopyAIOCB {
2767
    BlockAIOCB common;
2768
    BlockAIOCB *aiocb;
2769
    NvmeRequest *req;
2770
    NvmeCtrl *n;
2771
    int ret;
2772
2773
    void *ranges;
2774
    unsigned int format;
2775
    int nr;
2776
    int idx;
2777
2778
    uint8_t *bounce;
2779
    QEMUIOVector iov;
2780
    struct {
2781
        BlockAcctCookie read;
2782
        BlockAcctCookie write;
2783
    } acct;
2784
2785
    uint64_t reftag;
2786
    uint64_t slba;
2787
2788
    NvmeZone *zone;
2789
    NvmeNamespace *sns;
2790
    uint32_t tcl;
2791
} NvmeCopyAIOCB;
2792
2793
static void nvme_copy_cancel(BlockAIOCB *aiocb)
2794
0
{
2795
0
    NvmeCopyAIOCB *iocb = container_of(aiocb, NvmeCopyAIOCB, common);
2796
2797
0
    iocb->ret = -ECANCELED;
2798
2799
0
    if (iocb->aiocb) {
2800
0
        blk_aio_cancel_async(iocb->aiocb);
2801
0
        iocb->aiocb = NULL;
2802
0
    }
2803
0
}
2804
2805
static const AIOCBInfo nvme_copy_aiocb_info = {
2806
    .aiocb_size   = sizeof(NvmeCopyAIOCB),
2807
    .cancel_async = nvme_copy_cancel,
2808
};
2809
2810
static void nvme_copy_done(NvmeCopyAIOCB *iocb)
2811
0
{
2812
0
    NvmeRequest *req = iocb->req;
2813
2814
0
    if (iocb->idx != iocb->nr) {
2815
0
        req->cqe.result = cpu_to_le32(iocb->idx);
2816
0
    }
2817
2818
0
    qemu_iovec_destroy(&iocb->iov);
2819
0
    g_free(iocb->bounce);
2820
0
    g_free(iocb->ranges);
2821
2822
0
    iocb->common.cb(iocb->common.opaque, iocb->ret);
2823
0
    qemu_aio_unref(iocb);
2824
0
}
2825
2826
static void nvme_do_copy(NvmeCopyAIOCB *iocb);
2827
2828
static void nvme_copy_source_range_parse_format0_2(void *ranges,
2829
                                                   int idx, uint64_t *slba,
2830
                                                   uint32_t *nlb,
2831
                                                   uint32_t *snsid,
2832
                                                   uint16_t *apptag,
2833
                                                   uint16_t *appmask,
2834
                                                   uint64_t *reftag)
2835
0
{
2836
0
    NvmeCopySourceRangeFormat0_2 *_ranges = ranges;
2837
2838
0
    if (snsid) {
2839
0
        *snsid = le32_to_cpu(_ranges[idx].sparams);
2840
0
    }
2841
2842
0
    if (slba) {
2843
0
        *slba = le64_to_cpu(_ranges[idx].slba);
2844
0
    }
2845
2846
0
    if (nlb) {
2847
0
        *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2848
0
    }
2849
2850
0
    if (apptag) {
2851
0
        *apptag = le16_to_cpu(_ranges[idx].apptag);
2852
0
    }
2853
2854
0
    if (appmask) {
2855
0
        *appmask = le16_to_cpu(_ranges[idx].appmask);
2856
0
    }
2857
2858
0
    if (reftag) {
2859
0
        *reftag = le32_to_cpu(_ranges[idx].reftag);
2860
0
    }
2861
0
}
2862
2863
static void nvme_copy_source_range_parse_format1_3(void *ranges, int idx,
2864
                                                   uint64_t *slba,
2865
                                                   uint32_t *nlb,
2866
                                                   uint32_t *snsid,
2867
                                                   uint16_t *apptag,
2868
                                                   uint16_t *appmask,
2869
                                                   uint64_t *reftag)
2870
0
{
2871
0
    NvmeCopySourceRangeFormat1_3 *_ranges = ranges;
2872
2873
0
    if (snsid) {
2874
0
        *snsid = le32_to_cpu(_ranges[idx].sparams);
2875
0
    }
2876
2877
0
    if (slba) {
2878
0
        *slba = le64_to_cpu(_ranges[idx].slba);
2879
0
    }
2880
2881
0
    if (nlb) {
2882
0
        *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2883
0
    }
2884
2885
0
    if (apptag) {
2886
0
        *apptag = le16_to_cpu(_ranges[idx].apptag);
2887
0
    }
2888
2889
0
    if (appmask) {
2890
0
        *appmask = le16_to_cpu(_ranges[idx].appmask);
2891
0
    }
2892
2893
0
    if (reftag) {
2894
0
        *reftag = 0;
2895
2896
0
        *reftag |= (uint64_t)_ranges[idx].sr[4] << 40;
2897
0
        *reftag |= (uint64_t)_ranges[idx].sr[5] << 32;
2898
0
        *reftag |= (uint64_t)_ranges[idx].sr[6] << 24;
2899
0
        *reftag |= (uint64_t)_ranges[idx].sr[7] << 16;
2900
0
        *reftag |= (uint64_t)_ranges[idx].sr[8] << 8;
2901
0
        *reftag |= (uint64_t)_ranges[idx].sr[9];
2902
0
    }
2903
0
}
2904
2905
static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format,
2906
                                         uint64_t *slba, uint32_t *nlb,
2907
                                         uint32_t *snsid, uint16_t *apptag,
2908
                                         uint16_t *appmask, uint64_t *reftag)
2909
0
{
2910
0
    switch (format) {
2911
0
    case NVME_COPY_FORMAT_0:
2912
0
    case NVME_COPY_FORMAT_2:
2913
0
        nvme_copy_source_range_parse_format0_2(ranges, idx, slba, nlb, snsid,
2914
0
                                               apptag, appmask, reftag);
2915
0
        break;
2916
2917
0
    case NVME_COPY_FORMAT_1:
2918
0
    case NVME_COPY_FORMAT_3:
2919
0
        nvme_copy_source_range_parse_format1_3(ranges, idx, slba, nlb, snsid,
2920
0
                                               apptag, appmask, reftag);
2921
0
        break;
2922
2923
0
    default:
2924
0
        abort();
2925
0
    }
2926
0
}
2927
2928
static inline uint16_t nvme_check_copy_mcl(NvmeNamespace *ns,
2929
                                           NvmeCopyAIOCB *iocb, uint16_t nr)
2930
0
{
2931
0
    uint32_t copy_len = 0;
2932
2933
0
    for (int idx = 0; idx < nr; idx++) {
2934
0
        uint32_t nlb;
2935
0
        nvme_copy_source_range_parse(iocb->ranges, idx, iocb->format, NULL,
2936
0
                                     &nlb, NULL, NULL, NULL, NULL);
2937
0
        copy_len += nlb;
2938
0
    }
2939
0
    iocb->tcl = copy_len;
2940
0
    if (copy_len > ns->id_ns.mcl) {
2941
0
        return NVME_CMD_SIZE_LIMIT | NVME_DNR;
2942
0
    }
2943
2944
0
    return NVME_SUCCESS;
2945
0
}
2946
2947
static void nvme_copy_out_completed_cb(void *opaque, int ret)
2948
0
{
2949
0
    NvmeCopyAIOCB *iocb = opaque;
2950
0
    NvmeRequest *req = iocb->req;
2951
0
    NvmeNamespace *dns = req->ns;
2952
0
    BlockAcctStats *stats = blk_get_stats(dns->blkconf.blk);
2953
0
    uint32_t nlb;
2954
2955
0
    nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2956
0
                                 &nlb, NULL, NULL, NULL, NULL);
2957
2958
0
    if (ret < 0) {
2959
0
        iocb->ret = ret;
2960
0
        req->status = NVME_WRITE_FAULT;
2961
0
    }
2962
0
    if (iocb->ret < 0) {
2963
0
        block_acct_failed(stats, &iocb->acct.write);
2964
0
        goto out;
2965
0
    }
2966
0
    block_acct_done(stats, &iocb->acct.write);
2967
2968
0
    if (dns->params.zoned) {
2969
0
        nvme_advance_zone_wp(dns, iocb->zone, nlb);
2970
0
    }
2971
2972
0
    iocb->idx++;
2973
0
    iocb->slba += nlb;
2974
0
out:
2975
0
    nvme_do_copy(iocb);
2976
0
}
2977
2978
static void nvme_copy_out_cb(void *opaque, int ret)
2979
0
{
2980
0
    NvmeCopyAIOCB *iocb = opaque;
2981
0
    NvmeRequest *req = iocb->req;
2982
0
    NvmeNamespace *dns = req->ns;
2983
0
    BlockAcctStats *stats = blk_get_stats(dns->blkconf.blk);
2984
0
    uint32_t nlb;
2985
0
    size_t mlen;
2986
0
    uint8_t *mbounce;
2987
2988
0
    if (ret < 0 || iocb->ret < 0) {
2989
0
        block_acct_failed(stats, &iocb->acct.write);
2990
0
        goto out;
2991
0
    }
2992
0
    block_acct_done(stats, &iocb->acct.write);
2993
2994
0
    if (!dns->lbaf.ms) {
2995
0
        goto out;
2996
0
    }
2997
2998
0
    nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2999
0
                                 &nlb, NULL, NULL, NULL, NULL);
3000
3001
0
    mlen = nvme_m2b(dns, nlb);
3002
0
    mbounce = iocb->bounce + nvme_l2b(dns, nlb);
3003
3004
0
    qemu_iovec_reset(&iocb->iov);
3005
0
    qemu_iovec_add(&iocb->iov, mbounce, mlen);
3006
3007
0
    block_acct_start(stats, &iocb->acct.write, mlen, BLOCK_ACCT_WRITE);
3008
0
    iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_moff(dns, iocb->slba),
3009
0
                                  &iocb->iov, 0, nvme_copy_out_completed_cb,
3010
0
                                  iocb);
3011
3012
0
    return;
3013
3014
0
out:
3015
0
    nvme_copy_out_completed_cb(iocb, ret);
3016
0
}
3017
3018
static void nvme_copy_in_completed_cb(void *opaque, int ret)
3019
0
{
3020
0
    NvmeCopyAIOCB *iocb = opaque;
3021
0
    NvmeRequest *req = iocb->req;
3022
0
    NvmeNamespace *sns = iocb->sns;
3023
0
    NvmeNamespace *dns = req->ns;
3024
0
    BlockAcctStats *sstats = blk_get_stats(sns->blkconf.blk);
3025
0
    NvmeCopyCmd *copy = NULL;
3026
0
    uint8_t *mbounce = NULL;
3027
0
    uint32_t nlb;
3028
0
    uint64_t slba;
3029
0
    uint16_t apptag, appmask;
3030
0
    uint64_t reftag;
3031
0
    size_t len, mlen;
3032
0
    uint16_t status;
3033
3034
0
    if (ret < 0) {
3035
0
        iocb->ret = ret;
3036
0
        req->status = NVME_UNRECOVERED_READ;
3037
0
    }
3038
0
    if (iocb->ret < 0) {
3039
0
        block_acct_failed(sstats, &iocb->acct.read);
3040
0
        goto out;
3041
0
    }
3042
0
    block_acct_done(sstats, &iocb->acct.read);
3043
3044
0
    nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3045
0
                                 &nlb, NULL, &apptag, &appmask, &reftag);
3046
3047
0
    trace_pci_nvme_copy_out(iocb->slba, nlb);
3048
3049
0
    len = nvme_l2b(sns, nlb);
3050
3051
0
    if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps)) {
3052
0
        copy = (NvmeCopyCmd *)&req->cmd;
3053
3054
0
        uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
3055
3056
0
        mlen = nvme_m2b(sns, nlb);
3057
0
        mbounce = iocb->bounce + nvme_l2b(sns, nlb);
3058
3059
0
        status = nvme_dif_mangle_mdata(sns, mbounce, mlen, slba);
3060
0
        if (status) {
3061
0
            goto invalid;
3062
0
        }
3063
0
        status = nvme_dif_check(sns, iocb->bounce, len, mbounce, mlen, prinfor,
3064
0
                                slba, apptag, appmask, &reftag);
3065
0
        if (status) {
3066
0
            goto invalid;
3067
0
        }
3068
0
    }
3069
3070
0
    if (NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3071
0
        copy = (NvmeCopyCmd *)&req->cmd;
3072
0
        uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
3073
3074
0
        mlen = nvme_m2b(dns, nlb);
3075
0
        mbounce = iocb->bounce + nvme_l2b(dns, nlb);
3076
3077
0
        apptag = le16_to_cpu(copy->apptag);
3078
0
        appmask = le16_to_cpu(copy->appmask);
3079
3080
0
        if (prinfow & NVME_PRINFO_PRACT) {
3081
0
            status = nvme_check_prinfo(dns, prinfow, iocb->slba, iocb->reftag);
3082
0
            if (status) {
3083
0
                goto invalid;
3084
0
            }
3085
3086
0
            nvme_dif_pract_generate_dif(dns, iocb->bounce, len, mbounce, mlen,
3087
0
                                        apptag, &iocb->reftag);
3088
0
        } else {
3089
0
            status = nvme_dif_check(dns, iocb->bounce, len, mbounce, mlen,
3090
0
                                    prinfow, iocb->slba, apptag, appmask,
3091
0
                                    &iocb->reftag);
3092
0
            if (status) {
3093
0
                goto invalid;
3094
0
            }
3095
0
        }
3096
0
    }
3097
3098
0
    status = nvme_check_bounds(dns, iocb->slba, nlb);
3099
0
    if (status) {
3100
0
        goto invalid;
3101
0
    }
3102
3103
0
    if (dns->params.zoned) {
3104
0
        status = nvme_check_zone_write(dns, iocb->zone, iocb->slba, nlb);
3105
0
        if (status) {
3106
0
            goto invalid;
3107
0
        }
3108
3109
0
        if (!(iocb->zone->d.za & NVME_ZA_ZRWA_VALID)) {
3110
0
            iocb->zone->w_ptr += nlb;
3111
0
        }
3112
0
    }
3113
3114
0
    qemu_iovec_reset(&iocb->iov);
3115
0
    qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3116
3117
0
    block_acct_start(blk_get_stats(dns->blkconf.blk), &iocb->acct.write, len,
3118
0
                     BLOCK_ACCT_WRITE);
3119
3120
0
    iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_l2b(dns, iocb->slba),
3121
0
                                  &iocb->iov, 0, nvme_copy_out_cb, iocb);
3122
3123
0
    return;
3124
3125
0
invalid:
3126
0
    req->status = status;
3127
0
    iocb->ret = -1;
3128
0
out:
3129
0
    nvme_do_copy(iocb);
3130
0
}
3131
3132
static void nvme_copy_in_cb(void *opaque, int ret)
3133
0
{
3134
0
    NvmeCopyAIOCB *iocb = opaque;
3135
0
    NvmeNamespace *sns = iocb->sns;
3136
0
    BlockAcctStats *stats = blk_get_stats(sns->blkconf.blk);
3137
0
    uint64_t slba;
3138
0
    uint32_t nlb;
3139
0
    size_t mlen;
3140
3141
0
    if (ret < 0 || iocb->ret < 0) {
3142
0
        block_acct_failed(stats, &iocb->acct.read);
3143
0
        goto out;
3144
0
    }
3145
0
    block_acct_done(stats, &iocb->acct.read);
3146
3147
0
    if (!sns->lbaf.ms) {
3148
0
        goto out;
3149
0
    }
3150
3151
0
    nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3152
0
                                 &nlb, NULL, NULL, NULL, NULL);
3153
3154
0
    mlen = nvme_m2b(sns, nlb);
3155
0
    qemu_iovec_reset(&iocb->iov);
3156
0
    qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(sns, nlb), mlen);
3157
3158
0
    block_acct_start(stats, &iocb->acct.read, mlen, BLOCK_ACCT_READ);
3159
0
    iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_moff(sns, slba),
3160
0
                                 &iocb->iov, 0, nvme_copy_in_completed_cb,
3161
0
                                 iocb);
3162
0
    return;
3163
3164
0
out:
3165
0
    nvme_copy_in_completed_cb(iocb, ret);
3166
0
}
3167
3168
static inline bool nvme_csi_supports_copy(uint8_t csi)
3169
0
{
3170
0
    return csi == NVME_CSI_NVM || csi == NVME_CSI_ZONED;
3171
0
}
3172
3173
static inline bool nvme_copy_ns_format_match(NvmeNamespace *sns,
3174
                                             NvmeNamespace *dns)
3175
0
{
3176
0
    return sns->lbaf.ds == dns->lbaf.ds && sns->lbaf.ms == dns->lbaf.ms;
3177
0
}
3178
3179
static bool nvme_copy_matching_ns_format(NvmeNamespace *sns, NvmeNamespace *dns,
3180
                                         bool pi_enable)
3181
0
{
3182
0
    if (!nvme_csi_supports_copy(sns->csi) ||
3183
0
        !nvme_csi_supports_copy(dns->csi)) {
3184
0
        return false;
3185
0
    }
3186
3187
0
    if (!pi_enable && !nvme_copy_ns_format_match(sns, dns)) {
3188
0
            return false;
3189
0
    }
3190
3191
0
    if (pi_enable && (!nvme_copy_ns_format_match(sns, dns) ||
3192
0
        sns->id_ns.dps != dns->id_ns.dps)) {
3193
0
            return false;
3194
0
    }
3195
3196
0
    return true;
3197
0
}
3198
3199
static inline bool nvme_copy_corresp_pi_match(NvmeNamespace *sns,
3200
                                              NvmeNamespace *dns)
3201
0
{
3202
0
    return sns->lbaf.ms == 0 &&
3203
0
           ((dns->lbaf.ms == 8 && dns->pif == 0) ||
3204
0
           (dns->lbaf.ms == 16 && dns->pif == 1));
3205
0
}
3206
3207
static bool nvme_copy_corresp_pi_format(NvmeNamespace *sns, NvmeNamespace *dns,
3208
                                        bool sns_pi_en)
3209
0
{
3210
0
    if (!nvme_csi_supports_copy(sns->csi) ||
3211
0
        !nvme_csi_supports_copy(dns->csi)) {
3212
0
        return false;
3213
0
    }
3214
3215
0
    if (!sns_pi_en && !nvme_copy_corresp_pi_match(sns, dns)) {
3216
0
        return false;
3217
0
    }
3218
3219
0
    if (sns_pi_en && !nvme_copy_corresp_pi_match(dns, sns)) {
3220
0
        return false;
3221
0
    }
3222
3223
0
    return true;
3224
0
}
3225
3226
static void nvme_do_copy(NvmeCopyAIOCB *iocb)
3227
0
{
3228
0
    NvmeRequest *req = iocb->req;
3229
0
    NvmeNamespace *sns;
3230
0
    NvmeNamespace *dns = req->ns;
3231
0
    NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3232
0
    uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
3233
0
    uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
3234
0
    uint64_t slba;
3235
0
    uint32_t nlb;
3236
0
    size_t len, blen;
3237
0
    uint16_t status;
3238
0
    uint32_t dnsid = le32_to_cpu(req->cmd.nsid);
3239
0
    uint32_t snsid = dnsid;
3240
3241
0
    if (iocb->ret < 0) {
3242
0
        goto done;
3243
0
    }
3244
3245
0
    if (iocb->idx == iocb->nr) {
3246
0
        goto done;
3247
0
    }
3248
3249
0
    if (iocb->format == 2 || iocb->format == 3) {
3250
0
        nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format,
3251
0
                                     &slba, &nlb, &snsid, NULL, NULL, NULL);
3252
0
        if (snsid != dnsid) {
3253
0
            if (snsid == NVME_NSID_BROADCAST ||
3254
0
                !nvme_nsid_valid(iocb->n, snsid)) {
3255
0
                status = NVME_INVALID_NSID | NVME_DNR;
3256
0
                goto invalid;
3257
0
            }
3258
0
            iocb->sns = nvme_ns(iocb->n, snsid);
3259
0
            if (unlikely(!iocb->sns)) {
3260
0
                status = NVME_INVALID_FIELD | NVME_DNR;
3261
0
                goto invalid;
3262
0
            }
3263
0
        } else {
3264
0
            if (((slba + nlb) > iocb->slba) &&
3265
0
                ((slba + nlb) < (iocb->slba + iocb->tcl))) {
3266
0
                status = NVME_CMD_OVERLAP_IO_RANGE | NVME_DNR;
3267
0
                goto invalid;
3268
0
            }
3269
0
        }
3270
0
    } else {
3271
0
        nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format,
3272
0
                                     &slba, &nlb, NULL, NULL, NULL, NULL);
3273
0
    }
3274
3275
0
    sns = iocb->sns;
3276
0
    if ((snsid == dnsid) && NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3277
0
        ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) {
3278
0
        status = NVME_INVALID_FIELD | NVME_DNR;
3279
0
        goto invalid;
3280
0
    } else if (snsid != dnsid) {
3281
0
        if (sns->lbaf.ds != dns->lbaf.ds) {
3282
0
            status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3283
0
            goto invalid;
3284
0
        }
3285
3286
0
        if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3287
0
            !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3288
0
            if (!nvme_copy_matching_ns_format(sns, dns, false)) {
3289
0
                status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3290
0
                goto invalid;
3291
0
            }
3292
0
        }
3293
0
        if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3294
0
            NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3295
0
            if ((prinfor & NVME_PRINFO_PRACT) !=
3296
0
                (prinfow & NVME_PRINFO_PRACT)) {
3297
0
                status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3298
0
                goto invalid;
3299
0
            } else {
3300
0
                if (!nvme_copy_matching_ns_format(sns, dns, true)) {
3301
0
                    status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3302
0
                    goto invalid;
3303
0
                }
3304
0
            }
3305
0
        }
3306
3307
0
        if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3308
0
            NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3309
0
            if (!(prinfow & NVME_PRINFO_PRACT)) {
3310
0
                status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3311
0
                goto invalid;
3312
0
            } else {
3313
0
                if (!nvme_copy_corresp_pi_format(sns, dns, false)) {
3314
0
                    status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3315
0
                    goto invalid;
3316
0
                }
3317
0
            }
3318
0
        }
3319
3320
0
        if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3321
0
            !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3322
0
            if (!(prinfor & NVME_PRINFO_PRACT)) {
3323
0
                status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3324
0
                goto invalid;
3325
0
            } else {
3326
0
                if (!nvme_copy_corresp_pi_format(sns, dns, true)) {
3327
0
                    status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3328
0
                    goto invalid;
3329
0
                }
3330
0
            }
3331
0
        }
3332
0
    }
3333
0
    len = nvme_l2b(sns, nlb);
3334
3335
0
    trace_pci_nvme_copy_source_range(slba, nlb);
3336
3337
0
    if (nlb > le16_to_cpu(sns->id_ns.mssrl)) {
3338
0
        status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3339
0
        goto invalid;
3340
0
    }
3341
3342
0
    status = nvme_check_bounds(sns, slba, nlb);
3343
0
    if (status) {
3344
0
        goto invalid;
3345
0
    }
3346
3347
0
    if (NVME_ERR_REC_DULBE(sns->features.err_rec)) {
3348
0
        status = nvme_check_dulbe(sns, slba, nlb);
3349
0
        if (status) {
3350
0
            goto invalid;
3351
0
        }
3352
0
    }
3353
3354
0
    if (sns->params.zoned) {
3355
0
        status = nvme_check_zone_read(sns, slba, nlb);
3356
0
        if (status) {
3357
0
            goto invalid;
3358
0
        }
3359
0
    }
3360
3361
0
    g_free(iocb->bounce);
3362
0
    assert(g_size_checked_mul(&blen, le16_to_cpu(sns->id_ns.mssrl),
3363
0
                              sns->lbasz + MAX(sns->lbaf.ms, dns->lbaf.ms)));
3364
3365
0
    iocb->bounce = g_malloc(blen);
3366
3367
0
    qemu_iovec_reset(&iocb->iov);
3368
0
    assert(len <= blen);
3369
0
    qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3370
3371
0
    block_acct_start(blk_get_stats(sns->blkconf.blk), &iocb->acct.read, len,
3372
0
                     BLOCK_ACCT_READ);
3373
3374
0
    iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_l2b(sns, slba),
3375
0
                                 &iocb->iov, 0, nvme_copy_in_cb, iocb);
3376
0
    return;
3377
3378
0
invalid:
3379
0
    req->status = status;
3380
0
    iocb->ret = -1;
3381
0
done:
3382
0
    nvme_copy_done(iocb);
3383
0
}
3384
3385
static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req)
3386
0
{
3387
0
    NvmeNamespace *ns = req->ns;
3388
0
    NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3389
0
    NvmeCopyAIOCB *iocb = blk_aio_get(&nvme_copy_aiocb_info, ns->blkconf.blk,
3390
0
                                      nvme_misc_cb, req);
3391
0
    uint16_t nr = copy->nr + 1;
3392
0
    uint8_t format = copy->control[0] & 0xf;
3393
0
    size_t len = sizeof(NvmeCopySourceRangeFormat0_2);
3394
3395
0
    uint16_t status;
3396
3397
0
    trace_pci_nvme_copy(nvme_cid(req), nvme_nsid(ns), nr, format);
3398
3399
0
    iocb->ranges = NULL;
3400
0
    iocb->zone = NULL;
3401
3402
0
    if (!(n->id_ctrl.ocfs & (1 << format)) ||
3403
0
        ((format == 2 || format == 3) &&
3404
0
         !(n->features.hbs.cdfe & (1 << format)))) {
3405
0
        trace_pci_nvme_err_copy_invalid_format(format);
3406
0
        status = NVME_INVALID_FIELD | NVME_DNR;
3407
0
        goto invalid;
3408
0
    }
3409
3410
0
    if (nr > ns->id_ns.msrc + 1) {
3411
0
        status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3412
0
        goto invalid;
3413
0
    }
3414
3415
0
    if ((ns->pif == 0x0 && (format != 0x0 && format != 0x2)) ||
3416
0
        (ns->pif != 0x0 && (format != 0x1 && format != 0x3))) {
3417
0
        status = NVME_INVALID_FORMAT | NVME_DNR;
3418
0
        goto invalid;
3419
0
    }
3420
3421
0
    if (ns->pif) {
3422
0
        len = sizeof(NvmeCopySourceRangeFormat1_3);
3423
0
    }
3424
3425
0
    iocb->format = format;
3426
0
    iocb->ranges = g_malloc_n(nr, len);
3427
0
    status = nvme_h2c(n, (uint8_t *)iocb->ranges, len * nr, req);
3428
0
    if (status) {
3429
0
        goto invalid;
3430
0
    }
3431
3432
0
    iocb->slba = le64_to_cpu(copy->sdlba);
3433
3434
0
    if (ns->params.zoned) {
3435
0
        iocb->zone = nvme_get_zone_by_slba(ns, iocb->slba);
3436
0
        if (!iocb->zone) {
3437
0
            status = NVME_LBA_RANGE | NVME_DNR;
3438
0
            goto invalid;
3439
0
        }
3440
3441
0
        status = nvme_zrm_auto(n, ns, iocb->zone);
3442
0
        if (status) {
3443
0
            goto invalid;
3444
0
        }
3445
0
    }
3446
3447
0
    status = nvme_check_copy_mcl(ns, iocb, nr);
3448
0
    if (status) {
3449
0
        goto invalid;
3450
0
    }
3451
3452
0
    iocb->req = req;
3453
0
    iocb->ret = 0;
3454
0
    iocb->nr = nr;
3455
0
    iocb->idx = 0;
3456
0
    iocb->reftag = le32_to_cpu(copy->reftag);
3457
0
    iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32;
3458
3459
0
    qemu_iovec_init(&iocb->iov, 1);
3460
3461
0
    req->aiocb = &iocb->common;
3462
0
    iocb->sns = req->ns;
3463
0
    iocb->n = n;
3464
0
    iocb->bounce = NULL;
3465
0
    nvme_do_copy(iocb);
3466
3467
0
    return NVME_NO_COMPLETE;
3468
3469
0
invalid:
3470
0
    g_free(iocb->ranges);
3471
0
    qemu_aio_unref(iocb);
3472
0
    return status;
3473
0
}
3474
3475
static uint16_t nvme_compare(NvmeCtrl *n, NvmeRequest *req)
3476
0
{
3477
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3478
0
    NvmeNamespace *ns = req->ns;
3479
0
    BlockBackend *blk = ns->blkconf.blk;
3480
0
    uint64_t slba = le64_to_cpu(rw->slba);
3481
0
    uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
3482
0
    uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3483
0
    size_t data_len = nvme_l2b(ns, nlb);
3484
0
    size_t len = data_len;
3485
0
    int64_t offset = nvme_l2b(ns, slba);
3486
0
    struct nvme_compare_ctx *ctx = NULL;
3487
0
    uint16_t status;
3488
3489
0
    trace_pci_nvme_compare(nvme_cid(req), nvme_nsid(ns), slba, nlb);
3490
3491
0
    if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) && (prinfo & NVME_PRINFO_PRACT)) {
3492
0
        return NVME_INVALID_PROT_INFO | NVME_DNR;
3493
0
    }
3494
3495
0
    if (nvme_ns_ext(ns)) {
3496
0
        len += nvme_m2b(ns, nlb);
3497
0
    }
3498
3499
0
    if (NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt)) {
3500
0
        status = nvme_check_mdts(n, data_len);
3501
0
    } else {
3502
0
        status = nvme_check_mdts(n, len);
3503
0
    }
3504
0
    if (status) {
3505
0
        return status;
3506
0
    }
3507
3508
0
    status = nvme_check_bounds(ns, slba, nlb);
3509
0
    if (status) {
3510
0
        return status;
3511
0
    }
3512
3513
0
    if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3514
0
        status = nvme_check_dulbe(ns, slba, nlb);
3515
0
        if (status) {
3516
0
            return status;
3517
0
        }
3518
0
    }
3519
3520
0
    status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
3521
0
    if (status) {
3522
0
        return status;
3523
0
    }
3524
3525
0
    ctx = g_new(struct nvme_compare_ctx, 1);
3526
0
    ctx->data.bounce = g_malloc(data_len);
3527
3528
0
    req->opaque = ctx;
3529
3530
0
    qemu_iovec_init(&ctx->data.iov, 1);
3531
0
    qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, data_len);
3532
3533
0
    block_acct_start(blk_get_stats(blk), &req->acct, data_len,
3534
0
                     BLOCK_ACCT_READ);
3535
0
    req->aiocb = blk_aio_preadv(blk, offset, &ctx->data.iov, 0,
3536
0
                                nvme_compare_data_cb, req);
3537
3538
0
    return NVME_NO_COMPLETE;
3539
0
}
3540
3541
typedef struct NvmeFlushAIOCB {
3542
    BlockAIOCB common;
3543
    BlockAIOCB *aiocb;
3544
    NvmeRequest *req;
3545
    int ret;
3546
3547
    NvmeNamespace *ns;
3548
    uint32_t nsid;
3549
    bool broadcast;
3550
} NvmeFlushAIOCB;
3551
3552
static void nvme_flush_cancel(BlockAIOCB *acb)
3553
0
{
3554
0
    NvmeFlushAIOCB *iocb = container_of(acb, NvmeFlushAIOCB, common);
3555
3556
0
    iocb->ret = -ECANCELED;
3557
3558
0
    if (iocb->aiocb) {
3559
0
        blk_aio_cancel_async(iocb->aiocb);
3560
0
        iocb->aiocb = NULL;
3561
0
    }
3562
0
}
3563
3564
static const AIOCBInfo nvme_flush_aiocb_info = {
3565
    .aiocb_size = sizeof(NvmeFlushAIOCB),
3566
    .cancel_async = nvme_flush_cancel,
3567
};
3568
3569
static void nvme_do_flush(NvmeFlushAIOCB *iocb);
3570
3571
static void nvme_flush_ns_cb(void *opaque, int ret)
3572
0
{
3573
0
    NvmeFlushAIOCB *iocb = opaque;
3574
0
    NvmeNamespace *ns = iocb->ns;
3575
3576
0
    if (ret < 0) {
3577
0
        iocb->ret = ret;
3578
0
        iocb->req->status = NVME_WRITE_FAULT;
3579
0
        goto out;
3580
0
    } else if (iocb->ret < 0) {
3581
0
        goto out;
3582
0
    }
3583
3584
0
    if (ns) {
3585
0
        trace_pci_nvme_flush_ns(iocb->nsid);
3586
3587
0
        iocb->ns = NULL;
3588
0
        iocb->aiocb = blk_aio_flush(ns->blkconf.blk, nvme_flush_ns_cb, iocb);
3589
0
        return;
3590
0
    }
3591
3592
0
out:
3593
0
    nvme_do_flush(iocb);
3594
0
}
3595
3596
static void nvme_do_flush(NvmeFlushAIOCB *iocb)
3597
0
{
3598
0
    NvmeRequest *req = iocb->req;
3599
0
    NvmeCtrl *n = nvme_ctrl(req);
3600
0
    int i;
3601
3602
0
    if (iocb->ret < 0) {
3603
0
        goto done;
3604
0
    }
3605
3606
0
    if (iocb->broadcast) {
3607
0
        for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
3608
0
            iocb->ns = nvme_ns(n, i);
3609
0
            if (iocb->ns) {
3610
0
                iocb->nsid = i;
3611
0
                break;
3612
0
            }
3613
0
        }
3614
0
    }
3615
3616
0
    if (!iocb->ns) {
3617
0
        goto done;
3618
0
    }
3619
3620
0
    nvme_flush_ns_cb(iocb, 0);
3621
0
    return;
3622
3623
0
done:
3624
0
    iocb->common.cb(iocb->common.opaque, iocb->ret);
3625
0
    qemu_aio_unref(iocb);
3626
0
}
3627
3628
static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req)
3629
0
{
3630
0
    NvmeFlushAIOCB *iocb;
3631
0
    uint32_t nsid = le32_to_cpu(req->cmd.nsid);
3632
0
    uint16_t status;
3633
3634
0
    iocb = qemu_aio_get(&nvme_flush_aiocb_info, NULL, nvme_misc_cb, req);
3635
3636
0
    iocb->req = req;
3637
0
    iocb->ret = 0;
3638
0
    iocb->ns = NULL;
3639
0
    iocb->nsid = 0;
3640
0
    iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
3641
3642
0
    if (!iocb->broadcast) {
3643
0
        if (!nvme_nsid_valid(n, nsid)) {
3644
0
            status = NVME_INVALID_NSID | NVME_DNR;
3645
0
            goto out;
3646
0
        }
3647
3648
0
        iocb->ns = nvme_ns(n, nsid);
3649
0
        if (!iocb->ns) {
3650
0
            status = NVME_INVALID_FIELD | NVME_DNR;
3651
0
            goto out;
3652
0
        }
3653
3654
0
        iocb->nsid = nsid;
3655
0
    }
3656
3657
0
    req->aiocb = &iocb->common;
3658
0
    nvme_do_flush(iocb);
3659
3660
0
    return NVME_NO_COMPLETE;
3661
3662
0
out:
3663
0
    qemu_aio_unref(iocb);
3664
3665
0
    return status;
3666
0
}
3667
3668
static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req)
3669
0
{
3670
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3671
0
    NvmeNamespace *ns = req->ns;
3672
0
    uint64_t slba = le64_to_cpu(rw->slba);
3673
0
    uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3674
0
    uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3675
0
    uint64_t data_size = nvme_l2b(ns, nlb);
3676
0
    uint64_t mapped_size = data_size;
3677
0
    uint64_t data_offset;
3678
0
    BlockBackend *blk = ns->blkconf.blk;
3679
0
    uint16_t status;
3680
3681
0
    if (nvme_ns_ext(ns) && !(NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt))) {
3682
0
        mapped_size += nvme_m2b(ns, nlb);
3683
3684
0
        if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3685
0
            bool pract = prinfo & NVME_PRINFO_PRACT;
3686
3687
0
            if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3688
0
                mapped_size = data_size;
3689
0
            }
3690
0
        }
3691
0
    }
3692
3693
0
    trace_pci_nvme_read(nvme_cid(req), nvme_nsid(ns), nlb, mapped_size, slba);
3694
3695
0
    status = nvme_check_mdts(n, mapped_size);
3696
0
    if (status) {
3697
0
        goto invalid;
3698
0
    }
3699
3700
0
    status = nvme_check_bounds(ns, slba, nlb);
3701
0
    if (status) {
3702
0
        goto invalid;
3703
0
    }
3704
3705
0
    if (ns->params.zoned) {
3706
0
        status = nvme_check_zone_read(ns, slba, nlb);
3707
0
        if (status) {
3708
0
            trace_pci_nvme_err_zone_read_not_ok(slba, nlb, status);
3709
0
            goto invalid;
3710
0
        }
3711
0
    }
3712
3713
0
    if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3714
0
        status = nvme_check_dulbe(ns, slba, nlb);
3715
0
        if (status) {
3716
0
            goto invalid;
3717
0
        }
3718
0
    }
3719
3720
0
    if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3721
0
        return nvme_dif_rw(n, req);
3722
0
    }
3723
3724
0
    status = nvme_map_data(n, nlb, req);
3725
0
    if (status) {
3726
0
        goto invalid;
3727
0
    }
3728
3729
0
    data_offset = nvme_l2b(ns, slba);
3730
3731
0
    block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3732
0
                     BLOCK_ACCT_READ);
3733
0
    nvme_blk_read(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3734
0
    return NVME_NO_COMPLETE;
3735
3736
0
invalid:
3737
0
    block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
3738
0
    return status | NVME_DNR;
3739
0
}
3740
3741
static void nvme_do_write_fdp(NvmeCtrl *n, NvmeRequest *req, uint64_t slba,
3742
                              uint32_t nlb)
3743
0
{
3744
0
    NvmeNamespace *ns = req->ns;
3745
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3746
0
    uint64_t data_size = nvme_l2b(ns, nlb);
3747
0
    uint32_t dw12 = le32_to_cpu(req->cmd.cdw12);
3748
0
    uint8_t dtype = (dw12 >> 20) & 0xf;
3749
0
    uint16_t pid = le16_to_cpu(rw->dspec);
3750
0
    uint16_t ph, rg, ruhid;
3751
0
    NvmeReclaimUnit *ru;
3752
3753
0
    if (dtype != NVME_DIRECTIVE_DATA_PLACEMENT ||
3754
0
        !nvme_parse_pid(ns, pid, &ph, &rg)) {
3755
0
        ph = 0;
3756
0
        rg = 0;
3757
0
        pid = 0;
3758
0
    }
3759
3760
0
    ruhid = ns->fdp.phs[ph];
3761
0
    ru = &ns->endgrp->fdp.ruhs[ruhid].rus[rg];
3762
3763
0
    nvme_fdp_stat_inc(&ns->endgrp->fdp.hbmw, data_size);
3764
0
    nvme_fdp_stat_inc(&ns->endgrp->fdp.mbmw, data_size);
3765
3766
0
    while (nlb) {
3767
0
        if (nlb < ru->ruamw) {
3768
0
            ru->ruamw -= nlb;
3769
0
            break;
3770
0
        }
3771
3772
0
        nlb -= ru->ruamw;
3773
0
        nvme_update_ruh(n, ns, pid);
3774
0
    }
3775
0
}
3776
3777
static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append,
3778
                              bool wrz)
3779
0
{
3780
0
    NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3781
0
    NvmeNamespace *ns = req->ns;
3782
0
    uint64_t slba = le64_to_cpu(rw->slba);
3783
0
    uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3784
0
    uint16_t ctrl = le16_to_cpu(rw->control);
3785
0
    uint8_t prinfo = NVME_RW_PRINFO(ctrl);
3786
0
    uint64_t data_size = nvme_l2b(ns, nlb);
3787
0
    uint64_t mapped_size = data_size;
3788
0
    uint64_t data_offset;
3789
0
    NvmeZone *zone;
3790
0
    NvmeZonedResult *res = (NvmeZonedResult *)&req->cqe;
3791
0
    BlockBackend *blk = ns->blkconf.blk;
3792
0
    uint16_t status;
3793
3794
0
    if (nvme_ns_ext(ns) && !(NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt))) {
3795
0
        mapped_size += nvme_m2b(ns, nlb);
3796
3797
0
        if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3798
0
            bool pract = prinfo & NVME_PRINFO_PRACT;
3799
3800
0
            if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3801
0
                mapped_size -= nvme_m2b(ns, nlb);
3802
0
            }
3803
0
        }
3804
0
    }
3805
3806
0
    trace_pci_nvme_write(nvme_cid(req), nvme_io_opc_str(rw->opcode),
3807
0
                         nvme_nsid(ns), nlb, mapped_size, slba);
3808
3809
0
    if (!wrz) {
3810
0
        status = nvme_check_mdts(n, mapped_size);
3811
0
        if (status) {
3812
0
            goto invalid;
3813
0
        }
3814
0
    }
3815
3816
0
    status = nvme_check_bounds(ns, slba, nlb);
3817
0
    if (status) {
3818
0
        goto invalid;
3819
0
    }
3820
3821
0
    if (ns->params.zoned) {
3822
0
        zone = nvme_get_zone_by_slba(ns, slba);
3823
0
        assert(zone);
3824
3825
0
        if (append) {
3826
0
            bool piremap = !!(ctrl & NVME_RW_PIREMAP);
3827
3828
0
            if (unlikely(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3829
0
                return NVME_INVALID_ZONE_OP | NVME_DNR;
3830
0
            }
3831
3832
0
            if (unlikely(slba != zone->d.zslba)) {
3833
0
                trace_pci_nvme_err_append_not_at_start(slba, zone->d.zslba);
3834
0
                status = NVME_INVALID_FIELD;
3835
0
                goto invalid;
3836
0
            }
3837
3838
0
            if (n->params.zasl &&
3839
0
                data_size > (uint64_t)n->page_size << n->params.zasl) {
3840
0
                trace_pci_nvme_err_zasl(data_size);
3841
0
                return NVME_INVALID_FIELD | NVME_DNR;
3842
0
            }
3843
3844
0
            slba = zone->w_ptr;
3845
0
            rw->slba = cpu_to_le64(slba);
3846
0
            res->slba = cpu_to_le64(slba);
3847
3848
0
            switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3849
0
            case NVME_ID_NS_DPS_TYPE_1:
3850
0
                if (!piremap) {
3851
0
                    return NVME_INVALID_PROT_INFO | NVME_DNR;
3852
0
                }
3853
3854
                /* fallthrough */
3855
3856
0
            case NVME_ID_NS_DPS_TYPE_2:
3857
0
                if (piremap) {
3858
0
                    uint32_t reftag = le32_to_cpu(rw->reftag);
3859
0
                    rw->reftag = cpu_to_le32(reftag + (slba - zone->d.zslba));
3860
0
                }
3861
3862
0
                break;
3863
3864
0
            case NVME_ID_NS_DPS_TYPE_3:
3865
0
                if (piremap) {
3866
0
                    return NVME_INVALID_PROT_INFO | NVME_DNR;
3867
0
                }
3868
3869
0
                break;
3870
0
            }
3871
0
        }
3872
3873
0
        status = nvme_check_zone_write(ns, zone, slba, nlb);
3874
0
        if (status) {
3875
0
            goto invalid;
3876
0
        }
3877
3878
0
        status = nvme_zrm_auto(n, ns, zone);
3879
0
        if (status) {
3880
0
            goto invalid;
3881
0
        }
3882
3883
0
        if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3884
0
            zone->w_ptr += nlb;
3885
0
        }
3886
0
    } else if (ns->endgrp && ns->endgrp->fdp.enabled) {
3887
0
        nvme_do_write_fdp(n, req, slba, nlb);
3888
0
    }
3889
3890
0
    data_offset = nvme_l2b(ns, slba);
3891
3892
0
    if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3893
0
        return nvme_dif_rw(n, req);
3894
0
    }
3895
3896
0
    if (!wrz) {
3897
0
        status = nvme_map_data(n, nlb, req);
3898
0
        if (status) {
3899
0
            goto invalid;
3900
0
        }
3901
3902
0
        block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3903
0
                         BLOCK_ACCT_WRITE);
3904
0
        nvme_blk_write(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3905
0
    } else {
3906
0
        req->aiocb = blk_aio_pwrite_zeroes(blk, data_offset, data_size,
3907
0
                                           BDRV_REQ_MAY_UNMAP, nvme_rw_cb,
3908
0
                                           req);
3909
0
    }
3910
3911
0
    return NVME_NO_COMPLETE;
3912
3913
0
invalid:
3914
0
    block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
3915
0
    return status | NVME_DNR;
3916
0
}
3917
3918
static inline uint16_t nvme_write(NvmeCtrl *n, NvmeRequest *req)
3919
0
{
3920
0
    return nvme_do_write(n, req, false, false);
3921
0
}
3922
3923
static inline uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req)
3924
0
{
3925
0
    return nvme_do_write(n, req, false, true);
3926
0
}
3927
3928
static inline uint16_t nvme_zone_append(NvmeCtrl *n, NvmeRequest *req)
3929
0
{
3930
0
    return nvme_do_write(n, req, true, false);
3931
0
}
3932
3933
static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace *ns, NvmeCmd *c,
3934
                                            uint64_t *slba, uint32_t *zone_idx)
3935
0
{
3936
0
    uint32_t dw10 = le32_to_cpu(c->cdw10);
3937
0
    uint32_t dw11 = le32_to_cpu(c->cdw11);
3938
3939
0
    if (!ns->params.zoned) {
3940
0
        trace_pci_nvme_err_invalid_opc(c->opcode);
3941
0
        return NVME_INVALID_OPCODE | NVME_DNR;
3942
0
    }
3943
3944
0
    *slba = ((uint64_t)dw11) << 32 | dw10;
3945
0
    if (unlikely(*slba >= ns->id_ns.nsze)) {
3946
0
        trace_pci_nvme_err_invalid_lba_range(*slba, 0, ns->id_ns.nsze);
3947
0
        *slba = 0;
3948
0
        return NVME_LBA_RANGE | NVME_DNR;
3949
0
    }
3950
3951
0
    *zone_idx = nvme_zone_idx(ns, *slba);
3952
0
    assert(*zone_idx < ns->num_zones);
3953
3954
0
    return NVME_SUCCESS;
3955
0
}
3956
3957
typedef uint16_t (*op_handler_t)(NvmeNamespace *, NvmeZone *, NvmeZoneState,
3958
                                 NvmeRequest *);
3959
3960
enum NvmeZoneProcessingMask {
3961
    NVME_PROC_CURRENT_ZONE    = 0,
3962
    NVME_PROC_OPENED_ZONES    = 1 << 0,
3963
    NVME_PROC_CLOSED_ZONES    = 1 << 1,
3964
    NVME_PROC_READ_ONLY_ZONES = 1 << 2,
3965
    NVME_PROC_FULL_ZONES      = 1 << 3,
3966
};
3967
3968
static uint16_t nvme_open_zone(NvmeNamespace *ns, NvmeZone *zone,
3969
                               NvmeZoneState state, NvmeRequest *req)
3970
0
{
3971
0
    NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
3972
0
    int flags = 0;
3973
3974
0
    if (cmd->zsflags & NVME_ZSFLAG_ZRWA_ALLOC) {
3975
0
        uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3976
3977
0
        if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3978
0
            return NVME_INVALID_ZONE_OP | NVME_DNR;
3979
0
        }
3980
3981
0
        if (zone->w_ptr % ns->zns.zrwafg) {
3982
0
            return NVME_NOZRWA | NVME_DNR;
3983
0
        }
3984
3985
0
        flags = NVME_ZRM_ZRWA;
3986
0
    }
3987
3988
0
    return nvme_zrm_open_flags(nvme_ctrl(req), ns, zone, flags);
3989
0
}
3990
3991
static uint16_t nvme_close_zone(NvmeNamespace *ns, NvmeZone *zone,
3992
                                NvmeZoneState state, NvmeRequest *req)
3993
0
{
3994
0
    return nvme_zrm_close(ns, zone);
3995
0
}
3996
3997
static uint16_t nvme_finish_zone(NvmeNamespace *ns, NvmeZone *zone,
3998
                                 NvmeZoneState state, NvmeRequest *req)
3999
0
{
4000
0
    return nvme_zrm_finish(ns, zone);
4001
0
}
4002
4003
static uint16_t nvme_offline_zone(NvmeNamespace *ns, NvmeZone *zone,
4004
                                  NvmeZoneState state, NvmeRequest *req)
4005
0
{
4006
0
    switch (state) {
4007
0
    case NVME_ZONE_STATE_READ_ONLY:
4008
0
        nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_OFFLINE);
4009
        /* fall through */
4010
0
    case NVME_ZONE_STATE_OFFLINE:
4011
0
        return NVME_SUCCESS;
4012
0
    default:
4013
0
        return NVME_ZONE_INVAL_TRANSITION;
4014
0
    }
4015
0
}
4016
4017
static uint16_t nvme_set_zd_ext(NvmeNamespace *ns, NvmeZone *zone)
4018
0
{
4019
0
    uint16_t status;
4020
0
    uint8_t state = nvme_get_zone_state(zone);
4021
4022
0
    if (state == NVME_ZONE_STATE_EMPTY) {
4023
0
        status = nvme_aor_check(ns, 1, 0);
4024
0
        if (status) {
4025
0
            return status;
4026
0
        }
4027
0
        nvme_aor_inc_active(ns);
4028
0
        zone->d.za |= NVME_ZA_ZD_EXT_VALID;
4029
0
        nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
4030
0
        return NVME_SUCCESS;
4031
0
    }
4032
4033
0
    return NVME_ZONE_INVAL_TRANSITION;
4034
0
}
4035
4036
static uint16_t nvme_bulk_proc_zone(NvmeNamespace *ns, NvmeZone *zone,
4037
                                    enum NvmeZoneProcessingMask proc_mask,
4038
                                    op_handler_t op_hndlr, NvmeRequest *req)
4039
0
{
4040
0
    uint16_t status = NVME_SUCCESS;
4041
0
    NvmeZoneState zs = nvme_get_zone_state(zone);
4042
0
    bool proc_zone;
4043
4044
0
    switch (zs) {
4045
0
    case NVME_ZONE_STATE_IMPLICITLY_OPEN:
4046
0
    case NVME_ZONE_STATE_EXPLICITLY_OPEN:
4047
0
        proc_zone = proc_mask & NVME_PROC_OPENED_ZONES;
4048
0
        break;
4049
0
    case NVME_ZONE_STATE_CLOSED:
4050
0
        proc_zone = proc_mask & NVME_PROC_CLOSED_ZONES;
4051
0
        break;
4052
0
    case NVME_ZONE_STATE_READ_ONLY:
4053
0
        proc_zone = proc_mask & NVME_PROC_READ_ONLY_ZONES;
4054
0
        break;
4055
0
    case NVME_ZONE_STATE_FULL:
4056
0
        proc_zone = proc_mask & NVME_PROC_FULL_ZONES;
4057
0
        break;
4058
0
    default:
4059
0
        proc_zone = false;
4060
0
    }
4061
4062
0
    if (proc_zone) {
4063
0
        status = op_hndlr(ns, zone, zs, req);
4064
0
    }
4065
4066
0
    return status;
4067
0
}
4068
4069
static uint16_t nvme_do_zone_op(NvmeNamespace *ns, NvmeZone *zone,
4070
                                enum NvmeZoneProcessingMask proc_mask,
4071
                                op_handler_t op_hndlr, NvmeRequest *req)
4072
0
{
4073
0
    NvmeZone *next;
4074
0
    uint16_t status = NVME_SUCCESS;
4075
0
    int i;
4076
4077
0
    if (!proc_mask) {
4078
0
        status = op_hndlr(ns, zone, nvme_get_zone_state(zone), req);
4079
0
    } else {
4080
0
        if (proc_mask & NVME_PROC_CLOSED_ZONES) {
4081
0
            QTAILQ_FOREACH_SAFE(zone, &ns->closed_zones, entry, next) {
4082
0
                status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4083
0
                                             req);
4084
0
                if (status && status != NVME_NO_COMPLETE) {
4085
0
                    goto out;
4086
0
                }
4087
0
            }
4088
0
        }
4089
0
        if (proc_mask & NVME_PROC_OPENED_ZONES) {
4090
0
            QTAILQ_FOREACH_SAFE(zone, &ns->imp_open_zones, entry, next) {
4091
0
                status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4092
0
                                             req);
4093
0
                if (status && status != NVME_NO_COMPLETE) {
4094
0
                    goto out;
4095
0
                }
4096
0
            }
4097
4098
0
            QTAILQ_FOREACH_SAFE(zone, &ns->exp_open_zones, entry, next) {
4099
0
                status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4100
0
                                             req);
4101
0
                if (status && status != NVME_NO_COMPLETE) {
4102
0
                    goto out;
4103
0
                }
4104
0
            }
4105
0
        }
4106
0
        if (proc_mask & NVME_PROC_FULL_ZONES) {
4107
0
            QTAILQ_FOREACH_SAFE(zone, &ns->full_zones, entry, next) {
4108
0
                status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4109
0
                                             req);
4110
0
                if (status && status != NVME_NO_COMPLETE) {
4111
0
                    goto out;
4112
0
                }
4113
0
            }
4114
0
        }
4115
4116
0
        if (proc_mask & NVME_PROC_READ_ONLY_ZONES) {
4117
0
            for (i = 0; i < ns->num_zones; i++, zone++) {
4118
0
                status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4119
0
                                             req);
4120
0
                if (status && status != NVME_NO_COMPLETE) {
4121
0
                    goto out;
4122
0
                }
4123
0
            }
4124
0
        }
4125
0
    }
4126
4127
0
out:
4128
0
    return status;
4129
0
}
4130
4131
typedef struct NvmeZoneResetAIOCB {
4132
    BlockAIOCB common;
4133
    BlockAIOCB *aiocb;
4134
    NvmeRequest *req;
4135
    int ret;
4136
4137
    bool all;
4138
    int idx;
4139
    NvmeZone *zone;
4140
} NvmeZoneResetAIOCB;
4141
4142
static void nvme_zone_reset_cancel(BlockAIOCB *aiocb)
4143
0
{
4144
0
    NvmeZoneResetAIOCB *iocb = container_of(aiocb, NvmeZoneResetAIOCB, common);
4145
0
    NvmeRequest *req = iocb->req;
4146
0
    NvmeNamespace *ns = req->ns;
4147
4148
0
    iocb->idx = ns->num_zones;
4149
4150
0
    iocb->ret = -ECANCELED;
4151
4152
0
    if (iocb->aiocb) {
4153
0
        blk_aio_cancel_async(iocb->aiocb);
4154
0
        iocb->aiocb = NULL;
4155
0
    }
4156
0
}
4157
4158
static const AIOCBInfo nvme_zone_reset_aiocb_info = {
4159
    .aiocb_size = sizeof(NvmeZoneResetAIOCB),
4160
    .cancel_async = nvme_zone_reset_cancel,
4161
};
4162
4163
static void nvme_zone_reset_cb(void *opaque, int ret);
4164
4165
static void nvme_zone_reset_epilogue_cb(void *opaque, int ret)
4166
0
{
4167
0
    NvmeZoneResetAIOCB *iocb = opaque;
4168
0
    NvmeRequest *req = iocb->req;
4169
0
    NvmeNamespace *ns = req->ns;
4170
0
    int64_t moff;
4171
0
    int count;
4172
4173
0
    if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
4174
0
        goto out;
4175
0
    }
4176
4177
0
    moff = nvme_moff(ns, iocb->zone->d.zslba);
4178
0
    count = nvme_m2b(ns, ns->zone_size);
4179
4180
0
    iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, moff, count,
4181
0
                                        BDRV_REQ_MAY_UNMAP,
4182
0
                                        nvme_zone_reset_cb, iocb);
4183
0
    return;
4184
4185
0
out:
4186
0
    nvme_zone_reset_cb(iocb, ret);
4187
0
}
4188
4189
static void nvme_zone_reset_cb(void *opaque, int ret)
4190
0
{
4191
0
    NvmeZoneResetAIOCB *iocb = opaque;
4192
0
    NvmeRequest *req = iocb->req;
4193
0
    NvmeNamespace *ns = req->ns;
4194
4195
0
    if (iocb->ret < 0) {
4196
0
        goto done;
4197
0
    } else if (ret < 0) {
4198
0
        iocb->ret = ret;
4199
0
        goto done;
4200
0
    }
4201
4202
0
    if (iocb->zone) {
4203
0
        nvme_zrm_reset(ns, iocb->zone);
4204
4205
0
        if (!iocb->all) {
4206
0
            goto done;
4207
0
        }
4208
0
    }
4209
4210
0
    while (iocb->idx < ns->num_zones) {
4211
0
        NvmeZone *zone = &ns->zone_array[iocb->idx++];
4212
4213
0
        switch (nvme_get_zone_state(zone)) {
4214
0
        case NVME_ZONE_STATE_EMPTY:
4215
0
            if (!iocb->all) {
4216
0
                goto done;
4217
0
            }
4218
4219
0
            continue;
4220
4221
0
        case NVME_ZONE_STATE_EXPLICITLY_OPEN:
4222
0
        case NVME_ZONE_STATE_IMPLICITLY_OPEN:
4223
0
        case NVME_ZONE_STATE_CLOSED:
4224
0
        case NVME_ZONE_STATE_FULL:
4225
0
            iocb->zone = zone;
4226
0
            break;
4227
4228
0
        default:
4229
0
            continue;
4230
0
        }
4231
4232
0
        trace_pci_nvme_zns_zone_reset(zone->d.zslba);
4233
4234
0
        iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk,
4235
0
                                            nvme_l2b(ns, zone->d.zslba),
4236
0
                                            nvme_l2b(ns, ns->zone_size),
4237
0
                                            BDRV_REQ_MAY_UNMAP,
4238
0
                                            nvme_zone_reset_epilogue_cb,
4239
0
                                            iocb);
4240
0
        return;
4241
0
    }
4242
4243
0
done:
4244
0
    iocb->aiocb = NULL;
4245
4246
0
    iocb->common.cb(iocb->common.opaque, iocb->ret);
4247
0
    qemu_aio_unref(iocb);
4248
0
}
4249
4250
static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl *n, NvmeZone *zone,
4251
                                               uint64_t elba, NvmeRequest *req)
4252
0
{
4253
0
    NvmeNamespace *ns = req->ns;
4254
0
    uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
4255
0
    uint64_t wp = zone->d.wp;
4256
0
    uint32_t nlb = elba - wp + 1;
4257
0
    uint16_t status;
4258
4259
4260
0
    if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
4261
0
        return NVME_INVALID_ZONE_OP | NVME_DNR;
4262
0
    }
4263
4264
0
    if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
4265
0
        return NVME_INVALID_FIELD | NVME_DNR;
4266
0
    }
4267
4268
0
    if (elba < wp || elba > wp + ns->zns.zrwas) {
4269
0
        return NVME_ZONE_BOUNDARY_ERROR | NVME_DNR;
4270
0
    }
4271
4272
0
    if (nlb % ns->zns.zrwafg) {
4273
0
        return NVME_INVALID_FIELD | NVME_DNR;
4274
0
    }
4275
4276
0
    status = nvme_zrm_auto(n, ns, zone);
4277
0
    if (status) {
4278
0
        return status;
4279
0
    }
4280
4281
0
    zone->w_ptr += nlb;
4282
4283
0
    nvme_advance_zone_wp(ns, zone, nlb);
4284
4285
0
    return NVME_SUCCESS;
4286
0
}
4287
4288
static uint16_t nvme_zone_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4289
0
{
4290
0
    NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
4291
0
    NvmeNamespace *ns = req->ns;
4292
0
    NvmeZone *zone;
4293
0
    NvmeZoneResetAIOCB *iocb;
4294
0
    uint8_t *zd_ext;
4295
0
    uint64_t slba = 0;
4296
0
    uint32_t zone_idx = 0;
4297
0
    uint16_t status;
4298
0
    uint8_t action = cmd->zsa;
4299
0
    bool all;
4300
0
    enum NvmeZoneProcessingMask proc_mask = NVME_PROC_CURRENT_ZONE;
4301
4302
0
    all = cmd->zsflags & NVME_ZSFLAG_SELECT_ALL;
4303
4304
0
    req->status = NVME_SUCCESS;
4305
4306
0
    if (!all) {
4307
0
        status = nvme_get_mgmt_zone_slba_idx(ns, &req->cmd, &slba, &zone_idx);
4308
0
        if (status) {
4309
0
            return status;
4310
0
        }
4311
0
    }
4312
4313
0
    zone = &ns->zone_array[zone_idx];
4314
0
    if (slba != zone->d.zslba && action != NVME_ZONE_ACTION_ZRWA_FLUSH) {
4315
0
        trace_pci_nvme_err_unaligned_zone_cmd(action, slba, zone->d.zslba);
4316
0
        return NVME_INVALID_FIELD | NVME_DNR;
4317
0
    }
4318
4319
0
    switch (action) {
4320
4321
0
    case NVME_ZONE_ACTION_OPEN:
4322
0
        if (all) {
4323
0
            proc_mask = NVME_PROC_CLOSED_ZONES;
4324
0
        }
4325
0
        trace_pci_nvme_open_zone(slba, zone_idx, all);
4326
0
        status = nvme_do_zone_op(ns, zone, proc_mask, nvme_open_zone, req);
4327
0
        break;
4328
4329
0
    case NVME_ZONE_ACTION_CLOSE:
4330
0
        if (all) {
4331
0
            proc_mask = NVME_PROC_OPENED_ZONES;
4332
0
        }
4333
0
        trace_pci_nvme_close_zone(slba, zone_idx, all);
4334
0
        status = nvme_do_zone_op(ns, zone, proc_mask, nvme_close_zone, req);
4335
0
        break;
4336
4337
0
    case NVME_ZONE_ACTION_FINISH:
4338
0
        if (all) {
4339
0
            proc_mask = NVME_PROC_OPENED_ZONES | NVME_PROC_CLOSED_ZONES;
4340
0
        }
4341
0
        trace_pci_nvme_finish_zone(slba, zone_idx, all);
4342
0
        status = nvme_do_zone_op(ns, zone, proc_mask, nvme_finish_zone, req);
4343
0
        break;
4344
4345
0
    case NVME_ZONE_ACTION_RESET:
4346
0
        trace_pci_nvme_reset_zone(slba, zone_idx, all);
4347
4348
0
        iocb = blk_aio_get(&nvme_zone_reset_aiocb_info, ns->blkconf.blk,
4349
0
                           nvme_misc_cb, req);
4350
4351
0
        iocb->req = req;
4352
0
        iocb->ret = 0;
4353
0
        iocb->all = all;
4354
0
        iocb->idx = zone_idx;
4355
0
        iocb->zone = NULL;
4356
4357
0
        req->aiocb = &iocb->common;
4358
0
        nvme_zone_reset_cb(iocb, 0);
4359
4360
0
        return NVME_NO_COMPLETE;
4361
4362
0
    case NVME_ZONE_ACTION_OFFLINE:
4363
0
        if (all) {
4364
0
            proc_mask = NVME_PROC_READ_ONLY_ZONES;
4365
0
        }
4366
0
        trace_pci_nvme_offline_zone(slba, zone_idx, all);
4367
0
        status = nvme_do_zone_op(ns, zone, proc_mask, nvme_offline_zone, req);
4368
0
        break;
4369
4370
0
    case NVME_ZONE_ACTION_SET_ZD_EXT:
4371
0
        trace_pci_nvme_set_descriptor_extension(slba, zone_idx);
4372
0
        if (all || !ns->params.zd_extension_size) {
4373
0
            return NVME_INVALID_FIELD | NVME_DNR;
4374
0
        }
4375
0
        zd_ext = nvme_get_zd_extension(ns, zone_idx);
4376
0
        status = nvme_h2c(n, zd_ext, ns->params.zd_extension_size, req);
4377
0
        if (status) {
4378
0
            trace_pci_nvme_err_zd_extension_map_error(zone_idx);
4379
0
            return status;
4380
0
        }
4381
4382
0
        status = nvme_set_zd_ext(ns, zone);
4383
0
        if (status == NVME_SUCCESS) {
4384
0
            trace_pci_nvme_zd_extension_set(zone_idx);
4385
0
            return status;
4386
0
        }
4387
0
        break;
4388
4389
0
    case NVME_ZONE_ACTION_ZRWA_FLUSH:
4390
0
        if (all) {
4391
0
            return NVME_INVALID_FIELD | NVME_DNR;
4392
0
        }
4393
4394
0
        return nvme_zone_mgmt_send_zrwa_flush(n, zone, slba, req);
4395
4396
0
    default:
4397
0
        trace_pci_nvme_err_invalid_mgmt_action(action);
4398
0
        status = NVME_INVALID_FIELD;
4399
0
    }
4400
4401
0
    if (status == NVME_ZONE_INVAL_TRANSITION) {
4402
0
        trace_pci_nvme_err_invalid_zone_state_transition(action, slba,
4403
0
                                                         zone->d.za);
4404
0
    }
4405
0
    if (status) {
4406
0
        status |= NVME_DNR;
4407
0
    }
4408
4409
0
    return status;
4410
0
}
4411
4412
static bool nvme_zone_matches_filter(uint32_t zafs, NvmeZone *zl)
4413
0
{
4414
0
    NvmeZoneState zs = nvme_get_zone_state(zl);
4415
4416
0
    switch (zafs) {
4417
0
    case NVME_ZONE_REPORT_ALL:
4418
0
        return true;
4419
0
    case NVME_ZONE_REPORT_EMPTY:
4420
0
        return zs == NVME_ZONE_STATE_EMPTY;
4421
0
    case NVME_ZONE_REPORT_IMPLICITLY_OPEN:
4422
0
        return zs == NVME_ZONE_STATE_IMPLICITLY_OPEN;
4423
0
    case NVME_ZONE_REPORT_EXPLICITLY_OPEN:
4424
0
        return zs == NVME_ZONE_STATE_EXPLICITLY_OPEN;
4425
0
    case NVME_ZONE_REPORT_CLOSED:
4426
0
        return zs == NVME_ZONE_STATE_CLOSED;
4427
0
    case NVME_ZONE_REPORT_FULL:
4428
0
        return zs == NVME_ZONE_STATE_FULL;
4429
0
    case NVME_ZONE_REPORT_READ_ONLY:
4430
0
        return zs == NVME_ZONE_STATE_READ_ONLY;
4431
0
    case NVME_ZONE_REPORT_OFFLINE:
4432
0
        return zs == NVME_ZONE_STATE_OFFLINE;
4433
0
    default:
4434
0
        return false;
4435
0
    }
4436
0
}
4437
4438
static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4439
0
{
4440
0
    NvmeCmd *cmd = &req->cmd;
4441
0
    NvmeNamespace *ns = req->ns;
4442
    /* cdw12 is zero-based number of dwords to return. Convert to bytes */
4443
0
    uint32_t data_size = (le32_to_cpu(cmd->cdw12) + 1) << 2;
4444
0
    uint32_t dw13 = le32_to_cpu(cmd->cdw13);
4445
0
    uint32_t zone_idx, zra, zrasf, partial;
4446
0
    uint64_t max_zones, nr_zones = 0;
4447
0
    uint16_t status;
4448
0
    uint64_t slba;
4449
0
    NvmeZoneDescr *z;
4450
0
    NvmeZone *zone;
4451
0
    NvmeZoneReportHeader *header;
4452
0
    void *buf, *buf_p;
4453
0
    size_t zone_entry_sz;
4454
0
    int i;
4455
4456
0
    req->status = NVME_SUCCESS;
4457
4458
0
    status = nvme_get_mgmt_zone_slba_idx(ns, cmd, &slba, &zone_idx);
4459
0
    if (status) {
4460
0
        return status;
4461
0
    }
4462
4463
0
    zra = dw13 & 0xff;
4464
0
    if (zra != NVME_ZONE_REPORT && zra != NVME_ZONE_REPORT_EXTENDED) {
4465
0
        return NVME_INVALID_FIELD | NVME_DNR;
4466
0
    }
4467
0
    if (zra == NVME_ZONE_REPORT_EXTENDED && !ns->params.zd_extension_size) {
4468
0
        return NVME_INVALID_FIELD | NVME_DNR;
4469
0
    }
4470
4471
0
    zrasf = (dw13 >> 8) & 0xff;
4472
0
    if (zrasf > NVME_ZONE_REPORT_OFFLINE) {
4473
0
        return NVME_INVALID_FIELD | NVME_DNR;
4474
0
    }
4475
4476
0
    if (data_size < sizeof(NvmeZoneReportHeader)) {
4477
0
        return NVME_INVALID_FIELD | NVME_DNR;
4478
0
    }
4479
4480
0
    status = nvme_check_mdts(n, data_size);
4481
0
    if (status) {
4482
0
        return status;
4483
0
    }
4484
4485
0
    partial = (dw13 >> 16) & 0x01;
4486
4487
0
    zone_entry_sz = sizeof(NvmeZoneDescr);
4488
0
    if (zra == NVME_ZONE_REPORT_EXTENDED) {
4489
0
        zone_entry_sz += ns->params.zd_extension_size;
4490
0
    }
4491
4492
0
    max_zones = (data_size - sizeof(NvmeZoneReportHeader)) / zone_entry_sz;
4493
0
    buf = g_malloc0(data_size);
4494
4495
0
    zone = &ns->zone_array[zone_idx];
4496
0
    for (i = zone_idx; i < ns->num_zones; i++) {
4497
0
        if (partial && nr_zones >= max_zones) {
4498
0
            break;
4499
0
        }
4500
0
        if (nvme_zone_matches_filter(zrasf, zone++)) {
4501
0
            nr_zones++;
4502
0
        }
4503
0
    }
4504
0
    header = buf;
4505
0
    header->nr_zones = cpu_to_le64(nr_zones);
4506
4507
0
    buf_p = buf + sizeof(NvmeZoneReportHeader);
4508
0
    for (; zone_idx < ns->num_zones && max_zones > 0; zone_idx++) {
4509
0
        zone = &ns->zone_array[zone_idx];
4510
0
        if (nvme_zone_matches_filter(zrasf, zone)) {
4511
0
            z = buf_p;
4512
0
            buf_p += sizeof(NvmeZoneDescr);
4513
4514
0
            z->zt = zone->d.zt;
4515
0
            z->zs = zone->d.zs;
4516
0
            z->zcap = cpu_to_le64(zone->d.zcap);
4517
0
            z->zslba = cpu_to_le64(zone->d.zslba);
4518
0
            z->za = zone->d.za;
4519
4520
0
            if (nvme_wp_is_valid(zone)) {
4521
0
                z->wp = cpu_to_le64(zone->d.wp);
4522
0
            } else {
4523
0
                z->wp = cpu_to_le64(~0ULL);
4524
0
            }
4525
4526
0
            if (zra == NVME_ZONE_REPORT_EXTENDED) {
4527
0
                if (zone->d.za & NVME_ZA_ZD_EXT_VALID) {
4528
0
                    memcpy(buf_p, nvme_get_zd_extension(ns, zone_idx),
4529
0
                           ns->params.zd_extension_size);
4530
0
                }
4531
0
                buf_p += ns->params.zd_extension_size;
4532
0
            }
4533
4534
0
            max_zones--;
4535
0
        }
4536
0
    }
4537
4538
0
    status = nvme_c2h(n, (uint8_t *)buf, data_size, req);
4539
4540
0
    g_free(buf);
4541
4542
0
    return status;
4543
0
}
4544
4545
static uint16_t nvme_io_mgmt_recv_ruhs(NvmeCtrl *n, NvmeRequest *req,
4546
                                       size_t len)
4547
0
{
4548
0
    NvmeNamespace *ns = req->ns;
4549
0
    NvmeEnduranceGroup *endgrp;
4550
0
    NvmeRuhStatus *hdr;
4551
0
    NvmeRuhStatusDescr *ruhsd;
4552
0
    unsigned int nruhsd;
4553
0
    uint16_t rg, ph, *ruhid;
4554
0
    size_t trans_len;
4555
0
    g_autofree uint8_t *buf = NULL;
4556
4557
0
    if (!n->subsys) {
4558
0
        return NVME_INVALID_FIELD | NVME_DNR;
4559
0
    }
4560
4561
0
    if (ns->params.nsid == 0 || ns->params.nsid == 0xffffffff) {
4562
0
        return NVME_INVALID_NSID | NVME_DNR;
4563
0
    }
4564
4565
0
    if (!n->subsys->endgrp.fdp.enabled) {
4566
0
        return NVME_FDP_DISABLED | NVME_DNR;
4567
0
    }
4568
4569
0
    endgrp = ns->endgrp;
4570
4571
0
    nruhsd = ns->fdp.nphs * endgrp->fdp.nrg;
4572
0
    trans_len = sizeof(NvmeRuhStatus) + nruhsd * sizeof(NvmeRuhStatusDescr);
4573
0
    buf = g_malloc0(trans_len);
4574
4575
0
    trans_len = MIN(trans_len, len);
4576
4577
0
    hdr = (NvmeRuhStatus *)buf;
4578
0
    ruhsd = (NvmeRuhStatusDescr *)(buf + sizeof(NvmeRuhStatus));
4579
4580
0
    hdr->nruhsd = cpu_to_le16(nruhsd);
4581
4582
0
    ruhid = ns->fdp.phs;
4583
4584
0
    for (ph = 0; ph < ns->fdp.nphs; ph++, ruhid++) {
4585
0
        NvmeRuHandle *ruh = &endgrp->fdp.ruhs[*ruhid];
4586
4587
0
        for (rg = 0; rg < endgrp->fdp.nrg; rg++, ruhsd++) {
4588
0
            uint16_t pid = nvme_make_pid(ns, rg, ph);
4589
4590
0
            ruhsd->pid = cpu_to_le16(pid);
4591
0
            ruhsd->ruhid = *ruhid;
4592
0
            ruhsd->earutr = 0;
4593
0
            ruhsd->ruamw = cpu_to_le64(ruh->rus[rg].ruamw);
4594
0
        }
4595
0
    }
4596
4597
0
    return nvme_c2h(n, buf, trans_len, req);
4598
0
}
4599
4600
static uint16_t nvme_io_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4601
0
{
4602
0
    NvmeCmd *cmd = &req->cmd;
4603
0
    uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4604
0
    uint32_t numd = le32_to_cpu(cmd->cdw11);
4605
0
    uint8_t mo = (cdw10 & 0xff);
4606
0
    size_t len = (numd + 1) << 2;
4607
4608
0
    switch (mo) {
4609
0
    case NVME_IOMR_MO_NOP:
4610
0
        return 0;
4611
0
    case NVME_IOMR_MO_RUH_STATUS:
4612
0
        return nvme_io_mgmt_recv_ruhs(n, req, len);
4613
0
    default:
4614
0
        return NVME_INVALID_FIELD | NVME_DNR;
4615
0
    };
4616
0
}
4617
4618
static uint16_t nvme_io_mgmt_send_ruh_update(NvmeCtrl *n, NvmeRequest *req)
4619
0
{
4620
0
    NvmeCmd *cmd = &req->cmd;
4621
0
    NvmeNamespace *ns = req->ns;
4622
0
    uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4623
0
    uint16_t ret = NVME_SUCCESS;
4624
0
    uint32_t npid = (cdw10 >> 16) + 1;
4625
0
    unsigned int i = 0;
4626
0
    g_autofree uint16_t *pids = NULL;
4627
0
    uint32_t maxnpid;
4628
4629
0
    if (!ns->endgrp || !ns->endgrp->fdp.enabled) {
4630
0
        return NVME_FDP_DISABLED | NVME_DNR;
4631
0
    }
4632
4633
0
    maxnpid = n->subsys->endgrp.fdp.nrg * n->subsys->endgrp.fdp.nruh;
4634
4635
0
    if (unlikely(npid >= MIN(NVME_FDP_MAXPIDS, maxnpid))) {
4636
0
        return NVME_INVALID_FIELD | NVME_DNR;
4637
0
    }
4638
4639
0
    pids = g_new(uint16_t, npid);
4640
4641
0
    ret = nvme_h2c(n, pids, npid * sizeof(uint16_t), req);
4642
0
    if (ret) {
4643
0
        return ret;
4644
0
    }
4645
4646
0
    for (; i < npid; i++) {
4647
0
        if (!nvme_update_ruh(n, ns, pids[i])) {
4648
0
            return NVME_INVALID_FIELD | NVME_DNR;
4649
0
        }
4650
0
    }
4651
4652
0
    return ret;
4653
0
}
4654
4655
static uint16_t nvme_io_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4656
0
{
4657
0
    NvmeCmd *cmd = &req->cmd;
4658
0
    uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4659
0
    uint8_t mo = (cdw10 & 0xff);
4660
4661
0
    switch (mo) {
4662
0
    case NVME_IOMS_MO_NOP:
4663
0
        return 0;
4664
0
    case NVME_IOMS_MO_RUH_UPDATE:
4665
0
        return nvme_io_mgmt_send_ruh_update(n, req);
4666
    /* if you add something here, please update nvme_set_migration_blockers() */
4667
0
    default:
4668
0
        return NVME_INVALID_FIELD | NVME_DNR;
4669
0
    };
4670
0
}
4671
4672
static uint16_t __nvme_io_cmd_nvm(NvmeCtrl *n, NvmeRequest *req)
4673
0
{
4674
0
    switch (req->cmd.opcode) {
4675
0
    case NVME_CMD_WRITE:
4676
0
        return nvme_write(n, req);
4677
0
    case NVME_CMD_READ:
4678
0
        return nvme_read(n, req);
4679
0
    case NVME_CMD_COMPARE:
4680
0
        return nvme_compare(n, req);
4681
0
    case NVME_CMD_WRITE_ZEROES:
4682
0
        return nvme_write_zeroes(n, req);
4683
0
    case NVME_CMD_DSM:
4684
0
        return nvme_dsm(n, req);
4685
0
    case NVME_CMD_VERIFY:
4686
0
        return nvme_verify(n, req);
4687
0
    case NVME_CMD_COPY:
4688
0
        return nvme_copy(n, req);
4689
0
    case NVME_CMD_IO_MGMT_RECV:
4690
0
        return nvme_io_mgmt_recv(n, req);
4691
0
    case NVME_CMD_IO_MGMT_SEND:
4692
0
        return nvme_io_mgmt_send(n, req);
4693
0
    }
4694
4695
0
    g_assert_not_reached();
4696
0
}
4697
4698
static uint16_t nvme_io_cmd_nvm(NvmeCtrl *n, NvmeRequest *req)
4699
0
{
4700
0
    if (!(n->cse.iocs.nvm[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
4701
0
        trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
4702
0
        return NVME_INVALID_OPCODE | NVME_DNR;
4703
0
    }
4704
4705
0
    return __nvme_io_cmd_nvm(n, req);
4706
0
}
4707
4708
static uint16_t nvme_io_cmd_zoned(NvmeCtrl *n, NvmeRequest *req)
4709
0
{
4710
0
    if (!(n->cse.iocs.zoned[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
4711
0
        trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
4712
0
        return NVME_INVALID_OPCODE | NVME_DNR;
4713
0
    }
4714
4715
0
    switch (req->cmd.opcode) {
4716
0
    case NVME_CMD_ZONE_APPEND:
4717
0
        return nvme_zone_append(n, req);
4718
0
    case NVME_CMD_ZONE_MGMT_SEND:
4719
0
        return nvme_zone_mgmt_send(n, req);
4720
0
    case NVME_CMD_ZONE_MGMT_RECV:
4721
0
        return nvme_zone_mgmt_recv(n, req);
4722
0
    }
4723
4724
0
    return __nvme_io_cmd_nvm(n, req);
4725
0
}
4726
4727
static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req)
4728
0
{
4729
0
    NvmeNamespace *ns;
4730
0
    uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4731
4732
0
    trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req),
4733
0
                          req->cmd.opcode, nvme_io_opc_str(req->cmd.opcode));
4734
4735
    /*
4736
     * In the base NVM command set, Flush may apply to all namespaces
4737
     * (indicated by NSID being set to FFFFFFFFh). But if that feature is used
4738
     * along with TP 4056 (Namespace Types), it may be pretty screwed up.
4739
     *
4740
     * If NSID is indeed set to FFFFFFFFh, we simply cannot associate the
4741
     * opcode with a specific command since we cannot determine a unique I/O
4742
     * command set. Opcode 0h could have any other meaning than something
4743
     * equivalent to flushing and say it DOES have completely different
4744
     * semantics in some other command set - does an NSID of FFFFFFFFh then
4745
     * mean "for all namespaces, apply whatever command set specific command
4746
     * that uses the 0h opcode?" Or does it mean "for all namespaces, apply
4747
     * whatever command that uses the 0h opcode if, and only if, it allows NSID
4748
     * to be FFFFFFFFh"?
4749
     *
4750
     * Anyway (and luckily), for now, we do not care about this since the
4751
     * device only supports namespace types that includes the NVM Flush command
4752
     * (NVM and Zoned), so always do an NVM Flush.
4753
     */
4754
4755
0
    if (req->cmd.opcode == NVME_CMD_FLUSH) {
4756
0
        return nvme_flush(n, req);
4757
0
    }
4758
4759
0
    if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
4760
0
        return NVME_INVALID_NSID | NVME_DNR;
4761
0
    }
4762
4763
0
    ns = nvme_ns(n, nsid);
4764
0
    if (unlikely(!ns)) {
4765
0
        return NVME_INVALID_FIELD | NVME_DNR;
4766
0
    }
4767
4768
0
    if (ns->status) {
4769
0
        return ns->status;
4770
0
    }
4771
4772
0
    if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
4773
0
        return NVME_INVALID_FIELD;
4774
0
    }
4775
4776
0
    req->ns = ns;
4777
4778
0
    switch (ns->csi) {
4779
0
    case NVME_CSI_NVM:
4780
0
        return nvme_io_cmd_nvm(n, req);
4781
0
    case NVME_CSI_ZONED:
4782
0
        return nvme_io_cmd_zoned(n, req);
4783
0
    }
4784
4785
0
    g_assert_not_reached();
4786
0
}
4787
4788
static void nvme_cq_notifier(EventNotifier *e)
4789
0
{
4790
0
    NvmeCQueue *cq = container_of(e, NvmeCQueue, notifier);
4791
0
    NvmeCtrl *n = cq->ctrl;
4792
0
    bool pending = cq->tail != cq->head;
4793
4794
0
    if (!event_notifier_test_and_clear(e)) {
4795
0
        return;
4796
0
    }
4797
4798
0
    nvme_update_cq_eventidx(cq);
4799
0
    nvme_update_cq_head(cq);
4800
4801
0
    if (cq->tail == cq->head) {
4802
0
        if (cq->irq_enabled && pending) {
4803
0
            n->cq_pending--;
4804
0
        }
4805
4806
0
        nvme_irq_deassert(n, cq);
4807
0
    }
4808
4809
0
    qemu_bh_schedule(cq->bh);
4810
0
}
4811
4812
static int nvme_init_cq_ioeventfd(NvmeCQueue *cq)
4813
0
{
4814
0
    NvmeCtrl *n = cq->ctrl;
4815
0
    uint16_t offset = (cq->cqid << 3) + (1 << 2);
4816
0
    int ret;
4817
4818
0
    ret = event_notifier_init(&cq->notifier, 0);
4819
0
    if (ret < 0) {
4820
0
        return ret;
4821
0
    }
4822
4823
0
    event_notifier_set_handler(&cq->notifier, nvme_cq_notifier);
4824
0
    memory_region_add_eventfd(&n->iomem,
4825
0
                              0x1000 + offset, 4, false, 0, &cq->notifier);
4826
4827
0
    return 0;
4828
0
}
4829
4830
static void nvme_sq_notifier(EventNotifier *e)
4831
0
{
4832
0
    NvmeSQueue *sq = container_of(e, NvmeSQueue, notifier);
4833
4834
0
    if (!event_notifier_test_and_clear(e)) {
4835
0
        return;
4836
0
    }
4837
4838
0
    nvme_process_sq(sq);
4839
0
}
4840
4841
static int nvme_init_sq_ioeventfd(NvmeSQueue *sq)
4842
0
{
4843
0
    NvmeCtrl *n = sq->ctrl;
4844
0
    uint16_t offset = sq->sqid << 3;
4845
0
    int ret;
4846
4847
0
    ret = event_notifier_init(&sq->notifier, 0);
4848
0
    if (ret < 0) {
4849
0
        return ret;
4850
0
    }
4851
4852
0
    event_notifier_set_handler(&sq->notifier, nvme_sq_notifier);
4853
0
    memory_region_add_eventfd(&n->iomem,
4854
0
                              0x1000 + offset, 4, false, 0, &sq->notifier);
4855
4856
0
    return 0;
4857
0
}
4858
4859
/*
4860
 * A pending Async Event Request has no aiocb (nvme_aer() parks it without
4861
 * issuing any block I/O), so there is nothing to cancel; just drop it.
4862
 */
4863
static void nvme_sq_cancel_inflight(NvmeSQueue *sq, uint16_t status)
4864
0
{
4865
0
    NvmeRequest *r;
4866
4867
0
    while (!QTAILQ_EMPTY(&sq->out_req_list)) {
4868
0
        r = QTAILQ_FIRST(&sq->out_req_list);
4869
0
        r->status = status;
4870
4871
0
        if (r->aiocb) {
4872
0
            blk_aio_cancel(r->aiocb);
4873
0
        } else {
4874
0
            QTAILQ_REMOVE(&sq->out_req_list, r, entry);
4875
0
        }
4876
0
    }
4877
0
}
4878
4879
static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
4880
0
{
4881
0
    uint16_t offset = sq->sqid << 3;
4882
4883
0
    n->sq[sq->sqid] = NULL;
4884
0
    qemu_bh_delete(sq->bh);
4885
0
    if (sq->ioeventfd_enabled) {
4886
0
        memory_region_del_eventfd(&n->iomem,
4887
0
                                  0x1000 + offset, 4, false, 0, &sq->notifier);
4888
0
        event_notifier_set_handler(&sq->notifier, NULL);
4889
0
        event_notifier_cleanup(&sq->notifier);
4890
0
    }
4891
0
    g_free(sq->io_req);
4892
0
    if (sq->sqid) {
4893
0
        g_free(sq);
4894
0
    }
4895
0
}
4896
4897
static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req)
4898
0
{
4899
0
    NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
4900
0
    NvmeRequest *r, *next;
4901
0
    NvmeSQueue *sq;
4902
0
    NvmeCQueue *cq;
4903
0
    uint16_t qid = le16_to_cpu(c->qid);
4904
4905
0
    if (unlikely(!qid || nvme_check_sqid(n, qid))) {
4906
0
        trace_pci_nvme_err_invalid_del_sq(qid);
4907
0
        return NVME_INVALID_QID | NVME_DNR;
4908
0
    }
4909
4910
0
    trace_pci_nvme_del_sq(qid);
4911
4912
0
    sq = n->sq[qid];
4913
0
    nvme_sq_cancel_inflight(sq, NVME_CMD_ABORT_SQ_DEL);
4914
4915
0
    if (!nvme_check_cqid(n, sq->cqid)) {
4916
0
        cq = n->cq[sq->cqid];
4917
0
        QTAILQ_REMOVE(&cq->sq_list, sq, entry);
4918
4919
0
        nvme_post_cqes(cq);
4920
0
        QTAILQ_FOREACH_SAFE(r, &cq->req_list, entry, next) {
4921
0
            if (r->sq == sq) {
4922
0
                QTAILQ_REMOVE(&cq->req_list, r, entry);
4923
0
                QTAILQ_INSERT_TAIL(&sq->req_list, r, entry);
4924
0
            }
4925
0
        }
4926
0
    }
4927
4928
0
    nvme_free_sq(sq, n);
4929
0
    return NVME_SUCCESS;
4930
0
}
4931
4932
static void __nvme_init_sq(NvmeSQueue *sq)
4933
0
{
4934
0
    NvmeCtrl *n = sq->ctrl;
4935
0
    uint16_t sqid = sq->sqid;
4936
0
    uint16_t cqid = sq->cqid;
4937
0
    int i;
4938
0
    NvmeCQueue *cq;
4939
4940
0
    sq->io_req = g_new0(NvmeRequest, sq->size);
4941
4942
0
    QTAILQ_INIT(&sq->req_list);
4943
0
    QTAILQ_INIT(&sq->out_req_list);
4944
0
    for (i = 0; i < sq->size; i++) {
4945
0
        sq->io_req[i].sq = sq;
4946
0
        QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
4947
0
    }
4948
4949
0
    sq->bh = qemu_bh_new_guarded(nvme_process_sq, sq,
4950
0
                                 &DEVICE(sq->ctrl)->mem_reentrancy_guard);
4951
4952
0
    if (n->dbbuf_enabled) {
4953
0
        sq->db_addr = n->dbbuf_dbs + (sqid << 3);
4954
0
        sq->ei_addr = n->dbbuf_eis + (sqid << 3);
4955
4956
0
        if (n->params.ioeventfd && sq->sqid != 0) {
4957
0
            if (!nvme_init_sq_ioeventfd(sq)) {
4958
0
                sq->ioeventfd_enabled = true;
4959
0
            }
4960
0
        }
4961
0
    }
4962
4963
0
    assert(n->cq[cqid]);
4964
0
    cq = n->cq[cqid];
4965
0
    QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
4966
0
    n->sq[sqid] = sq;
4967
0
}
4968
4969
static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
4970
                         uint16_t sqid, uint16_t cqid, uint16_t size)
4971
0
{
4972
0
    sq->ctrl = n;
4973
0
    sq->dma_addr = dma_addr;
4974
0
    sq->sqid = sqid;
4975
0
    sq->size = size;
4976
0
    sq->cqid = cqid;
4977
0
    sq->head = sq->tail = 0;
4978
0
    __nvme_init_sq(sq);
4979
0
}
4980
4981
static void nvme_restore_sq(NvmeSQueue *sq_from)
4982
0
{
4983
0
    NvmeCtrl *n = sq_from->ctrl;
4984
0
    NvmeSQueue *sq = sq_from;
4985
4986
0
    if (sq_from->sqid == 0) {
4987
0
        sq = &n->admin_sq;
4988
0
        sq->ctrl = n;
4989
0
        sq->dma_addr = sq_from->dma_addr;
4990
0
        sq->sqid = sq_from->sqid;
4991
0
        sq->size = sq_from->size;
4992
0
        sq->cqid = sq_from->cqid;
4993
0
        sq->head = sq_from->head;
4994
0
        sq->tail = sq_from->tail;
4995
0
    }
4996
4997
0
    __nvme_init_sq(sq);
4998
0
}
4999
5000
static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req)
5001
0
{
5002
0
    NvmeSQueue *sq;
5003
0
    NvmeCreateSq *c = (NvmeCreateSq *)&req->cmd;
5004
5005
0
    uint16_t cqid = le16_to_cpu(c->cqid);
5006
0
    uint16_t sqid = le16_to_cpu(c->sqid);
5007
0
    uint16_t qsize = le16_to_cpu(c->qsize);
5008
0
    uint16_t qflags = le16_to_cpu(c->sq_flags);
5009
0
    uint64_t prp1 = le64_to_cpu(c->prp1);
5010
5011
0
    trace_pci_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
5012
5013
0
    if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
5014
0
        trace_pci_nvme_err_invalid_create_sq_cqid(cqid);
5015
0
        return NVME_INVALID_CQID | NVME_DNR;
5016
0
    }
5017
0
    if (unlikely(!sqid || sqid > n->conf_ioqpairs || n->sq[sqid] != NULL)) {
5018
0
        trace_pci_nvme_err_invalid_create_sq_sqid(sqid);
5019
0
        return NVME_INVALID_QID | NVME_DNR;
5020
0
    }
5021
0
    if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
5022
0
        trace_pci_nvme_err_invalid_create_sq_size(qsize);
5023
0
        return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
5024
0
    }
5025
0
    if (unlikely(prp1 & (n->page_size - 1))) {
5026
0
        trace_pci_nvme_err_invalid_create_sq_addr(prp1);
5027
0
        return NVME_INVALID_PRP_OFFSET | NVME_DNR;
5028
0
    }
5029
0
    if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
5030
0
        trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
5031
0
        return NVME_INVALID_FIELD | NVME_DNR;
5032
0
    }
5033
0
    sq = g_malloc0(sizeof(*sq));
5034
0
    nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
5035
0
    return NVME_SUCCESS;
5036
0
}
5037
5038
struct nvme_stats {
5039
    uint64_t units_read;
5040
    uint64_t units_written;
5041
    uint64_t read_commands;
5042
    uint64_t write_commands;
5043
};
5044
5045
static void nvme_set_blk_stats(NvmeNamespace *ns, struct nvme_stats *stats)
5046
0
{
5047
0
    BlockAcctStats *s = blk_get_stats(ns->blkconf.blk);
5048
5049
0
    stats->units_read += s->nr_bytes[BLOCK_ACCT_READ];
5050
0
    stats->units_written += s->nr_bytes[BLOCK_ACCT_WRITE];
5051
0
    stats->read_commands += s->nr_ops[BLOCK_ACCT_READ];
5052
0
    stats->write_commands += s->nr_ops[BLOCK_ACCT_WRITE];
5053
0
}
5054
5055
static uint16_t nvme_ocp_extended_smart_info(NvmeCtrl *n, uint8_t rae,
5056
                                             uint32_t buf_len, uint64_t off,
5057
                                             NvmeRequest *req)
5058
0
{
5059
0
    NvmeNamespace *ns = NULL;
5060
0
    NvmeSmartLogExtended smart_l = { 0 };
5061
0
    struct nvme_stats stats = { 0 };
5062
0
    uint32_t trans_len;
5063
5064
0
    if (off >= sizeof(smart_l)) {
5065
0
        return NVME_INVALID_FIELD | NVME_DNR;
5066
0
    }
5067
5068
    /* accumulate all stats from all namespaces */
5069
0
    for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5070
0
        ns = nvme_ns(n, i);
5071
0
        if (ns) {
5072
0
            nvme_set_blk_stats(ns, &stats);
5073
0
        }
5074
0
    }
5075
5076
0
    smart_l.physical_media_units_written[0] = cpu_to_le64(stats.units_written);
5077
0
    smart_l.physical_media_units_read[0] = cpu_to_le64(stats.units_read);
5078
0
    smart_l.log_page_version = 0x0005;
5079
5080
0
    static const uint8_t guid[16] = {
5081
0
        0xC5, 0xAF, 0x10, 0x28, 0xEA, 0xBF, 0xF2, 0xA4,
5082
0
        0x9C, 0x4F, 0x6F, 0x7C, 0xC9, 0x14, 0xD5, 0xAF
5083
0
    };
5084
0
    memcpy(smart_l.log_page_guid, guid, sizeof(smart_l.log_page_guid));
5085
5086
0
    if (!rae) {
5087
0
        nvme_clear_events(n, NVME_AER_TYPE_SMART);
5088
0
    }
5089
5090
0
    trans_len = MIN(sizeof(smart_l) - off, buf_len);
5091
0
    return nvme_c2h(n, (uint8_t *) &smart_l + off, trans_len, req);
5092
0
}
5093
5094
static uint16_t nvme_smart_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
5095
                                uint64_t off, NvmeRequest *req)
5096
0
{
5097
0
    uint32_t nsid = le32_to_cpu(req->cmd.nsid);
5098
0
    struct nvme_stats stats = { 0 };
5099
0
    NvmeSmartLog smart = { 0 };
5100
0
    uint32_t trans_len;
5101
0
    NvmeNamespace *ns;
5102
0
    time_t current_ms;
5103
0
    uint64_t u_read, u_written;
5104
5105
0
    if (off >= sizeof(smart)) {
5106
0
        return NVME_INVALID_FIELD | NVME_DNR;
5107
0
    }
5108
5109
0
    if (nsid != 0xffffffff) {
5110
0
        ns = nvme_ns(n, nsid);
5111
0
        if (!ns) {
5112
0
            return NVME_INVALID_NSID | NVME_DNR;
5113
0
        }
5114
0
        nvme_set_blk_stats(ns, &stats);
5115
0
    } else {
5116
0
        int i;
5117
5118
0
        for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5119
0
            ns = nvme_ns(n, i);
5120
0
            if (!ns) {
5121
0
                continue;
5122
0
            }
5123
0
            nvme_set_blk_stats(ns, &stats);
5124
0
        }
5125
0
    }
5126
5127
0
    trans_len = MIN(sizeof(smart) - off, buf_len);
5128
0
    smart.critical_warning = n->smart_critical_warning;
5129
5130
0
    u_read = DIV_ROUND_UP(stats.units_read >> BDRV_SECTOR_BITS, 1000);
5131
0
    u_written = DIV_ROUND_UP(stats.units_written >> BDRV_SECTOR_BITS, 1000);
5132
5133
0
    smart.data_units_read[0] = cpu_to_le64(u_read);
5134
0
    smart.data_units_written[0] = cpu_to_le64(u_written);
5135
0
    smart.host_read_commands[0] = cpu_to_le64(stats.read_commands);
5136
0
    smart.host_write_commands[0] = cpu_to_le64(stats.write_commands);
5137
5138
0
    smart.temperature = cpu_to_le16(n->temperature);
5139
5140
0
    if ((n->temperature >= n->features.temp_thresh_hi) ||
5141
0
        (n->temperature <= n->features.temp_thresh_low)) {
5142
0
        smart.critical_warning |= NVME_SMART_TEMPERATURE;
5143
0
    }
5144
5145
0
    current_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
5146
0
    smart.power_on_hours[0] =
5147
0
        cpu_to_le64((((current_ms - n->starttime_ms) / 1000) / 60) / 60);
5148
5149
0
    if (!rae) {
5150
0
        nvme_clear_events(n, NVME_AER_TYPE_SMART);
5151
0
    }
5152
5153
0
    return nvme_c2h(n, (uint8_t *) &smart + off, trans_len, req);
5154
0
}
5155
5156
static uint16_t nvme_endgrp_info(NvmeCtrl *n,  uint8_t rae, uint32_t buf_len,
5157
                                 uint64_t off, NvmeRequest *req)
5158
0
{
5159
0
    uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
5160
0
    uint16_t endgrpid = (dw11 >> 16) & 0xffff;
5161
0
    struct nvme_stats stats = {};
5162
0
    NvmeEndGrpLog info = {};
5163
0
    int i;
5164
5165
0
    if (!n->subsys || endgrpid != 0x1) {
5166
0
        return NVME_INVALID_FIELD | NVME_DNR;
5167
0
    }
5168
5169
0
    if (off >= sizeof(info)) {
5170
0
        return NVME_INVALID_FIELD | NVME_DNR;
5171
0
    }
5172
5173
0
    for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5174
0
        NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
5175
0
        if (!ns) {
5176
0
            continue;
5177
0
        }
5178
5179
0
        nvme_set_blk_stats(ns, &stats);
5180
0
    }
5181
5182
0
    info.data_units_read[0] =
5183
0
        cpu_to_le64(DIV_ROUND_UP(stats.units_read / 1000000000, 1000000000));
5184
0
    info.data_units_written[0] =
5185
0
        cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
5186
0
    info.media_units_written[0] =
5187
0
        cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
5188
5189
0
    info.host_read_commands[0] = cpu_to_le64(stats.read_commands);
5190
0
    info.host_write_commands[0] = cpu_to_le64(stats.write_commands);
5191
5192
0
    buf_len = MIN(sizeof(info) - off, buf_len);
5193
5194
0
    return nvme_c2h(n, (uint8_t *)&info + off, buf_len, req);
5195
0
}
5196
5197
5198
static uint16_t nvme_fw_log_info(NvmeCtrl *n, uint32_t buf_len, uint64_t off,
5199
                                 NvmeRequest *req)
5200
0
{
5201
0
    uint32_t trans_len;
5202
0
    NvmeFwSlotInfoLog fw_log = {
5203
0
        .afi = 0x1,
5204
0
    };
5205
5206
0
    if (off >= sizeof(fw_log)) {
5207
0
        return NVME_INVALID_FIELD | NVME_DNR;
5208
0
    }
5209
5210
0
    strpadcpy((char *)&fw_log.frs1, sizeof(fw_log.frs1), "1.0", ' ');
5211
0
    trans_len = MIN(sizeof(fw_log) - off, buf_len);
5212
5213
0
    return nvme_c2h(n, (uint8_t *) &fw_log + off, trans_len, req);
5214
0
}
5215
5216
static uint16_t nvme_error_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
5217
                                uint64_t off, NvmeRequest *req)
5218
0
{
5219
0
    uint32_t trans_len;
5220
0
    NvmeErrorLog errlog;
5221
5222
0
    if (off >= sizeof(errlog)) {
5223
0
        return NVME_INVALID_FIELD | NVME_DNR;
5224
0
    }
5225
5226
0
    if (!rae) {
5227
0
        nvme_clear_events(n, NVME_AER_TYPE_ERROR);
5228
0
    }
5229
5230
0
    memset(&errlog, 0x0, sizeof(errlog));
5231
0
    trans_len = MIN(sizeof(errlog) - off, buf_len);
5232
5233
0
    return nvme_c2h(n, (uint8_t *)&errlog, trans_len, req);
5234
0
}
5235
5236
static uint16_t nvme_changed_nslist(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
5237
                                    uint64_t off, NvmeRequest *req)
5238
0
{
5239
0
    uint32_t nslist[1024] = {};
5240
0
    uint32_t trans_len;
5241
0
    int i = 0;
5242
0
    uint32_t nsid;
5243
5244
0
    if (off >= sizeof(nslist)) {
5245
0
        trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(nslist));
5246
0
        return NVME_INVALID_FIELD | NVME_DNR;
5247
0
    }
5248
5249
0
    trans_len = MIN(sizeof(nslist) - off, buf_len);
5250
5251
0
    while ((nsid = find_first_bit(n->changed_nsids, NVME_CHANGED_NSID_SIZE)) !=
5252
0
            NVME_CHANGED_NSID_SIZE) {
5253
        /*
5254
         * If more than 1024 namespaces, the first entry in the log page should
5255
         * be set to FFFFFFFFh and the others to 0 as spec.
5256
         */
5257
0
        if (i == ARRAY_SIZE(nslist)) {
5258
0
            memset(nslist, 0x0, sizeof(nslist));
5259
0
            nslist[0] = 0xffffffff;
5260
0
            break;
5261
0
        }
5262
5263
0
        nslist[i++] = nsid;
5264
0
        clear_bit(nsid, n->changed_nsids);
5265
0
    }
5266
5267
    /*
5268
     * Remove all the remaining list entries in case returns directly due to
5269
     * more than 1024 namespaces.
5270
     */
5271
0
    if (nslist[0] == 0xffffffff) {
5272
0
        bitmap_zero(n->changed_nsids, NVME_CHANGED_NSID_SIZE);
5273
0
    }
5274
5275
0
    if (!rae) {
5276
0
        nvme_clear_events(n, NVME_AER_TYPE_NOTICE);
5277
0
    }
5278
5279
0
    return nvme_c2h(n, ((uint8_t *)nslist) + off, trans_len, req);
5280
0
}
5281
5282
static uint16_t nvme_cmd_effects(NvmeCtrl *n, uint8_t csi, uint32_t buf_len,
5283
                                 uint64_t off, NvmeRequest *req)
5284
0
{
5285
0
    NvmeEffectsLog log = {};
5286
0
    const uint32_t *iocs = NULL;
5287
0
    uint32_t trans_len;
5288
5289
0
    if (off >= sizeof(log)) {
5290
0
        trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(log));
5291
0
        return NVME_INVALID_FIELD | NVME_DNR;
5292
0
    }
5293
5294
0
    switch (NVME_CC_CSS(ldl_le_p(&n->bar.cc))) {
5295
0
    case NVME_CC_CSS_NVM:
5296
0
        iocs = n->cse.iocs.nvm;
5297
0
        break;
5298
5299
0
    case NVME_CC_CSS_ALL:
5300
0
        switch (csi) {
5301
0
        case NVME_CSI_NVM:
5302
0
            iocs = n->cse.iocs.nvm;
5303
0
            break;
5304
0
        case NVME_CSI_ZONED:
5305
0
            iocs = n->cse.iocs.zoned;
5306
0
            break;
5307
0
        }
5308
5309
0
        break;
5310
0
    }
5311
5312
0
    memcpy(log.acs, n->cse.acs, sizeof(log.acs));
5313
5314
0
    if (iocs) {
5315
0
        memcpy(log.iocs, iocs, sizeof(log.iocs));
5316
0
    }
5317
5318
0
    trans_len = MIN(sizeof(log) - off, buf_len);
5319
5320
0
    return nvme_c2h(n, ((uint8_t *)&log) + off, trans_len, req);
5321
0
}
5322
5323
static uint16_t nvme_vendor_specific_log(NvmeCtrl *n, uint8_t rae,
5324
                                         uint32_t buf_len, uint64_t off,
5325
                                         NvmeRequest *req, uint8_t lid)
5326
0
{
5327
0
    switch (lid) {
5328
0
    case NVME_OCP_EXTENDED_SMART_INFO:
5329
0
        if (n->params.ocp) {
5330
0
            return nvme_ocp_extended_smart_info(n, rae, buf_len, off, req);
5331
0
        }
5332
0
        break;
5333
        /* add a case for each additional vendor specific log id */
5334
0
    }
5335
5336
0
    trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid);
5337
0
    return NVME_INVALID_FIELD | NVME_DNR;
5338
0
}
5339
5340
static size_t sizeof_fdp_conf_descr(size_t nruh, size_t vss)
5341
0
{
5342
0
    size_t entry_siz = sizeof(NvmeFdpDescrHdr) + nruh * sizeof(NvmeRuhDescr)
5343
0
                       + vss;
5344
0
    return ROUND_UP(entry_siz, 8);
5345
0
}
5346
5347
static uint16_t nvme_fdp_confs(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
5348
                               uint64_t off, NvmeRequest *req)
5349
0
{
5350
0
    uint32_t log_size, trans_len;
5351
0
    g_autofree uint8_t *buf = NULL;
5352
0
    NvmeFdpDescrHdr *hdr;
5353
0
    NvmeRuhDescr *ruhd;
5354
0
    NvmeEnduranceGroup *endgrp;
5355
0
    NvmeFdpConfsHdr *log;
5356
0
    size_t nruh, fdp_descr_size;
5357
0
    int i;
5358
5359
0
    if (endgrpid != 1 || !n->subsys) {
5360
0
        return NVME_INVALID_FIELD | NVME_DNR;
5361
0
    }
5362
5363
0
    endgrp = &n->subsys->endgrp;
5364
5365
0
    if (endgrp->fdp.enabled) {
5366
0
        nruh = endgrp->fdp.nruh;
5367
0
    } else {
5368
0
        nruh = 1;
5369
0
    }
5370
5371
0
    fdp_descr_size = sizeof_fdp_conf_descr(nruh, FDPVSS);
5372
0
    log_size = sizeof(NvmeFdpConfsHdr) + fdp_descr_size;
5373
5374
0
    if (off >= log_size) {
5375
0
        return NVME_INVALID_FIELD | NVME_DNR;
5376
0
    }
5377
5378
0
    trans_len = MIN(log_size - off, buf_len);
5379
5380
0
    buf = g_malloc0(log_size);
5381
0
    log = (NvmeFdpConfsHdr *)buf;
5382
0
    hdr = (NvmeFdpDescrHdr *)(log + 1);
5383
0
    ruhd = (NvmeRuhDescr *)(buf + sizeof(*log) + sizeof(*hdr));
5384
5385
0
    log->num_confs = cpu_to_le16(0);
5386
0
    log->size = cpu_to_le32(log_size);
5387
5388
0
    hdr->descr_size = cpu_to_le16(fdp_descr_size);
5389
0
    if (endgrp->fdp.enabled) {
5390
0
        hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, VALID, 1);
5391
0
        hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, RGIF, endgrp->fdp.rgif);
5392
0
        hdr->nrg = cpu_to_le16(endgrp->fdp.nrg);
5393
0
        hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
5394
0
        hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
5395
0
        hdr->nnss = cpu_to_le32(NVME_MAX_NAMESPACES);
5396
0
        hdr->runs = cpu_to_le64(endgrp->fdp.runs);
5397
5398
0
        for (i = 0; i < nruh; i++) {
5399
0
            ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
5400
0
            ruhd++;
5401
0
        }
5402
0
    } else {
5403
        /* 1 bit for RUH in PIF -> 2 RUHs max. */
5404
0
        hdr->nrg = cpu_to_le16(1);
5405
0
        hdr->nruh = cpu_to_le16(1);
5406
0
        hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
5407
0
        hdr->nnss = cpu_to_le32(1);
5408
0
        hdr->runs = cpu_to_le64(96 * MiB);
5409
5410
0
        ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
5411
0
    }
5412
5413
0
    return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
5414
0
}
5415
5416
static uint16_t nvme_fdp_ruh_usage(NvmeCtrl *n, uint32_t endgrpid,
5417
                                   uint32_t dw10, uint32_t dw12,
5418
                                   uint32_t buf_len, uint64_t off,
5419
                                   NvmeRequest *req)
5420
0
{
5421
0
    NvmeRuHandle *ruh;
5422
0
    NvmeRuhuLog *hdr;
5423
0
    NvmeRuhuDescr *ruhud;
5424
0
    NvmeEnduranceGroup *endgrp;
5425
0
    g_autofree uint8_t *buf = NULL;
5426
0
    uint32_t log_size, trans_len;
5427
0
    uint16_t i;
5428
5429
0
    if (endgrpid != 1 || !n->subsys) {
5430
0
        return NVME_INVALID_FIELD | NVME_DNR;
5431
0
    }
5432
5433
0
    endgrp = &n->subsys->endgrp;
5434
5435
0
    if (!endgrp->fdp.enabled) {
5436
0
        return NVME_FDP_DISABLED | NVME_DNR;
5437
0
    }
5438
5439
0
    log_size = sizeof(NvmeRuhuLog) + endgrp->fdp.nruh * sizeof(NvmeRuhuDescr);
5440
5441
0
    if (off >= log_size) {
5442
0
        return NVME_INVALID_FIELD | NVME_DNR;
5443
0
    }
5444
5445
0
    trans_len = MIN(log_size - off, buf_len);
5446
5447
0
    buf = g_malloc0(log_size);
5448
0
    hdr = (NvmeRuhuLog *)buf;
5449
0
    ruhud = (NvmeRuhuDescr *)(hdr + 1);
5450
5451
0
    ruh = endgrp->fdp.ruhs;
5452
0
    hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
5453
5454
0
    for (i = 0; i < endgrp->fdp.nruh; i++, ruhud++, ruh++) {
5455
0
        ruhud->ruha = ruh->ruha;
5456
0
    }
5457
5458
0
    return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
5459
0
}
5460
5461
static uint16_t nvme_fdp_stats(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
5462
                               uint64_t off, NvmeRequest *req)
5463
0
{
5464
0
    NvmeEnduranceGroup *endgrp;
5465
0
    NvmeFdpStatsLog log = {};
5466
0
    uint32_t trans_len;
5467
5468
0
    if (off >= sizeof(NvmeFdpStatsLog)) {
5469
0
        return NVME_INVALID_FIELD | NVME_DNR;
5470
0
    }
5471
5472
0
    if (endgrpid != 1 || !n->subsys) {
5473
0
        return NVME_INVALID_FIELD | NVME_DNR;
5474
0
    }
5475
5476
0
    if (!n->subsys->endgrp.fdp.enabled) {
5477
0
        return NVME_FDP_DISABLED | NVME_DNR;
5478
0
    }
5479
5480
0
    endgrp = &n->subsys->endgrp;
5481
5482
0
    trans_len = MIN(sizeof(log) - off, buf_len);
5483
5484
    /* spec value is 128 bit, we only use 64 bit */
5485
0
    log.hbmw[0] = cpu_to_le64(endgrp->fdp.hbmw);
5486
0
    log.mbmw[0] = cpu_to_le64(endgrp->fdp.mbmw);
5487
0
    log.mbe[0] = cpu_to_le64(endgrp->fdp.mbe);
5488
5489
0
    return nvme_c2h(n, (uint8_t *)&log + off, trans_len, req);
5490
0
}
5491
5492
static uint16_t nvme_fdp_events(NvmeCtrl *n, uint32_t endgrpid,
5493
                                uint32_t buf_len, uint64_t off,
5494
                                NvmeRequest *req)
5495
0
{
5496
0
    NvmeEnduranceGroup *endgrp;
5497
0
    NvmeCmd *cmd = &req->cmd;
5498
0
    bool host_events = (cmd->cdw10 >> 8) & 0x1;
5499
0
    uint32_t log_size, trans_len;
5500
0
    NvmeFdpEventBuffer *ebuf;
5501
0
    g_autofree NvmeFdpEventsLog *elog = NULL;
5502
0
    NvmeFdpEvent *event;
5503
5504
0
    if (endgrpid != 1 || !n->subsys) {
5505
0
        return NVME_INVALID_FIELD | NVME_DNR;
5506
0
    }
5507
5508
0
    endgrp = &n->subsys->endgrp;
5509
5510
0
    if (!endgrp->fdp.enabled) {
5511
0
        return NVME_FDP_DISABLED | NVME_DNR;
5512
0
    }
5513
5514
0
    if (host_events) {
5515
0
        ebuf = &endgrp->fdp.host_events;
5516
0
    } else {
5517
0
        ebuf = &endgrp->fdp.ctrl_events;
5518
0
    }
5519
5520
0
    log_size = sizeof(NvmeFdpEventsLog) + ebuf->nelems * sizeof(NvmeFdpEvent);
5521
5522
0
    if (off >= log_size) {
5523
0
        return NVME_INVALID_FIELD | NVME_DNR;
5524
0
    }
5525
5526
0
    trans_len = MIN(log_size - off, buf_len);
5527
0
    elog = g_malloc0(log_size);
5528
0
    elog->num_events = cpu_to_le32(ebuf->nelems);
5529
0
    event = (NvmeFdpEvent *)(elog + 1);
5530
5531
0
    if (ebuf->nelems && ebuf->start == ebuf->next) {
5532
0
        unsigned int nelems = (NVME_FDP_MAX_EVENTS - ebuf->start);
5533
        /* wrap over, copy [start;NVME_FDP_MAX_EVENTS[ and [0; next[ */
5534
0
        memcpy(event, &ebuf->events[ebuf->start],
5535
0
               sizeof(NvmeFdpEvent) * nelems);
5536
0
        memcpy(event + nelems, ebuf->events,
5537
0
               sizeof(NvmeFdpEvent) * ebuf->next);
5538
0
    } else if (ebuf->start < ebuf->next) {
5539
0
        memcpy(event, &ebuf->events[ebuf->start],
5540
0
               sizeof(NvmeFdpEvent) * (ebuf->next - ebuf->start));
5541
0
    }
5542
5543
0
    return nvme_c2h(n, (uint8_t *)elog + off, trans_len, req);
5544
0
}
5545
5546
static uint16_t nvme_get_log(NvmeCtrl *n, NvmeRequest *req)
5547
0
{
5548
0
    NvmeCmd *cmd = &req->cmd;
5549
5550
0
    uint32_t dw10 = le32_to_cpu(cmd->cdw10);
5551
0
    uint32_t dw11 = le32_to_cpu(cmd->cdw11);
5552
0
    uint32_t dw12 = le32_to_cpu(cmd->cdw12);
5553
0
    uint32_t dw13 = le32_to_cpu(cmd->cdw13);
5554
0
    uint8_t  lid = dw10 & 0xff;
5555
0
    uint8_t  lsp = (dw10 >> 8) & 0xf;
5556
0
    uint8_t  rae = (dw10 >> 15) & 0x1;
5557
0
    uint8_t  csi = le32_to_cpu(cmd->cdw14) >> 24;
5558
0
    uint32_t numdl, numdu, lspi;
5559
0
    uint64_t off, lpol, lpou;
5560
0
    size_t   len;
5561
0
    uint16_t status;
5562
5563
0
    numdl = (dw10 >> 16);
5564
0
    numdu = (dw11 & 0xffff);
5565
0
    lspi = (dw11 >> 16);
5566
0
    lpol = dw12;
5567
0
    lpou = dw13;
5568
5569
0
    len = (((numdu << 16) | numdl) + 1) << 2;
5570
0
    off = (lpou << 32ULL) | lpol;
5571
5572
0
    if (off & 0x3) {
5573
0
        return NVME_INVALID_FIELD | NVME_DNR;
5574
0
    }
5575
5576
0
    trace_pci_nvme_get_log(nvme_cid(req), lid, lsp, rae, len, off);
5577
5578
0
    status = nvme_check_mdts(n, len);
5579
0
    if (status) {
5580
0
        return status;
5581
0
    }
5582
5583
0
    switch (lid) {
5584
0
    case NVME_LOG_ERROR_INFO:
5585
0
        return nvme_error_info(n, rae, len, off, req);
5586
0
    case NVME_LOG_SMART_INFO:
5587
0
        return nvme_smart_info(n, rae, len, off, req);
5588
0
    case NVME_LOG_FW_SLOT_INFO:
5589
0
        return nvme_fw_log_info(n, len, off, req);
5590
0
    case NVME_LOG_VENDOR_START...NVME_LOG_VENDOR_END:
5591
0
        return nvme_vendor_specific_log(n, rae, len, off, req, lid);
5592
0
    case NVME_LOG_CHANGED_NSLIST:
5593
0
        return nvme_changed_nslist(n, rae, len, off, req);
5594
0
    case NVME_LOG_CMD_EFFECTS:
5595
0
        return nvme_cmd_effects(n, csi, len, off, req);
5596
0
    case NVME_LOG_ENDGRP:
5597
0
        return nvme_endgrp_info(n, rae, len, off, req);
5598
0
    case NVME_LOG_FDP_CONFS:
5599
0
        return nvme_fdp_confs(n, lspi, len, off, req);
5600
0
    case NVME_LOG_FDP_RUH_USAGE:
5601
0
        return nvme_fdp_ruh_usage(n, lspi, dw10, dw12, len, off, req);
5602
0
    case NVME_LOG_FDP_STATS:
5603
0
        return nvme_fdp_stats(n, lspi, len, off, req);
5604
0
    case NVME_LOG_FDP_EVENTS:
5605
0
        return nvme_fdp_events(n, lspi, len, off, req);
5606
0
    default:
5607
0
        trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid);
5608
0
        return NVME_INVALID_FIELD | NVME_DNR;
5609
0
    }
5610
0
}
5611
5612
static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
5613
0
{
5614
0
    PCIDevice *pci = PCI_DEVICE(n);
5615
0
    uint16_t offset = (cq->cqid << 3) + (1 << 2);
5616
5617
0
    n->cq[cq->cqid] = NULL;
5618
0
    qemu_bh_delete(cq->bh);
5619
0
    if (cq->ioeventfd_enabled) {
5620
0
        memory_region_del_eventfd(&n->iomem,
5621
0
                                  0x1000 + offset, 4, false, 0, &cq->notifier);
5622
0
        event_notifier_set_handler(&cq->notifier, NULL);
5623
0
        event_notifier_cleanup(&cq->notifier);
5624
0
    }
5625
0
    if (msix_present(pci) && cq->irq_enabled) {
5626
0
        msix_vector_unuse(pci, cq->vector);
5627
0
    }
5628
0
    if (cq->cqid) {
5629
0
        g_free(cq);
5630
0
    }
5631
0
}
5632
5633
static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
5634
0
{
5635
0
    NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
5636
0
    NvmeCQueue *cq;
5637
0
    uint16_t qid = le16_to_cpu(c->qid);
5638
5639
0
    if (unlikely(!qid || nvme_check_cqid(n, qid))) {
5640
0
        trace_pci_nvme_err_invalid_del_cq_cqid(qid);
5641
0
        return NVME_INVALID_CQID | NVME_DNR;
5642
0
    }
5643
5644
0
    cq = n->cq[qid];
5645
0
    if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
5646
0
        trace_pci_nvme_err_invalid_del_cq_notempty(qid);
5647
0
        return NVME_INVALID_QUEUE_DEL;
5648
0
    }
5649
5650
0
    if (cq->irq_enabled && cq->tail != cq->head) {
5651
0
        n->cq_pending--;
5652
0
    }
5653
5654
0
    nvme_irq_deassert(n, cq);
5655
0
    trace_pci_nvme_del_cq(qid);
5656
0
    nvme_free_cq(cq, n);
5657
0
    return NVME_SUCCESS;
5658
0
}
5659
5660
static void __nvme_init_cq(NvmeCQueue *cq)
5661
0
{
5662
0
    NvmeCtrl *n = cq->ctrl;
5663
0
    PCIDevice *pci = PCI_DEVICE(n);
5664
0
    uint16_t cqid = cq->cqid;
5665
5666
0
    if (msix_present(pci) && cq->irq_enabled) {
5667
0
        msix_vector_use(pci, cq->vector);
5668
0
    }
5669
5670
0
    QTAILQ_INIT(&cq->sq_list);
5671
0
    if (n->dbbuf_enabled) {
5672
0
        cq->db_addr = n->dbbuf_dbs + (cqid << 3) + (1 << 2);
5673
0
        cq->ei_addr = n->dbbuf_eis + (cqid << 3) + (1 << 2);
5674
5675
0
        if (n->params.ioeventfd && cqid != 0) {
5676
0
            if (!nvme_init_cq_ioeventfd(cq)) {
5677
0
                cq->ioeventfd_enabled = true;
5678
0
            }
5679
0
        }
5680
0
    }
5681
0
    n->cq[cqid] = cq;
5682
0
    cq->bh = qemu_bh_new_guarded(nvme_post_cqes, cq,
5683
0
                                 &DEVICE(cq->ctrl)->mem_reentrancy_guard);
5684
0
}
5685
5686
static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
5687
                         uint16_t cqid, uint16_t vector, uint16_t size,
5688
                         uint16_t irq_enabled)
5689
0
{
5690
0
    cq->ctrl = n;
5691
0
    cq->cqid = cqid;
5692
0
    cq->size = size;
5693
0
    cq->dma_addr = dma_addr;
5694
0
    cq->phase = 1;
5695
0
    cq->irq_enabled = irq_enabled;
5696
0
    cq->vector = vector;
5697
0
    cq->head = cq->tail = 0;
5698
0
    QTAILQ_INIT(&cq->req_list);
5699
0
    __nvme_init_cq(cq);
5700
0
}
5701
5702
static void move_cq_req_list(NvmeCQueue *cq_to, NvmeCQueue *cq_from)
5703
0
{
5704
0
    NvmeRequest *req, *next;
5705
5706
0
    QTAILQ_FOREACH_SAFE(req, &cq_from->req_list, entry, next) {
5707
0
        QTAILQ_REMOVE(&cq_from->req_list, req, entry);
5708
0
        QTAILQ_INSERT_TAIL(&cq_to->req_list, req, entry);
5709
0
    }
5710
0
}
5711
5712
static void nvme_restore_cq(NvmeCQueue *cq_from)
5713
0
{
5714
0
    NvmeCtrl *n = cq_from->ctrl;
5715
0
    NvmeCQueue *cq = cq_from;
5716
5717
0
    if (cq_from->cqid == 0) {
5718
0
        cq = &n->admin_cq;
5719
0
        cq->ctrl = n;
5720
0
        cq->cqid = cq_from->cqid;
5721
0
        cq->size = cq_from->size;
5722
0
        cq->dma_addr = cq_from->dma_addr;
5723
0
        cq->phase = cq_from->phase;
5724
0
        cq->irq_enabled = cq_from->irq_enabled;
5725
0
        cq->vector = cq_from->vector;
5726
0
        cq->head = cq_from->head;
5727
0
        cq->tail = cq_from->tail;
5728
0
        QTAILQ_INIT(&cq->req_list);
5729
0
        move_cq_req_list(cq, cq_from);
5730
0
    }
5731
5732
0
    __nvme_init_cq(cq);
5733
0
}
5734
5735
static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeRequest *req)
5736
0
{
5737
0
    NvmeCQueue *cq;
5738
0
    NvmeCreateCq *c = (NvmeCreateCq *)&req->cmd;
5739
0
    uint16_t cqid = le16_to_cpu(c->cqid);
5740
0
    uint16_t vector = le16_to_cpu(c->irq_vector);
5741
0
    uint16_t qsize = le16_to_cpu(c->qsize);
5742
0
    uint16_t qflags = le16_to_cpu(c->cq_flags);
5743
0
    uint64_t prp1 = le64_to_cpu(c->prp1);
5744
0
    uint32_t cc = ldq_le_p(&n->bar.cc);
5745
0
    uint8_t iocqes = NVME_CC_IOCQES(cc);
5746
0
    uint8_t iosqes = NVME_CC_IOSQES(cc);
5747
5748
0
    trace_pci_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
5749
0
                             NVME_CQ_FLAGS_IEN(qflags) != 0);
5750
5751
0
    if (iosqes != NVME_SQES || iocqes != NVME_CQES) {
5752
0
        trace_pci_nvme_err_invalid_create_cq_entry_size(iosqes, iocqes);
5753
0
        return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
5754
0
    }
5755
5756
0
    if (unlikely(!cqid || cqid > n->conf_ioqpairs || n->cq[cqid] != NULL)) {
5757
0
        trace_pci_nvme_err_invalid_create_cq_cqid(cqid);
5758
0
        return NVME_INVALID_QID | NVME_DNR;
5759
0
    }
5760
0
    if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
5761
0
        trace_pci_nvme_err_invalid_create_cq_size(qsize);
5762
0
        return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
5763
0
    }
5764
0
    if (unlikely(prp1 & (n->page_size - 1))) {
5765
0
        trace_pci_nvme_err_invalid_create_cq_addr(prp1);
5766
0
        return NVME_INVALID_PRP_OFFSET | NVME_DNR;
5767
0
    }
5768
0
    if (unlikely(!msix_enabled(PCI_DEVICE(n)) && vector)) {
5769
0
        trace_pci_nvme_err_invalid_create_cq_vector(vector);
5770
0
        return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5771
0
    }
5772
0
    if (unlikely(vector >= n->conf_msix_qsize)) {
5773
0
        trace_pci_nvme_err_invalid_create_cq_vector(vector);
5774
0
        return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5775
0
    }
5776
0
    if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
5777
0
        trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
5778
0
        return NVME_INVALID_FIELD | NVME_DNR;
5779
0
    }
5780
5781
0
    cq = g_malloc0(sizeof(*cq));
5782
0
    nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
5783
0
                 NVME_CQ_FLAGS_IEN(qflags));
5784
5785
    /*
5786
     * It is only required to set qs_created when creating a completion queue;
5787
     * creating a submission queue without a matching completion queue will
5788
     * fail.
5789
     */
5790
0
    n->qs_created = true;
5791
0
    return NVME_SUCCESS;
5792
0
}
5793
5794
static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl *n, NvmeRequest *req)
5795
0
{
5796
0
    uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5797
5798
0
    return nvme_c2h(n, id, sizeof(id), req);
5799
0
}
5800
5801
static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeRequest *req)
5802
0
{
5803
0
    trace_pci_nvme_identify_ctrl();
5804
5805
0
    return nvme_c2h(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl), req);
5806
0
}
5807
5808
static uint16_t nvme_identify_ctrl_csi(NvmeCtrl *n, NvmeRequest *req)
5809
0
{
5810
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5811
0
    uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5812
0
    NvmeIdCtrlNvm *id_nvm = (NvmeIdCtrlNvm *)&id;
5813
5814
0
    trace_pci_nvme_identify_ctrl_csi(c->csi);
5815
5816
0
    switch (c->csi) {
5817
0
    case NVME_CSI_NVM:
5818
0
        id_nvm->vsl = n->params.vsl;
5819
0
        id_nvm->dmrl = NVME_ID_CTRL_NVM_DMRL_MAX;
5820
0
        id_nvm->dmrsl = cpu_to_le32(n->dmrsl);
5821
0
        id_nvm->dmsl = NVME_ID_CTRL_NVM_DMRL_MAX * n->dmrsl;
5822
0
        break;
5823
5824
0
    case NVME_CSI_ZONED:
5825
0
        ((NvmeIdCtrlZoned *)&id)->zasl = n->params.zasl;
5826
0
        break;
5827
5828
0
    default:
5829
0
        return NVME_INVALID_FIELD | NVME_DNR;
5830
0
    }
5831
5832
0
    return nvme_c2h(n, id, sizeof(id), req);
5833
0
}
5834
5835
static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeRequest *req, bool active)
5836
0
{
5837
0
    NvmeNamespace *ns;
5838
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5839
0
    uint32_t nsid = le32_to_cpu(c->nsid);
5840
5841
0
    trace_pci_nvme_identify_ns(nsid);
5842
5843
0
    if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5844
0
        return NVME_INVALID_NSID | NVME_DNR;
5845
0
    }
5846
5847
0
    ns = nvme_ns(n, nsid);
5848
0
    if (unlikely(!ns)) {
5849
0
        if (!active) {
5850
0
            ns = nvme_subsys_ns(n->subsys, nsid);
5851
0
            if (!ns) {
5852
0
                return nvme_rpt_empty_id_struct(n, req);
5853
0
            }
5854
0
        } else {
5855
0
            return nvme_rpt_empty_id_struct(n, req);
5856
0
        }
5857
0
    }
5858
5859
0
    if (active || ns->csi == NVME_CSI_NVM) {
5860
0
        return nvme_c2h(n, (uint8_t *)&ns->id_ns, sizeof(NvmeIdNs), req);
5861
0
    }
5862
5863
0
    return NVME_INVALID_IOCS | NVME_DNR;
5864
0
}
5865
5866
static uint16_t nvme_identify_ctrl_list(NvmeCtrl *n, NvmeRequest *req,
5867
                                        bool attached)
5868
0
{
5869
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5870
0
    uint32_t nsid = le32_to_cpu(c->nsid);
5871
0
    uint16_t min_id = le16_to_cpu(c->ctrlid);
5872
0
    uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
5873
0
    uint16_t *ids = &list[1];
5874
0
    NvmeNamespace *ns;
5875
0
    NvmeCtrl *ctrl;
5876
0
    int cntlid, nr_ids = 0;
5877
5878
0
    trace_pci_nvme_identify_ctrl_list(c->cns, min_id);
5879
5880
0
    if (!n->subsys) {
5881
0
        return NVME_INVALID_FIELD | NVME_DNR;
5882
0
    }
5883
5884
0
    if (attached) {
5885
0
        if (nsid == NVME_NSID_BROADCAST) {
5886
0
            return NVME_INVALID_FIELD | NVME_DNR;
5887
0
        }
5888
5889
0
        ns = nvme_subsys_ns(n->subsys, nsid);
5890
0
        if (!ns) {
5891
0
            return NVME_INVALID_FIELD | NVME_DNR;
5892
0
        }
5893
0
    }
5894
5895
0
    for (cntlid = min_id; cntlid < ARRAY_SIZE(n->subsys->ctrls); cntlid++) {
5896
0
        ctrl = nvme_subsys_ctrl(n->subsys, cntlid);
5897
0
        if (!ctrl) {
5898
0
            continue;
5899
0
        }
5900
5901
0
        if (attached && !nvme_ns(ctrl, nsid)) {
5902
0
            continue;
5903
0
        }
5904
5905
0
        ids[nr_ids++] = cntlid;
5906
0
    }
5907
5908
0
    list[0] = nr_ids;
5909
5910
0
    return nvme_c2h(n, (uint8_t *)list, sizeof(list), req);
5911
0
}
5912
5913
static uint16_t nvme_identify_pri_ctrl_cap(NvmeCtrl *n, NvmeRequest *req)
5914
0
{
5915
0
    trace_pci_nvme_identify_pri_ctrl_cap(le16_to_cpu(n->pri_ctrl_cap.cntlid));
5916
5917
0
    return nvme_c2h(n, (uint8_t *)&n->pri_ctrl_cap,
5918
0
                    sizeof(NvmePriCtrlCap), req);
5919
0
}
5920
5921
static uint16_t nvme_identify_sec_ctrl_list(NvmeCtrl *n, NvmeRequest *req)
5922
0
{
5923
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5924
0
    uint16_t pri_ctrl_id = le16_to_cpu(n->pri_ctrl_cap.cntlid);
5925
0
    uint16_t min_id = le16_to_cpu(c->ctrlid);
5926
0
    uint8_t num_sec_ctrl = n->nr_sec_ctrls;
5927
0
    NvmeSecCtrlList list = {0};
5928
0
    uint8_t i;
5929
5930
0
    for (i = 0; i < num_sec_ctrl; i++) {
5931
0
        if (n->sec_ctrl_list[i].scid >= min_id) {
5932
0
            list.numcntl = MIN(num_sec_ctrl - i, 127);
5933
0
            memcpy(&list.sec, n->sec_ctrl_list + i,
5934
0
                   list.numcntl * sizeof(NvmeSecCtrlEntry));
5935
0
            break;
5936
0
        }
5937
0
    }
5938
5939
0
    trace_pci_nvme_identify_sec_ctrl_list(pri_ctrl_id, list.numcntl);
5940
5941
0
    return nvme_c2h(n, (uint8_t *)&list, sizeof(list), req);
5942
0
}
5943
5944
static uint16_t nvme_identify_ns_ind(NvmeCtrl *n, NvmeRequest *req, bool alloc)
5945
0
{
5946
0
    NvmeNamespace *ns;
5947
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5948
0
    uint32_t nsid = le32_to_cpu(c->nsid);
5949
5950
0
    trace_pci_nvme_identify_ns_ind(nsid);
5951
5952
0
    if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5953
0
        return NVME_INVALID_NSID | NVME_DNR;
5954
0
    }
5955
5956
0
    ns = nvme_ns(n, nsid);
5957
0
    if (unlikely(!ns)) {
5958
0
        if (alloc) {
5959
0
            ns = nvme_subsys_ns(n->subsys, nsid);
5960
0
            if (!ns) {
5961
0
                return nvme_rpt_empty_id_struct(n, req);
5962
0
            }
5963
0
        } else {
5964
0
            return nvme_rpt_empty_id_struct(n, req);
5965
0
        }
5966
0
    }
5967
5968
0
    return nvme_c2h(n, (uint8_t *)&ns->id_ns_ind, sizeof(NvmeIdNsInd), req);
5969
0
}
5970
5971
static uint16_t nvme_identify_ns_csi(NvmeCtrl *n, NvmeRequest *req,
5972
                                     bool active)
5973
0
{
5974
0
    NvmeNamespace *ns;
5975
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5976
0
    uint32_t nsid = le32_to_cpu(c->nsid);
5977
5978
0
    trace_pci_nvme_identify_ns_csi(nsid, c->csi);
5979
5980
0
    if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5981
0
        return NVME_INVALID_NSID | NVME_DNR;
5982
0
    }
5983
5984
0
    ns = nvme_ns(n, nsid);
5985
0
    if (unlikely(!ns)) {
5986
0
        if (!active) {
5987
0
            ns = nvme_subsys_ns(n->subsys, nsid);
5988
0
            if (!ns) {
5989
0
                return nvme_rpt_empty_id_struct(n, req);
5990
0
            }
5991
0
        } else {
5992
0
            return nvme_rpt_empty_id_struct(n, req);
5993
0
        }
5994
0
    }
5995
5996
0
    if (c->csi == NVME_CSI_NVM) {
5997
0
        return nvme_c2h(n, (uint8_t *)&ns->id_ns_nvm, sizeof(NvmeIdNsNvm),
5998
0
                        req);
5999
0
    } else if (c->csi == NVME_CSI_ZONED && ns->csi == NVME_CSI_ZONED) {
6000
0
        return nvme_c2h(n, (uint8_t *)ns->id_ns_zoned, sizeof(NvmeIdNsZoned),
6001
0
                        req);
6002
0
    }
6003
6004
0
    return NVME_INVALID_FIELD | NVME_DNR;
6005
0
}
6006
6007
static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req,
6008
                                     bool active)
6009
0
{
6010
0
    NvmeNamespace *ns;
6011
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
6012
0
    uint32_t min_nsid = le32_to_cpu(c->nsid);
6013
0
    uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
6014
0
    static const int data_len = sizeof(list);
6015
0
    uint32_t *list_ptr = (uint32_t *)list;
6016
0
    int i, j = 0;
6017
6018
0
    trace_pci_nvme_identify_nslist(min_nsid);
6019
6020
    /*
6021
     * Both FFFFFFFFh (NVME_NSID_BROADCAST) and FFFFFFFFEh are invalid values
6022
     * since the Active Namespace ID List should return namespaces with ids
6023
     * *higher* than the NSID specified in the command. This is also specified
6024
     * in the spec (NVM Express v1.3d, Section 5.15.4).
6025
     */
6026
0
    if (min_nsid >= NVME_NSID_BROADCAST - 1) {
6027
0
        return NVME_INVALID_NSID | NVME_DNR;
6028
0
    }
6029
6030
0
    for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6031
0
        ns = nvme_ns(n, i);
6032
0
        if (!ns) {
6033
0
            if (!active) {
6034
0
                ns = nvme_subsys_ns(n->subsys, i);
6035
0
                if (!ns) {
6036
0
                    continue;
6037
0
                }
6038
0
            } else {
6039
0
                continue;
6040
0
            }
6041
0
        }
6042
0
        if (ns->params.nsid <= min_nsid) {
6043
0
            continue;
6044
0
        }
6045
0
        list_ptr[j++] = cpu_to_le32(ns->params.nsid);
6046
0
        if (j == data_len / sizeof(uint32_t)) {
6047
0
            break;
6048
0
        }
6049
0
    }
6050
6051
0
    return nvme_c2h(n, list, data_len, req);
6052
0
}
6053
6054
static uint16_t nvme_identify_nslist_csi(NvmeCtrl *n, NvmeRequest *req,
6055
                                         bool active)
6056
0
{
6057
0
    NvmeNamespace *ns;
6058
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
6059
0
    uint32_t min_nsid = le32_to_cpu(c->nsid);
6060
0
    uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
6061
0
    static const int data_len = sizeof(list);
6062
0
    uint32_t *list_ptr = (uint32_t *)list;
6063
0
    int i, j = 0;
6064
6065
0
    trace_pci_nvme_identify_nslist_csi(min_nsid, c->csi);
6066
6067
    /*
6068
     * Same as in nvme_identify_nslist(), FFFFFFFFh/FFFFFFFFEh are invalid.
6069
     */
6070
0
    if (min_nsid >= NVME_NSID_BROADCAST - 1) {
6071
0
        return NVME_INVALID_NSID | NVME_DNR;
6072
0
    }
6073
6074
0
    if (c->csi != NVME_CSI_NVM && c->csi != NVME_CSI_ZONED) {
6075
0
        return NVME_INVALID_FIELD | NVME_DNR;
6076
0
    }
6077
6078
0
    for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6079
0
        ns = nvme_ns(n, i);
6080
0
        if (!ns) {
6081
0
            if (!active) {
6082
0
                ns = nvme_subsys_ns(n->subsys, i);
6083
0
                if (!ns) {
6084
0
                    continue;
6085
0
                }
6086
0
            } else {
6087
0
                continue;
6088
0
            }
6089
0
        }
6090
0
        if (ns->params.nsid <= min_nsid || c->csi != ns->csi) {
6091
0
            continue;
6092
0
        }
6093
0
        list_ptr[j++] = cpu_to_le32(ns->params.nsid);
6094
0
        if (j == data_len / sizeof(uint32_t)) {
6095
0
            break;
6096
0
        }
6097
0
    }
6098
6099
0
    return nvme_c2h(n, list, data_len, req);
6100
0
}
6101
6102
static uint16_t nvme_endurance_group_list(NvmeCtrl *n, NvmeRequest *req)
6103
0
{
6104
0
    uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
6105
0
    uint16_t *nr_ids = &list[0];
6106
0
    uint16_t *ids = &list[1];
6107
0
    uint16_t endgid = le32_to_cpu(req->cmd.cdw11) & 0xffff;
6108
6109
    /*
6110
     * The current nvme-subsys only supports Endurance Group #1.
6111
     */
6112
0
    if (!endgid) {
6113
0
        *nr_ids = 1;
6114
0
        ids[0] = 1;
6115
0
    } else {
6116
0
        *nr_ids = 0;
6117
0
    }
6118
6119
0
    return nvme_c2h(n, list, sizeof(list), req);
6120
0
}
6121
6122
static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeRequest *req)
6123
0
{
6124
0
    NvmeNamespace *ns;
6125
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
6126
0
    uint32_t nsid = le32_to_cpu(c->nsid);
6127
0
    uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
6128
0
    uint8_t *pos = list;
6129
0
    struct {
6130
0
        NvmeIdNsDescr hdr;
6131
0
        uint8_t v[NVME_NIDL_UUID];
6132
0
    } QEMU_PACKED uuid = {};
6133
0
    struct {
6134
0
        NvmeIdNsDescr hdr;
6135
0
        uint8_t v[NVME_NIDL_NGUID];
6136
0
    } QEMU_PACKED nguid = {};
6137
0
    struct {
6138
0
        NvmeIdNsDescr hdr;
6139
0
        uint64_t v;
6140
0
    } QEMU_PACKED eui64 = {};
6141
0
    struct {
6142
0
        NvmeIdNsDescr hdr;
6143
0
        uint8_t v;
6144
0
    } QEMU_PACKED csi = {};
6145
6146
0
    trace_pci_nvme_identify_ns_descr_list(nsid);
6147
6148
0
    if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
6149
0
        return NVME_INVALID_NSID | NVME_DNR;
6150
0
    }
6151
6152
0
    ns = nvme_ns(n, nsid);
6153
0
    if (unlikely(!ns)) {
6154
0
        return NVME_INVALID_FIELD | NVME_DNR;
6155
0
    }
6156
6157
0
    if (!qemu_uuid_is_null(&ns->params.uuid)) {
6158
0
        uuid.hdr.nidt = NVME_NIDT_UUID;
6159
0
        uuid.hdr.nidl = NVME_NIDL_UUID;
6160
0
        memcpy(uuid.v, ns->params.uuid.data, NVME_NIDL_UUID);
6161
0
        memcpy(pos, &uuid, sizeof(uuid));
6162
0
        pos += sizeof(uuid);
6163
0
    }
6164
6165
0
    if (!nvme_nguid_is_null(&ns->params.nguid)) {
6166
0
        nguid.hdr.nidt = NVME_NIDT_NGUID;
6167
0
        nguid.hdr.nidl = NVME_NIDL_NGUID;
6168
0
        memcpy(nguid.v, ns->params.nguid.data, NVME_NIDL_NGUID);
6169
0
        memcpy(pos, &nguid, sizeof(nguid));
6170
0
        pos += sizeof(nguid);
6171
0
    }
6172
6173
0
    if (ns->params.eui64) {
6174
0
        eui64.hdr.nidt = NVME_NIDT_EUI64;
6175
0
        eui64.hdr.nidl = NVME_NIDL_EUI64;
6176
0
        eui64.v = cpu_to_be64(ns->params.eui64);
6177
0
        memcpy(pos, &eui64, sizeof(eui64));
6178
0
        pos += sizeof(eui64);
6179
0
    }
6180
6181
0
    csi.hdr.nidt = NVME_NIDT_CSI;
6182
0
    csi.hdr.nidl = NVME_NIDL_CSI;
6183
0
    csi.v = ns->csi;
6184
0
    memcpy(pos, &csi, sizeof(csi));
6185
0
    pos += sizeof(csi);
6186
6187
0
    return nvme_c2h(n, list, sizeof(list), req);
6188
0
}
6189
6190
static uint16_t nvme_identify_cmd_set(NvmeCtrl *n, NvmeRequest *req)
6191
0
{
6192
0
    uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
6193
0
    static const int data_len = sizeof(list);
6194
6195
0
    trace_pci_nvme_identify_cmd_set();
6196
6197
0
    NVME_SET_CSI(*list, NVME_CSI_NVM);
6198
0
    NVME_SET_CSI(*list, NVME_CSI_ZONED);
6199
6200
0
    return nvme_c2h(n, list, data_len, req);
6201
0
}
6202
6203
static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req)
6204
0
{
6205
0
    NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
6206
6207
0
    trace_pci_nvme_identify(nvme_cid(req), c->cns, le16_to_cpu(c->ctrlid),
6208
0
                            c->csi);
6209
6210
0
    switch (c->cns) {
6211
0
    case NVME_ID_CNS_NS:
6212
0
        return nvme_identify_ns(n, req, true);
6213
0
    case NVME_ID_CNS_NS_PRESENT:
6214
0
        return nvme_identify_ns(n, req, false);
6215
0
    case NVME_ID_CNS_NS_ATTACHED_CTRL_LIST:
6216
0
        return nvme_identify_ctrl_list(n, req, true);
6217
0
    case NVME_ID_CNS_CTRL_LIST:
6218
0
        return nvme_identify_ctrl_list(n, req, false);
6219
0
    case NVME_ID_CNS_PRIMARY_CTRL_CAP:
6220
0
        return nvme_identify_pri_ctrl_cap(n, req);
6221
0
    case NVME_ID_CNS_SECONDARY_CTRL_LIST:
6222
0
        return nvme_identify_sec_ctrl_list(n, req);
6223
0
    case NVME_ID_CNS_CS_NS:
6224
0
        return nvme_identify_ns_csi(n, req, true);
6225
0
    case NVME_ID_CNS_CS_IND_NS:
6226
0
        return nvme_identify_ns_ind(n, req, false);
6227
0
    case NVME_ID_CNS_CS_IND_NS_ALLOCATED:
6228
0
        return nvme_identify_ns_ind(n, req, true);
6229
0
    case NVME_ID_CNS_CS_NS_PRESENT:
6230
0
        return nvme_identify_ns_csi(n, req, false);
6231
0
    case NVME_ID_CNS_CTRL:
6232
0
        return nvme_identify_ctrl(n, req);
6233
0
    case NVME_ID_CNS_CS_CTRL:
6234
0
        return nvme_identify_ctrl_csi(n, req);
6235
0
    case NVME_ID_CNS_NS_ACTIVE_LIST:
6236
0
        return nvme_identify_nslist(n, req, true);
6237
0
    case NVME_ID_CNS_NS_PRESENT_LIST:
6238
0
        return nvme_identify_nslist(n, req, false);
6239
0
    case NVME_ID_CNS_CS_NS_ACTIVE_LIST:
6240
0
        return nvme_identify_nslist_csi(n, req, true);
6241
0
    case NVME_ID_CNS_ENDURANCE_GROUP_LIST:
6242
0
        return nvme_endurance_group_list(n, req);
6243
0
    case NVME_ID_CNS_CS_NS_PRESENT_LIST:
6244
0
        return nvme_identify_nslist_csi(n, req, false);
6245
0
    case NVME_ID_CNS_NS_DESCR_LIST:
6246
0
        return nvme_identify_ns_descr_list(n, req);
6247
0
    case NVME_ID_CNS_IO_COMMAND_SET:
6248
0
        return nvme_identify_cmd_set(n, req);
6249
0
    default:
6250
0
        trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
6251
0
        return NVME_INVALID_FIELD | NVME_DNR;
6252
0
    }
6253
0
}
6254
6255
static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req)
6256
0
{
6257
0
    uint16_t sqid = le32_to_cpu(req->cmd.cdw10) & 0xffff;
6258
0
    uint16_t cid  = (le32_to_cpu(req->cmd.cdw10) >> 16) & 0xffff;
6259
0
    NvmeSQueue *sq;
6260
0
    NvmeRequest *r, *next;
6261
0
    int i;
6262
6263
0
    req->cqe.result = 1;
6264
0
    if (nvme_check_sqid(n, sqid)) {
6265
0
        return NVME_INVALID_FIELD | NVME_DNR;
6266
0
    }
6267
6268
0
    sq = n->sq[sqid];
6269
6270
0
    if (sqid == 0) {
6271
0
        for (i = 0; i < n->outstanding_aers; i++) {
6272
0
            NvmeRequest *re = n->aer_reqs[i];
6273
0
            if (re->cqe.cid == cid) {
6274
0
                memmove(n->aer_reqs + i, n->aer_reqs + i + 1,
6275
0
                         (n->outstanding_aers - i - 1) * sizeof(NvmeRequest *));
6276
0
                n->outstanding_aers--;
6277
0
                re->status = NVME_CMD_ABORT_REQ;
6278
0
                req->cqe.result = 0;
6279
0
                nvme_enqueue_req_completion(&n->admin_cq, re);
6280
0
                return NVME_SUCCESS;
6281
0
            }
6282
0
        }
6283
0
    }
6284
6285
0
    QTAILQ_FOREACH_SAFE(r, &sq->out_req_list, entry, next) {
6286
0
        if (r->cqe.cid == cid) {
6287
0
            if (r->aiocb) {
6288
0
                r->status = NVME_CMD_ABORT_REQ;
6289
0
                blk_aio_cancel_async(r->aiocb);
6290
0
            }
6291
0
            break;
6292
0
        }
6293
0
    }
6294
6295
0
    return NVME_SUCCESS;
6296
0
}
6297
6298
static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts)
6299
0
{
6300
0
    trace_pci_nvme_setfeat_timestamp(ts);
6301
6302
0
    n->host_timestamp = le64_to_cpu(ts);
6303
0
    n->timestamp_set_qemu_clock_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
6304
0
}
6305
6306
static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)
6307
0
{
6308
0
    uint64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
6309
0
    uint64_t elapsed_time = current_time - n->timestamp_set_qemu_clock_ms;
6310
6311
0
    union nvme_timestamp {
6312
0
        struct {
6313
0
            uint64_t timestamp:48;
6314
0
            uint64_t sync:1;
6315
0
            uint64_t origin:3;
6316
0
            uint64_t rsvd1:12;
6317
0
        };
6318
0
        uint64_t all;
6319
0
    };
6320
6321
0
    union nvme_timestamp ts;
6322
0
    ts.all = 0;
6323
0
    ts.timestamp = n->host_timestamp + elapsed_time;
6324
6325
    /* If the host timestamp is non-zero, set the timestamp origin */
6326
0
    ts.origin = n->host_timestamp ? 0x01 : 0x00;
6327
6328
0
    trace_pci_nvme_getfeat_timestamp(ts.all);
6329
6330
0
    return cpu_to_le64(ts.all);
6331
0
}
6332
6333
static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
6334
0
{
6335
0
    uint64_t timestamp = nvme_get_timestamp(n);
6336
6337
0
    return nvme_c2h(n, (uint8_t *)&timestamp, sizeof(timestamp), req);
6338
0
}
6339
6340
static int nvme_get_feature_fdp(NvmeCtrl *n, uint32_t endgrpid,
6341
                                uint32_t *result)
6342
0
{
6343
0
    *result = 0;
6344
6345
0
    if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
6346
0
        return NVME_INVALID_FIELD | NVME_DNR;
6347
0
    }
6348
6349
0
    *result = FIELD_DP16(0, FEAT_FDP, FDPE, 1);
6350
0
    *result = FIELD_DP16(*result, FEAT_FDP, CONF_NDX, 0);
6351
6352
0
    return NVME_SUCCESS;
6353
0
}
6354
6355
static uint16_t nvme_get_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
6356
                                            NvmeRequest *req, uint32_t *result)
6357
0
{
6358
0
    NvmeCmd *cmd = &req->cmd;
6359
0
    uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
6360
0
    uint16_t ph = cdw11 & 0xffff;
6361
0
    uint8_t noet = (cdw11 >> 16) & 0xff;
6362
0
    uint16_t ruhid, ret;
6363
0
    uint32_t nentries = 0;
6364
0
    uint8_t s_events_ndx = 0;
6365
0
    size_t s_events_siz = sizeof(NvmeFdpEventDescr) * noet;
6366
0
    g_autofree NvmeFdpEventDescr *s_events = g_malloc0(s_events_siz);
6367
0
    NvmeRuHandle *ruh;
6368
0
    NvmeFdpEventDescr *s_event;
6369
6370
0
    if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
6371
0
        return NVME_FDP_DISABLED | NVME_DNR;
6372
0
    }
6373
6374
0
    if (!nvme_ph_valid(ns, ph)) {
6375
0
        return NVME_INVALID_FIELD | NVME_DNR;
6376
0
    }
6377
6378
0
    ruhid = ns->fdp.phs[ph];
6379
0
    ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
6380
6381
0
    assert(ruh);
6382
6383
0
    if (unlikely(noet == 0)) {
6384
0
        return NVME_INVALID_FIELD | NVME_DNR;
6385
0
    }
6386
6387
0
    for (uint8_t event_type = 0; event_type < FDP_EVT_MAX; event_type++) {
6388
0
        uint8_t shift = nvme_fdp_evf_shifts[event_type];
6389
0
        if (!shift && event_type) {
6390
0
            continue;
6391
0
        }
6392
6393
0
        nentries++;
6394
6395
0
        s_event = &s_events[s_events_ndx];
6396
0
        s_event->evt = event_type;
6397
0
        s_event->evta = (ruh->event_filter >> shift) & 0x1;
6398
6399
        /* break if all `noet` entries are filled */
6400
0
        if ((++s_events_ndx) == noet) {
6401
0
            break;
6402
0
        }
6403
0
    }
6404
6405
0
    ret = nvme_c2h(n, s_events, s_events_siz, req);
6406
0
    if (ret) {
6407
0
        return ret;
6408
0
    }
6409
6410
0
    *result = nentries;
6411
0
    return NVME_SUCCESS;
6412
0
}
6413
6414
static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req)
6415
0
{
6416
0
    NvmeCmd *cmd = &req->cmd;
6417
0
    uint32_t dw10 = le32_to_cpu(cmd->cdw10);
6418
0
    uint32_t dw11 = le32_to_cpu(cmd->cdw11);
6419
0
    uint32_t nsid = le32_to_cpu(cmd->nsid);
6420
0
    uint32_t result = 0;
6421
0
    uint8_t fid = NVME_GETSETFEAT_FID(dw10);
6422
0
    NvmeGetFeatureSelect sel = NVME_GETFEAT_SELECT(dw10);
6423
0
    uint16_t iv;
6424
0
    NvmeNamespace *ns;
6425
0
    int i;
6426
0
    uint16_t endgrpid = 0, ret = NVME_SUCCESS;
6427
6428
0
    static const uint32_t nvme_feature_default[NVME_FID_MAX] = {
6429
0
        [NVME_ARBITRATION] = NVME_ARB_AB_NOLIMIT,
6430
0
    };
6431
6432
0
    trace_pci_nvme_getfeat(nvme_cid(req), nsid, fid, sel, dw11);
6433
6434
0
    if (!nvme_feature_support[fid]) {
6435
0
        return NVME_INVALID_FIELD | NVME_DNR;
6436
0
    }
6437
6438
0
    if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
6439
0
        if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
6440
            /*
6441
             * The Reservation Notification Mask and Reservation Persistence
6442
             * features require a status code of Invalid Field in Command when
6443
             * NSID is FFFFFFFFh. Since the device does not support those
6444
             * features we can always return Invalid Namespace or Format as we
6445
             * should do for all other features.
6446
             */
6447
0
            return NVME_INVALID_NSID | NVME_DNR;
6448
0
        }
6449
6450
0
        if (!nvme_ns(n, nsid)) {
6451
0
            return NVME_INVALID_FIELD | NVME_DNR;
6452
0
        }
6453
0
    }
6454
6455
0
    switch (sel) {
6456
0
    case NVME_GETFEAT_SELECT_CURRENT:
6457
0
        break;
6458
0
    case NVME_GETFEAT_SELECT_SAVED:
6459
        /* no features are saveable by the controller; fallthrough */
6460
0
    case NVME_GETFEAT_SELECT_DEFAULT:
6461
0
        goto defaults;
6462
0
    case NVME_GETFEAT_SELECT_CAP:
6463
0
        result = nvme_feature_cap[fid];
6464
0
        goto out;
6465
0
    }
6466
6467
0
    switch (fid) {
6468
0
    case NVME_TEMPERATURE_THRESHOLD:
6469
0
        result = 0;
6470
6471
        /*
6472
         * The controller only implements the Composite Temperature sensor, so
6473
         * return 0 for all other sensors.
6474
         */
6475
0
        if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6476
0
            goto out;
6477
0
        }
6478
6479
0
        switch (NVME_TEMP_THSEL(dw11)) {
6480
0
        case NVME_TEMP_THSEL_OVER:
6481
0
            result = n->features.temp_thresh_hi;
6482
0
            goto out;
6483
0
        case NVME_TEMP_THSEL_UNDER:
6484
0
            result = n->features.temp_thresh_low;
6485
0
            goto out;
6486
0
        }
6487
6488
0
        return NVME_INVALID_FIELD | NVME_DNR;
6489
0
    case NVME_ERROR_RECOVERY:
6490
0
        if (!nvme_nsid_valid(n, nsid)) {
6491
0
            return NVME_INVALID_NSID | NVME_DNR;
6492
0
        }
6493
6494
0
        ns = nvme_ns(n, nsid);
6495
0
        if (unlikely(!ns)) {
6496
0
            return NVME_INVALID_FIELD | NVME_DNR;
6497
0
        }
6498
6499
0
        result = ns->features.err_rec;
6500
0
        goto out;
6501
0
    case NVME_VOLATILE_WRITE_CACHE:
6502
0
        result = 0;
6503
0
        for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6504
0
            ns = nvme_ns(n, i);
6505
0
            if (!ns) {
6506
0
                continue;
6507
0
            }
6508
6509
0
            result = blk_enable_write_cache(ns->blkconf.blk);
6510
0
            if (result) {
6511
0
                break;
6512
0
            }
6513
0
        }
6514
0
        trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
6515
0
        goto out;
6516
0
    case NVME_ASYNCHRONOUS_EVENT_CONF:
6517
0
        result = n->features.async_config;
6518
0
        goto out;
6519
0
    case NVME_TIMESTAMP:
6520
0
        return nvme_get_feature_timestamp(n, req);
6521
0
    case NVME_HOST_BEHAVIOR_SUPPORT:
6522
0
        return nvme_c2h(n, (uint8_t *)&n->features.hbs,
6523
0
                        sizeof(n->features.hbs), req);
6524
0
    case NVME_FDP_MODE:
6525
0
        endgrpid = dw11 & 0xff;
6526
6527
0
        if (endgrpid != 0x1) {
6528
0
            return NVME_INVALID_FIELD | NVME_DNR;
6529
0
        }
6530
6531
0
        ret = nvme_get_feature_fdp(n, endgrpid, &result);
6532
0
        if (ret) {
6533
0
            return ret;
6534
0
        }
6535
0
        goto out;
6536
0
    case NVME_FDP_EVENTS:
6537
0
        if (!nvme_nsid_valid(n, nsid)) {
6538
0
            return NVME_INVALID_NSID | NVME_DNR;
6539
0
        }
6540
6541
0
        ns = nvme_ns(n, nsid);
6542
0
        if (unlikely(!ns)) {
6543
0
            return NVME_INVALID_FIELD | NVME_DNR;
6544
0
        }
6545
6546
0
        ret = nvme_get_feature_fdp_events(n, ns, req, &result);
6547
0
        if (ret) {
6548
0
            return ret;
6549
0
        }
6550
0
        goto out;
6551
0
    default:
6552
0
        break;
6553
0
    }
6554
6555
0
defaults:
6556
0
    switch (fid) {
6557
0
    case NVME_TEMPERATURE_THRESHOLD:
6558
0
        result = 0;
6559
6560
0
        if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6561
0
            break;
6562
0
        }
6563
6564
0
        if (NVME_TEMP_THSEL(dw11) == NVME_TEMP_THSEL_OVER) {
6565
0
            result = NVME_TEMPERATURE_WARNING;
6566
0
        }
6567
6568
0
        break;
6569
0
    case NVME_NUMBER_OF_QUEUES:
6570
0
        result = (n->conf_ioqpairs - 1) | ((n->conf_ioqpairs - 1) << 16);
6571
0
        trace_pci_nvme_getfeat_numq(result);
6572
0
        break;
6573
0
    case NVME_INTERRUPT_VECTOR_CONF:
6574
0
        iv = dw11 & 0xffff;
6575
0
        if (iv >= n->conf_ioqpairs + 1) {
6576
0
            return NVME_INVALID_FIELD | NVME_DNR;
6577
0
        }
6578
6579
0
        result = iv;
6580
0
        if (iv == n->admin_cq.vector) {
6581
0
            result |= NVME_INTVC_NOCOALESCING;
6582
0
        }
6583
0
        break;
6584
0
    case NVME_FDP_MODE:
6585
0
        endgrpid = dw11 & 0xff;
6586
6587
0
        if (endgrpid != 0x1) {
6588
0
            return NVME_INVALID_FIELD | NVME_DNR;
6589
0
        }
6590
6591
0
        ret = nvme_get_feature_fdp(n, endgrpid, &result);
6592
0
        if (ret) {
6593
0
            return ret;
6594
0
        }
6595
0
        break;
6596
6597
0
    case NVME_WRITE_ATOMICITY:
6598
0
        result = n->dn;
6599
0
        break;
6600
0
    default:
6601
0
        result = nvme_feature_default[fid];
6602
0
        break;
6603
0
    }
6604
6605
0
out:
6606
0
    req->cqe.result = cpu_to_le32(result);
6607
0
    return ret;
6608
0
}
6609
6610
static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
6611
0
{
6612
0
    uint16_t ret;
6613
0
    uint64_t timestamp;
6614
6615
0
    ret = nvme_h2c(n, (uint8_t *)&timestamp, sizeof(timestamp), req);
6616
0
    if (ret) {
6617
0
        return ret;
6618
0
    }
6619
6620
0
    nvme_set_timestamp(n, timestamp);
6621
6622
0
    return NVME_SUCCESS;
6623
0
}
6624
6625
static uint16_t nvme_set_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
6626
                                            NvmeRequest *req)
6627
0
{
6628
0
    NvmeCmd *cmd = &req->cmd;
6629
0
    uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
6630
0
    uint16_t ph = cdw11 & 0xffff;
6631
0
    uint8_t noet = (cdw11 >> 16) & 0xff;
6632
0
    uint16_t ret, ruhid;
6633
0
    uint8_t enable = le32_to_cpu(cmd->cdw12) & 0x1;
6634
0
    uint64_t event_mask = 0;
6635
0
    unsigned int i;
6636
0
    g_autofree uint8_t *events = NULL;
6637
0
    NvmeRuHandle *ruh = NULL;
6638
6639
0
    if (!ns) {
6640
0
        return NVME_INVALID_FIELD | NVME_DNR;
6641
0
    }
6642
6643
0
    if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
6644
0
        return NVME_FDP_DISABLED | NVME_DNR;
6645
0
    }
6646
6647
0
    if (!nvme_ph_valid(ns, ph)) {
6648
0
        return NVME_INVALID_FIELD | NVME_DNR;
6649
0
    }
6650
6651
0
    if (unlikely(noet == 0)) {
6652
0
        return NVME_SUCCESS;
6653
0
    }
6654
6655
0
    ruhid = ns->fdp.phs[ph];
6656
0
    ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
6657
6658
0
    events = g_malloc0(noet);
6659
6660
0
    ret = nvme_h2c(n, events, noet, req);
6661
0
    if (ret) {
6662
0
        return ret;
6663
0
    }
6664
6665
0
    for (i = 0; i < noet; i++) {
6666
        /*
6667
         * We ignore requests to enable tracking of unsupported FDP event types
6668
         */
6669
0
        uint8_t event_type = events[i];
6670
0
        uint8_t shift = nvme_fdp_evf_shifts[event_type];
6671
0
        if (!shift && event_type) {
6672
0
            continue;
6673
0
        }
6674
0
        event_mask =
6675
0
            deposit64(event_mask, nvme_fdp_evf_shifts[events[i]], 1, 1);
6676
0
    }
6677
6678
0
    if (enable) {
6679
0
        ruh->event_filter |= event_mask;
6680
0
    } else {
6681
0
        ruh->event_filter = ruh->event_filter & ~event_mask;
6682
0
    }
6683
6684
0
    return NVME_SUCCESS;
6685
0
}
6686
6687
void nvme_atomic_configure_max_write_size(bool dn, uint16_t awun,
6688
                                          uint16_t awupf, NvmeAtomic *atomic)
6689
0
{
6690
0
    atomic->atomic_max_write_size = (dn ? awupf : awun) + 1;
6691
6692
0
    if (atomic->atomic_max_write_size > 1) {
6693
0
        atomic->atomic_writes = 1;
6694
0
    }
6695
0
}
6696
6697
static uint16_t nvme_set_feature_write_atomicity(NvmeCtrl *n, NvmeRequest *req)
6698
0
{
6699
0
    NvmeCmd *cmd = &req->cmd;
6700
6701
0
    uint32_t dw11 = le32_to_cpu(cmd->cdw11);
6702
6703
0
    uint16_t awun = le16_to_cpu(n->id_ctrl.awun);
6704
0
    uint16_t awupf = le16_to_cpu(n->id_ctrl.awupf);
6705
6706
0
    n->dn = dw11 & 0x1;
6707
6708
0
    nvme_atomic_configure_max_write_size(n->dn, awun, awupf, &n->atomic);
6709
6710
0
    for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6711
0
        uint16_t nawun, nawupf, nabsn, nabspf;
6712
6713
0
        NvmeNamespace *ns = nvme_ns(n, i);
6714
0
        if (!ns) {
6715
0
            continue;
6716
0
        }
6717
6718
0
        nawun = le16_to_cpu(ns->id_ns.nawun);
6719
0
        nawupf = le16_to_cpu(ns->id_ns.nawupf);
6720
6721
0
        nvme_atomic_configure_max_write_size(n->dn, nawun, nawupf,
6722
0
                                             &ns->atomic);
6723
6724
0
        nabsn = le16_to_cpu(ns->id_ns.nabsn);
6725
0
        nabspf = le16_to_cpu(ns->id_ns.nabspf);
6726
6727
0
        nvme_ns_atomic_configure_boundary(n->dn, nabsn, nabspf,
6728
0
                                          &ns->atomic);
6729
0
    }
6730
6731
0
    return NVME_SUCCESS;
6732
0
}
6733
6734
static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req)
6735
0
{
6736
0
    NvmeNamespace *ns = NULL;
6737
6738
0
    NvmeCmd *cmd = &req->cmd;
6739
0
    uint32_t dw10 = le32_to_cpu(cmd->cdw10);
6740
0
    uint32_t dw11 = le32_to_cpu(cmd->cdw11);
6741
0
    uint32_t nsid = le32_to_cpu(cmd->nsid);
6742
0
    uint8_t fid = NVME_GETSETFEAT_FID(dw10);
6743
0
    uint8_t save = NVME_SETFEAT_SAVE(dw10);
6744
0
    uint16_t status;
6745
0
    int i;
6746
6747
0
    trace_pci_nvme_setfeat(nvme_cid(req), nsid, fid, save, dw11);
6748
6749
0
    if (save && !(nvme_feature_cap[fid] & NVME_FEAT_CAP_SAVE)) {
6750
0
        return NVME_FID_NOT_SAVEABLE | NVME_DNR;
6751
0
    }
6752
6753
0
    if (!nvme_feature_support[fid]) {
6754
0
        return NVME_INVALID_FIELD | NVME_DNR;
6755
0
    }
6756
6757
0
    if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
6758
0
        if (nsid != NVME_NSID_BROADCAST) {
6759
0
            if (!nvme_nsid_valid(n, nsid)) {
6760
0
                return NVME_INVALID_NSID | NVME_DNR;
6761
0
            }
6762
6763
0
            ns = nvme_ns(n, nsid);
6764
0
            if (unlikely(!ns)) {
6765
0
                return NVME_INVALID_FIELD | NVME_DNR;
6766
0
            }
6767
0
        }
6768
0
    } else if (nsid && nsid != NVME_NSID_BROADCAST) {
6769
0
        if (!nvme_nsid_valid(n, nsid)) {
6770
0
            return NVME_INVALID_NSID | NVME_DNR;
6771
0
        }
6772
6773
0
        return NVME_FEAT_NOT_NS_SPEC | NVME_DNR;
6774
0
    }
6775
6776
0
    if (!(nvme_feature_cap[fid] & NVME_FEAT_CAP_CHANGE)) {
6777
0
        return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6778
0
    }
6779
6780
0
    switch (fid) {
6781
0
    case NVME_TEMPERATURE_THRESHOLD:
6782
0
        if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6783
0
            break;
6784
0
        }
6785
6786
0
        switch (NVME_TEMP_THSEL(dw11)) {
6787
0
        case NVME_TEMP_THSEL_OVER:
6788
0
            n->features.temp_thresh_hi = NVME_TEMP_TMPTH(dw11);
6789
0
            break;
6790
0
        case NVME_TEMP_THSEL_UNDER:
6791
0
            n->features.temp_thresh_low = NVME_TEMP_TMPTH(dw11);
6792
0
            break;
6793
0
        default:
6794
0
            return NVME_INVALID_FIELD | NVME_DNR;
6795
0
        }
6796
6797
0
        if ((n->temperature >= n->features.temp_thresh_hi) ||
6798
0
            (n->temperature <= n->features.temp_thresh_low)) {
6799
0
            nvme_smart_event(n, NVME_SMART_TEMPERATURE);
6800
0
        }
6801
6802
0
        break;
6803
0
    case NVME_ERROR_RECOVERY:
6804
0
        if (nsid == NVME_NSID_BROADCAST) {
6805
0
            for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6806
0
                ns = nvme_ns(n, i);
6807
6808
0
                if (!ns) {
6809
0
                    continue;
6810
0
                }
6811
6812
0
                if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
6813
0
                    ns->features.err_rec = dw11;
6814
0
                }
6815
0
            }
6816
6817
0
            break;
6818
0
        }
6819
6820
0
        assert(ns);
6821
0
        if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat))  {
6822
0
            ns->features.err_rec = dw11;
6823
0
        }
6824
0
        break;
6825
0
    case NVME_VOLATILE_WRITE_CACHE:
6826
0
        for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6827
0
            ns = nvme_ns(n, i);
6828
0
            if (!ns) {
6829
0
                continue;
6830
0
            }
6831
6832
0
            if (!(dw11 & 0x1) && blk_enable_write_cache(ns->blkconf.blk)) {
6833
0
                blk_flush(ns->blkconf.blk);
6834
0
            }
6835
6836
0
            blk_set_enable_write_cache(ns->blkconf.blk, dw11 & 1);
6837
0
        }
6838
6839
0
        break;
6840
6841
0
    case NVME_NUMBER_OF_QUEUES:
6842
0
        if (n->qs_created) {
6843
0
            return NVME_CMD_SEQ_ERROR | NVME_DNR;
6844
0
        }
6845
6846
        /*
6847
         * NVMe v1.3, Section 5.21.1.7: FFFFh is not an allowed value for NCQR
6848
         * and NSQR.
6849
         */
6850
0
        if ((dw11 & 0xffff) == 0xffff || ((dw11 >> 16) & 0xffff) == 0xffff) {
6851
0
            return NVME_INVALID_FIELD | NVME_DNR;
6852
0
        }
6853
6854
0
        trace_pci_nvme_setfeat_numq((dw11 & 0xffff) + 1,
6855
0
                                    ((dw11 >> 16) & 0xffff) + 1,
6856
0
                                    n->conf_ioqpairs,
6857
0
                                    n->conf_ioqpairs);
6858
0
        req->cqe.result = cpu_to_le32((n->conf_ioqpairs - 1) |
6859
0
                                      ((n->conf_ioqpairs - 1) << 16));
6860
0
        break;
6861
0
    case NVME_ASYNCHRONOUS_EVENT_CONF:
6862
0
        n->features.async_config = dw11;
6863
0
        break;
6864
0
    case NVME_TIMESTAMP:
6865
0
        return nvme_set_feature_timestamp(n, req);
6866
0
    case NVME_HOST_BEHAVIOR_SUPPORT:
6867
0
        status = nvme_h2c(n, (uint8_t *)&n->features.hbs,
6868
0
                          sizeof(n->features.hbs), req);
6869
0
        if (status) {
6870
0
            return status;
6871
0
        }
6872
6873
0
        for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6874
0
            ns = nvme_ns(n, i);
6875
6876
0
            if (!ns) {
6877
0
                continue;
6878
0
            }
6879
6880
0
            ns->id_ns.nlbaf = ns->nlbaf - 1;
6881
0
            if (!n->features.hbs.lbafee) {
6882
0
                ns->id_ns.nlbaf = MIN(ns->id_ns.nlbaf, 15);
6883
0
            }
6884
0
        }
6885
6886
0
        return status;
6887
0
    case NVME_COMMAND_SET_PROFILE:
6888
0
        if (dw11 & 0x1ff) {
6889
0
            trace_pci_nvme_err_invalid_iocsci(dw11 & 0x1ff);
6890
0
            return NVME_IOCS_COMBINATION_REJECTED | NVME_DNR;
6891
0
        }
6892
0
        break;
6893
0
    case NVME_FDP_MODE:
6894
        /* spec: abort with cmd seq err if there's one or more NS' in endgrp */
6895
0
        return NVME_CMD_SEQ_ERROR | NVME_DNR;
6896
0
    case NVME_FDP_EVENTS:
6897
0
        return nvme_set_feature_fdp_events(n, ns, req);
6898
0
    case NVME_WRITE_ATOMICITY:
6899
0
        return nvme_set_feature_write_atomicity(n, req);
6900
0
    default:
6901
0
        return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6902
0
    }
6903
0
    return NVME_SUCCESS;
6904
0
}
6905
6906
static uint16_t nvme_aer(NvmeCtrl *n, NvmeRequest *req)
6907
0
{
6908
0
    trace_pci_nvme_aer(nvme_cid(req));
6909
6910
0
    if (n->outstanding_aers > n->params.aerl) {
6911
0
        trace_pci_nvme_aer_aerl_exceeded();
6912
0
        return NVME_AER_LIMIT_EXCEEDED;
6913
0
    }
6914
6915
0
    n->aer_reqs[n->outstanding_aers] = req;
6916
0
    n->outstanding_aers++;
6917
6918
0
    if (!QTAILQ_EMPTY(&n->aer_queue)) {
6919
0
        nvme_process_aers(n);
6920
0
    }
6921
6922
0
    return NVME_NO_COMPLETE;
6923
0
}
6924
6925
static void nvme_update_dsm_limits(NvmeCtrl *n, NvmeNamespace *ns)
6926
0
{
6927
0
    if (ns) {
6928
0
        n->dmrsl =
6929
0
            MIN_NON_ZERO(n->dmrsl, BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
6930
6931
0
        return;
6932
0
    }
6933
6934
0
    for (uint32_t nsid = 1; nsid <= NVME_MAX_NAMESPACES; nsid++) {
6935
0
        ns = nvme_ns(n, nsid);
6936
0
        if (!ns) {
6937
0
            continue;
6938
0
        }
6939
6940
0
        n->dmrsl =
6941
0
            MIN_NON_ZERO(n->dmrsl, BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
6942
0
    }
6943
0
}
6944
6945
static bool nvme_csi_supported(NvmeCtrl *n, uint8_t csi)
6946
0
{
6947
0
    uint32_t cc;
6948
6949
0
    switch (csi) {
6950
0
    case NVME_CSI_NVM:
6951
0
        return true;
6952
6953
0
    case NVME_CSI_ZONED:
6954
0
        cc = ldl_le_p(&n->bar.cc);
6955
6956
0
        return NVME_CC_CSS(cc) == NVME_CC_CSS_ALL;
6957
0
    }
6958
6959
0
    g_assert_not_reached();
6960
0
}
6961
6962
static void nvme_detach_ns(NvmeCtrl *n, NvmeNamespace *ns)
6963
0
{
6964
0
    assert(ns->attached > 0);
6965
6966
0
    n->namespaces[ns->params.nsid] = NULL;
6967
0
    ns->attached--;
6968
0
}
6969
6970
static uint16_t nvme_ns_attachment(NvmeCtrl *n, NvmeRequest *req)
6971
0
{
6972
0
    NvmeNamespace *ns;
6973
0
    NvmeCtrl *ctrl;
6974
0
    uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
6975
0
    uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6976
0
    uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6977
0
    uint8_t sel = dw10 & 0xf;
6978
0
    uint16_t *nr_ids = &list[0];
6979
0
    uint16_t *ids = &list[1];
6980
0
    uint16_t ret;
6981
0
    int i;
6982
6983
0
    trace_pci_nvme_ns_attachment(nvme_cid(req), dw10 & 0xf);
6984
6985
0
    if (!nvme_nsid_valid(n, nsid)) {
6986
0
        return NVME_INVALID_NSID | NVME_DNR;
6987
0
    }
6988
6989
0
    ns = nvme_subsys_ns(n->subsys, nsid);
6990
0
    if (!ns) {
6991
0
        return NVME_INVALID_FIELD | NVME_DNR;
6992
0
    }
6993
6994
0
    ret = nvme_h2c(n, (uint8_t *)list, 4096, req);
6995
0
    if (ret) {
6996
0
        return ret;
6997
0
    }
6998
6999
0
    if (!*nr_ids) {
7000
0
        return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
7001
0
    }
7002
7003
0
    *nr_ids = MIN(*nr_ids, NVME_CONTROLLER_LIST_SIZE - 1);
7004
0
    for (i = 0; i < *nr_ids; i++) {
7005
0
        ctrl = nvme_subsys_ctrl(n->subsys, ids[i]);
7006
0
        if (!ctrl) {
7007
0
            return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
7008
0
        }
7009
7010
0
        switch (sel) {
7011
0
        case NVME_NS_ATTACHMENT_ATTACH:
7012
0
            if (nvme_ns(ctrl, nsid)) {
7013
0
                return NVME_NS_ALREADY_ATTACHED | NVME_DNR;
7014
0
            }
7015
7016
0
            if (ns->attached && !ns->params.shared) {
7017
0
                return NVME_NS_PRIVATE | NVME_DNR;
7018
0
            }
7019
7020
0
            if (!nvme_csi_supported(ctrl, ns->csi)) {
7021
0
                return NVME_IOCS_NOT_SUPPORTED | NVME_DNR;
7022
0
            }
7023
7024
0
            nvme_attach_ns(ctrl, ns);
7025
0
            nvme_update_dsm_limits(ctrl, ns);
7026
7027
0
            break;
7028
7029
0
        case NVME_NS_ATTACHMENT_DETACH:
7030
0
            if (!nvme_ns(ctrl, nsid)) {
7031
0
                return NVME_NS_NOT_ATTACHED | NVME_DNR;
7032
0
            }
7033
7034
0
            nvme_detach_ns(ctrl, ns);
7035
0
            nvme_update_dsm_limits(ctrl, NULL);
7036
7037
0
            break;
7038
7039
0
        default:
7040
0
            return NVME_INVALID_FIELD | NVME_DNR;
7041
0
        }
7042
7043
        /*
7044
         * Add namespace id to the changed namespace id list for event clearing
7045
         * via Get Log Page command.
7046
         */
7047
0
        if (!test_and_set_bit(nsid, ctrl->changed_nsids)) {
7048
0
            nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE,
7049
0
                               NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED,
7050
0
                               NVME_LOG_CHANGED_NSLIST);
7051
0
        }
7052
0
    }
7053
7054
0
    return NVME_SUCCESS;
7055
0
}
7056
7057
void nvme_ctrl_notify_ns_resize(NvmeCtrl *ctrl, NvmeNamespace *ns)
7058
0
{
7059
0
    if (!test_and_set_bit(ns->params.nsid, ctrl->changed_nsids)) {
7060
0
        nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE,
7061
0
                           NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED,
7062
0
                           NVME_LOG_CHANGED_NSLIST);
7063
0
    }
7064
0
}
7065
7066
typedef struct NvmeFormatAIOCB {
7067
    BlockAIOCB common;
7068
    BlockAIOCB *aiocb;
7069
    NvmeRequest *req;
7070
    int ret;
7071
7072
    NvmeNamespace *ns;
7073
    uint32_t nsid;
7074
    bool broadcast;
7075
    int64_t offset;
7076
7077
    uint8_t lbaf;
7078
    uint8_t mset;
7079
    uint8_t pi;
7080
    uint8_t pil;
7081
} NvmeFormatAIOCB;
7082
7083
static void nvme_format_cancel(BlockAIOCB *aiocb)
7084
0
{
7085
0
    NvmeFormatAIOCB *iocb = container_of(aiocb, NvmeFormatAIOCB, common);
7086
7087
0
    iocb->ret = -ECANCELED;
7088
7089
0
    if (iocb->aiocb) {
7090
0
        blk_aio_cancel_async(iocb->aiocb);
7091
0
        iocb->aiocb = NULL;
7092
0
    }
7093
0
}
7094
7095
static const AIOCBInfo nvme_format_aiocb_info = {
7096
    .aiocb_size = sizeof(NvmeFormatAIOCB),
7097
    .cancel_async = nvme_format_cancel,
7098
};
7099
7100
static void nvme_format_set(NvmeNamespace *ns, uint8_t lbaf, uint8_t mset,
7101
                            uint8_t pi, uint8_t pil)
7102
0
{
7103
0
    uint8_t lbafl = lbaf & 0xf;
7104
0
    uint8_t lbafu = lbaf >> 4;
7105
7106
0
    trace_pci_nvme_format_set(ns->params.nsid, lbaf, mset, pi, pil);
7107
7108
0
    ns->id_ns.dps = (pil << 3) | pi;
7109
0
    ns->id_ns.flbas = (lbafu << 5) | (mset << 4) | lbafl;
7110
7111
0
    nvme_ns_init_format(ns);
7112
0
}
7113
7114
static void nvme_do_format(NvmeFormatAIOCB *iocb);
7115
7116
static void nvme_format_ns_cb(void *opaque, int ret)
7117
0
{
7118
0
    NvmeFormatAIOCB *iocb = opaque;
7119
0
    NvmeNamespace *ns = iocb->ns;
7120
0
    int bytes;
7121
7122
0
    if (iocb->ret < 0) {
7123
0
        goto done;
7124
0
    } else if (ret < 0) {
7125
0
        iocb->ret = ret;
7126
0
        goto done;
7127
0
    }
7128
7129
0
    assert(ns);
7130
7131
0
    if (iocb->offset < ns->size) {
7132
0
        bytes = MIN(BDRV_REQUEST_MAX_BYTES, ns->size - iocb->offset);
7133
7134
0
        iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, iocb->offset,
7135
0
                                            bytes, BDRV_REQ_MAY_UNMAP,
7136
0
                                            nvme_format_ns_cb, iocb);
7137
7138
0
        iocb->offset += bytes;
7139
0
        return;
7140
0
    }
7141
7142
0
    nvme_format_set(ns, iocb->lbaf, iocb->mset, iocb->pi, iocb->pil);
7143
0
    ns->status = 0x0;
7144
0
    iocb->ns = NULL;
7145
0
    iocb->offset = 0;
7146
7147
0
done:
7148
0
    nvme_do_format(iocb);
7149
0
}
7150
7151
static uint16_t nvme_format_check(NvmeNamespace *ns, uint8_t lbaf, uint8_t pi)
7152
0
{
7153
0
    if (ns->params.zoned) {
7154
0
        return NVME_INVALID_FORMAT | NVME_DNR;
7155
0
    }
7156
7157
0
    if (lbaf > ns->id_ns.nlbaf) {
7158
0
        return NVME_INVALID_FORMAT | NVME_DNR;
7159
0
    }
7160
7161
0
    if (pi && (ns->id_ns.lbaf[lbaf].ms < nvme_pi_tuple_size(ns))) {
7162
0
        return NVME_INVALID_FORMAT | NVME_DNR;
7163
0
    }
7164
7165
0
    if (pi && pi > NVME_ID_NS_DPS_TYPE_3) {
7166
0
        return NVME_INVALID_FIELD | NVME_DNR;
7167
0
    }
7168
7169
0
    return NVME_SUCCESS;
7170
0
}
7171
7172
static void nvme_do_format(NvmeFormatAIOCB *iocb)
7173
0
{
7174
0
    NvmeRequest *req = iocb->req;
7175
0
    NvmeCtrl *n = nvme_ctrl(req);
7176
0
    uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7177
0
    uint8_t lbaf = dw10 & 0xf;
7178
0
    uint8_t pi = (dw10 >> 5) & 0x7;
7179
0
    uint16_t status;
7180
0
    int i;
7181
7182
0
    if (iocb->ret < 0) {
7183
0
        goto done;
7184
0
    }
7185
7186
0
    if (iocb->broadcast) {
7187
0
        for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
7188
0
            iocb->ns = nvme_ns(n, i);
7189
0
            if (iocb->ns) {
7190
0
                iocb->nsid = i;
7191
0
                break;
7192
0
            }
7193
0
        }
7194
0
    }
7195
7196
0
    if (!iocb->ns) {
7197
0
        goto done;
7198
0
    }
7199
7200
0
    status = nvme_format_check(iocb->ns, lbaf, pi);
7201
0
    if (status) {
7202
0
        req->status = status;
7203
0
        goto done;
7204
0
    }
7205
7206
0
    iocb->ns->status = NVME_FORMAT_IN_PROGRESS;
7207
0
    nvme_format_ns_cb(iocb, 0);
7208
0
    return;
7209
7210
0
done:
7211
0
    iocb->common.cb(iocb->common.opaque, iocb->ret);
7212
0
    qemu_aio_unref(iocb);
7213
0
}
7214
7215
static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req)
7216
0
{
7217
0
    NvmeFormatAIOCB *iocb;
7218
0
    uint32_t nsid = le32_to_cpu(req->cmd.nsid);
7219
0
    uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7220
0
    uint8_t lbaf = dw10 & 0xf;
7221
0
    uint8_t mset = (dw10 >> 4) & 0x1;
7222
0
    uint8_t pi = (dw10 >> 5) & 0x7;
7223
0
    uint8_t pil = (dw10 >> 8) & 0x1;
7224
0
    uint8_t lbafu = (dw10 >> 12) & 0x3;
7225
0
    uint16_t status;
7226
7227
0
    iocb = qemu_aio_get(&nvme_format_aiocb_info, NULL, nvme_misc_cb, req);
7228
7229
0
    iocb->req = req;
7230
0
    iocb->ret = 0;
7231
0
    iocb->ns = NULL;
7232
0
    iocb->nsid = 0;
7233
0
    iocb->lbaf = lbaf;
7234
0
    iocb->mset = mset;
7235
0
    iocb->pi = pi;
7236
0
    iocb->pil = pil;
7237
0
    iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
7238
0
    iocb->offset = 0;
7239
7240
0
    if (n->features.hbs.lbafee) {
7241
0
        iocb->lbaf |= lbafu << 4;
7242
0
    }
7243
7244
0
    if (!iocb->broadcast) {
7245
0
        if (!nvme_nsid_valid(n, nsid)) {
7246
0
            status = NVME_INVALID_NSID | NVME_DNR;
7247
0
            goto out;
7248
0
        }
7249
7250
0
        iocb->ns = nvme_ns(n, nsid);
7251
0
        if (!iocb->ns) {
7252
0
            status = NVME_INVALID_FIELD | NVME_DNR;
7253
0
            goto out;
7254
0
        }
7255
0
    }
7256
7257
0
    req->aiocb = &iocb->common;
7258
0
    nvme_do_format(iocb);
7259
7260
0
    return NVME_NO_COMPLETE;
7261
7262
0
out:
7263
0
    qemu_aio_unref(iocb);
7264
7265
0
    return status;
7266
0
}
7267
7268
static void nvme_get_virt_res_num(NvmeCtrl *n, uint8_t rt, int *num_total,
7269
                                  int *num_prim, int *num_sec)
7270
0
{
7271
0
    *num_total = le32_to_cpu(rt ?
7272
0
                             n->pri_ctrl_cap.vifrt : n->pri_ctrl_cap.vqfrt);
7273
0
    *num_prim = le16_to_cpu(rt ?
7274
0
                            n->pri_ctrl_cap.virfap : n->pri_ctrl_cap.vqrfap);
7275
0
    *num_sec = le16_to_cpu(rt ? n->pri_ctrl_cap.virfa : n->pri_ctrl_cap.vqrfa);
7276
0
}
7277
7278
static uint16_t nvme_assign_virt_res_to_prim(NvmeCtrl *n, NvmeRequest *req,
7279
                                             uint16_t cntlid, uint8_t rt,
7280
                                             int nr)
7281
0
{
7282
0
    int num_total, num_prim, num_sec;
7283
7284
0
    if (cntlid != n->cntlid) {
7285
0
        return NVME_INVALID_CTRL_ID | NVME_DNR;
7286
0
    }
7287
7288
0
    nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
7289
7290
0
    if (nr > num_total) {
7291
0
        return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
7292
0
    }
7293
7294
0
    if (nr > num_total - num_sec) {
7295
0
        return NVME_INVALID_RESOURCE_ID | NVME_DNR;
7296
0
    }
7297
7298
0
    if (rt) {
7299
0
        n->next_pri_ctrl_cap.virfap = cpu_to_le16(nr);
7300
0
    } else {
7301
0
        n->next_pri_ctrl_cap.vqrfap = cpu_to_le16(nr);
7302
0
    }
7303
7304
0
    req->cqe.result = cpu_to_le32(nr);
7305
0
    return req->status;
7306
0
}
7307
7308
static void nvme_update_virt_res(NvmeCtrl *n, NvmeSecCtrlEntry *sctrl,
7309
                                 uint8_t rt, int nr)
7310
0
{
7311
0
    int prev_nr, prev_total;
7312
7313
0
    if (rt) {
7314
0
        prev_nr = le16_to_cpu(sctrl->nvi);
7315
0
        prev_total = le32_to_cpu(n->pri_ctrl_cap.virfa);
7316
0
        sctrl->nvi = cpu_to_le16(nr);
7317
0
        n->pri_ctrl_cap.virfa = cpu_to_le32(prev_total + nr - prev_nr);
7318
0
    } else {
7319
0
        prev_nr = le16_to_cpu(sctrl->nvq);
7320
0
        prev_total = le32_to_cpu(n->pri_ctrl_cap.vqrfa);
7321
0
        sctrl->nvq = cpu_to_le16(nr);
7322
0
        n->pri_ctrl_cap.vqrfa = cpu_to_le32(prev_total + nr - prev_nr);
7323
0
    }
7324
0
}
7325
7326
static uint16_t nvme_assign_virt_res_to_sec(NvmeCtrl *n, NvmeRequest *req,
7327
                                            uint16_t cntlid, uint8_t rt, int nr)
7328
0
{
7329
0
    int num_total, num_prim, num_sec, num_free, diff, limit;
7330
0
    NvmeSecCtrlEntry *sctrl;
7331
7332
0
    sctrl = nvme_sctrl_for_cntlid(n, cntlid);
7333
0
    if (!sctrl) {
7334
0
        return NVME_INVALID_CTRL_ID | NVME_DNR;
7335
0
    }
7336
7337
0
    if (sctrl->scs) {
7338
0
        return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
7339
0
    }
7340
7341
0
    limit = le16_to_cpu(rt ? n->pri_ctrl_cap.vifrsm : n->pri_ctrl_cap.vqfrsm);
7342
0
    if (nr > limit) {
7343
0
        return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
7344
0
    }
7345
7346
0
    nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
7347
0
    num_free = num_total - num_prim - num_sec;
7348
0
    diff = nr - le16_to_cpu(rt ? sctrl->nvi : sctrl->nvq);
7349
7350
0
    if (diff > num_free) {
7351
0
        return NVME_INVALID_RESOURCE_ID | NVME_DNR;
7352
0
    }
7353
7354
0
    nvme_update_virt_res(n, sctrl, rt, nr);
7355
0
    req->cqe.result = cpu_to_le32(nr);
7356
7357
0
    return req->status;
7358
0
}
7359
7360
static uint16_t nvme_virt_set_state(NvmeCtrl *n, uint16_t cntlid, bool online)
7361
0
{
7362
0
    PCIDevice *pci = PCI_DEVICE(n);
7363
0
    NvmeCtrl *sn = NULL;
7364
0
    NvmeSecCtrlEntry *sctrl;
7365
0
    int vf_index;
7366
7367
0
    sctrl = nvme_sctrl_for_cntlid(n, cntlid);
7368
0
    if (!sctrl) {
7369
0
        return NVME_INVALID_CTRL_ID | NVME_DNR;
7370
0
    }
7371
7372
0
    if (!pci_is_vf(pci)) {
7373
0
        vf_index = le16_to_cpu(sctrl->vfn) - 1;
7374
0
        sn = NVME(pcie_sriov_get_vf_at_index(pci, vf_index));
7375
0
    }
7376
7377
0
    if (online) {
7378
0
        if (!sctrl->nvi || (le16_to_cpu(sctrl->nvq) < 2) || !sn) {
7379
0
            return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
7380
0
        }
7381
7382
0
        if (!sctrl->scs) {
7383
0
            sctrl->scs = 0x1;
7384
0
            nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
7385
0
        }
7386
0
    } else {
7387
0
        nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_INTERRUPT, 0);
7388
0
        nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_QUEUE, 0);
7389
7390
0
        if (sctrl->scs) {
7391
0
            sctrl->scs = 0x0;
7392
0
            if (sn) {
7393
0
                nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
7394
0
            }
7395
0
        }
7396
0
    }
7397
7398
0
    return NVME_SUCCESS;
7399
0
}
7400
7401
static uint16_t nvme_virt_mngmt(NvmeCtrl *n, NvmeRequest *req)
7402
0
{
7403
0
    uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7404
0
    uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
7405
0
    uint8_t act = dw10 & 0xf;
7406
0
    uint8_t rt = (dw10 >> 8) & 0x7;
7407
0
    uint16_t cntlid = (dw10 >> 16) & 0xffff;
7408
0
    int nr = dw11 & 0xffff;
7409
7410
0
    trace_pci_nvme_virt_mngmt(nvme_cid(req), act, cntlid, rt ? "VI" : "VQ", nr);
7411
7412
0
    if (rt != NVME_VIRT_RES_QUEUE && rt != NVME_VIRT_RES_INTERRUPT) {
7413
0
        return NVME_INVALID_RESOURCE_ID | NVME_DNR;
7414
0
    }
7415
7416
0
    switch (act) {
7417
0
    case NVME_VIRT_MNGMT_ACTION_SEC_ASSIGN:
7418
0
        return nvme_assign_virt_res_to_sec(n, req, cntlid, rt, nr);
7419
0
    case NVME_VIRT_MNGMT_ACTION_PRM_ALLOC:
7420
0
        return nvme_assign_virt_res_to_prim(n, req, cntlid, rt, nr);
7421
0
    case NVME_VIRT_MNGMT_ACTION_SEC_ONLINE:
7422
0
        return nvme_virt_set_state(n, cntlid, true);
7423
0
    case NVME_VIRT_MNGMT_ACTION_SEC_OFFLINE:
7424
0
        return nvme_virt_set_state(n, cntlid, false);
7425
0
    default:
7426
0
        return NVME_INVALID_FIELD | NVME_DNR;
7427
0
    }
7428
0
}
7429
7430
static uint16_t nvme_dbbuf_config(NvmeCtrl *n, const NvmeRequest *req)
7431
0
{
7432
0
    PCIDevice *pci = PCI_DEVICE(n);
7433
0
    uint64_t dbs_addr = le64_to_cpu(req->cmd.dptr.prp1);
7434
0
    uint64_t eis_addr = le64_to_cpu(req->cmd.dptr.prp2);
7435
0
    int i;
7436
7437
    /* Address should be page aligned */
7438
0
    if (dbs_addr & (n->page_size - 1) || eis_addr & (n->page_size - 1)) {
7439
0
        return NVME_INVALID_FIELD | NVME_DNR;
7440
0
    }
7441
7442
    /* Save shadow buffer base addr for use during queue creation */
7443
0
    n->dbbuf_dbs = dbs_addr;
7444
0
    n->dbbuf_eis = eis_addr;
7445
0
    n->dbbuf_enabled = true;
7446
7447
0
    for (i = 0; i < n->num_queues; i++) {
7448
0
        NvmeSQueue *sq = n->sq[i];
7449
0
        NvmeCQueue *cq = n->cq[i];
7450
7451
0
        if (sq) {
7452
            /*
7453
             * CAP.DSTRD is 0, so offset of ith sq db_addr is (i<<3)
7454
             * nvme_process_db() uses this hard-coded way to calculate
7455
             * doorbell offsets. Be consistent with that here.
7456
             */
7457
0
            sq->db_addr = dbs_addr + (i << 3);
7458
0
            sq->ei_addr = eis_addr + (i << 3);
7459
0
            stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED);
7460
7461
0
            if (n->params.ioeventfd && sq->sqid != 0) {
7462
0
                if (!nvme_init_sq_ioeventfd(sq)) {
7463
0
                    sq->ioeventfd_enabled = true;
7464
0
                }
7465
0
            }
7466
0
        }
7467
7468
0
        if (cq) {
7469
            /* CAP.DSTRD is 0, so offset of ith cq db_addr is (i<<3)+(1<<2) */
7470
0
            cq->db_addr = dbs_addr + (i << 3) + (1 << 2);
7471
0
            cq->ei_addr = eis_addr + (i << 3) + (1 << 2);
7472
0
            stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED);
7473
7474
0
            if (n->params.ioeventfd && cq->cqid != 0) {
7475
0
                if (!nvme_init_cq_ioeventfd(cq)) {
7476
0
                    cq->ioeventfd_enabled = true;
7477
0
                }
7478
0
            }
7479
0
        }
7480
0
    }
7481
7482
0
    trace_pci_nvme_dbbuf_config(dbs_addr, eis_addr);
7483
7484
0
    return NVME_SUCCESS;
7485
0
}
7486
7487
static uint16_t nvme_sec_prot_spdm_send(NvmeCtrl *n, NvmeRequest *req)
7488
0
{
7489
0
    StorageSpdmTransportHeader hdr = {0};
7490
0
    g_autofree uint8_t *sec_buf = NULL;
7491
0
    uint32_t transfer_len = le32_to_cpu(req->cmd.cdw11);
7492
0
    uint32_t transport_transfer_len = transfer_len;
7493
0
    uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7494
0
    uint32_t recvd;
7495
0
    uint16_t nvme_cmd_status, ret;
7496
0
    uint8_t secp = extract32(dw10, 24, 8);
7497
0
    uint16_t spsp = extract32(dw10, 8, 16);
7498
0
    bool spdm_res;
7499
7500
0
    if (transport_transfer_len > UINT32_MAX - sizeof(hdr)) {
7501
0
        return NVME_INVALID_FIELD | NVME_DNR;
7502
0
    }
7503
7504
0
    transport_transfer_len += sizeof(hdr);
7505
0
    if (transport_transfer_len > SPDM_SOCKET_MAX_MESSAGE_BUFFER_SIZE) {
7506
0
        return NVME_INVALID_FIELD | NVME_DNR;
7507
0
    }
7508
7509
0
    ret = nvme_check_mdts(n, transport_transfer_len);
7510
0
    if (ret != NVME_SUCCESS) {
7511
0
        return ret;
7512
0
    }
7513
7514
    /* Generate the NVMe transport header */
7515
0
    hdr.security_protocol = secp;
7516
0
    hdr.security_protocol_specific = cpu_to_le16(spsp);
7517
0
    hdr.length = cpu_to_le32(transfer_len);
7518
7519
0
    sec_buf = g_try_malloc0(transport_transfer_len);
7520
0
    if (!sec_buf) {
7521
0
        return NVME_INTERNAL_DEV_ERROR;
7522
0
    }
7523
7524
    /* Attach the transport header */
7525
0
    memcpy(sec_buf, &hdr, sizeof(hdr));
7526
0
    ret = nvme_h2c(n, sec_buf + sizeof(hdr), transfer_len, req);
7527
0
    if (ret) {
7528
0
        return ret;
7529
0
    }
7530
7531
0
    spdm_res = spdm_socket_send(n->spdm_socket, SPDM_SOCKET_STORAGE_CMD_IF_SEND,
7532
0
                                SPDM_TRANSPORT_TYPE_NVME, sec_buf,
7533
0
                                transport_transfer_len);
7534
0
    if (!spdm_res) {
7535
0
        return NVME_DATA_TRAS_ERROR | NVME_DNR;
7536
0
    }
7537
7538
    /* The responder shall ack with message status */
7539
0
    recvd = spdm_socket_receive(n->spdm_socket, SPDM_TRANSPORT_TYPE_NVME,
7540
0
                                &nvme_cmd_status,
7541
0
                                SPDM_SOCKET_MAX_MSG_STATUS_LEN);
7542
7543
0
    nvme_cmd_status = be16_to_cpu(nvme_cmd_status);
7544
7545
0
    if (recvd < SPDM_SOCKET_MAX_MSG_STATUS_LEN) {
7546
0
        return NVME_DATA_TRAS_ERROR | NVME_DNR;
7547
0
    }
7548
7549
0
    return nvme_cmd_status;
7550
0
}
7551
7552
/* From host to controller */
7553
static uint16_t nvme_security_send(NvmeCtrl *n, NvmeRequest *req)
7554
0
{
7555
0
    uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7556
0
    uint8_t secp = extract32(dw10, 24, 8);
7557
7558
0
    switch (secp) {
7559
0
    case NVME_SEC_PROT_DMTF_SPDM:
7560
0
        if (n->spdm_socket < 0) {
7561
0
            return NVME_INVALID_FIELD | NVME_DNR;
7562
0
        }
7563
0
        return nvme_sec_prot_spdm_send(n, req);
7564
0
    default:
7565
        /* Unsupported Security Protocol Type */
7566
0
        return NVME_INVALID_FIELD | NVME_DNR;
7567
0
    }
7568
7569
0
    return NVME_INVALID_FIELD | NVME_DNR;
7570
0
}
7571
7572
static uint16_t nvme_sec_prot_spdm_receive(NvmeCtrl *n, NvmeRequest *req)
7573
0
{
7574
0
    StorageSpdmTransportHeader hdr;
7575
0
    g_autofree uint8_t *rsp_spdm_buf = NULL;
7576
0
    uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7577
0
    uint32_t alloc_len = le32_to_cpu(req->cmd.cdw11);
7578
0
    uint32_t recvd, spdm_res;
7579
0
    uint16_t nvme_cmd_status, ret;
7580
0
    uint8_t secp = extract32(dw10, 24, 8);
7581
0
    uint8_t spsp = extract32(dw10, 8, 16);
7582
0
    if (!alloc_len) {
7583
0
        return NVME_INVALID_FIELD | NVME_DNR;
7584
0
    }
7585
7586
    /* Generate the NVMe transport header */
7587
0
    hdr = (StorageSpdmTransportHeader) {
7588
0
        .security_protocol = secp,
7589
0
        .security_protocol_specific = cpu_to_le16(spsp),
7590
0
        .length = cpu_to_le32(alloc_len),
7591
0
    };
7592
7593
    /* Forward if_recv to the SPDM Server with SPSP0 */
7594
0
    spdm_res = spdm_socket_send(n->spdm_socket, SPDM_SOCKET_STORAGE_CMD_IF_RECV,
7595
0
                                SPDM_TRANSPORT_TYPE_NVME,
7596
0
                                &hdr, sizeof(hdr));
7597
0
    if (!spdm_res) {
7598
0
        return NVME_DATA_TRAS_ERROR | NVME_DNR;
7599
0
    }
7600
7601
    /* The responder shall ack with message status */
7602
0
    recvd = spdm_socket_receive(n->spdm_socket, SPDM_TRANSPORT_TYPE_NVME,
7603
0
                                &nvme_cmd_status,
7604
0
                                SPDM_SOCKET_MAX_MSG_STATUS_LEN);
7605
0
    if (recvd < SPDM_SOCKET_MAX_MSG_STATUS_LEN) {
7606
0
        return NVME_DATA_TRAS_ERROR | NVME_DNR;
7607
0
    }
7608
7609
0
    nvme_cmd_status = be16_to_cpu(nvme_cmd_status);
7610
    /* An error here implies the prior if_recv from requester was spurious */
7611
0
    if (nvme_cmd_status != NVME_SUCCESS) {
7612
0
        return nvme_cmd_status;
7613
0
    }
7614
7615
    /* Clear to start receiving data from the server */
7616
0
    rsp_spdm_buf = g_try_malloc0(alloc_len);
7617
0
    if (!rsp_spdm_buf) {
7618
0
        return NVME_INTERNAL_DEV_ERROR;
7619
0
    }
7620
7621
0
    recvd = spdm_socket_receive(n->spdm_socket,
7622
0
                                SPDM_TRANSPORT_TYPE_NVME,
7623
0
                                rsp_spdm_buf, alloc_len);
7624
0
    if (!recvd) {
7625
0
        return NVME_DATA_TRAS_ERROR | NVME_DNR;
7626
0
    }
7627
7628
0
    ret = nvme_c2h(n, rsp_spdm_buf, MIN(recvd, alloc_len), req);
7629
0
    if (ret) {
7630
0
        return ret;
7631
0
    }
7632
7633
0
    return NVME_SUCCESS;
7634
0
}
7635
7636
static uint16_t nvme_get_sec_prot_info(NvmeCtrl *n, NvmeRequest *req)
7637
0
{
7638
0
    uint32_t alloc_len = le32_to_cpu(req->cmd.cdw11);
7639
0
    uint8_t resp[10] = {
7640
        /* Support Security Protol List Length */
7641
0
        [6] = 0, /* MSB */
7642
0
        [7] = 2, /* LSB */
7643
        /* Support Security Protocol List */
7644
0
        [8] = SFSC_SECURITY_PROT_INFO,
7645
0
        [9] = 0,
7646
0
    };
7647
7648
0
    if (n->spdm_socket >= 0) {
7649
0
        resp[9] = NVME_SEC_PROT_DMTF_SPDM;
7650
0
    }
7651
7652
0
    if (alloc_len < 10) {
7653
0
        return NVME_INVALID_FIELD | NVME_DNR;
7654
0
    }
7655
7656
0
    return nvme_c2h(n, resp, sizeof(resp), req);
7657
0
}
7658
7659
/* From controller to host */
7660
static uint16_t nvme_security_receive(NvmeCtrl *n, NvmeRequest *req)
7661
0
{
7662
0
    uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7663
0
    uint16_t spsp = extract32(dw10, 8, 16);
7664
0
    uint8_t secp = extract32(dw10, 24, 8);
7665
7666
0
    switch (secp) {
7667
0
    case SFSC_SECURITY_PROT_INFO:
7668
0
        switch (spsp) {
7669
0
        case 0:
7670
            /* Supported security protocol list */
7671
0
            return nvme_get_sec_prot_info(n, req);
7672
0
        case 1:
7673
            /* Certificate data */
7674
            /* fallthrough */
7675
0
        default:
7676
0
            return NVME_INVALID_FIELD | NVME_DNR;
7677
0
        }
7678
0
    case NVME_SEC_PROT_DMTF_SPDM:
7679
0
        if (n->spdm_socket < 0) {
7680
0
            return NVME_INVALID_FIELD | NVME_DNR;
7681
0
        }
7682
0
        return nvme_sec_prot_spdm_receive(n, req);
7683
0
    default:
7684
0
        return NVME_INVALID_FIELD | NVME_DNR;
7685
0
    }
7686
0
}
7687
7688
static uint16_t nvme_directive_send(NvmeCtrl *n, NvmeRequest *req)
7689
0
{
7690
    /*
7691
     * When adding a new dtype handling here,
7692
     * please also update nvme_set_migration_blockers().
7693
     */
7694
0
    return NVME_INVALID_FIELD | NVME_DNR;
7695
0
}
7696
7697
static uint16_t nvme_directive_receive(NvmeCtrl *n, NvmeRequest *req)
7698
0
{
7699
0
    NvmeNamespace *ns;
7700
0
    uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7701
0
    uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
7702
0
    uint32_t nsid = le32_to_cpu(req->cmd.nsid);
7703
0
    uint8_t doper, dtype;
7704
0
    uint32_t numd, trans_len;
7705
0
    NvmeDirectiveIdentify id = {
7706
0
        .supported = 1 << NVME_DIRECTIVE_IDENTIFY,
7707
0
        .enabled = 1 << NVME_DIRECTIVE_IDENTIFY,
7708
0
    };
7709
7710
0
    numd = dw10 + 1;
7711
0
    doper = dw11 & 0xff;
7712
0
    dtype = (dw11 >> 8) & 0xff;
7713
7714
0
    trans_len = MIN(sizeof(NvmeDirectiveIdentify), numd << 2);
7715
7716
0
    if (nsid == NVME_NSID_BROADCAST || dtype != NVME_DIRECTIVE_IDENTIFY ||
7717
0
        doper != NVME_DIRECTIVE_RETURN_PARAMS) {
7718
0
        return NVME_INVALID_FIELD | NVME_DNR;
7719
0
    }
7720
7721
0
    ns = nvme_ns(n, nsid);
7722
0
    if (!ns) {
7723
0
        return NVME_INVALID_FIELD | NVME_DNR;
7724
0
    }
7725
7726
0
    switch (dtype) {
7727
0
    case NVME_DIRECTIVE_IDENTIFY:
7728
0
        switch (doper) {
7729
0
        case NVME_DIRECTIVE_RETURN_PARAMS:
7730
0
            if (ns->endgrp && ns->endgrp->fdp.enabled) {
7731
0
                id.supported |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
7732
0
                id.enabled |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
7733
0
                id.persistent |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
7734
0
            }
7735
7736
0
            return nvme_c2h(n, (uint8_t *)&id, trans_len, req);
7737
7738
0
        default:
7739
0
            return NVME_INVALID_FIELD | NVME_DNR;
7740
0
        }
7741
7742
0
    default:
7743
0
        return NVME_INVALID_FIELD;
7744
0
    }
7745
0
}
7746
7747
static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeRequest *req)
7748
0
{
7749
0
    trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), req->cmd.opcode,
7750
0
                             nvme_adm_opc_str(req->cmd.opcode));
7751
7752
0
    if (!(n->cse.acs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
7753
0
        trace_pci_nvme_err_invalid_admin_opc(req->cmd.opcode);
7754
0
        return NVME_INVALID_OPCODE | NVME_DNR;
7755
0
    }
7756
7757
    /* SGLs shall not be used for Admin commands in NVMe over PCIe */
7758
0
    if (NVME_CMD_FLAGS_PSDT(req->cmd.flags) != NVME_PSDT_PRP) {
7759
0
        return NVME_INVALID_FIELD | NVME_DNR;
7760
0
    }
7761
7762
0
    if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
7763
0
        return NVME_INVALID_FIELD;
7764
0
    }
7765
7766
0
    switch (req->cmd.opcode) {
7767
0
    case NVME_ADM_CMD_DELETE_SQ:
7768
0
        return nvme_del_sq(n, req);
7769
0
    case NVME_ADM_CMD_CREATE_SQ:
7770
0
        return nvme_create_sq(n, req);
7771
0
    case NVME_ADM_CMD_GET_LOG_PAGE:
7772
0
        return nvme_get_log(n, req);
7773
0
    case NVME_ADM_CMD_DELETE_CQ:
7774
0
        return nvme_del_cq(n, req);
7775
0
    case NVME_ADM_CMD_CREATE_CQ:
7776
0
        return nvme_create_cq(n, req);
7777
0
    case NVME_ADM_CMD_IDENTIFY:
7778
0
        return nvme_identify(n, req);
7779
0
    case NVME_ADM_CMD_ABORT:
7780
0
        return nvme_abort(n, req);
7781
0
    case NVME_ADM_CMD_SET_FEATURES:
7782
0
        return nvme_set_feature(n, req);
7783
0
    case NVME_ADM_CMD_GET_FEATURES:
7784
0
        return nvme_get_feature(n, req);
7785
0
    case NVME_ADM_CMD_ASYNC_EV_REQ:
7786
0
        return nvme_aer(n, req);
7787
0
    case NVME_ADM_CMD_NS_ATTACHMENT:
7788
0
        return nvme_ns_attachment(n, req);
7789
0
    case NVME_ADM_CMD_VIRT_MNGMT:
7790
0
        return nvme_virt_mngmt(n, req);
7791
0
    case NVME_ADM_CMD_DBBUF_CONFIG:
7792
0
        return nvme_dbbuf_config(n, req);
7793
0
    case NVME_ADM_CMD_FORMAT_NVM:
7794
0
        return nvme_format(n, req);
7795
0
    case NVME_ADM_CMD_DIRECTIVE_SEND:
7796
0
        return nvme_directive_send(n, req);
7797
0
    case NVME_ADM_CMD_DIRECTIVE_RECV:
7798
0
        return nvme_directive_receive(n, req);
7799
0
    case NVME_ADM_CMD_SECURITY_SEND:
7800
0
        return nvme_security_send(n, req);
7801
0
    case NVME_ADM_CMD_SECURITY_RECV:
7802
0
        return nvme_security_receive(n, req);
7803
0
    default:
7804
0
        g_assert_not_reached();
7805
0
    }
7806
7807
0
    return NVME_INVALID_OPCODE | NVME_DNR;
7808
0
}
7809
7810
static void nvme_update_sq_eventidx(const NvmeSQueue *sq)
7811
0
{
7812
0
    trace_pci_nvme_update_sq_eventidx(sq->sqid, sq->tail);
7813
7814
0
    stl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->ei_addr, sq->tail,
7815
0
                   MEMTXATTRS_UNSPECIFIED);
7816
0
}
7817
7818
static void nvme_update_sq_tail(NvmeSQueue *sq)
7819
0
{
7820
0
    ldl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->db_addr, &sq->tail,
7821
0
                   MEMTXATTRS_UNSPECIFIED);
7822
7823
0
    trace_pci_nvme_update_sq_tail(sq->sqid, sq->tail);
7824
0
}
7825
7826
static int nvme_atomic_boundary_check(NvmeCtrl *n, NvmeCmd *cmd,
7827
    NvmeAtomic *atomic)
7828
0
{
7829
0
    NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
7830
7831
0
    if (atomic->atomic_boundary) {
7832
0
        uint64_t slba = le64_to_cpu(rw->slba);
7833
0
        uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb);
7834
0
        uint64_t elba = slba + nlb;
7835
0
        uint64_t imask;
7836
7837
0
        if ((slba < atomic->atomic_nabo) || (elba < atomic->atomic_nabo)) {
7838
0
            return 0;
7839
0
        }
7840
7841
        /* Update slba/elba based on boundary offset */
7842
0
        slba = slba - atomic->atomic_nabo;
7843
0
        elba = slba + nlb;
7844
7845
0
        imask = ~(atomic->atomic_boundary - 1);
7846
0
        if ((slba & imask) != (elba & imask)) {
7847
            /*
7848
             * The write crosses an atomic boundary and the controller provides
7849
             * no atomicity guarantees unless AWUN/AWUPF are non-zero.
7850
             */
7851
0
            if (n->atomic.atomic_max_write_size &&
7852
0
                ((nlb + 1) <= n->atomic.atomic_max_write_size)) {
7853
0
                return 1;
7854
0
            }
7855
0
            return 0;
7856
0
        }
7857
0
    }
7858
0
    return 1;
7859
0
}
7860
0
#define NVME_ATOMIC_NO_START        0
7861
0
#define NVME_ATOMIC_START_ATOMIC    1
7862
0
#define NVME_ATOMIC_START_NONATOMIC 2
7863
7864
static int nvme_atomic_write_check(NvmeCtrl *n, NvmeCmd *cmd,
7865
    NvmeAtomic *atomic)
7866
0
{
7867
0
    NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
7868
0
    uint64_t slba = le64_to_cpu(rw->slba);
7869
0
    uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb);
7870
0
    uint64_t elba = slba + nlb;
7871
0
    bool cmd_atomic_wr = true;
7872
0
    int i;
7873
7874
0
    if ((cmd->opcode == NVME_CMD_READ) || ((cmd->opcode == NVME_CMD_WRITE) &&
7875
0
        ((rw->nlb + 1) > atomic->atomic_max_write_size))) {
7876
0
        cmd_atomic_wr = false;
7877
0
    }
7878
7879
    /*
7880
     * Check if a write crosses an atomic boundary.
7881
     */
7882
0
    if (cmd->opcode == NVME_CMD_WRITE) {
7883
0
        if (!nvme_atomic_boundary_check(n, cmd, atomic)) {
7884
0
            cmd_atomic_wr = false;
7885
0
        }
7886
0
    }
7887
7888
    /*
7889
     * Walk the queues to see if there are any atomic conflicts.
7890
     */
7891
0
    for (i = 1; i < n->num_queues; i++) {
7892
0
        NvmeSQueue *sq;
7893
0
        NvmeRequest *req;
7894
0
        NvmeRwCmd *req_rw;
7895
0
        uint64_t req_slba;
7896
0
        uint32_t req_nlb;
7897
0
        uint64_t req_elba;
7898
7899
0
        sq = n->sq[i];
7900
0
        if (!sq) {
7901
0
            continue;
7902
0
        }
7903
7904
        /*
7905
         * Walk all the requests on a given queue.
7906
         */
7907
0
        QTAILQ_FOREACH(req, &sq->out_req_list, entry) {
7908
0
            req_rw = (NvmeRwCmd *)&req->cmd;
7909
7910
0
            if (((req_rw->opcode == NVME_CMD_WRITE) ||
7911
0
                 (req_rw->opcode == NVME_CMD_READ)) &&
7912
0
                (cmd->nsid == req->ns->params.nsid)) {
7913
0
                req_slba = le64_to_cpu(req_rw->slba);
7914
0
                req_nlb = (uint32_t)le16_to_cpu(req_rw->nlb);
7915
0
                req_elba = req_slba + req_nlb;
7916
7917
0
                if (cmd_atomic_wr) {
7918
0
                    if ((elba >= req_slba) && (slba <= req_elba)) {
7919
0
                        return NVME_ATOMIC_NO_START;
7920
0
                    }
7921
0
                } else {
7922
0
                    if (req->atomic_write && ((elba >= req_slba) &&
7923
0
                        (slba <= req_elba))) {
7924
0
                        return NVME_ATOMIC_NO_START;
7925
0
                    }
7926
0
                }
7927
0
            }
7928
0
        }
7929
0
    }
7930
0
    if (cmd_atomic_wr) {
7931
0
        return NVME_ATOMIC_START_ATOMIC;
7932
0
    }
7933
0
    return NVME_ATOMIC_START_NONATOMIC;
7934
0
}
7935
7936
static NvmeAtomic *nvme_get_atomic(NvmeCtrl *n, NvmeCmd *cmd)
7937
0
{
7938
0
    NvmeNamespace *ns = nvme_ns(n, cmd->nsid);
7939
7940
0
    if (ns && ns->atomic.atomic_writes) {
7941
0
        return &ns->atomic;
7942
0
    }
7943
7944
0
    if (n->atomic.atomic_writes) {
7945
0
        return &n->atomic;
7946
0
    }
7947
0
    return NULL;
7948
0
}
7949
7950
static void nvme_process_sq(void *opaque)
7951
0
{
7952
0
    NvmeSQueue *sq = opaque;
7953
0
    NvmeCtrl *n = sq->ctrl;
7954
0
    NvmeCQueue *cq = n->cq[sq->cqid];
7955
7956
0
    uint16_t status;
7957
0
    hwaddr addr;
7958
0
    NvmeCmd cmd;
7959
0
    NvmeRequest *req;
7960
7961
    /*
7962
     * We don't want to have a race with nvme_ctrl_pre_save().
7963
     * What implicitly protects us from this is BQL.
7964
     */
7965
0
    assert(bql_locked());
7966
7967
0
    if (n->dbbuf_enabled) {
7968
0
        nvme_update_sq_tail(sq);
7969
0
    }
7970
7971
0
    while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
7972
0
        NvmeAtomic *atomic;
7973
0
        bool cmd_is_atomic;
7974
7975
0
        addr = sq->dma_addr + (sq->head << NVME_SQES);
7976
0
        if (nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd))) {
7977
0
            trace_pci_nvme_err_addr_read(addr);
7978
0
            trace_pci_nvme_err_cfs();
7979
0
            stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
7980
0
            break;
7981
0
        }
7982
7983
0
        atomic = nvme_get_atomic(n, &cmd);
7984
7985
0
        cmd_is_atomic = false;
7986
0
        if (sq->sqid && atomic) {
7987
0
            int ret;
7988
7989
0
            ret = nvme_atomic_write_check(n, &cmd, atomic);
7990
0
            switch (ret) {
7991
0
            case NVME_ATOMIC_NO_START:
7992
0
                qemu_bh_schedule(sq->bh);
7993
0
                return;
7994
0
            case NVME_ATOMIC_START_ATOMIC:
7995
0
                cmd_is_atomic = true;
7996
0
                break;
7997
0
            case NVME_ATOMIC_START_NONATOMIC:
7998
0
            default:
7999
0
                break;
8000
0
            }
8001
0
        }
8002
0
        nvme_inc_sq_head(sq);
8003
8004
0
        req = QTAILQ_FIRST(&sq->req_list);
8005
0
        QTAILQ_REMOVE(&sq->req_list, req, entry);
8006
0
        QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
8007
0
        nvme_req_clear(req);
8008
0
        req->cqe.sq_id = cpu_to_le16(sq->sqid);
8009
0
        req->cqe.cid = cmd.cid;
8010
0
        memcpy(&req->cmd, &cmd, sizeof(NvmeCmd));
8011
8012
0
        if (sq->sqid && atomic) {
8013
0
            req->atomic_write = cmd_is_atomic;
8014
0
        }
8015
8016
0
        status = sq->sqid ? nvme_io_cmd(n, req) :
8017
0
            nvme_admin_cmd(n, req);
8018
0
        if (status != NVME_NO_COMPLETE) {
8019
0
            req->status = status;
8020
0
            nvme_enqueue_req_completion(cq, req);
8021
0
        }
8022
8023
0
        if (n->dbbuf_enabled) {
8024
0
            nvme_update_sq_eventidx(sq);
8025
0
            nvme_update_sq_tail(sq);
8026
0
        }
8027
0
    }
8028
0
}
8029
8030
static void nvme_update_msixcap_ts(PCIDevice *pci_dev, uint32_t table_size)
8031
0
{
8032
0
    uint8_t *config;
8033
8034
0
    if (!msix_present(pci_dev)) {
8035
0
        return;
8036
0
    }
8037
8038
0
    assert(table_size > 0 && table_size <= pci_dev->msix_entries_nr);
8039
8040
0
    config = pci_dev->config + pci_dev->msix_cap;
8041
0
    pci_set_word_by_mask(config + PCI_MSIX_FLAGS, PCI_MSIX_FLAGS_QSIZE,
8042
0
                         table_size - 1);
8043
0
}
8044
8045
static void nvme_activate_virt_res(NvmeCtrl *n)
8046
0
{
8047
0
    PCIDevice *pci_dev = PCI_DEVICE(n);
8048
0
    NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
8049
0
    NvmeSecCtrlEntry *sctrl;
8050
8051
    /* -1 to account for the admin queue */
8052
0
    if (pci_is_vf(pci_dev)) {
8053
0
        sctrl = nvme_sctrl(n);
8054
0
        cap->vqprt = sctrl->nvq;
8055
0
        cap->viprt = sctrl->nvi;
8056
0
        n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
8057
0
        n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
8058
0
    } else {
8059
0
        cap->vqrfap = n->next_pri_ctrl_cap.vqrfap;
8060
0
        cap->virfap = n->next_pri_ctrl_cap.virfap;
8061
0
        n->conf_ioqpairs = le16_to_cpu(cap->vqprt) +
8062
0
                           le16_to_cpu(cap->vqrfap) - 1;
8063
0
        n->conf_msix_qsize = le16_to_cpu(cap->viprt) +
8064
0
                             le16_to_cpu(cap->virfap);
8065
0
    }
8066
0
}
8067
8068
static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst)
8069
0
{
8070
0
    PCIDevice *pci_dev = PCI_DEVICE(n);
8071
0
    NvmeSecCtrlEntry *sctrl;
8072
0
    NvmeNamespace *ns;
8073
0
    int i;
8074
8075
0
    for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
8076
0
        ns = nvme_ns(n, i);
8077
0
        if (!ns) {
8078
0
            continue;
8079
0
        }
8080
8081
0
        nvme_ns_drain(ns);
8082
0
    }
8083
8084
    /*
8085
     * Cancel and wait out every inflight command on every queue first. A
8086
     * reset is not required to be preceded by the guest's graceful
8087
     * Delete I/O SQ/CQ sequence, so sq/cq must not be freed below while a
8088
     * blk_aio_* completion for them could still be in flight.
8089
     */
8090
0
    for (i = 0; i < n->num_queues; i++) {
8091
0
        if (n->sq[i] != NULL) {
8092
0
            nvme_sq_cancel_inflight(n->sq[i], NVME_CMD_ABORT_SQ_DEL);
8093
0
        }
8094
0
    }
8095
8096
0
    for (i = 0; i < n->num_queues; i++) {
8097
0
        if (n->sq[i] != NULL) {
8098
0
            nvme_free_sq(n->sq[i], n);
8099
0
        }
8100
0
    }
8101
0
    for (i = 0; i < n->num_queues; i++) {
8102
0
        if (n->cq[i] != NULL) {
8103
0
            nvme_free_cq(n->cq[i], n);
8104
0
        }
8105
0
    }
8106
8107
0
    while (!QTAILQ_EMPTY(&n->aer_queue)) {
8108
0
        NvmeAsyncEvent *event = QTAILQ_FIRST(&n->aer_queue);
8109
0
        QTAILQ_REMOVE(&n->aer_queue, event, entry);
8110
0
        g_free(event);
8111
0
    }
8112
8113
0
    if (n->params.sriov_max_vfs) {
8114
0
        if (!pci_is_vf(pci_dev)) {
8115
0
            for (i = 0; i < n->nr_sec_ctrls; i++) {
8116
0
                sctrl = &n->sec_ctrl_list[i];
8117
0
                nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
8118
0
            }
8119
0
        }
8120
8121
0
        if (rst != NVME_RESET_CONTROLLER) {
8122
0
            nvme_activate_virt_res(n);
8123
0
        }
8124
0
    }
8125
8126
0
    n->aer_queued = 0;
8127
0
    n->aer_mask = 0;
8128
0
    n->outstanding_aers = 0;
8129
0
    n->qs_created = false;
8130
8131
0
    n->dn = n->params.atomic_dn; /* Set Disable Normal */
8132
8133
0
    nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
8134
8135
0
    if (pci_is_vf(pci_dev)) {
8136
0
        sctrl = nvme_sctrl(n);
8137
8138
0
        stl_le_p(&n->bar.csts, sctrl->scs ? 0 : NVME_CSTS_FAILED);
8139
0
    } else {
8140
0
        stl_le_p(&n->bar.csts, 0);
8141
0
    }
8142
8143
0
    stl_le_p(&n->bar.intms, 0);
8144
0
    stl_le_p(&n->bar.intmc, 0);
8145
0
    stl_le_p(&n->bar.cc, 0);
8146
8147
0
    n->cq_pending = 0;
8148
0
    n->irq_status = 0;
8149
0
    nvme_irq_check(n);
8150
8151
0
    n->dbbuf_dbs = 0;
8152
0
    n->dbbuf_eis = 0;
8153
0
    n->dbbuf_enabled = false;
8154
0
}
8155
8156
static void nvme_ctrl_shutdown(NvmeCtrl *n)
8157
0
{
8158
0
    NvmeNamespace *ns;
8159
0
    int i;
8160
8161
0
    if (n->pmr.dev) {
8162
0
        memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
8163
0
    }
8164
8165
0
    for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
8166
0
        ns = nvme_ns(n, i);
8167
0
        if (!ns) {
8168
0
            continue;
8169
0
        }
8170
8171
0
        nvme_ns_shutdown(ns);
8172
0
    }
8173
0
}
8174
8175
static int nvme_start_ctrl(NvmeCtrl *n)
8176
0
{
8177
0
    uint64_t cap = ldq_le_p(&n->bar.cap);
8178
0
    uint32_t cc = ldl_le_p(&n->bar.cc);
8179
0
    uint32_t aqa = ldl_le_p(&n->bar.aqa);
8180
0
    uint64_t asq = ldq_le_p(&n->bar.asq);
8181
0
    uint64_t acq = ldq_le_p(&n->bar.acq);
8182
0
    uint32_t page_bits = NVME_CC_MPS(cc) + 12;
8183
0
    uint32_t page_size = 1 << page_bits;
8184
0
    NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
8185
8186
0
    if (pci_is_vf(PCI_DEVICE(n)) && !sctrl->scs) {
8187
0
        trace_pci_nvme_err_startfail_virt_state(le16_to_cpu(sctrl->nvi),
8188
0
                                                le16_to_cpu(sctrl->nvq));
8189
0
        return -1;
8190
0
    }
8191
0
    if (unlikely(n->cq[0])) {
8192
0
        trace_pci_nvme_err_startfail_cq();
8193
0
        return -1;
8194
0
    }
8195
0
    if (unlikely(n->sq[0])) {
8196
0
        trace_pci_nvme_err_startfail_sq();
8197
0
        return -1;
8198
0
    }
8199
0
    if (unlikely(asq & (page_size - 1))) {
8200
0
        trace_pci_nvme_err_startfail_asq_misaligned(asq);
8201
0
        return -1;
8202
0
    }
8203
0
    if (unlikely(acq & (page_size - 1))) {
8204
0
        trace_pci_nvme_err_startfail_acq_misaligned(acq);
8205
0
        return -1;
8206
0
    }
8207
0
    if (unlikely(!(NVME_CAP_CSS(cap) & (1 << NVME_CC_CSS(cc))))) {
8208
0
        trace_pci_nvme_err_startfail_css(NVME_CC_CSS(cc));
8209
0
        return -1;
8210
0
    }
8211
0
    if (unlikely(NVME_CC_MPS(cc) < NVME_CAP_MPSMIN(cap))) {
8212
0
        trace_pci_nvme_err_startfail_page_too_small(
8213
0
                    NVME_CC_MPS(cc),
8214
0
                    NVME_CAP_MPSMIN(cap));
8215
0
        return -1;
8216
0
    }
8217
0
    if (unlikely(NVME_CC_MPS(cc) >
8218
0
                 NVME_CAP_MPSMAX(cap))) {
8219
0
        trace_pci_nvme_err_startfail_page_too_large(
8220
0
                    NVME_CC_MPS(cc),
8221
0
                    NVME_CAP_MPSMAX(cap));
8222
0
        return -1;
8223
0
    }
8224
0
    if (unlikely(!NVME_AQA_ASQS(aqa))) {
8225
0
        trace_pci_nvme_err_startfail_asqent_sz_zero();
8226
0
        return -1;
8227
0
    }
8228
0
    if (unlikely(!NVME_AQA_ACQS(aqa))) {
8229
0
        trace_pci_nvme_err_startfail_acqent_sz_zero();
8230
0
        return -1;
8231
0
    }
8232
8233
0
    n->page_bits = page_bits;
8234
0
    n->page_size = page_size;
8235
0
    n->max_prp_ents = n->page_size / sizeof(uint64_t);
8236
0
    nvme_init_cq(&n->admin_cq, n, acq, 0, 0, NVME_AQA_ACQS(aqa) + 1, 1);
8237
0
    nvme_init_sq(&n->admin_sq, n, asq, 0, 0, NVME_AQA_ASQS(aqa) + 1);
8238
8239
0
    nvme_set_timestamp(n, 0ULL);
8240
8241
    /* verify that the command sets of attached namespaces are supported */
8242
0
    for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) {
8243
0
        NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
8244
8245
0
        if (!ns || (!ns->params.shared && ns->ctrl != n)) {
8246
0
            continue;
8247
0
        }
8248
8249
0
        if (nvme_csi_supported(n, ns->csi) && !ns->params.detached) {
8250
0
            if (!ns->attached || ns->params.shared) {
8251
0
                nvme_attach_ns(n, ns);
8252
0
            }
8253
0
        }
8254
0
    }
8255
8256
0
    nvme_update_dsm_limits(n, NULL);
8257
8258
0
    return 0;
8259
0
}
8260
8261
static void nvme_cmb_enable_regs(NvmeCtrl *n)
8262
0
{
8263
0
    uint32_t cmbloc = ldl_le_p(&n->bar.cmbloc);
8264
0
    uint32_t cmbsz = ldl_le_p(&n->bar.cmbsz);
8265
8266
0
    NVME_CMBLOC_SET_CDPCILS(cmbloc, 1);
8267
0
    NVME_CMBLOC_SET_CDPMLS(cmbloc, 1);
8268
0
    NVME_CMBLOC_SET_BIR(cmbloc, NVME_CMB_BIR);
8269
0
    stl_le_p(&n->bar.cmbloc, cmbloc);
8270
8271
0
    NVME_CMBSZ_SET_SQS(cmbsz, 1);
8272
0
    NVME_CMBSZ_SET_CQS(cmbsz, 0);
8273
0
    NVME_CMBSZ_SET_LISTS(cmbsz, 1);
8274
0
    NVME_CMBSZ_SET_RDS(cmbsz, 1);
8275
0
    NVME_CMBSZ_SET_WDS(cmbsz, 1);
8276
0
    NVME_CMBSZ_SET_SZU(cmbsz, 2); /* MBs */
8277
0
    NVME_CMBSZ_SET_SZ(cmbsz, n->params.cmb_size_mb);
8278
0
    stl_le_p(&n->bar.cmbsz, cmbsz);
8279
0
}
8280
8281
static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
8282
                           unsigned size)
8283
0
{
8284
0
    PCIDevice *pci = PCI_DEVICE(n);
8285
0
    uint64_t cap = ldq_le_p(&n->bar.cap);
8286
0
    uint32_t cc = ldl_le_p(&n->bar.cc);
8287
0
    uint32_t intms = ldl_le_p(&n->bar.intms);
8288
0
    uint32_t csts = ldl_le_p(&n->bar.csts);
8289
0
    uint32_t pmrsts = ldl_le_p(&n->bar.pmrsts);
8290
8291
0
    if (unlikely(offset & (sizeof(uint32_t) - 1))) {
8292
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32,
8293
0
                       "MMIO write not 32-bit aligned,"
8294
0
                       " offset=0x%"PRIx64"", offset);
8295
        /* should be ignored, fall through for now */
8296
0
    }
8297
8298
0
    if (unlikely(size < sizeof(uint32_t))) {
8299
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall,
8300
0
                       "MMIO write smaller than 32-bits,"
8301
0
                       " offset=0x%"PRIx64", size=%u",
8302
0
                       offset, size);
8303
        /* should be ignored, fall through for now */
8304
0
    }
8305
8306
0
    switch (offset) {
8307
0
    case NVME_REG_INTMS:
8308
0
        if (unlikely(msix_enabled(pci))) {
8309
0
            NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
8310
0
                           "undefined access to interrupt mask set"
8311
0
                           " when MSI-X is enabled");
8312
            /* should be ignored, fall through for now */
8313
0
        }
8314
0
        intms |= data;
8315
0
        stl_le_p(&n->bar.intms, intms);
8316
0
        n->bar.intmc = n->bar.intms;
8317
0
        trace_pci_nvme_mmio_intm_set(data & 0xffffffff, intms);
8318
0
        nvme_irq_check(n);
8319
0
        break;
8320
0
    case NVME_REG_INTMC:
8321
0
        if (unlikely(msix_enabled(pci))) {
8322
0
            NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
8323
0
                           "undefined access to interrupt mask clr"
8324
0
                           " when MSI-X is enabled");
8325
            /* should be ignored, fall through for now */
8326
0
        }
8327
0
        intms &= ~data;
8328
0
        stl_le_p(&n->bar.intms, intms);
8329
0
        n->bar.intmc = n->bar.intms;
8330
0
        trace_pci_nvme_mmio_intm_clr(data & 0xffffffff, intms);
8331
0
        nvme_irq_check(n);
8332
0
        break;
8333
0
    case NVME_REG_CC:
8334
0
        stl_le_p(&n->bar.cc, data);
8335
8336
0
        trace_pci_nvme_mmio_cfg(data & 0xffffffff);
8337
8338
0
        if (NVME_CC_SHN(data) && !(NVME_CC_SHN(cc))) {
8339
0
            trace_pci_nvme_mmio_shutdown_set();
8340
0
            nvme_ctrl_shutdown(n);
8341
0
            csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
8342
0
            csts |= NVME_CSTS_SHST_COMPLETE;
8343
0
        } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(cc)) {
8344
0
            trace_pci_nvme_mmio_shutdown_cleared();
8345
0
            csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
8346
0
        }
8347
8348
0
        if (NVME_CC_EN(data) && !NVME_CC_EN(cc)) {
8349
0
            if (unlikely(nvme_start_ctrl(n))) {
8350
0
                trace_pci_nvme_err_startfail();
8351
0
                csts = NVME_CSTS_FAILED;
8352
0
            } else {
8353
0
                trace_pci_nvme_mmio_start_success();
8354
0
                csts = NVME_CSTS_READY;
8355
0
            }
8356
0
        } else if (!NVME_CC_EN(data) && NVME_CC_EN(cc)) {
8357
0
            trace_pci_nvme_mmio_stopped();
8358
0
            nvme_ctrl_reset(n, NVME_RESET_CONTROLLER);
8359
8360
0
            break;
8361
0
        }
8362
8363
0
        stl_le_p(&n->bar.csts, csts);
8364
8365
0
        break;
8366
0
    case NVME_REG_CSTS:
8367
0
        if (data & (1 << 4)) {
8368
0
            NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported,
8369
0
                           "attempted to W1C CSTS.NSSRO"
8370
0
                           " but CAP.NSSRS is zero (not supported)");
8371
0
        } else if (data != 0) {
8372
0
            NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts,
8373
0
                           "attempted to set a read only bit"
8374
0
                           " of controller status");
8375
0
        }
8376
0
        break;
8377
0
    case NVME_REG_NSSR:
8378
0
        if (data == 0x4e564d65) {
8379
0
            trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
8380
0
        } else {
8381
            /* The spec says that writes of other values have no effect */
8382
0
            return;
8383
0
        }
8384
0
        break;
8385
0
    case NVME_REG_AQA:
8386
0
        stl_le_p(&n->bar.aqa, data);
8387
0
        trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
8388
0
        break;
8389
0
    case NVME_REG_ASQ:
8390
0
        stn_le_p(&n->bar.asq, size, data);
8391
0
        trace_pci_nvme_mmio_asqaddr(data);
8392
0
        break;
8393
0
    case NVME_REG_ASQ + 4:
8394
0
        stl_le_p((uint8_t *)&n->bar.asq + 4, data);
8395
0
        trace_pci_nvme_mmio_asqaddr_hi(data, ldq_le_p(&n->bar.asq));
8396
0
        break;
8397
0
    case NVME_REG_ACQ:
8398
0
        trace_pci_nvme_mmio_acqaddr(data);
8399
0
        stn_le_p(&n->bar.acq, size, data);
8400
0
        break;
8401
0
    case NVME_REG_ACQ + 4:
8402
0
        stl_le_p((uint8_t *)&n->bar.acq + 4, data);
8403
0
        trace_pci_nvme_mmio_acqaddr_hi(data, ldq_le_p(&n->bar.acq));
8404
0
        break;
8405
0
    case NVME_REG_CMBLOC:
8406
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved,
8407
0
                       "invalid write to reserved CMBLOC"
8408
0
                       " when CMBSZ is zero, ignored");
8409
0
        return;
8410
0
    case NVME_REG_CMBSZ:
8411
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly,
8412
0
                       "invalid write to read only CMBSZ, ignored");
8413
0
        return;
8414
0
    case NVME_REG_CMBMSC:
8415
0
        if (!NVME_CAP_CMBS(cap)) {
8416
0
            return;
8417
0
        }
8418
8419
0
        stn_le_p(&n->bar.cmbmsc, size, data);
8420
0
        n->cmb.cmse = false;
8421
8422
0
        if (NVME_CMBMSC_CRE(data)) {
8423
0
            nvme_cmb_enable_regs(n);
8424
8425
0
            if (NVME_CMBMSC_CMSE(data)) {
8426
0
                uint64_t cmbmsc = ldq_le_p(&n->bar.cmbmsc);
8427
0
                hwaddr cba = NVME_CMBMSC_CBA(cmbmsc) << CMBMSC_CBA_SHIFT;
8428
0
                if (cba + int128_get64(n->cmb.mem.size) < cba) {
8429
0
                    uint32_t cmbsts = ldl_le_p(&n->bar.cmbsts);
8430
0
                    NVME_CMBSTS_SET_CBAI(cmbsts, 1);
8431
0
                    stl_le_p(&n->bar.cmbsts, cmbsts);
8432
0
                    return;
8433
0
                }
8434
8435
0
                n->cmb.cba = cba;
8436
0
                n->cmb.cmse = true;
8437
0
            }
8438
0
        } else {
8439
0
            n->bar.cmbsz = 0;
8440
0
            n->bar.cmbloc = 0;
8441
0
        }
8442
8443
0
        return;
8444
0
    case NVME_REG_CMBMSC + 4:
8445
0
        stl_le_p((uint8_t *)&n->bar.cmbmsc + 4, data);
8446
0
        return;
8447
8448
0
    case NVME_REG_PMRCAP:
8449
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly,
8450
0
                       "invalid write to PMRCAP register, ignored");
8451
0
        return;
8452
0
    case NVME_REG_PMRCTL:
8453
0
        if (!NVME_CAP_PMRS(cap)) {
8454
0
            return;
8455
0
        }
8456
8457
0
        stl_le_p(&n->bar.pmrctl, data);
8458
0
        if (NVME_PMRCTL_EN(data)) {
8459
0
            memory_region_set_enabled(&n->pmr.dev->mr, true);
8460
0
            pmrsts = 0;
8461
0
        } else {
8462
0
            memory_region_set_enabled(&n->pmr.dev->mr, false);
8463
0
            NVME_PMRSTS_SET_NRDY(pmrsts, 1);
8464
0
            n->pmr.cmse = false;
8465
0
        }
8466
0
        stl_le_p(&n->bar.pmrsts, pmrsts);
8467
0
        return;
8468
0
    case NVME_REG_PMRSTS:
8469
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly,
8470
0
                       "invalid write to PMRSTS register, ignored");
8471
0
        return;
8472
0
    case NVME_REG_PMREBS:
8473
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly,
8474
0
                       "invalid write to PMREBS register, ignored");
8475
0
        return;
8476
0
    case NVME_REG_PMRSWTP:
8477
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly,
8478
0
                       "invalid write to PMRSWTP register, ignored");
8479
0
        return;
8480
0
    case NVME_REG_PMRMSCL:
8481
0
        if (!NVME_CAP_PMRS(cap)) {
8482
0
            return;
8483
0
        }
8484
8485
0
        stl_le_p(&n->bar.pmrmscl, data);
8486
0
        n->pmr.cmse = false;
8487
8488
0
        if (NVME_PMRMSCL_CMSE(data)) {
8489
0
            uint64_t pmrmscu = ldl_le_p(&n->bar.pmrmscu);
8490
0
            hwaddr cba = pmrmscu << 32 |
8491
0
                (NVME_PMRMSCL_CBA(data) << PMRMSCL_CBA_SHIFT);
8492
0
            if (cba + int128_get64(n->pmr.dev->mr.size) < cba) {
8493
0
                NVME_PMRSTS_SET_CBAI(pmrsts, 1);
8494
0
                stl_le_p(&n->bar.pmrsts, pmrsts);
8495
0
                return;
8496
0
            }
8497
8498
0
            n->pmr.cmse = true;
8499
0
            n->pmr.cba = cba;
8500
0
        }
8501
8502
0
        return;
8503
0
    case NVME_REG_PMRMSCU:
8504
0
        if (!NVME_CAP_PMRS(cap)) {
8505
0
            return;
8506
0
        }
8507
8508
0
        stl_le_p(&n->bar.pmrmscu, data);
8509
0
        return;
8510
0
    default:
8511
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid,
8512
0
                       "invalid MMIO write,"
8513
0
                       " offset=0x%"PRIx64", data=%"PRIx64"",
8514
0
                       offset, data);
8515
0
        break;
8516
0
    }
8517
0
}
8518
8519
static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
8520
0
{
8521
0
    NvmeCtrl *n = (NvmeCtrl *)opaque;
8522
0
    uint8_t *ptr = (uint8_t *)&n->bar;
8523
8524
0
    trace_pci_nvme_mmio_read(addr, size);
8525
8526
0
    if (unlikely(addr & (sizeof(uint32_t) - 1))) {
8527
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32,
8528
0
                       "MMIO read not 32-bit aligned,"
8529
0
                       " offset=0x%"PRIx64"", addr);
8530
        /* should RAZ, fall through for now */
8531
0
    } else if (unlikely(size < sizeof(uint32_t))) {
8532
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall,
8533
0
                       "MMIO read smaller than 32-bits,"
8534
0
                       " offset=0x%"PRIx64"", addr);
8535
        /* should RAZ, fall through for now */
8536
0
    }
8537
8538
0
    if (addr > sizeof(n->bar) - size) {
8539
0
        NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs,
8540
0
                       "MMIO read beyond last register,"
8541
0
                       " offset=0x%"PRIx64", returning 0", addr);
8542
8543
0
        return 0;
8544
0
    }
8545
8546
0
    if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
8547
0
        addr != NVME_REG_CSTS) {
8548
0
        trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
8549
0
        return 0;
8550
0
    }
8551
8552
    /*
8553
     * When PMRWBM bit 1 is set then read from
8554
     * from PMRSTS should ensure prior writes
8555
     * made it to persistent media
8556
     */
8557
0
    if (addr == NVME_REG_PMRSTS &&
8558
0
        (NVME_PMRCAP_PMRWBM(ldl_le_p(&n->bar.pmrcap)) & 0x02)) {
8559
0
        memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
8560
0
    }
8561
8562
0
    return ldn_le_p(ptr + addr, size);
8563
0
}
8564
8565
static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
8566
0
{
8567
0
    PCIDevice *pci = PCI_DEVICE(n);
8568
0
    uint32_t qid;
8569
8570
0
    if (unlikely(addr & ((1 << 2) - 1))) {
8571
0
        NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned,
8572
0
                       "doorbell write not 32-bit aligned,"
8573
0
                       " offset=0x%"PRIx64", ignoring", addr);
8574
0
        return;
8575
0
    }
8576
8577
0
    if (((addr - 0x1000) >> 2) & 1) {
8578
        /* Completion queue doorbell write */
8579
8580
0
        uint16_t new_head = val & 0xffff;
8581
0
        NvmeCQueue *cq;
8582
0
        bool pending;
8583
8584
0
        qid = (addr - (0x1000 + (1 << 2))) >> 3;
8585
0
        if (unlikely(nvme_check_cqid(n, qid))) {
8586
0
            NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq,
8587
0
                           "completion queue doorbell write"
8588
0
                           " for nonexistent queue,"
8589
0
                           " sqid=%"PRIu32", ignoring", qid);
8590
8591
            /*
8592
             * NVM Express v1.3d, Section 4.1 state: "If host software writes
8593
             * an invalid value to the Submission Queue Tail Doorbell or
8594
             * Completion Queue Head Doorbell register and an Asynchronous Event
8595
             * Request command is outstanding, then an asynchronous event is
8596
             * posted to the Admin Completion Queue with a status code of
8597
             * Invalid Doorbell Write Value."
8598
             *
8599
             * Also note that the spec includes the "Invalid Doorbell Register"
8600
             * status code, but nowhere does it specify when to use it.
8601
             * However, it seems reasonable to use it here in a similar
8602
             * fashion.
8603
             */
8604
0
            if (n->outstanding_aers) {
8605
0
                nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
8606
0
                                   NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
8607
0
                                   NVME_LOG_ERROR_INFO);
8608
0
            }
8609
8610
0
            return;
8611
0
        }
8612
8613
0
        cq = n->cq[qid];
8614
0
        if (unlikely(new_head >= cq->size)) {
8615
0
            NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead,
8616
0
                           "completion queue doorbell write value"
8617
0
                           " beyond queue size, sqid=%"PRIu32","
8618
0
                           " new_head=%"PRIu16", ignoring",
8619
0
                           qid, new_head);
8620
8621
0
            if (n->outstanding_aers) {
8622
0
                nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
8623
0
                                   NVME_AER_INFO_ERR_INVALID_DB_VALUE,
8624
0
                                   NVME_LOG_ERROR_INFO);
8625
0
            }
8626
8627
0
            return;
8628
0
        }
8629
8630
0
        trace_pci_nvme_mmio_doorbell_cq(cq->cqid, new_head);
8631
8632
        /* scheduled deferred cqe posting if queue was previously full */
8633
0
        if (nvme_cq_full(cq)) {
8634
0
            qemu_bh_schedule(cq->bh);
8635
0
        }
8636
8637
0
        pending = cq->tail != cq->head;
8638
0
        cq->head = new_head;
8639
0
        if (!qid && n->dbbuf_enabled) {
8640
0
            stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED);
8641
0
        }
8642
8643
0
        if (cq->tail == cq->head) {
8644
0
            if (cq->irq_enabled && pending) {
8645
0
                n->cq_pending--;
8646
0
            }
8647
8648
0
            nvme_irq_deassert(n, cq);
8649
0
        } else if (n->dbbuf_enabled) {
8650
0
            nvme_update_cq_eventidx(cq);
8651
0
        }
8652
0
    } else {
8653
        /* Submission queue doorbell write */
8654
8655
0
        uint16_t new_tail = val & 0xffff;
8656
0
        NvmeSQueue *sq;
8657
8658
0
        qid = (addr - 0x1000) >> 3;
8659
0
        if (unlikely(nvme_check_sqid(n, qid))) {
8660
0
            NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq,
8661
0
                           "submission queue doorbell write"
8662
0
                           " for nonexistent queue,"
8663
0
                           " sqid=%"PRIu32", ignoring", qid);
8664
8665
0
            if (n->outstanding_aers) {
8666
0
                nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
8667
0
                                   NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
8668
0
                                   NVME_LOG_ERROR_INFO);
8669
0
            }
8670
8671
0
            return;
8672
0
        }
8673
8674
0
        sq = n->sq[qid];
8675
0
        if (unlikely(new_tail >= sq->size)) {
8676
0
            NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail,
8677
0
                           "submission queue doorbell write value"
8678
0
                           " beyond queue size, sqid=%"PRIu32","
8679
0
                           " new_tail=%"PRIu16", ignoring",
8680
0
                           qid, new_tail);
8681
8682
0
            if (n->outstanding_aers) {
8683
0
                nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
8684
0
                                   NVME_AER_INFO_ERR_INVALID_DB_VALUE,
8685
0
                                   NVME_LOG_ERROR_INFO);
8686
0
            }
8687
8688
0
            return;
8689
0
        }
8690
8691
0
        trace_pci_nvme_mmio_doorbell_sq(sq->sqid, new_tail);
8692
8693
0
        sq->tail = new_tail;
8694
0
        if (!qid && n->dbbuf_enabled) {
8695
            /*
8696
             * The spec states "the host shall also update the controller's
8697
             * corresponding doorbell property to match the value of that entry
8698
             * in the Shadow Doorbell buffer."
8699
             *
8700
             * Since this context is currently a VM trap, we can safely enforce
8701
             * the requirement from the device side in case the host is
8702
             * misbehaving.
8703
             *
8704
             * Note, we shouldn't have to do this, but various drivers
8705
             * including ones that run on Linux, are not updating Admin Queues,
8706
             * so we can't trust reading it for an appropriate sq tail.
8707
             */
8708
0
            stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED);
8709
0
        }
8710
8711
0
        qemu_bh_schedule(sq->bh);
8712
0
    }
8713
0
}
8714
8715
static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
8716
                            unsigned size)
8717
0
{
8718
0
    NvmeCtrl *n = (NvmeCtrl *)opaque;
8719
8720
0
    trace_pci_nvme_mmio_write(addr, data, size);
8721
8722
0
    if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
8723
0
        addr != NVME_REG_CSTS) {
8724
0
        trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
8725
0
        return;
8726
0
    }
8727
8728
0
    if (addr < sizeof(n->bar)) {
8729
0
        nvme_write_bar(n, addr, data, size);
8730
0
    } else {
8731
0
        nvme_process_db(n, addr, data);
8732
0
    }
8733
0
}
8734
8735
static const MemoryRegionOps nvme_mmio_ops = {
8736
    .read = nvme_mmio_read,
8737
    .write = nvme_mmio_write,
8738
    .endianness = DEVICE_LITTLE_ENDIAN,
8739
    .impl = {
8740
        .min_access_size = 2,
8741
        .max_access_size = 8,
8742
    },
8743
};
8744
8745
static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data,
8746
                           unsigned size)
8747
0
{
8748
0
    NvmeCtrl *n = (NvmeCtrl *)opaque;
8749
0
    stn_le_p(&n->cmb.buf[addr], size, data);
8750
0
}
8751
8752
static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size)
8753
0
{
8754
0
    NvmeCtrl *n = (NvmeCtrl *)opaque;
8755
0
    return ldn_le_p(&n->cmb.buf[addr], size);
8756
0
}
8757
8758
static const MemoryRegionOps nvme_cmb_ops = {
8759
    .read = nvme_cmb_read,
8760
    .write = nvme_cmb_write,
8761
    .endianness = DEVICE_LITTLE_ENDIAN,
8762
    .impl = {
8763
        .min_access_size = 1,
8764
        .max_access_size = 8,
8765
    },
8766
};
8767
8768
static bool nvme_check_params(NvmeCtrl *n, Error **errp)
8769
0
{
8770
0
    NvmeParams *params = &n->params;
8771
8772
0
    if (params->num_queues) {
8773
0
        warn_report("num_queues is deprecated; please use max_ioqpairs "
8774
0
                    "instead");
8775
8776
0
        params->max_ioqpairs = params->num_queues - 1;
8777
0
    }
8778
8779
0
    n->num_queues = params->max_ioqpairs + 1;
8780
8781
0
    if (n->namespace.blkconf.blk && n->subsys) {
8782
0
        error_setg(errp, "subsystem support is unavailable with legacy "
8783
0
                   "namespace ('drive' property)");
8784
0
        return false;
8785
0
    }
8786
8787
0
    if (params->max_ioqpairs < 1 ||
8788
0
        params->max_ioqpairs > NVME_MAX_IOQPAIRS) {
8789
0
        error_setg(errp, "max_ioqpairs must be between 1 and %d",
8790
0
                   NVME_MAX_IOQPAIRS);
8791
0
        return false;
8792
0
    }
8793
8794
0
    if (params->msix_qsize < 1 ||
8795
0
        params->msix_qsize > PCI_MSIX_FLAGS_QSIZE + 1) {
8796
0
        error_setg(errp, "msix_qsize must be between 1 and %d",
8797
0
                   PCI_MSIX_FLAGS_QSIZE + 1);
8798
0
        return false;
8799
0
    }
8800
8801
0
    if (!params->serial) {
8802
0
        error_setg(errp, "serial property not set");
8803
0
        return false;
8804
0
    }
8805
0
    if (strlen(params->serial) > NVME_ID_CTRL_SN_MAX_LEN) {
8806
0
        error_setg(errp, "'serial' parameter '%s' can be at most '%d' characters",
8807
0
                   params->serial, NVME_ID_CTRL_SN_MAX_LEN);
8808
0
        return false;
8809
0
    }
8810
8811
0
    if (params->model &&
8812
0
        strlen(params->model) > NVME_ID_CTRL_MN_MAX_LEN) {
8813
0
        error_setg(errp, "'model' parameter '%s' can be at most '%d' characters",
8814
0
                   params->model, NVME_ID_CTRL_MN_MAX_LEN);
8815
0
        return false;
8816
0
    }
8817
8818
0
    if (params->firmware_version &&
8819
0
        strlen(params->firmware_version) > NVME_ID_CTRL_FR_MAX_LEN) {
8820
0
        error_setg(errp, "'firmware-version' parameter '%s' can be at most '%d' characters",
8821
0
                   params->firmware_version, NVME_ID_CTRL_FR_MAX_LEN);
8822
0
        return false;
8823
0
    }
8824
8825
0
    if (params->mqes < 1) {
8826
0
        error_setg(errp, "mqes property cannot be less than 1");
8827
0
        return false;
8828
0
    }
8829
8830
0
    if (n->pmr.dev) {
8831
0
        if (params->msix_exclusive_bar) {
8832
0
            error_setg(errp, "not enough BARs available to enable PMR");
8833
0
            return false;
8834
0
        }
8835
8836
0
        if (host_memory_backend_is_mapped(n->pmr.dev)) {
8837
0
            error_setg(errp, "can't use already busy memdev: %s",
8838
0
                       object_get_canonical_path_component(OBJECT(n->pmr.dev)));
8839
0
            return false;
8840
0
        }
8841
8842
0
        if (!is_power_of_2(n->pmr.dev->size)) {
8843
0
            error_setg(errp, "pmr backend size needs to be power of 2 in size");
8844
0
            return false;
8845
0
        }
8846
8847
0
        host_memory_backend_set_mapped(n->pmr.dev, true);
8848
0
    }
8849
8850
0
    if (!n->params.mdts || ((1 << n->params.mdts) + 1) > IOV_MAX) {
8851
0
        error_setg(errp, "mdts exceeds IOV_MAX");
8852
0
        return false;
8853
0
    }
8854
8855
0
    if (n->params.zasl > n->params.mdts) {
8856
0
        error_setg(errp, "zoned.zasl (Zone Append Size Limit) must be less "
8857
0
                   "than or equal to mdts (Maximum Data Transfer Size)");
8858
0
        return false;
8859
0
    }
8860
8861
0
    if (!n->params.vsl) {
8862
0
        error_setg(errp, "vsl must be non-zero");
8863
0
        return false;
8864
0
    }
8865
8866
0
    if (params->sriov_max_vfs) {
8867
0
        if (!n->subsys) {
8868
0
            error_setg(errp, "subsystem is required for the use of SR-IOV");
8869
0
            return false;
8870
0
        }
8871
8872
0
        if (params->cmb_size_mb) {
8873
0
            error_setg(errp, "CMB is not supported with SR-IOV");
8874
0
            return false;
8875
0
        }
8876
8877
0
        if (n->pmr.dev) {
8878
0
            error_setg(errp, "PMR is not supported with SR-IOV");
8879
0
            return false;
8880
0
        }
8881
8882
0
        if (!params->sriov_vq_flexible || !params->sriov_vi_flexible) {
8883
0
            error_setg(errp, "both sriov_vq_flexible and sriov_vi_flexible"
8884
0
                       " must be set for the use of SR-IOV");
8885
0
            return false;
8886
0
        }
8887
8888
0
        if (params->sriov_vq_flexible < params->sriov_max_vfs * 2) {
8889
0
            error_setg(errp, "sriov_vq_flexible must be greater than or equal"
8890
0
                       " to %d (sriov_max_vfs * 2)", params->sriov_max_vfs * 2);
8891
0
            return false;
8892
0
        }
8893
8894
0
        if (params->max_ioqpairs < params->sriov_vq_flexible + 2) {
8895
0
            error_setg(errp, "(max_ioqpairs - sriov_vq_flexible) must be"
8896
0
                       " greater than or equal to 2");
8897
0
            return false;
8898
0
        }
8899
8900
0
        if (params->sriov_vi_flexible < params->sriov_max_vfs) {
8901
0
            error_setg(errp, "sriov_vi_flexible must be greater than or equal"
8902
0
                       " to %d (sriov_max_vfs)", params->sriov_max_vfs);
8903
0
            return false;
8904
0
        }
8905
8906
0
        if (params->msix_qsize < params->sriov_vi_flexible + 1) {
8907
0
            error_setg(errp, "(msix_qsize - sriov_vi_flexible) must be"
8908
0
                       " greater than or equal to 1");
8909
0
            return false;
8910
0
        }
8911
8912
0
        if (params->sriov_max_vi_per_vf &&
8913
0
            (params->sriov_max_vi_per_vf - 1) % NVME_VF_RES_GRANULARITY) {
8914
0
            error_setg(errp, "sriov_max_vi_per_vf must meet:"
8915
0
                       " (sriov_max_vi_per_vf - 1) %% %d == 0 and"
8916
0
                       " sriov_max_vi_per_vf >= 1", NVME_VF_RES_GRANULARITY);
8917
0
            return false;
8918
0
        }
8919
8920
0
        if (params->sriov_max_vq_per_vf &&
8921
0
            (params->sriov_max_vq_per_vf < 2 ||
8922
0
             (params->sriov_max_vq_per_vf - 1) % NVME_VF_RES_GRANULARITY)) {
8923
0
            error_setg(errp, "sriov_max_vq_per_vf must meet:"
8924
0
                       " (sriov_max_vq_per_vf - 1) %% %d == 0 and"
8925
0
                       " sriov_max_vq_per_vf >= 2", NVME_VF_RES_GRANULARITY);
8926
0
            return false;
8927
0
        }
8928
0
    }
8929
8930
0
    return true;
8931
0
}
8932
8933
static void nvme_init_state(NvmeCtrl *n)
8934
0
{
8935
0
    NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
8936
0
    NvmeSecCtrlEntry *list = n->sec_ctrl_list;
8937
0
    NvmeSecCtrlEntry *sctrl;
8938
0
    PCIDevice *pci = PCI_DEVICE(n);
8939
0
    NvmeIdCtrl *id = &n->id_ctrl;
8940
0
    uint8_t max_vfs;
8941
0
    int i;
8942
8943
0
    if (pci_is_vf(pci)) {
8944
0
        sctrl = nvme_sctrl(n);
8945
0
        max_vfs = 0;
8946
0
        n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
8947
0
        n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
8948
0
    } else {
8949
0
        max_vfs = n->params.sriov_max_vfs;
8950
0
        n->conf_ioqpairs = n->params.max_ioqpairs;
8951
0
        n->conf_msix_qsize = n->params.msix_qsize;
8952
0
    }
8953
8954
0
    n->sq = g_new0(NvmeSQueue *, n->num_queues);
8955
0
    n->cq = g_new0(NvmeCQueue *, n->num_queues);
8956
0
    n->temperature = NVME_TEMPERATURE;
8957
0
    n->features.temp_thresh_hi = NVME_TEMPERATURE_WARNING;
8958
0
    n->starttime_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
8959
0
    n->aer_reqs = g_new0(NvmeRequest *, n->params.aerl + 1);
8960
0
    QTAILQ_INIT(&n->aer_queue);
8961
8962
0
    n->nr_sec_ctrls = max_vfs;
8963
0
    for (i = 0; i < max_vfs; i++) {
8964
0
        sctrl = &list[i];
8965
0
        sctrl->pcid = cpu_to_le16(n->cntlid);
8966
0
        sctrl->vfn = cpu_to_le16(i + 1);
8967
0
    }
8968
8969
0
    n->spdm_socket = -1;
8970
8971
0
    cap->cntlid = cpu_to_le16(n->cntlid);
8972
0
    cap->crt = NVME_CRT_VQ | NVME_CRT_VI;
8973
8974
0
    if (pci_is_vf(pci)) {
8975
0
        cap->vqprt = cpu_to_le16(1 + n->conf_ioqpairs);
8976
0
    } else {
8977
0
        cap->vqprt = cpu_to_le16(1 + n->params.max_ioqpairs -
8978
0
                                 n->params.sriov_vq_flexible);
8979
0
        cap->vqfrt = cpu_to_le32(n->params.sriov_vq_flexible);
8980
0
        cap->vqrfap = cap->vqfrt;
8981
0
        cap->vqgran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
8982
0
        cap->vqfrsm = n->params.sriov_max_vq_per_vf ?
8983
0
                        cpu_to_le16(n->params.sriov_max_vq_per_vf) :
8984
0
                        cap->vqfrt / MAX(max_vfs, 1);
8985
0
    }
8986
8987
0
    if (pci_is_vf(pci)) {
8988
0
        cap->viprt = cpu_to_le16(n->conf_msix_qsize);
8989
0
    } else {
8990
0
        cap->viprt = cpu_to_le16(n->params.msix_qsize -
8991
0
                                 n->params.sriov_vi_flexible);
8992
0
        cap->vifrt = cpu_to_le32(n->params.sriov_vi_flexible);
8993
0
        cap->virfap = cap->vifrt;
8994
0
        cap->vigran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
8995
0
        cap->vifrsm = n->params.sriov_max_vi_per_vf ?
8996
0
                        cpu_to_le16(n->params.sriov_max_vi_per_vf) :
8997
0
                        cap->vifrt / MAX(max_vfs, 1);
8998
0
    }
8999
9000
    /* Atomic Write */
9001
0
    id->awun = cpu_to_le16(n->params.atomic_awun);
9002
0
    id->awupf = cpu_to_le16(n->params.atomic_awupf);
9003
0
    n->dn = n->params.atomic_dn;
9004
9005
0
    if (id->awun || id->awupf) {
9006
0
        if (id->awupf > id->awun) {
9007
0
            id->awupf = 0;
9008
0
        }
9009
9010
0
        nvme_atomic_configure_max_write_size(n->dn, n->params.atomic_awun,
9011
0
                                             n->params.atomic_awupf,
9012
0
                                             &n->atomic);
9013
0
    }
9014
0
}
9015
9016
static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
9017
0
{
9018
0
    uint64_t cmb_size = n->params.cmb_size_mb * MiB;
9019
0
    uint64_t cap = ldq_le_p(&n->bar.cap);
9020
9021
0
    n->cmb.buf = g_malloc0(cmb_size);
9022
0
    memory_region_init_io(&n->cmb.mem, OBJECT(n), &nvme_cmb_ops, n,
9023
0
                          "nvme-cmb", cmb_size);
9024
0
    pci_register_bar(pci_dev, NVME_CMB_BIR,
9025
0
                     PCI_BASE_ADDRESS_SPACE_MEMORY |
9026
0
                     PCI_BASE_ADDRESS_MEM_TYPE_64 |
9027
0
                     PCI_BASE_ADDRESS_MEM_PREFETCH, &n->cmb.mem);
9028
9029
0
    NVME_CAP_SET_CMBS(cap, 1);
9030
0
    stq_le_p(&n->bar.cap, cap);
9031
9032
0
    if (n->params.legacy_cmb) {
9033
0
        nvme_cmb_enable_regs(n);
9034
0
        n->cmb.cmse = true;
9035
0
    }
9036
0
}
9037
9038
static bool nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
9039
0
{
9040
0
    uint32_t pmrcap = ldl_le_p(&n->bar.pmrcap);
9041
9042
0
    if (memory_region_size(&n->pmr.dev->mr) < 16) {
9043
0
        error_setg(errp, "PMR device must have at least 16 bytes");
9044
0
        return false;
9045
0
    }
9046
9047
0
    NVME_PMRCAP_SET_RDS(pmrcap, 1);
9048
0
    NVME_PMRCAP_SET_WDS(pmrcap, 1);
9049
0
    NVME_PMRCAP_SET_BIR(pmrcap, NVME_PMR_BIR);
9050
    /* Turn on bit 1 support */
9051
0
    NVME_PMRCAP_SET_PMRWBM(pmrcap, 0x02);
9052
0
    NVME_PMRCAP_SET_CMSS(pmrcap, 1);
9053
0
    stl_le_p(&n->bar.pmrcap, pmrcap);
9054
9055
0
    pci_register_bar(pci_dev, NVME_PMR_BIR,
9056
0
                     PCI_BASE_ADDRESS_SPACE_MEMORY |
9057
0
                     PCI_BASE_ADDRESS_MEM_TYPE_64 |
9058
0
                     PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmr.dev->mr);
9059
9060
0
    memory_region_set_enabled(&n->pmr.dev->mr, false);
9061
9062
0
    return true;
9063
0
}
9064
9065
static uint64_t nvme_mbar_size(unsigned total_queues, unsigned total_irqs,
9066
                               unsigned *msix_table_offset,
9067
                               unsigned *msix_pba_offset)
9068
0
{
9069
0
    uint64_t bar_size, msix_table_size;
9070
9071
0
    bar_size = sizeof(NvmeBar) + 2 * total_queues * NVME_DB_SIZE;
9072
9073
0
    if (total_irqs == 0) {
9074
0
        goto out;
9075
0
    }
9076
9077
0
    bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
9078
9079
0
    if (msix_table_offset) {
9080
0
        *msix_table_offset = bar_size;
9081
0
    }
9082
9083
0
    msix_table_size = PCI_MSIX_ENTRY_SIZE * total_irqs;
9084
0
    bar_size += msix_table_size;
9085
0
    bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
9086
9087
0
    if (msix_pba_offset) {
9088
0
        *msix_pba_offset = bar_size;
9089
0
    }
9090
9091
0
    bar_size += QEMU_ALIGN_UP(total_irqs, 64) / 8;
9092
9093
0
out:
9094
0
    return pow2ceil(bar_size);
9095
0
}
9096
9097
static bool nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset,
9098
                            Error **errp)
9099
0
{
9100
0
    uint16_t vf_dev_id = n->params.use_intel_id ?
9101
0
                         PCI_DEVICE_ID_INTEL_NVME : PCI_DEVICE_ID_REDHAT_NVME;
9102
0
    NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
9103
0
    uint64_t bar_size = nvme_mbar_size(le16_to_cpu(cap->vqfrsm),
9104
0
                                      le16_to_cpu(cap->vifrsm),
9105
0
                                      NULL, NULL);
9106
9107
0
    if (!pcie_sriov_pf_init(pci_dev, offset, "nvme", vf_dev_id,
9108
0
                            n->params.sriov_max_vfs, n->params.sriov_max_vfs,
9109
0
                            NVME_VF_OFFSET, NVME_VF_STRIDE, errp)) {
9110
0
        return false;
9111
0
    }
9112
9113
0
    pcie_sriov_pf_init_vf_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
9114
0
                              PCI_BASE_ADDRESS_MEM_TYPE_64, bar_size);
9115
9116
0
    return true;
9117
0
}
9118
9119
static int nvme_add_pm_capability(PCIDevice *pci_dev, uint8_t offset)
9120
0
{
9121
0
    Error *err = NULL;
9122
0
    int ret;
9123
9124
0
    ret = pci_pm_init(pci_dev, offset, &err);
9125
0
    if (err) {
9126
0
        error_report_err(err);
9127
0
        return ret;
9128
0
    }
9129
9130
0
    pci_set_word(pci_dev->config + offset + PCI_PM_PMC,
9131
0
                 PCI_PM_CAP_VER_1_2);
9132
0
    pci_set_word(pci_dev->config + offset + PCI_PM_CTRL,
9133
0
                 PCI_PM_CTRL_NO_SOFT_RESET);
9134
0
    pci_set_word(pci_dev->wmask + offset + PCI_PM_CTRL,
9135
0
                 PCI_PM_CTRL_STATE_MASK);
9136
9137
0
    return 0;
9138
0
}
9139
9140
static bool pcie_doe_spdm_rsp(DOECap *doe_cap)
9141
0
{
9142
0
    void *req = pcie_doe_get_write_mbox_ptr(doe_cap);
9143
0
    uint32_t req_len = pcie_doe_get_obj_len(req) * 4;
9144
0
    void *rsp = doe_cap->read_mbox;
9145
0
    uint32_t rsp_len = SPDM_SOCKET_MAX_MESSAGE_BUFFER_SIZE;
9146
9147
0
    uint32_t recvd = spdm_socket_rsp(doe_cap->spdm_socket,
9148
0
                             SPDM_TRANSPORT_TYPE_DOE,
9149
0
                             req, req_len, rsp, rsp_len);
9150
0
    doe_cap->read_mbox_len += DIV_ROUND_UP(recvd, 4);
9151
9152
0
    return recvd != 0;
9153
0
}
9154
9155
static DOEProtocol doe_spdm_prot[] = {
9156
    { PCI_VENDOR_ID_PCI_SIG, PCI_SIG_DOE_CMA, pcie_doe_spdm_rsp },
9157
    { PCI_VENDOR_ID_PCI_SIG, PCI_SIG_DOE_SECURED_CMA, pcie_doe_spdm_rsp },
9158
    { }
9159
};
9160
9161
static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
9162
0
{
9163
0
    ERRP_GUARD();
9164
0
    uint8_t *pci_conf = pci_dev->config;
9165
0
    uint64_t bar_size;
9166
0
    unsigned msix_table_offset = 0, msix_pba_offset = 0;
9167
0
    unsigned nr_vectors;
9168
0
    int ret;
9169
9170
0
    pci_conf[PCI_INTERRUPT_PIN] = pci_is_vf(pci_dev) ? 0 : 1;
9171
0
    pci_config_set_prog_interface(pci_conf, 0x2);
9172
9173
0
    if (n->params.use_intel_id) {
9174
0
        pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL);
9175
0
        pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_INTEL_NVME);
9176
0
    } else {
9177
0
        pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_REDHAT);
9178
0
        pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_REDHAT_NVME);
9179
0
    }
9180
9181
0
    pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
9182
0
    nvme_add_pm_capability(pci_dev, 0x60);
9183
0
    pcie_endpoint_cap_init(pci_dev, 0x80);
9184
0
    pcie_cap_flr_init(pci_dev);
9185
0
    if (n->params.sriov_max_vfs) {
9186
0
        pcie_ari_init(pci_dev, 0x100);
9187
0
    }
9188
9189
0
    if (n->params.msix_exclusive_bar && !pci_is_vf(pci_dev)) {
9190
0
        bar_size = nvme_mbar_size(n->num_queues, 0, NULL, NULL);
9191
0
        memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
9192
0
                              bar_size);
9193
0
        pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
9194
0
                         PCI_BASE_ADDRESS_MEM_TYPE_64, &n->iomem);
9195
0
        ret = msix_init_exclusive_bar(pci_dev, n->params.msix_qsize, 4, errp);
9196
0
    } else {
9197
0
        assert(n->params.msix_qsize >= 1);
9198
9199
        /* add one to max_ioqpairs to account for the admin queue pair */
9200
0
        if (!pci_is_vf(pci_dev)) {
9201
0
            nr_vectors = n->params.msix_qsize;
9202
0
            bar_size = nvme_mbar_size(n->num_queues,
9203
0
                                      nr_vectors, &msix_table_offset,
9204
0
                                      &msix_pba_offset);
9205
0
        } else {
9206
0
            NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));
9207
0
            NvmePriCtrlCap *cap = &pn->pri_ctrl_cap;
9208
9209
0
            nr_vectors = le16_to_cpu(cap->vifrsm);
9210
0
            bar_size = nvme_mbar_size(le16_to_cpu(cap->vqfrsm), nr_vectors,
9211
0
                                      &msix_table_offset, &msix_pba_offset);
9212
0
        }
9213
9214
0
        memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
9215
0
        memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
9216
0
                              msix_table_offset);
9217
0
        memory_region_add_subregion(&n->bar0, 0, &n->iomem);
9218
9219
0
        pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
9220
0
                         PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
9221
9222
0
        ret = msix_init(pci_dev, nr_vectors,
9223
0
                        &n->bar0, 0, msix_table_offset,
9224
0
                        &n->bar0, 0, msix_pba_offset, 0, errp);
9225
0
    }
9226
9227
0
    if (ret == -ENOTSUP) {
9228
        /* report that msix is not supported, but do not error out */
9229
0
        warn_report_err(*errp);
9230
0
        *errp = NULL;
9231
0
    } else if (ret < 0) {
9232
        /* propagate error to caller */
9233
0
        return false;
9234
0
    }
9235
9236
0
    if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs &&
9237
0
        !nvme_init_sriov(n, pci_dev, 0x120, errp)) {
9238
0
        return false;
9239
0
    }
9240
9241
0
    nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
9242
9243
0
    pcie_cap_deverr_init(pci_dev);
9244
9245
    /* SPDM Initialisation */
9246
0
    if (pci_dev->spdm_port) {
9247
0
        uint16_t doe_offset = PCI_CONFIG_SPACE_SIZE;
9248
9249
0
        switch  (pci_dev->spdm_trans) {
9250
0
        case SPDM_TRANSPORT_TYPE_DOE:
9251
0
            if (n->params.sriov_max_vfs) {
9252
0
                doe_offset += PCI_ARI_SIZEOF;
9253
0
            }
9254
9255
0
            pcie_doe_init(pci_dev, &pci_dev->doe_spdm, doe_offset,
9256
0
                          doe_spdm_prot, true, 0);
9257
9258
0
            pci_dev->doe_spdm.spdm_socket =
9259
0
                spdm_socket_connect(pci_dev->spdm_port, errp);
9260
9261
0
            if (pci_dev->doe_spdm.spdm_socket < 0) {
9262
0
                return false;
9263
0
            }
9264
0
            break;
9265
0
        case SPDM_TRANSPORT_TYPE_NVME:
9266
0
            n->spdm_socket = spdm_socket_connect(pci_dev->spdm_port, errp);
9267
0
            if (n->spdm_socket < 0) {
9268
0
                return false;
9269
0
            }
9270
0
            break;
9271
0
        default:
9272
0
            return false;
9273
0
        }
9274
0
    }
9275
9276
0
    if (n->params.cmb_size_mb) {
9277
0
        nvme_init_cmb(n, pci_dev);
9278
0
    }
9279
9280
0
    if (n->pmr.dev) {
9281
0
        if (!nvme_init_pmr(n, pci_dev, errp)) {
9282
0
            return false;
9283
0
        }
9284
0
    }
9285
9286
0
    return true;
9287
0
}
9288
9289
static void nvme_init_subnqn(NvmeCtrl *n)
9290
0
{
9291
0
    NvmeSubsystem *subsys = n->subsys;
9292
0
    NvmeIdCtrl *id = &n->id_ctrl;
9293
9294
0
    if (!subsys) {
9295
0
        snprintf((char *)id->subnqn, sizeof(id->subnqn),
9296
0
                 "nqn.2019-08.org.qemu:%s", n->params.serial);
9297
0
    } else {
9298
0
        pstrcpy((char *)id->subnqn, sizeof(id->subnqn), (char*)subsys->subnqn);
9299
0
    }
9300
0
}
9301
9302
static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
9303
0
{
9304
0
    NvmeIdCtrl *id = &n->id_ctrl;
9305
0
    uint8_t *pci_conf = pci_dev->config;
9306
0
    uint64_t cap = ldq_le_p(&n->bar.cap);
9307
0
    NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
9308
0
    uint32_t ctratt = le32_to_cpu(id->ctratt);
9309
0
    uint16_t oacs;
9310
9311
0
    memcpy(n->cse.acs, nvme_cse_acs_default, sizeof(n->cse.acs));
9312
0
    memcpy(n->cse.iocs.nvm, nvme_cse_iocs_nvm_default, sizeof(n->cse.iocs.nvm));
9313
0
    memcpy(n->cse.iocs.zoned, nvme_cse_iocs_zoned_default,
9314
0
           sizeof(n->cse.iocs.zoned));
9315
9316
0
    id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
9317
0
    id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
9318
0
    strpadcpy((char *)id->mn, sizeof(id->mn),
9319
0
              n->params.model ? n->params.model : "QEMU NVMe Ctrl", ' ');
9320
0
    strpadcpy((char *)id->fr, sizeof(id->fr),
9321
0
              n->params.firmware_version ? n->params.firmware_version : QEMU_VERSION, ' ');
9322
0
    strpadcpy((char *)id->sn, sizeof(id->sn), n->params.serial, ' ');
9323
9324
0
    id->cntlid = cpu_to_le16(n->cntlid);
9325
9326
0
    id->oaes = cpu_to_le32(NVME_OAES_NS_ATTR);
9327
9328
0
    ctratt |= NVME_CTRATT_ELBAS;
9329
0
    if (n->params.ctratt.mem) {
9330
0
        ctratt |= NVME_CTRATT_MEM;
9331
0
    }
9332
0
    id->ctratt = cpu_to_le32(ctratt);
9333
9334
0
    id->rab = 6;
9335
9336
0
    if (n->params.use_intel_id) {
9337
0
        id->ieee[0] = 0xb3;
9338
0
        id->ieee[1] = 0x02;
9339
0
        id->ieee[2] = 0x00;
9340
0
    } else {
9341
0
        id->ieee[0] = 0x00;
9342
0
        id->ieee[1] = 0x54;
9343
0
        id->ieee[2] = 0x52;
9344
0
    }
9345
9346
0
    id->mdts = n->params.mdts;
9347
0
    id->ver = cpu_to_le32(NVME_SPEC_VER);
9348
9349
0
    oacs = NVME_OACS_NMS | NVME_OACS_FORMAT | NVME_OACS_DIRECTIVES |
9350
0
           NVME_OACS_SECURITY;
9351
9352
0
    if (n->params.dbcs) {
9353
0
        oacs |= NVME_OACS_DBCS;
9354
9355
0
        n->cse.acs[NVME_ADM_CMD_DBBUF_CONFIG] = NVME_CMD_EFF_CSUPP;
9356
0
    }
9357
9358
0
    if (n->params.sriov_max_vfs) {
9359
0
        oacs |= NVME_OACS_VMS;
9360
9361
0
        n->cse.acs[NVME_ADM_CMD_VIRT_MNGMT] = NVME_CMD_EFF_CSUPP;
9362
0
    }
9363
9364
0
    id->oacs = cpu_to_le16(oacs);
9365
9366
0
    id->cntrltype = 0x1;
9367
9368
    /*
9369
     * Because the controller always completes the Abort command immediately,
9370
     * there can never be more than one concurrently executing Abort command,
9371
     * so this value is never used for anything. Note that there can easily be
9372
     * many Abort commands in the queues, but they are not considered
9373
     * "executing" until processed by nvme_abort.
9374
     *
9375
     * The specification recommends a value of 3 for Abort Command Limit (four
9376
     * concurrently outstanding Abort commands), so lets use that though it is
9377
     * inconsequential.
9378
     */
9379
0
    id->acl = 3;
9380
0
    id->aerl = n->params.aerl;
9381
0
    id->frmw = (NVME_NUM_FW_SLOTS << 1) | NVME_FRMW_SLOT1_RO;
9382
0
    id->lpa = NVME_LPA_NS_SMART | NVME_LPA_CSE | NVME_LPA_EXTENDED;
9383
9384
    /* recommended default value (~70 C) */
9385
0
    id->wctemp = cpu_to_le16(NVME_TEMPERATURE_WARNING);
9386
0
    id->cctemp = cpu_to_le16(NVME_TEMPERATURE_CRITICAL);
9387
9388
0
    id->sqes = (NVME_SQES << 4) | NVME_SQES;
9389
0
    id->cqes = (NVME_CQES << 4) | NVME_CQES;
9390
0
    id->nn = cpu_to_le32(NVME_MAX_NAMESPACES);
9391
0
    id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROES | NVME_ONCS_TIMESTAMP |
9392
0
                           NVME_ONCS_FEATURES | NVME_ONCS_DSM |
9393
0
                           NVME_ONCS_COMPARE | NVME_ONCS_COPY |
9394
0
                           NVME_ONCS_NVMCSA | NVME_ONCS_NVMAFC);
9395
9396
    /*
9397
     * NOTE: If this device ever supports a command set that does NOT use 0x0
9398
     * as a Flush-equivalent operation, support for the broadcast NSID in Flush
9399
     * should probably be removed.
9400
     *
9401
     * See comment in nvme_io_cmd.
9402
     */
9403
0
    id->vwc = NVME_VWC_NSID_BROADCAST_SUPPORT | NVME_VWC_PRESENT;
9404
9405
0
    id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1 |
9406
0
                            NVME_OCFS_COPY_FORMAT_2 | NVME_OCFS_COPY_FORMAT_3);
9407
0
    id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN |
9408
0
                           NVME_CTRL_SGLS_MPTR_SGL);
9409
9410
0
    nvme_init_subnqn(n);
9411
9412
0
    id->psd[0].mp = cpu_to_le16(0x9c4);
9413
0
    id->psd[0].enlat = cpu_to_le32(0x10);
9414
0
    id->psd[0].exlat = cpu_to_le32(0x4);
9415
9416
0
    NVME_CAP_SET_MQES(cap, n->params.mqes);
9417
0
    NVME_CAP_SET_CQR(cap, 1);
9418
0
    NVME_CAP_SET_TO(cap, 0xf);
9419
0
    NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_NCSS);
9420
0
    NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_IOCSS);
9421
0
    NVME_CAP_SET_MPSMAX(cap, 4);
9422
0
    NVME_CAP_SET_CMBS(cap, n->params.cmb_size_mb ? 1 : 0);
9423
0
    NVME_CAP_SET_PMRS(cap, n->pmr.dev ? 1 : 0);
9424
0
    stq_le_p(&n->bar.cap, cap);
9425
9426
0
    stl_le_p(&n->bar.vs, NVME_SPEC_VER);
9427
0
    n->bar.intmc = n->bar.intms = 0;
9428
9429
0
    if (pci_is_vf(pci_dev) && !sctrl->scs) {
9430
0
        stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
9431
0
    }
9432
0
}
9433
9434
static bool nvme_set_migration_blockers(NvmeCtrl *n, PCIDevice *pci_dev,
9435
                                        Error **errp)
9436
0
{
9437
0
    uint64_t unsupported_cap, cap = ldq_le_p(&n->bar.cap);
9438
0
    g_autoptr(GPtrArray) blocker_features = g_ptr_array_new();
9439
0
    bool adm_cmd_security_checked = false;
9440
0
    bool cmd_io_mgmt_checked = false;
9441
0
    bool cmd_zone_checked = false;
9442
9443
    /*
9444
     * Idea of this function is simple, we iterate over all Command Sets and
9445
     * for each supported command we provide a special handling logic to
9446
     * determine if we should block migration or not.
9447
     *
9448
     * For instance, we have NVME_ADM_CMD_NS_ATTACHMENT and it is always
9449
     * available to the guest, but if there is only 1 namespace, then it is
9450
     * safe to allow migration, but if there are more, then we need to block
9451
     * migration because we don't handle this in migration code yet.
9452
     */
9453
0
    for (int opcode = 0; opcode < ARRAY_SIZE(n->cse.acs); opcode++) {
9454
        /* Is command supported? */
9455
0
        if (!n->cse.acs[opcode]) {
9456
0
            continue;
9457
0
        }
9458
9459
0
        switch (opcode) {
9460
0
        case NVME_ADM_CMD_DELETE_SQ:
9461
0
        case NVME_ADM_CMD_CREATE_SQ:
9462
0
        case NVME_ADM_CMD_GET_LOG_PAGE:
9463
0
        case NVME_ADM_CMD_DELETE_CQ:
9464
0
        case NVME_ADM_CMD_CREATE_CQ:
9465
0
        case NVME_ADM_CMD_IDENTIFY:
9466
0
        case NVME_ADM_CMD_ABORT:
9467
0
        case NVME_ADM_CMD_SET_FEATURES:
9468
0
        case NVME_ADM_CMD_GET_FEATURES:
9469
0
        case NVME_ADM_CMD_ASYNC_EV_REQ:
9470
0
        case NVME_ADM_CMD_DBBUF_CONFIG:
9471
0
        case NVME_ADM_CMD_FORMAT_NVM:
9472
0
        case NVME_ADM_CMD_DIRECTIVE_SEND:
9473
0
        case NVME_ADM_CMD_DIRECTIVE_RECV:
9474
0
            break;
9475
0
        case NVME_ADM_CMD_NS_ATTACHMENT: {
9476
0
            int namespaces_num = 0;
9477
0
            for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) {
9478
0
                NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
9479
0
                if (!ns) {
9480
0
                    continue;
9481
0
                }
9482
9483
0
                namespaces_num++;
9484
0
            }
9485
9486
0
            if (namespaces_num > 1) {
9487
0
                g_ptr_array_add(blocker_features,
9488
0
                                (void *) "Namespace Attachment");
9489
0
            }
9490
9491
0
            break;
9492
0
        }
9493
0
        case NVME_ADM_CMD_VIRT_MNGMT:
9494
0
            if (n->params.sriov_max_vfs) {
9495
0
                g_ptr_array_add(blocker_features, (void *) "SR-IOV");
9496
0
            }
9497
9498
0
            break;
9499
0
        case NVME_ADM_CMD_SECURITY_SEND:
9500
0
        case NVME_ADM_CMD_SECURITY_RECV:
9501
0
            if (adm_cmd_security_checked) {
9502
0
                break;
9503
0
            }
9504
9505
0
            if (pci_dev->spdm_port) {
9506
0
                g_ptr_array_add(blocker_features, (void *) "SPDM");
9507
0
            }
9508
9509
0
            adm_cmd_security_checked = true;
9510
9511
0
            break;
9512
0
        default:
9513
0
            g_assert_not_reached();
9514
0
        }
9515
0
    }
9516
9517
0
    for (int opcode = 0; opcode < ARRAY_SIZE(n->cse.iocs.nvm); opcode++) {
9518
0
        if (!n->cse.iocs.nvm[opcode]) {
9519
0
            continue;
9520
0
        }
9521
9522
0
        switch (opcode) {
9523
0
        case NVME_CMD_FLUSH:
9524
0
        case NVME_CMD_WRITE:
9525
0
        case NVME_CMD_READ:
9526
0
        case NVME_CMD_COMPARE:
9527
0
        case NVME_CMD_WRITE_ZEROES:
9528
0
        case NVME_CMD_DSM:
9529
0
        case NVME_CMD_VERIFY:
9530
0
        case NVME_CMD_COPY:
9531
0
            break;
9532
0
        case NVME_CMD_IO_MGMT_RECV:
9533
0
        case NVME_CMD_IO_MGMT_SEND:
9534
0
            if (cmd_io_mgmt_checked) {
9535
0
                break;
9536
0
            }
9537
9538
            /* check for NVME_IOMS_MO_RUH_UPDATE */
9539
0
            if (n->subsys->params.fdp.enabled) {
9540
0
                g_ptr_array_add(blocker_features, (void *) "FDP");
9541
0
            }
9542
9543
0
            cmd_io_mgmt_checked = true;
9544
9545
0
            break;
9546
0
        default:
9547
0
            g_assert_not_reached();
9548
0
        }
9549
0
    }
9550
9551
0
    for (int opcode = 0; opcode < ARRAY_SIZE(n->cse.iocs.zoned); opcode++) {
9552
        /*
9553
         * If command isn't supported or we have the same command
9554
         * in n->cse.iocs.nvm, then we can skip it here.
9555
         */
9556
0
        if (!n->cse.iocs.zoned[opcode] || n->cse.iocs.nvm[opcode]) {
9557
0
            continue;
9558
0
        }
9559
9560
0
        switch (opcode) {
9561
0
        case NVME_CMD_ZONE_APPEND:
9562
0
        case NVME_CMD_ZONE_MGMT_SEND:
9563
0
        case NVME_CMD_ZONE_MGMT_RECV:
9564
0
            if (cmd_zone_checked) {
9565
0
                break;
9566
0
            }
9567
9568
0
            for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) {
9569
0
                NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
9570
0
                if (!ns) {
9571
0
                    continue;
9572
0
                }
9573
9574
0
                if (ns->params.zoned) {
9575
0
                    g_ptr_array_add(blocker_features,
9576
0
                                    (void *) "Zoned Namespace");
9577
0
                    break;
9578
0
                }
9579
0
            }
9580
9581
0
            cmd_zone_checked = true;
9582
9583
0
            break;
9584
0
        default:
9585
0
            g_assert_not_reached();
9586
0
        }
9587
0
    }
9588
9589
    /*
9590
     * Try our best to explicitly detect all not supported caps,
9591
     * to let users know what features cause migration to be blocked,
9592
     * but in case we miss handling here, everything else will be
9593
     * covered by unsupported_cap check.
9594
     */
9595
0
    if (NVME_CAP_CMBS(cap)) {
9596
0
        g_ptr_array_add(blocker_features, (void *) "CMB");
9597
0
        cap &= ~((uint64_t)CAP_CMBS_MASK << CAP_CMBS_SHIFT);
9598
0
    }
9599
9600
0
    if (NVME_CAP_PMRS(cap)) {
9601
0
        g_ptr_array_add(blocker_features, (void *) "PMR");
9602
0
        cap &= ~((uint64_t)CAP_PMRS_MASK << CAP_PMRS_SHIFT);
9603
0
    }
9604
9605
0
    unsupported_cap = cap & ~NVME_MIGRATION_SUPPORTED_CAP_BITS;
9606
0
    if (unsupported_cap) {
9607
0
        g_ptr_array_add(blocker_features, (void *) "unknown capability");
9608
0
    }
9609
9610
0
    assert(n->migration_blocker == NULL);
9611
0
    if (blocker_features->len > 0) {
9612
0
        g_autofree char *blocker_list = NULL;
9613
9614
0
        g_ptr_array_add(blocker_features, NULL);
9615
0
        blocker_list = g_strjoinv(", ", (void *)blocker_features->pdata);
9616
0
        error_setg(&n->migration_blocker,
9617
0
                   "Migration is not supported for %s", blocker_list);
9618
0
        if (migrate_add_blocker(&n->migration_blocker, errp) < 0) {
9619
0
            return false;
9620
0
        }
9621
0
    }
9622
9623
0
    return true;
9624
0
}
9625
9626
static int nvme_init_subsys(NvmeCtrl *n, Error **errp)
9627
0
{
9628
0
    int cntlid;
9629
9630
0
    if (!n->subsys) {
9631
0
        DeviceState *dev = qdev_new(TYPE_NVME_SUBSYS);
9632
9633
0
        qdev_prop_set_string(dev, "nqn", n->params.serial);
9634
9635
0
        if (!qdev_realize(dev, NULL, errp)) {
9636
0
            return -1;
9637
0
        }
9638
9639
0
        n->subsys = NVME_SUBSYS(dev);
9640
0
    } else {
9641
0
        NvmeIdCtrl *id = &n->id_ctrl;
9642
0
        uint32_t ctratt = le32_to_cpu(id->ctratt);
9643
9644
0
        id->cmic |= NVME_CMIC_MULTI_CTRL;
9645
0
        ctratt |= NVME_CTRATT_ENDGRPS;
9646
9647
0
        id->endgidmax = cpu_to_le16(0x1);
9648
9649
0
        if (n->subsys->endgrp.fdp.enabled) {
9650
0
            ctratt |= NVME_CTRATT_FDPS;
9651
0
        }
9652
9653
0
        id->ctratt = cpu_to_le32(ctratt);
9654
0
    }
9655
9656
0
    cntlid = nvme_subsys_register_ctrl(n, errp);
9657
0
    if (cntlid < 0) {
9658
0
        return -1;
9659
0
    }
9660
9661
0
    n->cntlid = cntlid;
9662
9663
0
    return 0;
9664
0
}
9665
9666
void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns)
9667
0
{
9668
0
    uint32_t nsid = ns->params.nsid;
9669
0
    assert(nsid && nsid <= NVME_MAX_NAMESPACES);
9670
9671
0
    n->namespaces[nsid] = ns;
9672
0
    ns->attached++;
9673
0
}
9674
9675
static void nvme_realize(PCIDevice *pci_dev, Error **errp)
9676
0
{
9677
0
    NvmeCtrl *n = NVME(pci_dev);
9678
0
    DeviceState *dev = DEVICE(pci_dev);
9679
0
    NvmeNamespace *ns;
9680
0
    NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));
9681
9682
0
    if (pci_is_vf(pci_dev)) {
9683
        /*
9684
         * VFs derive settings from the parent. PF's lifespan exceeds
9685
         * that of VF's.
9686
         */
9687
0
        memcpy(&n->params, &pn->params, sizeof(NvmeParams));
9688
9689
        /*
9690
         * Set PF's serial value to a new string memory to prevent 'serial'
9691
         * property object release of PF when a VF is removed from the system.
9692
         */
9693
0
        n->params.serial = g_strdup(pn->params.serial);
9694
0
        n->subsys = pn->subsys;
9695
9696
        /*
9697
         * Assigning this link (strong link) causes an `object_unref` later in
9698
         * `object_release_link_property`. Increment the refcount to balance
9699
         * this out.
9700
         */
9701
0
        object_ref(OBJECT(pn->subsys));
9702
0
    }
9703
9704
0
    if (!nvme_check_params(n, errp)) {
9705
0
        return;
9706
0
    }
9707
9708
0
    qbus_init(&n->bus, sizeof(NvmeBus), TYPE_NVME_BUS, dev, dev->id);
9709
0
    qbus_set_bus_hotplug_handler(BUS(&n->bus));
9710
9711
0
    if (nvme_init_subsys(n, errp)) {
9712
0
        return;
9713
0
    }
9714
0
    nvme_init_state(n);
9715
0
    if (!nvme_init_pci(n, pci_dev, errp)) {
9716
0
        return;
9717
0
    }
9718
0
    nvme_init_ctrl(n, pci_dev);
9719
9720
    /* setup a namespace if the controller drive property was given */
9721
0
    if (n->namespace.blkconf.blk) {
9722
0
        ns = &n->namespace;
9723
0
        ns->params.nsid = 1;
9724
0
        ns->ctrl = n;
9725
0
        ns->subsys = n->subsys;
9726
9727
0
        if (nvme_ns_setup(ns, errp)) {
9728
0
            return;
9729
0
        }
9730
9731
0
        n->subsys->namespaces[ns->params.nsid] = ns;
9732
0
    }
9733
9734
0
    if (!nvme_set_migration_blockers(n, pci_dev, errp)) {
9735
0
        return;
9736
0
    }
9737
0
}
9738
9739
static void nvme_exit(PCIDevice *pci_dev)
9740
0
{
9741
0
    NvmeCtrl *n = NVME(pci_dev);
9742
0
    NvmeNamespace *ns;
9743
0
    int i;
9744
9745
0
    nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
9746
9747
0
    for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
9748
0
        ns = nvme_ns(n, i);
9749
0
        if (ns) {
9750
0
            ns->attached--;
9751
0
        }
9752
0
    }
9753
9754
0
    if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
9755
0
        pcie_sriov_pf_exit(pci_dev);
9756
0
    }
9757
9758
0
    nvme_subsys_unregister_ctrl(n->subsys, n);
9759
9760
0
    if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
9761
0
        g_free(n->sec_ctrl_list);
9762
0
    }
9763
9764
0
    g_free(n->cq);
9765
0
    g_free(n->sq);
9766
0
    g_free(n->aer_reqs);
9767
9768
0
    if (n->params.cmb_size_mb) {
9769
0
        g_free(n->cmb.buf);
9770
0
    }
9771
9772
    /* Only one of the `spdm_socket`s below should have been setup */
9773
0
    assert(!(pci_dev->doe_spdm.spdm_socket > 0 && n->spdm_socket >= 0));
9774
0
    if (pci_dev->doe_spdm.spdm_socket > 0) {
9775
0
        spdm_socket_close(pci_dev->doe_spdm.spdm_socket,
9776
0
                          SPDM_TRANSPORT_TYPE_DOE);
9777
0
    } else if (n->spdm_socket >= 0) {
9778
0
        spdm_socket_close(pci_dev->doe_spdm.spdm_socket,
9779
0
                          SPDM_TRANSPORT_TYPE_NVME);
9780
0
    }
9781
9782
0
    if (n->pmr.dev) {
9783
0
        host_memory_backend_set_mapped(n->pmr.dev, false);
9784
0
    }
9785
9786
0
    if (n->params.msix_exclusive_bar && !pci_is_vf(pci_dev)) {
9787
0
        msix_uninit_exclusive_bar(pci_dev);
9788
0
    } else {
9789
0
        msix_uninit(pci_dev, &n->bar0, &n->bar0);
9790
0
        memory_region_del_subregion(&n->bar0, &n->iomem);
9791
0
    }
9792
9793
0
    migrate_del_blocker(&n->migration_blocker);
9794
0
}
9795
9796
static const Property nvme_props[] = {
9797
    DEFINE_BLOCK_PROPERTIES(NvmeCtrl, namespace.blkconf),
9798
    DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmr.dev, TYPE_MEMORY_BACKEND,
9799
                     HostMemoryBackend *),
9800
    DEFINE_PROP_LINK("subsys", NvmeCtrl, subsys, TYPE_NVME_SUBSYS,
9801
                     NvmeSubsystem *),
9802
    DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial),
9803
    DEFINE_PROP_STRING("model", NvmeCtrl, params.model),
9804
    DEFINE_PROP_STRING("firmware-version", NvmeCtrl, params.firmware_version),
9805
    DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0),
9806
    DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0),
9807
    DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl, params.max_ioqpairs, 64),
9808
    DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl, params.msix_qsize, 65),
9809
    DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
9810
    DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64),
9811
    DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7),
9812
    DEFINE_PROP_UINT8("vsl", NvmeCtrl, params.vsl, 7),
9813
    DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl, params.use_intel_id, false),
9814
    DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl, params.legacy_cmb, false),
9815
    DEFINE_PROP_BOOL("ioeventfd", NvmeCtrl, params.ioeventfd, false),
9816
    DEFINE_PROP_BOOL("dbcs", NvmeCtrl, params.dbcs, true),
9817
    DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl, params.zasl, 0),
9818
    DEFINE_PROP_BOOL("zoned.auto_transition", NvmeCtrl,
9819
                     params.auto_transition_zones, true),
9820
    DEFINE_PROP_UINT16("sriov_max_vfs", NvmeCtrl, params.sriov_max_vfs, 0),
9821
    DEFINE_PROP_UINT16("sriov_vq_flexible", NvmeCtrl,
9822
                       params.sriov_vq_flexible, 0),
9823
    DEFINE_PROP_UINT16("sriov_vi_flexible", NvmeCtrl,
9824
                       params.sriov_vi_flexible, 0),
9825
    DEFINE_PROP_UINT32("sriov_max_vi_per_vf", NvmeCtrl,
9826
                       params.sriov_max_vi_per_vf, 0),
9827
    DEFINE_PROP_UINT32("sriov_max_vq_per_vf", NvmeCtrl,
9828
                       params.sriov_max_vq_per_vf, 0),
9829
    DEFINE_PROP_BOOL("msix-exclusive-bar", NvmeCtrl, params.msix_exclusive_bar,
9830
                     false),
9831
    DEFINE_PROP_UINT16("mqes", NvmeCtrl, params.mqes, 0x7ff),
9832
    DEFINE_PROP_UINT16("spdm_port", PCIDevice, spdm_port, 0),
9833
    DEFINE_PROP_SPDM_TRANS("spdm_trans", PCIDevice, spdm_trans,
9834
                           SPDM_TRANSPORT_TYPE_DOE),
9835
    DEFINE_PROP_BOOL("ctratt.mem", NvmeCtrl, params.ctratt.mem, false),
9836
    DEFINE_PROP_BOOL("atomic.dn", NvmeCtrl, params.atomic_dn, 0),
9837
    DEFINE_PROP_UINT16("atomic.awun", NvmeCtrl, params.atomic_awun, 0),
9838
    DEFINE_PROP_UINT16("atomic.awupf", NvmeCtrl, params.atomic_awupf, 0),
9839
    DEFINE_PROP_BOOL("ocp", NvmeCtrl, params.ocp, false),
9840
};
9841
9842
static void nvme_get_smart_warning(Object *obj, Visitor *v, const char *name,
9843
                                   void *opaque, Error **errp)
9844
0
{
9845
0
    NvmeCtrl *n = NVME(obj);
9846
0
    uint8_t value = n->smart_critical_warning;
9847
9848
0
    visit_type_uint8(v, name, &value, errp);
9849
0
}
9850
9851
static void nvme_set_smart_warning(Object *obj, Visitor *v, const char *name,
9852
                                   void *opaque, Error **errp)
9853
0
{
9854
0
    NvmeCtrl *n = NVME(obj);
9855
0
    uint8_t value, old_value, cap = 0, index, event;
9856
9857
0
    if (!visit_type_uint8(v, name, &value, errp)) {
9858
0
        return;
9859
0
    }
9860
9861
0
    cap = NVME_SMART_SPARE | NVME_SMART_TEMPERATURE | NVME_SMART_RELIABILITY
9862
0
          | NVME_SMART_MEDIA_READ_ONLY | NVME_SMART_FAILED_VOLATILE_MEDIA;
9863
0
    if (NVME_CAP_PMRS(ldq_le_p(&n->bar.cap))) {
9864
0
        cap |= NVME_SMART_PMR_UNRELIABLE;
9865
0
    }
9866
9867
0
    if ((value & cap) != value) {
9868
0
        error_setg(errp, "unsupported smart critical warning bits: 0x%x",
9869
0
                   value & ~cap);
9870
0
        return;
9871
0
    }
9872
9873
0
    old_value = n->smart_critical_warning;
9874
0
    n->smart_critical_warning = value;
9875
9876
    /* only inject new bits of smart critical warning */
9877
0
    for (index = 0; index < NVME_SMART_WARN_MAX; index++) {
9878
0
        event = 1 << index;
9879
0
        if (value & ~old_value & event)
9880
0
            nvme_smart_event(n, event);
9881
0
    }
9882
0
}
9883
9884
static void nvme_pci_reset(DeviceState *qdev)
9885
0
{
9886
0
    PCIDevice *pci_dev = PCI_DEVICE(qdev);
9887
0
    NvmeCtrl *n = NVME(pci_dev);
9888
9889
0
    trace_pci_nvme_pci_reset();
9890
0
    nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
9891
0
}
9892
9893
static void nvme_sriov_post_write_config(PCIDevice *dev, uint16_t old_num_vfs)
9894
0
{
9895
0
    NvmeCtrl *n = NVME(dev);
9896
0
    NvmeSecCtrlEntry *sctrl;
9897
0
    int i;
9898
9899
0
    for (i = pcie_sriov_num_vfs(dev); i < old_num_vfs; i++) {
9900
0
        sctrl = &n->sec_ctrl_list[i];
9901
0
        nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
9902
0
    }
9903
0
}
9904
9905
static void nvme_pci_write_config(PCIDevice *dev, uint32_t address,
9906
                                  uint32_t val, int len)
9907
0
{
9908
0
    uint16_t old_num_vfs = pcie_sriov_num_vfs(dev);
9909
9910
    /* DOE is only initialised if SPDM over DOE is used */
9911
0
    if (pcie_find_capability(dev, PCI_EXT_CAP_ID_DOE) &&
9912
0
        dev->spdm_trans == SPDM_TRANSPORT_TYPE_DOE) {
9913
0
        pcie_doe_write_config(&dev->doe_spdm, address, val, len);
9914
0
    }
9915
0
    pci_default_write_config(dev, address, val, len);
9916
0
    pcie_cap_flr_write_config(dev, address, val, len);
9917
0
    nvme_sriov_post_write_config(dev, old_num_vfs);
9918
0
}
9919
9920
static uint32_t nvme_pci_read_config(PCIDevice *dev, uint32_t address, int len)
9921
0
{
9922
0
    uint32_t val;
9923
9924
0
    if (dev->spdm_port && pcie_find_capability(dev, PCI_EXT_CAP_ID_DOE) &&
9925
0
        (dev->spdm_trans == SPDM_TRANSPORT_TYPE_DOE)) {
9926
0
        if (pcie_doe_read_config(&dev->doe_spdm, address, len, &val)) {
9927
0
            return val;
9928
0
        }
9929
0
    }
9930
0
    return pci_default_read_config(dev, address, len);
9931
0
}
9932
9933
static const VMStateDescription nvme_vmstate_cqe = {
9934
    .name = "nvme-cqe",
9935
    .version_id = 1,
9936
    .minimum_version_id = 1,
9937
    .fields = (const VMStateField[]) {
9938
        VMSTATE_UINT32(result, NvmeCqe),
9939
        VMSTATE_UINT32(dw1, NvmeCqe),
9940
        VMSTATE_UINT16(sq_head, NvmeCqe),
9941
        VMSTATE_UINT16(sq_id, NvmeCqe),
9942
        VMSTATE_UINT16(cid, NvmeCqe),
9943
        VMSTATE_UINT16(status, NvmeCqe),
9944
        VMSTATE_END_OF_LIST()
9945
    }
9946
};
9947
9948
static const VMStateDescription nvme_vmstate_cmd_dptr_sgl = {
9949
    .name = "nvme-request-cmd-dptr-sgl",
9950
    .version_id = 1,
9951
    .minimum_version_id = 1,
9952
    .fields = (const VMStateField[]) {
9953
        VMSTATE_UINT64(addr, NvmeSglDescriptor),
9954
        VMSTATE_UINT32(len, NvmeSglDescriptor),
9955
        VMSTATE_UINT8_ARRAY(rsvd, NvmeSglDescriptor, 3),
9956
        VMSTATE_UINT8(type, NvmeSglDescriptor),
9957
        VMSTATE_END_OF_LIST()
9958
    }
9959
};
9960
9961
static const VMStateDescription nvme_vmstate_cmd_dptr = {
9962
    .name = "nvme-request-cmd-dptr",
9963
    .version_id = 1,
9964
    .minimum_version_id = 1,
9965
    .fields = (const VMStateField[]) {
9966
        VMSTATE_UINT64(prp1, NvmeCmdDptr),
9967
        VMSTATE_UINT64(prp2, NvmeCmdDptr),
9968
        VMSTATE_STRUCT(sgl, NvmeCmdDptr, 0,
9969
                       nvme_vmstate_cmd_dptr_sgl, NvmeSglDescriptor),
9970
        VMSTATE_END_OF_LIST()
9971
    }
9972
};
9973
9974
static const VMStateDescription nvme_vmstate_cmd = {
9975
    .name = "nvme-request-cmd",
9976
    .version_id = 1,
9977
    .minimum_version_id = 1,
9978
    .fields = (const VMStateField[]) {
9979
        VMSTATE_UINT8(opcode, NvmeCmd),
9980
        VMSTATE_UINT8(flags, NvmeCmd),
9981
        VMSTATE_UINT16(cid, NvmeCmd),
9982
        VMSTATE_UINT32(nsid, NvmeCmd),
9983
        VMSTATE_UINT64(res1, NvmeCmd),
9984
        VMSTATE_UINT64(mptr, NvmeCmd),
9985
        VMSTATE_STRUCT(dptr, NvmeCmd, 0, nvme_vmstate_cmd_dptr, NvmeCmdDptr),
9986
        VMSTATE_UINT32(cdw10, NvmeCmd),
9987
        VMSTATE_UINT32(cdw11, NvmeCmd),
9988
        VMSTATE_UINT32(cdw12, NvmeCmd),
9989
        VMSTATE_UINT32(cdw13, NvmeCmd),
9990
        VMSTATE_UINT32(cdw14, NvmeCmd),
9991
        VMSTATE_UINT32(cdw15, NvmeCmd),
9992
        VMSTATE_END_OF_LIST()
9993
    }
9994
};
9995
9996
static bool nvme_req_pre_load(void *opaque, Error **errp)
9997
0
{
9998
0
    memset(opaque, 0x0, sizeof(NvmeRequest));
9999
0
    return true;
10000
0
}
10001
10002
static const VMStateDescription nvme_vmstate_request = {
10003
    .name = "nvme-request",
10004
    .version_id = 1,
10005
    .minimum_version_id = 1,
10006
    .pre_load_errp = nvme_req_pre_load,
10007
    .fields = (const VMStateField[]) {
10008
        VMSTATE_UINT16(status, NvmeRequest),
10009
        VMSTATE_STRUCT(cqe, NvmeRequest, 0, nvme_vmstate_cqe, NvmeCqe),
10010
        VMSTATE_STRUCT(cmd, NvmeRequest, 0, nvme_vmstate_cmd, NvmeCmd),
10011
        VMSTATE_END_OF_LIST()
10012
    }
10013
};
10014
10015
static const VMStateDescription nvme_vmstate_bar = {
10016
    .name = "nvme-bar",
10017
    .minimum_version_id = 1,
10018
    .version_id = 1,
10019
    .fields = (const VMStateField[]) {
10020
        VMSTATE_UINT64(cap, NvmeBar),
10021
        VMSTATE_UINT32(vs, NvmeBar),
10022
        VMSTATE_UINT32(intms, NvmeBar),
10023
        VMSTATE_UINT32(intmc, NvmeBar),
10024
        VMSTATE_UINT32(cc, NvmeBar),
10025
        VMSTATE_UINT8_ARRAY(rsvd24, NvmeBar, 4),
10026
        VMSTATE_UINT32(csts, NvmeBar),
10027
        VMSTATE_UINT32(nssr, NvmeBar),
10028
        VMSTATE_UINT32(aqa, NvmeBar),
10029
        VMSTATE_UINT64(asq, NvmeBar),
10030
        VMSTATE_UINT64(acq, NvmeBar),
10031
        VMSTATE_UINT32(cmbloc, NvmeBar),
10032
        VMSTATE_UINT32(cmbsz, NvmeBar),
10033
        VMSTATE_UINT32(bpinfo, NvmeBar),
10034
        VMSTATE_UINT32(bprsel, NvmeBar),
10035
        VMSTATE_UINT64(bpmbl, NvmeBar),
10036
        VMSTATE_UINT64(cmbmsc, NvmeBar),
10037
        VMSTATE_UINT32(cmbsts, NvmeBar),
10038
        VMSTATE_UINT8_ARRAY(rsvd92, NvmeBar, 3492),
10039
        VMSTATE_UINT32(pmrcap, NvmeBar),
10040
        VMSTATE_UINT32(pmrctl, NvmeBar),
10041
        VMSTATE_UINT32(pmrsts, NvmeBar),
10042
        VMSTATE_UINT32(pmrebs, NvmeBar),
10043
        VMSTATE_UINT32(pmrswtp, NvmeBar),
10044
        VMSTATE_UINT32(pmrmscl, NvmeBar),
10045
        VMSTATE_UINT32(pmrmscu, NvmeBar),
10046
        VMSTATE_UINT8_ARRAY(css, NvmeBar, 484),
10047
        VMSTATE_END_OF_LIST()
10048
    },
10049
};
10050
10051
static bool nvme_cqueue_pre_load(void *opaque, Error **errp)
10052
0
{
10053
0
    NvmeCQueue *cq = opaque;
10054
10055
0
    QTAILQ_INIT(&cq->req_list);
10056
0
    return true;
10057
0
}
10058
10059
static const VMStateDescription nvme_vmstate_cqueue = {
10060
    .name = "nvme-cq",
10061
    .version_id = 1,
10062
    .minimum_version_id = 1,
10063
    .pre_load_errp = nvme_cqueue_pre_load,
10064
    .fields = (const VMStateField[]) {
10065
        VMSTATE_UINT8(phase, NvmeCQueue),
10066
        VMSTATE_UINT16(cqid, NvmeCQueue),
10067
        VMSTATE_UINT16(irq_enabled, NvmeCQueue),
10068
        VMSTATE_UINT32(head, NvmeCQueue),
10069
        VMSTATE_UINT32(tail, NvmeCQueue),
10070
        VMSTATE_UINT32(vector, NvmeCQueue),
10071
        VMSTATE_UINT32(size, NvmeCQueue),
10072
        VMSTATE_UINT64(dma_addr, NvmeCQueue),
10073
10074
        VMSTATE_QTAILQ_V(req_list, NvmeCQueue, 1, nvme_vmstate_request,
10075
                         NvmeRequest, entry),
10076
10077
        /* db_addr, ei_addr, etc will be recalculated */
10078
        VMSTATE_END_OF_LIST()
10079
    }
10080
};
10081
10082
static const VMStateDescription nvme_vmstate_squeue = {
10083
    .name = "nvme-sq",
10084
    .version_id = 1,
10085
    .minimum_version_id = 1,
10086
    .fields = (const VMStateField[]) {
10087
        VMSTATE_UINT16(sqid, NvmeSQueue),
10088
        VMSTATE_UINT16(cqid, NvmeSQueue),
10089
        VMSTATE_UINT32(head, NvmeSQueue),
10090
        VMSTATE_UINT32(tail, NvmeSQueue),
10091
        VMSTATE_UINT32(size, NvmeSQueue),
10092
        VMSTATE_UINT64(dma_addr, NvmeSQueue),
10093
        /* db_addr, ei_addr, etc will be recalculated */
10094
        VMSTATE_END_OF_LIST()
10095
    }
10096
};
10097
10098
static const VMStateDescription nvme_vmstate_async_event_result = {
10099
    .name = "nvme-async-event-result",
10100
    .version_id = 1,
10101
    .minimum_version_id = 1,
10102
    .fields = (const VMStateField[]) {
10103
        VMSTATE_UINT8(event_type, NvmeAerResult),
10104
        VMSTATE_UINT8(event_info, NvmeAerResult),
10105
        VMSTATE_UINT8(log_page, NvmeAerResult),
10106
        VMSTATE_UINT8(resv, NvmeAerResult),
10107
        VMSTATE_END_OF_LIST()
10108
    }
10109
};
10110
10111
static const VMStateDescription nvme_vmstate_async_event = {
10112
    .name = "nvme-async-event",
10113
    .version_id = 1,
10114
    .minimum_version_id = 1,
10115
    .fields = (const VMStateField[]) {
10116
        VMSTATE_STRUCT(result, NvmeAsyncEvent, 0,
10117
                       nvme_vmstate_async_event_result, NvmeAerResult),
10118
        VMSTATE_END_OF_LIST()
10119
    }
10120
};
10121
10122
static const VMStateDescription nvme_vmstate_hbs = {
10123
    .name = "nvme-hbs",
10124
    .version_id = 1,
10125
    .minimum_version_id = 1,
10126
    .fields = (const VMStateField[]) {
10127
        VMSTATE_UINT8(acre, NvmeHostBehaviorSupport),
10128
        VMSTATE_UINT8(etdas, NvmeHostBehaviorSupport),
10129
        VMSTATE_UINT8(lbafee, NvmeHostBehaviorSupport),
10130
        VMSTATE_UINT8(rsvd3, NvmeHostBehaviorSupport),
10131
        VMSTATE_UINT16(cdfe, NvmeHostBehaviorSupport),
10132
        VMSTATE_UINT8_ARRAY(rsvd6, NvmeHostBehaviorSupport, 506),
10133
        VMSTATE_END_OF_LIST()
10134
    }
10135
};
10136
10137
const VMStateDescription nvme_vmstate_atomic = {
10138
    .name = "nvme-atomic",
10139
    .version_id = 1,
10140
    .minimum_version_id = 1,
10141
    .fields = (const VMStateField[]) {
10142
        VMSTATE_UINT32(atomic_max_write_size, NvmeAtomic),
10143
        VMSTATE_UINT64(atomic_boundary, NvmeAtomic),
10144
        VMSTATE_UINT64(atomic_nabo, NvmeAtomic),
10145
        VMSTATE_BOOL(atomic_writes, NvmeAtomic),
10146
        VMSTATE_END_OF_LIST()
10147
    }
10148
};
10149
10150
static bool pre_save_validate_aer_req(NvmeRequest *req, Error **errp)
10151
0
{
10152
    /*
10153
     * Can't use assert() here, because we don't want
10154
     * to just crash QEMU when user requests a migration.
10155
     */
10156
0
    if (!(req->cmd.opcode == NVME_ADM_CMD_ASYNC_EV_REQ)) {
10157
0
        error_setg(errp, "req->cmd.opcode (%u) != NVME_ADM_CMD_ASYNC_EV_REQ",
10158
0
                   req->cmd.opcode);
10159
0
        return false;
10160
0
    }
10161
10162
0
    if (!(req->ns == NULL)) {
10163
0
        error_setg(errp, "req->ns != NULL");
10164
0
        return false;
10165
0
    }
10166
10167
0
    if (!(req->sq == &req->sq->ctrl->admin_sq)) {
10168
0
        error_setg(errp, "req->sq != &req->sq->ctrl->admin_sq");
10169
0
        return false;
10170
0
    }
10171
10172
0
    if (!(req->aiocb == NULL)) {
10173
0
        error_setg(errp, "req->aiocb != NULL");
10174
0
        return false;
10175
0
    }
10176
10177
0
    if (!(req->opaque == NULL)) {
10178
0
        error_setg(errp, "req->opaque != NULL");
10179
0
        return false;
10180
0
    }
10181
10182
0
    if (!(req->atomic_write == false)) {
10183
0
        error_setg(errp, "req->atomic_write != false");
10184
0
        return false;
10185
0
    }
10186
10187
0
    if (req->sg.flags & NVME_SG_ALLOC) {
10188
0
        error_setg(errp, "unexpected NVME_SG_ALLOC flag in req->sg.flags");
10189
0
        return false;
10190
0
    }
10191
10192
0
    return true;
10193
0
}
10194
10195
static bool pre_save_validate_cq_req(NvmeRequest *req, Error **errp)
10196
0
{
10197
0
    if (!(req->ns == NULL)) {
10198
0
        error_setg(errp, "req->ns != NULL");
10199
0
        return false;
10200
0
    }
10201
10202
0
    if (!(req->aiocb == NULL)) {
10203
0
        error_setg(errp, "req->aiocb != NULL");
10204
0
        return false;
10205
0
    }
10206
10207
0
    if (!(req->opaque == NULL)) {
10208
0
        error_setg(errp, "req->opaque != NULL");
10209
0
        return false;
10210
0
    }
10211
10212
0
    if (!(req->atomic_write == false)) {
10213
0
        error_setg(errp, "req->atomic_write != false");
10214
0
        return false;
10215
0
    }
10216
10217
0
    if (req->sg.flags & NVME_SG_ALLOC) {
10218
0
        error_setg(errp, "unexpected NVME_SG_ALLOC flag in req->sg.flags");
10219
0
        return false;
10220
0
    }
10221
10222
0
    return true;
10223
0
}
10224
10225
static bool nvme_ctrl_pre_save(void *opaque, Error **errp)
10226
0
{
10227
0
    NvmeCtrl *n = opaque;
10228
0
    int i;
10229
10230
0
    trace_pci_nvme_pre_save_enter(n);
10231
10232
    /*
10233
     * We don't want to have a race with nvme_process_sq().
10234
     * What implicitly protects us from this is BQL.
10235
     */
10236
0
    assert(bql_locked());
10237
10238
    /* cancel all SQ processing BHs */
10239
0
    for (i = 0; i < n->num_queues; i++) {
10240
0
        NvmeSQueue *sq = n->sq[i];
10241
10242
0
        if (!sq) {
10243
0
            continue;
10244
0
        }
10245
10246
0
        qemu_bh_cancel(sq->bh);
10247
0
    }
10248
10249
    /* drain all IO */
10250
0
    for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
10251
0
        NvmeNamespace *ns;
10252
10253
0
        ns = nvme_ns(n, i);
10254
0
        if (!ns) {
10255
0
            continue;
10256
0
        }
10257
10258
0
        trace_pci_nvme_pre_save_ns_drain(n, i);
10259
0
        nvme_ns_drain(ns);
10260
0
    }
10261
10262
    /*
10263
     * Now, we should take care of AERs.
10264
     *
10265
     * 1. Save all queued events (n->aer_queue).
10266
     *    This is done automatically, see nvme_vmstate VMStateDescription.
10267
     *    Here we only need to print them for debugging purpose.
10268
     * 2. Go over outstanding AER requests (n->aer_reqs) and check they are
10269
     *    all have expected opcode (NVME_ADM_CMD_ASYNC_EV_REQ) and other fields.
10270
     *
10271
     * We must be really careful here, because in case of further
10272
     * QEMU NVMe changes, we may break migration without noticing it, or worse,
10273
     * introduce silent data corruption during migration.
10274
     */
10275
0
    if (n->aer_queued) {
10276
0
        NvmeAsyncEvent *event;
10277
10278
0
        QTAILQ_FOREACH(event, &n->aer_queue, entry) {
10279
0
            trace_pci_nvme_pre_save_aer(event->result.event_type,
10280
0
                                        event->result.event_info,
10281
0
                                        event->result.log_page);
10282
0
        }
10283
0
    }
10284
10285
0
    for (i = 0; i < n->outstanding_aers; i++) {
10286
0
        NvmeRequest *req = n->aer_reqs[i];
10287
10288
0
        if (!pre_save_validate_aer_req(req, errp)) {
10289
0
            return false;
10290
0
        }
10291
0
    }
10292
10293
    /*
10294
     * Make sure that all in-flight IO requests
10295
     * (except NVME_ADM_CMD_ASYNC_EV_REQ) are processed.
10296
     */
10297
0
    for (i = 0; i < n->num_queues; i++) {
10298
0
        NvmeRequest *req;
10299
0
        NvmeSQueue *sq = n->sq[i];
10300
10301
0
        if (!sq) {
10302
0
            continue;
10303
0
        }
10304
10305
0
        trace_pci_nvme_pre_save_sq_out_req_check(n, i,
10306
0
                                                 sq->head, sq->tail, sq->size);
10307
10308
0
        QTAILQ_FOREACH(req, &sq->out_req_list, entry) {
10309
0
            assert(req->cmd.opcode == NVME_ADM_CMD_ASYNC_EV_REQ);
10310
0
        }
10311
0
    }
10312
10313
    /* wait when all IO requests completions are written to guest memory */
10314
0
    for (i = 0; i < n->num_queues; i++) {
10315
0
        NvmeCQueue *cq = n->cq[i];
10316
10317
0
        if (!cq) {
10318
0
            continue;
10319
0
        }
10320
10321
0
        qemu_bh_cancel(cq->bh);
10322
        /* this should empty cq->req_list unless CQ is full */
10323
0
        nvme_post_cqes(cq);
10324
10325
0
        trace_pci_nvme_pre_save_cq_req_check(n, i,
10326
0
                                             cq->head, cq->tail, cq->size);
10327
10328
0
        if (!QTAILQ_EMPTY(&cq->req_list)) {
10329
0
            NvmeRequest *req;
10330
10331
0
            assert(nvme_cq_full(cq));
10332
10333
0
            QTAILQ_FOREACH(req, &cq->req_list, entry) {
10334
0
                trace_pci_nvme_pre_save_cq_unposted_cqe(
10335
0
                                            n, i, nvme_cid(req),
10336
0
                                            nvme_nsid(req->ns),
10337
0
                                            le32_to_cpu(req->cqe.result),
10338
0
                                            le32_to_cpu(req->cqe.dw1),
10339
0
                                            req->status, req->cmd.opcode);
10340
0
                if (!pre_save_validate_cq_req(req, errp)) {
10341
0
                    return false;
10342
0
                }
10343
0
            }
10344
0
        }
10345
0
    }
10346
10347
0
    for (uint32_t nsid = 0; nsid <= NVME_MAX_NAMESPACES; nsid++) {
10348
0
        NvmeNamespace *ns = n->namespaces[nsid];
10349
10350
0
        if (!ns) {
10351
0
            continue;
10352
0
        }
10353
10354
0
        if (ns != &n->namespace) {
10355
0
            error_setg(errp,
10356
0
                       "only one NVMe namespace is supported for migration");
10357
0
            return false;
10358
0
        }
10359
0
    }
10360
10361
0
    return true;
10362
0
}
10363
10364
static bool nvme_ctrl_post_load(void *opaque, int version_id, Error **errp)
10365
0
{
10366
0
    NvmeCtrl *n = opaque;
10367
0
    int i;
10368
10369
0
    trace_pci_nvme_post_load_enter(n);
10370
10371
    /* restore CQs first */
10372
0
    for (i = 0; i < n->num_queues; i++) {
10373
0
        NvmeCQueue *cq = n->cq[i];
10374
10375
0
        if (!cq) {
10376
0
            continue;
10377
0
        }
10378
10379
0
        if (cq->cqid != i) {
10380
0
            error_setg(errp, "inconsistent migration stream (cq->cqid != i)");
10381
0
            return false;
10382
0
        }
10383
10384
0
        cq->ctrl = n;
10385
0
        nvme_restore_cq(cq);
10386
0
        trace_pci_nvme_post_load_restore_cq(n, i, cq->head, cq->tail, cq->size);
10387
10388
0
        if (i == 0) {
10389
            /*
10390
             * Admin CQ lives in n->admin_cq, we don't need
10391
             * memory allocated for it in get_ptrs_array_entry() anymore.
10392
             *
10393
             * nvme_restore_cq() also takes care of:
10394
             * n->cq[0] = &n->admin_cq;
10395
             * so n->cq[0] remains valid.
10396
             */
10397
0
            g_free(cq);
10398
0
        }
10399
0
    }
10400
10401
0
    for (i = 0; i < n->num_queues; i++) {
10402
0
        NvmeSQueue *sq = n->sq[i];
10403
10404
0
        if (!sq) {
10405
0
            continue;
10406
0
        }
10407
10408
0
        if (sq->sqid != i) {
10409
0
            error_setg(errp, "inconsistent migration stream (sq->sqid != i)");
10410
0
            return false;
10411
0
        }
10412
10413
0
        if (!n->cq[sq->cqid]) {
10414
0
            error_setg(errp,
10415
0
                       "inconsistent migration stream (n->cq[sq->cqid] is NULL)");
10416
0
            return false;
10417
0
        }
10418
10419
0
        sq->ctrl = n;
10420
0
        nvme_restore_sq(sq);
10421
0
        trace_pci_nvme_post_load_restore_sq(n, i, sq->head, sq->tail, sq->size);
10422
10423
0
        if (i == 0) {
10424
            /* same as for CQ */
10425
0
            g_free(sq);
10426
0
        }
10427
0
    }
10428
10429
    /* restore cq->req_list-s */
10430
0
    for (i = 0; i < n->num_queues; i++) {
10431
0
        NvmeRequest *req_from, *next;
10432
0
        typeof_field(NvmeCQueue, req_list) req_list;
10433
0
        NvmeCQueue *cq = n->cq[i];
10434
10435
0
        if (!cq || QTAILQ_EMPTY(&cq->req_list)) {
10436
0
            continue;
10437
0
        }
10438
10439
        /*
10440
         * We use nvme_vmstate_request VMStateDescription to save/restore
10441
         * NvmeRequest structures, but tricky thing here is that
10442
         * memory for each cq->req_list item is allocated separately
10443
         * during restore. It doesn't work for us. We need to take
10444
         * an existing NvmeRequest structure from SQ's req_list pool
10445
         * and fill it with data from the newly allocated one (req_from).
10446
         * Then, we can safely release allocated memory for it.
10447
         */
10448
10449
        /* make a copy of cq->req_list (QTAILQ head) and clean cq->req_list */
10450
0
        QTAILQ_INIT(&req_list);
10451
0
        QTAILQ_FOREACH_SAFE(req_from, &cq->req_list, entry, next) {
10452
0
            QTAILQ_REMOVE(&cq->req_list, req_from, entry);
10453
0
            QTAILQ_INSERT_TAIL(&req_list, req_from, entry);
10454
0
        }
10455
0
        QTAILQ_INIT(&cq->req_list);
10456
10457
0
        QTAILQ_FOREACH_SAFE(req_from, &req_list, entry, next) {
10458
0
            uint16_t sqid = le16_to_cpu(req_from->cqe.sq_id);
10459
0
            NvmeRequest *req;
10460
0
            NvmeSQueue *sq;
10461
10462
0
            assert(!nvme_check_sqid(n, sqid));
10463
0
            sq = n->sq[sqid];
10464
10465
0
            req = QTAILQ_FIRST(&sq->req_list);
10466
0
            QTAILQ_REMOVE(&sq->req_list, req, entry);
10467
0
            QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
10468
0
            nvme_req_clear(req);
10469
10470
            /* copy data from the source NvmeRequest */
10471
0
            req->status = req_from->status;
10472
0
            memcpy(&req->cqe, &req_from->cqe, sizeof(NvmeCqe));
10473
0
            memcpy(&req->cmd, &req_from->cmd, sizeof(NvmeCmd));
10474
10475
0
            QTAILQ_REMOVE(&req_list, req_from, entry);
10476
0
            g_free(req_from);
10477
0
        }
10478
10479
0
        qemu_bh_schedule(cq->bh);
10480
0
    }
10481
10482
0
    if (n->aer_queued) {
10483
0
        NvmeAsyncEvent *event;
10484
10485
0
        QTAILQ_FOREACH(event, &n->aer_queue, entry) {
10486
0
            trace_pci_nvme_post_load_aer(event->result.event_type,
10487
0
                                         event->result.event_info,
10488
0
                                         event->result.log_page);
10489
0
        }
10490
0
    }
10491
10492
0
    for (i = 0; i < n->outstanding_aers; i++) {
10493
0
        NvmeSQueue *sq = &n->admin_sq;
10494
0
        NvmeRequest *req_from = n->aer_reqs[i];
10495
0
        NvmeRequest *req;
10496
10497
        /* Idea here is the same as for "restore cq->req_list-s" step */
10498
10499
        /* take an NvmeRequest struct from SQ */
10500
0
        req = QTAILQ_FIRST(&sq->req_list);
10501
0
        QTAILQ_REMOVE(&sq->req_list, req, entry);
10502
0
        QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
10503
0
        nvme_req_clear(req);
10504
10505
        /* copy data from the source NvmeRequest */
10506
0
        req->status = req_from->status;
10507
0
        memcpy(&req->cqe, &req_from->cqe, sizeof(NvmeCqe));
10508
0
        memcpy(&req->cmd, &req_from->cmd, sizeof(NvmeCmd));
10509
10510
0
        n->aer_reqs[i] = req;
10511
0
        g_free(req_from);
10512
0
    }
10513
10514
    /*
10515
     * We need to attach namespaces (currently, only one namespace is
10516
     * supported for migration).
10517
     * This logic comes from nvme_start_ctrl().
10518
     */
10519
0
    for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
10520
0
        NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
10521
10522
0
        if (!ns || (!ns->params.shared && ns->ctrl != n)) {
10523
0
            continue;
10524
0
        }
10525
10526
0
        if (nvme_csi_supported(n, ns->csi) && !ns->params.detached) {
10527
0
            if (!ns->attached || ns->params.shared) {
10528
0
                nvme_attach_ns(n, ns);
10529
0
            }
10530
0
        }
10531
0
    }
10532
10533
    /* schedule SQ processing */
10534
0
    for (i = 0; i < n->num_queues; i++) {
10535
0
        NvmeSQueue *sq = n->sq[i];
10536
10537
0
        if (!sq) {
10538
0
            continue;
10539
0
        }
10540
10541
0
        qemu_bh_schedule(sq->bh);
10542
0
    }
10543
10544
0
    return true;
10545
0
}
10546
10547
static const VMStateDescription nvme_vmstate = {
10548
    .name = "nvme",
10549
    .minimum_version_id = 1,
10550
    .version_id = 1,
10551
    .pre_save_errp = nvme_ctrl_pre_save,
10552
    .post_load_errp = nvme_ctrl_post_load,
10553
    .fields = (const VMStateField[]) {
10554
        VMSTATE_PCI_DEVICE(parent_obj, NvmeCtrl),
10555
        VMSTATE_MSIX(parent_obj, NvmeCtrl),
10556
        VMSTATE_STRUCT(bar, NvmeCtrl, 0, nvme_vmstate_bar, NvmeBar),
10557
10558
        VMSTATE_BOOL(qs_created, NvmeCtrl),
10559
        VMSTATE_UINT32(page_size, NvmeCtrl),
10560
        VMSTATE_UINT16(page_bits, NvmeCtrl),
10561
        VMSTATE_UINT16(max_prp_ents, NvmeCtrl),
10562
        VMSTATE_UINT32(max_q_ents, NvmeCtrl),
10563
        VMSTATE_UINT8(outstanding_aers, NvmeCtrl),
10564
        VMSTATE_UINT32(irq_status, NvmeCtrl),
10565
        VMSTATE_INT32(cq_pending, NvmeCtrl),
10566
10567
        VMSTATE_UINT64(host_timestamp, NvmeCtrl),
10568
        VMSTATE_UINT64(timestamp_set_qemu_clock_ms, NvmeCtrl),
10569
        VMSTATE_UINT64(starttime_ms, NvmeCtrl),
10570
        VMSTATE_UINT16(temperature, NvmeCtrl),
10571
        VMSTATE_UINT8(smart_critical_warning, NvmeCtrl),
10572
10573
        VMSTATE_UINT32(conf_msix_qsize, NvmeCtrl),
10574
        VMSTATE_UINT32(conf_ioqpairs, NvmeCtrl),
10575
        VMSTATE_UINT64(dbbuf_dbs, NvmeCtrl),
10576
        VMSTATE_UINT64(dbbuf_eis, NvmeCtrl),
10577
        VMSTATE_BOOL(dbbuf_enabled, NvmeCtrl),
10578
10579
        VMSTATE_UINT8(aer_mask, NvmeCtrl),
10580
        VMSTATE_VARRAY_OF_POINTER_TO_STRUCT_UINT8_ALLOC(
10581
            aer_reqs, NvmeCtrl, outstanding_aers, 0,
10582
            nvme_vmstate_request, NvmeRequest),
10583
        VMSTATE_QTAILQ_V(aer_queue, NvmeCtrl, 1, nvme_vmstate_async_event,
10584
                         NvmeAsyncEvent, entry),
10585
        VMSTATE_INT32(aer_queued, NvmeCtrl),
10586
10587
        VMSTATE_STRUCT(namespace, NvmeCtrl, 0, nvme_vmstate_ns, NvmeNamespace),
10588
10589
        VMSTATE_VARRAY_OF_POINTER_TO_STRUCT_UINT32_ALLOC(
10590
            sq, NvmeCtrl, num_queues, 0, nvme_vmstate_squeue, NvmeSQueue),
10591
        VMSTATE_VARRAY_OF_POINTER_TO_STRUCT_UINT32_ALLOC(
10592
            cq, NvmeCtrl, num_queues, 0, nvme_vmstate_cqueue, NvmeCQueue),
10593
10594
        VMSTATE_UINT16(features.temp_thresh_hi, NvmeCtrl),
10595
        VMSTATE_UINT16(features.temp_thresh_low, NvmeCtrl),
10596
        VMSTATE_UINT32(features.async_config, NvmeCtrl),
10597
        VMSTATE_STRUCT(features.hbs, NvmeCtrl, 0,
10598
                       nvme_vmstate_hbs, NvmeHostBehaviorSupport),
10599
10600
        VMSTATE_UINT32(dn, NvmeCtrl),
10601
        VMSTATE_STRUCT(atomic, NvmeCtrl, 0, nvme_vmstate_atomic, NvmeAtomic),
10602
10603
        VMSTATE_END_OF_LIST()
10604
    },
10605
};
10606
10607
static void nvme_class_init(ObjectClass *oc, const void *data)
10608
63
{
10609
63
    DeviceClass *dc = DEVICE_CLASS(oc);
10610
63
    PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
10611
10612
63
    pc->realize = nvme_realize;
10613
63
    pc->config_write = nvme_pci_write_config;
10614
63
    pc->config_read = nvme_pci_read_config;
10615
63
    pc->exit = nvme_exit;
10616
63
    pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
10617
63
    pc->revision = 2;
10618
10619
63
    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
10620
63
    dc->desc = "Non-Volatile Memory Express";
10621
63
    device_class_set_props(dc, nvme_props);
10622
63
    dc->vmsd = &nvme_vmstate;
10623
63
    device_class_set_legacy_reset(dc, nvme_pci_reset);
10624
63
}
10625
10626
static void nvme_instance_init(Object *obj)
10627
0
{
10628
0
    NvmeCtrl *n = NVME(obj);
10629
10630
0
    device_add_bootindex_property(obj, &n->namespace.blkconf.bootindex,
10631
0
                                  "bootindex", "/namespace@1,0",
10632
0
                                  DEVICE(obj));
10633
10634
0
    object_property_add(obj, "smart_critical_warning", "uint8",
10635
0
                        &uint8_type_info,
10636
0
                        nvme_get_smart_warning,
10637
0
                        nvme_set_smart_warning, NULL, NULL);
10638
0
}
10639
10640
static const TypeInfo nvme_info = {
10641
    .name          = TYPE_NVME,
10642
    .parent        = TYPE_PCI_DEVICE,
10643
    .instance_size = sizeof(NvmeCtrl),
10644
    .instance_init = nvme_instance_init,
10645
    .class_init    = nvme_class_init,
10646
    .interfaces = (const InterfaceInfo[]) {
10647
        { INTERFACE_PCIE_DEVICE },
10648
        { }
10649
    },
10650
};
10651
10652
static void nvme_ns_hot_plug(const HotplugHandler *hotplug_dev,
10653
                             DeviceState *dev, Error **errp)
10654
0
{
10655
0
    NvmeNamespace *ns = NVME_NS(dev);
10656
0
    NvmeSubsystem *subsys = ns->subsys;
10657
0
    uint32_t nsid = ns->params.nsid;
10658
0
    int i;
10659
10660
    /*
10661
     * Attach to all started controllers and notify via AEN.
10662
     * Skip controllers that haven't started yet (boot-time realize) —
10663
     * nvme_start_ctrl() will attach namespaces during controller init.
10664
     */
10665
0
    for (i = 0; i < NVME_MAX_CONTROLLERS; i++) {
10666
0
        NvmeCtrl *ctrl = nvme_subsys_ctrl(subsys, i);
10667
0
        if (!ctrl || !ctrl->qs_created) {
10668
0
            continue;
10669
0
        }
10670
10671
0
        if (nvme_csi_supported(ctrl, ns->csi) && !ns->params.detached) {
10672
0
            nvme_attach_ns(ctrl, ns);
10673
0
            nvme_update_dsm_limits(ctrl, ns);
10674
10675
0
            if (!test_and_set_bit(nsid, ctrl->changed_nsids)) {
10676
0
                nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE,
10677
0
                                   NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED,
10678
0
                                   NVME_LOG_CHANGED_NSLIST);
10679
0
            }
10680
0
        }
10681
0
    }
10682
0
}
10683
10684
static void nvme_ns_hot_unplug(const HotplugHandler *hotplug_dev,
10685
                               DeviceState *dev, Error **errp)
10686
0
{
10687
0
    NvmeNamespace *ns = NVME_NS(dev);
10688
0
    NvmeSubsystem *subsys = ns->subsys;
10689
0
    uint32_t nsid = ns->params.nsid;
10690
0
    int i;
10691
10692
    /*
10693
     * Drain in-flight I/O before tearing down the namespace.
10694
     * This must happen while the namespace is still attached to the
10695
     * controllers so any pending requests can complete normally.
10696
     */
10697
0
    nvme_ns_drain(ns);
10698
10699
    /*
10700
     * Detach from all controllers and notify the guest via AEN.
10701
     * The guest kernel will rescan namespaces and remove the block device.
10702
     */
10703
0
    for (i = 0; i < NVME_MAX_CONTROLLERS; i++) {
10704
0
        NvmeCtrl *ctrl = nvme_subsys_ctrl(subsys, i);
10705
0
        if (!ctrl || !nvme_ns(ctrl, nsid)) {
10706
0
            continue;
10707
0
        }
10708
10709
0
        nvme_detach_ns(ctrl, ns);
10710
0
        nvme_update_dsm_limits(ctrl, NULL);
10711
10712
0
        if (!test_and_set_bit(nsid, ctrl->changed_nsids)) {
10713
0
            nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE,
10714
0
                               NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED,
10715
0
                               NVME_LOG_CHANGED_NSLIST);
10716
0
        }
10717
0
    }
10718
10719
    /*
10720
     * Unrealize: removes from subsystem (in nvme_ns_unrealize), flushes,
10721
     * cleans up structures, and removes from QOM.
10722
     */
10723
0
    qdev_unrealize(dev);
10724
0
}
10725
10726
static void nvme_bus_class_init(ObjectClass *klass, const void *data)
10727
63
{
10728
63
    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
10729
63
    hc->plug = nvme_ns_hot_plug;
10730
63
    hc->unplug = nvme_ns_hot_unplug;
10731
63
}
10732
10733
static const TypeInfo nvme_bus_info = {
10734
    .name = TYPE_NVME_BUS,
10735
    .parent = TYPE_BUS,
10736
    .instance_size = sizeof(NvmeBus),
10737
    .class_init = nvme_bus_class_init,
10738
    .interfaces = (const InterfaceInfo[]) {
10739
        { TYPE_HOTPLUG_HANDLER },
10740
        { }
10741
    },
10742
};
10743
10744
static void nvme_register_types(void)
10745
63
{
10746
63
    type_register_static(&nvme_info);
10747
63
    type_register_static(&nvme_bus_info);
10748
63
}
10749
10750
type_init(nvme_register_types)