Line | Count | Source |
1 | | /* |
2 | | * QEMU NVM Express Controller |
3 | | * |
4 | | * Copyright (c) 2012, Intel Corporation |
5 | | * |
6 | | * Written by Keith Busch <keith.busch@intel.com> |
7 | | * |
8 | | * This code is licensed under the GNU GPL v2 or later. |
9 | | */ |
10 | | |
11 | | /** |
12 | | * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e |
13 | | * |
14 | | * https://nvmexpress.org/developers/nvme-specification/ |
15 | | * |
16 | | * |
17 | | * Notes on coding style |
18 | | * --------------------- |
19 | | * While QEMU coding style prefers lowercase hexadecimals in constants, the |
20 | | * NVMe subsystem use this format from the NVMe specifications in the comments |
21 | | * (i.e. 'h' suffix instead of '0x' prefix). |
22 | | * |
23 | | * Usage |
24 | | * ----- |
25 | | * See docs/system/devices/nvme.rst for extensive documentation. |
26 | | * |
27 | | * Add options: |
28 | | * -drive file=<file>,if=none,id=<drive_id> |
29 | | * -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id> |
30 | | * -device nvme,serial=<serial>,id=<bus_name>, \ |
31 | | * cmb_size_mb=<cmb_size_mb[optional]>, \ |
32 | | * [pmrdev=<mem_backend_file_id>,] \ |
33 | | * max_ioqpairs=<N[optional]>, \ |
34 | | * aerl=<N[optional]>,aer_max_queued=<N[optional]>, \ |
35 | | * mdts=<N[optional]>,vsl=<N[optional]>, \ |
36 | | * zoned.zasl=<N[optional]>, \ |
37 | | * zoned.auto_transition=<on|off[optional]>, \ |
38 | | * sriov_max_vfs=<N[optional]> \ |
39 | | * sriov_vq_flexible=<N[optional]> \ |
40 | | * sriov_vi_flexible=<N[optional]> \ |
41 | | * sriov_max_vi_per_vf=<N[optional]> \ |
42 | | * sriov_max_vq_per_vf=<N[optional]> \ |
43 | | * atomic.dn=<on|off[optional]>, \ |
44 | | * atomic.awun<N[optional]>, \ |
45 | | * atomic.awupf<N[optional]>, \ |
46 | | * subsys=<subsys_id>, \ |
47 | | * model=<model-str>, \ |
48 | | * firmware-version=<version-str> |
49 | | * -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\ |
50 | | * zoned=<true|false[optional]>, \ |
51 | | * subsys=<subsys_id>,shared=<true|false[optional]>, \ |
52 | | * detached=<true|false[optional]>, \ |
53 | | * zoned.zone_size=<N[optional]>, \ |
54 | | * zoned.zone_capacity=<N[optional]>, \ |
55 | | * zoned.descr_ext_size=<N[optional]>, \ |
56 | | * zoned.max_active=<N[optional]>, \ |
57 | | * zoned.max_open=<N[optional]>, \ |
58 | | * zoned.cross_read=<true|false[optional]> |
59 | | * |
60 | | * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at |
61 | | * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the |
62 | | * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to |
63 | | * always enable the CMBLOC and CMBSZ registers (v1.3 behavior). |
64 | | * |
65 | | * Enabling pmr emulation can be achieved by pointing to memory-backend-file. |
66 | | * For example: |
67 | | * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \ |
68 | | * size=<size> .... -device nvme,...,pmrdev=<mem_id> |
69 | | * |
70 | | * The PMR will use BAR 4/5 exclusively. |
71 | | * |
72 | | * To place controller(s) and namespace(s) to a subsystem, then provide |
73 | | * nvme-subsys device as above. |
74 | | * |
75 | | * nvme subsystem device parameters |
76 | | * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
77 | | * - `nqn` |
78 | | * This parameter provides the `<nqn_id>` part of the string |
79 | | * `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field |
80 | | * of subsystem controllers. Note that `<nqn_id>` should be unique per |
81 | | * subsystem, but this is not enforced by QEMU. If not specified, it will |
82 | | * default to the value of the `id` parameter (`<subsys_id>`). |
83 | | * |
84 | | * nvme device parameters |
85 | | * ~~~~~~~~~~~~~~~~~~~~~~ |
86 | | * - `subsys` |
87 | | * Specifying this parameter attaches the controller to the subsystem and |
88 | | * the SUBNQN field in the controller will report the NQN of the subsystem |
89 | | * device. This also enables multi controller capability represented in |
90 | | * Identify Controller data structure in CMIC (Controller Multi-path I/O and |
91 | | * Namespace Sharing Capabilities). |
92 | | * |
93 | | * - `aerl` |
94 | | * The Asynchronous Event Request Limit (AERL). Indicates the maximum number |
95 | | * of concurrently outstanding Asynchronous Event Request commands support |
96 | | * by the controller. This is a 0's based value. |
97 | | * |
98 | | * - `aer_max_queued` |
99 | | * This is the maximum number of events that the device will enqueue for |
100 | | * completion when there are no outstanding AERs. When the maximum number of |
101 | | * enqueued events are reached, subsequent events will be dropped. |
102 | | * |
103 | | * - `mdts` |
104 | | * Indicates the maximum data transfer size for a command that transfers data |
105 | | * between host-accessible memory and the controller. The value is specified |
106 | | * as a power of two (2^n) and is in units of the minimum memory page size |
107 | | * (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB). |
108 | | * |
109 | | * - `vsl` |
110 | | * Indicates the maximum data size limit for the Verify command. Like `mdts`, |
111 | | * this value is specified as a power of two (2^n) and is in units of the |
112 | | * minimum memory page size (CAP.MPSMIN). The default value is 7 (i.e. 512 |
113 | | * KiB). |
114 | | * |
115 | | * - `zoned.zasl` |
116 | | * Indicates the maximum data transfer size for the Zone Append command. Like |
117 | | * `mdts`, the value is specified as a power of two (2^n) and is in units of |
118 | | * the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e. |
119 | | * defaulting to the value of `mdts`). |
120 | | * |
121 | | * - `zoned.auto_transition` |
122 | | * Indicates if zones in zone state implicitly opened can be automatically |
123 | | * transitioned to zone state closed for resource management purposes. |
124 | | * Defaults to 'on'. |
125 | | * |
126 | | * - `sriov_max_vfs` |
127 | | * Indicates the maximum number of PCIe virtual functions supported |
128 | | * by the controller. The default value is 0. Specifying a non-zero value |
129 | | * enables reporting of both SR-IOV and ARI capabilities by the NVMe device. |
130 | | * Virtual function controllers will not report SR-IOV capability. |
131 | | * |
132 | | * NOTE: Single Root I/O Virtualization support is experimental. |
133 | | * All the related parameters may be subject to change. |
134 | | * |
135 | | * - `sriov_vq_flexible` |
136 | | * Indicates the total number of flexible queue resources assignable to all |
137 | | * the secondary controllers. Implicitly sets the number of primary |
138 | | * controller's private resources to `(max_ioqpairs - sriov_vq_flexible)`. |
139 | | * |
140 | | * - `sriov_vi_flexible` |
141 | | * Indicates the total number of flexible interrupt resources assignable to |
142 | | * all the secondary controllers. Implicitly sets the number of primary |
143 | | * controller's private resources to `(msix_qsize - sriov_vi_flexible)`. |
144 | | * |
145 | | * - `sriov_max_vi_per_vf` |
146 | | * Indicates the maximum number of virtual interrupt resources assignable |
147 | | * to a secondary controller. The default 0 resolves to |
148 | | * `(sriov_vi_flexible / sriov_max_vfs)`. |
149 | | * |
150 | | * - `sriov_max_vq_per_vf` |
151 | | * Indicates the maximum number of virtual queue resources assignable to |
152 | | * a secondary controller. The default 0 resolves to |
153 | | * `(sriov_vq_flexible / sriov_max_vfs)`. |
154 | | * |
155 | | * nvme namespace device parameters |
156 | | * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
157 | | * - `shared` |
158 | | * When the parent nvme device (as defined explicitly by the 'bus' parameter |
159 | | * or implicitly by the most recently defined NvmeBus) is linked to an |
160 | | * nvme-subsys device, the namespace will be attached to all controllers in |
161 | | * the subsystem. If set to 'off' (the default), the namespace will remain a |
162 | | * private namespace and may only be attached to a single controller at a |
163 | | * time. |
164 | | * |
165 | | * - `detached` |
166 | | * This parameter is only valid together with the `subsys` parameter. If left |
167 | | * at the default value (`false/off`), the namespace will be attached to all |
168 | | * controllers in the NVMe subsystem at boot-up. If set to `true/on`, the |
169 | | * namespace will be available in the subsystem but not attached to any |
170 | | * controllers. |
171 | | * |
172 | | * Setting `zoned` to true selects Zoned Command Set at the namespace. |
173 | | * In this case, the following namespace properties are available to configure |
174 | | * zoned operation: |
175 | | * zoned.zone_size=<zone size in bytes, default: 128MiB> |
176 | | * The number may be followed by K, M, G as in kilo-, mega- or giga-. |
177 | | * |
178 | | * zoned.zone_capacity=<zone capacity in bytes, default: zone size> |
179 | | * The value 0 (default) forces zone capacity to be the same as zone |
180 | | * size. The value of this property may not exceed zone size. |
181 | | * |
182 | | * zoned.descr_ext_size=<zone descriptor extension size, default 0> |
183 | | * This value needs to be specified in 64B units. If it is zero, |
184 | | * namespace(s) will not support zone descriptor extensions. |
185 | | * |
186 | | * zoned.max_active=<Maximum Active Resources (zones), default: 0> |
187 | | * The default value means there is no limit to the number of |
188 | | * concurrently active zones. |
189 | | * |
190 | | * zoned.max_open=<Maximum Open Resources (zones), default: 0> |
191 | | * The default value means there is no limit to the number of |
192 | | * concurrently open zones. |
193 | | * |
194 | | * zoned.cross_read=<enable RAZB, default: false> |
195 | | * Setting this property to true enables Read Across Zone Boundaries. |
196 | | */ |
197 | | |
198 | | #include "qemu/osdep.h" |
199 | | #include "qemu/bitops.h" |
200 | | #include "qemu/cutils.h" |
201 | | #include "qemu/error-report.h" |
202 | | #include "qemu/log.h" |
203 | | #include "qemu/units.h" |
204 | | #include "qemu/range.h" |
205 | | #include "qapi/error.h" |
206 | | #include "qapi/qapi-builtin-type-infos.h" |
207 | | #include "qapi/visitor.h" |
208 | | #include "system/system.h" |
209 | | #include "system/block-backend.h" |
210 | | #include "system/hostmem.h" |
211 | | #include "hw/pci/msix.h" |
212 | | #include "hw/pci/pcie_sriov.h" |
213 | | #include "system/spdm-socket.h" |
214 | | #include "migration/blocker.h" |
215 | | #include "migration/qemu-file-types.h" |
216 | | #include "migration/vmstate.h" |
217 | | |
218 | | #include "nvme.h" |
219 | | #include "dif.h" |
220 | | #include "trace.h" |
221 | | |
222 | 0 | #define NVME_MAX_IOQPAIRS 0xffff |
223 | 0 | #define NVME_DB_SIZE 4 |
224 | 0 | #define NVME_SPEC_VER 0x00010400 |
225 | 0 | #define NVME_CMB_BIR 2 |
226 | 0 | #define NVME_PMR_BIR 4 |
227 | 0 | #define NVME_TEMPERATURE 0x143 |
228 | 0 | #define NVME_TEMPERATURE_WARNING 0x157 |
229 | 0 | #define NVME_TEMPERATURE_CRITICAL 0x175 |
230 | 0 | #define NVME_NUM_FW_SLOTS 1 |
231 | | #define NVME_DEFAULT_MAX_ZA_SIZE (128 * KiB) |
232 | 0 | #define NVME_VF_RES_GRANULARITY 1 |
233 | 0 | #define NVME_VF_OFFSET 0x1 |
234 | 0 | #define NVME_VF_STRIDE 1 |
235 | | |
236 | | #define NVME_GUEST_ERR(trace, fmt, ...) \ |
237 | 0 | do { \ |
238 | 0 | (trace_##trace)(__VA_ARGS__); \ |
239 | 0 | qemu_log_mask(LOG_GUEST_ERROR, #trace \ |
240 | 0 | " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \ |
241 | 0 | } while (0) |
242 | | |
243 | | static const bool nvme_feature_support[NVME_FID_MAX] = { |
244 | | [NVME_ARBITRATION] = true, |
245 | | [NVME_POWER_MANAGEMENT] = true, |
246 | | [NVME_TEMPERATURE_THRESHOLD] = true, |
247 | | [NVME_ERROR_RECOVERY] = true, |
248 | | [NVME_VOLATILE_WRITE_CACHE] = true, |
249 | | [NVME_NUMBER_OF_QUEUES] = true, |
250 | | [NVME_INTERRUPT_COALESCING] = true, |
251 | | [NVME_INTERRUPT_VECTOR_CONF] = true, |
252 | | [NVME_WRITE_ATOMICITY] = true, |
253 | | [NVME_ASYNCHRONOUS_EVENT_CONF] = true, |
254 | | [NVME_TIMESTAMP] = true, |
255 | | [NVME_HOST_BEHAVIOR_SUPPORT] = true, |
256 | | [NVME_COMMAND_SET_PROFILE] = true, |
257 | | [NVME_FDP_MODE] = true, |
258 | | [NVME_FDP_EVENTS] = true, |
259 | | /* if you add something here, please update nvme_set_migration_blockers() */ |
260 | | }; |
261 | | |
262 | | static const uint32_t nvme_feature_cap[NVME_FID_MAX] = { |
263 | | [NVME_TEMPERATURE_THRESHOLD] = NVME_FEAT_CAP_CHANGE, |
264 | | [NVME_ERROR_RECOVERY] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS, |
265 | | [NVME_VOLATILE_WRITE_CACHE] = NVME_FEAT_CAP_CHANGE, |
266 | | [NVME_NUMBER_OF_QUEUES] = NVME_FEAT_CAP_CHANGE, |
267 | | [NVME_WRITE_ATOMICITY] = NVME_FEAT_CAP_CHANGE, |
268 | | [NVME_ASYNCHRONOUS_EVENT_CONF] = NVME_FEAT_CAP_CHANGE, |
269 | | [NVME_TIMESTAMP] = NVME_FEAT_CAP_CHANGE, |
270 | | [NVME_HOST_BEHAVIOR_SUPPORT] = NVME_FEAT_CAP_CHANGE, |
271 | | [NVME_COMMAND_SET_PROFILE] = NVME_FEAT_CAP_CHANGE, |
272 | | [NVME_FDP_MODE] = NVME_FEAT_CAP_CHANGE, |
273 | | [NVME_FDP_EVENTS] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS, |
274 | | }; |
275 | | |
276 | | static const uint32_t nvme_cse_acs_default[256] = { |
277 | | [NVME_ADM_CMD_DELETE_SQ] = NVME_CMD_EFF_CSUPP, |
278 | | [NVME_ADM_CMD_CREATE_SQ] = NVME_CMD_EFF_CSUPP, |
279 | | [NVME_ADM_CMD_GET_LOG_PAGE] = NVME_CMD_EFF_CSUPP, |
280 | | [NVME_ADM_CMD_DELETE_CQ] = NVME_CMD_EFF_CSUPP, |
281 | | [NVME_ADM_CMD_CREATE_CQ] = NVME_CMD_EFF_CSUPP, |
282 | | [NVME_ADM_CMD_IDENTIFY] = NVME_CMD_EFF_CSUPP, |
283 | | [NVME_ADM_CMD_ABORT] = NVME_CMD_EFF_CSUPP, |
284 | | [NVME_ADM_CMD_SET_FEATURES] = NVME_CMD_EFF_CSUPP, |
285 | | [NVME_ADM_CMD_GET_FEATURES] = NVME_CMD_EFF_CSUPP, |
286 | | [NVME_ADM_CMD_ASYNC_EV_REQ] = NVME_CMD_EFF_CSUPP, |
287 | | [NVME_ADM_CMD_NS_ATTACHMENT] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_NIC | |
288 | | NVME_CMD_EFF_CCC, |
289 | | [NVME_ADM_CMD_FORMAT_NVM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
290 | | [NVME_ADM_CMD_DIRECTIVE_RECV] = NVME_CMD_EFF_CSUPP, |
291 | | [NVME_ADM_CMD_DIRECTIVE_SEND] = NVME_CMD_EFF_CSUPP, |
292 | | [NVME_ADM_CMD_SECURITY_SEND] = NVME_CMD_EFF_CSUPP, |
293 | | [NVME_ADM_CMD_SECURITY_RECV] = NVME_CMD_EFF_CSUPP, |
294 | | }; |
295 | | |
296 | | static const uint32_t nvme_cse_iocs_nvm_default[256] = { |
297 | | [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
298 | | [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
299 | | [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
300 | | [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP, |
301 | | [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
302 | | [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP, |
303 | | [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
304 | | [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP, |
305 | | [NVME_CMD_IO_MGMT_RECV] = NVME_CMD_EFF_CSUPP, |
306 | | [NVME_CMD_IO_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
307 | | }; |
308 | | |
309 | | static const uint32_t nvme_cse_iocs_zoned_default[256] = { |
310 | | [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
311 | | [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
312 | | [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
313 | | [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP, |
314 | | [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
315 | | [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP, |
316 | | [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
317 | | [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP, |
318 | | [NVME_CMD_IO_MGMT_RECV] = NVME_CMD_EFF_CSUPP, |
319 | | [NVME_CMD_IO_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
320 | | |
321 | | [NVME_CMD_ZONE_APPEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
322 | | [NVME_CMD_ZONE_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC, |
323 | | [NVME_CMD_ZONE_MGMT_RECV] = NVME_CMD_EFF_CSUPP, |
324 | | }; |
325 | | |
326 | | static void nvme_process_sq(void *opaque); |
327 | | static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst); |
328 | | static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n); |
329 | | |
330 | | static uint16_t nvme_sqid(NvmeRequest *req) |
331 | 0 | { |
332 | 0 | return le16_to_cpu(req->sq->sqid); |
333 | 0 | } |
334 | | |
335 | | static inline uint16_t nvme_make_pid(NvmeNamespace *ns, uint16_t rg, |
336 | | uint16_t ph) |
337 | 0 | { |
338 | 0 | uint16_t rgif = ns->endgrp->fdp.rgif; |
339 | |
|
340 | 0 | if (!rgif) { |
341 | 0 | return ph; |
342 | 0 | } |
343 | | |
344 | 0 | return (rg << (16 - rgif)) | ph; |
345 | 0 | } |
346 | | |
347 | | static inline bool nvme_ph_valid(NvmeNamespace *ns, uint16_t ph) |
348 | 0 | { |
349 | 0 | return ph < ns->fdp.nphs; |
350 | 0 | } |
351 | | |
352 | | static inline bool nvme_rg_valid(NvmeEnduranceGroup *endgrp, uint16_t rg) |
353 | 0 | { |
354 | 0 | return rg < endgrp->fdp.nrg; |
355 | 0 | } |
356 | | |
357 | | static inline uint16_t nvme_pid2ph(NvmeNamespace *ns, uint16_t pid) |
358 | 0 | { |
359 | 0 | uint16_t rgif = ns->endgrp->fdp.rgif; |
360 | |
|
361 | 0 | if (!rgif) { |
362 | 0 | return pid; |
363 | 0 | } |
364 | | |
365 | 0 | return pid & ((1 << (15 - rgif)) - 1); |
366 | 0 | } |
367 | | |
368 | | static inline uint16_t nvme_pid2rg(NvmeNamespace *ns, uint16_t pid) |
369 | 0 | { |
370 | 0 | uint16_t rgif = ns->endgrp->fdp.rgif; |
371 | |
|
372 | 0 | if (!rgif) { |
373 | 0 | return 0; |
374 | 0 | } |
375 | | |
376 | 0 | return pid >> (16 - rgif); |
377 | 0 | } |
378 | | |
379 | | static inline bool nvme_parse_pid(NvmeNamespace *ns, uint16_t pid, |
380 | | uint16_t *ph, uint16_t *rg) |
381 | 0 | { |
382 | 0 | *rg = nvme_pid2rg(ns, pid); |
383 | 0 | *ph = nvme_pid2ph(ns, pid); |
384 | |
|
385 | 0 | return nvme_ph_valid(ns, *ph) && nvme_rg_valid(ns->endgrp, *rg); |
386 | 0 | } |
387 | | |
388 | | static void nvme_assign_zone_state(NvmeNamespace *ns, NvmeZone *zone, |
389 | | NvmeZoneState state) |
390 | 0 | { |
391 | 0 | if (QTAILQ_IN_USE(zone, entry)) { |
392 | 0 | switch (nvme_get_zone_state(zone)) { |
393 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
394 | 0 | QTAILQ_REMOVE(&ns->exp_open_zones, zone, entry); |
395 | 0 | break; |
396 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
397 | 0 | QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry); |
398 | 0 | break; |
399 | 0 | case NVME_ZONE_STATE_CLOSED: |
400 | 0 | QTAILQ_REMOVE(&ns->closed_zones, zone, entry); |
401 | 0 | break; |
402 | 0 | case NVME_ZONE_STATE_FULL: |
403 | 0 | QTAILQ_REMOVE(&ns->full_zones, zone, entry); |
404 | 0 | default: |
405 | 0 | ; |
406 | 0 | } |
407 | 0 | } |
408 | | |
409 | 0 | nvme_set_zone_state(zone, state); |
410 | |
|
411 | 0 | switch (state) { |
412 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
413 | 0 | QTAILQ_INSERT_TAIL(&ns->exp_open_zones, zone, entry); |
414 | 0 | break; |
415 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
416 | 0 | QTAILQ_INSERT_TAIL(&ns->imp_open_zones, zone, entry); |
417 | 0 | break; |
418 | 0 | case NVME_ZONE_STATE_CLOSED: |
419 | 0 | QTAILQ_INSERT_TAIL(&ns->closed_zones, zone, entry); |
420 | 0 | break; |
421 | 0 | case NVME_ZONE_STATE_FULL: |
422 | 0 | QTAILQ_INSERT_TAIL(&ns->full_zones, zone, entry); |
423 | 0 | case NVME_ZONE_STATE_READ_ONLY: |
424 | 0 | break; |
425 | 0 | default: |
426 | 0 | zone->d.za = 0; |
427 | 0 | } |
428 | 0 | } |
429 | | |
430 | | static uint16_t nvme_zns_check_resources(NvmeNamespace *ns, uint32_t act, |
431 | | uint32_t opn, uint32_t zrwa) |
432 | 0 | { |
433 | 0 | if (ns->params.max_active_zones != 0 && |
434 | 0 | ns->nr_active_zones + act > ns->params.max_active_zones) { |
435 | 0 | trace_pci_nvme_err_insuff_active_res(ns->params.max_active_zones); |
436 | 0 | return NVME_ZONE_TOO_MANY_ACTIVE | NVME_DNR; |
437 | 0 | } |
438 | | |
439 | 0 | if (ns->params.max_open_zones != 0 && |
440 | 0 | ns->nr_open_zones + opn > ns->params.max_open_zones) { |
441 | 0 | trace_pci_nvme_err_insuff_open_res(ns->params.max_open_zones); |
442 | 0 | return NVME_ZONE_TOO_MANY_OPEN | NVME_DNR; |
443 | 0 | } |
444 | | |
445 | 0 | if (zrwa > ns->zns.numzrwa) { |
446 | 0 | return NVME_NOZRWA | NVME_DNR; |
447 | 0 | } |
448 | | |
449 | 0 | return NVME_SUCCESS; |
450 | 0 | } |
451 | | |
452 | | /* |
453 | | * Check if we can open a zone without exceeding open/active limits. |
454 | | * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5). |
455 | | */ |
456 | | static uint16_t nvme_aor_check(NvmeNamespace *ns, uint32_t act, uint32_t opn) |
457 | 0 | { |
458 | 0 | return nvme_zns_check_resources(ns, act, opn, 0); |
459 | 0 | } |
460 | | |
461 | | static NvmeFdpEvent *nvme_fdp_alloc_event(NvmeCtrl *n, NvmeFdpEventBuffer *ebuf) |
462 | 0 | { |
463 | 0 | NvmeFdpEvent *ret = NULL; |
464 | 0 | bool is_full = ebuf->next == ebuf->start && ebuf->nelems; |
465 | |
|
466 | 0 | ret = &ebuf->events[ebuf->next++]; |
467 | 0 | if (unlikely(ebuf->next == NVME_FDP_MAX_EVENTS)) { |
468 | 0 | ebuf->next = 0; |
469 | 0 | } |
470 | 0 | if (is_full) { |
471 | 0 | ebuf->start = ebuf->next; |
472 | 0 | } else { |
473 | 0 | ebuf->nelems++; |
474 | 0 | } |
475 | |
|
476 | 0 | memset(ret, 0, sizeof(NvmeFdpEvent)); |
477 | 0 | ret->timestamp = nvme_get_timestamp(n); |
478 | |
|
479 | 0 | return ret; |
480 | 0 | } |
481 | | |
482 | | static inline int log_event(NvmeRuHandle *ruh, uint8_t event_type) |
483 | 0 | { |
484 | 0 | return (ruh->event_filter >> nvme_fdp_evf_shifts[event_type]) & 0x1; |
485 | 0 | } |
486 | | |
487 | | static bool nvme_update_ruh(NvmeCtrl *n, NvmeNamespace *ns, uint16_t pid) |
488 | 0 | { |
489 | 0 | NvmeEnduranceGroup *endgrp = ns->endgrp; |
490 | 0 | NvmeRuHandle *ruh; |
491 | 0 | NvmeReclaimUnit *ru; |
492 | 0 | NvmeFdpEvent *e = NULL; |
493 | 0 | uint16_t ph, rg, ruhid; |
494 | |
|
495 | 0 | if (!nvme_parse_pid(ns, pid, &ph, &rg)) { |
496 | 0 | return false; |
497 | 0 | } |
498 | | |
499 | 0 | ruhid = ns->fdp.phs[ph]; |
500 | |
|
501 | 0 | ruh = &endgrp->fdp.ruhs[ruhid]; |
502 | 0 | ru = &ruh->rus[rg]; |
503 | |
|
504 | 0 | if (ru->ruamw) { |
505 | 0 | if (log_event(ruh, FDP_EVT_RU_NOT_FULLY_WRITTEN)) { |
506 | 0 | e = nvme_fdp_alloc_event(n, &endgrp->fdp.host_events); |
507 | 0 | e->type = FDP_EVT_RU_NOT_FULLY_WRITTEN; |
508 | 0 | e->flags = FDPEF_PIV | FDPEF_NSIDV | FDPEF_LV; |
509 | 0 | e->pid = cpu_to_le16(pid); |
510 | 0 | e->nsid = cpu_to_le32(ns->params.nsid); |
511 | 0 | e->rgid = cpu_to_le16(rg); |
512 | 0 | e->ruhid = cpu_to_le16(ruhid); |
513 | 0 | } |
514 | | |
515 | | /* log (eventual) GC overhead of prematurely swapping the RU */ |
516 | 0 | nvme_fdp_stat_inc(&endgrp->fdp.mbmw, nvme_l2b(ns, ru->ruamw)); |
517 | 0 | } |
518 | |
|
519 | 0 | ru->ruamw = ruh->ruamw; |
520 | |
|
521 | 0 | return true; |
522 | 0 | } |
523 | | |
524 | | static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr) |
525 | 0 | { |
526 | 0 | hwaddr hi, lo; |
527 | |
|
528 | 0 | if (!n->cmb.cmse) { |
529 | 0 | return false; |
530 | 0 | } |
531 | | |
532 | 0 | lo = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba; |
533 | 0 | hi = lo + int128_get64(n->cmb.mem.size); |
534 | |
|
535 | 0 | return addr >= lo && addr < hi; |
536 | 0 | } |
537 | | |
538 | | static inline void *nvme_addr_to_cmb(NvmeCtrl *n, hwaddr addr) |
539 | 0 | { |
540 | 0 | hwaddr base = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba; |
541 | 0 | return &n->cmb.buf[addr - base]; |
542 | 0 | } |
543 | | |
544 | | static bool nvme_addr_is_pmr(NvmeCtrl *n, hwaddr addr) |
545 | 0 | { |
546 | 0 | hwaddr hi; |
547 | |
|
548 | 0 | if (!n->pmr.cmse) { |
549 | 0 | return false; |
550 | 0 | } |
551 | | |
552 | 0 | hi = n->pmr.cba + int128_get64(n->pmr.dev->mr.size); |
553 | |
|
554 | 0 | return addr >= n->pmr.cba && addr < hi; |
555 | 0 | } |
556 | | |
557 | | static inline void *nvme_addr_to_pmr(NvmeCtrl *n, hwaddr addr) |
558 | 0 | { |
559 | 0 | return memory_region_get_ram_ptr(&n->pmr.dev->mr) + (addr - n->pmr.cba); |
560 | 0 | } |
561 | | |
562 | | static inline bool nvme_addr_is_iomem(NvmeCtrl *n, hwaddr addr) |
563 | 0 | { |
564 | 0 | hwaddr hi, lo; |
565 | | |
566 | | /* |
567 | | * The purpose of this check is to guard against invalid "local" access to |
568 | | * the iomem (i.e. controller registers). Thus, we check against the range |
569 | | * covered by the 'bar0' MemoryRegion since that is currently composed of |
570 | | * two subregions (the NVMe "MBAR" and the MSI-X table/pba). Note, however, |
571 | | * that if the device model is ever changed to allow the CMB to be located |
572 | | * in BAR0 as well, then this must be changed. |
573 | | */ |
574 | 0 | lo = n->bar0.addr; |
575 | 0 | hi = lo + int128_get64(n->bar0.size); |
576 | |
|
577 | 0 | return addr >= lo && addr < hi; |
578 | 0 | } |
579 | | |
580 | | static int nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size) |
581 | 0 | { |
582 | 0 | hwaddr hi = addr + size - 1; |
583 | 0 | if (hi < addr) { |
584 | 0 | return 1; |
585 | 0 | } |
586 | | |
587 | 0 | if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) { |
588 | 0 | memcpy(buf, nvme_addr_to_cmb(n, addr), size); |
589 | 0 | return 0; |
590 | 0 | } |
591 | | |
592 | 0 | if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) { |
593 | 0 | memcpy(buf, nvme_addr_to_pmr(n, addr), size); |
594 | 0 | return 0; |
595 | 0 | } |
596 | | |
597 | 0 | return pci_dma_read(PCI_DEVICE(n), addr, buf, size); |
598 | 0 | } |
599 | | |
600 | | static int nvme_addr_write(NvmeCtrl *n, hwaddr addr, const void *buf, int size) |
601 | 0 | { |
602 | 0 | hwaddr hi = addr + size - 1; |
603 | 0 | if (hi < addr) { |
604 | 0 | return 1; |
605 | 0 | } |
606 | | |
607 | 0 | if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) { |
608 | 0 | memcpy(nvme_addr_to_cmb(n, addr), buf, size); |
609 | 0 | return 0; |
610 | 0 | } |
611 | | |
612 | 0 | if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) { |
613 | 0 | memcpy(nvme_addr_to_pmr(n, addr), buf, size); |
614 | 0 | return 0; |
615 | 0 | } |
616 | | |
617 | 0 | return pci_dma_write(PCI_DEVICE(n), addr, buf, size); |
618 | 0 | } |
619 | | |
620 | | static bool nvme_nsid_valid(NvmeCtrl *n, uint32_t nsid) |
621 | 0 | { |
622 | 0 | return nsid && |
623 | 0 | (nsid == NVME_NSID_BROADCAST || nsid <= NVME_MAX_NAMESPACES); |
624 | 0 | } |
625 | | |
626 | | static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid) |
627 | 0 | { |
628 | 0 | return sqid < n->conf_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1; |
629 | 0 | } |
630 | | |
631 | | static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid) |
632 | 0 | { |
633 | 0 | return cqid < n->conf_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1; |
634 | 0 | } |
635 | | |
636 | | static void nvme_inc_cq_tail(NvmeCQueue *cq) |
637 | 0 | { |
638 | 0 | cq->tail++; |
639 | 0 | if (cq->tail >= cq->size) { |
640 | 0 | cq->tail = 0; |
641 | 0 | cq->phase = !cq->phase; |
642 | 0 | } |
643 | 0 | } |
644 | | |
645 | | static void nvme_inc_sq_head(NvmeSQueue *sq) |
646 | 0 | { |
647 | 0 | sq->head = (sq->head + 1) % sq->size; |
648 | 0 | } |
649 | | |
650 | | static uint8_t nvme_cq_full(NvmeCQueue *cq) |
651 | 0 | { |
652 | 0 | return (cq->tail + 1) % cq->size == cq->head; |
653 | 0 | } |
654 | | |
655 | | static uint8_t nvme_sq_empty(NvmeSQueue *sq) |
656 | 0 | { |
657 | 0 | return sq->head == sq->tail; |
658 | 0 | } |
659 | | |
660 | | static void nvme_irq_check(NvmeCtrl *n) |
661 | 0 | { |
662 | 0 | PCIDevice *pci = PCI_DEVICE(n); |
663 | 0 | uint32_t intms = ldl_le_p(&n->bar.intms); |
664 | |
|
665 | 0 | if (msix_enabled(pci)) { |
666 | 0 | return; |
667 | 0 | } |
668 | | |
669 | | /* vfs does not implement intx */ |
670 | 0 | if (pci_is_vf(pci)) { |
671 | 0 | return; |
672 | 0 | } |
673 | | |
674 | 0 | if (~intms & n->irq_status) { |
675 | 0 | pci_irq_assert(pci); |
676 | 0 | } else { |
677 | 0 | pci_irq_deassert(pci); |
678 | 0 | } |
679 | 0 | } |
680 | | |
681 | | static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq) |
682 | 0 | { |
683 | 0 | PCIDevice *pci = PCI_DEVICE(n); |
684 | |
|
685 | 0 | if (cq->irq_enabled) { |
686 | 0 | if (msix_enabled(pci)) { |
687 | 0 | trace_pci_nvme_irq_msix(cq->vector); |
688 | 0 | msix_notify(pci, cq->vector); |
689 | 0 | } else { |
690 | 0 | trace_pci_nvme_irq_pin(); |
691 | 0 | assert(cq->vector < 32); |
692 | 0 | n->irq_status |= 1 << cq->vector; |
693 | 0 | nvme_irq_check(n); |
694 | 0 | } |
695 | 0 | } else { |
696 | 0 | trace_pci_nvme_irq_masked(); |
697 | 0 | } |
698 | 0 | } |
699 | | |
700 | | static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq) |
701 | 0 | { |
702 | 0 | if (cq->irq_enabled) { |
703 | 0 | if (msix_enabled(PCI_DEVICE(n))) { |
704 | 0 | return; |
705 | 0 | } else { |
706 | 0 | assert(cq->vector < 32); |
707 | 0 | if (!n->cq_pending) { |
708 | 0 | n->irq_status &= ~(1 << cq->vector); |
709 | 0 | } |
710 | 0 | nvme_irq_check(n); |
711 | 0 | } |
712 | 0 | } |
713 | 0 | } |
714 | | |
715 | | static void nvme_req_clear(NvmeRequest *req) |
716 | 0 | { |
717 | 0 | req->ns = NULL; |
718 | 0 | req->opaque = NULL; |
719 | 0 | req->aiocb = NULL; |
720 | 0 | memset(&req->cqe, 0x0, sizeof(req->cqe)); |
721 | 0 | req->status = NVME_SUCCESS; |
722 | 0 | } |
723 | | |
724 | | static inline void nvme_sg_init(NvmeCtrl *n, NvmeSg *sg, bool dma) |
725 | 0 | { |
726 | 0 | if (dma) { |
727 | 0 | pci_dma_sglist_init(&sg->qsg, PCI_DEVICE(n), 0); |
728 | 0 | sg->flags = NVME_SG_DMA; |
729 | 0 | } else { |
730 | 0 | qemu_iovec_init(&sg->iov, 0); |
731 | 0 | } |
732 | |
|
733 | 0 | sg->flags |= NVME_SG_ALLOC; |
734 | 0 | } |
735 | | |
736 | | static inline void nvme_sg_unmap(NvmeSg *sg) |
737 | 0 | { |
738 | 0 | if (!(sg->flags & NVME_SG_ALLOC)) { |
739 | 0 | return; |
740 | 0 | } |
741 | | |
742 | 0 | if (sg->flags & NVME_SG_DMA) { |
743 | 0 | qemu_sglist_destroy(&sg->qsg); |
744 | 0 | } else { |
745 | 0 | qemu_iovec_destroy(&sg->iov); |
746 | 0 | } |
747 | |
|
748 | 0 | memset(sg, 0x0, sizeof(*sg)); |
749 | 0 | } |
750 | | |
751 | | /* |
752 | | * When metadata is transferred as extended LBAs, the DPTR mapped into `sg` |
753 | | * holds both data and metadata. This function splits the data and metadata |
754 | | * into two separate QSG/IOVs. |
755 | | */ |
756 | | static void nvme_sg_split(NvmeSg *sg, NvmeNamespace *ns, NvmeSg *data, |
757 | | NvmeSg *mdata) |
758 | 0 | { |
759 | 0 | NvmeSg *dst = data; |
760 | 0 | uint32_t trans_len, count = ns->lbasz; |
761 | 0 | uint64_t offset = 0; |
762 | 0 | bool dma = sg->flags & NVME_SG_DMA; |
763 | 0 | size_t sge_len; |
764 | 0 | size_t sg_len = dma ? sg->qsg.size : sg->iov.size; |
765 | 0 | int sg_idx = 0; |
766 | |
|
767 | 0 | assert(sg->flags & NVME_SG_ALLOC); |
768 | |
|
769 | 0 | while (sg_len) { |
770 | 0 | sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len; |
771 | |
|
772 | 0 | trans_len = MIN(sg_len, count); |
773 | 0 | trans_len = MIN(trans_len, sge_len - offset); |
774 | |
|
775 | 0 | if (dst) { |
776 | 0 | if (dma) { |
777 | 0 | qemu_sglist_add(&dst->qsg, sg->qsg.sg[sg_idx].base + offset, |
778 | 0 | trans_len); |
779 | 0 | } else { |
780 | 0 | qemu_iovec_add(&dst->iov, |
781 | 0 | sg->iov.iov[sg_idx].iov_base + offset, |
782 | 0 | trans_len); |
783 | 0 | } |
784 | 0 | } |
785 | |
|
786 | 0 | sg_len -= trans_len; |
787 | 0 | count -= trans_len; |
788 | 0 | offset += trans_len; |
789 | |
|
790 | 0 | if (count == 0) { |
791 | 0 | dst = (dst == data) ? mdata : data; |
792 | 0 | count = (dst == data) ? ns->lbasz : ns->lbaf.ms; |
793 | 0 | } |
794 | |
|
795 | 0 | if (sge_len == offset) { |
796 | 0 | offset = 0; |
797 | 0 | sg_idx++; |
798 | 0 | } |
799 | 0 | } |
800 | 0 | } |
801 | | |
802 | | static uint16_t nvme_map_addr_cmb(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr, |
803 | | size_t len) |
804 | 0 | { |
805 | 0 | if (!len) { |
806 | 0 | return NVME_SUCCESS; |
807 | 0 | } |
808 | | |
809 | 0 | trace_pci_nvme_map_addr_cmb(addr, len); |
810 | |
|
811 | 0 | if (!nvme_addr_is_cmb(n, addr) || !nvme_addr_is_cmb(n, addr + len - 1)) { |
812 | 0 | return NVME_DATA_TRAS_ERROR; |
813 | 0 | } |
814 | | |
815 | 0 | qemu_iovec_add(iov, nvme_addr_to_cmb(n, addr), len); |
816 | |
|
817 | 0 | return NVME_SUCCESS; |
818 | 0 | } |
819 | | |
820 | | static uint16_t nvme_map_addr_pmr(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr, |
821 | | size_t len) |
822 | 0 | { |
823 | 0 | if (!len) { |
824 | 0 | return NVME_SUCCESS; |
825 | 0 | } |
826 | | |
827 | 0 | if (!nvme_addr_is_pmr(n, addr) || !nvme_addr_is_pmr(n, addr + len - 1)) { |
828 | 0 | return NVME_DATA_TRAS_ERROR; |
829 | 0 | } |
830 | | |
831 | 0 | qemu_iovec_add(iov, nvme_addr_to_pmr(n, addr), len); |
832 | |
|
833 | 0 | return NVME_SUCCESS; |
834 | 0 | } |
835 | | |
836 | | static uint16_t nvme_map_addr(NvmeCtrl *n, NvmeSg *sg, hwaddr addr, size_t len) |
837 | 0 | { |
838 | 0 | bool cmb = false, pmr = false; |
839 | |
|
840 | 0 | if (!len) { |
841 | 0 | return NVME_SUCCESS; |
842 | 0 | } |
843 | | |
844 | 0 | trace_pci_nvme_map_addr(addr, len); |
845 | |
|
846 | 0 | if (nvme_addr_is_iomem(n, addr)) { |
847 | 0 | return NVME_DATA_TRAS_ERROR; |
848 | 0 | } |
849 | | |
850 | 0 | if (nvme_addr_is_cmb(n, addr)) { |
851 | 0 | cmb = true; |
852 | 0 | } else if (nvme_addr_is_pmr(n, addr)) { |
853 | 0 | pmr = true; |
854 | 0 | } |
855 | |
|
856 | 0 | if (cmb || pmr) { |
857 | 0 | if (sg->flags & NVME_SG_DMA) { |
858 | 0 | return NVME_INVALID_USE_OF_CMB | NVME_DNR; |
859 | 0 | } |
860 | | |
861 | 0 | if (sg->iov.niov + 1 > IOV_MAX) { |
862 | 0 | goto max_mappings_exceeded; |
863 | 0 | } |
864 | | |
865 | 0 | if (cmb) { |
866 | 0 | return nvme_map_addr_cmb(n, &sg->iov, addr, len); |
867 | 0 | } else { |
868 | 0 | return nvme_map_addr_pmr(n, &sg->iov, addr, len); |
869 | 0 | } |
870 | 0 | } |
871 | | |
872 | 0 | if (!(sg->flags & NVME_SG_DMA)) { |
873 | 0 | return NVME_INVALID_USE_OF_CMB | NVME_DNR; |
874 | 0 | } |
875 | | |
876 | 0 | if (sg->qsg.nsg + 1 > IOV_MAX) { |
877 | 0 | goto max_mappings_exceeded; |
878 | 0 | } |
879 | | |
880 | 0 | qemu_sglist_add(&sg->qsg, addr, len); |
881 | |
|
882 | 0 | return NVME_SUCCESS; |
883 | | |
884 | 0 | max_mappings_exceeded: |
885 | 0 | NVME_GUEST_ERR(pci_nvme_ub_too_many_mappings, |
886 | 0 | "number of mappings exceed 1024"); |
887 | 0 | return NVME_INTERNAL_DEV_ERROR | NVME_DNR; |
888 | 0 | } |
889 | | |
890 | | static inline bool nvme_addr_is_dma(NvmeCtrl *n, hwaddr addr) |
891 | 0 | { |
892 | 0 | return !(nvme_addr_is_cmb(n, addr) || nvme_addr_is_pmr(n, addr)); |
893 | 0 | } |
894 | | |
895 | | static uint16_t nvme_map_prp(NvmeCtrl *n, NvmeSg *sg, uint64_t prp1, |
896 | | uint64_t prp2, uint32_t len) |
897 | 0 | { |
898 | 0 | hwaddr trans_len = n->page_size - (prp1 % n->page_size); |
899 | 0 | trans_len = MIN(len, trans_len); |
900 | 0 | int num_prps = (len >> n->page_bits) + 1; |
901 | 0 | uint16_t status; |
902 | 0 | int ret; |
903 | |
|
904 | 0 | trace_pci_nvme_map_prp(trans_len, len, prp1, prp2, num_prps); |
905 | |
|
906 | 0 | nvme_sg_init(n, sg, nvme_addr_is_dma(n, prp1)); |
907 | |
|
908 | 0 | status = nvme_map_addr(n, sg, prp1, trans_len); |
909 | 0 | if (status) { |
910 | 0 | goto unmap; |
911 | 0 | } |
912 | | |
913 | 0 | len -= trans_len; |
914 | 0 | if (len) { |
915 | 0 | if (len > n->page_size) { |
916 | 0 | g_autofree uint64_t *prp_list = g_new(uint64_t, n->max_prp_ents); |
917 | 0 | uint32_t nents, prp_trans; |
918 | 0 | int i = 0; |
919 | | |
920 | | /* |
921 | | * The first PRP list entry, pointed to by PRP2 may contain offset. |
922 | | * Hence, we need to calculate the number of entries in based on |
923 | | * that offset. |
924 | | */ |
925 | 0 | nents = (n->page_size - (prp2 & (n->page_size - 1))) >> 3; |
926 | 0 | prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t); |
927 | 0 | ret = nvme_addr_read(n, prp2, (void *)prp_list, prp_trans); |
928 | 0 | if (ret) { |
929 | 0 | trace_pci_nvme_err_addr_read(prp2); |
930 | 0 | status = NVME_DATA_TRAS_ERROR; |
931 | 0 | goto unmap; |
932 | 0 | } |
933 | 0 | while (len != 0) { |
934 | 0 | uint64_t prp_ent = le64_to_cpu(prp_list[i]); |
935 | |
|
936 | 0 | if (i == nents - 1 && len > n->page_size) { |
937 | 0 | if (unlikely(prp_ent & (n->page_size - 1))) { |
938 | 0 | trace_pci_nvme_err_invalid_prplist_ent(prp_ent); |
939 | 0 | status = NVME_INVALID_PRP_OFFSET | NVME_DNR; |
940 | 0 | goto unmap; |
941 | 0 | } |
942 | | |
943 | 0 | i = 0; |
944 | 0 | nents = (len + n->page_size - 1) >> n->page_bits; |
945 | 0 | nents = MIN(nents, n->max_prp_ents); |
946 | 0 | prp_trans = nents * sizeof(uint64_t); |
947 | 0 | ret = nvme_addr_read(n, prp_ent, (void *)prp_list, |
948 | 0 | prp_trans); |
949 | 0 | if (ret) { |
950 | 0 | trace_pci_nvme_err_addr_read(prp_ent); |
951 | 0 | status = NVME_DATA_TRAS_ERROR; |
952 | 0 | goto unmap; |
953 | 0 | } |
954 | 0 | prp_ent = le64_to_cpu(prp_list[i]); |
955 | 0 | } |
956 | | |
957 | 0 | if (unlikely(prp_ent & (n->page_size - 1))) { |
958 | 0 | trace_pci_nvme_err_invalid_prplist_ent(prp_ent); |
959 | 0 | status = NVME_INVALID_PRP_OFFSET | NVME_DNR; |
960 | 0 | goto unmap; |
961 | 0 | } |
962 | | |
963 | 0 | trans_len = MIN(len, n->page_size); |
964 | 0 | status = nvme_map_addr(n, sg, prp_ent, trans_len); |
965 | 0 | if (status) { |
966 | 0 | goto unmap; |
967 | 0 | } |
968 | | |
969 | 0 | len -= trans_len; |
970 | 0 | i++; |
971 | 0 | } |
972 | 0 | } else { |
973 | 0 | if (unlikely(prp2 & (n->page_size - 1))) { |
974 | 0 | trace_pci_nvme_err_invalid_prp2_align(prp2); |
975 | 0 | status = NVME_INVALID_PRP_OFFSET | NVME_DNR; |
976 | 0 | goto unmap; |
977 | 0 | } |
978 | 0 | status = nvme_map_addr(n, sg, prp2, len); |
979 | 0 | if (status) { |
980 | 0 | goto unmap; |
981 | 0 | } |
982 | 0 | } |
983 | 0 | } |
984 | | |
985 | 0 | return NVME_SUCCESS; |
986 | | |
987 | 0 | unmap: |
988 | 0 | nvme_sg_unmap(sg); |
989 | 0 | return status; |
990 | 0 | } |
991 | | |
992 | | /* |
993 | | * Map 'nsgld' data descriptors from 'segment'. The function will subtract the |
994 | | * number of bytes mapped in len. |
995 | | */ |
996 | | static uint16_t nvme_map_sgl_data(NvmeCtrl *n, NvmeSg *sg, |
997 | | NvmeSglDescriptor *segment, uint64_t nsgld, |
998 | | size_t *len, NvmeCmd *cmd) |
999 | 0 | { |
1000 | 0 | dma_addr_t addr, trans_len; |
1001 | 0 | uint32_t dlen; |
1002 | 0 | uint16_t status; |
1003 | |
|
1004 | 0 | for (int i = 0; i < nsgld; i++) { |
1005 | 0 | uint8_t type = NVME_SGL_TYPE(segment[i].type); |
1006 | |
|
1007 | 0 | switch (type) { |
1008 | 0 | case NVME_SGL_DESCR_TYPE_DATA_BLOCK: |
1009 | 0 | break; |
1010 | 0 | case NVME_SGL_DESCR_TYPE_SEGMENT: |
1011 | 0 | case NVME_SGL_DESCR_TYPE_LAST_SEGMENT: |
1012 | 0 | return NVME_INVALID_NUM_SGL_DESCRS | NVME_DNR; |
1013 | 0 | default: |
1014 | 0 | return NVME_SGL_DESCR_TYPE_INVALID | NVME_DNR; |
1015 | 0 | } |
1016 | | |
1017 | 0 | dlen = le32_to_cpu(segment[i].len); |
1018 | |
|
1019 | 0 | if (!dlen) { |
1020 | 0 | continue; |
1021 | 0 | } |
1022 | | |
1023 | 0 | if (*len == 0) { |
1024 | | /* |
1025 | | * All data has been mapped, but the SGL contains additional |
1026 | | * segments and/or descriptors. The controller might accept |
1027 | | * ignoring the rest of the SGL. |
1028 | | */ |
1029 | 0 | uint32_t sgls = le32_to_cpu(n->id_ctrl.sgls); |
1030 | 0 | if (sgls & NVME_CTRL_SGLS_EXCESS_LENGTH) { |
1031 | 0 | break; |
1032 | 0 | } |
1033 | | |
1034 | 0 | trace_pci_nvme_err_invalid_sgl_excess_length(dlen); |
1035 | 0 | return NVME_DATA_SGL_LEN_INVALID | NVME_DNR; |
1036 | 0 | } |
1037 | | |
1038 | 0 | trans_len = MIN(*len, dlen); |
1039 | |
|
1040 | 0 | addr = le64_to_cpu(segment[i].addr); |
1041 | |
|
1042 | 0 | if (UINT64_MAX - addr < dlen) { |
1043 | 0 | return NVME_DATA_SGL_LEN_INVALID | NVME_DNR; |
1044 | 0 | } |
1045 | | |
1046 | 0 | status = nvme_map_addr(n, sg, addr, trans_len); |
1047 | 0 | if (status) { |
1048 | 0 | return status; |
1049 | 0 | } |
1050 | | |
1051 | 0 | *len -= trans_len; |
1052 | 0 | } |
1053 | | |
1054 | 0 | return NVME_SUCCESS; |
1055 | 0 | } |
1056 | | |
1057 | | static uint16_t nvme_map_sgl(NvmeCtrl *n, NvmeSg *sg, NvmeSglDescriptor sgl, |
1058 | | size_t len, NvmeCmd *cmd) |
1059 | 0 | { |
1060 | | /* |
1061 | | * Read the segment in chunks of 256 descriptors (one 4k page) to avoid |
1062 | | * dynamically allocating a potentially huge SGL. The spec allows the SGL |
1063 | | * to be larger (as in number of bytes required to describe the SGL |
1064 | | * descriptors and segment chain) than the command transfer size, so it is |
1065 | | * not bounded by MDTS. |
1066 | | */ |
1067 | 0 | #define SEG_CHUNK_SIZE 256 |
1068 | |
|
1069 | 0 | QEMU_UNINITIALIZED NvmeSglDescriptor segment[SEG_CHUNK_SIZE]; |
1070 | 0 | NvmeSglDescriptor *sgld, *last_sgld; |
1071 | 0 | uint64_t nsgld; |
1072 | 0 | uint32_t seg_len; |
1073 | 0 | uint16_t status; |
1074 | 0 | hwaddr addr; |
1075 | 0 | int ret; |
1076 | |
|
1077 | 0 | sgld = &sgl; |
1078 | 0 | addr = le64_to_cpu(sgl.addr); |
1079 | |
|
1080 | 0 | trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl.type), len); |
1081 | |
|
1082 | 0 | nvme_sg_init(n, sg, nvme_addr_is_dma(n, addr)); |
1083 | | |
1084 | | /* |
1085 | | * If the entire transfer can be described with a single data block it can |
1086 | | * be mapped directly. |
1087 | | */ |
1088 | 0 | if (NVME_SGL_TYPE(sgl.type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) { |
1089 | 0 | status = nvme_map_sgl_data(n, sg, sgld, 1, &len, cmd); |
1090 | 0 | if (status) { |
1091 | 0 | goto unmap; |
1092 | 0 | } |
1093 | | |
1094 | 0 | goto out; |
1095 | 0 | } |
1096 | | |
1097 | 0 | for (;;) { |
1098 | 0 | size_t prev_len = len; |
1099 | |
|
1100 | 0 | switch (NVME_SGL_TYPE(sgld->type)) { |
1101 | 0 | case NVME_SGL_DESCR_TYPE_SEGMENT: |
1102 | 0 | case NVME_SGL_DESCR_TYPE_LAST_SEGMENT: |
1103 | 0 | break; |
1104 | 0 | default: |
1105 | 0 | return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR; |
1106 | 0 | } |
1107 | | |
1108 | 0 | seg_len = le32_to_cpu(sgld->len); |
1109 | | |
1110 | | /* check the length of the (Last) Segment descriptor */ |
1111 | 0 | if (!seg_len || seg_len & 0xf) { |
1112 | 0 | return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR; |
1113 | 0 | } |
1114 | | |
1115 | 0 | if (UINT64_MAX - addr < seg_len) { |
1116 | 0 | return NVME_DATA_SGL_LEN_INVALID | NVME_DNR; |
1117 | 0 | } |
1118 | | |
1119 | 0 | nsgld = seg_len / sizeof(NvmeSglDescriptor); |
1120 | |
|
1121 | 0 | while (nsgld > SEG_CHUNK_SIZE) { |
1122 | 0 | if (nvme_addr_read(n, addr, segment, sizeof(segment))) { |
1123 | 0 | trace_pci_nvme_err_addr_read(addr); |
1124 | 0 | status = NVME_DATA_TRAS_ERROR; |
1125 | 0 | goto unmap; |
1126 | 0 | } |
1127 | | |
1128 | 0 | status = nvme_map_sgl_data(n, sg, segment, SEG_CHUNK_SIZE, |
1129 | 0 | &len, cmd); |
1130 | 0 | if (status) { |
1131 | 0 | goto unmap; |
1132 | 0 | } |
1133 | | |
1134 | 0 | nsgld -= SEG_CHUNK_SIZE; |
1135 | 0 | addr += SEG_CHUNK_SIZE * sizeof(NvmeSglDescriptor); |
1136 | 0 | } |
1137 | | |
1138 | 0 | ret = nvme_addr_read(n, addr, segment, nsgld * |
1139 | 0 | sizeof(NvmeSglDescriptor)); |
1140 | 0 | if (ret) { |
1141 | 0 | trace_pci_nvme_err_addr_read(addr); |
1142 | 0 | status = NVME_DATA_TRAS_ERROR; |
1143 | 0 | goto unmap; |
1144 | 0 | } |
1145 | | |
1146 | 0 | last_sgld = &segment[nsgld - 1]; |
1147 | | |
1148 | | /* |
1149 | | * If the segment ends with a Data Block, then we are done. |
1150 | | */ |
1151 | 0 | if (NVME_SGL_TYPE(last_sgld->type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) { |
1152 | 0 | status = nvme_map_sgl_data(n, sg, segment, nsgld, &len, cmd); |
1153 | 0 | if (status) { |
1154 | 0 | goto unmap; |
1155 | 0 | } |
1156 | | |
1157 | 0 | goto out; |
1158 | 0 | } |
1159 | | |
1160 | | /* |
1161 | | * If the last descriptor was not a Data Block, then the current |
1162 | | * segment must not be a Last Segment. |
1163 | | */ |
1164 | 0 | if (NVME_SGL_TYPE(sgld->type) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT) { |
1165 | 0 | status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR; |
1166 | 0 | goto unmap; |
1167 | 0 | } |
1168 | | |
1169 | 0 | sgld = last_sgld; |
1170 | 0 | addr = le64_to_cpu(sgld->addr); |
1171 | | |
1172 | | /* |
1173 | | * Do not map the last descriptor; it will be a Segment or Last Segment |
1174 | | * descriptor and is handled by the next iteration. |
1175 | | */ |
1176 | 0 | status = nvme_map_sgl_data(n, sg, segment, nsgld - 1, &len, cmd); |
1177 | 0 | if (status) { |
1178 | 0 | goto unmap; |
1179 | 0 | } |
1180 | | |
1181 | | /* |
1182 | | * Reject if this segment made no forward progress. The host should |
1183 | | * have skipped linking an empty segment. While not strictly spec |
1184 | | * compliant, allowing this makes it easy for a pathological host to |
1185 | | * create an infinite loop. |
1186 | | */ |
1187 | 0 | if (len == prev_len) { |
1188 | 0 | status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR; |
1189 | 0 | goto unmap; |
1190 | 0 | } |
1191 | 0 | } |
1192 | | |
1193 | 0 | out: |
1194 | | /* if there is any residual left in len, the SGL was too short */ |
1195 | 0 | if (len) { |
1196 | 0 | status = NVME_DATA_SGL_LEN_INVALID | NVME_DNR; |
1197 | 0 | goto unmap; |
1198 | 0 | } |
1199 | | |
1200 | 0 | return NVME_SUCCESS; |
1201 | | |
1202 | 0 | unmap: |
1203 | 0 | nvme_sg_unmap(sg); |
1204 | 0 | return status; |
1205 | 0 | } |
1206 | | |
1207 | | uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len, |
1208 | | NvmeCmd *cmd) |
1209 | 0 | { |
1210 | 0 | uint64_t prp1, prp2; |
1211 | |
|
1212 | 0 | switch (NVME_CMD_FLAGS_PSDT(cmd->flags)) { |
1213 | 0 | case NVME_PSDT_PRP: |
1214 | 0 | prp1 = le64_to_cpu(cmd->dptr.prp1); |
1215 | 0 | prp2 = le64_to_cpu(cmd->dptr.prp2); |
1216 | |
|
1217 | 0 | return nvme_map_prp(n, sg, prp1, prp2, len); |
1218 | 0 | case NVME_PSDT_SGL_MPTR_CONTIGUOUS: |
1219 | 0 | case NVME_PSDT_SGL_MPTR_SGL: |
1220 | 0 | return nvme_map_sgl(n, sg, cmd->dptr.sgl, len, cmd); |
1221 | 0 | default: |
1222 | 0 | return NVME_INVALID_FIELD; |
1223 | 0 | } |
1224 | 0 | } |
1225 | | |
1226 | | static uint16_t nvme_map_mptr(NvmeCtrl *n, NvmeSg *sg, size_t len, |
1227 | | NvmeCmd *cmd) |
1228 | 0 | { |
1229 | 0 | int psdt = NVME_CMD_FLAGS_PSDT(cmd->flags); |
1230 | 0 | hwaddr mptr = le64_to_cpu(cmd->mptr); |
1231 | 0 | uint16_t status; |
1232 | |
|
1233 | 0 | if (psdt == NVME_PSDT_SGL_MPTR_SGL) { |
1234 | 0 | NvmeSglDescriptor sgl; |
1235 | |
|
1236 | 0 | if (nvme_addr_read(n, mptr, &sgl, sizeof(sgl))) { |
1237 | 0 | return NVME_DATA_TRAS_ERROR; |
1238 | 0 | } |
1239 | | |
1240 | 0 | status = nvme_map_sgl(n, sg, sgl, len, cmd); |
1241 | 0 | if (status && (status & 0x7ff) == NVME_DATA_SGL_LEN_INVALID) { |
1242 | 0 | status = NVME_MD_SGL_LEN_INVALID | NVME_DNR; |
1243 | 0 | } |
1244 | |
|
1245 | 0 | return status; |
1246 | 0 | } |
1247 | | |
1248 | 0 | nvme_sg_init(n, sg, nvme_addr_is_dma(n, mptr)); |
1249 | 0 | status = nvme_map_addr(n, sg, mptr, len); |
1250 | 0 | if (status) { |
1251 | 0 | nvme_sg_unmap(sg); |
1252 | 0 | } |
1253 | |
|
1254 | 0 | return status; |
1255 | 0 | } |
1256 | | |
1257 | | static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req) |
1258 | 0 | { |
1259 | 0 | NvmeNamespace *ns = req->ns; |
1260 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
1261 | 0 | bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps); |
1262 | 0 | bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT); |
1263 | 0 | size_t len = nvme_l2b(ns, nlb); |
1264 | 0 | uint16_t status; |
1265 | |
|
1266 | 0 | if (nvme_ns_ext(ns) && |
1267 | 0 | !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) { |
1268 | 0 | NvmeSg sg; |
1269 | |
|
1270 | 0 | len += nvme_m2b(ns, nlb); |
1271 | |
|
1272 | 0 | status = nvme_map_dptr(n, &sg, len, &req->cmd); |
1273 | 0 | if (status) { |
1274 | 0 | return status; |
1275 | 0 | } |
1276 | | |
1277 | 0 | nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA); |
1278 | 0 | nvme_sg_split(&sg, ns, &req->sg, NULL); |
1279 | 0 | nvme_sg_unmap(&sg); |
1280 | |
|
1281 | 0 | return NVME_SUCCESS; |
1282 | 0 | } |
1283 | | |
1284 | 0 | return nvme_map_dptr(n, &req->sg, len, &req->cmd); |
1285 | 0 | } |
1286 | | |
1287 | | static uint16_t nvme_map_mdata(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req) |
1288 | 0 | { |
1289 | 0 | NvmeNamespace *ns = req->ns; |
1290 | 0 | size_t len = nvme_m2b(ns, nlb); |
1291 | 0 | uint16_t status; |
1292 | |
|
1293 | 0 | if (nvme_ns_ext(ns)) { |
1294 | 0 | NvmeSg sg; |
1295 | |
|
1296 | 0 | len += nvme_l2b(ns, nlb); |
1297 | |
|
1298 | 0 | status = nvme_map_dptr(n, &sg, len, &req->cmd); |
1299 | 0 | if (status) { |
1300 | 0 | return status; |
1301 | 0 | } |
1302 | | |
1303 | 0 | nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA); |
1304 | 0 | nvme_sg_split(&sg, ns, NULL, &req->sg); |
1305 | 0 | nvme_sg_unmap(&sg); |
1306 | |
|
1307 | 0 | return NVME_SUCCESS; |
1308 | 0 | } |
1309 | | |
1310 | 0 | return nvme_map_mptr(n, &req->sg, len, &req->cmd); |
1311 | 0 | } |
1312 | | |
1313 | | static uint16_t nvme_tx_interleaved(NvmeCtrl *n, NvmeSg *sg, uint8_t *ptr, |
1314 | | uint32_t len, uint32_t bytes, |
1315 | | int32_t skip_bytes, int64_t offset, |
1316 | | NvmeTxDirection dir) |
1317 | 0 | { |
1318 | 0 | hwaddr addr; |
1319 | 0 | uint32_t trans_len, count = bytes; |
1320 | 0 | bool dma = sg->flags & NVME_SG_DMA; |
1321 | 0 | int64_t sge_len; |
1322 | 0 | int sg_idx = 0; |
1323 | 0 | int ret; |
1324 | |
|
1325 | 0 | assert(sg->flags & NVME_SG_ALLOC); |
1326 | |
|
1327 | 0 | while (len) { |
1328 | 0 | sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len; |
1329 | |
|
1330 | 0 | if (sge_len - offset < 0) { |
1331 | 0 | offset -= sge_len; |
1332 | 0 | sg_idx++; |
1333 | 0 | continue; |
1334 | 0 | } |
1335 | | |
1336 | 0 | if (sge_len == offset) { |
1337 | 0 | offset = 0; |
1338 | 0 | sg_idx++; |
1339 | 0 | continue; |
1340 | 0 | } |
1341 | | |
1342 | 0 | trans_len = MIN(len, count); |
1343 | 0 | trans_len = MIN(trans_len, sge_len - offset); |
1344 | |
|
1345 | 0 | if (dma) { |
1346 | 0 | addr = sg->qsg.sg[sg_idx].base + offset; |
1347 | 0 | } else { |
1348 | 0 | addr = (hwaddr)(uintptr_t)sg->iov.iov[sg_idx].iov_base + offset; |
1349 | 0 | } |
1350 | |
|
1351 | 0 | if (dir == NVME_TX_DIRECTION_TO_DEVICE) { |
1352 | 0 | ret = nvme_addr_read(n, addr, ptr, trans_len); |
1353 | 0 | } else { |
1354 | 0 | ret = nvme_addr_write(n, addr, ptr, trans_len); |
1355 | 0 | } |
1356 | |
|
1357 | 0 | if (ret) { |
1358 | 0 | return NVME_DATA_TRAS_ERROR; |
1359 | 0 | } |
1360 | | |
1361 | 0 | ptr += trans_len; |
1362 | 0 | len -= trans_len; |
1363 | 0 | count -= trans_len; |
1364 | 0 | offset += trans_len; |
1365 | |
|
1366 | 0 | if (count == 0) { |
1367 | 0 | count = bytes; |
1368 | 0 | offset += skip_bytes; |
1369 | 0 | } |
1370 | 0 | } |
1371 | | |
1372 | 0 | return NVME_SUCCESS; |
1373 | 0 | } |
1374 | | |
1375 | | static uint16_t nvme_tx(NvmeCtrl *n, NvmeSg *sg, void *ptr, uint32_t len, |
1376 | | NvmeTxDirection dir) |
1377 | 0 | { |
1378 | 0 | assert(sg->flags & NVME_SG_ALLOC); |
1379 | |
|
1380 | 0 | if (sg->flags & NVME_SG_DMA) { |
1381 | 0 | const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED; |
1382 | 0 | dma_addr_t residual; |
1383 | |
|
1384 | 0 | if (dir == NVME_TX_DIRECTION_TO_DEVICE) { |
1385 | 0 | dma_buf_write(ptr, len, &residual, &sg->qsg, attrs); |
1386 | 0 | } else { |
1387 | 0 | dma_buf_read(ptr, len, &residual, &sg->qsg, attrs); |
1388 | 0 | } |
1389 | |
|
1390 | 0 | if (unlikely(residual)) { |
1391 | 0 | trace_pci_nvme_err_invalid_dma(); |
1392 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
1393 | 0 | } |
1394 | 0 | } else { |
1395 | 0 | size_t bytes; |
1396 | |
|
1397 | 0 | if (dir == NVME_TX_DIRECTION_TO_DEVICE) { |
1398 | 0 | bytes = qemu_iovec_to_buf(&sg->iov, 0, ptr, len); |
1399 | 0 | } else { |
1400 | 0 | bytes = qemu_iovec_from_buf(&sg->iov, 0, ptr, len); |
1401 | 0 | } |
1402 | |
|
1403 | 0 | if (unlikely(bytes != len)) { |
1404 | 0 | trace_pci_nvme_err_invalid_dma(); |
1405 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
1406 | 0 | } |
1407 | 0 | } |
1408 | | |
1409 | 0 | return NVME_SUCCESS; |
1410 | 0 | } |
1411 | | |
1412 | | static inline uint16_t nvme_c2h(NvmeCtrl *n, void *ptr, uint32_t len, |
1413 | | NvmeRequest *req) |
1414 | 0 | { |
1415 | 0 | uint16_t status; |
1416 | |
|
1417 | 0 | status = nvme_map_dptr(n, &req->sg, len, &req->cmd); |
1418 | 0 | if (status) { |
1419 | 0 | return status; |
1420 | 0 | } |
1421 | | |
1422 | 0 | return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_FROM_DEVICE); |
1423 | 0 | } |
1424 | | |
1425 | | static inline uint16_t nvme_h2c(NvmeCtrl *n, void *ptr, uint32_t len, |
1426 | | NvmeRequest *req) |
1427 | 0 | { |
1428 | 0 | uint16_t status; |
1429 | |
|
1430 | 0 | status = nvme_map_dptr(n, &req->sg, len, &req->cmd); |
1431 | 0 | if (status) { |
1432 | 0 | return status; |
1433 | 0 | } |
1434 | | |
1435 | 0 | return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_TO_DEVICE); |
1436 | 0 | } |
1437 | | |
1438 | | uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len, |
1439 | | NvmeTxDirection dir, NvmeRequest *req) |
1440 | 0 | { |
1441 | 0 | NvmeNamespace *ns = req->ns; |
1442 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
1443 | 0 | bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps); |
1444 | 0 | bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT); |
1445 | |
|
1446 | 0 | if (nvme_ns_ext(ns) && |
1447 | 0 | !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) { |
1448 | 0 | return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbasz, |
1449 | 0 | ns->lbaf.ms, 0, dir); |
1450 | 0 | } |
1451 | | |
1452 | 0 | return nvme_tx(n, &req->sg, ptr, len, dir); |
1453 | 0 | } |
1454 | | |
1455 | | uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len, |
1456 | | NvmeTxDirection dir, NvmeRequest *req) |
1457 | 0 | { |
1458 | 0 | NvmeNamespace *ns = req->ns; |
1459 | 0 | uint16_t status; |
1460 | |
|
1461 | 0 | if (nvme_ns_ext(ns)) { |
1462 | 0 | return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbaf.ms, |
1463 | 0 | ns->lbasz, ns->lbasz, dir); |
1464 | 0 | } |
1465 | | |
1466 | 0 | nvme_sg_unmap(&req->sg); |
1467 | |
|
1468 | 0 | status = nvme_map_mptr(n, &req->sg, len, &req->cmd); |
1469 | 0 | if (status) { |
1470 | 0 | return status; |
1471 | 0 | } |
1472 | | |
1473 | 0 | return nvme_tx(n, &req->sg, ptr, len, dir); |
1474 | 0 | } |
1475 | | |
1476 | | static inline void nvme_blk_read(BlockBackend *blk, int64_t offset, |
1477 | | uint32_t align, BlockCompletionFunc *cb, |
1478 | | NvmeRequest *req) |
1479 | 0 | { |
1480 | 0 | assert(req->sg.flags & NVME_SG_ALLOC); |
1481 | |
|
1482 | 0 | if (req->sg.flags & NVME_SG_DMA) { |
1483 | 0 | req->aiocb = dma_blk_read(blk, &req->sg.qsg, offset, align, cb, req); |
1484 | 0 | } else { |
1485 | 0 | req->aiocb = blk_aio_preadv(blk, offset, &req->sg.iov, 0, cb, req); |
1486 | 0 | } |
1487 | 0 | } |
1488 | | |
1489 | | static inline void nvme_blk_write(BlockBackend *blk, int64_t offset, |
1490 | | uint32_t align, BlockCompletionFunc *cb, |
1491 | | NvmeRequest *req) |
1492 | 0 | { |
1493 | 0 | assert(req->sg.flags & NVME_SG_ALLOC); |
1494 | |
|
1495 | 0 | if (req->sg.flags & NVME_SG_DMA) { |
1496 | 0 | req->aiocb = dma_blk_write(blk, &req->sg.qsg, offset, align, cb, req); |
1497 | 0 | } else { |
1498 | 0 | req->aiocb = blk_aio_pwritev(blk, offset, &req->sg.iov, 0, cb, req); |
1499 | 0 | } |
1500 | 0 | } |
1501 | | |
1502 | | static void nvme_update_cq_eventidx(const NvmeCQueue *cq) |
1503 | 0 | { |
1504 | 0 | trace_pci_nvme_update_cq_eventidx(cq->cqid, cq->head); |
1505 | |
|
1506 | 0 | stl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->ei_addr, cq->head, |
1507 | 0 | MEMTXATTRS_UNSPECIFIED); |
1508 | 0 | } |
1509 | | |
1510 | | static void nvme_update_cq_head(NvmeCQueue *cq) |
1511 | 0 | { |
1512 | 0 | ldl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->db_addr, &cq->head, |
1513 | 0 | MEMTXATTRS_UNSPECIFIED); |
1514 | |
|
1515 | 0 | trace_pci_nvme_update_cq_head(cq->cqid, cq->head); |
1516 | 0 | } |
1517 | | |
1518 | | static void nvme_post_cqes(void *opaque) |
1519 | 0 | { |
1520 | 0 | NvmeCQueue *cq = opaque; |
1521 | 0 | NvmeCtrl *n = cq->ctrl; |
1522 | 0 | NvmeRequest *req, *next; |
1523 | 0 | bool pending = cq->head != cq->tail; |
1524 | 0 | int ret; |
1525 | |
|
1526 | 0 | QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) { |
1527 | 0 | NvmeSQueue *sq; |
1528 | 0 | hwaddr addr; |
1529 | |
|
1530 | 0 | if (n->dbbuf_enabled) { |
1531 | 0 | nvme_update_cq_eventidx(cq); |
1532 | 0 | nvme_update_cq_head(cq); |
1533 | 0 | } |
1534 | |
|
1535 | 0 | if (nvme_cq_full(cq)) { |
1536 | 0 | break; |
1537 | 0 | } |
1538 | | |
1539 | | /* |
1540 | | * Here we take the following fields from NvmeRequest structure |
1541 | | * and write cqe to the guest RAM based on them: |
1542 | | * - req->sq |
1543 | | * - req->status |
1544 | | * - req->cqe |
1545 | | * |
1546 | | * If you change this code and more fields from NvmeRequest are |
1547 | | * used, please make sure that you have handled this in: |
1548 | | * nvme_vmstate_request and nvme_ctrl_pre_save(). |
1549 | | */ |
1550 | | |
1551 | 0 | sq = req->sq; |
1552 | 0 | req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase); |
1553 | 0 | req->cqe.sq_head = cpu_to_le16(sq->head); |
1554 | 0 | addr = cq->dma_addr + (cq->tail << NVME_CQES); |
1555 | 0 | ret = pci_dma_write(PCI_DEVICE(n), addr, (void *)&req->cqe, |
1556 | 0 | sizeof(req->cqe)); |
1557 | 0 | if (ret) { |
1558 | 0 | trace_pci_nvme_err_addr_write(addr); |
1559 | 0 | trace_pci_nvme_err_cfs(); |
1560 | 0 | stl_le_p(&n->bar.csts, NVME_CSTS_FAILED); |
1561 | 0 | break; |
1562 | 0 | } |
1563 | | |
1564 | 0 | QTAILQ_REMOVE(&cq->req_list, req, entry); |
1565 | |
|
1566 | 0 | nvme_inc_cq_tail(cq); |
1567 | |
|
1568 | 0 | if (QTAILQ_EMPTY(&sq->req_list) && !nvme_sq_empty(sq)) { |
1569 | 0 | qemu_bh_schedule(sq->bh); |
1570 | 0 | } |
1571 | |
|
1572 | 0 | QTAILQ_INSERT_TAIL(&sq->req_list, req, entry); |
1573 | 0 | } |
1574 | 0 | if (cq->tail != cq->head) { |
1575 | 0 | if (cq->irq_enabled && !pending) { |
1576 | 0 | n->cq_pending++; |
1577 | 0 | } |
1578 | |
|
1579 | 0 | nvme_irq_assert(n, cq); |
1580 | 0 | } |
1581 | 0 | } |
1582 | | |
1583 | | static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req) |
1584 | 0 | { |
1585 | 0 | assert(cq->cqid == req->sq->cqid); |
1586 | 0 | trace_pci_nvme_enqueue_req_completion(nvme_cid(req), cq->cqid, |
1587 | 0 | le32_to_cpu(req->cqe.result), |
1588 | 0 | le32_to_cpu(req->cqe.dw1), |
1589 | 0 | req->status); |
1590 | |
|
1591 | 0 | if (req->status) { |
1592 | 0 | trace_pci_nvme_err_req_status(nvme_cid(req), nvme_nsid(req->ns), |
1593 | 0 | req->status, req->cmd.opcode); |
1594 | 0 | } |
1595 | |
|
1596 | 0 | nvme_sg_unmap(&req->sg); |
1597 | |
|
1598 | 0 | QTAILQ_REMOVE(&req->sq->out_req_list, req, entry); |
1599 | 0 | QTAILQ_INSERT_TAIL(&cq->req_list, req, entry); |
1600 | |
|
1601 | 0 | qemu_bh_schedule(cq->bh); |
1602 | 0 | } |
1603 | | |
1604 | | static void nvme_process_aers(void *opaque) |
1605 | 0 | { |
1606 | 0 | NvmeCtrl *n = opaque; |
1607 | 0 | NvmeAsyncEvent *event, *next; |
1608 | |
|
1609 | 0 | trace_pci_nvme_process_aers(n->aer_queued); |
1610 | |
|
1611 | 0 | QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) { |
1612 | 0 | NvmeRequest *req; |
1613 | 0 | NvmeAerResult *result; |
1614 | | |
1615 | | /* can't post cqe if there is nothing to complete */ |
1616 | 0 | if (!n->outstanding_aers) { |
1617 | 0 | trace_pci_nvme_no_outstanding_aers(); |
1618 | 0 | break; |
1619 | 0 | } |
1620 | | |
1621 | | /* ignore if masked (cqe posted, but event not cleared) */ |
1622 | 0 | if (n->aer_mask & (1 << event->result.event_type)) { |
1623 | 0 | trace_pci_nvme_aer_masked(event->result.event_type, n->aer_mask); |
1624 | 0 | continue; |
1625 | 0 | } |
1626 | | |
1627 | 0 | QTAILQ_REMOVE(&n->aer_queue, event, entry); |
1628 | 0 | n->aer_queued--; |
1629 | |
|
1630 | 0 | n->aer_mask |= 1 << event->result.event_type; |
1631 | 0 | n->outstanding_aers--; |
1632 | |
|
1633 | 0 | req = n->aer_reqs[n->outstanding_aers]; |
1634 | |
|
1635 | 0 | result = (NvmeAerResult *) &req->cqe.result; |
1636 | 0 | result->event_type = event->result.event_type; |
1637 | 0 | result->event_info = event->result.event_info; |
1638 | 0 | result->log_page = event->result.log_page; |
1639 | 0 | g_free(event); |
1640 | |
|
1641 | 0 | trace_pci_nvme_aer_post_cqe(result->event_type, result->event_info, |
1642 | 0 | result->log_page); |
1643 | |
|
1644 | 0 | nvme_enqueue_req_completion(&n->admin_cq, req); |
1645 | 0 | } |
1646 | 0 | } |
1647 | | |
1648 | | static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type, |
1649 | | uint8_t event_info, uint8_t log_page) |
1650 | 0 | { |
1651 | 0 | NvmeAsyncEvent *event; |
1652 | |
|
1653 | 0 | trace_pci_nvme_enqueue_event(event_type, event_info, log_page); |
1654 | |
|
1655 | 0 | if (n->aer_queued == n->params.aer_max_queued) { |
1656 | 0 | trace_pci_nvme_enqueue_event_noqueue(n->aer_queued); |
1657 | 0 | return; |
1658 | 0 | } |
1659 | | |
1660 | 0 | event = g_new(NvmeAsyncEvent, 1); |
1661 | 0 | event->result = (NvmeAerResult) { |
1662 | 0 | .event_type = event_type, |
1663 | 0 | .event_info = event_info, |
1664 | 0 | .log_page = log_page, |
1665 | 0 | }; |
1666 | |
|
1667 | 0 | QTAILQ_INSERT_TAIL(&n->aer_queue, event, entry); |
1668 | 0 | n->aer_queued++; |
1669 | |
|
1670 | 0 | nvme_process_aers(n); |
1671 | 0 | } |
1672 | | |
1673 | | static void nvme_smart_event(NvmeCtrl *n, uint8_t event) |
1674 | 0 | { |
1675 | 0 | uint8_t aer_info; |
1676 | | |
1677 | | /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */ |
1678 | 0 | if (!(NVME_AEC_SMART(n->features.async_config) & event)) { |
1679 | 0 | return; |
1680 | 0 | } |
1681 | | |
1682 | 0 | switch (event) { |
1683 | 0 | case NVME_SMART_SPARE: |
1684 | 0 | aer_info = NVME_AER_INFO_SMART_SPARE_THRESH; |
1685 | 0 | break; |
1686 | 0 | case NVME_SMART_TEMPERATURE: |
1687 | 0 | aer_info = NVME_AER_INFO_SMART_TEMP_THRESH; |
1688 | 0 | break; |
1689 | 0 | case NVME_SMART_RELIABILITY: |
1690 | 0 | case NVME_SMART_MEDIA_READ_ONLY: |
1691 | 0 | case NVME_SMART_FAILED_VOLATILE_MEDIA: |
1692 | 0 | case NVME_SMART_PMR_UNRELIABLE: |
1693 | 0 | aer_info = NVME_AER_INFO_SMART_RELIABILITY; |
1694 | 0 | break; |
1695 | 0 | default: |
1696 | 0 | return; |
1697 | 0 | } |
1698 | | |
1699 | 0 | nvme_enqueue_event(n, NVME_AER_TYPE_SMART, aer_info, NVME_LOG_SMART_INFO); |
1700 | 0 | } |
1701 | | |
1702 | | static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type) |
1703 | 0 | { |
1704 | 0 | NvmeAsyncEvent *event, *next; |
1705 | |
|
1706 | 0 | n->aer_mask &= ~(1 << event_type); |
1707 | |
|
1708 | 0 | QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) { |
1709 | 0 | if (event->result.event_type == event_type) { |
1710 | 0 | QTAILQ_REMOVE(&n->aer_queue, event, entry); |
1711 | 0 | n->aer_queued--; |
1712 | 0 | g_free(event); |
1713 | 0 | } |
1714 | 0 | } |
1715 | 0 | } |
1716 | | |
1717 | | static inline uint16_t nvme_check_mdts(NvmeCtrl *n, size_t len) |
1718 | 0 | { |
1719 | 0 | uint8_t mdts = n->params.mdts; |
1720 | |
|
1721 | 0 | if (mdts && len > n->page_size << mdts) { |
1722 | 0 | trace_pci_nvme_err_mdts(len); |
1723 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
1724 | 0 | } |
1725 | | |
1726 | 0 | return NVME_SUCCESS; |
1727 | 0 | } |
1728 | | |
1729 | | static inline uint16_t nvme_check_bounds(NvmeNamespace *ns, uint64_t slba, |
1730 | | uint32_t nlb) |
1731 | 0 | { |
1732 | 0 | uint64_t nsze = le64_to_cpu(ns->id_ns.nsze); |
1733 | |
|
1734 | 0 | if (unlikely(UINT64_MAX - slba < nlb || slba + nlb > nsze)) { |
1735 | 0 | trace_pci_nvme_err_invalid_lba_range(slba, nlb, nsze); |
1736 | 0 | return NVME_LBA_RANGE | NVME_DNR; |
1737 | 0 | } |
1738 | | |
1739 | 0 | return NVME_SUCCESS; |
1740 | 0 | } |
1741 | | |
1742 | | static int nvme_block_status_all(NvmeNamespace *ns, uint64_t slba, |
1743 | | uint32_t nlb, int flags) |
1744 | 0 | { |
1745 | 0 | BlockDriverState *bs = blk_bs(ns->blkconf.blk); |
1746 | |
|
1747 | 0 | int64_t pnum = 0, bytes = nvme_l2b(ns, nlb); |
1748 | 0 | int64_t offset = nvme_l2b(ns, slba); |
1749 | 0 | int ret; |
1750 | | |
1751 | | /* |
1752 | | * `pnum` holds the number of bytes after offset that shares the same |
1753 | | * allocation status as the byte at offset. If `pnum` is different from |
1754 | | * `bytes`, we should check the allocation status of the next range and |
1755 | | * continue this until all bytes have been checked. |
1756 | | */ |
1757 | 0 | do { |
1758 | 0 | bytes -= pnum; |
1759 | |
|
1760 | 0 | ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL); |
1761 | 0 | if (ret < 0) { |
1762 | 0 | return ret; |
1763 | 0 | } |
1764 | | |
1765 | | |
1766 | 0 | trace_pci_nvme_block_status(offset, bytes, pnum, ret, |
1767 | 0 | !!(ret & BDRV_BLOCK_ZERO)); |
1768 | |
|
1769 | 0 | if (!(ret & flags)) { |
1770 | 0 | return 1; |
1771 | 0 | } |
1772 | | |
1773 | 0 | offset += pnum; |
1774 | 0 | } while (pnum != bytes); |
1775 | | |
1776 | 0 | return 0; |
1777 | 0 | } |
1778 | | |
1779 | | static uint16_t nvme_check_dulbe(NvmeNamespace *ns, uint64_t slba, |
1780 | | uint32_t nlb) |
1781 | 0 | { |
1782 | 0 | int ret; |
1783 | 0 | Error *err = NULL; |
1784 | |
|
1785 | 0 | ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_DATA); |
1786 | 0 | if (ret) { |
1787 | 0 | if (ret < 0) { |
1788 | 0 | error_setg_errno(&err, -ret, "unable to get block status"); |
1789 | 0 | error_report_err(err); |
1790 | |
|
1791 | 0 | return NVME_INTERNAL_DEV_ERROR; |
1792 | 0 | } |
1793 | | |
1794 | 0 | return NVME_DULB; |
1795 | 0 | } |
1796 | | |
1797 | 0 | return NVME_SUCCESS; |
1798 | 0 | } |
1799 | | |
1800 | | static inline uint32_t nvme_zone_idx(NvmeNamespace *ns, uint64_t slba) |
1801 | 0 | { |
1802 | 0 | return ns->zone_size_log2 > 0 ? slba >> ns->zone_size_log2 : |
1803 | 0 | slba / ns->zone_size; |
1804 | 0 | } |
1805 | | |
1806 | | static inline NvmeZone *nvme_get_zone_by_slba(NvmeNamespace *ns, uint64_t slba) |
1807 | 0 | { |
1808 | 0 | uint32_t zone_idx = nvme_zone_idx(ns, slba); |
1809 | |
|
1810 | 0 | if (zone_idx >= ns->num_zones) { |
1811 | 0 | return NULL; |
1812 | 0 | } |
1813 | | |
1814 | 0 | return &ns->zone_array[zone_idx]; |
1815 | 0 | } |
1816 | | |
1817 | | static uint16_t nvme_check_zone_state_for_write(NvmeZone *zone) |
1818 | 0 | { |
1819 | 0 | uint64_t zslba = zone->d.zslba; |
1820 | |
|
1821 | 0 | switch (nvme_get_zone_state(zone)) { |
1822 | 0 | case NVME_ZONE_STATE_EMPTY: |
1823 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
1824 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
1825 | 0 | case NVME_ZONE_STATE_CLOSED: |
1826 | 0 | return NVME_SUCCESS; |
1827 | 0 | case NVME_ZONE_STATE_FULL: |
1828 | 0 | trace_pci_nvme_err_zone_is_full(zslba); |
1829 | 0 | return NVME_ZONE_FULL; |
1830 | 0 | case NVME_ZONE_STATE_OFFLINE: |
1831 | 0 | trace_pci_nvme_err_zone_is_offline(zslba); |
1832 | 0 | return NVME_ZONE_OFFLINE; |
1833 | 0 | case NVME_ZONE_STATE_READ_ONLY: |
1834 | 0 | trace_pci_nvme_err_zone_is_read_only(zslba); |
1835 | 0 | return NVME_ZONE_READ_ONLY; |
1836 | 0 | default: |
1837 | 0 | g_assert_not_reached(); |
1838 | 0 | } |
1839 | | |
1840 | 0 | return NVME_INTERNAL_DEV_ERROR; |
1841 | 0 | } |
1842 | | |
1843 | | static uint16_t nvme_check_zone_write(NvmeNamespace *ns, NvmeZone *zone, |
1844 | | uint64_t slba, uint32_t nlb) |
1845 | 0 | { |
1846 | 0 | uint64_t zcap = nvme_zone_wr_boundary(zone); |
1847 | 0 | uint16_t status; |
1848 | |
|
1849 | 0 | status = nvme_check_zone_state_for_write(zone); |
1850 | 0 | if (status) { |
1851 | 0 | return status; |
1852 | 0 | } |
1853 | | |
1854 | 0 | if (zone->d.za & NVME_ZA_ZRWA_VALID) { |
1855 | 0 | uint64_t ezrwa = zone->w_ptr + 2 * ns->zns.zrwas; |
1856 | |
|
1857 | 0 | if (slba < zone->w_ptr || slba + nlb > ezrwa) { |
1858 | 0 | trace_pci_nvme_err_zone_invalid_write(slba, zone->w_ptr); |
1859 | 0 | return NVME_ZONE_INVALID_WRITE; |
1860 | 0 | } |
1861 | 0 | } else { |
1862 | 0 | if (unlikely(slba != zone->w_ptr)) { |
1863 | 0 | trace_pci_nvme_err_write_not_at_wp(slba, zone->d.zslba, |
1864 | 0 | zone->w_ptr); |
1865 | 0 | return NVME_ZONE_INVALID_WRITE; |
1866 | 0 | } |
1867 | 0 | } |
1868 | | |
1869 | 0 | if (unlikely((slba + nlb) > zcap)) { |
1870 | 0 | trace_pci_nvme_err_zone_boundary(slba, nlb, zcap); |
1871 | 0 | return NVME_ZONE_BOUNDARY_ERROR; |
1872 | 0 | } |
1873 | | |
1874 | 0 | return NVME_SUCCESS; |
1875 | 0 | } |
1876 | | |
1877 | | static uint16_t nvme_check_zone_state_for_read(NvmeZone *zone) |
1878 | 0 | { |
1879 | 0 | switch (nvme_get_zone_state(zone)) { |
1880 | 0 | case NVME_ZONE_STATE_EMPTY: |
1881 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
1882 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
1883 | 0 | case NVME_ZONE_STATE_FULL: |
1884 | 0 | case NVME_ZONE_STATE_CLOSED: |
1885 | 0 | case NVME_ZONE_STATE_READ_ONLY: |
1886 | 0 | return NVME_SUCCESS; |
1887 | 0 | case NVME_ZONE_STATE_OFFLINE: |
1888 | 0 | trace_pci_nvme_err_zone_is_offline(zone->d.zslba); |
1889 | 0 | return NVME_ZONE_OFFLINE; |
1890 | 0 | default: |
1891 | 0 | g_assert_not_reached(); |
1892 | 0 | } |
1893 | | |
1894 | 0 | return NVME_INTERNAL_DEV_ERROR; |
1895 | 0 | } |
1896 | | |
1897 | | static uint16_t nvme_check_zone_read(NvmeNamespace *ns, uint64_t slba, |
1898 | | uint32_t nlb) |
1899 | 0 | { |
1900 | 0 | NvmeZone *zone; |
1901 | 0 | uint64_t bndry, end; |
1902 | 0 | uint16_t status; |
1903 | |
|
1904 | 0 | zone = nvme_get_zone_by_slba(ns, slba); |
1905 | 0 | assert(zone); |
1906 | |
|
1907 | 0 | bndry = nvme_zone_rd_boundary(ns, zone); |
1908 | 0 | end = slba + nlb; |
1909 | |
|
1910 | 0 | status = nvme_check_zone_state_for_read(zone); |
1911 | 0 | if (status) { |
1912 | 0 | ; |
1913 | 0 | } else if (unlikely(end > bndry)) { |
1914 | 0 | if (!ns->params.cross_zone_read) { |
1915 | 0 | status = NVME_ZONE_BOUNDARY_ERROR; |
1916 | 0 | } else { |
1917 | | /* |
1918 | | * Read across zone boundary - check that all subsequent |
1919 | | * zones that are being read have an appropriate state. |
1920 | | */ |
1921 | 0 | do { |
1922 | 0 | zone++; |
1923 | 0 | status = nvme_check_zone_state_for_read(zone); |
1924 | 0 | if (status) { |
1925 | 0 | break; |
1926 | 0 | } |
1927 | 0 | } while (end > nvme_zone_rd_boundary(ns, zone)); |
1928 | 0 | } |
1929 | 0 | } |
1930 | |
|
1931 | 0 | return status; |
1932 | 0 | } |
1933 | | |
1934 | | static uint16_t nvme_zrm_finish(NvmeNamespace *ns, NvmeZone *zone) |
1935 | 0 | { |
1936 | 0 | switch (nvme_get_zone_state(zone)) { |
1937 | 0 | case NVME_ZONE_STATE_FULL: |
1938 | 0 | return NVME_SUCCESS; |
1939 | | |
1940 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
1941 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
1942 | 0 | nvme_aor_dec_open(ns); |
1943 | | /* fallthrough */ |
1944 | 0 | case NVME_ZONE_STATE_CLOSED: |
1945 | 0 | nvme_aor_dec_active(ns); |
1946 | |
|
1947 | 0 | if (zone->d.za & NVME_ZA_ZRWA_VALID) { |
1948 | 0 | zone->d.za &= ~NVME_ZA_ZRWA_VALID; |
1949 | 0 | if (ns->params.numzrwa) { |
1950 | 0 | ns->zns.numzrwa++; |
1951 | 0 | } |
1952 | 0 | } |
1953 | | |
1954 | | /* fallthrough */ |
1955 | 0 | case NVME_ZONE_STATE_EMPTY: |
1956 | 0 | nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_FULL); |
1957 | 0 | return NVME_SUCCESS; |
1958 | | |
1959 | 0 | default: |
1960 | 0 | return NVME_ZONE_INVAL_TRANSITION; |
1961 | 0 | } |
1962 | 0 | } |
1963 | | |
1964 | | static uint16_t nvme_zrm_close(NvmeNamespace *ns, NvmeZone *zone) |
1965 | 0 | { |
1966 | 0 | switch (nvme_get_zone_state(zone)) { |
1967 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
1968 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
1969 | 0 | nvme_aor_dec_open(ns); |
1970 | 0 | nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED); |
1971 | | /* fall through */ |
1972 | 0 | case NVME_ZONE_STATE_CLOSED: |
1973 | 0 | return NVME_SUCCESS; |
1974 | | |
1975 | 0 | default: |
1976 | 0 | return NVME_ZONE_INVAL_TRANSITION; |
1977 | 0 | } |
1978 | 0 | } |
1979 | | |
1980 | | static uint16_t nvme_zrm_reset(NvmeNamespace *ns, NvmeZone *zone) |
1981 | 0 | { |
1982 | 0 | switch (nvme_get_zone_state(zone)) { |
1983 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
1984 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
1985 | 0 | nvme_aor_dec_open(ns); |
1986 | | /* fallthrough */ |
1987 | 0 | case NVME_ZONE_STATE_CLOSED: |
1988 | 0 | nvme_aor_dec_active(ns); |
1989 | |
|
1990 | 0 | if (zone->d.za & NVME_ZA_ZRWA_VALID) { |
1991 | 0 | if (ns->params.numzrwa) { |
1992 | 0 | ns->zns.numzrwa++; |
1993 | 0 | } |
1994 | 0 | } |
1995 | | |
1996 | | /* fallthrough */ |
1997 | 0 | case NVME_ZONE_STATE_FULL: |
1998 | 0 | zone->w_ptr = zone->d.zslba; |
1999 | 0 | zone->d.wp = zone->w_ptr; |
2000 | 0 | nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EMPTY); |
2001 | | /* fallthrough */ |
2002 | 0 | case NVME_ZONE_STATE_EMPTY: |
2003 | 0 | return NVME_SUCCESS; |
2004 | | |
2005 | 0 | default: |
2006 | 0 | return NVME_ZONE_INVAL_TRANSITION; |
2007 | 0 | } |
2008 | 0 | } |
2009 | | |
2010 | | static void nvme_zrm_auto_transition_zone(NvmeNamespace *ns) |
2011 | 0 | { |
2012 | 0 | NvmeZone *zone; |
2013 | |
|
2014 | 0 | if (ns->params.max_open_zones && |
2015 | 0 | ns->nr_open_zones == ns->params.max_open_zones) { |
2016 | 0 | zone = QTAILQ_FIRST(&ns->imp_open_zones); |
2017 | 0 | if (zone) { |
2018 | | /* |
2019 | | * Automatically close this implicitly open zone. |
2020 | | */ |
2021 | 0 | QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry); |
2022 | 0 | nvme_zrm_close(ns, zone); |
2023 | 0 | } |
2024 | 0 | } |
2025 | 0 | } |
2026 | | |
2027 | | enum { |
2028 | | NVME_ZRM_AUTO = 1 << 0, |
2029 | | NVME_ZRM_ZRWA = 1 << 1, |
2030 | | }; |
2031 | | |
2032 | | static uint16_t nvme_zrm_open_flags(NvmeCtrl *n, NvmeNamespace *ns, |
2033 | | NvmeZone *zone, int flags) |
2034 | 0 | { |
2035 | 0 | int act = 0; |
2036 | 0 | uint16_t status; |
2037 | |
|
2038 | 0 | switch (nvme_get_zone_state(zone)) { |
2039 | 0 | case NVME_ZONE_STATE_EMPTY: |
2040 | 0 | act = 1; |
2041 | | |
2042 | | /* fallthrough */ |
2043 | |
|
2044 | 0 | case NVME_ZONE_STATE_CLOSED: |
2045 | 0 | if (n->params.auto_transition_zones) { |
2046 | 0 | nvme_zrm_auto_transition_zone(ns); |
2047 | 0 | } |
2048 | 0 | status = nvme_zns_check_resources(ns, act, 1, |
2049 | 0 | (flags & NVME_ZRM_ZRWA) ? 1 : 0); |
2050 | 0 | if (status) { |
2051 | 0 | return status; |
2052 | 0 | } |
2053 | | |
2054 | 0 | if (act) { |
2055 | 0 | nvme_aor_inc_active(ns); |
2056 | 0 | } |
2057 | |
|
2058 | 0 | nvme_aor_inc_open(ns); |
2059 | |
|
2060 | 0 | if (flags & NVME_ZRM_AUTO) { |
2061 | 0 | nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_IMPLICITLY_OPEN); |
2062 | 0 | return NVME_SUCCESS; |
2063 | 0 | } |
2064 | | |
2065 | | /* fallthrough */ |
2066 | | |
2067 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
2068 | 0 | if (flags & NVME_ZRM_AUTO) { |
2069 | 0 | return NVME_SUCCESS; |
2070 | 0 | } |
2071 | | |
2072 | 0 | nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EXPLICITLY_OPEN); |
2073 | | |
2074 | | /* fallthrough */ |
2075 | |
|
2076 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
2077 | 0 | if (flags & NVME_ZRM_ZRWA) { |
2078 | 0 | ns->zns.numzrwa--; |
2079 | |
|
2080 | 0 | zone->d.za |= NVME_ZA_ZRWA_VALID; |
2081 | 0 | } |
2082 | |
|
2083 | 0 | return NVME_SUCCESS; |
2084 | | |
2085 | 0 | default: |
2086 | 0 | return NVME_ZONE_INVAL_TRANSITION; |
2087 | 0 | } |
2088 | 0 | } |
2089 | | |
2090 | | static inline uint16_t nvme_zrm_auto(NvmeCtrl *n, NvmeNamespace *ns, |
2091 | | NvmeZone *zone) |
2092 | 0 | { |
2093 | 0 | return nvme_zrm_open_flags(n, ns, zone, NVME_ZRM_AUTO); |
2094 | 0 | } |
2095 | | |
2096 | | static void nvme_advance_zone_wp(NvmeNamespace *ns, NvmeZone *zone, |
2097 | | uint32_t nlb) |
2098 | 0 | { |
2099 | 0 | zone->d.wp += nlb; |
2100 | |
|
2101 | 0 | if (zone->d.wp == nvme_zone_wr_boundary(zone)) { |
2102 | 0 | nvme_zrm_finish(ns, zone); |
2103 | 0 | } |
2104 | 0 | } |
2105 | | |
2106 | | static void nvme_zoned_zrwa_implicit_flush(NvmeNamespace *ns, NvmeZone *zone, |
2107 | | uint32_t nlbc) |
2108 | 0 | { |
2109 | 0 | uint16_t nzrwafgs = DIV_ROUND_UP(nlbc, ns->zns.zrwafg); |
2110 | |
|
2111 | 0 | nlbc = nzrwafgs * ns->zns.zrwafg; |
2112 | |
|
2113 | 0 | trace_pci_nvme_zoned_zrwa_implicit_flush(zone->d.zslba, nlbc); |
2114 | |
|
2115 | 0 | zone->w_ptr += nlbc; |
2116 | |
|
2117 | 0 | nvme_advance_zone_wp(ns, zone, nlbc); |
2118 | 0 | } |
2119 | | |
2120 | | static void nvme_finalize_zoned_write(NvmeNamespace *ns, NvmeRequest *req) |
2121 | 0 | { |
2122 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
2123 | 0 | NvmeZone *zone; |
2124 | 0 | uint64_t slba; |
2125 | 0 | uint32_t nlb; |
2126 | |
|
2127 | 0 | slba = le64_to_cpu(rw->slba); |
2128 | 0 | nlb = le16_to_cpu(rw->nlb) + 1; |
2129 | 0 | zone = nvme_get_zone_by_slba(ns, slba); |
2130 | 0 | assert(zone); |
2131 | |
|
2132 | 0 | if (zone->d.za & NVME_ZA_ZRWA_VALID) { |
2133 | 0 | uint64_t ezrwa = zone->w_ptr + ns->zns.zrwas - 1; |
2134 | 0 | uint64_t elba = slba + nlb - 1; |
2135 | |
|
2136 | 0 | if (elba > ezrwa) { |
2137 | 0 | nvme_zoned_zrwa_implicit_flush(ns, zone, elba - ezrwa); |
2138 | 0 | } |
2139 | |
|
2140 | 0 | return; |
2141 | 0 | } |
2142 | | |
2143 | 0 | nvme_advance_zone_wp(ns, zone, nlb); |
2144 | 0 | } |
2145 | | |
2146 | | static inline bool nvme_is_write(NvmeRequest *req) |
2147 | 0 | { |
2148 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
2149 | |
|
2150 | 0 | return rw->opcode == NVME_CMD_WRITE || |
2151 | 0 | rw->opcode == NVME_CMD_ZONE_APPEND || |
2152 | 0 | rw->opcode == NVME_CMD_WRITE_ZEROES; |
2153 | 0 | } |
2154 | | |
2155 | | static void nvme_misc_cb(void *opaque, int ret) |
2156 | 0 | { |
2157 | 0 | NvmeRequest *req = opaque; |
2158 | 0 | uint16_t cid = nvme_cid(req); |
2159 | |
|
2160 | 0 | trace_pci_nvme_misc_cb(cid); |
2161 | |
|
2162 | 0 | if (ret) { |
2163 | 0 | if (!req->status) { |
2164 | 0 | req->status = NVME_INTERNAL_DEV_ERROR; |
2165 | 0 | } |
2166 | |
|
2167 | 0 | trace_pci_nvme_err_aio(cid, strerror(-ret), req->status); |
2168 | 0 | } |
2169 | |
|
2170 | 0 | nvme_enqueue_req_completion(nvme_cq(req), req); |
2171 | 0 | } |
2172 | | |
2173 | | void nvme_rw_complete_cb(void *opaque, int ret) |
2174 | 0 | { |
2175 | 0 | NvmeRequest *req = opaque; |
2176 | 0 | NvmeNamespace *ns = req->ns; |
2177 | 0 | BlockBackend *blk = ns->blkconf.blk; |
2178 | 0 | BlockAcctCookie *acct = &req->acct; |
2179 | 0 | BlockAcctStats *stats = blk_get_stats(blk); |
2180 | |
|
2181 | 0 | trace_pci_nvme_rw_complete_cb(nvme_cid(req), blk_name(blk)); |
2182 | |
|
2183 | 0 | if (ret) { |
2184 | 0 | Error *err = NULL; |
2185 | |
|
2186 | 0 | block_acct_failed(stats, acct); |
2187 | |
|
2188 | 0 | switch (req->cmd.opcode) { |
2189 | 0 | case NVME_CMD_READ: |
2190 | 0 | req->status = NVME_UNRECOVERED_READ; |
2191 | 0 | break; |
2192 | | |
2193 | 0 | case NVME_CMD_WRITE: |
2194 | 0 | case NVME_CMD_WRITE_ZEROES: |
2195 | 0 | case NVME_CMD_ZONE_APPEND: |
2196 | 0 | req->status = NVME_WRITE_FAULT; |
2197 | 0 | break; |
2198 | | |
2199 | 0 | default: |
2200 | 0 | req->status = NVME_INTERNAL_DEV_ERROR; |
2201 | 0 | break; |
2202 | 0 | } |
2203 | | |
2204 | 0 | trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status); |
2205 | |
|
2206 | 0 | error_setg_errno(&err, -ret, "aio failed"); |
2207 | 0 | error_report_err(err); |
2208 | 0 | } else { |
2209 | 0 | block_acct_done(stats, acct); |
2210 | 0 | } |
2211 | | |
2212 | 0 | if (ns->params.zoned && nvme_is_write(req)) { |
2213 | 0 | nvme_finalize_zoned_write(ns, req); |
2214 | 0 | } |
2215 | |
|
2216 | 0 | nvme_enqueue_req_completion(nvme_cq(req), req); |
2217 | 0 | } |
2218 | | |
2219 | | static void nvme_rw_cb(void *opaque, int ret) |
2220 | 0 | { |
2221 | 0 | NvmeRequest *req = opaque; |
2222 | 0 | NvmeNamespace *ns = req->ns; |
2223 | |
|
2224 | 0 | BlockBackend *blk = ns->blkconf.blk; |
2225 | |
|
2226 | 0 | trace_pci_nvme_rw_cb(nvme_cid(req), blk_name(blk)); |
2227 | |
|
2228 | 0 | if (ret) { |
2229 | 0 | goto out; |
2230 | 0 | } |
2231 | | |
2232 | 0 | if (ns->lbaf.ms) { |
2233 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
2234 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
2235 | 0 | uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1; |
2236 | 0 | uint64_t offset = nvme_moff(ns, slba); |
2237 | |
|
2238 | 0 | if (req->cmd.opcode == NVME_CMD_WRITE_ZEROES) { |
2239 | 0 | size_t mlen = nvme_m2b(ns, nlb); |
2240 | |
|
2241 | 0 | req->aiocb = blk_aio_pwrite_zeroes(blk, offset, mlen, |
2242 | 0 | BDRV_REQ_MAY_UNMAP, |
2243 | 0 | nvme_rw_complete_cb, req); |
2244 | 0 | return; |
2245 | 0 | } |
2246 | | |
2247 | 0 | if (nvme_ns_ext(ns) || req->cmd.mptr) { |
2248 | 0 | uint16_t status; |
2249 | |
|
2250 | 0 | nvme_sg_unmap(&req->sg); |
2251 | 0 | status = nvme_map_mdata(nvme_ctrl(req), nlb, req); |
2252 | 0 | if (status) { |
2253 | 0 | ret = -EFAULT; |
2254 | 0 | goto out; |
2255 | 0 | } |
2256 | | |
2257 | 0 | if (req->cmd.opcode == NVME_CMD_READ) { |
2258 | 0 | return nvme_blk_read(blk, offset, 1, nvme_rw_complete_cb, req); |
2259 | 0 | } |
2260 | | |
2261 | 0 | return nvme_blk_write(blk, offset, 1, nvme_rw_complete_cb, req); |
2262 | 0 | } |
2263 | 0 | } |
2264 | | |
2265 | 0 | out: |
2266 | 0 | nvme_rw_complete_cb(req, ret); |
2267 | 0 | } |
2268 | | |
2269 | | static void nvme_verify_cb(void *opaque, int ret) |
2270 | 0 | { |
2271 | 0 | NvmeBounceContext *ctx = opaque; |
2272 | 0 | NvmeRequest *req = ctx->req; |
2273 | 0 | NvmeNamespace *ns = req->ns; |
2274 | 0 | BlockBackend *blk = ns->blkconf.blk; |
2275 | 0 | BlockAcctCookie *acct = &req->acct; |
2276 | 0 | BlockAcctStats *stats = blk_get_stats(blk); |
2277 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
2278 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
2279 | 0 | uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); |
2280 | 0 | uint16_t apptag = le16_to_cpu(rw->apptag); |
2281 | 0 | uint16_t appmask = le16_to_cpu(rw->appmask); |
2282 | 0 | uint64_t reftag = le32_to_cpu(rw->reftag); |
2283 | 0 | uint64_t cdw3 = le32_to_cpu(rw->cdw3); |
2284 | 0 | uint16_t status; |
2285 | |
|
2286 | 0 | reftag |= cdw3 << 32; |
2287 | |
|
2288 | 0 | trace_pci_nvme_verify_cb(nvme_cid(req), prinfo, apptag, appmask, reftag); |
2289 | |
|
2290 | 0 | if (ret) { |
2291 | 0 | block_acct_failed(stats, acct); |
2292 | 0 | req->status = NVME_UNRECOVERED_READ; |
2293 | |
|
2294 | 0 | trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status); |
2295 | |
|
2296 | 0 | goto out; |
2297 | 0 | } |
2298 | | |
2299 | 0 | block_acct_done(stats, acct); |
2300 | |
|
2301 | 0 | if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { |
2302 | 0 | status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce, |
2303 | 0 | ctx->mdata.iov.size, slba); |
2304 | 0 | if (status) { |
2305 | 0 | req->status = status; |
2306 | 0 | goto out; |
2307 | 0 | } |
2308 | | |
2309 | 0 | req->status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size, |
2310 | 0 | ctx->mdata.bounce, ctx->mdata.iov.size, |
2311 | 0 | prinfo, slba, apptag, appmask, &reftag); |
2312 | 0 | } |
2313 | | |
2314 | 0 | out: |
2315 | 0 | qemu_iovec_destroy(&ctx->data.iov); |
2316 | 0 | g_free(ctx->data.bounce); |
2317 | |
|
2318 | 0 | qemu_iovec_destroy(&ctx->mdata.iov); |
2319 | 0 | g_free(ctx->mdata.bounce); |
2320 | |
|
2321 | 0 | g_free(ctx); |
2322 | |
|
2323 | 0 | nvme_enqueue_req_completion(nvme_cq(req), req); |
2324 | 0 | } |
2325 | | |
2326 | | |
2327 | | static void nvme_verify_mdata_in_cb(void *opaque, int ret) |
2328 | 0 | { |
2329 | 0 | NvmeBounceContext *ctx = opaque; |
2330 | 0 | NvmeRequest *req = ctx->req; |
2331 | 0 | NvmeNamespace *ns = req->ns; |
2332 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
2333 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
2334 | 0 | uint32_t nlb = le16_to_cpu(rw->nlb) + 1; |
2335 | 0 | size_t mlen = nvme_m2b(ns, nlb); |
2336 | 0 | uint64_t offset = nvme_moff(ns, slba); |
2337 | 0 | BlockBackend *blk = ns->blkconf.blk; |
2338 | |
|
2339 | 0 | trace_pci_nvme_verify_mdata_in_cb(nvme_cid(req), blk_name(blk)); |
2340 | |
|
2341 | 0 | if (ret) { |
2342 | 0 | goto out; |
2343 | 0 | } |
2344 | | |
2345 | 0 | ctx->mdata.bounce = g_malloc(mlen); |
2346 | |
|
2347 | 0 | qemu_iovec_reset(&ctx->mdata.iov); |
2348 | 0 | qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen); |
2349 | |
|
2350 | 0 | req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0, |
2351 | 0 | nvme_verify_cb, ctx); |
2352 | 0 | return; |
2353 | | |
2354 | 0 | out: |
2355 | 0 | nvme_verify_cb(ctx, ret); |
2356 | 0 | } |
2357 | | |
2358 | | struct nvme_compare_ctx { |
2359 | | struct { |
2360 | | QEMUIOVector iov; |
2361 | | uint8_t *bounce; |
2362 | | } data; |
2363 | | |
2364 | | struct { |
2365 | | QEMUIOVector iov; |
2366 | | uint8_t *bounce; |
2367 | | } mdata; |
2368 | | }; |
2369 | | |
2370 | | static void nvme_compare_mdata_cb(void *opaque, int ret) |
2371 | 0 | { |
2372 | 0 | NvmeRequest *req = opaque; |
2373 | 0 | NvmeNamespace *ns = req->ns; |
2374 | 0 | NvmeCtrl *n = nvme_ctrl(req); |
2375 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
2376 | 0 | uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); |
2377 | 0 | uint16_t apptag = le16_to_cpu(rw->apptag); |
2378 | 0 | uint16_t appmask = le16_to_cpu(rw->appmask); |
2379 | 0 | uint64_t reftag = le32_to_cpu(rw->reftag); |
2380 | 0 | uint64_t cdw3 = le32_to_cpu(rw->cdw3); |
2381 | 0 | struct nvme_compare_ctx *ctx = req->opaque; |
2382 | 0 | g_autofree uint8_t *buf = NULL; |
2383 | 0 | BlockBackend *blk = ns->blkconf.blk; |
2384 | 0 | BlockAcctCookie *acct = &req->acct; |
2385 | 0 | BlockAcctStats *stats = blk_get_stats(blk); |
2386 | 0 | uint16_t status = NVME_SUCCESS; |
2387 | |
|
2388 | 0 | reftag |= cdw3 << 32; |
2389 | |
|
2390 | 0 | trace_pci_nvme_compare_mdata_cb(nvme_cid(req)); |
2391 | |
|
2392 | 0 | if (ret) { |
2393 | 0 | req->status = NVME_UNRECOVERED_READ; |
2394 | |
|
2395 | 0 | trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status); |
2396 | |
|
2397 | 0 | goto out; |
2398 | 0 | } |
2399 | | |
2400 | 0 | buf = g_malloc(ctx->mdata.iov.size); |
2401 | |
|
2402 | 0 | status = nvme_bounce_mdata(n, buf, ctx->mdata.iov.size, |
2403 | 0 | NVME_TX_DIRECTION_TO_DEVICE, req); |
2404 | 0 | if (status) { |
2405 | 0 | req->status = status; |
2406 | 0 | goto out; |
2407 | 0 | } |
2408 | | |
2409 | 0 | if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { |
2410 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
2411 | 0 | uint8_t *bufp; |
2412 | 0 | uint8_t *mbufp = ctx->mdata.bounce; |
2413 | 0 | uint8_t *end = mbufp + ctx->mdata.iov.size; |
2414 | 0 | int16_t pil = 0; |
2415 | |
|
2416 | 0 | status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size, |
2417 | 0 | ctx->mdata.bounce, ctx->mdata.iov.size, prinfo, |
2418 | 0 | slba, apptag, appmask, &reftag); |
2419 | 0 | if (status) { |
2420 | 0 | req->status = status; |
2421 | 0 | goto out; |
2422 | 0 | } |
2423 | | |
2424 | | /* |
2425 | | * When formatted with protection information, do not compare the DIF |
2426 | | * tuple. |
2427 | | */ |
2428 | 0 | if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { |
2429 | 0 | pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); |
2430 | 0 | } |
2431 | |
|
2432 | 0 | for (bufp = buf; mbufp < end; bufp += ns->lbaf.ms, mbufp += ns->lbaf.ms) { |
2433 | 0 | if (memcmp(bufp + pil, mbufp + pil, ns->lbaf.ms - pil)) { |
2434 | 0 | req->status = NVME_CMP_FAILURE | NVME_DNR; |
2435 | 0 | goto out; |
2436 | 0 | } |
2437 | 0 | } |
2438 | | |
2439 | 0 | goto out; |
2440 | 0 | } |
2441 | | |
2442 | 0 | if (memcmp(buf, ctx->mdata.bounce, ctx->mdata.iov.size)) { |
2443 | 0 | req->status = NVME_CMP_FAILURE | NVME_DNR; |
2444 | 0 | goto out; |
2445 | 0 | } |
2446 | | |
2447 | | |
2448 | 0 | out: |
2449 | 0 | if (req->status == NVME_SUCCESS) { |
2450 | 0 | block_acct_done(stats, acct); |
2451 | 0 | } else { |
2452 | 0 | block_acct_failed(stats, acct); |
2453 | 0 | } |
2454 | |
|
2455 | 0 | qemu_iovec_destroy(&ctx->data.iov); |
2456 | 0 | g_free(ctx->data.bounce); |
2457 | |
|
2458 | 0 | qemu_iovec_destroy(&ctx->mdata.iov); |
2459 | 0 | g_free(ctx->mdata.bounce); |
2460 | |
|
2461 | 0 | g_free(ctx); |
2462 | |
|
2463 | 0 | nvme_enqueue_req_completion(nvme_cq(req), req); |
2464 | 0 | } |
2465 | | |
2466 | | static void nvme_compare_data_cb(void *opaque, int ret) |
2467 | 0 | { |
2468 | 0 | NvmeRequest *req = opaque; |
2469 | 0 | NvmeCtrl *n = nvme_ctrl(req); |
2470 | 0 | NvmeNamespace *ns = req->ns; |
2471 | 0 | BlockBackend *blk = ns->blkconf.blk; |
2472 | 0 | BlockAcctCookie *acct = &req->acct; |
2473 | 0 | BlockAcctStats *stats = blk_get_stats(blk); |
2474 | |
|
2475 | 0 | struct nvme_compare_ctx *ctx = req->opaque; |
2476 | 0 | g_autofree uint8_t *buf = NULL; |
2477 | 0 | uint16_t status; |
2478 | |
|
2479 | 0 | trace_pci_nvme_compare_data_cb(nvme_cid(req)); |
2480 | |
|
2481 | 0 | if (ret) { |
2482 | 0 | req->status = NVME_UNRECOVERED_READ; |
2483 | |
|
2484 | 0 | trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status); |
2485 | |
|
2486 | 0 | goto out; |
2487 | 0 | } |
2488 | | |
2489 | 0 | buf = g_malloc(ctx->data.iov.size); |
2490 | |
|
2491 | 0 | status = nvme_bounce_data(n, buf, ctx->data.iov.size, |
2492 | 0 | NVME_TX_DIRECTION_TO_DEVICE, req); |
2493 | 0 | if (status) { |
2494 | 0 | req->status = status; |
2495 | 0 | goto out; |
2496 | 0 | } |
2497 | | |
2498 | 0 | if (memcmp(buf, ctx->data.bounce, ctx->data.iov.size)) { |
2499 | 0 | req->status = NVME_CMP_FAILURE | NVME_DNR; |
2500 | 0 | goto out; |
2501 | 0 | } |
2502 | | |
2503 | 0 | if (ns->lbaf.ms) { |
2504 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
2505 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
2506 | 0 | uint32_t nlb = le16_to_cpu(rw->nlb) + 1; |
2507 | 0 | size_t mlen = nvme_m2b(ns, nlb); |
2508 | 0 | uint64_t offset = nvme_moff(ns, slba); |
2509 | |
|
2510 | 0 | ctx->mdata.bounce = g_malloc(mlen); |
2511 | |
|
2512 | 0 | qemu_iovec_init(&ctx->mdata.iov, 1); |
2513 | 0 | qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen); |
2514 | |
|
2515 | 0 | req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0, |
2516 | 0 | nvme_compare_mdata_cb, req); |
2517 | 0 | return; |
2518 | 0 | } |
2519 | | |
2520 | 0 | out: |
2521 | 0 | if (req->status == NVME_SUCCESS) { |
2522 | 0 | block_acct_done(stats, acct); |
2523 | 0 | } else { |
2524 | 0 | block_acct_failed(stats, acct); |
2525 | 0 | } |
2526 | |
|
2527 | 0 | qemu_iovec_destroy(&ctx->data.iov); |
2528 | 0 | g_free(ctx->data.bounce); |
2529 | 0 | g_free(ctx); |
2530 | |
|
2531 | 0 | nvme_enqueue_req_completion(nvme_cq(req), req); |
2532 | 0 | } |
2533 | | |
2534 | | typedef struct NvmeDSMAIOCB { |
2535 | | BlockAIOCB common; |
2536 | | BlockAIOCB *aiocb; |
2537 | | NvmeRequest *req; |
2538 | | int ret; |
2539 | | |
2540 | | NvmeDsmRange *range; |
2541 | | unsigned int nr; |
2542 | | unsigned int idx; |
2543 | | } NvmeDSMAIOCB; |
2544 | | |
2545 | | static void nvme_dsm_cancel(BlockAIOCB *aiocb) |
2546 | 0 | { |
2547 | 0 | NvmeDSMAIOCB *iocb = container_of(aiocb, NvmeDSMAIOCB, common); |
2548 | | |
2549 | | /* break nvme_dsm_cb loop */ |
2550 | 0 | iocb->idx = iocb->nr; |
2551 | 0 | iocb->ret = -ECANCELED; |
2552 | |
|
2553 | 0 | if (iocb->aiocb) { |
2554 | 0 | blk_aio_cancel_async(iocb->aiocb); |
2555 | 0 | iocb->aiocb = NULL; |
2556 | 0 | } else { |
2557 | | /* |
2558 | | * We only reach this if nvme_dsm_cancel() has already been called or |
2559 | | * the command ran to completion. |
2560 | | */ |
2561 | 0 | assert(iocb->idx == iocb->nr); |
2562 | 0 | } |
2563 | 0 | } |
2564 | | |
2565 | | static const AIOCBInfo nvme_dsm_aiocb_info = { |
2566 | | .aiocb_size = sizeof(NvmeDSMAIOCB), |
2567 | | .cancel_async = nvme_dsm_cancel, |
2568 | | }; |
2569 | | |
2570 | | static void nvme_dsm_cb(void *opaque, int ret); |
2571 | | |
2572 | | static void nvme_dsm_md_cb(void *opaque, int ret) |
2573 | 0 | { |
2574 | 0 | NvmeDSMAIOCB *iocb = opaque; |
2575 | 0 | NvmeRequest *req = iocb->req; |
2576 | 0 | NvmeNamespace *ns = req->ns; |
2577 | 0 | NvmeDsmRange *range; |
2578 | 0 | uint64_t slba; |
2579 | 0 | uint32_t nlb; |
2580 | |
|
2581 | 0 | if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) { |
2582 | 0 | goto done; |
2583 | 0 | } |
2584 | | |
2585 | 0 | range = &iocb->range[iocb->idx - 1]; |
2586 | 0 | slba = le64_to_cpu(range->slba); |
2587 | 0 | nlb = le32_to_cpu(range->nlb); |
2588 | | |
2589 | | /* |
2590 | | * Check that all block were discarded (zeroed); otherwise we do not zero |
2591 | | * the metadata. |
2592 | | */ |
2593 | |
|
2594 | 0 | ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_ZERO); |
2595 | 0 | if (ret) { |
2596 | 0 | if (ret < 0) { |
2597 | 0 | goto done; |
2598 | 0 | } |
2599 | | |
2600 | 0 | nvme_dsm_cb(iocb, 0); |
2601 | 0 | return; |
2602 | 0 | } |
2603 | | |
2604 | 0 | iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, nvme_moff(ns, slba), |
2605 | 0 | nvme_m2b(ns, nlb), BDRV_REQ_MAY_UNMAP, |
2606 | 0 | nvme_dsm_cb, iocb); |
2607 | 0 | return; |
2608 | | |
2609 | 0 | done: |
2610 | 0 | nvme_dsm_cb(iocb, ret); |
2611 | 0 | } |
2612 | | |
2613 | | static void nvme_dsm_cb(void *opaque, int ret) |
2614 | 0 | { |
2615 | 0 | NvmeDSMAIOCB *iocb = opaque; |
2616 | 0 | NvmeRequest *req = iocb->req; |
2617 | 0 | NvmeCtrl *n = nvme_ctrl(req); |
2618 | 0 | NvmeNamespace *ns = req->ns; |
2619 | 0 | NvmeDsmRange *range; |
2620 | 0 | uint64_t slba; |
2621 | 0 | uint32_t nlb; |
2622 | |
|
2623 | 0 | if (iocb->ret < 0) { |
2624 | 0 | goto done; |
2625 | 0 | } else if (ret < 0) { |
2626 | 0 | iocb->ret = ret; |
2627 | 0 | goto done; |
2628 | 0 | } |
2629 | | |
2630 | 0 | next: |
2631 | 0 | if (iocb->idx == iocb->nr) { |
2632 | 0 | goto done; |
2633 | 0 | } |
2634 | | |
2635 | 0 | range = &iocb->range[iocb->idx++]; |
2636 | 0 | slba = le64_to_cpu(range->slba); |
2637 | 0 | nlb = le32_to_cpu(range->nlb); |
2638 | |
|
2639 | 0 | trace_pci_nvme_dsm_deallocate(slba, nlb); |
2640 | |
|
2641 | 0 | if (nlb > n->dmrsl) { |
2642 | 0 | trace_pci_nvme_dsm_single_range_limit_exceeded(nlb, n->dmrsl); |
2643 | 0 | goto next; |
2644 | 0 | } |
2645 | | |
2646 | 0 | if (nvme_check_bounds(ns, slba, nlb)) { |
2647 | 0 | trace_pci_nvme_err_invalid_lba_range(slba, nlb, |
2648 | 0 | ns->id_ns.nsze); |
2649 | 0 | goto next; |
2650 | 0 | } |
2651 | | |
2652 | 0 | iocb->aiocb = blk_aio_pdiscard(ns->blkconf.blk, nvme_l2b(ns, slba), |
2653 | 0 | nvme_l2b(ns, nlb), |
2654 | 0 | nvme_dsm_md_cb, iocb); |
2655 | 0 | return; |
2656 | | |
2657 | 0 | done: |
2658 | 0 | iocb->aiocb = NULL; |
2659 | 0 | iocb->common.cb(iocb->common.opaque, iocb->ret); |
2660 | 0 | g_free(iocb->range); |
2661 | 0 | qemu_aio_unref(iocb); |
2662 | 0 | } |
2663 | | |
2664 | | static uint16_t nvme_dsm(NvmeCtrl *n, NvmeRequest *req) |
2665 | 0 | { |
2666 | 0 | NvmeNamespace *ns = req->ns; |
2667 | 0 | NvmeDsmCmd *dsm = (NvmeDsmCmd *) &req->cmd; |
2668 | 0 | uint32_t attr = le32_to_cpu(dsm->attributes); |
2669 | 0 | uint32_t nr = (le32_to_cpu(dsm->nr) & 0xff) + 1; |
2670 | 0 | uint16_t status = NVME_SUCCESS; |
2671 | |
|
2672 | 0 | trace_pci_nvme_dsm(nr, attr); |
2673 | |
|
2674 | 0 | if (attr & NVME_DSMGMT_AD) { |
2675 | 0 | NvmeDSMAIOCB *iocb = blk_aio_get(&nvme_dsm_aiocb_info, ns->blkconf.blk, |
2676 | 0 | nvme_misc_cb, req); |
2677 | |
|
2678 | 0 | iocb->req = req; |
2679 | 0 | iocb->ret = 0; |
2680 | 0 | iocb->range = g_new(NvmeDsmRange, nr); |
2681 | 0 | iocb->nr = nr; |
2682 | 0 | iocb->idx = 0; |
2683 | |
|
2684 | 0 | status = nvme_h2c(n, (uint8_t *)iocb->range, sizeof(NvmeDsmRange) * nr, |
2685 | 0 | req); |
2686 | 0 | if (status) { |
2687 | 0 | g_free(iocb->range); |
2688 | 0 | qemu_aio_unref(iocb); |
2689 | |
|
2690 | 0 | return status; |
2691 | 0 | } |
2692 | | |
2693 | 0 | req->aiocb = &iocb->common; |
2694 | 0 | nvme_dsm_cb(iocb, 0); |
2695 | |
|
2696 | 0 | return NVME_NO_COMPLETE; |
2697 | 0 | } |
2698 | | |
2699 | 0 | return status; |
2700 | 0 | } |
2701 | | |
2702 | | static uint16_t nvme_verify(NvmeCtrl *n, NvmeRequest *req) |
2703 | 0 | { |
2704 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
2705 | 0 | NvmeNamespace *ns = req->ns; |
2706 | 0 | BlockBackend *blk = ns->blkconf.blk; |
2707 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
2708 | 0 | uint32_t nlb = le16_to_cpu(rw->nlb) + 1; |
2709 | 0 | size_t len = nvme_l2b(ns, nlb); |
2710 | 0 | size_t data_len = len; |
2711 | 0 | int64_t offset = nvme_l2b(ns, slba); |
2712 | 0 | uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); |
2713 | 0 | uint32_t reftag = le32_to_cpu(rw->reftag); |
2714 | 0 | NvmeBounceContext *ctx = NULL; |
2715 | 0 | uint16_t status; |
2716 | |
|
2717 | 0 | trace_pci_nvme_verify(nvme_cid(req), nvme_nsid(ns), slba, nlb); |
2718 | |
|
2719 | 0 | if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { |
2720 | 0 | status = nvme_check_prinfo(ns, prinfo, slba, reftag); |
2721 | 0 | if (status) { |
2722 | 0 | return status; |
2723 | 0 | } |
2724 | | |
2725 | 0 | if (prinfo & NVME_PRINFO_PRACT) { |
2726 | 0 | return NVME_INVALID_PROT_INFO | NVME_DNR; |
2727 | 0 | } |
2728 | 0 | } |
2729 | | |
2730 | 0 | if (nvme_ns_ext(ns) && !(NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt))) { |
2731 | 0 | data_len += nvme_m2b(ns, nlb); |
2732 | 0 | } |
2733 | |
|
2734 | 0 | if (data_len > (n->page_size << n->params.vsl)) { |
2735 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
2736 | 0 | } |
2737 | | |
2738 | 0 | status = nvme_check_bounds(ns, slba, nlb); |
2739 | 0 | if (status) { |
2740 | 0 | return status; |
2741 | 0 | } |
2742 | | |
2743 | 0 | if (NVME_ERR_REC_DULBE(ns->features.err_rec)) { |
2744 | 0 | status = nvme_check_dulbe(ns, slba, nlb); |
2745 | 0 | if (status) { |
2746 | 0 | return status; |
2747 | 0 | } |
2748 | 0 | } |
2749 | | |
2750 | 0 | ctx = g_new0(NvmeBounceContext, 1); |
2751 | 0 | ctx->req = req; |
2752 | |
|
2753 | 0 | ctx->data.bounce = g_malloc(len); |
2754 | |
|
2755 | 0 | qemu_iovec_init(&ctx->data.iov, 1); |
2756 | 0 | qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len); |
2757 | |
|
2758 | 0 | block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size, |
2759 | 0 | BLOCK_ACCT_READ); |
2760 | |
|
2761 | 0 | req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0, |
2762 | 0 | nvme_verify_mdata_in_cb, ctx); |
2763 | 0 | return NVME_NO_COMPLETE; |
2764 | 0 | } |
2765 | | |
2766 | | typedef struct NvmeCopyAIOCB { |
2767 | | BlockAIOCB common; |
2768 | | BlockAIOCB *aiocb; |
2769 | | NvmeRequest *req; |
2770 | | NvmeCtrl *n; |
2771 | | int ret; |
2772 | | |
2773 | | void *ranges; |
2774 | | unsigned int format; |
2775 | | int nr; |
2776 | | int idx; |
2777 | | |
2778 | | uint8_t *bounce; |
2779 | | QEMUIOVector iov; |
2780 | | struct { |
2781 | | BlockAcctCookie read; |
2782 | | BlockAcctCookie write; |
2783 | | } acct; |
2784 | | |
2785 | | uint64_t reftag; |
2786 | | uint64_t slba; |
2787 | | |
2788 | | NvmeZone *zone; |
2789 | | NvmeNamespace *sns; |
2790 | | uint32_t tcl; |
2791 | | } NvmeCopyAIOCB; |
2792 | | |
2793 | | static void nvme_copy_cancel(BlockAIOCB *aiocb) |
2794 | 0 | { |
2795 | 0 | NvmeCopyAIOCB *iocb = container_of(aiocb, NvmeCopyAIOCB, common); |
2796 | |
|
2797 | 0 | iocb->ret = -ECANCELED; |
2798 | |
|
2799 | 0 | if (iocb->aiocb) { |
2800 | 0 | blk_aio_cancel_async(iocb->aiocb); |
2801 | 0 | iocb->aiocb = NULL; |
2802 | 0 | } |
2803 | 0 | } |
2804 | | |
2805 | | static const AIOCBInfo nvme_copy_aiocb_info = { |
2806 | | .aiocb_size = sizeof(NvmeCopyAIOCB), |
2807 | | .cancel_async = nvme_copy_cancel, |
2808 | | }; |
2809 | | |
2810 | | static void nvme_copy_done(NvmeCopyAIOCB *iocb) |
2811 | 0 | { |
2812 | 0 | NvmeRequest *req = iocb->req; |
2813 | |
|
2814 | 0 | if (iocb->idx != iocb->nr) { |
2815 | 0 | req->cqe.result = cpu_to_le32(iocb->idx); |
2816 | 0 | } |
2817 | |
|
2818 | 0 | qemu_iovec_destroy(&iocb->iov); |
2819 | 0 | g_free(iocb->bounce); |
2820 | 0 | g_free(iocb->ranges); |
2821 | |
|
2822 | 0 | iocb->common.cb(iocb->common.opaque, iocb->ret); |
2823 | 0 | qemu_aio_unref(iocb); |
2824 | 0 | } |
2825 | | |
2826 | | static void nvme_do_copy(NvmeCopyAIOCB *iocb); |
2827 | | |
2828 | | static void nvme_copy_source_range_parse_format0_2(void *ranges, |
2829 | | int idx, uint64_t *slba, |
2830 | | uint32_t *nlb, |
2831 | | uint32_t *snsid, |
2832 | | uint16_t *apptag, |
2833 | | uint16_t *appmask, |
2834 | | uint64_t *reftag) |
2835 | 0 | { |
2836 | 0 | NvmeCopySourceRangeFormat0_2 *_ranges = ranges; |
2837 | |
|
2838 | 0 | if (snsid) { |
2839 | 0 | *snsid = le32_to_cpu(_ranges[idx].sparams); |
2840 | 0 | } |
2841 | |
|
2842 | 0 | if (slba) { |
2843 | 0 | *slba = le64_to_cpu(_ranges[idx].slba); |
2844 | 0 | } |
2845 | |
|
2846 | 0 | if (nlb) { |
2847 | 0 | *nlb = le16_to_cpu(_ranges[idx].nlb) + 1; |
2848 | 0 | } |
2849 | |
|
2850 | 0 | if (apptag) { |
2851 | 0 | *apptag = le16_to_cpu(_ranges[idx].apptag); |
2852 | 0 | } |
2853 | |
|
2854 | 0 | if (appmask) { |
2855 | 0 | *appmask = le16_to_cpu(_ranges[idx].appmask); |
2856 | 0 | } |
2857 | |
|
2858 | 0 | if (reftag) { |
2859 | 0 | *reftag = le32_to_cpu(_ranges[idx].reftag); |
2860 | 0 | } |
2861 | 0 | } |
2862 | | |
2863 | | static void nvme_copy_source_range_parse_format1_3(void *ranges, int idx, |
2864 | | uint64_t *slba, |
2865 | | uint32_t *nlb, |
2866 | | uint32_t *snsid, |
2867 | | uint16_t *apptag, |
2868 | | uint16_t *appmask, |
2869 | | uint64_t *reftag) |
2870 | 0 | { |
2871 | 0 | NvmeCopySourceRangeFormat1_3 *_ranges = ranges; |
2872 | |
|
2873 | 0 | if (snsid) { |
2874 | 0 | *snsid = le32_to_cpu(_ranges[idx].sparams); |
2875 | 0 | } |
2876 | |
|
2877 | 0 | if (slba) { |
2878 | 0 | *slba = le64_to_cpu(_ranges[idx].slba); |
2879 | 0 | } |
2880 | |
|
2881 | 0 | if (nlb) { |
2882 | 0 | *nlb = le16_to_cpu(_ranges[idx].nlb) + 1; |
2883 | 0 | } |
2884 | |
|
2885 | 0 | if (apptag) { |
2886 | 0 | *apptag = le16_to_cpu(_ranges[idx].apptag); |
2887 | 0 | } |
2888 | |
|
2889 | 0 | if (appmask) { |
2890 | 0 | *appmask = le16_to_cpu(_ranges[idx].appmask); |
2891 | 0 | } |
2892 | |
|
2893 | 0 | if (reftag) { |
2894 | 0 | *reftag = 0; |
2895 | |
|
2896 | 0 | *reftag |= (uint64_t)_ranges[idx].sr[4] << 40; |
2897 | 0 | *reftag |= (uint64_t)_ranges[idx].sr[5] << 32; |
2898 | 0 | *reftag |= (uint64_t)_ranges[idx].sr[6] << 24; |
2899 | 0 | *reftag |= (uint64_t)_ranges[idx].sr[7] << 16; |
2900 | 0 | *reftag |= (uint64_t)_ranges[idx].sr[8] << 8; |
2901 | 0 | *reftag |= (uint64_t)_ranges[idx].sr[9]; |
2902 | 0 | } |
2903 | 0 | } |
2904 | | |
2905 | | static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format, |
2906 | | uint64_t *slba, uint32_t *nlb, |
2907 | | uint32_t *snsid, uint16_t *apptag, |
2908 | | uint16_t *appmask, uint64_t *reftag) |
2909 | 0 | { |
2910 | 0 | switch (format) { |
2911 | 0 | case NVME_COPY_FORMAT_0: |
2912 | 0 | case NVME_COPY_FORMAT_2: |
2913 | 0 | nvme_copy_source_range_parse_format0_2(ranges, idx, slba, nlb, snsid, |
2914 | 0 | apptag, appmask, reftag); |
2915 | 0 | break; |
2916 | | |
2917 | 0 | case NVME_COPY_FORMAT_1: |
2918 | 0 | case NVME_COPY_FORMAT_3: |
2919 | 0 | nvme_copy_source_range_parse_format1_3(ranges, idx, slba, nlb, snsid, |
2920 | 0 | apptag, appmask, reftag); |
2921 | 0 | break; |
2922 | | |
2923 | 0 | default: |
2924 | 0 | abort(); |
2925 | 0 | } |
2926 | 0 | } |
2927 | | |
2928 | | static inline uint16_t nvme_check_copy_mcl(NvmeNamespace *ns, |
2929 | | NvmeCopyAIOCB *iocb, uint16_t nr) |
2930 | 0 | { |
2931 | 0 | uint32_t copy_len = 0; |
2932 | |
|
2933 | 0 | for (int idx = 0; idx < nr; idx++) { |
2934 | 0 | uint32_t nlb; |
2935 | 0 | nvme_copy_source_range_parse(iocb->ranges, idx, iocb->format, NULL, |
2936 | 0 | &nlb, NULL, NULL, NULL, NULL); |
2937 | 0 | copy_len += nlb; |
2938 | 0 | } |
2939 | 0 | iocb->tcl = copy_len; |
2940 | 0 | if (copy_len > ns->id_ns.mcl) { |
2941 | 0 | return NVME_CMD_SIZE_LIMIT | NVME_DNR; |
2942 | 0 | } |
2943 | | |
2944 | 0 | return NVME_SUCCESS; |
2945 | 0 | } |
2946 | | |
2947 | | static void nvme_copy_out_completed_cb(void *opaque, int ret) |
2948 | 0 | { |
2949 | 0 | NvmeCopyAIOCB *iocb = opaque; |
2950 | 0 | NvmeRequest *req = iocb->req; |
2951 | 0 | NvmeNamespace *dns = req->ns; |
2952 | 0 | BlockAcctStats *stats = blk_get_stats(dns->blkconf.blk); |
2953 | 0 | uint32_t nlb; |
2954 | |
|
2955 | 0 | nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL, |
2956 | 0 | &nlb, NULL, NULL, NULL, NULL); |
2957 | |
|
2958 | 0 | if (ret < 0) { |
2959 | 0 | iocb->ret = ret; |
2960 | 0 | req->status = NVME_WRITE_FAULT; |
2961 | 0 | } |
2962 | 0 | if (iocb->ret < 0) { |
2963 | 0 | block_acct_failed(stats, &iocb->acct.write); |
2964 | 0 | goto out; |
2965 | 0 | } |
2966 | 0 | block_acct_done(stats, &iocb->acct.write); |
2967 | |
|
2968 | 0 | if (dns->params.zoned) { |
2969 | 0 | nvme_advance_zone_wp(dns, iocb->zone, nlb); |
2970 | 0 | } |
2971 | |
|
2972 | 0 | iocb->idx++; |
2973 | 0 | iocb->slba += nlb; |
2974 | 0 | out: |
2975 | 0 | nvme_do_copy(iocb); |
2976 | 0 | } |
2977 | | |
2978 | | static void nvme_copy_out_cb(void *opaque, int ret) |
2979 | 0 | { |
2980 | 0 | NvmeCopyAIOCB *iocb = opaque; |
2981 | 0 | NvmeRequest *req = iocb->req; |
2982 | 0 | NvmeNamespace *dns = req->ns; |
2983 | 0 | BlockAcctStats *stats = blk_get_stats(dns->blkconf.blk); |
2984 | 0 | uint32_t nlb; |
2985 | 0 | size_t mlen; |
2986 | 0 | uint8_t *mbounce; |
2987 | |
|
2988 | 0 | if (ret < 0 || iocb->ret < 0) { |
2989 | 0 | block_acct_failed(stats, &iocb->acct.write); |
2990 | 0 | goto out; |
2991 | 0 | } |
2992 | 0 | block_acct_done(stats, &iocb->acct.write); |
2993 | |
|
2994 | 0 | if (!dns->lbaf.ms) { |
2995 | 0 | goto out; |
2996 | 0 | } |
2997 | | |
2998 | 0 | nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL, |
2999 | 0 | &nlb, NULL, NULL, NULL, NULL); |
3000 | |
|
3001 | 0 | mlen = nvme_m2b(dns, nlb); |
3002 | 0 | mbounce = iocb->bounce + nvme_l2b(dns, nlb); |
3003 | |
|
3004 | 0 | qemu_iovec_reset(&iocb->iov); |
3005 | 0 | qemu_iovec_add(&iocb->iov, mbounce, mlen); |
3006 | |
|
3007 | 0 | block_acct_start(stats, &iocb->acct.write, mlen, BLOCK_ACCT_WRITE); |
3008 | 0 | iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_moff(dns, iocb->slba), |
3009 | 0 | &iocb->iov, 0, nvme_copy_out_completed_cb, |
3010 | 0 | iocb); |
3011 | |
|
3012 | 0 | return; |
3013 | | |
3014 | 0 | out: |
3015 | 0 | nvme_copy_out_completed_cb(iocb, ret); |
3016 | 0 | } |
3017 | | |
3018 | | static void nvme_copy_in_completed_cb(void *opaque, int ret) |
3019 | 0 | { |
3020 | 0 | NvmeCopyAIOCB *iocb = opaque; |
3021 | 0 | NvmeRequest *req = iocb->req; |
3022 | 0 | NvmeNamespace *sns = iocb->sns; |
3023 | 0 | NvmeNamespace *dns = req->ns; |
3024 | 0 | BlockAcctStats *sstats = blk_get_stats(sns->blkconf.blk); |
3025 | 0 | NvmeCopyCmd *copy = NULL; |
3026 | 0 | uint8_t *mbounce = NULL; |
3027 | 0 | uint32_t nlb; |
3028 | 0 | uint64_t slba; |
3029 | 0 | uint16_t apptag, appmask; |
3030 | 0 | uint64_t reftag; |
3031 | 0 | size_t len, mlen; |
3032 | 0 | uint16_t status; |
3033 | |
|
3034 | 0 | if (ret < 0) { |
3035 | 0 | iocb->ret = ret; |
3036 | 0 | req->status = NVME_UNRECOVERED_READ; |
3037 | 0 | } |
3038 | 0 | if (iocb->ret < 0) { |
3039 | 0 | block_acct_failed(sstats, &iocb->acct.read); |
3040 | 0 | goto out; |
3041 | 0 | } |
3042 | 0 | block_acct_done(sstats, &iocb->acct.read); |
3043 | |
|
3044 | 0 | nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba, |
3045 | 0 | &nlb, NULL, &apptag, &appmask, &reftag); |
3046 | |
|
3047 | 0 | trace_pci_nvme_copy_out(iocb->slba, nlb); |
3048 | |
|
3049 | 0 | len = nvme_l2b(sns, nlb); |
3050 | |
|
3051 | 0 | if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps)) { |
3052 | 0 | copy = (NvmeCopyCmd *)&req->cmd; |
3053 | |
|
3054 | 0 | uint16_t prinfor = ((copy->control[0] >> 4) & 0xf); |
3055 | |
|
3056 | 0 | mlen = nvme_m2b(sns, nlb); |
3057 | 0 | mbounce = iocb->bounce + nvme_l2b(sns, nlb); |
3058 | |
|
3059 | 0 | status = nvme_dif_mangle_mdata(sns, mbounce, mlen, slba); |
3060 | 0 | if (status) { |
3061 | 0 | goto invalid; |
3062 | 0 | } |
3063 | 0 | status = nvme_dif_check(sns, iocb->bounce, len, mbounce, mlen, prinfor, |
3064 | 0 | slba, apptag, appmask, &reftag); |
3065 | 0 | if (status) { |
3066 | 0 | goto invalid; |
3067 | 0 | } |
3068 | 0 | } |
3069 | | |
3070 | 0 | if (NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { |
3071 | 0 | copy = (NvmeCopyCmd *)&req->cmd; |
3072 | 0 | uint16_t prinfow = ((copy->control[2] >> 2) & 0xf); |
3073 | |
|
3074 | 0 | mlen = nvme_m2b(dns, nlb); |
3075 | 0 | mbounce = iocb->bounce + nvme_l2b(dns, nlb); |
3076 | |
|
3077 | 0 | apptag = le16_to_cpu(copy->apptag); |
3078 | 0 | appmask = le16_to_cpu(copy->appmask); |
3079 | |
|
3080 | 0 | if (prinfow & NVME_PRINFO_PRACT) { |
3081 | 0 | status = nvme_check_prinfo(dns, prinfow, iocb->slba, iocb->reftag); |
3082 | 0 | if (status) { |
3083 | 0 | goto invalid; |
3084 | 0 | } |
3085 | | |
3086 | 0 | nvme_dif_pract_generate_dif(dns, iocb->bounce, len, mbounce, mlen, |
3087 | 0 | apptag, &iocb->reftag); |
3088 | 0 | } else { |
3089 | 0 | status = nvme_dif_check(dns, iocb->bounce, len, mbounce, mlen, |
3090 | 0 | prinfow, iocb->slba, apptag, appmask, |
3091 | 0 | &iocb->reftag); |
3092 | 0 | if (status) { |
3093 | 0 | goto invalid; |
3094 | 0 | } |
3095 | 0 | } |
3096 | 0 | } |
3097 | | |
3098 | 0 | status = nvme_check_bounds(dns, iocb->slba, nlb); |
3099 | 0 | if (status) { |
3100 | 0 | goto invalid; |
3101 | 0 | } |
3102 | | |
3103 | 0 | if (dns->params.zoned) { |
3104 | 0 | status = nvme_check_zone_write(dns, iocb->zone, iocb->slba, nlb); |
3105 | 0 | if (status) { |
3106 | 0 | goto invalid; |
3107 | 0 | } |
3108 | | |
3109 | 0 | if (!(iocb->zone->d.za & NVME_ZA_ZRWA_VALID)) { |
3110 | 0 | iocb->zone->w_ptr += nlb; |
3111 | 0 | } |
3112 | 0 | } |
3113 | | |
3114 | 0 | qemu_iovec_reset(&iocb->iov); |
3115 | 0 | qemu_iovec_add(&iocb->iov, iocb->bounce, len); |
3116 | |
|
3117 | 0 | block_acct_start(blk_get_stats(dns->blkconf.blk), &iocb->acct.write, len, |
3118 | 0 | BLOCK_ACCT_WRITE); |
3119 | |
|
3120 | 0 | iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_l2b(dns, iocb->slba), |
3121 | 0 | &iocb->iov, 0, nvme_copy_out_cb, iocb); |
3122 | |
|
3123 | 0 | return; |
3124 | | |
3125 | 0 | invalid: |
3126 | 0 | req->status = status; |
3127 | 0 | iocb->ret = -1; |
3128 | 0 | out: |
3129 | 0 | nvme_do_copy(iocb); |
3130 | 0 | } |
3131 | | |
3132 | | static void nvme_copy_in_cb(void *opaque, int ret) |
3133 | 0 | { |
3134 | 0 | NvmeCopyAIOCB *iocb = opaque; |
3135 | 0 | NvmeNamespace *sns = iocb->sns; |
3136 | 0 | BlockAcctStats *stats = blk_get_stats(sns->blkconf.blk); |
3137 | 0 | uint64_t slba; |
3138 | 0 | uint32_t nlb; |
3139 | 0 | size_t mlen; |
3140 | |
|
3141 | 0 | if (ret < 0 || iocb->ret < 0) { |
3142 | 0 | block_acct_failed(stats, &iocb->acct.read); |
3143 | 0 | goto out; |
3144 | 0 | } |
3145 | 0 | block_acct_done(stats, &iocb->acct.read); |
3146 | |
|
3147 | 0 | if (!sns->lbaf.ms) { |
3148 | 0 | goto out; |
3149 | 0 | } |
3150 | | |
3151 | 0 | nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba, |
3152 | 0 | &nlb, NULL, NULL, NULL, NULL); |
3153 | |
|
3154 | 0 | mlen = nvme_m2b(sns, nlb); |
3155 | 0 | qemu_iovec_reset(&iocb->iov); |
3156 | 0 | qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(sns, nlb), mlen); |
3157 | |
|
3158 | 0 | block_acct_start(stats, &iocb->acct.read, mlen, BLOCK_ACCT_READ); |
3159 | 0 | iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_moff(sns, slba), |
3160 | 0 | &iocb->iov, 0, nvme_copy_in_completed_cb, |
3161 | 0 | iocb); |
3162 | 0 | return; |
3163 | | |
3164 | 0 | out: |
3165 | 0 | nvme_copy_in_completed_cb(iocb, ret); |
3166 | 0 | } |
3167 | | |
3168 | | static inline bool nvme_csi_supports_copy(uint8_t csi) |
3169 | 0 | { |
3170 | 0 | return csi == NVME_CSI_NVM || csi == NVME_CSI_ZONED; |
3171 | 0 | } |
3172 | | |
3173 | | static inline bool nvme_copy_ns_format_match(NvmeNamespace *sns, |
3174 | | NvmeNamespace *dns) |
3175 | 0 | { |
3176 | 0 | return sns->lbaf.ds == dns->lbaf.ds && sns->lbaf.ms == dns->lbaf.ms; |
3177 | 0 | } |
3178 | | |
3179 | | static bool nvme_copy_matching_ns_format(NvmeNamespace *sns, NvmeNamespace *dns, |
3180 | | bool pi_enable) |
3181 | 0 | { |
3182 | 0 | if (!nvme_csi_supports_copy(sns->csi) || |
3183 | 0 | !nvme_csi_supports_copy(dns->csi)) { |
3184 | 0 | return false; |
3185 | 0 | } |
3186 | | |
3187 | 0 | if (!pi_enable && !nvme_copy_ns_format_match(sns, dns)) { |
3188 | 0 | return false; |
3189 | 0 | } |
3190 | | |
3191 | 0 | if (pi_enable && (!nvme_copy_ns_format_match(sns, dns) || |
3192 | 0 | sns->id_ns.dps != dns->id_ns.dps)) { |
3193 | 0 | return false; |
3194 | 0 | } |
3195 | | |
3196 | 0 | return true; |
3197 | 0 | } |
3198 | | |
3199 | | static inline bool nvme_copy_corresp_pi_match(NvmeNamespace *sns, |
3200 | | NvmeNamespace *dns) |
3201 | 0 | { |
3202 | 0 | return sns->lbaf.ms == 0 && |
3203 | 0 | ((dns->lbaf.ms == 8 && dns->pif == 0) || |
3204 | 0 | (dns->lbaf.ms == 16 && dns->pif == 1)); |
3205 | 0 | } |
3206 | | |
3207 | | static bool nvme_copy_corresp_pi_format(NvmeNamespace *sns, NvmeNamespace *dns, |
3208 | | bool sns_pi_en) |
3209 | 0 | { |
3210 | 0 | if (!nvme_csi_supports_copy(sns->csi) || |
3211 | 0 | !nvme_csi_supports_copy(dns->csi)) { |
3212 | 0 | return false; |
3213 | 0 | } |
3214 | | |
3215 | 0 | if (!sns_pi_en && !nvme_copy_corresp_pi_match(sns, dns)) { |
3216 | 0 | return false; |
3217 | 0 | } |
3218 | | |
3219 | 0 | if (sns_pi_en && !nvme_copy_corresp_pi_match(dns, sns)) { |
3220 | 0 | return false; |
3221 | 0 | } |
3222 | | |
3223 | 0 | return true; |
3224 | 0 | } |
3225 | | |
3226 | | static void nvme_do_copy(NvmeCopyAIOCB *iocb) |
3227 | 0 | { |
3228 | 0 | NvmeRequest *req = iocb->req; |
3229 | 0 | NvmeNamespace *sns; |
3230 | 0 | NvmeNamespace *dns = req->ns; |
3231 | 0 | NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd; |
3232 | 0 | uint16_t prinfor = ((copy->control[0] >> 4) & 0xf); |
3233 | 0 | uint16_t prinfow = ((copy->control[2] >> 2) & 0xf); |
3234 | 0 | uint64_t slba; |
3235 | 0 | uint32_t nlb; |
3236 | 0 | size_t len, blen; |
3237 | 0 | uint16_t status; |
3238 | 0 | uint32_t dnsid = le32_to_cpu(req->cmd.nsid); |
3239 | 0 | uint32_t snsid = dnsid; |
3240 | |
|
3241 | 0 | if (iocb->ret < 0) { |
3242 | 0 | goto done; |
3243 | 0 | } |
3244 | | |
3245 | 0 | if (iocb->idx == iocb->nr) { |
3246 | 0 | goto done; |
3247 | 0 | } |
3248 | | |
3249 | 0 | if (iocb->format == 2 || iocb->format == 3) { |
3250 | 0 | nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, |
3251 | 0 | &slba, &nlb, &snsid, NULL, NULL, NULL); |
3252 | 0 | if (snsid != dnsid) { |
3253 | 0 | if (snsid == NVME_NSID_BROADCAST || |
3254 | 0 | !nvme_nsid_valid(iocb->n, snsid)) { |
3255 | 0 | status = NVME_INVALID_NSID | NVME_DNR; |
3256 | 0 | goto invalid; |
3257 | 0 | } |
3258 | 0 | iocb->sns = nvme_ns(iocb->n, snsid); |
3259 | 0 | if (unlikely(!iocb->sns)) { |
3260 | 0 | status = NVME_INVALID_FIELD | NVME_DNR; |
3261 | 0 | goto invalid; |
3262 | 0 | } |
3263 | 0 | } else { |
3264 | 0 | if (((slba + nlb) > iocb->slba) && |
3265 | 0 | ((slba + nlb) < (iocb->slba + iocb->tcl))) { |
3266 | 0 | status = NVME_CMD_OVERLAP_IO_RANGE | NVME_DNR; |
3267 | 0 | goto invalid; |
3268 | 0 | } |
3269 | 0 | } |
3270 | 0 | } else { |
3271 | 0 | nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, |
3272 | 0 | &slba, &nlb, NULL, NULL, NULL, NULL); |
3273 | 0 | } |
3274 | | |
3275 | 0 | sns = iocb->sns; |
3276 | 0 | if ((snsid == dnsid) && NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && |
3277 | 0 | ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) { |
3278 | 0 | status = NVME_INVALID_FIELD | NVME_DNR; |
3279 | 0 | goto invalid; |
3280 | 0 | } else if (snsid != dnsid) { |
3281 | 0 | if (sns->lbaf.ds != dns->lbaf.ds) { |
3282 | 0 | status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; |
3283 | 0 | goto invalid; |
3284 | 0 | } |
3285 | | |
3286 | 0 | if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && |
3287 | 0 | !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { |
3288 | 0 | if (!nvme_copy_matching_ns_format(sns, dns, false)) { |
3289 | 0 | status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; |
3290 | 0 | goto invalid; |
3291 | 0 | } |
3292 | 0 | } |
3293 | 0 | if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && |
3294 | 0 | NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { |
3295 | 0 | if ((prinfor & NVME_PRINFO_PRACT) != |
3296 | 0 | (prinfow & NVME_PRINFO_PRACT)) { |
3297 | 0 | status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; |
3298 | 0 | goto invalid; |
3299 | 0 | } else { |
3300 | 0 | if (!nvme_copy_matching_ns_format(sns, dns, true)) { |
3301 | 0 | status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; |
3302 | 0 | goto invalid; |
3303 | 0 | } |
3304 | 0 | } |
3305 | 0 | } |
3306 | | |
3307 | 0 | if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && |
3308 | 0 | NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { |
3309 | 0 | if (!(prinfow & NVME_PRINFO_PRACT)) { |
3310 | 0 | status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; |
3311 | 0 | goto invalid; |
3312 | 0 | } else { |
3313 | 0 | if (!nvme_copy_corresp_pi_format(sns, dns, false)) { |
3314 | 0 | status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; |
3315 | 0 | goto invalid; |
3316 | 0 | } |
3317 | 0 | } |
3318 | 0 | } |
3319 | | |
3320 | 0 | if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && |
3321 | 0 | !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { |
3322 | 0 | if (!(prinfor & NVME_PRINFO_PRACT)) { |
3323 | 0 | status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; |
3324 | 0 | goto invalid; |
3325 | 0 | } else { |
3326 | 0 | if (!nvme_copy_corresp_pi_format(sns, dns, true)) { |
3327 | 0 | status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; |
3328 | 0 | goto invalid; |
3329 | 0 | } |
3330 | 0 | } |
3331 | 0 | } |
3332 | 0 | } |
3333 | 0 | len = nvme_l2b(sns, nlb); |
3334 | |
|
3335 | 0 | trace_pci_nvme_copy_source_range(slba, nlb); |
3336 | |
|
3337 | 0 | if (nlb > le16_to_cpu(sns->id_ns.mssrl)) { |
3338 | 0 | status = NVME_CMD_SIZE_LIMIT | NVME_DNR; |
3339 | 0 | goto invalid; |
3340 | 0 | } |
3341 | | |
3342 | 0 | status = nvme_check_bounds(sns, slba, nlb); |
3343 | 0 | if (status) { |
3344 | 0 | goto invalid; |
3345 | 0 | } |
3346 | | |
3347 | 0 | if (NVME_ERR_REC_DULBE(sns->features.err_rec)) { |
3348 | 0 | status = nvme_check_dulbe(sns, slba, nlb); |
3349 | 0 | if (status) { |
3350 | 0 | goto invalid; |
3351 | 0 | } |
3352 | 0 | } |
3353 | | |
3354 | 0 | if (sns->params.zoned) { |
3355 | 0 | status = nvme_check_zone_read(sns, slba, nlb); |
3356 | 0 | if (status) { |
3357 | 0 | goto invalid; |
3358 | 0 | } |
3359 | 0 | } |
3360 | | |
3361 | 0 | g_free(iocb->bounce); |
3362 | 0 | assert(g_size_checked_mul(&blen, le16_to_cpu(sns->id_ns.mssrl), |
3363 | 0 | sns->lbasz + MAX(sns->lbaf.ms, dns->lbaf.ms))); |
3364 | |
|
3365 | 0 | iocb->bounce = g_malloc(blen); |
3366 | |
|
3367 | 0 | qemu_iovec_reset(&iocb->iov); |
3368 | 0 | assert(len <= blen); |
3369 | 0 | qemu_iovec_add(&iocb->iov, iocb->bounce, len); |
3370 | |
|
3371 | 0 | block_acct_start(blk_get_stats(sns->blkconf.blk), &iocb->acct.read, len, |
3372 | 0 | BLOCK_ACCT_READ); |
3373 | |
|
3374 | 0 | iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_l2b(sns, slba), |
3375 | 0 | &iocb->iov, 0, nvme_copy_in_cb, iocb); |
3376 | 0 | return; |
3377 | | |
3378 | 0 | invalid: |
3379 | 0 | req->status = status; |
3380 | 0 | iocb->ret = -1; |
3381 | 0 | done: |
3382 | 0 | nvme_copy_done(iocb); |
3383 | 0 | } |
3384 | | |
3385 | | static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req) |
3386 | 0 | { |
3387 | 0 | NvmeNamespace *ns = req->ns; |
3388 | 0 | NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd; |
3389 | 0 | NvmeCopyAIOCB *iocb = blk_aio_get(&nvme_copy_aiocb_info, ns->blkconf.blk, |
3390 | 0 | nvme_misc_cb, req); |
3391 | 0 | uint16_t nr = copy->nr + 1; |
3392 | 0 | uint8_t format = copy->control[0] & 0xf; |
3393 | 0 | size_t len = sizeof(NvmeCopySourceRangeFormat0_2); |
3394 | |
|
3395 | 0 | uint16_t status; |
3396 | |
|
3397 | 0 | trace_pci_nvme_copy(nvme_cid(req), nvme_nsid(ns), nr, format); |
3398 | |
|
3399 | 0 | iocb->ranges = NULL; |
3400 | 0 | iocb->zone = NULL; |
3401 | |
|
3402 | 0 | if (!(n->id_ctrl.ocfs & (1 << format)) || |
3403 | 0 | ((format == 2 || format == 3) && |
3404 | 0 | !(n->features.hbs.cdfe & (1 << format)))) { |
3405 | 0 | trace_pci_nvme_err_copy_invalid_format(format); |
3406 | 0 | status = NVME_INVALID_FIELD | NVME_DNR; |
3407 | 0 | goto invalid; |
3408 | 0 | } |
3409 | | |
3410 | 0 | if (nr > ns->id_ns.msrc + 1) { |
3411 | 0 | status = NVME_CMD_SIZE_LIMIT | NVME_DNR; |
3412 | 0 | goto invalid; |
3413 | 0 | } |
3414 | | |
3415 | 0 | if ((ns->pif == 0x0 && (format != 0x0 && format != 0x2)) || |
3416 | 0 | (ns->pif != 0x0 && (format != 0x1 && format != 0x3))) { |
3417 | 0 | status = NVME_INVALID_FORMAT | NVME_DNR; |
3418 | 0 | goto invalid; |
3419 | 0 | } |
3420 | | |
3421 | 0 | if (ns->pif) { |
3422 | 0 | len = sizeof(NvmeCopySourceRangeFormat1_3); |
3423 | 0 | } |
3424 | |
|
3425 | 0 | iocb->format = format; |
3426 | 0 | iocb->ranges = g_malloc_n(nr, len); |
3427 | 0 | status = nvme_h2c(n, (uint8_t *)iocb->ranges, len * nr, req); |
3428 | 0 | if (status) { |
3429 | 0 | goto invalid; |
3430 | 0 | } |
3431 | | |
3432 | 0 | iocb->slba = le64_to_cpu(copy->sdlba); |
3433 | |
|
3434 | 0 | if (ns->params.zoned) { |
3435 | 0 | iocb->zone = nvme_get_zone_by_slba(ns, iocb->slba); |
3436 | 0 | if (!iocb->zone) { |
3437 | 0 | status = NVME_LBA_RANGE | NVME_DNR; |
3438 | 0 | goto invalid; |
3439 | 0 | } |
3440 | | |
3441 | 0 | status = nvme_zrm_auto(n, ns, iocb->zone); |
3442 | 0 | if (status) { |
3443 | 0 | goto invalid; |
3444 | 0 | } |
3445 | 0 | } |
3446 | | |
3447 | 0 | status = nvme_check_copy_mcl(ns, iocb, nr); |
3448 | 0 | if (status) { |
3449 | 0 | goto invalid; |
3450 | 0 | } |
3451 | | |
3452 | 0 | iocb->req = req; |
3453 | 0 | iocb->ret = 0; |
3454 | 0 | iocb->nr = nr; |
3455 | 0 | iocb->idx = 0; |
3456 | 0 | iocb->reftag = le32_to_cpu(copy->reftag); |
3457 | 0 | iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32; |
3458 | |
|
3459 | 0 | qemu_iovec_init(&iocb->iov, 1); |
3460 | |
|
3461 | 0 | req->aiocb = &iocb->common; |
3462 | 0 | iocb->sns = req->ns; |
3463 | 0 | iocb->n = n; |
3464 | 0 | iocb->bounce = NULL; |
3465 | 0 | nvme_do_copy(iocb); |
3466 | |
|
3467 | 0 | return NVME_NO_COMPLETE; |
3468 | | |
3469 | 0 | invalid: |
3470 | 0 | g_free(iocb->ranges); |
3471 | 0 | qemu_aio_unref(iocb); |
3472 | 0 | return status; |
3473 | 0 | } |
3474 | | |
3475 | | static uint16_t nvme_compare(NvmeCtrl *n, NvmeRequest *req) |
3476 | 0 | { |
3477 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
3478 | 0 | NvmeNamespace *ns = req->ns; |
3479 | 0 | BlockBackend *blk = ns->blkconf.blk; |
3480 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
3481 | 0 | uint32_t nlb = le16_to_cpu(rw->nlb) + 1; |
3482 | 0 | uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); |
3483 | 0 | size_t data_len = nvme_l2b(ns, nlb); |
3484 | 0 | size_t len = data_len; |
3485 | 0 | int64_t offset = nvme_l2b(ns, slba); |
3486 | 0 | struct nvme_compare_ctx *ctx = NULL; |
3487 | 0 | uint16_t status; |
3488 | |
|
3489 | 0 | trace_pci_nvme_compare(nvme_cid(req), nvme_nsid(ns), slba, nlb); |
3490 | |
|
3491 | 0 | if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) && (prinfo & NVME_PRINFO_PRACT)) { |
3492 | 0 | return NVME_INVALID_PROT_INFO | NVME_DNR; |
3493 | 0 | } |
3494 | | |
3495 | 0 | if (nvme_ns_ext(ns)) { |
3496 | 0 | len += nvme_m2b(ns, nlb); |
3497 | 0 | } |
3498 | |
|
3499 | 0 | if (NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt)) { |
3500 | 0 | status = nvme_check_mdts(n, data_len); |
3501 | 0 | } else { |
3502 | 0 | status = nvme_check_mdts(n, len); |
3503 | 0 | } |
3504 | 0 | if (status) { |
3505 | 0 | return status; |
3506 | 0 | } |
3507 | | |
3508 | 0 | status = nvme_check_bounds(ns, slba, nlb); |
3509 | 0 | if (status) { |
3510 | 0 | return status; |
3511 | 0 | } |
3512 | | |
3513 | 0 | if (NVME_ERR_REC_DULBE(ns->features.err_rec)) { |
3514 | 0 | status = nvme_check_dulbe(ns, slba, nlb); |
3515 | 0 | if (status) { |
3516 | 0 | return status; |
3517 | 0 | } |
3518 | 0 | } |
3519 | | |
3520 | 0 | status = nvme_map_dptr(n, &req->sg, len, &req->cmd); |
3521 | 0 | if (status) { |
3522 | 0 | return status; |
3523 | 0 | } |
3524 | | |
3525 | 0 | ctx = g_new(struct nvme_compare_ctx, 1); |
3526 | 0 | ctx->data.bounce = g_malloc(data_len); |
3527 | |
|
3528 | 0 | req->opaque = ctx; |
3529 | |
|
3530 | 0 | qemu_iovec_init(&ctx->data.iov, 1); |
3531 | 0 | qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, data_len); |
3532 | |
|
3533 | 0 | block_acct_start(blk_get_stats(blk), &req->acct, data_len, |
3534 | 0 | BLOCK_ACCT_READ); |
3535 | 0 | req->aiocb = blk_aio_preadv(blk, offset, &ctx->data.iov, 0, |
3536 | 0 | nvme_compare_data_cb, req); |
3537 | |
|
3538 | 0 | return NVME_NO_COMPLETE; |
3539 | 0 | } |
3540 | | |
3541 | | typedef struct NvmeFlushAIOCB { |
3542 | | BlockAIOCB common; |
3543 | | BlockAIOCB *aiocb; |
3544 | | NvmeRequest *req; |
3545 | | int ret; |
3546 | | |
3547 | | NvmeNamespace *ns; |
3548 | | uint32_t nsid; |
3549 | | bool broadcast; |
3550 | | } NvmeFlushAIOCB; |
3551 | | |
3552 | | static void nvme_flush_cancel(BlockAIOCB *acb) |
3553 | 0 | { |
3554 | 0 | NvmeFlushAIOCB *iocb = container_of(acb, NvmeFlushAIOCB, common); |
3555 | |
|
3556 | 0 | iocb->ret = -ECANCELED; |
3557 | |
|
3558 | 0 | if (iocb->aiocb) { |
3559 | 0 | blk_aio_cancel_async(iocb->aiocb); |
3560 | 0 | iocb->aiocb = NULL; |
3561 | 0 | } |
3562 | 0 | } |
3563 | | |
3564 | | static const AIOCBInfo nvme_flush_aiocb_info = { |
3565 | | .aiocb_size = sizeof(NvmeFlushAIOCB), |
3566 | | .cancel_async = nvme_flush_cancel, |
3567 | | }; |
3568 | | |
3569 | | static void nvme_do_flush(NvmeFlushAIOCB *iocb); |
3570 | | |
3571 | | static void nvme_flush_ns_cb(void *opaque, int ret) |
3572 | 0 | { |
3573 | 0 | NvmeFlushAIOCB *iocb = opaque; |
3574 | 0 | NvmeNamespace *ns = iocb->ns; |
3575 | |
|
3576 | 0 | if (ret < 0) { |
3577 | 0 | iocb->ret = ret; |
3578 | 0 | iocb->req->status = NVME_WRITE_FAULT; |
3579 | 0 | goto out; |
3580 | 0 | } else if (iocb->ret < 0) { |
3581 | 0 | goto out; |
3582 | 0 | } |
3583 | | |
3584 | 0 | if (ns) { |
3585 | 0 | trace_pci_nvme_flush_ns(iocb->nsid); |
3586 | |
|
3587 | 0 | iocb->ns = NULL; |
3588 | 0 | iocb->aiocb = blk_aio_flush(ns->blkconf.blk, nvme_flush_ns_cb, iocb); |
3589 | 0 | return; |
3590 | 0 | } |
3591 | | |
3592 | 0 | out: |
3593 | 0 | nvme_do_flush(iocb); |
3594 | 0 | } |
3595 | | |
3596 | | static void nvme_do_flush(NvmeFlushAIOCB *iocb) |
3597 | 0 | { |
3598 | 0 | NvmeRequest *req = iocb->req; |
3599 | 0 | NvmeCtrl *n = nvme_ctrl(req); |
3600 | 0 | int i; |
3601 | |
|
3602 | 0 | if (iocb->ret < 0) { |
3603 | 0 | goto done; |
3604 | 0 | } |
3605 | | |
3606 | 0 | if (iocb->broadcast) { |
3607 | 0 | for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) { |
3608 | 0 | iocb->ns = nvme_ns(n, i); |
3609 | 0 | if (iocb->ns) { |
3610 | 0 | iocb->nsid = i; |
3611 | 0 | break; |
3612 | 0 | } |
3613 | 0 | } |
3614 | 0 | } |
3615 | |
|
3616 | 0 | if (!iocb->ns) { |
3617 | 0 | goto done; |
3618 | 0 | } |
3619 | | |
3620 | 0 | nvme_flush_ns_cb(iocb, 0); |
3621 | 0 | return; |
3622 | | |
3623 | 0 | done: |
3624 | 0 | iocb->common.cb(iocb->common.opaque, iocb->ret); |
3625 | 0 | qemu_aio_unref(iocb); |
3626 | 0 | } |
3627 | | |
3628 | | static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req) |
3629 | 0 | { |
3630 | 0 | NvmeFlushAIOCB *iocb; |
3631 | 0 | uint32_t nsid = le32_to_cpu(req->cmd.nsid); |
3632 | 0 | uint16_t status; |
3633 | |
|
3634 | 0 | iocb = qemu_aio_get(&nvme_flush_aiocb_info, NULL, nvme_misc_cb, req); |
3635 | |
|
3636 | 0 | iocb->req = req; |
3637 | 0 | iocb->ret = 0; |
3638 | 0 | iocb->ns = NULL; |
3639 | 0 | iocb->nsid = 0; |
3640 | 0 | iocb->broadcast = (nsid == NVME_NSID_BROADCAST); |
3641 | |
|
3642 | 0 | if (!iocb->broadcast) { |
3643 | 0 | if (!nvme_nsid_valid(n, nsid)) { |
3644 | 0 | status = NVME_INVALID_NSID | NVME_DNR; |
3645 | 0 | goto out; |
3646 | 0 | } |
3647 | | |
3648 | 0 | iocb->ns = nvme_ns(n, nsid); |
3649 | 0 | if (!iocb->ns) { |
3650 | 0 | status = NVME_INVALID_FIELD | NVME_DNR; |
3651 | 0 | goto out; |
3652 | 0 | } |
3653 | | |
3654 | 0 | iocb->nsid = nsid; |
3655 | 0 | } |
3656 | | |
3657 | 0 | req->aiocb = &iocb->common; |
3658 | 0 | nvme_do_flush(iocb); |
3659 | |
|
3660 | 0 | return NVME_NO_COMPLETE; |
3661 | | |
3662 | 0 | out: |
3663 | 0 | qemu_aio_unref(iocb); |
3664 | |
|
3665 | 0 | return status; |
3666 | 0 | } |
3667 | | |
3668 | | static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req) |
3669 | 0 | { |
3670 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
3671 | 0 | NvmeNamespace *ns = req->ns; |
3672 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
3673 | 0 | uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1; |
3674 | 0 | uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); |
3675 | 0 | uint64_t data_size = nvme_l2b(ns, nlb); |
3676 | 0 | uint64_t mapped_size = data_size; |
3677 | 0 | uint64_t data_offset; |
3678 | 0 | BlockBackend *blk = ns->blkconf.blk; |
3679 | 0 | uint16_t status; |
3680 | |
|
3681 | 0 | if (nvme_ns_ext(ns) && !(NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt))) { |
3682 | 0 | mapped_size += nvme_m2b(ns, nlb); |
3683 | |
|
3684 | 0 | if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { |
3685 | 0 | bool pract = prinfo & NVME_PRINFO_PRACT; |
3686 | |
|
3687 | 0 | if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) { |
3688 | 0 | mapped_size = data_size; |
3689 | 0 | } |
3690 | 0 | } |
3691 | 0 | } |
3692 | |
|
3693 | 0 | trace_pci_nvme_read(nvme_cid(req), nvme_nsid(ns), nlb, mapped_size, slba); |
3694 | |
|
3695 | 0 | status = nvme_check_mdts(n, mapped_size); |
3696 | 0 | if (status) { |
3697 | 0 | goto invalid; |
3698 | 0 | } |
3699 | | |
3700 | 0 | status = nvme_check_bounds(ns, slba, nlb); |
3701 | 0 | if (status) { |
3702 | 0 | goto invalid; |
3703 | 0 | } |
3704 | | |
3705 | 0 | if (ns->params.zoned) { |
3706 | 0 | status = nvme_check_zone_read(ns, slba, nlb); |
3707 | 0 | if (status) { |
3708 | 0 | trace_pci_nvme_err_zone_read_not_ok(slba, nlb, status); |
3709 | 0 | goto invalid; |
3710 | 0 | } |
3711 | 0 | } |
3712 | | |
3713 | 0 | if (NVME_ERR_REC_DULBE(ns->features.err_rec)) { |
3714 | 0 | status = nvme_check_dulbe(ns, slba, nlb); |
3715 | 0 | if (status) { |
3716 | 0 | goto invalid; |
3717 | 0 | } |
3718 | 0 | } |
3719 | | |
3720 | 0 | if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { |
3721 | 0 | return nvme_dif_rw(n, req); |
3722 | 0 | } |
3723 | | |
3724 | 0 | status = nvme_map_data(n, nlb, req); |
3725 | 0 | if (status) { |
3726 | 0 | goto invalid; |
3727 | 0 | } |
3728 | | |
3729 | 0 | data_offset = nvme_l2b(ns, slba); |
3730 | |
|
3731 | 0 | block_acct_start(blk_get_stats(blk), &req->acct, data_size, |
3732 | 0 | BLOCK_ACCT_READ); |
3733 | 0 | nvme_blk_read(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req); |
3734 | 0 | return NVME_NO_COMPLETE; |
3735 | | |
3736 | 0 | invalid: |
3737 | 0 | block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ); |
3738 | 0 | return status | NVME_DNR; |
3739 | 0 | } |
3740 | | |
3741 | | static void nvme_do_write_fdp(NvmeCtrl *n, NvmeRequest *req, uint64_t slba, |
3742 | | uint32_t nlb) |
3743 | 0 | { |
3744 | 0 | NvmeNamespace *ns = req->ns; |
3745 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
3746 | 0 | uint64_t data_size = nvme_l2b(ns, nlb); |
3747 | 0 | uint32_t dw12 = le32_to_cpu(req->cmd.cdw12); |
3748 | 0 | uint8_t dtype = (dw12 >> 20) & 0xf; |
3749 | 0 | uint16_t pid = le16_to_cpu(rw->dspec); |
3750 | 0 | uint16_t ph, rg, ruhid; |
3751 | 0 | NvmeReclaimUnit *ru; |
3752 | |
|
3753 | 0 | if (dtype != NVME_DIRECTIVE_DATA_PLACEMENT || |
3754 | 0 | !nvme_parse_pid(ns, pid, &ph, &rg)) { |
3755 | 0 | ph = 0; |
3756 | 0 | rg = 0; |
3757 | 0 | pid = 0; |
3758 | 0 | } |
3759 | |
|
3760 | 0 | ruhid = ns->fdp.phs[ph]; |
3761 | 0 | ru = &ns->endgrp->fdp.ruhs[ruhid].rus[rg]; |
3762 | |
|
3763 | 0 | nvme_fdp_stat_inc(&ns->endgrp->fdp.hbmw, data_size); |
3764 | 0 | nvme_fdp_stat_inc(&ns->endgrp->fdp.mbmw, data_size); |
3765 | |
|
3766 | 0 | while (nlb) { |
3767 | 0 | if (nlb < ru->ruamw) { |
3768 | 0 | ru->ruamw -= nlb; |
3769 | 0 | break; |
3770 | 0 | } |
3771 | | |
3772 | 0 | nlb -= ru->ruamw; |
3773 | 0 | nvme_update_ruh(n, ns, pid); |
3774 | 0 | } |
3775 | 0 | } |
3776 | | |
3777 | | static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append, |
3778 | | bool wrz) |
3779 | 0 | { |
3780 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; |
3781 | 0 | NvmeNamespace *ns = req->ns; |
3782 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
3783 | 0 | uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1; |
3784 | 0 | uint16_t ctrl = le16_to_cpu(rw->control); |
3785 | 0 | uint8_t prinfo = NVME_RW_PRINFO(ctrl); |
3786 | 0 | uint64_t data_size = nvme_l2b(ns, nlb); |
3787 | 0 | uint64_t mapped_size = data_size; |
3788 | 0 | uint64_t data_offset; |
3789 | 0 | NvmeZone *zone; |
3790 | 0 | NvmeZonedResult *res = (NvmeZonedResult *)&req->cqe; |
3791 | 0 | BlockBackend *blk = ns->blkconf.blk; |
3792 | 0 | uint16_t status; |
3793 | |
|
3794 | 0 | if (nvme_ns_ext(ns) && !(NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt))) { |
3795 | 0 | mapped_size += nvme_m2b(ns, nlb); |
3796 | |
|
3797 | 0 | if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { |
3798 | 0 | bool pract = prinfo & NVME_PRINFO_PRACT; |
3799 | |
|
3800 | 0 | if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) { |
3801 | 0 | mapped_size -= nvme_m2b(ns, nlb); |
3802 | 0 | } |
3803 | 0 | } |
3804 | 0 | } |
3805 | |
|
3806 | 0 | trace_pci_nvme_write(nvme_cid(req), nvme_io_opc_str(rw->opcode), |
3807 | 0 | nvme_nsid(ns), nlb, mapped_size, slba); |
3808 | |
|
3809 | 0 | if (!wrz) { |
3810 | 0 | status = nvme_check_mdts(n, mapped_size); |
3811 | 0 | if (status) { |
3812 | 0 | goto invalid; |
3813 | 0 | } |
3814 | 0 | } |
3815 | | |
3816 | 0 | status = nvme_check_bounds(ns, slba, nlb); |
3817 | 0 | if (status) { |
3818 | 0 | goto invalid; |
3819 | 0 | } |
3820 | | |
3821 | 0 | if (ns->params.zoned) { |
3822 | 0 | zone = nvme_get_zone_by_slba(ns, slba); |
3823 | 0 | assert(zone); |
3824 | |
|
3825 | 0 | if (append) { |
3826 | 0 | bool piremap = !!(ctrl & NVME_RW_PIREMAP); |
3827 | |
|
3828 | 0 | if (unlikely(zone->d.za & NVME_ZA_ZRWA_VALID)) { |
3829 | 0 | return NVME_INVALID_ZONE_OP | NVME_DNR; |
3830 | 0 | } |
3831 | | |
3832 | 0 | if (unlikely(slba != zone->d.zslba)) { |
3833 | 0 | trace_pci_nvme_err_append_not_at_start(slba, zone->d.zslba); |
3834 | 0 | status = NVME_INVALID_FIELD; |
3835 | 0 | goto invalid; |
3836 | 0 | } |
3837 | | |
3838 | 0 | if (n->params.zasl && |
3839 | 0 | data_size > (uint64_t)n->page_size << n->params.zasl) { |
3840 | 0 | trace_pci_nvme_err_zasl(data_size); |
3841 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
3842 | 0 | } |
3843 | | |
3844 | 0 | slba = zone->w_ptr; |
3845 | 0 | rw->slba = cpu_to_le64(slba); |
3846 | 0 | res->slba = cpu_to_le64(slba); |
3847 | |
|
3848 | 0 | switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { |
3849 | 0 | case NVME_ID_NS_DPS_TYPE_1: |
3850 | 0 | if (!piremap) { |
3851 | 0 | return NVME_INVALID_PROT_INFO | NVME_DNR; |
3852 | 0 | } |
3853 | | |
3854 | | /* fallthrough */ |
3855 | | |
3856 | 0 | case NVME_ID_NS_DPS_TYPE_2: |
3857 | 0 | if (piremap) { |
3858 | 0 | uint32_t reftag = le32_to_cpu(rw->reftag); |
3859 | 0 | rw->reftag = cpu_to_le32(reftag + (slba - zone->d.zslba)); |
3860 | 0 | } |
3861 | |
|
3862 | 0 | break; |
3863 | | |
3864 | 0 | case NVME_ID_NS_DPS_TYPE_3: |
3865 | 0 | if (piremap) { |
3866 | 0 | return NVME_INVALID_PROT_INFO | NVME_DNR; |
3867 | 0 | } |
3868 | | |
3869 | 0 | break; |
3870 | 0 | } |
3871 | 0 | } |
3872 | | |
3873 | 0 | status = nvme_check_zone_write(ns, zone, slba, nlb); |
3874 | 0 | if (status) { |
3875 | 0 | goto invalid; |
3876 | 0 | } |
3877 | | |
3878 | 0 | status = nvme_zrm_auto(n, ns, zone); |
3879 | 0 | if (status) { |
3880 | 0 | goto invalid; |
3881 | 0 | } |
3882 | | |
3883 | 0 | if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) { |
3884 | 0 | zone->w_ptr += nlb; |
3885 | 0 | } |
3886 | 0 | } else if (ns->endgrp && ns->endgrp->fdp.enabled) { |
3887 | 0 | nvme_do_write_fdp(n, req, slba, nlb); |
3888 | 0 | } |
3889 | | |
3890 | 0 | data_offset = nvme_l2b(ns, slba); |
3891 | |
|
3892 | 0 | if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { |
3893 | 0 | return nvme_dif_rw(n, req); |
3894 | 0 | } |
3895 | | |
3896 | 0 | if (!wrz) { |
3897 | 0 | status = nvme_map_data(n, nlb, req); |
3898 | 0 | if (status) { |
3899 | 0 | goto invalid; |
3900 | 0 | } |
3901 | | |
3902 | 0 | block_acct_start(blk_get_stats(blk), &req->acct, data_size, |
3903 | 0 | BLOCK_ACCT_WRITE); |
3904 | 0 | nvme_blk_write(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req); |
3905 | 0 | } else { |
3906 | 0 | req->aiocb = blk_aio_pwrite_zeroes(blk, data_offset, data_size, |
3907 | 0 | BDRV_REQ_MAY_UNMAP, nvme_rw_cb, |
3908 | 0 | req); |
3909 | 0 | } |
3910 | | |
3911 | 0 | return NVME_NO_COMPLETE; |
3912 | | |
3913 | 0 | invalid: |
3914 | 0 | block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE); |
3915 | 0 | return status | NVME_DNR; |
3916 | 0 | } |
3917 | | |
3918 | | static inline uint16_t nvme_write(NvmeCtrl *n, NvmeRequest *req) |
3919 | 0 | { |
3920 | 0 | return nvme_do_write(n, req, false, false); |
3921 | 0 | } |
3922 | | |
3923 | | static inline uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req) |
3924 | 0 | { |
3925 | 0 | return nvme_do_write(n, req, false, true); |
3926 | 0 | } |
3927 | | |
3928 | | static inline uint16_t nvme_zone_append(NvmeCtrl *n, NvmeRequest *req) |
3929 | 0 | { |
3930 | 0 | return nvme_do_write(n, req, true, false); |
3931 | 0 | } |
3932 | | |
3933 | | static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace *ns, NvmeCmd *c, |
3934 | | uint64_t *slba, uint32_t *zone_idx) |
3935 | 0 | { |
3936 | 0 | uint32_t dw10 = le32_to_cpu(c->cdw10); |
3937 | 0 | uint32_t dw11 = le32_to_cpu(c->cdw11); |
3938 | |
|
3939 | 0 | if (!ns->params.zoned) { |
3940 | 0 | trace_pci_nvme_err_invalid_opc(c->opcode); |
3941 | 0 | return NVME_INVALID_OPCODE | NVME_DNR; |
3942 | 0 | } |
3943 | | |
3944 | 0 | *slba = ((uint64_t)dw11) << 32 | dw10; |
3945 | 0 | if (unlikely(*slba >= ns->id_ns.nsze)) { |
3946 | 0 | trace_pci_nvme_err_invalid_lba_range(*slba, 0, ns->id_ns.nsze); |
3947 | 0 | *slba = 0; |
3948 | 0 | return NVME_LBA_RANGE | NVME_DNR; |
3949 | 0 | } |
3950 | | |
3951 | 0 | *zone_idx = nvme_zone_idx(ns, *slba); |
3952 | 0 | assert(*zone_idx < ns->num_zones); |
3953 | |
|
3954 | 0 | return NVME_SUCCESS; |
3955 | 0 | } |
3956 | | |
3957 | | typedef uint16_t (*op_handler_t)(NvmeNamespace *, NvmeZone *, NvmeZoneState, |
3958 | | NvmeRequest *); |
3959 | | |
3960 | | enum NvmeZoneProcessingMask { |
3961 | | NVME_PROC_CURRENT_ZONE = 0, |
3962 | | NVME_PROC_OPENED_ZONES = 1 << 0, |
3963 | | NVME_PROC_CLOSED_ZONES = 1 << 1, |
3964 | | NVME_PROC_READ_ONLY_ZONES = 1 << 2, |
3965 | | NVME_PROC_FULL_ZONES = 1 << 3, |
3966 | | }; |
3967 | | |
3968 | | static uint16_t nvme_open_zone(NvmeNamespace *ns, NvmeZone *zone, |
3969 | | NvmeZoneState state, NvmeRequest *req) |
3970 | 0 | { |
3971 | 0 | NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd; |
3972 | 0 | int flags = 0; |
3973 | |
|
3974 | 0 | if (cmd->zsflags & NVME_ZSFLAG_ZRWA_ALLOC) { |
3975 | 0 | uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs); |
3976 | |
|
3977 | 0 | if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) { |
3978 | 0 | return NVME_INVALID_ZONE_OP | NVME_DNR; |
3979 | 0 | } |
3980 | | |
3981 | 0 | if (zone->w_ptr % ns->zns.zrwafg) { |
3982 | 0 | return NVME_NOZRWA | NVME_DNR; |
3983 | 0 | } |
3984 | | |
3985 | 0 | flags = NVME_ZRM_ZRWA; |
3986 | 0 | } |
3987 | | |
3988 | 0 | return nvme_zrm_open_flags(nvme_ctrl(req), ns, zone, flags); |
3989 | 0 | } |
3990 | | |
3991 | | static uint16_t nvme_close_zone(NvmeNamespace *ns, NvmeZone *zone, |
3992 | | NvmeZoneState state, NvmeRequest *req) |
3993 | 0 | { |
3994 | 0 | return nvme_zrm_close(ns, zone); |
3995 | 0 | } |
3996 | | |
3997 | | static uint16_t nvme_finish_zone(NvmeNamespace *ns, NvmeZone *zone, |
3998 | | NvmeZoneState state, NvmeRequest *req) |
3999 | 0 | { |
4000 | 0 | return nvme_zrm_finish(ns, zone); |
4001 | 0 | } |
4002 | | |
4003 | | static uint16_t nvme_offline_zone(NvmeNamespace *ns, NvmeZone *zone, |
4004 | | NvmeZoneState state, NvmeRequest *req) |
4005 | 0 | { |
4006 | 0 | switch (state) { |
4007 | 0 | case NVME_ZONE_STATE_READ_ONLY: |
4008 | 0 | nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_OFFLINE); |
4009 | | /* fall through */ |
4010 | 0 | case NVME_ZONE_STATE_OFFLINE: |
4011 | 0 | return NVME_SUCCESS; |
4012 | 0 | default: |
4013 | 0 | return NVME_ZONE_INVAL_TRANSITION; |
4014 | 0 | } |
4015 | 0 | } |
4016 | | |
4017 | | static uint16_t nvme_set_zd_ext(NvmeNamespace *ns, NvmeZone *zone) |
4018 | 0 | { |
4019 | 0 | uint16_t status; |
4020 | 0 | uint8_t state = nvme_get_zone_state(zone); |
4021 | |
|
4022 | 0 | if (state == NVME_ZONE_STATE_EMPTY) { |
4023 | 0 | status = nvme_aor_check(ns, 1, 0); |
4024 | 0 | if (status) { |
4025 | 0 | return status; |
4026 | 0 | } |
4027 | 0 | nvme_aor_inc_active(ns); |
4028 | 0 | zone->d.za |= NVME_ZA_ZD_EXT_VALID; |
4029 | 0 | nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED); |
4030 | 0 | return NVME_SUCCESS; |
4031 | 0 | } |
4032 | | |
4033 | 0 | return NVME_ZONE_INVAL_TRANSITION; |
4034 | 0 | } |
4035 | | |
4036 | | static uint16_t nvme_bulk_proc_zone(NvmeNamespace *ns, NvmeZone *zone, |
4037 | | enum NvmeZoneProcessingMask proc_mask, |
4038 | | op_handler_t op_hndlr, NvmeRequest *req) |
4039 | 0 | { |
4040 | 0 | uint16_t status = NVME_SUCCESS; |
4041 | 0 | NvmeZoneState zs = nvme_get_zone_state(zone); |
4042 | 0 | bool proc_zone; |
4043 | |
|
4044 | 0 | switch (zs) { |
4045 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
4046 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
4047 | 0 | proc_zone = proc_mask & NVME_PROC_OPENED_ZONES; |
4048 | 0 | break; |
4049 | 0 | case NVME_ZONE_STATE_CLOSED: |
4050 | 0 | proc_zone = proc_mask & NVME_PROC_CLOSED_ZONES; |
4051 | 0 | break; |
4052 | 0 | case NVME_ZONE_STATE_READ_ONLY: |
4053 | 0 | proc_zone = proc_mask & NVME_PROC_READ_ONLY_ZONES; |
4054 | 0 | break; |
4055 | 0 | case NVME_ZONE_STATE_FULL: |
4056 | 0 | proc_zone = proc_mask & NVME_PROC_FULL_ZONES; |
4057 | 0 | break; |
4058 | 0 | default: |
4059 | 0 | proc_zone = false; |
4060 | 0 | } |
4061 | | |
4062 | 0 | if (proc_zone) { |
4063 | 0 | status = op_hndlr(ns, zone, zs, req); |
4064 | 0 | } |
4065 | |
|
4066 | 0 | return status; |
4067 | 0 | } |
4068 | | |
4069 | | static uint16_t nvme_do_zone_op(NvmeNamespace *ns, NvmeZone *zone, |
4070 | | enum NvmeZoneProcessingMask proc_mask, |
4071 | | op_handler_t op_hndlr, NvmeRequest *req) |
4072 | 0 | { |
4073 | 0 | NvmeZone *next; |
4074 | 0 | uint16_t status = NVME_SUCCESS; |
4075 | 0 | int i; |
4076 | |
|
4077 | 0 | if (!proc_mask) { |
4078 | 0 | status = op_hndlr(ns, zone, nvme_get_zone_state(zone), req); |
4079 | 0 | } else { |
4080 | 0 | if (proc_mask & NVME_PROC_CLOSED_ZONES) { |
4081 | 0 | QTAILQ_FOREACH_SAFE(zone, &ns->closed_zones, entry, next) { |
4082 | 0 | status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr, |
4083 | 0 | req); |
4084 | 0 | if (status && status != NVME_NO_COMPLETE) { |
4085 | 0 | goto out; |
4086 | 0 | } |
4087 | 0 | } |
4088 | 0 | } |
4089 | 0 | if (proc_mask & NVME_PROC_OPENED_ZONES) { |
4090 | 0 | QTAILQ_FOREACH_SAFE(zone, &ns->imp_open_zones, entry, next) { |
4091 | 0 | status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr, |
4092 | 0 | req); |
4093 | 0 | if (status && status != NVME_NO_COMPLETE) { |
4094 | 0 | goto out; |
4095 | 0 | } |
4096 | 0 | } |
4097 | | |
4098 | 0 | QTAILQ_FOREACH_SAFE(zone, &ns->exp_open_zones, entry, next) { |
4099 | 0 | status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr, |
4100 | 0 | req); |
4101 | 0 | if (status && status != NVME_NO_COMPLETE) { |
4102 | 0 | goto out; |
4103 | 0 | } |
4104 | 0 | } |
4105 | 0 | } |
4106 | 0 | if (proc_mask & NVME_PROC_FULL_ZONES) { |
4107 | 0 | QTAILQ_FOREACH_SAFE(zone, &ns->full_zones, entry, next) { |
4108 | 0 | status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr, |
4109 | 0 | req); |
4110 | 0 | if (status && status != NVME_NO_COMPLETE) { |
4111 | 0 | goto out; |
4112 | 0 | } |
4113 | 0 | } |
4114 | 0 | } |
4115 | | |
4116 | 0 | if (proc_mask & NVME_PROC_READ_ONLY_ZONES) { |
4117 | 0 | for (i = 0; i < ns->num_zones; i++, zone++) { |
4118 | 0 | status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr, |
4119 | 0 | req); |
4120 | 0 | if (status && status != NVME_NO_COMPLETE) { |
4121 | 0 | goto out; |
4122 | 0 | } |
4123 | 0 | } |
4124 | 0 | } |
4125 | 0 | } |
4126 | | |
4127 | 0 | out: |
4128 | 0 | return status; |
4129 | 0 | } |
4130 | | |
4131 | | typedef struct NvmeZoneResetAIOCB { |
4132 | | BlockAIOCB common; |
4133 | | BlockAIOCB *aiocb; |
4134 | | NvmeRequest *req; |
4135 | | int ret; |
4136 | | |
4137 | | bool all; |
4138 | | int idx; |
4139 | | NvmeZone *zone; |
4140 | | } NvmeZoneResetAIOCB; |
4141 | | |
4142 | | static void nvme_zone_reset_cancel(BlockAIOCB *aiocb) |
4143 | 0 | { |
4144 | 0 | NvmeZoneResetAIOCB *iocb = container_of(aiocb, NvmeZoneResetAIOCB, common); |
4145 | 0 | NvmeRequest *req = iocb->req; |
4146 | 0 | NvmeNamespace *ns = req->ns; |
4147 | |
|
4148 | 0 | iocb->idx = ns->num_zones; |
4149 | |
|
4150 | 0 | iocb->ret = -ECANCELED; |
4151 | |
|
4152 | 0 | if (iocb->aiocb) { |
4153 | 0 | blk_aio_cancel_async(iocb->aiocb); |
4154 | 0 | iocb->aiocb = NULL; |
4155 | 0 | } |
4156 | 0 | } |
4157 | | |
4158 | | static const AIOCBInfo nvme_zone_reset_aiocb_info = { |
4159 | | .aiocb_size = sizeof(NvmeZoneResetAIOCB), |
4160 | | .cancel_async = nvme_zone_reset_cancel, |
4161 | | }; |
4162 | | |
4163 | | static void nvme_zone_reset_cb(void *opaque, int ret); |
4164 | | |
4165 | | static void nvme_zone_reset_epilogue_cb(void *opaque, int ret) |
4166 | 0 | { |
4167 | 0 | NvmeZoneResetAIOCB *iocb = opaque; |
4168 | 0 | NvmeRequest *req = iocb->req; |
4169 | 0 | NvmeNamespace *ns = req->ns; |
4170 | 0 | int64_t moff; |
4171 | 0 | int count; |
4172 | |
|
4173 | 0 | if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) { |
4174 | 0 | goto out; |
4175 | 0 | } |
4176 | | |
4177 | 0 | moff = nvme_moff(ns, iocb->zone->d.zslba); |
4178 | 0 | count = nvme_m2b(ns, ns->zone_size); |
4179 | |
|
4180 | 0 | iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, moff, count, |
4181 | 0 | BDRV_REQ_MAY_UNMAP, |
4182 | 0 | nvme_zone_reset_cb, iocb); |
4183 | 0 | return; |
4184 | | |
4185 | 0 | out: |
4186 | 0 | nvme_zone_reset_cb(iocb, ret); |
4187 | 0 | } |
4188 | | |
4189 | | static void nvme_zone_reset_cb(void *opaque, int ret) |
4190 | 0 | { |
4191 | 0 | NvmeZoneResetAIOCB *iocb = opaque; |
4192 | 0 | NvmeRequest *req = iocb->req; |
4193 | 0 | NvmeNamespace *ns = req->ns; |
4194 | |
|
4195 | 0 | if (iocb->ret < 0) { |
4196 | 0 | goto done; |
4197 | 0 | } else if (ret < 0) { |
4198 | 0 | iocb->ret = ret; |
4199 | 0 | goto done; |
4200 | 0 | } |
4201 | | |
4202 | 0 | if (iocb->zone) { |
4203 | 0 | nvme_zrm_reset(ns, iocb->zone); |
4204 | |
|
4205 | 0 | if (!iocb->all) { |
4206 | 0 | goto done; |
4207 | 0 | } |
4208 | 0 | } |
4209 | | |
4210 | 0 | while (iocb->idx < ns->num_zones) { |
4211 | 0 | NvmeZone *zone = &ns->zone_array[iocb->idx++]; |
4212 | |
|
4213 | 0 | switch (nvme_get_zone_state(zone)) { |
4214 | 0 | case NVME_ZONE_STATE_EMPTY: |
4215 | 0 | if (!iocb->all) { |
4216 | 0 | goto done; |
4217 | 0 | } |
4218 | | |
4219 | 0 | continue; |
4220 | | |
4221 | 0 | case NVME_ZONE_STATE_EXPLICITLY_OPEN: |
4222 | 0 | case NVME_ZONE_STATE_IMPLICITLY_OPEN: |
4223 | 0 | case NVME_ZONE_STATE_CLOSED: |
4224 | 0 | case NVME_ZONE_STATE_FULL: |
4225 | 0 | iocb->zone = zone; |
4226 | 0 | break; |
4227 | | |
4228 | 0 | default: |
4229 | 0 | continue; |
4230 | 0 | } |
4231 | | |
4232 | 0 | trace_pci_nvme_zns_zone_reset(zone->d.zslba); |
4233 | |
|
4234 | 0 | iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, |
4235 | 0 | nvme_l2b(ns, zone->d.zslba), |
4236 | 0 | nvme_l2b(ns, ns->zone_size), |
4237 | 0 | BDRV_REQ_MAY_UNMAP, |
4238 | 0 | nvme_zone_reset_epilogue_cb, |
4239 | 0 | iocb); |
4240 | 0 | return; |
4241 | 0 | } |
4242 | | |
4243 | 0 | done: |
4244 | 0 | iocb->aiocb = NULL; |
4245 | |
|
4246 | 0 | iocb->common.cb(iocb->common.opaque, iocb->ret); |
4247 | 0 | qemu_aio_unref(iocb); |
4248 | 0 | } |
4249 | | |
4250 | | static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl *n, NvmeZone *zone, |
4251 | | uint64_t elba, NvmeRequest *req) |
4252 | 0 | { |
4253 | 0 | NvmeNamespace *ns = req->ns; |
4254 | 0 | uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs); |
4255 | 0 | uint64_t wp = zone->d.wp; |
4256 | 0 | uint32_t nlb = elba - wp + 1; |
4257 | 0 | uint16_t status; |
4258 | | |
4259 | |
|
4260 | 0 | if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) { |
4261 | 0 | return NVME_INVALID_ZONE_OP | NVME_DNR; |
4262 | 0 | } |
4263 | | |
4264 | 0 | if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) { |
4265 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4266 | 0 | } |
4267 | | |
4268 | 0 | if (elba < wp || elba > wp + ns->zns.zrwas) { |
4269 | 0 | return NVME_ZONE_BOUNDARY_ERROR | NVME_DNR; |
4270 | 0 | } |
4271 | | |
4272 | 0 | if (nlb % ns->zns.zrwafg) { |
4273 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4274 | 0 | } |
4275 | | |
4276 | 0 | status = nvme_zrm_auto(n, ns, zone); |
4277 | 0 | if (status) { |
4278 | 0 | return status; |
4279 | 0 | } |
4280 | | |
4281 | 0 | zone->w_ptr += nlb; |
4282 | |
|
4283 | 0 | nvme_advance_zone_wp(ns, zone, nlb); |
4284 | |
|
4285 | 0 | return NVME_SUCCESS; |
4286 | 0 | } |
4287 | | |
4288 | | static uint16_t nvme_zone_mgmt_send(NvmeCtrl *n, NvmeRequest *req) |
4289 | 0 | { |
4290 | 0 | NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd; |
4291 | 0 | NvmeNamespace *ns = req->ns; |
4292 | 0 | NvmeZone *zone; |
4293 | 0 | NvmeZoneResetAIOCB *iocb; |
4294 | 0 | uint8_t *zd_ext; |
4295 | 0 | uint64_t slba = 0; |
4296 | 0 | uint32_t zone_idx = 0; |
4297 | 0 | uint16_t status; |
4298 | 0 | uint8_t action = cmd->zsa; |
4299 | 0 | bool all; |
4300 | 0 | enum NvmeZoneProcessingMask proc_mask = NVME_PROC_CURRENT_ZONE; |
4301 | |
|
4302 | 0 | all = cmd->zsflags & NVME_ZSFLAG_SELECT_ALL; |
4303 | |
|
4304 | 0 | req->status = NVME_SUCCESS; |
4305 | |
|
4306 | 0 | if (!all) { |
4307 | 0 | status = nvme_get_mgmt_zone_slba_idx(ns, &req->cmd, &slba, &zone_idx); |
4308 | 0 | if (status) { |
4309 | 0 | return status; |
4310 | 0 | } |
4311 | 0 | } |
4312 | | |
4313 | 0 | zone = &ns->zone_array[zone_idx]; |
4314 | 0 | if (slba != zone->d.zslba && action != NVME_ZONE_ACTION_ZRWA_FLUSH) { |
4315 | 0 | trace_pci_nvme_err_unaligned_zone_cmd(action, slba, zone->d.zslba); |
4316 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4317 | 0 | } |
4318 | | |
4319 | 0 | switch (action) { |
4320 | | |
4321 | 0 | case NVME_ZONE_ACTION_OPEN: |
4322 | 0 | if (all) { |
4323 | 0 | proc_mask = NVME_PROC_CLOSED_ZONES; |
4324 | 0 | } |
4325 | 0 | trace_pci_nvme_open_zone(slba, zone_idx, all); |
4326 | 0 | status = nvme_do_zone_op(ns, zone, proc_mask, nvme_open_zone, req); |
4327 | 0 | break; |
4328 | | |
4329 | 0 | case NVME_ZONE_ACTION_CLOSE: |
4330 | 0 | if (all) { |
4331 | 0 | proc_mask = NVME_PROC_OPENED_ZONES; |
4332 | 0 | } |
4333 | 0 | trace_pci_nvme_close_zone(slba, zone_idx, all); |
4334 | 0 | status = nvme_do_zone_op(ns, zone, proc_mask, nvme_close_zone, req); |
4335 | 0 | break; |
4336 | | |
4337 | 0 | case NVME_ZONE_ACTION_FINISH: |
4338 | 0 | if (all) { |
4339 | 0 | proc_mask = NVME_PROC_OPENED_ZONES | NVME_PROC_CLOSED_ZONES; |
4340 | 0 | } |
4341 | 0 | trace_pci_nvme_finish_zone(slba, zone_idx, all); |
4342 | 0 | status = nvme_do_zone_op(ns, zone, proc_mask, nvme_finish_zone, req); |
4343 | 0 | break; |
4344 | | |
4345 | 0 | case NVME_ZONE_ACTION_RESET: |
4346 | 0 | trace_pci_nvme_reset_zone(slba, zone_idx, all); |
4347 | |
|
4348 | 0 | iocb = blk_aio_get(&nvme_zone_reset_aiocb_info, ns->blkconf.blk, |
4349 | 0 | nvme_misc_cb, req); |
4350 | |
|
4351 | 0 | iocb->req = req; |
4352 | 0 | iocb->ret = 0; |
4353 | 0 | iocb->all = all; |
4354 | 0 | iocb->idx = zone_idx; |
4355 | 0 | iocb->zone = NULL; |
4356 | |
|
4357 | 0 | req->aiocb = &iocb->common; |
4358 | 0 | nvme_zone_reset_cb(iocb, 0); |
4359 | |
|
4360 | 0 | return NVME_NO_COMPLETE; |
4361 | | |
4362 | 0 | case NVME_ZONE_ACTION_OFFLINE: |
4363 | 0 | if (all) { |
4364 | 0 | proc_mask = NVME_PROC_READ_ONLY_ZONES; |
4365 | 0 | } |
4366 | 0 | trace_pci_nvme_offline_zone(slba, zone_idx, all); |
4367 | 0 | status = nvme_do_zone_op(ns, zone, proc_mask, nvme_offline_zone, req); |
4368 | 0 | break; |
4369 | | |
4370 | 0 | case NVME_ZONE_ACTION_SET_ZD_EXT: |
4371 | 0 | trace_pci_nvme_set_descriptor_extension(slba, zone_idx); |
4372 | 0 | if (all || !ns->params.zd_extension_size) { |
4373 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4374 | 0 | } |
4375 | 0 | zd_ext = nvme_get_zd_extension(ns, zone_idx); |
4376 | 0 | status = nvme_h2c(n, zd_ext, ns->params.zd_extension_size, req); |
4377 | 0 | if (status) { |
4378 | 0 | trace_pci_nvme_err_zd_extension_map_error(zone_idx); |
4379 | 0 | return status; |
4380 | 0 | } |
4381 | | |
4382 | 0 | status = nvme_set_zd_ext(ns, zone); |
4383 | 0 | if (status == NVME_SUCCESS) { |
4384 | 0 | trace_pci_nvme_zd_extension_set(zone_idx); |
4385 | 0 | return status; |
4386 | 0 | } |
4387 | 0 | break; |
4388 | | |
4389 | 0 | case NVME_ZONE_ACTION_ZRWA_FLUSH: |
4390 | 0 | if (all) { |
4391 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4392 | 0 | } |
4393 | | |
4394 | 0 | return nvme_zone_mgmt_send_zrwa_flush(n, zone, slba, req); |
4395 | | |
4396 | 0 | default: |
4397 | 0 | trace_pci_nvme_err_invalid_mgmt_action(action); |
4398 | 0 | status = NVME_INVALID_FIELD; |
4399 | 0 | } |
4400 | | |
4401 | 0 | if (status == NVME_ZONE_INVAL_TRANSITION) { |
4402 | 0 | trace_pci_nvme_err_invalid_zone_state_transition(action, slba, |
4403 | 0 | zone->d.za); |
4404 | 0 | } |
4405 | 0 | if (status) { |
4406 | 0 | status |= NVME_DNR; |
4407 | 0 | } |
4408 | |
|
4409 | 0 | return status; |
4410 | 0 | } |
4411 | | |
4412 | | static bool nvme_zone_matches_filter(uint32_t zafs, NvmeZone *zl) |
4413 | 0 | { |
4414 | 0 | NvmeZoneState zs = nvme_get_zone_state(zl); |
4415 | |
|
4416 | 0 | switch (zafs) { |
4417 | 0 | case NVME_ZONE_REPORT_ALL: |
4418 | 0 | return true; |
4419 | 0 | case NVME_ZONE_REPORT_EMPTY: |
4420 | 0 | return zs == NVME_ZONE_STATE_EMPTY; |
4421 | 0 | case NVME_ZONE_REPORT_IMPLICITLY_OPEN: |
4422 | 0 | return zs == NVME_ZONE_STATE_IMPLICITLY_OPEN; |
4423 | 0 | case NVME_ZONE_REPORT_EXPLICITLY_OPEN: |
4424 | 0 | return zs == NVME_ZONE_STATE_EXPLICITLY_OPEN; |
4425 | 0 | case NVME_ZONE_REPORT_CLOSED: |
4426 | 0 | return zs == NVME_ZONE_STATE_CLOSED; |
4427 | 0 | case NVME_ZONE_REPORT_FULL: |
4428 | 0 | return zs == NVME_ZONE_STATE_FULL; |
4429 | 0 | case NVME_ZONE_REPORT_READ_ONLY: |
4430 | 0 | return zs == NVME_ZONE_STATE_READ_ONLY; |
4431 | 0 | case NVME_ZONE_REPORT_OFFLINE: |
4432 | 0 | return zs == NVME_ZONE_STATE_OFFLINE; |
4433 | 0 | default: |
4434 | 0 | return false; |
4435 | 0 | } |
4436 | 0 | } |
4437 | | |
4438 | | static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req) |
4439 | 0 | { |
4440 | 0 | NvmeCmd *cmd = &req->cmd; |
4441 | 0 | NvmeNamespace *ns = req->ns; |
4442 | | /* cdw12 is zero-based number of dwords to return. Convert to bytes */ |
4443 | 0 | uint32_t data_size = (le32_to_cpu(cmd->cdw12) + 1) << 2; |
4444 | 0 | uint32_t dw13 = le32_to_cpu(cmd->cdw13); |
4445 | 0 | uint32_t zone_idx, zra, zrasf, partial; |
4446 | 0 | uint64_t max_zones, nr_zones = 0; |
4447 | 0 | uint16_t status; |
4448 | 0 | uint64_t slba; |
4449 | 0 | NvmeZoneDescr *z; |
4450 | 0 | NvmeZone *zone; |
4451 | 0 | NvmeZoneReportHeader *header; |
4452 | 0 | void *buf, *buf_p; |
4453 | 0 | size_t zone_entry_sz; |
4454 | 0 | int i; |
4455 | |
|
4456 | 0 | req->status = NVME_SUCCESS; |
4457 | |
|
4458 | 0 | status = nvme_get_mgmt_zone_slba_idx(ns, cmd, &slba, &zone_idx); |
4459 | 0 | if (status) { |
4460 | 0 | return status; |
4461 | 0 | } |
4462 | | |
4463 | 0 | zra = dw13 & 0xff; |
4464 | 0 | if (zra != NVME_ZONE_REPORT && zra != NVME_ZONE_REPORT_EXTENDED) { |
4465 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4466 | 0 | } |
4467 | 0 | if (zra == NVME_ZONE_REPORT_EXTENDED && !ns->params.zd_extension_size) { |
4468 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4469 | 0 | } |
4470 | | |
4471 | 0 | zrasf = (dw13 >> 8) & 0xff; |
4472 | 0 | if (zrasf > NVME_ZONE_REPORT_OFFLINE) { |
4473 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4474 | 0 | } |
4475 | | |
4476 | 0 | if (data_size < sizeof(NvmeZoneReportHeader)) { |
4477 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4478 | 0 | } |
4479 | | |
4480 | 0 | status = nvme_check_mdts(n, data_size); |
4481 | 0 | if (status) { |
4482 | 0 | return status; |
4483 | 0 | } |
4484 | | |
4485 | 0 | partial = (dw13 >> 16) & 0x01; |
4486 | |
|
4487 | 0 | zone_entry_sz = sizeof(NvmeZoneDescr); |
4488 | 0 | if (zra == NVME_ZONE_REPORT_EXTENDED) { |
4489 | 0 | zone_entry_sz += ns->params.zd_extension_size; |
4490 | 0 | } |
4491 | |
|
4492 | 0 | max_zones = (data_size - sizeof(NvmeZoneReportHeader)) / zone_entry_sz; |
4493 | 0 | buf = g_malloc0(data_size); |
4494 | |
|
4495 | 0 | zone = &ns->zone_array[zone_idx]; |
4496 | 0 | for (i = zone_idx; i < ns->num_zones; i++) { |
4497 | 0 | if (partial && nr_zones >= max_zones) { |
4498 | 0 | break; |
4499 | 0 | } |
4500 | 0 | if (nvme_zone_matches_filter(zrasf, zone++)) { |
4501 | 0 | nr_zones++; |
4502 | 0 | } |
4503 | 0 | } |
4504 | 0 | header = buf; |
4505 | 0 | header->nr_zones = cpu_to_le64(nr_zones); |
4506 | |
|
4507 | 0 | buf_p = buf + sizeof(NvmeZoneReportHeader); |
4508 | 0 | for (; zone_idx < ns->num_zones && max_zones > 0; zone_idx++) { |
4509 | 0 | zone = &ns->zone_array[zone_idx]; |
4510 | 0 | if (nvme_zone_matches_filter(zrasf, zone)) { |
4511 | 0 | z = buf_p; |
4512 | 0 | buf_p += sizeof(NvmeZoneDescr); |
4513 | |
|
4514 | 0 | z->zt = zone->d.zt; |
4515 | 0 | z->zs = zone->d.zs; |
4516 | 0 | z->zcap = cpu_to_le64(zone->d.zcap); |
4517 | 0 | z->zslba = cpu_to_le64(zone->d.zslba); |
4518 | 0 | z->za = zone->d.za; |
4519 | |
|
4520 | 0 | if (nvme_wp_is_valid(zone)) { |
4521 | 0 | z->wp = cpu_to_le64(zone->d.wp); |
4522 | 0 | } else { |
4523 | 0 | z->wp = cpu_to_le64(~0ULL); |
4524 | 0 | } |
4525 | |
|
4526 | 0 | if (zra == NVME_ZONE_REPORT_EXTENDED) { |
4527 | 0 | if (zone->d.za & NVME_ZA_ZD_EXT_VALID) { |
4528 | 0 | memcpy(buf_p, nvme_get_zd_extension(ns, zone_idx), |
4529 | 0 | ns->params.zd_extension_size); |
4530 | 0 | } |
4531 | 0 | buf_p += ns->params.zd_extension_size; |
4532 | 0 | } |
4533 | |
|
4534 | 0 | max_zones--; |
4535 | 0 | } |
4536 | 0 | } |
4537 | |
|
4538 | 0 | status = nvme_c2h(n, (uint8_t *)buf, data_size, req); |
4539 | |
|
4540 | 0 | g_free(buf); |
4541 | |
|
4542 | 0 | return status; |
4543 | 0 | } |
4544 | | |
4545 | | static uint16_t nvme_io_mgmt_recv_ruhs(NvmeCtrl *n, NvmeRequest *req, |
4546 | | size_t len) |
4547 | 0 | { |
4548 | 0 | NvmeNamespace *ns = req->ns; |
4549 | 0 | NvmeEnduranceGroup *endgrp; |
4550 | 0 | NvmeRuhStatus *hdr; |
4551 | 0 | NvmeRuhStatusDescr *ruhsd; |
4552 | 0 | unsigned int nruhsd; |
4553 | 0 | uint16_t rg, ph, *ruhid; |
4554 | 0 | size_t trans_len; |
4555 | 0 | g_autofree uint8_t *buf = NULL; |
4556 | |
|
4557 | 0 | if (!n->subsys) { |
4558 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4559 | 0 | } |
4560 | | |
4561 | 0 | if (ns->params.nsid == 0 || ns->params.nsid == 0xffffffff) { |
4562 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
4563 | 0 | } |
4564 | | |
4565 | 0 | if (!n->subsys->endgrp.fdp.enabled) { |
4566 | 0 | return NVME_FDP_DISABLED | NVME_DNR; |
4567 | 0 | } |
4568 | | |
4569 | 0 | endgrp = ns->endgrp; |
4570 | |
|
4571 | 0 | nruhsd = ns->fdp.nphs * endgrp->fdp.nrg; |
4572 | 0 | trans_len = sizeof(NvmeRuhStatus) + nruhsd * sizeof(NvmeRuhStatusDescr); |
4573 | 0 | buf = g_malloc0(trans_len); |
4574 | |
|
4575 | 0 | trans_len = MIN(trans_len, len); |
4576 | |
|
4577 | 0 | hdr = (NvmeRuhStatus *)buf; |
4578 | 0 | ruhsd = (NvmeRuhStatusDescr *)(buf + sizeof(NvmeRuhStatus)); |
4579 | |
|
4580 | 0 | hdr->nruhsd = cpu_to_le16(nruhsd); |
4581 | |
|
4582 | 0 | ruhid = ns->fdp.phs; |
4583 | |
|
4584 | 0 | for (ph = 0; ph < ns->fdp.nphs; ph++, ruhid++) { |
4585 | 0 | NvmeRuHandle *ruh = &endgrp->fdp.ruhs[*ruhid]; |
4586 | |
|
4587 | 0 | for (rg = 0; rg < endgrp->fdp.nrg; rg++, ruhsd++) { |
4588 | 0 | uint16_t pid = nvme_make_pid(ns, rg, ph); |
4589 | |
|
4590 | 0 | ruhsd->pid = cpu_to_le16(pid); |
4591 | 0 | ruhsd->ruhid = *ruhid; |
4592 | 0 | ruhsd->earutr = 0; |
4593 | 0 | ruhsd->ruamw = cpu_to_le64(ruh->rus[rg].ruamw); |
4594 | 0 | } |
4595 | 0 | } |
4596 | |
|
4597 | 0 | return nvme_c2h(n, buf, trans_len, req); |
4598 | 0 | } |
4599 | | |
4600 | | static uint16_t nvme_io_mgmt_recv(NvmeCtrl *n, NvmeRequest *req) |
4601 | 0 | { |
4602 | 0 | NvmeCmd *cmd = &req->cmd; |
4603 | 0 | uint32_t cdw10 = le32_to_cpu(cmd->cdw10); |
4604 | 0 | uint32_t numd = le32_to_cpu(cmd->cdw11); |
4605 | 0 | uint8_t mo = (cdw10 & 0xff); |
4606 | 0 | size_t len = (numd + 1) << 2; |
4607 | |
|
4608 | 0 | switch (mo) { |
4609 | 0 | case NVME_IOMR_MO_NOP: |
4610 | 0 | return 0; |
4611 | 0 | case NVME_IOMR_MO_RUH_STATUS: |
4612 | 0 | return nvme_io_mgmt_recv_ruhs(n, req, len); |
4613 | 0 | default: |
4614 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4615 | 0 | }; |
4616 | 0 | } |
4617 | | |
4618 | | static uint16_t nvme_io_mgmt_send_ruh_update(NvmeCtrl *n, NvmeRequest *req) |
4619 | 0 | { |
4620 | 0 | NvmeCmd *cmd = &req->cmd; |
4621 | 0 | NvmeNamespace *ns = req->ns; |
4622 | 0 | uint32_t cdw10 = le32_to_cpu(cmd->cdw10); |
4623 | 0 | uint16_t ret = NVME_SUCCESS; |
4624 | 0 | uint32_t npid = (cdw10 >> 16) + 1; |
4625 | 0 | unsigned int i = 0; |
4626 | 0 | g_autofree uint16_t *pids = NULL; |
4627 | 0 | uint32_t maxnpid; |
4628 | |
|
4629 | 0 | if (!ns->endgrp || !ns->endgrp->fdp.enabled) { |
4630 | 0 | return NVME_FDP_DISABLED | NVME_DNR; |
4631 | 0 | } |
4632 | | |
4633 | 0 | maxnpid = n->subsys->endgrp.fdp.nrg * n->subsys->endgrp.fdp.nruh; |
4634 | |
|
4635 | 0 | if (unlikely(npid >= MIN(NVME_FDP_MAXPIDS, maxnpid))) { |
4636 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4637 | 0 | } |
4638 | | |
4639 | 0 | pids = g_new(uint16_t, npid); |
4640 | |
|
4641 | 0 | ret = nvme_h2c(n, pids, npid * sizeof(uint16_t), req); |
4642 | 0 | if (ret) { |
4643 | 0 | return ret; |
4644 | 0 | } |
4645 | | |
4646 | 0 | for (; i < npid; i++) { |
4647 | 0 | if (!nvme_update_ruh(n, ns, pids[i])) { |
4648 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4649 | 0 | } |
4650 | 0 | } |
4651 | | |
4652 | 0 | return ret; |
4653 | 0 | } |
4654 | | |
4655 | | static uint16_t nvme_io_mgmt_send(NvmeCtrl *n, NvmeRequest *req) |
4656 | 0 | { |
4657 | 0 | NvmeCmd *cmd = &req->cmd; |
4658 | 0 | uint32_t cdw10 = le32_to_cpu(cmd->cdw10); |
4659 | 0 | uint8_t mo = (cdw10 & 0xff); |
4660 | |
|
4661 | 0 | switch (mo) { |
4662 | 0 | case NVME_IOMS_MO_NOP: |
4663 | 0 | return 0; |
4664 | 0 | case NVME_IOMS_MO_RUH_UPDATE: |
4665 | 0 | return nvme_io_mgmt_send_ruh_update(n, req); |
4666 | | /* if you add something here, please update nvme_set_migration_blockers() */ |
4667 | 0 | default: |
4668 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4669 | 0 | }; |
4670 | 0 | } |
4671 | | |
4672 | | static uint16_t __nvme_io_cmd_nvm(NvmeCtrl *n, NvmeRequest *req) |
4673 | 0 | { |
4674 | 0 | switch (req->cmd.opcode) { |
4675 | 0 | case NVME_CMD_WRITE: |
4676 | 0 | return nvme_write(n, req); |
4677 | 0 | case NVME_CMD_READ: |
4678 | 0 | return nvme_read(n, req); |
4679 | 0 | case NVME_CMD_COMPARE: |
4680 | 0 | return nvme_compare(n, req); |
4681 | 0 | case NVME_CMD_WRITE_ZEROES: |
4682 | 0 | return nvme_write_zeroes(n, req); |
4683 | 0 | case NVME_CMD_DSM: |
4684 | 0 | return nvme_dsm(n, req); |
4685 | 0 | case NVME_CMD_VERIFY: |
4686 | 0 | return nvme_verify(n, req); |
4687 | 0 | case NVME_CMD_COPY: |
4688 | 0 | return nvme_copy(n, req); |
4689 | 0 | case NVME_CMD_IO_MGMT_RECV: |
4690 | 0 | return nvme_io_mgmt_recv(n, req); |
4691 | 0 | case NVME_CMD_IO_MGMT_SEND: |
4692 | 0 | return nvme_io_mgmt_send(n, req); |
4693 | 0 | } |
4694 | | |
4695 | 0 | g_assert_not_reached(); |
4696 | 0 | } |
4697 | | |
4698 | | static uint16_t nvme_io_cmd_nvm(NvmeCtrl *n, NvmeRequest *req) |
4699 | 0 | { |
4700 | 0 | if (!(n->cse.iocs.nvm[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) { |
4701 | 0 | trace_pci_nvme_err_invalid_opc(req->cmd.opcode); |
4702 | 0 | return NVME_INVALID_OPCODE | NVME_DNR; |
4703 | 0 | } |
4704 | | |
4705 | 0 | return __nvme_io_cmd_nvm(n, req); |
4706 | 0 | } |
4707 | | |
4708 | | static uint16_t nvme_io_cmd_zoned(NvmeCtrl *n, NvmeRequest *req) |
4709 | 0 | { |
4710 | 0 | if (!(n->cse.iocs.zoned[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) { |
4711 | 0 | trace_pci_nvme_err_invalid_opc(req->cmd.opcode); |
4712 | 0 | return NVME_INVALID_OPCODE | NVME_DNR; |
4713 | 0 | } |
4714 | | |
4715 | 0 | switch (req->cmd.opcode) { |
4716 | 0 | case NVME_CMD_ZONE_APPEND: |
4717 | 0 | return nvme_zone_append(n, req); |
4718 | 0 | case NVME_CMD_ZONE_MGMT_SEND: |
4719 | 0 | return nvme_zone_mgmt_send(n, req); |
4720 | 0 | case NVME_CMD_ZONE_MGMT_RECV: |
4721 | 0 | return nvme_zone_mgmt_recv(n, req); |
4722 | 0 | } |
4723 | | |
4724 | 0 | return __nvme_io_cmd_nvm(n, req); |
4725 | 0 | } |
4726 | | |
4727 | | static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req) |
4728 | 0 | { |
4729 | 0 | NvmeNamespace *ns; |
4730 | 0 | uint32_t nsid = le32_to_cpu(req->cmd.nsid); |
4731 | |
|
4732 | 0 | trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req), |
4733 | 0 | req->cmd.opcode, nvme_io_opc_str(req->cmd.opcode)); |
4734 | | |
4735 | | /* |
4736 | | * In the base NVM command set, Flush may apply to all namespaces |
4737 | | * (indicated by NSID being set to FFFFFFFFh). But if that feature is used |
4738 | | * along with TP 4056 (Namespace Types), it may be pretty screwed up. |
4739 | | * |
4740 | | * If NSID is indeed set to FFFFFFFFh, we simply cannot associate the |
4741 | | * opcode with a specific command since we cannot determine a unique I/O |
4742 | | * command set. Opcode 0h could have any other meaning than something |
4743 | | * equivalent to flushing and say it DOES have completely different |
4744 | | * semantics in some other command set - does an NSID of FFFFFFFFh then |
4745 | | * mean "for all namespaces, apply whatever command set specific command |
4746 | | * that uses the 0h opcode?" Or does it mean "for all namespaces, apply |
4747 | | * whatever command that uses the 0h opcode if, and only if, it allows NSID |
4748 | | * to be FFFFFFFFh"? |
4749 | | * |
4750 | | * Anyway (and luckily), for now, we do not care about this since the |
4751 | | * device only supports namespace types that includes the NVM Flush command |
4752 | | * (NVM and Zoned), so always do an NVM Flush. |
4753 | | */ |
4754 | |
|
4755 | 0 | if (req->cmd.opcode == NVME_CMD_FLUSH) { |
4756 | 0 | return nvme_flush(n, req); |
4757 | 0 | } |
4758 | | |
4759 | 0 | if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) { |
4760 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
4761 | 0 | } |
4762 | | |
4763 | 0 | ns = nvme_ns(n, nsid); |
4764 | 0 | if (unlikely(!ns)) { |
4765 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
4766 | 0 | } |
4767 | | |
4768 | 0 | if (ns->status) { |
4769 | 0 | return ns->status; |
4770 | 0 | } |
4771 | | |
4772 | 0 | if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) { |
4773 | 0 | return NVME_INVALID_FIELD; |
4774 | 0 | } |
4775 | | |
4776 | 0 | req->ns = ns; |
4777 | |
|
4778 | 0 | switch (ns->csi) { |
4779 | 0 | case NVME_CSI_NVM: |
4780 | 0 | return nvme_io_cmd_nvm(n, req); |
4781 | 0 | case NVME_CSI_ZONED: |
4782 | 0 | return nvme_io_cmd_zoned(n, req); |
4783 | 0 | } |
4784 | | |
4785 | 0 | g_assert_not_reached(); |
4786 | 0 | } |
4787 | | |
4788 | | static void nvme_cq_notifier(EventNotifier *e) |
4789 | 0 | { |
4790 | 0 | NvmeCQueue *cq = container_of(e, NvmeCQueue, notifier); |
4791 | 0 | NvmeCtrl *n = cq->ctrl; |
4792 | 0 | bool pending = cq->tail != cq->head; |
4793 | |
|
4794 | 0 | if (!event_notifier_test_and_clear(e)) { |
4795 | 0 | return; |
4796 | 0 | } |
4797 | | |
4798 | 0 | nvme_update_cq_eventidx(cq); |
4799 | 0 | nvme_update_cq_head(cq); |
4800 | |
|
4801 | 0 | if (cq->tail == cq->head) { |
4802 | 0 | if (cq->irq_enabled && pending) { |
4803 | 0 | n->cq_pending--; |
4804 | 0 | } |
4805 | |
|
4806 | 0 | nvme_irq_deassert(n, cq); |
4807 | 0 | } |
4808 | |
|
4809 | 0 | qemu_bh_schedule(cq->bh); |
4810 | 0 | } |
4811 | | |
4812 | | static int nvme_init_cq_ioeventfd(NvmeCQueue *cq) |
4813 | 0 | { |
4814 | 0 | NvmeCtrl *n = cq->ctrl; |
4815 | 0 | uint16_t offset = (cq->cqid << 3) + (1 << 2); |
4816 | 0 | int ret; |
4817 | |
|
4818 | 0 | ret = event_notifier_init(&cq->notifier, 0); |
4819 | 0 | if (ret < 0) { |
4820 | 0 | return ret; |
4821 | 0 | } |
4822 | | |
4823 | 0 | event_notifier_set_handler(&cq->notifier, nvme_cq_notifier); |
4824 | 0 | memory_region_add_eventfd(&n->iomem, |
4825 | 0 | 0x1000 + offset, 4, false, 0, &cq->notifier); |
4826 | |
|
4827 | 0 | return 0; |
4828 | 0 | } |
4829 | | |
4830 | | static void nvme_sq_notifier(EventNotifier *e) |
4831 | 0 | { |
4832 | 0 | NvmeSQueue *sq = container_of(e, NvmeSQueue, notifier); |
4833 | |
|
4834 | 0 | if (!event_notifier_test_and_clear(e)) { |
4835 | 0 | return; |
4836 | 0 | } |
4837 | | |
4838 | 0 | nvme_process_sq(sq); |
4839 | 0 | } |
4840 | | |
4841 | | static int nvme_init_sq_ioeventfd(NvmeSQueue *sq) |
4842 | 0 | { |
4843 | 0 | NvmeCtrl *n = sq->ctrl; |
4844 | 0 | uint16_t offset = sq->sqid << 3; |
4845 | 0 | int ret; |
4846 | |
|
4847 | 0 | ret = event_notifier_init(&sq->notifier, 0); |
4848 | 0 | if (ret < 0) { |
4849 | 0 | return ret; |
4850 | 0 | } |
4851 | | |
4852 | 0 | event_notifier_set_handler(&sq->notifier, nvme_sq_notifier); |
4853 | 0 | memory_region_add_eventfd(&n->iomem, |
4854 | 0 | 0x1000 + offset, 4, false, 0, &sq->notifier); |
4855 | |
|
4856 | 0 | return 0; |
4857 | 0 | } |
4858 | | |
4859 | | /* |
4860 | | * A pending Async Event Request has no aiocb (nvme_aer() parks it without |
4861 | | * issuing any block I/O), so there is nothing to cancel; just drop it. |
4862 | | */ |
4863 | | static void nvme_sq_cancel_inflight(NvmeSQueue *sq, uint16_t status) |
4864 | 0 | { |
4865 | 0 | NvmeRequest *r; |
4866 | |
|
4867 | 0 | while (!QTAILQ_EMPTY(&sq->out_req_list)) { |
4868 | 0 | r = QTAILQ_FIRST(&sq->out_req_list); |
4869 | 0 | r->status = status; |
4870 | |
|
4871 | 0 | if (r->aiocb) { |
4872 | 0 | blk_aio_cancel(r->aiocb); |
4873 | 0 | } else { |
4874 | 0 | QTAILQ_REMOVE(&sq->out_req_list, r, entry); |
4875 | 0 | } |
4876 | 0 | } |
4877 | 0 | } |
4878 | | |
4879 | | static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n) |
4880 | 0 | { |
4881 | 0 | uint16_t offset = sq->sqid << 3; |
4882 | |
|
4883 | 0 | n->sq[sq->sqid] = NULL; |
4884 | 0 | qemu_bh_delete(sq->bh); |
4885 | 0 | if (sq->ioeventfd_enabled) { |
4886 | 0 | memory_region_del_eventfd(&n->iomem, |
4887 | 0 | 0x1000 + offset, 4, false, 0, &sq->notifier); |
4888 | 0 | event_notifier_set_handler(&sq->notifier, NULL); |
4889 | 0 | event_notifier_cleanup(&sq->notifier); |
4890 | 0 | } |
4891 | 0 | g_free(sq->io_req); |
4892 | 0 | if (sq->sqid) { |
4893 | 0 | g_free(sq); |
4894 | 0 | } |
4895 | 0 | } |
4896 | | |
4897 | | static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req) |
4898 | 0 | { |
4899 | 0 | NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd; |
4900 | 0 | NvmeRequest *r, *next; |
4901 | 0 | NvmeSQueue *sq; |
4902 | 0 | NvmeCQueue *cq; |
4903 | 0 | uint16_t qid = le16_to_cpu(c->qid); |
4904 | |
|
4905 | 0 | if (unlikely(!qid || nvme_check_sqid(n, qid))) { |
4906 | 0 | trace_pci_nvme_err_invalid_del_sq(qid); |
4907 | 0 | return NVME_INVALID_QID | NVME_DNR; |
4908 | 0 | } |
4909 | | |
4910 | 0 | trace_pci_nvme_del_sq(qid); |
4911 | |
|
4912 | 0 | sq = n->sq[qid]; |
4913 | 0 | nvme_sq_cancel_inflight(sq, NVME_CMD_ABORT_SQ_DEL); |
4914 | |
|
4915 | 0 | if (!nvme_check_cqid(n, sq->cqid)) { |
4916 | 0 | cq = n->cq[sq->cqid]; |
4917 | 0 | QTAILQ_REMOVE(&cq->sq_list, sq, entry); |
4918 | |
|
4919 | 0 | nvme_post_cqes(cq); |
4920 | 0 | QTAILQ_FOREACH_SAFE(r, &cq->req_list, entry, next) { |
4921 | 0 | if (r->sq == sq) { |
4922 | 0 | QTAILQ_REMOVE(&cq->req_list, r, entry); |
4923 | 0 | QTAILQ_INSERT_TAIL(&sq->req_list, r, entry); |
4924 | 0 | } |
4925 | 0 | } |
4926 | 0 | } |
4927 | |
|
4928 | 0 | nvme_free_sq(sq, n); |
4929 | 0 | return NVME_SUCCESS; |
4930 | 0 | } |
4931 | | |
4932 | | static void __nvme_init_sq(NvmeSQueue *sq) |
4933 | 0 | { |
4934 | 0 | NvmeCtrl *n = sq->ctrl; |
4935 | 0 | uint16_t sqid = sq->sqid; |
4936 | 0 | uint16_t cqid = sq->cqid; |
4937 | 0 | int i; |
4938 | 0 | NvmeCQueue *cq; |
4939 | |
|
4940 | 0 | sq->io_req = g_new0(NvmeRequest, sq->size); |
4941 | |
|
4942 | 0 | QTAILQ_INIT(&sq->req_list); |
4943 | 0 | QTAILQ_INIT(&sq->out_req_list); |
4944 | 0 | for (i = 0; i < sq->size; i++) { |
4945 | 0 | sq->io_req[i].sq = sq; |
4946 | 0 | QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry); |
4947 | 0 | } |
4948 | |
|
4949 | 0 | sq->bh = qemu_bh_new_guarded(nvme_process_sq, sq, |
4950 | 0 | &DEVICE(sq->ctrl)->mem_reentrancy_guard); |
4951 | |
|
4952 | 0 | if (n->dbbuf_enabled) { |
4953 | 0 | sq->db_addr = n->dbbuf_dbs + (sqid << 3); |
4954 | 0 | sq->ei_addr = n->dbbuf_eis + (sqid << 3); |
4955 | |
|
4956 | 0 | if (n->params.ioeventfd && sq->sqid != 0) { |
4957 | 0 | if (!nvme_init_sq_ioeventfd(sq)) { |
4958 | 0 | sq->ioeventfd_enabled = true; |
4959 | 0 | } |
4960 | 0 | } |
4961 | 0 | } |
4962 | |
|
4963 | 0 | assert(n->cq[cqid]); |
4964 | 0 | cq = n->cq[cqid]; |
4965 | 0 | QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry); |
4966 | 0 | n->sq[sqid] = sq; |
4967 | 0 | } |
4968 | | |
4969 | | static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr, |
4970 | | uint16_t sqid, uint16_t cqid, uint16_t size) |
4971 | 0 | { |
4972 | 0 | sq->ctrl = n; |
4973 | 0 | sq->dma_addr = dma_addr; |
4974 | 0 | sq->sqid = sqid; |
4975 | 0 | sq->size = size; |
4976 | 0 | sq->cqid = cqid; |
4977 | 0 | sq->head = sq->tail = 0; |
4978 | 0 | __nvme_init_sq(sq); |
4979 | 0 | } |
4980 | | |
4981 | | static void nvme_restore_sq(NvmeSQueue *sq_from) |
4982 | 0 | { |
4983 | 0 | NvmeCtrl *n = sq_from->ctrl; |
4984 | 0 | NvmeSQueue *sq = sq_from; |
4985 | |
|
4986 | 0 | if (sq_from->sqid == 0) { |
4987 | 0 | sq = &n->admin_sq; |
4988 | 0 | sq->ctrl = n; |
4989 | 0 | sq->dma_addr = sq_from->dma_addr; |
4990 | 0 | sq->sqid = sq_from->sqid; |
4991 | 0 | sq->size = sq_from->size; |
4992 | 0 | sq->cqid = sq_from->cqid; |
4993 | 0 | sq->head = sq_from->head; |
4994 | 0 | sq->tail = sq_from->tail; |
4995 | 0 | } |
4996 | |
|
4997 | 0 | __nvme_init_sq(sq); |
4998 | 0 | } |
4999 | | |
5000 | | static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req) |
5001 | 0 | { |
5002 | 0 | NvmeSQueue *sq; |
5003 | 0 | NvmeCreateSq *c = (NvmeCreateSq *)&req->cmd; |
5004 | |
|
5005 | 0 | uint16_t cqid = le16_to_cpu(c->cqid); |
5006 | 0 | uint16_t sqid = le16_to_cpu(c->sqid); |
5007 | 0 | uint16_t qsize = le16_to_cpu(c->qsize); |
5008 | 0 | uint16_t qflags = le16_to_cpu(c->sq_flags); |
5009 | 0 | uint64_t prp1 = le64_to_cpu(c->prp1); |
5010 | |
|
5011 | 0 | trace_pci_nvme_create_sq(prp1, sqid, cqid, qsize, qflags); |
5012 | |
|
5013 | 0 | if (unlikely(!cqid || nvme_check_cqid(n, cqid))) { |
5014 | 0 | trace_pci_nvme_err_invalid_create_sq_cqid(cqid); |
5015 | 0 | return NVME_INVALID_CQID | NVME_DNR; |
5016 | 0 | } |
5017 | 0 | if (unlikely(!sqid || sqid > n->conf_ioqpairs || n->sq[sqid] != NULL)) { |
5018 | 0 | trace_pci_nvme_err_invalid_create_sq_sqid(sqid); |
5019 | 0 | return NVME_INVALID_QID | NVME_DNR; |
5020 | 0 | } |
5021 | 0 | if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) { |
5022 | 0 | trace_pci_nvme_err_invalid_create_sq_size(qsize); |
5023 | 0 | return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR; |
5024 | 0 | } |
5025 | 0 | if (unlikely(prp1 & (n->page_size - 1))) { |
5026 | 0 | trace_pci_nvme_err_invalid_create_sq_addr(prp1); |
5027 | 0 | return NVME_INVALID_PRP_OFFSET | NVME_DNR; |
5028 | 0 | } |
5029 | 0 | if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) { |
5030 | 0 | trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags)); |
5031 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5032 | 0 | } |
5033 | 0 | sq = g_malloc0(sizeof(*sq)); |
5034 | 0 | nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1); |
5035 | 0 | return NVME_SUCCESS; |
5036 | 0 | } |
5037 | | |
5038 | | struct nvme_stats { |
5039 | | uint64_t units_read; |
5040 | | uint64_t units_written; |
5041 | | uint64_t read_commands; |
5042 | | uint64_t write_commands; |
5043 | | }; |
5044 | | |
5045 | | static void nvme_set_blk_stats(NvmeNamespace *ns, struct nvme_stats *stats) |
5046 | 0 | { |
5047 | 0 | BlockAcctStats *s = blk_get_stats(ns->blkconf.blk); |
5048 | |
|
5049 | 0 | stats->units_read += s->nr_bytes[BLOCK_ACCT_READ]; |
5050 | 0 | stats->units_written += s->nr_bytes[BLOCK_ACCT_WRITE]; |
5051 | 0 | stats->read_commands += s->nr_ops[BLOCK_ACCT_READ]; |
5052 | 0 | stats->write_commands += s->nr_ops[BLOCK_ACCT_WRITE]; |
5053 | 0 | } |
5054 | | |
5055 | | static uint16_t nvme_ocp_extended_smart_info(NvmeCtrl *n, uint8_t rae, |
5056 | | uint32_t buf_len, uint64_t off, |
5057 | | NvmeRequest *req) |
5058 | 0 | { |
5059 | 0 | NvmeNamespace *ns = NULL; |
5060 | 0 | NvmeSmartLogExtended smart_l = { 0 }; |
5061 | 0 | struct nvme_stats stats = { 0 }; |
5062 | 0 | uint32_t trans_len; |
5063 | |
|
5064 | 0 | if (off >= sizeof(smart_l)) { |
5065 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5066 | 0 | } |
5067 | | |
5068 | | /* accumulate all stats from all namespaces */ |
5069 | 0 | for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
5070 | 0 | ns = nvme_ns(n, i); |
5071 | 0 | if (ns) { |
5072 | 0 | nvme_set_blk_stats(ns, &stats); |
5073 | 0 | } |
5074 | 0 | } |
5075 | |
|
5076 | 0 | smart_l.physical_media_units_written[0] = cpu_to_le64(stats.units_written); |
5077 | 0 | smart_l.physical_media_units_read[0] = cpu_to_le64(stats.units_read); |
5078 | 0 | smart_l.log_page_version = 0x0005; |
5079 | |
|
5080 | 0 | static const uint8_t guid[16] = { |
5081 | 0 | 0xC5, 0xAF, 0x10, 0x28, 0xEA, 0xBF, 0xF2, 0xA4, |
5082 | 0 | 0x9C, 0x4F, 0x6F, 0x7C, 0xC9, 0x14, 0xD5, 0xAF |
5083 | 0 | }; |
5084 | 0 | memcpy(smart_l.log_page_guid, guid, sizeof(smart_l.log_page_guid)); |
5085 | |
|
5086 | 0 | if (!rae) { |
5087 | 0 | nvme_clear_events(n, NVME_AER_TYPE_SMART); |
5088 | 0 | } |
5089 | |
|
5090 | 0 | trans_len = MIN(sizeof(smart_l) - off, buf_len); |
5091 | 0 | return nvme_c2h(n, (uint8_t *) &smart_l + off, trans_len, req); |
5092 | 0 | } |
5093 | | |
5094 | | static uint16_t nvme_smart_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len, |
5095 | | uint64_t off, NvmeRequest *req) |
5096 | 0 | { |
5097 | 0 | uint32_t nsid = le32_to_cpu(req->cmd.nsid); |
5098 | 0 | struct nvme_stats stats = { 0 }; |
5099 | 0 | NvmeSmartLog smart = { 0 }; |
5100 | 0 | uint32_t trans_len; |
5101 | 0 | NvmeNamespace *ns; |
5102 | 0 | time_t current_ms; |
5103 | 0 | uint64_t u_read, u_written; |
5104 | |
|
5105 | 0 | if (off >= sizeof(smart)) { |
5106 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5107 | 0 | } |
5108 | | |
5109 | 0 | if (nsid != 0xffffffff) { |
5110 | 0 | ns = nvme_ns(n, nsid); |
5111 | 0 | if (!ns) { |
5112 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
5113 | 0 | } |
5114 | 0 | nvme_set_blk_stats(ns, &stats); |
5115 | 0 | } else { |
5116 | 0 | int i; |
5117 | |
|
5118 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
5119 | 0 | ns = nvme_ns(n, i); |
5120 | 0 | if (!ns) { |
5121 | 0 | continue; |
5122 | 0 | } |
5123 | 0 | nvme_set_blk_stats(ns, &stats); |
5124 | 0 | } |
5125 | 0 | } |
5126 | | |
5127 | 0 | trans_len = MIN(sizeof(smart) - off, buf_len); |
5128 | 0 | smart.critical_warning = n->smart_critical_warning; |
5129 | |
|
5130 | 0 | u_read = DIV_ROUND_UP(stats.units_read >> BDRV_SECTOR_BITS, 1000); |
5131 | 0 | u_written = DIV_ROUND_UP(stats.units_written >> BDRV_SECTOR_BITS, 1000); |
5132 | |
|
5133 | 0 | smart.data_units_read[0] = cpu_to_le64(u_read); |
5134 | 0 | smart.data_units_written[0] = cpu_to_le64(u_written); |
5135 | 0 | smart.host_read_commands[0] = cpu_to_le64(stats.read_commands); |
5136 | 0 | smart.host_write_commands[0] = cpu_to_le64(stats.write_commands); |
5137 | |
|
5138 | 0 | smart.temperature = cpu_to_le16(n->temperature); |
5139 | |
|
5140 | 0 | if ((n->temperature >= n->features.temp_thresh_hi) || |
5141 | 0 | (n->temperature <= n->features.temp_thresh_low)) { |
5142 | 0 | smart.critical_warning |= NVME_SMART_TEMPERATURE; |
5143 | 0 | } |
5144 | |
|
5145 | 0 | current_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL); |
5146 | 0 | smart.power_on_hours[0] = |
5147 | 0 | cpu_to_le64((((current_ms - n->starttime_ms) / 1000) / 60) / 60); |
5148 | |
|
5149 | 0 | if (!rae) { |
5150 | 0 | nvme_clear_events(n, NVME_AER_TYPE_SMART); |
5151 | 0 | } |
5152 | |
|
5153 | 0 | return nvme_c2h(n, (uint8_t *) &smart + off, trans_len, req); |
5154 | 0 | } |
5155 | | |
5156 | | static uint16_t nvme_endgrp_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len, |
5157 | | uint64_t off, NvmeRequest *req) |
5158 | 0 | { |
5159 | 0 | uint32_t dw11 = le32_to_cpu(req->cmd.cdw11); |
5160 | 0 | uint16_t endgrpid = (dw11 >> 16) & 0xffff; |
5161 | 0 | struct nvme_stats stats = {}; |
5162 | 0 | NvmeEndGrpLog info = {}; |
5163 | 0 | int i; |
5164 | |
|
5165 | 0 | if (!n->subsys || endgrpid != 0x1) { |
5166 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5167 | 0 | } |
5168 | | |
5169 | 0 | if (off >= sizeof(info)) { |
5170 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5171 | 0 | } |
5172 | | |
5173 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
5174 | 0 | NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i); |
5175 | 0 | if (!ns) { |
5176 | 0 | continue; |
5177 | 0 | } |
5178 | | |
5179 | 0 | nvme_set_blk_stats(ns, &stats); |
5180 | 0 | } |
5181 | |
|
5182 | 0 | info.data_units_read[0] = |
5183 | 0 | cpu_to_le64(DIV_ROUND_UP(stats.units_read / 1000000000, 1000000000)); |
5184 | 0 | info.data_units_written[0] = |
5185 | 0 | cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000)); |
5186 | 0 | info.media_units_written[0] = |
5187 | 0 | cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000)); |
5188 | |
|
5189 | 0 | info.host_read_commands[0] = cpu_to_le64(stats.read_commands); |
5190 | 0 | info.host_write_commands[0] = cpu_to_le64(stats.write_commands); |
5191 | |
|
5192 | 0 | buf_len = MIN(sizeof(info) - off, buf_len); |
5193 | |
|
5194 | 0 | return nvme_c2h(n, (uint8_t *)&info + off, buf_len, req); |
5195 | 0 | } |
5196 | | |
5197 | | |
5198 | | static uint16_t nvme_fw_log_info(NvmeCtrl *n, uint32_t buf_len, uint64_t off, |
5199 | | NvmeRequest *req) |
5200 | 0 | { |
5201 | 0 | uint32_t trans_len; |
5202 | 0 | NvmeFwSlotInfoLog fw_log = { |
5203 | 0 | .afi = 0x1, |
5204 | 0 | }; |
5205 | |
|
5206 | 0 | if (off >= sizeof(fw_log)) { |
5207 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5208 | 0 | } |
5209 | | |
5210 | 0 | strpadcpy((char *)&fw_log.frs1, sizeof(fw_log.frs1), "1.0", ' '); |
5211 | 0 | trans_len = MIN(sizeof(fw_log) - off, buf_len); |
5212 | |
|
5213 | 0 | return nvme_c2h(n, (uint8_t *) &fw_log + off, trans_len, req); |
5214 | 0 | } |
5215 | | |
5216 | | static uint16_t nvme_error_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len, |
5217 | | uint64_t off, NvmeRequest *req) |
5218 | 0 | { |
5219 | 0 | uint32_t trans_len; |
5220 | 0 | NvmeErrorLog errlog; |
5221 | |
|
5222 | 0 | if (off >= sizeof(errlog)) { |
5223 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5224 | 0 | } |
5225 | | |
5226 | 0 | if (!rae) { |
5227 | 0 | nvme_clear_events(n, NVME_AER_TYPE_ERROR); |
5228 | 0 | } |
5229 | |
|
5230 | 0 | memset(&errlog, 0x0, sizeof(errlog)); |
5231 | 0 | trans_len = MIN(sizeof(errlog) - off, buf_len); |
5232 | |
|
5233 | 0 | return nvme_c2h(n, (uint8_t *)&errlog, trans_len, req); |
5234 | 0 | } |
5235 | | |
5236 | | static uint16_t nvme_changed_nslist(NvmeCtrl *n, uint8_t rae, uint32_t buf_len, |
5237 | | uint64_t off, NvmeRequest *req) |
5238 | 0 | { |
5239 | 0 | uint32_t nslist[1024] = {}; |
5240 | 0 | uint32_t trans_len; |
5241 | 0 | int i = 0; |
5242 | 0 | uint32_t nsid; |
5243 | |
|
5244 | 0 | if (off >= sizeof(nslist)) { |
5245 | 0 | trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(nslist)); |
5246 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5247 | 0 | } |
5248 | | |
5249 | 0 | trans_len = MIN(sizeof(nslist) - off, buf_len); |
5250 | |
|
5251 | 0 | while ((nsid = find_first_bit(n->changed_nsids, NVME_CHANGED_NSID_SIZE)) != |
5252 | 0 | NVME_CHANGED_NSID_SIZE) { |
5253 | | /* |
5254 | | * If more than 1024 namespaces, the first entry in the log page should |
5255 | | * be set to FFFFFFFFh and the others to 0 as spec. |
5256 | | */ |
5257 | 0 | if (i == ARRAY_SIZE(nslist)) { |
5258 | 0 | memset(nslist, 0x0, sizeof(nslist)); |
5259 | 0 | nslist[0] = 0xffffffff; |
5260 | 0 | break; |
5261 | 0 | } |
5262 | | |
5263 | 0 | nslist[i++] = nsid; |
5264 | 0 | clear_bit(nsid, n->changed_nsids); |
5265 | 0 | } |
5266 | | |
5267 | | /* |
5268 | | * Remove all the remaining list entries in case returns directly due to |
5269 | | * more than 1024 namespaces. |
5270 | | */ |
5271 | 0 | if (nslist[0] == 0xffffffff) { |
5272 | 0 | bitmap_zero(n->changed_nsids, NVME_CHANGED_NSID_SIZE); |
5273 | 0 | } |
5274 | |
|
5275 | 0 | if (!rae) { |
5276 | 0 | nvme_clear_events(n, NVME_AER_TYPE_NOTICE); |
5277 | 0 | } |
5278 | |
|
5279 | 0 | return nvme_c2h(n, ((uint8_t *)nslist) + off, trans_len, req); |
5280 | 0 | } |
5281 | | |
5282 | | static uint16_t nvme_cmd_effects(NvmeCtrl *n, uint8_t csi, uint32_t buf_len, |
5283 | | uint64_t off, NvmeRequest *req) |
5284 | 0 | { |
5285 | 0 | NvmeEffectsLog log = {}; |
5286 | 0 | const uint32_t *iocs = NULL; |
5287 | 0 | uint32_t trans_len; |
5288 | |
|
5289 | 0 | if (off >= sizeof(log)) { |
5290 | 0 | trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(log)); |
5291 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5292 | 0 | } |
5293 | | |
5294 | 0 | switch (NVME_CC_CSS(ldl_le_p(&n->bar.cc))) { |
5295 | 0 | case NVME_CC_CSS_NVM: |
5296 | 0 | iocs = n->cse.iocs.nvm; |
5297 | 0 | break; |
5298 | | |
5299 | 0 | case NVME_CC_CSS_ALL: |
5300 | 0 | switch (csi) { |
5301 | 0 | case NVME_CSI_NVM: |
5302 | 0 | iocs = n->cse.iocs.nvm; |
5303 | 0 | break; |
5304 | 0 | case NVME_CSI_ZONED: |
5305 | 0 | iocs = n->cse.iocs.zoned; |
5306 | 0 | break; |
5307 | 0 | } |
5308 | | |
5309 | 0 | break; |
5310 | 0 | } |
5311 | | |
5312 | 0 | memcpy(log.acs, n->cse.acs, sizeof(log.acs)); |
5313 | |
|
5314 | 0 | if (iocs) { |
5315 | 0 | memcpy(log.iocs, iocs, sizeof(log.iocs)); |
5316 | 0 | } |
5317 | |
|
5318 | 0 | trans_len = MIN(sizeof(log) - off, buf_len); |
5319 | |
|
5320 | 0 | return nvme_c2h(n, ((uint8_t *)&log) + off, trans_len, req); |
5321 | 0 | } |
5322 | | |
5323 | | static uint16_t nvme_vendor_specific_log(NvmeCtrl *n, uint8_t rae, |
5324 | | uint32_t buf_len, uint64_t off, |
5325 | | NvmeRequest *req, uint8_t lid) |
5326 | 0 | { |
5327 | 0 | switch (lid) { |
5328 | 0 | case NVME_OCP_EXTENDED_SMART_INFO: |
5329 | 0 | if (n->params.ocp) { |
5330 | 0 | return nvme_ocp_extended_smart_info(n, rae, buf_len, off, req); |
5331 | 0 | } |
5332 | 0 | break; |
5333 | | /* add a case for each additional vendor specific log id */ |
5334 | 0 | } |
5335 | | |
5336 | 0 | trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid); |
5337 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5338 | 0 | } |
5339 | | |
5340 | | static size_t sizeof_fdp_conf_descr(size_t nruh, size_t vss) |
5341 | 0 | { |
5342 | 0 | size_t entry_siz = sizeof(NvmeFdpDescrHdr) + nruh * sizeof(NvmeRuhDescr) |
5343 | 0 | + vss; |
5344 | 0 | return ROUND_UP(entry_siz, 8); |
5345 | 0 | } |
5346 | | |
5347 | | static uint16_t nvme_fdp_confs(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len, |
5348 | | uint64_t off, NvmeRequest *req) |
5349 | 0 | { |
5350 | 0 | uint32_t log_size, trans_len; |
5351 | 0 | g_autofree uint8_t *buf = NULL; |
5352 | 0 | NvmeFdpDescrHdr *hdr; |
5353 | 0 | NvmeRuhDescr *ruhd; |
5354 | 0 | NvmeEnduranceGroup *endgrp; |
5355 | 0 | NvmeFdpConfsHdr *log; |
5356 | 0 | size_t nruh, fdp_descr_size; |
5357 | 0 | int i; |
5358 | |
|
5359 | 0 | if (endgrpid != 1 || !n->subsys) { |
5360 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5361 | 0 | } |
5362 | | |
5363 | 0 | endgrp = &n->subsys->endgrp; |
5364 | |
|
5365 | 0 | if (endgrp->fdp.enabled) { |
5366 | 0 | nruh = endgrp->fdp.nruh; |
5367 | 0 | } else { |
5368 | 0 | nruh = 1; |
5369 | 0 | } |
5370 | |
|
5371 | 0 | fdp_descr_size = sizeof_fdp_conf_descr(nruh, FDPVSS); |
5372 | 0 | log_size = sizeof(NvmeFdpConfsHdr) + fdp_descr_size; |
5373 | |
|
5374 | 0 | if (off >= log_size) { |
5375 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5376 | 0 | } |
5377 | | |
5378 | 0 | trans_len = MIN(log_size - off, buf_len); |
5379 | |
|
5380 | 0 | buf = g_malloc0(log_size); |
5381 | 0 | log = (NvmeFdpConfsHdr *)buf; |
5382 | 0 | hdr = (NvmeFdpDescrHdr *)(log + 1); |
5383 | 0 | ruhd = (NvmeRuhDescr *)(buf + sizeof(*log) + sizeof(*hdr)); |
5384 | |
|
5385 | 0 | log->num_confs = cpu_to_le16(0); |
5386 | 0 | log->size = cpu_to_le32(log_size); |
5387 | |
|
5388 | 0 | hdr->descr_size = cpu_to_le16(fdp_descr_size); |
5389 | 0 | if (endgrp->fdp.enabled) { |
5390 | 0 | hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, VALID, 1); |
5391 | 0 | hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, RGIF, endgrp->fdp.rgif); |
5392 | 0 | hdr->nrg = cpu_to_le16(endgrp->fdp.nrg); |
5393 | 0 | hdr->nruh = cpu_to_le16(endgrp->fdp.nruh); |
5394 | 0 | hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1); |
5395 | 0 | hdr->nnss = cpu_to_le32(NVME_MAX_NAMESPACES); |
5396 | 0 | hdr->runs = cpu_to_le64(endgrp->fdp.runs); |
5397 | |
|
5398 | 0 | for (i = 0; i < nruh; i++) { |
5399 | 0 | ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED; |
5400 | 0 | ruhd++; |
5401 | 0 | } |
5402 | 0 | } else { |
5403 | | /* 1 bit for RUH in PIF -> 2 RUHs max. */ |
5404 | 0 | hdr->nrg = cpu_to_le16(1); |
5405 | 0 | hdr->nruh = cpu_to_le16(1); |
5406 | 0 | hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1); |
5407 | 0 | hdr->nnss = cpu_to_le32(1); |
5408 | 0 | hdr->runs = cpu_to_le64(96 * MiB); |
5409 | |
|
5410 | 0 | ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED; |
5411 | 0 | } |
5412 | |
|
5413 | 0 | return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req); |
5414 | 0 | } |
5415 | | |
5416 | | static uint16_t nvme_fdp_ruh_usage(NvmeCtrl *n, uint32_t endgrpid, |
5417 | | uint32_t dw10, uint32_t dw12, |
5418 | | uint32_t buf_len, uint64_t off, |
5419 | | NvmeRequest *req) |
5420 | 0 | { |
5421 | 0 | NvmeRuHandle *ruh; |
5422 | 0 | NvmeRuhuLog *hdr; |
5423 | 0 | NvmeRuhuDescr *ruhud; |
5424 | 0 | NvmeEnduranceGroup *endgrp; |
5425 | 0 | g_autofree uint8_t *buf = NULL; |
5426 | 0 | uint32_t log_size, trans_len; |
5427 | 0 | uint16_t i; |
5428 | |
|
5429 | 0 | if (endgrpid != 1 || !n->subsys) { |
5430 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5431 | 0 | } |
5432 | | |
5433 | 0 | endgrp = &n->subsys->endgrp; |
5434 | |
|
5435 | 0 | if (!endgrp->fdp.enabled) { |
5436 | 0 | return NVME_FDP_DISABLED | NVME_DNR; |
5437 | 0 | } |
5438 | | |
5439 | 0 | log_size = sizeof(NvmeRuhuLog) + endgrp->fdp.nruh * sizeof(NvmeRuhuDescr); |
5440 | |
|
5441 | 0 | if (off >= log_size) { |
5442 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5443 | 0 | } |
5444 | | |
5445 | 0 | trans_len = MIN(log_size - off, buf_len); |
5446 | |
|
5447 | 0 | buf = g_malloc0(log_size); |
5448 | 0 | hdr = (NvmeRuhuLog *)buf; |
5449 | 0 | ruhud = (NvmeRuhuDescr *)(hdr + 1); |
5450 | |
|
5451 | 0 | ruh = endgrp->fdp.ruhs; |
5452 | 0 | hdr->nruh = cpu_to_le16(endgrp->fdp.nruh); |
5453 | |
|
5454 | 0 | for (i = 0; i < endgrp->fdp.nruh; i++, ruhud++, ruh++) { |
5455 | 0 | ruhud->ruha = ruh->ruha; |
5456 | 0 | } |
5457 | |
|
5458 | 0 | return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req); |
5459 | 0 | } |
5460 | | |
5461 | | static uint16_t nvme_fdp_stats(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len, |
5462 | | uint64_t off, NvmeRequest *req) |
5463 | 0 | { |
5464 | 0 | NvmeEnduranceGroup *endgrp; |
5465 | 0 | NvmeFdpStatsLog log = {}; |
5466 | 0 | uint32_t trans_len; |
5467 | |
|
5468 | 0 | if (off >= sizeof(NvmeFdpStatsLog)) { |
5469 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5470 | 0 | } |
5471 | | |
5472 | 0 | if (endgrpid != 1 || !n->subsys) { |
5473 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5474 | 0 | } |
5475 | | |
5476 | 0 | if (!n->subsys->endgrp.fdp.enabled) { |
5477 | 0 | return NVME_FDP_DISABLED | NVME_DNR; |
5478 | 0 | } |
5479 | | |
5480 | 0 | endgrp = &n->subsys->endgrp; |
5481 | |
|
5482 | 0 | trans_len = MIN(sizeof(log) - off, buf_len); |
5483 | | |
5484 | | /* spec value is 128 bit, we only use 64 bit */ |
5485 | 0 | log.hbmw[0] = cpu_to_le64(endgrp->fdp.hbmw); |
5486 | 0 | log.mbmw[0] = cpu_to_le64(endgrp->fdp.mbmw); |
5487 | 0 | log.mbe[0] = cpu_to_le64(endgrp->fdp.mbe); |
5488 | |
|
5489 | 0 | return nvme_c2h(n, (uint8_t *)&log + off, trans_len, req); |
5490 | 0 | } |
5491 | | |
5492 | | static uint16_t nvme_fdp_events(NvmeCtrl *n, uint32_t endgrpid, |
5493 | | uint32_t buf_len, uint64_t off, |
5494 | | NvmeRequest *req) |
5495 | 0 | { |
5496 | 0 | NvmeEnduranceGroup *endgrp; |
5497 | 0 | NvmeCmd *cmd = &req->cmd; |
5498 | 0 | bool host_events = (cmd->cdw10 >> 8) & 0x1; |
5499 | 0 | uint32_t log_size, trans_len; |
5500 | 0 | NvmeFdpEventBuffer *ebuf; |
5501 | 0 | g_autofree NvmeFdpEventsLog *elog = NULL; |
5502 | 0 | NvmeFdpEvent *event; |
5503 | |
|
5504 | 0 | if (endgrpid != 1 || !n->subsys) { |
5505 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5506 | 0 | } |
5507 | | |
5508 | 0 | endgrp = &n->subsys->endgrp; |
5509 | |
|
5510 | 0 | if (!endgrp->fdp.enabled) { |
5511 | 0 | return NVME_FDP_DISABLED | NVME_DNR; |
5512 | 0 | } |
5513 | | |
5514 | 0 | if (host_events) { |
5515 | 0 | ebuf = &endgrp->fdp.host_events; |
5516 | 0 | } else { |
5517 | 0 | ebuf = &endgrp->fdp.ctrl_events; |
5518 | 0 | } |
5519 | |
|
5520 | 0 | log_size = sizeof(NvmeFdpEventsLog) + ebuf->nelems * sizeof(NvmeFdpEvent); |
5521 | |
|
5522 | 0 | if (off >= log_size) { |
5523 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5524 | 0 | } |
5525 | | |
5526 | 0 | trans_len = MIN(log_size - off, buf_len); |
5527 | 0 | elog = g_malloc0(log_size); |
5528 | 0 | elog->num_events = cpu_to_le32(ebuf->nelems); |
5529 | 0 | event = (NvmeFdpEvent *)(elog + 1); |
5530 | |
|
5531 | 0 | if (ebuf->nelems && ebuf->start == ebuf->next) { |
5532 | 0 | unsigned int nelems = (NVME_FDP_MAX_EVENTS - ebuf->start); |
5533 | | /* wrap over, copy [start;NVME_FDP_MAX_EVENTS[ and [0; next[ */ |
5534 | 0 | memcpy(event, &ebuf->events[ebuf->start], |
5535 | 0 | sizeof(NvmeFdpEvent) * nelems); |
5536 | 0 | memcpy(event + nelems, ebuf->events, |
5537 | 0 | sizeof(NvmeFdpEvent) * ebuf->next); |
5538 | 0 | } else if (ebuf->start < ebuf->next) { |
5539 | 0 | memcpy(event, &ebuf->events[ebuf->start], |
5540 | 0 | sizeof(NvmeFdpEvent) * (ebuf->next - ebuf->start)); |
5541 | 0 | } |
5542 | |
|
5543 | 0 | return nvme_c2h(n, (uint8_t *)elog + off, trans_len, req); |
5544 | 0 | } |
5545 | | |
5546 | | static uint16_t nvme_get_log(NvmeCtrl *n, NvmeRequest *req) |
5547 | 0 | { |
5548 | 0 | NvmeCmd *cmd = &req->cmd; |
5549 | |
|
5550 | 0 | uint32_t dw10 = le32_to_cpu(cmd->cdw10); |
5551 | 0 | uint32_t dw11 = le32_to_cpu(cmd->cdw11); |
5552 | 0 | uint32_t dw12 = le32_to_cpu(cmd->cdw12); |
5553 | 0 | uint32_t dw13 = le32_to_cpu(cmd->cdw13); |
5554 | 0 | uint8_t lid = dw10 & 0xff; |
5555 | 0 | uint8_t lsp = (dw10 >> 8) & 0xf; |
5556 | 0 | uint8_t rae = (dw10 >> 15) & 0x1; |
5557 | 0 | uint8_t csi = le32_to_cpu(cmd->cdw14) >> 24; |
5558 | 0 | uint32_t numdl, numdu, lspi; |
5559 | 0 | uint64_t off, lpol, lpou; |
5560 | 0 | size_t len; |
5561 | 0 | uint16_t status; |
5562 | |
|
5563 | 0 | numdl = (dw10 >> 16); |
5564 | 0 | numdu = (dw11 & 0xffff); |
5565 | 0 | lspi = (dw11 >> 16); |
5566 | 0 | lpol = dw12; |
5567 | 0 | lpou = dw13; |
5568 | |
|
5569 | 0 | len = (((numdu << 16) | numdl) + 1) << 2; |
5570 | 0 | off = (lpou << 32ULL) | lpol; |
5571 | |
|
5572 | 0 | if (off & 0x3) { |
5573 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5574 | 0 | } |
5575 | | |
5576 | 0 | trace_pci_nvme_get_log(nvme_cid(req), lid, lsp, rae, len, off); |
5577 | |
|
5578 | 0 | status = nvme_check_mdts(n, len); |
5579 | 0 | if (status) { |
5580 | 0 | return status; |
5581 | 0 | } |
5582 | | |
5583 | 0 | switch (lid) { |
5584 | 0 | case NVME_LOG_ERROR_INFO: |
5585 | 0 | return nvme_error_info(n, rae, len, off, req); |
5586 | 0 | case NVME_LOG_SMART_INFO: |
5587 | 0 | return nvme_smart_info(n, rae, len, off, req); |
5588 | 0 | case NVME_LOG_FW_SLOT_INFO: |
5589 | 0 | return nvme_fw_log_info(n, len, off, req); |
5590 | 0 | case NVME_LOG_VENDOR_START...NVME_LOG_VENDOR_END: |
5591 | 0 | return nvme_vendor_specific_log(n, rae, len, off, req, lid); |
5592 | 0 | case NVME_LOG_CHANGED_NSLIST: |
5593 | 0 | return nvme_changed_nslist(n, rae, len, off, req); |
5594 | 0 | case NVME_LOG_CMD_EFFECTS: |
5595 | 0 | return nvme_cmd_effects(n, csi, len, off, req); |
5596 | 0 | case NVME_LOG_ENDGRP: |
5597 | 0 | return nvme_endgrp_info(n, rae, len, off, req); |
5598 | 0 | case NVME_LOG_FDP_CONFS: |
5599 | 0 | return nvme_fdp_confs(n, lspi, len, off, req); |
5600 | 0 | case NVME_LOG_FDP_RUH_USAGE: |
5601 | 0 | return nvme_fdp_ruh_usage(n, lspi, dw10, dw12, len, off, req); |
5602 | 0 | case NVME_LOG_FDP_STATS: |
5603 | 0 | return nvme_fdp_stats(n, lspi, len, off, req); |
5604 | 0 | case NVME_LOG_FDP_EVENTS: |
5605 | 0 | return nvme_fdp_events(n, lspi, len, off, req); |
5606 | 0 | default: |
5607 | 0 | trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid); |
5608 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5609 | 0 | } |
5610 | 0 | } |
5611 | | |
5612 | | static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n) |
5613 | 0 | { |
5614 | 0 | PCIDevice *pci = PCI_DEVICE(n); |
5615 | 0 | uint16_t offset = (cq->cqid << 3) + (1 << 2); |
5616 | |
|
5617 | 0 | n->cq[cq->cqid] = NULL; |
5618 | 0 | qemu_bh_delete(cq->bh); |
5619 | 0 | if (cq->ioeventfd_enabled) { |
5620 | 0 | memory_region_del_eventfd(&n->iomem, |
5621 | 0 | 0x1000 + offset, 4, false, 0, &cq->notifier); |
5622 | 0 | event_notifier_set_handler(&cq->notifier, NULL); |
5623 | 0 | event_notifier_cleanup(&cq->notifier); |
5624 | 0 | } |
5625 | 0 | if (msix_present(pci) && cq->irq_enabled) { |
5626 | 0 | msix_vector_unuse(pci, cq->vector); |
5627 | 0 | } |
5628 | 0 | if (cq->cqid) { |
5629 | 0 | g_free(cq); |
5630 | 0 | } |
5631 | 0 | } |
5632 | | |
5633 | | static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req) |
5634 | 0 | { |
5635 | 0 | NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd; |
5636 | 0 | NvmeCQueue *cq; |
5637 | 0 | uint16_t qid = le16_to_cpu(c->qid); |
5638 | |
|
5639 | 0 | if (unlikely(!qid || nvme_check_cqid(n, qid))) { |
5640 | 0 | trace_pci_nvme_err_invalid_del_cq_cqid(qid); |
5641 | 0 | return NVME_INVALID_CQID | NVME_DNR; |
5642 | 0 | } |
5643 | | |
5644 | 0 | cq = n->cq[qid]; |
5645 | 0 | if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) { |
5646 | 0 | trace_pci_nvme_err_invalid_del_cq_notempty(qid); |
5647 | 0 | return NVME_INVALID_QUEUE_DEL; |
5648 | 0 | } |
5649 | | |
5650 | 0 | if (cq->irq_enabled && cq->tail != cq->head) { |
5651 | 0 | n->cq_pending--; |
5652 | 0 | } |
5653 | |
|
5654 | 0 | nvme_irq_deassert(n, cq); |
5655 | 0 | trace_pci_nvme_del_cq(qid); |
5656 | 0 | nvme_free_cq(cq, n); |
5657 | 0 | return NVME_SUCCESS; |
5658 | 0 | } |
5659 | | |
5660 | | static void __nvme_init_cq(NvmeCQueue *cq) |
5661 | 0 | { |
5662 | 0 | NvmeCtrl *n = cq->ctrl; |
5663 | 0 | PCIDevice *pci = PCI_DEVICE(n); |
5664 | 0 | uint16_t cqid = cq->cqid; |
5665 | |
|
5666 | 0 | if (msix_present(pci) && cq->irq_enabled) { |
5667 | 0 | msix_vector_use(pci, cq->vector); |
5668 | 0 | } |
5669 | |
|
5670 | 0 | QTAILQ_INIT(&cq->sq_list); |
5671 | 0 | if (n->dbbuf_enabled) { |
5672 | 0 | cq->db_addr = n->dbbuf_dbs + (cqid << 3) + (1 << 2); |
5673 | 0 | cq->ei_addr = n->dbbuf_eis + (cqid << 3) + (1 << 2); |
5674 | |
|
5675 | 0 | if (n->params.ioeventfd && cqid != 0) { |
5676 | 0 | if (!nvme_init_cq_ioeventfd(cq)) { |
5677 | 0 | cq->ioeventfd_enabled = true; |
5678 | 0 | } |
5679 | 0 | } |
5680 | 0 | } |
5681 | 0 | n->cq[cqid] = cq; |
5682 | 0 | cq->bh = qemu_bh_new_guarded(nvme_post_cqes, cq, |
5683 | 0 | &DEVICE(cq->ctrl)->mem_reentrancy_guard); |
5684 | 0 | } |
5685 | | |
5686 | | static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr, |
5687 | | uint16_t cqid, uint16_t vector, uint16_t size, |
5688 | | uint16_t irq_enabled) |
5689 | 0 | { |
5690 | 0 | cq->ctrl = n; |
5691 | 0 | cq->cqid = cqid; |
5692 | 0 | cq->size = size; |
5693 | 0 | cq->dma_addr = dma_addr; |
5694 | 0 | cq->phase = 1; |
5695 | 0 | cq->irq_enabled = irq_enabled; |
5696 | 0 | cq->vector = vector; |
5697 | 0 | cq->head = cq->tail = 0; |
5698 | 0 | QTAILQ_INIT(&cq->req_list); |
5699 | 0 | __nvme_init_cq(cq); |
5700 | 0 | } |
5701 | | |
5702 | | static void move_cq_req_list(NvmeCQueue *cq_to, NvmeCQueue *cq_from) |
5703 | 0 | { |
5704 | 0 | NvmeRequest *req, *next; |
5705 | |
|
5706 | 0 | QTAILQ_FOREACH_SAFE(req, &cq_from->req_list, entry, next) { |
5707 | 0 | QTAILQ_REMOVE(&cq_from->req_list, req, entry); |
5708 | 0 | QTAILQ_INSERT_TAIL(&cq_to->req_list, req, entry); |
5709 | 0 | } |
5710 | 0 | } |
5711 | | |
5712 | | static void nvme_restore_cq(NvmeCQueue *cq_from) |
5713 | 0 | { |
5714 | 0 | NvmeCtrl *n = cq_from->ctrl; |
5715 | 0 | NvmeCQueue *cq = cq_from; |
5716 | |
|
5717 | 0 | if (cq_from->cqid == 0) { |
5718 | 0 | cq = &n->admin_cq; |
5719 | 0 | cq->ctrl = n; |
5720 | 0 | cq->cqid = cq_from->cqid; |
5721 | 0 | cq->size = cq_from->size; |
5722 | 0 | cq->dma_addr = cq_from->dma_addr; |
5723 | 0 | cq->phase = cq_from->phase; |
5724 | 0 | cq->irq_enabled = cq_from->irq_enabled; |
5725 | 0 | cq->vector = cq_from->vector; |
5726 | 0 | cq->head = cq_from->head; |
5727 | 0 | cq->tail = cq_from->tail; |
5728 | 0 | QTAILQ_INIT(&cq->req_list); |
5729 | 0 | move_cq_req_list(cq, cq_from); |
5730 | 0 | } |
5731 | |
|
5732 | 0 | __nvme_init_cq(cq); |
5733 | 0 | } |
5734 | | |
5735 | | static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeRequest *req) |
5736 | 0 | { |
5737 | 0 | NvmeCQueue *cq; |
5738 | 0 | NvmeCreateCq *c = (NvmeCreateCq *)&req->cmd; |
5739 | 0 | uint16_t cqid = le16_to_cpu(c->cqid); |
5740 | 0 | uint16_t vector = le16_to_cpu(c->irq_vector); |
5741 | 0 | uint16_t qsize = le16_to_cpu(c->qsize); |
5742 | 0 | uint16_t qflags = le16_to_cpu(c->cq_flags); |
5743 | 0 | uint64_t prp1 = le64_to_cpu(c->prp1); |
5744 | 0 | uint32_t cc = ldq_le_p(&n->bar.cc); |
5745 | 0 | uint8_t iocqes = NVME_CC_IOCQES(cc); |
5746 | 0 | uint8_t iosqes = NVME_CC_IOSQES(cc); |
5747 | |
|
5748 | 0 | trace_pci_nvme_create_cq(prp1, cqid, vector, qsize, qflags, |
5749 | 0 | NVME_CQ_FLAGS_IEN(qflags) != 0); |
5750 | |
|
5751 | 0 | if (iosqes != NVME_SQES || iocqes != NVME_CQES) { |
5752 | 0 | trace_pci_nvme_err_invalid_create_cq_entry_size(iosqes, iocqes); |
5753 | 0 | return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR; |
5754 | 0 | } |
5755 | | |
5756 | 0 | if (unlikely(!cqid || cqid > n->conf_ioqpairs || n->cq[cqid] != NULL)) { |
5757 | 0 | trace_pci_nvme_err_invalid_create_cq_cqid(cqid); |
5758 | 0 | return NVME_INVALID_QID | NVME_DNR; |
5759 | 0 | } |
5760 | 0 | if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) { |
5761 | 0 | trace_pci_nvme_err_invalid_create_cq_size(qsize); |
5762 | 0 | return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR; |
5763 | 0 | } |
5764 | 0 | if (unlikely(prp1 & (n->page_size - 1))) { |
5765 | 0 | trace_pci_nvme_err_invalid_create_cq_addr(prp1); |
5766 | 0 | return NVME_INVALID_PRP_OFFSET | NVME_DNR; |
5767 | 0 | } |
5768 | 0 | if (unlikely(!msix_enabled(PCI_DEVICE(n)) && vector)) { |
5769 | 0 | trace_pci_nvme_err_invalid_create_cq_vector(vector); |
5770 | 0 | return NVME_INVALID_IRQ_VECTOR | NVME_DNR; |
5771 | 0 | } |
5772 | 0 | if (unlikely(vector >= n->conf_msix_qsize)) { |
5773 | 0 | trace_pci_nvme_err_invalid_create_cq_vector(vector); |
5774 | 0 | return NVME_INVALID_IRQ_VECTOR | NVME_DNR; |
5775 | 0 | } |
5776 | 0 | if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) { |
5777 | 0 | trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags)); |
5778 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5779 | 0 | } |
5780 | | |
5781 | 0 | cq = g_malloc0(sizeof(*cq)); |
5782 | 0 | nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1, |
5783 | 0 | NVME_CQ_FLAGS_IEN(qflags)); |
5784 | | |
5785 | | /* |
5786 | | * It is only required to set qs_created when creating a completion queue; |
5787 | | * creating a submission queue without a matching completion queue will |
5788 | | * fail. |
5789 | | */ |
5790 | 0 | n->qs_created = true; |
5791 | 0 | return NVME_SUCCESS; |
5792 | 0 | } |
5793 | | |
5794 | | static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl *n, NvmeRequest *req) |
5795 | 0 | { |
5796 | 0 | uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {}; |
5797 | |
|
5798 | 0 | return nvme_c2h(n, id, sizeof(id), req); |
5799 | 0 | } |
5800 | | |
5801 | | static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeRequest *req) |
5802 | 0 | { |
5803 | 0 | trace_pci_nvme_identify_ctrl(); |
5804 | |
|
5805 | 0 | return nvme_c2h(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl), req); |
5806 | 0 | } |
5807 | | |
5808 | | static uint16_t nvme_identify_ctrl_csi(NvmeCtrl *n, NvmeRequest *req) |
5809 | 0 | { |
5810 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
5811 | 0 | uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {}; |
5812 | 0 | NvmeIdCtrlNvm *id_nvm = (NvmeIdCtrlNvm *)&id; |
5813 | |
|
5814 | 0 | trace_pci_nvme_identify_ctrl_csi(c->csi); |
5815 | |
|
5816 | 0 | switch (c->csi) { |
5817 | 0 | case NVME_CSI_NVM: |
5818 | 0 | id_nvm->vsl = n->params.vsl; |
5819 | 0 | id_nvm->dmrl = NVME_ID_CTRL_NVM_DMRL_MAX; |
5820 | 0 | id_nvm->dmrsl = cpu_to_le32(n->dmrsl); |
5821 | 0 | id_nvm->dmsl = NVME_ID_CTRL_NVM_DMRL_MAX * n->dmrsl; |
5822 | 0 | break; |
5823 | | |
5824 | 0 | case NVME_CSI_ZONED: |
5825 | 0 | ((NvmeIdCtrlZoned *)&id)->zasl = n->params.zasl; |
5826 | 0 | break; |
5827 | | |
5828 | 0 | default: |
5829 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5830 | 0 | } |
5831 | | |
5832 | 0 | return nvme_c2h(n, id, sizeof(id), req); |
5833 | 0 | } |
5834 | | |
5835 | | static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeRequest *req, bool active) |
5836 | 0 | { |
5837 | 0 | NvmeNamespace *ns; |
5838 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
5839 | 0 | uint32_t nsid = le32_to_cpu(c->nsid); |
5840 | |
|
5841 | 0 | trace_pci_nvme_identify_ns(nsid); |
5842 | |
|
5843 | 0 | if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) { |
5844 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
5845 | 0 | } |
5846 | | |
5847 | 0 | ns = nvme_ns(n, nsid); |
5848 | 0 | if (unlikely(!ns)) { |
5849 | 0 | if (!active) { |
5850 | 0 | ns = nvme_subsys_ns(n->subsys, nsid); |
5851 | 0 | if (!ns) { |
5852 | 0 | return nvme_rpt_empty_id_struct(n, req); |
5853 | 0 | } |
5854 | 0 | } else { |
5855 | 0 | return nvme_rpt_empty_id_struct(n, req); |
5856 | 0 | } |
5857 | 0 | } |
5858 | | |
5859 | 0 | if (active || ns->csi == NVME_CSI_NVM) { |
5860 | 0 | return nvme_c2h(n, (uint8_t *)&ns->id_ns, sizeof(NvmeIdNs), req); |
5861 | 0 | } |
5862 | | |
5863 | 0 | return NVME_INVALID_IOCS | NVME_DNR; |
5864 | 0 | } |
5865 | | |
5866 | | static uint16_t nvme_identify_ctrl_list(NvmeCtrl *n, NvmeRequest *req, |
5867 | | bool attached) |
5868 | 0 | { |
5869 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
5870 | 0 | uint32_t nsid = le32_to_cpu(c->nsid); |
5871 | 0 | uint16_t min_id = le16_to_cpu(c->ctrlid); |
5872 | 0 | uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {}; |
5873 | 0 | uint16_t *ids = &list[1]; |
5874 | 0 | NvmeNamespace *ns; |
5875 | 0 | NvmeCtrl *ctrl; |
5876 | 0 | int cntlid, nr_ids = 0; |
5877 | |
|
5878 | 0 | trace_pci_nvme_identify_ctrl_list(c->cns, min_id); |
5879 | |
|
5880 | 0 | if (!n->subsys) { |
5881 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5882 | 0 | } |
5883 | | |
5884 | 0 | if (attached) { |
5885 | 0 | if (nsid == NVME_NSID_BROADCAST) { |
5886 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5887 | 0 | } |
5888 | | |
5889 | 0 | ns = nvme_subsys_ns(n->subsys, nsid); |
5890 | 0 | if (!ns) { |
5891 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
5892 | 0 | } |
5893 | 0 | } |
5894 | | |
5895 | 0 | for (cntlid = min_id; cntlid < ARRAY_SIZE(n->subsys->ctrls); cntlid++) { |
5896 | 0 | ctrl = nvme_subsys_ctrl(n->subsys, cntlid); |
5897 | 0 | if (!ctrl) { |
5898 | 0 | continue; |
5899 | 0 | } |
5900 | | |
5901 | 0 | if (attached && !nvme_ns(ctrl, nsid)) { |
5902 | 0 | continue; |
5903 | 0 | } |
5904 | | |
5905 | 0 | ids[nr_ids++] = cntlid; |
5906 | 0 | } |
5907 | |
|
5908 | 0 | list[0] = nr_ids; |
5909 | |
|
5910 | 0 | return nvme_c2h(n, (uint8_t *)list, sizeof(list), req); |
5911 | 0 | } |
5912 | | |
5913 | | static uint16_t nvme_identify_pri_ctrl_cap(NvmeCtrl *n, NvmeRequest *req) |
5914 | 0 | { |
5915 | 0 | trace_pci_nvme_identify_pri_ctrl_cap(le16_to_cpu(n->pri_ctrl_cap.cntlid)); |
5916 | |
|
5917 | 0 | return nvme_c2h(n, (uint8_t *)&n->pri_ctrl_cap, |
5918 | 0 | sizeof(NvmePriCtrlCap), req); |
5919 | 0 | } |
5920 | | |
5921 | | static uint16_t nvme_identify_sec_ctrl_list(NvmeCtrl *n, NvmeRequest *req) |
5922 | 0 | { |
5923 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
5924 | 0 | uint16_t pri_ctrl_id = le16_to_cpu(n->pri_ctrl_cap.cntlid); |
5925 | 0 | uint16_t min_id = le16_to_cpu(c->ctrlid); |
5926 | 0 | uint8_t num_sec_ctrl = n->nr_sec_ctrls; |
5927 | 0 | NvmeSecCtrlList list = {0}; |
5928 | 0 | uint8_t i; |
5929 | |
|
5930 | 0 | for (i = 0; i < num_sec_ctrl; i++) { |
5931 | 0 | if (n->sec_ctrl_list[i].scid >= min_id) { |
5932 | 0 | list.numcntl = MIN(num_sec_ctrl - i, 127); |
5933 | 0 | memcpy(&list.sec, n->sec_ctrl_list + i, |
5934 | 0 | list.numcntl * sizeof(NvmeSecCtrlEntry)); |
5935 | 0 | break; |
5936 | 0 | } |
5937 | 0 | } |
5938 | |
|
5939 | 0 | trace_pci_nvme_identify_sec_ctrl_list(pri_ctrl_id, list.numcntl); |
5940 | |
|
5941 | 0 | return nvme_c2h(n, (uint8_t *)&list, sizeof(list), req); |
5942 | 0 | } |
5943 | | |
5944 | | static uint16_t nvme_identify_ns_ind(NvmeCtrl *n, NvmeRequest *req, bool alloc) |
5945 | 0 | { |
5946 | 0 | NvmeNamespace *ns; |
5947 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
5948 | 0 | uint32_t nsid = le32_to_cpu(c->nsid); |
5949 | |
|
5950 | 0 | trace_pci_nvme_identify_ns_ind(nsid); |
5951 | |
|
5952 | 0 | if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) { |
5953 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
5954 | 0 | } |
5955 | | |
5956 | 0 | ns = nvme_ns(n, nsid); |
5957 | 0 | if (unlikely(!ns)) { |
5958 | 0 | if (alloc) { |
5959 | 0 | ns = nvme_subsys_ns(n->subsys, nsid); |
5960 | 0 | if (!ns) { |
5961 | 0 | return nvme_rpt_empty_id_struct(n, req); |
5962 | 0 | } |
5963 | 0 | } else { |
5964 | 0 | return nvme_rpt_empty_id_struct(n, req); |
5965 | 0 | } |
5966 | 0 | } |
5967 | | |
5968 | 0 | return nvme_c2h(n, (uint8_t *)&ns->id_ns_ind, sizeof(NvmeIdNsInd), req); |
5969 | 0 | } |
5970 | | |
5971 | | static uint16_t nvme_identify_ns_csi(NvmeCtrl *n, NvmeRequest *req, |
5972 | | bool active) |
5973 | 0 | { |
5974 | 0 | NvmeNamespace *ns; |
5975 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
5976 | 0 | uint32_t nsid = le32_to_cpu(c->nsid); |
5977 | |
|
5978 | 0 | trace_pci_nvme_identify_ns_csi(nsid, c->csi); |
5979 | |
|
5980 | 0 | if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) { |
5981 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
5982 | 0 | } |
5983 | | |
5984 | 0 | ns = nvme_ns(n, nsid); |
5985 | 0 | if (unlikely(!ns)) { |
5986 | 0 | if (!active) { |
5987 | 0 | ns = nvme_subsys_ns(n->subsys, nsid); |
5988 | 0 | if (!ns) { |
5989 | 0 | return nvme_rpt_empty_id_struct(n, req); |
5990 | 0 | } |
5991 | 0 | } else { |
5992 | 0 | return nvme_rpt_empty_id_struct(n, req); |
5993 | 0 | } |
5994 | 0 | } |
5995 | | |
5996 | 0 | if (c->csi == NVME_CSI_NVM) { |
5997 | 0 | return nvme_c2h(n, (uint8_t *)&ns->id_ns_nvm, sizeof(NvmeIdNsNvm), |
5998 | 0 | req); |
5999 | 0 | } else if (c->csi == NVME_CSI_ZONED && ns->csi == NVME_CSI_ZONED) { |
6000 | 0 | return nvme_c2h(n, (uint8_t *)ns->id_ns_zoned, sizeof(NvmeIdNsZoned), |
6001 | 0 | req); |
6002 | 0 | } |
6003 | | |
6004 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6005 | 0 | } |
6006 | | |
6007 | | static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req, |
6008 | | bool active) |
6009 | 0 | { |
6010 | 0 | NvmeNamespace *ns; |
6011 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
6012 | 0 | uint32_t min_nsid = le32_to_cpu(c->nsid); |
6013 | 0 | uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {}; |
6014 | 0 | static const int data_len = sizeof(list); |
6015 | 0 | uint32_t *list_ptr = (uint32_t *)list; |
6016 | 0 | int i, j = 0; |
6017 | |
|
6018 | 0 | trace_pci_nvme_identify_nslist(min_nsid); |
6019 | | |
6020 | | /* |
6021 | | * Both FFFFFFFFh (NVME_NSID_BROADCAST) and FFFFFFFFEh are invalid values |
6022 | | * since the Active Namespace ID List should return namespaces with ids |
6023 | | * *higher* than the NSID specified in the command. This is also specified |
6024 | | * in the spec (NVM Express v1.3d, Section 5.15.4). |
6025 | | */ |
6026 | 0 | if (min_nsid >= NVME_NSID_BROADCAST - 1) { |
6027 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
6028 | 0 | } |
6029 | | |
6030 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
6031 | 0 | ns = nvme_ns(n, i); |
6032 | 0 | if (!ns) { |
6033 | 0 | if (!active) { |
6034 | 0 | ns = nvme_subsys_ns(n->subsys, i); |
6035 | 0 | if (!ns) { |
6036 | 0 | continue; |
6037 | 0 | } |
6038 | 0 | } else { |
6039 | 0 | continue; |
6040 | 0 | } |
6041 | 0 | } |
6042 | 0 | if (ns->params.nsid <= min_nsid) { |
6043 | 0 | continue; |
6044 | 0 | } |
6045 | 0 | list_ptr[j++] = cpu_to_le32(ns->params.nsid); |
6046 | 0 | if (j == data_len / sizeof(uint32_t)) { |
6047 | 0 | break; |
6048 | 0 | } |
6049 | 0 | } |
6050 | |
|
6051 | 0 | return nvme_c2h(n, list, data_len, req); |
6052 | 0 | } |
6053 | | |
6054 | | static uint16_t nvme_identify_nslist_csi(NvmeCtrl *n, NvmeRequest *req, |
6055 | | bool active) |
6056 | 0 | { |
6057 | 0 | NvmeNamespace *ns; |
6058 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
6059 | 0 | uint32_t min_nsid = le32_to_cpu(c->nsid); |
6060 | 0 | uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {}; |
6061 | 0 | static const int data_len = sizeof(list); |
6062 | 0 | uint32_t *list_ptr = (uint32_t *)list; |
6063 | 0 | int i, j = 0; |
6064 | |
|
6065 | 0 | trace_pci_nvme_identify_nslist_csi(min_nsid, c->csi); |
6066 | | |
6067 | | /* |
6068 | | * Same as in nvme_identify_nslist(), FFFFFFFFh/FFFFFFFFEh are invalid. |
6069 | | */ |
6070 | 0 | if (min_nsid >= NVME_NSID_BROADCAST - 1) { |
6071 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
6072 | 0 | } |
6073 | | |
6074 | 0 | if (c->csi != NVME_CSI_NVM && c->csi != NVME_CSI_ZONED) { |
6075 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6076 | 0 | } |
6077 | | |
6078 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
6079 | 0 | ns = nvme_ns(n, i); |
6080 | 0 | if (!ns) { |
6081 | 0 | if (!active) { |
6082 | 0 | ns = nvme_subsys_ns(n->subsys, i); |
6083 | 0 | if (!ns) { |
6084 | 0 | continue; |
6085 | 0 | } |
6086 | 0 | } else { |
6087 | 0 | continue; |
6088 | 0 | } |
6089 | 0 | } |
6090 | 0 | if (ns->params.nsid <= min_nsid || c->csi != ns->csi) { |
6091 | 0 | continue; |
6092 | 0 | } |
6093 | 0 | list_ptr[j++] = cpu_to_le32(ns->params.nsid); |
6094 | 0 | if (j == data_len / sizeof(uint32_t)) { |
6095 | 0 | break; |
6096 | 0 | } |
6097 | 0 | } |
6098 | |
|
6099 | 0 | return nvme_c2h(n, list, data_len, req); |
6100 | 0 | } |
6101 | | |
6102 | | static uint16_t nvme_endurance_group_list(NvmeCtrl *n, NvmeRequest *req) |
6103 | 0 | { |
6104 | 0 | uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {}; |
6105 | 0 | uint16_t *nr_ids = &list[0]; |
6106 | 0 | uint16_t *ids = &list[1]; |
6107 | 0 | uint16_t endgid = le32_to_cpu(req->cmd.cdw11) & 0xffff; |
6108 | | |
6109 | | /* |
6110 | | * The current nvme-subsys only supports Endurance Group #1. |
6111 | | */ |
6112 | 0 | if (!endgid) { |
6113 | 0 | *nr_ids = 1; |
6114 | 0 | ids[0] = 1; |
6115 | 0 | } else { |
6116 | 0 | *nr_ids = 0; |
6117 | 0 | } |
6118 | |
|
6119 | 0 | return nvme_c2h(n, list, sizeof(list), req); |
6120 | 0 | } |
6121 | | |
6122 | | static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeRequest *req) |
6123 | 0 | { |
6124 | 0 | NvmeNamespace *ns; |
6125 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
6126 | 0 | uint32_t nsid = le32_to_cpu(c->nsid); |
6127 | 0 | uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {}; |
6128 | 0 | uint8_t *pos = list; |
6129 | 0 | struct { |
6130 | 0 | NvmeIdNsDescr hdr; |
6131 | 0 | uint8_t v[NVME_NIDL_UUID]; |
6132 | 0 | } QEMU_PACKED uuid = {}; |
6133 | 0 | struct { |
6134 | 0 | NvmeIdNsDescr hdr; |
6135 | 0 | uint8_t v[NVME_NIDL_NGUID]; |
6136 | 0 | } QEMU_PACKED nguid = {}; |
6137 | 0 | struct { |
6138 | 0 | NvmeIdNsDescr hdr; |
6139 | 0 | uint64_t v; |
6140 | 0 | } QEMU_PACKED eui64 = {}; |
6141 | 0 | struct { |
6142 | 0 | NvmeIdNsDescr hdr; |
6143 | 0 | uint8_t v; |
6144 | 0 | } QEMU_PACKED csi = {}; |
6145 | |
|
6146 | 0 | trace_pci_nvme_identify_ns_descr_list(nsid); |
6147 | |
|
6148 | 0 | if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) { |
6149 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
6150 | 0 | } |
6151 | | |
6152 | 0 | ns = nvme_ns(n, nsid); |
6153 | 0 | if (unlikely(!ns)) { |
6154 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6155 | 0 | } |
6156 | | |
6157 | 0 | if (!qemu_uuid_is_null(&ns->params.uuid)) { |
6158 | 0 | uuid.hdr.nidt = NVME_NIDT_UUID; |
6159 | 0 | uuid.hdr.nidl = NVME_NIDL_UUID; |
6160 | 0 | memcpy(uuid.v, ns->params.uuid.data, NVME_NIDL_UUID); |
6161 | 0 | memcpy(pos, &uuid, sizeof(uuid)); |
6162 | 0 | pos += sizeof(uuid); |
6163 | 0 | } |
6164 | |
|
6165 | 0 | if (!nvme_nguid_is_null(&ns->params.nguid)) { |
6166 | 0 | nguid.hdr.nidt = NVME_NIDT_NGUID; |
6167 | 0 | nguid.hdr.nidl = NVME_NIDL_NGUID; |
6168 | 0 | memcpy(nguid.v, ns->params.nguid.data, NVME_NIDL_NGUID); |
6169 | 0 | memcpy(pos, &nguid, sizeof(nguid)); |
6170 | 0 | pos += sizeof(nguid); |
6171 | 0 | } |
6172 | |
|
6173 | 0 | if (ns->params.eui64) { |
6174 | 0 | eui64.hdr.nidt = NVME_NIDT_EUI64; |
6175 | 0 | eui64.hdr.nidl = NVME_NIDL_EUI64; |
6176 | 0 | eui64.v = cpu_to_be64(ns->params.eui64); |
6177 | 0 | memcpy(pos, &eui64, sizeof(eui64)); |
6178 | 0 | pos += sizeof(eui64); |
6179 | 0 | } |
6180 | |
|
6181 | 0 | csi.hdr.nidt = NVME_NIDT_CSI; |
6182 | 0 | csi.hdr.nidl = NVME_NIDL_CSI; |
6183 | 0 | csi.v = ns->csi; |
6184 | 0 | memcpy(pos, &csi, sizeof(csi)); |
6185 | 0 | pos += sizeof(csi); |
6186 | |
|
6187 | 0 | return nvme_c2h(n, list, sizeof(list), req); |
6188 | 0 | } |
6189 | | |
6190 | | static uint16_t nvme_identify_cmd_set(NvmeCtrl *n, NvmeRequest *req) |
6191 | 0 | { |
6192 | 0 | uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {}; |
6193 | 0 | static const int data_len = sizeof(list); |
6194 | |
|
6195 | 0 | trace_pci_nvme_identify_cmd_set(); |
6196 | |
|
6197 | 0 | NVME_SET_CSI(*list, NVME_CSI_NVM); |
6198 | 0 | NVME_SET_CSI(*list, NVME_CSI_ZONED); |
6199 | |
|
6200 | 0 | return nvme_c2h(n, list, data_len, req); |
6201 | 0 | } |
6202 | | |
6203 | | static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req) |
6204 | 0 | { |
6205 | 0 | NvmeIdentify *c = (NvmeIdentify *)&req->cmd; |
6206 | |
|
6207 | 0 | trace_pci_nvme_identify(nvme_cid(req), c->cns, le16_to_cpu(c->ctrlid), |
6208 | 0 | c->csi); |
6209 | |
|
6210 | 0 | switch (c->cns) { |
6211 | 0 | case NVME_ID_CNS_NS: |
6212 | 0 | return nvme_identify_ns(n, req, true); |
6213 | 0 | case NVME_ID_CNS_NS_PRESENT: |
6214 | 0 | return nvme_identify_ns(n, req, false); |
6215 | 0 | case NVME_ID_CNS_NS_ATTACHED_CTRL_LIST: |
6216 | 0 | return nvme_identify_ctrl_list(n, req, true); |
6217 | 0 | case NVME_ID_CNS_CTRL_LIST: |
6218 | 0 | return nvme_identify_ctrl_list(n, req, false); |
6219 | 0 | case NVME_ID_CNS_PRIMARY_CTRL_CAP: |
6220 | 0 | return nvme_identify_pri_ctrl_cap(n, req); |
6221 | 0 | case NVME_ID_CNS_SECONDARY_CTRL_LIST: |
6222 | 0 | return nvme_identify_sec_ctrl_list(n, req); |
6223 | 0 | case NVME_ID_CNS_CS_NS: |
6224 | 0 | return nvme_identify_ns_csi(n, req, true); |
6225 | 0 | case NVME_ID_CNS_CS_IND_NS: |
6226 | 0 | return nvme_identify_ns_ind(n, req, false); |
6227 | 0 | case NVME_ID_CNS_CS_IND_NS_ALLOCATED: |
6228 | 0 | return nvme_identify_ns_ind(n, req, true); |
6229 | 0 | case NVME_ID_CNS_CS_NS_PRESENT: |
6230 | 0 | return nvme_identify_ns_csi(n, req, false); |
6231 | 0 | case NVME_ID_CNS_CTRL: |
6232 | 0 | return nvme_identify_ctrl(n, req); |
6233 | 0 | case NVME_ID_CNS_CS_CTRL: |
6234 | 0 | return nvme_identify_ctrl_csi(n, req); |
6235 | 0 | case NVME_ID_CNS_NS_ACTIVE_LIST: |
6236 | 0 | return nvme_identify_nslist(n, req, true); |
6237 | 0 | case NVME_ID_CNS_NS_PRESENT_LIST: |
6238 | 0 | return nvme_identify_nslist(n, req, false); |
6239 | 0 | case NVME_ID_CNS_CS_NS_ACTIVE_LIST: |
6240 | 0 | return nvme_identify_nslist_csi(n, req, true); |
6241 | 0 | case NVME_ID_CNS_ENDURANCE_GROUP_LIST: |
6242 | 0 | return nvme_endurance_group_list(n, req); |
6243 | 0 | case NVME_ID_CNS_CS_NS_PRESENT_LIST: |
6244 | 0 | return nvme_identify_nslist_csi(n, req, false); |
6245 | 0 | case NVME_ID_CNS_NS_DESCR_LIST: |
6246 | 0 | return nvme_identify_ns_descr_list(n, req); |
6247 | 0 | case NVME_ID_CNS_IO_COMMAND_SET: |
6248 | 0 | return nvme_identify_cmd_set(n, req); |
6249 | 0 | default: |
6250 | 0 | trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns)); |
6251 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6252 | 0 | } |
6253 | 0 | } |
6254 | | |
6255 | | static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req) |
6256 | 0 | { |
6257 | 0 | uint16_t sqid = le32_to_cpu(req->cmd.cdw10) & 0xffff; |
6258 | 0 | uint16_t cid = (le32_to_cpu(req->cmd.cdw10) >> 16) & 0xffff; |
6259 | 0 | NvmeSQueue *sq; |
6260 | 0 | NvmeRequest *r, *next; |
6261 | 0 | int i; |
6262 | |
|
6263 | 0 | req->cqe.result = 1; |
6264 | 0 | if (nvme_check_sqid(n, sqid)) { |
6265 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6266 | 0 | } |
6267 | | |
6268 | 0 | sq = n->sq[sqid]; |
6269 | |
|
6270 | 0 | if (sqid == 0) { |
6271 | 0 | for (i = 0; i < n->outstanding_aers; i++) { |
6272 | 0 | NvmeRequest *re = n->aer_reqs[i]; |
6273 | 0 | if (re->cqe.cid == cid) { |
6274 | 0 | memmove(n->aer_reqs + i, n->aer_reqs + i + 1, |
6275 | 0 | (n->outstanding_aers - i - 1) * sizeof(NvmeRequest *)); |
6276 | 0 | n->outstanding_aers--; |
6277 | 0 | re->status = NVME_CMD_ABORT_REQ; |
6278 | 0 | req->cqe.result = 0; |
6279 | 0 | nvme_enqueue_req_completion(&n->admin_cq, re); |
6280 | 0 | return NVME_SUCCESS; |
6281 | 0 | } |
6282 | 0 | } |
6283 | 0 | } |
6284 | | |
6285 | 0 | QTAILQ_FOREACH_SAFE(r, &sq->out_req_list, entry, next) { |
6286 | 0 | if (r->cqe.cid == cid) { |
6287 | 0 | if (r->aiocb) { |
6288 | 0 | r->status = NVME_CMD_ABORT_REQ; |
6289 | 0 | blk_aio_cancel_async(r->aiocb); |
6290 | 0 | } |
6291 | 0 | break; |
6292 | 0 | } |
6293 | 0 | } |
6294 | |
|
6295 | 0 | return NVME_SUCCESS; |
6296 | 0 | } |
6297 | | |
6298 | | static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts) |
6299 | 0 | { |
6300 | 0 | trace_pci_nvme_setfeat_timestamp(ts); |
6301 | |
|
6302 | 0 | n->host_timestamp = le64_to_cpu(ts); |
6303 | 0 | n->timestamp_set_qemu_clock_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL); |
6304 | 0 | } |
6305 | | |
6306 | | static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n) |
6307 | 0 | { |
6308 | 0 | uint64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL); |
6309 | 0 | uint64_t elapsed_time = current_time - n->timestamp_set_qemu_clock_ms; |
6310 | |
|
6311 | 0 | union nvme_timestamp { |
6312 | 0 | struct { |
6313 | 0 | uint64_t timestamp:48; |
6314 | 0 | uint64_t sync:1; |
6315 | 0 | uint64_t origin:3; |
6316 | 0 | uint64_t rsvd1:12; |
6317 | 0 | }; |
6318 | 0 | uint64_t all; |
6319 | 0 | }; |
6320 | |
|
6321 | 0 | union nvme_timestamp ts; |
6322 | 0 | ts.all = 0; |
6323 | 0 | ts.timestamp = n->host_timestamp + elapsed_time; |
6324 | | |
6325 | | /* If the host timestamp is non-zero, set the timestamp origin */ |
6326 | 0 | ts.origin = n->host_timestamp ? 0x01 : 0x00; |
6327 | |
|
6328 | 0 | trace_pci_nvme_getfeat_timestamp(ts.all); |
6329 | |
|
6330 | 0 | return cpu_to_le64(ts.all); |
6331 | 0 | } |
6332 | | |
6333 | | static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeRequest *req) |
6334 | 0 | { |
6335 | 0 | uint64_t timestamp = nvme_get_timestamp(n); |
6336 | |
|
6337 | 0 | return nvme_c2h(n, (uint8_t *)×tamp, sizeof(timestamp), req); |
6338 | 0 | } |
6339 | | |
6340 | | static int nvme_get_feature_fdp(NvmeCtrl *n, uint32_t endgrpid, |
6341 | | uint32_t *result) |
6342 | 0 | { |
6343 | 0 | *result = 0; |
6344 | |
|
6345 | 0 | if (!n->subsys || !n->subsys->endgrp.fdp.enabled) { |
6346 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6347 | 0 | } |
6348 | | |
6349 | 0 | *result = FIELD_DP16(0, FEAT_FDP, FDPE, 1); |
6350 | 0 | *result = FIELD_DP16(*result, FEAT_FDP, CONF_NDX, 0); |
6351 | |
|
6352 | 0 | return NVME_SUCCESS; |
6353 | 0 | } |
6354 | | |
6355 | | static uint16_t nvme_get_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns, |
6356 | | NvmeRequest *req, uint32_t *result) |
6357 | 0 | { |
6358 | 0 | NvmeCmd *cmd = &req->cmd; |
6359 | 0 | uint32_t cdw11 = le32_to_cpu(cmd->cdw11); |
6360 | 0 | uint16_t ph = cdw11 & 0xffff; |
6361 | 0 | uint8_t noet = (cdw11 >> 16) & 0xff; |
6362 | 0 | uint16_t ruhid, ret; |
6363 | 0 | uint32_t nentries = 0; |
6364 | 0 | uint8_t s_events_ndx = 0; |
6365 | 0 | size_t s_events_siz = sizeof(NvmeFdpEventDescr) * noet; |
6366 | 0 | g_autofree NvmeFdpEventDescr *s_events = g_malloc0(s_events_siz); |
6367 | 0 | NvmeRuHandle *ruh; |
6368 | 0 | NvmeFdpEventDescr *s_event; |
6369 | |
|
6370 | 0 | if (!n->subsys || !n->subsys->endgrp.fdp.enabled) { |
6371 | 0 | return NVME_FDP_DISABLED | NVME_DNR; |
6372 | 0 | } |
6373 | | |
6374 | 0 | if (!nvme_ph_valid(ns, ph)) { |
6375 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6376 | 0 | } |
6377 | | |
6378 | 0 | ruhid = ns->fdp.phs[ph]; |
6379 | 0 | ruh = &n->subsys->endgrp.fdp.ruhs[ruhid]; |
6380 | |
|
6381 | 0 | assert(ruh); |
6382 | |
|
6383 | 0 | if (unlikely(noet == 0)) { |
6384 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6385 | 0 | } |
6386 | | |
6387 | 0 | for (uint8_t event_type = 0; event_type < FDP_EVT_MAX; event_type++) { |
6388 | 0 | uint8_t shift = nvme_fdp_evf_shifts[event_type]; |
6389 | 0 | if (!shift && event_type) { |
6390 | 0 | continue; |
6391 | 0 | } |
6392 | | |
6393 | 0 | nentries++; |
6394 | |
|
6395 | 0 | s_event = &s_events[s_events_ndx]; |
6396 | 0 | s_event->evt = event_type; |
6397 | 0 | s_event->evta = (ruh->event_filter >> shift) & 0x1; |
6398 | | |
6399 | | /* break if all `noet` entries are filled */ |
6400 | 0 | if ((++s_events_ndx) == noet) { |
6401 | 0 | break; |
6402 | 0 | } |
6403 | 0 | } |
6404 | |
|
6405 | 0 | ret = nvme_c2h(n, s_events, s_events_siz, req); |
6406 | 0 | if (ret) { |
6407 | 0 | return ret; |
6408 | 0 | } |
6409 | | |
6410 | 0 | *result = nentries; |
6411 | 0 | return NVME_SUCCESS; |
6412 | 0 | } |
6413 | | |
6414 | | static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req) |
6415 | 0 | { |
6416 | 0 | NvmeCmd *cmd = &req->cmd; |
6417 | 0 | uint32_t dw10 = le32_to_cpu(cmd->cdw10); |
6418 | 0 | uint32_t dw11 = le32_to_cpu(cmd->cdw11); |
6419 | 0 | uint32_t nsid = le32_to_cpu(cmd->nsid); |
6420 | 0 | uint32_t result = 0; |
6421 | 0 | uint8_t fid = NVME_GETSETFEAT_FID(dw10); |
6422 | 0 | NvmeGetFeatureSelect sel = NVME_GETFEAT_SELECT(dw10); |
6423 | 0 | uint16_t iv; |
6424 | 0 | NvmeNamespace *ns; |
6425 | 0 | int i; |
6426 | 0 | uint16_t endgrpid = 0, ret = NVME_SUCCESS; |
6427 | |
|
6428 | 0 | static const uint32_t nvme_feature_default[NVME_FID_MAX] = { |
6429 | 0 | [NVME_ARBITRATION] = NVME_ARB_AB_NOLIMIT, |
6430 | 0 | }; |
6431 | |
|
6432 | 0 | trace_pci_nvme_getfeat(nvme_cid(req), nsid, fid, sel, dw11); |
6433 | |
|
6434 | 0 | if (!nvme_feature_support[fid]) { |
6435 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6436 | 0 | } |
6437 | | |
6438 | 0 | if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) { |
6439 | 0 | if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) { |
6440 | | /* |
6441 | | * The Reservation Notification Mask and Reservation Persistence |
6442 | | * features require a status code of Invalid Field in Command when |
6443 | | * NSID is FFFFFFFFh. Since the device does not support those |
6444 | | * features we can always return Invalid Namespace or Format as we |
6445 | | * should do for all other features. |
6446 | | */ |
6447 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
6448 | 0 | } |
6449 | | |
6450 | 0 | if (!nvme_ns(n, nsid)) { |
6451 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6452 | 0 | } |
6453 | 0 | } |
6454 | | |
6455 | 0 | switch (sel) { |
6456 | 0 | case NVME_GETFEAT_SELECT_CURRENT: |
6457 | 0 | break; |
6458 | 0 | case NVME_GETFEAT_SELECT_SAVED: |
6459 | | /* no features are saveable by the controller; fallthrough */ |
6460 | 0 | case NVME_GETFEAT_SELECT_DEFAULT: |
6461 | 0 | goto defaults; |
6462 | 0 | case NVME_GETFEAT_SELECT_CAP: |
6463 | 0 | result = nvme_feature_cap[fid]; |
6464 | 0 | goto out; |
6465 | 0 | } |
6466 | | |
6467 | 0 | switch (fid) { |
6468 | 0 | case NVME_TEMPERATURE_THRESHOLD: |
6469 | 0 | result = 0; |
6470 | | |
6471 | | /* |
6472 | | * The controller only implements the Composite Temperature sensor, so |
6473 | | * return 0 for all other sensors. |
6474 | | */ |
6475 | 0 | if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) { |
6476 | 0 | goto out; |
6477 | 0 | } |
6478 | | |
6479 | 0 | switch (NVME_TEMP_THSEL(dw11)) { |
6480 | 0 | case NVME_TEMP_THSEL_OVER: |
6481 | 0 | result = n->features.temp_thresh_hi; |
6482 | 0 | goto out; |
6483 | 0 | case NVME_TEMP_THSEL_UNDER: |
6484 | 0 | result = n->features.temp_thresh_low; |
6485 | 0 | goto out; |
6486 | 0 | } |
6487 | | |
6488 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6489 | 0 | case NVME_ERROR_RECOVERY: |
6490 | 0 | if (!nvme_nsid_valid(n, nsid)) { |
6491 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
6492 | 0 | } |
6493 | | |
6494 | 0 | ns = nvme_ns(n, nsid); |
6495 | 0 | if (unlikely(!ns)) { |
6496 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6497 | 0 | } |
6498 | | |
6499 | 0 | result = ns->features.err_rec; |
6500 | 0 | goto out; |
6501 | 0 | case NVME_VOLATILE_WRITE_CACHE: |
6502 | 0 | result = 0; |
6503 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
6504 | 0 | ns = nvme_ns(n, i); |
6505 | 0 | if (!ns) { |
6506 | 0 | continue; |
6507 | 0 | } |
6508 | | |
6509 | 0 | result = blk_enable_write_cache(ns->blkconf.blk); |
6510 | 0 | if (result) { |
6511 | 0 | break; |
6512 | 0 | } |
6513 | 0 | } |
6514 | 0 | trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled"); |
6515 | 0 | goto out; |
6516 | 0 | case NVME_ASYNCHRONOUS_EVENT_CONF: |
6517 | 0 | result = n->features.async_config; |
6518 | 0 | goto out; |
6519 | 0 | case NVME_TIMESTAMP: |
6520 | 0 | return nvme_get_feature_timestamp(n, req); |
6521 | 0 | case NVME_HOST_BEHAVIOR_SUPPORT: |
6522 | 0 | return nvme_c2h(n, (uint8_t *)&n->features.hbs, |
6523 | 0 | sizeof(n->features.hbs), req); |
6524 | 0 | case NVME_FDP_MODE: |
6525 | 0 | endgrpid = dw11 & 0xff; |
6526 | |
|
6527 | 0 | if (endgrpid != 0x1) { |
6528 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6529 | 0 | } |
6530 | | |
6531 | 0 | ret = nvme_get_feature_fdp(n, endgrpid, &result); |
6532 | 0 | if (ret) { |
6533 | 0 | return ret; |
6534 | 0 | } |
6535 | 0 | goto out; |
6536 | 0 | case NVME_FDP_EVENTS: |
6537 | 0 | if (!nvme_nsid_valid(n, nsid)) { |
6538 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
6539 | 0 | } |
6540 | | |
6541 | 0 | ns = nvme_ns(n, nsid); |
6542 | 0 | if (unlikely(!ns)) { |
6543 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6544 | 0 | } |
6545 | | |
6546 | 0 | ret = nvme_get_feature_fdp_events(n, ns, req, &result); |
6547 | 0 | if (ret) { |
6548 | 0 | return ret; |
6549 | 0 | } |
6550 | 0 | goto out; |
6551 | 0 | default: |
6552 | 0 | break; |
6553 | 0 | } |
6554 | | |
6555 | 0 | defaults: |
6556 | 0 | switch (fid) { |
6557 | 0 | case NVME_TEMPERATURE_THRESHOLD: |
6558 | 0 | result = 0; |
6559 | |
|
6560 | 0 | if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) { |
6561 | 0 | break; |
6562 | 0 | } |
6563 | | |
6564 | 0 | if (NVME_TEMP_THSEL(dw11) == NVME_TEMP_THSEL_OVER) { |
6565 | 0 | result = NVME_TEMPERATURE_WARNING; |
6566 | 0 | } |
6567 | |
|
6568 | 0 | break; |
6569 | 0 | case NVME_NUMBER_OF_QUEUES: |
6570 | 0 | result = (n->conf_ioqpairs - 1) | ((n->conf_ioqpairs - 1) << 16); |
6571 | 0 | trace_pci_nvme_getfeat_numq(result); |
6572 | 0 | break; |
6573 | 0 | case NVME_INTERRUPT_VECTOR_CONF: |
6574 | 0 | iv = dw11 & 0xffff; |
6575 | 0 | if (iv >= n->conf_ioqpairs + 1) { |
6576 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6577 | 0 | } |
6578 | | |
6579 | 0 | result = iv; |
6580 | 0 | if (iv == n->admin_cq.vector) { |
6581 | 0 | result |= NVME_INTVC_NOCOALESCING; |
6582 | 0 | } |
6583 | 0 | break; |
6584 | 0 | case NVME_FDP_MODE: |
6585 | 0 | endgrpid = dw11 & 0xff; |
6586 | |
|
6587 | 0 | if (endgrpid != 0x1) { |
6588 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6589 | 0 | } |
6590 | | |
6591 | 0 | ret = nvme_get_feature_fdp(n, endgrpid, &result); |
6592 | 0 | if (ret) { |
6593 | 0 | return ret; |
6594 | 0 | } |
6595 | 0 | break; |
6596 | | |
6597 | 0 | case NVME_WRITE_ATOMICITY: |
6598 | 0 | result = n->dn; |
6599 | 0 | break; |
6600 | 0 | default: |
6601 | 0 | result = nvme_feature_default[fid]; |
6602 | 0 | break; |
6603 | 0 | } |
6604 | | |
6605 | 0 | out: |
6606 | 0 | req->cqe.result = cpu_to_le32(result); |
6607 | 0 | return ret; |
6608 | 0 | } |
6609 | | |
6610 | | static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeRequest *req) |
6611 | 0 | { |
6612 | 0 | uint16_t ret; |
6613 | 0 | uint64_t timestamp; |
6614 | |
|
6615 | 0 | ret = nvme_h2c(n, (uint8_t *)×tamp, sizeof(timestamp), req); |
6616 | 0 | if (ret) { |
6617 | 0 | return ret; |
6618 | 0 | } |
6619 | | |
6620 | 0 | nvme_set_timestamp(n, timestamp); |
6621 | |
|
6622 | 0 | return NVME_SUCCESS; |
6623 | 0 | } |
6624 | | |
6625 | | static uint16_t nvme_set_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns, |
6626 | | NvmeRequest *req) |
6627 | 0 | { |
6628 | 0 | NvmeCmd *cmd = &req->cmd; |
6629 | 0 | uint32_t cdw11 = le32_to_cpu(cmd->cdw11); |
6630 | 0 | uint16_t ph = cdw11 & 0xffff; |
6631 | 0 | uint8_t noet = (cdw11 >> 16) & 0xff; |
6632 | 0 | uint16_t ret, ruhid; |
6633 | 0 | uint8_t enable = le32_to_cpu(cmd->cdw12) & 0x1; |
6634 | 0 | uint64_t event_mask = 0; |
6635 | 0 | unsigned int i; |
6636 | 0 | g_autofree uint8_t *events = NULL; |
6637 | 0 | NvmeRuHandle *ruh = NULL; |
6638 | |
|
6639 | 0 | if (!ns) { |
6640 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6641 | 0 | } |
6642 | | |
6643 | 0 | if (!n->subsys || !n->subsys->endgrp.fdp.enabled) { |
6644 | 0 | return NVME_FDP_DISABLED | NVME_DNR; |
6645 | 0 | } |
6646 | | |
6647 | 0 | if (!nvme_ph_valid(ns, ph)) { |
6648 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6649 | 0 | } |
6650 | | |
6651 | 0 | if (unlikely(noet == 0)) { |
6652 | 0 | return NVME_SUCCESS; |
6653 | 0 | } |
6654 | | |
6655 | 0 | ruhid = ns->fdp.phs[ph]; |
6656 | 0 | ruh = &n->subsys->endgrp.fdp.ruhs[ruhid]; |
6657 | |
|
6658 | 0 | events = g_malloc0(noet); |
6659 | |
|
6660 | 0 | ret = nvme_h2c(n, events, noet, req); |
6661 | 0 | if (ret) { |
6662 | 0 | return ret; |
6663 | 0 | } |
6664 | | |
6665 | 0 | for (i = 0; i < noet; i++) { |
6666 | | /* |
6667 | | * We ignore requests to enable tracking of unsupported FDP event types |
6668 | | */ |
6669 | 0 | uint8_t event_type = events[i]; |
6670 | 0 | uint8_t shift = nvme_fdp_evf_shifts[event_type]; |
6671 | 0 | if (!shift && event_type) { |
6672 | 0 | continue; |
6673 | 0 | } |
6674 | 0 | event_mask = |
6675 | 0 | deposit64(event_mask, nvme_fdp_evf_shifts[events[i]], 1, 1); |
6676 | 0 | } |
6677 | |
|
6678 | 0 | if (enable) { |
6679 | 0 | ruh->event_filter |= event_mask; |
6680 | 0 | } else { |
6681 | 0 | ruh->event_filter = ruh->event_filter & ~event_mask; |
6682 | 0 | } |
6683 | |
|
6684 | 0 | return NVME_SUCCESS; |
6685 | 0 | } |
6686 | | |
6687 | | void nvme_atomic_configure_max_write_size(bool dn, uint16_t awun, |
6688 | | uint16_t awupf, NvmeAtomic *atomic) |
6689 | 0 | { |
6690 | 0 | atomic->atomic_max_write_size = (dn ? awupf : awun) + 1; |
6691 | |
|
6692 | 0 | if (atomic->atomic_max_write_size > 1) { |
6693 | 0 | atomic->atomic_writes = 1; |
6694 | 0 | } |
6695 | 0 | } |
6696 | | |
6697 | | static uint16_t nvme_set_feature_write_atomicity(NvmeCtrl *n, NvmeRequest *req) |
6698 | 0 | { |
6699 | 0 | NvmeCmd *cmd = &req->cmd; |
6700 | |
|
6701 | 0 | uint32_t dw11 = le32_to_cpu(cmd->cdw11); |
6702 | |
|
6703 | 0 | uint16_t awun = le16_to_cpu(n->id_ctrl.awun); |
6704 | 0 | uint16_t awupf = le16_to_cpu(n->id_ctrl.awupf); |
6705 | |
|
6706 | 0 | n->dn = dw11 & 0x1; |
6707 | |
|
6708 | 0 | nvme_atomic_configure_max_write_size(n->dn, awun, awupf, &n->atomic); |
6709 | |
|
6710 | 0 | for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
6711 | 0 | uint16_t nawun, nawupf, nabsn, nabspf; |
6712 | |
|
6713 | 0 | NvmeNamespace *ns = nvme_ns(n, i); |
6714 | 0 | if (!ns) { |
6715 | 0 | continue; |
6716 | 0 | } |
6717 | | |
6718 | 0 | nawun = le16_to_cpu(ns->id_ns.nawun); |
6719 | 0 | nawupf = le16_to_cpu(ns->id_ns.nawupf); |
6720 | |
|
6721 | 0 | nvme_atomic_configure_max_write_size(n->dn, nawun, nawupf, |
6722 | 0 | &ns->atomic); |
6723 | |
|
6724 | 0 | nabsn = le16_to_cpu(ns->id_ns.nabsn); |
6725 | 0 | nabspf = le16_to_cpu(ns->id_ns.nabspf); |
6726 | |
|
6727 | 0 | nvme_ns_atomic_configure_boundary(n->dn, nabsn, nabspf, |
6728 | 0 | &ns->atomic); |
6729 | 0 | } |
6730 | |
|
6731 | 0 | return NVME_SUCCESS; |
6732 | 0 | } |
6733 | | |
6734 | | static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req) |
6735 | 0 | { |
6736 | 0 | NvmeNamespace *ns = NULL; |
6737 | |
|
6738 | 0 | NvmeCmd *cmd = &req->cmd; |
6739 | 0 | uint32_t dw10 = le32_to_cpu(cmd->cdw10); |
6740 | 0 | uint32_t dw11 = le32_to_cpu(cmd->cdw11); |
6741 | 0 | uint32_t nsid = le32_to_cpu(cmd->nsid); |
6742 | 0 | uint8_t fid = NVME_GETSETFEAT_FID(dw10); |
6743 | 0 | uint8_t save = NVME_SETFEAT_SAVE(dw10); |
6744 | 0 | uint16_t status; |
6745 | 0 | int i; |
6746 | |
|
6747 | 0 | trace_pci_nvme_setfeat(nvme_cid(req), nsid, fid, save, dw11); |
6748 | |
|
6749 | 0 | if (save && !(nvme_feature_cap[fid] & NVME_FEAT_CAP_SAVE)) { |
6750 | 0 | return NVME_FID_NOT_SAVEABLE | NVME_DNR; |
6751 | 0 | } |
6752 | | |
6753 | 0 | if (!nvme_feature_support[fid]) { |
6754 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6755 | 0 | } |
6756 | | |
6757 | 0 | if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) { |
6758 | 0 | if (nsid != NVME_NSID_BROADCAST) { |
6759 | 0 | if (!nvme_nsid_valid(n, nsid)) { |
6760 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
6761 | 0 | } |
6762 | | |
6763 | 0 | ns = nvme_ns(n, nsid); |
6764 | 0 | if (unlikely(!ns)) { |
6765 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6766 | 0 | } |
6767 | 0 | } |
6768 | 0 | } else if (nsid && nsid != NVME_NSID_BROADCAST) { |
6769 | 0 | if (!nvme_nsid_valid(n, nsid)) { |
6770 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
6771 | 0 | } |
6772 | | |
6773 | 0 | return NVME_FEAT_NOT_NS_SPEC | NVME_DNR; |
6774 | 0 | } |
6775 | | |
6776 | 0 | if (!(nvme_feature_cap[fid] & NVME_FEAT_CAP_CHANGE)) { |
6777 | 0 | return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR; |
6778 | 0 | } |
6779 | | |
6780 | 0 | switch (fid) { |
6781 | 0 | case NVME_TEMPERATURE_THRESHOLD: |
6782 | 0 | if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) { |
6783 | 0 | break; |
6784 | 0 | } |
6785 | | |
6786 | 0 | switch (NVME_TEMP_THSEL(dw11)) { |
6787 | 0 | case NVME_TEMP_THSEL_OVER: |
6788 | 0 | n->features.temp_thresh_hi = NVME_TEMP_TMPTH(dw11); |
6789 | 0 | break; |
6790 | 0 | case NVME_TEMP_THSEL_UNDER: |
6791 | 0 | n->features.temp_thresh_low = NVME_TEMP_TMPTH(dw11); |
6792 | 0 | break; |
6793 | 0 | default: |
6794 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6795 | 0 | } |
6796 | | |
6797 | 0 | if ((n->temperature >= n->features.temp_thresh_hi) || |
6798 | 0 | (n->temperature <= n->features.temp_thresh_low)) { |
6799 | 0 | nvme_smart_event(n, NVME_SMART_TEMPERATURE); |
6800 | 0 | } |
6801 | |
|
6802 | 0 | break; |
6803 | 0 | case NVME_ERROR_RECOVERY: |
6804 | 0 | if (nsid == NVME_NSID_BROADCAST) { |
6805 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
6806 | 0 | ns = nvme_ns(n, i); |
6807 | |
|
6808 | 0 | if (!ns) { |
6809 | 0 | continue; |
6810 | 0 | } |
6811 | | |
6812 | 0 | if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) { |
6813 | 0 | ns->features.err_rec = dw11; |
6814 | 0 | } |
6815 | 0 | } |
6816 | |
|
6817 | 0 | break; |
6818 | 0 | } |
6819 | | |
6820 | 0 | assert(ns); |
6821 | 0 | if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) { |
6822 | 0 | ns->features.err_rec = dw11; |
6823 | 0 | } |
6824 | 0 | break; |
6825 | 0 | case NVME_VOLATILE_WRITE_CACHE: |
6826 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
6827 | 0 | ns = nvme_ns(n, i); |
6828 | 0 | if (!ns) { |
6829 | 0 | continue; |
6830 | 0 | } |
6831 | | |
6832 | 0 | if (!(dw11 & 0x1) && blk_enable_write_cache(ns->blkconf.blk)) { |
6833 | 0 | blk_flush(ns->blkconf.blk); |
6834 | 0 | } |
6835 | |
|
6836 | 0 | blk_set_enable_write_cache(ns->blkconf.blk, dw11 & 1); |
6837 | 0 | } |
6838 | |
|
6839 | 0 | break; |
6840 | | |
6841 | 0 | case NVME_NUMBER_OF_QUEUES: |
6842 | 0 | if (n->qs_created) { |
6843 | 0 | return NVME_CMD_SEQ_ERROR | NVME_DNR; |
6844 | 0 | } |
6845 | | |
6846 | | /* |
6847 | | * NVMe v1.3, Section 5.21.1.7: FFFFh is not an allowed value for NCQR |
6848 | | * and NSQR. |
6849 | | */ |
6850 | 0 | if ((dw11 & 0xffff) == 0xffff || ((dw11 >> 16) & 0xffff) == 0xffff) { |
6851 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6852 | 0 | } |
6853 | | |
6854 | 0 | trace_pci_nvme_setfeat_numq((dw11 & 0xffff) + 1, |
6855 | 0 | ((dw11 >> 16) & 0xffff) + 1, |
6856 | 0 | n->conf_ioqpairs, |
6857 | 0 | n->conf_ioqpairs); |
6858 | 0 | req->cqe.result = cpu_to_le32((n->conf_ioqpairs - 1) | |
6859 | 0 | ((n->conf_ioqpairs - 1) << 16)); |
6860 | 0 | break; |
6861 | 0 | case NVME_ASYNCHRONOUS_EVENT_CONF: |
6862 | 0 | n->features.async_config = dw11; |
6863 | 0 | break; |
6864 | 0 | case NVME_TIMESTAMP: |
6865 | 0 | return nvme_set_feature_timestamp(n, req); |
6866 | 0 | case NVME_HOST_BEHAVIOR_SUPPORT: |
6867 | 0 | status = nvme_h2c(n, (uint8_t *)&n->features.hbs, |
6868 | 0 | sizeof(n->features.hbs), req); |
6869 | 0 | if (status) { |
6870 | 0 | return status; |
6871 | 0 | } |
6872 | | |
6873 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
6874 | 0 | ns = nvme_ns(n, i); |
6875 | |
|
6876 | 0 | if (!ns) { |
6877 | 0 | continue; |
6878 | 0 | } |
6879 | | |
6880 | 0 | ns->id_ns.nlbaf = ns->nlbaf - 1; |
6881 | 0 | if (!n->features.hbs.lbafee) { |
6882 | 0 | ns->id_ns.nlbaf = MIN(ns->id_ns.nlbaf, 15); |
6883 | 0 | } |
6884 | 0 | } |
6885 | |
|
6886 | 0 | return status; |
6887 | 0 | case NVME_COMMAND_SET_PROFILE: |
6888 | 0 | if (dw11 & 0x1ff) { |
6889 | 0 | trace_pci_nvme_err_invalid_iocsci(dw11 & 0x1ff); |
6890 | 0 | return NVME_IOCS_COMBINATION_REJECTED | NVME_DNR; |
6891 | 0 | } |
6892 | 0 | break; |
6893 | 0 | case NVME_FDP_MODE: |
6894 | | /* spec: abort with cmd seq err if there's one or more NS' in endgrp */ |
6895 | 0 | return NVME_CMD_SEQ_ERROR | NVME_DNR; |
6896 | 0 | case NVME_FDP_EVENTS: |
6897 | 0 | return nvme_set_feature_fdp_events(n, ns, req); |
6898 | 0 | case NVME_WRITE_ATOMICITY: |
6899 | 0 | return nvme_set_feature_write_atomicity(n, req); |
6900 | 0 | default: |
6901 | 0 | return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR; |
6902 | 0 | } |
6903 | 0 | return NVME_SUCCESS; |
6904 | 0 | } |
6905 | | |
6906 | | static uint16_t nvme_aer(NvmeCtrl *n, NvmeRequest *req) |
6907 | 0 | { |
6908 | 0 | trace_pci_nvme_aer(nvme_cid(req)); |
6909 | |
|
6910 | 0 | if (n->outstanding_aers > n->params.aerl) { |
6911 | 0 | trace_pci_nvme_aer_aerl_exceeded(); |
6912 | 0 | return NVME_AER_LIMIT_EXCEEDED; |
6913 | 0 | } |
6914 | | |
6915 | 0 | n->aer_reqs[n->outstanding_aers] = req; |
6916 | 0 | n->outstanding_aers++; |
6917 | |
|
6918 | 0 | if (!QTAILQ_EMPTY(&n->aer_queue)) { |
6919 | 0 | nvme_process_aers(n); |
6920 | 0 | } |
6921 | |
|
6922 | 0 | return NVME_NO_COMPLETE; |
6923 | 0 | } |
6924 | | |
6925 | | static void nvme_update_dsm_limits(NvmeCtrl *n, NvmeNamespace *ns) |
6926 | 0 | { |
6927 | 0 | if (ns) { |
6928 | 0 | n->dmrsl = |
6929 | 0 | MIN_NON_ZERO(n->dmrsl, BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1)); |
6930 | |
|
6931 | 0 | return; |
6932 | 0 | } |
6933 | | |
6934 | 0 | for (uint32_t nsid = 1; nsid <= NVME_MAX_NAMESPACES; nsid++) { |
6935 | 0 | ns = nvme_ns(n, nsid); |
6936 | 0 | if (!ns) { |
6937 | 0 | continue; |
6938 | 0 | } |
6939 | | |
6940 | 0 | n->dmrsl = |
6941 | 0 | MIN_NON_ZERO(n->dmrsl, BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1)); |
6942 | 0 | } |
6943 | 0 | } |
6944 | | |
6945 | | static bool nvme_csi_supported(NvmeCtrl *n, uint8_t csi) |
6946 | 0 | { |
6947 | 0 | uint32_t cc; |
6948 | |
|
6949 | 0 | switch (csi) { |
6950 | 0 | case NVME_CSI_NVM: |
6951 | 0 | return true; |
6952 | | |
6953 | 0 | case NVME_CSI_ZONED: |
6954 | 0 | cc = ldl_le_p(&n->bar.cc); |
6955 | |
|
6956 | 0 | return NVME_CC_CSS(cc) == NVME_CC_CSS_ALL; |
6957 | 0 | } |
6958 | | |
6959 | 0 | g_assert_not_reached(); |
6960 | 0 | } |
6961 | | |
6962 | | static void nvme_detach_ns(NvmeCtrl *n, NvmeNamespace *ns) |
6963 | 0 | { |
6964 | 0 | assert(ns->attached > 0); |
6965 | |
|
6966 | 0 | n->namespaces[ns->params.nsid] = NULL; |
6967 | 0 | ns->attached--; |
6968 | 0 | } |
6969 | | |
6970 | | static uint16_t nvme_ns_attachment(NvmeCtrl *n, NvmeRequest *req) |
6971 | 0 | { |
6972 | 0 | NvmeNamespace *ns; |
6973 | 0 | NvmeCtrl *ctrl; |
6974 | 0 | uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {}; |
6975 | 0 | uint32_t nsid = le32_to_cpu(req->cmd.nsid); |
6976 | 0 | uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); |
6977 | 0 | uint8_t sel = dw10 & 0xf; |
6978 | 0 | uint16_t *nr_ids = &list[0]; |
6979 | 0 | uint16_t *ids = &list[1]; |
6980 | 0 | uint16_t ret; |
6981 | 0 | int i; |
6982 | |
|
6983 | 0 | trace_pci_nvme_ns_attachment(nvme_cid(req), dw10 & 0xf); |
6984 | |
|
6985 | 0 | if (!nvme_nsid_valid(n, nsid)) { |
6986 | 0 | return NVME_INVALID_NSID | NVME_DNR; |
6987 | 0 | } |
6988 | | |
6989 | 0 | ns = nvme_subsys_ns(n->subsys, nsid); |
6990 | 0 | if (!ns) { |
6991 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
6992 | 0 | } |
6993 | | |
6994 | 0 | ret = nvme_h2c(n, (uint8_t *)list, 4096, req); |
6995 | 0 | if (ret) { |
6996 | 0 | return ret; |
6997 | 0 | } |
6998 | | |
6999 | 0 | if (!*nr_ids) { |
7000 | 0 | return NVME_NS_CTRL_LIST_INVALID | NVME_DNR; |
7001 | 0 | } |
7002 | | |
7003 | 0 | *nr_ids = MIN(*nr_ids, NVME_CONTROLLER_LIST_SIZE - 1); |
7004 | 0 | for (i = 0; i < *nr_ids; i++) { |
7005 | 0 | ctrl = nvme_subsys_ctrl(n->subsys, ids[i]); |
7006 | 0 | if (!ctrl) { |
7007 | 0 | return NVME_NS_CTRL_LIST_INVALID | NVME_DNR; |
7008 | 0 | } |
7009 | | |
7010 | 0 | switch (sel) { |
7011 | 0 | case NVME_NS_ATTACHMENT_ATTACH: |
7012 | 0 | if (nvme_ns(ctrl, nsid)) { |
7013 | 0 | return NVME_NS_ALREADY_ATTACHED | NVME_DNR; |
7014 | 0 | } |
7015 | | |
7016 | 0 | if (ns->attached && !ns->params.shared) { |
7017 | 0 | return NVME_NS_PRIVATE | NVME_DNR; |
7018 | 0 | } |
7019 | | |
7020 | 0 | if (!nvme_csi_supported(ctrl, ns->csi)) { |
7021 | 0 | return NVME_IOCS_NOT_SUPPORTED | NVME_DNR; |
7022 | 0 | } |
7023 | | |
7024 | 0 | nvme_attach_ns(ctrl, ns); |
7025 | 0 | nvme_update_dsm_limits(ctrl, ns); |
7026 | |
|
7027 | 0 | break; |
7028 | | |
7029 | 0 | case NVME_NS_ATTACHMENT_DETACH: |
7030 | 0 | if (!nvme_ns(ctrl, nsid)) { |
7031 | 0 | return NVME_NS_NOT_ATTACHED | NVME_DNR; |
7032 | 0 | } |
7033 | | |
7034 | 0 | nvme_detach_ns(ctrl, ns); |
7035 | 0 | nvme_update_dsm_limits(ctrl, NULL); |
7036 | |
|
7037 | 0 | break; |
7038 | | |
7039 | 0 | default: |
7040 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7041 | 0 | } |
7042 | | |
7043 | | /* |
7044 | | * Add namespace id to the changed namespace id list for event clearing |
7045 | | * via Get Log Page command. |
7046 | | */ |
7047 | 0 | if (!test_and_set_bit(nsid, ctrl->changed_nsids)) { |
7048 | 0 | nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE, |
7049 | 0 | NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED, |
7050 | 0 | NVME_LOG_CHANGED_NSLIST); |
7051 | 0 | } |
7052 | 0 | } |
7053 | | |
7054 | 0 | return NVME_SUCCESS; |
7055 | 0 | } |
7056 | | |
7057 | | void nvme_ctrl_notify_ns_resize(NvmeCtrl *ctrl, NvmeNamespace *ns) |
7058 | 0 | { |
7059 | 0 | if (!test_and_set_bit(ns->params.nsid, ctrl->changed_nsids)) { |
7060 | 0 | nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE, |
7061 | 0 | NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED, |
7062 | 0 | NVME_LOG_CHANGED_NSLIST); |
7063 | 0 | } |
7064 | 0 | } |
7065 | | |
7066 | | typedef struct NvmeFormatAIOCB { |
7067 | | BlockAIOCB common; |
7068 | | BlockAIOCB *aiocb; |
7069 | | NvmeRequest *req; |
7070 | | int ret; |
7071 | | |
7072 | | NvmeNamespace *ns; |
7073 | | uint32_t nsid; |
7074 | | bool broadcast; |
7075 | | int64_t offset; |
7076 | | |
7077 | | uint8_t lbaf; |
7078 | | uint8_t mset; |
7079 | | uint8_t pi; |
7080 | | uint8_t pil; |
7081 | | } NvmeFormatAIOCB; |
7082 | | |
7083 | | static void nvme_format_cancel(BlockAIOCB *aiocb) |
7084 | 0 | { |
7085 | 0 | NvmeFormatAIOCB *iocb = container_of(aiocb, NvmeFormatAIOCB, common); |
7086 | |
|
7087 | 0 | iocb->ret = -ECANCELED; |
7088 | |
|
7089 | 0 | if (iocb->aiocb) { |
7090 | 0 | blk_aio_cancel_async(iocb->aiocb); |
7091 | 0 | iocb->aiocb = NULL; |
7092 | 0 | } |
7093 | 0 | } |
7094 | | |
7095 | | static const AIOCBInfo nvme_format_aiocb_info = { |
7096 | | .aiocb_size = sizeof(NvmeFormatAIOCB), |
7097 | | .cancel_async = nvme_format_cancel, |
7098 | | }; |
7099 | | |
7100 | | static void nvme_format_set(NvmeNamespace *ns, uint8_t lbaf, uint8_t mset, |
7101 | | uint8_t pi, uint8_t pil) |
7102 | 0 | { |
7103 | 0 | uint8_t lbafl = lbaf & 0xf; |
7104 | 0 | uint8_t lbafu = lbaf >> 4; |
7105 | |
|
7106 | 0 | trace_pci_nvme_format_set(ns->params.nsid, lbaf, mset, pi, pil); |
7107 | |
|
7108 | 0 | ns->id_ns.dps = (pil << 3) | pi; |
7109 | 0 | ns->id_ns.flbas = (lbafu << 5) | (mset << 4) | lbafl; |
7110 | |
|
7111 | 0 | nvme_ns_init_format(ns); |
7112 | 0 | } |
7113 | | |
7114 | | static void nvme_do_format(NvmeFormatAIOCB *iocb); |
7115 | | |
7116 | | static void nvme_format_ns_cb(void *opaque, int ret) |
7117 | 0 | { |
7118 | 0 | NvmeFormatAIOCB *iocb = opaque; |
7119 | 0 | NvmeNamespace *ns = iocb->ns; |
7120 | 0 | int bytes; |
7121 | |
|
7122 | 0 | if (iocb->ret < 0) { |
7123 | 0 | goto done; |
7124 | 0 | } else if (ret < 0) { |
7125 | 0 | iocb->ret = ret; |
7126 | 0 | goto done; |
7127 | 0 | } |
7128 | | |
7129 | 0 | assert(ns); |
7130 | |
|
7131 | 0 | if (iocb->offset < ns->size) { |
7132 | 0 | bytes = MIN(BDRV_REQUEST_MAX_BYTES, ns->size - iocb->offset); |
7133 | |
|
7134 | 0 | iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, iocb->offset, |
7135 | 0 | bytes, BDRV_REQ_MAY_UNMAP, |
7136 | 0 | nvme_format_ns_cb, iocb); |
7137 | |
|
7138 | 0 | iocb->offset += bytes; |
7139 | 0 | return; |
7140 | 0 | } |
7141 | | |
7142 | 0 | nvme_format_set(ns, iocb->lbaf, iocb->mset, iocb->pi, iocb->pil); |
7143 | 0 | ns->status = 0x0; |
7144 | 0 | iocb->ns = NULL; |
7145 | 0 | iocb->offset = 0; |
7146 | |
|
7147 | 0 | done: |
7148 | 0 | nvme_do_format(iocb); |
7149 | 0 | } |
7150 | | |
7151 | | static uint16_t nvme_format_check(NvmeNamespace *ns, uint8_t lbaf, uint8_t pi) |
7152 | 0 | { |
7153 | 0 | if (ns->params.zoned) { |
7154 | 0 | return NVME_INVALID_FORMAT | NVME_DNR; |
7155 | 0 | } |
7156 | | |
7157 | 0 | if (lbaf > ns->id_ns.nlbaf) { |
7158 | 0 | return NVME_INVALID_FORMAT | NVME_DNR; |
7159 | 0 | } |
7160 | | |
7161 | 0 | if (pi && (ns->id_ns.lbaf[lbaf].ms < nvme_pi_tuple_size(ns))) { |
7162 | 0 | return NVME_INVALID_FORMAT | NVME_DNR; |
7163 | 0 | } |
7164 | | |
7165 | 0 | if (pi && pi > NVME_ID_NS_DPS_TYPE_3) { |
7166 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7167 | 0 | } |
7168 | | |
7169 | 0 | return NVME_SUCCESS; |
7170 | 0 | } |
7171 | | |
7172 | | static void nvme_do_format(NvmeFormatAIOCB *iocb) |
7173 | 0 | { |
7174 | 0 | NvmeRequest *req = iocb->req; |
7175 | 0 | NvmeCtrl *n = nvme_ctrl(req); |
7176 | 0 | uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); |
7177 | 0 | uint8_t lbaf = dw10 & 0xf; |
7178 | 0 | uint8_t pi = (dw10 >> 5) & 0x7; |
7179 | 0 | uint16_t status; |
7180 | 0 | int i; |
7181 | |
|
7182 | 0 | if (iocb->ret < 0) { |
7183 | 0 | goto done; |
7184 | 0 | } |
7185 | | |
7186 | 0 | if (iocb->broadcast) { |
7187 | 0 | for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) { |
7188 | 0 | iocb->ns = nvme_ns(n, i); |
7189 | 0 | if (iocb->ns) { |
7190 | 0 | iocb->nsid = i; |
7191 | 0 | break; |
7192 | 0 | } |
7193 | 0 | } |
7194 | 0 | } |
7195 | |
|
7196 | 0 | if (!iocb->ns) { |
7197 | 0 | goto done; |
7198 | 0 | } |
7199 | | |
7200 | 0 | status = nvme_format_check(iocb->ns, lbaf, pi); |
7201 | 0 | if (status) { |
7202 | 0 | req->status = status; |
7203 | 0 | goto done; |
7204 | 0 | } |
7205 | | |
7206 | 0 | iocb->ns->status = NVME_FORMAT_IN_PROGRESS; |
7207 | 0 | nvme_format_ns_cb(iocb, 0); |
7208 | 0 | return; |
7209 | | |
7210 | 0 | done: |
7211 | 0 | iocb->common.cb(iocb->common.opaque, iocb->ret); |
7212 | 0 | qemu_aio_unref(iocb); |
7213 | 0 | } |
7214 | | |
7215 | | static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req) |
7216 | 0 | { |
7217 | 0 | NvmeFormatAIOCB *iocb; |
7218 | 0 | uint32_t nsid = le32_to_cpu(req->cmd.nsid); |
7219 | 0 | uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); |
7220 | 0 | uint8_t lbaf = dw10 & 0xf; |
7221 | 0 | uint8_t mset = (dw10 >> 4) & 0x1; |
7222 | 0 | uint8_t pi = (dw10 >> 5) & 0x7; |
7223 | 0 | uint8_t pil = (dw10 >> 8) & 0x1; |
7224 | 0 | uint8_t lbafu = (dw10 >> 12) & 0x3; |
7225 | 0 | uint16_t status; |
7226 | |
|
7227 | 0 | iocb = qemu_aio_get(&nvme_format_aiocb_info, NULL, nvme_misc_cb, req); |
7228 | |
|
7229 | 0 | iocb->req = req; |
7230 | 0 | iocb->ret = 0; |
7231 | 0 | iocb->ns = NULL; |
7232 | 0 | iocb->nsid = 0; |
7233 | 0 | iocb->lbaf = lbaf; |
7234 | 0 | iocb->mset = mset; |
7235 | 0 | iocb->pi = pi; |
7236 | 0 | iocb->pil = pil; |
7237 | 0 | iocb->broadcast = (nsid == NVME_NSID_BROADCAST); |
7238 | 0 | iocb->offset = 0; |
7239 | |
|
7240 | 0 | if (n->features.hbs.lbafee) { |
7241 | 0 | iocb->lbaf |= lbafu << 4; |
7242 | 0 | } |
7243 | |
|
7244 | 0 | if (!iocb->broadcast) { |
7245 | 0 | if (!nvme_nsid_valid(n, nsid)) { |
7246 | 0 | status = NVME_INVALID_NSID | NVME_DNR; |
7247 | 0 | goto out; |
7248 | 0 | } |
7249 | | |
7250 | 0 | iocb->ns = nvme_ns(n, nsid); |
7251 | 0 | if (!iocb->ns) { |
7252 | 0 | status = NVME_INVALID_FIELD | NVME_DNR; |
7253 | 0 | goto out; |
7254 | 0 | } |
7255 | 0 | } |
7256 | | |
7257 | 0 | req->aiocb = &iocb->common; |
7258 | 0 | nvme_do_format(iocb); |
7259 | |
|
7260 | 0 | return NVME_NO_COMPLETE; |
7261 | | |
7262 | 0 | out: |
7263 | 0 | qemu_aio_unref(iocb); |
7264 | |
|
7265 | 0 | return status; |
7266 | 0 | } |
7267 | | |
7268 | | static void nvme_get_virt_res_num(NvmeCtrl *n, uint8_t rt, int *num_total, |
7269 | | int *num_prim, int *num_sec) |
7270 | 0 | { |
7271 | 0 | *num_total = le32_to_cpu(rt ? |
7272 | 0 | n->pri_ctrl_cap.vifrt : n->pri_ctrl_cap.vqfrt); |
7273 | 0 | *num_prim = le16_to_cpu(rt ? |
7274 | 0 | n->pri_ctrl_cap.virfap : n->pri_ctrl_cap.vqrfap); |
7275 | 0 | *num_sec = le16_to_cpu(rt ? n->pri_ctrl_cap.virfa : n->pri_ctrl_cap.vqrfa); |
7276 | 0 | } |
7277 | | |
7278 | | static uint16_t nvme_assign_virt_res_to_prim(NvmeCtrl *n, NvmeRequest *req, |
7279 | | uint16_t cntlid, uint8_t rt, |
7280 | | int nr) |
7281 | 0 | { |
7282 | 0 | int num_total, num_prim, num_sec; |
7283 | |
|
7284 | 0 | if (cntlid != n->cntlid) { |
7285 | 0 | return NVME_INVALID_CTRL_ID | NVME_DNR; |
7286 | 0 | } |
7287 | | |
7288 | 0 | nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec); |
7289 | |
|
7290 | 0 | if (nr > num_total) { |
7291 | 0 | return NVME_INVALID_NUM_RESOURCES | NVME_DNR; |
7292 | 0 | } |
7293 | | |
7294 | 0 | if (nr > num_total - num_sec) { |
7295 | 0 | return NVME_INVALID_RESOURCE_ID | NVME_DNR; |
7296 | 0 | } |
7297 | | |
7298 | 0 | if (rt) { |
7299 | 0 | n->next_pri_ctrl_cap.virfap = cpu_to_le16(nr); |
7300 | 0 | } else { |
7301 | 0 | n->next_pri_ctrl_cap.vqrfap = cpu_to_le16(nr); |
7302 | 0 | } |
7303 | |
|
7304 | 0 | req->cqe.result = cpu_to_le32(nr); |
7305 | 0 | return req->status; |
7306 | 0 | } |
7307 | | |
7308 | | static void nvme_update_virt_res(NvmeCtrl *n, NvmeSecCtrlEntry *sctrl, |
7309 | | uint8_t rt, int nr) |
7310 | 0 | { |
7311 | 0 | int prev_nr, prev_total; |
7312 | |
|
7313 | 0 | if (rt) { |
7314 | 0 | prev_nr = le16_to_cpu(sctrl->nvi); |
7315 | 0 | prev_total = le32_to_cpu(n->pri_ctrl_cap.virfa); |
7316 | 0 | sctrl->nvi = cpu_to_le16(nr); |
7317 | 0 | n->pri_ctrl_cap.virfa = cpu_to_le32(prev_total + nr - prev_nr); |
7318 | 0 | } else { |
7319 | 0 | prev_nr = le16_to_cpu(sctrl->nvq); |
7320 | 0 | prev_total = le32_to_cpu(n->pri_ctrl_cap.vqrfa); |
7321 | 0 | sctrl->nvq = cpu_to_le16(nr); |
7322 | 0 | n->pri_ctrl_cap.vqrfa = cpu_to_le32(prev_total + nr - prev_nr); |
7323 | 0 | } |
7324 | 0 | } |
7325 | | |
7326 | | static uint16_t nvme_assign_virt_res_to_sec(NvmeCtrl *n, NvmeRequest *req, |
7327 | | uint16_t cntlid, uint8_t rt, int nr) |
7328 | 0 | { |
7329 | 0 | int num_total, num_prim, num_sec, num_free, diff, limit; |
7330 | 0 | NvmeSecCtrlEntry *sctrl; |
7331 | |
|
7332 | 0 | sctrl = nvme_sctrl_for_cntlid(n, cntlid); |
7333 | 0 | if (!sctrl) { |
7334 | 0 | return NVME_INVALID_CTRL_ID | NVME_DNR; |
7335 | 0 | } |
7336 | | |
7337 | 0 | if (sctrl->scs) { |
7338 | 0 | return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR; |
7339 | 0 | } |
7340 | | |
7341 | 0 | limit = le16_to_cpu(rt ? n->pri_ctrl_cap.vifrsm : n->pri_ctrl_cap.vqfrsm); |
7342 | 0 | if (nr > limit) { |
7343 | 0 | return NVME_INVALID_NUM_RESOURCES | NVME_DNR; |
7344 | 0 | } |
7345 | | |
7346 | 0 | nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec); |
7347 | 0 | num_free = num_total - num_prim - num_sec; |
7348 | 0 | diff = nr - le16_to_cpu(rt ? sctrl->nvi : sctrl->nvq); |
7349 | |
|
7350 | 0 | if (diff > num_free) { |
7351 | 0 | return NVME_INVALID_RESOURCE_ID | NVME_DNR; |
7352 | 0 | } |
7353 | | |
7354 | 0 | nvme_update_virt_res(n, sctrl, rt, nr); |
7355 | 0 | req->cqe.result = cpu_to_le32(nr); |
7356 | |
|
7357 | 0 | return req->status; |
7358 | 0 | } |
7359 | | |
7360 | | static uint16_t nvme_virt_set_state(NvmeCtrl *n, uint16_t cntlid, bool online) |
7361 | 0 | { |
7362 | 0 | PCIDevice *pci = PCI_DEVICE(n); |
7363 | 0 | NvmeCtrl *sn = NULL; |
7364 | 0 | NvmeSecCtrlEntry *sctrl; |
7365 | 0 | int vf_index; |
7366 | |
|
7367 | 0 | sctrl = nvme_sctrl_for_cntlid(n, cntlid); |
7368 | 0 | if (!sctrl) { |
7369 | 0 | return NVME_INVALID_CTRL_ID | NVME_DNR; |
7370 | 0 | } |
7371 | | |
7372 | 0 | if (!pci_is_vf(pci)) { |
7373 | 0 | vf_index = le16_to_cpu(sctrl->vfn) - 1; |
7374 | 0 | sn = NVME(pcie_sriov_get_vf_at_index(pci, vf_index)); |
7375 | 0 | } |
7376 | |
|
7377 | 0 | if (online) { |
7378 | 0 | if (!sctrl->nvi || (le16_to_cpu(sctrl->nvq) < 2) || !sn) { |
7379 | 0 | return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR; |
7380 | 0 | } |
7381 | | |
7382 | 0 | if (!sctrl->scs) { |
7383 | 0 | sctrl->scs = 0x1; |
7384 | 0 | nvme_ctrl_reset(sn, NVME_RESET_FUNCTION); |
7385 | 0 | } |
7386 | 0 | } else { |
7387 | 0 | nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_INTERRUPT, 0); |
7388 | 0 | nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_QUEUE, 0); |
7389 | |
|
7390 | 0 | if (sctrl->scs) { |
7391 | 0 | sctrl->scs = 0x0; |
7392 | 0 | if (sn) { |
7393 | 0 | nvme_ctrl_reset(sn, NVME_RESET_FUNCTION); |
7394 | 0 | } |
7395 | 0 | } |
7396 | 0 | } |
7397 | | |
7398 | 0 | return NVME_SUCCESS; |
7399 | 0 | } |
7400 | | |
7401 | | static uint16_t nvme_virt_mngmt(NvmeCtrl *n, NvmeRequest *req) |
7402 | 0 | { |
7403 | 0 | uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); |
7404 | 0 | uint32_t dw11 = le32_to_cpu(req->cmd.cdw11); |
7405 | 0 | uint8_t act = dw10 & 0xf; |
7406 | 0 | uint8_t rt = (dw10 >> 8) & 0x7; |
7407 | 0 | uint16_t cntlid = (dw10 >> 16) & 0xffff; |
7408 | 0 | int nr = dw11 & 0xffff; |
7409 | |
|
7410 | 0 | trace_pci_nvme_virt_mngmt(nvme_cid(req), act, cntlid, rt ? "VI" : "VQ", nr); |
7411 | |
|
7412 | 0 | if (rt != NVME_VIRT_RES_QUEUE && rt != NVME_VIRT_RES_INTERRUPT) { |
7413 | 0 | return NVME_INVALID_RESOURCE_ID | NVME_DNR; |
7414 | 0 | } |
7415 | | |
7416 | 0 | switch (act) { |
7417 | 0 | case NVME_VIRT_MNGMT_ACTION_SEC_ASSIGN: |
7418 | 0 | return nvme_assign_virt_res_to_sec(n, req, cntlid, rt, nr); |
7419 | 0 | case NVME_VIRT_MNGMT_ACTION_PRM_ALLOC: |
7420 | 0 | return nvme_assign_virt_res_to_prim(n, req, cntlid, rt, nr); |
7421 | 0 | case NVME_VIRT_MNGMT_ACTION_SEC_ONLINE: |
7422 | 0 | return nvme_virt_set_state(n, cntlid, true); |
7423 | 0 | case NVME_VIRT_MNGMT_ACTION_SEC_OFFLINE: |
7424 | 0 | return nvme_virt_set_state(n, cntlid, false); |
7425 | 0 | default: |
7426 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7427 | 0 | } |
7428 | 0 | } |
7429 | | |
7430 | | static uint16_t nvme_dbbuf_config(NvmeCtrl *n, const NvmeRequest *req) |
7431 | 0 | { |
7432 | 0 | PCIDevice *pci = PCI_DEVICE(n); |
7433 | 0 | uint64_t dbs_addr = le64_to_cpu(req->cmd.dptr.prp1); |
7434 | 0 | uint64_t eis_addr = le64_to_cpu(req->cmd.dptr.prp2); |
7435 | 0 | int i; |
7436 | | |
7437 | | /* Address should be page aligned */ |
7438 | 0 | if (dbs_addr & (n->page_size - 1) || eis_addr & (n->page_size - 1)) { |
7439 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7440 | 0 | } |
7441 | | |
7442 | | /* Save shadow buffer base addr for use during queue creation */ |
7443 | 0 | n->dbbuf_dbs = dbs_addr; |
7444 | 0 | n->dbbuf_eis = eis_addr; |
7445 | 0 | n->dbbuf_enabled = true; |
7446 | |
|
7447 | 0 | for (i = 0; i < n->num_queues; i++) { |
7448 | 0 | NvmeSQueue *sq = n->sq[i]; |
7449 | 0 | NvmeCQueue *cq = n->cq[i]; |
7450 | |
|
7451 | 0 | if (sq) { |
7452 | | /* |
7453 | | * CAP.DSTRD is 0, so offset of ith sq db_addr is (i<<3) |
7454 | | * nvme_process_db() uses this hard-coded way to calculate |
7455 | | * doorbell offsets. Be consistent with that here. |
7456 | | */ |
7457 | 0 | sq->db_addr = dbs_addr + (i << 3); |
7458 | 0 | sq->ei_addr = eis_addr + (i << 3); |
7459 | 0 | stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED); |
7460 | |
|
7461 | 0 | if (n->params.ioeventfd && sq->sqid != 0) { |
7462 | 0 | if (!nvme_init_sq_ioeventfd(sq)) { |
7463 | 0 | sq->ioeventfd_enabled = true; |
7464 | 0 | } |
7465 | 0 | } |
7466 | 0 | } |
7467 | |
|
7468 | 0 | if (cq) { |
7469 | | /* CAP.DSTRD is 0, so offset of ith cq db_addr is (i<<3)+(1<<2) */ |
7470 | 0 | cq->db_addr = dbs_addr + (i << 3) + (1 << 2); |
7471 | 0 | cq->ei_addr = eis_addr + (i << 3) + (1 << 2); |
7472 | 0 | stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED); |
7473 | |
|
7474 | 0 | if (n->params.ioeventfd && cq->cqid != 0) { |
7475 | 0 | if (!nvme_init_cq_ioeventfd(cq)) { |
7476 | 0 | cq->ioeventfd_enabled = true; |
7477 | 0 | } |
7478 | 0 | } |
7479 | 0 | } |
7480 | 0 | } |
7481 | |
|
7482 | 0 | trace_pci_nvme_dbbuf_config(dbs_addr, eis_addr); |
7483 | |
|
7484 | 0 | return NVME_SUCCESS; |
7485 | 0 | } |
7486 | | |
7487 | | static uint16_t nvme_sec_prot_spdm_send(NvmeCtrl *n, NvmeRequest *req) |
7488 | 0 | { |
7489 | 0 | StorageSpdmTransportHeader hdr = {0}; |
7490 | 0 | g_autofree uint8_t *sec_buf = NULL; |
7491 | 0 | uint32_t transfer_len = le32_to_cpu(req->cmd.cdw11); |
7492 | 0 | uint32_t transport_transfer_len = transfer_len; |
7493 | 0 | uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); |
7494 | 0 | uint32_t recvd; |
7495 | 0 | uint16_t nvme_cmd_status, ret; |
7496 | 0 | uint8_t secp = extract32(dw10, 24, 8); |
7497 | 0 | uint16_t spsp = extract32(dw10, 8, 16); |
7498 | 0 | bool spdm_res; |
7499 | |
|
7500 | 0 | if (transport_transfer_len > UINT32_MAX - sizeof(hdr)) { |
7501 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7502 | 0 | } |
7503 | | |
7504 | 0 | transport_transfer_len += sizeof(hdr); |
7505 | 0 | if (transport_transfer_len > SPDM_SOCKET_MAX_MESSAGE_BUFFER_SIZE) { |
7506 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7507 | 0 | } |
7508 | | |
7509 | 0 | ret = nvme_check_mdts(n, transport_transfer_len); |
7510 | 0 | if (ret != NVME_SUCCESS) { |
7511 | 0 | return ret; |
7512 | 0 | } |
7513 | | |
7514 | | /* Generate the NVMe transport header */ |
7515 | 0 | hdr.security_protocol = secp; |
7516 | 0 | hdr.security_protocol_specific = cpu_to_le16(spsp); |
7517 | 0 | hdr.length = cpu_to_le32(transfer_len); |
7518 | |
|
7519 | 0 | sec_buf = g_try_malloc0(transport_transfer_len); |
7520 | 0 | if (!sec_buf) { |
7521 | 0 | return NVME_INTERNAL_DEV_ERROR; |
7522 | 0 | } |
7523 | | |
7524 | | /* Attach the transport header */ |
7525 | 0 | memcpy(sec_buf, &hdr, sizeof(hdr)); |
7526 | 0 | ret = nvme_h2c(n, sec_buf + sizeof(hdr), transfer_len, req); |
7527 | 0 | if (ret) { |
7528 | 0 | return ret; |
7529 | 0 | } |
7530 | | |
7531 | 0 | spdm_res = spdm_socket_send(n->spdm_socket, SPDM_SOCKET_STORAGE_CMD_IF_SEND, |
7532 | 0 | SPDM_TRANSPORT_TYPE_NVME, sec_buf, |
7533 | 0 | transport_transfer_len); |
7534 | 0 | if (!spdm_res) { |
7535 | 0 | return NVME_DATA_TRAS_ERROR | NVME_DNR; |
7536 | 0 | } |
7537 | | |
7538 | | /* The responder shall ack with message status */ |
7539 | 0 | recvd = spdm_socket_receive(n->spdm_socket, SPDM_TRANSPORT_TYPE_NVME, |
7540 | 0 | &nvme_cmd_status, |
7541 | 0 | SPDM_SOCKET_MAX_MSG_STATUS_LEN); |
7542 | |
|
7543 | 0 | nvme_cmd_status = be16_to_cpu(nvme_cmd_status); |
7544 | |
|
7545 | 0 | if (recvd < SPDM_SOCKET_MAX_MSG_STATUS_LEN) { |
7546 | 0 | return NVME_DATA_TRAS_ERROR | NVME_DNR; |
7547 | 0 | } |
7548 | | |
7549 | 0 | return nvme_cmd_status; |
7550 | 0 | } |
7551 | | |
7552 | | /* From host to controller */ |
7553 | | static uint16_t nvme_security_send(NvmeCtrl *n, NvmeRequest *req) |
7554 | 0 | { |
7555 | 0 | uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); |
7556 | 0 | uint8_t secp = extract32(dw10, 24, 8); |
7557 | |
|
7558 | 0 | switch (secp) { |
7559 | 0 | case NVME_SEC_PROT_DMTF_SPDM: |
7560 | 0 | if (n->spdm_socket < 0) { |
7561 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7562 | 0 | } |
7563 | 0 | return nvme_sec_prot_spdm_send(n, req); |
7564 | 0 | default: |
7565 | | /* Unsupported Security Protocol Type */ |
7566 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7567 | 0 | } |
7568 | | |
7569 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7570 | 0 | } |
7571 | | |
7572 | | static uint16_t nvme_sec_prot_spdm_receive(NvmeCtrl *n, NvmeRequest *req) |
7573 | 0 | { |
7574 | 0 | StorageSpdmTransportHeader hdr; |
7575 | 0 | g_autofree uint8_t *rsp_spdm_buf = NULL; |
7576 | 0 | uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); |
7577 | 0 | uint32_t alloc_len = le32_to_cpu(req->cmd.cdw11); |
7578 | 0 | uint32_t recvd, spdm_res; |
7579 | 0 | uint16_t nvme_cmd_status, ret; |
7580 | 0 | uint8_t secp = extract32(dw10, 24, 8); |
7581 | 0 | uint8_t spsp = extract32(dw10, 8, 16); |
7582 | 0 | if (!alloc_len) { |
7583 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7584 | 0 | } |
7585 | | |
7586 | | /* Generate the NVMe transport header */ |
7587 | 0 | hdr = (StorageSpdmTransportHeader) { |
7588 | 0 | .security_protocol = secp, |
7589 | 0 | .security_protocol_specific = cpu_to_le16(spsp), |
7590 | 0 | .length = cpu_to_le32(alloc_len), |
7591 | 0 | }; |
7592 | | |
7593 | | /* Forward if_recv to the SPDM Server with SPSP0 */ |
7594 | 0 | spdm_res = spdm_socket_send(n->spdm_socket, SPDM_SOCKET_STORAGE_CMD_IF_RECV, |
7595 | 0 | SPDM_TRANSPORT_TYPE_NVME, |
7596 | 0 | &hdr, sizeof(hdr)); |
7597 | 0 | if (!spdm_res) { |
7598 | 0 | return NVME_DATA_TRAS_ERROR | NVME_DNR; |
7599 | 0 | } |
7600 | | |
7601 | | /* The responder shall ack with message status */ |
7602 | 0 | recvd = spdm_socket_receive(n->spdm_socket, SPDM_TRANSPORT_TYPE_NVME, |
7603 | 0 | &nvme_cmd_status, |
7604 | 0 | SPDM_SOCKET_MAX_MSG_STATUS_LEN); |
7605 | 0 | if (recvd < SPDM_SOCKET_MAX_MSG_STATUS_LEN) { |
7606 | 0 | return NVME_DATA_TRAS_ERROR | NVME_DNR; |
7607 | 0 | } |
7608 | | |
7609 | 0 | nvme_cmd_status = be16_to_cpu(nvme_cmd_status); |
7610 | | /* An error here implies the prior if_recv from requester was spurious */ |
7611 | 0 | if (nvme_cmd_status != NVME_SUCCESS) { |
7612 | 0 | return nvme_cmd_status; |
7613 | 0 | } |
7614 | | |
7615 | | /* Clear to start receiving data from the server */ |
7616 | 0 | rsp_spdm_buf = g_try_malloc0(alloc_len); |
7617 | 0 | if (!rsp_spdm_buf) { |
7618 | 0 | return NVME_INTERNAL_DEV_ERROR; |
7619 | 0 | } |
7620 | | |
7621 | 0 | recvd = spdm_socket_receive(n->spdm_socket, |
7622 | 0 | SPDM_TRANSPORT_TYPE_NVME, |
7623 | 0 | rsp_spdm_buf, alloc_len); |
7624 | 0 | if (!recvd) { |
7625 | 0 | return NVME_DATA_TRAS_ERROR | NVME_DNR; |
7626 | 0 | } |
7627 | | |
7628 | 0 | ret = nvme_c2h(n, rsp_spdm_buf, MIN(recvd, alloc_len), req); |
7629 | 0 | if (ret) { |
7630 | 0 | return ret; |
7631 | 0 | } |
7632 | | |
7633 | 0 | return NVME_SUCCESS; |
7634 | 0 | } |
7635 | | |
7636 | | static uint16_t nvme_get_sec_prot_info(NvmeCtrl *n, NvmeRequest *req) |
7637 | 0 | { |
7638 | 0 | uint32_t alloc_len = le32_to_cpu(req->cmd.cdw11); |
7639 | 0 | uint8_t resp[10] = { |
7640 | | /* Support Security Protol List Length */ |
7641 | 0 | [6] = 0, /* MSB */ |
7642 | 0 | [7] = 2, /* LSB */ |
7643 | | /* Support Security Protocol List */ |
7644 | 0 | [8] = SFSC_SECURITY_PROT_INFO, |
7645 | 0 | [9] = 0, |
7646 | 0 | }; |
7647 | |
|
7648 | 0 | if (n->spdm_socket >= 0) { |
7649 | 0 | resp[9] = NVME_SEC_PROT_DMTF_SPDM; |
7650 | 0 | } |
7651 | |
|
7652 | 0 | if (alloc_len < 10) { |
7653 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7654 | 0 | } |
7655 | | |
7656 | 0 | return nvme_c2h(n, resp, sizeof(resp), req); |
7657 | 0 | } |
7658 | | |
7659 | | /* From controller to host */ |
7660 | | static uint16_t nvme_security_receive(NvmeCtrl *n, NvmeRequest *req) |
7661 | 0 | { |
7662 | 0 | uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); |
7663 | 0 | uint16_t spsp = extract32(dw10, 8, 16); |
7664 | 0 | uint8_t secp = extract32(dw10, 24, 8); |
7665 | |
|
7666 | 0 | switch (secp) { |
7667 | 0 | case SFSC_SECURITY_PROT_INFO: |
7668 | 0 | switch (spsp) { |
7669 | 0 | case 0: |
7670 | | /* Supported security protocol list */ |
7671 | 0 | return nvme_get_sec_prot_info(n, req); |
7672 | 0 | case 1: |
7673 | | /* Certificate data */ |
7674 | | /* fallthrough */ |
7675 | 0 | default: |
7676 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7677 | 0 | } |
7678 | 0 | case NVME_SEC_PROT_DMTF_SPDM: |
7679 | 0 | if (n->spdm_socket < 0) { |
7680 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7681 | 0 | } |
7682 | 0 | return nvme_sec_prot_spdm_receive(n, req); |
7683 | 0 | default: |
7684 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7685 | 0 | } |
7686 | 0 | } |
7687 | | |
7688 | | static uint16_t nvme_directive_send(NvmeCtrl *n, NvmeRequest *req) |
7689 | 0 | { |
7690 | | /* |
7691 | | * When adding a new dtype handling here, |
7692 | | * please also update nvme_set_migration_blockers(). |
7693 | | */ |
7694 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7695 | 0 | } |
7696 | | |
7697 | | static uint16_t nvme_directive_receive(NvmeCtrl *n, NvmeRequest *req) |
7698 | 0 | { |
7699 | 0 | NvmeNamespace *ns; |
7700 | 0 | uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); |
7701 | 0 | uint32_t dw11 = le32_to_cpu(req->cmd.cdw11); |
7702 | 0 | uint32_t nsid = le32_to_cpu(req->cmd.nsid); |
7703 | 0 | uint8_t doper, dtype; |
7704 | 0 | uint32_t numd, trans_len; |
7705 | 0 | NvmeDirectiveIdentify id = { |
7706 | 0 | .supported = 1 << NVME_DIRECTIVE_IDENTIFY, |
7707 | 0 | .enabled = 1 << NVME_DIRECTIVE_IDENTIFY, |
7708 | 0 | }; |
7709 | |
|
7710 | 0 | numd = dw10 + 1; |
7711 | 0 | doper = dw11 & 0xff; |
7712 | 0 | dtype = (dw11 >> 8) & 0xff; |
7713 | |
|
7714 | 0 | trans_len = MIN(sizeof(NvmeDirectiveIdentify), numd << 2); |
7715 | |
|
7716 | 0 | if (nsid == NVME_NSID_BROADCAST || dtype != NVME_DIRECTIVE_IDENTIFY || |
7717 | 0 | doper != NVME_DIRECTIVE_RETURN_PARAMS) { |
7718 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7719 | 0 | } |
7720 | | |
7721 | 0 | ns = nvme_ns(n, nsid); |
7722 | 0 | if (!ns) { |
7723 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7724 | 0 | } |
7725 | | |
7726 | 0 | switch (dtype) { |
7727 | 0 | case NVME_DIRECTIVE_IDENTIFY: |
7728 | 0 | switch (doper) { |
7729 | 0 | case NVME_DIRECTIVE_RETURN_PARAMS: |
7730 | 0 | if (ns->endgrp && ns->endgrp->fdp.enabled) { |
7731 | 0 | id.supported |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT; |
7732 | 0 | id.enabled |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT; |
7733 | 0 | id.persistent |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT; |
7734 | 0 | } |
7735 | |
|
7736 | 0 | return nvme_c2h(n, (uint8_t *)&id, trans_len, req); |
7737 | | |
7738 | 0 | default: |
7739 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7740 | 0 | } |
7741 | | |
7742 | 0 | default: |
7743 | 0 | return NVME_INVALID_FIELD; |
7744 | 0 | } |
7745 | 0 | } |
7746 | | |
7747 | | static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeRequest *req) |
7748 | 0 | { |
7749 | 0 | trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), req->cmd.opcode, |
7750 | 0 | nvme_adm_opc_str(req->cmd.opcode)); |
7751 | |
|
7752 | 0 | if (!(n->cse.acs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) { |
7753 | 0 | trace_pci_nvme_err_invalid_admin_opc(req->cmd.opcode); |
7754 | 0 | return NVME_INVALID_OPCODE | NVME_DNR; |
7755 | 0 | } |
7756 | | |
7757 | | /* SGLs shall not be used for Admin commands in NVMe over PCIe */ |
7758 | 0 | if (NVME_CMD_FLAGS_PSDT(req->cmd.flags) != NVME_PSDT_PRP) { |
7759 | 0 | return NVME_INVALID_FIELD | NVME_DNR; |
7760 | 0 | } |
7761 | | |
7762 | 0 | if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) { |
7763 | 0 | return NVME_INVALID_FIELD; |
7764 | 0 | } |
7765 | | |
7766 | 0 | switch (req->cmd.opcode) { |
7767 | 0 | case NVME_ADM_CMD_DELETE_SQ: |
7768 | 0 | return nvme_del_sq(n, req); |
7769 | 0 | case NVME_ADM_CMD_CREATE_SQ: |
7770 | 0 | return nvme_create_sq(n, req); |
7771 | 0 | case NVME_ADM_CMD_GET_LOG_PAGE: |
7772 | 0 | return nvme_get_log(n, req); |
7773 | 0 | case NVME_ADM_CMD_DELETE_CQ: |
7774 | 0 | return nvme_del_cq(n, req); |
7775 | 0 | case NVME_ADM_CMD_CREATE_CQ: |
7776 | 0 | return nvme_create_cq(n, req); |
7777 | 0 | case NVME_ADM_CMD_IDENTIFY: |
7778 | 0 | return nvme_identify(n, req); |
7779 | 0 | case NVME_ADM_CMD_ABORT: |
7780 | 0 | return nvme_abort(n, req); |
7781 | 0 | case NVME_ADM_CMD_SET_FEATURES: |
7782 | 0 | return nvme_set_feature(n, req); |
7783 | 0 | case NVME_ADM_CMD_GET_FEATURES: |
7784 | 0 | return nvme_get_feature(n, req); |
7785 | 0 | case NVME_ADM_CMD_ASYNC_EV_REQ: |
7786 | 0 | return nvme_aer(n, req); |
7787 | 0 | case NVME_ADM_CMD_NS_ATTACHMENT: |
7788 | 0 | return nvme_ns_attachment(n, req); |
7789 | 0 | case NVME_ADM_CMD_VIRT_MNGMT: |
7790 | 0 | return nvme_virt_mngmt(n, req); |
7791 | 0 | case NVME_ADM_CMD_DBBUF_CONFIG: |
7792 | 0 | return nvme_dbbuf_config(n, req); |
7793 | 0 | case NVME_ADM_CMD_FORMAT_NVM: |
7794 | 0 | return nvme_format(n, req); |
7795 | 0 | case NVME_ADM_CMD_DIRECTIVE_SEND: |
7796 | 0 | return nvme_directive_send(n, req); |
7797 | 0 | case NVME_ADM_CMD_DIRECTIVE_RECV: |
7798 | 0 | return nvme_directive_receive(n, req); |
7799 | 0 | case NVME_ADM_CMD_SECURITY_SEND: |
7800 | 0 | return nvme_security_send(n, req); |
7801 | 0 | case NVME_ADM_CMD_SECURITY_RECV: |
7802 | 0 | return nvme_security_receive(n, req); |
7803 | 0 | default: |
7804 | 0 | g_assert_not_reached(); |
7805 | 0 | } |
7806 | | |
7807 | 0 | return NVME_INVALID_OPCODE | NVME_DNR; |
7808 | 0 | } |
7809 | | |
7810 | | static void nvme_update_sq_eventidx(const NvmeSQueue *sq) |
7811 | 0 | { |
7812 | 0 | trace_pci_nvme_update_sq_eventidx(sq->sqid, sq->tail); |
7813 | |
|
7814 | 0 | stl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->ei_addr, sq->tail, |
7815 | 0 | MEMTXATTRS_UNSPECIFIED); |
7816 | 0 | } |
7817 | | |
7818 | | static void nvme_update_sq_tail(NvmeSQueue *sq) |
7819 | 0 | { |
7820 | 0 | ldl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->db_addr, &sq->tail, |
7821 | 0 | MEMTXATTRS_UNSPECIFIED); |
7822 | |
|
7823 | 0 | trace_pci_nvme_update_sq_tail(sq->sqid, sq->tail); |
7824 | 0 | } |
7825 | | |
7826 | | static int nvme_atomic_boundary_check(NvmeCtrl *n, NvmeCmd *cmd, |
7827 | | NvmeAtomic *atomic) |
7828 | 0 | { |
7829 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)cmd; |
7830 | |
|
7831 | 0 | if (atomic->atomic_boundary) { |
7832 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
7833 | 0 | uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb); |
7834 | 0 | uint64_t elba = slba + nlb; |
7835 | 0 | uint64_t imask; |
7836 | |
|
7837 | 0 | if ((slba < atomic->atomic_nabo) || (elba < atomic->atomic_nabo)) { |
7838 | 0 | return 0; |
7839 | 0 | } |
7840 | | |
7841 | | /* Update slba/elba based on boundary offset */ |
7842 | 0 | slba = slba - atomic->atomic_nabo; |
7843 | 0 | elba = slba + nlb; |
7844 | |
|
7845 | 0 | imask = ~(atomic->atomic_boundary - 1); |
7846 | 0 | if ((slba & imask) != (elba & imask)) { |
7847 | | /* |
7848 | | * The write crosses an atomic boundary and the controller provides |
7849 | | * no atomicity guarantees unless AWUN/AWUPF are non-zero. |
7850 | | */ |
7851 | 0 | if (n->atomic.atomic_max_write_size && |
7852 | 0 | ((nlb + 1) <= n->atomic.atomic_max_write_size)) { |
7853 | 0 | return 1; |
7854 | 0 | } |
7855 | 0 | return 0; |
7856 | 0 | } |
7857 | 0 | } |
7858 | 0 | return 1; |
7859 | 0 | } |
7860 | 0 | #define NVME_ATOMIC_NO_START 0 |
7861 | 0 | #define NVME_ATOMIC_START_ATOMIC 1 |
7862 | 0 | #define NVME_ATOMIC_START_NONATOMIC 2 |
7863 | | |
7864 | | static int nvme_atomic_write_check(NvmeCtrl *n, NvmeCmd *cmd, |
7865 | | NvmeAtomic *atomic) |
7866 | 0 | { |
7867 | 0 | NvmeRwCmd *rw = (NvmeRwCmd *)cmd; |
7868 | 0 | uint64_t slba = le64_to_cpu(rw->slba); |
7869 | 0 | uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb); |
7870 | 0 | uint64_t elba = slba + nlb; |
7871 | 0 | bool cmd_atomic_wr = true; |
7872 | 0 | int i; |
7873 | |
|
7874 | 0 | if ((cmd->opcode == NVME_CMD_READ) || ((cmd->opcode == NVME_CMD_WRITE) && |
7875 | 0 | ((rw->nlb + 1) > atomic->atomic_max_write_size))) { |
7876 | 0 | cmd_atomic_wr = false; |
7877 | 0 | } |
7878 | | |
7879 | | /* |
7880 | | * Check if a write crosses an atomic boundary. |
7881 | | */ |
7882 | 0 | if (cmd->opcode == NVME_CMD_WRITE) { |
7883 | 0 | if (!nvme_atomic_boundary_check(n, cmd, atomic)) { |
7884 | 0 | cmd_atomic_wr = false; |
7885 | 0 | } |
7886 | 0 | } |
7887 | | |
7888 | | /* |
7889 | | * Walk the queues to see if there are any atomic conflicts. |
7890 | | */ |
7891 | 0 | for (i = 1; i < n->num_queues; i++) { |
7892 | 0 | NvmeSQueue *sq; |
7893 | 0 | NvmeRequest *req; |
7894 | 0 | NvmeRwCmd *req_rw; |
7895 | 0 | uint64_t req_slba; |
7896 | 0 | uint32_t req_nlb; |
7897 | 0 | uint64_t req_elba; |
7898 | |
|
7899 | 0 | sq = n->sq[i]; |
7900 | 0 | if (!sq) { |
7901 | 0 | continue; |
7902 | 0 | } |
7903 | | |
7904 | | /* |
7905 | | * Walk all the requests on a given queue. |
7906 | | */ |
7907 | 0 | QTAILQ_FOREACH(req, &sq->out_req_list, entry) { |
7908 | 0 | req_rw = (NvmeRwCmd *)&req->cmd; |
7909 | |
|
7910 | 0 | if (((req_rw->opcode == NVME_CMD_WRITE) || |
7911 | 0 | (req_rw->opcode == NVME_CMD_READ)) && |
7912 | 0 | (cmd->nsid == req->ns->params.nsid)) { |
7913 | 0 | req_slba = le64_to_cpu(req_rw->slba); |
7914 | 0 | req_nlb = (uint32_t)le16_to_cpu(req_rw->nlb); |
7915 | 0 | req_elba = req_slba + req_nlb; |
7916 | |
|
7917 | 0 | if (cmd_atomic_wr) { |
7918 | 0 | if ((elba >= req_slba) && (slba <= req_elba)) { |
7919 | 0 | return NVME_ATOMIC_NO_START; |
7920 | 0 | } |
7921 | 0 | } else { |
7922 | 0 | if (req->atomic_write && ((elba >= req_slba) && |
7923 | 0 | (slba <= req_elba))) { |
7924 | 0 | return NVME_ATOMIC_NO_START; |
7925 | 0 | } |
7926 | 0 | } |
7927 | 0 | } |
7928 | 0 | } |
7929 | 0 | } |
7930 | 0 | if (cmd_atomic_wr) { |
7931 | 0 | return NVME_ATOMIC_START_ATOMIC; |
7932 | 0 | } |
7933 | 0 | return NVME_ATOMIC_START_NONATOMIC; |
7934 | 0 | } |
7935 | | |
7936 | | static NvmeAtomic *nvme_get_atomic(NvmeCtrl *n, NvmeCmd *cmd) |
7937 | 0 | { |
7938 | 0 | NvmeNamespace *ns = nvme_ns(n, cmd->nsid); |
7939 | |
|
7940 | 0 | if (ns && ns->atomic.atomic_writes) { |
7941 | 0 | return &ns->atomic; |
7942 | 0 | } |
7943 | | |
7944 | 0 | if (n->atomic.atomic_writes) { |
7945 | 0 | return &n->atomic; |
7946 | 0 | } |
7947 | 0 | return NULL; |
7948 | 0 | } |
7949 | | |
7950 | | static void nvme_process_sq(void *opaque) |
7951 | 0 | { |
7952 | 0 | NvmeSQueue *sq = opaque; |
7953 | 0 | NvmeCtrl *n = sq->ctrl; |
7954 | 0 | NvmeCQueue *cq = n->cq[sq->cqid]; |
7955 | |
|
7956 | 0 | uint16_t status; |
7957 | 0 | hwaddr addr; |
7958 | 0 | NvmeCmd cmd; |
7959 | 0 | NvmeRequest *req; |
7960 | | |
7961 | | /* |
7962 | | * We don't want to have a race with nvme_ctrl_pre_save(). |
7963 | | * What implicitly protects us from this is BQL. |
7964 | | */ |
7965 | 0 | assert(bql_locked()); |
7966 | |
|
7967 | 0 | if (n->dbbuf_enabled) { |
7968 | 0 | nvme_update_sq_tail(sq); |
7969 | 0 | } |
7970 | |
|
7971 | 0 | while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) { |
7972 | 0 | NvmeAtomic *atomic; |
7973 | 0 | bool cmd_is_atomic; |
7974 | |
|
7975 | 0 | addr = sq->dma_addr + (sq->head << NVME_SQES); |
7976 | 0 | if (nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd))) { |
7977 | 0 | trace_pci_nvme_err_addr_read(addr); |
7978 | 0 | trace_pci_nvme_err_cfs(); |
7979 | 0 | stl_le_p(&n->bar.csts, NVME_CSTS_FAILED); |
7980 | 0 | break; |
7981 | 0 | } |
7982 | | |
7983 | 0 | atomic = nvme_get_atomic(n, &cmd); |
7984 | |
|
7985 | 0 | cmd_is_atomic = false; |
7986 | 0 | if (sq->sqid && atomic) { |
7987 | 0 | int ret; |
7988 | |
|
7989 | 0 | ret = nvme_atomic_write_check(n, &cmd, atomic); |
7990 | 0 | switch (ret) { |
7991 | 0 | case NVME_ATOMIC_NO_START: |
7992 | 0 | qemu_bh_schedule(sq->bh); |
7993 | 0 | return; |
7994 | 0 | case NVME_ATOMIC_START_ATOMIC: |
7995 | 0 | cmd_is_atomic = true; |
7996 | 0 | break; |
7997 | 0 | case NVME_ATOMIC_START_NONATOMIC: |
7998 | 0 | default: |
7999 | 0 | break; |
8000 | 0 | } |
8001 | 0 | } |
8002 | 0 | nvme_inc_sq_head(sq); |
8003 | |
|
8004 | 0 | req = QTAILQ_FIRST(&sq->req_list); |
8005 | 0 | QTAILQ_REMOVE(&sq->req_list, req, entry); |
8006 | 0 | QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry); |
8007 | 0 | nvme_req_clear(req); |
8008 | 0 | req->cqe.sq_id = cpu_to_le16(sq->sqid); |
8009 | 0 | req->cqe.cid = cmd.cid; |
8010 | 0 | memcpy(&req->cmd, &cmd, sizeof(NvmeCmd)); |
8011 | |
|
8012 | 0 | if (sq->sqid && atomic) { |
8013 | 0 | req->atomic_write = cmd_is_atomic; |
8014 | 0 | } |
8015 | |
|
8016 | 0 | status = sq->sqid ? nvme_io_cmd(n, req) : |
8017 | 0 | nvme_admin_cmd(n, req); |
8018 | 0 | if (status != NVME_NO_COMPLETE) { |
8019 | 0 | req->status = status; |
8020 | 0 | nvme_enqueue_req_completion(cq, req); |
8021 | 0 | } |
8022 | |
|
8023 | 0 | if (n->dbbuf_enabled) { |
8024 | 0 | nvme_update_sq_eventidx(sq); |
8025 | 0 | nvme_update_sq_tail(sq); |
8026 | 0 | } |
8027 | 0 | } |
8028 | 0 | } |
8029 | | |
8030 | | static void nvme_update_msixcap_ts(PCIDevice *pci_dev, uint32_t table_size) |
8031 | 0 | { |
8032 | 0 | uint8_t *config; |
8033 | |
|
8034 | 0 | if (!msix_present(pci_dev)) { |
8035 | 0 | return; |
8036 | 0 | } |
8037 | | |
8038 | 0 | assert(table_size > 0 && table_size <= pci_dev->msix_entries_nr); |
8039 | |
|
8040 | 0 | config = pci_dev->config + pci_dev->msix_cap; |
8041 | 0 | pci_set_word_by_mask(config + PCI_MSIX_FLAGS, PCI_MSIX_FLAGS_QSIZE, |
8042 | 0 | table_size - 1); |
8043 | 0 | } |
8044 | | |
8045 | | static void nvme_activate_virt_res(NvmeCtrl *n) |
8046 | 0 | { |
8047 | 0 | PCIDevice *pci_dev = PCI_DEVICE(n); |
8048 | 0 | NvmePriCtrlCap *cap = &n->pri_ctrl_cap; |
8049 | 0 | NvmeSecCtrlEntry *sctrl; |
8050 | | |
8051 | | /* -1 to account for the admin queue */ |
8052 | 0 | if (pci_is_vf(pci_dev)) { |
8053 | 0 | sctrl = nvme_sctrl(n); |
8054 | 0 | cap->vqprt = sctrl->nvq; |
8055 | 0 | cap->viprt = sctrl->nvi; |
8056 | 0 | n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0; |
8057 | 0 | n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1; |
8058 | 0 | } else { |
8059 | 0 | cap->vqrfap = n->next_pri_ctrl_cap.vqrfap; |
8060 | 0 | cap->virfap = n->next_pri_ctrl_cap.virfap; |
8061 | 0 | n->conf_ioqpairs = le16_to_cpu(cap->vqprt) + |
8062 | 0 | le16_to_cpu(cap->vqrfap) - 1; |
8063 | 0 | n->conf_msix_qsize = le16_to_cpu(cap->viprt) + |
8064 | 0 | le16_to_cpu(cap->virfap); |
8065 | 0 | } |
8066 | 0 | } |
8067 | | |
8068 | | static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst) |
8069 | 0 | { |
8070 | 0 | PCIDevice *pci_dev = PCI_DEVICE(n); |
8071 | 0 | NvmeSecCtrlEntry *sctrl; |
8072 | 0 | NvmeNamespace *ns; |
8073 | 0 | int i; |
8074 | |
|
8075 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
8076 | 0 | ns = nvme_ns(n, i); |
8077 | 0 | if (!ns) { |
8078 | 0 | continue; |
8079 | 0 | } |
8080 | | |
8081 | 0 | nvme_ns_drain(ns); |
8082 | 0 | } |
8083 | | |
8084 | | /* |
8085 | | * Cancel and wait out every inflight command on every queue first. A |
8086 | | * reset is not required to be preceded by the guest's graceful |
8087 | | * Delete I/O SQ/CQ sequence, so sq/cq must not be freed below while a |
8088 | | * blk_aio_* completion for them could still be in flight. |
8089 | | */ |
8090 | 0 | for (i = 0; i < n->num_queues; i++) { |
8091 | 0 | if (n->sq[i] != NULL) { |
8092 | 0 | nvme_sq_cancel_inflight(n->sq[i], NVME_CMD_ABORT_SQ_DEL); |
8093 | 0 | } |
8094 | 0 | } |
8095 | |
|
8096 | 0 | for (i = 0; i < n->num_queues; i++) { |
8097 | 0 | if (n->sq[i] != NULL) { |
8098 | 0 | nvme_free_sq(n->sq[i], n); |
8099 | 0 | } |
8100 | 0 | } |
8101 | 0 | for (i = 0; i < n->num_queues; i++) { |
8102 | 0 | if (n->cq[i] != NULL) { |
8103 | 0 | nvme_free_cq(n->cq[i], n); |
8104 | 0 | } |
8105 | 0 | } |
8106 | |
|
8107 | 0 | while (!QTAILQ_EMPTY(&n->aer_queue)) { |
8108 | 0 | NvmeAsyncEvent *event = QTAILQ_FIRST(&n->aer_queue); |
8109 | 0 | QTAILQ_REMOVE(&n->aer_queue, event, entry); |
8110 | 0 | g_free(event); |
8111 | 0 | } |
8112 | |
|
8113 | 0 | if (n->params.sriov_max_vfs) { |
8114 | 0 | if (!pci_is_vf(pci_dev)) { |
8115 | 0 | for (i = 0; i < n->nr_sec_ctrls; i++) { |
8116 | 0 | sctrl = &n->sec_ctrl_list[i]; |
8117 | 0 | nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false); |
8118 | 0 | } |
8119 | 0 | } |
8120 | |
|
8121 | 0 | if (rst != NVME_RESET_CONTROLLER) { |
8122 | 0 | nvme_activate_virt_res(n); |
8123 | 0 | } |
8124 | 0 | } |
8125 | |
|
8126 | 0 | n->aer_queued = 0; |
8127 | 0 | n->aer_mask = 0; |
8128 | 0 | n->outstanding_aers = 0; |
8129 | 0 | n->qs_created = false; |
8130 | |
|
8131 | 0 | n->dn = n->params.atomic_dn; /* Set Disable Normal */ |
8132 | |
|
8133 | 0 | nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize); |
8134 | |
|
8135 | 0 | if (pci_is_vf(pci_dev)) { |
8136 | 0 | sctrl = nvme_sctrl(n); |
8137 | |
|
8138 | 0 | stl_le_p(&n->bar.csts, sctrl->scs ? 0 : NVME_CSTS_FAILED); |
8139 | 0 | } else { |
8140 | 0 | stl_le_p(&n->bar.csts, 0); |
8141 | 0 | } |
8142 | |
|
8143 | 0 | stl_le_p(&n->bar.intms, 0); |
8144 | 0 | stl_le_p(&n->bar.intmc, 0); |
8145 | 0 | stl_le_p(&n->bar.cc, 0); |
8146 | |
|
8147 | 0 | n->cq_pending = 0; |
8148 | 0 | n->irq_status = 0; |
8149 | 0 | nvme_irq_check(n); |
8150 | |
|
8151 | 0 | n->dbbuf_dbs = 0; |
8152 | 0 | n->dbbuf_eis = 0; |
8153 | 0 | n->dbbuf_enabled = false; |
8154 | 0 | } |
8155 | | |
8156 | | static void nvme_ctrl_shutdown(NvmeCtrl *n) |
8157 | 0 | { |
8158 | 0 | NvmeNamespace *ns; |
8159 | 0 | int i; |
8160 | |
|
8161 | 0 | if (n->pmr.dev) { |
8162 | 0 | memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size); |
8163 | 0 | } |
8164 | |
|
8165 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
8166 | 0 | ns = nvme_ns(n, i); |
8167 | 0 | if (!ns) { |
8168 | 0 | continue; |
8169 | 0 | } |
8170 | | |
8171 | 0 | nvme_ns_shutdown(ns); |
8172 | 0 | } |
8173 | 0 | } |
8174 | | |
8175 | | static int nvme_start_ctrl(NvmeCtrl *n) |
8176 | 0 | { |
8177 | 0 | uint64_t cap = ldq_le_p(&n->bar.cap); |
8178 | 0 | uint32_t cc = ldl_le_p(&n->bar.cc); |
8179 | 0 | uint32_t aqa = ldl_le_p(&n->bar.aqa); |
8180 | 0 | uint64_t asq = ldq_le_p(&n->bar.asq); |
8181 | 0 | uint64_t acq = ldq_le_p(&n->bar.acq); |
8182 | 0 | uint32_t page_bits = NVME_CC_MPS(cc) + 12; |
8183 | 0 | uint32_t page_size = 1 << page_bits; |
8184 | 0 | NvmeSecCtrlEntry *sctrl = nvme_sctrl(n); |
8185 | |
|
8186 | 0 | if (pci_is_vf(PCI_DEVICE(n)) && !sctrl->scs) { |
8187 | 0 | trace_pci_nvme_err_startfail_virt_state(le16_to_cpu(sctrl->nvi), |
8188 | 0 | le16_to_cpu(sctrl->nvq)); |
8189 | 0 | return -1; |
8190 | 0 | } |
8191 | 0 | if (unlikely(n->cq[0])) { |
8192 | 0 | trace_pci_nvme_err_startfail_cq(); |
8193 | 0 | return -1; |
8194 | 0 | } |
8195 | 0 | if (unlikely(n->sq[0])) { |
8196 | 0 | trace_pci_nvme_err_startfail_sq(); |
8197 | 0 | return -1; |
8198 | 0 | } |
8199 | 0 | if (unlikely(asq & (page_size - 1))) { |
8200 | 0 | trace_pci_nvme_err_startfail_asq_misaligned(asq); |
8201 | 0 | return -1; |
8202 | 0 | } |
8203 | 0 | if (unlikely(acq & (page_size - 1))) { |
8204 | 0 | trace_pci_nvme_err_startfail_acq_misaligned(acq); |
8205 | 0 | return -1; |
8206 | 0 | } |
8207 | 0 | if (unlikely(!(NVME_CAP_CSS(cap) & (1 << NVME_CC_CSS(cc))))) { |
8208 | 0 | trace_pci_nvme_err_startfail_css(NVME_CC_CSS(cc)); |
8209 | 0 | return -1; |
8210 | 0 | } |
8211 | 0 | if (unlikely(NVME_CC_MPS(cc) < NVME_CAP_MPSMIN(cap))) { |
8212 | 0 | trace_pci_nvme_err_startfail_page_too_small( |
8213 | 0 | NVME_CC_MPS(cc), |
8214 | 0 | NVME_CAP_MPSMIN(cap)); |
8215 | 0 | return -1; |
8216 | 0 | } |
8217 | 0 | if (unlikely(NVME_CC_MPS(cc) > |
8218 | 0 | NVME_CAP_MPSMAX(cap))) { |
8219 | 0 | trace_pci_nvme_err_startfail_page_too_large( |
8220 | 0 | NVME_CC_MPS(cc), |
8221 | 0 | NVME_CAP_MPSMAX(cap)); |
8222 | 0 | return -1; |
8223 | 0 | } |
8224 | 0 | if (unlikely(!NVME_AQA_ASQS(aqa))) { |
8225 | 0 | trace_pci_nvme_err_startfail_asqent_sz_zero(); |
8226 | 0 | return -1; |
8227 | 0 | } |
8228 | 0 | if (unlikely(!NVME_AQA_ACQS(aqa))) { |
8229 | 0 | trace_pci_nvme_err_startfail_acqent_sz_zero(); |
8230 | 0 | return -1; |
8231 | 0 | } |
8232 | | |
8233 | 0 | n->page_bits = page_bits; |
8234 | 0 | n->page_size = page_size; |
8235 | 0 | n->max_prp_ents = n->page_size / sizeof(uint64_t); |
8236 | 0 | nvme_init_cq(&n->admin_cq, n, acq, 0, 0, NVME_AQA_ACQS(aqa) + 1, 1); |
8237 | 0 | nvme_init_sq(&n->admin_sq, n, asq, 0, 0, NVME_AQA_ASQS(aqa) + 1); |
8238 | |
|
8239 | 0 | nvme_set_timestamp(n, 0ULL); |
8240 | | |
8241 | | /* verify that the command sets of attached namespaces are supported */ |
8242 | 0 | for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
8243 | 0 | NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i); |
8244 | |
|
8245 | 0 | if (!ns || (!ns->params.shared && ns->ctrl != n)) { |
8246 | 0 | continue; |
8247 | 0 | } |
8248 | | |
8249 | 0 | if (nvme_csi_supported(n, ns->csi) && !ns->params.detached) { |
8250 | 0 | if (!ns->attached || ns->params.shared) { |
8251 | 0 | nvme_attach_ns(n, ns); |
8252 | 0 | } |
8253 | 0 | } |
8254 | 0 | } |
8255 | |
|
8256 | 0 | nvme_update_dsm_limits(n, NULL); |
8257 | |
|
8258 | 0 | return 0; |
8259 | 0 | } |
8260 | | |
8261 | | static void nvme_cmb_enable_regs(NvmeCtrl *n) |
8262 | 0 | { |
8263 | 0 | uint32_t cmbloc = ldl_le_p(&n->bar.cmbloc); |
8264 | 0 | uint32_t cmbsz = ldl_le_p(&n->bar.cmbsz); |
8265 | |
|
8266 | 0 | NVME_CMBLOC_SET_CDPCILS(cmbloc, 1); |
8267 | 0 | NVME_CMBLOC_SET_CDPMLS(cmbloc, 1); |
8268 | 0 | NVME_CMBLOC_SET_BIR(cmbloc, NVME_CMB_BIR); |
8269 | 0 | stl_le_p(&n->bar.cmbloc, cmbloc); |
8270 | |
|
8271 | 0 | NVME_CMBSZ_SET_SQS(cmbsz, 1); |
8272 | 0 | NVME_CMBSZ_SET_CQS(cmbsz, 0); |
8273 | 0 | NVME_CMBSZ_SET_LISTS(cmbsz, 1); |
8274 | 0 | NVME_CMBSZ_SET_RDS(cmbsz, 1); |
8275 | 0 | NVME_CMBSZ_SET_WDS(cmbsz, 1); |
8276 | 0 | NVME_CMBSZ_SET_SZU(cmbsz, 2); /* MBs */ |
8277 | 0 | NVME_CMBSZ_SET_SZ(cmbsz, n->params.cmb_size_mb); |
8278 | 0 | stl_le_p(&n->bar.cmbsz, cmbsz); |
8279 | 0 | } |
8280 | | |
8281 | | static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data, |
8282 | | unsigned size) |
8283 | 0 | { |
8284 | 0 | PCIDevice *pci = PCI_DEVICE(n); |
8285 | 0 | uint64_t cap = ldq_le_p(&n->bar.cap); |
8286 | 0 | uint32_t cc = ldl_le_p(&n->bar.cc); |
8287 | 0 | uint32_t intms = ldl_le_p(&n->bar.intms); |
8288 | 0 | uint32_t csts = ldl_le_p(&n->bar.csts); |
8289 | 0 | uint32_t pmrsts = ldl_le_p(&n->bar.pmrsts); |
8290 | |
|
8291 | 0 | if (unlikely(offset & (sizeof(uint32_t) - 1))) { |
8292 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32, |
8293 | 0 | "MMIO write not 32-bit aligned," |
8294 | 0 | " offset=0x%"PRIx64"", offset); |
8295 | | /* should be ignored, fall through for now */ |
8296 | 0 | } |
8297 | |
|
8298 | 0 | if (unlikely(size < sizeof(uint32_t))) { |
8299 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall, |
8300 | 0 | "MMIO write smaller than 32-bits," |
8301 | 0 | " offset=0x%"PRIx64", size=%u", |
8302 | 0 | offset, size); |
8303 | | /* should be ignored, fall through for now */ |
8304 | 0 | } |
8305 | |
|
8306 | 0 | switch (offset) { |
8307 | 0 | case NVME_REG_INTMS: |
8308 | 0 | if (unlikely(msix_enabled(pci))) { |
8309 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix, |
8310 | 0 | "undefined access to interrupt mask set" |
8311 | 0 | " when MSI-X is enabled"); |
8312 | | /* should be ignored, fall through for now */ |
8313 | 0 | } |
8314 | 0 | intms |= data; |
8315 | 0 | stl_le_p(&n->bar.intms, intms); |
8316 | 0 | n->bar.intmc = n->bar.intms; |
8317 | 0 | trace_pci_nvme_mmio_intm_set(data & 0xffffffff, intms); |
8318 | 0 | nvme_irq_check(n); |
8319 | 0 | break; |
8320 | 0 | case NVME_REG_INTMC: |
8321 | 0 | if (unlikely(msix_enabled(pci))) { |
8322 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix, |
8323 | 0 | "undefined access to interrupt mask clr" |
8324 | 0 | " when MSI-X is enabled"); |
8325 | | /* should be ignored, fall through for now */ |
8326 | 0 | } |
8327 | 0 | intms &= ~data; |
8328 | 0 | stl_le_p(&n->bar.intms, intms); |
8329 | 0 | n->bar.intmc = n->bar.intms; |
8330 | 0 | trace_pci_nvme_mmio_intm_clr(data & 0xffffffff, intms); |
8331 | 0 | nvme_irq_check(n); |
8332 | 0 | break; |
8333 | 0 | case NVME_REG_CC: |
8334 | 0 | stl_le_p(&n->bar.cc, data); |
8335 | |
|
8336 | 0 | trace_pci_nvme_mmio_cfg(data & 0xffffffff); |
8337 | |
|
8338 | 0 | if (NVME_CC_SHN(data) && !(NVME_CC_SHN(cc))) { |
8339 | 0 | trace_pci_nvme_mmio_shutdown_set(); |
8340 | 0 | nvme_ctrl_shutdown(n); |
8341 | 0 | csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT); |
8342 | 0 | csts |= NVME_CSTS_SHST_COMPLETE; |
8343 | 0 | } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(cc)) { |
8344 | 0 | trace_pci_nvme_mmio_shutdown_cleared(); |
8345 | 0 | csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT); |
8346 | 0 | } |
8347 | |
|
8348 | 0 | if (NVME_CC_EN(data) && !NVME_CC_EN(cc)) { |
8349 | 0 | if (unlikely(nvme_start_ctrl(n))) { |
8350 | 0 | trace_pci_nvme_err_startfail(); |
8351 | 0 | csts = NVME_CSTS_FAILED; |
8352 | 0 | } else { |
8353 | 0 | trace_pci_nvme_mmio_start_success(); |
8354 | 0 | csts = NVME_CSTS_READY; |
8355 | 0 | } |
8356 | 0 | } else if (!NVME_CC_EN(data) && NVME_CC_EN(cc)) { |
8357 | 0 | trace_pci_nvme_mmio_stopped(); |
8358 | 0 | nvme_ctrl_reset(n, NVME_RESET_CONTROLLER); |
8359 | |
|
8360 | 0 | break; |
8361 | 0 | } |
8362 | | |
8363 | 0 | stl_le_p(&n->bar.csts, csts); |
8364 | |
|
8365 | 0 | break; |
8366 | 0 | case NVME_REG_CSTS: |
8367 | 0 | if (data & (1 << 4)) { |
8368 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported, |
8369 | 0 | "attempted to W1C CSTS.NSSRO" |
8370 | 0 | " but CAP.NSSRS is zero (not supported)"); |
8371 | 0 | } else if (data != 0) { |
8372 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts, |
8373 | 0 | "attempted to set a read only bit" |
8374 | 0 | " of controller status"); |
8375 | 0 | } |
8376 | 0 | break; |
8377 | 0 | case NVME_REG_NSSR: |
8378 | 0 | if (data == 0x4e564d65) { |
8379 | 0 | trace_pci_nvme_ub_mmiowr_ssreset_unsupported(); |
8380 | 0 | } else { |
8381 | | /* The spec says that writes of other values have no effect */ |
8382 | 0 | return; |
8383 | 0 | } |
8384 | 0 | break; |
8385 | 0 | case NVME_REG_AQA: |
8386 | 0 | stl_le_p(&n->bar.aqa, data); |
8387 | 0 | trace_pci_nvme_mmio_aqattr(data & 0xffffffff); |
8388 | 0 | break; |
8389 | 0 | case NVME_REG_ASQ: |
8390 | 0 | stn_le_p(&n->bar.asq, size, data); |
8391 | 0 | trace_pci_nvme_mmio_asqaddr(data); |
8392 | 0 | break; |
8393 | 0 | case NVME_REG_ASQ + 4: |
8394 | 0 | stl_le_p((uint8_t *)&n->bar.asq + 4, data); |
8395 | 0 | trace_pci_nvme_mmio_asqaddr_hi(data, ldq_le_p(&n->bar.asq)); |
8396 | 0 | break; |
8397 | 0 | case NVME_REG_ACQ: |
8398 | 0 | trace_pci_nvme_mmio_acqaddr(data); |
8399 | 0 | stn_le_p(&n->bar.acq, size, data); |
8400 | 0 | break; |
8401 | 0 | case NVME_REG_ACQ + 4: |
8402 | 0 | stl_le_p((uint8_t *)&n->bar.acq + 4, data); |
8403 | 0 | trace_pci_nvme_mmio_acqaddr_hi(data, ldq_le_p(&n->bar.acq)); |
8404 | 0 | break; |
8405 | 0 | case NVME_REG_CMBLOC: |
8406 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved, |
8407 | 0 | "invalid write to reserved CMBLOC" |
8408 | 0 | " when CMBSZ is zero, ignored"); |
8409 | 0 | return; |
8410 | 0 | case NVME_REG_CMBSZ: |
8411 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly, |
8412 | 0 | "invalid write to read only CMBSZ, ignored"); |
8413 | 0 | return; |
8414 | 0 | case NVME_REG_CMBMSC: |
8415 | 0 | if (!NVME_CAP_CMBS(cap)) { |
8416 | 0 | return; |
8417 | 0 | } |
8418 | | |
8419 | 0 | stn_le_p(&n->bar.cmbmsc, size, data); |
8420 | 0 | n->cmb.cmse = false; |
8421 | |
|
8422 | 0 | if (NVME_CMBMSC_CRE(data)) { |
8423 | 0 | nvme_cmb_enable_regs(n); |
8424 | |
|
8425 | 0 | if (NVME_CMBMSC_CMSE(data)) { |
8426 | 0 | uint64_t cmbmsc = ldq_le_p(&n->bar.cmbmsc); |
8427 | 0 | hwaddr cba = NVME_CMBMSC_CBA(cmbmsc) << CMBMSC_CBA_SHIFT; |
8428 | 0 | if (cba + int128_get64(n->cmb.mem.size) < cba) { |
8429 | 0 | uint32_t cmbsts = ldl_le_p(&n->bar.cmbsts); |
8430 | 0 | NVME_CMBSTS_SET_CBAI(cmbsts, 1); |
8431 | 0 | stl_le_p(&n->bar.cmbsts, cmbsts); |
8432 | 0 | return; |
8433 | 0 | } |
8434 | | |
8435 | 0 | n->cmb.cba = cba; |
8436 | 0 | n->cmb.cmse = true; |
8437 | 0 | } |
8438 | 0 | } else { |
8439 | 0 | n->bar.cmbsz = 0; |
8440 | 0 | n->bar.cmbloc = 0; |
8441 | 0 | } |
8442 | | |
8443 | 0 | return; |
8444 | 0 | case NVME_REG_CMBMSC + 4: |
8445 | 0 | stl_le_p((uint8_t *)&n->bar.cmbmsc + 4, data); |
8446 | 0 | return; |
8447 | | |
8448 | 0 | case NVME_REG_PMRCAP: |
8449 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly, |
8450 | 0 | "invalid write to PMRCAP register, ignored"); |
8451 | 0 | return; |
8452 | 0 | case NVME_REG_PMRCTL: |
8453 | 0 | if (!NVME_CAP_PMRS(cap)) { |
8454 | 0 | return; |
8455 | 0 | } |
8456 | | |
8457 | 0 | stl_le_p(&n->bar.pmrctl, data); |
8458 | 0 | if (NVME_PMRCTL_EN(data)) { |
8459 | 0 | memory_region_set_enabled(&n->pmr.dev->mr, true); |
8460 | 0 | pmrsts = 0; |
8461 | 0 | } else { |
8462 | 0 | memory_region_set_enabled(&n->pmr.dev->mr, false); |
8463 | 0 | NVME_PMRSTS_SET_NRDY(pmrsts, 1); |
8464 | 0 | n->pmr.cmse = false; |
8465 | 0 | } |
8466 | 0 | stl_le_p(&n->bar.pmrsts, pmrsts); |
8467 | 0 | return; |
8468 | 0 | case NVME_REG_PMRSTS: |
8469 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly, |
8470 | 0 | "invalid write to PMRSTS register, ignored"); |
8471 | 0 | return; |
8472 | 0 | case NVME_REG_PMREBS: |
8473 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly, |
8474 | 0 | "invalid write to PMREBS register, ignored"); |
8475 | 0 | return; |
8476 | 0 | case NVME_REG_PMRSWTP: |
8477 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly, |
8478 | 0 | "invalid write to PMRSWTP register, ignored"); |
8479 | 0 | return; |
8480 | 0 | case NVME_REG_PMRMSCL: |
8481 | 0 | if (!NVME_CAP_PMRS(cap)) { |
8482 | 0 | return; |
8483 | 0 | } |
8484 | | |
8485 | 0 | stl_le_p(&n->bar.pmrmscl, data); |
8486 | 0 | n->pmr.cmse = false; |
8487 | |
|
8488 | 0 | if (NVME_PMRMSCL_CMSE(data)) { |
8489 | 0 | uint64_t pmrmscu = ldl_le_p(&n->bar.pmrmscu); |
8490 | 0 | hwaddr cba = pmrmscu << 32 | |
8491 | 0 | (NVME_PMRMSCL_CBA(data) << PMRMSCL_CBA_SHIFT); |
8492 | 0 | if (cba + int128_get64(n->pmr.dev->mr.size) < cba) { |
8493 | 0 | NVME_PMRSTS_SET_CBAI(pmrsts, 1); |
8494 | 0 | stl_le_p(&n->bar.pmrsts, pmrsts); |
8495 | 0 | return; |
8496 | 0 | } |
8497 | | |
8498 | 0 | n->pmr.cmse = true; |
8499 | 0 | n->pmr.cba = cba; |
8500 | 0 | } |
8501 | | |
8502 | 0 | return; |
8503 | 0 | case NVME_REG_PMRMSCU: |
8504 | 0 | if (!NVME_CAP_PMRS(cap)) { |
8505 | 0 | return; |
8506 | 0 | } |
8507 | | |
8508 | 0 | stl_le_p(&n->bar.pmrmscu, data); |
8509 | 0 | return; |
8510 | 0 | default: |
8511 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid, |
8512 | 0 | "invalid MMIO write," |
8513 | 0 | " offset=0x%"PRIx64", data=%"PRIx64"", |
8514 | 0 | offset, data); |
8515 | 0 | break; |
8516 | 0 | } |
8517 | 0 | } |
8518 | | |
8519 | | static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size) |
8520 | 0 | { |
8521 | 0 | NvmeCtrl *n = (NvmeCtrl *)opaque; |
8522 | 0 | uint8_t *ptr = (uint8_t *)&n->bar; |
8523 | |
|
8524 | 0 | trace_pci_nvme_mmio_read(addr, size); |
8525 | |
|
8526 | 0 | if (unlikely(addr & (sizeof(uint32_t) - 1))) { |
8527 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32, |
8528 | 0 | "MMIO read not 32-bit aligned," |
8529 | 0 | " offset=0x%"PRIx64"", addr); |
8530 | | /* should RAZ, fall through for now */ |
8531 | 0 | } else if (unlikely(size < sizeof(uint32_t))) { |
8532 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall, |
8533 | 0 | "MMIO read smaller than 32-bits," |
8534 | 0 | " offset=0x%"PRIx64"", addr); |
8535 | | /* should RAZ, fall through for now */ |
8536 | 0 | } |
8537 | |
|
8538 | 0 | if (addr > sizeof(n->bar) - size) { |
8539 | 0 | NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs, |
8540 | 0 | "MMIO read beyond last register," |
8541 | 0 | " offset=0x%"PRIx64", returning 0", addr); |
8542 | |
|
8543 | 0 | return 0; |
8544 | 0 | } |
8545 | | |
8546 | 0 | if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs && |
8547 | 0 | addr != NVME_REG_CSTS) { |
8548 | 0 | trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size); |
8549 | 0 | return 0; |
8550 | 0 | } |
8551 | | |
8552 | | /* |
8553 | | * When PMRWBM bit 1 is set then read from |
8554 | | * from PMRSTS should ensure prior writes |
8555 | | * made it to persistent media |
8556 | | */ |
8557 | 0 | if (addr == NVME_REG_PMRSTS && |
8558 | 0 | (NVME_PMRCAP_PMRWBM(ldl_le_p(&n->bar.pmrcap)) & 0x02)) { |
8559 | 0 | memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size); |
8560 | 0 | } |
8561 | |
|
8562 | 0 | return ldn_le_p(ptr + addr, size); |
8563 | 0 | } |
8564 | | |
8565 | | static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val) |
8566 | 0 | { |
8567 | 0 | PCIDevice *pci = PCI_DEVICE(n); |
8568 | 0 | uint32_t qid; |
8569 | |
|
8570 | 0 | if (unlikely(addr & ((1 << 2) - 1))) { |
8571 | 0 | NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned, |
8572 | 0 | "doorbell write not 32-bit aligned," |
8573 | 0 | " offset=0x%"PRIx64", ignoring", addr); |
8574 | 0 | return; |
8575 | 0 | } |
8576 | | |
8577 | 0 | if (((addr - 0x1000) >> 2) & 1) { |
8578 | | /* Completion queue doorbell write */ |
8579 | |
|
8580 | 0 | uint16_t new_head = val & 0xffff; |
8581 | 0 | NvmeCQueue *cq; |
8582 | 0 | bool pending; |
8583 | |
|
8584 | 0 | qid = (addr - (0x1000 + (1 << 2))) >> 3; |
8585 | 0 | if (unlikely(nvme_check_cqid(n, qid))) { |
8586 | 0 | NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq, |
8587 | 0 | "completion queue doorbell write" |
8588 | 0 | " for nonexistent queue," |
8589 | 0 | " sqid=%"PRIu32", ignoring", qid); |
8590 | | |
8591 | | /* |
8592 | | * NVM Express v1.3d, Section 4.1 state: "If host software writes |
8593 | | * an invalid value to the Submission Queue Tail Doorbell or |
8594 | | * Completion Queue Head Doorbell register and an Asynchronous Event |
8595 | | * Request command is outstanding, then an asynchronous event is |
8596 | | * posted to the Admin Completion Queue with a status code of |
8597 | | * Invalid Doorbell Write Value." |
8598 | | * |
8599 | | * Also note that the spec includes the "Invalid Doorbell Register" |
8600 | | * status code, but nowhere does it specify when to use it. |
8601 | | * However, it seems reasonable to use it here in a similar |
8602 | | * fashion. |
8603 | | */ |
8604 | 0 | if (n->outstanding_aers) { |
8605 | 0 | nvme_enqueue_event(n, NVME_AER_TYPE_ERROR, |
8606 | 0 | NVME_AER_INFO_ERR_INVALID_DB_REGISTER, |
8607 | 0 | NVME_LOG_ERROR_INFO); |
8608 | 0 | } |
8609 | |
|
8610 | 0 | return; |
8611 | 0 | } |
8612 | | |
8613 | 0 | cq = n->cq[qid]; |
8614 | 0 | if (unlikely(new_head >= cq->size)) { |
8615 | 0 | NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead, |
8616 | 0 | "completion queue doorbell write value" |
8617 | 0 | " beyond queue size, sqid=%"PRIu32"," |
8618 | 0 | " new_head=%"PRIu16", ignoring", |
8619 | 0 | qid, new_head); |
8620 | |
|
8621 | 0 | if (n->outstanding_aers) { |
8622 | 0 | nvme_enqueue_event(n, NVME_AER_TYPE_ERROR, |
8623 | 0 | NVME_AER_INFO_ERR_INVALID_DB_VALUE, |
8624 | 0 | NVME_LOG_ERROR_INFO); |
8625 | 0 | } |
8626 | |
|
8627 | 0 | return; |
8628 | 0 | } |
8629 | | |
8630 | 0 | trace_pci_nvme_mmio_doorbell_cq(cq->cqid, new_head); |
8631 | | |
8632 | | /* scheduled deferred cqe posting if queue was previously full */ |
8633 | 0 | if (nvme_cq_full(cq)) { |
8634 | 0 | qemu_bh_schedule(cq->bh); |
8635 | 0 | } |
8636 | |
|
8637 | 0 | pending = cq->tail != cq->head; |
8638 | 0 | cq->head = new_head; |
8639 | 0 | if (!qid && n->dbbuf_enabled) { |
8640 | 0 | stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED); |
8641 | 0 | } |
8642 | |
|
8643 | 0 | if (cq->tail == cq->head) { |
8644 | 0 | if (cq->irq_enabled && pending) { |
8645 | 0 | n->cq_pending--; |
8646 | 0 | } |
8647 | |
|
8648 | 0 | nvme_irq_deassert(n, cq); |
8649 | 0 | } else if (n->dbbuf_enabled) { |
8650 | 0 | nvme_update_cq_eventidx(cq); |
8651 | 0 | } |
8652 | 0 | } else { |
8653 | | /* Submission queue doorbell write */ |
8654 | |
|
8655 | 0 | uint16_t new_tail = val & 0xffff; |
8656 | 0 | NvmeSQueue *sq; |
8657 | |
|
8658 | 0 | qid = (addr - 0x1000) >> 3; |
8659 | 0 | if (unlikely(nvme_check_sqid(n, qid))) { |
8660 | 0 | NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq, |
8661 | 0 | "submission queue doorbell write" |
8662 | 0 | " for nonexistent queue," |
8663 | 0 | " sqid=%"PRIu32", ignoring", qid); |
8664 | |
|
8665 | 0 | if (n->outstanding_aers) { |
8666 | 0 | nvme_enqueue_event(n, NVME_AER_TYPE_ERROR, |
8667 | 0 | NVME_AER_INFO_ERR_INVALID_DB_REGISTER, |
8668 | 0 | NVME_LOG_ERROR_INFO); |
8669 | 0 | } |
8670 | |
|
8671 | 0 | return; |
8672 | 0 | } |
8673 | | |
8674 | 0 | sq = n->sq[qid]; |
8675 | 0 | if (unlikely(new_tail >= sq->size)) { |
8676 | 0 | NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail, |
8677 | 0 | "submission queue doorbell write value" |
8678 | 0 | " beyond queue size, sqid=%"PRIu32"," |
8679 | 0 | " new_tail=%"PRIu16", ignoring", |
8680 | 0 | qid, new_tail); |
8681 | |
|
8682 | 0 | if (n->outstanding_aers) { |
8683 | 0 | nvme_enqueue_event(n, NVME_AER_TYPE_ERROR, |
8684 | 0 | NVME_AER_INFO_ERR_INVALID_DB_VALUE, |
8685 | 0 | NVME_LOG_ERROR_INFO); |
8686 | 0 | } |
8687 | |
|
8688 | 0 | return; |
8689 | 0 | } |
8690 | | |
8691 | 0 | trace_pci_nvme_mmio_doorbell_sq(sq->sqid, new_tail); |
8692 | |
|
8693 | 0 | sq->tail = new_tail; |
8694 | 0 | if (!qid && n->dbbuf_enabled) { |
8695 | | /* |
8696 | | * The spec states "the host shall also update the controller's |
8697 | | * corresponding doorbell property to match the value of that entry |
8698 | | * in the Shadow Doorbell buffer." |
8699 | | * |
8700 | | * Since this context is currently a VM trap, we can safely enforce |
8701 | | * the requirement from the device side in case the host is |
8702 | | * misbehaving. |
8703 | | * |
8704 | | * Note, we shouldn't have to do this, but various drivers |
8705 | | * including ones that run on Linux, are not updating Admin Queues, |
8706 | | * so we can't trust reading it for an appropriate sq tail. |
8707 | | */ |
8708 | 0 | stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED); |
8709 | 0 | } |
8710 | |
|
8711 | 0 | qemu_bh_schedule(sq->bh); |
8712 | 0 | } |
8713 | 0 | } |
8714 | | |
8715 | | static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data, |
8716 | | unsigned size) |
8717 | 0 | { |
8718 | 0 | NvmeCtrl *n = (NvmeCtrl *)opaque; |
8719 | |
|
8720 | 0 | trace_pci_nvme_mmio_write(addr, data, size); |
8721 | |
|
8722 | 0 | if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs && |
8723 | 0 | addr != NVME_REG_CSTS) { |
8724 | 0 | trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size); |
8725 | 0 | return; |
8726 | 0 | } |
8727 | | |
8728 | 0 | if (addr < sizeof(n->bar)) { |
8729 | 0 | nvme_write_bar(n, addr, data, size); |
8730 | 0 | } else { |
8731 | 0 | nvme_process_db(n, addr, data); |
8732 | 0 | } |
8733 | 0 | } |
8734 | | |
8735 | | static const MemoryRegionOps nvme_mmio_ops = { |
8736 | | .read = nvme_mmio_read, |
8737 | | .write = nvme_mmio_write, |
8738 | | .endianness = DEVICE_LITTLE_ENDIAN, |
8739 | | .impl = { |
8740 | | .min_access_size = 2, |
8741 | | .max_access_size = 8, |
8742 | | }, |
8743 | | }; |
8744 | | |
8745 | | static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data, |
8746 | | unsigned size) |
8747 | 0 | { |
8748 | 0 | NvmeCtrl *n = (NvmeCtrl *)opaque; |
8749 | 0 | stn_le_p(&n->cmb.buf[addr], size, data); |
8750 | 0 | } |
8751 | | |
8752 | | static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size) |
8753 | 0 | { |
8754 | 0 | NvmeCtrl *n = (NvmeCtrl *)opaque; |
8755 | 0 | return ldn_le_p(&n->cmb.buf[addr], size); |
8756 | 0 | } |
8757 | | |
8758 | | static const MemoryRegionOps nvme_cmb_ops = { |
8759 | | .read = nvme_cmb_read, |
8760 | | .write = nvme_cmb_write, |
8761 | | .endianness = DEVICE_LITTLE_ENDIAN, |
8762 | | .impl = { |
8763 | | .min_access_size = 1, |
8764 | | .max_access_size = 8, |
8765 | | }, |
8766 | | }; |
8767 | | |
8768 | | static bool nvme_check_params(NvmeCtrl *n, Error **errp) |
8769 | 0 | { |
8770 | 0 | NvmeParams *params = &n->params; |
8771 | |
|
8772 | 0 | if (params->num_queues) { |
8773 | 0 | warn_report("num_queues is deprecated; please use max_ioqpairs " |
8774 | 0 | "instead"); |
8775 | |
|
8776 | 0 | params->max_ioqpairs = params->num_queues - 1; |
8777 | 0 | } |
8778 | |
|
8779 | 0 | n->num_queues = params->max_ioqpairs + 1; |
8780 | |
|
8781 | 0 | if (n->namespace.blkconf.blk && n->subsys) { |
8782 | 0 | error_setg(errp, "subsystem support is unavailable with legacy " |
8783 | 0 | "namespace ('drive' property)"); |
8784 | 0 | return false; |
8785 | 0 | } |
8786 | | |
8787 | 0 | if (params->max_ioqpairs < 1 || |
8788 | 0 | params->max_ioqpairs > NVME_MAX_IOQPAIRS) { |
8789 | 0 | error_setg(errp, "max_ioqpairs must be between 1 and %d", |
8790 | 0 | NVME_MAX_IOQPAIRS); |
8791 | 0 | return false; |
8792 | 0 | } |
8793 | | |
8794 | 0 | if (params->msix_qsize < 1 || |
8795 | 0 | params->msix_qsize > PCI_MSIX_FLAGS_QSIZE + 1) { |
8796 | 0 | error_setg(errp, "msix_qsize must be between 1 and %d", |
8797 | 0 | PCI_MSIX_FLAGS_QSIZE + 1); |
8798 | 0 | return false; |
8799 | 0 | } |
8800 | | |
8801 | 0 | if (!params->serial) { |
8802 | 0 | error_setg(errp, "serial property not set"); |
8803 | 0 | return false; |
8804 | 0 | } |
8805 | 0 | if (strlen(params->serial) > NVME_ID_CTRL_SN_MAX_LEN) { |
8806 | 0 | error_setg(errp, "'serial' parameter '%s' can be at most '%d' characters", |
8807 | 0 | params->serial, NVME_ID_CTRL_SN_MAX_LEN); |
8808 | 0 | return false; |
8809 | 0 | } |
8810 | | |
8811 | 0 | if (params->model && |
8812 | 0 | strlen(params->model) > NVME_ID_CTRL_MN_MAX_LEN) { |
8813 | 0 | error_setg(errp, "'model' parameter '%s' can be at most '%d' characters", |
8814 | 0 | params->model, NVME_ID_CTRL_MN_MAX_LEN); |
8815 | 0 | return false; |
8816 | 0 | } |
8817 | | |
8818 | 0 | if (params->firmware_version && |
8819 | 0 | strlen(params->firmware_version) > NVME_ID_CTRL_FR_MAX_LEN) { |
8820 | 0 | error_setg(errp, "'firmware-version' parameter '%s' can be at most '%d' characters", |
8821 | 0 | params->firmware_version, NVME_ID_CTRL_FR_MAX_LEN); |
8822 | 0 | return false; |
8823 | 0 | } |
8824 | | |
8825 | 0 | if (params->mqes < 1) { |
8826 | 0 | error_setg(errp, "mqes property cannot be less than 1"); |
8827 | 0 | return false; |
8828 | 0 | } |
8829 | | |
8830 | 0 | if (n->pmr.dev) { |
8831 | 0 | if (params->msix_exclusive_bar) { |
8832 | 0 | error_setg(errp, "not enough BARs available to enable PMR"); |
8833 | 0 | return false; |
8834 | 0 | } |
8835 | | |
8836 | 0 | if (host_memory_backend_is_mapped(n->pmr.dev)) { |
8837 | 0 | error_setg(errp, "can't use already busy memdev: %s", |
8838 | 0 | object_get_canonical_path_component(OBJECT(n->pmr.dev))); |
8839 | 0 | return false; |
8840 | 0 | } |
8841 | | |
8842 | 0 | if (!is_power_of_2(n->pmr.dev->size)) { |
8843 | 0 | error_setg(errp, "pmr backend size needs to be power of 2 in size"); |
8844 | 0 | return false; |
8845 | 0 | } |
8846 | | |
8847 | 0 | host_memory_backend_set_mapped(n->pmr.dev, true); |
8848 | 0 | } |
8849 | | |
8850 | 0 | if (!n->params.mdts || ((1 << n->params.mdts) + 1) > IOV_MAX) { |
8851 | 0 | error_setg(errp, "mdts exceeds IOV_MAX"); |
8852 | 0 | return false; |
8853 | 0 | } |
8854 | | |
8855 | 0 | if (n->params.zasl > n->params.mdts) { |
8856 | 0 | error_setg(errp, "zoned.zasl (Zone Append Size Limit) must be less " |
8857 | 0 | "than or equal to mdts (Maximum Data Transfer Size)"); |
8858 | 0 | return false; |
8859 | 0 | } |
8860 | | |
8861 | 0 | if (!n->params.vsl) { |
8862 | 0 | error_setg(errp, "vsl must be non-zero"); |
8863 | 0 | return false; |
8864 | 0 | } |
8865 | | |
8866 | 0 | if (params->sriov_max_vfs) { |
8867 | 0 | if (!n->subsys) { |
8868 | 0 | error_setg(errp, "subsystem is required for the use of SR-IOV"); |
8869 | 0 | return false; |
8870 | 0 | } |
8871 | | |
8872 | 0 | if (params->cmb_size_mb) { |
8873 | 0 | error_setg(errp, "CMB is not supported with SR-IOV"); |
8874 | 0 | return false; |
8875 | 0 | } |
8876 | | |
8877 | 0 | if (n->pmr.dev) { |
8878 | 0 | error_setg(errp, "PMR is not supported with SR-IOV"); |
8879 | 0 | return false; |
8880 | 0 | } |
8881 | | |
8882 | 0 | if (!params->sriov_vq_flexible || !params->sriov_vi_flexible) { |
8883 | 0 | error_setg(errp, "both sriov_vq_flexible and sriov_vi_flexible" |
8884 | 0 | " must be set for the use of SR-IOV"); |
8885 | 0 | return false; |
8886 | 0 | } |
8887 | | |
8888 | 0 | if (params->sriov_vq_flexible < params->sriov_max_vfs * 2) { |
8889 | 0 | error_setg(errp, "sriov_vq_flexible must be greater than or equal" |
8890 | 0 | " to %d (sriov_max_vfs * 2)", params->sriov_max_vfs * 2); |
8891 | 0 | return false; |
8892 | 0 | } |
8893 | | |
8894 | 0 | if (params->max_ioqpairs < params->sriov_vq_flexible + 2) { |
8895 | 0 | error_setg(errp, "(max_ioqpairs - sriov_vq_flexible) must be" |
8896 | 0 | " greater than or equal to 2"); |
8897 | 0 | return false; |
8898 | 0 | } |
8899 | | |
8900 | 0 | if (params->sriov_vi_flexible < params->sriov_max_vfs) { |
8901 | 0 | error_setg(errp, "sriov_vi_flexible must be greater than or equal" |
8902 | 0 | " to %d (sriov_max_vfs)", params->sriov_max_vfs); |
8903 | 0 | return false; |
8904 | 0 | } |
8905 | | |
8906 | 0 | if (params->msix_qsize < params->sriov_vi_flexible + 1) { |
8907 | 0 | error_setg(errp, "(msix_qsize - sriov_vi_flexible) must be" |
8908 | 0 | " greater than or equal to 1"); |
8909 | 0 | return false; |
8910 | 0 | } |
8911 | | |
8912 | 0 | if (params->sriov_max_vi_per_vf && |
8913 | 0 | (params->sriov_max_vi_per_vf - 1) % NVME_VF_RES_GRANULARITY) { |
8914 | 0 | error_setg(errp, "sriov_max_vi_per_vf must meet:" |
8915 | 0 | " (sriov_max_vi_per_vf - 1) %% %d == 0 and" |
8916 | 0 | " sriov_max_vi_per_vf >= 1", NVME_VF_RES_GRANULARITY); |
8917 | 0 | return false; |
8918 | 0 | } |
8919 | | |
8920 | 0 | if (params->sriov_max_vq_per_vf && |
8921 | 0 | (params->sriov_max_vq_per_vf < 2 || |
8922 | 0 | (params->sriov_max_vq_per_vf - 1) % NVME_VF_RES_GRANULARITY)) { |
8923 | 0 | error_setg(errp, "sriov_max_vq_per_vf must meet:" |
8924 | 0 | " (sriov_max_vq_per_vf - 1) %% %d == 0 and" |
8925 | 0 | " sriov_max_vq_per_vf >= 2", NVME_VF_RES_GRANULARITY); |
8926 | 0 | return false; |
8927 | 0 | } |
8928 | 0 | } |
8929 | | |
8930 | 0 | return true; |
8931 | 0 | } |
8932 | | |
8933 | | static void nvme_init_state(NvmeCtrl *n) |
8934 | 0 | { |
8935 | 0 | NvmePriCtrlCap *cap = &n->pri_ctrl_cap; |
8936 | 0 | NvmeSecCtrlEntry *list = n->sec_ctrl_list; |
8937 | 0 | NvmeSecCtrlEntry *sctrl; |
8938 | 0 | PCIDevice *pci = PCI_DEVICE(n); |
8939 | 0 | NvmeIdCtrl *id = &n->id_ctrl; |
8940 | 0 | uint8_t max_vfs; |
8941 | 0 | int i; |
8942 | |
|
8943 | 0 | if (pci_is_vf(pci)) { |
8944 | 0 | sctrl = nvme_sctrl(n); |
8945 | 0 | max_vfs = 0; |
8946 | 0 | n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0; |
8947 | 0 | n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1; |
8948 | 0 | } else { |
8949 | 0 | max_vfs = n->params.sriov_max_vfs; |
8950 | 0 | n->conf_ioqpairs = n->params.max_ioqpairs; |
8951 | 0 | n->conf_msix_qsize = n->params.msix_qsize; |
8952 | 0 | } |
8953 | |
|
8954 | 0 | n->sq = g_new0(NvmeSQueue *, n->num_queues); |
8955 | 0 | n->cq = g_new0(NvmeCQueue *, n->num_queues); |
8956 | 0 | n->temperature = NVME_TEMPERATURE; |
8957 | 0 | n->features.temp_thresh_hi = NVME_TEMPERATURE_WARNING; |
8958 | 0 | n->starttime_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL); |
8959 | 0 | n->aer_reqs = g_new0(NvmeRequest *, n->params.aerl + 1); |
8960 | 0 | QTAILQ_INIT(&n->aer_queue); |
8961 | |
|
8962 | 0 | n->nr_sec_ctrls = max_vfs; |
8963 | 0 | for (i = 0; i < max_vfs; i++) { |
8964 | 0 | sctrl = &list[i]; |
8965 | 0 | sctrl->pcid = cpu_to_le16(n->cntlid); |
8966 | 0 | sctrl->vfn = cpu_to_le16(i + 1); |
8967 | 0 | } |
8968 | |
|
8969 | 0 | n->spdm_socket = -1; |
8970 | |
|
8971 | 0 | cap->cntlid = cpu_to_le16(n->cntlid); |
8972 | 0 | cap->crt = NVME_CRT_VQ | NVME_CRT_VI; |
8973 | |
|
8974 | 0 | if (pci_is_vf(pci)) { |
8975 | 0 | cap->vqprt = cpu_to_le16(1 + n->conf_ioqpairs); |
8976 | 0 | } else { |
8977 | 0 | cap->vqprt = cpu_to_le16(1 + n->params.max_ioqpairs - |
8978 | 0 | n->params.sriov_vq_flexible); |
8979 | 0 | cap->vqfrt = cpu_to_le32(n->params.sriov_vq_flexible); |
8980 | 0 | cap->vqrfap = cap->vqfrt; |
8981 | 0 | cap->vqgran = cpu_to_le16(NVME_VF_RES_GRANULARITY); |
8982 | 0 | cap->vqfrsm = n->params.sriov_max_vq_per_vf ? |
8983 | 0 | cpu_to_le16(n->params.sriov_max_vq_per_vf) : |
8984 | 0 | cap->vqfrt / MAX(max_vfs, 1); |
8985 | 0 | } |
8986 | |
|
8987 | 0 | if (pci_is_vf(pci)) { |
8988 | 0 | cap->viprt = cpu_to_le16(n->conf_msix_qsize); |
8989 | 0 | } else { |
8990 | 0 | cap->viprt = cpu_to_le16(n->params.msix_qsize - |
8991 | 0 | n->params.sriov_vi_flexible); |
8992 | 0 | cap->vifrt = cpu_to_le32(n->params.sriov_vi_flexible); |
8993 | 0 | cap->virfap = cap->vifrt; |
8994 | 0 | cap->vigran = cpu_to_le16(NVME_VF_RES_GRANULARITY); |
8995 | 0 | cap->vifrsm = n->params.sriov_max_vi_per_vf ? |
8996 | 0 | cpu_to_le16(n->params.sriov_max_vi_per_vf) : |
8997 | 0 | cap->vifrt / MAX(max_vfs, 1); |
8998 | 0 | } |
8999 | | |
9000 | | /* Atomic Write */ |
9001 | 0 | id->awun = cpu_to_le16(n->params.atomic_awun); |
9002 | 0 | id->awupf = cpu_to_le16(n->params.atomic_awupf); |
9003 | 0 | n->dn = n->params.atomic_dn; |
9004 | |
|
9005 | 0 | if (id->awun || id->awupf) { |
9006 | 0 | if (id->awupf > id->awun) { |
9007 | 0 | id->awupf = 0; |
9008 | 0 | } |
9009 | |
|
9010 | 0 | nvme_atomic_configure_max_write_size(n->dn, n->params.atomic_awun, |
9011 | 0 | n->params.atomic_awupf, |
9012 | 0 | &n->atomic); |
9013 | 0 | } |
9014 | 0 | } |
9015 | | |
9016 | | static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev) |
9017 | 0 | { |
9018 | 0 | uint64_t cmb_size = n->params.cmb_size_mb * MiB; |
9019 | 0 | uint64_t cap = ldq_le_p(&n->bar.cap); |
9020 | |
|
9021 | 0 | n->cmb.buf = g_malloc0(cmb_size); |
9022 | 0 | memory_region_init_io(&n->cmb.mem, OBJECT(n), &nvme_cmb_ops, n, |
9023 | 0 | "nvme-cmb", cmb_size); |
9024 | 0 | pci_register_bar(pci_dev, NVME_CMB_BIR, |
9025 | 0 | PCI_BASE_ADDRESS_SPACE_MEMORY | |
9026 | 0 | PCI_BASE_ADDRESS_MEM_TYPE_64 | |
9027 | 0 | PCI_BASE_ADDRESS_MEM_PREFETCH, &n->cmb.mem); |
9028 | |
|
9029 | 0 | NVME_CAP_SET_CMBS(cap, 1); |
9030 | 0 | stq_le_p(&n->bar.cap, cap); |
9031 | |
|
9032 | 0 | if (n->params.legacy_cmb) { |
9033 | 0 | nvme_cmb_enable_regs(n); |
9034 | 0 | n->cmb.cmse = true; |
9035 | 0 | } |
9036 | 0 | } |
9037 | | |
9038 | | static bool nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp) |
9039 | 0 | { |
9040 | 0 | uint32_t pmrcap = ldl_le_p(&n->bar.pmrcap); |
9041 | |
|
9042 | 0 | if (memory_region_size(&n->pmr.dev->mr) < 16) { |
9043 | 0 | error_setg(errp, "PMR device must have at least 16 bytes"); |
9044 | 0 | return false; |
9045 | 0 | } |
9046 | | |
9047 | 0 | NVME_PMRCAP_SET_RDS(pmrcap, 1); |
9048 | 0 | NVME_PMRCAP_SET_WDS(pmrcap, 1); |
9049 | 0 | NVME_PMRCAP_SET_BIR(pmrcap, NVME_PMR_BIR); |
9050 | | /* Turn on bit 1 support */ |
9051 | 0 | NVME_PMRCAP_SET_PMRWBM(pmrcap, 0x02); |
9052 | 0 | NVME_PMRCAP_SET_CMSS(pmrcap, 1); |
9053 | 0 | stl_le_p(&n->bar.pmrcap, pmrcap); |
9054 | |
|
9055 | 0 | pci_register_bar(pci_dev, NVME_PMR_BIR, |
9056 | 0 | PCI_BASE_ADDRESS_SPACE_MEMORY | |
9057 | 0 | PCI_BASE_ADDRESS_MEM_TYPE_64 | |
9058 | 0 | PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmr.dev->mr); |
9059 | |
|
9060 | 0 | memory_region_set_enabled(&n->pmr.dev->mr, false); |
9061 | |
|
9062 | 0 | return true; |
9063 | 0 | } |
9064 | | |
9065 | | static uint64_t nvme_mbar_size(unsigned total_queues, unsigned total_irqs, |
9066 | | unsigned *msix_table_offset, |
9067 | | unsigned *msix_pba_offset) |
9068 | 0 | { |
9069 | 0 | uint64_t bar_size, msix_table_size; |
9070 | |
|
9071 | 0 | bar_size = sizeof(NvmeBar) + 2 * total_queues * NVME_DB_SIZE; |
9072 | |
|
9073 | 0 | if (total_irqs == 0) { |
9074 | 0 | goto out; |
9075 | 0 | } |
9076 | | |
9077 | 0 | bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB); |
9078 | |
|
9079 | 0 | if (msix_table_offset) { |
9080 | 0 | *msix_table_offset = bar_size; |
9081 | 0 | } |
9082 | |
|
9083 | 0 | msix_table_size = PCI_MSIX_ENTRY_SIZE * total_irqs; |
9084 | 0 | bar_size += msix_table_size; |
9085 | 0 | bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB); |
9086 | |
|
9087 | 0 | if (msix_pba_offset) { |
9088 | 0 | *msix_pba_offset = bar_size; |
9089 | 0 | } |
9090 | |
|
9091 | 0 | bar_size += QEMU_ALIGN_UP(total_irqs, 64) / 8; |
9092 | |
|
9093 | 0 | out: |
9094 | 0 | return pow2ceil(bar_size); |
9095 | 0 | } |
9096 | | |
9097 | | static bool nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset, |
9098 | | Error **errp) |
9099 | 0 | { |
9100 | 0 | uint16_t vf_dev_id = n->params.use_intel_id ? |
9101 | 0 | PCI_DEVICE_ID_INTEL_NVME : PCI_DEVICE_ID_REDHAT_NVME; |
9102 | 0 | NvmePriCtrlCap *cap = &n->pri_ctrl_cap; |
9103 | 0 | uint64_t bar_size = nvme_mbar_size(le16_to_cpu(cap->vqfrsm), |
9104 | 0 | le16_to_cpu(cap->vifrsm), |
9105 | 0 | NULL, NULL); |
9106 | |
|
9107 | 0 | if (!pcie_sriov_pf_init(pci_dev, offset, "nvme", vf_dev_id, |
9108 | 0 | n->params.sriov_max_vfs, n->params.sriov_max_vfs, |
9109 | 0 | NVME_VF_OFFSET, NVME_VF_STRIDE, errp)) { |
9110 | 0 | return false; |
9111 | 0 | } |
9112 | | |
9113 | 0 | pcie_sriov_pf_init_vf_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY | |
9114 | 0 | PCI_BASE_ADDRESS_MEM_TYPE_64, bar_size); |
9115 | |
|
9116 | 0 | return true; |
9117 | 0 | } |
9118 | | |
9119 | | static int nvme_add_pm_capability(PCIDevice *pci_dev, uint8_t offset) |
9120 | 0 | { |
9121 | 0 | Error *err = NULL; |
9122 | 0 | int ret; |
9123 | |
|
9124 | 0 | ret = pci_pm_init(pci_dev, offset, &err); |
9125 | 0 | if (err) { |
9126 | 0 | error_report_err(err); |
9127 | 0 | return ret; |
9128 | 0 | } |
9129 | | |
9130 | 0 | pci_set_word(pci_dev->config + offset + PCI_PM_PMC, |
9131 | 0 | PCI_PM_CAP_VER_1_2); |
9132 | 0 | pci_set_word(pci_dev->config + offset + PCI_PM_CTRL, |
9133 | 0 | PCI_PM_CTRL_NO_SOFT_RESET); |
9134 | 0 | pci_set_word(pci_dev->wmask + offset + PCI_PM_CTRL, |
9135 | 0 | PCI_PM_CTRL_STATE_MASK); |
9136 | |
|
9137 | 0 | return 0; |
9138 | 0 | } |
9139 | | |
9140 | | static bool pcie_doe_spdm_rsp(DOECap *doe_cap) |
9141 | 0 | { |
9142 | 0 | void *req = pcie_doe_get_write_mbox_ptr(doe_cap); |
9143 | 0 | uint32_t req_len = pcie_doe_get_obj_len(req) * 4; |
9144 | 0 | void *rsp = doe_cap->read_mbox; |
9145 | 0 | uint32_t rsp_len = SPDM_SOCKET_MAX_MESSAGE_BUFFER_SIZE; |
9146 | |
|
9147 | 0 | uint32_t recvd = spdm_socket_rsp(doe_cap->spdm_socket, |
9148 | 0 | SPDM_TRANSPORT_TYPE_DOE, |
9149 | 0 | req, req_len, rsp, rsp_len); |
9150 | 0 | doe_cap->read_mbox_len += DIV_ROUND_UP(recvd, 4); |
9151 | |
|
9152 | 0 | return recvd != 0; |
9153 | 0 | } |
9154 | | |
9155 | | static DOEProtocol doe_spdm_prot[] = { |
9156 | | { PCI_VENDOR_ID_PCI_SIG, PCI_SIG_DOE_CMA, pcie_doe_spdm_rsp }, |
9157 | | { PCI_VENDOR_ID_PCI_SIG, PCI_SIG_DOE_SECURED_CMA, pcie_doe_spdm_rsp }, |
9158 | | { } |
9159 | | }; |
9160 | | |
9161 | | static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp) |
9162 | 0 | { |
9163 | 0 | ERRP_GUARD(); |
9164 | 0 | uint8_t *pci_conf = pci_dev->config; |
9165 | 0 | uint64_t bar_size; |
9166 | 0 | unsigned msix_table_offset = 0, msix_pba_offset = 0; |
9167 | 0 | unsigned nr_vectors; |
9168 | 0 | int ret; |
9169 | |
|
9170 | 0 | pci_conf[PCI_INTERRUPT_PIN] = pci_is_vf(pci_dev) ? 0 : 1; |
9171 | 0 | pci_config_set_prog_interface(pci_conf, 0x2); |
9172 | |
|
9173 | 0 | if (n->params.use_intel_id) { |
9174 | 0 | pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL); |
9175 | 0 | pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_INTEL_NVME); |
9176 | 0 | } else { |
9177 | 0 | pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_REDHAT); |
9178 | 0 | pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_REDHAT_NVME); |
9179 | 0 | } |
9180 | |
|
9181 | 0 | pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS); |
9182 | 0 | nvme_add_pm_capability(pci_dev, 0x60); |
9183 | 0 | pcie_endpoint_cap_init(pci_dev, 0x80); |
9184 | 0 | pcie_cap_flr_init(pci_dev); |
9185 | 0 | if (n->params.sriov_max_vfs) { |
9186 | 0 | pcie_ari_init(pci_dev, 0x100); |
9187 | 0 | } |
9188 | |
|
9189 | 0 | if (n->params.msix_exclusive_bar && !pci_is_vf(pci_dev)) { |
9190 | 0 | bar_size = nvme_mbar_size(n->num_queues, 0, NULL, NULL); |
9191 | 0 | memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme", |
9192 | 0 | bar_size); |
9193 | 0 | pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY | |
9194 | 0 | PCI_BASE_ADDRESS_MEM_TYPE_64, &n->iomem); |
9195 | 0 | ret = msix_init_exclusive_bar(pci_dev, n->params.msix_qsize, 4, errp); |
9196 | 0 | } else { |
9197 | 0 | assert(n->params.msix_qsize >= 1); |
9198 | | |
9199 | | /* add one to max_ioqpairs to account for the admin queue pair */ |
9200 | 0 | if (!pci_is_vf(pci_dev)) { |
9201 | 0 | nr_vectors = n->params.msix_qsize; |
9202 | 0 | bar_size = nvme_mbar_size(n->num_queues, |
9203 | 0 | nr_vectors, &msix_table_offset, |
9204 | 0 | &msix_pba_offset); |
9205 | 0 | } else { |
9206 | 0 | NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev)); |
9207 | 0 | NvmePriCtrlCap *cap = &pn->pri_ctrl_cap; |
9208 | |
|
9209 | 0 | nr_vectors = le16_to_cpu(cap->vifrsm); |
9210 | 0 | bar_size = nvme_mbar_size(le16_to_cpu(cap->vqfrsm), nr_vectors, |
9211 | 0 | &msix_table_offset, &msix_pba_offset); |
9212 | 0 | } |
9213 | |
|
9214 | 0 | memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size); |
9215 | 0 | memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme", |
9216 | 0 | msix_table_offset); |
9217 | 0 | memory_region_add_subregion(&n->bar0, 0, &n->iomem); |
9218 | |
|
9219 | 0 | pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY | |
9220 | 0 | PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0); |
9221 | |
|
9222 | 0 | ret = msix_init(pci_dev, nr_vectors, |
9223 | 0 | &n->bar0, 0, msix_table_offset, |
9224 | 0 | &n->bar0, 0, msix_pba_offset, 0, errp); |
9225 | 0 | } |
9226 | |
|
9227 | 0 | if (ret == -ENOTSUP) { |
9228 | | /* report that msix is not supported, but do not error out */ |
9229 | 0 | warn_report_err(*errp); |
9230 | 0 | *errp = NULL; |
9231 | 0 | } else if (ret < 0) { |
9232 | | /* propagate error to caller */ |
9233 | 0 | return false; |
9234 | 0 | } |
9235 | | |
9236 | 0 | if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs && |
9237 | 0 | !nvme_init_sriov(n, pci_dev, 0x120, errp)) { |
9238 | 0 | return false; |
9239 | 0 | } |
9240 | | |
9241 | 0 | nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize); |
9242 | |
|
9243 | 0 | pcie_cap_deverr_init(pci_dev); |
9244 | | |
9245 | | /* SPDM Initialisation */ |
9246 | 0 | if (pci_dev->spdm_port) { |
9247 | 0 | uint16_t doe_offset = PCI_CONFIG_SPACE_SIZE; |
9248 | |
|
9249 | 0 | switch (pci_dev->spdm_trans) { |
9250 | 0 | case SPDM_TRANSPORT_TYPE_DOE: |
9251 | 0 | if (n->params.sriov_max_vfs) { |
9252 | 0 | doe_offset += PCI_ARI_SIZEOF; |
9253 | 0 | } |
9254 | |
|
9255 | 0 | pcie_doe_init(pci_dev, &pci_dev->doe_spdm, doe_offset, |
9256 | 0 | doe_spdm_prot, true, 0); |
9257 | |
|
9258 | 0 | pci_dev->doe_spdm.spdm_socket = |
9259 | 0 | spdm_socket_connect(pci_dev->spdm_port, errp); |
9260 | |
|
9261 | 0 | if (pci_dev->doe_spdm.spdm_socket < 0) { |
9262 | 0 | return false; |
9263 | 0 | } |
9264 | 0 | break; |
9265 | 0 | case SPDM_TRANSPORT_TYPE_NVME: |
9266 | 0 | n->spdm_socket = spdm_socket_connect(pci_dev->spdm_port, errp); |
9267 | 0 | if (n->spdm_socket < 0) { |
9268 | 0 | return false; |
9269 | 0 | } |
9270 | 0 | break; |
9271 | 0 | default: |
9272 | 0 | return false; |
9273 | 0 | } |
9274 | 0 | } |
9275 | | |
9276 | 0 | if (n->params.cmb_size_mb) { |
9277 | 0 | nvme_init_cmb(n, pci_dev); |
9278 | 0 | } |
9279 | |
|
9280 | 0 | if (n->pmr.dev) { |
9281 | 0 | if (!nvme_init_pmr(n, pci_dev, errp)) { |
9282 | 0 | return false; |
9283 | 0 | } |
9284 | 0 | } |
9285 | | |
9286 | 0 | return true; |
9287 | 0 | } |
9288 | | |
9289 | | static void nvme_init_subnqn(NvmeCtrl *n) |
9290 | 0 | { |
9291 | 0 | NvmeSubsystem *subsys = n->subsys; |
9292 | 0 | NvmeIdCtrl *id = &n->id_ctrl; |
9293 | |
|
9294 | 0 | if (!subsys) { |
9295 | 0 | snprintf((char *)id->subnqn, sizeof(id->subnqn), |
9296 | 0 | "nqn.2019-08.org.qemu:%s", n->params.serial); |
9297 | 0 | } else { |
9298 | 0 | pstrcpy((char *)id->subnqn, sizeof(id->subnqn), (char*)subsys->subnqn); |
9299 | 0 | } |
9300 | 0 | } |
9301 | | |
9302 | | static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) |
9303 | 0 | { |
9304 | 0 | NvmeIdCtrl *id = &n->id_ctrl; |
9305 | 0 | uint8_t *pci_conf = pci_dev->config; |
9306 | 0 | uint64_t cap = ldq_le_p(&n->bar.cap); |
9307 | 0 | NvmeSecCtrlEntry *sctrl = nvme_sctrl(n); |
9308 | 0 | uint32_t ctratt = le32_to_cpu(id->ctratt); |
9309 | 0 | uint16_t oacs; |
9310 | |
|
9311 | 0 | memcpy(n->cse.acs, nvme_cse_acs_default, sizeof(n->cse.acs)); |
9312 | 0 | memcpy(n->cse.iocs.nvm, nvme_cse_iocs_nvm_default, sizeof(n->cse.iocs.nvm)); |
9313 | 0 | memcpy(n->cse.iocs.zoned, nvme_cse_iocs_zoned_default, |
9314 | 0 | sizeof(n->cse.iocs.zoned)); |
9315 | |
|
9316 | 0 | id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID)); |
9317 | 0 | id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID)); |
9318 | 0 | strpadcpy((char *)id->mn, sizeof(id->mn), |
9319 | 0 | n->params.model ? n->params.model : "QEMU NVMe Ctrl", ' '); |
9320 | 0 | strpadcpy((char *)id->fr, sizeof(id->fr), |
9321 | 0 | n->params.firmware_version ? n->params.firmware_version : QEMU_VERSION, ' '); |
9322 | 0 | strpadcpy((char *)id->sn, sizeof(id->sn), n->params.serial, ' '); |
9323 | |
|
9324 | 0 | id->cntlid = cpu_to_le16(n->cntlid); |
9325 | |
|
9326 | 0 | id->oaes = cpu_to_le32(NVME_OAES_NS_ATTR); |
9327 | |
|
9328 | 0 | ctratt |= NVME_CTRATT_ELBAS; |
9329 | 0 | if (n->params.ctratt.mem) { |
9330 | 0 | ctratt |= NVME_CTRATT_MEM; |
9331 | 0 | } |
9332 | 0 | id->ctratt = cpu_to_le32(ctratt); |
9333 | |
|
9334 | 0 | id->rab = 6; |
9335 | |
|
9336 | 0 | if (n->params.use_intel_id) { |
9337 | 0 | id->ieee[0] = 0xb3; |
9338 | 0 | id->ieee[1] = 0x02; |
9339 | 0 | id->ieee[2] = 0x00; |
9340 | 0 | } else { |
9341 | 0 | id->ieee[0] = 0x00; |
9342 | 0 | id->ieee[1] = 0x54; |
9343 | 0 | id->ieee[2] = 0x52; |
9344 | 0 | } |
9345 | |
|
9346 | 0 | id->mdts = n->params.mdts; |
9347 | 0 | id->ver = cpu_to_le32(NVME_SPEC_VER); |
9348 | |
|
9349 | 0 | oacs = NVME_OACS_NMS | NVME_OACS_FORMAT | NVME_OACS_DIRECTIVES | |
9350 | 0 | NVME_OACS_SECURITY; |
9351 | |
|
9352 | 0 | if (n->params.dbcs) { |
9353 | 0 | oacs |= NVME_OACS_DBCS; |
9354 | |
|
9355 | 0 | n->cse.acs[NVME_ADM_CMD_DBBUF_CONFIG] = NVME_CMD_EFF_CSUPP; |
9356 | 0 | } |
9357 | |
|
9358 | 0 | if (n->params.sriov_max_vfs) { |
9359 | 0 | oacs |= NVME_OACS_VMS; |
9360 | |
|
9361 | 0 | n->cse.acs[NVME_ADM_CMD_VIRT_MNGMT] = NVME_CMD_EFF_CSUPP; |
9362 | 0 | } |
9363 | |
|
9364 | 0 | id->oacs = cpu_to_le16(oacs); |
9365 | |
|
9366 | 0 | id->cntrltype = 0x1; |
9367 | | |
9368 | | /* |
9369 | | * Because the controller always completes the Abort command immediately, |
9370 | | * there can never be more than one concurrently executing Abort command, |
9371 | | * so this value is never used for anything. Note that there can easily be |
9372 | | * many Abort commands in the queues, but they are not considered |
9373 | | * "executing" until processed by nvme_abort. |
9374 | | * |
9375 | | * The specification recommends a value of 3 for Abort Command Limit (four |
9376 | | * concurrently outstanding Abort commands), so lets use that though it is |
9377 | | * inconsequential. |
9378 | | */ |
9379 | 0 | id->acl = 3; |
9380 | 0 | id->aerl = n->params.aerl; |
9381 | 0 | id->frmw = (NVME_NUM_FW_SLOTS << 1) | NVME_FRMW_SLOT1_RO; |
9382 | 0 | id->lpa = NVME_LPA_NS_SMART | NVME_LPA_CSE | NVME_LPA_EXTENDED; |
9383 | | |
9384 | | /* recommended default value (~70 C) */ |
9385 | 0 | id->wctemp = cpu_to_le16(NVME_TEMPERATURE_WARNING); |
9386 | 0 | id->cctemp = cpu_to_le16(NVME_TEMPERATURE_CRITICAL); |
9387 | |
|
9388 | 0 | id->sqes = (NVME_SQES << 4) | NVME_SQES; |
9389 | 0 | id->cqes = (NVME_CQES << 4) | NVME_CQES; |
9390 | 0 | id->nn = cpu_to_le32(NVME_MAX_NAMESPACES); |
9391 | 0 | id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROES | NVME_ONCS_TIMESTAMP | |
9392 | 0 | NVME_ONCS_FEATURES | NVME_ONCS_DSM | |
9393 | 0 | NVME_ONCS_COMPARE | NVME_ONCS_COPY | |
9394 | 0 | NVME_ONCS_NVMCSA | NVME_ONCS_NVMAFC); |
9395 | | |
9396 | | /* |
9397 | | * NOTE: If this device ever supports a command set that does NOT use 0x0 |
9398 | | * as a Flush-equivalent operation, support for the broadcast NSID in Flush |
9399 | | * should probably be removed. |
9400 | | * |
9401 | | * See comment in nvme_io_cmd. |
9402 | | */ |
9403 | 0 | id->vwc = NVME_VWC_NSID_BROADCAST_SUPPORT | NVME_VWC_PRESENT; |
9404 | |
|
9405 | 0 | id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1 | |
9406 | 0 | NVME_OCFS_COPY_FORMAT_2 | NVME_OCFS_COPY_FORMAT_3); |
9407 | 0 | id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN | |
9408 | 0 | NVME_CTRL_SGLS_MPTR_SGL); |
9409 | |
|
9410 | 0 | nvme_init_subnqn(n); |
9411 | |
|
9412 | 0 | id->psd[0].mp = cpu_to_le16(0x9c4); |
9413 | 0 | id->psd[0].enlat = cpu_to_le32(0x10); |
9414 | 0 | id->psd[0].exlat = cpu_to_le32(0x4); |
9415 | |
|
9416 | 0 | NVME_CAP_SET_MQES(cap, n->params.mqes); |
9417 | 0 | NVME_CAP_SET_CQR(cap, 1); |
9418 | 0 | NVME_CAP_SET_TO(cap, 0xf); |
9419 | 0 | NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_NCSS); |
9420 | 0 | NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_IOCSS); |
9421 | 0 | NVME_CAP_SET_MPSMAX(cap, 4); |
9422 | 0 | NVME_CAP_SET_CMBS(cap, n->params.cmb_size_mb ? 1 : 0); |
9423 | 0 | NVME_CAP_SET_PMRS(cap, n->pmr.dev ? 1 : 0); |
9424 | 0 | stq_le_p(&n->bar.cap, cap); |
9425 | |
|
9426 | 0 | stl_le_p(&n->bar.vs, NVME_SPEC_VER); |
9427 | 0 | n->bar.intmc = n->bar.intms = 0; |
9428 | |
|
9429 | 0 | if (pci_is_vf(pci_dev) && !sctrl->scs) { |
9430 | 0 | stl_le_p(&n->bar.csts, NVME_CSTS_FAILED); |
9431 | 0 | } |
9432 | 0 | } |
9433 | | |
9434 | | static bool nvme_set_migration_blockers(NvmeCtrl *n, PCIDevice *pci_dev, |
9435 | | Error **errp) |
9436 | 0 | { |
9437 | 0 | uint64_t unsupported_cap, cap = ldq_le_p(&n->bar.cap); |
9438 | 0 | g_autoptr(GPtrArray) blocker_features = g_ptr_array_new(); |
9439 | 0 | bool adm_cmd_security_checked = false; |
9440 | 0 | bool cmd_io_mgmt_checked = false; |
9441 | 0 | bool cmd_zone_checked = false; |
9442 | | |
9443 | | /* |
9444 | | * Idea of this function is simple, we iterate over all Command Sets and |
9445 | | * for each supported command we provide a special handling logic to |
9446 | | * determine if we should block migration or not. |
9447 | | * |
9448 | | * For instance, we have NVME_ADM_CMD_NS_ATTACHMENT and it is always |
9449 | | * available to the guest, but if there is only 1 namespace, then it is |
9450 | | * safe to allow migration, but if there are more, then we need to block |
9451 | | * migration because we don't handle this in migration code yet. |
9452 | | */ |
9453 | 0 | for (int opcode = 0; opcode < ARRAY_SIZE(n->cse.acs); opcode++) { |
9454 | | /* Is command supported? */ |
9455 | 0 | if (!n->cse.acs[opcode]) { |
9456 | 0 | continue; |
9457 | 0 | } |
9458 | | |
9459 | 0 | switch (opcode) { |
9460 | 0 | case NVME_ADM_CMD_DELETE_SQ: |
9461 | 0 | case NVME_ADM_CMD_CREATE_SQ: |
9462 | 0 | case NVME_ADM_CMD_GET_LOG_PAGE: |
9463 | 0 | case NVME_ADM_CMD_DELETE_CQ: |
9464 | 0 | case NVME_ADM_CMD_CREATE_CQ: |
9465 | 0 | case NVME_ADM_CMD_IDENTIFY: |
9466 | 0 | case NVME_ADM_CMD_ABORT: |
9467 | 0 | case NVME_ADM_CMD_SET_FEATURES: |
9468 | 0 | case NVME_ADM_CMD_GET_FEATURES: |
9469 | 0 | case NVME_ADM_CMD_ASYNC_EV_REQ: |
9470 | 0 | case NVME_ADM_CMD_DBBUF_CONFIG: |
9471 | 0 | case NVME_ADM_CMD_FORMAT_NVM: |
9472 | 0 | case NVME_ADM_CMD_DIRECTIVE_SEND: |
9473 | 0 | case NVME_ADM_CMD_DIRECTIVE_RECV: |
9474 | 0 | break; |
9475 | 0 | case NVME_ADM_CMD_NS_ATTACHMENT: { |
9476 | 0 | int namespaces_num = 0; |
9477 | 0 | for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
9478 | 0 | NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i); |
9479 | 0 | if (!ns) { |
9480 | 0 | continue; |
9481 | 0 | } |
9482 | | |
9483 | 0 | namespaces_num++; |
9484 | 0 | } |
9485 | |
|
9486 | 0 | if (namespaces_num > 1) { |
9487 | 0 | g_ptr_array_add(blocker_features, |
9488 | 0 | (void *) "Namespace Attachment"); |
9489 | 0 | } |
9490 | |
|
9491 | 0 | break; |
9492 | 0 | } |
9493 | 0 | case NVME_ADM_CMD_VIRT_MNGMT: |
9494 | 0 | if (n->params.sriov_max_vfs) { |
9495 | 0 | g_ptr_array_add(blocker_features, (void *) "SR-IOV"); |
9496 | 0 | } |
9497 | |
|
9498 | 0 | break; |
9499 | 0 | case NVME_ADM_CMD_SECURITY_SEND: |
9500 | 0 | case NVME_ADM_CMD_SECURITY_RECV: |
9501 | 0 | if (adm_cmd_security_checked) { |
9502 | 0 | break; |
9503 | 0 | } |
9504 | | |
9505 | 0 | if (pci_dev->spdm_port) { |
9506 | 0 | g_ptr_array_add(blocker_features, (void *) "SPDM"); |
9507 | 0 | } |
9508 | |
|
9509 | 0 | adm_cmd_security_checked = true; |
9510 | |
|
9511 | 0 | break; |
9512 | 0 | default: |
9513 | 0 | g_assert_not_reached(); |
9514 | 0 | } |
9515 | 0 | } |
9516 | | |
9517 | 0 | for (int opcode = 0; opcode < ARRAY_SIZE(n->cse.iocs.nvm); opcode++) { |
9518 | 0 | if (!n->cse.iocs.nvm[opcode]) { |
9519 | 0 | continue; |
9520 | 0 | } |
9521 | | |
9522 | 0 | switch (opcode) { |
9523 | 0 | case NVME_CMD_FLUSH: |
9524 | 0 | case NVME_CMD_WRITE: |
9525 | 0 | case NVME_CMD_READ: |
9526 | 0 | case NVME_CMD_COMPARE: |
9527 | 0 | case NVME_CMD_WRITE_ZEROES: |
9528 | 0 | case NVME_CMD_DSM: |
9529 | 0 | case NVME_CMD_VERIFY: |
9530 | 0 | case NVME_CMD_COPY: |
9531 | 0 | break; |
9532 | 0 | case NVME_CMD_IO_MGMT_RECV: |
9533 | 0 | case NVME_CMD_IO_MGMT_SEND: |
9534 | 0 | if (cmd_io_mgmt_checked) { |
9535 | 0 | break; |
9536 | 0 | } |
9537 | | |
9538 | | /* check for NVME_IOMS_MO_RUH_UPDATE */ |
9539 | 0 | if (n->subsys->params.fdp.enabled) { |
9540 | 0 | g_ptr_array_add(blocker_features, (void *) "FDP"); |
9541 | 0 | } |
9542 | |
|
9543 | 0 | cmd_io_mgmt_checked = true; |
9544 | |
|
9545 | 0 | break; |
9546 | 0 | default: |
9547 | 0 | g_assert_not_reached(); |
9548 | 0 | } |
9549 | 0 | } |
9550 | | |
9551 | 0 | for (int opcode = 0; opcode < ARRAY_SIZE(n->cse.iocs.zoned); opcode++) { |
9552 | | /* |
9553 | | * If command isn't supported or we have the same command |
9554 | | * in n->cse.iocs.nvm, then we can skip it here. |
9555 | | */ |
9556 | 0 | if (!n->cse.iocs.zoned[opcode] || n->cse.iocs.nvm[opcode]) { |
9557 | 0 | continue; |
9558 | 0 | } |
9559 | | |
9560 | 0 | switch (opcode) { |
9561 | 0 | case NVME_CMD_ZONE_APPEND: |
9562 | 0 | case NVME_CMD_ZONE_MGMT_SEND: |
9563 | 0 | case NVME_CMD_ZONE_MGMT_RECV: |
9564 | 0 | if (cmd_zone_checked) { |
9565 | 0 | break; |
9566 | 0 | } |
9567 | | |
9568 | 0 | for (int i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
9569 | 0 | NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i); |
9570 | 0 | if (!ns) { |
9571 | 0 | continue; |
9572 | 0 | } |
9573 | | |
9574 | 0 | if (ns->params.zoned) { |
9575 | 0 | g_ptr_array_add(blocker_features, |
9576 | 0 | (void *) "Zoned Namespace"); |
9577 | 0 | break; |
9578 | 0 | } |
9579 | 0 | } |
9580 | |
|
9581 | 0 | cmd_zone_checked = true; |
9582 | |
|
9583 | 0 | break; |
9584 | 0 | default: |
9585 | 0 | g_assert_not_reached(); |
9586 | 0 | } |
9587 | 0 | } |
9588 | | |
9589 | | /* |
9590 | | * Try our best to explicitly detect all not supported caps, |
9591 | | * to let users know what features cause migration to be blocked, |
9592 | | * but in case we miss handling here, everything else will be |
9593 | | * covered by unsupported_cap check. |
9594 | | */ |
9595 | 0 | if (NVME_CAP_CMBS(cap)) { |
9596 | 0 | g_ptr_array_add(blocker_features, (void *) "CMB"); |
9597 | 0 | cap &= ~((uint64_t)CAP_CMBS_MASK << CAP_CMBS_SHIFT); |
9598 | 0 | } |
9599 | |
|
9600 | 0 | if (NVME_CAP_PMRS(cap)) { |
9601 | 0 | g_ptr_array_add(blocker_features, (void *) "PMR"); |
9602 | 0 | cap &= ~((uint64_t)CAP_PMRS_MASK << CAP_PMRS_SHIFT); |
9603 | 0 | } |
9604 | |
|
9605 | 0 | unsupported_cap = cap & ~NVME_MIGRATION_SUPPORTED_CAP_BITS; |
9606 | 0 | if (unsupported_cap) { |
9607 | 0 | g_ptr_array_add(blocker_features, (void *) "unknown capability"); |
9608 | 0 | } |
9609 | |
|
9610 | 0 | assert(n->migration_blocker == NULL); |
9611 | 0 | if (blocker_features->len > 0) { |
9612 | 0 | g_autofree char *blocker_list = NULL; |
9613 | |
|
9614 | 0 | g_ptr_array_add(blocker_features, NULL); |
9615 | 0 | blocker_list = g_strjoinv(", ", (void *)blocker_features->pdata); |
9616 | 0 | error_setg(&n->migration_blocker, |
9617 | 0 | "Migration is not supported for %s", blocker_list); |
9618 | 0 | if (migrate_add_blocker(&n->migration_blocker, errp) < 0) { |
9619 | 0 | return false; |
9620 | 0 | } |
9621 | 0 | } |
9622 | | |
9623 | 0 | return true; |
9624 | 0 | } |
9625 | | |
9626 | | static int nvme_init_subsys(NvmeCtrl *n, Error **errp) |
9627 | 0 | { |
9628 | 0 | int cntlid; |
9629 | |
|
9630 | 0 | if (!n->subsys) { |
9631 | 0 | DeviceState *dev = qdev_new(TYPE_NVME_SUBSYS); |
9632 | |
|
9633 | 0 | qdev_prop_set_string(dev, "nqn", n->params.serial); |
9634 | |
|
9635 | 0 | if (!qdev_realize(dev, NULL, errp)) { |
9636 | 0 | return -1; |
9637 | 0 | } |
9638 | | |
9639 | 0 | n->subsys = NVME_SUBSYS(dev); |
9640 | 0 | } else { |
9641 | 0 | NvmeIdCtrl *id = &n->id_ctrl; |
9642 | 0 | uint32_t ctratt = le32_to_cpu(id->ctratt); |
9643 | |
|
9644 | 0 | id->cmic |= NVME_CMIC_MULTI_CTRL; |
9645 | 0 | ctratt |= NVME_CTRATT_ENDGRPS; |
9646 | |
|
9647 | 0 | id->endgidmax = cpu_to_le16(0x1); |
9648 | |
|
9649 | 0 | if (n->subsys->endgrp.fdp.enabled) { |
9650 | 0 | ctratt |= NVME_CTRATT_FDPS; |
9651 | 0 | } |
9652 | |
|
9653 | 0 | id->ctratt = cpu_to_le32(ctratt); |
9654 | 0 | } |
9655 | | |
9656 | 0 | cntlid = nvme_subsys_register_ctrl(n, errp); |
9657 | 0 | if (cntlid < 0) { |
9658 | 0 | return -1; |
9659 | 0 | } |
9660 | | |
9661 | 0 | n->cntlid = cntlid; |
9662 | |
|
9663 | 0 | return 0; |
9664 | 0 | } |
9665 | | |
9666 | | void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns) |
9667 | 0 | { |
9668 | 0 | uint32_t nsid = ns->params.nsid; |
9669 | 0 | assert(nsid && nsid <= NVME_MAX_NAMESPACES); |
9670 | |
|
9671 | 0 | n->namespaces[nsid] = ns; |
9672 | 0 | ns->attached++; |
9673 | 0 | } |
9674 | | |
9675 | | static void nvme_realize(PCIDevice *pci_dev, Error **errp) |
9676 | 0 | { |
9677 | 0 | NvmeCtrl *n = NVME(pci_dev); |
9678 | 0 | DeviceState *dev = DEVICE(pci_dev); |
9679 | 0 | NvmeNamespace *ns; |
9680 | 0 | NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev)); |
9681 | |
|
9682 | 0 | if (pci_is_vf(pci_dev)) { |
9683 | | /* |
9684 | | * VFs derive settings from the parent. PF's lifespan exceeds |
9685 | | * that of VF's. |
9686 | | */ |
9687 | 0 | memcpy(&n->params, &pn->params, sizeof(NvmeParams)); |
9688 | | |
9689 | | /* |
9690 | | * Set PF's serial value to a new string memory to prevent 'serial' |
9691 | | * property object release of PF when a VF is removed from the system. |
9692 | | */ |
9693 | 0 | n->params.serial = g_strdup(pn->params.serial); |
9694 | 0 | n->subsys = pn->subsys; |
9695 | | |
9696 | | /* |
9697 | | * Assigning this link (strong link) causes an `object_unref` later in |
9698 | | * `object_release_link_property`. Increment the refcount to balance |
9699 | | * this out. |
9700 | | */ |
9701 | 0 | object_ref(OBJECT(pn->subsys)); |
9702 | 0 | } |
9703 | |
|
9704 | 0 | if (!nvme_check_params(n, errp)) { |
9705 | 0 | return; |
9706 | 0 | } |
9707 | | |
9708 | 0 | qbus_init(&n->bus, sizeof(NvmeBus), TYPE_NVME_BUS, dev, dev->id); |
9709 | 0 | qbus_set_bus_hotplug_handler(BUS(&n->bus)); |
9710 | |
|
9711 | 0 | if (nvme_init_subsys(n, errp)) { |
9712 | 0 | return; |
9713 | 0 | } |
9714 | 0 | nvme_init_state(n); |
9715 | 0 | if (!nvme_init_pci(n, pci_dev, errp)) { |
9716 | 0 | return; |
9717 | 0 | } |
9718 | 0 | nvme_init_ctrl(n, pci_dev); |
9719 | | |
9720 | | /* setup a namespace if the controller drive property was given */ |
9721 | 0 | if (n->namespace.blkconf.blk) { |
9722 | 0 | ns = &n->namespace; |
9723 | 0 | ns->params.nsid = 1; |
9724 | 0 | ns->ctrl = n; |
9725 | 0 | ns->subsys = n->subsys; |
9726 | |
|
9727 | 0 | if (nvme_ns_setup(ns, errp)) { |
9728 | 0 | return; |
9729 | 0 | } |
9730 | | |
9731 | 0 | n->subsys->namespaces[ns->params.nsid] = ns; |
9732 | 0 | } |
9733 | | |
9734 | 0 | if (!nvme_set_migration_blockers(n, pci_dev, errp)) { |
9735 | 0 | return; |
9736 | 0 | } |
9737 | 0 | } |
9738 | | |
9739 | | static void nvme_exit(PCIDevice *pci_dev) |
9740 | 0 | { |
9741 | 0 | NvmeCtrl *n = NVME(pci_dev); |
9742 | 0 | NvmeNamespace *ns; |
9743 | 0 | int i; |
9744 | |
|
9745 | 0 | nvme_ctrl_reset(n, NVME_RESET_FUNCTION); |
9746 | |
|
9747 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
9748 | 0 | ns = nvme_ns(n, i); |
9749 | 0 | if (ns) { |
9750 | 0 | ns->attached--; |
9751 | 0 | } |
9752 | 0 | } |
9753 | |
|
9754 | 0 | if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) { |
9755 | 0 | pcie_sriov_pf_exit(pci_dev); |
9756 | 0 | } |
9757 | |
|
9758 | 0 | nvme_subsys_unregister_ctrl(n->subsys, n); |
9759 | |
|
9760 | 0 | if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) { |
9761 | 0 | g_free(n->sec_ctrl_list); |
9762 | 0 | } |
9763 | |
|
9764 | 0 | g_free(n->cq); |
9765 | 0 | g_free(n->sq); |
9766 | 0 | g_free(n->aer_reqs); |
9767 | |
|
9768 | 0 | if (n->params.cmb_size_mb) { |
9769 | 0 | g_free(n->cmb.buf); |
9770 | 0 | } |
9771 | | |
9772 | | /* Only one of the `spdm_socket`s below should have been setup */ |
9773 | 0 | assert(!(pci_dev->doe_spdm.spdm_socket > 0 && n->spdm_socket >= 0)); |
9774 | 0 | if (pci_dev->doe_spdm.spdm_socket > 0) { |
9775 | 0 | spdm_socket_close(pci_dev->doe_spdm.spdm_socket, |
9776 | 0 | SPDM_TRANSPORT_TYPE_DOE); |
9777 | 0 | } else if (n->spdm_socket >= 0) { |
9778 | 0 | spdm_socket_close(pci_dev->doe_spdm.spdm_socket, |
9779 | 0 | SPDM_TRANSPORT_TYPE_NVME); |
9780 | 0 | } |
9781 | |
|
9782 | 0 | if (n->pmr.dev) { |
9783 | 0 | host_memory_backend_set_mapped(n->pmr.dev, false); |
9784 | 0 | } |
9785 | |
|
9786 | 0 | if (n->params.msix_exclusive_bar && !pci_is_vf(pci_dev)) { |
9787 | 0 | msix_uninit_exclusive_bar(pci_dev); |
9788 | 0 | } else { |
9789 | 0 | msix_uninit(pci_dev, &n->bar0, &n->bar0); |
9790 | 0 | memory_region_del_subregion(&n->bar0, &n->iomem); |
9791 | 0 | } |
9792 | |
|
9793 | 0 | migrate_del_blocker(&n->migration_blocker); |
9794 | 0 | } |
9795 | | |
9796 | | static const Property nvme_props[] = { |
9797 | | DEFINE_BLOCK_PROPERTIES(NvmeCtrl, namespace.blkconf), |
9798 | | DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmr.dev, TYPE_MEMORY_BACKEND, |
9799 | | HostMemoryBackend *), |
9800 | | DEFINE_PROP_LINK("subsys", NvmeCtrl, subsys, TYPE_NVME_SUBSYS, |
9801 | | NvmeSubsystem *), |
9802 | | DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial), |
9803 | | DEFINE_PROP_STRING("model", NvmeCtrl, params.model), |
9804 | | DEFINE_PROP_STRING("firmware-version", NvmeCtrl, params.firmware_version), |
9805 | | DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0), |
9806 | | DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0), |
9807 | | DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl, params.max_ioqpairs, 64), |
9808 | | DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl, params.msix_qsize, 65), |
9809 | | DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3), |
9810 | | DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64), |
9811 | | DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7), |
9812 | | DEFINE_PROP_UINT8("vsl", NvmeCtrl, params.vsl, 7), |
9813 | | DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl, params.use_intel_id, false), |
9814 | | DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl, params.legacy_cmb, false), |
9815 | | DEFINE_PROP_BOOL("ioeventfd", NvmeCtrl, params.ioeventfd, false), |
9816 | | DEFINE_PROP_BOOL("dbcs", NvmeCtrl, params.dbcs, true), |
9817 | | DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl, params.zasl, 0), |
9818 | | DEFINE_PROP_BOOL("zoned.auto_transition", NvmeCtrl, |
9819 | | params.auto_transition_zones, true), |
9820 | | DEFINE_PROP_UINT16("sriov_max_vfs", NvmeCtrl, params.sriov_max_vfs, 0), |
9821 | | DEFINE_PROP_UINT16("sriov_vq_flexible", NvmeCtrl, |
9822 | | params.sriov_vq_flexible, 0), |
9823 | | DEFINE_PROP_UINT16("sriov_vi_flexible", NvmeCtrl, |
9824 | | params.sriov_vi_flexible, 0), |
9825 | | DEFINE_PROP_UINT32("sriov_max_vi_per_vf", NvmeCtrl, |
9826 | | params.sriov_max_vi_per_vf, 0), |
9827 | | DEFINE_PROP_UINT32("sriov_max_vq_per_vf", NvmeCtrl, |
9828 | | params.sriov_max_vq_per_vf, 0), |
9829 | | DEFINE_PROP_BOOL("msix-exclusive-bar", NvmeCtrl, params.msix_exclusive_bar, |
9830 | | false), |
9831 | | DEFINE_PROP_UINT16("mqes", NvmeCtrl, params.mqes, 0x7ff), |
9832 | | DEFINE_PROP_UINT16("spdm_port", PCIDevice, spdm_port, 0), |
9833 | | DEFINE_PROP_SPDM_TRANS("spdm_trans", PCIDevice, spdm_trans, |
9834 | | SPDM_TRANSPORT_TYPE_DOE), |
9835 | | DEFINE_PROP_BOOL("ctratt.mem", NvmeCtrl, params.ctratt.mem, false), |
9836 | | DEFINE_PROP_BOOL("atomic.dn", NvmeCtrl, params.atomic_dn, 0), |
9837 | | DEFINE_PROP_UINT16("atomic.awun", NvmeCtrl, params.atomic_awun, 0), |
9838 | | DEFINE_PROP_UINT16("atomic.awupf", NvmeCtrl, params.atomic_awupf, 0), |
9839 | | DEFINE_PROP_BOOL("ocp", NvmeCtrl, params.ocp, false), |
9840 | | }; |
9841 | | |
9842 | | static void nvme_get_smart_warning(Object *obj, Visitor *v, const char *name, |
9843 | | void *opaque, Error **errp) |
9844 | 0 | { |
9845 | 0 | NvmeCtrl *n = NVME(obj); |
9846 | 0 | uint8_t value = n->smart_critical_warning; |
9847 | |
|
9848 | 0 | visit_type_uint8(v, name, &value, errp); |
9849 | 0 | } |
9850 | | |
9851 | | static void nvme_set_smart_warning(Object *obj, Visitor *v, const char *name, |
9852 | | void *opaque, Error **errp) |
9853 | 0 | { |
9854 | 0 | NvmeCtrl *n = NVME(obj); |
9855 | 0 | uint8_t value, old_value, cap = 0, index, event; |
9856 | |
|
9857 | 0 | if (!visit_type_uint8(v, name, &value, errp)) { |
9858 | 0 | return; |
9859 | 0 | } |
9860 | | |
9861 | 0 | cap = NVME_SMART_SPARE | NVME_SMART_TEMPERATURE | NVME_SMART_RELIABILITY |
9862 | 0 | | NVME_SMART_MEDIA_READ_ONLY | NVME_SMART_FAILED_VOLATILE_MEDIA; |
9863 | 0 | if (NVME_CAP_PMRS(ldq_le_p(&n->bar.cap))) { |
9864 | 0 | cap |= NVME_SMART_PMR_UNRELIABLE; |
9865 | 0 | } |
9866 | |
|
9867 | 0 | if ((value & cap) != value) { |
9868 | 0 | error_setg(errp, "unsupported smart critical warning bits: 0x%x", |
9869 | 0 | value & ~cap); |
9870 | 0 | return; |
9871 | 0 | } |
9872 | | |
9873 | 0 | old_value = n->smart_critical_warning; |
9874 | 0 | n->smart_critical_warning = value; |
9875 | | |
9876 | | /* only inject new bits of smart critical warning */ |
9877 | 0 | for (index = 0; index < NVME_SMART_WARN_MAX; index++) { |
9878 | 0 | event = 1 << index; |
9879 | 0 | if (value & ~old_value & event) |
9880 | 0 | nvme_smart_event(n, event); |
9881 | 0 | } |
9882 | 0 | } |
9883 | | |
9884 | | static void nvme_pci_reset(DeviceState *qdev) |
9885 | 0 | { |
9886 | 0 | PCIDevice *pci_dev = PCI_DEVICE(qdev); |
9887 | 0 | NvmeCtrl *n = NVME(pci_dev); |
9888 | |
|
9889 | 0 | trace_pci_nvme_pci_reset(); |
9890 | 0 | nvme_ctrl_reset(n, NVME_RESET_FUNCTION); |
9891 | 0 | } |
9892 | | |
9893 | | static void nvme_sriov_post_write_config(PCIDevice *dev, uint16_t old_num_vfs) |
9894 | 0 | { |
9895 | 0 | NvmeCtrl *n = NVME(dev); |
9896 | 0 | NvmeSecCtrlEntry *sctrl; |
9897 | 0 | int i; |
9898 | |
|
9899 | 0 | for (i = pcie_sriov_num_vfs(dev); i < old_num_vfs; i++) { |
9900 | 0 | sctrl = &n->sec_ctrl_list[i]; |
9901 | 0 | nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false); |
9902 | 0 | } |
9903 | 0 | } |
9904 | | |
9905 | | static void nvme_pci_write_config(PCIDevice *dev, uint32_t address, |
9906 | | uint32_t val, int len) |
9907 | 0 | { |
9908 | 0 | uint16_t old_num_vfs = pcie_sriov_num_vfs(dev); |
9909 | | |
9910 | | /* DOE is only initialised if SPDM over DOE is used */ |
9911 | 0 | if (pcie_find_capability(dev, PCI_EXT_CAP_ID_DOE) && |
9912 | 0 | dev->spdm_trans == SPDM_TRANSPORT_TYPE_DOE) { |
9913 | 0 | pcie_doe_write_config(&dev->doe_spdm, address, val, len); |
9914 | 0 | } |
9915 | 0 | pci_default_write_config(dev, address, val, len); |
9916 | 0 | pcie_cap_flr_write_config(dev, address, val, len); |
9917 | 0 | nvme_sriov_post_write_config(dev, old_num_vfs); |
9918 | 0 | } |
9919 | | |
9920 | | static uint32_t nvme_pci_read_config(PCIDevice *dev, uint32_t address, int len) |
9921 | 0 | { |
9922 | 0 | uint32_t val; |
9923 | |
|
9924 | 0 | if (dev->spdm_port && pcie_find_capability(dev, PCI_EXT_CAP_ID_DOE) && |
9925 | 0 | (dev->spdm_trans == SPDM_TRANSPORT_TYPE_DOE)) { |
9926 | 0 | if (pcie_doe_read_config(&dev->doe_spdm, address, len, &val)) { |
9927 | 0 | return val; |
9928 | 0 | } |
9929 | 0 | } |
9930 | 0 | return pci_default_read_config(dev, address, len); |
9931 | 0 | } |
9932 | | |
9933 | | static const VMStateDescription nvme_vmstate_cqe = { |
9934 | | .name = "nvme-cqe", |
9935 | | .version_id = 1, |
9936 | | .minimum_version_id = 1, |
9937 | | .fields = (const VMStateField[]) { |
9938 | | VMSTATE_UINT32(result, NvmeCqe), |
9939 | | VMSTATE_UINT32(dw1, NvmeCqe), |
9940 | | VMSTATE_UINT16(sq_head, NvmeCqe), |
9941 | | VMSTATE_UINT16(sq_id, NvmeCqe), |
9942 | | VMSTATE_UINT16(cid, NvmeCqe), |
9943 | | VMSTATE_UINT16(status, NvmeCqe), |
9944 | | VMSTATE_END_OF_LIST() |
9945 | | } |
9946 | | }; |
9947 | | |
9948 | | static const VMStateDescription nvme_vmstate_cmd_dptr_sgl = { |
9949 | | .name = "nvme-request-cmd-dptr-sgl", |
9950 | | .version_id = 1, |
9951 | | .minimum_version_id = 1, |
9952 | | .fields = (const VMStateField[]) { |
9953 | | VMSTATE_UINT64(addr, NvmeSglDescriptor), |
9954 | | VMSTATE_UINT32(len, NvmeSglDescriptor), |
9955 | | VMSTATE_UINT8_ARRAY(rsvd, NvmeSglDescriptor, 3), |
9956 | | VMSTATE_UINT8(type, NvmeSglDescriptor), |
9957 | | VMSTATE_END_OF_LIST() |
9958 | | } |
9959 | | }; |
9960 | | |
9961 | | static const VMStateDescription nvme_vmstate_cmd_dptr = { |
9962 | | .name = "nvme-request-cmd-dptr", |
9963 | | .version_id = 1, |
9964 | | .minimum_version_id = 1, |
9965 | | .fields = (const VMStateField[]) { |
9966 | | VMSTATE_UINT64(prp1, NvmeCmdDptr), |
9967 | | VMSTATE_UINT64(prp2, NvmeCmdDptr), |
9968 | | VMSTATE_STRUCT(sgl, NvmeCmdDptr, 0, |
9969 | | nvme_vmstate_cmd_dptr_sgl, NvmeSglDescriptor), |
9970 | | VMSTATE_END_OF_LIST() |
9971 | | } |
9972 | | }; |
9973 | | |
9974 | | static const VMStateDescription nvme_vmstate_cmd = { |
9975 | | .name = "nvme-request-cmd", |
9976 | | .version_id = 1, |
9977 | | .minimum_version_id = 1, |
9978 | | .fields = (const VMStateField[]) { |
9979 | | VMSTATE_UINT8(opcode, NvmeCmd), |
9980 | | VMSTATE_UINT8(flags, NvmeCmd), |
9981 | | VMSTATE_UINT16(cid, NvmeCmd), |
9982 | | VMSTATE_UINT32(nsid, NvmeCmd), |
9983 | | VMSTATE_UINT64(res1, NvmeCmd), |
9984 | | VMSTATE_UINT64(mptr, NvmeCmd), |
9985 | | VMSTATE_STRUCT(dptr, NvmeCmd, 0, nvme_vmstate_cmd_dptr, NvmeCmdDptr), |
9986 | | VMSTATE_UINT32(cdw10, NvmeCmd), |
9987 | | VMSTATE_UINT32(cdw11, NvmeCmd), |
9988 | | VMSTATE_UINT32(cdw12, NvmeCmd), |
9989 | | VMSTATE_UINT32(cdw13, NvmeCmd), |
9990 | | VMSTATE_UINT32(cdw14, NvmeCmd), |
9991 | | VMSTATE_UINT32(cdw15, NvmeCmd), |
9992 | | VMSTATE_END_OF_LIST() |
9993 | | } |
9994 | | }; |
9995 | | |
9996 | | static bool nvme_req_pre_load(void *opaque, Error **errp) |
9997 | 0 | { |
9998 | 0 | memset(opaque, 0x0, sizeof(NvmeRequest)); |
9999 | 0 | return true; |
10000 | 0 | } |
10001 | | |
10002 | | static const VMStateDescription nvme_vmstate_request = { |
10003 | | .name = "nvme-request", |
10004 | | .version_id = 1, |
10005 | | .minimum_version_id = 1, |
10006 | | .pre_load_errp = nvme_req_pre_load, |
10007 | | .fields = (const VMStateField[]) { |
10008 | | VMSTATE_UINT16(status, NvmeRequest), |
10009 | | VMSTATE_STRUCT(cqe, NvmeRequest, 0, nvme_vmstate_cqe, NvmeCqe), |
10010 | | VMSTATE_STRUCT(cmd, NvmeRequest, 0, nvme_vmstate_cmd, NvmeCmd), |
10011 | | VMSTATE_END_OF_LIST() |
10012 | | } |
10013 | | }; |
10014 | | |
10015 | | static const VMStateDescription nvme_vmstate_bar = { |
10016 | | .name = "nvme-bar", |
10017 | | .minimum_version_id = 1, |
10018 | | .version_id = 1, |
10019 | | .fields = (const VMStateField[]) { |
10020 | | VMSTATE_UINT64(cap, NvmeBar), |
10021 | | VMSTATE_UINT32(vs, NvmeBar), |
10022 | | VMSTATE_UINT32(intms, NvmeBar), |
10023 | | VMSTATE_UINT32(intmc, NvmeBar), |
10024 | | VMSTATE_UINT32(cc, NvmeBar), |
10025 | | VMSTATE_UINT8_ARRAY(rsvd24, NvmeBar, 4), |
10026 | | VMSTATE_UINT32(csts, NvmeBar), |
10027 | | VMSTATE_UINT32(nssr, NvmeBar), |
10028 | | VMSTATE_UINT32(aqa, NvmeBar), |
10029 | | VMSTATE_UINT64(asq, NvmeBar), |
10030 | | VMSTATE_UINT64(acq, NvmeBar), |
10031 | | VMSTATE_UINT32(cmbloc, NvmeBar), |
10032 | | VMSTATE_UINT32(cmbsz, NvmeBar), |
10033 | | VMSTATE_UINT32(bpinfo, NvmeBar), |
10034 | | VMSTATE_UINT32(bprsel, NvmeBar), |
10035 | | VMSTATE_UINT64(bpmbl, NvmeBar), |
10036 | | VMSTATE_UINT64(cmbmsc, NvmeBar), |
10037 | | VMSTATE_UINT32(cmbsts, NvmeBar), |
10038 | | VMSTATE_UINT8_ARRAY(rsvd92, NvmeBar, 3492), |
10039 | | VMSTATE_UINT32(pmrcap, NvmeBar), |
10040 | | VMSTATE_UINT32(pmrctl, NvmeBar), |
10041 | | VMSTATE_UINT32(pmrsts, NvmeBar), |
10042 | | VMSTATE_UINT32(pmrebs, NvmeBar), |
10043 | | VMSTATE_UINT32(pmrswtp, NvmeBar), |
10044 | | VMSTATE_UINT32(pmrmscl, NvmeBar), |
10045 | | VMSTATE_UINT32(pmrmscu, NvmeBar), |
10046 | | VMSTATE_UINT8_ARRAY(css, NvmeBar, 484), |
10047 | | VMSTATE_END_OF_LIST() |
10048 | | }, |
10049 | | }; |
10050 | | |
10051 | | static bool nvme_cqueue_pre_load(void *opaque, Error **errp) |
10052 | 0 | { |
10053 | 0 | NvmeCQueue *cq = opaque; |
10054 | |
|
10055 | 0 | QTAILQ_INIT(&cq->req_list); |
10056 | 0 | return true; |
10057 | 0 | } |
10058 | | |
10059 | | static const VMStateDescription nvme_vmstate_cqueue = { |
10060 | | .name = "nvme-cq", |
10061 | | .version_id = 1, |
10062 | | .minimum_version_id = 1, |
10063 | | .pre_load_errp = nvme_cqueue_pre_load, |
10064 | | .fields = (const VMStateField[]) { |
10065 | | VMSTATE_UINT8(phase, NvmeCQueue), |
10066 | | VMSTATE_UINT16(cqid, NvmeCQueue), |
10067 | | VMSTATE_UINT16(irq_enabled, NvmeCQueue), |
10068 | | VMSTATE_UINT32(head, NvmeCQueue), |
10069 | | VMSTATE_UINT32(tail, NvmeCQueue), |
10070 | | VMSTATE_UINT32(vector, NvmeCQueue), |
10071 | | VMSTATE_UINT32(size, NvmeCQueue), |
10072 | | VMSTATE_UINT64(dma_addr, NvmeCQueue), |
10073 | | |
10074 | | VMSTATE_QTAILQ_V(req_list, NvmeCQueue, 1, nvme_vmstate_request, |
10075 | | NvmeRequest, entry), |
10076 | | |
10077 | | /* db_addr, ei_addr, etc will be recalculated */ |
10078 | | VMSTATE_END_OF_LIST() |
10079 | | } |
10080 | | }; |
10081 | | |
10082 | | static const VMStateDescription nvme_vmstate_squeue = { |
10083 | | .name = "nvme-sq", |
10084 | | .version_id = 1, |
10085 | | .minimum_version_id = 1, |
10086 | | .fields = (const VMStateField[]) { |
10087 | | VMSTATE_UINT16(sqid, NvmeSQueue), |
10088 | | VMSTATE_UINT16(cqid, NvmeSQueue), |
10089 | | VMSTATE_UINT32(head, NvmeSQueue), |
10090 | | VMSTATE_UINT32(tail, NvmeSQueue), |
10091 | | VMSTATE_UINT32(size, NvmeSQueue), |
10092 | | VMSTATE_UINT64(dma_addr, NvmeSQueue), |
10093 | | /* db_addr, ei_addr, etc will be recalculated */ |
10094 | | VMSTATE_END_OF_LIST() |
10095 | | } |
10096 | | }; |
10097 | | |
10098 | | static const VMStateDescription nvme_vmstate_async_event_result = { |
10099 | | .name = "nvme-async-event-result", |
10100 | | .version_id = 1, |
10101 | | .minimum_version_id = 1, |
10102 | | .fields = (const VMStateField[]) { |
10103 | | VMSTATE_UINT8(event_type, NvmeAerResult), |
10104 | | VMSTATE_UINT8(event_info, NvmeAerResult), |
10105 | | VMSTATE_UINT8(log_page, NvmeAerResult), |
10106 | | VMSTATE_UINT8(resv, NvmeAerResult), |
10107 | | VMSTATE_END_OF_LIST() |
10108 | | } |
10109 | | }; |
10110 | | |
10111 | | static const VMStateDescription nvme_vmstate_async_event = { |
10112 | | .name = "nvme-async-event", |
10113 | | .version_id = 1, |
10114 | | .minimum_version_id = 1, |
10115 | | .fields = (const VMStateField[]) { |
10116 | | VMSTATE_STRUCT(result, NvmeAsyncEvent, 0, |
10117 | | nvme_vmstate_async_event_result, NvmeAerResult), |
10118 | | VMSTATE_END_OF_LIST() |
10119 | | } |
10120 | | }; |
10121 | | |
10122 | | static const VMStateDescription nvme_vmstate_hbs = { |
10123 | | .name = "nvme-hbs", |
10124 | | .version_id = 1, |
10125 | | .minimum_version_id = 1, |
10126 | | .fields = (const VMStateField[]) { |
10127 | | VMSTATE_UINT8(acre, NvmeHostBehaviorSupport), |
10128 | | VMSTATE_UINT8(etdas, NvmeHostBehaviorSupport), |
10129 | | VMSTATE_UINT8(lbafee, NvmeHostBehaviorSupport), |
10130 | | VMSTATE_UINT8(rsvd3, NvmeHostBehaviorSupport), |
10131 | | VMSTATE_UINT16(cdfe, NvmeHostBehaviorSupport), |
10132 | | VMSTATE_UINT8_ARRAY(rsvd6, NvmeHostBehaviorSupport, 506), |
10133 | | VMSTATE_END_OF_LIST() |
10134 | | } |
10135 | | }; |
10136 | | |
10137 | | const VMStateDescription nvme_vmstate_atomic = { |
10138 | | .name = "nvme-atomic", |
10139 | | .version_id = 1, |
10140 | | .minimum_version_id = 1, |
10141 | | .fields = (const VMStateField[]) { |
10142 | | VMSTATE_UINT32(atomic_max_write_size, NvmeAtomic), |
10143 | | VMSTATE_UINT64(atomic_boundary, NvmeAtomic), |
10144 | | VMSTATE_UINT64(atomic_nabo, NvmeAtomic), |
10145 | | VMSTATE_BOOL(atomic_writes, NvmeAtomic), |
10146 | | VMSTATE_END_OF_LIST() |
10147 | | } |
10148 | | }; |
10149 | | |
10150 | | static bool pre_save_validate_aer_req(NvmeRequest *req, Error **errp) |
10151 | 0 | { |
10152 | | /* |
10153 | | * Can't use assert() here, because we don't want |
10154 | | * to just crash QEMU when user requests a migration. |
10155 | | */ |
10156 | 0 | if (!(req->cmd.opcode == NVME_ADM_CMD_ASYNC_EV_REQ)) { |
10157 | 0 | error_setg(errp, "req->cmd.opcode (%u) != NVME_ADM_CMD_ASYNC_EV_REQ", |
10158 | 0 | req->cmd.opcode); |
10159 | 0 | return false; |
10160 | 0 | } |
10161 | | |
10162 | 0 | if (!(req->ns == NULL)) { |
10163 | 0 | error_setg(errp, "req->ns != NULL"); |
10164 | 0 | return false; |
10165 | 0 | } |
10166 | | |
10167 | 0 | if (!(req->sq == &req->sq->ctrl->admin_sq)) { |
10168 | 0 | error_setg(errp, "req->sq != &req->sq->ctrl->admin_sq"); |
10169 | 0 | return false; |
10170 | 0 | } |
10171 | | |
10172 | 0 | if (!(req->aiocb == NULL)) { |
10173 | 0 | error_setg(errp, "req->aiocb != NULL"); |
10174 | 0 | return false; |
10175 | 0 | } |
10176 | | |
10177 | 0 | if (!(req->opaque == NULL)) { |
10178 | 0 | error_setg(errp, "req->opaque != NULL"); |
10179 | 0 | return false; |
10180 | 0 | } |
10181 | | |
10182 | 0 | if (!(req->atomic_write == false)) { |
10183 | 0 | error_setg(errp, "req->atomic_write != false"); |
10184 | 0 | return false; |
10185 | 0 | } |
10186 | | |
10187 | 0 | if (req->sg.flags & NVME_SG_ALLOC) { |
10188 | 0 | error_setg(errp, "unexpected NVME_SG_ALLOC flag in req->sg.flags"); |
10189 | 0 | return false; |
10190 | 0 | } |
10191 | | |
10192 | 0 | return true; |
10193 | 0 | } |
10194 | | |
10195 | | static bool pre_save_validate_cq_req(NvmeRequest *req, Error **errp) |
10196 | 0 | { |
10197 | 0 | if (!(req->ns == NULL)) { |
10198 | 0 | error_setg(errp, "req->ns != NULL"); |
10199 | 0 | return false; |
10200 | 0 | } |
10201 | | |
10202 | 0 | if (!(req->aiocb == NULL)) { |
10203 | 0 | error_setg(errp, "req->aiocb != NULL"); |
10204 | 0 | return false; |
10205 | 0 | } |
10206 | | |
10207 | 0 | if (!(req->opaque == NULL)) { |
10208 | 0 | error_setg(errp, "req->opaque != NULL"); |
10209 | 0 | return false; |
10210 | 0 | } |
10211 | | |
10212 | 0 | if (!(req->atomic_write == false)) { |
10213 | 0 | error_setg(errp, "req->atomic_write != false"); |
10214 | 0 | return false; |
10215 | 0 | } |
10216 | | |
10217 | 0 | if (req->sg.flags & NVME_SG_ALLOC) { |
10218 | 0 | error_setg(errp, "unexpected NVME_SG_ALLOC flag in req->sg.flags"); |
10219 | 0 | return false; |
10220 | 0 | } |
10221 | | |
10222 | 0 | return true; |
10223 | 0 | } |
10224 | | |
10225 | | static bool nvme_ctrl_pre_save(void *opaque, Error **errp) |
10226 | 0 | { |
10227 | 0 | NvmeCtrl *n = opaque; |
10228 | 0 | int i; |
10229 | |
|
10230 | 0 | trace_pci_nvme_pre_save_enter(n); |
10231 | | |
10232 | | /* |
10233 | | * We don't want to have a race with nvme_process_sq(). |
10234 | | * What implicitly protects us from this is BQL. |
10235 | | */ |
10236 | 0 | assert(bql_locked()); |
10237 | | |
10238 | | /* cancel all SQ processing BHs */ |
10239 | 0 | for (i = 0; i < n->num_queues; i++) { |
10240 | 0 | NvmeSQueue *sq = n->sq[i]; |
10241 | |
|
10242 | 0 | if (!sq) { |
10243 | 0 | continue; |
10244 | 0 | } |
10245 | | |
10246 | 0 | qemu_bh_cancel(sq->bh); |
10247 | 0 | } |
10248 | | |
10249 | | /* drain all IO */ |
10250 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
10251 | 0 | NvmeNamespace *ns; |
10252 | |
|
10253 | 0 | ns = nvme_ns(n, i); |
10254 | 0 | if (!ns) { |
10255 | 0 | continue; |
10256 | 0 | } |
10257 | | |
10258 | 0 | trace_pci_nvme_pre_save_ns_drain(n, i); |
10259 | 0 | nvme_ns_drain(ns); |
10260 | 0 | } |
10261 | | |
10262 | | /* |
10263 | | * Now, we should take care of AERs. |
10264 | | * |
10265 | | * 1. Save all queued events (n->aer_queue). |
10266 | | * This is done automatically, see nvme_vmstate VMStateDescription. |
10267 | | * Here we only need to print them for debugging purpose. |
10268 | | * 2. Go over outstanding AER requests (n->aer_reqs) and check they are |
10269 | | * all have expected opcode (NVME_ADM_CMD_ASYNC_EV_REQ) and other fields. |
10270 | | * |
10271 | | * We must be really careful here, because in case of further |
10272 | | * QEMU NVMe changes, we may break migration without noticing it, or worse, |
10273 | | * introduce silent data corruption during migration. |
10274 | | */ |
10275 | 0 | if (n->aer_queued) { |
10276 | 0 | NvmeAsyncEvent *event; |
10277 | |
|
10278 | 0 | QTAILQ_FOREACH(event, &n->aer_queue, entry) { |
10279 | 0 | trace_pci_nvme_pre_save_aer(event->result.event_type, |
10280 | 0 | event->result.event_info, |
10281 | 0 | event->result.log_page); |
10282 | 0 | } |
10283 | 0 | } |
10284 | |
|
10285 | 0 | for (i = 0; i < n->outstanding_aers; i++) { |
10286 | 0 | NvmeRequest *req = n->aer_reqs[i]; |
10287 | |
|
10288 | 0 | if (!pre_save_validate_aer_req(req, errp)) { |
10289 | 0 | return false; |
10290 | 0 | } |
10291 | 0 | } |
10292 | | |
10293 | | /* |
10294 | | * Make sure that all in-flight IO requests |
10295 | | * (except NVME_ADM_CMD_ASYNC_EV_REQ) are processed. |
10296 | | */ |
10297 | 0 | for (i = 0; i < n->num_queues; i++) { |
10298 | 0 | NvmeRequest *req; |
10299 | 0 | NvmeSQueue *sq = n->sq[i]; |
10300 | |
|
10301 | 0 | if (!sq) { |
10302 | 0 | continue; |
10303 | 0 | } |
10304 | | |
10305 | 0 | trace_pci_nvme_pre_save_sq_out_req_check(n, i, |
10306 | 0 | sq->head, sq->tail, sq->size); |
10307 | |
|
10308 | 0 | QTAILQ_FOREACH(req, &sq->out_req_list, entry) { |
10309 | 0 | assert(req->cmd.opcode == NVME_ADM_CMD_ASYNC_EV_REQ); |
10310 | 0 | } |
10311 | 0 | } |
10312 | | |
10313 | | /* wait when all IO requests completions are written to guest memory */ |
10314 | 0 | for (i = 0; i < n->num_queues; i++) { |
10315 | 0 | NvmeCQueue *cq = n->cq[i]; |
10316 | |
|
10317 | 0 | if (!cq) { |
10318 | 0 | continue; |
10319 | 0 | } |
10320 | | |
10321 | 0 | qemu_bh_cancel(cq->bh); |
10322 | | /* this should empty cq->req_list unless CQ is full */ |
10323 | 0 | nvme_post_cqes(cq); |
10324 | |
|
10325 | 0 | trace_pci_nvme_pre_save_cq_req_check(n, i, |
10326 | 0 | cq->head, cq->tail, cq->size); |
10327 | |
|
10328 | 0 | if (!QTAILQ_EMPTY(&cq->req_list)) { |
10329 | 0 | NvmeRequest *req; |
10330 | |
|
10331 | 0 | assert(nvme_cq_full(cq)); |
10332 | |
|
10333 | 0 | QTAILQ_FOREACH(req, &cq->req_list, entry) { |
10334 | 0 | trace_pci_nvme_pre_save_cq_unposted_cqe( |
10335 | 0 | n, i, nvme_cid(req), |
10336 | 0 | nvme_nsid(req->ns), |
10337 | 0 | le32_to_cpu(req->cqe.result), |
10338 | 0 | le32_to_cpu(req->cqe.dw1), |
10339 | 0 | req->status, req->cmd.opcode); |
10340 | 0 | if (!pre_save_validate_cq_req(req, errp)) { |
10341 | 0 | return false; |
10342 | 0 | } |
10343 | 0 | } |
10344 | 0 | } |
10345 | 0 | } |
10346 | | |
10347 | 0 | for (uint32_t nsid = 0; nsid <= NVME_MAX_NAMESPACES; nsid++) { |
10348 | 0 | NvmeNamespace *ns = n->namespaces[nsid]; |
10349 | |
|
10350 | 0 | if (!ns) { |
10351 | 0 | continue; |
10352 | 0 | } |
10353 | | |
10354 | 0 | if (ns != &n->namespace) { |
10355 | 0 | error_setg(errp, |
10356 | 0 | "only one NVMe namespace is supported for migration"); |
10357 | 0 | return false; |
10358 | 0 | } |
10359 | 0 | } |
10360 | | |
10361 | 0 | return true; |
10362 | 0 | } |
10363 | | |
10364 | | static bool nvme_ctrl_post_load(void *opaque, int version_id, Error **errp) |
10365 | 0 | { |
10366 | 0 | NvmeCtrl *n = opaque; |
10367 | 0 | int i; |
10368 | |
|
10369 | 0 | trace_pci_nvme_post_load_enter(n); |
10370 | | |
10371 | | /* restore CQs first */ |
10372 | 0 | for (i = 0; i < n->num_queues; i++) { |
10373 | 0 | NvmeCQueue *cq = n->cq[i]; |
10374 | |
|
10375 | 0 | if (!cq) { |
10376 | 0 | continue; |
10377 | 0 | } |
10378 | | |
10379 | 0 | if (cq->cqid != i) { |
10380 | 0 | error_setg(errp, "inconsistent migration stream (cq->cqid != i)"); |
10381 | 0 | return false; |
10382 | 0 | } |
10383 | | |
10384 | 0 | cq->ctrl = n; |
10385 | 0 | nvme_restore_cq(cq); |
10386 | 0 | trace_pci_nvme_post_load_restore_cq(n, i, cq->head, cq->tail, cq->size); |
10387 | |
|
10388 | 0 | if (i == 0) { |
10389 | | /* |
10390 | | * Admin CQ lives in n->admin_cq, we don't need |
10391 | | * memory allocated for it in get_ptrs_array_entry() anymore. |
10392 | | * |
10393 | | * nvme_restore_cq() also takes care of: |
10394 | | * n->cq[0] = &n->admin_cq; |
10395 | | * so n->cq[0] remains valid. |
10396 | | */ |
10397 | 0 | g_free(cq); |
10398 | 0 | } |
10399 | 0 | } |
10400 | | |
10401 | 0 | for (i = 0; i < n->num_queues; i++) { |
10402 | 0 | NvmeSQueue *sq = n->sq[i]; |
10403 | |
|
10404 | 0 | if (!sq) { |
10405 | 0 | continue; |
10406 | 0 | } |
10407 | | |
10408 | 0 | if (sq->sqid != i) { |
10409 | 0 | error_setg(errp, "inconsistent migration stream (sq->sqid != i)"); |
10410 | 0 | return false; |
10411 | 0 | } |
10412 | | |
10413 | 0 | if (!n->cq[sq->cqid]) { |
10414 | 0 | error_setg(errp, |
10415 | 0 | "inconsistent migration stream (n->cq[sq->cqid] is NULL)"); |
10416 | 0 | return false; |
10417 | 0 | } |
10418 | | |
10419 | 0 | sq->ctrl = n; |
10420 | 0 | nvme_restore_sq(sq); |
10421 | 0 | trace_pci_nvme_post_load_restore_sq(n, i, sq->head, sq->tail, sq->size); |
10422 | |
|
10423 | 0 | if (i == 0) { |
10424 | | /* same as for CQ */ |
10425 | 0 | g_free(sq); |
10426 | 0 | } |
10427 | 0 | } |
10428 | | |
10429 | | /* restore cq->req_list-s */ |
10430 | 0 | for (i = 0; i < n->num_queues; i++) { |
10431 | 0 | NvmeRequest *req_from, *next; |
10432 | 0 | typeof_field(NvmeCQueue, req_list) req_list; |
10433 | 0 | NvmeCQueue *cq = n->cq[i]; |
10434 | |
|
10435 | 0 | if (!cq || QTAILQ_EMPTY(&cq->req_list)) { |
10436 | 0 | continue; |
10437 | 0 | } |
10438 | | |
10439 | | /* |
10440 | | * We use nvme_vmstate_request VMStateDescription to save/restore |
10441 | | * NvmeRequest structures, but tricky thing here is that |
10442 | | * memory for each cq->req_list item is allocated separately |
10443 | | * during restore. It doesn't work for us. We need to take |
10444 | | * an existing NvmeRequest structure from SQ's req_list pool |
10445 | | * and fill it with data from the newly allocated one (req_from). |
10446 | | * Then, we can safely release allocated memory for it. |
10447 | | */ |
10448 | | |
10449 | | /* make a copy of cq->req_list (QTAILQ head) and clean cq->req_list */ |
10450 | 0 | QTAILQ_INIT(&req_list); |
10451 | 0 | QTAILQ_FOREACH_SAFE(req_from, &cq->req_list, entry, next) { |
10452 | 0 | QTAILQ_REMOVE(&cq->req_list, req_from, entry); |
10453 | 0 | QTAILQ_INSERT_TAIL(&req_list, req_from, entry); |
10454 | 0 | } |
10455 | 0 | QTAILQ_INIT(&cq->req_list); |
10456 | |
|
10457 | 0 | QTAILQ_FOREACH_SAFE(req_from, &req_list, entry, next) { |
10458 | 0 | uint16_t sqid = le16_to_cpu(req_from->cqe.sq_id); |
10459 | 0 | NvmeRequest *req; |
10460 | 0 | NvmeSQueue *sq; |
10461 | |
|
10462 | 0 | assert(!nvme_check_sqid(n, sqid)); |
10463 | 0 | sq = n->sq[sqid]; |
10464 | |
|
10465 | 0 | req = QTAILQ_FIRST(&sq->req_list); |
10466 | 0 | QTAILQ_REMOVE(&sq->req_list, req, entry); |
10467 | 0 | QTAILQ_INSERT_TAIL(&cq->req_list, req, entry); |
10468 | 0 | nvme_req_clear(req); |
10469 | | |
10470 | | /* copy data from the source NvmeRequest */ |
10471 | 0 | req->status = req_from->status; |
10472 | 0 | memcpy(&req->cqe, &req_from->cqe, sizeof(NvmeCqe)); |
10473 | 0 | memcpy(&req->cmd, &req_from->cmd, sizeof(NvmeCmd)); |
10474 | |
|
10475 | 0 | QTAILQ_REMOVE(&req_list, req_from, entry); |
10476 | 0 | g_free(req_from); |
10477 | 0 | } |
10478 | |
|
10479 | 0 | qemu_bh_schedule(cq->bh); |
10480 | 0 | } |
10481 | | |
10482 | 0 | if (n->aer_queued) { |
10483 | 0 | NvmeAsyncEvent *event; |
10484 | |
|
10485 | 0 | QTAILQ_FOREACH(event, &n->aer_queue, entry) { |
10486 | 0 | trace_pci_nvme_post_load_aer(event->result.event_type, |
10487 | 0 | event->result.event_info, |
10488 | 0 | event->result.log_page); |
10489 | 0 | } |
10490 | 0 | } |
10491 | |
|
10492 | 0 | for (i = 0; i < n->outstanding_aers; i++) { |
10493 | 0 | NvmeSQueue *sq = &n->admin_sq; |
10494 | 0 | NvmeRequest *req_from = n->aer_reqs[i]; |
10495 | 0 | NvmeRequest *req; |
10496 | | |
10497 | | /* Idea here is the same as for "restore cq->req_list-s" step */ |
10498 | | |
10499 | | /* take an NvmeRequest struct from SQ */ |
10500 | 0 | req = QTAILQ_FIRST(&sq->req_list); |
10501 | 0 | QTAILQ_REMOVE(&sq->req_list, req, entry); |
10502 | 0 | QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry); |
10503 | 0 | nvme_req_clear(req); |
10504 | | |
10505 | | /* copy data from the source NvmeRequest */ |
10506 | 0 | req->status = req_from->status; |
10507 | 0 | memcpy(&req->cqe, &req_from->cqe, sizeof(NvmeCqe)); |
10508 | 0 | memcpy(&req->cmd, &req_from->cmd, sizeof(NvmeCmd)); |
10509 | |
|
10510 | 0 | n->aer_reqs[i] = req; |
10511 | 0 | g_free(req_from); |
10512 | 0 | } |
10513 | | |
10514 | | /* |
10515 | | * We need to attach namespaces (currently, only one namespace is |
10516 | | * supported for migration). |
10517 | | * This logic comes from nvme_start_ctrl(). |
10518 | | */ |
10519 | 0 | for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { |
10520 | 0 | NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i); |
10521 | |
|
10522 | 0 | if (!ns || (!ns->params.shared && ns->ctrl != n)) { |
10523 | 0 | continue; |
10524 | 0 | } |
10525 | | |
10526 | 0 | if (nvme_csi_supported(n, ns->csi) && !ns->params.detached) { |
10527 | 0 | if (!ns->attached || ns->params.shared) { |
10528 | 0 | nvme_attach_ns(n, ns); |
10529 | 0 | } |
10530 | 0 | } |
10531 | 0 | } |
10532 | | |
10533 | | /* schedule SQ processing */ |
10534 | 0 | for (i = 0; i < n->num_queues; i++) { |
10535 | 0 | NvmeSQueue *sq = n->sq[i]; |
10536 | |
|
10537 | 0 | if (!sq) { |
10538 | 0 | continue; |
10539 | 0 | } |
10540 | | |
10541 | 0 | qemu_bh_schedule(sq->bh); |
10542 | 0 | } |
10543 | |
|
10544 | 0 | return true; |
10545 | 0 | } |
10546 | | |
10547 | | static const VMStateDescription nvme_vmstate = { |
10548 | | .name = "nvme", |
10549 | | .minimum_version_id = 1, |
10550 | | .version_id = 1, |
10551 | | .pre_save_errp = nvme_ctrl_pre_save, |
10552 | | .post_load_errp = nvme_ctrl_post_load, |
10553 | | .fields = (const VMStateField[]) { |
10554 | | VMSTATE_PCI_DEVICE(parent_obj, NvmeCtrl), |
10555 | | VMSTATE_MSIX(parent_obj, NvmeCtrl), |
10556 | | VMSTATE_STRUCT(bar, NvmeCtrl, 0, nvme_vmstate_bar, NvmeBar), |
10557 | | |
10558 | | VMSTATE_BOOL(qs_created, NvmeCtrl), |
10559 | | VMSTATE_UINT32(page_size, NvmeCtrl), |
10560 | | VMSTATE_UINT16(page_bits, NvmeCtrl), |
10561 | | VMSTATE_UINT16(max_prp_ents, NvmeCtrl), |
10562 | | VMSTATE_UINT32(max_q_ents, NvmeCtrl), |
10563 | | VMSTATE_UINT8(outstanding_aers, NvmeCtrl), |
10564 | | VMSTATE_UINT32(irq_status, NvmeCtrl), |
10565 | | VMSTATE_INT32(cq_pending, NvmeCtrl), |
10566 | | |
10567 | | VMSTATE_UINT64(host_timestamp, NvmeCtrl), |
10568 | | VMSTATE_UINT64(timestamp_set_qemu_clock_ms, NvmeCtrl), |
10569 | | VMSTATE_UINT64(starttime_ms, NvmeCtrl), |
10570 | | VMSTATE_UINT16(temperature, NvmeCtrl), |
10571 | | VMSTATE_UINT8(smart_critical_warning, NvmeCtrl), |
10572 | | |
10573 | | VMSTATE_UINT32(conf_msix_qsize, NvmeCtrl), |
10574 | | VMSTATE_UINT32(conf_ioqpairs, NvmeCtrl), |
10575 | | VMSTATE_UINT64(dbbuf_dbs, NvmeCtrl), |
10576 | | VMSTATE_UINT64(dbbuf_eis, NvmeCtrl), |
10577 | | VMSTATE_BOOL(dbbuf_enabled, NvmeCtrl), |
10578 | | |
10579 | | VMSTATE_UINT8(aer_mask, NvmeCtrl), |
10580 | | VMSTATE_VARRAY_OF_POINTER_TO_STRUCT_UINT8_ALLOC( |
10581 | | aer_reqs, NvmeCtrl, outstanding_aers, 0, |
10582 | | nvme_vmstate_request, NvmeRequest), |
10583 | | VMSTATE_QTAILQ_V(aer_queue, NvmeCtrl, 1, nvme_vmstate_async_event, |
10584 | | NvmeAsyncEvent, entry), |
10585 | | VMSTATE_INT32(aer_queued, NvmeCtrl), |
10586 | | |
10587 | | VMSTATE_STRUCT(namespace, NvmeCtrl, 0, nvme_vmstate_ns, NvmeNamespace), |
10588 | | |
10589 | | VMSTATE_VARRAY_OF_POINTER_TO_STRUCT_UINT32_ALLOC( |
10590 | | sq, NvmeCtrl, num_queues, 0, nvme_vmstate_squeue, NvmeSQueue), |
10591 | | VMSTATE_VARRAY_OF_POINTER_TO_STRUCT_UINT32_ALLOC( |
10592 | | cq, NvmeCtrl, num_queues, 0, nvme_vmstate_cqueue, NvmeCQueue), |
10593 | | |
10594 | | VMSTATE_UINT16(features.temp_thresh_hi, NvmeCtrl), |
10595 | | VMSTATE_UINT16(features.temp_thresh_low, NvmeCtrl), |
10596 | | VMSTATE_UINT32(features.async_config, NvmeCtrl), |
10597 | | VMSTATE_STRUCT(features.hbs, NvmeCtrl, 0, |
10598 | | nvme_vmstate_hbs, NvmeHostBehaviorSupport), |
10599 | | |
10600 | | VMSTATE_UINT32(dn, NvmeCtrl), |
10601 | | VMSTATE_STRUCT(atomic, NvmeCtrl, 0, nvme_vmstate_atomic, NvmeAtomic), |
10602 | | |
10603 | | VMSTATE_END_OF_LIST() |
10604 | | }, |
10605 | | }; |
10606 | | |
10607 | | static void nvme_class_init(ObjectClass *oc, const void *data) |
10608 | 63 | { |
10609 | 63 | DeviceClass *dc = DEVICE_CLASS(oc); |
10610 | 63 | PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc); |
10611 | | |
10612 | 63 | pc->realize = nvme_realize; |
10613 | 63 | pc->config_write = nvme_pci_write_config; |
10614 | 63 | pc->config_read = nvme_pci_read_config; |
10615 | 63 | pc->exit = nvme_exit; |
10616 | 63 | pc->class_id = PCI_CLASS_STORAGE_EXPRESS; |
10617 | 63 | pc->revision = 2; |
10618 | | |
10619 | 63 | set_bit(DEVICE_CATEGORY_STORAGE, dc->categories); |
10620 | 63 | dc->desc = "Non-Volatile Memory Express"; |
10621 | 63 | device_class_set_props(dc, nvme_props); |
10622 | 63 | dc->vmsd = &nvme_vmstate; |
10623 | 63 | device_class_set_legacy_reset(dc, nvme_pci_reset); |
10624 | 63 | } |
10625 | | |
10626 | | static void nvme_instance_init(Object *obj) |
10627 | 0 | { |
10628 | 0 | NvmeCtrl *n = NVME(obj); |
10629 | |
|
10630 | 0 | device_add_bootindex_property(obj, &n->namespace.blkconf.bootindex, |
10631 | 0 | "bootindex", "/namespace@1,0", |
10632 | 0 | DEVICE(obj)); |
10633 | |
|
10634 | 0 | object_property_add(obj, "smart_critical_warning", "uint8", |
10635 | 0 | &uint8_type_info, |
10636 | 0 | nvme_get_smart_warning, |
10637 | 0 | nvme_set_smart_warning, NULL, NULL); |
10638 | 0 | } |
10639 | | |
10640 | | static const TypeInfo nvme_info = { |
10641 | | .name = TYPE_NVME, |
10642 | | .parent = TYPE_PCI_DEVICE, |
10643 | | .instance_size = sizeof(NvmeCtrl), |
10644 | | .instance_init = nvme_instance_init, |
10645 | | .class_init = nvme_class_init, |
10646 | | .interfaces = (const InterfaceInfo[]) { |
10647 | | { INTERFACE_PCIE_DEVICE }, |
10648 | | { } |
10649 | | }, |
10650 | | }; |
10651 | | |
10652 | | static void nvme_ns_hot_plug(const HotplugHandler *hotplug_dev, |
10653 | | DeviceState *dev, Error **errp) |
10654 | 0 | { |
10655 | 0 | NvmeNamespace *ns = NVME_NS(dev); |
10656 | 0 | NvmeSubsystem *subsys = ns->subsys; |
10657 | 0 | uint32_t nsid = ns->params.nsid; |
10658 | 0 | int i; |
10659 | | |
10660 | | /* |
10661 | | * Attach to all started controllers and notify via AEN. |
10662 | | * Skip controllers that haven't started yet (boot-time realize) — |
10663 | | * nvme_start_ctrl() will attach namespaces during controller init. |
10664 | | */ |
10665 | 0 | for (i = 0; i < NVME_MAX_CONTROLLERS; i++) { |
10666 | 0 | NvmeCtrl *ctrl = nvme_subsys_ctrl(subsys, i); |
10667 | 0 | if (!ctrl || !ctrl->qs_created) { |
10668 | 0 | continue; |
10669 | 0 | } |
10670 | | |
10671 | 0 | if (nvme_csi_supported(ctrl, ns->csi) && !ns->params.detached) { |
10672 | 0 | nvme_attach_ns(ctrl, ns); |
10673 | 0 | nvme_update_dsm_limits(ctrl, ns); |
10674 | |
|
10675 | 0 | if (!test_and_set_bit(nsid, ctrl->changed_nsids)) { |
10676 | 0 | nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE, |
10677 | 0 | NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED, |
10678 | 0 | NVME_LOG_CHANGED_NSLIST); |
10679 | 0 | } |
10680 | 0 | } |
10681 | 0 | } |
10682 | 0 | } |
10683 | | |
10684 | | static void nvme_ns_hot_unplug(const HotplugHandler *hotplug_dev, |
10685 | | DeviceState *dev, Error **errp) |
10686 | 0 | { |
10687 | 0 | NvmeNamespace *ns = NVME_NS(dev); |
10688 | 0 | NvmeSubsystem *subsys = ns->subsys; |
10689 | 0 | uint32_t nsid = ns->params.nsid; |
10690 | 0 | int i; |
10691 | | |
10692 | | /* |
10693 | | * Drain in-flight I/O before tearing down the namespace. |
10694 | | * This must happen while the namespace is still attached to the |
10695 | | * controllers so any pending requests can complete normally. |
10696 | | */ |
10697 | 0 | nvme_ns_drain(ns); |
10698 | | |
10699 | | /* |
10700 | | * Detach from all controllers and notify the guest via AEN. |
10701 | | * The guest kernel will rescan namespaces and remove the block device. |
10702 | | */ |
10703 | 0 | for (i = 0; i < NVME_MAX_CONTROLLERS; i++) { |
10704 | 0 | NvmeCtrl *ctrl = nvme_subsys_ctrl(subsys, i); |
10705 | 0 | if (!ctrl || !nvme_ns(ctrl, nsid)) { |
10706 | 0 | continue; |
10707 | 0 | } |
10708 | | |
10709 | 0 | nvme_detach_ns(ctrl, ns); |
10710 | 0 | nvme_update_dsm_limits(ctrl, NULL); |
10711 | |
|
10712 | 0 | if (!test_and_set_bit(nsid, ctrl->changed_nsids)) { |
10713 | 0 | nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE, |
10714 | 0 | NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED, |
10715 | 0 | NVME_LOG_CHANGED_NSLIST); |
10716 | 0 | } |
10717 | 0 | } |
10718 | | |
10719 | | /* |
10720 | | * Unrealize: removes from subsystem (in nvme_ns_unrealize), flushes, |
10721 | | * cleans up structures, and removes from QOM. |
10722 | | */ |
10723 | 0 | qdev_unrealize(dev); |
10724 | 0 | } |
10725 | | |
10726 | | static void nvme_bus_class_init(ObjectClass *klass, const void *data) |
10727 | 63 | { |
10728 | 63 | HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass); |
10729 | 63 | hc->plug = nvme_ns_hot_plug; |
10730 | 63 | hc->unplug = nvme_ns_hot_unplug; |
10731 | 63 | } |
10732 | | |
10733 | | static const TypeInfo nvme_bus_info = { |
10734 | | .name = TYPE_NVME_BUS, |
10735 | | .parent = TYPE_BUS, |
10736 | | .instance_size = sizeof(NvmeBus), |
10737 | | .class_init = nvme_bus_class_init, |
10738 | | .interfaces = (const InterfaceInfo[]) { |
10739 | | { TYPE_HOTPLUG_HANDLER }, |
10740 | | { } |
10741 | | }, |
10742 | | }; |
10743 | | |
10744 | | static void nvme_register_types(void) |
10745 | 63 | { |
10746 | 63 | type_register_static(&nvme_info); |
10747 | 63 | type_register_static(&nvme_bus_info); |
10748 | 63 | } |
10749 | | |
10750 | | type_init(nvme_register_types) |