Coverage Report

Created: 2026-11-20 08:13

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qemu/block/io.c
Line
Count
Source
1
/*
2
 * Block layer I/O functions
3
 *
4
 * Copyright (c) 2003 Fabrice Bellard
5
 *
6
 * Permission is hereby granted, free of charge, to any person obtaining a copy
7
 * of this software and associated documentation files (the "Software"), to deal
8
 * in the Software without restriction, including without limitation the rights
9
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
 * copies of the Software, and to permit persons to whom the Software is
11
 * furnished to do so, subject to the following conditions:
12
 *
13
 * The above copyright notice and this permission notice shall be included in
14
 * all copies or substantial portions of the Software.
15
 *
16
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22
 * THE SOFTWARE.
23
 */
24
25
#include "qemu/osdep.h"
26
#include "trace.h"
27
#include "system/block-backend.h"
28
#include "qemu/aio-wait.h"
29
#include "block/blockjob.h"
30
#include "block/blockjob_int.h"
31
#include "block/block_int.h"
32
#include "block/coroutines.h"
33
#include "block/dirty-bitmap.h"
34
#include "block/write-threshold.h"
35
#include "qemu/cutils.h"
36
#include "qemu/memalign.h"
37
#include "qapi/error.h"
38
#include "qemu/error-report.h"
39
#include "qemu/main-loop.h"
40
#include "system/replay.h"
41
#include "qemu/units.h"
42
#include "qemu/atomic.h"
43
44
/* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
45
#define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
46
47
/* Maximum read size for checking if data reads as zero, in bytes */
48
0
#define MAX_ZERO_CHECK_BUFFER (128 * KiB)
49
50
static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
51
    int64_t offset, int64_t bytes, BdrvRequestFlags flags);
52
53
static void coroutine_fn
54
bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self);
55
56
static void GRAPH_RDLOCK
57
bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore)
58
288k
{
59
288k
    BdrvChild *c, *next;
60
288k
    IO_OR_GS_CODE();
61
288k
    assert_bdrv_graph_readable();
62
63
288k
    QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
64
288k
        if (c == ignore) {
65
0
            continue;
66
0
        }
67
288k
        bdrv_parent_drained_begin_single(c);
68
288k
    }
69
288k
}
70
71
void bdrv_parent_drained_end_single(BdrvChild *c)
72
288k
{
73
288k
    GLOBAL_STATE_CODE();
74
75
288k
    assert(c->quiesced_parent);
76
288k
    c->quiesced_parent = false;
77
78
288k
    if (c->klass->drained_end) {
79
288k
        c->klass->drained_end(c);
80
288k
    }
81
288k
}
82
83
static void GRAPH_RDLOCK
84
bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore)
85
288k
{
86
288k
    BdrvChild *c;
87
288k
    IO_OR_GS_CODE();
88
288k
    assert_bdrv_graph_readable();
89
90
288k
    QLIST_FOREACH(c, &bs->parents, next_parent) {
91
288k
        if (c == ignore) {
92
0
            continue;
93
0
        }
94
288k
        bdrv_parent_drained_end_single(c);
95
288k
    }
96
288k
}
97
98
bool bdrv_parent_drained_poll_single(BdrvChild *c)
99
203k
{
100
203k
    IO_OR_GS_CODE();
101
102
203k
    if (c->klass->drained_poll) {
103
203k
        return c->klass->drained_poll(c);
104
203k
    }
105
0
    return false;
106
203k
}
107
108
static bool GRAPH_RDLOCK
109
bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
110
                         bool ignore_bds_parents)
111
289k
{
112
289k
    BdrvChild *c, *next;
113
289k
    bool busy = false;
114
289k
    IO_OR_GS_CODE();
115
289k
    assert_bdrv_graph_readable();
116
117
289k
    QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
118
289k
        if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
119
85.7k
            continue;
120
85.7k
        }
121
203k
        busy |= bdrv_parent_drained_poll_single(c);
122
203k
    }
123
124
289k
    return busy;
125
289k
}
126
127
void bdrv_parent_drained_begin_single(BdrvChild *c)
128
288k
{
129
288k
    GLOBAL_STATE_CODE();
130
131
288k
    assert(!c->quiesced_parent);
132
288k
    c->quiesced_parent = true;
133
134
288k
    if (c->klass->drained_begin) {
135
        /* called with rdlock taken, but it doesn't really need it. */
136
288k
        c->klass->drained_begin(c);
137
288k
    }
138
288k
}
139
140
static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
141
15
{
142
15
    dst->pdiscard_alignment = MAX(dst->pdiscard_alignment,
143
15
                                  src->pdiscard_alignment);
144
15
    dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
145
15
    dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
146
15
    dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer,
147
15
                                        src->max_hw_transfer);
148
15
    dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
149
15
                                 src->opt_mem_alignment);
150
15
    dst->min_mem_alignment = MAX(dst->min_mem_alignment,
151
15
                                 src->min_mem_alignment);
152
15
    dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
153
15
    dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov);
154
15
}
155
156
typedef struct BdrvRefreshLimitsState {
157
    BlockDriverState *bs;
158
    BlockLimits old_bl;
159
} BdrvRefreshLimitsState;
160
161
static void bdrv_refresh_limits_abort(void *opaque)
162
0
{
163
0
    BdrvRefreshLimitsState *s = opaque;
164
165
0
    s->bs->bl = s->old_bl;
166
0
}
167
168
static TransactionActionDrv bdrv_refresh_limits_drv = {
169
    .abort = bdrv_refresh_limits_abort,
170
    .clean = g_free,
171
};
172
173
/* @tran is allowed to be NULL, in this case no rollback is possible. */
174
void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp)
175
31
{
176
31
    ERRP_GUARD();
177
31
    BlockDriver *drv = bs->drv;
178
31
    BdrvChild *c;
179
31
    bool have_limits;
180
181
31
    GLOBAL_STATE_CODE();
182
183
31
    if (tran) {
184
0
        BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1);
185
0
        *s = (BdrvRefreshLimitsState) {
186
0
            .bs = bs,
187
0
            .old_bl = bs->bl,
188
0
        };
189
0
        tran_add(tran, &bdrv_refresh_limits_drv, s);
190
0
    }
191
192
31
    memset(&bs->bl, 0, sizeof(bs->bl));
193
194
31
    if (!drv) {
195
0
        return;
196
0
    }
197
198
    /* Default alignment based on whether driver has byte interface */
199
31
    bs->bl.request_alignment = (drv->bdrv_co_preadv ||
200
0
                                drv->bdrv_aio_preadv ||
201
31
                                drv->bdrv_co_preadv_part) ? 1 : 512;
202
203
    /* Take some limits from the children as a default */
204
31
    have_limits = false;
205
31
    QLIST_FOREACH(c, &bs->children, next) {
206
15
        if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW))
207
15
        {
208
15
            bdrv_merge_limits(&bs->bl, &c->bs->bl);
209
15
            have_limits = true;
210
15
        }
211
212
15
        if (c->role & BDRV_CHILD_FILTERED) {
213
15
            bs->bl.has_variable_length |= c->bs->bl.has_variable_length;
214
15
        }
215
15
    }
216
217
31
    if (!have_limits) {
218
16
        bs->bl.min_mem_alignment = 512;
219
16
        bs->bl.opt_mem_alignment = qemu_real_host_page_size();
220
221
        /* Safe default since most protocols use readv()/writev()/etc */
222
16
        bs->bl.max_iov = IOV_MAX;
223
16
    }
224
225
    /* Then let the driver override it */
226
31
    if (drv->bdrv_refresh_limits) {
227
15
        drv->bdrv_refresh_limits(bs, errp);
228
15
        if (*errp) {
229
0
            return;
230
0
        }
231
15
    }
232
233
31
    if (bs->bl.zone_size) {
234
0
        assert(is_power_of_2(bs->bl.zone_size));
235
0
        bs->bl.zone_size_bits = ctz64(bs->bl.zone_size);
236
0
    }
237
238
31
    if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) {
239
0
        error_setg(errp, "Driver requires too large request alignment");
240
0
    }
241
31
}
242
243
/**
244
 * The copy-on-read flag is actually a reference count so multiple users may
245
 * use the feature without worrying about clobbering its previous state.
246
 * Copy-on-read stays enabled until all users have called to disable it.
247
 */
248
void bdrv_enable_copy_on_read(BlockDriverState *bs)
249
0
{
250
0
    IO_CODE();
251
0
    qatomic_inc(&bs->copy_on_read);
252
0
}
253
254
void bdrv_disable_copy_on_read(BlockDriverState *bs)
255
0
{
256
0
    int old = qatomic_fetch_dec(&bs->copy_on_read);
257
0
    IO_CODE();
258
0
    assert(old >= 1);
259
0
}
260
261
typedef struct {
262
    Coroutine *co;
263
    BlockDriverState *bs;
264
    bool done;
265
    bool begin;
266
    bool poll;
267
    BdrvChild *parent;
268
} BdrvCoDrainData;
269
270
/* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
271
bool bdrv_drain_poll(BlockDriverState *bs, BdrvChild *ignore_parent,
272
                     bool ignore_bds_parents)
273
289k
{
274
289k
    GLOBAL_STATE_CODE();
275
276
289k
    if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
277
908
        return true;
278
908
    }
279
280
288k
    if (qatomic_read(&bs->in_flight)) {
281
0
        return true;
282
0
    }
283
284
288k
    return false;
285
288k
}
286
287
static bool bdrv_drain_poll_top_level(BlockDriverState *bs,
288
                                      BdrvChild *ignore_parent)
289
118k
{
290
118k
    GLOBAL_STATE_CODE();
291
118k
    GRAPH_RDLOCK_GUARD_MAINLOOP();
292
293
118k
    return bdrv_drain_poll(bs, ignore_parent, false);
294
118k
}
295
296
static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
297
                                  bool poll);
298
static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent);
299
300
static void bdrv_co_drain_bh_cb(void *opaque)
301
0
{
302
0
    BdrvCoDrainData *data = opaque;
303
0
    Coroutine *co = data->co;
304
0
    BlockDriverState *bs = data->bs;
305
306
0
    if (bs) {
307
0
        bdrv_dec_in_flight(bs);
308
0
        if (data->begin) {
309
0
            bdrv_do_drained_begin(bs, data->parent, data->poll);
310
0
        } else {
311
0
            assert(!data->poll);
312
0
            bdrv_do_drained_end(bs, data->parent);
313
0
        }
314
0
    } else {
315
0
        assert(data->begin);
316
0
        bdrv_drain_all_begin();
317
0
    }
318
319
0
    data->done = true;
320
0
    aio_co_wake(co);
321
0
}
322
323
static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
324
                                                bool begin,
325
                                                BdrvChild *parent,
326
                                                bool poll)
327
0
{
328
0
    BdrvCoDrainData data;
329
0
    Coroutine *self = qemu_coroutine_self();
330
331
    /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
332
     * other coroutines run if they were queued by aio_co_enter(). */
333
334
0
    assert(qemu_in_coroutine());
335
0
    data = (BdrvCoDrainData) {
336
0
        .co = self,
337
0
        .bs = bs,
338
0
        .done = false,
339
0
        .begin = begin,
340
0
        .parent = parent,
341
0
        .poll = poll,
342
0
    };
343
344
0
    if (bs) {
345
0
        bdrv_inc_in_flight(bs);
346
0
    }
347
348
0
    replay_bh_schedule_oneshot_event(qemu_get_aio_context(),
349
0
                                     bdrv_co_drain_bh_cb, &data);
350
351
0
    qemu_coroutine_yield();
352
    /* If we are resumed from some other event (such as an aio completion or a
353
     * timer callback), it is a bug in the caller that should be fixed. */
354
0
    assert(data.done);
355
0
}
356
357
static void coroutine_mixed_fn
358
bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
359
                      bool poll)
360
374k
{
361
374k
    IO_OR_GS_CODE();
362
363
374k
    if (qemu_in_coroutine()) {
364
0
        bdrv_co_yield_to_drain(bs, true, parent, poll);
365
0
        return;
366
0
    }
367
368
374k
    GLOBAL_STATE_CODE();
369
370
    /* Stop things in parent-to-child order */
371
374k
    if (bs->quiesce_counter++ == 0) {
372
288k
        GRAPH_RDLOCK_GUARD_MAINLOOP();
373
288k
        bdrv_parent_drained_begin(bs, parent);
374
288k
        if (bs->drv && bs->drv->bdrv_drain_begin) {
375
0
            bs->drv->bdrv_drain_begin(bs);
376
0
        }
377
288k
    }
378
379
    /*
380
     * Wait for drained requests to finish.
381
     *
382
     * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
383
     * call is needed so things in this AioContext can make progress even
384
     * though we don't return to the main AioContext loop - this automatically
385
     * includes other nodes in the same AioContext and therefore all child
386
     * nodes.
387
     */
388
374k
    if (poll) {
389
117k
        BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, parent));
390
117k
    }
391
374k
}
392
393
void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, BdrvChild *parent)
394
85.5k
{
395
85.5k
    bdrv_do_drained_begin(bs, parent, false);
396
85.5k
}
397
398
void coroutine_mixed_fn
399
bdrv_drained_begin(BlockDriverState *bs)
400
117k
{
401
117k
    IO_OR_GS_CODE();
402
117k
    bdrv_do_drained_begin(bs, NULL, true);
403
117k
}
404
405
/**
406
 * This function does not poll, nor must any of its recursively called
407
 * functions.
408
 */
409
static void coroutine_mixed_fn
410
bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent)
411
374k
{
412
374k
    IO_OR_GS_CODE();
413
414
374k
    if (qemu_in_coroutine()) {
415
0
        bdrv_co_yield_to_drain(bs, false, parent, false);
416
0
        return;
417
0
    }
418
419
    /* At this point, we should be always running in the main loop. */
420
374k
    GLOBAL_STATE_CODE();
421
374k
    assert(bs->quiesce_counter > 0);
422
423
    /* Re-enable things in child-to-parent order */
424
374k
    if (--bs->quiesce_counter == 0) {
425
288k
        GRAPH_RDLOCK_GUARD_MAINLOOP();
426
288k
        if (bs->drv && bs->drv->bdrv_drain_end) {
427
0
            bs->drv->bdrv_drain_end(bs);
428
0
        }
429
288k
        bdrv_parent_drained_end(bs, parent);
430
288k
    }
431
374k
}
432
433
void bdrv_drained_end(BlockDriverState *bs)
434
203k
{
435
203k
    IO_OR_GS_CODE();
436
203k
    bdrv_do_drained_end(bs, NULL);
437
203k
}
438
439
void bdrv_drain(BlockDriverState *bs)
440
0
{
441
0
    IO_OR_GS_CODE();
442
0
    bdrv_drained_begin(bs);
443
0
    bdrv_drained_end(bs);
444
0
}
445
446
static void bdrv_drain_assert_idle(BlockDriverState *bs)
447
256k
{
448
256k
    BdrvChild *child, *next;
449
256k
    GLOBAL_STATE_CODE();
450
256k
    GRAPH_RDLOCK_GUARD_MAINLOOP();
451
452
256k
    assert(qatomic_read(&bs->in_flight) == 0);
453
256k
    QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
454
85.4k
        bdrv_drain_assert_idle(child->bs);
455
85.4k
    }
456
256k
}
457
458
unsigned int bdrv_drain_all_count = 0;
459
460
static bool bdrv_drain_all_poll(void)
461
85.7k
{
462
85.7k
    BlockDriverState *bs = NULL;
463
85.7k
    bool result = false;
464
465
85.7k
    GLOBAL_STATE_CODE();
466
85.7k
    GRAPH_RDLOCK_GUARD_MAINLOOP();
467
468
    /*
469
     * bdrv_drain_poll() can't make changes to the graph and we hold the BQL,
470
     * so iterating bdrv_next_all_states() is safe.
471
     */
472
257k
    while ((bs = bdrv_next_all_states(bs))) {
473
171k
        result |= bdrv_drain_poll(bs, NULL, true);
474
171k
    }
475
476
85.7k
    return result;
477
85.7k
}
478
479
/*
480
 * Wait for pending requests to complete across all BlockDriverStates
481
 *
482
 * This function does not flush data to disk, use bdrv_flush_all() for that
483
 * after calling this function.
484
 *
485
 * This pauses all block jobs and disables external clients. It must
486
 * be paired with bdrv_drain_all_end().
487
 *
488
 * NOTE: no new block jobs or BlockDriverStates can be created between
489
 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
490
 */
491
void bdrv_drain_all_begin_nopoll(void)
492
85.5k
{
493
85.5k
    BlockDriverState *bs = NULL;
494
85.5k
    GLOBAL_STATE_CODE();
495
496
    /*
497
     * bdrv queue is managed by record/replay,
498
     * waiting for finishing the I/O requests may
499
     * be infinite
500
     */
501
85.5k
    if (replay_events_enabled()) {
502
0
        return;
503
0
    }
504
505
    /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
506
     * loop AioContext, so make sure we're in the main context. */
507
85.5k
    assert(qemu_get_current_aio_context() == qemu_get_aio_context());
508
85.5k
    assert(bdrv_drain_all_count < INT_MAX);
509
85.5k
    bdrv_drain_all_count++;
510
511
    /* Quiesce all nodes, without polling in-flight requests yet. The graph
512
     * cannot change during this loop. */
513
256k
    while ((bs = bdrv_next_all_states(bs))) {
514
170k
        bdrv_do_drained_begin(bs, NULL, false);
515
170k
    }
516
85.5k
}
517
518
void coroutine_mixed_fn bdrv_drain_all_begin(void)
519
85.5k
{
520
85.5k
    BlockDriverState *bs = NULL;
521
522
85.5k
    if (qemu_in_coroutine()) {
523
0
        bdrv_co_yield_to_drain(NULL, true, NULL, true);
524
0
        return;
525
0
    }
526
527
    /*
528
     * bdrv queue is managed by record/replay,
529
     * waiting for finishing the I/O requests may
530
     * be infinite
531
     */
532
85.5k
    if (replay_events_enabled()) {
533
0
        return;
534
0
    }
535
536
85.5k
    bdrv_drain_all_begin_nopoll();
537
538
    /* Now poll the in-flight requests */
539
85.5k
    AIO_WAIT_WHILE_UNLOCKED(NULL, bdrv_drain_all_poll());
540
541
256k
    while ((bs = bdrv_next_all_states(bs))) {
542
170k
        bdrv_drain_assert_idle(bs);
543
170k
    }
544
85.5k
}
545
546
void bdrv_drain_all_end_quiesce(BlockDriverState *bs)
547
0
{
548
0
    GLOBAL_STATE_CODE();
549
550
0
    g_assert(bs->quiesce_counter > 0);
551
0
    g_assert(!bs->refcnt);
552
553
0
    while (bs->quiesce_counter) {
554
0
        bdrv_do_drained_end(bs, NULL);
555
0
    }
556
0
}
557
558
void bdrv_drain_all_end(void)
559
85.5k
{
560
85.5k
    BlockDriverState *bs = NULL;
561
85.5k
    GLOBAL_STATE_CODE();
562
563
    /*
564
     * bdrv queue is managed by record/replay,
565
     * waiting for finishing the I/O requests may
566
     * be endless
567
     */
568
85.5k
    if (replay_events_enabled()) {
569
0
        return;
570
0
    }
571
572
256k
    while ((bs = bdrv_next_all_states(bs))) {
573
170k
        bdrv_do_drained_end(bs, NULL);
574
170k
    }
575
576
85.5k
    assert(qemu_get_current_aio_context() == qemu_get_aio_context());
577
85.5k
    assert(bdrv_drain_all_count > 0);
578
85.5k
    bdrv_drain_all_count--;
579
85.5k
}
580
581
void bdrv_drain_all(void)
582
0
{
583
0
    GLOBAL_STATE_CODE();
584
0
    bdrv_drain_all_begin();
585
0
    bdrv_drain_all_end();
586
0
}
587
588
/**
589
 * Remove an active request from the tracked requests list
590
 *
591
 * This function should be called when a tracked request is completing.
592
 */
593
static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req)
594
40.2M
{
595
40.2M
    if (req->serialising) {
596
0
        qatomic_dec(&req->bs->serialising_in_flight);
597
0
    }
598
599
40.2M
    qemu_mutex_lock(&req->bs->reqs_lock);
600
40.2M
    QLIST_REMOVE(req, list);
601
40.2M
    qemu_mutex_unlock(&req->bs->reqs_lock);
602
603
    /*
604
     * At this point qemu_co_queue_wait(&req->wait_queue, ...) won't be called
605
     * anymore because the request has been removed from the list, so it's safe
606
     * to restart the queue outside reqs_lock to minimize the critical section.
607
     */
608
40.2M
    qemu_co_queue_restart_all(&req->wait_queue);
609
40.2M
}
610
611
/**
612
 * Add an active request to the tracked requests list
613
 */
614
static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req,
615
                                               BlockDriverState *bs,
616
                                               int64_t offset,
617
                                               int64_t bytes,
618
                                               enum BdrvTrackedRequestType type)
619
40.2M
{
620
40.2M
    bdrv_check_request(offset, bytes, &error_abort);
621
622
40.2M
    *req = (BdrvTrackedRequest){
623
40.2M
        .bs = bs,
624
40.2M
        .offset         = offset,
625
40.2M
        .bytes          = bytes,
626
40.2M
        .type           = type,
627
40.2M
        .co             = qemu_coroutine_self(),
628
40.2M
        .serialising    = false,
629
40.2M
        .active_owner   = false,
630
40.2M
        .overlap_offset = offset,
631
40.2M
        .overlap_bytes  = bytes,
632
40.2M
    };
633
634
40.2M
    qemu_co_queue_init(&req->wait_queue);
635
636
40.2M
    qemu_mutex_lock(&bs->reqs_lock);
637
40.2M
    QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
638
639
    /*
640
     * Set .waiting_for while we're holding the lock, otherwise we'd open a race
641
     * window where another request in the middle of its RMW cycle starts
642
     * waiting for us, leading to potential data corruption if this request
643
     * overwrites a block the RMW request has already read.
644
     */
645
40.2M
    bdrv_wait_serialising_requests_locked(req);
646
647
40.2M
    qemu_mutex_unlock(&bs->reqs_lock);
648
40.2M
}
649
650
static bool tracked_request_overlaps(BdrvTrackedRequest *req,
651
                                     int64_t offset, int64_t bytes)
652
0
{
653
0
    bdrv_check_request(offset, bytes, &error_abort);
654
655
    /*        aaaa   bbbb */
656
0
    if (offset >= req->overlap_offset + req->overlap_bytes) {
657
0
        return false;
658
0
    }
659
    /* bbbb   aaaa        */
660
0
    if (req->overlap_offset >= offset + bytes) {
661
0
        return false;
662
0
    }
663
0
    return true;
664
0
}
665
666
/* Called with self->bs->reqs_lock held */
667
static coroutine_fn BdrvTrackedRequest *
668
bdrv_find_conflicting_request(BdrvTrackedRequest *self)
669
40.2M
{
670
40.2M
    BdrvTrackedRequest *req;
671
672
40.2M
    QLIST_FOREACH(req, &self->bs->tracked_requests, list) {
673
40.2M
        if (req == self || (!req->serialising && !self->serialising)) {
674
40.2M
            continue;
675
40.2M
        }
676
0
        if (tracked_request_overlaps(req, self->overlap_offset,
677
0
                                     self->overlap_bytes))
678
0
        {
679
            /*
680
             * Hitting this means there was a reentrant request, for
681
             * example, a block driver issuing nested requests.  This must
682
             * never happen since it means deadlock.
683
             */
684
0
            assert(qemu_coroutine_self() != req->co);
685
686
            /*
687
             * If the request is already (indirectly) waiting for us, or
688
             * will wait for us as soon as it wakes up, then just go on
689
             * (instead of producing a deadlock in the former case).
690
             */
691
0
            if (!req->waiting_for) {
692
0
                return req;
693
0
            }
694
0
        }
695
0
    }
696
697
40.2M
    return NULL;
698
40.2M
}
699
700
/* Called with self->bs->reqs_lock held */
701
static void coroutine_fn
702
bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self)
703
40.2M
{
704
40.2M
    BdrvTrackedRequest *req;
705
706
40.2M
    while ((req = bdrv_find_conflicting_request(self))) {
707
        /*
708
         * If this request is the active owner of the data blocks it accesses,
709
         * no other conflicting request may exist; other requests have to wait
710
         * for this one, not the other way around. If this condition is
711
         * violated, RMW operations may be interrupted in the middle, operate on
712
         * stale data and introduce data corruption.
713
         */
714
0
        assert(!self->active_owner);
715
716
0
        self->waiting_for = req;
717
0
        qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock);
718
0
        self->waiting_for = NULL;
719
0
    }
720
40.2M
}
721
722
/* Called with req->bs->reqs_lock held */
723
static void tracked_request_set_serialising(BdrvTrackedRequest *req,
724
                                            uint64_t align)
725
0
{
726
0
    int64_t overlap_offset = req->offset & ~(align - 1);
727
0
    int64_t overlap_bytes =
728
0
        ROUND_UP(req->offset + req->bytes, align) - overlap_offset;
729
730
0
    bdrv_check_request(req->offset, req->bytes, &error_abort);
731
732
0
    if (!req->serialising) {
733
0
        qatomic_inc(&req->bs->serialising_in_flight);
734
0
        req->serialising = true;
735
0
    }
736
737
0
    req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
738
0
    req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
739
0
}
740
741
/**
742
 * Return the tracked request on @bs for the current coroutine, or
743
 * NULL if there is none.
744
 */
745
BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
746
0
{
747
0
    BdrvTrackedRequest *req;
748
0
    Coroutine *self = qemu_coroutine_self();
749
0
    IO_CODE();
750
751
0
    QEMU_LOCK_GUARD(&bs->reqs_lock);
752
0
    QLIST_FOREACH(req, &bs->tracked_requests, list) {
753
0
        if (req->co == self) {
754
0
            return req;
755
0
        }
756
0
    }
757
758
0
    return NULL;
759
0
}
760
761
/**
762
 * Round a region to subcluster (if supported) or cluster boundaries
763
 */
764
void coroutine_fn GRAPH_RDLOCK
765
bdrv_round_to_subclusters(BlockDriverState *bs, int64_t offset, int64_t bytes,
766
                          int64_t *align_offset, int64_t *align_bytes)
767
0
{
768
0
    BlockDriverInfo bdi;
769
0
    IO_CODE();
770
0
    if (bdrv_co_get_info(bs, &bdi) < 0 || bdi.subcluster_size == 0) {
771
0
        *align_offset = offset;
772
0
        *align_bytes = bytes;
773
0
    } else {
774
0
        int64_t c = bdi.subcluster_size;
775
0
        *align_offset = QEMU_ALIGN_DOWN(offset, c);
776
0
        *align_bytes = QEMU_ALIGN_UP(offset - *align_offset + bytes, c);
777
0
    }
778
0
}
779
780
static int coroutine_fn GRAPH_RDLOCK bdrv_get_cluster_size(BlockDriverState *bs)
781
0
{
782
0
    BlockDriverInfo bdi;
783
0
    int ret;
784
785
0
    ret = bdrv_co_get_info(bs, &bdi);
786
0
    if (ret < 0 || bdi.cluster_size == 0) {
787
0
        return bs->bl.request_alignment;
788
0
    } else {
789
0
        return bdi.cluster_size;
790
0
    }
791
0
}
792
793
void bdrv_inc_in_flight(BlockDriverState *bs)
794
60.4M
{
795
60.4M
    IO_CODE();
796
60.4M
    qatomic_inc(&bs->in_flight);
797
60.4M
}
798
799
void bdrv_wakeup(BlockDriverState *bs)
800
60.4M
{
801
60.4M
    IO_CODE();
802
60.4M
    aio_wait_kick();
803
60.4M
}
804
805
void bdrv_dec_in_flight(BlockDriverState *bs)
806
60.4M
{
807
60.4M
    IO_CODE();
808
60.4M
    qatomic_dec(&bs->in_flight);
809
60.4M
    bdrv_wakeup(bs);
810
60.4M
}
811
812
void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req,
813
                                                uint64_t align)
814
0
{
815
0
    IO_CODE();
816
817
0
    qemu_mutex_lock(&req->bs->reqs_lock);
818
819
0
    tracked_request_set_serialising(req, align);
820
0
    bdrv_wait_serialising_requests_locked(req);
821
0
    req->active_owner = true;
822
823
0
    qemu_mutex_unlock(&req->bs->reqs_lock);
824
0
}
825
826
int bdrv_check_qiov_request(int64_t offset, int64_t bytes,
827
                            QEMUIOVector *qiov, size_t qiov_offset,
828
                            Error **errp)
829
245M
{
830
    /*
831
     * Check generic offset/bytes correctness
832
     */
833
834
245M
    if (offset < 0) {
835
0
        error_setg(errp, "offset is negative: %" PRIi64, offset);
836
0
        return -EIO;
837
0
    }
838
839
245M
    if (bytes < 0) {
840
0
        error_setg(errp, "bytes is negative: %" PRIi64, bytes);
841
0
        return -EIO;
842
0
    }
843
844
245M
    if (bytes > BDRV_MAX_LENGTH) {
845
0
        error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
846
0
                   bytes, BDRV_MAX_LENGTH);
847
0
        return -EIO;
848
0
    }
849
850
245M
    if (offset > BDRV_MAX_LENGTH) {
851
0
        error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
852
0
                   offset, BDRV_MAX_LENGTH);
853
0
        return -EIO;
854
0
    }
855
856
245M
    if (offset > BDRV_MAX_LENGTH - bytes) {
857
0
        error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") "
858
0
                   "exceeds maximum(%" PRIi64 ")", offset, bytes,
859
0
                   BDRV_MAX_LENGTH);
860
0
        return -EIO;
861
0
    }
862
863
245M
    if (!qiov) {
864
83.9M
        return 0;
865
83.9M
    }
866
867
    /*
868
     * Check qiov and qiov_offset
869
     */
870
871
161M
    if (qiov_offset > qiov->size) {
872
0
        error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)",
873
0
                   qiov_offset, qiov->size);
874
0
        return -EIO;
875
0
    }
876
877
161M
    if (bytes > qiov->size - qiov_offset) {
878
0
        error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io "
879
0
                   "vector size(%zu)", bytes, qiov_offset, qiov->size);
880
0
        return -EIO;
881
0
    }
882
883
161M
    return 0;
884
161M
}
885
886
int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp)
887
83.9M
{
888
83.9M
    return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp);
889
83.9M
}
890
891
static int bdrv_check_request32(int64_t offset, int64_t bytes,
892
                                QEMUIOVector *qiov, size_t qiov_offset)
893
80.5M
{
894
80.5M
    int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
895
80.5M
    if (ret < 0) {
896
0
        return ret;
897
0
    }
898
899
80.5M
    if (bytes > BDRV_REQUEST_MAX_BYTES) {
900
0
        return -EIO;
901
0
    }
902
903
80.5M
    return 0;
904
80.5M
}
905
906
/*
907
 * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
908
 * The operation is sped up by checking the block status and only writing
909
 * zeroes to the device if they currently do not return zeroes. Optional
910
 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
911
 * BDRV_REQ_FUA).
912
 *
913
 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
914
 */
915
int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
916
0
{
917
0
    int ret;
918
0
    int64_t target_size, bytes, offset = 0;
919
0
    BlockDriverState *bs = child->bs;
920
0
    IO_CODE();
921
922
0
    target_size = bdrv_getlength(bs);
923
0
    if (target_size < 0) {
924
0
        return target_size;
925
0
    }
926
927
0
    for (;;) {
928
0
        bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
929
0
        if (bytes <= 0) {
930
0
            return 0;
931
0
        }
932
0
        ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
933
0
        if (ret < 0) {
934
0
            return ret;
935
0
        }
936
0
        if (ret & BDRV_BLOCK_ZERO) {
937
0
            offset += bytes;
938
0
            continue;
939
0
        }
940
0
        ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
941
0
        if (ret < 0) {
942
0
            return ret;
943
0
        }
944
0
        offset += bytes;
945
0
    }
946
0
}
947
948
/*
949
 * Writes to the file and ensures that no writes are reordered across this
950
 * request (acts as a barrier)
951
 *
952
 * Returns 0 on success, -errno in error cases.
953
 */
954
int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset,
955
                                     int64_t bytes, const void *buf,
956
                                     BdrvRequestFlags flags)
957
0
{
958
0
    int ret;
959
0
    IO_CODE();
960
0
    assert_bdrv_graph_readable();
961
962
0
    ret = bdrv_co_pwrite(child, offset, bytes, buf, flags);
963
0
    if (ret < 0) {
964
0
        return ret;
965
0
    }
966
967
0
    ret = bdrv_co_flush(child->bs);
968
0
    if (ret < 0) {
969
0
        return ret;
970
0
    }
971
972
0
    return 0;
973
0
}
974
975
typedef struct CoroutineIOCompletion {
976
    Coroutine *coroutine;
977
    int ret;
978
} CoroutineIOCompletion;
979
980
static void bdrv_co_io_em_complete(void *opaque, int ret)
981
0
{
982
0
    CoroutineIOCompletion *co = opaque;
983
984
0
    co->ret = ret;
985
0
    aio_co_wake(co->coroutine);
986
0
}
987
988
static int coroutine_fn GRAPH_RDLOCK
989
bdrv_driver_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes,
990
                   QEMUIOVector *qiov, size_t qiov_offset, int flags)
991
38.6M
{
992
38.6M
    BlockDriver *drv = bs->drv;
993
38.6M
    int64_t sector_num;
994
38.6M
    unsigned int nb_sectors;
995
38.6M
    QEMUIOVector local_qiov;
996
38.6M
    int ret;
997
38.6M
    assert_bdrv_graph_readable();
998
999
38.6M
    bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1000
38.6M
    assert(!(flags & ~bs->supported_read_flags));
1001
1002
38.6M
    if (!drv) {
1003
0
        return -ENOMEDIUM;
1004
0
    }
1005
1006
38.6M
    if (drv->bdrv_co_preadv_part) {
1007
0
        return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
1008
0
                                        flags);
1009
0
    }
1010
1011
38.6M
    if (qiov_offset > 0 || bytes != qiov->size) {
1012
0
        qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1013
0
        qiov = &local_qiov;
1014
0
    }
1015
1016
38.6M
    if (drv->bdrv_co_preadv) {
1017
38.6M
        ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
1018
38.6M
        goto out;
1019
38.6M
    }
1020
1021
0
    if (drv->bdrv_aio_preadv) {
1022
0
        BlockAIOCB *acb;
1023
0
        CoroutineIOCompletion co = {
1024
0
            .coroutine = qemu_coroutine_self(),
1025
0
        };
1026
1027
0
        acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
1028
0
                                   bdrv_co_io_em_complete, &co);
1029
0
        if (acb == NULL) {
1030
0
            ret = -EIO;
1031
0
            goto out;
1032
0
        } else {
1033
0
            qemu_coroutine_yield();
1034
0
            ret = co.ret;
1035
0
            goto out;
1036
0
        }
1037
0
    }
1038
1039
0
    sector_num = offset >> BDRV_SECTOR_BITS;
1040
0
    nb_sectors = bytes >> BDRV_SECTOR_BITS;
1041
1042
0
    assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1043
0
    assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1044
0
    assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1045
0
    assert(drv->bdrv_co_readv);
1046
1047
0
    ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1048
1049
38.6M
out:
1050
38.6M
    if (qiov == &local_qiov) {
1051
0
        qemu_iovec_destroy(&local_qiov);
1052
0
    }
1053
1054
38.6M
    return ret;
1055
0
}
1056
1057
static int coroutine_fn GRAPH_RDLOCK
1058
bdrv_driver_pwritev(BlockDriverState *bs, int64_t offset, int64_t bytes,
1059
                    QEMUIOVector *qiov, size_t qiov_offset,
1060
                    BdrvRequestFlags flags)
1061
1.62M
{
1062
1.62M
    BlockDriver *drv = bs->drv;
1063
1.62M
    bool emulate_fua = false;
1064
1.62M
    int64_t sector_num;
1065
1.62M
    unsigned int nb_sectors;
1066
1.62M
    QEMUIOVector local_qiov;
1067
1.62M
    int ret;
1068
1.62M
    assert_bdrv_graph_readable();
1069
1070
1.62M
    bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1071
1072
1.62M
    if (!drv) {
1073
0
        return -ENOMEDIUM;
1074
0
    }
1075
1076
1.62M
    if (bs->open_flags & BDRV_O_NO_FLUSH) {
1077
0
        flags &= ~BDRV_REQ_FUA;
1078
0
    }
1079
1080
1.62M
    if ((flags & BDRV_REQ_FUA) &&
1081
157k
        (~bs->supported_write_flags & BDRV_REQ_FUA)) {
1082
0
        flags &= ~BDRV_REQ_FUA;
1083
0
        emulate_fua = true;
1084
0
    }
1085
1086
1.62M
    flags &= bs->supported_write_flags;
1087
1088
1.62M
    if (drv->bdrv_co_pwritev_part) {
1089
0
        ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1090
0
                                        flags);
1091
0
        goto emulate_flags;
1092
0
    }
1093
1094
1.62M
    if (qiov_offset > 0 || bytes != qiov->size) {
1095
0
        qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1096
0
        qiov = &local_qiov;
1097
0
    }
1098
1099
1.62M
    if (drv->bdrv_co_pwritev) {
1100
1.62M
        ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
1101
1.62M
        goto emulate_flags;
1102
1.62M
    }
1103
1104
0
    if (drv->bdrv_aio_pwritev) {
1105
0
        BlockAIOCB *acb;
1106
0
        CoroutineIOCompletion co = {
1107
0
            .coroutine = qemu_coroutine_self(),
1108
0
        };
1109
1110
0
        acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, flags,
1111
0
                                    bdrv_co_io_em_complete, &co);
1112
0
        if (acb == NULL) {
1113
0
            ret = -EIO;
1114
0
        } else {
1115
0
            qemu_coroutine_yield();
1116
0
            ret = co.ret;
1117
0
        }
1118
0
        goto emulate_flags;
1119
0
    }
1120
1121
0
    sector_num = offset >> BDRV_SECTOR_BITS;
1122
0
    nb_sectors = bytes >> BDRV_SECTOR_BITS;
1123
1124
0
    assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1125
0
    assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1126
0
    assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1127
1128
0
    assert(drv->bdrv_co_writev);
1129
0
    ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, flags);
1130
1131
1.62M
emulate_flags:
1132
1.62M
    if (ret == 0 && emulate_fua) {
1133
0
        ret = bdrv_co_flush(bs);
1134
0
    }
1135
1136
1.62M
    if (qiov == &local_qiov) {
1137
0
        qemu_iovec_destroy(&local_qiov);
1138
0
    }
1139
1140
1.62M
    return ret;
1141
0
}
1142
1143
static int coroutine_fn GRAPH_RDLOCK
1144
bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset,
1145
                               int64_t bytes, QEMUIOVector *qiov,
1146
                               size_t qiov_offset)
1147
0
{
1148
0
    BlockDriver *drv = bs->drv;
1149
0
    QEMUIOVector local_qiov;
1150
0
    int ret;
1151
0
    assert_bdrv_graph_readable();
1152
1153
0
    bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1154
1155
0
    if (!drv) {
1156
0
        return -ENOMEDIUM;
1157
0
    }
1158
1159
0
    if (!block_driver_can_compress(drv)) {
1160
0
        return -ENOTSUP;
1161
0
    }
1162
1163
0
    if (drv->bdrv_co_pwritev_compressed_part) {
1164
0
        return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1165
0
                                                    qiov, qiov_offset);
1166
0
    }
1167
1168
0
    if (qiov_offset == 0) {
1169
0
        return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
1170
0
    }
1171
1172
0
    qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1173
0
    ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1174
0
    qemu_iovec_destroy(&local_qiov);
1175
1176
0
    return ret;
1177
0
}
1178
1179
static int coroutine_fn GRAPH_RDLOCK
1180
bdrv_co_do_copy_on_readv(BdrvChild *child, int64_t offset, int64_t bytes,
1181
                         QEMUIOVector *qiov, size_t qiov_offset, int flags)
1182
0
{
1183
0
    BlockDriverState *bs = child->bs;
1184
1185
    /* Perform I/O through a temporary buffer so that users who scribble over
1186
     * their read buffer while the operation is in progress do not end up
1187
     * modifying the image file.  This is critical for zero-copy guest I/O
1188
     * where anything might happen inside guest memory.
1189
     */
1190
0
    void *bounce_buffer = NULL;
1191
1192
0
    BlockDriver *drv = bs->drv;
1193
0
    int64_t align_offset;
1194
0
    int64_t align_bytes;
1195
0
    int64_t skip_bytes;
1196
0
    int ret;
1197
0
    int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1198
0
                                    BDRV_REQUEST_MAX_BYTES);
1199
0
    int64_t progress = 0;
1200
0
    bool skip_write;
1201
1202
0
    bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1203
1204
0
    if (!drv) {
1205
0
        return -ENOMEDIUM;
1206
0
    }
1207
1208
    /*
1209
     * Do not write anything when the BDS is inactive.  That is not
1210
     * allowed, and it would not help.
1211
     */
1212
0
    skip_write = (bs->open_flags & BDRV_O_INACTIVE);
1213
1214
    /* FIXME We cannot require callers to have write permissions when all they
1215
     * are doing is a read request. If we did things right, write permissions
1216
     * would be obtained anyway, but internally by the copy-on-read code. As
1217
     * long as it is implemented here rather than in a separate filter driver,
1218
     * the copy-on-read code doesn't have its own BdrvChild, however, for which
1219
     * it could request permissions. Therefore we have to bypass the permission
1220
     * system for the moment. */
1221
    // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1222
1223
    /* Cover entire cluster so no additional backing file I/O is required when
1224
     * allocating cluster in the image file.  Note that this value may exceed
1225
     * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1226
     * is one reason we loop rather than doing it all at once.
1227
     */
1228
0
    bdrv_round_to_subclusters(bs, offset, bytes, &align_offset, &align_bytes);
1229
0
    skip_bytes = offset - align_offset;
1230
1231
0
    trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1232
0
                                   align_offset, align_bytes);
1233
1234
0
    while (align_bytes) {
1235
0
        int64_t pnum;
1236
1237
0
        if (skip_write) {
1238
0
            ret = 1; /* "already allocated", so nothing will be copied */
1239
0
            pnum = MIN(align_bytes, max_transfer);
1240
0
        } else {
1241
0
            ret = bdrv_co_is_allocated(bs, align_offset,
1242
0
                                       MIN(align_bytes, max_transfer), &pnum);
1243
0
            if (ret < 0) {
1244
                /*
1245
                 * Safe to treat errors in querying allocation as if
1246
                 * unallocated; we'll probably fail again soon on the
1247
                 * read, but at least that will set a decent errno.
1248
                 */
1249
0
                pnum = MIN(align_bytes, max_transfer);
1250
0
            }
1251
1252
            /* Stop at EOF if the image ends in the middle of the cluster */
1253
0
            if (ret == 0 && pnum == 0) {
1254
0
                assert(progress >= bytes);
1255
0
                break;
1256
0
            }
1257
1258
0
            assert(skip_bytes < pnum);
1259
0
        }
1260
1261
0
        if (ret <= 0) {
1262
0
            QEMUIOVector local_qiov;
1263
1264
            /* Must copy-on-read; use the bounce buffer */
1265
0
            pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
1266
0
            if (!bounce_buffer) {
1267
0
                int64_t max_we_need = MAX(pnum, align_bytes - pnum);
1268
0
                int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
1269
0
                int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
1270
1271
0
                bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
1272
0
                if (!bounce_buffer) {
1273
0
                    ret = -ENOMEM;
1274
0
                    goto err;
1275
0
                }
1276
0
            }
1277
0
            qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1278
1279
0
            ret = bdrv_driver_preadv(bs, align_offset, pnum,
1280
0
                                     &local_qiov, 0, 0);
1281
0
            if (ret < 0) {
1282
0
                goto err;
1283
0
            }
1284
1285
0
            bdrv_co_debug_event(bs, BLKDBG_COR_WRITE);
1286
0
            if (drv->bdrv_co_pwrite_zeroes &&
1287
0
                buffer_is_zero(bounce_buffer, pnum)) {
1288
                /* FIXME: Should we (perhaps conditionally) be setting
1289
                 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1290
                 * that still correctly reads as zero? */
1291
0
                ret = bdrv_co_do_pwrite_zeroes(bs, align_offset, pnum,
1292
0
                                               BDRV_REQ_WRITE_UNCHANGED);
1293
0
            } else {
1294
                /* This does not change the data on the disk, it is not
1295
                 * necessary to flush even in cache=writethrough mode.
1296
                 */
1297
0
                ret = bdrv_driver_pwritev(bs, align_offset, pnum,
1298
0
                                          &local_qiov, 0,
1299
0
                                          BDRV_REQ_WRITE_UNCHANGED);
1300
0
            }
1301
1302
0
            if (ret < 0) {
1303
                /* It might be okay to ignore write errors for guest
1304
                 * requests.  If this is a deliberate copy-on-read
1305
                 * then we don't want to ignore the error.  Simply
1306
                 * report it in all cases.
1307
                 */
1308
0
                goto err;
1309
0
            }
1310
1311
0
            if (!(flags & BDRV_REQ_PREFETCH)) {
1312
0
                qemu_iovec_from_buf(qiov, qiov_offset + progress,
1313
0
                                    bounce_buffer + skip_bytes,
1314
0
                                    MIN(pnum - skip_bytes, bytes - progress));
1315
0
            }
1316
0
        } else if (!(flags & BDRV_REQ_PREFETCH)) {
1317
            /* Read directly into the destination */
1318
0
            ret = bdrv_driver_preadv(bs, offset + progress,
1319
0
                                     MIN(pnum - skip_bytes, bytes - progress),
1320
0
                                     qiov, qiov_offset + progress, 0);
1321
0
            if (ret < 0) {
1322
0
                goto err;
1323
0
            }
1324
0
        }
1325
1326
0
        align_offset += pnum;
1327
0
        align_bytes -= pnum;
1328
0
        progress += pnum - skip_bytes;
1329
0
        skip_bytes = 0;
1330
0
    }
1331
0
    ret = 0;
1332
1333
0
err:
1334
0
    qemu_vfree(bounce_buffer);
1335
0
    return ret;
1336
0
}
1337
1338
/*
1339
 * Forwards an already correctly aligned request to the BlockDriver. This
1340
 * handles copy on read, zeroing after EOF, and fragmentation of large
1341
 * reads; any other features must be implemented by the caller.
1342
 */
1343
static int coroutine_fn GRAPH_RDLOCK
1344
bdrv_aligned_preadv(BdrvChild *child, BdrvTrackedRequest *req,
1345
                    int64_t offset, int64_t bytes, int64_t align,
1346
                    QEMUIOVector *qiov, size_t qiov_offset, int flags)
1347
38.6M
{
1348
38.6M
    BlockDriverState *bs = child->bs;
1349
38.6M
    int64_t total_bytes, max_bytes;
1350
38.6M
    int ret = 0;
1351
38.6M
    int64_t bytes_remaining = bytes;
1352
38.6M
    int max_transfer;
1353
1354
38.6M
    bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1355
38.6M
    assert(is_power_of_2(align));
1356
38.6M
    assert((offset & (align - 1)) == 0);
1357
38.6M
    assert((bytes & (align - 1)) == 0);
1358
38.6M
    assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1359
38.6M
    max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1360
38.6M
                                   align);
1361
1362
    /*
1363
     * TODO: We would need a per-BDS .supported_read_flags and
1364
     * potential fallback support, if we ever implement any read flags
1365
     * to pass through to drivers.  For now, there aren't any
1366
     * passthrough flags except the BDRV_REQ_REGISTERED_BUF optimization hint.
1367
     */
1368
38.6M
    assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH |
1369
38.6M
                       BDRV_REQ_REGISTERED_BUF)));
1370
1371
    /* Handle Copy on Read and associated serialisation */
1372
38.6M
    if (flags & BDRV_REQ_COPY_ON_READ) {
1373
        /* If we touch the same cluster it counts as an overlap.  This
1374
         * guarantees that allocating writes will be serialized and not race
1375
         * with each other for the same cluster.  For example, in copy-on-read
1376
         * it ensures that the CoR read and write operations are atomic and
1377
         * guest writes cannot interleave between them. */
1378
0
        bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs));
1379
0
    }
1380
1381
38.6M
    if (flags & BDRV_REQ_COPY_ON_READ) {
1382
0
        int64_t pnum;
1383
1384
        /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */
1385
0
        flags &= ~BDRV_REQ_COPY_ON_READ;
1386
1387
0
        ret = bdrv_co_is_allocated(bs, offset, bytes, &pnum);
1388
0
        if (ret < 0) {
1389
0
            goto out;
1390
0
        }
1391
1392
0
        if (!ret || pnum != bytes) {
1393
0
            ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
1394
0
                                           qiov, qiov_offset, flags);
1395
0
            goto out;
1396
0
        } else if (flags & BDRV_REQ_PREFETCH) {
1397
0
            goto out;
1398
0
        }
1399
0
    }
1400
1401
    /* Forward the request to the BlockDriver, possibly fragmenting it */
1402
38.6M
    total_bytes = bdrv_co_getlength(bs);
1403
38.6M
    if (total_bytes < 0) {
1404
0
        ret = total_bytes;
1405
0
        goto out;
1406
0
    }
1407
1408
38.6M
    assert(!(flags & ~(bs->supported_read_flags | BDRV_REQ_REGISTERED_BUF)));
1409
1410
38.6M
    max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1411
38.6M
    if (bytes <= max_bytes && bytes <= max_transfer) {
1412
38.6M
        ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags);
1413
38.6M
        goto out;
1414
38.6M
    }
1415
1416
0
    while (bytes_remaining) {
1417
0
        int64_t num;
1418
1419
0
        if (max_bytes) {
1420
0
            num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1421
0
            assert(num);
1422
1423
0
            ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1424
0
                                     num, qiov,
1425
0
                                     qiov_offset + bytes - bytes_remaining,
1426
0
                                     flags);
1427
0
            max_bytes -= num;
1428
0
        } else {
1429
0
            num = bytes_remaining;
1430
0
            ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining,
1431
0
                                    0, bytes_remaining);
1432
0
        }
1433
0
        if (ret < 0) {
1434
0
            goto out;
1435
0
        }
1436
0
        bytes_remaining -= num;
1437
0
    }
1438
1439
38.6M
out:
1440
38.6M
    return ret < 0 ? ret : 0;
1441
0
}
1442
1443
/*
1444
 * Request padding
1445
 *
1446
 *  |<---- align ----->|                     |<----- align ---->|
1447
 *  |<- head ->|<------------- bytes ------------->|<-- tail -->|
1448
 *  |          |       |                     |     |            |
1449
 * -*----------$-------*-------- ... --------*-----$------------*---
1450
 *  |          |       |                     |     |            |
1451
 *  |          offset  |                     |     end          |
1452
 *  ALIGN_DOWN(offset) ALIGN_UP(offset)      ALIGN_DOWN(end)   ALIGN_UP(end)
1453
 *  [buf   ... )                             [tail_buf          )
1454
 *
1455
 * @buf is an aligned allocation needed to store @head and @tail paddings. @head
1456
 * is placed at the beginning of @buf and @tail at the @end.
1457
 *
1458
 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
1459
 * around tail, if tail exists.
1460
 *
1461
 * @merge_reads is true for small requests,
1462
 * if @buf_len == @head + bytes + @tail. In this case it is possible that both
1463
 * head and tail exist but @buf_len == align and @tail_buf == @buf.
1464
 *
1465
 * @write is true for write requests, false for read requests.
1466
 *
1467
 * If padding makes the vector too long (exceeding IOV_MAX), then we need to
1468
 * merge existing vector elements into a single one.  @collapse_bounce_buf acts
1469
 * as the bounce buffer in such cases.  @pre_collapse_qiov has the pre-collapse
1470
 * I/O vector elements so for read requests, the data can be copied back after
1471
 * the read is done.
1472
 */
1473
typedef struct BdrvRequestPadding {
1474
    uint8_t *buf;
1475
    size_t buf_len;
1476
    uint8_t *tail_buf;
1477
    size_t head;
1478
    size_t tail;
1479
    bool merge_reads;
1480
    bool write;
1481
    QEMUIOVector local_qiov;
1482
1483
    uint8_t *collapse_bounce_buf;
1484
    size_t collapse_len;
1485
    QEMUIOVector pre_collapse_qiov;
1486
} BdrvRequestPadding;
1487
1488
static bool bdrv_init_padding(BlockDriverState *bs,
1489
                              int64_t offset, int64_t bytes,
1490
                              bool write,
1491
                              BdrvRequestPadding *pad)
1492
40.2M
{
1493
40.2M
    int64_t align = bs->bl.request_alignment;
1494
40.2M
    int64_t sum;
1495
1496
40.2M
    bdrv_check_request(offset, bytes, &error_abort);
1497
40.2M
    assert(align <= INT_MAX); /* documented in block/block_int.h */
1498
40.2M
    assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */
1499
1500
40.2M
    memset(pad, 0, sizeof(*pad));
1501
1502
40.2M
    pad->head = offset & (align - 1);
1503
40.2M
    pad->tail = ((offset + bytes) & (align - 1));
1504
40.2M
    if (pad->tail) {
1505
0
        pad->tail = align - pad->tail;
1506
0
    }
1507
1508
40.2M
    if (!pad->head && !pad->tail) {
1509
40.2M
        return false;
1510
40.2M
    }
1511
1512
40.2M
    assert(bytes); /* Nothing good in aligning zero-length requests */
1513
1514
0
    sum = pad->head + bytes + pad->tail;
1515
0
    pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
1516
0
    pad->buf = qemu_blockalign(bs, pad->buf_len);
1517
0
    pad->merge_reads = sum == pad->buf_len;
1518
0
    if (pad->tail) {
1519
0
        pad->tail_buf = pad->buf + pad->buf_len - align;
1520
0
    }
1521
1522
0
    pad->write = write;
1523
1524
0
    return true;
1525
0
}
1526
1527
static int coroutine_fn GRAPH_RDLOCK
1528
bdrv_padding_rmw_read(BdrvChild *child, BdrvTrackedRequest *req,
1529
                      BdrvRequestPadding *pad, bool zero_middle)
1530
0
{
1531
0
    QEMUIOVector local_qiov;
1532
0
    BlockDriverState *bs = child->bs;
1533
0
    uint64_t align = bs->bl.request_alignment;
1534
0
    int ret;
1535
1536
0
    assert(req->serialising && pad->buf);
1537
1538
0
    if (pad->head || pad->merge_reads) {
1539
0
        int64_t bytes = pad->merge_reads ? pad->buf_len : align;
1540
1541
0
        qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
1542
1543
0
        if (pad->head) {
1544
0
            bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1545
0
        }
1546
0
        if (pad->merge_reads && pad->tail) {
1547
0
            bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1548
0
        }
1549
0
        ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
1550
0
                                  align, &local_qiov, 0, 0);
1551
0
        if (ret < 0) {
1552
0
            return ret;
1553
0
        }
1554
0
        if (pad->head) {
1555
0
            bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1556
0
        }
1557
0
        if (pad->merge_reads && pad->tail) {
1558
0
            bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1559
0
        }
1560
1561
0
        if (pad->merge_reads) {
1562
0
            goto zero_mem;
1563
0
        }
1564
0
    }
1565
1566
0
    if (pad->tail) {
1567
0
        qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
1568
1569
0
        bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1570
0
        ret = bdrv_aligned_preadv(
1571
0
                child, req,
1572
0
                req->overlap_offset + req->overlap_bytes - align,
1573
0
                align, align, &local_qiov, 0, 0);
1574
0
        if (ret < 0) {
1575
0
            return ret;
1576
0
        }
1577
0
        bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1578
0
    }
1579
1580
0
zero_mem:
1581
0
    if (zero_middle) {
1582
0
        memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
1583
0
    }
1584
1585
0
    return 0;
1586
0
}
1587
1588
/**
1589
 * Free *pad's associated buffers, and perform any necessary finalization steps.
1590
 */
1591
static void bdrv_padding_finalize(BdrvRequestPadding *pad)
1592
40.2M
{
1593
40.2M
    if (pad->collapse_bounce_buf) {
1594
0
        if (!pad->write) {
1595
            /*
1596
             * If padding required elements in the vector to be collapsed into a
1597
             * bounce buffer, copy the bounce buffer content back
1598
             */
1599
0
            qemu_iovec_from_buf(&pad->pre_collapse_qiov, 0,
1600
0
                                pad->collapse_bounce_buf, pad->collapse_len);
1601
0
        }
1602
0
        qemu_vfree(pad->collapse_bounce_buf);
1603
0
        qemu_iovec_destroy(&pad->pre_collapse_qiov);
1604
0
    }
1605
40.2M
    if (pad->buf) {
1606
0
        qemu_vfree(pad->buf);
1607
0
        qemu_iovec_destroy(&pad->local_qiov);
1608
0
    }
1609
40.2M
    memset(pad, 0, sizeof(*pad));
1610
40.2M
}
1611
1612
/*
1613
 * Create pad->local_qiov by wrapping @iov in the padding head and tail, while
1614
 * ensuring that the resulting vector will not exceed IOV_MAX elements.
1615
 *
1616
 * To ensure this, when necessary, the first two or three elements of @iov are
1617
 * merged into pad->collapse_bounce_buf and replaced by a reference to that
1618
 * bounce buffer in pad->local_qiov.
1619
 *
1620
 * After performing a read request, the data from the bounce buffer must be
1621
 * copied back into pad->pre_collapse_qiov (e.g. by bdrv_padding_finalize()).
1622
 */
1623
static int bdrv_create_padded_qiov(BlockDriverState *bs,
1624
                                   BdrvRequestPadding *pad,
1625
                                   struct iovec *iov, int niov,
1626
                                   size_t iov_offset, size_t bytes)
1627
0
{
1628
0
    int padded_niov, surplus_count, collapse_count;
1629
1630
    /* Assert this invariant */
1631
0
    assert(niov <= IOV_MAX);
1632
1633
    /*
1634
     * Cannot pad if resulting length would exceed SIZE_MAX.  Returning an error
1635
     * to the guest is not ideal, but there is little else we can do.  At least
1636
     * this will practically never happen on 64-bit systems.
1637
     */
1638
0
    if (SIZE_MAX - pad->head < bytes ||
1639
0
        SIZE_MAX - pad->head - bytes < pad->tail)
1640
0
    {
1641
0
        return -EINVAL;
1642
0
    }
1643
1644
    /* Length of the resulting IOV if we just concatenated everything */
1645
0
    padded_niov = !!pad->head + niov + !!pad->tail;
1646
1647
0
    qemu_iovec_init(&pad->local_qiov, MIN(padded_niov, IOV_MAX));
1648
1649
0
    if (pad->head) {
1650
0
        qemu_iovec_add(&pad->local_qiov, pad->buf, pad->head);
1651
0
    }
1652
1653
    /*
1654
     * If padded_niov > IOV_MAX, we cannot just concatenate everything.
1655
     * Instead, merge the first two or three elements of @iov to reduce the
1656
     * number of vector elements as necessary.
1657
     */
1658
0
    if (padded_niov > IOV_MAX) {
1659
        /*
1660
         * Only head and tail can have lead to the number of entries exceeding
1661
         * IOV_MAX, so we can exceed it by the head and tail at most.  We need
1662
         * to reduce the number of elements by `surplus_count`, so we merge that
1663
         * many elements plus one into one element.
1664
         */
1665
0
        surplus_count = padded_niov - IOV_MAX;
1666
0
        assert(surplus_count <= !!pad->head + !!pad->tail);
1667
0
        collapse_count = surplus_count + 1;
1668
1669
        /*
1670
         * Move the elements to collapse into `pad->pre_collapse_qiov`, then
1671
         * advance `iov` (and associated variables) by those elements.
1672
         */
1673
0
        qemu_iovec_init(&pad->pre_collapse_qiov, collapse_count);
1674
0
        qemu_iovec_concat_iov(&pad->pre_collapse_qiov, iov,
1675
0
                              collapse_count, iov_offset, SIZE_MAX);
1676
0
        iov += collapse_count;
1677
0
        iov_offset = 0;
1678
0
        niov -= collapse_count;
1679
0
        bytes -= pad->pre_collapse_qiov.size;
1680
1681
        /*
1682
         * Construct the bounce buffer to match the length of the to-collapse
1683
         * vector elements, and for write requests, initialize it with the data
1684
         * from those elements.  Then add it to `pad->local_qiov`.
1685
         */
1686
0
        pad->collapse_len = pad->pre_collapse_qiov.size;
1687
0
        pad->collapse_bounce_buf = qemu_blockalign(bs, pad->collapse_len);
1688
0
        if (pad->write) {
1689
0
            qemu_iovec_to_buf(&pad->pre_collapse_qiov, 0,
1690
0
                              pad->collapse_bounce_buf, pad->collapse_len);
1691
0
        }
1692
0
        qemu_iovec_add(&pad->local_qiov,
1693
0
                       pad->collapse_bounce_buf, pad->collapse_len);
1694
0
    }
1695
1696
0
    qemu_iovec_concat_iov(&pad->local_qiov, iov, niov, iov_offset, bytes);
1697
1698
0
    if (pad->tail) {
1699
0
        qemu_iovec_add(&pad->local_qiov,
1700
0
                       pad->buf + pad->buf_len - pad->tail, pad->tail);
1701
0
    }
1702
1703
0
    assert(pad->local_qiov.niov == MIN(padded_niov, IOV_MAX));
1704
0
    return 0;
1705
0
}
1706
1707
/*
1708
 * bdrv_pad_request
1709
 *
1710
 * Exchange request parameters with padded request if needed. Don't include RMW
1711
 * read of padding, bdrv_padding_rmw_read() should be called separately if
1712
 * needed.
1713
 *
1714
 * @write is true for write requests, false for read requests.
1715
 *
1716
 * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out:
1717
 *  - on function start they represent original request
1718
 *  - on failure or when padding is not needed they are unchanged
1719
 *  - on success when padding is needed they represent padded request
1720
 */
1721
static int bdrv_pad_request(BlockDriverState *bs,
1722
                            QEMUIOVector **qiov, size_t *qiov_offset,
1723
                            int64_t *offset, int64_t *bytes,
1724
                            bool write,
1725
                            BdrvRequestPadding *pad, bool *padded,
1726
                            BdrvRequestFlags *flags)
1727
40.2M
{
1728
40.2M
    int ret;
1729
40.2M
    struct iovec *sliced_iov;
1730
40.2M
    int sliced_niov;
1731
40.2M
    size_t sliced_head, sliced_tail;
1732
1733
    /* Should have been checked by the caller already */
1734
40.2M
    ret = bdrv_check_request32(*offset, *bytes, *qiov, *qiov_offset);
1735
40.2M
    if (ret < 0) {
1736
0
        return ret;
1737
0
    }
1738
1739
40.2M
    if (!bdrv_init_padding(bs, *offset, *bytes, write, pad)) {
1740
40.2M
        if (padded) {
1741
1.61M
            *padded = false;
1742
1.61M
        }
1743
40.2M
        return 0;
1744
40.2M
    }
1745
1746
    /*
1747
     * For prefetching in stream_populate(), no qiov is passed along, because
1748
     * only copy-on-read matters.
1749
     */
1750
0
    if (*qiov) {
1751
0
        sliced_iov = qemu_iovec_slice(*qiov, *qiov_offset, *bytes,
1752
0
                                      &sliced_head, &sliced_tail,
1753
0
                                      &sliced_niov);
1754
1755
        /* Guaranteed by bdrv_check_request32() */
1756
0
        assert(*bytes <= SIZE_MAX);
1757
0
        ret = bdrv_create_padded_qiov(bs, pad, sliced_iov, sliced_niov,
1758
0
                                      sliced_head, *bytes);
1759
0
        if (ret < 0) {
1760
0
            bdrv_padding_finalize(pad);
1761
0
            return ret;
1762
0
        }
1763
0
        *qiov = &pad->local_qiov;
1764
0
        *qiov_offset = 0;
1765
0
    }
1766
1767
0
    *bytes += pad->head + pad->tail;
1768
0
    *offset -= pad->head;
1769
0
    if (padded) {
1770
0
        *padded = true;
1771
0
    }
1772
0
    if (flags) {
1773
        /* Can't use optimization hint with bounce buffer */
1774
0
        *flags &= ~BDRV_REQ_REGISTERED_BUF;
1775
0
    }
1776
1777
0
    return 0;
1778
0
}
1779
1780
int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1781
    int64_t offset, int64_t bytes, QEMUIOVector *qiov,
1782
    BdrvRequestFlags flags)
1783
19.3M
{
1784
19.3M
    IO_CODE();
1785
19.3M
    return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
1786
19.3M
}
1787
1788
int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
1789
    int64_t offset, int64_t bytes,
1790
    QEMUIOVector *qiov, size_t qiov_offset,
1791
    BdrvRequestFlags flags)
1792
38.6M
{
1793
38.6M
    BlockDriverState *bs = child->bs;
1794
38.6M
    BdrvTrackedRequest req;
1795
38.6M
    BdrvRequestPadding pad;
1796
38.6M
    int ret;
1797
38.6M
    IO_CODE();
1798
1799
38.6M
    trace_bdrv_co_preadv_part(bs, offset, bytes, flags);
1800
1801
38.6M
    if (!bdrv_co_is_inserted(bs)) {
1802
0
        return -ENOMEDIUM;
1803
0
    }
1804
1805
38.6M
    ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
1806
38.6M
    if (ret < 0) {
1807
0
        return ret;
1808
0
    }
1809
1810
38.6M
    if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1811
        /*
1812
         * Aligning zero request is nonsense. Even if driver has special meaning
1813
         * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1814
         * it to driver due to request_alignment.
1815
         *
1816
         * Still, no reason to return an error if someone do unaligned
1817
         * zero-length read occasionally.
1818
         */
1819
0
        return 0;
1820
0
    }
1821
1822
38.6M
    bdrv_inc_in_flight(bs);
1823
1824
    /* Don't do copy-on-read if we read data before write operation */
1825
38.6M
    if (qatomic_read(&bs->copy_on_read)) {
1826
0
        flags |= BDRV_REQ_COPY_ON_READ;
1827
0
    }
1828
1829
38.6M
    ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, false,
1830
38.6M
                           &pad, NULL, &flags);
1831
38.6M
    if (ret < 0) {
1832
0
        goto fail;
1833
0
    }
1834
1835
38.6M
    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1836
38.6M
    ret = bdrv_aligned_preadv(child, &req, offset, bytes,
1837
38.6M
                              bs->bl.request_alignment,
1838
38.6M
                              qiov, qiov_offset, flags);
1839
38.6M
    tracked_request_end(&req);
1840
38.6M
    bdrv_padding_finalize(&pad);
1841
1842
38.6M
fail:
1843
38.6M
    bdrv_dec_in_flight(bs);
1844
1845
38.6M
    return ret;
1846
38.6M
}
1847
1848
static int coroutine_fn GRAPH_RDLOCK
1849
bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int64_t bytes,
1850
                         BdrvRequestFlags flags)
1851
932
{
1852
932
    BlockDriver *drv = bs->drv;
1853
932
    QEMUIOVector qiov;
1854
932
    void *buf = NULL;
1855
932
    int ret = 0;
1856
932
    bool need_flush = false;
1857
932
    int head = 0;
1858
932
    int tail = 0;
1859
1860
932
    int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes,
1861
932
                                            INT64_MAX);
1862
932
    int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1863
932
                        bs->bl.request_alignment);
1864
932
    int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1865
1866
932
    assert_bdrv_graph_readable();
1867
932
    bdrv_check_request(offset, bytes, &error_abort);
1868
1869
932
    if (!drv) {
1870
0
        return -ENOMEDIUM;
1871
0
    }
1872
1873
932
    if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1874
0
        return -ENOTSUP;
1875
0
    }
1876
1877
    /* By definition there is no user buffer so this flag doesn't make sense */
1878
932
    if (flags & BDRV_REQ_REGISTERED_BUF) {
1879
0
        return -EINVAL;
1880
0
    }
1881
1882
    /* If opened with discard=off we should never unmap. */
1883
932
    if (!(bs->open_flags & BDRV_O_UNMAP)) {
1884
466
        flags &= ~BDRV_REQ_MAY_UNMAP;
1885
466
    }
1886
1887
    /* Invalidate the cached block-status data range if this write overlaps */
1888
932
    bdrv_bsc_invalidate_range(bs, offset, bytes);
1889
1890
932
    assert(alignment % bs->bl.request_alignment == 0);
1891
932
    head = offset % alignment;
1892
932
    tail = (offset + bytes) % alignment;
1893
932
    max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1894
932
    assert(max_write_zeroes >= bs->bl.request_alignment);
1895
1896
6.53k
    while (bytes > 0 && !ret) {
1897
5.59k
        int64_t num = bytes;
1898
1899
        /* Align request.  Block drivers can expect the "bulk" of the request
1900
         * to be aligned, and that unaligned requests do not cross cluster
1901
         * boundaries.
1902
         */
1903
5.59k
        if (head) {
1904
            /* Make a small request up to the first aligned sector. For
1905
             * convenience, limit this request to max_transfer even if
1906
             * we don't need to fall back to writes.  */
1907
0
            num = MIN(MIN(bytes, max_transfer), alignment - head);
1908
0
            head = (head + num) % alignment;
1909
0
            assert(num < max_write_zeroes);
1910
5.59k
        } else if (tail && num > alignment) {
1911
            /* Shorten the request to the last aligned sector.  */
1912
0
            num -= tail;
1913
0
        }
1914
1915
        /* limit request size */
1916
5.59k
        if (num > max_write_zeroes) {
1917
0
            num = max_write_zeroes;
1918
0
        }
1919
1920
5.59k
        ret = -ENOTSUP;
1921
        /* First try the efficient write zeroes operation */
1922
5.59k
        if (drv->bdrv_co_pwrite_zeroes) {
1923
466
            ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1924
466
                                             flags & bs->supported_zero_flags);
1925
466
            if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1926
88
                !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1927
88
                need_flush = true;
1928
88
            }
1929
5.13k
        } else {
1930
5.13k
            assert(!bs->supported_zero_flags);
1931
5.13k
        }
1932
1933
        /*
1934
         * TODO The ret == -EINVAL && num < alignment case is a workaround for
1935
         * when request_alignment is 1 on files with cache=writeback. The Linux
1936
         * ioctl(BLKZEROOUT) requires block alignment and will fail with
1937
         * EINVAL. The block layer should align the request to
1938
         * write_zeroes_alignment instead of trying the syscall, failing, and
1939
         * falling back to a bounce buffer. Doing that is not easy so for now
1940
         * we use a bounce buffer:
1941
         * https://lore.kernel.org/qemu-devel/20260109120837.2772961-1-f.ebner@proxmox.com/
1942
         */
1943
5.59k
        if ((ret == -ENOTSUP || (ret == -EINVAL && num < alignment)) &&
1944
5.13k
            !(flags & BDRV_REQ_NO_FALLBACK)) {
1945
            /* Fall back to bounce buffer if write zeroes is unsupported */
1946
5.13k
            BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1947
1948
5.13k
            if ((flags & BDRV_REQ_FUA) &&
1949
0
                !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1950
                /* No need for bdrv_driver_pwrite() to do a fallback
1951
                 * flush on each chunk; use just one at the end */
1952
0
                write_flags &= ~BDRV_REQ_FUA;
1953
0
                need_flush = true;
1954
0
            }
1955
5.13k
            num = MIN(num, max_transfer);
1956
5.13k
            if (buf == NULL) {
1957
466
                buf = qemu_try_blockalign0(bs, num);
1958
466
                if (buf == NULL) {
1959
0
                    ret = -ENOMEM;
1960
0
                    goto fail;
1961
0
                }
1962
466
            }
1963
5.13k
            qemu_iovec_init_buf(&qiov, buf, num);
1964
1965
5.13k
            ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
1966
1967
            /* Keep bounce buffer around if it is big enough for all
1968
             * all future requests.
1969
             */
1970
5.13k
            if (num < max_transfer) {
1971
211
                qemu_vfree(buf);
1972
211
                buf = NULL;
1973
211
            }
1974
5.13k
        }
1975
1976
5.59k
        offset += num;
1977
5.59k
        bytes -= num;
1978
5.59k
    }
1979
1980
932
fail:
1981
932
    if (ret == 0 && need_flush) {
1982
88
        ret = bdrv_co_flush(bs);
1983
88
    }
1984
932
    qemu_vfree(buf);
1985
932
    return ret;
1986
932
}
1987
1988
static inline int coroutine_fn GRAPH_RDLOCK
1989
bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes,
1990
                          BdrvTrackedRequest *req, int flags)
1991
1.61M
{
1992
1.61M
    BlockDriverState *bs = child->bs;
1993
1994
1.61M
    bdrv_check_request(offset, bytes, &error_abort);
1995
1996
1.61M
    if (bdrv_is_read_only(bs)) {
1997
0
        return -EPERM;
1998
0
    }
1999
2000
1.61M
    assert(!(bs->open_flags & BDRV_O_INACTIVE));
2001
1.61M
    assert((bs->open_flags & BDRV_O_NO_IO) == 0);
2002
1.61M
    assert(!(flags & ~BDRV_REQ_MASK));
2003
1.61M
    assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING)));
2004
2005
1.61M
    if (flags & BDRV_REQ_SERIALISING) {
2006
0
        QEMU_LOCK_GUARD(&bs->reqs_lock);
2007
2008
0
        tracked_request_set_serialising(req, bdrv_get_cluster_size(bs));
2009
2010
0
        if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) {
2011
0
            return -EBUSY;
2012
0
        }
2013
2014
0
        bdrv_wait_serialising_requests_locked(req);
2015
0
    }
2016
2017
1.61M
    assert(req->overlap_offset <= offset);
2018
1.61M
    assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
2019
1.61M
    assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE ||
2020
1.61M
           child->perm & BLK_PERM_RESIZE);
2021
2022
1.61M
    switch (req->type) {
2023
1.61M
    case BDRV_TRACKED_WRITE:
2024
1.61M
    case BDRV_TRACKED_DISCARD:
2025
1.61M
        if (flags & BDRV_REQ_WRITE_UNCHANGED) {
2026
0
            assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
2027
1.61M
        } else {
2028
1.61M
            assert(child->perm & BLK_PERM_WRITE);
2029
1.61M
        }
2030
1.61M
        bdrv_write_threshold_check_write(bs, offset, bytes);
2031
1.61M
        return 0;
2032
0
    case BDRV_TRACKED_TRUNCATE:
2033
0
        assert(child->perm & BLK_PERM_RESIZE);
2034
0
        return 0;
2035
0
    default:
2036
0
        abort();
2037
1.61M
    }
2038
1.61M
}
2039
2040
static inline void coroutine_fn GRAPH_RDLOCK
2041
bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes,
2042
                         BdrvTrackedRequest *req, int ret)
2043
1.61M
{
2044
1.61M
    int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
2045
1.61M
    BlockDriverState *bs = child->bs;
2046
2047
1.61M
    bdrv_check_request(offset, bytes, &error_abort);
2048
2049
1.61M
    qatomic_inc(&bs->write_gen);
2050
2051
    /*
2052
     * Discard cannot extend the image, but in error handling cases, such as
2053
     * when reverting a qcow2 cluster allocation, the discarded range can pass
2054
     * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
2055
     * here. Instead, just skip it, since semantically a discard request
2056
     * beyond EOF cannot expand the image anyway.
2057
     */
2058
1.61M
    if (ret == 0 &&
2059
1.61M
        (req->type == BDRV_TRACKED_TRUNCATE ||
2060
1.61M
         end_sector > bs->total_sectors) &&
2061
0
        req->type != BDRV_TRACKED_DISCARD) {
2062
0
        bs->total_sectors = end_sector;
2063
0
        bdrv_co_parent_cb_resize(bs);
2064
0
        bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
2065
0
    }
2066
1.61M
    if (req->bytes) {
2067
1.61M
        switch (req->type) {
2068
1.61M
        case BDRV_TRACKED_WRITE:
2069
1.61M
            {
2070
1.61M
                uint64_t new = offset + bytes;
2071
1.61M
                uint64_t old = qatomic_read(&bs->wr_highest_offset);
2072
2073
1.61M
                while (old < new) {
2074
1.65k
                    old = qatomic_cmpxchg(&bs->wr_highest_offset, old, new);
2075
1.65k
                }
2076
1.61M
            }
2077
            /* fall through, to set dirty bits */
2078
1.61M
        case BDRV_TRACKED_DISCARD:
2079
1.61M
            bdrv_set_dirty(bs, offset, bytes);
2080
1.61M
            break;
2081
0
        default:
2082
0
            break;
2083
1.61M
        }
2084
1.61M
    }
2085
1.61M
}
2086
2087
/*
2088
 * Forwards an already correctly aligned write request to the BlockDriver,
2089
 * after possibly fragmenting it.
2090
 */
2091
static int coroutine_fn GRAPH_RDLOCK
2092
bdrv_aligned_pwritev(BdrvChild *child, BdrvTrackedRequest *req,
2093
                     int64_t offset, int64_t bytes, int64_t align,
2094
                     QEMUIOVector *qiov, size_t qiov_offset,
2095
                     BdrvRequestFlags flags)
2096
1.61M
{
2097
1.61M
    BlockDriverState *bs = child->bs;
2098
1.61M
    BlockDriver *drv = bs->drv;
2099
1.61M
    int ret;
2100
2101
1.61M
    int64_t bytes_remaining = bytes;
2102
1.61M
    int max_transfer;
2103
2104
1.61M
    bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
2105
2106
1.61M
    if (!drv) {
2107
0
        return -ENOMEDIUM;
2108
0
    }
2109
2110
1.61M
    if (bdrv_has_readonly_bitmaps(bs)) {
2111
0
        return -EPERM;
2112
0
    }
2113
2114
1.61M
    assert(is_power_of_2(align));
2115
1.61M
    assert((offset & (align - 1)) == 0);
2116
1.61M
    assert((bytes & (align - 1)) == 0);
2117
1.61M
    max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
2118
1.61M
                                   align);
2119
2120
1.61M
    ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
2121
2122
1.61M
    if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
2123
0
        !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
2124
0
        qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
2125
0
        flags |= BDRV_REQ_ZERO_WRITE;
2126
0
        if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
2127
0
            flags |= BDRV_REQ_MAY_UNMAP;
2128
0
        }
2129
2130
        /* Can't use optimization hint with bufferless zero write */
2131
0
        flags &= ~BDRV_REQ_REGISTERED_BUF;
2132
0
    }
2133
2134
1.61M
    if (ret < 0) {
2135
        /* Do nothing, write notifier decided to fail this request */
2136
1.61M
    } else if (flags & BDRV_REQ_ZERO_WRITE) {
2137
932
        bdrv_co_debug_event(bs, BLKDBG_PWRITEV_ZERO);
2138
932
        ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
2139
1.61M
    } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
2140
0
        ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
2141
0
                                             qiov, qiov_offset);
2142
1.61M
    } else if (bytes <= max_transfer) {
2143
1.61M
        bdrv_co_debug_event(bs, BLKDBG_PWRITEV);
2144
1.61M
        ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
2145
1.61M
    } else {
2146
0
        bdrv_co_debug_event(bs, BLKDBG_PWRITEV);
2147
0
        while (bytes_remaining) {
2148
0
            int num = MIN(bytes_remaining, max_transfer);
2149
0
            int local_flags = flags;
2150
2151
0
            assert(num);
2152
0
            if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
2153
0
                !(bs->supported_write_flags & BDRV_REQ_FUA)) {
2154
                /* If FUA is going to be emulated by flush, we only
2155
                 * need to flush on the last iteration */
2156
0
                local_flags &= ~BDRV_REQ_FUA;
2157
0
            }
2158
2159
0
            ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
2160
0
                                      num, qiov,
2161
0
                                      qiov_offset + bytes - bytes_remaining,
2162
0
                                      local_flags);
2163
0
            if (ret < 0) {
2164
0
                break;
2165
0
            }
2166
0
            bytes_remaining -= num;
2167
0
        }
2168
0
    }
2169
1.61M
    bdrv_co_debug_event(bs, BLKDBG_PWRITEV_DONE);
2170
2171
1.61M
    if (ret >= 0) {
2172
1.61M
        ret = 0;
2173
1.61M
    }
2174
1.61M
    bdrv_co_write_req_finish(child, offset, bytes, req, ret);
2175
2176
1.61M
    return ret;
2177
1.61M
}
2178
2179
static int coroutine_fn GRAPH_RDLOCK
2180
bdrv_co_do_zero_pwritev(BdrvChild *child, int64_t offset, int64_t bytes,
2181
                        BdrvRequestFlags flags, BdrvTrackedRequest *req)
2182
932
{
2183
932
    BlockDriverState *bs = child->bs;
2184
932
    QEMUIOVector local_qiov;
2185
932
    uint64_t align = bs->bl.request_alignment;
2186
932
    int ret = 0;
2187
932
    bool padding;
2188
932
    BdrvRequestPadding pad;
2189
2190
    /* This flag doesn't make sense for padding or zero writes */
2191
932
    flags &= ~BDRV_REQ_REGISTERED_BUF;
2192
2193
932
    padding = bdrv_init_padding(bs, offset, bytes, true, &pad);
2194
932
    if (padding) {
2195
0
        assert(!(flags & BDRV_REQ_NO_WAIT));
2196
0
        bdrv_make_request_serialising(req, align);
2197
2198
0
        bdrv_padding_rmw_read(child, req, &pad, true);
2199
2200
0
        if (pad.head || pad.merge_reads) {
2201
0
            int64_t aligned_offset = offset & ~(align - 1);
2202
0
            int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
2203
2204
0
            qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
2205
0
            ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
2206
0
                                       align, &local_qiov, 0,
2207
0
                                       flags & ~BDRV_REQ_ZERO_WRITE);
2208
0
            if (ret < 0 || pad.merge_reads) {
2209
                /* Error or all work is done */
2210
0
                goto out;
2211
0
            }
2212
0
            offset += write_bytes - pad.head;
2213
0
            bytes -= write_bytes - pad.head;
2214
0
        }
2215
0
    }
2216
2217
932
    assert(!bytes || (offset & (align - 1)) == 0);
2218
932
    if (bytes >= align) {
2219
        /* Write the aligned part in the middle. */
2220
932
        int64_t aligned_bytes = bytes & ~(align - 1);
2221
932
        ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
2222
932
                                   NULL, 0, flags);
2223
932
        if (ret < 0) {
2224
0
            goto out;
2225
0
        }
2226
932
        bytes -= aligned_bytes;
2227
932
        offset += aligned_bytes;
2228
932
    }
2229
2230
932
    assert(!bytes || (offset & (align - 1)) == 0);
2231
932
    if (bytes) {
2232
0
        assert(align == pad.tail + bytes);
2233
2234
0
        qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
2235
0
        ret = bdrv_aligned_pwritev(child, req, offset, align, align,
2236
0
                                   &local_qiov, 0,
2237
0
                                   flags & ~BDRV_REQ_ZERO_WRITE);
2238
0
    }
2239
2240
932
out:
2241
932
    bdrv_padding_finalize(&pad);
2242
2243
932
    return ret;
2244
932
}
2245
2246
/*
2247
 * Handle a write request in coroutine context
2248
 */
2249
int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
2250
    int64_t offset, int64_t bytes, QEMUIOVector *qiov,
2251
    BdrvRequestFlags flags)
2252
808k
{
2253
808k
    IO_CODE();
2254
808k
    return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
2255
808k
}
2256
2257
int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
2258
    int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset,
2259
    BdrvRequestFlags flags)
2260
1.61M
{
2261
1.61M
    BlockDriverState *bs = child->bs;
2262
1.61M
    BdrvTrackedRequest req;
2263
1.61M
    uint64_t align = bs->bl.request_alignment;
2264
1.61M
    BdrvRequestPadding pad;
2265
1.61M
    int ret;
2266
1.61M
    bool padded = false;
2267
1.61M
    IO_CODE();
2268
2269
1.61M
    trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags);
2270
2271
1.61M
    if (!bdrv_co_is_inserted(bs)) {
2272
0
        return -ENOMEDIUM;
2273
0
    }
2274
2275
1.61M
    if (flags & BDRV_REQ_ZERO_WRITE) {
2276
932
        ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
2277
1.61M
    } else {
2278
1.61M
        ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
2279
1.61M
    }
2280
1.61M
    if (ret < 0) {
2281
0
        return ret;
2282
0
    }
2283
2284
    /* If the request is misaligned then we can't make it efficient */
2285
1.61M
    if ((flags & BDRV_REQ_NO_FALLBACK) &&
2286
0
        !QEMU_IS_ALIGNED(offset | bytes, align))
2287
0
    {
2288
0
        return -ENOTSUP;
2289
0
    }
2290
2291
1.61M
    if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2292
        /*
2293
         * Aligning zero request is nonsense. Even if driver has special meaning
2294
         * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2295
         * it to driver due to request_alignment.
2296
         *
2297
         * Still, no reason to return an error if someone do unaligned
2298
         * zero-length write occasionally.
2299
         */
2300
0
        return 0;
2301
0
    }
2302
2303
1.61M
    if (!(flags & BDRV_REQ_ZERO_WRITE)) {
2304
        /*
2305
         * Pad request for following read-modify-write cycle.
2306
         * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do
2307
         * alignment only if there is no ZERO flag.
2308
         */
2309
1.61M
        ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, true,
2310
1.61M
                               &pad, &padded, &flags);
2311
1.61M
        if (ret < 0) {
2312
0
            return ret;
2313
0
        }
2314
1.61M
    }
2315
2316
1.61M
    bdrv_inc_in_flight(bs);
2317
1.61M
    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
2318
2319
1.61M
    if (flags & BDRV_REQ_ZERO_WRITE) {
2320
932
        assert(!padded);
2321
932
        ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
2322
932
        goto out;
2323
932
    }
2324
2325
1.61M
    if (padded) {
2326
        /*
2327
         * Request was unaligned to request_alignment and therefore
2328
         * padded.  We are going to do read-modify-write, and must
2329
         * serialize the request to prevent interactions of the
2330
         * widened region with other transactions.
2331
         */
2332
0
        assert(!(flags & BDRV_REQ_NO_WAIT));
2333
0
        bdrv_make_request_serialising(&req, align);
2334
0
        bdrv_padding_rmw_read(child, &req, &pad, false);
2335
0
    }
2336
2337
1.61M
    ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
2338
1.61M
                               qiov, qiov_offset, flags);
2339
2340
1.61M
    bdrv_padding_finalize(&pad);
2341
2342
1.61M
out:
2343
1.61M
    tracked_request_end(&req);
2344
1.61M
    bdrv_dec_in_flight(bs);
2345
2346
1.61M
    return ret;
2347
1.61M
}
2348
2349
int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2350
                                       int64_t bytes, BdrvRequestFlags flags)
2351
466
{
2352
466
    IO_CODE();
2353
466
    trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
2354
466
    assert_bdrv_graph_readable();
2355
2356
466
    return bdrv_co_pwritev(child, offset, bytes, NULL,
2357
466
                           BDRV_REQ_ZERO_WRITE | flags);
2358
466
}
2359
2360
/*
2361
 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
2362
 */
2363
int bdrv_flush_all(void)
2364
0
{
2365
0
    BdrvNextIterator it;
2366
0
    BlockDriverState *bs = NULL;
2367
0
    int result = 0;
2368
2369
0
    GLOBAL_STATE_CODE();
2370
0
    GRAPH_RDLOCK_GUARD_MAINLOOP();
2371
2372
    /*
2373
     * bdrv queue is managed by record/replay,
2374
     * creating new flush request for stopping
2375
     * the VM may break the determinism
2376
     */
2377
0
    if (replay_events_enabled()) {
2378
0
        return result;
2379
0
    }
2380
2381
0
    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
2382
0
        int ret = bdrv_flush(bs);
2383
0
        if (ret < 0 && !result) {
2384
0
            result = ret;
2385
0
        }
2386
0
    }
2387
2388
0
    return result;
2389
0
}
2390
2391
/*
2392
 * Returns the allocation status of the specified sectors.
2393
 * Drivers not implementing the functionality are assumed to not support
2394
 * backing files, hence all their sectors are reported as allocated.
2395
 *
2396
 * 'mode' serves as a hint as to which results are favored; see the
2397
 * BDRV_WANT_* macros for details.
2398
 *
2399
 * If 'offset' is beyond the end of the disk image the return value is
2400
 * BDRV_BLOCK_EOF and 'pnum' is set to 0.
2401
 *
2402
 * 'bytes' is the max value 'pnum' should be set to.  If bytes goes
2403
 * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2404
 * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
2405
 *
2406
 * 'pnum' is set to the number of bytes (including and immediately
2407
 * following the specified offset) that are easily known to be in the
2408
 * same allocated/unallocated state.  Note that a second call starting
2409
 * at the original offset plus returned pnum may have the same status.
2410
 * The returned value is non-zero on success except at end-of-file.
2411
 *
2412
 * Returns negative errno on failure.  Otherwise, if the
2413
 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
2414
 * set to the host mapping and BDS corresponding to the guest offset.
2415
 */
2416
static int coroutine_fn GRAPH_RDLOCK
2417
bdrv_co_do_block_status(BlockDriverState *bs, unsigned int mode,
2418
                        int64_t offset, int64_t bytes,
2419
                        int64_t *pnum, int64_t *map, BlockDriverState **file)
2420
0
{
2421
0
    int64_t total_size;
2422
0
    int64_t n; /* bytes */
2423
0
    int ret;
2424
0
    int64_t local_map = 0;
2425
0
    BlockDriverState *local_file = NULL;
2426
0
    int64_t aligned_offset, aligned_bytes;
2427
0
    uint32_t align;
2428
0
    bool has_filtered_child;
2429
2430
0
    assert(pnum);
2431
0
    assert_bdrv_graph_readable();
2432
0
    *pnum = 0;
2433
0
    total_size = bdrv_co_getlength(bs);
2434
0
    if (total_size < 0) {
2435
0
        ret = total_size;
2436
0
        goto early_out;
2437
0
    }
2438
2439
0
    if (offset >= total_size) {
2440
0
        ret = BDRV_BLOCK_EOF;
2441
0
        goto early_out;
2442
0
    }
2443
0
    if (!bytes) {
2444
0
        ret = 0;
2445
0
        goto early_out;
2446
0
    }
2447
2448
0
    n = total_size - offset;
2449
0
    if (n < bytes) {
2450
0
        bytes = n;
2451
0
    }
2452
2453
    /* Must be non-NULL or bdrv_co_getlength() would have failed */
2454
0
    assert(bs->drv);
2455
0
    has_filtered_child = bdrv_filter_child(bs);
2456
0
    if (!bs->drv->bdrv_co_block_status && !has_filtered_child) {
2457
0
        *pnum = bytes;
2458
0
        ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
2459
0
        if (offset + bytes == total_size) {
2460
0
            ret |= BDRV_BLOCK_EOF;
2461
0
        }
2462
0
        if (bs->drv->protocol_name) {
2463
0
            ret |= BDRV_BLOCK_OFFSET_VALID;
2464
0
            local_map = offset;
2465
0
            local_file = bs;
2466
0
        }
2467
0
        goto early_out;
2468
0
    }
2469
2470
0
    bdrv_inc_in_flight(bs);
2471
2472
    /* Round out to request_alignment boundaries */
2473
0
    align = bs->bl.request_alignment;
2474
0
    aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2475
0
    aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2476
2477
0
    if (bs->drv->bdrv_co_block_status) {
2478
        /*
2479
         * Use the block-status cache only for protocol nodes: Format
2480
         * drivers are generally quick to inquire the status, but protocol
2481
         * drivers often need to get information from outside of qemu, so
2482
         * we do not have control over the actual implementation.  There
2483
         * have been cases where inquiring the status took an unreasonably
2484
         * long time, and we can do nothing in qemu to fix it.
2485
         * This is especially problematic for images with large data areas,
2486
         * because finding the few holes in them and giving them special
2487
         * treatment does not gain much performance.  Therefore, we try to
2488
         * cache the last-identified data region.
2489
         *
2490
         * Second, limiting ourselves to protocol nodes allows us to assume
2491
         * the block status for data regions to be DATA | OFFSET_VALID, and
2492
         * that the host offset is the same as the guest offset.
2493
         *
2494
         * Note that it is possible that external writers zero parts of
2495
         * the cached regions without the cache being invalidated, and so
2496
         * we may report zeroes as data.  This is not catastrophic,
2497
         * however, because reporting zeroes as data is fine.
2498
         */
2499
0
        if (QLIST_EMPTY(&bs->children) &&
2500
0
            bdrv_bsc_is_data(bs, aligned_offset, pnum))
2501
0
        {
2502
0
            ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
2503
0
            local_file = bs;
2504
0
            local_map = aligned_offset;
2505
0
        } else {
2506
0
            ret = bs->drv->bdrv_co_block_status(bs, mode, aligned_offset,
2507
0
                                                aligned_bytes, pnum, &local_map,
2508
0
                                                &local_file);
2509
2510
            /*
2511
             * Note that checking QLIST_EMPTY(&bs->children) is also done when
2512
             * the cache is queried above.  Technically, we do not need to check
2513
             * it here; the worst that can happen is that we fill the cache for
2514
             * non-protocol nodes, and then it is never used.  However, filling
2515
             * the cache requires an RCU update, so double check here to avoid
2516
             * such an update if possible.
2517
             *
2518
             * Check mode, because we only want to update the cache when we
2519
             * have accurate information about what is zero and what is data.
2520
             */
2521
0
            if (mode == BDRV_WANT_PRECISE &&
2522
0
                ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) &&
2523
0
                QLIST_EMPTY(&bs->children))
2524
0
            {
2525
                /*
2526
                 * When a protocol driver reports BLOCK_OFFSET_VALID, the
2527
                 * returned local_map value must be the same as the offset we
2528
                 * have passed (aligned_offset), and local_bs must be the node
2529
                 * itself.
2530
                 * Assert this, because we follow this rule when reading from
2531
                 * the cache (see the `local_file = bs` and
2532
                 * `local_map = aligned_offset` assignments above), and the
2533
                 * result the cache delivers must be the same as the driver
2534
                 * would deliver.
2535
                 */
2536
0
                assert(local_file == bs);
2537
0
                assert(local_map == aligned_offset);
2538
0
                bdrv_bsc_fill(bs, aligned_offset, *pnum);
2539
0
            }
2540
0
        }
2541
0
    } else {
2542
        /* Default code for filters */
2543
2544
0
        local_file = bdrv_filter_bs(bs);
2545
0
        assert(local_file);
2546
2547
0
        *pnum = aligned_bytes;
2548
0
        local_map = aligned_offset;
2549
0
        ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2550
0
    }
2551
0
    if (ret < 0) {
2552
0
        *pnum = 0;
2553
0
        goto out;
2554
0
    }
2555
2556
    /*
2557
     * The driver's result must be a non-zero multiple of request_alignment.
2558
     * Clamp pnum and adjust map to original request.
2559
     */
2560
0
    assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2561
0
           align > offset - aligned_offset);
2562
0
    if (ret & BDRV_BLOCK_RECURSE) {
2563
0
        assert(ret & BDRV_BLOCK_DATA);
2564
0
        assert(ret & BDRV_BLOCK_OFFSET_VALID);
2565
0
        assert(!(ret & BDRV_BLOCK_ZERO));
2566
0
    }
2567
2568
0
    *pnum -= offset - aligned_offset;
2569
0
    if (*pnum > bytes) {
2570
0
        *pnum = bytes;
2571
0
    }
2572
0
    if (ret & BDRV_BLOCK_OFFSET_VALID) {
2573
0
        local_map += offset - aligned_offset;
2574
0
    }
2575
2576
0
    if (ret & BDRV_BLOCK_RAW) {
2577
0
        assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
2578
0
        ret = bdrv_co_do_block_status(local_file, mode, local_map,
2579
0
                                      *pnum, pnum, &local_map, &local_file);
2580
0
        goto out;
2581
0
    }
2582
2583
0
    if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
2584
0
        ret |= BDRV_BLOCK_ALLOCATED;
2585
0
    } else if (bs->drv->supports_backing) {
2586
0
        BlockDriverState *cow_bs = bdrv_cow_bs(bs);
2587
2588
0
        if (!cow_bs) {
2589
0
            ret |= BDRV_BLOCK_ZERO;
2590
0
        } else if (mode == BDRV_WANT_PRECISE) {
2591
0
            int64_t size2 = bdrv_co_getlength(cow_bs);
2592
2593
0
            if (size2 >= 0 && offset >= size2) {
2594
0
                ret |= BDRV_BLOCK_ZERO;
2595
0
            }
2596
0
        }
2597
0
    }
2598
2599
0
    if (mode == BDRV_WANT_PRECISE && ret & BDRV_BLOCK_RECURSE &&
2600
0
        local_file && local_file != bs &&
2601
0
        (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
2602
0
        (ret & BDRV_BLOCK_OFFSET_VALID)) {
2603
0
        int64_t file_pnum;
2604
0
        int ret2;
2605
2606
0
        ret2 = bdrv_co_do_block_status(local_file, mode, local_map,
2607
0
                                       *pnum, &file_pnum, NULL, NULL);
2608
0
        if (ret2 >= 0) {
2609
            /* Ignore errors.  This is just providing extra information, it
2610
             * is useful but not necessary.
2611
             */
2612
0
            if (ret2 & BDRV_BLOCK_EOF &&
2613
0
                (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2614
                /*
2615
                 * It is valid for the format block driver to read
2616
                 * beyond the end of the underlying file's current
2617
                 * size; such areas read as zero.
2618
                 */
2619
0
                ret |= BDRV_BLOCK_ZERO;
2620
0
            } else {
2621
                /* Limit request to the range reported by the protocol driver */
2622
0
                *pnum = file_pnum;
2623
0
                ret |= (ret2 & BDRV_BLOCK_ZERO);
2624
0
            }
2625
0
        }
2626
2627
        /*
2628
         * Now that the recursive search was done, clear the flag. Otherwise,
2629
         * with more complicated block graphs like snapshot-access ->
2630
         * copy-before-write -> qcow2, where the return value will be propagated
2631
         * further up to a parent bdrv_co_do_block_status() call, both the
2632
         * BDRV_BLOCK_RECURSE and BDRV_BLOCK_ZERO flags would be set, which is
2633
         * not allowed.
2634
         */
2635
0
        ret &= ~BDRV_BLOCK_RECURSE;
2636
0
    }
2637
2638
0
out:
2639
0
    bdrv_dec_in_flight(bs);
2640
0
    if (ret >= 0 && offset + *pnum == total_size) {
2641
0
        ret |= BDRV_BLOCK_EOF;
2642
0
    }
2643
0
early_out:
2644
0
    if (file) {
2645
0
        *file = local_file;
2646
0
    }
2647
0
    if (map) {
2648
0
        *map = local_map;
2649
0
    }
2650
0
    return ret;
2651
0
}
2652
2653
int coroutine_fn
2654
bdrv_co_common_block_status_above(BlockDriverState *bs,
2655
                                  BlockDriverState *base,
2656
                                  bool include_base,
2657
                                  unsigned int mode,
2658
                                  int64_t offset,
2659
                                  int64_t bytes,
2660
                                  int64_t *pnum,
2661
                                  int64_t *map,
2662
                                  BlockDriverState **file,
2663
                                  int *depth)
2664
0
{
2665
0
    int ret;
2666
0
    BlockDriverState *p;
2667
0
    int64_t eof = 0;
2668
0
    int dummy;
2669
0
    IO_CODE();
2670
2671
0
    assert(!include_base || base); /* Can't include NULL base */
2672
0
    assert_bdrv_graph_readable();
2673
2674
0
    if (!depth) {
2675
0
        depth = &dummy;
2676
0
    }
2677
0
    *depth = 0;
2678
2679
0
    if (!include_base && bs == base) {
2680
0
        *pnum = bytes;
2681
0
        return 0;
2682
0
    }
2683
2684
0
    ret = bdrv_co_do_block_status(bs, mode, offset, bytes, pnum,
2685
0
                                  map, file);
2686
0
    ++*depth;
2687
0
    if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) {
2688
0
        return ret;
2689
0
    }
2690
2691
0
    if (ret & BDRV_BLOCK_EOF) {
2692
0
        eof = offset + *pnum;
2693
0
    }
2694
2695
0
    assert(*pnum <= bytes);
2696
0
    bytes = *pnum;
2697
2698
0
    for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base;
2699
0
         p = bdrv_filter_or_cow_bs(p))
2700
0
    {
2701
0
        ret = bdrv_co_do_block_status(p, mode, offset, bytes, pnum,
2702
0
                                      map, file);
2703
0
        ++*depth;
2704
0
        if (ret < 0) {
2705
0
            return ret;
2706
0
        }
2707
0
        if (*pnum == 0) {
2708
            /*
2709
             * The top layer deferred to this layer, and because this layer is
2710
             * short, any zeroes that we synthesize beyond EOF behave as if they
2711
             * were allocated at this layer.
2712
             *
2713
             * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be
2714
             * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2715
             * below.
2716
             */
2717
0
            assert(ret & BDRV_BLOCK_EOF);
2718
0
            *pnum = bytes;
2719
0
            if (file) {
2720
0
                *file = p;
2721
0
            }
2722
0
            ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED;
2723
0
            break;
2724
0
        }
2725
0
        if (ret & BDRV_BLOCK_ALLOCATED) {
2726
            /*
2727
             * We've found the node and the status, we must break.
2728
             *
2729
             * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be
2730
             * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2731
             * below.
2732
             */
2733
0
            ret &= ~BDRV_BLOCK_EOF;
2734
0
            break;
2735
0
        }
2736
2737
0
        if (p == base) {
2738
0
            assert(include_base);
2739
0
            break;
2740
0
        }
2741
2742
        /*
2743
         * OK, [offset, offset + *pnum) region is unallocated on this layer,
2744
         * let's continue the diving.
2745
         */
2746
0
        assert(*pnum <= bytes);
2747
0
        bytes = *pnum;
2748
0
    }
2749
2750
0
    if (offset + *pnum == eof) {
2751
0
        ret |= BDRV_BLOCK_EOF;
2752
0
    }
2753
2754
0
    return ret;
2755
0
}
2756
2757
int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs,
2758
                                            BlockDriverState *base,
2759
                                            int64_t offset, int64_t bytes,
2760
                                            int64_t *pnum, int64_t *map,
2761
                                            BlockDriverState **file)
2762
0
{
2763
0
    IO_CODE();
2764
0
    return bdrv_co_common_block_status_above(bs, base, false,
2765
0
                                             BDRV_WANT_PRECISE, offset,
2766
0
                                             bytes, pnum, map, file, NULL);
2767
0
}
2768
2769
int coroutine_fn bdrv_co_block_status(BlockDriverState *bs, int64_t offset,
2770
                                      int64_t bytes, int64_t *pnum,
2771
                                      int64_t *map, BlockDriverState **file)
2772
0
{
2773
0
    IO_CODE();
2774
0
    return bdrv_co_block_status_above(bs, bdrv_filter_or_cow_bs(bs),
2775
0
                                      offset, bytes, pnum, map, file);
2776
0
}
2777
2778
/*
2779
 * Check @bs (and its backing chain) to see if the range defined
2780
 * by @offset and @bytes is known to read as zeroes.
2781
 * Return 1 if that is the case, 0 otherwise and -errno on error.
2782
 * This test is meant to be fast rather than accurate so returning 0
2783
 * does not guarantee non-zero data; but a return of 1 is reliable.
2784
 */
2785
int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset,
2786
                                      int64_t bytes)
2787
0
{
2788
0
    int ret;
2789
0
    int64_t pnum;
2790
0
    IO_CODE();
2791
2792
0
    while (bytes) {
2793
0
        ret = bdrv_co_common_block_status_above(bs, NULL, false,
2794
0
                                                BDRV_WANT_ZERO, offset, bytes,
2795
0
                                                &pnum, NULL, NULL, NULL);
2796
2797
0
        if (ret < 0) {
2798
0
            return ret;
2799
0
        }
2800
0
        if (!(ret & BDRV_BLOCK_ZERO)) {
2801
0
            return 0;
2802
0
        }
2803
0
        offset += pnum;
2804
0
        bytes -= pnum;
2805
0
    }
2806
2807
0
    return 1;
2808
0
}
2809
2810
/*
2811
 * Check @bs (and its backing chain) to see if the entire image is known
2812
 * to read as zeroes.
2813
 * Return 1 if that is the case, 0 otherwise and -errno on error.
2814
 * This test is meant to be fast rather than accurate so returning 0
2815
 * does not guarantee non-zero data; however, a return of 1 is reliable,
2816
 * and this function can report 1 in more cases than bdrv_co_is_zero_fast.
2817
 */
2818
int coroutine_fn bdrv_co_is_all_zeroes(BlockDriverState *bs)
2819
0
{
2820
0
    int ret;
2821
0
    int64_t pnum, bytes;
2822
0
    char *buf;
2823
0
    QEMUIOVector local_qiov;
2824
0
    IO_CODE();
2825
2826
0
    bytes = bdrv_co_getlength(bs);
2827
0
    if (bytes < 0) {
2828
0
        return bytes;
2829
0
    }
2830
2831
    /* First probe - see if the entire image reads as zero */
2832
0
    ret = bdrv_co_common_block_status_above(bs, NULL, false, BDRV_WANT_ZERO,
2833
0
                                            0, bytes, &pnum, NULL, NULL,
2834
0
                                            NULL);
2835
0
    if (ret < 0) {
2836
0
        return ret;
2837
0
    }
2838
0
    if (ret & BDRV_BLOCK_ZERO) {
2839
0
        return bdrv_co_is_zero_fast(bs, pnum, bytes - pnum);
2840
0
    }
2841
2842
    /*
2843
     * Because of the way 'blockdev-create' works, raw files tend to
2844
     * be created with a non-sparse region at the front to make
2845
     * alignment probing easier.  If the block starts with only a
2846
     * small allocated region, it is still worth the effort to see if
2847
     * the rest of the image is still sparse, coupled with manually
2848
     * reading the first region to see if it reads zero after all.
2849
     */
2850
0
    if (pnum > MAX_ZERO_CHECK_BUFFER) {
2851
0
        return 0;
2852
0
    }
2853
0
    ret = bdrv_co_is_zero_fast(bs, pnum, bytes - pnum);
2854
0
    if (ret <= 0) {
2855
0
        return ret;
2856
0
    }
2857
    /* Only the head of the image is unknown, and it's small.  Read it.  */
2858
0
    buf = qemu_blockalign(bs, pnum);
2859
0
    qemu_iovec_init_buf(&local_qiov, buf, pnum);
2860
0
    ret = bdrv_driver_preadv(bs, 0, pnum, &local_qiov, 0, 0);
2861
0
    if (ret >= 0) {
2862
0
        ret = buffer_is_zero(buf, pnum);
2863
0
    }
2864
0
    qemu_vfree(buf);
2865
0
    return ret;
2866
0
}
2867
2868
int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t offset,
2869
                                      int64_t bytes, int64_t *pnum)
2870
0
{
2871
0
    int ret;
2872
0
    int64_t dummy;
2873
0
    IO_CODE();
2874
2875
0
    ret = bdrv_co_common_block_status_above(bs, bs, true, BDRV_WANT_ALLOCATED,
2876
0
                                            offset, bytes, pnum ? pnum : &dummy,
2877
0
                                            NULL, NULL, NULL);
2878
0
    if (ret < 0) {
2879
0
        return ret;
2880
0
    }
2881
0
    return !!(ret & BDRV_BLOCK_ALLOCATED);
2882
0
}
2883
2884
/*
2885
 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
2886
 *
2887
 * Return a positive depth if (a prefix of) the given range is allocated
2888
 * in any image between BASE and TOP (BASE is only included if include_base
2889
 * is set).  Depth 1 is TOP, 2 is the first backing layer, and so forth.
2890
 * BASE can be NULL to check if the given offset is allocated in any
2891
 * image of the chain.  Return 0 otherwise, or negative errno on
2892
 * failure.
2893
 *
2894
 * 'pnum' is set to the number of bytes (including and immediately
2895
 * following the specified offset) that are known to be in the same
2896
 * allocated/unallocated state.  Note that a subsequent call starting
2897
 * at 'offset + *pnum' may return the same allocation status (in other
2898
 * words, the result is not necessarily the maximum possible range);
2899
 * but 'pnum' will only be 0 when end of file is reached.
2900
 */
2901
int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *bs,
2902
                                            BlockDriverState *base,
2903
                                            bool include_base, int64_t offset,
2904
                                            int64_t bytes, int64_t *pnum)
2905
0
{
2906
0
    int depth;
2907
0
    int ret;
2908
0
    IO_CODE();
2909
2910
0
    ret = bdrv_co_common_block_status_above(bs, base, include_base,
2911
0
                                            BDRV_WANT_ALLOCATED,
2912
0
                                            offset, bytes, pnum, NULL, NULL,
2913
0
                                            &depth);
2914
0
    if (ret < 0) {
2915
0
        return ret;
2916
0
    }
2917
2918
0
    if (ret & BDRV_BLOCK_ALLOCATED) {
2919
0
        return depth;
2920
0
    }
2921
0
    return 0;
2922
0
}
2923
2924
int coroutine_fn
2925
bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2926
0
{
2927
0
    BlockDriver *drv = bs->drv;
2928
0
    BlockDriverState *child_bs = bdrv_primary_bs(bs);
2929
0
    int ret;
2930
0
    IO_CODE();
2931
0
    assert_bdrv_graph_readable();
2932
2933
0
    ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2934
0
    if (ret < 0) {
2935
0
        return ret;
2936
0
    }
2937
2938
0
    if (!drv) {
2939
0
        return -ENOMEDIUM;
2940
0
    }
2941
2942
0
    bdrv_inc_in_flight(bs);
2943
2944
0
    if (drv->bdrv_co_load_vmstate) {
2945
0
        ret = drv->bdrv_co_load_vmstate(bs, qiov, pos);
2946
0
    } else if (child_bs) {
2947
0
        ret = bdrv_co_readv_vmstate(child_bs, qiov, pos);
2948
0
    } else {
2949
0
        ret = -ENOTSUP;
2950
0
    }
2951
2952
0
    bdrv_dec_in_flight(bs);
2953
2954
0
    return ret;
2955
0
}
2956
2957
int coroutine_fn
2958
bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2959
0
{
2960
0
    BlockDriver *drv = bs->drv;
2961
0
    BlockDriverState *child_bs = bdrv_primary_bs(bs);
2962
0
    int ret;
2963
0
    IO_CODE();
2964
0
    assert_bdrv_graph_readable();
2965
2966
0
    ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2967
0
    if (ret < 0) {
2968
0
        return ret;
2969
0
    }
2970
2971
0
    if (!drv) {
2972
0
        return -ENOMEDIUM;
2973
0
    }
2974
2975
0
    bdrv_inc_in_flight(bs);
2976
2977
0
    if (drv->bdrv_co_save_vmstate) {
2978
0
        ret = drv->bdrv_co_save_vmstate(bs, qiov, pos);
2979
0
    } else if (child_bs) {
2980
0
        ret = bdrv_co_writev_vmstate(child_bs, qiov, pos);
2981
0
    } else {
2982
0
        ret = -ENOTSUP;
2983
0
    }
2984
2985
0
    bdrv_dec_in_flight(bs);
2986
2987
0
    return ret;
2988
0
}
2989
2990
int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
2991
                      int64_t pos, int size)
2992
0
{
2993
0
    QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2994
0
    int ret = bdrv_writev_vmstate(bs, &qiov, pos);
2995
0
    IO_CODE();
2996
2997
0
    return ret < 0 ? ret : size;
2998
0
}
2999
3000
int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
3001
                      int64_t pos, int size)
3002
0
{
3003
0
    QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
3004
0
    int ret = bdrv_readv_vmstate(bs, &qiov, pos);
3005
0
    IO_CODE();
3006
3007
0
    return ret < 0 ? ret : size;
3008
0
}
3009
3010
/**************************************************************/
3011
/* async I/Os */
3012
3013
/**
3014
 * Synchronously cancels an acb. Must be called with the BQL held and the acb
3015
 * must be processed with the BQL held too (IOThreads are not allowed).
3016
 *
3017
 * Use bdrv_aio_cancel_async() instead when possible.
3018
 */
3019
void bdrv_aio_cancel(BlockAIOCB *acb)
3020
317
{
3021
317
    GLOBAL_STATE_CODE();
3022
317
    qemu_aio_ref(acb);
3023
317
    bdrv_aio_cancel_async(acb);
3024
317
    AIO_WAIT_WHILE_UNLOCKED(NULL, acb->refcnt > 1);
3025
317
    qemu_aio_unref(acb);
3026
317
}
3027
3028
/* Async version of aio cancel. The caller is not blocked if the acb implements
3029
 * cancel_async, otherwise we do nothing and let the request normally complete.
3030
 * In either case the completion callback must be called. */
3031
void bdrv_aio_cancel_async(BlockAIOCB *acb)
3032
1.80k
{
3033
1.80k
    IO_CODE();
3034
1.80k
    if (acb->aiocb_info->cancel_async) {
3035
0
        acb->aiocb_info->cancel_async(acb);
3036
0
    }
3037
1.80k
}
3038
3039
/**************************************************************/
3040
/* Coroutine block device emulation */
3041
3042
int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
3043
7.32k
{
3044
7.32k
    BdrvChild *primary_child = bdrv_primary_child(bs);
3045
7.32k
    BdrvChild *child;
3046
7.32k
    int current_gen;
3047
7.32k
    int ret = 0;
3048
7.32k
    IO_CODE();
3049
3050
7.32k
    assert_bdrv_graph_readable();
3051
7.32k
    bdrv_inc_in_flight(bs);
3052
3053
7.32k
    if (!bdrv_co_is_inserted(bs) || bdrv_is_read_only(bs) ||
3054
7.32k
        bdrv_is_sg(bs)) {
3055
0
        goto early_exit;
3056
0
    }
3057
3058
14.6k
    qemu_mutex_lock(&bs->reqs_lock);
3059
14.6k
    current_gen = qatomic_read(&bs->write_gen);
3060
3061
    /* Wait until any previous flushes are completed */
3062
7.32k
    while (bs->active_flush_req) {
3063
0
        qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
3064
0
    }
3065
3066
    /* Flushes reach this point in nondecreasing current_gen order.  */
3067
7.32k
    bs->active_flush_req = true;
3068
7.32k
    qemu_mutex_unlock(&bs->reqs_lock);
3069
3070
    /* Write back all layers by calling one driver function */
3071
7.32k
    if (bs->drv->bdrv_co_flush) {
3072
0
        ret = bs->drv->bdrv_co_flush(bs);
3073
0
        goto out;
3074
0
    }
3075
3076
    /* Write back cached data to the OS even with cache=unsafe */
3077
7.32k
    BLKDBG_CO_EVENT(primary_child, BLKDBG_FLUSH_TO_OS);
3078
7.32k
    if (bs->drv->bdrv_co_flush_to_os) {
3079
0
        ret = bs->drv->bdrv_co_flush_to_os(bs);
3080
0
        if (ret < 0) {
3081
0
            goto out;
3082
0
        }
3083
0
    }
3084
3085
    /* But don't actually force it to the disk with cache=unsafe */
3086
7.32k
    if (bs->open_flags & BDRV_O_NO_FLUSH) {
3087
0
        goto flush_children;
3088
0
    }
3089
3090
    /* Check if we really need to flush anything */
3091
7.32k
    if (bs->flushed_gen == current_gen) {
3092
5.57k
        goto flush_children;
3093
5.57k
    }
3094
3095
1.75k
    BLKDBG_CO_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK);
3096
1.75k
    if (!bs->drv) {
3097
        /* bs->drv->bdrv_co_flush() might have ejected the BDS
3098
         * (even in case of apparent success) */
3099
0
        ret = -ENOMEDIUM;
3100
0
        goto out;
3101
0
    }
3102
1.75k
    if (bs->drv->bdrv_co_flush_to_disk) {
3103
877
        ret = bs->drv->bdrv_co_flush_to_disk(bs);
3104
877
    } else if (bs->drv->bdrv_aio_flush) {
3105
0
        BlockAIOCB *acb;
3106
0
        CoroutineIOCompletion co = {
3107
0
            .coroutine = qemu_coroutine_self(),
3108
0
        };
3109
3110
0
        acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
3111
0
        if (acb == NULL) {
3112
0
            ret = -EIO;
3113
0
        } else {
3114
0
            qemu_coroutine_yield();
3115
0
            ret = co.ret;
3116
0
        }
3117
877
    } else {
3118
        /*
3119
         * Some block drivers always operate in either writethrough or unsafe
3120
         * mode and don't support bdrv_flush therefore. Usually qemu doesn't
3121
         * know how the server works (because the behaviour is hardcoded or
3122
         * depends on server-side configuration), so we can't ensure that
3123
         * everything is safe on disk. Returning an error doesn't work because
3124
         * that would break guests even if the server operates in writethrough
3125
         * mode.
3126
         *
3127
         * Let's hope the user knows what he's doing.
3128
         */
3129
877
        ret = 0;
3130
877
    }
3131
3132
1.75k
    if (ret < 0) {
3133
0
        goto out;
3134
0
    }
3135
3136
    /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
3137
     * in the case of cache=unsafe, so there are no useless flushes.
3138
     */
3139
7.32k
flush_children:
3140
7.32k
    ret = 0;
3141
7.32k
    QLIST_FOREACH(child, &bs->children, next) {
3142
3.66k
        if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
3143
3.66k
            int this_child_ret = bdrv_co_flush(child->bs);
3144
3.66k
            if (!ret) {
3145
3.66k
                ret = this_child_ret;
3146
3.66k
            }
3147
3.66k
        }
3148
3.66k
    }
3149
3150
7.32k
out:
3151
    /* Notify any pending flushes that we have completed */
3152
7.32k
    if (ret == 0) {
3153
7.32k
        bs->flushed_gen = current_gen;
3154
7.32k
    }
3155
3156
7.32k
    qemu_mutex_lock(&bs->reqs_lock);
3157
7.32k
    bs->active_flush_req = false;
3158
    /* Return value is ignored - it's ok if wait queue is empty */
3159
7.32k
    qemu_co_queue_next(&bs->flush_queue);
3160
7.32k
    qemu_mutex_unlock(&bs->reqs_lock);
3161
3162
7.32k
early_exit:
3163
7.32k
    bdrv_dec_in_flight(bs);
3164
7.32k
    return ret;
3165
7.32k
}
3166
3167
int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
3168
                                  int64_t bytes)
3169
140k
{
3170
140k
    BdrvTrackedRequest req;
3171
140k
    int ret;
3172
140k
    int64_t max_pdiscard;
3173
140k
    int head, tail, align;
3174
140k
    BlockDriverState *bs = child->bs;
3175
140k
    IO_CODE();
3176
140k
    assert_bdrv_graph_readable();
3177
3178
140k
    if (!bs || !bs->drv || !bdrv_co_is_inserted(bs)) {
3179
0
        return -ENOMEDIUM;
3180
0
    }
3181
3182
140k
    if (bdrv_has_readonly_bitmaps(bs)) {
3183
0
        return -EPERM;
3184
0
    }
3185
3186
140k
    ret = bdrv_check_request(offset, bytes, NULL);
3187
140k
    if (ret < 0) {
3188
0
        return ret;
3189
0
    }
3190
3191
    /* Do nothing if disabled.  */
3192
140k
    if (!(bs->open_flags & BDRV_O_UNMAP)) {
3193
140k
        return 0;
3194
140k
    }
3195
3196
0
    if (!bs->drv->bdrv_co_pdiscard) {
3197
0
        return 0;
3198
0
    }
3199
3200
    /* Invalidate the cached block-status data range if this discard overlaps */
3201
0
    bdrv_bsc_invalidate_range(bs, offset, bytes);
3202
3203
    /*
3204
     * Discard is advisory, but some devices track and coalesce
3205
     * unaligned requests, so we must pass everything down rather than
3206
     * round here.  Still, most devices reject unaligned requests with
3207
     * -EINVAL or -ENOTSUP, so we must fragment the request accordingly.
3208
     */
3209
0
    align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
3210
0
    assert(align % bs->bl.request_alignment == 0);
3211
0
    head = offset % align;
3212
0
    tail = (offset + bytes) % align;
3213
3214
0
    bdrv_inc_in_flight(bs);
3215
0
    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
3216
3217
0
    ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
3218
0
    if (ret < 0) {
3219
0
        goto out;
3220
0
    }
3221
3222
0
    max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX),
3223
0
                                   align);
3224
0
    assert(max_pdiscard >= bs->bl.request_alignment);
3225
3226
0
    while (bytes > 0) {
3227
0
        int64_t num = bytes;
3228
3229
0
        if (head) {
3230
            /* Make small requests to get to alignment boundaries. */
3231
0
            num = MIN(bytes, align - head);
3232
0
            if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
3233
0
                num %= bs->bl.request_alignment;
3234
0
            }
3235
0
            head = (head + num) % align;
3236
0
            assert(num < max_pdiscard);
3237
0
        } else if (tail) {
3238
0
            if (num > align) {
3239
                /* Shorten the request to the last aligned cluster.  */
3240
0
                num -= tail;
3241
0
            } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
3242
0
                       tail > bs->bl.request_alignment) {
3243
0
                tail %= bs->bl.request_alignment;
3244
0
                num -= tail;
3245
0
            }
3246
0
        }
3247
        /* limit request size */
3248
0
        if (num > max_pdiscard) {
3249
0
            num = max_pdiscard;
3250
0
        }
3251
3252
0
        if (!bs->drv) {
3253
0
            ret = -ENOMEDIUM;
3254
0
            goto out;
3255
0
        }
3256
3257
0
        ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
3258
0
        if (ret && ret != -ENOTSUP) {
3259
0
            if (ret == -EINVAL && (offset % align != 0 || num % align != 0)) {
3260
                /* Silently skip rejected unaligned head/tail requests */
3261
0
            } else {
3262
0
                goto out; /* bail out */
3263
0
            }
3264
0
        }
3265
3266
0
        offset += num;
3267
0
        bytes -= num;
3268
0
    }
3269
0
    ret = 0;
3270
0
out:
3271
0
    bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
3272
0
    tracked_request_end(&req);
3273
0
    bdrv_dec_in_flight(bs);
3274
0
    return ret;
3275
0
}
3276
3277
int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
3278
0
{
3279
0
    BlockDriver *drv = bs->drv;
3280
0
    CoroutineIOCompletion co = {
3281
0
        .coroutine = qemu_coroutine_self(),
3282
0
    };
3283
0
    BlockAIOCB *acb;
3284
0
    IO_CODE();
3285
0
    assert_bdrv_graph_readable();
3286
3287
0
    bdrv_inc_in_flight(bs);
3288
0
    if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
3289
0
        co.ret = -ENOTSUP;
3290
0
        goto out;
3291
0
    }
3292
3293
0
    if (drv->bdrv_co_ioctl) {
3294
0
        co.ret = drv->bdrv_co_ioctl(bs, req, buf);
3295
0
    } else {
3296
0
        acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
3297
0
        if (!acb) {
3298
0
            co.ret = -ENOTSUP;
3299
0
            goto out;
3300
0
        }
3301
0
        qemu_coroutine_yield();
3302
0
    }
3303
0
out:
3304
0
    bdrv_dec_in_flight(bs);
3305
0
    return co.ret;
3306
0
}
3307
3308
int coroutine_fn bdrv_co_zone_report(BlockDriverState *bs, int64_t offset,
3309
                        unsigned int *nr_zones,
3310
                        BlockZoneDescriptor *zones)
3311
0
{
3312
0
    BlockDriver *drv = bs->drv;
3313
0
    int ret;
3314
0
    IO_CODE();
3315
3316
0
    bdrv_inc_in_flight(bs);
3317
0
    if (!drv || !drv->bdrv_co_zone_report || bs->bl.zoned == BLK_Z_NONE) {
3318
0
        ret = -ENOTSUP;
3319
0
        goto out;
3320
0
    }
3321
0
    ret = drv->bdrv_co_zone_report(bs, offset, nr_zones, zones);
3322
0
out:
3323
0
    bdrv_dec_in_flight(bs);
3324
0
    return ret;
3325
0
}
3326
3327
int coroutine_fn bdrv_co_zone_mgmt(BlockDriverState *bs, BlockZoneOp op,
3328
        int64_t offset, int64_t len)
3329
0
{
3330
0
    BlockDriver *drv = bs->drv;
3331
0
    int ret;
3332
0
    IO_CODE();
3333
3334
0
    bdrv_inc_in_flight(bs);
3335
0
    if (!drv || !drv->bdrv_co_zone_mgmt || bs->bl.zoned == BLK_Z_NONE) {
3336
0
        ret = -ENOTSUP;
3337
0
        goto out;
3338
0
    }
3339
0
    ret = drv->bdrv_co_zone_mgmt(bs, op, offset, len);
3340
0
out:
3341
0
    bdrv_dec_in_flight(bs);
3342
0
    return ret;
3343
0
}
3344
3345
int coroutine_fn bdrv_co_zone_append(BlockDriverState *bs, int64_t *offset,
3346
                        QEMUIOVector *qiov,
3347
                        BdrvRequestFlags flags)
3348
0
{
3349
0
    int ret;
3350
0
    BlockDriver *drv = bs->drv;
3351
0
    IO_CODE();
3352
3353
0
    ret = bdrv_check_qiov_request(*offset, qiov->size, qiov, 0, NULL);
3354
0
    if (ret < 0) {
3355
0
        return ret;
3356
0
    }
3357
3358
    /*
3359
     * Zone write pointers are kept and reported in units of BDRV_SECTOR_SIZE,
3360
     * so an append that would leave a write pointer at a finer granularity
3361
     * cannot be represented. Drivers may impose a coarser granularity of their
3362
     * own, see BlockLimits.write_granularity.
3363
     */
3364
0
    if (!QEMU_IS_ALIGNED(qiov->size, BDRV_SECTOR_SIZE)) {
3365
0
        return -EINVAL;
3366
0
    }
3367
3368
    /*
3369
     * An append cannot cross a zone boundary, so one that is larger than a
3370
     * zone can never be carried out, wherever the write pointer of the zone
3371
     * is. The exact bound is the part of the zone that is still writable,
3372
     * which only a driver can check, as only it holds the write pointer.
3373
     * zone_size is zero when the device is not zoned, which is reported as
3374
     * unsupported below.
3375
     */
3376
0
    if (bs->bl.zone_size && qiov->size > bs->bl.zone_size) {
3377
0
        return -EINVAL;
3378
0
    }
3379
3380
0
    bdrv_inc_in_flight(bs);
3381
0
    if (!drv || !drv->bdrv_co_zone_append || bs->bl.zoned == BLK_Z_NONE) {
3382
0
        ret = -ENOTSUP;
3383
0
        goto out;
3384
0
    }
3385
0
    ret = drv->bdrv_co_zone_append(bs, offset, qiov, flags);
3386
0
out:
3387
0
    bdrv_dec_in_flight(bs);
3388
0
    return ret;
3389
0
}
3390
3391
uint32_t bdrv_zone_index(BlockDriverState *bs, uint64_t offset)
3392
0
{
3393
0
    IO_CODE();
3394
3395
0
    return offset >> bs->bl.zone_size_bits;
3396
0
}
3397
3398
bool bdrv_zone_is_conv(BlockDriverState *bs, uint32_t index)
3399
0
{
3400
0
    IO_CODE();
3401
3402
0
    return BDRV_ZT_IS_CONV(bs->wps->wp[index]);
3403
0
}
3404
3405
bool bdrv_zone_is_full(BlockDriverState *bs, uint32_t index)
3406
0
{
3407
0
    uint64_t zone_end = MIN((uint64_t)(index + 1) * bs->bl.zone_size,
3408
0
                            (uint64_t)bs->total_sectors << BDRV_SECTOR_BITS);
3409
0
    IO_CODE();
3410
3411
0
    return bs->wps->wp[index] >= zone_end;
3412
0
}
3413
3414
void *qemu_blockalign(BlockDriverState *bs, size_t size)
3415
77.4k
{
3416
77.4k
    IO_CODE();
3417
77.4k
    return qemu_memalign(bdrv_opt_mem_align(bs), size);
3418
77.4k
}
3419
3420
void *qemu_blockalign0(BlockDriverState *bs, size_t size)
3421
0
{
3422
0
    IO_CODE();
3423
0
    return memset(qemu_blockalign(bs, size), 0, size);
3424
0
}
3425
3426
void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
3427
466
{
3428
466
    size_t align = bdrv_opt_mem_align(bs);
3429
466
    IO_CODE();
3430
3431
    /* Ensure that NULL is never returned on success */
3432
466
    assert(align > 0);
3433
466
    if (size == 0) {
3434
0
        size = align;
3435
0
    }
3436
3437
466
    return qemu_try_memalign(align, size);
3438
466
}
3439
3440
void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
3441
466
{
3442
466
    void *mem = qemu_try_blockalign(bs, size);
3443
466
    IO_CODE();
3444
3445
466
    if (mem) {
3446
466
        memset(mem, 0, size);
3447
466
    }
3448
3449
466
    return mem;
3450
466
}
3451
3452
/* Helper that undoes bdrv_register_buf() when it fails partway through */
3453
static void GRAPH_RDLOCK
3454
bdrv_register_buf_rollback(BlockDriverState *bs, void *host, size_t size,
3455
                           BdrvChild *final_child)
3456
0
{
3457
0
    BdrvChild *child;
3458
3459
0
    GLOBAL_STATE_CODE();
3460
0
    assert_bdrv_graph_readable();
3461
3462
0
    QLIST_FOREACH(child, &bs->children, next) {
3463
0
        if (child == final_child) {
3464
0
            break;
3465
0
        }
3466
3467
0
        bdrv_unregister_buf(child->bs, host, size);
3468
0
    }
3469
3470
0
    if (bs->drv && bs->drv->bdrv_unregister_buf) {
3471
0
        bs->drv->bdrv_unregister_buf(bs, host, size);
3472
0
    }
3473
0
}
3474
3475
bool bdrv_register_buf(BlockDriverState *bs, void *host, size_t size,
3476
                       Error **errp)
3477
36
{
3478
36
    BdrvChild *child;
3479
3480
36
    GLOBAL_STATE_CODE();
3481
36
    GRAPH_RDLOCK_GUARD_MAINLOOP();
3482
3483
36
    if (bs->drv && bs->drv->bdrv_register_buf) {
3484
0
        if (!bs->drv->bdrv_register_buf(bs, host, size, errp)) {
3485
0
            return false;
3486
0
        }
3487
0
    }
3488
36
    QLIST_FOREACH(child, &bs->children, next) {
3489
18
        if (!bdrv_register_buf(child->bs, host, size, errp)) {
3490
0
            bdrv_register_buf_rollback(bs, host, size, child);
3491
0
            return false;
3492
0
        }
3493
18
    }
3494
36
    return true;
3495
36
}
3496
3497
void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size)
3498
0
{
3499
0
    BdrvChild *child;
3500
3501
0
    GLOBAL_STATE_CODE();
3502
0
    GRAPH_RDLOCK_GUARD_MAINLOOP();
3503
3504
0
    if (bs->drv && bs->drv->bdrv_unregister_buf) {
3505
0
        bs->drv->bdrv_unregister_buf(bs, host, size);
3506
0
    }
3507
0
    QLIST_FOREACH(child, &bs->children, next) {
3508
0
        bdrv_unregister_buf(child->bs, host, size);
3509
0
    }
3510
0
}
3511
3512
static int coroutine_fn GRAPH_RDLOCK bdrv_co_copy_range_internal(
3513
        BdrvChild *src, int64_t src_offset, BdrvChild *dst,
3514
        int64_t dst_offset, int64_t bytes,
3515
        BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
3516
        bool recurse_src)
3517
0
{
3518
0
    BdrvTrackedRequest req;
3519
0
    int ret;
3520
0
    assert_bdrv_graph_readable();
3521
3522
    /* TODO We can support BDRV_REQ_NO_FALLBACK here */
3523
0
    assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
3524
0
    assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
3525
0
    assert(!(read_flags & BDRV_REQ_NO_WAIT));
3526
0
    assert(!(write_flags & BDRV_REQ_NO_WAIT));
3527
3528
0
    if (!dst || !dst->bs || !bdrv_co_is_inserted(dst->bs)) {
3529
0
        return -ENOMEDIUM;
3530
0
    }
3531
0
    ret = bdrv_check_request32(dst_offset, bytes, NULL, 0);
3532
0
    if (ret) {
3533
0
        return ret;
3534
0
    }
3535
0
    if (write_flags & BDRV_REQ_ZERO_WRITE) {
3536
0
        return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
3537
0
    }
3538
3539
0
    if (!src || !src->bs || !bdrv_co_is_inserted(src->bs)) {
3540
0
        return -ENOMEDIUM;
3541
0
    }
3542
0
    ret = bdrv_check_request32(src_offset, bytes, NULL, 0);
3543
0
    if (ret) {
3544
0
        return ret;
3545
0
    }
3546
3547
0
    if (!src->bs->drv->bdrv_co_copy_range_from
3548
0
        || !dst->bs->drv->bdrv_co_copy_range_to
3549
0
        || src->bs->encrypted || dst->bs->encrypted) {
3550
0
        return -ENOTSUP;
3551
0
    }
3552
3553
0
    if (recurse_src) {
3554
0
        bdrv_inc_in_flight(src->bs);
3555
0
        tracked_request_begin(&req, src->bs, src_offset, bytes,
3556
0
                              BDRV_TRACKED_READ);
3557
3558
        /* BDRV_REQ_SERIALISING is only for write operation */
3559
0
        assert(!(read_flags & BDRV_REQ_SERIALISING));
3560
3561
0
        ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3562
0
                                                    src, src_offset,
3563
0
                                                    dst, dst_offset,
3564
0
                                                    bytes,
3565
0
                                                    read_flags, write_flags);
3566
3567
0
        tracked_request_end(&req);
3568
0
        bdrv_dec_in_flight(src->bs);
3569
0
    } else {
3570
0
        bdrv_inc_in_flight(dst->bs);
3571
0
        tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3572
0
                              BDRV_TRACKED_WRITE);
3573
0
        ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
3574
0
                                        write_flags);
3575
0
        if (!ret) {
3576
0
            ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3577
0
                                                      src, src_offset,
3578
0
                                                      dst, dst_offset,
3579
0
                                                      bytes,
3580
0
                                                      read_flags, write_flags);
3581
0
        }
3582
0
        bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3583
0
        tracked_request_end(&req);
3584
0
        bdrv_dec_in_flight(dst->bs);
3585
0
    }
3586
3587
0
    return ret;
3588
0
}
3589
3590
/* Copy range from @src to @dst.
3591
 *
3592
 * See the comment of bdrv_co_copy_range for the parameter and return value
3593
 * semantics. */
3594
int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset,
3595
                                         BdrvChild *dst, int64_t dst_offset,
3596
                                         int64_t bytes,
3597
                                         BdrvRequestFlags read_flags,
3598
                                         BdrvRequestFlags write_flags)
3599
0
{
3600
0
    IO_CODE();
3601
0
    assert_bdrv_graph_readable();
3602
0
    trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3603
0
                                  read_flags, write_flags);
3604
0
    return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3605
0
                                       bytes, read_flags, write_flags, true);
3606
0
}
3607
3608
/* Copy range from @src to @dst.
3609
 *
3610
 * See the comment of bdrv_co_copy_range for the parameter and return value
3611
 * semantics. */
3612
int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset,
3613
                                       BdrvChild *dst, int64_t dst_offset,
3614
                                       int64_t bytes,
3615
                                       BdrvRequestFlags read_flags,
3616
                                       BdrvRequestFlags write_flags)
3617
0
{
3618
0
    IO_CODE();
3619
0
    assert_bdrv_graph_readable();
3620
0
    trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3621
0
                                read_flags, write_flags);
3622
0
    return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3623
0
                                       bytes, read_flags, write_flags, false);
3624
0
}
3625
3626
int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset,
3627
                                    BdrvChild *dst, int64_t dst_offset,
3628
                                    int64_t bytes, BdrvRequestFlags read_flags,
3629
                                    BdrvRequestFlags write_flags)
3630
0
{
3631
0
    IO_CODE();
3632
0
    assert_bdrv_graph_readable();
3633
3634
0
    return bdrv_co_copy_range_from(src, src_offset,
3635
0
                                   dst, dst_offset,
3636
0
                                   bytes, read_flags, write_flags);
3637
0
}
3638
3639
void coroutine_fn bdrv_co_child_notify_resize(BdrvChild *c)
3640
0
{
3641
0
    IO_CODE();
3642
0
    assert_bdrv_graph_readable();
3643
3644
0
    if (c->klass->resize) {
3645
0
        c->klass->resize(c);
3646
0
    }
3647
0
}
3648
3649
void coroutine_fn bdrv_co_parent_cb_resize(BlockDriverState *bs)
3650
0
{
3651
0
    BdrvChild *c;
3652
3653
0
    IO_CODE();
3654
0
    assert_bdrv_graph_readable();
3655
3656
0
    QLIST_FOREACH(c, &bs->parents, next_parent) {
3657
0
        bdrv_co_child_notify_resize(c);
3658
0
    }
3659
0
}
3660
3661
/**
3662
 * Truncate file to 'offset' bytes (needed only for file protocols)
3663
 *
3664
 * If 'exact' is true, the file must be resized to exactly the given
3665
 * 'offset'.  Otherwise, it is sufficient for the node to be at least
3666
 * 'offset' bytes in length.
3667
 */
3668
int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
3669
                                  PreallocMode prealloc, BdrvRequestFlags flags,
3670
                                  Error **errp)
3671
0
{
3672
0
    BlockDriverState *bs = child->bs;
3673
0
    BdrvChild *filtered, *backing;
3674
0
    BlockDriver *drv = bs->drv;
3675
0
    BdrvTrackedRequest req;
3676
0
    int64_t old_size, new_bytes;
3677
0
    int ret;
3678
0
    IO_CODE();
3679
0
    assert_bdrv_graph_readable();
3680
3681
    /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
3682
0
    if (!drv) {
3683
0
        error_setg(errp, "No medium inserted");
3684
0
        return -ENOMEDIUM;
3685
0
    }
3686
0
    if (offset < 0) {
3687
0
        error_setg(errp, "Image size cannot be negative");
3688
0
        return -EINVAL;
3689
0
    }
3690
3691
0
    ret = bdrv_check_request(offset, 0, errp);
3692
0
    if (ret < 0) {
3693
0
        return ret;
3694
0
    }
3695
3696
0
    old_size = bdrv_co_getlength(bs);
3697
0
    if (old_size < 0) {
3698
0
        error_setg_errno(errp, -old_size, "Failed to get old image size");
3699
0
        return old_size;
3700
0
    }
3701
3702
0
    if (bdrv_is_read_only(bs)) {
3703
0
        error_setg(errp, "Image is read-only");
3704
0
        return -EACCES;
3705
0
    }
3706
3707
0
    if (offset > old_size) {
3708
0
        new_bytes = offset - old_size;
3709
0
    } else {
3710
0
        new_bytes = 0;
3711
0
    }
3712
3713
0
    bdrv_inc_in_flight(bs);
3714
0
    tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
3715
0
                          BDRV_TRACKED_TRUNCATE);
3716
3717
    /* If we are growing the image and potentially using preallocation for the
3718
     * new area, we need to make sure that no write requests are made to it
3719
     * concurrently or they might be overwritten by preallocation. */
3720
0
    if (new_bytes) {
3721
0
        bdrv_make_request_serialising(&req, 1);
3722
0
    }
3723
0
    ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3724
0
                                    0);
3725
0
    if (ret < 0) {
3726
0
        error_setg_errno(errp, -ret,
3727
0
                         "Failed to prepare request for truncation");
3728
0
        goto out;
3729
0
    }
3730
3731
0
    filtered = bdrv_filter_child(bs);
3732
0
    backing = bdrv_cow_child(bs);
3733
3734
    /*
3735
     * If the image has a backing file that is large enough that it would
3736
     * provide data for the new area, we cannot leave it unallocated because
3737
     * then the backing file content would become visible. Instead, zero-fill
3738
     * the new area.
3739
     *
3740
     * Note that if the image has a backing file, but was opened without the
3741
     * backing file, taking care of keeping things consistent with that backing
3742
     * file is the user's responsibility.
3743
     */
3744
0
    if (new_bytes && backing) {
3745
0
        int64_t backing_len;
3746
3747
0
        backing_len = bdrv_co_getlength(backing->bs);
3748
0
        if (backing_len < 0) {
3749
0
            ret = backing_len;
3750
0
            error_setg_errno(errp, -ret, "Could not get backing file size");
3751
0
            goto out;
3752
0
        }
3753
3754
0
        if (backing_len > old_size) {
3755
0
            flags |= BDRV_REQ_ZERO_WRITE;
3756
0
        }
3757
0
    }
3758
3759
0
    if (drv->bdrv_co_truncate) {
3760
0
        if (flags & ~bs->supported_truncate_flags) {
3761
0
            error_setg(errp, "Block driver does not support requested flags");
3762
0
            ret = -ENOTSUP;
3763
0
            goto out;
3764
0
        }
3765
0
        ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp);
3766
0
    } else if (filtered) {
3767
0
        ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp);
3768
0
    } else {
3769
0
        error_setg(errp, "Image format driver does not support resize");
3770
0
        ret = -ENOTSUP;
3771
0
        goto out;
3772
0
    }
3773
0
    if (ret < 0) {
3774
0
        goto out;
3775
0
    }
3776
3777
0
    ret = bdrv_co_refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
3778
0
    if (ret < 0) {
3779
0
        error_setg_errno(errp, -ret, "Could not refresh total sector count");
3780
0
    } else {
3781
0
        offset = bs->total_sectors * BDRV_SECTOR_SIZE;
3782
0
    }
3783
    /*
3784
     * It's possible that truncation succeeded but bdrv_refresh_total_sectors
3785
     * failed, but the latter doesn't affect how we should finish the request.
3786
     * Pass 0 as the last parameter so that dirty bitmaps etc. are handled.
3787
     */
3788
0
    bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
3789
3790
0
out:
3791
0
    tracked_request_end(&req);
3792
0
    bdrv_dec_in_flight(bs);
3793
3794
0
    return ret;
3795
0
}
3796
3797
void bdrv_cancel_in_flight(BlockDriverState *bs)
3798
0
{
3799
0
    GLOBAL_STATE_CODE();
3800
0
    GRAPH_RDLOCK_GUARD_MAINLOOP();
3801
3802
0
    if (!bs || !bs->drv) {
3803
0
        return;
3804
0
    }
3805
3806
0
    if (bs->drv->bdrv_cancel_in_flight) {
3807
0
        bs->drv->bdrv_cancel_in_flight(bs);
3808
0
    }
3809
0
}
3810
3811
int coroutine_fn
3812
bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes,
3813
                        QEMUIOVector *qiov, size_t qiov_offset)
3814
0
{
3815
0
    BlockDriverState *bs = child->bs;
3816
0
    BlockDriver *drv = bs->drv;
3817
0
    int ret;
3818
0
    IO_CODE();
3819
0
    assert_bdrv_graph_readable();
3820
3821
0
    if (!drv) {
3822
0
        return -ENOMEDIUM;
3823
0
    }
3824
3825
0
    if (!drv->bdrv_co_preadv_snapshot) {
3826
0
        return -ENOTSUP;
3827
0
    }
3828
3829
0
    bdrv_inc_in_flight(bs);
3830
0
    ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset);
3831
0
    bdrv_dec_in_flight(bs);
3832
3833
0
    return ret;
3834
0
}
3835
3836
int coroutine_fn
3837
bdrv_co_snapshot_block_status(BlockDriverState *bs, unsigned int mode,
3838
                              int64_t offset, int64_t bytes,
3839
                              int64_t *pnum, int64_t *map,
3840
                              BlockDriverState **file)
3841
0
{
3842
0
    BlockDriver *drv = bs->drv;
3843
0
    int ret;
3844
0
    IO_CODE();
3845
0
    assert_bdrv_graph_readable();
3846
3847
0
    if (!drv) {
3848
0
        return -ENOMEDIUM;
3849
0
    }
3850
3851
0
    if (!drv->bdrv_co_snapshot_block_status) {
3852
0
        return -ENOTSUP;
3853
0
    }
3854
3855
0
    bdrv_inc_in_flight(bs);
3856
0
    ret = drv->bdrv_co_snapshot_block_status(bs, mode, offset, bytes,
3857
0
                                             pnum, map, file);
3858
0
    bdrv_dec_in_flight(bs);
3859
3860
0
    return ret;
3861
0
}
3862
3863
int coroutine_fn
3864
bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes)
3865
0
{
3866
0
    BlockDriver *drv = bs->drv;
3867
0
    int ret;
3868
0
    IO_CODE();
3869
0
    assert_bdrv_graph_readable();
3870
3871
0
    if (!drv) {
3872
0
        return -ENOMEDIUM;
3873
0
    }
3874
3875
0
    if (!drv->bdrv_co_pdiscard_snapshot) {
3876
0
        return -ENOTSUP;
3877
0
    }
3878
3879
0
    bdrv_inc_in_flight(bs);
3880
0
    ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes);
3881
0
    bdrv_dec_in_flight(bs);
3882
3883
0
    return ret;
3884
0
}