xref: /qemu/block/io.c (revision 09147930)
1 /*
2  * Block layer I/O functions
3  *
4  * Copyright (c) 2003 Fabrice Bellard
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/aio-wait.h"
29 #include "block/blockjob.h"
30 #include "block/blockjob_int.h"
31 #include "block/block_int.h"
32 #include "qemu/cutils.h"
33 #include "qapi/error.h"
34 #include "qemu/error-report.h"
35 #include "qemu/main-loop.h"
36 #include "sysemu/replay.h"
37 
38 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
39 
40 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
41 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
42 
43 static void bdrv_parent_cb_resize(BlockDriverState *bs);
44 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
45     int64_t offset, int bytes, BdrvRequestFlags flags);
46 
47 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
48                                       bool ignore_bds_parents)
49 {
50     BdrvChild *c, *next;
51 
52     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
53         if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) {
54             continue;
55         }
56         bdrv_parent_drained_begin_single(c, false);
57     }
58 }
59 
60 static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c,
61                                                    int *drained_end_counter)
62 {
63     assert(c->parent_quiesce_counter > 0);
64     c->parent_quiesce_counter--;
65     if (c->role->drained_end) {
66         c->role->drained_end(c, drained_end_counter);
67     }
68 }
69 
70 void bdrv_parent_drained_end_single(BdrvChild *c)
71 {
72     int drained_end_counter = 0;
73     bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter);
74     BDRV_POLL_WHILE(c->bs, atomic_read(&drained_end_counter) > 0);
75 }
76 
77 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
78                                     bool ignore_bds_parents,
79                                     int *drained_end_counter)
80 {
81     BdrvChild *c;
82 
83     QLIST_FOREACH(c, &bs->parents, next_parent) {
84         if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) {
85             continue;
86         }
87         bdrv_parent_drained_end_single_no_poll(c, drained_end_counter);
88     }
89 }
90 
91 static bool bdrv_parent_drained_poll_single(BdrvChild *c)
92 {
93     if (c->role->drained_poll) {
94         return c->role->drained_poll(c);
95     }
96     return false;
97 }
98 
99 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
100                                      bool ignore_bds_parents)
101 {
102     BdrvChild *c, *next;
103     bool busy = false;
104 
105     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
106         if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) {
107             continue;
108         }
109         busy |= bdrv_parent_drained_poll_single(c);
110     }
111 
112     return busy;
113 }
114 
115 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll)
116 {
117     c->parent_quiesce_counter++;
118     if (c->role->drained_begin) {
119         c->role->drained_begin(c);
120     }
121     if (poll) {
122         BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c));
123     }
124 }
125 
126 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
127 {
128     dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
129     dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
130     dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
131                                  src->opt_mem_alignment);
132     dst->min_mem_alignment = MAX(dst->min_mem_alignment,
133                                  src->min_mem_alignment);
134     dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
135 }
136 
137 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
138 {
139     BlockDriver *drv = bs->drv;
140     Error *local_err = NULL;
141 
142     memset(&bs->bl, 0, sizeof(bs->bl));
143 
144     if (!drv) {
145         return;
146     }
147 
148     /* Default alignment based on whether driver has byte interface */
149     bs->bl.request_alignment = (drv->bdrv_co_preadv ||
150                                 drv->bdrv_aio_preadv ||
151                                 drv->bdrv_co_preadv_part) ? 1 : 512;
152 
153     /* Take some limits from the children as a default */
154     if (bs->file) {
155         bdrv_refresh_limits(bs->file->bs, &local_err);
156         if (local_err) {
157             error_propagate(errp, local_err);
158             return;
159         }
160         bdrv_merge_limits(&bs->bl, &bs->file->bs->bl);
161     } else {
162         bs->bl.min_mem_alignment = 512;
163         bs->bl.opt_mem_alignment = qemu_real_host_page_size;
164 
165         /* Safe default since most protocols use readv()/writev()/etc */
166         bs->bl.max_iov = IOV_MAX;
167     }
168 
169     if (bs->backing) {
170         bdrv_refresh_limits(bs->backing->bs, &local_err);
171         if (local_err) {
172             error_propagate(errp, local_err);
173             return;
174         }
175         bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl);
176     }
177 
178     /* Then let the driver override it */
179     if (drv->bdrv_refresh_limits) {
180         drv->bdrv_refresh_limits(bs, errp);
181     }
182 }
183 
184 /**
185  * The copy-on-read flag is actually a reference count so multiple users may
186  * use the feature without worrying about clobbering its previous state.
187  * Copy-on-read stays enabled until all users have called to disable it.
188  */
189 void bdrv_enable_copy_on_read(BlockDriverState *bs)
190 {
191     atomic_inc(&bs->copy_on_read);
192 }
193 
194 void bdrv_disable_copy_on_read(BlockDriverState *bs)
195 {
196     int old = atomic_fetch_dec(&bs->copy_on_read);
197     assert(old >= 1);
198 }
199 
200 typedef struct {
201     Coroutine *co;
202     BlockDriverState *bs;
203     bool done;
204     bool begin;
205     bool recursive;
206     bool poll;
207     BdrvChild *parent;
208     bool ignore_bds_parents;
209     int *drained_end_counter;
210 } BdrvCoDrainData;
211 
212 static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
213 {
214     BdrvCoDrainData *data = opaque;
215     BlockDriverState *bs = data->bs;
216 
217     if (data->begin) {
218         bs->drv->bdrv_co_drain_begin(bs);
219     } else {
220         bs->drv->bdrv_co_drain_end(bs);
221     }
222 
223     /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */
224     atomic_mb_set(&data->done, true);
225     if (!data->begin) {
226         atomic_dec(data->drained_end_counter);
227     }
228     bdrv_dec_in_flight(bs);
229 
230     g_free(data);
231 }
232 
233 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
234 static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
235                               int *drained_end_counter)
236 {
237     BdrvCoDrainData *data;
238 
239     if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) ||
240             (!begin && !bs->drv->bdrv_co_drain_end)) {
241         return;
242     }
243 
244     data = g_new(BdrvCoDrainData, 1);
245     *data = (BdrvCoDrainData) {
246         .bs = bs,
247         .done = false,
248         .begin = begin,
249         .drained_end_counter = drained_end_counter,
250     };
251 
252     if (!begin) {
253         atomic_inc(drained_end_counter);
254     }
255 
256     /* Make sure the driver callback completes during the polling phase for
257      * drain_begin. */
258     bdrv_inc_in_flight(bs);
259     data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
260     aio_co_schedule(bdrv_get_aio_context(bs), data->co);
261 }
262 
263 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
264 bool bdrv_drain_poll(BlockDriverState *bs, bool recursive,
265                      BdrvChild *ignore_parent, bool ignore_bds_parents)
266 {
267     BdrvChild *child, *next;
268 
269     if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
270         return true;
271     }
272 
273     if (atomic_read(&bs->in_flight)) {
274         return true;
275     }
276 
277     if (recursive) {
278         assert(!ignore_bds_parents);
279         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
280             if (bdrv_drain_poll(child->bs, recursive, child, false)) {
281                 return true;
282             }
283         }
284     }
285 
286     return false;
287 }
288 
289 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive,
290                                       BdrvChild *ignore_parent)
291 {
292     return bdrv_drain_poll(bs, recursive, ignore_parent, false);
293 }
294 
295 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
296                                   BdrvChild *parent, bool ignore_bds_parents,
297                                   bool poll);
298 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
299                                 BdrvChild *parent, bool ignore_bds_parents,
300                                 int *drained_end_counter);
301 
302 static void bdrv_co_drain_bh_cb(void *opaque)
303 {
304     BdrvCoDrainData *data = opaque;
305     Coroutine *co = data->co;
306     BlockDriverState *bs = data->bs;
307 
308     if (bs) {
309         AioContext *ctx = bdrv_get_aio_context(bs);
310         AioContext *co_ctx = qemu_coroutine_get_aio_context(co);
311 
312         /*
313          * When the coroutine yielded, the lock for its home context was
314          * released, so we need to re-acquire it here. If it explicitly
315          * acquired a different context, the lock is still held and we don't
316          * want to lock it a second time (or AIO_WAIT_WHILE() would hang).
317          */
318         if (ctx == co_ctx) {
319             aio_context_acquire(ctx);
320         }
321         bdrv_dec_in_flight(bs);
322         if (data->begin) {
323             assert(!data->drained_end_counter);
324             bdrv_do_drained_begin(bs, data->recursive, data->parent,
325                                   data->ignore_bds_parents, data->poll);
326         } else {
327             assert(!data->poll);
328             bdrv_do_drained_end(bs, data->recursive, data->parent,
329                                 data->ignore_bds_parents,
330                                 data->drained_end_counter);
331         }
332         if (ctx == co_ctx) {
333             aio_context_release(ctx);
334         }
335     } else {
336         assert(data->begin);
337         bdrv_drain_all_begin();
338     }
339 
340     data->done = true;
341     aio_co_wake(co);
342 }
343 
344 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
345                                                 bool begin, bool recursive,
346                                                 BdrvChild *parent,
347                                                 bool ignore_bds_parents,
348                                                 bool poll,
349                                                 int *drained_end_counter)
350 {
351     BdrvCoDrainData data;
352 
353     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
354      * other coroutines run if they were queued by aio_co_enter(). */
355 
356     assert(qemu_in_coroutine());
357     data = (BdrvCoDrainData) {
358         .co = qemu_coroutine_self(),
359         .bs = bs,
360         .done = false,
361         .begin = begin,
362         .recursive = recursive,
363         .parent = parent,
364         .ignore_bds_parents = ignore_bds_parents,
365         .poll = poll,
366         .drained_end_counter = drained_end_counter,
367     };
368 
369     if (bs) {
370         bdrv_inc_in_flight(bs);
371     }
372     replay_bh_schedule_oneshot_event(bdrv_get_aio_context(bs),
373                                      bdrv_co_drain_bh_cb, &data);
374 
375     qemu_coroutine_yield();
376     /* If we are resumed from some other event (such as an aio completion or a
377      * timer callback), it is a bug in the caller that should be fixed. */
378     assert(data.done);
379 }
380 
381 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs,
382                                    BdrvChild *parent, bool ignore_bds_parents)
383 {
384     assert(!qemu_in_coroutine());
385 
386     /* Stop things in parent-to-child order */
387     if (atomic_fetch_inc(&bs->quiesce_counter) == 0) {
388         aio_disable_external(bdrv_get_aio_context(bs));
389     }
390 
391     bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
392     bdrv_drain_invoke(bs, true, NULL);
393 }
394 
395 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
396                                   BdrvChild *parent, bool ignore_bds_parents,
397                                   bool poll)
398 {
399     BdrvChild *child, *next;
400 
401     if (qemu_in_coroutine()) {
402         bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents,
403                                poll, NULL);
404         return;
405     }
406 
407     bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents);
408 
409     if (recursive) {
410         assert(!ignore_bds_parents);
411         bs->recursive_quiesce_counter++;
412         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
413             bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents,
414                                   false);
415         }
416     }
417 
418     /*
419      * Wait for drained requests to finish.
420      *
421      * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
422      * call is needed so things in this AioContext can make progress even
423      * though we don't return to the main AioContext loop - this automatically
424      * includes other nodes in the same AioContext and therefore all child
425      * nodes.
426      */
427     if (poll) {
428         assert(!ignore_bds_parents);
429         BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent));
430     }
431 }
432 
433 void bdrv_drained_begin(BlockDriverState *bs)
434 {
435     bdrv_do_drained_begin(bs, false, NULL, false, true);
436 }
437 
438 void bdrv_subtree_drained_begin(BlockDriverState *bs)
439 {
440     bdrv_do_drained_begin(bs, true, NULL, false, true);
441 }
442 
443 /**
444  * This function does not poll, nor must any of its recursively called
445  * functions.  The *drained_end_counter pointee will be incremented
446  * once for every background operation scheduled, and decremented once
447  * the operation settles.  Therefore, the pointer must remain valid
448  * until the pointee reaches 0.  That implies that whoever sets up the
449  * pointee has to poll until it is 0.
450  *
451  * We use atomic operations to access *drained_end_counter, because
452  * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of
453  *     @bs may contain nodes in different AioContexts,
454  * (2) bdrv_drain_all_end() uses the same counter for all nodes,
455  *     regardless of which AioContext they are in.
456  */
457 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
458                                 BdrvChild *parent, bool ignore_bds_parents,
459                                 int *drained_end_counter)
460 {
461     BdrvChild *child;
462     int old_quiesce_counter;
463 
464     assert(drained_end_counter != NULL);
465 
466     if (qemu_in_coroutine()) {
467         bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
468                                false, drained_end_counter);
469         return;
470     }
471     assert(bs->quiesce_counter > 0);
472 
473     /* Re-enable things in child-to-parent order */
474     bdrv_drain_invoke(bs, false, drained_end_counter);
475     bdrv_parent_drained_end(bs, parent, ignore_bds_parents,
476                             drained_end_counter);
477 
478     old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter);
479     if (old_quiesce_counter == 1) {
480         aio_enable_external(bdrv_get_aio_context(bs));
481     }
482 
483     if (recursive) {
484         assert(!ignore_bds_parents);
485         bs->recursive_quiesce_counter--;
486         QLIST_FOREACH(child, &bs->children, next) {
487             bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
488                                 drained_end_counter);
489         }
490     }
491 }
492 
493 void bdrv_drained_end(BlockDriverState *bs)
494 {
495     int drained_end_counter = 0;
496     bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter);
497     BDRV_POLL_WHILE(bs, atomic_read(&drained_end_counter) > 0);
498 }
499 
500 void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter)
501 {
502     bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter);
503 }
504 
505 void bdrv_subtree_drained_end(BlockDriverState *bs)
506 {
507     int drained_end_counter = 0;
508     bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter);
509     BDRV_POLL_WHILE(bs, atomic_read(&drained_end_counter) > 0);
510 }
511 
512 void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
513 {
514     int i;
515 
516     for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
517         bdrv_do_drained_begin(child->bs, true, child, false, true);
518     }
519 }
520 
521 void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
522 {
523     int drained_end_counter = 0;
524     int i;
525 
526     for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
527         bdrv_do_drained_end(child->bs, true, child, false,
528                             &drained_end_counter);
529     }
530 
531     BDRV_POLL_WHILE(child->bs, atomic_read(&drained_end_counter) > 0);
532 }
533 
534 /*
535  * Wait for pending requests to complete on a single BlockDriverState subtree,
536  * and suspend block driver's internal I/O until next request arrives.
537  *
538  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
539  * AioContext.
540  */
541 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
542 {
543     assert(qemu_in_coroutine());
544     bdrv_drained_begin(bs);
545     bdrv_drained_end(bs);
546 }
547 
548 void bdrv_drain(BlockDriverState *bs)
549 {
550     bdrv_drained_begin(bs);
551     bdrv_drained_end(bs);
552 }
553 
554 static void bdrv_drain_assert_idle(BlockDriverState *bs)
555 {
556     BdrvChild *child, *next;
557 
558     assert(atomic_read(&bs->in_flight) == 0);
559     QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
560         bdrv_drain_assert_idle(child->bs);
561     }
562 }
563 
564 unsigned int bdrv_drain_all_count = 0;
565 
566 static bool bdrv_drain_all_poll(void)
567 {
568     BlockDriverState *bs = NULL;
569     bool result = false;
570 
571     /* bdrv_drain_poll() can't make changes to the graph and we are holding the
572      * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
573     while ((bs = bdrv_next_all_states(bs))) {
574         AioContext *aio_context = bdrv_get_aio_context(bs);
575         aio_context_acquire(aio_context);
576         result |= bdrv_drain_poll(bs, false, NULL, true);
577         aio_context_release(aio_context);
578     }
579 
580     return result;
581 }
582 
583 /*
584  * Wait for pending requests to complete across all BlockDriverStates
585  *
586  * This function does not flush data to disk, use bdrv_flush_all() for that
587  * after calling this function.
588  *
589  * This pauses all block jobs and disables external clients. It must
590  * be paired with bdrv_drain_all_end().
591  *
592  * NOTE: no new block jobs or BlockDriverStates can be created between
593  * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
594  */
595 void bdrv_drain_all_begin(void)
596 {
597     BlockDriverState *bs = NULL;
598 
599     if (qemu_in_coroutine()) {
600         bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL);
601         return;
602     }
603 
604     /*
605      * bdrv queue is managed by record/replay,
606      * waiting for finishing the I/O requests may
607      * be infinite
608      */
609     if (replay_events_enabled()) {
610         return;
611     }
612 
613     /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
614      * loop AioContext, so make sure we're in the main context. */
615     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
616     assert(bdrv_drain_all_count < INT_MAX);
617     bdrv_drain_all_count++;
618 
619     /* Quiesce all nodes, without polling in-flight requests yet. The graph
620      * cannot change during this loop. */
621     while ((bs = bdrv_next_all_states(bs))) {
622         AioContext *aio_context = bdrv_get_aio_context(bs);
623 
624         aio_context_acquire(aio_context);
625         bdrv_do_drained_begin(bs, false, NULL, true, false);
626         aio_context_release(aio_context);
627     }
628 
629     /* Now poll the in-flight requests */
630     AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
631 
632     while ((bs = bdrv_next_all_states(bs))) {
633         bdrv_drain_assert_idle(bs);
634     }
635 }
636 
637 void bdrv_drain_all_end(void)
638 {
639     BlockDriverState *bs = NULL;
640     int drained_end_counter = 0;
641 
642     /*
643      * bdrv queue is managed by record/replay,
644      * waiting for finishing the I/O requests may
645      * be endless
646      */
647     if (replay_events_enabled()) {
648         return;
649     }
650 
651     while ((bs = bdrv_next_all_states(bs))) {
652         AioContext *aio_context = bdrv_get_aio_context(bs);
653 
654         aio_context_acquire(aio_context);
655         bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
656         aio_context_release(aio_context);
657     }
658 
659     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
660     AIO_WAIT_WHILE(NULL, atomic_read(&drained_end_counter) > 0);
661 
662     assert(bdrv_drain_all_count > 0);
663     bdrv_drain_all_count--;
664 }
665 
666 void bdrv_drain_all(void)
667 {
668     bdrv_drain_all_begin();
669     bdrv_drain_all_end();
670 }
671 
672 /**
673  * Remove an active request from the tracked requests list
674  *
675  * This function should be called when a tracked request is completing.
676  */
677 static void tracked_request_end(BdrvTrackedRequest *req)
678 {
679     if (req->serialising) {
680         atomic_dec(&req->bs->serialising_in_flight);
681     }
682 
683     qemu_co_mutex_lock(&req->bs->reqs_lock);
684     QLIST_REMOVE(req, list);
685     qemu_co_queue_restart_all(&req->wait_queue);
686     qemu_co_mutex_unlock(&req->bs->reqs_lock);
687 }
688 
689 /**
690  * Add an active request to the tracked requests list
691  */
692 static void tracked_request_begin(BdrvTrackedRequest *req,
693                                   BlockDriverState *bs,
694                                   int64_t offset,
695                                   uint64_t bytes,
696                                   enum BdrvTrackedRequestType type)
697 {
698     assert(bytes <= INT64_MAX && offset <= INT64_MAX - bytes);
699 
700     *req = (BdrvTrackedRequest){
701         .bs = bs,
702         .offset         = offset,
703         .bytes          = bytes,
704         .type           = type,
705         .co             = qemu_coroutine_self(),
706         .serialising    = false,
707         .overlap_offset = offset,
708         .overlap_bytes  = bytes,
709     };
710 
711     qemu_co_queue_init(&req->wait_queue);
712 
713     qemu_co_mutex_lock(&bs->reqs_lock);
714     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
715     qemu_co_mutex_unlock(&bs->reqs_lock);
716 }
717 
718 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
719                                      int64_t offset, uint64_t bytes)
720 {
721     /*        aaaa   bbbb */
722     if (offset >= req->overlap_offset + req->overlap_bytes) {
723         return false;
724     }
725     /* bbbb   aaaa        */
726     if (req->overlap_offset >= offset + bytes) {
727         return false;
728     }
729     return true;
730 }
731 
732 static bool coroutine_fn
733 bdrv_wait_serialising_requests_locked(BlockDriverState *bs,
734                                       BdrvTrackedRequest *self)
735 {
736     BdrvTrackedRequest *req;
737     bool retry;
738     bool waited = false;
739 
740     do {
741         retry = false;
742         QLIST_FOREACH(req, &bs->tracked_requests, list) {
743             if (req == self || (!req->serialising && !self->serialising)) {
744                 continue;
745             }
746             if (tracked_request_overlaps(req, self->overlap_offset,
747                                          self->overlap_bytes))
748             {
749                 /* Hitting this means there was a reentrant request, for
750                  * example, a block driver issuing nested requests.  This must
751                  * never happen since it means deadlock.
752                  */
753                 assert(qemu_coroutine_self() != req->co);
754 
755                 /* If the request is already (indirectly) waiting for us, or
756                  * will wait for us as soon as it wakes up, then just go on
757                  * (instead of producing a deadlock in the former case). */
758                 if (!req->waiting_for) {
759                     self->waiting_for = req;
760                     qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock);
761                     self->waiting_for = NULL;
762                     retry = true;
763                     waited = true;
764                     break;
765                 }
766             }
767         }
768     } while (retry);
769     return waited;
770 }
771 
772 bool bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
773 {
774     BlockDriverState *bs = req->bs;
775     int64_t overlap_offset = req->offset & ~(align - 1);
776     uint64_t overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
777                                - overlap_offset;
778     bool waited;
779 
780     qemu_co_mutex_lock(&bs->reqs_lock);
781     if (!req->serialising) {
782         atomic_inc(&req->bs->serialising_in_flight);
783         req->serialising = true;
784     }
785 
786     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
787     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
788     waited = bdrv_wait_serialising_requests_locked(bs, req);
789     qemu_co_mutex_unlock(&bs->reqs_lock);
790     return waited;
791 }
792 
793 /**
794  * Return the tracked request on @bs for the current coroutine, or
795  * NULL if there is none.
796  */
797 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
798 {
799     BdrvTrackedRequest *req;
800     Coroutine *self = qemu_coroutine_self();
801 
802     QLIST_FOREACH(req, &bs->tracked_requests, list) {
803         if (req->co == self) {
804             return req;
805         }
806     }
807 
808     return NULL;
809 }
810 
811 /**
812  * Round a region to cluster boundaries
813  */
814 void bdrv_round_to_clusters(BlockDriverState *bs,
815                             int64_t offset, int64_t bytes,
816                             int64_t *cluster_offset,
817                             int64_t *cluster_bytes)
818 {
819     BlockDriverInfo bdi;
820 
821     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
822         *cluster_offset = offset;
823         *cluster_bytes = bytes;
824     } else {
825         int64_t c = bdi.cluster_size;
826         *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
827         *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
828     }
829 }
830 
831 static int bdrv_get_cluster_size(BlockDriverState *bs)
832 {
833     BlockDriverInfo bdi;
834     int ret;
835 
836     ret = bdrv_get_info(bs, &bdi);
837     if (ret < 0 || bdi.cluster_size == 0) {
838         return bs->bl.request_alignment;
839     } else {
840         return bdi.cluster_size;
841     }
842 }
843 
844 void bdrv_inc_in_flight(BlockDriverState *bs)
845 {
846     atomic_inc(&bs->in_flight);
847 }
848 
849 void bdrv_wakeup(BlockDriverState *bs)
850 {
851     aio_wait_kick();
852 }
853 
854 void bdrv_dec_in_flight(BlockDriverState *bs)
855 {
856     atomic_dec(&bs->in_flight);
857     bdrv_wakeup(bs);
858 }
859 
860 static bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
861 {
862     BlockDriverState *bs = self->bs;
863     bool waited = false;
864 
865     if (!atomic_read(&bs->serialising_in_flight)) {
866         return false;
867     }
868 
869     qemu_co_mutex_lock(&bs->reqs_lock);
870     waited = bdrv_wait_serialising_requests_locked(bs, self);
871     qemu_co_mutex_unlock(&bs->reqs_lock);
872 
873     return waited;
874 }
875 
876 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
877                                    size_t size)
878 {
879     if (size > BDRV_REQUEST_MAX_BYTES) {
880         return -EIO;
881     }
882 
883     if (!bdrv_is_inserted(bs)) {
884         return -ENOMEDIUM;
885     }
886 
887     if (offset < 0) {
888         return -EIO;
889     }
890 
891     return 0;
892 }
893 
894 typedef struct RwCo {
895     BdrvChild *child;
896     int64_t offset;
897     QEMUIOVector *qiov;
898     bool is_write;
899     int ret;
900     BdrvRequestFlags flags;
901 } RwCo;
902 
903 static void coroutine_fn bdrv_rw_co_entry(void *opaque)
904 {
905     RwCo *rwco = opaque;
906 
907     if (!rwco->is_write) {
908         rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset,
909                                    rwco->qiov->size, rwco->qiov,
910                                    rwco->flags);
911     } else {
912         rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset,
913                                     rwco->qiov->size, rwco->qiov,
914                                     rwco->flags);
915     }
916     aio_wait_kick();
917 }
918 
919 /*
920  * Process a vectored synchronous request using coroutines
921  */
922 static int bdrv_prwv_co(BdrvChild *child, int64_t offset,
923                         QEMUIOVector *qiov, bool is_write,
924                         BdrvRequestFlags flags)
925 {
926     Coroutine *co;
927     RwCo rwco = {
928         .child = child,
929         .offset = offset,
930         .qiov = qiov,
931         .is_write = is_write,
932         .ret = NOT_DONE,
933         .flags = flags,
934     };
935 
936     if (qemu_in_coroutine()) {
937         /* Fast-path if already in coroutine context */
938         bdrv_rw_co_entry(&rwco);
939     } else {
940         co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco);
941         bdrv_coroutine_enter(child->bs, co);
942         BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE);
943     }
944     return rwco.ret;
945 }
946 
947 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
948                        int bytes, BdrvRequestFlags flags)
949 {
950     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, NULL, bytes);
951 
952     return bdrv_prwv_co(child, offset, &qiov, true,
953                         BDRV_REQ_ZERO_WRITE | flags);
954 }
955 
956 /*
957  * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
958  * The operation is sped up by checking the block status and only writing
959  * zeroes to the device if they currently do not return zeroes. Optional
960  * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
961  * BDRV_REQ_FUA).
962  *
963  * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
964  */
965 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
966 {
967     int ret;
968     int64_t target_size, bytes, offset = 0;
969     BlockDriverState *bs = child->bs;
970 
971     target_size = bdrv_getlength(bs);
972     if (target_size < 0) {
973         return target_size;
974     }
975 
976     for (;;) {
977         bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
978         if (bytes <= 0) {
979             return 0;
980         }
981         ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
982         if (ret < 0) {
983             return ret;
984         }
985         if (ret & BDRV_BLOCK_ZERO) {
986             offset += bytes;
987             continue;
988         }
989         ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
990         if (ret < 0) {
991             return ret;
992         }
993         offset += bytes;
994     }
995 }
996 
997 int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
998 {
999     int ret;
1000 
1001     ret = bdrv_prwv_co(child, offset, qiov, false, 0);
1002     if (ret < 0) {
1003         return ret;
1004     }
1005 
1006     return qiov->size;
1007 }
1008 
1009 /* See bdrv_pwrite() for the return codes */
1010 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes)
1011 {
1012     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1013 
1014     if (bytes < 0) {
1015         return -EINVAL;
1016     }
1017 
1018     return bdrv_preadv(child, offset, &qiov);
1019 }
1020 
1021 int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
1022 {
1023     int ret;
1024 
1025     ret = bdrv_prwv_co(child, offset, qiov, true, 0);
1026     if (ret < 0) {
1027         return ret;
1028     }
1029 
1030     return qiov->size;
1031 }
1032 
1033 /* Return no. of bytes on success or < 0 on error. Important errors are:
1034   -EIO         generic I/O error (may happen for all errors)
1035   -ENOMEDIUM   No media inserted.
1036   -EINVAL      Invalid offset or number of bytes
1037   -EACCES      Trying to write a read-only device
1038 */
1039 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes)
1040 {
1041     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1042 
1043     if (bytes < 0) {
1044         return -EINVAL;
1045     }
1046 
1047     return bdrv_pwritev(child, offset, &qiov);
1048 }
1049 
1050 /*
1051  * Writes to the file and ensures that no writes are reordered across this
1052  * request (acts as a barrier)
1053  *
1054  * Returns 0 on success, -errno in error cases.
1055  */
1056 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
1057                      const void *buf, int count)
1058 {
1059     int ret;
1060 
1061     ret = bdrv_pwrite(child, offset, buf, count);
1062     if (ret < 0) {
1063         return ret;
1064     }
1065 
1066     ret = bdrv_flush(child->bs);
1067     if (ret < 0) {
1068         return ret;
1069     }
1070 
1071     return 0;
1072 }
1073 
1074 typedef struct CoroutineIOCompletion {
1075     Coroutine *coroutine;
1076     int ret;
1077 } CoroutineIOCompletion;
1078 
1079 static void bdrv_co_io_em_complete(void *opaque, int ret)
1080 {
1081     CoroutineIOCompletion *co = opaque;
1082 
1083     co->ret = ret;
1084     aio_co_wake(co->coroutine);
1085 }
1086 
1087 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
1088                                            uint64_t offset, uint64_t bytes,
1089                                            QEMUIOVector *qiov,
1090                                            size_t qiov_offset, int flags)
1091 {
1092     BlockDriver *drv = bs->drv;
1093     int64_t sector_num;
1094     unsigned int nb_sectors;
1095     QEMUIOVector local_qiov;
1096     int ret;
1097 
1098     assert(!(flags & ~BDRV_REQ_MASK));
1099     assert(!(flags & BDRV_REQ_NO_FALLBACK));
1100 
1101     if (!drv) {
1102         return -ENOMEDIUM;
1103     }
1104 
1105     if (drv->bdrv_co_preadv_part) {
1106         return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
1107                                         flags);
1108     }
1109 
1110     if (qiov_offset > 0 || bytes != qiov->size) {
1111         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1112         qiov = &local_qiov;
1113     }
1114 
1115     if (drv->bdrv_co_preadv) {
1116         ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
1117         goto out;
1118     }
1119 
1120     if (drv->bdrv_aio_preadv) {
1121         BlockAIOCB *acb;
1122         CoroutineIOCompletion co = {
1123             .coroutine = qemu_coroutine_self(),
1124         };
1125 
1126         acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
1127                                    bdrv_co_io_em_complete, &co);
1128         if (acb == NULL) {
1129             ret = -EIO;
1130             goto out;
1131         } else {
1132             qemu_coroutine_yield();
1133             ret = co.ret;
1134             goto out;
1135         }
1136     }
1137 
1138     sector_num = offset >> BDRV_SECTOR_BITS;
1139     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1140 
1141     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1142     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1143     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1144     assert(drv->bdrv_co_readv);
1145 
1146     ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1147 
1148 out:
1149     if (qiov == &local_qiov) {
1150         qemu_iovec_destroy(&local_qiov);
1151     }
1152 
1153     return ret;
1154 }
1155 
1156 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
1157                                             uint64_t offset, uint64_t bytes,
1158                                             QEMUIOVector *qiov,
1159                                             size_t qiov_offset, int flags)
1160 {
1161     BlockDriver *drv = bs->drv;
1162     int64_t sector_num;
1163     unsigned int nb_sectors;
1164     QEMUIOVector local_qiov;
1165     int ret;
1166 
1167     assert(!(flags & ~BDRV_REQ_MASK));
1168     assert(!(flags & BDRV_REQ_NO_FALLBACK));
1169 
1170     if (!drv) {
1171         return -ENOMEDIUM;
1172     }
1173 
1174     if (drv->bdrv_co_pwritev_part) {
1175         ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1176                                         flags & bs->supported_write_flags);
1177         flags &= ~bs->supported_write_flags;
1178         goto emulate_flags;
1179     }
1180 
1181     if (qiov_offset > 0 || bytes != qiov->size) {
1182         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1183         qiov = &local_qiov;
1184     }
1185 
1186     if (drv->bdrv_co_pwritev) {
1187         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
1188                                    flags & bs->supported_write_flags);
1189         flags &= ~bs->supported_write_flags;
1190         goto emulate_flags;
1191     }
1192 
1193     if (drv->bdrv_aio_pwritev) {
1194         BlockAIOCB *acb;
1195         CoroutineIOCompletion co = {
1196             .coroutine = qemu_coroutine_self(),
1197         };
1198 
1199         acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov,
1200                                     flags & bs->supported_write_flags,
1201                                     bdrv_co_io_em_complete, &co);
1202         flags &= ~bs->supported_write_flags;
1203         if (acb == NULL) {
1204             ret = -EIO;
1205         } else {
1206             qemu_coroutine_yield();
1207             ret = co.ret;
1208         }
1209         goto emulate_flags;
1210     }
1211 
1212     sector_num = offset >> BDRV_SECTOR_BITS;
1213     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1214 
1215     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1216     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1217     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1218 
1219     assert(drv->bdrv_co_writev);
1220     ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov,
1221                               flags & bs->supported_write_flags);
1222     flags &= ~bs->supported_write_flags;
1223 
1224 emulate_flags:
1225     if (ret == 0 && (flags & BDRV_REQ_FUA)) {
1226         ret = bdrv_co_flush(bs);
1227     }
1228 
1229     if (qiov == &local_qiov) {
1230         qemu_iovec_destroy(&local_qiov);
1231     }
1232 
1233     return ret;
1234 }
1235 
1236 static int coroutine_fn
1237 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset,
1238                                uint64_t bytes, QEMUIOVector *qiov,
1239                                size_t qiov_offset)
1240 {
1241     BlockDriver *drv = bs->drv;
1242     QEMUIOVector local_qiov;
1243     int ret;
1244 
1245     if (!drv) {
1246         return -ENOMEDIUM;
1247     }
1248 
1249     if (!block_driver_can_compress(drv)) {
1250         return -ENOTSUP;
1251     }
1252 
1253     if (drv->bdrv_co_pwritev_compressed_part) {
1254         return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1255                                                     qiov, qiov_offset);
1256     }
1257 
1258     if (qiov_offset == 0) {
1259         return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
1260     }
1261 
1262     qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1263     ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1264     qemu_iovec_destroy(&local_qiov);
1265 
1266     return ret;
1267 }
1268 
1269 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
1270         int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1271         size_t qiov_offset, int flags)
1272 {
1273     BlockDriverState *bs = child->bs;
1274 
1275     /* Perform I/O through a temporary buffer so that users who scribble over
1276      * their read buffer while the operation is in progress do not end up
1277      * modifying the image file.  This is critical for zero-copy guest I/O
1278      * where anything might happen inside guest memory.
1279      */
1280     void *bounce_buffer = NULL;
1281 
1282     BlockDriver *drv = bs->drv;
1283     int64_t cluster_offset;
1284     int64_t cluster_bytes;
1285     size_t skip_bytes;
1286     int ret;
1287     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1288                                     BDRV_REQUEST_MAX_BYTES);
1289     unsigned int progress = 0;
1290     bool skip_write;
1291 
1292     if (!drv) {
1293         return -ENOMEDIUM;
1294     }
1295 
1296     /*
1297      * Do not write anything when the BDS is inactive.  That is not
1298      * allowed, and it would not help.
1299      */
1300     skip_write = (bs->open_flags & BDRV_O_INACTIVE);
1301 
1302     /* FIXME We cannot require callers to have write permissions when all they
1303      * are doing is a read request. If we did things right, write permissions
1304      * would be obtained anyway, but internally by the copy-on-read code. As
1305      * long as it is implemented here rather than in a separate filter driver,
1306      * the copy-on-read code doesn't have its own BdrvChild, however, for which
1307      * it could request permissions. Therefore we have to bypass the permission
1308      * system for the moment. */
1309     // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1310 
1311     /* Cover entire cluster so no additional backing file I/O is required when
1312      * allocating cluster in the image file.  Note that this value may exceed
1313      * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1314      * is one reason we loop rather than doing it all at once.
1315      */
1316     bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1317     skip_bytes = offset - cluster_offset;
1318 
1319     trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1320                                    cluster_offset, cluster_bytes);
1321 
1322     while (cluster_bytes) {
1323         int64_t pnum;
1324 
1325         if (skip_write) {
1326             ret = 1; /* "already allocated", so nothing will be copied */
1327             pnum = MIN(cluster_bytes, max_transfer);
1328         } else {
1329             ret = bdrv_is_allocated(bs, cluster_offset,
1330                                     MIN(cluster_bytes, max_transfer), &pnum);
1331             if (ret < 0) {
1332                 /*
1333                  * Safe to treat errors in querying allocation as if
1334                  * unallocated; we'll probably fail again soon on the
1335                  * read, but at least that will set a decent errno.
1336                  */
1337                 pnum = MIN(cluster_bytes, max_transfer);
1338             }
1339 
1340             /* Stop at EOF if the image ends in the middle of the cluster */
1341             if (ret == 0 && pnum == 0) {
1342                 assert(progress >= bytes);
1343                 break;
1344             }
1345 
1346             assert(skip_bytes < pnum);
1347         }
1348 
1349         if (ret <= 0) {
1350             QEMUIOVector local_qiov;
1351 
1352             /* Must copy-on-read; use the bounce buffer */
1353             pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
1354             if (!bounce_buffer) {
1355                 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
1356                 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
1357                 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
1358 
1359                 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
1360                 if (!bounce_buffer) {
1361                     ret = -ENOMEM;
1362                     goto err;
1363                 }
1364             }
1365             qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1366 
1367             ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1368                                      &local_qiov, 0, 0);
1369             if (ret < 0) {
1370                 goto err;
1371             }
1372 
1373             bdrv_debug_event(bs, BLKDBG_COR_WRITE);
1374             if (drv->bdrv_co_pwrite_zeroes &&
1375                 buffer_is_zero(bounce_buffer, pnum)) {
1376                 /* FIXME: Should we (perhaps conditionally) be setting
1377                  * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1378                  * that still correctly reads as zero? */
1379                 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum,
1380                                                BDRV_REQ_WRITE_UNCHANGED);
1381             } else {
1382                 /* This does not change the data on the disk, it is not
1383                  * necessary to flush even in cache=writethrough mode.
1384                  */
1385                 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1386                                           &local_qiov, 0,
1387                                           BDRV_REQ_WRITE_UNCHANGED);
1388             }
1389 
1390             if (ret < 0) {
1391                 /* It might be okay to ignore write errors for guest
1392                  * requests.  If this is a deliberate copy-on-read
1393                  * then we don't want to ignore the error.  Simply
1394                  * report it in all cases.
1395                  */
1396                 goto err;
1397             }
1398 
1399             if (!(flags & BDRV_REQ_PREFETCH)) {
1400                 qemu_iovec_from_buf(qiov, qiov_offset + progress,
1401                                     bounce_buffer + skip_bytes,
1402                                     pnum - skip_bytes);
1403             }
1404         } else if (!(flags & BDRV_REQ_PREFETCH)) {
1405             /* Read directly into the destination */
1406             ret = bdrv_driver_preadv(bs, offset + progress,
1407                                      MIN(pnum - skip_bytes, bytes - progress),
1408                                      qiov, qiov_offset + progress, 0);
1409             if (ret < 0) {
1410                 goto err;
1411             }
1412         }
1413 
1414         cluster_offset += pnum;
1415         cluster_bytes -= pnum;
1416         progress += pnum - skip_bytes;
1417         skip_bytes = 0;
1418     }
1419     ret = 0;
1420 
1421 err:
1422     qemu_vfree(bounce_buffer);
1423     return ret;
1424 }
1425 
1426 /*
1427  * Forwards an already correctly aligned request to the BlockDriver. This
1428  * handles copy on read, zeroing after EOF, and fragmentation of large
1429  * reads; any other features must be implemented by the caller.
1430  */
1431 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
1432     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1433     int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1434 {
1435     BlockDriverState *bs = child->bs;
1436     int64_t total_bytes, max_bytes;
1437     int ret = 0;
1438     uint64_t bytes_remaining = bytes;
1439     int max_transfer;
1440 
1441     assert(is_power_of_2(align));
1442     assert((offset & (align - 1)) == 0);
1443     assert((bytes & (align - 1)) == 0);
1444     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1445     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1446                                    align);
1447 
1448     /* TODO: We would need a per-BDS .supported_read_flags and
1449      * potential fallback support, if we ever implement any read flags
1450      * to pass through to drivers.  For now, there aren't any
1451      * passthrough flags.  */
1452     assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH)));
1453 
1454     /* Handle Copy on Read and associated serialisation */
1455     if (flags & BDRV_REQ_COPY_ON_READ) {
1456         /* If we touch the same cluster it counts as an overlap.  This
1457          * guarantees that allocating writes will be serialized and not race
1458          * with each other for the same cluster.  For example, in copy-on-read
1459          * it ensures that the CoR read and write operations are atomic and
1460          * guest writes cannot interleave between them. */
1461         bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
1462     } else {
1463         bdrv_wait_serialising_requests(req);
1464     }
1465 
1466     if (flags & BDRV_REQ_COPY_ON_READ) {
1467         int64_t pnum;
1468 
1469         ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
1470         if (ret < 0) {
1471             goto out;
1472         }
1473 
1474         if (!ret || pnum != bytes) {
1475             ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
1476                                            qiov, qiov_offset, flags);
1477             goto out;
1478         } else if (flags & BDRV_REQ_PREFETCH) {
1479             goto out;
1480         }
1481     }
1482 
1483     /* Forward the request to the BlockDriver, possibly fragmenting it */
1484     total_bytes = bdrv_getlength(bs);
1485     if (total_bytes < 0) {
1486         ret = total_bytes;
1487         goto out;
1488     }
1489 
1490     max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1491     if (bytes <= max_bytes && bytes <= max_transfer) {
1492         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0);
1493         goto out;
1494     }
1495 
1496     while (bytes_remaining) {
1497         int num;
1498 
1499         if (max_bytes) {
1500             num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1501             assert(num);
1502 
1503             ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1504                                      num, qiov, bytes - bytes_remaining, 0);
1505             max_bytes -= num;
1506         } else {
1507             num = bytes_remaining;
1508             ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0,
1509                                     bytes_remaining);
1510         }
1511         if (ret < 0) {
1512             goto out;
1513         }
1514         bytes_remaining -= num;
1515     }
1516 
1517 out:
1518     return ret < 0 ? ret : 0;
1519 }
1520 
1521 /*
1522  * Request padding
1523  *
1524  *  |<---- align ----->|                     |<----- align ---->|
1525  *  |<- head ->|<------------- bytes ------------->|<-- tail -->|
1526  *  |          |       |                     |     |            |
1527  * -*----------$-------*-------- ... --------*-----$------------*---
1528  *  |          |       |                     |     |            |
1529  *  |          offset  |                     |     end          |
1530  *  ALIGN_DOWN(offset) ALIGN_UP(offset)      ALIGN_DOWN(end)   ALIGN_UP(end)
1531  *  [buf   ... )                             [tail_buf          )
1532  *
1533  * @buf is an aligned allocation needed to store @head and @tail paddings. @head
1534  * is placed at the beginning of @buf and @tail at the @end.
1535  *
1536  * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
1537  * around tail, if tail exists.
1538  *
1539  * @merge_reads is true for small requests,
1540  * if @buf_len == @head + bytes + @tail. In this case it is possible that both
1541  * head and tail exist but @buf_len == align and @tail_buf == @buf.
1542  */
1543 typedef struct BdrvRequestPadding {
1544     uint8_t *buf;
1545     size_t buf_len;
1546     uint8_t *tail_buf;
1547     size_t head;
1548     size_t tail;
1549     bool merge_reads;
1550     QEMUIOVector local_qiov;
1551 } BdrvRequestPadding;
1552 
1553 static bool bdrv_init_padding(BlockDriverState *bs,
1554                               int64_t offset, int64_t bytes,
1555                               BdrvRequestPadding *pad)
1556 {
1557     uint64_t align = bs->bl.request_alignment;
1558     size_t sum;
1559 
1560     memset(pad, 0, sizeof(*pad));
1561 
1562     pad->head = offset & (align - 1);
1563     pad->tail = ((offset + bytes) & (align - 1));
1564     if (pad->tail) {
1565         pad->tail = align - pad->tail;
1566     }
1567 
1568     if (!pad->head && !pad->tail) {
1569         return false;
1570     }
1571 
1572     assert(bytes); /* Nothing good in aligning zero-length requests */
1573 
1574     sum = pad->head + bytes + pad->tail;
1575     pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
1576     pad->buf = qemu_blockalign(bs, pad->buf_len);
1577     pad->merge_reads = sum == pad->buf_len;
1578     if (pad->tail) {
1579         pad->tail_buf = pad->buf + pad->buf_len - align;
1580     }
1581 
1582     return true;
1583 }
1584 
1585 static int bdrv_padding_rmw_read(BdrvChild *child,
1586                                  BdrvTrackedRequest *req,
1587                                  BdrvRequestPadding *pad,
1588                                  bool zero_middle)
1589 {
1590     QEMUIOVector local_qiov;
1591     BlockDriverState *bs = child->bs;
1592     uint64_t align = bs->bl.request_alignment;
1593     int ret;
1594 
1595     assert(req->serialising && pad->buf);
1596 
1597     if (pad->head || pad->merge_reads) {
1598         uint64_t bytes = pad->merge_reads ? pad->buf_len : align;
1599 
1600         qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
1601 
1602         if (pad->head) {
1603             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1604         }
1605         if (pad->merge_reads && pad->tail) {
1606             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1607         }
1608         ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
1609                                   align, &local_qiov, 0, 0);
1610         if (ret < 0) {
1611             return ret;
1612         }
1613         if (pad->head) {
1614             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1615         }
1616         if (pad->merge_reads && pad->tail) {
1617             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1618         }
1619 
1620         if (pad->merge_reads) {
1621             goto zero_mem;
1622         }
1623     }
1624 
1625     if (pad->tail) {
1626         qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
1627 
1628         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1629         ret = bdrv_aligned_preadv(
1630                 child, req,
1631                 req->overlap_offset + req->overlap_bytes - align,
1632                 align, align, &local_qiov, 0, 0);
1633         if (ret < 0) {
1634             return ret;
1635         }
1636         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1637     }
1638 
1639 zero_mem:
1640     if (zero_middle) {
1641         memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
1642     }
1643 
1644     return 0;
1645 }
1646 
1647 static void bdrv_padding_destroy(BdrvRequestPadding *pad)
1648 {
1649     if (pad->buf) {
1650         qemu_vfree(pad->buf);
1651         qemu_iovec_destroy(&pad->local_qiov);
1652     }
1653 }
1654 
1655 /*
1656  * bdrv_pad_request
1657  *
1658  * Exchange request parameters with padded request if needed. Don't include RMW
1659  * read of padding, bdrv_padding_rmw_read() should be called separately if
1660  * needed.
1661  *
1662  * All parameters except @bs are in-out: they represent original request at
1663  * function call and padded (if padding needed) at function finish.
1664  *
1665  * Function always succeeds.
1666  */
1667 static bool bdrv_pad_request(BlockDriverState *bs,
1668                              QEMUIOVector **qiov, size_t *qiov_offset,
1669                              int64_t *offset, unsigned int *bytes,
1670                              BdrvRequestPadding *pad)
1671 {
1672     if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
1673         return false;
1674     }
1675 
1676     qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
1677                              *qiov, *qiov_offset, *bytes,
1678                              pad->buf + pad->buf_len - pad->tail, pad->tail);
1679     *bytes += pad->head + pad->tail;
1680     *offset -= pad->head;
1681     *qiov = &pad->local_qiov;
1682     *qiov_offset = 0;
1683 
1684     return true;
1685 }
1686 
1687 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1688     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1689     BdrvRequestFlags flags)
1690 {
1691     return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
1692 }
1693 
1694 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
1695     int64_t offset, unsigned int bytes,
1696     QEMUIOVector *qiov, size_t qiov_offset,
1697     BdrvRequestFlags flags)
1698 {
1699     BlockDriverState *bs = child->bs;
1700     BdrvTrackedRequest req;
1701     BdrvRequestPadding pad;
1702     int ret;
1703 
1704     trace_bdrv_co_preadv(bs, offset, bytes, flags);
1705 
1706     ret = bdrv_check_byte_request(bs, offset, bytes);
1707     if (ret < 0) {
1708         return ret;
1709     }
1710 
1711     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1712         /*
1713          * Aligning zero request is nonsense. Even if driver has special meaning
1714          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1715          * it to driver due to request_alignment.
1716          *
1717          * Still, no reason to return an error if someone do unaligned
1718          * zero-length read occasionally.
1719          */
1720         return 0;
1721     }
1722 
1723     bdrv_inc_in_flight(bs);
1724 
1725     /* Don't do copy-on-read if we read data before write operation */
1726     if (atomic_read(&bs->copy_on_read)) {
1727         flags |= BDRV_REQ_COPY_ON_READ;
1728     }
1729 
1730     bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad);
1731 
1732     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1733     ret = bdrv_aligned_preadv(child, &req, offset, bytes,
1734                               bs->bl.request_alignment,
1735                               qiov, qiov_offset, flags);
1736     tracked_request_end(&req);
1737     bdrv_dec_in_flight(bs);
1738 
1739     bdrv_padding_destroy(&pad);
1740 
1741     return ret;
1742 }
1743 
1744 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1745     int64_t offset, int bytes, BdrvRequestFlags flags)
1746 {
1747     BlockDriver *drv = bs->drv;
1748     QEMUIOVector qiov;
1749     void *buf = NULL;
1750     int ret = 0;
1751     bool need_flush = false;
1752     int head = 0;
1753     int tail = 0;
1754 
1755     int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX);
1756     int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1757                         bs->bl.request_alignment);
1758     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1759 
1760     if (!drv) {
1761         return -ENOMEDIUM;
1762     }
1763 
1764     if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1765         return -ENOTSUP;
1766     }
1767 
1768     assert(alignment % bs->bl.request_alignment == 0);
1769     head = offset % alignment;
1770     tail = (offset + bytes) % alignment;
1771     max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1772     assert(max_write_zeroes >= bs->bl.request_alignment);
1773 
1774     while (bytes > 0 && !ret) {
1775         int num = bytes;
1776 
1777         /* Align request.  Block drivers can expect the "bulk" of the request
1778          * to be aligned, and that unaligned requests do not cross cluster
1779          * boundaries.
1780          */
1781         if (head) {
1782             /* Make a small request up to the first aligned sector. For
1783              * convenience, limit this request to max_transfer even if
1784              * we don't need to fall back to writes.  */
1785             num = MIN(MIN(bytes, max_transfer), alignment - head);
1786             head = (head + num) % alignment;
1787             assert(num < max_write_zeroes);
1788         } else if (tail && num > alignment) {
1789             /* Shorten the request to the last aligned sector.  */
1790             num -= tail;
1791         }
1792 
1793         /* limit request size */
1794         if (num > max_write_zeroes) {
1795             num = max_write_zeroes;
1796         }
1797 
1798         ret = -ENOTSUP;
1799         /* First try the efficient write zeroes operation */
1800         if (drv->bdrv_co_pwrite_zeroes) {
1801             ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1802                                              flags & bs->supported_zero_flags);
1803             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1804                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1805                 need_flush = true;
1806             }
1807         } else {
1808             assert(!bs->supported_zero_flags);
1809         }
1810 
1811         if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
1812             /* Fall back to bounce buffer if write zeroes is unsupported */
1813             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1814 
1815             if ((flags & BDRV_REQ_FUA) &&
1816                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1817                 /* No need for bdrv_driver_pwrite() to do a fallback
1818                  * flush on each chunk; use just one at the end */
1819                 write_flags &= ~BDRV_REQ_FUA;
1820                 need_flush = true;
1821             }
1822             num = MIN(num, max_transfer);
1823             if (buf == NULL) {
1824                 buf = qemu_try_blockalign0(bs, num);
1825                 if (buf == NULL) {
1826                     ret = -ENOMEM;
1827                     goto fail;
1828                 }
1829             }
1830             qemu_iovec_init_buf(&qiov, buf, num);
1831 
1832             ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
1833 
1834             /* Keep bounce buffer around if it is big enough for all
1835              * all future requests.
1836              */
1837             if (num < max_transfer) {
1838                 qemu_vfree(buf);
1839                 buf = NULL;
1840             }
1841         }
1842 
1843         offset += num;
1844         bytes -= num;
1845     }
1846 
1847 fail:
1848     if (ret == 0 && need_flush) {
1849         ret = bdrv_co_flush(bs);
1850     }
1851     qemu_vfree(buf);
1852     return ret;
1853 }
1854 
1855 static inline int coroutine_fn
1856 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, uint64_t bytes,
1857                           BdrvTrackedRequest *req, int flags)
1858 {
1859     BlockDriverState *bs = child->bs;
1860     bool waited;
1861     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1862 
1863     if (bs->read_only) {
1864         return -EPERM;
1865     }
1866 
1867     assert(!(bs->open_flags & BDRV_O_INACTIVE));
1868     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1869     assert(!(flags & ~BDRV_REQ_MASK));
1870 
1871     if (flags & BDRV_REQ_SERIALISING) {
1872         waited = bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
1873         /*
1874          * For a misaligned request we should have already waited earlier,
1875          * because we come after bdrv_padding_rmw_read which must be called
1876          * with the request already marked as serialising.
1877          */
1878         assert(!waited ||
1879                (req->offset == req->overlap_offset &&
1880                 req->bytes == req->overlap_bytes));
1881     } else {
1882         bdrv_wait_serialising_requests(req);
1883     }
1884 
1885     assert(req->overlap_offset <= offset);
1886     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1887     assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE);
1888 
1889     switch (req->type) {
1890     case BDRV_TRACKED_WRITE:
1891     case BDRV_TRACKED_DISCARD:
1892         if (flags & BDRV_REQ_WRITE_UNCHANGED) {
1893             assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1894         } else {
1895             assert(child->perm & BLK_PERM_WRITE);
1896         }
1897         return notifier_with_return_list_notify(&bs->before_write_notifiers,
1898                                                 req);
1899     case BDRV_TRACKED_TRUNCATE:
1900         assert(child->perm & BLK_PERM_RESIZE);
1901         return 0;
1902     default:
1903         abort();
1904     }
1905 }
1906 
1907 static inline void coroutine_fn
1908 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, uint64_t bytes,
1909                          BdrvTrackedRequest *req, int ret)
1910 {
1911     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1912     BlockDriverState *bs = child->bs;
1913 
1914     atomic_inc(&bs->write_gen);
1915 
1916     /*
1917      * Discard cannot extend the image, but in error handling cases, such as
1918      * when reverting a qcow2 cluster allocation, the discarded range can pass
1919      * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
1920      * here. Instead, just skip it, since semantically a discard request
1921      * beyond EOF cannot expand the image anyway.
1922      */
1923     if (ret == 0 &&
1924         (req->type == BDRV_TRACKED_TRUNCATE ||
1925          end_sector > bs->total_sectors) &&
1926         req->type != BDRV_TRACKED_DISCARD) {
1927         bs->total_sectors = end_sector;
1928         bdrv_parent_cb_resize(bs);
1929         bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
1930     }
1931     if (req->bytes) {
1932         switch (req->type) {
1933         case BDRV_TRACKED_WRITE:
1934             stat64_max(&bs->wr_highest_offset, offset + bytes);
1935             /* fall through, to set dirty bits */
1936         case BDRV_TRACKED_DISCARD:
1937             bdrv_set_dirty(bs, offset, bytes);
1938             break;
1939         default:
1940             break;
1941         }
1942     }
1943 }
1944 
1945 /*
1946  * Forwards an already correctly aligned write request to the BlockDriver,
1947  * after possibly fragmenting it.
1948  */
1949 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
1950     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1951     int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1952 {
1953     BlockDriverState *bs = child->bs;
1954     BlockDriver *drv = bs->drv;
1955     int ret;
1956 
1957     uint64_t bytes_remaining = bytes;
1958     int max_transfer;
1959 
1960     if (!drv) {
1961         return -ENOMEDIUM;
1962     }
1963 
1964     if (bdrv_has_readonly_bitmaps(bs)) {
1965         return -EPERM;
1966     }
1967 
1968     assert(is_power_of_2(align));
1969     assert((offset & (align - 1)) == 0);
1970     assert((bytes & (align - 1)) == 0);
1971     assert(!qiov || qiov_offset + bytes <= qiov->size);
1972     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1973                                    align);
1974 
1975     ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
1976 
1977     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1978         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
1979         qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
1980         flags |= BDRV_REQ_ZERO_WRITE;
1981         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1982             flags |= BDRV_REQ_MAY_UNMAP;
1983         }
1984     }
1985 
1986     if (ret < 0) {
1987         /* Do nothing, write notifier decided to fail this request */
1988     } else if (flags & BDRV_REQ_ZERO_WRITE) {
1989         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1990         ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
1991     } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
1992         ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
1993                                              qiov, qiov_offset);
1994     } else if (bytes <= max_transfer) {
1995         bdrv_debug_event(bs, BLKDBG_PWRITEV);
1996         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
1997     } else {
1998         bdrv_debug_event(bs, BLKDBG_PWRITEV);
1999         while (bytes_remaining) {
2000             int num = MIN(bytes_remaining, max_transfer);
2001             int local_flags = flags;
2002 
2003             assert(num);
2004             if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
2005                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
2006                 /* If FUA is going to be emulated by flush, we only
2007                  * need to flush on the last iteration */
2008                 local_flags &= ~BDRV_REQ_FUA;
2009             }
2010 
2011             ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
2012                                       num, qiov, bytes - bytes_remaining,
2013                                       local_flags);
2014             if (ret < 0) {
2015                 break;
2016             }
2017             bytes_remaining -= num;
2018         }
2019     }
2020     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
2021 
2022     if (ret >= 0) {
2023         ret = 0;
2024     }
2025     bdrv_co_write_req_finish(child, offset, bytes, req, ret);
2026 
2027     return ret;
2028 }
2029 
2030 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
2031                                                 int64_t offset,
2032                                                 unsigned int bytes,
2033                                                 BdrvRequestFlags flags,
2034                                                 BdrvTrackedRequest *req)
2035 {
2036     BlockDriverState *bs = child->bs;
2037     QEMUIOVector local_qiov;
2038     uint64_t align = bs->bl.request_alignment;
2039     int ret = 0;
2040     bool padding;
2041     BdrvRequestPadding pad;
2042 
2043     padding = bdrv_init_padding(bs, offset, bytes, &pad);
2044     if (padding) {
2045         bdrv_mark_request_serialising(req, align);
2046 
2047         bdrv_padding_rmw_read(child, req, &pad, true);
2048 
2049         if (pad.head || pad.merge_reads) {
2050             int64_t aligned_offset = offset & ~(align - 1);
2051             int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
2052 
2053             qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
2054             ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
2055                                        align, &local_qiov, 0,
2056                                        flags & ~BDRV_REQ_ZERO_WRITE);
2057             if (ret < 0 || pad.merge_reads) {
2058                 /* Error or all work is done */
2059                 goto out;
2060             }
2061             offset += write_bytes - pad.head;
2062             bytes -= write_bytes - pad.head;
2063         }
2064     }
2065 
2066     assert(!bytes || (offset & (align - 1)) == 0);
2067     if (bytes >= align) {
2068         /* Write the aligned part in the middle. */
2069         uint64_t aligned_bytes = bytes & ~(align - 1);
2070         ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
2071                                    NULL, 0, flags);
2072         if (ret < 0) {
2073             goto out;
2074         }
2075         bytes -= aligned_bytes;
2076         offset += aligned_bytes;
2077     }
2078 
2079     assert(!bytes || (offset & (align - 1)) == 0);
2080     if (bytes) {
2081         assert(align == pad.tail + bytes);
2082 
2083         qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
2084         ret = bdrv_aligned_pwritev(child, req, offset, align, align,
2085                                    &local_qiov, 0,
2086                                    flags & ~BDRV_REQ_ZERO_WRITE);
2087     }
2088 
2089 out:
2090     bdrv_padding_destroy(&pad);
2091 
2092     return ret;
2093 }
2094 
2095 /*
2096  * Handle a write request in coroutine context
2097  */
2098 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
2099     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
2100     BdrvRequestFlags flags)
2101 {
2102     return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
2103 }
2104 
2105 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
2106     int64_t offset, unsigned int bytes, QEMUIOVector *qiov, size_t qiov_offset,
2107     BdrvRequestFlags flags)
2108 {
2109     BlockDriverState *bs = child->bs;
2110     BdrvTrackedRequest req;
2111     uint64_t align = bs->bl.request_alignment;
2112     BdrvRequestPadding pad;
2113     int ret;
2114 
2115     trace_bdrv_co_pwritev(child->bs, offset, bytes, flags);
2116 
2117     if (!bs->drv) {
2118         return -ENOMEDIUM;
2119     }
2120 
2121     ret = bdrv_check_byte_request(bs, offset, bytes);
2122     if (ret < 0) {
2123         return ret;
2124     }
2125 
2126     /* If the request is misaligned then we can't make it efficient */
2127     if ((flags & BDRV_REQ_NO_FALLBACK) &&
2128         !QEMU_IS_ALIGNED(offset | bytes, align))
2129     {
2130         return -ENOTSUP;
2131     }
2132 
2133     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2134         /*
2135          * Aligning zero request is nonsense. Even if driver has special meaning
2136          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2137          * it to driver due to request_alignment.
2138          *
2139          * Still, no reason to return an error if someone do unaligned
2140          * zero-length write occasionally.
2141          */
2142         return 0;
2143     }
2144 
2145     bdrv_inc_in_flight(bs);
2146     /*
2147      * Align write if necessary by performing a read-modify-write cycle.
2148      * Pad qiov with the read parts and be sure to have a tracked request not
2149      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
2150      */
2151     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
2152 
2153     if (flags & BDRV_REQ_ZERO_WRITE) {
2154         ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
2155         goto out;
2156     }
2157 
2158     if (bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad)) {
2159         bdrv_mark_request_serialising(&req, align);
2160         bdrv_padding_rmw_read(child, &req, &pad, false);
2161     }
2162 
2163     ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
2164                                qiov, qiov_offset, flags);
2165 
2166     bdrv_padding_destroy(&pad);
2167 
2168 out:
2169     tracked_request_end(&req);
2170     bdrv_dec_in_flight(bs);
2171 
2172     return ret;
2173 }
2174 
2175 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2176                                        int bytes, BdrvRequestFlags flags)
2177 {
2178     trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
2179 
2180     if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
2181         flags &= ~BDRV_REQ_MAY_UNMAP;
2182     }
2183 
2184     return bdrv_co_pwritev(child, offset, bytes, NULL,
2185                            BDRV_REQ_ZERO_WRITE | flags);
2186 }
2187 
2188 /*
2189  * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
2190  */
2191 int bdrv_flush_all(void)
2192 {
2193     BdrvNextIterator it;
2194     BlockDriverState *bs = NULL;
2195     int result = 0;
2196 
2197     /*
2198      * bdrv queue is managed by record/replay,
2199      * creating new flush request for stopping
2200      * the VM may break the determinism
2201      */
2202     if (replay_events_enabled()) {
2203         return result;
2204     }
2205 
2206     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
2207         AioContext *aio_context = bdrv_get_aio_context(bs);
2208         int ret;
2209 
2210         aio_context_acquire(aio_context);
2211         ret = bdrv_flush(bs);
2212         if (ret < 0 && !result) {
2213             result = ret;
2214         }
2215         aio_context_release(aio_context);
2216     }
2217 
2218     return result;
2219 }
2220 
2221 
2222 typedef struct BdrvCoBlockStatusData {
2223     BlockDriverState *bs;
2224     BlockDriverState *base;
2225     bool want_zero;
2226     int64_t offset;
2227     int64_t bytes;
2228     int64_t *pnum;
2229     int64_t *map;
2230     BlockDriverState **file;
2231     int ret;
2232     bool done;
2233 } BdrvCoBlockStatusData;
2234 
2235 int coroutine_fn bdrv_co_block_status_from_file(BlockDriverState *bs,
2236                                                 bool want_zero,
2237                                                 int64_t offset,
2238                                                 int64_t bytes,
2239                                                 int64_t *pnum,
2240                                                 int64_t *map,
2241                                                 BlockDriverState **file)
2242 {
2243     assert(bs->file && bs->file->bs);
2244     *pnum = bytes;
2245     *map = offset;
2246     *file = bs->file->bs;
2247     return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2248 }
2249 
2250 int coroutine_fn bdrv_co_block_status_from_backing(BlockDriverState *bs,
2251                                                    bool want_zero,
2252                                                    int64_t offset,
2253                                                    int64_t bytes,
2254                                                    int64_t *pnum,
2255                                                    int64_t *map,
2256                                                    BlockDriverState **file)
2257 {
2258     assert(bs->backing && bs->backing->bs);
2259     *pnum = bytes;
2260     *map = offset;
2261     *file = bs->backing->bs;
2262     return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2263 }
2264 
2265 /*
2266  * Returns the allocation status of the specified sectors.
2267  * Drivers not implementing the functionality are assumed to not support
2268  * backing files, hence all their sectors are reported as allocated.
2269  *
2270  * If 'want_zero' is true, the caller is querying for mapping
2271  * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
2272  * _ZERO where possible; otherwise, the result favors larger 'pnum',
2273  * with a focus on accurate BDRV_BLOCK_ALLOCATED.
2274  *
2275  * If 'offset' is beyond the end of the disk image the return value is
2276  * BDRV_BLOCK_EOF and 'pnum' is set to 0.
2277  *
2278  * 'bytes' is the max value 'pnum' should be set to.  If bytes goes
2279  * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2280  * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
2281  *
2282  * 'pnum' is set to the number of bytes (including and immediately
2283  * following the specified offset) that are easily known to be in the
2284  * same allocated/unallocated state.  Note that a second call starting
2285  * at the original offset plus returned pnum may have the same status.
2286  * The returned value is non-zero on success except at end-of-file.
2287  *
2288  * Returns negative errno on failure.  Otherwise, if the
2289  * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
2290  * set to the host mapping and BDS corresponding to the guest offset.
2291  */
2292 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
2293                                              bool want_zero,
2294                                              int64_t offset, int64_t bytes,
2295                                              int64_t *pnum, int64_t *map,
2296                                              BlockDriverState **file)
2297 {
2298     int64_t total_size;
2299     int64_t n; /* bytes */
2300     int ret;
2301     int64_t local_map = 0;
2302     BlockDriverState *local_file = NULL;
2303     int64_t aligned_offset, aligned_bytes;
2304     uint32_t align;
2305 
2306     assert(pnum);
2307     *pnum = 0;
2308     total_size = bdrv_getlength(bs);
2309     if (total_size < 0) {
2310         ret = total_size;
2311         goto early_out;
2312     }
2313 
2314     if (offset >= total_size) {
2315         ret = BDRV_BLOCK_EOF;
2316         goto early_out;
2317     }
2318     if (!bytes) {
2319         ret = 0;
2320         goto early_out;
2321     }
2322 
2323     n = total_size - offset;
2324     if (n < bytes) {
2325         bytes = n;
2326     }
2327 
2328     /* Must be non-NULL or bdrv_getlength() would have failed */
2329     assert(bs->drv);
2330     if (!bs->drv->bdrv_co_block_status) {
2331         *pnum = bytes;
2332         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
2333         if (offset + bytes == total_size) {
2334             ret |= BDRV_BLOCK_EOF;
2335         }
2336         if (bs->drv->protocol_name) {
2337             ret |= BDRV_BLOCK_OFFSET_VALID;
2338             local_map = offset;
2339             local_file = bs;
2340         }
2341         goto early_out;
2342     }
2343 
2344     bdrv_inc_in_flight(bs);
2345 
2346     /* Round out to request_alignment boundaries */
2347     align = bs->bl.request_alignment;
2348     aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2349     aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2350 
2351     ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
2352                                         aligned_bytes, pnum, &local_map,
2353                                         &local_file);
2354     if (ret < 0) {
2355         *pnum = 0;
2356         goto out;
2357     }
2358 
2359     /*
2360      * The driver's result must be a non-zero multiple of request_alignment.
2361      * Clamp pnum and adjust map to original request.
2362      */
2363     assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2364            align > offset - aligned_offset);
2365     if (ret & BDRV_BLOCK_RECURSE) {
2366         assert(ret & BDRV_BLOCK_DATA);
2367         assert(ret & BDRV_BLOCK_OFFSET_VALID);
2368         assert(!(ret & BDRV_BLOCK_ZERO));
2369     }
2370 
2371     *pnum -= offset - aligned_offset;
2372     if (*pnum > bytes) {
2373         *pnum = bytes;
2374     }
2375     if (ret & BDRV_BLOCK_OFFSET_VALID) {
2376         local_map += offset - aligned_offset;
2377     }
2378 
2379     if (ret & BDRV_BLOCK_RAW) {
2380         assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
2381         ret = bdrv_co_block_status(local_file, want_zero, local_map,
2382                                    *pnum, pnum, &local_map, &local_file);
2383         goto out;
2384     }
2385 
2386     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
2387         ret |= BDRV_BLOCK_ALLOCATED;
2388     } else if (want_zero) {
2389         if (bdrv_unallocated_blocks_are_zero(bs)) {
2390             ret |= BDRV_BLOCK_ZERO;
2391         } else if (bs->backing) {
2392             BlockDriverState *bs2 = bs->backing->bs;
2393             int64_t size2 = bdrv_getlength(bs2);
2394 
2395             if (size2 >= 0 && offset >= size2) {
2396                 ret |= BDRV_BLOCK_ZERO;
2397             }
2398         }
2399     }
2400 
2401     if (want_zero && ret & BDRV_BLOCK_RECURSE &&
2402         local_file && local_file != bs &&
2403         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
2404         (ret & BDRV_BLOCK_OFFSET_VALID)) {
2405         int64_t file_pnum;
2406         int ret2;
2407 
2408         ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
2409                                     *pnum, &file_pnum, NULL, NULL);
2410         if (ret2 >= 0) {
2411             /* Ignore errors.  This is just providing extra information, it
2412              * is useful but not necessary.
2413              */
2414             if (ret2 & BDRV_BLOCK_EOF &&
2415                 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2416                 /*
2417                  * It is valid for the format block driver to read
2418                  * beyond the end of the underlying file's current
2419                  * size; such areas read as zero.
2420                  */
2421                 ret |= BDRV_BLOCK_ZERO;
2422             } else {
2423                 /* Limit request to the range reported by the protocol driver */
2424                 *pnum = file_pnum;
2425                 ret |= (ret2 & BDRV_BLOCK_ZERO);
2426             }
2427         }
2428     }
2429 
2430 out:
2431     bdrv_dec_in_flight(bs);
2432     if (ret >= 0 && offset + *pnum == total_size) {
2433         ret |= BDRV_BLOCK_EOF;
2434     }
2435 early_out:
2436     if (file) {
2437         *file = local_file;
2438     }
2439     if (map) {
2440         *map = local_map;
2441     }
2442     return ret;
2443 }
2444 
2445 static int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs,
2446                                                    BlockDriverState *base,
2447                                                    bool want_zero,
2448                                                    int64_t offset,
2449                                                    int64_t bytes,
2450                                                    int64_t *pnum,
2451                                                    int64_t *map,
2452                                                    BlockDriverState **file)
2453 {
2454     BlockDriverState *p;
2455     int ret = 0;
2456     bool first = true;
2457 
2458     assert(bs != base);
2459     for (p = bs; p != base; p = backing_bs(p)) {
2460         ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
2461                                    file);
2462         if (ret < 0) {
2463             break;
2464         }
2465         if (ret & BDRV_BLOCK_ZERO && ret & BDRV_BLOCK_EOF && !first) {
2466             /*
2467              * Reading beyond the end of the file continues to read
2468              * zeroes, but we can only widen the result to the
2469              * unallocated length we learned from an earlier
2470              * iteration.
2471              */
2472             *pnum = bytes;
2473         }
2474         if (ret & (BDRV_BLOCK_ZERO | BDRV_BLOCK_DATA)) {
2475             break;
2476         }
2477         /* [offset, pnum] unallocated on this layer, which could be only
2478          * the first part of [offset, bytes].  */
2479         bytes = MIN(bytes, *pnum);
2480         first = false;
2481     }
2482     return ret;
2483 }
2484 
2485 /* Coroutine wrapper for bdrv_block_status_above() */
2486 static void coroutine_fn bdrv_block_status_above_co_entry(void *opaque)
2487 {
2488     BdrvCoBlockStatusData *data = opaque;
2489 
2490     data->ret = bdrv_co_block_status_above(data->bs, data->base,
2491                                            data->want_zero,
2492                                            data->offset, data->bytes,
2493                                            data->pnum, data->map, data->file);
2494     data->done = true;
2495     aio_wait_kick();
2496 }
2497 
2498 /*
2499  * Synchronous wrapper around bdrv_co_block_status_above().
2500  *
2501  * See bdrv_co_block_status_above() for details.
2502  */
2503 static int bdrv_common_block_status_above(BlockDriverState *bs,
2504                                           BlockDriverState *base,
2505                                           bool want_zero, int64_t offset,
2506                                           int64_t bytes, int64_t *pnum,
2507                                           int64_t *map,
2508                                           BlockDriverState **file)
2509 {
2510     Coroutine *co;
2511     BdrvCoBlockStatusData data = {
2512         .bs = bs,
2513         .base = base,
2514         .want_zero = want_zero,
2515         .offset = offset,
2516         .bytes = bytes,
2517         .pnum = pnum,
2518         .map = map,
2519         .file = file,
2520         .done = false,
2521     };
2522 
2523     if (qemu_in_coroutine()) {
2524         /* Fast-path if already in coroutine context */
2525         bdrv_block_status_above_co_entry(&data);
2526     } else {
2527         co = qemu_coroutine_create(bdrv_block_status_above_co_entry, &data);
2528         bdrv_coroutine_enter(bs, co);
2529         BDRV_POLL_WHILE(bs, !data.done);
2530     }
2531     return data.ret;
2532 }
2533 
2534 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
2535                             int64_t offset, int64_t bytes, int64_t *pnum,
2536                             int64_t *map, BlockDriverState **file)
2537 {
2538     return bdrv_common_block_status_above(bs, base, true, offset, bytes,
2539                                           pnum, map, file);
2540 }
2541 
2542 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2543                       int64_t *pnum, int64_t *map, BlockDriverState **file)
2544 {
2545     return bdrv_block_status_above(bs, backing_bs(bs),
2546                                    offset, bytes, pnum, map, file);
2547 }
2548 
2549 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset,
2550                                    int64_t bytes, int64_t *pnum)
2551 {
2552     int ret;
2553     int64_t dummy;
2554 
2555     ret = bdrv_common_block_status_above(bs, backing_bs(bs), false, offset,
2556                                          bytes, pnum ? pnum : &dummy, NULL,
2557                                          NULL);
2558     if (ret < 0) {
2559         return ret;
2560     }
2561     return !!(ret & BDRV_BLOCK_ALLOCATED);
2562 }
2563 
2564 /*
2565  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
2566  *
2567  * Return 1 if (a prefix of) the given range is allocated in any image
2568  * between BASE and TOP (BASE is only included if include_base is set).
2569  * BASE can be NULL to check if the given offset is allocated in any
2570  * image of the chain.  Return 0 otherwise, or negative errno on
2571  * failure.
2572  *
2573  * 'pnum' is set to the number of bytes (including and immediately
2574  * following the specified offset) that are known to be in the same
2575  * allocated/unallocated state.  Note that a subsequent call starting
2576  * at 'offset + *pnum' may return the same allocation status (in other
2577  * words, the result is not necessarily the maximum possible range);
2578  * but 'pnum' will only be 0 when end of file is reached.
2579  *
2580  */
2581 int bdrv_is_allocated_above(BlockDriverState *top,
2582                             BlockDriverState *base,
2583                             bool include_base, int64_t offset,
2584                             int64_t bytes, int64_t *pnum)
2585 {
2586     BlockDriverState *intermediate;
2587     int ret;
2588     int64_t n = bytes;
2589 
2590     assert(base || !include_base);
2591 
2592     intermediate = top;
2593     while (include_base || intermediate != base) {
2594         int64_t pnum_inter;
2595         int64_t size_inter;
2596 
2597         assert(intermediate);
2598         ret = bdrv_is_allocated(intermediate, offset, bytes, &pnum_inter);
2599         if (ret < 0) {
2600             return ret;
2601         }
2602         if (ret) {
2603             *pnum = pnum_inter;
2604             return 1;
2605         }
2606 
2607         size_inter = bdrv_getlength(intermediate);
2608         if (size_inter < 0) {
2609             return size_inter;
2610         }
2611         if (n > pnum_inter &&
2612             (intermediate == top || offset + pnum_inter < size_inter)) {
2613             n = pnum_inter;
2614         }
2615 
2616         if (intermediate == base) {
2617             break;
2618         }
2619 
2620         intermediate = backing_bs(intermediate);
2621     }
2622 
2623     *pnum = n;
2624     return 0;
2625 }
2626 
2627 typedef struct BdrvVmstateCo {
2628     BlockDriverState    *bs;
2629     QEMUIOVector        *qiov;
2630     int64_t             pos;
2631     bool                is_read;
2632     int                 ret;
2633 } BdrvVmstateCo;
2634 
2635 static int coroutine_fn
2636 bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
2637                    bool is_read)
2638 {
2639     BlockDriver *drv = bs->drv;
2640     int ret = -ENOTSUP;
2641 
2642     bdrv_inc_in_flight(bs);
2643 
2644     if (!drv) {
2645         ret = -ENOMEDIUM;
2646     } else if (drv->bdrv_load_vmstate) {
2647         if (is_read) {
2648             ret = drv->bdrv_load_vmstate(bs, qiov, pos);
2649         } else {
2650             ret = drv->bdrv_save_vmstate(bs, qiov, pos);
2651         }
2652     } else if (bs->file) {
2653         ret = bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read);
2654     }
2655 
2656     bdrv_dec_in_flight(bs);
2657     return ret;
2658 }
2659 
2660 static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque)
2661 {
2662     BdrvVmstateCo *co = opaque;
2663     co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read);
2664     aio_wait_kick();
2665 }
2666 
2667 static inline int
2668 bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
2669                 bool is_read)
2670 {
2671     if (qemu_in_coroutine()) {
2672         return bdrv_co_rw_vmstate(bs, qiov, pos, is_read);
2673     } else {
2674         BdrvVmstateCo data = {
2675             .bs         = bs,
2676             .qiov       = qiov,
2677             .pos        = pos,
2678             .is_read    = is_read,
2679             .ret        = -EINPROGRESS,
2680         };
2681         Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data);
2682 
2683         bdrv_coroutine_enter(bs, co);
2684         BDRV_POLL_WHILE(bs, data.ret == -EINPROGRESS);
2685         return data.ret;
2686     }
2687 }
2688 
2689 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
2690                       int64_t pos, int size)
2691 {
2692     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2693     int ret;
2694 
2695     ret = bdrv_writev_vmstate(bs, &qiov, pos);
2696     if (ret < 0) {
2697         return ret;
2698     }
2699 
2700     return size;
2701 }
2702 
2703 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2704 {
2705     return bdrv_rw_vmstate(bs, qiov, pos, false);
2706 }
2707 
2708 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
2709                       int64_t pos, int size)
2710 {
2711     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2712     int ret;
2713 
2714     ret = bdrv_readv_vmstate(bs, &qiov, pos);
2715     if (ret < 0) {
2716         return ret;
2717     }
2718 
2719     return size;
2720 }
2721 
2722 int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2723 {
2724     return bdrv_rw_vmstate(bs, qiov, pos, true);
2725 }
2726 
2727 /**************************************************************/
2728 /* async I/Os */
2729 
2730 void bdrv_aio_cancel(BlockAIOCB *acb)
2731 {
2732     qemu_aio_ref(acb);
2733     bdrv_aio_cancel_async(acb);
2734     while (acb->refcnt > 1) {
2735         if (acb->aiocb_info->get_aio_context) {
2736             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2737         } else if (acb->bs) {
2738             /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
2739              * assert that we're not using an I/O thread.  Thread-safe
2740              * code should use bdrv_aio_cancel_async exclusively.
2741              */
2742             assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
2743             aio_poll(bdrv_get_aio_context(acb->bs), true);
2744         } else {
2745             abort();
2746         }
2747     }
2748     qemu_aio_unref(acb);
2749 }
2750 
2751 /* Async version of aio cancel. The caller is not blocked if the acb implements
2752  * cancel_async, otherwise we do nothing and let the request normally complete.
2753  * In either case the completion callback must be called. */
2754 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2755 {
2756     if (acb->aiocb_info->cancel_async) {
2757         acb->aiocb_info->cancel_async(acb);
2758     }
2759 }
2760 
2761 /**************************************************************/
2762 /* Coroutine block device emulation */
2763 
2764 typedef struct FlushCo {
2765     BlockDriverState *bs;
2766     int ret;
2767 } FlushCo;
2768 
2769 
2770 static void coroutine_fn bdrv_flush_co_entry(void *opaque)
2771 {
2772     FlushCo *rwco = opaque;
2773 
2774     rwco->ret = bdrv_co_flush(rwco->bs);
2775     aio_wait_kick();
2776 }
2777 
2778 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2779 {
2780     int current_gen;
2781     int ret = 0;
2782 
2783     bdrv_inc_in_flight(bs);
2784 
2785     if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2786         bdrv_is_sg(bs)) {
2787         goto early_exit;
2788     }
2789 
2790     qemu_co_mutex_lock(&bs->reqs_lock);
2791     current_gen = atomic_read(&bs->write_gen);
2792 
2793     /* Wait until any previous flushes are completed */
2794     while (bs->active_flush_req) {
2795         qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
2796     }
2797 
2798     /* Flushes reach this point in nondecreasing current_gen order.  */
2799     bs->active_flush_req = true;
2800     qemu_co_mutex_unlock(&bs->reqs_lock);
2801 
2802     /* Write back all layers by calling one driver function */
2803     if (bs->drv->bdrv_co_flush) {
2804         ret = bs->drv->bdrv_co_flush(bs);
2805         goto out;
2806     }
2807 
2808     /* Write back cached data to the OS even with cache=unsafe */
2809     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
2810     if (bs->drv->bdrv_co_flush_to_os) {
2811         ret = bs->drv->bdrv_co_flush_to_os(bs);
2812         if (ret < 0) {
2813             goto out;
2814         }
2815     }
2816 
2817     /* But don't actually force it to the disk with cache=unsafe */
2818     if (bs->open_flags & BDRV_O_NO_FLUSH) {
2819         goto flush_parent;
2820     }
2821 
2822     /* Check if we really need to flush anything */
2823     if (bs->flushed_gen == current_gen) {
2824         goto flush_parent;
2825     }
2826 
2827     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2828     if (!bs->drv) {
2829         /* bs->drv->bdrv_co_flush() might have ejected the BDS
2830          * (even in case of apparent success) */
2831         ret = -ENOMEDIUM;
2832         goto out;
2833     }
2834     if (bs->drv->bdrv_co_flush_to_disk) {
2835         ret = bs->drv->bdrv_co_flush_to_disk(bs);
2836     } else if (bs->drv->bdrv_aio_flush) {
2837         BlockAIOCB *acb;
2838         CoroutineIOCompletion co = {
2839             .coroutine = qemu_coroutine_self(),
2840         };
2841 
2842         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2843         if (acb == NULL) {
2844             ret = -EIO;
2845         } else {
2846             qemu_coroutine_yield();
2847             ret = co.ret;
2848         }
2849     } else {
2850         /*
2851          * Some block drivers always operate in either writethrough or unsafe
2852          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2853          * know how the server works (because the behaviour is hardcoded or
2854          * depends on server-side configuration), so we can't ensure that
2855          * everything is safe on disk. Returning an error doesn't work because
2856          * that would break guests even if the server operates in writethrough
2857          * mode.
2858          *
2859          * Let's hope the user knows what he's doing.
2860          */
2861         ret = 0;
2862     }
2863 
2864     if (ret < 0) {
2865         goto out;
2866     }
2867 
2868     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
2869      * in the case of cache=unsafe, so there are no useless flushes.
2870      */
2871 flush_parent:
2872     ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2873 out:
2874     /* Notify any pending flushes that we have completed */
2875     if (ret == 0) {
2876         bs->flushed_gen = current_gen;
2877     }
2878 
2879     qemu_co_mutex_lock(&bs->reqs_lock);
2880     bs->active_flush_req = false;
2881     /* Return value is ignored - it's ok if wait queue is empty */
2882     qemu_co_queue_next(&bs->flush_queue);
2883     qemu_co_mutex_unlock(&bs->reqs_lock);
2884 
2885 early_exit:
2886     bdrv_dec_in_flight(bs);
2887     return ret;
2888 }
2889 
2890 int bdrv_flush(BlockDriverState *bs)
2891 {
2892     Coroutine *co;
2893     FlushCo flush_co = {
2894         .bs = bs,
2895         .ret = NOT_DONE,
2896     };
2897 
2898     if (qemu_in_coroutine()) {
2899         /* Fast-path if already in coroutine context */
2900         bdrv_flush_co_entry(&flush_co);
2901     } else {
2902         co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co);
2903         bdrv_coroutine_enter(bs, co);
2904         BDRV_POLL_WHILE(bs, flush_co.ret == NOT_DONE);
2905     }
2906 
2907     return flush_co.ret;
2908 }
2909 
2910 typedef struct DiscardCo {
2911     BdrvChild *child;
2912     int64_t offset;
2913     int64_t bytes;
2914     int ret;
2915 } DiscardCo;
2916 static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque)
2917 {
2918     DiscardCo *rwco = opaque;
2919 
2920     rwco->ret = bdrv_co_pdiscard(rwco->child, rwco->offset, rwco->bytes);
2921     aio_wait_kick();
2922 }
2923 
2924 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
2925                                   int64_t bytes)
2926 {
2927     BdrvTrackedRequest req;
2928     int max_pdiscard, ret;
2929     int head, tail, align;
2930     BlockDriverState *bs = child->bs;
2931 
2932     if (!bs || !bs->drv || !bdrv_is_inserted(bs)) {
2933         return -ENOMEDIUM;
2934     }
2935 
2936     if (bdrv_has_readonly_bitmaps(bs)) {
2937         return -EPERM;
2938     }
2939 
2940     if (offset < 0 || bytes < 0 || bytes > INT64_MAX - offset) {
2941         return -EIO;
2942     }
2943 
2944     /* Do nothing if disabled.  */
2945     if (!(bs->open_flags & BDRV_O_UNMAP)) {
2946         return 0;
2947     }
2948 
2949     if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
2950         return 0;
2951     }
2952 
2953     /* Discard is advisory, but some devices track and coalesce
2954      * unaligned requests, so we must pass everything down rather than
2955      * round here.  Still, most devices will just silently ignore
2956      * unaligned requests (by returning -ENOTSUP), so we must fragment
2957      * the request accordingly.  */
2958     align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2959     assert(align % bs->bl.request_alignment == 0);
2960     head = offset % align;
2961     tail = (offset + bytes) % align;
2962 
2963     bdrv_inc_in_flight(bs);
2964     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
2965 
2966     ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
2967     if (ret < 0) {
2968         goto out;
2969     }
2970 
2971     max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX),
2972                                    align);
2973     assert(max_pdiscard >= bs->bl.request_alignment);
2974 
2975     while (bytes > 0) {
2976         int64_t num = bytes;
2977 
2978         if (head) {
2979             /* Make small requests to get to alignment boundaries. */
2980             num = MIN(bytes, align - head);
2981             if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
2982                 num %= bs->bl.request_alignment;
2983             }
2984             head = (head + num) % align;
2985             assert(num < max_pdiscard);
2986         } else if (tail) {
2987             if (num > align) {
2988                 /* Shorten the request to the last aligned cluster.  */
2989                 num -= tail;
2990             } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
2991                        tail > bs->bl.request_alignment) {
2992                 tail %= bs->bl.request_alignment;
2993                 num -= tail;
2994             }
2995         }
2996         /* limit request size */
2997         if (num > max_pdiscard) {
2998             num = max_pdiscard;
2999         }
3000 
3001         if (!bs->drv) {
3002             ret = -ENOMEDIUM;
3003             goto out;
3004         }
3005         if (bs->drv->bdrv_co_pdiscard) {
3006             ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
3007         } else {
3008             BlockAIOCB *acb;
3009             CoroutineIOCompletion co = {
3010                 .coroutine = qemu_coroutine_self(),
3011             };
3012 
3013             acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
3014                                              bdrv_co_io_em_complete, &co);
3015             if (acb == NULL) {
3016                 ret = -EIO;
3017                 goto out;
3018             } else {
3019                 qemu_coroutine_yield();
3020                 ret = co.ret;
3021             }
3022         }
3023         if (ret && ret != -ENOTSUP) {
3024             goto out;
3025         }
3026 
3027         offset += num;
3028         bytes -= num;
3029     }
3030     ret = 0;
3031 out:
3032     bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
3033     tracked_request_end(&req);
3034     bdrv_dec_in_flight(bs);
3035     return ret;
3036 }
3037 
3038 int bdrv_pdiscard(BdrvChild *child, int64_t offset, int64_t bytes)
3039 {
3040     Coroutine *co;
3041     DiscardCo rwco = {
3042         .child = child,
3043         .offset = offset,
3044         .bytes = bytes,
3045         .ret = NOT_DONE,
3046     };
3047 
3048     if (qemu_in_coroutine()) {
3049         /* Fast-path if already in coroutine context */
3050         bdrv_pdiscard_co_entry(&rwco);
3051     } else {
3052         co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco);
3053         bdrv_coroutine_enter(child->bs, co);
3054         BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE);
3055     }
3056 
3057     return rwco.ret;
3058 }
3059 
3060 int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
3061 {
3062     BlockDriver *drv = bs->drv;
3063     CoroutineIOCompletion co = {
3064         .coroutine = qemu_coroutine_self(),
3065     };
3066     BlockAIOCB *acb;
3067 
3068     bdrv_inc_in_flight(bs);
3069     if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
3070         co.ret = -ENOTSUP;
3071         goto out;
3072     }
3073 
3074     if (drv->bdrv_co_ioctl) {
3075         co.ret = drv->bdrv_co_ioctl(bs, req, buf);
3076     } else {
3077         acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
3078         if (!acb) {
3079             co.ret = -ENOTSUP;
3080             goto out;
3081         }
3082         qemu_coroutine_yield();
3083     }
3084 out:
3085     bdrv_dec_in_flight(bs);
3086     return co.ret;
3087 }
3088 
3089 void *qemu_blockalign(BlockDriverState *bs, size_t size)
3090 {
3091     return qemu_memalign(bdrv_opt_mem_align(bs), size);
3092 }
3093 
3094 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
3095 {
3096     return memset(qemu_blockalign(bs, size), 0, size);
3097 }
3098 
3099 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
3100 {
3101     size_t align = bdrv_opt_mem_align(bs);
3102 
3103     /* Ensure that NULL is never returned on success */
3104     assert(align > 0);
3105     if (size == 0) {
3106         size = align;
3107     }
3108 
3109     return qemu_try_memalign(align, size);
3110 }
3111 
3112 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
3113 {
3114     void *mem = qemu_try_blockalign(bs, size);
3115 
3116     if (mem) {
3117         memset(mem, 0, size);
3118     }
3119 
3120     return mem;
3121 }
3122 
3123 /*
3124  * Check if all memory in this vector is sector aligned.
3125  */
3126 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
3127 {
3128     int i;
3129     size_t alignment = bdrv_min_mem_align(bs);
3130 
3131     for (i = 0; i < qiov->niov; i++) {
3132         if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
3133             return false;
3134         }
3135         if (qiov->iov[i].iov_len % alignment) {
3136             return false;
3137         }
3138     }
3139 
3140     return true;
3141 }
3142 
3143 void bdrv_add_before_write_notifier(BlockDriverState *bs,
3144                                     NotifierWithReturn *notifier)
3145 {
3146     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
3147 }
3148 
3149 void bdrv_io_plug(BlockDriverState *bs)
3150 {
3151     BdrvChild *child;
3152 
3153     QLIST_FOREACH(child, &bs->children, next) {
3154         bdrv_io_plug(child->bs);
3155     }
3156 
3157     if (atomic_fetch_inc(&bs->io_plugged) == 0) {
3158         BlockDriver *drv = bs->drv;
3159         if (drv && drv->bdrv_io_plug) {
3160             drv->bdrv_io_plug(bs);
3161         }
3162     }
3163 }
3164 
3165 void bdrv_io_unplug(BlockDriverState *bs)
3166 {
3167     BdrvChild *child;
3168 
3169     assert(bs->io_plugged);
3170     if (atomic_fetch_dec(&bs->io_plugged) == 1) {
3171         BlockDriver *drv = bs->drv;
3172         if (drv && drv->bdrv_io_unplug) {
3173             drv->bdrv_io_unplug(bs);
3174         }
3175     }
3176 
3177     QLIST_FOREACH(child, &bs->children, next) {
3178         bdrv_io_unplug(child->bs);
3179     }
3180 }
3181 
3182 void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size)
3183 {
3184     BdrvChild *child;
3185 
3186     if (bs->drv && bs->drv->bdrv_register_buf) {
3187         bs->drv->bdrv_register_buf(bs, host, size);
3188     }
3189     QLIST_FOREACH(child, &bs->children, next) {
3190         bdrv_register_buf(child->bs, host, size);
3191     }
3192 }
3193 
3194 void bdrv_unregister_buf(BlockDriverState *bs, void *host)
3195 {
3196     BdrvChild *child;
3197 
3198     if (bs->drv && bs->drv->bdrv_unregister_buf) {
3199         bs->drv->bdrv_unregister_buf(bs, host);
3200     }
3201     QLIST_FOREACH(child, &bs->children, next) {
3202         bdrv_unregister_buf(child->bs, host);
3203     }
3204 }
3205 
3206 static int coroutine_fn bdrv_co_copy_range_internal(
3207         BdrvChild *src, uint64_t src_offset, BdrvChild *dst,
3208         uint64_t dst_offset, uint64_t bytes,
3209         BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
3210         bool recurse_src)
3211 {
3212     BdrvTrackedRequest req;
3213     int ret;
3214 
3215     /* TODO We can support BDRV_REQ_NO_FALLBACK here */
3216     assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
3217     assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
3218 
3219     if (!dst || !dst->bs) {
3220         return -ENOMEDIUM;
3221     }
3222     ret = bdrv_check_byte_request(dst->bs, dst_offset, bytes);
3223     if (ret) {
3224         return ret;
3225     }
3226     if (write_flags & BDRV_REQ_ZERO_WRITE) {
3227         return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
3228     }
3229 
3230     if (!src || !src->bs) {
3231         return -ENOMEDIUM;
3232     }
3233     ret = bdrv_check_byte_request(src->bs, src_offset, bytes);
3234     if (ret) {
3235         return ret;
3236     }
3237 
3238     if (!src->bs->drv->bdrv_co_copy_range_from
3239         || !dst->bs->drv->bdrv_co_copy_range_to
3240         || src->bs->encrypted || dst->bs->encrypted) {
3241         return -ENOTSUP;
3242     }
3243 
3244     if (recurse_src) {
3245         bdrv_inc_in_flight(src->bs);
3246         tracked_request_begin(&req, src->bs, src_offset, bytes,
3247                               BDRV_TRACKED_READ);
3248 
3249         /* BDRV_REQ_SERIALISING is only for write operation */
3250         assert(!(read_flags & BDRV_REQ_SERIALISING));
3251         bdrv_wait_serialising_requests(&req);
3252 
3253         ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3254                                                     src, src_offset,
3255                                                     dst, dst_offset,
3256                                                     bytes,
3257                                                     read_flags, write_flags);
3258 
3259         tracked_request_end(&req);
3260         bdrv_dec_in_flight(src->bs);
3261     } else {
3262         bdrv_inc_in_flight(dst->bs);
3263         tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3264                               BDRV_TRACKED_WRITE);
3265         ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
3266                                         write_flags);
3267         if (!ret) {
3268             ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3269                                                       src, src_offset,
3270                                                       dst, dst_offset,
3271                                                       bytes,
3272                                                       read_flags, write_flags);
3273         }
3274         bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3275         tracked_request_end(&req);
3276         bdrv_dec_in_flight(dst->bs);
3277     }
3278 
3279     return ret;
3280 }
3281 
3282 /* Copy range from @src to @dst.
3283  *
3284  * See the comment of bdrv_co_copy_range for the parameter and return value
3285  * semantics. */
3286 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, uint64_t src_offset,
3287                                          BdrvChild *dst, uint64_t dst_offset,
3288                                          uint64_t bytes,
3289                                          BdrvRequestFlags read_flags,
3290                                          BdrvRequestFlags write_flags)
3291 {
3292     trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3293                                   read_flags, write_flags);
3294     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3295                                        bytes, read_flags, write_flags, true);
3296 }
3297 
3298 /* Copy range from @src to @dst.
3299  *
3300  * See the comment of bdrv_co_copy_range for the parameter and return value
3301  * semantics. */
3302 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, uint64_t src_offset,
3303                                        BdrvChild *dst, uint64_t dst_offset,
3304                                        uint64_t bytes,
3305                                        BdrvRequestFlags read_flags,
3306                                        BdrvRequestFlags write_flags)
3307 {
3308     trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3309                                 read_flags, write_flags);
3310     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3311                                        bytes, read_flags, write_flags, false);
3312 }
3313 
3314 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, uint64_t src_offset,
3315                                     BdrvChild *dst, uint64_t dst_offset,
3316                                     uint64_t bytes, BdrvRequestFlags read_flags,
3317                                     BdrvRequestFlags write_flags)
3318 {
3319     return bdrv_co_copy_range_from(src, src_offset,
3320                                    dst, dst_offset,
3321                                    bytes, read_flags, write_flags);
3322 }
3323 
3324 static void bdrv_parent_cb_resize(BlockDriverState *bs)
3325 {
3326     BdrvChild *c;
3327     QLIST_FOREACH(c, &bs->parents, next_parent) {
3328         if (c->role->resize) {
3329             c->role->resize(c);
3330         }
3331     }
3332 }
3333 
3334 /**
3335  * Truncate file to 'offset' bytes (needed only for file protocols)
3336  *
3337  * If 'exact' is true, the file must be resized to exactly the given
3338  * 'offset'.  Otherwise, it is sufficient for the node to be at least
3339  * 'offset' bytes in length.
3340  */
3341 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
3342                                   PreallocMode prealloc, Error **errp)
3343 {
3344     BlockDriverState *bs = child->bs;
3345     BlockDriver *drv = bs->drv;
3346     BdrvTrackedRequest req;
3347     int64_t old_size, new_bytes;
3348     int ret;
3349 
3350 
3351     /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
3352     if (!drv) {
3353         error_setg(errp, "No medium inserted");
3354         return -ENOMEDIUM;
3355     }
3356     if (offset < 0) {
3357         error_setg(errp, "Image size cannot be negative");
3358         return -EINVAL;
3359     }
3360 
3361     old_size = bdrv_getlength(bs);
3362     if (old_size < 0) {
3363         error_setg_errno(errp, -old_size, "Failed to get old image size");
3364         return old_size;
3365     }
3366 
3367     if (offset > old_size) {
3368         new_bytes = offset - old_size;
3369     } else {
3370         new_bytes = 0;
3371     }
3372 
3373     bdrv_inc_in_flight(bs);
3374     tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
3375                           BDRV_TRACKED_TRUNCATE);
3376 
3377     /* If we are growing the image and potentially using preallocation for the
3378      * new area, we need to make sure that no write requests are made to it
3379      * concurrently or they might be overwritten by preallocation. */
3380     if (new_bytes) {
3381         bdrv_mark_request_serialising(&req, 1);
3382     }
3383     if (bs->read_only) {
3384         error_setg(errp, "Image is read-only");
3385         ret = -EACCES;
3386         goto out;
3387     }
3388     ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3389                                     0);
3390     if (ret < 0) {
3391         error_setg_errno(errp, -ret,
3392                          "Failed to prepare request for truncation");
3393         goto out;
3394     }
3395 
3396     if (drv->bdrv_co_truncate) {
3397         ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, errp);
3398     } else if (bs->file && drv->is_filter) {
3399         ret = bdrv_co_truncate(bs->file, offset, exact, prealloc, errp);
3400     } else {
3401         error_setg(errp, "Image format driver does not support resize");
3402         ret = -ENOTSUP;
3403         goto out;
3404     }
3405     if (ret < 0) {
3406         goto out;
3407     }
3408 
3409     ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
3410     if (ret < 0) {
3411         error_setg_errno(errp, -ret, "Could not refresh total sector count");
3412     } else {
3413         offset = bs->total_sectors * BDRV_SECTOR_SIZE;
3414     }
3415     /* It's possible that truncation succeeded but refresh_total_sectors
3416      * failed, but the latter doesn't affect how we should finish the request.
3417      * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */
3418     bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
3419 
3420 out:
3421     tracked_request_end(&req);
3422     bdrv_dec_in_flight(bs);
3423 
3424     return ret;
3425 }
3426 
3427 typedef struct TruncateCo {
3428     BdrvChild *child;
3429     int64_t offset;
3430     bool exact;
3431     PreallocMode prealloc;
3432     Error **errp;
3433     int ret;
3434 } TruncateCo;
3435 
3436 static void coroutine_fn bdrv_truncate_co_entry(void *opaque)
3437 {
3438     TruncateCo *tco = opaque;
3439     tco->ret = bdrv_co_truncate(tco->child, tco->offset, tco->exact,
3440                                 tco->prealloc, tco->errp);
3441     aio_wait_kick();
3442 }
3443 
3444 int bdrv_truncate(BdrvChild *child, int64_t offset, bool exact,
3445                   PreallocMode prealloc, Error **errp)
3446 {
3447     Coroutine *co;
3448     TruncateCo tco = {
3449         .child      = child,
3450         .offset     = offset,
3451         .exact      = exact,
3452         .prealloc   = prealloc,
3453         .errp       = errp,
3454         .ret        = NOT_DONE,
3455     };
3456 
3457     if (qemu_in_coroutine()) {
3458         /* Fast-path if already in coroutine context */
3459         bdrv_truncate_co_entry(&tco);
3460     } else {
3461         co = qemu_coroutine_create(bdrv_truncate_co_entry, &tco);
3462         bdrv_coroutine_enter(child->bs, co);
3463         BDRV_POLL_WHILE(child->bs, tco.ret == NOT_DONE);
3464     }
3465 
3466     return tco.ret;
3467 }
3468