1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/aio-wait.h" 29 #include "block/blockjob.h" 30 #include "block/blockjob_int.h" 31 #include "block/block_int.h" 32 #include "block/coroutines.h" 33 #include "block/dirty-bitmap.h" 34 #include "block/write-threshold.h" 35 #include "qemu/cutils.h" 36 #include "qemu/memalign.h" 37 #include "qapi/error.h" 38 #include "qemu/error-report.h" 39 #include "qemu/main-loop.h" 40 #include "sysemu/replay.h" 41 42 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 43 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 44 45 static void bdrv_parent_cb_resize(BlockDriverState *bs); 46 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 47 int64_t offset, int64_t bytes, BdrvRequestFlags flags); 48 49 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore) 50 { 51 BdrvChild *c, *next; 52 53 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 54 if (c == ignore) { 55 continue; 56 } 57 bdrv_parent_drained_begin_single(c); 58 } 59 } 60 61 void bdrv_parent_drained_end_single(BdrvChild *c) 62 { 63 IO_OR_GS_CODE(); 64 65 assert(c->quiesced_parent); 66 c->quiesced_parent = false; 67 68 if (c->klass->drained_end) { 69 c->klass->drained_end(c); 70 } 71 } 72 73 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore) 74 { 75 BdrvChild *c; 76 77 QLIST_FOREACH(c, &bs->parents, next_parent) { 78 if (c == ignore) { 79 continue; 80 } 81 bdrv_parent_drained_end_single(c); 82 } 83 } 84 85 bool bdrv_parent_drained_poll_single(BdrvChild *c) 86 { 87 if (c->klass->drained_poll) { 88 return c->klass->drained_poll(c); 89 } 90 return false; 91 } 92 93 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore, 94 bool ignore_bds_parents) 95 { 96 BdrvChild *c, *next; 97 bool busy = false; 98 99 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 100 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 101 continue; 102 } 103 busy |= bdrv_parent_drained_poll_single(c); 104 } 105 106 return busy; 107 } 108 109 void bdrv_parent_drained_begin_single(BdrvChild *c) 110 { 111 IO_OR_GS_CODE(); 112 113 assert(!c->quiesced_parent); 114 c->quiesced_parent = true; 115 116 if (c->klass->drained_begin) { 117 c->klass->drained_begin(c); 118 } 119 } 120 121 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 122 { 123 dst->pdiscard_alignment = MAX(dst->pdiscard_alignment, 124 src->pdiscard_alignment); 125 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 126 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 127 dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer, 128 src->max_hw_transfer); 129 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 130 src->opt_mem_alignment); 131 dst->min_mem_alignment = MAX(dst->min_mem_alignment, 132 src->min_mem_alignment); 133 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 134 dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov); 135 } 136 137 typedef struct BdrvRefreshLimitsState { 138 BlockDriverState *bs; 139 BlockLimits old_bl; 140 } BdrvRefreshLimitsState; 141 142 static void bdrv_refresh_limits_abort(void *opaque) 143 { 144 BdrvRefreshLimitsState *s = opaque; 145 146 s->bs->bl = s->old_bl; 147 } 148 149 static TransactionActionDrv bdrv_refresh_limits_drv = { 150 .abort = bdrv_refresh_limits_abort, 151 .clean = g_free, 152 }; 153 154 /* @tran is allowed to be NULL, in this case no rollback is possible. */ 155 void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp) 156 { 157 ERRP_GUARD(); 158 BlockDriver *drv = bs->drv; 159 BdrvChild *c; 160 bool have_limits; 161 162 GLOBAL_STATE_CODE(); 163 assume_graph_lock(); /* FIXME */ 164 165 if (tran) { 166 BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1); 167 *s = (BdrvRefreshLimitsState) { 168 .bs = bs, 169 .old_bl = bs->bl, 170 }; 171 tran_add(tran, &bdrv_refresh_limits_drv, s); 172 } 173 174 memset(&bs->bl, 0, sizeof(bs->bl)); 175 176 if (!drv) { 177 return; 178 } 179 180 /* Default alignment based on whether driver has byte interface */ 181 bs->bl.request_alignment = (drv->bdrv_co_preadv || 182 drv->bdrv_aio_preadv || 183 drv->bdrv_co_preadv_part) ? 1 : 512; 184 185 bs->bl.has_variable_length = drv->has_variable_length; 186 187 /* Take some limits from the children as a default */ 188 have_limits = false; 189 QLIST_FOREACH(c, &bs->children, next) { 190 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW)) 191 { 192 bdrv_merge_limits(&bs->bl, &c->bs->bl); 193 have_limits = true; 194 } 195 196 if (c->role & BDRV_CHILD_FILTERED) { 197 bs->bl.has_variable_length |= c->bs->bl.has_variable_length; 198 } 199 } 200 201 if (!have_limits) { 202 bs->bl.min_mem_alignment = 512; 203 bs->bl.opt_mem_alignment = qemu_real_host_page_size(); 204 205 /* Safe default since most protocols use readv()/writev()/etc */ 206 bs->bl.max_iov = IOV_MAX; 207 } 208 209 /* Then let the driver override it */ 210 if (drv->bdrv_refresh_limits) { 211 drv->bdrv_refresh_limits(bs, errp); 212 if (*errp) { 213 return; 214 } 215 } 216 217 if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) { 218 error_setg(errp, "Driver requires too large request alignment"); 219 } 220 } 221 222 /** 223 * The copy-on-read flag is actually a reference count so multiple users may 224 * use the feature without worrying about clobbering its previous state. 225 * Copy-on-read stays enabled until all users have called to disable it. 226 */ 227 void bdrv_enable_copy_on_read(BlockDriverState *bs) 228 { 229 IO_CODE(); 230 qatomic_inc(&bs->copy_on_read); 231 } 232 233 void bdrv_disable_copy_on_read(BlockDriverState *bs) 234 { 235 int old = qatomic_fetch_dec(&bs->copy_on_read); 236 IO_CODE(); 237 assert(old >= 1); 238 } 239 240 typedef struct { 241 Coroutine *co; 242 BlockDriverState *bs; 243 bool done; 244 bool begin; 245 bool poll; 246 BdrvChild *parent; 247 } BdrvCoDrainData; 248 249 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */ 250 bool bdrv_drain_poll(BlockDriverState *bs, BdrvChild *ignore_parent, 251 bool ignore_bds_parents) 252 { 253 IO_OR_GS_CODE(); 254 255 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) { 256 return true; 257 } 258 259 if (qatomic_read(&bs->in_flight)) { 260 return true; 261 } 262 263 return false; 264 } 265 266 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, 267 BdrvChild *ignore_parent) 268 { 269 return bdrv_drain_poll(bs, ignore_parent, false); 270 } 271 272 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent, 273 bool poll); 274 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent); 275 276 static void bdrv_co_drain_bh_cb(void *opaque) 277 { 278 BdrvCoDrainData *data = opaque; 279 Coroutine *co = data->co; 280 BlockDriverState *bs = data->bs; 281 282 if (bs) { 283 AioContext *ctx = bdrv_get_aio_context(bs); 284 aio_context_acquire(ctx); 285 bdrv_dec_in_flight(bs); 286 if (data->begin) { 287 bdrv_do_drained_begin(bs, data->parent, data->poll); 288 } else { 289 assert(!data->poll); 290 bdrv_do_drained_end(bs, data->parent); 291 } 292 aio_context_release(ctx); 293 } else { 294 assert(data->begin); 295 bdrv_drain_all_begin(); 296 } 297 298 data->done = true; 299 aio_co_wake(co); 300 } 301 302 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 303 bool begin, 304 BdrvChild *parent, 305 bool poll) 306 { 307 BdrvCoDrainData data; 308 Coroutine *self = qemu_coroutine_self(); 309 AioContext *ctx = bdrv_get_aio_context(bs); 310 AioContext *co_ctx = qemu_coroutine_get_aio_context(self); 311 312 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 313 * other coroutines run if they were queued by aio_co_enter(). */ 314 315 assert(qemu_in_coroutine()); 316 data = (BdrvCoDrainData) { 317 .co = self, 318 .bs = bs, 319 .done = false, 320 .begin = begin, 321 .parent = parent, 322 .poll = poll, 323 }; 324 325 if (bs) { 326 bdrv_inc_in_flight(bs); 327 } 328 329 /* 330 * Temporarily drop the lock across yield or we would get deadlocks. 331 * bdrv_co_drain_bh_cb() reaquires the lock as needed. 332 * 333 * When we yield below, the lock for the current context will be 334 * released, so if this is actually the lock that protects bs, don't drop 335 * it a second time. 336 */ 337 if (ctx != co_ctx) { 338 aio_context_release(ctx); 339 } 340 replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data); 341 342 qemu_coroutine_yield(); 343 /* If we are resumed from some other event (such as an aio completion or a 344 * timer callback), it is a bug in the caller that should be fixed. */ 345 assert(data.done); 346 347 /* Reaquire the AioContext of bs if we dropped it */ 348 if (ctx != co_ctx) { 349 aio_context_acquire(ctx); 350 } 351 } 352 353 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent, 354 bool poll) 355 { 356 IO_OR_GS_CODE(); 357 358 if (qemu_in_coroutine()) { 359 bdrv_co_yield_to_drain(bs, true, parent, poll); 360 return; 361 } 362 363 /* Stop things in parent-to-child order */ 364 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) { 365 aio_disable_external(bdrv_get_aio_context(bs)); 366 bdrv_parent_drained_begin(bs, parent); 367 if (bs->drv && bs->drv->bdrv_drain_begin) { 368 bs->drv->bdrv_drain_begin(bs); 369 } 370 } 371 372 /* 373 * Wait for drained requests to finish. 374 * 375 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The 376 * call is needed so things in this AioContext can make progress even 377 * though we don't return to the main AioContext loop - this automatically 378 * includes other nodes in the same AioContext and therefore all child 379 * nodes. 380 */ 381 if (poll) { 382 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, parent)); 383 } 384 } 385 386 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, BdrvChild *parent) 387 { 388 bdrv_do_drained_begin(bs, parent, false); 389 } 390 391 void bdrv_drained_begin(BlockDriverState *bs) 392 { 393 IO_OR_GS_CODE(); 394 bdrv_do_drained_begin(bs, NULL, true); 395 } 396 397 /** 398 * This function does not poll, nor must any of its recursively called 399 * functions. 400 */ 401 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent) 402 { 403 int old_quiesce_counter; 404 405 if (qemu_in_coroutine()) { 406 bdrv_co_yield_to_drain(bs, false, parent, false); 407 return; 408 } 409 assert(bs->quiesce_counter > 0); 410 411 /* Re-enable things in child-to-parent order */ 412 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter); 413 if (old_quiesce_counter == 1) { 414 if (bs->drv && bs->drv->bdrv_drain_end) { 415 bs->drv->bdrv_drain_end(bs); 416 } 417 bdrv_parent_drained_end(bs, parent); 418 aio_enable_external(bdrv_get_aio_context(bs)); 419 } 420 } 421 422 void bdrv_drained_end(BlockDriverState *bs) 423 { 424 IO_OR_GS_CODE(); 425 bdrv_do_drained_end(bs, NULL); 426 } 427 428 void bdrv_drain(BlockDriverState *bs) 429 { 430 IO_OR_GS_CODE(); 431 bdrv_drained_begin(bs); 432 bdrv_drained_end(bs); 433 } 434 435 static void bdrv_drain_assert_idle(BlockDriverState *bs) 436 { 437 BdrvChild *child, *next; 438 439 assert(qatomic_read(&bs->in_flight) == 0); 440 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 441 bdrv_drain_assert_idle(child->bs); 442 } 443 } 444 445 unsigned int bdrv_drain_all_count = 0; 446 447 static bool bdrv_drain_all_poll(void) 448 { 449 BlockDriverState *bs = NULL; 450 bool result = false; 451 GLOBAL_STATE_CODE(); 452 453 /* bdrv_drain_poll() can't make changes to the graph and we are holding the 454 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ 455 while ((bs = bdrv_next_all_states(bs))) { 456 AioContext *aio_context = bdrv_get_aio_context(bs); 457 aio_context_acquire(aio_context); 458 result |= bdrv_drain_poll(bs, NULL, true); 459 aio_context_release(aio_context); 460 } 461 462 return result; 463 } 464 465 /* 466 * Wait for pending requests to complete across all BlockDriverStates 467 * 468 * This function does not flush data to disk, use bdrv_flush_all() for that 469 * after calling this function. 470 * 471 * This pauses all block jobs and disables external clients. It must 472 * be paired with bdrv_drain_all_end(). 473 * 474 * NOTE: no new block jobs or BlockDriverStates can be created between 475 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 476 */ 477 void bdrv_drain_all_begin_nopoll(void) 478 { 479 BlockDriverState *bs = NULL; 480 GLOBAL_STATE_CODE(); 481 482 /* 483 * bdrv queue is managed by record/replay, 484 * waiting for finishing the I/O requests may 485 * be infinite 486 */ 487 if (replay_events_enabled()) { 488 return; 489 } 490 491 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main 492 * loop AioContext, so make sure we're in the main context. */ 493 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 494 assert(bdrv_drain_all_count < INT_MAX); 495 bdrv_drain_all_count++; 496 497 /* Quiesce all nodes, without polling in-flight requests yet. The graph 498 * cannot change during this loop. */ 499 while ((bs = bdrv_next_all_states(bs))) { 500 AioContext *aio_context = bdrv_get_aio_context(bs); 501 502 aio_context_acquire(aio_context); 503 bdrv_do_drained_begin(bs, NULL, false); 504 aio_context_release(aio_context); 505 } 506 } 507 508 void bdrv_drain_all_begin(void) 509 { 510 BlockDriverState *bs = NULL; 511 512 if (qemu_in_coroutine()) { 513 bdrv_co_yield_to_drain(NULL, true, NULL, true); 514 return; 515 } 516 517 /* 518 * bdrv queue is managed by record/replay, 519 * waiting for finishing the I/O requests may 520 * be infinite 521 */ 522 if (replay_events_enabled()) { 523 return; 524 } 525 526 bdrv_drain_all_begin_nopoll(); 527 528 /* Now poll the in-flight requests */ 529 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll()); 530 531 while ((bs = bdrv_next_all_states(bs))) { 532 bdrv_drain_assert_idle(bs); 533 } 534 } 535 536 void bdrv_drain_all_end_quiesce(BlockDriverState *bs) 537 { 538 GLOBAL_STATE_CODE(); 539 540 g_assert(bs->quiesce_counter > 0); 541 g_assert(!bs->refcnt); 542 543 while (bs->quiesce_counter) { 544 bdrv_do_drained_end(bs, NULL); 545 } 546 } 547 548 void bdrv_drain_all_end(void) 549 { 550 BlockDriverState *bs = NULL; 551 GLOBAL_STATE_CODE(); 552 553 /* 554 * bdrv queue is managed by record/replay, 555 * waiting for finishing the I/O requests may 556 * be endless 557 */ 558 if (replay_events_enabled()) { 559 return; 560 } 561 562 while ((bs = bdrv_next_all_states(bs))) { 563 AioContext *aio_context = bdrv_get_aio_context(bs); 564 565 aio_context_acquire(aio_context); 566 bdrv_do_drained_end(bs, NULL); 567 aio_context_release(aio_context); 568 } 569 570 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 571 assert(bdrv_drain_all_count > 0); 572 bdrv_drain_all_count--; 573 } 574 575 void bdrv_drain_all(void) 576 { 577 GLOBAL_STATE_CODE(); 578 bdrv_drain_all_begin(); 579 bdrv_drain_all_end(); 580 } 581 582 /** 583 * Remove an active request from the tracked requests list 584 * 585 * This function should be called when a tracked request is completing. 586 */ 587 static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req) 588 { 589 if (req->serialising) { 590 qatomic_dec(&req->bs->serialising_in_flight); 591 } 592 593 qemu_co_mutex_lock(&req->bs->reqs_lock); 594 QLIST_REMOVE(req, list); 595 qemu_co_queue_restart_all(&req->wait_queue); 596 qemu_co_mutex_unlock(&req->bs->reqs_lock); 597 } 598 599 /** 600 * Add an active request to the tracked requests list 601 */ 602 static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req, 603 BlockDriverState *bs, 604 int64_t offset, 605 int64_t bytes, 606 enum BdrvTrackedRequestType type) 607 { 608 bdrv_check_request(offset, bytes, &error_abort); 609 610 *req = (BdrvTrackedRequest){ 611 .bs = bs, 612 .offset = offset, 613 .bytes = bytes, 614 .type = type, 615 .co = qemu_coroutine_self(), 616 .serialising = false, 617 .overlap_offset = offset, 618 .overlap_bytes = bytes, 619 }; 620 621 qemu_co_queue_init(&req->wait_queue); 622 623 qemu_co_mutex_lock(&bs->reqs_lock); 624 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 625 qemu_co_mutex_unlock(&bs->reqs_lock); 626 } 627 628 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 629 int64_t offset, int64_t bytes) 630 { 631 bdrv_check_request(offset, bytes, &error_abort); 632 633 /* aaaa bbbb */ 634 if (offset >= req->overlap_offset + req->overlap_bytes) { 635 return false; 636 } 637 /* bbbb aaaa */ 638 if (req->overlap_offset >= offset + bytes) { 639 return false; 640 } 641 return true; 642 } 643 644 /* Called with self->bs->reqs_lock held */ 645 static coroutine_fn BdrvTrackedRequest * 646 bdrv_find_conflicting_request(BdrvTrackedRequest *self) 647 { 648 BdrvTrackedRequest *req; 649 650 QLIST_FOREACH(req, &self->bs->tracked_requests, list) { 651 if (req == self || (!req->serialising && !self->serialising)) { 652 continue; 653 } 654 if (tracked_request_overlaps(req, self->overlap_offset, 655 self->overlap_bytes)) 656 { 657 /* 658 * Hitting this means there was a reentrant request, for 659 * example, a block driver issuing nested requests. This must 660 * never happen since it means deadlock. 661 */ 662 assert(qemu_coroutine_self() != req->co); 663 664 /* 665 * If the request is already (indirectly) waiting for us, or 666 * will wait for us as soon as it wakes up, then just go on 667 * (instead of producing a deadlock in the former case). 668 */ 669 if (!req->waiting_for) { 670 return req; 671 } 672 } 673 } 674 675 return NULL; 676 } 677 678 /* Called with self->bs->reqs_lock held */ 679 static void coroutine_fn 680 bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self) 681 { 682 BdrvTrackedRequest *req; 683 684 while ((req = bdrv_find_conflicting_request(self))) { 685 self->waiting_for = req; 686 qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock); 687 self->waiting_for = NULL; 688 } 689 } 690 691 /* Called with req->bs->reqs_lock held */ 692 static void tracked_request_set_serialising(BdrvTrackedRequest *req, 693 uint64_t align) 694 { 695 int64_t overlap_offset = req->offset & ~(align - 1); 696 int64_t overlap_bytes = 697 ROUND_UP(req->offset + req->bytes, align) - overlap_offset; 698 699 bdrv_check_request(req->offset, req->bytes, &error_abort); 700 701 if (!req->serialising) { 702 qatomic_inc(&req->bs->serialising_in_flight); 703 req->serialising = true; 704 } 705 706 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 707 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 708 } 709 710 /** 711 * Return the tracked request on @bs for the current coroutine, or 712 * NULL if there is none. 713 */ 714 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs) 715 { 716 BdrvTrackedRequest *req; 717 Coroutine *self = qemu_coroutine_self(); 718 IO_CODE(); 719 720 QLIST_FOREACH(req, &bs->tracked_requests, list) { 721 if (req->co == self) { 722 return req; 723 } 724 } 725 726 return NULL; 727 } 728 729 /** 730 * Round a region to cluster boundaries 731 */ 732 void coroutine_fn bdrv_round_to_clusters(BlockDriverState *bs, 733 int64_t offset, int64_t bytes, 734 int64_t *cluster_offset, 735 int64_t *cluster_bytes) 736 { 737 BlockDriverInfo bdi; 738 IO_CODE(); 739 if (bdrv_co_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 740 *cluster_offset = offset; 741 *cluster_bytes = bytes; 742 } else { 743 int64_t c = bdi.cluster_size; 744 *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 745 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 746 } 747 } 748 749 static coroutine_fn int bdrv_get_cluster_size(BlockDriverState *bs) 750 { 751 BlockDriverInfo bdi; 752 int ret; 753 754 ret = bdrv_co_get_info(bs, &bdi); 755 if (ret < 0 || bdi.cluster_size == 0) { 756 return bs->bl.request_alignment; 757 } else { 758 return bdi.cluster_size; 759 } 760 } 761 762 void bdrv_inc_in_flight(BlockDriverState *bs) 763 { 764 IO_CODE(); 765 qatomic_inc(&bs->in_flight); 766 } 767 768 void bdrv_wakeup(BlockDriverState *bs) 769 { 770 IO_CODE(); 771 aio_wait_kick(); 772 } 773 774 void bdrv_dec_in_flight(BlockDriverState *bs) 775 { 776 IO_CODE(); 777 qatomic_dec(&bs->in_flight); 778 bdrv_wakeup(bs); 779 } 780 781 static void coroutine_fn 782 bdrv_wait_serialising_requests(BdrvTrackedRequest *self) 783 { 784 BlockDriverState *bs = self->bs; 785 786 if (!qatomic_read(&bs->serialising_in_flight)) { 787 return; 788 } 789 790 qemu_co_mutex_lock(&bs->reqs_lock); 791 bdrv_wait_serialising_requests_locked(self); 792 qemu_co_mutex_unlock(&bs->reqs_lock); 793 } 794 795 void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req, 796 uint64_t align) 797 { 798 IO_CODE(); 799 800 qemu_co_mutex_lock(&req->bs->reqs_lock); 801 802 tracked_request_set_serialising(req, align); 803 bdrv_wait_serialising_requests_locked(req); 804 805 qemu_co_mutex_unlock(&req->bs->reqs_lock); 806 } 807 808 int bdrv_check_qiov_request(int64_t offset, int64_t bytes, 809 QEMUIOVector *qiov, size_t qiov_offset, 810 Error **errp) 811 { 812 /* 813 * Check generic offset/bytes correctness 814 */ 815 816 if (offset < 0) { 817 error_setg(errp, "offset is negative: %" PRIi64, offset); 818 return -EIO; 819 } 820 821 if (bytes < 0) { 822 error_setg(errp, "bytes is negative: %" PRIi64, bytes); 823 return -EIO; 824 } 825 826 if (bytes > BDRV_MAX_LENGTH) { 827 error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 828 bytes, BDRV_MAX_LENGTH); 829 return -EIO; 830 } 831 832 if (offset > BDRV_MAX_LENGTH) { 833 error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 834 offset, BDRV_MAX_LENGTH); 835 return -EIO; 836 } 837 838 if (offset > BDRV_MAX_LENGTH - bytes) { 839 error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") " 840 "exceeds maximum(%" PRIi64 ")", offset, bytes, 841 BDRV_MAX_LENGTH); 842 return -EIO; 843 } 844 845 if (!qiov) { 846 return 0; 847 } 848 849 /* 850 * Check qiov and qiov_offset 851 */ 852 853 if (qiov_offset > qiov->size) { 854 error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)", 855 qiov_offset, qiov->size); 856 return -EIO; 857 } 858 859 if (bytes > qiov->size - qiov_offset) { 860 error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io " 861 "vector size(%zu)", bytes, qiov_offset, qiov->size); 862 return -EIO; 863 } 864 865 return 0; 866 } 867 868 int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp) 869 { 870 return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp); 871 } 872 873 static int bdrv_check_request32(int64_t offset, int64_t bytes, 874 QEMUIOVector *qiov, size_t qiov_offset) 875 { 876 int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 877 if (ret < 0) { 878 return ret; 879 } 880 881 if (bytes > BDRV_REQUEST_MAX_BYTES) { 882 return -EIO; 883 } 884 885 return 0; 886 } 887 888 /* 889 * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 890 * The operation is sped up by checking the block status and only writing 891 * zeroes to the device if they currently do not return zeroes. Optional 892 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 893 * BDRV_REQ_FUA). 894 * 895 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite(). 896 */ 897 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 898 { 899 int ret; 900 int64_t target_size, bytes, offset = 0; 901 BlockDriverState *bs = child->bs; 902 IO_CODE(); 903 904 target_size = bdrv_getlength(bs); 905 if (target_size < 0) { 906 return target_size; 907 } 908 909 for (;;) { 910 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 911 if (bytes <= 0) { 912 return 0; 913 } 914 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 915 if (ret < 0) { 916 return ret; 917 } 918 if (ret & BDRV_BLOCK_ZERO) { 919 offset += bytes; 920 continue; 921 } 922 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 923 if (ret < 0) { 924 return ret; 925 } 926 offset += bytes; 927 } 928 } 929 930 /* 931 * Writes to the file and ensures that no writes are reordered across this 932 * request (acts as a barrier) 933 * 934 * Returns 0 on success, -errno in error cases. 935 */ 936 int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset, 937 int64_t bytes, const void *buf, 938 BdrvRequestFlags flags) 939 { 940 int ret; 941 IO_CODE(); 942 assert_bdrv_graph_readable(); 943 944 ret = bdrv_co_pwrite(child, offset, bytes, buf, flags); 945 if (ret < 0) { 946 return ret; 947 } 948 949 ret = bdrv_co_flush(child->bs); 950 if (ret < 0) { 951 return ret; 952 } 953 954 return 0; 955 } 956 957 typedef struct CoroutineIOCompletion { 958 Coroutine *coroutine; 959 int ret; 960 } CoroutineIOCompletion; 961 962 static void bdrv_co_io_em_complete(void *opaque, int ret) 963 { 964 CoroutineIOCompletion *co = opaque; 965 966 co->ret = ret; 967 aio_co_wake(co->coroutine); 968 } 969 970 static int coroutine_fn GRAPH_RDLOCK 971 bdrv_driver_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes, 972 QEMUIOVector *qiov, size_t qiov_offset, int flags) 973 { 974 BlockDriver *drv = bs->drv; 975 int64_t sector_num; 976 unsigned int nb_sectors; 977 QEMUIOVector local_qiov; 978 int ret; 979 assert_bdrv_graph_readable(); 980 981 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 982 assert(!(flags & ~bs->supported_read_flags)); 983 984 if (!drv) { 985 return -ENOMEDIUM; 986 } 987 988 if (drv->bdrv_co_preadv_part) { 989 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset, 990 flags); 991 } 992 993 if (qiov_offset > 0 || bytes != qiov->size) { 994 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 995 qiov = &local_qiov; 996 } 997 998 if (drv->bdrv_co_preadv) { 999 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 1000 goto out; 1001 } 1002 1003 if (drv->bdrv_aio_preadv) { 1004 BlockAIOCB *acb; 1005 CoroutineIOCompletion co = { 1006 .coroutine = qemu_coroutine_self(), 1007 }; 1008 1009 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags, 1010 bdrv_co_io_em_complete, &co); 1011 if (acb == NULL) { 1012 ret = -EIO; 1013 goto out; 1014 } else { 1015 qemu_coroutine_yield(); 1016 ret = co.ret; 1017 goto out; 1018 } 1019 } 1020 1021 sector_num = offset >> BDRV_SECTOR_BITS; 1022 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1023 1024 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1025 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1026 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1027 assert(drv->bdrv_co_readv); 1028 1029 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 1030 1031 out: 1032 if (qiov == &local_qiov) { 1033 qemu_iovec_destroy(&local_qiov); 1034 } 1035 1036 return ret; 1037 } 1038 1039 static int coroutine_fn GRAPH_RDLOCK 1040 bdrv_driver_pwritev(BlockDriverState *bs, int64_t offset, int64_t bytes, 1041 QEMUIOVector *qiov, size_t qiov_offset, 1042 BdrvRequestFlags flags) 1043 { 1044 BlockDriver *drv = bs->drv; 1045 bool emulate_fua = false; 1046 int64_t sector_num; 1047 unsigned int nb_sectors; 1048 QEMUIOVector local_qiov; 1049 int ret; 1050 assert_bdrv_graph_readable(); 1051 1052 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1053 1054 if (!drv) { 1055 return -ENOMEDIUM; 1056 } 1057 1058 if ((flags & BDRV_REQ_FUA) && 1059 (~bs->supported_write_flags & BDRV_REQ_FUA)) { 1060 flags &= ~BDRV_REQ_FUA; 1061 emulate_fua = true; 1062 } 1063 1064 flags &= bs->supported_write_flags; 1065 1066 if (drv->bdrv_co_pwritev_part) { 1067 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset, 1068 flags); 1069 goto emulate_flags; 1070 } 1071 1072 if (qiov_offset > 0 || bytes != qiov->size) { 1073 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1074 qiov = &local_qiov; 1075 } 1076 1077 if (drv->bdrv_co_pwritev) { 1078 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags); 1079 goto emulate_flags; 1080 } 1081 1082 if (drv->bdrv_aio_pwritev) { 1083 BlockAIOCB *acb; 1084 CoroutineIOCompletion co = { 1085 .coroutine = qemu_coroutine_self(), 1086 }; 1087 1088 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, flags, 1089 bdrv_co_io_em_complete, &co); 1090 if (acb == NULL) { 1091 ret = -EIO; 1092 } else { 1093 qemu_coroutine_yield(); 1094 ret = co.ret; 1095 } 1096 goto emulate_flags; 1097 } 1098 1099 sector_num = offset >> BDRV_SECTOR_BITS; 1100 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1101 1102 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1103 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1104 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1105 1106 assert(drv->bdrv_co_writev); 1107 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, flags); 1108 1109 emulate_flags: 1110 if (ret == 0 && emulate_fua) { 1111 ret = bdrv_co_flush(bs); 1112 } 1113 1114 if (qiov == &local_qiov) { 1115 qemu_iovec_destroy(&local_qiov); 1116 } 1117 1118 return ret; 1119 } 1120 1121 static int coroutine_fn GRAPH_RDLOCK 1122 bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset, 1123 int64_t bytes, QEMUIOVector *qiov, 1124 size_t qiov_offset) 1125 { 1126 BlockDriver *drv = bs->drv; 1127 QEMUIOVector local_qiov; 1128 int ret; 1129 assert_bdrv_graph_readable(); 1130 1131 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1132 1133 if (!drv) { 1134 return -ENOMEDIUM; 1135 } 1136 1137 if (!block_driver_can_compress(drv)) { 1138 return -ENOTSUP; 1139 } 1140 1141 if (drv->bdrv_co_pwritev_compressed_part) { 1142 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes, 1143 qiov, qiov_offset); 1144 } 1145 1146 if (qiov_offset == 0) { 1147 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 1148 } 1149 1150 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1151 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov); 1152 qemu_iovec_destroy(&local_qiov); 1153 1154 return ret; 1155 } 1156 1157 static int coroutine_fn GRAPH_RDLOCK 1158 bdrv_co_do_copy_on_readv(BdrvChild *child, int64_t offset, int64_t bytes, 1159 QEMUIOVector *qiov, size_t qiov_offset, int flags) 1160 { 1161 BlockDriverState *bs = child->bs; 1162 1163 /* Perform I/O through a temporary buffer so that users who scribble over 1164 * their read buffer while the operation is in progress do not end up 1165 * modifying the image file. This is critical for zero-copy guest I/O 1166 * where anything might happen inside guest memory. 1167 */ 1168 void *bounce_buffer = NULL; 1169 1170 BlockDriver *drv = bs->drv; 1171 int64_t cluster_offset; 1172 int64_t cluster_bytes; 1173 int64_t skip_bytes; 1174 int ret; 1175 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1176 BDRV_REQUEST_MAX_BYTES); 1177 int64_t progress = 0; 1178 bool skip_write; 1179 1180 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1181 1182 if (!drv) { 1183 return -ENOMEDIUM; 1184 } 1185 1186 /* 1187 * Do not write anything when the BDS is inactive. That is not 1188 * allowed, and it would not help. 1189 */ 1190 skip_write = (bs->open_flags & BDRV_O_INACTIVE); 1191 1192 /* FIXME We cannot require callers to have write permissions when all they 1193 * are doing is a read request. If we did things right, write permissions 1194 * would be obtained anyway, but internally by the copy-on-read code. As 1195 * long as it is implemented here rather than in a separate filter driver, 1196 * the copy-on-read code doesn't have its own BdrvChild, however, for which 1197 * it could request permissions. Therefore we have to bypass the permission 1198 * system for the moment. */ 1199 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1200 1201 /* Cover entire cluster so no additional backing file I/O is required when 1202 * allocating cluster in the image file. Note that this value may exceed 1203 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1204 * is one reason we loop rather than doing it all at once. 1205 */ 1206 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 1207 skip_bytes = offset - cluster_offset; 1208 1209 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1210 cluster_offset, cluster_bytes); 1211 1212 while (cluster_bytes) { 1213 int64_t pnum; 1214 1215 if (skip_write) { 1216 ret = 1; /* "already allocated", so nothing will be copied */ 1217 pnum = MIN(cluster_bytes, max_transfer); 1218 } else { 1219 ret = bdrv_is_allocated(bs, cluster_offset, 1220 MIN(cluster_bytes, max_transfer), &pnum); 1221 if (ret < 0) { 1222 /* 1223 * Safe to treat errors in querying allocation as if 1224 * unallocated; we'll probably fail again soon on the 1225 * read, but at least that will set a decent errno. 1226 */ 1227 pnum = MIN(cluster_bytes, max_transfer); 1228 } 1229 1230 /* Stop at EOF if the image ends in the middle of the cluster */ 1231 if (ret == 0 && pnum == 0) { 1232 assert(progress >= bytes); 1233 break; 1234 } 1235 1236 assert(skip_bytes < pnum); 1237 } 1238 1239 if (ret <= 0) { 1240 QEMUIOVector local_qiov; 1241 1242 /* Must copy-on-read; use the bounce buffer */ 1243 pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 1244 if (!bounce_buffer) { 1245 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum); 1246 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER); 1247 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed); 1248 1249 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len); 1250 if (!bounce_buffer) { 1251 ret = -ENOMEM; 1252 goto err; 1253 } 1254 } 1255 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum); 1256 1257 ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1258 &local_qiov, 0, 0); 1259 if (ret < 0) { 1260 goto err; 1261 } 1262 1263 bdrv_co_debug_event(bs, BLKDBG_COR_WRITE); 1264 if (drv->bdrv_co_pwrite_zeroes && 1265 buffer_is_zero(bounce_buffer, pnum)) { 1266 /* FIXME: Should we (perhaps conditionally) be setting 1267 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1268 * that still correctly reads as zero? */ 1269 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 1270 BDRV_REQ_WRITE_UNCHANGED); 1271 } else { 1272 /* This does not change the data on the disk, it is not 1273 * necessary to flush even in cache=writethrough mode. 1274 */ 1275 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 1276 &local_qiov, 0, 1277 BDRV_REQ_WRITE_UNCHANGED); 1278 } 1279 1280 if (ret < 0) { 1281 /* It might be okay to ignore write errors for guest 1282 * requests. If this is a deliberate copy-on-read 1283 * then we don't want to ignore the error. Simply 1284 * report it in all cases. 1285 */ 1286 goto err; 1287 } 1288 1289 if (!(flags & BDRV_REQ_PREFETCH)) { 1290 qemu_iovec_from_buf(qiov, qiov_offset + progress, 1291 bounce_buffer + skip_bytes, 1292 MIN(pnum - skip_bytes, bytes - progress)); 1293 } 1294 } else if (!(flags & BDRV_REQ_PREFETCH)) { 1295 /* Read directly into the destination */ 1296 ret = bdrv_driver_preadv(bs, offset + progress, 1297 MIN(pnum - skip_bytes, bytes - progress), 1298 qiov, qiov_offset + progress, 0); 1299 if (ret < 0) { 1300 goto err; 1301 } 1302 } 1303 1304 cluster_offset += pnum; 1305 cluster_bytes -= pnum; 1306 progress += pnum - skip_bytes; 1307 skip_bytes = 0; 1308 } 1309 ret = 0; 1310 1311 err: 1312 qemu_vfree(bounce_buffer); 1313 return ret; 1314 } 1315 1316 /* 1317 * Forwards an already correctly aligned request to the BlockDriver. This 1318 * handles copy on read, zeroing after EOF, and fragmentation of large 1319 * reads; any other features must be implemented by the caller. 1320 */ 1321 static int coroutine_fn GRAPH_RDLOCK 1322 bdrv_aligned_preadv(BdrvChild *child, BdrvTrackedRequest *req, 1323 int64_t offset, int64_t bytes, int64_t align, 1324 QEMUIOVector *qiov, size_t qiov_offset, int flags) 1325 { 1326 BlockDriverState *bs = child->bs; 1327 int64_t total_bytes, max_bytes; 1328 int ret = 0; 1329 int64_t bytes_remaining = bytes; 1330 int max_transfer; 1331 1332 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1333 assert(is_power_of_2(align)); 1334 assert((offset & (align - 1)) == 0); 1335 assert((bytes & (align - 1)) == 0); 1336 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1337 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1338 align); 1339 1340 /* 1341 * TODO: We would need a per-BDS .supported_read_flags and 1342 * potential fallback support, if we ever implement any read flags 1343 * to pass through to drivers. For now, there aren't any 1344 * passthrough flags except the BDRV_REQ_REGISTERED_BUF optimization hint. 1345 */ 1346 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH | 1347 BDRV_REQ_REGISTERED_BUF))); 1348 1349 /* Handle Copy on Read and associated serialisation */ 1350 if (flags & BDRV_REQ_COPY_ON_READ) { 1351 /* If we touch the same cluster it counts as an overlap. This 1352 * guarantees that allocating writes will be serialized and not race 1353 * with each other for the same cluster. For example, in copy-on-read 1354 * it ensures that the CoR read and write operations are atomic and 1355 * guest writes cannot interleave between them. */ 1356 bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs)); 1357 } else { 1358 bdrv_wait_serialising_requests(req); 1359 } 1360 1361 if (flags & BDRV_REQ_COPY_ON_READ) { 1362 int64_t pnum; 1363 1364 /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */ 1365 flags &= ~BDRV_REQ_COPY_ON_READ; 1366 1367 ret = bdrv_is_allocated(bs, offset, bytes, &pnum); 1368 if (ret < 0) { 1369 goto out; 1370 } 1371 1372 if (!ret || pnum != bytes) { 1373 ret = bdrv_co_do_copy_on_readv(child, offset, bytes, 1374 qiov, qiov_offset, flags); 1375 goto out; 1376 } else if (flags & BDRV_REQ_PREFETCH) { 1377 goto out; 1378 } 1379 } 1380 1381 /* Forward the request to the BlockDriver, possibly fragmenting it */ 1382 total_bytes = bdrv_getlength(bs); 1383 if (total_bytes < 0) { 1384 ret = total_bytes; 1385 goto out; 1386 } 1387 1388 assert(!(flags & ~(bs->supported_read_flags | BDRV_REQ_REGISTERED_BUF))); 1389 1390 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 1391 if (bytes <= max_bytes && bytes <= max_transfer) { 1392 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags); 1393 goto out; 1394 } 1395 1396 while (bytes_remaining) { 1397 int64_t num; 1398 1399 if (max_bytes) { 1400 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 1401 assert(num); 1402 1403 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1404 num, qiov, 1405 qiov_offset + bytes - bytes_remaining, 1406 flags); 1407 max_bytes -= num; 1408 } else { 1409 num = bytes_remaining; 1410 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining, 1411 0, bytes_remaining); 1412 } 1413 if (ret < 0) { 1414 goto out; 1415 } 1416 bytes_remaining -= num; 1417 } 1418 1419 out: 1420 return ret < 0 ? ret : 0; 1421 } 1422 1423 /* 1424 * Request padding 1425 * 1426 * |<---- align ----->| |<----- align ---->| 1427 * |<- head ->|<------------- bytes ------------->|<-- tail -->| 1428 * | | | | | | 1429 * -*----------$-------*-------- ... --------*-----$------------*--- 1430 * | | | | | | 1431 * | offset | | end | 1432 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end) 1433 * [buf ... ) [tail_buf ) 1434 * 1435 * @buf is an aligned allocation needed to store @head and @tail paddings. @head 1436 * is placed at the beginning of @buf and @tail at the @end. 1437 * 1438 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk 1439 * around tail, if tail exists. 1440 * 1441 * @merge_reads is true for small requests, 1442 * if @buf_len == @head + bytes + @tail. In this case it is possible that both 1443 * head and tail exist but @buf_len == align and @tail_buf == @buf. 1444 */ 1445 typedef struct BdrvRequestPadding { 1446 uint8_t *buf; 1447 size_t buf_len; 1448 uint8_t *tail_buf; 1449 size_t head; 1450 size_t tail; 1451 bool merge_reads; 1452 QEMUIOVector local_qiov; 1453 } BdrvRequestPadding; 1454 1455 static bool bdrv_init_padding(BlockDriverState *bs, 1456 int64_t offset, int64_t bytes, 1457 BdrvRequestPadding *pad) 1458 { 1459 int64_t align = bs->bl.request_alignment; 1460 int64_t sum; 1461 1462 bdrv_check_request(offset, bytes, &error_abort); 1463 assert(align <= INT_MAX); /* documented in block/block_int.h */ 1464 assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */ 1465 1466 memset(pad, 0, sizeof(*pad)); 1467 1468 pad->head = offset & (align - 1); 1469 pad->tail = ((offset + bytes) & (align - 1)); 1470 if (pad->tail) { 1471 pad->tail = align - pad->tail; 1472 } 1473 1474 if (!pad->head && !pad->tail) { 1475 return false; 1476 } 1477 1478 assert(bytes); /* Nothing good in aligning zero-length requests */ 1479 1480 sum = pad->head + bytes + pad->tail; 1481 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align; 1482 pad->buf = qemu_blockalign(bs, pad->buf_len); 1483 pad->merge_reads = sum == pad->buf_len; 1484 if (pad->tail) { 1485 pad->tail_buf = pad->buf + pad->buf_len - align; 1486 } 1487 1488 return true; 1489 } 1490 1491 static int coroutine_fn GRAPH_RDLOCK 1492 bdrv_padding_rmw_read(BdrvChild *child, BdrvTrackedRequest *req, 1493 BdrvRequestPadding *pad, bool zero_middle) 1494 { 1495 QEMUIOVector local_qiov; 1496 BlockDriverState *bs = child->bs; 1497 uint64_t align = bs->bl.request_alignment; 1498 int ret; 1499 1500 assert(req->serialising && pad->buf); 1501 1502 if (pad->head || pad->merge_reads) { 1503 int64_t bytes = pad->merge_reads ? pad->buf_len : align; 1504 1505 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes); 1506 1507 if (pad->head) { 1508 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1509 } 1510 if (pad->merge_reads && pad->tail) { 1511 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1512 } 1513 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes, 1514 align, &local_qiov, 0, 0); 1515 if (ret < 0) { 1516 return ret; 1517 } 1518 if (pad->head) { 1519 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1520 } 1521 if (pad->merge_reads && pad->tail) { 1522 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1523 } 1524 1525 if (pad->merge_reads) { 1526 goto zero_mem; 1527 } 1528 } 1529 1530 if (pad->tail) { 1531 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align); 1532 1533 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1534 ret = bdrv_aligned_preadv( 1535 child, req, 1536 req->overlap_offset + req->overlap_bytes - align, 1537 align, align, &local_qiov, 0, 0); 1538 if (ret < 0) { 1539 return ret; 1540 } 1541 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1542 } 1543 1544 zero_mem: 1545 if (zero_middle) { 1546 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail); 1547 } 1548 1549 return 0; 1550 } 1551 1552 static void bdrv_padding_destroy(BdrvRequestPadding *pad) 1553 { 1554 if (pad->buf) { 1555 qemu_vfree(pad->buf); 1556 qemu_iovec_destroy(&pad->local_qiov); 1557 } 1558 memset(pad, 0, sizeof(*pad)); 1559 } 1560 1561 /* 1562 * bdrv_pad_request 1563 * 1564 * Exchange request parameters with padded request if needed. Don't include RMW 1565 * read of padding, bdrv_padding_rmw_read() should be called separately if 1566 * needed. 1567 * 1568 * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out: 1569 * - on function start they represent original request 1570 * - on failure or when padding is not needed they are unchanged 1571 * - on success when padding is needed they represent padded request 1572 */ 1573 static int bdrv_pad_request(BlockDriverState *bs, 1574 QEMUIOVector **qiov, size_t *qiov_offset, 1575 int64_t *offset, int64_t *bytes, 1576 BdrvRequestPadding *pad, bool *padded, 1577 BdrvRequestFlags *flags) 1578 { 1579 int ret; 1580 1581 bdrv_check_qiov_request(*offset, *bytes, *qiov, *qiov_offset, &error_abort); 1582 1583 if (!bdrv_init_padding(bs, *offset, *bytes, pad)) { 1584 if (padded) { 1585 *padded = false; 1586 } 1587 return 0; 1588 } 1589 1590 ret = qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head, 1591 *qiov, *qiov_offset, *bytes, 1592 pad->buf + pad->buf_len - pad->tail, 1593 pad->tail); 1594 if (ret < 0) { 1595 bdrv_padding_destroy(pad); 1596 return ret; 1597 } 1598 *bytes += pad->head + pad->tail; 1599 *offset -= pad->head; 1600 *qiov = &pad->local_qiov; 1601 *qiov_offset = 0; 1602 if (padded) { 1603 *padded = true; 1604 } 1605 if (flags) { 1606 /* Can't use optimization hint with bounce buffer */ 1607 *flags &= ~BDRV_REQ_REGISTERED_BUF; 1608 } 1609 1610 return 0; 1611 } 1612 1613 int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1614 int64_t offset, int64_t bytes, QEMUIOVector *qiov, 1615 BdrvRequestFlags flags) 1616 { 1617 IO_CODE(); 1618 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags); 1619 } 1620 1621 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child, 1622 int64_t offset, int64_t bytes, 1623 QEMUIOVector *qiov, size_t qiov_offset, 1624 BdrvRequestFlags flags) 1625 { 1626 BlockDriverState *bs = child->bs; 1627 BdrvTrackedRequest req; 1628 BdrvRequestPadding pad; 1629 int ret; 1630 IO_CODE(); 1631 1632 trace_bdrv_co_preadv_part(bs, offset, bytes, flags); 1633 1634 if (!bdrv_co_is_inserted(bs)) { 1635 return -ENOMEDIUM; 1636 } 1637 1638 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 1639 if (ret < 0) { 1640 return ret; 1641 } 1642 1643 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 1644 /* 1645 * Aligning zero request is nonsense. Even if driver has special meaning 1646 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 1647 * it to driver due to request_alignment. 1648 * 1649 * Still, no reason to return an error if someone do unaligned 1650 * zero-length read occasionally. 1651 */ 1652 return 0; 1653 } 1654 1655 bdrv_inc_in_flight(bs); 1656 1657 /* Don't do copy-on-read if we read data before write operation */ 1658 if (qatomic_read(&bs->copy_on_read)) { 1659 flags |= BDRV_REQ_COPY_ON_READ; 1660 } 1661 1662 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad, 1663 NULL, &flags); 1664 if (ret < 0) { 1665 goto fail; 1666 } 1667 1668 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1669 ret = bdrv_aligned_preadv(child, &req, offset, bytes, 1670 bs->bl.request_alignment, 1671 qiov, qiov_offset, flags); 1672 tracked_request_end(&req); 1673 bdrv_padding_destroy(&pad); 1674 1675 fail: 1676 bdrv_dec_in_flight(bs); 1677 1678 return ret; 1679 } 1680 1681 static int coroutine_fn GRAPH_RDLOCK 1682 bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int64_t bytes, 1683 BdrvRequestFlags flags) 1684 { 1685 BlockDriver *drv = bs->drv; 1686 QEMUIOVector qiov; 1687 void *buf = NULL; 1688 int ret = 0; 1689 bool need_flush = false; 1690 int head = 0; 1691 int tail = 0; 1692 1693 int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, 1694 INT64_MAX); 1695 int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1696 bs->bl.request_alignment); 1697 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1698 1699 assert_bdrv_graph_readable(); 1700 bdrv_check_request(offset, bytes, &error_abort); 1701 1702 if (!drv) { 1703 return -ENOMEDIUM; 1704 } 1705 1706 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) { 1707 return -ENOTSUP; 1708 } 1709 1710 /* By definition there is no user buffer so this flag doesn't make sense */ 1711 if (flags & BDRV_REQ_REGISTERED_BUF) { 1712 return -EINVAL; 1713 } 1714 1715 /* Invalidate the cached block-status data range if this write overlaps */ 1716 bdrv_bsc_invalidate_range(bs, offset, bytes); 1717 1718 assert(alignment % bs->bl.request_alignment == 0); 1719 head = offset % alignment; 1720 tail = (offset + bytes) % alignment; 1721 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1722 assert(max_write_zeroes >= bs->bl.request_alignment); 1723 1724 while (bytes > 0 && !ret) { 1725 int64_t num = bytes; 1726 1727 /* Align request. Block drivers can expect the "bulk" of the request 1728 * to be aligned, and that unaligned requests do not cross cluster 1729 * boundaries. 1730 */ 1731 if (head) { 1732 /* Make a small request up to the first aligned sector. For 1733 * convenience, limit this request to max_transfer even if 1734 * we don't need to fall back to writes. */ 1735 num = MIN(MIN(bytes, max_transfer), alignment - head); 1736 head = (head + num) % alignment; 1737 assert(num < max_write_zeroes); 1738 } else if (tail && num > alignment) { 1739 /* Shorten the request to the last aligned sector. */ 1740 num -= tail; 1741 } 1742 1743 /* limit request size */ 1744 if (num > max_write_zeroes) { 1745 num = max_write_zeroes; 1746 } 1747 1748 ret = -ENOTSUP; 1749 /* First try the efficient write zeroes operation */ 1750 if (drv->bdrv_co_pwrite_zeroes) { 1751 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1752 flags & bs->supported_zero_flags); 1753 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1754 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1755 need_flush = true; 1756 } 1757 } else { 1758 assert(!bs->supported_zero_flags); 1759 } 1760 1761 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) { 1762 /* Fall back to bounce buffer if write zeroes is unsupported */ 1763 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1764 1765 if ((flags & BDRV_REQ_FUA) && 1766 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1767 /* No need for bdrv_driver_pwrite() to do a fallback 1768 * flush on each chunk; use just one at the end */ 1769 write_flags &= ~BDRV_REQ_FUA; 1770 need_flush = true; 1771 } 1772 num = MIN(num, max_transfer); 1773 if (buf == NULL) { 1774 buf = qemu_try_blockalign0(bs, num); 1775 if (buf == NULL) { 1776 ret = -ENOMEM; 1777 goto fail; 1778 } 1779 } 1780 qemu_iovec_init_buf(&qiov, buf, num); 1781 1782 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags); 1783 1784 /* Keep bounce buffer around if it is big enough for all 1785 * all future requests. 1786 */ 1787 if (num < max_transfer) { 1788 qemu_vfree(buf); 1789 buf = NULL; 1790 } 1791 } 1792 1793 offset += num; 1794 bytes -= num; 1795 } 1796 1797 fail: 1798 if (ret == 0 && need_flush) { 1799 ret = bdrv_co_flush(bs); 1800 } 1801 qemu_vfree(buf); 1802 return ret; 1803 } 1804 1805 static inline int coroutine_fn 1806 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes, 1807 BdrvTrackedRequest *req, int flags) 1808 { 1809 BlockDriverState *bs = child->bs; 1810 1811 bdrv_check_request(offset, bytes, &error_abort); 1812 1813 if (bdrv_is_read_only(bs)) { 1814 return -EPERM; 1815 } 1816 1817 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1818 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1819 assert(!(flags & ~BDRV_REQ_MASK)); 1820 assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING))); 1821 1822 if (flags & BDRV_REQ_SERIALISING) { 1823 QEMU_LOCK_GUARD(&bs->reqs_lock); 1824 1825 tracked_request_set_serialising(req, bdrv_get_cluster_size(bs)); 1826 1827 if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) { 1828 return -EBUSY; 1829 } 1830 1831 bdrv_wait_serialising_requests_locked(req); 1832 } else { 1833 bdrv_wait_serialising_requests(req); 1834 } 1835 1836 assert(req->overlap_offset <= offset); 1837 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1838 assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE || 1839 child->perm & BLK_PERM_RESIZE); 1840 1841 switch (req->type) { 1842 case BDRV_TRACKED_WRITE: 1843 case BDRV_TRACKED_DISCARD: 1844 if (flags & BDRV_REQ_WRITE_UNCHANGED) { 1845 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1846 } else { 1847 assert(child->perm & BLK_PERM_WRITE); 1848 } 1849 bdrv_write_threshold_check_write(bs, offset, bytes); 1850 return 0; 1851 case BDRV_TRACKED_TRUNCATE: 1852 assert(child->perm & BLK_PERM_RESIZE); 1853 return 0; 1854 default: 1855 abort(); 1856 } 1857 } 1858 1859 static inline void coroutine_fn 1860 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes, 1861 BdrvTrackedRequest *req, int ret) 1862 { 1863 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1864 BlockDriverState *bs = child->bs; 1865 1866 bdrv_check_request(offset, bytes, &error_abort); 1867 1868 qatomic_inc(&bs->write_gen); 1869 1870 /* 1871 * Discard cannot extend the image, but in error handling cases, such as 1872 * when reverting a qcow2 cluster allocation, the discarded range can pass 1873 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD 1874 * here. Instead, just skip it, since semantically a discard request 1875 * beyond EOF cannot expand the image anyway. 1876 */ 1877 if (ret == 0 && 1878 (req->type == BDRV_TRACKED_TRUNCATE || 1879 end_sector > bs->total_sectors) && 1880 req->type != BDRV_TRACKED_DISCARD) { 1881 bs->total_sectors = end_sector; 1882 bdrv_parent_cb_resize(bs); 1883 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS); 1884 } 1885 if (req->bytes) { 1886 switch (req->type) { 1887 case BDRV_TRACKED_WRITE: 1888 stat64_max(&bs->wr_highest_offset, offset + bytes); 1889 /* fall through, to set dirty bits */ 1890 case BDRV_TRACKED_DISCARD: 1891 bdrv_set_dirty(bs, offset, bytes); 1892 break; 1893 default: 1894 break; 1895 } 1896 } 1897 } 1898 1899 /* 1900 * Forwards an already correctly aligned write request to the BlockDriver, 1901 * after possibly fragmenting it. 1902 */ 1903 static int coroutine_fn GRAPH_RDLOCK 1904 bdrv_aligned_pwritev(BdrvChild *child, BdrvTrackedRequest *req, 1905 int64_t offset, int64_t bytes, int64_t align, 1906 QEMUIOVector *qiov, size_t qiov_offset, 1907 BdrvRequestFlags flags) 1908 { 1909 BlockDriverState *bs = child->bs; 1910 BlockDriver *drv = bs->drv; 1911 int ret; 1912 1913 int64_t bytes_remaining = bytes; 1914 int max_transfer; 1915 1916 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1917 1918 if (!drv) { 1919 return -ENOMEDIUM; 1920 } 1921 1922 if (bdrv_has_readonly_bitmaps(bs)) { 1923 return -EPERM; 1924 } 1925 1926 assert(is_power_of_2(align)); 1927 assert((offset & (align - 1)) == 0); 1928 assert((bytes & (align - 1)) == 0); 1929 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1930 align); 1931 1932 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags); 1933 1934 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1935 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 1936 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) { 1937 flags |= BDRV_REQ_ZERO_WRITE; 1938 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 1939 flags |= BDRV_REQ_MAY_UNMAP; 1940 } 1941 1942 /* Can't use optimization hint with bufferless zero write */ 1943 flags &= ~BDRV_REQ_REGISTERED_BUF; 1944 } 1945 1946 if (ret < 0) { 1947 /* Do nothing, write notifier decided to fail this request */ 1948 } else if (flags & BDRV_REQ_ZERO_WRITE) { 1949 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_ZERO); 1950 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 1951 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 1952 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, 1953 qiov, qiov_offset); 1954 } else if (bytes <= max_transfer) { 1955 bdrv_co_debug_event(bs, BLKDBG_PWRITEV); 1956 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags); 1957 } else { 1958 bdrv_co_debug_event(bs, BLKDBG_PWRITEV); 1959 while (bytes_remaining) { 1960 int num = MIN(bytes_remaining, max_transfer); 1961 int local_flags = flags; 1962 1963 assert(num); 1964 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 1965 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1966 /* If FUA is going to be emulated by flush, we only 1967 * need to flush on the last iteration */ 1968 local_flags &= ~BDRV_REQ_FUA; 1969 } 1970 1971 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 1972 num, qiov, 1973 qiov_offset + bytes - bytes_remaining, 1974 local_flags); 1975 if (ret < 0) { 1976 break; 1977 } 1978 bytes_remaining -= num; 1979 } 1980 } 1981 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_DONE); 1982 1983 if (ret >= 0) { 1984 ret = 0; 1985 } 1986 bdrv_co_write_req_finish(child, offset, bytes, req, ret); 1987 1988 return ret; 1989 } 1990 1991 static int coroutine_fn GRAPH_RDLOCK 1992 bdrv_co_do_zero_pwritev(BdrvChild *child, int64_t offset, int64_t bytes, 1993 BdrvRequestFlags flags, BdrvTrackedRequest *req) 1994 { 1995 BlockDriverState *bs = child->bs; 1996 QEMUIOVector local_qiov; 1997 uint64_t align = bs->bl.request_alignment; 1998 int ret = 0; 1999 bool padding; 2000 BdrvRequestPadding pad; 2001 2002 /* This flag doesn't make sense for padding or zero writes */ 2003 flags &= ~BDRV_REQ_REGISTERED_BUF; 2004 2005 padding = bdrv_init_padding(bs, offset, bytes, &pad); 2006 if (padding) { 2007 assert(!(flags & BDRV_REQ_NO_WAIT)); 2008 bdrv_make_request_serialising(req, align); 2009 2010 bdrv_padding_rmw_read(child, req, &pad, true); 2011 2012 if (pad.head || pad.merge_reads) { 2013 int64_t aligned_offset = offset & ~(align - 1); 2014 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align; 2015 2016 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes); 2017 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes, 2018 align, &local_qiov, 0, 2019 flags & ~BDRV_REQ_ZERO_WRITE); 2020 if (ret < 0 || pad.merge_reads) { 2021 /* Error or all work is done */ 2022 goto out; 2023 } 2024 offset += write_bytes - pad.head; 2025 bytes -= write_bytes - pad.head; 2026 } 2027 } 2028 2029 assert(!bytes || (offset & (align - 1)) == 0); 2030 if (bytes >= align) { 2031 /* Write the aligned part in the middle. */ 2032 int64_t aligned_bytes = bytes & ~(align - 1); 2033 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 2034 NULL, 0, flags); 2035 if (ret < 0) { 2036 goto out; 2037 } 2038 bytes -= aligned_bytes; 2039 offset += aligned_bytes; 2040 } 2041 2042 assert(!bytes || (offset & (align - 1)) == 0); 2043 if (bytes) { 2044 assert(align == pad.tail + bytes); 2045 2046 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align); 2047 ret = bdrv_aligned_pwritev(child, req, offset, align, align, 2048 &local_qiov, 0, 2049 flags & ~BDRV_REQ_ZERO_WRITE); 2050 } 2051 2052 out: 2053 bdrv_padding_destroy(&pad); 2054 2055 return ret; 2056 } 2057 2058 /* 2059 * Handle a write request in coroutine context 2060 */ 2061 int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 2062 int64_t offset, int64_t bytes, QEMUIOVector *qiov, 2063 BdrvRequestFlags flags) 2064 { 2065 IO_CODE(); 2066 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags); 2067 } 2068 2069 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child, 2070 int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset, 2071 BdrvRequestFlags flags) 2072 { 2073 BlockDriverState *bs = child->bs; 2074 BdrvTrackedRequest req; 2075 uint64_t align = bs->bl.request_alignment; 2076 BdrvRequestPadding pad; 2077 int ret; 2078 bool padded = false; 2079 IO_CODE(); 2080 2081 trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags); 2082 2083 if (!bdrv_co_is_inserted(bs)) { 2084 return -ENOMEDIUM; 2085 } 2086 2087 if (flags & BDRV_REQ_ZERO_WRITE) { 2088 ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 2089 } else { 2090 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 2091 } 2092 if (ret < 0) { 2093 return ret; 2094 } 2095 2096 /* If the request is misaligned then we can't make it efficient */ 2097 if ((flags & BDRV_REQ_NO_FALLBACK) && 2098 !QEMU_IS_ALIGNED(offset | bytes, align)) 2099 { 2100 return -ENOTSUP; 2101 } 2102 2103 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 2104 /* 2105 * Aligning zero request is nonsense. Even if driver has special meaning 2106 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 2107 * it to driver due to request_alignment. 2108 * 2109 * Still, no reason to return an error if someone do unaligned 2110 * zero-length write occasionally. 2111 */ 2112 return 0; 2113 } 2114 2115 if (!(flags & BDRV_REQ_ZERO_WRITE)) { 2116 /* 2117 * Pad request for following read-modify-write cycle. 2118 * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do 2119 * alignment only if there is no ZERO flag. 2120 */ 2121 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad, 2122 &padded, &flags); 2123 if (ret < 0) { 2124 return ret; 2125 } 2126 } 2127 2128 bdrv_inc_in_flight(bs); 2129 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 2130 2131 if (flags & BDRV_REQ_ZERO_WRITE) { 2132 assert(!padded); 2133 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 2134 goto out; 2135 } 2136 2137 if (padded) { 2138 /* 2139 * Request was unaligned to request_alignment and therefore 2140 * padded. We are going to do read-modify-write, and must 2141 * serialize the request to prevent interactions of the 2142 * widened region with other transactions. 2143 */ 2144 assert(!(flags & BDRV_REQ_NO_WAIT)); 2145 bdrv_make_request_serialising(&req, align); 2146 bdrv_padding_rmw_read(child, &req, &pad, false); 2147 } 2148 2149 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 2150 qiov, qiov_offset, flags); 2151 2152 bdrv_padding_destroy(&pad); 2153 2154 out: 2155 tracked_request_end(&req); 2156 bdrv_dec_in_flight(bs); 2157 2158 return ret; 2159 } 2160 2161 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 2162 int64_t bytes, BdrvRequestFlags flags) 2163 { 2164 IO_CODE(); 2165 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 2166 assert_bdrv_graph_readable(); 2167 2168 if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 2169 flags &= ~BDRV_REQ_MAY_UNMAP; 2170 } 2171 2172 return bdrv_co_pwritev(child, offset, bytes, NULL, 2173 BDRV_REQ_ZERO_WRITE | flags); 2174 } 2175 2176 /* 2177 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 2178 */ 2179 int bdrv_flush_all(void) 2180 { 2181 BdrvNextIterator it; 2182 BlockDriverState *bs = NULL; 2183 int result = 0; 2184 2185 GLOBAL_STATE_CODE(); 2186 2187 /* 2188 * bdrv queue is managed by record/replay, 2189 * creating new flush request for stopping 2190 * the VM may break the determinism 2191 */ 2192 if (replay_events_enabled()) { 2193 return result; 2194 } 2195 2196 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 2197 AioContext *aio_context = bdrv_get_aio_context(bs); 2198 int ret; 2199 2200 aio_context_acquire(aio_context); 2201 ret = bdrv_flush(bs); 2202 if (ret < 0 && !result) { 2203 result = ret; 2204 } 2205 aio_context_release(aio_context); 2206 } 2207 2208 return result; 2209 } 2210 2211 /* 2212 * Returns the allocation status of the specified sectors. 2213 * Drivers not implementing the functionality are assumed to not support 2214 * backing files, hence all their sectors are reported as allocated. 2215 * 2216 * If 'want_zero' is true, the caller is querying for mapping 2217 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 2218 * _ZERO where possible; otherwise, the result favors larger 'pnum', 2219 * with a focus on accurate BDRV_BLOCK_ALLOCATED. 2220 * 2221 * If 'offset' is beyond the end of the disk image the return value is 2222 * BDRV_BLOCK_EOF and 'pnum' is set to 0. 2223 * 2224 * 'bytes' is the max value 'pnum' should be set to. If bytes goes 2225 * beyond the end of the disk image it will be clamped; if 'pnum' is set to 2226 * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 2227 * 2228 * 'pnum' is set to the number of bytes (including and immediately 2229 * following the specified offset) that are easily known to be in the 2230 * same allocated/unallocated state. Note that a second call starting 2231 * at the original offset plus returned pnum may have the same status. 2232 * The returned value is non-zero on success except at end-of-file. 2233 * 2234 * Returns negative errno on failure. Otherwise, if the 2235 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 2236 * set to the host mapping and BDS corresponding to the guest offset. 2237 */ 2238 static int coroutine_fn GRAPH_RDLOCK 2239 bdrv_co_block_status(BlockDriverState *bs, bool want_zero, 2240 int64_t offset, int64_t bytes, 2241 int64_t *pnum, int64_t *map, BlockDriverState **file) 2242 { 2243 int64_t total_size; 2244 int64_t n; /* bytes */ 2245 int ret; 2246 int64_t local_map = 0; 2247 BlockDriverState *local_file = NULL; 2248 int64_t aligned_offset, aligned_bytes; 2249 uint32_t align; 2250 bool has_filtered_child; 2251 2252 assert(pnum); 2253 assert_bdrv_graph_readable(); 2254 *pnum = 0; 2255 total_size = bdrv_getlength(bs); 2256 if (total_size < 0) { 2257 ret = total_size; 2258 goto early_out; 2259 } 2260 2261 if (offset >= total_size) { 2262 ret = BDRV_BLOCK_EOF; 2263 goto early_out; 2264 } 2265 if (!bytes) { 2266 ret = 0; 2267 goto early_out; 2268 } 2269 2270 n = total_size - offset; 2271 if (n < bytes) { 2272 bytes = n; 2273 } 2274 2275 /* Must be non-NULL or bdrv_getlength() would have failed */ 2276 assert(bs->drv); 2277 has_filtered_child = bdrv_filter_child(bs); 2278 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) { 2279 *pnum = bytes; 2280 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 2281 if (offset + bytes == total_size) { 2282 ret |= BDRV_BLOCK_EOF; 2283 } 2284 if (bs->drv->protocol_name) { 2285 ret |= BDRV_BLOCK_OFFSET_VALID; 2286 local_map = offset; 2287 local_file = bs; 2288 } 2289 goto early_out; 2290 } 2291 2292 bdrv_inc_in_flight(bs); 2293 2294 /* Round out to request_alignment boundaries */ 2295 align = bs->bl.request_alignment; 2296 aligned_offset = QEMU_ALIGN_DOWN(offset, align); 2297 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 2298 2299 if (bs->drv->bdrv_co_block_status) { 2300 /* 2301 * Use the block-status cache only for protocol nodes: Format 2302 * drivers are generally quick to inquire the status, but protocol 2303 * drivers often need to get information from outside of qemu, so 2304 * we do not have control over the actual implementation. There 2305 * have been cases where inquiring the status took an unreasonably 2306 * long time, and we can do nothing in qemu to fix it. 2307 * This is especially problematic for images with large data areas, 2308 * because finding the few holes in them and giving them special 2309 * treatment does not gain much performance. Therefore, we try to 2310 * cache the last-identified data region. 2311 * 2312 * Second, limiting ourselves to protocol nodes allows us to assume 2313 * the block status for data regions to be DATA | OFFSET_VALID, and 2314 * that the host offset is the same as the guest offset. 2315 * 2316 * Note that it is possible that external writers zero parts of 2317 * the cached regions without the cache being invalidated, and so 2318 * we may report zeroes as data. This is not catastrophic, 2319 * however, because reporting zeroes as data is fine. 2320 */ 2321 if (QLIST_EMPTY(&bs->children) && 2322 bdrv_bsc_is_data(bs, aligned_offset, pnum)) 2323 { 2324 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID; 2325 local_file = bs; 2326 local_map = aligned_offset; 2327 } else { 2328 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 2329 aligned_bytes, pnum, &local_map, 2330 &local_file); 2331 2332 /* 2333 * Note that checking QLIST_EMPTY(&bs->children) is also done when 2334 * the cache is queried above. Technically, we do not need to check 2335 * it here; the worst that can happen is that we fill the cache for 2336 * non-protocol nodes, and then it is never used. However, filling 2337 * the cache requires an RCU update, so double check here to avoid 2338 * such an update if possible. 2339 * 2340 * Check want_zero, because we only want to update the cache when we 2341 * have accurate information about what is zero and what is data. 2342 */ 2343 if (want_zero && 2344 ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) && 2345 QLIST_EMPTY(&bs->children)) 2346 { 2347 /* 2348 * When a protocol driver reports BLOCK_OFFSET_VALID, the 2349 * returned local_map value must be the same as the offset we 2350 * have passed (aligned_offset), and local_bs must be the node 2351 * itself. 2352 * Assert this, because we follow this rule when reading from 2353 * the cache (see the `local_file = bs` and 2354 * `local_map = aligned_offset` assignments above), and the 2355 * result the cache delivers must be the same as the driver 2356 * would deliver. 2357 */ 2358 assert(local_file == bs); 2359 assert(local_map == aligned_offset); 2360 bdrv_bsc_fill(bs, aligned_offset, *pnum); 2361 } 2362 } 2363 } else { 2364 /* Default code for filters */ 2365 2366 local_file = bdrv_filter_bs(bs); 2367 assert(local_file); 2368 2369 *pnum = aligned_bytes; 2370 local_map = aligned_offset; 2371 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2372 } 2373 if (ret < 0) { 2374 *pnum = 0; 2375 goto out; 2376 } 2377 2378 /* 2379 * The driver's result must be a non-zero multiple of request_alignment. 2380 * Clamp pnum and adjust map to original request. 2381 */ 2382 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2383 align > offset - aligned_offset); 2384 if (ret & BDRV_BLOCK_RECURSE) { 2385 assert(ret & BDRV_BLOCK_DATA); 2386 assert(ret & BDRV_BLOCK_OFFSET_VALID); 2387 assert(!(ret & BDRV_BLOCK_ZERO)); 2388 } 2389 2390 *pnum -= offset - aligned_offset; 2391 if (*pnum > bytes) { 2392 *pnum = bytes; 2393 } 2394 if (ret & BDRV_BLOCK_OFFSET_VALID) { 2395 local_map += offset - aligned_offset; 2396 } 2397 2398 if (ret & BDRV_BLOCK_RAW) { 2399 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 2400 ret = bdrv_co_block_status(local_file, want_zero, local_map, 2401 *pnum, pnum, &local_map, &local_file); 2402 goto out; 2403 } 2404 2405 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 2406 ret |= BDRV_BLOCK_ALLOCATED; 2407 } else if (bs->drv->supports_backing) { 2408 BlockDriverState *cow_bs = bdrv_cow_bs(bs); 2409 2410 if (!cow_bs) { 2411 ret |= BDRV_BLOCK_ZERO; 2412 } else if (want_zero) { 2413 int64_t size2 = bdrv_getlength(cow_bs); 2414 2415 if (size2 >= 0 && offset >= size2) { 2416 ret |= BDRV_BLOCK_ZERO; 2417 } 2418 } 2419 } 2420 2421 if (want_zero && ret & BDRV_BLOCK_RECURSE && 2422 local_file && local_file != bs && 2423 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 2424 (ret & BDRV_BLOCK_OFFSET_VALID)) { 2425 int64_t file_pnum; 2426 int ret2; 2427 2428 ret2 = bdrv_co_block_status(local_file, want_zero, local_map, 2429 *pnum, &file_pnum, NULL, NULL); 2430 if (ret2 >= 0) { 2431 /* Ignore errors. This is just providing extra information, it 2432 * is useful but not necessary. 2433 */ 2434 if (ret2 & BDRV_BLOCK_EOF && 2435 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2436 /* 2437 * It is valid for the format block driver to read 2438 * beyond the end of the underlying file's current 2439 * size; such areas read as zero. 2440 */ 2441 ret |= BDRV_BLOCK_ZERO; 2442 } else { 2443 /* Limit request to the range reported by the protocol driver */ 2444 *pnum = file_pnum; 2445 ret |= (ret2 & BDRV_BLOCK_ZERO); 2446 } 2447 } 2448 } 2449 2450 out: 2451 bdrv_dec_in_flight(bs); 2452 if (ret >= 0 && offset + *pnum == total_size) { 2453 ret |= BDRV_BLOCK_EOF; 2454 } 2455 early_out: 2456 if (file) { 2457 *file = local_file; 2458 } 2459 if (map) { 2460 *map = local_map; 2461 } 2462 return ret; 2463 } 2464 2465 int coroutine_fn 2466 bdrv_co_common_block_status_above(BlockDriverState *bs, 2467 BlockDriverState *base, 2468 bool include_base, 2469 bool want_zero, 2470 int64_t offset, 2471 int64_t bytes, 2472 int64_t *pnum, 2473 int64_t *map, 2474 BlockDriverState **file, 2475 int *depth) 2476 { 2477 int ret; 2478 BlockDriverState *p; 2479 int64_t eof = 0; 2480 int dummy; 2481 IO_CODE(); 2482 2483 assert(!include_base || base); /* Can't include NULL base */ 2484 assert_bdrv_graph_readable(); 2485 2486 if (!depth) { 2487 depth = &dummy; 2488 } 2489 *depth = 0; 2490 2491 if (!include_base && bs == base) { 2492 *pnum = bytes; 2493 return 0; 2494 } 2495 2496 ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file); 2497 ++*depth; 2498 if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) { 2499 return ret; 2500 } 2501 2502 if (ret & BDRV_BLOCK_EOF) { 2503 eof = offset + *pnum; 2504 } 2505 2506 assert(*pnum <= bytes); 2507 bytes = *pnum; 2508 2509 for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base; 2510 p = bdrv_filter_or_cow_bs(p)) 2511 { 2512 ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map, 2513 file); 2514 ++*depth; 2515 if (ret < 0) { 2516 return ret; 2517 } 2518 if (*pnum == 0) { 2519 /* 2520 * The top layer deferred to this layer, and because this layer is 2521 * short, any zeroes that we synthesize beyond EOF behave as if they 2522 * were allocated at this layer. 2523 * 2524 * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be 2525 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2526 * below. 2527 */ 2528 assert(ret & BDRV_BLOCK_EOF); 2529 *pnum = bytes; 2530 if (file) { 2531 *file = p; 2532 } 2533 ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED; 2534 break; 2535 } 2536 if (ret & BDRV_BLOCK_ALLOCATED) { 2537 /* 2538 * We've found the node and the status, we must break. 2539 * 2540 * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be 2541 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2542 * below. 2543 */ 2544 ret &= ~BDRV_BLOCK_EOF; 2545 break; 2546 } 2547 2548 if (p == base) { 2549 assert(include_base); 2550 break; 2551 } 2552 2553 /* 2554 * OK, [offset, offset + *pnum) region is unallocated on this layer, 2555 * let's continue the diving. 2556 */ 2557 assert(*pnum <= bytes); 2558 bytes = *pnum; 2559 } 2560 2561 if (offset + *pnum == eof) { 2562 ret |= BDRV_BLOCK_EOF; 2563 } 2564 2565 return ret; 2566 } 2567 2568 int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs, 2569 BlockDriverState *base, 2570 int64_t offset, int64_t bytes, 2571 int64_t *pnum, int64_t *map, 2572 BlockDriverState **file) 2573 { 2574 IO_CODE(); 2575 return bdrv_co_common_block_status_above(bs, base, false, true, offset, 2576 bytes, pnum, map, file, NULL); 2577 } 2578 2579 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base, 2580 int64_t offset, int64_t bytes, int64_t *pnum, 2581 int64_t *map, BlockDriverState **file) 2582 { 2583 IO_CODE(); 2584 return bdrv_common_block_status_above(bs, base, false, true, offset, bytes, 2585 pnum, map, file, NULL); 2586 } 2587 2588 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2589 int64_t *pnum, int64_t *map, BlockDriverState **file) 2590 { 2591 IO_CODE(); 2592 return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs), 2593 offset, bytes, pnum, map, file); 2594 } 2595 2596 /* 2597 * Check @bs (and its backing chain) to see if the range defined 2598 * by @offset and @bytes is known to read as zeroes. 2599 * Return 1 if that is the case, 0 otherwise and -errno on error. 2600 * This test is meant to be fast rather than accurate so returning 0 2601 * does not guarantee non-zero data. 2602 */ 2603 int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset, 2604 int64_t bytes) 2605 { 2606 int ret; 2607 int64_t pnum = bytes; 2608 IO_CODE(); 2609 2610 if (!bytes) { 2611 return 1; 2612 } 2613 2614 ret = bdrv_co_common_block_status_above(bs, NULL, false, false, offset, 2615 bytes, &pnum, NULL, NULL, NULL); 2616 2617 if (ret < 0) { 2618 return ret; 2619 } 2620 2621 return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO); 2622 } 2623 2624 int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t offset, 2625 int64_t bytes, int64_t *pnum) 2626 { 2627 int ret; 2628 int64_t dummy; 2629 IO_CODE(); 2630 2631 ret = bdrv_co_common_block_status_above(bs, bs, true, false, offset, 2632 bytes, pnum ? pnum : &dummy, NULL, 2633 NULL, NULL); 2634 if (ret < 0) { 2635 return ret; 2636 } 2637 return !!(ret & BDRV_BLOCK_ALLOCATED); 2638 } 2639 2640 int bdrv_is_allocated(BlockDriverState *bs, int64_t offset, int64_t bytes, 2641 int64_t *pnum) 2642 { 2643 int ret; 2644 int64_t dummy; 2645 IO_CODE(); 2646 2647 ret = bdrv_common_block_status_above(bs, bs, true, false, offset, 2648 bytes, pnum ? pnum : &dummy, NULL, 2649 NULL, NULL); 2650 if (ret < 0) { 2651 return ret; 2652 } 2653 return !!(ret & BDRV_BLOCK_ALLOCATED); 2654 } 2655 2656 /* See bdrv_is_allocated_above for documentation */ 2657 int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top, 2658 BlockDriverState *base, 2659 bool include_base, int64_t offset, 2660 int64_t bytes, int64_t *pnum) 2661 { 2662 int depth; 2663 int ret; 2664 IO_CODE(); 2665 2666 ret = bdrv_co_common_block_status_above(top, base, include_base, false, 2667 offset, bytes, pnum, NULL, NULL, 2668 &depth); 2669 if (ret < 0) { 2670 return ret; 2671 } 2672 2673 if (ret & BDRV_BLOCK_ALLOCATED) { 2674 return depth; 2675 } 2676 return 0; 2677 } 2678 2679 /* 2680 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 2681 * 2682 * Return a positive depth if (a prefix of) the given range is allocated 2683 * in any image between BASE and TOP (BASE is only included if include_base 2684 * is set). Depth 1 is TOP, 2 is the first backing layer, and so forth. 2685 * BASE can be NULL to check if the given offset is allocated in any 2686 * image of the chain. Return 0 otherwise, or negative errno on 2687 * failure. 2688 * 2689 * 'pnum' is set to the number of bytes (including and immediately 2690 * following the specified offset) that are known to be in the same 2691 * allocated/unallocated state. Note that a subsequent call starting 2692 * at 'offset + *pnum' may return the same allocation status (in other 2693 * words, the result is not necessarily the maximum possible range); 2694 * but 'pnum' will only be 0 when end of file is reached. 2695 */ 2696 int bdrv_is_allocated_above(BlockDriverState *top, 2697 BlockDriverState *base, 2698 bool include_base, int64_t offset, 2699 int64_t bytes, int64_t *pnum) 2700 { 2701 int depth; 2702 int ret; 2703 IO_CODE(); 2704 2705 ret = bdrv_common_block_status_above(top, base, include_base, false, 2706 offset, bytes, pnum, NULL, NULL, 2707 &depth); 2708 if (ret < 0) { 2709 return ret; 2710 } 2711 2712 if (ret & BDRV_BLOCK_ALLOCATED) { 2713 return depth; 2714 } 2715 return 0; 2716 } 2717 2718 int coroutine_fn 2719 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2720 { 2721 BlockDriver *drv = bs->drv; 2722 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2723 int ret; 2724 IO_CODE(); 2725 assert_bdrv_graph_readable(); 2726 2727 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2728 if (ret < 0) { 2729 return ret; 2730 } 2731 2732 if (!drv) { 2733 return -ENOMEDIUM; 2734 } 2735 2736 bdrv_inc_in_flight(bs); 2737 2738 if (drv->bdrv_co_load_vmstate) { 2739 ret = drv->bdrv_co_load_vmstate(bs, qiov, pos); 2740 } else if (child_bs) { 2741 ret = bdrv_co_readv_vmstate(child_bs, qiov, pos); 2742 } else { 2743 ret = -ENOTSUP; 2744 } 2745 2746 bdrv_dec_in_flight(bs); 2747 2748 return ret; 2749 } 2750 2751 int coroutine_fn 2752 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2753 { 2754 BlockDriver *drv = bs->drv; 2755 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2756 int ret; 2757 IO_CODE(); 2758 assert_bdrv_graph_readable(); 2759 2760 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2761 if (ret < 0) { 2762 return ret; 2763 } 2764 2765 if (!drv) { 2766 return -ENOMEDIUM; 2767 } 2768 2769 bdrv_inc_in_flight(bs); 2770 2771 if (drv->bdrv_co_save_vmstate) { 2772 ret = drv->bdrv_co_save_vmstate(bs, qiov, pos); 2773 } else if (child_bs) { 2774 ret = bdrv_co_writev_vmstate(child_bs, qiov, pos); 2775 } else { 2776 ret = -ENOTSUP; 2777 } 2778 2779 bdrv_dec_in_flight(bs); 2780 2781 return ret; 2782 } 2783 2784 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 2785 int64_t pos, int size) 2786 { 2787 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2788 int ret = bdrv_writev_vmstate(bs, &qiov, pos); 2789 IO_CODE(); 2790 2791 return ret < 0 ? ret : size; 2792 } 2793 2794 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 2795 int64_t pos, int size) 2796 { 2797 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2798 int ret = bdrv_readv_vmstate(bs, &qiov, pos); 2799 IO_CODE(); 2800 2801 return ret < 0 ? ret : size; 2802 } 2803 2804 /**************************************************************/ 2805 /* async I/Os */ 2806 2807 void bdrv_aio_cancel(BlockAIOCB *acb) 2808 { 2809 IO_CODE(); 2810 qemu_aio_ref(acb); 2811 bdrv_aio_cancel_async(acb); 2812 while (acb->refcnt > 1) { 2813 if (acb->aiocb_info->get_aio_context) { 2814 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 2815 } else if (acb->bs) { 2816 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 2817 * assert that we're not using an I/O thread. Thread-safe 2818 * code should use bdrv_aio_cancel_async exclusively. 2819 */ 2820 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 2821 aio_poll(bdrv_get_aio_context(acb->bs), true); 2822 } else { 2823 abort(); 2824 } 2825 } 2826 qemu_aio_unref(acb); 2827 } 2828 2829 /* Async version of aio cancel. The caller is not blocked if the acb implements 2830 * cancel_async, otherwise we do nothing and let the request normally complete. 2831 * In either case the completion callback must be called. */ 2832 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2833 { 2834 IO_CODE(); 2835 if (acb->aiocb_info->cancel_async) { 2836 acb->aiocb_info->cancel_async(acb); 2837 } 2838 } 2839 2840 /**************************************************************/ 2841 /* Coroutine block device emulation */ 2842 2843 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2844 { 2845 BdrvChild *primary_child = bdrv_primary_child(bs); 2846 BdrvChild *child; 2847 int current_gen; 2848 int ret = 0; 2849 IO_CODE(); 2850 2851 assert_bdrv_graph_readable(); 2852 bdrv_inc_in_flight(bs); 2853 2854 if (!bdrv_co_is_inserted(bs) || bdrv_is_read_only(bs) || 2855 bdrv_is_sg(bs)) { 2856 goto early_exit; 2857 } 2858 2859 qemu_co_mutex_lock(&bs->reqs_lock); 2860 current_gen = qatomic_read(&bs->write_gen); 2861 2862 /* Wait until any previous flushes are completed */ 2863 while (bs->active_flush_req) { 2864 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 2865 } 2866 2867 /* Flushes reach this point in nondecreasing current_gen order. */ 2868 bs->active_flush_req = true; 2869 qemu_co_mutex_unlock(&bs->reqs_lock); 2870 2871 /* Write back all layers by calling one driver function */ 2872 if (bs->drv->bdrv_co_flush) { 2873 ret = bs->drv->bdrv_co_flush(bs); 2874 goto out; 2875 } 2876 2877 /* Write back cached data to the OS even with cache=unsafe */ 2878 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS); 2879 if (bs->drv->bdrv_co_flush_to_os) { 2880 ret = bs->drv->bdrv_co_flush_to_os(bs); 2881 if (ret < 0) { 2882 goto out; 2883 } 2884 } 2885 2886 /* But don't actually force it to the disk with cache=unsafe */ 2887 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2888 goto flush_children; 2889 } 2890 2891 /* Check if we really need to flush anything */ 2892 if (bs->flushed_gen == current_gen) { 2893 goto flush_children; 2894 } 2895 2896 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK); 2897 if (!bs->drv) { 2898 /* bs->drv->bdrv_co_flush() might have ejected the BDS 2899 * (even in case of apparent success) */ 2900 ret = -ENOMEDIUM; 2901 goto out; 2902 } 2903 if (bs->drv->bdrv_co_flush_to_disk) { 2904 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2905 } else if (bs->drv->bdrv_aio_flush) { 2906 BlockAIOCB *acb; 2907 CoroutineIOCompletion co = { 2908 .coroutine = qemu_coroutine_self(), 2909 }; 2910 2911 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 2912 if (acb == NULL) { 2913 ret = -EIO; 2914 } else { 2915 qemu_coroutine_yield(); 2916 ret = co.ret; 2917 } 2918 } else { 2919 /* 2920 * Some block drivers always operate in either writethrough or unsafe 2921 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 2922 * know how the server works (because the behaviour is hardcoded or 2923 * depends on server-side configuration), so we can't ensure that 2924 * everything is safe on disk. Returning an error doesn't work because 2925 * that would break guests even if the server operates in writethrough 2926 * mode. 2927 * 2928 * Let's hope the user knows what he's doing. 2929 */ 2930 ret = 0; 2931 } 2932 2933 if (ret < 0) { 2934 goto out; 2935 } 2936 2937 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 2938 * in the case of cache=unsafe, so there are no useless flushes. 2939 */ 2940 flush_children: 2941 ret = 0; 2942 QLIST_FOREACH(child, &bs->children, next) { 2943 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) { 2944 int this_child_ret = bdrv_co_flush(child->bs); 2945 if (!ret) { 2946 ret = this_child_ret; 2947 } 2948 } 2949 } 2950 2951 out: 2952 /* Notify any pending flushes that we have completed */ 2953 if (ret == 0) { 2954 bs->flushed_gen = current_gen; 2955 } 2956 2957 qemu_co_mutex_lock(&bs->reqs_lock); 2958 bs->active_flush_req = false; 2959 /* Return value is ignored - it's ok if wait queue is empty */ 2960 qemu_co_queue_next(&bs->flush_queue); 2961 qemu_co_mutex_unlock(&bs->reqs_lock); 2962 2963 early_exit: 2964 bdrv_dec_in_flight(bs); 2965 return ret; 2966 } 2967 2968 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset, 2969 int64_t bytes) 2970 { 2971 BdrvTrackedRequest req; 2972 int ret; 2973 int64_t max_pdiscard; 2974 int head, tail, align; 2975 BlockDriverState *bs = child->bs; 2976 IO_CODE(); 2977 assert_bdrv_graph_readable(); 2978 2979 if (!bs || !bs->drv || !bdrv_co_is_inserted(bs)) { 2980 return -ENOMEDIUM; 2981 } 2982 2983 if (bdrv_has_readonly_bitmaps(bs)) { 2984 return -EPERM; 2985 } 2986 2987 ret = bdrv_check_request(offset, bytes, NULL); 2988 if (ret < 0) { 2989 return ret; 2990 } 2991 2992 /* Do nothing if disabled. */ 2993 if (!(bs->open_flags & BDRV_O_UNMAP)) { 2994 return 0; 2995 } 2996 2997 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 2998 return 0; 2999 } 3000 3001 /* Invalidate the cached block-status data range if this discard overlaps */ 3002 bdrv_bsc_invalidate_range(bs, offset, bytes); 3003 3004 /* Discard is advisory, but some devices track and coalesce 3005 * unaligned requests, so we must pass everything down rather than 3006 * round here. Still, most devices will just silently ignore 3007 * unaligned requests (by returning -ENOTSUP), so we must fragment 3008 * the request accordingly. */ 3009 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 3010 assert(align % bs->bl.request_alignment == 0); 3011 head = offset % align; 3012 tail = (offset + bytes) % align; 3013 3014 bdrv_inc_in_flight(bs); 3015 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 3016 3017 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0); 3018 if (ret < 0) { 3019 goto out; 3020 } 3021 3022 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX), 3023 align); 3024 assert(max_pdiscard >= bs->bl.request_alignment); 3025 3026 while (bytes > 0) { 3027 int64_t num = bytes; 3028 3029 if (head) { 3030 /* Make small requests to get to alignment boundaries. */ 3031 num = MIN(bytes, align - head); 3032 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 3033 num %= bs->bl.request_alignment; 3034 } 3035 head = (head + num) % align; 3036 assert(num < max_pdiscard); 3037 } else if (tail) { 3038 if (num > align) { 3039 /* Shorten the request to the last aligned cluster. */ 3040 num -= tail; 3041 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 3042 tail > bs->bl.request_alignment) { 3043 tail %= bs->bl.request_alignment; 3044 num -= tail; 3045 } 3046 } 3047 /* limit request size */ 3048 if (num > max_pdiscard) { 3049 num = max_pdiscard; 3050 } 3051 3052 if (!bs->drv) { 3053 ret = -ENOMEDIUM; 3054 goto out; 3055 } 3056 if (bs->drv->bdrv_co_pdiscard) { 3057 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 3058 } else { 3059 BlockAIOCB *acb; 3060 CoroutineIOCompletion co = { 3061 .coroutine = qemu_coroutine_self(), 3062 }; 3063 3064 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 3065 bdrv_co_io_em_complete, &co); 3066 if (acb == NULL) { 3067 ret = -EIO; 3068 goto out; 3069 } else { 3070 qemu_coroutine_yield(); 3071 ret = co.ret; 3072 } 3073 } 3074 if (ret && ret != -ENOTSUP) { 3075 goto out; 3076 } 3077 3078 offset += num; 3079 bytes -= num; 3080 } 3081 ret = 0; 3082 out: 3083 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret); 3084 tracked_request_end(&req); 3085 bdrv_dec_in_flight(bs); 3086 return ret; 3087 } 3088 3089 int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 3090 { 3091 BlockDriver *drv = bs->drv; 3092 CoroutineIOCompletion co = { 3093 .coroutine = qemu_coroutine_self(), 3094 }; 3095 BlockAIOCB *acb; 3096 IO_CODE(); 3097 assert_bdrv_graph_readable(); 3098 3099 bdrv_inc_in_flight(bs); 3100 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 3101 co.ret = -ENOTSUP; 3102 goto out; 3103 } 3104 3105 if (drv->bdrv_co_ioctl) { 3106 co.ret = drv->bdrv_co_ioctl(bs, req, buf); 3107 } else { 3108 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 3109 if (!acb) { 3110 co.ret = -ENOTSUP; 3111 goto out; 3112 } 3113 qemu_coroutine_yield(); 3114 } 3115 out: 3116 bdrv_dec_in_flight(bs); 3117 return co.ret; 3118 } 3119 3120 void *qemu_blockalign(BlockDriverState *bs, size_t size) 3121 { 3122 IO_CODE(); 3123 return qemu_memalign(bdrv_opt_mem_align(bs), size); 3124 } 3125 3126 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 3127 { 3128 IO_CODE(); 3129 return memset(qemu_blockalign(bs, size), 0, size); 3130 } 3131 3132 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 3133 { 3134 size_t align = bdrv_opt_mem_align(bs); 3135 IO_CODE(); 3136 3137 /* Ensure that NULL is never returned on success */ 3138 assert(align > 0); 3139 if (size == 0) { 3140 size = align; 3141 } 3142 3143 return qemu_try_memalign(align, size); 3144 } 3145 3146 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 3147 { 3148 void *mem = qemu_try_blockalign(bs, size); 3149 IO_CODE(); 3150 3151 if (mem) { 3152 memset(mem, 0, size); 3153 } 3154 3155 return mem; 3156 } 3157 3158 void coroutine_fn bdrv_co_io_plug(BlockDriverState *bs) 3159 { 3160 BdrvChild *child; 3161 IO_CODE(); 3162 assert_bdrv_graph_readable(); 3163 3164 QLIST_FOREACH(child, &bs->children, next) { 3165 bdrv_co_io_plug(child->bs); 3166 } 3167 3168 if (qatomic_fetch_inc(&bs->io_plugged) == 0) { 3169 BlockDriver *drv = bs->drv; 3170 if (drv && drv->bdrv_co_io_plug) { 3171 drv->bdrv_co_io_plug(bs); 3172 } 3173 } 3174 } 3175 3176 void coroutine_fn bdrv_co_io_unplug(BlockDriverState *bs) 3177 { 3178 BdrvChild *child; 3179 IO_CODE(); 3180 assert_bdrv_graph_readable(); 3181 3182 assert(bs->io_plugged); 3183 if (qatomic_fetch_dec(&bs->io_plugged) == 1) { 3184 BlockDriver *drv = bs->drv; 3185 if (drv && drv->bdrv_co_io_unplug) { 3186 drv->bdrv_co_io_unplug(bs); 3187 } 3188 } 3189 3190 QLIST_FOREACH(child, &bs->children, next) { 3191 bdrv_co_io_unplug(child->bs); 3192 } 3193 } 3194 3195 /* Helper that undoes bdrv_register_buf() when it fails partway through */ 3196 static void GRAPH_RDLOCK 3197 bdrv_register_buf_rollback(BlockDriverState *bs, void *host, size_t size, 3198 BdrvChild *final_child) 3199 { 3200 BdrvChild *child; 3201 3202 GLOBAL_STATE_CODE(); 3203 assert_bdrv_graph_readable(); 3204 3205 QLIST_FOREACH(child, &bs->children, next) { 3206 if (child == final_child) { 3207 break; 3208 } 3209 3210 bdrv_unregister_buf(child->bs, host, size); 3211 } 3212 3213 if (bs->drv && bs->drv->bdrv_unregister_buf) { 3214 bs->drv->bdrv_unregister_buf(bs, host, size); 3215 } 3216 } 3217 3218 bool bdrv_register_buf(BlockDriverState *bs, void *host, size_t size, 3219 Error **errp) 3220 { 3221 BdrvChild *child; 3222 3223 GLOBAL_STATE_CODE(); 3224 GRAPH_RDLOCK_GUARD_MAINLOOP(); 3225 3226 if (bs->drv && bs->drv->bdrv_register_buf) { 3227 if (!bs->drv->bdrv_register_buf(bs, host, size, errp)) { 3228 return false; 3229 } 3230 } 3231 QLIST_FOREACH(child, &bs->children, next) { 3232 if (!bdrv_register_buf(child->bs, host, size, errp)) { 3233 bdrv_register_buf_rollback(bs, host, size, child); 3234 return false; 3235 } 3236 } 3237 return true; 3238 } 3239 3240 void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size) 3241 { 3242 BdrvChild *child; 3243 3244 GLOBAL_STATE_CODE(); 3245 GRAPH_RDLOCK_GUARD_MAINLOOP(); 3246 3247 if (bs->drv && bs->drv->bdrv_unregister_buf) { 3248 bs->drv->bdrv_unregister_buf(bs, host, size); 3249 } 3250 QLIST_FOREACH(child, &bs->children, next) { 3251 bdrv_unregister_buf(child->bs, host, size); 3252 } 3253 } 3254 3255 static int coroutine_fn GRAPH_RDLOCK bdrv_co_copy_range_internal( 3256 BdrvChild *src, int64_t src_offset, BdrvChild *dst, 3257 int64_t dst_offset, int64_t bytes, 3258 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags, 3259 bool recurse_src) 3260 { 3261 BdrvTrackedRequest req; 3262 int ret; 3263 assert_bdrv_graph_readable(); 3264 3265 /* TODO We can support BDRV_REQ_NO_FALLBACK here */ 3266 assert(!(read_flags & BDRV_REQ_NO_FALLBACK)); 3267 assert(!(write_flags & BDRV_REQ_NO_FALLBACK)); 3268 assert(!(read_flags & BDRV_REQ_NO_WAIT)); 3269 assert(!(write_flags & BDRV_REQ_NO_WAIT)); 3270 3271 if (!dst || !dst->bs || !bdrv_co_is_inserted(dst->bs)) { 3272 return -ENOMEDIUM; 3273 } 3274 ret = bdrv_check_request32(dst_offset, bytes, NULL, 0); 3275 if (ret) { 3276 return ret; 3277 } 3278 if (write_flags & BDRV_REQ_ZERO_WRITE) { 3279 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags); 3280 } 3281 3282 if (!src || !src->bs || !bdrv_co_is_inserted(src->bs)) { 3283 return -ENOMEDIUM; 3284 } 3285 ret = bdrv_check_request32(src_offset, bytes, NULL, 0); 3286 if (ret) { 3287 return ret; 3288 } 3289 3290 if (!src->bs->drv->bdrv_co_copy_range_from 3291 || !dst->bs->drv->bdrv_co_copy_range_to 3292 || src->bs->encrypted || dst->bs->encrypted) { 3293 return -ENOTSUP; 3294 } 3295 3296 if (recurse_src) { 3297 bdrv_inc_in_flight(src->bs); 3298 tracked_request_begin(&req, src->bs, src_offset, bytes, 3299 BDRV_TRACKED_READ); 3300 3301 /* BDRV_REQ_SERIALISING is only for write operation */ 3302 assert(!(read_flags & BDRV_REQ_SERIALISING)); 3303 bdrv_wait_serialising_requests(&req); 3304 3305 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs, 3306 src, src_offset, 3307 dst, dst_offset, 3308 bytes, 3309 read_flags, write_flags); 3310 3311 tracked_request_end(&req); 3312 bdrv_dec_in_flight(src->bs); 3313 } else { 3314 bdrv_inc_in_flight(dst->bs); 3315 tracked_request_begin(&req, dst->bs, dst_offset, bytes, 3316 BDRV_TRACKED_WRITE); 3317 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req, 3318 write_flags); 3319 if (!ret) { 3320 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs, 3321 src, src_offset, 3322 dst, dst_offset, 3323 bytes, 3324 read_flags, write_flags); 3325 } 3326 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret); 3327 tracked_request_end(&req); 3328 bdrv_dec_in_flight(dst->bs); 3329 } 3330 3331 return ret; 3332 } 3333 3334 /* Copy range from @src to @dst. 3335 * 3336 * See the comment of bdrv_co_copy_range for the parameter and return value 3337 * semantics. */ 3338 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset, 3339 BdrvChild *dst, int64_t dst_offset, 3340 int64_t bytes, 3341 BdrvRequestFlags read_flags, 3342 BdrvRequestFlags write_flags) 3343 { 3344 IO_CODE(); 3345 assert_bdrv_graph_readable(); 3346 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes, 3347 read_flags, write_flags); 3348 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3349 bytes, read_flags, write_flags, true); 3350 } 3351 3352 /* Copy range from @src to @dst. 3353 * 3354 * See the comment of bdrv_co_copy_range for the parameter and return value 3355 * semantics. */ 3356 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset, 3357 BdrvChild *dst, int64_t dst_offset, 3358 int64_t bytes, 3359 BdrvRequestFlags read_flags, 3360 BdrvRequestFlags write_flags) 3361 { 3362 IO_CODE(); 3363 assert_bdrv_graph_readable(); 3364 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes, 3365 read_flags, write_flags); 3366 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3367 bytes, read_flags, write_flags, false); 3368 } 3369 3370 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset, 3371 BdrvChild *dst, int64_t dst_offset, 3372 int64_t bytes, BdrvRequestFlags read_flags, 3373 BdrvRequestFlags write_flags) 3374 { 3375 IO_CODE(); 3376 assert_bdrv_graph_readable(); 3377 3378 return bdrv_co_copy_range_from(src, src_offset, 3379 dst, dst_offset, 3380 bytes, read_flags, write_flags); 3381 } 3382 3383 static void bdrv_parent_cb_resize(BlockDriverState *bs) 3384 { 3385 BdrvChild *c; 3386 QLIST_FOREACH(c, &bs->parents, next_parent) { 3387 if (c->klass->resize) { 3388 c->klass->resize(c); 3389 } 3390 } 3391 } 3392 3393 /** 3394 * Truncate file to 'offset' bytes (needed only for file protocols) 3395 * 3396 * If 'exact' is true, the file must be resized to exactly the given 3397 * 'offset'. Otherwise, it is sufficient for the node to be at least 3398 * 'offset' bytes in length. 3399 */ 3400 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact, 3401 PreallocMode prealloc, BdrvRequestFlags flags, 3402 Error **errp) 3403 { 3404 BlockDriverState *bs = child->bs; 3405 BdrvChild *filtered, *backing; 3406 BlockDriver *drv = bs->drv; 3407 BdrvTrackedRequest req; 3408 int64_t old_size, new_bytes; 3409 int ret; 3410 IO_CODE(); 3411 assert_bdrv_graph_readable(); 3412 3413 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */ 3414 if (!drv) { 3415 error_setg(errp, "No medium inserted"); 3416 return -ENOMEDIUM; 3417 } 3418 if (offset < 0) { 3419 error_setg(errp, "Image size cannot be negative"); 3420 return -EINVAL; 3421 } 3422 3423 ret = bdrv_check_request(offset, 0, errp); 3424 if (ret < 0) { 3425 return ret; 3426 } 3427 3428 old_size = bdrv_getlength(bs); 3429 if (old_size < 0) { 3430 error_setg_errno(errp, -old_size, "Failed to get old image size"); 3431 return old_size; 3432 } 3433 3434 if (bdrv_is_read_only(bs)) { 3435 error_setg(errp, "Image is read-only"); 3436 return -EACCES; 3437 } 3438 3439 if (offset > old_size) { 3440 new_bytes = offset - old_size; 3441 } else { 3442 new_bytes = 0; 3443 } 3444 3445 bdrv_inc_in_flight(bs); 3446 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes, 3447 BDRV_TRACKED_TRUNCATE); 3448 3449 /* If we are growing the image and potentially using preallocation for the 3450 * new area, we need to make sure that no write requests are made to it 3451 * concurrently or they might be overwritten by preallocation. */ 3452 if (new_bytes) { 3453 bdrv_make_request_serialising(&req, 1); 3454 } 3455 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req, 3456 0); 3457 if (ret < 0) { 3458 error_setg_errno(errp, -ret, 3459 "Failed to prepare request for truncation"); 3460 goto out; 3461 } 3462 3463 filtered = bdrv_filter_child(bs); 3464 backing = bdrv_cow_child(bs); 3465 3466 /* 3467 * If the image has a backing file that is large enough that it would 3468 * provide data for the new area, we cannot leave it unallocated because 3469 * then the backing file content would become visible. Instead, zero-fill 3470 * the new area. 3471 * 3472 * Note that if the image has a backing file, but was opened without the 3473 * backing file, taking care of keeping things consistent with that backing 3474 * file is the user's responsibility. 3475 */ 3476 if (new_bytes && backing) { 3477 int64_t backing_len; 3478 3479 backing_len = bdrv_co_getlength(backing->bs); 3480 if (backing_len < 0) { 3481 ret = backing_len; 3482 error_setg_errno(errp, -ret, "Could not get backing file size"); 3483 goto out; 3484 } 3485 3486 if (backing_len > old_size) { 3487 flags |= BDRV_REQ_ZERO_WRITE; 3488 } 3489 } 3490 3491 if (drv->bdrv_co_truncate) { 3492 if (flags & ~bs->supported_truncate_flags) { 3493 error_setg(errp, "Block driver does not support requested flags"); 3494 ret = -ENOTSUP; 3495 goto out; 3496 } 3497 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp); 3498 } else if (filtered) { 3499 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp); 3500 } else { 3501 error_setg(errp, "Image format driver does not support resize"); 3502 ret = -ENOTSUP; 3503 goto out; 3504 } 3505 if (ret < 0) { 3506 goto out; 3507 } 3508 3509 ret = bdrv_co_refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS); 3510 if (ret < 0) { 3511 error_setg_errno(errp, -ret, "Could not refresh total sector count"); 3512 } else { 3513 offset = bs->total_sectors * BDRV_SECTOR_SIZE; 3514 } 3515 /* 3516 * It's possible that truncation succeeded but bdrv_refresh_total_sectors 3517 * failed, but the latter doesn't affect how we should finish the request. 3518 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. 3519 */ 3520 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0); 3521 3522 out: 3523 tracked_request_end(&req); 3524 bdrv_dec_in_flight(bs); 3525 3526 return ret; 3527 } 3528 3529 void bdrv_cancel_in_flight(BlockDriverState *bs) 3530 { 3531 GLOBAL_STATE_CODE(); 3532 if (!bs || !bs->drv) { 3533 return; 3534 } 3535 3536 if (bs->drv->bdrv_cancel_in_flight) { 3537 bs->drv->bdrv_cancel_in_flight(bs); 3538 } 3539 } 3540 3541 int coroutine_fn 3542 bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes, 3543 QEMUIOVector *qiov, size_t qiov_offset) 3544 { 3545 BlockDriverState *bs = child->bs; 3546 BlockDriver *drv = bs->drv; 3547 int ret; 3548 IO_CODE(); 3549 assert_bdrv_graph_readable(); 3550 3551 if (!drv) { 3552 return -ENOMEDIUM; 3553 } 3554 3555 if (!drv->bdrv_co_preadv_snapshot) { 3556 return -ENOTSUP; 3557 } 3558 3559 bdrv_inc_in_flight(bs); 3560 ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset); 3561 bdrv_dec_in_flight(bs); 3562 3563 return ret; 3564 } 3565 3566 int coroutine_fn 3567 bdrv_co_snapshot_block_status(BlockDriverState *bs, 3568 bool want_zero, int64_t offset, int64_t bytes, 3569 int64_t *pnum, int64_t *map, 3570 BlockDriverState **file) 3571 { 3572 BlockDriver *drv = bs->drv; 3573 int ret; 3574 IO_CODE(); 3575 assert_bdrv_graph_readable(); 3576 3577 if (!drv) { 3578 return -ENOMEDIUM; 3579 } 3580 3581 if (!drv->bdrv_co_snapshot_block_status) { 3582 return -ENOTSUP; 3583 } 3584 3585 bdrv_inc_in_flight(bs); 3586 ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes, 3587 pnum, map, file); 3588 bdrv_dec_in_flight(bs); 3589 3590 return ret; 3591 } 3592 3593 int coroutine_fn 3594 bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes) 3595 { 3596 BlockDriver *drv = bs->drv; 3597 int ret; 3598 IO_CODE(); 3599 assert_bdrv_graph_readable(); 3600 3601 if (!drv) { 3602 return -ENOMEDIUM; 3603 } 3604 3605 if (!drv->bdrv_co_pdiscard_snapshot) { 3606 return -ENOTSUP; 3607 } 3608 3609 bdrv_inc_in_flight(bs); 3610 ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes); 3611 bdrv_dec_in_flight(bs); 3612 3613 return ret; 3614 } 3615