1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/aio-wait.h" 29 #include "block/blockjob.h" 30 #include "block/blockjob_int.h" 31 #include "block/block_int.h" 32 #include "block/coroutines.h" 33 #include "block/dirty-bitmap.h" 34 #include "block/write-threshold.h" 35 #include "qemu/cutils.h" 36 #include "qemu/memalign.h" 37 #include "qapi/error.h" 38 #include "qemu/error-report.h" 39 #include "qemu/main-loop.h" 40 #include "sysemu/replay.h" 41 42 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 43 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 44 45 static void bdrv_parent_cb_resize(BlockDriverState *bs); 46 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 47 int64_t offset, int64_t bytes, BdrvRequestFlags flags); 48 49 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore) 50 { 51 BdrvChild *c, *next; 52 53 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 54 if (c == ignore) { 55 continue; 56 } 57 bdrv_parent_drained_begin_single(c); 58 } 59 } 60 61 void bdrv_parent_drained_end_single(BdrvChild *c) 62 { 63 IO_OR_GS_CODE(); 64 65 assert(c->quiesced_parent); 66 c->quiesced_parent = false; 67 68 if (c->klass->drained_end) { 69 c->klass->drained_end(c); 70 } 71 } 72 73 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore) 74 { 75 BdrvChild *c; 76 77 QLIST_FOREACH(c, &bs->parents, next_parent) { 78 if (c == ignore) { 79 continue; 80 } 81 bdrv_parent_drained_end_single(c); 82 } 83 } 84 85 bool bdrv_parent_drained_poll_single(BdrvChild *c) 86 { 87 if (c->klass->drained_poll) { 88 return c->klass->drained_poll(c); 89 } 90 return false; 91 } 92 93 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore, 94 bool ignore_bds_parents) 95 { 96 BdrvChild *c, *next; 97 bool busy = false; 98 99 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 100 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 101 continue; 102 } 103 busy |= bdrv_parent_drained_poll_single(c); 104 } 105 106 return busy; 107 } 108 109 void bdrv_parent_drained_begin_single(BdrvChild *c) 110 { 111 IO_OR_GS_CODE(); 112 113 assert(!c->quiesced_parent); 114 c->quiesced_parent = true; 115 116 if (c->klass->drained_begin) { 117 c->klass->drained_begin(c); 118 } 119 } 120 121 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 122 { 123 dst->pdiscard_alignment = MAX(dst->pdiscard_alignment, 124 src->pdiscard_alignment); 125 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 126 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 127 dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer, 128 src->max_hw_transfer); 129 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 130 src->opt_mem_alignment); 131 dst->min_mem_alignment = MAX(dst->min_mem_alignment, 132 src->min_mem_alignment); 133 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 134 dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov); 135 } 136 137 typedef struct BdrvRefreshLimitsState { 138 BlockDriverState *bs; 139 BlockLimits old_bl; 140 } BdrvRefreshLimitsState; 141 142 static void bdrv_refresh_limits_abort(void *opaque) 143 { 144 BdrvRefreshLimitsState *s = opaque; 145 146 s->bs->bl = s->old_bl; 147 } 148 149 static TransactionActionDrv bdrv_refresh_limits_drv = { 150 .abort = bdrv_refresh_limits_abort, 151 .clean = g_free, 152 }; 153 154 /* @tran is allowed to be NULL, in this case no rollback is possible. */ 155 void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp) 156 { 157 ERRP_GUARD(); 158 BlockDriver *drv = bs->drv; 159 BdrvChild *c; 160 bool have_limits; 161 162 GLOBAL_STATE_CODE(); 163 assume_graph_lock(); /* FIXME */ 164 165 if (tran) { 166 BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1); 167 *s = (BdrvRefreshLimitsState) { 168 .bs = bs, 169 .old_bl = bs->bl, 170 }; 171 tran_add(tran, &bdrv_refresh_limits_drv, s); 172 } 173 174 memset(&bs->bl, 0, sizeof(bs->bl)); 175 176 if (!drv) { 177 return; 178 } 179 180 /* Default alignment based on whether driver has byte interface */ 181 bs->bl.request_alignment = (drv->bdrv_co_preadv || 182 drv->bdrv_aio_preadv || 183 drv->bdrv_co_preadv_part) ? 1 : 512; 184 185 /* Take some limits from the children as a default */ 186 have_limits = false; 187 QLIST_FOREACH(c, &bs->children, next) { 188 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW)) 189 { 190 bdrv_merge_limits(&bs->bl, &c->bs->bl); 191 have_limits = true; 192 } 193 } 194 195 if (!have_limits) { 196 bs->bl.min_mem_alignment = 512; 197 bs->bl.opt_mem_alignment = qemu_real_host_page_size(); 198 199 /* Safe default since most protocols use readv()/writev()/etc */ 200 bs->bl.max_iov = IOV_MAX; 201 } 202 203 /* Then let the driver override it */ 204 if (drv->bdrv_refresh_limits) { 205 drv->bdrv_refresh_limits(bs, errp); 206 if (*errp) { 207 return; 208 } 209 } 210 211 if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) { 212 error_setg(errp, "Driver requires too large request alignment"); 213 } 214 } 215 216 /** 217 * The copy-on-read flag is actually a reference count so multiple users may 218 * use the feature without worrying about clobbering its previous state. 219 * Copy-on-read stays enabled until all users have called to disable it. 220 */ 221 void bdrv_enable_copy_on_read(BlockDriverState *bs) 222 { 223 IO_CODE(); 224 qatomic_inc(&bs->copy_on_read); 225 } 226 227 void bdrv_disable_copy_on_read(BlockDriverState *bs) 228 { 229 int old = qatomic_fetch_dec(&bs->copy_on_read); 230 IO_CODE(); 231 assert(old >= 1); 232 } 233 234 typedef struct { 235 Coroutine *co; 236 BlockDriverState *bs; 237 bool done; 238 bool begin; 239 bool poll; 240 BdrvChild *parent; 241 } BdrvCoDrainData; 242 243 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */ 244 bool bdrv_drain_poll(BlockDriverState *bs, BdrvChild *ignore_parent, 245 bool ignore_bds_parents) 246 { 247 IO_OR_GS_CODE(); 248 249 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) { 250 return true; 251 } 252 253 if (qatomic_read(&bs->in_flight)) { 254 return true; 255 } 256 257 return false; 258 } 259 260 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, 261 BdrvChild *ignore_parent) 262 { 263 return bdrv_drain_poll(bs, ignore_parent, false); 264 } 265 266 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent, 267 bool poll); 268 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent); 269 270 static void bdrv_co_drain_bh_cb(void *opaque) 271 { 272 BdrvCoDrainData *data = opaque; 273 Coroutine *co = data->co; 274 BlockDriverState *bs = data->bs; 275 276 if (bs) { 277 AioContext *ctx = bdrv_get_aio_context(bs); 278 aio_context_acquire(ctx); 279 bdrv_dec_in_flight(bs); 280 if (data->begin) { 281 bdrv_do_drained_begin(bs, data->parent, data->poll); 282 } else { 283 assert(!data->poll); 284 bdrv_do_drained_end(bs, data->parent); 285 } 286 aio_context_release(ctx); 287 } else { 288 assert(data->begin); 289 bdrv_drain_all_begin(); 290 } 291 292 data->done = true; 293 aio_co_wake(co); 294 } 295 296 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 297 bool begin, 298 BdrvChild *parent, 299 bool poll) 300 { 301 BdrvCoDrainData data; 302 Coroutine *self = qemu_coroutine_self(); 303 AioContext *ctx = bdrv_get_aio_context(bs); 304 AioContext *co_ctx = qemu_coroutine_get_aio_context(self); 305 306 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 307 * other coroutines run if they were queued by aio_co_enter(). */ 308 309 assert(qemu_in_coroutine()); 310 data = (BdrvCoDrainData) { 311 .co = self, 312 .bs = bs, 313 .done = false, 314 .begin = begin, 315 .parent = parent, 316 .poll = poll, 317 }; 318 319 if (bs) { 320 bdrv_inc_in_flight(bs); 321 } 322 323 /* 324 * Temporarily drop the lock across yield or we would get deadlocks. 325 * bdrv_co_drain_bh_cb() reaquires the lock as needed. 326 * 327 * When we yield below, the lock for the current context will be 328 * released, so if this is actually the lock that protects bs, don't drop 329 * it a second time. 330 */ 331 if (ctx != co_ctx) { 332 aio_context_release(ctx); 333 } 334 replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data); 335 336 qemu_coroutine_yield(); 337 /* If we are resumed from some other event (such as an aio completion or a 338 * timer callback), it is a bug in the caller that should be fixed. */ 339 assert(data.done); 340 341 /* Reaquire the AioContext of bs if we dropped it */ 342 if (ctx != co_ctx) { 343 aio_context_acquire(ctx); 344 } 345 } 346 347 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent, 348 bool poll) 349 { 350 IO_OR_GS_CODE(); 351 352 if (qemu_in_coroutine()) { 353 bdrv_co_yield_to_drain(bs, true, parent, poll); 354 return; 355 } 356 357 /* Stop things in parent-to-child order */ 358 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) { 359 aio_disable_external(bdrv_get_aio_context(bs)); 360 bdrv_parent_drained_begin(bs, parent); 361 if (bs->drv && bs->drv->bdrv_drain_begin) { 362 bs->drv->bdrv_drain_begin(bs); 363 } 364 } 365 366 /* 367 * Wait for drained requests to finish. 368 * 369 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The 370 * call is needed so things in this AioContext can make progress even 371 * though we don't return to the main AioContext loop - this automatically 372 * includes other nodes in the same AioContext and therefore all child 373 * nodes. 374 */ 375 if (poll) { 376 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, parent)); 377 } 378 } 379 380 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, BdrvChild *parent) 381 { 382 bdrv_do_drained_begin(bs, parent, false); 383 } 384 385 void bdrv_drained_begin(BlockDriverState *bs) 386 { 387 IO_OR_GS_CODE(); 388 bdrv_do_drained_begin(bs, NULL, true); 389 } 390 391 /** 392 * This function does not poll, nor must any of its recursively called 393 * functions. 394 */ 395 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent) 396 { 397 int old_quiesce_counter; 398 399 if (qemu_in_coroutine()) { 400 bdrv_co_yield_to_drain(bs, false, parent, false); 401 return; 402 } 403 assert(bs->quiesce_counter > 0); 404 405 /* Re-enable things in child-to-parent order */ 406 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter); 407 if (old_quiesce_counter == 1) { 408 if (bs->drv && bs->drv->bdrv_drain_end) { 409 bs->drv->bdrv_drain_end(bs); 410 } 411 bdrv_parent_drained_end(bs, parent); 412 aio_enable_external(bdrv_get_aio_context(bs)); 413 } 414 } 415 416 void bdrv_drained_end(BlockDriverState *bs) 417 { 418 IO_OR_GS_CODE(); 419 bdrv_do_drained_end(bs, NULL); 420 } 421 422 void bdrv_drain(BlockDriverState *bs) 423 { 424 IO_OR_GS_CODE(); 425 bdrv_drained_begin(bs); 426 bdrv_drained_end(bs); 427 } 428 429 static void bdrv_drain_assert_idle(BlockDriverState *bs) 430 { 431 BdrvChild *child, *next; 432 433 assert(qatomic_read(&bs->in_flight) == 0); 434 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 435 bdrv_drain_assert_idle(child->bs); 436 } 437 } 438 439 unsigned int bdrv_drain_all_count = 0; 440 441 static bool bdrv_drain_all_poll(void) 442 { 443 BlockDriverState *bs = NULL; 444 bool result = false; 445 GLOBAL_STATE_CODE(); 446 447 /* bdrv_drain_poll() can't make changes to the graph and we are holding the 448 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ 449 while ((bs = bdrv_next_all_states(bs))) { 450 AioContext *aio_context = bdrv_get_aio_context(bs); 451 aio_context_acquire(aio_context); 452 result |= bdrv_drain_poll(bs, NULL, true); 453 aio_context_release(aio_context); 454 } 455 456 return result; 457 } 458 459 /* 460 * Wait for pending requests to complete across all BlockDriverStates 461 * 462 * This function does not flush data to disk, use bdrv_flush_all() for that 463 * after calling this function. 464 * 465 * This pauses all block jobs and disables external clients. It must 466 * be paired with bdrv_drain_all_end(). 467 * 468 * NOTE: no new block jobs or BlockDriverStates can be created between 469 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 470 */ 471 void bdrv_drain_all_begin_nopoll(void) 472 { 473 BlockDriverState *bs = NULL; 474 GLOBAL_STATE_CODE(); 475 476 /* 477 * bdrv queue is managed by record/replay, 478 * waiting for finishing the I/O requests may 479 * be infinite 480 */ 481 if (replay_events_enabled()) { 482 return; 483 } 484 485 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main 486 * loop AioContext, so make sure we're in the main context. */ 487 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 488 assert(bdrv_drain_all_count < INT_MAX); 489 bdrv_drain_all_count++; 490 491 /* Quiesce all nodes, without polling in-flight requests yet. The graph 492 * cannot change during this loop. */ 493 while ((bs = bdrv_next_all_states(bs))) { 494 AioContext *aio_context = bdrv_get_aio_context(bs); 495 496 aio_context_acquire(aio_context); 497 bdrv_do_drained_begin(bs, NULL, false); 498 aio_context_release(aio_context); 499 } 500 } 501 502 void bdrv_drain_all_begin(void) 503 { 504 BlockDriverState *bs = NULL; 505 506 if (qemu_in_coroutine()) { 507 bdrv_co_yield_to_drain(NULL, true, NULL, true); 508 return; 509 } 510 511 /* 512 * bdrv queue is managed by record/replay, 513 * waiting for finishing the I/O requests may 514 * be infinite 515 */ 516 if (replay_events_enabled()) { 517 return; 518 } 519 520 bdrv_drain_all_begin_nopoll(); 521 522 /* Now poll the in-flight requests */ 523 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll()); 524 525 while ((bs = bdrv_next_all_states(bs))) { 526 bdrv_drain_assert_idle(bs); 527 } 528 } 529 530 void bdrv_drain_all_end_quiesce(BlockDriverState *bs) 531 { 532 GLOBAL_STATE_CODE(); 533 534 g_assert(bs->quiesce_counter > 0); 535 g_assert(!bs->refcnt); 536 537 while (bs->quiesce_counter) { 538 bdrv_do_drained_end(bs, NULL); 539 } 540 } 541 542 void bdrv_drain_all_end(void) 543 { 544 BlockDriverState *bs = NULL; 545 GLOBAL_STATE_CODE(); 546 547 /* 548 * bdrv queue is managed by record/replay, 549 * waiting for finishing the I/O requests may 550 * be endless 551 */ 552 if (replay_events_enabled()) { 553 return; 554 } 555 556 while ((bs = bdrv_next_all_states(bs))) { 557 AioContext *aio_context = bdrv_get_aio_context(bs); 558 559 aio_context_acquire(aio_context); 560 bdrv_do_drained_end(bs, NULL); 561 aio_context_release(aio_context); 562 } 563 564 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 565 assert(bdrv_drain_all_count > 0); 566 bdrv_drain_all_count--; 567 } 568 569 void bdrv_drain_all(void) 570 { 571 GLOBAL_STATE_CODE(); 572 bdrv_drain_all_begin(); 573 bdrv_drain_all_end(); 574 } 575 576 /** 577 * Remove an active request from the tracked requests list 578 * 579 * This function should be called when a tracked request is completing. 580 */ 581 static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req) 582 { 583 if (req->serialising) { 584 qatomic_dec(&req->bs->serialising_in_flight); 585 } 586 587 qemu_co_mutex_lock(&req->bs->reqs_lock); 588 QLIST_REMOVE(req, list); 589 qemu_co_queue_restart_all(&req->wait_queue); 590 qemu_co_mutex_unlock(&req->bs->reqs_lock); 591 } 592 593 /** 594 * Add an active request to the tracked requests list 595 */ 596 static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req, 597 BlockDriverState *bs, 598 int64_t offset, 599 int64_t bytes, 600 enum BdrvTrackedRequestType type) 601 { 602 bdrv_check_request(offset, bytes, &error_abort); 603 604 *req = (BdrvTrackedRequest){ 605 .bs = bs, 606 .offset = offset, 607 .bytes = bytes, 608 .type = type, 609 .co = qemu_coroutine_self(), 610 .serialising = false, 611 .overlap_offset = offset, 612 .overlap_bytes = bytes, 613 }; 614 615 qemu_co_queue_init(&req->wait_queue); 616 617 qemu_co_mutex_lock(&bs->reqs_lock); 618 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 619 qemu_co_mutex_unlock(&bs->reqs_lock); 620 } 621 622 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 623 int64_t offset, int64_t bytes) 624 { 625 bdrv_check_request(offset, bytes, &error_abort); 626 627 /* aaaa bbbb */ 628 if (offset >= req->overlap_offset + req->overlap_bytes) { 629 return false; 630 } 631 /* bbbb aaaa */ 632 if (req->overlap_offset >= offset + bytes) { 633 return false; 634 } 635 return true; 636 } 637 638 /* Called with self->bs->reqs_lock held */ 639 static coroutine_fn BdrvTrackedRequest * 640 bdrv_find_conflicting_request(BdrvTrackedRequest *self) 641 { 642 BdrvTrackedRequest *req; 643 644 QLIST_FOREACH(req, &self->bs->tracked_requests, list) { 645 if (req == self || (!req->serialising && !self->serialising)) { 646 continue; 647 } 648 if (tracked_request_overlaps(req, self->overlap_offset, 649 self->overlap_bytes)) 650 { 651 /* 652 * Hitting this means there was a reentrant request, for 653 * example, a block driver issuing nested requests. This must 654 * never happen since it means deadlock. 655 */ 656 assert(qemu_coroutine_self() != req->co); 657 658 /* 659 * If the request is already (indirectly) waiting for us, or 660 * will wait for us as soon as it wakes up, then just go on 661 * (instead of producing a deadlock in the former case). 662 */ 663 if (!req->waiting_for) { 664 return req; 665 } 666 } 667 } 668 669 return NULL; 670 } 671 672 /* Called with self->bs->reqs_lock held */ 673 static void coroutine_fn 674 bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self) 675 { 676 BdrvTrackedRequest *req; 677 678 while ((req = bdrv_find_conflicting_request(self))) { 679 self->waiting_for = req; 680 qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock); 681 self->waiting_for = NULL; 682 } 683 } 684 685 /* Called with req->bs->reqs_lock held */ 686 static void tracked_request_set_serialising(BdrvTrackedRequest *req, 687 uint64_t align) 688 { 689 int64_t overlap_offset = req->offset & ~(align - 1); 690 int64_t overlap_bytes = 691 ROUND_UP(req->offset + req->bytes, align) - overlap_offset; 692 693 bdrv_check_request(req->offset, req->bytes, &error_abort); 694 695 if (!req->serialising) { 696 qatomic_inc(&req->bs->serialising_in_flight); 697 req->serialising = true; 698 } 699 700 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 701 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 702 } 703 704 /** 705 * Return the tracked request on @bs for the current coroutine, or 706 * NULL if there is none. 707 */ 708 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs) 709 { 710 BdrvTrackedRequest *req; 711 Coroutine *self = qemu_coroutine_self(); 712 IO_CODE(); 713 714 QLIST_FOREACH(req, &bs->tracked_requests, list) { 715 if (req->co == self) { 716 return req; 717 } 718 } 719 720 return NULL; 721 } 722 723 /** 724 * Round a region to cluster boundaries 725 */ 726 void coroutine_fn bdrv_round_to_clusters(BlockDriverState *bs, 727 int64_t offset, int64_t bytes, 728 int64_t *cluster_offset, 729 int64_t *cluster_bytes) 730 { 731 BlockDriverInfo bdi; 732 IO_CODE(); 733 if (bdrv_co_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 734 *cluster_offset = offset; 735 *cluster_bytes = bytes; 736 } else { 737 int64_t c = bdi.cluster_size; 738 *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 739 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 740 } 741 } 742 743 static coroutine_fn int bdrv_get_cluster_size(BlockDriverState *bs) 744 { 745 BlockDriverInfo bdi; 746 int ret; 747 748 ret = bdrv_co_get_info(bs, &bdi); 749 if (ret < 0 || bdi.cluster_size == 0) { 750 return bs->bl.request_alignment; 751 } else { 752 return bdi.cluster_size; 753 } 754 } 755 756 void bdrv_inc_in_flight(BlockDriverState *bs) 757 { 758 IO_CODE(); 759 qatomic_inc(&bs->in_flight); 760 } 761 762 void bdrv_wakeup(BlockDriverState *bs) 763 { 764 IO_CODE(); 765 aio_wait_kick(); 766 } 767 768 void bdrv_dec_in_flight(BlockDriverState *bs) 769 { 770 IO_CODE(); 771 qatomic_dec(&bs->in_flight); 772 bdrv_wakeup(bs); 773 } 774 775 static void coroutine_fn 776 bdrv_wait_serialising_requests(BdrvTrackedRequest *self) 777 { 778 BlockDriverState *bs = self->bs; 779 780 if (!qatomic_read(&bs->serialising_in_flight)) { 781 return; 782 } 783 784 qemu_co_mutex_lock(&bs->reqs_lock); 785 bdrv_wait_serialising_requests_locked(self); 786 qemu_co_mutex_unlock(&bs->reqs_lock); 787 } 788 789 void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req, 790 uint64_t align) 791 { 792 IO_CODE(); 793 794 qemu_co_mutex_lock(&req->bs->reqs_lock); 795 796 tracked_request_set_serialising(req, align); 797 bdrv_wait_serialising_requests_locked(req); 798 799 qemu_co_mutex_unlock(&req->bs->reqs_lock); 800 } 801 802 int bdrv_check_qiov_request(int64_t offset, int64_t bytes, 803 QEMUIOVector *qiov, size_t qiov_offset, 804 Error **errp) 805 { 806 /* 807 * Check generic offset/bytes correctness 808 */ 809 810 if (offset < 0) { 811 error_setg(errp, "offset is negative: %" PRIi64, offset); 812 return -EIO; 813 } 814 815 if (bytes < 0) { 816 error_setg(errp, "bytes is negative: %" PRIi64, bytes); 817 return -EIO; 818 } 819 820 if (bytes > BDRV_MAX_LENGTH) { 821 error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 822 bytes, BDRV_MAX_LENGTH); 823 return -EIO; 824 } 825 826 if (offset > BDRV_MAX_LENGTH) { 827 error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 828 offset, BDRV_MAX_LENGTH); 829 return -EIO; 830 } 831 832 if (offset > BDRV_MAX_LENGTH - bytes) { 833 error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") " 834 "exceeds maximum(%" PRIi64 ")", offset, bytes, 835 BDRV_MAX_LENGTH); 836 return -EIO; 837 } 838 839 if (!qiov) { 840 return 0; 841 } 842 843 /* 844 * Check qiov and qiov_offset 845 */ 846 847 if (qiov_offset > qiov->size) { 848 error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)", 849 qiov_offset, qiov->size); 850 return -EIO; 851 } 852 853 if (bytes > qiov->size - qiov_offset) { 854 error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io " 855 "vector size(%zu)", bytes, qiov_offset, qiov->size); 856 return -EIO; 857 } 858 859 return 0; 860 } 861 862 int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp) 863 { 864 return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp); 865 } 866 867 static int bdrv_check_request32(int64_t offset, int64_t bytes, 868 QEMUIOVector *qiov, size_t qiov_offset) 869 { 870 int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 871 if (ret < 0) { 872 return ret; 873 } 874 875 if (bytes > BDRV_REQUEST_MAX_BYTES) { 876 return -EIO; 877 } 878 879 return 0; 880 } 881 882 /* 883 * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 884 * The operation is sped up by checking the block status and only writing 885 * zeroes to the device if they currently do not return zeroes. Optional 886 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 887 * BDRV_REQ_FUA). 888 * 889 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite(). 890 */ 891 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 892 { 893 int ret; 894 int64_t target_size, bytes, offset = 0; 895 BlockDriverState *bs = child->bs; 896 IO_CODE(); 897 898 target_size = bdrv_getlength(bs); 899 if (target_size < 0) { 900 return target_size; 901 } 902 903 for (;;) { 904 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 905 if (bytes <= 0) { 906 return 0; 907 } 908 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 909 if (ret < 0) { 910 return ret; 911 } 912 if (ret & BDRV_BLOCK_ZERO) { 913 offset += bytes; 914 continue; 915 } 916 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 917 if (ret < 0) { 918 return ret; 919 } 920 offset += bytes; 921 } 922 } 923 924 /* 925 * Writes to the file and ensures that no writes are reordered across this 926 * request (acts as a barrier) 927 * 928 * Returns 0 on success, -errno in error cases. 929 */ 930 int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset, 931 int64_t bytes, const void *buf, 932 BdrvRequestFlags flags) 933 { 934 int ret; 935 IO_CODE(); 936 937 assume_graph_lock(); /* FIXME */ 938 939 ret = bdrv_co_pwrite(child, offset, bytes, buf, flags); 940 if (ret < 0) { 941 return ret; 942 } 943 944 ret = bdrv_co_flush(child->bs); 945 if (ret < 0) { 946 return ret; 947 } 948 949 return 0; 950 } 951 952 typedef struct CoroutineIOCompletion { 953 Coroutine *coroutine; 954 int ret; 955 } CoroutineIOCompletion; 956 957 static void bdrv_co_io_em_complete(void *opaque, int ret) 958 { 959 CoroutineIOCompletion *co = opaque; 960 961 co->ret = ret; 962 aio_co_wake(co->coroutine); 963 } 964 965 static int coroutine_fn GRAPH_RDLOCK 966 bdrv_driver_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes, 967 QEMUIOVector *qiov, size_t qiov_offset, int flags) 968 { 969 BlockDriver *drv = bs->drv; 970 int64_t sector_num; 971 unsigned int nb_sectors; 972 QEMUIOVector local_qiov; 973 int ret; 974 assert_bdrv_graph_readable(); 975 976 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 977 assert(!(flags & ~bs->supported_read_flags)); 978 979 if (!drv) { 980 return -ENOMEDIUM; 981 } 982 983 if (drv->bdrv_co_preadv_part) { 984 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset, 985 flags); 986 } 987 988 if (qiov_offset > 0 || bytes != qiov->size) { 989 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 990 qiov = &local_qiov; 991 } 992 993 if (drv->bdrv_co_preadv) { 994 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 995 goto out; 996 } 997 998 if (drv->bdrv_aio_preadv) { 999 BlockAIOCB *acb; 1000 CoroutineIOCompletion co = { 1001 .coroutine = qemu_coroutine_self(), 1002 }; 1003 1004 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags, 1005 bdrv_co_io_em_complete, &co); 1006 if (acb == NULL) { 1007 ret = -EIO; 1008 goto out; 1009 } else { 1010 qemu_coroutine_yield(); 1011 ret = co.ret; 1012 goto out; 1013 } 1014 } 1015 1016 sector_num = offset >> BDRV_SECTOR_BITS; 1017 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1018 1019 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1020 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1021 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1022 assert(drv->bdrv_co_readv); 1023 1024 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 1025 1026 out: 1027 if (qiov == &local_qiov) { 1028 qemu_iovec_destroy(&local_qiov); 1029 } 1030 1031 return ret; 1032 } 1033 1034 static int coroutine_fn GRAPH_RDLOCK 1035 bdrv_driver_pwritev(BlockDriverState *bs, int64_t offset, int64_t bytes, 1036 QEMUIOVector *qiov, size_t qiov_offset, 1037 BdrvRequestFlags flags) 1038 { 1039 BlockDriver *drv = bs->drv; 1040 bool emulate_fua = false; 1041 int64_t sector_num; 1042 unsigned int nb_sectors; 1043 QEMUIOVector local_qiov; 1044 int ret; 1045 assert_bdrv_graph_readable(); 1046 1047 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1048 1049 if (!drv) { 1050 return -ENOMEDIUM; 1051 } 1052 1053 if ((flags & BDRV_REQ_FUA) && 1054 (~bs->supported_write_flags & BDRV_REQ_FUA)) { 1055 flags &= ~BDRV_REQ_FUA; 1056 emulate_fua = true; 1057 } 1058 1059 flags &= bs->supported_write_flags; 1060 1061 if (drv->bdrv_co_pwritev_part) { 1062 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset, 1063 flags); 1064 goto emulate_flags; 1065 } 1066 1067 if (qiov_offset > 0 || bytes != qiov->size) { 1068 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1069 qiov = &local_qiov; 1070 } 1071 1072 if (drv->bdrv_co_pwritev) { 1073 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags); 1074 goto emulate_flags; 1075 } 1076 1077 if (drv->bdrv_aio_pwritev) { 1078 BlockAIOCB *acb; 1079 CoroutineIOCompletion co = { 1080 .coroutine = qemu_coroutine_self(), 1081 }; 1082 1083 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, flags, 1084 bdrv_co_io_em_complete, &co); 1085 if (acb == NULL) { 1086 ret = -EIO; 1087 } else { 1088 qemu_coroutine_yield(); 1089 ret = co.ret; 1090 } 1091 goto emulate_flags; 1092 } 1093 1094 sector_num = offset >> BDRV_SECTOR_BITS; 1095 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1096 1097 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1098 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1099 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1100 1101 assert(drv->bdrv_co_writev); 1102 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, flags); 1103 1104 emulate_flags: 1105 if (ret == 0 && emulate_fua) { 1106 ret = bdrv_co_flush(bs); 1107 } 1108 1109 if (qiov == &local_qiov) { 1110 qemu_iovec_destroy(&local_qiov); 1111 } 1112 1113 return ret; 1114 } 1115 1116 static int coroutine_fn GRAPH_RDLOCK 1117 bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset, 1118 int64_t bytes, QEMUIOVector *qiov, 1119 size_t qiov_offset) 1120 { 1121 BlockDriver *drv = bs->drv; 1122 QEMUIOVector local_qiov; 1123 int ret; 1124 assert_bdrv_graph_readable(); 1125 1126 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1127 1128 if (!drv) { 1129 return -ENOMEDIUM; 1130 } 1131 1132 if (!block_driver_can_compress(drv)) { 1133 return -ENOTSUP; 1134 } 1135 1136 if (drv->bdrv_co_pwritev_compressed_part) { 1137 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes, 1138 qiov, qiov_offset); 1139 } 1140 1141 if (qiov_offset == 0) { 1142 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 1143 } 1144 1145 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1146 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov); 1147 qemu_iovec_destroy(&local_qiov); 1148 1149 return ret; 1150 } 1151 1152 static int coroutine_fn GRAPH_RDLOCK 1153 bdrv_co_do_copy_on_readv(BdrvChild *child, int64_t offset, int64_t bytes, 1154 QEMUIOVector *qiov, size_t qiov_offset, int flags) 1155 { 1156 BlockDriverState *bs = child->bs; 1157 1158 /* Perform I/O through a temporary buffer so that users who scribble over 1159 * their read buffer while the operation is in progress do not end up 1160 * modifying the image file. This is critical for zero-copy guest I/O 1161 * where anything might happen inside guest memory. 1162 */ 1163 void *bounce_buffer = NULL; 1164 1165 BlockDriver *drv = bs->drv; 1166 int64_t cluster_offset; 1167 int64_t cluster_bytes; 1168 int64_t skip_bytes; 1169 int ret; 1170 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1171 BDRV_REQUEST_MAX_BYTES); 1172 int64_t progress = 0; 1173 bool skip_write; 1174 1175 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1176 1177 if (!drv) { 1178 return -ENOMEDIUM; 1179 } 1180 1181 /* 1182 * Do not write anything when the BDS is inactive. That is not 1183 * allowed, and it would not help. 1184 */ 1185 skip_write = (bs->open_flags & BDRV_O_INACTIVE); 1186 1187 /* FIXME We cannot require callers to have write permissions when all they 1188 * are doing is a read request. If we did things right, write permissions 1189 * would be obtained anyway, but internally by the copy-on-read code. As 1190 * long as it is implemented here rather than in a separate filter driver, 1191 * the copy-on-read code doesn't have its own BdrvChild, however, for which 1192 * it could request permissions. Therefore we have to bypass the permission 1193 * system for the moment. */ 1194 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1195 1196 /* Cover entire cluster so no additional backing file I/O is required when 1197 * allocating cluster in the image file. Note that this value may exceed 1198 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1199 * is one reason we loop rather than doing it all at once. 1200 */ 1201 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 1202 skip_bytes = offset - cluster_offset; 1203 1204 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1205 cluster_offset, cluster_bytes); 1206 1207 while (cluster_bytes) { 1208 int64_t pnum; 1209 1210 if (skip_write) { 1211 ret = 1; /* "already allocated", so nothing will be copied */ 1212 pnum = MIN(cluster_bytes, max_transfer); 1213 } else { 1214 ret = bdrv_is_allocated(bs, cluster_offset, 1215 MIN(cluster_bytes, max_transfer), &pnum); 1216 if (ret < 0) { 1217 /* 1218 * Safe to treat errors in querying allocation as if 1219 * unallocated; we'll probably fail again soon on the 1220 * read, but at least that will set a decent errno. 1221 */ 1222 pnum = MIN(cluster_bytes, max_transfer); 1223 } 1224 1225 /* Stop at EOF if the image ends in the middle of the cluster */ 1226 if (ret == 0 && pnum == 0) { 1227 assert(progress >= bytes); 1228 break; 1229 } 1230 1231 assert(skip_bytes < pnum); 1232 } 1233 1234 if (ret <= 0) { 1235 QEMUIOVector local_qiov; 1236 1237 /* Must copy-on-read; use the bounce buffer */ 1238 pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 1239 if (!bounce_buffer) { 1240 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum); 1241 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER); 1242 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed); 1243 1244 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len); 1245 if (!bounce_buffer) { 1246 ret = -ENOMEM; 1247 goto err; 1248 } 1249 } 1250 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum); 1251 1252 ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1253 &local_qiov, 0, 0); 1254 if (ret < 0) { 1255 goto err; 1256 } 1257 1258 bdrv_co_debug_event(bs, BLKDBG_COR_WRITE); 1259 if (drv->bdrv_co_pwrite_zeroes && 1260 buffer_is_zero(bounce_buffer, pnum)) { 1261 /* FIXME: Should we (perhaps conditionally) be setting 1262 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1263 * that still correctly reads as zero? */ 1264 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 1265 BDRV_REQ_WRITE_UNCHANGED); 1266 } else { 1267 /* This does not change the data on the disk, it is not 1268 * necessary to flush even in cache=writethrough mode. 1269 */ 1270 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 1271 &local_qiov, 0, 1272 BDRV_REQ_WRITE_UNCHANGED); 1273 } 1274 1275 if (ret < 0) { 1276 /* It might be okay to ignore write errors for guest 1277 * requests. If this is a deliberate copy-on-read 1278 * then we don't want to ignore the error. Simply 1279 * report it in all cases. 1280 */ 1281 goto err; 1282 } 1283 1284 if (!(flags & BDRV_REQ_PREFETCH)) { 1285 qemu_iovec_from_buf(qiov, qiov_offset + progress, 1286 bounce_buffer + skip_bytes, 1287 MIN(pnum - skip_bytes, bytes - progress)); 1288 } 1289 } else if (!(flags & BDRV_REQ_PREFETCH)) { 1290 /* Read directly into the destination */ 1291 ret = bdrv_driver_preadv(bs, offset + progress, 1292 MIN(pnum - skip_bytes, bytes - progress), 1293 qiov, qiov_offset + progress, 0); 1294 if (ret < 0) { 1295 goto err; 1296 } 1297 } 1298 1299 cluster_offset += pnum; 1300 cluster_bytes -= pnum; 1301 progress += pnum - skip_bytes; 1302 skip_bytes = 0; 1303 } 1304 ret = 0; 1305 1306 err: 1307 qemu_vfree(bounce_buffer); 1308 return ret; 1309 } 1310 1311 /* 1312 * Forwards an already correctly aligned request to the BlockDriver. This 1313 * handles copy on read, zeroing after EOF, and fragmentation of large 1314 * reads; any other features must be implemented by the caller. 1315 */ 1316 static int coroutine_fn GRAPH_RDLOCK 1317 bdrv_aligned_preadv(BdrvChild *child, BdrvTrackedRequest *req, 1318 int64_t offset, int64_t bytes, int64_t align, 1319 QEMUIOVector *qiov, size_t qiov_offset, int flags) 1320 { 1321 BlockDriverState *bs = child->bs; 1322 int64_t total_bytes, max_bytes; 1323 int ret = 0; 1324 int64_t bytes_remaining = bytes; 1325 int max_transfer; 1326 1327 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1328 assert(is_power_of_2(align)); 1329 assert((offset & (align - 1)) == 0); 1330 assert((bytes & (align - 1)) == 0); 1331 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1332 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1333 align); 1334 1335 /* 1336 * TODO: We would need a per-BDS .supported_read_flags and 1337 * potential fallback support, if we ever implement any read flags 1338 * to pass through to drivers. For now, there aren't any 1339 * passthrough flags except the BDRV_REQ_REGISTERED_BUF optimization hint. 1340 */ 1341 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH | 1342 BDRV_REQ_REGISTERED_BUF))); 1343 1344 /* Handle Copy on Read and associated serialisation */ 1345 if (flags & BDRV_REQ_COPY_ON_READ) { 1346 /* If we touch the same cluster it counts as an overlap. This 1347 * guarantees that allocating writes will be serialized and not race 1348 * with each other for the same cluster. For example, in copy-on-read 1349 * it ensures that the CoR read and write operations are atomic and 1350 * guest writes cannot interleave between them. */ 1351 bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs)); 1352 } else { 1353 bdrv_wait_serialising_requests(req); 1354 } 1355 1356 if (flags & BDRV_REQ_COPY_ON_READ) { 1357 int64_t pnum; 1358 1359 /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */ 1360 flags &= ~BDRV_REQ_COPY_ON_READ; 1361 1362 ret = bdrv_is_allocated(bs, offset, bytes, &pnum); 1363 if (ret < 0) { 1364 goto out; 1365 } 1366 1367 if (!ret || pnum != bytes) { 1368 ret = bdrv_co_do_copy_on_readv(child, offset, bytes, 1369 qiov, qiov_offset, flags); 1370 goto out; 1371 } else if (flags & BDRV_REQ_PREFETCH) { 1372 goto out; 1373 } 1374 } 1375 1376 /* Forward the request to the BlockDriver, possibly fragmenting it */ 1377 total_bytes = bdrv_getlength(bs); 1378 if (total_bytes < 0) { 1379 ret = total_bytes; 1380 goto out; 1381 } 1382 1383 assert(!(flags & ~(bs->supported_read_flags | BDRV_REQ_REGISTERED_BUF))); 1384 1385 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 1386 if (bytes <= max_bytes && bytes <= max_transfer) { 1387 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags); 1388 goto out; 1389 } 1390 1391 while (bytes_remaining) { 1392 int64_t num; 1393 1394 if (max_bytes) { 1395 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 1396 assert(num); 1397 1398 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1399 num, qiov, 1400 qiov_offset + bytes - bytes_remaining, 1401 flags); 1402 max_bytes -= num; 1403 } else { 1404 num = bytes_remaining; 1405 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining, 1406 0, bytes_remaining); 1407 } 1408 if (ret < 0) { 1409 goto out; 1410 } 1411 bytes_remaining -= num; 1412 } 1413 1414 out: 1415 return ret < 0 ? ret : 0; 1416 } 1417 1418 /* 1419 * Request padding 1420 * 1421 * |<---- align ----->| |<----- align ---->| 1422 * |<- head ->|<------------- bytes ------------->|<-- tail -->| 1423 * | | | | | | 1424 * -*----------$-------*-------- ... --------*-----$------------*--- 1425 * | | | | | | 1426 * | offset | | end | 1427 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end) 1428 * [buf ... ) [tail_buf ) 1429 * 1430 * @buf is an aligned allocation needed to store @head and @tail paddings. @head 1431 * is placed at the beginning of @buf and @tail at the @end. 1432 * 1433 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk 1434 * around tail, if tail exists. 1435 * 1436 * @merge_reads is true for small requests, 1437 * if @buf_len == @head + bytes + @tail. In this case it is possible that both 1438 * head and tail exist but @buf_len == align and @tail_buf == @buf. 1439 */ 1440 typedef struct BdrvRequestPadding { 1441 uint8_t *buf; 1442 size_t buf_len; 1443 uint8_t *tail_buf; 1444 size_t head; 1445 size_t tail; 1446 bool merge_reads; 1447 QEMUIOVector local_qiov; 1448 } BdrvRequestPadding; 1449 1450 static bool bdrv_init_padding(BlockDriverState *bs, 1451 int64_t offset, int64_t bytes, 1452 BdrvRequestPadding *pad) 1453 { 1454 int64_t align = bs->bl.request_alignment; 1455 int64_t sum; 1456 1457 bdrv_check_request(offset, bytes, &error_abort); 1458 assert(align <= INT_MAX); /* documented in block/block_int.h */ 1459 assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */ 1460 1461 memset(pad, 0, sizeof(*pad)); 1462 1463 pad->head = offset & (align - 1); 1464 pad->tail = ((offset + bytes) & (align - 1)); 1465 if (pad->tail) { 1466 pad->tail = align - pad->tail; 1467 } 1468 1469 if (!pad->head && !pad->tail) { 1470 return false; 1471 } 1472 1473 assert(bytes); /* Nothing good in aligning zero-length requests */ 1474 1475 sum = pad->head + bytes + pad->tail; 1476 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align; 1477 pad->buf = qemu_blockalign(bs, pad->buf_len); 1478 pad->merge_reads = sum == pad->buf_len; 1479 if (pad->tail) { 1480 pad->tail_buf = pad->buf + pad->buf_len - align; 1481 } 1482 1483 return true; 1484 } 1485 1486 static int coroutine_fn GRAPH_RDLOCK 1487 bdrv_padding_rmw_read(BdrvChild *child, BdrvTrackedRequest *req, 1488 BdrvRequestPadding *pad, bool zero_middle) 1489 { 1490 QEMUIOVector local_qiov; 1491 BlockDriverState *bs = child->bs; 1492 uint64_t align = bs->bl.request_alignment; 1493 int ret; 1494 1495 assert(req->serialising && pad->buf); 1496 1497 if (pad->head || pad->merge_reads) { 1498 int64_t bytes = pad->merge_reads ? pad->buf_len : align; 1499 1500 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes); 1501 1502 if (pad->head) { 1503 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1504 } 1505 if (pad->merge_reads && pad->tail) { 1506 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1507 } 1508 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes, 1509 align, &local_qiov, 0, 0); 1510 if (ret < 0) { 1511 return ret; 1512 } 1513 if (pad->head) { 1514 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1515 } 1516 if (pad->merge_reads && pad->tail) { 1517 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1518 } 1519 1520 if (pad->merge_reads) { 1521 goto zero_mem; 1522 } 1523 } 1524 1525 if (pad->tail) { 1526 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align); 1527 1528 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1529 ret = bdrv_aligned_preadv( 1530 child, req, 1531 req->overlap_offset + req->overlap_bytes - align, 1532 align, align, &local_qiov, 0, 0); 1533 if (ret < 0) { 1534 return ret; 1535 } 1536 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1537 } 1538 1539 zero_mem: 1540 if (zero_middle) { 1541 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail); 1542 } 1543 1544 return 0; 1545 } 1546 1547 static void bdrv_padding_destroy(BdrvRequestPadding *pad) 1548 { 1549 if (pad->buf) { 1550 qemu_vfree(pad->buf); 1551 qemu_iovec_destroy(&pad->local_qiov); 1552 } 1553 memset(pad, 0, sizeof(*pad)); 1554 } 1555 1556 /* 1557 * bdrv_pad_request 1558 * 1559 * Exchange request parameters with padded request if needed. Don't include RMW 1560 * read of padding, bdrv_padding_rmw_read() should be called separately if 1561 * needed. 1562 * 1563 * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out: 1564 * - on function start they represent original request 1565 * - on failure or when padding is not needed they are unchanged 1566 * - on success when padding is needed they represent padded request 1567 */ 1568 static int bdrv_pad_request(BlockDriverState *bs, 1569 QEMUIOVector **qiov, size_t *qiov_offset, 1570 int64_t *offset, int64_t *bytes, 1571 BdrvRequestPadding *pad, bool *padded, 1572 BdrvRequestFlags *flags) 1573 { 1574 int ret; 1575 1576 bdrv_check_qiov_request(*offset, *bytes, *qiov, *qiov_offset, &error_abort); 1577 1578 if (!bdrv_init_padding(bs, *offset, *bytes, pad)) { 1579 if (padded) { 1580 *padded = false; 1581 } 1582 return 0; 1583 } 1584 1585 ret = qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head, 1586 *qiov, *qiov_offset, *bytes, 1587 pad->buf + pad->buf_len - pad->tail, 1588 pad->tail); 1589 if (ret < 0) { 1590 bdrv_padding_destroy(pad); 1591 return ret; 1592 } 1593 *bytes += pad->head + pad->tail; 1594 *offset -= pad->head; 1595 *qiov = &pad->local_qiov; 1596 *qiov_offset = 0; 1597 if (padded) { 1598 *padded = true; 1599 } 1600 if (flags) { 1601 /* Can't use optimization hint with bounce buffer */ 1602 *flags &= ~BDRV_REQ_REGISTERED_BUF; 1603 } 1604 1605 return 0; 1606 } 1607 1608 int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1609 int64_t offset, int64_t bytes, QEMUIOVector *qiov, 1610 BdrvRequestFlags flags) 1611 { 1612 IO_CODE(); 1613 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags); 1614 } 1615 1616 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child, 1617 int64_t offset, int64_t bytes, 1618 QEMUIOVector *qiov, size_t qiov_offset, 1619 BdrvRequestFlags flags) 1620 { 1621 BlockDriverState *bs = child->bs; 1622 BdrvTrackedRequest req; 1623 BdrvRequestPadding pad; 1624 int ret; 1625 IO_CODE(); 1626 1627 trace_bdrv_co_preadv_part(bs, offset, bytes, flags); 1628 1629 if (!bdrv_co_is_inserted(bs)) { 1630 return -ENOMEDIUM; 1631 } 1632 1633 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 1634 if (ret < 0) { 1635 return ret; 1636 } 1637 1638 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 1639 /* 1640 * Aligning zero request is nonsense. Even if driver has special meaning 1641 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 1642 * it to driver due to request_alignment. 1643 * 1644 * Still, no reason to return an error if someone do unaligned 1645 * zero-length read occasionally. 1646 */ 1647 return 0; 1648 } 1649 1650 bdrv_inc_in_flight(bs); 1651 1652 /* Don't do copy-on-read if we read data before write operation */ 1653 if (qatomic_read(&bs->copy_on_read)) { 1654 flags |= BDRV_REQ_COPY_ON_READ; 1655 } 1656 1657 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad, 1658 NULL, &flags); 1659 if (ret < 0) { 1660 goto fail; 1661 } 1662 1663 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1664 ret = bdrv_aligned_preadv(child, &req, offset, bytes, 1665 bs->bl.request_alignment, 1666 qiov, qiov_offset, flags); 1667 tracked_request_end(&req); 1668 bdrv_padding_destroy(&pad); 1669 1670 fail: 1671 bdrv_dec_in_flight(bs); 1672 1673 return ret; 1674 } 1675 1676 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1677 int64_t offset, int64_t bytes, BdrvRequestFlags flags) 1678 { 1679 BlockDriver *drv = bs->drv; 1680 QEMUIOVector qiov; 1681 void *buf = NULL; 1682 int ret = 0; 1683 bool need_flush = false; 1684 int head = 0; 1685 int tail = 0; 1686 1687 assume_graph_lock(); /* FIXME */ 1688 1689 int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, 1690 INT64_MAX); 1691 int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1692 bs->bl.request_alignment); 1693 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1694 1695 assert_bdrv_graph_readable(); 1696 bdrv_check_request(offset, bytes, &error_abort); 1697 1698 if (!drv) { 1699 return -ENOMEDIUM; 1700 } 1701 1702 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) { 1703 return -ENOTSUP; 1704 } 1705 1706 /* By definition there is no user buffer so this flag doesn't make sense */ 1707 if (flags & BDRV_REQ_REGISTERED_BUF) { 1708 return -EINVAL; 1709 } 1710 1711 /* Invalidate the cached block-status data range if this write overlaps */ 1712 bdrv_bsc_invalidate_range(bs, offset, bytes); 1713 1714 assert(alignment % bs->bl.request_alignment == 0); 1715 head = offset % alignment; 1716 tail = (offset + bytes) % alignment; 1717 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1718 assert(max_write_zeroes >= bs->bl.request_alignment); 1719 1720 while (bytes > 0 && !ret) { 1721 int64_t num = bytes; 1722 1723 /* Align request. Block drivers can expect the "bulk" of the request 1724 * to be aligned, and that unaligned requests do not cross cluster 1725 * boundaries. 1726 */ 1727 if (head) { 1728 /* Make a small request up to the first aligned sector. For 1729 * convenience, limit this request to max_transfer even if 1730 * we don't need to fall back to writes. */ 1731 num = MIN(MIN(bytes, max_transfer), alignment - head); 1732 head = (head + num) % alignment; 1733 assert(num < max_write_zeroes); 1734 } else if (tail && num > alignment) { 1735 /* Shorten the request to the last aligned sector. */ 1736 num -= tail; 1737 } 1738 1739 /* limit request size */ 1740 if (num > max_write_zeroes) { 1741 num = max_write_zeroes; 1742 } 1743 1744 ret = -ENOTSUP; 1745 /* First try the efficient write zeroes operation */ 1746 if (drv->bdrv_co_pwrite_zeroes) { 1747 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1748 flags & bs->supported_zero_flags); 1749 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1750 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1751 need_flush = true; 1752 } 1753 } else { 1754 assert(!bs->supported_zero_flags); 1755 } 1756 1757 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) { 1758 /* Fall back to bounce buffer if write zeroes is unsupported */ 1759 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1760 1761 if ((flags & BDRV_REQ_FUA) && 1762 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1763 /* No need for bdrv_driver_pwrite() to do a fallback 1764 * flush on each chunk; use just one at the end */ 1765 write_flags &= ~BDRV_REQ_FUA; 1766 need_flush = true; 1767 } 1768 num = MIN(num, max_transfer); 1769 if (buf == NULL) { 1770 buf = qemu_try_blockalign0(bs, num); 1771 if (buf == NULL) { 1772 ret = -ENOMEM; 1773 goto fail; 1774 } 1775 } 1776 qemu_iovec_init_buf(&qiov, buf, num); 1777 1778 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags); 1779 1780 /* Keep bounce buffer around if it is big enough for all 1781 * all future requests. 1782 */ 1783 if (num < max_transfer) { 1784 qemu_vfree(buf); 1785 buf = NULL; 1786 } 1787 } 1788 1789 offset += num; 1790 bytes -= num; 1791 } 1792 1793 fail: 1794 if (ret == 0 && need_flush) { 1795 ret = bdrv_co_flush(bs); 1796 } 1797 qemu_vfree(buf); 1798 return ret; 1799 } 1800 1801 static inline int coroutine_fn 1802 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes, 1803 BdrvTrackedRequest *req, int flags) 1804 { 1805 BlockDriverState *bs = child->bs; 1806 1807 bdrv_check_request(offset, bytes, &error_abort); 1808 1809 if (bdrv_is_read_only(bs)) { 1810 return -EPERM; 1811 } 1812 1813 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1814 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1815 assert(!(flags & ~BDRV_REQ_MASK)); 1816 assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING))); 1817 1818 if (flags & BDRV_REQ_SERIALISING) { 1819 QEMU_LOCK_GUARD(&bs->reqs_lock); 1820 1821 tracked_request_set_serialising(req, bdrv_get_cluster_size(bs)); 1822 1823 if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) { 1824 return -EBUSY; 1825 } 1826 1827 bdrv_wait_serialising_requests_locked(req); 1828 } else { 1829 bdrv_wait_serialising_requests(req); 1830 } 1831 1832 assert(req->overlap_offset <= offset); 1833 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1834 assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE || 1835 child->perm & BLK_PERM_RESIZE); 1836 1837 switch (req->type) { 1838 case BDRV_TRACKED_WRITE: 1839 case BDRV_TRACKED_DISCARD: 1840 if (flags & BDRV_REQ_WRITE_UNCHANGED) { 1841 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1842 } else { 1843 assert(child->perm & BLK_PERM_WRITE); 1844 } 1845 bdrv_write_threshold_check_write(bs, offset, bytes); 1846 return 0; 1847 case BDRV_TRACKED_TRUNCATE: 1848 assert(child->perm & BLK_PERM_RESIZE); 1849 return 0; 1850 default: 1851 abort(); 1852 } 1853 } 1854 1855 static inline void coroutine_fn 1856 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes, 1857 BdrvTrackedRequest *req, int ret) 1858 { 1859 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1860 BlockDriverState *bs = child->bs; 1861 1862 bdrv_check_request(offset, bytes, &error_abort); 1863 1864 qatomic_inc(&bs->write_gen); 1865 1866 /* 1867 * Discard cannot extend the image, but in error handling cases, such as 1868 * when reverting a qcow2 cluster allocation, the discarded range can pass 1869 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD 1870 * here. Instead, just skip it, since semantically a discard request 1871 * beyond EOF cannot expand the image anyway. 1872 */ 1873 if (ret == 0 && 1874 (req->type == BDRV_TRACKED_TRUNCATE || 1875 end_sector > bs->total_sectors) && 1876 req->type != BDRV_TRACKED_DISCARD) { 1877 bs->total_sectors = end_sector; 1878 bdrv_parent_cb_resize(bs); 1879 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS); 1880 } 1881 if (req->bytes) { 1882 switch (req->type) { 1883 case BDRV_TRACKED_WRITE: 1884 stat64_max(&bs->wr_highest_offset, offset + bytes); 1885 /* fall through, to set dirty bits */ 1886 case BDRV_TRACKED_DISCARD: 1887 bdrv_set_dirty(bs, offset, bytes); 1888 break; 1889 default: 1890 break; 1891 } 1892 } 1893 } 1894 1895 /* 1896 * Forwards an already correctly aligned write request to the BlockDriver, 1897 * after possibly fragmenting it. 1898 */ 1899 static int coroutine_fn GRAPH_RDLOCK 1900 bdrv_aligned_pwritev(BdrvChild *child, BdrvTrackedRequest *req, 1901 int64_t offset, int64_t bytes, int64_t align, 1902 QEMUIOVector *qiov, size_t qiov_offset, 1903 BdrvRequestFlags flags) 1904 { 1905 BlockDriverState *bs = child->bs; 1906 BlockDriver *drv = bs->drv; 1907 int ret; 1908 1909 int64_t bytes_remaining = bytes; 1910 int max_transfer; 1911 1912 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1913 1914 if (!drv) { 1915 return -ENOMEDIUM; 1916 } 1917 1918 if (bdrv_has_readonly_bitmaps(bs)) { 1919 return -EPERM; 1920 } 1921 1922 assert(is_power_of_2(align)); 1923 assert((offset & (align - 1)) == 0); 1924 assert((bytes & (align - 1)) == 0); 1925 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1926 align); 1927 1928 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags); 1929 1930 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1931 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 1932 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) { 1933 flags |= BDRV_REQ_ZERO_WRITE; 1934 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 1935 flags |= BDRV_REQ_MAY_UNMAP; 1936 } 1937 1938 /* Can't use optimization hint with bufferless zero write */ 1939 flags &= ~BDRV_REQ_REGISTERED_BUF; 1940 } 1941 1942 if (ret < 0) { 1943 /* Do nothing, write notifier decided to fail this request */ 1944 } else if (flags & BDRV_REQ_ZERO_WRITE) { 1945 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_ZERO); 1946 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 1947 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 1948 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, 1949 qiov, qiov_offset); 1950 } else if (bytes <= max_transfer) { 1951 bdrv_co_debug_event(bs, BLKDBG_PWRITEV); 1952 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags); 1953 } else { 1954 bdrv_co_debug_event(bs, BLKDBG_PWRITEV); 1955 while (bytes_remaining) { 1956 int num = MIN(bytes_remaining, max_transfer); 1957 int local_flags = flags; 1958 1959 assert(num); 1960 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 1961 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1962 /* If FUA is going to be emulated by flush, we only 1963 * need to flush on the last iteration */ 1964 local_flags &= ~BDRV_REQ_FUA; 1965 } 1966 1967 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 1968 num, qiov, 1969 qiov_offset + bytes - bytes_remaining, 1970 local_flags); 1971 if (ret < 0) { 1972 break; 1973 } 1974 bytes_remaining -= num; 1975 } 1976 } 1977 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_DONE); 1978 1979 if (ret >= 0) { 1980 ret = 0; 1981 } 1982 bdrv_co_write_req_finish(child, offset, bytes, req, ret); 1983 1984 return ret; 1985 } 1986 1987 static int coroutine_fn GRAPH_RDLOCK 1988 bdrv_co_do_zero_pwritev(BdrvChild *child, int64_t offset, int64_t bytes, 1989 BdrvRequestFlags flags, BdrvTrackedRequest *req) 1990 { 1991 BlockDriverState *bs = child->bs; 1992 QEMUIOVector local_qiov; 1993 uint64_t align = bs->bl.request_alignment; 1994 int ret = 0; 1995 bool padding; 1996 BdrvRequestPadding pad; 1997 1998 /* This flag doesn't make sense for padding or zero writes */ 1999 flags &= ~BDRV_REQ_REGISTERED_BUF; 2000 2001 padding = bdrv_init_padding(bs, offset, bytes, &pad); 2002 if (padding) { 2003 assert(!(flags & BDRV_REQ_NO_WAIT)); 2004 bdrv_make_request_serialising(req, align); 2005 2006 bdrv_padding_rmw_read(child, req, &pad, true); 2007 2008 if (pad.head || pad.merge_reads) { 2009 int64_t aligned_offset = offset & ~(align - 1); 2010 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align; 2011 2012 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes); 2013 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes, 2014 align, &local_qiov, 0, 2015 flags & ~BDRV_REQ_ZERO_WRITE); 2016 if (ret < 0 || pad.merge_reads) { 2017 /* Error or all work is done */ 2018 goto out; 2019 } 2020 offset += write_bytes - pad.head; 2021 bytes -= write_bytes - pad.head; 2022 } 2023 } 2024 2025 assert(!bytes || (offset & (align - 1)) == 0); 2026 if (bytes >= align) { 2027 /* Write the aligned part in the middle. */ 2028 int64_t aligned_bytes = bytes & ~(align - 1); 2029 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 2030 NULL, 0, flags); 2031 if (ret < 0) { 2032 goto out; 2033 } 2034 bytes -= aligned_bytes; 2035 offset += aligned_bytes; 2036 } 2037 2038 assert(!bytes || (offset & (align - 1)) == 0); 2039 if (bytes) { 2040 assert(align == pad.tail + bytes); 2041 2042 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align); 2043 ret = bdrv_aligned_pwritev(child, req, offset, align, align, 2044 &local_qiov, 0, 2045 flags & ~BDRV_REQ_ZERO_WRITE); 2046 } 2047 2048 out: 2049 bdrv_padding_destroy(&pad); 2050 2051 return ret; 2052 } 2053 2054 /* 2055 * Handle a write request in coroutine context 2056 */ 2057 int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 2058 int64_t offset, int64_t bytes, QEMUIOVector *qiov, 2059 BdrvRequestFlags flags) 2060 { 2061 IO_CODE(); 2062 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags); 2063 } 2064 2065 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child, 2066 int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset, 2067 BdrvRequestFlags flags) 2068 { 2069 BlockDriverState *bs = child->bs; 2070 BdrvTrackedRequest req; 2071 uint64_t align = bs->bl.request_alignment; 2072 BdrvRequestPadding pad; 2073 int ret; 2074 bool padded = false; 2075 IO_CODE(); 2076 2077 trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags); 2078 2079 if (!bdrv_co_is_inserted(bs)) { 2080 return -ENOMEDIUM; 2081 } 2082 2083 if (flags & BDRV_REQ_ZERO_WRITE) { 2084 ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 2085 } else { 2086 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 2087 } 2088 if (ret < 0) { 2089 return ret; 2090 } 2091 2092 /* If the request is misaligned then we can't make it efficient */ 2093 if ((flags & BDRV_REQ_NO_FALLBACK) && 2094 !QEMU_IS_ALIGNED(offset | bytes, align)) 2095 { 2096 return -ENOTSUP; 2097 } 2098 2099 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 2100 /* 2101 * Aligning zero request is nonsense. Even if driver has special meaning 2102 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 2103 * it to driver due to request_alignment. 2104 * 2105 * Still, no reason to return an error if someone do unaligned 2106 * zero-length write occasionally. 2107 */ 2108 return 0; 2109 } 2110 2111 if (!(flags & BDRV_REQ_ZERO_WRITE)) { 2112 /* 2113 * Pad request for following read-modify-write cycle. 2114 * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do 2115 * alignment only if there is no ZERO flag. 2116 */ 2117 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad, 2118 &padded, &flags); 2119 if (ret < 0) { 2120 return ret; 2121 } 2122 } 2123 2124 bdrv_inc_in_flight(bs); 2125 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 2126 2127 if (flags & BDRV_REQ_ZERO_WRITE) { 2128 assert(!padded); 2129 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 2130 goto out; 2131 } 2132 2133 if (padded) { 2134 /* 2135 * Request was unaligned to request_alignment and therefore 2136 * padded. We are going to do read-modify-write, and must 2137 * serialize the request to prevent interactions of the 2138 * widened region with other transactions. 2139 */ 2140 assert(!(flags & BDRV_REQ_NO_WAIT)); 2141 bdrv_make_request_serialising(&req, align); 2142 bdrv_padding_rmw_read(child, &req, &pad, false); 2143 } 2144 2145 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 2146 qiov, qiov_offset, flags); 2147 2148 bdrv_padding_destroy(&pad); 2149 2150 out: 2151 tracked_request_end(&req); 2152 bdrv_dec_in_flight(bs); 2153 2154 return ret; 2155 } 2156 2157 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 2158 int64_t bytes, BdrvRequestFlags flags) 2159 { 2160 IO_CODE(); 2161 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 2162 assert_bdrv_graph_readable(); 2163 2164 if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 2165 flags &= ~BDRV_REQ_MAY_UNMAP; 2166 } 2167 2168 return bdrv_co_pwritev(child, offset, bytes, NULL, 2169 BDRV_REQ_ZERO_WRITE | flags); 2170 } 2171 2172 /* 2173 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 2174 */ 2175 int bdrv_flush_all(void) 2176 { 2177 BdrvNextIterator it; 2178 BlockDriverState *bs = NULL; 2179 int result = 0; 2180 2181 GLOBAL_STATE_CODE(); 2182 2183 /* 2184 * bdrv queue is managed by record/replay, 2185 * creating new flush request for stopping 2186 * the VM may break the determinism 2187 */ 2188 if (replay_events_enabled()) { 2189 return result; 2190 } 2191 2192 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 2193 AioContext *aio_context = bdrv_get_aio_context(bs); 2194 int ret; 2195 2196 aio_context_acquire(aio_context); 2197 ret = bdrv_flush(bs); 2198 if (ret < 0 && !result) { 2199 result = ret; 2200 } 2201 aio_context_release(aio_context); 2202 } 2203 2204 return result; 2205 } 2206 2207 /* 2208 * Returns the allocation status of the specified sectors. 2209 * Drivers not implementing the functionality are assumed to not support 2210 * backing files, hence all their sectors are reported as allocated. 2211 * 2212 * If 'want_zero' is true, the caller is querying for mapping 2213 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 2214 * _ZERO where possible; otherwise, the result favors larger 'pnum', 2215 * with a focus on accurate BDRV_BLOCK_ALLOCATED. 2216 * 2217 * If 'offset' is beyond the end of the disk image the return value is 2218 * BDRV_BLOCK_EOF and 'pnum' is set to 0. 2219 * 2220 * 'bytes' is the max value 'pnum' should be set to. If bytes goes 2221 * beyond the end of the disk image it will be clamped; if 'pnum' is set to 2222 * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 2223 * 2224 * 'pnum' is set to the number of bytes (including and immediately 2225 * following the specified offset) that are easily known to be in the 2226 * same allocated/unallocated state. Note that a second call starting 2227 * at the original offset plus returned pnum may have the same status. 2228 * The returned value is non-zero on success except at end-of-file. 2229 * 2230 * Returns negative errno on failure. Otherwise, if the 2231 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 2232 * set to the host mapping and BDS corresponding to the guest offset. 2233 */ 2234 static int coroutine_fn GRAPH_RDLOCK 2235 bdrv_co_block_status(BlockDriverState *bs, bool want_zero, 2236 int64_t offset, int64_t bytes, 2237 int64_t *pnum, int64_t *map, BlockDriverState **file) 2238 { 2239 int64_t total_size; 2240 int64_t n; /* bytes */ 2241 int ret; 2242 int64_t local_map = 0; 2243 BlockDriverState *local_file = NULL; 2244 int64_t aligned_offset, aligned_bytes; 2245 uint32_t align; 2246 bool has_filtered_child; 2247 2248 assert(pnum); 2249 assert_bdrv_graph_readable(); 2250 *pnum = 0; 2251 total_size = bdrv_getlength(bs); 2252 if (total_size < 0) { 2253 ret = total_size; 2254 goto early_out; 2255 } 2256 2257 if (offset >= total_size) { 2258 ret = BDRV_BLOCK_EOF; 2259 goto early_out; 2260 } 2261 if (!bytes) { 2262 ret = 0; 2263 goto early_out; 2264 } 2265 2266 n = total_size - offset; 2267 if (n < bytes) { 2268 bytes = n; 2269 } 2270 2271 /* Must be non-NULL or bdrv_getlength() would have failed */ 2272 assert(bs->drv); 2273 has_filtered_child = bdrv_filter_child(bs); 2274 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) { 2275 *pnum = bytes; 2276 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 2277 if (offset + bytes == total_size) { 2278 ret |= BDRV_BLOCK_EOF; 2279 } 2280 if (bs->drv->protocol_name) { 2281 ret |= BDRV_BLOCK_OFFSET_VALID; 2282 local_map = offset; 2283 local_file = bs; 2284 } 2285 goto early_out; 2286 } 2287 2288 bdrv_inc_in_flight(bs); 2289 2290 /* Round out to request_alignment boundaries */ 2291 align = bs->bl.request_alignment; 2292 aligned_offset = QEMU_ALIGN_DOWN(offset, align); 2293 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 2294 2295 if (bs->drv->bdrv_co_block_status) { 2296 /* 2297 * Use the block-status cache only for protocol nodes: Format 2298 * drivers are generally quick to inquire the status, but protocol 2299 * drivers often need to get information from outside of qemu, so 2300 * we do not have control over the actual implementation. There 2301 * have been cases where inquiring the status took an unreasonably 2302 * long time, and we can do nothing in qemu to fix it. 2303 * This is especially problematic for images with large data areas, 2304 * because finding the few holes in them and giving them special 2305 * treatment does not gain much performance. Therefore, we try to 2306 * cache the last-identified data region. 2307 * 2308 * Second, limiting ourselves to protocol nodes allows us to assume 2309 * the block status for data regions to be DATA | OFFSET_VALID, and 2310 * that the host offset is the same as the guest offset. 2311 * 2312 * Note that it is possible that external writers zero parts of 2313 * the cached regions without the cache being invalidated, and so 2314 * we may report zeroes as data. This is not catastrophic, 2315 * however, because reporting zeroes as data is fine. 2316 */ 2317 if (QLIST_EMPTY(&bs->children) && 2318 bdrv_bsc_is_data(bs, aligned_offset, pnum)) 2319 { 2320 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID; 2321 local_file = bs; 2322 local_map = aligned_offset; 2323 } else { 2324 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 2325 aligned_bytes, pnum, &local_map, 2326 &local_file); 2327 2328 /* 2329 * Note that checking QLIST_EMPTY(&bs->children) is also done when 2330 * the cache is queried above. Technically, we do not need to check 2331 * it here; the worst that can happen is that we fill the cache for 2332 * non-protocol nodes, and then it is never used. However, filling 2333 * the cache requires an RCU update, so double check here to avoid 2334 * such an update if possible. 2335 * 2336 * Check want_zero, because we only want to update the cache when we 2337 * have accurate information about what is zero and what is data. 2338 */ 2339 if (want_zero && 2340 ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) && 2341 QLIST_EMPTY(&bs->children)) 2342 { 2343 /* 2344 * When a protocol driver reports BLOCK_OFFSET_VALID, the 2345 * returned local_map value must be the same as the offset we 2346 * have passed (aligned_offset), and local_bs must be the node 2347 * itself. 2348 * Assert this, because we follow this rule when reading from 2349 * the cache (see the `local_file = bs` and 2350 * `local_map = aligned_offset` assignments above), and the 2351 * result the cache delivers must be the same as the driver 2352 * would deliver. 2353 */ 2354 assert(local_file == bs); 2355 assert(local_map == aligned_offset); 2356 bdrv_bsc_fill(bs, aligned_offset, *pnum); 2357 } 2358 } 2359 } else { 2360 /* Default code for filters */ 2361 2362 local_file = bdrv_filter_bs(bs); 2363 assert(local_file); 2364 2365 *pnum = aligned_bytes; 2366 local_map = aligned_offset; 2367 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2368 } 2369 if (ret < 0) { 2370 *pnum = 0; 2371 goto out; 2372 } 2373 2374 /* 2375 * The driver's result must be a non-zero multiple of request_alignment. 2376 * Clamp pnum and adjust map to original request. 2377 */ 2378 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2379 align > offset - aligned_offset); 2380 if (ret & BDRV_BLOCK_RECURSE) { 2381 assert(ret & BDRV_BLOCK_DATA); 2382 assert(ret & BDRV_BLOCK_OFFSET_VALID); 2383 assert(!(ret & BDRV_BLOCK_ZERO)); 2384 } 2385 2386 *pnum -= offset - aligned_offset; 2387 if (*pnum > bytes) { 2388 *pnum = bytes; 2389 } 2390 if (ret & BDRV_BLOCK_OFFSET_VALID) { 2391 local_map += offset - aligned_offset; 2392 } 2393 2394 if (ret & BDRV_BLOCK_RAW) { 2395 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 2396 ret = bdrv_co_block_status(local_file, want_zero, local_map, 2397 *pnum, pnum, &local_map, &local_file); 2398 goto out; 2399 } 2400 2401 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 2402 ret |= BDRV_BLOCK_ALLOCATED; 2403 } else if (bs->drv->supports_backing) { 2404 BlockDriverState *cow_bs = bdrv_cow_bs(bs); 2405 2406 if (!cow_bs) { 2407 ret |= BDRV_BLOCK_ZERO; 2408 } else if (want_zero) { 2409 int64_t size2 = bdrv_getlength(cow_bs); 2410 2411 if (size2 >= 0 && offset >= size2) { 2412 ret |= BDRV_BLOCK_ZERO; 2413 } 2414 } 2415 } 2416 2417 if (want_zero && ret & BDRV_BLOCK_RECURSE && 2418 local_file && local_file != bs && 2419 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 2420 (ret & BDRV_BLOCK_OFFSET_VALID)) { 2421 int64_t file_pnum; 2422 int ret2; 2423 2424 ret2 = bdrv_co_block_status(local_file, want_zero, local_map, 2425 *pnum, &file_pnum, NULL, NULL); 2426 if (ret2 >= 0) { 2427 /* Ignore errors. This is just providing extra information, it 2428 * is useful but not necessary. 2429 */ 2430 if (ret2 & BDRV_BLOCK_EOF && 2431 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2432 /* 2433 * It is valid for the format block driver to read 2434 * beyond the end of the underlying file's current 2435 * size; such areas read as zero. 2436 */ 2437 ret |= BDRV_BLOCK_ZERO; 2438 } else { 2439 /* Limit request to the range reported by the protocol driver */ 2440 *pnum = file_pnum; 2441 ret |= (ret2 & BDRV_BLOCK_ZERO); 2442 } 2443 } 2444 } 2445 2446 out: 2447 bdrv_dec_in_flight(bs); 2448 if (ret >= 0 && offset + *pnum == total_size) { 2449 ret |= BDRV_BLOCK_EOF; 2450 } 2451 early_out: 2452 if (file) { 2453 *file = local_file; 2454 } 2455 if (map) { 2456 *map = local_map; 2457 } 2458 return ret; 2459 } 2460 2461 int coroutine_fn 2462 bdrv_co_common_block_status_above(BlockDriverState *bs, 2463 BlockDriverState *base, 2464 bool include_base, 2465 bool want_zero, 2466 int64_t offset, 2467 int64_t bytes, 2468 int64_t *pnum, 2469 int64_t *map, 2470 BlockDriverState **file, 2471 int *depth) 2472 { 2473 int ret; 2474 BlockDriverState *p; 2475 int64_t eof = 0; 2476 int dummy; 2477 IO_CODE(); 2478 2479 assert(!include_base || base); /* Can't include NULL base */ 2480 assert_bdrv_graph_readable(); 2481 2482 if (!depth) { 2483 depth = &dummy; 2484 } 2485 *depth = 0; 2486 2487 if (!include_base && bs == base) { 2488 *pnum = bytes; 2489 return 0; 2490 } 2491 2492 ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file); 2493 ++*depth; 2494 if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) { 2495 return ret; 2496 } 2497 2498 if (ret & BDRV_BLOCK_EOF) { 2499 eof = offset + *pnum; 2500 } 2501 2502 assert(*pnum <= bytes); 2503 bytes = *pnum; 2504 2505 for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base; 2506 p = bdrv_filter_or_cow_bs(p)) 2507 { 2508 ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map, 2509 file); 2510 ++*depth; 2511 if (ret < 0) { 2512 return ret; 2513 } 2514 if (*pnum == 0) { 2515 /* 2516 * The top layer deferred to this layer, and because this layer is 2517 * short, any zeroes that we synthesize beyond EOF behave as if they 2518 * were allocated at this layer. 2519 * 2520 * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be 2521 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2522 * below. 2523 */ 2524 assert(ret & BDRV_BLOCK_EOF); 2525 *pnum = bytes; 2526 if (file) { 2527 *file = p; 2528 } 2529 ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED; 2530 break; 2531 } 2532 if (ret & BDRV_BLOCK_ALLOCATED) { 2533 /* 2534 * We've found the node and the status, we must break. 2535 * 2536 * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be 2537 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2538 * below. 2539 */ 2540 ret &= ~BDRV_BLOCK_EOF; 2541 break; 2542 } 2543 2544 if (p == base) { 2545 assert(include_base); 2546 break; 2547 } 2548 2549 /* 2550 * OK, [offset, offset + *pnum) region is unallocated on this layer, 2551 * let's continue the diving. 2552 */ 2553 assert(*pnum <= bytes); 2554 bytes = *pnum; 2555 } 2556 2557 if (offset + *pnum == eof) { 2558 ret |= BDRV_BLOCK_EOF; 2559 } 2560 2561 return ret; 2562 } 2563 2564 int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs, 2565 BlockDriverState *base, 2566 int64_t offset, int64_t bytes, 2567 int64_t *pnum, int64_t *map, 2568 BlockDriverState **file) 2569 { 2570 IO_CODE(); 2571 return bdrv_co_common_block_status_above(bs, base, false, true, offset, 2572 bytes, pnum, map, file, NULL); 2573 } 2574 2575 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base, 2576 int64_t offset, int64_t bytes, int64_t *pnum, 2577 int64_t *map, BlockDriverState **file) 2578 { 2579 IO_CODE(); 2580 return bdrv_common_block_status_above(bs, base, false, true, offset, bytes, 2581 pnum, map, file, NULL); 2582 } 2583 2584 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2585 int64_t *pnum, int64_t *map, BlockDriverState **file) 2586 { 2587 IO_CODE(); 2588 return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs), 2589 offset, bytes, pnum, map, file); 2590 } 2591 2592 /* 2593 * Check @bs (and its backing chain) to see if the range defined 2594 * by @offset and @bytes is known to read as zeroes. 2595 * Return 1 if that is the case, 0 otherwise and -errno on error. 2596 * This test is meant to be fast rather than accurate so returning 0 2597 * does not guarantee non-zero data. 2598 */ 2599 int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset, 2600 int64_t bytes) 2601 { 2602 int ret; 2603 int64_t pnum = bytes; 2604 IO_CODE(); 2605 2606 if (!bytes) { 2607 return 1; 2608 } 2609 2610 ret = bdrv_co_common_block_status_above(bs, NULL, false, false, offset, 2611 bytes, &pnum, NULL, NULL, NULL); 2612 2613 if (ret < 0) { 2614 return ret; 2615 } 2616 2617 return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO); 2618 } 2619 2620 int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t offset, 2621 int64_t bytes, int64_t *pnum) 2622 { 2623 int ret; 2624 int64_t dummy; 2625 IO_CODE(); 2626 2627 ret = bdrv_co_common_block_status_above(bs, bs, true, false, offset, 2628 bytes, pnum ? pnum : &dummy, NULL, 2629 NULL, NULL); 2630 if (ret < 0) { 2631 return ret; 2632 } 2633 return !!(ret & BDRV_BLOCK_ALLOCATED); 2634 } 2635 2636 int bdrv_is_allocated(BlockDriverState *bs, int64_t offset, int64_t bytes, 2637 int64_t *pnum) 2638 { 2639 int ret; 2640 int64_t dummy; 2641 IO_CODE(); 2642 2643 ret = bdrv_common_block_status_above(bs, bs, true, false, offset, 2644 bytes, pnum ? pnum : &dummy, NULL, 2645 NULL, NULL); 2646 if (ret < 0) { 2647 return ret; 2648 } 2649 return !!(ret & BDRV_BLOCK_ALLOCATED); 2650 } 2651 2652 /* See bdrv_is_allocated_above for documentation */ 2653 int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top, 2654 BlockDriverState *base, 2655 bool include_base, int64_t offset, 2656 int64_t bytes, int64_t *pnum) 2657 { 2658 int depth; 2659 int ret; 2660 IO_CODE(); 2661 2662 ret = bdrv_co_common_block_status_above(top, base, include_base, false, 2663 offset, bytes, pnum, NULL, NULL, 2664 &depth); 2665 if (ret < 0) { 2666 return ret; 2667 } 2668 2669 if (ret & BDRV_BLOCK_ALLOCATED) { 2670 return depth; 2671 } 2672 return 0; 2673 } 2674 2675 /* 2676 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 2677 * 2678 * Return a positive depth if (a prefix of) the given range is allocated 2679 * in any image between BASE and TOP (BASE is only included if include_base 2680 * is set). Depth 1 is TOP, 2 is the first backing layer, and so forth. 2681 * BASE can be NULL to check if the given offset is allocated in any 2682 * image of the chain. Return 0 otherwise, or negative errno on 2683 * failure. 2684 * 2685 * 'pnum' is set to the number of bytes (including and immediately 2686 * following the specified offset) that are known to be in the same 2687 * allocated/unallocated state. Note that a subsequent call starting 2688 * at 'offset + *pnum' may return the same allocation status (in other 2689 * words, the result is not necessarily the maximum possible range); 2690 * but 'pnum' will only be 0 when end of file is reached. 2691 */ 2692 int bdrv_is_allocated_above(BlockDriverState *top, 2693 BlockDriverState *base, 2694 bool include_base, int64_t offset, 2695 int64_t bytes, int64_t *pnum) 2696 { 2697 int depth; 2698 int ret; 2699 IO_CODE(); 2700 2701 ret = bdrv_common_block_status_above(top, base, include_base, false, 2702 offset, bytes, pnum, NULL, NULL, 2703 &depth); 2704 if (ret < 0) { 2705 return ret; 2706 } 2707 2708 if (ret & BDRV_BLOCK_ALLOCATED) { 2709 return depth; 2710 } 2711 return 0; 2712 } 2713 2714 int coroutine_fn 2715 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2716 { 2717 BlockDriver *drv = bs->drv; 2718 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2719 int ret; 2720 IO_CODE(); 2721 assert_bdrv_graph_readable(); 2722 2723 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2724 if (ret < 0) { 2725 return ret; 2726 } 2727 2728 if (!drv) { 2729 return -ENOMEDIUM; 2730 } 2731 2732 bdrv_inc_in_flight(bs); 2733 2734 if (drv->bdrv_co_load_vmstate) { 2735 ret = drv->bdrv_co_load_vmstate(bs, qiov, pos); 2736 } else if (child_bs) { 2737 ret = bdrv_co_readv_vmstate(child_bs, qiov, pos); 2738 } else { 2739 ret = -ENOTSUP; 2740 } 2741 2742 bdrv_dec_in_flight(bs); 2743 2744 return ret; 2745 } 2746 2747 int coroutine_fn 2748 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2749 { 2750 BlockDriver *drv = bs->drv; 2751 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2752 int ret; 2753 IO_CODE(); 2754 assert_bdrv_graph_readable(); 2755 2756 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2757 if (ret < 0) { 2758 return ret; 2759 } 2760 2761 if (!drv) { 2762 return -ENOMEDIUM; 2763 } 2764 2765 bdrv_inc_in_flight(bs); 2766 2767 if (drv->bdrv_co_save_vmstate) { 2768 ret = drv->bdrv_co_save_vmstate(bs, qiov, pos); 2769 } else if (child_bs) { 2770 ret = bdrv_co_writev_vmstate(child_bs, qiov, pos); 2771 } else { 2772 ret = -ENOTSUP; 2773 } 2774 2775 bdrv_dec_in_flight(bs); 2776 2777 return ret; 2778 } 2779 2780 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 2781 int64_t pos, int size) 2782 { 2783 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2784 int ret = bdrv_writev_vmstate(bs, &qiov, pos); 2785 IO_CODE(); 2786 2787 return ret < 0 ? ret : size; 2788 } 2789 2790 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 2791 int64_t pos, int size) 2792 { 2793 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2794 int ret = bdrv_readv_vmstate(bs, &qiov, pos); 2795 IO_CODE(); 2796 2797 return ret < 0 ? ret : size; 2798 } 2799 2800 /**************************************************************/ 2801 /* async I/Os */ 2802 2803 void bdrv_aio_cancel(BlockAIOCB *acb) 2804 { 2805 IO_CODE(); 2806 qemu_aio_ref(acb); 2807 bdrv_aio_cancel_async(acb); 2808 while (acb->refcnt > 1) { 2809 if (acb->aiocb_info->get_aio_context) { 2810 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 2811 } else if (acb->bs) { 2812 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 2813 * assert that we're not using an I/O thread. Thread-safe 2814 * code should use bdrv_aio_cancel_async exclusively. 2815 */ 2816 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 2817 aio_poll(bdrv_get_aio_context(acb->bs), true); 2818 } else { 2819 abort(); 2820 } 2821 } 2822 qemu_aio_unref(acb); 2823 } 2824 2825 /* Async version of aio cancel. The caller is not blocked if the acb implements 2826 * cancel_async, otherwise we do nothing and let the request normally complete. 2827 * In either case the completion callback must be called. */ 2828 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2829 { 2830 IO_CODE(); 2831 if (acb->aiocb_info->cancel_async) { 2832 acb->aiocb_info->cancel_async(acb); 2833 } 2834 } 2835 2836 /**************************************************************/ 2837 /* Coroutine block device emulation */ 2838 2839 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2840 { 2841 BdrvChild *primary_child = bdrv_primary_child(bs); 2842 BdrvChild *child; 2843 int current_gen; 2844 int ret = 0; 2845 IO_CODE(); 2846 2847 assert_bdrv_graph_readable(); 2848 bdrv_inc_in_flight(bs); 2849 2850 if (!bdrv_co_is_inserted(bs) || bdrv_is_read_only(bs) || 2851 bdrv_is_sg(bs)) { 2852 goto early_exit; 2853 } 2854 2855 qemu_co_mutex_lock(&bs->reqs_lock); 2856 current_gen = qatomic_read(&bs->write_gen); 2857 2858 /* Wait until any previous flushes are completed */ 2859 while (bs->active_flush_req) { 2860 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 2861 } 2862 2863 /* Flushes reach this point in nondecreasing current_gen order. */ 2864 bs->active_flush_req = true; 2865 qemu_co_mutex_unlock(&bs->reqs_lock); 2866 2867 /* Write back all layers by calling one driver function */ 2868 if (bs->drv->bdrv_co_flush) { 2869 ret = bs->drv->bdrv_co_flush(bs); 2870 goto out; 2871 } 2872 2873 /* Write back cached data to the OS even with cache=unsafe */ 2874 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS); 2875 if (bs->drv->bdrv_co_flush_to_os) { 2876 ret = bs->drv->bdrv_co_flush_to_os(bs); 2877 if (ret < 0) { 2878 goto out; 2879 } 2880 } 2881 2882 /* But don't actually force it to the disk with cache=unsafe */ 2883 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2884 goto flush_children; 2885 } 2886 2887 /* Check if we really need to flush anything */ 2888 if (bs->flushed_gen == current_gen) { 2889 goto flush_children; 2890 } 2891 2892 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK); 2893 if (!bs->drv) { 2894 /* bs->drv->bdrv_co_flush() might have ejected the BDS 2895 * (even in case of apparent success) */ 2896 ret = -ENOMEDIUM; 2897 goto out; 2898 } 2899 if (bs->drv->bdrv_co_flush_to_disk) { 2900 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2901 } else if (bs->drv->bdrv_aio_flush) { 2902 BlockAIOCB *acb; 2903 CoroutineIOCompletion co = { 2904 .coroutine = qemu_coroutine_self(), 2905 }; 2906 2907 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 2908 if (acb == NULL) { 2909 ret = -EIO; 2910 } else { 2911 qemu_coroutine_yield(); 2912 ret = co.ret; 2913 } 2914 } else { 2915 /* 2916 * Some block drivers always operate in either writethrough or unsafe 2917 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 2918 * know how the server works (because the behaviour is hardcoded or 2919 * depends on server-side configuration), so we can't ensure that 2920 * everything is safe on disk. Returning an error doesn't work because 2921 * that would break guests even if the server operates in writethrough 2922 * mode. 2923 * 2924 * Let's hope the user knows what he's doing. 2925 */ 2926 ret = 0; 2927 } 2928 2929 if (ret < 0) { 2930 goto out; 2931 } 2932 2933 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 2934 * in the case of cache=unsafe, so there are no useless flushes. 2935 */ 2936 flush_children: 2937 ret = 0; 2938 QLIST_FOREACH(child, &bs->children, next) { 2939 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) { 2940 int this_child_ret = bdrv_co_flush(child->bs); 2941 if (!ret) { 2942 ret = this_child_ret; 2943 } 2944 } 2945 } 2946 2947 out: 2948 /* Notify any pending flushes that we have completed */ 2949 if (ret == 0) { 2950 bs->flushed_gen = current_gen; 2951 } 2952 2953 qemu_co_mutex_lock(&bs->reqs_lock); 2954 bs->active_flush_req = false; 2955 /* Return value is ignored - it's ok if wait queue is empty */ 2956 qemu_co_queue_next(&bs->flush_queue); 2957 qemu_co_mutex_unlock(&bs->reqs_lock); 2958 2959 early_exit: 2960 bdrv_dec_in_flight(bs); 2961 return ret; 2962 } 2963 2964 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset, 2965 int64_t bytes) 2966 { 2967 BdrvTrackedRequest req; 2968 int ret; 2969 int64_t max_pdiscard; 2970 int head, tail, align; 2971 BlockDriverState *bs = child->bs; 2972 IO_CODE(); 2973 assert_bdrv_graph_readable(); 2974 2975 if (!bs || !bs->drv || !bdrv_co_is_inserted(bs)) { 2976 return -ENOMEDIUM; 2977 } 2978 2979 if (bdrv_has_readonly_bitmaps(bs)) { 2980 return -EPERM; 2981 } 2982 2983 ret = bdrv_check_request(offset, bytes, NULL); 2984 if (ret < 0) { 2985 return ret; 2986 } 2987 2988 /* Do nothing if disabled. */ 2989 if (!(bs->open_flags & BDRV_O_UNMAP)) { 2990 return 0; 2991 } 2992 2993 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 2994 return 0; 2995 } 2996 2997 /* Invalidate the cached block-status data range if this discard overlaps */ 2998 bdrv_bsc_invalidate_range(bs, offset, bytes); 2999 3000 /* Discard is advisory, but some devices track and coalesce 3001 * unaligned requests, so we must pass everything down rather than 3002 * round here. Still, most devices will just silently ignore 3003 * unaligned requests (by returning -ENOTSUP), so we must fragment 3004 * the request accordingly. */ 3005 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 3006 assert(align % bs->bl.request_alignment == 0); 3007 head = offset % align; 3008 tail = (offset + bytes) % align; 3009 3010 bdrv_inc_in_flight(bs); 3011 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 3012 3013 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0); 3014 if (ret < 0) { 3015 goto out; 3016 } 3017 3018 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX), 3019 align); 3020 assert(max_pdiscard >= bs->bl.request_alignment); 3021 3022 while (bytes > 0) { 3023 int64_t num = bytes; 3024 3025 if (head) { 3026 /* Make small requests to get to alignment boundaries. */ 3027 num = MIN(bytes, align - head); 3028 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 3029 num %= bs->bl.request_alignment; 3030 } 3031 head = (head + num) % align; 3032 assert(num < max_pdiscard); 3033 } else if (tail) { 3034 if (num > align) { 3035 /* Shorten the request to the last aligned cluster. */ 3036 num -= tail; 3037 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 3038 tail > bs->bl.request_alignment) { 3039 tail %= bs->bl.request_alignment; 3040 num -= tail; 3041 } 3042 } 3043 /* limit request size */ 3044 if (num > max_pdiscard) { 3045 num = max_pdiscard; 3046 } 3047 3048 if (!bs->drv) { 3049 ret = -ENOMEDIUM; 3050 goto out; 3051 } 3052 if (bs->drv->bdrv_co_pdiscard) { 3053 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 3054 } else { 3055 BlockAIOCB *acb; 3056 CoroutineIOCompletion co = { 3057 .coroutine = qemu_coroutine_self(), 3058 }; 3059 3060 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 3061 bdrv_co_io_em_complete, &co); 3062 if (acb == NULL) { 3063 ret = -EIO; 3064 goto out; 3065 } else { 3066 qemu_coroutine_yield(); 3067 ret = co.ret; 3068 } 3069 } 3070 if (ret && ret != -ENOTSUP) { 3071 goto out; 3072 } 3073 3074 offset += num; 3075 bytes -= num; 3076 } 3077 ret = 0; 3078 out: 3079 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret); 3080 tracked_request_end(&req); 3081 bdrv_dec_in_flight(bs); 3082 return ret; 3083 } 3084 3085 int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 3086 { 3087 BlockDriver *drv = bs->drv; 3088 CoroutineIOCompletion co = { 3089 .coroutine = qemu_coroutine_self(), 3090 }; 3091 BlockAIOCB *acb; 3092 IO_CODE(); 3093 assert_bdrv_graph_readable(); 3094 3095 bdrv_inc_in_flight(bs); 3096 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 3097 co.ret = -ENOTSUP; 3098 goto out; 3099 } 3100 3101 if (drv->bdrv_co_ioctl) { 3102 co.ret = drv->bdrv_co_ioctl(bs, req, buf); 3103 } else { 3104 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 3105 if (!acb) { 3106 co.ret = -ENOTSUP; 3107 goto out; 3108 } 3109 qemu_coroutine_yield(); 3110 } 3111 out: 3112 bdrv_dec_in_flight(bs); 3113 return co.ret; 3114 } 3115 3116 void *qemu_blockalign(BlockDriverState *bs, size_t size) 3117 { 3118 IO_CODE(); 3119 return qemu_memalign(bdrv_opt_mem_align(bs), size); 3120 } 3121 3122 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 3123 { 3124 IO_CODE(); 3125 return memset(qemu_blockalign(bs, size), 0, size); 3126 } 3127 3128 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 3129 { 3130 size_t align = bdrv_opt_mem_align(bs); 3131 IO_CODE(); 3132 3133 /* Ensure that NULL is never returned on success */ 3134 assert(align > 0); 3135 if (size == 0) { 3136 size = align; 3137 } 3138 3139 return qemu_try_memalign(align, size); 3140 } 3141 3142 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 3143 { 3144 void *mem = qemu_try_blockalign(bs, size); 3145 IO_CODE(); 3146 3147 if (mem) { 3148 memset(mem, 0, size); 3149 } 3150 3151 return mem; 3152 } 3153 3154 void coroutine_fn bdrv_co_io_plug(BlockDriverState *bs) 3155 { 3156 BdrvChild *child; 3157 IO_CODE(); 3158 3159 QLIST_FOREACH(child, &bs->children, next) { 3160 bdrv_co_io_plug(child->bs); 3161 } 3162 3163 if (qatomic_fetch_inc(&bs->io_plugged) == 0) { 3164 BlockDriver *drv = bs->drv; 3165 if (drv && drv->bdrv_co_io_plug) { 3166 drv->bdrv_co_io_plug(bs); 3167 } 3168 } 3169 } 3170 3171 void coroutine_fn bdrv_co_io_unplug(BlockDriverState *bs) 3172 { 3173 BdrvChild *child; 3174 IO_CODE(); 3175 3176 assert(bs->io_plugged); 3177 if (qatomic_fetch_dec(&bs->io_plugged) == 1) { 3178 BlockDriver *drv = bs->drv; 3179 if (drv && drv->bdrv_co_io_unplug) { 3180 drv->bdrv_co_io_unplug(bs); 3181 } 3182 } 3183 3184 QLIST_FOREACH(child, &bs->children, next) { 3185 bdrv_co_io_unplug(child->bs); 3186 } 3187 } 3188 3189 /* Helper that undoes bdrv_register_buf() when it fails partway through */ 3190 static void bdrv_register_buf_rollback(BlockDriverState *bs, 3191 void *host, 3192 size_t size, 3193 BdrvChild *final_child) 3194 { 3195 BdrvChild *child; 3196 3197 QLIST_FOREACH(child, &bs->children, next) { 3198 if (child == final_child) { 3199 break; 3200 } 3201 3202 bdrv_unregister_buf(child->bs, host, size); 3203 } 3204 3205 if (bs->drv && bs->drv->bdrv_unregister_buf) { 3206 bs->drv->bdrv_unregister_buf(bs, host, size); 3207 } 3208 } 3209 3210 bool bdrv_register_buf(BlockDriverState *bs, void *host, size_t size, 3211 Error **errp) 3212 { 3213 BdrvChild *child; 3214 3215 GLOBAL_STATE_CODE(); 3216 if (bs->drv && bs->drv->bdrv_register_buf) { 3217 if (!bs->drv->bdrv_register_buf(bs, host, size, errp)) { 3218 return false; 3219 } 3220 } 3221 QLIST_FOREACH(child, &bs->children, next) { 3222 if (!bdrv_register_buf(child->bs, host, size, errp)) { 3223 bdrv_register_buf_rollback(bs, host, size, child); 3224 return false; 3225 } 3226 } 3227 return true; 3228 } 3229 3230 void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size) 3231 { 3232 BdrvChild *child; 3233 3234 GLOBAL_STATE_CODE(); 3235 if (bs->drv && bs->drv->bdrv_unregister_buf) { 3236 bs->drv->bdrv_unregister_buf(bs, host, size); 3237 } 3238 QLIST_FOREACH(child, &bs->children, next) { 3239 bdrv_unregister_buf(child->bs, host, size); 3240 } 3241 } 3242 3243 static int coroutine_fn GRAPH_RDLOCK bdrv_co_copy_range_internal( 3244 BdrvChild *src, int64_t src_offset, BdrvChild *dst, 3245 int64_t dst_offset, int64_t bytes, 3246 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags, 3247 bool recurse_src) 3248 { 3249 BdrvTrackedRequest req; 3250 int ret; 3251 3252 /* TODO We can support BDRV_REQ_NO_FALLBACK here */ 3253 assert(!(read_flags & BDRV_REQ_NO_FALLBACK)); 3254 assert(!(write_flags & BDRV_REQ_NO_FALLBACK)); 3255 assert(!(read_flags & BDRV_REQ_NO_WAIT)); 3256 assert(!(write_flags & BDRV_REQ_NO_WAIT)); 3257 3258 if (!dst || !dst->bs || !bdrv_co_is_inserted(dst->bs)) { 3259 return -ENOMEDIUM; 3260 } 3261 ret = bdrv_check_request32(dst_offset, bytes, NULL, 0); 3262 if (ret) { 3263 return ret; 3264 } 3265 if (write_flags & BDRV_REQ_ZERO_WRITE) { 3266 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags); 3267 } 3268 3269 if (!src || !src->bs || !bdrv_co_is_inserted(src->bs)) { 3270 return -ENOMEDIUM; 3271 } 3272 ret = bdrv_check_request32(src_offset, bytes, NULL, 0); 3273 if (ret) { 3274 return ret; 3275 } 3276 3277 if (!src->bs->drv->bdrv_co_copy_range_from 3278 || !dst->bs->drv->bdrv_co_copy_range_to 3279 || src->bs->encrypted || dst->bs->encrypted) { 3280 return -ENOTSUP; 3281 } 3282 3283 if (recurse_src) { 3284 bdrv_inc_in_flight(src->bs); 3285 tracked_request_begin(&req, src->bs, src_offset, bytes, 3286 BDRV_TRACKED_READ); 3287 3288 /* BDRV_REQ_SERIALISING is only for write operation */ 3289 assert(!(read_flags & BDRV_REQ_SERIALISING)); 3290 bdrv_wait_serialising_requests(&req); 3291 3292 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs, 3293 src, src_offset, 3294 dst, dst_offset, 3295 bytes, 3296 read_flags, write_flags); 3297 3298 tracked_request_end(&req); 3299 bdrv_dec_in_flight(src->bs); 3300 } else { 3301 bdrv_inc_in_flight(dst->bs); 3302 tracked_request_begin(&req, dst->bs, dst_offset, bytes, 3303 BDRV_TRACKED_WRITE); 3304 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req, 3305 write_flags); 3306 if (!ret) { 3307 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs, 3308 src, src_offset, 3309 dst, dst_offset, 3310 bytes, 3311 read_flags, write_flags); 3312 } 3313 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret); 3314 tracked_request_end(&req); 3315 bdrv_dec_in_flight(dst->bs); 3316 } 3317 3318 return ret; 3319 } 3320 3321 /* Copy range from @src to @dst. 3322 * 3323 * See the comment of bdrv_co_copy_range for the parameter and return value 3324 * semantics. */ 3325 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset, 3326 BdrvChild *dst, int64_t dst_offset, 3327 int64_t bytes, 3328 BdrvRequestFlags read_flags, 3329 BdrvRequestFlags write_flags) 3330 { 3331 IO_CODE(); 3332 assume_graph_lock(); /* FIXME */ 3333 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes, 3334 read_flags, write_flags); 3335 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3336 bytes, read_flags, write_flags, true); 3337 } 3338 3339 /* Copy range from @src to @dst. 3340 * 3341 * See the comment of bdrv_co_copy_range for the parameter and return value 3342 * semantics. */ 3343 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset, 3344 BdrvChild *dst, int64_t dst_offset, 3345 int64_t bytes, 3346 BdrvRequestFlags read_flags, 3347 BdrvRequestFlags write_flags) 3348 { 3349 IO_CODE(); 3350 assume_graph_lock(); /* FIXME */ 3351 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes, 3352 read_flags, write_flags); 3353 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3354 bytes, read_flags, write_flags, false); 3355 } 3356 3357 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset, 3358 BdrvChild *dst, int64_t dst_offset, 3359 int64_t bytes, BdrvRequestFlags read_flags, 3360 BdrvRequestFlags write_flags) 3361 { 3362 IO_CODE(); 3363 return bdrv_co_copy_range_from(src, src_offset, 3364 dst, dst_offset, 3365 bytes, read_flags, write_flags); 3366 } 3367 3368 static void bdrv_parent_cb_resize(BlockDriverState *bs) 3369 { 3370 BdrvChild *c; 3371 QLIST_FOREACH(c, &bs->parents, next_parent) { 3372 if (c->klass->resize) { 3373 c->klass->resize(c); 3374 } 3375 } 3376 } 3377 3378 /** 3379 * Truncate file to 'offset' bytes (needed only for file protocols) 3380 * 3381 * If 'exact' is true, the file must be resized to exactly the given 3382 * 'offset'. Otherwise, it is sufficient for the node to be at least 3383 * 'offset' bytes in length. 3384 */ 3385 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact, 3386 PreallocMode prealloc, BdrvRequestFlags flags, 3387 Error **errp) 3388 { 3389 BlockDriverState *bs = child->bs; 3390 BdrvChild *filtered, *backing; 3391 BlockDriver *drv = bs->drv; 3392 BdrvTrackedRequest req; 3393 int64_t old_size, new_bytes; 3394 int ret; 3395 IO_CODE(); 3396 assert_bdrv_graph_readable(); 3397 3398 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */ 3399 if (!drv) { 3400 error_setg(errp, "No medium inserted"); 3401 return -ENOMEDIUM; 3402 } 3403 if (offset < 0) { 3404 error_setg(errp, "Image size cannot be negative"); 3405 return -EINVAL; 3406 } 3407 3408 ret = bdrv_check_request(offset, 0, errp); 3409 if (ret < 0) { 3410 return ret; 3411 } 3412 3413 old_size = bdrv_getlength(bs); 3414 if (old_size < 0) { 3415 error_setg_errno(errp, -old_size, "Failed to get old image size"); 3416 return old_size; 3417 } 3418 3419 if (bdrv_is_read_only(bs)) { 3420 error_setg(errp, "Image is read-only"); 3421 return -EACCES; 3422 } 3423 3424 if (offset > old_size) { 3425 new_bytes = offset - old_size; 3426 } else { 3427 new_bytes = 0; 3428 } 3429 3430 bdrv_inc_in_flight(bs); 3431 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes, 3432 BDRV_TRACKED_TRUNCATE); 3433 3434 /* If we are growing the image and potentially using preallocation for the 3435 * new area, we need to make sure that no write requests are made to it 3436 * concurrently or they might be overwritten by preallocation. */ 3437 if (new_bytes) { 3438 bdrv_make_request_serialising(&req, 1); 3439 } 3440 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req, 3441 0); 3442 if (ret < 0) { 3443 error_setg_errno(errp, -ret, 3444 "Failed to prepare request for truncation"); 3445 goto out; 3446 } 3447 3448 filtered = bdrv_filter_child(bs); 3449 backing = bdrv_cow_child(bs); 3450 3451 /* 3452 * If the image has a backing file that is large enough that it would 3453 * provide data for the new area, we cannot leave it unallocated because 3454 * then the backing file content would become visible. Instead, zero-fill 3455 * the new area. 3456 * 3457 * Note that if the image has a backing file, but was opened without the 3458 * backing file, taking care of keeping things consistent with that backing 3459 * file is the user's responsibility. 3460 */ 3461 if (new_bytes && backing) { 3462 int64_t backing_len; 3463 3464 backing_len = bdrv_co_getlength(backing->bs); 3465 if (backing_len < 0) { 3466 ret = backing_len; 3467 error_setg_errno(errp, -ret, "Could not get backing file size"); 3468 goto out; 3469 } 3470 3471 if (backing_len > old_size) { 3472 flags |= BDRV_REQ_ZERO_WRITE; 3473 } 3474 } 3475 3476 if (drv->bdrv_co_truncate) { 3477 if (flags & ~bs->supported_truncate_flags) { 3478 error_setg(errp, "Block driver does not support requested flags"); 3479 ret = -ENOTSUP; 3480 goto out; 3481 } 3482 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp); 3483 } else if (filtered) { 3484 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp); 3485 } else { 3486 error_setg(errp, "Image format driver does not support resize"); 3487 ret = -ENOTSUP; 3488 goto out; 3489 } 3490 if (ret < 0) { 3491 goto out; 3492 } 3493 3494 ret = bdrv_co_refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS); 3495 if (ret < 0) { 3496 error_setg_errno(errp, -ret, "Could not refresh total sector count"); 3497 } else { 3498 offset = bs->total_sectors * BDRV_SECTOR_SIZE; 3499 } 3500 /* 3501 * It's possible that truncation succeeded but bdrv_refresh_total_sectors 3502 * failed, but the latter doesn't affect how we should finish the request. 3503 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. 3504 */ 3505 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0); 3506 3507 out: 3508 tracked_request_end(&req); 3509 bdrv_dec_in_flight(bs); 3510 3511 return ret; 3512 } 3513 3514 void bdrv_cancel_in_flight(BlockDriverState *bs) 3515 { 3516 GLOBAL_STATE_CODE(); 3517 if (!bs || !bs->drv) { 3518 return; 3519 } 3520 3521 if (bs->drv->bdrv_cancel_in_flight) { 3522 bs->drv->bdrv_cancel_in_flight(bs); 3523 } 3524 } 3525 3526 int coroutine_fn 3527 bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes, 3528 QEMUIOVector *qiov, size_t qiov_offset) 3529 { 3530 BlockDriverState *bs = child->bs; 3531 BlockDriver *drv = bs->drv; 3532 int ret; 3533 IO_CODE(); 3534 3535 if (!drv) { 3536 return -ENOMEDIUM; 3537 } 3538 3539 if (!drv->bdrv_co_preadv_snapshot) { 3540 return -ENOTSUP; 3541 } 3542 3543 bdrv_inc_in_flight(bs); 3544 ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset); 3545 bdrv_dec_in_flight(bs); 3546 3547 return ret; 3548 } 3549 3550 int coroutine_fn 3551 bdrv_co_snapshot_block_status(BlockDriverState *bs, 3552 bool want_zero, int64_t offset, int64_t bytes, 3553 int64_t *pnum, int64_t *map, 3554 BlockDriverState **file) 3555 { 3556 BlockDriver *drv = bs->drv; 3557 int ret; 3558 IO_CODE(); 3559 3560 if (!drv) { 3561 return -ENOMEDIUM; 3562 } 3563 3564 if (!drv->bdrv_co_snapshot_block_status) { 3565 return -ENOTSUP; 3566 } 3567 3568 bdrv_inc_in_flight(bs); 3569 ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes, 3570 pnum, map, file); 3571 bdrv_dec_in_flight(bs); 3572 3573 return ret; 3574 } 3575 3576 int coroutine_fn 3577 bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes) 3578 { 3579 BlockDriver *drv = bs->drv; 3580 int ret; 3581 IO_CODE(); 3582 assert_bdrv_graph_readable(); 3583 3584 if (!drv) { 3585 return -ENOMEDIUM; 3586 } 3587 3588 if (!drv->bdrv_co_pdiscard_snapshot) { 3589 return -ENOTSUP; 3590 } 3591 3592 bdrv_inc_in_flight(bs); 3593 ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes); 3594 bdrv_dec_in_flight(bs); 3595 3596 return ret; 3597 } 3598