1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/blockjob.h" 29 #include "block/blockjob_int.h" 30 #include "block/block_int.h" 31 #include "qemu/cutils.h" 32 #include "qapi/error.h" 33 #include "qemu/error-report.h" 34 35 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 36 37 static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child, 38 int64_t offset, 39 QEMUIOVector *qiov, 40 BdrvRequestFlags flags, 41 BlockCompletionFunc *cb, 42 void *opaque, 43 bool is_write); 44 static void coroutine_fn bdrv_co_do_rw(void *opaque); 45 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 46 int64_t offset, int count, BdrvRequestFlags flags); 47 48 void bdrv_parent_drained_begin(BlockDriverState *bs) 49 { 50 BdrvChild *c; 51 52 QLIST_FOREACH(c, &bs->parents, next_parent) { 53 if (c->role->drained_begin) { 54 c->role->drained_begin(c); 55 } 56 } 57 } 58 59 void bdrv_parent_drained_end(BlockDriverState *bs) 60 { 61 BdrvChild *c; 62 63 QLIST_FOREACH(c, &bs->parents, next_parent) { 64 if (c->role->drained_end) { 65 c->role->drained_end(c); 66 } 67 } 68 } 69 70 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 71 { 72 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 73 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 74 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 75 src->opt_mem_alignment); 76 dst->min_mem_alignment = MAX(dst->min_mem_alignment, 77 src->min_mem_alignment); 78 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 79 } 80 81 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 82 { 83 BlockDriver *drv = bs->drv; 84 Error *local_err = NULL; 85 86 memset(&bs->bl, 0, sizeof(bs->bl)); 87 88 if (!drv) { 89 return; 90 } 91 92 /* Default alignment based on whether driver has byte interface */ 93 bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512; 94 95 /* Take some limits from the children as a default */ 96 if (bs->file) { 97 bdrv_refresh_limits(bs->file->bs, &local_err); 98 if (local_err) { 99 error_propagate(errp, local_err); 100 return; 101 } 102 bdrv_merge_limits(&bs->bl, &bs->file->bs->bl); 103 } else { 104 bs->bl.min_mem_alignment = 512; 105 bs->bl.opt_mem_alignment = getpagesize(); 106 107 /* Safe default since most protocols use readv()/writev()/etc */ 108 bs->bl.max_iov = IOV_MAX; 109 } 110 111 if (bs->backing) { 112 bdrv_refresh_limits(bs->backing->bs, &local_err); 113 if (local_err) { 114 error_propagate(errp, local_err); 115 return; 116 } 117 bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl); 118 } 119 120 /* Then let the driver override it */ 121 if (drv->bdrv_refresh_limits) { 122 drv->bdrv_refresh_limits(bs, errp); 123 } 124 } 125 126 /** 127 * The copy-on-read flag is actually a reference count so multiple users may 128 * use the feature without worrying about clobbering its previous state. 129 * Copy-on-read stays enabled until all users have called to disable it. 130 */ 131 void bdrv_enable_copy_on_read(BlockDriverState *bs) 132 { 133 bs->copy_on_read++; 134 } 135 136 void bdrv_disable_copy_on_read(BlockDriverState *bs) 137 { 138 assert(bs->copy_on_read > 0); 139 bs->copy_on_read--; 140 } 141 142 /* Check if any requests are in-flight (including throttled requests) */ 143 bool bdrv_requests_pending(BlockDriverState *bs) 144 { 145 BdrvChild *child; 146 147 if (atomic_read(&bs->in_flight)) { 148 return true; 149 } 150 151 QLIST_FOREACH(child, &bs->children, next) { 152 if (bdrv_requests_pending(child->bs)) { 153 return true; 154 } 155 } 156 157 return false; 158 } 159 160 static bool bdrv_drain_recurse(BlockDriverState *bs) 161 { 162 BdrvChild *child, *tmp; 163 bool waited; 164 165 waited = BDRV_POLL_WHILE(bs, atomic_read(&bs->in_flight) > 0); 166 167 if (bs->drv && bs->drv->bdrv_drain) { 168 bs->drv->bdrv_drain(bs); 169 } 170 171 QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) { 172 BlockDriverState *bs = child->bs; 173 bool in_main_loop = 174 qemu_get_current_aio_context() == qemu_get_aio_context(); 175 assert(bs->refcnt > 0); 176 if (in_main_loop) { 177 /* In case the recursive bdrv_drain_recurse processes a 178 * block_job_defer_to_main_loop BH and modifies the graph, 179 * let's hold a reference to bs until we are done. 180 * 181 * IOThread doesn't have such a BH, and it is not safe to call 182 * bdrv_unref without BQL, so skip doing it there. 183 */ 184 bdrv_ref(bs); 185 } 186 waited |= bdrv_drain_recurse(bs); 187 if (in_main_loop) { 188 bdrv_unref(bs); 189 } 190 } 191 192 return waited; 193 } 194 195 typedef struct { 196 Coroutine *co; 197 BlockDriverState *bs; 198 bool done; 199 } BdrvCoDrainData; 200 201 static void bdrv_co_drain_bh_cb(void *opaque) 202 { 203 BdrvCoDrainData *data = opaque; 204 Coroutine *co = data->co; 205 BlockDriverState *bs = data->bs; 206 207 bdrv_dec_in_flight(bs); 208 bdrv_drained_begin(bs); 209 data->done = true; 210 aio_co_wake(co); 211 } 212 213 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs) 214 { 215 BdrvCoDrainData data; 216 217 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 218 * other coroutines run if they were queued from 219 * qemu_co_queue_run_restart(). */ 220 221 assert(qemu_in_coroutine()); 222 data = (BdrvCoDrainData) { 223 .co = qemu_coroutine_self(), 224 .bs = bs, 225 .done = false, 226 }; 227 bdrv_inc_in_flight(bs); 228 aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), 229 bdrv_co_drain_bh_cb, &data); 230 231 qemu_coroutine_yield(); 232 /* If we are resumed from some other event (such as an aio completion or a 233 * timer callback), it is a bug in the caller that should be fixed. */ 234 assert(data.done); 235 } 236 237 void bdrv_drained_begin(BlockDriverState *bs) 238 { 239 if (qemu_in_coroutine()) { 240 bdrv_co_yield_to_drain(bs); 241 return; 242 } 243 244 if (!bs->quiesce_counter++) { 245 aio_disable_external(bdrv_get_aio_context(bs)); 246 bdrv_parent_drained_begin(bs); 247 } 248 249 bdrv_drain_recurse(bs); 250 } 251 252 void bdrv_drained_end(BlockDriverState *bs) 253 { 254 assert(bs->quiesce_counter > 0); 255 if (--bs->quiesce_counter > 0) { 256 return; 257 } 258 259 bdrv_parent_drained_end(bs); 260 aio_enable_external(bdrv_get_aio_context(bs)); 261 } 262 263 /* 264 * Wait for pending requests to complete on a single BlockDriverState subtree, 265 * and suspend block driver's internal I/O until next request arrives. 266 * 267 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 268 * AioContext. 269 * 270 * Only this BlockDriverState's AioContext is run, so in-flight requests must 271 * not depend on events in other AioContexts. In that case, use 272 * bdrv_drain_all() instead. 273 */ 274 void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 275 { 276 assert(qemu_in_coroutine()); 277 bdrv_drained_begin(bs); 278 bdrv_drained_end(bs); 279 } 280 281 void bdrv_drain(BlockDriverState *bs) 282 { 283 bdrv_drained_begin(bs); 284 bdrv_drained_end(bs); 285 } 286 287 /* 288 * Wait for pending requests to complete across all BlockDriverStates 289 * 290 * This function does not flush data to disk, use bdrv_flush_all() for that 291 * after calling this function. 292 * 293 * This pauses all block jobs and disables external clients. It must 294 * be paired with bdrv_drain_all_end(). 295 * 296 * NOTE: no new block jobs or BlockDriverStates can be created between 297 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 298 */ 299 void bdrv_drain_all_begin(void) 300 { 301 /* Always run first iteration so any pending completion BHs run */ 302 bool waited = true; 303 BlockDriverState *bs; 304 BdrvNextIterator it; 305 GSList *aio_ctxs = NULL, *ctx; 306 307 block_job_pause_all(); 308 309 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 310 AioContext *aio_context = bdrv_get_aio_context(bs); 311 312 aio_context_acquire(aio_context); 313 bdrv_parent_drained_begin(bs); 314 aio_disable_external(aio_context); 315 aio_context_release(aio_context); 316 317 if (!g_slist_find(aio_ctxs, aio_context)) { 318 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 319 } 320 } 321 322 /* Note that completion of an asynchronous I/O operation can trigger any 323 * number of other I/O operations on other devices---for example a 324 * coroutine can submit an I/O request to another device in response to 325 * request completion. Therefore we must keep looping until there was no 326 * more activity rather than simply draining each device independently. 327 */ 328 while (waited) { 329 waited = false; 330 331 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 332 AioContext *aio_context = ctx->data; 333 334 aio_context_acquire(aio_context); 335 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 336 if (aio_context == bdrv_get_aio_context(bs)) { 337 waited |= bdrv_drain_recurse(bs); 338 } 339 } 340 aio_context_release(aio_context); 341 } 342 } 343 344 g_slist_free(aio_ctxs); 345 } 346 347 void bdrv_drain_all_end(void) 348 { 349 BlockDriverState *bs; 350 BdrvNextIterator it; 351 352 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 353 AioContext *aio_context = bdrv_get_aio_context(bs); 354 355 aio_context_acquire(aio_context); 356 aio_enable_external(aio_context); 357 bdrv_parent_drained_end(bs); 358 aio_context_release(aio_context); 359 } 360 361 block_job_resume_all(); 362 } 363 364 void bdrv_drain_all(void) 365 { 366 bdrv_drain_all_begin(); 367 bdrv_drain_all_end(); 368 } 369 370 /** 371 * Remove an active request from the tracked requests list 372 * 373 * This function should be called when a tracked request is completing. 374 */ 375 static void tracked_request_end(BdrvTrackedRequest *req) 376 { 377 if (req->serialising) { 378 req->bs->serialising_in_flight--; 379 } 380 381 QLIST_REMOVE(req, list); 382 qemu_co_queue_restart_all(&req->wait_queue); 383 } 384 385 /** 386 * Add an active request to the tracked requests list 387 */ 388 static void tracked_request_begin(BdrvTrackedRequest *req, 389 BlockDriverState *bs, 390 int64_t offset, 391 unsigned int bytes, 392 enum BdrvTrackedRequestType type) 393 { 394 *req = (BdrvTrackedRequest){ 395 .bs = bs, 396 .offset = offset, 397 .bytes = bytes, 398 .type = type, 399 .co = qemu_coroutine_self(), 400 .serialising = false, 401 .overlap_offset = offset, 402 .overlap_bytes = bytes, 403 }; 404 405 qemu_co_queue_init(&req->wait_queue); 406 407 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 408 } 409 410 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 411 { 412 int64_t overlap_offset = req->offset & ~(align - 1); 413 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 414 - overlap_offset; 415 416 if (!req->serialising) { 417 req->bs->serialising_in_flight++; 418 req->serialising = true; 419 } 420 421 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 422 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 423 } 424 425 /** 426 * Round a region to cluster boundaries (sector-based) 427 */ 428 void bdrv_round_sectors_to_clusters(BlockDriverState *bs, 429 int64_t sector_num, int nb_sectors, 430 int64_t *cluster_sector_num, 431 int *cluster_nb_sectors) 432 { 433 BlockDriverInfo bdi; 434 435 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 436 *cluster_sector_num = sector_num; 437 *cluster_nb_sectors = nb_sectors; 438 } else { 439 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE; 440 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c); 441 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num + 442 nb_sectors, c); 443 } 444 } 445 446 /** 447 * Round a region to cluster boundaries 448 */ 449 void bdrv_round_to_clusters(BlockDriverState *bs, 450 int64_t offset, unsigned int bytes, 451 int64_t *cluster_offset, 452 unsigned int *cluster_bytes) 453 { 454 BlockDriverInfo bdi; 455 456 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 457 *cluster_offset = offset; 458 *cluster_bytes = bytes; 459 } else { 460 int64_t c = bdi.cluster_size; 461 *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 462 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 463 } 464 } 465 466 static int bdrv_get_cluster_size(BlockDriverState *bs) 467 { 468 BlockDriverInfo bdi; 469 int ret; 470 471 ret = bdrv_get_info(bs, &bdi); 472 if (ret < 0 || bdi.cluster_size == 0) { 473 return bs->bl.request_alignment; 474 } else { 475 return bdi.cluster_size; 476 } 477 } 478 479 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 480 int64_t offset, unsigned int bytes) 481 { 482 /* aaaa bbbb */ 483 if (offset >= req->overlap_offset + req->overlap_bytes) { 484 return false; 485 } 486 /* bbbb aaaa */ 487 if (req->overlap_offset >= offset + bytes) { 488 return false; 489 } 490 return true; 491 } 492 493 void bdrv_inc_in_flight(BlockDriverState *bs) 494 { 495 atomic_inc(&bs->in_flight); 496 } 497 498 static void dummy_bh_cb(void *opaque) 499 { 500 } 501 502 void bdrv_wakeup(BlockDriverState *bs) 503 { 504 if (bs->wakeup) { 505 aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL); 506 } 507 } 508 509 void bdrv_dec_in_flight(BlockDriverState *bs) 510 { 511 atomic_dec(&bs->in_flight); 512 bdrv_wakeup(bs); 513 } 514 515 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 516 { 517 BlockDriverState *bs = self->bs; 518 BdrvTrackedRequest *req; 519 bool retry; 520 bool waited = false; 521 522 if (!bs->serialising_in_flight) { 523 return false; 524 } 525 526 do { 527 retry = false; 528 QLIST_FOREACH(req, &bs->tracked_requests, list) { 529 if (req == self || (!req->serialising && !self->serialising)) { 530 continue; 531 } 532 if (tracked_request_overlaps(req, self->overlap_offset, 533 self->overlap_bytes)) 534 { 535 /* Hitting this means there was a reentrant request, for 536 * example, a block driver issuing nested requests. This must 537 * never happen since it means deadlock. 538 */ 539 assert(qemu_coroutine_self() != req->co); 540 541 /* If the request is already (indirectly) waiting for us, or 542 * will wait for us as soon as it wakes up, then just go on 543 * (instead of producing a deadlock in the former case). */ 544 if (!req->waiting_for) { 545 self->waiting_for = req; 546 qemu_co_queue_wait(&req->wait_queue, NULL); 547 self->waiting_for = NULL; 548 retry = true; 549 waited = true; 550 break; 551 } 552 } 553 } 554 } while (retry); 555 556 return waited; 557 } 558 559 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 560 size_t size) 561 { 562 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 563 return -EIO; 564 } 565 566 if (!bdrv_is_inserted(bs)) { 567 return -ENOMEDIUM; 568 } 569 570 if (offset < 0) { 571 return -EIO; 572 } 573 574 return 0; 575 } 576 577 typedef struct RwCo { 578 BdrvChild *child; 579 int64_t offset; 580 QEMUIOVector *qiov; 581 bool is_write; 582 int ret; 583 BdrvRequestFlags flags; 584 } RwCo; 585 586 static void coroutine_fn bdrv_rw_co_entry(void *opaque) 587 { 588 RwCo *rwco = opaque; 589 590 if (!rwco->is_write) { 591 rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset, 592 rwco->qiov->size, rwco->qiov, 593 rwco->flags); 594 } else { 595 rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset, 596 rwco->qiov->size, rwco->qiov, 597 rwco->flags); 598 } 599 } 600 601 /* 602 * Process a vectored synchronous request using coroutines 603 */ 604 static int bdrv_prwv_co(BdrvChild *child, int64_t offset, 605 QEMUIOVector *qiov, bool is_write, 606 BdrvRequestFlags flags) 607 { 608 Coroutine *co; 609 RwCo rwco = { 610 .child = child, 611 .offset = offset, 612 .qiov = qiov, 613 .is_write = is_write, 614 .ret = NOT_DONE, 615 .flags = flags, 616 }; 617 618 if (qemu_in_coroutine()) { 619 /* Fast-path if already in coroutine context */ 620 bdrv_rw_co_entry(&rwco); 621 } else { 622 co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco); 623 bdrv_coroutine_enter(child->bs, co); 624 BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE); 625 } 626 return rwco.ret; 627 } 628 629 /* 630 * Process a synchronous request using coroutines 631 */ 632 static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf, 633 int nb_sectors, bool is_write, BdrvRequestFlags flags) 634 { 635 QEMUIOVector qiov; 636 struct iovec iov = { 637 .iov_base = (void *)buf, 638 .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 639 }; 640 641 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 642 return -EINVAL; 643 } 644 645 qemu_iovec_init_external(&qiov, &iov, 1); 646 return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS, 647 &qiov, is_write, flags); 648 } 649 650 /* return < 0 if error. See bdrv_write() for the return codes */ 651 int bdrv_read(BdrvChild *child, int64_t sector_num, 652 uint8_t *buf, int nb_sectors) 653 { 654 return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0); 655 } 656 657 /* Return < 0 if error. Important errors are: 658 -EIO generic I/O error (may happen for all errors) 659 -ENOMEDIUM No media inserted. 660 -EINVAL Invalid sector number or nb_sectors 661 -EACCES Trying to write a read-only device 662 */ 663 int bdrv_write(BdrvChild *child, int64_t sector_num, 664 const uint8_t *buf, int nb_sectors) 665 { 666 return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 667 } 668 669 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 670 int count, BdrvRequestFlags flags) 671 { 672 QEMUIOVector qiov; 673 struct iovec iov = { 674 .iov_base = NULL, 675 .iov_len = count, 676 }; 677 678 qemu_iovec_init_external(&qiov, &iov, 1); 679 return bdrv_prwv_co(child, offset, &qiov, true, 680 BDRV_REQ_ZERO_WRITE | flags); 681 } 682 683 /* 684 * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 685 * The operation is sped up by checking the block status and only writing 686 * zeroes to the device if they currently do not return zeroes. Optional 687 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 688 * BDRV_REQ_FUA). 689 * 690 * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 691 */ 692 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 693 { 694 int64_t target_sectors, ret, nb_sectors, sector_num = 0; 695 BlockDriverState *bs = child->bs; 696 BlockDriverState *file; 697 int n; 698 699 target_sectors = bdrv_nb_sectors(bs); 700 if (target_sectors < 0) { 701 return target_sectors; 702 } 703 704 for (;;) { 705 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS); 706 if (nb_sectors <= 0) { 707 return 0; 708 } 709 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file); 710 if (ret < 0) { 711 error_report("error getting block status at sector %" PRId64 ": %s", 712 sector_num, strerror(-ret)); 713 return ret; 714 } 715 if (ret & BDRV_BLOCK_ZERO) { 716 sector_num += n; 717 continue; 718 } 719 ret = bdrv_pwrite_zeroes(child, sector_num << BDRV_SECTOR_BITS, 720 n << BDRV_SECTOR_BITS, flags); 721 if (ret < 0) { 722 error_report("error writing zeroes at sector %" PRId64 ": %s", 723 sector_num, strerror(-ret)); 724 return ret; 725 } 726 sector_num += n; 727 } 728 } 729 730 int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 731 { 732 int ret; 733 734 ret = bdrv_prwv_co(child, offset, qiov, false, 0); 735 if (ret < 0) { 736 return ret; 737 } 738 739 return qiov->size; 740 } 741 742 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 743 { 744 QEMUIOVector qiov; 745 struct iovec iov = { 746 .iov_base = (void *)buf, 747 .iov_len = bytes, 748 }; 749 750 if (bytes < 0) { 751 return -EINVAL; 752 } 753 754 qemu_iovec_init_external(&qiov, &iov, 1); 755 return bdrv_preadv(child, offset, &qiov); 756 } 757 758 int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 759 { 760 int ret; 761 762 ret = bdrv_prwv_co(child, offset, qiov, true, 0); 763 if (ret < 0) { 764 return ret; 765 } 766 767 return qiov->size; 768 } 769 770 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 771 { 772 QEMUIOVector qiov; 773 struct iovec iov = { 774 .iov_base = (void *) buf, 775 .iov_len = bytes, 776 }; 777 778 if (bytes < 0) { 779 return -EINVAL; 780 } 781 782 qemu_iovec_init_external(&qiov, &iov, 1); 783 return bdrv_pwritev(child, offset, &qiov); 784 } 785 786 /* 787 * Writes to the file and ensures that no writes are reordered across this 788 * request (acts as a barrier) 789 * 790 * Returns 0 on success, -errno in error cases. 791 */ 792 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 793 const void *buf, int count) 794 { 795 int ret; 796 797 ret = bdrv_pwrite(child, offset, buf, count); 798 if (ret < 0) { 799 return ret; 800 } 801 802 ret = bdrv_flush(child->bs); 803 if (ret < 0) { 804 return ret; 805 } 806 807 return 0; 808 } 809 810 typedef struct CoroutineIOCompletion { 811 Coroutine *coroutine; 812 int ret; 813 } CoroutineIOCompletion; 814 815 static void bdrv_co_io_em_complete(void *opaque, int ret) 816 { 817 CoroutineIOCompletion *co = opaque; 818 819 co->ret = ret; 820 aio_co_wake(co->coroutine); 821 } 822 823 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 824 uint64_t offset, uint64_t bytes, 825 QEMUIOVector *qiov, int flags) 826 { 827 BlockDriver *drv = bs->drv; 828 int64_t sector_num; 829 unsigned int nb_sectors; 830 831 assert(!(flags & ~BDRV_REQ_MASK)); 832 833 if (drv->bdrv_co_preadv) { 834 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 835 } 836 837 sector_num = offset >> BDRV_SECTOR_BITS; 838 nb_sectors = bytes >> BDRV_SECTOR_BITS; 839 840 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 841 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 842 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 843 844 if (drv->bdrv_co_readv) { 845 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 846 } else { 847 BlockAIOCB *acb; 848 CoroutineIOCompletion co = { 849 .coroutine = qemu_coroutine_self(), 850 }; 851 852 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors, 853 bdrv_co_io_em_complete, &co); 854 if (acb == NULL) { 855 return -EIO; 856 } else { 857 qemu_coroutine_yield(); 858 return co.ret; 859 } 860 } 861 } 862 863 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 864 uint64_t offset, uint64_t bytes, 865 QEMUIOVector *qiov, int flags) 866 { 867 BlockDriver *drv = bs->drv; 868 int64_t sector_num; 869 unsigned int nb_sectors; 870 int ret; 871 872 assert(!(flags & ~BDRV_REQ_MASK)); 873 874 if (drv->bdrv_co_pwritev) { 875 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 876 flags & bs->supported_write_flags); 877 flags &= ~bs->supported_write_flags; 878 goto emulate_flags; 879 } 880 881 sector_num = offset >> BDRV_SECTOR_BITS; 882 nb_sectors = bytes >> BDRV_SECTOR_BITS; 883 884 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 885 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 886 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 887 888 if (drv->bdrv_co_writev_flags) { 889 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov, 890 flags & bs->supported_write_flags); 891 flags &= ~bs->supported_write_flags; 892 } else if (drv->bdrv_co_writev) { 893 assert(!bs->supported_write_flags); 894 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 895 } else { 896 BlockAIOCB *acb; 897 CoroutineIOCompletion co = { 898 .coroutine = qemu_coroutine_self(), 899 }; 900 901 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors, 902 bdrv_co_io_em_complete, &co); 903 if (acb == NULL) { 904 ret = -EIO; 905 } else { 906 qemu_coroutine_yield(); 907 ret = co.ret; 908 } 909 } 910 911 emulate_flags: 912 if (ret == 0 && (flags & BDRV_REQ_FUA)) { 913 ret = bdrv_co_flush(bs); 914 } 915 916 return ret; 917 } 918 919 static int coroutine_fn 920 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset, 921 uint64_t bytes, QEMUIOVector *qiov) 922 { 923 BlockDriver *drv = bs->drv; 924 925 if (!drv->bdrv_co_pwritev_compressed) { 926 return -ENOTSUP; 927 } 928 929 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 930 } 931 932 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 933 int64_t offset, unsigned int bytes, QEMUIOVector *qiov) 934 { 935 BlockDriverState *bs = child->bs; 936 937 /* Perform I/O through a temporary buffer so that users who scribble over 938 * their read buffer while the operation is in progress do not end up 939 * modifying the image file. This is critical for zero-copy guest I/O 940 * where anything might happen inside guest memory. 941 */ 942 void *bounce_buffer; 943 944 BlockDriver *drv = bs->drv; 945 struct iovec iov; 946 QEMUIOVector bounce_qiov; 947 int64_t cluster_offset; 948 unsigned int cluster_bytes; 949 size_t skip_bytes; 950 int ret; 951 952 /* FIXME We cannot require callers to have write permissions when all they 953 * are doing is a read request. If we did things right, write permissions 954 * would be obtained anyway, but internally by the copy-on-read code. As 955 * long as it is implemented here rather than in a separat filter driver, 956 * the copy-on-read code doesn't have its own BdrvChild, however, for which 957 * it could request permissions. Therefore we have to bypass the permission 958 * system for the moment. */ 959 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 960 961 /* Cover entire cluster so no additional backing file I/O is required when 962 * allocating cluster in the image file. 963 */ 964 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 965 966 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 967 cluster_offset, cluster_bytes); 968 969 iov.iov_len = cluster_bytes; 970 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len); 971 if (bounce_buffer == NULL) { 972 ret = -ENOMEM; 973 goto err; 974 } 975 976 qemu_iovec_init_external(&bounce_qiov, &iov, 1); 977 978 ret = bdrv_driver_preadv(bs, cluster_offset, cluster_bytes, 979 &bounce_qiov, 0); 980 if (ret < 0) { 981 goto err; 982 } 983 984 if (drv->bdrv_co_pwrite_zeroes && 985 buffer_is_zero(bounce_buffer, iov.iov_len)) { 986 /* FIXME: Should we (perhaps conditionally) be setting 987 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 988 * that still correctly reads as zero? */ 989 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, cluster_bytes, 0); 990 } else { 991 /* This does not change the data on the disk, it is not necessary 992 * to flush even in cache=writethrough mode. 993 */ 994 ret = bdrv_driver_pwritev(bs, cluster_offset, cluster_bytes, 995 &bounce_qiov, 0); 996 } 997 998 if (ret < 0) { 999 /* It might be okay to ignore write errors for guest requests. If this 1000 * is a deliberate copy-on-read then we don't want to ignore the error. 1001 * Simply report it in all cases. 1002 */ 1003 goto err; 1004 } 1005 1006 skip_bytes = offset - cluster_offset; 1007 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, bytes); 1008 1009 err: 1010 qemu_vfree(bounce_buffer); 1011 return ret; 1012 } 1013 1014 /* 1015 * Forwards an already correctly aligned request to the BlockDriver. This 1016 * handles copy on read, zeroing after EOF, and fragmentation of large 1017 * reads; any other features must be implemented by the caller. 1018 */ 1019 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 1020 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1021 int64_t align, QEMUIOVector *qiov, int flags) 1022 { 1023 BlockDriverState *bs = child->bs; 1024 int64_t total_bytes, max_bytes; 1025 int ret = 0; 1026 uint64_t bytes_remaining = bytes; 1027 int max_transfer; 1028 1029 assert(is_power_of_2(align)); 1030 assert((offset & (align - 1)) == 0); 1031 assert((bytes & (align - 1)) == 0); 1032 assert(!qiov || bytes == qiov->size); 1033 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1034 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1035 align); 1036 1037 /* TODO: We would need a per-BDS .supported_read_flags and 1038 * potential fallback support, if we ever implement any read flags 1039 * to pass through to drivers. For now, there aren't any 1040 * passthrough flags. */ 1041 assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ))); 1042 1043 /* Handle Copy on Read and associated serialisation */ 1044 if (flags & BDRV_REQ_COPY_ON_READ) { 1045 /* If we touch the same cluster it counts as an overlap. This 1046 * guarantees that allocating writes will be serialized and not race 1047 * with each other for the same cluster. For example, in copy-on-read 1048 * it ensures that the CoR read and write operations are atomic and 1049 * guest writes cannot interleave between them. */ 1050 mark_request_serialising(req, bdrv_get_cluster_size(bs)); 1051 } 1052 1053 if (!(flags & BDRV_REQ_NO_SERIALISING)) { 1054 wait_serialising_requests(req); 1055 } 1056 1057 if (flags & BDRV_REQ_COPY_ON_READ) { 1058 int64_t start_sector = offset >> BDRV_SECTOR_BITS; 1059 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1060 unsigned int nb_sectors = end_sector - start_sector; 1061 int pnum; 1062 1063 ret = bdrv_is_allocated(bs, start_sector, nb_sectors, &pnum); 1064 if (ret < 0) { 1065 goto out; 1066 } 1067 1068 if (!ret || pnum != nb_sectors) { 1069 ret = bdrv_co_do_copy_on_readv(child, offset, bytes, qiov); 1070 goto out; 1071 } 1072 } 1073 1074 /* Forward the request to the BlockDriver, possibly fragmenting it */ 1075 total_bytes = bdrv_getlength(bs); 1076 if (total_bytes < 0) { 1077 ret = total_bytes; 1078 goto out; 1079 } 1080 1081 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 1082 if (bytes <= max_bytes && bytes <= max_transfer) { 1083 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 1084 goto out; 1085 } 1086 1087 while (bytes_remaining) { 1088 int num; 1089 1090 if (max_bytes) { 1091 QEMUIOVector local_qiov; 1092 1093 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 1094 assert(num); 1095 qemu_iovec_init(&local_qiov, qiov->niov); 1096 qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 1097 1098 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1099 num, &local_qiov, 0); 1100 max_bytes -= num; 1101 qemu_iovec_destroy(&local_qiov); 1102 } else { 1103 num = bytes_remaining; 1104 ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0, 1105 bytes_remaining); 1106 } 1107 if (ret < 0) { 1108 goto out; 1109 } 1110 bytes_remaining -= num; 1111 } 1112 1113 out: 1114 return ret < 0 ? ret : 0; 1115 } 1116 1117 /* 1118 * Handle a read request in coroutine context 1119 */ 1120 int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1121 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1122 BdrvRequestFlags flags) 1123 { 1124 BlockDriverState *bs = child->bs; 1125 BlockDriver *drv = bs->drv; 1126 BdrvTrackedRequest req; 1127 1128 uint64_t align = bs->bl.request_alignment; 1129 uint8_t *head_buf = NULL; 1130 uint8_t *tail_buf = NULL; 1131 QEMUIOVector local_qiov; 1132 bool use_local_qiov = false; 1133 int ret; 1134 1135 if (!drv) { 1136 return -ENOMEDIUM; 1137 } 1138 1139 ret = bdrv_check_byte_request(bs, offset, bytes); 1140 if (ret < 0) { 1141 return ret; 1142 } 1143 1144 bdrv_inc_in_flight(bs); 1145 1146 /* Don't do copy-on-read if we read data before write operation */ 1147 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) { 1148 flags |= BDRV_REQ_COPY_ON_READ; 1149 } 1150 1151 /* Align read if necessary by padding qiov */ 1152 if (offset & (align - 1)) { 1153 head_buf = qemu_blockalign(bs, align); 1154 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1155 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1156 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1157 use_local_qiov = true; 1158 1159 bytes += offset & (align - 1); 1160 offset = offset & ~(align - 1); 1161 } 1162 1163 if ((offset + bytes) & (align - 1)) { 1164 if (!use_local_qiov) { 1165 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1166 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1167 use_local_qiov = true; 1168 } 1169 tail_buf = qemu_blockalign(bs, align); 1170 qemu_iovec_add(&local_qiov, tail_buf, 1171 align - ((offset + bytes) & (align - 1))); 1172 1173 bytes = ROUND_UP(bytes, align); 1174 } 1175 1176 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1177 ret = bdrv_aligned_preadv(child, &req, offset, bytes, align, 1178 use_local_qiov ? &local_qiov : qiov, 1179 flags); 1180 tracked_request_end(&req); 1181 bdrv_dec_in_flight(bs); 1182 1183 if (use_local_qiov) { 1184 qemu_iovec_destroy(&local_qiov); 1185 qemu_vfree(head_buf); 1186 qemu_vfree(tail_buf); 1187 } 1188 1189 return ret; 1190 } 1191 1192 static int coroutine_fn bdrv_co_do_readv(BdrvChild *child, 1193 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1194 BdrvRequestFlags flags) 1195 { 1196 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1197 return -EINVAL; 1198 } 1199 1200 return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS, 1201 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1202 } 1203 1204 int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num, 1205 int nb_sectors, QEMUIOVector *qiov) 1206 { 1207 trace_bdrv_co_readv(child->bs, sector_num, nb_sectors); 1208 1209 return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0); 1210 } 1211 1212 /* Maximum buffer for write zeroes fallback, in bytes */ 1213 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 1214 1215 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1216 int64_t offset, int count, BdrvRequestFlags flags) 1217 { 1218 BlockDriver *drv = bs->drv; 1219 QEMUIOVector qiov; 1220 struct iovec iov = {0}; 1221 int ret = 0; 1222 bool need_flush = false; 1223 int head = 0; 1224 int tail = 0; 1225 1226 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1227 int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1228 bs->bl.request_alignment); 1229 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1230 MAX_WRITE_ZEROES_BOUNCE_BUFFER); 1231 1232 assert(alignment % bs->bl.request_alignment == 0); 1233 head = offset % alignment; 1234 tail = (offset + count) % alignment; 1235 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1236 assert(max_write_zeroes >= bs->bl.request_alignment); 1237 1238 while (count > 0 && !ret) { 1239 int num = count; 1240 1241 /* Align request. Block drivers can expect the "bulk" of the request 1242 * to be aligned, and that unaligned requests do not cross cluster 1243 * boundaries. 1244 */ 1245 if (head) { 1246 /* Make a small request up to the first aligned sector. For 1247 * convenience, limit this request to max_transfer even if 1248 * we don't need to fall back to writes. */ 1249 num = MIN(MIN(count, max_transfer), alignment - head); 1250 head = (head + num) % alignment; 1251 assert(num < max_write_zeroes); 1252 } else if (tail && num > alignment) { 1253 /* Shorten the request to the last aligned sector. */ 1254 num -= tail; 1255 } 1256 1257 /* limit request size */ 1258 if (num > max_write_zeroes) { 1259 num = max_write_zeroes; 1260 } 1261 1262 ret = -ENOTSUP; 1263 /* First try the efficient write zeroes operation */ 1264 if (drv->bdrv_co_pwrite_zeroes) { 1265 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1266 flags & bs->supported_zero_flags); 1267 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1268 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1269 need_flush = true; 1270 } 1271 } else { 1272 assert(!bs->supported_zero_flags); 1273 } 1274 1275 if (ret == -ENOTSUP) { 1276 /* Fall back to bounce buffer if write zeroes is unsupported */ 1277 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1278 1279 if ((flags & BDRV_REQ_FUA) && 1280 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1281 /* No need for bdrv_driver_pwrite() to do a fallback 1282 * flush on each chunk; use just one at the end */ 1283 write_flags &= ~BDRV_REQ_FUA; 1284 need_flush = true; 1285 } 1286 num = MIN(num, max_transfer); 1287 iov.iov_len = num; 1288 if (iov.iov_base == NULL) { 1289 iov.iov_base = qemu_try_blockalign(bs, num); 1290 if (iov.iov_base == NULL) { 1291 ret = -ENOMEM; 1292 goto fail; 1293 } 1294 memset(iov.iov_base, 0, num); 1295 } 1296 qemu_iovec_init_external(&qiov, &iov, 1); 1297 1298 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags); 1299 1300 /* Keep bounce buffer around if it is big enough for all 1301 * all future requests. 1302 */ 1303 if (num < max_transfer) { 1304 qemu_vfree(iov.iov_base); 1305 iov.iov_base = NULL; 1306 } 1307 } 1308 1309 offset += num; 1310 count -= num; 1311 } 1312 1313 fail: 1314 if (ret == 0 && need_flush) { 1315 ret = bdrv_co_flush(bs); 1316 } 1317 qemu_vfree(iov.iov_base); 1318 return ret; 1319 } 1320 1321 /* 1322 * Forwards an already correctly aligned write request to the BlockDriver, 1323 * after possibly fragmenting it. 1324 */ 1325 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 1326 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1327 int64_t align, QEMUIOVector *qiov, int flags) 1328 { 1329 BlockDriverState *bs = child->bs; 1330 BlockDriver *drv = bs->drv; 1331 bool waited; 1332 int ret; 1333 1334 int64_t start_sector = offset >> BDRV_SECTOR_BITS; 1335 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1336 uint64_t bytes_remaining = bytes; 1337 int max_transfer; 1338 1339 assert(is_power_of_2(align)); 1340 assert((offset & (align - 1)) == 0); 1341 assert((bytes & (align - 1)) == 0); 1342 assert(!qiov || bytes == qiov->size); 1343 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1344 assert(!(flags & ~BDRV_REQ_MASK)); 1345 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1346 align); 1347 1348 waited = wait_serialising_requests(req); 1349 assert(!waited || !req->serialising); 1350 assert(req->overlap_offset <= offset); 1351 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1352 assert(child->perm & BLK_PERM_WRITE); 1353 assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE); 1354 1355 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 1356 1357 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1358 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 1359 qemu_iovec_is_zero(qiov)) { 1360 flags |= BDRV_REQ_ZERO_WRITE; 1361 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 1362 flags |= BDRV_REQ_MAY_UNMAP; 1363 } 1364 } 1365 1366 if (ret < 0) { 1367 /* Do nothing, write notifier decided to fail this request */ 1368 } else if (flags & BDRV_REQ_ZERO_WRITE) { 1369 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 1370 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 1371 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 1372 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov); 1373 } else if (bytes <= max_transfer) { 1374 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1375 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags); 1376 } else { 1377 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1378 while (bytes_remaining) { 1379 int num = MIN(bytes_remaining, max_transfer); 1380 QEMUIOVector local_qiov; 1381 int local_flags = flags; 1382 1383 assert(num); 1384 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 1385 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1386 /* If FUA is going to be emulated by flush, we only 1387 * need to flush on the last iteration */ 1388 local_flags &= ~BDRV_REQ_FUA; 1389 } 1390 qemu_iovec_init(&local_qiov, qiov->niov); 1391 qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 1392 1393 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 1394 num, &local_qiov, local_flags); 1395 qemu_iovec_destroy(&local_qiov); 1396 if (ret < 0) { 1397 break; 1398 } 1399 bytes_remaining -= num; 1400 } 1401 } 1402 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 1403 1404 ++bs->write_gen; 1405 bdrv_set_dirty(bs, start_sector, end_sector - start_sector); 1406 1407 if (bs->wr_highest_offset < offset + bytes) { 1408 bs->wr_highest_offset = offset + bytes; 1409 } 1410 1411 if (ret >= 0) { 1412 bs->total_sectors = MAX(bs->total_sectors, end_sector); 1413 ret = 0; 1414 } 1415 1416 return ret; 1417 } 1418 1419 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 1420 int64_t offset, 1421 unsigned int bytes, 1422 BdrvRequestFlags flags, 1423 BdrvTrackedRequest *req) 1424 { 1425 BlockDriverState *bs = child->bs; 1426 uint8_t *buf = NULL; 1427 QEMUIOVector local_qiov; 1428 struct iovec iov; 1429 uint64_t align = bs->bl.request_alignment; 1430 unsigned int head_padding_bytes, tail_padding_bytes; 1431 int ret = 0; 1432 1433 head_padding_bytes = offset & (align - 1); 1434 tail_padding_bytes = (align - (offset + bytes)) & (align - 1); 1435 1436 1437 assert(flags & BDRV_REQ_ZERO_WRITE); 1438 if (head_padding_bytes || tail_padding_bytes) { 1439 buf = qemu_blockalign(bs, align); 1440 iov = (struct iovec) { 1441 .iov_base = buf, 1442 .iov_len = align, 1443 }; 1444 qemu_iovec_init_external(&local_qiov, &iov, 1); 1445 } 1446 if (head_padding_bytes) { 1447 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 1448 1449 /* RMW the unaligned part before head. */ 1450 mark_request_serialising(req, align); 1451 wait_serialising_requests(req); 1452 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1453 ret = bdrv_aligned_preadv(child, req, offset & ~(align - 1), align, 1454 align, &local_qiov, 0); 1455 if (ret < 0) { 1456 goto fail; 1457 } 1458 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1459 1460 memset(buf + head_padding_bytes, 0, zero_bytes); 1461 ret = bdrv_aligned_pwritev(child, req, offset & ~(align - 1), align, 1462 align, &local_qiov, 1463 flags & ~BDRV_REQ_ZERO_WRITE); 1464 if (ret < 0) { 1465 goto fail; 1466 } 1467 offset += zero_bytes; 1468 bytes -= zero_bytes; 1469 } 1470 1471 assert(!bytes || (offset & (align - 1)) == 0); 1472 if (bytes >= align) { 1473 /* Write the aligned part in the middle. */ 1474 uint64_t aligned_bytes = bytes & ~(align - 1); 1475 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 1476 NULL, flags); 1477 if (ret < 0) { 1478 goto fail; 1479 } 1480 bytes -= aligned_bytes; 1481 offset += aligned_bytes; 1482 } 1483 1484 assert(!bytes || (offset & (align - 1)) == 0); 1485 if (bytes) { 1486 assert(align == tail_padding_bytes + bytes); 1487 /* RMW the unaligned part after tail. */ 1488 mark_request_serialising(req, align); 1489 wait_serialising_requests(req); 1490 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1491 ret = bdrv_aligned_preadv(child, req, offset, align, 1492 align, &local_qiov, 0); 1493 if (ret < 0) { 1494 goto fail; 1495 } 1496 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1497 1498 memset(buf, 0, bytes); 1499 ret = bdrv_aligned_pwritev(child, req, offset, align, align, 1500 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 1501 } 1502 fail: 1503 qemu_vfree(buf); 1504 return ret; 1505 1506 } 1507 1508 /* 1509 * Handle a write request in coroutine context 1510 */ 1511 int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 1512 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1513 BdrvRequestFlags flags) 1514 { 1515 BlockDriverState *bs = child->bs; 1516 BdrvTrackedRequest req; 1517 uint64_t align = bs->bl.request_alignment; 1518 uint8_t *head_buf = NULL; 1519 uint8_t *tail_buf = NULL; 1520 QEMUIOVector local_qiov; 1521 bool use_local_qiov = false; 1522 int ret; 1523 1524 if (!bs->drv) { 1525 return -ENOMEDIUM; 1526 } 1527 if (bs->read_only) { 1528 return -EPERM; 1529 } 1530 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1531 1532 ret = bdrv_check_byte_request(bs, offset, bytes); 1533 if (ret < 0) { 1534 return ret; 1535 } 1536 1537 bdrv_inc_in_flight(bs); 1538 /* 1539 * Align write if necessary by performing a read-modify-write cycle. 1540 * Pad qiov with the read parts and be sure to have a tracked request not 1541 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 1542 */ 1543 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 1544 1545 if (!qiov) { 1546 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 1547 goto out; 1548 } 1549 1550 if (offset & (align - 1)) { 1551 QEMUIOVector head_qiov; 1552 struct iovec head_iov; 1553 1554 mark_request_serialising(&req, align); 1555 wait_serialising_requests(&req); 1556 1557 head_buf = qemu_blockalign(bs, align); 1558 head_iov = (struct iovec) { 1559 .iov_base = head_buf, 1560 .iov_len = align, 1561 }; 1562 qemu_iovec_init_external(&head_qiov, &head_iov, 1); 1563 1564 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1565 ret = bdrv_aligned_preadv(child, &req, offset & ~(align - 1), align, 1566 align, &head_qiov, 0); 1567 if (ret < 0) { 1568 goto fail; 1569 } 1570 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1571 1572 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1573 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1574 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1575 use_local_qiov = true; 1576 1577 bytes += offset & (align - 1); 1578 offset = offset & ~(align - 1); 1579 1580 /* We have read the tail already if the request is smaller 1581 * than one aligned block. 1582 */ 1583 if (bytes < align) { 1584 qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes); 1585 bytes = align; 1586 } 1587 } 1588 1589 if ((offset + bytes) & (align - 1)) { 1590 QEMUIOVector tail_qiov; 1591 struct iovec tail_iov; 1592 size_t tail_bytes; 1593 bool waited; 1594 1595 mark_request_serialising(&req, align); 1596 waited = wait_serialising_requests(&req); 1597 assert(!waited || !use_local_qiov); 1598 1599 tail_buf = qemu_blockalign(bs, align); 1600 tail_iov = (struct iovec) { 1601 .iov_base = tail_buf, 1602 .iov_len = align, 1603 }; 1604 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 1605 1606 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1607 ret = bdrv_aligned_preadv(child, &req, (offset + bytes) & ~(align - 1), 1608 align, align, &tail_qiov, 0); 1609 if (ret < 0) { 1610 goto fail; 1611 } 1612 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1613 1614 if (!use_local_qiov) { 1615 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1616 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1617 use_local_qiov = true; 1618 } 1619 1620 tail_bytes = (offset + bytes) & (align - 1); 1621 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 1622 1623 bytes = ROUND_UP(bytes, align); 1624 } 1625 1626 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 1627 use_local_qiov ? &local_qiov : qiov, 1628 flags); 1629 1630 fail: 1631 1632 if (use_local_qiov) { 1633 qemu_iovec_destroy(&local_qiov); 1634 } 1635 qemu_vfree(head_buf); 1636 qemu_vfree(tail_buf); 1637 out: 1638 tracked_request_end(&req); 1639 bdrv_dec_in_flight(bs); 1640 return ret; 1641 } 1642 1643 static int coroutine_fn bdrv_co_do_writev(BdrvChild *child, 1644 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1645 BdrvRequestFlags flags) 1646 { 1647 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1648 return -EINVAL; 1649 } 1650 1651 return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS, 1652 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1653 } 1654 1655 int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num, 1656 int nb_sectors, QEMUIOVector *qiov) 1657 { 1658 trace_bdrv_co_writev(child->bs, sector_num, nb_sectors); 1659 1660 return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0); 1661 } 1662 1663 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 1664 int count, BdrvRequestFlags flags) 1665 { 1666 trace_bdrv_co_pwrite_zeroes(child->bs, offset, count, flags); 1667 1668 if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 1669 flags &= ~BDRV_REQ_MAY_UNMAP; 1670 } 1671 1672 return bdrv_co_pwritev(child, offset, count, NULL, 1673 BDRV_REQ_ZERO_WRITE | flags); 1674 } 1675 1676 /* 1677 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 1678 */ 1679 int bdrv_flush_all(void) 1680 { 1681 BdrvNextIterator it; 1682 BlockDriverState *bs = NULL; 1683 int result = 0; 1684 1685 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 1686 AioContext *aio_context = bdrv_get_aio_context(bs); 1687 int ret; 1688 1689 aio_context_acquire(aio_context); 1690 ret = bdrv_flush(bs); 1691 if (ret < 0 && !result) { 1692 result = ret; 1693 } 1694 aio_context_release(aio_context); 1695 } 1696 1697 return result; 1698 } 1699 1700 1701 typedef struct BdrvCoGetBlockStatusData { 1702 BlockDriverState *bs; 1703 BlockDriverState *base; 1704 BlockDriverState **file; 1705 int64_t sector_num; 1706 int nb_sectors; 1707 int *pnum; 1708 int64_t ret; 1709 bool done; 1710 } BdrvCoGetBlockStatusData; 1711 1712 /* 1713 * Returns the allocation status of the specified sectors. 1714 * Drivers not implementing the functionality are assumed to not support 1715 * backing files, hence all their sectors are reported as allocated. 1716 * 1717 * If 'sector_num' is beyond the end of the disk image the return value is 0 1718 * and 'pnum' is set to 0. 1719 * 1720 * 'pnum' is set to the number of sectors (including and immediately following 1721 * the specified sector) that are known to be in the same 1722 * allocated/unallocated state. 1723 * 1724 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes 1725 * beyond the end of the disk image it will be clamped. 1726 * 1727 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file' 1728 * points to the BDS which the sector range is allocated in. 1729 */ 1730 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs, 1731 int64_t sector_num, 1732 int nb_sectors, int *pnum, 1733 BlockDriverState **file) 1734 { 1735 int64_t total_sectors; 1736 int64_t n; 1737 int64_t ret, ret2; 1738 1739 total_sectors = bdrv_nb_sectors(bs); 1740 if (total_sectors < 0) { 1741 return total_sectors; 1742 } 1743 1744 if (sector_num >= total_sectors) { 1745 *pnum = 0; 1746 return 0; 1747 } 1748 1749 n = total_sectors - sector_num; 1750 if (n < nb_sectors) { 1751 nb_sectors = n; 1752 } 1753 1754 if (!bs->drv->bdrv_co_get_block_status) { 1755 *pnum = nb_sectors; 1756 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 1757 if (bs->drv->protocol_name) { 1758 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE); 1759 } 1760 return ret; 1761 } 1762 1763 *file = NULL; 1764 bdrv_inc_in_flight(bs); 1765 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum, 1766 file); 1767 if (ret < 0) { 1768 *pnum = 0; 1769 goto out; 1770 } 1771 1772 if (ret & BDRV_BLOCK_RAW) { 1773 assert(ret & BDRV_BLOCK_OFFSET_VALID); 1774 ret = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS, 1775 *pnum, pnum, file); 1776 goto out; 1777 } 1778 1779 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 1780 ret |= BDRV_BLOCK_ALLOCATED; 1781 } else { 1782 if (bdrv_unallocated_blocks_are_zero(bs)) { 1783 ret |= BDRV_BLOCK_ZERO; 1784 } else if (bs->backing) { 1785 BlockDriverState *bs2 = bs->backing->bs; 1786 int64_t nb_sectors2 = bdrv_nb_sectors(bs2); 1787 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) { 1788 ret |= BDRV_BLOCK_ZERO; 1789 } 1790 } 1791 } 1792 1793 if (*file && *file != bs && 1794 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 1795 (ret & BDRV_BLOCK_OFFSET_VALID)) { 1796 BlockDriverState *file2; 1797 int file_pnum; 1798 1799 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS, 1800 *pnum, &file_pnum, &file2); 1801 if (ret2 >= 0) { 1802 /* Ignore errors. This is just providing extra information, it 1803 * is useful but not necessary. 1804 */ 1805 if (!file_pnum) { 1806 /* !file_pnum indicates an offset at or beyond the EOF; it is 1807 * perfectly valid for the format block driver to point to such 1808 * offsets, so catch it and mark everything as zero */ 1809 ret |= BDRV_BLOCK_ZERO; 1810 } else { 1811 /* Limit request to the range reported by the protocol driver */ 1812 *pnum = file_pnum; 1813 ret |= (ret2 & BDRV_BLOCK_ZERO); 1814 } 1815 } 1816 } 1817 1818 out: 1819 bdrv_dec_in_flight(bs); 1820 return ret; 1821 } 1822 1823 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs, 1824 BlockDriverState *base, 1825 int64_t sector_num, 1826 int nb_sectors, 1827 int *pnum, 1828 BlockDriverState **file) 1829 { 1830 BlockDriverState *p; 1831 int64_t ret = 0; 1832 1833 assert(bs != base); 1834 for (p = bs; p != base; p = backing_bs(p)) { 1835 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file); 1836 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) { 1837 break; 1838 } 1839 /* [sector_num, pnum] unallocated on this layer, which could be only 1840 * the first part of [sector_num, nb_sectors]. */ 1841 nb_sectors = MIN(nb_sectors, *pnum); 1842 } 1843 return ret; 1844 } 1845 1846 /* Coroutine wrapper for bdrv_get_block_status_above() */ 1847 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque) 1848 { 1849 BdrvCoGetBlockStatusData *data = opaque; 1850 1851 data->ret = bdrv_co_get_block_status_above(data->bs, data->base, 1852 data->sector_num, 1853 data->nb_sectors, 1854 data->pnum, 1855 data->file); 1856 data->done = true; 1857 } 1858 1859 /* 1860 * Synchronous wrapper around bdrv_co_get_block_status_above(). 1861 * 1862 * See bdrv_co_get_block_status_above() for details. 1863 */ 1864 int64_t bdrv_get_block_status_above(BlockDriverState *bs, 1865 BlockDriverState *base, 1866 int64_t sector_num, 1867 int nb_sectors, int *pnum, 1868 BlockDriverState **file) 1869 { 1870 Coroutine *co; 1871 BdrvCoGetBlockStatusData data = { 1872 .bs = bs, 1873 .base = base, 1874 .file = file, 1875 .sector_num = sector_num, 1876 .nb_sectors = nb_sectors, 1877 .pnum = pnum, 1878 .done = false, 1879 }; 1880 1881 if (qemu_in_coroutine()) { 1882 /* Fast-path if already in coroutine context */ 1883 bdrv_get_block_status_above_co_entry(&data); 1884 } else { 1885 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry, 1886 &data); 1887 bdrv_coroutine_enter(bs, co); 1888 BDRV_POLL_WHILE(bs, !data.done); 1889 } 1890 return data.ret; 1891 } 1892 1893 int64_t bdrv_get_block_status(BlockDriverState *bs, 1894 int64_t sector_num, 1895 int nb_sectors, int *pnum, 1896 BlockDriverState **file) 1897 { 1898 return bdrv_get_block_status_above(bs, backing_bs(bs), 1899 sector_num, nb_sectors, pnum, file); 1900 } 1901 1902 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, 1903 int nb_sectors, int *pnum) 1904 { 1905 BlockDriverState *file; 1906 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum, 1907 &file); 1908 if (ret < 0) { 1909 return ret; 1910 } 1911 return !!(ret & BDRV_BLOCK_ALLOCATED); 1912 } 1913 1914 /* 1915 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 1916 * 1917 * Return true if the given sector is allocated in any image between 1918 * BASE and TOP (inclusive). BASE can be NULL to check if the given 1919 * sector is allocated in any image of the chain. Return false otherwise. 1920 * 1921 * 'pnum' is set to the number of sectors (including and immediately following 1922 * the specified sector) that are known to be in the same 1923 * allocated/unallocated state. 1924 * 1925 */ 1926 int bdrv_is_allocated_above(BlockDriverState *top, 1927 BlockDriverState *base, 1928 int64_t sector_num, 1929 int nb_sectors, int *pnum) 1930 { 1931 BlockDriverState *intermediate; 1932 int ret, n = nb_sectors; 1933 1934 intermediate = top; 1935 while (intermediate && intermediate != base) { 1936 int pnum_inter; 1937 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors, 1938 &pnum_inter); 1939 if (ret < 0) { 1940 return ret; 1941 } else if (ret) { 1942 *pnum = pnum_inter; 1943 return 1; 1944 } 1945 1946 /* 1947 * [sector_num, nb_sectors] is unallocated on top but intermediate 1948 * might have 1949 * 1950 * [sector_num+x, nr_sectors] allocated. 1951 */ 1952 if (n > pnum_inter && 1953 (intermediate == top || 1954 sector_num + pnum_inter < intermediate->total_sectors)) { 1955 n = pnum_inter; 1956 } 1957 1958 intermediate = backing_bs(intermediate); 1959 } 1960 1961 *pnum = n; 1962 return 0; 1963 } 1964 1965 typedef struct BdrvVmstateCo { 1966 BlockDriverState *bs; 1967 QEMUIOVector *qiov; 1968 int64_t pos; 1969 bool is_read; 1970 int ret; 1971 } BdrvVmstateCo; 1972 1973 static int coroutine_fn 1974 bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 1975 bool is_read) 1976 { 1977 BlockDriver *drv = bs->drv; 1978 1979 if (!drv) { 1980 return -ENOMEDIUM; 1981 } else if (drv->bdrv_load_vmstate) { 1982 return is_read ? drv->bdrv_load_vmstate(bs, qiov, pos) 1983 : drv->bdrv_save_vmstate(bs, qiov, pos); 1984 } else if (bs->file) { 1985 return bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read); 1986 } 1987 1988 return -ENOTSUP; 1989 } 1990 1991 static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque) 1992 { 1993 BdrvVmstateCo *co = opaque; 1994 co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read); 1995 } 1996 1997 static inline int 1998 bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 1999 bool is_read) 2000 { 2001 if (qemu_in_coroutine()) { 2002 return bdrv_co_rw_vmstate(bs, qiov, pos, is_read); 2003 } else { 2004 BdrvVmstateCo data = { 2005 .bs = bs, 2006 .qiov = qiov, 2007 .pos = pos, 2008 .is_read = is_read, 2009 .ret = -EINPROGRESS, 2010 }; 2011 Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data); 2012 2013 bdrv_coroutine_enter(bs, co); 2014 while (data.ret == -EINPROGRESS) { 2015 aio_poll(bdrv_get_aio_context(bs), true); 2016 } 2017 return data.ret; 2018 } 2019 } 2020 2021 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 2022 int64_t pos, int size) 2023 { 2024 QEMUIOVector qiov; 2025 struct iovec iov = { 2026 .iov_base = (void *) buf, 2027 .iov_len = size, 2028 }; 2029 int ret; 2030 2031 qemu_iovec_init_external(&qiov, &iov, 1); 2032 2033 ret = bdrv_writev_vmstate(bs, &qiov, pos); 2034 if (ret < 0) { 2035 return ret; 2036 } 2037 2038 return size; 2039 } 2040 2041 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2042 { 2043 return bdrv_rw_vmstate(bs, qiov, pos, false); 2044 } 2045 2046 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 2047 int64_t pos, int size) 2048 { 2049 QEMUIOVector qiov; 2050 struct iovec iov = { 2051 .iov_base = buf, 2052 .iov_len = size, 2053 }; 2054 int ret; 2055 2056 qemu_iovec_init_external(&qiov, &iov, 1); 2057 ret = bdrv_readv_vmstate(bs, &qiov, pos); 2058 if (ret < 0) { 2059 return ret; 2060 } 2061 2062 return size; 2063 } 2064 2065 int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2066 { 2067 return bdrv_rw_vmstate(bs, qiov, pos, true); 2068 } 2069 2070 /**************************************************************/ 2071 /* async I/Os */ 2072 2073 BlockAIOCB *bdrv_aio_readv(BdrvChild *child, int64_t sector_num, 2074 QEMUIOVector *qiov, int nb_sectors, 2075 BlockCompletionFunc *cb, void *opaque) 2076 { 2077 trace_bdrv_aio_readv(child->bs, sector_num, nb_sectors, opaque); 2078 2079 assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size); 2080 return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov, 2081 0, cb, opaque, false); 2082 } 2083 2084 BlockAIOCB *bdrv_aio_writev(BdrvChild *child, int64_t sector_num, 2085 QEMUIOVector *qiov, int nb_sectors, 2086 BlockCompletionFunc *cb, void *opaque) 2087 { 2088 trace_bdrv_aio_writev(child->bs, sector_num, nb_sectors, opaque); 2089 2090 assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size); 2091 return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov, 2092 0, cb, opaque, true); 2093 } 2094 2095 void bdrv_aio_cancel(BlockAIOCB *acb) 2096 { 2097 qemu_aio_ref(acb); 2098 bdrv_aio_cancel_async(acb); 2099 while (acb->refcnt > 1) { 2100 if (acb->aiocb_info->get_aio_context) { 2101 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 2102 } else if (acb->bs) { 2103 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 2104 * assert that we're not using an I/O thread. Thread-safe 2105 * code should use bdrv_aio_cancel_async exclusively. 2106 */ 2107 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 2108 aio_poll(bdrv_get_aio_context(acb->bs), true); 2109 } else { 2110 abort(); 2111 } 2112 } 2113 qemu_aio_unref(acb); 2114 } 2115 2116 /* Async version of aio cancel. The caller is not blocked if the acb implements 2117 * cancel_async, otherwise we do nothing and let the request normally complete. 2118 * In either case the completion callback must be called. */ 2119 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2120 { 2121 if (acb->aiocb_info->cancel_async) { 2122 acb->aiocb_info->cancel_async(acb); 2123 } 2124 } 2125 2126 /**************************************************************/ 2127 /* async block device emulation */ 2128 2129 typedef struct BlockRequest { 2130 union { 2131 /* Used during read, write, trim */ 2132 struct { 2133 int64_t offset; 2134 int bytes; 2135 int flags; 2136 QEMUIOVector *qiov; 2137 }; 2138 /* Used during ioctl */ 2139 struct { 2140 int req; 2141 void *buf; 2142 }; 2143 }; 2144 BlockCompletionFunc *cb; 2145 void *opaque; 2146 2147 int error; 2148 } BlockRequest; 2149 2150 typedef struct BlockAIOCBCoroutine { 2151 BlockAIOCB common; 2152 BdrvChild *child; 2153 BlockRequest req; 2154 bool is_write; 2155 bool need_bh; 2156 bool *done; 2157 } BlockAIOCBCoroutine; 2158 2159 static const AIOCBInfo bdrv_em_co_aiocb_info = { 2160 .aiocb_size = sizeof(BlockAIOCBCoroutine), 2161 }; 2162 2163 static void bdrv_co_complete(BlockAIOCBCoroutine *acb) 2164 { 2165 if (!acb->need_bh) { 2166 bdrv_dec_in_flight(acb->common.bs); 2167 acb->common.cb(acb->common.opaque, acb->req.error); 2168 qemu_aio_unref(acb); 2169 } 2170 } 2171 2172 static void bdrv_co_em_bh(void *opaque) 2173 { 2174 BlockAIOCBCoroutine *acb = opaque; 2175 2176 assert(!acb->need_bh); 2177 bdrv_co_complete(acb); 2178 } 2179 2180 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb) 2181 { 2182 acb->need_bh = false; 2183 if (acb->req.error != -EINPROGRESS) { 2184 BlockDriverState *bs = acb->common.bs; 2185 2186 aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb); 2187 } 2188 } 2189 2190 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */ 2191 static void coroutine_fn bdrv_co_do_rw(void *opaque) 2192 { 2193 BlockAIOCBCoroutine *acb = opaque; 2194 2195 if (!acb->is_write) { 2196 acb->req.error = bdrv_co_preadv(acb->child, acb->req.offset, 2197 acb->req.qiov->size, acb->req.qiov, acb->req.flags); 2198 } else { 2199 acb->req.error = bdrv_co_pwritev(acb->child, acb->req.offset, 2200 acb->req.qiov->size, acb->req.qiov, acb->req.flags); 2201 } 2202 2203 bdrv_co_complete(acb); 2204 } 2205 2206 static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child, 2207 int64_t offset, 2208 QEMUIOVector *qiov, 2209 BdrvRequestFlags flags, 2210 BlockCompletionFunc *cb, 2211 void *opaque, 2212 bool is_write) 2213 { 2214 Coroutine *co; 2215 BlockAIOCBCoroutine *acb; 2216 2217 /* Matched by bdrv_co_complete's bdrv_dec_in_flight. */ 2218 bdrv_inc_in_flight(child->bs); 2219 2220 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, child->bs, cb, opaque); 2221 acb->child = child; 2222 acb->need_bh = true; 2223 acb->req.error = -EINPROGRESS; 2224 acb->req.offset = offset; 2225 acb->req.qiov = qiov; 2226 acb->req.flags = flags; 2227 acb->is_write = is_write; 2228 2229 co = qemu_coroutine_create(bdrv_co_do_rw, acb); 2230 bdrv_coroutine_enter(child->bs, co); 2231 2232 bdrv_co_maybe_schedule_bh(acb); 2233 return &acb->common; 2234 } 2235 2236 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque) 2237 { 2238 BlockAIOCBCoroutine *acb = opaque; 2239 BlockDriverState *bs = acb->common.bs; 2240 2241 acb->req.error = bdrv_co_flush(bs); 2242 bdrv_co_complete(acb); 2243 } 2244 2245 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs, 2246 BlockCompletionFunc *cb, void *opaque) 2247 { 2248 trace_bdrv_aio_flush(bs, opaque); 2249 2250 Coroutine *co; 2251 BlockAIOCBCoroutine *acb; 2252 2253 /* Matched by bdrv_co_complete's bdrv_dec_in_flight. */ 2254 bdrv_inc_in_flight(bs); 2255 2256 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2257 acb->need_bh = true; 2258 acb->req.error = -EINPROGRESS; 2259 2260 co = qemu_coroutine_create(bdrv_aio_flush_co_entry, acb); 2261 bdrv_coroutine_enter(bs, co); 2262 2263 bdrv_co_maybe_schedule_bh(acb); 2264 return &acb->common; 2265 } 2266 2267 /**************************************************************/ 2268 /* Coroutine block device emulation */ 2269 2270 typedef struct FlushCo { 2271 BlockDriverState *bs; 2272 int ret; 2273 } FlushCo; 2274 2275 2276 static void coroutine_fn bdrv_flush_co_entry(void *opaque) 2277 { 2278 FlushCo *rwco = opaque; 2279 2280 rwco->ret = bdrv_co_flush(rwco->bs); 2281 } 2282 2283 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2284 { 2285 int current_gen; 2286 int ret = 0; 2287 2288 bdrv_inc_in_flight(bs); 2289 2290 if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 2291 bdrv_is_sg(bs)) { 2292 goto early_exit; 2293 } 2294 2295 current_gen = bs->write_gen; 2296 2297 /* Wait until any previous flushes are completed */ 2298 while (bs->active_flush_req) { 2299 qemu_co_queue_wait(&bs->flush_queue, NULL); 2300 } 2301 2302 bs->active_flush_req = true; 2303 2304 /* Write back all layers by calling one driver function */ 2305 if (bs->drv->bdrv_co_flush) { 2306 ret = bs->drv->bdrv_co_flush(bs); 2307 goto out; 2308 } 2309 2310 /* Write back cached data to the OS even with cache=unsafe */ 2311 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 2312 if (bs->drv->bdrv_co_flush_to_os) { 2313 ret = bs->drv->bdrv_co_flush_to_os(bs); 2314 if (ret < 0) { 2315 goto out; 2316 } 2317 } 2318 2319 /* But don't actually force it to the disk with cache=unsafe */ 2320 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2321 goto flush_parent; 2322 } 2323 2324 /* Check if we really need to flush anything */ 2325 if (bs->flushed_gen == current_gen) { 2326 goto flush_parent; 2327 } 2328 2329 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 2330 if (bs->drv->bdrv_co_flush_to_disk) { 2331 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2332 } else if (bs->drv->bdrv_aio_flush) { 2333 BlockAIOCB *acb; 2334 CoroutineIOCompletion co = { 2335 .coroutine = qemu_coroutine_self(), 2336 }; 2337 2338 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 2339 if (acb == NULL) { 2340 ret = -EIO; 2341 } else { 2342 qemu_coroutine_yield(); 2343 ret = co.ret; 2344 } 2345 } else { 2346 /* 2347 * Some block drivers always operate in either writethrough or unsafe 2348 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 2349 * know how the server works (because the behaviour is hardcoded or 2350 * depends on server-side configuration), so we can't ensure that 2351 * everything is safe on disk. Returning an error doesn't work because 2352 * that would break guests even if the server operates in writethrough 2353 * mode. 2354 * 2355 * Let's hope the user knows what he's doing. 2356 */ 2357 ret = 0; 2358 } 2359 2360 if (ret < 0) { 2361 goto out; 2362 } 2363 2364 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 2365 * in the case of cache=unsafe, so there are no useless flushes. 2366 */ 2367 flush_parent: 2368 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2369 out: 2370 /* Notify any pending flushes that we have completed */ 2371 if (ret == 0) { 2372 bs->flushed_gen = current_gen; 2373 } 2374 bs->active_flush_req = false; 2375 /* Return value is ignored - it's ok if wait queue is empty */ 2376 qemu_co_queue_next(&bs->flush_queue); 2377 2378 early_exit: 2379 bdrv_dec_in_flight(bs); 2380 return ret; 2381 } 2382 2383 int bdrv_flush(BlockDriverState *bs) 2384 { 2385 Coroutine *co; 2386 FlushCo flush_co = { 2387 .bs = bs, 2388 .ret = NOT_DONE, 2389 }; 2390 2391 if (qemu_in_coroutine()) { 2392 /* Fast-path if already in coroutine context */ 2393 bdrv_flush_co_entry(&flush_co); 2394 } else { 2395 co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co); 2396 bdrv_coroutine_enter(bs, co); 2397 BDRV_POLL_WHILE(bs, flush_co.ret == NOT_DONE); 2398 } 2399 2400 return flush_co.ret; 2401 } 2402 2403 typedef struct DiscardCo { 2404 BlockDriverState *bs; 2405 int64_t offset; 2406 int count; 2407 int ret; 2408 } DiscardCo; 2409 static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque) 2410 { 2411 DiscardCo *rwco = opaque; 2412 2413 rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->count); 2414 } 2415 2416 int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset, 2417 int count) 2418 { 2419 BdrvTrackedRequest req; 2420 int max_pdiscard, ret; 2421 int head, tail, align; 2422 2423 if (!bs->drv) { 2424 return -ENOMEDIUM; 2425 } 2426 2427 ret = bdrv_check_byte_request(bs, offset, count); 2428 if (ret < 0) { 2429 return ret; 2430 } else if (bs->read_only) { 2431 return -EPERM; 2432 } 2433 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 2434 2435 /* Do nothing if disabled. */ 2436 if (!(bs->open_flags & BDRV_O_UNMAP)) { 2437 return 0; 2438 } 2439 2440 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 2441 return 0; 2442 } 2443 2444 /* Discard is advisory, but some devices track and coalesce 2445 * unaligned requests, so we must pass everything down rather than 2446 * round here. Still, most devices will just silently ignore 2447 * unaligned requests (by returning -ENOTSUP), so we must fragment 2448 * the request accordingly. */ 2449 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 2450 assert(align % bs->bl.request_alignment == 0); 2451 head = offset % align; 2452 tail = (offset + count) % align; 2453 2454 bdrv_inc_in_flight(bs); 2455 tracked_request_begin(&req, bs, offset, count, BDRV_TRACKED_DISCARD); 2456 2457 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req); 2458 if (ret < 0) { 2459 goto out; 2460 } 2461 2462 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX), 2463 align); 2464 assert(max_pdiscard >= bs->bl.request_alignment); 2465 2466 while (count > 0) { 2467 int ret; 2468 int num = count; 2469 2470 if (head) { 2471 /* Make small requests to get to alignment boundaries. */ 2472 num = MIN(count, align - head); 2473 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 2474 num %= bs->bl.request_alignment; 2475 } 2476 head = (head + num) % align; 2477 assert(num < max_pdiscard); 2478 } else if (tail) { 2479 if (num > align) { 2480 /* Shorten the request to the last aligned cluster. */ 2481 num -= tail; 2482 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 2483 tail > bs->bl.request_alignment) { 2484 tail %= bs->bl.request_alignment; 2485 num -= tail; 2486 } 2487 } 2488 /* limit request size */ 2489 if (num > max_pdiscard) { 2490 num = max_pdiscard; 2491 } 2492 2493 if (bs->drv->bdrv_co_pdiscard) { 2494 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 2495 } else { 2496 BlockAIOCB *acb; 2497 CoroutineIOCompletion co = { 2498 .coroutine = qemu_coroutine_self(), 2499 }; 2500 2501 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 2502 bdrv_co_io_em_complete, &co); 2503 if (acb == NULL) { 2504 ret = -EIO; 2505 goto out; 2506 } else { 2507 qemu_coroutine_yield(); 2508 ret = co.ret; 2509 } 2510 } 2511 if (ret && ret != -ENOTSUP) { 2512 goto out; 2513 } 2514 2515 offset += num; 2516 count -= num; 2517 } 2518 ret = 0; 2519 out: 2520 ++bs->write_gen; 2521 bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS, 2522 req.bytes >> BDRV_SECTOR_BITS); 2523 tracked_request_end(&req); 2524 bdrv_dec_in_flight(bs); 2525 return ret; 2526 } 2527 2528 int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int count) 2529 { 2530 Coroutine *co; 2531 DiscardCo rwco = { 2532 .bs = bs, 2533 .offset = offset, 2534 .count = count, 2535 .ret = NOT_DONE, 2536 }; 2537 2538 if (qemu_in_coroutine()) { 2539 /* Fast-path if already in coroutine context */ 2540 bdrv_pdiscard_co_entry(&rwco); 2541 } else { 2542 co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco); 2543 bdrv_coroutine_enter(bs, co); 2544 BDRV_POLL_WHILE(bs, rwco.ret == NOT_DONE); 2545 } 2546 2547 return rwco.ret; 2548 } 2549 2550 int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 2551 { 2552 BlockDriver *drv = bs->drv; 2553 CoroutineIOCompletion co = { 2554 .coroutine = qemu_coroutine_self(), 2555 }; 2556 BlockAIOCB *acb; 2557 2558 bdrv_inc_in_flight(bs); 2559 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 2560 co.ret = -ENOTSUP; 2561 goto out; 2562 } 2563 2564 if (drv->bdrv_co_ioctl) { 2565 co.ret = drv->bdrv_co_ioctl(bs, req, buf); 2566 } else { 2567 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 2568 if (!acb) { 2569 co.ret = -ENOTSUP; 2570 goto out; 2571 } 2572 qemu_coroutine_yield(); 2573 } 2574 out: 2575 bdrv_dec_in_flight(bs); 2576 return co.ret; 2577 } 2578 2579 void *qemu_blockalign(BlockDriverState *bs, size_t size) 2580 { 2581 return qemu_memalign(bdrv_opt_mem_align(bs), size); 2582 } 2583 2584 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 2585 { 2586 return memset(qemu_blockalign(bs, size), 0, size); 2587 } 2588 2589 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 2590 { 2591 size_t align = bdrv_opt_mem_align(bs); 2592 2593 /* Ensure that NULL is never returned on success */ 2594 assert(align > 0); 2595 if (size == 0) { 2596 size = align; 2597 } 2598 2599 return qemu_try_memalign(align, size); 2600 } 2601 2602 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 2603 { 2604 void *mem = qemu_try_blockalign(bs, size); 2605 2606 if (mem) { 2607 memset(mem, 0, size); 2608 } 2609 2610 return mem; 2611 } 2612 2613 /* 2614 * Check if all memory in this vector is sector aligned. 2615 */ 2616 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 2617 { 2618 int i; 2619 size_t alignment = bdrv_min_mem_align(bs); 2620 2621 for (i = 0; i < qiov->niov; i++) { 2622 if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 2623 return false; 2624 } 2625 if (qiov->iov[i].iov_len % alignment) { 2626 return false; 2627 } 2628 } 2629 2630 return true; 2631 } 2632 2633 void bdrv_add_before_write_notifier(BlockDriverState *bs, 2634 NotifierWithReturn *notifier) 2635 { 2636 notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 2637 } 2638 2639 void bdrv_io_plug(BlockDriverState *bs) 2640 { 2641 BdrvChild *child; 2642 2643 QLIST_FOREACH(child, &bs->children, next) { 2644 bdrv_io_plug(child->bs); 2645 } 2646 2647 if (bs->io_plugged++ == 0) { 2648 BlockDriver *drv = bs->drv; 2649 if (drv && drv->bdrv_io_plug) { 2650 drv->bdrv_io_plug(bs); 2651 } 2652 } 2653 } 2654 2655 void bdrv_io_unplug(BlockDriverState *bs) 2656 { 2657 BdrvChild *child; 2658 2659 assert(bs->io_plugged); 2660 if (--bs->io_plugged == 0) { 2661 BlockDriver *drv = bs->drv; 2662 if (drv && drv->bdrv_io_unplug) { 2663 drv->bdrv_io_unplug(bs); 2664 } 2665 } 2666 2667 QLIST_FOREACH(child, &bs->children, next) { 2668 bdrv_io_unplug(child->bs); 2669 } 2670 } 2671