1 /* 2 * Image mirroring 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Paolo Bonzini <pbonzini@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU LGPL, version 2 or later. 10 * See the COPYING.LIB file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "trace.h" 16 #include "block/blockjob.h" 17 #include "block/block_int.h" 18 #include "sysemu/block-backend.h" 19 #include "qapi/qmp/qerror.h" 20 #include "qemu/ratelimit.h" 21 #include "qemu/bitmap.h" 22 #include "qemu/error-report.h" 23 24 #define SLICE_TIME 100000000ULL /* ns */ 25 #define MAX_IN_FLIGHT 16 26 #define DEFAULT_MIRROR_BUF_SIZE (10 << 20) 27 28 /* The mirroring buffer is a list of granularity-sized chunks. 29 * Free chunks are organized in a list. 30 */ 31 typedef struct MirrorBuffer { 32 QSIMPLEQ_ENTRY(MirrorBuffer) next; 33 } MirrorBuffer; 34 35 typedef struct MirrorBlockJob { 36 BlockJob common; 37 RateLimit limit; 38 BlockDriverState *target; 39 BlockDriverState *base; 40 /* The name of the graph node to replace */ 41 char *replaces; 42 /* The BDS to replace */ 43 BlockDriverState *to_replace; 44 /* Used to block operations on the drive-mirror-replace target */ 45 Error *replace_blocker; 46 bool is_none_mode; 47 BlockdevOnError on_source_error, on_target_error; 48 bool synced; 49 bool should_complete; 50 int64_t granularity; 51 size_t buf_size; 52 int64_t bdev_length; 53 unsigned long *cow_bitmap; 54 BdrvDirtyBitmap *dirty_bitmap; 55 HBitmapIter hbi; 56 uint8_t *buf; 57 QSIMPLEQ_HEAD(, MirrorBuffer) buf_free; 58 int buf_free_count; 59 60 unsigned long *in_flight_bitmap; 61 int in_flight; 62 int sectors_in_flight; 63 int ret; 64 bool unmap; 65 bool waiting_for_io; 66 int target_cluster_sectors; 67 int max_iov; 68 } MirrorBlockJob; 69 70 typedef struct MirrorOp { 71 MirrorBlockJob *s; 72 QEMUIOVector qiov; 73 int64_t sector_num; 74 int nb_sectors; 75 } MirrorOp; 76 77 static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read, 78 int error) 79 { 80 s->synced = false; 81 if (read) { 82 return block_job_error_action(&s->common, s->common.bs, 83 s->on_source_error, true, error); 84 } else { 85 return block_job_error_action(&s->common, s->target, 86 s->on_target_error, false, error); 87 } 88 } 89 90 static void mirror_iteration_done(MirrorOp *op, int ret) 91 { 92 MirrorBlockJob *s = op->s; 93 struct iovec *iov; 94 int64_t chunk_num; 95 int i, nb_chunks, sectors_per_chunk; 96 97 trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret); 98 99 s->in_flight--; 100 s->sectors_in_flight -= op->nb_sectors; 101 iov = op->qiov.iov; 102 for (i = 0; i < op->qiov.niov; i++) { 103 MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base; 104 QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next); 105 s->buf_free_count++; 106 } 107 108 sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 109 chunk_num = op->sector_num / sectors_per_chunk; 110 nb_chunks = op->nb_sectors / sectors_per_chunk; 111 bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks); 112 if (ret >= 0) { 113 if (s->cow_bitmap) { 114 bitmap_set(s->cow_bitmap, chunk_num, nb_chunks); 115 } 116 s->common.offset += (uint64_t)op->nb_sectors * BDRV_SECTOR_SIZE; 117 } 118 119 qemu_iovec_destroy(&op->qiov); 120 g_free(op); 121 122 if (s->waiting_for_io) { 123 qemu_coroutine_enter(s->common.co, NULL); 124 } 125 } 126 127 static void mirror_write_complete(void *opaque, int ret) 128 { 129 MirrorOp *op = opaque; 130 MirrorBlockJob *s = op->s; 131 if (ret < 0) { 132 BlockErrorAction action; 133 134 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->sector_num, op->nb_sectors); 135 action = mirror_error_action(s, false, -ret); 136 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) { 137 s->ret = ret; 138 } 139 } 140 mirror_iteration_done(op, ret); 141 } 142 143 static void mirror_read_complete(void *opaque, int ret) 144 { 145 MirrorOp *op = opaque; 146 MirrorBlockJob *s = op->s; 147 if (ret < 0) { 148 BlockErrorAction action; 149 150 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->sector_num, op->nb_sectors); 151 action = mirror_error_action(s, true, -ret); 152 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) { 153 s->ret = ret; 154 } 155 156 mirror_iteration_done(op, ret); 157 return; 158 } 159 bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors, 160 mirror_write_complete, op); 161 } 162 163 /* Round sector_num and/or nb_sectors to target cluster if COW is needed, and 164 * return the offset of the adjusted tail sector against original. */ 165 static int mirror_cow_align(MirrorBlockJob *s, 166 int64_t *sector_num, 167 int *nb_sectors) 168 { 169 bool need_cow; 170 int ret = 0; 171 int chunk_sectors = s->granularity >> BDRV_SECTOR_BITS; 172 int64_t align_sector_num = *sector_num; 173 int align_nb_sectors = *nb_sectors; 174 int max_sectors = chunk_sectors * s->max_iov; 175 176 need_cow = !test_bit(*sector_num / chunk_sectors, s->cow_bitmap); 177 need_cow |= !test_bit((*sector_num + *nb_sectors - 1) / chunk_sectors, 178 s->cow_bitmap); 179 if (need_cow) { 180 bdrv_round_to_clusters(s->target, *sector_num, *nb_sectors, 181 &align_sector_num, &align_nb_sectors); 182 } 183 184 if (align_nb_sectors > max_sectors) { 185 align_nb_sectors = max_sectors; 186 if (need_cow) { 187 align_nb_sectors = QEMU_ALIGN_DOWN(align_nb_sectors, 188 s->target_cluster_sectors); 189 } 190 } 191 192 ret = align_sector_num + align_nb_sectors - (*sector_num + *nb_sectors); 193 *sector_num = align_sector_num; 194 *nb_sectors = align_nb_sectors; 195 assert(ret >= 0); 196 return ret; 197 } 198 199 static inline void mirror_wait_for_io(MirrorBlockJob *s) 200 { 201 assert(!s->waiting_for_io); 202 s->waiting_for_io = true; 203 qemu_coroutine_yield(); 204 s->waiting_for_io = false; 205 } 206 207 /* Submit async read while handling COW. 208 * Returns: nb_sectors if no alignment is necessary, or 209 * (new_end - sector_num) if tail is rounded up or down due to 210 * alignment or buffer limit. 211 */ 212 static int mirror_do_read(MirrorBlockJob *s, int64_t sector_num, 213 int nb_sectors) 214 { 215 BlockDriverState *source = s->common.bs; 216 int sectors_per_chunk, nb_chunks; 217 int ret = nb_sectors; 218 MirrorOp *op; 219 220 sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 221 222 /* We can only handle as much as buf_size at a time. */ 223 nb_sectors = MIN(s->buf_size >> BDRV_SECTOR_BITS, nb_sectors); 224 assert(nb_sectors); 225 226 if (s->cow_bitmap) { 227 ret += mirror_cow_align(s, §or_num, &nb_sectors); 228 } 229 assert(nb_sectors << BDRV_SECTOR_BITS <= s->buf_size); 230 /* The sector range must meet granularity because: 231 * 1) Caller passes in aligned values; 232 * 2) mirror_cow_align is used only when target cluster is larger. */ 233 assert(!(nb_sectors % sectors_per_chunk)); 234 assert(!(sector_num % sectors_per_chunk)); 235 nb_chunks = nb_sectors / sectors_per_chunk; 236 237 while (s->buf_free_count < nb_chunks) { 238 trace_mirror_yield_in_flight(s, sector_num, s->in_flight); 239 mirror_wait_for_io(s); 240 } 241 242 /* Allocate a MirrorOp that is used as an AIO callback. */ 243 op = g_new(MirrorOp, 1); 244 op->s = s; 245 op->sector_num = sector_num; 246 op->nb_sectors = nb_sectors; 247 248 /* Now make a QEMUIOVector taking enough granularity-sized chunks 249 * from s->buf_free. 250 */ 251 qemu_iovec_init(&op->qiov, nb_chunks); 252 while (nb_chunks-- > 0) { 253 MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free); 254 size_t remaining = nb_sectors * BDRV_SECTOR_SIZE - op->qiov.size; 255 256 QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next); 257 s->buf_free_count--; 258 qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining)); 259 } 260 261 /* Copy the dirty cluster. */ 262 s->in_flight++; 263 s->sectors_in_flight += nb_sectors; 264 trace_mirror_one_iteration(s, sector_num, nb_sectors); 265 266 bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors, 267 mirror_read_complete, op); 268 return ret; 269 } 270 271 static void mirror_do_zero_or_discard(MirrorBlockJob *s, 272 int64_t sector_num, 273 int nb_sectors, 274 bool is_discard) 275 { 276 MirrorOp *op; 277 278 /* Allocate a MirrorOp that is used as an AIO callback. The qiov is zeroed 279 * so the freeing in mirror_iteration_done is nop. */ 280 op = g_new0(MirrorOp, 1); 281 op->s = s; 282 op->sector_num = sector_num; 283 op->nb_sectors = nb_sectors; 284 285 s->in_flight++; 286 s->sectors_in_flight += nb_sectors; 287 if (is_discard) { 288 bdrv_aio_discard(s->target, sector_num, op->nb_sectors, 289 mirror_write_complete, op); 290 } else { 291 bdrv_aio_write_zeroes(s->target, sector_num, op->nb_sectors, 292 s->unmap ? BDRV_REQ_MAY_UNMAP : 0, 293 mirror_write_complete, op); 294 } 295 } 296 297 static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) 298 { 299 BlockDriverState *source = s->common.bs; 300 int64_t sector_num; 301 uint64_t delay_ns = 0; 302 /* At least the first dirty chunk is mirrored in one iteration. */ 303 int nb_chunks = 1; 304 int64_t end = s->bdev_length / BDRV_SECTOR_SIZE; 305 int sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 306 307 sector_num = hbitmap_iter_next(&s->hbi); 308 if (sector_num < 0) { 309 bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi); 310 sector_num = hbitmap_iter_next(&s->hbi); 311 trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap)); 312 assert(sector_num >= 0); 313 } 314 315 /* Find the number of consective dirty chunks following the first dirty 316 * one, and wait for in flight requests in them. */ 317 while (nb_chunks * sectors_per_chunk < (s->buf_size >> BDRV_SECTOR_BITS)) { 318 int64_t hbitmap_next; 319 int64_t next_sector = sector_num + nb_chunks * sectors_per_chunk; 320 int64_t next_chunk = next_sector / sectors_per_chunk; 321 if (next_sector >= end || 322 !bdrv_get_dirty(source, s->dirty_bitmap, next_sector)) { 323 break; 324 } 325 if (test_bit(next_chunk, s->in_flight_bitmap)) { 326 if (nb_chunks > 0) { 327 break; 328 } 329 trace_mirror_yield_in_flight(s, next_sector, s->in_flight); 330 mirror_wait_for_io(s); 331 /* Now retry. */ 332 } else { 333 hbitmap_next = hbitmap_iter_next(&s->hbi); 334 assert(hbitmap_next == next_sector); 335 nb_chunks++; 336 } 337 } 338 339 /* Clear dirty bits before querying the block status, because 340 * calling bdrv_get_block_status_above could yield - if some blocks are 341 * marked dirty in this window, we need to know. 342 */ 343 bdrv_reset_dirty_bitmap(s->dirty_bitmap, sector_num, 344 nb_chunks * sectors_per_chunk); 345 bitmap_set(s->in_flight_bitmap, sector_num / sectors_per_chunk, nb_chunks); 346 while (nb_chunks > 0 && sector_num < end) { 347 int ret; 348 int io_sectors; 349 BlockDriverState *file; 350 enum MirrorMethod { 351 MIRROR_METHOD_COPY, 352 MIRROR_METHOD_ZERO, 353 MIRROR_METHOD_DISCARD 354 } mirror_method = MIRROR_METHOD_COPY; 355 356 assert(!(sector_num % sectors_per_chunk)); 357 ret = bdrv_get_block_status_above(source, NULL, sector_num, 358 nb_chunks * sectors_per_chunk, 359 &io_sectors, &file); 360 if (ret < 0) { 361 io_sectors = nb_chunks * sectors_per_chunk; 362 } 363 364 io_sectors -= io_sectors % sectors_per_chunk; 365 if (io_sectors < sectors_per_chunk) { 366 io_sectors = sectors_per_chunk; 367 } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) { 368 int64_t target_sector_num; 369 int target_nb_sectors; 370 bdrv_round_to_clusters(s->target, sector_num, io_sectors, 371 &target_sector_num, &target_nb_sectors); 372 if (target_sector_num == sector_num && 373 target_nb_sectors == io_sectors) { 374 mirror_method = ret & BDRV_BLOCK_ZERO ? 375 MIRROR_METHOD_ZERO : 376 MIRROR_METHOD_DISCARD; 377 } 378 } 379 380 switch (mirror_method) { 381 case MIRROR_METHOD_COPY: 382 io_sectors = mirror_do_read(s, sector_num, io_sectors); 383 break; 384 case MIRROR_METHOD_ZERO: 385 mirror_do_zero_or_discard(s, sector_num, io_sectors, false); 386 break; 387 case MIRROR_METHOD_DISCARD: 388 mirror_do_zero_or_discard(s, sector_num, io_sectors, true); 389 break; 390 default: 391 abort(); 392 } 393 assert(io_sectors); 394 sector_num += io_sectors; 395 nb_chunks -= io_sectors / sectors_per_chunk; 396 delay_ns += ratelimit_calculate_delay(&s->limit, io_sectors); 397 } 398 return delay_ns; 399 } 400 401 static void mirror_free_init(MirrorBlockJob *s) 402 { 403 int granularity = s->granularity; 404 size_t buf_size = s->buf_size; 405 uint8_t *buf = s->buf; 406 407 assert(s->buf_free_count == 0); 408 QSIMPLEQ_INIT(&s->buf_free); 409 while (buf_size != 0) { 410 MirrorBuffer *cur = (MirrorBuffer *)buf; 411 QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next); 412 s->buf_free_count++; 413 buf_size -= granularity; 414 buf += granularity; 415 } 416 } 417 418 static void mirror_drain(MirrorBlockJob *s) 419 { 420 while (s->in_flight > 0) { 421 mirror_wait_for_io(s); 422 } 423 } 424 425 typedef struct { 426 int ret; 427 } MirrorExitData; 428 429 static void mirror_exit(BlockJob *job, void *opaque) 430 { 431 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 432 MirrorExitData *data = opaque; 433 AioContext *replace_aio_context = NULL; 434 BlockDriverState *src = s->common.bs; 435 436 /* Make sure that the source BDS doesn't go away before we called 437 * block_job_completed(). */ 438 bdrv_ref(src); 439 440 if (s->to_replace) { 441 replace_aio_context = bdrv_get_aio_context(s->to_replace); 442 aio_context_acquire(replace_aio_context); 443 } 444 445 if (s->should_complete && data->ret == 0) { 446 BlockDriverState *to_replace = s->common.bs; 447 if (s->to_replace) { 448 to_replace = s->to_replace; 449 } 450 451 /* This was checked in mirror_start_job(), but meanwhile one of the 452 * nodes could have been newly attached to a BlockBackend. */ 453 if (to_replace->blk && s->target->blk) { 454 error_report("block job: Can't create node with two BlockBackends"); 455 data->ret = -EINVAL; 456 goto out; 457 } 458 459 if (bdrv_get_flags(s->target) != bdrv_get_flags(to_replace)) { 460 bdrv_reopen(s->target, bdrv_get_flags(to_replace), NULL); 461 } 462 bdrv_replace_in_backing_chain(to_replace, s->target); 463 } 464 465 out: 466 if (s->to_replace) { 467 bdrv_op_unblock_all(s->to_replace, s->replace_blocker); 468 error_free(s->replace_blocker); 469 bdrv_unref(s->to_replace); 470 } 471 if (replace_aio_context) { 472 aio_context_release(replace_aio_context); 473 } 474 g_free(s->replaces); 475 bdrv_op_unblock_all(s->target, s->common.blocker); 476 bdrv_unref(s->target); 477 block_job_completed(&s->common, data->ret); 478 g_free(data); 479 bdrv_drained_end(src); 480 bdrv_unref(src); 481 } 482 483 static void coroutine_fn mirror_run(void *opaque) 484 { 485 MirrorBlockJob *s = opaque; 486 MirrorExitData *data; 487 BlockDriverState *bs = s->common.bs; 488 int64_t sector_num, end, length; 489 uint64_t last_pause_ns; 490 BlockDriverInfo bdi; 491 char backing_filename[2]; /* we only need 2 characters because we are only 492 checking for a NULL string */ 493 int ret = 0; 494 int n; 495 int target_cluster_size = BDRV_SECTOR_SIZE; 496 497 if (block_job_is_cancelled(&s->common)) { 498 goto immediate_exit; 499 } 500 501 s->bdev_length = bdrv_getlength(bs); 502 if (s->bdev_length < 0) { 503 ret = s->bdev_length; 504 goto immediate_exit; 505 } else if (s->bdev_length == 0) { 506 /* Report BLOCK_JOB_READY and wait for complete. */ 507 block_job_event_ready(&s->common); 508 s->synced = true; 509 while (!block_job_is_cancelled(&s->common) && !s->should_complete) { 510 block_job_yield(&s->common); 511 } 512 s->common.cancelled = false; 513 goto immediate_exit; 514 } 515 516 length = DIV_ROUND_UP(s->bdev_length, s->granularity); 517 s->in_flight_bitmap = bitmap_new(length); 518 519 /* If we have no backing file yet in the destination, we cannot let 520 * the destination do COW. Instead, we copy sectors around the 521 * dirty data if needed. We need a bitmap to do that. 522 */ 523 bdrv_get_backing_filename(s->target, backing_filename, 524 sizeof(backing_filename)); 525 if (!bdrv_get_info(s->target, &bdi) && bdi.cluster_size) { 526 target_cluster_size = bdi.cluster_size; 527 } 528 if (backing_filename[0] && !s->target->backing 529 && s->granularity < target_cluster_size) { 530 s->buf_size = MAX(s->buf_size, target_cluster_size); 531 s->cow_bitmap = bitmap_new(length); 532 } 533 s->target_cluster_sectors = target_cluster_size >> BDRV_SECTOR_BITS; 534 s->max_iov = MIN(s->common.bs->bl.max_iov, s->target->bl.max_iov); 535 536 end = s->bdev_length / BDRV_SECTOR_SIZE; 537 s->buf = qemu_try_blockalign(bs, s->buf_size); 538 if (s->buf == NULL) { 539 ret = -ENOMEM; 540 goto immediate_exit; 541 } 542 543 mirror_free_init(s); 544 545 last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 546 if (!s->is_none_mode) { 547 /* First part, loop on the sectors and initialize the dirty bitmap. */ 548 BlockDriverState *base = s->base; 549 bool mark_all_dirty = s->base == NULL && !bdrv_has_zero_init(s->target); 550 551 for (sector_num = 0; sector_num < end; ) { 552 /* Just to make sure we are not exceeding int limit. */ 553 int nb_sectors = MIN(INT_MAX >> BDRV_SECTOR_BITS, 554 end - sector_num); 555 int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 556 557 if (now - last_pause_ns > SLICE_TIME) { 558 last_pause_ns = now; 559 block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, 0); 560 } 561 562 if (block_job_is_cancelled(&s->common)) { 563 goto immediate_exit; 564 } 565 566 ret = bdrv_is_allocated_above(bs, base, sector_num, nb_sectors, &n); 567 568 if (ret < 0) { 569 goto immediate_exit; 570 } 571 572 assert(n > 0); 573 if (ret == 1 || mark_all_dirty) { 574 bdrv_set_dirty_bitmap(s->dirty_bitmap, sector_num, n); 575 } 576 sector_num += n; 577 } 578 } 579 580 bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi); 581 for (;;) { 582 uint64_t delay_ns = 0; 583 int64_t cnt; 584 bool should_complete; 585 586 if (s->ret < 0) { 587 ret = s->ret; 588 goto immediate_exit; 589 } 590 591 cnt = bdrv_get_dirty_count(s->dirty_bitmap); 592 /* s->common.offset contains the number of bytes already processed so 593 * far, cnt is the number of dirty sectors remaining and 594 * s->sectors_in_flight is the number of sectors currently being 595 * processed; together those are the current total operation length */ 596 s->common.len = s->common.offset + 597 (cnt + s->sectors_in_flight) * BDRV_SECTOR_SIZE; 598 599 /* Note that even when no rate limit is applied we need to yield 600 * periodically with no pending I/O so that bdrv_drain_all() returns. 601 * We do so every SLICE_TIME nanoseconds, or when there is an error, 602 * or when the source is clean, whichever comes first. 603 */ 604 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - last_pause_ns < SLICE_TIME && 605 s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) { 606 if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 || 607 (cnt == 0 && s->in_flight > 0)) { 608 trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt); 609 mirror_wait_for_io(s); 610 continue; 611 } else if (cnt != 0) { 612 delay_ns = mirror_iteration(s); 613 } 614 } 615 616 should_complete = false; 617 if (s->in_flight == 0 && cnt == 0) { 618 trace_mirror_before_flush(s); 619 ret = bdrv_flush(s->target); 620 if (ret < 0) { 621 if (mirror_error_action(s, false, -ret) == 622 BLOCK_ERROR_ACTION_REPORT) { 623 goto immediate_exit; 624 } 625 } else { 626 /* We're out of the streaming phase. From now on, if the job 627 * is cancelled we will actually complete all pending I/O and 628 * report completion. This way, block-job-cancel will leave 629 * the target in a consistent state. 630 */ 631 if (!s->synced) { 632 block_job_event_ready(&s->common); 633 s->synced = true; 634 } 635 636 should_complete = s->should_complete || 637 block_job_is_cancelled(&s->common); 638 cnt = bdrv_get_dirty_count(s->dirty_bitmap); 639 } 640 } 641 642 if (cnt == 0 && should_complete) { 643 /* The dirty bitmap is not updated while operations are pending. 644 * If we're about to exit, wait for pending operations before 645 * calling bdrv_get_dirty_count(bs), or we may exit while the 646 * source has dirty data to copy! 647 * 648 * Note that I/O can be submitted by the guest while 649 * mirror_populate runs. 650 */ 651 trace_mirror_before_drain(s, cnt); 652 bdrv_drain(bs); 653 cnt = bdrv_get_dirty_count(s->dirty_bitmap); 654 } 655 656 ret = 0; 657 trace_mirror_before_sleep(s, cnt, s->synced, delay_ns); 658 if (!s->synced) { 659 block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 660 if (block_job_is_cancelled(&s->common)) { 661 break; 662 } 663 } else if (!should_complete) { 664 delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0); 665 block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 666 } else if (cnt == 0) { 667 /* The two disks are in sync. Exit and report successful 668 * completion. 669 */ 670 assert(QLIST_EMPTY(&bs->tracked_requests)); 671 s->common.cancelled = false; 672 break; 673 } 674 last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 675 } 676 677 immediate_exit: 678 if (s->in_flight > 0) { 679 /* We get here only if something went wrong. Either the job failed, 680 * or it was cancelled prematurely so that we do not guarantee that 681 * the target is a copy of the source. 682 */ 683 assert(ret < 0 || (!s->synced && block_job_is_cancelled(&s->common))); 684 mirror_drain(s); 685 } 686 687 assert(s->in_flight == 0); 688 qemu_vfree(s->buf); 689 g_free(s->cow_bitmap); 690 g_free(s->in_flight_bitmap); 691 bdrv_release_dirty_bitmap(bs, s->dirty_bitmap); 692 if (s->target->blk) { 693 blk_iostatus_disable(s->target->blk); 694 } 695 696 data = g_malloc(sizeof(*data)); 697 data->ret = ret; 698 /* Before we switch to target in mirror_exit, make sure data doesn't 699 * change. */ 700 bdrv_drained_begin(s->common.bs); 701 block_job_defer_to_main_loop(&s->common, mirror_exit, data); 702 } 703 704 static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp) 705 { 706 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 707 708 if (speed < 0) { 709 error_setg(errp, QERR_INVALID_PARAMETER, "speed"); 710 return; 711 } 712 ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME); 713 } 714 715 static void mirror_iostatus_reset(BlockJob *job) 716 { 717 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 718 719 if (s->target->blk) { 720 blk_iostatus_reset(s->target->blk); 721 } 722 } 723 724 static void mirror_complete(BlockJob *job, Error **errp) 725 { 726 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 727 Error *local_err = NULL; 728 int ret; 729 730 ret = bdrv_open_backing_file(s->target, NULL, "backing", &local_err); 731 if (ret < 0) { 732 error_propagate(errp, local_err); 733 return; 734 } 735 if (!s->synced) { 736 error_setg(errp, QERR_BLOCK_JOB_NOT_READY, job->id); 737 return; 738 } 739 740 /* check the target bs is not blocked and block all operations on it */ 741 if (s->replaces) { 742 AioContext *replace_aio_context; 743 744 s->to_replace = bdrv_find_node(s->replaces); 745 if (!s->to_replace) { 746 error_setg(errp, "Node name '%s' not found", s->replaces); 747 return; 748 } 749 750 replace_aio_context = bdrv_get_aio_context(s->to_replace); 751 aio_context_acquire(replace_aio_context); 752 753 error_setg(&s->replace_blocker, 754 "block device is in use by block-job-complete"); 755 bdrv_op_block_all(s->to_replace, s->replace_blocker); 756 bdrv_ref(s->to_replace); 757 758 aio_context_release(replace_aio_context); 759 } 760 761 s->should_complete = true; 762 block_job_enter(&s->common); 763 } 764 765 static const BlockJobDriver mirror_job_driver = { 766 .instance_size = sizeof(MirrorBlockJob), 767 .job_type = BLOCK_JOB_TYPE_MIRROR, 768 .set_speed = mirror_set_speed, 769 .iostatus_reset= mirror_iostatus_reset, 770 .complete = mirror_complete, 771 }; 772 773 static const BlockJobDriver commit_active_job_driver = { 774 .instance_size = sizeof(MirrorBlockJob), 775 .job_type = BLOCK_JOB_TYPE_COMMIT, 776 .set_speed = mirror_set_speed, 777 .iostatus_reset 778 = mirror_iostatus_reset, 779 .complete = mirror_complete, 780 }; 781 782 static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target, 783 const char *replaces, 784 int64_t speed, uint32_t granularity, 785 int64_t buf_size, 786 BlockdevOnError on_source_error, 787 BlockdevOnError on_target_error, 788 bool unmap, 789 BlockCompletionFunc *cb, 790 void *opaque, Error **errp, 791 const BlockJobDriver *driver, 792 bool is_none_mode, BlockDriverState *base) 793 { 794 MirrorBlockJob *s; 795 BlockDriverState *replaced_bs; 796 797 if (granularity == 0) { 798 granularity = bdrv_get_default_bitmap_granularity(target); 799 } 800 801 assert ((granularity & (granularity - 1)) == 0); 802 803 if ((on_source_error == BLOCKDEV_ON_ERROR_STOP || 804 on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) && 805 (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) { 806 error_setg(errp, QERR_INVALID_PARAMETER, "on-source-error"); 807 return; 808 } 809 810 if (buf_size < 0) { 811 error_setg(errp, "Invalid parameter 'buf-size'"); 812 return; 813 } 814 815 if (buf_size == 0) { 816 buf_size = DEFAULT_MIRROR_BUF_SIZE; 817 } 818 819 /* We can't support this case as long as the block layer can't handle 820 * multiple BlockBackends per BlockDriverState. */ 821 if (replaces) { 822 replaced_bs = bdrv_lookup_bs(replaces, replaces, errp); 823 if (replaced_bs == NULL) { 824 return; 825 } 826 } else { 827 replaced_bs = bs; 828 } 829 if (replaced_bs->blk && target->blk) { 830 error_setg(errp, "Can't create node with two BlockBackends"); 831 return; 832 } 833 834 s = block_job_create(driver, bs, speed, cb, opaque, errp); 835 if (!s) { 836 return; 837 } 838 839 s->replaces = g_strdup(replaces); 840 s->on_source_error = on_source_error; 841 s->on_target_error = on_target_error; 842 s->target = target; 843 s->is_none_mode = is_none_mode; 844 s->base = base; 845 s->granularity = granularity; 846 s->buf_size = ROUND_UP(buf_size, granularity); 847 s->unmap = unmap; 848 849 s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp); 850 if (!s->dirty_bitmap) { 851 g_free(s->replaces); 852 block_job_unref(&s->common); 853 return; 854 } 855 856 bdrv_op_block_all(s->target, s->common.blocker); 857 858 bdrv_set_enable_write_cache(s->target, true); 859 if (s->target->blk) { 860 blk_set_on_error(s->target->blk, on_target_error, on_target_error); 861 blk_iostatus_enable(s->target->blk); 862 } 863 s->common.co = qemu_coroutine_create(mirror_run); 864 trace_mirror_start(bs, s, s->common.co, opaque); 865 qemu_coroutine_enter(s->common.co, s); 866 } 867 868 void mirror_start(BlockDriverState *bs, BlockDriverState *target, 869 const char *replaces, 870 int64_t speed, uint32_t granularity, int64_t buf_size, 871 MirrorSyncMode mode, BlockdevOnError on_source_error, 872 BlockdevOnError on_target_error, 873 bool unmap, 874 BlockCompletionFunc *cb, 875 void *opaque, Error **errp) 876 { 877 bool is_none_mode; 878 BlockDriverState *base; 879 880 if (mode == MIRROR_SYNC_MODE_INCREMENTAL) { 881 error_setg(errp, "Sync mode 'incremental' not supported"); 882 return; 883 } 884 is_none_mode = mode == MIRROR_SYNC_MODE_NONE; 885 base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL; 886 mirror_start_job(bs, target, replaces, 887 speed, granularity, buf_size, 888 on_source_error, on_target_error, unmap, cb, opaque, errp, 889 &mirror_job_driver, is_none_mode, base); 890 } 891 892 void commit_active_start(BlockDriverState *bs, BlockDriverState *base, 893 int64_t speed, 894 BlockdevOnError on_error, 895 BlockCompletionFunc *cb, 896 void *opaque, Error **errp) 897 { 898 int64_t length, base_length; 899 int orig_base_flags; 900 int ret; 901 Error *local_err = NULL; 902 903 orig_base_flags = bdrv_get_flags(base); 904 905 if (bdrv_reopen(base, bs->open_flags, errp)) { 906 return; 907 } 908 909 length = bdrv_getlength(bs); 910 if (length < 0) { 911 error_setg_errno(errp, -length, 912 "Unable to determine length of %s", bs->filename); 913 goto error_restore_flags; 914 } 915 916 base_length = bdrv_getlength(base); 917 if (base_length < 0) { 918 error_setg_errno(errp, -base_length, 919 "Unable to determine length of %s", base->filename); 920 goto error_restore_flags; 921 } 922 923 if (length > base_length) { 924 ret = bdrv_truncate(base, length); 925 if (ret < 0) { 926 error_setg_errno(errp, -ret, 927 "Top image %s is larger than base image %s, and " 928 "resize of base image failed", 929 bs->filename, base->filename); 930 goto error_restore_flags; 931 } 932 } 933 934 bdrv_ref(base); 935 mirror_start_job(bs, base, NULL, speed, 0, 0, 936 on_error, on_error, false, cb, opaque, &local_err, 937 &commit_active_job_driver, false, base); 938 if (local_err) { 939 error_propagate(errp, local_err); 940 goto error_restore_flags; 941 } 942 943 return; 944 945 error_restore_flags: 946 /* ignore error and errp for bdrv_reopen, because we want to propagate 947 * the original error */ 948 bdrv_reopen(base, orig_base_flags, NULL); 949 return; 950 } 951