1 /* 2 * Image mirroring 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Paolo Bonzini <pbonzini@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU LGPL, version 2 or later. 10 * See the COPYING.LIB file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "trace.h" 16 #include "block/blockjob.h" 17 #include "block/block_int.h" 18 #include "sysemu/block-backend.h" 19 #include "qapi/error.h" 20 #include "qapi/qmp/qerror.h" 21 #include "qemu/ratelimit.h" 22 #include "qemu/bitmap.h" 23 #include "qemu/error-report.h" 24 25 #define SLICE_TIME 100000000ULL /* ns */ 26 #define MAX_IN_FLIGHT 16 27 #define DEFAULT_MIRROR_BUF_SIZE (10 << 20) 28 29 /* The mirroring buffer is a list of granularity-sized chunks. 30 * Free chunks are organized in a list. 31 */ 32 typedef struct MirrorBuffer { 33 QSIMPLEQ_ENTRY(MirrorBuffer) next; 34 } MirrorBuffer; 35 36 typedef struct MirrorBlockJob { 37 BlockJob common; 38 RateLimit limit; 39 BlockDriverState *target; 40 BlockDriverState *base; 41 /* The name of the graph node to replace */ 42 char *replaces; 43 /* The BDS to replace */ 44 BlockDriverState *to_replace; 45 /* Used to block operations on the drive-mirror-replace target */ 46 Error *replace_blocker; 47 bool is_none_mode; 48 BlockdevOnError on_source_error, on_target_error; 49 bool synced; 50 bool should_complete; 51 int64_t granularity; 52 size_t buf_size; 53 int64_t bdev_length; 54 unsigned long *cow_bitmap; 55 BdrvDirtyBitmap *dirty_bitmap; 56 HBitmapIter hbi; 57 uint8_t *buf; 58 QSIMPLEQ_HEAD(, MirrorBuffer) buf_free; 59 int buf_free_count; 60 61 unsigned long *in_flight_bitmap; 62 int in_flight; 63 int sectors_in_flight; 64 int ret; 65 bool unmap; 66 bool waiting_for_io; 67 int target_cluster_sectors; 68 int max_iov; 69 } MirrorBlockJob; 70 71 typedef struct MirrorOp { 72 MirrorBlockJob *s; 73 QEMUIOVector qiov; 74 int64_t sector_num; 75 int nb_sectors; 76 } MirrorOp; 77 78 static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read, 79 int error) 80 { 81 s->synced = false; 82 if (read) { 83 return block_job_error_action(&s->common, s->common.bs, 84 s->on_source_error, true, error); 85 } else { 86 return block_job_error_action(&s->common, s->target, 87 s->on_target_error, false, error); 88 } 89 } 90 91 static void mirror_iteration_done(MirrorOp *op, int ret) 92 { 93 MirrorBlockJob *s = op->s; 94 struct iovec *iov; 95 int64_t chunk_num; 96 int i, nb_chunks, sectors_per_chunk; 97 98 trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret); 99 100 s->in_flight--; 101 s->sectors_in_flight -= op->nb_sectors; 102 iov = op->qiov.iov; 103 for (i = 0; i < op->qiov.niov; i++) { 104 MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base; 105 QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next); 106 s->buf_free_count++; 107 } 108 109 sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 110 chunk_num = op->sector_num / sectors_per_chunk; 111 nb_chunks = op->nb_sectors / sectors_per_chunk; 112 bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks); 113 if (ret >= 0) { 114 if (s->cow_bitmap) { 115 bitmap_set(s->cow_bitmap, chunk_num, nb_chunks); 116 } 117 s->common.offset += (uint64_t)op->nb_sectors * BDRV_SECTOR_SIZE; 118 } 119 120 qemu_iovec_destroy(&op->qiov); 121 g_free(op); 122 123 if (s->waiting_for_io) { 124 qemu_coroutine_enter(s->common.co, NULL); 125 } 126 } 127 128 static void mirror_write_complete(void *opaque, int ret) 129 { 130 MirrorOp *op = opaque; 131 MirrorBlockJob *s = op->s; 132 if (ret < 0) { 133 BlockErrorAction action; 134 135 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->sector_num, op->nb_sectors); 136 action = mirror_error_action(s, false, -ret); 137 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) { 138 s->ret = ret; 139 } 140 } 141 mirror_iteration_done(op, ret); 142 } 143 144 static void mirror_read_complete(void *opaque, int ret) 145 { 146 MirrorOp *op = opaque; 147 MirrorBlockJob *s = op->s; 148 if (ret < 0) { 149 BlockErrorAction action; 150 151 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->sector_num, op->nb_sectors); 152 action = mirror_error_action(s, true, -ret); 153 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) { 154 s->ret = ret; 155 } 156 157 mirror_iteration_done(op, ret); 158 return; 159 } 160 bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors, 161 mirror_write_complete, op); 162 } 163 164 /* Round sector_num and/or nb_sectors to target cluster if COW is needed, and 165 * return the offset of the adjusted tail sector against original. */ 166 static int mirror_cow_align(MirrorBlockJob *s, 167 int64_t *sector_num, 168 int *nb_sectors) 169 { 170 bool need_cow; 171 int ret = 0; 172 int chunk_sectors = s->granularity >> BDRV_SECTOR_BITS; 173 int64_t align_sector_num = *sector_num; 174 int align_nb_sectors = *nb_sectors; 175 int max_sectors = chunk_sectors * s->max_iov; 176 177 need_cow = !test_bit(*sector_num / chunk_sectors, s->cow_bitmap); 178 need_cow |= !test_bit((*sector_num + *nb_sectors - 1) / chunk_sectors, 179 s->cow_bitmap); 180 if (need_cow) { 181 bdrv_round_to_clusters(s->target, *sector_num, *nb_sectors, 182 &align_sector_num, &align_nb_sectors); 183 } 184 185 if (align_nb_sectors > max_sectors) { 186 align_nb_sectors = max_sectors; 187 if (need_cow) { 188 align_nb_sectors = QEMU_ALIGN_DOWN(align_nb_sectors, 189 s->target_cluster_sectors); 190 } 191 } 192 193 ret = align_sector_num + align_nb_sectors - (*sector_num + *nb_sectors); 194 *sector_num = align_sector_num; 195 *nb_sectors = align_nb_sectors; 196 assert(ret >= 0); 197 return ret; 198 } 199 200 static inline void mirror_wait_for_io(MirrorBlockJob *s) 201 { 202 assert(!s->waiting_for_io); 203 s->waiting_for_io = true; 204 qemu_coroutine_yield(); 205 s->waiting_for_io = false; 206 } 207 208 /* Submit async read while handling COW. 209 * Returns: nb_sectors if no alignment is necessary, or 210 * (new_end - sector_num) if tail is rounded up or down due to 211 * alignment or buffer limit. 212 */ 213 static int mirror_do_read(MirrorBlockJob *s, int64_t sector_num, 214 int nb_sectors) 215 { 216 BlockDriverState *source = s->common.bs; 217 int sectors_per_chunk, nb_chunks; 218 int ret = nb_sectors; 219 MirrorOp *op; 220 221 sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 222 223 /* We can only handle as much as buf_size at a time. */ 224 nb_sectors = MIN(s->buf_size >> BDRV_SECTOR_BITS, nb_sectors); 225 assert(nb_sectors); 226 227 if (s->cow_bitmap) { 228 ret += mirror_cow_align(s, §or_num, &nb_sectors); 229 } 230 assert(nb_sectors << BDRV_SECTOR_BITS <= s->buf_size); 231 /* The sector range must meet granularity because: 232 * 1) Caller passes in aligned values; 233 * 2) mirror_cow_align is used only when target cluster is larger. */ 234 assert(!(nb_sectors % sectors_per_chunk)); 235 assert(!(sector_num % sectors_per_chunk)); 236 nb_chunks = nb_sectors / sectors_per_chunk; 237 238 while (s->buf_free_count < nb_chunks) { 239 trace_mirror_yield_in_flight(s, sector_num, s->in_flight); 240 mirror_wait_for_io(s); 241 } 242 243 /* Allocate a MirrorOp that is used as an AIO callback. */ 244 op = g_new(MirrorOp, 1); 245 op->s = s; 246 op->sector_num = sector_num; 247 op->nb_sectors = nb_sectors; 248 249 /* Now make a QEMUIOVector taking enough granularity-sized chunks 250 * from s->buf_free. 251 */ 252 qemu_iovec_init(&op->qiov, nb_chunks); 253 while (nb_chunks-- > 0) { 254 MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free); 255 size_t remaining = nb_sectors * BDRV_SECTOR_SIZE - op->qiov.size; 256 257 QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next); 258 s->buf_free_count--; 259 qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining)); 260 } 261 262 /* Copy the dirty cluster. */ 263 s->in_flight++; 264 s->sectors_in_flight += nb_sectors; 265 trace_mirror_one_iteration(s, sector_num, nb_sectors); 266 267 bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors, 268 mirror_read_complete, op); 269 return ret; 270 } 271 272 static void mirror_do_zero_or_discard(MirrorBlockJob *s, 273 int64_t sector_num, 274 int nb_sectors, 275 bool is_discard) 276 { 277 MirrorOp *op; 278 279 /* Allocate a MirrorOp that is used as an AIO callback. The qiov is zeroed 280 * so the freeing in mirror_iteration_done is nop. */ 281 op = g_new0(MirrorOp, 1); 282 op->s = s; 283 op->sector_num = sector_num; 284 op->nb_sectors = nb_sectors; 285 286 s->in_flight++; 287 s->sectors_in_flight += nb_sectors; 288 if (is_discard) { 289 bdrv_aio_discard(s->target, sector_num, op->nb_sectors, 290 mirror_write_complete, op); 291 } else { 292 bdrv_aio_write_zeroes(s->target, sector_num, op->nb_sectors, 293 s->unmap ? BDRV_REQ_MAY_UNMAP : 0, 294 mirror_write_complete, op); 295 } 296 } 297 298 static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) 299 { 300 BlockDriverState *source = s->common.bs; 301 int64_t sector_num; 302 uint64_t delay_ns = 0; 303 /* At least the first dirty chunk is mirrored in one iteration. */ 304 int nb_chunks = 1; 305 int64_t end = s->bdev_length / BDRV_SECTOR_SIZE; 306 int sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 307 308 sector_num = hbitmap_iter_next(&s->hbi); 309 if (sector_num < 0) { 310 bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi); 311 sector_num = hbitmap_iter_next(&s->hbi); 312 trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap)); 313 assert(sector_num >= 0); 314 } 315 316 /* Find the number of consective dirty chunks following the first dirty 317 * one, and wait for in flight requests in them. */ 318 while (nb_chunks * sectors_per_chunk < (s->buf_size >> BDRV_SECTOR_BITS)) { 319 int64_t hbitmap_next; 320 int64_t next_sector = sector_num + nb_chunks * sectors_per_chunk; 321 int64_t next_chunk = next_sector / sectors_per_chunk; 322 if (next_sector >= end || 323 !bdrv_get_dirty(source, s->dirty_bitmap, next_sector)) { 324 break; 325 } 326 if (test_bit(next_chunk, s->in_flight_bitmap)) { 327 if (nb_chunks > 0) { 328 break; 329 } 330 trace_mirror_yield_in_flight(s, next_sector, s->in_flight); 331 mirror_wait_for_io(s); 332 /* Now retry. */ 333 } else { 334 hbitmap_next = hbitmap_iter_next(&s->hbi); 335 assert(hbitmap_next == next_sector); 336 nb_chunks++; 337 } 338 } 339 340 /* Clear dirty bits before querying the block status, because 341 * calling bdrv_get_block_status_above could yield - if some blocks are 342 * marked dirty in this window, we need to know. 343 */ 344 bdrv_reset_dirty_bitmap(s->dirty_bitmap, sector_num, 345 nb_chunks * sectors_per_chunk); 346 bitmap_set(s->in_flight_bitmap, sector_num / sectors_per_chunk, nb_chunks); 347 while (nb_chunks > 0 && sector_num < end) { 348 int ret; 349 int io_sectors; 350 BlockDriverState *file; 351 enum MirrorMethod { 352 MIRROR_METHOD_COPY, 353 MIRROR_METHOD_ZERO, 354 MIRROR_METHOD_DISCARD 355 } mirror_method = MIRROR_METHOD_COPY; 356 357 assert(!(sector_num % sectors_per_chunk)); 358 ret = bdrv_get_block_status_above(source, NULL, sector_num, 359 nb_chunks * sectors_per_chunk, 360 &io_sectors, &file); 361 if (ret < 0) { 362 io_sectors = nb_chunks * sectors_per_chunk; 363 } 364 365 io_sectors -= io_sectors % sectors_per_chunk; 366 if (io_sectors < sectors_per_chunk) { 367 io_sectors = sectors_per_chunk; 368 } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) { 369 int64_t target_sector_num; 370 int target_nb_sectors; 371 bdrv_round_to_clusters(s->target, sector_num, io_sectors, 372 &target_sector_num, &target_nb_sectors); 373 if (target_sector_num == sector_num && 374 target_nb_sectors == io_sectors) { 375 mirror_method = ret & BDRV_BLOCK_ZERO ? 376 MIRROR_METHOD_ZERO : 377 MIRROR_METHOD_DISCARD; 378 } 379 } 380 381 switch (mirror_method) { 382 case MIRROR_METHOD_COPY: 383 io_sectors = mirror_do_read(s, sector_num, io_sectors); 384 break; 385 case MIRROR_METHOD_ZERO: 386 mirror_do_zero_or_discard(s, sector_num, io_sectors, false); 387 break; 388 case MIRROR_METHOD_DISCARD: 389 mirror_do_zero_or_discard(s, sector_num, io_sectors, true); 390 break; 391 default: 392 abort(); 393 } 394 assert(io_sectors); 395 sector_num += io_sectors; 396 nb_chunks -= io_sectors / sectors_per_chunk; 397 delay_ns += ratelimit_calculate_delay(&s->limit, io_sectors); 398 } 399 return delay_ns; 400 } 401 402 static void mirror_free_init(MirrorBlockJob *s) 403 { 404 int granularity = s->granularity; 405 size_t buf_size = s->buf_size; 406 uint8_t *buf = s->buf; 407 408 assert(s->buf_free_count == 0); 409 QSIMPLEQ_INIT(&s->buf_free); 410 while (buf_size != 0) { 411 MirrorBuffer *cur = (MirrorBuffer *)buf; 412 QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next); 413 s->buf_free_count++; 414 buf_size -= granularity; 415 buf += granularity; 416 } 417 } 418 419 static void mirror_drain(MirrorBlockJob *s) 420 { 421 while (s->in_flight > 0) { 422 mirror_wait_for_io(s); 423 } 424 } 425 426 typedef struct { 427 int ret; 428 } MirrorExitData; 429 430 static void mirror_exit(BlockJob *job, void *opaque) 431 { 432 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 433 MirrorExitData *data = opaque; 434 AioContext *replace_aio_context = NULL; 435 BlockDriverState *src = s->common.bs; 436 437 /* Make sure that the source BDS doesn't go away before we called 438 * block_job_completed(). */ 439 bdrv_ref(src); 440 441 if (s->to_replace) { 442 replace_aio_context = bdrv_get_aio_context(s->to_replace); 443 aio_context_acquire(replace_aio_context); 444 } 445 446 if (s->should_complete && data->ret == 0) { 447 BlockDriverState *to_replace = s->common.bs; 448 if (s->to_replace) { 449 to_replace = s->to_replace; 450 } 451 452 /* This was checked in mirror_start_job(), but meanwhile one of the 453 * nodes could have been newly attached to a BlockBackend. */ 454 if (to_replace->blk && s->target->blk) { 455 error_report("block job: Can't create node with two BlockBackends"); 456 data->ret = -EINVAL; 457 goto out; 458 } 459 460 if (bdrv_get_flags(s->target) != bdrv_get_flags(to_replace)) { 461 bdrv_reopen(s->target, bdrv_get_flags(to_replace), NULL); 462 } 463 bdrv_replace_in_backing_chain(to_replace, s->target); 464 } 465 466 out: 467 if (s->to_replace) { 468 bdrv_op_unblock_all(s->to_replace, s->replace_blocker); 469 error_free(s->replace_blocker); 470 bdrv_unref(s->to_replace); 471 } 472 if (replace_aio_context) { 473 aio_context_release(replace_aio_context); 474 } 475 g_free(s->replaces); 476 bdrv_op_unblock_all(s->target, s->common.blocker); 477 bdrv_unref(s->target); 478 block_job_completed(&s->common, data->ret); 479 g_free(data); 480 bdrv_drained_end(src); 481 bdrv_unref(src); 482 } 483 484 static void coroutine_fn mirror_run(void *opaque) 485 { 486 MirrorBlockJob *s = opaque; 487 MirrorExitData *data; 488 BlockDriverState *bs = s->common.bs; 489 int64_t sector_num, end, length; 490 uint64_t last_pause_ns; 491 BlockDriverInfo bdi; 492 char backing_filename[2]; /* we only need 2 characters because we are only 493 checking for a NULL string */ 494 int ret = 0; 495 int n; 496 int target_cluster_size = BDRV_SECTOR_SIZE; 497 498 if (block_job_is_cancelled(&s->common)) { 499 goto immediate_exit; 500 } 501 502 s->bdev_length = bdrv_getlength(bs); 503 if (s->bdev_length < 0) { 504 ret = s->bdev_length; 505 goto immediate_exit; 506 } else if (s->bdev_length == 0) { 507 /* Report BLOCK_JOB_READY and wait for complete. */ 508 block_job_event_ready(&s->common); 509 s->synced = true; 510 while (!block_job_is_cancelled(&s->common) && !s->should_complete) { 511 block_job_yield(&s->common); 512 } 513 s->common.cancelled = false; 514 goto immediate_exit; 515 } 516 517 length = DIV_ROUND_UP(s->bdev_length, s->granularity); 518 s->in_flight_bitmap = bitmap_new(length); 519 520 /* If we have no backing file yet in the destination, we cannot let 521 * the destination do COW. Instead, we copy sectors around the 522 * dirty data if needed. We need a bitmap to do that. 523 */ 524 bdrv_get_backing_filename(s->target, backing_filename, 525 sizeof(backing_filename)); 526 if (!bdrv_get_info(s->target, &bdi) && bdi.cluster_size) { 527 target_cluster_size = bdi.cluster_size; 528 } 529 if (backing_filename[0] && !s->target->backing 530 && s->granularity < target_cluster_size) { 531 s->buf_size = MAX(s->buf_size, target_cluster_size); 532 s->cow_bitmap = bitmap_new(length); 533 } 534 s->target_cluster_sectors = target_cluster_size >> BDRV_SECTOR_BITS; 535 s->max_iov = MIN(s->common.bs->bl.max_iov, s->target->bl.max_iov); 536 537 end = s->bdev_length / BDRV_SECTOR_SIZE; 538 s->buf = qemu_try_blockalign(bs, s->buf_size); 539 if (s->buf == NULL) { 540 ret = -ENOMEM; 541 goto immediate_exit; 542 } 543 544 mirror_free_init(s); 545 546 last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 547 if (!s->is_none_mode) { 548 /* First part, loop on the sectors and initialize the dirty bitmap. */ 549 BlockDriverState *base = s->base; 550 bool mark_all_dirty = s->base == NULL && !bdrv_has_zero_init(s->target); 551 552 for (sector_num = 0; sector_num < end; ) { 553 /* Just to make sure we are not exceeding int limit. */ 554 int nb_sectors = MIN(INT_MAX >> BDRV_SECTOR_BITS, 555 end - sector_num); 556 int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 557 558 if (now - last_pause_ns > SLICE_TIME) { 559 last_pause_ns = now; 560 block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, 0); 561 } 562 563 if (block_job_is_cancelled(&s->common)) { 564 goto immediate_exit; 565 } 566 567 ret = bdrv_is_allocated_above(bs, base, sector_num, nb_sectors, &n); 568 569 if (ret < 0) { 570 goto immediate_exit; 571 } 572 573 assert(n > 0); 574 if (ret == 1 || mark_all_dirty) { 575 bdrv_set_dirty_bitmap(s->dirty_bitmap, sector_num, n); 576 } 577 sector_num += n; 578 } 579 } 580 581 bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi); 582 for (;;) { 583 uint64_t delay_ns = 0; 584 int64_t cnt; 585 bool should_complete; 586 587 if (s->ret < 0) { 588 ret = s->ret; 589 goto immediate_exit; 590 } 591 592 cnt = bdrv_get_dirty_count(s->dirty_bitmap); 593 /* s->common.offset contains the number of bytes already processed so 594 * far, cnt is the number of dirty sectors remaining and 595 * s->sectors_in_flight is the number of sectors currently being 596 * processed; together those are the current total operation length */ 597 s->common.len = s->common.offset + 598 (cnt + s->sectors_in_flight) * BDRV_SECTOR_SIZE; 599 600 /* Note that even when no rate limit is applied we need to yield 601 * periodically with no pending I/O so that bdrv_drain_all() returns. 602 * We do so every SLICE_TIME nanoseconds, or when there is an error, 603 * or when the source is clean, whichever comes first. 604 */ 605 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - last_pause_ns < SLICE_TIME && 606 s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) { 607 if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 || 608 (cnt == 0 && s->in_flight > 0)) { 609 trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt); 610 mirror_wait_for_io(s); 611 continue; 612 } else if (cnt != 0) { 613 delay_ns = mirror_iteration(s); 614 } 615 } 616 617 should_complete = false; 618 if (s->in_flight == 0 && cnt == 0) { 619 trace_mirror_before_flush(s); 620 ret = bdrv_flush(s->target); 621 if (ret < 0) { 622 if (mirror_error_action(s, false, -ret) == 623 BLOCK_ERROR_ACTION_REPORT) { 624 goto immediate_exit; 625 } 626 } else { 627 /* We're out of the streaming phase. From now on, if the job 628 * is cancelled we will actually complete all pending I/O and 629 * report completion. This way, block-job-cancel will leave 630 * the target in a consistent state. 631 */ 632 if (!s->synced) { 633 block_job_event_ready(&s->common); 634 s->synced = true; 635 } 636 637 should_complete = s->should_complete || 638 block_job_is_cancelled(&s->common); 639 cnt = bdrv_get_dirty_count(s->dirty_bitmap); 640 } 641 } 642 643 if (cnt == 0 && should_complete) { 644 /* The dirty bitmap is not updated while operations are pending. 645 * If we're about to exit, wait for pending operations before 646 * calling bdrv_get_dirty_count(bs), or we may exit while the 647 * source has dirty data to copy! 648 * 649 * Note that I/O can be submitted by the guest while 650 * mirror_populate runs. 651 */ 652 trace_mirror_before_drain(s, cnt); 653 bdrv_drain(bs); 654 cnt = bdrv_get_dirty_count(s->dirty_bitmap); 655 } 656 657 ret = 0; 658 trace_mirror_before_sleep(s, cnt, s->synced, delay_ns); 659 if (!s->synced) { 660 block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 661 if (block_job_is_cancelled(&s->common)) { 662 break; 663 } 664 } else if (!should_complete) { 665 delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0); 666 block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 667 } else if (cnt == 0) { 668 /* The two disks are in sync. Exit and report successful 669 * completion. 670 */ 671 assert(QLIST_EMPTY(&bs->tracked_requests)); 672 s->common.cancelled = false; 673 break; 674 } 675 last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 676 } 677 678 immediate_exit: 679 if (s->in_flight > 0) { 680 /* We get here only if something went wrong. Either the job failed, 681 * or it was cancelled prematurely so that we do not guarantee that 682 * the target is a copy of the source. 683 */ 684 assert(ret < 0 || (!s->synced && block_job_is_cancelled(&s->common))); 685 mirror_drain(s); 686 } 687 688 assert(s->in_flight == 0); 689 qemu_vfree(s->buf); 690 g_free(s->cow_bitmap); 691 g_free(s->in_flight_bitmap); 692 bdrv_release_dirty_bitmap(bs, s->dirty_bitmap); 693 if (s->target->blk) { 694 blk_iostatus_disable(s->target->blk); 695 } 696 697 data = g_malloc(sizeof(*data)); 698 data->ret = ret; 699 /* Before we switch to target in mirror_exit, make sure data doesn't 700 * change. */ 701 bdrv_drained_begin(s->common.bs); 702 block_job_defer_to_main_loop(&s->common, mirror_exit, data); 703 } 704 705 static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp) 706 { 707 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 708 709 if (speed < 0) { 710 error_setg(errp, QERR_INVALID_PARAMETER, "speed"); 711 return; 712 } 713 ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME); 714 } 715 716 static void mirror_iostatus_reset(BlockJob *job) 717 { 718 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 719 720 if (s->target->blk) { 721 blk_iostatus_reset(s->target->blk); 722 } 723 } 724 725 static void mirror_complete(BlockJob *job, Error **errp) 726 { 727 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 728 Error *local_err = NULL; 729 int ret; 730 731 ret = bdrv_open_backing_file(s->target, NULL, "backing", &local_err); 732 if (ret < 0) { 733 error_propagate(errp, local_err); 734 return; 735 } 736 if (!s->synced) { 737 error_setg(errp, QERR_BLOCK_JOB_NOT_READY, job->id); 738 return; 739 } 740 741 /* check the target bs is not blocked and block all operations on it */ 742 if (s->replaces) { 743 AioContext *replace_aio_context; 744 745 s->to_replace = bdrv_find_node(s->replaces); 746 if (!s->to_replace) { 747 error_setg(errp, "Node name '%s' not found", s->replaces); 748 return; 749 } 750 751 replace_aio_context = bdrv_get_aio_context(s->to_replace); 752 aio_context_acquire(replace_aio_context); 753 754 error_setg(&s->replace_blocker, 755 "block device is in use by block-job-complete"); 756 bdrv_op_block_all(s->to_replace, s->replace_blocker); 757 bdrv_ref(s->to_replace); 758 759 aio_context_release(replace_aio_context); 760 } 761 762 s->should_complete = true; 763 block_job_enter(&s->common); 764 } 765 766 static const BlockJobDriver mirror_job_driver = { 767 .instance_size = sizeof(MirrorBlockJob), 768 .job_type = BLOCK_JOB_TYPE_MIRROR, 769 .set_speed = mirror_set_speed, 770 .iostatus_reset= mirror_iostatus_reset, 771 .complete = mirror_complete, 772 }; 773 774 static const BlockJobDriver commit_active_job_driver = { 775 .instance_size = sizeof(MirrorBlockJob), 776 .job_type = BLOCK_JOB_TYPE_COMMIT, 777 .set_speed = mirror_set_speed, 778 .iostatus_reset 779 = mirror_iostatus_reset, 780 .complete = mirror_complete, 781 }; 782 783 static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target, 784 const char *replaces, 785 int64_t speed, uint32_t granularity, 786 int64_t buf_size, 787 BlockdevOnError on_source_error, 788 BlockdevOnError on_target_error, 789 bool unmap, 790 BlockCompletionFunc *cb, 791 void *opaque, Error **errp, 792 const BlockJobDriver *driver, 793 bool is_none_mode, BlockDriverState *base) 794 { 795 MirrorBlockJob *s; 796 BlockDriverState *replaced_bs; 797 798 if (granularity == 0) { 799 granularity = bdrv_get_default_bitmap_granularity(target); 800 } 801 802 assert ((granularity & (granularity - 1)) == 0); 803 804 if ((on_source_error == BLOCKDEV_ON_ERROR_STOP || 805 on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) && 806 (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) { 807 error_setg(errp, QERR_INVALID_PARAMETER, "on-source-error"); 808 return; 809 } 810 811 if (buf_size < 0) { 812 error_setg(errp, "Invalid parameter 'buf-size'"); 813 return; 814 } 815 816 if (buf_size == 0) { 817 buf_size = DEFAULT_MIRROR_BUF_SIZE; 818 } 819 820 /* We can't support this case as long as the block layer can't handle 821 * multiple BlockBackends per BlockDriverState. */ 822 if (replaces) { 823 replaced_bs = bdrv_lookup_bs(replaces, replaces, errp); 824 if (replaced_bs == NULL) { 825 return; 826 } 827 } else { 828 replaced_bs = bs; 829 } 830 if (replaced_bs->blk && target->blk) { 831 error_setg(errp, "Can't create node with two BlockBackends"); 832 return; 833 } 834 835 s = block_job_create(driver, bs, speed, cb, opaque, errp); 836 if (!s) { 837 return; 838 } 839 840 s->replaces = g_strdup(replaces); 841 s->on_source_error = on_source_error; 842 s->on_target_error = on_target_error; 843 s->target = target; 844 s->is_none_mode = is_none_mode; 845 s->base = base; 846 s->granularity = granularity; 847 s->buf_size = ROUND_UP(buf_size, granularity); 848 s->unmap = unmap; 849 850 s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp); 851 if (!s->dirty_bitmap) { 852 g_free(s->replaces); 853 block_job_unref(&s->common); 854 return; 855 } 856 857 bdrv_op_block_all(s->target, s->common.blocker); 858 859 if (s->target->blk) { 860 blk_set_on_error(s->target->blk, on_target_error, on_target_error); 861 blk_iostatus_enable(s->target->blk); 862 } 863 s->common.co = qemu_coroutine_create(mirror_run); 864 trace_mirror_start(bs, s, s->common.co, opaque); 865 qemu_coroutine_enter(s->common.co, s); 866 } 867 868 void mirror_start(BlockDriverState *bs, BlockDriverState *target, 869 const char *replaces, 870 int64_t speed, uint32_t granularity, int64_t buf_size, 871 MirrorSyncMode mode, BlockdevOnError on_source_error, 872 BlockdevOnError on_target_error, 873 bool unmap, 874 BlockCompletionFunc *cb, 875 void *opaque, Error **errp) 876 { 877 bool is_none_mode; 878 BlockDriverState *base; 879 880 if (mode == MIRROR_SYNC_MODE_INCREMENTAL) { 881 error_setg(errp, "Sync mode 'incremental' not supported"); 882 return; 883 } 884 is_none_mode = mode == MIRROR_SYNC_MODE_NONE; 885 base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL; 886 mirror_start_job(bs, target, replaces, 887 speed, granularity, buf_size, 888 on_source_error, on_target_error, unmap, cb, opaque, errp, 889 &mirror_job_driver, is_none_mode, base); 890 } 891 892 void commit_active_start(BlockDriverState *bs, BlockDriverState *base, 893 int64_t speed, 894 BlockdevOnError on_error, 895 BlockCompletionFunc *cb, 896 void *opaque, Error **errp) 897 { 898 int64_t length, base_length; 899 int orig_base_flags; 900 int ret; 901 Error *local_err = NULL; 902 903 orig_base_flags = bdrv_get_flags(base); 904 905 if (bdrv_reopen(base, bs->open_flags, errp)) { 906 return; 907 } 908 909 length = bdrv_getlength(bs); 910 if (length < 0) { 911 error_setg_errno(errp, -length, 912 "Unable to determine length of %s", bs->filename); 913 goto error_restore_flags; 914 } 915 916 base_length = bdrv_getlength(base); 917 if (base_length < 0) { 918 error_setg_errno(errp, -base_length, 919 "Unable to determine length of %s", base->filename); 920 goto error_restore_flags; 921 } 922 923 if (length > base_length) { 924 ret = bdrv_truncate(base, length); 925 if (ret < 0) { 926 error_setg_errno(errp, -ret, 927 "Top image %s is larger than base image %s, and " 928 "resize of base image failed", 929 bs->filename, base->filename); 930 goto error_restore_flags; 931 } 932 } 933 934 bdrv_ref(base); 935 mirror_start_job(bs, base, NULL, speed, 0, 0, 936 on_error, on_error, false, cb, opaque, &local_err, 937 &commit_active_job_driver, false, base); 938 if (local_err) { 939 error_propagate(errp, local_err); 940 goto error_restore_flags; 941 } 942 943 return; 944 945 error_restore_flags: 946 /* ignore error and errp for bdrv_reopen, because we want to propagate 947 * the original error */ 948 bdrv_reopen(base, orig_base_flags, NULL); 949 return; 950 } 951