1893f7ebaSPaolo Bonzini /* 2893f7ebaSPaolo Bonzini * Image mirroring 3893f7ebaSPaolo Bonzini * 4893f7ebaSPaolo Bonzini * Copyright Red Hat, Inc. 2012 5893f7ebaSPaolo Bonzini * 6893f7ebaSPaolo Bonzini * Authors: 7893f7ebaSPaolo Bonzini * Paolo Bonzini <pbonzini@redhat.com> 8893f7ebaSPaolo Bonzini * 9893f7ebaSPaolo Bonzini * This work is licensed under the terms of the GNU LGPL, version 2 or later. 10893f7ebaSPaolo Bonzini * See the COPYING.LIB file in the top-level directory. 11893f7ebaSPaolo Bonzini * 12893f7ebaSPaolo Bonzini */ 13893f7ebaSPaolo Bonzini 14893f7ebaSPaolo Bonzini #include "trace.h" 15737e150eSPaolo Bonzini #include "block/blockjob.h" 16737e150eSPaolo Bonzini #include "block/block_int.h" 17893f7ebaSPaolo Bonzini #include "qemu/ratelimit.h" 18b812f671SPaolo Bonzini #include "qemu/bitmap.h" 19893f7ebaSPaolo Bonzini 20893f7ebaSPaolo Bonzini #define SLICE_TIME 100000000ULL /* ns */ 21402a4741SPaolo Bonzini #define MAX_IN_FLIGHT 16 22402a4741SPaolo Bonzini 23402a4741SPaolo Bonzini /* The mirroring buffer is a list of granularity-sized chunks. 24402a4741SPaolo Bonzini * Free chunks are organized in a list. 25402a4741SPaolo Bonzini */ 26402a4741SPaolo Bonzini typedef struct MirrorBuffer { 27402a4741SPaolo Bonzini QSIMPLEQ_ENTRY(MirrorBuffer) next; 28402a4741SPaolo Bonzini } MirrorBuffer; 29893f7ebaSPaolo Bonzini 30893f7ebaSPaolo Bonzini typedef struct MirrorBlockJob { 31893f7ebaSPaolo Bonzini BlockJob common; 32893f7ebaSPaolo Bonzini RateLimit limit; 33893f7ebaSPaolo Bonzini BlockDriverState *target; 34893f7ebaSPaolo Bonzini MirrorSyncMode mode; 35b952b558SPaolo Bonzini BlockdevOnError on_source_error, on_target_error; 36d63ffd87SPaolo Bonzini bool synced; 37d63ffd87SPaolo Bonzini bool should_complete; 38893f7ebaSPaolo Bonzini int64_t sector_num; 39eee13dfeSPaolo Bonzini int64_t granularity; 40b812f671SPaolo Bonzini size_t buf_size; 41b812f671SPaolo Bonzini unsigned long *cow_bitmap; 428f0720ecSPaolo Bonzini HBitmapIter hbi; 43893f7ebaSPaolo Bonzini uint8_t *buf; 44402a4741SPaolo Bonzini QSIMPLEQ_HEAD(, MirrorBuffer) buf_free; 45402a4741SPaolo Bonzini int buf_free_count; 46bd48bde8SPaolo Bonzini 47402a4741SPaolo Bonzini unsigned long *in_flight_bitmap; 48bd48bde8SPaolo Bonzini int in_flight; 49bd48bde8SPaolo Bonzini int ret; 50893f7ebaSPaolo Bonzini } MirrorBlockJob; 51893f7ebaSPaolo Bonzini 52bd48bde8SPaolo Bonzini typedef struct MirrorOp { 53bd48bde8SPaolo Bonzini MirrorBlockJob *s; 54bd48bde8SPaolo Bonzini QEMUIOVector qiov; 55bd48bde8SPaolo Bonzini int64_t sector_num; 56bd48bde8SPaolo Bonzini int nb_sectors; 57bd48bde8SPaolo Bonzini } MirrorOp; 58bd48bde8SPaolo Bonzini 59b952b558SPaolo Bonzini static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read, 60b952b558SPaolo Bonzini int error) 61b952b558SPaolo Bonzini { 62b952b558SPaolo Bonzini s->synced = false; 63b952b558SPaolo Bonzini if (read) { 64b952b558SPaolo Bonzini return block_job_error_action(&s->common, s->common.bs, 65b952b558SPaolo Bonzini s->on_source_error, true, error); 66b952b558SPaolo Bonzini } else { 67b952b558SPaolo Bonzini return block_job_error_action(&s->common, s->target, 68b952b558SPaolo Bonzini s->on_target_error, false, error); 69b952b558SPaolo Bonzini } 70b952b558SPaolo Bonzini } 71b952b558SPaolo Bonzini 72bd48bde8SPaolo Bonzini static void mirror_iteration_done(MirrorOp *op, int ret) 73bd48bde8SPaolo Bonzini { 74bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 75402a4741SPaolo Bonzini struct iovec *iov; 76bd48bde8SPaolo Bonzini int64_t chunk_num; 77402a4741SPaolo Bonzini int i, nb_chunks, sectors_per_chunk; 78bd48bde8SPaolo Bonzini 79bd48bde8SPaolo Bonzini trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret); 80bd48bde8SPaolo Bonzini 81bd48bde8SPaolo Bonzini s->in_flight--; 82402a4741SPaolo Bonzini iov = op->qiov.iov; 83402a4741SPaolo Bonzini for (i = 0; i < op->qiov.niov; i++) { 84402a4741SPaolo Bonzini MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base; 85402a4741SPaolo Bonzini QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next); 86402a4741SPaolo Bonzini s->buf_free_count++; 87402a4741SPaolo Bonzini } 88402a4741SPaolo Bonzini 89bd48bde8SPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 90bd48bde8SPaolo Bonzini chunk_num = op->sector_num / sectors_per_chunk; 91bd48bde8SPaolo Bonzini nb_chunks = op->nb_sectors / sectors_per_chunk; 92402a4741SPaolo Bonzini bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks); 93bd48bde8SPaolo Bonzini if (s->cow_bitmap && ret >= 0) { 94bd48bde8SPaolo Bonzini bitmap_set(s->cow_bitmap, chunk_num, nb_chunks); 95bd48bde8SPaolo Bonzini } 96bd48bde8SPaolo Bonzini 97bd48bde8SPaolo Bonzini g_slice_free(MirrorOp, op); 98bd48bde8SPaolo Bonzini qemu_coroutine_enter(s->common.co, NULL); 99bd48bde8SPaolo Bonzini } 100bd48bde8SPaolo Bonzini 101bd48bde8SPaolo Bonzini static void mirror_write_complete(void *opaque, int ret) 102bd48bde8SPaolo Bonzini { 103bd48bde8SPaolo Bonzini MirrorOp *op = opaque; 104bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 105bd48bde8SPaolo Bonzini if (ret < 0) { 106bd48bde8SPaolo Bonzini BlockDriverState *source = s->common.bs; 107bd48bde8SPaolo Bonzini BlockErrorAction action; 108bd48bde8SPaolo Bonzini 109bd48bde8SPaolo Bonzini bdrv_set_dirty(source, op->sector_num, op->nb_sectors); 110bd48bde8SPaolo Bonzini action = mirror_error_action(s, false, -ret); 111bd48bde8SPaolo Bonzini if (action == BDRV_ACTION_REPORT && s->ret >= 0) { 112bd48bde8SPaolo Bonzini s->ret = ret; 113bd48bde8SPaolo Bonzini } 114bd48bde8SPaolo Bonzini } 115bd48bde8SPaolo Bonzini mirror_iteration_done(op, ret); 116bd48bde8SPaolo Bonzini } 117bd48bde8SPaolo Bonzini 118bd48bde8SPaolo Bonzini static void mirror_read_complete(void *opaque, int ret) 119bd48bde8SPaolo Bonzini { 120bd48bde8SPaolo Bonzini MirrorOp *op = opaque; 121bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 122bd48bde8SPaolo Bonzini if (ret < 0) { 123bd48bde8SPaolo Bonzini BlockDriverState *source = s->common.bs; 124bd48bde8SPaolo Bonzini BlockErrorAction action; 125bd48bde8SPaolo Bonzini 126bd48bde8SPaolo Bonzini bdrv_set_dirty(source, op->sector_num, op->nb_sectors); 127bd48bde8SPaolo Bonzini action = mirror_error_action(s, true, -ret); 128bd48bde8SPaolo Bonzini if (action == BDRV_ACTION_REPORT && s->ret >= 0) { 129bd48bde8SPaolo Bonzini s->ret = ret; 130bd48bde8SPaolo Bonzini } 131bd48bde8SPaolo Bonzini 132bd48bde8SPaolo Bonzini mirror_iteration_done(op, ret); 133bd48bde8SPaolo Bonzini return; 134bd48bde8SPaolo Bonzini } 135bd48bde8SPaolo Bonzini bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors, 136bd48bde8SPaolo Bonzini mirror_write_complete, op); 137bd48bde8SPaolo Bonzini } 138bd48bde8SPaolo Bonzini 139bd48bde8SPaolo Bonzini static void coroutine_fn mirror_iteration(MirrorBlockJob *s) 140893f7ebaSPaolo Bonzini { 141893f7ebaSPaolo Bonzini BlockDriverState *source = s->common.bs; 142402a4741SPaolo Bonzini int nb_sectors, sectors_per_chunk, nb_chunks; 143884fea4eSPaolo Bonzini int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector; 144bd48bde8SPaolo Bonzini MirrorOp *op; 145893f7ebaSPaolo Bonzini 1468f0720ecSPaolo Bonzini s->sector_num = hbitmap_iter_next(&s->hbi); 1478f0720ecSPaolo Bonzini if (s->sector_num < 0) { 1488f0720ecSPaolo Bonzini bdrv_dirty_iter_init(source, &s->hbi); 1498f0720ecSPaolo Bonzini s->sector_num = hbitmap_iter_next(&s->hbi); 1508f0720ecSPaolo Bonzini trace_mirror_restart_iter(s, bdrv_get_dirty_count(source)); 1518f0720ecSPaolo Bonzini assert(s->sector_num >= 0); 1528f0720ecSPaolo Bonzini } 1538f0720ecSPaolo Bonzini 154402a4741SPaolo Bonzini hbitmap_next_sector = s->sector_num; 155b812f671SPaolo Bonzini sector_num = s->sector_num; 156884fea4eSPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 157884fea4eSPaolo Bonzini end = s->common.len >> BDRV_SECTOR_BITS; 158402a4741SPaolo Bonzini 159884fea4eSPaolo Bonzini /* Extend the QEMUIOVector to include all adjacent blocks that will 160884fea4eSPaolo Bonzini * be copied in this operation. 161884fea4eSPaolo Bonzini * 162884fea4eSPaolo Bonzini * We have to do this if we have no backing file yet in the destination, 163884fea4eSPaolo Bonzini * and the cluster size is very large. Then we need to do COW ourselves. 164884fea4eSPaolo Bonzini * The first time a cluster is copied, copy it entirely. Note that, 165884fea4eSPaolo Bonzini * because both the granularity and the cluster size are powers of two, 166884fea4eSPaolo Bonzini * the number of sectors to copy cannot exceed one cluster. 167884fea4eSPaolo Bonzini * 168884fea4eSPaolo Bonzini * We also want to extend the QEMUIOVector to include more adjacent 169884fea4eSPaolo Bonzini * dirty blocks if possible, to limit the number of I/O operations and 170884fea4eSPaolo Bonzini * run efficiently even with a small granularity. 171402a4741SPaolo Bonzini */ 172884fea4eSPaolo Bonzini nb_chunks = 0; 173884fea4eSPaolo Bonzini nb_sectors = 0; 174884fea4eSPaolo Bonzini next_sector = sector_num; 175884fea4eSPaolo Bonzini next_chunk = sector_num / sectors_per_chunk; 176402a4741SPaolo Bonzini 177402a4741SPaolo Bonzini /* Wait for I/O to this cluster (from a previous iteration) to be done. */ 178884fea4eSPaolo Bonzini while (test_bit(next_chunk, s->in_flight_bitmap)) { 179402a4741SPaolo Bonzini trace_mirror_yield_in_flight(s, sector_num, s->in_flight); 180402a4741SPaolo Bonzini qemu_coroutine_yield(); 181b812f671SPaolo Bonzini } 182b812f671SPaolo Bonzini 183884fea4eSPaolo Bonzini do { 184884fea4eSPaolo Bonzini int added_sectors, added_chunks; 185884fea4eSPaolo Bonzini 186884fea4eSPaolo Bonzini if (!bdrv_get_dirty(source, next_sector) || 187884fea4eSPaolo Bonzini test_bit(next_chunk, s->in_flight_bitmap)) { 188884fea4eSPaolo Bonzini assert(nb_sectors > 0); 189884fea4eSPaolo Bonzini break; 190884fea4eSPaolo Bonzini } 191884fea4eSPaolo Bonzini 192884fea4eSPaolo Bonzini added_sectors = sectors_per_chunk; 193884fea4eSPaolo Bonzini if (s->cow_bitmap && !test_bit(next_chunk, s->cow_bitmap)) { 194884fea4eSPaolo Bonzini bdrv_round_to_clusters(s->target, 195884fea4eSPaolo Bonzini next_sector, added_sectors, 196884fea4eSPaolo Bonzini &next_sector, &added_sectors); 197884fea4eSPaolo Bonzini 198884fea4eSPaolo Bonzini /* On the first iteration, the rounding may make us copy 199884fea4eSPaolo Bonzini * sectors before the first dirty one. 200884fea4eSPaolo Bonzini */ 201884fea4eSPaolo Bonzini if (next_sector < sector_num) { 202884fea4eSPaolo Bonzini assert(nb_sectors == 0); 203884fea4eSPaolo Bonzini sector_num = next_sector; 204884fea4eSPaolo Bonzini next_chunk = next_sector / sectors_per_chunk; 205884fea4eSPaolo Bonzini } 206884fea4eSPaolo Bonzini } 207884fea4eSPaolo Bonzini 208884fea4eSPaolo Bonzini added_sectors = MIN(added_sectors, end - (sector_num + nb_sectors)); 209884fea4eSPaolo Bonzini added_chunks = (added_sectors + sectors_per_chunk - 1) / sectors_per_chunk; 210884fea4eSPaolo Bonzini 211884fea4eSPaolo Bonzini /* When doing COW, it may happen that there is not enough space for 212884fea4eSPaolo Bonzini * a full cluster. Wait if that is the case. 213884fea4eSPaolo Bonzini */ 214884fea4eSPaolo Bonzini while (nb_chunks == 0 && s->buf_free_count < added_chunks) { 215402a4741SPaolo Bonzini trace_mirror_yield_buf_busy(s, nb_chunks, s->in_flight); 216402a4741SPaolo Bonzini qemu_coroutine_yield(); 217402a4741SPaolo Bonzini } 218884fea4eSPaolo Bonzini if (s->buf_free_count < nb_chunks + added_chunks) { 219884fea4eSPaolo Bonzini trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight); 220884fea4eSPaolo Bonzini break; 221884fea4eSPaolo Bonzini } 222402a4741SPaolo Bonzini 223402a4741SPaolo Bonzini /* We have enough free space to copy these sectors. */ 224884fea4eSPaolo Bonzini bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks); 225884fea4eSPaolo Bonzini 226884fea4eSPaolo Bonzini nb_sectors += added_sectors; 227884fea4eSPaolo Bonzini nb_chunks += added_chunks; 228884fea4eSPaolo Bonzini next_sector += added_sectors; 229884fea4eSPaolo Bonzini next_chunk += added_chunks; 230884fea4eSPaolo Bonzini } while (next_sector < end); 231bd48bde8SPaolo Bonzini 232bd48bde8SPaolo Bonzini /* Allocate a MirrorOp that is used as an AIO callback. */ 233bd48bde8SPaolo Bonzini op = g_slice_new(MirrorOp); 234bd48bde8SPaolo Bonzini op->s = s; 235bd48bde8SPaolo Bonzini op->sector_num = sector_num; 236bd48bde8SPaolo Bonzini op->nb_sectors = nb_sectors; 237402a4741SPaolo Bonzini 238402a4741SPaolo Bonzini /* Now make a QEMUIOVector taking enough granularity-sized chunks 239402a4741SPaolo Bonzini * from s->buf_free. 240402a4741SPaolo Bonzini */ 241402a4741SPaolo Bonzini qemu_iovec_init(&op->qiov, nb_chunks); 242402a4741SPaolo Bonzini next_sector = sector_num; 243402a4741SPaolo Bonzini while (nb_chunks-- > 0) { 244402a4741SPaolo Bonzini MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free); 245402a4741SPaolo Bonzini QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next); 246402a4741SPaolo Bonzini s->buf_free_count--; 247402a4741SPaolo Bonzini qemu_iovec_add(&op->qiov, buf, s->granularity); 248402a4741SPaolo Bonzini 249402a4741SPaolo Bonzini /* Advance the HBitmapIter in parallel, so that we do not examine 250402a4741SPaolo Bonzini * the same sector twice. 251402a4741SPaolo Bonzini */ 252402a4741SPaolo Bonzini if (next_sector > hbitmap_next_sector && bdrv_get_dirty(source, next_sector)) { 253402a4741SPaolo Bonzini hbitmap_next_sector = hbitmap_iter_next(&s->hbi); 254402a4741SPaolo Bonzini } 255402a4741SPaolo Bonzini 256402a4741SPaolo Bonzini next_sector += sectors_per_chunk; 257402a4741SPaolo Bonzini } 258bd48bde8SPaolo Bonzini 259b812f671SPaolo Bonzini bdrv_reset_dirty(source, sector_num, nb_sectors); 260893f7ebaSPaolo Bonzini 261893f7ebaSPaolo Bonzini /* Copy the dirty cluster. */ 262bd48bde8SPaolo Bonzini s->in_flight++; 263b812f671SPaolo Bonzini trace_mirror_one_iteration(s, sector_num, nb_sectors); 264bd48bde8SPaolo Bonzini bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors, 265bd48bde8SPaolo Bonzini mirror_read_complete, op); 266893f7ebaSPaolo Bonzini } 267b952b558SPaolo Bonzini 268402a4741SPaolo Bonzini static void mirror_free_init(MirrorBlockJob *s) 269402a4741SPaolo Bonzini { 270402a4741SPaolo Bonzini int granularity = s->granularity; 271402a4741SPaolo Bonzini size_t buf_size = s->buf_size; 272402a4741SPaolo Bonzini uint8_t *buf = s->buf; 273402a4741SPaolo Bonzini 274402a4741SPaolo Bonzini assert(s->buf_free_count == 0); 275402a4741SPaolo Bonzini QSIMPLEQ_INIT(&s->buf_free); 276402a4741SPaolo Bonzini while (buf_size != 0) { 277402a4741SPaolo Bonzini MirrorBuffer *cur = (MirrorBuffer *)buf; 278402a4741SPaolo Bonzini QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next); 279402a4741SPaolo Bonzini s->buf_free_count++; 280402a4741SPaolo Bonzini buf_size -= granularity; 281402a4741SPaolo Bonzini buf += granularity; 282402a4741SPaolo Bonzini } 283402a4741SPaolo Bonzini } 284402a4741SPaolo Bonzini 285bd48bde8SPaolo Bonzini static void mirror_drain(MirrorBlockJob *s) 286bd48bde8SPaolo Bonzini { 287bd48bde8SPaolo Bonzini while (s->in_flight > 0) { 288bd48bde8SPaolo Bonzini qemu_coroutine_yield(); 289bd48bde8SPaolo Bonzini } 290893f7ebaSPaolo Bonzini } 291893f7ebaSPaolo Bonzini 292893f7ebaSPaolo Bonzini static void coroutine_fn mirror_run(void *opaque) 293893f7ebaSPaolo Bonzini { 294893f7ebaSPaolo Bonzini MirrorBlockJob *s = opaque; 295893f7ebaSPaolo Bonzini BlockDriverState *bs = s->common.bs; 296eee13dfeSPaolo Bonzini int64_t sector_num, end, sectors_per_chunk, length; 297bd48bde8SPaolo Bonzini uint64_t last_pause_ns; 298b812f671SPaolo Bonzini BlockDriverInfo bdi; 299b812f671SPaolo Bonzini char backing_filename[1024]; 300893f7ebaSPaolo Bonzini int ret = 0; 301893f7ebaSPaolo Bonzini int n; 302893f7ebaSPaolo Bonzini 303893f7ebaSPaolo Bonzini if (block_job_is_cancelled(&s->common)) { 304893f7ebaSPaolo Bonzini goto immediate_exit; 305893f7ebaSPaolo Bonzini } 306893f7ebaSPaolo Bonzini 307893f7ebaSPaolo Bonzini s->common.len = bdrv_getlength(bs); 30888ff0e48SPaolo Bonzini if (s->common.len <= 0) { 309893f7ebaSPaolo Bonzini block_job_completed(&s->common, s->common.len); 310893f7ebaSPaolo Bonzini return; 311893f7ebaSPaolo Bonzini } 312893f7ebaSPaolo Bonzini 313402a4741SPaolo Bonzini length = (bdrv_getlength(bs) + s->granularity - 1) / s->granularity; 314402a4741SPaolo Bonzini s->in_flight_bitmap = bitmap_new(length); 315402a4741SPaolo Bonzini 316b812f671SPaolo Bonzini /* If we have no backing file yet in the destination, we cannot let 317b812f671SPaolo Bonzini * the destination do COW. Instead, we copy sectors around the 318b812f671SPaolo Bonzini * dirty data if needed. We need a bitmap to do that. 319b812f671SPaolo Bonzini */ 320b812f671SPaolo Bonzini bdrv_get_backing_filename(s->target, backing_filename, 321b812f671SPaolo Bonzini sizeof(backing_filename)); 322b812f671SPaolo Bonzini if (backing_filename[0] && !s->target->backing_hd) { 323b812f671SPaolo Bonzini bdrv_get_info(s->target, &bdi); 324eee13dfeSPaolo Bonzini if (s->granularity < bdi.cluster_size) { 32508e4ed6cSPaolo Bonzini s->buf_size = MAX(s->buf_size, bdi.cluster_size); 326b812f671SPaolo Bonzini s->cow_bitmap = bitmap_new(length); 327b812f671SPaolo Bonzini } 328b812f671SPaolo Bonzini } 329b812f671SPaolo Bonzini 330893f7ebaSPaolo Bonzini end = s->common.len >> BDRV_SECTOR_BITS; 331b812f671SPaolo Bonzini s->buf = qemu_blockalign(bs, s->buf_size); 332eee13dfeSPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 333402a4741SPaolo Bonzini mirror_free_init(s); 334893f7ebaSPaolo Bonzini 335893f7ebaSPaolo Bonzini if (s->mode != MIRROR_SYNC_MODE_NONE) { 336893f7ebaSPaolo Bonzini /* First part, loop on the sectors and initialize the dirty bitmap. */ 337893f7ebaSPaolo Bonzini BlockDriverState *base; 338893f7ebaSPaolo Bonzini base = s->mode == MIRROR_SYNC_MODE_FULL ? NULL : bs->backing_hd; 339893f7ebaSPaolo Bonzini for (sector_num = 0; sector_num < end; ) { 340eee13dfeSPaolo Bonzini int64_t next = (sector_num | (sectors_per_chunk - 1)) + 1; 3414f578637SPaolo Bonzini ret = bdrv_is_allocated_above(bs, base, 342893f7ebaSPaolo Bonzini sector_num, next - sector_num, &n); 343893f7ebaSPaolo Bonzini 344893f7ebaSPaolo Bonzini if (ret < 0) { 345893f7ebaSPaolo Bonzini goto immediate_exit; 346893f7ebaSPaolo Bonzini } 347893f7ebaSPaolo Bonzini 348893f7ebaSPaolo Bonzini assert(n > 0); 349893f7ebaSPaolo Bonzini if (ret == 1) { 350893f7ebaSPaolo Bonzini bdrv_set_dirty(bs, sector_num, n); 351893f7ebaSPaolo Bonzini sector_num = next; 352893f7ebaSPaolo Bonzini } else { 353893f7ebaSPaolo Bonzini sector_num += n; 354893f7ebaSPaolo Bonzini } 355893f7ebaSPaolo Bonzini } 356893f7ebaSPaolo Bonzini } 357893f7ebaSPaolo Bonzini 3588f0720ecSPaolo Bonzini bdrv_dirty_iter_init(bs, &s->hbi); 359bc72ad67SAlex Bligh last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 360893f7ebaSPaolo Bonzini for (;;) { 361893f7ebaSPaolo Bonzini uint64_t delay_ns; 362893f7ebaSPaolo Bonzini int64_t cnt; 363893f7ebaSPaolo Bonzini bool should_complete; 364893f7ebaSPaolo Bonzini 365bd48bde8SPaolo Bonzini if (s->ret < 0) { 366bd48bde8SPaolo Bonzini ret = s->ret; 367893f7ebaSPaolo Bonzini goto immediate_exit; 368893f7ebaSPaolo Bonzini } 369bd48bde8SPaolo Bonzini 370893f7ebaSPaolo Bonzini cnt = bdrv_get_dirty_count(bs); 371bd48bde8SPaolo Bonzini 372bd48bde8SPaolo Bonzini /* Note that even when no rate limit is applied we need to yield 373bd48bde8SPaolo Bonzini * periodically with no pending I/O so that qemu_aio_flush() returns. 374bd48bde8SPaolo Bonzini * We do so every SLICE_TIME nanoseconds, or when there is an error, 375bd48bde8SPaolo Bonzini * or when the source is clean, whichever comes first. 376bd48bde8SPaolo Bonzini */ 377bc72ad67SAlex Bligh if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - last_pause_ns < SLICE_TIME && 378bd48bde8SPaolo Bonzini s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) { 379402a4741SPaolo Bonzini if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 || 380402a4741SPaolo Bonzini (cnt == 0 && s->in_flight > 0)) { 381402a4741SPaolo Bonzini trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt); 382bd48bde8SPaolo Bonzini qemu_coroutine_yield(); 383bd48bde8SPaolo Bonzini continue; 384bd48bde8SPaolo Bonzini } else if (cnt != 0) { 385bd48bde8SPaolo Bonzini mirror_iteration(s); 386bd48bde8SPaolo Bonzini continue; 387bd48bde8SPaolo Bonzini } 388893f7ebaSPaolo Bonzini } 389893f7ebaSPaolo Bonzini 390893f7ebaSPaolo Bonzini should_complete = false; 391bd48bde8SPaolo Bonzini if (s->in_flight == 0 && cnt == 0) { 392893f7ebaSPaolo Bonzini trace_mirror_before_flush(s); 393893f7ebaSPaolo Bonzini ret = bdrv_flush(s->target); 394893f7ebaSPaolo Bonzini if (ret < 0) { 395b952b558SPaolo Bonzini if (mirror_error_action(s, false, -ret) == BDRV_ACTION_REPORT) { 396893f7ebaSPaolo Bonzini goto immediate_exit; 397893f7ebaSPaolo Bonzini } 398b952b558SPaolo Bonzini } else { 399893f7ebaSPaolo Bonzini /* We're out of the streaming phase. From now on, if the job 400893f7ebaSPaolo Bonzini * is cancelled we will actually complete all pending I/O and 401893f7ebaSPaolo Bonzini * report completion. This way, block-job-cancel will leave 402893f7ebaSPaolo Bonzini * the target in a consistent state. 403893f7ebaSPaolo Bonzini */ 404893f7ebaSPaolo Bonzini s->common.offset = end * BDRV_SECTOR_SIZE; 405d63ffd87SPaolo Bonzini if (!s->synced) { 406d63ffd87SPaolo Bonzini block_job_ready(&s->common); 407d63ffd87SPaolo Bonzini s->synced = true; 408d63ffd87SPaolo Bonzini } 409d63ffd87SPaolo Bonzini 410d63ffd87SPaolo Bonzini should_complete = s->should_complete || 411d63ffd87SPaolo Bonzini block_job_is_cancelled(&s->common); 412893f7ebaSPaolo Bonzini cnt = bdrv_get_dirty_count(bs); 413893f7ebaSPaolo Bonzini } 414b952b558SPaolo Bonzini } 415893f7ebaSPaolo Bonzini 416893f7ebaSPaolo Bonzini if (cnt == 0 && should_complete) { 417893f7ebaSPaolo Bonzini /* The dirty bitmap is not updated while operations are pending. 418893f7ebaSPaolo Bonzini * If we're about to exit, wait for pending operations before 419893f7ebaSPaolo Bonzini * calling bdrv_get_dirty_count(bs), or we may exit while the 420893f7ebaSPaolo Bonzini * source has dirty data to copy! 421893f7ebaSPaolo Bonzini * 422893f7ebaSPaolo Bonzini * Note that I/O can be submitted by the guest while 423893f7ebaSPaolo Bonzini * mirror_populate runs. 424893f7ebaSPaolo Bonzini */ 425893f7ebaSPaolo Bonzini trace_mirror_before_drain(s, cnt); 426893f7ebaSPaolo Bonzini bdrv_drain_all(); 427893f7ebaSPaolo Bonzini cnt = bdrv_get_dirty_count(bs); 428893f7ebaSPaolo Bonzini } 429893f7ebaSPaolo Bonzini 430893f7ebaSPaolo Bonzini ret = 0; 431d63ffd87SPaolo Bonzini trace_mirror_before_sleep(s, cnt, s->synced); 432d63ffd87SPaolo Bonzini if (!s->synced) { 433893f7ebaSPaolo Bonzini /* Publish progress */ 434acc906c6SPaolo Bonzini s->common.offset = (end - cnt) * BDRV_SECTOR_SIZE; 435893f7ebaSPaolo Bonzini 436893f7ebaSPaolo Bonzini if (s->common.speed) { 437eee13dfeSPaolo Bonzini delay_ns = ratelimit_calculate_delay(&s->limit, sectors_per_chunk); 438893f7ebaSPaolo Bonzini } else { 439893f7ebaSPaolo Bonzini delay_ns = 0; 440893f7ebaSPaolo Bonzini } 441893f7ebaSPaolo Bonzini 4427483d1e5SAlex Bligh block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 443893f7ebaSPaolo Bonzini if (block_job_is_cancelled(&s->common)) { 444893f7ebaSPaolo Bonzini break; 445893f7ebaSPaolo Bonzini } 446893f7ebaSPaolo Bonzini } else if (!should_complete) { 447bd48bde8SPaolo Bonzini delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0); 4487483d1e5SAlex Bligh block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 449893f7ebaSPaolo Bonzini } else if (cnt == 0) { 450893f7ebaSPaolo Bonzini /* The two disks are in sync. Exit and report successful 451893f7ebaSPaolo Bonzini * completion. 452893f7ebaSPaolo Bonzini */ 453893f7ebaSPaolo Bonzini assert(QLIST_EMPTY(&bs->tracked_requests)); 454893f7ebaSPaolo Bonzini s->common.cancelled = false; 455893f7ebaSPaolo Bonzini break; 456893f7ebaSPaolo Bonzini } 457bc72ad67SAlex Bligh last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 458893f7ebaSPaolo Bonzini } 459893f7ebaSPaolo Bonzini 460893f7ebaSPaolo Bonzini immediate_exit: 461bd48bde8SPaolo Bonzini if (s->in_flight > 0) { 462bd48bde8SPaolo Bonzini /* We get here only if something went wrong. Either the job failed, 463bd48bde8SPaolo Bonzini * or it was cancelled prematurely so that we do not guarantee that 464bd48bde8SPaolo Bonzini * the target is a copy of the source. 465bd48bde8SPaolo Bonzini */ 466bd48bde8SPaolo Bonzini assert(ret < 0 || (!s->synced && block_job_is_cancelled(&s->common))); 467bd48bde8SPaolo Bonzini mirror_drain(s); 468bd48bde8SPaolo Bonzini } 469bd48bde8SPaolo Bonzini 470bd48bde8SPaolo Bonzini assert(s->in_flight == 0); 4717191bf31SMarkus Armbruster qemu_vfree(s->buf); 472b812f671SPaolo Bonzini g_free(s->cow_bitmap); 473402a4741SPaolo Bonzini g_free(s->in_flight_bitmap); 47450717e94SPaolo Bonzini bdrv_set_dirty_tracking(bs, 0); 475b952b558SPaolo Bonzini bdrv_iostatus_disable(s->target); 476d63ffd87SPaolo Bonzini if (s->should_complete && ret == 0) { 477d63ffd87SPaolo Bonzini if (bdrv_get_flags(s->target) != bdrv_get_flags(s->common.bs)) { 478d63ffd87SPaolo Bonzini bdrv_reopen(s->target, bdrv_get_flags(s->common.bs), NULL); 479d63ffd87SPaolo Bonzini } 480d63ffd87SPaolo Bonzini bdrv_swap(s->target, s->common.bs); 481d63ffd87SPaolo Bonzini } 482893f7ebaSPaolo Bonzini bdrv_close(s->target); 4834f6fd349SFam Zheng bdrv_unref(s->target); 484893f7ebaSPaolo Bonzini block_job_completed(&s->common, ret); 485893f7ebaSPaolo Bonzini } 486893f7ebaSPaolo Bonzini 487893f7ebaSPaolo Bonzini static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp) 488893f7ebaSPaolo Bonzini { 489893f7ebaSPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 490893f7ebaSPaolo Bonzini 491893f7ebaSPaolo Bonzini if (speed < 0) { 492893f7ebaSPaolo Bonzini error_set(errp, QERR_INVALID_PARAMETER, "speed"); 493893f7ebaSPaolo Bonzini return; 494893f7ebaSPaolo Bonzini } 495893f7ebaSPaolo Bonzini ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME); 496893f7ebaSPaolo Bonzini } 497893f7ebaSPaolo Bonzini 498b952b558SPaolo Bonzini static void mirror_iostatus_reset(BlockJob *job) 499b952b558SPaolo Bonzini { 500b952b558SPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 501b952b558SPaolo Bonzini 502b952b558SPaolo Bonzini bdrv_iostatus_reset(s->target); 503b952b558SPaolo Bonzini } 504b952b558SPaolo Bonzini 505d63ffd87SPaolo Bonzini static void mirror_complete(BlockJob *job, Error **errp) 506d63ffd87SPaolo Bonzini { 507d63ffd87SPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 50834b5d2c6SMax Reitz Error *local_err = NULL; 509d63ffd87SPaolo Bonzini int ret; 510d63ffd87SPaolo Bonzini 51134b5d2c6SMax Reitz ret = bdrv_open_backing_file(s->target, NULL, &local_err); 512d63ffd87SPaolo Bonzini if (ret < 0) { 513d63ffd87SPaolo Bonzini char backing_filename[PATH_MAX]; 514d63ffd87SPaolo Bonzini bdrv_get_full_backing_filename(s->target, backing_filename, 515d63ffd87SPaolo Bonzini sizeof(backing_filename)); 51634b5d2c6SMax Reitz error_propagate(errp, local_err); 517d63ffd87SPaolo Bonzini return; 518d63ffd87SPaolo Bonzini } 519d63ffd87SPaolo Bonzini if (!s->synced) { 520d63ffd87SPaolo Bonzini error_set(errp, QERR_BLOCK_JOB_NOT_READY, job->bs->device_name); 521d63ffd87SPaolo Bonzini return; 522d63ffd87SPaolo Bonzini } 523d63ffd87SPaolo Bonzini 524d63ffd87SPaolo Bonzini s->should_complete = true; 525d63ffd87SPaolo Bonzini block_job_resume(job); 526d63ffd87SPaolo Bonzini } 527d63ffd87SPaolo Bonzini 5283fc4b10aSFam Zheng static const BlockJobDriver mirror_job_driver = { 529893f7ebaSPaolo Bonzini .instance_size = sizeof(MirrorBlockJob), 530*79e14bf7SFam Zheng .job_type = BLOCK_JOB_TYPE_MIRROR, 531893f7ebaSPaolo Bonzini .set_speed = mirror_set_speed, 532b952b558SPaolo Bonzini .iostatus_reset= mirror_iostatus_reset, 533d63ffd87SPaolo Bonzini .complete = mirror_complete, 534893f7ebaSPaolo Bonzini }; 535893f7ebaSPaolo Bonzini 536893f7ebaSPaolo Bonzini void mirror_start(BlockDriverState *bs, BlockDriverState *target, 53708e4ed6cSPaolo Bonzini int64_t speed, int64_t granularity, int64_t buf_size, 53808e4ed6cSPaolo Bonzini MirrorSyncMode mode, BlockdevOnError on_source_error, 539b952b558SPaolo Bonzini BlockdevOnError on_target_error, 540893f7ebaSPaolo Bonzini BlockDriverCompletionFunc *cb, 541893f7ebaSPaolo Bonzini void *opaque, Error **errp) 542893f7ebaSPaolo Bonzini { 543893f7ebaSPaolo Bonzini MirrorBlockJob *s; 544893f7ebaSPaolo Bonzini 545eee13dfeSPaolo Bonzini if (granularity == 0) { 546eee13dfeSPaolo Bonzini /* Choose the default granularity based on the target file's cluster 547eee13dfeSPaolo Bonzini * size, clamped between 4k and 64k. */ 548eee13dfeSPaolo Bonzini BlockDriverInfo bdi; 549eee13dfeSPaolo Bonzini if (bdrv_get_info(target, &bdi) >= 0 && bdi.cluster_size != 0) { 550eee13dfeSPaolo Bonzini granularity = MAX(4096, bdi.cluster_size); 551eee13dfeSPaolo Bonzini granularity = MIN(65536, granularity); 552eee13dfeSPaolo Bonzini } else { 553eee13dfeSPaolo Bonzini granularity = 65536; 554eee13dfeSPaolo Bonzini } 555eee13dfeSPaolo Bonzini } 556eee13dfeSPaolo Bonzini 557eee13dfeSPaolo Bonzini assert ((granularity & (granularity - 1)) == 0); 558eee13dfeSPaolo Bonzini 559b952b558SPaolo Bonzini if ((on_source_error == BLOCKDEV_ON_ERROR_STOP || 560b952b558SPaolo Bonzini on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) && 561b952b558SPaolo Bonzini !bdrv_iostatus_is_enabled(bs)) { 562b952b558SPaolo Bonzini error_set(errp, QERR_INVALID_PARAMETER, "on-source-error"); 563b952b558SPaolo Bonzini return; 564b952b558SPaolo Bonzini } 565b952b558SPaolo Bonzini 5663fc4b10aSFam Zheng s = block_job_create(&mirror_job_driver, bs, speed, cb, opaque, errp); 567893f7ebaSPaolo Bonzini if (!s) { 568893f7ebaSPaolo Bonzini return; 569893f7ebaSPaolo Bonzini } 570893f7ebaSPaolo Bonzini 571b952b558SPaolo Bonzini s->on_source_error = on_source_error; 572b952b558SPaolo Bonzini s->on_target_error = on_target_error; 573893f7ebaSPaolo Bonzini s->target = target; 574893f7ebaSPaolo Bonzini s->mode = mode; 575eee13dfeSPaolo Bonzini s->granularity = granularity; 57608e4ed6cSPaolo Bonzini s->buf_size = MAX(buf_size, granularity); 577b812f671SPaolo Bonzini 578eee13dfeSPaolo Bonzini bdrv_set_dirty_tracking(bs, granularity); 579893f7ebaSPaolo Bonzini bdrv_set_enable_write_cache(s->target, true); 580b952b558SPaolo Bonzini bdrv_set_on_error(s->target, on_target_error, on_target_error); 581b952b558SPaolo Bonzini bdrv_iostatus_enable(s->target); 582893f7ebaSPaolo Bonzini s->common.co = qemu_coroutine_create(mirror_run); 583893f7ebaSPaolo Bonzini trace_mirror_start(bs, s, s->common.co, opaque); 584893f7ebaSPaolo Bonzini qemu_coroutine_enter(s->common.co, s); 585893f7ebaSPaolo Bonzini } 586