1893f7ebaSPaolo Bonzini /* 2893f7ebaSPaolo Bonzini * Image mirroring 3893f7ebaSPaolo Bonzini * 4893f7ebaSPaolo Bonzini * Copyright Red Hat, Inc. 2012 5893f7ebaSPaolo Bonzini * 6893f7ebaSPaolo Bonzini * Authors: 7893f7ebaSPaolo Bonzini * Paolo Bonzini <pbonzini@redhat.com> 8893f7ebaSPaolo Bonzini * 9893f7ebaSPaolo Bonzini * This work is licensed under the terms of the GNU LGPL, version 2 or later. 10893f7ebaSPaolo Bonzini * See the COPYING.LIB file in the top-level directory. 11893f7ebaSPaolo Bonzini * 12893f7ebaSPaolo Bonzini */ 13893f7ebaSPaolo Bonzini 14893f7ebaSPaolo Bonzini #include "trace.h" 15737e150eSPaolo Bonzini #include "block/blockjob.h" 16737e150eSPaolo Bonzini #include "block/block_int.h" 17893f7ebaSPaolo Bonzini #include "qemu/ratelimit.h" 18b812f671SPaolo Bonzini #include "qemu/bitmap.h" 19893f7ebaSPaolo Bonzini 20893f7ebaSPaolo Bonzini #define SLICE_TIME 100000000ULL /* ns */ 21402a4741SPaolo Bonzini #define MAX_IN_FLIGHT 16 22402a4741SPaolo Bonzini 23402a4741SPaolo Bonzini /* The mirroring buffer is a list of granularity-sized chunks. 24402a4741SPaolo Bonzini * Free chunks are organized in a list. 25402a4741SPaolo Bonzini */ 26402a4741SPaolo Bonzini typedef struct MirrorBuffer { 27402a4741SPaolo Bonzini QSIMPLEQ_ENTRY(MirrorBuffer) next; 28402a4741SPaolo Bonzini } MirrorBuffer; 29893f7ebaSPaolo Bonzini 30893f7ebaSPaolo Bonzini typedef struct MirrorBlockJob { 31893f7ebaSPaolo Bonzini BlockJob common; 32893f7ebaSPaolo Bonzini RateLimit limit; 33893f7ebaSPaolo Bonzini BlockDriverState *target; 345bc361b8SFam Zheng BlockDriverState *base; 3509158f00SBenoît Canet /* The name of the graph node to replace */ 3609158f00SBenoît Canet char *replaces; 3709158f00SBenoît Canet /* The BDS to replace */ 3809158f00SBenoît Canet BlockDriverState *to_replace; 3909158f00SBenoît Canet /* Used to block operations on the drive-mirror-replace target */ 4009158f00SBenoît Canet Error *replace_blocker; 4103544a6eSFam Zheng bool is_none_mode; 42b952b558SPaolo Bonzini BlockdevOnError on_source_error, on_target_error; 43d63ffd87SPaolo Bonzini bool synced; 44d63ffd87SPaolo Bonzini bool should_complete; 45893f7ebaSPaolo Bonzini int64_t sector_num; 46eee13dfeSPaolo Bonzini int64_t granularity; 47b812f671SPaolo Bonzini size_t buf_size; 48b21c7652SMax Reitz int64_t bdev_length; 49b812f671SPaolo Bonzini unsigned long *cow_bitmap; 50e4654d2dSFam Zheng BdrvDirtyBitmap *dirty_bitmap; 518f0720ecSPaolo Bonzini HBitmapIter hbi; 52893f7ebaSPaolo Bonzini uint8_t *buf; 53402a4741SPaolo Bonzini QSIMPLEQ_HEAD(, MirrorBuffer) buf_free; 54402a4741SPaolo Bonzini int buf_free_count; 55bd48bde8SPaolo Bonzini 56402a4741SPaolo Bonzini unsigned long *in_flight_bitmap; 57bd48bde8SPaolo Bonzini int in_flight; 58b21c7652SMax Reitz int sectors_in_flight; 59bd48bde8SPaolo Bonzini int ret; 60893f7ebaSPaolo Bonzini } MirrorBlockJob; 61893f7ebaSPaolo Bonzini 62bd48bde8SPaolo Bonzini typedef struct MirrorOp { 63bd48bde8SPaolo Bonzini MirrorBlockJob *s; 64bd48bde8SPaolo Bonzini QEMUIOVector qiov; 65bd48bde8SPaolo Bonzini int64_t sector_num; 66bd48bde8SPaolo Bonzini int nb_sectors; 67bd48bde8SPaolo Bonzini } MirrorOp; 68bd48bde8SPaolo Bonzini 69b952b558SPaolo Bonzini static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read, 70b952b558SPaolo Bonzini int error) 71b952b558SPaolo Bonzini { 72b952b558SPaolo Bonzini s->synced = false; 73b952b558SPaolo Bonzini if (read) { 74b952b558SPaolo Bonzini return block_job_error_action(&s->common, s->common.bs, 75b952b558SPaolo Bonzini s->on_source_error, true, error); 76b952b558SPaolo Bonzini } else { 77b952b558SPaolo Bonzini return block_job_error_action(&s->common, s->target, 78b952b558SPaolo Bonzini s->on_target_error, false, error); 79b952b558SPaolo Bonzini } 80b952b558SPaolo Bonzini } 81b952b558SPaolo Bonzini 82bd48bde8SPaolo Bonzini static void mirror_iteration_done(MirrorOp *op, int ret) 83bd48bde8SPaolo Bonzini { 84bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 85402a4741SPaolo Bonzini struct iovec *iov; 86bd48bde8SPaolo Bonzini int64_t chunk_num; 87402a4741SPaolo Bonzini int i, nb_chunks, sectors_per_chunk; 88bd48bde8SPaolo Bonzini 89bd48bde8SPaolo Bonzini trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret); 90bd48bde8SPaolo Bonzini 91bd48bde8SPaolo Bonzini s->in_flight--; 92b21c7652SMax Reitz s->sectors_in_flight -= op->nb_sectors; 93402a4741SPaolo Bonzini iov = op->qiov.iov; 94402a4741SPaolo Bonzini for (i = 0; i < op->qiov.niov; i++) { 95402a4741SPaolo Bonzini MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base; 96402a4741SPaolo Bonzini QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next); 97402a4741SPaolo Bonzini s->buf_free_count++; 98402a4741SPaolo Bonzini } 99402a4741SPaolo Bonzini 100bd48bde8SPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 101bd48bde8SPaolo Bonzini chunk_num = op->sector_num / sectors_per_chunk; 102bd48bde8SPaolo Bonzini nb_chunks = op->nb_sectors / sectors_per_chunk; 103402a4741SPaolo Bonzini bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks); 104b21c7652SMax Reitz if (ret >= 0) { 105b21c7652SMax Reitz if (s->cow_bitmap) { 106bd48bde8SPaolo Bonzini bitmap_set(s->cow_bitmap, chunk_num, nb_chunks); 107bd48bde8SPaolo Bonzini } 108b21c7652SMax Reitz s->common.offset += (uint64_t)op->nb_sectors * BDRV_SECTOR_SIZE; 109b21c7652SMax Reitz } 110bd48bde8SPaolo Bonzini 1116df3bf8eSZhang Min qemu_iovec_destroy(&op->qiov); 112bd48bde8SPaolo Bonzini g_slice_free(MirrorOp, op); 1137b770c72SStefan Hajnoczi 1147b770c72SStefan Hajnoczi /* Enter coroutine when it is not sleeping. The coroutine sleeps to 1157b770c72SStefan Hajnoczi * rate-limit itself. The coroutine will eventually resume since there is 1167b770c72SStefan Hajnoczi * a sleep timeout so don't wake it early. 1177b770c72SStefan Hajnoczi */ 1187b770c72SStefan Hajnoczi if (s->common.busy) { 119bd48bde8SPaolo Bonzini qemu_coroutine_enter(s->common.co, NULL); 120bd48bde8SPaolo Bonzini } 1217b770c72SStefan Hajnoczi } 122bd48bde8SPaolo Bonzini 123bd48bde8SPaolo Bonzini static void mirror_write_complete(void *opaque, int ret) 124bd48bde8SPaolo Bonzini { 125bd48bde8SPaolo Bonzini MirrorOp *op = opaque; 126bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 127bd48bde8SPaolo Bonzini if (ret < 0) { 128bd48bde8SPaolo Bonzini BlockDriverState *source = s->common.bs; 129bd48bde8SPaolo Bonzini BlockErrorAction action; 130bd48bde8SPaolo Bonzini 131c4237dfaSVladimir Sementsov-Ogievskiy bdrv_set_dirty_bitmap(source, s->dirty_bitmap, op->sector_num, 132c4237dfaSVladimir Sementsov-Ogievskiy op->nb_sectors); 133bd48bde8SPaolo Bonzini action = mirror_error_action(s, false, -ret); 134a589569fSWenchao Xia if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) { 135bd48bde8SPaolo Bonzini s->ret = ret; 136bd48bde8SPaolo Bonzini } 137bd48bde8SPaolo Bonzini } 138bd48bde8SPaolo Bonzini mirror_iteration_done(op, ret); 139bd48bde8SPaolo Bonzini } 140bd48bde8SPaolo Bonzini 141bd48bde8SPaolo Bonzini static void mirror_read_complete(void *opaque, int ret) 142bd48bde8SPaolo Bonzini { 143bd48bde8SPaolo Bonzini MirrorOp *op = opaque; 144bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 145bd48bde8SPaolo Bonzini if (ret < 0) { 146bd48bde8SPaolo Bonzini BlockDriverState *source = s->common.bs; 147bd48bde8SPaolo Bonzini BlockErrorAction action; 148bd48bde8SPaolo Bonzini 149c4237dfaSVladimir Sementsov-Ogievskiy bdrv_set_dirty_bitmap(source, s->dirty_bitmap, op->sector_num, 150c4237dfaSVladimir Sementsov-Ogievskiy op->nb_sectors); 151bd48bde8SPaolo Bonzini action = mirror_error_action(s, true, -ret); 152a589569fSWenchao Xia if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) { 153bd48bde8SPaolo Bonzini s->ret = ret; 154bd48bde8SPaolo Bonzini } 155bd48bde8SPaolo Bonzini 156bd48bde8SPaolo Bonzini mirror_iteration_done(op, ret); 157bd48bde8SPaolo Bonzini return; 158bd48bde8SPaolo Bonzini } 159bd48bde8SPaolo Bonzini bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors, 160bd48bde8SPaolo Bonzini mirror_write_complete, op); 161bd48bde8SPaolo Bonzini } 162bd48bde8SPaolo Bonzini 163cc8c9d6cSPaolo Bonzini static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) 164893f7ebaSPaolo Bonzini { 165893f7ebaSPaolo Bonzini BlockDriverState *source = s->common.bs; 166402a4741SPaolo Bonzini int nb_sectors, sectors_per_chunk, nb_chunks; 167884fea4eSPaolo Bonzini int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector; 1686d0de8ebSStefan Hajnoczi uint64_t delay_ns = 0; 169bd48bde8SPaolo Bonzini MirrorOp *op; 170893f7ebaSPaolo Bonzini 1718f0720ecSPaolo Bonzini s->sector_num = hbitmap_iter_next(&s->hbi); 1728f0720ecSPaolo Bonzini if (s->sector_num < 0) { 173e4654d2dSFam Zheng bdrv_dirty_iter_init(source, s->dirty_bitmap, &s->hbi); 1748f0720ecSPaolo Bonzini s->sector_num = hbitmap_iter_next(&s->hbi); 175e4654d2dSFam Zheng trace_mirror_restart_iter(s, 176e4654d2dSFam Zheng bdrv_get_dirty_count(source, s->dirty_bitmap)); 1778f0720ecSPaolo Bonzini assert(s->sector_num >= 0); 1788f0720ecSPaolo Bonzini } 1798f0720ecSPaolo Bonzini 180402a4741SPaolo Bonzini hbitmap_next_sector = s->sector_num; 181b812f671SPaolo Bonzini sector_num = s->sector_num; 182884fea4eSPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 183b21c7652SMax Reitz end = s->bdev_length / BDRV_SECTOR_SIZE; 184402a4741SPaolo Bonzini 185884fea4eSPaolo Bonzini /* Extend the QEMUIOVector to include all adjacent blocks that will 186884fea4eSPaolo Bonzini * be copied in this operation. 187884fea4eSPaolo Bonzini * 188884fea4eSPaolo Bonzini * We have to do this if we have no backing file yet in the destination, 189884fea4eSPaolo Bonzini * and the cluster size is very large. Then we need to do COW ourselves. 190884fea4eSPaolo Bonzini * The first time a cluster is copied, copy it entirely. Note that, 191884fea4eSPaolo Bonzini * because both the granularity and the cluster size are powers of two, 192884fea4eSPaolo Bonzini * the number of sectors to copy cannot exceed one cluster. 193884fea4eSPaolo Bonzini * 194884fea4eSPaolo Bonzini * We also want to extend the QEMUIOVector to include more adjacent 195884fea4eSPaolo Bonzini * dirty blocks if possible, to limit the number of I/O operations and 196884fea4eSPaolo Bonzini * run efficiently even with a small granularity. 197402a4741SPaolo Bonzini */ 198884fea4eSPaolo Bonzini nb_chunks = 0; 199884fea4eSPaolo Bonzini nb_sectors = 0; 200884fea4eSPaolo Bonzini next_sector = sector_num; 201884fea4eSPaolo Bonzini next_chunk = sector_num / sectors_per_chunk; 202402a4741SPaolo Bonzini 203402a4741SPaolo Bonzini /* Wait for I/O to this cluster (from a previous iteration) to be done. */ 204884fea4eSPaolo Bonzini while (test_bit(next_chunk, s->in_flight_bitmap)) { 205402a4741SPaolo Bonzini trace_mirror_yield_in_flight(s, sector_num, s->in_flight); 206402a4741SPaolo Bonzini qemu_coroutine_yield(); 207b812f671SPaolo Bonzini } 208b812f671SPaolo Bonzini 209884fea4eSPaolo Bonzini do { 210884fea4eSPaolo Bonzini int added_sectors, added_chunks; 211884fea4eSPaolo Bonzini 212e4654d2dSFam Zheng if (!bdrv_get_dirty(source, s->dirty_bitmap, next_sector) || 213884fea4eSPaolo Bonzini test_bit(next_chunk, s->in_flight_bitmap)) { 214884fea4eSPaolo Bonzini assert(nb_sectors > 0); 215884fea4eSPaolo Bonzini break; 216884fea4eSPaolo Bonzini } 217884fea4eSPaolo Bonzini 218884fea4eSPaolo Bonzini added_sectors = sectors_per_chunk; 219884fea4eSPaolo Bonzini if (s->cow_bitmap && !test_bit(next_chunk, s->cow_bitmap)) { 220884fea4eSPaolo Bonzini bdrv_round_to_clusters(s->target, 221884fea4eSPaolo Bonzini next_sector, added_sectors, 222884fea4eSPaolo Bonzini &next_sector, &added_sectors); 223884fea4eSPaolo Bonzini 224884fea4eSPaolo Bonzini /* On the first iteration, the rounding may make us copy 225884fea4eSPaolo Bonzini * sectors before the first dirty one. 226884fea4eSPaolo Bonzini */ 227884fea4eSPaolo Bonzini if (next_sector < sector_num) { 228884fea4eSPaolo Bonzini assert(nb_sectors == 0); 229884fea4eSPaolo Bonzini sector_num = next_sector; 230884fea4eSPaolo Bonzini next_chunk = next_sector / sectors_per_chunk; 231884fea4eSPaolo Bonzini } 232884fea4eSPaolo Bonzini } 233884fea4eSPaolo Bonzini 234884fea4eSPaolo Bonzini added_sectors = MIN(added_sectors, end - (sector_num + nb_sectors)); 235884fea4eSPaolo Bonzini added_chunks = (added_sectors + sectors_per_chunk - 1) / sectors_per_chunk; 236884fea4eSPaolo Bonzini 237884fea4eSPaolo Bonzini /* When doing COW, it may happen that there is not enough space for 238884fea4eSPaolo Bonzini * a full cluster. Wait if that is the case. 239884fea4eSPaolo Bonzini */ 240884fea4eSPaolo Bonzini while (nb_chunks == 0 && s->buf_free_count < added_chunks) { 241402a4741SPaolo Bonzini trace_mirror_yield_buf_busy(s, nb_chunks, s->in_flight); 242402a4741SPaolo Bonzini qemu_coroutine_yield(); 243402a4741SPaolo Bonzini } 244884fea4eSPaolo Bonzini if (s->buf_free_count < nb_chunks + added_chunks) { 245884fea4eSPaolo Bonzini trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight); 246884fea4eSPaolo Bonzini break; 247884fea4eSPaolo Bonzini } 248402a4741SPaolo Bonzini 249402a4741SPaolo Bonzini /* We have enough free space to copy these sectors. */ 250884fea4eSPaolo Bonzini bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks); 251884fea4eSPaolo Bonzini 252884fea4eSPaolo Bonzini nb_sectors += added_sectors; 253884fea4eSPaolo Bonzini nb_chunks += added_chunks; 254884fea4eSPaolo Bonzini next_sector += added_sectors; 255884fea4eSPaolo Bonzini next_chunk += added_chunks; 256cc8c9d6cSPaolo Bonzini if (!s->synced && s->common.speed) { 257cc8c9d6cSPaolo Bonzini delay_ns = ratelimit_calculate_delay(&s->limit, added_sectors); 258cc8c9d6cSPaolo Bonzini } 259cc8c9d6cSPaolo Bonzini } while (delay_ns == 0 && next_sector < end); 260bd48bde8SPaolo Bonzini 261bd48bde8SPaolo Bonzini /* Allocate a MirrorOp that is used as an AIO callback. */ 262bd48bde8SPaolo Bonzini op = g_slice_new(MirrorOp); 263bd48bde8SPaolo Bonzini op->s = s; 264bd48bde8SPaolo Bonzini op->sector_num = sector_num; 265bd48bde8SPaolo Bonzini op->nb_sectors = nb_sectors; 266402a4741SPaolo Bonzini 267402a4741SPaolo Bonzini /* Now make a QEMUIOVector taking enough granularity-sized chunks 268402a4741SPaolo Bonzini * from s->buf_free. 269402a4741SPaolo Bonzini */ 270402a4741SPaolo Bonzini qemu_iovec_init(&op->qiov, nb_chunks); 271402a4741SPaolo Bonzini next_sector = sector_num; 272402a4741SPaolo Bonzini while (nb_chunks-- > 0) { 273402a4741SPaolo Bonzini MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free); 2745a0f6fd5SKevin Wolf size_t remaining = (nb_sectors * BDRV_SECTOR_SIZE) - op->qiov.size; 2755a0f6fd5SKevin Wolf 276402a4741SPaolo Bonzini QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next); 277402a4741SPaolo Bonzini s->buf_free_count--; 2785a0f6fd5SKevin Wolf qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining)); 279402a4741SPaolo Bonzini 280402a4741SPaolo Bonzini /* Advance the HBitmapIter in parallel, so that we do not examine 281402a4741SPaolo Bonzini * the same sector twice. 282402a4741SPaolo Bonzini */ 283e4654d2dSFam Zheng if (next_sector > hbitmap_next_sector 284e4654d2dSFam Zheng && bdrv_get_dirty(source, s->dirty_bitmap, next_sector)) { 285402a4741SPaolo Bonzini hbitmap_next_sector = hbitmap_iter_next(&s->hbi); 286402a4741SPaolo Bonzini } 287402a4741SPaolo Bonzini 288402a4741SPaolo Bonzini next_sector += sectors_per_chunk; 289402a4741SPaolo Bonzini } 290bd48bde8SPaolo Bonzini 291c4237dfaSVladimir Sementsov-Ogievskiy bdrv_reset_dirty_bitmap(source, s->dirty_bitmap, sector_num, 292c4237dfaSVladimir Sementsov-Ogievskiy nb_sectors); 293893f7ebaSPaolo Bonzini 294893f7ebaSPaolo Bonzini /* Copy the dirty cluster. */ 295bd48bde8SPaolo Bonzini s->in_flight++; 296b21c7652SMax Reitz s->sectors_in_flight += nb_sectors; 297b812f671SPaolo Bonzini trace_mirror_one_iteration(s, sector_num, nb_sectors); 298bd48bde8SPaolo Bonzini bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors, 299bd48bde8SPaolo Bonzini mirror_read_complete, op); 300cc8c9d6cSPaolo Bonzini return delay_ns; 301893f7ebaSPaolo Bonzini } 302b952b558SPaolo Bonzini 303402a4741SPaolo Bonzini static void mirror_free_init(MirrorBlockJob *s) 304402a4741SPaolo Bonzini { 305402a4741SPaolo Bonzini int granularity = s->granularity; 306402a4741SPaolo Bonzini size_t buf_size = s->buf_size; 307402a4741SPaolo Bonzini uint8_t *buf = s->buf; 308402a4741SPaolo Bonzini 309402a4741SPaolo Bonzini assert(s->buf_free_count == 0); 310402a4741SPaolo Bonzini QSIMPLEQ_INIT(&s->buf_free); 311402a4741SPaolo Bonzini while (buf_size != 0) { 312402a4741SPaolo Bonzini MirrorBuffer *cur = (MirrorBuffer *)buf; 313402a4741SPaolo Bonzini QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next); 314402a4741SPaolo Bonzini s->buf_free_count++; 315402a4741SPaolo Bonzini buf_size -= granularity; 316402a4741SPaolo Bonzini buf += granularity; 317402a4741SPaolo Bonzini } 318402a4741SPaolo Bonzini } 319402a4741SPaolo Bonzini 320bd48bde8SPaolo Bonzini static void mirror_drain(MirrorBlockJob *s) 321bd48bde8SPaolo Bonzini { 322bd48bde8SPaolo Bonzini while (s->in_flight > 0) { 323bd48bde8SPaolo Bonzini qemu_coroutine_yield(); 324bd48bde8SPaolo Bonzini } 325893f7ebaSPaolo Bonzini } 326893f7ebaSPaolo Bonzini 3275a7e7a0bSStefan Hajnoczi typedef struct { 3285a7e7a0bSStefan Hajnoczi int ret; 3295a7e7a0bSStefan Hajnoczi } MirrorExitData; 3305a7e7a0bSStefan Hajnoczi 3315a7e7a0bSStefan Hajnoczi static void mirror_exit(BlockJob *job, void *opaque) 3325a7e7a0bSStefan Hajnoczi { 3335a7e7a0bSStefan Hajnoczi MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 3345a7e7a0bSStefan Hajnoczi MirrorExitData *data = opaque; 3355a7e7a0bSStefan Hajnoczi AioContext *replace_aio_context = NULL; 3365a7e7a0bSStefan Hajnoczi 3375a7e7a0bSStefan Hajnoczi if (s->to_replace) { 3385a7e7a0bSStefan Hajnoczi replace_aio_context = bdrv_get_aio_context(s->to_replace); 3395a7e7a0bSStefan Hajnoczi aio_context_acquire(replace_aio_context); 3405a7e7a0bSStefan Hajnoczi } 3415a7e7a0bSStefan Hajnoczi 3425a7e7a0bSStefan Hajnoczi if (s->should_complete && data->ret == 0) { 3435a7e7a0bSStefan Hajnoczi BlockDriverState *to_replace = s->common.bs; 3445a7e7a0bSStefan Hajnoczi if (s->to_replace) { 3455a7e7a0bSStefan Hajnoczi to_replace = s->to_replace; 3465a7e7a0bSStefan Hajnoczi } 3475a7e7a0bSStefan Hajnoczi if (bdrv_get_flags(s->target) != bdrv_get_flags(to_replace)) { 3485a7e7a0bSStefan Hajnoczi bdrv_reopen(s->target, bdrv_get_flags(to_replace), NULL); 3495a7e7a0bSStefan Hajnoczi } 3505a7e7a0bSStefan Hajnoczi bdrv_swap(s->target, to_replace); 3515a7e7a0bSStefan Hajnoczi if (s->common.driver->job_type == BLOCK_JOB_TYPE_COMMIT) { 3525a7e7a0bSStefan Hajnoczi /* drop the bs loop chain formed by the swap: break the loop then 3535a7e7a0bSStefan Hajnoczi * trigger the unref from the top one */ 3545a7e7a0bSStefan Hajnoczi BlockDriverState *p = s->base->backing_hd; 3555a7e7a0bSStefan Hajnoczi bdrv_set_backing_hd(s->base, NULL); 3565a7e7a0bSStefan Hajnoczi bdrv_unref(p); 3575a7e7a0bSStefan Hajnoczi } 3585a7e7a0bSStefan Hajnoczi } 3595a7e7a0bSStefan Hajnoczi if (s->to_replace) { 3605a7e7a0bSStefan Hajnoczi bdrv_op_unblock_all(s->to_replace, s->replace_blocker); 3615a7e7a0bSStefan Hajnoczi error_free(s->replace_blocker); 3625a7e7a0bSStefan Hajnoczi bdrv_unref(s->to_replace); 3635a7e7a0bSStefan Hajnoczi } 3645a7e7a0bSStefan Hajnoczi if (replace_aio_context) { 3655a7e7a0bSStefan Hajnoczi aio_context_release(replace_aio_context); 3665a7e7a0bSStefan Hajnoczi } 3675a7e7a0bSStefan Hajnoczi g_free(s->replaces); 3685a7e7a0bSStefan Hajnoczi bdrv_unref(s->target); 3695a7e7a0bSStefan Hajnoczi block_job_completed(&s->common, data->ret); 3705a7e7a0bSStefan Hajnoczi g_free(data); 3715a7e7a0bSStefan Hajnoczi } 3725a7e7a0bSStefan Hajnoczi 373893f7ebaSPaolo Bonzini static void coroutine_fn mirror_run(void *opaque) 374893f7ebaSPaolo Bonzini { 375893f7ebaSPaolo Bonzini MirrorBlockJob *s = opaque; 3765a7e7a0bSStefan Hajnoczi MirrorExitData *data; 377893f7ebaSPaolo Bonzini BlockDriverState *bs = s->common.bs; 378eee13dfeSPaolo Bonzini int64_t sector_num, end, sectors_per_chunk, length; 379bd48bde8SPaolo Bonzini uint64_t last_pause_ns; 380b812f671SPaolo Bonzini BlockDriverInfo bdi; 3811d33936eSJeff Cody char backing_filename[2]; /* we only need 2 characters because we are only 3821d33936eSJeff Cody checking for a NULL string */ 383893f7ebaSPaolo Bonzini int ret = 0; 384893f7ebaSPaolo Bonzini int n; 385893f7ebaSPaolo Bonzini 386893f7ebaSPaolo Bonzini if (block_job_is_cancelled(&s->common)) { 387893f7ebaSPaolo Bonzini goto immediate_exit; 388893f7ebaSPaolo Bonzini } 389893f7ebaSPaolo Bonzini 390b21c7652SMax Reitz s->bdev_length = bdrv_getlength(bs); 391b21c7652SMax Reitz if (s->bdev_length < 0) { 392b21c7652SMax Reitz ret = s->bdev_length; 393373df5b1SFam Zheng goto immediate_exit; 394b21c7652SMax Reitz } else if (s->bdev_length == 0) { 3959e48b025SFam Zheng /* Report BLOCK_JOB_READY and wait for complete. */ 3969e48b025SFam Zheng block_job_event_ready(&s->common); 3979e48b025SFam Zheng s->synced = true; 3989e48b025SFam Zheng while (!block_job_is_cancelled(&s->common) && !s->should_complete) { 3999e48b025SFam Zheng block_job_yield(&s->common); 4009e48b025SFam Zheng } 4019e48b025SFam Zheng s->common.cancelled = false; 4029e48b025SFam Zheng goto immediate_exit; 403893f7ebaSPaolo Bonzini } 404893f7ebaSPaolo Bonzini 405b21c7652SMax Reitz length = DIV_ROUND_UP(s->bdev_length, s->granularity); 406402a4741SPaolo Bonzini s->in_flight_bitmap = bitmap_new(length); 407402a4741SPaolo Bonzini 408b812f671SPaolo Bonzini /* If we have no backing file yet in the destination, we cannot let 409b812f671SPaolo Bonzini * the destination do COW. Instead, we copy sectors around the 410b812f671SPaolo Bonzini * dirty data if needed. We need a bitmap to do that. 411b812f671SPaolo Bonzini */ 412b812f671SPaolo Bonzini bdrv_get_backing_filename(s->target, backing_filename, 413b812f671SPaolo Bonzini sizeof(backing_filename)); 414b812f671SPaolo Bonzini if (backing_filename[0] && !s->target->backing_hd) { 415c3cc95bdSFam Zheng ret = bdrv_get_info(s->target, &bdi); 416c3cc95bdSFam Zheng if (ret < 0) { 417c3cc95bdSFam Zheng goto immediate_exit; 418c3cc95bdSFam Zheng } 419eee13dfeSPaolo Bonzini if (s->granularity < bdi.cluster_size) { 42008e4ed6cSPaolo Bonzini s->buf_size = MAX(s->buf_size, bdi.cluster_size); 421b812f671SPaolo Bonzini s->cow_bitmap = bitmap_new(length); 422b812f671SPaolo Bonzini } 423b812f671SPaolo Bonzini } 424b812f671SPaolo Bonzini 425b21c7652SMax Reitz end = s->bdev_length / BDRV_SECTOR_SIZE; 4267504edf4SKevin Wolf s->buf = qemu_try_blockalign(bs, s->buf_size); 4277504edf4SKevin Wolf if (s->buf == NULL) { 4287504edf4SKevin Wolf ret = -ENOMEM; 4297504edf4SKevin Wolf goto immediate_exit; 4307504edf4SKevin Wolf } 4317504edf4SKevin Wolf 432eee13dfeSPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 433402a4741SPaolo Bonzini mirror_free_init(s); 434893f7ebaSPaolo Bonzini 43503544a6eSFam Zheng if (!s->is_none_mode) { 436893f7ebaSPaolo Bonzini /* First part, loop on the sectors and initialize the dirty bitmap. */ 4375bc361b8SFam Zheng BlockDriverState *base = s->base; 438893f7ebaSPaolo Bonzini for (sector_num = 0; sector_num < end; ) { 439eee13dfeSPaolo Bonzini int64_t next = (sector_num | (sectors_per_chunk - 1)) + 1; 4404f578637SPaolo Bonzini ret = bdrv_is_allocated_above(bs, base, 441893f7ebaSPaolo Bonzini sector_num, next - sector_num, &n); 442893f7ebaSPaolo Bonzini 443893f7ebaSPaolo Bonzini if (ret < 0) { 444893f7ebaSPaolo Bonzini goto immediate_exit; 445893f7ebaSPaolo Bonzini } 446893f7ebaSPaolo Bonzini 447893f7ebaSPaolo Bonzini assert(n > 0); 448893f7ebaSPaolo Bonzini if (ret == 1) { 449c4237dfaSVladimir Sementsov-Ogievskiy bdrv_set_dirty_bitmap(bs, s->dirty_bitmap, sector_num, n); 450893f7ebaSPaolo Bonzini sector_num = next; 451893f7ebaSPaolo Bonzini } else { 452893f7ebaSPaolo Bonzini sector_num += n; 453893f7ebaSPaolo Bonzini } 454893f7ebaSPaolo Bonzini } 455893f7ebaSPaolo Bonzini } 456893f7ebaSPaolo Bonzini 457e4654d2dSFam Zheng bdrv_dirty_iter_init(bs, s->dirty_bitmap, &s->hbi); 458bc72ad67SAlex Bligh last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 459893f7ebaSPaolo Bonzini for (;;) { 460cc8c9d6cSPaolo Bonzini uint64_t delay_ns = 0; 461893f7ebaSPaolo Bonzini int64_t cnt; 462893f7ebaSPaolo Bonzini bool should_complete; 463893f7ebaSPaolo Bonzini 464bd48bde8SPaolo Bonzini if (s->ret < 0) { 465bd48bde8SPaolo Bonzini ret = s->ret; 466893f7ebaSPaolo Bonzini goto immediate_exit; 467893f7ebaSPaolo Bonzini } 468bd48bde8SPaolo Bonzini 469e4654d2dSFam Zheng cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap); 470b21c7652SMax Reitz /* s->common.offset contains the number of bytes already processed so 471b21c7652SMax Reitz * far, cnt is the number of dirty sectors remaining and 472b21c7652SMax Reitz * s->sectors_in_flight is the number of sectors currently being 473b21c7652SMax Reitz * processed; together those are the current total operation length */ 474b21c7652SMax Reitz s->common.len = s->common.offset + 475b21c7652SMax Reitz (cnt + s->sectors_in_flight) * BDRV_SECTOR_SIZE; 476bd48bde8SPaolo Bonzini 477bd48bde8SPaolo Bonzini /* Note that even when no rate limit is applied we need to yield 478bd48bde8SPaolo Bonzini * periodically with no pending I/O so that qemu_aio_flush() returns. 479bd48bde8SPaolo Bonzini * We do so every SLICE_TIME nanoseconds, or when there is an error, 480bd48bde8SPaolo Bonzini * or when the source is clean, whichever comes first. 481bd48bde8SPaolo Bonzini */ 482bc72ad67SAlex Bligh if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - last_pause_ns < SLICE_TIME && 483bd48bde8SPaolo Bonzini s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) { 484402a4741SPaolo Bonzini if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 || 485402a4741SPaolo Bonzini (cnt == 0 && s->in_flight > 0)) { 486402a4741SPaolo Bonzini trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt); 487bd48bde8SPaolo Bonzini qemu_coroutine_yield(); 488bd48bde8SPaolo Bonzini continue; 489bd48bde8SPaolo Bonzini } else if (cnt != 0) { 490cc8c9d6cSPaolo Bonzini delay_ns = mirror_iteration(s); 491cc8c9d6cSPaolo Bonzini if (delay_ns == 0) { 492bd48bde8SPaolo Bonzini continue; 493bd48bde8SPaolo Bonzini } 494893f7ebaSPaolo Bonzini } 495cc8c9d6cSPaolo Bonzini } 496893f7ebaSPaolo Bonzini 497893f7ebaSPaolo Bonzini should_complete = false; 498bd48bde8SPaolo Bonzini if (s->in_flight == 0 && cnt == 0) { 499893f7ebaSPaolo Bonzini trace_mirror_before_flush(s); 500893f7ebaSPaolo Bonzini ret = bdrv_flush(s->target); 501893f7ebaSPaolo Bonzini if (ret < 0) { 502a589569fSWenchao Xia if (mirror_error_action(s, false, -ret) == 503a589569fSWenchao Xia BLOCK_ERROR_ACTION_REPORT) { 504893f7ebaSPaolo Bonzini goto immediate_exit; 505893f7ebaSPaolo Bonzini } 506b952b558SPaolo Bonzini } else { 507893f7ebaSPaolo Bonzini /* We're out of the streaming phase. From now on, if the job 508893f7ebaSPaolo Bonzini * is cancelled we will actually complete all pending I/O and 509893f7ebaSPaolo Bonzini * report completion. This way, block-job-cancel will leave 510893f7ebaSPaolo Bonzini * the target in a consistent state. 511893f7ebaSPaolo Bonzini */ 512d63ffd87SPaolo Bonzini if (!s->synced) { 513bcada37bSWenchao Xia block_job_event_ready(&s->common); 514d63ffd87SPaolo Bonzini s->synced = true; 515d63ffd87SPaolo Bonzini } 516d63ffd87SPaolo Bonzini 517d63ffd87SPaolo Bonzini should_complete = s->should_complete || 518d63ffd87SPaolo Bonzini block_job_is_cancelled(&s->common); 519e4654d2dSFam Zheng cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap); 520893f7ebaSPaolo Bonzini } 521b952b558SPaolo Bonzini } 522893f7ebaSPaolo Bonzini 523893f7ebaSPaolo Bonzini if (cnt == 0 && should_complete) { 524893f7ebaSPaolo Bonzini /* The dirty bitmap is not updated while operations are pending. 525893f7ebaSPaolo Bonzini * If we're about to exit, wait for pending operations before 526893f7ebaSPaolo Bonzini * calling bdrv_get_dirty_count(bs), or we may exit while the 527893f7ebaSPaolo Bonzini * source has dirty data to copy! 528893f7ebaSPaolo Bonzini * 529893f7ebaSPaolo Bonzini * Note that I/O can be submitted by the guest while 530893f7ebaSPaolo Bonzini * mirror_populate runs. 531893f7ebaSPaolo Bonzini */ 532893f7ebaSPaolo Bonzini trace_mirror_before_drain(s, cnt); 5335a7e7a0bSStefan Hajnoczi bdrv_drain(bs); 534e4654d2dSFam Zheng cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap); 535893f7ebaSPaolo Bonzini } 536893f7ebaSPaolo Bonzini 537893f7ebaSPaolo Bonzini ret = 0; 538cc8c9d6cSPaolo Bonzini trace_mirror_before_sleep(s, cnt, s->synced, delay_ns); 539d63ffd87SPaolo Bonzini if (!s->synced) { 5407483d1e5SAlex Bligh block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 541893f7ebaSPaolo Bonzini if (block_job_is_cancelled(&s->common)) { 542893f7ebaSPaolo Bonzini break; 543893f7ebaSPaolo Bonzini } 544893f7ebaSPaolo Bonzini } else if (!should_complete) { 545bd48bde8SPaolo Bonzini delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0); 5467483d1e5SAlex Bligh block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 547893f7ebaSPaolo Bonzini } else if (cnt == 0) { 548893f7ebaSPaolo Bonzini /* The two disks are in sync. Exit and report successful 549893f7ebaSPaolo Bonzini * completion. 550893f7ebaSPaolo Bonzini */ 551893f7ebaSPaolo Bonzini assert(QLIST_EMPTY(&bs->tracked_requests)); 552893f7ebaSPaolo Bonzini s->common.cancelled = false; 553893f7ebaSPaolo Bonzini break; 554893f7ebaSPaolo Bonzini } 555bc72ad67SAlex Bligh last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 556893f7ebaSPaolo Bonzini } 557893f7ebaSPaolo Bonzini 558893f7ebaSPaolo Bonzini immediate_exit: 559bd48bde8SPaolo Bonzini if (s->in_flight > 0) { 560bd48bde8SPaolo Bonzini /* We get here only if something went wrong. Either the job failed, 561bd48bde8SPaolo Bonzini * or it was cancelled prematurely so that we do not guarantee that 562bd48bde8SPaolo Bonzini * the target is a copy of the source. 563bd48bde8SPaolo Bonzini */ 564bd48bde8SPaolo Bonzini assert(ret < 0 || (!s->synced && block_job_is_cancelled(&s->common))); 565bd48bde8SPaolo Bonzini mirror_drain(s); 566bd48bde8SPaolo Bonzini } 567bd48bde8SPaolo Bonzini 568bd48bde8SPaolo Bonzini assert(s->in_flight == 0); 5697191bf31SMarkus Armbruster qemu_vfree(s->buf); 570b812f671SPaolo Bonzini g_free(s->cow_bitmap); 571402a4741SPaolo Bonzini g_free(s->in_flight_bitmap); 572e4654d2dSFam Zheng bdrv_release_dirty_bitmap(bs, s->dirty_bitmap); 573b952b558SPaolo Bonzini bdrv_iostatus_disable(s->target); 5745a7e7a0bSStefan Hajnoczi 5755a7e7a0bSStefan Hajnoczi data = g_malloc(sizeof(*data)); 5765a7e7a0bSStefan Hajnoczi data->ret = ret; 5775a7e7a0bSStefan Hajnoczi block_job_defer_to_main_loop(&s->common, mirror_exit, data); 578893f7ebaSPaolo Bonzini } 579893f7ebaSPaolo Bonzini 580893f7ebaSPaolo Bonzini static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp) 581893f7ebaSPaolo Bonzini { 582893f7ebaSPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 583893f7ebaSPaolo Bonzini 584893f7ebaSPaolo Bonzini if (speed < 0) { 585893f7ebaSPaolo Bonzini error_set(errp, QERR_INVALID_PARAMETER, "speed"); 586893f7ebaSPaolo Bonzini return; 587893f7ebaSPaolo Bonzini } 588893f7ebaSPaolo Bonzini ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME); 589893f7ebaSPaolo Bonzini } 590893f7ebaSPaolo Bonzini 591b952b558SPaolo Bonzini static void mirror_iostatus_reset(BlockJob *job) 592b952b558SPaolo Bonzini { 593b952b558SPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 594b952b558SPaolo Bonzini 595b952b558SPaolo Bonzini bdrv_iostatus_reset(s->target); 596b952b558SPaolo Bonzini } 597b952b558SPaolo Bonzini 598d63ffd87SPaolo Bonzini static void mirror_complete(BlockJob *job, Error **errp) 599d63ffd87SPaolo Bonzini { 600d63ffd87SPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 60134b5d2c6SMax Reitz Error *local_err = NULL; 602d63ffd87SPaolo Bonzini int ret; 603d63ffd87SPaolo Bonzini 60434b5d2c6SMax Reitz ret = bdrv_open_backing_file(s->target, NULL, &local_err); 605d63ffd87SPaolo Bonzini if (ret < 0) { 60634b5d2c6SMax Reitz error_propagate(errp, local_err); 607d63ffd87SPaolo Bonzini return; 608d63ffd87SPaolo Bonzini } 609d63ffd87SPaolo Bonzini if (!s->synced) { 610bfb197e0SMarkus Armbruster error_set(errp, QERR_BLOCK_JOB_NOT_READY, 611bfb197e0SMarkus Armbruster bdrv_get_device_name(job->bs)); 612d63ffd87SPaolo Bonzini return; 613d63ffd87SPaolo Bonzini } 614d63ffd87SPaolo Bonzini 61509158f00SBenoît Canet /* check the target bs is not blocked and block all operations on it */ 61609158f00SBenoît Canet if (s->replaces) { 6175a7e7a0bSStefan Hajnoczi AioContext *replace_aio_context; 6185a7e7a0bSStefan Hajnoczi 61909158f00SBenoît Canet s->to_replace = check_to_replace_node(s->replaces, &local_err); 62009158f00SBenoît Canet if (!s->to_replace) { 62109158f00SBenoît Canet error_propagate(errp, local_err); 62209158f00SBenoît Canet return; 62309158f00SBenoît Canet } 62409158f00SBenoît Canet 6255a7e7a0bSStefan Hajnoczi replace_aio_context = bdrv_get_aio_context(s->to_replace); 6265a7e7a0bSStefan Hajnoczi aio_context_acquire(replace_aio_context); 6275a7e7a0bSStefan Hajnoczi 62809158f00SBenoît Canet error_setg(&s->replace_blocker, 62909158f00SBenoît Canet "block device is in use by block-job-complete"); 63009158f00SBenoît Canet bdrv_op_block_all(s->to_replace, s->replace_blocker); 63109158f00SBenoît Canet bdrv_ref(s->to_replace); 6325a7e7a0bSStefan Hajnoczi 6335a7e7a0bSStefan Hajnoczi aio_context_release(replace_aio_context); 63409158f00SBenoît Canet } 63509158f00SBenoît Canet 636d63ffd87SPaolo Bonzini s->should_complete = true; 637*751ebd76SFam Zheng block_job_enter(&s->common); 638d63ffd87SPaolo Bonzini } 639d63ffd87SPaolo Bonzini 6403fc4b10aSFam Zheng static const BlockJobDriver mirror_job_driver = { 641893f7ebaSPaolo Bonzini .instance_size = sizeof(MirrorBlockJob), 64279e14bf7SFam Zheng .job_type = BLOCK_JOB_TYPE_MIRROR, 643893f7ebaSPaolo Bonzini .set_speed = mirror_set_speed, 644b952b558SPaolo Bonzini .iostatus_reset= mirror_iostatus_reset, 645d63ffd87SPaolo Bonzini .complete = mirror_complete, 646893f7ebaSPaolo Bonzini }; 647893f7ebaSPaolo Bonzini 64803544a6eSFam Zheng static const BlockJobDriver commit_active_job_driver = { 64903544a6eSFam Zheng .instance_size = sizeof(MirrorBlockJob), 65003544a6eSFam Zheng .job_type = BLOCK_JOB_TYPE_COMMIT, 65103544a6eSFam Zheng .set_speed = mirror_set_speed, 65203544a6eSFam Zheng .iostatus_reset 65303544a6eSFam Zheng = mirror_iostatus_reset, 65403544a6eSFam Zheng .complete = mirror_complete, 65503544a6eSFam Zheng }; 65603544a6eSFam Zheng 65703544a6eSFam Zheng static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target, 65809158f00SBenoît Canet const char *replaces, 65903544a6eSFam Zheng int64_t speed, int64_t granularity, 66003544a6eSFam Zheng int64_t buf_size, 66103544a6eSFam Zheng BlockdevOnError on_source_error, 662b952b558SPaolo Bonzini BlockdevOnError on_target_error, 663097310b5SMarkus Armbruster BlockCompletionFunc *cb, 66403544a6eSFam Zheng void *opaque, Error **errp, 66503544a6eSFam Zheng const BlockJobDriver *driver, 66603544a6eSFam Zheng bool is_none_mode, BlockDriverState *base) 667893f7ebaSPaolo Bonzini { 668893f7ebaSPaolo Bonzini MirrorBlockJob *s; 669893f7ebaSPaolo Bonzini 670eee13dfeSPaolo Bonzini if (granularity == 0) { 671eee13dfeSPaolo Bonzini /* Choose the default granularity based on the target file's cluster 672eee13dfeSPaolo Bonzini * size, clamped between 4k and 64k. */ 673eee13dfeSPaolo Bonzini BlockDriverInfo bdi; 674eee13dfeSPaolo Bonzini if (bdrv_get_info(target, &bdi) >= 0 && bdi.cluster_size != 0) { 675eee13dfeSPaolo Bonzini granularity = MAX(4096, bdi.cluster_size); 676eee13dfeSPaolo Bonzini granularity = MIN(65536, granularity); 677eee13dfeSPaolo Bonzini } else { 678eee13dfeSPaolo Bonzini granularity = 65536; 679eee13dfeSPaolo Bonzini } 680eee13dfeSPaolo Bonzini } 681eee13dfeSPaolo Bonzini 682eee13dfeSPaolo Bonzini assert ((granularity & (granularity - 1)) == 0); 683eee13dfeSPaolo Bonzini 684b952b558SPaolo Bonzini if ((on_source_error == BLOCKDEV_ON_ERROR_STOP || 685b952b558SPaolo Bonzini on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) && 686b952b558SPaolo Bonzini !bdrv_iostatus_is_enabled(bs)) { 687b952b558SPaolo Bonzini error_set(errp, QERR_INVALID_PARAMETER, "on-source-error"); 688b952b558SPaolo Bonzini return; 689b952b558SPaolo Bonzini } 690b952b558SPaolo Bonzini 6915bc361b8SFam Zheng 69203544a6eSFam Zheng s = block_job_create(driver, bs, speed, cb, opaque, errp); 693893f7ebaSPaolo Bonzini if (!s) { 694893f7ebaSPaolo Bonzini return; 695893f7ebaSPaolo Bonzini } 696893f7ebaSPaolo Bonzini 69709158f00SBenoît Canet s->replaces = g_strdup(replaces); 698b952b558SPaolo Bonzini s->on_source_error = on_source_error; 699b952b558SPaolo Bonzini s->on_target_error = on_target_error; 700893f7ebaSPaolo Bonzini s->target = target; 70103544a6eSFam Zheng s->is_none_mode = is_none_mode; 7025bc361b8SFam Zheng s->base = base; 703eee13dfeSPaolo Bonzini s->granularity = granularity; 70408e4ed6cSPaolo Bonzini s->buf_size = MAX(buf_size, granularity); 705b812f671SPaolo Bonzini 706b8afb520SFam Zheng s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, errp); 707b8afb520SFam Zheng if (!s->dirty_bitmap) { 708b8afb520SFam Zheng return; 709b8afb520SFam Zheng } 710893f7ebaSPaolo Bonzini bdrv_set_enable_write_cache(s->target, true); 711b952b558SPaolo Bonzini bdrv_set_on_error(s->target, on_target_error, on_target_error); 712b952b558SPaolo Bonzini bdrv_iostatus_enable(s->target); 713893f7ebaSPaolo Bonzini s->common.co = qemu_coroutine_create(mirror_run); 714893f7ebaSPaolo Bonzini trace_mirror_start(bs, s, s->common.co, opaque); 715893f7ebaSPaolo Bonzini qemu_coroutine_enter(s->common.co, s); 716893f7ebaSPaolo Bonzini } 71703544a6eSFam Zheng 71803544a6eSFam Zheng void mirror_start(BlockDriverState *bs, BlockDriverState *target, 71909158f00SBenoît Canet const char *replaces, 72003544a6eSFam Zheng int64_t speed, int64_t granularity, int64_t buf_size, 72103544a6eSFam Zheng MirrorSyncMode mode, BlockdevOnError on_source_error, 72203544a6eSFam Zheng BlockdevOnError on_target_error, 723097310b5SMarkus Armbruster BlockCompletionFunc *cb, 72403544a6eSFam Zheng void *opaque, Error **errp) 72503544a6eSFam Zheng { 72603544a6eSFam Zheng bool is_none_mode; 72703544a6eSFam Zheng BlockDriverState *base; 72803544a6eSFam Zheng 72903544a6eSFam Zheng is_none_mode = mode == MIRROR_SYNC_MODE_NONE; 73003544a6eSFam Zheng base = mode == MIRROR_SYNC_MODE_TOP ? bs->backing_hd : NULL; 73109158f00SBenoît Canet mirror_start_job(bs, target, replaces, 73209158f00SBenoît Canet speed, granularity, buf_size, 73303544a6eSFam Zheng on_source_error, on_target_error, cb, opaque, errp, 73403544a6eSFam Zheng &mirror_job_driver, is_none_mode, base); 73503544a6eSFam Zheng } 73603544a6eSFam Zheng 73703544a6eSFam Zheng void commit_active_start(BlockDriverState *bs, BlockDriverState *base, 73803544a6eSFam Zheng int64_t speed, 73903544a6eSFam Zheng BlockdevOnError on_error, 740097310b5SMarkus Armbruster BlockCompletionFunc *cb, 74103544a6eSFam Zheng void *opaque, Error **errp) 74203544a6eSFam Zheng { 7434da83585SJeff Cody int64_t length, base_length; 7444da83585SJeff Cody int orig_base_flags; 74539a611a3SJeff Cody int ret; 746cc67f4d1SJeff Cody Error *local_err = NULL; 7474da83585SJeff Cody 7484da83585SJeff Cody orig_base_flags = bdrv_get_flags(base); 7494da83585SJeff Cody 75020a63d2cSFam Zheng if (bdrv_reopen(base, bs->open_flags, errp)) { 75120a63d2cSFam Zheng return; 75220a63d2cSFam Zheng } 7534da83585SJeff Cody 7544da83585SJeff Cody length = bdrv_getlength(bs); 7554da83585SJeff Cody if (length < 0) { 75639a611a3SJeff Cody error_setg_errno(errp, -length, 75739a611a3SJeff Cody "Unable to determine length of %s", bs->filename); 7584da83585SJeff Cody goto error_restore_flags; 7594da83585SJeff Cody } 7604da83585SJeff Cody 7614da83585SJeff Cody base_length = bdrv_getlength(base); 7624da83585SJeff Cody if (base_length < 0) { 76339a611a3SJeff Cody error_setg_errno(errp, -base_length, 76439a611a3SJeff Cody "Unable to determine length of %s", base->filename); 7654da83585SJeff Cody goto error_restore_flags; 7664da83585SJeff Cody } 7674da83585SJeff Cody 7684da83585SJeff Cody if (length > base_length) { 76939a611a3SJeff Cody ret = bdrv_truncate(base, length); 77039a611a3SJeff Cody if (ret < 0) { 77139a611a3SJeff Cody error_setg_errno(errp, -ret, 77239a611a3SJeff Cody "Top image %s is larger than base image %s, and " 7734da83585SJeff Cody "resize of base image failed", 7744da83585SJeff Cody bs->filename, base->filename); 7754da83585SJeff Cody goto error_restore_flags; 7764da83585SJeff Cody } 7774da83585SJeff Cody } 7784da83585SJeff Cody 77920a63d2cSFam Zheng bdrv_ref(base); 78009158f00SBenoît Canet mirror_start_job(bs, base, NULL, speed, 0, 0, 781cc67f4d1SJeff Cody on_error, on_error, cb, opaque, &local_err, 78203544a6eSFam Zheng &commit_active_job_driver, false, base); 7830fb6395cSMarkus Armbruster if (local_err) { 784cc67f4d1SJeff Cody error_propagate(errp, local_err); 7854da83585SJeff Cody goto error_restore_flags; 7864da83585SJeff Cody } 7874da83585SJeff Cody 7884da83585SJeff Cody return; 7894da83585SJeff Cody 7904da83585SJeff Cody error_restore_flags: 7914da83585SJeff Cody /* ignore error and errp for bdrv_reopen, because we want to propagate 7924da83585SJeff Cody * the original error */ 7934da83585SJeff Cody bdrv_reopen(base, orig_base_flags, NULL); 7944da83585SJeff Cody return; 79503544a6eSFam Zheng } 796