1893f7ebaSPaolo Bonzini /* 2893f7ebaSPaolo Bonzini * Image mirroring 3893f7ebaSPaolo Bonzini * 4893f7ebaSPaolo Bonzini * Copyright Red Hat, Inc. 2012 5893f7ebaSPaolo Bonzini * 6893f7ebaSPaolo Bonzini * Authors: 7893f7ebaSPaolo Bonzini * Paolo Bonzini <pbonzini@redhat.com> 8893f7ebaSPaolo Bonzini * 9893f7ebaSPaolo Bonzini * This work is licensed under the terms of the GNU LGPL, version 2 or later. 10893f7ebaSPaolo Bonzini * See the COPYING.LIB file in the top-level directory. 11893f7ebaSPaolo Bonzini * 12893f7ebaSPaolo Bonzini */ 13893f7ebaSPaolo Bonzini 14893f7ebaSPaolo Bonzini #include "trace.h" 15737e150eSPaolo Bonzini #include "block/blockjob.h" 16737e150eSPaolo Bonzini #include "block/block_int.h" 17cc7a8ea7SMarkus Armbruster #include "qapi/qmp/qerror.h" 18893f7ebaSPaolo Bonzini #include "qemu/ratelimit.h" 19b812f671SPaolo Bonzini #include "qemu/bitmap.h" 20893f7ebaSPaolo Bonzini 21893f7ebaSPaolo Bonzini #define SLICE_TIME 100000000ULL /* ns */ 22402a4741SPaolo Bonzini #define MAX_IN_FLIGHT 16 2348ac0a4dSWen Congyang #define DEFAULT_MIRROR_BUF_SIZE (10 << 20) 24402a4741SPaolo Bonzini 25402a4741SPaolo Bonzini /* The mirroring buffer is a list of granularity-sized chunks. 26402a4741SPaolo Bonzini * Free chunks are organized in a list. 27402a4741SPaolo Bonzini */ 28402a4741SPaolo Bonzini typedef struct MirrorBuffer { 29402a4741SPaolo Bonzini QSIMPLEQ_ENTRY(MirrorBuffer) next; 30402a4741SPaolo Bonzini } MirrorBuffer; 31893f7ebaSPaolo Bonzini 32893f7ebaSPaolo Bonzini typedef struct MirrorBlockJob { 33893f7ebaSPaolo Bonzini BlockJob common; 34893f7ebaSPaolo Bonzini RateLimit limit; 35893f7ebaSPaolo Bonzini BlockDriverState *target; 365bc361b8SFam Zheng BlockDriverState *base; 3709158f00SBenoît Canet /* The name of the graph node to replace */ 3809158f00SBenoît Canet char *replaces; 3909158f00SBenoît Canet /* The BDS to replace */ 4009158f00SBenoît Canet BlockDriverState *to_replace; 4109158f00SBenoît Canet /* Used to block operations on the drive-mirror-replace target */ 4209158f00SBenoît Canet Error *replace_blocker; 4303544a6eSFam Zheng bool is_none_mode; 44b952b558SPaolo Bonzini BlockdevOnError on_source_error, on_target_error; 45d63ffd87SPaolo Bonzini bool synced; 46d63ffd87SPaolo Bonzini bool should_complete; 47893f7ebaSPaolo Bonzini int64_t sector_num; 48eee13dfeSPaolo Bonzini int64_t granularity; 49b812f671SPaolo Bonzini size_t buf_size; 50b21c7652SMax Reitz int64_t bdev_length; 51b812f671SPaolo Bonzini unsigned long *cow_bitmap; 52e4654d2dSFam Zheng BdrvDirtyBitmap *dirty_bitmap; 538f0720ecSPaolo Bonzini HBitmapIter hbi; 54893f7ebaSPaolo Bonzini uint8_t *buf; 55402a4741SPaolo Bonzini QSIMPLEQ_HEAD(, MirrorBuffer) buf_free; 56402a4741SPaolo Bonzini int buf_free_count; 57bd48bde8SPaolo Bonzini 58402a4741SPaolo Bonzini unsigned long *in_flight_bitmap; 59bd48bde8SPaolo Bonzini int in_flight; 60b21c7652SMax Reitz int sectors_in_flight; 61bd48bde8SPaolo Bonzini int ret; 620fc9f8eaSFam Zheng bool unmap; 63*e424aff5SKevin Wolf bool waiting_for_io; 64893f7ebaSPaolo Bonzini } MirrorBlockJob; 65893f7ebaSPaolo Bonzini 66bd48bde8SPaolo Bonzini typedef struct MirrorOp { 67bd48bde8SPaolo Bonzini MirrorBlockJob *s; 68bd48bde8SPaolo Bonzini QEMUIOVector qiov; 69bd48bde8SPaolo Bonzini int64_t sector_num; 70bd48bde8SPaolo Bonzini int nb_sectors; 71bd48bde8SPaolo Bonzini } MirrorOp; 72bd48bde8SPaolo Bonzini 73b952b558SPaolo Bonzini static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read, 74b952b558SPaolo Bonzini int error) 75b952b558SPaolo Bonzini { 76b952b558SPaolo Bonzini s->synced = false; 77b952b558SPaolo Bonzini if (read) { 78b952b558SPaolo Bonzini return block_job_error_action(&s->common, s->common.bs, 79b952b558SPaolo Bonzini s->on_source_error, true, error); 80b952b558SPaolo Bonzini } else { 81b952b558SPaolo Bonzini return block_job_error_action(&s->common, s->target, 82b952b558SPaolo Bonzini s->on_target_error, false, error); 83b952b558SPaolo Bonzini } 84b952b558SPaolo Bonzini } 85b952b558SPaolo Bonzini 86bd48bde8SPaolo Bonzini static void mirror_iteration_done(MirrorOp *op, int ret) 87bd48bde8SPaolo Bonzini { 88bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 89402a4741SPaolo Bonzini struct iovec *iov; 90bd48bde8SPaolo Bonzini int64_t chunk_num; 91402a4741SPaolo Bonzini int i, nb_chunks, sectors_per_chunk; 92bd48bde8SPaolo Bonzini 93bd48bde8SPaolo Bonzini trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret); 94bd48bde8SPaolo Bonzini 95bd48bde8SPaolo Bonzini s->in_flight--; 96b21c7652SMax Reitz s->sectors_in_flight -= op->nb_sectors; 97402a4741SPaolo Bonzini iov = op->qiov.iov; 98402a4741SPaolo Bonzini for (i = 0; i < op->qiov.niov; i++) { 99402a4741SPaolo Bonzini MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base; 100402a4741SPaolo Bonzini QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next); 101402a4741SPaolo Bonzini s->buf_free_count++; 102402a4741SPaolo Bonzini } 103402a4741SPaolo Bonzini 104bd48bde8SPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 105bd48bde8SPaolo Bonzini chunk_num = op->sector_num / sectors_per_chunk; 106bd48bde8SPaolo Bonzini nb_chunks = op->nb_sectors / sectors_per_chunk; 107402a4741SPaolo Bonzini bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks); 108b21c7652SMax Reitz if (ret >= 0) { 109b21c7652SMax Reitz if (s->cow_bitmap) { 110bd48bde8SPaolo Bonzini bitmap_set(s->cow_bitmap, chunk_num, nb_chunks); 111bd48bde8SPaolo Bonzini } 112b21c7652SMax Reitz s->common.offset += (uint64_t)op->nb_sectors * BDRV_SECTOR_SIZE; 113b21c7652SMax Reitz } 114bd48bde8SPaolo Bonzini 1156df3bf8eSZhang Min qemu_iovec_destroy(&op->qiov); 116bd48bde8SPaolo Bonzini g_slice_free(MirrorOp, op); 1177b770c72SStefan Hajnoczi 118*e424aff5SKevin Wolf if (s->waiting_for_io) { 119bd48bde8SPaolo Bonzini qemu_coroutine_enter(s->common.co, NULL); 120bd48bde8SPaolo Bonzini } 1217b770c72SStefan Hajnoczi } 122bd48bde8SPaolo Bonzini 123bd48bde8SPaolo Bonzini static void mirror_write_complete(void *opaque, int ret) 124bd48bde8SPaolo Bonzini { 125bd48bde8SPaolo Bonzini MirrorOp *op = opaque; 126bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 127bd48bde8SPaolo Bonzini if (ret < 0) { 128bd48bde8SPaolo Bonzini BlockErrorAction action; 129bd48bde8SPaolo Bonzini 13020dca810SJohn Snow bdrv_set_dirty_bitmap(s->dirty_bitmap, op->sector_num, op->nb_sectors); 131bd48bde8SPaolo Bonzini action = mirror_error_action(s, false, -ret); 132a589569fSWenchao Xia if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) { 133bd48bde8SPaolo Bonzini s->ret = ret; 134bd48bde8SPaolo Bonzini } 135bd48bde8SPaolo Bonzini } 136bd48bde8SPaolo Bonzini mirror_iteration_done(op, ret); 137bd48bde8SPaolo Bonzini } 138bd48bde8SPaolo Bonzini 139bd48bde8SPaolo Bonzini static void mirror_read_complete(void *opaque, int ret) 140bd48bde8SPaolo Bonzini { 141bd48bde8SPaolo Bonzini MirrorOp *op = opaque; 142bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 143bd48bde8SPaolo Bonzini if (ret < 0) { 144bd48bde8SPaolo Bonzini BlockErrorAction action; 145bd48bde8SPaolo Bonzini 14620dca810SJohn Snow bdrv_set_dirty_bitmap(s->dirty_bitmap, op->sector_num, op->nb_sectors); 147bd48bde8SPaolo Bonzini action = mirror_error_action(s, true, -ret); 148a589569fSWenchao Xia if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) { 149bd48bde8SPaolo Bonzini s->ret = ret; 150bd48bde8SPaolo Bonzini } 151bd48bde8SPaolo Bonzini 152bd48bde8SPaolo Bonzini mirror_iteration_done(op, ret); 153bd48bde8SPaolo Bonzini return; 154bd48bde8SPaolo Bonzini } 155bd48bde8SPaolo Bonzini bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors, 156bd48bde8SPaolo Bonzini mirror_write_complete, op); 157bd48bde8SPaolo Bonzini } 158bd48bde8SPaolo Bonzini 159cc8c9d6cSPaolo Bonzini static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) 160893f7ebaSPaolo Bonzini { 161893f7ebaSPaolo Bonzini BlockDriverState *source = s->common.bs; 162402a4741SPaolo Bonzini int nb_sectors, sectors_per_chunk, nb_chunks; 163884fea4eSPaolo Bonzini int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector; 1646d0de8ebSStefan Hajnoczi uint64_t delay_ns = 0; 165bd48bde8SPaolo Bonzini MirrorOp *op; 166dcfb3bebSFam Zheng int pnum; 167dcfb3bebSFam Zheng int64_t ret; 168893f7ebaSPaolo Bonzini 1698f0720ecSPaolo Bonzini s->sector_num = hbitmap_iter_next(&s->hbi); 1708f0720ecSPaolo Bonzini if (s->sector_num < 0) { 17120dca810SJohn Snow bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi); 1728f0720ecSPaolo Bonzini s->sector_num = hbitmap_iter_next(&s->hbi); 17320dca810SJohn Snow trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap)); 1748f0720ecSPaolo Bonzini assert(s->sector_num >= 0); 1758f0720ecSPaolo Bonzini } 1768f0720ecSPaolo Bonzini 177402a4741SPaolo Bonzini hbitmap_next_sector = s->sector_num; 178b812f671SPaolo Bonzini sector_num = s->sector_num; 179884fea4eSPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 180b21c7652SMax Reitz end = s->bdev_length / BDRV_SECTOR_SIZE; 181402a4741SPaolo Bonzini 182884fea4eSPaolo Bonzini /* Extend the QEMUIOVector to include all adjacent blocks that will 183884fea4eSPaolo Bonzini * be copied in this operation. 184884fea4eSPaolo Bonzini * 185884fea4eSPaolo Bonzini * We have to do this if we have no backing file yet in the destination, 186884fea4eSPaolo Bonzini * and the cluster size is very large. Then we need to do COW ourselves. 187884fea4eSPaolo Bonzini * The first time a cluster is copied, copy it entirely. Note that, 188884fea4eSPaolo Bonzini * because both the granularity and the cluster size are powers of two, 189884fea4eSPaolo Bonzini * the number of sectors to copy cannot exceed one cluster. 190884fea4eSPaolo Bonzini * 191884fea4eSPaolo Bonzini * We also want to extend the QEMUIOVector to include more adjacent 192884fea4eSPaolo Bonzini * dirty blocks if possible, to limit the number of I/O operations and 193884fea4eSPaolo Bonzini * run efficiently even with a small granularity. 194402a4741SPaolo Bonzini */ 195884fea4eSPaolo Bonzini nb_chunks = 0; 196884fea4eSPaolo Bonzini nb_sectors = 0; 197884fea4eSPaolo Bonzini next_sector = sector_num; 198884fea4eSPaolo Bonzini next_chunk = sector_num / sectors_per_chunk; 199402a4741SPaolo Bonzini 200402a4741SPaolo Bonzini /* Wait for I/O to this cluster (from a previous iteration) to be done. */ 201884fea4eSPaolo Bonzini while (test_bit(next_chunk, s->in_flight_bitmap)) { 202402a4741SPaolo Bonzini trace_mirror_yield_in_flight(s, sector_num, s->in_flight); 203*e424aff5SKevin Wolf s->waiting_for_io = true; 204402a4741SPaolo Bonzini qemu_coroutine_yield(); 205*e424aff5SKevin Wolf s->waiting_for_io = false; 206b812f671SPaolo Bonzini } 207b812f671SPaolo Bonzini 208884fea4eSPaolo Bonzini do { 209884fea4eSPaolo Bonzini int added_sectors, added_chunks; 210884fea4eSPaolo Bonzini 211e4654d2dSFam Zheng if (!bdrv_get_dirty(source, s->dirty_bitmap, next_sector) || 212884fea4eSPaolo Bonzini test_bit(next_chunk, s->in_flight_bitmap)) { 213884fea4eSPaolo Bonzini assert(nb_sectors > 0); 214884fea4eSPaolo Bonzini break; 215884fea4eSPaolo Bonzini } 216884fea4eSPaolo Bonzini 217884fea4eSPaolo Bonzini added_sectors = sectors_per_chunk; 218884fea4eSPaolo Bonzini if (s->cow_bitmap && !test_bit(next_chunk, s->cow_bitmap)) { 219884fea4eSPaolo Bonzini bdrv_round_to_clusters(s->target, 220884fea4eSPaolo Bonzini next_sector, added_sectors, 221884fea4eSPaolo Bonzini &next_sector, &added_sectors); 222884fea4eSPaolo Bonzini 223884fea4eSPaolo Bonzini /* On the first iteration, the rounding may make us copy 224884fea4eSPaolo Bonzini * sectors before the first dirty one. 225884fea4eSPaolo Bonzini */ 226884fea4eSPaolo Bonzini if (next_sector < sector_num) { 227884fea4eSPaolo Bonzini assert(nb_sectors == 0); 228884fea4eSPaolo Bonzini sector_num = next_sector; 229884fea4eSPaolo Bonzini next_chunk = next_sector / sectors_per_chunk; 230884fea4eSPaolo Bonzini } 231884fea4eSPaolo Bonzini } 232884fea4eSPaolo Bonzini 233884fea4eSPaolo Bonzini added_sectors = MIN(added_sectors, end - (sector_num + nb_sectors)); 234884fea4eSPaolo Bonzini added_chunks = (added_sectors + sectors_per_chunk - 1) / sectors_per_chunk; 235884fea4eSPaolo Bonzini 236884fea4eSPaolo Bonzini /* When doing COW, it may happen that there is not enough space for 237884fea4eSPaolo Bonzini * a full cluster. Wait if that is the case. 238884fea4eSPaolo Bonzini */ 239884fea4eSPaolo Bonzini while (nb_chunks == 0 && s->buf_free_count < added_chunks) { 240402a4741SPaolo Bonzini trace_mirror_yield_buf_busy(s, nb_chunks, s->in_flight); 241*e424aff5SKevin Wolf s->waiting_for_io = true; 242402a4741SPaolo Bonzini qemu_coroutine_yield(); 243*e424aff5SKevin Wolf s->waiting_for_io = false; 244402a4741SPaolo Bonzini } 245884fea4eSPaolo Bonzini if (s->buf_free_count < nb_chunks + added_chunks) { 246884fea4eSPaolo Bonzini trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight); 247884fea4eSPaolo Bonzini break; 248884fea4eSPaolo Bonzini } 249cae98cb8SStefan Hajnoczi if (IOV_MAX < nb_chunks + added_chunks) { 250cae98cb8SStefan Hajnoczi trace_mirror_break_iov_max(s, nb_chunks, added_chunks); 251cae98cb8SStefan Hajnoczi break; 252cae98cb8SStefan Hajnoczi } 253402a4741SPaolo Bonzini 254402a4741SPaolo Bonzini /* We have enough free space to copy these sectors. */ 255884fea4eSPaolo Bonzini bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks); 256884fea4eSPaolo Bonzini 257884fea4eSPaolo Bonzini nb_sectors += added_sectors; 258884fea4eSPaolo Bonzini nb_chunks += added_chunks; 259884fea4eSPaolo Bonzini next_sector += added_sectors; 260884fea4eSPaolo Bonzini next_chunk += added_chunks; 261cc8c9d6cSPaolo Bonzini if (!s->synced && s->common.speed) { 262cc8c9d6cSPaolo Bonzini delay_ns = ratelimit_calculate_delay(&s->limit, added_sectors); 263cc8c9d6cSPaolo Bonzini } 264cc8c9d6cSPaolo Bonzini } while (delay_ns == 0 && next_sector < end); 265bd48bde8SPaolo Bonzini 266bd48bde8SPaolo Bonzini /* Allocate a MirrorOp that is used as an AIO callback. */ 267bd48bde8SPaolo Bonzini op = g_slice_new(MirrorOp); 268bd48bde8SPaolo Bonzini op->s = s; 269bd48bde8SPaolo Bonzini op->sector_num = sector_num; 270bd48bde8SPaolo Bonzini op->nb_sectors = nb_sectors; 271402a4741SPaolo Bonzini 272402a4741SPaolo Bonzini /* Now make a QEMUIOVector taking enough granularity-sized chunks 273402a4741SPaolo Bonzini * from s->buf_free. 274402a4741SPaolo Bonzini */ 275402a4741SPaolo Bonzini qemu_iovec_init(&op->qiov, nb_chunks); 276402a4741SPaolo Bonzini next_sector = sector_num; 277402a4741SPaolo Bonzini while (nb_chunks-- > 0) { 278402a4741SPaolo Bonzini MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free); 2795a0f6fd5SKevin Wolf size_t remaining = (nb_sectors * BDRV_SECTOR_SIZE) - op->qiov.size; 2805a0f6fd5SKevin Wolf 281402a4741SPaolo Bonzini QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next); 282402a4741SPaolo Bonzini s->buf_free_count--; 2835a0f6fd5SKevin Wolf qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining)); 284402a4741SPaolo Bonzini 285402a4741SPaolo Bonzini /* Advance the HBitmapIter in parallel, so that we do not examine 286402a4741SPaolo Bonzini * the same sector twice. 287402a4741SPaolo Bonzini */ 288e4654d2dSFam Zheng if (next_sector > hbitmap_next_sector 289e4654d2dSFam Zheng && bdrv_get_dirty(source, s->dirty_bitmap, next_sector)) { 290402a4741SPaolo Bonzini hbitmap_next_sector = hbitmap_iter_next(&s->hbi); 291402a4741SPaolo Bonzini } 292402a4741SPaolo Bonzini 293402a4741SPaolo Bonzini next_sector += sectors_per_chunk; 294402a4741SPaolo Bonzini } 295bd48bde8SPaolo Bonzini 29620dca810SJohn Snow bdrv_reset_dirty_bitmap(s->dirty_bitmap, sector_num, nb_sectors); 297893f7ebaSPaolo Bonzini 298893f7ebaSPaolo Bonzini /* Copy the dirty cluster. */ 299bd48bde8SPaolo Bonzini s->in_flight++; 300b21c7652SMax Reitz s->sectors_in_flight += nb_sectors; 301b812f671SPaolo Bonzini trace_mirror_one_iteration(s, sector_num, nb_sectors); 302dcfb3bebSFam Zheng 303dcfb3bebSFam Zheng ret = bdrv_get_block_status_above(source, NULL, sector_num, 304dcfb3bebSFam Zheng nb_sectors, &pnum); 305dcfb3bebSFam Zheng if (ret < 0 || pnum < nb_sectors || 306dcfb3bebSFam Zheng (ret & BDRV_BLOCK_DATA && !(ret & BDRV_BLOCK_ZERO))) { 307bd48bde8SPaolo Bonzini bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors, 308bd48bde8SPaolo Bonzini mirror_read_complete, op); 309dcfb3bebSFam Zheng } else if (ret & BDRV_BLOCK_ZERO) { 310dcfb3bebSFam Zheng bdrv_aio_write_zeroes(s->target, sector_num, op->nb_sectors, 311dcfb3bebSFam Zheng s->unmap ? BDRV_REQ_MAY_UNMAP : 0, 312dcfb3bebSFam Zheng mirror_write_complete, op); 313dcfb3bebSFam Zheng } else { 314dcfb3bebSFam Zheng assert(!(ret & BDRV_BLOCK_DATA)); 315dcfb3bebSFam Zheng bdrv_aio_discard(s->target, sector_num, op->nb_sectors, 316dcfb3bebSFam Zheng mirror_write_complete, op); 317dcfb3bebSFam Zheng } 318cc8c9d6cSPaolo Bonzini return delay_ns; 319893f7ebaSPaolo Bonzini } 320b952b558SPaolo Bonzini 321402a4741SPaolo Bonzini static void mirror_free_init(MirrorBlockJob *s) 322402a4741SPaolo Bonzini { 323402a4741SPaolo Bonzini int granularity = s->granularity; 324402a4741SPaolo Bonzini size_t buf_size = s->buf_size; 325402a4741SPaolo Bonzini uint8_t *buf = s->buf; 326402a4741SPaolo Bonzini 327402a4741SPaolo Bonzini assert(s->buf_free_count == 0); 328402a4741SPaolo Bonzini QSIMPLEQ_INIT(&s->buf_free); 329402a4741SPaolo Bonzini while (buf_size != 0) { 330402a4741SPaolo Bonzini MirrorBuffer *cur = (MirrorBuffer *)buf; 331402a4741SPaolo Bonzini QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next); 332402a4741SPaolo Bonzini s->buf_free_count++; 333402a4741SPaolo Bonzini buf_size -= granularity; 334402a4741SPaolo Bonzini buf += granularity; 335402a4741SPaolo Bonzini } 336402a4741SPaolo Bonzini } 337402a4741SPaolo Bonzini 338bd48bde8SPaolo Bonzini static void mirror_drain(MirrorBlockJob *s) 339bd48bde8SPaolo Bonzini { 340bd48bde8SPaolo Bonzini while (s->in_flight > 0) { 341*e424aff5SKevin Wolf s->waiting_for_io = true; 342bd48bde8SPaolo Bonzini qemu_coroutine_yield(); 343*e424aff5SKevin Wolf s->waiting_for_io = false; 344bd48bde8SPaolo Bonzini } 345893f7ebaSPaolo Bonzini } 346893f7ebaSPaolo Bonzini 3475a7e7a0bSStefan Hajnoczi typedef struct { 3485a7e7a0bSStefan Hajnoczi int ret; 3495a7e7a0bSStefan Hajnoczi } MirrorExitData; 3505a7e7a0bSStefan Hajnoczi 3515a7e7a0bSStefan Hajnoczi static void mirror_exit(BlockJob *job, void *opaque) 3525a7e7a0bSStefan Hajnoczi { 3535a7e7a0bSStefan Hajnoczi MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 3545a7e7a0bSStefan Hajnoczi MirrorExitData *data = opaque; 3555a7e7a0bSStefan Hajnoczi AioContext *replace_aio_context = NULL; 3565a7e7a0bSStefan Hajnoczi 3575a7e7a0bSStefan Hajnoczi if (s->to_replace) { 3585a7e7a0bSStefan Hajnoczi replace_aio_context = bdrv_get_aio_context(s->to_replace); 3595a7e7a0bSStefan Hajnoczi aio_context_acquire(replace_aio_context); 3605a7e7a0bSStefan Hajnoczi } 3615a7e7a0bSStefan Hajnoczi 3625a7e7a0bSStefan Hajnoczi if (s->should_complete && data->ret == 0) { 3635a7e7a0bSStefan Hajnoczi BlockDriverState *to_replace = s->common.bs; 3645a7e7a0bSStefan Hajnoczi if (s->to_replace) { 3655a7e7a0bSStefan Hajnoczi to_replace = s->to_replace; 3665a7e7a0bSStefan Hajnoczi } 3675a7e7a0bSStefan Hajnoczi if (bdrv_get_flags(s->target) != bdrv_get_flags(to_replace)) { 3685a7e7a0bSStefan Hajnoczi bdrv_reopen(s->target, bdrv_get_flags(to_replace), NULL); 3695a7e7a0bSStefan Hajnoczi } 3705a7e7a0bSStefan Hajnoczi bdrv_swap(s->target, to_replace); 3715a7e7a0bSStefan Hajnoczi if (s->common.driver->job_type == BLOCK_JOB_TYPE_COMMIT) { 3725a7e7a0bSStefan Hajnoczi /* drop the bs loop chain formed by the swap: break the loop then 3735a7e7a0bSStefan Hajnoczi * trigger the unref from the top one */ 3745a7e7a0bSStefan Hajnoczi BlockDriverState *p = s->base->backing_hd; 3755a7e7a0bSStefan Hajnoczi bdrv_set_backing_hd(s->base, NULL); 3765a7e7a0bSStefan Hajnoczi bdrv_unref(p); 3775a7e7a0bSStefan Hajnoczi } 3785a7e7a0bSStefan Hajnoczi } 3795a7e7a0bSStefan Hajnoczi if (s->to_replace) { 3805a7e7a0bSStefan Hajnoczi bdrv_op_unblock_all(s->to_replace, s->replace_blocker); 3815a7e7a0bSStefan Hajnoczi error_free(s->replace_blocker); 3825a7e7a0bSStefan Hajnoczi bdrv_unref(s->to_replace); 3835a7e7a0bSStefan Hajnoczi } 3845a7e7a0bSStefan Hajnoczi if (replace_aio_context) { 3855a7e7a0bSStefan Hajnoczi aio_context_release(replace_aio_context); 3865a7e7a0bSStefan Hajnoczi } 3875a7e7a0bSStefan Hajnoczi g_free(s->replaces); 3885a7e7a0bSStefan Hajnoczi bdrv_unref(s->target); 3895a7e7a0bSStefan Hajnoczi block_job_completed(&s->common, data->ret); 3905a7e7a0bSStefan Hajnoczi g_free(data); 3915a7e7a0bSStefan Hajnoczi } 3925a7e7a0bSStefan Hajnoczi 393893f7ebaSPaolo Bonzini static void coroutine_fn mirror_run(void *opaque) 394893f7ebaSPaolo Bonzini { 395893f7ebaSPaolo Bonzini MirrorBlockJob *s = opaque; 3965a7e7a0bSStefan Hajnoczi MirrorExitData *data; 397893f7ebaSPaolo Bonzini BlockDriverState *bs = s->common.bs; 39899900697SFam Zheng int64_t sector_num, end, length; 399bd48bde8SPaolo Bonzini uint64_t last_pause_ns; 400b812f671SPaolo Bonzini BlockDriverInfo bdi; 4011d33936eSJeff Cody char backing_filename[2]; /* we only need 2 characters because we are only 4021d33936eSJeff Cody checking for a NULL string */ 403893f7ebaSPaolo Bonzini int ret = 0; 404893f7ebaSPaolo Bonzini int n; 405893f7ebaSPaolo Bonzini 406893f7ebaSPaolo Bonzini if (block_job_is_cancelled(&s->common)) { 407893f7ebaSPaolo Bonzini goto immediate_exit; 408893f7ebaSPaolo Bonzini } 409893f7ebaSPaolo Bonzini 410b21c7652SMax Reitz s->bdev_length = bdrv_getlength(bs); 411b21c7652SMax Reitz if (s->bdev_length < 0) { 412b21c7652SMax Reitz ret = s->bdev_length; 413373df5b1SFam Zheng goto immediate_exit; 414b21c7652SMax Reitz } else if (s->bdev_length == 0) { 4159e48b025SFam Zheng /* Report BLOCK_JOB_READY and wait for complete. */ 4169e48b025SFam Zheng block_job_event_ready(&s->common); 4179e48b025SFam Zheng s->synced = true; 4189e48b025SFam Zheng while (!block_job_is_cancelled(&s->common) && !s->should_complete) { 4199e48b025SFam Zheng block_job_yield(&s->common); 4209e48b025SFam Zheng } 4219e48b025SFam Zheng s->common.cancelled = false; 4229e48b025SFam Zheng goto immediate_exit; 423893f7ebaSPaolo Bonzini } 424893f7ebaSPaolo Bonzini 425b21c7652SMax Reitz length = DIV_ROUND_UP(s->bdev_length, s->granularity); 426402a4741SPaolo Bonzini s->in_flight_bitmap = bitmap_new(length); 427402a4741SPaolo Bonzini 428b812f671SPaolo Bonzini /* If we have no backing file yet in the destination, we cannot let 429b812f671SPaolo Bonzini * the destination do COW. Instead, we copy sectors around the 430b812f671SPaolo Bonzini * dirty data if needed. We need a bitmap to do that. 431b812f671SPaolo Bonzini */ 432b812f671SPaolo Bonzini bdrv_get_backing_filename(s->target, backing_filename, 433b812f671SPaolo Bonzini sizeof(backing_filename)); 434b812f671SPaolo Bonzini if (backing_filename[0] && !s->target->backing_hd) { 435c3cc95bdSFam Zheng ret = bdrv_get_info(s->target, &bdi); 436c3cc95bdSFam Zheng if (ret < 0) { 437c3cc95bdSFam Zheng goto immediate_exit; 438c3cc95bdSFam Zheng } 439eee13dfeSPaolo Bonzini if (s->granularity < bdi.cluster_size) { 44008e4ed6cSPaolo Bonzini s->buf_size = MAX(s->buf_size, bdi.cluster_size); 441b812f671SPaolo Bonzini s->cow_bitmap = bitmap_new(length); 442b812f671SPaolo Bonzini } 443b812f671SPaolo Bonzini } 444b812f671SPaolo Bonzini 445b21c7652SMax Reitz end = s->bdev_length / BDRV_SECTOR_SIZE; 4467504edf4SKevin Wolf s->buf = qemu_try_blockalign(bs, s->buf_size); 4477504edf4SKevin Wolf if (s->buf == NULL) { 4487504edf4SKevin Wolf ret = -ENOMEM; 4497504edf4SKevin Wolf goto immediate_exit; 4507504edf4SKevin Wolf } 4517504edf4SKevin Wolf 452402a4741SPaolo Bonzini mirror_free_init(s); 453893f7ebaSPaolo Bonzini 4544c0cbd6fSFam Zheng last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 45503544a6eSFam Zheng if (!s->is_none_mode) { 456893f7ebaSPaolo Bonzini /* First part, loop on the sectors and initialize the dirty bitmap. */ 4575bc361b8SFam Zheng BlockDriverState *base = s->base; 458893f7ebaSPaolo Bonzini for (sector_num = 0; sector_num < end; ) { 45999900697SFam Zheng /* Just to make sure we are not exceeding int limit. */ 46099900697SFam Zheng int nb_sectors = MIN(INT_MAX >> BDRV_SECTOR_BITS, 46199900697SFam Zheng end - sector_num); 4624c0cbd6fSFam Zheng int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 4634c0cbd6fSFam Zheng 4644c0cbd6fSFam Zheng if (now - last_pause_ns > SLICE_TIME) { 4654c0cbd6fSFam Zheng last_pause_ns = now; 4664c0cbd6fSFam Zheng block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, 0); 4674c0cbd6fSFam Zheng } 4684c0cbd6fSFam Zheng 4694c0cbd6fSFam Zheng if (block_job_is_cancelled(&s->common)) { 4704c0cbd6fSFam Zheng goto immediate_exit; 4714c0cbd6fSFam Zheng } 4724c0cbd6fSFam Zheng 47399900697SFam Zheng ret = bdrv_is_allocated_above(bs, base, sector_num, nb_sectors, &n); 474893f7ebaSPaolo Bonzini 475893f7ebaSPaolo Bonzini if (ret < 0) { 476893f7ebaSPaolo Bonzini goto immediate_exit; 477893f7ebaSPaolo Bonzini } 478893f7ebaSPaolo Bonzini 479893f7ebaSPaolo Bonzini assert(n > 0); 480893f7ebaSPaolo Bonzini if (ret == 1) { 48120dca810SJohn Snow bdrv_set_dirty_bitmap(s->dirty_bitmap, sector_num, n); 482893f7ebaSPaolo Bonzini } 48399900697SFam Zheng sector_num += n; 484893f7ebaSPaolo Bonzini } 485893f7ebaSPaolo Bonzini } 486893f7ebaSPaolo Bonzini 48720dca810SJohn Snow bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi); 488893f7ebaSPaolo Bonzini for (;;) { 489cc8c9d6cSPaolo Bonzini uint64_t delay_ns = 0; 490893f7ebaSPaolo Bonzini int64_t cnt; 491893f7ebaSPaolo Bonzini bool should_complete; 492893f7ebaSPaolo Bonzini 493bd48bde8SPaolo Bonzini if (s->ret < 0) { 494bd48bde8SPaolo Bonzini ret = s->ret; 495893f7ebaSPaolo Bonzini goto immediate_exit; 496893f7ebaSPaolo Bonzini } 497bd48bde8SPaolo Bonzini 49820dca810SJohn Snow cnt = bdrv_get_dirty_count(s->dirty_bitmap); 499b21c7652SMax Reitz /* s->common.offset contains the number of bytes already processed so 500b21c7652SMax Reitz * far, cnt is the number of dirty sectors remaining and 501b21c7652SMax Reitz * s->sectors_in_flight is the number of sectors currently being 502b21c7652SMax Reitz * processed; together those are the current total operation length */ 503b21c7652SMax Reitz s->common.len = s->common.offset + 504b21c7652SMax Reitz (cnt + s->sectors_in_flight) * BDRV_SECTOR_SIZE; 505bd48bde8SPaolo Bonzini 506bd48bde8SPaolo Bonzini /* Note that even when no rate limit is applied we need to yield 507a7282330SFam Zheng * periodically with no pending I/O so that bdrv_drain_all() returns. 508bd48bde8SPaolo Bonzini * We do so every SLICE_TIME nanoseconds, or when there is an error, 509bd48bde8SPaolo Bonzini * or when the source is clean, whichever comes first. 510bd48bde8SPaolo Bonzini */ 511bc72ad67SAlex Bligh if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - last_pause_ns < SLICE_TIME && 512bd48bde8SPaolo Bonzini s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) { 513402a4741SPaolo Bonzini if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 || 514402a4741SPaolo Bonzini (cnt == 0 && s->in_flight > 0)) { 515402a4741SPaolo Bonzini trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt); 516*e424aff5SKevin Wolf s->waiting_for_io = true; 517bd48bde8SPaolo Bonzini qemu_coroutine_yield(); 518*e424aff5SKevin Wolf s->waiting_for_io = false; 519bd48bde8SPaolo Bonzini continue; 520bd48bde8SPaolo Bonzini } else if (cnt != 0) { 521cc8c9d6cSPaolo Bonzini delay_ns = mirror_iteration(s); 522893f7ebaSPaolo Bonzini } 523cc8c9d6cSPaolo Bonzini } 524893f7ebaSPaolo Bonzini 525893f7ebaSPaolo Bonzini should_complete = false; 526bd48bde8SPaolo Bonzini if (s->in_flight == 0 && cnt == 0) { 527893f7ebaSPaolo Bonzini trace_mirror_before_flush(s); 528893f7ebaSPaolo Bonzini ret = bdrv_flush(s->target); 529893f7ebaSPaolo Bonzini if (ret < 0) { 530a589569fSWenchao Xia if (mirror_error_action(s, false, -ret) == 531a589569fSWenchao Xia BLOCK_ERROR_ACTION_REPORT) { 532893f7ebaSPaolo Bonzini goto immediate_exit; 533893f7ebaSPaolo Bonzini } 534b952b558SPaolo Bonzini } else { 535893f7ebaSPaolo Bonzini /* We're out of the streaming phase. From now on, if the job 536893f7ebaSPaolo Bonzini * is cancelled we will actually complete all pending I/O and 537893f7ebaSPaolo Bonzini * report completion. This way, block-job-cancel will leave 538893f7ebaSPaolo Bonzini * the target in a consistent state. 539893f7ebaSPaolo Bonzini */ 540d63ffd87SPaolo Bonzini if (!s->synced) { 541bcada37bSWenchao Xia block_job_event_ready(&s->common); 542d63ffd87SPaolo Bonzini s->synced = true; 543d63ffd87SPaolo Bonzini } 544d63ffd87SPaolo Bonzini 545d63ffd87SPaolo Bonzini should_complete = s->should_complete || 546d63ffd87SPaolo Bonzini block_job_is_cancelled(&s->common); 54720dca810SJohn Snow cnt = bdrv_get_dirty_count(s->dirty_bitmap); 548893f7ebaSPaolo Bonzini } 549b952b558SPaolo Bonzini } 550893f7ebaSPaolo Bonzini 551893f7ebaSPaolo Bonzini if (cnt == 0 && should_complete) { 552893f7ebaSPaolo Bonzini /* The dirty bitmap is not updated while operations are pending. 553893f7ebaSPaolo Bonzini * If we're about to exit, wait for pending operations before 554893f7ebaSPaolo Bonzini * calling bdrv_get_dirty_count(bs), or we may exit while the 555893f7ebaSPaolo Bonzini * source has dirty data to copy! 556893f7ebaSPaolo Bonzini * 557893f7ebaSPaolo Bonzini * Note that I/O can be submitted by the guest while 558893f7ebaSPaolo Bonzini * mirror_populate runs. 559893f7ebaSPaolo Bonzini */ 560893f7ebaSPaolo Bonzini trace_mirror_before_drain(s, cnt); 5615a7e7a0bSStefan Hajnoczi bdrv_drain(bs); 56220dca810SJohn Snow cnt = bdrv_get_dirty_count(s->dirty_bitmap); 563893f7ebaSPaolo Bonzini } 564893f7ebaSPaolo Bonzini 565893f7ebaSPaolo Bonzini ret = 0; 566cc8c9d6cSPaolo Bonzini trace_mirror_before_sleep(s, cnt, s->synced, delay_ns); 567d63ffd87SPaolo Bonzini if (!s->synced) { 5687483d1e5SAlex Bligh block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 569893f7ebaSPaolo Bonzini if (block_job_is_cancelled(&s->common)) { 570893f7ebaSPaolo Bonzini break; 571893f7ebaSPaolo Bonzini } 572893f7ebaSPaolo Bonzini } else if (!should_complete) { 573bd48bde8SPaolo Bonzini delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0); 5747483d1e5SAlex Bligh block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 575893f7ebaSPaolo Bonzini } else if (cnt == 0) { 576893f7ebaSPaolo Bonzini /* The two disks are in sync. Exit and report successful 577893f7ebaSPaolo Bonzini * completion. 578893f7ebaSPaolo Bonzini */ 579893f7ebaSPaolo Bonzini assert(QLIST_EMPTY(&bs->tracked_requests)); 580893f7ebaSPaolo Bonzini s->common.cancelled = false; 581893f7ebaSPaolo Bonzini break; 582893f7ebaSPaolo Bonzini } 583bc72ad67SAlex Bligh last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 584893f7ebaSPaolo Bonzini } 585893f7ebaSPaolo Bonzini 586893f7ebaSPaolo Bonzini immediate_exit: 587bd48bde8SPaolo Bonzini if (s->in_flight > 0) { 588bd48bde8SPaolo Bonzini /* We get here only if something went wrong. Either the job failed, 589bd48bde8SPaolo Bonzini * or it was cancelled prematurely so that we do not guarantee that 590bd48bde8SPaolo Bonzini * the target is a copy of the source. 591bd48bde8SPaolo Bonzini */ 592bd48bde8SPaolo Bonzini assert(ret < 0 || (!s->synced && block_job_is_cancelled(&s->common))); 593bd48bde8SPaolo Bonzini mirror_drain(s); 594bd48bde8SPaolo Bonzini } 595bd48bde8SPaolo Bonzini 596bd48bde8SPaolo Bonzini assert(s->in_flight == 0); 5977191bf31SMarkus Armbruster qemu_vfree(s->buf); 598b812f671SPaolo Bonzini g_free(s->cow_bitmap); 599402a4741SPaolo Bonzini g_free(s->in_flight_bitmap); 600e4654d2dSFam Zheng bdrv_release_dirty_bitmap(bs, s->dirty_bitmap); 601b952b558SPaolo Bonzini bdrv_iostatus_disable(s->target); 6025a7e7a0bSStefan Hajnoczi 6035a7e7a0bSStefan Hajnoczi data = g_malloc(sizeof(*data)); 6045a7e7a0bSStefan Hajnoczi data->ret = ret; 6055a7e7a0bSStefan Hajnoczi block_job_defer_to_main_loop(&s->common, mirror_exit, data); 606893f7ebaSPaolo Bonzini } 607893f7ebaSPaolo Bonzini 608893f7ebaSPaolo Bonzini static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp) 609893f7ebaSPaolo Bonzini { 610893f7ebaSPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 611893f7ebaSPaolo Bonzini 612893f7ebaSPaolo Bonzini if (speed < 0) { 613c6bd8c70SMarkus Armbruster error_setg(errp, QERR_INVALID_PARAMETER, "speed"); 614893f7ebaSPaolo Bonzini return; 615893f7ebaSPaolo Bonzini } 616893f7ebaSPaolo Bonzini ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME); 617893f7ebaSPaolo Bonzini } 618893f7ebaSPaolo Bonzini 619b952b558SPaolo Bonzini static void mirror_iostatus_reset(BlockJob *job) 620b952b558SPaolo Bonzini { 621b952b558SPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 622b952b558SPaolo Bonzini 623b952b558SPaolo Bonzini bdrv_iostatus_reset(s->target); 624b952b558SPaolo Bonzini } 625b952b558SPaolo Bonzini 626d63ffd87SPaolo Bonzini static void mirror_complete(BlockJob *job, Error **errp) 627d63ffd87SPaolo Bonzini { 628d63ffd87SPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 62934b5d2c6SMax Reitz Error *local_err = NULL; 630d63ffd87SPaolo Bonzini int ret; 631d63ffd87SPaolo Bonzini 63234b5d2c6SMax Reitz ret = bdrv_open_backing_file(s->target, NULL, &local_err); 633d63ffd87SPaolo Bonzini if (ret < 0) { 63434b5d2c6SMax Reitz error_propagate(errp, local_err); 635d63ffd87SPaolo Bonzini return; 636d63ffd87SPaolo Bonzini } 637d63ffd87SPaolo Bonzini if (!s->synced) { 638c6bd8c70SMarkus Armbruster error_setg(errp, QERR_BLOCK_JOB_NOT_READY, 639bfb197e0SMarkus Armbruster bdrv_get_device_name(job->bs)); 640d63ffd87SPaolo Bonzini return; 641d63ffd87SPaolo Bonzini } 642d63ffd87SPaolo Bonzini 64309158f00SBenoît Canet /* check the target bs is not blocked and block all operations on it */ 64409158f00SBenoît Canet if (s->replaces) { 6455a7e7a0bSStefan Hajnoczi AioContext *replace_aio_context; 6465a7e7a0bSStefan Hajnoczi 64709158f00SBenoît Canet s->to_replace = check_to_replace_node(s->replaces, &local_err); 64809158f00SBenoît Canet if (!s->to_replace) { 64909158f00SBenoît Canet error_propagate(errp, local_err); 65009158f00SBenoît Canet return; 65109158f00SBenoît Canet } 65209158f00SBenoît Canet 6535a7e7a0bSStefan Hajnoczi replace_aio_context = bdrv_get_aio_context(s->to_replace); 6545a7e7a0bSStefan Hajnoczi aio_context_acquire(replace_aio_context); 6555a7e7a0bSStefan Hajnoczi 65609158f00SBenoît Canet error_setg(&s->replace_blocker, 65709158f00SBenoît Canet "block device is in use by block-job-complete"); 65809158f00SBenoît Canet bdrv_op_block_all(s->to_replace, s->replace_blocker); 65909158f00SBenoît Canet bdrv_ref(s->to_replace); 6605a7e7a0bSStefan Hajnoczi 6615a7e7a0bSStefan Hajnoczi aio_context_release(replace_aio_context); 66209158f00SBenoît Canet } 66309158f00SBenoît Canet 664d63ffd87SPaolo Bonzini s->should_complete = true; 665751ebd76SFam Zheng block_job_enter(&s->common); 666d63ffd87SPaolo Bonzini } 667d63ffd87SPaolo Bonzini 6683fc4b10aSFam Zheng static const BlockJobDriver mirror_job_driver = { 669893f7ebaSPaolo Bonzini .instance_size = sizeof(MirrorBlockJob), 67079e14bf7SFam Zheng .job_type = BLOCK_JOB_TYPE_MIRROR, 671893f7ebaSPaolo Bonzini .set_speed = mirror_set_speed, 672b952b558SPaolo Bonzini .iostatus_reset= mirror_iostatus_reset, 673d63ffd87SPaolo Bonzini .complete = mirror_complete, 674893f7ebaSPaolo Bonzini }; 675893f7ebaSPaolo Bonzini 67603544a6eSFam Zheng static const BlockJobDriver commit_active_job_driver = { 67703544a6eSFam Zheng .instance_size = sizeof(MirrorBlockJob), 67803544a6eSFam Zheng .job_type = BLOCK_JOB_TYPE_COMMIT, 67903544a6eSFam Zheng .set_speed = mirror_set_speed, 68003544a6eSFam Zheng .iostatus_reset 68103544a6eSFam Zheng = mirror_iostatus_reset, 68203544a6eSFam Zheng .complete = mirror_complete, 68303544a6eSFam Zheng }; 68403544a6eSFam Zheng 68503544a6eSFam Zheng static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target, 68609158f00SBenoît Canet const char *replaces, 6875fba6c0eSJohn Snow int64_t speed, uint32_t granularity, 68803544a6eSFam Zheng int64_t buf_size, 68903544a6eSFam Zheng BlockdevOnError on_source_error, 690b952b558SPaolo Bonzini BlockdevOnError on_target_error, 6910fc9f8eaSFam Zheng bool unmap, 692097310b5SMarkus Armbruster BlockCompletionFunc *cb, 69303544a6eSFam Zheng void *opaque, Error **errp, 69403544a6eSFam Zheng const BlockJobDriver *driver, 69503544a6eSFam Zheng bool is_none_mode, BlockDriverState *base) 696893f7ebaSPaolo Bonzini { 697893f7ebaSPaolo Bonzini MirrorBlockJob *s; 698893f7ebaSPaolo Bonzini 699eee13dfeSPaolo Bonzini if (granularity == 0) { 700341ebc2fSJohn Snow granularity = bdrv_get_default_bitmap_granularity(target); 701eee13dfeSPaolo Bonzini } 702eee13dfeSPaolo Bonzini 703eee13dfeSPaolo Bonzini assert ((granularity & (granularity - 1)) == 0); 704eee13dfeSPaolo Bonzini 705b952b558SPaolo Bonzini if ((on_source_error == BLOCKDEV_ON_ERROR_STOP || 706b952b558SPaolo Bonzini on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) && 707b952b558SPaolo Bonzini !bdrv_iostatus_is_enabled(bs)) { 708c6bd8c70SMarkus Armbruster error_setg(errp, QERR_INVALID_PARAMETER, "on-source-error"); 709b952b558SPaolo Bonzini return; 710b952b558SPaolo Bonzini } 711b952b558SPaolo Bonzini 71248ac0a4dSWen Congyang if (buf_size < 0) { 71348ac0a4dSWen Congyang error_setg(errp, "Invalid parameter 'buf-size'"); 71448ac0a4dSWen Congyang return; 71548ac0a4dSWen Congyang } 71648ac0a4dSWen Congyang 71748ac0a4dSWen Congyang if (buf_size == 0) { 71848ac0a4dSWen Congyang buf_size = DEFAULT_MIRROR_BUF_SIZE; 71948ac0a4dSWen Congyang } 7205bc361b8SFam Zheng 72103544a6eSFam Zheng s = block_job_create(driver, bs, speed, cb, opaque, errp); 722893f7ebaSPaolo Bonzini if (!s) { 723893f7ebaSPaolo Bonzini return; 724893f7ebaSPaolo Bonzini } 725893f7ebaSPaolo Bonzini 72609158f00SBenoît Canet s->replaces = g_strdup(replaces); 727b952b558SPaolo Bonzini s->on_source_error = on_source_error; 728b952b558SPaolo Bonzini s->on_target_error = on_target_error; 729893f7ebaSPaolo Bonzini s->target = target; 73003544a6eSFam Zheng s->is_none_mode = is_none_mode; 7315bc361b8SFam Zheng s->base = base; 732eee13dfeSPaolo Bonzini s->granularity = granularity; 73348ac0a4dSWen Congyang s->buf_size = ROUND_UP(buf_size, granularity); 7340fc9f8eaSFam Zheng s->unmap = unmap; 735b812f671SPaolo Bonzini 7360db6e54aSFam Zheng s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp); 737b8afb520SFam Zheng if (!s->dirty_bitmap) { 73897031164STing Wang g_free(s->replaces); 73997031164STing Wang block_job_release(bs); 740b8afb520SFam Zheng return; 741b8afb520SFam Zheng } 742893f7ebaSPaolo Bonzini bdrv_set_enable_write_cache(s->target, true); 743b952b558SPaolo Bonzini bdrv_set_on_error(s->target, on_target_error, on_target_error); 744b952b558SPaolo Bonzini bdrv_iostatus_enable(s->target); 745893f7ebaSPaolo Bonzini s->common.co = qemu_coroutine_create(mirror_run); 746893f7ebaSPaolo Bonzini trace_mirror_start(bs, s, s->common.co, opaque); 747893f7ebaSPaolo Bonzini qemu_coroutine_enter(s->common.co, s); 748893f7ebaSPaolo Bonzini } 74903544a6eSFam Zheng 75003544a6eSFam Zheng void mirror_start(BlockDriverState *bs, BlockDriverState *target, 75109158f00SBenoît Canet const char *replaces, 7525fba6c0eSJohn Snow int64_t speed, uint32_t granularity, int64_t buf_size, 75303544a6eSFam Zheng MirrorSyncMode mode, BlockdevOnError on_source_error, 75403544a6eSFam Zheng BlockdevOnError on_target_error, 7550fc9f8eaSFam Zheng bool unmap, 756097310b5SMarkus Armbruster BlockCompletionFunc *cb, 75703544a6eSFam Zheng void *opaque, Error **errp) 75803544a6eSFam Zheng { 75903544a6eSFam Zheng bool is_none_mode; 76003544a6eSFam Zheng BlockDriverState *base; 76103544a6eSFam Zheng 7624b80ab2bSJohn Snow if (mode == MIRROR_SYNC_MODE_INCREMENTAL) { 7634b80ab2bSJohn Snow error_setg(errp, "Sync mode 'incremental' not supported"); 764d58d8453SJohn Snow return; 765d58d8453SJohn Snow } 76603544a6eSFam Zheng is_none_mode = mode == MIRROR_SYNC_MODE_NONE; 76703544a6eSFam Zheng base = mode == MIRROR_SYNC_MODE_TOP ? bs->backing_hd : NULL; 76809158f00SBenoît Canet mirror_start_job(bs, target, replaces, 76909158f00SBenoît Canet speed, granularity, buf_size, 7700fc9f8eaSFam Zheng on_source_error, on_target_error, unmap, cb, opaque, errp, 77103544a6eSFam Zheng &mirror_job_driver, is_none_mode, base); 77203544a6eSFam Zheng } 77303544a6eSFam Zheng 77403544a6eSFam Zheng void commit_active_start(BlockDriverState *bs, BlockDriverState *base, 77503544a6eSFam Zheng int64_t speed, 77603544a6eSFam Zheng BlockdevOnError on_error, 777097310b5SMarkus Armbruster BlockCompletionFunc *cb, 77803544a6eSFam Zheng void *opaque, Error **errp) 77903544a6eSFam Zheng { 7804da83585SJeff Cody int64_t length, base_length; 7814da83585SJeff Cody int orig_base_flags; 78239a611a3SJeff Cody int ret; 783cc67f4d1SJeff Cody Error *local_err = NULL; 7844da83585SJeff Cody 7854da83585SJeff Cody orig_base_flags = bdrv_get_flags(base); 7864da83585SJeff Cody 78720a63d2cSFam Zheng if (bdrv_reopen(base, bs->open_flags, errp)) { 78820a63d2cSFam Zheng return; 78920a63d2cSFam Zheng } 7904da83585SJeff Cody 7914da83585SJeff Cody length = bdrv_getlength(bs); 7924da83585SJeff Cody if (length < 0) { 79339a611a3SJeff Cody error_setg_errno(errp, -length, 79439a611a3SJeff Cody "Unable to determine length of %s", bs->filename); 7954da83585SJeff Cody goto error_restore_flags; 7964da83585SJeff Cody } 7974da83585SJeff Cody 7984da83585SJeff Cody base_length = bdrv_getlength(base); 7994da83585SJeff Cody if (base_length < 0) { 80039a611a3SJeff Cody error_setg_errno(errp, -base_length, 80139a611a3SJeff Cody "Unable to determine length of %s", base->filename); 8024da83585SJeff Cody goto error_restore_flags; 8034da83585SJeff Cody } 8044da83585SJeff Cody 8054da83585SJeff Cody if (length > base_length) { 80639a611a3SJeff Cody ret = bdrv_truncate(base, length); 80739a611a3SJeff Cody if (ret < 0) { 80839a611a3SJeff Cody error_setg_errno(errp, -ret, 80939a611a3SJeff Cody "Top image %s is larger than base image %s, and " 8104da83585SJeff Cody "resize of base image failed", 8114da83585SJeff Cody bs->filename, base->filename); 8124da83585SJeff Cody goto error_restore_flags; 8134da83585SJeff Cody } 8144da83585SJeff Cody } 8154da83585SJeff Cody 81620a63d2cSFam Zheng bdrv_ref(base); 81709158f00SBenoît Canet mirror_start_job(bs, base, NULL, speed, 0, 0, 8180fc9f8eaSFam Zheng on_error, on_error, false, cb, opaque, &local_err, 81903544a6eSFam Zheng &commit_active_job_driver, false, base); 8200fb6395cSMarkus Armbruster if (local_err) { 821cc67f4d1SJeff Cody error_propagate(errp, local_err); 8224da83585SJeff Cody goto error_restore_flags; 8234da83585SJeff Cody } 8244da83585SJeff Cody 8254da83585SJeff Cody return; 8264da83585SJeff Cody 8274da83585SJeff Cody error_restore_flags: 8284da83585SJeff Cody /* ignore error and errp for bdrv_reopen, because we want to propagate 8294da83585SJeff Cody * the original error */ 8304da83585SJeff Cody bdrv_reopen(base, orig_base_flags, NULL); 8314da83585SJeff Cody return; 83203544a6eSFam Zheng } 833