1eb59db53SDr. David Alan Gilbert /* 2eb59db53SDr. David Alan Gilbert * Postcopy migration for RAM 3eb59db53SDr. David Alan Gilbert * 4eb59db53SDr. David Alan Gilbert * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates 5eb59db53SDr. David Alan Gilbert * 6eb59db53SDr. David Alan Gilbert * Authors: 7eb59db53SDr. David Alan Gilbert * Dave Gilbert <dgilbert@redhat.com> 8eb59db53SDr. David Alan Gilbert * 9eb59db53SDr. David Alan Gilbert * This work is licensed under the terms of the GNU GPL, version 2 or later. 10eb59db53SDr. David Alan Gilbert * See the COPYING file in the top-level directory. 11eb59db53SDr. David Alan Gilbert * 12eb59db53SDr. David Alan Gilbert */ 13eb59db53SDr. David Alan Gilbert 14eb59db53SDr. David Alan Gilbert /* 15eb59db53SDr. David Alan Gilbert * Postcopy is a migration technique where the execution flips from the 16eb59db53SDr. David Alan Gilbert * source to the destination before all the data has been copied. 17eb59db53SDr. David Alan Gilbert */ 18eb59db53SDr. David Alan Gilbert 191393a485SPeter Maydell #include "qemu/osdep.h" 20b85ea5faSPeter Maydell #include "qemu/madvise.h" 2151180423SJuan Quintela #include "exec/target_page.h" 226666c96aSJuan Quintela #include "migration.h" 2308a0aee1SJuan Quintela #include "qemu-file.h" 2420a519a0SJuan Quintela #include "savevm.h" 25be07b0acSJuan Quintela #include "postcopy-ram.h" 267b1e1a22SJuan Quintela #include "ram.h" 271693c64cSDr. David Alan Gilbert #include "qapi/error.h" 281693c64cSDr. David Alan Gilbert #include "qemu/notify.h" 29d4842052SMarkus Armbruster #include "qemu/rcu.h" 30eb59db53SDr. David Alan Gilbert #include "sysemu/sysemu.h" 31eb59db53SDr. David Alan Gilbert #include "qemu/error-report.h" 32eb59db53SDr. David Alan Gilbert #include "trace.h" 335cc8767dSLike Xu #include "hw/boards.h" 34898ba906SDavid Hildenbrand #include "exec/ramblock.h" 3536f62f11SPeter Xu #include "socket.h" 3636f62f11SPeter Xu #include "yank_functions.h" 37f0afaf6cSPeter Xu #include "tls.h" 38d5890ea0SPeter Xu #include "qemu/userfaultfd.h" 39eb59db53SDr. David Alan Gilbert 40e0b266f0SDr. David Alan Gilbert /* Arbitrary limit on size of each discard command, 41e0b266f0SDr. David Alan Gilbert * keeps them around ~200 bytes 42e0b266f0SDr. David Alan Gilbert */ 43e0b266f0SDr. David Alan Gilbert #define MAX_DISCARDS_PER_COMMAND 12 44e0b266f0SDr. David Alan Gilbert 45e0b266f0SDr. David Alan Gilbert struct PostcopyDiscardState { 46e0b266f0SDr. David Alan Gilbert const char *ramblock_name; 47e0b266f0SDr. David Alan Gilbert uint16_t cur_entry; 48e0b266f0SDr. David Alan Gilbert /* 49e0b266f0SDr. David Alan Gilbert * Start and length of a discard range (bytes) 50e0b266f0SDr. David Alan Gilbert */ 51e0b266f0SDr. David Alan Gilbert uint64_t start_list[MAX_DISCARDS_PER_COMMAND]; 52e0b266f0SDr. David Alan Gilbert uint64_t length_list[MAX_DISCARDS_PER_COMMAND]; 53e0b266f0SDr. David Alan Gilbert unsigned int nsentwords; 54e0b266f0SDr. David Alan Gilbert unsigned int nsentcmds; 55e0b266f0SDr. David Alan Gilbert }; 56e0b266f0SDr. David Alan Gilbert 571693c64cSDr. David Alan Gilbert static NotifierWithReturnList postcopy_notifier_list; 581693c64cSDr. David Alan Gilbert 591693c64cSDr. David Alan Gilbert void postcopy_infrastructure_init(void) 601693c64cSDr. David Alan Gilbert { 611693c64cSDr. David Alan Gilbert notifier_with_return_list_init(&postcopy_notifier_list); 621693c64cSDr. David Alan Gilbert } 631693c64cSDr. David Alan Gilbert 641693c64cSDr. David Alan Gilbert void postcopy_add_notifier(NotifierWithReturn *nn) 651693c64cSDr. David Alan Gilbert { 661693c64cSDr. David Alan Gilbert notifier_with_return_list_add(&postcopy_notifier_list, nn); 671693c64cSDr. David Alan Gilbert } 681693c64cSDr. David Alan Gilbert 691693c64cSDr. David Alan Gilbert void postcopy_remove_notifier(NotifierWithReturn *n) 701693c64cSDr. David Alan Gilbert { 711693c64cSDr. David Alan Gilbert notifier_with_return_remove(n); 721693c64cSDr. David Alan Gilbert } 731693c64cSDr. David Alan Gilbert 741693c64cSDr. David Alan Gilbert int postcopy_notify(enum PostcopyNotifyReason reason, Error **errp) 751693c64cSDr. David Alan Gilbert { 761693c64cSDr. David Alan Gilbert struct PostcopyNotifyData pnd; 771693c64cSDr. David Alan Gilbert pnd.reason = reason; 781693c64cSDr. David Alan Gilbert pnd.errp = errp; 791693c64cSDr. David Alan Gilbert 801693c64cSDr. David Alan Gilbert return notifier_with_return_list_notify(&postcopy_notifier_list, 811693c64cSDr. David Alan Gilbert &pnd); 821693c64cSDr. David Alan Gilbert } 831693c64cSDr. David Alan Gilbert 84095c12a4SPeter Xu /* 85095c12a4SPeter Xu * NOTE: this routine is not thread safe, we can't call it concurrently. But it 86095c12a4SPeter Xu * should be good enough for migration's purposes. 87095c12a4SPeter Xu */ 88095c12a4SPeter Xu void postcopy_thread_create(MigrationIncomingState *mis, 89095c12a4SPeter Xu QemuThread *thread, const char *name, 90095c12a4SPeter Xu void *(*fn)(void *), int joinable) 91095c12a4SPeter Xu { 92095c12a4SPeter Xu qemu_sem_init(&mis->thread_sync_sem, 0); 93095c12a4SPeter Xu qemu_thread_create(thread, name, fn, mis, joinable); 94095c12a4SPeter Xu qemu_sem_wait(&mis->thread_sync_sem); 95095c12a4SPeter Xu qemu_sem_destroy(&mis->thread_sync_sem); 96095c12a4SPeter Xu } 97095c12a4SPeter Xu 98eb59db53SDr. David Alan Gilbert /* Postcopy needs to detect accesses to pages that haven't yet been copied 99eb59db53SDr. David Alan Gilbert * across, and efficiently map new pages in, the techniques for doing this 100eb59db53SDr. David Alan Gilbert * are target OS specific. 101eb59db53SDr. David Alan Gilbert */ 102eb59db53SDr. David Alan Gilbert #if defined(__linux__) 103eb59db53SDr. David Alan Gilbert 104c4faeed2SDr. David Alan Gilbert #include <poll.h> 105eb59db53SDr. David Alan Gilbert #include <sys/ioctl.h> 106eb59db53SDr. David Alan Gilbert #include <sys/syscall.h> 107eb59db53SDr. David Alan Gilbert #include <asm/types.h> /* for __u64 */ 108eb59db53SDr. David Alan Gilbert #endif 109eb59db53SDr. David Alan Gilbert 110d8b9d771SMatthew Fortune #if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD) 111d8b9d771SMatthew Fortune #include <sys/eventfd.h> 112eb59db53SDr. David Alan Gilbert #include <linux/userfaultfd.h> 113eb59db53SDr. David Alan Gilbert 1142a4c42f1SAlexey Perevalov typedef struct PostcopyBlocktimeContext { 1152a4c42f1SAlexey Perevalov /* time when page fault initiated per vCPU */ 1162a4c42f1SAlexey Perevalov uint32_t *page_fault_vcpu_time; 1172a4c42f1SAlexey Perevalov /* page address per vCPU */ 1182a4c42f1SAlexey Perevalov uintptr_t *vcpu_addr; 1192a4c42f1SAlexey Perevalov uint32_t total_blocktime; 1202a4c42f1SAlexey Perevalov /* blocktime per vCPU */ 1212a4c42f1SAlexey Perevalov uint32_t *vcpu_blocktime; 1222a4c42f1SAlexey Perevalov /* point in time when last page fault was initiated */ 1232a4c42f1SAlexey Perevalov uint32_t last_begin; 1242a4c42f1SAlexey Perevalov /* number of vCPU are suspended */ 1252a4c42f1SAlexey Perevalov int smp_cpus_down; 1262a4c42f1SAlexey Perevalov uint64_t start_time; 1272a4c42f1SAlexey Perevalov 1282a4c42f1SAlexey Perevalov /* 1292a4c42f1SAlexey Perevalov * Handler for exit event, necessary for 1302a4c42f1SAlexey Perevalov * releasing whole blocktime_ctx 1312a4c42f1SAlexey Perevalov */ 1322a4c42f1SAlexey Perevalov Notifier exit_notifier; 1332a4c42f1SAlexey Perevalov } PostcopyBlocktimeContext; 1342a4c42f1SAlexey Perevalov 1352a4c42f1SAlexey Perevalov static void destroy_blocktime_context(struct PostcopyBlocktimeContext *ctx) 1362a4c42f1SAlexey Perevalov { 1372a4c42f1SAlexey Perevalov g_free(ctx->page_fault_vcpu_time); 1382a4c42f1SAlexey Perevalov g_free(ctx->vcpu_addr); 1392a4c42f1SAlexey Perevalov g_free(ctx->vcpu_blocktime); 1402a4c42f1SAlexey Perevalov g_free(ctx); 1412a4c42f1SAlexey Perevalov } 1422a4c42f1SAlexey Perevalov 1432a4c42f1SAlexey Perevalov static void migration_exit_cb(Notifier *n, void *data) 1442a4c42f1SAlexey Perevalov { 1452a4c42f1SAlexey Perevalov PostcopyBlocktimeContext *ctx = container_of(n, PostcopyBlocktimeContext, 1462a4c42f1SAlexey Perevalov exit_notifier); 1472a4c42f1SAlexey Perevalov destroy_blocktime_context(ctx); 1482a4c42f1SAlexey Perevalov } 1492a4c42f1SAlexey Perevalov 1502a4c42f1SAlexey Perevalov static struct PostcopyBlocktimeContext *blocktime_context_new(void) 1512a4c42f1SAlexey Perevalov { 1525cc8767dSLike Xu MachineState *ms = MACHINE(qdev_get_machine()); 1535cc8767dSLike Xu unsigned int smp_cpus = ms->smp.cpus; 1542a4c42f1SAlexey Perevalov PostcopyBlocktimeContext *ctx = g_new0(PostcopyBlocktimeContext, 1); 1552a4c42f1SAlexey Perevalov ctx->page_fault_vcpu_time = g_new0(uint32_t, smp_cpus); 1562a4c42f1SAlexey Perevalov ctx->vcpu_addr = g_new0(uintptr_t, smp_cpus); 1572a4c42f1SAlexey Perevalov ctx->vcpu_blocktime = g_new0(uint32_t, smp_cpus); 1582a4c42f1SAlexey Perevalov 1592a4c42f1SAlexey Perevalov ctx->exit_notifier.notify = migration_exit_cb; 1602a4c42f1SAlexey Perevalov ctx->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 1612a4c42f1SAlexey Perevalov qemu_add_exit_notifier(&ctx->exit_notifier); 1622a4c42f1SAlexey Perevalov return ctx; 1632a4c42f1SAlexey Perevalov } 164ca6011c2SAlexey Perevalov 16565ace060SAlexey Perevalov static uint32List *get_vcpu_blocktime_list(PostcopyBlocktimeContext *ctx) 16665ace060SAlexey Perevalov { 1675cc8767dSLike Xu MachineState *ms = MACHINE(qdev_get_machine()); 16854aa3de7SEric Blake uint32List *list = NULL; 16965ace060SAlexey Perevalov int i; 17065ace060SAlexey Perevalov 1715cc8767dSLike Xu for (i = ms->smp.cpus - 1; i >= 0; i--) { 17254aa3de7SEric Blake QAPI_LIST_PREPEND(list, ctx->vcpu_blocktime[i]); 17365ace060SAlexey Perevalov } 17465ace060SAlexey Perevalov 17565ace060SAlexey Perevalov return list; 17665ace060SAlexey Perevalov } 17765ace060SAlexey Perevalov 17865ace060SAlexey Perevalov /* 17965ace060SAlexey Perevalov * This function just populates MigrationInfo from postcopy's 18065ace060SAlexey Perevalov * blocktime context. It will not populate MigrationInfo, 18165ace060SAlexey Perevalov * unless postcopy-blocktime capability was set. 18265ace060SAlexey Perevalov * 18365ace060SAlexey Perevalov * @info: pointer to MigrationInfo to populate 18465ace060SAlexey Perevalov */ 18565ace060SAlexey Perevalov void fill_destination_postcopy_migration_info(MigrationInfo *info) 18665ace060SAlexey Perevalov { 18765ace060SAlexey Perevalov MigrationIncomingState *mis = migration_incoming_get_current(); 18865ace060SAlexey Perevalov PostcopyBlocktimeContext *bc = mis->blocktime_ctx; 18965ace060SAlexey Perevalov 19065ace060SAlexey Perevalov if (!bc) { 19165ace060SAlexey Perevalov return; 19265ace060SAlexey Perevalov } 19365ace060SAlexey Perevalov 19465ace060SAlexey Perevalov info->has_postcopy_blocktime = true; 19565ace060SAlexey Perevalov info->postcopy_blocktime = bc->total_blocktime; 19665ace060SAlexey Perevalov info->has_postcopy_vcpu_blocktime = true; 19765ace060SAlexey Perevalov info->postcopy_vcpu_blocktime = get_vcpu_blocktime_list(bc); 19865ace060SAlexey Perevalov } 19965ace060SAlexey Perevalov 20065ace060SAlexey Perevalov static uint32_t get_postcopy_total_blocktime(void) 20165ace060SAlexey Perevalov { 20265ace060SAlexey Perevalov MigrationIncomingState *mis = migration_incoming_get_current(); 20365ace060SAlexey Perevalov PostcopyBlocktimeContext *bc = mis->blocktime_ctx; 20465ace060SAlexey Perevalov 20565ace060SAlexey Perevalov if (!bc) { 20665ace060SAlexey Perevalov return 0; 20765ace060SAlexey Perevalov } 20865ace060SAlexey Perevalov 20965ace060SAlexey Perevalov return bc->total_blocktime; 21065ace060SAlexey Perevalov } 21165ace060SAlexey Perevalov 21254ae0886SAlexey Perevalov /** 21354ae0886SAlexey Perevalov * receive_ufd_features: check userfault fd features, to request only supported 21454ae0886SAlexey Perevalov * features in the future. 21554ae0886SAlexey Perevalov * 21654ae0886SAlexey Perevalov * Returns: true on success 21754ae0886SAlexey Perevalov * 21854ae0886SAlexey Perevalov * __NR_userfaultfd - should be checked before 21954ae0886SAlexey Perevalov * @features: out parameter will contain uffdio_api.features provided by kernel 22054ae0886SAlexey Perevalov * in case of success 22154ae0886SAlexey Perevalov */ 22254ae0886SAlexey Perevalov static bool receive_ufd_features(uint64_t *features) 22354ae0886SAlexey Perevalov { 22454ae0886SAlexey Perevalov struct uffdio_api api_struct = {0}; 22554ae0886SAlexey Perevalov int ufd; 22654ae0886SAlexey Perevalov bool ret = true; 22754ae0886SAlexey Perevalov 228d5890ea0SPeter Xu ufd = uffd_open(O_CLOEXEC); 22954ae0886SAlexey Perevalov if (ufd == -1) { 230d5890ea0SPeter Xu error_report("%s: uffd_open() failed: %s", __func__, strerror(errno)); 23154ae0886SAlexey Perevalov return false; 23254ae0886SAlexey Perevalov } 23354ae0886SAlexey Perevalov 23454ae0886SAlexey Perevalov /* ask features */ 235eb59db53SDr. David Alan Gilbert api_struct.api = UFFD_API; 236eb59db53SDr. David Alan Gilbert api_struct.features = 0; 237eb59db53SDr. David Alan Gilbert if (ioctl(ufd, UFFDIO_API, &api_struct)) { 2385553499fSAlexey Perevalov error_report("%s: UFFDIO_API failed: %s", __func__, 239eb59db53SDr. David Alan Gilbert strerror(errno)); 24054ae0886SAlexey Perevalov ret = false; 24154ae0886SAlexey Perevalov goto release_ufd; 24254ae0886SAlexey Perevalov } 24354ae0886SAlexey Perevalov 24454ae0886SAlexey Perevalov *features = api_struct.features; 24554ae0886SAlexey Perevalov 24654ae0886SAlexey Perevalov release_ufd: 24754ae0886SAlexey Perevalov close(ufd); 24854ae0886SAlexey Perevalov return ret; 24954ae0886SAlexey Perevalov } 25054ae0886SAlexey Perevalov 25154ae0886SAlexey Perevalov /** 25254ae0886SAlexey Perevalov * request_ufd_features: this function should be called only once on a newly 25354ae0886SAlexey Perevalov * opened ufd, subsequent calls will lead to error. 25454ae0886SAlexey Perevalov * 2553a4452d8Szhaolichang * Returns: true on success 25654ae0886SAlexey Perevalov * 25754ae0886SAlexey Perevalov * @ufd: fd obtained from userfaultfd syscall 25854ae0886SAlexey Perevalov * @features: bit mask see UFFD_API_FEATURES 25954ae0886SAlexey Perevalov */ 26054ae0886SAlexey Perevalov static bool request_ufd_features(int ufd, uint64_t features) 26154ae0886SAlexey Perevalov { 26254ae0886SAlexey Perevalov struct uffdio_api api_struct = {0}; 26354ae0886SAlexey Perevalov uint64_t ioctl_mask; 26454ae0886SAlexey Perevalov 26554ae0886SAlexey Perevalov api_struct.api = UFFD_API; 26654ae0886SAlexey Perevalov api_struct.features = features; 26754ae0886SAlexey Perevalov if (ioctl(ufd, UFFDIO_API, &api_struct)) { 26854ae0886SAlexey Perevalov error_report("%s failed: UFFDIO_API failed: %s", __func__, 26954ae0886SAlexey Perevalov strerror(errno)); 270eb59db53SDr. David Alan Gilbert return false; 271eb59db53SDr. David Alan Gilbert } 272eb59db53SDr. David Alan Gilbert 273eb59db53SDr. David Alan Gilbert ioctl_mask = (__u64)1 << _UFFDIO_REGISTER | 274eb59db53SDr. David Alan Gilbert (__u64)1 << _UFFDIO_UNREGISTER; 275eb59db53SDr. David Alan Gilbert if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) { 276eb59db53SDr. David Alan Gilbert error_report("Missing userfault features: %" PRIx64, 277eb59db53SDr. David Alan Gilbert (uint64_t)(~api_struct.ioctls & ioctl_mask)); 278eb59db53SDr. David Alan Gilbert return false; 279eb59db53SDr. David Alan Gilbert } 280eb59db53SDr. David Alan Gilbert 28154ae0886SAlexey Perevalov return true; 28254ae0886SAlexey Perevalov } 28354ae0886SAlexey Perevalov 28454ae0886SAlexey Perevalov static bool ufd_check_and_apply(int ufd, MigrationIncomingState *mis) 28554ae0886SAlexey Perevalov { 28654ae0886SAlexey Perevalov uint64_t asked_features = 0; 28754ae0886SAlexey Perevalov static uint64_t supported_features; 28854ae0886SAlexey Perevalov 28954ae0886SAlexey Perevalov /* 29054ae0886SAlexey Perevalov * it's not possible to 29154ae0886SAlexey Perevalov * request UFFD_API twice per one fd 29254ae0886SAlexey Perevalov * userfault fd features is persistent 29354ae0886SAlexey Perevalov */ 29454ae0886SAlexey Perevalov if (!supported_features) { 29554ae0886SAlexey Perevalov if (!receive_ufd_features(&supported_features)) { 29654ae0886SAlexey Perevalov error_report("%s failed", __func__); 29754ae0886SAlexey Perevalov return false; 29854ae0886SAlexey Perevalov } 29954ae0886SAlexey Perevalov } 30054ae0886SAlexey Perevalov 3012a4c42f1SAlexey Perevalov #ifdef UFFD_FEATURE_THREAD_ID 3022d1c37c6SPeter Xu if (UFFD_FEATURE_THREAD_ID & supported_features) { 3032d1c37c6SPeter Xu asked_features |= UFFD_FEATURE_THREAD_ID; 3042d1c37c6SPeter Xu if (migrate_postcopy_blocktime()) { 3052a4c42f1SAlexey Perevalov if (!mis->blocktime_ctx) { 3062a4c42f1SAlexey Perevalov mis->blocktime_ctx = blocktime_context_new(); 3072a4c42f1SAlexey Perevalov } 3082d1c37c6SPeter Xu } 3092a4c42f1SAlexey Perevalov } 3102a4c42f1SAlexey Perevalov #endif 3112a4c42f1SAlexey Perevalov 31254ae0886SAlexey Perevalov /* 31354ae0886SAlexey Perevalov * request features, even if asked_features is 0, due to 31454ae0886SAlexey Perevalov * kernel expects UFFD_API before UFFDIO_REGISTER, per 31554ae0886SAlexey Perevalov * userfault file descriptor 31654ae0886SAlexey Perevalov */ 31754ae0886SAlexey Perevalov if (!request_ufd_features(ufd, asked_features)) { 31854ae0886SAlexey Perevalov error_report("%s failed: features %" PRIu64, __func__, 31954ae0886SAlexey Perevalov asked_features); 32054ae0886SAlexey Perevalov return false; 32154ae0886SAlexey Perevalov } 32254ae0886SAlexey Perevalov 3238e3b0cbbSMarc-André Lureau if (qemu_real_host_page_size() != ram_pagesize_summary()) { 3247e8cafb7SDr. David Alan Gilbert bool have_hp = false; 3257e8cafb7SDr. David Alan Gilbert /* We've got a huge page */ 3267e8cafb7SDr. David Alan Gilbert #ifdef UFFD_FEATURE_MISSING_HUGETLBFS 32754ae0886SAlexey Perevalov have_hp = supported_features & UFFD_FEATURE_MISSING_HUGETLBFS; 3287e8cafb7SDr. David Alan Gilbert #endif 3297e8cafb7SDr. David Alan Gilbert if (!have_hp) { 3307e8cafb7SDr. David Alan Gilbert error_report("Userfault on this host does not support huge pages"); 3317e8cafb7SDr. David Alan Gilbert return false; 3327e8cafb7SDr. David Alan Gilbert } 3337e8cafb7SDr. David Alan Gilbert } 334eb59db53SDr. David Alan Gilbert return true; 335eb59db53SDr. David Alan Gilbert } 336eb59db53SDr. David Alan Gilbert 3378679638bSDr. David Alan Gilbert /* Callback from postcopy_ram_supported_by_host block iterator. 3388679638bSDr. David Alan Gilbert */ 339754cb9c0SYury Kotov static int test_ramblock_postcopiable(RAMBlock *rb, void *opaque) 3408679638bSDr. David Alan Gilbert { 341754cb9c0SYury Kotov const char *block_name = qemu_ram_get_idstr(rb); 342754cb9c0SYury Kotov ram_addr_t length = qemu_ram_get_used_length(rb); 3435d214a92SDr. David Alan Gilbert size_t pagesize = qemu_ram_pagesize(rb); 3445d214a92SDr. David Alan Gilbert 3455d214a92SDr. David Alan Gilbert if (length % pagesize) { 3465d214a92SDr. David Alan Gilbert error_report("Postcopy requires RAM blocks to be a page size multiple," 3475d214a92SDr. David Alan Gilbert " block %s is 0x" RAM_ADDR_FMT " bytes with a " 3485d214a92SDr. David Alan Gilbert "page size of 0x%zx", block_name, length, pagesize); 3495d214a92SDr. David Alan Gilbert return 1; 3505d214a92SDr. David Alan Gilbert } 3518679638bSDr. David Alan Gilbert return 0; 3528679638bSDr. David Alan Gilbert } 3538679638bSDr. David Alan Gilbert 35458b7c17eSDr. David Alan Gilbert /* 35558b7c17eSDr. David Alan Gilbert * Note: This has the side effect of munlock'ing all of RAM, that's 35658b7c17eSDr. David Alan Gilbert * normally fine since if the postcopy succeeds it gets turned back on at the 35758b7c17eSDr. David Alan Gilbert * end. 35858b7c17eSDr. David Alan Gilbert */ 359d7651f15SAlexey Perevalov bool postcopy_ram_supported_by_host(MigrationIncomingState *mis) 360eb59db53SDr. David Alan Gilbert { 3618e3b0cbbSMarc-André Lureau long pagesize = qemu_real_host_page_size(); 362eb59db53SDr. David Alan Gilbert int ufd = -1; 363eb59db53SDr. David Alan Gilbert bool ret = false; /* Error unless we change it */ 364eb59db53SDr. David Alan Gilbert void *testarea = NULL; 365eb59db53SDr. David Alan Gilbert struct uffdio_register reg_struct; 366eb59db53SDr. David Alan Gilbert struct uffdio_range range_struct; 367eb59db53SDr. David Alan Gilbert uint64_t feature_mask; 3681693c64cSDr. David Alan Gilbert Error *local_err = NULL; 369eb59db53SDr. David Alan Gilbert 37020afaed9SJuan Quintela if (qemu_target_page_size() > pagesize) { 371eb59db53SDr. David Alan Gilbert error_report("Target page size bigger than host page size"); 372eb59db53SDr. David Alan Gilbert goto out; 373eb59db53SDr. David Alan Gilbert } 374eb59db53SDr. David Alan Gilbert 375d5890ea0SPeter Xu ufd = uffd_open(O_CLOEXEC); 376eb59db53SDr. David Alan Gilbert if (ufd == -1) { 377eb59db53SDr. David Alan Gilbert error_report("%s: userfaultfd not available: %s", __func__, 378eb59db53SDr. David Alan Gilbert strerror(errno)); 379eb59db53SDr. David Alan Gilbert goto out; 380eb59db53SDr. David Alan Gilbert } 381eb59db53SDr. David Alan Gilbert 3821693c64cSDr. David Alan Gilbert /* Give devices a chance to object */ 3831693c64cSDr. David Alan Gilbert if (postcopy_notify(POSTCOPY_NOTIFY_PROBE, &local_err)) { 3841693c64cSDr. David Alan Gilbert error_report_err(local_err); 3851693c64cSDr. David Alan Gilbert goto out; 3861693c64cSDr. David Alan Gilbert } 3871693c64cSDr. David Alan Gilbert 388eb59db53SDr. David Alan Gilbert /* Version and features check */ 38954ae0886SAlexey Perevalov if (!ufd_check_and_apply(ufd, mis)) { 390eb59db53SDr. David Alan Gilbert goto out; 391eb59db53SDr. David Alan Gilbert } 392eb59db53SDr. David Alan Gilbert 3938679638bSDr. David Alan Gilbert /* We don't support postcopy with shared RAM yet */ 394fbd162e6SYury Kotov if (foreach_not_ignored_block(test_ramblock_postcopiable, NULL)) { 3958679638bSDr. David Alan Gilbert goto out; 3968679638bSDr. David Alan Gilbert } 3978679638bSDr. David Alan Gilbert 398eb59db53SDr. David Alan Gilbert /* 39958b7c17eSDr. David Alan Gilbert * userfault and mlock don't go together; we'll put it back later if 40058b7c17eSDr. David Alan Gilbert * it was enabled. 40158b7c17eSDr. David Alan Gilbert */ 40258b7c17eSDr. David Alan Gilbert if (munlockall()) { 40358b7c17eSDr. David Alan Gilbert error_report("%s: munlockall: %s", __func__, strerror(errno)); 404617a32f5SDr. David Alan Gilbert goto out; 40558b7c17eSDr. David Alan Gilbert } 40658b7c17eSDr. David Alan Gilbert 40758b7c17eSDr. David Alan Gilbert /* 408eb59db53SDr. David Alan Gilbert * We need to check that the ops we need are supported on anon memory 409eb59db53SDr. David Alan Gilbert * To do that we need to register a chunk and see the flags that 410eb59db53SDr. David Alan Gilbert * are returned. 411eb59db53SDr. David Alan Gilbert */ 412eb59db53SDr. David Alan Gilbert testarea = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE | 413eb59db53SDr. David Alan Gilbert MAP_ANONYMOUS, -1, 0); 414eb59db53SDr. David Alan Gilbert if (testarea == MAP_FAILED) { 415eb59db53SDr. David Alan Gilbert error_report("%s: Failed to map test area: %s", __func__, 416eb59db53SDr. David Alan Gilbert strerror(errno)); 417eb59db53SDr. David Alan Gilbert goto out; 418eb59db53SDr. David Alan Gilbert } 4197648297dSDavid Hildenbrand g_assert(QEMU_PTR_IS_ALIGNED(testarea, pagesize)); 420eb59db53SDr. David Alan Gilbert 421eb59db53SDr. David Alan Gilbert reg_struct.range.start = (uintptr_t)testarea; 422eb59db53SDr. David Alan Gilbert reg_struct.range.len = pagesize; 423eb59db53SDr. David Alan Gilbert reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; 424eb59db53SDr. David Alan Gilbert 425eb59db53SDr. David Alan Gilbert if (ioctl(ufd, UFFDIO_REGISTER, ®_struct)) { 426eb59db53SDr. David Alan Gilbert error_report("%s userfault register: %s", __func__, strerror(errno)); 427eb59db53SDr. David Alan Gilbert goto out; 428eb59db53SDr. David Alan Gilbert } 429eb59db53SDr. David Alan Gilbert 430eb59db53SDr. David Alan Gilbert range_struct.start = (uintptr_t)testarea; 431eb59db53SDr. David Alan Gilbert range_struct.len = pagesize; 432eb59db53SDr. David Alan Gilbert if (ioctl(ufd, UFFDIO_UNREGISTER, &range_struct)) { 433eb59db53SDr. David Alan Gilbert error_report("%s userfault unregister: %s", __func__, strerror(errno)); 434eb59db53SDr. David Alan Gilbert goto out; 435eb59db53SDr. David Alan Gilbert } 436eb59db53SDr. David Alan Gilbert 437eb59db53SDr. David Alan Gilbert feature_mask = (__u64)1 << _UFFDIO_WAKE | 438eb59db53SDr. David Alan Gilbert (__u64)1 << _UFFDIO_COPY | 439eb59db53SDr. David Alan Gilbert (__u64)1 << _UFFDIO_ZEROPAGE; 440eb59db53SDr. David Alan Gilbert if ((reg_struct.ioctls & feature_mask) != feature_mask) { 441eb59db53SDr. David Alan Gilbert error_report("Missing userfault map features: %" PRIx64, 442eb59db53SDr. David Alan Gilbert (uint64_t)(~reg_struct.ioctls & feature_mask)); 443eb59db53SDr. David Alan Gilbert goto out; 444eb59db53SDr. David Alan Gilbert } 445eb59db53SDr. David Alan Gilbert 446eb59db53SDr. David Alan Gilbert /* Success! */ 447eb59db53SDr. David Alan Gilbert ret = true; 448eb59db53SDr. David Alan Gilbert out: 449eb59db53SDr. David Alan Gilbert if (testarea) { 450eb59db53SDr. David Alan Gilbert munmap(testarea, pagesize); 451eb59db53SDr. David Alan Gilbert } 452eb59db53SDr. David Alan Gilbert if (ufd != -1) { 453eb59db53SDr. David Alan Gilbert close(ufd); 454eb59db53SDr. David Alan Gilbert } 455eb59db53SDr. David Alan Gilbert return ret; 456eb59db53SDr. David Alan Gilbert } 457eb59db53SDr. David Alan Gilbert 4581caddf8aSDr. David Alan Gilbert /* 4591caddf8aSDr. David Alan Gilbert * Setup an area of RAM so that it *can* be used for postcopy later; this 4601caddf8aSDr. David Alan Gilbert * must be done right at the start prior to pre-copy. 4611caddf8aSDr. David Alan Gilbert * opaque should be the MIS. 4621caddf8aSDr. David Alan Gilbert */ 463754cb9c0SYury Kotov static int init_range(RAMBlock *rb, void *opaque) 4641caddf8aSDr. David Alan Gilbert { 465754cb9c0SYury Kotov const char *block_name = qemu_ram_get_idstr(rb); 466754cb9c0SYury Kotov void *host_addr = qemu_ram_get_host_addr(rb); 467754cb9c0SYury Kotov ram_addr_t offset = qemu_ram_get_offset(rb); 468754cb9c0SYury Kotov ram_addr_t length = qemu_ram_get_used_length(rb); 4691caddf8aSDr. David Alan Gilbert trace_postcopy_init_range(block_name, host_addr, offset, length); 4701caddf8aSDr. David Alan Gilbert 4711caddf8aSDr. David Alan Gilbert /* 472898ba906SDavid Hildenbrand * Save the used_length before running the guest. In case we have to 473898ba906SDavid Hildenbrand * resize RAM blocks when syncing RAM block sizes from the source during 474898ba906SDavid Hildenbrand * precopy, we'll update it manually via the ram block notifier. 475898ba906SDavid Hildenbrand */ 476898ba906SDavid Hildenbrand rb->postcopy_length = length; 477898ba906SDavid Hildenbrand 478898ba906SDavid Hildenbrand /* 4791caddf8aSDr. David Alan Gilbert * We need the whole of RAM to be truly empty for postcopy, so things 4801caddf8aSDr. David Alan Gilbert * like ROMs and any data tables built during init must be zero'd 4811caddf8aSDr. David Alan Gilbert * - we're going to get the copy from the source anyway. 4821caddf8aSDr. David Alan Gilbert * (Precopy will just overwrite this data, so doesn't need the discard) 4831caddf8aSDr. David Alan Gilbert */ 484aaa2064cSJuan Quintela if (ram_discard_range(block_name, 0, length)) { 4851caddf8aSDr. David Alan Gilbert return -1; 4861caddf8aSDr. David Alan Gilbert } 4871caddf8aSDr. David Alan Gilbert 4881caddf8aSDr. David Alan Gilbert return 0; 4891caddf8aSDr. David Alan Gilbert } 4901caddf8aSDr. David Alan Gilbert 4911caddf8aSDr. David Alan Gilbert /* 4921caddf8aSDr. David Alan Gilbert * At the end of migration, undo the effects of init_range 4931caddf8aSDr. David Alan Gilbert * opaque should be the MIS. 4941caddf8aSDr. David Alan Gilbert */ 495754cb9c0SYury Kotov static int cleanup_range(RAMBlock *rb, void *opaque) 4961caddf8aSDr. David Alan Gilbert { 497754cb9c0SYury Kotov const char *block_name = qemu_ram_get_idstr(rb); 498754cb9c0SYury Kotov void *host_addr = qemu_ram_get_host_addr(rb); 499754cb9c0SYury Kotov ram_addr_t offset = qemu_ram_get_offset(rb); 500898ba906SDavid Hildenbrand ram_addr_t length = rb->postcopy_length; 5011caddf8aSDr. David Alan Gilbert MigrationIncomingState *mis = opaque; 5021caddf8aSDr. David Alan Gilbert struct uffdio_range range_struct; 5031caddf8aSDr. David Alan Gilbert trace_postcopy_cleanup_range(block_name, host_addr, offset, length); 5041caddf8aSDr. David Alan Gilbert 5051caddf8aSDr. David Alan Gilbert /* 5061caddf8aSDr. David Alan Gilbert * We turned off hugepage for the precopy stage with postcopy enabled 5071caddf8aSDr. David Alan Gilbert * we can turn it back on now. 5081caddf8aSDr. David Alan Gilbert */ 5091d741439SDr. David Alan Gilbert qemu_madvise(host_addr, length, QEMU_MADV_HUGEPAGE); 5101caddf8aSDr. David Alan Gilbert 5111caddf8aSDr. David Alan Gilbert /* 5121caddf8aSDr. David Alan Gilbert * We can also turn off userfault now since we should have all the 5131caddf8aSDr. David Alan Gilbert * pages. It can be useful to leave it on to debug postcopy 5141caddf8aSDr. David Alan Gilbert * if you're not sure it's always getting every page. 5151caddf8aSDr. David Alan Gilbert */ 5161caddf8aSDr. David Alan Gilbert range_struct.start = (uintptr_t)host_addr; 5171caddf8aSDr. David Alan Gilbert range_struct.len = length; 5181caddf8aSDr. David Alan Gilbert 5191caddf8aSDr. David Alan Gilbert if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) { 5201caddf8aSDr. David Alan Gilbert error_report("%s: userfault unregister %s", __func__, strerror(errno)); 5211caddf8aSDr. David Alan Gilbert 5221caddf8aSDr. David Alan Gilbert return -1; 5231caddf8aSDr. David Alan Gilbert } 5241caddf8aSDr. David Alan Gilbert 5251caddf8aSDr. David Alan Gilbert return 0; 5261caddf8aSDr. David Alan Gilbert } 5271caddf8aSDr. David Alan Gilbert 5281caddf8aSDr. David Alan Gilbert /* 5291caddf8aSDr. David Alan Gilbert * Initialise postcopy-ram, setting the RAM to a state where we can go into 5301caddf8aSDr. David Alan Gilbert * postcopy later; must be called prior to any precopy. 5311caddf8aSDr. David Alan Gilbert * called from arch_init's similarly named ram_postcopy_incoming_init 5321caddf8aSDr. David Alan Gilbert */ 533c136180cSDavid Hildenbrand int postcopy_ram_incoming_init(MigrationIncomingState *mis) 5341caddf8aSDr. David Alan Gilbert { 535fbd162e6SYury Kotov if (foreach_not_ignored_block(init_range, NULL)) { 5361caddf8aSDr. David Alan Gilbert return -1; 5371caddf8aSDr. David Alan Gilbert } 5381caddf8aSDr. David Alan Gilbert 5391caddf8aSDr. David Alan Gilbert return 0; 5401caddf8aSDr. David Alan Gilbert } 5411caddf8aSDr. David Alan Gilbert 542476ebf77SPeter Xu static void postcopy_temp_pages_cleanup(MigrationIncomingState *mis) 543476ebf77SPeter Xu { 54477dadc3fSPeter Xu int i; 54577dadc3fSPeter Xu 54677dadc3fSPeter Xu if (mis->postcopy_tmp_pages) { 54777dadc3fSPeter Xu for (i = 0; i < mis->postcopy_channels; i++) { 54877dadc3fSPeter Xu if (mis->postcopy_tmp_pages[i].tmp_huge_page) { 54977dadc3fSPeter Xu munmap(mis->postcopy_tmp_pages[i].tmp_huge_page, 55077dadc3fSPeter Xu mis->largest_page_size); 55177dadc3fSPeter Xu mis->postcopy_tmp_pages[i].tmp_huge_page = NULL; 55277dadc3fSPeter Xu } 55377dadc3fSPeter Xu } 55477dadc3fSPeter Xu g_free(mis->postcopy_tmp_pages); 55577dadc3fSPeter Xu mis->postcopy_tmp_pages = NULL; 556476ebf77SPeter Xu } 557476ebf77SPeter Xu 558476ebf77SPeter Xu if (mis->postcopy_tmp_zero_page) { 559476ebf77SPeter Xu munmap(mis->postcopy_tmp_zero_page, mis->largest_page_size); 560476ebf77SPeter Xu mis->postcopy_tmp_zero_page = NULL; 561476ebf77SPeter Xu } 562476ebf77SPeter Xu } 563476ebf77SPeter Xu 5641caddf8aSDr. David Alan Gilbert /* 5651caddf8aSDr. David Alan Gilbert * At the end of a migration where postcopy_ram_incoming_init was called. 5661caddf8aSDr. David Alan Gilbert */ 5671caddf8aSDr. David Alan Gilbert int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) 5681caddf8aSDr. David Alan Gilbert { 569c4faeed2SDr. David Alan Gilbert trace_postcopy_ram_incoming_cleanup_entry(); 570c4faeed2SDr. David Alan Gilbert 5716621883fSPeter Xu if (mis->preempt_thread_status == PREEMPT_THREAD_CREATED) { 5726621883fSPeter Xu /* Notify the fast load thread to quit */ 5736621883fSPeter Xu mis->preempt_thread_status = PREEMPT_THREAD_QUIT; 5746621883fSPeter Xu if (mis->postcopy_qemufile_dst) { 5756621883fSPeter Xu qemu_file_shutdown(mis->postcopy_qemufile_dst); 5766621883fSPeter Xu } 57736f62f11SPeter Xu qemu_thread_join(&mis->postcopy_prio_thread); 5786621883fSPeter Xu mis->preempt_thread_status = PREEMPT_THREAD_NONE; 57936f62f11SPeter Xu } 58036f62f11SPeter Xu 581c4faeed2SDr. David Alan Gilbert if (mis->have_fault_thread) { 58246343570SDr. David Alan Gilbert Error *local_err = NULL; 58346343570SDr. David Alan Gilbert 58455d0fe82SIlya Maximets /* Let the fault thread quit */ 585d73415a3SStefan Hajnoczi qatomic_set(&mis->fault_thread_quit, 1); 58655d0fe82SIlya Maximets postcopy_fault_thread_notify(mis); 58755d0fe82SIlya Maximets trace_postcopy_ram_incoming_cleanup_join(); 58855d0fe82SIlya Maximets qemu_thread_join(&mis->fault_thread); 58955d0fe82SIlya Maximets 59046343570SDr. David Alan Gilbert if (postcopy_notify(POSTCOPY_NOTIFY_INBOUND_END, &local_err)) { 59146343570SDr. David Alan Gilbert error_report_err(local_err); 59246343570SDr. David Alan Gilbert return -1; 59346343570SDr. David Alan Gilbert } 59446343570SDr. David Alan Gilbert 595fbd162e6SYury Kotov if (foreach_not_ignored_block(cleanup_range, mis)) { 5961caddf8aSDr. David Alan Gilbert return -1; 5971caddf8aSDr. David Alan Gilbert } 5989ab7ef9bSPeter Xu 599c4faeed2SDr. David Alan Gilbert trace_postcopy_ram_incoming_cleanup_closeuf(); 600c4faeed2SDr. David Alan Gilbert close(mis->userfault_fd); 60164f615feSPeter Xu close(mis->userfault_event_fd); 602c4faeed2SDr. David Alan Gilbert mis->have_fault_thread = false; 603c4faeed2SDr. David Alan Gilbert } 604c4faeed2SDr. David Alan Gilbert 60558b7c17eSDr. David Alan Gilbert if (enable_mlock) { 60658b7c17eSDr. David Alan Gilbert if (os_mlock() < 0) { 60758b7c17eSDr. David Alan Gilbert error_report("mlock: %s", strerror(errno)); 60858b7c17eSDr. David Alan Gilbert /* 60958b7c17eSDr. David Alan Gilbert * It doesn't feel right to fail at this point, we have a valid 61058b7c17eSDr. David Alan Gilbert * VM state. 61158b7c17eSDr. David Alan Gilbert */ 61258b7c17eSDr. David Alan Gilbert } 61358b7c17eSDr. David Alan Gilbert } 61458b7c17eSDr. David Alan Gilbert 615476ebf77SPeter Xu postcopy_temp_pages_cleanup(mis); 616476ebf77SPeter Xu 61765ace060SAlexey Perevalov trace_postcopy_ram_incoming_cleanup_blocktime( 61865ace060SAlexey Perevalov get_postcopy_total_blocktime()); 61965ace060SAlexey Perevalov 620c4faeed2SDr. David Alan Gilbert trace_postcopy_ram_incoming_cleanup_exit(); 6211caddf8aSDr. David Alan Gilbert return 0; 6221caddf8aSDr. David Alan Gilbert } 6231caddf8aSDr. David Alan Gilbert 624f0a227adSDr. David Alan Gilbert /* 625f9527107SDr. David Alan Gilbert * Disable huge pages on an area 626f9527107SDr. David Alan Gilbert */ 627754cb9c0SYury Kotov static int nhp_range(RAMBlock *rb, void *opaque) 628f9527107SDr. David Alan Gilbert { 629754cb9c0SYury Kotov const char *block_name = qemu_ram_get_idstr(rb); 630754cb9c0SYury Kotov void *host_addr = qemu_ram_get_host_addr(rb); 631754cb9c0SYury Kotov ram_addr_t offset = qemu_ram_get_offset(rb); 632898ba906SDavid Hildenbrand ram_addr_t length = rb->postcopy_length; 633f9527107SDr. David Alan Gilbert trace_postcopy_nhp_range(block_name, host_addr, offset, length); 634f9527107SDr. David Alan Gilbert 635f9527107SDr. David Alan Gilbert /* 636f9527107SDr. David Alan Gilbert * Before we do discards we need to ensure those discards really 637f9527107SDr. David Alan Gilbert * do delete areas of the page, even if THP thinks a hugepage would 638f9527107SDr. David Alan Gilbert * be a good idea, so force hugepages off. 639f9527107SDr. David Alan Gilbert */ 6401d741439SDr. David Alan Gilbert qemu_madvise(host_addr, length, QEMU_MADV_NOHUGEPAGE); 641f9527107SDr. David Alan Gilbert 642f9527107SDr. David Alan Gilbert return 0; 643f9527107SDr. David Alan Gilbert } 644f9527107SDr. David Alan Gilbert 645f9527107SDr. David Alan Gilbert /* 646f9527107SDr. David Alan Gilbert * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard 647f9527107SDr. David Alan Gilbert * however leaving it until after precopy means that most of the precopy 648f9527107SDr. David Alan Gilbert * data is still THPd 649f9527107SDr. David Alan Gilbert */ 650f9527107SDr. David Alan Gilbert int postcopy_ram_prepare_discard(MigrationIncomingState *mis) 651f9527107SDr. David Alan Gilbert { 652fbd162e6SYury Kotov if (foreach_not_ignored_block(nhp_range, mis)) { 653f9527107SDr. David Alan Gilbert return -1; 654f9527107SDr. David Alan Gilbert } 655f9527107SDr. David Alan Gilbert 656f9527107SDr. David Alan Gilbert postcopy_state_set(POSTCOPY_INCOMING_DISCARD); 657f9527107SDr. David Alan Gilbert 658f9527107SDr. David Alan Gilbert return 0; 659f9527107SDr. David Alan Gilbert } 660f9527107SDr. David Alan Gilbert 661f9527107SDr. David Alan Gilbert /* 662f0a227adSDr. David Alan Gilbert * Mark the given area of RAM as requiring notification to unwritten areas 663fbd162e6SYury Kotov * Used as a callback on foreach_not_ignored_block. 664f0a227adSDr. David Alan Gilbert * host_addr: Base of area to mark 665f0a227adSDr. David Alan Gilbert * offset: Offset in the whole ram arena 666f0a227adSDr. David Alan Gilbert * length: Length of the section 667f0a227adSDr. David Alan Gilbert * opaque: MigrationIncomingState pointer 668f0a227adSDr. David Alan Gilbert * Returns 0 on success 669f0a227adSDr. David Alan Gilbert */ 670754cb9c0SYury Kotov static int ram_block_enable_notify(RAMBlock *rb, void *opaque) 671f0a227adSDr. David Alan Gilbert { 672f0a227adSDr. David Alan Gilbert MigrationIncomingState *mis = opaque; 673f0a227adSDr. David Alan Gilbert struct uffdio_register reg_struct; 674f0a227adSDr. David Alan Gilbert 675754cb9c0SYury Kotov reg_struct.range.start = (uintptr_t)qemu_ram_get_host_addr(rb); 676898ba906SDavid Hildenbrand reg_struct.range.len = rb->postcopy_length; 677f0a227adSDr. David Alan Gilbert reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; 678f0a227adSDr. David Alan Gilbert 679f0a227adSDr. David Alan Gilbert /* Now tell our userfault_fd that it's responsible for this area */ 680f0a227adSDr. David Alan Gilbert if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, ®_struct)) { 681f0a227adSDr. David Alan Gilbert error_report("%s userfault register: %s", __func__, strerror(errno)); 682f0a227adSDr. David Alan Gilbert return -1; 683f0a227adSDr. David Alan Gilbert } 684665414adSDr. David Alan Gilbert if (!(reg_struct.ioctls & ((__u64)1 << _UFFDIO_COPY))) { 685665414adSDr. David Alan Gilbert error_report("%s userfault: Region doesn't support COPY", __func__); 686665414adSDr. David Alan Gilbert return -1; 687665414adSDr. David Alan Gilbert } 6882ce16640SDr. David Alan Gilbert if (reg_struct.ioctls & ((__u64)1 << _UFFDIO_ZEROPAGE)) { 6892ce16640SDr. David Alan Gilbert qemu_ram_set_uf_zeroable(rb); 6902ce16640SDr. David Alan Gilbert } 691f0a227adSDr. David Alan Gilbert 692f0a227adSDr. David Alan Gilbert return 0; 693f0a227adSDr. David Alan Gilbert } 694f0a227adSDr. David Alan Gilbert 6955efc3564SDr. David Alan Gilbert int postcopy_wake_shared(struct PostCopyFD *pcfd, 6965efc3564SDr. David Alan Gilbert uint64_t client_addr, 6975efc3564SDr. David Alan Gilbert RAMBlock *rb) 6985efc3564SDr. David Alan Gilbert { 6995efc3564SDr. David Alan Gilbert size_t pagesize = qemu_ram_pagesize(rb); 7005efc3564SDr. David Alan Gilbert struct uffdio_range range; 7015efc3564SDr. David Alan Gilbert int ret; 7025efc3564SDr. David Alan Gilbert trace_postcopy_wake_shared(client_addr, qemu_ram_get_idstr(rb)); 7037648297dSDavid Hildenbrand range.start = ROUND_DOWN(client_addr, pagesize); 7045efc3564SDr. David Alan Gilbert range.len = pagesize; 7055efc3564SDr. David Alan Gilbert ret = ioctl(pcfd->fd, UFFDIO_WAKE, &range); 7065efc3564SDr. David Alan Gilbert if (ret) { 7075efc3564SDr. David Alan Gilbert error_report("%s: Failed to wake: %zx in %s (%s)", 7085efc3564SDr. David Alan Gilbert __func__, (size_t)client_addr, qemu_ram_get_idstr(rb), 7095efc3564SDr. David Alan Gilbert strerror(errno)); 7105efc3564SDr. David Alan Gilbert } 7115efc3564SDr. David Alan Gilbert return ret; 7125efc3564SDr. David Alan Gilbert } 7135efc3564SDr. David Alan Gilbert 7149470c5e0SDavid Hildenbrand static int postcopy_request_page(MigrationIncomingState *mis, RAMBlock *rb, 7159470c5e0SDavid Hildenbrand ram_addr_t start, uint64_t haddr) 7169470c5e0SDavid Hildenbrand { 7179470c5e0SDavid Hildenbrand void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb)); 7189470c5e0SDavid Hildenbrand 7199470c5e0SDavid Hildenbrand /* 7209470c5e0SDavid Hildenbrand * Discarded pages (via RamDiscardManager) are never migrated. On unlikely 7219470c5e0SDavid Hildenbrand * access, place a zeropage, which will also set the relevant bits in the 7229470c5e0SDavid Hildenbrand * recv_bitmap accordingly, so we won't try placing a zeropage twice. 7239470c5e0SDavid Hildenbrand * 7249470c5e0SDavid Hildenbrand * Checking a single bit is sufficient to handle pagesize > TPS as either 7259470c5e0SDavid Hildenbrand * all relevant bits are set or not. 7269470c5e0SDavid Hildenbrand */ 7279470c5e0SDavid Hildenbrand assert(QEMU_IS_ALIGNED(start, qemu_ram_pagesize(rb))); 7289470c5e0SDavid Hildenbrand if (ramblock_page_is_discarded(rb, start)) { 7299470c5e0SDavid Hildenbrand bool received = ramblock_recv_bitmap_test_byte_offset(rb, start); 7309470c5e0SDavid Hildenbrand 7319470c5e0SDavid Hildenbrand return received ? 0 : postcopy_place_page_zero(mis, aligned, rb); 7329470c5e0SDavid Hildenbrand } 7339470c5e0SDavid Hildenbrand 7349470c5e0SDavid Hildenbrand return migrate_send_rp_req_pages(mis, rb, start, haddr); 7359470c5e0SDavid Hildenbrand } 7369470c5e0SDavid Hildenbrand 737f0a227adSDr. David Alan Gilbert /* 738096bf4c8SDr. David Alan Gilbert * Callback from shared fault handlers to ask for a page, 739096bf4c8SDr. David Alan Gilbert * the page must be specified by a RAMBlock and an offset in that rb 740096bf4c8SDr. David Alan Gilbert * Note: Only for use by shared fault handlers (in fault thread) 741096bf4c8SDr. David Alan Gilbert */ 742096bf4c8SDr. David Alan Gilbert int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb, 743096bf4c8SDr. David Alan Gilbert uint64_t client_addr, uint64_t rb_offset) 744096bf4c8SDr. David Alan Gilbert { 7457648297dSDavid Hildenbrand uint64_t aligned_rbo = ROUND_DOWN(rb_offset, qemu_ram_pagesize(rb)); 746096bf4c8SDr. David Alan Gilbert MigrationIncomingState *mis = migration_incoming_get_current(); 747096bf4c8SDr. David Alan Gilbert 748096bf4c8SDr. David Alan Gilbert trace_postcopy_request_shared_page(pcfd->idstr, qemu_ram_get_idstr(rb), 749096bf4c8SDr. David Alan Gilbert rb_offset); 750dedfb4b2SDr. David Alan Gilbert if (ramblock_recv_bitmap_test_byte_offset(rb, aligned_rbo)) { 751dedfb4b2SDr. David Alan Gilbert trace_postcopy_request_shared_page_present(pcfd->idstr, 752dedfb4b2SDr. David Alan Gilbert qemu_ram_get_idstr(rb), rb_offset); 753dedfb4b2SDr. David Alan Gilbert return postcopy_wake_shared(pcfd, client_addr, rb); 754dedfb4b2SDr. David Alan Gilbert } 7559470c5e0SDavid Hildenbrand postcopy_request_page(mis, rb, aligned_rbo, client_addr); 756096bf4c8SDr. David Alan Gilbert return 0; 757096bf4c8SDr. David Alan Gilbert } 758096bf4c8SDr. David Alan Gilbert 759575b0b33SAlexey Perevalov static int get_mem_fault_cpu_index(uint32_t pid) 760575b0b33SAlexey Perevalov { 761575b0b33SAlexey Perevalov CPUState *cpu_iter; 762575b0b33SAlexey Perevalov 763575b0b33SAlexey Perevalov CPU_FOREACH(cpu_iter) { 764575b0b33SAlexey Perevalov if (cpu_iter->thread_id == pid) { 765575b0b33SAlexey Perevalov trace_get_mem_fault_cpu_index(cpu_iter->cpu_index, pid); 766575b0b33SAlexey Perevalov return cpu_iter->cpu_index; 767575b0b33SAlexey Perevalov } 768575b0b33SAlexey Perevalov } 769575b0b33SAlexey Perevalov trace_get_mem_fault_cpu_index(-1, pid); 770575b0b33SAlexey Perevalov return -1; 771575b0b33SAlexey Perevalov } 772575b0b33SAlexey Perevalov 773575b0b33SAlexey Perevalov static uint32_t get_low_time_offset(PostcopyBlocktimeContext *dc) 774575b0b33SAlexey Perevalov { 775575b0b33SAlexey Perevalov int64_t start_time_offset = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - 776575b0b33SAlexey Perevalov dc->start_time; 777575b0b33SAlexey Perevalov return start_time_offset < 1 ? 1 : start_time_offset & UINT32_MAX; 778575b0b33SAlexey Perevalov } 779575b0b33SAlexey Perevalov 780575b0b33SAlexey Perevalov /* 781575b0b33SAlexey Perevalov * This function is being called when pagefault occurs. It 782575b0b33SAlexey Perevalov * tracks down vCPU blocking time. 783575b0b33SAlexey Perevalov * 784575b0b33SAlexey Perevalov * @addr: faulted host virtual address 785575b0b33SAlexey Perevalov * @ptid: faulted process thread id 786575b0b33SAlexey Perevalov * @rb: ramblock appropriate to addr 787575b0b33SAlexey Perevalov */ 788575b0b33SAlexey Perevalov static void mark_postcopy_blocktime_begin(uintptr_t addr, uint32_t ptid, 789575b0b33SAlexey Perevalov RAMBlock *rb) 790575b0b33SAlexey Perevalov { 791575b0b33SAlexey Perevalov int cpu, already_received; 792575b0b33SAlexey Perevalov MigrationIncomingState *mis = migration_incoming_get_current(); 793575b0b33SAlexey Perevalov PostcopyBlocktimeContext *dc = mis->blocktime_ctx; 794575b0b33SAlexey Perevalov uint32_t low_time_offset; 795575b0b33SAlexey Perevalov 796575b0b33SAlexey Perevalov if (!dc || ptid == 0) { 797575b0b33SAlexey Perevalov return; 798575b0b33SAlexey Perevalov } 799575b0b33SAlexey Perevalov cpu = get_mem_fault_cpu_index(ptid); 800575b0b33SAlexey Perevalov if (cpu < 0) { 801575b0b33SAlexey Perevalov return; 802575b0b33SAlexey Perevalov } 803575b0b33SAlexey Perevalov 804575b0b33SAlexey Perevalov low_time_offset = get_low_time_offset(dc); 805575b0b33SAlexey Perevalov if (dc->vcpu_addr[cpu] == 0) { 806d73415a3SStefan Hajnoczi qatomic_inc(&dc->smp_cpus_down); 807575b0b33SAlexey Perevalov } 808575b0b33SAlexey Perevalov 809d73415a3SStefan Hajnoczi qatomic_xchg(&dc->last_begin, low_time_offset); 810d73415a3SStefan Hajnoczi qatomic_xchg(&dc->page_fault_vcpu_time[cpu], low_time_offset); 811d73415a3SStefan Hajnoczi qatomic_xchg(&dc->vcpu_addr[cpu], addr); 812575b0b33SAlexey Perevalov 813da1725d3SWei Yang /* 814da1725d3SWei Yang * check it here, not at the beginning of the function, 815da1725d3SWei Yang * due to, check could occur early than bitmap_set in 816da1725d3SWei Yang * qemu_ufd_copy_ioctl 817da1725d3SWei Yang */ 818575b0b33SAlexey Perevalov already_received = ramblock_recv_bitmap_test(rb, (void *)addr); 819575b0b33SAlexey Perevalov if (already_received) { 820d73415a3SStefan Hajnoczi qatomic_xchg(&dc->vcpu_addr[cpu], 0); 821d73415a3SStefan Hajnoczi qatomic_xchg(&dc->page_fault_vcpu_time[cpu], 0); 822d73415a3SStefan Hajnoczi qatomic_dec(&dc->smp_cpus_down); 823575b0b33SAlexey Perevalov } 824575b0b33SAlexey Perevalov trace_mark_postcopy_blocktime_begin(addr, dc, dc->page_fault_vcpu_time[cpu], 825575b0b33SAlexey Perevalov cpu, already_received); 826575b0b33SAlexey Perevalov } 827575b0b33SAlexey Perevalov 828575b0b33SAlexey Perevalov /* 829575b0b33SAlexey Perevalov * This function just provide calculated blocktime per cpu and trace it. 830575b0b33SAlexey Perevalov * Total blocktime is calculated in mark_postcopy_blocktime_end. 831575b0b33SAlexey Perevalov * 832575b0b33SAlexey Perevalov * 833575b0b33SAlexey Perevalov * Assume we have 3 CPU 834575b0b33SAlexey Perevalov * 835575b0b33SAlexey Perevalov * S1 E1 S1 E1 836575b0b33SAlexey Perevalov * -----***********------------xxx***************------------------------> CPU1 837575b0b33SAlexey Perevalov * 838575b0b33SAlexey Perevalov * S2 E2 839575b0b33SAlexey Perevalov * ------------****************xxx---------------------------------------> CPU2 840575b0b33SAlexey Perevalov * 841575b0b33SAlexey Perevalov * S3 E3 842575b0b33SAlexey Perevalov * ------------------------****xxx********-------------------------------> CPU3 843575b0b33SAlexey Perevalov * 844575b0b33SAlexey Perevalov * We have sequence S1,S2,E1,S3,S1,E2,E3,E1 845575b0b33SAlexey Perevalov * S2,E1 - doesn't match condition due to sequence S1,S2,E1 doesn't include CPU3 846575b0b33SAlexey Perevalov * S3,S1,E2 - sequence includes all CPUs, in this case overlap will be S1,E2 - 847575b0b33SAlexey Perevalov * it's a part of total blocktime. 848575b0b33SAlexey Perevalov * S1 - here is last_begin 849575b0b33SAlexey Perevalov * Legend of the picture is following: 850575b0b33SAlexey Perevalov * * - means blocktime per vCPU 851575b0b33SAlexey Perevalov * x - means overlapped blocktime (total blocktime) 852575b0b33SAlexey Perevalov * 853575b0b33SAlexey Perevalov * @addr: host virtual address 854575b0b33SAlexey Perevalov */ 855575b0b33SAlexey Perevalov static void mark_postcopy_blocktime_end(uintptr_t addr) 856575b0b33SAlexey Perevalov { 857575b0b33SAlexey Perevalov MigrationIncomingState *mis = migration_incoming_get_current(); 858575b0b33SAlexey Perevalov PostcopyBlocktimeContext *dc = mis->blocktime_ctx; 8595cc8767dSLike Xu MachineState *ms = MACHINE(qdev_get_machine()); 8605cc8767dSLike Xu unsigned int smp_cpus = ms->smp.cpus; 861575b0b33SAlexey Perevalov int i, affected_cpu = 0; 862575b0b33SAlexey Perevalov bool vcpu_total_blocktime = false; 863575b0b33SAlexey Perevalov uint32_t read_vcpu_time, low_time_offset; 864575b0b33SAlexey Perevalov 865575b0b33SAlexey Perevalov if (!dc) { 866575b0b33SAlexey Perevalov return; 867575b0b33SAlexey Perevalov } 868575b0b33SAlexey Perevalov 869575b0b33SAlexey Perevalov low_time_offset = get_low_time_offset(dc); 870575b0b33SAlexey Perevalov /* lookup cpu, to clear it, 8713a4452d8Szhaolichang * that algorithm looks straightforward, but it's not 872575b0b33SAlexey Perevalov * optimal, more optimal algorithm is keeping tree or hash 873575b0b33SAlexey Perevalov * where key is address value is a list of */ 874575b0b33SAlexey Perevalov for (i = 0; i < smp_cpus; i++) { 875575b0b33SAlexey Perevalov uint32_t vcpu_blocktime = 0; 876575b0b33SAlexey Perevalov 877d73415a3SStefan Hajnoczi read_vcpu_time = qatomic_fetch_add(&dc->page_fault_vcpu_time[i], 0); 878d73415a3SStefan Hajnoczi if (qatomic_fetch_add(&dc->vcpu_addr[i], 0) != addr || 879575b0b33SAlexey Perevalov read_vcpu_time == 0) { 880575b0b33SAlexey Perevalov continue; 881575b0b33SAlexey Perevalov } 882d73415a3SStefan Hajnoczi qatomic_xchg(&dc->vcpu_addr[i], 0); 883575b0b33SAlexey Perevalov vcpu_blocktime = low_time_offset - read_vcpu_time; 884575b0b33SAlexey Perevalov affected_cpu += 1; 885575b0b33SAlexey Perevalov /* we need to know is that mark_postcopy_end was due to 886575b0b33SAlexey Perevalov * faulted page, another possible case it's prefetched 887575b0b33SAlexey Perevalov * page and in that case we shouldn't be here */ 888575b0b33SAlexey Perevalov if (!vcpu_total_blocktime && 889d73415a3SStefan Hajnoczi qatomic_fetch_add(&dc->smp_cpus_down, 0) == smp_cpus) { 890575b0b33SAlexey Perevalov vcpu_total_blocktime = true; 891575b0b33SAlexey Perevalov } 892575b0b33SAlexey Perevalov /* continue cycle, due to one page could affect several vCPUs */ 893575b0b33SAlexey Perevalov dc->vcpu_blocktime[i] += vcpu_blocktime; 894575b0b33SAlexey Perevalov } 895575b0b33SAlexey Perevalov 896d73415a3SStefan Hajnoczi qatomic_sub(&dc->smp_cpus_down, affected_cpu); 897575b0b33SAlexey Perevalov if (vcpu_total_blocktime) { 898d73415a3SStefan Hajnoczi dc->total_blocktime += low_time_offset - qatomic_fetch_add( 899575b0b33SAlexey Perevalov &dc->last_begin, 0); 900575b0b33SAlexey Perevalov } 901575b0b33SAlexey Perevalov trace_mark_postcopy_blocktime_end(addr, dc, dc->total_blocktime, 902575b0b33SAlexey Perevalov affected_cpu); 903575b0b33SAlexey Perevalov } 904575b0b33SAlexey Perevalov 90527dd21b4SPeter Xu static void postcopy_pause_fault_thread(MigrationIncomingState *mis) 9063a7804c3SPeter Xu { 9073a7804c3SPeter Xu trace_postcopy_pause_fault_thread(); 9083a7804c3SPeter Xu qemu_sem_wait(&mis->postcopy_pause_sem_fault); 9093a7804c3SPeter Xu trace_postcopy_pause_fault_thread_continued(); 9103a7804c3SPeter Xu } 9113a7804c3SPeter Xu 912096bf4c8SDr. David Alan Gilbert /* 913f0a227adSDr. David Alan Gilbert * Handle faults detected by the USERFAULT markings 914f0a227adSDr. David Alan Gilbert */ 915f0a227adSDr. David Alan Gilbert static void *postcopy_ram_fault_thread(void *opaque) 916f0a227adSDr. David Alan Gilbert { 917f0a227adSDr. David Alan Gilbert MigrationIncomingState *mis = opaque; 918c4faeed2SDr. David Alan Gilbert struct uffd_msg msg; 919c4faeed2SDr. David Alan Gilbert int ret; 92000fa4fc8SDr. David Alan Gilbert size_t index; 921c4faeed2SDr. David Alan Gilbert RAMBlock *rb = NULL; 922f0a227adSDr. David Alan Gilbert 923c4faeed2SDr. David Alan Gilbert trace_postcopy_ram_fault_thread_entry(); 92474637e6fSLidong Chen rcu_register_thread(); 925096bf4c8SDr. David Alan Gilbert mis->last_rb = NULL; /* last RAMBlock we sent part of */ 926095c12a4SPeter Xu qemu_sem_post(&mis->thread_sync_sem); 927c4faeed2SDr. David Alan Gilbert 92800fa4fc8SDr. David Alan Gilbert struct pollfd *pfd; 92900fa4fc8SDr. David Alan Gilbert size_t pfd_len = 2 + mis->postcopy_remote_fds->len; 93000fa4fc8SDr. David Alan Gilbert 93100fa4fc8SDr. David Alan Gilbert pfd = g_new0(struct pollfd, pfd_len); 93200fa4fc8SDr. David Alan Gilbert 93300fa4fc8SDr. David Alan Gilbert pfd[0].fd = mis->userfault_fd; 93400fa4fc8SDr. David Alan Gilbert pfd[0].events = POLLIN; 93500fa4fc8SDr. David Alan Gilbert pfd[1].fd = mis->userfault_event_fd; 93600fa4fc8SDr. David Alan Gilbert pfd[1].events = POLLIN; /* Waiting for eventfd to go positive */ 93700fa4fc8SDr. David Alan Gilbert trace_postcopy_ram_fault_thread_fds_core(pfd[0].fd, pfd[1].fd); 93800fa4fc8SDr. David Alan Gilbert for (index = 0; index < mis->postcopy_remote_fds->len; index++) { 93900fa4fc8SDr. David Alan Gilbert struct PostCopyFD *pcfd = &g_array_index(mis->postcopy_remote_fds, 94000fa4fc8SDr. David Alan Gilbert struct PostCopyFD, index); 94100fa4fc8SDr. David Alan Gilbert pfd[2 + index].fd = pcfd->fd; 94200fa4fc8SDr. David Alan Gilbert pfd[2 + index].events = POLLIN; 94300fa4fc8SDr. David Alan Gilbert trace_postcopy_ram_fault_thread_fds_extra(2 + index, pcfd->idstr, 94400fa4fc8SDr. David Alan Gilbert pcfd->fd); 94500fa4fc8SDr. David Alan Gilbert } 94600fa4fc8SDr. David Alan Gilbert 947c4faeed2SDr. David Alan Gilbert while (true) { 948c4faeed2SDr. David Alan Gilbert ram_addr_t rb_offset; 94900fa4fc8SDr. David Alan Gilbert int poll_result; 950c4faeed2SDr. David Alan Gilbert 951c4faeed2SDr. David Alan Gilbert /* 952c4faeed2SDr. David Alan Gilbert * We're mainly waiting for the kernel to give us a faulting HVA, 953c4faeed2SDr. David Alan Gilbert * however we can be told to quit via userfault_quit_fd which is 954c4faeed2SDr. David Alan Gilbert * an eventfd 955c4faeed2SDr. David Alan Gilbert */ 956c4faeed2SDr. David Alan Gilbert 95700fa4fc8SDr. David Alan Gilbert poll_result = poll(pfd, pfd_len, -1 /* Wait forever */); 95800fa4fc8SDr. David Alan Gilbert if (poll_result == -1) { 959c4faeed2SDr. David Alan Gilbert error_report("%s: userfault poll: %s", __func__, strerror(errno)); 960c4faeed2SDr. David Alan Gilbert break; 961f0a227adSDr. David Alan Gilbert } 962f0a227adSDr. David Alan Gilbert 9633a7804c3SPeter Xu if (!mis->to_src_file) { 9643a7804c3SPeter Xu /* 9653a7804c3SPeter Xu * Possibly someone tells us that the return path is 9663a7804c3SPeter Xu * broken already using the event. We should hold until 9673a7804c3SPeter Xu * the channel is rebuilt. 9683a7804c3SPeter Xu */ 96927dd21b4SPeter Xu postcopy_pause_fault_thread(mis); 9703a7804c3SPeter Xu } 9713a7804c3SPeter Xu 972c4faeed2SDr. David Alan Gilbert if (pfd[1].revents) { 97364f615feSPeter Xu uint64_t tmp64 = 0; 97464f615feSPeter Xu 97564f615feSPeter Xu /* Consume the signal */ 97664f615feSPeter Xu if (read(mis->userfault_event_fd, &tmp64, 8) != 8) { 97764f615feSPeter Xu /* Nothing obviously nicer than posting this error. */ 97864f615feSPeter Xu error_report("%s: read() failed", __func__); 97964f615feSPeter Xu } 98064f615feSPeter Xu 981d73415a3SStefan Hajnoczi if (qatomic_read(&mis->fault_thread_quit)) { 982c4faeed2SDr. David Alan Gilbert trace_postcopy_ram_fault_thread_quit(); 983c4faeed2SDr. David Alan Gilbert break; 984c4faeed2SDr. David Alan Gilbert } 98564f615feSPeter Xu } 986c4faeed2SDr. David Alan Gilbert 98700fa4fc8SDr. David Alan Gilbert if (pfd[0].revents) { 98800fa4fc8SDr. David Alan Gilbert poll_result--; 989c4faeed2SDr. David Alan Gilbert ret = read(mis->userfault_fd, &msg, sizeof(msg)); 990c4faeed2SDr. David Alan Gilbert if (ret != sizeof(msg)) { 991c4faeed2SDr. David Alan Gilbert if (errno == EAGAIN) { 992c4faeed2SDr. David Alan Gilbert /* 993c4faeed2SDr. David Alan Gilbert * if a wake up happens on the other thread just after 994c4faeed2SDr. David Alan Gilbert * the poll, there is nothing to read. 995c4faeed2SDr. David Alan Gilbert */ 996c4faeed2SDr. David Alan Gilbert continue; 997c4faeed2SDr. David Alan Gilbert } 998c4faeed2SDr. David Alan Gilbert if (ret < 0) { 99900fa4fc8SDr. David Alan Gilbert error_report("%s: Failed to read full userfault " 100000fa4fc8SDr. David Alan Gilbert "message: %s", 1001c4faeed2SDr. David Alan Gilbert __func__, strerror(errno)); 1002c4faeed2SDr. David Alan Gilbert break; 1003c4faeed2SDr. David Alan Gilbert } else { 100400fa4fc8SDr. David Alan Gilbert error_report("%s: Read %d bytes from userfaultfd " 100500fa4fc8SDr. David Alan Gilbert "expected %zd", 1006c4faeed2SDr. David Alan Gilbert __func__, ret, sizeof(msg)); 1007c4faeed2SDr. David Alan Gilbert break; /* Lost alignment, don't know what we'd read next */ 1008c4faeed2SDr. David Alan Gilbert } 1009c4faeed2SDr. David Alan Gilbert } 1010c4faeed2SDr. David Alan Gilbert if (msg.event != UFFD_EVENT_PAGEFAULT) { 1011c4faeed2SDr. David Alan Gilbert error_report("%s: Read unexpected event %ud from userfaultfd", 1012c4faeed2SDr. David Alan Gilbert __func__, msg.event); 1013c4faeed2SDr. David Alan Gilbert continue; /* It's not a page fault, shouldn't happen */ 1014c4faeed2SDr. David Alan Gilbert } 1015c4faeed2SDr. David Alan Gilbert 1016c4faeed2SDr. David Alan Gilbert rb = qemu_ram_block_from_host( 1017c4faeed2SDr. David Alan Gilbert (void *)(uintptr_t)msg.arg.pagefault.address, 1018f615f396SPaolo Bonzini true, &rb_offset); 1019c4faeed2SDr. David Alan Gilbert if (!rb) { 1020c4faeed2SDr. David Alan Gilbert error_report("postcopy_ram_fault_thread: Fault outside guest: %" 1021c4faeed2SDr. David Alan Gilbert PRIx64, (uint64_t)msg.arg.pagefault.address); 1022c4faeed2SDr. David Alan Gilbert break; 1023c4faeed2SDr. David Alan Gilbert } 1024c4faeed2SDr. David Alan Gilbert 10257648297dSDavid Hildenbrand rb_offset = ROUND_DOWN(rb_offset, qemu_ram_pagesize(rb)); 1026c4faeed2SDr. David Alan Gilbert trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address, 1027c4faeed2SDr. David Alan Gilbert qemu_ram_get_idstr(rb), 1028575b0b33SAlexey Perevalov rb_offset, 1029575b0b33SAlexey Perevalov msg.arg.pagefault.feat.ptid); 1030575b0b33SAlexey Perevalov mark_postcopy_blocktime_begin( 1031575b0b33SAlexey Perevalov (uintptr_t)(msg.arg.pagefault.address), 1032575b0b33SAlexey Perevalov msg.arg.pagefault.feat.ptid, rb); 1033575b0b33SAlexey Perevalov 10343a7804c3SPeter Xu retry: 1035c4faeed2SDr. David Alan Gilbert /* 1036c4faeed2SDr. David Alan Gilbert * Send the request to the source - we want to request one 1037c4faeed2SDr. David Alan Gilbert * of our host page sizes (which is >= TPS) 1038c4faeed2SDr. David Alan Gilbert */ 10399470c5e0SDavid Hildenbrand ret = postcopy_request_page(mis, rb, rb_offset, 10408f8bfffcSPeter Xu msg.arg.pagefault.address); 10413a7804c3SPeter Xu if (ret) { 10423a7804c3SPeter Xu /* May be network failure, try to wait for recovery */ 104327dd21b4SPeter Xu postcopy_pause_fault_thread(mis); 10443a7804c3SPeter Xu goto retry; 1045c4faeed2SDr. David Alan Gilbert } 1046c4faeed2SDr. David Alan Gilbert } 104700fa4fc8SDr. David Alan Gilbert 104800fa4fc8SDr. David Alan Gilbert /* Now handle any requests from external processes on shared memory */ 104900fa4fc8SDr. David Alan Gilbert /* TODO: May need to handle devices deregistering during postcopy */ 105000fa4fc8SDr. David Alan Gilbert for (index = 2; index < pfd_len && poll_result; index++) { 105100fa4fc8SDr. David Alan Gilbert if (pfd[index].revents) { 105200fa4fc8SDr. David Alan Gilbert struct PostCopyFD *pcfd = 105300fa4fc8SDr. David Alan Gilbert &g_array_index(mis->postcopy_remote_fds, 105400fa4fc8SDr. David Alan Gilbert struct PostCopyFD, index - 2); 105500fa4fc8SDr. David Alan Gilbert 105600fa4fc8SDr. David Alan Gilbert poll_result--; 105700fa4fc8SDr. David Alan Gilbert if (pfd[index].revents & POLLERR) { 105800fa4fc8SDr. David Alan Gilbert error_report("%s: POLLERR on poll %zd fd=%d", 105900fa4fc8SDr. David Alan Gilbert __func__, index, pcfd->fd); 106000fa4fc8SDr. David Alan Gilbert pfd[index].events = 0; 106100fa4fc8SDr. David Alan Gilbert continue; 106200fa4fc8SDr. David Alan Gilbert } 106300fa4fc8SDr. David Alan Gilbert 106400fa4fc8SDr. David Alan Gilbert ret = read(pcfd->fd, &msg, sizeof(msg)); 106500fa4fc8SDr. David Alan Gilbert if (ret != sizeof(msg)) { 106600fa4fc8SDr. David Alan Gilbert if (errno == EAGAIN) { 106700fa4fc8SDr. David Alan Gilbert /* 106800fa4fc8SDr. David Alan Gilbert * if a wake up happens on the other thread just after 106900fa4fc8SDr. David Alan Gilbert * the poll, there is nothing to read. 107000fa4fc8SDr. David Alan Gilbert */ 107100fa4fc8SDr. David Alan Gilbert continue; 107200fa4fc8SDr. David Alan Gilbert } 107300fa4fc8SDr. David Alan Gilbert if (ret < 0) { 107400fa4fc8SDr. David Alan Gilbert error_report("%s: Failed to read full userfault " 107500fa4fc8SDr. David Alan Gilbert "message: %s (shared) revents=%d", 107600fa4fc8SDr. David Alan Gilbert __func__, strerror(errno), 107700fa4fc8SDr. David Alan Gilbert pfd[index].revents); 107800fa4fc8SDr. David Alan Gilbert /*TODO: Could just disable this sharer */ 107900fa4fc8SDr. David Alan Gilbert break; 108000fa4fc8SDr. David Alan Gilbert } else { 108100fa4fc8SDr. David Alan Gilbert error_report("%s: Read %d bytes from userfaultfd " 108200fa4fc8SDr. David Alan Gilbert "expected %zd (shared)", 108300fa4fc8SDr. David Alan Gilbert __func__, ret, sizeof(msg)); 108400fa4fc8SDr. David Alan Gilbert /*TODO: Could just disable this sharer */ 108500fa4fc8SDr. David Alan Gilbert break; /*Lost alignment,don't know what we'd read next*/ 108600fa4fc8SDr. David Alan Gilbert } 108700fa4fc8SDr. David Alan Gilbert } 108800fa4fc8SDr. David Alan Gilbert if (msg.event != UFFD_EVENT_PAGEFAULT) { 108900fa4fc8SDr. David Alan Gilbert error_report("%s: Read unexpected event %ud " 109000fa4fc8SDr. David Alan Gilbert "from userfaultfd (shared)", 109100fa4fc8SDr. David Alan Gilbert __func__, msg.event); 109200fa4fc8SDr. David Alan Gilbert continue; /* It's not a page fault, shouldn't happen */ 109300fa4fc8SDr. David Alan Gilbert } 109400fa4fc8SDr. David Alan Gilbert /* Call the device handler registered with us */ 109500fa4fc8SDr. David Alan Gilbert ret = pcfd->handler(pcfd, &msg); 109600fa4fc8SDr. David Alan Gilbert if (ret) { 109700fa4fc8SDr. David Alan Gilbert error_report("%s: Failed to resolve shared fault on %zd/%s", 109800fa4fc8SDr. David Alan Gilbert __func__, index, pcfd->idstr); 109900fa4fc8SDr. David Alan Gilbert /* TODO: Fail? Disable this sharer? */ 110000fa4fc8SDr. David Alan Gilbert } 110100fa4fc8SDr. David Alan Gilbert } 110200fa4fc8SDr. David Alan Gilbert } 110300fa4fc8SDr. David Alan Gilbert } 110474637e6fSLidong Chen rcu_unregister_thread(); 1105c4faeed2SDr. David Alan Gilbert trace_postcopy_ram_fault_thread_exit(); 1106fc6008f3SMarc-André Lureau g_free(pfd); 1107f0a227adSDr. David Alan Gilbert return NULL; 1108f0a227adSDr. David Alan Gilbert } 1109f0a227adSDr. David Alan Gilbert 1110476ebf77SPeter Xu static int postcopy_temp_pages_setup(MigrationIncomingState *mis) 1111476ebf77SPeter Xu { 111277dadc3fSPeter Xu PostcopyTmpPage *tmp_page; 111377dadc3fSPeter Xu int err, i, channels; 111477dadc3fSPeter Xu void *temp_page; 1115476ebf77SPeter Xu 111636f62f11SPeter Xu if (migrate_postcopy_preempt()) { 111736f62f11SPeter Xu /* If preemption enabled, need extra channel for urgent requests */ 111836f62f11SPeter Xu mis->postcopy_channels = RAM_CHANNEL_MAX; 111936f62f11SPeter Xu } else { 112036f62f11SPeter Xu /* Both precopy/postcopy on the same channel */ 112177dadc3fSPeter Xu mis->postcopy_channels = 1; 112236f62f11SPeter Xu } 112377dadc3fSPeter Xu 112477dadc3fSPeter Xu channels = mis->postcopy_channels; 112577dadc3fSPeter Xu mis->postcopy_tmp_pages = g_malloc0_n(sizeof(PostcopyTmpPage), channels); 112677dadc3fSPeter Xu 112777dadc3fSPeter Xu for (i = 0; i < channels; i++) { 112877dadc3fSPeter Xu tmp_page = &mis->postcopy_tmp_pages[i]; 112977dadc3fSPeter Xu temp_page = mmap(NULL, mis->largest_page_size, PROT_READ | PROT_WRITE, 1130476ebf77SPeter Xu MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 113177dadc3fSPeter Xu if (temp_page == MAP_FAILED) { 1132476ebf77SPeter Xu err = errno; 113377dadc3fSPeter Xu error_report("%s: Failed to map postcopy_tmp_pages[%d]: %s", 113477dadc3fSPeter Xu __func__, i, strerror(err)); 113577dadc3fSPeter Xu /* Clean up will be done later */ 1136476ebf77SPeter Xu return -err; 1137476ebf77SPeter Xu } 113877dadc3fSPeter Xu tmp_page->tmp_huge_page = temp_page; 113977dadc3fSPeter Xu /* Initialize default states for each tmp page */ 114077dadc3fSPeter Xu postcopy_temp_page_reset(tmp_page); 114177dadc3fSPeter Xu } 1142476ebf77SPeter Xu 1143476ebf77SPeter Xu /* 1144476ebf77SPeter Xu * Map large zero page when kernel can't use UFFDIO_ZEROPAGE for hugepages 1145476ebf77SPeter Xu */ 1146476ebf77SPeter Xu mis->postcopy_tmp_zero_page = mmap(NULL, mis->largest_page_size, 1147476ebf77SPeter Xu PROT_READ | PROT_WRITE, 1148476ebf77SPeter Xu MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1149476ebf77SPeter Xu if (mis->postcopy_tmp_zero_page == MAP_FAILED) { 1150476ebf77SPeter Xu err = errno; 1151476ebf77SPeter Xu mis->postcopy_tmp_zero_page = NULL; 1152476ebf77SPeter Xu error_report("%s: Failed to map large zero page %s", 1153476ebf77SPeter Xu __func__, strerror(err)); 1154476ebf77SPeter Xu return -err; 1155476ebf77SPeter Xu } 1156476ebf77SPeter Xu 1157476ebf77SPeter Xu memset(mis->postcopy_tmp_zero_page, '\0', mis->largest_page_size); 1158476ebf77SPeter Xu 1159476ebf77SPeter Xu return 0; 1160476ebf77SPeter Xu } 1161476ebf77SPeter Xu 11622a7eb148SWei Yang int postcopy_ram_incoming_setup(MigrationIncomingState *mis) 1163f0a227adSDr. David Alan Gilbert { 1164c4faeed2SDr. David Alan Gilbert /* Open the fd for the kernel to give us userfaults */ 1165d5890ea0SPeter Xu mis->userfault_fd = uffd_open(O_CLOEXEC | O_NONBLOCK); 1166c4faeed2SDr. David Alan Gilbert if (mis->userfault_fd == -1) { 1167c4faeed2SDr. David Alan Gilbert error_report("%s: Failed to open userfault fd: %s", __func__, 1168c4faeed2SDr. David Alan Gilbert strerror(errno)); 1169c4faeed2SDr. David Alan Gilbert return -1; 1170c4faeed2SDr. David Alan Gilbert } 1171c4faeed2SDr. David Alan Gilbert 1172c4faeed2SDr. David Alan Gilbert /* 1173c4faeed2SDr. David Alan Gilbert * Although the host check already tested the API, we need to 1174c4faeed2SDr. David Alan Gilbert * do the check again as an ABI handshake on the new fd. 1175c4faeed2SDr. David Alan Gilbert */ 117654ae0886SAlexey Perevalov if (!ufd_check_and_apply(mis->userfault_fd, mis)) { 1177c4faeed2SDr. David Alan Gilbert return -1; 1178c4faeed2SDr. David Alan Gilbert } 1179c4faeed2SDr. David Alan Gilbert 1180c4faeed2SDr. David Alan Gilbert /* Now an eventfd we use to tell the fault-thread to quit */ 118164f615feSPeter Xu mis->userfault_event_fd = eventfd(0, EFD_CLOEXEC); 118264f615feSPeter Xu if (mis->userfault_event_fd == -1) { 118364f615feSPeter Xu error_report("%s: Opening userfault_event_fd: %s", __func__, 1184c4faeed2SDr. David Alan Gilbert strerror(errno)); 1185c4faeed2SDr. David Alan Gilbert close(mis->userfault_fd); 1186c4faeed2SDr. David Alan Gilbert return -1; 1187c4faeed2SDr. David Alan Gilbert } 1188c4faeed2SDr. David Alan Gilbert 118936f62f11SPeter Xu postcopy_thread_create(mis, &mis->fault_thread, "fault-default", 1190095c12a4SPeter Xu postcopy_ram_fault_thread, QEMU_THREAD_JOINABLE); 1191c4faeed2SDr. David Alan Gilbert mis->have_fault_thread = true; 1192f0a227adSDr. David Alan Gilbert 1193f0a227adSDr. David Alan Gilbert /* Mark so that we get notified of accesses to unwritten areas */ 1194fbd162e6SYury Kotov if (foreach_not_ignored_block(ram_block_enable_notify, mis)) { 119591b02dc7SFei Li error_report("ram_block_enable_notify failed"); 1196f0a227adSDr. David Alan Gilbert return -1; 1197f0a227adSDr. David Alan Gilbert } 1198f0a227adSDr. David Alan Gilbert 1199476ebf77SPeter Xu if (postcopy_temp_pages_setup(mis)) { 1200476ebf77SPeter Xu /* Error dumped in the sub-function */ 12013414322aSWei Yang return -1; 12023414322aSWei Yang } 12033414322aSWei Yang 120436f62f11SPeter Xu if (migrate_postcopy_preempt()) { 120536f62f11SPeter Xu /* 120636f62f11SPeter Xu * This thread needs to be created after the temp pages because 120736f62f11SPeter Xu * it'll fetch RAM_CHANNEL_POSTCOPY PostcopyTmpPage immediately. 120836f62f11SPeter Xu */ 120936f62f11SPeter Xu postcopy_thread_create(mis, &mis->postcopy_prio_thread, "fault-fast", 121036f62f11SPeter Xu postcopy_preempt_thread, QEMU_THREAD_JOINABLE); 12116621883fSPeter Xu mis->preempt_thread_status = PREEMPT_THREAD_CREATED; 121236f62f11SPeter Xu } 121336f62f11SPeter Xu 1214c4faeed2SDr. David Alan Gilbert trace_postcopy_ram_enable_notify(); 1215c4faeed2SDr. David Alan Gilbert 1216f0a227adSDr. David Alan Gilbert return 0; 1217f0a227adSDr. David Alan Gilbert } 1218f0a227adSDr. David Alan Gilbert 1219eef621c4SPeter Xu static int qemu_ufd_copy_ioctl(MigrationIncomingState *mis, void *host_addr, 1220f9494614SAlexey Perevalov void *from_addr, uint64_t pagesize, RAMBlock *rb) 1221727b9d7eSAlexey Perevalov { 1222eef621c4SPeter Xu int userfault_fd = mis->userfault_fd; 1223f9494614SAlexey Perevalov int ret; 1224eef621c4SPeter Xu 1225727b9d7eSAlexey Perevalov if (from_addr) { 1226727b9d7eSAlexey Perevalov struct uffdio_copy copy_struct; 1227727b9d7eSAlexey Perevalov copy_struct.dst = (uint64_t)(uintptr_t)host_addr; 1228727b9d7eSAlexey Perevalov copy_struct.src = (uint64_t)(uintptr_t)from_addr; 1229727b9d7eSAlexey Perevalov copy_struct.len = pagesize; 1230727b9d7eSAlexey Perevalov copy_struct.mode = 0; 1231f9494614SAlexey Perevalov ret = ioctl(userfault_fd, UFFDIO_COPY, ©_struct); 1232727b9d7eSAlexey Perevalov } else { 1233727b9d7eSAlexey Perevalov struct uffdio_zeropage zero_struct; 1234727b9d7eSAlexey Perevalov zero_struct.range.start = (uint64_t)(uintptr_t)host_addr; 1235727b9d7eSAlexey Perevalov zero_struct.range.len = pagesize; 1236727b9d7eSAlexey Perevalov zero_struct.mode = 0; 1237f9494614SAlexey Perevalov ret = ioctl(userfault_fd, UFFDIO_ZEROPAGE, &zero_struct); 1238727b9d7eSAlexey Perevalov } 1239f9494614SAlexey Perevalov if (!ret) { 12408f8bfffcSPeter Xu qemu_mutex_lock(&mis->page_request_mutex); 1241f9494614SAlexey Perevalov ramblock_recv_bitmap_set_range(rb, host_addr, 1242f9494614SAlexey Perevalov pagesize / qemu_target_page_size()); 12438f8bfffcSPeter Xu /* 12448f8bfffcSPeter Xu * If this page resolves a page fault for a previous recorded faulted 12458f8bfffcSPeter Xu * address, take a special note to maintain the requested page list. 12468f8bfffcSPeter Xu */ 12478f8bfffcSPeter Xu if (g_tree_lookup(mis->page_requested, host_addr)) { 12488f8bfffcSPeter Xu g_tree_remove(mis->page_requested, host_addr); 12498f8bfffcSPeter Xu mis->page_requested_count--; 12508f8bfffcSPeter Xu trace_postcopy_page_req_del(host_addr, mis->page_requested_count); 12518f8bfffcSPeter Xu } 12528f8bfffcSPeter Xu qemu_mutex_unlock(&mis->page_request_mutex); 1253575b0b33SAlexey Perevalov mark_postcopy_blocktime_end((uintptr_t)host_addr); 1254f9494614SAlexey Perevalov } 1255f9494614SAlexey Perevalov return ret; 1256727b9d7eSAlexey Perevalov } 1257727b9d7eSAlexey Perevalov 1258d488b349SDr. David Alan Gilbert int postcopy_notify_shared_wake(RAMBlock *rb, uint64_t offset) 1259d488b349SDr. David Alan Gilbert { 1260d488b349SDr. David Alan Gilbert int i; 1261d488b349SDr. David Alan Gilbert MigrationIncomingState *mis = migration_incoming_get_current(); 1262d488b349SDr. David Alan Gilbert GArray *pcrfds = mis->postcopy_remote_fds; 1263d488b349SDr. David Alan Gilbert 1264d488b349SDr. David Alan Gilbert for (i = 0; i < pcrfds->len; i++) { 1265d488b349SDr. David Alan Gilbert struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i); 1266d488b349SDr. David Alan Gilbert int ret = cur->waker(cur, rb, offset); 1267d488b349SDr. David Alan Gilbert if (ret) { 1268d488b349SDr. David Alan Gilbert return ret; 1269d488b349SDr. David Alan Gilbert } 1270d488b349SDr. David Alan Gilbert } 1271d488b349SDr. David Alan Gilbert return 0; 1272d488b349SDr. David Alan Gilbert } 1273d488b349SDr. David Alan Gilbert 1274696ed9a9SDr. David Alan Gilbert /* 1275696ed9a9SDr. David Alan Gilbert * Place a host page (from) at (host) atomically 1276696ed9a9SDr. David Alan Gilbert * returns 0 on success 1277696ed9a9SDr. David Alan Gilbert */ 1278df9ff5e1SDr. David Alan Gilbert int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from, 12798be4620bSAlexey Perevalov RAMBlock *rb) 1280696ed9a9SDr. David Alan Gilbert { 12818be4620bSAlexey Perevalov size_t pagesize = qemu_ram_pagesize(rb); 1282696ed9a9SDr. David Alan Gilbert 1283696ed9a9SDr. David Alan Gilbert /* copy also acks to the kernel waking the stalled thread up 1284696ed9a9SDr. David Alan Gilbert * TODO: We can inhibit that ack and only do it if it was requested 1285696ed9a9SDr. David Alan Gilbert * which would be slightly cheaper, but we'd have to be careful 1286696ed9a9SDr. David Alan Gilbert * of the order of updating our page state. 1287696ed9a9SDr. David Alan Gilbert */ 1288eef621c4SPeter Xu if (qemu_ufd_copy_ioctl(mis, host, from, pagesize, rb)) { 1289696ed9a9SDr. David Alan Gilbert int e = errno; 1290df9ff5e1SDr. David Alan Gilbert error_report("%s: %s copy host: %p from: %p (size: %zd)", 1291df9ff5e1SDr. David Alan Gilbert __func__, strerror(e), host, from, pagesize); 1292696ed9a9SDr. David Alan Gilbert 1293696ed9a9SDr. David Alan Gilbert return -e; 1294696ed9a9SDr. David Alan Gilbert } 1295696ed9a9SDr. David Alan Gilbert 1296696ed9a9SDr. David Alan Gilbert trace_postcopy_place_page(host); 1297dedfb4b2SDr. David Alan Gilbert return postcopy_notify_shared_wake(rb, 1298dedfb4b2SDr. David Alan Gilbert qemu_ram_block_host_offset(rb, host)); 1299696ed9a9SDr. David Alan Gilbert } 1300696ed9a9SDr. David Alan Gilbert 1301696ed9a9SDr. David Alan Gilbert /* 1302696ed9a9SDr. David Alan Gilbert * Place a zero page at (host) atomically 1303696ed9a9SDr. David Alan Gilbert * returns 0 on success 1304696ed9a9SDr. David Alan Gilbert */ 1305df9ff5e1SDr. David Alan Gilbert int postcopy_place_page_zero(MigrationIncomingState *mis, void *host, 13068be4620bSAlexey Perevalov RAMBlock *rb) 1307696ed9a9SDr. David Alan Gilbert { 13082ce16640SDr. David Alan Gilbert size_t pagesize = qemu_ram_pagesize(rb); 1309df9ff5e1SDr. David Alan Gilbert trace_postcopy_place_page_zero(host); 1310696ed9a9SDr. David Alan Gilbert 13112ce16640SDr. David Alan Gilbert /* Normal RAMBlocks can zero a page using UFFDIO_ZEROPAGE 13122ce16640SDr. David Alan Gilbert * but it's not available for everything (e.g. hugetlbpages) 13132ce16640SDr. David Alan Gilbert */ 13142ce16640SDr. David Alan Gilbert if (qemu_ram_is_uf_zeroable(rb)) { 1315eef621c4SPeter Xu if (qemu_ufd_copy_ioctl(mis, host, NULL, pagesize, rb)) { 1316696ed9a9SDr. David Alan Gilbert int e = errno; 1317696ed9a9SDr. David Alan Gilbert error_report("%s: %s zero host: %p", 1318696ed9a9SDr. David Alan Gilbert __func__, strerror(e), host); 1319696ed9a9SDr. David Alan Gilbert 1320696ed9a9SDr. David Alan Gilbert return -e; 1321696ed9a9SDr. David Alan Gilbert } 1322dedfb4b2SDr. David Alan Gilbert return postcopy_notify_shared_wake(rb, 1323dedfb4b2SDr. David Alan Gilbert qemu_ram_block_host_offset(rb, 1324dedfb4b2SDr. David Alan Gilbert host)); 1325df9ff5e1SDr. David Alan Gilbert } else { 13266629890dSWei Yang return postcopy_place_page(mis, host, mis->postcopy_tmp_zero_page, rb); 1327df9ff5e1SDr. David Alan Gilbert } 1328696ed9a9SDr. David Alan Gilbert } 1329696ed9a9SDr. David Alan Gilbert 1330eb59db53SDr. David Alan Gilbert #else 1331eb59db53SDr. David Alan Gilbert /* No target OS support, stubs just fail */ 133265ace060SAlexey Perevalov void fill_destination_postcopy_migration_info(MigrationInfo *info) 133365ace060SAlexey Perevalov { 133465ace060SAlexey Perevalov } 133565ace060SAlexey Perevalov 1336d7651f15SAlexey Perevalov bool postcopy_ram_supported_by_host(MigrationIncomingState *mis) 1337eb59db53SDr. David Alan Gilbert { 1338eb59db53SDr. David Alan Gilbert error_report("%s: No OS support", __func__); 1339eb59db53SDr. David Alan Gilbert return false; 1340eb59db53SDr. David Alan Gilbert } 1341eb59db53SDr. David Alan Gilbert 1342c136180cSDavid Hildenbrand int postcopy_ram_incoming_init(MigrationIncomingState *mis) 13431caddf8aSDr. David Alan Gilbert { 13441caddf8aSDr. David Alan Gilbert error_report("postcopy_ram_incoming_init: No OS support"); 13451caddf8aSDr. David Alan Gilbert return -1; 13461caddf8aSDr. David Alan Gilbert } 13471caddf8aSDr. David Alan Gilbert 13481caddf8aSDr. David Alan Gilbert int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) 13491caddf8aSDr. David Alan Gilbert { 13501caddf8aSDr. David Alan Gilbert assert(0); 13511caddf8aSDr. David Alan Gilbert return -1; 13521caddf8aSDr. David Alan Gilbert } 13531caddf8aSDr. David Alan Gilbert 1354f9527107SDr. David Alan Gilbert int postcopy_ram_prepare_discard(MigrationIncomingState *mis) 1355f9527107SDr. David Alan Gilbert { 1356f9527107SDr. David Alan Gilbert assert(0); 1357f9527107SDr. David Alan Gilbert return -1; 1358f9527107SDr. David Alan Gilbert } 1359f9527107SDr. David Alan Gilbert 1360c188c539SMichael S. Tsirkin int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb, 1361c188c539SMichael S. Tsirkin uint64_t client_addr, uint64_t rb_offset) 1362c188c539SMichael S. Tsirkin { 1363c188c539SMichael S. Tsirkin assert(0); 1364c188c539SMichael S. Tsirkin return -1; 1365c188c539SMichael S. Tsirkin } 1366c188c539SMichael S. Tsirkin 13672a7eb148SWei Yang int postcopy_ram_incoming_setup(MigrationIncomingState *mis) 1368f0a227adSDr. David Alan Gilbert { 1369f0a227adSDr. David Alan Gilbert assert(0); 1370f0a227adSDr. David Alan Gilbert return -1; 1371f0a227adSDr. David Alan Gilbert } 1372696ed9a9SDr. David Alan Gilbert 1373df9ff5e1SDr. David Alan Gilbert int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from, 13748be4620bSAlexey Perevalov RAMBlock *rb) 1375696ed9a9SDr. David Alan Gilbert { 1376696ed9a9SDr. David Alan Gilbert assert(0); 1377696ed9a9SDr. David Alan Gilbert return -1; 1378696ed9a9SDr. David Alan Gilbert } 1379696ed9a9SDr. David Alan Gilbert 1380df9ff5e1SDr. David Alan Gilbert int postcopy_place_page_zero(MigrationIncomingState *mis, void *host, 13818be4620bSAlexey Perevalov RAMBlock *rb) 1382696ed9a9SDr. David Alan Gilbert { 1383696ed9a9SDr. David Alan Gilbert assert(0); 1384696ed9a9SDr. David Alan Gilbert return -1; 1385696ed9a9SDr. David Alan Gilbert } 1386696ed9a9SDr. David Alan Gilbert 13875efc3564SDr. David Alan Gilbert int postcopy_wake_shared(struct PostCopyFD *pcfd, 13885efc3564SDr. David Alan Gilbert uint64_t client_addr, 13895efc3564SDr. David Alan Gilbert RAMBlock *rb) 13905efc3564SDr. David Alan Gilbert { 13915efc3564SDr. David Alan Gilbert assert(0); 13925efc3564SDr. David Alan Gilbert return -1; 13935efc3564SDr. David Alan Gilbert } 1394eb59db53SDr. David Alan Gilbert #endif 1395eb59db53SDr. David Alan Gilbert 1396e0b266f0SDr. David Alan Gilbert /* ------------------------------------------------------------------------- */ 139777dadc3fSPeter Xu void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page) 139877dadc3fSPeter Xu { 139977dadc3fSPeter Xu tmp_page->target_pages = 0; 140077dadc3fSPeter Xu tmp_page->host_addr = NULL; 140177dadc3fSPeter Xu /* 140277dadc3fSPeter Xu * This is set to true when reset, and cleared as long as we received any 140377dadc3fSPeter Xu * of the non-zero small page within this huge page. 140477dadc3fSPeter Xu */ 140577dadc3fSPeter Xu tmp_page->all_zero = true; 140677dadc3fSPeter Xu } 1407e0b266f0SDr. David Alan Gilbert 14089ab7ef9bSPeter Xu void postcopy_fault_thread_notify(MigrationIncomingState *mis) 14099ab7ef9bSPeter Xu { 14109ab7ef9bSPeter Xu uint64_t tmp64 = 1; 14119ab7ef9bSPeter Xu 14129ab7ef9bSPeter Xu /* 14139ab7ef9bSPeter Xu * Wakeup the fault_thread. It's an eventfd that should currently 14149ab7ef9bSPeter Xu * be at 0, we're going to increment it to 1 14159ab7ef9bSPeter Xu */ 14169ab7ef9bSPeter Xu if (write(mis->userfault_event_fd, &tmp64, 8) != 8) { 14179ab7ef9bSPeter Xu /* Not much we can do here, but may as well report it */ 14189ab7ef9bSPeter Xu error_report("%s: incrementing failed: %s", __func__, 14199ab7ef9bSPeter Xu strerror(errno)); 14209ab7ef9bSPeter Xu } 14219ab7ef9bSPeter Xu } 14229ab7ef9bSPeter Xu 1423e0b266f0SDr. David Alan Gilbert /** 1424e0b266f0SDr. David Alan Gilbert * postcopy_discard_send_init: Called at the start of each RAMBlock before 1425e0b266f0SDr. David Alan Gilbert * asking to discard individual ranges. 1426e0b266f0SDr. David Alan Gilbert * 1427e0b266f0SDr. David Alan Gilbert * @ms: The current migration state. 1428810cf2bbSWei Yang * @offset: the bitmap offset of the named RAMBlock in the migration bitmap. 1429e0b266f0SDr. David Alan Gilbert * @name: RAMBlock that discards will operate on. 1430e0b266f0SDr. David Alan Gilbert */ 1431810cf2bbSWei Yang static PostcopyDiscardState pds = {0}; 1432810cf2bbSWei Yang void postcopy_discard_send_init(MigrationState *ms, const char *name) 1433e0b266f0SDr. David Alan Gilbert { 1434810cf2bbSWei Yang pds.ramblock_name = name; 1435810cf2bbSWei Yang pds.cur_entry = 0; 1436810cf2bbSWei Yang pds.nsentwords = 0; 1437810cf2bbSWei Yang pds.nsentcmds = 0; 1438e0b266f0SDr. David Alan Gilbert } 1439e0b266f0SDr. David Alan Gilbert 1440e0b266f0SDr. David Alan Gilbert /** 1441e0b266f0SDr. David Alan Gilbert * postcopy_discard_send_range: Called by the bitmap code for each chunk to 1442e0b266f0SDr. David Alan Gilbert * discard. May send a discard message, may just leave it queued to 1443e0b266f0SDr. David Alan Gilbert * be sent later. 1444e0b266f0SDr. David Alan Gilbert * 1445e0b266f0SDr. David Alan Gilbert * @ms: Current migration state. 1446e0b266f0SDr. David Alan Gilbert * @start,@length: a range of pages in the migration bitmap in the 1447e0b266f0SDr. David Alan Gilbert * RAM block passed to postcopy_discard_send_init() (length=1 is one page) 1448e0b266f0SDr. David Alan Gilbert */ 1449810cf2bbSWei Yang void postcopy_discard_send_range(MigrationState *ms, unsigned long start, 1450810cf2bbSWei Yang unsigned long length) 1451e0b266f0SDr. David Alan Gilbert { 145220afaed9SJuan Quintela size_t tp_size = qemu_target_page_size(); 1453e0b266f0SDr. David Alan Gilbert /* Convert to byte offsets within the RAM block */ 1454810cf2bbSWei Yang pds.start_list[pds.cur_entry] = start * tp_size; 1455810cf2bbSWei Yang pds.length_list[pds.cur_entry] = length * tp_size; 1456810cf2bbSWei Yang trace_postcopy_discard_send_range(pds.ramblock_name, start, length); 1457810cf2bbSWei Yang pds.cur_entry++; 1458810cf2bbSWei Yang pds.nsentwords++; 1459e0b266f0SDr. David Alan Gilbert 1460810cf2bbSWei Yang if (pds.cur_entry == MAX_DISCARDS_PER_COMMAND) { 1461e0b266f0SDr. David Alan Gilbert /* Full set, ship it! */ 146289a02a9fSzhanghailiang qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, 1463810cf2bbSWei Yang pds.ramblock_name, 1464810cf2bbSWei Yang pds.cur_entry, 1465810cf2bbSWei Yang pds.start_list, 1466810cf2bbSWei Yang pds.length_list); 1467810cf2bbSWei Yang pds.nsentcmds++; 1468810cf2bbSWei Yang pds.cur_entry = 0; 1469e0b266f0SDr. David Alan Gilbert } 1470e0b266f0SDr. David Alan Gilbert } 1471e0b266f0SDr. David Alan Gilbert 1472e0b266f0SDr. David Alan Gilbert /** 1473e0b266f0SDr. David Alan Gilbert * postcopy_discard_send_finish: Called at the end of each RAMBlock by the 1474e0b266f0SDr. David Alan Gilbert * bitmap code. Sends any outstanding discard messages, frees the PDS 1475e0b266f0SDr. David Alan Gilbert * 1476e0b266f0SDr. David Alan Gilbert * @ms: Current migration state. 1477e0b266f0SDr. David Alan Gilbert */ 1478810cf2bbSWei Yang void postcopy_discard_send_finish(MigrationState *ms) 1479e0b266f0SDr. David Alan Gilbert { 1480e0b266f0SDr. David Alan Gilbert /* Anything unsent? */ 1481810cf2bbSWei Yang if (pds.cur_entry) { 148289a02a9fSzhanghailiang qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, 1483810cf2bbSWei Yang pds.ramblock_name, 1484810cf2bbSWei Yang pds.cur_entry, 1485810cf2bbSWei Yang pds.start_list, 1486810cf2bbSWei Yang pds.length_list); 1487810cf2bbSWei Yang pds.nsentcmds++; 1488e0b266f0SDr. David Alan Gilbert } 1489e0b266f0SDr. David Alan Gilbert 1490810cf2bbSWei Yang trace_postcopy_discard_send_finish(pds.ramblock_name, pds.nsentwords, 1491810cf2bbSWei Yang pds.nsentcmds); 1492e0b266f0SDr. David Alan Gilbert } 1493bac3b212SJuan Quintela 1494bac3b212SJuan Quintela /* 1495bac3b212SJuan Quintela * Current state of incoming postcopy; note this is not part of 1496bac3b212SJuan Quintela * MigrationIncomingState since it's state is used during cleanup 1497bac3b212SJuan Quintela * at the end as MIS is being freed. 1498bac3b212SJuan Quintela */ 1499bac3b212SJuan Quintela static PostcopyState incoming_postcopy_state; 1500bac3b212SJuan Quintela 1501bac3b212SJuan Quintela PostcopyState postcopy_state_get(void) 1502bac3b212SJuan Quintela { 1503d73415a3SStefan Hajnoczi return qatomic_mb_read(&incoming_postcopy_state); 1504bac3b212SJuan Quintela } 1505bac3b212SJuan Quintela 1506bac3b212SJuan Quintela /* Set the state and return the old state */ 1507bac3b212SJuan Quintela PostcopyState postcopy_state_set(PostcopyState new_state) 1508bac3b212SJuan Quintela { 1509d73415a3SStefan Hajnoczi return qatomic_xchg(&incoming_postcopy_state, new_state); 1510bac3b212SJuan Quintela } 151100fa4fc8SDr. David Alan Gilbert 151200fa4fc8SDr. David Alan Gilbert /* Register a handler for external shared memory postcopy 151300fa4fc8SDr. David Alan Gilbert * called on the destination. 151400fa4fc8SDr. David Alan Gilbert */ 151500fa4fc8SDr. David Alan Gilbert void postcopy_register_shared_ufd(struct PostCopyFD *pcfd) 151600fa4fc8SDr. David Alan Gilbert { 151700fa4fc8SDr. David Alan Gilbert MigrationIncomingState *mis = migration_incoming_get_current(); 151800fa4fc8SDr. David Alan Gilbert 151900fa4fc8SDr. David Alan Gilbert mis->postcopy_remote_fds = g_array_append_val(mis->postcopy_remote_fds, 152000fa4fc8SDr. David Alan Gilbert *pcfd); 152100fa4fc8SDr. David Alan Gilbert } 152200fa4fc8SDr. David Alan Gilbert 152300fa4fc8SDr. David Alan Gilbert /* Unregister a handler for external shared memory postcopy 152400fa4fc8SDr. David Alan Gilbert */ 152500fa4fc8SDr. David Alan Gilbert void postcopy_unregister_shared_ufd(struct PostCopyFD *pcfd) 152600fa4fc8SDr. David Alan Gilbert { 152700fa4fc8SDr. David Alan Gilbert guint i; 152800fa4fc8SDr. David Alan Gilbert MigrationIncomingState *mis = migration_incoming_get_current(); 152900fa4fc8SDr. David Alan Gilbert GArray *pcrfds = mis->postcopy_remote_fds; 153000fa4fc8SDr. David Alan Gilbert 153156559980SJuan Quintela if (!pcrfds) { 153256559980SJuan Quintela /* migration has already finished and freed the array */ 153356559980SJuan Quintela return; 153456559980SJuan Quintela } 153500fa4fc8SDr. David Alan Gilbert for (i = 0; i < pcrfds->len; i++) { 153600fa4fc8SDr. David Alan Gilbert struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i); 153700fa4fc8SDr. David Alan Gilbert if (cur->fd == pcfd->fd) { 153800fa4fc8SDr. David Alan Gilbert mis->postcopy_remote_fds = g_array_remove_index(pcrfds, i); 153900fa4fc8SDr. David Alan Gilbert return; 154000fa4fc8SDr. David Alan Gilbert } 154100fa4fc8SDr. David Alan Gilbert } 154200fa4fc8SDr. David Alan Gilbert } 154336f62f11SPeter Xu 15446720c2b3Smanish.mishra void postcopy_preempt_new_channel(MigrationIncomingState *mis, QEMUFile *file) 154536f62f11SPeter Xu { 154636f62f11SPeter Xu /* 154736f62f11SPeter Xu * The new loading channel has its own threads, so it needs to be 154836f62f11SPeter Xu * blocked too. It's by default true, just be explicit. 154936f62f11SPeter Xu */ 155036f62f11SPeter Xu qemu_file_set_blocking(file, true); 155136f62f11SPeter Xu mis->postcopy_qemufile_dst = file; 15525655aab0SPeter Xu qemu_sem_post(&mis->postcopy_qemufile_dst_done); 155336f62f11SPeter Xu trace_postcopy_preempt_new_channel(); 155436f62f11SPeter Xu } 155536f62f11SPeter Xu 1556f0afaf6cSPeter Xu /* 1557f0afaf6cSPeter Xu * Setup the postcopy preempt channel with the IOC. If ERROR is specified, 1558f0afaf6cSPeter Xu * setup the error instead. This helper will free the ERROR if specified. 1559f0afaf6cSPeter Xu */ 1560d0edb8a1SPeter Xu static void 1561f0afaf6cSPeter Xu postcopy_preempt_send_channel_done(MigrationState *s, 1562f0afaf6cSPeter Xu QIOChannel *ioc, Error *local_err) 1563d0edb8a1SPeter Xu { 1564f0afaf6cSPeter Xu if (local_err) { 1565d0edb8a1SPeter Xu migrate_set_error(s, local_err); 1566d0edb8a1SPeter Xu error_free(local_err); 1567d0edb8a1SPeter Xu } else { 1568d0edb8a1SPeter Xu migration_ioc_register_yank(ioc); 1569d0edb8a1SPeter Xu s->postcopy_qemufile_src = qemu_file_new_output(ioc); 1570d0edb8a1SPeter Xu trace_postcopy_preempt_new_channel(); 1571d0edb8a1SPeter Xu } 1572d0edb8a1SPeter Xu 1573d0edb8a1SPeter Xu /* 1574d0edb8a1SPeter Xu * Kick the waiter in all cases. The waiter should check upon 1575d0edb8a1SPeter Xu * postcopy_qemufile_src to know whether it failed or not. 1576d0edb8a1SPeter Xu */ 1577d0edb8a1SPeter Xu qemu_sem_post(&s->postcopy_qemufile_src_sem); 1578f0afaf6cSPeter Xu } 1579f0afaf6cSPeter Xu 1580f0afaf6cSPeter Xu static void 1581f0afaf6cSPeter Xu postcopy_preempt_tls_handshake(QIOTask *task, gpointer opaque) 1582f0afaf6cSPeter Xu { 1583f0afaf6cSPeter Xu g_autoptr(QIOChannel) ioc = QIO_CHANNEL(qio_task_get_source(task)); 1584f0afaf6cSPeter Xu MigrationState *s = opaque; 1585f0afaf6cSPeter Xu Error *local_err = NULL; 1586f0afaf6cSPeter Xu 1587f0afaf6cSPeter Xu qio_task_propagate_error(task, &local_err); 1588f0afaf6cSPeter Xu postcopy_preempt_send_channel_done(s, ioc, local_err); 1589f0afaf6cSPeter Xu } 1590f0afaf6cSPeter Xu 1591f0afaf6cSPeter Xu static void 1592f0afaf6cSPeter Xu postcopy_preempt_send_channel_new(QIOTask *task, gpointer opaque) 1593f0afaf6cSPeter Xu { 1594f0afaf6cSPeter Xu g_autoptr(QIOChannel) ioc = QIO_CHANNEL(qio_task_get_source(task)); 1595f0afaf6cSPeter Xu MigrationState *s = opaque; 1596f0afaf6cSPeter Xu QIOChannelTLS *tioc; 1597f0afaf6cSPeter Xu Error *local_err = NULL; 1598f0afaf6cSPeter Xu 1599f0afaf6cSPeter Xu if (qio_task_propagate_error(task, &local_err)) { 1600f0afaf6cSPeter Xu goto out; 1601f0afaf6cSPeter Xu } 1602f0afaf6cSPeter Xu 1603f0afaf6cSPeter Xu if (migrate_channel_requires_tls_upgrade(ioc)) { 1604f0afaf6cSPeter Xu tioc = migration_tls_client_create(s, ioc, s->hostname, &local_err); 1605f0afaf6cSPeter Xu if (!tioc) { 1606f0afaf6cSPeter Xu goto out; 1607f0afaf6cSPeter Xu } 1608f0afaf6cSPeter Xu trace_postcopy_preempt_tls_handshake(); 1609f0afaf6cSPeter Xu qio_channel_set_name(QIO_CHANNEL(tioc), "migration-tls-preempt"); 1610f0afaf6cSPeter Xu qio_channel_tls_handshake(tioc, postcopy_preempt_tls_handshake, 1611f0afaf6cSPeter Xu s, NULL, NULL); 1612f0afaf6cSPeter Xu /* Setup the channel until TLS handshake finished */ 1613f0afaf6cSPeter Xu return; 1614f0afaf6cSPeter Xu } 1615f0afaf6cSPeter Xu 1616f0afaf6cSPeter Xu out: 1617f0afaf6cSPeter Xu /* This handles both good and error cases */ 1618f0afaf6cSPeter Xu postcopy_preempt_send_channel_done(s, ioc, local_err); 1619d0edb8a1SPeter Xu } 1620d0edb8a1SPeter Xu 16215655aab0SPeter Xu /* 16225655aab0SPeter Xu * This function will kick off an async task to establish the preempt 16235655aab0SPeter Xu * channel, and wait until the connection setup completed. Returns 0 if 16245655aab0SPeter Xu * channel established, -1 for error. 16255655aab0SPeter Xu */ 16265655aab0SPeter Xu int postcopy_preempt_establish_channel(MigrationState *s) 1627d0edb8a1SPeter Xu { 1628d0edb8a1SPeter Xu /* If preempt not enabled, no need to wait */ 1629d0edb8a1SPeter Xu if (!migrate_postcopy_preempt()) { 1630d0edb8a1SPeter Xu return 0; 1631d0edb8a1SPeter Xu } 1632d0edb8a1SPeter Xu 163306064a67SPeter Xu /* 163406064a67SPeter Xu * Kick off async task to establish preempt channel. Only do so with 163506064a67SPeter Xu * 8.0+ machines, because 7.1/7.2 require the channel to be created in 163606064a67SPeter Xu * setup phase of migration (even if racy in an unreliable network). 163706064a67SPeter Xu */ 163806064a67SPeter Xu if (!s->preempt_pre_7_2) { 16395655aab0SPeter Xu postcopy_preempt_setup(s); 164006064a67SPeter Xu } 16415655aab0SPeter Xu 1642d0edb8a1SPeter Xu /* 1643d0edb8a1SPeter Xu * We need the postcopy preempt channel to be established before 1644d0edb8a1SPeter Xu * starting doing anything. 1645d0edb8a1SPeter Xu */ 1646d0edb8a1SPeter Xu qemu_sem_wait(&s->postcopy_qemufile_src_sem); 1647d0edb8a1SPeter Xu 1648d0edb8a1SPeter Xu return s->postcopy_qemufile_src ? 0 : -1; 1649d0edb8a1SPeter Xu } 1650d0edb8a1SPeter Xu 1651fc063a7bSPeter Xu void postcopy_preempt_setup(MigrationState *s) 165236f62f11SPeter Xu { 1653d0edb8a1SPeter Xu /* Kick an async task to connect */ 1654d0edb8a1SPeter Xu socket_send_channel_create(postcopy_preempt_send_channel_new, s); 165536f62f11SPeter Xu } 165636f62f11SPeter Xu 165760bb3c58SPeter Xu static void postcopy_pause_ram_fast_load(MigrationIncomingState *mis) 165860bb3c58SPeter Xu { 165960bb3c58SPeter Xu trace_postcopy_pause_fast_load(); 166060bb3c58SPeter Xu qemu_mutex_unlock(&mis->postcopy_prio_thread_mutex); 166160bb3c58SPeter Xu qemu_sem_wait(&mis->postcopy_pause_sem_fast_load); 166260bb3c58SPeter Xu qemu_mutex_lock(&mis->postcopy_prio_thread_mutex); 166360bb3c58SPeter Xu trace_postcopy_pause_fast_load_continued(); 166460bb3c58SPeter Xu } 166560bb3c58SPeter Xu 16666621883fSPeter Xu static bool preempt_thread_should_run(MigrationIncomingState *mis) 16676621883fSPeter Xu { 16686621883fSPeter Xu return mis->preempt_thread_status != PREEMPT_THREAD_QUIT; 16696621883fSPeter Xu } 16706621883fSPeter Xu 167136f62f11SPeter Xu void *postcopy_preempt_thread(void *opaque) 167236f62f11SPeter Xu { 167336f62f11SPeter Xu MigrationIncomingState *mis = opaque; 167436f62f11SPeter Xu int ret; 167536f62f11SPeter Xu 167636f62f11SPeter Xu trace_postcopy_preempt_thread_entry(); 167736f62f11SPeter Xu 167836f62f11SPeter Xu rcu_register_thread(); 167936f62f11SPeter Xu 168036f62f11SPeter Xu qemu_sem_post(&mis->thread_sync_sem); 168136f62f11SPeter Xu 1682a5d35dc7SPeter Xu /* 1683a5d35dc7SPeter Xu * The preempt channel is established in asynchronous way. Wait 1684a5d35dc7SPeter Xu * for its completion. 1685a5d35dc7SPeter Xu */ 1686a5d35dc7SPeter Xu qemu_sem_wait(&mis->postcopy_qemufile_dst_done); 1687a5d35dc7SPeter Xu 168836f62f11SPeter Xu /* Sending RAM_SAVE_FLAG_EOS to terminate this thread */ 168960bb3c58SPeter Xu qemu_mutex_lock(&mis->postcopy_prio_thread_mutex); 16906621883fSPeter Xu while (preempt_thread_should_run(mis)) { 169160bb3c58SPeter Xu ret = ram_load_postcopy(mis->postcopy_qemufile_dst, 169260bb3c58SPeter Xu RAM_CHANNEL_POSTCOPY); 169360bb3c58SPeter Xu /* If error happened, go into recovery routine */ 16946621883fSPeter Xu if (ret && preempt_thread_should_run(mis)) { 169560bb3c58SPeter Xu postcopy_pause_ram_fast_load(mis); 169660bb3c58SPeter Xu } else { 169760bb3c58SPeter Xu /* We're done */ 169860bb3c58SPeter Xu break; 169960bb3c58SPeter Xu } 170060bb3c58SPeter Xu } 170160bb3c58SPeter Xu qemu_mutex_unlock(&mis->postcopy_prio_thread_mutex); 170236f62f11SPeter Xu 170336f62f11SPeter Xu rcu_unregister_thread(); 170436f62f11SPeter Xu 170536f62f11SPeter Xu trace_postcopy_preempt_thread_exit(); 170636f62f11SPeter Xu 170760bb3c58SPeter Xu return NULL; 170836f62f11SPeter Xu } 1709