1 /*
2 * QEMU System Emulator
3 *
4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
6 *
7 * Authors:
8 * Juan Quintela <quintela@redhat.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 * THE SOFTWARE.
27 */
28
29 #include "qemu/osdep.h"
30 #include "qemu/cutils.h"
31 #include "qemu/bitops.h"
32 #include "qemu/bitmap.h"
33 #include "qemu/madvise.h"
34 #include "qemu/main-loop.h"
35 #include "xbzrle.h"
36 #include "ram.h"
37 #include "migration.h"
38 #include "migration-stats.h"
39 #include "migration/register.h"
40 #include "migration/misc.h"
41 #include "qemu-file.h"
42 #include "postcopy-ram.h"
43 #include "page_cache.h"
44 #include "qemu/error-report.h"
45 #include "qapi/error.h"
46 #include "qapi/qapi-types-migration.h"
47 #include "qapi/qapi-events-migration.h"
48 #include "qapi/qapi-commands-migration.h"
49 #include "qapi/qmp/qerror.h"
50 #include "trace.h"
51 #include "exec/ram_addr.h"
52 #include "exec/target_page.h"
53 #include "qemu/rcu_queue.h"
54 #include "migration/colo.h"
55 #include "sysemu/cpu-throttle.h"
56 #include "savevm.h"
57 #include "qemu/iov.h"
58 #include "multifd.h"
59 #include "sysemu/runstate.h"
60 #include "rdma.h"
61 #include "options.h"
62 #include "sysemu/dirtylimit.h"
63 #include "sysemu/kvm.h"
64
65 #include "hw/boards.h" /* for machine_dump_guest_core() */
66
67 #if defined(__linux__)
68 #include "qemu/userfaultfd.h"
69 #endif /* defined(__linux__) */
70
71 /***********************************************************/
72 /* ram save/restore */
73
74 /*
75 * RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
76 * worked for pages that were filled with the same char. We switched
77 * it to only search for the zero value. And to avoid confusion with
78 * RAM_SAVE_FLAG_COMPRESS_PAGE just rename it.
79 *
80 * RAM_SAVE_FLAG_FULL was obsoleted in 2009.
81 *
82 * RAM_SAVE_FLAG_COMPRESS_PAGE (0x100) was removed in QEMU 9.1.
83 */
84 #define RAM_SAVE_FLAG_FULL 0x01
85 #define RAM_SAVE_FLAG_ZERO 0x02
86 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
87 #define RAM_SAVE_FLAG_PAGE 0x08
88 #define RAM_SAVE_FLAG_EOS 0x10
89 #define RAM_SAVE_FLAG_CONTINUE 0x20
90 #define RAM_SAVE_FLAG_XBZRLE 0x40
91 /* 0x80 is reserved in rdma.h for RAM_SAVE_FLAG_HOOK */
92 #define RAM_SAVE_FLAG_MULTIFD_FLUSH 0x200
93 /* We can't use any flag that is bigger than 0x200 */
94
95 /*
96 * mapped-ram migration supports O_DIRECT, so we need to make sure the
97 * userspace buffer, the IO operation size and the file offset are
98 * aligned according to the underlying device's block size. The first
99 * two are already aligned to page size, but we need to add padding to
100 * the file to align the offset. We cannot read the block size
101 * dynamically because the migration file can be moved between
102 * different systems, so use 1M to cover most block sizes and to keep
103 * the file offset aligned at page size as well.
104 */
105 #define MAPPED_RAM_FILE_OFFSET_ALIGNMENT 0x100000
106
107 /*
108 * When doing mapped-ram migration, this is the amount we read from
109 * the pages region in the migration file at a time.
110 */
111 #define MAPPED_RAM_LOAD_BUF_SIZE 0x100000
112
113 XBZRLECacheStats xbzrle_counters;
114
115 /* used by the search for pages to send */
116 struct PageSearchStatus {
117 /* The migration channel used for a specific host page */
118 QEMUFile *pss_channel;
119 /* Last block from where we have sent data */
120 RAMBlock *last_sent_block;
121 /* Current block being searched */
122 RAMBlock *block;
123 /* Current page to search from */
124 unsigned long page;
125 /* Set once we wrap around */
126 bool complete_round;
127 /* Whether we're sending a host page */
128 bool host_page_sending;
129 /* The start/end of current host page. Invalid if host_page_sending==false */
130 unsigned long host_page_start;
131 unsigned long host_page_end;
132 };
133 typedef struct PageSearchStatus PageSearchStatus;
134
135 /* struct contains XBZRLE cache and a static page
136 used by the compression */
137 static struct {
138 /* buffer used for XBZRLE encoding */
139 uint8_t *encoded_buf;
140 /* buffer for storing page content */
141 uint8_t *current_buf;
142 /* Cache for XBZRLE, Protected by lock. */
143 PageCache *cache;
144 QemuMutex lock;
145 /* it will store a page full of zeros */
146 uint8_t *zero_target_page;
147 /* buffer used for XBZRLE decoding */
148 uint8_t *decoded_buf;
149 } XBZRLE;
150
XBZRLE_cache_lock(void)151 static void XBZRLE_cache_lock(void)
152 {
153 if (migrate_xbzrle()) {
154 qemu_mutex_lock(&XBZRLE.lock);
155 }
156 }
157
XBZRLE_cache_unlock(void)158 static void XBZRLE_cache_unlock(void)
159 {
160 if (migrate_xbzrle()) {
161 qemu_mutex_unlock(&XBZRLE.lock);
162 }
163 }
164
165 /**
166 * xbzrle_cache_resize: resize the xbzrle cache
167 *
168 * This function is called from migrate_params_apply in main
169 * thread, possibly while a migration is in progress. A running
170 * migration may be using the cache and might finish during this call,
171 * hence changes to the cache are protected by XBZRLE.lock().
172 *
173 * Returns 0 for success or -1 for error
174 *
175 * @new_size: new cache size
176 * @errp: set *errp if the check failed, with reason
177 */
xbzrle_cache_resize(uint64_t new_size,Error ** errp)178 int xbzrle_cache_resize(uint64_t new_size, Error **errp)
179 {
180 PageCache *new_cache;
181 int64_t ret = 0;
182
183 /* Check for truncation */
184 if (new_size != (size_t)new_size) {
185 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
186 "exceeding address space");
187 return -1;
188 }
189
190 if (new_size == migrate_xbzrle_cache_size()) {
191 /* nothing to do */
192 return 0;
193 }
194
195 XBZRLE_cache_lock();
196
197 if (XBZRLE.cache != NULL) {
198 new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp);
199 if (!new_cache) {
200 ret = -1;
201 goto out;
202 }
203
204 cache_fini(XBZRLE.cache);
205 XBZRLE.cache = new_cache;
206 }
207 out:
208 XBZRLE_cache_unlock();
209 return ret;
210 }
211
postcopy_preempt_active(void)212 static bool postcopy_preempt_active(void)
213 {
214 return migrate_postcopy_preempt() && migration_in_postcopy();
215 }
216
migrate_ram_is_ignored(RAMBlock * block)217 bool migrate_ram_is_ignored(RAMBlock *block)
218 {
219 return !qemu_ram_is_migratable(block) ||
220 (migrate_ignore_shared() && qemu_ram_is_shared(block)
221 && qemu_ram_is_named_file(block));
222 }
223
224 #undef RAMBLOCK_FOREACH
225
foreach_not_ignored_block(RAMBlockIterFunc func,void * opaque)226 int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
227 {
228 RAMBlock *block;
229 int ret = 0;
230
231 RCU_READ_LOCK_GUARD();
232
233 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
234 ret = func(block, opaque);
235 if (ret) {
236 break;
237 }
238 }
239 return ret;
240 }
241
ramblock_recv_map_init(void)242 static void ramblock_recv_map_init(void)
243 {
244 RAMBlock *rb;
245
246 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
247 assert(!rb->receivedmap);
248 rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
249 }
250 }
251
ramblock_recv_bitmap_test(RAMBlock * rb,void * host_addr)252 int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr)
253 {
254 return test_bit(ramblock_recv_bitmap_offset(host_addr, rb),
255 rb->receivedmap);
256 }
257
ramblock_recv_bitmap_test_byte_offset(RAMBlock * rb,uint64_t byte_offset)258 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset)
259 {
260 return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
261 }
262
ramblock_recv_bitmap_set(RAMBlock * rb,void * host_addr)263 void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr)
264 {
265 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
266 }
267
ramblock_recv_bitmap_set_range(RAMBlock * rb,void * host_addr,size_t nr)268 void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr,
269 size_t nr)
270 {
271 bitmap_set_atomic(rb->receivedmap,
272 ramblock_recv_bitmap_offset(host_addr, rb),
273 nr);
274 }
275
ramblock_recv_bitmap_set_offset(RAMBlock * rb,uint64_t byte_offset)276 void ramblock_recv_bitmap_set_offset(RAMBlock *rb, uint64_t byte_offset)
277 {
278 set_bit_atomic(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
279 }
280 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
281
282 /*
283 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
284 *
285 * Returns >0 if success with sent bytes, or <0 if error.
286 */
ramblock_recv_bitmap_send(QEMUFile * file,const char * block_name)287 int64_t ramblock_recv_bitmap_send(QEMUFile *file,
288 const char *block_name)
289 {
290 RAMBlock *block = qemu_ram_block_by_name(block_name);
291 unsigned long *le_bitmap, nbits;
292 uint64_t size;
293
294 if (!block) {
295 error_report("%s: invalid block name: %s", __func__, block_name);
296 return -1;
297 }
298
299 nbits = block->postcopy_length >> TARGET_PAGE_BITS;
300
301 /*
302 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
303 * machines we may need 4 more bytes for padding (see below
304 * comment). So extend it a bit before hand.
305 */
306 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
307
308 /*
309 * Always use little endian when sending the bitmap. This is
310 * required that when source and destination VMs are not using the
311 * same endianness. (Note: big endian won't work.)
312 */
313 bitmap_to_le(le_bitmap, block->receivedmap, nbits);
314
315 /* Size of the bitmap, in bytes */
316 size = DIV_ROUND_UP(nbits, 8);
317
318 /*
319 * size is always aligned to 8 bytes for 64bit machines, but it
320 * may not be true for 32bit machines. We need this padding to
321 * make sure the migration can survive even between 32bit and
322 * 64bit machines.
323 */
324 size = ROUND_UP(size, 8);
325
326 qemu_put_be64(file, size);
327 qemu_put_buffer(file, (const uint8_t *)le_bitmap, size);
328 g_free(le_bitmap);
329 /*
330 * Mark as an end, in case the middle part is screwed up due to
331 * some "mysterious" reason.
332 */
333 qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING);
334 int ret = qemu_fflush(file);
335 if (ret) {
336 return ret;
337 }
338
339 return size + sizeof(size);
340 }
341
342 /*
343 * An outstanding page request, on the source, having been received
344 * and queued
345 */
346 struct RAMSrcPageRequest {
347 RAMBlock *rb;
348 hwaddr offset;
349 hwaddr len;
350
351 QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req;
352 };
353
354 /* State of RAM for migration */
355 struct RAMState {
356 /*
357 * PageSearchStatus structures for the channels when send pages.
358 * Protected by the bitmap_mutex.
359 */
360 PageSearchStatus pss[RAM_CHANNEL_MAX];
361 /* UFFD file descriptor, used in 'write-tracking' migration */
362 int uffdio_fd;
363 /* total ram size in bytes */
364 uint64_t ram_bytes_total;
365 /* Last block that we have visited searching for dirty pages */
366 RAMBlock *last_seen_block;
367 /* Last dirty target page we have sent */
368 ram_addr_t last_page;
369 /* last ram version we have seen */
370 uint32_t last_version;
371 /* How many times we have dirty too many pages */
372 int dirty_rate_high_cnt;
373 /* these variables are used for bitmap sync */
374 /* last time we did a full bitmap_sync */
375 int64_t time_last_bitmap_sync;
376 /* bytes transferred at start_time */
377 uint64_t bytes_xfer_prev;
378 /* number of dirty pages since start_time */
379 uint64_t num_dirty_pages_period;
380 /* xbzrle misses since the beginning of the period */
381 uint64_t xbzrle_cache_miss_prev;
382 /* Amount of xbzrle pages since the beginning of the period */
383 uint64_t xbzrle_pages_prev;
384 /* Amount of xbzrle encoded bytes since the beginning of the period */
385 uint64_t xbzrle_bytes_prev;
386 /* Are we really using XBZRLE (e.g., after the first round). */
387 bool xbzrle_started;
388 /* Are we on the last stage of migration */
389 bool last_stage;
390
391 /* total handled target pages at the beginning of period */
392 uint64_t target_page_count_prev;
393 /* total handled target pages since start */
394 uint64_t target_page_count;
395 /* number of dirty bits in the bitmap */
396 uint64_t migration_dirty_pages;
397 /*
398 * Protects:
399 * - dirty/clear bitmap
400 * - migration_dirty_pages
401 * - pss structures
402 */
403 QemuMutex bitmap_mutex;
404 /* The RAMBlock used in the last src_page_requests */
405 RAMBlock *last_req_rb;
406 /* Queue of outstanding page requests from the destination */
407 QemuMutex src_page_req_mutex;
408 QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
409
410 /*
411 * This is only used when postcopy is in recovery phase, to communicate
412 * between the migration thread and the return path thread on dirty
413 * bitmap synchronizations. This field is unused in other stages of
414 * RAM migration.
415 */
416 unsigned int postcopy_bmap_sync_requested;
417 };
418 typedef struct RAMState RAMState;
419
420 static RAMState *ram_state;
421
422 static NotifierWithReturnList precopy_notifier_list;
423
424 /* Whether postcopy has queued requests? */
postcopy_has_request(RAMState * rs)425 static bool postcopy_has_request(RAMState *rs)
426 {
427 return !QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests);
428 }
429
precopy_infrastructure_init(void)430 void precopy_infrastructure_init(void)
431 {
432 notifier_with_return_list_init(&precopy_notifier_list);
433 }
434
precopy_add_notifier(NotifierWithReturn * n)435 void precopy_add_notifier(NotifierWithReturn *n)
436 {
437 notifier_with_return_list_add(&precopy_notifier_list, n);
438 }
439
precopy_remove_notifier(NotifierWithReturn * n)440 void precopy_remove_notifier(NotifierWithReturn *n)
441 {
442 notifier_with_return_remove(n);
443 }
444
precopy_notify(PrecopyNotifyReason reason,Error ** errp)445 int precopy_notify(PrecopyNotifyReason reason, Error **errp)
446 {
447 PrecopyNotifyData pnd;
448 pnd.reason = reason;
449
450 return notifier_with_return_list_notify(&precopy_notifier_list, &pnd, errp);
451 }
452
ram_bytes_remaining(void)453 uint64_t ram_bytes_remaining(void)
454 {
455 return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
456 0;
457 }
458
ram_transferred_add(uint64_t bytes)459 void ram_transferred_add(uint64_t bytes)
460 {
461 if (runstate_is_running()) {
462 stat64_add(&mig_stats.precopy_bytes, bytes);
463 } else if (migration_in_postcopy()) {
464 stat64_add(&mig_stats.postcopy_bytes, bytes);
465 } else {
466 stat64_add(&mig_stats.downtime_bytes, bytes);
467 }
468 }
469
470 struct MigrationOps {
471 int (*ram_save_target_page)(RAMState *rs, PageSearchStatus *pss);
472 };
473 typedef struct MigrationOps MigrationOps;
474
475 MigrationOps *migration_ops;
476
477 static int ram_save_host_page_urgent(PageSearchStatus *pss);
478
479 /* NOTE: page is the PFN not real ram_addr_t. */
pss_init(PageSearchStatus * pss,RAMBlock * rb,ram_addr_t page)480 static void pss_init(PageSearchStatus *pss, RAMBlock *rb, ram_addr_t page)
481 {
482 pss->block = rb;
483 pss->page = page;
484 pss->complete_round = false;
485 }
486
487 /*
488 * Check whether two PSSs are actively sending the same page. Return true
489 * if it is, false otherwise.
490 */
pss_overlap(PageSearchStatus * pss1,PageSearchStatus * pss2)491 static bool pss_overlap(PageSearchStatus *pss1, PageSearchStatus *pss2)
492 {
493 return pss1->host_page_sending && pss2->host_page_sending &&
494 (pss1->host_page_start == pss2->host_page_start);
495 }
496
497 /**
498 * save_page_header: write page header to wire
499 *
500 * If this is the 1st block, it also writes the block identification
501 *
502 * Returns the number of bytes written
503 *
504 * @pss: current PSS channel status
505 * @block: block that contains the page we want to send
506 * @offset: offset inside the block for the page
507 * in the lower bits, it contains flags
508 */
save_page_header(PageSearchStatus * pss,QEMUFile * f,RAMBlock * block,ram_addr_t offset)509 static size_t save_page_header(PageSearchStatus *pss, QEMUFile *f,
510 RAMBlock *block, ram_addr_t offset)
511 {
512 size_t size, len;
513 bool same_block = (block == pss->last_sent_block);
514
515 if (same_block) {
516 offset |= RAM_SAVE_FLAG_CONTINUE;
517 }
518 qemu_put_be64(f, offset);
519 size = 8;
520
521 if (!same_block) {
522 len = strlen(block->idstr);
523 qemu_put_byte(f, len);
524 qemu_put_buffer(f, (uint8_t *)block->idstr, len);
525 size += 1 + len;
526 pss->last_sent_block = block;
527 }
528 return size;
529 }
530
531 /**
532 * mig_throttle_guest_down: throttle down the guest
533 *
534 * Reduce amount of guest cpu execution to hopefully slow down memory
535 * writes. If guest dirty memory rate is reduced below the rate at
536 * which we can transfer pages to the destination then we should be
537 * able to complete migration. Some workloads dirty memory way too
538 * fast and will not effectively converge, even with auto-converge.
539 */
mig_throttle_guest_down(uint64_t bytes_dirty_period,uint64_t bytes_dirty_threshold)540 static void mig_throttle_guest_down(uint64_t bytes_dirty_period,
541 uint64_t bytes_dirty_threshold)
542 {
543 uint64_t pct_initial = migrate_cpu_throttle_initial();
544 uint64_t pct_increment = migrate_cpu_throttle_increment();
545 bool pct_tailslow = migrate_cpu_throttle_tailslow();
546 int pct_max = migrate_max_cpu_throttle();
547
548 uint64_t throttle_now = cpu_throttle_get_percentage();
549 uint64_t cpu_now, cpu_ideal, throttle_inc;
550
551 /* We have not started throttling yet. Let's start it. */
552 if (!cpu_throttle_active()) {
553 cpu_throttle_set(pct_initial);
554 } else {
555 /* Throttling already on, just increase the rate */
556 if (!pct_tailslow) {
557 throttle_inc = pct_increment;
558 } else {
559 /* Compute the ideal CPU percentage used by Guest, which may
560 * make the dirty rate match the dirty rate threshold. */
561 cpu_now = 100 - throttle_now;
562 cpu_ideal = cpu_now * (bytes_dirty_threshold * 1.0 /
563 bytes_dirty_period);
564 throttle_inc = MIN(cpu_now - cpu_ideal, pct_increment);
565 }
566 cpu_throttle_set(MIN(throttle_now + throttle_inc, pct_max));
567 }
568 }
569
mig_throttle_counter_reset(void)570 void mig_throttle_counter_reset(void)
571 {
572 RAMState *rs = ram_state;
573
574 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
575 rs->num_dirty_pages_period = 0;
576 rs->bytes_xfer_prev = migration_transferred_bytes();
577 }
578
579 /**
580 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
581 *
582 * @current_addr: address for the zero page
583 *
584 * Update the xbzrle cache to reflect a page that's been sent as all 0.
585 * The important thing is that a stale (not-yet-0'd) page be replaced
586 * by the new data.
587 * As a bonus, if the page wasn't in the cache it gets added so that
588 * when a small write is made into the 0'd page it gets XBZRLE sent.
589 */
xbzrle_cache_zero_page(ram_addr_t current_addr)590 static void xbzrle_cache_zero_page(ram_addr_t current_addr)
591 {
592 /* We don't care if this fails to allocate a new cache page
593 * as long as it updated an old one */
594 cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page,
595 stat64_get(&mig_stats.dirty_sync_count));
596 }
597
598 #define ENCODING_FLAG_XBZRLE 0x1
599
600 /**
601 * save_xbzrle_page: compress and send current page
602 *
603 * Returns: 1 means that we wrote the page
604 * 0 means that page is identical to the one already sent
605 * -1 means that xbzrle would be longer than normal
606 *
607 * @rs: current RAM state
608 * @pss: current PSS channel
609 * @current_data: pointer to the address of the page contents
610 * @current_addr: addr of the page
611 * @block: block that contains the page we want to send
612 * @offset: offset inside the block for the page
613 */
save_xbzrle_page(RAMState * rs,PageSearchStatus * pss,uint8_t ** current_data,ram_addr_t current_addr,RAMBlock * block,ram_addr_t offset)614 static int save_xbzrle_page(RAMState *rs, PageSearchStatus *pss,
615 uint8_t **current_data, ram_addr_t current_addr,
616 RAMBlock *block, ram_addr_t offset)
617 {
618 int encoded_len = 0, bytes_xbzrle;
619 uint8_t *prev_cached_page;
620 QEMUFile *file = pss->pss_channel;
621 uint64_t generation = stat64_get(&mig_stats.dirty_sync_count);
622
623 if (!cache_is_cached(XBZRLE.cache, current_addr, generation)) {
624 xbzrle_counters.cache_miss++;
625 if (!rs->last_stage) {
626 if (cache_insert(XBZRLE.cache, current_addr, *current_data,
627 generation) == -1) {
628 return -1;
629 } else {
630 /* update *current_data when the page has been
631 inserted into cache */
632 *current_data = get_cached_data(XBZRLE.cache, current_addr);
633 }
634 }
635 return -1;
636 }
637
638 /*
639 * Reaching here means the page has hit the xbzrle cache, no matter what
640 * encoding result it is (normal encoding, overflow or skipping the page),
641 * count the page as encoded. This is used to calculate the encoding rate.
642 *
643 * Example: 2 pages (8KB) being encoded, first page encoding generates 2KB,
644 * 2nd page turns out to be skipped (i.e. no new bytes written to the
645 * page), the overall encoding rate will be 8KB / 2KB = 4, which has the
646 * skipped page included. In this way, the encoding rate can tell if the
647 * guest page is good for xbzrle encoding.
648 */
649 xbzrle_counters.pages++;
650 prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
651
652 /* save current buffer into memory */
653 memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
654
655 /* XBZRLE encoding (if there is no overflow) */
656 encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
657 TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
658 TARGET_PAGE_SIZE);
659
660 /*
661 * Update the cache contents, so that it corresponds to the data
662 * sent, in all cases except where we skip the page.
663 */
664 if (!rs->last_stage && encoded_len != 0) {
665 memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
666 /*
667 * In the case where we couldn't compress, ensure that the caller
668 * sends the data from the cache, since the guest might have
669 * changed the RAM since we copied it.
670 */
671 *current_data = prev_cached_page;
672 }
673
674 if (encoded_len == 0) {
675 trace_save_xbzrle_page_skipping();
676 return 0;
677 } else if (encoded_len == -1) {
678 trace_save_xbzrle_page_overflow();
679 xbzrle_counters.overflow++;
680 xbzrle_counters.bytes += TARGET_PAGE_SIZE;
681 return -1;
682 }
683
684 /* Send XBZRLE based compressed page */
685 bytes_xbzrle = save_page_header(pss, pss->pss_channel, block,
686 offset | RAM_SAVE_FLAG_XBZRLE);
687 qemu_put_byte(file, ENCODING_FLAG_XBZRLE);
688 qemu_put_be16(file, encoded_len);
689 qemu_put_buffer(file, XBZRLE.encoded_buf, encoded_len);
690 bytes_xbzrle += encoded_len + 1 + 2;
691 /*
692 * The xbzrle encoded bytes don't count the 8 byte header with
693 * RAM_SAVE_FLAG_CONTINUE.
694 */
695 xbzrle_counters.bytes += bytes_xbzrle - 8;
696 ram_transferred_add(bytes_xbzrle);
697
698 return 1;
699 }
700
701 /**
702 * pss_find_next_dirty: find the next dirty page of current ramblock
703 *
704 * This function updates pss->page to point to the next dirty page index
705 * within the ramblock to migrate, or the end of ramblock when nothing
706 * found. Note that when pss->host_page_sending==true it means we're
707 * during sending a host page, so we won't look for dirty page that is
708 * outside the host page boundary.
709 *
710 * @pss: the current page search status
711 */
pss_find_next_dirty(PageSearchStatus * pss)712 static void pss_find_next_dirty(PageSearchStatus *pss)
713 {
714 RAMBlock *rb = pss->block;
715 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
716 unsigned long *bitmap = rb->bmap;
717
718 if (migrate_ram_is_ignored(rb)) {
719 /* Points directly to the end, so we know no dirty page */
720 pss->page = size;
721 return;
722 }
723
724 /*
725 * If during sending a host page, only look for dirty pages within the
726 * current host page being send.
727 */
728 if (pss->host_page_sending) {
729 assert(pss->host_page_end);
730 size = MIN(size, pss->host_page_end);
731 }
732
733 pss->page = find_next_bit(bitmap, size, pss->page);
734 }
735
migration_clear_memory_region_dirty_bitmap(RAMBlock * rb,unsigned long page)736 static void migration_clear_memory_region_dirty_bitmap(RAMBlock *rb,
737 unsigned long page)
738 {
739 uint8_t shift;
740 hwaddr size, start;
741
742 if (!rb->clear_bmap || !clear_bmap_test_and_clear(rb, page)) {
743 return;
744 }
745
746 shift = rb->clear_bmap_shift;
747 /*
748 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
749 * can make things easier sometimes since then start address
750 * of the small chunk will always be 64 pages aligned so the
751 * bitmap will always be aligned to unsigned long. We should
752 * even be able to remove this restriction but I'm simply
753 * keeping it.
754 */
755 assert(shift >= 6);
756
757 size = 1ULL << (TARGET_PAGE_BITS + shift);
758 start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size);
759 trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page);
760 memory_region_clear_dirty_bitmap(rb->mr, start, size);
761 }
762
763 static void
migration_clear_memory_region_dirty_bitmap_range(RAMBlock * rb,unsigned long start,unsigned long npages)764 migration_clear_memory_region_dirty_bitmap_range(RAMBlock *rb,
765 unsigned long start,
766 unsigned long npages)
767 {
768 unsigned long i, chunk_pages = 1UL << rb->clear_bmap_shift;
769 unsigned long chunk_start = QEMU_ALIGN_DOWN(start, chunk_pages);
770 unsigned long chunk_end = QEMU_ALIGN_UP(start + npages, chunk_pages);
771
772 /*
773 * Clear pages from start to start + npages - 1, so the end boundary is
774 * exclusive.
775 */
776 for (i = chunk_start; i < chunk_end; i += chunk_pages) {
777 migration_clear_memory_region_dirty_bitmap(rb, i);
778 }
779 }
780
781 /*
782 * colo_bitmap_find_diry:find contiguous dirty pages from start
783 *
784 * Returns the page offset within memory region of the start of the contiguout
785 * dirty page
786 *
787 * @rs: current RAM state
788 * @rb: RAMBlock where to search for dirty pages
789 * @start: page where we start the search
790 * @num: the number of contiguous dirty pages
791 */
792 static inline
colo_bitmap_find_dirty(RAMState * rs,RAMBlock * rb,unsigned long start,unsigned long * num)793 unsigned long colo_bitmap_find_dirty(RAMState *rs, RAMBlock *rb,
794 unsigned long start, unsigned long *num)
795 {
796 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
797 unsigned long *bitmap = rb->bmap;
798 unsigned long first, next;
799
800 *num = 0;
801
802 if (migrate_ram_is_ignored(rb)) {
803 return size;
804 }
805
806 first = find_next_bit(bitmap, size, start);
807 if (first >= size) {
808 return first;
809 }
810 next = find_next_zero_bit(bitmap, size, first + 1);
811 assert(next >= first);
812 *num = next - first;
813 return first;
814 }
815
migration_bitmap_clear_dirty(RAMState * rs,RAMBlock * rb,unsigned long page)816 static inline bool migration_bitmap_clear_dirty(RAMState *rs,
817 RAMBlock *rb,
818 unsigned long page)
819 {
820 bool ret;
821
822 /*
823 * Clear dirty bitmap if needed. This _must_ be called before we
824 * send any of the page in the chunk because we need to make sure
825 * we can capture further page content changes when we sync dirty
826 * log the next time. So as long as we are going to send any of
827 * the page in the chunk we clear the remote dirty bitmap for all.
828 * Clearing it earlier won't be a problem, but too late will.
829 */
830 migration_clear_memory_region_dirty_bitmap(rb, page);
831
832 ret = test_and_clear_bit(page, rb->bmap);
833 if (ret) {
834 rs->migration_dirty_pages--;
835 }
836
837 return ret;
838 }
839
dirty_bitmap_clear_section(MemoryRegionSection * section,void * opaque)840 static void dirty_bitmap_clear_section(MemoryRegionSection *section,
841 void *opaque)
842 {
843 const hwaddr offset = section->offset_within_region;
844 const hwaddr size = int128_get64(section->size);
845 const unsigned long start = offset >> TARGET_PAGE_BITS;
846 const unsigned long npages = size >> TARGET_PAGE_BITS;
847 RAMBlock *rb = section->mr->ram_block;
848 uint64_t *cleared_bits = opaque;
849
850 /*
851 * We don't grab ram_state->bitmap_mutex because we expect to run
852 * only when starting migration or during postcopy recovery where
853 * we don't have concurrent access.
854 */
855 if (!migration_in_postcopy() && !migrate_background_snapshot()) {
856 migration_clear_memory_region_dirty_bitmap_range(rb, start, npages);
857 }
858 *cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages);
859 bitmap_clear(rb->bmap, start, npages);
860 }
861
862 /*
863 * Exclude all dirty pages from migration that fall into a discarded range as
864 * managed by a RamDiscardManager responsible for the mapped memory region of
865 * the RAMBlock. Clear the corresponding bits in the dirty bitmaps.
866 *
867 * Discarded pages ("logically unplugged") have undefined content and must
868 * not get migrated, because even reading these pages for migration might
869 * result in undesired behavior.
870 *
871 * Returns the number of cleared bits in the RAMBlock dirty bitmap.
872 *
873 * Note: The result is only stable while migrating (precopy/postcopy).
874 */
ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock * rb)875 static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock *rb)
876 {
877 uint64_t cleared_bits = 0;
878
879 if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) {
880 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
881 MemoryRegionSection section = {
882 .mr = rb->mr,
883 .offset_within_region = 0,
884 .size = int128_make64(qemu_ram_get_used_length(rb)),
885 };
886
887 ram_discard_manager_replay_discarded(rdm, §ion,
888 dirty_bitmap_clear_section,
889 &cleared_bits);
890 }
891 return cleared_bits;
892 }
893
894 /*
895 * Check if a host-page aligned page falls into a discarded range as managed by
896 * a RamDiscardManager responsible for the mapped memory region of the RAMBlock.
897 *
898 * Note: The result is only stable while migrating (precopy/postcopy).
899 */
ramblock_page_is_discarded(RAMBlock * rb,ram_addr_t start)900 bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start)
901 {
902 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
903 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
904 MemoryRegionSection section = {
905 .mr = rb->mr,
906 .offset_within_region = start,
907 .size = int128_make64(qemu_ram_pagesize(rb)),
908 };
909
910 return !ram_discard_manager_is_populated(rdm, §ion);
911 }
912 return false;
913 }
914
915 /* Called with RCU critical section */
ramblock_sync_dirty_bitmap(RAMState * rs,RAMBlock * rb)916 static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb)
917 {
918 uint64_t new_dirty_pages =
919 cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length);
920
921 rs->migration_dirty_pages += new_dirty_pages;
922 rs->num_dirty_pages_period += new_dirty_pages;
923 }
924
925 /**
926 * ram_pagesize_summary: calculate all the pagesizes of a VM
927 *
928 * Returns a summary bitmap of the page sizes of all RAMBlocks
929 *
930 * For VMs with just normal pages this is equivalent to the host page
931 * size. If it's got some huge pages then it's the OR of all the
932 * different page sizes.
933 */
ram_pagesize_summary(void)934 uint64_t ram_pagesize_summary(void)
935 {
936 RAMBlock *block;
937 uint64_t summary = 0;
938
939 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
940 summary |= block->page_size;
941 }
942
943 return summary;
944 }
945
ram_get_total_transferred_pages(void)946 uint64_t ram_get_total_transferred_pages(void)
947 {
948 return stat64_get(&mig_stats.normal_pages) +
949 stat64_get(&mig_stats.zero_pages) +
950 xbzrle_counters.pages;
951 }
952
migration_update_rates(RAMState * rs,int64_t end_time)953 static void migration_update_rates(RAMState *rs, int64_t end_time)
954 {
955 uint64_t page_count = rs->target_page_count - rs->target_page_count_prev;
956
957 /* calculate period counters */
958 stat64_set(&mig_stats.dirty_pages_rate,
959 rs->num_dirty_pages_period * 1000 /
960 (end_time - rs->time_last_bitmap_sync));
961
962 if (!page_count) {
963 return;
964 }
965
966 if (migrate_xbzrle()) {
967 double encoded_size, unencoded_size;
968
969 xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
970 rs->xbzrle_cache_miss_prev) / page_count;
971 rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
972 unencoded_size = (xbzrle_counters.pages - rs->xbzrle_pages_prev) *
973 TARGET_PAGE_SIZE;
974 encoded_size = xbzrle_counters.bytes - rs->xbzrle_bytes_prev;
975 if (xbzrle_counters.pages == rs->xbzrle_pages_prev || !encoded_size) {
976 xbzrle_counters.encoding_rate = 0;
977 } else {
978 xbzrle_counters.encoding_rate = unencoded_size / encoded_size;
979 }
980 rs->xbzrle_pages_prev = xbzrle_counters.pages;
981 rs->xbzrle_bytes_prev = xbzrle_counters.bytes;
982 }
983 }
984
985 /*
986 * Enable dirty-limit to throttle down the guest
987 */
migration_dirty_limit_guest(void)988 static void migration_dirty_limit_guest(void)
989 {
990 /*
991 * dirty page rate quota for all vCPUs fetched from
992 * migration parameter 'vcpu_dirty_limit'
993 */
994 static int64_t quota_dirtyrate;
995 MigrationState *s = migrate_get_current();
996
997 /*
998 * If dirty limit already enabled and migration parameter
999 * vcpu-dirty-limit untouched.
1000 */
1001 if (dirtylimit_in_service() &&
1002 quota_dirtyrate == s->parameters.vcpu_dirty_limit) {
1003 return;
1004 }
1005
1006 quota_dirtyrate = s->parameters.vcpu_dirty_limit;
1007
1008 /*
1009 * Set all vCPU a quota dirtyrate, note that the second
1010 * parameter will be ignored if setting all vCPU for the vm
1011 */
1012 qmp_set_vcpu_dirty_limit(false, -1, quota_dirtyrate, NULL);
1013 trace_migration_dirty_limit_guest(quota_dirtyrate);
1014 }
1015
migration_trigger_throttle(RAMState * rs)1016 static void migration_trigger_throttle(RAMState *rs)
1017 {
1018 uint64_t threshold = migrate_throttle_trigger_threshold();
1019 uint64_t bytes_xfer_period =
1020 migration_transferred_bytes() - rs->bytes_xfer_prev;
1021 uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE;
1022 uint64_t bytes_dirty_threshold = bytes_xfer_period * threshold / 100;
1023
1024 /*
1025 * The following detection logic can be refined later. For now:
1026 * Check to see if the ratio between dirtied bytes and the approx.
1027 * amount of bytes that just got transferred since the last time
1028 * we were in this routine reaches the threshold. If that happens
1029 * twice, start or increase throttling.
1030 */
1031 if ((bytes_dirty_period > bytes_dirty_threshold) &&
1032 (++rs->dirty_rate_high_cnt >= 2)) {
1033 rs->dirty_rate_high_cnt = 0;
1034 if (migrate_auto_converge()) {
1035 trace_migration_throttle();
1036 mig_throttle_guest_down(bytes_dirty_period,
1037 bytes_dirty_threshold);
1038 } else if (migrate_dirty_limit()) {
1039 migration_dirty_limit_guest();
1040 }
1041 }
1042 }
1043
migration_bitmap_sync(RAMState * rs,bool last_stage)1044 static void migration_bitmap_sync(RAMState *rs, bool last_stage)
1045 {
1046 RAMBlock *block;
1047 int64_t end_time;
1048
1049 stat64_add(&mig_stats.dirty_sync_count, 1);
1050
1051 if (!rs->time_last_bitmap_sync) {
1052 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1053 }
1054
1055 trace_migration_bitmap_sync_start();
1056 memory_global_dirty_log_sync(last_stage);
1057
1058 WITH_QEMU_LOCK_GUARD(&rs->bitmap_mutex) {
1059 WITH_RCU_READ_LOCK_GUARD() {
1060 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1061 ramblock_sync_dirty_bitmap(rs, block);
1062 }
1063 stat64_set(&mig_stats.dirty_bytes_last_sync, ram_bytes_remaining());
1064 }
1065 }
1066
1067 memory_global_after_dirty_log_sync();
1068 trace_migration_bitmap_sync_end(rs->num_dirty_pages_period);
1069
1070 end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1071
1072 /* more than 1 second = 1000 millisecons */
1073 if (end_time > rs->time_last_bitmap_sync + 1000) {
1074 migration_trigger_throttle(rs);
1075
1076 migration_update_rates(rs, end_time);
1077
1078 rs->target_page_count_prev = rs->target_page_count;
1079
1080 /* reset period counters */
1081 rs->time_last_bitmap_sync = end_time;
1082 rs->num_dirty_pages_period = 0;
1083 rs->bytes_xfer_prev = migration_transferred_bytes();
1084 }
1085 if (migrate_events()) {
1086 uint64_t generation = stat64_get(&mig_stats.dirty_sync_count);
1087 qapi_event_send_migration_pass(generation);
1088 }
1089 }
1090
migration_bitmap_sync_precopy(RAMState * rs,bool last_stage)1091 static void migration_bitmap_sync_precopy(RAMState *rs, bool last_stage)
1092 {
1093 Error *local_err = NULL;
1094
1095 /*
1096 * The current notifier usage is just an optimization to migration, so we
1097 * don't stop the normal migration process in the error case.
1098 */
1099 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) {
1100 error_report_err(local_err);
1101 local_err = NULL;
1102 }
1103
1104 migration_bitmap_sync(rs, last_stage);
1105
1106 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) {
1107 error_report_err(local_err);
1108 }
1109 }
1110
ram_release_page(const char * rbname,uint64_t offset)1111 void ram_release_page(const char *rbname, uint64_t offset)
1112 {
1113 if (!migrate_release_ram() || !migration_in_postcopy()) {
1114 return;
1115 }
1116
1117 ram_discard_range(rbname, offset, TARGET_PAGE_SIZE);
1118 }
1119
1120 /**
1121 * save_zero_page: send the zero page to the stream
1122 *
1123 * Returns the number of pages written.
1124 *
1125 * @rs: current RAM state
1126 * @pss: current PSS channel
1127 * @offset: offset inside the block for the page
1128 */
save_zero_page(RAMState * rs,PageSearchStatus * pss,ram_addr_t offset)1129 static int save_zero_page(RAMState *rs, PageSearchStatus *pss,
1130 ram_addr_t offset)
1131 {
1132 uint8_t *p = pss->block->host + offset;
1133 QEMUFile *file = pss->pss_channel;
1134 int len = 0;
1135
1136 if (migrate_zero_page_detection() == ZERO_PAGE_DETECTION_NONE) {
1137 return 0;
1138 }
1139
1140 if (!buffer_is_zero(p, TARGET_PAGE_SIZE)) {
1141 return 0;
1142 }
1143
1144 stat64_add(&mig_stats.zero_pages, 1);
1145
1146 if (migrate_mapped_ram()) {
1147 /* zero pages are not transferred with mapped-ram */
1148 clear_bit_atomic(offset >> TARGET_PAGE_BITS, pss->block->file_bmap);
1149 return 1;
1150 }
1151
1152 len += save_page_header(pss, file, pss->block, offset | RAM_SAVE_FLAG_ZERO);
1153 qemu_put_byte(file, 0);
1154 len += 1;
1155 ram_release_page(pss->block->idstr, offset);
1156 ram_transferred_add(len);
1157
1158 /*
1159 * Must let xbzrle know, otherwise a previous (now 0'd) cached
1160 * page would be stale.
1161 */
1162 if (rs->xbzrle_started) {
1163 XBZRLE_cache_lock();
1164 xbzrle_cache_zero_page(pss->block->offset + offset);
1165 XBZRLE_cache_unlock();
1166 }
1167
1168 return len;
1169 }
1170
1171 /*
1172 * @pages: the number of pages written by the control path,
1173 * < 0 - error
1174 * > 0 - number of pages written
1175 *
1176 * Return true if the pages has been saved, otherwise false is returned.
1177 */
control_save_page(PageSearchStatus * pss,ram_addr_t offset,int * pages)1178 static bool control_save_page(PageSearchStatus *pss,
1179 ram_addr_t offset, int *pages)
1180 {
1181 int ret;
1182
1183 ret = rdma_control_save_page(pss->pss_channel, pss->block->offset, offset,
1184 TARGET_PAGE_SIZE);
1185 if (ret == RAM_SAVE_CONTROL_NOT_SUPP) {
1186 return false;
1187 }
1188
1189 if (ret == RAM_SAVE_CONTROL_DELAYED) {
1190 *pages = 1;
1191 return true;
1192 }
1193 *pages = ret;
1194 return true;
1195 }
1196
1197 /*
1198 * directly send the page to the stream
1199 *
1200 * Returns the number of pages written.
1201 *
1202 * @pss: current PSS channel
1203 * @block: block that contains the page we want to send
1204 * @offset: offset inside the block for the page
1205 * @buf: the page to be sent
1206 * @async: send to page asyncly
1207 */
save_normal_page(PageSearchStatus * pss,RAMBlock * block,ram_addr_t offset,uint8_t * buf,bool async)1208 static int save_normal_page(PageSearchStatus *pss, RAMBlock *block,
1209 ram_addr_t offset, uint8_t *buf, bool async)
1210 {
1211 QEMUFile *file = pss->pss_channel;
1212
1213 if (migrate_mapped_ram()) {
1214 qemu_put_buffer_at(file, buf, TARGET_PAGE_SIZE,
1215 block->pages_offset + offset);
1216 set_bit(offset >> TARGET_PAGE_BITS, block->file_bmap);
1217 } else {
1218 ram_transferred_add(save_page_header(pss, pss->pss_channel, block,
1219 offset | RAM_SAVE_FLAG_PAGE));
1220 if (async) {
1221 qemu_put_buffer_async(file, buf, TARGET_PAGE_SIZE,
1222 migrate_release_ram() &&
1223 migration_in_postcopy());
1224 } else {
1225 qemu_put_buffer(file, buf, TARGET_PAGE_SIZE);
1226 }
1227 }
1228 ram_transferred_add(TARGET_PAGE_SIZE);
1229 stat64_add(&mig_stats.normal_pages, 1);
1230 return 1;
1231 }
1232
1233 /**
1234 * ram_save_page: send the given page to the stream
1235 *
1236 * Returns the number of pages written.
1237 * < 0 - error
1238 * >=0 - Number of pages written - this might legally be 0
1239 * if xbzrle noticed the page was the same.
1240 *
1241 * @rs: current RAM state
1242 * @block: block that contains the page we want to send
1243 * @offset: offset inside the block for the page
1244 */
ram_save_page(RAMState * rs,PageSearchStatus * pss)1245 static int ram_save_page(RAMState *rs, PageSearchStatus *pss)
1246 {
1247 int pages = -1;
1248 uint8_t *p;
1249 bool send_async = true;
1250 RAMBlock *block = pss->block;
1251 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
1252 ram_addr_t current_addr = block->offset + offset;
1253
1254 p = block->host + offset;
1255 trace_ram_save_page(block->idstr, (uint64_t)offset, p);
1256
1257 XBZRLE_cache_lock();
1258 if (rs->xbzrle_started && !migration_in_postcopy()) {
1259 pages = save_xbzrle_page(rs, pss, &p, current_addr,
1260 block, offset);
1261 if (!rs->last_stage) {
1262 /* Can't send this cached data async, since the cache page
1263 * might get updated before it gets to the wire
1264 */
1265 send_async = false;
1266 }
1267 }
1268
1269 /* XBZRLE overflow or normal page */
1270 if (pages == -1) {
1271 pages = save_normal_page(pss, block, offset, p, send_async);
1272 }
1273
1274 XBZRLE_cache_unlock();
1275
1276 return pages;
1277 }
1278
ram_save_multifd_page(RAMBlock * block,ram_addr_t offset)1279 static int ram_save_multifd_page(RAMBlock *block, ram_addr_t offset)
1280 {
1281 if (!multifd_queue_page(block, offset)) {
1282 return -1;
1283 }
1284
1285 return 1;
1286 }
1287
1288
1289 #define PAGE_ALL_CLEAN 0
1290 #define PAGE_TRY_AGAIN 1
1291 #define PAGE_DIRTY_FOUND 2
1292 /**
1293 * find_dirty_block: find the next dirty page and update any state
1294 * associated with the search process.
1295 *
1296 * Returns:
1297 * <0: An error happened
1298 * PAGE_ALL_CLEAN: no dirty page found, give up
1299 * PAGE_TRY_AGAIN: no dirty page found, retry for next block
1300 * PAGE_DIRTY_FOUND: dirty page found
1301 *
1302 * @rs: current RAM state
1303 * @pss: data about the state of the current dirty page scan
1304 * @again: set to false if the search has scanned the whole of RAM
1305 */
find_dirty_block(RAMState * rs,PageSearchStatus * pss)1306 static int find_dirty_block(RAMState *rs, PageSearchStatus *pss)
1307 {
1308 /* Update pss->page for the next dirty bit in ramblock */
1309 pss_find_next_dirty(pss);
1310
1311 if (pss->complete_round && pss->block == rs->last_seen_block &&
1312 pss->page >= rs->last_page) {
1313 /*
1314 * We've been once around the RAM and haven't found anything.
1315 * Give up.
1316 */
1317 return PAGE_ALL_CLEAN;
1318 }
1319 if (!offset_in_ramblock(pss->block,
1320 ((ram_addr_t)pss->page) << TARGET_PAGE_BITS)) {
1321 /* Didn't find anything in this RAM Block */
1322 pss->page = 0;
1323 pss->block = QLIST_NEXT_RCU(pss->block, next);
1324 if (!pss->block) {
1325 if (migrate_multifd() &&
1326 (!migrate_multifd_flush_after_each_section() ||
1327 migrate_mapped_ram())) {
1328 QEMUFile *f = rs->pss[RAM_CHANNEL_PRECOPY].pss_channel;
1329 int ret = multifd_ram_flush_and_sync();
1330 if (ret < 0) {
1331 return ret;
1332 }
1333
1334 if (!migrate_mapped_ram()) {
1335 qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
1336 qemu_fflush(f);
1337 }
1338 }
1339
1340 /* Hit the end of the list */
1341 pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
1342 /* Flag that we've looped */
1343 pss->complete_round = true;
1344 /* After the first round, enable XBZRLE. */
1345 if (migrate_xbzrle()) {
1346 rs->xbzrle_started = true;
1347 }
1348 }
1349 /* Didn't find anything this time, but try again on the new block */
1350 return PAGE_TRY_AGAIN;
1351 } else {
1352 /* We've found something */
1353 return PAGE_DIRTY_FOUND;
1354 }
1355 }
1356
1357 /**
1358 * unqueue_page: gets a page of the queue
1359 *
1360 * Helper for 'get_queued_page' - gets a page off the queue
1361 *
1362 * Returns the block of the page (or NULL if none available)
1363 *
1364 * @rs: current RAM state
1365 * @offset: used to return the offset within the RAMBlock
1366 */
unqueue_page(RAMState * rs,ram_addr_t * offset)1367 static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset)
1368 {
1369 struct RAMSrcPageRequest *entry;
1370 RAMBlock *block = NULL;
1371
1372 if (!postcopy_has_request(rs)) {
1373 return NULL;
1374 }
1375
1376 QEMU_LOCK_GUARD(&rs->src_page_req_mutex);
1377
1378 /*
1379 * This should _never_ change even after we take the lock, because no one
1380 * should be taking anything off the request list other than us.
1381 */
1382 assert(postcopy_has_request(rs));
1383
1384 entry = QSIMPLEQ_FIRST(&rs->src_page_requests);
1385 block = entry->rb;
1386 *offset = entry->offset;
1387
1388 if (entry->len > TARGET_PAGE_SIZE) {
1389 entry->len -= TARGET_PAGE_SIZE;
1390 entry->offset += TARGET_PAGE_SIZE;
1391 } else {
1392 memory_region_unref(block->mr);
1393 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1394 g_free(entry);
1395 migration_consume_urgent_request();
1396 }
1397
1398 return block;
1399 }
1400
1401 #if defined(__linux__)
1402 /**
1403 * poll_fault_page: try to get next UFFD write fault page and, if pending fault
1404 * is found, return RAM block pointer and page offset
1405 *
1406 * Returns pointer to the RAMBlock containing faulting page,
1407 * NULL if no write faults are pending
1408 *
1409 * @rs: current RAM state
1410 * @offset: page offset from the beginning of the block
1411 */
poll_fault_page(RAMState * rs,ram_addr_t * offset)1412 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
1413 {
1414 struct uffd_msg uffd_msg;
1415 void *page_address;
1416 RAMBlock *block;
1417 int res;
1418
1419 if (!migrate_background_snapshot()) {
1420 return NULL;
1421 }
1422
1423 res = uffd_read_events(rs->uffdio_fd, &uffd_msg, 1);
1424 if (res <= 0) {
1425 return NULL;
1426 }
1427
1428 page_address = (void *)(uintptr_t) uffd_msg.arg.pagefault.address;
1429 block = qemu_ram_block_from_host(page_address, false, offset);
1430 assert(block && (block->flags & RAM_UF_WRITEPROTECT) != 0);
1431 return block;
1432 }
1433
1434 /**
1435 * ram_save_release_protection: release UFFD write protection after
1436 * a range of pages has been saved
1437 *
1438 * @rs: current RAM state
1439 * @pss: page-search-status structure
1440 * @start_page: index of the first page in the range relative to pss->block
1441 *
1442 * Returns 0 on success, negative value in case of an error
1443 */
ram_save_release_protection(RAMState * rs,PageSearchStatus * pss,unsigned long start_page)1444 static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
1445 unsigned long start_page)
1446 {
1447 int res = 0;
1448
1449 /* Check if page is from UFFD-managed region. */
1450 if (pss->block->flags & RAM_UF_WRITEPROTECT) {
1451 void *page_address = pss->block->host + (start_page << TARGET_PAGE_BITS);
1452 uint64_t run_length = (pss->page - start_page) << TARGET_PAGE_BITS;
1453
1454 /* Flush async buffers before un-protect. */
1455 qemu_fflush(pss->pss_channel);
1456 /* Un-protect memory range. */
1457 res = uffd_change_protection(rs->uffdio_fd, page_address, run_length,
1458 false, false);
1459 }
1460
1461 return res;
1462 }
1463
1464 /* ram_write_tracking_available: check if kernel supports required UFFD features
1465 *
1466 * Returns true if supports, false otherwise
1467 */
ram_write_tracking_available(void)1468 bool ram_write_tracking_available(void)
1469 {
1470 uint64_t uffd_features;
1471 int res;
1472
1473 res = uffd_query_features(&uffd_features);
1474 return (res == 0 &&
1475 (uffd_features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) != 0);
1476 }
1477
1478 /* ram_write_tracking_compatible: check if guest configuration is
1479 * compatible with 'write-tracking'
1480 *
1481 * Returns true if compatible, false otherwise
1482 */
ram_write_tracking_compatible(void)1483 bool ram_write_tracking_compatible(void)
1484 {
1485 const uint64_t uffd_ioctls_mask = BIT(_UFFDIO_WRITEPROTECT);
1486 int uffd_fd;
1487 RAMBlock *block;
1488 bool ret = false;
1489
1490 /* Open UFFD file descriptor */
1491 uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, false);
1492 if (uffd_fd < 0) {
1493 return false;
1494 }
1495
1496 RCU_READ_LOCK_GUARD();
1497
1498 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1499 uint64_t uffd_ioctls;
1500
1501 /* Nothing to do with read-only and MMIO-writable regions */
1502 if (block->mr->readonly || block->mr->rom_device) {
1503 continue;
1504 }
1505 /* Try to register block memory via UFFD-IO to track writes */
1506 if (uffd_register_memory(uffd_fd, block->host, block->max_length,
1507 UFFDIO_REGISTER_MODE_WP, &uffd_ioctls)) {
1508 goto out;
1509 }
1510 if ((uffd_ioctls & uffd_ioctls_mask) != uffd_ioctls_mask) {
1511 goto out;
1512 }
1513 }
1514 ret = true;
1515
1516 out:
1517 uffd_close_fd(uffd_fd);
1518 return ret;
1519 }
1520
populate_read_range(RAMBlock * block,ram_addr_t offset,ram_addr_t size)1521 static inline void populate_read_range(RAMBlock *block, ram_addr_t offset,
1522 ram_addr_t size)
1523 {
1524 const ram_addr_t end = offset + size;
1525
1526 /*
1527 * We read one byte of each page; this will preallocate page tables if
1528 * required and populate the shared zeropage on MAP_PRIVATE anonymous memory
1529 * where no page was populated yet. This might require adaption when
1530 * supporting other mappings, like shmem.
1531 */
1532 for (; offset < end; offset += block->page_size) {
1533 char tmp = *((char *)block->host + offset);
1534
1535 /* Don't optimize the read out */
1536 asm volatile("" : "+r" (tmp));
1537 }
1538 }
1539
populate_read_section(MemoryRegionSection * section,void * opaque)1540 static inline int populate_read_section(MemoryRegionSection *section,
1541 void *opaque)
1542 {
1543 const hwaddr size = int128_get64(section->size);
1544 hwaddr offset = section->offset_within_region;
1545 RAMBlock *block = section->mr->ram_block;
1546
1547 populate_read_range(block, offset, size);
1548 return 0;
1549 }
1550
1551 /*
1552 * ram_block_populate_read: preallocate page tables and populate pages in the
1553 * RAM block by reading a byte of each page.
1554 *
1555 * Since it's solely used for userfault_fd WP feature, here we just
1556 * hardcode page size to qemu_real_host_page_size.
1557 *
1558 * @block: RAM block to populate
1559 */
ram_block_populate_read(RAMBlock * rb)1560 static void ram_block_populate_read(RAMBlock *rb)
1561 {
1562 /*
1563 * Skip populating all pages that fall into a discarded range as managed by
1564 * a RamDiscardManager responsible for the mapped memory region of the
1565 * RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock
1566 * must not get populated automatically. We don't have to track
1567 * modifications via userfaultfd WP reliably, because these pages will
1568 * not be part of the migration stream either way -- see
1569 * ramblock_dirty_bitmap_exclude_discarded_pages().
1570 *
1571 * Note: The result is only stable while migrating (precopy/postcopy).
1572 */
1573 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1574 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1575 MemoryRegionSection section = {
1576 .mr = rb->mr,
1577 .offset_within_region = 0,
1578 .size = rb->mr->size,
1579 };
1580
1581 ram_discard_manager_replay_populated(rdm, §ion,
1582 populate_read_section, NULL);
1583 } else {
1584 populate_read_range(rb, 0, rb->used_length);
1585 }
1586 }
1587
1588 /*
1589 * ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
1590 */
ram_write_tracking_prepare(void)1591 void ram_write_tracking_prepare(void)
1592 {
1593 RAMBlock *block;
1594
1595 RCU_READ_LOCK_GUARD();
1596
1597 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1598 /* Nothing to do with read-only and MMIO-writable regions */
1599 if (block->mr->readonly || block->mr->rom_device) {
1600 continue;
1601 }
1602
1603 /*
1604 * Populate pages of the RAM block before enabling userfault_fd
1605 * write protection.
1606 *
1607 * This stage is required since ioctl(UFFDIO_WRITEPROTECT) with
1608 * UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
1609 * pages with pte_none() entries in page table.
1610 */
1611 ram_block_populate_read(block);
1612 }
1613 }
1614
uffd_protect_section(MemoryRegionSection * section,void * opaque)1615 static inline int uffd_protect_section(MemoryRegionSection *section,
1616 void *opaque)
1617 {
1618 const hwaddr size = int128_get64(section->size);
1619 const hwaddr offset = section->offset_within_region;
1620 RAMBlock *rb = section->mr->ram_block;
1621 int uffd_fd = (uintptr_t)opaque;
1622
1623 return uffd_change_protection(uffd_fd, rb->host + offset, size, true,
1624 false);
1625 }
1626
ram_block_uffd_protect(RAMBlock * rb,int uffd_fd)1627 static int ram_block_uffd_protect(RAMBlock *rb, int uffd_fd)
1628 {
1629 assert(rb->flags & RAM_UF_WRITEPROTECT);
1630
1631 /* See ram_block_populate_read() */
1632 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1633 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1634 MemoryRegionSection section = {
1635 .mr = rb->mr,
1636 .offset_within_region = 0,
1637 .size = rb->mr->size,
1638 };
1639
1640 return ram_discard_manager_replay_populated(rdm, §ion,
1641 uffd_protect_section,
1642 (void *)(uintptr_t)uffd_fd);
1643 }
1644 return uffd_change_protection(uffd_fd, rb->host,
1645 rb->used_length, true, false);
1646 }
1647
1648 /*
1649 * ram_write_tracking_start: start UFFD-WP memory tracking
1650 *
1651 * Returns 0 for success or negative value in case of error
1652 */
ram_write_tracking_start(void)1653 int ram_write_tracking_start(void)
1654 {
1655 int uffd_fd;
1656 RAMState *rs = ram_state;
1657 RAMBlock *block;
1658
1659 /* Open UFFD file descriptor */
1660 uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, true);
1661 if (uffd_fd < 0) {
1662 return uffd_fd;
1663 }
1664 rs->uffdio_fd = uffd_fd;
1665
1666 RCU_READ_LOCK_GUARD();
1667
1668 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1669 /* Nothing to do with read-only and MMIO-writable regions */
1670 if (block->mr->readonly || block->mr->rom_device) {
1671 continue;
1672 }
1673
1674 /* Register block memory with UFFD to track writes */
1675 if (uffd_register_memory(rs->uffdio_fd, block->host,
1676 block->max_length, UFFDIO_REGISTER_MODE_WP, NULL)) {
1677 goto fail;
1678 }
1679 block->flags |= RAM_UF_WRITEPROTECT;
1680 memory_region_ref(block->mr);
1681
1682 /* Apply UFFD write protection to the block memory range */
1683 if (ram_block_uffd_protect(block, uffd_fd)) {
1684 goto fail;
1685 }
1686
1687 trace_ram_write_tracking_ramblock_start(block->idstr, block->page_size,
1688 block->host, block->max_length);
1689 }
1690
1691 return 0;
1692
1693 fail:
1694 error_report("ram_write_tracking_start() failed: restoring initial memory state");
1695
1696 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1697 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
1698 continue;
1699 }
1700 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
1701 /* Cleanup flags and remove reference */
1702 block->flags &= ~RAM_UF_WRITEPROTECT;
1703 memory_region_unref(block->mr);
1704 }
1705
1706 uffd_close_fd(uffd_fd);
1707 rs->uffdio_fd = -1;
1708 return -1;
1709 }
1710
1711 /**
1712 * ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
1713 */
ram_write_tracking_stop(void)1714 void ram_write_tracking_stop(void)
1715 {
1716 RAMState *rs = ram_state;
1717 RAMBlock *block;
1718
1719 RCU_READ_LOCK_GUARD();
1720
1721 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1722 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
1723 continue;
1724 }
1725 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
1726
1727 trace_ram_write_tracking_ramblock_stop(block->idstr, block->page_size,
1728 block->host, block->max_length);
1729
1730 /* Cleanup flags and remove reference */
1731 block->flags &= ~RAM_UF_WRITEPROTECT;
1732 memory_region_unref(block->mr);
1733 }
1734
1735 /* Finally close UFFD file descriptor */
1736 uffd_close_fd(rs->uffdio_fd);
1737 rs->uffdio_fd = -1;
1738 }
1739
1740 #else
1741 /* No target OS support, stubs just fail or ignore */
1742
poll_fault_page(RAMState * rs,ram_addr_t * offset)1743 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
1744 {
1745 (void) rs;
1746 (void) offset;
1747
1748 return NULL;
1749 }
1750
ram_save_release_protection(RAMState * rs,PageSearchStatus * pss,unsigned long start_page)1751 static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
1752 unsigned long start_page)
1753 {
1754 (void) rs;
1755 (void) pss;
1756 (void) start_page;
1757
1758 return 0;
1759 }
1760
ram_write_tracking_available(void)1761 bool ram_write_tracking_available(void)
1762 {
1763 return false;
1764 }
1765
ram_write_tracking_compatible(void)1766 bool ram_write_tracking_compatible(void)
1767 {
1768 assert(0);
1769 return false;
1770 }
1771
ram_write_tracking_start(void)1772 int ram_write_tracking_start(void)
1773 {
1774 assert(0);
1775 return -1;
1776 }
1777
ram_write_tracking_stop(void)1778 void ram_write_tracking_stop(void)
1779 {
1780 assert(0);
1781 }
1782 #endif /* defined(__linux__) */
1783
1784 /**
1785 * get_queued_page: unqueue a page from the postcopy requests
1786 *
1787 * Skips pages that are already sent (!dirty)
1788 *
1789 * Returns true if a queued page is found
1790 *
1791 * @rs: current RAM state
1792 * @pss: data about the state of the current dirty page scan
1793 */
get_queued_page(RAMState * rs,PageSearchStatus * pss)1794 static bool get_queued_page(RAMState *rs, PageSearchStatus *pss)
1795 {
1796 RAMBlock *block;
1797 ram_addr_t offset;
1798 bool dirty;
1799
1800 do {
1801 block = unqueue_page(rs, &offset);
1802 /*
1803 * We're sending this page, and since it's postcopy nothing else
1804 * will dirty it, and we must make sure it doesn't get sent again
1805 * even if this queue request was received after the background
1806 * search already sent it.
1807 */
1808 if (block) {
1809 unsigned long page;
1810
1811 page = offset >> TARGET_PAGE_BITS;
1812 dirty = test_bit(page, block->bmap);
1813 if (!dirty) {
1814 trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset,
1815 page);
1816 } else {
1817 trace_get_queued_page(block->idstr, (uint64_t)offset, page);
1818 }
1819 }
1820
1821 } while (block && !dirty);
1822
1823 if (!block) {
1824 /*
1825 * Poll write faults too if background snapshot is enabled; that's
1826 * when we have vcpus got blocked by the write protected pages.
1827 */
1828 block = poll_fault_page(rs, &offset);
1829 }
1830
1831 if (block) {
1832 /*
1833 * We want the background search to continue from the queued page
1834 * since the guest is likely to want other pages near to the page
1835 * it just requested.
1836 */
1837 pss->block = block;
1838 pss->page = offset >> TARGET_PAGE_BITS;
1839
1840 /*
1841 * This unqueued page would break the "one round" check, even is
1842 * really rare.
1843 */
1844 pss->complete_round = false;
1845 }
1846
1847 return !!block;
1848 }
1849
1850 /**
1851 * migration_page_queue_free: drop any remaining pages in the ram
1852 * request queue
1853 *
1854 * It should be empty at the end anyway, but in error cases there may
1855 * be some left. in case that there is any page left, we drop it.
1856 *
1857 */
migration_page_queue_free(RAMState * rs)1858 static void migration_page_queue_free(RAMState *rs)
1859 {
1860 struct RAMSrcPageRequest *mspr, *next_mspr;
1861 /* This queue generally should be empty - but in the case of a failed
1862 * migration might have some droppings in.
1863 */
1864 RCU_READ_LOCK_GUARD();
1865 QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) {
1866 memory_region_unref(mspr->rb->mr);
1867 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1868 g_free(mspr);
1869 }
1870 }
1871
1872 /**
1873 * ram_save_queue_pages: queue the page for transmission
1874 *
1875 * A request from postcopy destination for example.
1876 *
1877 * Returns zero on success or negative on error
1878 *
1879 * @rbname: Name of the RAMBLock of the request. NULL means the
1880 * same that last one.
1881 * @start: starting address from the start of the RAMBlock
1882 * @len: length (in bytes) to send
1883 */
ram_save_queue_pages(const char * rbname,ram_addr_t start,ram_addr_t len,Error ** errp)1884 int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len,
1885 Error **errp)
1886 {
1887 RAMBlock *ramblock;
1888 RAMState *rs = ram_state;
1889
1890 stat64_add(&mig_stats.postcopy_requests, 1);
1891 RCU_READ_LOCK_GUARD();
1892
1893 if (!rbname) {
1894 /* Reuse last RAMBlock */
1895 ramblock = rs->last_req_rb;
1896
1897 if (!ramblock) {
1898 /*
1899 * Shouldn't happen, we can't reuse the last RAMBlock if
1900 * it's the 1st request.
1901 */
1902 error_setg(errp, "MIG_RP_MSG_REQ_PAGES has no previous block");
1903 return -1;
1904 }
1905 } else {
1906 ramblock = qemu_ram_block_by_name(rbname);
1907
1908 if (!ramblock) {
1909 /* We shouldn't be asked for a non-existent RAMBlock */
1910 error_setg(errp, "MIG_RP_MSG_REQ_PAGES has no block '%s'", rbname);
1911 return -1;
1912 }
1913 rs->last_req_rb = ramblock;
1914 }
1915 trace_ram_save_queue_pages(ramblock->idstr, start, len);
1916 if (!offset_in_ramblock(ramblock, start + len - 1)) {
1917 error_setg(errp, "MIG_RP_MSG_REQ_PAGES request overrun, "
1918 "start=" RAM_ADDR_FMT " len="
1919 RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
1920 start, len, ramblock->used_length);
1921 return -1;
1922 }
1923
1924 /*
1925 * When with postcopy preempt, we send back the page directly in the
1926 * rp-return thread.
1927 */
1928 if (postcopy_preempt_active()) {
1929 ram_addr_t page_start = start >> TARGET_PAGE_BITS;
1930 size_t page_size = qemu_ram_pagesize(ramblock);
1931 PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_POSTCOPY];
1932 int ret = 0;
1933
1934 qemu_mutex_lock(&rs->bitmap_mutex);
1935
1936 pss_init(pss, ramblock, page_start);
1937 /*
1938 * Always use the preempt channel, and make sure it's there. It's
1939 * safe to access without lock, because when rp-thread is running
1940 * we should be the only one who operates on the qemufile
1941 */
1942 pss->pss_channel = migrate_get_current()->postcopy_qemufile_src;
1943 assert(pss->pss_channel);
1944
1945 /*
1946 * It must be either one or multiple of host page size. Just
1947 * assert; if something wrong we're mostly split brain anyway.
1948 */
1949 assert(len % page_size == 0);
1950 while (len) {
1951 if (ram_save_host_page_urgent(pss)) {
1952 error_setg(errp, "ram_save_host_page_urgent() failed: "
1953 "ramblock=%s, start_addr=0x"RAM_ADDR_FMT,
1954 ramblock->idstr, start);
1955 ret = -1;
1956 break;
1957 }
1958 /*
1959 * NOTE: after ram_save_host_page_urgent() succeeded, pss->page
1960 * will automatically be moved and point to the next host page
1961 * we're going to send, so no need to update here.
1962 *
1963 * Normally QEMU never sends >1 host page in requests, so
1964 * logically we don't even need that as the loop should only
1965 * run once, but just to be consistent.
1966 */
1967 len -= page_size;
1968 };
1969 qemu_mutex_unlock(&rs->bitmap_mutex);
1970
1971 return ret;
1972 }
1973
1974 struct RAMSrcPageRequest *new_entry =
1975 g_new0(struct RAMSrcPageRequest, 1);
1976 new_entry->rb = ramblock;
1977 new_entry->offset = start;
1978 new_entry->len = len;
1979
1980 memory_region_ref(ramblock->mr);
1981 qemu_mutex_lock(&rs->src_page_req_mutex);
1982 QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
1983 migration_make_urgent_request();
1984 qemu_mutex_unlock(&rs->src_page_req_mutex);
1985
1986 return 0;
1987 }
1988
1989 /**
1990 * ram_save_target_page_legacy: save one target page
1991 *
1992 * Returns the number of pages written
1993 *
1994 * @rs: current RAM state
1995 * @pss: data about the page we want to send
1996 */
ram_save_target_page_legacy(RAMState * rs,PageSearchStatus * pss)1997 static int ram_save_target_page_legacy(RAMState *rs, PageSearchStatus *pss)
1998 {
1999 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2000 int res;
2001
2002 if (control_save_page(pss, offset, &res)) {
2003 return res;
2004 }
2005
2006 if (save_zero_page(rs, pss, offset)) {
2007 return 1;
2008 }
2009
2010 return ram_save_page(rs, pss);
2011 }
2012
2013 /**
2014 * ram_save_target_page_multifd: send one target page to multifd workers
2015 *
2016 * Returns 1 if the page was queued, -1 otherwise.
2017 *
2018 * @rs: current RAM state
2019 * @pss: data about the page we want to send
2020 */
ram_save_target_page_multifd(RAMState * rs,PageSearchStatus * pss)2021 static int ram_save_target_page_multifd(RAMState *rs, PageSearchStatus *pss)
2022 {
2023 RAMBlock *block = pss->block;
2024 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2025
2026 /*
2027 * While using multifd live migration, we still need to handle zero
2028 * page checking on the migration main thread.
2029 */
2030 if (migrate_zero_page_detection() == ZERO_PAGE_DETECTION_LEGACY) {
2031 if (save_zero_page(rs, pss, offset)) {
2032 return 1;
2033 }
2034 }
2035
2036 return ram_save_multifd_page(block, offset);
2037 }
2038
2039 /* Should be called before sending a host page */
pss_host_page_prepare(PageSearchStatus * pss)2040 static void pss_host_page_prepare(PageSearchStatus *pss)
2041 {
2042 /* How many guest pages are there in one host page? */
2043 size_t guest_pfns = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2044
2045 pss->host_page_sending = true;
2046 if (guest_pfns <= 1) {
2047 /*
2048 * This covers both when guest psize == host psize, or when guest
2049 * has larger psize than the host (guest_pfns==0).
2050 *
2051 * For the latter, we always send one whole guest page per
2052 * iteration of the host page (example: an Alpha VM on x86 host
2053 * will have guest psize 8K while host psize 4K).
2054 */
2055 pss->host_page_start = pss->page;
2056 pss->host_page_end = pss->page + 1;
2057 } else {
2058 /*
2059 * The host page spans over multiple guest pages, we send them
2060 * within the same host page iteration.
2061 */
2062 pss->host_page_start = ROUND_DOWN(pss->page, guest_pfns);
2063 pss->host_page_end = ROUND_UP(pss->page + 1, guest_pfns);
2064 }
2065 }
2066
2067 /*
2068 * Whether the page pointed by PSS is within the host page being sent.
2069 * Must be called after a previous pss_host_page_prepare().
2070 */
pss_within_range(PageSearchStatus * pss)2071 static bool pss_within_range(PageSearchStatus *pss)
2072 {
2073 ram_addr_t ram_addr;
2074
2075 assert(pss->host_page_sending);
2076
2077 /* Over host-page boundary? */
2078 if (pss->page >= pss->host_page_end) {
2079 return false;
2080 }
2081
2082 ram_addr = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2083
2084 return offset_in_ramblock(pss->block, ram_addr);
2085 }
2086
pss_host_page_finish(PageSearchStatus * pss)2087 static void pss_host_page_finish(PageSearchStatus *pss)
2088 {
2089 pss->host_page_sending = false;
2090 /* This is not needed, but just to reset it */
2091 pss->host_page_start = pss->host_page_end = 0;
2092 }
2093
2094 /*
2095 * Send an urgent host page specified by `pss'. Need to be called with
2096 * bitmap_mutex held.
2097 *
2098 * Returns 0 if save host page succeeded, false otherwise.
2099 */
ram_save_host_page_urgent(PageSearchStatus * pss)2100 static int ram_save_host_page_urgent(PageSearchStatus *pss)
2101 {
2102 bool page_dirty, sent = false;
2103 RAMState *rs = ram_state;
2104 int ret = 0;
2105
2106 trace_postcopy_preempt_send_host_page(pss->block->idstr, pss->page);
2107 pss_host_page_prepare(pss);
2108
2109 /*
2110 * If precopy is sending the same page, let it be done in precopy, or
2111 * we could send the same page in two channels and none of them will
2112 * receive the whole page.
2113 */
2114 if (pss_overlap(pss, &ram_state->pss[RAM_CHANNEL_PRECOPY])) {
2115 trace_postcopy_preempt_hit(pss->block->idstr,
2116 pss->page << TARGET_PAGE_BITS);
2117 return 0;
2118 }
2119
2120 do {
2121 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2122
2123 if (page_dirty) {
2124 /* Be strict to return code; it must be 1, or what else? */
2125 if (migration_ops->ram_save_target_page(rs, pss) != 1) {
2126 error_report_once("%s: ram_save_target_page failed", __func__);
2127 ret = -1;
2128 goto out;
2129 }
2130 sent = true;
2131 }
2132 pss_find_next_dirty(pss);
2133 } while (pss_within_range(pss));
2134 out:
2135 pss_host_page_finish(pss);
2136 /* For urgent requests, flush immediately if sent */
2137 if (sent) {
2138 qemu_fflush(pss->pss_channel);
2139 }
2140 return ret;
2141 }
2142
2143 /**
2144 * ram_save_host_page: save a whole host page
2145 *
2146 * Starting at *offset send pages up to the end of the current host
2147 * page. It's valid for the initial offset to point into the middle of
2148 * a host page in which case the remainder of the hostpage is sent.
2149 * Only dirty target pages are sent. Note that the host page size may
2150 * be a huge page for this block.
2151 *
2152 * The saving stops at the boundary of the used_length of the block
2153 * if the RAMBlock isn't a multiple of the host page size.
2154 *
2155 * The caller must be with ram_state.bitmap_mutex held to call this
2156 * function. Note that this function can temporarily release the lock, but
2157 * when the function is returned it'll make sure the lock is still held.
2158 *
2159 * Returns the number of pages written or negative on error
2160 *
2161 * @rs: current RAM state
2162 * @pss: data about the page we want to send
2163 */
ram_save_host_page(RAMState * rs,PageSearchStatus * pss)2164 static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss)
2165 {
2166 bool page_dirty, preempt_active = postcopy_preempt_active();
2167 int tmppages, pages = 0;
2168 size_t pagesize_bits =
2169 qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2170 unsigned long start_page = pss->page;
2171 int res;
2172
2173 if (migrate_ram_is_ignored(pss->block)) {
2174 error_report("block %s should not be migrated !", pss->block->idstr);
2175 return 0;
2176 }
2177
2178 /* Update host page boundary information */
2179 pss_host_page_prepare(pss);
2180
2181 do {
2182 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2183
2184 /* Check the pages is dirty and if it is send it */
2185 if (page_dirty) {
2186 /*
2187 * Properly yield the lock only in postcopy preempt mode
2188 * because both migration thread and rp-return thread can
2189 * operate on the bitmaps.
2190 */
2191 if (preempt_active) {
2192 qemu_mutex_unlock(&rs->bitmap_mutex);
2193 }
2194 tmppages = migration_ops->ram_save_target_page(rs, pss);
2195 if (tmppages >= 0) {
2196 pages += tmppages;
2197 /*
2198 * Allow rate limiting to happen in the middle of huge pages if
2199 * something is sent in the current iteration.
2200 */
2201 if (pagesize_bits > 1 && tmppages > 0) {
2202 migration_rate_limit();
2203 }
2204 }
2205 if (preempt_active) {
2206 qemu_mutex_lock(&rs->bitmap_mutex);
2207 }
2208 } else {
2209 tmppages = 0;
2210 }
2211
2212 if (tmppages < 0) {
2213 pss_host_page_finish(pss);
2214 return tmppages;
2215 }
2216
2217 pss_find_next_dirty(pss);
2218 } while (pss_within_range(pss));
2219
2220 pss_host_page_finish(pss);
2221
2222 res = ram_save_release_protection(rs, pss, start_page);
2223 return (res < 0 ? res : pages);
2224 }
2225
2226 /**
2227 * ram_find_and_save_block: finds a dirty page and sends it to f
2228 *
2229 * Called within an RCU critical section.
2230 *
2231 * Returns the number of pages written where zero means no dirty pages,
2232 * or negative on error
2233 *
2234 * @rs: current RAM state
2235 *
2236 * On systems where host-page-size > target-page-size it will send all the
2237 * pages in a host page that are dirty.
2238 */
ram_find_and_save_block(RAMState * rs)2239 static int ram_find_and_save_block(RAMState *rs)
2240 {
2241 PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY];
2242 int pages = 0;
2243
2244 /* No dirty page as there is zero RAM */
2245 if (!rs->ram_bytes_total) {
2246 return pages;
2247 }
2248
2249 /*
2250 * Always keep last_seen_block/last_page valid during this procedure,
2251 * because find_dirty_block() relies on these values (e.g., we compare
2252 * last_seen_block with pss.block to see whether we searched all the
2253 * ramblocks) to detect the completion of migration. Having NULL value
2254 * of last_seen_block can conditionally cause below loop to run forever.
2255 */
2256 if (!rs->last_seen_block) {
2257 rs->last_seen_block = QLIST_FIRST_RCU(&ram_list.blocks);
2258 rs->last_page = 0;
2259 }
2260
2261 pss_init(pss, rs->last_seen_block, rs->last_page);
2262
2263 while (true){
2264 if (!get_queued_page(rs, pss)) {
2265 /* priority queue empty, so just search for something dirty */
2266 int res = find_dirty_block(rs, pss);
2267 if (res != PAGE_DIRTY_FOUND) {
2268 if (res == PAGE_ALL_CLEAN) {
2269 break;
2270 } else if (res == PAGE_TRY_AGAIN) {
2271 continue;
2272 } else if (res < 0) {
2273 pages = res;
2274 break;
2275 }
2276 }
2277 }
2278 pages = ram_save_host_page(rs, pss);
2279 if (pages) {
2280 break;
2281 }
2282 }
2283
2284 rs->last_seen_block = pss->block;
2285 rs->last_page = pss->page;
2286
2287 return pages;
2288 }
2289
ram_bytes_total_with_ignored(void)2290 static uint64_t ram_bytes_total_with_ignored(void)
2291 {
2292 RAMBlock *block;
2293 uint64_t total = 0;
2294
2295 RCU_READ_LOCK_GUARD();
2296
2297 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2298 total += block->used_length;
2299 }
2300 return total;
2301 }
2302
ram_bytes_total(void)2303 uint64_t ram_bytes_total(void)
2304 {
2305 RAMBlock *block;
2306 uint64_t total = 0;
2307
2308 RCU_READ_LOCK_GUARD();
2309
2310 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2311 total += block->used_length;
2312 }
2313 return total;
2314 }
2315
xbzrle_load_setup(void)2316 static void xbzrle_load_setup(void)
2317 {
2318 XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
2319 }
2320
xbzrle_load_cleanup(void)2321 static void xbzrle_load_cleanup(void)
2322 {
2323 g_free(XBZRLE.decoded_buf);
2324 XBZRLE.decoded_buf = NULL;
2325 }
2326
ram_state_cleanup(RAMState ** rsp)2327 static void ram_state_cleanup(RAMState **rsp)
2328 {
2329 if (*rsp) {
2330 migration_page_queue_free(*rsp);
2331 qemu_mutex_destroy(&(*rsp)->bitmap_mutex);
2332 qemu_mutex_destroy(&(*rsp)->src_page_req_mutex);
2333 g_free(*rsp);
2334 *rsp = NULL;
2335 }
2336 }
2337
xbzrle_cleanup(void)2338 static void xbzrle_cleanup(void)
2339 {
2340 XBZRLE_cache_lock();
2341 if (XBZRLE.cache) {
2342 cache_fini(XBZRLE.cache);
2343 g_free(XBZRLE.encoded_buf);
2344 g_free(XBZRLE.current_buf);
2345 g_free(XBZRLE.zero_target_page);
2346 XBZRLE.cache = NULL;
2347 XBZRLE.encoded_buf = NULL;
2348 XBZRLE.current_buf = NULL;
2349 XBZRLE.zero_target_page = NULL;
2350 }
2351 XBZRLE_cache_unlock();
2352 }
2353
ram_bitmaps_destroy(void)2354 static void ram_bitmaps_destroy(void)
2355 {
2356 RAMBlock *block;
2357
2358 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2359 g_free(block->clear_bmap);
2360 block->clear_bmap = NULL;
2361 g_free(block->bmap);
2362 block->bmap = NULL;
2363 g_free(block->file_bmap);
2364 block->file_bmap = NULL;
2365 }
2366 }
2367
ram_save_cleanup(void * opaque)2368 static void ram_save_cleanup(void *opaque)
2369 {
2370 RAMState **rsp = opaque;
2371
2372 /* We don't use dirty log with background snapshots */
2373 if (!migrate_background_snapshot()) {
2374 /* caller have hold BQL or is in a bh, so there is
2375 * no writing race against the migration bitmap
2376 */
2377 if (global_dirty_tracking & GLOBAL_DIRTY_MIGRATION) {
2378 /*
2379 * do not stop dirty log without starting it, since
2380 * memory_global_dirty_log_stop will assert that
2381 * memory_global_dirty_log_start/stop used in pairs
2382 */
2383 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
2384 }
2385 }
2386
2387 ram_bitmaps_destroy();
2388
2389 xbzrle_cleanup();
2390 multifd_ram_save_cleanup();
2391 ram_state_cleanup(rsp);
2392 g_free(migration_ops);
2393 migration_ops = NULL;
2394 }
2395
ram_state_reset(RAMState * rs)2396 static void ram_state_reset(RAMState *rs)
2397 {
2398 int i;
2399
2400 for (i = 0; i < RAM_CHANNEL_MAX; i++) {
2401 rs->pss[i].last_sent_block = NULL;
2402 }
2403
2404 rs->last_seen_block = NULL;
2405 rs->last_page = 0;
2406 rs->last_version = ram_list.version;
2407 rs->xbzrle_started = false;
2408 }
2409
2410 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2411
2412 /* **** functions for postcopy ***** */
2413
ram_postcopy_migrated_memory_release(MigrationState * ms)2414 void ram_postcopy_migrated_memory_release(MigrationState *ms)
2415 {
2416 struct RAMBlock *block;
2417
2418 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2419 unsigned long *bitmap = block->bmap;
2420 unsigned long range = block->used_length >> TARGET_PAGE_BITS;
2421 unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
2422
2423 while (run_start < range) {
2424 unsigned long run_end = find_next_bit(bitmap, range, run_start + 1);
2425 ram_discard_range(block->idstr,
2426 ((ram_addr_t)run_start) << TARGET_PAGE_BITS,
2427 ((ram_addr_t)(run_end - run_start))
2428 << TARGET_PAGE_BITS);
2429 run_start = find_next_zero_bit(bitmap, range, run_end + 1);
2430 }
2431 }
2432 }
2433
2434 /**
2435 * postcopy_send_discard_bm_ram: discard a RAMBlock
2436 *
2437 * Callback from postcopy_each_ram_send_discard for each RAMBlock
2438 *
2439 * @ms: current migration state
2440 * @block: RAMBlock to discard
2441 */
postcopy_send_discard_bm_ram(MigrationState * ms,RAMBlock * block)2442 static void postcopy_send_discard_bm_ram(MigrationState *ms, RAMBlock *block)
2443 {
2444 unsigned long end = block->used_length >> TARGET_PAGE_BITS;
2445 unsigned long current;
2446 unsigned long *bitmap = block->bmap;
2447
2448 for (current = 0; current < end; ) {
2449 unsigned long one = find_next_bit(bitmap, end, current);
2450 unsigned long zero, discard_length;
2451
2452 if (one >= end) {
2453 break;
2454 }
2455
2456 zero = find_next_zero_bit(bitmap, end, one + 1);
2457
2458 if (zero >= end) {
2459 discard_length = end - one;
2460 } else {
2461 discard_length = zero - one;
2462 }
2463 postcopy_discard_send_range(ms, one, discard_length);
2464 current = one + discard_length;
2465 }
2466 }
2467
2468 static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block);
2469
2470 /**
2471 * postcopy_each_ram_send_discard: discard all RAMBlocks
2472 *
2473 * Utility for the outgoing postcopy code.
2474 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2475 * passing it bitmap indexes and name.
2476 * (qemu_ram_foreach_block ends up passing unscaled lengths
2477 * which would mean postcopy code would have to deal with target page)
2478 *
2479 * @ms: current migration state
2480 */
postcopy_each_ram_send_discard(MigrationState * ms)2481 static void postcopy_each_ram_send_discard(MigrationState *ms)
2482 {
2483 struct RAMBlock *block;
2484
2485 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2486 postcopy_discard_send_init(ms, block->idstr);
2487
2488 /*
2489 * Deal with TPS != HPS and huge pages. It discard any partially sent
2490 * host-page size chunks, mark any partially dirty host-page size
2491 * chunks as all dirty. In this case the host-page is the host-page
2492 * for the particular RAMBlock, i.e. it might be a huge page.
2493 */
2494 postcopy_chunk_hostpages_pass(ms, block);
2495
2496 /*
2497 * Postcopy sends chunks of bitmap over the wire, but it
2498 * just needs indexes at this point, avoids it having
2499 * target page specific code.
2500 */
2501 postcopy_send_discard_bm_ram(ms, block);
2502 postcopy_discard_send_finish(ms);
2503 }
2504 }
2505
2506 /**
2507 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2508 *
2509 * Helper for postcopy_chunk_hostpages; it's called twice to
2510 * canonicalize the two bitmaps, that are similar, but one is
2511 * inverted.
2512 *
2513 * Postcopy requires that all target pages in a hostpage are dirty or
2514 * clean, not a mix. This function canonicalizes the bitmaps.
2515 *
2516 * @ms: current migration state
2517 * @block: block that contains the page we want to canonicalize
2518 */
postcopy_chunk_hostpages_pass(MigrationState * ms,RAMBlock * block)2519 static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block)
2520 {
2521 RAMState *rs = ram_state;
2522 unsigned long *bitmap = block->bmap;
2523 unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE;
2524 unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
2525 unsigned long run_start;
2526
2527 if (block->page_size == TARGET_PAGE_SIZE) {
2528 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2529 return;
2530 }
2531
2532 /* Find a dirty page */
2533 run_start = find_next_bit(bitmap, pages, 0);
2534
2535 while (run_start < pages) {
2536
2537 /*
2538 * If the start of this run of pages is in the middle of a host
2539 * page, then we need to fixup this host page.
2540 */
2541 if (QEMU_IS_ALIGNED(run_start, host_ratio)) {
2542 /* Find the end of this run */
2543 run_start = find_next_zero_bit(bitmap, pages, run_start + 1);
2544 /*
2545 * If the end isn't at the start of a host page, then the
2546 * run doesn't finish at the end of a host page
2547 * and we need to discard.
2548 */
2549 }
2550
2551 if (!QEMU_IS_ALIGNED(run_start, host_ratio)) {
2552 unsigned long page;
2553 unsigned long fixup_start_addr = QEMU_ALIGN_DOWN(run_start,
2554 host_ratio);
2555 run_start = QEMU_ALIGN_UP(run_start, host_ratio);
2556
2557 /* Clean up the bitmap */
2558 for (page = fixup_start_addr;
2559 page < fixup_start_addr + host_ratio; page++) {
2560 /*
2561 * Remark them as dirty, updating the count for any pages
2562 * that weren't previously dirty.
2563 */
2564 rs->migration_dirty_pages += !test_and_set_bit(page, bitmap);
2565 }
2566 }
2567
2568 /* Find the next dirty page for the next iteration */
2569 run_start = find_next_bit(bitmap, pages, run_start);
2570 }
2571 }
2572
2573 /**
2574 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2575 *
2576 * Transmit the set of pages to be discarded after precopy to the target
2577 * these are pages that:
2578 * a) Have been previously transmitted but are now dirty again
2579 * b) Pages that have never been transmitted, this ensures that
2580 * any pages on the destination that have been mapped by background
2581 * tasks get discarded (transparent huge pages is the specific concern)
2582 * Hopefully this is pretty sparse
2583 *
2584 * @ms: current migration state
2585 */
ram_postcopy_send_discard_bitmap(MigrationState * ms)2586 void ram_postcopy_send_discard_bitmap(MigrationState *ms)
2587 {
2588 RAMState *rs = ram_state;
2589
2590 RCU_READ_LOCK_GUARD();
2591
2592 /* This should be our last sync, the src is now paused */
2593 migration_bitmap_sync(rs, false);
2594
2595 /* Easiest way to make sure we don't resume in the middle of a host-page */
2596 rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL;
2597 rs->last_seen_block = NULL;
2598 rs->last_page = 0;
2599
2600 postcopy_each_ram_send_discard(ms);
2601
2602 trace_ram_postcopy_send_discard_bitmap();
2603 }
2604
2605 /**
2606 * ram_discard_range: discard dirtied pages at the beginning of postcopy
2607 *
2608 * Returns zero on success
2609 *
2610 * @rbname: name of the RAMBlock of the request. NULL means the
2611 * same that last one.
2612 * @start: RAMBlock starting page
2613 * @length: RAMBlock size
2614 */
ram_discard_range(const char * rbname,uint64_t start,size_t length)2615 int ram_discard_range(const char *rbname, uint64_t start, size_t length)
2616 {
2617 trace_ram_discard_range(rbname, start, length);
2618
2619 RCU_READ_LOCK_GUARD();
2620 RAMBlock *rb = qemu_ram_block_by_name(rbname);
2621
2622 if (!rb) {
2623 error_report("ram_discard_range: Failed to find block '%s'", rbname);
2624 return -1;
2625 }
2626
2627 /*
2628 * On source VM, we don't need to update the received bitmap since
2629 * we don't even have one.
2630 */
2631 if (rb->receivedmap) {
2632 bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
2633 length >> qemu_target_page_bits());
2634 }
2635
2636 return ram_block_discard_range(rb, start, length);
2637 }
2638
2639 /*
2640 * For every allocation, we will try not to crash the VM if the
2641 * allocation failed.
2642 */
xbzrle_init(Error ** errp)2643 static bool xbzrle_init(Error **errp)
2644 {
2645 if (!migrate_xbzrle()) {
2646 return true;
2647 }
2648
2649 XBZRLE_cache_lock();
2650
2651 XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE);
2652 if (!XBZRLE.zero_target_page) {
2653 error_setg(errp, "%s: Error allocating zero page", __func__);
2654 goto err_out;
2655 }
2656
2657 XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(),
2658 TARGET_PAGE_SIZE, errp);
2659 if (!XBZRLE.cache) {
2660 goto free_zero_page;
2661 }
2662
2663 XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
2664 if (!XBZRLE.encoded_buf) {
2665 error_setg(errp, "%s: Error allocating encoded_buf", __func__);
2666 goto free_cache;
2667 }
2668
2669 XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
2670 if (!XBZRLE.current_buf) {
2671 error_setg(errp, "%s: Error allocating current_buf", __func__);
2672 goto free_encoded_buf;
2673 }
2674
2675 /* We are all good */
2676 XBZRLE_cache_unlock();
2677 return true;
2678
2679 free_encoded_buf:
2680 g_free(XBZRLE.encoded_buf);
2681 XBZRLE.encoded_buf = NULL;
2682 free_cache:
2683 cache_fini(XBZRLE.cache);
2684 XBZRLE.cache = NULL;
2685 free_zero_page:
2686 g_free(XBZRLE.zero_target_page);
2687 XBZRLE.zero_target_page = NULL;
2688 err_out:
2689 XBZRLE_cache_unlock();
2690 return false;
2691 }
2692
ram_state_init(RAMState ** rsp,Error ** errp)2693 static bool ram_state_init(RAMState **rsp, Error **errp)
2694 {
2695 *rsp = g_try_new0(RAMState, 1);
2696
2697 if (!*rsp) {
2698 error_setg(errp, "%s: Init ramstate fail", __func__);
2699 return false;
2700 }
2701
2702 qemu_mutex_init(&(*rsp)->bitmap_mutex);
2703 qemu_mutex_init(&(*rsp)->src_page_req_mutex);
2704 QSIMPLEQ_INIT(&(*rsp)->src_page_requests);
2705 (*rsp)->ram_bytes_total = ram_bytes_total();
2706
2707 /*
2708 * Count the total number of pages used by ram blocks not including any
2709 * gaps due to alignment or unplugs.
2710 * This must match with the initial values of dirty bitmap.
2711 */
2712 (*rsp)->migration_dirty_pages = (*rsp)->ram_bytes_total >> TARGET_PAGE_BITS;
2713 ram_state_reset(*rsp);
2714
2715 return true;
2716 }
2717
ram_list_init_bitmaps(void)2718 static void ram_list_init_bitmaps(void)
2719 {
2720 MigrationState *ms = migrate_get_current();
2721 RAMBlock *block;
2722 unsigned long pages;
2723 uint8_t shift;
2724
2725 /* Skip setting bitmap if there is no RAM */
2726 if (ram_bytes_total()) {
2727 shift = ms->clear_bitmap_shift;
2728 if (shift > CLEAR_BITMAP_SHIFT_MAX) {
2729 error_report("clear_bitmap_shift (%u) too big, using "
2730 "max value (%u)", shift, CLEAR_BITMAP_SHIFT_MAX);
2731 shift = CLEAR_BITMAP_SHIFT_MAX;
2732 } else if (shift < CLEAR_BITMAP_SHIFT_MIN) {
2733 error_report("clear_bitmap_shift (%u) too small, using "
2734 "min value (%u)", shift, CLEAR_BITMAP_SHIFT_MIN);
2735 shift = CLEAR_BITMAP_SHIFT_MIN;
2736 }
2737
2738 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2739 pages = block->max_length >> TARGET_PAGE_BITS;
2740 /*
2741 * The initial dirty bitmap for migration must be set with all
2742 * ones to make sure we'll migrate every guest RAM page to
2743 * destination.
2744 * Here we set RAMBlock.bmap all to 1 because when rebegin a
2745 * new migration after a failed migration, ram_list.
2746 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
2747 * guest memory.
2748 */
2749 block->bmap = bitmap_new(pages);
2750 bitmap_set(block->bmap, 0, pages);
2751 if (migrate_mapped_ram()) {
2752 block->file_bmap = bitmap_new(pages);
2753 }
2754 block->clear_bmap_shift = shift;
2755 block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift));
2756 }
2757 }
2758 }
2759
migration_bitmap_clear_discarded_pages(RAMState * rs)2760 static void migration_bitmap_clear_discarded_pages(RAMState *rs)
2761 {
2762 unsigned long pages;
2763 RAMBlock *rb;
2764
2765 RCU_READ_LOCK_GUARD();
2766
2767 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
2768 pages = ramblock_dirty_bitmap_clear_discarded_pages(rb);
2769 rs->migration_dirty_pages -= pages;
2770 }
2771 }
2772
ram_init_bitmaps(RAMState * rs,Error ** errp)2773 static bool ram_init_bitmaps(RAMState *rs, Error **errp)
2774 {
2775 bool ret = true;
2776
2777 qemu_mutex_lock_ramlist();
2778
2779 WITH_RCU_READ_LOCK_GUARD() {
2780 ram_list_init_bitmaps();
2781 /* We don't use dirty log with background snapshots */
2782 if (!migrate_background_snapshot()) {
2783 ret = memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION, errp);
2784 if (!ret) {
2785 goto out_unlock;
2786 }
2787 migration_bitmap_sync_precopy(rs, false);
2788 }
2789 }
2790 out_unlock:
2791 qemu_mutex_unlock_ramlist();
2792
2793 if (!ret) {
2794 ram_bitmaps_destroy();
2795 return false;
2796 }
2797
2798 /*
2799 * After an eventual first bitmap sync, fixup the initial bitmap
2800 * containing all 1s to exclude any discarded pages from migration.
2801 */
2802 migration_bitmap_clear_discarded_pages(rs);
2803 return true;
2804 }
2805
ram_init_all(RAMState ** rsp,Error ** errp)2806 static int ram_init_all(RAMState **rsp, Error **errp)
2807 {
2808 if (!ram_state_init(rsp, errp)) {
2809 return -1;
2810 }
2811
2812 if (!xbzrle_init(errp)) {
2813 ram_state_cleanup(rsp);
2814 return -1;
2815 }
2816
2817 if (!ram_init_bitmaps(*rsp, errp)) {
2818 return -1;
2819 }
2820
2821 return 0;
2822 }
2823
ram_state_resume_prepare(RAMState * rs,QEMUFile * out)2824 static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out)
2825 {
2826 RAMBlock *block;
2827 uint64_t pages = 0;
2828
2829 /*
2830 * Postcopy is not using xbzrle/compression, so no need for that.
2831 * Also, since source are already halted, we don't need to care
2832 * about dirty page logging as well.
2833 */
2834
2835 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2836 pages += bitmap_count_one(block->bmap,
2837 block->used_length >> TARGET_PAGE_BITS);
2838 }
2839
2840 /* This may not be aligned with current bitmaps. Recalculate. */
2841 rs->migration_dirty_pages = pages;
2842
2843 ram_state_reset(rs);
2844
2845 /* Update RAMState cache of output QEMUFile */
2846 rs->pss[RAM_CHANNEL_PRECOPY].pss_channel = out;
2847
2848 trace_ram_state_resume_prepare(pages);
2849 }
2850
2851 /*
2852 * This function clears bits of the free pages reported by the caller from the
2853 * migration dirty bitmap. @addr is the host address corresponding to the
2854 * start of the continuous guest free pages, and @len is the total bytes of
2855 * those pages.
2856 */
qemu_guest_free_page_hint(void * addr,size_t len)2857 void qemu_guest_free_page_hint(void *addr, size_t len)
2858 {
2859 RAMBlock *block;
2860 ram_addr_t offset;
2861 size_t used_len, start, npages;
2862
2863 /* This function is currently expected to be used during live migration */
2864 if (!migration_is_setup_or_active()) {
2865 return;
2866 }
2867
2868 for (; len > 0; len -= used_len, addr += used_len) {
2869 block = qemu_ram_block_from_host(addr, false, &offset);
2870 if (unlikely(!block || offset >= block->used_length)) {
2871 /*
2872 * The implementation might not support RAMBlock resize during
2873 * live migration, but it could happen in theory with future
2874 * updates. So we add a check here to capture that case.
2875 */
2876 error_report_once("%s unexpected error", __func__);
2877 return;
2878 }
2879
2880 if (len <= block->used_length - offset) {
2881 used_len = len;
2882 } else {
2883 used_len = block->used_length - offset;
2884 }
2885
2886 start = offset >> TARGET_PAGE_BITS;
2887 npages = used_len >> TARGET_PAGE_BITS;
2888
2889 qemu_mutex_lock(&ram_state->bitmap_mutex);
2890 /*
2891 * The skipped free pages are equavalent to be sent from clear_bmap's
2892 * perspective, so clear the bits from the memory region bitmap which
2893 * are initially set. Otherwise those skipped pages will be sent in
2894 * the next round after syncing from the memory region bitmap.
2895 */
2896 migration_clear_memory_region_dirty_bitmap_range(block, start, npages);
2897 ram_state->migration_dirty_pages -=
2898 bitmap_count_one_with_offset(block->bmap, start, npages);
2899 bitmap_clear(block->bmap, start, npages);
2900 qemu_mutex_unlock(&ram_state->bitmap_mutex);
2901 }
2902 }
2903
2904 #define MAPPED_RAM_HDR_VERSION 1
2905 struct MappedRamHeader {
2906 uint32_t version;
2907 /*
2908 * The target's page size, so we know how many pages are in the
2909 * bitmap.
2910 */
2911 uint64_t page_size;
2912 /*
2913 * The offset in the migration file where the pages bitmap is
2914 * stored.
2915 */
2916 uint64_t bitmap_offset;
2917 /*
2918 * The offset in the migration file where the actual pages (data)
2919 * are stored.
2920 */
2921 uint64_t pages_offset;
2922 } QEMU_PACKED;
2923 typedef struct MappedRamHeader MappedRamHeader;
2924
mapped_ram_setup_ramblock(QEMUFile * file,RAMBlock * block)2925 static void mapped_ram_setup_ramblock(QEMUFile *file, RAMBlock *block)
2926 {
2927 g_autofree MappedRamHeader *header = NULL;
2928 size_t header_size, bitmap_size;
2929 long num_pages;
2930
2931 header = g_new0(MappedRamHeader, 1);
2932 header_size = sizeof(MappedRamHeader);
2933
2934 num_pages = block->used_length >> TARGET_PAGE_BITS;
2935 bitmap_size = BITS_TO_LONGS(num_pages) * sizeof(unsigned long);
2936
2937 /*
2938 * Save the file offsets of where the bitmap and the pages should
2939 * go as they are written at the end of migration and during the
2940 * iterative phase, respectively.
2941 */
2942 block->bitmap_offset = qemu_get_offset(file) + header_size;
2943 block->pages_offset = ROUND_UP(block->bitmap_offset +
2944 bitmap_size,
2945 MAPPED_RAM_FILE_OFFSET_ALIGNMENT);
2946
2947 header->version = cpu_to_be32(MAPPED_RAM_HDR_VERSION);
2948 header->page_size = cpu_to_be64(TARGET_PAGE_SIZE);
2949 header->bitmap_offset = cpu_to_be64(block->bitmap_offset);
2950 header->pages_offset = cpu_to_be64(block->pages_offset);
2951
2952 qemu_put_buffer(file, (uint8_t *) header, header_size);
2953
2954 /* prepare offset for next ramblock */
2955 qemu_set_offset(file, block->pages_offset + block->used_length, SEEK_SET);
2956 }
2957
mapped_ram_read_header(QEMUFile * file,MappedRamHeader * header,Error ** errp)2958 static bool mapped_ram_read_header(QEMUFile *file, MappedRamHeader *header,
2959 Error **errp)
2960 {
2961 size_t ret, header_size = sizeof(MappedRamHeader);
2962
2963 ret = qemu_get_buffer(file, (uint8_t *)header, header_size);
2964 if (ret != header_size) {
2965 error_setg(errp, "Could not read whole mapped-ram migration header "
2966 "(expected %zd, got %zd bytes)", header_size, ret);
2967 return false;
2968 }
2969
2970 /* migration stream is big-endian */
2971 header->version = be32_to_cpu(header->version);
2972
2973 if (header->version > MAPPED_RAM_HDR_VERSION) {
2974 error_setg(errp, "Migration mapped-ram capability version not "
2975 "supported (expected <= %d, got %d)", MAPPED_RAM_HDR_VERSION,
2976 header->version);
2977 return false;
2978 }
2979
2980 header->page_size = be64_to_cpu(header->page_size);
2981 header->bitmap_offset = be64_to_cpu(header->bitmap_offset);
2982 header->pages_offset = be64_to_cpu(header->pages_offset);
2983
2984 return true;
2985 }
2986
2987 /*
2988 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
2989 * long-running RCU critical section. When rcu-reclaims in the code
2990 * start to become numerous it will be necessary to reduce the
2991 * granularity of these critical sections.
2992 */
2993
2994 /**
2995 * ram_save_setup: Setup RAM for migration
2996 *
2997 * Returns zero to indicate success and negative for error
2998 *
2999 * @f: QEMUFile where to send the data
3000 * @opaque: RAMState pointer
3001 * @errp: pointer to Error*, to store an error if it happens.
3002 */
ram_save_setup(QEMUFile * f,void * opaque,Error ** errp)3003 static int ram_save_setup(QEMUFile *f, void *opaque, Error **errp)
3004 {
3005 RAMState **rsp = opaque;
3006 RAMBlock *block;
3007 int ret, max_hg_page_size;
3008
3009 /* migration has already setup the bitmap, reuse it. */
3010 if (!migration_in_colo_state()) {
3011 if (ram_init_all(rsp, errp) != 0) {
3012 return -1;
3013 }
3014 }
3015 (*rsp)->pss[RAM_CHANNEL_PRECOPY].pss_channel = f;
3016
3017 /*
3018 * ??? Mirrors the previous value of qemu_host_page_size,
3019 * but is this really what was intended for the migration?
3020 */
3021 max_hg_page_size = MAX(qemu_real_host_page_size(), TARGET_PAGE_SIZE);
3022
3023 WITH_RCU_READ_LOCK_GUARD() {
3024 qemu_put_be64(f, ram_bytes_total_with_ignored()
3025 | RAM_SAVE_FLAG_MEM_SIZE);
3026
3027 RAMBLOCK_FOREACH_MIGRATABLE(block) {
3028 qemu_put_byte(f, strlen(block->idstr));
3029 qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
3030 qemu_put_be64(f, block->used_length);
3031 if (migrate_postcopy_ram() &&
3032 block->page_size != max_hg_page_size) {
3033 qemu_put_be64(f, block->page_size);
3034 }
3035 if (migrate_ignore_shared()) {
3036 qemu_put_be64(f, block->mr->addr);
3037 }
3038
3039 if (migrate_mapped_ram()) {
3040 mapped_ram_setup_ramblock(f, block);
3041 }
3042 }
3043 }
3044
3045 ret = rdma_registration_start(f, RAM_CONTROL_SETUP);
3046 if (ret < 0) {
3047 error_setg(errp, "%s: failed to start RDMA registration", __func__);
3048 qemu_file_set_error(f, ret);
3049 return ret;
3050 }
3051
3052 ret = rdma_registration_stop(f, RAM_CONTROL_SETUP);
3053 if (ret < 0) {
3054 error_setg(errp, "%s: failed to stop RDMA registration", __func__);
3055 qemu_file_set_error(f, ret);
3056 return ret;
3057 }
3058
3059 migration_ops = g_malloc0(sizeof(MigrationOps));
3060
3061 if (migrate_multifd()) {
3062 multifd_ram_save_setup();
3063 migration_ops->ram_save_target_page = ram_save_target_page_multifd;
3064 } else {
3065 migration_ops->ram_save_target_page = ram_save_target_page_legacy;
3066 }
3067
3068 bql_unlock();
3069 ret = multifd_ram_flush_and_sync();
3070 bql_lock();
3071 if (ret < 0) {
3072 error_setg(errp, "%s: multifd synchronization failed", __func__);
3073 return ret;
3074 }
3075
3076 if (migrate_multifd() && !migrate_multifd_flush_after_each_section()
3077 && !migrate_mapped_ram()) {
3078 qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
3079 }
3080
3081 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3082 ret = qemu_fflush(f);
3083 if (ret < 0) {
3084 error_setg_errno(errp, -ret, "%s failed", __func__);
3085 }
3086 return ret;
3087 }
3088
ram_save_file_bmap(QEMUFile * f)3089 static void ram_save_file_bmap(QEMUFile *f)
3090 {
3091 RAMBlock *block;
3092
3093 RAMBLOCK_FOREACH_MIGRATABLE(block) {
3094 long num_pages = block->used_length >> TARGET_PAGE_BITS;
3095 long bitmap_size = BITS_TO_LONGS(num_pages) * sizeof(unsigned long);
3096
3097 qemu_put_buffer_at(f, (uint8_t *)block->file_bmap, bitmap_size,
3098 block->bitmap_offset);
3099 ram_transferred_add(bitmap_size);
3100
3101 /*
3102 * Free the bitmap here to catch any synchronization issues
3103 * with multifd channels. No channels should be sending pages
3104 * after we've written the bitmap to file.
3105 */
3106 g_free(block->file_bmap);
3107 block->file_bmap = NULL;
3108 }
3109 }
3110
ramblock_set_file_bmap_atomic(RAMBlock * block,ram_addr_t offset,bool set)3111 void ramblock_set_file_bmap_atomic(RAMBlock *block, ram_addr_t offset, bool set)
3112 {
3113 if (set) {
3114 set_bit_atomic(offset >> TARGET_PAGE_BITS, block->file_bmap);
3115 } else {
3116 clear_bit_atomic(offset >> TARGET_PAGE_BITS, block->file_bmap);
3117 }
3118 }
3119
3120 /**
3121 * ram_save_iterate: iterative stage for migration
3122 *
3123 * Returns zero to indicate success and negative for error
3124 *
3125 * @f: QEMUFile where to send the data
3126 * @opaque: RAMState pointer
3127 */
ram_save_iterate(QEMUFile * f,void * opaque)3128 static int ram_save_iterate(QEMUFile *f, void *opaque)
3129 {
3130 RAMState **temp = opaque;
3131 RAMState *rs = *temp;
3132 int ret = 0;
3133 int i;
3134 int64_t t0;
3135 int done = 0;
3136
3137 /*
3138 * We'll take this lock a little bit long, but it's okay for two reasons.
3139 * Firstly, the only possible other thread to take it is who calls
3140 * qemu_guest_free_page_hint(), which should be rare; secondly, see
3141 * MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which
3142 * guarantees that we'll at least released it in a regular basis.
3143 */
3144 WITH_QEMU_LOCK_GUARD(&rs->bitmap_mutex) {
3145 WITH_RCU_READ_LOCK_GUARD() {
3146 if (ram_list.version != rs->last_version) {
3147 ram_state_reset(rs);
3148 }
3149
3150 /* Read version before ram_list.blocks */
3151 smp_rmb();
3152
3153 ret = rdma_registration_start(f, RAM_CONTROL_ROUND);
3154 if (ret < 0) {
3155 qemu_file_set_error(f, ret);
3156 goto out;
3157 }
3158
3159 t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
3160 i = 0;
3161 while ((ret = migration_rate_exceeded(f)) == 0 ||
3162 postcopy_has_request(rs)) {
3163 int pages;
3164
3165 if (qemu_file_get_error(f)) {
3166 break;
3167 }
3168
3169 pages = ram_find_and_save_block(rs);
3170 /* no more pages to sent */
3171 if (pages == 0) {
3172 done = 1;
3173 break;
3174 }
3175
3176 if (pages < 0) {
3177 qemu_file_set_error(f, pages);
3178 break;
3179 }
3180
3181 rs->target_page_count += pages;
3182
3183 /*
3184 * we want to check in the 1st loop, just in case it was the 1st
3185 * time and we had to sync the dirty bitmap.
3186 * qemu_clock_get_ns() is a bit expensive, so we only check each
3187 * some iterations
3188 */
3189 if ((i & 63) == 0) {
3190 uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) /
3191 1000000;
3192 if (t1 > MAX_WAIT) {
3193 trace_ram_save_iterate_big_wait(t1, i);
3194 break;
3195 }
3196 }
3197 i++;
3198 }
3199 }
3200 }
3201
3202 /*
3203 * Must occur before EOS (or any QEMUFile operation)
3204 * because of RDMA protocol.
3205 */
3206 ret = rdma_registration_stop(f, RAM_CONTROL_ROUND);
3207 if (ret < 0) {
3208 qemu_file_set_error(f, ret);
3209 }
3210
3211 out:
3212 if (ret >= 0
3213 && migration_is_setup_or_active()) {
3214 if (migrate_multifd() && migrate_multifd_flush_after_each_section() &&
3215 !migrate_mapped_ram()) {
3216 ret = multifd_ram_flush_and_sync();
3217 if (ret < 0) {
3218 return ret;
3219 }
3220 }
3221
3222 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3223 ram_transferred_add(8);
3224 ret = qemu_fflush(f);
3225 }
3226 if (ret < 0) {
3227 return ret;
3228 }
3229
3230 return done;
3231 }
3232
3233 /**
3234 * ram_save_complete: function called to send the remaining amount of ram
3235 *
3236 * Returns zero to indicate success or negative on error
3237 *
3238 * Called with the BQL
3239 *
3240 * @f: QEMUFile where to send the data
3241 * @opaque: RAMState pointer
3242 */
ram_save_complete(QEMUFile * f,void * opaque)3243 static int ram_save_complete(QEMUFile *f, void *opaque)
3244 {
3245 RAMState **temp = opaque;
3246 RAMState *rs = *temp;
3247 int ret = 0;
3248
3249 rs->last_stage = !migration_in_colo_state();
3250
3251 WITH_RCU_READ_LOCK_GUARD() {
3252 if (!migration_in_postcopy()) {
3253 migration_bitmap_sync_precopy(rs, true);
3254 }
3255
3256 ret = rdma_registration_start(f, RAM_CONTROL_FINISH);
3257 if (ret < 0) {
3258 qemu_file_set_error(f, ret);
3259 return ret;
3260 }
3261
3262 /* try transferring iterative blocks of memory */
3263
3264 /* flush all remaining blocks regardless of rate limiting */
3265 qemu_mutex_lock(&rs->bitmap_mutex);
3266 while (true) {
3267 int pages;
3268
3269 pages = ram_find_and_save_block(rs);
3270 /* no more blocks to sent */
3271 if (pages == 0) {
3272 break;
3273 }
3274 if (pages < 0) {
3275 qemu_mutex_unlock(&rs->bitmap_mutex);
3276 return pages;
3277 }
3278 }
3279 qemu_mutex_unlock(&rs->bitmap_mutex);
3280
3281 ret = rdma_registration_stop(f, RAM_CONTROL_FINISH);
3282 if (ret < 0) {
3283 qemu_file_set_error(f, ret);
3284 return ret;
3285 }
3286 }
3287
3288 ret = multifd_ram_flush_and_sync();
3289 if (ret < 0) {
3290 return ret;
3291 }
3292
3293 if (migrate_mapped_ram()) {
3294 ram_save_file_bmap(f);
3295
3296 if (qemu_file_get_error(f)) {
3297 Error *local_err = NULL;
3298 int err = qemu_file_get_error_obj(f, &local_err);
3299
3300 error_reportf_err(local_err, "Failed to write bitmap to file: ");
3301 return -err;
3302 }
3303 }
3304
3305 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3306 return qemu_fflush(f);
3307 }
3308
ram_state_pending_estimate(void * opaque,uint64_t * must_precopy,uint64_t * can_postcopy)3309 static void ram_state_pending_estimate(void *opaque, uint64_t *must_precopy,
3310 uint64_t *can_postcopy)
3311 {
3312 RAMState **temp = opaque;
3313 RAMState *rs = *temp;
3314
3315 uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3316
3317 if (migrate_postcopy_ram()) {
3318 /* We can do postcopy, and all the data is postcopiable */
3319 *can_postcopy += remaining_size;
3320 } else {
3321 *must_precopy += remaining_size;
3322 }
3323 }
3324
ram_state_pending_exact(void * opaque,uint64_t * must_precopy,uint64_t * can_postcopy)3325 static void ram_state_pending_exact(void *opaque, uint64_t *must_precopy,
3326 uint64_t *can_postcopy)
3327 {
3328 RAMState **temp = opaque;
3329 RAMState *rs = *temp;
3330 uint64_t remaining_size;
3331
3332 if (!migration_in_postcopy()) {
3333 bql_lock();
3334 WITH_RCU_READ_LOCK_GUARD() {
3335 migration_bitmap_sync_precopy(rs, false);
3336 }
3337 bql_unlock();
3338 }
3339
3340 remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3341
3342 if (migrate_postcopy_ram()) {
3343 /* We can do postcopy, and all the data is postcopiable */
3344 *can_postcopy += remaining_size;
3345 } else {
3346 *must_precopy += remaining_size;
3347 }
3348 }
3349
load_xbzrle(QEMUFile * f,ram_addr_t addr,void * host)3350 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
3351 {
3352 unsigned int xh_len;
3353 int xh_flags;
3354 uint8_t *loaded_data;
3355
3356 /* extract RLE header */
3357 xh_flags = qemu_get_byte(f);
3358 xh_len = qemu_get_be16(f);
3359
3360 if (xh_flags != ENCODING_FLAG_XBZRLE) {
3361 error_report("Failed to load XBZRLE page - wrong compression!");
3362 return -1;
3363 }
3364
3365 if (xh_len > TARGET_PAGE_SIZE) {
3366 error_report("Failed to load XBZRLE page - len overflow!");
3367 return -1;
3368 }
3369 loaded_data = XBZRLE.decoded_buf;
3370 /* load data and decode */
3371 /* it can change loaded_data to point to an internal buffer */
3372 qemu_get_buffer_in_place(f, &loaded_data, xh_len);
3373
3374 /* decode RLE */
3375 if (xbzrle_decode_buffer(loaded_data, xh_len, host,
3376 TARGET_PAGE_SIZE) == -1) {
3377 error_report("Failed to load XBZRLE page - decode error!");
3378 return -1;
3379 }
3380
3381 return 0;
3382 }
3383
3384 /**
3385 * ram_block_from_stream: read a RAMBlock id from the migration stream
3386 *
3387 * Must be called from within a rcu critical section.
3388 *
3389 * Returns a pointer from within the RCU-protected ram_list.
3390 *
3391 * @mis: the migration incoming state pointer
3392 * @f: QEMUFile where to read the data from
3393 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3394 * @channel: the channel we're using
3395 */
ram_block_from_stream(MigrationIncomingState * mis,QEMUFile * f,int flags,int channel)3396 static inline RAMBlock *ram_block_from_stream(MigrationIncomingState *mis,
3397 QEMUFile *f, int flags,
3398 int channel)
3399 {
3400 RAMBlock *block = mis->last_recv_block[channel];
3401 char id[256];
3402 uint8_t len;
3403
3404 if (flags & RAM_SAVE_FLAG_CONTINUE) {
3405 if (!block) {
3406 error_report("Ack, bad migration stream!");
3407 return NULL;
3408 }
3409 return block;
3410 }
3411
3412 len = qemu_get_byte(f);
3413 qemu_get_buffer(f, (uint8_t *)id, len);
3414 id[len] = 0;
3415
3416 block = qemu_ram_block_by_name(id);
3417 if (!block) {
3418 error_report("Can't find block %s", id);
3419 return NULL;
3420 }
3421
3422 if (migrate_ram_is_ignored(block)) {
3423 error_report("block %s should not be migrated !", id);
3424 return NULL;
3425 }
3426
3427 mis->last_recv_block[channel] = block;
3428
3429 return block;
3430 }
3431
host_from_ram_block_offset(RAMBlock * block,ram_addr_t offset)3432 static inline void *host_from_ram_block_offset(RAMBlock *block,
3433 ram_addr_t offset)
3434 {
3435 if (!offset_in_ramblock(block, offset)) {
3436 return NULL;
3437 }
3438
3439 return block->host + offset;
3440 }
3441
host_page_from_ram_block_offset(RAMBlock * block,ram_addr_t offset)3442 static void *host_page_from_ram_block_offset(RAMBlock *block,
3443 ram_addr_t offset)
3444 {
3445 /* Note: Explicitly no check against offset_in_ramblock(). */
3446 return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block->host + offset),
3447 block->page_size);
3448 }
3449
host_page_offset_from_ram_block_offset(RAMBlock * block,ram_addr_t offset)3450 static ram_addr_t host_page_offset_from_ram_block_offset(RAMBlock *block,
3451 ram_addr_t offset)
3452 {
3453 return ((uintptr_t)block->host + offset) & (block->page_size - 1);
3454 }
3455
colo_record_bitmap(RAMBlock * block,ram_addr_t * normal,uint32_t pages)3456 void colo_record_bitmap(RAMBlock *block, ram_addr_t *normal, uint32_t pages)
3457 {
3458 qemu_mutex_lock(&ram_state->bitmap_mutex);
3459 for (int i = 0; i < pages; i++) {
3460 ram_addr_t offset = normal[i];
3461 ram_state->migration_dirty_pages += !test_and_set_bit(
3462 offset >> TARGET_PAGE_BITS,
3463 block->bmap);
3464 }
3465 qemu_mutex_unlock(&ram_state->bitmap_mutex);
3466 }
3467
colo_cache_from_block_offset(RAMBlock * block,ram_addr_t offset,bool record_bitmap)3468 static inline void *colo_cache_from_block_offset(RAMBlock *block,
3469 ram_addr_t offset, bool record_bitmap)
3470 {
3471 if (!offset_in_ramblock(block, offset)) {
3472 return NULL;
3473 }
3474 if (!block->colo_cache) {
3475 error_report("%s: colo_cache is NULL in block :%s",
3476 __func__, block->idstr);
3477 return NULL;
3478 }
3479
3480 /*
3481 * During colo checkpoint, we need bitmap of these migrated pages.
3482 * It help us to decide which pages in ram cache should be flushed
3483 * into VM's RAM later.
3484 */
3485 if (record_bitmap) {
3486 colo_record_bitmap(block, &offset, 1);
3487 }
3488 return block->colo_cache + offset;
3489 }
3490
3491 /**
3492 * ram_handle_zero: handle the zero page case
3493 *
3494 * If a page (or a whole RDMA chunk) has been
3495 * determined to be zero, then zap it.
3496 *
3497 * @host: host address for the zero page
3498 * @ch: what the page is filled from. We only support zero
3499 * @size: size of the zero page
3500 */
ram_handle_zero(void * host,uint64_t size)3501 void ram_handle_zero(void *host, uint64_t size)
3502 {
3503 if (!buffer_is_zero(host, size)) {
3504 memset(host, 0, size);
3505 }
3506 }
3507
colo_init_ram_state(void)3508 static void colo_init_ram_state(void)
3509 {
3510 Error *local_err = NULL;
3511
3512 if (!ram_state_init(&ram_state, &local_err)) {
3513 error_report_err(local_err);
3514 }
3515 }
3516
3517 /*
3518 * colo cache: this is for secondary VM, we cache the whole
3519 * memory of the secondary VM, it is need to hold the global lock
3520 * to call this helper.
3521 */
colo_init_ram_cache(void)3522 int colo_init_ram_cache(void)
3523 {
3524 RAMBlock *block;
3525
3526 WITH_RCU_READ_LOCK_GUARD() {
3527 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3528 block->colo_cache = qemu_anon_ram_alloc(block->used_length,
3529 NULL, false, false);
3530 if (!block->colo_cache) {
3531 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3532 "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
3533 block->used_length);
3534 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3535 if (block->colo_cache) {
3536 qemu_anon_ram_free(block->colo_cache, block->used_length);
3537 block->colo_cache = NULL;
3538 }
3539 }
3540 return -errno;
3541 }
3542 if (!machine_dump_guest_core(current_machine)) {
3543 qemu_madvise(block->colo_cache, block->used_length,
3544 QEMU_MADV_DONTDUMP);
3545 }
3546 }
3547 }
3548
3549 /*
3550 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3551 * with to decide which page in cache should be flushed into SVM's RAM. Here
3552 * we use the same name 'ram_bitmap' as for migration.
3553 */
3554 if (ram_bytes_total()) {
3555 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3556 unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
3557 block->bmap = bitmap_new(pages);
3558 }
3559 }
3560
3561 colo_init_ram_state();
3562 return 0;
3563 }
3564
3565 /* TODO: duplicated with ram_init_bitmaps */
colo_incoming_start_dirty_log(void)3566 void colo_incoming_start_dirty_log(void)
3567 {
3568 RAMBlock *block = NULL;
3569 Error *local_err = NULL;
3570
3571 /* For memory_global_dirty_log_start below. */
3572 bql_lock();
3573 qemu_mutex_lock_ramlist();
3574
3575 memory_global_dirty_log_sync(false);
3576 WITH_RCU_READ_LOCK_GUARD() {
3577 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3578 ramblock_sync_dirty_bitmap(ram_state, block);
3579 /* Discard this dirty bitmap record */
3580 bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS);
3581 }
3582 if (!memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION,
3583 &local_err)) {
3584 error_report_err(local_err);
3585 }
3586 }
3587 ram_state->migration_dirty_pages = 0;
3588 qemu_mutex_unlock_ramlist();
3589 bql_unlock();
3590 }
3591
3592 /* It is need to hold the global lock to call this helper */
colo_release_ram_cache(void)3593 void colo_release_ram_cache(void)
3594 {
3595 RAMBlock *block;
3596
3597 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
3598 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3599 g_free(block->bmap);
3600 block->bmap = NULL;
3601 }
3602
3603 WITH_RCU_READ_LOCK_GUARD() {
3604 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3605 if (block->colo_cache) {
3606 qemu_anon_ram_free(block->colo_cache, block->used_length);
3607 block->colo_cache = NULL;
3608 }
3609 }
3610 }
3611 ram_state_cleanup(&ram_state);
3612 }
3613
3614 /**
3615 * ram_load_setup: Setup RAM for migration incoming side
3616 *
3617 * Returns zero to indicate success and negative for error
3618 *
3619 * @f: QEMUFile where to receive the data
3620 * @opaque: RAMState pointer
3621 * @errp: pointer to Error*, to store an error if it happens.
3622 */
ram_load_setup(QEMUFile * f,void * opaque,Error ** errp)3623 static int ram_load_setup(QEMUFile *f, void *opaque, Error **errp)
3624 {
3625 xbzrle_load_setup();
3626 ramblock_recv_map_init();
3627
3628 return 0;
3629 }
3630
ram_load_cleanup(void * opaque)3631 static int ram_load_cleanup(void *opaque)
3632 {
3633 RAMBlock *rb;
3634
3635 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3636 qemu_ram_block_writeback(rb);
3637 }
3638
3639 xbzrle_load_cleanup();
3640
3641 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3642 g_free(rb->receivedmap);
3643 rb->receivedmap = NULL;
3644 }
3645
3646 return 0;
3647 }
3648
3649 /**
3650 * ram_postcopy_incoming_init: allocate postcopy data structures
3651 *
3652 * Returns 0 for success and negative if there was one error
3653 *
3654 * @mis: current migration incoming state
3655 *
3656 * Allocate data structures etc needed by incoming migration with
3657 * postcopy-ram. postcopy-ram's similarly names
3658 * postcopy_ram_incoming_init does the work.
3659 */
ram_postcopy_incoming_init(MigrationIncomingState * mis)3660 int ram_postcopy_incoming_init(MigrationIncomingState *mis)
3661 {
3662 return postcopy_ram_incoming_init(mis);
3663 }
3664
3665 /**
3666 * ram_load_postcopy: load a page in postcopy case
3667 *
3668 * Returns 0 for success or -errno in case of error
3669 *
3670 * Called in postcopy mode by ram_load().
3671 * rcu_read_lock is taken prior to this being called.
3672 *
3673 * @f: QEMUFile where to send the data
3674 * @channel: the channel to use for loading
3675 */
ram_load_postcopy(QEMUFile * f,int channel)3676 int ram_load_postcopy(QEMUFile *f, int channel)
3677 {
3678 int flags = 0, ret = 0;
3679 bool place_needed = false;
3680 bool matches_target_page_size = false;
3681 MigrationIncomingState *mis = migration_incoming_get_current();
3682 PostcopyTmpPage *tmp_page = &mis->postcopy_tmp_pages[channel];
3683
3684 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
3685 ram_addr_t addr;
3686 void *page_buffer = NULL;
3687 void *place_source = NULL;
3688 RAMBlock *block = NULL;
3689 uint8_t ch;
3690
3691 addr = qemu_get_be64(f);
3692
3693 /*
3694 * If qemu file error, we should stop here, and then "addr"
3695 * may be invalid
3696 */
3697 ret = qemu_file_get_error(f);
3698 if (ret) {
3699 break;
3700 }
3701
3702 flags = addr & ~TARGET_PAGE_MASK;
3703 addr &= TARGET_PAGE_MASK;
3704
3705 trace_ram_load_postcopy_loop(channel, (uint64_t)addr, flags);
3706 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE)) {
3707 block = ram_block_from_stream(mis, f, flags, channel);
3708 if (!block) {
3709 ret = -EINVAL;
3710 break;
3711 }
3712
3713 /*
3714 * Relying on used_length is racy and can result in false positives.
3715 * We might place pages beyond used_length in case RAM was shrunk
3716 * while in postcopy, which is fine - trying to place via
3717 * UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault.
3718 */
3719 if (!block->host || addr >= block->postcopy_length) {
3720 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
3721 ret = -EINVAL;
3722 break;
3723 }
3724 tmp_page->target_pages++;
3725 matches_target_page_size = block->page_size == TARGET_PAGE_SIZE;
3726 /*
3727 * Postcopy requires that we place whole host pages atomically;
3728 * these may be huge pages for RAMBlocks that are backed by
3729 * hugetlbfs.
3730 * To make it atomic, the data is read into a temporary page
3731 * that's moved into place later.
3732 * The migration protocol uses, possibly smaller, target-pages
3733 * however the source ensures it always sends all the components
3734 * of a host page in one chunk.
3735 */
3736 page_buffer = tmp_page->tmp_huge_page +
3737 host_page_offset_from_ram_block_offset(block, addr);
3738 /* If all TP are zero then we can optimise the place */
3739 if (tmp_page->target_pages == 1) {
3740 tmp_page->host_addr =
3741 host_page_from_ram_block_offset(block, addr);
3742 } else if (tmp_page->host_addr !=
3743 host_page_from_ram_block_offset(block, addr)) {
3744 /* not the 1st TP within the HP */
3745 error_report("Non-same host page detected on channel %d: "
3746 "Target host page %p, received host page %p "
3747 "(rb %s offset 0x"RAM_ADDR_FMT" target_pages %d)",
3748 channel, tmp_page->host_addr,
3749 host_page_from_ram_block_offset(block, addr),
3750 block->idstr, addr, tmp_page->target_pages);
3751 ret = -EINVAL;
3752 break;
3753 }
3754
3755 /*
3756 * If it's the last part of a host page then we place the host
3757 * page
3758 */
3759 if (tmp_page->target_pages ==
3760 (block->page_size / TARGET_PAGE_SIZE)) {
3761 place_needed = true;
3762 }
3763 place_source = tmp_page->tmp_huge_page;
3764 }
3765
3766 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
3767 case RAM_SAVE_FLAG_ZERO:
3768 ch = qemu_get_byte(f);
3769 if (ch != 0) {
3770 error_report("Found a zero page with value %d", ch);
3771 ret = -EINVAL;
3772 break;
3773 }
3774 /*
3775 * Can skip to set page_buffer when
3776 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
3777 */
3778 if (!matches_target_page_size) {
3779 memset(page_buffer, ch, TARGET_PAGE_SIZE);
3780 }
3781 break;
3782
3783 case RAM_SAVE_FLAG_PAGE:
3784 tmp_page->all_zero = false;
3785 if (!matches_target_page_size) {
3786 /* For huge pages, we always use temporary buffer */
3787 qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
3788 } else {
3789 /*
3790 * For small pages that matches target page size, we
3791 * avoid the qemu_file copy. Instead we directly use
3792 * the buffer of QEMUFile to place the page. Note: we
3793 * cannot do any QEMUFile operation before using that
3794 * buffer to make sure the buffer is valid when
3795 * placing the page.
3796 */
3797 qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
3798 TARGET_PAGE_SIZE);
3799 }
3800 break;
3801 case RAM_SAVE_FLAG_MULTIFD_FLUSH:
3802 multifd_recv_sync_main();
3803 break;
3804 case RAM_SAVE_FLAG_EOS:
3805 /* normal exit */
3806 if (migrate_multifd() &&
3807 migrate_multifd_flush_after_each_section()) {
3808 multifd_recv_sync_main();
3809 }
3810 break;
3811 default:
3812 error_report("Unknown combination of migration flags: 0x%x"
3813 " (postcopy mode)", flags);
3814 ret = -EINVAL;
3815 break;
3816 }
3817
3818 /* Detect for any possible file errors */
3819 if (!ret && qemu_file_get_error(f)) {
3820 ret = qemu_file_get_error(f);
3821 }
3822
3823 if (!ret && place_needed) {
3824 if (tmp_page->all_zero) {
3825 ret = postcopy_place_page_zero(mis, tmp_page->host_addr, block);
3826 } else {
3827 ret = postcopy_place_page(mis, tmp_page->host_addr,
3828 place_source, block);
3829 }
3830 place_needed = false;
3831 postcopy_temp_page_reset(tmp_page);
3832 }
3833 }
3834
3835 return ret;
3836 }
3837
postcopy_is_running(void)3838 static bool postcopy_is_running(void)
3839 {
3840 PostcopyState ps = postcopy_state_get();
3841 return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
3842 }
3843
3844 /*
3845 * Flush content of RAM cache into SVM's memory.
3846 * Only flush the pages that be dirtied by PVM or SVM or both.
3847 */
colo_flush_ram_cache(void)3848 void colo_flush_ram_cache(void)
3849 {
3850 RAMBlock *block = NULL;
3851 void *dst_host;
3852 void *src_host;
3853 unsigned long offset = 0;
3854
3855 memory_global_dirty_log_sync(false);
3856 qemu_mutex_lock(&ram_state->bitmap_mutex);
3857 WITH_RCU_READ_LOCK_GUARD() {
3858 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3859 ramblock_sync_dirty_bitmap(ram_state, block);
3860 }
3861 }
3862
3863 trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
3864 WITH_RCU_READ_LOCK_GUARD() {
3865 block = QLIST_FIRST_RCU(&ram_list.blocks);
3866
3867 while (block) {
3868 unsigned long num = 0;
3869
3870 offset = colo_bitmap_find_dirty(ram_state, block, offset, &num);
3871 if (!offset_in_ramblock(block,
3872 ((ram_addr_t)offset) << TARGET_PAGE_BITS)) {
3873 offset = 0;
3874 num = 0;
3875 block = QLIST_NEXT_RCU(block, next);
3876 } else {
3877 unsigned long i = 0;
3878
3879 for (i = 0; i < num; i++) {
3880 migration_bitmap_clear_dirty(ram_state, block, offset + i);
3881 }
3882 dst_host = block->host
3883 + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
3884 src_host = block->colo_cache
3885 + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
3886 memcpy(dst_host, src_host, TARGET_PAGE_SIZE * num);
3887 offset += num;
3888 }
3889 }
3890 }
3891 qemu_mutex_unlock(&ram_state->bitmap_mutex);
3892 trace_colo_flush_ram_cache_end();
3893 }
3894
ram_load_multifd_pages(void * host_addr,size_t size,uint64_t offset)3895 static size_t ram_load_multifd_pages(void *host_addr, size_t size,
3896 uint64_t offset)
3897 {
3898 MultiFDRecvData *data = multifd_get_recv_data();
3899
3900 data->opaque = host_addr;
3901 data->file_offset = offset;
3902 data->size = size;
3903
3904 if (!multifd_recv()) {
3905 return 0;
3906 }
3907
3908 return size;
3909 }
3910
read_ramblock_mapped_ram(QEMUFile * f,RAMBlock * block,long num_pages,unsigned long * bitmap,Error ** errp)3911 static bool read_ramblock_mapped_ram(QEMUFile *f, RAMBlock *block,
3912 long num_pages, unsigned long *bitmap,
3913 Error **errp)
3914 {
3915 ERRP_GUARD();
3916 unsigned long set_bit_idx, clear_bit_idx;
3917 ram_addr_t offset;
3918 void *host;
3919 size_t read, unread, size;
3920
3921 for (set_bit_idx = find_first_bit(bitmap, num_pages);
3922 set_bit_idx < num_pages;
3923 set_bit_idx = find_next_bit(bitmap, num_pages, clear_bit_idx + 1)) {
3924
3925 clear_bit_idx = find_next_zero_bit(bitmap, num_pages, set_bit_idx + 1);
3926
3927 unread = TARGET_PAGE_SIZE * (clear_bit_idx - set_bit_idx);
3928 offset = set_bit_idx << TARGET_PAGE_BITS;
3929
3930 while (unread > 0) {
3931 host = host_from_ram_block_offset(block, offset);
3932 if (!host) {
3933 error_setg(errp, "page outside of ramblock %s range",
3934 block->idstr);
3935 return false;
3936 }
3937
3938 size = MIN(unread, MAPPED_RAM_LOAD_BUF_SIZE);
3939
3940 if (migrate_multifd()) {
3941 read = ram_load_multifd_pages(host, size,
3942 block->pages_offset + offset);
3943 } else {
3944 read = qemu_get_buffer_at(f, host, size,
3945 block->pages_offset + offset);
3946 }
3947
3948 if (!read) {
3949 goto err;
3950 }
3951 offset += read;
3952 unread -= read;
3953 }
3954 }
3955
3956 return true;
3957
3958 err:
3959 qemu_file_get_error_obj(f, errp);
3960 error_prepend(errp, "(%s) failed to read page " RAM_ADDR_FMT
3961 "from file offset %" PRIx64 ": ", block->idstr, offset,
3962 block->pages_offset + offset);
3963 return false;
3964 }
3965
parse_ramblock_mapped_ram(QEMUFile * f,RAMBlock * block,ram_addr_t length,Error ** errp)3966 static void parse_ramblock_mapped_ram(QEMUFile *f, RAMBlock *block,
3967 ram_addr_t length, Error **errp)
3968 {
3969 g_autofree unsigned long *bitmap = NULL;
3970 MappedRamHeader header;
3971 size_t bitmap_size;
3972 long num_pages;
3973
3974 if (!mapped_ram_read_header(f, &header, errp)) {
3975 return;
3976 }
3977
3978 block->pages_offset = header.pages_offset;
3979
3980 /*
3981 * Check the alignment of the file region that contains pages. We
3982 * don't enforce MAPPED_RAM_FILE_OFFSET_ALIGNMENT to allow that
3983 * value to change in the future. Do only a sanity check with page
3984 * size alignment.
3985 */
3986 if (!QEMU_IS_ALIGNED(block->pages_offset, TARGET_PAGE_SIZE)) {
3987 error_setg(errp,
3988 "Error reading ramblock %s pages, region has bad alignment",
3989 block->idstr);
3990 return;
3991 }
3992
3993 num_pages = length / header.page_size;
3994 bitmap_size = BITS_TO_LONGS(num_pages) * sizeof(unsigned long);
3995
3996 bitmap = g_malloc0(bitmap_size);
3997 if (qemu_get_buffer_at(f, (uint8_t *)bitmap, bitmap_size,
3998 header.bitmap_offset) != bitmap_size) {
3999 error_setg(errp, "Error reading dirty bitmap");
4000 return;
4001 }
4002
4003 if (!read_ramblock_mapped_ram(f, block, num_pages, bitmap, errp)) {
4004 return;
4005 }
4006
4007 /* Skip pages array */
4008 qemu_set_offset(f, block->pages_offset + length, SEEK_SET);
4009
4010 return;
4011 }
4012
parse_ramblock(QEMUFile * f,RAMBlock * block,ram_addr_t length)4013 static int parse_ramblock(QEMUFile *f, RAMBlock *block, ram_addr_t length)
4014 {
4015 int ret = 0;
4016 /* ADVISE is earlier, it shows the source has the postcopy capability on */
4017 bool postcopy_advised = migration_incoming_postcopy_advised();
4018 int max_hg_page_size;
4019 Error *local_err = NULL;
4020
4021 assert(block);
4022
4023 if (migrate_mapped_ram()) {
4024 parse_ramblock_mapped_ram(f, block, length, &local_err);
4025 if (local_err) {
4026 error_report_err(local_err);
4027 return -EINVAL;
4028 }
4029 return 0;
4030 }
4031
4032 if (!qemu_ram_is_migratable(block)) {
4033 error_report("block %s should not be migrated !", block->idstr);
4034 return -EINVAL;
4035 }
4036
4037 if (length != block->used_length) {
4038 ret = qemu_ram_resize(block, length, &local_err);
4039 if (local_err) {
4040 error_report_err(local_err);
4041 return ret;
4042 }
4043 }
4044
4045 /*
4046 * ??? Mirrors the previous value of qemu_host_page_size,
4047 * but is this really what was intended for the migration?
4048 */
4049 max_hg_page_size = MAX(qemu_real_host_page_size(), TARGET_PAGE_SIZE);
4050
4051 /* For postcopy we need to check hugepage sizes match */
4052 if (postcopy_advised && migrate_postcopy_ram() &&
4053 block->page_size != max_hg_page_size) {
4054 uint64_t remote_page_size = qemu_get_be64(f);
4055 if (remote_page_size != block->page_size) {
4056 error_report("Mismatched RAM page size %s "
4057 "(local) %zd != %" PRId64, block->idstr,
4058 block->page_size, remote_page_size);
4059 return -EINVAL;
4060 }
4061 }
4062 if (migrate_ignore_shared()) {
4063 hwaddr addr = qemu_get_be64(f);
4064 if (migrate_ram_is_ignored(block) &&
4065 block->mr->addr != addr) {
4066 error_report("Mismatched GPAs for block %s "
4067 "%" PRId64 "!= %" PRId64, block->idstr,
4068 (uint64_t)addr, (uint64_t)block->mr->addr);
4069 return -EINVAL;
4070 }
4071 }
4072 ret = rdma_block_notification_handle(f, block->idstr);
4073 if (ret < 0) {
4074 qemu_file_set_error(f, ret);
4075 }
4076
4077 return ret;
4078 }
4079
parse_ramblocks(QEMUFile * f,ram_addr_t total_ram_bytes)4080 static int parse_ramblocks(QEMUFile *f, ram_addr_t total_ram_bytes)
4081 {
4082 int ret = 0;
4083
4084 /* Synchronize RAM block list */
4085 while (!ret && total_ram_bytes) {
4086 RAMBlock *block;
4087 char id[256];
4088 ram_addr_t length;
4089 int len = qemu_get_byte(f);
4090
4091 qemu_get_buffer(f, (uint8_t *)id, len);
4092 id[len] = 0;
4093 length = qemu_get_be64(f);
4094
4095 block = qemu_ram_block_by_name(id);
4096 if (block) {
4097 ret = parse_ramblock(f, block, length);
4098 } else {
4099 error_report("Unknown ramblock \"%s\", cannot accept "
4100 "migration", id);
4101 ret = -EINVAL;
4102 }
4103 total_ram_bytes -= length;
4104 }
4105
4106 return ret;
4107 }
4108
4109 /**
4110 * ram_load_precopy: load pages in precopy case
4111 *
4112 * Returns 0 for success or -errno in case of error
4113 *
4114 * Called in precopy mode by ram_load().
4115 * rcu_read_lock is taken prior to this being called.
4116 *
4117 * @f: QEMUFile where to send the data
4118 */
ram_load_precopy(QEMUFile * f)4119 static int ram_load_precopy(QEMUFile *f)
4120 {
4121 MigrationIncomingState *mis = migration_incoming_get_current();
4122 int flags = 0, ret = 0, invalid_flags = 0, i = 0;
4123
4124 if (migrate_mapped_ram()) {
4125 invalid_flags |= (RAM_SAVE_FLAG_HOOK | RAM_SAVE_FLAG_MULTIFD_FLUSH |
4126 RAM_SAVE_FLAG_PAGE | RAM_SAVE_FLAG_XBZRLE |
4127 RAM_SAVE_FLAG_ZERO);
4128 }
4129
4130 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
4131 ram_addr_t addr;
4132 void *host = NULL, *host_bak = NULL;
4133 uint8_t ch;
4134
4135 /*
4136 * Yield periodically to let main loop run, but an iteration of
4137 * the main loop is expensive, so do it each some iterations
4138 */
4139 if ((i & 32767) == 0 && qemu_in_coroutine()) {
4140 aio_co_schedule(qemu_get_current_aio_context(),
4141 qemu_coroutine_self());
4142 qemu_coroutine_yield();
4143 }
4144 i++;
4145
4146 addr = qemu_get_be64(f);
4147 ret = qemu_file_get_error(f);
4148 if (ret) {
4149 error_report("Getting RAM address failed");
4150 break;
4151 }
4152
4153 flags = addr & ~TARGET_PAGE_MASK;
4154 addr &= TARGET_PAGE_MASK;
4155
4156 if (flags & invalid_flags) {
4157 error_report("Unexpected RAM flags: %d", flags & invalid_flags);
4158
4159 ret = -EINVAL;
4160 break;
4161 }
4162
4163 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
4164 RAM_SAVE_FLAG_XBZRLE)) {
4165 RAMBlock *block = ram_block_from_stream(mis, f, flags,
4166 RAM_CHANNEL_PRECOPY);
4167
4168 host = host_from_ram_block_offset(block, addr);
4169 /*
4170 * After going into COLO stage, we should not load the page
4171 * into SVM's memory directly, we put them into colo_cache firstly.
4172 * NOTE: We need to keep a copy of SVM's ram in colo_cache.
4173 * Previously, we copied all these memory in preparing stage of COLO
4174 * while we need to stop VM, which is a time-consuming process.
4175 * Here we optimize it by a trick, back-up every page while in
4176 * migration process while COLO is enabled, though it affects the
4177 * speed of the migration, but it obviously reduce the downtime of
4178 * back-up all SVM'S memory in COLO preparing stage.
4179 */
4180 if (migration_incoming_colo_enabled()) {
4181 if (migration_incoming_in_colo_state()) {
4182 /* In COLO stage, put all pages into cache temporarily */
4183 host = colo_cache_from_block_offset(block, addr, true);
4184 } else {
4185 /*
4186 * In migration stage but before COLO stage,
4187 * Put all pages into both cache and SVM's memory.
4188 */
4189 host_bak = colo_cache_from_block_offset(block, addr, false);
4190 }
4191 }
4192 if (!host) {
4193 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
4194 ret = -EINVAL;
4195 break;
4196 }
4197 if (!migration_incoming_in_colo_state()) {
4198 ramblock_recv_bitmap_set(block, host);
4199 }
4200
4201 trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
4202 }
4203
4204 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
4205 case RAM_SAVE_FLAG_MEM_SIZE:
4206 ret = parse_ramblocks(f, addr);
4207 /*
4208 * For mapped-ram migration (to a file) using multifd, we sync
4209 * once and for all here to make sure all tasks we queued to
4210 * multifd threads are completed, so that all the ramblocks
4211 * (including all the guest memory pages within) are fully
4212 * loaded after this sync returns.
4213 */
4214 if (migrate_mapped_ram()) {
4215 multifd_recv_sync_main();
4216 }
4217 break;
4218
4219 case RAM_SAVE_FLAG_ZERO:
4220 ch = qemu_get_byte(f);
4221 if (ch != 0) {
4222 error_report("Found a zero page with value %d", ch);
4223 ret = -EINVAL;
4224 break;
4225 }
4226 ram_handle_zero(host, TARGET_PAGE_SIZE);
4227 break;
4228
4229 case RAM_SAVE_FLAG_PAGE:
4230 qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
4231 break;
4232
4233 case RAM_SAVE_FLAG_XBZRLE:
4234 if (load_xbzrle(f, addr, host) < 0) {
4235 error_report("Failed to decompress XBZRLE page at "
4236 RAM_ADDR_FMT, addr);
4237 ret = -EINVAL;
4238 break;
4239 }
4240 break;
4241 case RAM_SAVE_FLAG_MULTIFD_FLUSH:
4242 multifd_recv_sync_main();
4243 break;
4244 case RAM_SAVE_FLAG_EOS:
4245 /* normal exit */
4246 if (migrate_multifd() &&
4247 migrate_multifd_flush_after_each_section() &&
4248 /*
4249 * Mapped-ram migration flushes once and for all after
4250 * parsing ramblocks. Always ignore EOS for it.
4251 */
4252 !migrate_mapped_ram()) {
4253 multifd_recv_sync_main();
4254 }
4255 break;
4256 case RAM_SAVE_FLAG_HOOK:
4257 ret = rdma_registration_handle(f);
4258 if (ret < 0) {
4259 qemu_file_set_error(f, ret);
4260 }
4261 break;
4262 default:
4263 error_report("Unknown combination of migration flags: 0x%x", flags);
4264 ret = -EINVAL;
4265 }
4266 if (!ret) {
4267 ret = qemu_file_get_error(f);
4268 }
4269 if (!ret && host_bak) {
4270 memcpy(host_bak, host, TARGET_PAGE_SIZE);
4271 }
4272 }
4273
4274 return ret;
4275 }
4276
ram_load(QEMUFile * f,void * opaque,int version_id)4277 static int ram_load(QEMUFile *f, void *opaque, int version_id)
4278 {
4279 int ret = 0;
4280 static uint64_t seq_iter;
4281 /*
4282 * If system is running in postcopy mode, page inserts to host memory must
4283 * be atomic
4284 */
4285 bool postcopy_running = postcopy_is_running();
4286
4287 seq_iter++;
4288
4289 if (version_id != 4) {
4290 return -EINVAL;
4291 }
4292
4293 /*
4294 * This RCU critical section can be very long running.
4295 * When RCU reclaims in the code start to become numerous,
4296 * it will be necessary to reduce the granularity of this
4297 * critical section.
4298 */
4299 WITH_RCU_READ_LOCK_GUARD() {
4300 if (postcopy_running) {
4301 /*
4302 * Note! Here RAM_CHANNEL_PRECOPY is the precopy channel of
4303 * postcopy migration, we have another RAM_CHANNEL_POSTCOPY to
4304 * service fast page faults.
4305 */
4306 ret = ram_load_postcopy(f, RAM_CHANNEL_PRECOPY);
4307 } else {
4308 ret = ram_load_precopy(f);
4309 }
4310 }
4311 trace_ram_load_complete(ret, seq_iter);
4312
4313 return ret;
4314 }
4315
ram_has_postcopy(void * opaque)4316 static bool ram_has_postcopy(void *opaque)
4317 {
4318 RAMBlock *rb;
4319 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
4320 if (ramblock_is_pmem(rb)) {
4321 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4322 "is not supported now!", rb->idstr, rb->host);
4323 return false;
4324 }
4325 }
4326
4327 return migrate_postcopy_ram();
4328 }
4329
4330 /* Sync all the dirty bitmap with destination VM. */
ram_dirty_bitmap_sync_all(MigrationState * s,RAMState * rs)4331 static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs)
4332 {
4333 RAMBlock *block;
4334 QEMUFile *file = s->to_dst_file;
4335
4336 trace_ram_dirty_bitmap_sync_start();
4337
4338 qatomic_set(&rs->postcopy_bmap_sync_requested, 0);
4339 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
4340 qemu_savevm_send_recv_bitmap(file, block->idstr);
4341 trace_ram_dirty_bitmap_request(block->idstr);
4342 qatomic_inc(&rs->postcopy_bmap_sync_requested);
4343 }
4344
4345 trace_ram_dirty_bitmap_sync_wait();
4346
4347 /* Wait until all the ramblocks' dirty bitmap synced */
4348 while (qatomic_read(&rs->postcopy_bmap_sync_requested)) {
4349 if (migration_rp_wait(s)) {
4350 return -1;
4351 }
4352 }
4353
4354 trace_ram_dirty_bitmap_sync_complete();
4355
4356 return 0;
4357 }
4358
4359 /*
4360 * Read the received bitmap, revert it as the initial dirty bitmap.
4361 * This is only used when the postcopy migration is paused but wants
4362 * to resume from a middle point.
4363 *
4364 * Returns true if succeeded, false for errors.
4365 */
ram_dirty_bitmap_reload(MigrationState * s,RAMBlock * block,Error ** errp)4366 bool ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block, Error **errp)
4367 {
4368 /* from_dst_file is always valid because we're within rp_thread */
4369 QEMUFile *file = s->rp_state.from_dst_file;
4370 g_autofree unsigned long *le_bitmap = NULL;
4371 unsigned long nbits = block->used_length >> TARGET_PAGE_BITS;
4372 uint64_t local_size = DIV_ROUND_UP(nbits, 8);
4373 uint64_t size, end_mark;
4374 RAMState *rs = ram_state;
4375
4376 trace_ram_dirty_bitmap_reload_begin(block->idstr);
4377
4378 if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
4379 error_setg(errp, "Reload bitmap in incorrect state %s",
4380 MigrationStatus_str(s->state));
4381 return false;
4382 }
4383
4384 /*
4385 * Note: see comments in ramblock_recv_bitmap_send() on why we
4386 * need the endianness conversion, and the paddings.
4387 */
4388 local_size = ROUND_UP(local_size, 8);
4389
4390 /* Add paddings */
4391 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
4392
4393 size = qemu_get_be64(file);
4394
4395 /* The size of the bitmap should match with our ramblock */
4396 if (size != local_size) {
4397 error_setg(errp, "ramblock '%s' bitmap size mismatch (0x%"PRIx64
4398 " != 0x%"PRIx64")", block->idstr, size, local_size);
4399 return false;
4400 }
4401
4402 size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size);
4403 end_mark = qemu_get_be64(file);
4404
4405 if (qemu_file_get_error(file) || size != local_size) {
4406 error_setg(errp, "read bitmap failed for ramblock '%s': "
4407 "(size 0x%"PRIx64", got: 0x%"PRIx64")",
4408 block->idstr, local_size, size);
4409 return false;
4410 }
4411
4412 if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) {
4413 error_setg(errp, "ramblock '%s' end mark incorrect: 0x%"PRIx64,
4414 block->idstr, end_mark);
4415 return false;
4416 }
4417
4418 /*
4419 * Endianness conversion. We are during postcopy (though paused).
4420 * The dirty bitmap won't change. We can directly modify it.
4421 */
4422 bitmap_from_le(block->bmap, le_bitmap, nbits);
4423
4424 /*
4425 * What we received is "received bitmap". Revert it as the initial
4426 * dirty bitmap for this ramblock.
4427 */
4428 bitmap_complement(block->bmap, block->bmap, nbits);
4429
4430 /* Clear dirty bits of discarded ranges that we don't want to migrate. */
4431 ramblock_dirty_bitmap_clear_discarded_pages(block);
4432
4433 /* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */
4434 trace_ram_dirty_bitmap_reload_complete(block->idstr);
4435
4436 qatomic_dec(&rs->postcopy_bmap_sync_requested);
4437
4438 /*
4439 * We succeeded to sync bitmap for current ramblock. Always kick the
4440 * migration thread to check whether all requested bitmaps are
4441 * reloaded. NOTE: it's racy to only kick when requested==0, because
4442 * we don't know whether the migration thread may still be increasing
4443 * it.
4444 */
4445 migration_rp_kick(s);
4446
4447 return true;
4448 }
4449
ram_resume_prepare(MigrationState * s,void * opaque)4450 static int ram_resume_prepare(MigrationState *s, void *opaque)
4451 {
4452 RAMState *rs = *(RAMState **)opaque;
4453 int ret;
4454
4455 ret = ram_dirty_bitmap_sync_all(s, rs);
4456 if (ret) {
4457 return ret;
4458 }
4459
4460 ram_state_resume_prepare(rs, s->to_dst_file);
4461
4462 return 0;
4463 }
4464
postcopy_preempt_shutdown_file(MigrationState * s)4465 void postcopy_preempt_shutdown_file(MigrationState *s)
4466 {
4467 qemu_put_be64(s->postcopy_qemufile_src, RAM_SAVE_FLAG_EOS);
4468 qemu_fflush(s->postcopy_qemufile_src);
4469 }
4470
4471 static SaveVMHandlers savevm_ram_handlers = {
4472 .save_setup = ram_save_setup,
4473 .save_live_iterate = ram_save_iterate,
4474 .save_live_complete_postcopy = ram_save_complete,
4475 .save_live_complete_precopy = ram_save_complete,
4476 .has_postcopy = ram_has_postcopy,
4477 .state_pending_exact = ram_state_pending_exact,
4478 .state_pending_estimate = ram_state_pending_estimate,
4479 .load_state = ram_load,
4480 .save_cleanup = ram_save_cleanup,
4481 .load_setup = ram_load_setup,
4482 .load_cleanup = ram_load_cleanup,
4483 .resume_prepare = ram_resume_prepare,
4484 };
4485
ram_mig_ram_block_resized(RAMBlockNotifier * n,void * host,size_t old_size,size_t new_size)4486 static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host,
4487 size_t old_size, size_t new_size)
4488 {
4489 PostcopyState ps = postcopy_state_get();
4490 ram_addr_t offset;
4491 RAMBlock *rb = qemu_ram_block_from_host(host, false, &offset);
4492 Error *err = NULL;
4493
4494 if (!rb) {
4495 error_report("RAM block not found");
4496 return;
4497 }
4498
4499 if (migrate_ram_is_ignored(rb)) {
4500 return;
4501 }
4502
4503 if (!migration_is_idle()) {
4504 /*
4505 * Precopy code on the source cannot deal with the size of RAM blocks
4506 * changing at random points in time - especially after sending the
4507 * RAM block sizes in the migration stream, they must no longer change.
4508 * Abort and indicate a proper reason.
4509 */
4510 error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr);
4511 migration_cancel(err);
4512 error_free(err);
4513 }
4514
4515 switch (ps) {
4516 case POSTCOPY_INCOMING_ADVISE:
4517 /*
4518 * Update what ram_postcopy_incoming_init()->init_range() does at the
4519 * time postcopy was advised. Syncing RAM blocks with the source will
4520 * result in RAM resizes.
4521 */
4522 if (old_size < new_size) {
4523 if (ram_discard_range(rb->idstr, old_size, new_size - old_size)) {
4524 error_report("RAM block '%s' discard of resized RAM failed",
4525 rb->idstr);
4526 }
4527 }
4528 rb->postcopy_length = new_size;
4529 break;
4530 case POSTCOPY_INCOMING_NONE:
4531 case POSTCOPY_INCOMING_RUNNING:
4532 case POSTCOPY_INCOMING_END:
4533 /*
4534 * Once our guest is running, postcopy does no longer care about
4535 * resizes. When growing, the new memory was not available on the
4536 * source, no handler needed.
4537 */
4538 break;
4539 default:
4540 error_report("RAM block '%s' resized during postcopy state: %d",
4541 rb->idstr, ps);
4542 exit(-1);
4543 }
4544 }
4545
4546 static RAMBlockNotifier ram_mig_ram_notifier = {
4547 .ram_block_resized = ram_mig_ram_block_resized,
4548 };
4549
ram_mig_init(void)4550 void ram_mig_init(void)
4551 {
4552 qemu_mutex_init(&XBZRLE.lock);
4553 register_savevm_live("ram", 0, 4, &savevm_ram_handlers, &ram_state);
4554 ram_block_notifier_add(&ram_mig_ram_notifier);
4555 }
4556