xref: /qemu/migration/multifd-nocomp.c (revision aa54f5be) 
1 /*
2  * Multifd RAM migration without compression
3  *
4  * Copyright (c) 2019-2020 Red Hat Inc
5  *
6  * Authors:
7  *  Juan Quintela <quintela@redhat.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2 or later.
10  * See the COPYING file in the top-level directory.
11  */
12 
13 #include "qemu/osdep.h"
14 #include "exec/ramblock.h"
15 #include "exec/target_page.h"
16 #include "file.h"
17 #include "multifd.h"
18 #include "options.h"
19 #include "qapi/error.h"
20 #include "qemu/cutils.h"
21 #include "qemu/error-report.h"
22 #include "trace.h"
23 
24 static MultiFDSendData *multifd_ram_send;
25 
26 size_t multifd_ram_payload_size(void)
27 {
28     uint32_t n = multifd_ram_page_count();
29 
30     /*
31      * We keep an array of page offsets at the end of MultiFDPages_t,
32      * add space for it in the allocation.
33      */
34     return sizeof(MultiFDPages_t) + n * sizeof(ram_addr_t);
35 }
36 
37 void multifd_ram_save_setup(void)
38 {
39     multifd_ram_send = multifd_send_data_alloc();
40 }
41 
42 void multifd_ram_save_cleanup(void)
43 {
44     g_free(multifd_ram_send);
45     multifd_ram_send = NULL;
46 }
47 
48 static void multifd_set_file_bitmap(MultiFDSendParams *p)
49 {
50     MultiFDPages_t *pages = &p->data->u.ram;
51 
52     assert(pages->block);
53 
54     for (int i = 0; i < pages->normal_num; i++) {
55         ramblock_set_file_bmap_atomic(pages->block, pages->offset[i], true);
56     }
57 
58     for (int i = pages->normal_num; i < pages->num; i++) {
59         ramblock_set_file_bmap_atomic(pages->block, pages->offset[i], false);
60     }
61 }
62 
63 static int multifd_nocomp_send_setup(MultiFDSendParams *p, Error **errp)
64 {
65     uint32_t page_count = multifd_ram_page_count();
66 
67     if (migrate_zero_copy_send()) {
68         p->write_flags |= QIO_CHANNEL_WRITE_FLAG_ZERO_COPY;
69     }
70 
71     if (!migrate_mapped_ram()) {
72         /* We need one extra place for the packet header */
73         p->iov = g_new0(struct iovec, page_count + 1);
74     } else {
75         p->iov = g_new0(struct iovec, page_count);
76     }
77 
78     return 0;
79 }
80 
81 static void multifd_nocomp_send_cleanup(MultiFDSendParams *p, Error **errp)
82 {
83     g_free(p->iov);
84     p->iov = NULL;
85     return;
86 }
87 
88 static void multifd_send_prepare_iovs(MultiFDSendParams *p)
89 {
90     MultiFDPages_t *pages = &p->data->u.ram;
91     uint32_t page_size = multifd_ram_page_size();
92 
93     for (int i = 0; i < pages->normal_num; i++) {
94         p->iov[p->iovs_num].iov_base = pages->block->host + pages->offset[i];
95         p->iov[p->iovs_num].iov_len = page_size;
96         p->iovs_num++;
97     }
98 
99     p->next_packet_size = pages->normal_num * page_size;
100 }
101 
102 static int multifd_nocomp_send_prepare(MultiFDSendParams *p, Error **errp)
103 {
104     bool use_zero_copy_send = migrate_zero_copy_send();
105     int ret;
106 
107     multifd_send_zero_page_detect(p);
108 
109     if (migrate_mapped_ram()) {
110         multifd_send_prepare_iovs(p);
111         multifd_set_file_bitmap(p);
112 
113         return 0;
114     }
115 
116     if (!use_zero_copy_send) {
117         /*
118          * Only !zerocopy needs the header in IOV; zerocopy will
119          * send it separately.
120          */
121         multifd_send_prepare_header(p);
122     }
123 
124     multifd_send_prepare_iovs(p);
125     p->flags |= MULTIFD_FLAG_NOCOMP;
126 
127     multifd_send_fill_packet(p);
128 
129     if (use_zero_copy_send) {
130         /* Send header first, without zerocopy */
131         ret = qio_channel_write_all(p->c, (void *)p->packet,
132                                     p->packet_len, errp);
133         if (ret != 0) {
134             return -1;
135         }
136     }
137 
138     return 0;
139 }
140 
141 static int multifd_nocomp_recv_setup(MultiFDRecvParams *p, Error **errp)
142 {
143     p->iov = g_new0(struct iovec, multifd_ram_page_count());
144     return 0;
145 }
146 
147 static void multifd_nocomp_recv_cleanup(MultiFDRecvParams *p)
148 {
149     g_free(p->iov);
150     p->iov = NULL;
151 }
152 
153 static int multifd_nocomp_recv(MultiFDRecvParams *p, Error **errp)
154 {
155     uint32_t flags;
156 
157     if (migrate_mapped_ram()) {
158         return multifd_file_recv_data(p, errp);
159     }
160 
161     flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK;
162 
163     if (flags != MULTIFD_FLAG_NOCOMP) {
164         error_setg(errp, "multifd %u: flags received %x flags expected %x",
165                    p->id, flags, MULTIFD_FLAG_NOCOMP);
166         return -1;
167     }
168 
169     multifd_recv_zero_page_process(p);
170 
171     if (!p->normal_num) {
172         return 0;
173     }
174 
175     for (int i = 0; i < p->normal_num; i++) {
176         p->iov[i].iov_base = p->host + p->normal[i];
177         p->iov[i].iov_len = multifd_ram_page_size();
178         ramblock_recv_bitmap_set_offset(p->block, p->normal[i]);
179     }
180     return qio_channel_readv_all(p->c, p->iov, p->normal_num, errp);
181 }
182 
183 static void multifd_pages_reset(MultiFDPages_t *pages)
184 {
185     /*
186      * We don't need to touch offset[] array, because it will be
187      * overwritten later when reused.
188      */
189     pages->num = 0;
190     pages->normal_num = 0;
191     pages->block = NULL;
192 }
193 
194 void multifd_ram_fill_packet(MultiFDSendParams *p)
195 {
196     MultiFDPacket_t *packet = p->packet;
197     MultiFDPages_t *pages = &p->data->u.ram;
198     uint32_t zero_num = pages->num - pages->normal_num;
199 
200     packet->pages_alloc = cpu_to_be32(multifd_ram_page_count());
201     packet->normal_pages = cpu_to_be32(pages->normal_num);
202     packet->zero_pages = cpu_to_be32(zero_num);
203 
204     if (pages->block) {
205         pstrcpy(packet->ramblock, sizeof(packet->ramblock),
206                 pages->block->idstr);
207     }
208 
209     for (int i = 0; i < pages->num; i++) {
210         /* there are architectures where ram_addr_t is 32 bit */
211         uint64_t temp = pages->offset[i];
212 
213         packet->offset[i] = cpu_to_be64(temp);
214     }
215 
216     trace_multifd_send_ram_fill(p->id, pages->normal_num,
217                                 zero_num);
218 }
219 
220 int multifd_ram_unfill_packet(MultiFDRecvParams *p, Error **errp)
221 {
222     MultiFDPacket_t *packet = p->packet;
223     uint32_t page_count = multifd_ram_page_count();
224     uint32_t page_size = multifd_ram_page_size();
225     uint32_t pages_per_packet = be32_to_cpu(packet->pages_alloc);
226     int i;
227 
228     if (pages_per_packet > page_count) {
229         error_setg(errp, "multifd: received packet with %u pages, expected %u",
230                    pages_per_packet, page_count);
231         return -1;
232     }
233 
234     p->normal_num = be32_to_cpu(packet->normal_pages);
235     if (p->normal_num > pages_per_packet) {
236         error_setg(errp, "multifd: received packet with %u non-zero pages, "
237                    "which exceeds maximum expected pages %u",
238                    p->normal_num, pages_per_packet);
239         return -1;
240     }
241 
242     p->zero_num = be32_to_cpu(packet->zero_pages);
243     if (p->zero_num > pages_per_packet - p->normal_num) {
244         error_setg(errp,
245                    "multifd: received packet with %u zero pages, expected maximum %u",
246                    p->zero_num, pages_per_packet - p->normal_num);
247         return -1;
248     }
249 
250     if (p->normal_num == 0 && p->zero_num == 0) {
251         return 0;
252     }
253 
254     /* make sure that ramblock is 0 terminated */
255     packet->ramblock[255] = 0;
256     p->block = qemu_ram_block_by_name(packet->ramblock);
257     if (!p->block) {
258         error_setg(errp, "multifd: unknown ram block %s",
259                    packet->ramblock);
260         return -1;
261     }
262 
263     p->host = p->block->host;
264     for (i = 0; i < p->normal_num; i++) {
265         uint64_t offset = be64_to_cpu(packet->offset[i]);
266 
267         if (offset > (p->block->used_length - page_size)) {
268             error_setg(errp, "multifd: offset too long %" PRIu64
269                        " (max " RAM_ADDR_FMT ")",
270                        offset, p->block->used_length);
271             return -1;
272         }
273         p->normal[i] = offset;
274     }
275 
276     for (i = 0; i < p->zero_num; i++) {
277         uint64_t offset = be64_to_cpu(packet->offset[p->normal_num + i]);
278 
279         if (offset > (p->block->used_length - page_size)) {
280             error_setg(errp, "multifd: offset too long %" PRIu64
281                        " (max " RAM_ADDR_FMT ")",
282                        offset, p->block->used_length);
283             return -1;
284         }
285         p->zero[i] = offset;
286     }
287 
288     return 0;
289 }
290 
291 static inline bool multifd_queue_empty(MultiFDPages_t *pages)
292 {
293     return pages->num == 0;
294 }
295 
296 static inline bool multifd_queue_full(MultiFDPages_t *pages)
297 {
298     return pages->num == multifd_ram_page_count();
299 }
300 
301 static inline void multifd_enqueue(MultiFDPages_t *pages, ram_addr_t offset)
302 {
303     pages->offset[pages->num++] = offset;
304 }
305 
306 /* Returns true if enqueue successful, false otherwise */
307 bool multifd_queue_page(RAMBlock *block, ram_addr_t offset)
308 {
309     MultiFDPages_t *pages;
310 
311 retry:
312     pages = &multifd_ram_send->u.ram;
313 
314     if (multifd_payload_empty(multifd_ram_send)) {
315         multifd_pages_reset(pages);
316         multifd_set_payload_type(multifd_ram_send, MULTIFD_PAYLOAD_RAM);
317     }
318 
319     /* If the queue is empty, we can already enqueue now */
320     if (multifd_queue_empty(pages)) {
321         pages->block = block;
322         multifd_enqueue(pages, offset);
323         return true;
324     }
325 
326     /*
327      * Not empty, meanwhile we need a flush.  It can because of either:
328      *
329      * (1) The page is not on the same ramblock of previous ones, or,
330      * (2) The queue is full.
331      *
332      * After flush, always retry.
333      */
334     if (pages->block != block || multifd_queue_full(pages)) {
335         if (!multifd_send(&multifd_ram_send)) {
336             return false;
337         }
338         goto retry;
339     }
340 
341     /* Not empty, and we still have space, do it! */
342     multifd_enqueue(pages, offset);
343     return true;
344 }
345 
346 int multifd_ram_flush_and_sync(void)
347 {
348     if (!migrate_multifd()) {
349         return 0;
350     }
351 
352     if (!multifd_payload_empty(multifd_ram_send)) {
353         if (!multifd_send(&multifd_ram_send)) {
354             error_report("%s: multifd_send fail", __func__);
355             return -1;
356         }
357     }
358 
359     return multifd_send_sync_main();
360 }
361 
362 bool multifd_send_prepare_common(MultiFDSendParams *p)
363 {
364     MultiFDPages_t *pages = &p->data->u.ram;
365     multifd_send_zero_page_detect(p);
366 
367     if (!pages->normal_num) {
368         p->next_packet_size = 0;
369         return false;
370     }
371 
372     multifd_send_prepare_header(p);
373 
374     return true;
375 }
376 
377 static const MultiFDMethods multifd_nocomp_ops = {
378     .send_setup = multifd_nocomp_send_setup,
379     .send_cleanup = multifd_nocomp_send_cleanup,
380     .send_prepare = multifd_nocomp_send_prepare,
381     .recv_setup = multifd_nocomp_recv_setup,
382     .recv_cleanup = multifd_nocomp_recv_cleanup,
383     .recv = multifd_nocomp_recv
384 };
385 
386 static void multifd_nocomp_register(void)
387 {
388     multifd_register_ops(MULTIFD_COMPRESSION_NONE, &multifd_nocomp_ops);
389 }
390 
391 migration_init(multifd_nocomp_register);
392