1 /*
2 * Multifd common functions
3 *
4 * Copyright (c) 2019-2020 Red Hat Inc
5 *
6 * Authors:
7 * Juan Quintela <quintela@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12
13 #ifndef QEMU_MIGRATION_MULTIFD_H
14 #define QEMU_MIGRATION_MULTIFD_H
15
16 #include "exec/target_page.h"
17 #include "ram.h"
18
19 typedef struct MultiFDRecvData MultiFDRecvData;
20 typedef struct MultiFDSendData MultiFDSendData;
21
22 bool multifd_send_setup(void);
23 void multifd_send_shutdown(void);
24 void multifd_send_channel_created(void);
25 int multifd_recv_setup(Error **errp);
26 void multifd_recv_cleanup(void);
27 void multifd_recv_shutdown(void);
28 bool multifd_recv_all_channels_created(void);
29 void multifd_recv_new_channel(QIOChannel *ioc, Error **errp);
30 void multifd_recv_sync_main(void);
31 int multifd_send_sync_main(void);
32 bool multifd_queue_page(RAMBlock *block, ram_addr_t offset);
33 bool multifd_recv(void);
34 MultiFDRecvData *multifd_get_recv_data(void);
35
36 /* Multifd Compression flags */
37 #define MULTIFD_FLAG_SYNC (1 << 0)
38
39 /* We reserve 5 bits for compression methods */
40 #define MULTIFD_FLAG_COMPRESSION_MASK (0x1f << 1)
41 /* we need to be compatible. Before compression value was 0 */
42 #define MULTIFD_FLAG_NOCOMP (0 << 1)
43 #define MULTIFD_FLAG_ZLIB (1 << 1)
44 #define MULTIFD_FLAG_ZSTD (2 << 1)
45 #define MULTIFD_FLAG_QPL (4 << 1)
46 #define MULTIFD_FLAG_UADK (8 << 1)
47 #define MULTIFD_FLAG_QATZIP (16 << 1)
48
49 /* This value needs to be a multiple of qemu_target_page_size() */
50 #define MULTIFD_PACKET_SIZE (512 * 1024)
51
52 typedef struct {
53 uint32_t magic;
54 uint32_t version;
55 uint32_t flags;
56 /* maximum number of allocated pages */
57 uint32_t pages_alloc;
58 /* non zero pages */
59 uint32_t normal_pages;
60 /* size of the next packet that contains pages */
61 uint32_t next_packet_size;
62 uint64_t packet_num;
63 /* zero pages */
64 uint32_t zero_pages;
65 uint32_t unused32[1]; /* Reserved for future use */
66 uint64_t unused64[3]; /* Reserved for future use */
67 char ramblock[256];
68 /*
69 * This array contains the pointers to:
70 * - normal pages (initial normal_pages entries)
71 * - zero pages (following zero_pages entries)
72 */
73 uint64_t offset[];
74 } __attribute__((packed)) MultiFDPacket_t;
75
76 typedef struct {
77 /* number of used pages */
78 uint32_t num;
79 /* number of normal pages */
80 uint32_t normal_num;
81 RAMBlock *block;
82 /* offset of each page */
83 ram_addr_t offset[];
84 } MultiFDPages_t;
85
86 struct MultiFDRecvData {
87 void *opaque;
88 size_t size;
89 /* for preadv */
90 off_t file_offset;
91 };
92
93 typedef enum {
94 MULTIFD_PAYLOAD_NONE,
95 MULTIFD_PAYLOAD_RAM,
96 } MultiFDPayloadType;
97
98 typedef union MultiFDPayload {
99 MultiFDPages_t ram;
100 } MultiFDPayload;
101
102 struct MultiFDSendData {
103 MultiFDPayloadType type;
104 MultiFDPayload u;
105 };
106
multifd_payload_empty(MultiFDSendData * data)107 static inline bool multifd_payload_empty(MultiFDSendData *data)
108 {
109 return data->type == MULTIFD_PAYLOAD_NONE;
110 }
111
multifd_set_payload_type(MultiFDSendData * data,MultiFDPayloadType type)112 static inline void multifd_set_payload_type(MultiFDSendData *data,
113 MultiFDPayloadType type)
114 {
115 data->type = type;
116 }
117
118 typedef struct {
119 /* Fields are only written at creating/deletion time */
120 /* No lock required for them, they are read only */
121
122 /* channel number */
123 uint8_t id;
124 /* channel thread name */
125 char *name;
126 /* channel thread id */
127 QemuThread thread;
128 bool thread_created;
129 QemuThread tls_thread;
130 bool tls_thread_created;
131 /* communication channel */
132 QIOChannel *c;
133 /* packet allocated len */
134 uint32_t packet_len;
135 /* multifd flags for sending ram */
136 int write_flags;
137
138 /* sem where to wait for more work */
139 QemuSemaphore sem;
140 /* syncs main thread and channels */
141 QemuSemaphore sem_sync;
142
143 /* multifd flags for each packet */
144 uint32_t flags;
145 /*
146 * The sender thread has work to do if either of below boolean is set.
147 *
148 * @pending_job: a job is pending
149 * @pending_sync: a sync request is pending
150 *
151 * For both of these fields, they're only set by the requesters, and
152 * cleared by the multifd sender threads.
153 */
154 bool pending_job;
155 bool pending_sync;
156 MultiFDSendData *data;
157
158 /* thread local variables. No locking required */
159
160 /* pointer to the packet */
161 MultiFDPacket_t *packet;
162 /* size of the next packet that contains pages */
163 uint32_t next_packet_size;
164 /* packets sent through this channel */
165 uint64_t packets_sent;
166 /* buffers to send */
167 struct iovec *iov;
168 /* number of iovs used */
169 uint32_t iovs_num;
170 /* used for compression methods */
171 void *compress_data;
172 } MultiFDSendParams;
173
174 typedef struct {
175 /* Fields are only written at creating/deletion time */
176 /* No lock required for them, they are read only */
177
178 /* channel number */
179 uint8_t id;
180 /* channel thread name */
181 char *name;
182 /* channel thread id */
183 QemuThread thread;
184 bool thread_created;
185 /* communication channel */
186 QIOChannel *c;
187 /* packet allocated len */
188 uint32_t packet_len;
189
190 /* syncs main thread and channels */
191 QemuSemaphore sem_sync;
192 /* sem where to wait for more work */
193 QemuSemaphore sem;
194
195 /* this mutex protects the following parameters */
196 QemuMutex mutex;
197 /* should this thread finish */
198 bool quit;
199 /* multifd flags for each packet */
200 uint32_t flags;
201 /* global number of generated multifd packets */
202 uint64_t packet_num;
203 int pending_job;
204 MultiFDRecvData *data;
205
206 /* thread local variables. No locking required */
207
208 /* pointer to the packet */
209 MultiFDPacket_t *packet;
210 /* size of the next packet that contains pages */
211 uint32_t next_packet_size;
212 /* packets received through this channel */
213 uint64_t packets_recved;
214 /* ramblock */
215 RAMBlock *block;
216 /* ramblock host address */
217 uint8_t *host;
218 /* buffers to recv */
219 struct iovec *iov;
220 /* Pages that are not zero */
221 ram_addr_t *normal;
222 /* num of non zero pages */
223 uint32_t normal_num;
224 /* Pages that are zero */
225 ram_addr_t *zero;
226 /* num of zero pages */
227 uint32_t zero_num;
228 /* used for de-compression methods */
229 void *compress_data;
230 } MultiFDRecvParams;
231
232 typedef struct {
233 /*
234 * The send_setup, send_cleanup, send_prepare are only called on
235 * the QEMU instance at the migration source.
236 */
237
238 /*
239 * Setup for sending side. Called once per channel during channel
240 * setup phase.
241 *
242 * Must allocate p->iov. If packets are in use (default), one
243 * extra iovec must be allocated for the packet header. Any memory
244 * allocated in this hook must be released at send_cleanup.
245 *
246 * p->write_flags may be used for passing flags to the QIOChannel.
247 *
248 * p->compression_data may be used by compression methods to store
249 * compression data.
250 */
251 int (*send_setup)(MultiFDSendParams *p, Error **errp);
252
253 /*
254 * Cleanup for sending side. Called once per channel during
255 * channel cleanup phase.
256 */
257 void (*send_cleanup)(MultiFDSendParams *p, Error **errp);
258
259 /*
260 * Prepare the send packet. Called as a result of multifd_send()
261 * on the client side, with p pointing to the MultiFDSendParams of
262 * a channel that is currently idle.
263 *
264 * Must populate p->iov with the data to be sent, increment
265 * p->iovs_num to match the amount of iovecs used and set
266 * p->next_packet_size with the amount of data currently present
267 * in p->iov.
268 *
269 * Must indicate whether this is a compression packet by setting
270 * p->flags.
271 *
272 * As a last step, if packets are in use (default), must prepare
273 * the packet by calling multifd_send_fill_packet().
274 */
275 int (*send_prepare)(MultiFDSendParams *p, Error **errp);
276
277 /*
278 * The recv_setup, recv_cleanup, recv are only called on the QEMU
279 * instance at the migration destination.
280 */
281
282 /*
283 * Setup for receiving side. Called once per channel during
284 * channel setup phase. May be empty.
285 *
286 * May allocate data structures for the receiving of data. May use
287 * p->iov. Compression methods may use p->compress_data.
288 */
289 int (*recv_setup)(MultiFDRecvParams *p, Error **errp);
290
291 /*
292 * Cleanup for receiving side. Called once per channel during
293 * channel cleanup phase. May be empty.
294 */
295 void (*recv_cleanup)(MultiFDRecvParams *p);
296
297 /*
298 * Data receive method. Called as a result of multifd_recv() on
299 * the client side, with p pointing to the MultiFDRecvParams of a
300 * channel that is currently idle. Only called if there is data
301 * available to receive.
302 *
303 * Must validate p->flags according to what was set at
304 * send_prepare.
305 *
306 * Must read the data from the QIOChannel p->c.
307 */
308 int (*recv)(MultiFDRecvParams *p, Error **errp);
309 } MultiFDMethods;
310
311 void multifd_register_ops(int method, const MultiFDMethods *ops);
312 void multifd_send_fill_packet(MultiFDSendParams *p);
313 bool multifd_send_prepare_common(MultiFDSendParams *p);
314 void multifd_send_zero_page_detect(MultiFDSendParams *p);
315 void multifd_recv_zero_page_process(MultiFDRecvParams *p);
316
multifd_send_prepare_header(MultiFDSendParams * p)317 static inline void multifd_send_prepare_header(MultiFDSendParams *p)
318 {
319 p->iov[0].iov_len = p->packet_len;
320 p->iov[0].iov_base = p->packet;
321 p->iovs_num++;
322 }
323
324 void multifd_channel_connect(MultiFDSendParams *p, QIOChannel *ioc);
325 bool multifd_send(MultiFDSendData **send_data);
326 MultiFDSendData *multifd_send_data_alloc(void);
327
multifd_ram_page_size(void)328 static inline uint32_t multifd_ram_page_size(void)
329 {
330 return qemu_target_page_size();
331 }
332
multifd_ram_page_count(void)333 static inline uint32_t multifd_ram_page_count(void)
334 {
335 return MULTIFD_PACKET_SIZE / qemu_target_page_size();
336 }
337
338 void multifd_ram_save_setup(void);
339 void multifd_ram_save_cleanup(void);
340 int multifd_ram_flush_and_sync(void);
341 size_t multifd_ram_payload_size(void);
342 void multifd_ram_fill_packet(MultiFDSendParams *p);
343 int multifd_ram_unfill_packet(MultiFDRecvParams *p, Error **errp);
344 #endif
345