1 /*
2 * qemu_migration.c: QEMU migration handling
3 *
4 * Copyright (C) 2006-2015 Red Hat, Inc.
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library. If not, see
18 * <http://www.gnu.org/licenses/>.
19 *
20 */
21
22 #include <config.h>
23
24 #include <sys/time.h>
25 #include <fcntl.h>
26 #include <poll.h>
27
28 #include "qemu_migration.h"
29 #include "qemu_migration_cookie.h"
30 #include "qemu_migration_params.h"
31 #include "qemu_monitor.h"
32 #include "qemu_domain.h"
33 #include "qemu_process.h"
34 #include "qemu_capabilities.h"
35 #include "qemu_alias.h"
36 #include "qemu_cgroup.h"
37 #include "qemu_hotplug.h"
38 #include "qemu_blockjob.h"
39 #include "qemu_security.h"
40 #include "qemu_slirp.h"
41 #include "qemu_block.h"
42
43 #include "domain_audit.h"
44 #include "virlog.h"
45 #include "virerror.h"
46 #include "viralloc.h"
47 #include "virfile.h"
48 #include "virnetdevopenvswitch.h"
49 #include "datatypes.h"
50 #include "virfdstream.h"
51 #include "viruuid.h"
52 #include "virtime.h"
53 #include "locking/domain_lock.h"
54 #include "rpc/virnetsocket.h"
55 #include "storage_source_conf.h"
56 #include "viruri.h"
57 #include "virhook.h"
58 #include "virstring.h"
59 #include "virtypedparam.h"
60 #include "virprocess.h"
61 #include "nwfilter_conf.h"
62 #include "virdomainsnapshotobjlist.h"
63 #include "virsocket.h"
64 #include "virutil.h"
65
66 #define VIR_FROM_THIS VIR_FROM_QEMU
67
68 VIR_LOG_INIT("qemu.qemu_migration");
69
70 VIR_ENUM_IMPL(qemuMigrationJobPhase,
71 QEMU_MIGRATION_PHASE_LAST,
72 "none",
73 "perform2",
74 "begin3",
75 "perform3",
76 "perform3_done",
77 "confirm3_cancelled",
78 "confirm3",
79 "prepare",
80 "finish2",
81 "finish3",
82 );
83
84 static int
85 qemuMigrationJobStart(virQEMUDriver *driver,
86 virDomainObj *vm,
87 qemuDomainAsyncJob job,
88 unsigned long apiFlags)
89 ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT;
90
91 static void
92 qemuMigrationJobSetPhase(virQEMUDriver *driver,
93 virDomainObj *vm,
94 qemuMigrationJobPhase phase)
95 ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
96
97 static void
98 qemuMigrationJobStartPhase(virQEMUDriver *driver,
99 virDomainObj *vm,
100 qemuMigrationJobPhase phase)
101 ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
102
103 static void
104 qemuMigrationJobContinue(virDomainObj *obj)
105 ATTRIBUTE_NONNULL(1);
106
107 static bool
108 qemuMigrationJobIsActive(virDomainObj *vm,
109 qemuDomainAsyncJob job)
110 ATTRIBUTE_NONNULL(1);
111
112 static void
113 qemuMigrationJobFinish(virQEMUDriver *driver,
114 virDomainObj *obj)
115 ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
116
117 static void
qemuMigrationSrcStoreDomainState(virDomainObj * vm)118 qemuMigrationSrcStoreDomainState(virDomainObj *vm)
119 {
120 qemuDomainObjPrivate *priv = vm->privateData;
121 priv->preMigrationState = virDomainObjGetState(vm, NULL);
122
123 VIR_DEBUG("Storing pre-migration state=%d domain=%p",
124 priv->preMigrationState, vm);
125 }
126
127 /* Returns true if the domain was resumed, false otherwise */
128 static bool
qemuMigrationSrcRestoreDomainState(virQEMUDriver * driver,virDomainObj * vm)129 qemuMigrationSrcRestoreDomainState(virQEMUDriver *driver, virDomainObj *vm)
130 {
131 qemuDomainObjPrivate *priv = vm->privateData;
132 int reason;
133 virDomainState state = virDomainObjGetState(vm, &reason);
134 bool ret = false;
135
136 VIR_DEBUG("driver=%p, vm=%p, pre-mig-state=%s, state=%s, reason=%s",
137 driver, vm,
138 virDomainStateTypeToString(priv->preMigrationState),
139 virDomainStateTypeToString(state),
140 virDomainStateReasonToString(state, reason));
141
142 if (state != VIR_DOMAIN_PAUSED ||
143 reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED)
144 goto cleanup;
145
146 if (priv->preMigrationState == VIR_DOMAIN_RUNNING) {
147 /* This is basically the only restore possibility that's safe
148 * and we should attempt to do */
149
150 VIR_DEBUG("Restoring pre-migration state due to migration error");
151
152 /* we got here through some sort of failure; start the domain again */
153 if (qemuProcessStartCPUs(driver, vm,
154 VIR_DOMAIN_RUNNING_MIGRATION_CANCELED,
155 QEMU_ASYNC_JOB_MIGRATION_OUT) < 0) {
156 /* Hm, we already know we are in error here. We don't want to
157 * overwrite the previous error, though, so we just throw something
158 * to the logs and hope for the best */
159 VIR_ERROR(_("Failed to resume guest %s after failure"), vm->def->name);
160 goto cleanup;
161 }
162 ret = true;
163 }
164
165 cleanup:
166 priv->preMigrationState = VIR_DOMAIN_NOSTATE;
167 return ret;
168 }
169
170
171 static int
qemuMigrationDstPrecreateDisk(virConnectPtr * conn,virDomainDiskDef * disk,unsigned long long capacity)172 qemuMigrationDstPrecreateDisk(virConnectPtr *conn,
173 virDomainDiskDef *disk,
174 unsigned long long capacity)
175 {
176 int ret = -1;
177 virStoragePoolPtr pool = NULL;
178 virStorageVolPtr vol = NULL;
179 char *volName = NULL, *basePath = NULL;
180 char *volStr = NULL;
181 g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
182 const char *format = NULL;
183 const char *compat = NULL;
184 unsigned int flags = 0;
185
186 VIR_DEBUG("Precreate disk type=%s", virStorageTypeToString(disk->src->type));
187
188 switch ((virStorageType)disk->src->type) {
189 case VIR_STORAGE_TYPE_FILE:
190 if (!virDomainDiskGetSource(disk)) {
191 VIR_DEBUG("Dropping sourceless disk '%s'",
192 disk->dst);
193 return 0;
194 }
195
196 basePath = g_strdup(disk->src->path);
197
198 if (!(volName = strrchr(basePath, '/'))) {
199 virReportError(VIR_ERR_INVALID_ARG,
200 _("malformed disk path: %s"),
201 disk->src->path);
202 goto cleanup;
203 }
204
205 *volName = '\0';
206 volName++;
207
208 if (!*conn) {
209 if (!(*conn = virGetConnectStorage()))
210 goto cleanup;
211 }
212
213 if (!(pool = virStoragePoolLookupByTargetPath(*conn, basePath)))
214 goto cleanup;
215 format = virStorageFileFormatTypeToString(disk->src->format);
216 if (disk->src->format == VIR_STORAGE_FILE_QCOW2) {
217 flags |= VIR_STORAGE_VOL_CREATE_PREALLOC_METADATA;
218 /* format qcow2v3 image */
219 compat = "1.1";
220 }
221 break;
222
223 case VIR_STORAGE_TYPE_VOLUME:
224 if (!*conn) {
225 if (!(*conn = virGetConnectStorage()))
226 goto cleanup;
227 }
228
229 if (!(pool = virStoragePoolLookupByName(*conn, disk->src->srcpool->pool)))
230 goto cleanup;
231 format = virStorageFileFormatTypeToString(disk->src->format);
232 volName = disk->src->srcpool->volume;
233 if (disk->src->format == VIR_STORAGE_FILE_QCOW2)
234 flags |= VIR_STORAGE_VOL_CREATE_PREALLOC_METADATA;
235 break;
236
237 case VIR_STORAGE_TYPE_NETWORK:
238 VIR_DEBUG("Skipping creation of network disk '%s'",
239 disk->dst);
240 return 0;
241
242 case VIR_STORAGE_TYPE_BLOCK:
243 case VIR_STORAGE_TYPE_DIR:
244 case VIR_STORAGE_TYPE_NVME:
245 case VIR_STORAGE_TYPE_VHOST_USER:
246 case VIR_STORAGE_TYPE_NONE:
247 case VIR_STORAGE_TYPE_LAST:
248 virReportError(VIR_ERR_INTERNAL_ERROR,
249 _("cannot precreate storage for disk type '%s'"),
250 virStorageTypeToString(disk->src->type));
251 goto cleanup;
252 }
253
254 if ((vol = virStorageVolLookupByName(pool, volName))) {
255 VIR_DEBUG("Skipping creation of already existing volume of name '%s'",
256 volName);
257 ret = 0;
258 goto cleanup;
259 }
260
261 virBufferAddLit(&buf, "<volume>\n");
262 virBufferAdjustIndent(&buf, 2);
263 virBufferEscapeString(&buf, "<name>%s</name>\n", volName);
264 virBufferAsprintf(&buf, "<capacity>%llu</capacity>\n", capacity);
265 virBufferAddLit(&buf, "<target>\n");
266 virBufferAdjustIndent(&buf, 2);
267 virBufferAsprintf(&buf, "<format type='%s'/>\n", format);
268 if (compat)
269 virBufferAsprintf(&buf, "<compat>%s</compat>\n", compat);
270 virBufferAdjustIndent(&buf, -2);
271 virBufferAddLit(&buf, "</target>\n");
272 virBufferAdjustIndent(&buf, -2);
273 virBufferAddLit(&buf, "</volume>\n");
274
275 if (!(volStr = virBufferContentAndReset(&buf))) {
276 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
277 _("unable to create volume XML"));
278 goto cleanup;
279 }
280
281 if (!(vol = virStorageVolCreateXML(pool, volStr, flags)))
282 goto cleanup;
283
284 ret = 0;
285 cleanup:
286 VIR_FREE(basePath);
287 VIR_FREE(volStr);
288 virObjectUnref(vol);
289 virObjectUnref(pool);
290 return ret;
291 }
292
293 static bool
qemuMigrationAnyCopyDisk(virDomainDiskDef const * disk,size_t nmigrate_disks,const char ** migrate_disks)294 qemuMigrationAnyCopyDisk(virDomainDiskDef const *disk,
295 size_t nmigrate_disks, const char **migrate_disks)
296 {
297 size_t i;
298
299 /* Check if the disk alias is in the list */
300 if (nmigrate_disks) {
301 for (i = 0; i < nmigrate_disks; i++) {
302 if (STREQ(disk->dst, migrate_disks[i]))
303 return true;
304 }
305 return false;
306 }
307
308 /* Default is to migrate only non-shared non-readonly disks
309 * with source */
310 return !disk->src->shared && !disk->src->readonly &&
311 !virStorageSourceIsEmpty(disk->src);
312 }
313
314
315 static bool
qemuMigrationHasAnyStorageMigrationDisks(virDomainDef * def,const char ** migrate_disks,size_t nmigrate_disks)316 qemuMigrationHasAnyStorageMigrationDisks(virDomainDef *def,
317 const char **migrate_disks,
318 size_t nmigrate_disks)
319 {
320 size_t i;
321
322 for (i = 0; i < def->ndisks; i++) {
323 if (qemuMigrationAnyCopyDisk(def->disks[i], nmigrate_disks, migrate_disks))
324 return true;
325 }
326
327 return false;
328 }
329
330
331 static int
qemuMigrationDstPrecreateStorage(virDomainObj * vm,qemuMigrationCookieNBD * nbd,size_t nmigrate_disks,const char ** migrate_disks,bool incremental)332 qemuMigrationDstPrecreateStorage(virDomainObj *vm,
333 qemuMigrationCookieNBD *nbd,
334 size_t nmigrate_disks,
335 const char **migrate_disks,
336 bool incremental)
337 {
338 int ret = -1;
339 size_t i = 0;
340 virConnectPtr conn = NULL;
341
342 if (!nbd || !nbd->ndisks)
343 return 0;
344
345 for (i = 0; i < nbd->ndisks; i++) {
346 virDomainDiskDef *disk;
347 const char *diskSrcPath;
348 g_autofree char *nvmePath = NULL;
349
350 VIR_DEBUG("Looking up disk target '%s' (capacity=%llu)",
351 nbd->disks[i].target, nbd->disks[i].capacity);
352
353 if (!(disk = virDomainDiskByTarget(vm->def, nbd->disks[i].target))) {
354 virReportError(VIR_ERR_INTERNAL_ERROR,
355 _("unable to find disk by target: %s"),
356 nbd->disks[i].target);
357 goto cleanup;
358 }
359
360 if (disk->src->type == VIR_STORAGE_TYPE_NVME) {
361 virPCIDeviceAddressGetSysfsFile(&disk->src->nvme->pciAddr, &nvmePath);
362 diskSrcPath = nvmePath;
363 } else {
364 diskSrcPath = virDomainDiskGetSource(disk);
365 }
366
367 /* Skip disks we don't want to migrate and already existing disks. */
368 if (!qemuMigrationAnyCopyDisk(disk, nmigrate_disks, migrate_disks) ||
369 (diskSrcPath && virFileExists(diskSrcPath))) {
370 continue;
371 }
372
373 if (incremental) {
374 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
375 _("pre-creation of storage targets for incremental "
376 "storage migration is not supported"));
377 goto cleanup;
378 }
379
380 VIR_DEBUG("Proceeding with disk source %s", NULLSTR(diskSrcPath));
381
382 if (qemuMigrationDstPrecreateDisk(&conn,
383 disk, nbd->disks[i].capacity) < 0)
384 goto cleanup;
385 }
386
387 ret = 0;
388 cleanup:
389 virObjectUnref(conn);
390 return ret;
391 }
392
393
394 /**
395 * qemuMigrationDstStartNBDServer:
396 * @driver: qemu driver
397 * @vm: domain
398 *
399 * Starts NBD server. This is a newer method to copy
400 * storage during migration than using 'blk' and 'inc'
401 * arguments in 'migrate' monitor command.
402 * Error is reported here.
403 *
404 * Returns 0 on success, -1 otherwise.
405 */
406 static int
qemuMigrationDstStartNBDServer(virQEMUDriver * driver,virDomainObj * vm,const char * listenAddr,size_t nmigrate_disks,const char ** migrate_disks,int nbdPort,const char * nbdURI,const char * tls_alias)407 qemuMigrationDstStartNBDServer(virQEMUDriver *driver,
408 virDomainObj *vm,
409 const char *listenAddr,
410 size_t nmigrate_disks,
411 const char **migrate_disks,
412 int nbdPort,
413 const char *nbdURI,
414 const char *tls_alias)
415 {
416 int ret = -1;
417 qemuDomainObjPrivate *priv = vm->privateData;
418 size_t i;
419 virStorageNetHostDef server = {
420 .name = (char *)listenAddr, /* cast away const */
421 .transport = VIR_STORAGE_NET_HOST_TRANS_TCP,
422 .port = nbdPort,
423 };
424 bool server_started = false;
425 g_autoptr(virURI) uri = NULL;
426
427 /* Prefer nbdURI */
428 if (nbdURI) {
429 uri = virURIParse(nbdURI);
430
431 if (!uri)
432 return -1;
433
434 if (!uri->scheme) {
435 virReportError(VIR_ERR_INVALID_ARG, _("No URI scheme specified: %s"), nbdURI);
436 return -1;
437 }
438
439 if (STREQ(uri->scheme, "tcp")) {
440 server.transport = VIR_STORAGE_NET_HOST_TRANS_TCP;
441 if (!uri->server || STREQ(uri->server, "")) {
442 /* Since tcp://:<port>/ is parsed as server = NULL and port = 0
443 * we should rather error out instead of auto-allocating a port
444 * as that would be the exact opposite of what was requested. */
445 virReportError(VIR_ERR_INVALID_ARG,
446 _("URI with tcp scheme did not provide a server part: %s"),
447 nbdURI);
448 return -1;
449 }
450 server.name = (char *)uri->server;
451 if (uri->port)
452 server.port = uri->port;
453 } else if (STREQ(uri->scheme, "unix")) {
454 if (!uri->path) {
455 virReportError(VIR_ERR_INVALID_ARG, "%s",
456 _("UNIX disks URI does not include path"));
457 return -1;
458 }
459 server.transport = VIR_STORAGE_NET_HOST_TRANS_UNIX;
460 server.socket = (char *)uri->path;
461 } else {
462 virReportError(VIR_ERR_INVALID_ARG,
463 _("Unsupported scheme in disks URI: %s"),
464 uri->scheme);
465 return -1;
466 }
467 } else if (nbdPort < 0 || nbdPort > USHRT_MAX) {
468 virReportError(VIR_ERR_INVALID_ARG, "%s",
469 _("nbd port must be in range 0-65535"));
470 return -1;
471 }
472
473 for (i = 0; i < vm->def->ndisks; i++) {
474 virDomainDiskDef *disk = vm->def->disks[i];
475 g_autofree char *diskAlias = NULL;
476
477 /* check whether disk should be migrated */
478 if (!qemuMigrationAnyCopyDisk(disk, nmigrate_disks, migrate_disks))
479 continue;
480
481 if (disk->src->readonly || virStorageSourceIsEmpty(disk->src)) {
482 virReportError(VIR_ERR_OPERATION_UNSUPPORTED,
483 _("Cannot migrate empty or read-only disk %s"),
484 disk->dst);
485 goto cleanup;
486 }
487
488 if (!(diskAlias = qemuAliasDiskDriveFromDisk(disk)))
489 goto cleanup;
490
491 if (!server_started &&
492 server.transport == VIR_STORAGE_NET_HOST_TRANS_TCP) {
493 if (server.port) {
494 if (virPortAllocatorSetUsed(server.port) < 0)
495 goto cleanup;
496 } else {
497 unsigned short port = 0;
498
499 if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0)
500 goto cleanup;
501
502 server.port = port;
503 }
504 }
505
506 if (qemuDomainObjEnterMonitorAsync(driver, vm,
507 QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
508 goto cleanup;
509
510 if (!server_started) {
511 if (qemuMonitorNBDServerStart(priv->mon, &server, tls_alias) < 0)
512 goto exit_monitor;
513 server_started = true;
514 }
515
516 if (qemuBlockExportAddNBD(vm, diskAlias, disk->src, diskAlias, true, NULL) < 0)
517 goto exit_monitor;
518 if (qemuDomainObjExitMonitor(driver, vm) < 0)
519 goto cleanup;
520 }
521
522 if (server.transport == VIR_STORAGE_NET_HOST_TRANS_TCP)
523 priv->nbdPort = server.port;
524
525 ret = 0;
526
527 cleanup:
528 if (ret < 0)
529 virPortAllocatorRelease(server.port);
530 return ret;
531
532 exit_monitor:
533 ignore_value(qemuDomainObjExitMonitor(driver, vm));
534 goto cleanup;
535 }
536
537
538 static int
qemuMigrationDstStopNBDServer(virQEMUDriver * driver,virDomainObj * vm,qemuMigrationCookie * mig)539 qemuMigrationDstStopNBDServer(virQEMUDriver *driver,
540 virDomainObj *vm,
541 qemuMigrationCookie *mig)
542 {
543 qemuDomainObjPrivate *priv = vm->privateData;
544
545 if (!mig->nbd)
546 return 0;
547
548 if (qemuDomainObjEnterMonitorAsync(driver, vm,
549 QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
550 return -1;
551
552 if (qemuMonitorNBDServerStop(priv->mon) < 0)
553 VIR_WARN("Unable to stop NBD server");
554 if (qemuDomainObjExitMonitor(driver, vm) < 0)
555 return -1;
556
557 virPortAllocatorRelease(priv->nbdPort);
558 priv->nbdPort = 0;
559 return 0;
560 }
561
562
563 static void
qemuMigrationNBDReportMirrorError(qemuBlockJobData * job,const char * diskdst)564 qemuMigrationNBDReportMirrorError(qemuBlockJobData *job,
565 const char *diskdst)
566 {
567 if (job->errmsg) {
568 virReportError(VIR_ERR_OPERATION_FAILED,
569 _("migration of disk %s failed: %s"),
570 diskdst, job->errmsg);
571 } else {
572 virReportError(VIR_ERR_OPERATION_FAILED,
573 _("migration of disk %s failed"), diskdst);
574 }
575 }
576
577
578 /**
579 * qemuMigrationSrcNBDStorageCopyReady:
580 * @vm: domain
581 *
582 * Check the status of all drives copied via qemuMigrationSrcNBDStorageCopy.
583 * Any pending block job events for the mirrored disks will be processed.
584 *
585 * Returns 1 if all mirrors are "ready",
586 * 0 if some mirrors are still performing initial sync,
587 * -1 on error.
588 */
589 static int
qemuMigrationSrcNBDStorageCopyReady(virDomainObj * vm,qemuDomainAsyncJob asyncJob)590 qemuMigrationSrcNBDStorageCopyReady(virDomainObj *vm,
591 qemuDomainAsyncJob asyncJob)
592 {
593 size_t i;
594 size_t notReady = 0;
595
596 for (i = 0; i < vm->def->ndisks; i++) {
597 virDomainDiskDef *disk = vm->def->disks[i];
598 qemuDomainDiskPrivate *diskPriv = QEMU_DOMAIN_DISK_PRIVATE(disk);
599 qemuBlockJobData *job;
600
601 if (!diskPriv->migrating)
602 continue;
603
604 if (!(job = qemuBlockJobDiskGetJob(disk))) {
605 virReportError(VIR_ERR_INTERNAL_ERROR,
606 _("missing block job data for disk '%s'"), disk->dst);
607 return -1;
608 }
609
610 qemuBlockJobUpdate(vm, job, asyncJob);
611 if (job->state == VIR_DOMAIN_BLOCK_JOB_FAILED) {
612 qemuMigrationNBDReportMirrorError(job, disk->dst);
613 virObjectUnref(job);
614 return -1;
615 }
616
617 if (job->state != VIR_DOMAIN_BLOCK_JOB_READY)
618 notReady++;
619
620 virObjectUnref(job);
621 }
622
623 if (notReady) {
624 VIR_DEBUG("Waiting for %zu disk mirrors to get ready", notReady);
625 return 0;
626 } else {
627 VIR_DEBUG("All disk mirrors are ready");
628 return 1;
629 }
630 }
631
632
633 /*
634 * If @abortMigration is false, the function will report an error and return a
635 * different code in case a block job fails. This way we can properly abort
636 * migration in case some block jobs failed once all memory has already been
637 * transferred.
638 *
639 * Returns 1 if all mirrors are gone,
640 * 0 if some mirrors are still active,
641 * -1 some mirrors failed but some are still active,
642 * -2 all mirrors are gone but some of them failed.
643 */
644 static int
qemuMigrationSrcNBDCopyCancelled(virDomainObj * vm,qemuDomainAsyncJob asyncJob,bool abortMigration)645 qemuMigrationSrcNBDCopyCancelled(virDomainObj *vm,
646 qemuDomainAsyncJob asyncJob,
647 bool abortMigration)
648 {
649 size_t i;
650 size_t active = 0;
651 size_t completed = 0;
652 bool failed = false;
653
654 do {
655 active = 0;
656 completed = 0;
657
658 for (i = 0; i < vm->def->ndisks; i++) {
659 virDomainDiskDef *disk = vm->def->disks[i];
660 qemuDomainDiskPrivate *diskPriv = QEMU_DOMAIN_DISK_PRIVATE(disk);
661 qemuBlockJobData *job;
662
663 if (!diskPriv->migrating)
664 continue;
665
666 if (!(job = qemuBlockJobDiskGetJob(disk)))
667 continue;
668
669 qemuBlockJobUpdate(vm, job, asyncJob);
670 switch (job->state) {
671 case VIR_DOMAIN_BLOCK_JOB_FAILED:
672 if (!abortMigration) {
673 qemuMigrationNBDReportMirrorError(job, disk->dst);
674 failed = true;
675 }
676 G_GNUC_FALLTHROUGH;
677 case VIR_DOMAIN_BLOCK_JOB_CANCELED:
678 case VIR_DOMAIN_BLOCK_JOB_COMPLETED:
679 diskPriv->migrating = false;
680 break;
681
682 default:
683 active++;
684 }
685
686 if (job->state == VIR_DOMAIN_BLOCK_JOB_COMPLETED)
687 completed++;
688
689 virObjectUnref(job);
690 }
691
692 /* Updating completed block job drops the lock thus we have to recheck
693 * block jobs for disks that reside before the disk(s) with completed
694 * block job.
695 */
696 } while (completed > 0);
697
698 if (failed) {
699 if (active) {
700 VIR_DEBUG("Some disk mirrors failed; still waiting for %zu "
701 "disk mirrors to finish", active);
702 return -1;
703 } else {
704 VIR_DEBUG("All disk mirrors are gone; some of them failed");
705 return -2;
706 }
707 } else {
708 if (active) {
709 VIR_DEBUG("Waiting for %zu disk mirrors to finish", active);
710 return 0;
711 } else {
712 VIR_DEBUG("All disk mirrors are gone");
713 return 1;
714 }
715 }
716 }
717
718
719 /*
720 * Returns 0 on success,
721 * 1 when job is already completed or it failed and failNoJob is false,
722 * -1 on error or when job failed and failNoJob is true.
723 */
724 static int
qemuMigrationSrcNBDCopyCancelOne(virQEMUDriver * driver,virDomainObj * vm,virDomainDiskDef * disk,qemuBlockJobData * job,bool abortMigration,qemuDomainAsyncJob asyncJob)725 qemuMigrationSrcNBDCopyCancelOne(virQEMUDriver *driver,
726 virDomainObj *vm,
727 virDomainDiskDef *disk,
728 qemuBlockJobData *job,
729 bool abortMigration,
730 qemuDomainAsyncJob asyncJob)
731 {
732 qemuDomainObjPrivate *priv = vm->privateData;
733 int rv;
734
735 qemuBlockJobUpdate(vm, job, asyncJob);
736 switch (job->state) {
737 case VIR_DOMAIN_BLOCK_JOB_FAILED:
738 case VIR_DOMAIN_BLOCK_JOB_CANCELED:
739 if (!abortMigration) {
740 qemuMigrationNBDReportMirrorError(job, disk->dst);
741 return -1;
742 }
743 G_GNUC_FALLTHROUGH;
744 case VIR_DOMAIN_BLOCK_JOB_COMPLETED:
745 return 1;
746 }
747
748 if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
749 return -1;
750
751 /* when we are aborting the migration we don't care about the data
752 * consistency on the destination so that we can force cancel the mirror */
753 rv = qemuMonitorBlockJobCancel(priv->mon, job->name, abortMigration);
754
755 if (qemuDomainObjExitMonitor(driver, vm) < 0 || rv < 0)
756 return -1;
757
758 return 0;
759 }
760
761
762 /**
763 * qemuMigrationSrcNBDCopyCancel:
764 * @driver: qemu driver
765 * @vm: domain
766 * @abortMigration: The migration is being cancelled.
767 *
768 * Cancel all drive-mirrors started by qemuMigrationSrcNBDStorageCopy.
769 * Any pending block job events for the affected disks will be processed and
770 * synchronous block job terminated regardless of return value unless qemu
771 * has crashed.
772 *
773 * Returns 0 on success, -1 otherwise.
774 */
775 static int
qemuMigrationSrcNBDCopyCancel(virQEMUDriver * driver,virDomainObj * vm,bool abortMigration,qemuDomainAsyncJob asyncJob,virConnectPtr dconn)776 qemuMigrationSrcNBDCopyCancel(virQEMUDriver *driver,
777 virDomainObj *vm,
778 bool abortMigration,
779 qemuDomainAsyncJob asyncJob,
780 virConnectPtr dconn)
781 {
782 virErrorPtr err = NULL;
783 int ret = -1;
784 size_t i;
785 int rv;
786 bool failed = false;
787
788 VIR_DEBUG("Cancelling drive mirrors for domain %s", vm->def->name);
789
790 for (i = 0; i < vm->def->ndisks; i++) {
791 virDomainDiskDef *disk = vm->def->disks[i];
792 qemuDomainDiskPrivate *diskPriv = QEMU_DOMAIN_DISK_PRIVATE(disk);
793 qemuBlockJobData *job;
794
795 if (!(job = qemuBlockJobDiskGetJob(disk)) ||
796 !qemuBlockJobIsRunning(job))
797 diskPriv->migrating = false;
798
799 if (!diskPriv->migrating) {
800 virObjectUnref(job);
801 continue;
802 }
803
804 rv = qemuMigrationSrcNBDCopyCancelOne(driver, vm, disk, job,
805 abortMigration, asyncJob);
806 if (rv != 0) {
807 if (rv < 0) {
808 if (!err)
809 virErrorPreserveLast(&err);
810 failed = true;
811 }
812 qemuBlockJobSyncEnd(vm, job, asyncJob);
813 diskPriv->migrating = false;
814 }
815
816 virObjectUnref(job);
817 }
818
819 while ((rv = qemuMigrationSrcNBDCopyCancelled(vm, asyncJob, abortMigration)) != 1) {
820 if (!abortMigration && !failed &&
821 dconn && virConnectIsAlive(dconn) <= 0) {
822 virReportError(VIR_ERR_OPERATION_FAILED, "%s",
823 _("Lost connection to destination host"));
824 failed = true;
825 }
826
827 if (rv < 0) {
828 failed = true;
829 if (rv == -2)
830 break;
831 }
832
833 if (failed && !err)
834 virErrorPreserveLast(&err);
835
836 if (virDomainObjWait(vm) < 0)
837 goto cleanup;
838 }
839
840 for (i = 0; i < vm->def->ndisks; i++) {
841 virDomainDiskDef *disk = vm->def->disks[i];
842 qemuDomainDiskPrivate *diskPriv = QEMU_DOMAIN_DISK_PRIVATE(disk);
843
844 if (!diskPriv->migrSource)
845 continue;
846
847 qemuBlockStorageSourceDetachOneBlockdev(driver, vm, asyncJob,
848 diskPriv->migrSource);
849 virObjectUnref(diskPriv->migrSource);
850 diskPriv->migrSource = NULL;
851 }
852
853 ret = failed ? -1 : 0;
854
855 cleanup:
856 virErrorRestore(&err);
857 return ret;
858 }
859
860
861 static int
qemuMigrationSrcCancelRemoveTempBitmaps(virDomainObj * vm,qemuDomainAsyncJob asyncJob)862 qemuMigrationSrcCancelRemoveTempBitmaps(virDomainObj *vm,
863 qemuDomainAsyncJob asyncJob)
864 {
865 qemuDomainObjPrivate *priv = vm->privateData;
866 virQEMUDriver *driver = priv->driver;
867 qemuDomainJobPrivate *jobPriv = priv->job.privateData;
868 GSList *next;
869
870 for (next = jobPriv->migTempBitmaps; next; next = next->next) {
871 qemuDomainJobPrivateMigrateTempBitmap *t = next->data;
872
873 if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
874 return -1;
875 qemuMonitorBitmapRemove(priv->mon, t->nodename, t->bitmapname);
876 if (qemuDomainObjExitMonitor(driver, vm) < 0)
877 return -1;
878 }
879
880 return 0;
881 }
882
883
884 static virStorageSource *
qemuMigrationSrcNBDStorageCopyBlockdevPrepareSource(virDomainDiskDef * disk,const char * host,int port,const char * socket,const char * tlsAlias)885 qemuMigrationSrcNBDStorageCopyBlockdevPrepareSource(virDomainDiskDef *disk,
886 const char *host,
887 int port,
888 const char *socket,
889 const char *tlsAlias)
890 {
891 g_autoptr(virStorageSource) copysrc = NULL;
892
893 copysrc = virStorageSourceNew();
894 copysrc->type = VIR_STORAGE_TYPE_NETWORK;
895 copysrc->protocol = VIR_STORAGE_NET_PROTOCOL_NBD;
896 copysrc->format = VIR_STORAGE_FILE_RAW;
897
898 copysrc->backingStore = virStorageSourceNew();
899
900 if (!(copysrc->path = qemuAliasDiskDriveFromDisk(disk)))
901 return NULL;
902
903 copysrc->hosts = g_new0(virStorageNetHostDef, 1);
904
905 copysrc->nhosts = 1;
906 if (socket) {
907 copysrc->hosts->transport = VIR_STORAGE_NET_HOST_TRANS_UNIX;
908 copysrc->hosts->socket = g_strdup(socket);
909 } else {
910 copysrc->hosts->transport = VIR_STORAGE_NET_HOST_TRANS_TCP;
911 copysrc->hosts->port = port;
912 copysrc->hosts->name = g_strdup(host);
913 }
914
915 copysrc->tlsAlias = g_strdup(tlsAlias);
916
917 copysrc->nodestorage = g_strdup_printf("migration-%s-storage", disk->dst);
918 copysrc->nodeformat = g_strdup_printf("migration-%s-format", disk->dst);
919
920 return g_steal_pointer(©src);
921 }
922
923
924 static int
qemuMigrationSrcNBDStorageCopyBlockdev(virQEMUDriver * driver,virDomainObj * vm,virDomainDiskDef * disk,const char * jobname,const char * sourcename,bool persistjob,const char * host,int port,const char * socket,unsigned long long mirror_speed,unsigned int mirror_shallow,const char * tlsAlias)925 qemuMigrationSrcNBDStorageCopyBlockdev(virQEMUDriver *driver,
926 virDomainObj *vm,
927 virDomainDiskDef *disk,
928 const char *jobname,
929 const char *sourcename,
930 bool persistjob,
931 const char *host,
932 int port,
933 const char *socket,
934 unsigned long long mirror_speed,
935 unsigned int mirror_shallow,
936 const char *tlsAlias)
937 {
938 g_autoptr(qemuBlockStorageSourceAttachData) data = NULL;
939 qemuDomainDiskPrivate *diskPriv = QEMU_DOMAIN_DISK_PRIVATE(disk);
940 int mon_ret = 0;
941 g_autoptr(virStorageSource) copysrc = NULL;
942
943 VIR_DEBUG("starting blockdev mirror for disk=%s to host=%s", disk->dst, host);
944
945 if (!(copysrc = qemuMigrationSrcNBDStorageCopyBlockdevPrepareSource(disk, host, port, socket, tlsAlias)))
946 return -1;
947
948 /* Migration via blockdev-mirror was supported sooner than the auto-read-only
949 * feature was added to qemu */
950 if (!(data = qemuBlockStorageSourceAttachPrepareBlockdev(copysrc,
951 copysrc->backingStore,
952 false)))
953 return -1;
954
955 if (qemuDomainObjEnterMonitorAsync(driver, vm,
956 QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
957 return -1;
958
959 mon_ret = qemuBlockStorageSourceAttachApply(qemuDomainGetMonitor(vm), data);
960
961 if (mon_ret == 0)
962 mon_ret = qemuMonitorBlockdevMirror(qemuDomainGetMonitor(vm), jobname, persistjob,
963 sourcename, copysrc->nodeformat,
964 mirror_speed, 0, 0, mirror_shallow);
965
966 if (mon_ret != 0)
967 qemuBlockStorageSourceAttachRollback(qemuDomainGetMonitor(vm), data);
968
969 if (qemuDomainObjExitMonitor(driver, vm) < 0 || mon_ret < 0)
970 return -1;
971
972 diskPriv->migrSource = g_steal_pointer(©src);
973
974 return 0;
975 }
976
977
978 static int
qemuMigrationSrcNBDStorageCopyDriveMirror(virQEMUDriver * driver,virDomainObj * vm,const char * diskAlias,const char * host,int port,const char * socket,unsigned long long mirror_speed,bool mirror_shallow)979 qemuMigrationSrcNBDStorageCopyDriveMirror(virQEMUDriver *driver,
980 virDomainObj *vm,
981 const char *diskAlias,
982 const char *host,
983 int port,
984 const char *socket,
985 unsigned long long mirror_speed,
986 bool mirror_shallow)
987 {
988 g_autofree char *nbd_dest = NULL;
989 int mon_ret;
990
991 if (socket) {
992 nbd_dest = g_strdup_printf("nbd+unix:///%s?socket=%s",
993 diskAlias, socket);
994 } else if (strchr(host, ':')) {
995 nbd_dest = g_strdup_printf("nbd:[%s]:%d:exportname=%s", host, port,
996 diskAlias);
997 } else {
998 nbd_dest = g_strdup_printf("nbd:%s:%d:exportname=%s", host, port,
999 diskAlias);
1000 }
1001
1002 if (qemuDomainObjEnterMonitorAsync(driver, vm,
1003 QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
1004 return -1;
1005
1006 mon_ret = qemuMonitorDriveMirror(qemuDomainGetMonitor(vm),
1007 diskAlias, nbd_dest, "raw",
1008 mirror_speed, 0, 0, mirror_shallow, true);
1009
1010 if (qemuDomainObjExitMonitor(driver, vm) < 0 || mon_ret < 0)
1011 return -1;
1012
1013 return 0;
1014 }
1015
1016
1017 static int
qemuMigrationSrcNBDStorageCopyOne(virQEMUDriver * driver,virDomainObj * vm,virDomainDiskDef * disk,const char * host,int port,const char * socket,unsigned long long mirror_speed,bool mirror_shallow,const char * tlsAlias,unsigned int flags)1018 qemuMigrationSrcNBDStorageCopyOne(virQEMUDriver *driver,
1019 virDomainObj *vm,
1020 virDomainDiskDef *disk,
1021 const char *host,
1022 int port,
1023 const char *socket,
1024 unsigned long long mirror_speed,
1025 bool mirror_shallow,
1026 const char *tlsAlias,
1027 unsigned int flags)
1028 {
1029 qemuDomainObjPrivate *priv = vm->privateData;
1030 qemuDomainDiskPrivate *diskPriv = QEMU_DOMAIN_DISK_PRIVATE(disk);
1031 qemuBlockJobData *job = NULL;
1032 char *diskAlias = NULL;
1033 const char *jobname = NULL;
1034 const char *sourcename = NULL;
1035 bool persistjob = false;
1036 int rc;
1037 int ret = -1;
1038
1039 if (!(diskAlias = qemuAliasDiskDriveFromDisk(disk)))
1040 goto cleanup;
1041
1042 if (!(job = qemuBlockJobDiskNew(vm, disk, QEMU_BLOCKJOB_TYPE_COPY, diskAlias)))
1043 goto cleanup;
1044
1045 if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_BLOCKDEV)) {
1046 jobname = diskAlias;
1047 sourcename = qemuDomainDiskGetTopNodename(disk);
1048 persistjob = true;
1049 } else {
1050 jobname = NULL;
1051 sourcename = diskAlias;
1052 persistjob = false;
1053 }
1054
1055 qemuBlockJobSyncBegin(job);
1056
1057 if (flags & VIR_MIGRATE_TLS ||
1058 virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_BLOCKDEV)) {
1059 rc = qemuMigrationSrcNBDStorageCopyBlockdev(driver, vm,
1060 disk, jobname,
1061 sourcename, persistjob,
1062 host, port, socket,
1063 mirror_speed,
1064 mirror_shallow,
1065 tlsAlias);
1066 } else {
1067 rc = qemuMigrationSrcNBDStorageCopyDriveMirror(driver, vm, diskAlias,
1068 host, port, socket,
1069 mirror_speed,
1070 mirror_shallow);
1071 }
1072
1073 if (rc < 0)
1074 goto cleanup;
1075
1076 diskPriv->migrating = true;
1077 qemuBlockJobStarted(job, vm);
1078
1079 ret = 0;
1080
1081 cleanup:
1082 qemuBlockJobStartupFinalize(vm, job);
1083 VIR_FREE(diskAlias);
1084 return ret;
1085 }
1086
1087
1088 /**
1089 * qemuMigrationSrcNBDStorageCopy:
1090 * @driver: qemu driver
1091 * @vm: domain
1092 * @mig: migration cookie
1093 * @host: where are we migrating to
1094 * @speed: bandwidth limit in MiB/s
1095 *
1096 * Migrate non-shared storage using the NBD protocol to the server running
1097 * inside the qemu process on dst and wait until the copy converges.
1098 * On failure, the caller is expected to call qemuMigrationSrcNBDCopyCancel
1099 * to stop all running copy operations.
1100 *
1101 * Returns 0 on success (@migrate_flags updated),
1102 * -1 otherwise.
1103 */
1104 static int
qemuMigrationSrcNBDStorageCopy(virQEMUDriver * driver,virDomainObj * vm,qemuMigrationCookie * mig,const char * host,unsigned long speed,size_t nmigrate_disks,const char ** migrate_disks,virConnectPtr dconn,const char * tlsAlias,const char * nbdURI,unsigned int flags)1105 qemuMigrationSrcNBDStorageCopy(virQEMUDriver *driver,
1106 virDomainObj *vm,
1107 qemuMigrationCookie *mig,
1108 const char *host,
1109 unsigned long speed,
1110 size_t nmigrate_disks,
1111 const char **migrate_disks,
1112 virConnectPtr dconn,
1113 const char *tlsAlias,
1114 const char *nbdURI,
1115 unsigned int flags)
1116 {
1117 qemuDomainObjPrivate *priv = vm->privateData;
1118 int port;
1119 size_t i;
1120 unsigned long long mirror_speed = speed;
1121 bool mirror_shallow = flags & VIR_MIGRATE_NON_SHARED_INC;
1122 int rv;
1123 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
1124 g_autoptr(virURI) uri = NULL;
1125 const char *socket = NULL;
1126
1127 VIR_DEBUG("Starting drive mirrors for domain %s", vm->def->name);
1128
1129 if (mirror_speed > LLONG_MAX >> 20) {
1130 virReportError(VIR_ERR_OVERFLOW,
1131 _("bandwidth must be less than %llu"),
1132 LLONG_MAX >> 20);
1133 return -1;
1134 }
1135 mirror_speed <<= 20;
1136
1137 /* steal NBD port and thus prevent its propagation back to destination */
1138 port = mig->nbd->port;
1139 mig->nbd->port = 0;
1140
1141 if (nbdURI) {
1142 uri = virURIParse(nbdURI);
1143 if (!uri)
1144 return -1;
1145
1146 if (STREQ(uri->scheme, "tcp")) {
1147 if (uri->server && STRNEQ(uri->server, ""))
1148 host = (char *)uri->server;
1149 if (uri->port)
1150 port = uri->port;
1151 } else if (STREQ(uri->scheme, "unix")) {
1152 if (flags & VIR_MIGRATE_TLS) {
1153 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
1154 _("NBD migration with TLS is not supported over UNIX socket"));
1155 return -1;
1156 }
1157
1158 if (!uri->path) {
1159 virReportError(VIR_ERR_INVALID_ARG, "%s",
1160 _("UNIX disks URI does not include path"));
1161 return -1;
1162 }
1163 socket = uri->path;
1164
1165 if (qemuSecurityDomainSetPathLabel(driver, vm, socket, false) < 0)
1166 return -1;
1167 } else {
1168 virReportError(VIR_ERR_INVALID_ARG,
1169 _("Unsupported scheme in disks URI: %s"),
1170 uri->scheme);
1171 return -1;
1172 }
1173 }
1174
1175 for (i = 0; i < vm->def->ndisks; i++) {
1176 virDomainDiskDef *disk = vm->def->disks[i];
1177
1178 /* check whether disk should be migrated */
1179 if (!qemuMigrationAnyCopyDisk(disk, nmigrate_disks, migrate_disks))
1180 continue;
1181
1182 if (qemuMigrationSrcNBDStorageCopyOne(driver, vm, disk, host, port,
1183 socket,
1184 mirror_speed, mirror_shallow,
1185 tlsAlias, flags) < 0)
1186 return -1;
1187
1188 if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) {
1189 VIR_WARN("Failed to save status on vm %s", vm->def->name);
1190 return -1;
1191 }
1192 }
1193
1194 while ((rv = qemuMigrationSrcNBDStorageCopyReady(vm, QEMU_ASYNC_JOB_MIGRATION_OUT)) != 1) {
1195 if (rv < 0)
1196 return -1;
1197
1198 if (priv->job.abortJob) {
1199 priv->job.current->status = QEMU_DOMAIN_JOB_STATUS_CANCELED;
1200 virReportError(VIR_ERR_OPERATION_ABORTED, _("%s: %s"),
1201 qemuDomainAsyncJobTypeToString(priv->job.asyncJob),
1202 _("canceled by client"));
1203 return -1;
1204 }
1205
1206 if (dconn && virConnectIsAlive(dconn) <= 0) {
1207 virReportError(VIR_ERR_OPERATION_FAILED, "%s",
1208 _("Lost connection to destination host"));
1209 return -1;
1210 }
1211
1212 if (virDomainObjWait(vm) < 0)
1213 return -1;
1214 }
1215
1216 qemuMigrationSrcFetchMirrorStats(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
1217 priv->job.current);
1218 return 0;
1219 }
1220
1221
1222 /**
1223 * qemuMigrationSrcIsAllowedHostdev:
1224 * @def: domain definition
1225 *
1226 * Checks that @def does not contain any host devices unsupported across
1227 * migrations. Returns true if the vm is allowed to migrate.
1228 */
1229 static bool
qemuMigrationSrcIsAllowedHostdev(const virDomainDef * def)1230 qemuMigrationSrcIsAllowedHostdev(const virDomainDef *def)
1231 {
1232 size_t i;
1233
1234 /* Migration with USB host devices is allowed, all other devices are
1235 * forbidden. */
1236 for (i = 0; i < def->nhostdevs; i++) {
1237 virDomainHostdevDef *hostdev = def->hostdevs[i];
1238 switch ((virDomainHostdevMode)hostdev->mode) {
1239 case VIR_DOMAIN_HOSTDEV_MODE_CAPABILITIES:
1240 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
1241 _("cannot migrate a domain with <hostdev mode='capabilities'>"));
1242 return false;
1243
1244 case VIR_DOMAIN_HOSTDEV_MODE_SUBSYS:
1245 switch ((virDomainHostdevSubsysType)hostdev->source.subsys.type) {
1246 case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
1247 /* USB devices can be "migrated" */
1248 continue;
1249
1250 case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI:
1251 case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI_HOST:
1252 case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_MDEV:
1253 virReportError(VIR_ERR_OPERATION_UNSUPPORTED,
1254 _("cannot migrate a domain with <hostdev mode='subsystem' type='%s'>"),
1255 virDomainHostdevSubsysTypeToString(hostdev->source.subsys.type));
1256 return false;
1257
1258 case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
1259 /*
1260 * if the device has a <teaming type='transient'>
1261 * element, then migration *is* allowed because the
1262 * device will be auto-unplugged by QEMU during
1263 * migration. Generic <hostdev> and <interface
1264 * type='hostdev'> have their teaming configuration
1265 * stored in different places.
1266 */
1267 if ((hostdev->teaming &&
1268 hostdev->teaming->type == VIR_DOMAIN_NET_TEAMING_TYPE_TRANSIENT) ||
1269 (hostdev->parentnet && hostdev->parentnet->teaming &&
1270 hostdev->parentnet->teaming->type == VIR_DOMAIN_NET_TEAMING_TYPE_TRANSIENT)) {
1271 continue;
1272 }
1273
1274 /* all other PCI hostdevs can't be migrated */
1275 if (hostdev->parentnet) {
1276 virDomainNetType actualType = virDomainNetGetActualType(hostdev->parentnet);
1277
1278 virReportError(VIR_ERR_OPERATION_UNSUPPORTED,
1279 _("cannot migrate a domain with <interface type='%s'>"),
1280 virDomainNetTypeToString(actualType));
1281 } else {
1282 virReportError(VIR_ERR_OPERATION_UNSUPPORTED,
1283 _("cannot migrate a domain with <hostdev mode='subsystem' type='%s'>"),
1284 virDomainHostdevSubsysTypeToString(hostdev->source.subsys.type));
1285 }
1286 return false;
1287
1288 case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_LAST:
1289 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
1290 _("invalid hostdev subsystem type"));
1291 return false;
1292 }
1293 break;
1294
1295 case VIR_DOMAIN_HOSTDEV_MODE_LAST:
1296 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
1297 _("invalid hostdev mode"));
1298 return false;
1299 }
1300 }
1301
1302 return true;
1303 }
1304
1305
1306 /**
1307 * qemuMigrationSrcIsAllowed:
1308 * @driver: qemu driver struct
1309 * @vm: domain object
1310 * @remote: migration is remote
1311 * @flags: migration flags (see struct virDomainMigrateFlags)
1312 *
1313 * Validates that the configuration of @vm can be migrated in various
1314 * situations. If @remote is true, the migration happens to remote host. @flags
1315 * is used to check various special migration types according to the request.
1316 *
1317 * Returns true if migration is supported. Reports libvirt error and returns
1318 * false otherwise.
1319 */
1320 bool
qemuMigrationSrcIsAllowed(virQEMUDriver * driver,virDomainObj * vm,bool remote,unsigned int flags)1321 qemuMigrationSrcIsAllowed(virQEMUDriver *driver,
1322 virDomainObj *vm,
1323 bool remote,
1324 unsigned int flags)
1325 {
1326 qemuDomainObjPrivate *priv = vm->privateData;
1327 int nsnapshots;
1328 int pauseReason;
1329 size_t i;
1330
1331 /* perform these checks only when migrating to remote hosts */
1332 if (remote) {
1333 nsnapshots = virDomainSnapshotObjListNum(vm->snapshots, NULL, 0);
1334 if (nsnapshots < 0)
1335 return false;
1336
1337 if (nsnapshots > 0) {
1338 virReportError(VIR_ERR_OPERATION_INVALID,
1339 _("cannot migrate domain with %d snapshots"),
1340 nsnapshots);
1341 return false;
1342 }
1343 }
1344
1345 /* following checks don't make sense for offline migration */
1346 if (!(flags & VIR_MIGRATE_OFFLINE)) {
1347 if (remote) {
1348 /* cancel migration if disk I/O error is emitted while migrating */
1349 if (flags & VIR_MIGRATE_ABORT_ON_ERROR &&
1350 virDomainObjGetState(vm, &pauseReason) == VIR_DOMAIN_PAUSED &&
1351 pauseReason == VIR_DOMAIN_PAUSED_IOERROR) {
1352 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1353 _("cannot migrate domain with I/O error"));
1354 return false;
1355 }
1356
1357 if (qemuProcessAutoDestroyActive(driver, vm)) {
1358 virReportError(VIR_ERR_OPERATION_INVALID,
1359 "%s", _("domain is marked for auto destroy"));
1360 return false;
1361 }
1362 }
1363
1364
1365 if (qemuDomainHasBlockjob(vm, false)) {
1366 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1367 _("domain has active block job"));
1368 return false;
1369 }
1370
1371 if (!qemuMigrationSrcIsAllowedHostdev(vm->def))
1372 return false;
1373
1374 if (vm->def->cpu) {
1375 /* QEMU blocks migration and save with invariant TSC enabled
1376 * unless TSC frequency is explicitly set.
1377 */
1378 if (virCPUCheckFeature(vm->def->os.arch, vm->def->cpu,
1379 "invtsc") == 1) {
1380 bool block = true;
1381
1382 for (i = 0; i < vm->def->clock.ntimers; i++) {
1383 virDomainTimerDef *timer = vm->def->clock.timers[i];
1384
1385 if (timer->name == VIR_DOMAIN_TIMER_NAME_TSC &&
1386 timer->frequency > 0) {
1387 block = false;
1388 break;
1389 }
1390 }
1391
1392 if (block) {
1393 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1394 _("domain has 'invtsc' CPU feature but "
1395 "TSC frequency is not specified"));
1396 return false;
1397 }
1398 }
1399 }
1400
1401 /* Verify that memory device config can be transferred reliably */
1402 for (i = 0; i < vm->def->nmems; i++) {
1403 virDomainMemoryDef *mem = vm->def->mems[i];
1404
1405 if (mem->model == VIR_DOMAIN_MEMORY_MODEL_DIMM &&
1406 mem->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_DIMM) {
1407 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1408 _("domain's dimm info lacks slot ID "
1409 "or base address"));
1410
1411 return false;
1412 }
1413 }
1414
1415 for (i = 0; i < vm->def->nshmems; i++) {
1416 virDomainShmemDef *shmem = vm->def->shmems[i];
1417
1418 if (shmem->model == VIR_DOMAIN_SHMEM_MODEL_IVSHMEM) {
1419 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1420 _("migration with legacy shmem device is not supported"));
1421 return false;
1422 }
1423 if (shmem->role != VIR_DOMAIN_SHMEM_ROLE_MASTER) {
1424 virReportError(VIR_ERR_OPERATION_INVALID,
1425 _("shmem device '%s' cannot be migrated, "
1426 "only shmem with role='%s' can be migrated"),
1427 shmem->name,
1428 virDomainShmemRoleTypeToString(VIR_DOMAIN_SHMEM_ROLE_MASTER));
1429 return false;
1430 }
1431 }
1432
1433 for (i = 0; i < vm->def->nnets; i++) {
1434 virDomainNetDef *net = vm->def->nets[i];
1435 qemuSlirp *slirp;
1436
1437 if (net->type == VIR_DOMAIN_NET_TYPE_VDPA) {
1438 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1439 _("vDPA devices cannot be migrated"));
1440 return false;
1441 }
1442
1443 slirp = QEMU_DOMAIN_NETWORK_PRIVATE(net)->slirp;
1444
1445 if (slirp && !qemuSlirpHasFeature(slirp, QEMU_SLIRP_FEATURE_MIGRATE)) {
1446 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1447 _("a slirp-helper cannot be migrated"));
1448 return false;
1449 }
1450 }
1451
1452 for (i = 0; i < vm->def->nfss; i++) {
1453 virDomainFSDef *fs = vm->def->fss[i];
1454
1455 if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS) {
1456 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1457 _("migration with virtiofs device is not supported"));
1458 return false;
1459 }
1460 }
1461
1462 if (priv->dbusVMStateIds &&
1463 !virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DBUS_VMSTATE)) {
1464 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1465 _("cannot migrate this domain without dbus-vmstate support"));
1466 return false;
1467 }
1468
1469 for (i = 0; i < vm->def->ndisks; i++) {
1470 virDomainDiskDef *disk = vm->def->disks[i];
1471
1472 if (disk->transient) {
1473 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1474 _("migration with transient disk is not supported"));
1475 return false;
1476 }
1477 }
1478 }
1479
1480 return true;
1481 }
1482
1483 static bool
qemuMigrationSrcIsSafe(virDomainDef * def,virQEMUCaps * qemuCaps,size_t nmigrate_disks,const char ** migrate_disks,unsigned int flags)1484 qemuMigrationSrcIsSafe(virDomainDef *def,
1485 virQEMUCaps *qemuCaps,
1486 size_t nmigrate_disks,
1487 const char **migrate_disks,
1488 unsigned int flags)
1489
1490 {
1491 bool storagemigration = flags & (VIR_MIGRATE_NON_SHARED_DISK |
1492 VIR_MIGRATE_NON_SHARED_INC);
1493 size_t i;
1494 int rc;
1495
1496 for (i = 0; i < def->ndisks; i++) {
1497 virDomainDiskDef *disk = def->disks[i];
1498 const char *src = virDomainDiskGetSource(disk);
1499 int actualType = virStorageSourceGetActualType(disk->src);
1500 bool unsafe = false;
1501
1502 /* Disks without any source (i.e. floppies and CD-ROMs)
1503 * OR readonly are safe. */
1504 if (virStorageSourceIsEmpty(disk->src) ||
1505 disk->src->readonly)
1506 continue;
1507
1508 /* Disks which are migrated by qemu are safe too. */
1509 if (storagemigration &&
1510 qemuMigrationAnyCopyDisk(disk, nmigrate_disks, migrate_disks))
1511 continue;
1512
1513 /* However, disks on local FS (e.g. ext4) are not safe. */
1514 switch ((virStorageType) actualType) {
1515 case VIR_STORAGE_TYPE_FILE:
1516 if ((rc = virFileIsSharedFS(src)) < 0) {
1517 return false;
1518 } else if (rc == 0) {
1519 unsafe = true;
1520 }
1521 if ((rc = virFileIsClusterFS(src)) < 0)
1522 return false;
1523 else if (rc == 1)
1524 continue;
1525 break;
1526 case VIR_STORAGE_TYPE_NETWORK:
1527 /* But network disks are safe again. */
1528 continue;
1529
1530 case VIR_STORAGE_TYPE_NVME:
1531 unsafe = true;
1532 break;
1533
1534 case VIR_STORAGE_TYPE_VHOST_USER:
1535 case VIR_STORAGE_TYPE_NONE:
1536 case VIR_STORAGE_TYPE_BLOCK:
1537 case VIR_STORAGE_TYPE_DIR:
1538 case VIR_STORAGE_TYPE_VOLUME:
1539 case VIR_STORAGE_TYPE_LAST:
1540 break;
1541 }
1542
1543 if (unsafe) {
1544 virReportError(VIR_ERR_MIGRATE_UNSAFE, "%s",
1545 _("Migration without shared storage is unsafe"));
1546 return false;
1547 }
1548
1549 /* Our code elsewhere guarantees shared disks are either readonly (in
1550 * which case cache mode doesn't matter) or used with cache=none or used with cache=directsync */
1551 if (disk->src->shared ||
1552 disk->cachemode == VIR_DOMAIN_DISK_CACHE_DISABLE ||
1553 disk->cachemode == VIR_DOMAIN_DISK_CACHE_DIRECTSYNC)
1554 continue;
1555
1556 if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATION_FILE_DROP_CACHE)) {
1557 VIR_DEBUG("QEMU supports flushing caches; migration is safe");
1558 continue;
1559 }
1560
1561 virReportError(VIR_ERR_MIGRATE_UNSAFE, "%s",
1562 _("Migration may lead to data corruption if disks"
1563 " use cache other than none or directsync"));
1564 return false;
1565 }
1566
1567 return true;
1568 }
1569
1570
1571 void
qemuMigrationAnyPostcopyFailed(virQEMUDriver * driver,virDomainObj * vm)1572 qemuMigrationAnyPostcopyFailed(virQEMUDriver *driver,
1573 virDomainObj *vm)
1574 {
1575 virDomainState state;
1576 int reason;
1577
1578 state = virDomainObjGetState(vm, &reason);
1579
1580 if (state != VIR_DOMAIN_PAUSED &&
1581 state != VIR_DOMAIN_RUNNING)
1582 return;
1583
1584 if (state == VIR_DOMAIN_PAUSED &&
1585 reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED)
1586 return;
1587
1588 VIR_WARN("Migration of domain %s failed during post-copy; "
1589 "leaving the domain paused", vm->def->name);
1590
1591 if (state == VIR_DOMAIN_RUNNING) {
1592 if (qemuProcessStopCPUs(driver, vm,
1593 VIR_DOMAIN_PAUSED_POSTCOPY_FAILED,
1594 QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
1595 VIR_WARN("Unable to pause guest CPUs for %s", vm->def->name);
1596 } else {
1597 virDomainObjSetState(vm, VIR_DOMAIN_PAUSED,
1598 VIR_DOMAIN_PAUSED_POSTCOPY_FAILED);
1599 }
1600 }
1601
1602
1603 static int
qemuMigrationSrcWaitForSpice(virDomainObj * vm)1604 qemuMigrationSrcWaitForSpice(virDomainObj *vm)
1605 {
1606 qemuDomainObjPrivate *priv = vm->privateData;
1607 qemuDomainJobPrivate *jobPriv = priv->job.privateData;
1608
1609 if (!jobPriv->spiceMigration)
1610 return 0;
1611
1612 VIR_DEBUG("Waiting for SPICE to finish migration");
1613 while (!jobPriv->spiceMigrated && !priv->job.abortJob) {
1614 if (virDomainObjWait(vm) < 0)
1615 return -1;
1616 }
1617 return 0;
1618 }
1619
1620
1621 static void
qemuMigrationUpdateJobType(qemuDomainJobInfo * jobInfo)1622 qemuMigrationUpdateJobType(qemuDomainJobInfo *jobInfo)
1623 {
1624 switch ((qemuMonitorMigrationStatus) jobInfo->stats.mig.status) {
1625 case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY:
1626 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_POSTCOPY;
1627 break;
1628
1629 case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED:
1630 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_QEMU_COMPLETED;
1631 break;
1632
1633 case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE:
1634 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_NONE;
1635 break;
1636
1637 case QEMU_MONITOR_MIGRATION_STATUS_ERROR:
1638 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_FAILED;
1639 break;
1640
1641 case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED:
1642 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_CANCELED;
1643 break;
1644
1645 case QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER:
1646 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_PAUSED;
1647 break;
1648
1649 case QEMU_MONITOR_MIGRATION_STATUS_DEVICE:
1650 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_MIGRATING;
1651 break;
1652
1653 case QEMU_MONITOR_MIGRATION_STATUS_SETUP:
1654 case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE:
1655 case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING:
1656 case QEMU_MONITOR_MIGRATION_STATUS_WAIT_UNPLUG:
1657 case QEMU_MONITOR_MIGRATION_STATUS_LAST:
1658 break;
1659 }
1660 }
1661
1662
1663 int
qemuMigrationAnyFetchStats(virQEMUDriver * driver,virDomainObj * vm,qemuDomainAsyncJob asyncJob,qemuDomainJobInfo * jobInfo,char ** error)1664 qemuMigrationAnyFetchStats(virQEMUDriver *driver,
1665 virDomainObj *vm,
1666 qemuDomainAsyncJob asyncJob,
1667 qemuDomainJobInfo *jobInfo,
1668 char **error)
1669 {
1670 qemuDomainObjPrivate *priv = vm->privateData;
1671 qemuMonitorMigrationStats stats;
1672 int rv;
1673
1674 if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
1675 return -1;
1676
1677 rv = qemuMonitorGetMigrationStats(priv->mon, &stats, error);
1678
1679 if (qemuDomainObjExitMonitor(driver, vm) < 0 || rv < 0)
1680 return -1;
1681
1682 jobInfo->stats.mig = stats;
1683
1684 return 0;
1685 }
1686
1687
1688 static const char *
qemuMigrationJobName(virDomainObj * vm)1689 qemuMigrationJobName(virDomainObj *vm)
1690 {
1691 qemuDomainObjPrivate *priv = vm->privateData;
1692
1693 switch (priv->job.asyncJob) {
1694 case QEMU_ASYNC_JOB_MIGRATION_OUT:
1695 return _("migration out job");
1696 case QEMU_ASYNC_JOB_SAVE:
1697 return _("domain save job");
1698 case QEMU_ASYNC_JOB_DUMP:
1699 return _("domain core dump job");
1700 case QEMU_ASYNC_JOB_NONE:
1701 return _("undefined");
1702 case QEMU_ASYNC_JOB_MIGRATION_IN:
1703 return _("migration in job");
1704 case QEMU_ASYNC_JOB_SNAPSHOT:
1705 return _("snapshot job");
1706 case QEMU_ASYNC_JOB_START:
1707 return _("start job");
1708 case QEMU_ASYNC_JOB_BACKUP:
1709 return _("backup job");
1710 case QEMU_ASYNC_JOB_LAST:
1711 default:
1712 return _("job");
1713 }
1714 }
1715
1716
1717 static int
qemuMigrationJobCheckStatus(virQEMUDriver * driver,virDomainObj * vm,qemuDomainAsyncJob asyncJob)1718 qemuMigrationJobCheckStatus(virQEMUDriver *driver,
1719 virDomainObj *vm,
1720 qemuDomainAsyncJob asyncJob)
1721 {
1722 qemuDomainObjPrivate *priv = vm->privateData;
1723 qemuDomainJobInfo *jobInfo = priv->job.current;
1724 g_autofree char *error = NULL;
1725 bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT);
1726
1727 if (!events ||
1728 jobInfo->stats.mig.status == QEMU_MONITOR_MIGRATION_STATUS_ERROR) {
1729 if (qemuMigrationAnyFetchStats(driver, vm, asyncJob, jobInfo, &error) < 0)
1730 return -1;
1731 }
1732
1733 qemuMigrationUpdateJobType(jobInfo);
1734
1735 switch (jobInfo->status) {
1736 case QEMU_DOMAIN_JOB_STATUS_NONE:
1737 virReportError(VIR_ERR_OPERATION_FAILED, _("%s: %s"),
1738 qemuMigrationJobName(vm), _("is not active"));
1739 return -1;
1740
1741 case QEMU_DOMAIN_JOB_STATUS_FAILED:
1742 virReportError(VIR_ERR_OPERATION_FAILED, _("%s: %s"),
1743 qemuMigrationJobName(vm),
1744 error ? error : _("unexpectedly failed"));
1745 return -1;
1746
1747 case QEMU_DOMAIN_JOB_STATUS_CANCELED:
1748 virReportError(VIR_ERR_OPERATION_ABORTED, _("%s: %s"),
1749 qemuMigrationJobName(vm), _("canceled by client"));
1750 return -1;
1751
1752 case QEMU_DOMAIN_JOB_STATUS_COMPLETED:
1753 case QEMU_DOMAIN_JOB_STATUS_ACTIVE:
1754 case QEMU_DOMAIN_JOB_STATUS_MIGRATING:
1755 case QEMU_DOMAIN_JOB_STATUS_QEMU_COMPLETED:
1756 case QEMU_DOMAIN_JOB_STATUS_POSTCOPY:
1757 case QEMU_DOMAIN_JOB_STATUS_PAUSED:
1758 break;
1759 }
1760
1761 return 0;
1762 }
1763
1764
1765 enum qemuMigrationCompletedFlags {
1766 QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR = (1 << 0),
1767 /* This flag should only be set when run on src host */
1768 QEMU_MIGRATION_COMPLETED_CHECK_STORAGE = (1 << 1),
1769 QEMU_MIGRATION_COMPLETED_POSTCOPY = (1 << 2),
1770 QEMU_MIGRATION_COMPLETED_PRE_SWITCHOVER = (1 << 3),
1771 };
1772
1773
1774 /**
1775 * Returns 1 if migration completed successfully,
1776 * 0 if the domain is still being migrated,
1777 * -1 migration failed,
1778 * -2 something else failed, we need to cancel migration.
1779 */
1780 static int
qemuMigrationAnyCompleted(virQEMUDriver * driver,virDomainObj * vm,qemuDomainAsyncJob asyncJob,virConnectPtr dconn,unsigned int flags)1781 qemuMigrationAnyCompleted(virQEMUDriver *driver,
1782 virDomainObj *vm,
1783 qemuDomainAsyncJob asyncJob,
1784 virConnectPtr dconn,
1785 unsigned int flags)
1786 {
1787 qemuDomainObjPrivate *priv = vm->privateData;
1788 qemuDomainJobInfo *jobInfo = priv->job.current;
1789 int pauseReason;
1790
1791 if (qemuMigrationJobCheckStatus(driver, vm, asyncJob) < 0)
1792 goto error;
1793
1794 /* This flag should only be set when run on src host */
1795 if (flags & QEMU_MIGRATION_COMPLETED_CHECK_STORAGE &&
1796 qemuMigrationSrcNBDStorageCopyReady(vm, asyncJob) < 0)
1797 goto error;
1798
1799 if (flags & QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR &&
1800 virDomainObjGetState(vm, &pauseReason) == VIR_DOMAIN_PAUSED &&
1801 pauseReason == VIR_DOMAIN_PAUSED_IOERROR) {
1802 virReportError(VIR_ERR_OPERATION_FAILED, _("%s: %s"),
1803 qemuMigrationJobName(vm), _("failed due to I/O error"));
1804 goto error;
1805 }
1806
1807 if (dconn && virConnectIsAlive(dconn) <= 0) {
1808 virReportError(VIR_ERR_OPERATION_FAILED, "%s",
1809 _("Lost connection to destination host"));
1810 goto error;
1811 }
1812
1813 /* Migration was paused before serializing device state, let's return to
1814 * the caller so that it can finish all block jobs, resume migration, and
1815 * wait again for the real end of the migration.
1816 */
1817 if (flags & QEMU_MIGRATION_COMPLETED_PRE_SWITCHOVER &&
1818 jobInfo->status == QEMU_DOMAIN_JOB_STATUS_PAUSED) {
1819 VIR_DEBUG("Migration paused before switchover");
1820 return 1;
1821 }
1822
1823 /* In case of postcopy the source considers migration completed at the
1824 * moment it switched from active to postcopy-active state. The destination
1825 * will continue waiting until the migrate state changes to completed.
1826 */
1827 if (flags & QEMU_MIGRATION_COMPLETED_POSTCOPY &&
1828 jobInfo->status == QEMU_DOMAIN_JOB_STATUS_POSTCOPY) {
1829 VIR_DEBUG("Migration switched to post-copy");
1830 return 1;
1831 }
1832
1833 if (jobInfo->status == QEMU_DOMAIN_JOB_STATUS_QEMU_COMPLETED)
1834 return 1;
1835 else
1836 return 0;
1837
1838 error:
1839 switch (jobInfo->status) {
1840 case QEMU_DOMAIN_JOB_STATUS_MIGRATING:
1841 case QEMU_DOMAIN_JOB_STATUS_POSTCOPY:
1842 case QEMU_DOMAIN_JOB_STATUS_PAUSED:
1843 /* The migration was aborted by us rather than QEMU itself. */
1844 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_FAILED;
1845 return -2;
1846
1847 case QEMU_DOMAIN_JOB_STATUS_QEMU_COMPLETED:
1848 /* Something failed after QEMU already finished the migration. */
1849 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_FAILED;
1850 return -1;
1851
1852 case QEMU_DOMAIN_JOB_STATUS_FAILED:
1853 case QEMU_DOMAIN_JOB_STATUS_CANCELED:
1854 /* QEMU aborted the migration. */
1855 return -1;
1856
1857 case QEMU_DOMAIN_JOB_STATUS_ACTIVE:
1858 case QEMU_DOMAIN_JOB_STATUS_COMPLETED:
1859 case QEMU_DOMAIN_JOB_STATUS_NONE:
1860 /* Impossible. */
1861 break;
1862 }
1863
1864 return -1;
1865 }
1866
1867
1868 /* Returns 0 on success, -2 when migration needs to be cancelled, or -1 when
1869 * QEMU reports failed migration.
1870 */
1871 static int
qemuMigrationSrcWaitForCompletion(virQEMUDriver * driver,virDomainObj * vm,qemuDomainAsyncJob asyncJob,virConnectPtr dconn,unsigned int flags)1872 qemuMigrationSrcWaitForCompletion(virQEMUDriver *driver,
1873 virDomainObj *vm,
1874 qemuDomainAsyncJob asyncJob,
1875 virConnectPtr dconn,
1876 unsigned int flags)
1877 {
1878 qemuDomainObjPrivate *priv = vm->privateData;
1879 qemuDomainJobInfo *jobInfo = priv->job.current;
1880 bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT);
1881 int rv;
1882
1883 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_MIGRATING;
1884
1885 while ((rv = qemuMigrationAnyCompleted(driver, vm, asyncJob,
1886 dconn, flags)) != 1) {
1887 if (rv < 0)
1888 return rv;
1889
1890 if (events) {
1891 if (virDomainObjWait(vm) < 0) {
1892 if (virDomainObjIsActive(vm))
1893 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_FAILED;
1894 return -2;
1895 }
1896 } else {
1897 /* Poll every 50ms for progress & to allow cancellation */
1898 struct timespec ts = { .tv_sec = 0, .tv_nsec = 50 * 1000 * 1000ull };
1899
1900 virObjectUnlock(vm);
1901 nanosleep(&ts, NULL);
1902 virObjectLock(vm);
1903 }
1904 }
1905
1906 if (events)
1907 ignore_value(qemuMigrationAnyFetchStats(driver, vm, asyncJob, jobInfo, NULL));
1908
1909 qemuDomainJobInfoUpdateTime(jobInfo);
1910 qemuDomainJobInfoUpdateDowntime(jobInfo);
1911 g_clear_pointer(&priv->job.completed, qemuDomainJobInfoFree);
1912 priv->job.completed = qemuDomainJobInfoCopy(jobInfo);
1913 priv->job.completed->status = QEMU_DOMAIN_JOB_STATUS_COMPLETED;
1914
1915 if (asyncJob != QEMU_ASYNC_JOB_MIGRATION_OUT &&
1916 jobInfo->status == QEMU_DOMAIN_JOB_STATUS_QEMU_COMPLETED)
1917 jobInfo->status = QEMU_DOMAIN_JOB_STATUS_COMPLETED;
1918
1919 return 0;
1920 }
1921
1922
1923 static int
qemuMigrationDstWaitForCompletion(virQEMUDriver * driver,virDomainObj * vm,qemuDomainAsyncJob asyncJob,bool postcopy)1924 qemuMigrationDstWaitForCompletion(virQEMUDriver *driver,
1925 virDomainObj *vm,
1926 qemuDomainAsyncJob asyncJob,
1927 bool postcopy)
1928 {
1929 qemuDomainObjPrivate *priv = vm->privateData;
1930 unsigned int flags = 0;
1931 int rv;
1932
1933 if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT))
1934 return 0;
1935
1936 VIR_DEBUG("Waiting for incoming migration to complete");
1937
1938 if (postcopy)
1939 flags = QEMU_MIGRATION_COMPLETED_POSTCOPY;
1940
1941 while ((rv = qemuMigrationAnyCompleted(driver, vm, asyncJob,
1942 NULL, flags)) != 1) {
1943 if (rv < 0 || virDomainObjWait(vm) < 0)
1944 return -1;
1945 }
1946
1947 return 0;
1948 }
1949
1950
1951 static int
qemuMigrationSrcGraphicsRelocate(virQEMUDriver * driver,virDomainObj * vm,qemuMigrationCookie * cookie,const char * graphicsuri)1952 qemuMigrationSrcGraphicsRelocate(virQEMUDriver *driver,
1953 virDomainObj *vm,
1954 qemuMigrationCookie *cookie,
1955 const char *graphicsuri)
1956 {
1957 qemuDomainObjPrivate *priv = vm->privateData;
1958 int ret = -1;
1959 const char *listenAddress = NULL;
1960 virSocketAddr addr;
1961 virURI *uri = NULL;
1962 int type = -1;
1963 int port = -1;
1964 int tlsPort = -1;
1965 const char *tlsSubject = NULL;
1966
1967 if (!cookie || (!cookie->graphics && !graphicsuri))
1968 return 0;
1969
1970 if (graphicsuri && !(uri = virURIParse(graphicsuri)))
1971 goto cleanup;
1972
1973 if (cookie->graphics) {
1974 type = cookie->graphics->type;
1975
1976 listenAddress = cookie->graphics->listen;
1977
1978 if (!listenAddress ||
1979 (virSocketAddrParse(&addr, listenAddress, AF_UNSPEC) > 0 &&
1980 virSocketAddrIsWildcard(&addr)))
1981 listenAddress = cookie->remoteHostname;
1982
1983 port = cookie->graphics->port;
1984 tlsPort = cookie->graphics->tlsPort;
1985 tlsSubject = cookie->graphics->tlsSubject;
1986 }
1987
1988 if (uri) {
1989 size_t i;
1990
1991 if ((type = virDomainGraphicsTypeFromString(uri->scheme)) < 0) {
1992 virReportError(VIR_ERR_INVALID_ARG,
1993 _("unknown graphics type %s"), uri->scheme);
1994 goto cleanup;
1995 }
1996
1997 if (uri->server)
1998 listenAddress = uri->server;
1999 if (uri->port > 0)
2000 port = uri->port;
2001
2002 for (i = 0; i < uri->paramsCount; i++) {
2003 virURIParam *param = uri->params + i;
2004
2005 if (STRCASEEQ(param->name, "tlsPort")) {
2006 if (virStrToLong_i(param->value, NULL, 10, &tlsPort) < 0) {
2007 virReportError(VIR_ERR_INVALID_ARG,
2008 _("invalid tlsPort number: %s"),
2009 param->value);
2010 goto cleanup;
2011 }
2012 } else if (STRCASEEQ(param->name, "tlsSubject")) {
2013 tlsSubject = param->value;
2014 }
2015 }
2016 }
2017
2018 /* QEMU doesn't support VNC relocation yet, so
2019 * skip it to avoid generating an error
2020 */
2021 if (type != VIR_DOMAIN_GRAPHICS_TYPE_SPICE) {
2022 ret = 0;
2023 goto cleanup;
2024 }
2025
2026 /* Older libvirt sends port == 0 for listen type='none' graphics. It's
2027 * safe to ignore such requests since relocation to unknown port does
2028 * not make sense in general.
2029 */
2030 if (port <= 0 && tlsPort <= 0) {
2031 ret = 0;
2032 goto cleanup;
2033 }
2034
2035 if (qemuDomainObjEnterMonitorAsync(driver, vm,
2036 QEMU_ASYNC_JOB_MIGRATION_OUT) == 0) {
2037 qemuDomainJobPrivate *jobPriv = priv->job.privateData;
2038
2039 ret = qemuMonitorGraphicsRelocate(priv->mon, type, listenAddress,
2040 port, tlsPort, tlsSubject);
2041 jobPriv->spiceMigration = !ret;
2042 if (qemuDomainObjExitMonitor(driver, vm) < 0)
2043 ret = -1;
2044 }
2045
2046 cleanup:
2047 virURIFree(uri);
2048 return ret;
2049 }
2050
2051
2052 static int
qemuMigrationDstOPDRelocate(virQEMUDriver * driver G_GNUC_UNUSED,virDomainObj * vm,qemuMigrationCookie * cookie)2053 qemuMigrationDstOPDRelocate(virQEMUDriver *driver G_GNUC_UNUSED,
2054 virDomainObj *vm,
2055 qemuMigrationCookie *cookie)
2056 {
2057 virDomainNetDef *netptr;
2058 size_t i;
2059
2060 for (i = 0; i < cookie->network->nnets; i++) {
2061 netptr = vm->def->nets[i];
2062
2063 switch (cookie->network->net[i].vporttype) {
2064 case VIR_NETDEV_VPORT_PROFILE_NONE:
2065 case VIR_NETDEV_VPORT_PROFILE_8021QBG:
2066 case VIR_NETDEV_VPORT_PROFILE_8021QBH:
2067 break;
2068 case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH:
2069 if (virNetDevOpenvswitchSetMigrateData(cookie->network->net[i].portdata,
2070 netptr->ifname) != 0) {
2071 virReportError(VIR_ERR_INTERNAL_ERROR,
2072 _("Unable to run command to set OVS port data for "
2073 "interface %s"), netptr->ifname);
2074 return -1;
2075 }
2076 break;
2077 default:
2078 break;
2079 }
2080 }
2081
2082 return 0;
2083 }
2084
2085
2086 int
qemuMigrationDstCheckProtocol(virQEMUCaps * qemuCaps,const char * migrateFrom)2087 qemuMigrationDstCheckProtocol(virQEMUCaps *qemuCaps,
2088 const char *migrateFrom)
2089 {
2090 if (STRPREFIX(migrateFrom, "rdma")) {
2091 if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {
2092 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
2093 _("incoming RDMA migration is not supported "
2094 "with this QEMU binary"));
2095 return -1;
2096 }
2097 } else if (!STRPREFIX(migrateFrom, "tcp") &&
2098 !STRPREFIX(migrateFrom, "exec") &&
2099 !STRPREFIX(migrateFrom, "fd") &&
2100 !STRPREFIX(migrateFrom, "unix") &&
2101 STRNEQ(migrateFrom, "stdio")) {
2102 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
2103 _("unknown migration protocol"));
2104 return -1;
2105 }
2106
2107 return 0;
2108 }
2109
2110
2111 char *
qemuMigrationDstGetURI(const char * migrateFrom,int migrateFd)2112 qemuMigrationDstGetURI(const char *migrateFrom,
2113 int migrateFd)
2114 {
2115 char *uri = NULL;
2116
2117 if (STREQ(migrateFrom, "stdio"))
2118 uri = g_strdup_printf("fd:%d", migrateFd);
2119 else
2120 uri = g_strdup(migrateFrom);
2121
2122 return uri;
2123 }
2124
2125
2126 int
qemuMigrationDstRun(virQEMUDriver * driver,virDomainObj * vm,const char * uri,qemuDomainAsyncJob asyncJob)2127 qemuMigrationDstRun(virQEMUDriver *driver,
2128 virDomainObj *vm,
2129 const char *uri,
2130 qemuDomainAsyncJob asyncJob)
2131 {
2132 qemuDomainObjPrivate *priv = vm->privateData;
2133 int rv;
2134
2135 VIR_DEBUG("Setting up incoming migration with URI %s", uri);
2136
2137 if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
2138 return -1;
2139
2140 rv = qemuMonitorSetDBusVMStateIdList(priv->mon, priv->dbusVMStateIds);
2141 if (rv < 0)
2142 goto exit_monitor;
2143
2144 rv = qemuMonitorMigrateIncoming(priv->mon, uri);
2145
2146 exit_monitor:
2147 if (qemuDomainObjExitMonitor(driver, vm) < 0 || rv < 0)
2148 return -1;
2149
2150 if (asyncJob == QEMU_ASYNC_JOB_MIGRATION_IN) {
2151 /* qemuMigrationDstWaitForCompletion is called from the Finish phase */
2152 return 0;
2153 }
2154
2155 if (qemuMigrationDstWaitForCompletion(driver, vm, asyncJob, false) < 0)
2156 return -1;
2157
2158 return 0;
2159 }
2160
2161
2162 /* This is called for outgoing non-p2p migrations when a connection to the
2163 * client which initiated the migration was closed but we were waiting for it
2164 * to follow up with the next phase, that is, in between
2165 * qemuDomainMigrateBegin3 and qemuDomainMigratePerform3 or
2166 * qemuDomainMigratePerform3 and qemuDomainMigrateConfirm3.
2167 */
2168 static void
qemuMigrationSrcCleanup(virDomainObj * vm,virConnectPtr conn,void * opaque)2169 qemuMigrationSrcCleanup(virDomainObj *vm,
2170 virConnectPtr conn,
2171 void *opaque)
2172 {
2173 virQEMUDriver *driver = opaque;
2174 qemuDomainObjPrivate *priv = vm->privateData;
2175 qemuDomainJobPrivate *jobPriv = priv->job.privateData;
2176
2177 VIR_DEBUG("vm=%s, conn=%p, asyncJob=%s, phase=%s",
2178 vm->def->name, conn,
2179 qemuDomainAsyncJobTypeToString(priv->job.asyncJob),
2180 qemuDomainAsyncJobPhaseToString(priv->job.asyncJob,
2181 priv->job.phase));
2182
2183 if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT))
2184 return;
2185
2186 VIR_DEBUG("The connection which started outgoing migration of domain %s"
2187 " was closed; canceling the migration",
2188 vm->def->name);
2189
2190 switch ((qemuMigrationJobPhase) priv->job.phase) {
2191 case QEMU_MIGRATION_PHASE_BEGIN3:
2192 /* just forget we were about to migrate */
2193 qemuMigrationJobFinish(driver, vm);
2194 break;
2195
2196 case QEMU_MIGRATION_PHASE_PERFORM3_DONE:
2197 VIR_WARN("Migration of domain %s finished but we don't know if the"
2198 " domain was successfully started on destination or not",
2199 vm->def->name);
2200 qemuMigrationParamsReset(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
2201 jobPriv->migParams, priv->job.apiFlags);
2202 /* clear the job and let higher levels decide what to do */
2203 qemuMigrationJobFinish(driver, vm);
2204 break;
2205
2206 case QEMU_MIGRATION_PHASE_PERFORM3:
2207 /* cannot be seen without an active migration API; unreachable */
2208 case QEMU_MIGRATION_PHASE_CONFIRM3:
2209 case QEMU_MIGRATION_PHASE_CONFIRM3_CANCELLED:
2210 /* all done; unreachable */
2211 case QEMU_MIGRATION_PHASE_PREPARE:
2212 case QEMU_MIGRATION_PHASE_FINISH2:
2213 case QEMU_MIGRATION_PHASE_FINISH3:
2214 /* incoming migration; unreachable */
2215 case QEMU_MIGRATION_PHASE_PERFORM2:
2216 /* single phase outgoing migration; unreachable */
2217 case QEMU_MIGRATION_PHASE_NONE:
2218 case QEMU_MIGRATION_PHASE_LAST:
2219 /* unreachable */
2220 ;
2221 }
2222 }
2223
2224
2225 /**
2226 * qemuMigrationSrcBeginPhaseBlockDirtyBitmaps:
2227 * @mig: migration cookie struct
2228 * @vm: domain object
2229 * @migrate_disks: disks which are being migrated
2230 * @nmigrage_disks: number of @migrate_disks
2231 *
2232 * Enumerates block dirty bitmaps on disks which will undergo storage migration
2233 * and fills them into @mig to be offered to the destination.
2234 */
2235 static int
qemuMigrationSrcBeginPhaseBlockDirtyBitmaps(qemuMigrationCookie * mig,virDomainObj * vm,const char ** migrate_disks,size_t nmigrate_disks)2236 qemuMigrationSrcBeginPhaseBlockDirtyBitmaps(qemuMigrationCookie *mig,
2237 virDomainObj *vm,
2238 const char **migrate_disks,
2239 size_t nmigrate_disks)
2240
2241 {
2242 GSList *disks = NULL;
2243 qemuDomainObjPrivate *priv = vm->privateData;
2244 size_t i;
2245
2246 g_autoptr(GHashTable) blockNamedNodeData = NULL;
2247
2248 if (!(blockNamedNodeData = qemuBlockGetNamedNodeData(vm, priv->job.asyncJob)))
2249 return -1;
2250
2251 for (i = 0; i < vm->def->ndisks; i++) {
2252 qemuMigrationBlockDirtyBitmapsDisk *disk;
2253 GSList *bitmaps = NULL;
2254 virDomainDiskDef *diskdef = vm->def->disks[i];
2255 qemuBlockNamedNodeData *nodedata = virHashLookup(blockNamedNodeData, diskdef->src->nodeformat);
2256 size_t j;
2257
2258 if (!nodedata)
2259 continue;
2260
2261 if (migrate_disks) {
2262 bool migrating = false;
2263
2264 for (j = 0; j < nmigrate_disks; j++) {
2265 if (STREQ(migrate_disks[j], diskdef->dst)) {
2266 migrating = true;
2267 break;
2268 }
2269 }
2270
2271 if (!migrating)
2272 continue;
2273 }
2274
2275 for (j = 0; j < nodedata->nbitmaps; j++) {
2276 qemuMigrationBlockDirtyBitmapsDiskBitmap *bitmap;
2277
2278 if (!qemuBlockBitmapChainIsValid(diskdef->src,
2279 nodedata->bitmaps[j]->name,
2280 blockNamedNodeData))
2281 continue;
2282
2283 bitmap = g_new0(qemuMigrationBlockDirtyBitmapsDiskBitmap, 1);
2284 bitmap->bitmapname = g_strdup(nodedata->bitmaps[j]->name);
2285 bitmap->alias = g_strdup_printf("libvirt-%s-%s",
2286 diskdef->dst,
2287 nodedata->bitmaps[j]->name);
2288 bitmaps = g_slist_prepend(bitmaps, bitmap);
2289 }
2290
2291 if (!bitmaps)
2292 continue;
2293
2294 disk = g_new0(qemuMigrationBlockDirtyBitmapsDisk, 1);
2295 disk->target = g_strdup(diskdef->dst);
2296 disk->bitmaps = bitmaps;
2297 disks = g_slist_prepend(disks, disk);
2298 }
2299
2300 if (!disks)
2301 return 0;
2302
2303 mig->blockDirtyBitmaps = disks;
2304 mig->flags |= QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS;
2305
2306 return 0;
2307 }
2308
2309
2310 /* The caller is supposed to lock the vm and start a migration job. */
2311 static char *
qemuMigrationSrcBeginPhase(virQEMUDriver * driver,virDomainObj * vm,const char * xmlin,const char * dname,char ** cookieout,int * cookieoutlen,size_t nmigrate_disks,const char ** migrate_disks,unsigned long flags)2312 qemuMigrationSrcBeginPhase(virQEMUDriver *driver,
2313 virDomainObj *vm,
2314 const char *xmlin,
2315 const char *dname,
2316 char **cookieout,
2317 int *cookieoutlen,
2318 size_t nmigrate_disks,
2319 const char **migrate_disks,
2320 unsigned long flags)
2321 {
2322 g_autoptr(qemuMigrationCookie) mig = NULL;
2323 g_autoptr(virDomainDef) def = NULL;
2324 qemuDomainObjPrivate *priv = vm->privateData;
2325 unsigned int cookieFlags = QEMU_MIGRATION_COOKIE_LOCKSTATE;
2326
2327 VIR_DEBUG("driver=%p, vm=%p, xmlin=%s, dname=%s,"
2328 " cookieout=%p, cookieoutlen=%p,"
2329 " nmigrate_disks=%zu, migrate_disks=%p, flags=0x%lx",
2330 driver, vm, NULLSTR(xmlin), NULLSTR(dname),
2331 cookieout, cookieoutlen, nmigrate_disks,
2332 migrate_disks, flags);
2333
2334 /* Only set the phase if we are inside QEMU_ASYNC_JOB_MIGRATION_OUT.
2335 * Otherwise we will start the async job later in the perform phase losing
2336 * change protection.
2337 */
2338 if (priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_OUT)
2339 qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_BEGIN3);
2340
2341 if (!qemuMigrationSrcIsAllowed(driver, vm, true, flags))
2342 return NULL;
2343
2344 if (!(flags & (VIR_MIGRATE_UNSAFE | VIR_MIGRATE_OFFLINE)) &&
2345 !qemuMigrationSrcIsSafe(vm->def, priv->qemuCaps,
2346 nmigrate_disks, migrate_disks, flags))
2347 return NULL;
2348
2349 if (flags & VIR_MIGRATE_POSTCOPY &&
2350 (!(flags & VIR_MIGRATE_LIVE) ||
2351 flags & VIR_MIGRATE_PAUSED)) {
2352 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
2353 _("post-copy migration is not supported with non-live "
2354 "or paused migration"));
2355 return NULL;
2356 }
2357
2358 if (flags & VIR_MIGRATE_POSTCOPY && flags & VIR_MIGRATE_TUNNELLED) {
2359 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
2360 _("post-copy is not supported with tunnelled migration"));
2361 return NULL;
2362 }
2363
2364 if (flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC)) {
2365 if (flags & VIR_MIGRATE_TUNNELLED) {
2366 if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_BLOCKDEV)) {
2367 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
2368 _("migration of non-shared storage is not supported with tunnelled migration and this QEMU"));
2369 return NULL;
2370 }
2371
2372 if (nmigrate_disks) {
2373 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
2374 _("Selecting disks to migrate is not implemented for tunnelled migration"));
2375 return NULL;
2376 }
2377 } else {
2378 if (nmigrate_disks) {
2379 size_t i, j;
2380 /* Check user requested only known disk targets. */
2381 for (i = 0; i < nmigrate_disks; i++) {
2382 for (j = 0; j < vm->def->ndisks; j++) {
2383 if (STREQ(vm->def->disks[j]->dst, migrate_disks[i]))
2384 break;
2385 }
2386
2387 if (j == vm->def->ndisks) {
2388 virReportError(VIR_ERR_INVALID_ARG,
2389 _("disk target %s not found"),
2390 migrate_disks[i]);
2391 return NULL;
2392 }
2393 }
2394 }
2395
2396 priv->nbdPort = 0;
2397
2398 if (qemuMigrationHasAnyStorageMigrationDisks(vm->def,
2399 migrate_disks,
2400 nmigrate_disks))
2401 cookieFlags |= QEMU_MIGRATION_COOKIE_NBD;
2402 }
2403 }
2404
2405 if (virDomainDefHasMemoryHotplug(vm->def) ||
2406 ((flags & VIR_MIGRATE_PERSIST_DEST) &&
2407 vm->newDef && virDomainDefHasMemoryHotplug(vm->newDef)))
2408 cookieFlags |= QEMU_MIGRATION_COOKIE_MEMORY_HOTPLUG;
2409
2410 if (!qemuDomainVcpuHotplugIsInOrder(vm->def) ||
2411 ((flags & VIR_MIGRATE_PERSIST_DEST) &&
2412 vm->newDef && !qemuDomainVcpuHotplugIsInOrder(vm->newDef)))
2413 cookieFlags |= QEMU_MIGRATION_COOKIE_CPU_HOTPLUG;
2414
2415 if (priv->origCPU)
2416 cookieFlags |= QEMU_MIGRATION_COOKIE_CPU;
2417
2418 if (!(flags & VIR_MIGRATE_OFFLINE))
2419 cookieFlags |= QEMU_MIGRATION_COOKIE_CAPS;
2420
2421 if (!(mig = qemuMigrationCookieNew(vm->def, priv->origname)))
2422 return NULL;
2423
2424 if (cookieFlags & QEMU_MIGRATION_COOKIE_NBD &&
2425 virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_PARAM_BLOCK_BITMAP_MAPPING) &&
2426 qemuMigrationSrcBeginPhaseBlockDirtyBitmaps(mig, vm, migrate_disks,
2427 nmigrate_disks) < 0)
2428 return NULL;
2429
2430 if (qemuMigrationCookieFormat(mig, driver, vm,
2431 QEMU_MIGRATION_SOURCE,
2432 cookieout, cookieoutlen,
2433 cookieFlags) < 0)
2434 return NULL;
2435
2436 if (flags & VIR_MIGRATE_OFFLINE) {
2437 if (flags & (VIR_MIGRATE_NON_SHARED_DISK |
2438 VIR_MIGRATE_NON_SHARED_INC)) {
2439 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2440 _("offline migration cannot handle "
2441 "non-shared storage"));
2442 return NULL;
2443 }
2444 if (!(flags & VIR_MIGRATE_PERSIST_DEST)) {
2445 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2446 _("offline migration must be specified with "
2447 "the persistent flag set"));
2448 return NULL;
2449 }
2450 if (flags & VIR_MIGRATE_TUNNELLED) {
2451 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2452 _("tunnelled offline migration does not "
2453 "make sense"));
2454 return NULL;
2455 }
2456 }
2457
2458 if (xmlin) {
2459 if (!(def = virDomainDefParseString(xmlin, driver->xmlopt, priv->qemuCaps,
2460 VIR_DOMAIN_DEF_PARSE_INACTIVE |
2461 VIR_DOMAIN_DEF_PARSE_SKIP_VALIDATE)))
2462 return NULL;
2463
2464 if (!qemuDomainCheckABIStability(driver, vm, def))
2465 return NULL;
2466
2467 return qemuDomainDefFormatLive(driver, priv->qemuCaps, def, NULL, false, true);
2468 } else {
2469 return qemuDomainDefFormatLive(driver, priv->qemuCaps, vm->def, priv->origCPU,
2470 false, true);
2471 }
2472 }
2473
2474 char *
qemuMigrationSrcBegin(virConnectPtr conn,virDomainObj * vm,const char * xmlin,const char * dname,char ** cookieout,int * cookieoutlen,size_t nmigrate_disks,const char ** migrate_disks,unsigned long flags)2475 qemuMigrationSrcBegin(virConnectPtr conn,
2476 virDomainObj *vm,
2477 const char *xmlin,
2478 const char *dname,
2479 char **cookieout,
2480 int *cookieoutlen,
2481 size_t nmigrate_disks,
2482 const char **migrate_disks,
2483 unsigned long flags)
2484 {
2485 virQEMUDriver *driver = conn->privateData;
2486 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
2487 char *xml = NULL;
2488 qemuDomainAsyncJob asyncJob;
2489
2490 if (cfg->migrateTLSForce &&
2491 !(flags & VIR_MIGRATE_TUNNELLED) &&
2492 !(flags & VIR_MIGRATE_TLS)) {
2493 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2494 _("this libvirtd instance allows migration only with VIR_MIGRATE_TLS flag"));
2495 goto cleanup;
2496 }
2497
2498 if ((flags & VIR_MIGRATE_CHANGE_PROTECTION)) {
2499 if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
2500 flags) < 0)
2501 goto cleanup;
2502 asyncJob = QEMU_ASYNC_JOB_MIGRATION_OUT;
2503 } else {
2504 if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0)
2505 goto cleanup;
2506 asyncJob = QEMU_ASYNC_JOB_NONE;
2507 }
2508
2509 qemuMigrationSrcStoreDomainState(vm);
2510
2511 if (!(flags & VIR_MIGRATE_OFFLINE) && virDomainObjCheckActive(vm) < 0)
2512 goto endjob;
2513
2514 /* Check if there is any ejected media.
2515 * We don't want to require them on the destination.
2516 */
2517 if (!(flags & VIR_MIGRATE_OFFLINE) &&
2518 qemuProcessRefreshDisks(driver, vm, asyncJob) < 0)
2519 goto endjob;
2520
2521 if (!(xml = qemuMigrationSrcBeginPhase(driver, vm, xmlin, dname,
2522 cookieout, cookieoutlen,
2523 nmigrate_disks, migrate_disks, flags)))
2524 goto endjob;
2525
2526 if ((flags & VIR_MIGRATE_CHANGE_PROTECTION)) {
2527 /* We keep the job active across API calls until the confirm() call.
2528 * This prevents any other APIs being invoked while migration is taking
2529 * place.
2530 */
2531 if (virCloseCallbacksSet(driver->closeCallbacks, vm, conn,
2532 qemuMigrationSrcCleanup) < 0) {
2533 VIR_FREE(xml);
2534 goto endjob;
2535 }
2536 qemuMigrationJobContinue(vm);
2537 } else {
2538 goto endjob;
2539 }
2540
2541 cleanup:
2542 virDomainObjEndAPI(&vm);
2543 return xml;
2544
2545 endjob:
2546 if (flags & VIR_MIGRATE_CHANGE_PROTECTION)
2547 qemuMigrationJobFinish(driver, vm);
2548 else
2549 qemuDomainObjEndJob(driver, vm);
2550 goto cleanup;
2551 }
2552
2553
2554 /* Prepare is the first step, and it runs on the destination host.
2555 */
2556
2557 static void
qemuMigrationDstPrepareCleanup(virQEMUDriver * driver,virDomainObj * vm)2558 qemuMigrationDstPrepareCleanup(virQEMUDriver *driver,
2559 virDomainObj *vm)
2560 {
2561 qemuDomainObjPrivate *priv = vm->privateData;
2562
2563 VIR_DEBUG("driver=%p, vm=%s, job=%s, asyncJob=%s",
2564 driver,
2565 vm->def->name,
2566 qemuDomainJobTypeToString(priv->job.active),
2567 qemuDomainAsyncJobTypeToString(priv->job.asyncJob));
2568
2569 virPortAllocatorRelease(priv->migrationPort);
2570 priv->migrationPort = 0;
2571
2572 if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_IN))
2573 return;
2574 qemuDomainObjDiscardAsyncJob(driver, vm);
2575 }
2576
2577 static qemuProcessIncomingDef *
qemuMigrationDstPrepare(virDomainObj * vm,bool tunnel,const char * protocol,const char * listenAddress,unsigned short port,int fd)2578 qemuMigrationDstPrepare(virDomainObj *vm,
2579 bool tunnel,
2580 const char *protocol,
2581 const char *listenAddress,
2582 unsigned short port,
2583 int fd)
2584 {
2585 qemuDomainObjPrivate *priv = vm->privateData;
2586 g_autofree char *migrateFrom = NULL;
2587
2588 if (tunnel) {
2589 migrateFrom = g_strdup("stdio");
2590 } else if (g_strcmp0(protocol, "unix") == 0) {
2591 migrateFrom = g_strdup_printf("%s:%s", protocol, listenAddress);
2592 } else {
2593 bool encloseAddress = false;
2594 bool hostIPv6Capable = false;
2595 struct addrinfo *info = NULL;
2596 struct addrinfo hints = { .ai_flags = AI_ADDRCONFIG,
2597 .ai_socktype = SOCK_STREAM };
2598 const char *incFormat;
2599
2600 if (getaddrinfo("::", NULL, &hints, &info) == 0) {
2601 freeaddrinfo(info);
2602 hostIPv6Capable = true;
2603 }
2604
2605 if (listenAddress) {
2606 if (virSocketAddrNumericFamily(listenAddress) == AF_INET6) {
2607 if (!hostIPv6Capable) {
2608 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
2609 _("host isn't capable of IPv6"));
2610 return NULL;
2611 }
2612 /* IPv6 address must be escaped in brackets on the cmd line */
2613 encloseAddress = true;
2614 } else {
2615 /* listenAddress is a hostname or IPv4 */
2616 }
2617 } else if (hostIPv6Capable) {
2618 /* Listen on :: instead of 0.0.0.0 if QEMU understands it
2619 * and there is at least one IPv6 address configured
2620 */
2621 listenAddress = "::";
2622 encloseAddress = true;
2623 } else {
2624 listenAddress = "0.0.0.0";
2625 }
2626
2627 /* QEMU will be started with
2628 * -incoming protocol:[<IPv6 addr>]:port,
2629 * -incoming protocol:<IPv4 addr>:port, or
2630 * -incoming protocol:<hostname>:port
2631 */
2632 if (encloseAddress)
2633 incFormat = "%s:[%s]:%d";
2634 else
2635 incFormat = "%s:%s:%d";
2636 migrateFrom = g_strdup_printf(incFormat, protocol, listenAddress, port);
2637 }
2638
2639 return qemuProcessIncomingDefNew(priv->qemuCaps, listenAddress,
2640 migrateFrom, fd, NULL);
2641 }
2642
2643
2644 /**
2645 * qemuMigrationDstPrepareAnyBlockDirtyBitmaps:
2646 * @vm: domain object
2647 * @mig: migration cookie
2648 * @migParams: migration parameters
2649 * @flags: migration flags
2650 *
2651 * Checks whether block dirty bitmaps offered by the migration source are
2652 * to be migrated (e.g. they don't exist, the destination is compatible etc)
2653 * and sets up destination qemu for migrating the bitmaps as well as updates the
2654 * list of eligible bitmaps in the migration cookie to be sent back to the
2655 * source.
2656 */
2657 static int
qemuMigrationDstPrepareAnyBlockDirtyBitmaps(virDomainObj * vm,qemuMigrationCookie * mig,qemuMigrationParams * migParams,unsigned int flags)2658 qemuMigrationDstPrepareAnyBlockDirtyBitmaps(virDomainObj *vm,
2659 qemuMigrationCookie *mig,
2660 qemuMigrationParams *migParams,
2661 unsigned int flags)
2662 {
2663 qemuDomainObjPrivate *priv = vm->privateData;
2664 g_autoptr(virJSONValue) mapping = NULL;
2665 g_autoptr(GHashTable) blockNamedNodeData = NULL;
2666 GSList *nextdisk;
2667
2668 if (!mig->nbd ||
2669 !mig->blockDirtyBitmaps ||
2670 !(flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC)) ||
2671 !virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_PARAM_BLOCK_BITMAP_MAPPING))
2672 return 0;
2673
2674 if (qemuMigrationCookieBlockDirtyBitmapsMatchDisks(vm->def, mig->blockDirtyBitmaps) < 0)
2675 return -1;
2676
2677 if (!(blockNamedNodeData = qemuBlockGetNamedNodeData(vm, QEMU_ASYNC_JOB_MIGRATION_IN)))
2678 return -1;
2679
2680 for (nextdisk = mig->blockDirtyBitmaps; nextdisk; nextdisk = nextdisk->next) {
2681 qemuMigrationBlockDirtyBitmapsDisk *disk = nextdisk->data;
2682 qemuBlockNamedNodeData *nodedata;
2683 GSList *nextbitmap;
2684
2685 if (!(nodedata = virHashLookup(blockNamedNodeData, disk->nodename))) {
2686 virReportError(VIR_ERR_INTERNAL_ERROR,
2687 _("failed to find data for block node '%s'"),
2688 disk->nodename);
2689 return -1;
2690 }
2691
2692 /* Bitmaps can only be migrated to qcow2 v3+ */
2693 if (disk->disk->src->format != VIR_STORAGE_FILE_QCOW2 ||
2694 nodedata->qcow2v2) {
2695 disk->skip = true;
2696 continue;
2697 }
2698
2699 for (nextbitmap = disk->bitmaps; nextbitmap; nextbitmap = nextbitmap->next) {
2700 qemuMigrationBlockDirtyBitmapsDiskBitmap *bitmap = nextbitmap->data;
2701 size_t k;
2702
2703 /* don't migrate into existing bitmaps */
2704 for (k = 0; k < nodedata->nbitmaps; k++) {
2705 if (STREQ(bitmap->bitmapname, nodedata->bitmaps[k]->name)) {
2706 bitmap->skip = true;
2707 break;
2708 }
2709 }
2710
2711 if (bitmap->skip)
2712 continue;
2713 }
2714 }
2715
2716 if (qemuMigrationCookieBlockDirtyBitmapsToParams(mig->blockDirtyBitmaps,
2717 &mapping) < 0)
2718 return -1;
2719
2720 if (!mapping)
2721 return 0;
2722
2723 qemuMigrationParamsSetBlockDirtyBitmapMapping(migParams, &mapping);
2724 mig->flags |= QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS;
2725 return 0;
2726 }
2727
2728
2729 static int
qemuMigrationDstPrepareAny(virQEMUDriver * driver,virConnectPtr dconn,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,virDomainDef ** def,const char * origname,virStreamPtr st,const char * protocol,unsigned short port,bool autoPort,const char * listenAddress,size_t nmigrate_disks,const char ** migrate_disks,int nbdPort,const char * nbdURI,qemuMigrationParams * migParams,unsigned long flags)2730 qemuMigrationDstPrepareAny(virQEMUDriver *driver,
2731 virConnectPtr dconn,
2732 const char *cookiein,
2733 int cookieinlen,
2734 char **cookieout,
2735 int *cookieoutlen,
2736 virDomainDef **def,
2737 const char *origname,
2738 virStreamPtr st,
2739 const char *protocol,
2740 unsigned short port,
2741 bool autoPort,
2742 const char *listenAddress,
2743 size_t nmigrate_disks,
2744 const char **migrate_disks,
2745 int nbdPort,
2746 const char *nbdURI,
2747 qemuMigrationParams *migParams,
2748 unsigned long flags)
2749 {
2750 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
2751 virDomainObj *vm = NULL;
2752 virObjectEvent *event = NULL;
2753 virErrorPtr origErr;
2754 int ret = -1;
2755 int dataFD[2] = { -1, -1 };
2756 qemuDomainObjPrivate *priv = NULL;
2757 qemuMigrationCookie *mig = NULL;
2758 qemuDomainJobPrivate *jobPriv = NULL;
2759 bool tunnel = !!st;
2760 g_autofree char *xmlout = NULL;
2761 unsigned int cookieFlags;
2762 unsigned int startFlags;
2763 qemuProcessIncomingDef *incoming = NULL;
2764 bool taint_hook = false;
2765 bool stopProcess = false;
2766 bool relabel = false;
2767 int rv;
2768 g_autofree char *tlsAlias = NULL;
2769
2770 virNWFilterReadLockFilterUpdates();
2771
2772 if (flags & VIR_MIGRATE_OFFLINE) {
2773 if (flags & (VIR_MIGRATE_NON_SHARED_DISK |
2774 VIR_MIGRATE_NON_SHARED_INC)) {
2775 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2776 _("offline migration cannot handle "
2777 "non-shared storage"));
2778 goto cleanup;
2779 }
2780 if (!(flags & VIR_MIGRATE_PERSIST_DEST)) {
2781 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2782 _("offline migration must be specified with "
2783 "the persistent flag set"));
2784 goto cleanup;
2785 }
2786 if (tunnel) {
2787 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2788 _("tunnelled offline migration does not "
2789 "make sense"));
2790 goto cleanup;
2791 }
2792 cookieFlags = 0;
2793 } else {
2794 cookieFlags = QEMU_MIGRATION_COOKIE_GRAPHICS |
2795 QEMU_MIGRATION_COOKIE_CAPS;
2796 }
2797
2798 if (flags & VIR_MIGRATE_POSTCOPY &&
2799 (!(flags & VIR_MIGRATE_LIVE) ||
2800 flags & VIR_MIGRATE_PAUSED)) {
2801 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
2802 _("post-copy migration is not supported with non-live "
2803 "or paused migration"));
2804 goto cleanup;
2805 }
2806
2807 if (flags & VIR_MIGRATE_POSTCOPY && flags & VIR_MIGRATE_TUNNELLED) {
2808 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
2809 _("post-copy is not supported with tunnelled migration"));
2810 goto cleanup;
2811 }
2812
2813 if (cfg->migrateTLSForce &&
2814 !(flags & VIR_MIGRATE_TUNNELLED) &&
2815 !(flags & VIR_MIGRATE_TLS)) {
2816 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2817 _("this libvirtd instance allows migration only with VIR_MIGRATE_TLS flag"));
2818 goto cleanup;
2819 }
2820
2821 if (!qemuMigrationSrcIsAllowedHostdev(*def))
2822 goto cleanup;
2823
2824 /* Let migration hook filter domain XML */
2825 if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) {
2826 g_autofree char *xml = NULL;
2827 int hookret;
2828
2829 if (!(xml = qemuDomainDefFormatXML(driver, NULL, *def,
2830 VIR_DOMAIN_XML_SECURE |
2831 VIR_DOMAIN_XML_MIGRATABLE)))
2832 goto cleanup;
2833
2834 hookret = virHookCall(VIR_HOOK_DRIVER_QEMU, (*def)->name,
2835 VIR_HOOK_QEMU_OP_MIGRATE, VIR_HOOK_SUBOP_BEGIN,
2836 NULL, xml, &xmlout);
2837
2838 if (hookret < 0) {
2839 goto cleanup;
2840 } else if (hookret == 0) {
2841 if (virStringIsEmpty(xmlout)) {
2842 VIR_DEBUG("Migrate hook filter returned nothing; using the"
2843 " original XML");
2844 } else {
2845 g_autoptr(virDomainDef) newdef = NULL;
2846
2847 VIR_DEBUG("Using hook-filtered domain XML: %s", xmlout);
2848 newdef = virDomainDefParseString(xmlout, driver->xmlopt, NULL,
2849 VIR_DOMAIN_DEF_PARSE_INACTIVE |
2850 VIR_DOMAIN_DEF_PARSE_SKIP_VALIDATE);
2851 if (!newdef)
2852 goto cleanup;
2853
2854 if (!qemuDomainDefCheckABIStability(driver, NULL, *def, newdef))
2855 goto cleanup;
2856
2857 virDomainDefFree(*def);
2858 *def = g_steal_pointer(&newdef);
2859 /* We should taint the domain here. However, @vm and therefore
2860 * privateData too are still NULL, so just notice the fact and
2861 * taint it later. */
2862 taint_hook = true;
2863 }
2864 }
2865 }
2866
2867 /* Parse cookie earlier than adding the domain onto the
2868 * domain list. Parsing/validation may fail and there's no
2869 * point in having the domain in the list at that point. */
2870 if (!(mig = qemuMigrationCookieParse(driver, *def, origname, NULL,
2871 cookiein, cookieinlen,
2872 QEMU_MIGRATION_COOKIE_LOCKSTATE |
2873 QEMU_MIGRATION_COOKIE_NBD |
2874 QEMU_MIGRATION_COOKIE_MEMORY_HOTPLUG |
2875 QEMU_MIGRATION_COOKIE_CPU_HOTPLUG |
2876 QEMU_MIGRATION_COOKIE_CPU |
2877 QEMU_MIGRATION_COOKIE_CAPS |
2878 QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS)))
2879 goto cleanup;
2880
2881 if (!(vm = virDomainObjListAdd(driver->domains, def,
2882 driver->xmlopt,
2883 VIR_DOMAIN_OBJ_LIST_ADD_LIVE |
2884 VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE,
2885 NULL)))
2886 goto cleanup;
2887
2888 priv = vm->privateData;
2889 jobPriv = priv->job.privateData;
2890 priv->origname = g_strdup(origname);
2891
2892 if (taint_hook) {
2893 /* Domain XML has been altered by a hook script. */
2894 priv->hookRun = true;
2895 }
2896
2897 if (STREQ_NULLABLE(protocol, "rdma") &&
2898 !virMemoryLimitIsSet(vm->def->mem.hard_limit)) {
2899 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2900 _("cannot start RDMA migration with no memory hard "
2901 "limit set"));
2902 goto cleanup;
2903 }
2904
2905 if (qemuMigrationDstPrecreateStorage(vm, mig->nbd,
2906 nmigrate_disks, migrate_disks,
2907 !!(flags & VIR_MIGRATE_NON_SHARED_INC)) < 0)
2908 goto cleanup;
2909
2910 if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN,
2911 flags) < 0)
2912 goto cleanup;
2913 qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PREPARE);
2914
2915 /* Domain starts inactive, even if the domain XML had an id field. */
2916 vm->def->id = -1;
2917
2918 if (flags & VIR_MIGRATE_OFFLINE)
2919 goto done;
2920
2921 if (tunnel &&
2922 virPipe(dataFD) < 0)
2923 goto stopjob;
2924
2925 startFlags = VIR_QEMU_PROCESS_START_AUTODESTROY;
2926
2927 if (qemuProcessInit(driver, vm, mig->cpu, QEMU_ASYNC_JOB_MIGRATION_IN,
2928 true, startFlags) < 0)
2929 goto stopjob;
2930 stopProcess = true;
2931
2932 if (!(incoming = qemuMigrationDstPrepare(vm, tunnel, protocol,
2933 listenAddress, port,
2934 dataFD[0])))
2935 goto stopjob;
2936
2937 if (qemuProcessPrepareDomain(driver, vm, startFlags) < 0)
2938 goto stopjob;
2939
2940 if (qemuProcessPrepareHost(driver, vm, startFlags) < 0)
2941 goto stopjob;
2942
2943 rv = qemuProcessLaunch(dconn, driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN,
2944 incoming, NULL,
2945 VIR_NETDEV_VPORT_PROFILE_OP_MIGRATE_IN_START,
2946 startFlags);
2947 if (rv < 0) {
2948 if (rv == -2)
2949 relabel = true;
2950 goto stopjob;
2951 }
2952 relabel = true;
2953
2954 if (tunnel) {
2955 if (virFDStreamOpen(st, dataFD[1]) < 0) {
2956 virReportSystemError(errno, "%s",
2957 _("cannot pass pipe for tunnelled migration"));
2958 goto stopjob;
2959 }
2960 dataFD[1] = -1; /* 'st' owns the FD now & will close it */
2961 }
2962
2963 if (STREQ_NULLABLE(protocol, "rdma") &&
2964 vm->def->mem.hard_limit > 0 &&
2965 virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {
2966 goto stopjob;
2967 }
2968
2969 if (qemuMigrationDstPrepareAnyBlockDirtyBitmaps(vm, mig, migParams, flags) < 0)
2970 goto stopjob;
2971
2972 if (qemuMigrationParamsCheck(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN,
2973 migParams, mig->caps->automatic) < 0)
2974 goto stopjob;
2975
2976 /* Migrations using TLS need to add the "tls-creds-x509" object and
2977 * set the migration TLS parameters */
2978 if (flags & VIR_MIGRATE_TLS) {
2979 if (qemuMigrationParamsEnableTLS(driver, vm, true,
2980 QEMU_ASYNC_JOB_MIGRATION_IN,
2981 &tlsAlias, NULL,
2982 migParams) < 0)
2983 goto stopjob;
2984 } else {
2985 if (qemuMigrationParamsDisableTLS(vm, migParams) < 0)
2986 goto stopjob;
2987 }
2988
2989 if (qemuMigrationParamsApply(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN,
2990 migParams) < 0)
2991 goto stopjob;
2992
2993 if (mig->nbd &&
2994 flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC) &&
2995 virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_NBD_SERVER)) {
2996 const char *nbdTLSAlias = NULL;
2997
2998 if (flags & VIR_MIGRATE_TLS) {
2999 if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_NBD_TLS)) {
3000 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
3001 _("QEMU NBD server does not support TLS transport"));
3002 goto stopjob;
3003 }
3004
3005 nbdTLSAlias = tlsAlias;
3006 }
3007
3008 if (qemuMigrationDstStartNBDServer(driver, vm, incoming->address,
3009 nmigrate_disks, migrate_disks,
3010 nbdPort, nbdURI,
3011 nbdTLSAlias) < 0) {
3012 goto stopjob;
3013 }
3014 cookieFlags |= QEMU_MIGRATION_COOKIE_NBD;
3015 }
3016
3017 if (mig->lockState) {
3018 VIR_DEBUG("Received lockstate %s", mig->lockState);
3019 VIR_FREE(priv->lockState);
3020 priv->lockState = mig->lockState;
3021 mig->lockState = NULL;
3022 } else {
3023 VIR_DEBUG("Received no lockstate");
3024 }
3025
3026 if (incoming->deferredURI &&
3027 qemuMigrationDstRun(driver, vm, incoming->deferredURI,
3028 QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
3029 goto stopjob;
3030
3031 if (qemuProcessFinishStartup(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN,
3032 false, VIR_DOMAIN_PAUSED_MIGRATION) < 0)
3033 goto stopjob;
3034
3035 done:
3036 if (qemuMigrationCookieFormat(mig, driver, vm,
3037 QEMU_MIGRATION_DESTINATION,
3038 cookieout, cookieoutlen, cookieFlags) < 0) {
3039 /* We could tear down the whole guest here, but
3040 * cookie data is (so far) non-critical, so that
3041 * seems a little harsh. We'll just warn for now.
3042 */
3043 VIR_WARN("Unable to encode migration cookie");
3044 }
3045
3046 if (qemuDomainCleanupAdd(vm, qemuMigrationDstPrepareCleanup) < 0)
3047 goto stopjob;
3048
3049 if (!(flags & VIR_MIGRATE_OFFLINE)) {
3050 virDomainAuditStart(vm, "migrated", true);
3051 event = virDomainEventLifecycleNewFromObj(vm,
3052 VIR_DOMAIN_EVENT_STARTED,
3053 VIR_DOMAIN_EVENT_STARTED_MIGRATED);
3054 }
3055
3056 /* We keep the job active across API calls until the finish() call.
3057 * This prevents any other APIs being invoked while incoming
3058 * migration is taking place.
3059 */
3060 qemuMigrationJobContinue(vm);
3061
3062 if (autoPort)
3063 priv->migrationPort = port;
3064 /* in this case port is not auto selected and we don't need to manage it
3065 * anymore after cookie is baked
3066 */
3067 if (nbdPort != 0)
3068 priv->nbdPort = 0;
3069 ret = 0;
3070
3071 cleanup:
3072 virErrorPreserveLast(&origErr);
3073 qemuProcessIncomingDefFree(incoming);
3074 VIR_FORCE_CLOSE(dataFD[0]);
3075 VIR_FORCE_CLOSE(dataFD[1]);
3076 if (ret < 0 && priv) {
3077 /* priv is set right after vm is added to the list of domains
3078 * and there is no 'goto cleanup;' in the middle of those */
3079 VIR_FREE(priv->origname);
3080 /* release if port is auto selected which is not the case if
3081 * it is given in parameters
3082 */
3083 if (nbdPort == 0)
3084 virPortAllocatorRelease(priv->nbdPort);
3085 priv->nbdPort = 0;
3086 virDomainObjRemoveTransientDef(vm);
3087 qemuDomainRemoveInactiveJob(driver, vm);
3088 }
3089 virDomainObjEndAPI(&vm);
3090 virObjectEventStateQueue(driver->domainEventState, event);
3091 qemuMigrationCookieFree(mig);
3092 virNWFilterUnlockFilterUpdates();
3093 virErrorRestore(&origErr);
3094 return ret;
3095
3096 stopjob:
3097 qemuMigrationParamsReset(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN,
3098 jobPriv->migParams, priv->job.apiFlags);
3099
3100 if (stopProcess) {
3101 unsigned int stopFlags = VIR_QEMU_PROCESS_STOP_MIGRATED;
3102 if (!relabel)
3103 stopFlags |= VIR_QEMU_PROCESS_STOP_NO_RELABEL;
3104 virDomainAuditStart(vm, "migrated", false);
3105 qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED,
3106 QEMU_ASYNC_JOB_MIGRATION_IN, stopFlags);
3107 }
3108
3109 qemuMigrationJobFinish(driver, vm);
3110 goto cleanup;
3111 }
3112
3113
3114 /*
3115 * This version starts an empty VM listening on a localhost TCP port, and
3116 * sets up the corresponding virStream to handle the incoming data.
3117 */
3118 int
qemuMigrationDstPrepareTunnel(virQEMUDriver * driver,virConnectPtr dconn,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,virStreamPtr st,virDomainDef ** def,const char * origname,qemuMigrationParams * migParams,unsigned long flags)3119 qemuMigrationDstPrepareTunnel(virQEMUDriver *driver,
3120 virConnectPtr dconn,
3121 const char *cookiein,
3122 int cookieinlen,
3123 char **cookieout,
3124 int *cookieoutlen,
3125 virStreamPtr st,
3126 virDomainDef **def,
3127 const char *origname,
3128 qemuMigrationParams *migParams,
3129 unsigned long flags)
3130 {
3131 VIR_DEBUG("driver=%p, dconn=%p, cookiein=%s, cookieinlen=%d, "
3132 "cookieout=%p, cookieoutlen=%p, st=%p, def=%p, "
3133 "origname=%s, flags=0x%lx",
3134 driver, dconn, NULLSTR(cookiein), cookieinlen,
3135 cookieout, cookieoutlen, st, *def, origname, flags);
3136
3137 if (st == NULL) {
3138 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
3139 _("tunnelled migration requested but NULL stream passed"));
3140 return -1;
3141 }
3142
3143 return qemuMigrationDstPrepareAny(driver, dconn, cookiein, cookieinlen,
3144 cookieout, cookieoutlen, def, origname,
3145 st, NULL, 0, false, NULL, 0, NULL, 0,
3146 NULL, migParams, flags);
3147 }
3148
3149
3150 static virURI *
qemuMigrationAnyParseURI(const char * uri,bool * wellFormed)3151 qemuMigrationAnyParseURI(const char *uri, bool *wellFormed)
3152 {
3153 char *tmp = NULL;
3154 virURI *parsed;
3155
3156 /* For compatibility reasons tcp://... URIs are sent as tcp:...
3157 * We need to transform them to a well-formed URI before parsing. */
3158 if (STRPREFIX(uri, "tcp:") && !STRPREFIX(uri + 4, "//")) {
3159 tmp = g_strdup_printf("tcp://%s", uri + 4);
3160 uri = tmp;
3161 }
3162
3163 parsed = virURIParse(uri);
3164 if (parsed && wellFormed)
3165 *wellFormed = !tmp;
3166 VIR_FREE(tmp);
3167
3168 return parsed;
3169 }
3170
3171
3172 int
qemuMigrationDstPrepareDirect(virQEMUDriver * driver,virConnectPtr dconn,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,const char * uri_in,char ** uri_out,virDomainDef ** def,const char * origname,const char * listenAddress,size_t nmigrate_disks,const char ** migrate_disks,int nbdPort,const char * nbdURI,qemuMigrationParams * migParams,unsigned long flags)3173 qemuMigrationDstPrepareDirect(virQEMUDriver *driver,
3174 virConnectPtr dconn,
3175 const char *cookiein,
3176 int cookieinlen,
3177 char **cookieout,
3178 int *cookieoutlen,
3179 const char *uri_in,
3180 char **uri_out,
3181 virDomainDef **def,
3182 const char *origname,
3183 const char *listenAddress,
3184 size_t nmigrate_disks,
3185 const char **migrate_disks,
3186 int nbdPort,
3187 const char *nbdURI,
3188 qemuMigrationParams *migParams,
3189 unsigned long flags)
3190 {
3191 unsigned short port = 0;
3192 bool autoPort = true;
3193 g_autofree char *hostname = NULL;
3194 int ret = -1;
3195 g_autoptr(virURI) uri = NULL;
3196 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
3197 const char *migrateHost = cfg->migrateHost;
3198
3199 VIR_DEBUG("driver=%p, dconn=%p, cookiein=%s, cookieinlen=%d, "
3200 "cookieout=%p, cookieoutlen=%p, uri_in=%s, uri_out=%p, "
3201 "def=%p, origname=%s, listenAddress=%s, "
3202 "nmigrate_disks=%zu, migrate_disks=%p, nbdPort=%d, "
3203 "nbdURI=%s, flags=0x%lx",
3204 driver, dconn, NULLSTR(cookiein), cookieinlen,
3205 cookieout, cookieoutlen, NULLSTR(uri_in), uri_out,
3206 *def, origname, NULLSTR(listenAddress),
3207 nmigrate_disks, migrate_disks, nbdPort, NULLSTR(nbdURI),
3208 flags);
3209
3210 *uri_out = NULL;
3211
3212 /* The URI passed in may be NULL or a string "tcp://somehostname:port".
3213 *
3214 * If the URI passed in is NULL then we allocate a port number
3215 * from our pool of port numbers, and if the migrateHost is configured,
3216 * we return a URI of "tcp://migrateHost:port", otherwise return a URI
3217 * of "tcp://ourhostname:port".
3218 *
3219 * If the URI passed in is not NULL then we try to parse out the
3220 * port number and use that (note that the hostname is assumed
3221 * to be a correct hostname which refers to the target machine).
3222 */
3223 if (uri_in == NULL) {
3224 bool encloseAddress = false;
3225 const char *incFormat;
3226
3227 if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0)
3228 goto cleanup;
3229
3230 if (migrateHost != NULL) {
3231 if (virSocketAddrNumericFamily(migrateHost) == AF_INET6)
3232 encloseAddress = true;
3233
3234 hostname = g_strdup(migrateHost);
3235 } else {
3236 if ((hostname = virGetHostname()) == NULL)
3237 goto cleanup;
3238 }
3239
3240 if (STRPREFIX(hostname, "localhost")) {
3241 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
3242 _("hostname on destination resolved to localhost,"
3243 " but migration requires an FQDN"));
3244 goto cleanup;
3245 }
3246
3247 /* XXX this really should have been a properly well-formed
3248 * URI, but we can't add in tcp:// now without breaking
3249 * compatibility with old targets. We at least make the
3250 * new targets accept both syntaxes though.
3251 */
3252 if (encloseAddress)
3253 incFormat = "%s:[%s]:%d";
3254 else
3255 incFormat = "%s:%s:%d";
3256
3257 *uri_out = g_strdup_printf(incFormat, "tcp", hostname, port);
3258 } else {
3259 bool well_formed_uri = false;
3260
3261 if (!(uri = qemuMigrationAnyParseURI(uri_in, &well_formed_uri)))
3262 goto cleanup;
3263
3264 if (uri->scheme == NULL) {
3265 virReportError(VIR_ERR_INVALID_ARG,
3266 _("missing scheme in migration URI: %s"),
3267 uri_in);
3268 goto cleanup;
3269 }
3270
3271 if (STRNEQ(uri->scheme, "tcp") &&
3272 STRNEQ(uri->scheme, "rdma") &&
3273 STRNEQ(uri->scheme, "unix")) {
3274 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED,
3275 _("unsupported scheme %s in migration URI %s"),
3276 uri->scheme, uri_in);
3277 goto cleanup;
3278 }
3279
3280 if (STREQ(uri->scheme, "unix")) {
3281 autoPort = false;
3282 listenAddress = uri->path;
3283 } else {
3284 if (uri->server == NULL) {
3285 virReportError(VIR_ERR_INVALID_ARG, _("missing host in migration"
3286 " URI: %s"), uri_in);
3287 goto cleanup;
3288 }
3289
3290 if (uri->port == 0) {
3291 if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0)
3292 goto cleanup;
3293
3294 /* Send well-formed URI only if uri_in was well-formed */
3295 if (well_formed_uri) {
3296 uri->port = port;
3297 if (!(*uri_out = virURIFormat(uri)))
3298 goto cleanup;
3299 } else {
3300 *uri_out = g_strdup_printf("%s:%d", uri_in, port);
3301 }
3302 } else {
3303 port = uri->port;
3304 autoPort = false;
3305 }
3306 }
3307 }
3308
3309 if (*uri_out)
3310 VIR_DEBUG("Generated uri_out=%s", *uri_out);
3311
3312 ret = qemuMigrationDstPrepareAny(driver, dconn, cookiein, cookieinlen,
3313 cookieout, cookieoutlen, def, origname,
3314 NULL, uri ? uri->scheme : "tcp",
3315 port, autoPort, listenAddress,
3316 nmigrate_disks, migrate_disks, nbdPort,
3317 nbdURI, migParams, flags);
3318 cleanup:
3319 if (ret != 0) {
3320 VIR_FREE(*uri_out);
3321 if (autoPort)
3322 virPortAllocatorRelease(port);
3323 }
3324 return ret;
3325 }
3326
3327
3328 virDomainDef *
qemuMigrationAnyPrepareDef(virQEMUDriver * driver,virQEMUCaps * qemuCaps,const char * dom_xml,const char * dname,char ** origname)3329 qemuMigrationAnyPrepareDef(virQEMUDriver *driver,
3330 virQEMUCaps *qemuCaps,
3331 const char *dom_xml,
3332 const char *dname,
3333 char **origname)
3334 {
3335 virDomainDef *def;
3336 char *name = NULL;
3337
3338 if (!dom_xml) {
3339 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
3340 _("no domain XML passed"));
3341 return NULL;
3342 }
3343
3344 if (!(def = virDomainDefParseString(dom_xml, driver->xmlopt,
3345 qemuCaps,
3346 VIR_DOMAIN_DEF_PARSE_INACTIVE |
3347 VIR_DOMAIN_DEF_PARSE_SKIP_VALIDATE)))
3348 goto cleanup;
3349
3350 if (dname) {
3351 name = def->name;
3352 def->name = g_strdup(dname);
3353 }
3354
3355 cleanup:
3356 if (def && origname)
3357 *origname = name;
3358 else
3359 VIR_FREE(name);
3360 return def;
3361 }
3362
3363
3364 static int
qemuMigrationSrcConfirmPhase(virQEMUDriver * driver,virDomainObj * vm,const char * cookiein,int cookieinlen,unsigned int flags,int retcode)3365 qemuMigrationSrcConfirmPhase(virQEMUDriver *driver,
3366 virDomainObj *vm,
3367 const char *cookiein,
3368 int cookieinlen,
3369 unsigned int flags,
3370 int retcode)
3371 {
3372 g_autoptr(qemuMigrationCookie) mig = NULL;
3373 virObjectEvent *event;
3374 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
3375 qemuDomainObjPrivate *priv = vm->privateData;
3376 qemuDomainJobPrivate *jobPriv = priv->job.privateData;
3377 qemuDomainJobInfo *jobInfo = NULL;
3378
3379 VIR_DEBUG("driver=%p, vm=%p, cookiein=%s, cookieinlen=%d, "
3380 "flags=0x%x, retcode=%d",
3381 driver, vm, NULLSTR(cookiein), cookieinlen,
3382 flags, retcode);
3383
3384 virCheckFlags(QEMU_MIGRATION_FLAGS, -1);
3385
3386 qemuMigrationJobSetPhase(driver, vm,
3387 retcode == 0
3388 ? QEMU_MIGRATION_PHASE_CONFIRM3
3389 : QEMU_MIGRATION_PHASE_CONFIRM3_CANCELLED);
3390
3391 if (!(mig = qemuMigrationCookieParse(driver, vm->def, priv->origname, priv,
3392 cookiein, cookieinlen,
3393 QEMU_MIGRATION_COOKIE_STATS)))
3394 return -1;
3395
3396 if (retcode == 0)
3397 jobInfo = priv->job.completed;
3398 else
3399 g_clear_pointer(&priv->job.completed, qemuDomainJobInfoFree);
3400
3401 /* Update times with the values sent by the destination daemon */
3402 if (mig->jobInfo && jobInfo) {
3403 int reason;
3404
3405 /* We need to refresh migration statistics after a completed post-copy
3406 * migration since priv->job.completed contains obsolete data from the
3407 * time we switched to post-copy mode.
3408 */
3409 if (virDomainObjGetState(vm, &reason) == VIR_DOMAIN_PAUSED &&
3410 reason == VIR_DOMAIN_PAUSED_POSTCOPY &&
3411 qemuMigrationAnyFetchStats(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
3412 jobInfo, NULL) < 0)
3413 VIR_WARN("Could not refresh migration statistics");
3414
3415 qemuDomainJobInfoUpdateTime(jobInfo);
3416 jobInfo->timeDeltaSet = mig->jobInfo->timeDeltaSet;
3417 jobInfo->timeDelta = mig->jobInfo->timeDelta;
3418 jobInfo->stats.mig.downtime_set = mig->jobInfo->stats.mig.downtime_set;
3419 jobInfo->stats.mig.downtime = mig->jobInfo->stats.mig.downtime;
3420 }
3421
3422 if (flags & VIR_MIGRATE_OFFLINE)
3423 return 0;
3424
3425 /* Did the migration go as planned? If yes, kill off the domain object.
3426 * If something failed, resume CPUs, but only if we didn't use post-copy.
3427 */
3428 if (retcode == 0) {
3429 /* If guest uses SPICE and supports seamless migration we have to hold
3430 * up domain shutdown until SPICE server transfers its data */
3431 qemuMigrationSrcWaitForSpice(vm);
3432
3433 qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_MIGRATED,
3434 QEMU_ASYNC_JOB_MIGRATION_OUT,
3435 VIR_QEMU_PROCESS_STOP_MIGRATED);
3436 virDomainAuditStop(vm, "migrated");
3437
3438 event = virDomainEventLifecycleNewFromObj(vm,
3439 VIR_DOMAIN_EVENT_STOPPED,
3440 VIR_DOMAIN_EVENT_STOPPED_MIGRATED);
3441 virObjectEventStateQueue(driver->domainEventState, event);
3442 qemuDomainEventEmitJobCompleted(driver, vm);
3443 } else {
3444 virErrorPtr orig_err;
3445 int reason;
3446
3447 virErrorPreserveLast(&orig_err);
3448
3449 /* cancel any outstanding NBD jobs */
3450 qemuMigrationSrcNBDCopyCancel(driver, vm, false,
3451 QEMU_ASYNC_JOB_MIGRATION_OUT, NULL);
3452
3453 virErrorRestore(&orig_err);
3454
3455 if (virDomainObjGetState(vm, &reason) == VIR_DOMAIN_PAUSED &&
3456 reason == VIR_DOMAIN_PAUSED_POSTCOPY)
3457 qemuMigrationAnyPostcopyFailed(driver, vm);
3458 else
3459 qemuMigrationSrcRestoreDomainState(driver, vm);
3460
3461 qemuMigrationParamsReset(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
3462 jobPriv->migParams, priv->job.apiFlags);
3463
3464 if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0)
3465 VIR_WARN("Failed to save status on vm %s", vm->def->name);
3466 }
3467
3468 return 0;
3469 }
3470
3471 int
qemuMigrationSrcConfirm(virQEMUDriver * driver,virDomainObj * vm,const char * cookiein,int cookieinlen,unsigned int flags,int cancelled)3472 qemuMigrationSrcConfirm(virQEMUDriver *driver,
3473 virDomainObj *vm,
3474 const char *cookiein,
3475 int cookieinlen,
3476 unsigned int flags,
3477 int cancelled)
3478 {
3479 qemuMigrationJobPhase phase;
3480 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
3481 int ret = -1;
3482
3483 if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT))
3484 goto cleanup;
3485
3486 if (cancelled)
3487 phase = QEMU_MIGRATION_PHASE_CONFIRM3_CANCELLED;
3488 else
3489 phase = QEMU_MIGRATION_PHASE_CONFIRM3;
3490
3491 qemuMigrationJobStartPhase(driver, vm, phase);
3492 virCloseCallbacksUnset(driver->closeCallbacks, vm,
3493 qemuMigrationSrcCleanup);
3494
3495 ret = qemuMigrationSrcConfirmPhase(driver, vm,
3496 cookiein, cookieinlen,
3497 flags, cancelled);
3498
3499 qemuMigrationJobFinish(driver, vm);
3500 if (!virDomainObjIsActive(vm)) {
3501 if (!cancelled && ret == 0 && flags & VIR_MIGRATE_UNDEFINE_SOURCE) {
3502 virDomainDeleteConfig(cfg->configDir, cfg->autostartDir, vm);
3503 vm->persistent = 0;
3504 }
3505 qemuDomainRemoveInactiveJob(driver, vm);
3506 }
3507
3508 cleanup:
3509 virDomainObjEndAPI(&vm);
3510 return ret;
3511 }
3512
3513
3514 enum qemuMigrationDestinationType {
3515 MIGRATION_DEST_HOST,
3516 MIGRATION_DEST_CONNECT_HOST,
3517 MIGRATION_DEST_SOCKET,
3518 MIGRATION_DEST_CONNECT_SOCKET,
3519 MIGRATION_DEST_FD,
3520 };
3521
3522 enum qemuMigrationForwardType {
3523 MIGRATION_FWD_DIRECT,
3524 MIGRATION_FWD_STREAM,
3525 };
3526
3527 typedef struct _qemuMigrationSpec qemuMigrationSpec;
3528 struct _qemuMigrationSpec {
3529 enum qemuMigrationDestinationType destType;
3530 union {
3531 struct {
3532 const char *protocol;
3533 const char *name;
3534 int port;
3535 } host;
3536
3537 struct {
3538 const char *path;
3539 } socket;
3540
3541 struct {
3542 int qemu;
3543 int local;
3544 } fd;
3545 } dest;
3546
3547 enum qemuMigrationForwardType fwdType;
3548 union {
3549 virStreamPtr stream;
3550 } fwd;
3551 };
3552
3553 #define TUNNEL_SEND_BUF_SIZE 65536
3554
3555 typedef struct _qemuMigrationIOThread qemuMigrationIOThread;
3556 struct _qemuMigrationIOThread {
3557 virThread thread;
3558 virStreamPtr st;
3559 int sock;
3560 virError err;
3561 int wakeupRecvFD;
3562 int wakeupSendFD;
3563 };
3564
qemuMigrationSrcIOFunc(void * arg)3565 static void qemuMigrationSrcIOFunc(void *arg)
3566 {
3567 qemuMigrationIOThread *data = arg;
3568 char *buffer = NULL;
3569 struct pollfd fds[2];
3570 int timeout = -1;
3571 virErrorPtr err = NULL;
3572
3573 VIR_DEBUG("Running migration tunnel; stream=%p, sock=%d",
3574 data->st, data->sock);
3575
3576 buffer = g_new0(char, TUNNEL_SEND_BUF_SIZE);
3577
3578 fds[0].fd = data->sock;
3579 fds[1].fd = data->wakeupRecvFD;
3580
3581 for (;;) {
3582 int ret;
3583
3584 fds[0].events = fds[1].events = POLLIN;
3585 fds[0].revents = fds[1].revents = 0;
3586
3587 ret = poll(fds, G_N_ELEMENTS(fds), timeout);
3588
3589 if (ret < 0) {
3590 if (errno == EAGAIN || errno == EINTR)
3591 continue;
3592 virReportSystemError(errno, "%s",
3593 _("poll failed in migration tunnel"));
3594 goto abrt;
3595 }
3596
3597 if (ret == 0) {
3598 /* We were asked to gracefully stop but reading would block. This
3599 * can only happen if qemu told us migration finished but didn't
3600 * close the migration fd. We handle this in the same way as EOF.
3601 */
3602 VIR_DEBUG("QEMU forgot to close migration fd");
3603 break;
3604 }
3605
3606 if (fds[1].revents & (POLLIN | POLLERR | POLLHUP)) {
3607 char stop = 0;
3608
3609 if (saferead(data->wakeupRecvFD, &stop, 1) != 1) {
3610 virReportSystemError(errno, "%s",
3611 _("failed to read from wakeup fd"));
3612 goto abrt;
3613 }
3614
3615 VIR_DEBUG("Migration tunnel was asked to %s",
3616 stop ? "abort" : "finish");
3617 if (stop) {
3618 goto abrt;
3619 } else {
3620 timeout = 0;
3621 }
3622 }
3623
3624 if (fds[0].revents & (POLLIN | POLLERR | POLLHUP)) {
3625 int nbytes;
3626
3627 nbytes = saferead(data->sock, buffer, TUNNEL_SEND_BUF_SIZE);
3628 if (nbytes > 0) {
3629 if (virStreamSend(data->st, buffer, nbytes) < 0)
3630 goto error;
3631 } else if (nbytes < 0) {
3632 virReportSystemError(errno, "%s",
3633 _("tunnelled migration failed to read from qemu"));
3634 goto abrt;
3635 } else {
3636 /* EOF; get out of here */
3637 break;
3638 }
3639 }
3640 }
3641
3642 if (virStreamFinish(data->st) < 0)
3643 goto error;
3644
3645 VIR_FORCE_CLOSE(data->sock);
3646 VIR_FREE(buffer);
3647
3648 return;
3649
3650 abrt:
3651 virErrorPreserveLast(&err);
3652 if (err && err->code == VIR_ERR_OK) {
3653 virFreeError(err);
3654 err = NULL;
3655 }
3656 virStreamAbort(data->st);
3657 virErrorRestore(&err);
3658
3659 error:
3660 /* Let the source qemu know that the transfer can't continue anymore.
3661 * Don't copy the error for EPIPE as destination has the actual error. */
3662 VIR_FORCE_CLOSE(data->sock);
3663 if (!virLastErrorIsSystemErrno(EPIPE))
3664 virCopyLastError(&data->err);
3665 virResetLastError();
3666 VIR_FREE(buffer);
3667 }
3668
3669
3670 static qemuMigrationIOThread *
qemuMigrationSrcStartTunnel(virStreamPtr st,int sock)3671 qemuMigrationSrcStartTunnel(virStreamPtr st,
3672 int sock)
3673 {
3674 qemuMigrationIOThread *io = NULL;
3675 int wakeupFD[2] = { -1, -1 };
3676
3677 if (virPipe(wakeupFD) < 0)
3678 goto error;
3679
3680 io = g_new0(qemuMigrationIOThread, 1);
3681
3682 io->st = st;
3683 io->sock = sock;
3684 io->wakeupRecvFD = wakeupFD[0];
3685 io->wakeupSendFD = wakeupFD[1];
3686
3687 if (virThreadCreateFull(&io->thread, true,
3688 qemuMigrationSrcIOFunc,
3689 "qemu-mig-tunnel",
3690 false,
3691 io) < 0) {
3692 virReportSystemError(errno, "%s",
3693 _("Unable to create migration thread"));
3694 goto error;
3695 }
3696
3697 return io;
3698
3699 error:
3700 VIR_FORCE_CLOSE(wakeupFD[0]);
3701 VIR_FORCE_CLOSE(wakeupFD[1]);
3702 VIR_FREE(io);
3703 return NULL;
3704 }
3705
3706 static int
qemuMigrationSrcStopTunnel(qemuMigrationIOThread * io,bool error)3707 qemuMigrationSrcStopTunnel(qemuMigrationIOThread *io, bool error)
3708 {
3709 int rv = -1;
3710 char stop = error ? 1 : 0;
3711
3712 /* make sure the thread finishes its job and is joinable */
3713 if (safewrite(io->wakeupSendFD, &stop, 1) != 1) {
3714 virReportSystemError(errno, "%s",
3715 _("failed to wakeup migration tunnel"));
3716 goto cleanup;
3717 }
3718
3719 virThreadJoin(&io->thread);
3720
3721 /* Forward error from the IO thread, to this thread */
3722 if (io->err.code != VIR_ERR_OK) {
3723 if (error)
3724 rv = 0;
3725 else
3726 virSetError(&io->err);
3727 virResetError(&io->err);
3728 goto cleanup;
3729 }
3730
3731 rv = 0;
3732
3733 cleanup:
3734 VIR_FORCE_CLOSE(io->wakeupSendFD);
3735 VIR_FORCE_CLOSE(io->wakeupRecvFD);
3736 VIR_FREE(io);
3737 return rv;
3738 }
3739
3740 static int
qemuMigrationSrcConnect(virQEMUDriver * driver,virDomainObj * vm,qemuMigrationSpec * spec)3741 qemuMigrationSrcConnect(virQEMUDriver *driver,
3742 virDomainObj *vm,
3743 qemuMigrationSpec *spec)
3744 {
3745 virNetSocket *sock;
3746 g_autofree char *port = NULL;
3747 int fd_qemu = -1;
3748 int ret = -1;
3749
3750 if (qemuSecuritySetSocketLabel(driver->securityManager, vm->def) < 0)
3751 goto cleanup;
3752
3753 switch (spec->destType) {
3754 case MIGRATION_DEST_CONNECT_HOST:
3755 port = g_strdup_printf("%d", spec->dest.host.port);
3756 if (virNetSocketNewConnectTCP(spec->dest.host.name,
3757 port,
3758 AF_UNSPEC,
3759 &sock) == 0) {
3760 fd_qemu = virNetSocketDupFD(sock, true);
3761 virObjectUnref(sock);
3762 }
3763 break;
3764 case MIGRATION_DEST_CONNECT_SOCKET:
3765 if (virNetSocketNewConnectUNIX(spec->dest.socket.path,
3766 NULL, &sock) == 0) {
3767 fd_qemu = virNetSocketDupFD(sock, true);
3768 virObjectUnref(sock);
3769 }
3770 break;
3771 case MIGRATION_DEST_HOST:
3772 case MIGRATION_DEST_SOCKET:
3773 case MIGRATION_DEST_FD:
3774 break;
3775 }
3776
3777 spec->destType = MIGRATION_DEST_FD;
3778 spec->dest.fd.qemu = fd_qemu;
3779
3780 if (qemuSecurityClearSocketLabel(driver->securityManager, vm->def) < 0 ||
3781 spec->dest.fd.qemu == -1)
3782 goto cleanup;
3783
3784 /* Migration expects a blocking FD */
3785 if (virSetBlocking(spec->dest.fd.qemu, true) < 0) {
3786 virReportSystemError(errno, _("Unable to set FD %d blocking"),
3787 spec->dest.fd.qemu);
3788 goto cleanup;
3789 }
3790
3791 ret = 0;
3792
3793 cleanup:
3794 if (ret < 0)
3795 VIR_FORCE_CLOSE(spec->dest.fd.qemu);
3796 return ret;
3797 }
3798
3799
3800 static int
qemuMigrationSrcContinue(virQEMUDriver * driver,virDomainObj * vm,qemuMonitorMigrationStatus status,qemuDomainAsyncJob asyncJob)3801 qemuMigrationSrcContinue(virQEMUDriver *driver,
3802 virDomainObj *vm,
3803 qemuMonitorMigrationStatus status,
3804 qemuDomainAsyncJob asyncJob)
3805 {
3806 qemuDomainObjPrivate *priv = vm->privateData;
3807 int ret;
3808
3809 if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
3810 return -1;
3811
3812 ret = qemuMonitorMigrateContinue(priv->mon, status);
3813
3814 if (qemuDomainObjExitMonitor(driver, vm) < 0)
3815 ret = -1;
3816
3817 return ret;
3818 }
3819
3820
3821 static int
qemuMigrationSetDBusVMState(virQEMUDriver * driver,virDomainObj * vm)3822 qemuMigrationSetDBusVMState(virQEMUDriver *driver,
3823 virDomainObj *vm)
3824 {
3825 qemuDomainObjPrivate *priv = vm->privateData;
3826
3827 if (priv->dbusVMStateIds) {
3828 int rv;
3829
3830 if (qemuHotplugAttachDBusVMState(driver, vm, QEMU_ASYNC_JOB_NONE) < 0)
3831 return -1;
3832
3833 if (qemuDomainObjEnterMonitorAsync(driver, vm, QEMU_ASYNC_JOB_NONE) < 0)
3834 return -1;
3835
3836 rv = qemuMonitorSetDBusVMStateIdList(priv->mon, priv->dbusVMStateIds);
3837
3838 if (qemuDomainObjExitMonitor(driver, vm) < 0)
3839 rv = -1;
3840
3841 return rv;
3842 } else {
3843 if (qemuHotplugRemoveDBusVMState(driver, vm, QEMU_ASYNC_JOB_NONE) < 0)
3844 return -1;
3845 }
3846
3847 return 0;
3848 }
3849
3850
3851 /**
3852 * qemuMigrationSrcRunPrepareBlockDirtyBitmapsMerge:
3853 * @vm: domain object
3854 * @mig: migration cookie
3855 *
3856 * When migrating full disks, which means that the backing chain of the disk
3857 * will be squashed into a single image we need to calculate bitmaps
3858 * corresponding to the checkpoints which express the same set of changes
3859 * for migration.
3860 *
3861 * This function prepares temporary bitmaps and corresponding merges, updates
3862 * the data so that the temporary bitmaps are used and registers the temporary
3863 * bitmaps for deletion on failed migration.
3864 */
3865 static int
qemuMigrationSrcRunPrepareBlockDirtyBitmapsMerge(virDomainObj * vm,qemuMigrationCookie * mig)3866 qemuMigrationSrcRunPrepareBlockDirtyBitmapsMerge(virDomainObj *vm,
3867 qemuMigrationCookie *mig)
3868 {
3869 g_autoslist(qemuDomainJobPrivateMigrateTempBitmap) tmpbitmaps = NULL;
3870 qemuDomainObjPrivate *priv = vm->privateData;
3871 qemuDomainJobPrivate *jobPriv = priv->job.privateData;
3872 virQEMUDriver *driver = priv->driver;
3873 g_autoptr(virJSONValue) actions = virJSONValueNewArray();
3874 g_autoptr(GHashTable) blockNamedNodeData = NULL;
3875 GSList *nextdisk;
3876 int rc;
3877
3878 if (!(blockNamedNodeData = qemuBlockGetNamedNodeData(vm, QEMU_ASYNC_JOB_MIGRATION_OUT)))
3879 return -1;
3880
3881 for (nextdisk = mig->blockDirtyBitmaps; nextdisk; nextdisk = nextdisk->next) {
3882 qemuMigrationBlockDirtyBitmapsDisk *disk = nextdisk->data;
3883 GSList *nextbitmap;
3884
3885 /* if a disk doesn't have a backing chain we don't need the code below */
3886 if (!virStorageSourceHasBacking(disk->disk->src))
3887 continue;
3888
3889 for (nextbitmap = disk->bitmaps; nextbitmap; nextbitmap = nextbitmap->next) {
3890 qemuMigrationBlockDirtyBitmapsDiskBitmap *bitmap = nextbitmap->data;
3891 qemuDomainJobPrivateMigrateTempBitmap *tmpbmp;
3892 virStorageSource *n;
3893 unsigned long long granularity = 0;
3894 g_autoptr(virJSONValue) merge = virJSONValueNewArray();
3895
3896 for (n = disk->disk->src; virStorageSourceIsBacking(n); n = n->backingStore) {
3897 qemuBlockNamedNodeDataBitmap *b;
3898
3899 if (!(b = qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData, n,
3900 bitmap->bitmapname)))
3901 break;
3902
3903 if (granularity == 0)
3904 granularity = b->granularity;
3905
3906 if (qemuMonitorTransactionBitmapMergeSourceAddBitmap(merge,
3907 n->nodeformat,
3908 b->name) < 0)
3909 return -1;
3910 }
3911
3912 bitmap->sourcebitmap = g_strdup_printf("libvirt-migration-%s", bitmap->alias);
3913 bitmap->persistent = VIR_TRISTATE_BOOL_YES;
3914
3915 if (qemuMonitorTransactionBitmapAdd(actions,
3916 disk->disk->src->nodeformat,
3917 bitmap->sourcebitmap,
3918 false, false, granularity) < 0)
3919 return -1;
3920
3921 if (qemuMonitorTransactionBitmapMerge(actions,
3922 disk->disk->src->nodeformat,
3923 bitmap->sourcebitmap,
3924 &merge) < 0)
3925 return -1;
3926
3927 tmpbmp = g_new0(qemuDomainJobPrivateMigrateTempBitmap, 1);
3928 tmpbmp->nodename = g_strdup(disk->disk->src->nodeformat);
3929 tmpbmp->bitmapname = g_strdup(bitmap->sourcebitmap);
3930 tmpbitmaps = g_slist_prepend(tmpbitmaps, tmpbmp);
3931 }
3932 }
3933
3934 if (qemuDomainObjEnterMonitorAsync(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
3935 return -1;
3936
3937 rc = qemuMonitorTransaction(priv->mon, &actions);
3938
3939 if (qemuDomainObjExitMonitor(driver, vm) < 0 || rc < 0)
3940 return -1;
3941
3942 jobPriv->migTempBitmaps = g_steal_pointer(&tmpbitmaps);
3943
3944 return 0;
3945 }
3946
3947
3948 /**
3949 * qemuMigrationSrcRunPrepareBlockDirtyBitmaps:
3950 * @vm: domain object
3951 * @mig: migration cookie
3952 * @migParams: migration parameters
3953 * @flags: migration flags
3954 *
3955 * Configures the source for bitmap migration when the destination asks
3956 * for bitmaps.
3957 */
3958 static int
qemuMigrationSrcRunPrepareBlockDirtyBitmaps(virDomainObj * vm,qemuMigrationCookie * mig,qemuMigrationParams * migParams,unsigned int flags)3959 qemuMigrationSrcRunPrepareBlockDirtyBitmaps(virDomainObj *vm,
3960 qemuMigrationCookie *mig,
3961 qemuMigrationParams *migParams,
3962 unsigned int flags)
3963
3964 {
3965 g_autoptr(virJSONValue) mapping = NULL;
3966
3967 if (!mig->blockDirtyBitmaps)
3968 return 0;
3969
3970 if (qemuMigrationCookieBlockDirtyBitmapsMatchDisks(vm->def, mig->blockDirtyBitmaps) < 0)
3971 return -1;
3972
3973 /* For VIR_MIGRATE_NON_SHARED_INC we can migrate the bitmaps directly,
3974 * otherwise we must create merged bitmaps from the whole chain */
3975
3976 if (!(flags & VIR_MIGRATE_NON_SHARED_INC) &&
3977 qemuMigrationSrcRunPrepareBlockDirtyBitmapsMerge(vm, mig) < 0)
3978 return -1;
3979
3980 if (qemuMigrationCookieBlockDirtyBitmapsToParams(mig->blockDirtyBitmaps,
3981 &mapping) < 0)
3982 return -1;
3983
3984 qemuMigrationParamsSetBlockDirtyBitmapMapping(migParams, &mapping);
3985 return 0;
3986 }
3987
3988
3989 static int
qemuMigrationSrcRun(virQEMUDriver * driver,virDomainObj * vm,const char * persist_xml,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,unsigned long flags,unsigned long resource,qemuMigrationSpec * spec,virConnectPtr dconn,const char * graphicsuri,size_t nmigrate_disks,const char ** migrate_disks,qemuMigrationParams * migParams,const char * nbdURI)3990 qemuMigrationSrcRun(virQEMUDriver *driver,
3991 virDomainObj *vm,
3992 const char *persist_xml,
3993 const char *cookiein,
3994 int cookieinlen,
3995 char **cookieout,
3996 int *cookieoutlen,
3997 unsigned long flags,
3998 unsigned long resource,
3999 qemuMigrationSpec *spec,
4000 virConnectPtr dconn,
4001 const char *graphicsuri,
4002 size_t nmigrate_disks,
4003 const char **migrate_disks,
4004 qemuMigrationParams *migParams,
4005 const char *nbdURI)
4006 {
4007 int ret = -1;
4008 unsigned int migrate_flags = QEMU_MONITOR_MIGRATE_BACKGROUND;
4009 qemuDomainObjPrivate *priv = vm->privateData;
4010 g_autoptr(qemuMigrationCookie) mig = NULL;
4011 g_autofree char *tlsAlias = NULL;
4012 qemuMigrationIOThread *iothread = NULL;
4013 VIR_AUTOCLOSE fd = -1;
4014 unsigned long restore_max_bandwidth = priv->migMaxBandwidth;
4015 virErrorPtr orig_err = NULL;
4016 unsigned int cookieFlags = 0;
4017 bool abort_on_error = !!(flags & VIR_MIGRATE_ABORT_ON_ERROR);
4018 bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT);
4019 bool bwParam = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_PARAM_BANDWIDTH);
4020 bool storageMigration = flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC);
4021 bool cancel = false;
4022 unsigned int waitFlags;
4023 g_autoptr(virDomainDef) persistDef = NULL;
4024 g_autofree char *timestamp = NULL;
4025 int rc;
4026
4027 if (resource > 0)
4028 priv->migMaxBandwidth = resource;
4029
4030 VIR_DEBUG("driver=%p, vm=%p, cookiein=%s, cookieinlen=%d, "
4031 "cookieout=%p, cookieoutlen=%p, flags=0x%lx, resource=%lu, "
4032 "spec=%p (dest=%d, fwd=%d), dconn=%p, graphicsuri=%s, "
4033 "nmigrate_disks=%zu, migrate_disks=%p",
4034 driver, vm, NULLSTR(cookiein), cookieinlen,
4035 cookieout, cookieoutlen, flags, resource,
4036 spec, spec->destType, spec->fwdType, dconn,
4037 NULLSTR(graphicsuri), nmigrate_disks, migrate_disks);
4038
4039 if (storageMigration)
4040 storageMigration = qemuMigrationHasAnyStorageMigrationDisks(vm->def,
4041 migrate_disks,
4042 nmigrate_disks);
4043
4044 if (storageMigration) {
4045 cookieFlags |= QEMU_MIGRATION_COOKIE_NBD;
4046
4047 if (virQEMUCapsGet(priv->qemuCaps,
4048 QEMU_CAPS_MIGRATION_PARAM_BLOCK_BITMAP_MAPPING))
4049 cookieFlags |= QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS;
4050 }
4051
4052 if (virLockManagerPluginUsesState(driver->lockManager) &&
4053 !cookieout) {
4054 virReportError(VIR_ERR_INTERNAL_ERROR,
4055 _("Migration with lock driver %s requires"
4056 " cookie support"),
4057 virLockManagerPluginGetName(driver->lockManager));
4058 return -1;
4059 }
4060
4061 if (events)
4062 priv->signalIOError = abort_on_error;
4063
4064 if (flags & VIR_MIGRATE_PERSIST_DEST) {
4065 if (persist_xml) {
4066 if (!(persistDef = qemuMigrationAnyPrepareDef(driver,
4067 priv->qemuCaps,
4068 persist_xml,
4069 NULL, NULL)))
4070 goto error;
4071 } else {
4072 virDomainDef *def = vm->newDef ? vm->newDef : vm->def;
4073 if (!(persistDef = qemuDomainDefCopy(driver, priv->qemuCaps, def,
4074 VIR_DOMAIN_XML_SECURE |
4075 VIR_DOMAIN_XML_MIGRATABLE)))
4076 goto error;
4077 }
4078 }
4079
4080 mig = qemuMigrationCookieParse(driver, vm->def, priv->origname, priv,
4081 cookiein, cookieinlen,
4082 cookieFlags |
4083 QEMU_MIGRATION_COOKIE_GRAPHICS |
4084 QEMU_MIGRATION_COOKIE_CAPS |
4085 QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS);
4086 if (!mig)
4087 goto error;
4088
4089 if (qemuMigrationSrcGraphicsRelocate(driver, vm, mig, graphicsuri) < 0)
4090 VIR_WARN("unable to provide data for graphics client relocation");
4091
4092 if (mig->blockDirtyBitmaps &&
4093 qemuMigrationSrcRunPrepareBlockDirtyBitmaps(vm, mig, migParams, flags) < 0)
4094 goto error;
4095
4096 if (qemuMigrationParamsCheck(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
4097 migParams, mig->caps->automatic) < 0)
4098 goto error;
4099
4100 if (flags & VIR_MIGRATE_TLS) {
4101 const char *hostname = NULL;
4102
4103 /* We need to add tls-hostname whenever QEMU itself does not
4104 * connect directly to the destination. */
4105 if (spec->destType == MIGRATION_DEST_CONNECT_HOST ||
4106 spec->destType == MIGRATION_DEST_FD)
4107 hostname = spec->dest.host.name;
4108
4109 if (qemuMigrationParamsEnableTLS(driver, vm, false,
4110 QEMU_ASYNC_JOB_MIGRATION_OUT,
4111 &tlsAlias, hostname,
4112 migParams) < 0)
4113 goto error;
4114 } else {
4115 if (qemuMigrationParamsDisableTLS(vm, migParams) < 0)
4116 goto error;
4117 }
4118
4119 if (bwParam &&
4120 qemuMigrationParamsSetULL(migParams, QEMU_MIGRATION_PARAM_MAX_BANDWIDTH,
4121 priv->migMaxBandwidth * 1024 * 1024) < 0)
4122 goto error;
4123
4124 if (qemuMigrationParamsApply(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
4125 migParams) < 0)
4126 goto error;
4127
4128
4129 if (storageMigration) {
4130 if (mig->nbd) {
4131 const char *host = "";
4132
4133 if (spec->destType == MIGRATION_DEST_HOST ||
4134 spec->destType == MIGRATION_DEST_CONNECT_HOST) {
4135 host = spec->dest.host.name;
4136 }
4137
4138 /* Currently libvirt does not support setting up of the NBD
4139 * non-shared storage migration with TLS. As we need to honour the
4140 * VIR_MIGRATE_TLS flag, we need to reject such migration until
4141 * we implement TLS for NBD. */
4142 if (flags & VIR_MIGRATE_TLS &&
4143 !virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_BLOCKDEV_DEL)) {
4144 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
4145 _("NBD migration with TLS is not supported"));
4146 goto error;
4147 }
4148
4149 if (qemuMigrationSrcNBDStorageCopy(driver, vm, mig,
4150 host,
4151 priv->migMaxBandwidth,
4152 nmigrate_disks,
4153 migrate_disks,
4154 dconn, tlsAlias,
4155 nbdURI, flags) < 0) {
4156 goto error;
4157 }
4158 } else {
4159 /* Destination doesn't support NBD server.
4160 * Fall back to previous implementation. */
4161 VIR_DEBUG("Destination doesn't support NBD server "
4162 "Falling back to previous implementation.");
4163
4164 if (flags & VIR_MIGRATE_NON_SHARED_DISK)
4165 migrate_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_DISK;
4166
4167 if (flags & VIR_MIGRATE_NON_SHARED_INC)
4168 migrate_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_INC;
4169 }
4170 }
4171
4172 if (qemuMigrationSetDBusVMState(driver, vm) < 0)
4173 goto error;
4174
4175 /* Before EnterMonitor, since already qemuProcessStopCPUs does that */
4176 if (!(flags & VIR_MIGRATE_LIVE) &&
4177 virDomainObjGetState(vm, NULL) == VIR_DOMAIN_RUNNING) {
4178 if (qemuProcessStopCPUs(driver, vm, VIR_DOMAIN_PAUSED_MIGRATION,
4179 QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
4180 goto error;
4181 }
4182
4183 if (qemuDomainObjEnterMonitorAsync(driver, vm,
4184 QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
4185 goto error;
4186
4187 if (priv->job.abortJob) {
4188 /* explicitly do this *after* we entered the monitor,
4189 * as this is a critical section so we are guaranteed
4190 * priv->job.abortJob will not change */
4191 priv->job.current->status = QEMU_DOMAIN_JOB_STATUS_CANCELED;
4192 virReportError(VIR_ERR_OPERATION_ABORTED, _("%s: %s"),
4193 qemuDomainAsyncJobTypeToString(priv->job.asyncJob),
4194 _("canceled by client"));
4195 goto exit_monitor;
4196 }
4197
4198 if (!bwParam &&
4199 qemuMonitorSetMigrationSpeed(priv->mon, priv->migMaxBandwidth) < 0)
4200 goto exit_monitor;
4201
4202 /* connect to the destination qemu if needed */
4203 if ((spec->destType == MIGRATION_DEST_CONNECT_HOST ||
4204 spec->destType == MIGRATION_DEST_CONNECT_SOCKET) &&
4205 qemuMigrationSrcConnect(driver, vm, spec) < 0) {
4206 goto exit_monitor;
4207 }
4208
4209 /* log start of migration */
4210 if ((timestamp = virTimeStringNow()) != NULL)
4211 qemuDomainLogAppendMessage(driver, vm, "%s: initiating migration\n", timestamp);
4212
4213 rc = -1;
4214 switch (spec->destType) {
4215 case MIGRATION_DEST_HOST:
4216 if (STREQ(spec->dest.host.protocol, "rdma") &&
4217 vm->def->mem.hard_limit > 0 &&
4218 virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {
4219 goto exit_monitor;
4220 }
4221 rc = qemuMonitorMigrateToHost(priv->mon, migrate_flags,
4222 spec->dest.host.protocol,
4223 spec->dest.host.name,
4224 spec->dest.host.port);
4225 break;
4226
4227 case MIGRATION_DEST_SOCKET:
4228 qemuSecurityDomainSetPathLabel(driver, vm, spec->dest.socket.path, false);
4229 rc = qemuMonitorMigrateToSocket(priv->mon, migrate_flags,
4230 spec->dest.socket.path);
4231 break;
4232
4233 case MIGRATION_DEST_CONNECT_HOST:
4234 case MIGRATION_DEST_CONNECT_SOCKET:
4235 /* handled above and transformed into MIGRATION_DEST_FD */
4236 break;
4237
4238 case MIGRATION_DEST_FD:
4239 if (spec->fwdType != MIGRATION_FWD_DIRECT) {
4240 fd = spec->dest.fd.local;
4241 spec->dest.fd.local = -1;
4242 }
4243 rc = qemuMonitorMigrateToFd(priv->mon, migrate_flags,
4244 spec->dest.fd.qemu);
4245 VIR_FORCE_CLOSE(spec->dest.fd.qemu);
4246 break;
4247 }
4248
4249 if (qemuDomainObjExitMonitor(driver, vm) < 0 || rc < 0)
4250 goto error;
4251
4252 /* From this point onwards we *must* call cancel to abort the
4253 * migration on source if anything goes wrong */
4254 cancel = true;
4255
4256 if (spec->fwdType != MIGRATION_FWD_DIRECT) {
4257 if (!(iothread = qemuMigrationSrcStartTunnel(spec->fwd.stream, fd)))
4258 goto error;
4259 /* If we've created a tunnel, then the 'fd' will be closed in the
4260 * qemuMigrationIOFunc as data->sock.
4261 */
4262 fd = -1;
4263 }
4264
4265 waitFlags = QEMU_MIGRATION_COMPLETED_PRE_SWITCHOVER;
4266 if (abort_on_error)
4267 waitFlags |= QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR;
4268 if (mig->nbd)
4269 waitFlags |= QEMU_MIGRATION_COMPLETED_CHECK_STORAGE;
4270 if (flags & VIR_MIGRATE_POSTCOPY)
4271 waitFlags |= QEMU_MIGRATION_COMPLETED_POSTCOPY;
4272
4273 rc = qemuMigrationSrcWaitForCompletion(driver, vm,
4274 QEMU_ASYNC_JOB_MIGRATION_OUT,
4275 dconn, waitFlags);
4276 if (rc == -2) {
4277 goto error;
4278 } else if (rc == -1) {
4279 /* QEMU reported failed migration, nothing to cancel anymore */
4280 cancel = false;
4281 goto error;
4282 }
4283
4284 /* When migration completed, QEMU will have paused the CPUs for us.
4285 * Wait for the STOP event to be processed to release the lock state.
4286 */
4287 while (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_RUNNING) {
4288 priv->signalStop = true;
4289 rc = virDomainObjWait(vm);
4290 priv->signalStop = false;
4291 if (rc < 0)
4292 goto error;
4293 }
4294
4295 if (mig->nbd &&
4296 qemuMigrationSrcNBDCopyCancel(driver, vm, false,
4297 QEMU_ASYNC_JOB_MIGRATION_OUT,
4298 dconn) < 0)
4299 goto error;
4300
4301 /* When migration was paused before serializing device state we need to
4302 * resume it now once we finished all block jobs and wait for the real
4303 * end of the migration.
4304 */
4305 if (priv->job.current->status == QEMU_DOMAIN_JOB_STATUS_PAUSED) {
4306 if (qemuMigrationSrcContinue(driver, vm,
4307 QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER,
4308 QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
4309 goto error;
4310
4311 waitFlags ^= QEMU_MIGRATION_COMPLETED_PRE_SWITCHOVER;
4312
4313 rc = qemuMigrationSrcWaitForCompletion(driver, vm,
4314 QEMU_ASYNC_JOB_MIGRATION_OUT,
4315 dconn, waitFlags);
4316 if (rc == -2) {
4317 goto error;
4318 } else if (rc == -1) {
4319 /* QEMU reported failed migration, nothing to cancel anymore */
4320 cancel = false;
4321 goto error;
4322 }
4323 }
4324
4325 if (iothread) {
4326 qemuMigrationIOThread *io;
4327
4328 io = g_steal_pointer(&iothread);
4329 if (qemuMigrationSrcStopTunnel(io, false) < 0)
4330 goto error;
4331 }
4332
4333 if (priv->job.completed) {
4334 priv->job.completed->stopped = priv->job.current->stopped;
4335 qemuDomainJobInfoUpdateTime(priv->job.completed);
4336 qemuDomainJobInfoUpdateDowntime(priv->job.completed);
4337 ignore_value(virTimeMillisNow(&priv->job.completed->sent));
4338 }
4339
4340 cookieFlags |= QEMU_MIGRATION_COOKIE_NETWORK |
4341 QEMU_MIGRATION_COOKIE_STATS;
4342
4343 if (qemuMigrationCookieAddPersistent(mig, &persistDef) < 0 ||
4344 qemuMigrationCookieFormat(mig, driver, vm,
4345 QEMU_MIGRATION_SOURCE,
4346 cookieout, cookieoutlen, cookieFlags) < 0) {
4347 VIR_WARN("Unable to encode migration cookie");
4348 }
4349
4350 ret = 0;
4351
4352 cleanup:
4353 if (events)
4354 priv->signalIOError = false;
4355
4356 priv->migMaxBandwidth = restore_max_bandwidth;
4357 virErrorRestore(&orig_err);
4358
4359 return ret;
4360
4361 error:
4362 virErrorPreserveLast(&orig_err);
4363
4364 if (virDomainObjIsActive(vm)) {
4365 if (cancel &&
4366 priv->job.current->status != QEMU_DOMAIN_JOB_STATUS_QEMU_COMPLETED &&
4367 qemuDomainObjEnterMonitorAsync(driver, vm,
4368 QEMU_ASYNC_JOB_MIGRATION_OUT) == 0) {
4369 qemuMonitorMigrateCancel(priv->mon);
4370 ignore_value(qemuDomainObjExitMonitor(driver, vm));
4371 }
4372
4373 /* cancel any outstanding NBD jobs */
4374 if (mig && mig->nbd)
4375 qemuMigrationSrcNBDCopyCancel(driver, vm, true,
4376 QEMU_ASYNC_JOB_MIGRATION_OUT,
4377 dconn);
4378
4379 qemuMigrationSrcCancelRemoveTempBitmaps(vm, QEMU_ASYNC_JOB_MIGRATION_OUT);
4380
4381 if (priv->job.current->status != QEMU_DOMAIN_JOB_STATUS_CANCELED)
4382 priv->job.current->status = QEMU_DOMAIN_JOB_STATUS_FAILED;
4383 }
4384
4385 if (iothread)
4386 qemuMigrationSrcStopTunnel(iothread, true);
4387
4388 goto cleanup;
4389
4390 exit_monitor:
4391 ignore_value(qemuDomainObjExitMonitor(driver, vm));
4392 goto error;
4393 }
4394
4395 /* Perform migration using QEMU's native migrate support,
4396 * not encrypted obviously
4397 */
4398 static int
qemuMigrationSrcPerformNative(virQEMUDriver * driver,virDomainObj * vm,const char * persist_xml,const char * uri,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,unsigned long flags,unsigned long resource,virConnectPtr dconn,const char * graphicsuri,size_t nmigrate_disks,const char ** migrate_disks,qemuMigrationParams * migParams,const char * nbdURI)4399 qemuMigrationSrcPerformNative(virQEMUDriver *driver,
4400 virDomainObj *vm,
4401 const char *persist_xml,
4402 const char *uri,
4403 const char *cookiein,
4404 int cookieinlen,
4405 char **cookieout,
4406 int *cookieoutlen,
4407 unsigned long flags,
4408 unsigned long resource,
4409 virConnectPtr dconn,
4410 const char *graphicsuri,
4411 size_t nmigrate_disks,
4412 const char **migrate_disks,
4413 qemuMigrationParams *migParams,
4414 const char *nbdURI)
4415 {
4416 qemuDomainObjPrivate *priv = vm->privateData;
4417 g_autoptr(virURI) uribits = NULL;
4418 int ret = -1;
4419 qemuMigrationSpec spec;
4420
4421 VIR_DEBUG("driver=%p, vm=%p, uri=%s, cookiein=%s, cookieinlen=%d, "
4422 "cookieout=%p, cookieoutlen=%p, flags=0x%lx, resource=%lu, "
4423 "graphicsuri=%s, nmigrate_disks=%zu migrate_disks=%p",
4424 driver, vm, uri, NULLSTR(cookiein), cookieinlen,
4425 cookieout, cookieoutlen, flags, resource,
4426 NULLSTR(graphicsuri), nmigrate_disks, migrate_disks);
4427
4428 if (!(uribits = qemuMigrationAnyParseURI(uri, NULL)))
4429 return -1;
4430
4431 if (uribits->scheme == NULL) {
4432 virReportError(VIR_ERR_INTERNAL_ERROR,
4433 _("missing scheme in migration URI: %s"),
4434 uri);
4435 return -1;
4436 }
4437
4438 if (STREQ(uribits->scheme, "rdma")) {
4439 if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {
4440 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
4441 _("outgoing RDMA migration is not supported "
4442 "with this QEMU binary"));
4443 return -1;
4444 }
4445 if (!virMemoryLimitIsSet(vm->def->mem.hard_limit)) {
4446 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
4447 _("cannot start RDMA migration with no memory hard "
4448 "limit set"));
4449 return -1;
4450 }
4451 }
4452
4453 if (STREQ(uribits->scheme, "unix")) {
4454 if ((flags & VIR_MIGRATE_TLS) &&
4455 !qemuMigrationParamsTLSHostnameIsSet(migParams)) {
4456 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
4457 _("Explicit destination hostname is required "
4458 "for TLS migration over UNIX socket"));
4459 return -1;
4460 }
4461
4462 if (flags & VIR_MIGRATE_PARALLEL)
4463 spec.destType = MIGRATION_DEST_SOCKET;
4464 else
4465 spec.destType = MIGRATION_DEST_CONNECT_SOCKET;
4466
4467 spec.dest.socket.path = uribits->path;
4468 } else {
4469 /* RDMA and multi-fd migration requires QEMU to connect to the destination
4470 * itself.
4471 */
4472 if (STREQ(uribits->scheme, "rdma") || (flags & VIR_MIGRATE_PARALLEL))
4473 spec.destType = MIGRATION_DEST_HOST;
4474 else
4475 spec.destType = MIGRATION_DEST_CONNECT_HOST;
4476
4477 spec.dest.host.protocol = uribits->scheme;
4478 spec.dest.host.name = uribits->server;
4479 spec.dest.host.port = uribits->port;
4480 }
4481
4482 spec.fwdType = MIGRATION_FWD_DIRECT;
4483
4484 ret = qemuMigrationSrcRun(driver, vm, persist_xml, cookiein, cookieinlen, cookieout,
4485 cookieoutlen, flags, resource, &spec, dconn,
4486 graphicsuri, nmigrate_disks, migrate_disks,
4487 migParams, nbdURI);
4488
4489 if (spec.destType == MIGRATION_DEST_FD)
4490 VIR_FORCE_CLOSE(spec.dest.fd.qemu);
4491
4492 return ret;
4493 }
4494
4495
4496 static int
qemuMigrationSrcPerformTunnel(virQEMUDriver * driver,virDomainObj * vm,virStreamPtr st,const char * persist_xml,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,unsigned long flags,unsigned long resource,virConnectPtr dconn,const char * graphicsuri,size_t nmigrate_disks,const char ** migrate_disks,qemuMigrationParams * migParams)4497 qemuMigrationSrcPerformTunnel(virQEMUDriver *driver,
4498 virDomainObj *vm,
4499 virStreamPtr st,
4500 const char *persist_xml,
4501 const char *cookiein,
4502 int cookieinlen,
4503 char **cookieout,
4504 int *cookieoutlen,
4505 unsigned long flags,
4506 unsigned long resource,
4507 virConnectPtr dconn,
4508 const char *graphicsuri,
4509 size_t nmigrate_disks,
4510 const char **migrate_disks,
4511 qemuMigrationParams *migParams)
4512 {
4513 int ret = -1;
4514 qemuMigrationSpec spec;
4515 int fds[2] = { -1, -1 };
4516
4517 VIR_DEBUG("driver=%p, vm=%p, st=%p, cookiein=%s, cookieinlen=%d, "
4518 "cookieout=%p, cookieoutlen=%p, flags=0x%lx, resource=%lu, "
4519 "graphicsuri=%s, nmigrate_disks=%zu, migrate_disks=%p",
4520 driver, vm, st, NULLSTR(cookiein), cookieinlen,
4521 cookieout, cookieoutlen, flags, resource,
4522 NULLSTR(graphicsuri), nmigrate_disks, migrate_disks);
4523
4524 spec.fwdType = MIGRATION_FWD_STREAM;
4525 spec.fwd.stream = st;
4526
4527
4528 spec.destType = MIGRATION_DEST_FD;
4529 spec.dest.fd.qemu = -1;
4530 spec.dest.fd.local = -1;
4531
4532 if (virPipe(fds) < 0)
4533 goto cleanup;
4534
4535 spec.dest.fd.qemu = fds[1];
4536 spec.dest.fd.local = fds[0];
4537
4538 if (spec.dest.fd.qemu == -1 ||
4539 qemuSecuritySetImageFDLabel(driver->securityManager, vm->def,
4540 spec.dest.fd.qemu) < 0) {
4541 virReportSystemError(errno, "%s",
4542 _("cannot create pipe for tunnelled migration"));
4543 goto cleanup;
4544 }
4545
4546 ret = qemuMigrationSrcRun(driver, vm, persist_xml, cookiein, cookieinlen,
4547 cookieout, cookieoutlen, flags, resource, &spec,
4548 dconn, graphicsuri, nmigrate_disks, migrate_disks,
4549 migParams, NULL);
4550
4551 cleanup:
4552 VIR_FORCE_CLOSE(spec.dest.fd.qemu);
4553 VIR_FORCE_CLOSE(spec.dest.fd.local);
4554
4555 return ret;
4556 }
4557
4558
4559 /* This is essentially a re-impl of virDomainMigrateVersion2
4560 * from libvirt.c, but running in source libvirtd context,
4561 * instead of client app context & also adding in tunnel
4562 * handling */
4563 static int
qemuMigrationSrcPerformPeer2Peer2(virQEMUDriver * driver,virConnectPtr sconn,virConnectPtr dconn,virDomainObj * vm,const char * dconnuri,unsigned long flags,const char * dname,unsigned long resource,qemuMigrationParams * migParams)4564 qemuMigrationSrcPerformPeer2Peer2(virQEMUDriver *driver,
4565 virConnectPtr sconn,
4566 virConnectPtr dconn,
4567 virDomainObj *vm,
4568 const char *dconnuri,
4569 unsigned long flags,
4570 const char *dname,
4571 unsigned long resource,
4572 qemuMigrationParams *migParams)
4573 {
4574 virDomainPtr ddomain = NULL;
4575 char *uri_out = NULL;
4576 char *cookie = NULL;
4577 char *dom_xml = NULL;
4578 int cookielen = 0, ret;
4579 virErrorPtr orig_err = NULL;
4580 bool cancelled;
4581 virStreamPtr st = NULL;
4582 unsigned long destflags;
4583
4584 VIR_DEBUG("driver=%p, sconn=%p, dconn=%p, vm=%p, dconnuri=%s, "
4585 "flags=0x%lx, dname=%s, resource=%lu",
4586 driver, sconn, dconn, vm, NULLSTR(dconnuri),
4587 flags, NULLSTR(dname), resource);
4588
4589 /* In version 2 of the protocol, the prepare step is slightly
4590 * different. We fetch the domain XML of the source domain
4591 * and pass it to Prepare2.
4592 */
4593 if (!(dom_xml = qemuDomainFormatXML(driver, vm,
4594 QEMU_DOMAIN_FORMAT_LIVE_FLAGS |
4595 VIR_DOMAIN_XML_MIGRATABLE)))
4596 return -1;
4597
4598 if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_PAUSED)
4599 flags |= VIR_MIGRATE_PAUSED;
4600
4601 destflags = flags & ~(VIR_MIGRATE_ABORT_ON_ERROR |
4602 VIR_MIGRATE_AUTO_CONVERGE);
4603
4604 VIR_DEBUG("Prepare2 %p", dconn);
4605 if (flags & VIR_MIGRATE_TUNNELLED) {
4606 /*
4607 * Tunnelled Migrate Version 2 does not support cookies
4608 * due to missing parameters in the prepareTunnel() API.
4609 */
4610
4611 if (!(st = virStreamNew(dconn, 0)))
4612 goto cleanup;
4613
4614 qemuDomainObjEnterRemote(vm);
4615 ret = dconn->driver->domainMigratePrepareTunnel
4616 (dconn, st, destflags, dname, resource, dom_xml);
4617 if (qemuDomainObjExitRemote(vm, true) < 0)
4618 goto cleanup;
4619 } else {
4620 qemuDomainObjEnterRemote(vm);
4621 ret = dconn->driver->domainMigratePrepare2
4622 (dconn, &cookie, &cookielen, NULL, &uri_out,
4623 destflags, dname, resource, dom_xml);
4624 if (qemuDomainObjExitRemote(vm, true) < 0)
4625 goto cleanup;
4626 }
4627 VIR_FREE(dom_xml);
4628 if (ret == -1)
4629 goto cleanup;
4630
4631 if (!(flags & VIR_MIGRATE_TUNNELLED) &&
4632 (uri_out == NULL)) {
4633 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
4634 _("domainMigratePrepare2 did not set uri"));
4635 cancelled = true;
4636 virErrorPreserveLast(&orig_err);
4637 goto finish;
4638 }
4639
4640 /* Perform the migration. The driver isn't supposed to return
4641 * until the migration is complete.
4642 */
4643 VIR_DEBUG("Perform %p", sconn);
4644 qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM2);
4645 if (flags & VIR_MIGRATE_TUNNELLED)
4646 ret = qemuMigrationSrcPerformTunnel(driver, vm, st, NULL,
4647 NULL, 0, NULL, NULL,
4648 flags, resource, dconn,
4649 NULL, 0, NULL, migParams);
4650 else
4651 ret = qemuMigrationSrcPerformNative(driver, vm, NULL, uri_out,
4652 cookie, cookielen,
4653 NULL, NULL, /* No out cookie with v2 migration */
4654 flags, resource, dconn, NULL, 0, NULL,
4655 migParams, NULL);
4656
4657 /* Perform failed. Make sure Finish doesn't overwrite the error */
4658 if (ret < 0)
4659 virErrorPreserveLast(&orig_err);
4660
4661 /* If Perform returns < 0, then we need to cancel the VM
4662 * startup on the destination
4663 */
4664 cancelled = ret < 0;
4665
4666 finish:
4667 /* In version 2 of the migration protocol, we pass the
4668 * status code from the sender to the destination host,
4669 * so it can do any cleanup if the migration failed.
4670 */
4671 dname = dname ? dname : vm->def->name;
4672 VIR_DEBUG("Finish2 %p ret=%d", dconn, ret);
4673 qemuDomainObjEnterRemote(vm);
4674 ddomain = dconn->driver->domainMigrateFinish2
4675 (dconn, dname, cookie, cookielen,
4676 uri_out ? uri_out : dconnuri, destflags, cancelled);
4677 /* The domain is already gone at this point */
4678 ignore_value(qemuDomainObjExitRemote(vm, false));
4679 if (cancelled && ddomain)
4680 VIR_ERROR(_("finish step ignored that migration was cancelled"));
4681
4682 cleanup:
4683 if (ddomain) {
4684 virObjectUnref(ddomain);
4685 ret = 0;
4686 } else {
4687 ret = -1;
4688 }
4689
4690 virObjectUnref(st);
4691
4692 virErrorRestore(&orig_err);
4693 VIR_FREE(uri_out);
4694 VIR_FREE(cookie);
4695
4696 return ret;
4697 }
4698
4699
4700 /* This is essentially a re-impl of virDomainMigrateVersion3
4701 * from libvirt.c, but running in source libvirtd context,
4702 * instead of client app context & also adding in tunnel
4703 * handling */
4704 static int
qemuMigrationSrcPerformPeer2Peer3(virQEMUDriver * driver,virConnectPtr sconn,virConnectPtr dconn,const char * dconnuri,virDomainObj * vm,const char * xmlin,const char * persist_xml,const char * dname,const char * uri,const char * graphicsuri,const char * listenAddress,size_t nmigrate_disks,const char ** migrate_disks,int nbdPort,const char * nbdURI,qemuMigrationParams * migParams,unsigned long long bandwidth,bool useParams,unsigned long flags)4705 qemuMigrationSrcPerformPeer2Peer3(virQEMUDriver *driver,
4706 virConnectPtr sconn,
4707 virConnectPtr dconn,
4708 const char *dconnuri,
4709 virDomainObj *vm,
4710 const char *xmlin,
4711 const char *persist_xml,
4712 const char *dname,
4713 const char *uri,
4714 const char *graphicsuri,
4715 const char *listenAddress,
4716 size_t nmigrate_disks,
4717 const char **migrate_disks,
4718 int nbdPort,
4719 const char *nbdURI,
4720 qemuMigrationParams *migParams,
4721 unsigned long long bandwidth,
4722 bool useParams,
4723 unsigned long flags)
4724 {
4725 virDomainPtr ddomain = NULL;
4726 char *uri_out = NULL;
4727 char *cookiein = NULL;
4728 char *cookieout = NULL;
4729 g_autofree char *dom_xml = NULL;
4730 int cookieinlen = 0;
4731 int cookieoutlen = 0;
4732 int ret = -1;
4733 virErrorPtr orig_err = NULL;
4734 bool cancelled = true;
4735 virStreamPtr st = NULL;
4736 unsigned long destflags;
4737 virTypedParameterPtr params = NULL;
4738 int nparams = 0;
4739 int maxparams = 0;
4740 size_t i;
4741 bool offline = !!(flags & VIR_MIGRATE_OFFLINE);
4742
4743 VIR_DEBUG("driver=%p, sconn=%p, dconn=%p, dconnuri=%s, vm=%p, xmlin=%s, "
4744 "dname=%s, uri=%s, graphicsuri=%s, listenAddress=%s, "
4745 "nmigrate_disks=%zu, migrate_disks=%p, nbdPort=%d, nbdURI=%s, "
4746 "bandwidth=%llu, useParams=%d, flags=0x%lx",
4747 driver, sconn, dconn, NULLSTR(dconnuri), vm, NULLSTR(xmlin),
4748 NULLSTR(dname), NULLSTR(uri), NULLSTR(graphicsuri),
4749 NULLSTR(listenAddress), nmigrate_disks, migrate_disks, nbdPort,
4750 NULLSTR(nbdURI), bandwidth, useParams, flags);
4751
4752 /* Unlike the virDomainMigrateVersion3 counterpart, we don't need
4753 * to worry about auto-setting the VIR_MIGRATE_CHANGE_PROTECTION
4754 * bit here, because we are already running inside the context of
4755 * a single job. */
4756
4757 dom_xml = qemuMigrationSrcBeginPhase(driver, vm, xmlin, dname,
4758 &cookieout, &cookieoutlen,
4759 nmigrate_disks, migrate_disks, flags);
4760 if (!dom_xml)
4761 goto cleanup;
4762
4763 if (useParams) {
4764 if (virTypedParamsAddString(¶ms, &nparams, &maxparams,
4765 VIR_MIGRATE_PARAM_DEST_XML, dom_xml) < 0)
4766 goto cleanup;
4767
4768 if (dname &&
4769 virTypedParamsAddString(¶ms, &nparams, &maxparams,
4770 VIR_MIGRATE_PARAM_DEST_NAME, dname) < 0)
4771 goto cleanup;
4772
4773 if (uri &&
4774 virTypedParamsAddString(¶ms, &nparams, &maxparams,
4775 VIR_MIGRATE_PARAM_URI, uri) < 0)
4776 goto cleanup;
4777
4778 if (bandwidth &&
4779 virTypedParamsAddULLong(¶ms, &nparams, &maxparams,
4780 VIR_MIGRATE_PARAM_BANDWIDTH,
4781 bandwidth) < 0)
4782 goto cleanup;
4783
4784 if (graphicsuri &&
4785 virTypedParamsAddString(¶ms, &nparams, &maxparams,
4786 VIR_MIGRATE_PARAM_GRAPHICS_URI,
4787 graphicsuri) < 0)
4788 goto cleanup;
4789 if (listenAddress &&
4790 virTypedParamsAddString(¶ms, &nparams, &maxparams,
4791 VIR_MIGRATE_PARAM_LISTEN_ADDRESS,
4792 listenAddress) < 0)
4793 goto cleanup;
4794 for (i = 0; i < nmigrate_disks; i++)
4795 if (virTypedParamsAddString(¶ms, &nparams, &maxparams,
4796 VIR_MIGRATE_PARAM_MIGRATE_DISKS,
4797 migrate_disks[i]) < 0)
4798 goto cleanup;
4799 if (nbdPort &&
4800 virTypedParamsAddInt(¶ms, &nparams, &maxparams,
4801 VIR_MIGRATE_PARAM_DISKS_PORT,
4802 nbdPort) < 0)
4803 goto cleanup;
4804 if (nbdURI &&
4805 virTypedParamsAddString(¶ms, &nparams, &maxparams,
4806 VIR_MIGRATE_PARAM_DISKS_URI,
4807 nbdURI) < 0)
4808 goto cleanup;
4809
4810 if (qemuMigrationParamsDump(migParams, ¶ms, &nparams,
4811 &maxparams, &flags) < 0)
4812 goto cleanup;
4813 }
4814
4815 if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_PAUSED)
4816 flags |= VIR_MIGRATE_PAUSED;
4817
4818 destflags = flags & ~(VIR_MIGRATE_ABORT_ON_ERROR |
4819 VIR_MIGRATE_AUTO_CONVERGE);
4820
4821 VIR_DEBUG("Prepare3 %p", dconn);
4822 cookiein = g_steal_pointer(&cookieout);
4823 cookieinlen = cookieoutlen;
4824 cookieoutlen = 0;
4825 if (flags & VIR_MIGRATE_TUNNELLED) {
4826 if (!(st = virStreamNew(dconn, 0)))
4827 goto cleanup;
4828
4829 qemuDomainObjEnterRemote(vm);
4830 if (useParams) {
4831 ret = dconn->driver->domainMigratePrepareTunnel3Params
4832 (dconn, st, params, nparams, cookiein, cookieinlen,
4833 &cookieout, &cookieoutlen, destflags);
4834 } else {
4835 ret = dconn->driver->domainMigratePrepareTunnel3
4836 (dconn, st, cookiein, cookieinlen, &cookieout, &cookieoutlen,
4837 destflags, dname, bandwidth, dom_xml);
4838 }
4839 if (qemuDomainObjExitRemote(vm, !offline) < 0)
4840 goto cleanup;
4841 } else {
4842 qemuDomainObjEnterRemote(vm);
4843 if (useParams) {
4844 ret = dconn->driver->domainMigratePrepare3Params
4845 (dconn, params, nparams, cookiein, cookieinlen,
4846 &cookieout, &cookieoutlen, &uri_out, destflags);
4847 } else {
4848 ret = dconn->driver->domainMigratePrepare3
4849 (dconn, cookiein, cookieinlen, &cookieout, &cookieoutlen,
4850 uri, &uri_out, destflags, dname, bandwidth, dom_xml);
4851 }
4852 if (qemuDomainObjExitRemote(vm, !offline) < 0)
4853 goto cleanup;
4854 }
4855 VIR_FREE(dom_xml);
4856 if (ret == -1)
4857 goto cleanup;
4858
4859 if (offline) {
4860 VIR_DEBUG("Offline migration, skipping Perform phase");
4861 VIR_FREE(cookieout);
4862 cookieoutlen = 0;
4863 cancelled = false;
4864 goto finish;
4865 }
4866
4867 if (uri_out) {
4868 uri = uri_out;
4869 if (useParams &&
4870 virTypedParamsReplaceString(¶ms, &nparams,
4871 VIR_MIGRATE_PARAM_URI, uri_out) < 0) {
4872 virErrorPreserveLast(&orig_err);
4873 goto finish;
4874 }
4875 } else if (!uri && !(flags & VIR_MIGRATE_TUNNELLED)) {
4876 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
4877 _("domainMigratePrepare3 did not set uri"));
4878 virErrorPreserveLast(&orig_err);
4879 goto finish;
4880 }
4881
4882 /* Perform the migration. The driver isn't supposed to return
4883 * until the migration is complete. The src VM should remain
4884 * running, but in paused state until the destination can
4885 * confirm migration completion.
4886 */
4887 VIR_DEBUG("Perform3 %p uri=%s", sconn, NULLSTR(uri));
4888 qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM3);
4889 VIR_FREE(cookiein);
4890 cookiein = g_steal_pointer(&cookieout);
4891 cookieinlen = cookieoutlen;
4892 cookieoutlen = 0;
4893 if (flags & VIR_MIGRATE_TUNNELLED) {
4894 ret = qemuMigrationSrcPerformTunnel(driver, vm, st, persist_xml,
4895 cookiein, cookieinlen,
4896 &cookieout, &cookieoutlen,
4897 flags, bandwidth, dconn, graphicsuri,
4898 nmigrate_disks, migrate_disks,
4899 migParams);
4900 } else {
4901 ret = qemuMigrationSrcPerformNative(driver, vm, persist_xml, uri,
4902 cookiein, cookieinlen,
4903 &cookieout, &cookieoutlen,
4904 flags, bandwidth, dconn, graphicsuri,
4905 nmigrate_disks, migrate_disks,
4906 migParams, nbdURI);
4907 }
4908
4909 /* Perform failed. Make sure Finish doesn't overwrite the error */
4910 if (ret < 0) {
4911 virErrorPreserveLast(&orig_err);
4912 } else {
4913 qemuMigrationJobSetPhase(driver, vm,
4914 QEMU_MIGRATION_PHASE_PERFORM3_DONE);
4915 }
4916
4917 /* If Perform returns < 0, then we need to cancel the VM
4918 * startup on the destination
4919 */
4920 cancelled = ret < 0;
4921
4922 finish:
4923 /*
4924 * The status code from the source is passed to the destination.
4925 * The dest can cleanup in the source indicated it failed to
4926 * send all migration data. Returns NULL for ddomain if
4927 * the dest was unable to complete migration.
4928 */
4929 VIR_DEBUG("Finish3 %p ret=%d", dconn, ret);
4930 VIR_FREE(cookiein);
4931 cookiein = g_steal_pointer(&cookieout);
4932 cookieinlen = cookieoutlen;
4933 cookieoutlen = 0;
4934
4935 if (useParams) {
4936 if (virTypedParamsGetString(params, nparams,
4937 VIR_MIGRATE_PARAM_DEST_NAME, NULL) <= 0 &&
4938 virTypedParamsReplaceString(¶ms, &nparams,
4939 VIR_MIGRATE_PARAM_DEST_NAME,
4940 vm->def->name) < 0) {
4941 ddomain = NULL;
4942 } else {
4943 qemuDomainObjEnterRemote(vm);
4944 ddomain = dconn->driver->domainMigrateFinish3Params
4945 (dconn, params, nparams, cookiein, cookieinlen,
4946 &cookieout, &cookieoutlen, destflags, cancelled);
4947 if (qemuDomainObjExitRemote(vm, !offline) < 0)
4948 goto cleanup;
4949 }
4950 } else {
4951 dname = dname ? dname : vm->def->name;
4952 qemuDomainObjEnterRemote(vm);
4953 ddomain = dconn->driver->domainMigrateFinish3
4954 (dconn, dname, cookiein, cookieinlen, &cookieout, &cookieoutlen,
4955 dconnuri, uri, destflags, cancelled);
4956 if (qemuDomainObjExitRemote(vm, !offline) < 0)
4957 goto cleanup;
4958 }
4959
4960 if (cancelled) {
4961 if (ddomain) {
4962 VIR_ERROR(_("finish step ignored that migration was cancelled"));
4963 } else {
4964 /* If Finish reported a useful error, use it instead of the
4965 * original "migration unexpectedly failed" error.
4966 *
4967 * This is ugly but we can't do better with the APIs we have. We
4968 * only replace the error if Finish was called with cancelled == 1
4969 * and reported a real error (old libvirt would report an error
4970 * from RPC instead of MIGRATE_FINISH_OK), which only happens when
4971 * the domain died on destination. To further reduce a possibility
4972 * of false positives we also check that Perform returned
4973 * VIR_ERR_OPERATION_FAILED.
4974 */
4975 if (orig_err &&
4976 orig_err->domain == VIR_FROM_QEMU &&
4977 orig_err->code == VIR_ERR_OPERATION_FAILED) {
4978 virErrorPtr err = virGetLastError();
4979 if (err &&
4980 err->domain == VIR_FROM_QEMU &&
4981 err->code != VIR_ERR_MIGRATE_FINISH_OK) {
4982 virFreeError(orig_err);
4983 orig_err = NULL;
4984 }
4985 }
4986 }
4987 }
4988
4989 /* If ddomain is NULL, then we were unable to start
4990 * the guest on the target, and must restart on the
4991 * source. There is a small chance that the ddomain
4992 * is NULL due to an RPC failure, in which case
4993 * ddomain could in fact be running on the dest.
4994 * The lock manager plugins should take care of
4995 * safety in this scenario.
4996 */
4997 cancelled = ddomain == NULL;
4998
4999 /* If finish3 set an error, and we don't have an earlier
5000 * one we need to preserve it in case confirm3 overwrites
5001 */
5002 if (!orig_err)
5003 virErrorPreserveLast(&orig_err);
5004
5005 /*
5006 * If cancelled, then src VM will be restarted, else
5007 * it will be killed
5008 */
5009 VIR_DEBUG("Confirm3 %p cancelled=%d vm=%p", sconn, cancelled, vm);
5010 VIR_FREE(cookiein);
5011 cookiein = g_steal_pointer(&cookieout);
5012 cookieinlen = cookieoutlen;
5013 cookieoutlen = 0;
5014 ret = qemuMigrationSrcConfirmPhase(driver, vm,
5015 cookiein, cookieinlen,
5016 flags, cancelled);
5017 /* If Confirm3 returns -1, there's nothing more we can
5018 * do, but fortunately worst case is that there is a
5019 * domain left in 'paused' state on source.
5020 */
5021 if (ret < 0)
5022 VIR_WARN("Guest %s probably left in 'paused' state on source",
5023 vm->def->name);
5024
5025 cleanup:
5026 if (ddomain) {
5027 virObjectUnref(ddomain);
5028 ret = 0;
5029 } else {
5030 ret = -1;
5031 }
5032
5033 virObjectUnref(st);
5034
5035 virErrorRestore(&orig_err);
5036 VIR_FREE(uri_out);
5037 VIR_FREE(cookiein);
5038 VIR_FREE(cookieout);
5039 virTypedParamsFree(params, nparams);
5040 return ret;
5041 }
5042
5043
5044 static void
qemuMigrationSrcConnectionClosed(virConnectPtr conn,int reason,void * opaque)5045 qemuMigrationSrcConnectionClosed(virConnectPtr conn,
5046 int reason,
5047 void *opaque)
5048 {
5049 virDomainObj *vm = opaque;
5050
5051 VIR_DEBUG("conn=%p, reason=%d, vm=%s", conn, reason, vm->def->name);
5052 virDomainObjBroadcast(vm);
5053 }
5054
5055
5056 static int virConnectCredType[] = {
5057 VIR_CRED_AUTHNAME,
5058 VIR_CRED_PASSPHRASE,
5059 };
5060
5061
5062 static virConnectAuth virConnectAuthConfig = {
5063 .credtype = virConnectCredType,
5064 .ncredtype = G_N_ELEMENTS(virConnectCredType),
5065 };
5066
5067
5068 static int
qemuMigrationSrcPerformPeer2Peer(virQEMUDriver * driver,virConnectPtr sconn,virDomainObj * vm,const char * xmlin,const char * persist_xml,const char * dconnuri,const char * uri,const char * graphicsuri,const char * listenAddress,size_t nmigrate_disks,const char ** migrate_disks,int nbdPort,const char * nbdURI,qemuMigrationParams * migParams,unsigned long flags,const char * dname,unsigned long resource,bool * v3proto)5069 qemuMigrationSrcPerformPeer2Peer(virQEMUDriver *driver,
5070 virConnectPtr sconn,
5071 virDomainObj *vm,
5072 const char *xmlin,
5073 const char *persist_xml,
5074 const char *dconnuri,
5075 const char *uri,
5076 const char *graphicsuri,
5077 const char *listenAddress,
5078 size_t nmigrate_disks,
5079 const char **migrate_disks,
5080 int nbdPort,
5081 const char *nbdURI,
5082 qemuMigrationParams *migParams,
5083 unsigned long flags,
5084 const char *dname,
5085 unsigned long resource,
5086 bool *v3proto)
5087 {
5088 int ret = -1;
5089 g_autoptr(virConnect) dconn = NULL;
5090 int p2p;
5091 virErrorPtr orig_err = NULL;
5092 bool offline = !!(flags & VIR_MIGRATE_OFFLINE);
5093 int dstOffline = 0;
5094 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
5095 int useParams;
5096 int rc;
5097
5098 VIR_DEBUG("driver=%p, sconn=%p, vm=%p, xmlin=%s, dconnuri=%s, uri=%s, "
5099 "graphicsuri=%s, listenAddress=%s, nmigrate_disks=%zu, "
5100 "migrate_disks=%p, nbdPort=%d, nbdURI=%s, flags=0x%lx, "
5101 "dname=%s, resource=%lu",
5102 driver, sconn, vm, NULLSTR(xmlin), NULLSTR(dconnuri),
5103 NULLSTR(uri), NULLSTR(graphicsuri), NULLSTR(listenAddress),
5104 nmigrate_disks, migrate_disks, nbdPort, NULLSTR(nbdURI),
5105 flags, NULLSTR(dname), resource);
5106
5107 if (flags & VIR_MIGRATE_TUNNELLED && uri) {
5108 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
5109 _("migration URI is not supported by tunnelled "
5110 "migration"));
5111 goto cleanup;
5112 }
5113
5114 if (flags & VIR_MIGRATE_TUNNELLED && listenAddress) {
5115 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
5116 _("listen address is not supported by tunnelled "
5117 "migration"));
5118 goto cleanup;
5119 }
5120
5121 if (flags & VIR_MIGRATE_TUNNELLED && nbdPort) {
5122 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
5123 _("disk port address is not supported by tunnelled "
5124 "migration"));
5125 goto cleanup;
5126 }
5127
5128 /* the order of operations is important here; we make sure the
5129 * destination side is completely setup before we touch the source
5130 */
5131
5132 qemuDomainObjEnterRemote(vm);
5133 dconn = virConnectOpenAuth(dconnuri, &virConnectAuthConfig, 0);
5134 if (qemuDomainObjExitRemote(vm, !offline) < 0)
5135 goto cleanup;
5136
5137 if (dconn == NULL) {
5138 virReportError(VIR_ERR_OPERATION_FAILED,
5139 _("Failed to connect to remote libvirt URI %s: %s"),
5140 dconnuri, virGetLastErrorMessage());
5141 return -1;
5142 }
5143
5144 if (virConnectSetKeepAlive(dconn, cfg->keepAliveInterval,
5145 cfg->keepAliveCount) < 0)
5146 goto cleanup;
5147
5148 if (virConnectRegisterCloseCallback(dconn, qemuMigrationSrcConnectionClosed,
5149 vm, NULL) < 0) {
5150 goto cleanup;
5151 }
5152
5153 qemuDomainObjEnterRemote(vm);
5154 p2p = VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
5155 VIR_DRV_FEATURE_MIGRATION_P2P);
5156 if (p2p < 0)
5157 goto cleanup;
5158 /* v3proto reflects whether the caller used Perform3, but with
5159 * p2p migrate, regardless of whether Perform2 or Perform3
5160 * were used, we decide protocol based on what target supports
5161 */
5162 rc = VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
5163 VIR_DRV_FEATURE_MIGRATION_V3);
5164 if (rc < 0)
5165 goto cleanup;
5166 *v3proto = !!rc;
5167 useParams = VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
5168 VIR_DRV_FEATURE_MIGRATION_PARAMS);
5169 if (useParams < 0)
5170 goto cleanup;
5171 if (offline) {
5172 dstOffline = VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
5173 VIR_DRV_FEATURE_MIGRATION_OFFLINE);
5174 if (dstOffline < 0)
5175 goto cleanup;
5176 }
5177 if (qemuDomainObjExitRemote(vm, !offline) < 0)
5178 goto cleanup;
5179
5180 if (!p2p) {
5181 virReportError(VIR_ERR_OPERATION_FAILED, "%s",
5182 _("Destination libvirt does not support peer-to-peer migration protocol"));
5183 goto cleanup;
5184 }
5185
5186 /* Only xmlin, dname, uri, and bandwidth parameters can be used with
5187 * old-style APIs. */
5188 if (!useParams && (graphicsuri || listenAddress || nmigrate_disks)) {
5189 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
5190 _("Migration APIs with extensible parameters are not "
5191 "supported but extended parameters were passed"));
5192 goto cleanup;
5193 }
5194
5195 if (offline && !dstOffline) {
5196 virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
5197 _("offline migration is not supported by "
5198 "the destination host"));
5199 goto cleanup;
5200 }
5201
5202 /* Change protection is only required on the source side (us), and
5203 * only for v3 migration when begin and perform are separate jobs.
5204 * But peer-2-peer is already a single job, and we still want to
5205 * talk to older destinations that would reject the flag.
5206 * Therefore it is safe to clear the bit here. */
5207 flags &= ~VIR_MIGRATE_CHANGE_PROTECTION;
5208
5209 if (*v3proto) {
5210 ret = qemuMigrationSrcPerformPeer2Peer3(driver, sconn, dconn, dconnuri, vm, xmlin,
5211 persist_xml, dname, uri, graphicsuri,
5212 listenAddress, nmigrate_disks, migrate_disks,
5213 nbdPort, nbdURI, migParams, resource,
5214 !!useParams, flags);
5215 } else {
5216 ret = qemuMigrationSrcPerformPeer2Peer2(driver, sconn, dconn, vm,
5217 dconnuri, flags, dname, resource,
5218 migParams);
5219 }
5220
5221 cleanup:
5222 virErrorPreserveLast(&orig_err);
5223 if (dconn && virConnectIsAlive(dconn) == 1) {
5224 qemuDomainObjEnterRemote(vm);
5225 virConnectUnregisterCloseCallback(dconn, qemuMigrationSrcConnectionClosed);
5226 ignore_value(qemuDomainObjExitRemote(vm, false));
5227 }
5228 virErrorRestore(&orig_err);
5229 return ret;
5230 }
5231
5232
5233 /*
5234 * This implements perform part of the migration protocol when migration job
5235 * does not need to be active across several APIs, i.e., peer2peer migration or
5236 * perform phase of v2 non-peer2peer migration.
5237 */
5238 static int
qemuMigrationSrcPerformJob(virQEMUDriver * driver,virConnectPtr conn,virDomainObj * vm,const char * xmlin,const char * persist_xml,const char * dconnuri,const char * uri,const char * graphicsuri,const char * listenAddress,size_t nmigrate_disks,const char ** migrate_disks,int nbdPort,const char * nbdURI,qemuMigrationParams * migParams,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,unsigned long flags,const char * dname,unsigned long resource,bool v3proto)5239 qemuMigrationSrcPerformJob(virQEMUDriver *driver,
5240 virConnectPtr conn,
5241 virDomainObj *vm,
5242 const char *xmlin,
5243 const char *persist_xml,
5244 const char *dconnuri,
5245 const char *uri,
5246 const char *graphicsuri,
5247 const char *listenAddress,
5248 size_t nmigrate_disks,
5249 const char **migrate_disks,
5250 int nbdPort,
5251 const char *nbdURI,
5252 qemuMigrationParams *migParams,
5253 const char *cookiein,
5254 int cookieinlen,
5255 char **cookieout,
5256 int *cookieoutlen,
5257 unsigned long flags,
5258 const char *dname,
5259 unsigned long resource,
5260 bool v3proto)
5261 {
5262 virObjectEvent *event = NULL;
5263 int ret = -1;
5264 virErrorPtr orig_err = NULL;
5265 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
5266 qemuDomainObjPrivate *priv = vm->privateData;
5267 qemuDomainJobPrivate *jobPriv = priv->job.privateData;
5268
5269 if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
5270 flags) < 0)
5271 goto cleanup;
5272
5273 if (!(flags & VIR_MIGRATE_OFFLINE) && virDomainObjCheckActive(vm) < 0)
5274 goto endjob;
5275
5276 if (!qemuMigrationSrcIsAllowed(driver, vm, true, flags))
5277 goto endjob;
5278
5279 if (!(flags & (VIR_MIGRATE_UNSAFE | VIR_MIGRATE_OFFLINE)) &&
5280 !qemuMigrationSrcIsSafe(vm->def, priv->qemuCaps,
5281 nmigrate_disks, migrate_disks, flags))
5282 goto endjob;
5283
5284 qemuMigrationSrcStoreDomainState(vm);
5285
5286 if ((flags & (VIR_MIGRATE_TUNNELLED | VIR_MIGRATE_PEER2PEER))) {
5287 ret = qemuMigrationSrcPerformPeer2Peer(driver, conn, vm, xmlin, persist_xml,
5288 dconnuri, uri, graphicsuri, listenAddress,
5289 nmigrate_disks, migrate_disks, nbdPort,
5290 nbdURI,
5291 migParams, flags, dname, resource,
5292 &v3proto);
5293 } else {
5294 qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM2);
5295 ret = qemuMigrationSrcPerformNative(driver, vm, persist_xml, uri, cookiein, cookieinlen,
5296 cookieout, cookieoutlen,
5297 flags, resource, NULL, NULL, 0, NULL,
5298 migParams, nbdURI);
5299 }
5300 if (ret < 0)
5301 goto endjob;
5302
5303 /*
5304 * In v3 protocol, the source VM is not killed off until the
5305 * confirm step.
5306 */
5307 if (!v3proto) {
5308 qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_MIGRATED,
5309 QEMU_ASYNC_JOB_MIGRATION_OUT,
5310 VIR_QEMU_PROCESS_STOP_MIGRATED);
5311 virDomainAuditStop(vm, "migrated");
5312 event = virDomainEventLifecycleNewFromObj(vm,
5313 VIR_DOMAIN_EVENT_STOPPED,
5314 VIR_DOMAIN_EVENT_STOPPED_MIGRATED);
5315 }
5316
5317 endjob:
5318 if (ret < 0)
5319 virErrorPreserveLast(&orig_err);
5320
5321 /* v2 proto has no confirm phase so we need to reset migration parameters
5322 * here
5323 */
5324 if (!v3proto && ret < 0)
5325 qemuMigrationParamsReset(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
5326 jobPriv->migParams, priv->job.apiFlags);
5327
5328 qemuMigrationSrcRestoreDomainState(driver, vm);
5329
5330 qemuMigrationJobFinish(driver, vm);
5331 if (!virDomainObjIsActive(vm) && ret == 0) {
5332 if (flags & VIR_MIGRATE_UNDEFINE_SOURCE) {
5333 virDomainDeleteConfig(cfg->configDir, cfg->autostartDir, vm);
5334 vm->persistent = 0;
5335 }
5336 qemuDomainRemoveInactiveJob(driver, vm);
5337 }
5338
5339 virErrorRestore(&orig_err);
5340
5341 cleanup:
5342 virObjectEventStateQueue(driver->domainEventState, event);
5343 return ret;
5344 }
5345
5346 /*
5347 * This implements perform phase of v3 migration protocol.
5348 */
5349 static int
qemuMigrationSrcPerformPhase(virQEMUDriver * driver,virConnectPtr conn,virDomainObj * vm,const char * persist_xml,const char * uri,const char * graphicsuri,size_t nmigrate_disks,const char ** migrate_disks,qemuMigrationParams * migParams,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,unsigned long flags,unsigned long resource,const char * nbdURI)5350 qemuMigrationSrcPerformPhase(virQEMUDriver *driver,
5351 virConnectPtr conn,
5352 virDomainObj *vm,
5353 const char *persist_xml,
5354 const char *uri,
5355 const char *graphicsuri,
5356 size_t nmigrate_disks,
5357 const char **migrate_disks,
5358 qemuMigrationParams *migParams,
5359 const char *cookiein,
5360 int cookieinlen,
5361 char **cookieout,
5362 int *cookieoutlen,
5363 unsigned long flags,
5364 unsigned long resource,
5365 const char *nbdURI)
5366 {
5367 qemuDomainObjPrivate *priv = vm->privateData;
5368 qemuDomainJobPrivate *jobPriv = priv->job.privateData;
5369 int ret = -1;
5370
5371 /* If we didn't start the job in the begin phase, start it now. */
5372 if (!(flags & VIR_MIGRATE_CHANGE_PROTECTION)) {
5373 if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
5374 flags) < 0)
5375 return ret;
5376 } else if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT)) {
5377 return ret;
5378 }
5379
5380 qemuMigrationJobStartPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM3);
5381 virCloseCallbacksUnset(driver->closeCallbacks, vm,
5382 qemuMigrationSrcCleanup);
5383
5384 ret = qemuMigrationSrcPerformNative(driver, vm, persist_xml, uri, cookiein, cookieinlen,
5385 cookieout, cookieoutlen,
5386 flags, resource, NULL, graphicsuri,
5387 nmigrate_disks, migrate_disks, migParams, nbdURI);
5388
5389 if (ret < 0) {
5390 qemuMigrationSrcRestoreDomainState(driver, vm);
5391 goto endjob;
5392 }
5393
5394 qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM3_DONE);
5395
5396 if (virCloseCallbacksSet(driver->closeCallbacks, vm, conn,
5397 qemuMigrationSrcCleanup) < 0)
5398 goto endjob;
5399
5400 endjob:
5401 if (ret < 0) {
5402 qemuMigrationParamsReset(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
5403 jobPriv->migParams, priv->job.apiFlags);
5404 qemuMigrationJobFinish(driver, vm);
5405 } else {
5406 qemuMigrationJobContinue(vm);
5407 }
5408
5409 if (!virDomainObjIsActive(vm))
5410 qemuDomainRemoveInactiveJob(driver, vm);
5411
5412 return ret;
5413 }
5414
5415 int
qemuMigrationSrcPerform(virQEMUDriver * driver,virConnectPtr conn,virDomainObj * vm,const char * xmlin,const char * persist_xml,const char * dconnuri,const char * uri,const char * graphicsuri,const char * listenAddress,size_t nmigrate_disks,const char ** migrate_disks,int nbdPort,const char * nbdURI,qemuMigrationParams * migParams,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,unsigned long flags,const char * dname,unsigned long resource,bool v3proto)5416 qemuMigrationSrcPerform(virQEMUDriver *driver,
5417 virConnectPtr conn,
5418 virDomainObj *vm,
5419 const char *xmlin,
5420 const char *persist_xml,
5421 const char *dconnuri,
5422 const char *uri,
5423 const char *graphicsuri,
5424 const char *listenAddress,
5425 size_t nmigrate_disks,
5426 const char **migrate_disks,
5427 int nbdPort,
5428 const char *nbdURI,
5429 qemuMigrationParams *migParams,
5430 const char *cookiein,
5431 int cookieinlen,
5432 char **cookieout,
5433 int *cookieoutlen,
5434 unsigned long flags,
5435 const char *dname,
5436 unsigned long resource,
5437 bool v3proto)
5438 {
5439 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
5440
5441 VIR_DEBUG("driver=%p, conn=%p, vm=%p, xmlin=%s, dconnuri=%s, "
5442 "uri=%s, graphicsuri=%s, listenAddress=%s, "
5443 "nmigrate_disks=%zu, migrate_disks=%p, nbdPort=%d, "
5444 "nbdURI=%s, "
5445 "cookiein=%s, cookieinlen=%d, cookieout=%p, cookieoutlen=%p, "
5446 "flags=0x%lx, dname=%s, resource=%lu, v3proto=%d",
5447 driver, conn, vm, NULLSTR(xmlin), NULLSTR(dconnuri),
5448 NULLSTR(uri), NULLSTR(graphicsuri), NULLSTR(listenAddress),
5449 nmigrate_disks, migrate_disks, nbdPort, NULLSTR(nbdURI),
5450 NULLSTR(cookiein), cookieinlen, cookieout, cookieoutlen,
5451 flags, NULLSTR(dname), resource, v3proto);
5452
5453 if (cfg->migrateTLSForce &&
5454 !(flags & VIR_MIGRATE_TUNNELLED) &&
5455 !(flags & VIR_MIGRATE_TLS)) {
5456 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
5457 _("this libvirtd instance allows migration only with VIR_MIGRATE_TLS flag"));
5458 return -1;
5459 }
5460
5461 if ((flags & (VIR_MIGRATE_TUNNELLED | VIR_MIGRATE_PEER2PEER))) {
5462 if (cookieinlen) {
5463 virReportError(VIR_ERR_OPERATION_INVALID,
5464 "%s", _("received unexpected cookie with P2P migration"));
5465 return -1;
5466 }
5467
5468 return qemuMigrationSrcPerformJob(driver, conn, vm, xmlin, persist_xml, dconnuri, uri,
5469 graphicsuri, listenAddress,
5470 nmigrate_disks, migrate_disks, nbdPort,
5471 nbdURI, migParams,
5472 cookiein, cookieinlen,
5473 cookieout, cookieoutlen,
5474 flags, dname, resource, v3proto);
5475 } else {
5476 if (dconnuri) {
5477 virReportError(VIR_ERR_INTERNAL_ERROR,
5478 "%s", _("Unexpected dconnuri parameter with non-peer2peer migration"));
5479 return -1;
5480 }
5481
5482 if (v3proto) {
5483 return qemuMigrationSrcPerformPhase(driver, conn, vm, persist_xml, uri,
5484 graphicsuri,
5485 nmigrate_disks, migrate_disks,
5486 migParams,
5487 cookiein, cookieinlen,
5488 cookieout, cookieoutlen,
5489 flags, resource, nbdURI);
5490 } else {
5491 return qemuMigrationSrcPerformJob(driver, conn, vm, xmlin, persist_xml, NULL,
5492 uri, graphicsuri, listenAddress,
5493 nmigrate_disks, migrate_disks, nbdPort,
5494 nbdURI, migParams,
5495 cookiein, cookieinlen,
5496 cookieout, cookieoutlen, flags,
5497 dname, resource, v3proto);
5498 }
5499 }
5500 }
5501
5502 static int
qemuMigrationDstVPAssociatePortProfiles(virDomainDef * def)5503 qemuMigrationDstVPAssociatePortProfiles(virDomainDef *def)
5504 {
5505 size_t i;
5506 int last_good_net = -1;
5507 virDomainNetDef *net;
5508
5509 for (i = 0; i < def->nnets; i++) {
5510 net = def->nets[i];
5511 if (virDomainNetGetActualType(net) == VIR_DOMAIN_NET_TYPE_DIRECT) {
5512 if (virNetDevVPortProfileAssociate(net->ifname,
5513 virDomainNetGetActualVirtPortProfile(net),
5514 &net->mac,
5515 virDomainNetGetActualDirectDev(net),
5516 -1,
5517 def->uuid,
5518 VIR_NETDEV_VPORT_PROFILE_OP_MIGRATE_IN_FINISH,
5519 false) < 0) {
5520 virReportError(VIR_ERR_OPERATION_FAILED,
5521 _("Port profile Associate failed for %s"),
5522 net->ifname);
5523 goto err_exit;
5524 }
5525 last_good_net = i;
5526 VIR_DEBUG("Port profile Associate succeeded for %s", net->ifname);
5527
5528 if (virNetDevMacVLanVPortProfileRegisterCallback(net->ifname, &net->mac,
5529 virDomainNetGetActualDirectDev(net), def->uuid,
5530 virDomainNetGetActualVirtPortProfile(net),
5531 VIR_NETDEV_VPORT_PROFILE_OP_CREATE))
5532 goto err_exit;
5533 }
5534 }
5535
5536 return 0;
5537
5538 err_exit:
5539 for (i = 0; last_good_net != -1 && i <= last_good_net; i++) {
5540 net = def->nets[i];
5541 if (virDomainNetGetActualType(net) == VIR_DOMAIN_NET_TYPE_DIRECT) {
5542 ignore_value(virNetDevVPortProfileDisassociate(net->ifname,
5543 virDomainNetGetActualVirtPortProfile(net),
5544 &net->mac,
5545 virDomainNetGetActualDirectDev(net),
5546 -1,
5547 VIR_NETDEV_VPORT_PROFILE_OP_MIGRATE_IN_FINISH));
5548 }
5549 }
5550 return -1;
5551 }
5552
5553
5554 static int
qemuMigrationDstPersist(virQEMUDriver * driver,virDomainObj * vm,qemuMigrationCookie * mig,bool ignoreSaveError)5555 qemuMigrationDstPersist(virQEMUDriver *driver,
5556 virDomainObj *vm,
5557 qemuMigrationCookie *mig,
5558 bool ignoreSaveError)
5559 {
5560 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
5561 qemuDomainObjPrivate *priv = vm->privateData;
5562 virDomainDef *vmdef;
5563 g_autoptr(virDomainDef) oldDef = NULL;
5564 unsigned int oldPersist = vm->persistent;
5565 virObjectEvent *event;
5566
5567 vm->persistent = 1;
5568 oldDef = vm->newDef;
5569 vm->newDef = qemuMigrationCookieGetPersistent(mig);
5570
5571 if (!(vmdef = virDomainObjGetPersistentDef(driver->xmlopt, vm,
5572 priv->qemuCaps)))
5573 goto error;
5574
5575 if (!oldPersist && qemuDomainNamePathsCleanup(cfg, vmdef->name, false) < 0)
5576 goto error;
5577
5578 if (virDomainDefSave(vmdef, driver->xmlopt, cfg->configDir) < 0 &&
5579 !ignoreSaveError)
5580 goto error;
5581
5582 event = virDomainEventLifecycleNewFromObj(vm,
5583 VIR_DOMAIN_EVENT_DEFINED,
5584 oldPersist ?
5585 VIR_DOMAIN_EVENT_DEFINED_UPDATED :
5586 VIR_DOMAIN_EVENT_DEFINED_ADDED);
5587 virObjectEventStateQueue(driver->domainEventState, event);
5588
5589 return 0;
5590
5591 error:
5592 virDomainDefFree(vm->newDef);
5593 vm->persistent = oldPersist;
5594 vm->newDef = oldDef;
5595 oldDef = NULL;
5596 return -1;
5597 }
5598
5599
5600 virDomainPtr
qemuMigrationDstFinish(virQEMUDriver * driver,virConnectPtr dconn,virDomainObj * vm,const char * cookiein,int cookieinlen,char ** cookieout,int * cookieoutlen,unsigned long flags,int retcode,bool v3proto)5601 qemuMigrationDstFinish(virQEMUDriver *driver,
5602 virConnectPtr dconn,
5603 virDomainObj *vm,
5604 const char *cookiein,
5605 int cookieinlen,
5606 char **cookieout,
5607 int *cookieoutlen,
5608 unsigned long flags,
5609 int retcode,
5610 bool v3proto)
5611 {
5612 virDomainPtr dom = NULL;
5613 g_autoptr(qemuMigrationCookie) mig = NULL;
5614 virErrorPtr orig_err = NULL;
5615 int cookie_flags = 0;
5616 qemuDomainObjPrivate *priv = vm->privateData;
5617 qemuDomainJobPrivate *jobPriv = priv->job.privateData;
5618 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
5619 unsigned short port;
5620 unsigned long long timeReceived = 0;
5621 virObjectEvent *event;
5622 qemuDomainJobInfo *jobInfo = NULL;
5623 bool inPostCopy = false;
5624 bool doKill = true;
5625
5626 VIR_DEBUG("driver=%p, dconn=%p, vm=%p, cookiein=%s, cookieinlen=%d, "
5627 "cookieout=%p, cookieoutlen=%p, flags=0x%lx, retcode=%d",
5628 driver, dconn, vm, NULLSTR(cookiein), cookieinlen,
5629 cookieout, cookieoutlen, flags, retcode);
5630
5631 port = priv->migrationPort;
5632 priv->migrationPort = 0;
5633
5634 if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_IN)) {
5635 qemuMigrationDstErrorReport(driver, vm->def->name);
5636 goto cleanup;
5637 }
5638
5639 ignore_value(virTimeMillisNow(&timeReceived));
5640
5641 qemuMigrationJobStartPhase(driver, vm,
5642 v3proto ? QEMU_MIGRATION_PHASE_FINISH3
5643 : QEMU_MIGRATION_PHASE_FINISH2);
5644
5645 qemuDomainCleanupRemove(vm, qemuMigrationDstPrepareCleanup);
5646 g_clear_pointer(&priv->job.completed, qemuDomainJobInfoFree);
5647
5648 cookie_flags = QEMU_MIGRATION_COOKIE_NETWORK |
5649 QEMU_MIGRATION_COOKIE_STATS |
5650 QEMU_MIGRATION_COOKIE_NBD;
5651 /* Some older versions of libvirt always send persistent XML in the cookie
5652 * even though VIR_MIGRATE_PERSIST_DEST was not used. */
5653 cookie_flags |= QEMU_MIGRATION_COOKIE_PERSISTENT;
5654
5655 if (!(mig = qemuMigrationCookieParse(driver, vm->def, priv->origname, priv,
5656 cookiein, cookieinlen, cookie_flags)))
5657 goto endjob;
5658
5659 if (flags & VIR_MIGRATE_OFFLINE) {
5660 if (retcode == 0 &&
5661 qemuMigrationDstPersist(driver, vm, mig, false) == 0)
5662 dom = virGetDomain(dconn, vm->def->name, vm->def->uuid, -1);
5663 goto endjob;
5664 }
5665
5666 if (retcode != 0) {
5667 /* Check for a possible error on the monitor in case Finish was called
5668 * earlier than monitor EOF handler got a chance to process the error
5669 */
5670 qemuDomainCheckMonitor(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN);
5671 goto endjob;
5672 }
5673
5674 if (!virDomainObjIsActive(vm)) {
5675 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
5676 _("guest unexpectedly quit"));
5677 qemuMigrationDstErrorReport(driver, vm->def->name);
5678 goto endjob;
5679 }
5680
5681 if (qemuMigrationDstVPAssociatePortProfiles(vm->def) < 0)
5682 goto endjob;
5683
5684 if (mig->network && qemuMigrationDstOPDRelocate(driver, vm, mig) < 0)
5685 VIR_WARN("unable to provide network data for relocation");
5686
5687 if (qemuMigrationDstStopNBDServer(driver, vm, mig) < 0)
5688 goto endjob;
5689
5690 if (qemuRefreshVirtioChannelState(driver, vm,
5691 QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
5692 goto endjob;
5693
5694 if (qemuConnectAgent(driver, vm) < 0)
5695 goto endjob;
5696
5697 if (flags & VIR_MIGRATE_PERSIST_DEST) {
5698 if (qemuMigrationDstPersist(driver, vm, mig, !v3proto) < 0) {
5699 /* Hmpf. Migration was successful, but making it persistent
5700 * was not. If we report successful, then when this domain
5701 * shuts down, management tools are in for a surprise. On the
5702 * other hand, if we report failure, then the management tools
5703 * might try to restart the domain on the source side, even
5704 * though the domain is actually running on the destination.
5705 * Pretend success and hope that this is a rare situation and
5706 * management tools are smart.
5707 *
5708 * However, in v3 protocol, the source VM is still available
5709 * to restart during confirm() step, so we kill it off now.
5710 */
5711 if (v3proto)
5712 goto endjob;
5713 }
5714 }
5715
5716 /* We need to wait for QEMU to process all data sent by the source
5717 * before starting guest CPUs.
5718 */
5719 if (qemuMigrationDstWaitForCompletion(driver, vm,
5720 QEMU_ASYNC_JOB_MIGRATION_IN,
5721 !!(flags & VIR_MIGRATE_POSTCOPY)) < 0) {
5722 /* There's not much we can do for v2 protocol since the
5723 * original domain on the source host is already gone.
5724 */
5725 if (v3proto)
5726 goto endjob;
5727 }
5728
5729 /* Now that the state data was transferred we can refresh the actual state
5730 * of the devices */
5731 if (qemuProcessRefreshState(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN) < 0) {
5732 /* Similarly to the case above v2 protocol will not be able to recover
5733 * from this. Let's ignore this and perhaps stuff will not break. */
5734 if (v3proto)
5735 goto endjob;
5736 }
5737
5738 if (priv->job.current->status == QEMU_DOMAIN_JOB_STATUS_POSTCOPY)
5739 inPostCopy = true;
5740
5741 if (!(flags & VIR_MIGRATE_PAUSED)) {
5742 /* run 'cont' on the destination, which allows migration on qemu
5743 * >= 0.10.6 to work properly. This isn't strictly necessary on
5744 * older qemu's, but it also doesn't hurt anything there
5745 */
5746 if (qemuProcessStartCPUs(driver, vm,
5747 inPostCopy ? VIR_DOMAIN_RUNNING_POSTCOPY
5748 : VIR_DOMAIN_RUNNING_MIGRATED,
5749 QEMU_ASYNC_JOB_MIGRATION_IN) < 0) {
5750 if (virGetLastErrorCode() == VIR_ERR_OK)
5751 virReportError(VIR_ERR_INTERNAL_ERROR,
5752 "%s", _("resume operation failed"));
5753 /* Need to save the current error, in case shutting
5754 * down the process overwrites it
5755 */
5756 virErrorPreserveLast(&orig_err);
5757
5758 /*
5759 * In v3 protocol, the source VM is still available to
5760 * restart during confirm() step, so we kill it off
5761 * now.
5762 * In v2 protocol, the source is dead, so we leave
5763 * target in paused state, in case admin can fix
5764 * things up.
5765 */
5766 if (v3proto)
5767 goto endjob;
5768 }
5769
5770 if (inPostCopy)
5771 doKill = false;
5772 }
5773
5774 if (mig->jobInfo) {
5775 jobInfo = mig->jobInfo;
5776 mig->jobInfo = NULL;
5777
5778 if (jobInfo->sent && timeReceived) {
5779 jobInfo->timeDelta = timeReceived - jobInfo->sent;
5780 jobInfo->received = timeReceived;
5781 jobInfo->timeDeltaSet = true;
5782 }
5783 qemuDomainJobInfoUpdateTime(jobInfo);
5784 qemuDomainJobInfoUpdateDowntime(jobInfo);
5785 }
5786
5787 if (inPostCopy) {
5788 if (qemuMigrationDstWaitForCompletion(driver, vm,
5789 QEMU_ASYNC_JOB_MIGRATION_IN,
5790 false) < 0) {
5791 goto endjob;
5792 }
5793 if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_RUNNING) {
5794 virDomainObjSetState(vm,
5795 VIR_DOMAIN_RUNNING,
5796 VIR_DOMAIN_RUNNING_MIGRATED);
5797 }
5798 }
5799
5800 dom = virGetDomain(dconn, vm->def->name, vm->def->uuid, vm->def->id);
5801
5802 if (inPostCopy) {
5803 /* The only RESUME event during post-copy migration is triggered by
5804 * QEMU when the running domain moves from the source to the
5805 * destination host, but then the migration keeps running until all
5806 * modified memory is transferred from the source host. This will
5807 * result in VIR_DOMAIN_EVENT_RESUMED with RESUMED_POSTCOPY detail.
5808 * However, our API documentation says we need to fire another RESUMED
5809 * event at the very end of migration with RESUMED_MIGRATED detail.
5810 */
5811 event = virDomainEventLifecycleNewFromObj(vm,
5812 VIR_DOMAIN_EVENT_RESUMED,
5813 VIR_DOMAIN_EVENT_RESUMED_MIGRATED);
5814 virObjectEventStateQueue(driver->domainEventState, event);
5815 }
5816
5817 if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_PAUSED) {
5818 virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, VIR_DOMAIN_PAUSED_USER);
5819 event = virDomainEventLifecycleNewFromObj(vm,
5820 VIR_DOMAIN_EVENT_SUSPENDED,
5821 VIR_DOMAIN_EVENT_SUSPENDED_PAUSED);
5822 virObjectEventStateQueue(driver->domainEventState, event);
5823 }
5824
5825 if (virDomainObjIsActive(vm) &&
5826 virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0)
5827 VIR_WARN("Failed to save status on vm %s", vm->def->name);
5828
5829 /* Guest is successfully running, so cancel previous auto destroy */
5830 qemuProcessAutoDestroyRemove(driver, vm);
5831
5832 endjob:
5833 if (!dom &&
5834 !(flags & VIR_MIGRATE_OFFLINE) &&
5835 virDomainObjIsActive(vm)) {
5836 if (doKill) {
5837 qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED,
5838 QEMU_ASYNC_JOB_MIGRATION_IN,
5839 VIR_QEMU_PROCESS_STOP_MIGRATED);
5840 virDomainAuditStop(vm, "failed");
5841 event = virDomainEventLifecycleNewFromObj(vm,
5842 VIR_DOMAIN_EVENT_STOPPED,
5843 VIR_DOMAIN_EVENT_STOPPED_FAILED);
5844 virObjectEventStateQueue(driver->domainEventState, event);
5845 } else {
5846 qemuMigrationAnyPostcopyFailed(driver, vm);
5847 }
5848 }
5849
5850 if (dom) {
5851 if (jobInfo) {
5852 priv->job.completed = g_steal_pointer(&jobInfo);
5853 priv->job.completed->status = QEMU_DOMAIN_JOB_STATUS_COMPLETED;
5854 priv->job.completed->statsType = QEMU_DOMAIN_JOB_STATS_TYPE_MIGRATION;
5855 }
5856
5857 if (qemuMigrationCookieFormat(mig, driver, vm,
5858 QEMU_MIGRATION_DESTINATION,
5859 cookieout, cookieoutlen,
5860 QEMU_MIGRATION_COOKIE_STATS) < 0)
5861 VIR_WARN("Unable to encode migration cookie");
5862
5863 /* Remove completed stats for post-copy, everything but timing fields
5864 * is obsolete anyway.
5865 */
5866 if (inPostCopy)
5867 g_clear_pointer(&priv->job.completed, qemuDomainJobInfoFree);
5868 }
5869
5870 qemuMigrationParamsReset(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN,
5871 jobPriv->migParams, priv->job.apiFlags);
5872
5873 qemuMigrationJobFinish(driver, vm);
5874 if (!virDomainObjIsActive(vm))
5875 qemuDomainRemoveInactiveJob(driver, vm);
5876
5877 cleanup:
5878 g_clear_pointer(&jobInfo, qemuDomainJobInfoFree);
5879 virPortAllocatorRelease(port);
5880 if (priv->mon)
5881 qemuMonitorSetDomainLog(priv->mon, NULL, NULL, NULL);
5882 VIR_FREE(priv->origname);
5883 virDomainObjEndAPI(&vm);
5884 virErrorRestore(&orig_err);
5885
5886 /* Set a special error if Finish is expected to return NULL as a result of
5887 * successful call with retcode != 0
5888 */
5889 if (retcode != 0 && !dom && virGetLastErrorCode() == VIR_ERR_OK)
5890 virReportError(VIR_ERR_MIGRATE_FINISH_OK, NULL);
5891 return dom;
5892 }
5893
5894
5895 /* Helper function called while vm is active. */
5896 int
qemuMigrationSrcToFile(virQEMUDriver * driver,virDomainObj * vm,int fd,virCommand * compressor,qemuDomainAsyncJob asyncJob)5897 qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm,
5898 int fd,
5899 virCommand *compressor,
5900 qemuDomainAsyncJob asyncJob)
5901 {
5902 qemuDomainObjPrivate *priv = vm->privateData;
5903 bool bwParam = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_PARAM_BANDWIDTH);
5904 int rc;
5905 int ret = -1;
5906 int pipeFD[2] = { -1, -1 };
5907 unsigned long saveMigBandwidth = priv->migMaxBandwidth;
5908 char *errbuf = NULL;
5909 virErrorPtr orig_err = NULL;
5910 g_autoptr(qemuMigrationParams) migParams = NULL;
5911
5912 if (qemuMigrationSetDBusVMState(driver, vm) < 0)
5913 return -1;
5914
5915 /* Increase migration bandwidth to unlimited since target is a file.
5916 * Failure to change migration speed is not fatal. */
5917 if (bwParam) {
5918 if (!(migParams = qemuMigrationParamsNew()))
5919 return -1;
5920
5921 if (qemuMigrationParamsSetULL(migParams,
5922 QEMU_MIGRATION_PARAM_MAX_BANDWIDTH,
5923 QEMU_DOMAIN_MIG_BANDWIDTH_MAX * 1024 * 1024) < 0)
5924 return -1;
5925
5926 if (qemuMigrationParamsApply(driver, vm, asyncJob, migParams) < 0)
5927 return -1;
5928
5929 priv->migMaxBandwidth = QEMU_DOMAIN_MIG_BANDWIDTH_MAX;
5930 } else {
5931 if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) == 0) {
5932 qemuMonitorSetMigrationSpeed(priv->mon,
5933 QEMU_DOMAIN_MIG_BANDWIDTH_MAX);
5934 priv->migMaxBandwidth = QEMU_DOMAIN_MIG_BANDWIDTH_MAX;
5935 if (qemuDomainObjExitMonitor(driver, vm) < 0)
5936 return -1;
5937 }
5938 }
5939
5940 if (!virDomainObjIsActive(vm)) {
5941 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
5942 _("guest unexpectedly quit"));
5943 /* nothing to tear down */
5944 return -1;
5945 }
5946
5947 if (compressor && virPipe(pipeFD) < 0)
5948 return -1;
5949
5950 /* All right! We can use fd migration, which means that qemu
5951 * doesn't have to open() the file, so while we still have to
5952 * grant SELinux access, we can do it on fd and avoid cleanup
5953 * later, as well as skip futzing with cgroup. */
5954 if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def,
5955 compressor ? pipeFD[1] : fd) < 0)
5956 goto cleanup;
5957
5958 if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
5959 goto cleanup;
5960
5961 if (!compressor) {
5962 rc = qemuMonitorMigrateToFd(priv->mon,
5963 QEMU_MONITOR_MIGRATE_BACKGROUND,
5964 fd);
5965 } else {
5966 virCommandSetInputFD(compressor, pipeFD[0]);
5967 virCommandSetOutputFD(compressor, &fd);
5968 virCommandSetErrorBuffer(compressor, &errbuf);
5969 virCommandDoAsyncIO(compressor);
5970 if (virSetCloseExec(pipeFD[1]) < 0) {
5971 virReportSystemError(errno, "%s",
5972 _("Unable to set cloexec flag"));
5973 ignore_value(qemuDomainObjExitMonitor(driver, vm));
5974 goto cleanup;
5975 }
5976 if (virCommandRunAsync(compressor, NULL) < 0) {
5977 ignore_value(qemuDomainObjExitMonitor(driver, vm));
5978 goto cleanup;
5979 }
5980 rc = qemuMonitorMigrateToFd(priv->mon,
5981 QEMU_MONITOR_MIGRATE_BACKGROUND,
5982 pipeFD[1]);
5983 if (VIR_CLOSE(pipeFD[0]) < 0 ||
5984 VIR_CLOSE(pipeFD[1]) < 0)
5985 VIR_WARN("failed to close intermediate pipe");
5986 }
5987 if (qemuDomainObjExitMonitor(driver, vm) < 0)
5988 goto cleanup;
5989 if (rc < 0)
5990 goto cleanup;
5991
5992 rc = qemuMigrationSrcWaitForCompletion(driver, vm, asyncJob, NULL, 0);
5993
5994 if (rc < 0) {
5995 if (rc == -2) {
5996 virErrorPreserveLast(&orig_err);
5997 virCommandAbort(compressor);
5998 if (virDomainObjIsActive(vm) &&
5999 qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) == 0) {
6000 qemuMonitorMigrateCancel(priv->mon);
6001 ignore_value(qemuDomainObjExitMonitor(driver, vm));
6002 }
6003 }
6004 goto cleanup;
6005 }
6006
6007 if (compressor && virCommandWait(compressor, NULL) < 0)
6008 goto cleanup;
6009
6010 qemuDomainEventEmitJobCompleted(driver, vm);
6011 ret = 0;
6012
6013 cleanup:
6014 if (ret < 0 && !orig_err)
6015 virErrorPreserveLast(&orig_err);
6016
6017 /* Restore max migration bandwidth */
6018 if (virDomainObjIsActive(vm)) {
6019 if (bwParam) {
6020 if (qemuMigrationParamsSetULL(migParams,
6021 QEMU_MIGRATION_PARAM_MAX_BANDWIDTH,
6022 saveMigBandwidth * 1024 * 1024) == 0)
6023 ignore_value(qemuMigrationParamsApply(driver, vm, asyncJob,
6024 migParams));
6025 } else {
6026 if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) == 0) {
6027 qemuMonitorSetMigrationSpeed(priv->mon, saveMigBandwidth);
6028 ignore_value(qemuDomainObjExitMonitor(driver, vm));
6029 }
6030 }
6031 priv->migMaxBandwidth = saveMigBandwidth;
6032 }
6033
6034 VIR_FORCE_CLOSE(pipeFD[0]);
6035 VIR_FORCE_CLOSE(pipeFD[1]);
6036 if (errbuf) {
6037 VIR_DEBUG("Compression binary stderr: %s", NULLSTR(errbuf));
6038 VIR_FREE(errbuf);
6039 }
6040
6041 virErrorRestore(&orig_err);
6042
6043 return ret;
6044 }
6045
6046
6047 int
qemuMigrationSrcCancel(virQEMUDriver * driver,virDomainObj * vm)6048 qemuMigrationSrcCancel(virQEMUDriver *driver,
6049 virDomainObj *vm)
6050 {
6051 qemuDomainObjPrivate *priv = vm->privateData;
6052 bool storage = false;
6053 size_t i;
6054
6055 VIR_DEBUG("Canceling unfinished outgoing migration of domain %s",
6056 vm->def->name);
6057
6058 qemuDomainObjEnterMonitor(driver, vm);
6059 ignore_value(qemuMonitorMigrateCancel(priv->mon));
6060 if (qemuDomainObjExitMonitor(driver, vm) < 0)
6061 return -1;
6062
6063 for (i = 0; i < vm->def->ndisks; i++) {
6064 virDomainDiskDef *disk = vm->def->disks[i];
6065 qemuDomainDiskPrivate *diskPriv = QEMU_DOMAIN_DISK_PRIVATE(disk);
6066 qemuBlockJobData *job;
6067
6068 if (!(job = qemuBlockJobDiskGetJob(disk)) ||
6069 !qemuBlockJobIsRunning(job))
6070 diskPriv->migrating = false;
6071
6072 if (diskPriv->migrating) {
6073 qemuBlockJobSyncBegin(job);
6074 storage = true;
6075 }
6076
6077 virObjectUnref(job);
6078 }
6079
6080 if (storage &&
6081 qemuMigrationSrcNBDCopyCancel(driver, vm, true,
6082 QEMU_ASYNC_JOB_NONE, NULL) < 0)
6083 return -1;
6084
6085 if (qemuMigrationSrcCancelRemoveTempBitmaps(vm, QEMU_ASYNC_JOB_NONE) < 0)
6086 return -1;
6087
6088 return 0;
6089 }
6090
6091
6092 static int
qemuMigrationJobStart(virQEMUDriver * driver,virDomainObj * vm,qemuDomainAsyncJob job,unsigned long apiFlags)6093 qemuMigrationJobStart(virQEMUDriver *driver,
6094 virDomainObj *vm,
6095 qemuDomainAsyncJob job,
6096 unsigned long apiFlags)
6097 {
6098 qemuDomainObjPrivate *priv = vm->privateData;
6099 virDomainJobOperation op;
6100 unsigned long long mask;
6101
6102 if (job == QEMU_ASYNC_JOB_MIGRATION_IN) {
6103 op = VIR_DOMAIN_JOB_OPERATION_MIGRATION_IN;
6104 mask = QEMU_JOB_NONE;
6105 } else {
6106 op = VIR_DOMAIN_JOB_OPERATION_MIGRATION_OUT;
6107 mask = QEMU_JOB_DEFAULT_MASK |
6108 JOB_MASK(QEMU_JOB_SUSPEND) |
6109 JOB_MASK(QEMU_JOB_MIGRATION_OP);
6110 }
6111
6112 if (qemuDomainObjBeginAsyncJob(driver, vm, job, op, apiFlags) < 0)
6113 return -1;
6114
6115 priv->job.current->statsType = QEMU_DOMAIN_JOB_STATS_TYPE_MIGRATION;
6116
6117 qemuDomainObjSetAsyncJobMask(vm, mask);
6118 return 0;
6119 }
6120
6121 static void
qemuMigrationJobSetPhase(virQEMUDriver * driver,virDomainObj * vm,qemuMigrationJobPhase phase)6122 qemuMigrationJobSetPhase(virQEMUDriver *driver,
6123 virDomainObj *vm,
6124 qemuMigrationJobPhase phase)
6125 {
6126 qemuDomainObjPrivate *priv = vm->privateData;
6127
6128 if (phase < priv->job.phase) {
6129 VIR_ERROR(_("migration protocol going backwards %s => %s"),
6130 qemuMigrationJobPhaseTypeToString(priv->job.phase),
6131 qemuMigrationJobPhaseTypeToString(phase));
6132 return;
6133 }
6134
6135 qemuDomainObjSetJobPhase(driver, vm, phase);
6136 }
6137
6138 static void
qemuMigrationJobStartPhase(virQEMUDriver * driver,virDomainObj * vm,qemuMigrationJobPhase phase)6139 qemuMigrationJobStartPhase(virQEMUDriver *driver,
6140 virDomainObj *vm,
6141 qemuMigrationJobPhase phase)
6142 {
6143 qemuMigrationJobSetPhase(driver, vm, phase);
6144 }
6145
6146 static void
qemuMigrationJobContinue(virDomainObj * vm)6147 qemuMigrationJobContinue(virDomainObj *vm)
6148 {
6149 qemuDomainObjReleaseAsyncJob(vm);
6150 }
6151
6152 static bool
qemuMigrationJobIsActive(virDomainObj * vm,qemuDomainAsyncJob job)6153 qemuMigrationJobIsActive(virDomainObj *vm,
6154 qemuDomainAsyncJob job)
6155 {
6156 qemuDomainObjPrivate *priv = vm->privateData;
6157
6158 if (priv->job.asyncJob != job) {
6159 const char *msg;
6160
6161 if (job == QEMU_ASYNC_JOB_MIGRATION_IN)
6162 msg = _("domain '%s' is not processing incoming migration");
6163 else
6164 msg = _("domain '%s' is not being migrated");
6165
6166 virReportError(VIR_ERR_OPERATION_INVALID, msg, vm->def->name);
6167 return false;
6168 }
6169 return true;
6170 }
6171
6172 static void
qemuMigrationJobFinish(virQEMUDriver * driver,virDomainObj * vm)6173 qemuMigrationJobFinish(virQEMUDriver *driver, virDomainObj *vm)
6174 {
6175 qemuDomainObjEndAsyncJob(driver, vm);
6176 }
6177
6178
6179 static void
qemuMigrationDstErrorFree(void * data)6180 qemuMigrationDstErrorFree(void *data)
6181 {
6182 virErrorPtr err = data;
6183 virFreeError(err);
6184 }
6185
6186 int
qemuMigrationDstErrorInit(virQEMUDriver * driver)6187 qemuMigrationDstErrorInit(virQEMUDriver *driver)
6188 {
6189 driver->migrationErrors = virHashAtomicNew(qemuMigrationDstErrorFree);
6190 if (driver->migrationErrors)
6191 return 0;
6192 else
6193 return -1;
6194 }
6195
6196 /**
6197 * This function consumes @err; the caller should consider the @err pointer
6198 * invalid after calling this function.
6199 */
6200 void
qemuMigrationDstErrorSave(virQEMUDriver * driver,const char * name,virErrorPtr err)6201 qemuMigrationDstErrorSave(virQEMUDriver *driver,
6202 const char *name,
6203 virErrorPtr err)
6204 {
6205 if (!err)
6206 return;
6207
6208 VIR_DEBUG("Saving incoming migration error for domain %s: %s",
6209 name, err->message);
6210 if (virHashAtomicUpdate(driver->migrationErrors, name, err) < 0) {
6211 VIR_WARN("Failed to save migration error for domain '%s'", name);
6212 virFreeError(err);
6213 }
6214 }
6215
6216 void
qemuMigrationDstErrorReport(virQEMUDriver * driver,const char * name)6217 qemuMigrationDstErrorReport(virQEMUDriver *driver,
6218 const char *name)
6219 {
6220 virErrorPtr err;
6221
6222 if (!(err = virHashAtomicSteal(driver->migrationErrors, name)))
6223 return;
6224
6225 VIR_DEBUG("Restoring saved incoming migration error for domain %s: %s",
6226 name, err->message);
6227 virErrorRestore(&err);
6228 }
6229
6230
6231 int
qemuMigrationSrcFetchMirrorStats(virQEMUDriver * driver,virDomainObj * vm,qemuDomainAsyncJob asyncJob,qemuDomainJobInfo * jobInfo)6232 qemuMigrationSrcFetchMirrorStats(virQEMUDriver *driver,
6233 virDomainObj *vm,
6234 qemuDomainAsyncJob asyncJob,
6235 qemuDomainJobInfo *jobInfo)
6236 {
6237 size_t i;
6238 qemuDomainObjPrivate *priv = vm->privateData;
6239 bool nbd = false;
6240 GHashTable *blockinfo = NULL;
6241 qemuDomainMirrorStats *stats = &jobInfo->mirrorStats;
6242
6243 for (i = 0; i < vm->def->ndisks; i++) {
6244 virDomainDiskDef *disk = vm->def->disks[i];
6245 if (QEMU_DOMAIN_DISK_PRIVATE(disk)->migrating) {
6246 nbd = true;
6247 break;
6248 }
6249 }
6250
6251 if (!nbd)
6252 return 0;
6253
6254 if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
6255 return -1;
6256
6257 blockinfo = qemuMonitorGetAllBlockJobInfo(priv->mon, false);
6258
6259 if (qemuDomainObjExitMonitor(driver, vm) < 0 || !blockinfo)
6260 return -1;
6261
6262 memset(stats, 0, sizeof(*stats));
6263
6264 for (i = 0; i < vm->def->ndisks; i++) {
6265 virDomainDiskDef *disk = vm->def->disks[i];
6266 qemuDomainDiskPrivate *diskPriv = QEMU_DOMAIN_DISK_PRIVATE(disk);
6267 qemuMonitorBlockJobInfo *data;
6268
6269 if (!diskPriv->migrating ||
6270 !(data = virHashLookup(blockinfo, disk->info.alias)))
6271 continue;
6272
6273 stats->transferred += data->cur;
6274 stats->total += data->end;
6275 }
6276
6277 virHashFree(blockinfo);
6278 return 0;
6279 }
6280