1 /* 2 * vhost-user 3 * 4 * Copyright (c) 2013 Virtual Open Systems Sarl. 5 * 6 * This work is licensed under the terms of the GNU GPL, version 2 or later. 7 * See the COPYING file in the top-level directory. 8 * 9 */ 10 11 #include "qemu/osdep.h" 12 #include "qapi/error.h" 13 #include "hw/virtio/vhost.h" 14 #include "hw/virtio/vhost-user.h" 15 #include "hw/virtio/vhost-backend.h" 16 #include "hw/virtio/virtio.h" 17 #include "hw/virtio/virtio-net.h" 18 #include "chardev/char-fe.h" 19 #include "sysemu/kvm.h" 20 #include "qemu/error-report.h" 21 #include "qemu/main-loop.h" 22 #include "qemu/sockets.h" 23 #include "sysemu/cryptodev.h" 24 #include "migration/migration.h" 25 #include "migration/postcopy-ram.h" 26 #include "trace.h" 27 28 #include <sys/ioctl.h> 29 #include <sys/socket.h> 30 #include <sys/un.h> 31 32 #include "standard-headers/linux/vhost_types.h" 33 34 #ifdef CONFIG_LINUX 35 #include <linux/userfaultfd.h> 36 #endif 37 38 #define VHOST_MEMORY_MAX_NREGIONS 8 39 #define VHOST_USER_F_PROTOCOL_FEATURES 30 40 #define VHOST_USER_SLAVE_MAX_FDS 8 41 42 /* 43 * Maximum size of virtio device config space 44 */ 45 #define VHOST_USER_MAX_CONFIG_SIZE 256 46 47 enum VhostUserProtocolFeature { 48 VHOST_USER_PROTOCOL_F_MQ = 0, 49 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1, 50 VHOST_USER_PROTOCOL_F_RARP = 2, 51 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3, 52 VHOST_USER_PROTOCOL_F_NET_MTU = 4, 53 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5, 54 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6, 55 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7, 56 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8, 57 VHOST_USER_PROTOCOL_F_CONFIG = 9, 58 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10, 59 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11, 60 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12, 61 VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13, 62 VHOST_USER_PROTOCOL_F_MAX 63 }; 64 65 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1) 66 67 typedef enum VhostUserRequest { 68 VHOST_USER_NONE = 0, 69 VHOST_USER_GET_FEATURES = 1, 70 VHOST_USER_SET_FEATURES = 2, 71 VHOST_USER_SET_OWNER = 3, 72 VHOST_USER_RESET_OWNER = 4, 73 VHOST_USER_SET_MEM_TABLE = 5, 74 VHOST_USER_SET_LOG_BASE = 6, 75 VHOST_USER_SET_LOG_FD = 7, 76 VHOST_USER_SET_VRING_NUM = 8, 77 VHOST_USER_SET_VRING_ADDR = 9, 78 VHOST_USER_SET_VRING_BASE = 10, 79 VHOST_USER_GET_VRING_BASE = 11, 80 VHOST_USER_SET_VRING_KICK = 12, 81 VHOST_USER_SET_VRING_CALL = 13, 82 VHOST_USER_SET_VRING_ERR = 14, 83 VHOST_USER_GET_PROTOCOL_FEATURES = 15, 84 VHOST_USER_SET_PROTOCOL_FEATURES = 16, 85 VHOST_USER_GET_QUEUE_NUM = 17, 86 VHOST_USER_SET_VRING_ENABLE = 18, 87 VHOST_USER_SEND_RARP = 19, 88 VHOST_USER_NET_SET_MTU = 20, 89 VHOST_USER_SET_SLAVE_REQ_FD = 21, 90 VHOST_USER_IOTLB_MSG = 22, 91 VHOST_USER_SET_VRING_ENDIAN = 23, 92 VHOST_USER_GET_CONFIG = 24, 93 VHOST_USER_SET_CONFIG = 25, 94 VHOST_USER_CREATE_CRYPTO_SESSION = 26, 95 VHOST_USER_CLOSE_CRYPTO_SESSION = 27, 96 VHOST_USER_POSTCOPY_ADVISE = 28, 97 VHOST_USER_POSTCOPY_LISTEN = 29, 98 VHOST_USER_POSTCOPY_END = 30, 99 VHOST_USER_GET_INFLIGHT_FD = 31, 100 VHOST_USER_SET_INFLIGHT_FD = 32, 101 VHOST_USER_GPU_SET_SOCKET = 33, 102 VHOST_USER_RESET_DEVICE = 34, 103 VHOST_USER_MAX 104 } VhostUserRequest; 105 106 typedef enum VhostUserSlaveRequest { 107 VHOST_USER_SLAVE_NONE = 0, 108 VHOST_USER_SLAVE_IOTLB_MSG = 1, 109 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2, 110 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3, 111 VHOST_USER_SLAVE_MAX 112 } VhostUserSlaveRequest; 113 114 typedef struct VhostUserMemoryRegion { 115 uint64_t guest_phys_addr; 116 uint64_t memory_size; 117 uint64_t userspace_addr; 118 uint64_t mmap_offset; 119 } VhostUserMemoryRegion; 120 121 typedef struct VhostUserMemory { 122 uint32_t nregions; 123 uint32_t padding; 124 VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS]; 125 } VhostUserMemory; 126 127 typedef struct VhostUserLog { 128 uint64_t mmap_size; 129 uint64_t mmap_offset; 130 } VhostUserLog; 131 132 typedef struct VhostUserConfig { 133 uint32_t offset; 134 uint32_t size; 135 uint32_t flags; 136 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE]; 137 } VhostUserConfig; 138 139 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512 140 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64 141 142 typedef struct VhostUserCryptoSession { 143 /* session id for success, -1 on errors */ 144 int64_t session_id; 145 CryptoDevBackendSymSessionInfo session_setup_data; 146 uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN]; 147 uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN]; 148 } VhostUserCryptoSession; 149 150 static VhostUserConfig c __attribute__ ((unused)); 151 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \ 152 + sizeof(c.size) \ 153 + sizeof(c.flags)) 154 155 typedef struct VhostUserVringArea { 156 uint64_t u64; 157 uint64_t size; 158 uint64_t offset; 159 } VhostUserVringArea; 160 161 typedef struct VhostUserInflight { 162 uint64_t mmap_size; 163 uint64_t mmap_offset; 164 uint16_t num_queues; 165 uint16_t queue_size; 166 } VhostUserInflight; 167 168 typedef struct { 169 VhostUserRequest request; 170 171 #define VHOST_USER_VERSION_MASK (0x3) 172 #define VHOST_USER_REPLY_MASK (0x1<<2) 173 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3) 174 uint32_t flags; 175 uint32_t size; /* the following payload size */ 176 } QEMU_PACKED VhostUserHeader; 177 178 typedef union { 179 #define VHOST_USER_VRING_IDX_MASK (0xff) 180 #define VHOST_USER_VRING_NOFD_MASK (0x1<<8) 181 uint64_t u64; 182 struct vhost_vring_state state; 183 struct vhost_vring_addr addr; 184 VhostUserMemory memory; 185 VhostUserLog log; 186 struct vhost_iotlb_msg iotlb; 187 VhostUserConfig config; 188 VhostUserCryptoSession session; 189 VhostUserVringArea area; 190 VhostUserInflight inflight; 191 } VhostUserPayload; 192 193 typedef struct VhostUserMsg { 194 VhostUserHeader hdr; 195 VhostUserPayload payload; 196 } QEMU_PACKED VhostUserMsg; 197 198 static VhostUserMsg m __attribute__ ((unused)); 199 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader)) 200 201 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload)) 202 203 /* The version of the protocol we support */ 204 #define VHOST_USER_VERSION (0x1) 205 206 struct vhost_user { 207 struct vhost_dev *dev; 208 /* Shared between vhost devs of the same virtio device */ 209 VhostUserState *user; 210 int slave_fd; 211 NotifierWithReturn postcopy_notifier; 212 struct PostCopyFD postcopy_fd; 213 uint64_t postcopy_client_bases[VHOST_MEMORY_MAX_NREGIONS]; 214 /* Length of the region_rb and region_rb_offset arrays */ 215 size_t region_rb_len; 216 /* RAMBlock associated with a given region */ 217 RAMBlock **region_rb; 218 /* The offset from the start of the RAMBlock to the start of the 219 * vhost region. 220 */ 221 ram_addr_t *region_rb_offset; 222 223 /* True once we've entered postcopy_listen */ 224 bool postcopy_listen; 225 }; 226 227 static bool ioeventfd_enabled(void) 228 { 229 return !kvm_enabled() || kvm_eventfds_enabled(); 230 } 231 232 static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg) 233 { 234 struct vhost_user *u = dev->opaque; 235 CharBackend *chr = u->user->chr; 236 uint8_t *p = (uint8_t *) msg; 237 int r, size = VHOST_USER_HDR_SIZE; 238 239 r = qemu_chr_fe_read_all(chr, p, size); 240 if (r != size) { 241 error_report("Failed to read msg header. Read %d instead of %d." 242 " Original request %d.", r, size, msg->hdr.request); 243 return -1; 244 } 245 246 /* validate received flags */ 247 if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) { 248 error_report("Failed to read msg header." 249 " Flags 0x%x instead of 0x%x.", msg->hdr.flags, 250 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION); 251 return -1; 252 } 253 254 return 0; 255 } 256 257 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg) 258 { 259 struct vhost_user *u = dev->opaque; 260 CharBackend *chr = u->user->chr; 261 uint8_t *p = (uint8_t *) msg; 262 int r, size; 263 264 if (vhost_user_read_header(dev, msg) < 0) { 265 return -1; 266 } 267 268 /* validate message size is sane */ 269 if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) { 270 error_report("Failed to read msg header." 271 " Size %d exceeds the maximum %zu.", msg->hdr.size, 272 VHOST_USER_PAYLOAD_SIZE); 273 return -1; 274 } 275 276 if (msg->hdr.size) { 277 p += VHOST_USER_HDR_SIZE; 278 size = msg->hdr.size; 279 r = qemu_chr_fe_read_all(chr, p, size); 280 if (r != size) { 281 error_report("Failed to read msg payload." 282 " Read %d instead of %d.", r, msg->hdr.size); 283 return -1; 284 } 285 } 286 287 return 0; 288 } 289 290 static int process_message_reply(struct vhost_dev *dev, 291 const VhostUserMsg *msg) 292 { 293 VhostUserMsg msg_reply; 294 295 if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) { 296 return 0; 297 } 298 299 if (vhost_user_read(dev, &msg_reply) < 0) { 300 return -1; 301 } 302 303 if (msg_reply.hdr.request != msg->hdr.request) { 304 error_report("Received unexpected msg type." 305 "Expected %d received %d", 306 msg->hdr.request, msg_reply.hdr.request); 307 return -1; 308 } 309 310 return msg_reply.payload.u64 ? -1 : 0; 311 } 312 313 static bool vhost_user_one_time_request(VhostUserRequest request) 314 { 315 switch (request) { 316 case VHOST_USER_SET_OWNER: 317 case VHOST_USER_RESET_OWNER: 318 case VHOST_USER_SET_MEM_TABLE: 319 case VHOST_USER_GET_QUEUE_NUM: 320 case VHOST_USER_NET_SET_MTU: 321 return true; 322 default: 323 return false; 324 } 325 } 326 327 /* most non-init callers ignore the error */ 328 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg, 329 int *fds, int fd_num) 330 { 331 struct vhost_user *u = dev->opaque; 332 CharBackend *chr = u->user->chr; 333 int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size; 334 335 /* 336 * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE, 337 * we just need send it once in the first time. For later such 338 * request, we just ignore it. 339 */ 340 if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) { 341 msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK; 342 return 0; 343 } 344 345 if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) { 346 error_report("Failed to set msg fds."); 347 return -1; 348 } 349 350 ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size); 351 if (ret != size) { 352 error_report("Failed to write msg." 353 " Wrote %d instead of %d.", ret, size); 354 return -1; 355 } 356 357 return 0; 358 } 359 360 int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd) 361 { 362 VhostUserMsg msg = { 363 .hdr.request = VHOST_USER_GPU_SET_SOCKET, 364 .hdr.flags = VHOST_USER_VERSION, 365 }; 366 367 return vhost_user_write(dev, &msg, &fd, 1); 368 } 369 370 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base, 371 struct vhost_log *log) 372 { 373 int fds[VHOST_MEMORY_MAX_NREGIONS]; 374 size_t fd_num = 0; 375 bool shmfd = virtio_has_feature(dev->protocol_features, 376 VHOST_USER_PROTOCOL_F_LOG_SHMFD); 377 VhostUserMsg msg = { 378 .hdr.request = VHOST_USER_SET_LOG_BASE, 379 .hdr.flags = VHOST_USER_VERSION, 380 .payload.log.mmap_size = log->size * sizeof(*(log->log)), 381 .payload.log.mmap_offset = 0, 382 .hdr.size = sizeof(msg.payload.log), 383 }; 384 385 if (shmfd && log->fd != -1) { 386 fds[fd_num++] = log->fd; 387 } 388 389 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 390 return -1; 391 } 392 393 if (shmfd) { 394 msg.hdr.size = 0; 395 if (vhost_user_read(dev, &msg) < 0) { 396 return -1; 397 } 398 399 if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) { 400 error_report("Received unexpected msg type. " 401 "Expected %d received %d", 402 VHOST_USER_SET_LOG_BASE, msg.hdr.request); 403 return -1; 404 } 405 } 406 407 return 0; 408 } 409 410 static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev, 411 struct vhost_memory *mem) 412 { 413 struct vhost_user *u = dev->opaque; 414 int fds[VHOST_MEMORY_MAX_NREGIONS]; 415 int i, fd; 416 size_t fd_num = 0; 417 VhostUserMsg msg_reply; 418 int region_i, msg_i; 419 420 VhostUserMsg msg = { 421 .hdr.request = VHOST_USER_SET_MEM_TABLE, 422 .hdr.flags = VHOST_USER_VERSION, 423 }; 424 425 if (u->region_rb_len < dev->mem->nregions) { 426 u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions); 427 u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset, 428 dev->mem->nregions); 429 memset(&(u->region_rb[u->region_rb_len]), '\0', 430 sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len)); 431 memset(&(u->region_rb_offset[u->region_rb_len]), '\0', 432 sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len)); 433 u->region_rb_len = dev->mem->nregions; 434 } 435 436 for (i = 0; i < dev->mem->nregions; ++i) { 437 struct vhost_memory_region *reg = dev->mem->regions + i; 438 ram_addr_t offset; 439 MemoryRegion *mr; 440 441 assert((uintptr_t)reg->userspace_addr == reg->userspace_addr); 442 mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr, 443 &offset); 444 fd = memory_region_get_fd(mr); 445 if (fd > 0) { 446 assert(fd_num < VHOST_MEMORY_MAX_NREGIONS); 447 trace_vhost_user_set_mem_table_withfd(fd_num, mr->name, 448 reg->memory_size, 449 reg->guest_phys_addr, 450 reg->userspace_addr, offset); 451 u->region_rb_offset[i] = offset; 452 u->region_rb[i] = mr->ram_block; 453 msg.payload.memory.regions[fd_num].userspace_addr = 454 reg->userspace_addr; 455 msg.payload.memory.regions[fd_num].memory_size = reg->memory_size; 456 msg.payload.memory.regions[fd_num].guest_phys_addr = 457 reg->guest_phys_addr; 458 msg.payload.memory.regions[fd_num].mmap_offset = offset; 459 fds[fd_num++] = fd; 460 } else { 461 u->region_rb_offset[i] = 0; 462 u->region_rb[i] = NULL; 463 } 464 } 465 466 msg.payload.memory.nregions = fd_num; 467 468 if (!fd_num) { 469 error_report("Failed initializing vhost-user memory map, " 470 "consider using -object memory-backend-file share=on"); 471 return -1; 472 } 473 474 msg.hdr.size = sizeof(msg.payload.memory.nregions); 475 msg.hdr.size += sizeof(msg.payload.memory.padding); 476 msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion); 477 478 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 479 return -1; 480 } 481 482 if (vhost_user_read(dev, &msg_reply) < 0) { 483 return -1; 484 } 485 486 if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) { 487 error_report("%s: Received unexpected msg type." 488 "Expected %d received %d", __func__, 489 VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request); 490 return -1; 491 } 492 /* We're using the same structure, just reusing one of the 493 * fields, so it should be the same size. 494 */ 495 if (msg_reply.hdr.size != msg.hdr.size) { 496 error_report("%s: Unexpected size for postcopy reply " 497 "%d vs %d", __func__, msg_reply.hdr.size, msg.hdr.size); 498 return -1; 499 } 500 501 memset(u->postcopy_client_bases, 0, 502 sizeof(uint64_t) * VHOST_MEMORY_MAX_NREGIONS); 503 504 /* They're in the same order as the regions that were sent 505 * but some of the regions were skipped (above) if they 506 * didn't have fd's 507 */ 508 for (msg_i = 0, region_i = 0; 509 region_i < dev->mem->nregions; 510 region_i++) { 511 if (msg_i < fd_num && 512 msg_reply.payload.memory.regions[msg_i].guest_phys_addr == 513 dev->mem->regions[region_i].guest_phys_addr) { 514 u->postcopy_client_bases[region_i] = 515 msg_reply.payload.memory.regions[msg_i].userspace_addr; 516 trace_vhost_user_set_mem_table_postcopy( 517 msg_reply.payload.memory.regions[msg_i].userspace_addr, 518 msg.payload.memory.regions[msg_i].userspace_addr, 519 msg_i, region_i); 520 msg_i++; 521 } 522 } 523 if (msg_i != fd_num) { 524 error_report("%s: postcopy reply not fully consumed " 525 "%d vs %zd", 526 __func__, msg_i, fd_num); 527 return -1; 528 } 529 /* Now we've registered this with the postcopy code, we ack to the client, 530 * because now we're in the position to be able to deal with any faults 531 * it generates. 532 */ 533 /* TODO: Use this for failure cases as well with a bad value */ 534 msg.hdr.size = sizeof(msg.payload.u64); 535 msg.payload.u64 = 0; /* OK */ 536 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 537 return -1; 538 } 539 540 return 0; 541 } 542 543 static int vhost_user_set_mem_table(struct vhost_dev *dev, 544 struct vhost_memory *mem) 545 { 546 struct vhost_user *u = dev->opaque; 547 int fds[VHOST_MEMORY_MAX_NREGIONS]; 548 int i, fd; 549 size_t fd_num = 0; 550 bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler; 551 bool reply_supported = virtio_has_feature(dev->protocol_features, 552 VHOST_USER_PROTOCOL_F_REPLY_ACK); 553 554 if (do_postcopy) { 555 /* Postcopy has enough differences that it's best done in it's own 556 * version 557 */ 558 return vhost_user_set_mem_table_postcopy(dev, mem); 559 } 560 561 VhostUserMsg msg = { 562 .hdr.request = VHOST_USER_SET_MEM_TABLE, 563 .hdr.flags = VHOST_USER_VERSION, 564 }; 565 566 if (reply_supported) { 567 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 568 } 569 570 for (i = 0; i < dev->mem->nregions; ++i) { 571 struct vhost_memory_region *reg = dev->mem->regions + i; 572 ram_addr_t offset; 573 MemoryRegion *mr; 574 575 assert((uintptr_t)reg->userspace_addr == reg->userspace_addr); 576 mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr, 577 &offset); 578 fd = memory_region_get_fd(mr); 579 if (fd > 0) { 580 if (fd_num == VHOST_MEMORY_MAX_NREGIONS) { 581 error_report("Failed preparing vhost-user memory table msg"); 582 return -1; 583 } 584 msg.payload.memory.regions[fd_num].userspace_addr = 585 reg->userspace_addr; 586 msg.payload.memory.regions[fd_num].memory_size = reg->memory_size; 587 msg.payload.memory.regions[fd_num].guest_phys_addr = 588 reg->guest_phys_addr; 589 msg.payload.memory.regions[fd_num].mmap_offset = offset; 590 fds[fd_num++] = fd; 591 } 592 } 593 594 msg.payload.memory.nregions = fd_num; 595 596 if (!fd_num) { 597 error_report("Failed initializing vhost-user memory map, " 598 "consider using -object memory-backend-file share=on"); 599 return -1; 600 } 601 602 msg.hdr.size = sizeof(msg.payload.memory.nregions); 603 msg.hdr.size += sizeof(msg.payload.memory.padding); 604 msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion); 605 606 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 607 return -1; 608 } 609 610 if (reply_supported) { 611 return process_message_reply(dev, &msg); 612 } 613 614 return 0; 615 } 616 617 static int vhost_user_set_vring_addr(struct vhost_dev *dev, 618 struct vhost_vring_addr *addr) 619 { 620 VhostUserMsg msg = { 621 .hdr.request = VHOST_USER_SET_VRING_ADDR, 622 .hdr.flags = VHOST_USER_VERSION, 623 .payload.addr = *addr, 624 .hdr.size = sizeof(msg.payload.addr), 625 }; 626 627 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 628 return -1; 629 } 630 631 return 0; 632 } 633 634 static int vhost_user_set_vring_endian(struct vhost_dev *dev, 635 struct vhost_vring_state *ring) 636 { 637 bool cross_endian = virtio_has_feature(dev->protocol_features, 638 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN); 639 VhostUserMsg msg = { 640 .hdr.request = VHOST_USER_SET_VRING_ENDIAN, 641 .hdr.flags = VHOST_USER_VERSION, 642 .payload.state = *ring, 643 .hdr.size = sizeof(msg.payload.state), 644 }; 645 646 if (!cross_endian) { 647 error_report("vhost-user trying to send unhandled ioctl"); 648 return -1; 649 } 650 651 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 652 return -1; 653 } 654 655 return 0; 656 } 657 658 static int vhost_set_vring(struct vhost_dev *dev, 659 unsigned long int request, 660 struct vhost_vring_state *ring) 661 { 662 VhostUserMsg msg = { 663 .hdr.request = request, 664 .hdr.flags = VHOST_USER_VERSION, 665 .payload.state = *ring, 666 .hdr.size = sizeof(msg.payload.state), 667 }; 668 669 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 670 return -1; 671 } 672 673 return 0; 674 } 675 676 static int vhost_user_set_vring_num(struct vhost_dev *dev, 677 struct vhost_vring_state *ring) 678 { 679 return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring); 680 } 681 682 static void vhost_user_host_notifier_restore(struct vhost_dev *dev, 683 int queue_idx) 684 { 685 struct vhost_user *u = dev->opaque; 686 VhostUserHostNotifier *n = &u->user->notifier[queue_idx]; 687 VirtIODevice *vdev = dev->vdev; 688 689 if (n->addr && !n->set) { 690 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true); 691 n->set = true; 692 } 693 } 694 695 static void vhost_user_host_notifier_remove(struct vhost_dev *dev, 696 int queue_idx) 697 { 698 struct vhost_user *u = dev->opaque; 699 VhostUserHostNotifier *n = &u->user->notifier[queue_idx]; 700 VirtIODevice *vdev = dev->vdev; 701 702 if (n->addr && n->set) { 703 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false); 704 n->set = false; 705 } 706 } 707 708 static int vhost_user_set_vring_base(struct vhost_dev *dev, 709 struct vhost_vring_state *ring) 710 { 711 vhost_user_host_notifier_restore(dev, ring->index); 712 713 return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring); 714 } 715 716 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable) 717 { 718 int i; 719 720 if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) { 721 return -1; 722 } 723 724 for (i = 0; i < dev->nvqs; ++i) { 725 struct vhost_vring_state state = { 726 .index = dev->vq_index + i, 727 .num = enable, 728 }; 729 730 vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state); 731 } 732 733 return 0; 734 } 735 736 static int vhost_user_get_vring_base(struct vhost_dev *dev, 737 struct vhost_vring_state *ring) 738 { 739 VhostUserMsg msg = { 740 .hdr.request = VHOST_USER_GET_VRING_BASE, 741 .hdr.flags = VHOST_USER_VERSION, 742 .payload.state = *ring, 743 .hdr.size = sizeof(msg.payload.state), 744 }; 745 746 vhost_user_host_notifier_remove(dev, ring->index); 747 748 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 749 return -1; 750 } 751 752 if (vhost_user_read(dev, &msg) < 0) { 753 return -1; 754 } 755 756 if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) { 757 error_report("Received unexpected msg type. Expected %d received %d", 758 VHOST_USER_GET_VRING_BASE, msg.hdr.request); 759 return -1; 760 } 761 762 if (msg.hdr.size != sizeof(msg.payload.state)) { 763 error_report("Received bad msg size."); 764 return -1; 765 } 766 767 *ring = msg.payload.state; 768 769 return 0; 770 } 771 772 static int vhost_set_vring_file(struct vhost_dev *dev, 773 VhostUserRequest request, 774 struct vhost_vring_file *file) 775 { 776 int fds[VHOST_MEMORY_MAX_NREGIONS]; 777 size_t fd_num = 0; 778 VhostUserMsg msg = { 779 .hdr.request = request, 780 .hdr.flags = VHOST_USER_VERSION, 781 .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK, 782 .hdr.size = sizeof(msg.payload.u64), 783 }; 784 785 if (ioeventfd_enabled() && file->fd > 0) { 786 fds[fd_num++] = file->fd; 787 } else { 788 msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK; 789 } 790 791 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 792 return -1; 793 } 794 795 return 0; 796 } 797 798 static int vhost_user_set_vring_kick(struct vhost_dev *dev, 799 struct vhost_vring_file *file) 800 { 801 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file); 802 } 803 804 static int vhost_user_set_vring_call(struct vhost_dev *dev, 805 struct vhost_vring_file *file) 806 { 807 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file); 808 } 809 810 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) 811 { 812 VhostUserMsg msg = { 813 .hdr.request = request, 814 .hdr.flags = VHOST_USER_VERSION, 815 .payload.u64 = u64, 816 .hdr.size = sizeof(msg.payload.u64), 817 }; 818 819 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 820 return -1; 821 } 822 823 return 0; 824 } 825 826 static int vhost_user_set_features(struct vhost_dev *dev, 827 uint64_t features) 828 { 829 return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features); 830 } 831 832 static int vhost_user_set_protocol_features(struct vhost_dev *dev, 833 uint64_t features) 834 { 835 return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features); 836 } 837 838 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64) 839 { 840 VhostUserMsg msg = { 841 .hdr.request = request, 842 .hdr.flags = VHOST_USER_VERSION, 843 }; 844 845 if (vhost_user_one_time_request(request) && dev->vq_index != 0) { 846 return 0; 847 } 848 849 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 850 return -1; 851 } 852 853 if (vhost_user_read(dev, &msg) < 0) { 854 return -1; 855 } 856 857 if (msg.hdr.request != request) { 858 error_report("Received unexpected msg type. Expected %d received %d", 859 request, msg.hdr.request); 860 return -1; 861 } 862 863 if (msg.hdr.size != sizeof(msg.payload.u64)) { 864 error_report("Received bad msg size."); 865 return -1; 866 } 867 868 *u64 = msg.payload.u64; 869 870 return 0; 871 } 872 873 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features) 874 { 875 return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features); 876 } 877 878 static int vhost_user_set_owner(struct vhost_dev *dev) 879 { 880 VhostUserMsg msg = { 881 .hdr.request = VHOST_USER_SET_OWNER, 882 .hdr.flags = VHOST_USER_VERSION, 883 }; 884 885 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 886 return -1; 887 } 888 889 return 0; 890 } 891 892 static int vhost_user_reset_device(struct vhost_dev *dev) 893 { 894 VhostUserMsg msg = { 895 .hdr.flags = VHOST_USER_VERSION, 896 }; 897 898 msg.hdr.request = virtio_has_feature(dev->protocol_features, 899 VHOST_USER_PROTOCOL_F_RESET_DEVICE) 900 ? VHOST_USER_RESET_DEVICE 901 : VHOST_USER_RESET_OWNER; 902 903 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 904 return -1; 905 } 906 907 return 0; 908 } 909 910 static int vhost_user_slave_handle_config_change(struct vhost_dev *dev) 911 { 912 int ret = -1; 913 914 if (!dev->config_ops) { 915 return -1; 916 } 917 918 if (dev->config_ops->vhost_dev_config_notifier) { 919 ret = dev->config_ops->vhost_dev_config_notifier(dev); 920 } 921 922 return ret; 923 } 924 925 static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev, 926 VhostUserVringArea *area, 927 int fd) 928 { 929 int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK; 930 size_t page_size = qemu_real_host_page_size; 931 struct vhost_user *u = dev->opaque; 932 VhostUserState *user = u->user; 933 VirtIODevice *vdev = dev->vdev; 934 VhostUserHostNotifier *n; 935 void *addr; 936 char *name; 937 938 if (!virtio_has_feature(dev->protocol_features, 939 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) || 940 vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) { 941 return -1; 942 } 943 944 n = &user->notifier[queue_idx]; 945 946 if (n->addr) { 947 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false); 948 object_unparent(OBJECT(&n->mr)); 949 munmap(n->addr, page_size); 950 n->addr = NULL; 951 } 952 953 if (area->u64 & VHOST_USER_VRING_NOFD_MASK) { 954 return 0; 955 } 956 957 /* Sanity check. */ 958 if (area->size != page_size) { 959 return -1; 960 } 961 962 addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, 963 fd, area->offset); 964 if (addr == MAP_FAILED) { 965 return -1; 966 } 967 968 name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]", 969 user, queue_idx); 970 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, 971 page_size, addr); 972 g_free(name); 973 974 if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) { 975 munmap(addr, page_size); 976 return -1; 977 } 978 979 n->addr = addr; 980 n->set = true; 981 982 return 0; 983 } 984 985 static void slave_read(void *opaque) 986 { 987 struct vhost_dev *dev = opaque; 988 struct vhost_user *u = dev->opaque; 989 VhostUserHeader hdr = { 0, }; 990 VhostUserPayload payload = { 0, }; 991 int size, ret = 0; 992 struct iovec iov; 993 struct msghdr msgh; 994 int fd[VHOST_USER_SLAVE_MAX_FDS]; 995 char control[CMSG_SPACE(sizeof(fd))]; 996 struct cmsghdr *cmsg; 997 int i, fdsize = 0; 998 999 memset(&msgh, 0, sizeof(msgh)); 1000 msgh.msg_iov = &iov; 1001 msgh.msg_iovlen = 1; 1002 msgh.msg_control = control; 1003 msgh.msg_controllen = sizeof(control); 1004 1005 memset(fd, -1, sizeof(fd)); 1006 1007 /* Read header */ 1008 iov.iov_base = &hdr; 1009 iov.iov_len = VHOST_USER_HDR_SIZE; 1010 1011 do { 1012 size = recvmsg(u->slave_fd, &msgh, 0); 1013 } while (size < 0 && (errno == EINTR || errno == EAGAIN)); 1014 1015 if (size != VHOST_USER_HDR_SIZE) { 1016 error_report("Failed to read from slave."); 1017 goto err; 1018 } 1019 1020 if (msgh.msg_flags & MSG_CTRUNC) { 1021 error_report("Truncated message."); 1022 goto err; 1023 } 1024 1025 for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL; 1026 cmsg = CMSG_NXTHDR(&msgh, cmsg)) { 1027 if (cmsg->cmsg_level == SOL_SOCKET && 1028 cmsg->cmsg_type == SCM_RIGHTS) { 1029 fdsize = cmsg->cmsg_len - CMSG_LEN(0); 1030 memcpy(fd, CMSG_DATA(cmsg), fdsize); 1031 break; 1032 } 1033 } 1034 1035 if (hdr.size > VHOST_USER_PAYLOAD_SIZE) { 1036 error_report("Failed to read msg header." 1037 " Size %d exceeds the maximum %zu.", hdr.size, 1038 VHOST_USER_PAYLOAD_SIZE); 1039 goto err; 1040 } 1041 1042 /* Read payload */ 1043 do { 1044 size = read(u->slave_fd, &payload, hdr.size); 1045 } while (size < 0 && (errno == EINTR || errno == EAGAIN)); 1046 1047 if (size != hdr.size) { 1048 error_report("Failed to read payload from slave."); 1049 goto err; 1050 } 1051 1052 switch (hdr.request) { 1053 case VHOST_USER_SLAVE_IOTLB_MSG: 1054 ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb); 1055 break; 1056 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG : 1057 ret = vhost_user_slave_handle_config_change(dev); 1058 break; 1059 case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG: 1060 ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area, 1061 fd[0]); 1062 break; 1063 default: 1064 error_report("Received unexpected msg type: %d.", hdr.request); 1065 ret = -EINVAL; 1066 } 1067 1068 /* Close the remaining file descriptors. */ 1069 for (i = 0; i < fdsize; i++) { 1070 if (fd[i] != -1) { 1071 close(fd[i]); 1072 } 1073 } 1074 1075 /* 1076 * REPLY_ACK feature handling. Other reply types has to be managed 1077 * directly in their request handlers. 1078 */ 1079 if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) { 1080 struct iovec iovec[2]; 1081 1082 1083 hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK; 1084 hdr.flags |= VHOST_USER_REPLY_MASK; 1085 1086 payload.u64 = !!ret; 1087 hdr.size = sizeof(payload.u64); 1088 1089 iovec[0].iov_base = &hdr; 1090 iovec[0].iov_len = VHOST_USER_HDR_SIZE; 1091 iovec[1].iov_base = &payload; 1092 iovec[1].iov_len = hdr.size; 1093 1094 do { 1095 size = writev(u->slave_fd, iovec, ARRAY_SIZE(iovec)); 1096 } while (size < 0 && (errno == EINTR || errno == EAGAIN)); 1097 1098 if (size != VHOST_USER_HDR_SIZE + hdr.size) { 1099 error_report("Failed to send msg reply to slave."); 1100 goto err; 1101 } 1102 } 1103 1104 return; 1105 1106 err: 1107 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); 1108 close(u->slave_fd); 1109 u->slave_fd = -1; 1110 for (i = 0; i < fdsize; i++) { 1111 if (fd[i] != -1) { 1112 close(fd[i]); 1113 } 1114 } 1115 return; 1116 } 1117 1118 static int vhost_setup_slave_channel(struct vhost_dev *dev) 1119 { 1120 VhostUserMsg msg = { 1121 .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD, 1122 .hdr.flags = VHOST_USER_VERSION, 1123 }; 1124 struct vhost_user *u = dev->opaque; 1125 int sv[2], ret = 0; 1126 bool reply_supported = virtio_has_feature(dev->protocol_features, 1127 VHOST_USER_PROTOCOL_F_REPLY_ACK); 1128 1129 if (!virtio_has_feature(dev->protocol_features, 1130 VHOST_USER_PROTOCOL_F_SLAVE_REQ)) { 1131 return 0; 1132 } 1133 1134 if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) { 1135 error_report("socketpair() failed"); 1136 return -1; 1137 } 1138 1139 u->slave_fd = sv[0]; 1140 qemu_set_fd_handler(u->slave_fd, slave_read, NULL, dev); 1141 1142 if (reply_supported) { 1143 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 1144 } 1145 1146 ret = vhost_user_write(dev, &msg, &sv[1], 1); 1147 if (ret) { 1148 goto out; 1149 } 1150 1151 if (reply_supported) { 1152 ret = process_message_reply(dev, &msg); 1153 } 1154 1155 out: 1156 close(sv[1]); 1157 if (ret) { 1158 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); 1159 close(u->slave_fd); 1160 u->slave_fd = -1; 1161 } 1162 1163 return ret; 1164 } 1165 1166 #ifdef CONFIG_LINUX 1167 /* 1168 * Called back from the postcopy fault thread when a fault is received on our 1169 * ufd. 1170 * TODO: This is Linux specific 1171 */ 1172 static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd, 1173 void *ufd) 1174 { 1175 struct vhost_dev *dev = pcfd->data; 1176 struct vhost_user *u = dev->opaque; 1177 struct uffd_msg *msg = ufd; 1178 uint64_t faultaddr = msg->arg.pagefault.address; 1179 RAMBlock *rb = NULL; 1180 uint64_t rb_offset; 1181 int i; 1182 1183 trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr, 1184 dev->mem->nregions); 1185 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) { 1186 trace_vhost_user_postcopy_fault_handler_loop(i, 1187 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size); 1188 if (faultaddr >= u->postcopy_client_bases[i]) { 1189 /* Ofset of the fault address in the vhost region */ 1190 uint64_t region_offset = faultaddr - u->postcopy_client_bases[i]; 1191 if (region_offset < dev->mem->regions[i].memory_size) { 1192 rb_offset = region_offset + u->region_rb_offset[i]; 1193 trace_vhost_user_postcopy_fault_handler_found(i, 1194 region_offset, rb_offset); 1195 rb = u->region_rb[i]; 1196 return postcopy_request_shared_page(pcfd, rb, faultaddr, 1197 rb_offset); 1198 } 1199 } 1200 } 1201 error_report("%s: Failed to find region for fault %" PRIx64, 1202 __func__, faultaddr); 1203 return -1; 1204 } 1205 1206 static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb, 1207 uint64_t offset) 1208 { 1209 struct vhost_dev *dev = pcfd->data; 1210 struct vhost_user *u = dev->opaque; 1211 int i; 1212 1213 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset); 1214 1215 if (!u) { 1216 return 0; 1217 } 1218 /* Translate the offset into an address in the clients address space */ 1219 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) { 1220 if (u->region_rb[i] == rb && 1221 offset >= u->region_rb_offset[i] && 1222 offset < (u->region_rb_offset[i] + 1223 dev->mem->regions[i].memory_size)) { 1224 uint64_t client_addr = (offset - u->region_rb_offset[i]) + 1225 u->postcopy_client_bases[i]; 1226 trace_vhost_user_postcopy_waker_found(client_addr); 1227 return postcopy_wake_shared(pcfd, client_addr, rb); 1228 } 1229 } 1230 1231 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset); 1232 return 0; 1233 } 1234 #endif 1235 1236 /* 1237 * Called at the start of an inbound postcopy on reception of the 1238 * 'advise' command. 1239 */ 1240 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp) 1241 { 1242 #ifdef CONFIG_LINUX 1243 struct vhost_user *u = dev->opaque; 1244 CharBackend *chr = u->user->chr; 1245 int ufd; 1246 VhostUserMsg msg = { 1247 .hdr.request = VHOST_USER_POSTCOPY_ADVISE, 1248 .hdr.flags = VHOST_USER_VERSION, 1249 }; 1250 1251 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1252 error_setg(errp, "Failed to send postcopy_advise to vhost"); 1253 return -1; 1254 } 1255 1256 if (vhost_user_read(dev, &msg) < 0) { 1257 error_setg(errp, "Failed to get postcopy_advise reply from vhost"); 1258 return -1; 1259 } 1260 1261 if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) { 1262 error_setg(errp, "Unexpected msg type. Expected %d received %d", 1263 VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request); 1264 return -1; 1265 } 1266 1267 if (msg.hdr.size) { 1268 error_setg(errp, "Received bad msg size."); 1269 return -1; 1270 } 1271 ufd = qemu_chr_fe_get_msgfd(chr); 1272 if (ufd < 0) { 1273 error_setg(errp, "%s: Failed to get ufd", __func__); 1274 return -1; 1275 } 1276 qemu_set_nonblock(ufd); 1277 1278 /* register ufd with userfault thread */ 1279 u->postcopy_fd.fd = ufd; 1280 u->postcopy_fd.data = dev; 1281 u->postcopy_fd.handler = vhost_user_postcopy_fault_handler; 1282 u->postcopy_fd.waker = vhost_user_postcopy_waker; 1283 u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */ 1284 postcopy_register_shared_ufd(&u->postcopy_fd); 1285 return 0; 1286 #else 1287 error_setg(errp, "Postcopy not supported on non-Linux systems"); 1288 return -1; 1289 #endif 1290 } 1291 1292 /* 1293 * Called at the switch to postcopy on reception of the 'listen' command. 1294 */ 1295 static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp) 1296 { 1297 struct vhost_user *u = dev->opaque; 1298 int ret; 1299 VhostUserMsg msg = { 1300 .hdr.request = VHOST_USER_POSTCOPY_LISTEN, 1301 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, 1302 }; 1303 u->postcopy_listen = true; 1304 trace_vhost_user_postcopy_listen(); 1305 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1306 error_setg(errp, "Failed to send postcopy_listen to vhost"); 1307 return -1; 1308 } 1309 1310 ret = process_message_reply(dev, &msg); 1311 if (ret) { 1312 error_setg(errp, "Failed to receive reply to postcopy_listen"); 1313 return ret; 1314 } 1315 1316 return 0; 1317 } 1318 1319 /* 1320 * Called at the end of postcopy 1321 */ 1322 static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp) 1323 { 1324 VhostUserMsg msg = { 1325 .hdr.request = VHOST_USER_POSTCOPY_END, 1326 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, 1327 }; 1328 int ret; 1329 struct vhost_user *u = dev->opaque; 1330 1331 trace_vhost_user_postcopy_end_entry(); 1332 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1333 error_setg(errp, "Failed to send postcopy_end to vhost"); 1334 return -1; 1335 } 1336 1337 ret = process_message_reply(dev, &msg); 1338 if (ret) { 1339 error_setg(errp, "Failed to receive reply to postcopy_end"); 1340 return ret; 1341 } 1342 postcopy_unregister_shared_ufd(&u->postcopy_fd); 1343 close(u->postcopy_fd.fd); 1344 u->postcopy_fd.handler = NULL; 1345 1346 trace_vhost_user_postcopy_end_exit(); 1347 1348 return 0; 1349 } 1350 1351 static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier, 1352 void *opaque) 1353 { 1354 struct PostcopyNotifyData *pnd = opaque; 1355 struct vhost_user *u = container_of(notifier, struct vhost_user, 1356 postcopy_notifier); 1357 struct vhost_dev *dev = u->dev; 1358 1359 switch (pnd->reason) { 1360 case POSTCOPY_NOTIFY_PROBE: 1361 if (!virtio_has_feature(dev->protocol_features, 1362 VHOST_USER_PROTOCOL_F_PAGEFAULT)) { 1363 /* TODO: Get the device name into this error somehow */ 1364 error_setg(pnd->errp, 1365 "vhost-user backend not capable of postcopy"); 1366 return -ENOENT; 1367 } 1368 break; 1369 1370 case POSTCOPY_NOTIFY_INBOUND_ADVISE: 1371 return vhost_user_postcopy_advise(dev, pnd->errp); 1372 1373 case POSTCOPY_NOTIFY_INBOUND_LISTEN: 1374 return vhost_user_postcopy_listen(dev, pnd->errp); 1375 1376 case POSTCOPY_NOTIFY_INBOUND_END: 1377 return vhost_user_postcopy_end(dev, pnd->errp); 1378 1379 default: 1380 /* We ignore notifications we don't know */ 1381 break; 1382 } 1383 1384 return 0; 1385 } 1386 1387 static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque) 1388 { 1389 uint64_t features, protocol_features; 1390 struct vhost_user *u; 1391 int err; 1392 1393 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1394 1395 u = g_new0(struct vhost_user, 1); 1396 u->user = opaque; 1397 u->slave_fd = -1; 1398 u->dev = dev; 1399 dev->opaque = u; 1400 1401 err = vhost_user_get_features(dev, &features); 1402 if (err < 0) { 1403 return err; 1404 } 1405 1406 if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) { 1407 dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES; 1408 1409 err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES, 1410 &protocol_features); 1411 if (err < 0) { 1412 return err; 1413 } 1414 1415 dev->protocol_features = 1416 protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK; 1417 1418 if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) { 1419 /* Don't acknowledge CONFIG feature if device doesn't support it */ 1420 dev->protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG); 1421 } else if (!(protocol_features & 1422 (1ULL << VHOST_USER_PROTOCOL_F_CONFIG))) { 1423 error_report("Device expects VHOST_USER_PROTOCOL_F_CONFIG " 1424 "but backend does not support it."); 1425 return -1; 1426 } 1427 1428 err = vhost_user_set_protocol_features(dev, dev->protocol_features); 1429 if (err < 0) { 1430 return err; 1431 } 1432 1433 /* query the max queues we support if backend supports Multiple Queue */ 1434 if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) { 1435 err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM, 1436 &dev->max_queues); 1437 if (err < 0) { 1438 return err; 1439 } 1440 } 1441 1442 if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) && 1443 !(virtio_has_feature(dev->protocol_features, 1444 VHOST_USER_PROTOCOL_F_SLAVE_REQ) && 1445 virtio_has_feature(dev->protocol_features, 1446 VHOST_USER_PROTOCOL_F_REPLY_ACK))) { 1447 error_report("IOMMU support requires reply-ack and " 1448 "slave-req protocol features."); 1449 return -1; 1450 } 1451 } 1452 1453 if (dev->migration_blocker == NULL && 1454 !virtio_has_feature(dev->protocol_features, 1455 VHOST_USER_PROTOCOL_F_LOG_SHMFD)) { 1456 error_setg(&dev->migration_blocker, 1457 "Migration disabled: vhost-user backend lacks " 1458 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature."); 1459 } 1460 1461 if (dev->vq_index == 0) { 1462 err = vhost_setup_slave_channel(dev); 1463 if (err < 0) { 1464 return err; 1465 } 1466 } 1467 1468 u->postcopy_notifier.notify = vhost_user_postcopy_notifier; 1469 postcopy_add_notifier(&u->postcopy_notifier); 1470 1471 return 0; 1472 } 1473 1474 static int vhost_user_backend_cleanup(struct vhost_dev *dev) 1475 { 1476 struct vhost_user *u; 1477 1478 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1479 1480 u = dev->opaque; 1481 if (u->postcopy_notifier.notify) { 1482 postcopy_remove_notifier(&u->postcopy_notifier); 1483 u->postcopy_notifier.notify = NULL; 1484 } 1485 u->postcopy_listen = false; 1486 if (u->postcopy_fd.handler) { 1487 postcopy_unregister_shared_ufd(&u->postcopy_fd); 1488 close(u->postcopy_fd.fd); 1489 u->postcopy_fd.handler = NULL; 1490 } 1491 if (u->slave_fd >= 0) { 1492 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); 1493 close(u->slave_fd); 1494 u->slave_fd = -1; 1495 } 1496 g_free(u->region_rb); 1497 u->region_rb = NULL; 1498 g_free(u->region_rb_offset); 1499 u->region_rb_offset = NULL; 1500 u->region_rb_len = 0; 1501 g_free(u); 1502 dev->opaque = 0; 1503 1504 return 0; 1505 } 1506 1507 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx) 1508 { 1509 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 1510 1511 return idx; 1512 } 1513 1514 static int vhost_user_memslots_limit(struct vhost_dev *dev) 1515 { 1516 return VHOST_MEMORY_MAX_NREGIONS; 1517 } 1518 1519 static bool vhost_user_requires_shm_log(struct vhost_dev *dev) 1520 { 1521 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1522 1523 return virtio_has_feature(dev->protocol_features, 1524 VHOST_USER_PROTOCOL_F_LOG_SHMFD); 1525 } 1526 1527 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr) 1528 { 1529 VhostUserMsg msg = { }; 1530 1531 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1532 1533 /* If guest supports GUEST_ANNOUNCE do nothing */ 1534 if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) { 1535 return 0; 1536 } 1537 1538 /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */ 1539 if (virtio_has_feature(dev->protocol_features, 1540 VHOST_USER_PROTOCOL_F_RARP)) { 1541 msg.hdr.request = VHOST_USER_SEND_RARP; 1542 msg.hdr.flags = VHOST_USER_VERSION; 1543 memcpy((char *)&msg.payload.u64, mac_addr, 6); 1544 msg.hdr.size = sizeof(msg.payload.u64); 1545 1546 return vhost_user_write(dev, &msg, NULL, 0); 1547 } 1548 return -1; 1549 } 1550 1551 static bool vhost_user_can_merge(struct vhost_dev *dev, 1552 uint64_t start1, uint64_t size1, 1553 uint64_t start2, uint64_t size2) 1554 { 1555 ram_addr_t offset; 1556 int mfd, rfd; 1557 MemoryRegion *mr; 1558 1559 mr = memory_region_from_host((void *)(uintptr_t)start1, &offset); 1560 mfd = memory_region_get_fd(mr); 1561 1562 mr = memory_region_from_host((void *)(uintptr_t)start2, &offset); 1563 rfd = memory_region_get_fd(mr); 1564 1565 return mfd == rfd; 1566 } 1567 1568 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu) 1569 { 1570 VhostUserMsg msg; 1571 bool reply_supported = virtio_has_feature(dev->protocol_features, 1572 VHOST_USER_PROTOCOL_F_REPLY_ACK); 1573 1574 if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) { 1575 return 0; 1576 } 1577 1578 msg.hdr.request = VHOST_USER_NET_SET_MTU; 1579 msg.payload.u64 = mtu; 1580 msg.hdr.size = sizeof(msg.payload.u64); 1581 msg.hdr.flags = VHOST_USER_VERSION; 1582 if (reply_supported) { 1583 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 1584 } 1585 1586 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1587 return -1; 1588 } 1589 1590 /* If reply_ack supported, slave has to ack specified MTU is valid */ 1591 if (reply_supported) { 1592 return process_message_reply(dev, &msg); 1593 } 1594 1595 return 0; 1596 } 1597 1598 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev, 1599 struct vhost_iotlb_msg *imsg) 1600 { 1601 VhostUserMsg msg = { 1602 .hdr.request = VHOST_USER_IOTLB_MSG, 1603 .hdr.size = sizeof(msg.payload.iotlb), 1604 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, 1605 .payload.iotlb = *imsg, 1606 }; 1607 1608 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1609 return -EFAULT; 1610 } 1611 1612 return process_message_reply(dev, &msg); 1613 } 1614 1615 1616 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled) 1617 { 1618 /* No-op as the receive channel is not dedicated to IOTLB messages. */ 1619 } 1620 1621 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config, 1622 uint32_t config_len) 1623 { 1624 VhostUserMsg msg = { 1625 .hdr.request = VHOST_USER_GET_CONFIG, 1626 .hdr.flags = VHOST_USER_VERSION, 1627 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len, 1628 }; 1629 1630 if (!virtio_has_feature(dev->protocol_features, 1631 VHOST_USER_PROTOCOL_F_CONFIG)) { 1632 return -1; 1633 } 1634 1635 if (config_len > VHOST_USER_MAX_CONFIG_SIZE) { 1636 return -1; 1637 } 1638 1639 msg.payload.config.offset = 0; 1640 msg.payload.config.size = config_len; 1641 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1642 return -1; 1643 } 1644 1645 if (vhost_user_read(dev, &msg) < 0) { 1646 return -1; 1647 } 1648 1649 if (msg.hdr.request != VHOST_USER_GET_CONFIG) { 1650 error_report("Received unexpected msg type. Expected %d received %d", 1651 VHOST_USER_GET_CONFIG, msg.hdr.request); 1652 return -1; 1653 } 1654 1655 if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) { 1656 error_report("Received bad msg size."); 1657 return -1; 1658 } 1659 1660 memcpy(config, msg.payload.config.region, config_len); 1661 1662 return 0; 1663 } 1664 1665 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data, 1666 uint32_t offset, uint32_t size, uint32_t flags) 1667 { 1668 uint8_t *p; 1669 bool reply_supported = virtio_has_feature(dev->protocol_features, 1670 VHOST_USER_PROTOCOL_F_REPLY_ACK); 1671 1672 VhostUserMsg msg = { 1673 .hdr.request = VHOST_USER_SET_CONFIG, 1674 .hdr.flags = VHOST_USER_VERSION, 1675 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size, 1676 }; 1677 1678 if (!virtio_has_feature(dev->protocol_features, 1679 VHOST_USER_PROTOCOL_F_CONFIG)) { 1680 return -1; 1681 } 1682 1683 if (reply_supported) { 1684 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 1685 } 1686 1687 if (size > VHOST_USER_MAX_CONFIG_SIZE) { 1688 return -1; 1689 } 1690 1691 msg.payload.config.offset = offset, 1692 msg.payload.config.size = size, 1693 msg.payload.config.flags = flags, 1694 p = msg.payload.config.region; 1695 memcpy(p, data, size); 1696 1697 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1698 return -1; 1699 } 1700 1701 if (reply_supported) { 1702 return process_message_reply(dev, &msg); 1703 } 1704 1705 return 0; 1706 } 1707 1708 static int vhost_user_crypto_create_session(struct vhost_dev *dev, 1709 void *session_info, 1710 uint64_t *session_id) 1711 { 1712 bool crypto_session = virtio_has_feature(dev->protocol_features, 1713 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); 1714 CryptoDevBackendSymSessionInfo *sess_info = session_info; 1715 VhostUserMsg msg = { 1716 .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION, 1717 .hdr.flags = VHOST_USER_VERSION, 1718 .hdr.size = sizeof(msg.payload.session), 1719 }; 1720 1721 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1722 1723 if (!crypto_session) { 1724 error_report("vhost-user trying to send unhandled ioctl"); 1725 return -1; 1726 } 1727 1728 memcpy(&msg.payload.session.session_setup_data, sess_info, 1729 sizeof(CryptoDevBackendSymSessionInfo)); 1730 if (sess_info->key_len) { 1731 memcpy(&msg.payload.session.key, sess_info->cipher_key, 1732 sess_info->key_len); 1733 } 1734 if (sess_info->auth_key_len > 0) { 1735 memcpy(&msg.payload.session.auth_key, sess_info->auth_key, 1736 sess_info->auth_key_len); 1737 } 1738 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1739 error_report("vhost_user_write() return -1, create session failed"); 1740 return -1; 1741 } 1742 1743 if (vhost_user_read(dev, &msg) < 0) { 1744 error_report("vhost_user_read() return -1, create session failed"); 1745 return -1; 1746 } 1747 1748 if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) { 1749 error_report("Received unexpected msg type. Expected %d received %d", 1750 VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request); 1751 return -1; 1752 } 1753 1754 if (msg.hdr.size != sizeof(msg.payload.session)) { 1755 error_report("Received bad msg size."); 1756 return -1; 1757 } 1758 1759 if (msg.payload.session.session_id < 0) { 1760 error_report("Bad session id: %" PRId64 "", 1761 msg.payload.session.session_id); 1762 return -1; 1763 } 1764 *session_id = msg.payload.session.session_id; 1765 1766 return 0; 1767 } 1768 1769 static int 1770 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id) 1771 { 1772 bool crypto_session = virtio_has_feature(dev->protocol_features, 1773 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); 1774 VhostUserMsg msg = { 1775 .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION, 1776 .hdr.flags = VHOST_USER_VERSION, 1777 .hdr.size = sizeof(msg.payload.u64), 1778 }; 1779 msg.payload.u64 = session_id; 1780 1781 if (!crypto_session) { 1782 error_report("vhost-user trying to send unhandled ioctl"); 1783 return -1; 1784 } 1785 1786 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1787 error_report("vhost_user_write() return -1, close session failed"); 1788 return -1; 1789 } 1790 1791 return 0; 1792 } 1793 1794 static bool vhost_user_mem_section_filter(struct vhost_dev *dev, 1795 MemoryRegionSection *section) 1796 { 1797 bool result; 1798 1799 result = memory_region_get_fd(section->mr) >= 0; 1800 1801 return result; 1802 } 1803 1804 static int vhost_user_get_inflight_fd(struct vhost_dev *dev, 1805 uint16_t queue_size, 1806 struct vhost_inflight *inflight) 1807 { 1808 void *addr; 1809 int fd; 1810 struct vhost_user *u = dev->opaque; 1811 CharBackend *chr = u->user->chr; 1812 VhostUserMsg msg = { 1813 .hdr.request = VHOST_USER_GET_INFLIGHT_FD, 1814 .hdr.flags = VHOST_USER_VERSION, 1815 .payload.inflight.num_queues = dev->nvqs, 1816 .payload.inflight.queue_size = queue_size, 1817 .hdr.size = sizeof(msg.payload.inflight), 1818 }; 1819 1820 if (!virtio_has_feature(dev->protocol_features, 1821 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) { 1822 return 0; 1823 } 1824 1825 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1826 return -1; 1827 } 1828 1829 if (vhost_user_read(dev, &msg) < 0) { 1830 return -1; 1831 } 1832 1833 if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) { 1834 error_report("Received unexpected msg type. " 1835 "Expected %d received %d", 1836 VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request); 1837 return -1; 1838 } 1839 1840 if (msg.hdr.size != sizeof(msg.payload.inflight)) { 1841 error_report("Received bad msg size."); 1842 return -1; 1843 } 1844 1845 if (!msg.payload.inflight.mmap_size) { 1846 return 0; 1847 } 1848 1849 fd = qemu_chr_fe_get_msgfd(chr); 1850 if (fd < 0) { 1851 error_report("Failed to get mem fd"); 1852 return -1; 1853 } 1854 1855 addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ | PROT_WRITE, 1856 MAP_SHARED, fd, msg.payload.inflight.mmap_offset); 1857 1858 if (addr == MAP_FAILED) { 1859 error_report("Failed to mmap mem fd"); 1860 close(fd); 1861 return -1; 1862 } 1863 1864 inflight->addr = addr; 1865 inflight->fd = fd; 1866 inflight->size = msg.payload.inflight.mmap_size; 1867 inflight->offset = msg.payload.inflight.mmap_offset; 1868 inflight->queue_size = queue_size; 1869 1870 return 0; 1871 } 1872 1873 static int vhost_user_set_inflight_fd(struct vhost_dev *dev, 1874 struct vhost_inflight *inflight) 1875 { 1876 VhostUserMsg msg = { 1877 .hdr.request = VHOST_USER_SET_INFLIGHT_FD, 1878 .hdr.flags = VHOST_USER_VERSION, 1879 .payload.inflight.mmap_size = inflight->size, 1880 .payload.inflight.mmap_offset = inflight->offset, 1881 .payload.inflight.num_queues = dev->nvqs, 1882 .payload.inflight.queue_size = inflight->queue_size, 1883 .hdr.size = sizeof(msg.payload.inflight), 1884 }; 1885 1886 if (!virtio_has_feature(dev->protocol_features, 1887 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) { 1888 return 0; 1889 } 1890 1891 if (vhost_user_write(dev, &msg, &inflight->fd, 1) < 0) { 1892 return -1; 1893 } 1894 1895 return 0; 1896 } 1897 1898 bool vhost_user_init(VhostUserState *user, CharBackend *chr, Error **errp) 1899 { 1900 if (user->chr) { 1901 error_setg(errp, "Cannot initialize vhost-user state"); 1902 return false; 1903 } 1904 user->chr = chr; 1905 return true; 1906 } 1907 1908 void vhost_user_cleanup(VhostUserState *user) 1909 { 1910 int i; 1911 1912 if (!user->chr) { 1913 return; 1914 } 1915 1916 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1917 if (user->notifier[i].addr) { 1918 object_unparent(OBJECT(&user->notifier[i].mr)); 1919 munmap(user->notifier[i].addr, qemu_real_host_page_size); 1920 user->notifier[i].addr = NULL; 1921 } 1922 } 1923 user->chr = NULL; 1924 } 1925 1926 const VhostOps user_ops = { 1927 .backend_type = VHOST_BACKEND_TYPE_USER, 1928 .vhost_backend_init = vhost_user_backend_init, 1929 .vhost_backend_cleanup = vhost_user_backend_cleanup, 1930 .vhost_backend_memslots_limit = vhost_user_memslots_limit, 1931 .vhost_set_log_base = vhost_user_set_log_base, 1932 .vhost_set_mem_table = vhost_user_set_mem_table, 1933 .vhost_set_vring_addr = vhost_user_set_vring_addr, 1934 .vhost_set_vring_endian = vhost_user_set_vring_endian, 1935 .vhost_set_vring_num = vhost_user_set_vring_num, 1936 .vhost_set_vring_base = vhost_user_set_vring_base, 1937 .vhost_get_vring_base = vhost_user_get_vring_base, 1938 .vhost_set_vring_kick = vhost_user_set_vring_kick, 1939 .vhost_set_vring_call = vhost_user_set_vring_call, 1940 .vhost_set_features = vhost_user_set_features, 1941 .vhost_get_features = vhost_user_get_features, 1942 .vhost_set_owner = vhost_user_set_owner, 1943 .vhost_reset_device = vhost_user_reset_device, 1944 .vhost_get_vq_index = vhost_user_get_vq_index, 1945 .vhost_set_vring_enable = vhost_user_set_vring_enable, 1946 .vhost_requires_shm_log = vhost_user_requires_shm_log, 1947 .vhost_migration_done = vhost_user_migration_done, 1948 .vhost_backend_can_merge = vhost_user_can_merge, 1949 .vhost_net_set_mtu = vhost_user_net_set_mtu, 1950 .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback, 1951 .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg, 1952 .vhost_get_config = vhost_user_get_config, 1953 .vhost_set_config = vhost_user_set_config, 1954 .vhost_crypto_create_session = vhost_user_crypto_create_session, 1955 .vhost_crypto_close_session = vhost_user_crypto_close_session, 1956 .vhost_backend_mem_section_filter = vhost_user_mem_section_filter, 1957 .vhost_get_inflight_fd = vhost_user_get_inflight_fd, 1958 .vhost_set_inflight_fd = vhost_user_set_inflight_fd, 1959 }; 1960