1 /* 2 * Virtio MEM device 3 * 4 * Copyright (C) 2020 Red Hat, Inc. 5 * 6 * Authors: 7 * David Hildenbrand <david@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include "qemu/iov.h" 15 #include "qemu/cutils.h" 16 #include "qemu/error-report.h" 17 #include "qemu/units.h" 18 #include "sysemu/numa.h" 19 #include "sysemu/sysemu.h" 20 #include "sysemu/reset.h" 21 #include "hw/virtio/virtio.h" 22 #include "hw/virtio/virtio-bus.h" 23 #include "hw/virtio/virtio-access.h" 24 #include "hw/virtio/virtio-mem.h" 25 #include "qapi/error.h" 26 #include "qapi/visitor.h" 27 #include "exec/ram_addr.h" 28 #include "migration/misc.h" 29 #include "hw/boards.h" 30 #include "hw/qdev-properties.h" 31 #include CONFIG_DEVICES 32 #include "trace.h" 33 34 /* 35 * We only had legacy x86 guests that did not support 36 * VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE. Other targets don't have legacy guests. 37 */ 38 #if defined(TARGET_X86_64) || defined(TARGET_I386) 39 #define VIRTIO_MEM_HAS_LEGACY_GUESTS 40 #endif 41 42 /* 43 * Let's not allow blocks smaller than 1 MiB, for example, to keep the tracking 44 * bitmap small. 45 */ 46 #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)(1 * MiB)) 47 48 static uint32_t virtio_mem_default_thp_size(void) 49 { 50 uint32_t default_thp_size = VIRTIO_MEM_MIN_BLOCK_SIZE; 51 52 #if defined(__x86_64__) || defined(__arm__) || defined(__powerpc64__) 53 default_thp_size = 2 * MiB; 54 #elif defined(__aarch64__) 55 if (qemu_real_host_page_size() == 4 * KiB) { 56 default_thp_size = 2 * MiB; 57 } else if (qemu_real_host_page_size() == 16 * KiB) { 58 default_thp_size = 32 * MiB; 59 } else if (qemu_real_host_page_size() == 64 * KiB) { 60 default_thp_size = 512 * MiB; 61 } 62 #endif 63 64 return default_thp_size; 65 } 66 67 /* 68 * We want to have a reasonable default block size such that 69 * 1. We avoid splitting THPs when unplugging memory, which degrades 70 * performance. 71 * 2. We avoid placing THPs for plugged blocks that also cover unplugged 72 * blocks. 73 * 74 * The actual THP size might differ between Linux kernels, so we try to probe 75 * it. In the future (if we ever run into issues regarding 2.), we might want 76 * to disable THP in case we fail to properly probe the THP size, or if the 77 * block size is configured smaller than the THP size. 78 */ 79 static uint32_t thp_size; 80 81 #define HPAGE_PMD_SIZE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" 82 static uint32_t virtio_mem_thp_size(void) 83 { 84 gchar *content = NULL; 85 const char *endptr; 86 uint64_t tmp; 87 88 if (thp_size) { 89 return thp_size; 90 } 91 92 /* 93 * Try to probe the actual THP size, fallback to (sane but eventually 94 * incorrect) default sizes. 95 */ 96 if (g_file_get_contents(HPAGE_PMD_SIZE_PATH, &content, NULL, NULL) && 97 !qemu_strtou64(content, &endptr, 0, &tmp) && 98 (!endptr || *endptr == '\n')) { 99 /* Sanity-check the value and fallback to something reasonable. */ 100 if (!tmp || !is_power_of_2(tmp)) { 101 warn_report("Read unsupported THP size: %" PRIx64, tmp); 102 } else { 103 thp_size = tmp; 104 } 105 } 106 107 if (!thp_size) { 108 thp_size = virtio_mem_default_thp_size(); 109 warn_report("Could not detect THP size, falling back to %" PRIx64 110 " MiB.", thp_size / MiB); 111 } 112 113 g_free(content); 114 return thp_size; 115 } 116 117 static uint64_t virtio_mem_default_block_size(RAMBlock *rb) 118 { 119 const uint64_t page_size = qemu_ram_pagesize(rb); 120 121 /* We can have hugetlbfs with a page size smaller than the THP size. */ 122 if (page_size == qemu_real_host_page_size()) { 123 return MAX(page_size, virtio_mem_thp_size()); 124 } 125 return MAX(page_size, VIRTIO_MEM_MIN_BLOCK_SIZE); 126 } 127 128 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) 129 static bool virtio_mem_has_shared_zeropage(RAMBlock *rb) 130 { 131 /* 132 * We only have a guaranteed shared zeropage on ordinary MAP_PRIVATE 133 * anonymous RAM. In any other case, reading unplugged *can* populate a 134 * fresh page, consuming actual memory. 135 */ 136 return !qemu_ram_is_shared(rb) && rb->fd < 0 && 137 qemu_ram_pagesize(rb) == qemu_real_host_page_size(); 138 } 139 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ 140 141 /* 142 * Size the usable region bigger than the requested size if possible. Esp. 143 * Linux guests will only add (aligned) memory blocks in case they fully 144 * fit into the usable region, but plug+online only a subset of the pages. 145 * The memory block size corresponds mostly to the section size. 146 * 147 * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and 148 * a section size of 512MB on arm64 (as long as the start address is properly 149 * aligned, similar to ordinary DIMMs). 150 * 151 * We can change this at any time and maybe even make it configurable if 152 * necessary (as the section size can change). But it's more likely that the 153 * section size will rather get smaller and not bigger over time. 154 */ 155 #if defined(TARGET_X86_64) || defined(TARGET_I386) 156 #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB)) 157 #elif defined(TARGET_ARM) 158 #define VIRTIO_MEM_USABLE_EXTENT (2 * (512 * MiB)) 159 #else 160 #error VIRTIO_MEM_USABLE_EXTENT not defined 161 #endif 162 163 static bool virtio_mem_is_busy(void) 164 { 165 /* 166 * Postcopy cannot handle concurrent discards and we don't want to migrate 167 * pages on-demand with stale content when plugging new blocks. 168 * 169 * For precopy, we don't want unplugged blocks in our migration stream, and 170 * when plugging new blocks, the page content might differ between source 171 * and destination (observable by the guest when not initializing pages 172 * after plugging them) until we're running on the destination (as we didn't 173 * migrate these blocks when they were unplugged). 174 */ 175 return migration_in_incoming_postcopy() || !migration_is_idle(); 176 } 177 178 typedef int (*virtio_mem_range_cb)(const VirtIOMEM *vmem, void *arg, 179 uint64_t offset, uint64_t size); 180 181 static int virtio_mem_for_each_unplugged_range(const VirtIOMEM *vmem, void *arg, 182 virtio_mem_range_cb cb) 183 { 184 unsigned long first_zero_bit, last_zero_bit; 185 uint64_t offset, size; 186 int ret = 0; 187 188 first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size); 189 while (first_zero_bit < vmem->bitmap_size) { 190 offset = first_zero_bit * vmem->block_size; 191 last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, 192 first_zero_bit + 1) - 1; 193 size = (last_zero_bit - first_zero_bit + 1) * vmem->block_size; 194 195 ret = cb(vmem, arg, offset, size); 196 if (ret) { 197 break; 198 } 199 first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, 200 last_zero_bit + 2); 201 } 202 return ret; 203 } 204 205 /* 206 * Adjust the memory section to cover the intersection with the given range. 207 * 208 * Returns false if the intersection is empty, otherwise returns true. 209 */ 210 static bool virito_mem_intersect_memory_section(MemoryRegionSection *s, 211 uint64_t offset, uint64_t size) 212 { 213 uint64_t start = MAX(s->offset_within_region, offset); 214 uint64_t end = MIN(s->offset_within_region + int128_get64(s->size), 215 offset + size); 216 217 if (end <= start) { 218 return false; 219 } 220 221 s->offset_within_address_space += start - s->offset_within_region; 222 s->offset_within_region = start; 223 s->size = int128_make64(end - start); 224 return true; 225 } 226 227 typedef int (*virtio_mem_section_cb)(MemoryRegionSection *s, void *arg); 228 229 static int virtio_mem_for_each_plugged_section(const VirtIOMEM *vmem, 230 MemoryRegionSection *s, 231 void *arg, 232 virtio_mem_section_cb cb) 233 { 234 unsigned long first_bit, last_bit; 235 uint64_t offset, size; 236 int ret = 0; 237 238 first_bit = s->offset_within_region / vmem->bitmap_size; 239 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, first_bit); 240 while (first_bit < vmem->bitmap_size) { 241 MemoryRegionSection tmp = *s; 242 243 offset = first_bit * vmem->block_size; 244 last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, 245 first_bit + 1) - 1; 246 size = (last_bit - first_bit + 1) * vmem->block_size; 247 248 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 249 break; 250 } 251 ret = cb(&tmp, arg); 252 if (ret) { 253 break; 254 } 255 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, 256 last_bit + 2); 257 } 258 return ret; 259 } 260 261 static int virtio_mem_for_each_unplugged_section(const VirtIOMEM *vmem, 262 MemoryRegionSection *s, 263 void *arg, 264 virtio_mem_section_cb cb) 265 { 266 unsigned long first_bit, last_bit; 267 uint64_t offset, size; 268 int ret = 0; 269 270 first_bit = s->offset_within_region / vmem->bitmap_size; 271 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, first_bit); 272 while (first_bit < vmem->bitmap_size) { 273 MemoryRegionSection tmp = *s; 274 275 offset = first_bit * vmem->block_size; 276 last_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, 277 first_bit + 1) - 1; 278 size = (last_bit - first_bit + 1) * vmem->block_size; 279 280 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 281 break; 282 } 283 ret = cb(&tmp, arg); 284 if (ret) { 285 break; 286 } 287 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, 288 last_bit + 2); 289 } 290 return ret; 291 } 292 293 static int virtio_mem_notify_populate_cb(MemoryRegionSection *s, void *arg) 294 { 295 RamDiscardListener *rdl = arg; 296 297 return rdl->notify_populate(rdl, s); 298 } 299 300 static int virtio_mem_notify_discard_cb(MemoryRegionSection *s, void *arg) 301 { 302 RamDiscardListener *rdl = arg; 303 304 rdl->notify_discard(rdl, s); 305 return 0; 306 } 307 308 static void virtio_mem_notify_unplug(VirtIOMEM *vmem, uint64_t offset, 309 uint64_t size) 310 { 311 RamDiscardListener *rdl; 312 313 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 314 MemoryRegionSection tmp = *rdl->section; 315 316 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 317 continue; 318 } 319 rdl->notify_discard(rdl, &tmp); 320 } 321 } 322 323 static int virtio_mem_notify_plug(VirtIOMEM *vmem, uint64_t offset, 324 uint64_t size) 325 { 326 RamDiscardListener *rdl, *rdl2; 327 int ret = 0; 328 329 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 330 MemoryRegionSection tmp = *rdl->section; 331 332 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 333 continue; 334 } 335 ret = rdl->notify_populate(rdl, &tmp); 336 if (ret) { 337 break; 338 } 339 } 340 341 if (ret) { 342 /* Notify all already-notified listeners. */ 343 QLIST_FOREACH(rdl2, &vmem->rdl_list, next) { 344 MemoryRegionSection tmp = *rdl->section; 345 346 if (rdl2 == rdl) { 347 break; 348 } 349 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 350 continue; 351 } 352 rdl2->notify_discard(rdl2, &tmp); 353 } 354 } 355 return ret; 356 } 357 358 static void virtio_mem_notify_unplug_all(VirtIOMEM *vmem) 359 { 360 RamDiscardListener *rdl; 361 362 if (!vmem->size) { 363 return; 364 } 365 366 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 367 if (rdl->double_discard_supported) { 368 rdl->notify_discard(rdl, rdl->section); 369 } else { 370 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 371 virtio_mem_notify_discard_cb); 372 } 373 } 374 } 375 376 static bool virtio_mem_test_bitmap(const VirtIOMEM *vmem, uint64_t start_gpa, 377 uint64_t size, bool plugged) 378 { 379 const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; 380 const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; 381 unsigned long found_bit; 382 383 /* We fake a shorter bitmap to avoid searching too far. */ 384 if (plugged) { 385 found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit); 386 } else { 387 found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit); 388 } 389 return found_bit > last_bit; 390 } 391 392 static void virtio_mem_set_bitmap(VirtIOMEM *vmem, uint64_t start_gpa, 393 uint64_t size, bool plugged) 394 { 395 const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; 396 const unsigned long nbits = size / vmem->block_size; 397 398 if (plugged) { 399 bitmap_set(vmem->bitmap, bit, nbits); 400 } else { 401 bitmap_clear(vmem->bitmap, bit, nbits); 402 } 403 } 404 405 static void virtio_mem_send_response(VirtIOMEM *vmem, VirtQueueElement *elem, 406 struct virtio_mem_resp *resp) 407 { 408 VirtIODevice *vdev = VIRTIO_DEVICE(vmem); 409 VirtQueue *vq = vmem->vq; 410 411 trace_virtio_mem_send_response(le16_to_cpu(resp->type)); 412 iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp)); 413 414 virtqueue_push(vq, elem, sizeof(*resp)); 415 virtio_notify(vdev, vq); 416 } 417 418 static void virtio_mem_send_response_simple(VirtIOMEM *vmem, 419 VirtQueueElement *elem, 420 uint16_t type) 421 { 422 struct virtio_mem_resp resp = { 423 .type = cpu_to_le16(type), 424 }; 425 426 virtio_mem_send_response(vmem, elem, &resp); 427 } 428 429 static bool virtio_mem_valid_range(const VirtIOMEM *vmem, uint64_t gpa, 430 uint64_t size) 431 { 432 if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) { 433 return false; 434 } 435 if (gpa + size < gpa || !size) { 436 return false; 437 } 438 if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) { 439 return false; 440 } 441 if (gpa + size > vmem->addr + vmem->usable_region_size) { 442 return false; 443 } 444 return true; 445 } 446 447 static int virtio_mem_set_block_state(VirtIOMEM *vmem, uint64_t start_gpa, 448 uint64_t size, bool plug) 449 { 450 const uint64_t offset = start_gpa - vmem->addr; 451 RAMBlock *rb = vmem->memdev->mr.ram_block; 452 453 if (virtio_mem_is_busy()) { 454 return -EBUSY; 455 } 456 457 if (!plug) { 458 if (ram_block_discard_range(rb, offset, size)) { 459 return -EBUSY; 460 } 461 virtio_mem_notify_unplug(vmem, offset, size); 462 } else { 463 int ret = 0; 464 465 if (vmem->prealloc) { 466 void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; 467 int fd = memory_region_get_fd(&vmem->memdev->mr); 468 Error *local_err = NULL; 469 470 os_mem_prealloc(fd, area, size, 1, &local_err); 471 if (local_err) { 472 static bool warned; 473 474 /* 475 * Warn only once, we don't want to fill the log with these 476 * warnings. 477 */ 478 if (!warned) { 479 warn_report_err(local_err); 480 warned = true; 481 } else { 482 error_free(local_err); 483 } 484 ret = -EBUSY; 485 } 486 } 487 if (!ret) { 488 ret = virtio_mem_notify_plug(vmem, offset, size); 489 } 490 491 if (ret) { 492 /* Could be preallocation or a notifier populated memory. */ 493 ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size); 494 return -EBUSY; 495 } 496 } 497 virtio_mem_set_bitmap(vmem, start_gpa, size, plug); 498 return 0; 499 } 500 501 static int virtio_mem_state_change_request(VirtIOMEM *vmem, uint64_t gpa, 502 uint16_t nb_blocks, bool plug) 503 { 504 const uint64_t size = nb_blocks * vmem->block_size; 505 int ret; 506 507 if (!virtio_mem_valid_range(vmem, gpa, size)) { 508 return VIRTIO_MEM_RESP_ERROR; 509 } 510 511 if (plug && (vmem->size + size > vmem->requested_size)) { 512 return VIRTIO_MEM_RESP_NACK; 513 } 514 515 /* test if really all blocks are in the opposite state */ 516 if (!virtio_mem_test_bitmap(vmem, gpa, size, !plug)) { 517 return VIRTIO_MEM_RESP_ERROR; 518 } 519 520 ret = virtio_mem_set_block_state(vmem, gpa, size, plug); 521 if (ret) { 522 return VIRTIO_MEM_RESP_BUSY; 523 } 524 if (plug) { 525 vmem->size += size; 526 } else { 527 vmem->size -= size; 528 } 529 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); 530 return VIRTIO_MEM_RESP_ACK; 531 } 532 533 static void virtio_mem_plug_request(VirtIOMEM *vmem, VirtQueueElement *elem, 534 struct virtio_mem_req *req) 535 { 536 const uint64_t gpa = le64_to_cpu(req->u.plug.addr); 537 const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks); 538 uint16_t type; 539 540 trace_virtio_mem_plug_request(gpa, nb_blocks); 541 type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, true); 542 virtio_mem_send_response_simple(vmem, elem, type); 543 } 544 545 static void virtio_mem_unplug_request(VirtIOMEM *vmem, VirtQueueElement *elem, 546 struct virtio_mem_req *req) 547 { 548 const uint64_t gpa = le64_to_cpu(req->u.unplug.addr); 549 const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks); 550 uint16_t type; 551 552 trace_virtio_mem_unplug_request(gpa, nb_blocks); 553 type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, false); 554 virtio_mem_send_response_simple(vmem, elem, type); 555 } 556 557 static void virtio_mem_resize_usable_region(VirtIOMEM *vmem, 558 uint64_t requested_size, 559 bool can_shrink) 560 { 561 uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr), 562 requested_size + VIRTIO_MEM_USABLE_EXTENT); 563 564 /* The usable region size always has to be multiples of the block size. */ 565 newsize = QEMU_ALIGN_UP(newsize, vmem->block_size); 566 567 if (!requested_size) { 568 newsize = 0; 569 } 570 571 if (newsize < vmem->usable_region_size && !can_shrink) { 572 return; 573 } 574 575 trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize); 576 vmem->usable_region_size = newsize; 577 } 578 579 static int virtio_mem_unplug_all(VirtIOMEM *vmem) 580 { 581 RAMBlock *rb = vmem->memdev->mr.ram_block; 582 583 if (virtio_mem_is_busy()) { 584 return -EBUSY; 585 } 586 587 if (ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb))) { 588 return -EBUSY; 589 } 590 virtio_mem_notify_unplug_all(vmem); 591 592 bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size); 593 if (vmem->size) { 594 vmem->size = 0; 595 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); 596 } 597 trace_virtio_mem_unplugged_all(); 598 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); 599 return 0; 600 } 601 602 static void virtio_mem_unplug_all_request(VirtIOMEM *vmem, 603 VirtQueueElement *elem) 604 { 605 trace_virtio_mem_unplug_all_request(); 606 if (virtio_mem_unplug_all(vmem)) { 607 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_BUSY); 608 } else { 609 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ACK); 610 } 611 } 612 613 static void virtio_mem_state_request(VirtIOMEM *vmem, VirtQueueElement *elem, 614 struct virtio_mem_req *req) 615 { 616 const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks); 617 const uint64_t gpa = le64_to_cpu(req->u.state.addr); 618 const uint64_t size = nb_blocks * vmem->block_size; 619 struct virtio_mem_resp resp = { 620 .type = cpu_to_le16(VIRTIO_MEM_RESP_ACK), 621 }; 622 623 trace_virtio_mem_state_request(gpa, nb_blocks); 624 if (!virtio_mem_valid_range(vmem, gpa, size)) { 625 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ERROR); 626 return; 627 } 628 629 if (virtio_mem_test_bitmap(vmem, gpa, size, true)) { 630 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED); 631 } else if (virtio_mem_test_bitmap(vmem, gpa, size, false)) { 632 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED); 633 } else { 634 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_MIXED); 635 } 636 trace_virtio_mem_state_response(le16_to_cpu(resp.u.state.state)); 637 virtio_mem_send_response(vmem, elem, &resp); 638 } 639 640 static void virtio_mem_handle_request(VirtIODevice *vdev, VirtQueue *vq) 641 { 642 const int len = sizeof(struct virtio_mem_req); 643 VirtIOMEM *vmem = VIRTIO_MEM(vdev); 644 VirtQueueElement *elem; 645 struct virtio_mem_req req; 646 uint16_t type; 647 648 while (true) { 649 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 650 if (!elem) { 651 return; 652 } 653 654 if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) { 655 virtio_error(vdev, "virtio-mem protocol violation: invalid request" 656 " size: %d", len); 657 virtqueue_detach_element(vq, elem, 0); 658 g_free(elem); 659 return; 660 } 661 662 if (iov_size(elem->in_sg, elem->in_num) < 663 sizeof(struct virtio_mem_resp)) { 664 virtio_error(vdev, "virtio-mem protocol violation: not enough space" 665 " for response: %zu", 666 iov_size(elem->in_sg, elem->in_num)); 667 virtqueue_detach_element(vq, elem, 0); 668 g_free(elem); 669 return; 670 } 671 672 type = le16_to_cpu(req.type); 673 switch (type) { 674 case VIRTIO_MEM_REQ_PLUG: 675 virtio_mem_plug_request(vmem, elem, &req); 676 break; 677 case VIRTIO_MEM_REQ_UNPLUG: 678 virtio_mem_unplug_request(vmem, elem, &req); 679 break; 680 case VIRTIO_MEM_REQ_UNPLUG_ALL: 681 virtio_mem_unplug_all_request(vmem, elem); 682 break; 683 case VIRTIO_MEM_REQ_STATE: 684 virtio_mem_state_request(vmem, elem, &req); 685 break; 686 default: 687 virtio_error(vdev, "virtio-mem protocol violation: unknown request" 688 " type: %d", type); 689 virtqueue_detach_element(vq, elem, 0); 690 g_free(elem); 691 return; 692 } 693 694 g_free(elem); 695 } 696 } 697 698 static void virtio_mem_get_config(VirtIODevice *vdev, uint8_t *config_data) 699 { 700 VirtIOMEM *vmem = VIRTIO_MEM(vdev); 701 struct virtio_mem_config *config = (void *) config_data; 702 703 config->block_size = cpu_to_le64(vmem->block_size); 704 config->node_id = cpu_to_le16(vmem->node); 705 config->requested_size = cpu_to_le64(vmem->requested_size); 706 config->plugged_size = cpu_to_le64(vmem->size); 707 config->addr = cpu_to_le64(vmem->addr); 708 config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr)); 709 config->usable_region_size = cpu_to_le64(vmem->usable_region_size); 710 } 711 712 static uint64_t virtio_mem_get_features(VirtIODevice *vdev, uint64_t features, 713 Error **errp) 714 { 715 MachineState *ms = MACHINE(qdev_get_machine()); 716 VirtIOMEM *vmem = VIRTIO_MEM(vdev); 717 718 if (ms->numa_state) { 719 #if defined(CONFIG_ACPI) 720 virtio_add_feature(&features, VIRTIO_MEM_F_ACPI_PXM); 721 #endif 722 } 723 assert(vmem->unplugged_inaccessible != ON_OFF_AUTO_AUTO); 724 if (vmem->unplugged_inaccessible == ON_OFF_AUTO_ON) { 725 virtio_add_feature(&features, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE); 726 } 727 return features; 728 } 729 730 static int virtio_mem_validate_features(VirtIODevice *vdev) 731 { 732 if (virtio_host_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE) && 733 !virtio_vdev_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE)) { 734 return -EFAULT; 735 } 736 return 0; 737 } 738 739 static void virtio_mem_system_reset(void *opaque) 740 { 741 VirtIOMEM *vmem = VIRTIO_MEM(opaque); 742 743 /* 744 * During usual resets, we will unplug all memory and shrink the usable 745 * region size. This is, however, not possible in all scenarios. Then, 746 * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL). 747 */ 748 virtio_mem_unplug_all(vmem); 749 } 750 751 static void virtio_mem_device_realize(DeviceState *dev, Error **errp) 752 { 753 MachineState *ms = MACHINE(qdev_get_machine()); 754 int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0; 755 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 756 VirtIOMEM *vmem = VIRTIO_MEM(dev); 757 uint64_t page_size; 758 RAMBlock *rb; 759 int ret; 760 761 if (!vmem->memdev) { 762 error_setg(errp, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP); 763 return; 764 } else if (host_memory_backend_is_mapped(vmem->memdev)) { 765 error_setg(errp, "'%s' property specifies a busy memdev: %s", 766 VIRTIO_MEM_MEMDEV_PROP, 767 object_get_canonical_path_component(OBJECT(vmem->memdev))); 768 return; 769 } else if (!memory_region_is_ram(&vmem->memdev->mr) || 770 memory_region_is_rom(&vmem->memdev->mr) || 771 !vmem->memdev->mr.ram_block) { 772 error_setg(errp, "'%s' property specifies an unsupported memdev", 773 VIRTIO_MEM_MEMDEV_PROP); 774 return; 775 } 776 777 if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) || 778 (!nb_numa_nodes && vmem->node)) { 779 error_setg(errp, "'%s' property has value '%" PRIu32 "', which exceeds" 780 "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP, 781 vmem->node, nb_numa_nodes ? nb_numa_nodes : 1); 782 return; 783 } 784 785 if (enable_mlock) { 786 error_setg(errp, "Incompatible with mlock"); 787 return; 788 } 789 790 rb = vmem->memdev->mr.ram_block; 791 page_size = qemu_ram_pagesize(rb); 792 793 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) 794 switch (vmem->unplugged_inaccessible) { 795 case ON_OFF_AUTO_AUTO: 796 if (virtio_mem_has_shared_zeropage(rb)) { 797 vmem->unplugged_inaccessible = ON_OFF_AUTO_OFF; 798 } else { 799 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; 800 } 801 break; 802 case ON_OFF_AUTO_OFF: 803 if (!virtio_mem_has_shared_zeropage(rb)) { 804 warn_report("'%s' property set to 'off' with a memdev that does" 805 " not support the shared zeropage.", 806 VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP); 807 } 808 break; 809 default: 810 break; 811 } 812 #else /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ 813 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; 814 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ 815 816 /* 817 * If the block size wasn't configured by the user, use a sane default. This 818 * allows using hugetlbfs backends of any page size without manual 819 * intervention. 820 */ 821 if (!vmem->block_size) { 822 vmem->block_size = virtio_mem_default_block_size(rb); 823 } 824 825 if (vmem->block_size < page_size) { 826 error_setg(errp, "'%s' property has to be at least the page size (0x%" 827 PRIx64 ")", VIRTIO_MEM_BLOCK_SIZE_PROP, page_size); 828 return; 829 } else if (vmem->block_size < virtio_mem_default_block_size(rb)) { 830 warn_report("'%s' property is smaller than the default block size (%" 831 PRIx64 " MiB)", VIRTIO_MEM_BLOCK_SIZE_PROP, 832 virtio_mem_default_block_size(rb) / MiB); 833 } 834 if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) { 835 error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 836 ")", VIRTIO_MEM_REQUESTED_SIZE_PROP, 837 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); 838 return; 839 } else if (!QEMU_IS_ALIGNED(vmem->addr, vmem->block_size)) { 840 error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 841 ")", VIRTIO_MEM_ADDR_PROP, VIRTIO_MEM_BLOCK_SIZE_PROP, 842 vmem->block_size); 843 return; 844 } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr), 845 vmem->block_size)) { 846 error_setg(errp, "'%s' property memdev size has to be multiples of" 847 "'%s' (0x%" PRIx64 ")", VIRTIO_MEM_MEMDEV_PROP, 848 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); 849 return; 850 } 851 852 if (ram_block_coordinated_discard_require(true)) { 853 error_setg(errp, "Discarding RAM is disabled"); 854 return; 855 } 856 857 ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb)); 858 if (ret) { 859 error_setg_errno(errp, -ret, "Unexpected error discarding RAM"); 860 ram_block_coordinated_discard_require(false); 861 return; 862 } 863 864 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); 865 866 vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) / 867 vmem->block_size; 868 vmem->bitmap = bitmap_new(vmem->bitmap_size); 869 870 virtio_init(vdev, TYPE_VIRTIO_MEM, VIRTIO_ID_MEM, 871 sizeof(struct virtio_mem_config)); 872 vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request); 873 874 host_memory_backend_set_mapped(vmem->memdev, true); 875 vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem)); 876 qemu_register_reset(virtio_mem_system_reset, vmem); 877 878 /* 879 * Set ourselves as RamDiscardManager before the plug handler maps the 880 * memory region and exposes it via an address space. 881 */ 882 memory_region_set_ram_discard_manager(&vmem->memdev->mr, 883 RAM_DISCARD_MANAGER(vmem)); 884 } 885 886 static void virtio_mem_device_unrealize(DeviceState *dev) 887 { 888 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 889 VirtIOMEM *vmem = VIRTIO_MEM(dev); 890 891 /* 892 * The unplug handler unmapped the memory region, it cannot be 893 * found via an address space anymore. Unset ourselves. 894 */ 895 memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL); 896 qemu_unregister_reset(virtio_mem_system_reset, vmem); 897 vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem)); 898 host_memory_backend_set_mapped(vmem->memdev, false); 899 virtio_del_queue(vdev, 0); 900 virtio_cleanup(vdev); 901 g_free(vmem->bitmap); 902 ram_block_coordinated_discard_require(false); 903 } 904 905 static int virtio_mem_discard_range_cb(const VirtIOMEM *vmem, void *arg, 906 uint64_t offset, uint64_t size) 907 { 908 RAMBlock *rb = vmem->memdev->mr.ram_block; 909 910 return ram_block_discard_range(rb, offset, size) ? -EINVAL : 0; 911 } 912 913 static int virtio_mem_restore_unplugged(VirtIOMEM *vmem) 914 { 915 /* Make sure all memory is really discarded after migration. */ 916 return virtio_mem_for_each_unplugged_range(vmem, NULL, 917 virtio_mem_discard_range_cb); 918 } 919 920 static int virtio_mem_post_load(void *opaque, int version_id) 921 { 922 VirtIOMEM *vmem = VIRTIO_MEM(opaque); 923 RamDiscardListener *rdl; 924 int ret; 925 926 /* 927 * We started out with all memory discarded and our memory region is mapped 928 * into an address space. Replay, now that we updated the bitmap. 929 */ 930 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 931 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 932 virtio_mem_notify_populate_cb); 933 if (ret) { 934 return ret; 935 } 936 } 937 938 if (migration_in_incoming_postcopy()) { 939 return 0; 940 } 941 942 return virtio_mem_restore_unplugged(vmem); 943 } 944 945 typedef struct VirtIOMEMMigSanityChecks { 946 VirtIOMEM *parent; 947 uint64_t addr; 948 uint64_t region_size; 949 uint64_t block_size; 950 uint32_t node; 951 } VirtIOMEMMigSanityChecks; 952 953 static int virtio_mem_mig_sanity_checks_pre_save(void *opaque) 954 { 955 VirtIOMEMMigSanityChecks *tmp = opaque; 956 VirtIOMEM *vmem = tmp->parent; 957 958 tmp->addr = vmem->addr; 959 tmp->region_size = memory_region_size(&vmem->memdev->mr); 960 tmp->block_size = vmem->block_size; 961 tmp->node = vmem->node; 962 return 0; 963 } 964 965 static int virtio_mem_mig_sanity_checks_post_load(void *opaque, int version_id) 966 { 967 VirtIOMEMMigSanityChecks *tmp = opaque; 968 VirtIOMEM *vmem = tmp->parent; 969 const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr); 970 971 if (tmp->addr != vmem->addr) { 972 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, 973 VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr); 974 return -EINVAL; 975 } 976 /* 977 * Note: Preparation for resizeable memory regions. The maximum size 978 * of the memory region must not change during migration. 979 */ 980 if (tmp->region_size != new_region_size) { 981 error_report("Property '%s' size changed from 0x%" PRIx64 " to 0x%" 982 PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size, 983 new_region_size); 984 return -EINVAL; 985 } 986 if (tmp->block_size != vmem->block_size) { 987 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, 988 VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size, 989 vmem->block_size); 990 return -EINVAL; 991 } 992 if (tmp->node != vmem->node) { 993 error_report("Property '%s' changed from %" PRIu32 " to %" PRIu32, 994 VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node); 995 return -EINVAL; 996 } 997 return 0; 998 } 999 1000 static const VMStateDescription vmstate_virtio_mem_sanity_checks = { 1001 .name = "virtio-mem-device/sanity-checks", 1002 .pre_save = virtio_mem_mig_sanity_checks_pre_save, 1003 .post_load = virtio_mem_mig_sanity_checks_post_load, 1004 .fields = (VMStateField[]) { 1005 VMSTATE_UINT64(addr, VirtIOMEMMigSanityChecks), 1006 VMSTATE_UINT64(region_size, VirtIOMEMMigSanityChecks), 1007 VMSTATE_UINT64(block_size, VirtIOMEMMigSanityChecks), 1008 VMSTATE_UINT32(node, VirtIOMEMMigSanityChecks), 1009 VMSTATE_END_OF_LIST(), 1010 }, 1011 }; 1012 1013 static const VMStateDescription vmstate_virtio_mem_device = { 1014 .name = "virtio-mem-device", 1015 .minimum_version_id = 1, 1016 .version_id = 1, 1017 .priority = MIG_PRI_VIRTIO_MEM, 1018 .post_load = virtio_mem_post_load, 1019 .fields = (VMStateField[]) { 1020 VMSTATE_WITH_TMP(VirtIOMEM, VirtIOMEMMigSanityChecks, 1021 vmstate_virtio_mem_sanity_checks), 1022 VMSTATE_UINT64(usable_region_size, VirtIOMEM), 1023 VMSTATE_UINT64(size, VirtIOMEM), 1024 VMSTATE_UINT64(requested_size, VirtIOMEM), 1025 VMSTATE_BITMAP(bitmap, VirtIOMEM, 0, bitmap_size), 1026 VMSTATE_END_OF_LIST() 1027 }, 1028 }; 1029 1030 static const VMStateDescription vmstate_virtio_mem = { 1031 .name = "virtio-mem", 1032 .minimum_version_id = 1, 1033 .version_id = 1, 1034 .fields = (VMStateField[]) { 1035 VMSTATE_VIRTIO_DEVICE, 1036 VMSTATE_END_OF_LIST() 1037 }, 1038 }; 1039 1040 static void virtio_mem_fill_device_info(const VirtIOMEM *vmem, 1041 VirtioMEMDeviceInfo *vi) 1042 { 1043 vi->memaddr = vmem->addr; 1044 vi->node = vmem->node; 1045 vi->requested_size = vmem->requested_size; 1046 vi->size = vmem->size; 1047 vi->max_size = memory_region_size(&vmem->memdev->mr); 1048 vi->block_size = vmem->block_size; 1049 vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev)); 1050 } 1051 1052 static MemoryRegion *virtio_mem_get_memory_region(VirtIOMEM *vmem, Error **errp) 1053 { 1054 if (!vmem->memdev) { 1055 error_setg(errp, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP); 1056 return NULL; 1057 } 1058 1059 return &vmem->memdev->mr; 1060 } 1061 1062 static void virtio_mem_add_size_change_notifier(VirtIOMEM *vmem, 1063 Notifier *notifier) 1064 { 1065 notifier_list_add(&vmem->size_change_notifiers, notifier); 1066 } 1067 1068 static void virtio_mem_remove_size_change_notifier(VirtIOMEM *vmem, 1069 Notifier *notifier) 1070 { 1071 notifier_remove(notifier); 1072 } 1073 1074 static void virtio_mem_get_size(Object *obj, Visitor *v, const char *name, 1075 void *opaque, Error **errp) 1076 { 1077 const VirtIOMEM *vmem = VIRTIO_MEM(obj); 1078 uint64_t value = vmem->size; 1079 1080 visit_type_size(v, name, &value, errp); 1081 } 1082 1083 static void virtio_mem_get_requested_size(Object *obj, Visitor *v, 1084 const char *name, void *opaque, 1085 Error **errp) 1086 { 1087 const VirtIOMEM *vmem = VIRTIO_MEM(obj); 1088 uint64_t value = vmem->requested_size; 1089 1090 visit_type_size(v, name, &value, errp); 1091 } 1092 1093 static void virtio_mem_set_requested_size(Object *obj, Visitor *v, 1094 const char *name, void *opaque, 1095 Error **errp) 1096 { 1097 VirtIOMEM *vmem = VIRTIO_MEM(obj); 1098 Error *err = NULL; 1099 uint64_t value; 1100 1101 visit_type_size(v, name, &value, &err); 1102 if (err) { 1103 error_propagate(errp, err); 1104 return; 1105 } 1106 1107 /* 1108 * The block size and memory backend are not fixed until the device was 1109 * realized. realize() will verify these properties then. 1110 */ 1111 if (DEVICE(obj)->realized) { 1112 if (!QEMU_IS_ALIGNED(value, vmem->block_size)) { 1113 error_setg(errp, "'%s' has to be multiples of '%s' (0x%" PRIx64 1114 ")", name, VIRTIO_MEM_BLOCK_SIZE_PROP, 1115 vmem->block_size); 1116 return; 1117 } else if (value > memory_region_size(&vmem->memdev->mr)) { 1118 error_setg(errp, "'%s' cannot exceed the memory backend size" 1119 "(0x%" PRIx64 ")", name, 1120 memory_region_size(&vmem->memdev->mr)); 1121 return; 1122 } 1123 1124 if (value != vmem->requested_size) { 1125 virtio_mem_resize_usable_region(vmem, value, false); 1126 vmem->requested_size = value; 1127 } 1128 /* 1129 * Trigger a config update so the guest gets notified. We trigger 1130 * even if the size didn't change (especially helpful for debugging). 1131 */ 1132 virtio_notify_config(VIRTIO_DEVICE(vmem)); 1133 } else { 1134 vmem->requested_size = value; 1135 } 1136 } 1137 1138 static void virtio_mem_get_block_size(Object *obj, Visitor *v, const char *name, 1139 void *opaque, Error **errp) 1140 { 1141 const VirtIOMEM *vmem = VIRTIO_MEM(obj); 1142 uint64_t value = vmem->block_size; 1143 1144 /* 1145 * If not configured by the user (and we're not realized yet), use the 1146 * default block size we would use with the current memory backend. 1147 */ 1148 if (!value) { 1149 if (vmem->memdev && memory_region_is_ram(&vmem->memdev->mr)) { 1150 value = virtio_mem_default_block_size(vmem->memdev->mr.ram_block); 1151 } else { 1152 value = virtio_mem_thp_size(); 1153 } 1154 } 1155 1156 visit_type_size(v, name, &value, errp); 1157 } 1158 1159 static void virtio_mem_set_block_size(Object *obj, Visitor *v, const char *name, 1160 void *opaque, Error **errp) 1161 { 1162 VirtIOMEM *vmem = VIRTIO_MEM(obj); 1163 Error *err = NULL; 1164 uint64_t value; 1165 1166 if (DEVICE(obj)->realized) { 1167 error_setg(errp, "'%s' cannot be changed", name); 1168 return; 1169 } 1170 1171 visit_type_size(v, name, &value, &err); 1172 if (err) { 1173 error_propagate(errp, err); 1174 return; 1175 } 1176 1177 if (value < VIRTIO_MEM_MIN_BLOCK_SIZE) { 1178 error_setg(errp, "'%s' property has to be at least 0x%" PRIx32, name, 1179 VIRTIO_MEM_MIN_BLOCK_SIZE); 1180 return; 1181 } else if (!is_power_of_2(value)) { 1182 error_setg(errp, "'%s' property has to be a power of two", name); 1183 return; 1184 } 1185 vmem->block_size = value; 1186 } 1187 1188 static void virtio_mem_instance_init(Object *obj) 1189 { 1190 VirtIOMEM *vmem = VIRTIO_MEM(obj); 1191 1192 notifier_list_init(&vmem->size_change_notifiers); 1193 QLIST_INIT(&vmem->rdl_list); 1194 1195 object_property_add(obj, VIRTIO_MEM_SIZE_PROP, "size", virtio_mem_get_size, 1196 NULL, NULL, NULL); 1197 object_property_add(obj, VIRTIO_MEM_REQUESTED_SIZE_PROP, "size", 1198 virtio_mem_get_requested_size, 1199 virtio_mem_set_requested_size, NULL, NULL); 1200 object_property_add(obj, VIRTIO_MEM_BLOCK_SIZE_PROP, "size", 1201 virtio_mem_get_block_size, virtio_mem_set_block_size, 1202 NULL, NULL); 1203 } 1204 1205 static Property virtio_mem_properties[] = { 1206 DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP, VirtIOMEM, addr, 0), 1207 DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP, VirtIOMEM, node, 0), 1208 DEFINE_PROP_BOOL(VIRTIO_MEM_PREALLOC_PROP, VirtIOMEM, prealloc, false), 1209 DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP, VirtIOMEM, memdev, 1210 TYPE_MEMORY_BACKEND, HostMemoryBackend *), 1211 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) 1212 DEFINE_PROP_ON_OFF_AUTO(VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP, VirtIOMEM, 1213 unplugged_inaccessible, ON_OFF_AUTO_AUTO), 1214 #endif 1215 DEFINE_PROP_END_OF_LIST(), 1216 }; 1217 1218 static uint64_t virtio_mem_rdm_get_min_granularity(const RamDiscardManager *rdm, 1219 const MemoryRegion *mr) 1220 { 1221 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1222 1223 g_assert(mr == &vmem->memdev->mr); 1224 return vmem->block_size; 1225 } 1226 1227 static bool virtio_mem_rdm_is_populated(const RamDiscardManager *rdm, 1228 const MemoryRegionSection *s) 1229 { 1230 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1231 uint64_t start_gpa = vmem->addr + s->offset_within_region; 1232 uint64_t end_gpa = start_gpa + int128_get64(s->size); 1233 1234 g_assert(s->mr == &vmem->memdev->mr); 1235 1236 start_gpa = QEMU_ALIGN_DOWN(start_gpa, vmem->block_size); 1237 end_gpa = QEMU_ALIGN_UP(end_gpa, vmem->block_size); 1238 1239 if (!virtio_mem_valid_range(vmem, start_gpa, end_gpa - start_gpa)) { 1240 return false; 1241 } 1242 1243 return virtio_mem_test_bitmap(vmem, start_gpa, end_gpa - start_gpa, true); 1244 } 1245 1246 struct VirtIOMEMReplayData { 1247 void *fn; 1248 void *opaque; 1249 }; 1250 1251 static int virtio_mem_rdm_replay_populated_cb(MemoryRegionSection *s, void *arg) 1252 { 1253 struct VirtIOMEMReplayData *data = arg; 1254 1255 return ((ReplayRamPopulate)data->fn)(s, data->opaque); 1256 } 1257 1258 static int virtio_mem_rdm_replay_populated(const RamDiscardManager *rdm, 1259 MemoryRegionSection *s, 1260 ReplayRamPopulate replay_fn, 1261 void *opaque) 1262 { 1263 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1264 struct VirtIOMEMReplayData data = { 1265 .fn = replay_fn, 1266 .opaque = opaque, 1267 }; 1268 1269 g_assert(s->mr == &vmem->memdev->mr); 1270 return virtio_mem_for_each_plugged_section(vmem, s, &data, 1271 virtio_mem_rdm_replay_populated_cb); 1272 } 1273 1274 static int virtio_mem_rdm_replay_discarded_cb(MemoryRegionSection *s, 1275 void *arg) 1276 { 1277 struct VirtIOMEMReplayData *data = arg; 1278 1279 ((ReplayRamDiscard)data->fn)(s, data->opaque); 1280 return 0; 1281 } 1282 1283 static void virtio_mem_rdm_replay_discarded(const RamDiscardManager *rdm, 1284 MemoryRegionSection *s, 1285 ReplayRamDiscard replay_fn, 1286 void *opaque) 1287 { 1288 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1289 struct VirtIOMEMReplayData data = { 1290 .fn = replay_fn, 1291 .opaque = opaque, 1292 }; 1293 1294 g_assert(s->mr == &vmem->memdev->mr); 1295 virtio_mem_for_each_unplugged_section(vmem, s, &data, 1296 virtio_mem_rdm_replay_discarded_cb); 1297 } 1298 1299 static void virtio_mem_rdm_register_listener(RamDiscardManager *rdm, 1300 RamDiscardListener *rdl, 1301 MemoryRegionSection *s) 1302 { 1303 VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1304 int ret; 1305 1306 g_assert(s->mr == &vmem->memdev->mr); 1307 rdl->section = memory_region_section_new_copy(s); 1308 1309 QLIST_INSERT_HEAD(&vmem->rdl_list, rdl, next); 1310 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 1311 virtio_mem_notify_populate_cb); 1312 if (ret) { 1313 error_report("%s: Replaying plugged ranges failed: %s", __func__, 1314 strerror(-ret)); 1315 } 1316 } 1317 1318 static void virtio_mem_rdm_unregister_listener(RamDiscardManager *rdm, 1319 RamDiscardListener *rdl) 1320 { 1321 VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1322 1323 g_assert(rdl->section->mr == &vmem->memdev->mr); 1324 if (vmem->size) { 1325 if (rdl->double_discard_supported) { 1326 rdl->notify_discard(rdl, rdl->section); 1327 } else { 1328 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 1329 virtio_mem_notify_discard_cb); 1330 } 1331 } 1332 1333 memory_region_section_free_copy(rdl->section); 1334 rdl->section = NULL; 1335 QLIST_REMOVE(rdl, next); 1336 } 1337 1338 static void virtio_mem_class_init(ObjectClass *klass, void *data) 1339 { 1340 DeviceClass *dc = DEVICE_CLASS(klass); 1341 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 1342 VirtIOMEMClass *vmc = VIRTIO_MEM_CLASS(klass); 1343 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(klass); 1344 1345 device_class_set_props(dc, virtio_mem_properties); 1346 dc->vmsd = &vmstate_virtio_mem; 1347 1348 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1349 vdc->realize = virtio_mem_device_realize; 1350 vdc->unrealize = virtio_mem_device_unrealize; 1351 vdc->get_config = virtio_mem_get_config; 1352 vdc->get_features = virtio_mem_get_features; 1353 vdc->validate_features = virtio_mem_validate_features; 1354 vdc->vmsd = &vmstate_virtio_mem_device; 1355 1356 vmc->fill_device_info = virtio_mem_fill_device_info; 1357 vmc->get_memory_region = virtio_mem_get_memory_region; 1358 vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier; 1359 vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier; 1360 1361 rdmc->get_min_granularity = virtio_mem_rdm_get_min_granularity; 1362 rdmc->is_populated = virtio_mem_rdm_is_populated; 1363 rdmc->replay_populated = virtio_mem_rdm_replay_populated; 1364 rdmc->replay_discarded = virtio_mem_rdm_replay_discarded; 1365 rdmc->register_listener = virtio_mem_rdm_register_listener; 1366 rdmc->unregister_listener = virtio_mem_rdm_unregister_listener; 1367 } 1368 1369 static const TypeInfo virtio_mem_info = { 1370 .name = TYPE_VIRTIO_MEM, 1371 .parent = TYPE_VIRTIO_DEVICE, 1372 .instance_size = sizeof(VirtIOMEM), 1373 .instance_init = virtio_mem_instance_init, 1374 .class_init = virtio_mem_class_init, 1375 .class_size = sizeof(VirtIOMEMClass), 1376 .interfaces = (InterfaceInfo[]) { 1377 { TYPE_RAM_DISCARD_MANAGER }, 1378 { } 1379 }, 1380 }; 1381 1382 static void virtio_register_types(void) 1383 { 1384 type_register_static(&virtio_mem_info); 1385 } 1386 1387 type_init(virtio_register_types) 1388