1 /* 2 * Virtio MEM device 3 * 4 * Copyright (C) 2020 Red Hat, Inc. 5 * 6 * Authors: 7 * David Hildenbrand <david@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include "qemu/iov.h" 15 #include "qemu/cutils.h" 16 #include "qemu/error-report.h" 17 #include "qemu/units.h" 18 #include "sysemu/numa.h" 19 #include "sysemu/sysemu.h" 20 #include "sysemu/reset.h" 21 #include "hw/virtio/virtio.h" 22 #include "hw/virtio/virtio-bus.h" 23 #include "hw/virtio/virtio-access.h" 24 #include "hw/virtio/virtio-mem.h" 25 #include "qapi/error.h" 26 #include "qapi/visitor.h" 27 #include "exec/ram_addr.h" 28 #include "migration/misc.h" 29 #include "hw/boards.h" 30 #include "hw/qdev-properties.h" 31 #include CONFIG_DEVICES 32 #include "trace.h" 33 34 /* 35 * We only had legacy x86 guests that did not support 36 * VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE. Other targets don't have legacy guests. 37 */ 38 #if defined(TARGET_X86_64) || defined(TARGET_I386) 39 #define VIRTIO_MEM_HAS_LEGACY_GUESTS 40 #endif 41 42 /* 43 * Let's not allow blocks smaller than 1 MiB, for example, to keep the tracking 44 * bitmap small. 45 */ 46 #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)(1 * MiB)) 47 48 static uint32_t virtio_mem_default_thp_size(void) 49 { 50 uint32_t default_thp_size = VIRTIO_MEM_MIN_BLOCK_SIZE; 51 52 #if defined(__x86_64__) || defined(__arm__) || defined(__powerpc64__) 53 default_thp_size = 2 * MiB; 54 #elif defined(__aarch64__) 55 if (qemu_real_host_page_size() == 4 * KiB) { 56 default_thp_size = 2 * MiB; 57 } else if (qemu_real_host_page_size() == 16 * KiB) { 58 default_thp_size = 32 * MiB; 59 } else if (qemu_real_host_page_size() == 64 * KiB) { 60 default_thp_size = 512 * MiB; 61 } 62 #endif 63 64 return default_thp_size; 65 } 66 67 /* 68 * We want to have a reasonable default block size such that 69 * 1. We avoid splitting THPs when unplugging memory, which degrades 70 * performance. 71 * 2. We avoid placing THPs for plugged blocks that also cover unplugged 72 * blocks. 73 * 74 * The actual THP size might differ between Linux kernels, so we try to probe 75 * it. In the future (if we ever run into issues regarding 2.), we might want 76 * to disable THP in case we fail to properly probe the THP size, or if the 77 * block size is configured smaller than the THP size. 78 */ 79 static uint32_t thp_size; 80 81 #define HPAGE_PMD_SIZE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" 82 static uint32_t virtio_mem_thp_size(void) 83 { 84 gchar *content = NULL; 85 const char *endptr; 86 uint64_t tmp; 87 88 if (thp_size) { 89 return thp_size; 90 } 91 92 /* 93 * Try to probe the actual THP size, fallback to (sane but eventually 94 * incorrect) default sizes. 95 */ 96 if (g_file_get_contents(HPAGE_PMD_SIZE_PATH, &content, NULL, NULL) && 97 !qemu_strtou64(content, &endptr, 0, &tmp) && 98 (!endptr || *endptr == '\n')) { 99 /* Sanity-check the value and fallback to something reasonable. */ 100 if (!tmp || !is_power_of_2(tmp)) { 101 warn_report("Read unsupported THP size: %" PRIx64, tmp); 102 } else { 103 thp_size = tmp; 104 } 105 } 106 107 if (!thp_size) { 108 thp_size = virtio_mem_default_thp_size(); 109 warn_report("Could not detect THP size, falling back to %" PRIx64 110 " MiB.", thp_size / MiB); 111 } 112 113 g_free(content); 114 return thp_size; 115 } 116 117 static uint64_t virtio_mem_default_block_size(RAMBlock *rb) 118 { 119 const uint64_t page_size = qemu_ram_pagesize(rb); 120 121 /* We can have hugetlbfs with a page size smaller than the THP size. */ 122 if (page_size == qemu_real_host_page_size()) { 123 return MAX(page_size, virtio_mem_thp_size()); 124 } 125 return MAX(page_size, VIRTIO_MEM_MIN_BLOCK_SIZE); 126 } 127 128 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) 129 static bool virtio_mem_has_shared_zeropage(RAMBlock *rb) 130 { 131 /* 132 * We only have a guaranteed shared zeropage on ordinary MAP_PRIVATE 133 * anonymous RAM. In any other case, reading unplugged *can* populate a 134 * fresh page, consuming actual memory. 135 */ 136 return !qemu_ram_is_shared(rb) && rb->fd < 0 && 137 qemu_ram_pagesize(rb) == qemu_real_host_page_size(); 138 } 139 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ 140 141 /* 142 * Size the usable region bigger than the requested size if possible. Esp. 143 * Linux guests will only add (aligned) memory blocks in case they fully 144 * fit into the usable region, but plug+online only a subset of the pages. 145 * The memory block size corresponds mostly to the section size. 146 * 147 * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and 148 * a section size of 512MB on arm64 (as long as the start address is properly 149 * aligned, similar to ordinary DIMMs). 150 * 151 * We can change this at any time and maybe even make it configurable if 152 * necessary (as the section size can change). But it's more likely that the 153 * section size will rather get smaller and not bigger over time. 154 */ 155 #if defined(TARGET_X86_64) || defined(TARGET_I386) 156 #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB)) 157 #elif defined(TARGET_ARM) 158 #define VIRTIO_MEM_USABLE_EXTENT (2 * (512 * MiB)) 159 #else 160 #error VIRTIO_MEM_USABLE_EXTENT not defined 161 #endif 162 163 static bool virtio_mem_is_busy(void) 164 { 165 /* 166 * Postcopy cannot handle concurrent discards and we don't want to migrate 167 * pages on-demand with stale content when plugging new blocks. 168 * 169 * For precopy, we don't want unplugged blocks in our migration stream, and 170 * when plugging new blocks, the page content might differ between source 171 * and destination (observable by the guest when not initializing pages 172 * after plugging them) until we're running on the destination (as we didn't 173 * migrate these blocks when they were unplugged). 174 */ 175 return migration_in_incoming_postcopy() || !migration_is_idle(); 176 } 177 178 typedef int (*virtio_mem_range_cb)(const VirtIOMEM *vmem, void *arg, 179 uint64_t offset, uint64_t size); 180 181 static int virtio_mem_for_each_unplugged_range(const VirtIOMEM *vmem, void *arg, 182 virtio_mem_range_cb cb) 183 { 184 unsigned long first_zero_bit, last_zero_bit; 185 uint64_t offset, size; 186 int ret = 0; 187 188 first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size); 189 while (first_zero_bit < vmem->bitmap_size) { 190 offset = first_zero_bit * vmem->block_size; 191 last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, 192 first_zero_bit + 1) - 1; 193 size = (last_zero_bit - first_zero_bit + 1) * vmem->block_size; 194 195 ret = cb(vmem, arg, offset, size); 196 if (ret) { 197 break; 198 } 199 first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, 200 last_zero_bit + 2); 201 } 202 return ret; 203 } 204 205 /* 206 * Adjust the memory section to cover the intersection with the given range. 207 * 208 * Returns false if the intersection is empty, otherwise returns true. 209 */ 210 static bool virito_mem_intersect_memory_section(MemoryRegionSection *s, 211 uint64_t offset, uint64_t size) 212 { 213 uint64_t start = MAX(s->offset_within_region, offset); 214 uint64_t end = MIN(s->offset_within_region + int128_get64(s->size), 215 offset + size); 216 217 if (end <= start) { 218 return false; 219 } 220 221 s->offset_within_address_space += start - s->offset_within_region; 222 s->offset_within_region = start; 223 s->size = int128_make64(end - start); 224 return true; 225 } 226 227 typedef int (*virtio_mem_section_cb)(MemoryRegionSection *s, void *arg); 228 229 static int virtio_mem_for_each_plugged_section(const VirtIOMEM *vmem, 230 MemoryRegionSection *s, 231 void *arg, 232 virtio_mem_section_cb cb) 233 { 234 unsigned long first_bit, last_bit; 235 uint64_t offset, size; 236 int ret = 0; 237 238 first_bit = s->offset_within_region / vmem->bitmap_size; 239 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, first_bit); 240 while (first_bit < vmem->bitmap_size) { 241 MemoryRegionSection tmp = *s; 242 243 offset = first_bit * vmem->block_size; 244 last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, 245 first_bit + 1) - 1; 246 size = (last_bit - first_bit + 1) * vmem->block_size; 247 248 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 249 break; 250 } 251 ret = cb(&tmp, arg); 252 if (ret) { 253 break; 254 } 255 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, 256 last_bit + 2); 257 } 258 return ret; 259 } 260 261 static int virtio_mem_for_each_unplugged_section(const VirtIOMEM *vmem, 262 MemoryRegionSection *s, 263 void *arg, 264 virtio_mem_section_cb cb) 265 { 266 unsigned long first_bit, last_bit; 267 uint64_t offset, size; 268 int ret = 0; 269 270 first_bit = s->offset_within_region / vmem->bitmap_size; 271 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, first_bit); 272 while (first_bit < vmem->bitmap_size) { 273 MemoryRegionSection tmp = *s; 274 275 offset = first_bit * vmem->block_size; 276 last_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, 277 first_bit + 1) - 1; 278 size = (last_bit - first_bit + 1) * vmem->block_size; 279 280 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 281 break; 282 } 283 ret = cb(&tmp, arg); 284 if (ret) { 285 break; 286 } 287 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, 288 last_bit + 2); 289 } 290 return ret; 291 } 292 293 static int virtio_mem_notify_populate_cb(MemoryRegionSection *s, void *arg) 294 { 295 RamDiscardListener *rdl = arg; 296 297 return rdl->notify_populate(rdl, s); 298 } 299 300 static int virtio_mem_notify_discard_cb(MemoryRegionSection *s, void *arg) 301 { 302 RamDiscardListener *rdl = arg; 303 304 rdl->notify_discard(rdl, s); 305 return 0; 306 } 307 308 static void virtio_mem_notify_unplug(VirtIOMEM *vmem, uint64_t offset, 309 uint64_t size) 310 { 311 RamDiscardListener *rdl; 312 313 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 314 MemoryRegionSection tmp = *rdl->section; 315 316 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 317 continue; 318 } 319 rdl->notify_discard(rdl, &tmp); 320 } 321 } 322 323 static int virtio_mem_notify_plug(VirtIOMEM *vmem, uint64_t offset, 324 uint64_t size) 325 { 326 RamDiscardListener *rdl, *rdl2; 327 int ret = 0; 328 329 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 330 MemoryRegionSection tmp = *rdl->section; 331 332 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 333 continue; 334 } 335 ret = rdl->notify_populate(rdl, &tmp); 336 if (ret) { 337 break; 338 } 339 } 340 341 if (ret) { 342 /* Notify all already-notified listeners. */ 343 QLIST_FOREACH(rdl2, &vmem->rdl_list, next) { 344 MemoryRegionSection tmp = *rdl->section; 345 346 if (rdl2 == rdl) { 347 break; 348 } 349 if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { 350 continue; 351 } 352 rdl2->notify_discard(rdl2, &tmp); 353 } 354 } 355 return ret; 356 } 357 358 static void virtio_mem_notify_unplug_all(VirtIOMEM *vmem) 359 { 360 RamDiscardListener *rdl; 361 362 if (!vmem->size) { 363 return; 364 } 365 366 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 367 if (rdl->double_discard_supported) { 368 rdl->notify_discard(rdl, rdl->section); 369 } else { 370 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 371 virtio_mem_notify_discard_cb); 372 } 373 } 374 } 375 376 static bool virtio_mem_test_bitmap(const VirtIOMEM *vmem, uint64_t start_gpa, 377 uint64_t size, bool plugged) 378 { 379 const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; 380 const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; 381 unsigned long found_bit; 382 383 /* We fake a shorter bitmap to avoid searching too far. */ 384 if (plugged) { 385 found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit); 386 } else { 387 found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit); 388 } 389 return found_bit > last_bit; 390 } 391 392 static void virtio_mem_set_bitmap(VirtIOMEM *vmem, uint64_t start_gpa, 393 uint64_t size, bool plugged) 394 { 395 const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; 396 const unsigned long nbits = size / vmem->block_size; 397 398 if (plugged) { 399 bitmap_set(vmem->bitmap, bit, nbits); 400 } else { 401 bitmap_clear(vmem->bitmap, bit, nbits); 402 } 403 } 404 405 static void virtio_mem_send_response(VirtIOMEM *vmem, VirtQueueElement *elem, 406 struct virtio_mem_resp *resp) 407 { 408 VirtIODevice *vdev = VIRTIO_DEVICE(vmem); 409 VirtQueue *vq = vmem->vq; 410 411 trace_virtio_mem_send_response(le16_to_cpu(resp->type)); 412 iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp)); 413 414 virtqueue_push(vq, elem, sizeof(*resp)); 415 virtio_notify(vdev, vq); 416 } 417 418 static void virtio_mem_send_response_simple(VirtIOMEM *vmem, 419 VirtQueueElement *elem, 420 uint16_t type) 421 { 422 struct virtio_mem_resp resp = { 423 .type = cpu_to_le16(type), 424 }; 425 426 virtio_mem_send_response(vmem, elem, &resp); 427 } 428 429 static bool virtio_mem_valid_range(const VirtIOMEM *vmem, uint64_t gpa, 430 uint64_t size) 431 { 432 if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) { 433 return false; 434 } 435 if (gpa + size < gpa || !size) { 436 return false; 437 } 438 if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) { 439 return false; 440 } 441 if (gpa + size > vmem->addr + vmem->usable_region_size) { 442 return false; 443 } 444 return true; 445 } 446 447 static int virtio_mem_set_block_state(VirtIOMEM *vmem, uint64_t start_gpa, 448 uint64_t size, bool plug) 449 { 450 const uint64_t offset = start_gpa - vmem->addr; 451 RAMBlock *rb = vmem->memdev->mr.ram_block; 452 453 if (virtio_mem_is_busy()) { 454 return -EBUSY; 455 } 456 457 if (!plug) { 458 if (ram_block_discard_range(rb, offset, size)) { 459 return -EBUSY; 460 } 461 virtio_mem_notify_unplug(vmem, offset, size); 462 } else { 463 int ret = 0; 464 465 if (vmem->prealloc) { 466 void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; 467 int fd = memory_region_get_fd(&vmem->memdev->mr); 468 Error *local_err = NULL; 469 470 qemu_prealloc_mem(fd, area, size, 1, NULL, &local_err); 471 if (local_err) { 472 static bool warned; 473 474 /* 475 * Warn only once, we don't want to fill the log with these 476 * warnings. 477 */ 478 if (!warned) { 479 warn_report_err(local_err); 480 warned = true; 481 } else { 482 error_free(local_err); 483 } 484 ret = -EBUSY; 485 } 486 } 487 if (!ret) { 488 ret = virtio_mem_notify_plug(vmem, offset, size); 489 } 490 491 if (ret) { 492 /* Could be preallocation or a notifier populated memory. */ 493 ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size); 494 return -EBUSY; 495 } 496 } 497 virtio_mem_set_bitmap(vmem, start_gpa, size, plug); 498 return 0; 499 } 500 501 static int virtio_mem_state_change_request(VirtIOMEM *vmem, uint64_t gpa, 502 uint16_t nb_blocks, bool plug) 503 { 504 const uint64_t size = nb_blocks * vmem->block_size; 505 int ret; 506 507 if (!virtio_mem_valid_range(vmem, gpa, size)) { 508 return VIRTIO_MEM_RESP_ERROR; 509 } 510 511 if (plug && (vmem->size + size > vmem->requested_size)) { 512 return VIRTIO_MEM_RESP_NACK; 513 } 514 515 /* test if really all blocks are in the opposite state */ 516 if (!virtio_mem_test_bitmap(vmem, gpa, size, !plug)) { 517 return VIRTIO_MEM_RESP_ERROR; 518 } 519 520 ret = virtio_mem_set_block_state(vmem, gpa, size, plug); 521 if (ret) { 522 return VIRTIO_MEM_RESP_BUSY; 523 } 524 if (plug) { 525 vmem->size += size; 526 } else { 527 vmem->size -= size; 528 } 529 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); 530 return VIRTIO_MEM_RESP_ACK; 531 } 532 533 static void virtio_mem_plug_request(VirtIOMEM *vmem, VirtQueueElement *elem, 534 struct virtio_mem_req *req) 535 { 536 const uint64_t gpa = le64_to_cpu(req->u.plug.addr); 537 const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks); 538 uint16_t type; 539 540 trace_virtio_mem_plug_request(gpa, nb_blocks); 541 type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, true); 542 virtio_mem_send_response_simple(vmem, elem, type); 543 } 544 545 static void virtio_mem_unplug_request(VirtIOMEM *vmem, VirtQueueElement *elem, 546 struct virtio_mem_req *req) 547 { 548 const uint64_t gpa = le64_to_cpu(req->u.unplug.addr); 549 const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks); 550 uint16_t type; 551 552 trace_virtio_mem_unplug_request(gpa, nb_blocks); 553 type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, false); 554 virtio_mem_send_response_simple(vmem, elem, type); 555 } 556 557 static void virtio_mem_resize_usable_region(VirtIOMEM *vmem, 558 uint64_t requested_size, 559 bool can_shrink) 560 { 561 uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr), 562 requested_size + VIRTIO_MEM_USABLE_EXTENT); 563 564 /* The usable region size always has to be multiples of the block size. */ 565 newsize = QEMU_ALIGN_UP(newsize, vmem->block_size); 566 567 if (!requested_size) { 568 newsize = 0; 569 } 570 571 if (newsize < vmem->usable_region_size && !can_shrink) { 572 return; 573 } 574 575 trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize); 576 vmem->usable_region_size = newsize; 577 } 578 579 static int virtio_mem_unplug_all(VirtIOMEM *vmem) 580 { 581 RAMBlock *rb = vmem->memdev->mr.ram_block; 582 583 if (virtio_mem_is_busy()) { 584 return -EBUSY; 585 } 586 587 if (ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb))) { 588 return -EBUSY; 589 } 590 virtio_mem_notify_unplug_all(vmem); 591 592 bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size); 593 if (vmem->size) { 594 vmem->size = 0; 595 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); 596 } 597 trace_virtio_mem_unplugged_all(); 598 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); 599 return 0; 600 } 601 602 static void virtio_mem_unplug_all_request(VirtIOMEM *vmem, 603 VirtQueueElement *elem) 604 { 605 trace_virtio_mem_unplug_all_request(); 606 if (virtio_mem_unplug_all(vmem)) { 607 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_BUSY); 608 } else { 609 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ACK); 610 } 611 } 612 613 static void virtio_mem_state_request(VirtIOMEM *vmem, VirtQueueElement *elem, 614 struct virtio_mem_req *req) 615 { 616 const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks); 617 const uint64_t gpa = le64_to_cpu(req->u.state.addr); 618 const uint64_t size = nb_blocks * vmem->block_size; 619 struct virtio_mem_resp resp = { 620 .type = cpu_to_le16(VIRTIO_MEM_RESP_ACK), 621 }; 622 623 trace_virtio_mem_state_request(gpa, nb_blocks); 624 if (!virtio_mem_valid_range(vmem, gpa, size)) { 625 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ERROR); 626 return; 627 } 628 629 if (virtio_mem_test_bitmap(vmem, gpa, size, true)) { 630 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED); 631 } else if (virtio_mem_test_bitmap(vmem, gpa, size, false)) { 632 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED); 633 } else { 634 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_MIXED); 635 } 636 trace_virtio_mem_state_response(le16_to_cpu(resp.u.state.state)); 637 virtio_mem_send_response(vmem, elem, &resp); 638 } 639 640 static void virtio_mem_handle_request(VirtIODevice *vdev, VirtQueue *vq) 641 { 642 const int len = sizeof(struct virtio_mem_req); 643 VirtIOMEM *vmem = VIRTIO_MEM(vdev); 644 VirtQueueElement *elem; 645 struct virtio_mem_req req; 646 uint16_t type; 647 648 while (true) { 649 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 650 if (!elem) { 651 return; 652 } 653 654 if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) { 655 virtio_error(vdev, "virtio-mem protocol violation: invalid request" 656 " size: %d", len); 657 virtqueue_detach_element(vq, elem, 0); 658 g_free(elem); 659 return; 660 } 661 662 if (iov_size(elem->in_sg, elem->in_num) < 663 sizeof(struct virtio_mem_resp)) { 664 virtio_error(vdev, "virtio-mem protocol violation: not enough space" 665 " for response: %zu", 666 iov_size(elem->in_sg, elem->in_num)); 667 virtqueue_detach_element(vq, elem, 0); 668 g_free(elem); 669 return; 670 } 671 672 type = le16_to_cpu(req.type); 673 switch (type) { 674 case VIRTIO_MEM_REQ_PLUG: 675 virtio_mem_plug_request(vmem, elem, &req); 676 break; 677 case VIRTIO_MEM_REQ_UNPLUG: 678 virtio_mem_unplug_request(vmem, elem, &req); 679 break; 680 case VIRTIO_MEM_REQ_UNPLUG_ALL: 681 virtio_mem_unplug_all_request(vmem, elem); 682 break; 683 case VIRTIO_MEM_REQ_STATE: 684 virtio_mem_state_request(vmem, elem, &req); 685 break; 686 default: 687 virtio_error(vdev, "virtio-mem protocol violation: unknown request" 688 " type: %d", type); 689 virtqueue_detach_element(vq, elem, 0); 690 g_free(elem); 691 return; 692 } 693 694 g_free(elem); 695 } 696 } 697 698 static void virtio_mem_get_config(VirtIODevice *vdev, uint8_t *config_data) 699 { 700 VirtIOMEM *vmem = VIRTIO_MEM(vdev); 701 struct virtio_mem_config *config = (void *) config_data; 702 703 config->block_size = cpu_to_le64(vmem->block_size); 704 config->node_id = cpu_to_le16(vmem->node); 705 config->requested_size = cpu_to_le64(vmem->requested_size); 706 config->plugged_size = cpu_to_le64(vmem->size); 707 config->addr = cpu_to_le64(vmem->addr); 708 config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr)); 709 config->usable_region_size = cpu_to_le64(vmem->usable_region_size); 710 } 711 712 static uint64_t virtio_mem_get_features(VirtIODevice *vdev, uint64_t features, 713 Error **errp) 714 { 715 MachineState *ms = MACHINE(qdev_get_machine()); 716 VirtIOMEM *vmem = VIRTIO_MEM(vdev); 717 718 if (ms->numa_state) { 719 #if defined(CONFIG_ACPI) 720 virtio_add_feature(&features, VIRTIO_MEM_F_ACPI_PXM); 721 #endif 722 } 723 assert(vmem->unplugged_inaccessible != ON_OFF_AUTO_AUTO); 724 if (vmem->unplugged_inaccessible == ON_OFF_AUTO_ON) { 725 virtio_add_feature(&features, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE); 726 } 727 return features; 728 } 729 730 static int virtio_mem_validate_features(VirtIODevice *vdev) 731 { 732 if (virtio_host_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE) && 733 !virtio_vdev_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE)) { 734 return -EFAULT; 735 } 736 return 0; 737 } 738 739 static void virtio_mem_system_reset(void *opaque) 740 { 741 VirtIOMEM *vmem = VIRTIO_MEM(opaque); 742 743 /* 744 * During usual resets, we will unplug all memory and shrink the usable 745 * region size. This is, however, not possible in all scenarios. Then, 746 * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL). 747 */ 748 virtio_mem_unplug_all(vmem); 749 } 750 751 static void virtio_mem_device_realize(DeviceState *dev, Error **errp) 752 { 753 MachineState *ms = MACHINE(qdev_get_machine()); 754 int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0; 755 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 756 VirtIOMEM *vmem = VIRTIO_MEM(dev); 757 uint64_t page_size; 758 RAMBlock *rb; 759 int ret; 760 761 if (!vmem->memdev) { 762 error_setg(errp, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP); 763 return; 764 } else if (host_memory_backend_is_mapped(vmem->memdev)) { 765 error_setg(errp, "'%s' property specifies a busy memdev: %s", 766 VIRTIO_MEM_MEMDEV_PROP, 767 object_get_canonical_path_component(OBJECT(vmem->memdev))); 768 return; 769 } else if (!memory_region_is_ram(&vmem->memdev->mr) || 770 memory_region_is_rom(&vmem->memdev->mr) || 771 !vmem->memdev->mr.ram_block) { 772 error_setg(errp, "'%s' property specifies an unsupported memdev", 773 VIRTIO_MEM_MEMDEV_PROP); 774 return; 775 } 776 777 if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) || 778 (!nb_numa_nodes && vmem->node)) { 779 error_setg(errp, "'%s' property has value '%" PRIu32 "', which exceeds" 780 "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP, 781 vmem->node, nb_numa_nodes ? nb_numa_nodes : 1); 782 return; 783 } 784 785 if (enable_mlock) { 786 error_setg(errp, "Incompatible with mlock"); 787 return; 788 } 789 790 rb = vmem->memdev->mr.ram_block; 791 page_size = qemu_ram_pagesize(rb); 792 793 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) 794 switch (vmem->unplugged_inaccessible) { 795 case ON_OFF_AUTO_AUTO: 796 if (virtio_mem_has_shared_zeropage(rb)) { 797 vmem->unplugged_inaccessible = ON_OFF_AUTO_OFF; 798 } else { 799 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; 800 } 801 break; 802 case ON_OFF_AUTO_OFF: 803 if (!virtio_mem_has_shared_zeropage(rb)) { 804 warn_report("'%s' property set to 'off' with a memdev that does" 805 " not support the shared zeropage.", 806 VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP); 807 } 808 break; 809 default: 810 break; 811 } 812 #else /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ 813 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; 814 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ 815 816 /* 817 * If the block size wasn't configured by the user, use a sane default. This 818 * allows using hugetlbfs backends of any page size without manual 819 * intervention. 820 */ 821 if (!vmem->block_size) { 822 vmem->block_size = virtio_mem_default_block_size(rb); 823 } 824 825 if (vmem->block_size < page_size) { 826 error_setg(errp, "'%s' property has to be at least the page size (0x%" 827 PRIx64 ")", VIRTIO_MEM_BLOCK_SIZE_PROP, page_size); 828 return; 829 } else if (vmem->block_size < virtio_mem_default_block_size(rb)) { 830 warn_report("'%s' property is smaller than the default block size (%" 831 PRIx64 " MiB)", VIRTIO_MEM_BLOCK_SIZE_PROP, 832 virtio_mem_default_block_size(rb) / MiB); 833 } 834 if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) { 835 error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 836 ")", VIRTIO_MEM_REQUESTED_SIZE_PROP, 837 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); 838 return; 839 } else if (!QEMU_IS_ALIGNED(vmem->addr, vmem->block_size)) { 840 error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 841 ")", VIRTIO_MEM_ADDR_PROP, VIRTIO_MEM_BLOCK_SIZE_PROP, 842 vmem->block_size); 843 return; 844 } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr), 845 vmem->block_size)) { 846 error_setg(errp, "'%s' property memdev size has to be multiples of" 847 "'%s' (0x%" PRIx64 ")", VIRTIO_MEM_MEMDEV_PROP, 848 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); 849 return; 850 } 851 852 if (ram_block_coordinated_discard_require(true)) { 853 error_setg(errp, "Discarding RAM is disabled"); 854 return; 855 } 856 857 ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb)); 858 if (ret) { 859 error_setg_errno(errp, -ret, "Unexpected error discarding RAM"); 860 ram_block_coordinated_discard_require(false); 861 return; 862 } 863 864 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); 865 866 vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) / 867 vmem->block_size; 868 vmem->bitmap = bitmap_new(vmem->bitmap_size); 869 870 virtio_init(vdev, VIRTIO_ID_MEM, sizeof(struct virtio_mem_config)); 871 vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request); 872 873 host_memory_backend_set_mapped(vmem->memdev, true); 874 vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem)); 875 qemu_register_reset(virtio_mem_system_reset, vmem); 876 877 /* 878 * Set ourselves as RamDiscardManager before the plug handler maps the 879 * memory region and exposes it via an address space. 880 */ 881 memory_region_set_ram_discard_manager(&vmem->memdev->mr, 882 RAM_DISCARD_MANAGER(vmem)); 883 } 884 885 static void virtio_mem_device_unrealize(DeviceState *dev) 886 { 887 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 888 VirtIOMEM *vmem = VIRTIO_MEM(dev); 889 890 /* 891 * The unplug handler unmapped the memory region, it cannot be 892 * found via an address space anymore. Unset ourselves. 893 */ 894 memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL); 895 qemu_unregister_reset(virtio_mem_system_reset, vmem); 896 vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem)); 897 host_memory_backend_set_mapped(vmem->memdev, false); 898 virtio_del_queue(vdev, 0); 899 virtio_cleanup(vdev); 900 g_free(vmem->bitmap); 901 ram_block_coordinated_discard_require(false); 902 } 903 904 static int virtio_mem_discard_range_cb(const VirtIOMEM *vmem, void *arg, 905 uint64_t offset, uint64_t size) 906 { 907 RAMBlock *rb = vmem->memdev->mr.ram_block; 908 909 return ram_block_discard_range(rb, offset, size) ? -EINVAL : 0; 910 } 911 912 static int virtio_mem_restore_unplugged(VirtIOMEM *vmem) 913 { 914 /* Make sure all memory is really discarded after migration. */ 915 return virtio_mem_for_each_unplugged_range(vmem, NULL, 916 virtio_mem_discard_range_cb); 917 } 918 919 static int virtio_mem_post_load(void *opaque, int version_id) 920 { 921 VirtIOMEM *vmem = VIRTIO_MEM(opaque); 922 RamDiscardListener *rdl; 923 int ret; 924 925 /* 926 * We started out with all memory discarded and our memory region is mapped 927 * into an address space. Replay, now that we updated the bitmap. 928 */ 929 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 930 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 931 virtio_mem_notify_populate_cb); 932 if (ret) { 933 return ret; 934 } 935 } 936 937 if (migration_in_incoming_postcopy()) { 938 return 0; 939 } 940 941 return virtio_mem_restore_unplugged(vmem); 942 } 943 944 typedef struct VirtIOMEMMigSanityChecks { 945 VirtIOMEM *parent; 946 uint64_t addr; 947 uint64_t region_size; 948 uint64_t block_size; 949 uint32_t node; 950 } VirtIOMEMMigSanityChecks; 951 952 static int virtio_mem_mig_sanity_checks_pre_save(void *opaque) 953 { 954 VirtIOMEMMigSanityChecks *tmp = opaque; 955 VirtIOMEM *vmem = tmp->parent; 956 957 tmp->addr = vmem->addr; 958 tmp->region_size = memory_region_size(&vmem->memdev->mr); 959 tmp->block_size = vmem->block_size; 960 tmp->node = vmem->node; 961 return 0; 962 } 963 964 static int virtio_mem_mig_sanity_checks_post_load(void *opaque, int version_id) 965 { 966 VirtIOMEMMigSanityChecks *tmp = opaque; 967 VirtIOMEM *vmem = tmp->parent; 968 const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr); 969 970 if (tmp->addr != vmem->addr) { 971 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, 972 VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr); 973 return -EINVAL; 974 } 975 /* 976 * Note: Preparation for resizeable memory regions. The maximum size 977 * of the memory region must not change during migration. 978 */ 979 if (tmp->region_size != new_region_size) { 980 error_report("Property '%s' size changed from 0x%" PRIx64 " to 0x%" 981 PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size, 982 new_region_size); 983 return -EINVAL; 984 } 985 if (tmp->block_size != vmem->block_size) { 986 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, 987 VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size, 988 vmem->block_size); 989 return -EINVAL; 990 } 991 if (tmp->node != vmem->node) { 992 error_report("Property '%s' changed from %" PRIu32 " to %" PRIu32, 993 VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node); 994 return -EINVAL; 995 } 996 return 0; 997 } 998 999 static const VMStateDescription vmstate_virtio_mem_sanity_checks = { 1000 .name = "virtio-mem-device/sanity-checks", 1001 .pre_save = virtio_mem_mig_sanity_checks_pre_save, 1002 .post_load = virtio_mem_mig_sanity_checks_post_load, 1003 .fields = (VMStateField[]) { 1004 VMSTATE_UINT64(addr, VirtIOMEMMigSanityChecks), 1005 VMSTATE_UINT64(region_size, VirtIOMEMMigSanityChecks), 1006 VMSTATE_UINT64(block_size, VirtIOMEMMigSanityChecks), 1007 VMSTATE_UINT32(node, VirtIOMEMMigSanityChecks), 1008 VMSTATE_END_OF_LIST(), 1009 }, 1010 }; 1011 1012 static const VMStateDescription vmstate_virtio_mem_device = { 1013 .name = "virtio-mem-device", 1014 .minimum_version_id = 1, 1015 .version_id = 1, 1016 .priority = MIG_PRI_VIRTIO_MEM, 1017 .post_load = virtio_mem_post_load, 1018 .fields = (VMStateField[]) { 1019 VMSTATE_WITH_TMP(VirtIOMEM, VirtIOMEMMigSanityChecks, 1020 vmstate_virtio_mem_sanity_checks), 1021 VMSTATE_UINT64(usable_region_size, VirtIOMEM), 1022 VMSTATE_UINT64(size, VirtIOMEM), 1023 VMSTATE_UINT64(requested_size, VirtIOMEM), 1024 VMSTATE_BITMAP(bitmap, VirtIOMEM, 0, bitmap_size), 1025 VMSTATE_END_OF_LIST() 1026 }, 1027 }; 1028 1029 static const VMStateDescription vmstate_virtio_mem = { 1030 .name = "virtio-mem", 1031 .minimum_version_id = 1, 1032 .version_id = 1, 1033 .fields = (VMStateField[]) { 1034 VMSTATE_VIRTIO_DEVICE, 1035 VMSTATE_END_OF_LIST() 1036 }, 1037 }; 1038 1039 static void virtio_mem_fill_device_info(const VirtIOMEM *vmem, 1040 VirtioMEMDeviceInfo *vi) 1041 { 1042 vi->memaddr = vmem->addr; 1043 vi->node = vmem->node; 1044 vi->requested_size = vmem->requested_size; 1045 vi->size = vmem->size; 1046 vi->max_size = memory_region_size(&vmem->memdev->mr); 1047 vi->block_size = vmem->block_size; 1048 vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev)); 1049 } 1050 1051 static MemoryRegion *virtio_mem_get_memory_region(VirtIOMEM *vmem, Error **errp) 1052 { 1053 if (!vmem->memdev) { 1054 error_setg(errp, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP); 1055 return NULL; 1056 } 1057 1058 return &vmem->memdev->mr; 1059 } 1060 1061 static void virtio_mem_add_size_change_notifier(VirtIOMEM *vmem, 1062 Notifier *notifier) 1063 { 1064 notifier_list_add(&vmem->size_change_notifiers, notifier); 1065 } 1066 1067 static void virtio_mem_remove_size_change_notifier(VirtIOMEM *vmem, 1068 Notifier *notifier) 1069 { 1070 notifier_remove(notifier); 1071 } 1072 1073 static void virtio_mem_get_size(Object *obj, Visitor *v, const char *name, 1074 void *opaque, Error **errp) 1075 { 1076 const VirtIOMEM *vmem = VIRTIO_MEM(obj); 1077 uint64_t value = vmem->size; 1078 1079 visit_type_size(v, name, &value, errp); 1080 } 1081 1082 static void virtio_mem_get_requested_size(Object *obj, Visitor *v, 1083 const char *name, void *opaque, 1084 Error **errp) 1085 { 1086 const VirtIOMEM *vmem = VIRTIO_MEM(obj); 1087 uint64_t value = vmem->requested_size; 1088 1089 visit_type_size(v, name, &value, errp); 1090 } 1091 1092 static void virtio_mem_set_requested_size(Object *obj, Visitor *v, 1093 const char *name, void *opaque, 1094 Error **errp) 1095 { 1096 VirtIOMEM *vmem = VIRTIO_MEM(obj); 1097 Error *err = NULL; 1098 uint64_t value; 1099 1100 visit_type_size(v, name, &value, &err); 1101 if (err) { 1102 error_propagate(errp, err); 1103 return; 1104 } 1105 1106 /* 1107 * The block size and memory backend are not fixed until the device was 1108 * realized. realize() will verify these properties then. 1109 */ 1110 if (DEVICE(obj)->realized) { 1111 if (!QEMU_IS_ALIGNED(value, vmem->block_size)) { 1112 error_setg(errp, "'%s' has to be multiples of '%s' (0x%" PRIx64 1113 ")", name, VIRTIO_MEM_BLOCK_SIZE_PROP, 1114 vmem->block_size); 1115 return; 1116 } else if (value > memory_region_size(&vmem->memdev->mr)) { 1117 error_setg(errp, "'%s' cannot exceed the memory backend size" 1118 "(0x%" PRIx64 ")", name, 1119 memory_region_size(&vmem->memdev->mr)); 1120 return; 1121 } 1122 1123 if (value != vmem->requested_size) { 1124 virtio_mem_resize_usable_region(vmem, value, false); 1125 vmem->requested_size = value; 1126 } 1127 /* 1128 * Trigger a config update so the guest gets notified. We trigger 1129 * even if the size didn't change (especially helpful for debugging). 1130 */ 1131 virtio_notify_config(VIRTIO_DEVICE(vmem)); 1132 } else { 1133 vmem->requested_size = value; 1134 } 1135 } 1136 1137 static void virtio_mem_get_block_size(Object *obj, Visitor *v, const char *name, 1138 void *opaque, Error **errp) 1139 { 1140 const VirtIOMEM *vmem = VIRTIO_MEM(obj); 1141 uint64_t value = vmem->block_size; 1142 1143 /* 1144 * If not configured by the user (and we're not realized yet), use the 1145 * default block size we would use with the current memory backend. 1146 */ 1147 if (!value) { 1148 if (vmem->memdev && memory_region_is_ram(&vmem->memdev->mr)) { 1149 value = virtio_mem_default_block_size(vmem->memdev->mr.ram_block); 1150 } else { 1151 value = virtio_mem_thp_size(); 1152 } 1153 } 1154 1155 visit_type_size(v, name, &value, errp); 1156 } 1157 1158 static void virtio_mem_set_block_size(Object *obj, Visitor *v, const char *name, 1159 void *opaque, Error **errp) 1160 { 1161 VirtIOMEM *vmem = VIRTIO_MEM(obj); 1162 Error *err = NULL; 1163 uint64_t value; 1164 1165 if (DEVICE(obj)->realized) { 1166 error_setg(errp, "'%s' cannot be changed", name); 1167 return; 1168 } 1169 1170 visit_type_size(v, name, &value, &err); 1171 if (err) { 1172 error_propagate(errp, err); 1173 return; 1174 } 1175 1176 if (value < VIRTIO_MEM_MIN_BLOCK_SIZE) { 1177 error_setg(errp, "'%s' property has to be at least 0x%" PRIx32, name, 1178 VIRTIO_MEM_MIN_BLOCK_SIZE); 1179 return; 1180 } else if (!is_power_of_2(value)) { 1181 error_setg(errp, "'%s' property has to be a power of two", name); 1182 return; 1183 } 1184 vmem->block_size = value; 1185 } 1186 1187 static void virtio_mem_instance_init(Object *obj) 1188 { 1189 VirtIOMEM *vmem = VIRTIO_MEM(obj); 1190 1191 notifier_list_init(&vmem->size_change_notifiers); 1192 QLIST_INIT(&vmem->rdl_list); 1193 1194 object_property_add(obj, VIRTIO_MEM_SIZE_PROP, "size", virtio_mem_get_size, 1195 NULL, NULL, NULL); 1196 object_property_add(obj, VIRTIO_MEM_REQUESTED_SIZE_PROP, "size", 1197 virtio_mem_get_requested_size, 1198 virtio_mem_set_requested_size, NULL, NULL); 1199 object_property_add(obj, VIRTIO_MEM_BLOCK_SIZE_PROP, "size", 1200 virtio_mem_get_block_size, virtio_mem_set_block_size, 1201 NULL, NULL); 1202 } 1203 1204 static Property virtio_mem_properties[] = { 1205 DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP, VirtIOMEM, addr, 0), 1206 DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP, VirtIOMEM, node, 0), 1207 DEFINE_PROP_BOOL(VIRTIO_MEM_PREALLOC_PROP, VirtIOMEM, prealloc, false), 1208 DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP, VirtIOMEM, memdev, 1209 TYPE_MEMORY_BACKEND, HostMemoryBackend *), 1210 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) 1211 DEFINE_PROP_ON_OFF_AUTO(VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP, VirtIOMEM, 1212 unplugged_inaccessible, ON_OFF_AUTO_AUTO), 1213 #endif 1214 DEFINE_PROP_END_OF_LIST(), 1215 }; 1216 1217 static uint64_t virtio_mem_rdm_get_min_granularity(const RamDiscardManager *rdm, 1218 const MemoryRegion *mr) 1219 { 1220 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1221 1222 g_assert(mr == &vmem->memdev->mr); 1223 return vmem->block_size; 1224 } 1225 1226 static bool virtio_mem_rdm_is_populated(const RamDiscardManager *rdm, 1227 const MemoryRegionSection *s) 1228 { 1229 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1230 uint64_t start_gpa = vmem->addr + s->offset_within_region; 1231 uint64_t end_gpa = start_gpa + int128_get64(s->size); 1232 1233 g_assert(s->mr == &vmem->memdev->mr); 1234 1235 start_gpa = QEMU_ALIGN_DOWN(start_gpa, vmem->block_size); 1236 end_gpa = QEMU_ALIGN_UP(end_gpa, vmem->block_size); 1237 1238 if (!virtio_mem_valid_range(vmem, start_gpa, end_gpa - start_gpa)) { 1239 return false; 1240 } 1241 1242 return virtio_mem_test_bitmap(vmem, start_gpa, end_gpa - start_gpa, true); 1243 } 1244 1245 struct VirtIOMEMReplayData { 1246 void *fn; 1247 void *opaque; 1248 }; 1249 1250 static int virtio_mem_rdm_replay_populated_cb(MemoryRegionSection *s, void *arg) 1251 { 1252 struct VirtIOMEMReplayData *data = arg; 1253 1254 return ((ReplayRamPopulate)data->fn)(s, data->opaque); 1255 } 1256 1257 static int virtio_mem_rdm_replay_populated(const RamDiscardManager *rdm, 1258 MemoryRegionSection *s, 1259 ReplayRamPopulate replay_fn, 1260 void *opaque) 1261 { 1262 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1263 struct VirtIOMEMReplayData data = { 1264 .fn = replay_fn, 1265 .opaque = opaque, 1266 }; 1267 1268 g_assert(s->mr == &vmem->memdev->mr); 1269 return virtio_mem_for_each_plugged_section(vmem, s, &data, 1270 virtio_mem_rdm_replay_populated_cb); 1271 } 1272 1273 static int virtio_mem_rdm_replay_discarded_cb(MemoryRegionSection *s, 1274 void *arg) 1275 { 1276 struct VirtIOMEMReplayData *data = arg; 1277 1278 ((ReplayRamDiscard)data->fn)(s, data->opaque); 1279 return 0; 1280 } 1281 1282 static void virtio_mem_rdm_replay_discarded(const RamDiscardManager *rdm, 1283 MemoryRegionSection *s, 1284 ReplayRamDiscard replay_fn, 1285 void *opaque) 1286 { 1287 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1288 struct VirtIOMEMReplayData data = { 1289 .fn = replay_fn, 1290 .opaque = opaque, 1291 }; 1292 1293 g_assert(s->mr == &vmem->memdev->mr); 1294 virtio_mem_for_each_unplugged_section(vmem, s, &data, 1295 virtio_mem_rdm_replay_discarded_cb); 1296 } 1297 1298 static void virtio_mem_rdm_register_listener(RamDiscardManager *rdm, 1299 RamDiscardListener *rdl, 1300 MemoryRegionSection *s) 1301 { 1302 VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1303 int ret; 1304 1305 g_assert(s->mr == &vmem->memdev->mr); 1306 rdl->section = memory_region_section_new_copy(s); 1307 1308 QLIST_INSERT_HEAD(&vmem->rdl_list, rdl, next); 1309 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 1310 virtio_mem_notify_populate_cb); 1311 if (ret) { 1312 error_report("%s: Replaying plugged ranges failed: %s", __func__, 1313 strerror(-ret)); 1314 } 1315 } 1316 1317 static void virtio_mem_rdm_unregister_listener(RamDiscardManager *rdm, 1318 RamDiscardListener *rdl) 1319 { 1320 VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1321 1322 g_assert(rdl->section->mr == &vmem->memdev->mr); 1323 if (vmem->size) { 1324 if (rdl->double_discard_supported) { 1325 rdl->notify_discard(rdl, rdl->section); 1326 } else { 1327 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 1328 virtio_mem_notify_discard_cb); 1329 } 1330 } 1331 1332 memory_region_section_free_copy(rdl->section); 1333 rdl->section = NULL; 1334 QLIST_REMOVE(rdl, next); 1335 } 1336 1337 static void virtio_mem_class_init(ObjectClass *klass, void *data) 1338 { 1339 DeviceClass *dc = DEVICE_CLASS(klass); 1340 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 1341 VirtIOMEMClass *vmc = VIRTIO_MEM_CLASS(klass); 1342 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(klass); 1343 1344 device_class_set_props(dc, virtio_mem_properties); 1345 dc->vmsd = &vmstate_virtio_mem; 1346 1347 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1348 vdc->realize = virtio_mem_device_realize; 1349 vdc->unrealize = virtio_mem_device_unrealize; 1350 vdc->get_config = virtio_mem_get_config; 1351 vdc->get_features = virtio_mem_get_features; 1352 vdc->validate_features = virtio_mem_validate_features; 1353 vdc->vmsd = &vmstate_virtio_mem_device; 1354 1355 vmc->fill_device_info = virtio_mem_fill_device_info; 1356 vmc->get_memory_region = virtio_mem_get_memory_region; 1357 vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier; 1358 vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier; 1359 1360 rdmc->get_min_granularity = virtio_mem_rdm_get_min_granularity; 1361 rdmc->is_populated = virtio_mem_rdm_is_populated; 1362 rdmc->replay_populated = virtio_mem_rdm_replay_populated; 1363 rdmc->replay_discarded = virtio_mem_rdm_replay_discarded; 1364 rdmc->register_listener = virtio_mem_rdm_register_listener; 1365 rdmc->unregister_listener = virtio_mem_rdm_unregister_listener; 1366 } 1367 1368 static const TypeInfo virtio_mem_info = { 1369 .name = TYPE_VIRTIO_MEM, 1370 .parent = TYPE_VIRTIO_DEVICE, 1371 .instance_size = sizeof(VirtIOMEM), 1372 .instance_init = virtio_mem_instance_init, 1373 .class_init = virtio_mem_class_init, 1374 .class_size = sizeof(VirtIOMEMClass), 1375 .interfaces = (InterfaceInfo[]) { 1376 { TYPE_RAM_DISCARD_MANAGER }, 1377 { } 1378 }, 1379 }; 1380 1381 static void virtio_register_types(void) 1382 { 1383 type_register_static(&virtio_mem_info); 1384 } 1385 1386 type_init(virtio_register_types) 1387