1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * VMware Balloon driver. 4 * 5 * Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved. 6 * 7 * This is VMware physical memory management driver for Linux. The driver 8 * acts like a "balloon" that can be inflated to reclaim physical pages by 9 * reserving them in the guest and invalidating them in the monitor, 10 * freeing up the underlying machine pages so they can be allocated to 11 * other guests. The balloon can also be deflated to allow the guest to 12 * use more physical memory. Higher level policies can control the sizes 13 * of balloons in VMs in order to manage physical memory resources. 14 */ 15 16 //#define DEBUG 17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 18 19 #include <linux/types.h> 20 #include <linux/io.h> 21 #include <linux/kernel.h> 22 #include <linux/mm.h> 23 #include <linux/vmalloc.h> 24 #include <linux/sched.h> 25 #include <linux/module.h> 26 #include <linux/workqueue.h> 27 #include <linux/debugfs.h> 28 #include <linux/seq_file.h> 29 #include <linux/rwsem.h> 30 #include <linux/slab.h> 31 #include <linux/spinlock.h> 32 #include <linux/mount.h> 33 #include <linux/pseudo_fs.h> 34 #include <linux/balloon_compaction.h> 35 #include <linux/vmw_vmci_defs.h> 36 #include <linux/vmw_vmci_api.h> 37 #include <asm/hypervisor.h> 38 39 MODULE_AUTHOR("VMware, Inc."); 40 MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver"); 41 MODULE_ALIAS("dmi:*:svnVMware*:*"); 42 MODULE_ALIAS("vmware_vmmemctl"); 43 MODULE_LICENSE("GPL"); 44 45 static bool __read_mostly vmwballoon_shrinker_enable; 46 module_param(vmwballoon_shrinker_enable, bool, 0444); 47 MODULE_PARM_DESC(vmwballoon_shrinker_enable, 48 "Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performance."); 49 50 /* Delay in seconds after shrink before inflation. */ 51 #define VMBALLOON_SHRINK_DELAY (5) 52 53 /* Maximum number of refused pages we accumulate during inflation cycle */ 54 #define VMW_BALLOON_MAX_REFUSED 16 55 56 /* Magic number for the balloon mount-point */ 57 #define BALLOON_VMW_MAGIC 0x0ba11007 58 59 /* 60 * Hypervisor communication port definitions. 61 */ 62 #define VMW_BALLOON_HV_PORT 0x5670 63 #define VMW_BALLOON_HV_MAGIC 0x456c6d6f 64 #define VMW_BALLOON_GUEST_ID 1 /* Linux */ 65 66 enum vmwballoon_capabilities { 67 /* 68 * Bit 0 is reserved and not associated to any capability. 69 */ 70 VMW_BALLOON_BASIC_CMDS = (1 << 1), 71 VMW_BALLOON_BATCHED_CMDS = (1 << 2), 72 VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3), 73 VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4), 74 VMW_BALLOON_64_BIT_TARGET = (1 << 5) 75 }; 76 77 #define VMW_BALLOON_CAPABILITIES_COMMON (VMW_BALLOON_BASIC_CMDS \ 78 | VMW_BALLOON_BATCHED_CMDS \ 79 | VMW_BALLOON_BATCHED_2M_CMDS \ 80 | VMW_BALLOON_SIGNALLED_WAKEUP_CMD) 81 82 #define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT) 83 84 /* 85 * 64-bit targets are only supported in 64-bit 86 */ 87 #ifdef CONFIG_64BIT 88 #define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_CAPABILITIES_COMMON \ 89 | VMW_BALLOON_64_BIT_TARGET) 90 #else 91 #define VMW_BALLOON_CAPABILITIES VMW_BALLOON_CAPABILITIES_COMMON 92 #endif 93 94 enum vmballoon_page_size_type { 95 VMW_BALLOON_4K_PAGE, 96 VMW_BALLOON_2M_PAGE, 97 VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE 98 }; 99 100 #define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1) 101 102 static const char * const vmballoon_page_size_names[] = { 103 [VMW_BALLOON_4K_PAGE] = "4k", 104 [VMW_BALLOON_2M_PAGE] = "2M" 105 }; 106 107 enum vmballoon_op { 108 VMW_BALLOON_INFLATE, 109 VMW_BALLOON_DEFLATE 110 }; 111 112 enum vmballoon_op_stat_type { 113 VMW_BALLOON_OP_STAT, 114 VMW_BALLOON_OP_FAIL_STAT 115 }; 116 117 #define VMW_BALLOON_OP_STAT_TYPES (VMW_BALLOON_OP_FAIL_STAT + 1) 118 119 /** 120 * enum vmballoon_cmd_type - backdoor commands. 121 * 122 * Availability of the commands is as followed: 123 * 124 * %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and 125 * %VMW_BALLOON_CMD_GUEST_ID are always available. 126 * 127 * If the host reports %VMW_BALLOON_BASIC_CMDS are supported then 128 * %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available. 129 * 130 * If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then 131 * %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands 132 * are available. 133 * 134 * If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then 135 * %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK 136 * are supported. 137 * 138 * If the host reports VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then 139 * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported. 140 * 141 * @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor. 142 * @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size. 143 * @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page. 144 * @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about 145 * to be deflated from the balloon. 146 * @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that 147 * runs in the VM. 148 * @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of 149 * ballooned pages (up to 512). 150 * @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of 151 * pages that are about to be deflated from the 152 * balloon (up to 512). 153 * @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK 154 * for 2MB pages. 155 * @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to 156 * @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB 157 * pages. 158 * @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification 159 * that would be invoked when the balloon 160 * size changes. 161 * @VMW_BALLOON_CMD_LAST: Value of the last command. 162 */ 163 enum vmballoon_cmd_type { 164 VMW_BALLOON_CMD_START, 165 VMW_BALLOON_CMD_GET_TARGET, 166 VMW_BALLOON_CMD_LOCK, 167 VMW_BALLOON_CMD_UNLOCK, 168 VMW_BALLOON_CMD_GUEST_ID, 169 /* No command 5 */ 170 VMW_BALLOON_CMD_BATCHED_LOCK = 6, 171 VMW_BALLOON_CMD_BATCHED_UNLOCK, 172 VMW_BALLOON_CMD_BATCHED_2M_LOCK, 173 VMW_BALLOON_CMD_BATCHED_2M_UNLOCK, 174 VMW_BALLOON_CMD_VMCI_DOORBELL_SET, 175 VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET, 176 }; 177 178 #define VMW_BALLOON_CMD_NUM (VMW_BALLOON_CMD_LAST + 1) 179 180 enum vmballoon_error_codes { 181 VMW_BALLOON_SUCCESS, 182 VMW_BALLOON_ERROR_CMD_INVALID, 183 VMW_BALLOON_ERROR_PPN_INVALID, 184 VMW_BALLOON_ERROR_PPN_LOCKED, 185 VMW_BALLOON_ERROR_PPN_UNLOCKED, 186 VMW_BALLOON_ERROR_PPN_PINNED, 187 VMW_BALLOON_ERROR_PPN_NOTNEEDED, 188 VMW_BALLOON_ERROR_RESET, 189 VMW_BALLOON_ERROR_BUSY 190 }; 191 192 #define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000) 193 194 #define VMW_BALLOON_CMD_WITH_TARGET_MASK \ 195 ((1UL << VMW_BALLOON_CMD_GET_TARGET) | \ 196 (1UL << VMW_BALLOON_CMD_LOCK) | \ 197 (1UL << VMW_BALLOON_CMD_UNLOCK) | \ 198 (1UL << VMW_BALLOON_CMD_BATCHED_LOCK) | \ 199 (1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK) | \ 200 (1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK) | \ 201 (1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK)) 202 203 static const char * const vmballoon_cmd_names[] = { 204 [VMW_BALLOON_CMD_START] = "start", 205 [VMW_BALLOON_CMD_GET_TARGET] = "target", 206 [VMW_BALLOON_CMD_LOCK] = "lock", 207 [VMW_BALLOON_CMD_UNLOCK] = "unlock", 208 [VMW_BALLOON_CMD_GUEST_ID] = "guestType", 209 [VMW_BALLOON_CMD_BATCHED_LOCK] = "batchLock", 210 [VMW_BALLOON_CMD_BATCHED_UNLOCK] = "batchUnlock", 211 [VMW_BALLOON_CMD_BATCHED_2M_LOCK] = "2m-lock", 212 [VMW_BALLOON_CMD_BATCHED_2M_UNLOCK] = "2m-unlock", 213 [VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet" 214 }; 215 216 enum vmballoon_stat_page { 217 VMW_BALLOON_PAGE_STAT_ALLOC, 218 VMW_BALLOON_PAGE_STAT_ALLOC_FAIL, 219 VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC, 220 VMW_BALLOON_PAGE_STAT_REFUSED_FREE, 221 VMW_BALLOON_PAGE_STAT_FREE, 222 VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE 223 }; 224 225 #define VMW_BALLOON_PAGE_STAT_NUM (VMW_BALLOON_PAGE_STAT_LAST + 1) 226 227 enum vmballoon_stat_general { 228 VMW_BALLOON_STAT_TIMER, 229 VMW_BALLOON_STAT_DOORBELL, 230 VMW_BALLOON_STAT_RESET, 231 VMW_BALLOON_STAT_SHRINK, 232 VMW_BALLOON_STAT_SHRINK_FREE, 233 VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE 234 }; 235 236 #define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1) 237 238 static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching); 239 static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled); 240 241 struct vmballoon_ctl { 242 struct list_head pages; 243 struct list_head refused_pages; 244 struct list_head prealloc_pages; 245 unsigned int n_refused_pages; 246 unsigned int n_pages; 247 enum vmballoon_page_size_type page_size; 248 enum vmballoon_op op; 249 }; 250 251 /** 252 * struct vmballoon_batch_entry - a batch entry for lock or unlock. 253 * 254 * @status: the status of the operation, which is written by the hypervisor. 255 * @reserved: reserved for future use. Must be set to zero. 256 * @pfn: the physical frame number of the page to be locked or unlocked. 257 */ 258 struct vmballoon_batch_entry { 259 u64 status : 5; 260 u64 reserved : PAGE_SHIFT - 5; 261 u64 pfn : 52; 262 } __packed; 263 264 struct vmballoon { 265 /** 266 * @max_page_size: maximum supported page size for ballooning. 267 * 268 * Protected by @conf_sem 269 */ 270 enum vmballoon_page_size_type max_page_size; 271 272 /** 273 * @size: balloon actual size in basic page size (frames). 274 * 275 * While we currently do not support size which is bigger than 32-bit, 276 * in preparation for future support, use 64-bits. 277 */ 278 atomic64_t size; 279 280 /** 281 * @target: balloon target size in basic page size (frames). 282 * 283 * We do not protect the target under the assumption that setting the 284 * value is always done through a single write. If this assumption ever 285 * breaks, we would have to use X_ONCE for accesses, and suffer the less 286 * optimized code. Although we may read stale target value if multiple 287 * accesses happen at once, the performance impact should be minor. 288 */ 289 unsigned long target; 290 291 /** 292 * @reset_required: reset flag 293 * 294 * Setting this flag may introduce races, but the code is expected to 295 * handle them gracefully. In the worst case, another operation will 296 * fail as reset did not take place. Clearing the flag is done while 297 * holding @conf_sem for write. 298 */ 299 bool reset_required; 300 301 /** 302 * @capabilities: hypervisor balloon capabilities. 303 * 304 * Protected by @conf_sem. 305 */ 306 unsigned long capabilities; 307 308 /** 309 * @batch_page: pointer to communication batch page. 310 * 311 * When batching is used, batch_page points to a page, which holds up to 312 * %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking. 313 */ 314 struct vmballoon_batch_entry *batch_page; 315 316 /** 317 * @batch_max_pages: maximum pages that can be locked/unlocked. 318 * 319 * Indicates the number of pages that the hypervisor can lock or unlock 320 * at once, according to whether batching is enabled. If batching is 321 * disabled, only a single page can be locked/unlock on each operation. 322 * 323 * Protected by @conf_sem. 324 */ 325 unsigned int batch_max_pages; 326 327 /** 328 * @page: page to be locked/unlocked by the hypervisor 329 * 330 * @page is only used when batching is disabled and a single page is 331 * reclaimed on each iteration. 332 * 333 * Protected by @comm_lock. 334 */ 335 struct page *page; 336 337 /** 338 * @shrink_timeout: timeout until the next inflation. 339 * 340 * After an shrink event, indicates the time in jiffies after which 341 * inflation is allowed again. Can be written concurrently with reads, 342 * so must use READ_ONCE/WRITE_ONCE when accessing. 343 */ 344 unsigned long shrink_timeout; 345 346 /* statistics */ 347 struct vmballoon_stats *stats; 348 349 #ifdef CONFIG_DEBUG_FS 350 /* debugfs file exporting statistics */ 351 struct dentry *dbg_entry; 352 #endif 353 354 /** 355 * @b_dev_info: balloon device information descriptor. 356 */ 357 struct balloon_dev_info b_dev_info; 358 359 struct delayed_work dwork; 360 361 /** 362 * @huge_pages - list of the inflated 2MB pages. 363 * 364 * Protected by @b_dev_info.pages_lock . 365 */ 366 struct list_head huge_pages; 367 368 /** 369 * @vmci_doorbell. 370 * 371 * Protected by @conf_sem. 372 */ 373 struct vmci_handle vmci_doorbell; 374 375 /** 376 * @conf_sem: semaphore to protect the configuration and the statistics. 377 */ 378 struct rw_semaphore conf_sem; 379 380 /** 381 * @comm_lock: lock to protect the communication with the host. 382 * 383 * Lock ordering: @conf_sem -> @comm_lock . 384 */ 385 spinlock_t comm_lock; 386 387 /** 388 * @shrinker: shrinker interface that is used to avoid over-inflation. 389 */ 390 struct shrinker shrinker; 391 392 /** 393 * @shrinker_registered: whether the shrinker was registered. 394 * 395 * The shrinker interface does not handle gracefully the removal of 396 * shrinker that was not registered before. This indication allows to 397 * simplify the unregistration process. 398 */ 399 bool shrinker_registered; 400 }; 401 402 static struct vmballoon balloon; 403 404 struct vmballoon_stats { 405 /* timer / doorbell operations */ 406 atomic64_t general_stat[VMW_BALLOON_STAT_NUM]; 407 408 /* allocation statistics for huge and small pages */ 409 atomic64_t 410 page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES]; 411 412 /* Monitor operations: total operations, and failures */ 413 atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES]; 414 }; 415 416 static inline bool is_vmballoon_stats_on(void) 417 { 418 return IS_ENABLED(CONFIG_DEBUG_FS) && 419 static_branch_unlikely(&balloon_stat_enabled); 420 } 421 422 static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op, 423 enum vmballoon_op_stat_type type) 424 { 425 if (is_vmballoon_stats_on()) 426 atomic64_inc(&b->stats->ops[op][type]); 427 } 428 429 static inline void vmballoon_stats_gen_inc(struct vmballoon *b, 430 enum vmballoon_stat_general stat) 431 { 432 if (is_vmballoon_stats_on()) 433 atomic64_inc(&b->stats->general_stat[stat]); 434 } 435 436 static inline void vmballoon_stats_gen_add(struct vmballoon *b, 437 enum vmballoon_stat_general stat, 438 unsigned int val) 439 { 440 if (is_vmballoon_stats_on()) 441 atomic64_add(val, &b->stats->general_stat[stat]); 442 } 443 444 static inline void vmballoon_stats_page_inc(struct vmballoon *b, 445 enum vmballoon_stat_page stat, 446 enum vmballoon_page_size_type size) 447 { 448 if (is_vmballoon_stats_on()) 449 atomic64_inc(&b->stats->page_stat[stat][size]); 450 } 451 452 static inline void vmballoon_stats_page_add(struct vmballoon *b, 453 enum vmballoon_stat_page stat, 454 enum vmballoon_page_size_type size, 455 unsigned int val) 456 { 457 if (is_vmballoon_stats_on()) 458 atomic64_add(val, &b->stats->page_stat[stat][size]); 459 } 460 461 static inline unsigned long 462 __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1, 463 unsigned long arg2, unsigned long *result) 464 { 465 unsigned long status, dummy1, dummy2, dummy3, local_result; 466 467 vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT); 468 469 asm volatile ("inl %%dx" : 470 "=a"(status), 471 "=c"(dummy1), 472 "=d"(dummy2), 473 "=b"(local_result), 474 "=S"(dummy3) : 475 "0"(VMW_BALLOON_HV_MAGIC), 476 "1"(cmd), 477 "2"(VMW_BALLOON_HV_PORT), 478 "3"(arg1), 479 "4"(arg2) : 480 "memory"); 481 482 /* update the result if needed */ 483 if (result) 484 *result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 : 485 local_result; 486 487 /* update target when applicable */ 488 if (status == VMW_BALLOON_SUCCESS && 489 ((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK)) 490 WRITE_ONCE(b->target, local_result); 491 492 if (status != VMW_BALLOON_SUCCESS && 493 status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) { 494 vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT); 495 pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n", 496 __func__, vmballoon_cmd_names[cmd], arg1, arg2, 497 status); 498 } 499 500 /* mark reset required accordingly */ 501 if (status == VMW_BALLOON_ERROR_RESET) 502 b->reset_required = true; 503 504 return status; 505 } 506 507 static __always_inline unsigned long 508 vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1, 509 unsigned long arg2) 510 { 511 unsigned long dummy; 512 513 return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy); 514 } 515 516 /* 517 * Send "start" command to the host, communicating supported version 518 * of the protocol. 519 */ 520 static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps) 521 { 522 unsigned long status, capabilities; 523 524 status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0, 525 &capabilities); 526 527 switch (status) { 528 case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES: 529 b->capabilities = capabilities; 530 break; 531 case VMW_BALLOON_SUCCESS: 532 b->capabilities = VMW_BALLOON_BASIC_CMDS; 533 break; 534 default: 535 return -EIO; 536 } 537 538 /* 539 * 2MB pages are only supported with batching. If batching is for some 540 * reason disabled, do not use 2MB pages, since otherwise the legacy 541 * mechanism is used with 2MB pages, causing a failure. 542 */ 543 b->max_page_size = VMW_BALLOON_4K_PAGE; 544 if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) && 545 (b->capabilities & VMW_BALLOON_BATCHED_CMDS)) 546 b->max_page_size = VMW_BALLOON_2M_PAGE; 547 548 549 return 0; 550 } 551 552 /** 553 * vmballoon_send_guest_id - communicate guest type to the host. 554 * 555 * @b: pointer to the balloon. 556 * 557 * Communicate guest type to the host so that it can adjust ballooning 558 * algorithm to the one most appropriate for the guest. This command 559 * is normally issued after sending "start" command and is part of 560 * standard reset sequence. 561 * 562 * Return: zero on success or appropriate error code. 563 */ 564 static int vmballoon_send_guest_id(struct vmballoon *b) 565 { 566 unsigned long status; 567 568 status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID, 569 VMW_BALLOON_GUEST_ID, 0); 570 571 return status == VMW_BALLOON_SUCCESS ? 0 : -EIO; 572 } 573 574 /** 575 * vmballoon_page_order() - return the order of the page 576 * @page_size: the size of the page. 577 * 578 * Return: the allocation order. 579 */ 580 static inline 581 unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size) 582 { 583 return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0; 584 } 585 586 /** 587 * vmballoon_page_in_frames() - returns the number of frames in a page. 588 * @page_size: the size of the page. 589 * 590 * Return: the number of 4k frames. 591 */ 592 static inline unsigned int 593 vmballoon_page_in_frames(enum vmballoon_page_size_type page_size) 594 { 595 return 1 << vmballoon_page_order(page_size); 596 } 597 598 /** 599 * vmballoon_mark_page_offline() - mark a page as offline 600 * @page: pointer for the page. 601 * @page_size: the size of the page. 602 */ 603 static void 604 vmballoon_mark_page_offline(struct page *page, 605 enum vmballoon_page_size_type page_size) 606 { 607 int i; 608 609 for (i = 0; i < vmballoon_page_in_frames(page_size); i++) 610 __SetPageOffline(page + i); 611 } 612 613 /** 614 * vmballoon_mark_page_online() - mark a page as online 615 * @page: pointer for the page. 616 * @page_size: the size of the page. 617 */ 618 static void 619 vmballoon_mark_page_online(struct page *page, 620 enum vmballoon_page_size_type page_size) 621 { 622 int i; 623 624 for (i = 0; i < vmballoon_page_in_frames(page_size); i++) 625 __ClearPageOffline(page + i); 626 } 627 628 /** 629 * vmballoon_send_get_target() - Retrieve desired balloon size from the host. 630 * 631 * @b: pointer to the balloon. 632 * 633 * Return: zero on success, EINVAL if limit does not fit in 32-bit, as required 634 * by the host-guest protocol and EIO if an error occurred in communicating with 635 * the host. 636 */ 637 static int vmballoon_send_get_target(struct vmballoon *b) 638 { 639 unsigned long status; 640 unsigned long limit; 641 642 limit = totalram_pages(); 643 644 /* Ensure limit fits in 32-bits if 64-bit targets are not supported */ 645 if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) && 646 limit != (u32)limit) 647 return -EINVAL; 648 649 status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0); 650 651 return status == VMW_BALLOON_SUCCESS ? 0 : -EIO; 652 } 653 654 /** 655 * vmballoon_alloc_page_list - allocates a list of pages. 656 * 657 * @b: pointer to the balloon. 658 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation. 659 * @req_n_pages: the number of requested pages. 660 * 661 * Tries to allocate @req_n_pages. Add them to the list of balloon pages in 662 * @ctl.pages and updates @ctl.n_pages to reflect the number of pages. 663 * 664 * Return: zero on success or error code otherwise. 665 */ 666 static int vmballoon_alloc_page_list(struct vmballoon *b, 667 struct vmballoon_ctl *ctl, 668 unsigned int req_n_pages) 669 { 670 struct page *page; 671 unsigned int i; 672 673 for (i = 0; i < req_n_pages; i++) { 674 /* 675 * First check if we happen to have pages that were allocated 676 * before. This happens when 2MB page rejected during inflation 677 * by the hypervisor, and then split into 4KB pages. 678 */ 679 if (!list_empty(&ctl->prealloc_pages)) { 680 page = list_first_entry(&ctl->prealloc_pages, 681 struct page, lru); 682 list_del(&page->lru); 683 } else { 684 if (ctl->page_size == VMW_BALLOON_2M_PAGE) 685 page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN| 686 __GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER); 687 else 688 page = balloon_page_alloc(); 689 690 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC, 691 ctl->page_size); 692 } 693 694 if (page) { 695 /* Success. Add the page to the list and continue. */ 696 list_add(&page->lru, &ctl->pages); 697 continue; 698 } 699 700 /* Allocation failed. Update statistics and stop. */ 701 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL, 702 ctl->page_size); 703 break; 704 } 705 706 ctl->n_pages = i; 707 708 return req_n_pages == ctl->n_pages ? 0 : -ENOMEM; 709 } 710 711 /** 712 * vmballoon_handle_one_result - Handle lock/unlock result for a single page. 713 * 714 * @b: pointer for %struct vmballoon. 715 * @page: pointer for the page whose result should be handled. 716 * @page_size: size of the page. 717 * @status: status of the operation as provided by the hypervisor. 718 */ 719 static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page, 720 enum vmballoon_page_size_type page_size, 721 unsigned long status) 722 { 723 /* On success do nothing. The page is already on the balloon list. */ 724 if (likely(status == VMW_BALLOON_SUCCESS)) 725 return 0; 726 727 pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__, 728 page_to_pfn(page), status, 729 vmballoon_page_size_names[page_size]); 730 731 /* Error occurred */ 732 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC, 733 page_size); 734 735 return -EIO; 736 } 737 738 /** 739 * vmballoon_status_page - returns the status of (un)lock operation 740 * 741 * @b: pointer to the balloon. 742 * @idx: index for the page for which the operation is performed. 743 * @p: pointer to where the page struct is returned. 744 * 745 * Following a lock or unlock operation, returns the status of the operation for 746 * an individual page. Provides the page that the operation was performed on on 747 * the @page argument. 748 * 749 * Returns: The status of a lock or unlock operation for an individual page. 750 */ 751 static unsigned long vmballoon_status_page(struct vmballoon *b, int idx, 752 struct page **p) 753 { 754 if (static_branch_likely(&vmw_balloon_batching)) { 755 /* batching mode */ 756 *p = pfn_to_page(b->batch_page[idx].pfn); 757 return b->batch_page[idx].status; 758 } 759 760 /* non-batching mode */ 761 *p = b->page; 762 763 /* 764 * If a failure occurs, the indication will be provided in the status 765 * of the entire operation, which is considered before the individual 766 * page status. So for non-batching mode, the indication is always of 767 * success. 768 */ 769 return VMW_BALLOON_SUCCESS; 770 } 771 772 /** 773 * vmballoon_lock_op - notifies the host about inflated/deflated pages. 774 * @b: pointer to the balloon. 775 * @num_pages: number of inflated/deflated pages. 776 * @page_size: size of the page. 777 * @op: the type of operation (lock or unlock). 778 * 779 * Notify the host about page(s) that were ballooned (or removed from the 780 * balloon) so that host can use it without fear that guest will need it (or 781 * stop using them since the VM does). Host may reject some pages, we need to 782 * check the return value and maybe submit a different page. The pages that are 783 * inflated/deflated are pointed by @b->page. 784 * 785 * Return: result as provided by the hypervisor. 786 */ 787 static unsigned long vmballoon_lock_op(struct vmballoon *b, 788 unsigned int num_pages, 789 enum vmballoon_page_size_type page_size, 790 enum vmballoon_op op) 791 { 792 unsigned long cmd, pfn; 793 794 lockdep_assert_held(&b->comm_lock); 795 796 if (static_branch_likely(&vmw_balloon_batching)) { 797 if (op == VMW_BALLOON_INFLATE) 798 cmd = page_size == VMW_BALLOON_2M_PAGE ? 799 VMW_BALLOON_CMD_BATCHED_2M_LOCK : 800 VMW_BALLOON_CMD_BATCHED_LOCK; 801 else 802 cmd = page_size == VMW_BALLOON_2M_PAGE ? 803 VMW_BALLOON_CMD_BATCHED_2M_UNLOCK : 804 VMW_BALLOON_CMD_BATCHED_UNLOCK; 805 806 pfn = PHYS_PFN(virt_to_phys(b->batch_page)); 807 } else { 808 cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK : 809 VMW_BALLOON_CMD_UNLOCK; 810 pfn = page_to_pfn(b->page); 811 812 /* In non-batching mode, PFNs must fit in 32-bit */ 813 if (unlikely(pfn != (u32)pfn)) 814 return VMW_BALLOON_ERROR_PPN_INVALID; 815 } 816 817 return vmballoon_cmd(b, cmd, pfn, num_pages); 818 } 819 820 /** 821 * vmballoon_add_page - adds a page towards lock/unlock operation. 822 * 823 * @b: pointer to the balloon. 824 * @idx: index of the page to be ballooned in this batch. 825 * @p: pointer to the page that is about to be ballooned. 826 * 827 * Adds the page to be ballooned. Must be called while holding @comm_lock. 828 */ 829 static void vmballoon_add_page(struct vmballoon *b, unsigned int idx, 830 struct page *p) 831 { 832 lockdep_assert_held(&b->comm_lock); 833 834 if (static_branch_likely(&vmw_balloon_batching)) 835 b->batch_page[idx] = (struct vmballoon_batch_entry) 836 { .pfn = page_to_pfn(p) }; 837 else 838 b->page = p; 839 } 840 841 /** 842 * vmballoon_lock - lock or unlock a batch of pages. 843 * 844 * @b: pointer to the balloon. 845 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation. 846 * 847 * Notifies the host of about ballooned pages (after inflation or deflation, 848 * according to @ctl). If the host rejects the page put it on the 849 * @ctl refuse list. These refused page are then released when moving to the 850 * next size of pages. 851 * 852 * Note that we neither free any @page here nor put them back on the ballooned 853 * pages list. Instead we queue it for later processing. We do that for several 854 * reasons. First, we do not want to free the page under the lock. Second, it 855 * allows us to unify the handling of lock and unlock. In the inflate case, the 856 * caller will check if there are too many refused pages and release them. 857 * Although it is not identical to the past behavior, it should not affect 858 * performance. 859 */ 860 static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl) 861 { 862 unsigned long batch_status; 863 struct page *page; 864 unsigned int i, num_pages; 865 866 num_pages = ctl->n_pages; 867 if (num_pages == 0) 868 return 0; 869 870 /* communication with the host is done under the communication lock */ 871 spin_lock(&b->comm_lock); 872 873 i = 0; 874 list_for_each_entry(page, &ctl->pages, lru) 875 vmballoon_add_page(b, i++, page); 876 877 batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size, 878 ctl->op); 879 880 /* 881 * Iterate over the pages in the provided list. Since we are changing 882 * @ctl->n_pages we are saving the original value in @num_pages and 883 * use this value to bound the loop. 884 */ 885 for (i = 0; i < num_pages; i++) { 886 unsigned long status; 887 888 status = vmballoon_status_page(b, i, &page); 889 890 /* 891 * Failure of the whole batch overrides a single operation 892 * results. 893 */ 894 if (batch_status != VMW_BALLOON_SUCCESS) 895 status = batch_status; 896 897 /* Continue if no error happened */ 898 if (!vmballoon_handle_one_result(b, page, ctl->page_size, 899 status)) 900 continue; 901 902 /* 903 * Error happened. Move the pages to the refused list and update 904 * the pages number. 905 */ 906 list_move(&page->lru, &ctl->refused_pages); 907 ctl->n_pages--; 908 ctl->n_refused_pages++; 909 } 910 911 spin_unlock(&b->comm_lock); 912 913 return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO; 914 } 915 916 /** 917 * vmballoon_release_page_list() - Releases a page list 918 * 919 * @page_list: list of pages to release. 920 * @n_pages: pointer to the number of pages. 921 * @page_size: whether the pages in the list are 2MB (or else 4KB). 922 * 923 * Releases the list of pages and zeros the number of pages. 924 */ 925 static void vmballoon_release_page_list(struct list_head *page_list, 926 int *n_pages, 927 enum vmballoon_page_size_type page_size) 928 { 929 struct page *page, *tmp; 930 931 list_for_each_entry_safe(page, tmp, page_list, lru) { 932 list_del(&page->lru); 933 __free_pages(page, vmballoon_page_order(page_size)); 934 } 935 936 if (n_pages) 937 *n_pages = 0; 938 } 939 940 941 /* 942 * Release pages that were allocated while attempting to inflate the 943 * balloon but were refused by the host for one reason or another. 944 */ 945 static void vmballoon_release_refused_pages(struct vmballoon *b, 946 struct vmballoon_ctl *ctl) 947 { 948 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE, 949 ctl->page_size); 950 951 vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages, 952 ctl->page_size); 953 } 954 955 /** 956 * vmballoon_change - retrieve the required balloon change 957 * 958 * @b: pointer for the balloon. 959 * 960 * Return: the required change for the balloon size. A positive number 961 * indicates inflation, a negative number indicates a deflation. 962 */ 963 static int64_t vmballoon_change(struct vmballoon *b) 964 { 965 int64_t size, target; 966 967 size = atomic64_read(&b->size); 968 target = READ_ONCE(b->target); 969 970 /* 971 * We must cast first because of int sizes 972 * Otherwise we might get huge positives instead of negatives 973 */ 974 975 if (b->reset_required) 976 return 0; 977 978 /* consider a 2MB slack on deflate, unless the balloon is emptied */ 979 if (target < size && target != 0 && 980 size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)) 981 return 0; 982 983 /* If an out-of-memory recently occurred, inflation is disallowed. */ 984 if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout))) 985 return 0; 986 987 return target - size; 988 } 989 990 /** 991 * vmballoon_enqueue_page_list() - Enqueues list of pages after inflation. 992 * 993 * @b: pointer to balloon. 994 * @pages: list of pages to enqueue. 995 * @n_pages: pointer to number of pages in list. The value is zeroed. 996 * @page_size: whether the pages are 2MB or 4KB pages. 997 * 998 * Enqueues the provides list of pages in the ballooned page list, clears the 999 * list and zeroes the number of pages that was provided. 1000 */ 1001 static void vmballoon_enqueue_page_list(struct vmballoon *b, 1002 struct list_head *pages, 1003 unsigned int *n_pages, 1004 enum vmballoon_page_size_type page_size) 1005 { 1006 unsigned long flags; 1007 struct page *page; 1008 1009 if (page_size == VMW_BALLOON_4K_PAGE) { 1010 balloon_page_list_enqueue(&b->b_dev_info, pages); 1011 } else { 1012 /* 1013 * Keep the huge pages in a local list which is not available 1014 * for the balloon compaction mechanism. 1015 */ 1016 spin_lock_irqsave(&b->b_dev_info.pages_lock, flags); 1017 1018 list_for_each_entry(page, pages, lru) { 1019 vmballoon_mark_page_offline(page, VMW_BALLOON_2M_PAGE); 1020 } 1021 1022 list_splice_init(pages, &b->huge_pages); 1023 __count_vm_events(BALLOON_INFLATE, *n_pages * 1024 vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)); 1025 spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags); 1026 } 1027 1028 *n_pages = 0; 1029 } 1030 1031 /** 1032 * vmballoon_dequeue_page_list() - Dequeues page lists for deflation. 1033 * 1034 * @b: pointer to balloon. 1035 * @pages: list of pages to enqueue. 1036 * @n_pages: pointer to number of pages in list. The value is zeroed. 1037 * @page_size: whether the pages are 2MB or 4KB pages. 1038 * @n_req_pages: the number of requested pages. 1039 * 1040 * Dequeues the number of requested pages from the balloon for deflation. The 1041 * number of dequeued pages may be lower, if not enough pages in the requested 1042 * size are available. 1043 */ 1044 static void vmballoon_dequeue_page_list(struct vmballoon *b, 1045 struct list_head *pages, 1046 unsigned int *n_pages, 1047 enum vmballoon_page_size_type page_size, 1048 unsigned int n_req_pages) 1049 { 1050 struct page *page, *tmp; 1051 unsigned int i = 0; 1052 unsigned long flags; 1053 1054 /* In the case of 4k pages, use the compaction infrastructure */ 1055 if (page_size == VMW_BALLOON_4K_PAGE) { 1056 *n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages, 1057 n_req_pages); 1058 return; 1059 } 1060 1061 /* 2MB pages */ 1062 spin_lock_irqsave(&b->b_dev_info.pages_lock, flags); 1063 list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) { 1064 vmballoon_mark_page_online(page, VMW_BALLOON_2M_PAGE); 1065 1066 list_move(&page->lru, pages); 1067 if (++i == n_req_pages) 1068 break; 1069 } 1070 1071 __count_vm_events(BALLOON_DEFLATE, 1072 i * vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)); 1073 spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags); 1074 *n_pages = i; 1075 } 1076 1077 /** 1078 * vmballoon_split_refused_pages() - Split the 2MB refused pages to 4k. 1079 * 1080 * If inflation of 2MB pages was denied by the hypervisor, it is likely to be 1081 * due to one or few 4KB pages. These 2MB pages may keep being allocated and 1082 * then being refused. To prevent this case, this function splits the refused 1083 * pages into 4KB pages and adds them into @prealloc_pages list. 1084 * 1085 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation. 1086 */ 1087 static void vmballoon_split_refused_pages(struct vmballoon_ctl *ctl) 1088 { 1089 struct page *page, *tmp; 1090 unsigned int i, order; 1091 1092 order = vmballoon_page_order(ctl->page_size); 1093 1094 list_for_each_entry_safe(page, tmp, &ctl->refused_pages, lru) { 1095 list_del(&page->lru); 1096 split_page(page, order); 1097 for (i = 0; i < (1 << order); i++) 1098 list_add(&page[i].lru, &ctl->prealloc_pages); 1099 } 1100 ctl->n_refused_pages = 0; 1101 } 1102 1103 /** 1104 * vmballoon_inflate() - Inflate the balloon towards its target size. 1105 * 1106 * @b: pointer to the balloon. 1107 */ 1108 static void vmballoon_inflate(struct vmballoon *b) 1109 { 1110 int64_t to_inflate_frames; 1111 struct vmballoon_ctl ctl = { 1112 .pages = LIST_HEAD_INIT(ctl.pages), 1113 .refused_pages = LIST_HEAD_INIT(ctl.refused_pages), 1114 .prealloc_pages = LIST_HEAD_INIT(ctl.prealloc_pages), 1115 .page_size = b->max_page_size, 1116 .op = VMW_BALLOON_INFLATE 1117 }; 1118 1119 while ((to_inflate_frames = vmballoon_change(b)) > 0) { 1120 unsigned int to_inflate_pages, page_in_frames; 1121 int alloc_error, lock_error = 0; 1122 1123 VM_BUG_ON(!list_empty(&ctl.pages)); 1124 VM_BUG_ON(ctl.n_pages != 0); 1125 1126 page_in_frames = vmballoon_page_in_frames(ctl.page_size); 1127 1128 to_inflate_pages = min_t(unsigned long, b->batch_max_pages, 1129 DIV_ROUND_UP_ULL(to_inflate_frames, 1130 page_in_frames)); 1131 1132 /* Start by allocating */ 1133 alloc_error = vmballoon_alloc_page_list(b, &ctl, 1134 to_inflate_pages); 1135 1136 /* Actually lock the pages by telling the hypervisor */ 1137 lock_error = vmballoon_lock(b, &ctl); 1138 1139 /* 1140 * If an error indicates that something serious went wrong, 1141 * stop the inflation. 1142 */ 1143 if (lock_error) 1144 break; 1145 1146 /* Update the balloon size */ 1147 atomic64_add(ctl.n_pages * page_in_frames, &b->size); 1148 1149 vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages, 1150 ctl.page_size); 1151 1152 /* 1153 * If allocation failed or the number of refused pages exceeds 1154 * the maximum allowed, move to the next page size. 1155 */ 1156 if (alloc_error || 1157 ctl.n_refused_pages >= VMW_BALLOON_MAX_REFUSED) { 1158 if (ctl.page_size == VMW_BALLOON_4K_PAGE) 1159 break; 1160 1161 /* 1162 * Split the refused pages to 4k. This will also empty 1163 * the refused pages list. 1164 */ 1165 vmballoon_split_refused_pages(&ctl); 1166 ctl.page_size--; 1167 } 1168 1169 cond_resched(); 1170 } 1171 1172 /* 1173 * Release pages that were allocated while attempting to inflate the 1174 * balloon but were refused by the host for one reason or another, 1175 * and update the statistics. 1176 */ 1177 if (ctl.n_refused_pages != 0) 1178 vmballoon_release_refused_pages(b, &ctl); 1179 1180 vmballoon_release_page_list(&ctl.prealloc_pages, NULL, ctl.page_size); 1181 } 1182 1183 /** 1184 * vmballoon_deflate() - Decrease the size of the balloon. 1185 * 1186 * @b: pointer to the balloon 1187 * @n_frames: the number of frames to deflate. If zero, automatically 1188 * calculated according to the target size. 1189 * @coordinated: whether to coordinate with the host 1190 * 1191 * Decrease the size of the balloon allowing guest to use more memory. 1192 * 1193 * Return: The number of deflated frames (i.e., basic page size units) 1194 */ 1195 static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames, 1196 bool coordinated) 1197 { 1198 unsigned long deflated_frames = 0; 1199 unsigned long tried_frames = 0; 1200 struct vmballoon_ctl ctl = { 1201 .pages = LIST_HEAD_INIT(ctl.pages), 1202 .refused_pages = LIST_HEAD_INIT(ctl.refused_pages), 1203 .page_size = VMW_BALLOON_4K_PAGE, 1204 .op = VMW_BALLOON_DEFLATE 1205 }; 1206 1207 /* free pages to reach target */ 1208 while (true) { 1209 unsigned int to_deflate_pages, n_unlocked_frames; 1210 unsigned int page_in_frames; 1211 int64_t to_deflate_frames; 1212 bool deflated_all; 1213 1214 page_in_frames = vmballoon_page_in_frames(ctl.page_size); 1215 1216 VM_BUG_ON(!list_empty(&ctl.pages)); 1217 VM_BUG_ON(ctl.n_pages); 1218 VM_BUG_ON(!list_empty(&ctl.refused_pages)); 1219 VM_BUG_ON(ctl.n_refused_pages); 1220 1221 /* 1222 * If we were requested a specific number of frames, we try to 1223 * deflate this number of frames. Otherwise, deflation is 1224 * performed according to the target and balloon size. 1225 */ 1226 to_deflate_frames = n_frames ? n_frames - tried_frames : 1227 -vmballoon_change(b); 1228 1229 /* break if no work to do */ 1230 if (to_deflate_frames <= 0) 1231 break; 1232 1233 /* 1234 * Calculate the number of frames based on current page size, 1235 * but limit the deflated frames to a single chunk 1236 */ 1237 to_deflate_pages = min_t(unsigned long, b->batch_max_pages, 1238 DIV_ROUND_UP_ULL(to_deflate_frames, 1239 page_in_frames)); 1240 1241 /* First take the pages from the balloon pages. */ 1242 vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages, 1243 ctl.page_size, to_deflate_pages); 1244 1245 /* 1246 * Before pages are moving to the refused list, count their 1247 * frames as frames that we tried to deflate. 1248 */ 1249 tried_frames += ctl.n_pages * page_in_frames; 1250 1251 /* 1252 * Unlock the pages by communicating with the hypervisor if the 1253 * communication is coordinated (i.e., not pop). We ignore the 1254 * return code. Instead we check if all the pages we manage to 1255 * unlock all the pages. If we failed, we will move to the next 1256 * page size, and would eventually try again later. 1257 */ 1258 if (coordinated) 1259 vmballoon_lock(b, &ctl); 1260 1261 /* 1262 * Check if we deflated enough. We will move to the next page 1263 * size if we did not manage to do so. This calculation takes 1264 * place now, as once the pages are released, the number of 1265 * pages is zeroed. 1266 */ 1267 deflated_all = (ctl.n_pages == to_deflate_pages); 1268 1269 /* Update local and global counters */ 1270 n_unlocked_frames = ctl.n_pages * page_in_frames; 1271 atomic64_sub(n_unlocked_frames, &b->size); 1272 deflated_frames += n_unlocked_frames; 1273 1274 vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE, 1275 ctl.page_size, ctl.n_pages); 1276 1277 /* free the ballooned pages */ 1278 vmballoon_release_page_list(&ctl.pages, &ctl.n_pages, 1279 ctl.page_size); 1280 1281 /* Return the refused pages to the ballooned list. */ 1282 vmballoon_enqueue_page_list(b, &ctl.refused_pages, 1283 &ctl.n_refused_pages, 1284 ctl.page_size); 1285 1286 /* If we failed to unlock all the pages, move to next size. */ 1287 if (!deflated_all) { 1288 if (ctl.page_size == b->max_page_size) 1289 break; 1290 ctl.page_size++; 1291 } 1292 1293 cond_resched(); 1294 } 1295 1296 return deflated_frames; 1297 } 1298 1299 /** 1300 * vmballoon_deinit_batching - disables batching mode. 1301 * 1302 * @b: pointer to &struct vmballoon. 1303 * 1304 * Disables batching, by deallocating the page for communication with the 1305 * hypervisor and disabling the static key to indicate that batching is off. 1306 */ 1307 static void vmballoon_deinit_batching(struct vmballoon *b) 1308 { 1309 free_page((unsigned long)b->batch_page); 1310 b->batch_page = NULL; 1311 static_branch_disable(&vmw_balloon_batching); 1312 b->batch_max_pages = 1; 1313 } 1314 1315 /** 1316 * vmballoon_init_batching - enable batching mode. 1317 * 1318 * @b: pointer to &struct vmballoon. 1319 * 1320 * Enables batching, by allocating a page for communication with the hypervisor 1321 * and enabling the static_key to use batching. 1322 * 1323 * Return: zero on success or an appropriate error-code. 1324 */ 1325 static int vmballoon_init_batching(struct vmballoon *b) 1326 { 1327 struct page *page; 1328 1329 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1330 if (!page) 1331 return -ENOMEM; 1332 1333 b->batch_page = page_address(page); 1334 b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry); 1335 1336 static_branch_enable(&vmw_balloon_batching); 1337 1338 return 0; 1339 } 1340 1341 /* 1342 * Receive notification and resize balloon 1343 */ 1344 static void vmballoon_doorbell(void *client_data) 1345 { 1346 struct vmballoon *b = client_data; 1347 1348 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL); 1349 1350 mod_delayed_work(system_freezable_wq, &b->dwork, 0); 1351 } 1352 1353 /* 1354 * Clean up vmci doorbell 1355 */ 1356 static void vmballoon_vmci_cleanup(struct vmballoon *b) 1357 { 1358 vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET, 1359 VMCI_INVALID_ID, VMCI_INVALID_ID); 1360 1361 if (!vmci_handle_is_invalid(b->vmci_doorbell)) { 1362 vmci_doorbell_destroy(b->vmci_doorbell); 1363 b->vmci_doorbell = VMCI_INVALID_HANDLE; 1364 } 1365 } 1366 1367 /** 1368 * vmballoon_vmci_init - Initialize vmci doorbell. 1369 * 1370 * @b: pointer to the balloon. 1371 * 1372 * Return: zero on success or when wakeup command not supported. Error-code 1373 * otherwise. 1374 * 1375 * Initialize vmci doorbell, to get notified as soon as balloon changes. 1376 */ 1377 static int vmballoon_vmci_init(struct vmballoon *b) 1378 { 1379 unsigned long error; 1380 1381 if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0) 1382 return 0; 1383 1384 error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB, 1385 VMCI_PRIVILEGE_FLAG_RESTRICTED, 1386 vmballoon_doorbell, b); 1387 1388 if (error != VMCI_SUCCESS) 1389 goto fail; 1390 1391 error = __vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET, 1392 b->vmci_doorbell.context, 1393 b->vmci_doorbell.resource, NULL); 1394 1395 if (error != VMW_BALLOON_SUCCESS) 1396 goto fail; 1397 1398 return 0; 1399 fail: 1400 vmballoon_vmci_cleanup(b); 1401 return -EIO; 1402 } 1403 1404 /** 1405 * vmballoon_pop - Quickly release all pages allocate for the balloon. 1406 * 1407 * @b: pointer to the balloon. 1408 * 1409 * This function is called when host decides to "reset" balloon for one reason 1410 * or another. Unlike normal "deflate" we do not (shall not) notify host of the 1411 * pages being released. 1412 */ 1413 static void vmballoon_pop(struct vmballoon *b) 1414 { 1415 unsigned long size; 1416 1417 while ((size = atomic64_read(&b->size))) 1418 vmballoon_deflate(b, size, false); 1419 } 1420 1421 /* 1422 * Perform standard reset sequence by popping the balloon (in case it 1423 * is not empty) and then restarting protocol. This operation normally 1424 * happens when host responds with VMW_BALLOON_ERROR_RESET to a command. 1425 */ 1426 static void vmballoon_reset(struct vmballoon *b) 1427 { 1428 int error; 1429 1430 down_write(&b->conf_sem); 1431 1432 vmballoon_vmci_cleanup(b); 1433 1434 /* free all pages, skipping monitor unlock */ 1435 vmballoon_pop(b); 1436 1437 if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES)) 1438 goto unlock; 1439 1440 if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) { 1441 if (vmballoon_init_batching(b)) { 1442 /* 1443 * We failed to initialize batching, inform the monitor 1444 * about it by sending a null capability. 1445 * 1446 * The guest will retry in one second. 1447 */ 1448 vmballoon_send_start(b, 0); 1449 goto unlock; 1450 } 1451 } else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) { 1452 vmballoon_deinit_batching(b); 1453 } 1454 1455 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET); 1456 b->reset_required = false; 1457 1458 error = vmballoon_vmci_init(b); 1459 if (error) 1460 pr_err("failed to initialize vmci doorbell\n"); 1461 1462 if (vmballoon_send_guest_id(b)) 1463 pr_err("failed to send guest ID to the host\n"); 1464 1465 unlock: 1466 up_write(&b->conf_sem); 1467 } 1468 1469 /** 1470 * vmballoon_work - periodic balloon worker for reset, inflation and deflation. 1471 * 1472 * @work: pointer to the &work_struct which is provided by the workqueue. 1473 * 1474 * Resets the protocol if needed, gets the new size and adjusts balloon as 1475 * needed. Repeat in 1 sec. 1476 */ 1477 static void vmballoon_work(struct work_struct *work) 1478 { 1479 struct delayed_work *dwork = to_delayed_work(work); 1480 struct vmballoon *b = container_of(dwork, struct vmballoon, dwork); 1481 int64_t change = 0; 1482 1483 if (b->reset_required) 1484 vmballoon_reset(b); 1485 1486 down_read(&b->conf_sem); 1487 1488 /* 1489 * Update the stats while holding the semaphore to ensure that 1490 * @stats_enabled is consistent with whether the stats are actually 1491 * enabled 1492 */ 1493 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER); 1494 1495 if (!vmballoon_send_get_target(b)) 1496 change = vmballoon_change(b); 1497 1498 if (change != 0) { 1499 pr_debug("%s - size: %llu, target %lu\n", __func__, 1500 atomic64_read(&b->size), READ_ONCE(b->target)); 1501 1502 if (change > 0) 1503 vmballoon_inflate(b); 1504 else /* (change < 0) */ 1505 vmballoon_deflate(b, 0, true); 1506 } 1507 1508 up_read(&b->conf_sem); 1509 1510 /* 1511 * We are using a freezable workqueue so that balloon operations are 1512 * stopped while the system transitions to/from sleep/hibernation. 1513 */ 1514 queue_delayed_work(system_freezable_wq, 1515 dwork, round_jiffies_relative(HZ)); 1516 1517 } 1518 1519 /** 1520 * vmballoon_shrinker_scan() - deflate the balloon due to memory pressure. 1521 * @shrinker: pointer to the balloon shrinker. 1522 * @sc: page reclaim information. 1523 * 1524 * Returns: number of pages that were freed during deflation. 1525 */ 1526 static unsigned long vmballoon_shrinker_scan(struct shrinker *shrinker, 1527 struct shrink_control *sc) 1528 { 1529 struct vmballoon *b = &balloon; 1530 unsigned long deflated_frames; 1531 1532 pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size)); 1533 1534 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK); 1535 1536 /* 1537 * If the lock is also contended for read, we cannot easily reclaim and 1538 * we bail out. 1539 */ 1540 if (!down_read_trylock(&b->conf_sem)) 1541 return 0; 1542 1543 deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true); 1544 1545 vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE, 1546 deflated_frames); 1547 1548 /* 1549 * Delay future inflation for some time to mitigate the situations in 1550 * which balloon continuously grows and shrinks. Use WRITE_ONCE() since 1551 * the access is asynchronous. 1552 */ 1553 WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY); 1554 1555 up_read(&b->conf_sem); 1556 1557 return deflated_frames; 1558 } 1559 1560 /** 1561 * vmballoon_shrinker_count() - return the number of ballooned pages. 1562 * @shrinker: pointer to the balloon shrinker. 1563 * @sc: page reclaim information. 1564 * 1565 * Returns: number of 4k pages that are allocated for the balloon and can 1566 * therefore be reclaimed under pressure. 1567 */ 1568 static unsigned long vmballoon_shrinker_count(struct shrinker *shrinker, 1569 struct shrink_control *sc) 1570 { 1571 struct vmballoon *b = &balloon; 1572 1573 return atomic64_read(&b->size); 1574 } 1575 1576 static void vmballoon_unregister_shrinker(struct vmballoon *b) 1577 { 1578 if (b->shrinker_registered) 1579 unregister_shrinker(&b->shrinker); 1580 b->shrinker_registered = false; 1581 } 1582 1583 static int vmballoon_register_shrinker(struct vmballoon *b) 1584 { 1585 int r; 1586 1587 /* Do nothing if the shrinker is not enabled */ 1588 if (!vmwballoon_shrinker_enable) 1589 return 0; 1590 1591 b->shrinker.scan_objects = vmballoon_shrinker_scan; 1592 b->shrinker.count_objects = vmballoon_shrinker_count; 1593 b->shrinker.seeks = DEFAULT_SEEKS; 1594 1595 r = register_shrinker(&b->shrinker); 1596 1597 if (r == 0) 1598 b->shrinker_registered = true; 1599 1600 return r; 1601 } 1602 1603 /* 1604 * DEBUGFS Interface 1605 */ 1606 #ifdef CONFIG_DEBUG_FS 1607 1608 static const char * const vmballoon_stat_page_names[] = { 1609 [VMW_BALLOON_PAGE_STAT_ALLOC] = "alloc", 1610 [VMW_BALLOON_PAGE_STAT_ALLOC_FAIL] = "allocFail", 1611 [VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC] = "errAlloc", 1612 [VMW_BALLOON_PAGE_STAT_REFUSED_FREE] = "errFree", 1613 [VMW_BALLOON_PAGE_STAT_FREE] = "free" 1614 }; 1615 1616 static const char * const vmballoon_stat_names[] = { 1617 [VMW_BALLOON_STAT_TIMER] = "timer", 1618 [VMW_BALLOON_STAT_DOORBELL] = "doorbell", 1619 [VMW_BALLOON_STAT_RESET] = "reset", 1620 [VMW_BALLOON_STAT_SHRINK] = "shrink", 1621 [VMW_BALLOON_STAT_SHRINK_FREE] = "shrinkFree" 1622 }; 1623 1624 static int vmballoon_enable_stats(struct vmballoon *b) 1625 { 1626 int r = 0; 1627 1628 down_write(&b->conf_sem); 1629 1630 /* did we somehow race with another reader which enabled stats? */ 1631 if (b->stats) 1632 goto out; 1633 1634 b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL); 1635 1636 if (!b->stats) { 1637 /* allocation failed */ 1638 r = -ENOMEM; 1639 goto out; 1640 } 1641 static_key_enable(&balloon_stat_enabled.key); 1642 out: 1643 up_write(&b->conf_sem); 1644 return r; 1645 } 1646 1647 /** 1648 * vmballoon_debug_show - shows statistics of balloon operations. 1649 * @f: pointer to the &struct seq_file. 1650 * @offset: ignored. 1651 * 1652 * Provides the statistics that can be accessed in vmmemctl in the debugfs. 1653 * To avoid the overhead - mainly that of memory - of collecting the statistics, 1654 * we only collect statistics after the first time the counters are read. 1655 * 1656 * Return: zero on success or an error code. 1657 */ 1658 static int vmballoon_debug_show(struct seq_file *f, void *offset) 1659 { 1660 struct vmballoon *b = f->private; 1661 int i, j; 1662 1663 /* enables stats if they are disabled */ 1664 if (!b->stats) { 1665 int r = vmballoon_enable_stats(b); 1666 1667 if (r) 1668 return r; 1669 } 1670 1671 /* format capabilities info */ 1672 seq_printf(f, "%-22s: %#16x\n", "balloon capabilities", 1673 VMW_BALLOON_CAPABILITIES); 1674 seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities); 1675 seq_printf(f, "%-22s: %16s\n", "is resetting", 1676 b->reset_required ? "y" : "n"); 1677 1678 /* format size info */ 1679 seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target)); 1680 seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size)); 1681 1682 for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) { 1683 if (vmballoon_cmd_names[i] == NULL) 1684 continue; 1685 1686 seq_printf(f, "%-22s: %16llu (%llu failed)\n", 1687 vmballoon_cmd_names[i], 1688 atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]), 1689 atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT])); 1690 } 1691 1692 for (i = 0; i < VMW_BALLOON_STAT_NUM; i++) 1693 seq_printf(f, "%-22s: %16llu\n", 1694 vmballoon_stat_names[i], 1695 atomic64_read(&b->stats->general_stat[i])); 1696 1697 for (i = 0; i < VMW_BALLOON_PAGE_STAT_NUM; i++) { 1698 for (j = 0; j < VMW_BALLOON_NUM_PAGE_SIZES; j++) 1699 seq_printf(f, "%-18s(%s): %16llu\n", 1700 vmballoon_stat_page_names[i], 1701 vmballoon_page_size_names[j], 1702 atomic64_read(&b->stats->page_stat[i][j])); 1703 } 1704 1705 return 0; 1706 } 1707 1708 DEFINE_SHOW_ATTRIBUTE(vmballoon_debug); 1709 1710 static void __init vmballoon_debugfs_init(struct vmballoon *b) 1711 { 1712 b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b, 1713 &vmballoon_debug_fops); 1714 } 1715 1716 static void __exit vmballoon_debugfs_exit(struct vmballoon *b) 1717 { 1718 static_key_disable(&balloon_stat_enabled.key); 1719 debugfs_remove(b->dbg_entry); 1720 kfree(b->stats); 1721 b->stats = NULL; 1722 } 1723 1724 #else 1725 1726 static inline void vmballoon_debugfs_init(struct vmballoon *b) 1727 { 1728 } 1729 1730 static inline void vmballoon_debugfs_exit(struct vmballoon *b) 1731 { 1732 } 1733 1734 #endif /* CONFIG_DEBUG_FS */ 1735 1736 1737 #ifdef CONFIG_BALLOON_COMPACTION 1738 1739 static int vmballoon_init_fs_context(struct fs_context *fc) 1740 { 1741 return init_pseudo(fc, BALLOON_VMW_MAGIC) ? 0 : -ENOMEM; 1742 } 1743 1744 static struct file_system_type vmballoon_fs = { 1745 .name = "balloon-vmware", 1746 .init_fs_context = vmballoon_init_fs_context, 1747 .kill_sb = kill_anon_super, 1748 }; 1749 1750 static struct vfsmount *vmballoon_mnt; 1751 1752 /** 1753 * vmballoon_migratepage() - migrates a balloon page. 1754 * @b_dev_info: balloon device information descriptor. 1755 * @newpage: the page to which @page should be migrated. 1756 * @page: a ballooned page that should be migrated. 1757 * @mode: migration mode, ignored. 1758 * 1759 * This function is really open-coded, but that is according to the interface 1760 * that balloon_compaction provides. 1761 * 1762 * Return: zero on success, -EAGAIN when migration cannot be performed 1763 * momentarily, and -EBUSY if migration failed and should be retried 1764 * with that specific page. 1765 */ 1766 static int vmballoon_migratepage(struct balloon_dev_info *b_dev_info, 1767 struct page *newpage, struct page *page, 1768 enum migrate_mode mode) 1769 { 1770 unsigned long status, flags; 1771 struct vmballoon *b; 1772 int ret; 1773 1774 b = container_of(b_dev_info, struct vmballoon, b_dev_info); 1775 1776 /* 1777 * If the semaphore is taken, there is ongoing configuration change 1778 * (i.e., balloon reset), so try again. 1779 */ 1780 if (!down_read_trylock(&b->conf_sem)) 1781 return -EAGAIN; 1782 1783 spin_lock(&b->comm_lock); 1784 /* 1785 * We must start by deflating and not inflating, as otherwise the 1786 * hypervisor may tell us that it has enough memory and the new page is 1787 * not needed. Since the old page is isolated, we cannot use the list 1788 * interface to unlock it, as the LRU field is used for isolation. 1789 * Instead, we use the native interface directly. 1790 */ 1791 vmballoon_add_page(b, 0, page); 1792 status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE, 1793 VMW_BALLOON_DEFLATE); 1794 1795 if (status == VMW_BALLOON_SUCCESS) 1796 status = vmballoon_status_page(b, 0, &page); 1797 1798 /* 1799 * If a failure happened, let the migration mechanism know that it 1800 * should not retry. 1801 */ 1802 if (status != VMW_BALLOON_SUCCESS) { 1803 spin_unlock(&b->comm_lock); 1804 ret = -EBUSY; 1805 goto out_unlock; 1806 } 1807 1808 /* 1809 * The page is isolated, so it is safe to delete it without holding 1810 * @pages_lock . We keep holding @comm_lock since we will need it in a 1811 * second. 1812 */ 1813 balloon_page_delete(page); 1814 1815 put_page(page); 1816 1817 /* Inflate */ 1818 vmballoon_add_page(b, 0, newpage); 1819 status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE, 1820 VMW_BALLOON_INFLATE); 1821 1822 if (status == VMW_BALLOON_SUCCESS) 1823 status = vmballoon_status_page(b, 0, &newpage); 1824 1825 spin_unlock(&b->comm_lock); 1826 1827 if (status != VMW_BALLOON_SUCCESS) { 1828 /* 1829 * A failure happened. While we can deflate the page we just 1830 * inflated, this deflation can also encounter an error. Instead 1831 * we will decrease the size of the balloon to reflect the 1832 * change and report failure. 1833 */ 1834 atomic64_dec(&b->size); 1835 ret = -EBUSY; 1836 } else { 1837 /* 1838 * Success. Take a reference for the page, and we will add it to 1839 * the list after acquiring the lock. 1840 */ 1841 get_page(newpage); 1842 ret = MIGRATEPAGE_SUCCESS; 1843 } 1844 1845 /* Update the balloon list under the @pages_lock */ 1846 spin_lock_irqsave(&b->b_dev_info.pages_lock, flags); 1847 1848 /* 1849 * On inflation success, we already took a reference for the @newpage. 1850 * If we succeed just insert it to the list and update the statistics 1851 * under the lock. 1852 */ 1853 if (ret == MIGRATEPAGE_SUCCESS) { 1854 balloon_page_insert(&b->b_dev_info, newpage); 1855 __count_vm_event(BALLOON_MIGRATE); 1856 } 1857 1858 /* 1859 * We deflated successfully, so regardless to the inflation success, we 1860 * need to reduce the number of isolated_pages. 1861 */ 1862 b->b_dev_info.isolated_pages--; 1863 spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags); 1864 1865 out_unlock: 1866 up_read(&b->conf_sem); 1867 return ret; 1868 } 1869 1870 /** 1871 * vmballoon_compaction_deinit() - removes compaction related data. 1872 * 1873 * @b: pointer to the balloon. 1874 */ 1875 static void vmballoon_compaction_deinit(struct vmballoon *b) 1876 { 1877 if (!IS_ERR(b->b_dev_info.inode)) 1878 iput(b->b_dev_info.inode); 1879 1880 b->b_dev_info.inode = NULL; 1881 kern_unmount(vmballoon_mnt); 1882 vmballoon_mnt = NULL; 1883 } 1884 1885 /** 1886 * vmballoon_compaction_init() - initialized compaction for the balloon. 1887 * 1888 * @b: pointer to the balloon. 1889 * 1890 * If during the initialization a failure occurred, this function does not 1891 * perform cleanup. The caller must call vmballoon_compaction_deinit() in this 1892 * case. 1893 * 1894 * Return: zero on success or error code on failure. 1895 */ 1896 static __init int vmballoon_compaction_init(struct vmballoon *b) 1897 { 1898 vmballoon_mnt = kern_mount(&vmballoon_fs); 1899 if (IS_ERR(vmballoon_mnt)) 1900 return PTR_ERR(vmballoon_mnt); 1901 1902 b->b_dev_info.migratepage = vmballoon_migratepage; 1903 b->b_dev_info.inode = alloc_anon_inode(vmballoon_mnt->mnt_sb); 1904 1905 if (IS_ERR(b->b_dev_info.inode)) 1906 return PTR_ERR(b->b_dev_info.inode); 1907 1908 b->b_dev_info.inode->i_mapping->a_ops = &balloon_aops; 1909 return 0; 1910 } 1911 1912 #else /* CONFIG_BALLOON_COMPACTION */ 1913 1914 static void vmballoon_compaction_deinit(struct vmballoon *b) 1915 { 1916 } 1917 1918 static int vmballoon_compaction_init(struct vmballoon *b) 1919 { 1920 return 0; 1921 } 1922 1923 #endif /* CONFIG_BALLOON_COMPACTION */ 1924 1925 static int __init vmballoon_init(void) 1926 { 1927 int error; 1928 1929 /* 1930 * Check if we are running on VMware's hypervisor and bail out 1931 * if we are not. 1932 */ 1933 if (x86_hyper_type != X86_HYPER_VMWARE) 1934 return -ENODEV; 1935 1936 INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work); 1937 1938 error = vmballoon_register_shrinker(&balloon); 1939 if (error) 1940 goto fail; 1941 1942 /* 1943 * Initialization of compaction must be done after the call to 1944 * balloon_devinfo_init() . 1945 */ 1946 balloon_devinfo_init(&balloon.b_dev_info); 1947 error = vmballoon_compaction_init(&balloon); 1948 if (error) 1949 goto fail; 1950 1951 INIT_LIST_HEAD(&balloon.huge_pages); 1952 spin_lock_init(&balloon.comm_lock); 1953 init_rwsem(&balloon.conf_sem); 1954 balloon.vmci_doorbell = VMCI_INVALID_HANDLE; 1955 balloon.batch_page = NULL; 1956 balloon.page = NULL; 1957 balloon.reset_required = true; 1958 1959 queue_delayed_work(system_freezable_wq, &balloon.dwork, 0); 1960 1961 vmballoon_debugfs_init(&balloon); 1962 1963 return 0; 1964 fail: 1965 vmballoon_unregister_shrinker(&balloon); 1966 vmballoon_compaction_deinit(&balloon); 1967 return error; 1968 } 1969 1970 /* 1971 * Using late_initcall() instead of module_init() allows the balloon to use the 1972 * VMCI doorbell even when the balloon is built into the kernel. Otherwise the 1973 * VMCI is probed only after the balloon is initialized. If the balloon is used 1974 * as a module, late_initcall() is equivalent to module_init(). 1975 */ 1976 late_initcall(vmballoon_init); 1977 1978 static void __exit vmballoon_exit(void) 1979 { 1980 vmballoon_unregister_shrinker(&balloon); 1981 vmballoon_vmci_cleanup(&balloon); 1982 cancel_delayed_work_sync(&balloon.dwork); 1983 1984 vmballoon_debugfs_exit(&balloon); 1985 1986 /* 1987 * Deallocate all reserved memory, and reset connection with monitor. 1988 * Reset connection before deallocating memory to avoid potential for 1989 * additional spurious resets from guest touching deallocated pages. 1990 */ 1991 vmballoon_send_start(&balloon, 0); 1992 vmballoon_pop(&balloon); 1993 1994 /* Only once we popped the balloon, compaction can be deinit */ 1995 vmballoon_compaction_deinit(&balloon); 1996 } 1997 module_exit(vmballoon_exit); 1998