1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* memcontrol.h - Memory Controller 3 * 4 * Copyright IBM Corporation, 2007 5 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 6 * 7 * Copyright 2007 OpenVZ SWsoft Inc 8 * Author: Pavel Emelianov <xemul@openvz.org> 9 */ 10 11 #ifndef _LINUX_MEMCONTROL_H 12 #define _LINUX_MEMCONTROL_H 13 #include <linux/cgroup.h> 14 #include <linux/vm_event_item.h> 15 #include <linux/hardirq.h> 16 #include <linux/jump_label.h> 17 #include <linux/page_counter.h> 18 #include <linux/vmpressure.h> 19 #include <linux/eventfd.h> 20 #include <linux/mm.h> 21 #include <linux/vmstat.h> 22 #include <linux/writeback.h> 23 #include <linux/page-flags.h> 24 25 struct mem_cgroup; 26 struct obj_cgroup; 27 struct page; 28 struct mm_struct; 29 struct kmem_cache; 30 31 /* Cgroup-specific page state, on top of universal node page state */ 32 enum memcg_stat_item { 33 MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS, 34 MEMCG_SOCK, 35 MEMCG_PERCPU_B, 36 MEMCG_VMALLOC, 37 MEMCG_KMEM, 38 MEMCG_ZSWAP_B, 39 MEMCG_ZSWAPPED, 40 MEMCG_NR_STAT, 41 }; 42 43 enum memcg_memory_event { 44 MEMCG_LOW, 45 MEMCG_HIGH, 46 MEMCG_MAX, 47 MEMCG_OOM, 48 MEMCG_OOM_KILL, 49 MEMCG_OOM_GROUP_KILL, 50 MEMCG_SWAP_HIGH, 51 MEMCG_SWAP_MAX, 52 MEMCG_SWAP_FAIL, 53 MEMCG_NR_MEMORY_EVENTS, 54 }; 55 56 struct mem_cgroup_reclaim_cookie { 57 pg_data_t *pgdat; 58 unsigned int generation; 59 }; 60 61 #ifdef CONFIG_MEMCG 62 63 #define MEM_CGROUP_ID_SHIFT 16 64 #define MEM_CGROUP_ID_MAX USHRT_MAX 65 66 struct mem_cgroup_id { 67 int id; 68 refcount_t ref; 69 }; 70 71 /* 72 * Per memcg event counter is incremented at every pagein/pageout. With THP, 73 * it will be incremented by the number of pages. This counter is used 74 * to trigger some periodic events. This is straightforward and better 75 * than using jiffies etc. to handle periodic memcg event. 76 */ 77 enum mem_cgroup_events_target { 78 MEM_CGROUP_TARGET_THRESH, 79 MEM_CGROUP_TARGET_SOFTLIMIT, 80 MEM_CGROUP_NTARGETS, 81 }; 82 83 struct memcg_vmstats_percpu; 84 struct memcg_vmstats; 85 86 struct mem_cgroup_reclaim_iter { 87 struct mem_cgroup *position; 88 /* scan generation, increased every round-trip */ 89 unsigned int generation; 90 }; 91 92 /* 93 * Bitmap and deferred work of shrinker::id corresponding to memcg-aware 94 * shrinkers, which have elements charged to this memcg. 95 */ 96 struct shrinker_info { 97 struct rcu_head rcu; 98 atomic_long_t *nr_deferred; 99 unsigned long *map; 100 }; 101 102 struct lruvec_stats_percpu { 103 /* Local (CPU and cgroup) state */ 104 long state[NR_VM_NODE_STAT_ITEMS]; 105 106 /* Delta calculation for lockless upward propagation */ 107 long state_prev[NR_VM_NODE_STAT_ITEMS]; 108 }; 109 110 struct lruvec_stats { 111 /* Aggregated (CPU and subtree) state */ 112 long state[NR_VM_NODE_STAT_ITEMS]; 113 114 /* Pending child counts during tree propagation */ 115 long state_pending[NR_VM_NODE_STAT_ITEMS]; 116 }; 117 118 /* 119 * per-node information in memory controller. 120 */ 121 struct mem_cgroup_per_node { 122 struct lruvec lruvec; 123 124 struct lruvec_stats_percpu __percpu *lruvec_stats_percpu; 125 struct lruvec_stats lruvec_stats; 126 127 unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; 128 129 struct mem_cgroup_reclaim_iter iter; 130 131 struct shrinker_info __rcu *shrinker_info; 132 133 struct rb_node tree_node; /* RB tree node */ 134 unsigned long usage_in_excess;/* Set to the value by which */ 135 /* the soft limit is exceeded*/ 136 bool on_tree; 137 struct mem_cgroup *memcg; /* Back pointer, we cannot */ 138 /* use container_of */ 139 }; 140 141 struct mem_cgroup_threshold { 142 struct eventfd_ctx *eventfd; 143 unsigned long threshold; 144 }; 145 146 /* For threshold */ 147 struct mem_cgroup_threshold_ary { 148 /* An array index points to threshold just below or equal to usage. */ 149 int current_threshold; 150 /* Size of entries[] */ 151 unsigned int size; 152 /* Array of thresholds */ 153 struct mem_cgroup_threshold entries[]; 154 }; 155 156 struct mem_cgroup_thresholds { 157 /* Primary thresholds array */ 158 struct mem_cgroup_threshold_ary *primary; 159 /* 160 * Spare threshold array. 161 * This is needed to make mem_cgroup_unregister_event() "never fail". 162 * It must be able to store at least primary->size - 1 entries. 163 */ 164 struct mem_cgroup_threshold_ary *spare; 165 }; 166 167 /* 168 * Remember four most recent foreign writebacks with dirty pages in this 169 * cgroup. Inode sharing is expected to be uncommon and, even if we miss 170 * one in a given round, we're likely to catch it later if it keeps 171 * foreign-dirtying, so a fairly low count should be enough. 172 * 173 * See mem_cgroup_track_foreign_dirty_slowpath() for details. 174 */ 175 #define MEMCG_CGWB_FRN_CNT 4 176 177 struct memcg_cgwb_frn { 178 u64 bdi_id; /* bdi->id of the foreign inode */ 179 int memcg_id; /* memcg->css.id of foreign inode */ 180 u64 at; /* jiffies_64 at the time of dirtying */ 181 struct wb_completion done; /* tracks in-flight foreign writebacks */ 182 }; 183 184 /* 185 * Bucket for arbitrarily byte-sized objects charged to a memory 186 * cgroup. The bucket can be reparented in one piece when the cgroup 187 * is destroyed, without having to round up the individual references 188 * of all live memory objects in the wild. 189 */ 190 struct obj_cgroup { 191 struct percpu_ref refcnt; 192 struct mem_cgroup *memcg; 193 atomic_t nr_charged_bytes; 194 union { 195 struct list_head list; /* protected by objcg_lock */ 196 struct rcu_head rcu; 197 }; 198 }; 199 200 /* 201 * The memory controller data structure. The memory controller controls both 202 * page cache and RSS per cgroup. We would eventually like to provide 203 * statistics based on the statistics developed by Rik Van Riel for clock-pro, 204 * to help the administrator determine what knobs to tune. 205 */ 206 struct mem_cgroup { 207 struct cgroup_subsys_state css; 208 209 /* Private memcg ID. Used to ID objects that outlive the cgroup */ 210 struct mem_cgroup_id id; 211 212 /* Accounted resources */ 213 struct page_counter memory; /* Both v1 & v2 */ 214 215 union { 216 struct page_counter swap; /* v2 only */ 217 struct page_counter memsw; /* v1 only */ 218 }; 219 220 /* Legacy consumer-oriented counters */ 221 struct page_counter kmem; /* v1 only */ 222 struct page_counter tcpmem; /* v1 only */ 223 224 /* Range enforcement for interrupt charges */ 225 struct work_struct high_work; 226 227 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 228 unsigned long zswap_max; 229 #endif 230 231 unsigned long soft_limit; 232 233 /* vmpressure notifications */ 234 struct vmpressure vmpressure; 235 236 /* 237 * Should the OOM killer kill all belonging tasks, had it kill one? 238 */ 239 bool oom_group; 240 241 /* protected by memcg_oom_lock */ 242 bool oom_lock; 243 int under_oom; 244 245 int swappiness; 246 /* OOM-Killer disable */ 247 int oom_kill_disable; 248 249 /* memory.events and memory.events.local */ 250 struct cgroup_file events_file; 251 struct cgroup_file events_local_file; 252 253 /* handle for "memory.swap.events" */ 254 struct cgroup_file swap_events_file; 255 256 /* protect arrays of thresholds */ 257 struct mutex thresholds_lock; 258 259 /* thresholds for memory usage. RCU-protected */ 260 struct mem_cgroup_thresholds thresholds; 261 262 /* thresholds for mem+swap usage. RCU-protected */ 263 struct mem_cgroup_thresholds memsw_thresholds; 264 265 /* For oom notifier event fd */ 266 struct list_head oom_notify; 267 268 /* 269 * Should we move charges of a task when a task is moved into this 270 * mem_cgroup ? And what type of charges should we move ? 271 */ 272 unsigned long move_charge_at_immigrate; 273 /* taken only while moving_account > 0 */ 274 spinlock_t move_lock; 275 unsigned long move_lock_flags; 276 277 CACHELINE_PADDING(_pad1_); 278 279 /* memory.stat */ 280 struct memcg_vmstats *vmstats; 281 282 /* memory.events */ 283 atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; 284 atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS]; 285 286 unsigned long socket_pressure; 287 288 /* Legacy tcp memory accounting */ 289 bool tcpmem_active; 290 int tcpmem_pressure; 291 292 #ifdef CONFIG_MEMCG_KMEM 293 int kmemcg_id; 294 struct obj_cgroup __rcu *objcg; 295 /* list of inherited objcgs, protected by objcg_lock */ 296 struct list_head objcg_list; 297 #endif 298 299 CACHELINE_PADDING(_pad2_); 300 301 /* 302 * set > 0 if pages under this cgroup are moving to other cgroup. 303 */ 304 atomic_t moving_account; 305 struct task_struct *move_lock_task; 306 307 struct memcg_vmstats_percpu __percpu *vmstats_percpu; 308 309 #ifdef CONFIG_CGROUP_WRITEBACK 310 struct list_head cgwb_list; 311 struct wb_domain cgwb_domain; 312 struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; 313 #endif 314 315 /* List of events which userspace want to receive */ 316 struct list_head event_list; 317 spinlock_t event_list_lock; 318 319 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 320 struct deferred_split deferred_split_queue; 321 #endif 322 323 #ifdef CONFIG_LRU_GEN 324 /* per-memcg mm_struct list */ 325 struct lru_gen_mm_list mm_list; 326 #endif 327 328 struct mem_cgroup_per_node *nodeinfo[]; 329 }; 330 331 /* 332 * size of first charge trial. 333 * TODO: maybe necessary to use big numbers in big irons or dynamic based of the 334 * workload. 335 */ 336 #define MEMCG_CHARGE_BATCH 64U 337 338 extern struct mem_cgroup *root_mem_cgroup; 339 340 enum page_memcg_data_flags { 341 /* page->memcg_data is a pointer to an objcgs vector */ 342 MEMCG_DATA_OBJCGS = (1UL << 0), 343 /* page has been accounted as a non-slab kernel page */ 344 MEMCG_DATA_KMEM = (1UL << 1), 345 /* the next bit after the last actual flag */ 346 __NR_MEMCG_DATA_FLAGS = (1UL << 2), 347 }; 348 349 #define MEMCG_DATA_FLAGS_MASK (__NR_MEMCG_DATA_FLAGS - 1) 350 351 static inline bool folio_memcg_kmem(struct folio *folio); 352 353 /* 354 * After the initialization objcg->memcg is always pointing at 355 * a valid memcg, but can be atomically swapped to the parent memcg. 356 * 357 * The caller must ensure that the returned memcg won't be released: 358 * e.g. acquire the rcu_read_lock or css_set_lock. 359 */ 360 static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) 361 { 362 return READ_ONCE(objcg->memcg); 363 } 364 365 /* 366 * __folio_memcg - Get the memory cgroup associated with a non-kmem folio 367 * @folio: Pointer to the folio. 368 * 369 * Returns a pointer to the memory cgroup associated with the folio, 370 * or NULL. This function assumes that the folio is known to have a 371 * proper memory cgroup pointer. It's not safe to call this function 372 * against some type of folios, e.g. slab folios or ex-slab folios or 373 * kmem folios. 374 */ 375 static inline struct mem_cgroup *__folio_memcg(struct folio *folio) 376 { 377 unsigned long memcg_data = folio->memcg_data; 378 379 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); 380 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio); 381 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio); 382 383 return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); 384 } 385 386 /* 387 * __folio_objcg - get the object cgroup associated with a kmem folio. 388 * @folio: Pointer to the folio. 389 * 390 * Returns a pointer to the object cgroup associated with the folio, 391 * or NULL. This function assumes that the folio is known to have a 392 * proper object cgroup pointer. It's not safe to call this function 393 * against some type of folios, e.g. slab folios or ex-slab folios or 394 * LRU folios. 395 */ 396 static inline struct obj_cgroup *__folio_objcg(struct folio *folio) 397 { 398 unsigned long memcg_data = folio->memcg_data; 399 400 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); 401 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio); 402 VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio); 403 404 return (struct obj_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); 405 } 406 407 /* 408 * folio_memcg - Get the memory cgroup associated with a folio. 409 * @folio: Pointer to the folio. 410 * 411 * Returns a pointer to the memory cgroup associated with the folio, 412 * or NULL. This function assumes that the folio is known to have a 413 * proper memory cgroup pointer. It's not safe to call this function 414 * against some type of folios, e.g. slab folios or ex-slab folios. 415 * 416 * For a non-kmem folio any of the following ensures folio and memcg binding 417 * stability: 418 * 419 * - the folio lock 420 * - LRU isolation 421 * - lock_page_memcg() 422 * - exclusive reference 423 * - mem_cgroup_trylock_pages() 424 * 425 * For a kmem folio a caller should hold an rcu read lock to protect memcg 426 * associated with a kmem folio from being released. 427 */ 428 static inline struct mem_cgroup *folio_memcg(struct folio *folio) 429 { 430 if (folio_memcg_kmem(folio)) 431 return obj_cgroup_memcg(__folio_objcg(folio)); 432 return __folio_memcg(folio); 433 } 434 435 static inline struct mem_cgroup *page_memcg(struct page *page) 436 { 437 return folio_memcg(page_folio(page)); 438 } 439 440 /** 441 * folio_memcg_rcu - Locklessly get the memory cgroup associated with a folio. 442 * @folio: Pointer to the folio. 443 * 444 * This function assumes that the folio is known to have a 445 * proper memory cgroup pointer. It's not safe to call this function 446 * against some type of folios, e.g. slab folios or ex-slab folios. 447 * 448 * Return: A pointer to the memory cgroup associated with the folio, 449 * or NULL. 450 */ 451 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) 452 { 453 unsigned long memcg_data = READ_ONCE(folio->memcg_data); 454 455 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); 456 WARN_ON_ONCE(!rcu_read_lock_held()); 457 458 if (memcg_data & MEMCG_DATA_KMEM) { 459 struct obj_cgroup *objcg; 460 461 objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); 462 return obj_cgroup_memcg(objcg); 463 } 464 465 return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); 466 } 467 468 /* 469 * page_memcg_check - get the memory cgroup associated with a page 470 * @page: a pointer to the page struct 471 * 472 * Returns a pointer to the memory cgroup associated with the page, 473 * or NULL. This function unlike page_memcg() can take any page 474 * as an argument. It has to be used in cases when it's not known if a page 475 * has an associated memory cgroup pointer or an object cgroups vector or 476 * an object cgroup. 477 * 478 * For a non-kmem page any of the following ensures page and memcg binding 479 * stability: 480 * 481 * - the page lock 482 * - LRU isolation 483 * - lock_page_memcg() 484 * - exclusive reference 485 * - mem_cgroup_trylock_pages() 486 * 487 * For a kmem page a caller should hold an rcu read lock to protect memcg 488 * associated with a kmem page from being released. 489 */ 490 static inline struct mem_cgroup *page_memcg_check(struct page *page) 491 { 492 /* 493 * Because page->memcg_data might be changed asynchronously 494 * for slab pages, READ_ONCE() should be used here. 495 */ 496 unsigned long memcg_data = READ_ONCE(page->memcg_data); 497 498 if (memcg_data & MEMCG_DATA_OBJCGS) 499 return NULL; 500 501 if (memcg_data & MEMCG_DATA_KMEM) { 502 struct obj_cgroup *objcg; 503 504 objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); 505 return obj_cgroup_memcg(objcg); 506 } 507 508 return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); 509 } 510 511 static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) 512 { 513 struct mem_cgroup *memcg; 514 515 rcu_read_lock(); 516 retry: 517 memcg = obj_cgroup_memcg(objcg); 518 if (unlikely(!css_tryget(&memcg->css))) 519 goto retry; 520 rcu_read_unlock(); 521 522 return memcg; 523 } 524 525 #ifdef CONFIG_MEMCG_KMEM 526 /* 527 * folio_memcg_kmem - Check if the folio has the memcg_kmem flag set. 528 * @folio: Pointer to the folio. 529 * 530 * Checks if the folio has MemcgKmem flag set. The caller must ensure 531 * that the folio has an associated memory cgroup. It's not safe to call 532 * this function against some types of folios, e.g. slab folios. 533 */ 534 static inline bool folio_memcg_kmem(struct folio *folio) 535 { 536 VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page); 537 VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJCGS, folio); 538 return folio->memcg_data & MEMCG_DATA_KMEM; 539 } 540 541 542 #else 543 static inline bool folio_memcg_kmem(struct folio *folio) 544 { 545 return false; 546 } 547 548 #endif 549 550 static inline bool PageMemcgKmem(struct page *page) 551 { 552 return folio_memcg_kmem(page_folio(page)); 553 } 554 555 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) 556 { 557 return (memcg == root_mem_cgroup); 558 } 559 560 static inline bool mem_cgroup_disabled(void) 561 { 562 return !cgroup_subsys_enabled(memory_cgrp_subsys); 563 } 564 565 static inline void mem_cgroup_protection(struct mem_cgroup *root, 566 struct mem_cgroup *memcg, 567 unsigned long *min, 568 unsigned long *low) 569 { 570 *min = *low = 0; 571 572 if (mem_cgroup_disabled()) 573 return; 574 575 /* 576 * There is no reclaim protection applied to a targeted reclaim. 577 * We are special casing this specific case here because 578 * mem_cgroup_protected calculation is not robust enough to keep 579 * the protection invariant for calculated effective values for 580 * parallel reclaimers with different reclaim target. This is 581 * especially a problem for tail memcgs (as they have pages on LRU) 582 * which would want to have effective values 0 for targeted reclaim 583 * but a different value for external reclaim. 584 * 585 * Example 586 * Let's have global and A's reclaim in parallel: 587 * | 588 * A (low=2G, usage = 3G, max = 3G, children_low_usage = 1.5G) 589 * |\ 590 * | C (low = 1G, usage = 2.5G) 591 * B (low = 1G, usage = 0.5G) 592 * 593 * For the global reclaim 594 * A.elow = A.low 595 * B.elow = min(B.usage, B.low) because children_low_usage <= A.elow 596 * C.elow = min(C.usage, C.low) 597 * 598 * With the effective values resetting we have A reclaim 599 * A.elow = 0 600 * B.elow = B.low 601 * C.elow = C.low 602 * 603 * If the global reclaim races with A's reclaim then 604 * B.elow = C.elow = 0 because children_low_usage > A.elow) 605 * is possible and reclaiming B would be violating the protection. 606 * 607 */ 608 if (root == memcg) 609 return; 610 611 *min = READ_ONCE(memcg->memory.emin); 612 *low = READ_ONCE(memcg->memory.elow); 613 } 614 615 void mem_cgroup_calculate_protection(struct mem_cgroup *root, 616 struct mem_cgroup *memcg); 617 618 static inline bool mem_cgroup_supports_protection(struct mem_cgroup *memcg) 619 { 620 /* 621 * The root memcg doesn't account charges, and doesn't support 622 * protection. 623 */ 624 return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg); 625 626 } 627 628 static inline bool mem_cgroup_below_low(struct mem_cgroup *memcg) 629 { 630 if (!mem_cgroup_supports_protection(memcg)) 631 return false; 632 633 return READ_ONCE(memcg->memory.elow) >= 634 page_counter_read(&memcg->memory); 635 } 636 637 static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg) 638 { 639 if (!mem_cgroup_supports_protection(memcg)) 640 return false; 641 642 return READ_ONCE(memcg->memory.emin) >= 643 page_counter_read(&memcg->memory); 644 } 645 646 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp); 647 648 /** 649 * mem_cgroup_charge - Charge a newly allocated folio to a cgroup. 650 * @folio: Folio to charge. 651 * @mm: mm context of the allocating task. 652 * @gfp: Reclaim mode. 653 * 654 * Try to charge @folio to the memcg that @mm belongs to, reclaiming 655 * pages according to @gfp if necessary. If @mm is NULL, try to 656 * charge to the active memcg. 657 * 658 * Do not use this for folios allocated for swapin. 659 * 660 * Return: 0 on success. Otherwise, an error code is returned. 661 */ 662 static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, 663 gfp_t gfp) 664 { 665 if (mem_cgroup_disabled()) 666 return 0; 667 return __mem_cgroup_charge(folio, mm, gfp); 668 } 669 670 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm, 671 gfp_t gfp, swp_entry_t entry); 672 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry); 673 674 void __mem_cgroup_uncharge(struct folio *folio); 675 676 /** 677 * mem_cgroup_uncharge - Uncharge a folio. 678 * @folio: Folio to uncharge. 679 * 680 * Uncharge a folio previously charged with mem_cgroup_charge(). 681 */ 682 static inline void mem_cgroup_uncharge(struct folio *folio) 683 { 684 if (mem_cgroup_disabled()) 685 return; 686 __mem_cgroup_uncharge(folio); 687 } 688 689 void __mem_cgroup_uncharge_list(struct list_head *page_list); 690 static inline void mem_cgroup_uncharge_list(struct list_head *page_list) 691 { 692 if (mem_cgroup_disabled()) 693 return; 694 __mem_cgroup_uncharge_list(page_list); 695 } 696 697 void mem_cgroup_migrate(struct folio *old, struct folio *new); 698 699 /** 700 * mem_cgroup_lruvec - get the lru list vector for a memcg & node 701 * @memcg: memcg of the wanted lruvec 702 * @pgdat: pglist_data 703 * 704 * Returns the lru list vector holding pages for a given @memcg & 705 * @pgdat combination. This can be the node lruvec, if the memory 706 * controller is disabled. 707 */ 708 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, 709 struct pglist_data *pgdat) 710 { 711 struct mem_cgroup_per_node *mz; 712 struct lruvec *lruvec; 713 714 if (mem_cgroup_disabled()) { 715 lruvec = &pgdat->__lruvec; 716 goto out; 717 } 718 719 if (!memcg) 720 memcg = root_mem_cgroup; 721 722 mz = memcg->nodeinfo[pgdat->node_id]; 723 lruvec = &mz->lruvec; 724 out: 725 /* 726 * Since a node can be onlined after the mem_cgroup was created, 727 * we have to be prepared to initialize lruvec->pgdat here; 728 * and if offlined then reonlined, we need to reinitialize it. 729 */ 730 if (unlikely(lruvec->pgdat != pgdat)) 731 lruvec->pgdat = pgdat; 732 return lruvec; 733 } 734 735 /** 736 * folio_lruvec - return lruvec for isolating/putting an LRU folio 737 * @folio: Pointer to the folio. 738 * 739 * This function relies on folio->mem_cgroup being stable. 740 */ 741 static inline struct lruvec *folio_lruvec(struct folio *folio) 742 { 743 struct mem_cgroup *memcg = folio_memcg(folio); 744 745 VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio); 746 return mem_cgroup_lruvec(memcg, folio_pgdat(folio)); 747 } 748 749 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p); 750 751 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm); 752 753 struct lruvec *folio_lruvec_lock(struct folio *folio); 754 struct lruvec *folio_lruvec_lock_irq(struct folio *folio); 755 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, 756 unsigned long *flags); 757 758 #ifdef CONFIG_DEBUG_VM 759 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio); 760 #else 761 static inline 762 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) 763 { 764 } 765 #endif 766 767 static inline 768 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ 769 return css ? container_of(css, struct mem_cgroup, css) : NULL; 770 } 771 772 static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg) 773 { 774 return percpu_ref_tryget(&objcg->refcnt); 775 } 776 777 static inline void obj_cgroup_get(struct obj_cgroup *objcg) 778 { 779 percpu_ref_get(&objcg->refcnt); 780 } 781 782 static inline void obj_cgroup_get_many(struct obj_cgroup *objcg, 783 unsigned long nr) 784 { 785 percpu_ref_get_many(&objcg->refcnt, nr); 786 } 787 788 static inline void obj_cgroup_put(struct obj_cgroup *objcg) 789 { 790 percpu_ref_put(&objcg->refcnt); 791 } 792 793 static inline void mem_cgroup_put(struct mem_cgroup *memcg) 794 { 795 if (memcg) 796 css_put(&memcg->css); 797 } 798 799 #define mem_cgroup_from_counter(counter, member) \ 800 container_of(counter, struct mem_cgroup, member) 801 802 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *, 803 struct mem_cgroup *, 804 struct mem_cgroup_reclaim_cookie *); 805 void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); 806 int mem_cgroup_scan_tasks(struct mem_cgroup *, 807 int (*)(struct task_struct *, void *), void *); 808 809 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg) 810 { 811 if (mem_cgroup_disabled()) 812 return 0; 813 814 return memcg->id.id; 815 } 816 struct mem_cgroup *mem_cgroup_from_id(unsigned short id); 817 818 #ifdef CONFIG_SHRINKER_DEBUG 819 static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg) 820 { 821 return memcg ? cgroup_ino(memcg->css.cgroup) : 0; 822 } 823 824 struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino); 825 #endif 826 827 static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) 828 { 829 return mem_cgroup_from_css(seq_css(m)); 830 } 831 832 static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) 833 { 834 struct mem_cgroup_per_node *mz; 835 836 if (mem_cgroup_disabled()) 837 return NULL; 838 839 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 840 return mz->memcg; 841 } 842 843 /** 844 * parent_mem_cgroup - find the accounting parent of a memcg 845 * @memcg: memcg whose parent to find 846 * 847 * Returns the parent memcg, or NULL if this is the root or the memory 848 * controller is in legacy no-hierarchy mode. 849 */ 850 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) 851 { 852 return mem_cgroup_from_css(memcg->css.parent); 853 } 854 855 static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, 856 struct mem_cgroup *root) 857 { 858 if (root == memcg) 859 return true; 860 return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup); 861 } 862 863 static inline bool mm_match_cgroup(struct mm_struct *mm, 864 struct mem_cgroup *memcg) 865 { 866 struct mem_cgroup *task_memcg; 867 bool match = false; 868 869 rcu_read_lock(); 870 task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 871 if (task_memcg) 872 match = mem_cgroup_is_descendant(task_memcg, memcg); 873 rcu_read_unlock(); 874 return match; 875 } 876 877 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page); 878 ino_t page_cgroup_ino(struct page *page); 879 880 static inline bool mem_cgroup_online(struct mem_cgroup *memcg) 881 { 882 if (mem_cgroup_disabled()) 883 return true; 884 return !!(memcg->css.flags & CSS_ONLINE); 885 } 886 887 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 888 int zid, int nr_pages); 889 890 static inline 891 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, 892 enum lru_list lru, int zone_idx) 893 { 894 struct mem_cgroup_per_node *mz; 895 896 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 897 return READ_ONCE(mz->lru_zone_size[zone_idx][lru]); 898 } 899 900 void mem_cgroup_handle_over_high(void); 901 902 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg); 903 904 unsigned long mem_cgroup_size(struct mem_cgroup *memcg); 905 906 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, 907 struct task_struct *p); 908 909 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg); 910 911 static inline void mem_cgroup_enter_user_fault(void) 912 { 913 WARN_ON(current->in_user_fault); 914 current->in_user_fault = 1; 915 } 916 917 static inline void mem_cgroup_exit_user_fault(void) 918 { 919 WARN_ON(!current->in_user_fault); 920 current->in_user_fault = 0; 921 } 922 923 static inline bool task_in_memcg_oom(struct task_struct *p) 924 { 925 return p->memcg_in_oom; 926 } 927 928 bool mem_cgroup_oom_synchronize(bool wait); 929 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim, 930 struct mem_cgroup *oom_domain); 931 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg); 932 933 void folio_memcg_lock(struct folio *folio); 934 void folio_memcg_unlock(struct folio *folio); 935 void lock_page_memcg(struct page *page); 936 void unlock_page_memcg(struct page *page); 937 938 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); 939 940 /* try to stablize folio_memcg() for all the pages in a memcg */ 941 static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) 942 { 943 rcu_read_lock(); 944 945 if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account)) 946 return true; 947 948 rcu_read_unlock(); 949 return false; 950 } 951 952 static inline void mem_cgroup_unlock_pages(void) 953 { 954 rcu_read_unlock(); 955 } 956 957 /* idx can be of type enum memcg_stat_item or node_stat_item */ 958 static inline void mod_memcg_state(struct mem_cgroup *memcg, 959 int idx, int val) 960 { 961 unsigned long flags; 962 963 local_irq_save(flags); 964 __mod_memcg_state(memcg, idx, val); 965 local_irq_restore(flags); 966 } 967 968 static inline void mod_memcg_page_state(struct page *page, 969 int idx, int val) 970 { 971 struct mem_cgroup *memcg; 972 973 if (mem_cgroup_disabled()) 974 return; 975 976 rcu_read_lock(); 977 memcg = page_memcg(page); 978 if (memcg) 979 mod_memcg_state(memcg, idx, val); 980 rcu_read_unlock(); 981 } 982 983 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx); 984 985 static inline unsigned long lruvec_page_state(struct lruvec *lruvec, 986 enum node_stat_item idx) 987 { 988 struct mem_cgroup_per_node *pn; 989 long x; 990 991 if (mem_cgroup_disabled()) 992 return node_page_state(lruvec_pgdat(lruvec), idx); 993 994 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 995 x = READ_ONCE(pn->lruvec_stats.state[idx]); 996 #ifdef CONFIG_SMP 997 if (x < 0) 998 x = 0; 999 #endif 1000 return x; 1001 } 1002 1003 static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, 1004 enum node_stat_item idx) 1005 { 1006 struct mem_cgroup_per_node *pn; 1007 long x = 0; 1008 int cpu; 1009 1010 if (mem_cgroup_disabled()) 1011 return node_page_state(lruvec_pgdat(lruvec), idx); 1012 1013 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 1014 for_each_possible_cpu(cpu) 1015 x += per_cpu(pn->lruvec_stats_percpu->state[idx], cpu); 1016 #ifdef CONFIG_SMP 1017 if (x < 0) 1018 x = 0; 1019 #endif 1020 return x; 1021 } 1022 1023 void mem_cgroup_flush_stats(void); 1024 void mem_cgroup_flush_stats_delayed(void); 1025 1026 void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 1027 int val); 1028 void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val); 1029 1030 static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, 1031 int val) 1032 { 1033 unsigned long flags; 1034 1035 local_irq_save(flags); 1036 __mod_lruvec_kmem_state(p, idx, val); 1037 local_irq_restore(flags); 1038 } 1039 1040 static inline void mod_memcg_lruvec_state(struct lruvec *lruvec, 1041 enum node_stat_item idx, int val) 1042 { 1043 unsigned long flags; 1044 1045 local_irq_save(flags); 1046 __mod_memcg_lruvec_state(lruvec, idx, val); 1047 local_irq_restore(flags); 1048 } 1049 1050 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, 1051 unsigned long count); 1052 1053 static inline void count_memcg_events(struct mem_cgroup *memcg, 1054 enum vm_event_item idx, 1055 unsigned long count) 1056 { 1057 unsigned long flags; 1058 1059 local_irq_save(flags); 1060 __count_memcg_events(memcg, idx, count); 1061 local_irq_restore(flags); 1062 } 1063 1064 static inline void count_memcg_page_event(struct page *page, 1065 enum vm_event_item idx) 1066 { 1067 struct mem_cgroup *memcg = page_memcg(page); 1068 1069 if (memcg) 1070 count_memcg_events(memcg, idx, 1); 1071 } 1072 1073 static inline void count_memcg_folio_events(struct folio *folio, 1074 enum vm_event_item idx, unsigned long nr) 1075 { 1076 struct mem_cgroup *memcg = folio_memcg(folio); 1077 1078 if (memcg) 1079 count_memcg_events(memcg, idx, nr); 1080 } 1081 1082 static inline void count_memcg_event_mm(struct mm_struct *mm, 1083 enum vm_event_item idx) 1084 { 1085 struct mem_cgroup *memcg; 1086 1087 if (mem_cgroup_disabled()) 1088 return; 1089 1090 rcu_read_lock(); 1091 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 1092 if (likely(memcg)) 1093 count_memcg_events(memcg, idx, 1); 1094 rcu_read_unlock(); 1095 } 1096 1097 static inline void memcg_memory_event(struct mem_cgroup *memcg, 1098 enum memcg_memory_event event) 1099 { 1100 bool swap_event = event == MEMCG_SWAP_HIGH || event == MEMCG_SWAP_MAX || 1101 event == MEMCG_SWAP_FAIL; 1102 1103 atomic_long_inc(&memcg->memory_events_local[event]); 1104 if (!swap_event) 1105 cgroup_file_notify(&memcg->events_local_file); 1106 1107 do { 1108 atomic_long_inc(&memcg->memory_events[event]); 1109 if (swap_event) 1110 cgroup_file_notify(&memcg->swap_events_file); 1111 else 1112 cgroup_file_notify(&memcg->events_file); 1113 1114 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 1115 break; 1116 if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS) 1117 break; 1118 } while ((memcg = parent_mem_cgroup(memcg)) && 1119 !mem_cgroup_is_root(memcg)); 1120 } 1121 1122 static inline void memcg_memory_event_mm(struct mm_struct *mm, 1123 enum memcg_memory_event event) 1124 { 1125 struct mem_cgroup *memcg; 1126 1127 if (mem_cgroup_disabled()) 1128 return; 1129 1130 rcu_read_lock(); 1131 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 1132 if (likely(memcg)) 1133 memcg_memory_event(memcg, event); 1134 rcu_read_unlock(); 1135 } 1136 1137 void split_page_memcg(struct page *head, unsigned int nr); 1138 1139 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 1140 gfp_t gfp_mask, 1141 unsigned long *total_scanned); 1142 1143 #else /* CONFIG_MEMCG */ 1144 1145 #define MEM_CGROUP_ID_SHIFT 0 1146 #define MEM_CGROUP_ID_MAX 0 1147 1148 static inline struct mem_cgroup *folio_memcg(struct folio *folio) 1149 { 1150 return NULL; 1151 } 1152 1153 static inline struct mem_cgroup *page_memcg(struct page *page) 1154 { 1155 return NULL; 1156 } 1157 1158 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) 1159 { 1160 WARN_ON_ONCE(!rcu_read_lock_held()); 1161 return NULL; 1162 } 1163 1164 static inline struct mem_cgroup *page_memcg_check(struct page *page) 1165 { 1166 return NULL; 1167 } 1168 1169 static inline bool folio_memcg_kmem(struct folio *folio) 1170 { 1171 return false; 1172 } 1173 1174 static inline bool PageMemcgKmem(struct page *page) 1175 { 1176 return false; 1177 } 1178 1179 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) 1180 { 1181 return true; 1182 } 1183 1184 static inline bool mem_cgroup_disabled(void) 1185 { 1186 return true; 1187 } 1188 1189 static inline void memcg_memory_event(struct mem_cgroup *memcg, 1190 enum memcg_memory_event event) 1191 { 1192 } 1193 1194 static inline void memcg_memory_event_mm(struct mm_struct *mm, 1195 enum memcg_memory_event event) 1196 { 1197 } 1198 1199 static inline void mem_cgroup_protection(struct mem_cgroup *root, 1200 struct mem_cgroup *memcg, 1201 unsigned long *min, 1202 unsigned long *low) 1203 { 1204 *min = *low = 0; 1205 } 1206 1207 static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root, 1208 struct mem_cgroup *memcg) 1209 { 1210 } 1211 1212 static inline bool mem_cgroup_below_low(struct mem_cgroup *memcg) 1213 { 1214 return false; 1215 } 1216 1217 static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg) 1218 { 1219 return false; 1220 } 1221 1222 static inline int mem_cgroup_charge(struct folio *folio, 1223 struct mm_struct *mm, gfp_t gfp) 1224 { 1225 return 0; 1226 } 1227 1228 static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, 1229 struct mm_struct *mm, gfp_t gfp, swp_entry_t entry) 1230 { 1231 return 0; 1232 } 1233 1234 static inline void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry) 1235 { 1236 } 1237 1238 static inline void mem_cgroup_uncharge(struct folio *folio) 1239 { 1240 } 1241 1242 static inline void mem_cgroup_uncharge_list(struct list_head *page_list) 1243 { 1244 } 1245 1246 static inline void mem_cgroup_migrate(struct folio *old, struct folio *new) 1247 { 1248 } 1249 1250 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, 1251 struct pglist_data *pgdat) 1252 { 1253 return &pgdat->__lruvec; 1254 } 1255 1256 static inline struct lruvec *folio_lruvec(struct folio *folio) 1257 { 1258 struct pglist_data *pgdat = folio_pgdat(folio); 1259 return &pgdat->__lruvec; 1260 } 1261 1262 static inline 1263 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) 1264 { 1265 } 1266 1267 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) 1268 { 1269 return NULL; 1270 } 1271 1272 static inline bool mm_match_cgroup(struct mm_struct *mm, 1273 struct mem_cgroup *memcg) 1274 { 1275 return true; 1276 } 1277 1278 static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) 1279 { 1280 return NULL; 1281 } 1282 1283 static inline 1284 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css) 1285 { 1286 return NULL; 1287 } 1288 1289 static inline void obj_cgroup_put(struct obj_cgroup *objcg) 1290 { 1291 } 1292 1293 static inline void mem_cgroup_put(struct mem_cgroup *memcg) 1294 { 1295 } 1296 1297 static inline struct lruvec *folio_lruvec_lock(struct folio *folio) 1298 { 1299 struct pglist_data *pgdat = folio_pgdat(folio); 1300 1301 spin_lock(&pgdat->__lruvec.lru_lock); 1302 return &pgdat->__lruvec; 1303 } 1304 1305 static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) 1306 { 1307 struct pglist_data *pgdat = folio_pgdat(folio); 1308 1309 spin_lock_irq(&pgdat->__lruvec.lru_lock); 1310 return &pgdat->__lruvec; 1311 } 1312 1313 static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, 1314 unsigned long *flagsp) 1315 { 1316 struct pglist_data *pgdat = folio_pgdat(folio); 1317 1318 spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp); 1319 return &pgdat->__lruvec; 1320 } 1321 1322 static inline struct mem_cgroup * 1323 mem_cgroup_iter(struct mem_cgroup *root, 1324 struct mem_cgroup *prev, 1325 struct mem_cgroup_reclaim_cookie *reclaim) 1326 { 1327 return NULL; 1328 } 1329 1330 static inline void mem_cgroup_iter_break(struct mem_cgroup *root, 1331 struct mem_cgroup *prev) 1332 { 1333 } 1334 1335 static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 1336 int (*fn)(struct task_struct *, void *), void *arg) 1337 { 1338 return 0; 1339 } 1340 1341 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg) 1342 { 1343 return 0; 1344 } 1345 1346 static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id) 1347 { 1348 WARN_ON_ONCE(id); 1349 /* XXX: This should always return root_mem_cgroup */ 1350 return NULL; 1351 } 1352 1353 #ifdef CONFIG_SHRINKER_DEBUG 1354 static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg) 1355 { 1356 return 0; 1357 } 1358 1359 static inline struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino) 1360 { 1361 return NULL; 1362 } 1363 #endif 1364 1365 static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) 1366 { 1367 return NULL; 1368 } 1369 1370 static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) 1371 { 1372 return NULL; 1373 } 1374 1375 static inline bool mem_cgroup_online(struct mem_cgroup *memcg) 1376 { 1377 return true; 1378 } 1379 1380 static inline 1381 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, 1382 enum lru_list lru, int zone_idx) 1383 { 1384 return 0; 1385 } 1386 1387 static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) 1388 { 1389 return 0; 1390 } 1391 1392 static inline unsigned long mem_cgroup_size(struct mem_cgroup *memcg) 1393 { 1394 return 0; 1395 } 1396 1397 static inline void 1398 mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) 1399 { 1400 } 1401 1402 static inline void 1403 mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) 1404 { 1405 } 1406 1407 static inline void lock_page_memcg(struct page *page) 1408 { 1409 } 1410 1411 static inline void unlock_page_memcg(struct page *page) 1412 { 1413 } 1414 1415 static inline void folio_memcg_lock(struct folio *folio) 1416 { 1417 } 1418 1419 static inline void folio_memcg_unlock(struct folio *folio) 1420 { 1421 } 1422 1423 static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) 1424 { 1425 /* to match folio_memcg_rcu() */ 1426 rcu_read_lock(); 1427 return true; 1428 } 1429 1430 static inline void mem_cgroup_unlock_pages(void) 1431 { 1432 rcu_read_unlock(); 1433 } 1434 1435 static inline void mem_cgroup_handle_over_high(void) 1436 { 1437 } 1438 1439 static inline void mem_cgroup_enter_user_fault(void) 1440 { 1441 } 1442 1443 static inline void mem_cgroup_exit_user_fault(void) 1444 { 1445 } 1446 1447 static inline bool task_in_memcg_oom(struct task_struct *p) 1448 { 1449 return false; 1450 } 1451 1452 static inline bool mem_cgroup_oom_synchronize(bool wait) 1453 { 1454 return false; 1455 } 1456 1457 static inline struct mem_cgroup *mem_cgroup_get_oom_group( 1458 struct task_struct *victim, struct mem_cgroup *oom_domain) 1459 { 1460 return NULL; 1461 } 1462 1463 static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) 1464 { 1465 } 1466 1467 static inline void __mod_memcg_state(struct mem_cgroup *memcg, 1468 int idx, 1469 int nr) 1470 { 1471 } 1472 1473 static inline void mod_memcg_state(struct mem_cgroup *memcg, 1474 int idx, 1475 int nr) 1476 { 1477 } 1478 1479 static inline void mod_memcg_page_state(struct page *page, 1480 int idx, int val) 1481 { 1482 } 1483 1484 static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) 1485 { 1486 return 0; 1487 } 1488 1489 static inline unsigned long lruvec_page_state(struct lruvec *lruvec, 1490 enum node_stat_item idx) 1491 { 1492 return node_page_state(lruvec_pgdat(lruvec), idx); 1493 } 1494 1495 static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, 1496 enum node_stat_item idx) 1497 { 1498 return node_page_state(lruvec_pgdat(lruvec), idx); 1499 } 1500 1501 static inline void mem_cgroup_flush_stats(void) 1502 { 1503 } 1504 1505 static inline void mem_cgroup_flush_stats_delayed(void) 1506 { 1507 } 1508 1509 static inline void __mod_memcg_lruvec_state(struct lruvec *lruvec, 1510 enum node_stat_item idx, int val) 1511 { 1512 } 1513 1514 static inline void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, 1515 int val) 1516 { 1517 struct page *page = virt_to_head_page(p); 1518 1519 __mod_node_page_state(page_pgdat(page), idx, val); 1520 } 1521 1522 static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, 1523 int val) 1524 { 1525 struct page *page = virt_to_head_page(p); 1526 1527 mod_node_page_state(page_pgdat(page), idx, val); 1528 } 1529 1530 static inline void count_memcg_events(struct mem_cgroup *memcg, 1531 enum vm_event_item idx, 1532 unsigned long count) 1533 { 1534 } 1535 1536 static inline void __count_memcg_events(struct mem_cgroup *memcg, 1537 enum vm_event_item idx, 1538 unsigned long count) 1539 { 1540 } 1541 1542 static inline void count_memcg_page_event(struct page *page, 1543 int idx) 1544 { 1545 } 1546 1547 static inline void count_memcg_folio_events(struct folio *folio, 1548 enum vm_event_item idx, unsigned long nr) 1549 { 1550 } 1551 1552 static inline 1553 void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx) 1554 { 1555 } 1556 1557 static inline void split_page_memcg(struct page *head, unsigned int nr) 1558 { 1559 } 1560 1561 static inline 1562 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 1563 gfp_t gfp_mask, 1564 unsigned long *total_scanned) 1565 { 1566 return 0; 1567 } 1568 #endif /* CONFIG_MEMCG */ 1569 1570 static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx) 1571 { 1572 __mod_lruvec_kmem_state(p, idx, 1); 1573 } 1574 1575 static inline void __dec_lruvec_kmem_state(void *p, enum node_stat_item idx) 1576 { 1577 __mod_lruvec_kmem_state(p, idx, -1); 1578 } 1579 1580 static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) 1581 { 1582 struct mem_cgroup *memcg; 1583 1584 memcg = lruvec_memcg(lruvec); 1585 if (!memcg) 1586 return NULL; 1587 memcg = parent_mem_cgroup(memcg); 1588 if (!memcg) 1589 return NULL; 1590 return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec)); 1591 } 1592 1593 static inline void unlock_page_lruvec(struct lruvec *lruvec) 1594 { 1595 spin_unlock(&lruvec->lru_lock); 1596 } 1597 1598 static inline void unlock_page_lruvec_irq(struct lruvec *lruvec) 1599 { 1600 spin_unlock_irq(&lruvec->lru_lock); 1601 } 1602 1603 static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec, 1604 unsigned long flags) 1605 { 1606 spin_unlock_irqrestore(&lruvec->lru_lock, flags); 1607 } 1608 1609 /* Test requires a stable page->memcg binding, see page_memcg() */ 1610 static inline bool folio_matches_lruvec(struct folio *folio, 1611 struct lruvec *lruvec) 1612 { 1613 return lruvec_pgdat(lruvec) == folio_pgdat(folio) && 1614 lruvec_memcg(lruvec) == folio_memcg(folio); 1615 } 1616 1617 /* Don't lock again iff page's lruvec locked */ 1618 static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, 1619 struct lruvec *locked_lruvec) 1620 { 1621 if (locked_lruvec) { 1622 if (folio_matches_lruvec(folio, locked_lruvec)) 1623 return locked_lruvec; 1624 1625 unlock_page_lruvec_irq(locked_lruvec); 1626 } 1627 1628 return folio_lruvec_lock_irq(folio); 1629 } 1630 1631 /* Don't lock again iff page's lruvec locked */ 1632 static inline struct lruvec *folio_lruvec_relock_irqsave(struct folio *folio, 1633 struct lruvec *locked_lruvec, unsigned long *flags) 1634 { 1635 if (locked_lruvec) { 1636 if (folio_matches_lruvec(folio, locked_lruvec)) 1637 return locked_lruvec; 1638 1639 unlock_page_lruvec_irqrestore(locked_lruvec, *flags); 1640 } 1641 1642 return folio_lruvec_lock_irqsave(folio, flags); 1643 } 1644 1645 #ifdef CONFIG_CGROUP_WRITEBACK 1646 1647 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb); 1648 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, 1649 unsigned long *pheadroom, unsigned long *pdirty, 1650 unsigned long *pwriteback); 1651 1652 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio, 1653 struct bdi_writeback *wb); 1654 1655 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio, 1656 struct bdi_writeback *wb) 1657 { 1658 if (mem_cgroup_disabled()) 1659 return; 1660 1661 if (unlikely(&folio_memcg(folio)->css != wb->memcg_css)) 1662 mem_cgroup_track_foreign_dirty_slowpath(folio, wb); 1663 } 1664 1665 void mem_cgroup_flush_foreign(struct bdi_writeback *wb); 1666 1667 #else /* CONFIG_CGROUP_WRITEBACK */ 1668 1669 static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) 1670 { 1671 return NULL; 1672 } 1673 1674 static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb, 1675 unsigned long *pfilepages, 1676 unsigned long *pheadroom, 1677 unsigned long *pdirty, 1678 unsigned long *pwriteback) 1679 { 1680 } 1681 1682 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio, 1683 struct bdi_writeback *wb) 1684 { 1685 } 1686 1687 static inline void mem_cgroup_flush_foreign(struct bdi_writeback *wb) 1688 { 1689 } 1690 1691 #endif /* CONFIG_CGROUP_WRITEBACK */ 1692 1693 struct sock; 1694 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages, 1695 gfp_t gfp_mask); 1696 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages); 1697 #ifdef CONFIG_MEMCG 1698 extern struct static_key_false memcg_sockets_enabled_key; 1699 #define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key) 1700 void mem_cgroup_sk_alloc(struct sock *sk); 1701 void mem_cgroup_sk_free(struct sock *sk); 1702 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg) 1703 { 1704 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure) 1705 return true; 1706 do { 1707 if (time_before(jiffies, READ_ONCE(memcg->socket_pressure))) 1708 return true; 1709 } while ((memcg = parent_mem_cgroup(memcg))); 1710 return false; 1711 } 1712 1713 int alloc_shrinker_info(struct mem_cgroup *memcg); 1714 void free_shrinker_info(struct mem_cgroup *memcg); 1715 void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id); 1716 void reparent_shrinker_deferred(struct mem_cgroup *memcg); 1717 #else 1718 #define mem_cgroup_sockets_enabled 0 1719 static inline void mem_cgroup_sk_alloc(struct sock *sk) { }; 1720 static inline void mem_cgroup_sk_free(struct sock *sk) { }; 1721 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg) 1722 { 1723 return false; 1724 } 1725 1726 static inline void set_shrinker_bit(struct mem_cgroup *memcg, 1727 int nid, int shrinker_id) 1728 { 1729 } 1730 #endif 1731 1732 #ifdef CONFIG_MEMCG_KMEM 1733 bool mem_cgroup_kmem_disabled(void); 1734 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order); 1735 void __memcg_kmem_uncharge_page(struct page *page, int order); 1736 1737 struct obj_cgroup *get_obj_cgroup_from_current(void); 1738 struct obj_cgroup *get_obj_cgroup_from_page(struct page *page); 1739 1740 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size); 1741 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size); 1742 1743 extern struct static_key_false memcg_kmem_enabled_key; 1744 1745 static inline bool memcg_kmem_enabled(void) 1746 { 1747 return static_branch_likely(&memcg_kmem_enabled_key); 1748 } 1749 1750 static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp, 1751 int order) 1752 { 1753 if (memcg_kmem_enabled()) 1754 return __memcg_kmem_charge_page(page, gfp, order); 1755 return 0; 1756 } 1757 1758 static inline void memcg_kmem_uncharge_page(struct page *page, int order) 1759 { 1760 if (memcg_kmem_enabled()) 1761 __memcg_kmem_uncharge_page(page, order); 1762 } 1763 1764 /* 1765 * A helper for accessing memcg's kmem_id, used for getting 1766 * corresponding LRU lists. 1767 */ 1768 static inline int memcg_kmem_id(struct mem_cgroup *memcg) 1769 { 1770 return memcg ? memcg->kmemcg_id : -1; 1771 } 1772 1773 struct mem_cgroup *mem_cgroup_from_obj(void *p); 1774 struct mem_cgroup *mem_cgroup_from_slab_obj(void *p); 1775 1776 static inline void count_objcg_event(struct obj_cgroup *objcg, 1777 enum vm_event_item idx) 1778 { 1779 struct mem_cgroup *memcg; 1780 1781 if (!memcg_kmem_enabled()) 1782 return; 1783 1784 rcu_read_lock(); 1785 memcg = obj_cgroup_memcg(objcg); 1786 count_memcg_events(memcg, idx, 1); 1787 rcu_read_unlock(); 1788 } 1789 1790 #else 1791 static inline bool mem_cgroup_kmem_disabled(void) 1792 { 1793 return true; 1794 } 1795 1796 static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp, 1797 int order) 1798 { 1799 return 0; 1800 } 1801 1802 static inline void memcg_kmem_uncharge_page(struct page *page, int order) 1803 { 1804 } 1805 1806 static inline int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, 1807 int order) 1808 { 1809 return 0; 1810 } 1811 1812 static inline void __memcg_kmem_uncharge_page(struct page *page, int order) 1813 { 1814 } 1815 1816 static inline struct obj_cgroup *get_obj_cgroup_from_page(struct page *page) 1817 { 1818 return NULL; 1819 } 1820 1821 static inline bool memcg_kmem_enabled(void) 1822 { 1823 return false; 1824 } 1825 1826 static inline int memcg_kmem_id(struct mem_cgroup *memcg) 1827 { 1828 return -1; 1829 } 1830 1831 static inline struct mem_cgroup *mem_cgroup_from_obj(void *p) 1832 { 1833 return NULL; 1834 } 1835 1836 static inline struct mem_cgroup *mem_cgroup_from_slab_obj(void *p) 1837 { 1838 return NULL; 1839 } 1840 1841 static inline void count_objcg_event(struct obj_cgroup *objcg, 1842 enum vm_event_item idx) 1843 { 1844 } 1845 1846 #endif /* CONFIG_MEMCG_KMEM */ 1847 1848 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 1849 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg); 1850 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size); 1851 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size); 1852 #else 1853 static inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg) 1854 { 1855 return true; 1856 } 1857 static inline void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, 1858 size_t size) 1859 { 1860 } 1861 static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, 1862 size_t size) 1863 { 1864 } 1865 #endif 1866 1867 #endif /* _LINUX_MEMCONTROL_H */ 1868