/linux/include/linux/ |
H A D | mm_inline.h | 38 static __always_inline void __update_lru_size(struct lruvec *lruvec, in __update_lru_size() argument 52 static __always_inline void update_lru_size(struct lruvec *lruvec, in update_lru_size() argument 165 static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) in lru_gen_is_active() argument 175 static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, in lru_gen_update_size() argument 212 if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { in lru_gen_update_size() 218 VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); in lru_gen_update_size() 221 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) in lru_gen_add_folio() argument 272 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) in lru_gen_del_folio() argument 319 void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) in lruvec_add_folio() argument 333 void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) in lruvec_add_folio_tail() argument [all …]
|
H A D | memcontrol.h | 99 struct lruvec lruvec; member 741 struct lruvec *lruvec; in mem_cgroup_lruvec() local 752 lruvec = &mz->lruvec; in mem_cgroup_lruvec() 761 return lruvec; in mem_cgroup_lruvec() 881 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); in lruvec_memcg() 932 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, in mem_cgroup_get_zone_lru_size() argument 937 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); in mem_cgroup_get_zone_lru_size() 1025 unsigned long lruvec_page_state_local(struct lruvec *lruvec, 1610 static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) in parent_lruvec() argument 1623 static inline void unlock_page_lruvec(struct lruvec *lruvec) in unlock_page_lruvec() argument [all …]
|
H A D | zswap.h | 8 struct lruvec; 36 void zswap_lruvec_state_init(struct lruvec *lruvec); 60 static inline void zswap_lruvec_state_init(struct lruvec *lruvec) {} in zswap_lruvec_state_init() argument
|
H A D | vmstat.h | 538 void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 541 static inline void mod_lruvec_state(struct lruvec *lruvec, in mod_lruvec_state() argument 547 __mod_lruvec_state(lruvec, idx, val); in mod_lruvec_state() 572 static inline void __mod_lruvec_state(struct lruvec *lruvec, in __mod_lruvec_state() argument 575 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val); in __mod_lruvec_state() 578 static inline void mod_lruvec_state(struct lruvec *lruvec, in mod_lruvec_state() argument 581 mod_node_page_state(lruvec_pgdat(lruvec), idx, val); in mod_lruvec_state()
|
H A D | mmzone.h | 386 struct lruvec; 487 struct lruvec *lruvec; member 559 void lru_gen_init_lruvec(struct lruvec *lruvec); 575 static inline void lru_gen_init_lruvec(struct lruvec *lruvec) in lru_gen_init_lruvec() argument 609 struct lruvec { struct 1396 struct lruvec __lruvec; 1456 extern void lruvec_init(struct lruvec *lruvec); 1458 static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec) in lruvec_pgdat() argument 1461 return lruvec->pgdat; in lruvec_pgdat() 1463 return container_of(lruvec, struct pglist_data, __lruvec); in lruvec_pgdat()
|
H A D | swap.h | 358 void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); 371 void lru_note_cost(struct lruvec *lruvec, bool file,
|
/linux/mm/ |
H A D | swap.c | 107 struct lruvec *lruvec = NULL; in page_cache_release() local 111 if (lruvec) in page_cache_release() 166 static void lru_add_fn(struct lruvec *lruvec, struct folio *folio) in lru_add_fn() argument 209 struct lruvec *lruvec = NULL; in folio_batch_move_lru() local 225 if (lruvec) in folio_batch_move_lru() 271 void lru_note_cost(struct lruvec *lruvec, bool file, in lru_note_cost() argument 315 if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) { in lru_note_cost() 320 } while ((lruvec = parent_lruvec(lruvec))); in lru_note_cost() 375 struct lruvec *lruvec; in folio_activate() local 971 struct lruvec *lruvec = NULL; in folios_put_refs() local [all …]
|
H A D | workingset.c | 237 struct lruvec *lruvec; in lru_gen_eviction() local 249 lrugen = &lruvec->lrugen; in lru_gen_eviction() 263 static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec, in lru_gen_test_recent() argument 286 struct lruvec *lruvec; in lru_gen_refault() local 302 lrugen = &lruvec->lrugen; in lru_gen_refault() 334 static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec, in lru_gen_test_recent() argument 356 void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages) in workingset_age_nonresident() argument 371 } while ((lruvec = parent_lruvec(lruvec))); in workingset_age_nonresident() 386 struct lruvec *lruvec; in workingset_eviction() local 535 struct lruvec *lruvec; in workingset_refault() local [all …]
|
H A D | vmscan.c | 1753 struct lruvec *lruvec; in folio_isolate_lru() local 2911 struct lruvec *lruvec = walk->lruvec; in reset_mm_stats() local 2937 struct lruvec *lruvec = walk->lruvec; in iterate_mm_list() local 3188 struct lruvec *lruvec = walk->lruvec; in reset_batch_size() local 3629 struct lruvec *lruvec = walk->lruvec; in walk_mm() local 3880 walk->lruvec = lruvec; in try_to_inc_max_seq() 4594 walk->lruvec = lruvec; in evict_folios() 4815 struct lruvec *lruvec; in shrink_many() local 5408 struct lruvec *lruvec; in run_cmd() local 6256 struct lruvec *lruvec; in do_try_to_free_pages() local [all …]
|
H A D | mlock.c | 61 static struct lruvec *__mlock_folio(struct folio *folio, struct lruvec *lruvec) in __mlock_folio() argument 67 lruvec = folio_lruvec_relock_irq(folio, lruvec); in __mlock_folio() 103 static struct lruvec *__mlock_new_folio(struct folio *folio, struct lruvec *lruvec) in __mlock_new_folio() argument 107 lruvec = folio_lruvec_relock_irq(folio, lruvec); in __mlock_new_folio() 122 static struct lruvec *__munlock_folio(struct folio *folio, struct lruvec *lruvec) in __munlock_folio() argument 131 lruvec = folio_lruvec_relock_irq(folio, lruvec); in __munlock_folio() 188 struct lruvec *lruvec = NULL; in mlock_folio_batch() local 200 lruvec = __mlock_folio(folio, lruvec); in mlock_folio_batch() 202 lruvec = __mlock_new_folio(folio, lruvec); in mlock_folio_batch() 204 lruvec = __munlock_folio(folio, lruvec); in mlock_folio_batch() [all …]
|
H A D | mmzone.c | 75 void lruvec_init(struct lruvec *lruvec) in lruvec_init() argument 79 memset(lruvec, 0, sizeof(struct lruvec)); in lruvec_init() 80 spin_lock_init(&lruvec->lru_lock); in lruvec_init() 81 zswap_lruvec_state_init(lruvec); in lruvec_init() 84 INIT_LIST_HEAD(&lruvec->lists[lru]); in lruvec_init() 91 list_del(&lruvec->lists[LRU_UNEVICTABLE]); in lruvec_init() 93 lru_gen_init_lruvec(lruvec); in lruvec_init()
|
H A D | zswap.c | 717 struct lruvec *lruvec; in zswap_lru_add() local 738 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); in zswap_lru_add() 763 void zswap_lruvec_state_init(struct lruvec *lruvec) in zswap_lruvec_state_init() argument 765 atomic_long_set(&lruvec->zswap_lruvec_state.nr_zswap_protected, 0); in zswap_lruvec_state_init() 770 struct lruvec *lruvec; in zswap_folio_swapin() local 773 lruvec = folio_lruvec(folio); in zswap_folio_swapin() 774 atomic_long_inc(&lruvec->zswap_lruvec_state.nr_zswap_protected); in zswap_folio_swapin() 1175 struct lruvec *lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid)); in zswap_shrinker_scan() local 1186 atomic_long_read(&lruvec->zswap_lruvec_state.nr_zswap_protected); in zswap_shrinker_scan() 1217 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid)); in zswap_shrinker_count() local [all …]
|
H A D | memcontrol.c | 685 unsigned long lruvec_page_state_local(struct lruvec *lruvec, in lruvec_page_state_local() argument 990 static void __mod_memcg_lruvec_state(struct lruvec *lruvec, in __mod_memcg_lruvec_state() argument 1059 struct lruvec *lruvec; in __lruvec_stat_mod_folio() local 1080 struct lruvec *lruvec; in __mod_lruvec_kmem_state() local 1531 struct lruvec *lruvec = folio_lruvec(folio); in folio_lruvec_lock() local 1536 return lruvec; in folio_lruvec_lock() 1554 struct lruvec *lruvec = folio_lruvec(folio); in folio_lruvec_lock_irq() local 1559 return lruvec; in folio_lruvec_lock_irq() 1579 struct lruvec *lruvec = folio_lruvec(folio); in folio_lruvec_lock_irqsave() local 3148 struct lruvec *lruvec; in __mod_objcg_mlstate() local [all …]
|
H A D | compaction.c | 896 struct lruvec *lruvec; in isolate_migratepages_block() local 898 struct lruvec *locked = NULL; in isolate_migratepages_block() 1213 lruvec = folio_lruvec(folio); in isolate_migratepages_block() 1216 if (lruvec != locked) { in isolate_migratepages_block() 1220 compact_lock_irqsave(&lruvec->lru_lock, &flags, cc); in isolate_migratepages_block() 1221 locked = lruvec; in isolate_migratepages_block() 1223 lruvec_memcg_debug(lruvec, folio); in isolate_migratepages_block() 1257 lruvec_del_folio(lruvec, folio); in isolate_migratepages_block()
|
H A D | huge_memory.c | 2705 struct lruvec *lruvec, struct list_head *list) in lru_add_page_tail() argument 2709 lockdep_assert_held(&lruvec->lru_lock); in lru_add_page_tail() 2728 struct lruvec *lruvec, struct list_head *list, in __split_huge_page_tail() argument 2821 lru_add_page_tail(head, page_tail, lruvec, list); in __split_huge_page_tail() 2829 struct lruvec *lruvec; in __split_huge_page() local 2847 lruvec = folio_lruvec_lock(folio); in __split_huge_page() 2852 __split_huge_page_tail(folio, i, lruvec, list, new_order); in __split_huge_page() 2880 unlock_page_lruvec(lruvec); in __split_huge_page()
|
H A D | migrate.c | 489 struct lruvec *old_lruvec, *new_lruvec; in folio_migrate_mapping()
|
/linux/Documentation/mm/ |
H A D | multigen_lru.rst | 83 ``lruvec``. The youngest generation number is stored in 114 The aging produces young generations. Given an ``lruvec``, it 130 The eviction consumes old generations. Given an ``lruvec``, it 148 set, an ``lruvec`` is protected from the eviction when its oldest
|
H A D | physical_memory.rst | 267 Per-node lruvec holding LRU lists and related parameters. Used only when
|
/linux/Documentation/trace/ |
H A D | events-kmem.rst | 72 contention on the lruvec->lru_lock.
|
/linux/Documentation/admin-guide/cgroup-v1/ |
H A D | memory.rst | 307 lruvec->lru_lock. 310 lruvec->lru_lock; the folio LRU flag is cleared before 311 isolating a page from its LRU under lruvec->lru_lock.
|