1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_MM_H
3 #define _LINUX_MM_H
4
5 #include <linux/errno.h>
6 #include <linux/mmdebug.h>
7 #include <linux/gfp.h>
8 #include <linux/pgalloc_tag.h>
9 #include <linux/bug.h>
10 #include <linux/list.h>
11 #include <linux/mmzone.h>
12 #include <linux/rbtree.h>
13 #include <linux/atomic.h>
14 #include <linux/debug_locks.h>
15 #include <linux/mm_types.h>
16 #include <linux/mmap_lock.h>
17 #include <linux/range.h>
18 #include <linux/pfn.h>
19 #include <linux/percpu-refcount.h>
20 #include <linux/bit_spinlock.h>
21 #include <linux/shrinker.h>
22 #include <linux/resource.h>
23 #include <linux/page_ext.h>
24 #include <linux/err.h>
25 #include <linux/page-flags.h>
26 #include <linux/page_ref.h>
27 #include <linux/overflow.h>
28 #include <linux/sizes.h>
29 #include <linux/sched.h>
30 #include <linux/pgtable.h>
31 #include <linux/kasan.h>
32 #include <linux/memremap.h>
33 #include <linux/slab.h>
34
35 struct mempolicy;
36 struct anon_vma;
37 struct anon_vma_chain;
38 struct user_struct;
39 struct pt_regs;
40 struct folio_batch;
41
42 extern int sysctl_page_lock_unfairness;
43
44 void mm_core_init(void);
45 void init_mm_internals(void);
46
47 #ifndef CONFIG_NUMA /* Don't use mapnrs, do it properly */
48 extern unsigned long max_mapnr;
49
set_max_mapnr(unsigned long limit)50 static inline void set_max_mapnr(unsigned long limit)
51 {
52 max_mapnr = limit;
53 }
54 #else
set_max_mapnr(unsigned long limit)55 static inline void set_max_mapnr(unsigned long limit) { }
56 #endif
57
58 extern atomic_long_t _totalram_pages;
totalram_pages(void)59 static inline unsigned long totalram_pages(void)
60 {
61 return (unsigned long)atomic_long_read(&_totalram_pages);
62 }
63
totalram_pages_inc(void)64 static inline void totalram_pages_inc(void)
65 {
66 atomic_long_inc(&_totalram_pages);
67 }
68
totalram_pages_dec(void)69 static inline void totalram_pages_dec(void)
70 {
71 atomic_long_dec(&_totalram_pages);
72 }
73
totalram_pages_add(long count)74 static inline void totalram_pages_add(long count)
75 {
76 atomic_long_add(count, &_totalram_pages);
77 }
78
79 extern void * high_memory;
80 extern int page_cluster;
81 extern const int page_cluster_max;
82
83 #ifdef CONFIG_SYSCTL
84 extern int sysctl_legacy_va_layout;
85 #else
86 #define sysctl_legacy_va_layout 0
87 #endif
88
89 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
90 extern const int mmap_rnd_bits_min;
91 extern int mmap_rnd_bits_max __ro_after_init;
92 extern int mmap_rnd_bits __read_mostly;
93 #endif
94 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
95 extern const int mmap_rnd_compat_bits_min;
96 extern const int mmap_rnd_compat_bits_max;
97 extern int mmap_rnd_compat_bits __read_mostly;
98 #endif
99
100 #include <asm/page.h>
101 #include <asm/processor.h>
102
103 #ifndef __pa_symbol
104 #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
105 #endif
106
107 #ifndef page_to_virt
108 #define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x)))
109 #endif
110
111 #ifndef lm_alias
112 #define lm_alias(x) __va(__pa_symbol(x))
113 #endif
114
115 /*
116 * To prevent common memory management code establishing
117 * a zero page mapping on a read fault.
118 * This macro should be defined within <asm/pgtable.h>.
119 * s390 does this to prevent multiplexing of hardware bits
120 * related to the physical page in case of virtualization.
121 */
122 #ifndef mm_forbids_zeropage
123 #define mm_forbids_zeropage(X) (0)
124 #endif
125
126 /*
127 * On some architectures it is expensive to call memset() for small sizes.
128 * If an architecture decides to implement their own version of
129 * mm_zero_struct_page they should wrap the defines below in a #ifndef and
130 * define their own version of this macro in <asm/pgtable.h>
131 */
132 #if BITS_PER_LONG == 64
133 /* This function must be updated when the size of struct page grows above 96
134 * or reduces below 56. The idea that compiler optimizes out switch()
135 * statement, and only leaves move/store instructions. Also the compiler can
136 * combine write statements if they are both assignments and can be reordered,
137 * this can result in several of the writes here being dropped.
138 */
139 #define mm_zero_struct_page(pp) __mm_zero_struct_page(pp)
__mm_zero_struct_page(struct page * page)140 static inline void __mm_zero_struct_page(struct page *page)
141 {
142 unsigned long *_pp = (void *)page;
143
144 /* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */
145 BUILD_BUG_ON(sizeof(struct page) & 7);
146 BUILD_BUG_ON(sizeof(struct page) < 56);
147 BUILD_BUG_ON(sizeof(struct page) > 96);
148
149 switch (sizeof(struct page)) {
150 case 96:
151 _pp[11] = 0;
152 fallthrough;
153 case 88:
154 _pp[10] = 0;
155 fallthrough;
156 case 80:
157 _pp[9] = 0;
158 fallthrough;
159 case 72:
160 _pp[8] = 0;
161 fallthrough;
162 case 64:
163 _pp[7] = 0;
164 fallthrough;
165 case 56:
166 _pp[6] = 0;
167 _pp[5] = 0;
168 _pp[4] = 0;
169 _pp[3] = 0;
170 _pp[2] = 0;
171 _pp[1] = 0;
172 _pp[0] = 0;
173 }
174 }
175 #else
176 #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page)))
177 #endif
178
179 /*
180 * Default maximum number of active map areas, this limits the number of vmas
181 * per mm struct. Users can overwrite this number by sysctl but there is a
182 * problem.
183 *
184 * When a program's coredump is generated as ELF format, a section is created
185 * per a vma. In ELF, the number of sections is represented in unsigned short.
186 * This means the number of sections should be smaller than 65535 at coredump.
187 * Because the kernel adds some informative sections to a image of program at
188 * generating coredump, we need some margin. The number of extra sections is
189 * 1-3 now and depends on arch. We use "5" as safe margin, here.
190 *
191 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is
192 * not a hard limit any more. Although some userspace tools can be surprised by
193 * that.
194 */
195 #define MAPCOUNT_ELF_CORE_MARGIN (5)
196 #define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
197
198 extern int sysctl_max_map_count;
199
200 extern unsigned long sysctl_user_reserve_kbytes;
201 extern unsigned long sysctl_admin_reserve_kbytes;
202
203 extern int sysctl_overcommit_memory;
204 extern int sysctl_overcommit_ratio;
205 extern unsigned long sysctl_overcommit_kbytes;
206
207 int overcommit_ratio_handler(struct ctl_table *, int, void *, size_t *,
208 loff_t *);
209 int overcommit_kbytes_handler(struct ctl_table *, int, void *, size_t *,
210 loff_t *);
211 int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *,
212 loff_t *);
213
214 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
215 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
216 #define folio_page_idx(folio, p) (page_to_pfn(p) - folio_pfn(folio))
217 #else
218 #define nth_page(page,n) ((page) + (n))
219 #define folio_page_idx(folio, p) ((p) - &(folio)->page)
220 #endif
221
222 /* to align the pointer to the (next) page boundary */
223 #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
224
225 /* to align the pointer to the (prev) page boundary */
226 #define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE)
227
228 /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
229 #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
230
lru_to_folio(struct list_head * head)231 static inline struct folio *lru_to_folio(struct list_head *head)
232 {
233 return list_entry((head)->prev, struct folio, lru);
234 }
235
236 void setup_initial_init_mm(void *start_code, void *end_code,
237 void *end_data, void *brk);
238
239 /*
240 * Linux kernel virtual memory manager primitives.
241 * The idea being to have a "virtual" mm in the same way
242 * we have a virtual fs - giving a cleaner interface to the
243 * mm details, and allowing different kinds of memory mappings
244 * (from shared memory to executable loading to arbitrary
245 * mmap() functions).
246 */
247
248 struct vm_area_struct *vm_area_alloc(struct mm_struct *);
249 struct vm_area_struct *vm_area_dup(struct vm_area_struct *);
250 void vm_area_free(struct vm_area_struct *);
251 /* Use only if VMA has no other users */
252 void __vm_area_free(struct vm_area_struct *vma);
253
254 #ifndef CONFIG_MMU
255 extern struct rb_root nommu_region_tree;
256 extern struct rw_semaphore nommu_region_sem;
257
258 extern unsigned int kobjsize(const void *objp);
259 #endif
260
261 /*
262 * vm_flags in vm_area_struct, see mm_types.h.
263 * When changing, update also include/trace/events/mmflags.h
264 */
265 #define VM_NONE 0x00000000
266
267 #define VM_READ 0x00000001 /* currently active flags */
268 #define VM_WRITE 0x00000002
269 #define VM_EXEC 0x00000004
270 #define VM_SHARED 0x00000008
271
272 /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
273 #define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
274 #define VM_MAYWRITE 0x00000020
275 #define VM_MAYEXEC 0x00000040
276 #define VM_MAYSHARE 0x00000080
277
278 #define VM_GROWSDOWN 0x00000100 /* general info on the segment */
279 #ifdef CONFIG_MMU
280 #define VM_UFFD_MISSING 0x00000200 /* missing pages tracking */
281 #else /* CONFIG_MMU */
282 #define VM_MAYOVERLAY 0x00000200 /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */
283 #define VM_UFFD_MISSING 0
284 #endif /* CONFIG_MMU */
285 #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
286 #define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */
287
288 #define VM_LOCKED 0x00002000
289 #define VM_IO 0x00004000 /* Memory mapped I/O or similar */
290
291 /* Used by sys_madvise() */
292 #define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
293 #define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
294
295 #define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
296 #define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
297 #define VM_LOCKONFAULT 0x00080000 /* Lock the pages covered when they are faulted in */
298 #define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
299 #define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
300 #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
301 #define VM_SYNC 0x00800000 /* Synchronous page faults */
302 #define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
303 #define VM_WIPEONFORK 0x02000000 /* Wipe VMA contents in child. */
304 #define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
305
306 #ifdef CONFIG_MEM_SOFT_DIRTY
307 # define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */
308 #else
309 # define VM_SOFTDIRTY 0
310 #endif
311
312 #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
313 #define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
314 #define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
315 #define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
316
317 #ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS
318 #define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */
319 #define VM_HIGH_ARCH_BIT_1 33 /* bit only usable on 64-bit architectures */
320 #define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */
321 #define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */
322 #define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit architectures */
323 #define VM_HIGH_ARCH_BIT_5 37 /* bit only usable on 64-bit architectures */
324 #define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
325 #define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
326 #define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
327 #define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
328 #define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4)
329 #define VM_HIGH_ARCH_5 BIT(VM_HIGH_ARCH_BIT_5)
330 #endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
331
332 #ifdef CONFIG_ARCH_HAS_PKEYS
333 # define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0
334 # define VM_PKEY_BIT0 VM_HIGH_ARCH_0 /* A protection key is a 4-bit value */
335 # define VM_PKEY_BIT1 VM_HIGH_ARCH_1 /* on x86 and 5-bit value on ppc64 */
336 # define VM_PKEY_BIT2 VM_HIGH_ARCH_2
337 # define VM_PKEY_BIT3 VM_HIGH_ARCH_3
338 #ifdef CONFIG_PPC
339 # define VM_PKEY_BIT4 VM_HIGH_ARCH_4
340 #else
341 # define VM_PKEY_BIT4 0
342 #endif
343 #endif /* CONFIG_ARCH_HAS_PKEYS */
344
345 #ifdef CONFIG_X86_USER_SHADOW_STACK
346 /*
347 * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of
348 * support core mm.
349 *
350 * These VMAs will get a single end guard page. This helps userspace protect
351 * itself from attacks. A single page is enough for current shadow stack archs
352 * (x86). See the comments near alloc_shstk() in arch/x86/kernel/shstk.c
353 * for more details on the guard size.
354 */
355 # define VM_SHADOW_STACK VM_HIGH_ARCH_5
356 #else
357 # define VM_SHADOW_STACK VM_NONE
358 #endif
359
360 #if defined(CONFIG_X86)
361 # define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
362 #elif defined(CONFIG_PPC)
363 # define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
364 #elif defined(CONFIG_PARISC)
365 # define VM_GROWSUP VM_ARCH_1
366 #elif defined(CONFIG_SPARC64)
367 # define VM_SPARC_ADI VM_ARCH_1 /* Uses ADI tag for access control */
368 # define VM_ARCH_CLEAR VM_SPARC_ADI
369 #elif defined(CONFIG_ARM64)
370 # define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */
371 # define VM_ARCH_CLEAR VM_ARM64_BTI
372 #elif !defined(CONFIG_MMU)
373 # define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
374 #endif
375
376 #if defined(CONFIG_ARM64_MTE)
377 # define VM_MTE VM_HIGH_ARCH_0 /* Use Tagged memory for access control */
378 # define VM_MTE_ALLOWED VM_HIGH_ARCH_1 /* Tagged memory permitted */
379 #else
380 # define VM_MTE VM_NONE
381 # define VM_MTE_ALLOWED VM_NONE
382 #endif
383
384 #ifndef VM_GROWSUP
385 # define VM_GROWSUP VM_NONE
386 #endif
387
388 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
389 # define VM_UFFD_MINOR_BIT 38
390 # define VM_UFFD_MINOR BIT(VM_UFFD_MINOR_BIT) /* UFFD minor faults */
391 #else /* !CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
392 # define VM_UFFD_MINOR VM_NONE
393 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
394
395 /*
396 * This flag is used to connect VFIO to arch specific KVM code. It
397 * indicates that the memory under this VMA is safe for use with any
398 * non-cachable memory type inside KVM. Some VFIO devices, on some
399 * platforms, are thought to be unsafe and can cause machine crashes
400 * if KVM does not lock down the memory type.
401 */
402 #ifdef CONFIG_64BIT
403 #define VM_ALLOW_ANY_UNCACHED_BIT 39
404 #define VM_ALLOW_ANY_UNCACHED BIT(VM_ALLOW_ANY_UNCACHED_BIT)
405 #else
406 #define VM_ALLOW_ANY_UNCACHED VM_NONE
407 #endif
408
409 /* Bits set in the VMA until the stack is in its final location */
410 #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY)
411
412 #define TASK_EXEC ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0)
413
414 /* Common data flag combinations */
415 #define VM_DATA_FLAGS_TSK_EXEC (VM_READ | VM_WRITE | TASK_EXEC | \
416 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
417 #define VM_DATA_FLAGS_NON_EXEC (VM_READ | VM_WRITE | VM_MAYREAD | \
418 VM_MAYWRITE | VM_MAYEXEC)
419 #define VM_DATA_FLAGS_EXEC (VM_READ | VM_WRITE | VM_EXEC | \
420 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
421
422 #ifndef VM_DATA_DEFAULT_FLAGS /* arch can override this */
423 #define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_EXEC
424 #endif
425
426 #ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
427 #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
428 #endif
429
430 #define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK)
431
432 #ifdef CONFIG_STACK_GROWSUP
433 #define VM_STACK VM_GROWSUP
434 #define VM_STACK_EARLY VM_GROWSDOWN
435 #else
436 #define VM_STACK VM_GROWSDOWN
437 #define VM_STACK_EARLY 0
438 #endif
439
440 #define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
441
442 /* VMA basic access permission flags */
443 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
444
445
446 /*
447 * Special vmas that are non-mergable, non-mlock()able.
448 */
449 #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
450
451 /* This mask prevents VMA from being scanned with khugepaged */
452 #define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
453
454 /* This mask defines which mm->def_flags a process can inherit its parent */
455 #define VM_INIT_DEF_MASK VM_NOHUGEPAGE
456
457 /* This mask represents all the VMA flag bits used by mlock */
458 #define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT)
459
460 /* Arch-specific flags to clear when updating VM flags on protection change */
461 #ifndef VM_ARCH_CLEAR
462 # define VM_ARCH_CLEAR VM_NONE
463 #endif
464 #define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR)
465
466 /*
467 * mapping from the currently active vm_flags protection bits (the
468 * low four bits) to a page protection mask..
469 */
470
471 /*
472 * The default fault flags that should be used by most of the
473 * arch-specific page fault handlers.
474 */
475 #define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \
476 FAULT_FLAG_KILLABLE | \
477 FAULT_FLAG_INTERRUPTIBLE)
478
479 /**
480 * fault_flag_allow_retry_first - check ALLOW_RETRY the first time
481 * @flags: Fault flags.
482 *
483 * This is mostly used for places where we want to try to avoid taking
484 * the mmap_lock for too long a time when waiting for another condition
485 * to change, in which case we can try to be polite to release the
486 * mmap_lock in the first round to avoid potential starvation of other
487 * processes that would also want the mmap_lock.
488 *
489 * Return: true if the page fault allows retry and this is the first
490 * attempt of the fault handling; false otherwise.
491 */
fault_flag_allow_retry_first(enum fault_flag flags)492 static inline bool fault_flag_allow_retry_first(enum fault_flag flags)
493 {
494 return (flags & FAULT_FLAG_ALLOW_RETRY) &&
495 (!(flags & FAULT_FLAG_TRIED));
496 }
497
498 #define FAULT_FLAG_TRACE \
499 { FAULT_FLAG_WRITE, "WRITE" }, \
500 { FAULT_FLAG_MKWRITE, "MKWRITE" }, \
501 { FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \
502 { FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \
503 { FAULT_FLAG_KILLABLE, "KILLABLE" }, \
504 { FAULT_FLAG_TRIED, "TRIED" }, \
505 { FAULT_FLAG_USER, "USER" }, \
506 { FAULT_FLAG_REMOTE, "REMOTE" }, \
507 { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \
508 { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \
509 { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" }
510
511 /*
512 * vm_fault is filled by the pagefault handler and passed to the vma's
513 * ->fault function. The vma's ->fault is responsible for returning a bitmask
514 * of VM_FAULT_xxx flags that give details about how the fault was handled.
515 *
516 * MM layer fills up gfp_mask for page allocations but fault handler might
517 * alter it if its implementation requires a different allocation context.
518 *
519 * pgoff should be used in favour of virtual_address, if possible.
520 */
521 struct vm_fault {
522 const struct {
523 struct vm_area_struct *vma; /* Target VMA */
524 gfp_t gfp_mask; /* gfp mask to be used for allocations */
525 pgoff_t pgoff; /* Logical page offset based on vma */
526 unsigned long address; /* Faulting virtual address - masked */
527 unsigned long real_address; /* Faulting virtual address - unmasked */
528 };
529 enum fault_flag flags; /* FAULT_FLAG_xxx flags
530 * XXX: should really be 'const' */
531 pmd_t *pmd; /* Pointer to pmd entry matching
532 * the 'address' */
533 pud_t *pud; /* Pointer to pud entry matching
534 * the 'address'
535 */
536 union {
537 pte_t orig_pte; /* Value of PTE at the time of fault */
538 pmd_t orig_pmd; /* Value of PMD at the time of fault,
539 * used by PMD fault only.
540 */
541 };
542
543 struct page *cow_page; /* Page handler may use for COW fault */
544 struct page *page; /* ->fault handlers should return a
545 * page here, unless VM_FAULT_NOPAGE
546 * is set (which is also implied by
547 * VM_FAULT_ERROR).
548 */
549 /* These three entries are valid only while holding ptl lock */
550 pte_t *pte; /* Pointer to pte entry matching
551 * the 'address'. NULL if the page
552 * table hasn't been allocated.
553 */
554 spinlock_t *ptl; /* Page table lock.
555 * Protects pte page table if 'pte'
556 * is not NULL, otherwise pmd.
557 */
558 pgtable_t prealloc_pte; /* Pre-allocated pte page table.
559 * vm_ops->map_pages() sets up a page
560 * table from atomic context.
561 * do_fault_around() pre-allocates
562 * page table to avoid allocation from
563 * atomic context.
564 */
565 };
566
567 /*
568 * These are the virtual MM functions - opening of an area, closing and
569 * unmapping it (needed to keep files on disk up-to-date etc), pointer
570 * to the functions called when a no-page or a wp-page exception occurs.
571 */
572 struct vm_operations_struct {
573 void (*open)(struct vm_area_struct * area);
574 /**
575 * @close: Called when the VMA is being removed from the MM.
576 * Context: User context. May sleep. Caller holds mmap_lock.
577 */
578 void (*close)(struct vm_area_struct * area);
579 /* Called any time before splitting to check if it's allowed */
580 int (*may_split)(struct vm_area_struct *area, unsigned long addr);
581 int (*mremap)(struct vm_area_struct *area);
582 /*
583 * Called by mprotect() to make driver-specific permission
584 * checks before mprotect() is finalised. The VMA must not
585 * be modified. Returns 0 if mprotect() can proceed.
586 */
587 int (*mprotect)(struct vm_area_struct *vma, unsigned long start,
588 unsigned long end, unsigned long newflags);
589 vm_fault_t (*fault)(struct vm_fault *vmf);
590 vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order);
591 vm_fault_t (*map_pages)(struct vm_fault *vmf,
592 pgoff_t start_pgoff, pgoff_t end_pgoff);
593 unsigned long (*pagesize)(struct vm_area_struct * area);
594
595 /* notification that a previously read-only page is about to become
596 * writable, if an error is returned it will cause a SIGBUS */
597 vm_fault_t (*page_mkwrite)(struct vm_fault *vmf);
598
599 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
600 vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf);
601
602 /* called by access_process_vm when get_user_pages() fails, typically
603 * for use by special VMAs. See also generic_access_phys() for a generic
604 * implementation useful for any iomem mapping.
605 */
606 int (*access)(struct vm_area_struct *vma, unsigned long addr,
607 void *buf, int len, int write);
608
609 /* Called by the /proc/PID/maps code to ask the vma whether it
610 * has a special name. Returning non-NULL will also cause this
611 * vma to be dumped unconditionally. */
612 const char *(*name)(struct vm_area_struct *vma);
613
614 #ifdef CONFIG_NUMA
615 /*
616 * set_policy() op must add a reference to any non-NULL @new mempolicy
617 * to hold the policy upon return. Caller should pass NULL @new to
618 * remove a policy and fall back to surrounding context--i.e. do not
619 * install a MPOL_DEFAULT policy, nor the task or system default
620 * mempolicy.
621 */
622 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
623
624 /*
625 * get_policy() op must add reference [mpol_get()] to any policy at
626 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
627 * in mm/mempolicy.c will do this automatically.
628 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
629 * marked as MPOL_SHARED. vma policies are protected by the mmap_lock.
630 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
631 * must return NULL--i.e., do not "fallback" to task or system default
632 * policy.
633 */
634 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
635 unsigned long addr, pgoff_t *ilx);
636 #endif
637 /*
638 * Called by vm_normal_page() for special PTEs to find the
639 * page for @addr. This is useful if the default behavior
640 * (using pte_page()) would not find the correct page.
641 */
642 struct page *(*find_special_page)(struct vm_area_struct *vma,
643 unsigned long addr);
644 };
645
646 #ifdef CONFIG_NUMA_BALANCING
vma_numab_state_init(struct vm_area_struct * vma)647 static inline void vma_numab_state_init(struct vm_area_struct *vma)
648 {
649 vma->numab_state = NULL;
650 }
vma_numab_state_free(struct vm_area_struct * vma)651 static inline void vma_numab_state_free(struct vm_area_struct *vma)
652 {
653 kfree(vma->numab_state);
654 }
655 #else
vma_numab_state_init(struct vm_area_struct * vma)656 static inline void vma_numab_state_init(struct vm_area_struct *vma) {}
vma_numab_state_free(struct vm_area_struct * vma)657 static inline void vma_numab_state_free(struct vm_area_struct *vma) {}
658 #endif /* CONFIG_NUMA_BALANCING */
659
660 #ifdef CONFIG_PER_VMA_LOCK
661 /*
662 * Try to read-lock a vma. The function is allowed to occasionally yield false
663 * locked result to avoid performance overhead, in which case we fall back to
664 * using mmap_lock. The function should never yield false unlocked result.
665 */
vma_start_read(struct vm_area_struct * vma)666 static inline bool vma_start_read(struct vm_area_struct *vma)
667 {
668 /*
669 * Check before locking. A race might cause false locked result.
670 * We can use READ_ONCE() for the mm_lock_seq here, and don't need
671 * ACQUIRE semantics, because this is just a lockless check whose result
672 * we don't rely on for anything - the mm_lock_seq read against which we
673 * need ordering is below.
674 */
675 if (READ_ONCE(vma->vm_lock_seq) == READ_ONCE(vma->vm_mm->mm_lock_seq))
676 return false;
677
678 if (unlikely(down_read_trylock(&vma->vm_lock->lock) == 0))
679 return false;
680
681 /*
682 * Overflow might produce false locked result.
683 * False unlocked result is impossible because we modify and check
684 * vma->vm_lock_seq under vma->vm_lock protection and mm->mm_lock_seq
685 * modification invalidates all existing locks.
686 *
687 * We must use ACQUIRE semantics for the mm_lock_seq so that if we are
688 * racing with vma_end_write_all(), we only start reading from the VMA
689 * after it has been unlocked.
690 * This pairs with RELEASE semantics in vma_end_write_all().
691 */
692 if (unlikely(vma->vm_lock_seq == smp_load_acquire(&vma->vm_mm->mm_lock_seq))) {
693 up_read(&vma->vm_lock->lock);
694 return false;
695 }
696 return true;
697 }
698
vma_end_read(struct vm_area_struct * vma)699 static inline void vma_end_read(struct vm_area_struct *vma)
700 {
701 rcu_read_lock(); /* keeps vma alive till the end of up_read */
702 up_read(&vma->vm_lock->lock);
703 rcu_read_unlock();
704 }
705
706 /* WARNING! Can only be used if mmap_lock is expected to be write-locked */
__is_vma_write_locked(struct vm_area_struct * vma,int * mm_lock_seq)707 static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)
708 {
709 mmap_assert_write_locked(vma->vm_mm);
710
711 /*
712 * current task is holding mmap_write_lock, both vma->vm_lock_seq and
713 * mm->mm_lock_seq can't be concurrently modified.
714 */
715 *mm_lock_seq = vma->vm_mm->mm_lock_seq;
716 return (vma->vm_lock_seq == *mm_lock_seq);
717 }
718
719 /*
720 * Begin writing to a VMA.
721 * Exclude concurrent readers under the per-VMA lock until the currently
722 * write-locked mmap_lock is dropped or downgraded.
723 */
vma_start_write(struct vm_area_struct * vma)724 static inline void vma_start_write(struct vm_area_struct *vma)
725 {
726 int mm_lock_seq;
727
728 if (__is_vma_write_locked(vma, &mm_lock_seq))
729 return;
730
731 down_write(&vma->vm_lock->lock);
732 /*
733 * We should use WRITE_ONCE() here because we can have concurrent reads
734 * from the early lockless pessimistic check in vma_start_read().
735 * We don't really care about the correctness of that early check, but
736 * we should use WRITE_ONCE() for cleanliness and to keep KCSAN happy.
737 */
738 WRITE_ONCE(vma->vm_lock_seq, mm_lock_seq);
739 up_write(&vma->vm_lock->lock);
740 }
741
vma_assert_write_locked(struct vm_area_struct * vma)742 static inline void vma_assert_write_locked(struct vm_area_struct *vma)
743 {
744 int mm_lock_seq;
745
746 VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma);
747 }
748
vma_assert_locked(struct vm_area_struct * vma)749 static inline void vma_assert_locked(struct vm_area_struct *vma)
750 {
751 if (!rwsem_is_locked(&vma->vm_lock->lock))
752 vma_assert_write_locked(vma);
753 }
754
vma_mark_detached(struct vm_area_struct * vma,bool detached)755 static inline void vma_mark_detached(struct vm_area_struct *vma, bool detached)
756 {
757 /* When detaching vma should be write-locked */
758 if (detached)
759 vma_assert_write_locked(vma);
760 vma->detached = detached;
761 }
762
release_fault_lock(struct vm_fault * vmf)763 static inline void release_fault_lock(struct vm_fault *vmf)
764 {
765 if (vmf->flags & FAULT_FLAG_VMA_LOCK)
766 vma_end_read(vmf->vma);
767 else
768 mmap_read_unlock(vmf->vma->vm_mm);
769 }
770
assert_fault_locked(struct vm_fault * vmf)771 static inline void assert_fault_locked(struct vm_fault *vmf)
772 {
773 if (vmf->flags & FAULT_FLAG_VMA_LOCK)
774 vma_assert_locked(vmf->vma);
775 else
776 mmap_assert_locked(vmf->vma->vm_mm);
777 }
778
779 struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
780 unsigned long address);
781
782 #else /* CONFIG_PER_VMA_LOCK */
783
vma_start_read(struct vm_area_struct * vma)784 static inline bool vma_start_read(struct vm_area_struct *vma)
785 { return false; }
vma_end_read(struct vm_area_struct * vma)786 static inline void vma_end_read(struct vm_area_struct *vma) {}
vma_start_write(struct vm_area_struct * vma)787 static inline void vma_start_write(struct vm_area_struct *vma) {}
vma_assert_write_locked(struct vm_area_struct * vma)788 static inline void vma_assert_write_locked(struct vm_area_struct *vma)
789 { mmap_assert_write_locked(vma->vm_mm); }
vma_mark_detached(struct vm_area_struct * vma,bool detached)790 static inline void vma_mark_detached(struct vm_area_struct *vma,
791 bool detached) {}
792
lock_vma_under_rcu(struct mm_struct * mm,unsigned long address)793 static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
794 unsigned long address)
795 {
796 return NULL;
797 }
798
vma_assert_locked(struct vm_area_struct * vma)799 static inline void vma_assert_locked(struct vm_area_struct *vma)
800 {
801 mmap_assert_locked(vma->vm_mm);
802 }
803
release_fault_lock(struct vm_fault * vmf)804 static inline void release_fault_lock(struct vm_fault *vmf)
805 {
806 mmap_read_unlock(vmf->vma->vm_mm);
807 }
808
assert_fault_locked(struct vm_fault * vmf)809 static inline void assert_fault_locked(struct vm_fault *vmf)
810 {
811 mmap_assert_locked(vmf->vma->vm_mm);
812 }
813
814 #endif /* CONFIG_PER_VMA_LOCK */
815
816 extern const struct vm_operations_struct vma_dummy_vm_ops;
817
818 /*
819 * WARNING: vma_init does not initialize vma->vm_lock.
820 * Use vm_area_alloc()/vm_area_free() if vma needs locking.
821 */
vma_init(struct vm_area_struct * vma,struct mm_struct * mm)822 static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm)
823 {
824 memset(vma, 0, sizeof(*vma));
825 vma->vm_mm = mm;
826 vma->vm_ops = &vma_dummy_vm_ops;
827 INIT_LIST_HEAD(&vma->anon_vma_chain);
828 vma_mark_detached(vma, false);
829 vma_numab_state_init(vma);
830 }
831
832 /* Use when VMA is not part of the VMA tree and needs no locking */
vm_flags_init(struct vm_area_struct * vma,vm_flags_t flags)833 static inline void vm_flags_init(struct vm_area_struct *vma,
834 vm_flags_t flags)
835 {
836 ACCESS_PRIVATE(vma, __vm_flags) = flags;
837 }
838
839 /*
840 * Use when VMA is part of the VMA tree and modifications need coordination
841 * Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and
842 * it should be locked explicitly beforehand.
843 */
vm_flags_reset(struct vm_area_struct * vma,vm_flags_t flags)844 static inline void vm_flags_reset(struct vm_area_struct *vma,
845 vm_flags_t flags)
846 {
847 vma_assert_write_locked(vma);
848 vm_flags_init(vma, flags);
849 }
850
vm_flags_reset_once(struct vm_area_struct * vma,vm_flags_t flags)851 static inline void vm_flags_reset_once(struct vm_area_struct *vma,
852 vm_flags_t flags)
853 {
854 vma_assert_write_locked(vma);
855 WRITE_ONCE(ACCESS_PRIVATE(vma, __vm_flags), flags);
856 }
857
vm_flags_set(struct vm_area_struct * vma,vm_flags_t flags)858 static inline void vm_flags_set(struct vm_area_struct *vma,
859 vm_flags_t flags)
860 {
861 vma_start_write(vma);
862 ACCESS_PRIVATE(vma, __vm_flags) |= flags;
863 }
864
vm_flags_clear(struct vm_area_struct * vma,vm_flags_t flags)865 static inline void vm_flags_clear(struct vm_area_struct *vma,
866 vm_flags_t flags)
867 {
868 vma_start_write(vma);
869 ACCESS_PRIVATE(vma, __vm_flags) &= ~flags;
870 }
871
872 /*
873 * Use only if VMA is not part of the VMA tree or has no other users and
874 * therefore needs no locking.
875 */
__vm_flags_mod(struct vm_area_struct * vma,vm_flags_t set,vm_flags_t clear)876 static inline void __vm_flags_mod(struct vm_area_struct *vma,
877 vm_flags_t set, vm_flags_t clear)
878 {
879 vm_flags_init(vma, (vma->vm_flags | set) & ~clear);
880 }
881
882 /*
883 * Use only when the order of set/clear operations is unimportant, otherwise
884 * use vm_flags_{set|clear} explicitly.
885 */
vm_flags_mod(struct vm_area_struct * vma,vm_flags_t set,vm_flags_t clear)886 static inline void vm_flags_mod(struct vm_area_struct *vma,
887 vm_flags_t set, vm_flags_t clear)
888 {
889 vma_start_write(vma);
890 __vm_flags_mod(vma, set, clear);
891 }
892
vma_set_anonymous(struct vm_area_struct * vma)893 static inline void vma_set_anonymous(struct vm_area_struct *vma)
894 {
895 vma->vm_ops = NULL;
896 }
897
vma_is_anonymous(struct vm_area_struct * vma)898 static inline bool vma_is_anonymous(struct vm_area_struct *vma)
899 {
900 return !vma->vm_ops;
901 }
902
903 /*
904 * Indicate if the VMA is a heap for the given task; for
905 * /proc/PID/maps that is the heap of the main task.
906 */
vma_is_initial_heap(const struct vm_area_struct * vma)907 static inline bool vma_is_initial_heap(const struct vm_area_struct *vma)
908 {
909 return vma->vm_start < vma->vm_mm->brk &&
910 vma->vm_end > vma->vm_mm->start_brk;
911 }
912
913 /*
914 * Indicate if the VMA is a stack for the given task; for
915 * /proc/PID/maps that is the stack of the main task.
916 */
vma_is_initial_stack(const struct vm_area_struct * vma)917 static inline bool vma_is_initial_stack(const struct vm_area_struct *vma)
918 {
919 /*
920 * We make no effort to guess what a given thread considers to be
921 * its "stack". It's not even well-defined for programs written
922 * languages like Go.
923 */
924 return vma->vm_start <= vma->vm_mm->start_stack &&
925 vma->vm_end >= vma->vm_mm->start_stack;
926 }
927
vma_is_temporary_stack(struct vm_area_struct * vma)928 static inline bool vma_is_temporary_stack(struct vm_area_struct *vma)
929 {
930 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
931
932 if (!maybe_stack)
933 return false;
934
935 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
936 VM_STACK_INCOMPLETE_SETUP)
937 return true;
938
939 return false;
940 }
941
vma_is_foreign(struct vm_area_struct * vma)942 static inline bool vma_is_foreign(struct vm_area_struct *vma)
943 {
944 if (!current->mm)
945 return true;
946
947 if (current->mm != vma->vm_mm)
948 return true;
949
950 return false;
951 }
952
vma_is_accessible(struct vm_area_struct * vma)953 static inline bool vma_is_accessible(struct vm_area_struct *vma)
954 {
955 return vma->vm_flags & VM_ACCESS_FLAGS;
956 }
957
is_shared_maywrite(vm_flags_t vm_flags)958 static inline bool is_shared_maywrite(vm_flags_t vm_flags)
959 {
960 return (vm_flags & (VM_SHARED | VM_MAYWRITE)) ==
961 (VM_SHARED | VM_MAYWRITE);
962 }
963
vma_is_shared_maywrite(struct vm_area_struct * vma)964 static inline bool vma_is_shared_maywrite(struct vm_area_struct *vma)
965 {
966 return is_shared_maywrite(vma->vm_flags);
967 }
968
969 static inline
vma_find(struct vma_iterator * vmi,unsigned long max)970 struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max)
971 {
972 return mas_find(&vmi->mas, max - 1);
973 }
974
vma_next(struct vma_iterator * vmi)975 static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi)
976 {
977 /*
978 * Uses mas_find() to get the first VMA when the iterator starts.
979 * Calling mas_next() could skip the first entry.
980 */
981 return mas_find(&vmi->mas, ULONG_MAX);
982 }
983
984 static inline
vma_iter_next_range(struct vma_iterator * vmi)985 struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi)
986 {
987 return mas_next_range(&vmi->mas, ULONG_MAX);
988 }
989
990
vma_prev(struct vma_iterator * vmi)991 static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi)
992 {
993 return mas_prev(&vmi->mas, 0);
994 }
995
996 static inline
vma_iter_prev_range(struct vma_iterator * vmi)997 struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi)
998 {
999 return mas_prev_range(&vmi->mas, 0);
1000 }
1001
vma_iter_addr(struct vma_iterator * vmi)1002 static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
1003 {
1004 return vmi->mas.index;
1005 }
1006
vma_iter_end(struct vma_iterator * vmi)1007 static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
1008 {
1009 return vmi->mas.last + 1;
1010 }
vma_iter_bulk_alloc(struct vma_iterator * vmi,unsigned long count)1011 static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
1012 unsigned long count)
1013 {
1014 return mas_expected_entries(&vmi->mas, count);
1015 }
1016
vma_iter_clear_gfp(struct vma_iterator * vmi,unsigned long start,unsigned long end,gfp_t gfp)1017 static inline int vma_iter_clear_gfp(struct vma_iterator *vmi,
1018 unsigned long start, unsigned long end, gfp_t gfp)
1019 {
1020 __mas_set_range(&vmi->mas, start, end - 1);
1021 mas_store_gfp(&vmi->mas, NULL, gfp);
1022 if (unlikely(mas_is_err(&vmi->mas)))
1023 return -ENOMEM;
1024
1025 return 0;
1026 }
1027
1028 /* Free any unused preallocations */
vma_iter_free(struct vma_iterator * vmi)1029 static inline void vma_iter_free(struct vma_iterator *vmi)
1030 {
1031 mas_destroy(&vmi->mas);
1032 }
1033
vma_iter_bulk_store(struct vma_iterator * vmi,struct vm_area_struct * vma)1034 static inline int vma_iter_bulk_store(struct vma_iterator *vmi,
1035 struct vm_area_struct *vma)
1036 {
1037 vmi->mas.index = vma->vm_start;
1038 vmi->mas.last = vma->vm_end - 1;
1039 mas_store(&vmi->mas, vma);
1040 if (unlikely(mas_is_err(&vmi->mas)))
1041 return -ENOMEM;
1042
1043 return 0;
1044 }
1045
vma_iter_invalidate(struct vma_iterator * vmi)1046 static inline void vma_iter_invalidate(struct vma_iterator *vmi)
1047 {
1048 mas_pause(&vmi->mas);
1049 }
1050
vma_iter_set(struct vma_iterator * vmi,unsigned long addr)1051 static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr)
1052 {
1053 mas_set(&vmi->mas, addr);
1054 }
1055
1056 #define for_each_vma(__vmi, __vma) \
1057 while (((__vma) = vma_next(&(__vmi))) != NULL)
1058
1059 /* The MM code likes to work with exclusive end addresses */
1060 #define for_each_vma_range(__vmi, __vma, __end) \
1061 while (((__vma) = vma_find(&(__vmi), (__end))) != NULL)
1062
1063 #ifdef CONFIG_SHMEM
1064 /*
1065 * The vma_is_shmem is not inline because it is used only by slow
1066 * paths in userfault.
1067 */
1068 bool vma_is_shmem(struct vm_area_struct *vma);
1069 bool vma_is_anon_shmem(struct vm_area_struct *vma);
1070 #else
vma_is_shmem(struct vm_area_struct * vma)1071 static inline bool vma_is_shmem(struct vm_area_struct *vma) { return false; }
vma_is_anon_shmem(struct vm_area_struct * vma)1072 static inline bool vma_is_anon_shmem(struct vm_area_struct *vma) { return false; }
1073 #endif
1074
1075 int vma_is_stack_for_current(struct vm_area_struct *vma);
1076
1077 /* flush_tlb_range() takes a vma, not a mm, and can care about flags */
1078 #define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) }
1079
1080 struct mmu_gather;
1081 struct inode;
1082
1083 /*
1084 * compound_order() can be called without holding a reference, which means
1085 * that niceties like page_folio() don't work. These callers should be
1086 * prepared to handle wild return values. For example, PG_head may be
1087 * set before the order is initialised, or this may be a tail page.
1088 * See compaction.c for some good examples.
1089 */
compound_order(struct page * page)1090 static inline unsigned int compound_order(struct page *page)
1091 {
1092 struct folio *folio = (struct folio *)page;
1093
1094 if (!test_bit(PG_head, &folio->flags))
1095 return 0;
1096 return folio->_flags_1 & 0xff;
1097 }
1098
1099 /**
1100 * folio_order - The allocation order of a folio.
1101 * @folio: The folio.
1102 *
1103 * A folio is composed of 2^order pages. See get_order() for the definition
1104 * of order.
1105 *
1106 * Return: The order of the folio.
1107 */
folio_order(struct folio * folio)1108 static inline unsigned int folio_order(struct folio *folio)
1109 {
1110 if (!folio_test_large(folio))
1111 return 0;
1112 return folio->_flags_1 & 0xff;
1113 }
1114
1115 #include <linux/huge_mm.h>
1116
1117 /*
1118 * Methods to modify the page usage count.
1119 *
1120 * What counts for a page usage:
1121 * - cache mapping (page->mapping)
1122 * - private data (page->private)
1123 * - page mapped in a task's page tables, each mapping
1124 * is counted separately
1125 *
1126 * Also, many kernel routines increase the page count before a critical
1127 * routine so they can be sure the page doesn't go away from under them.
1128 */
1129
1130 /*
1131 * Drop a ref, return true if the refcount fell to zero (the page has no users)
1132 */
put_page_testzero(struct page * page)1133 static inline int put_page_testzero(struct page *page)
1134 {
1135 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
1136 return page_ref_dec_and_test(page);
1137 }
1138
folio_put_testzero(struct folio * folio)1139 static inline int folio_put_testzero(struct folio *folio)
1140 {
1141 return put_page_testzero(&folio->page);
1142 }
1143
1144 /*
1145 * Try to grab a ref unless the page has a refcount of zero, return false if
1146 * that is the case.
1147 * This can be called when MMU is off so it must not access
1148 * any of the virtual mappings.
1149 */
get_page_unless_zero(struct page * page)1150 static inline bool get_page_unless_zero(struct page *page)
1151 {
1152 return page_ref_add_unless(page, 1, 0);
1153 }
1154
folio_get_nontail_page(struct page * page)1155 static inline struct folio *folio_get_nontail_page(struct page *page)
1156 {
1157 if (unlikely(!get_page_unless_zero(page)))
1158 return NULL;
1159 return (struct folio *)page;
1160 }
1161
1162 extern int page_is_ram(unsigned long pfn);
1163
1164 enum {
1165 REGION_INTERSECTS,
1166 REGION_DISJOINT,
1167 REGION_MIXED,
1168 };
1169
1170 int region_intersects(resource_size_t offset, size_t size, unsigned long flags,
1171 unsigned long desc);
1172
1173 /* Support for virtually mapped pages */
1174 struct page *vmalloc_to_page(const void *addr);
1175 unsigned long vmalloc_to_pfn(const void *addr);
1176
1177 /*
1178 * Determine if an address is within the vmalloc range
1179 *
1180 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
1181 * is no special casing required.
1182 */
1183 #ifdef CONFIG_MMU
1184 extern bool is_vmalloc_addr(const void *x);
1185 extern int is_vmalloc_or_module_addr(const void *x);
1186 #else
is_vmalloc_addr(const void * x)1187 static inline bool is_vmalloc_addr(const void *x)
1188 {
1189 return false;
1190 }
is_vmalloc_or_module_addr(const void * x)1191 static inline int is_vmalloc_or_module_addr(const void *x)
1192 {
1193 return 0;
1194 }
1195 #endif
1196
1197 /*
1198 * How many times the entire folio is mapped as a single unit (eg by a
1199 * PMD or PUD entry). This is probably not what you want, except for
1200 * debugging purposes - it does not include PTE-mapped sub-pages; look
1201 * at folio_mapcount() or page_mapcount() instead.
1202 */
folio_entire_mapcount(const struct folio * folio)1203 static inline int folio_entire_mapcount(const struct folio *folio)
1204 {
1205 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
1206 return atomic_read(&folio->_entire_mapcount) + 1;
1207 }
1208
1209 /*
1210 * The atomic page->_mapcount, starts from -1: so that transitions
1211 * both from it and to it can be tracked, using atomic_inc_and_test
1212 * and atomic_add_negative(-1).
1213 */
page_mapcount_reset(struct page * page)1214 static inline void page_mapcount_reset(struct page *page)
1215 {
1216 atomic_set(&(page)->_mapcount, -1);
1217 }
1218
1219 /**
1220 * page_mapcount() - Number of times this precise page is mapped.
1221 * @page: The page.
1222 *
1223 * The number of times this page is mapped. If this page is part of
1224 * a large folio, it includes the number of times this page is mapped
1225 * as part of that folio.
1226 *
1227 * Will report 0 for pages which cannot be mapped into userspace, eg
1228 * slab, page tables and similar.
1229 */
page_mapcount(struct page * page)1230 static inline int page_mapcount(struct page *page)
1231 {
1232 int mapcount = atomic_read(&page->_mapcount) + 1;
1233
1234 /* Handle page_has_type() pages */
1235 if (mapcount < PAGE_MAPCOUNT_RESERVE + 1)
1236 mapcount = 0;
1237 if (unlikely(PageCompound(page)))
1238 mapcount += folio_entire_mapcount(page_folio(page));
1239
1240 return mapcount;
1241 }
1242
folio_large_mapcount(const struct folio * folio)1243 static inline int folio_large_mapcount(const struct folio *folio)
1244 {
1245 VM_WARN_ON_FOLIO(!folio_test_large(folio), folio);
1246 return atomic_read(&folio->_large_mapcount) + 1;
1247 }
1248
1249 /**
1250 * folio_mapcount() - Number of mappings of this folio.
1251 * @folio: The folio.
1252 *
1253 * The folio mapcount corresponds to the number of present user page table
1254 * entries that reference any part of a folio. Each such present user page
1255 * table entry must be paired with exactly on folio reference.
1256 *
1257 * For ordindary folios, each user page table entry (PTE/PMD/PUD/...) counts
1258 * exactly once.
1259 *
1260 * For hugetlb folios, each abstracted "hugetlb" user page table entry that
1261 * references the entire folio counts exactly once, even when such special
1262 * page table entries are comprised of multiple ordinary page table entries.
1263 *
1264 * Will report 0 for pages which cannot be mapped into userspace, such as
1265 * slab, page tables and similar.
1266 *
1267 * Return: The number of times this folio is mapped.
1268 */
folio_mapcount(const struct folio * folio)1269 static inline int folio_mapcount(const struct folio *folio)
1270 {
1271 int mapcount;
1272
1273 if (likely(!folio_test_large(folio))) {
1274 mapcount = atomic_read(&folio->_mapcount) + 1;
1275 /* Handle page_has_type() pages */
1276 if (mapcount < PAGE_MAPCOUNT_RESERVE + 1)
1277 mapcount = 0;
1278 return mapcount;
1279 }
1280 return folio_large_mapcount(folio);
1281 }
1282
1283 /**
1284 * folio_mapped - Is this folio mapped into userspace?
1285 * @folio: The folio.
1286 *
1287 * Return: True if any page in this folio is referenced by user page tables.
1288 */
folio_mapped(const struct folio * folio)1289 static inline bool folio_mapped(const struct folio *folio)
1290 {
1291 return folio_mapcount(folio) >= 1;
1292 }
1293
1294 /*
1295 * Return true if this page is mapped into pagetables.
1296 * For compound page it returns true if any sub-page of compound page is mapped,
1297 * even if this particular sub-page is not itself mapped by any PTE or PMD.
1298 */
page_mapped(const struct page * page)1299 static inline bool page_mapped(const struct page *page)
1300 {
1301 return folio_mapped(page_folio(page));
1302 }
1303
virt_to_head_page(const void * x)1304 static inline struct page *virt_to_head_page(const void *x)
1305 {
1306 struct page *page = virt_to_page(x);
1307
1308 return compound_head(page);
1309 }
1310
virt_to_folio(const void * x)1311 static inline struct folio *virt_to_folio(const void *x)
1312 {
1313 struct page *page = virt_to_page(x);
1314
1315 return page_folio(page);
1316 }
1317
1318 void __folio_put(struct folio *folio);
1319
1320 void put_pages_list(struct list_head *pages);
1321
1322 void split_page(struct page *page, unsigned int order);
1323 void folio_copy(struct folio *dst, struct folio *src);
1324
1325 unsigned long nr_free_buffer_pages(void);
1326
1327 /* Returns the number of bytes in this potentially compound page. */
page_size(struct page * page)1328 static inline unsigned long page_size(struct page *page)
1329 {
1330 return PAGE_SIZE << compound_order(page);
1331 }
1332
1333 /* Returns the number of bits needed for the number of bytes in a page */
page_shift(struct page * page)1334 static inline unsigned int page_shift(struct page *page)
1335 {
1336 return PAGE_SHIFT + compound_order(page);
1337 }
1338
1339 /**
1340 * thp_order - Order of a transparent huge page.
1341 * @page: Head page of a transparent huge page.
1342 */
thp_order(struct page * page)1343 static inline unsigned int thp_order(struct page *page)
1344 {
1345 VM_BUG_ON_PGFLAGS(PageTail(page), page);
1346 return compound_order(page);
1347 }
1348
1349 /**
1350 * thp_size - Size of a transparent huge page.
1351 * @page: Head page of a transparent huge page.
1352 *
1353 * Return: Number of bytes in this page.
1354 */
thp_size(struct page * page)1355 static inline unsigned long thp_size(struct page *page)
1356 {
1357 return PAGE_SIZE << thp_order(page);
1358 }
1359
1360 #ifdef CONFIG_MMU
1361 /*
1362 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
1363 * servicing faults for write access. In the normal case, do always want
1364 * pte_mkwrite. But get_user_pages can cause write faults for mappings
1365 * that do not have writing enabled, when used by access_process_vm.
1366 */
maybe_mkwrite(pte_t pte,struct vm_area_struct * vma)1367 static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
1368 {
1369 if (likely(vma->vm_flags & VM_WRITE))
1370 pte = pte_mkwrite(pte, vma);
1371 return pte;
1372 }
1373
1374 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page);
1375 void set_pte_range(struct vm_fault *vmf, struct folio *folio,
1376 struct page *page, unsigned int nr, unsigned long addr);
1377
1378 vm_fault_t finish_fault(struct vm_fault *vmf);
1379 #endif
1380
1381 /*
1382 * Multiple processes may "see" the same page. E.g. for untouched
1383 * mappings of /dev/null, all processes see the same page full of
1384 * zeroes, and text pages of executables and shared libraries have
1385 * only one copy in memory, at most, normally.
1386 *
1387 * For the non-reserved pages, page_count(page) denotes a reference count.
1388 * page_count() == 0 means the page is free. page->lru is then used for
1389 * freelist management in the buddy allocator.
1390 * page_count() > 0 means the page has been allocated.
1391 *
1392 * Pages are allocated by the slab allocator in order to provide memory
1393 * to kmalloc and kmem_cache_alloc. In this case, the management of the
1394 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
1395 * unless a particular usage is carefully commented. (the responsibility of
1396 * freeing the kmalloc memory is the caller's, of course).
1397 *
1398 * A page may be used by anyone else who does a __get_free_page().
1399 * In this case, page_count still tracks the references, and should only
1400 * be used through the normal accessor functions. The top bits of page->flags
1401 * and page->virtual store page management information, but all other fields
1402 * are unused and could be used privately, carefully. The management of this
1403 * page is the responsibility of the one who allocated it, and those who have
1404 * subsequently been given references to it.
1405 *
1406 * The other pages (we may call them "pagecache pages") are completely
1407 * managed by the Linux memory manager: I/O, buffers, swapping etc.
1408 * The following discussion applies only to them.
1409 *
1410 * A pagecache page contains an opaque `private' member, which belongs to the
1411 * page's address_space. Usually, this is the address of a circular list of
1412 * the page's disk buffers. PG_private must be set to tell the VM to call
1413 * into the filesystem to release these pages.
1414 *
1415 * A page may belong to an inode's memory mapping. In this case, page->mapping
1416 * is the pointer to the inode, and page->index is the file offset of the page,
1417 * in units of PAGE_SIZE.
1418 *
1419 * If pagecache pages are not associated with an inode, they are said to be
1420 * anonymous pages. These may become associated with the swapcache, and in that
1421 * case PG_swapcache is set, and page->private is an offset into the swapcache.
1422 *
1423 * In either case (swapcache or inode backed), the pagecache itself holds one
1424 * reference to the page. Setting PG_private should also increment the
1425 * refcount. The each user mapping also has a reference to the page.
1426 *
1427 * The pagecache pages are stored in a per-mapping radix tree, which is
1428 * rooted at mapping->i_pages, and indexed by offset.
1429 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
1430 * lists, we instead now tag pages as dirty/writeback in the radix tree.
1431 *
1432 * All pagecache pages may be subject to I/O:
1433 * - inode pages may need to be read from disk,
1434 * - inode pages which have been modified and are MAP_SHARED may need
1435 * to be written back to the inode on disk,
1436 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
1437 * modified may need to be swapped out to swap space and (later) to be read
1438 * back into memory.
1439 */
1440
1441 #if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX)
1442 DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
1443
1444 bool __put_devmap_managed_folio_refs(struct folio *folio, int refs);
put_devmap_managed_folio_refs(struct folio * folio,int refs)1445 static inline bool put_devmap_managed_folio_refs(struct folio *folio, int refs)
1446 {
1447 if (!static_branch_unlikely(&devmap_managed_key))
1448 return false;
1449 if (!folio_is_zone_device(folio))
1450 return false;
1451 return __put_devmap_managed_folio_refs(folio, refs);
1452 }
1453 #else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
put_devmap_managed_folio_refs(struct folio * folio,int refs)1454 static inline bool put_devmap_managed_folio_refs(struct folio *folio, int refs)
1455 {
1456 return false;
1457 }
1458 #endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
1459
1460 /* 127: arbitrary random number, small enough to assemble well */
1461 #define folio_ref_zero_or_close_to_overflow(folio) \
1462 ((unsigned int) folio_ref_count(folio) + 127u <= 127u)
1463
1464 /**
1465 * folio_get - Increment the reference count on a folio.
1466 * @folio: The folio.
1467 *
1468 * Context: May be called in any context, as long as you know that
1469 * you have a refcount on the folio. If you do not already have one,
1470 * folio_try_get() may be the right interface for you to use.
1471 */
folio_get(struct folio * folio)1472 static inline void folio_get(struct folio *folio)
1473 {
1474 VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio);
1475 folio_ref_inc(folio);
1476 }
1477
get_page(struct page * page)1478 static inline void get_page(struct page *page)
1479 {
1480 folio_get(page_folio(page));
1481 }
1482
try_get_page(struct page * page)1483 static inline __must_check bool try_get_page(struct page *page)
1484 {
1485 page = compound_head(page);
1486 if (WARN_ON_ONCE(page_ref_count(page) <= 0))
1487 return false;
1488 page_ref_inc(page);
1489 return true;
1490 }
1491
1492 /**
1493 * folio_put - Decrement the reference count on a folio.
1494 * @folio: The folio.
1495 *
1496 * If the folio's reference count reaches zero, the memory will be
1497 * released back to the page allocator and may be used by another
1498 * allocation immediately. Do not access the memory or the struct folio
1499 * after calling folio_put() unless you can be sure that it wasn't the
1500 * last reference.
1501 *
1502 * Context: May be called in process or interrupt context, but not in NMI
1503 * context. May be called while holding a spinlock.
1504 */
folio_put(struct folio * folio)1505 static inline void folio_put(struct folio *folio)
1506 {
1507 if (folio_put_testzero(folio))
1508 __folio_put(folio);
1509 }
1510
1511 /**
1512 * folio_put_refs - Reduce the reference count on a folio.
1513 * @folio: The folio.
1514 * @refs: The amount to subtract from the folio's reference count.
1515 *
1516 * If the folio's reference count reaches zero, the memory will be
1517 * released back to the page allocator and may be used by another
1518 * allocation immediately. Do not access the memory or the struct folio
1519 * after calling folio_put_refs() unless you can be sure that these weren't
1520 * the last references.
1521 *
1522 * Context: May be called in process or interrupt context, but not in NMI
1523 * context. May be called while holding a spinlock.
1524 */
folio_put_refs(struct folio * folio,int refs)1525 static inline void folio_put_refs(struct folio *folio, int refs)
1526 {
1527 if (folio_ref_sub_and_test(folio, refs))
1528 __folio_put(folio);
1529 }
1530
1531 void folios_put_refs(struct folio_batch *folios, unsigned int *refs);
1532
1533 /*
1534 * union release_pages_arg - an array of pages or folios
1535 *
1536 * release_pages() releases a simple array of multiple pages, and
1537 * accepts various different forms of said page array: either
1538 * a regular old boring array of pages, an array of folios, or
1539 * an array of encoded page pointers.
1540 *
1541 * The transparent union syntax for this kind of "any of these
1542 * argument types" is all kinds of ugly, so look away.
1543 */
1544 typedef union {
1545 struct page **pages;
1546 struct folio **folios;
1547 struct encoded_page **encoded_pages;
1548 } release_pages_arg __attribute__ ((__transparent_union__));
1549
1550 void release_pages(release_pages_arg, int nr);
1551
1552 /**
1553 * folios_put - Decrement the reference count on an array of folios.
1554 * @folios: The folios.
1555 *
1556 * Like folio_put(), but for a batch of folios. This is more efficient
1557 * than writing the loop yourself as it will optimise the locks which need
1558 * to be taken if the folios are freed. The folios batch is returned
1559 * empty and ready to be reused for another batch; there is no need to
1560 * reinitialise it.
1561 *
1562 * Context: May be called in process or interrupt context, but not in NMI
1563 * context. May be called while holding a spinlock.
1564 */
folios_put(struct folio_batch * folios)1565 static inline void folios_put(struct folio_batch *folios)
1566 {
1567 folios_put_refs(folios, NULL);
1568 }
1569
put_page(struct page * page)1570 static inline void put_page(struct page *page)
1571 {
1572 struct folio *folio = page_folio(page);
1573
1574 /*
1575 * For some devmap managed pages we need to catch refcount transition
1576 * from 2 to 1:
1577 */
1578 if (put_devmap_managed_folio_refs(folio, 1))
1579 return;
1580 folio_put(folio);
1581 }
1582
1583 /*
1584 * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload
1585 * the page's refcount so that two separate items are tracked: the original page
1586 * reference count, and also a new count of how many pin_user_pages() calls were
1587 * made against the page. ("gup-pinned" is another term for the latter).
1588 *
1589 * With this scheme, pin_user_pages() becomes special: such pages are marked as
1590 * distinct from normal pages. As such, the unpin_user_page() call (and its
1591 * variants) must be used in order to release gup-pinned pages.
1592 *
1593 * Choice of value:
1594 *
1595 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference
1596 * counts with respect to pin_user_pages() and unpin_user_page() becomes
1597 * simpler, due to the fact that adding an even power of two to the page
1598 * refcount has the effect of using only the upper N bits, for the code that
1599 * counts up using the bias value. This means that the lower bits are left for
1600 * the exclusive use of the original code that increments and decrements by one
1601 * (or at least, by much smaller values than the bias value).
1602 *
1603 * Of course, once the lower bits overflow into the upper bits (and this is
1604 * OK, because subtraction recovers the original values), then visual inspection
1605 * no longer suffices to directly view the separate counts. However, for normal
1606 * applications that don't have huge page reference counts, this won't be an
1607 * issue.
1608 *
1609 * Locking: the lockless algorithm described in folio_try_get_rcu()
1610 * provides safe operation for get_user_pages(), page_mkclean() and
1611 * other calls that race to set up page table entries.
1612 */
1613 #define GUP_PIN_COUNTING_BIAS (1U << 10)
1614
1615 void unpin_user_page(struct page *page);
1616 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
1617 bool make_dirty);
1618 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
1619 bool make_dirty);
1620 void unpin_user_pages(struct page **pages, unsigned long npages);
1621
is_cow_mapping(vm_flags_t flags)1622 static inline bool is_cow_mapping(vm_flags_t flags)
1623 {
1624 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
1625 }
1626
1627 #ifndef CONFIG_MMU
is_nommu_shared_mapping(vm_flags_t flags)1628 static inline bool is_nommu_shared_mapping(vm_flags_t flags)
1629 {
1630 /*
1631 * NOMMU shared mappings are ordinary MAP_SHARED mappings and selected
1632 * R/O MAP_PRIVATE file mappings that are an effective R/O overlay of
1633 * a file mapping. R/O MAP_PRIVATE mappings might still modify
1634 * underlying memory if ptrace is active, so this is only possible if
1635 * ptrace does not apply. Note that there is no mprotect() to upgrade
1636 * write permissions later.
1637 */
1638 return flags & (VM_MAYSHARE | VM_MAYOVERLAY);
1639 }
1640 #endif
1641
1642 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
1643 #define SECTION_IN_PAGE_FLAGS
1644 #endif
1645
1646 /*
1647 * The identification function is mainly used by the buddy allocator for
1648 * determining if two pages could be buddies. We are not really identifying
1649 * the zone since we could be using the section number id if we do not have
1650 * node id available in page flags.
1651 * We only guarantee that it will return the same value for two combinable
1652 * pages in a zone.
1653 */
page_zone_id(struct page * page)1654 static inline int page_zone_id(struct page *page)
1655 {
1656 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
1657 }
1658
1659 #ifdef NODE_NOT_IN_PAGE_FLAGS
1660 int page_to_nid(const struct page *page);
1661 #else
page_to_nid(const struct page * page)1662 static inline int page_to_nid(const struct page *page)
1663 {
1664 return (PF_POISONED_CHECK(page)->flags >> NODES_PGSHIFT) & NODES_MASK;
1665 }
1666 #endif
1667
folio_nid(const struct folio * folio)1668 static inline int folio_nid(const struct folio *folio)
1669 {
1670 return page_to_nid(&folio->page);
1671 }
1672
1673 #ifdef CONFIG_NUMA_BALANCING
1674 /* page access time bits needs to hold at least 4 seconds */
1675 #define PAGE_ACCESS_TIME_MIN_BITS 12
1676 #if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS
1677 #define PAGE_ACCESS_TIME_BUCKETS \
1678 (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT)
1679 #else
1680 #define PAGE_ACCESS_TIME_BUCKETS 0
1681 #endif
1682
1683 #define PAGE_ACCESS_TIME_MASK \
1684 (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS)
1685
cpu_pid_to_cpupid(int cpu,int pid)1686 static inline int cpu_pid_to_cpupid(int cpu, int pid)
1687 {
1688 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
1689 }
1690
cpupid_to_pid(int cpupid)1691 static inline int cpupid_to_pid(int cpupid)
1692 {
1693 return cpupid & LAST__PID_MASK;
1694 }
1695
cpupid_to_cpu(int cpupid)1696 static inline int cpupid_to_cpu(int cpupid)
1697 {
1698 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
1699 }
1700
cpupid_to_nid(int cpupid)1701 static inline int cpupid_to_nid(int cpupid)
1702 {
1703 return cpu_to_node(cpupid_to_cpu(cpupid));
1704 }
1705
cpupid_pid_unset(int cpupid)1706 static inline bool cpupid_pid_unset(int cpupid)
1707 {
1708 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
1709 }
1710
cpupid_cpu_unset(int cpupid)1711 static inline bool cpupid_cpu_unset(int cpupid)
1712 {
1713 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
1714 }
1715
__cpupid_match_pid(pid_t task_pid,int cpupid)1716 static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
1717 {
1718 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
1719 }
1720
1721 #define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
1722 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
folio_xchg_last_cpupid(struct folio * folio,int cpupid)1723 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
1724 {
1725 return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK);
1726 }
1727
folio_last_cpupid(struct folio * folio)1728 static inline int folio_last_cpupid(struct folio *folio)
1729 {
1730 return folio->_last_cpupid;
1731 }
page_cpupid_reset_last(struct page * page)1732 static inline void page_cpupid_reset_last(struct page *page)
1733 {
1734 page->_last_cpupid = -1 & LAST_CPUPID_MASK;
1735 }
1736 #else
folio_last_cpupid(struct folio * folio)1737 static inline int folio_last_cpupid(struct folio *folio)
1738 {
1739 return (folio->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
1740 }
1741
1742 int folio_xchg_last_cpupid(struct folio *folio, int cpupid);
1743
page_cpupid_reset_last(struct page * page)1744 static inline void page_cpupid_reset_last(struct page *page)
1745 {
1746 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
1747 }
1748 #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
1749
folio_xchg_access_time(struct folio * folio,int time)1750 static inline int folio_xchg_access_time(struct folio *folio, int time)
1751 {
1752 int last_time;
1753
1754 last_time = folio_xchg_last_cpupid(folio,
1755 time >> PAGE_ACCESS_TIME_BUCKETS);
1756 return last_time << PAGE_ACCESS_TIME_BUCKETS;
1757 }
1758
vma_set_access_pid_bit(struct vm_area_struct * vma)1759 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
1760 {
1761 unsigned int pid_bit;
1762
1763 pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG));
1764 if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->pids_active[1])) {
1765 __set_bit(pid_bit, &vma->numab_state->pids_active[1]);
1766 }
1767 }
1768 #else /* !CONFIG_NUMA_BALANCING */
folio_xchg_last_cpupid(struct folio * folio,int cpupid)1769 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
1770 {
1771 return folio_nid(folio); /* XXX */
1772 }
1773
folio_xchg_access_time(struct folio * folio,int time)1774 static inline int folio_xchg_access_time(struct folio *folio, int time)
1775 {
1776 return 0;
1777 }
1778
folio_last_cpupid(struct folio * folio)1779 static inline int folio_last_cpupid(struct folio *folio)
1780 {
1781 return folio_nid(folio); /* XXX */
1782 }
1783
cpupid_to_nid(int cpupid)1784 static inline int cpupid_to_nid(int cpupid)
1785 {
1786 return -1;
1787 }
1788
cpupid_to_pid(int cpupid)1789 static inline int cpupid_to_pid(int cpupid)
1790 {
1791 return -1;
1792 }
1793
cpupid_to_cpu(int cpupid)1794 static inline int cpupid_to_cpu(int cpupid)
1795 {
1796 return -1;
1797 }
1798
cpu_pid_to_cpupid(int nid,int pid)1799 static inline int cpu_pid_to_cpupid(int nid, int pid)
1800 {
1801 return -1;
1802 }
1803
cpupid_pid_unset(int cpupid)1804 static inline bool cpupid_pid_unset(int cpupid)
1805 {
1806 return true;
1807 }
1808
page_cpupid_reset_last(struct page * page)1809 static inline void page_cpupid_reset_last(struct page *page)
1810 {
1811 }
1812
cpupid_match_pid(struct task_struct * task,int cpupid)1813 static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
1814 {
1815 return false;
1816 }
1817
vma_set_access_pid_bit(struct vm_area_struct * vma)1818 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
1819 {
1820 }
1821 #endif /* CONFIG_NUMA_BALANCING */
1822
1823 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
1824
1825 /*
1826 * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid
1827 * setting tags for all pages to native kernel tag value 0xff, as the default
1828 * value 0x00 maps to 0xff.
1829 */
1830
page_kasan_tag(const struct page * page)1831 static inline u8 page_kasan_tag(const struct page *page)
1832 {
1833 u8 tag = KASAN_TAG_KERNEL;
1834
1835 if (kasan_enabled()) {
1836 tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;
1837 tag ^= 0xff;
1838 }
1839
1840 return tag;
1841 }
1842
page_kasan_tag_set(struct page * page,u8 tag)1843 static inline void page_kasan_tag_set(struct page *page, u8 tag)
1844 {
1845 unsigned long old_flags, flags;
1846
1847 if (!kasan_enabled())
1848 return;
1849
1850 tag ^= 0xff;
1851 old_flags = READ_ONCE(page->flags);
1852 do {
1853 flags = old_flags;
1854 flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);
1855 flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;
1856 } while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags)));
1857 }
1858
page_kasan_tag_reset(struct page * page)1859 static inline void page_kasan_tag_reset(struct page *page)
1860 {
1861 if (kasan_enabled())
1862 page_kasan_tag_set(page, KASAN_TAG_KERNEL);
1863 }
1864
1865 #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
1866
page_kasan_tag(const struct page * page)1867 static inline u8 page_kasan_tag(const struct page *page)
1868 {
1869 return 0xff;
1870 }
1871
page_kasan_tag_set(struct page * page,u8 tag)1872 static inline void page_kasan_tag_set(struct page *page, u8 tag) { }
page_kasan_tag_reset(struct page * page)1873 static inline void page_kasan_tag_reset(struct page *page) { }
1874
1875 #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
1876
page_zone(const struct page * page)1877 static inline struct zone *page_zone(const struct page *page)
1878 {
1879 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
1880 }
1881
page_pgdat(const struct page * page)1882 static inline pg_data_t *page_pgdat(const struct page *page)
1883 {
1884 return NODE_DATA(page_to_nid(page));
1885 }
1886
folio_zone(const struct folio * folio)1887 static inline struct zone *folio_zone(const struct folio *folio)
1888 {
1889 return page_zone(&folio->page);
1890 }
1891
folio_pgdat(const struct folio * folio)1892 static inline pg_data_t *folio_pgdat(const struct folio *folio)
1893 {
1894 return page_pgdat(&folio->page);
1895 }
1896
1897 #ifdef SECTION_IN_PAGE_FLAGS
set_page_section(struct page * page,unsigned long section)1898 static inline void set_page_section(struct page *page, unsigned long section)
1899 {
1900 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
1901 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
1902 }
1903
page_to_section(const struct page * page)1904 static inline unsigned long page_to_section(const struct page *page)
1905 {
1906 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
1907 }
1908 #endif
1909
1910 /**
1911 * folio_pfn - Return the Page Frame Number of a folio.
1912 * @folio: The folio.
1913 *
1914 * A folio may contain multiple pages. The pages have consecutive
1915 * Page Frame Numbers.
1916 *
1917 * Return: The Page Frame Number of the first page in the folio.
1918 */
folio_pfn(struct folio * folio)1919 static inline unsigned long folio_pfn(struct folio *folio)
1920 {
1921 return page_to_pfn(&folio->page);
1922 }
1923
pfn_folio(unsigned long pfn)1924 static inline struct folio *pfn_folio(unsigned long pfn)
1925 {
1926 return page_folio(pfn_to_page(pfn));
1927 }
1928
1929 /**
1930 * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA.
1931 * @folio: The folio.
1932 *
1933 * This function checks if a folio has been pinned via a call to
1934 * a function in the pin_user_pages() family.
1935 *
1936 * For small folios, the return value is partially fuzzy: false is not fuzzy,
1937 * because it means "definitely not pinned for DMA", but true means "probably
1938 * pinned for DMA, but possibly a false positive due to having at least
1939 * GUP_PIN_COUNTING_BIAS worth of normal folio references".
1940 *
1941 * False positives are OK, because: a) it's unlikely for a folio to
1942 * get that many refcounts, and b) all the callers of this routine are
1943 * expected to be able to deal gracefully with a false positive.
1944 *
1945 * For large folios, the result will be exactly correct. That's because
1946 * we have more tracking data available: the _pincount field is used
1947 * instead of the GUP_PIN_COUNTING_BIAS scheme.
1948 *
1949 * For more information, please see Documentation/core-api/pin_user_pages.rst.
1950 *
1951 * Return: True, if it is likely that the page has been "dma-pinned".
1952 * False, if the page is definitely not dma-pinned.
1953 */
folio_maybe_dma_pinned(struct folio * folio)1954 static inline bool folio_maybe_dma_pinned(struct folio *folio)
1955 {
1956 if (folio_test_large(folio))
1957 return atomic_read(&folio->_pincount) > 0;
1958
1959 /*
1960 * folio_ref_count() is signed. If that refcount overflows, then
1961 * folio_ref_count() returns a negative value, and callers will avoid
1962 * further incrementing the refcount.
1963 *
1964 * Here, for that overflow case, use the sign bit to count a little
1965 * bit higher via unsigned math, and thus still get an accurate result.
1966 */
1967 return ((unsigned int)folio_ref_count(folio)) >=
1968 GUP_PIN_COUNTING_BIAS;
1969 }
1970
page_maybe_dma_pinned(struct page * page)1971 static inline bool page_maybe_dma_pinned(struct page *page)
1972 {
1973 return folio_maybe_dma_pinned(page_folio(page));
1974 }
1975
1976 /*
1977 * This should most likely only be called during fork() to see whether we
1978 * should break the cow immediately for an anon page on the src mm.
1979 *
1980 * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq.
1981 */
folio_needs_cow_for_dma(struct vm_area_struct * vma,struct folio * folio)1982 static inline bool folio_needs_cow_for_dma(struct vm_area_struct *vma,
1983 struct folio *folio)
1984 {
1985 VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1));
1986
1987 if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
1988 return false;
1989
1990 return folio_maybe_dma_pinned(folio);
1991 }
1992
1993 /**
1994 * is_zero_page - Query if a page is a zero page
1995 * @page: The page to query
1996 *
1997 * This returns true if @page is one of the permanent zero pages.
1998 */
is_zero_page(const struct page * page)1999 static inline bool is_zero_page(const struct page *page)
2000 {
2001 return is_zero_pfn(page_to_pfn(page));
2002 }
2003
2004 /**
2005 * is_zero_folio - Query if a folio is a zero page
2006 * @folio: The folio to query
2007 *
2008 * This returns true if @folio is one of the permanent zero pages.
2009 */
is_zero_folio(const struct folio * folio)2010 static inline bool is_zero_folio(const struct folio *folio)
2011 {
2012 return is_zero_page(&folio->page);
2013 }
2014
2015 /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */
2016 #ifdef CONFIG_MIGRATION
folio_is_longterm_pinnable(struct folio * folio)2017 static inline bool folio_is_longterm_pinnable(struct folio *folio)
2018 {
2019 #ifdef CONFIG_CMA
2020 int mt = folio_migratetype(folio);
2021
2022 if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE)
2023 return false;
2024 #endif
2025 /* The zero page can be "pinned" but gets special handling. */
2026 if (is_zero_folio(folio))
2027 return true;
2028
2029 /* Coherent device memory must always allow eviction. */
2030 if (folio_is_device_coherent(folio))
2031 return false;
2032
2033 /* Otherwise, non-movable zone folios can be pinned. */
2034 return !folio_is_zone_movable(folio);
2035
2036 }
2037 #else
folio_is_longterm_pinnable(struct folio * folio)2038 static inline bool folio_is_longterm_pinnable(struct folio *folio)
2039 {
2040 return true;
2041 }
2042 #endif
2043
set_page_zone(struct page * page,enum zone_type zone)2044 static inline void set_page_zone(struct page *page, enum zone_type zone)
2045 {
2046 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
2047 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
2048 }
2049
set_page_node(struct page * page,unsigned long node)2050 static inline void set_page_node(struct page *page, unsigned long node)
2051 {
2052 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
2053 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
2054 }
2055
set_page_links(struct page * page,enum zone_type zone,unsigned long node,unsigned long pfn)2056 static inline void set_page_links(struct page *page, enum zone_type zone,
2057 unsigned long node, unsigned long pfn)
2058 {
2059 set_page_zone(page, zone);
2060 set_page_node(page, node);
2061 #ifdef SECTION_IN_PAGE_FLAGS
2062 set_page_section(page, pfn_to_section_nr(pfn));
2063 #endif
2064 }
2065
2066 /**
2067 * folio_nr_pages - The number of pages in the folio.
2068 * @folio: The folio.
2069 *
2070 * Return: A positive power of two.
2071 */
folio_nr_pages(const struct folio * folio)2072 static inline long folio_nr_pages(const struct folio *folio)
2073 {
2074 if (!folio_test_large(folio))
2075 return 1;
2076 #ifdef CONFIG_64BIT
2077 return folio->_folio_nr_pages;
2078 #else
2079 return 1L << (folio->_flags_1 & 0xff);
2080 #endif
2081 }
2082
2083 /* Only hugetlbfs can allocate folios larger than MAX_ORDER */
2084 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
2085 #define MAX_FOLIO_NR_PAGES (1UL << PUD_ORDER)
2086 #else
2087 #define MAX_FOLIO_NR_PAGES MAX_ORDER_NR_PAGES
2088 #endif
2089
2090 /*
2091 * compound_nr() returns the number of pages in this potentially compound
2092 * page. compound_nr() can be called on a tail page, and is defined to
2093 * return 1 in that case.
2094 */
compound_nr(struct page * page)2095 static inline unsigned long compound_nr(struct page *page)
2096 {
2097 struct folio *folio = (struct folio *)page;
2098
2099 if (!test_bit(PG_head, &folio->flags))
2100 return 1;
2101 #ifdef CONFIG_64BIT
2102 return folio->_folio_nr_pages;
2103 #else
2104 return 1L << (folio->_flags_1 & 0xff);
2105 #endif
2106 }
2107
2108 /**
2109 * thp_nr_pages - The number of regular pages in this huge page.
2110 * @page: The head page of a huge page.
2111 */
thp_nr_pages(struct page * page)2112 static inline int thp_nr_pages(struct page *page)
2113 {
2114 return folio_nr_pages((struct folio *)page);
2115 }
2116
2117 /**
2118 * folio_next - Move to the next physical folio.
2119 * @folio: The folio we're currently operating on.
2120 *
2121 * If you have physically contiguous memory which may span more than
2122 * one folio (eg a &struct bio_vec), use this function to move from one
2123 * folio to the next. Do not use it if the memory is only virtually
2124 * contiguous as the folios are almost certainly not adjacent to each
2125 * other. This is the folio equivalent to writing ``page++``.
2126 *
2127 * Context: We assume that the folios are refcounted and/or locked at a
2128 * higher level and do not adjust the reference counts.
2129 * Return: The next struct folio.
2130 */
folio_next(struct folio * folio)2131 static inline struct folio *folio_next(struct folio *folio)
2132 {
2133 return (struct folio *)folio_page(folio, folio_nr_pages(folio));
2134 }
2135
2136 /**
2137 * folio_shift - The size of the memory described by this folio.
2138 * @folio: The folio.
2139 *
2140 * A folio represents a number of bytes which is a power-of-two in size.
2141 * This function tells you which power-of-two the folio is. See also
2142 * folio_size() and folio_order().
2143 *
2144 * Context: The caller should have a reference on the folio to prevent
2145 * it from being split. It is not necessary for the folio to be locked.
2146 * Return: The base-2 logarithm of the size of this folio.
2147 */
folio_shift(struct folio * folio)2148 static inline unsigned int folio_shift(struct folio *folio)
2149 {
2150 return PAGE_SHIFT + folio_order(folio);
2151 }
2152
2153 /**
2154 * folio_size - The number of bytes in a folio.
2155 * @folio: The folio.
2156 *
2157 * Context: The caller should have a reference on the folio to prevent
2158 * it from being split. It is not necessary for the folio to be locked.
2159 * Return: The number of bytes in this folio.
2160 */
folio_size(struct folio * folio)2161 static inline size_t folio_size(struct folio *folio)
2162 {
2163 return PAGE_SIZE << folio_order(folio);
2164 }
2165
2166 /**
2167 * folio_likely_mapped_shared - Estimate if the folio is mapped into the page
2168 * tables of more than one MM
2169 * @folio: The folio.
2170 *
2171 * This function checks if the folio is currently mapped into more than one
2172 * MM ("mapped shared"), or if the folio is only mapped into a single MM
2173 * ("mapped exclusively").
2174 *
2175 * As precise information is not easily available for all folios, this function
2176 * estimates the number of MMs ("sharers") that are currently mapping a folio
2177 * using the number of times the first page of the folio is currently mapped
2178 * into page tables.
2179 *
2180 * For small anonymous folios (except KSM folios) and anonymous hugetlb folios,
2181 * the return value will be exactly correct, because they can only be mapped
2182 * at most once into an MM, and they cannot be partially mapped.
2183 *
2184 * For other folios, the result can be fuzzy:
2185 * #. For partially-mappable large folios (THP), the return value can wrongly
2186 * indicate "mapped exclusively" (false negative) when the folio is
2187 * only partially mapped into at least one MM.
2188 * #. For pagecache folios (including hugetlb), the return value can wrongly
2189 * indicate "mapped shared" (false positive) when two VMAs in the same MM
2190 * cover the same file range.
2191 * #. For (small) KSM folios, the return value can wrongly indicate "mapped
2192 * shared" (false positive), when the folio is mapped multiple times into
2193 * the same MM.
2194 *
2195 * Further, this function only considers current page table mappings that
2196 * are tracked using the folio mapcount(s).
2197 *
2198 * This function does not consider:
2199 * #. If the folio might get mapped in the (near) future (e.g., swapcache,
2200 * pagecache, temporary unmapping for migration).
2201 * #. If the folio is mapped differently (VM_PFNMAP).
2202 * #. If hugetlb page table sharing applies. Callers might want to check
2203 * hugetlb_pmd_shared().
2204 *
2205 * Return: Whether the folio is estimated to be mapped into more than one MM.
2206 */
folio_likely_mapped_shared(struct folio * folio)2207 static inline bool folio_likely_mapped_shared(struct folio *folio)
2208 {
2209 int mapcount = folio_mapcount(folio);
2210
2211 /* Only partially-mappable folios require more care. */
2212 if (!folio_test_large(folio) || unlikely(folio_test_hugetlb(folio)))
2213 return mapcount > 1;
2214
2215 /* A single mapping implies "mapped exclusively". */
2216 if (mapcount <= 1)
2217 return false;
2218
2219 /* If any page is mapped more than once we treat it "mapped shared". */
2220 if (folio_entire_mapcount(folio) || mapcount > folio_nr_pages(folio))
2221 return true;
2222
2223 /* Let's guess based on the first subpage. */
2224 return atomic_read(&folio->_mapcount) > 0;
2225 }
2226
2227 #ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE
arch_make_page_accessible(struct page * page)2228 static inline int arch_make_page_accessible(struct page *page)
2229 {
2230 return 0;
2231 }
2232 #endif
2233
2234 #ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE
arch_make_folio_accessible(struct folio * folio)2235 static inline int arch_make_folio_accessible(struct folio *folio)
2236 {
2237 int ret;
2238 long i, nr = folio_nr_pages(folio);
2239
2240 for (i = 0; i < nr; i++) {
2241 ret = arch_make_page_accessible(folio_page(folio, i));
2242 if (ret)
2243 break;
2244 }
2245
2246 return ret;
2247 }
2248 #endif
2249
2250 /*
2251 * Some inline functions in vmstat.h depend on page_zone()
2252 */
2253 #include <linux/vmstat.h>
2254
2255 #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
2256 #define HASHED_PAGE_VIRTUAL
2257 #endif
2258
2259 #if defined(WANT_PAGE_VIRTUAL)
page_address(const struct page * page)2260 static inline void *page_address(const struct page *page)
2261 {
2262 return page->virtual;
2263 }
set_page_address(struct page * page,void * address)2264 static inline void set_page_address(struct page *page, void *address)
2265 {
2266 page->virtual = address;
2267 }
2268 #define page_address_init() do { } while(0)
2269 #endif
2270
2271 #if defined(HASHED_PAGE_VIRTUAL)
2272 void *page_address(const struct page *page);
2273 void set_page_address(struct page *page, void *virtual);
2274 void page_address_init(void);
2275 #endif
2276
lowmem_page_address(const struct page * page)2277 static __always_inline void *lowmem_page_address(const struct page *page)
2278 {
2279 return page_to_virt(page);
2280 }
2281
2282 #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
2283 #define page_address(page) lowmem_page_address(page)
2284 #define set_page_address(page, address) do { } while(0)
2285 #define page_address_init() do { } while(0)
2286 #endif
2287
folio_address(const struct folio * folio)2288 static inline void *folio_address(const struct folio *folio)
2289 {
2290 return page_address(&folio->page);
2291 }
2292
2293 extern pgoff_t __page_file_index(struct page *page);
2294
2295 /*
2296 * Return the pagecache index of the passed page. Regular pagecache pages
2297 * use ->index whereas swapcache pages use swp_offset(->private)
2298 */
page_index(struct page * page)2299 static inline pgoff_t page_index(struct page *page)
2300 {
2301 if (unlikely(PageSwapCache(page)))
2302 return __page_file_index(page);
2303 return page->index;
2304 }
2305
2306 /*
2307 * Return true only if the page has been allocated with
2308 * ALLOC_NO_WATERMARKS and the low watermark was not
2309 * met implying that the system is under some pressure.
2310 */
page_is_pfmemalloc(const struct page * page)2311 static inline bool page_is_pfmemalloc(const struct page *page)
2312 {
2313 /*
2314 * lru.next has bit 1 set if the page is allocated from the
2315 * pfmemalloc reserves. Callers may simply overwrite it if
2316 * they do not need to preserve that information.
2317 */
2318 return (uintptr_t)page->lru.next & BIT(1);
2319 }
2320
2321 /*
2322 * Return true only if the folio has been allocated with
2323 * ALLOC_NO_WATERMARKS and the low watermark was not
2324 * met implying that the system is under some pressure.
2325 */
folio_is_pfmemalloc(const struct folio * folio)2326 static inline bool folio_is_pfmemalloc(const struct folio *folio)
2327 {
2328 /*
2329 * lru.next has bit 1 set if the page is allocated from the
2330 * pfmemalloc reserves. Callers may simply overwrite it if
2331 * they do not need to preserve that information.
2332 */
2333 return (uintptr_t)folio->lru.next & BIT(1);
2334 }
2335
2336 /*
2337 * Only to be called by the page allocator on a freshly allocated
2338 * page.
2339 */
set_page_pfmemalloc(struct page * page)2340 static inline void set_page_pfmemalloc(struct page *page)
2341 {
2342 page->lru.next = (void *)BIT(1);
2343 }
2344
clear_page_pfmemalloc(struct page * page)2345 static inline void clear_page_pfmemalloc(struct page *page)
2346 {
2347 page->lru.next = NULL;
2348 }
2349
2350 /*
2351 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
2352 */
2353 extern void pagefault_out_of_memory(void);
2354
2355 #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
2356 #define offset_in_thp(page, p) ((unsigned long)(p) & (thp_size(page) - 1))
2357 #define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1))
2358
2359 /*
2360 * Parameter block passed down to zap_pte_range in exceptional cases.
2361 */
2362 struct zap_details {
2363 struct folio *single_folio; /* Locked folio to be unmapped */
2364 bool even_cows; /* Zap COWed private pages too? */
2365 zap_flags_t zap_flags; /* Extra flags for zapping */
2366 };
2367
2368 /*
2369 * Whether to drop the pte markers, for example, the uffd-wp information for
2370 * file-backed memory. This should only be specified when we will completely
2371 * drop the page in the mm, either by truncation or unmapping of the vma. By
2372 * default, the flag is not set.
2373 */
2374 #define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0))
2375 /* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */
2376 #define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1))
2377
2378 #ifdef CONFIG_SCHED_MM_CID
2379 void sched_mm_cid_before_execve(struct task_struct *t);
2380 void sched_mm_cid_after_execve(struct task_struct *t);
2381 void sched_mm_cid_fork(struct task_struct *t);
2382 void sched_mm_cid_exit_signals(struct task_struct *t);
task_mm_cid(struct task_struct * t)2383 static inline int task_mm_cid(struct task_struct *t)
2384 {
2385 return t->mm_cid;
2386 }
2387 #else
sched_mm_cid_before_execve(struct task_struct * t)2388 static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
sched_mm_cid_after_execve(struct task_struct * t)2389 static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
sched_mm_cid_fork(struct task_struct * t)2390 static inline void sched_mm_cid_fork(struct task_struct *t) { }
sched_mm_cid_exit_signals(struct task_struct * t)2391 static inline void sched_mm_cid_exit_signals(struct task_struct *t) { }
task_mm_cid(struct task_struct * t)2392 static inline int task_mm_cid(struct task_struct *t)
2393 {
2394 /*
2395 * Use the processor id as a fall-back when the mm cid feature is
2396 * disabled. This provides functional per-cpu data structure accesses
2397 * in user-space, althrough it won't provide the memory usage benefits.
2398 */
2399 return raw_smp_processor_id();
2400 }
2401 #endif
2402
2403 #ifdef CONFIG_MMU
2404 extern bool can_do_mlock(void);
2405 #else
can_do_mlock(void)2406 static inline bool can_do_mlock(void) { return false; }
2407 #endif
2408 extern int user_shm_lock(size_t, struct ucounts *);
2409 extern void user_shm_unlock(size_t, struct ucounts *);
2410
2411 struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
2412 pte_t pte);
2413 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
2414 pte_t pte);
2415 struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
2416 unsigned long addr, pmd_t pmd);
2417 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
2418 pmd_t pmd);
2419
2420 void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
2421 unsigned long size);
2422 void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
2423 unsigned long size, struct zap_details *details);
zap_vma_pages(struct vm_area_struct * vma)2424 static inline void zap_vma_pages(struct vm_area_struct *vma)
2425 {
2426 zap_page_range_single(vma, vma->vm_start,
2427 vma->vm_end - vma->vm_start, NULL);
2428 }
2429 void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
2430 struct vm_area_struct *start_vma, unsigned long start,
2431 unsigned long end, unsigned long tree_end, bool mm_wr_locked);
2432
2433 struct mmu_notifier_range;
2434
2435 void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
2436 unsigned long end, unsigned long floor, unsigned long ceiling);
2437 int
2438 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
2439 int follow_pte(struct vm_area_struct *vma, unsigned long address,
2440 pte_t **ptepp, spinlock_t **ptlp);
2441 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
2442 void *buf, int len, int write);
2443
2444 extern void truncate_pagecache(struct inode *inode, loff_t new);
2445 extern void truncate_setsize(struct inode *inode, loff_t newsize);
2446 void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
2447 void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
2448 int generic_error_remove_folio(struct address_space *mapping,
2449 struct folio *folio);
2450
2451 struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
2452 unsigned long address, struct pt_regs *regs);
2453
2454 #ifdef CONFIG_MMU
2455 extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
2456 unsigned long address, unsigned int flags,
2457 struct pt_regs *regs);
2458 extern int fixup_user_fault(struct mm_struct *mm,
2459 unsigned long address, unsigned int fault_flags,
2460 bool *unlocked);
2461 void unmap_mapping_pages(struct address_space *mapping,
2462 pgoff_t start, pgoff_t nr, bool even_cows);
2463 void unmap_mapping_range(struct address_space *mapping,
2464 loff_t const holebegin, loff_t const holelen, int even_cows);
2465 #else
handle_mm_fault(struct vm_area_struct * vma,unsigned long address,unsigned int flags,struct pt_regs * regs)2466 static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
2467 unsigned long address, unsigned int flags,
2468 struct pt_regs *regs)
2469 {
2470 /* should never happen if there's no MMU */
2471 BUG();
2472 return VM_FAULT_SIGBUS;
2473 }
fixup_user_fault(struct mm_struct * mm,unsigned long address,unsigned int fault_flags,bool * unlocked)2474 static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address,
2475 unsigned int fault_flags, bool *unlocked)
2476 {
2477 /* should never happen if there's no MMU */
2478 BUG();
2479 return -EFAULT;
2480 }
unmap_mapping_pages(struct address_space * mapping,pgoff_t start,pgoff_t nr,bool even_cows)2481 static inline void unmap_mapping_pages(struct address_space *mapping,
2482 pgoff_t start, pgoff_t nr, bool even_cows) { }
unmap_mapping_range(struct address_space * mapping,loff_t const holebegin,loff_t const holelen,int even_cows)2483 static inline void unmap_mapping_range(struct address_space *mapping,
2484 loff_t const holebegin, loff_t const holelen, int even_cows) { }
2485 #endif
2486
unmap_shared_mapping_range(struct address_space * mapping,loff_t const holebegin,loff_t const holelen)2487 static inline void unmap_shared_mapping_range(struct address_space *mapping,
2488 loff_t const holebegin, loff_t const holelen)
2489 {
2490 unmap_mapping_range(mapping, holebegin, holelen, 0);
2491 }
2492
2493 static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm,
2494 unsigned long addr);
2495
2496 extern int access_process_vm(struct task_struct *tsk, unsigned long addr,
2497 void *buf, int len, unsigned int gup_flags);
2498 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
2499 void *buf, int len, unsigned int gup_flags);
2500
2501 long get_user_pages_remote(struct mm_struct *mm,
2502 unsigned long start, unsigned long nr_pages,
2503 unsigned int gup_flags, struct page **pages,
2504 int *locked);
2505 long pin_user_pages_remote(struct mm_struct *mm,
2506 unsigned long start, unsigned long nr_pages,
2507 unsigned int gup_flags, struct page **pages,
2508 int *locked);
2509
2510 /*
2511 * Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT.
2512 */
get_user_page_vma_remote(struct mm_struct * mm,unsigned long addr,int gup_flags,struct vm_area_struct ** vmap)2513 static inline struct page *get_user_page_vma_remote(struct mm_struct *mm,
2514 unsigned long addr,
2515 int gup_flags,
2516 struct vm_area_struct **vmap)
2517 {
2518 struct page *page;
2519 struct vm_area_struct *vma;
2520 int got;
2521
2522 if (WARN_ON_ONCE(unlikely(gup_flags & FOLL_NOWAIT)))
2523 return ERR_PTR(-EINVAL);
2524
2525 got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL);
2526
2527 if (got < 0)
2528 return ERR_PTR(got);
2529
2530 vma = vma_lookup(mm, addr);
2531 if (WARN_ON_ONCE(!vma)) {
2532 put_page(page);
2533 return ERR_PTR(-EINVAL);
2534 }
2535
2536 *vmap = vma;
2537 return page;
2538 }
2539
2540 long get_user_pages(unsigned long start, unsigned long nr_pages,
2541 unsigned int gup_flags, struct page **pages);
2542 long pin_user_pages(unsigned long start, unsigned long nr_pages,
2543 unsigned int gup_flags, struct page **pages);
2544 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2545 struct page **pages, unsigned int gup_flags);
2546 long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2547 struct page **pages, unsigned int gup_flags);
2548
2549 int get_user_pages_fast(unsigned long start, int nr_pages,
2550 unsigned int gup_flags, struct page **pages);
2551 int pin_user_pages_fast(unsigned long start, int nr_pages,
2552 unsigned int gup_flags, struct page **pages);
2553 void folio_add_pin(struct folio *folio);
2554
2555 int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);
2556 int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
2557 struct task_struct *task, bool bypass_rlim);
2558
2559 struct kvec;
2560 struct page *get_dump_page(unsigned long addr);
2561
2562 bool folio_mark_dirty(struct folio *folio);
2563 bool set_page_dirty(struct page *page);
2564 int set_page_dirty_lock(struct page *page);
2565
2566 int get_cmdline(struct task_struct *task, char *buffer, int buflen);
2567
2568 extern unsigned long move_page_tables(struct vm_area_struct *vma,
2569 unsigned long old_addr, struct vm_area_struct *new_vma,
2570 unsigned long new_addr, unsigned long len,
2571 bool need_rmap_locks, bool for_stack);
2572
2573 /*
2574 * Flags used by change_protection(). For now we make it a bitmap so
2575 * that we can pass in multiple flags just like parameters. However
2576 * for now all the callers are only use one of the flags at the same
2577 * time.
2578 */
2579 /*
2580 * Whether we should manually check if we can map individual PTEs writable,
2581 * because something (e.g., COW, uffd-wp) blocks that from happening for all
2582 * PTEs automatically in a writable mapping.
2583 */
2584 #define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0)
2585 /* Whether this protection change is for NUMA hints */
2586 #define MM_CP_PROT_NUMA (1UL << 1)
2587 /* Whether this change is for write protecting */
2588 #define MM_CP_UFFD_WP (1UL << 2) /* do wp */
2589 #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */
2590 #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \
2591 MM_CP_UFFD_WP_RESOLVE)
2592
2593 bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
2594 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
vma_wants_manual_pte_write_upgrade(struct vm_area_struct * vma)2595 static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
2596 {
2597 /*
2598 * We want to check manually if we can change individual PTEs writable
2599 * if we can't do that automatically for all PTEs in a mapping. For
2600 * private mappings, that's always the case when we have write
2601 * permissions as we properly have to handle COW.
2602 */
2603 if (vma->vm_flags & VM_SHARED)
2604 return vma_wants_writenotify(vma, vma->vm_page_prot);
2605 return !!(vma->vm_flags & VM_WRITE);
2606
2607 }
2608 bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
2609 pte_t pte);
2610 extern long change_protection(struct mmu_gather *tlb,
2611 struct vm_area_struct *vma, unsigned long start,
2612 unsigned long end, unsigned long cp_flags);
2613 extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
2614 struct vm_area_struct *vma, struct vm_area_struct **pprev,
2615 unsigned long start, unsigned long end, unsigned long newflags);
2616
2617 /*
2618 * doesn't attempt to fault and will return short.
2619 */
2620 int get_user_pages_fast_only(unsigned long start, int nr_pages,
2621 unsigned int gup_flags, struct page **pages);
2622
get_user_page_fast_only(unsigned long addr,unsigned int gup_flags,struct page ** pagep)2623 static inline bool get_user_page_fast_only(unsigned long addr,
2624 unsigned int gup_flags, struct page **pagep)
2625 {
2626 return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1;
2627 }
2628 /*
2629 * per-process(per-mm_struct) statistics.
2630 */
get_mm_counter(struct mm_struct * mm,int member)2631 static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
2632 {
2633 return percpu_counter_read_positive(&mm->rss_stat[member]);
2634 }
2635
2636 void mm_trace_rss_stat(struct mm_struct *mm, int member);
2637
add_mm_counter(struct mm_struct * mm,int member,long value)2638 static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
2639 {
2640 percpu_counter_add(&mm->rss_stat[member], value);
2641
2642 mm_trace_rss_stat(mm, member);
2643 }
2644
inc_mm_counter(struct mm_struct * mm,int member)2645 static inline void inc_mm_counter(struct mm_struct *mm, int member)
2646 {
2647 percpu_counter_inc(&mm->rss_stat[member]);
2648
2649 mm_trace_rss_stat(mm, member);
2650 }
2651
dec_mm_counter(struct mm_struct * mm,int member)2652 static inline void dec_mm_counter(struct mm_struct *mm, int member)
2653 {
2654 percpu_counter_dec(&mm->rss_stat[member]);
2655
2656 mm_trace_rss_stat(mm, member);
2657 }
2658
2659 /* Optimized variant when folio is already known not to be anon */
mm_counter_file(struct folio * folio)2660 static inline int mm_counter_file(struct folio *folio)
2661 {
2662 if (folio_test_swapbacked(folio))
2663 return MM_SHMEMPAGES;
2664 return MM_FILEPAGES;
2665 }
2666
mm_counter(struct folio * folio)2667 static inline int mm_counter(struct folio *folio)
2668 {
2669 if (folio_test_anon(folio))
2670 return MM_ANONPAGES;
2671 return mm_counter_file(folio);
2672 }
2673
get_mm_rss(struct mm_struct * mm)2674 static inline unsigned long get_mm_rss(struct mm_struct *mm)
2675 {
2676 return get_mm_counter(mm, MM_FILEPAGES) +
2677 get_mm_counter(mm, MM_ANONPAGES) +
2678 get_mm_counter(mm, MM_SHMEMPAGES);
2679 }
2680
get_mm_hiwater_rss(struct mm_struct * mm)2681 static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
2682 {
2683 return max(mm->hiwater_rss, get_mm_rss(mm));
2684 }
2685
get_mm_hiwater_vm(struct mm_struct * mm)2686 static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
2687 {
2688 return max(mm->hiwater_vm, mm->total_vm);
2689 }
2690
update_hiwater_rss(struct mm_struct * mm)2691 static inline void update_hiwater_rss(struct mm_struct *mm)
2692 {
2693 unsigned long _rss = get_mm_rss(mm);
2694
2695 if ((mm)->hiwater_rss < _rss)
2696 (mm)->hiwater_rss = _rss;
2697 }
2698
update_hiwater_vm(struct mm_struct * mm)2699 static inline void update_hiwater_vm(struct mm_struct *mm)
2700 {
2701 if (mm->hiwater_vm < mm->total_vm)
2702 mm->hiwater_vm = mm->total_vm;
2703 }
2704
reset_mm_hiwater_rss(struct mm_struct * mm)2705 static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
2706 {
2707 mm->hiwater_rss = get_mm_rss(mm);
2708 }
2709
setmax_mm_hiwater_rss(unsigned long * maxrss,struct mm_struct * mm)2710 static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
2711 struct mm_struct *mm)
2712 {
2713 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
2714
2715 if (*maxrss < hiwater_rss)
2716 *maxrss = hiwater_rss;
2717 }
2718
2719 #ifndef CONFIG_ARCH_HAS_PTE_SPECIAL
pte_special(pte_t pte)2720 static inline int pte_special(pte_t pte)
2721 {
2722 return 0;
2723 }
2724
pte_mkspecial(pte_t pte)2725 static inline pte_t pte_mkspecial(pte_t pte)
2726 {
2727 return pte;
2728 }
2729 #endif
2730
2731 #ifndef CONFIG_ARCH_HAS_PTE_DEVMAP
pte_devmap(pte_t pte)2732 static inline int pte_devmap(pte_t pte)
2733 {
2734 return 0;
2735 }
2736 #endif
2737
2738 extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
2739 spinlock_t **ptl);
get_locked_pte(struct mm_struct * mm,unsigned long addr,spinlock_t ** ptl)2740 static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
2741 spinlock_t **ptl)
2742 {
2743 pte_t *ptep;
2744 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
2745 return ptep;
2746 }
2747
2748 #ifdef __PAGETABLE_P4D_FOLDED
__p4d_alloc(struct mm_struct * mm,pgd_t * pgd,unsigned long address)2749 static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
2750 unsigned long address)
2751 {
2752 return 0;
2753 }
2754 #else
2755 int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
2756 #endif
2757
2758 #if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU)
__pud_alloc(struct mm_struct * mm,p4d_t * p4d,unsigned long address)2759 static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d,
2760 unsigned long address)
2761 {
2762 return 0;
2763 }
mm_inc_nr_puds(struct mm_struct * mm)2764 static inline void mm_inc_nr_puds(struct mm_struct *mm) {}
mm_dec_nr_puds(struct mm_struct * mm)2765 static inline void mm_dec_nr_puds(struct mm_struct *mm) {}
2766
2767 #else
2768 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address);
2769
mm_inc_nr_puds(struct mm_struct * mm)2770 static inline void mm_inc_nr_puds(struct mm_struct *mm)
2771 {
2772 if (mm_pud_folded(mm))
2773 return;
2774 atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
2775 }
2776
mm_dec_nr_puds(struct mm_struct * mm)2777 static inline void mm_dec_nr_puds(struct mm_struct *mm)
2778 {
2779 if (mm_pud_folded(mm))
2780 return;
2781 atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
2782 }
2783 #endif
2784
2785 #if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
__pmd_alloc(struct mm_struct * mm,pud_t * pud,unsigned long address)2786 static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
2787 unsigned long address)
2788 {
2789 return 0;
2790 }
2791
mm_inc_nr_pmds(struct mm_struct * mm)2792 static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
mm_dec_nr_pmds(struct mm_struct * mm)2793 static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
2794
2795 #else
2796 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
2797
mm_inc_nr_pmds(struct mm_struct * mm)2798 static inline void mm_inc_nr_pmds(struct mm_struct *mm)
2799 {
2800 if (mm_pmd_folded(mm))
2801 return;
2802 atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
2803 }
2804
mm_dec_nr_pmds(struct mm_struct * mm)2805 static inline void mm_dec_nr_pmds(struct mm_struct *mm)
2806 {
2807 if (mm_pmd_folded(mm))
2808 return;
2809 atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
2810 }
2811 #endif
2812
2813 #ifdef CONFIG_MMU
mm_pgtables_bytes_init(struct mm_struct * mm)2814 static inline void mm_pgtables_bytes_init(struct mm_struct *mm)
2815 {
2816 atomic_long_set(&mm->pgtables_bytes, 0);
2817 }
2818
mm_pgtables_bytes(const struct mm_struct * mm)2819 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
2820 {
2821 return atomic_long_read(&mm->pgtables_bytes);
2822 }
2823
mm_inc_nr_ptes(struct mm_struct * mm)2824 static inline void mm_inc_nr_ptes(struct mm_struct *mm)
2825 {
2826 atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
2827 }
2828
mm_dec_nr_ptes(struct mm_struct * mm)2829 static inline void mm_dec_nr_ptes(struct mm_struct *mm)
2830 {
2831 atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
2832 }
2833 #else
2834
mm_pgtables_bytes_init(struct mm_struct * mm)2835 static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {}
mm_pgtables_bytes(const struct mm_struct * mm)2836 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
2837 {
2838 return 0;
2839 }
2840
mm_inc_nr_ptes(struct mm_struct * mm)2841 static inline void mm_inc_nr_ptes(struct mm_struct *mm) {}
mm_dec_nr_ptes(struct mm_struct * mm)2842 static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}
2843 #endif
2844
2845 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd);
2846 int __pte_alloc_kernel(pmd_t *pmd);
2847
2848 #if defined(CONFIG_MMU)
2849
p4d_alloc(struct mm_struct * mm,pgd_t * pgd,unsigned long address)2850 static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
2851 unsigned long address)
2852 {
2853 return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ?
2854 NULL : p4d_offset(pgd, address);
2855 }
2856
pud_alloc(struct mm_struct * mm,p4d_t * p4d,unsigned long address)2857 static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,
2858 unsigned long address)
2859 {
2860 return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ?
2861 NULL : pud_offset(p4d, address);
2862 }
2863
pmd_alloc(struct mm_struct * mm,pud_t * pud,unsigned long address)2864 static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
2865 {
2866 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
2867 NULL: pmd_offset(pud, address);
2868 }
2869 #endif /* CONFIG_MMU */
2870
virt_to_ptdesc(const void * x)2871 static inline struct ptdesc *virt_to_ptdesc(const void *x)
2872 {
2873 return page_ptdesc(virt_to_page(x));
2874 }
2875
ptdesc_to_virt(const struct ptdesc * pt)2876 static inline void *ptdesc_to_virt(const struct ptdesc *pt)
2877 {
2878 return page_to_virt(ptdesc_page(pt));
2879 }
2880
ptdesc_address(const struct ptdesc * pt)2881 static inline void *ptdesc_address(const struct ptdesc *pt)
2882 {
2883 return folio_address(ptdesc_folio(pt));
2884 }
2885
pagetable_is_reserved(struct ptdesc * pt)2886 static inline bool pagetable_is_reserved(struct ptdesc *pt)
2887 {
2888 return folio_test_reserved(ptdesc_folio(pt));
2889 }
2890
2891 /**
2892 * pagetable_alloc - Allocate pagetables
2893 * @gfp: GFP flags
2894 * @order: desired pagetable order
2895 *
2896 * pagetable_alloc allocates memory for page tables as well as a page table
2897 * descriptor to describe that memory.
2898 *
2899 * Return: The ptdesc describing the allocated page tables.
2900 */
pagetable_alloc_noprof(gfp_t gfp,unsigned int order)2901 static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order)
2902 {
2903 struct page *page = alloc_pages_noprof(gfp | __GFP_COMP, order);
2904
2905 return page_ptdesc(page);
2906 }
2907 #define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__))
2908
2909 /**
2910 * pagetable_free - Free pagetables
2911 * @pt: The page table descriptor
2912 *
2913 * pagetable_free frees the memory of all page tables described by a page
2914 * table descriptor and the memory for the descriptor itself.
2915 */
pagetable_free(struct ptdesc * pt)2916 static inline void pagetable_free(struct ptdesc *pt)
2917 {
2918 struct page *page = ptdesc_page(pt);
2919
2920 __free_pages(page, compound_order(page));
2921 }
2922
2923 #if USE_SPLIT_PTE_PTLOCKS
2924 #if ALLOC_SPLIT_PTLOCKS
2925 void __init ptlock_cache_init(void);
2926 bool ptlock_alloc(struct ptdesc *ptdesc);
2927 void ptlock_free(struct ptdesc *ptdesc);
2928
ptlock_ptr(struct ptdesc * ptdesc)2929 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc)
2930 {
2931 return ptdesc->ptl;
2932 }
2933 #else /* ALLOC_SPLIT_PTLOCKS */
ptlock_cache_init(void)2934 static inline void ptlock_cache_init(void)
2935 {
2936 }
2937
ptlock_alloc(struct ptdesc * ptdesc)2938 static inline bool ptlock_alloc(struct ptdesc *ptdesc)
2939 {
2940 return true;
2941 }
2942
ptlock_free(struct ptdesc * ptdesc)2943 static inline void ptlock_free(struct ptdesc *ptdesc)
2944 {
2945 }
2946
ptlock_ptr(struct ptdesc * ptdesc)2947 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc)
2948 {
2949 return &ptdesc->ptl;
2950 }
2951 #endif /* ALLOC_SPLIT_PTLOCKS */
2952
pte_lockptr(struct mm_struct * mm,pmd_t * pmd)2953 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
2954 {
2955 return ptlock_ptr(page_ptdesc(pmd_page(*pmd)));
2956 }
2957
ptlock_init(struct ptdesc * ptdesc)2958 static inline bool ptlock_init(struct ptdesc *ptdesc)
2959 {
2960 /*
2961 * prep_new_page() initialize page->private (and therefore page->ptl)
2962 * with 0. Make sure nobody took it in use in between.
2963 *
2964 * It can happen if arch try to use slab for page table allocation:
2965 * slab code uses page->slab_cache, which share storage with page->ptl.
2966 */
2967 VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc));
2968 if (!ptlock_alloc(ptdesc))
2969 return false;
2970 spin_lock_init(ptlock_ptr(ptdesc));
2971 return true;
2972 }
2973
2974 #else /* !USE_SPLIT_PTE_PTLOCKS */
2975 /*
2976 * We use mm->page_table_lock to guard all pagetable pages of the mm.
2977 */
pte_lockptr(struct mm_struct * mm,pmd_t * pmd)2978 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
2979 {
2980 return &mm->page_table_lock;
2981 }
ptlock_cache_init(void)2982 static inline void ptlock_cache_init(void) {}
ptlock_init(struct ptdesc * ptdesc)2983 static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; }
ptlock_free(struct ptdesc * ptdesc)2984 static inline void ptlock_free(struct ptdesc *ptdesc) {}
2985 #endif /* USE_SPLIT_PTE_PTLOCKS */
2986
pagetable_pte_ctor(struct ptdesc * ptdesc)2987 static inline bool pagetable_pte_ctor(struct ptdesc *ptdesc)
2988 {
2989 struct folio *folio = ptdesc_folio(ptdesc);
2990
2991 if (!ptlock_init(ptdesc))
2992 return false;
2993 __folio_set_pgtable(folio);
2994 lruvec_stat_add_folio(folio, NR_PAGETABLE);
2995 return true;
2996 }
2997
pagetable_pte_dtor(struct ptdesc * ptdesc)2998 static inline void pagetable_pte_dtor(struct ptdesc *ptdesc)
2999 {
3000 struct folio *folio = ptdesc_folio(ptdesc);
3001
3002 ptlock_free(ptdesc);
3003 __folio_clear_pgtable(folio);
3004 lruvec_stat_sub_folio(folio, NR_PAGETABLE);
3005 }
3006
3007 pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp);
pte_offset_map(pmd_t * pmd,unsigned long addr)3008 static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr)
3009 {
3010 return __pte_offset_map(pmd, addr, NULL);
3011 }
3012
3013 pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
3014 unsigned long addr, spinlock_t **ptlp);
pte_offset_map_lock(struct mm_struct * mm,pmd_t * pmd,unsigned long addr,spinlock_t ** ptlp)3015 static inline pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
3016 unsigned long addr, spinlock_t **ptlp)
3017 {
3018 pte_t *pte;
3019
3020 __cond_lock(*ptlp, pte = __pte_offset_map_lock(mm, pmd, addr, ptlp));
3021 return pte;
3022 }
3023
3024 pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd,
3025 unsigned long addr, spinlock_t **ptlp);
3026
3027 #define pte_unmap_unlock(pte, ptl) do { \
3028 spin_unlock(ptl); \
3029 pte_unmap(pte); \
3030 } while (0)
3031
3032 #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd))
3033
3034 #define pte_alloc_map(mm, pmd, address) \
3035 (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address))
3036
3037 #define pte_alloc_map_lock(mm, pmd, address, ptlp) \
3038 (pte_alloc(mm, pmd) ? \
3039 NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
3040
3041 #define pte_alloc_kernel(pmd, address) \
3042 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \
3043 NULL: pte_offset_kernel(pmd, address))
3044
3045 #if USE_SPLIT_PMD_PTLOCKS
3046
pmd_pgtable_page(pmd_t * pmd)3047 static inline struct page *pmd_pgtable_page(pmd_t *pmd)
3048 {
3049 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
3050 return virt_to_page((void *)((unsigned long) pmd & mask));
3051 }
3052
pmd_ptdesc(pmd_t * pmd)3053 static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd)
3054 {
3055 return page_ptdesc(pmd_pgtable_page(pmd));
3056 }
3057
pmd_lockptr(struct mm_struct * mm,pmd_t * pmd)3058 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
3059 {
3060 return ptlock_ptr(pmd_ptdesc(pmd));
3061 }
3062
pmd_ptlock_init(struct ptdesc * ptdesc)3063 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc)
3064 {
3065 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3066 ptdesc->pmd_huge_pte = NULL;
3067 #endif
3068 return ptlock_init(ptdesc);
3069 }
3070
pmd_ptlock_free(struct ptdesc * ptdesc)3071 static inline void pmd_ptlock_free(struct ptdesc *ptdesc)
3072 {
3073 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3074 VM_BUG_ON_PAGE(ptdesc->pmd_huge_pte, ptdesc_page(ptdesc));
3075 #endif
3076 ptlock_free(ptdesc);
3077 }
3078
3079 #define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte)
3080
3081 #else
3082
pmd_lockptr(struct mm_struct * mm,pmd_t * pmd)3083 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
3084 {
3085 return &mm->page_table_lock;
3086 }
3087
pmd_ptlock_init(struct ptdesc * ptdesc)3088 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; }
pmd_ptlock_free(struct ptdesc * ptdesc)3089 static inline void pmd_ptlock_free(struct ptdesc *ptdesc) {}
3090
3091 #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
3092
3093 #endif
3094
pmd_lock(struct mm_struct * mm,pmd_t * pmd)3095 static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
3096 {
3097 spinlock_t *ptl = pmd_lockptr(mm, pmd);
3098 spin_lock(ptl);
3099 return ptl;
3100 }
3101
pagetable_pmd_ctor(struct ptdesc * ptdesc)3102 static inline bool pagetable_pmd_ctor(struct ptdesc *ptdesc)
3103 {
3104 struct folio *folio = ptdesc_folio(ptdesc);
3105
3106 if (!pmd_ptlock_init(ptdesc))
3107 return false;
3108 __folio_set_pgtable(folio);
3109 lruvec_stat_add_folio(folio, NR_PAGETABLE);
3110 return true;
3111 }
3112
pagetable_pmd_dtor(struct ptdesc * ptdesc)3113 static inline void pagetable_pmd_dtor(struct ptdesc *ptdesc)
3114 {
3115 struct folio *folio = ptdesc_folio(ptdesc);
3116
3117 pmd_ptlock_free(ptdesc);
3118 __folio_clear_pgtable(folio);
3119 lruvec_stat_sub_folio(folio, NR_PAGETABLE);
3120 }
3121
3122 /*
3123 * No scalability reason to split PUD locks yet, but follow the same pattern
3124 * as the PMD locks to make it easier if we decide to. The VM should not be
3125 * considered ready to switch to split PUD locks yet; there may be places
3126 * which need to be converted from page_table_lock.
3127 */
pud_lockptr(struct mm_struct * mm,pud_t * pud)3128 static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud)
3129 {
3130 return &mm->page_table_lock;
3131 }
3132
pud_lock(struct mm_struct * mm,pud_t * pud)3133 static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud)
3134 {
3135 spinlock_t *ptl = pud_lockptr(mm, pud);
3136
3137 spin_lock(ptl);
3138 return ptl;
3139 }
3140
pagetable_pud_ctor(struct ptdesc * ptdesc)3141 static inline void pagetable_pud_ctor(struct ptdesc *ptdesc)
3142 {
3143 struct folio *folio = ptdesc_folio(ptdesc);
3144
3145 __folio_set_pgtable(folio);
3146 lruvec_stat_add_folio(folio, NR_PAGETABLE);
3147 }
3148
pagetable_pud_dtor(struct ptdesc * ptdesc)3149 static inline void pagetable_pud_dtor(struct ptdesc *ptdesc)
3150 {
3151 struct folio *folio = ptdesc_folio(ptdesc);
3152
3153 __folio_clear_pgtable(folio);
3154 lruvec_stat_sub_folio(folio, NR_PAGETABLE);
3155 }
3156
3157 extern void __init pagecache_init(void);
3158 extern void free_initmem(void);
3159
3160 /*
3161 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
3162 * into the buddy system. The freed pages will be poisoned with pattern
3163 * "poison" if it's within range [0, UCHAR_MAX].
3164 * Return pages freed into the buddy system.
3165 */
3166 extern unsigned long free_reserved_area(void *start, void *end,
3167 int poison, const char *s);
3168
3169 extern void adjust_managed_page_count(struct page *page, long count);
3170
3171 extern void reserve_bootmem_region(phys_addr_t start,
3172 phys_addr_t end, int nid);
3173
3174 /* Free the reserved page into the buddy system, so it gets managed. */
free_reserved_page(struct page * page)3175 static inline void free_reserved_page(struct page *page)
3176 {
3177 if (mem_alloc_profiling_enabled()) {
3178 union codetag_ref *ref = get_page_tag_ref(page);
3179
3180 if (ref) {
3181 set_codetag_empty(ref);
3182 put_page_tag_ref(ref);
3183 }
3184 }
3185 ClearPageReserved(page);
3186 init_page_count(page);
3187 __free_page(page);
3188 adjust_managed_page_count(page, 1);
3189 }
3190 #define free_highmem_page(page) free_reserved_page(page)
3191
mark_page_reserved(struct page * page)3192 static inline void mark_page_reserved(struct page *page)
3193 {
3194 SetPageReserved(page);
3195 adjust_managed_page_count(page, -1);
3196 }
3197
free_reserved_ptdesc(struct ptdesc * pt)3198 static inline void free_reserved_ptdesc(struct ptdesc *pt)
3199 {
3200 free_reserved_page(ptdesc_page(pt));
3201 }
3202
3203 /*
3204 * Default method to free all the __init memory into the buddy system.
3205 * The freed pages will be poisoned with pattern "poison" if it's within
3206 * range [0, UCHAR_MAX].
3207 * Return pages freed into the buddy system.
3208 */
free_initmem_default(int poison)3209 static inline unsigned long free_initmem_default(int poison)
3210 {
3211 extern char __init_begin[], __init_end[];
3212
3213 return free_reserved_area(&__init_begin, &__init_end,
3214 poison, "unused kernel image (initmem)");
3215 }
3216
get_num_physpages(void)3217 static inline unsigned long get_num_physpages(void)
3218 {
3219 int nid;
3220 unsigned long phys_pages = 0;
3221
3222 for_each_online_node(nid)
3223 phys_pages += node_present_pages(nid);
3224
3225 return phys_pages;
3226 }
3227
3228 /*
3229 * Using memblock node mappings, an architecture may initialise its
3230 * zones, allocate the backing mem_map and account for memory holes in an
3231 * architecture independent manner.
3232 *
3233 * An architecture is expected to register range of page frames backed by
3234 * physical memory with memblock_add[_node]() before calling
3235 * free_area_init() passing in the PFN each zone ends at. At a basic
3236 * usage, an architecture is expected to do something like
3237 *
3238 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
3239 * max_highmem_pfn};
3240 * for_each_valid_physical_page_range()
3241 * memblock_add_node(base, size, nid, MEMBLOCK_NONE)
3242 * free_area_init(max_zone_pfns);
3243 */
3244 void free_area_init(unsigned long *max_zone_pfn);
3245 unsigned long node_map_pfn_alignment(void);
3246 extern unsigned long absent_pages_in_range(unsigned long start_pfn,
3247 unsigned long end_pfn);
3248 extern void get_pfn_range_for_nid(unsigned int nid,
3249 unsigned long *start_pfn, unsigned long *end_pfn);
3250
3251 #ifndef CONFIG_NUMA
early_pfn_to_nid(unsigned long pfn)3252 static inline int early_pfn_to_nid(unsigned long pfn)
3253 {
3254 return 0;
3255 }
3256 #else
3257 /* please see mm/page_alloc.c */
3258 extern int __meminit early_pfn_to_nid(unsigned long pfn);
3259 #endif
3260
3261 extern void mem_init(void);
3262 extern void __init mmap_init(void);
3263
3264 extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx);
show_mem(void)3265 static inline void show_mem(void)
3266 {
3267 __show_mem(0, NULL, MAX_NR_ZONES - 1);
3268 }
3269 extern long si_mem_available(void);
3270 extern void si_meminfo(struct sysinfo * val);
3271 extern void si_meminfo_node(struct sysinfo *val, int nid);
3272
3273 extern __printf(3, 4)
3274 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...);
3275
3276 extern void setup_per_cpu_pageset(void);
3277
3278 /* nommu.c */
3279 extern atomic_long_t mmap_pages_allocated;
3280 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
3281
3282 /* interval_tree.c */
3283 void vma_interval_tree_insert(struct vm_area_struct *node,
3284 struct rb_root_cached *root);
3285 void vma_interval_tree_insert_after(struct vm_area_struct *node,
3286 struct vm_area_struct *prev,
3287 struct rb_root_cached *root);
3288 void vma_interval_tree_remove(struct vm_area_struct *node,
3289 struct rb_root_cached *root);
3290 struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root,
3291 unsigned long start, unsigned long last);
3292 struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
3293 unsigned long start, unsigned long last);
3294
3295 #define vma_interval_tree_foreach(vma, root, start, last) \
3296 for (vma = vma_interval_tree_iter_first(root, start, last); \
3297 vma; vma = vma_interval_tree_iter_next(vma, start, last))
3298
3299 void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
3300 struct rb_root_cached *root);
3301 void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
3302 struct rb_root_cached *root);
3303 struct anon_vma_chain *
3304 anon_vma_interval_tree_iter_first(struct rb_root_cached *root,
3305 unsigned long start, unsigned long last);
3306 struct anon_vma_chain *anon_vma_interval_tree_iter_next(
3307 struct anon_vma_chain *node, unsigned long start, unsigned long last);
3308 #ifdef CONFIG_DEBUG_VM_RB
3309 void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
3310 #endif
3311
3312 #define anon_vma_interval_tree_foreach(avc, root, start, last) \
3313 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
3314 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
3315
3316 /* mmap.c */
3317 extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
3318 extern int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
3319 unsigned long start, unsigned long end, pgoff_t pgoff,
3320 struct vm_area_struct *next);
3321 extern int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
3322 unsigned long start, unsigned long end, pgoff_t pgoff);
3323 extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
3324 extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
3325 extern void unlink_file_vma(struct vm_area_struct *);
3326 extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
3327 unsigned long addr, unsigned long len, pgoff_t pgoff,
3328 bool *need_rmap_locks);
3329 extern void exit_mmap(struct mm_struct *);
3330 struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
3331 struct vm_area_struct *prev,
3332 struct vm_area_struct *vma,
3333 unsigned long start, unsigned long end,
3334 unsigned long vm_flags,
3335 struct mempolicy *policy,
3336 struct vm_userfaultfd_ctx uffd_ctx,
3337 struct anon_vma_name *anon_name);
3338
3339 /* We are about to modify the VMA's flags. */
3340 static inline struct vm_area_struct
vma_modify_flags(struct vma_iterator * vmi,struct vm_area_struct * prev,struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long new_flags)3341 *vma_modify_flags(struct vma_iterator *vmi,
3342 struct vm_area_struct *prev,
3343 struct vm_area_struct *vma,
3344 unsigned long start, unsigned long end,
3345 unsigned long new_flags)
3346 {
3347 return vma_modify(vmi, prev, vma, start, end, new_flags,
3348 vma_policy(vma), vma->vm_userfaultfd_ctx,
3349 anon_vma_name(vma));
3350 }
3351
3352 /* We are about to modify the VMA's flags and/or anon_name. */
3353 static inline struct vm_area_struct
vma_modify_flags_name(struct vma_iterator * vmi,struct vm_area_struct * prev,struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long new_flags,struct anon_vma_name * new_name)3354 *vma_modify_flags_name(struct vma_iterator *vmi,
3355 struct vm_area_struct *prev,
3356 struct vm_area_struct *vma,
3357 unsigned long start,
3358 unsigned long end,
3359 unsigned long new_flags,
3360 struct anon_vma_name *new_name)
3361 {
3362 return vma_modify(vmi, prev, vma, start, end, new_flags,
3363 vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);
3364 }
3365
3366 /* We are about to modify the VMA's memory policy. */
3367 static inline struct vm_area_struct
vma_modify_policy(struct vma_iterator * vmi,struct vm_area_struct * prev,struct vm_area_struct * vma,unsigned long start,unsigned long end,struct mempolicy * new_pol)3368 *vma_modify_policy(struct vma_iterator *vmi,
3369 struct vm_area_struct *prev,
3370 struct vm_area_struct *vma,
3371 unsigned long start, unsigned long end,
3372 struct mempolicy *new_pol)
3373 {
3374 return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,
3375 new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));
3376 }
3377
3378 /* We are about to modify the VMA's flags and/or uffd context. */
3379 static inline struct vm_area_struct
vma_modify_flags_uffd(struct vma_iterator * vmi,struct vm_area_struct * prev,struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long new_flags,struct vm_userfaultfd_ctx new_ctx)3380 *vma_modify_flags_uffd(struct vma_iterator *vmi,
3381 struct vm_area_struct *prev,
3382 struct vm_area_struct *vma,
3383 unsigned long start, unsigned long end,
3384 unsigned long new_flags,
3385 struct vm_userfaultfd_ctx new_ctx)
3386 {
3387 return vma_modify(vmi, prev, vma, start, end, new_flags,
3388 vma_policy(vma), new_ctx, anon_vma_name(vma));
3389 }
3390
check_data_rlimit(unsigned long rlim,unsigned long new,unsigned long start,unsigned long end_data,unsigned long start_data)3391 static inline int check_data_rlimit(unsigned long rlim,
3392 unsigned long new,
3393 unsigned long start,
3394 unsigned long end_data,
3395 unsigned long start_data)
3396 {
3397 if (rlim < RLIM_INFINITY) {
3398 if (((new - start) + (end_data - start_data)) > rlim)
3399 return -ENOSPC;
3400 }
3401
3402 return 0;
3403 }
3404
3405 extern int mm_take_all_locks(struct mm_struct *mm);
3406 extern void mm_drop_all_locks(struct mm_struct *mm);
3407
3408 extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
3409 extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
3410 extern struct file *get_mm_exe_file(struct mm_struct *mm);
3411 extern struct file *get_task_exe_file(struct task_struct *task);
3412
3413 extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages);
3414 extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
3415
3416 extern bool vma_is_special_mapping(const struct vm_area_struct *vma,
3417 const struct vm_special_mapping *sm);
3418 extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
3419 unsigned long addr, unsigned long len,
3420 unsigned long flags,
3421 const struct vm_special_mapping *spec);
3422 /* This is an obsolete alternative to _install_special_mapping. */
3423 extern int install_special_mapping(struct mm_struct *mm,
3424 unsigned long addr, unsigned long len,
3425 unsigned long flags, struct page **pages);
3426
3427 unsigned long randomize_stack_top(unsigned long stack_top);
3428 unsigned long randomize_page(unsigned long start, unsigned long range);
3429
3430 unsigned long
3431 __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
3432 unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags);
3433
3434 static inline unsigned long
get_unmapped_area(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)3435 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
3436 unsigned long pgoff, unsigned long flags)
3437 {
3438 return __get_unmapped_area(file, addr, len, pgoff, flags, 0);
3439 }
3440
3441 extern unsigned long mmap_region(struct file *file, unsigned long addr,
3442 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
3443 struct list_head *uf);
3444 extern unsigned long do_mmap(struct file *file, unsigned long addr,
3445 unsigned long len, unsigned long prot, unsigned long flags,
3446 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate,
3447 struct list_head *uf);
3448 extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
3449 unsigned long start, size_t len, struct list_head *uf,
3450 bool unlock);
3451 extern int do_munmap(struct mm_struct *, unsigned long, size_t,
3452 struct list_head *uf);
3453 extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior);
3454
3455 #ifdef CONFIG_MMU
3456 extern int do_vma_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
3457 unsigned long start, unsigned long end,
3458 struct list_head *uf, bool unlock);
3459 extern int __mm_populate(unsigned long addr, unsigned long len,
3460 int ignore_errors);
mm_populate(unsigned long addr,unsigned long len)3461 static inline void mm_populate(unsigned long addr, unsigned long len)
3462 {
3463 /* Ignore errors */
3464 (void) __mm_populate(addr, len, 1);
3465 }
3466 #else
mm_populate(unsigned long addr,unsigned long len)3467 static inline void mm_populate(unsigned long addr, unsigned long len) {}
3468 #endif
3469
3470 /* This takes the mm semaphore itself */
3471 extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long);
3472 extern int vm_munmap(unsigned long, size_t);
3473 extern unsigned long __must_check vm_mmap(struct file *, unsigned long,
3474 unsigned long, unsigned long,
3475 unsigned long, unsigned long);
3476
3477 struct vm_unmapped_area_info {
3478 #define VM_UNMAPPED_AREA_TOPDOWN 1
3479 unsigned long flags;
3480 unsigned long length;
3481 unsigned long low_limit;
3482 unsigned long high_limit;
3483 unsigned long align_mask;
3484 unsigned long align_offset;
3485 unsigned long start_gap;
3486 };
3487
3488 extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info);
3489
3490 /* truncate.c */
3491 extern void truncate_inode_pages(struct address_space *, loff_t);
3492 extern void truncate_inode_pages_range(struct address_space *,
3493 loff_t lstart, loff_t lend);
3494 extern void truncate_inode_pages_final(struct address_space *);
3495
3496 /* generic vm_area_ops exported for stackable file systems */
3497 extern vm_fault_t filemap_fault(struct vm_fault *vmf);
3498 extern vm_fault_t filemap_map_pages(struct vm_fault *vmf,
3499 pgoff_t start_pgoff, pgoff_t end_pgoff);
3500 extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf);
3501
3502 extern unsigned long stack_guard_gap;
3503 /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
3504 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address);
3505 struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr);
3506
3507 /* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */
3508 int expand_downwards(struct vm_area_struct *vma, unsigned long address);
3509
3510 /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
3511 extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
3512 extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
3513 struct vm_area_struct **pprev);
3514
3515 /*
3516 * Look up the first VMA which intersects the interval [start_addr, end_addr)
3517 * NULL if none. Assume start_addr < end_addr.
3518 */
3519 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
3520 unsigned long start_addr, unsigned long end_addr);
3521
3522 /**
3523 * vma_lookup() - Find a VMA at a specific address
3524 * @mm: The process address space.
3525 * @addr: The user address.
3526 *
3527 * Return: The vm_area_struct at the given address, %NULL otherwise.
3528 */
3529 static inline
vma_lookup(struct mm_struct * mm,unsigned long addr)3530 struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr)
3531 {
3532 return mtree_load(&mm->mm_mt, addr);
3533 }
3534
stack_guard_start_gap(struct vm_area_struct * vma)3535 static inline unsigned long stack_guard_start_gap(struct vm_area_struct *vma)
3536 {
3537 if (vma->vm_flags & VM_GROWSDOWN)
3538 return stack_guard_gap;
3539
3540 /* See reasoning around the VM_SHADOW_STACK definition */
3541 if (vma->vm_flags & VM_SHADOW_STACK)
3542 return PAGE_SIZE;
3543
3544 return 0;
3545 }
3546
vm_start_gap(struct vm_area_struct * vma)3547 static inline unsigned long vm_start_gap(struct vm_area_struct *vma)
3548 {
3549 unsigned long gap = stack_guard_start_gap(vma);
3550 unsigned long vm_start = vma->vm_start;
3551
3552 vm_start -= gap;
3553 if (vm_start > vma->vm_start)
3554 vm_start = 0;
3555 return vm_start;
3556 }
3557
vm_end_gap(struct vm_area_struct * vma)3558 static inline unsigned long vm_end_gap(struct vm_area_struct *vma)
3559 {
3560 unsigned long vm_end = vma->vm_end;
3561
3562 if (vma->vm_flags & VM_GROWSUP) {
3563 vm_end += stack_guard_gap;
3564 if (vm_end < vma->vm_end)
3565 vm_end = -PAGE_SIZE;
3566 }
3567 return vm_end;
3568 }
3569
vma_pages(struct vm_area_struct * vma)3570 static inline unsigned long vma_pages(struct vm_area_struct *vma)
3571 {
3572 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
3573 }
3574
3575 /* Look up the first VMA which exactly match the interval vm_start ... vm_end */
find_exact_vma(struct mm_struct * mm,unsigned long vm_start,unsigned long vm_end)3576 static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
3577 unsigned long vm_start, unsigned long vm_end)
3578 {
3579 struct vm_area_struct *vma = vma_lookup(mm, vm_start);
3580
3581 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
3582 vma = NULL;
3583
3584 return vma;
3585 }
3586
range_in_vma(struct vm_area_struct * vma,unsigned long start,unsigned long end)3587 static inline bool range_in_vma(struct vm_area_struct *vma,
3588 unsigned long start, unsigned long end)
3589 {
3590 return (vma && vma->vm_start <= start && end <= vma->vm_end);
3591 }
3592
3593 #ifdef CONFIG_MMU
3594 pgprot_t vm_get_page_prot(unsigned long vm_flags);
3595 void vma_set_page_prot(struct vm_area_struct *vma);
3596 #else
vm_get_page_prot(unsigned long vm_flags)3597 static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
3598 {
3599 return __pgprot(0);
3600 }
vma_set_page_prot(struct vm_area_struct * vma)3601 static inline void vma_set_page_prot(struct vm_area_struct *vma)
3602 {
3603 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
3604 }
3605 #endif
3606
3607 void vma_set_file(struct vm_area_struct *vma, struct file *file);
3608
3609 #ifdef CONFIG_NUMA_BALANCING
3610 unsigned long change_prot_numa(struct vm_area_struct *vma,
3611 unsigned long start, unsigned long end);
3612 #endif
3613
3614 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *,
3615 unsigned long addr);
3616 int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
3617 unsigned long pfn, unsigned long size, pgprot_t);
3618 int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
3619 unsigned long pfn, unsigned long size, pgprot_t prot);
3620 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
3621 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
3622 struct page **pages, unsigned long *num);
3623 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
3624 unsigned long num);
3625 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
3626 unsigned long num);
3627 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
3628 unsigned long pfn);
3629 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
3630 unsigned long pfn, pgprot_t pgprot);
3631 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
3632 pfn_t pfn);
3633 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
3634 unsigned long addr, pfn_t pfn);
3635 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
3636
vmf_insert_page(struct vm_area_struct * vma,unsigned long addr,struct page * page)3637 static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma,
3638 unsigned long addr, struct page *page)
3639 {
3640 int err = vm_insert_page(vma, addr, page);
3641
3642 if (err == -ENOMEM)
3643 return VM_FAULT_OOM;
3644 if (err < 0 && err != -EBUSY)
3645 return VM_FAULT_SIGBUS;
3646
3647 return VM_FAULT_NOPAGE;
3648 }
3649
3650 #ifndef io_remap_pfn_range
io_remap_pfn_range(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)3651 static inline int io_remap_pfn_range(struct vm_area_struct *vma,
3652 unsigned long addr, unsigned long pfn,
3653 unsigned long size, pgprot_t prot)
3654 {
3655 return remap_pfn_range(vma, addr, pfn, size, pgprot_decrypted(prot));
3656 }
3657 #endif
3658
vmf_error(int err)3659 static inline vm_fault_t vmf_error(int err)
3660 {
3661 if (err == -ENOMEM)
3662 return VM_FAULT_OOM;
3663 else if (err == -EHWPOISON)
3664 return VM_FAULT_HWPOISON;
3665 return VM_FAULT_SIGBUS;
3666 }
3667
3668 /*
3669 * Convert errno to return value for ->page_mkwrite() calls.
3670 *
3671 * This should eventually be merged with vmf_error() above, but will need a
3672 * careful audit of all vmf_error() callers.
3673 */
vmf_fs_error(int err)3674 static inline vm_fault_t vmf_fs_error(int err)
3675 {
3676 if (err == 0)
3677 return VM_FAULT_LOCKED;
3678 if (err == -EFAULT || err == -EAGAIN)
3679 return VM_FAULT_NOPAGE;
3680 if (err == -ENOMEM)
3681 return VM_FAULT_OOM;
3682 /* -ENOSPC, -EDQUOT, -EIO ... */
3683 return VM_FAULT_SIGBUS;
3684 }
3685
3686 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
3687 unsigned int foll_flags);
3688
vm_fault_to_errno(vm_fault_t vm_fault,int foll_flags)3689 static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags)
3690 {
3691 if (vm_fault & VM_FAULT_OOM)
3692 return -ENOMEM;
3693 if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE))
3694 return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT;
3695 if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV))
3696 return -EFAULT;
3697 return 0;
3698 }
3699
3700 /*
3701 * Indicates whether GUP can follow a PROT_NONE mapped page, or whether
3702 * a (NUMA hinting) fault is required.
3703 */
gup_can_follow_protnone(struct vm_area_struct * vma,unsigned int flags)3704 static inline bool gup_can_follow_protnone(struct vm_area_struct *vma,
3705 unsigned int flags)
3706 {
3707 /*
3708 * If callers don't want to honor NUMA hinting faults, no need to
3709 * determine if we would actually have to trigger a NUMA hinting fault.
3710 */
3711 if (!(flags & FOLL_HONOR_NUMA_FAULT))
3712 return true;
3713
3714 /*
3715 * NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs.
3716 *
3717 * Requiring a fault here even for inaccessible VMAs would mean that
3718 * FOLL_FORCE cannot make any progress, because handle_mm_fault()
3719 * refuses to process NUMA hinting faults in inaccessible VMAs.
3720 */
3721 return !vma_is_accessible(vma);
3722 }
3723
3724 typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data);
3725 extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
3726 unsigned long size, pte_fn_t fn, void *data);
3727 extern int apply_to_existing_page_range(struct mm_struct *mm,
3728 unsigned long address, unsigned long size,
3729 pte_fn_t fn, void *data);
3730
3731 #ifdef CONFIG_PAGE_POISONING
3732 extern void __kernel_poison_pages(struct page *page, int numpages);
3733 extern void __kernel_unpoison_pages(struct page *page, int numpages);
3734 extern bool _page_poisoning_enabled_early;
3735 DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled);
page_poisoning_enabled(void)3736 static inline bool page_poisoning_enabled(void)
3737 {
3738 return _page_poisoning_enabled_early;
3739 }
3740 /*
3741 * For use in fast paths after init_mem_debugging() has run, or when a
3742 * false negative result is not harmful when called too early.
3743 */
page_poisoning_enabled_static(void)3744 static inline bool page_poisoning_enabled_static(void)
3745 {
3746 return static_branch_unlikely(&_page_poisoning_enabled);
3747 }
kernel_poison_pages(struct page * page,int numpages)3748 static inline void kernel_poison_pages(struct page *page, int numpages)
3749 {
3750 if (page_poisoning_enabled_static())
3751 __kernel_poison_pages(page, numpages);
3752 }
kernel_unpoison_pages(struct page * page,int numpages)3753 static inline void kernel_unpoison_pages(struct page *page, int numpages)
3754 {
3755 if (page_poisoning_enabled_static())
3756 __kernel_unpoison_pages(page, numpages);
3757 }
3758 #else
page_poisoning_enabled(void)3759 static inline bool page_poisoning_enabled(void) { return false; }
page_poisoning_enabled_static(void)3760 static inline bool page_poisoning_enabled_static(void) { return false; }
__kernel_poison_pages(struct page * page,int nunmpages)3761 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { }
kernel_poison_pages(struct page * page,int numpages)3762 static inline void kernel_poison_pages(struct page *page, int numpages) { }
kernel_unpoison_pages(struct page * page,int numpages)3763 static inline void kernel_unpoison_pages(struct page *page, int numpages) { }
3764 #endif
3765
3766 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
want_init_on_alloc(gfp_t flags)3767 static inline bool want_init_on_alloc(gfp_t flags)
3768 {
3769 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
3770 &init_on_alloc))
3771 return true;
3772 return flags & __GFP_ZERO;
3773 }
3774
3775 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
want_init_on_free(void)3776 static inline bool want_init_on_free(void)
3777 {
3778 return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
3779 &init_on_free);
3780 }
3781
3782 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON, init_mlocked_on_free);
want_init_mlocked_on_free(void)3783 static inline bool want_init_mlocked_on_free(void)
3784 {
3785 return static_branch_maybe(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON,
3786 &init_mlocked_on_free);
3787 }
3788
3789 extern bool _debug_pagealloc_enabled_early;
3790 DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
3791
debug_pagealloc_enabled(void)3792 static inline bool debug_pagealloc_enabled(void)
3793 {
3794 return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) &&
3795 _debug_pagealloc_enabled_early;
3796 }
3797
3798 /*
3799 * For use in fast paths after mem_debugging_and_hardening_init() has run,
3800 * or when a false negative result is not harmful when called too early.
3801 */
debug_pagealloc_enabled_static(void)3802 static inline bool debug_pagealloc_enabled_static(void)
3803 {
3804 if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC))
3805 return false;
3806
3807 return static_branch_unlikely(&_debug_pagealloc_enabled);
3808 }
3809
3810 /*
3811 * To support DEBUG_PAGEALLOC architecture must ensure that
3812 * __kernel_map_pages() never fails
3813 */
3814 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
3815 #ifdef CONFIG_DEBUG_PAGEALLOC
debug_pagealloc_map_pages(struct page * page,int numpages)3816 static inline void debug_pagealloc_map_pages(struct page *page, int numpages)
3817 {
3818 if (debug_pagealloc_enabled_static())
3819 __kernel_map_pages(page, numpages, 1);
3820 }
3821
debug_pagealloc_unmap_pages(struct page * page,int numpages)3822 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages)
3823 {
3824 if (debug_pagealloc_enabled_static())
3825 __kernel_map_pages(page, numpages, 0);
3826 }
3827
3828 extern unsigned int _debug_guardpage_minorder;
3829 DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
3830
debug_guardpage_minorder(void)3831 static inline unsigned int debug_guardpage_minorder(void)
3832 {
3833 return _debug_guardpage_minorder;
3834 }
3835
debug_guardpage_enabled(void)3836 static inline bool debug_guardpage_enabled(void)
3837 {
3838 return static_branch_unlikely(&_debug_guardpage_enabled);
3839 }
3840
page_is_guard(struct page * page)3841 static inline bool page_is_guard(struct page *page)
3842 {
3843 if (!debug_guardpage_enabled())
3844 return false;
3845
3846 return PageGuard(page);
3847 }
3848
3849 bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order);
set_page_guard(struct zone * zone,struct page * page,unsigned int order)3850 static inline bool set_page_guard(struct zone *zone, struct page *page,
3851 unsigned int order)
3852 {
3853 if (!debug_guardpage_enabled())
3854 return false;
3855 return __set_page_guard(zone, page, order);
3856 }
3857
3858 void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order);
clear_page_guard(struct zone * zone,struct page * page,unsigned int order)3859 static inline void clear_page_guard(struct zone *zone, struct page *page,
3860 unsigned int order)
3861 {
3862 if (!debug_guardpage_enabled())
3863 return;
3864 __clear_page_guard(zone, page, order);
3865 }
3866
3867 #else /* CONFIG_DEBUG_PAGEALLOC */
debug_pagealloc_map_pages(struct page * page,int numpages)3868 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {}
debug_pagealloc_unmap_pages(struct page * page,int numpages)3869 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {}
debug_guardpage_minorder(void)3870 static inline unsigned int debug_guardpage_minorder(void) { return 0; }
debug_guardpage_enabled(void)3871 static inline bool debug_guardpage_enabled(void) { return false; }
page_is_guard(struct page * page)3872 static inline bool page_is_guard(struct page *page) { return false; }
set_page_guard(struct zone * zone,struct page * page,unsigned int order)3873 static inline bool set_page_guard(struct zone *zone, struct page *page,
3874 unsigned int order) { return false; }
clear_page_guard(struct zone * zone,struct page * page,unsigned int order)3875 static inline void clear_page_guard(struct zone *zone, struct page *page,
3876 unsigned int order) {}
3877 #endif /* CONFIG_DEBUG_PAGEALLOC */
3878
3879 #ifdef __HAVE_ARCH_GATE_AREA
3880 extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
3881 extern int in_gate_area_no_mm(unsigned long addr);
3882 extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
3883 #else
get_gate_vma(struct mm_struct * mm)3884 static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
3885 {
3886 return NULL;
3887 }
in_gate_area_no_mm(unsigned long addr)3888 static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
in_gate_area(struct mm_struct * mm,unsigned long addr)3889 static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
3890 {
3891 return 0;
3892 }
3893 #endif /* __HAVE_ARCH_GATE_AREA */
3894
3895 extern bool process_shares_mm(struct task_struct *p, struct mm_struct *mm);
3896
3897 #ifdef CONFIG_SYSCTL
3898 extern int sysctl_drop_caches;
3899 int drop_caches_sysctl_handler(struct ctl_table *, int, void *, size_t *,
3900 loff_t *);
3901 #endif
3902
3903 void drop_slab(void);
3904
3905 #ifndef CONFIG_MMU
3906 #define randomize_va_space 0
3907 #else
3908 extern int randomize_va_space;
3909 #endif
3910
3911 const char * arch_vma_name(struct vm_area_struct *vma);
3912 #ifdef CONFIG_MMU
3913 void print_vma_addr(char *prefix, unsigned long rip);
3914 #else
print_vma_addr(char * prefix,unsigned long rip)3915 static inline void print_vma_addr(char *prefix, unsigned long rip)
3916 {
3917 }
3918 #endif
3919
3920 void *sparse_buffer_alloc(unsigned long size);
3921 struct page * __populate_section_memmap(unsigned long pfn,
3922 unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
3923 struct dev_pagemap *pgmap);
3924 void pmd_init(void *addr);
3925 void pud_init(void *addr);
3926 pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
3927 p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
3928 pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
3929 pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
3930 pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
3931 struct vmem_altmap *altmap, struct page *reuse);
3932 void *vmemmap_alloc_block(unsigned long size, int node);
3933 struct vmem_altmap;
3934 void *vmemmap_alloc_block_buf(unsigned long size, int node,
3935 struct vmem_altmap *altmap);
3936 void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
3937 void vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
3938 unsigned long addr, unsigned long next);
3939 int vmemmap_check_pmd(pmd_t *pmd, int node,
3940 unsigned long addr, unsigned long next);
3941 int vmemmap_populate_basepages(unsigned long start, unsigned long end,
3942 int node, struct vmem_altmap *altmap);
3943 int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
3944 int node, struct vmem_altmap *altmap);
3945 int vmemmap_populate(unsigned long start, unsigned long end, int node,
3946 struct vmem_altmap *altmap);
3947 void vmemmap_populate_print_last(void);
3948 #ifdef CONFIG_MEMORY_HOTPLUG
3949 void vmemmap_free(unsigned long start, unsigned long end,
3950 struct vmem_altmap *altmap);
3951 #endif
3952
3953 #ifdef CONFIG_SPARSEMEM_VMEMMAP
vmem_altmap_offset(struct vmem_altmap * altmap)3954 static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
3955 {
3956 /* number of pfns from base where pfn_to_page() is valid */
3957 if (altmap)
3958 return altmap->reserve + altmap->free;
3959 return 0;
3960 }
3961
vmem_altmap_free(struct vmem_altmap * altmap,unsigned long nr_pfns)3962 static inline void vmem_altmap_free(struct vmem_altmap *altmap,
3963 unsigned long nr_pfns)
3964 {
3965 altmap->alloc -= nr_pfns;
3966 }
3967 #else
vmem_altmap_offset(struct vmem_altmap * altmap)3968 static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
3969 {
3970 return 0;
3971 }
3972
vmem_altmap_free(struct vmem_altmap * altmap,unsigned long nr_pfns)3973 static inline void vmem_altmap_free(struct vmem_altmap *altmap,
3974 unsigned long nr_pfns)
3975 {
3976 }
3977 #endif
3978
3979 #define VMEMMAP_RESERVE_NR 2
3980 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
__vmemmap_can_optimize(struct vmem_altmap * altmap,struct dev_pagemap * pgmap)3981 static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap,
3982 struct dev_pagemap *pgmap)
3983 {
3984 unsigned long nr_pages;
3985 unsigned long nr_vmemmap_pages;
3986
3987 if (!pgmap || !is_power_of_2(sizeof(struct page)))
3988 return false;
3989
3990 nr_pages = pgmap_vmemmap_nr(pgmap);
3991 nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT);
3992 /*
3993 * For vmemmap optimization with DAX we need minimum 2 vmemmap
3994 * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst
3995 */
3996 return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR);
3997 }
3998 /*
3999 * If we don't have an architecture override, use the generic rule
4000 */
4001 #ifndef vmemmap_can_optimize
4002 #define vmemmap_can_optimize __vmemmap_can_optimize
4003 #endif
4004
4005 #else
vmemmap_can_optimize(struct vmem_altmap * altmap,struct dev_pagemap * pgmap)4006 static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap,
4007 struct dev_pagemap *pgmap)
4008 {
4009 return false;
4010 }
4011 #endif
4012
4013 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
4014 unsigned long nr_pages);
4015
4016 enum mf_flags {
4017 MF_COUNT_INCREASED = 1 << 0,
4018 MF_ACTION_REQUIRED = 1 << 1,
4019 MF_MUST_KILL = 1 << 2,
4020 MF_SOFT_OFFLINE = 1 << 3,
4021 MF_UNPOISON = 1 << 4,
4022 MF_SW_SIMULATED = 1 << 5,
4023 MF_NO_RETRY = 1 << 6,
4024 MF_MEM_PRE_REMOVE = 1 << 7,
4025 };
4026 int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index,
4027 unsigned long count, int mf_flags);
4028 extern int memory_failure(unsigned long pfn, int flags);
4029 extern void memory_failure_queue_kick(int cpu);
4030 extern int unpoison_memory(unsigned long pfn);
4031 extern atomic_long_t num_poisoned_pages __read_mostly;
4032 extern int soft_offline_page(unsigned long pfn, int flags);
4033 #ifdef CONFIG_MEMORY_FAILURE
4034 /*
4035 * Sysfs entries for memory failure handling statistics.
4036 */
4037 extern const struct attribute_group memory_failure_attr_group;
4038 extern void memory_failure_queue(unsigned long pfn, int flags);
4039 extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
4040 bool *migratable_cleared);
4041 void num_poisoned_pages_inc(unsigned long pfn);
4042 void num_poisoned_pages_sub(unsigned long pfn, long i);
4043 struct task_struct *task_early_kill(struct task_struct *tsk, int force_early);
4044 #else
memory_failure_queue(unsigned long pfn,int flags)4045 static inline void memory_failure_queue(unsigned long pfn, int flags)
4046 {
4047 }
4048
__get_huge_page_for_hwpoison(unsigned long pfn,int flags,bool * migratable_cleared)4049 static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
4050 bool *migratable_cleared)
4051 {
4052 return 0;
4053 }
4054
num_poisoned_pages_inc(unsigned long pfn)4055 static inline void num_poisoned_pages_inc(unsigned long pfn)
4056 {
4057 }
4058
num_poisoned_pages_sub(unsigned long pfn,long i)4059 static inline void num_poisoned_pages_sub(unsigned long pfn, long i)
4060 {
4061 }
4062 #endif
4063
4064 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_KSM)
4065 void add_to_kill_ksm(struct task_struct *tsk, struct page *p,
4066 struct vm_area_struct *vma, struct list_head *to_kill,
4067 unsigned long ksm_addr);
4068 #endif
4069
4070 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)
4071 extern void memblk_nr_poison_inc(unsigned long pfn);
4072 extern void memblk_nr_poison_sub(unsigned long pfn, long i);
4073 #else
memblk_nr_poison_inc(unsigned long pfn)4074 static inline void memblk_nr_poison_inc(unsigned long pfn)
4075 {
4076 }
4077
memblk_nr_poison_sub(unsigned long pfn,long i)4078 static inline void memblk_nr_poison_sub(unsigned long pfn, long i)
4079 {
4080 }
4081 #endif
4082
4083 #ifndef arch_memory_failure
arch_memory_failure(unsigned long pfn,int flags)4084 static inline int arch_memory_failure(unsigned long pfn, int flags)
4085 {
4086 return -ENXIO;
4087 }
4088 #endif
4089
4090 #ifndef arch_is_platform_page
arch_is_platform_page(u64 paddr)4091 static inline bool arch_is_platform_page(u64 paddr)
4092 {
4093 return false;
4094 }
4095 #endif
4096
4097 /*
4098 * Error handlers for various types of pages.
4099 */
4100 enum mf_result {
4101 MF_IGNORED, /* Error: cannot be handled */
4102 MF_FAILED, /* Error: handling failed */
4103 MF_DELAYED, /* Will be handled later */
4104 MF_RECOVERED, /* Successfully recovered */
4105 };
4106
4107 enum mf_action_page_type {
4108 MF_MSG_KERNEL,
4109 MF_MSG_KERNEL_HIGH_ORDER,
4110 MF_MSG_SLAB,
4111 MF_MSG_DIFFERENT_COMPOUND,
4112 MF_MSG_HUGE,
4113 MF_MSG_FREE_HUGE,
4114 MF_MSG_UNMAP_FAILED,
4115 MF_MSG_DIRTY_SWAPCACHE,
4116 MF_MSG_CLEAN_SWAPCACHE,
4117 MF_MSG_DIRTY_MLOCKED_LRU,
4118 MF_MSG_CLEAN_MLOCKED_LRU,
4119 MF_MSG_DIRTY_UNEVICTABLE_LRU,
4120 MF_MSG_CLEAN_UNEVICTABLE_LRU,
4121 MF_MSG_DIRTY_LRU,
4122 MF_MSG_CLEAN_LRU,
4123 MF_MSG_TRUNCATED_LRU,
4124 MF_MSG_BUDDY,
4125 MF_MSG_DAX,
4126 MF_MSG_UNSPLIT_THP,
4127 MF_MSG_UNKNOWN,
4128 };
4129
4130 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
4131 extern void clear_huge_page(struct page *page,
4132 unsigned long addr_hint,
4133 unsigned int pages_per_huge_page);
4134 int copy_user_large_folio(struct folio *dst, struct folio *src,
4135 unsigned long addr_hint,
4136 struct vm_area_struct *vma);
4137 long copy_folio_from_user(struct folio *dst_folio,
4138 const void __user *usr_src,
4139 bool allow_pagefault);
4140
4141 /**
4142 * vma_is_special_huge - Are transhuge page-table entries considered special?
4143 * @vma: Pointer to the struct vm_area_struct to consider
4144 *
4145 * Whether transhuge page-table entries are considered "special" following
4146 * the definition in vm_normal_page().
4147 *
4148 * Return: true if transhuge page-table entries should be considered special,
4149 * false otherwise.
4150 */
vma_is_special_huge(const struct vm_area_struct * vma)4151 static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
4152 {
4153 return vma_is_dax(vma) || (vma->vm_file &&
4154 (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
4155 }
4156
4157 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
4158
4159 #if MAX_NUMNODES > 1
4160 void __init setup_nr_node_ids(void);
4161 #else
setup_nr_node_ids(void)4162 static inline void setup_nr_node_ids(void) {}
4163 #endif
4164
4165 extern int memcmp_pages(struct page *page1, struct page *page2);
4166
pages_identical(struct page * page1,struct page * page2)4167 static inline int pages_identical(struct page *page1, struct page *page2)
4168 {
4169 return !memcmp_pages(page1, page2);
4170 }
4171
4172 #ifdef CONFIG_MAPPING_DIRTY_HELPERS
4173 unsigned long clean_record_shared_mapping_range(struct address_space *mapping,
4174 pgoff_t first_index, pgoff_t nr,
4175 pgoff_t bitmap_pgoff,
4176 unsigned long *bitmap,
4177 pgoff_t *start,
4178 pgoff_t *end);
4179
4180 unsigned long wp_shared_mapping_range(struct address_space *mapping,
4181 pgoff_t first_index, pgoff_t nr);
4182 #endif
4183
4184 extern int sysctl_nr_trim_pages;
4185
4186 #ifdef CONFIG_PRINTK
4187 void mem_dump_obj(void *object);
4188 #else
mem_dump_obj(void * object)4189 static inline void mem_dump_obj(void *object) {}
4190 #endif
4191
4192 /**
4193 * seal_check_write - Check for F_SEAL_WRITE or F_SEAL_FUTURE_WRITE flags and
4194 * handle them.
4195 * @seals: the seals to check
4196 * @vma: the vma to operate on
4197 *
4198 * Check whether F_SEAL_WRITE or F_SEAL_FUTURE_WRITE are set; if so, do proper
4199 * check/handling on the vma flags. Return 0 if check pass, or <0 for errors.
4200 */
seal_check_write(int seals,struct vm_area_struct * vma)4201 static inline int seal_check_write(int seals, struct vm_area_struct *vma)
4202 {
4203 if (seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
4204 /*
4205 * New PROT_WRITE and MAP_SHARED mmaps are not allowed when
4206 * write seals are active.
4207 */
4208 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
4209 return -EPERM;
4210
4211 /*
4212 * Since an F_SEAL_[FUTURE_]WRITE sealed memfd can be mapped as
4213 * MAP_SHARED and read-only, take care to not allow mprotect to
4214 * revert protections on such mappings. Do this only for shared
4215 * mappings. For private mappings, don't need to mask
4216 * VM_MAYWRITE as we still want them to be COW-writable.
4217 */
4218 if (vma->vm_flags & VM_SHARED)
4219 vm_flags_clear(vma, VM_MAYWRITE);
4220 }
4221
4222 return 0;
4223 }
4224
4225 #ifdef CONFIG_ANON_VMA_NAME
4226 int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
4227 unsigned long len_in,
4228 struct anon_vma_name *anon_name);
4229 #else
4230 static inline int
madvise_set_anon_name(struct mm_struct * mm,unsigned long start,unsigned long len_in,struct anon_vma_name * anon_name)4231 madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
4232 unsigned long len_in, struct anon_vma_name *anon_name) {
4233 return 0;
4234 }
4235 #endif
4236
4237 #ifdef CONFIG_UNACCEPTED_MEMORY
4238
4239 bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end);
4240 void accept_memory(phys_addr_t start, phys_addr_t end);
4241
4242 #else
4243
range_contains_unaccepted_memory(phys_addr_t start,phys_addr_t end)4244 static inline bool range_contains_unaccepted_memory(phys_addr_t start,
4245 phys_addr_t end)
4246 {
4247 return false;
4248 }
4249
accept_memory(phys_addr_t start,phys_addr_t end)4250 static inline void accept_memory(phys_addr_t start, phys_addr_t end)
4251 {
4252 }
4253
4254 #endif
4255
pfn_is_unaccepted_memory(unsigned long pfn)4256 static inline bool pfn_is_unaccepted_memory(unsigned long pfn)
4257 {
4258 phys_addr_t paddr = pfn << PAGE_SHIFT;
4259
4260 return range_contains_unaccepted_memory(paddr, paddr + PAGE_SIZE);
4261 }
4262
4263 void vma_pgtable_walk_begin(struct vm_area_struct *vma);
4264 void vma_pgtable_walk_end(struct vm_area_struct *vma);
4265
4266 #endif /* _LINUX_MM_H */
4267