1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_SCHED_MM_H 3 #define _LINUX_SCHED_MM_H 4 5 #include <linux/kernel.h> 6 #include <linux/atomic.h> 7 #include <linux/sched.h> 8 #include <linux/mm_types.h> 9 #include <linux/gfp.h> 10 #include <linux/sync_core.h> 11 12 /* 13 * Routines for handling mm_structs 14 */ 15 extern struct mm_struct *mm_alloc(void); 16 17 /** 18 * mmgrab() - Pin a &struct mm_struct. 19 * @mm: The &struct mm_struct to pin. 20 * 21 * Make sure that @mm will not get freed even after the owning task 22 * exits. This doesn't guarantee that the associated address space 23 * will still exist later on and mmget_not_zero() has to be used before 24 * accessing it. 25 * 26 * This is a preferred way to to pin @mm for a longer/unbounded amount 27 * of time. 28 * 29 * Use mmdrop() to release the reference acquired by mmgrab(). 30 * 31 * See also <Documentation/vm/active_mm.rst> for an in-depth explanation 32 * of &mm_struct.mm_count vs &mm_struct.mm_users. 33 */ 34 static inline void mmgrab(struct mm_struct *mm) 35 { 36 atomic_inc(&mm->mm_count); 37 } 38 39 extern void __mmdrop(struct mm_struct *mm); 40 41 static inline void mmdrop(struct mm_struct *mm) 42 { 43 /* 44 * The implicit full barrier implied by atomic_dec_and_test() is 45 * required by the membarrier system call before returning to 46 * user-space, after storing to rq->curr. 47 */ 48 if (unlikely(atomic_dec_and_test(&mm->mm_count))) 49 __mmdrop(mm); 50 } 51 52 /* 53 * This has to be called after a get_task_mm()/mmget_not_zero() 54 * followed by taking the mmap_sem for writing before modifying the 55 * vmas or anything the coredump pretends not to change from under it. 56 * 57 * It also has to be called when mmgrab() is used in the context of 58 * the process, but then the mm_count refcount is transferred outside 59 * the context of the process to run down_write() on that pinned mm. 60 * 61 * NOTE: find_extend_vma() called from GUP context is the only place 62 * that can modify the "mm" (notably the vm_start/end) under mmap_sem 63 * for reading and outside the context of the process, so it is also 64 * the only case that holds the mmap_sem for reading that must call 65 * this function. Generally if the mmap_sem is hold for reading 66 * there's no need of this check after get_task_mm()/mmget_not_zero(). 67 * 68 * This function can be obsoleted and the check can be removed, after 69 * the coredump code will hold the mmap_sem for writing before 70 * invoking the ->core_dump methods. 71 */ 72 static inline bool mmget_still_valid(struct mm_struct *mm) 73 { 74 return likely(!mm->core_state); 75 } 76 77 /** 78 * mmget() - Pin the address space associated with a &struct mm_struct. 79 * @mm: The address space to pin. 80 * 81 * Make sure that the address space of the given &struct mm_struct doesn't 82 * go away. This does not protect against parts of the address space being 83 * modified or freed, however. 84 * 85 * Never use this function to pin this address space for an 86 * unbounded/indefinite amount of time. 87 * 88 * Use mmput() to release the reference acquired by mmget(). 89 * 90 * See also <Documentation/vm/active_mm.rst> for an in-depth explanation 91 * of &mm_struct.mm_count vs &mm_struct.mm_users. 92 */ 93 static inline void mmget(struct mm_struct *mm) 94 { 95 atomic_inc(&mm->mm_users); 96 } 97 98 static inline bool mmget_not_zero(struct mm_struct *mm) 99 { 100 return atomic_inc_not_zero(&mm->mm_users); 101 } 102 103 /* mmput gets rid of the mappings and all user-space */ 104 extern void mmput(struct mm_struct *); 105 #ifdef CONFIG_MMU 106 /* same as above but performs the slow path from the async context. Can 107 * be called from the atomic context as well 108 */ 109 void mmput_async(struct mm_struct *); 110 #endif 111 112 /* Grab a reference to a task's mm, if it is not already going away */ 113 extern struct mm_struct *get_task_mm(struct task_struct *task); 114 /* 115 * Grab a reference to a task's mm, if it is not already going away 116 * and ptrace_may_access with the mode parameter passed to it 117 * succeeds. 118 */ 119 extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); 120 /* Remove the current tasks stale references to the old mm_struct on exit() */ 121 extern void exit_mm_release(struct task_struct *, struct mm_struct *); 122 /* Remove the current tasks stale references to the old mm_struct on exec() */ 123 extern void exec_mm_release(struct task_struct *, struct mm_struct *); 124 125 #ifdef CONFIG_MEMCG 126 extern void mm_update_next_owner(struct mm_struct *mm); 127 #else 128 static inline void mm_update_next_owner(struct mm_struct *mm) 129 { 130 } 131 #endif /* CONFIG_MEMCG */ 132 133 #ifdef CONFIG_MMU 134 extern void arch_pick_mmap_layout(struct mm_struct *mm, 135 struct rlimit *rlim_stack); 136 extern unsigned long 137 arch_get_unmapped_area(struct file *, unsigned long, unsigned long, 138 unsigned long, unsigned long); 139 extern unsigned long 140 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 141 unsigned long len, unsigned long pgoff, 142 unsigned long flags); 143 #else 144 static inline void arch_pick_mmap_layout(struct mm_struct *mm, 145 struct rlimit *rlim_stack) {} 146 #endif 147 148 static inline bool in_vfork(struct task_struct *tsk) 149 { 150 bool ret; 151 152 /* 153 * need RCU to access ->real_parent if CLONE_VM was used along with 154 * CLONE_PARENT. 155 * 156 * We check real_parent->mm == tsk->mm because CLONE_VFORK does not 157 * imply CLONE_VM 158 * 159 * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus 160 * ->real_parent is not necessarily the task doing vfork(), so in 161 * theory we can't rely on task_lock() if we want to dereference it. 162 * 163 * And in this case we can't trust the real_parent->mm == tsk->mm 164 * check, it can be false negative. But we do not care, if init or 165 * another oom-unkillable task does this it should blame itself. 166 */ 167 rcu_read_lock(); 168 ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm; 169 rcu_read_unlock(); 170 171 return ret; 172 } 173 174 /* 175 * Applies per-task gfp context to the given allocation flags. 176 * PF_MEMALLOC_NOIO implies GFP_NOIO 177 * PF_MEMALLOC_NOFS implies GFP_NOFS 178 * PF_MEMALLOC_NOCMA implies no allocation from CMA region. 179 */ 180 static inline gfp_t current_gfp_context(gfp_t flags) 181 { 182 if (unlikely(current->flags & 183 (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_NOCMA))) { 184 /* 185 * NOIO implies both NOIO and NOFS and it is a weaker context 186 * so always make sure it makes precedence 187 */ 188 if (current->flags & PF_MEMALLOC_NOIO) 189 flags &= ~(__GFP_IO | __GFP_FS); 190 else if (current->flags & PF_MEMALLOC_NOFS) 191 flags &= ~__GFP_FS; 192 #ifdef CONFIG_CMA 193 if (current->flags & PF_MEMALLOC_NOCMA) 194 flags &= ~__GFP_MOVABLE; 195 #endif 196 } 197 return flags; 198 } 199 200 #ifdef CONFIG_LOCKDEP 201 extern void __fs_reclaim_acquire(void); 202 extern void __fs_reclaim_release(void); 203 extern void fs_reclaim_acquire(gfp_t gfp_mask); 204 extern void fs_reclaim_release(gfp_t gfp_mask); 205 #else 206 static inline void __fs_reclaim_acquire(void) { } 207 static inline void __fs_reclaim_release(void) { } 208 static inline void fs_reclaim_acquire(gfp_t gfp_mask) { } 209 static inline void fs_reclaim_release(gfp_t gfp_mask) { } 210 #endif 211 212 /** 213 * memalloc_noio_save - Marks implicit GFP_NOIO allocation scope. 214 * 215 * This functions marks the beginning of the GFP_NOIO allocation scope. 216 * All further allocations will implicitly drop __GFP_IO flag and so 217 * they are safe for the IO critical section from the allocation recursion 218 * point of view. Use memalloc_noio_restore to end the scope with flags 219 * returned by this function. 220 * 221 * This function is safe to be used from any context. 222 */ 223 static inline unsigned int memalloc_noio_save(void) 224 { 225 unsigned int flags = current->flags & PF_MEMALLOC_NOIO; 226 current->flags |= PF_MEMALLOC_NOIO; 227 return flags; 228 } 229 230 /** 231 * memalloc_noio_restore - Ends the implicit GFP_NOIO scope. 232 * @flags: Flags to restore. 233 * 234 * Ends the implicit GFP_NOIO scope started by memalloc_noio_save function. 235 * Always make sure that that the given flags is the return value from the 236 * pairing memalloc_noio_save call. 237 */ 238 static inline void memalloc_noio_restore(unsigned int flags) 239 { 240 current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; 241 } 242 243 /** 244 * memalloc_nofs_save - Marks implicit GFP_NOFS allocation scope. 245 * 246 * This functions marks the beginning of the GFP_NOFS allocation scope. 247 * All further allocations will implicitly drop __GFP_FS flag and so 248 * they are safe for the FS critical section from the allocation recursion 249 * point of view. Use memalloc_nofs_restore to end the scope with flags 250 * returned by this function. 251 * 252 * This function is safe to be used from any context. 253 */ 254 static inline unsigned int memalloc_nofs_save(void) 255 { 256 unsigned int flags = current->flags & PF_MEMALLOC_NOFS; 257 current->flags |= PF_MEMALLOC_NOFS; 258 return flags; 259 } 260 261 /** 262 * memalloc_nofs_restore - Ends the implicit GFP_NOFS scope. 263 * @flags: Flags to restore. 264 * 265 * Ends the implicit GFP_NOFS scope started by memalloc_nofs_save function. 266 * Always make sure that that the given flags is the return value from the 267 * pairing memalloc_nofs_save call. 268 */ 269 static inline void memalloc_nofs_restore(unsigned int flags) 270 { 271 current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags; 272 } 273 274 static inline unsigned int memalloc_noreclaim_save(void) 275 { 276 unsigned int flags = current->flags & PF_MEMALLOC; 277 current->flags |= PF_MEMALLOC; 278 return flags; 279 } 280 281 static inline void memalloc_noreclaim_restore(unsigned int flags) 282 { 283 current->flags = (current->flags & ~PF_MEMALLOC) | flags; 284 } 285 286 #ifdef CONFIG_CMA 287 static inline unsigned int memalloc_nocma_save(void) 288 { 289 unsigned int flags = current->flags & PF_MEMALLOC_NOCMA; 290 291 current->flags |= PF_MEMALLOC_NOCMA; 292 return flags; 293 } 294 295 static inline void memalloc_nocma_restore(unsigned int flags) 296 { 297 current->flags = (current->flags & ~PF_MEMALLOC_NOCMA) | flags; 298 } 299 #else 300 static inline unsigned int memalloc_nocma_save(void) 301 { 302 return 0; 303 } 304 305 static inline void memalloc_nocma_restore(unsigned int flags) 306 { 307 } 308 #endif 309 310 #ifdef CONFIG_MEMCG 311 /** 312 * memalloc_use_memcg - Starts the remote memcg charging scope. 313 * @memcg: memcg to charge. 314 * 315 * This function marks the beginning of the remote memcg charging scope. All the 316 * __GFP_ACCOUNT allocations till the end of the scope will be charged to the 317 * given memcg. 318 * 319 * NOTE: This function is not nesting safe. 320 */ 321 static inline void memalloc_use_memcg(struct mem_cgroup *memcg) 322 { 323 WARN_ON_ONCE(current->active_memcg); 324 current->active_memcg = memcg; 325 } 326 327 /** 328 * memalloc_unuse_memcg - Ends the remote memcg charging scope. 329 * 330 * This function marks the end of the remote memcg charging scope started by 331 * memalloc_use_memcg(). 332 */ 333 static inline void memalloc_unuse_memcg(void) 334 { 335 current->active_memcg = NULL; 336 } 337 #else 338 static inline void memalloc_use_memcg(struct mem_cgroup *memcg) 339 { 340 } 341 342 static inline void memalloc_unuse_memcg(void) 343 { 344 } 345 #endif 346 347 #ifdef CONFIG_MEMBARRIER 348 enum { 349 MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = (1U << 0), 350 MEMBARRIER_STATE_PRIVATE_EXPEDITED = (1U << 1), 351 MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY = (1U << 2), 352 MEMBARRIER_STATE_GLOBAL_EXPEDITED = (1U << 3), 353 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = (1U << 4), 354 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = (1U << 5), 355 }; 356 357 enum { 358 MEMBARRIER_FLAG_SYNC_CORE = (1U << 0), 359 }; 360 361 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS 362 #include <asm/membarrier.h> 363 #endif 364 365 static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) 366 { 367 if (current->mm != mm) 368 return; 369 if (likely(!(atomic_read(&mm->membarrier_state) & 370 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) 371 return; 372 sync_core_before_usermode(); 373 } 374 375 extern void membarrier_exec_mmap(struct mm_struct *mm); 376 377 #else 378 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS 379 static inline void membarrier_arch_switch_mm(struct mm_struct *prev, 380 struct mm_struct *next, 381 struct task_struct *tsk) 382 { 383 } 384 #endif 385 static inline void membarrier_exec_mmap(struct mm_struct *mm) 386 { 387 } 388 static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) 389 { 390 } 391 #endif 392 393 #endif /* _LINUX_SCHED_MM_H */ 394