1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_SCHED_MM_H 3 #define _LINUX_SCHED_MM_H 4 5 #include <linux/kernel.h> 6 #include <linux/atomic.h> 7 #include <linux/sched.h> 8 #include <linux/mm_types.h> 9 #include <linux/gfp.h> 10 #include <linux/sync_core.h> 11 12 /* 13 * Routines for handling mm_structs 14 */ 15 extern struct mm_struct *mm_alloc(void); 16 17 /** 18 * mmgrab() - Pin a &struct mm_struct. 19 * @mm: The &struct mm_struct to pin. 20 * 21 * Make sure that @mm will not get freed even after the owning task 22 * exits. This doesn't guarantee that the associated address space 23 * will still exist later on and mmget_not_zero() has to be used before 24 * accessing it. 25 * 26 * This is a preferred way to to pin @mm for a longer/unbounded amount 27 * of time. 28 * 29 * Use mmdrop() to release the reference acquired by mmgrab(). 30 * 31 * See also <Documentation/vm/active_mm.rst> for an in-depth explanation 32 * of &mm_struct.mm_count vs &mm_struct.mm_users. 33 */ 34 static inline void mmgrab(struct mm_struct *mm) 35 { 36 atomic_inc(&mm->mm_count); 37 } 38 39 extern void __mmdrop(struct mm_struct *mm); 40 41 static inline void mmdrop(struct mm_struct *mm) 42 { 43 /* 44 * The implicit full barrier implied by atomic_dec_and_test() is 45 * required by the membarrier system call before returning to 46 * user-space, after storing to rq->curr. 47 */ 48 if (unlikely(atomic_dec_and_test(&mm->mm_count))) 49 __mmdrop(mm); 50 } 51 52 /* 53 * This has to be called after a get_task_mm()/mmget_not_zero() 54 * followed by taking the mmap_sem for writing before modifying the 55 * vmas or anything the coredump pretends not to change from under it. 56 * 57 * It also has to be called when mmgrab() is used in the context of 58 * the process, but then the mm_count refcount is transferred outside 59 * the context of the process to run down_write() on that pinned mm. 60 * 61 * NOTE: find_extend_vma() called from GUP context is the only place 62 * that can modify the "mm" (notably the vm_start/end) under mmap_sem 63 * for reading and outside the context of the process, so it is also 64 * the only case that holds the mmap_sem for reading that must call 65 * this function. Generally if the mmap_sem is hold for reading 66 * there's no need of this check after get_task_mm()/mmget_not_zero(). 67 * 68 * This function can be obsoleted and the check can be removed, after 69 * the coredump code will hold the mmap_sem for writing before 70 * invoking the ->core_dump methods. 71 */ 72 static inline bool mmget_still_valid(struct mm_struct *mm) 73 { 74 return likely(!mm->core_state); 75 } 76 77 /** 78 * mmget() - Pin the address space associated with a &struct mm_struct. 79 * @mm: The address space to pin. 80 * 81 * Make sure that the address space of the given &struct mm_struct doesn't 82 * go away. This does not protect against parts of the address space being 83 * modified or freed, however. 84 * 85 * Never use this function to pin this address space for an 86 * unbounded/indefinite amount of time. 87 * 88 * Use mmput() to release the reference acquired by mmget(). 89 * 90 * See also <Documentation/vm/active_mm.rst> for an in-depth explanation 91 * of &mm_struct.mm_count vs &mm_struct.mm_users. 92 */ 93 static inline void mmget(struct mm_struct *mm) 94 { 95 atomic_inc(&mm->mm_users); 96 } 97 98 static inline bool mmget_not_zero(struct mm_struct *mm) 99 { 100 return atomic_inc_not_zero(&mm->mm_users); 101 } 102 103 /* mmput gets rid of the mappings and all user-space */ 104 extern void mmput(struct mm_struct *); 105 #ifdef CONFIG_MMU 106 /* same as above but performs the slow path from the async context. Can 107 * be called from the atomic context as well 108 */ 109 void mmput_async(struct mm_struct *); 110 #endif 111 112 /* Grab a reference to a task's mm, if it is not already going away */ 113 extern struct mm_struct *get_task_mm(struct task_struct *task); 114 /* 115 * Grab a reference to a task's mm, if it is not already going away 116 * and ptrace_may_access with the mode parameter passed to it 117 * succeeds. 118 */ 119 extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); 120 /* Remove the current tasks stale references to the old mm_struct */ 121 extern void mm_release(struct task_struct *, struct mm_struct *); 122 123 #ifdef CONFIG_MEMCG 124 extern void mm_update_next_owner(struct mm_struct *mm); 125 #else 126 static inline void mm_update_next_owner(struct mm_struct *mm) 127 { 128 } 129 #endif /* CONFIG_MEMCG */ 130 131 #ifdef CONFIG_MMU 132 extern void arch_pick_mmap_layout(struct mm_struct *mm, 133 struct rlimit *rlim_stack); 134 extern unsigned long 135 arch_get_unmapped_area(struct file *, unsigned long, unsigned long, 136 unsigned long, unsigned long); 137 extern unsigned long 138 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 139 unsigned long len, unsigned long pgoff, 140 unsigned long flags); 141 #else 142 static inline void arch_pick_mmap_layout(struct mm_struct *mm, 143 struct rlimit *rlim_stack) {} 144 #endif 145 146 static inline bool in_vfork(struct task_struct *tsk) 147 { 148 bool ret; 149 150 /* 151 * need RCU to access ->real_parent if CLONE_VM was used along with 152 * CLONE_PARENT. 153 * 154 * We check real_parent->mm == tsk->mm because CLONE_VFORK does not 155 * imply CLONE_VM 156 * 157 * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus 158 * ->real_parent is not necessarily the task doing vfork(), so in 159 * theory we can't rely on task_lock() if we want to dereference it. 160 * 161 * And in this case we can't trust the real_parent->mm == tsk->mm 162 * check, it can be false negative. But we do not care, if init or 163 * another oom-unkillable task does this it should blame itself. 164 */ 165 rcu_read_lock(); 166 ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm; 167 rcu_read_unlock(); 168 169 return ret; 170 } 171 172 /* 173 * Applies per-task gfp context to the given allocation flags. 174 * PF_MEMALLOC_NOIO implies GFP_NOIO 175 * PF_MEMALLOC_NOFS implies GFP_NOFS 176 * PF_MEMALLOC_NOCMA implies no allocation from CMA region. 177 */ 178 static inline gfp_t current_gfp_context(gfp_t flags) 179 { 180 if (unlikely(current->flags & 181 (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_NOCMA))) { 182 /* 183 * NOIO implies both NOIO and NOFS and it is a weaker context 184 * so always make sure it makes precedence 185 */ 186 if (current->flags & PF_MEMALLOC_NOIO) 187 flags &= ~(__GFP_IO | __GFP_FS); 188 else if (current->flags & PF_MEMALLOC_NOFS) 189 flags &= ~__GFP_FS; 190 #ifdef CONFIG_CMA 191 if (current->flags & PF_MEMALLOC_NOCMA) 192 flags &= ~__GFP_MOVABLE; 193 #endif 194 } 195 return flags; 196 } 197 198 #ifdef CONFIG_LOCKDEP 199 extern void __fs_reclaim_acquire(void); 200 extern void __fs_reclaim_release(void); 201 extern void fs_reclaim_acquire(gfp_t gfp_mask); 202 extern void fs_reclaim_release(gfp_t gfp_mask); 203 #else 204 static inline void __fs_reclaim_acquire(void) { } 205 static inline void __fs_reclaim_release(void) { } 206 static inline void fs_reclaim_acquire(gfp_t gfp_mask) { } 207 static inline void fs_reclaim_release(gfp_t gfp_mask) { } 208 #endif 209 210 /** 211 * memalloc_noio_save - Marks implicit GFP_NOIO allocation scope. 212 * 213 * This functions marks the beginning of the GFP_NOIO allocation scope. 214 * All further allocations will implicitly drop __GFP_IO flag and so 215 * they are safe for the IO critical section from the allocation recursion 216 * point of view. Use memalloc_noio_restore to end the scope with flags 217 * returned by this function. 218 * 219 * This function is safe to be used from any context. 220 */ 221 static inline unsigned int memalloc_noio_save(void) 222 { 223 unsigned int flags = current->flags & PF_MEMALLOC_NOIO; 224 current->flags |= PF_MEMALLOC_NOIO; 225 return flags; 226 } 227 228 /** 229 * memalloc_noio_restore - Ends the implicit GFP_NOIO scope. 230 * @flags: Flags to restore. 231 * 232 * Ends the implicit GFP_NOIO scope started by memalloc_noio_save function. 233 * Always make sure that that the given flags is the return value from the 234 * pairing memalloc_noio_save call. 235 */ 236 static inline void memalloc_noio_restore(unsigned int flags) 237 { 238 current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; 239 } 240 241 /** 242 * memalloc_nofs_save - Marks implicit GFP_NOFS allocation scope. 243 * 244 * This functions marks the beginning of the GFP_NOFS allocation scope. 245 * All further allocations will implicitly drop __GFP_FS flag and so 246 * they are safe for the FS critical section from the allocation recursion 247 * point of view. Use memalloc_nofs_restore to end the scope with flags 248 * returned by this function. 249 * 250 * This function is safe to be used from any context. 251 */ 252 static inline unsigned int memalloc_nofs_save(void) 253 { 254 unsigned int flags = current->flags & PF_MEMALLOC_NOFS; 255 current->flags |= PF_MEMALLOC_NOFS; 256 return flags; 257 } 258 259 /** 260 * memalloc_nofs_restore - Ends the implicit GFP_NOFS scope. 261 * @flags: Flags to restore. 262 * 263 * Ends the implicit GFP_NOFS scope started by memalloc_nofs_save function. 264 * Always make sure that that the given flags is the return value from the 265 * pairing memalloc_nofs_save call. 266 */ 267 static inline void memalloc_nofs_restore(unsigned int flags) 268 { 269 current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags; 270 } 271 272 static inline unsigned int memalloc_noreclaim_save(void) 273 { 274 unsigned int flags = current->flags & PF_MEMALLOC; 275 current->flags |= PF_MEMALLOC; 276 return flags; 277 } 278 279 static inline void memalloc_noreclaim_restore(unsigned int flags) 280 { 281 current->flags = (current->flags & ~PF_MEMALLOC) | flags; 282 } 283 284 #ifdef CONFIG_CMA 285 static inline unsigned int memalloc_nocma_save(void) 286 { 287 unsigned int flags = current->flags & PF_MEMALLOC_NOCMA; 288 289 current->flags |= PF_MEMALLOC_NOCMA; 290 return flags; 291 } 292 293 static inline void memalloc_nocma_restore(unsigned int flags) 294 { 295 current->flags = (current->flags & ~PF_MEMALLOC_NOCMA) | flags; 296 } 297 #else 298 static inline unsigned int memalloc_nocma_save(void) 299 { 300 return 0; 301 } 302 303 static inline void memalloc_nocma_restore(unsigned int flags) 304 { 305 } 306 #endif 307 308 #ifdef CONFIG_MEMCG 309 /** 310 * memalloc_use_memcg - Starts the remote memcg charging scope. 311 * @memcg: memcg to charge. 312 * 313 * This function marks the beginning of the remote memcg charging scope. All the 314 * __GFP_ACCOUNT allocations till the end of the scope will be charged to the 315 * given memcg. 316 * 317 * NOTE: This function is not nesting safe. 318 */ 319 static inline void memalloc_use_memcg(struct mem_cgroup *memcg) 320 { 321 WARN_ON_ONCE(current->active_memcg); 322 current->active_memcg = memcg; 323 } 324 325 /** 326 * memalloc_unuse_memcg - Ends the remote memcg charging scope. 327 * 328 * This function marks the end of the remote memcg charging scope started by 329 * memalloc_use_memcg(). 330 */ 331 static inline void memalloc_unuse_memcg(void) 332 { 333 current->active_memcg = NULL; 334 } 335 #else 336 static inline void memalloc_use_memcg(struct mem_cgroup *memcg) 337 { 338 } 339 340 static inline void memalloc_unuse_memcg(void) 341 { 342 } 343 #endif 344 345 #ifdef CONFIG_MEMBARRIER 346 enum { 347 MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = (1U << 0), 348 MEMBARRIER_STATE_PRIVATE_EXPEDITED = (1U << 1), 349 MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY = (1U << 2), 350 MEMBARRIER_STATE_GLOBAL_EXPEDITED = (1U << 3), 351 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = (1U << 4), 352 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = (1U << 5), 353 }; 354 355 enum { 356 MEMBARRIER_FLAG_SYNC_CORE = (1U << 0), 357 }; 358 359 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS 360 #include <asm/membarrier.h> 361 #endif 362 363 static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) 364 { 365 if (current->mm != mm) 366 return; 367 if (likely(!(atomic_read(&mm->membarrier_state) & 368 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) 369 return; 370 sync_core_before_usermode(); 371 } 372 373 extern void membarrier_exec_mmap(struct mm_struct *mm); 374 375 #else 376 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS 377 static inline void membarrier_arch_switch_mm(struct mm_struct *prev, 378 struct mm_struct *next, 379 struct task_struct *tsk) 380 { 381 } 382 #endif 383 static inline void membarrier_exec_mmap(struct mm_struct *mm) 384 { 385 } 386 static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) 387 { 388 } 389 #endif 390 391 #endif /* _LINUX_SCHED_MM_H */ 392