1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Wound/Wait Mutexes: blocking mutual exclusion locks with deadlock avoidance 4 * 5 * Original mutex implementation started by Ingo Molnar: 6 * 7 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 8 * 9 * Wait/Die implementation: 10 * Copyright (C) 2013 Canonical Ltd. 11 * Choice of algorithm: 12 * Copyright (C) 2018 WMWare Inc. 13 * 14 * This file contains the main data structure and API definitions. 15 */ 16 17 #ifndef __LINUX_WW_MUTEX_H 18 #define __LINUX_WW_MUTEX_H 19 20 #include <linux/mutex.h> 21 22 #define WW_MUTEX_BASE mutex 23 #define ww_mutex_base_init(l,n,k) __mutex_init(l,n,k) 24 #define ww_mutex_base_trylock(l) mutex_trylock(l) 25 #define ww_mutex_base_is_locked(b) mutex_is_locked((b)) 26 27 struct ww_class { 28 atomic_long_t stamp; 29 struct lock_class_key acquire_key; 30 struct lock_class_key mutex_key; 31 const char *acquire_name; 32 const char *mutex_name; 33 unsigned int is_wait_die; 34 }; 35 36 struct ww_mutex { 37 struct WW_MUTEX_BASE base; 38 struct ww_acquire_ctx *ctx; 39 #ifdef CONFIG_DEBUG_MUTEXES 40 struct ww_class *ww_class; 41 #endif 42 }; 43 44 struct ww_acquire_ctx { 45 struct task_struct *task; 46 unsigned long stamp; 47 unsigned int acquired; 48 unsigned short wounded; 49 unsigned short is_wait_die; 50 #ifdef CONFIG_DEBUG_MUTEXES 51 unsigned int done_acquire; 52 struct ww_class *ww_class; 53 struct ww_mutex *contending_lock; 54 #endif 55 #ifdef CONFIG_DEBUG_LOCK_ALLOC 56 struct lockdep_map dep_map; 57 #endif 58 #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH 59 unsigned int deadlock_inject_interval; 60 unsigned int deadlock_inject_countdown; 61 #endif 62 }; 63 64 #define __WW_CLASS_INITIALIZER(ww_class, _is_wait_die) \ 65 { .stamp = ATOMIC_LONG_INIT(0) \ 66 , .acquire_name = #ww_class "_acquire" \ 67 , .mutex_name = #ww_class "_mutex" \ 68 , .is_wait_die = _is_wait_die } 69 70 #define DEFINE_WD_CLASS(classname) \ 71 struct ww_class classname = __WW_CLASS_INITIALIZER(classname, 1) 72 73 #define DEFINE_WW_CLASS(classname) \ 74 struct ww_class classname = __WW_CLASS_INITIALIZER(classname, 0) 75 76 /** 77 * ww_mutex_init - initialize the w/w mutex 78 * @lock: the mutex to be initialized 79 * @ww_class: the w/w class the mutex should belong to 80 * 81 * Initialize the w/w mutex to unlocked state and associate it with the given 82 * class. Static define macro for w/w mutex is not provided and this function 83 * is the only way to properly initialize the w/w mutex. 84 * 85 * It is not allowed to initialize an already locked mutex. 86 */ 87 static inline void ww_mutex_init(struct ww_mutex *lock, 88 struct ww_class *ww_class) 89 { 90 ww_mutex_base_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key); 91 lock->ctx = NULL; 92 #ifdef CONFIG_DEBUG_MUTEXES 93 lock->ww_class = ww_class; 94 #endif 95 } 96 97 /** 98 * ww_acquire_init - initialize a w/w acquire context 99 * @ctx: w/w acquire context to initialize 100 * @ww_class: w/w class of the context 101 * 102 * Initializes an context to acquire multiple mutexes of the given w/w class. 103 * 104 * Context-based w/w mutex acquiring can be done in any order whatsoever within 105 * a given lock class. Deadlocks will be detected and handled with the 106 * wait/die logic. 107 * 108 * Mixing of context-based w/w mutex acquiring and single w/w mutex locking can 109 * result in undetected deadlocks and is so forbidden. Mixing different contexts 110 * for the same w/w class when acquiring mutexes can also result in undetected 111 * deadlocks, and is hence also forbidden. Both types of abuse will be caught by 112 * enabling CONFIG_PROVE_LOCKING. 113 * 114 * Nesting of acquire contexts for _different_ w/w classes is possible, subject 115 * to the usual locking rules between different lock classes. 116 * 117 * An acquire context must be released with ww_acquire_fini by the same task 118 * before the memory is freed. It is recommended to allocate the context itself 119 * on the stack. 120 */ 121 static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, 122 struct ww_class *ww_class) 123 { 124 ctx->task = current; 125 ctx->stamp = atomic_long_inc_return_relaxed(&ww_class->stamp); 126 ctx->acquired = 0; 127 ctx->wounded = false; 128 ctx->is_wait_die = ww_class->is_wait_die; 129 #ifdef CONFIG_DEBUG_MUTEXES 130 ctx->ww_class = ww_class; 131 ctx->done_acquire = 0; 132 ctx->contending_lock = NULL; 133 #endif 134 #ifdef CONFIG_DEBUG_LOCK_ALLOC 135 debug_check_no_locks_freed((void *)ctx, sizeof(*ctx)); 136 lockdep_init_map(&ctx->dep_map, ww_class->acquire_name, 137 &ww_class->acquire_key, 0); 138 mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_); 139 #endif 140 #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH 141 ctx->deadlock_inject_interval = 1; 142 ctx->deadlock_inject_countdown = ctx->stamp & 0xf; 143 #endif 144 } 145 146 /** 147 * ww_acquire_done - marks the end of the acquire phase 148 * @ctx: the acquire context 149 * 150 * Marks the end of the acquire phase, any further w/w mutex lock calls using 151 * this context are forbidden. 152 * 153 * Calling this function is optional, it is just useful to document w/w mutex 154 * code and clearly designated the acquire phase from actually using the locked 155 * data structures. 156 */ 157 static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) 158 { 159 #ifdef CONFIG_DEBUG_MUTEXES 160 lockdep_assert_held(ctx); 161 162 DEBUG_LOCKS_WARN_ON(ctx->done_acquire); 163 ctx->done_acquire = 1; 164 #endif 165 } 166 167 /** 168 * ww_acquire_fini - releases a w/w acquire context 169 * @ctx: the acquire context to free 170 * 171 * Releases a w/w acquire context. This must be called _after_ all acquired w/w 172 * mutexes have been released with ww_mutex_unlock. 173 */ 174 static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) 175 { 176 #ifdef CONFIG_DEBUG_LOCK_ALLOC 177 mutex_release(&ctx->dep_map, _THIS_IP_); 178 #endif 179 #ifdef CONFIG_DEBUG_MUTEXES 180 DEBUG_LOCKS_WARN_ON(ctx->acquired); 181 if (!IS_ENABLED(CONFIG_PROVE_LOCKING)) 182 /* 183 * lockdep will normally handle this, 184 * but fail without anyway 185 */ 186 ctx->done_acquire = 1; 187 188 if (!IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC)) 189 /* ensure ww_acquire_fini will still fail if called twice */ 190 ctx->acquired = ~0U; 191 #endif 192 } 193 194 /** 195 * ww_mutex_lock - acquire the w/w mutex 196 * @lock: the mutex to be acquired 197 * @ctx: w/w acquire context, or NULL to acquire only a single lock. 198 * 199 * Lock the w/w mutex exclusively for this task. 200 * 201 * Deadlocks within a given w/w class of locks are detected and handled with the 202 * wait/die algorithm. If the lock isn't immediately available this function 203 * will either sleep until it is (wait case). Or it selects the current context 204 * for backing off by returning -EDEADLK (die case). Trying to acquire the 205 * same lock with the same context twice is also detected and signalled by 206 * returning -EALREADY. Returns 0 if the mutex was successfully acquired. 207 * 208 * In the die case the caller must release all currently held w/w mutexes for 209 * the given context and then wait for this contending lock to be available by 210 * calling ww_mutex_lock_slow. Alternatively callers can opt to not acquire this 211 * lock and proceed with trying to acquire further w/w mutexes (e.g. when 212 * scanning through lru lists trying to free resources). 213 * 214 * The mutex must later on be released by the same task that 215 * acquired it. The task may not exit without first unlocking the mutex. Also, 216 * kernel memory where the mutex resides must not be freed with the mutex still 217 * locked. The mutex must first be initialized (or statically defined) before it 218 * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be 219 * of the same w/w lock class as was used to initialize the acquire context. 220 * 221 * A mutex acquired with this function must be released with ww_mutex_unlock. 222 */ 223 extern int /* __must_check */ ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx); 224 225 /** 226 * ww_mutex_lock_interruptible - acquire the w/w mutex, interruptible 227 * @lock: the mutex to be acquired 228 * @ctx: w/w acquire context 229 * 230 * Lock the w/w mutex exclusively for this task. 231 * 232 * Deadlocks within a given w/w class of locks are detected and handled with the 233 * wait/die algorithm. If the lock isn't immediately available this function 234 * will either sleep until it is (wait case). Or it selects the current context 235 * for backing off by returning -EDEADLK (die case). Trying to acquire the 236 * same lock with the same context twice is also detected and signalled by 237 * returning -EALREADY. Returns 0 if the mutex was successfully acquired. If a 238 * signal arrives while waiting for the lock then this function returns -EINTR. 239 * 240 * In the die case the caller must release all currently held w/w mutexes for 241 * the given context and then wait for this contending lock to be available by 242 * calling ww_mutex_lock_slow_interruptible. Alternatively callers can opt to 243 * not acquire this lock and proceed with trying to acquire further w/w mutexes 244 * (e.g. when scanning through lru lists trying to free resources). 245 * 246 * The mutex must later on be released by the same task that 247 * acquired it. The task may not exit without first unlocking the mutex. Also, 248 * kernel memory where the mutex resides must not be freed with the mutex still 249 * locked. The mutex must first be initialized (or statically defined) before it 250 * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be 251 * of the same w/w lock class as was used to initialize the acquire context. 252 * 253 * A mutex acquired with this function must be released with ww_mutex_unlock. 254 */ 255 extern int __must_check ww_mutex_lock_interruptible(struct ww_mutex *lock, 256 struct ww_acquire_ctx *ctx); 257 258 /** 259 * ww_mutex_lock_slow - slowpath acquiring of the w/w mutex 260 * @lock: the mutex to be acquired 261 * @ctx: w/w acquire context 262 * 263 * Acquires a w/w mutex with the given context after a die case. This function 264 * will sleep until the lock becomes available. 265 * 266 * The caller must have released all w/w mutexes already acquired with the 267 * context and then call this function on the contended lock. 268 * 269 * Afterwards the caller may continue to (re)acquire the other w/w mutexes it 270 * needs with ww_mutex_lock. Note that the -EALREADY return code from 271 * ww_mutex_lock can be used to avoid locking this contended mutex twice. 272 * 273 * It is forbidden to call this function with any other w/w mutexes associated 274 * with the context held. It is forbidden to call this on anything else than the 275 * contending mutex. 276 * 277 * Note that the slowpath lock acquiring can also be done by calling 278 * ww_mutex_lock directly. This function here is simply to help w/w mutex 279 * locking code readability by clearly denoting the slowpath. 280 */ 281 static inline void 282 ww_mutex_lock_slow(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 283 { 284 int ret; 285 #ifdef CONFIG_DEBUG_MUTEXES 286 DEBUG_LOCKS_WARN_ON(!ctx->contending_lock); 287 #endif 288 ret = ww_mutex_lock(lock, ctx); 289 (void)ret; 290 } 291 292 /** 293 * ww_mutex_lock_slow_interruptible - slowpath acquiring of the w/w mutex, interruptible 294 * @lock: the mutex to be acquired 295 * @ctx: w/w acquire context 296 * 297 * Acquires a w/w mutex with the given context after a die case. This function 298 * will sleep until the lock becomes available and returns 0 when the lock has 299 * been acquired. If a signal arrives while waiting for the lock then this 300 * function returns -EINTR. 301 * 302 * The caller must have released all w/w mutexes already acquired with the 303 * context and then call this function on the contended lock. 304 * 305 * Afterwards the caller may continue to (re)acquire the other w/w mutexes it 306 * needs with ww_mutex_lock. Note that the -EALREADY return code from 307 * ww_mutex_lock can be used to avoid locking this contended mutex twice. 308 * 309 * It is forbidden to call this function with any other w/w mutexes associated 310 * with the given context held. It is forbidden to call this on anything else 311 * than the contending mutex. 312 * 313 * Note that the slowpath lock acquiring can also be done by calling 314 * ww_mutex_lock_interruptible directly. This function here is simply to help 315 * w/w mutex locking code readability by clearly denoting the slowpath. 316 */ 317 static inline int __must_check 318 ww_mutex_lock_slow_interruptible(struct ww_mutex *lock, 319 struct ww_acquire_ctx *ctx) 320 { 321 #ifdef CONFIG_DEBUG_MUTEXES 322 DEBUG_LOCKS_WARN_ON(!ctx->contending_lock); 323 #endif 324 return ww_mutex_lock_interruptible(lock, ctx); 325 } 326 327 extern void ww_mutex_unlock(struct ww_mutex *lock); 328 329 /** 330 * ww_mutex_trylock - tries to acquire the w/w mutex without acquire context 331 * @lock: mutex to lock 332 * 333 * Trylocks a mutex without acquire context, so no deadlock detection is 334 * possible. Returns 1 if the mutex has been acquired successfully, 0 otherwise. 335 */ 336 static inline int __must_check ww_mutex_trylock(struct ww_mutex *lock) 337 { 338 return ww_mutex_base_trylock(&lock->base); 339 } 340 341 /*** 342 * ww_mutex_destroy - mark a w/w mutex unusable 343 * @lock: the mutex to be destroyed 344 * 345 * This function marks the mutex uninitialized, and any subsequent 346 * use of the mutex is forbidden. The mutex must not be locked when 347 * this function is called. 348 */ 349 static inline void ww_mutex_destroy(struct ww_mutex *lock) 350 { 351 mutex_destroy(&lock->base); 352 } 353 354 /** 355 * ww_mutex_is_locked - is the w/w mutex locked 356 * @lock: the mutex to be queried 357 * 358 * Returns 1 if the mutex is locked, 0 if unlocked. 359 */ 360 static inline bool ww_mutex_is_locked(struct ww_mutex *lock) 361 { 362 return ww_mutex_base_is_locked(&lock->base); 363 } 364 365 #endif 366