1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 */ 29 30 #ifndef _SYS_UMTXVAR_H_ 31 #define _SYS_UMTXVAR_H_ 32 33 #ifdef _KERNEL 34 35 /* 36 * The umtx_key structure is used by both the Linux futex code and the 37 * umtx implementation to map userland addresses to unique keys. 38 */ 39 enum { 40 TYPE_SIMPLE_WAIT, 41 TYPE_CV, 42 TYPE_SEM, 43 TYPE_SIMPLE_LOCK, 44 TYPE_NORMAL_UMUTEX, 45 TYPE_PI_UMUTEX, 46 TYPE_PP_UMUTEX, 47 TYPE_RWLOCK, 48 TYPE_FUTEX, 49 TYPE_SHM, 50 TYPE_PI_ROBUST_UMUTEX, 51 TYPE_PP_ROBUST_UMUTEX, 52 TYPE_PI_FUTEX, 53 }; 54 55 /* Key to represent a unique userland synchronous object */ 56 struct umtx_key { 57 int hash; 58 int type; 59 int shared; 60 union { 61 struct { 62 struct vm_object *object; 63 uintptr_t offset; 64 } shared; 65 struct { 66 struct vmspace *vs; 67 uintptr_t addr; 68 } private; 69 struct { 70 void *a; 71 uintptr_t b; 72 } both; 73 } info; 74 }; 75 76 #define THREAD_SHARE 0 77 #define PROCESS_SHARE 1 78 #define AUTO_SHARE 2 79 80 struct umtx_abs_timeout { 81 int clockid; 82 bool is_abs_real; /* TIMER_ABSTIME && CLOCK_REALTIME* */ 83 struct timespec cur; 84 struct timespec end; 85 }; 86 87 struct thread; 88 89 /* Priority inheritance mutex info. */ 90 struct umtx_pi { 91 /* Owner thread */ 92 struct thread *pi_owner; 93 94 /* Reference count */ 95 int pi_refcount; 96 97 /* List entry to link umtx holding by thread */ 98 TAILQ_ENTRY(umtx_pi) pi_link; 99 100 /* List entry in hash */ 101 TAILQ_ENTRY(umtx_pi) pi_hashlink; 102 103 /* List for waiters */ 104 TAILQ_HEAD(,umtx_q) pi_blocked; 105 106 /* Identify a userland lock object */ 107 struct umtx_key pi_key; 108 }; 109 110 /* A userland synchronous object user. */ 111 struct umtx_q { 112 /* Linked list for the hash. */ 113 TAILQ_ENTRY(umtx_q) uq_link; 114 115 /* Umtx key. */ 116 struct umtx_key uq_key; 117 118 /* Umtx flags. */ 119 int uq_flags; 120 #define UQF_UMTXQ 0x0001 121 122 /* Futex bitset mask */ 123 u_int uq_bitset; 124 125 /* The thread waits on. */ 126 struct thread *uq_thread; 127 128 /* 129 * Blocked on PI mutex. read can use chain lock 130 * or umtx_lock, write must have both chain lock and 131 * umtx_lock being hold. 132 */ 133 struct umtx_pi *uq_pi_blocked; 134 135 /* On blocked list */ 136 TAILQ_ENTRY(umtx_q) uq_lockq; 137 138 /* Thread contending with us */ 139 TAILQ_HEAD(,umtx_pi) uq_pi_contested; 140 141 /* Inherited priority from PP mutex */ 142 u_char uq_inherited_pri; 143 144 /* Spare queue ready to be reused */ 145 struct umtxq_queue *uq_spare_queue; 146 147 /* The queue we on */ 148 struct umtxq_queue *uq_cur_queue; 149 }; 150 151 TAILQ_HEAD(umtxq_head, umtx_q); 152 153 /* Per-key wait-queue */ 154 struct umtxq_queue { 155 struct umtxq_head head; 156 struct umtx_key key; 157 LIST_ENTRY(umtxq_queue) link; 158 int length; 159 }; 160 161 LIST_HEAD(umtxq_list, umtxq_queue); 162 163 /* Userland lock object's wait-queue chain */ 164 struct umtxq_chain { 165 /* Lock for this chain. */ 166 struct mtx uc_lock; 167 168 /* List of sleep queues. */ 169 struct umtxq_list uc_queue[2]; 170 #define UMTX_SHARED_QUEUE 0 171 #define UMTX_EXCLUSIVE_QUEUE 1 172 173 LIST_HEAD(, umtxq_queue) uc_spare_queue; 174 175 /* Busy flag */ 176 char uc_busy; 177 178 /* Chain lock waiters */ 179 int uc_waiters; 180 181 /* All PI in the list */ 182 TAILQ_HEAD(,umtx_pi) uc_pi_list; 183 184 #ifdef UMTX_PROFILING 185 u_int length; 186 u_int max_length; 187 #endif 188 }; 189 190 static inline int 191 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2) 192 { 193 194 return (k1->type == k2->type && 195 k1->info.both.a == k2->info.both.a && 196 k1->info.both.b == k2->info.both.b); 197 } 198 199 void umtx_abs_timeout_init(struct umtx_abs_timeout *, int, int, 200 const struct timespec *); 201 int umtx_copyin_timeout(const void *, struct timespec *); 202 void umtx_exec(struct proc *p); 203 int umtx_key_get(const void *, int, int, struct umtx_key *); 204 void umtx_key_release(struct umtx_key *); 205 struct umtx_q *umtxq_alloc(void); 206 void umtxq_busy(struct umtx_key *); 207 int umtxq_count(struct umtx_key *); 208 void umtxq_free(struct umtx_q *); 209 struct umtxq_chain *umtxq_getchain(struct umtx_key *); 210 void umtxq_insert_queue(struct umtx_q *, int); 211 void umtxq_remove_queue(struct umtx_q *, int); 212 int umtxq_requeue(struct umtx_key *, int, struct umtx_key *, int); 213 int umtxq_signal_mask(struct umtx_key *, int, u_int); 214 int umtxq_sleep(struct umtx_q *, const char *, 215 struct umtx_abs_timeout *); 216 int umtxq_sleep_pi(struct umtx_q *, struct umtx_pi *, uint32_t, 217 const char *, struct umtx_abs_timeout *, bool); 218 void umtxq_unbusy(struct umtx_key *); 219 void umtxq_unbusy_unlocked(struct umtx_key *); 220 int kern_umtx_wake(struct thread *, void *, int, int); 221 void umtx_pi_adjust(struct thread *, u_char); 222 struct umtx_pi *umtx_pi_alloc(int); 223 int umtx_pi_claim(struct umtx_pi *, struct thread *); 224 int umtx_pi_drop(struct thread *, struct umtx_key *, bool, int *); 225 void umtx_pi_free(struct umtx_pi *); 226 void umtx_pi_insert(struct umtx_pi *); 227 struct umtx_pi *umtx_pi_lookup(struct umtx_key *); 228 void umtx_pi_ref(struct umtx_pi *); 229 void umtx_pi_unref(struct umtx_pi *); 230 void umtx_thread_init(struct thread *); 231 void umtx_thread_fini(struct thread *); 232 void umtx_thread_alloc(struct thread *); 233 void umtx_thread_exit(struct thread *); 234 235 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE) 236 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE) 237 238 /* 239 * Lock a chain. 240 * 241 * The code is a macro so that file/line information is taken from the caller. 242 */ 243 #define umtxq_lock(key) do { \ 244 struct umtx_key *_key = (key); \ 245 struct umtxq_chain *_uc; \ 246 \ 247 _uc = umtxq_getchain(_key); \ 248 mtx_lock(&_uc->uc_lock); \ 249 } while (0) 250 251 /* 252 * Unlock a chain. 253 */ 254 static inline void 255 umtxq_unlock(struct umtx_key *key) 256 { 257 struct umtxq_chain *uc; 258 259 uc = umtxq_getchain(key); 260 mtx_unlock(&uc->uc_lock); 261 } 262 263 #endif /* _KERNEL */ 264 #endif /* !_SYS_UMTXVAR_H_ */ 265