1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2006, David Xu <davidxu@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * A lockless rwlock for rtld. 31 */ 32 #include <sys/mman.h> 33 #include <sys/syscall.h> 34 #include <link.h> 35 #include <stdlib.h> 36 #include <string.h> 37 38 #include "libc_private.h" 39 #include "rtld_lock.h" 40 #include "thr_private.h" 41 42 #undef errno 43 extern int errno; 44 45 static int _thr_rtld_clr_flag(int); 46 static void *_thr_rtld_lock_create(void); 47 static void _thr_rtld_lock_destroy(void *); 48 static void _thr_rtld_lock_release(void *); 49 static void _thr_rtld_rlock_acquire(void *); 50 static int _thr_rtld_set_flag(int); 51 static void _thr_rtld_wlock_acquire(void *); 52 53 struct rtld_lock { 54 struct urwlock lock; 55 char _pad[CACHE_LINE_SIZE - sizeof(struct urwlock)]; 56 }; 57 58 static struct rtld_lock lock_place[MAX_RTLD_LOCKS] __aligned(CACHE_LINE_SIZE); 59 static int busy_places; 60 61 static void * 62 _thr_rtld_lock_create(void) 63 { 64 int locki; 65 struct rtld_lock *l; 66 static const char fail[] = "_thr_rtld_lock_create failed\n"; 67 68 for (locki = 0; locki < MAX_RTLD_LOCKS; locki++) { 69 if ((busy_places & (1 << locki)) == 0) 70 break; 71 } 72 if (locki == MAX_RTLD_LOCKS) { 73 write(2, fail, sizeof(fail) - 1); 74 return (NULL); 75 } 76 busy_places |= (1 << locki); 77 78 l = &lock_place[locki]; 79 l->lock.rw_flags = URWLOCK_PREFER_READER; 80 return (l); 81 } 82 83 static void 84 _thr_rtld_lock_destroy(void *lock) 85 { 86 int locki; 87 size_t i; 88 89 locki = (struct rtld_lock *)lock - &lock_place[0]; 90 for (i = 0; i < sizeof(struct rtld_lock); ++i) 91 ((char *)lock)[i] = 0; 92 busy_places &= ~(1 << locki); 93 } 94 95 #define SAVE_ERRNO() { \ 96 if (curthread != _thr_initial) \ 97 errsave = curthread->error; \ 98 else \ 99 errsave = errno; \ 100 } 101 102 #define RESTORE_ERRNO() { \ 103 if (curthread != _thr_initial) \ 104 curthread->error = errsave; \ 105 else \ 106 errno = errsave; \ 107 } 108 109 static void 110 _thr_rtld_rlock_acquire(void *lock) 111 { 112 struct pthread *curthread; 113 struct rtld_lock *l; 114 int errsave; 115 116 curthread = _get_curthread(); 117 SAVE_ERRNO(); 118 l = (struct rtld_lock *)lock; 119 120 THR_CRITICAL_ENTER(curthread); 121 while (_thr_rwlock_rdlock(&l->lock, 0, NULL) != 0) 122 ; 123 curthread->rdlock_count++; 124 RESTORE_ERRNO(); 125 } 126 127 static void 128 _thr_rtld_wlock_acquire(void *lock) 129 { 130 struct pthread *curthread; 131 struct rtld_lock *l; 132 int errsave; 133 134 curthread = _get_curthread(); 135 SAVE_ERRNO(); 136 l = (struct rtld_lock *)lock; 137 138 THR_CRITICAL_ENTER(curthread); 139 while (_thr_rwlock_wrlock(&l->lock, NULL) != 0) 140 ; 141 RESTORE_ERRNO(); 142 } 143 144 static void 145 _thr_rtld_lock_release(void *lock) 146 { 147 struct pthread *curthread; 148 struct rtld_lock *l; 149 int32_t state; 150 int errsave; 151 152 curthread = _get_curthread(); 153 SAVE_ERRNO(); 154 l = (struct rtld_lock *)lock; 155 156 state = l->lock.rw_state; 157 if (__predict_false(_thr_after_fork)) { 158 /* 159 * After fork, only this thread is running, there is no 160 * waiters. Keeping waiters recorded in rwlock breaks 161 * wake logic. 162 */ 163 atomic_clear_int(&l->lock.rw_state, 164 URWLOCK_WRITE_WAITERS | URWLOCK_READ_WAITERS); 165 l->lock.rw_blocked_readers = 0; 166 l->lock.rw_blocked_writers = 0; 167 } 168 if (_thr_rwlock_unlock(&l->lock) == 0) { 169 if ((state & URWLOCK_WRITE_OWNER) == 0) 170 curthread->rdlock_count--; 171 THR_CRITICAL_LEAVE(curthread); 172 } 173 RESTORE_ERRNO(); 174 } 175 176 static int 177 _thr_rtld_set_flag(int mask __unused) 178 { 179 /* 180 * The caller's code in rtld-elf is broken, it is not signal safe, 181 * just return zero to fool it. 182 */ 183 return (0); 184 } 185 186 static int 187 _thr_rtld_clr_flag(int mask __unused) 188 { 189 return (0); 190 } 191 192 /* 193 * ABI bug workaround: This symbol must be present for rtld to accept 194 * RTLI_VERSION from RtldLockInfo 195 */ 196 extern char _pli_rtli_version; 197 char _pli_rtli_version; 198 199 static char * 200 _thr_dlerror_loc(void) 201 { 202 struct pthread *curthread; 203 204 curthread = _get_curthread(); 205 return (curthread->dlerror_msg); 206 } 207 208 static int * 209 _thr_dlerror_seen(void) 210 { 211 struct pthread *curthread; 212 213 curthread = _get_curthread(); 214 return (&curthread->dlerror_seen); 215 } 216 217 void 218 _thr_rtld_init(void) 219 { 220 struct RtldLockInfo li; 221 struct pthread *curthread; 222 ucontext_t *uc; 223 long dummy = -1; 224 int uc_len; 225 226 curthread = _get_curthread(); 227 228 /* force to resolve _umtx_op PLT */ 229 _umtx_op_err((struct umtx *)&dummy, UMTX_OP_WAKE, 1, 0, 0); 230 231 /* force to resolve errno() PLT */ 232 __error(); 233 234 /* force to resolve memcpy PLT */ 235 memcpy(&dummy, &dummy, sizeof(dummy)); 236 237 mprotect(NULL, 0, 0); 238 _rtld_get_stack_prot(); 239 thr_wake(-1); 240 241 li.rtli_version = RTLI_VERSION; 242 li.lock_create = _thr_rtld_lock_create; 243 li.lock_destroy = _thr_rtld_lock_destroy; 244 li.rlock_acquire = _thr_rtld_rlock_acquire; 245 li.wlock_acquire = _thr_rtld_wlock_acquire; 246 li.lock_release = _thr_rtld_lock_release; 247 li.thread_set_flag = _thr_rtld_set_flag; 248 li.thread_clr_flag = _thr_rtld_clr_flag; 249 li.at_fork = NULL; 250 li.dlerror_loc = _thr_dlerror_loc; 251 li.dlerror_loc_sz = sizeof(curthread->dlerror_msg); 252 li.dlerror_seen = _thr_dlerror_seen; 253 254 /* 255 * Preresolve the symbols needed for the fork interposer. We 256 * call _rtld_atfork_pre() and _rtld_atfork_post() with NULL 257 * argument to indicate that no actual locking inside the 258 * functions should happen. Neither rtld compat locks nor 259 * libthr rtld locks cannot work there: 260 * - compat locks do not handle the case of two locks taken 261 * in write mode (the signal mask for the thread is corrupted); 262 * - libthr locks would work, but locked rtld_bind_lock prevents 263 * symbol resolution for _rtld_atfork_post. 264 */ 265 _rtld_atfork_pre(NULL); 266 _rtld_atfork_post(NULL); 267 _malloc_prefork(); 268 _malloc_postfork(); 269 getpid(); 270 syscall(SYS_getpid); 271 272 /* mask signals, also force to resolve __sys_sigprocmask PLT */ 273 _thr_signal_block(curthread); 274 _rtld_thread_init(&li); 275 _thr_signal_unblock(curthread); 276 _thr_signal_block_check_fast(); 277 _thr_signal_block_setup(curthread); 278 279 uc_len = __getcontextx_size(); 280 uc = alloca(uc_len); 281 getcontext(uc); 282 __fillcontextx2((char *)uc); 283 } 284