1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2006, David Xu <davidxu@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 /* 33 * A lockless rwlock for rtld. 34 */ 35 #include <sys/cdefs.h> 36 #include <sys/mman.h> 37 #include <sys/syscall.h> 38 #include <link.h> 39 #include <stdlib.h> 40 #include <string.h> 41 42 #include "libc_private.h" 43 #include "rtld_lock.h" 44 #include "thr_private.h" 45 46 #undef errno 47 extern int errno; 48 49 static int _thr_rtld_clr_flag(int); 50 static void *_thr_rtld_lock_create(void); 51 static void _thr_rtld_lock_destroy(void *); 52 static void _thr_rtld_lock_release(void *); 53 static void _thr_rtld_rlock_acquire(void *); 54 static int _thr_rtld_set_flag(int); 55 static void _thr_rtld_wlock_acquire(void *); 56 57 struct rtld_lock { 58 struct urwlock lock; 59 char _pad[CACHE_LINE_SIZE - sizeof(struct urwlock)]; 60 }; 61 62 static struct rtld_lock lock_place[MAX_RTLD_LOCKS] __aligned(CACHE_LINE_SIZE); 63 static int busy_places; 64 65 static void * 66 _thr_rtld_lock_create(void) 67 { 68 int locki; 69 struct rtld_lock *l; 70 static const char fail[] = "_thr_rtld_lock_create failed\n"; 71 72 for (locki = 0; locki < MAX_RTLD_LOCKS; locki++) { 73 if ((busy_places & (1 << locki)) == 0) 74 break; 75 } 76 if (locki == MAX_RTLD_LOCKS) { 77 write(2, fail, sizeof(fail) - 1); 78 return (NULL); 79 } 80 busy_places |= (1 << locki); 81 82 l = &lock_place[locki]; 83 l->lock.rw_flags = URWLOCK_PREFER_READER; 84 return (l); 85 } 86 87 static void 88 _thr_rtld_lock_destroy(void *lock) 89 { 90 int locki; 91 size_t i; 92 93 locki = (struct rtld_lock *)lock - &lock_place[0]; 94 for (i = 0; i < sizeof(struct rtld_lock); ++i) 95 ((char *)lock)[i] = 0; 96 busy_places &= ~(1 << locki); 97 } 98 99 #define SAVE_ERRNO() { \ 100 if (curthread != _thr_initial) \ 101 errsave = curthread->error; \ 102 else \ 103 errsave = errno; \ 104 } 105 106 #define RESTORE_ERRNO() { \ 107 if (curthread != _thr_initial) \ 108 curthread->error = errsave; \ 109 else \ 110 errno = errsave; \ 111 } 112 113 static void 114 _thr_rtld_rlock_acquire(void *lock) 115 { 116 struct pthread *curthread; 117 struct rtld_lock *l; 118 int errsave; 119 120 curthread = _get_curthread(); 121 SAVE_ERRNO(); 122 l = (struct rtld_lock *)lock; 123 124 THR_CRITICAL_ENTER(curthread); 125 while (_thr_rwlock_rdlock(&l->lock, 0, NULL) != 0) 126 ; 127 curthread->rdlock_count++; 128 RESTORE_ERRNO(); 129 } 130 131 static void 132 _thr_rtld_wlock_acquire(void *lock) 133 { 134 struct pthread *curthread; 135 struct rtld_lock *l; 136 int errsave; 137 138 curthread = _get_curthread(); 139 SAVE_ERRNO(); 140 l = (struct rtld_lock *)lock; 141 142 THR_CRITICAL_ENTER(curthread); 143 while (_thr_rwlock_wrlock(&l->lock, NULL) != 0) 144 ; 145 RESTORE_ERRNO(); 146 } 147 148 static void 149 _thr_rtld_lock_release(void *lock) 150 { 151 struct pthread *curthread; 152 struct rtld_lock *l; 153 int32_t state; 154 int errsave; 155 156 curthread = _get_curthread(); 157 SAVE_ERRNO(); 158 l = (struct rtld_lock *)lock; 159 160 state = l->lock.rw_state; 161 if (__predict_false(_thr_after_fork)) { 162 /* 163 * After fork, only this thread is running, there is no 164 * waiters. Keeping waiters recorded in rwlock breaks 165 * wake logic. 166 */ 167 atomic_clear_int(&l->lock.rw_state, 168 URWLOCK_WRITE_WAITERS | URWLOCK_READ_WAITERS); 169 l->lock.rw_blocked_readers = 0; 170 l->lock.rw_blocked_writers = 0; 171 } 172 if (_thr_rwlock_unlock(&l->lock) == 0) { 173 if ((state & URWLOCK_WRITE_OWNER) == 0) 174 curthread->rdlock_count--; 175 THR_CRITICAL_LEAVE(curthread); 176 } 177 RESTORE_ERRNO(); 178 } 179 180 static int 181 _thr_rtld_set_flag(int mask __unused) 182 { 183 /* 184 * The caller's code in rtld-elf is broken, it is not signal safe, 185 * just return zero to fool it. 186 */ 187 return (0); 188 } 189 190 static int 191 _thr_rtld_clr_flag(int mask __unused) 192 { 193 return (0); 194 } 195 196 /* 197 * ABI bug workaround: This symbol must be present for rtld to accept 198 * RTLI_VERSION from RtldLockInfo 199 */ 200 extern char _pli_rtli_version; 201 char _pli_rtli_version; 202 203 static char * 204 _thr_dlerror_loc(void) 205 { 206 struct pthread *curthread; 207 208 curthread = _get_curthread(); 209 return (curthread->dlerror_msg); 210 } 211 212 static int * 213 _thr_dlerror_seen(void) 214 { 215 struct pthread *curthread; 216 217 curthread = _get_curthread(); 218 return (&curthread->dlerror_seen); 219 } 220 221 void 222 _thr_rtld_init(void) 223 { 224 struct RtldLockInfo li; 225 struct pthread *curthread; 226 ucontext_t *uc; 227 long dummy = -1; 228 int uc_len; 229 230 curthread = _get_curthread(); 231 232 /* force to resolve _umtx_op PLT */ 233 _umtx_op_err((struct umtx *)&dummy, UMTX_OP_WAKE, 1, 0, 0); 234 235 /* force to resolve errno() PLT */ 236 __error(); 237 238 /* force to resolve memcpy PLT */ 239 memcpy(&dummy, &dummy, sizeof(dummy)); 240 241 mprotect(NULL, 0, 0); 242 _rtld_get_stack_prot(); 243 244 li.rtli_version = RTLI_VERSION; 245 li.lock_create = _thr_rtld_lock_create; 246 li.lock_destroy = _thr_rtld_lock_destroy; 247 li.rlock_acquire = _thr_rtld_rlock_acquire; 248 li.wlock_acquire = _thr_rtld_wlock_acquire; 249 li.lock_release = _thr_rtld_lock_release; 250 li.thread_set_flag = _thr_rtld_set_flag; 251 li.thread_clr_flag = _thr_rtld_clr_flag; 252 li.at_fork = NULL; 253 li.dlerror_loc = _thr_dlerror_loc; 254 li.dlerror_loc_sz = sizeof(curthread->dlerror_msg); 255 li.dlerror_seen = _thr_dlerror_seen; 256 257 /* 258 * Preresolve the symbols needed for the fork interposer. We 259 * call _rtld_atfork_pre() and _rtld_atfork_post() with NULL 260 * argument to indicate that no actual locking inside the 261 * functions should happen. Neither rtld compat locks nor 262 * libthr rtld locks cannot work there: 263 * - compat locks do not handle the case of two locks taken 264 * in write mode (the signal mask for the thread is corrupted); 265 * - libthr locks would work, but locked rtld_bind_lock prevents 266 * symbol resolution for _rtld_atfork_post. 267 */ 268 _rtld_atfork_pre(NULL); 269 _rtld_atfork_post(NULL); 270 _malloc_prefork(); 271 _malloc_postfork(); 272 getpid(); 273 syscall(SYS_getpid); 274 275 /* mask signals, also force to resolve __sys_sigprocmask PLT */ 276 _thr_signal_block(curthread); 277 _rtld_thread_init(&li); 278 _thr_signal_unblock(curthread); 279 _thr_signal_block_check_fast(); 280 _thr_signal_block_setup(curthread); 281 282 uc_len = __getcontextx_size(); 283 uc = alloca(uc_len); 284 getcontext(uc); 285 __fillcontextx2((char *)uc); 286 } 287