1 /* 2 * Copyright (c) 2003,2004,2010,2017 The DragonFly Project. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Matthew Dillon <dillon@backplane.com> and David Xu <davidxu@freebsd.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * 3. Neither the name of The DragonFly Project nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific, prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 */ 35 36 /* 37 * This module implements userland mutex helper functions. umtx_sleep() 38 * handling blocking and umtx_wakeup() handles wakeups. The sleep/wakeup 39 * functions operate on user addresses. 40 */ 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/cdefs.h> 45 #include <sys/kernel.h> 46 #include <sys/sysmsg.h> 47 #include <sys/sysent.h> 48 #include <sys/syscall.h> 49 #include <sys/sysctl.h> 50 #include <sys/module.h> 51 #include <sys/thread.h> 52 #include <sys/proc.h> 53 54 #include <cpu/lwbuf.h> 55 56 #include <vm/vm.h> 57 #include <vm/vm_param.h> 58 #include <sys/lock.h> 59 #include <vm/pmap.h> 60 #include <vm/vm_map.h> 61 #include <vm/vm_object.h> 62 #include <vm/vm_page.h> 63 #include <vm/vm_pager.h> 64 #include <vm/vm_pageout.h> 65 #include <vm/vm_extern.h> 66 #include <vm/vm_kern.h> 67 68 #include <vm/vm_page2.h> 69 70 #include <machine/vmm.h> 71 72 /* 73 * Improve umtx performance by polling for 4000nS before going to sleep. 74 * This can avoid many IPIs in typical pthreads mutex situations. 75 */ 76 #ifdef _RDTSC_SUPPORTED_ 77 static int umtx_delay = 4000; /* nS */ 78 SYSCTL_INT(_kern, OID_AUTO, umtx_delay, CTLFLAG_RW, 79 &umtx_delay, 0, ""); 80 #endif 81 static int umtx_timeout_max = 2000000; /* microseconds */ 82 SYSCTL_INT(_kern, OID_AUTO, umtx_timeout_max, CTLFLAG_RW, 83 &umtx_timeout_max, 0, ""); 84 85 /* 86 * If the contents of the userland-supplied pointer matches the specified 87 * value enter an interruptable sleep for up to <timeout> microseconds. 88 * If the contents does not match then return immediately. 89 * 90 * Returns 0 if we slept and were woken up, -1 and EWOULDBLOCK if we slept 91 * and timed out, and EBUSY if the contents of the pointer already does 92 * not match the specified value. A timeout of 0 indicates an unlimited sleep. 93 * EINTR is returned if the call was interrupted by a signal (even if 94 * the signal specifies that the system call should restart). 95 * 96 * This function interlocks against call to umtx_wakeup. It does NOT interlock 97 * against changes in *ptr. However, it does not have to. The standard use 98 * of *ptr is to differentiate between an uncontested and a contested mutex 99 * and call umtx_wakeup when releasing a contested mutex. Therefore we can 100 * safely race against changes in *ptr as long as we are properly interlocked 101 * against the umtx_wakeup() call. 102 * 103 * For performance reasons, we do not try to track the underlying page for 104 * mapping changes. Instead, the timeout is capped at kern.umtx_timeout_max 105 * (default 1 second) and the caller is expected to retry. The kernel 106 * will wake all umtx_sleep()s if the process fork()s, but not if it vfork()s. 107 * Other mapping changes must be caught by the timeout. 108 * 109 * umtx_sleep { const int *ptr, int value, int timeout } 110 */ 111 int 112 sys_umtx_sleep(struct sysmsg *sysmsg, const struct umtx_sleep_args *uap) 113 { 114 void *waddr; 115 void *uptr; 116 int offset; 117 int timeout; 118 int error; 119 int value; 120 int fail_counter; 121 thread_t td; 122 volatile const int *ptr = uap->ptr; 123 124 if (uap->timeout < 0) 125 return (EINVAL); 126 td = curthread; 127 128 if (td->td_vmm) { 129 register_t gpa; 130 vmm_vm_get_gpa(td->td_proc, &gpa, (register_t)ptr); 131 ptr = (const int *)gpa; 132 } 133 134 uptr = __DEQUALIFY(void *, ptr); 135 if ((vm_offset_t)uptr & (sizeof(int) - 1)) 136 return EFAULT; 137 138 offset = (vm_offset_t)uptr & PAGE_MASK; 139 140 /* 141 * Resolve the physical address. We allow the case where there are 142 * sometimes discontinuities (causing a 2 second retry timeout). 143 */ 144 retry_on_discontinuity: 145 fail_counter = 10000; 146 do { 147 if (--fail_counter == 0) { 148 kprintf("umtx_sleep() (X): ERROR Discontinuity %p (%s %d/%d)\n", 149 uptr, td->td_comm, 150 (int)td->td_proc->p_pid, 151 (int)td->td_lwp->lwp_tid); 152 return EINVAL; 153 } 154 value = fuwordadd32(uptr, 0); 155 waddr = (void *)(intptr_t)uservtophys((intptr_t)uptr); 156 } while (waddr == (void *)(intptr_t)-1 && value != -1); 157 158 if (value == -1 && waddr == (void *)(intptr_t)-1) { 159 kprintf("umtx_sleep() (A): WARNING can't translate %p (%s %d/%d)\n", 160 uptr, td->td_comm, 161 (int)td->td_proc->p_pid, 162 (int)td->td_lwp->lwp_tid); 163 return EINVAL; 164 } 165 166 error = EBUSY; 167 if (value == uap->value) { 168 #ifdef _RDTSC_SUPPORTED_ 169 /* 170 * Poll a little while before sleeping, most mutexes are 171 * short-lived. 172 */ 173 if (umtx_delay) { 174 int64_t tsc_target; 175 int good = 0; 176 177 tsc_target = tsc_get_target(umtx_delay); 178 while (tsc_test_target(tsc_target) == 0) { 179 cpu_lfence(); 180 if (fuwordadd32(uptr, 0) != uap->value) { 181 good = 1; 182 break; 183 } 184 cpu_pause(); 185 } 186 if (good) { 187 error = EBUSY; 188 goto done; 189 } 190 } 191 #endif 192 /* 193 * Calculate the timeout. This will be acccurate to within ~2 ticks. 194 * uap->timeout is in microseconds. 195 */ 196 timeout = umtx_timeout_max; 197 if (uap->timeout && uap->timeout < timeout) 198 timeout = uap->timeout; 199 timeout = (timeout / 1000000) * hz + 200 ((timeout % 1000000) * hz + 999999) / 1000000; 201 202 /* 203 * Wake us up if the memory location COWs while we are sleeping. 204 * Use a critical section to tighten up the interlock. Also, 205 * tsleep_remove() requires the caller be in a critical section. 206 */ 207 crit_enter(); 208 209 /* 210 * We must interlock just before sleeping. If we interlock before 211 * registration the lock operations done by the registration can 212 * interfere with it. 213 * 214 * We cannot leave our interlock hanging on return because this 215 * will interfere with umtx_wakeup() calls with limited wakeup 216 * counts. 217 */ 218 tsleep_interlock(waddr, PCATCH | PDOMAIN_UMTX); 219 220 /* 221 * Check physical address changed 222 */ 223 cpu_lfence(); 224 if ((void *)(intptr_t)uservtophys((intptr_t)uptr) != waddr) { 225 crit_exit(); 226 goto retry_on_discontinuity; 227 } 228 229 /* 230 * Re-read value 231 */ 232 value = fuwordadd32(uptr, 0); 233 234 if (value == uap->value) { 235 error = tsleep(waddr, PCATCH | PINTERLOCKED | PDOMAIN_UMTX, 236 "umtxsl", timeout); 237 } else { 238 error = EBUSY; 239 } 240 crit_exit(); 241 /* Always break out in case of signal, even if restartable */ 242 if (error == ERESTART) 243 error = EINTR; 244 } else { 245 error = EBUSY; 246 } 247 done: 248 return(error); 249 } 250 251 /* 252 * umtx_wakeup { const int *ptr, int count } 253 * 254 * Wakeup the specified number of processes held in umtx_sleep() on the 255 * specified user address. A count of 0 wakes up all waiting processes. 256 */ 257 int 258 sys_umtx_wakeup(struct sysmsg *sysmsg, const struct umtx_wakeup_args *uap) 259 { 260 int offset; 261 int error; 262 int fail_counter; 263 int32_t value; 264 void *waddr; 265 void *uptr; 266 volatile const int *ptr = uap->ptr; 267 thread_t td; 268 269 td = curthread; 270 271 if (td->td_vmm) { 272 register_t gpa; 273 vmm_vm_get_gpa(td->td_proc, &gpa, (register_t)ptr); 274 ptr = (const int *)gpa; 275 } 276 277 /* 278 * WARNING! We can only use vm_fault_page*() for reading data. We 279 * cannot use it for writing data because there is no pmap 280 * interlock to protect against flushes/pageouts. 281 */ 282 cpu_mfence(); 283 if ((vm_offset_t)ptr & (sizeof(int) - 1)) 284 return EFAULT; 285 286 offset = (vm_offset_t)ptr & PAGE_MASK; 287 uptr = __DEQUALIFY(void *, ptr); 288 289 fail_counter = 10000; 290 do { 291 if (--fail_counter == 0) { 292 kprintf("umtx_wakeup() (X): ERROR Discontinuity " 293 "%p (%s %d/%d)\n", 294 uptr, td->td_comm, 295 (int)td->td_proc->p_pid, 296 (int)td->td_lwp->lwp_tid); 297 return EINVAL; 298 } 299 value = fuwordadd32(uptr, 0); 300 waddr = (void *)(intptr_t)uservtophys((intptr_t)uptr); 301 } while (waddr == (void *)(intptr_t)-1 && value != -1); 302 303 if (value == -1 && waddr == (void *)(intptr_t)-1) { 304 kprintf("umtx_wakeup() (A): WARNING can't translate %p (%s %d/%d)\n", 305 uptr, td->td_comm, 306 (int)td->td_proc->p_pid, 307 (int)td->td_lwp->lwp_tid); 308 return EINVAL; 309 } 310 311 if (uap->count == 1) { 312 wakeup_domain_one(waddr, PDOMAIN_UMTX); 313 } else { 314 /* XXX wakes them all up for now */ 315 wakeup_domain(waddr, PDOMAIN_UMTX); 316 } 317 error = 0; 318 319 return(error); 320 } 321