1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2018 Turing Robotic Industries Inc. 5 * Copyright (C) 2022 Dmitry Chagin <dchagin@FreeBSD.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31#include <machine/asm.h> 32__FBSDID("$FreeBSD$"); 33 34#include <machine/param.h> 35#include <machine/vmparam.h> 36 37#include <sys/errno.h> 38 39#include "assym.inc" 40 41.macro check_user_access user_arg, limit, bad_addr_func 42 ldr x7, =(\limit) 43 cmp x\user_arg, x7 44 b.cs \bad_addr_func 45.endm 46 47futex_fault: 48 SET_FAULT_HANDLER(xzr, x1) 49 EXIT_USER_ACCESS_CHECK(w0, x1) 50futex_fault_nopcb: 51 mov x0, #EFAULT 52 ret 53 54#define LINUX_FUTEX_MAX_LOOPS 128 55 56/* 57 * int oparg, uint32_t *uaddr, int *oldval 58 * 59 * Return 0 on success, errno on failure, 60 * EAGAIN is returned if LL/SC operation fails. 61 * 62 * XXX. VM_MAXUSER_ADDRESS is not applicable here, should be replaced 63 * by something like LINUX_SHAREDPAGE. 64 */ 65 66/* (int *)uaddr2 = oparg */ 67ENTRY(futex_xchgl) 68 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 69 adr x9, futex_fault /* Load the fault handler */ 70 SET_FAULT_HANDLER(x9, x4) /* And set it */ 71 ENTER_USER_ACCESS(w9, x4) 72 mov w5, #LINUX_FUTEX_MAX_LOOPS 73 prfm pstl1strm, [x1] 74 mov w6, w0 /* Save oparg */ 751: ldxr w4, [x1] /* Load oldval from uaddr */ 76 stlxr w0, w6, [x1] /* Store oparg to uaddr */ 77 cbz w0, 3f /* Exit on success */ 78 sub w5, w5, w0 /* Dec loop counter, w0 is 1 */ 79 cbnz w5, 1b /* Loop */ 80 mov x0, #EAGAIN /* Store of newval failed */ 813: dmb ish 82 EXIT_USER_ACCESS(w9) 83 SET_FAULT_HANDLER(xzr, x9) /* Reset the fault handler */ 84 str w4, [x2] /* Store oldval */ 85 ret 86END(futex_xchgl) 87 88/* (int *)uaddr2 += oparg */ 89ENTRY(futex_addl) 90 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 91 adr x9, futex_fault 92 SET_FAULT_HANDLER(x9, x4) 93 ENTER_USER_ACCESS(w9, x4) 94 mov w5, #LINUX_FUTEX_MAX_LOOPS 95 prfm pstl1strm, [x1] 96 mov w6, w0 971: ldxr w4, [x1] 98 add w3, w4, w6 /* oldval + oparg */ 99 stlxr w0, w3, [x1] 100 cbz w0, 3f 101 sub w5, w5, w0 102 cbnz w5, 1b 103 mov x0, #EAGAIN 1043: dmb ish 105 EXIT_USER_ACCESS(w9) 106 SET_FAULT_HANDLER(xzr, x9) 107 str w4, [x2] 108 ret 109END(futex_addl) 110 111/* (int *)uaddr2 |= oparg */ 112ENTRY(futex_orl) 113 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 114 adr x9, futex_fault 115 SET_FAULT_HANDLER(x9, x4) 116 ENTER_USER_ACCESS(w9, x4) 117 mov w5, #LINUX_FUTEX_MAX_LOOPS 118 prfm pstl1strm, [x1] 119 mov w6, w0 1201: ldxr w4, [x1] 121 orr w3, w4, w6 /* oldavl |= oparg */ 122 stlxr w0, w3, [x1] 123 cbz w0, 3f 124 sub w5, w5, w0 125 cbnz w5, 1b 126 mov x0, #EAGAIN 1273: dmb ish 128 EXIT_USER_ACCESS(w9) 129 SET_FAULT_HANDLER(xzr, x9) 130 str w4, [x2] 131 ret 132END(futex_orl) 133 134/* (int *)uaddr2 &= oparg */ 135ENTRY(futex_andl) 136 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 137 adr x9, futex_fault 138 SET_FAULT_HANDLER(x9, x4) 139 ENTER_USER_ACCESS(w9, x4) 140 mov w5, #LINUX_FUTEX_MAX_LOOPS 141 prfm pstl1strm, [x1] 142 mov w6, w0 1431: ldxr w4, [x1] 144 and w3, w4, w6 /* oldval &= oparg */ 145 stlxr w0, w3, [x1] 146 cbz w0, 3f 147 sub w5, w5, w0 148 cbnz w5, 1b 149 mov x0, #EAGAIN 1503: dmb ish 151 EXIT_USER_ACCESS(w9) 152 SET_FAULT_HANDLER(xzr, x9) 153 str w4, [x2] 154 ret 155END(futex_andl) 156 157/* (int *)uaddr2 ^= oparg */ 158ENTRY(futex_xorl) 159 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 160 adr x9, futex_fault 161 SET_FAULT_HANDLER(x9, x4) 162 ENTER_USER_ACCESS(w9, x4) 163 mov w5, #LINUX_FUTEX_MAX_LOOPS 164 prfm pstl1strm, [x1] 165 mov w6, w0 1661: ldxr w4, [x1] 167 eor w3, w4, w6 /* oldval ^= oparg */ 168 stlxr w0, w3, [x1] 169 cbz w0, 3f 170 sub w5, w5, w0 171 cbnz w5, 1b 172 mov x0, #EAGAIN 1733: dmb ish 174 EXIT_USER_ACCESS(w9) 175 SET_FAULT_HANDLER(xzr, x9) 176 str w4, [x2] 177 ret 178END(futex_xorl) 179