1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Atomic operations. 4 * 5 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 6 */ 7 #ifndef _ASM_ATOMIC_H 8 #define _ASM_ATOMIC_H 9 10 #include <linux/types.h> 11 #include <asm/barrier.h> 12 #include <asm/cmpxchg.h> 13 #include <asm/compiler.h> 14 15 #if __SIZEOF_LONG__ == 4 16 #define __LL "ll.w " 17 #define __SC "sc.w " 18 #define __AMADD "amadd.w " 19 #define __AMAND_DB "amand_db.w " 20 #define __AMOR_DB "amor_db.w " 21 #define __AMXOR_DB "amxor_db.w " 22 #elif __SIZEOF_LONG__ == 8 23 #define __LL "ll.d " 24 #define __SC "sc.d " 25 #define __AMADD "amadd.d " 26 #define __AMAND_DB "amand_db.d " 27 #define __AMOR_DB "amor_db.d " 28 #define __AMXOR_DB "amxor_db.d " 29 #endif 30 31 #define ATOMIC_INIT(i) { (i) } 32 33 /* 34 * arch_atomic_read - read atomic variable 35 * @v: pointer of type atomic_t 36 * 37 * Atomically reads the value of @v. 38 */ 39 #define arch_atomic_read(v) READ_ONCE((v)->counter) 40 41 /* 42 * arch_atomic_set - set atomic variable 43 * @v: pointer of type atomic_t 44 * @i: required value 45 * 46 * Atomically sets the value of @v to @i. 47 */ 48 #define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i)) 49 50 #define ATOMIC_OP(op, I, asm_op) \ 51 static inline void arch_atomic_##op(int i, atomic_t *v) \ 52 { \ 53 __asm__ __volatile__( \ 54 "am"#asm_op"_db.w" " $zero, %1, %0 \n" \ 55 : "+ZB" (v->counter) \ 56 : "r" (I) \ 57 : "memory"); \ 58 } 59 60 #define ATOMIC_OP_RETURN(op, I, asm_op, c_op) \ 61 static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ 62 { \ 63 int result; \ 64 \ 65 __asm__ __volatile__( \ 66 "am"#asm_op"_db.w" " %1, %2, %0 \n" \ 67 : "+ZB" (v->counter), "=&r" (result) \ 68 : "r" (I) \ 69 : "memory"); \ 70 \ 71 return result c_op I; \ 72 } 73 74 #define ATOMIC_FETCH_OP(op, I, asm_op) \ 75 static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ 76 { \ 77 int result; \ 78 \ 79 __asm__ __volatile__( \ 80 "am"#asm_op"_db.w" " %1, %2, %0 \n" \ 81 : "+ZB" (v->counter), "=&r" (result) \ 82 : "r" (I) \ 83 : "memory"); \ 84 \ 85 return result; \ 86 } 87 88 #define ATOMIC_OPS(op, I, asm_op, c_op) \ 89 ATOMIC_OP(op, I, asm_op) \ 90 ATOMIC_OP_RETURN(op, I, asm_op, c_op) \ 91 ATOMIC_FETCH_OP(op, I, asm_op) 92 93 ATOMIC_OPS(add, i, add, +) 94 ATOMIC_OPS(sub, -i, add, +) 95 96 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 97 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 98 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 99 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 100 101 #undef ATOMIC_OPS 102 103 #define ATOMIC_OPS(op, I, asm_op) \ 104 ATOMIC_OP(op, I, asm_op) \ 105 ATOMIC_FETCH_OP(op, I, asm_op) 106 107 ATOMIC_OPS(and, i, and) 108 ATOMIC_OPS(or, i, or) 109 ATOMIC_OPS(xor, i, xor) 110 111 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 112 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 113 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 114 115 #undef ATOMIC_OPS 116 #undef ATOMIC_FETCH_OP 117 #undef ATOMIC_OP_RETURN 118 #undef ATOMIC_OP 119 120 static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 121 { 122 int prev, rc; 123 124 __asm__ __volatile__ ( 125 "0: ll.w %[p], %[c]\n" 126 " beq %[p], %[u], 1f\n" 127 " add.w %[rc], %[p], %[a]\n" 128 " sc.w %[rc], %[c]\n" 129 " beqz %[rc], 0b\n" 130 " b 2f\n" 131 "1:\n" 132 __WEAK_LLSC_MB 133 "2:\n" 134 : [p]"=&r" (prev), [rc]"=&r" (rc), 135 [c]"=ZB" (v->counter) 136 : [a]"r" (a), [u]"r" (u) 137 : "memory"); 138 139 return prev; 140 } 141 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 142 143 /* 144 * arch_atomic_sub_if_positive - conditionally subtract integer from atomic variable 145 * @i: integer value to subtract 146 * @v: pointer of type atomic_t 147 * 148 * Atomically test @v and subtract @i if @v is greater or equal than @i. 149 * The function returns the old value of @v minus @i. 150 */ 151 static inline int arch_atomic_sub_if_positive(int i, atomic_t *v) 152 { 153 int result; 154 int temp; 155 156 if (__builtin_constant_p(i)) { 157 __asm__ __volatile__( 158 "1: ll.w %1, %2 # atomic_sub_if_positive\n" 159 " addi.w %0, %1, %3 \n" 160 " or %1, %0, $zero \n" 161 " blt %0, $zero, 2f \n" 162 " sc.w %1, %2 \n" 163 " beq $zero, %1, 1b \n" 164 "2: \n" 165 __WEAK_LLSC_MB 166 : "=&r" (result), "=&r" (temp), 167 "+" GCC_OFF_SMALL_ASM() (v->counter) 168 : "I" (-i)); 169 } else { 170 __asm__ __volatile__( 171 "1: ll.w %1, %2 # atomic_sub_if_positive\n" 172 " sub.w %0, %1, %3 \n" 173 " or %1, %0, $zero \n" 174 " blt %0, $zero, 2f \n" 175 " sc.w %1, %2 \n" 176 " beq $zero, %1, 1b \n" 177 "2: \n" 178 __WEAK_LLSC_MB 179 : "=&r" (result), "=&r" (temp), 180 "+" GCC_OFF_SMALL_ASM() (v->counter) 181 : "r" (i)); 182 } 183 184 return result; 185 } 186 187 #define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n))) 188 #define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new))) 189 190 /* 191 * arch_atomic_dec_if_positive - decrement by 1 if old value positive 192 * @v: pointer of type atomic_t 193 */ 194 #define arch_atomic_dec_if_positive(v) arch_atomic_sub_if_positive(1, v) 195 196 #ifdef CONFIG_64BIT 197 198 #define ATOMIC64_INIT(i) { (i) } 199 200 /* 201 * arch_atomic64_read - read atomic variable 202 * @v: pointer of type atomic64_t 203 * 204 */ 205 #define arch_atomic64_read(v) READ_ONCE((v)->counter) 206 207 /* 208 * arch_atomic64_set - set atomic variable 209 * @v: pointer of type atomic64_t 210 * @i: required value 211 */ 212 #define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i)) 213 214 #define ATOMIC64_OP(op, I, asm_op) \ 215 static inline void arch_atomic64_##op(long i, atomic64_t *v) \ 216 { \ 217 __asm__ __volatile__( \ 218 "am"#asm_op"_db.d " " $zero, %1, %0 \n" \ 219 : "+ZB" (v->counter) \ 220 : "r" (I) \ 221 : "memory"); \ 222 } 223 224 #define ATOMIC64_OP_RETURN(op, I, asm_op, c_op) \ 225 static inline long arch_atomic64_##op##_return_relaxed(long i, atomic64_t *v) \ 226 { \ 227 long result; \ 228 __asm__ __volatile__( \ 229 "am"#asm_op"_db.d " " %1, %2, %0 \n" \ 230 : "+ZB" (v->counter), "=&r" (result) \ 231 : "r" (I) \ 232 : "memory"); \ 233 \ 234 return result c_op I; \ 235 } 236 237 #define ATOMIC64_FETCH_OP(op, I, asm_op) \ 238 static inline long arch_atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ 239 { \ 240 long result; \ 241 \ 242 __asm__ __volatile__( \ 243 "am"#asm_op"_db.d " " %1, %2, %0 \n" \ 244 : "+ZB" (v->counter), "=&r" (result) \ 245 : "r" (I) \ 246 : "memory"); \ 247 \ 248 return result; \ 249 } 250 251 #define ATOMIC64_OPS(op, I, asm_op, c_op) \ 252 ATOMIC64_OP(op, I, asm_op) \ 253 ATOMIC64_OP_RETURN(op, I, asm_op, c_op) \ 254 ATOMIC64_FETCH_OP(op, I, asm_op) 255 256 ATOMIC64_OPS(add, i, add, +) 257 ATOMIC64_OPS(sub, -i, add, +) 258 259 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 260 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 261 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 262 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 263 264 #undef ATOMIC64_OPS 265 266 #define ATOMIC64_OPS(op, I, asm_op) \ 267 ATOMIC64_OP(op, I, asm_op) \ 268 ATOMIC64_FETCH_OP(op, I, asm_op) 269 270 ATOMIC64_OPS(and, i, and) 271 ATOMIC64_OPS(or, i, or) 272 ATOMIC64_OPS(xor, i, xor) 273 274 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 275 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 276 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 277 278 #undef ATOMIC64_OPS 279 #undef ATOMIC64_FETCH_OP 280 #undef ATOMIC64_OP_RETURN 281 #undef ATOMIC64_OP 282 283 static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u) 284 { 285 long prev, rc; 286 287 __asm__ __volatile__ ( 288 "0: ll.d %[p], %[c]\n" 289 " beq %[p], %[u], 1f\n" 290 " add.d %[rc], %[p], %[a]\n" 291 " sc.d %[rc], %[c]\n" 292 " beqz %[rc], 0b\n" 293 " b 2f\n" 294 "1:\n" 295 __WEAK_LLSC_MB 296 "2:\n" 297 : [p]"=&r" (prev), [rc]"=&r" (rc), 298 [c] "=ZB" (v->counter) 299 : [a]"r" (a), [u]"r" (u) 300 : "memory"); 301 302 return prev; 303 } 304 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 305 306 /* 307 * arch_atomic64_sub_if_positive - conditionally subtract integer from atomic variable 308 * @i: integer value to subtract 309 * @v: pointer of type atomic64_t 310 * 311 * Atomically test @v and subtract @i if @v is greater or equal than @i. 312 * The function returns the old value of @v minus @i. 313 */ 314 static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v) 315 { 316 long result; 317 long temp; 318 319 if (__builtin_constant_p(i)) { 320 __asm__ __volatile__( 321 "1: ll.d %1, %2 # atomic64_sub_if_positive \n" 322 " addi.d %0, %1, %3 \n" 323 " or %1, %0, $zero \n" 324 " blt %0, $zero, 2f \n" 325 " sc.d %1, %2 \n" 326 " beq %1, $zero, 1b \n" 327 "2: \n" 328 __WEAK_LLSC_MB 329 : "=&r" (result), "=&r" (temp), 330 "+" GCC_OFF_SMALL_ASM() (v->counter) 331 : "I" (-i)); 332 } else { 333 __asm__ __volatile__( 334 "1: ll.d %1, %2 # atomic64_sub_if_positive \n" 335 " sub.d %0, %1, %3 \n" 336 " or %1, %0, $zero \n" 337 " blt %0, $zero, 2f \n" 338 " sc.d %1, %2 \n" 339 " beq %1, $zero, 1b \n" 340 "2: \n" 341 __WEAK_LLSC_MB 342 : "=&r" (result), "=&r" (temp), 343 "+" GCC_OFF_SMALL_ASM() (v->counter) 344 : "r" (i)); 345 } 346 347 return result; 348 } 349 350 #define arch_atomic64_cmpxchg(v, o, n) \ 351 ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n))) 352 #define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), (new))) 353 354 /* 355 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive 356 * @v: pointer of type atomic64_t 357 */ 358 #define arch_atomic64_dec_if_positive(v) arch_atomic64_sub_if_positive(1, v) 359 360 #endif /* CONFIG_64BIT */ 361 362 #endif /* _ASM_ATOMIC_H */ 363