1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike 4 * other architectures, the ebreak instruction has no immediate field for 5 * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG. 6 * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it 7 * is an uncommon instruction and will raise an illegal instruction 8 * exception when executed in all modes. 9 */ 10 #include <endian.h> 11 12 #if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN) 13 #define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */ 14 #else 15 #error "Currently, RSEQ only supports Little-Endian version" 16 #endif 17 18 #if __riscv_xlen == 64 19 #define __REG_SEL(a, b) a 20 #elif __riscv_xlen == 32 21 #define __REG_SEL(a, b) b 22 #endif 23 24 #define REG_L __REG_SEL("ld ", "lw ") 25 #define REG_S __REG_SEL("sd ", "sw ") 26 27 #define RISCV_FENCE(p, s) \ 28 __asm__ __volatile__ ("fence " #p "," #s : : : "memory") 29 #define rseq_smp_mb() RISCV_FENCE(rw, rw) 30 #define rseq_smp_rmb() RISCV_FENCE(r, r) 31 #define rseq_smp_wmb() RISCV_FENCE(w, w) 32 #define RSEQ_ASM_TMP_REG_1 "t6" 33 #define RSEQ_ASM_TMP_REG_2 "t5" 34 #define RSEQ_ASM_TMP_REG_3 "t4" 35 #define RSEQ_ASM_TMP_REG_4 "t3" 36 37 #define rseq_smp_load_acquire(p) \ 38 __extension__ ({ \ 39 __typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ 40 RISCV_FENCE(r, rw) \ 41 ____p1; \ 42 }) 43 44 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 45 46 #define rseq_smp_store_release(p, v) \ 47 do { \ 48 RISCV_FENCE(rw, w); \ 49 RSEQ_WRITE_ONCE(*(p), v); \ 50 } while (0) 51 52 #ifdef RSEQ_SKIP_FASTPATH 53 #include "rseq-skip.h" 54 #else /* !RSEQ_SKIP_FASTPATH */ 55 56 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 57 post_commit_offset, abort_ip) \ 58 ".pushsection __rseq_cs, \"aw\"\n" \ 59 ".balign 32\n" \ 60 __rseq_str(label) ":\n" \ 61 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ 62 ".quad " __rseq_str(start_ip) ", " \ 63 __rseq_str(post_commit_offset) ", " \ 64 __rseq_str(abort_ip) "\n" \ 65 ".popsection\n\t" \ 66 ".pushsection __rseq_cs_ptr_array, \"aw\"\n" \ 67 ".quad " __rseq_str(label) "b\n" \ 68 ".popsection\n" 69 70 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 71 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 72 ((post_commit_ip) - (start_ip)), abort_ip) 73 74 /* 75 * Exit points of a rseq critical section consist of all instructions outside 76 * of the critical section where a critical section can either branch to or 77 * reach through the normal course of its execution. The abort IP and the 78 * post-commit IP are already part of the __rseq_cs section and should not be 79 * explicitly defined as additional exit points. Knowing all exit points is 80 * useful to assist debuggers stepping over the critical section. 81 */ 82 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 83 ".pushsection __rseq_exit_point_array, \"aw\"\n" \ 84 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \ 85 ".popsection\n" 86 87 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 88 RSEQ_INJECT_ASM(1) \ 89 "la "RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \ 90 REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \ 91 __rseq_str(label) ":\n" 92 93 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 94 "j 222f\n" \ 95 ".balign 4\n" \ 96 ".long " __rseq_str(RSEQ_SIG) "\n" \ 97 __rseq_str(label) ":\n" \ 98 "j %l[" __rseq_str(abort_label) "]\n" \ 99 "222:\n" 100 101 #define RSEQ_ASM_OP_STORE(value, var) \ 102 REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 103 104 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ 105 REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ 106 "bne "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ 107 __rseq_str(label) "\n" 108 109 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ 110 "lw "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ 111 "bne "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ 112 __rseq_str(label) "\n" 113 114 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \ 115 REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ 116 "beq "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ 117 __rseq_str(label) "\n" 118 119 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 120 RSEQ_INJECT_ASM(2) \ 121 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label) 122 123 #define RSEQ_ASM_OP_R_LOAD(var) \ 124 REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" 125 126 #define RSEQ_ASM_OP_R_STORE(var) \ 127 REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" 128 129 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \ 130 "add "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \ 131 RSEQ_ASM_TMP_REG_1 "\n" \ 132 REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n" 133 134 #define RSEQ_ASM_OP_R_ADD(count) \ 135 "add "RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \ 136 ", %[" __rseq_str(count) "]\n" 137 138 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ 139 RSEQ_ASM_OP_STORE(value, var) \ 140 __rseq_str(post_commit_label) ":\n" 141 142 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \ 143 "fence rw, w\n" \ 144 RSEQ_ASM_OP_STORE(value, var) \ 145 __rseq_str(post_commit_label) ":\n" 146 147 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ 148 REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ 149 __rseq_str(post_commit_label) ":\n" 150 151 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \ 152 "beqz %[" __rseq_str(len) "], 333f\n" \ 153 "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n" \ 154 "mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n" \ 155 "mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n" \ 156 "222:\n" \ 157 "lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n" \ 158 "sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n" \ 159 "addi " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n" \ 160 "addi " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n" \ 161 "addi " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n" \ 162 "bnez " RSEQ_ASM_TMP_REG_1 ", 222b\n" \ 163 "333:\n" 164 165 #define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label) \ 166 "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n" \ 167 RSEQ_ASM_OP_R_ADD(off) \ 168 REG_L RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \ 169 RSEQ_ASM_OP_R_ADD(inc) \ 170 __rseq_str(post_commit_label) ":\n" 171 172 static inline __always_inline 173 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 174 { 175 RSEQ_INJECT_C(9) 176 177 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 178 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 179 #ifdef RSEQ_COMPARE_TWICE 180 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 181 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 182 #endif 183 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 184 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 185 RSEQ_INJECT_ASM(3) 186 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 187 RSEQ_INJECT_ASM(4) 188 #ifdef RSEQ_COMPARE_TWICE 189 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 190 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 191 #endif 192 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 193 RSEQ_INJECT_ASM(5) 194 RSEQ_ASM_DEFINE_ABORT(4, abort) 195 : /* gcc asm goto does not allow outputs */ 196 : [cpu_id] "r" (cpu), 197 [current_cpu_id] "m" (__rseq_abi.cpu_id), 198 [rseq_cs] "m" (__rseq_abi.rseq_cs), 199 [v] "m" (*v), 200 [expect] "r" (expect), 201 [newv] "r" (newv) 202 RSEQ_INJECT_INPUT 203 : "memory", RSEQ_ASM_TMP_REG_1 204 RSEQ_INJECT_CLOBBER 205 : abort, cmpfail 206 #ifdef RSEQ_COMPARE_TWICE 207 , error1, error2 208 #endif 209 ); 210 211 return 0; 212 abort: 213 RSEQ_INJECT_FAILED 214 return -1; 215 cmpfail: 216 return 1; 217 #ifdef RSEQ_COMPARE_TWICE 218 error1: 219 rseq_bug("cpu_id comparison failed"); 220 error2: 221 rseq_bug("expected value comparison failed"); 222 #endif 223 } 224 225 static inline __always_inline 226 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 227 off_t voffp, intptr_t *load, int cpu) 228 { 229 RSEQ_INJECT_C(9) 230 231 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 232 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 233 #ifdef RSEQ_COMPARE_TWICE 234 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 235 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 236 #endif 237 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 238 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 239 RSEQ_INJECT_ASM(3) 240 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]") 241 RSEQ_INJECT_ASM(4) 242 #ifdef RSEQ_COMPARE_TWICE 243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 244 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]") 245 #endif 246 RSEQ_ASM_OP_R_LOAD(v) 247 RSEQ_ASM_OP_R_STORE(load) 248 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 249 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 250 RSEQ_INJECT_ASM(5) 251 RSEQ_ASM_DEFINE_ABORT(4, abort) 252 : /* gcc asm goto does not allow outputs */ 253 : [cpu_id] "r" (cpu), 254 [current_cpu_id] "m" (__rseq_abi.cpu_id), 255 [rseq_cs] "m" (__rseq_abi.rseq_cs), 256 [v] "m" (*v), 257 [expectnot] "r" (expectnot), 258 [load] "m" (*load), 259 [voffp] "r" (voffp) 260 RSEQ_INJECT_INPUT 261 : "memory", RSEQ_ASM_TMP_REG_1 262 RSEQ_INJECT_CLOBBER 263 : abort, cmpfail 264 #ifdef RSEQ_COMPARE_TWICE 265 , error1, error2 266 #endif 267 ); 268 return 0; 269 abort: 270 RSEQ_INJECT_FAILED 271 return -1; 272 cmpfail: 273 return 1; 274 #ifdef RSEQ_COMPARE_TWICE 275 error1: 276 rseq_bug("cpu_id comparison failed"); 277 error2: 278 rseq_bug("expected value comparison failed"); 279 #endif 280 } 281 282 static inline __always_inline 283 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 284 { 285 RSEQ_INJECT_C(9) 286 287 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 288 #ifdef RSEQ_COMPARE_TWICE 289 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 290 #endif 291 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 292 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 293 RSEQ_INJECT_ASM(3) 294 #ifdef RSEQ_COMPARE_TWICE 295 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 296 #endif 297 RSEQ_ASM_OP_R_LOAD(v) 298 RSEQ_ASM_OP_R_ADD(count) 299 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 300 RSEQ_INJECT_ASM(4) 301 RSEQ_ASM_DEFINE_ABORT(4, abort) 302 : /* gcc asm goto does not allow outputs */ 303 : [cpu_id] "r" (cpu), 304 [current_cpu_id] "m" (__rseq_abi.cpu_id), 305 [rseq_cs] "m" (__rseq_abi.rseq_cs), 306 [v] "m" (*v), 307 [count] "r" (count) 308 RSEQ_INJECT_INPUT 309 : "memory", RSEQ_ASM_TMP_REG_1 310 RSEQ_INJECT_CLOBBER 311 : abort 312 #ifdef RSEQ_COMPARE_TWICE 313 , error1 314 #endif 315 ); 316 return 0; 317 abort: 318 RSEQ_INJECT_FAILED 319 return -1; 320 #ifdef RSEQ_COMPARE_TWICE 321 error1: 322 rseq_bug("cpu_id comparison failed"); 323 #endif 324 } 325 326 static inline __always_inline 327 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 328 intptr_t *v2, intptr_t newv2, 329 intptr_t newv, int cpu) 330 { 331 RSEQ_INJECT_C(9) 332 333 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 334 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 335 #ifdef RSEQ_COMPARE_TWICE 336 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 337 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 338 #endif 339 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 340 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 341 RSEQ_INJECT_ASM(3) 342 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 343 RSEQ_INJECT_ASM(4) 344 #ifdef RSEQ_COMPARE_TWICE 345 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 346 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 347 #endif 348 RSEQ_ASM_OP_STORE(newv2, v2) 349 RSEQ_INJECT_ASM(5) 350 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 351 RSEQ_INJECT_ASM(6) 352 RSEQ_ASM_DEFINE_ABORT(4, abort) 353 : /* gcc asm goto does not allow outputs */ 354 : [cpu_id] "r" (cpu), 355 [current_cpu_id] "m" (__rseq_abi.cpu_id), 356 [rseq_cs] "m" (__rseq_abi.rseq_cs), 357 [expect] "r" (expect), 358 [v] "m" (*v), 359 [newv] "r" (newv), 360 [v2] "m" (*v2), 361 [newv2] "r" (newv2) 362 RSEQ_INJECT_INPUT 363 : "memory", RSEQ_ASM_TMP_REG_1 364 RSEQ_INJECT_CLOBBER 365 : abort, cmpfail 366 #ifdef RSEQ_COMPARE_TWICE 367 , error1, error2 368 #endif 369 ); 370 371 return 0; 372 abort: 373 RSEQ_INJECT_FAILED 374 return -1; 375 cmpfail: 376 return 1; 377 #ifdef RSEQ_COMPARE_TWICE 378 error1: 379 rseq_bug("cpu_id comparison failed"); 380 error2: 381 rseq_bug("expected value comparison failed"); 382 #endif 383 } 384 385 static inline __always_inline 386 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 387 intptr_t *v2, intptr_t newv2, 388 intptr_t newv, int cpu) 389 { 390 RSEQ_INJECT_C(9) 391 392 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 393 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 394 #ifdef RSEQ_COMPARE_TWICE 395 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 396 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 397 #endif 398 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 399 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 400 RSEQ_INJECT_ASM(3) 401 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 402 RSEQ_INJECT_ASM(4) 403 #ifdef RSEQ_COMPARE_TWICE 404 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 405 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 406 #endif 407 RSEQ_ASM_OP_STORE(newv2, v2) 408 RSEQ_INJECT_ASM(5) 409 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 410 RSEQ_INJECT_ASM(6) 411 RSEQ_ASM_DEFINE_ABORT(4, abort) 412 : /* gcc asm goto does not allow outputs */ 413 : [cpu_id] "r" (cpu), 414 [current_cpu_id] "m" (__rseq_abi.cpu_id), 415 [rseq_cs] "m" (__rseq_abi.rseq_cs), 416 [expect] "r" (expect), 417 [v] "m" (*v), 418 [newv] "r" (newv), 419 [v2] "m" (*v2), 420 [newv2] "r" (newv2) 421 RSEQ_INJECT_INPUT 422 : "memory", RSEQ_ASM_TMP_REG_1 423 RSEQ_INJECT_CLOBBER 424 : abort, cmpfail 425 #ifdef RSEQ_COMPARE_TWICE 426 , error1, error2 427 #endif 428 ); 429 430 return 0; 431 abort: 432 RSEQ_INJECT_FAILED 433 return -1; 434 cmpfail: 435 return 1; 436 #ifdef RSEQ_COMPARE_TWICE 437 error1: 438 rseq_bug("cpu_id comparison failed"); 439 error2: 440 rseq_bug("expected value comparison failed"); 441 #endif 442 } 443 444 static inline __always_inline 445 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 446 intptr_t *v2, intptr_t expect2, 447 intptr_t newv, int cpu) 448 { 449 RSEQ_INJECT_C(9) 450 451 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 452 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 453 #ifdef RSEQ_COMPARE_TWICE 454 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 455 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 456 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]") 457 #endif 458 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 459 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 460 RSEQ_INJECT_ASM(3) 461 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 462 RSEQ_INJECT_ASM(4) 463 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]") 464 RSEQ_INJECT_ASM(5) 465 #ifdef RSEQ_COMPARE_TWICE 466 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 467 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 468 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]") 469 #endif 470 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 471 RSEQ_INJECT_ASM(6) 472 RSEQ_ASM_DEFINE_ABORT(4, abort) 473 : /* gcc asm goto does not allow outputs */ 474 : [cpu_id] "r" (cpu), 475 [current_cpu_id] "m" (__rseq_abi.cpu_id), 476 [rseq_cs] "m" (__rseq_abi.rseq_cs), 477 [v] "m" (*v), 478 [expect] "r" (expect), 479 [v2] "m" (*v2), 480 [expect2] "r" (expect2), 481 [newv] "r" (newv) 482 RSEQ_INJECT_INPUT 483 : "memory", RSEQ_ASM_TMP_REG_1 484 RSEQ_INJECT_CLOBBER 485 : abort, cmpfail 486 #ifdef RSEQ_COMPARE_TWICE 487 , error1, error2, error3 488 #endif 489 ); 490 491 return 0; 492 abort: 493 RSEQ_INJECT_FAILED 494 return -1; 495 cmpfail: 496 return 1; 497 #ifdef RSEQ_COMPARE_TWICE 498 error1: 499 rseq_bug("cpu_id comparison failed"); 500 error2: 501 rseq_bug("expected value comparison failed"); 502 error3: 503 rseq_bug("2nd expected value comparison failed"); 504 #endif 505 } 506 507 static inline __always_inline 508 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 509 void *dst, void *src, size_t len, 510 intptr_t newv, int cpu) 511 { 512 RSEQ_INJECT_C(9) 513 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 514 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 515 #ifdef RSEQ_COMPARE_TWICE 516 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 517 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 518 #endif 519 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 520 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 521 RSEQ_INJECT_ASM(3) 522 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 523 RSEQ_INJECT_ASM(4) 524 #ifdef RSEQ_COMPARE_TWICE 525 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 526 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 527 #endif 528 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 529 RSEQ_INJECT_ASM(5) 530 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 531 RSEQ_INJECT_ASM(6) 532 RSEQ_ASM_DEFINE_ABORT(4, abort) 533 : /* gcc asm goto does not allow outputs */ 534 : [cpu_id] "r" (cpu), 535 [current_cpu_id] "m" (__rseq_abi.cpu_id), 536 [rseq_cs] "m" (__rseq_abi.rseq_cs), 537 [expect] "r" (expect), 538 [v] "m" (*v), 539 [newv] "r" (newv), 540 [dst] "r" (dst), 541 [src] "r" (src), 542 [len] "r" (len) 543 RSEQ_INJECT_INPUT 544 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, 545 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 546 RSEQ_INJECT_CLOBBER 547 : abort, cmpfail 548 #ifdef RSEQ_COMPARE_TWICE 549 , error1, error2 550 #endif 551 ); 552 553 return 0; 554 abort: 555 RSEQ_INJECT_FAILED 556 return -1; 557 cmpfail: 558 return 1; 559 #ifdef RSEQ_COMPARE_TWICE 560 error1: 561 rseq_bug("cpu_id comparison failed"); 562 error2: 563 rseq_bug("expected value comparison failed"); 564 #endif 565 } 566 567 static inline __always_inline 568 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 569 void *dst, void *src, size_t len, 570 intptr_t newv, int cpu) 571 { 572 RSEQ_INJECT_C(9) 573 574 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 575 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 576 #ifdef RSEQ_COMPARE_TWICE 577 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 578 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 579 #endif 580 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 581 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 582 RSEQ_INJECT_ASM(3) 583 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 584 RSEQ_INJECT_ASM(4) 585 #ifdef RSEQ_COMPARE_TWICE 586 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 587 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 588 #endif 589 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 590 RSEQ_INJECT_ASM(5) 591 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 592 RSEQ_INJECT_ASM(6) 593 RSEQ_ASM_DEFINE_ABORT(4, abort) 594 : /* gcc asm goto does not allow outputs */ 595 : [cpu_id] "r" (cpu), 596 [current_cpu_id] "m" (__rseq_abi.cpu_id), 597 [rseq_cs] "m" (__rseq_abi.rseq_cs), 598 [expect] "r" (expect), 599 [v] "m" (*v), 600 [newv] "r" (newv), 601 [dst] "r" (dst), 602 [src] "r" (src), 603 [len] "r" (len) 604 RSEQ_INJECT_INPUT 605 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, 606 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 607 RSEQ_INJECT_CLOBBER 608 : abort, cmpfail 609 #ifdef RSEQ_COMPARE_TWICE 610 , error1, error2 611 #endif 612 ); 613 614 return 0; 615 abort: 616 RSEQ_INJECT_FAILED 617 return -1; 618 cmpfail: 619 return 1; 620 #ifdef RSEQ_COMPARE_TWICE 621 error1: 622 rseq_bug("cpu_id comparison failed"); 623 error2: 624 rseq_bug("expected value comparison failed"); 625 #endif 626 } 627 628 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 629 630 /* 631 * pval = *(ptr+off) 632 * *pval += inc; 633 */ 634 static inline __always_inline 635 int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu) 636 { 637 RSEQ_INJECT_C(9) 638 639 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 640 #ifdef RSEQ_COMPARE_TWICE 641 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 642 #endif 643 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 644 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 645 RSEQ_INJECT_ASM(3) 646 #ifdef RSEQ_COMPARE_TWICE 647 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 648 #endif 649 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3) 650 RSEQ_INJECT_ASM(4) 651 RSEQ_ASM_DEFINE_ABORT(4, abort) 652 : /* gcc asm goto does not allow outputs */ 653 : [cpu_id] "r" (cpu), 654 [current_cpu_id] "m" (__rseq_abi.cpu_id), 655 [rseq_cs] "m" (__rseq_abi.rseq_cs), 656 [ptr] "r" (ptr), 657 [off] "er" (off), 658 [inc] "er" (inc) 659 RSEQ_INJECT_INPUT 660 : "memory", RSEQ_ASM_TMP_REG_1 661 RSEQ_INJECT_CLOBBER 662 : abort 663 #ifdef RSEQ_COMPARE_TWICE 664 , error1 665 #endif 666 ); 667 return 0; 668 abort: 669 RSEQ_INJECT_FAILED 670 return -1; 671 #ifdef RSEQ_COMPARE_TWICE 672 error1: 673 rseq_bug("cpu_id comparison failed"); 674 #endif 675 } 676 677 #endif /* !RSEQ_SKIP_FASTPATH */ 678