1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * Author: Paul Burton <paul.burton@mips.com> 4 * (C) Copyright 2018 MIPS Tech LLC 5 * 6 * Based on rseq-arm.h: 7 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 8 */ 9 10 /* 11 * RSEQ_SIG uses the break instruction. The instruction pattern is: 12 * 13 * On MIPS: 14 * 0350000d break 0x350 15 * 16 * On nanoMIPS: 17 * 00100350 break 0x350 18 * 19 * On microMIPS: 20 * 0000d407 break 0x350 21 * 22 * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit 23 * halfwords, so the signature halfwords need to be swapped accordingly for 24 * little-endian. 25 */ 26 #if defined(__nanomips__) 27 # ifdef __MIPSEL__ 28 # define RSEQ_SIG 0x03500010 29 # else 30 # define RSEQ_SIG 0x00100350 31 # endif 32 #elif defined(__mips_micromips) 33 # ifdef __MIPSEL__ 34 # define RSEQ_SIG 0xd4070000 35 # else 36 # define RSEQ_SIG 0x0000d407 37 # endif 38 #elif defined(__mips__) 39 # define RSEQ_SIG 0x0350000d 40 #else 41 /* Unknown MIPS architecture. */ 42 #endif 43 44 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory") 45 #define rseq_smp_rmb() rseq_smp_mb() 46 #define rseq_smp_wmb() rseq_smp_mb() 47 48 #define rseq_smp_load_acquire(p) \ 49 __extension__ ({ \ 50 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 51 rseq_smp_mb(); \ 52 ____p1; \ 53 }) 54 55 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 56 57 #define rseq_smp_store_release(p, v) \ 58 do { \ 59 rseq_smp_mb(); \ 60 RSEQ_WRITE_ONCE(*p, v); \ 61 } while (0) 62 63 #ifdef RSEQ_SKIP_FASTPATH 64 #include "rseq-skip.h" 65 #else /* !RSEQ_SKIP_FASTPATH */ 66 67 #if _MIPS_SZLONG == 64 68 # define LONG ".dword" 69 # define LONG_LA "dla" 70 # define LONG_L "ld" 71 # define LONG_S "sd" 72 # define LONG_ADDI "daddiu" 73 # define U32_U64_PAD(x) x 74 #elif _MIPS_SZLONG == 32 75 # define LONG ".word" 76 # define LONG_LA "la" 77 # define LONG_L "lw" 78 # define LONG_S "sw" 79 # define LONG_ADDI "addiu" 80 # ifdef __BIG_ENDIAN 81 # define U32_U64_PAD(x) "0x0, " x 82 # else 83 # define U32_U64_PAD(x) x ", 0x0" 84 # endif 85 #else 86 # error unsupported _MIPS_SZLONG 87 #endif 88 89 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 90 post_commit_offset, abort_ip) \ 91 ".pushsection __rseq_cs, \"aw\"\n\t" \ 92 ".balign 32\n\t" \ 93 __rseq_str(label) ":\n\t" \ 94 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 95 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 96 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ 97 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ 98 ".popsection\n\t" \ 99 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 100 LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \ 101 ".popsection\n\t" 102 103 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 104 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 105 (post_commit_ip - start_ip), abort_ip) 106 107 /* 108 * Exit points of a rseq critical section consist of all instructions outside 109 * of the critical section where a critical section can either branch to or 110 * reach through the normal course of its execution. The abort IP and the 111 * post-commit IP are already part of the __rseq_cs section and should not be 112 * explicitly defined as additional exit points. Knowing all exit points is 113 * useful to assist debuggers stepping over the critical section. 114 */ 115 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 116 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 117 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 118 LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \ 119 ".popsection\n\t" 120 121 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 122 RSEQ_INJECT_ASM(1) \ 123 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \ 124 LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \ 125 __rseq_str(label) ":\n\t" 126 127 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 128 RSEQ_INJECT_ASM(2) \ 129 "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \ 130 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t" 131 132 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 133 abort_label, version, flags, \ 134 start_ip, post_commit_offset, abort_ip) \ 135 ".balign 32\n\t" \ 136 __rseq_str(table_label) ":\n\t" \ 137 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 138 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 139 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ 140 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ 141 ".word " __rseq_str(RSEQ_SIG) "\n\t" \ 142 __rseq_str(label) ":\n\t" \ 143 teardown \ 144 "b %l[" __rseq_str(abort_label) "]\n\t" 145 146 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \ 147 start_ip, post_commit_ip, abort_ip) \ 148 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 149 abort_label, 0x0, 0x0, start_ip, \ 150 (post_commit_ip - start_ip), abort_ip) 151 152 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 153 __rseq_str(label) ":\n\t" \ 154 teardown \ 155 "b %l[" __rseq_str(cmpfail_label) "]\n\t" 156 157 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") 158 159 static inline __attribute__((always_inline)) 160 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 161 { 162 RSEQ_INJECT_C(9) 163 164 rseq_workaround_gcc_asm_size_guess(); 165 __asm__ __volatile__ goto ( 166 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 167 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 168 #ifdef RSEQ_COMPARE_TWICE 169 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 170 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 171 #endif 172 /* Start rseq by storing table entry pointer into rseq_cs. */ 173 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 174 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 175 RSEQ_INJECT_ASM(3) 176 LONG_L " $4, %[v]\n\t" 177 "bne $4, %[expect], %l[cmpfail]\n\t" 178 RSEQ_INJECT_ASM(4) 179 #ifdef RSEQ_COMPARE_TWICE 180 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 181 LONG_L " $4, %[v]\n\t" 182 "bne $4, %[expect], %l[error2]\n\t" 183 #endif 184 /* final store */ 185 LONG_S " %[newv], %[v]\n\t" 186 "2:\n\t" 187 RSEQ_INJECT_ASM(5) 188 "b 5f\n\t" 189 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 190 "5:\n\t" 191 : /* gcc asm goto does not allow outputs */ 192 : [cpu_id] "r" (cpu), 193 [current_cpu_id] "m" (__rseq_abi.cpu_id), 194 [rseq_cs] "m" (__rseq_abi.rseq_cs), 195 [v] "m" (*v), 196 [expect] "r" (expect), 197 [newv] "r" (newv) 198 RSEQ_INJECT_INPUT 199 : "$4", "memory" 200 RSEQ_INJECT_CLOBBER 201 : abort, cmpfail 202 #ifdef RSEQ_COMPARE_TWICE 203 , error1, error2 204 #endif 205 ); 206 rseq_workaround_gcc_asm_size_guess(); 207 return 0; 208 abort: 209 rseq_workaround_gcc_asm_size_guess(); 210 RSEQ_INJECT_FAILED 211 return -1; 212 cmpfail: 213 rseq_workaround_gcc_asm_size_guess(); 214 return 1; 215 #ifdef RSEQ_COMPARE_TWICE 216 error1: 217 rseq_bug("cpu_id comparison failed"); 218 error2: 219 rseq_bug("expected value comparison failed"); 220 #endif 221 } 222 223 static inline __attribute__((always_inline)) 224 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 225 off_t voffp, intptr_t *load, int cpu) 226 { 227 RSEQ_INJECT_C(9) 228 229 rseq_workaround_gcc_asm_size_guess(); 230 __asm__ __volatile__ goto ( 231 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 232 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 233 #ifdef RSEQ_COMPARE_TWICE 234 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 235 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 236 #endif 237 /* Start rseq by storing table entry pointer into rseq_cs. */ 238 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 239 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 240 RSEQ_INJECT_ASM(3) 241 LONG_L " $4, %[v]\n\t" 242 "beq $4, %[expectnot], %l[cmpfail]\n\t" 243 RSEQ_INJECT_ASM(4) 244 #ifdef RSEQ_COMPARE_TWICE 245 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 246 LONG_L " $4, %[v]\n\t" 247 "beq $4, %[expectnot], %l[error2]\n\t" 248 #endif 249 LONG_S " $4, %[load]\n\t" 250 LONG_ADDI " $4, %[voffp]\n\t" 251 LONG_L " $4, 0($4)\n\t" 252 /* final store */ 253 LONG_S " $4, %[v]\n\t" 254 "2:\n\t" 255 RSEQ_INJECT_ASM(5) 256 "b 5f\n\t" 257 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 258 "5:\n\t" 259 : /* gcc asm goto does not allow outputs */ 260 : [cpu_id] "r" (cpu), 261 [current_cpu_id] "m" (__rseq_abi.cpu_id), 262 [rseq_cs] "m" (__rseq_abi.rseq_cs), 263 /* final store input */ 264 [v] "m" (*v), 265 [expectnot] "r" (expectnot), 266 [voffp] "Ir" (voffp), 267 [load] "m" (*load) 268 RSEQ_INJECT_INPUT 269 : "$4", "memory" 270 RSEQ_INJECT_CLOBBER 271 : abort, cmpfail 272 #ifdef RSEQ_COMPARE_TWICE 273 , error1, error2 274 #endif 275 ); 276 rseq_workaround_gcc_asm_size_guess(); 277 return 0; 278 abort: 279 rseq_workaround_gcc_asm_size_guess(); 280 RSEQ_INJECT_FAILED 281 return -1; 282 cmpfail: 283 rseq_workaround_gcc_asm_size_guess(); 284 return 1; 285 #ifdef RSEQ_COMPARE_TWICE 286 error1: 287 rseq_bug("cpu_id comparison failed"); 288 error2: 289 rseq_bug("expected value comparison failed"); 290 #endif 291 } 292 293 static inline __attribute__((always_inline)) 294 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 295 { 296 RSEQ_INJECT_C(9) 297 298 rseq_workaround_gcc_asm_size_guess(); 299 __asm__ __volatile__ goto ( 300 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 301 #ifdef RSEQ_COMPARE_TWICE 302 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 303 #endif 304 /* Start rseq by storing table entry pointer into rseq_cs. */ 305 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 306 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 307 RSEQ_INJECT_ASM(3) 308 #ifdef RSEQ_COMPARE_TWICE 309 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 310 #endif 311 LONG_L " $4, %[v]\n\t" 312 LONG_ADDI " $4, %[count]\n\t" 313 /* final store */ 314 LONG_S " $4, %[v]\n\t" 315 "2:\n\t" 316 RSEQ_INJECT_ASM(4) 317 "b 5f\n\t" 318 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 319 "5:\n\t" 320 : /* gcc asm goto does not allow outputs */ 321 : [cpu_id] "r" (cpu), 322 [current_cpu_id] "m" (__rseq_abi.cpu_id), 323 [rseq_cs] "m" (__rseq_abi.rseq_cs), 324 [v] "m" (*v), 325 [count] "Ir" (count) 326 RSEQ_INJECT_INPUT 327 : "$4", "memory" 328 RSEQ_INJECT_CLOBBER 329 : abort 330 #ifdef RSEQ_COMPARE_TWICE 331 , error1 332 #endif 333 ); 334 rseq_workaround_gcc_asm_size_guess(); 335 return 0; 336 abort: 337 rseq_workaround_gcc_asm_size_guess(); 338 RSEQ_INJECT_FAILED 339 return -1; 340 #ifdef RSEQ_COMPARE_TWICE 341 error1: 342 rseq_bug("cpu_id comparison failed"); 343 #endif 344 } 345 346 static inline __attribute__((always_inline)) 347 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 348 intptr_t *v2, intptr_t newv2, 349 intptr_t newv, int cpu) 350 { 351 RSEQ_INJECT_C(9) 352 353 rseq_workaround_gcc_asm_size_guess(); 354 __asm__ __volatile__ goto ( 355 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 356 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 357 #ifdef RSEQ_COMPARE_TWICE 358 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 359 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 360 #endif 361 /* Start rseq by storing table entry pointer into rseq_cs. */ 362 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 364 RSEQ_INJECT_ASM(3) 365 LONG_L " $4, %[v]\n\t" 366 "bne $4, %[expect], %l[cmpfail]\n\t" 367 RSEQ_INJECT_ASM(4) 368 #ifdef RSEQ_COMPARE_TWICE 369 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 370 LONG_L " $4, %[v]\n\t" 371 "bne $4, %[expect], %l[error2]\n\t" 372 #endif 373 /* try store */ 374 LONG_S " %[newv2], %[v2]\n\t" 375 RSEQ_INJECT_ASM(5) 376 /* final store */ 377 LONG_S " %[newv], %[v]\n\t" 378 "2:\n\t" 379 RSEQ_INJECT_ASM(6) 380 "b 5f\n\t" 381 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 382 "5:\n\t" 383 : /* gcc asm goto does not allow outputs */ 384 : [cpu_id] "r" (cpu), 385 [current_cpu_id] "m" (__rseq_abi.cpu_id), 386 [rseq_cs] "m" (__rseq_abi.rseq_cs), 387 /* try store input */ 388 [v2] "m" (*v2), 389 [newv2] "r" (newv2), 390 /* final store input */ 391 [v] "m" (*v), 392 [expect] "r" (expect), 393 [newv] "r" (newv) 394 RSEQ_INJECT_INPUT 395 : "$4", "memory" 396 RSEQ_INJECT_CLOBBER 397 : abort, cmpfail 398 #ifdef RSEQ_COMPARE_TWICE 399 , error1, error2 400 #endif 401 ); 402 rseq_workaround_gcc_asm_size_guess(); 403 return 0; 404 abort: 405 rseq_workaround_gcc_asm_size_guess(); 406 RSEQ_INJECT_FAILED 407 return -1; 408 cmpfail: 409 rseq_workaround_gcc_asm_size_guess(); 410 return 1; 411 #ifdef RSEQ_COMPARE_TWICE 412 error1: 413 rseq_bug("cpu_id comparison failed"); 414 error2: 415 rseq_bug("expected value comparison failed"); 416 #endif 417 } 418 419 static inline __attribute__((always_inline)) 420 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 421 intptr_t *v2, intptr_t newv2, 422 intptr_t newv, int cpu) 423 { 424 RSEQ_INJECT_C(9) 425 426 rseq_workaround_gcc_asm_size_guess(); 427 __asm__ __volatile__ goto ( 428 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 429 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 430 #ifdef RSEQ_COMPARE_TWICE 431 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 432 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 433 #endif 434 /* Start rseq by storing table entry pointer into rseq_cs. */ 435 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 436 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 437 RSEQ_INJECT_ASM(3) 438 LONG_L " $4, %[v]\n\t" 439 "bne $4, %[expect], %l[cmpfail]\n\t" 440 RSEQ_INJECT_ASM(4) 441 #ifdef RSEQ_COMPARE_TWICE 442 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 443 LONG_L " $4, %[v]\n\t" 444 "bne $4, %[expect], %l[error2]\n\t" 445 #endif 446 /* try store */ 447 LONG_S " %[newv2], %[v2]\n\t" 448 RSEQ_INJECT_ASM(5) 449 "sync\n\t" /* full sync provides store-release */ 450 /* final store */ 451 LONG_S " %[newv], %[v]\n\t" 452 "2:\n\t" 453 RSEQ_INJECT_ASM(6) 454 "b 5f\n\t" 455 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 456 "5:\n\t" 457 : /* gcc asm goto does not allow outputs */ 458 : [cpu_id] "r" (cpu), 459 [current_cpu_id] "m" (__rseq_abi.cpu_id), 460 [rseq_cs] "m" (__rseq_abi.rseq_cs), 461 /* try store input */ 462 [v2] "m" (*v2), 463 [newv2] "r" (newv2), 464 /* final store input */ 465 [v] "m" (*v), 466 [expect] "r" (expect), 467 [newv] "r" (newv) 468 RSEQ_INJECT_INPUT 469 : "$4", "memory" 470 RSEQ_INJECT_CLOBBER 471 : abort, cmpfail 472 #ifdef RSEQ_COMPARE_TWICE 473 , error1, error2 474 #endif 475 ); 476 rseq_workaround_gcc_asm_size_guess(); 477 return 0; 478 abort: 479 rseq_workaround_gcc_asm_size_guess(); 480 RSEQ_INJECT_FAILED 481 return -1; 482 cmpfail: 483 rseq_workaround_gcc_asm_size_guess(); 484 return 1; 485 #ifdef RSEQ_COMPARE_TWICE 486 error1: 487 rseq_bug("cpu_id comparison failed"); 488 error2: 489 rseq_bug("expected value comparison failed"); 490 #endif 491 } 492 493 static inline __attribute__((always_inline)) 494 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 495 intptr_t *v2, intptr_t expect2, 496 intptr_t newv, int cpu) 497 { 498 RSEQ_INJECT_C(9) 499 500 rseq_workaround_gcc_asm_size_guess(); 501 __asm__ __volatile__ goto ( 502 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 503 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 504 #ifdef RSEQ_COMPARE_TWICE 505 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 506 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 507 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 508 #endif 509 /* Start rseq by storing table entry pointer into rseq_cs. */ 510 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 511 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 512 RSEQ_INJECT_ASM(3) 513 LONG_L " $4, %[v]\n\t" 514 "bne $4, %[expect], %l[cmpfail]\n\t" 515 RSEQ_INJECT_ASM(4) 516 LONG_L " $4, %[v2]\n\t" 517 "bne $4, %[expect2], %l[cmpfail]\n\t" 518 RSEQ_INJECT_ASM(5) 519 #ifdef RSEQ_COMPARE_TWICE 520 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 521 LONG_L " $4, %[v]\n\t" 522 "bne $4, %[expect], %l[error2]\n\t" 523 LONG_L " $4, %[v2]\n\t" 524 "bne $4, %[expect2], %l[error3]\n\t" 525 #endif 526 /* final store */ 527 LONG_S " %[newv], %[v]\n\t" 528 "2:\n\t" 529 RSEQ_INJECT_ASM(6) 530 "b 5f\n\t" 531 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 532 "5:\n\t" 533 : /* gcc asm goto does not allow outputs */ 534 : [cpu_id] "r" (cpu), 535 [current_cpu_id] "m" (__rseq_abi.cpu_id), 536 [rseq_cs] "m" (__rseq_abi.rseq_cs), 537 /* cmp2 input */ 538 [v2] "m" (*v2), 539 [expect2] "r" (expect2), 540 /* final store input */ 541 [v] "m" (*v), 542 [expect] "r" (expect), 543 [newv] "r" (newv) 544 RSEQ_INJECT_INPUT 545 : "$4", "memory" 546 RSEQ_INJECT_CLOBBER 547 : abort, cmpfail 548 #ifdef RSEQ_COMPARE_TWICE 549 , error1, error2, error3 550 #endif 551 ); 552 rseq_workaround_gcc_asm_size_guess(); 553 return 0; 554 abort: 555 rseq_workaround_gcc_asm_size_guess(); 556 RSEQ_INJECT_FAILED 557 return -1; 558 cmpfail: 559 rseq_workaround_gcc_asm_size_guess(); 560 return 1; 561 #ifdef RSEQ_COMPARE_TWICE 562 error1: 563 rseq_bug("cpu_id comparison failed"); 564 error2: 565 rseq_bug("1st expected value comparison failed"); 566 error3: 567 rseq_bug("2nd expected value comparison failed"); 568 #endif 569 } 570 571 static inline __attribute__((always_inline)) 572 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 573 void *dst, void *src, size_t len, 574 intptr_t newv, int cpu) 575 { 576 uintptr_t rseq_scratch[3]; 577 578 RSEQ_INJECT_C(9) 579 580 rseq_workaround_gcc_asm_size_guess(); 581 __asm__ __volatile__ goto ( 582 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 583 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 584 #ifdef RSEQ_COMPARE_TWICE 585 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 586 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 587 #endif 588 LONG_S " %[src], %[rseq_scratch0]\n\t" 589 LONG_S " %[dst], %[rseq_scratch1]\n\t" 590 LONG_S " %[len], %[rseq_scratch2]\n\t" 591 /* Start rseq by storing table entry pointer into rseq_cs. */ 592 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 593 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 594 RSEQ_INJECT_ASM(3) 595 LONG_L " $4, %[v]\n\t" 596 "bne $4, %[expect], 5f\n\t" 597 RSEQ_INJECT_ASM(4) 598 #ifdef RSEQ_COMPARE_TWICE 599 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 600 LONG_L " $4, %[v]\n\t" 601 "bne $4, %[expect], 7f\n\t" 602 #endif 603 /* try memcpy */ 604 "beqz %[len], 333f\n\t" \ 605 "222:\n\t" \ 606 "lb $4, 0(%[src])\n\t" \ 607 "sb $4, 0(%[dst])\n\t" \ 608 LONG_ADDI " %[src], 1\n\t" \ 609 LONG_ADDI " %[dst], 1\n\t" \ 610 LONG_ADDI " %[len], -1\n\t" \ 611 "bnez %[len], 222b\n\t" \ 612 "333:\n\t" \ 613 RSEQ_INJECT_ASM(5) 614 /* final store */ 615 LONG_S " %[newv], %[v]\n\t" 616 "2:\n\t" 617 RSEQ_INJECT_ASM(6) 618 /* teardown */ 619 LONG_L " %[len], %[rseq_scratch2]\n\t" 620 LONG_L " %[dst], %[rseq_scratch1]\n\t" 621 LONG_L " %[src], %[rseq_scratch0]\n\t" 622 "b 8f\n\t" 623 RSEQ_ASM_DEFINE_ABORT(3, 4, 624 /* teardown */ 625 LONG_L " %[len], %[rseq_scratch2]\n\t" 626 LONG_L " %[dst], %[rseq_scratch1]\n\t" 627 LONG_L " %[src], %[rseq_scratch0]\n\t", 628 abort, 1b, 2b, 4f) 629 RSEQ_ASM_DEFINE_CMPFAIL(5, 630 /* teardown */ 631 LONG_L " %[len], %[rseq_scratch2]\n\t" 632 LONG_L " %[dst], %[rseq_scratch1]\n\t" 633 LONG_L " %[src], %[rseq_scratch0]\n\t", 634 cmpfail) 635 #ifdef RSEQ_COMPARE_TWICE 636 RSEQ_ASM_DEFINE_CMPFAIL(6, 637 /* teardown */ 638 LONG_L " %[len], %[rseq_scratch2]\n\t" 639 LONG_L " %[dst], %[rseq_scratch1]\n\t" 640 LONG_L " %[src], %[rseq_scratch0]\n\t", 641 error1) 642 RSEQ_ASM_DEFINE_CMPFAIL(7, 643 /* teardown */ 644 LONG_L " %[len], %[rseq_scratch2]\n\t" 645 LONG_L " %[dst], %[rseq_scratch1]\n\t" 646 LONG_L " %[src], %[rseq_scratch0]\n\t", 647 error2) 648 #endif 649 "8:\n\t" 650 : /* gcc asm goto does not allow outputs */ 651 : [cpu_id] "r" (cpu), 652 [current_cpu_id] "m" (__rseq_abi.cpu_id), 653 [rseq_cs] "m" (__rseq_abi.rseq_cs), 654 /* final store input */ 655 [v] "m" (*v), 656 [expect] "r" (expect), 657 [newv] "r" (newv), 658 /* try memcpy input */ 659 [dst] "r" (dst), 660 [src] "r" (src), 661 [len] "r" (len), 662 [rseq_scratch0] "m" (rseq_scratch[0]), 663 [rseq_scratch1] "m" (rseq_scratch[1]), 664 [rseq_scratch2] "m" (rseq_scratch[2]) 665 RSEQ_INJECT_INPUT 666 : "$4", "memory" 667 RSEQ_INJECT_CLOBBER 668 : abort, cmpfail 669 #ifdef RSEQ_COMPARE_TWICE 670 , error1, error2 671 #endif 672 ); 673 rseq_workaround_gcc_asm_size_guess(); 674 return 0; 675 abort: 676 rseq_workaround_gcc_asm_size_guess(); 677 RSEQ_INJECT_FAILED 678 return -1; 679 cmpfail: 680 rseq_workaround_gcc_asm_size_guess(); 681 return 1; 682 #ifdef RSEQ_COMPARE_TWICE 683 error1: 684 rseq_workaround_gcc_asm_size_guess(); 685 rseq_bug("cpu_id comparison failed"); 686 error2: 687 rseq_workaround_gcc_asm_size_guess(); 688 rseq_bug("expected value comparison failed"); 689 #endif 690 } 691 692 static inline __attribute__((always_inline)) 693 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 694 void *dst, void *src, size_t len, 695 intptr_t newv, int cpu) 696 { 697 uintptr_t rseq_scratch[3]; 698 699 RSEQ_INJECT_C(9) 700 701 rseq_workaround_gcc_asm_size_guess(); 702 __asm__ __volatile__ goto ( 703 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 704 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 705 #ifdef RSEQ_COMPARE_TWICE 706 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 707 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 708 #endif 709 LONG_S " %[src], %[rseq_scratch0]\n\t" 710 LONG_S " %[dst], %[rseq_scratch1]\n\t" 711 LONG_S " %[len], %[rseq_scratch2]\n\t" 712 /* Start rseq by storing table entry pointer into rseq_cs. */ 713 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 714 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 715 RSEQ_INJECT_ASM(3) 716 LONG_L " $4, %[v]\n\t" 717 "bne $4, %[expect], 5f\n\t" 718 RSEQ_INJECT_ASM(4) 719 #ifdef RSEQ_COMPARE_TWICE 720 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 721 LONG_L " $4, %[v]\n\t" 722 "bne $4, %[expect], 7f\n\t" 723 #endif 724 /* try memcpy */ 725 "beqz %[len], 333f\n\t" \ 726 "222:\n\t" \ 727 "lb $4, 0(%[src])\n\t" \ 728 "sb $4, 0(%[dst])\n\t" \ 729 LONG_ADDI " %[src], 1\n\t" \ 730 LONG_ADDI " %[dst], 1\n\t" \ 731 LONG_ADDI " %[len], -1\n\t" \ 732 "bnez %[len], 222b\n\t" \ 733 "333:\n\t" \ 734 RSEQ_INJECT_ASM(5) 735 "sync\n\t" /* full sync provides store-release */ 736 /* final store */ 737 LONG_S " %[newv], %[v]\n\t" 738 "2:\n\t" 739 RSEQ_INJECT_ASM(6) 740 /* teardown */ 741 LONG_L " %[len], %[rseq_scratch2]\n\t" 742 LONG_L " %[dst], %[rseq_scratch1]\n\t" 743 LONG_L " %[src], %[rseq_scratch0]\n\t" 744 "b 8f\n\t" 745 RSEQ_ASM_DEFINE_ABORT(3, 4, 746 /* teardown */ 747 LONG_L " %[len], %[rseq_scratch2]\n\t" 748 LONG_L " %[dst], %[rseq_scratch1]\n\t" 749 LONG_L " %[src], %[rseq_scratch0]\n\t", 750 abort, 1b, 2b, 4f) 751 RSEQ_ASM_DEFINE_CMPFAIL(5, 752 /* teardown */ 753 LONG_L " %[len], %[rseq_scratch2]\n\t" 754 LONG_L " %[dst], %[rseq_scratch1]\n\t" 755 LONG_L " %[src], %[rseq_scratch0]\n\t", 756 cmpfail) 757 #ifdef RSEQ_COMPARE_TWICE 758 RSEQ_ASM_DEFINE_CMPFAIL(6, 759 /* teardown */ 760 LONG_L " %[len], %[rseq_scratch2]\n\t" 761 LONG_L " %[dst], %[rseq_scratch1]\n\t" 762 LONG_L " %[src], %[rseq_scratch0]\n\t", 763 error1) 764 RSEQ_ASM_DEFINE_CMPFAIL(7, 765 /* teardown */ 766 LONG_L " %[len], %[rseq_scratch2]\n\t" 767 LONG_L " %[dst], %[rseq_scratch1]\n\t" 768 LONG_L " %[src], %[rseq_scratch0]\n\t", 769 error2) 770 #endif 771 "8:\n\t" 772 : /* gcc asm goto does not allow outputs */ 773 : [cpu_id] "r" (cpu), 774 [current_cpu_id] "m" (__rseq_abi.cpu_id), 775 [rseq_cs] "m" (__rseq_abi.rseq_cs), 776 /* final store input */ 777 [v] "m" (*v), 778 [expect] "r" (expect), 779 [newv] "r" (newv), 780 /* try memcpy input */ 781 [dst] "r" (dst), 782 [src] "r" (src), 783 [len] "r" (len), 784 [rseq_scratch0] "m" (rseq_scratch[0]), 785 [rseq_scratch1] "m" (rseq_scratch[1]), 786 [rseq_scratch2] "m" (rseq_scratch[2]) 787 RSEQ_INJECT_INPUT 788 : "$4", "memory" 789 RSEQ_INJECT_CLOBBER 790 : abort, cmpfail 791 #ifdef RSEQ_COMPARE_TWICE 792 , error1, error2 793 #endif 794 ); 795 rseq_workaround_gcc_asm_size_guess(); 796 return 0; 797 abort: 798 rseq_workaround_gcc_asm_size_guess(); 799 RSEQ_INJECT_FAILED 800 return -1; 801 cmpfail: 802 rseq_workaround_gcc_asm_size_guess(); 803 return 1; 804 #ifdef RSEQ_COMPARE_TWICE 805 error1: 806 rseq_workaround_gcc_asm_size_guess(); 807 rseq_bug("cpu_id comparison failed"); 808 error2: 809 rseq_workaround_gcc_asm_size_guess(); 810 rseq_bug("expected value comparison failed"); 811 #endif 812 } 813 814 #endif /* !RSEQ_SKIP_FASTPATH */ 815