10b57cec5SDimitry Andric// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 20b57cec5SDimitry Andric// 30b57cec5SDimitry Andric// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric// See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric// 70b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric/// 90b57cec5SDimitry Andric/// \file 100b57cec5SDimitry Andric/// WebAssembly Atomic operand code-gen constructs. 110b57cec5SDimitry Andric/// 120b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 130b57cec5SDimitry Andric 140b57cec5SDimitry Andriclet UseNamedOperandTable = 1 in 150b57cec5SDimitry Andricmulticlass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 165ffd83dbSDimitry Andric list<dag> pattern_r, string asmstr_r, 175ffd83dbSDimitry Andric string asmstr_s, bits<32> atomic_op, 185ffd83dbSDimitry Andric string is64 = "false"> { 190b57cec5SDimitry Andric defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 205ffd83dbSDimitry Andric !or(0xfe00, !and(0xff, atomic_op)), is64>, 210b57cec5SDimitry Andric Requires<[HasAtomics]>; 220b57cec5SDimitry Andric} 230b57cec5SDimitry Andric 240b57cec5SDimitry Andricmulticlass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 250b57cec5SDimitry Andric bits<32> atomic_op = -1> { 260b57cec5SDimitry Andric defm "" : NRI<oops, iops, pattern, asmstr, 270b57cec5SDimitry Andric !or(0xfe00, !and(0xff, atomic_op))>, 280b57cec5SDimitry Andric Requires<[HasAtomics]>; 290b57cec5SDimitry Andric} 300b57cec5SDimitry Andric 310b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 320b57cec5SDimitry Andric// Atomic wait / notify 330b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 340b57cec5SDimitry Andric 350b57cec5SDimitry Andriclet hasSideEffects = 1 in { 36e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_NOTIFY_A32 : 370b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 380b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 390b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 40e8d8bef9SDimitry Andric "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 41e8d8bef9SDimitry Andric "memory.atomic.notify \t${off}${p2align}", 0x00, "false">; 42e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_NOTIFY_A64 : 435ffd83dbSDimitry Andric ATOMIC_I<(outs I32:$dst), 445ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$count), 455ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 46e8d8bef9SDimitry Andric "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 47e8d8bef9SDimitry Andric "memory.atomic.notify \t${off}${p2align}", 0x00, "true">; 480b57cec5SDimitry Andriclet mayLoad = 1 in { 49e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT32_A32 : 500b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 510b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 520b57cec5SDimitry Andric I64:$timeout), 530b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54e8d8bef9SDimitry Andric "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 55e8d8bef9SDimitry Andric "memory.atomic.wait32 \t${off}${p2align}", 0x01, "false">; 56e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT32_A64 : 575ffd83dbSDimitry Andric ATOMIC_I<(outs I32:$dst), 585ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$exp, 595ffd83dbSDimitry Andric I64:$timeout), 605ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 61e8d8bef9SDimitry Andric "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 62e8d8bef9SDimitry Andric "memory.atomic.wait32 \t${off}${p2align}", 0x01, "true">; 63e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT64_A32 : 640b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 650b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 660b57cec5SDimitry Andric I64:$timeout), 670b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 68e8d8bef9SDimitry Andric "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 69e8d8bef9SDimitry Andric "memory.atomic.wait64 \t${off}${p2align}", 0x02, "false">; 70e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT64_A64 : 715ffd83dbSDimitry Andric ATOMIC_I<(outs I32:$dst), 725ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp, 735ffd83dbSDimitry Andric I64:$timeout), 745ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 75e8d8bef9SDimitry Andric "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 76e8d8bef9SDimitry Andric "memory.atomic.wait64 \t${off}${p2align}", 0x02, "true">; 770b57cec5SDimitry Andric} // mayLoad = 1 780b57cec5SDimitry Andric} // hasSideEffects = 1 790b57cec5SDimitry Andric 800b57cec5SDimitry Andric// Select notifys with no constant offset. 815ffd83dbSDimitry Andricdef NotifyPatNoOffset_A32 : 82e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify I32:$addr, I32:$count)), 83e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A32 0, 0, I32:$addr, I32:$count)>, 84e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 855ffd83dbSDimitry Andricdef NotifyPatNoOffset_A64 : 86e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify I64:$addr, I32:$count)), 87e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A64 0, 0, I64:$addr, I32:$count)>, 88e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 890b57cec5SDimitry Andric 900b57cec5SDimitry Andric// Select notifys with a constant offset. 910b57cec5SDimitry Andric 920b57cec5SDimitry Andric// Pattern with address + immediate offset 935ffd83dbSDimitry Andricmulticlass NotifyPatImmOff<PatFrag operand, string inst> { 94e8d8bef9SDimitry Andric def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I32:$addr, imm:$off), 955ffd83dbSDimitry Andric I32:$count)), 965ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, I32:$count)>, 97e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 98e8d8bef9SDimitry Andric def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I64:$addr, imm:$off), 995ffd83dbSDimitry Andric I32:$count)), 1005ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, I32:$count)>, 101e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1025ffd83dbSDimitry Andric} 103e8d8bef9SDimitry Andricdefm : NotifyPatImmOff<regPlusImm, "MEMORY_ATOMIC_NOTIFY">; 104e8d8bef9SDimitry Andricdefm : NotifyPatImmOff<or_is_add, "MEMORY_ATOMIC_NOTIFY">; 1050b57cec5SDimitry Andric 1060b57cec5SDimitry Andric// Select notifys with just a constant offset. 1075ffd83dbSDimitry Andricdef NotifyPatOffsetOnly_A32 : 108e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)), 109e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A32 0, imm:$off, (CONST_I32 0), I32:$count)>, 110e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 1115ffd83dbSDimitry Andricdef NotifyPatOffsetOnly_A64 : 112e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)), 113e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A64 0, imm:$off, (CONST_I64 0), I32:$count)>, 114e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1150b57cec5SDimitry Andric 1165ffd83dbSDimitry Andricdef NotifyPatGlobalAddrOffOnly_A32 : 117e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 1180b57cec5SDimitry Andric I32:$count)), 119e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A32 0, tglobaladdr:$off, (CONST_I32 0), I32:$count) 120e8d8bef9SDimitry Andric >, 121e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 1225ffd83dbSDimitry Andricdef NotifyPatGlobalAddrOffOnly_A64 : 123e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 1245ffd83dbSDimitry Andric I32:$count)), 125e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A64 0, tglobaladdr:$off, (CONST_I64 0), I32:$count) 126e8d8bef9SDimitry Andric >, 127e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 1280b57cec5SDimitry Andric 1290b57cec5SDimitry Andric// Select waits with no constant offset. 1305ffd83dbSDimitry Andricmulticlass WaitPatNoOffset<ValueType ty, Intrinsic kind, 1315ffd83dbSDimitry Andric string inst> { 1325ffd83dbSDimitry Andric def : Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 1335ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, I64:$timeout)>, 134e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 1355ffd83dbSDimitry Andric def : Pat<(i32 (kind I64:$addr, ty:$exp, I64:$timeout)), 1365ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, I64:$timeout)>, 137e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1385ffd83dbSDimitry Andric} 139e8d8bef9SDimitry Andricdefm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32, 140e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 141e8d8bef9SDimitry Andricdefm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64, 142e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 143e8d8bef9SDimitry Andricdefm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32, 144e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 145e8d8bef9SDimitry Andricdefm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64, 146e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 1470b57cec5SDimitry Andric 1480b57cec5SDimitry Andric// Select waits with a constant offset. 1490b57cec5SDimitry Andric 1500b57cec5SDimitry Andric// Pattern with address + immediate offset 1515ffd83dbSDimitry Andricmulticlass WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, 1525ffd83dbSDimitry Andric string inst> { 1535ffd83dbSDimitry Andric def : Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 1545ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, 1555ffd83dbSDimitry Andric I64:$timeout)>, 156e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 1575ffd83dbSDimitry Andric def : Pat<(i32 (kind (operand I64:$addr, imm:$off), ty:$exp, I64:$timeout)), 1585ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, 1595ffd83dbSDimitry Andric I64:$timeout)>, 160e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1615ffd83dbSDimitry Andric} 162e8d8bef9SDimitry Andricdefm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, regPlusImm, 163e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 164e8d8bef9SDimitry Andricdefm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, or_is_add, 165e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 166e8d8bef9SDimitry Andricdefm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, regPlusImm, 167e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 168e8d8bef9SDimitry Andricdefm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, or_is_add, 169e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 1700b57cec5SDimitry Andric 171e8d8bef9SDimitry Andric// Select waits with just a constant offset. 1725ffd83dbSDimitry Andricmulticlass WaitPatOffsetOnly<ValueType ty, Intrinsic kind, string inst> { 1735ffd83dbSDimitry Andric def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 1745ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 1755ffd83dbSDimitry Andric I64:$timeout)>, 176e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 1775ffd83dbSDimitry Andric def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 1785ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 1795ffd83dbSDimitry Andric I64:$timeout)>, 180e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1815ffd83dbSDimitry Andric} 182e8d8bef9SDimitry Andricdefm : WaitPatOffsetOnly<i32, int_wasm_memory_atomic_wait32, 183e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 184e8d8bef9SDimitry Andricdefm : WaitPatOffsetOnly<i64, int_wasm_memory_atomic_wait64, 185e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 1860b57cec5SDimitry Andric 1875ffd83dbSDimitry Andricmulticlass WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, string inst> { 1885ffd83dbSDimitry Andric def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 1895ffd83dbSDimitry Andric I64:$timeout)), 1905ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 1915ffd83dbSDimitry Andric I64:$timeout)>, 192e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 1935ffd83dbSDimitry Andric def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 1945ffd83dbSDimitry Andric I64:$timeout)), 1955ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 1965ffd83dbSDimitry Andric I64:$timeout)>, 197e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 1985ffd83dbSDimitry Andric} 199e8d8bef9SDimitry Andricdefm : WaitPatGlobalAddrOffOnly<i32, int_wasm_memory_atomic_wait32, 200e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 201e8d8bef9SDimitry Andricdefm : WaitPatGlobalAddrOffOnly<i64, int_wasm_memory_atomic_wait64, 202e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 2030b57cec5SDimitry Andric 2040b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2058bcb0991SDimitry Andric// Atomic fences 2068bcb0991SDimitry Andric//===----------------------------------------------------------------------===// 2078bcb0991SDimitry Andric 2088bcb0991SDimitry Andric// A compiler fence instruction that prevents reordering of instructions. 2098bcb0991SDimitry Andriclet Defs = [ARGUMENTS] in { 2108bcb0991SDimitry Andriclet isPseudo = 1, hasSideEffects = 1 in 2118bcb0991SDimitry Andricdefm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 2128bcb0991SDimitry Andriclet hasSideEffects = 1 in 2138bcb0991SDimitry Andricdefm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", 2148bcb0991SDimitry Andric 0x03>; 2158bcb0991SDimitry Andric} // Defs = [ARGUMENTS] 2168bcb0991SDimitry Andric 2178bcb0991SDimitry Andric//===----------------------------------------------------------------------===// 2180b57cec5SDimitry Andric// Atomic loads 2190b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2200b57cec5SDimitry Andric 2210b57cec5SDimitry Andricmulticlass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 2225ffd83dbSDimitry Andric defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 2235ffd83dbSDimitry Andric [HasAtomics]>; 2240b57cec5SDimitry Andric} 2250b57cec5SDimitry Andric 2260b57cec5SDimitry Andricdefm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 2270b57cec5SDimitry Andricdefm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 2280b57cec5SDimitry Andric 2290b57cec5SDimitry Andric// Select loads with no constant offset. 2305ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 2315ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 2320b57cec5SDimitry Andric 2330b57cec5SDimitry Andric// Select loads with a constant offset. 2340b57cec5SDimitry Andric 2350b57cec5SDimitry Andric// Pattern with address + immediate offset 2365ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_32, regPlusImm, "ATOMIC_LOAD_I32">; 2375ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, atomic_load_64, regPlusImm, "ATOMIC_LOAD_I64">; 2385ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_32, or_is_add, "ATOMIC_LOAD_I32">; 2395ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, atomic_load_64, or_is_add, "ATOMIC_LOAD_I64">; 2400b57cec5SDimitry Andric 2410b57cec5SDimitry Andric// Select loads with just a constant offset. 2425ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 2435ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 2440b57cec5SDimitry Andric 2455ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 2465ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 2470b57cec5SDimitry Andric 2480b57cec5SDimitry Andric 2490b57cec5SDimitry Andric// Extending loads. Note that there are only zero-extending atomic loads, no 2500b57cec5SDimitry Andric// sign-extending loads. 2510b57cec5SDimitry Andricdefm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 2520b57cec5SDimitry Andricdefm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 2530b57cec5SDimitry Andricdefm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 2540b57cec5SDimitry Andricdefm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 2550b57cec5SDimitry Andricdefm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 2560b57cec5SDimitry Andric 2570b57cec5SDimitry Andric// Fragments for extending loads. These are different from regular loads because 2580b57cec5SDimitry Andric// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 2590b57cec5SDimitry Andric// therefore don't have the extension type field. So instead of matching that, 2600b57cec5SDimitry Andric// we match the patterns that the type legalizer expands them to. 2610b57cec5SDimitry Andric 2620b57cec5SDimitry Andric// We directly match zext patterns and select the zext atomic loads. 2630b57cec5SDimitry Andric// i32 (zext (i8 (atomic_load_8))) gets legalized to 2640b57cec5SDimitry Andric// i32 (and (i32 (atomic_load_8)), 255) 2650b57cec5SDimitry Andric// These can be selected to a single zero-extending atomic load instruction. 2660b57cec5SDimitry Andricdef zext_aload_8_32 : 2670b57cec5SDimitry Andric PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 2680b57cec5SDimitry Andricdef zext_aload_16_32 : 2690b57cec5SDimitry Andric PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 2700b57cec5SDimitry Andric// Unlike regular loads, extension to i64 is handled differently than i32. 2710b57cec5SDimitry Andric// i64 (zext (i8 (atomic_load_8))) gets legalized to 2720b57cec5SDimitry Andric// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 2730b57cec5SDimitry Andricdef zext_aload_8_64 : 2740b57cec5SDimitry Andric PatFrag<(ops node:$addr), 2750b57cec5SDimitry Andric (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 2760b57cec5SDimitry Andricdef zext_aload_16_64 : 2770b57cec5SDimitry Andric PatFrag<(ops node:$addr), 2780b57cec5SDimitry Andric (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 2790b57cec5SDimitry Andricdef zext_aload_32_64 : 2800b57cec5SDimitry Andric PatFrag<(ops node:$addr), 2810b57cec5SDimitry Andric (zext (i32 (atomic_load node:$addr)))>; 2820b57cec5SDimitry Andric 2830b57cec5SDimitry Andric// We don't have single sext atomic load instructions. So for sext loads, we 2840b57cec5SDimitry Andric// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 2850b57cec5SDimitry Andric// results) and select a zext load; the next instruction will be sext_inreg 2860b57cec5SDimitry Andric// which is selected by itself. 2870b57cec5SDimitry Andricdef sext_aload_8_64 : 2880b57cec5SDimitry Andric PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 2890b57cec5SDimitry Andricdef sext_aload_16_64 : 2900b57cec5SDimitry Andric PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 2910b57cec5SDimitry Andric 2920b57cec5SDimitry Andric// Select zero-extending loads with no constant offset. 2935ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">; 2945ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">; 2955ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 2965ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 2975ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 2980b57cec5SDimitry Andric 2990b57cec5SDimitry Andric// Select sign-extending loads with no constant offset 3005ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 3015ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 3025ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 3035ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 3040b57cec5SDimitry Andric// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 3050b57cec5SDimitry Andric 3060b57cec5SDimitry Andric// Zero-extending loads with constant offset 3075ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, "ATOMIC_LOAD8_U_I32">; 3085ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, "ATOMIC_LOAD16_U_I32">; 3095ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, "ATOMIC_LOAD8_U_I32">; 3105ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, "ATOMIC_LOAD16_U_I32">; 3115ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 3125ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 3135ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, "ATOMIC_LOAD32_U_I64">; 3145ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 3155ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 3165ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, "ATOMIC_LOAD32_U_I64">; 3170b57cec5SDimitry Andric 3180b57cec5SDimitry Andric// Sign-extending loads with constant offset 3195ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_8, regPlusImm, "ATOMIC_LOAD8_U_I32">; 3205ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_16, regPlusImm, "ATOMIC_LOAD16_U_I32">; 3215ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_8, or_is_add, "ATOMIC_LOAD8_U_I32">; 3225ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_16, or_is_add, "ATOMIC_LOAD16_U_I32">; 3235ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 3245ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 3255ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 3265ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 3270b57cec5SDimitry Andric// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 3280b57cec5SDimitry Andric 3290b57cec5SDimitry Andric// Extending loads with just a constant offset 3305ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">; 3315ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">; 3325ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 3335ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 3345ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 3355ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 3365ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 3375ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 3385ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 3390b57cec5SDimitry Andric 3405ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">; 3415ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">; 3425ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 3435ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 3445ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 3455ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 3465ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 3475ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 3485ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 3490b57cec5SDimitry Andric 3500b57cec5SDimitry Andric 3510b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 3520b57cec5SDimitry Andric// Atomic stores 3530b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 3540b57cec5SDimitry Andric 3550b57cec5SDimitry Andricmulticlass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 3565ffd83dbSDimitry Andric defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 3575ffd83dbSDimitry Andric [HasAtomics]>; 3580b57cec5SDimitry Andric} 3590b57cec5SDimitry Andric 3600b57cec5SDimitry Andricdefm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 3610b57cec5SDimitry Andricdefm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 3620b57cec5SDimitry Andric 3630b57cec5SDimitry Andric// We need an 'atomic' version of store patterns because store and atomic_store 3640b57cec5SDimitry Andric// nodes have different operand orders: 3650b57cec5SDimitry Andric// store: (store $val, $ptr) 3660b57cec5SDimitry Andric// atomic_store: (store $ptr, $val) 3670b57cec5SDimitry Andric 3680b57cec5SDimitry Andric 3690b57cec5SDimitry Andric// Select stores with no constant offset. 3705ffd83dbSDimitry Andricmulticlass AStorePatNoOffset<ValueType ty, PatFrag kind, string inst> { 3715ffd83dbSDimitry Andric def : Pat<(kind I32:$addr, ty:$val), 3725ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 373e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 3745ffd83dbSDimitry Andric def : Pat<(kind I64:$addr, ty:$val), 3755ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 376e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 3775ffd83dbSDimitry Andric} 3785ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i32, atomic_store_32, "ATOMIC_STORE_I32">; 3795ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i64, atomic_store_64, "ATOMIC_STORE_I64">; 3800b57cec5SDimitry Andric 3810b57cec5SDimitry Andric// Select stores with a constant offset. 3820b57cec5SDimitry Andric 3830b57cec5SDimitry Andric// Pattern with address + immediate offset 3845ffd83dbSDimitry Andricmulticlass AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 3855ffd83dbSDimitry Andric string inst> { 3865ffd83dbSDimitry Andric def : Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 3875ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 388e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 3895ffd83dbSDimitry Andric def : Pat<(kind (operand I64:$addr, imm:$off), ty:$val), 3905ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 391e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 3925ffd83dbSDimitry Andric} 3935ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_32, regPlusImm, "ATOMIC_STORE_I32">; 3945ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, atomic_store_64, regPlusImm, "ATOMIC_STORE_I64">; 3950b57cec5SDimitry Andric 3960b57cec5SDimitry Andric// Select stores with just a constant offset. 3975ffd83dbSDimitry Andricmulticlass AStorePatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 3985ffd83dbSDimitry Andric def : Pat<(kind imm:$off, ty:$val), 3995ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 400e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 4015ffd83dbSDimitry Andric def : Pat<(kind imm:$off, ty:$val), 4025ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 403e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 4045ffd83dbSDimitry Andric} 4055ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 4065ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 4070b57cec5SDimitry Andric 4085ffd83dbSDimitry Andricmulticlass AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 4095ffd83dbSDimitry Andric def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 4105ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 411e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 4125ffd83dbSDimitry Andric def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 4135ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 414e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 4155ffd83dbSDimitry Andric} 4165ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 4175ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 4180b57cec5SDimitry Andric 4190b57cec5SDimitry Andric 4200b57cec5SDimitry Andric// Truncating stores. 4210b57cec5SDimitry Andricdefm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 4220b57cec5SDimitry Andricdefm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 4230b57cec5SDimitry Andricdefm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 4240b57cec5SDimitry Andricdefm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 4250b57cec5SDimitry Andricdefm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 4260b57cec5SDimitry Andric 4270b57cec5SDimitry Andric// Fragments for truncating stores. 4280b57cec5SDimitry Andric 4290b57cec5SDimitry Andric// We don't have single truncating atomic store instructions. For 32-bit 4300b57cec5SDimitry Andric// instructions, we just need to match bare atomic stores. On the other hand, 4310b57cec5SDimitry Andric// truncating stores from i64 values are once truncated to i32 first. 4320b57cec5SDimitry Andricclass trunc_astore_64<PatFrag kind> : 4330b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 4340b57cec5SDimitry Andric (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 4350b57cec5SDimitry Andricdef trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 4360b57cec5SDimitry Andricdef trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 4370b57cec5SDimitry Andricdef trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 4380b57cec5SDimitry Andric 4390b57cec5SDimitry Andric 4400b57cec5SDimitry Andric// Truncating stores with no constant offset 4415ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 4425ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 4435ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 4445ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 4455ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 4460b57cec5SDimitry Andric 4470b57cec5SDimitry Andric// Truncating stores with a constant offset 4485ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_8, regPlusImm, "ATOMIC_STORE8_I32">; 4495ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_16, regPlusImm, "ATOMIC_STORE16_I32">; 4505ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, "ATOMIC_STORE8_I64">; 4515ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, 4525ffd83dbSDimitry Andric "ATOMIC_STORE16_I64">; 4535ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, 4545ffd83dbSDimitry Andric "ATOMIC_STORE32_I64">; 4555ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_8, or_is_add, "ATOMIC_STORE8_I32">; 4565ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_16, or_is_add, "ATOMIC_STORE16_I32">; 4575ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, "ATOMIC_STORE8_I64">; 4585ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, 4595ffd83dbSDimitry Andric "ATOMIC_STORE16_I64">; 4605ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, 4615ffd83dbSDimitry Andric "ATOMIC_STORE32_I64">; 4620b57cec5SDimitry Andric 4630b57cec5SDimitry Andric// Truncating stores with just a constant offset 4645ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 4655ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 4665ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 4675ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 4685ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 4690b57cec5SDimitry Andric 4705ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 4715ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 4725ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 4735ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 4745ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 4750b57cec5SDimitry Andric 4760b57cec5SDimitry Andric 4770b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 4780b57cec5SDimitry Andric// Atomic binary read-modify-writes 4790b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 4800b57cec5SDimitry Andric 4810b57cec5SDimitry Andricmulticlass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 4820b57cec5SDimitry Andric int atomic_op> { 4835ffd83dbSDimitry Andric defm "_A32" : 4840b57cec5SDimitry Andric ATOMIC_I<(outs rc:$dst), 4850b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 4860b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 4870b57cec5SDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 4885ffd83dbSDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 4895ffd83dbSDimitry Andric defm "_A64" : 4905ffd83dbSDimitry Andric ATOMIC_I<(outs rc:$dst), 4915ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$val), 4925ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 4935ffd83dbSDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 4945ffd83dbSDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 4950b57cec5SDimitry Andric} 4960b57cec5SDimitry Andric 4970b57cec5SDimitry Andricdefm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 4980b57cec5SDimitry Andricdefm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 4990b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_ADD_I32 : 5000b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 5010b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_ADD_I32 : 5020b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 5030b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_ADD_I64 : 5040b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 5050b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_ADD_I64 : 5060b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 5070b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_ADD_I64 : 5080b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 5090b57cec5SDimitry Andric 5100b57cec5SDimitry Andricdefm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 5110b57cec5SDimitry Andricdefm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 5120b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_SUB_I32 : 5130b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 5140b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_SUB_I32 : 5150b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 5160b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_SUB_I64 : 5170b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 5180b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_SUB_I64 : 5190b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 5200b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_SUB_I64 : 5210b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 5220b57cec5SDimitry Andric 5230b57cec5SDimitry Andricdefm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 5240b57cec5SDimitry Andricdefm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 5250b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_AND_I32 : 5260b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 5270b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_AND_I32 : 5280b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 5290b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_AND_I64 : 5300b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 5310b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_AND_I64 : 5320b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 5330b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_AND_I64 : 5340b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 5350b57cec5SDimitry Andric 5360b57cec5SDimitry Andricdefm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 5370b57cec5SDimitry Andricdefm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 5380b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_OR_I32 : 5390b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 5400b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_OR_I32 : 5410b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 5420b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_OR_I64 : 5430b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 5440b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_OR_I64 : 5450b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 5460b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_OR_I64 : 5470b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 5480b57cec5SDimitry Andric 5490b57cec5SDimitry Andricdefm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 5500b57cec5SDimitry Andricdefm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 5510b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XOR_I32 : 5520b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 5530b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XOR_I32 : 5540b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 5550b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XOR_I64 : 5560b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 5570b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XOR_I64 : 5580b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 5590b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_XOR_I64 : 5600b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 5610b57cec5SDimitry Andric 5620b57cec5SDimitry Andricdefm ATOMIC_RMW_XCHG_I32 : 5630b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 5640b57cec5SDimitry Andricdefm ATOMIC_RMW_XCHG_I64 : 5650b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 5660b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XCHG_I32 : 5670b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 5680b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XCHG_I32 : 5690b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 5700b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XCHG_I64 : 5710b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 5720b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XCHG_I64 : 5730b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 5740b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_XCHG_I64 : 5750b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 5760b57cec5SDimitry Andric 5770b57cec5SDimitry Andric// Select binary RMWs with no constant offset. 5785ffd83dbSDimitry Andricmulticlass BinRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 5795ffd83dbSDimitry Andric def : Pat<(ty (kind I32:$addr, ty:$val)), 5805ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 581e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 5825ffd83dbSDimitry Andric def : Pat<(ty (kind I64:$addr, ty:$val)), 5835ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 584e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 5855ffd83dbSDimitry Andric} 5860b57cec5SDimitry Andric 5870b57cec5SDimitry Andric// Select binary RMWs with a constant offset. 5880b57cec5SDimitry Andric 5890b57cec5SDimitry Andric// Pattern with address + immediate offset 5905ffd83dbSDimitry Andricmulticlass BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 5915ffd83dbSDimitry Andric string inst> { 5925ffd83dbSDimitry Andric def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 5935ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 594e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 5955ffd83dbSDimitry Andric def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$val)), 5965ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 597e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 5985ffd83dbSDimitry Andric} 5990b57cec5SDimitry Andric 6000b57cec5SDimitry Andric// Select binary RMWs with just a constant offset. 6015ffd83dbSDimitry Andricmulticlass BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 6025ffd83dbSDimitry Andric def : Pat<(ty (kind imm:$off, ty:$val)), 6035ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 604e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 6055ffd83dbSDimitry Andric def : Pat<(ty (kind imm:$off, ty:$val)), 6065ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 607e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 6085ffd83dbSDimitry Andric} 6090b57cec5SDimitry Andric 6105ffd83dbSDimitry Andricmulticlass BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> { 6115ffd83dbSDimitry Andric def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 6125ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 613e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 6145ffd83dbSDimitry Andric def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 6155ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 616e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 6175ffd83dbSDimitry Andric} 6180b57cec5SDimitry Andric 6190b57cec5SDimitry Andric// Patterns for various addressing modes. 6205ffd83dbSDimitry Andricmulticlass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 6215ffd83dbSDimitry Andric string inst_64> { 6225ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 6235ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 6240b57cec5SDimitry Andric 6255ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 6265ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 6275ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 6285ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 6290b57cec5SDimitry Andric 6305ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 6315ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 6320b57cec5SDimitry Andric 6335ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 6345ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 6350b57cec5SDimitry Andric} 6360b57cec5SDimitry Andric 6375ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, 6385ffd83dbSDimitry Andric "ATOMIC_RMW_ADD_I32", "ATOMIC_RMW_ADD_I64">; 6395ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, 6405ffd83dbSDimitry Andric "ATOMIC_RMW_SUB_I32", "ATOMIC_RMW_SUB_I64">; 6415ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, 6425ffd83dbSDimitry Andric "ATOMIC_RMW_AND_I32", "ATOMIC_RMW_AND_I64">; 6435ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, 6445ffd83dbSDimitry Andric "ATOMIC_RMW_OR_I32", "ATOMIC_RMW_OR_I64">; 6455ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, 6465ffd83dbSDimitry Andric "ATOMIC_RMW_XOR_I32", "ATOMIC_RMW_XOR_I64">; 6475ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_swap_32, atomic_swap_64, 6485ffd83dbSDimitry Andric "ATOMIC_RMW_XCHG_I32", "ATOMIC_RMW_XCHG_I64">; 6490b57cec5SDimitry Andric 6500b57cec5SDimitry Andric// Truncating & zero-extending binary RMW patterns. 6510b57cec5SDimitry Andric// These are combined patterns of truncating store patterns and zero-extending 6520b57cec5SDimitry Andric// load patterns above. 6530b57cec5SDimitry Andricclass zext_bin_rmw_8_32<PatFrag kind> : 6540b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 6550b57cec5SDimitry Andric (and (i32 (kind node:$addr, node:$val)), 255)>; 6560b57cec5SDimitry Andricclass zext_bin_rmw_16_32<PatFrag kind> : 6570b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 6580b57cec5SDimitry Andric (and (i32 (kind node:$addr, node:$val)), 65535)>; 6590b57cec5SDimitry Andricclass zext_bin_rmw_8_64<PatFrag kind> : 6600b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 6610b57cec5SDimitry Andric (and (i64 (anyext (i32 (kind node:$addr, 6620b57cec5SDimitry Andric (i32 (trunc (i64 node:$val))))))), 255)>; 6630b57cec5SDimitry Andricclass zext_bin_rmw_16_64<PatFrag kind> : 6640b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 6650b57cec5SDimitry Andric (and (i64 (anyext (i32 (kind node:$addr, 6660b57cec5SDimitry Andric (i32 (trunc (i64 node:$val))))))), 65535)>; 6670b57cec5SDimitry Andricclass zext_bin_rmw_32_64<PatFrag kind> : 6680b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 6690b57cec5SDimitry Andric (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 6700b57cec5SDimitry Andric 6710b57cec5SDimitry Andric// Truncating & sign-extending binary RMW patterns. 6720b57cec5SDimitry Andric// These are combined patterns of truncating store patterns and sign-extending 6730b57cec5SDimitry Andric// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 6740b57cec5SDimitry Andric// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 6750b57cec5SDimitry Andric// is selected by itself. 6760b57cec5SDimitry Andricclass sext_bin_rmw_8_32<PatFrag kind> : 6770b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 6780b57cec5SDimitry Andricclass sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 6790b57cec5SDimitry Andricclass sext_bin_rmw_8_64<PatFrag kind> : 6800b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 6810b57cec5SDimitry Andric (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 6820b57cec5SDimitry Andricclass sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 6830b57cec5SDimitry Andric// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 6840b57cec5SDimitry Andric 6850b57cec5SDimitry Andric// Patterns for various addressing modes for truncating-extending binary RMWs. 6860b57cec5SDimitry Andricmulticlass BinRMWTruncExtPattern< 6870b57cec5SDimitry Andric PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 6880b57cec5SDimitry Andric NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 6890b57cec5SDimitry Andric // Truncating-extending binary RMWs with no constant offset 6905ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 6915ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 6925ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 6935ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 6945ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 6950b57cec5SDimitry Andric 6965ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 6975ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 6985ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 6995ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 7000b57cec5SDimitry Andric 7010b57cec5SDimitry Andric // Truncating-extending binary RMWs with a constant offset 7025ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 7035ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, 7045ffd83dbSDimitry Andric inst16_32>; 7055ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 7065ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, 7075ffd83dbSDimitry Andric inst16_64>; 7085ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, 7095ffd83dbSDimitry Andric inst32_64>; 7105ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 7115ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 7125ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 7135ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 7145ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 7150b57cec5SDimitry Andric 7165ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 7175ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, 7185ffd83dbSDimitry Andric inst16_32>; 7195ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 7205ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, 7215ffd83dbSDimitry Andric inst16_64>; 7225ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 7235ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 7245ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 7255ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 7260b57cec5SDimitry Andric 7270b57cec5SDimitry Andric // Truncating-extending binary RMWs with just a constant offset 7285ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 7295ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 7305ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 7315ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 7325ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 7330b57cec5SDimitry Andric 7345ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 7355ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 7365ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 7375ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 7380b57cec5SDimitry Andric 7395ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 7405ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 7415ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 7425ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 7435ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 7440b57cec5SDimitry Andric 7455ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 7465ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 7475ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 7485ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 7490b57cec5SDimitry Andric} 7500b57cec5SDimitry Andric 7510b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7520b57cec5SDimitry Andric atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 7535ffd83dbSDimitry Andric "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32", 7545ffd83dbSDimitry Andric "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">; 7550b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7560b57cec5SDimitry Andric atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 7575ffd83dbSDimitry Andric "ATOMIC_RMW8_U_SUB_I32", "ATOMIC_RMW16_U_SUB_I32", 7585ffd83dbSDimitry Andric "ATOMIC_RMW8_U_SUB_I64", "ATOMIC_RMW16_U_SUB_I64", "ATOMIC_RMW32_U_SUB_I64">; 7590b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7600b57cec5SDimitry Andric atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 7615ffd83dbSDimitry Andric "ATOMIC_RMW8_U_AND_I32", "ATOMIC_RMW16_U_AND_I32", 7625ffd83dbSDimitry Andric "ATOMIC_RMW8_U_AND_I64", "ATOMIC_RMW16_U_AND_I64", "ATOMIC_RMW32_U_AND_I64">; 7630b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7640b57cec5SDimitry Andric atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 7655ffd83dbSDimitry Andric "ATOMIC_RMW8_U_OR_I32", "ATOMIC_RMW16_U_OR_I32", 7665ffd83dbSDimitry Andric "ATOMIC_RMW8_U_OR_I64", "ATOMIC_RMW16_U_OR_I64", "ATOMIC_RMW32_U_OR_I64">; 7670b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7680b57cec5SDimitry Andric atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 7695ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XOR_I32", "ATOMIC_RMW16_U_XOR_I32", 7705ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XOR_I64", "ATOMIC_RMW16_U_XOR_I64", "ATOMIC_RMW32_U_XOR_I64">; 7710b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7720b57cec5SDimitry Andric atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 7735ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32", 7745ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64", 7755ffd83dbSDimitry Andric "ATOMIC_RMW32_U_XCHG_I64">; 7760b57cec5SDimitry Andric 7770b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 7780b57cec5SDimitry Andric// Atomic ternary read-modify-writes 7790b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 7800b57cec5SDimitry Andric 7810b57cec5SDimitry Andric// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 7820b57cec5SDimitry Andric// flag}. When we use the success flag or both values, we can't make use of i64 7830b57cec5SDimitry Andric// truncate/extend versions of instructions for now, which is suboptimal. 7840b57cec5SDimitry Andric// Consider adding a pass after instruction selection that optimizes this case 7850b57cec5SDimitry Andric// if it is frequent. 7860b57cec5SDimitry Andric 7870b57cec5SDimitry Andricmulticlass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 7880b57cec5SDimitry Andric int atomic_op> { 7895ffd83dbSDimitry Andric defm "_A32" : 7900b57cec5SDimitry Andric ATOMIC_I<(outs rc:$dst), 7910b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 7920b57cec5SDimitry Andric rc:$new_), 7930b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 7940b57cec5SDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 7955ffd83dbSDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 7965ffd83dbSDimitry Andric defm "_A64" : 7975ffd83dbSDimitry Andric ATOMIC_I<(outs rc:$dst), 7985ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$exp, 7995ffd83dbSDimitry Andric rc:$new_), 8005ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 8015ffd83dbSDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 8025ffd83dbSDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 8030b57cec5SDimitry Andric} 8040b57cec5SDimitry Andric 8050b57cec5SDimitry Andricdefm ATOMIC_RMW_CMPXCHG_I32 : 8060b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 8070b57cec5SDimitry Andricdefm ATOMIC_RMW_CMPXCHG_I64 : 8080b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 8090b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_CMPXCHG_I32 : 8100b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 8110b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_CMPXCHG_I32 : 8120b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 8130b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_CMPXCHG_I64 : 8140b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 8150b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_CMPXCHG_I64 : 8160b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 8170b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_CMPXCHG_I64 : 8180b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 8190b57cec5SDimitry Andric 8200b57cec5SDimitry Andric// Select ternary RMWs with no constant offset. 8215ffd83dbSDimitry Andricmulticlass TerRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 8225ffd83dbSDimitry Andric def : Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 8235ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, ty:$new)>, 824e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 8255ffd83dbSDimitry Andric def : Pat<(ty (kind I64:$addr, ty:$exp, ty:$new)), 8265ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, ty:$new)>, 827e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 8285ffd83dbSDimitry Andric} 8290b57cec5SDimitry Andric 8300b57cec5SDimitry Andric// Select ternary RMWs with a constant offset. 8310b57cec5SDimitry Andric 8320b57cec5SDimitry Andric// Pattern with address + immediate offset 8335ffd83dbSDimitry Andricmulticlass TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 8345ffd83dbSDimitry Andric string inst> { 8355ffd83dbSDimitry Andric def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 8365ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>, 837e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 8385ffd83dbSDimitry Andric def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$exp, ty:$new)), 8395ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, ty:$new)>, 840e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 8415ffd83dbSDimitry Andric} 8420b57cec5SDimitry Andric 8430b57cec5SDimitry Andric// Select ternary RMWs with just a constant offset. 8445ffd83dbSDimitry Andricmulticlass TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 8455ffd83dbSDimitry Andric def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 8465ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 8475ffd83dbSDimitry Andric ty:$new)>; 8485ffd83dbSDimitry Andric def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 8495ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 8505ffd83dbSDimitry Andric ty:$new)>; 8515ffd83dbSDimitry Andric} 8520b57cec5SDimitry Andric 8535ffd83dbSDimitry Andricmulticlass TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 8545ffd83dbSDimitry Andric def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 8555ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 8565ffd83dbSDimitry Andric ty:$new)>, 857e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 8585ffd83dbSDimitry Andric def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 8595ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 8605ffd83dbSDimitry Andric ty:$new)>, 861e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 8625ffd83dbSDimitry Andric} 8630b57cec5SDimitry Andric 8640b57cec5SDimitry Andric// Patterns for various addressing modes. 8655ffd83dbSDimitry Andricmulticlass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 8665ffd83dbSDimitry Andric string inst_64> { 8675ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 8685ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 8690b57cec5SDimitry Andric 8705ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 8715ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 8725ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 8735ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 8740b57cec5SDimitry Andric 8755ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 8765ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 8770b57cec5SDimitry Andric 8785ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 8795ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 8800b57cec5SDimitry Andric} 8810b57cec5SDimitry Andric 8820b57cec5SDimitry Andricdefm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 8835ffd83dbSDimitry Andric "ATOMIC_RMW_CMPXCHG_I32", "ATOMIC_RMW_CMPXCHG_I64">; 8840b57cec5SDimitry Andric 8850b57cec5SDimitry Andric// Truncating & zero-extending ternary RMW patterns. 8860b57cec5SDimitry Andric// DAG legalization & optimization before instruction selection may introduce 8870b57cec5SDimitry Andric// additional nodes such as anyext or assertzext depending on operand types. 8880b57cec5SDimitry Andricclass zext_ter_rmw_8_32<PatFrag kind> : 8890b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 8900b57cec5SDimitry Andric (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 8910b57cec5SDimitry Andricclass zext_ter_rmw_16_32<PatFrag kind> : 8920b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 8930b57cec5SDimitry Andric (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 8940b57cec5SDimitry Andricclass zext_ter_rmw_8_64<PatFrag kind> : 8950b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 8960b57cec5SDimitry Andric (zext (i32 (assertzext (i32 (kind node:$addr, 8970b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 8980b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))))>; 8990b57cec5SDimitry Andricclass zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 9000b57cec5SDimitry Andricclass zext_ter_rmw_32_64<PatFrag kind> : 9010b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 9020b57cec5SDimitry Andric (zext (i32 (kind node:$addr, 9030b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 9040b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))>; 9050b57cec5SDimitry Andric 9060b57cec5SDimitry Andric// Truncating & sign-extending ternary RMW patterns. 9070b57cec5SDimitry Andric// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 9080b57cec5SDimitry Andric// zext RMW; the next instruction will be sext_inreg which is selected by 9090b57cec5SDimitry Andric// itself. 9100b57cec5SDimitry Andricclass sext_ter_rmw_8_32<PatFrag kind> : 9110b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 9120b57cec5SDimitry Andric (kind node:$addr, node:$exp, node:$new)>; 9130b57cec5SDimitry Andricclass sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 9140b57cec5SDimitry Andricclass sext_ter_rmw_8_64<PatFrag kind> : 9150b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 9160b57cec5SDimitry Andric (anyext (i32 (assertzext (i32 9170b57cec5SDimitry Andric (kind node:$addr, 9180b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 9190b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))))>; 9200b57cec5SDimitry Andricclass sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 9210b57cec5SDimitry Andric// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 9220b57cec5SDimitry Andric 9230b57cec5SDimitry Andric// Patterns for various addressing modes for truncating-extending ternary RMWs. 9240b57cec5SDimitry Andricmulticlass TerRMWTruncExtPattern< 9250b57cec5SDimitry Andric PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 9265ffd83dbSDimitry Andric string inst8_32, string inst16_32, string inst8_64, string inst16_64, 9275ffd83dbSDimitry Andric string inst32_64> { 9280b57cec5SDimitry Andric // Truncating-extending ternary RMWs with no constant offset 9295ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 9305ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 9315ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 9325ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 9335ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 9340b57cec5SDimitry Andric 9355ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 9365ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 9375ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 9385ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 9390b57cec5SDimitry Andric 9400b57cec5SDimitry Andric // Truncating-extending ternary RMWs with a constant offset 9415ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 9425ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, 9435ffd83dbSDimitry Andric inst16_32>; 9445ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 9455ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, 9465ffd83dbSDimitry Andric inst16_64>; 9475ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, 9485ffd83dbSDimitry Andric inst32_64>; 9495ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 9505ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 9515ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 9525ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 9535ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 9540b57cec5SDimitry Andric 9555ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 9565ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, 9575ffd83dbSDimitry Andric inst16_32>; 9585ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 9595ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, 9605ffd83dbSDimitry Andric inst16_64>; 9615ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 9625ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 9635ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 9645ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 9650b57cec5SDimitry Andric 9660b57cec5SDimitry Andric // Truncating-extending ternary RMWs with just a constant offset 9675ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 9685ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 9695ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 9705ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 9715ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 9720b57cec5SDimitry Andric 9735ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 9745ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 9755ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 9765ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 9770b57cec5SDimitry Andric 9785ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 9795ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 9805ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 9815ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 9825ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 9830b57cec5SDimitry Andric 9845ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 9855ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 9865ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 9875ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 9880b57cec5SDimitry Andric} 9890b57cec5SDimitry Andric 9900b57cec5SDimitry Andricdefm : TerRMWTruncExtPattern< 9910b57cec5SDimitry Andric atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 9925ffd83dbSDimitry Andric "ATOMIC_RMW8_U_CMPXCHG_I32", "ATOMIC_RMW16_U_CMPXCHG_I32", 9935ffd83dbSDimitry Andric "ATOMIC_RMW8_U_CMPXCHG_I64", "ATOMIC_RMW16_U_CMPXCHG_I64", 9945ffd83dbSDimitry Andric "ATOMIC_RMW32_U_CMPXCHG_I64">; 995