10b57cec5SDimitry Andric// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 20b57cec5SDimitry Andric// 30b57cec5SDimitry Andric// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric// See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric// 70b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric/// 90b57cec5SDimitry Andric/// \file 100b57cec5SDimitry Andric/// WebAssembly Atomic operand code-gen constructs. 110b57cec5SDimitry Andric/// 120b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 130b57cec5SDimitry Andric 140b57cec5SDimitry Andriclet UseNamedOperandTable = 1 in 150b57cec5SDimitry Andricmulticlass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 165ffd83dbSDimitry Andric list<dag> pattern_r, string asmstr_r, 175ffd83dbSDimitry Andric string asmstr_s, bits<32> atomic_op, 185ffd83dbSDimitry Andric string is64 = "false"> { 190b57cec5SDimitry Andric defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 205ffd83dbSDimitry Andric !or(0xfe00, !and(0xff, atomic_op)), is64>, 210b57cec5SDimitry Andric Requires<[HasAtomics]>; 220b57cec5SDimitry Andric} 230b57cec5SDimitry Andric 240b57cec5SDimitry Andricmulticlass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 250b57cec5SDimitry Andric bits<32> atomic_op = -1> { 260b57cec5SDimitry Andric defm "" : NRI<oops, iops, pattern, asmstr, 270b57cec5SDimitry Andric !or(0xfe00, !and(0xff, atomic_op))>, 280b57cec5SDimitry Andric Requires<[HasAtomics]>; 290b57cec5SDimitry Andric} 300b57cec5SDimitry Andric 310b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 320b57cec5SDimitry Andric// Atomic wait / notify 330b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 340b57cec5SDimitry Andric 350b57cec5SDimitry Andriclet hasSideEffects = 1 in { 36e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_NOTIFY_A32 : 370b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 380b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 390b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 40e8d8bef9SDimitry Andric "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 41e8d8bef9SDimitry Andric "memory.atomic.notify \t${off}${p2align}", 0x00, "false">; 42e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_NOTIFY_A64 : 435ffd83dbSDimitry Andric ATOMIC_I<(outs I32:$dst), 445ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$count), 455ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 46e8d8bef9SDimitry Andric "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 47e8d8bef9SDimitry Andric "memory.atomic.notify \t${off}${p2align}", 0x00, "true">; 480b57cec5SDimitry Andriclet mayLoad = 1 in { 49e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT32_A32 : 500b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 510b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 520b57cec5SDimitry Andric I64:$timeout), 530b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54e8d8bef9SDimitry Andric "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 55e8d8bef9SDimitry Andric "memory.atomic.wait32 \t${off}${p2align}", 0x01, "false">; 56e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT32_A64 : 575ffd83dbSDimitry Andric ATOMIC_I<(outs I32:$dst), 585ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$exp, 595ffd83dbSDimitry Andric I64:$timeout), 605ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 61e8d8bef9SDimitry Andric "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 62e8d8bef9SDimitry Andric "memory.atomic.wait32 \t${off}${p2align}", 0x01, "true">; 63e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT64_A32 : 640b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 650b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 660b57cec5SDimitry Andric I64:$timeout), 670b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 68e8d8bef9SDimitry Andric "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 69e8d8bef9SDimitry Andric "memory.atomic.wait64 \t${off}${p2align}", 0x02, "false">; 70e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT64_A64 : 715ffd83dbSDimitry Andric ATOMIC_I<(outs I32:$dst), 725ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp, 735ffd83dbSDimitry Andric I64:$timeout), 745ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 75e8d8bef9SDimitry Andric "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 76e8d8bef9SDimitry Andric "memory.atomic.wait64 \t${off}${p2align}", 0x02, "true">; 770b57cec5SDimitry Andric} // mayLoad = 1 780b57cec5SDimitry Andric} // hasSideEffects = 1 790b57cec5SDimitry Andric 800b57cec5SDimitry Andric// Select notifys with no constant offset. 815ffd83dbSDimitry Andricdef NotifyPatNoOffset_A32 : 82e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify I32:$addr, I32:$count)), 83e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A32 0, 0, I32:$addr, I32:$count)>, 84e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 855ffd83dbSDimitry Andricdef NotifyPatNoOffset_A64 : 86e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify I64:$addr, I32:$count)), 87e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A64 0, 0, I64:$addr, I32:$count)>, 88e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 890b57cec5SDimitry Andric 900b57cec5SDimitry Andric// Select notifys with a constant offset. 910b57cec5SDimitry Andric 920b57cec5SDimitry Andric// Pattern with address + immediate offset 935ffd83dbSDimitry Andricmulticlass NotifyPatImmOff<PatFrag operand, string inst> { 94e8d8bef9SDimitry Andric def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I32:$addr, imm:$off), 955ffd83dbSDimitry Andric I32:$count)), 965ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, I32:$count)>, 97e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 98e8d8bef9SDimitry Andric def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I64:$addr, imm:$off), 995ffd83dbSDimitry Andric I32:$count)), 1005ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, I32:$count)>, 101e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1025ffd83dbSDimitry Andric} 103e8d8bef9SDimitry Andricdefm : NotifyPatImmOff<regPlusImm, "MEMORY_ATOMIC_NOTIFY">; 104e8d8bef9SDimitry Andricdefm : NotifyPatImmOff<or_is_add, "MEMORY_ATOMIC_NOTIFY">; 1050b57cec5SDimitry Andric 1060b57cec5SDimitry Andric// Select notifys with just a constant offset. 1075ffd83dbSDimitry Andricdef NotifyPatOffsetOnly_A32 : 108e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)), 109e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A32 0, imm:$off, (CONST_I32 0), I32:$count)>, 110e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 1115ffd83dbSDimitry Andricdef NotifyPatOffsetOnly_A64 : 112e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)), 113e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A64 0, imm:$off, (CONST_I64 0), I32:$count)>, 114e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1150b57cec5SDimitry Andric 1165ffd83dbSDimitry Andricdef NotifyPatGlobalAddrOffOnly_A32 : 117e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 1180b57cec5SDimitry Andric I32:$count)), 119e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A32 0, tglobaladdr:$off, (CONST_I32 0), I32:$count) 120e8d8bef9SDimitry Andric >, 121e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 1225ffd83dbSDimitry Andricdef NotifyPatGlobalAddrOffOnly_A64 : 123e8d8bef9SDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 1245ffd83dbSDimitry Andric I32:$count)), 125e8d8bef9SDimitry Andric (MEMORY_ATOMIC_NOTIFY_A64 0, tglobaladdr:$off, (CONST_I64 0), I32:$count) 126e8d8bef9SDimitry Andric >, 127e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 1280b57cec5SDimitry Andric 1290b57cec5SDimitry Andric// Select waits with no constant offset. 1305ffd83dbSDimitry Andricmulticlass WaitPatNoOffset<ValueType ty, Intrinsic kind, 1315ffd83dbSDimitry Andric string inst> { 1325ffd83dbSDimitry Andric def : Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 1335ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, I64:$timeout)>, 134e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 1355ffd83dbSDimitry Andric def : Pat<(i32 (kind I64:$addr, ty:$exp, I64:$timeout)), 1365ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, I64:$timeout)>, 137e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1385ffd83dbSDimitry Andric} 139e8d8bef9SDimitry Andricdefm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32, 140e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 141e8d8bef9SDimitry Andricdefm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64, 142e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 143e8d8bef9SDimitry Andricdefm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32, 144e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 145e8d8bef9SDimitry Andricdefm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64, 146e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 1470b57cec5SDimitry Andric 1480b57cec5SDimitry Andric// Select waits with a constant offset. 1490b57cec5SDimitry Andric 1500b57cec5SDimitry Andric// Pattern with address + immediate offset 1515ffd83dbSDimitry Andricmulticlass WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, 1525ffd83dbSDimitry Andric string inst> { 1535ffd83dbSDimitry Andric def : Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 1545ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, 1555ffd83dbSDimitry Andric I64:$timeout)>, 156e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 1575ffd83dbSDimitry Andric def : Pat<(i32 (kind (operand I64:$addr, imm:$off), ty:$exp, I64:$timeout)), 1585ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, 1595ffd83dbSDimitry Andric I64:$timeout)>, 160e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1615ffd83dbSDimitry Andric} 162e8d8bef9SDimitry Andricdefm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, regPlusImm, 163e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 164e8d8bef9SDimitry Andricdefm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, or_is_add, 165e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 166e8d8bef9SDimitry Andricdefm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, regPlusImm, 167e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 168e8d8bef9SDimitry Andricdefm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, or_is_add, 169e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 1700b57cec5SDimitry Andric 171e8d8bef9SDimitry Andric// Select waits with just a constant offset. 1725ffd83dbSDimitry Andricmulticlass WaitPatOffsetOnly<ValueType ty, Intrinsic kind, string inst> { 1735ffd83dbSDimitry Andric def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 1745ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 1755ffd83dbSDimitry Andric I64:$timeout)>, 176e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 1775ffd83dbSDimitry Andric def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 1785ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 1795ffd83dbSDimitry Andric I64:$timeout)>, 180e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 1815ffd83dbSDimitry Andric} 182e8d8bef9SDimitry Andricdefm : WaitPatOffsetOnly<i32, int_wasm_memory_atomic_wait32, 183e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 184e8d8bef9SDimitry Andricdefm : WaitPatOffsetOnly<i64, int_wasm_memory_atomic_wait64, 185e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 1860b57cec5SDimitry Andric 1875ffd83dbSDimitry Andricmulticlass WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, string inst> { 1885ffd83dbSDimitry Andric def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 1895ffd83dbSDimitry Andric I64:$timeout)), 1905ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 1915ffd83dbSDimitry Andric I64:$timeout)>, 192e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 1935ffd83dbSDimitry Andric def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 1945ffd83dbSDimitry Andric I64:$timeout)), 1955ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 1965ffd83dbSDimitry Andric I64:$timeout)>, 197e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 1985ffd83dbSDimitry Andric} 199e8d8bef9SDimitry Andricdefm : WaitPatGlobalAddrOffOnly<i32, int_wasm_memory_atomic_wait32, 200e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT32">; 201e8d8bef9SDimitry Andricdefm : WaitPatGlobalAddrOffOnly<i64, int_wasm_memory_atomic_wait64, 202e8d8bef9SDimitry Andric "MEMORY_ATOMIC_WAIT64">; 2030b57cec5SDimitry Andric 2040b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2058bcb0991SDimitry Andric// Atomic fences 2068bcb0991SDimitry Andric//===----------------------------------------------------------------------===// 2078bcb0991SDimitry Andric 2088bcb0991SDimitry Andric// A compiler fence instruction that prevents reordering of instructions. 2098bcb0991SDimitry Andriclet Defs = [ARGUMENTS] in { 2108bcb0991SDimitry Andriclet isPseudo = 1, hasSideEffects = 1 in 2118bcb0991SDimitry Andricdefm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 2128bcb0991SDimitry Andriclet hasSideEffects = 1 in 2138bcb0991SDimitry Andricdefm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", 2148bcb0991SDimitry Andric 0x03>; 2158bcb0991SDimitry Andric} // Defs = [ARGUMENTS] 2168bcb0991SDimitry Andric 2178bcb0991SDimitry Andric//===----------------------------------------------------------------------===// 2180b57cec5SDimitry Andric// Atomic loads 2190b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2200b57cec5SDimitry Andric 2210b57cec5SDimitry Andricmulticlass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 2225ffd83dbSDimitry Andric defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 2235ffd83dbSDimitry Andric [HasAtomics]>; 2240b57cec5SDimitry Andric} 2250b57cec5SDimitry Andric 2260b57cec5SDimitry Andricdefm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 2270b57cec5SDimitry Andricdefm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 2280b57cec5SDimitry Andric 2290b57cec5SDimitry Andric// Select loads with no constant offset. 2305ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 2315ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 2320b57cec5SDimitry Andric 2330b57cec5SDimitry Andric// Select loads with a constant offset. 2340b57cec5SDimitry Andric 2350b57cec5SDimitry Andric// Pattern with address + immediate offset 2365ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_32, regPlusImm, "ATOMIC_LOAD_I32">; 2375ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, atomic_load_64, regPlusImm, "ATOMIC_LOAD_I64">; 2385ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_32, or_is_add, "ATOMIC_LOAD_I32">; 2395ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, atomic_load_64, or_is_add, "ATOMIC_LOAD_I64">; 2400b57cec5SDimitry Andric 2410b57cec5SDimitry Andric// Select loads with just a constant offset. 2425ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 2435ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 2440b57cec5SDimitry Andric 2455ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 2465ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 2470b57cec5SDimitry Andric 2480b57cec5SDimitry Andric 2490b57cec5SDimitry Andric// Extending loads. Note that there are only zero-extending atomic loads, no 2500b57cec5SDimitry Andric// sign-extending loads. 2510b57cec5SDimitry Andricdefm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 2520b57cec5SDimitry Andricdefm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 2530b57cec5SDimitry Andricdefm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 2540b57cec5SDimitry Andricdefm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 2550b57cec5SDimitry Andricdefm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 2560b57cec5SDimitry Andric 2570b57cec5SDimitry Andric// Fragments for extending loads. These are different from regular loads because 2580b57cec5SDimitry Andric// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 2590b57cec5SDimitry Andric// therefore don't have the extension type field. So instead of matching that, 2600b57cec5SDimitry Andric// we match the patterns that the type legalizer expands them to. 2610b57cec5SDimitry Andric 2620b57cec5SDimitry Andric// Unlike regular loads, extension to i64 is handled differently than i32. 2630b57cec5SDimitry Andric// i64 (zext (i8 (atomic_load_8))) gets legalized to 2640b57cec5SDimitry Andric// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 265fe6060f1SDimitry Andric// Extension to i32 is elided by SelectionDAG as our atomic loads are 266fe6060f1SDimitry Andric// zero-extending. 2670b57cec5SDimitry Andricdef zext_aload_8_64 : 2680b57cec5SDimitry Andric PatFrag<(ops node:$addr), 269fe6060f1SDimitry Andric (i64 (zext (i32 (atomic_load_8 node:$addr))))>; 2700b57cec5SDimitry Andricdef zext_aload_16_64 : 2710b57cec5SDimitry Andric PatFrag<(ops node:$addr), 272fe6060f1SDimitry Andric (i64 (zext (i32 (atomic_load_16 node:$addr))))>; 2730b57cec5SDimitry Andricdef zext_aload_32_64 : 2740b57cec5SDimitry Andric PatFrag<(ops node:$addr), 275fe6060f1SDimitry Andric (i64 (zext (i32 (atomic_load_32 node:$addr))))>; 2760b57cec5SDimitry Andric 2770b57cec5SDimitry Andric// We don't have single sext atomic load instructions. So for sext loads, we 2780b57cec5SDimitry Andric// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 2790b57cec5SDimitry Andric// results) and select a zext load; the next instruction will be sext_inreg 2800b57cec5SDimitry Andric// which is selected by itself. 2810b57cec5SDimitry Andricdef sext_aload_8_64 : 2820b57cec5SDimitry Andric PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 2830b57cec5SDimitry Andricdef sext_aload_16_64 : 2840b57cec5SDimitry Andric PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 2850b57cec5SDimitry Andric 2860b57cec5SDimitry Andric// Select zero-extending loads with no constant offset. 2875ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 2885ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 2895ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 2900b57cec5SDimitry Andric 2910b57cec5SDimitry Andric// Select sign-extending loads with no constant offset 2925ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 2935ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 2945ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 2955ffd83dbSDimitry Andricdefm : LoadPatNoOffset<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 2960b57cec5SDimitry Andric// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 2970b57cec5SDimitry Andric 2980b57cec5SDimitry Andric// Zero-extending loads with constant offset 2995ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 3005ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 3015ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, "ATOMIC_LOAD32_U_I64">; 3025ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 3035ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 3045ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, "ATOMIC_LOAD32_U_I64">; 3050b57cec5SDimitry Andric 3060b57cec5SDimitry Andric// Sign-extending loads with constant offset 3075ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_8, regPlusImm, "ATOMIC_LOAD8_U_I32">; 3085ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_16, regPlusImm, "ATOMIC_LOAD16_U_I32">; 3095ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_8, or_is_add, "ATOMIC_LOAD8_U_I32">; 3105ffd83dbSDimitry Andricdefm : LoadPatImmOff<i32, atomic_load_16, or_is_add, "ATOMIC_LOAD16_U_I32">; 3115ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 3125ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 3135ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 3145ffd83dbSDimitry Andricdefm : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 3150b57cec5SDimitry Andric// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 3160b57cec5SDimitry Andric 3170b57cec5SDimitry Andric// Extending loads with just a constant offset 3185ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 3195ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 3205ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 3215ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 3225ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 3235ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 3245ffd83dbSDimitry Andricdefm : LoadPatOffsetOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 3250b57cec5SDimitry Andric 3265ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 3275ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 3285ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 3295ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 3305ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 3315ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 3325ffd83dbSDimitry Andricdefm : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 3330b57cec5SDimitry Andric 3340b57cec5SDimitry Andric 3350b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 3360b57cec5SDimitry Andric// Atomic stores 3370b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 3380b57cec5SDimitry Andric 3390b57cec5SDimitry Andricmulticlass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 3405ffd83dbSDimitry Andric defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 3415ffd83dbSDimitry Andric [HasAtomics]>; 3420b57cec5SDimitry Andric} 3430b57cec5SDimitry Andric 3440b57cec5SDimitry Andricdefm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 3450b57cec5SDimitry Andricdefm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 3460b57cec5SDimitry Andric 3470b57cec5SDimitry Andric// We need an 'atomic' version of store patterns because store and atomic_store 3480b57cec5SDimitry Andric// nodes have different operand orders: 3490b57cec5SDimitry Andric// store: (store $val, $ptr) 3500b57cec5SDimitry Andric// atomic_store: (store $ptr, $val) 3510b57cec5SDimitry Andric 3520b57cec5SDimitry Andric 3530b57cec5SDimitry Andric// Select stores with no constant offset. 3545ffd83dbSDimitry Andricmulticlass AStorePatNoOffset<ValueType ty, PatFrag kind, string inst> { 3555ffd83dbSDimitry Andric def : Pat<(kind I32:$addr, ty:$val), 3565ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 357e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 3585ffd83dbSDimitry Andric def : Pat<(kind I64:$addr, ty:$val), 3595ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 360e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 3615ffd83dbSDimitry Andric} 3625ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i32, atomic_store_32, "ATOMIC_STORE_I32">; 3635ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i64, atomic_store_64, "ATOMIC_STORE_I64">; 3640b57cec5SDimitry Andric 3650b57cec5SDimitry Andric// Select stores with a constant offset. 3660b57cec5SDimitry Andric 3670b57cec5SDimitry Andric// Pattern with address + immediate offset 3685ffd83dbSDimitry Andricmulticlass AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 3695ffd83dbSDimitry Andric string inst> { 3705ffd83dbSDimitry Andric def : Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 3715ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 372e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 3735ffd83dbSDimitry Andric def : Pat<(kind (operand I64:$addr, imm:$off), ty:$val), 3745ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 375e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 3765ffd83dbSDimitry Andric} 3775ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_32, regPlusImm, "ATOMIC_STORE_I32">; 3785ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, atomic_store_64, regPlusImm, "ATOMIC_STORE_I64">; 3790b57cec5SDimitry Andric 3800b57cec5SDimitry Andric// Select stores with just a constant offset. 3815ffd83dbSDimitry Andricmulticlass AStorePatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 3825ffd83dbSDimitry Andric def : Pat<(kind imm:$off, ty:$val), 3835ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 384e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 3855ffd83dbSDimitry Andric def : Pat<(kind imm:$off, ty:$val), 3865ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 387e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 3885ffd83dbSDimitry Andric} 3895ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 3905ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 3910b57cec5SDimitry Andric 3925ffd83dbSDimitry Andricmulticlass AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 3935ffd83dbSDimitry Andric def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 3945ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 395e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 3965ffd83dbSDimitry Andric def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 3975ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 398e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 3995ffd83dbSDimitry Andric} 4005ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 4015ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 4020b57cec5SDimitry Andric 4030b57cec5SDimitry Andric 4040b57cec5SDimitry Andric// Truncating stores. 4050b57cec5SDimitry Andricdefm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 4060b57cec5SDimitry Andricdefm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 4070b57cec5SDimitry Andricdefm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 4080b57cec5SDimitry Andricdefm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 4090b57cec5SDimitry Andricdefm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 4100b57cec5SDimitry Andric 4110b57cec5SDimitry Andric// Fragments for truncating stores. 4120b57cec5SDimitry Andric 4130b57cec5SDimitry Andric// We don't have single truncating atomic store instructions. For 32-bit 4140b57cec5SDimitry Andric// instructions, we just need to match bare atomic stores. On the other hand, 4150b57cec5SDimitry Andric// truncating stores from i64 values are once truncated to i32 first. 4160b57cec5SDimitry Andricclass trunc_astore_64<PatFrag kind> : 4170b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 4180b57cec5SDimitry Andric (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 4190b57cec5SDimitry Andricdef trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 4200b57cec5SDimitry Andricdef trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 4210b57cec5SDimitry Andricdef trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 4220b57cec5SDimitry Andric 4230b57cec5SDimitry Andric 4240b57cec5SDimitry Andric// Truncating stores with no constant offset 4255ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 4265ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 4275ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 4285ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 4295ffd83dbSDimitry Andricdefm : AStorePatNoOffset<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 4300b57cec5SDimitry Andric 4310b57cec5SDimitry Andric// Truncating stores with a constant offset 4325ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_8, regPlusImm, "ATOMIC_STORE8_I32">; 4335ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_16, regPlusImm, "ATOMIC_STORE16_I32">; 4345ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, "ATOMIC_STORE8_I64">; 4355ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, 4365ffd83dbSDimitry Andric "ATOMIC_STORE16_I64">; 4375ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, 4385ffd83dbSDimitry Andric "ATOMIC_STORE32_I64">; 4395ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_8, or_is_add, "ATOMIC_STORE8_I32">; 4405ffd83dbSDimitry Andricdefm : AStorePatImmOff<i32, atomic_store_16, or_is_add, "ATOMIC_STORE16_I32">; 4415ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, "ATOMIC_STORE8_I64">; 4425ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, 4435ffd83dbSDimitry Andric "ATOMIC_STORE16_I64">; 4445ffd83dbSDimitry Andricdefm : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, 4455ffd83dbSDimitry Andric "ATOMIC_STORE32_I64">; 4460b57cec5SDimitry Andric 4470b57cec5SDimitry Andric// Truncating stores with just a constant offset 4485ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 4495ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 4505ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 4515ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 4525ffd83dbSDimitry Andricdefm : AStorePatOffsetOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 4530b57cec5SDimitry Andric 4545ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 4555ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 4565ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 4575ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 4585ffd83dbSDimitry Andricdefm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 4590b57cec5SDimitry Andric 4600b57cec5SDimitry Andric 4610b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 4620b57cec5SDimitry Andric// Atomic binary read-modify-writes 4630b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 4640b57cec5SDimitry Andric 4650b57cec5SDimitry Andricmulticlass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 4660b57cec5SDimitry Andric int atomic_op> { 4675ffd83dbSDimitry Andric defm "_A32" : 4680b57cec5SDimitry Andric ATOMIC_I<(outs rc:$dst), 4690b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 4700b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 4710b57cec5SDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 4725ffd83dbSDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 4735ffd83dbSDimitry Andric defm "_A64" : 4745ffd83dbSDimitry Andric ATOMIC_I<(outs rc:$dst), 4755ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$val), 4765ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 4775ffd83dbSDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 4785ffd83dbSDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 4790b57cec5SDimitry Andric} 4800b57cec5SDimitry Andric 4810b57cec5SDimitry Andricdefm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 4820b57cec5SDimitry Andricdefm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 4830b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_ADD_I32 : 4840b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 4850b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_ADD_I32 : 4860b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 4870b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_ADD_I64 : 4880b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 4890b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_ADD_I64 : 4900b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 4910b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_ADD_I64 : 4920b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 4930b57cec5SDimitry Andric 4940b57cec5SDimitry Andricdefm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 4950b57cec5SDimitry Andricdefm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 4960b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_SUB_I32 : 4970b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 4980b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_SUB_I32 : 4990b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 5000b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_SUB_I64 : 5010b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 5020b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_SUB_I64 : 5030b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 5040b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_SUB_I64 : 5050b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 5060b57cec5SDimitry Andric 5070b57cec5SDimitry Andricdefm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 5080b57cec5SDimitry Andricdefm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 5090b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_AND_I32 : 5100b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 5110b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_AND_I32 : 5120b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 5130b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_AND_I64 : 5140b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 5150b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_AND_I64 : 5160b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 5170b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_AND_I64 : 5180b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 5190b57cec5SDimitry Andric 5200b57cec5SDimitry Andricdefm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 5210b57cec5SDimitry Andricdefm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 5220b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_OR_I32 : 5230b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 5240b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_OR_I32 : 5250b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 5260b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_OR_I64 : 5270b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 5280b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_OR_I64 : 5290b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 5300b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_OR_I64 : 5310b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 5320b57cec5SDimitry Andric 5330b57cec5SDimitry Andricdefm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 5340b57cec5SDimitry Andricdefm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 5350b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XOR_I32 : 5360b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 5370b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XOR_I32 : 5380b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 5390b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XOR_I64 : 5400b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 5410b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XOR_I64 : 5420b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 5430b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_XOR_I64 : 5440b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 5450b57cec5SDimitry Andric 5460b57cec5SDimitry Andricdefm ATOMIC_RMW_XCHG_I32 : 5470b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 5480b57cec5SDimitry Andricdefm ATOMIC_RMW_XCHG_I64 : 5490b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 5500b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XCHG_I32 : 5510b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 5520b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XCHG_I32 : 5530b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 5540b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XCHG_I64 : 5550b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 5560b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XCHG_I64 : 5570b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 5580b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_XCHG_I64 : 5590b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 5600b57cec5SDimitry Andric 5610b57cec5SDimitry Andric// Select binary RMWs with no constant offset. 5625ffd83dbSDimitry Andricmulticlass BinRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 5635ffd83dbSDimitry Andric def : Pat<(ty (kind I32:$addr, ty:$val)), 5645ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 565e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 5665ffd83dbSDimitry Andric def : Pat<(ty (kind I64:$addr, ty:$val)), 5675ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 568e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 5695ffd83dbSDimitry Andric} 5700b57cec5SDimitry Andric 5710b57cec5SDimitry Andric// Select binary RMWs with a constant offset. 5720b57cec5SDimitry Andric 5730b57cec5SDimitry Andric// Pattern with address + immediate offset 5745ffd83dbSDimitry Andricmulticlass BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 5755ffd83dbSDimitry Andric string inst> { 5765ffd83dbSDimitry Andric def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 5775ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 578e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 5795ffd83dbSDimitry Andric def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$val)), 5805ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 581e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 5825ffd83dbSDimitry Andric} 5830b57cec5SDimitry Andric 5840b57cec5SDimitry Andric// Select binary RMWs with just a constant offset. 5855ffd83dbSDimitry Andricmulticlass BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 5865ffd83dbSDimitry Andric def : Pat<(ty (kind imm:$off, ty:$val)), 5875ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 588e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 5895ffd83dbSDimitry Andric def : Pat<(ty (kind imm:$off, ty:$val)), 5905ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 591e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 5925ffd83dbSDimitry Andric} 5930b57cec5SDimitry Andric 594fe6060f1SDimitry Andricmulticlass BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 5955ffd83dbSDimitry Andric def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 5965ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 597e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 5985ffd83dbSDimitry Andric def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 5995ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 600e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 6015ffd83dbSDimitry Andric} 6020b57cec5SDimitry Andric 6030b57cec5SDimitry Andric// Patterns for various addressing modes. 6045ffd83dbSDimitry Andricmulticlass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 6055ffd83dbSDimitry Andric string inst_64> { 6065ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 6075ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 6080b57cec5SDimitry Andric 6095ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 6105ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 6115ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 6125ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 6130b57cec5SDimitry Andric 6145ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 6155ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 6160b57cec5SDimitry Andric 6175ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 6185ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 6190b57cec5SDimitry Andric} 6200b57cec5SDimitry Andric 6215ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, 6225ffd83dbSDimitry Andric "ATOMIC_RMW_ADD_I32", "ATOMIC_RMW_ADD_I64">; 6235ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, 6245ffd83dbSDimitry Andric "ATOMIC_RMW_SUB_I32", "ATOMIC_RMW_SUB_I64">; 6255ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, 6265ffd83dbSDimitry Andric "ATOMIC_RMW_AND_I32", "ATOMIC_RMW_AND_I64">; 6275ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, 6285ffd83dbSDimitry Andric "ATOMIC_RMW_OR_I32", "ATOMIC_RMW_OR_I64">; 6295ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, 6305ffd83dbSDimitry Andric "ATOMIC_RMW_XOR_I32", "ATOMIC_RMW_XOR_I64">; 6315ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_swap_32, atomic_swap_64, 6325ffd83dbSDimitry Andric "ATOMIC_RMW_XCHG_I32", "ATOMIC_RMW_XCHG_I64">; 6330b57cec5SDimitry Andric 6340b57cec5SDimitry Andric// Truncating & zero-extending binary RMW patterns. 6350b57cec5SDimitry Andric// These are combined patterns of truncating store patterns and zero-extending 6360b57cec5SDimitry Andric// load patterns above. 6370b57cec5SDimitry Andricclass zext_bin_rmw_8_32<PatFrag kind> : 638fe6060f1SDimitry Andric PatFrag<(ops node:$addr, node:$val), (i32 (kind node:$addr, node:$val))>; 639fe6060f1SDimitry Andricclass zext_bin_rmw_16_32<PatFrag kind> : zext_bin_rmw_8_32<kind>; 6400b57cec5SDimitry Andricclass zext_bin_rmw_8_64<PatFrag kind> : 6410b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 6420b57cec5SDimitry Andric (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 643fe6060f1SDimitry Andricclass zext_bin_rmw_16_64<PatFrag kind> : zext_bin_rmw_8_64<kind>; 644fe6060f1SDimitry Andricclass zext_bin_rmw_32_64<PatFrag kind> : zext_bin_rmw_8_64<kind>; 6450b57cec5SDimitry Andric 6460b57cec5SDimitry Andric// Truncating & sign-extending binary RMW patterns. 6470b57cec5SDimitry Andric// These are combined patterns of truncating store patterns and sign-extending 6480b57cec5SDimitry Andric// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 6490b57cec5SDimitry Andric// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 6500b57cec5SDimitry Andric// is selected by itself. 6510b57cec5SDimitry Andricclass sext_bin_rmw_8_32<PatFrag kind> : 6520b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 6530b57cec5SDimitry Andricclass sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 6540b57cec5SDimitry Andricclass sext_bin_rmw_8_64<PatFrag kind> : 6550b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 6560b57cec5SDimitry Andric (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 6570b57cec5SDimitry Andricclass sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 6580b57cec5SDimitry Andric// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 6590b57cec5SDimitry Andric 6600b57cec5SDimitry Andric// Patterns for various addressing modes for truncating-extending binary RMWs. 6610b57cec5SDimitry Andricmulticlass BinRMWTruncExtPattern< 6620b57cec5SDimitry Andric PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 663fe6060f1SDimitry Andric string inst8_32, string inst16_32, string inst8_64, string inst16_64, string inst32_64> { 6640b57cec5SDimitry Andric // Truncating-extending binary RMWs with no constant offset 6655ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 6665ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 6675ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 6685ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 6695ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 6700b57cec5SDimitry Andric 6715ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 6725ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 6735ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 6745ffd83dbSDimitry Andric defm : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 6750b57cec5SDimitry Andric 6760b57cec5SDimitry Andric // Truncating-extending binary RMWs with a constant offset 6775ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 6785ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, 6795ffd83dbSDimitry Andric inst16_32>; 6805ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 6815ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, 6825ffd83dbSDimitry Andric inst16_64>; 6835ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, 6845ffd83dbSDimitry Andric inst32_64>; 6855ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 6865ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 6875ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 6885ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 6895ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 6900b57cec5SDimitry Andric 6915ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 6925ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, 6935ffd83dbSDimitry Andric inst16_32>; 6945ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 6955ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, 6965ffd83dbSDimitry Andric inst16_64>; 6975ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 6985ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 6995ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 7005ffd83dbSDimitry Andric defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 7010b57cec5SDimitry Andric 7020b57cec5SDimitry Andric // Truncating-extending binary RMWs with just a constant offset 7035ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 7045ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 7055ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 7065ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 7075ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 7080b57cec5SDimitry Andric 7095ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 7105ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 7115ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 7125ffd83dbSDimitry Andric defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 7130b57cec5SDimitry Andric 7145ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 7155ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 7165ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 7175ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 7185ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 7190b57cec5SDimitry Andric 7205ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 7215ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 7225ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 7235ffd83dbSDimitry Andric defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 7240b57cec5SDimitry Andric} 7250b57cec5SDimitry Andric 7260b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7270b57cec5SDimitry Andric atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 7285ffd83dbSDimitry Andric "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32", 7295ffd83dbSDimitry Andric "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">; 7300b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7310b57cec5SDimitry Andric atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 7325ffd83dbSDimitry Andric "ATOMIC_RMW8_U_SUB_I32", "ATOMIC_RMW16_U_SUB_I32", 7335ffd83dbSDimitry Andric "ATOMIC_RMW8_U_SUB_I64", "ATOMIC_RMW16_U_SUB_I64", "ATOMIC_RMW32_U_SUB_I64">; 7340b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7350b57cec5SDimitry Andric atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 7365ffd83dbSDimitry Andric "ATOMIC_RMW8_U_AND_I32", "ATOMIC_RMW16_U_AND_I32", 7375ffd83dbSDimitry Andric "ATOMIC_RMW8_U_AND_I64", "ATOMIC_RMW16_U_AND_I64", "ATOMIC_RMW32_U_AND_I64">; 7380b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7390b57cec5SDimitry Andric atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 7405ffd83dbSDimitry Andric "ATOMIC_RMW8_U_OR_I32", "ATOMIC_RMW16_U_OR_I32", 7415ffd83dbSDimitry Andric "ATOMIC_RMW8_U_OR_I64", "ATOMIC_RMW16_U_OR_I64", "ATOMIC_RMW32_U_OR_I64">; 7420b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7430b57cec5SDimitry Andric atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 7445ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XOR_I32", "ATOMIC_RMW16_U_XOR_I32", 7455ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XOR_I64", "ATOMIC_RMW16_U_XOR_I64", "ATOMIC_RMW32_U_XOR_I64">; 7460b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 7470b57cec5SDimitry Andric atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 7485ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32", 7495ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64", 7505ffd83dbSDimitry Andric "ATOMIC_RMW32_U_XCHG_I64">; 7510b57cec5SDimitry Andric 7520b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 7530b57cec5SDimitry Andric// Atomic ternary read-modify-writes 7540b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 7550b57cec5SDimitry Andric 7560b57cec5SDimitry Andric// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 7570b57cec5SDimitry Andric// flag}. When we use the success flag or both values, we can't make use of i64 7580b57cec5SDimitry Andric// truncate/extend versions of instructions for now, which is suboptimal. 7590b57cec5SDimitry Andric// Consider adding a pass after instruction selection that optimizes this case 7600b57cec5SDimitry Andric// if it is frequent. 7610b57cec5SDimitry Andric 7620b57cec5SDimitry Andricmulticlass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 7630b57cec5SDimitry Andric int atomic_op> { 7645ffd83dbSDimitry Andric defm "_A32" : 7650b57cec5SDimitry Andric ATOMIC_I<(outs rc:$dst), 7660b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 7670b57cec5SDimitry Andric rc:$new_), 7680b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 7690b57cec5SDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 7705ffd83dbSDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 7715ffd83dbSDimitry Andric defm "_A64" : 7725ffd83dbSDimitry Andric ATOMIC_I<(outs rc:$dst), 7735ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$exp, 7745ffd83dbSDimitry Andric rc:$new_), 7755ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 7765ffd83dbSDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 7775ffd83dbSDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 7780b57cec5SDimitry Andric} 7790b57cec5SDimitry Andric 7800b57cec5SDimitry Andricdefm ATOMIC_RMW_CMPXCHG_I32 : 7810b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 7820b57cec5SDimitry Andricdefm ATOMIC_RMW_CMPXCHG_I64 : 7830b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 7840b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_CMPXCHG_I32 : 7850b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 7860b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_CMPXCHG_I32 : 7870b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 7880b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_CMPXCHG_I64 : 7890b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 7900b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_CMPXCHG_I64 : 7910b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 7920b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_CMPXCHG_I64 : 7930b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 7940b57cec5SDimitry Andric 7950b57cec5SDimitry Andric// Select ternary RMWs with no constant offset. 7965ffd83dbSDimitry Andricmulticlass TerRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 7975ffd83dbSDimitry Andric def : Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 7985ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, ty:$new)>, 799e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 8005ffd83dbSDimitry Andric def : Pat<(ty (kind I64:$addr, ty:$exp, ty:$new)), 8015ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, ty:$new)>, 802e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 8035ffd83dbSDimitry Andric} 8040b57cec5SDimitry Andric 8050b57cec5SDimitry Andric// Select ternary RMWs with a constant offset. 8060b57cec5SDimitry Andric 8070b57cec5SDimitry Andric// Pattern with address + immediate offset 8085ffd83dbSDimitry Andricmulticlass TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 8095ffd83dbSDimitry Andric string inst> { 8105ffd83dbSDimitry Andric def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 8115ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>, 812e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 8135ffd83dbSDimitry Andric def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$exp, ty:$new)), 8145ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, ty:$new)>, 815e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 8165ffd83dbSDimitry Andric} 8170b57cec5SDimitry Andric 8180b57cec5SDimitry Andric// Select ternary RMWs with just a constant offset. 8195ffd83dbSDimitry Andricmulticlass TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 8205ffd83dbSDimitry Andric def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 8215ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 8225ffd83dbSDimitry Andric ty:$new)>; 8235ffd83dbSDimitry Andric def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 8245ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 8255ffd83dbSDimitry Andric ty:$new)>; 8265ffd83dbSDimitry Andric} 8270b57cec5SDimitry Andric 8285ffd83dbSDimitry Andricmulticlass TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 8295ffd83dbSDimitry Andric def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 8305ffd83dbSDimitry Andric (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 8315ffd83dbSDimitry Andric ty:$new)>, 832e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 8335ffd83dbSDimitry Andric def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 8345ffd83dbSDimitry Andric (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 8355ffd83dbSDimitry Andric ty:$new)>, 836e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 8375ffd83dbSDimitry Andric} 8380b57cec5SDimitry Andric 8390b57cec5SDimitry Andric// Patterns for various addressing modes. 8405ffd83dbSDimitry Andricmulticlass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 8415ffd83dbSDimitry Andric string inst_64> { 8425ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 8435ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 8440b57cec5SDimitry Andric 8455ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 8465ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 8475ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 8485ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 8490b57cec5SDimitry Andric 8505ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 8515ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 8520b57cec5SDimitry Andric 8535ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 8545ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 8550b57cec5SDimitry Andric} 8560b57cec5SDimitry Andric 8570b57cec5SDimitry Andricdefm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 8585ffd83dbSDimitry Andric "ATOMIC_RMW_CMPXCHG_I32", "ATOMIC_RMW_CMPXCHG_I64">; 8590b57cec5SDimitry Andric 8600b57cec5SDimitry Andric// Truncating & zero-extending ternary RMW patterns. 8610b57cec5SDimitry Andric// DAG legalization & optimization before instruction selection may introduce 8620b57cec5SDimitry Andric// additional nodes such as anyext or assertzext depending on operand types. 8630b57cec5SDimitry Andricclass zext_ter_rmw_8_32<PatFrag kind> : 8640b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 865fe6060f1SDimitry Andric (i32 (kind node:$addr, node:$exp, node:$new))>; 866fe6060f1SDimitry Andricclass zext_ter_rmw_16_32<PatFrag kind> : zext_ter_rmw_8_32<kind>; 8670b57cec5SDimitry Andricclass zext_ter_rmw_8_64<PatFrag kind> : 8680b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 8690b57cec5SDimitry Andric (zext (i32 (assertzext (i32 (kind node:$addr, 8700b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 8710b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))))>; 8720b57cec5SDimitry Andricclass zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 8730b57cec5SDimitry Andricclass zext_ter_rmw_32_64<PatFrag kind> : 8740b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 8750b57cec5SDimitry Andric (zext (i32 (kind node:$addr, 8760b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 8770b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))>; 8780b57cec5SDimitry Andric 8790b57cec5SDimitry Andric// Truncating & sign-extending ternary RMW patterns. 8800b57cec5SDimitry Andric// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 8810b57cec5SDimitry Andric// zext RMW; the next instruction will be sext_inreg which is selected by 8820b57cec5SDimitry Andric// itself. 8830b57cec5SDimitry Andricclass sext_ter_rmw_8_32<PatFrag kind> : 8840b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 8850b57cec5SDimitry Andric (kind node:$addr, node:$exp, node:$new)>; 8860b57cec5SDimitry Andricclass sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 8870b57cec5SDimitry Andricclass sext_ter_rmw_8_64<PatFrag kind> : 8880b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 8890b57cec5SDimitry Andric (anyext (i32 (assertzext (i32 8900b57cec5SDimitry Andric (kind node:$addr, 8910b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 8920b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))))>; 8930b57cec5SDimitry Andricclass sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 8940b57cec5SDimitry Andric// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 8950b57cec5SDimitry Andric 8960b57cec5SDimitry Andric// Patterns for various addressing modes for truncating-extending ternary RMWs. 8970b57cec5SDimitry Andricmulticlass TerRMWTruncExtPattern< 8980b57cec5SDimitry Andric PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 8995ffd83dbSDimitry Andric string inst8_32, string inst16_32, string inst8_64, string inst16_64, 9005ffd83dbSDimitry Andric string inst32_64> { 9010b57cec5SDimitry Andric // Truncating-extending ternary RMWs with no constant offset 9025ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 9035ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 9045ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 9055ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 9065ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 9070b57cec5SDimitry Andric 9085ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 9095ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 9105ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 9115ffd83dbSDimitry Andric defm : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 9120b57cec5SDimitry Andric 9130b57cec5SDimitry Andric // Truncating-extending ternary RMWs with a constant offset 9145ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 9155ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, 9165ffd83dbSDimitry Andric inst16_32>; 9175ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 9185ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, 9195ffd83dbSDimitry Andric inst16_64>; 9205ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, 9215ffd83dbSDimitry Andric inst32_64>; 9225ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 9235ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 9245ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 9255ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 9265ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 9270b57cec5SDimitry Andric 9285ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 9295ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, 9305ffd83dbSDimitry Andric inst16_32>; 9315ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 9325ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, 9335ffd83dbSDimitry Andric inst16_64>; 9345ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 9355ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 9365ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 9375ffd83dbSDimitry Andric defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 9380b57cec5SDimitry Andric 9390b57cec5SDimitry Andric // Truncating-extending ternary RMWs with just a constant offset 9405ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 9415ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 9425ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 9435ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 9445ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 9450b57cec5SDimitry Andric 9465ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 9475ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 9485ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 9495ffd83dbSDimitry Andric defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 9500b57cec5SDimitry Andric 9515ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 9525ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 9535ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 9545ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 9555ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 9560b57cec5SDimitry Andric 9575ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 9585ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 9595ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 9605ffd83dbSDimitry Andric defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 9610b57cec5SDimitry Andric} 9620b57cec5SDimitry Andric 9630b57cec5SDimitry Andricdefm : TerRMWTruncExtPattern< 9640b57cec5SDimitry Andric atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 9655ffd83dbSDimitry Andric "ATOMIC_RMW8_U_CMPXCHG_I32", "ATOMIC_RMW16_U_CMPXCHG_I32", 9665ffd83dbSDimitry Andric "ATOMIC_RMW8_U_CMPXCHG_I64", "ATOMIC_RMW16_U_CMPXCHG_I64", 9675ffd83dbSDimitry Andric "ATOMIC_RMW32_U_CMPXCHG_I64">; 968