10b57cec5SDimitry Andric// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 20b57cec5SDimitry Andric// 30b57cec5SDimitry Andric// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric// See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric// 70b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric/// 90b57cec5SDimitry Andric/// \file 100b57cec5SDimitry Andric/// WebAssembly Atomic operand code-gen constructs. 110b57cec5SDimitry Andric/// 120b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 130b57cec5SDimitry Andric 140b57cec5SDimitry Andriclet UseNamedOperandTable = 1 in 150b57cec5SDimitry Andricmulticlass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 165ffd83dbSDimitry Andric list<dag> pattern_r, string asmstr_r, 175ffd83dbSDimitry Andric string asmstr_s, bits<32> atomic_op, 1881ad6265SDimitry Andric bit is64 = false> { 190b57cec5SDimitry Andric defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 205ffd83dbSDimitry Andric !or(0xfe00, !and(0xff, atomic_op)), is64>, 210b57cec5SDimitry Andric Requires<[HasAtomics]>; 220b57cec5SDimitry Andric} 230b57cec5SDimitry Andric 240b57cec5SDimitry Andricmulticlass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 250b57cec5SDimitry Andric bits<32> atomic_op = -1> { 260b57cec5SDimitry Andric defm "" : NRI<oops, iops, pattern, asmstr, 270b57cec5SDimitry Andric !or(0xfe00, !and(0xff, atomic_op))>, 280b57cec5SDimitry Andric Requires<[HasAtomics]>; 290b57cec5SDimitry Andric} 300b57cec5SDimitry Andric 310b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 320b57cec5SDimitry Andric// Atomic wait / notify 330b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 340b57cec5SDimitry Andric 350b57cec5SDimitry Andriclet hasSideEffects = 1 in { 36e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_NOTIFY_A32 : 370b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 380b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 390b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 40e8d8bef9SDimitry Andric "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 4181ad6265SDimitry Andric "memory.atomic.notify \t${off}${p2align}", 0x00, false>; 42e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_NOTIFY_A64 : 435ffd83dbSDimitry Andric ATOMIC_I<(outs I32:$dst), 445ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$count), 455ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 46e8d8bef9SDimitry Andric "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 4781ad6265SDimitry Andric "memory.atomic.notify \t${off}${p2align}", 0x00, true>; 480b57cec5SDimitry Andriclet mayLoad = 1 in { 49e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT32_A32 : 500b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 510b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 520b57cec5SDimitry Andric I64:$timeout), 530b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54e8d8bef9SDimitry Andric "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 5581ad6265SDimitry Andric "memory.atomic.wait32 \t${off}${p2align}", 0x01, false>; 56e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT32_A64 : 575ffd83dbSDimitry Andric ATOMIC_I<(outs I32:$dst), 585ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$exp, 595ffd83dbSDimitry Andric I64:$timeout), 605ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 61e8d8bef9SDimitry Andric "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 6281ad6265SDimitry Andric "memory.atomic.wait32 \t${off}${p2align}", 0x01, true>; 63e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT64_A32 : 640b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 650b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 660b57cec5SDimitry Andric I64:$timeout), 670b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 68e8d8bef9SDimitry Andric "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 6981ad6265SDimitry Andric "memory.atomic.wait64 \t${off}${p2align}", 0x02, false>; 70e8d8bef9SDimitry Andricdefm MEMORY_ATOMIC_WAIT64_A64 : 715ffd83dbSDimitry Andric ATOMIC_I<(outs I32:$dst), 725ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp, 735ffd83dbSDimitry Andric I64:$timeout), 745ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 75e8d8bef9SDimitry Andric "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 7681ad6265SDimitry Andric "memory.atomic.wait64 \t${off}${p2align}", 0x02, true>; 770b57cec5SDimitry Andric} // mayLoad = 1 780b57cec5SDimitry Andric} // hasSideEffects = 1 790b57cec5SDimitry Andric 80bdd1243dSDimitry Andricdef NotifyPat_A32 : 81bdd1243dSDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify (AddrOps32 offset32_op:$offset, I32:$addr), I32:$count)), 82bdd1243dSDimitry Andric (MEMORY_ATOMIC_NOTIFY_A32 0, $offset, $addr, $count)>, 83e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 84bdd1243dSDimitry Andricdef NotifyPat_A64 : 85bdd1243dSDimitry Andric Pat<(i32 (int_wasm_memory_atomic_notify (AddrOps64 offset64_op:$offset, I64:$addr), I32:$count)), 86bdd1243dSDimitry Andric (MEMORY_ATOMIC_NOTIFY_A64 0, $offset, $addr, $count)>, 87e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 880b57cec5SDimitry Andric 890b57cec5SDimitry Andric 90bdd1243dSDimitry Andricmulticlass WaitPat<ValueType ty, Intrinsic kind, string inst> { 91bdd1243dSDimitry Andric def WaitPat_A32 : 92bdd1243dSDimitry Andric Pat<(i32 (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$exp, I64:$timeout)), 93bdd1243dSDimitry Andric (!cast<NI>(inst#_A32) 0, $offset, $addr, $exp, $timeout)>, 94e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 95bdd1243dSDimitry Andric def WaitPat_A64 : 96bdd1243dSDimitry Andric Pat<(i32 (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$exp, I64:$timeout)), 97bdd1243dSDimitry Andric (!cast<NI>(inst#_A64) 0, $offset, $addr, $exp, $timeout)>, 98e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 995ffd83dbSDimitry Andric} 1000b57cec5SDimitry Andric 101bdd1243dSDimitry Andricdefm : WaitPat<i32, int_wasm_memory_atomic_wait32, "MEMORY_ATOMIC_WAIT32">; 102bdd1243dSDimitry Andricdefm : WaitPat<i64, int_wasm_memory_atomic_wait64, "MEMORY_ATOMIC_WAIT64">; 1030b57cec5SDimitry Andric 1040b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 1058bcb0991SDimitry Andric// Atomic fences 1068bcb0991SDimitry Andric//===----------------------------------------------------------------------===// 1078bcb0991SDimitry Andric 1088bcb0991SDimitry Andric// A compiler fence instruction that prevents reordering of instructions. 1098bcb0991SDimitry Andriclet Defs = [ARGUMENTS] in { 1108bcb0991SDimitry Andriclet isPseudo = 1, hasSideEffects = 1 in 1118bcb0991SDimitry Andricdefm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 1128bcb0991SDimitry Andriclet hasSideEffects = 1 in 1138bcb0991SDimitry Andricdefm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", 1148bcb0991SDimitry Andric 0x03>; 1158bcb0991SDimitry Andric} // Defs = [ARGUMENTS] 1168bcb0991SDimitry Andric 1178bcb0991SDimitry Andric//===----------------------------------------------------------------------===// 1180b57cec5SDimitry Andric// Atomic loads 1190b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 1200b57cec5SDimitry Andric 1210b57cec5SDimitry Andricmulticlass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 1225ffd83dbSDimitry Andric defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 1235ffd83dbSDimitry Andric [HasAtomics]>; 1240b57cec5SDimitry Andric} 1250b57cec5SDimitry Andric 1260b57cec5SDimitry Andricdefm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 1270b57cec5SDimitry Andricdefm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 1280b57cec5SDimitry Andric 129bdd1243dSDimitry Andric// Select loads 130bdd1243dSDimitry Andricdefm : LoadPat<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 131bdd1243dSDimitry Andricdefm : LoadPat<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 1320b57cec5SDimitry Andric 1330b57cec5SDimitry Andric// Extending loads. Note that there are only zero-extending atomic loads, no 1340b57cec5SDimitry Andric// sign-extending loads. 1350b57cec5SDimitry Andricdefm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 1360b57cec5SDimitry Andricdefm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 1370b57cec5SDimitry Andricdefm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 1380b57cec5SDimitry Andricdefm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 1390b57cec5SDimitry Andricdefm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 1400b57cec5SDimitry Andric 1410b57cec5SDimitry Andric// Fragments for extending loads. These are different from regular loads because 1420b57cec5SDimitry Andric// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 1430b57cec5SDimitry Andric// therefore don't have the extension type field. So instead of matching that, 1440b57cec5SDimitry Andric// we match the patterns that the type legalizer expands them to. 1450b57cec5SDimitry Andric 1460b57cec5SDimitry Andric// Unlike regular loads, extension to i64 is handled differently than i32. 1470b57cec5SDimitry Andric// i64 (zext (i8 (atomic_load_8))) gets legalized to 1480b57cec5SDimitry Andric// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 149fe6060f1SDimitry Andric// Extension to i32 is elided by SelectionDAG as our atomic loads are 150fe6060f1SDimitry Andric// zero-extending. 1510b57cec5SDimitry Andricdef zext_aload_8_64 : 1520b57cec5SDimitry Andric PatFrag<(ops node:$addr), 153fe6060f1SDimitry Andric (i64 (zext (i32 (atomic_load_8 node:$addr))))>; 1540b57cec5SDimitry Andricdef zext_aload_16_64 : 1550b57cec5SDimitry Andric PatFrag<(ops node:$addr), 156fe6060f1SDimitry Andric (i64 (zext (i32 (atomic_load_16 node:$addr))))>; 1570b57cec5SDimitry Andricdef zext_aload_32_64 : 1580b57cec5SDimitry Andric PatFrag<(ops node:$addr), 159fe6060f1SDimitry Andric (i64 (zext (i32 (atomic_load_32 node:$addr))))>; 1600b57cec5SDimitry Andric 1610b57cec5SDimitry Andric// We don't have single sext atomic load instructions. So for sext loads, we 1620b57cec5SDimitry Andric// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 1630b57cec5SDimitry Andric// results) and select a zext load; the next instruction will be sext_inreg 1640b57cec5SDimitry Andric// which is selected by itself. 1650b57cec5SDimitry Andricdef sext_aload_8_64 : 1660b57cec5SDimitry Andric PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 1670b57cec5SDimitry Andricdef sext_aload_16_64 : 1680b57cec5SDimitry Andric PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 1690b57cec5SDimitry Andric 170bdd1243dSDimitry Andric// Select zero-extending loads 171bdd1243dSDimitry Andricdefm : LoadPat<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 172bdd1243dSDimitry Andricdefm : LoadPat<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 173bdd1243dSDimitry Andricdefm : LoadPat<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 1740b57cec5SDimitry Andric 175bdd1243dSDimitry Andric// Select sign-extending loads 176bdd1243dSDimitry Andricdefm : LoadPat<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 177bdd1243dSDimitry Andricdefm : LoadPat<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 178bdd1243dSDimitry Andricdefm : LoadPat<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 179bdd1243dSDimitry Andricdefm : LoadPat<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 1800b57cec5SDimitry Andric// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 1810b57cec5SDimitry Andric 1820b57cec5SDimitry Andric 1830b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 1840b57cec5SDimitry Andric// Atomic stores 1850b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 1860b57cec5SDimitry Andric 1870b57cec5SDimitry Andricmulticlass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 1885ffd83dbSDimitry Andric defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 1895ffd83dbSDimitry Andric [HasAtomics]>; 1900b57cec5SDimitry Andric} 1910b57cec5SDimitry Andric 1920b57cec5SDimitry Andricdefm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 1930b57cec5SDimitry Andricdefm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 1940b57cec5SDimitry Andric 1955f757f3fSDimitry Andric// We used to need an 'atomic' version of store patterns because store and atomic_store 1965f757f3fSDimitry Andric// nodes have different operand orders. 1975f757f3fSDimitry Andric// 1985f757f3fSDimitry Andric// TODO: This is no longer true and atomic_store and store patterns 1995f757f3fSDimitry Andric// can be unified. 2000b57cec5SDimitry Andric 201bdd1243dSDimitry Andricmulticlass AStorePat<ValueType ty, PatFrag kind, string inst> { 2025f757f3fSDimitry Andric def : Pat<(kind ty:$val, (AddrOps32 offset32_op:$offset, I32:$addr)), 203bdd1243dSDimitry Andric (!cast<NI>(inst#_A32) 0, $offset, $addr, $val)>, 204e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 2055f757f3fSDimitry Andric def : Pat<(kind ty:$val, (AddrOps64 offset64_op:$offset, I64:$addr)), 206bdd1243dSDimitry Andric (!cast<NI>(inst#_A64) 0, $offset, $addr, $val)>, 207e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 2085ffd83dbSDimitry Andric} 209bdd1243dSDimitry Andricdefm : AStorePat<i32, atomic_store_32, "ATOMIC_STORE_I32">; 210bdd1243dSDimitry Andricdefm : AStorePat<i64, atomic_store_64, "ATOMIC_STORE_I64">; 2110b57cec5SDimitry Andric 2120b57cec5SDimitry Andric// Truncating stores. 2130b57cec5SDimitry Andricdefm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 2140b57cec5SDimitry Andricdefm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 2150b57cec5SDimitry Andricdefm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 2160b57cec5SDimitry Andricdefm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 2170b57cec5SDimitry Andricdefm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 2180b57cec5SDimitry Andric 2190b57cec5SDimitry Andric// Fragments for truncating stores. 2200b57cec5SDimitry Andric 2210b57cec5SDimitry Andric// We don't have single truncating atomic store instructions. For 32-bit 2220b57cec5SDimitry Andric// instructions, we just need to match bare atomic stores. On the other hand, 2230b57cec5SDimitry Andric// truncating stores from i64 values are once truncated to i32 first. 2240b57cec5SDimitry Andricclass trunc_astore_64<PatFrag kind> : 2255f757f3fSDimitry Andric PatFrag<(ops node:$val, node:$addr), 2265f757f3fSDimitry Andric (kind (i32 (trunc (i64 node:$val))), node:$addr)>; 2270b57cec5SDimitry Andricdef trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 2280b57cec5SDimitry Andricdef trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 2290b57cec5SDimitry Andricdef trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 2300b57cec5SDimitry Andric 2310b57cec5SDimitry Andric// Truncating stores with no constant offset 232bdd1243dSDimitry Andricdefm : AStorePat<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 233bdd1243dSDimitry Andricdefm : AStorePat<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 234bdd1243dSDimitry Andricdefm : AStorePat<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 235bdd1243dSDimitry Andricdefm : AStorePat<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 236bdd1243dSDimitry Andricdefm : AStorePat<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 2370b57cec5SDimitry Andric 2380b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2390b57cec5SDimitry Andric// Atomic binary read-modify-writes 2400b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2410b57cec5SDimitry Andric 2420b57cec5SDimitry Andricmulticlass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 2430b57cec5SDimitry Andric int atomic_op> { 2445ffd83dbSDimitry Andric defm "_A32" : 2450b57cec5SDimitry Andric ATOMIC_I<(outs rc:$dst), 2460b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 2470b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 2480b57cec5SDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 24981ad6265SDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, false>; 2505ffd83dbSDimitry Andric defm "_A64" : 2515ffd83dbSDimitry Andric ATOMIC_I<(outs rc:$dst), 2525ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$val), 2535ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 2545ffd83dbSDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 25581ad6265SDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, true>; 2560b57cec5SDimitry Andric} 2570b57cec5SDimitry Andric 2580b57cec5SDimitry Andricdefm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 2590b57cec5SDimitry Andricdefm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 2600b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_ADD_I32 : 2610b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 2620b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_ADD_I32 : 2630b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 2640b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_ADD_I64 : 2650b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 2660b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_ADD_I64 : 2670b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 2680b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_ADD_I64 : 2690b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 2700b57cec5SDimitry Andric 2710b57cec5SDimitry Andricdefm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 2720b57cec5SDimitry Andricdefm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 2730b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_SUB_I32 : 2740b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 2750b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_SUB_I32 : 2760b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 2770b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_SUB_I64 : 2780b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 2790b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_SUB_I64 : 2800b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 2810b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_SUB_I64 : 2820b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 2830b57cec5SDimitry Andric 2840b57cec5SDimitry Andricdefm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 2850b57cec5SDimitry Andricdefm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 2860b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_AND_I32 : 2870b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 2880b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_AND_I32 : 2890b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 2900b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_AND_I64 : 2910b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 2920b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_AND_I64 : 2930b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 2940b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_AND_I64 : 2950b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 2960b57cec5SDimitry Andric 2970b57cec5SDimitry Andricdefm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 2980b57cec5SDimitry Andricdefm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 2990b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_OR_I32 : 3000b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 3010b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_OR_I32 : 3020b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 3030b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_OR_I64 : 3040b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 3050b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_OR_I64 : 3060b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 3070b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_OR_I64 : 3080b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 3090b57cec5SDimitry Andric 3100b57cec5SDimitry Andricdefm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 3110b57cec5SDimitry Andricdefm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 3120b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XOR_I32 : 3130b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 3140b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XOR_I32 : 3150b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 3160b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XOR_I64 : 3170b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 3180b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XOR_I64 : 3190b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 3200b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_XOR_I64 : 3210b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 3220b57cec5SDimitry Andric 3230b57cec5SDimitry Andricdefm ATOMIC_RMW_XCHG_I32 : 3240b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 3250b57cec5SDimitry Andricdefm ATOMIC_RMW_XCHG_I64 : 3260b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 3270b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XCHG_I32 : 3280b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 3290b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XCHG_I32 : 3300b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 3310b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XCHG_I64 : 3320b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 3330b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XCHG_I64 : 3340b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 3350b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_XCHG_I64 : 3360b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 3370b57cec5SDimitry Andric 338bdd1243dSDimitry Andricmulticlass BinRMWPat<ValueType ty, PatFrag kind, string inst> { 339bdd1243dSDimitry Andric def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$val)), 340bdd1243dSDimitry Andric (!cast<NI>(inst#_A32) 0, $offset, $addr, $val)>, 341e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 342bdd1243dSDimitry Andric def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$val)), 343bdd1243dSDimitry Andric (!cast<NI>(inst#_A64) 0, $offset, $addr, $val)>, 344e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 3455ffd83dbSDimitry Andric} 3460b57cec5SDimitry Andric 3470b57cec5SDimitry Andric// Patterns for various addressing modes. 3485ffd83dbSDimitry Andricmulticlass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 3495ffd83dbSDimitry Andric string inst_64> { 350bdd1243dSDimitry Andric defm : BinRMWPat<i32, rmw_32, inst_32>; 351bdd1243dSDimitry Andric defm : BinRMWPat<i64, rmw_64, inst_64>; 3520b57cec5SDimitry Andric} 3530b57cec5SDimitry Andric 3545ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, 3555ffd83dbSDimitry Andric "ATOMIC_RMW_ADD_I32", "ATOMIC_RMW_ADD_I64">; 3565ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, 3575ffd83dbSDimitry Andric "ATOMIC_RMW_SUB_I32", "ATOMIC_RMW_SUB_I64">; 3585ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, 3595ffd83dbSDimitry Andric "ATOMIC_RMW_AND_I32", "ATOMIC_RMW_AND_I64">; 3605ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, 3615ffd83dbSDimitry Andric "ATOMIC_RMW_OR_I32", "ATOMIC_RMW_OR_I64">; 3625ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, 3635ffd83dbSDimitry Andric "ATOMIC_RMW_XOR_I32", "ATOMIC_RMW_XOR_I64">; 3645ffd83dbSDimitry Andricdefm : BinRMWPattern<atomic_swap_32, atomic_swap_64, 3655ffd83dbSDimitry Andric "ATOMIC_RMW_XCHG_I32", "ATOMIC_RMW_XCHG_I64">; 3660b57cec5SDimitry Andric 3670b57cec5SDimitry Andric// Truncating & zero-extending binary RMW patterns. 3680b57cec5SDimitry Andric// These are combined patterns of truncating store patterns and zero-extending 3690b57cec5SDimitry Andric// load patterns above. 3700b57cec5SDimitry Andricclass zext_bin_rmw_8_32<PatFrag kind> : 371fe6060f1SDimitry Andric PatFrag<(ops node:$addr, node:$val), (i32 (kind node:$addr, node:$val))>; 372fe6060f1SDimitry Andricclass zext_bin_rmw_16_32<PatFrag kind> : zext_bin_rmw_8_32<kind>; 3730b57cec5SDimitry Andricclass zext_bin_rmw_8_64<PatFrag kind> : 3740b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 3750b57cec5SDimitry Andric (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 376fe6060f1SDimitry Andricclass zext_bin_rmw_16_64<PatFrag kind> : zext_bin_rmw_8_64<kind>; 377fe6060f1SDimitry Andricclass zext_bin_rmw_32_64<PatFrag kind> : zext_bin_rmw_8_64<kind>; 3780b57cec5SDimitry Andric 3790b57cec5SDimitry Andric// Truncating & sign-extending binary RMW patterns. 3800b57cec5SDimitry Andric// These are combined patterns of truncating store patterns and sign-extending 3810b57cec5SDimitry Andric// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 3820b57cec5SDimitry Andric// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 3830b57cec5SDimitry Andric// is selected by itself. 3840b57cec5SDimitry Andricclass sext_bin_rmw_8_32<PatFrag kind> : 3850b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 3860b57cec5SDimitry Andricclass sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 3870b57cec5SDimitry Andricclass sext_bin_rmw_8_64<PatFrag kind> : 3880b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 3890b57cec5SDimitry Andric (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 3900b57cec5SDimitry Andricclass sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 3910b57cec5SDimitry Andric// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 3920b57cec5SDimitry Andric 3930b57cec5SDimitry Andric// Patterns for various addressing modes for truncating-extending binary RMWs. 3940b57cec5SDimitry Andricmulticlass BinRMWTruncExtPattern< 395349cc55cSDimitry Andric PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, 396fe6060f1SDimitry Andric string inst8_32, string inst16_32, string inst8_64, string inst16_64, string inst32_64> { 397bdd1243dSDimitry Andric // Truncating-extending binary RMWs 398bdd1243dSDimitry Andric defm : BinRMWPat<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 399bdd1243dSDimitry Andric defm : BinRMWPat<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 400bdd1243dSDimitry Andric defm : BinRMWPat<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 401bdd1243dSDimitry Andric defm : BinRMWPat<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 402bdd1243dSDimitry Andric defm : BinRMWPat<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 4030b57cec5SDimitry Andric 404bdd1243dSDimitry Andric defm : BinRMWPat<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 405bdd1243dSDimitry Andric defm : BinRMWPat<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 406bdd1243dSDimitry Andric defm : BinRMWPat<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 407bdd1243dSDimitry Andric defm : BinRMWPat<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 4080b57cec5SDimitry Andric} 4090b57cec5SDimitry Andric 4100b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 411349cc55cSDimitry Andric atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, 4125ffd83dbSDimitry Andric "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32", 4135ffd83dbSDimitry Andric "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">; 4140b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 415349cc55cSDimitry Andric atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, 4165ffd83dbSDimitry Andric "ATOMIC_RMW8_U_SUB_I32", "ATOMIC_RMW16_U_SUB_I32", 4175ffd83dbSDimitry Andric "ATOMIC_RMW8_U_SUB_I64", "ATOMIC_RMW16_U_SUB_I64", "ATOMIC_RMW32_U_SUB_I64">; 4180b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 419349cc55cSDimitry Andric atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, 4205ffd83dbSDimitry Andric "ATOMIC_RMW8_U_AND_I32", "ATOMIC_RMW16_U_AND_I32", 4215ffd83dbSDimitry Andric "ATOMIC_RMW8_U_AND_I64", "ATOMIC_RMW16_U_AND_I64", "ATOMIC_RMW32_U_AND_I64">; 4220b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 423349cc55cSDimitry Andric atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, 4245ffd83dbSDimitry Andric "ATOMIC_RMW8_U_OR_I32", "ATOMIC_RMW16_U_OR_I32", 4255ffd83dbSDimitry Andric "ATOMIC_RMW8_U_OR_I64", "ATOMIC_RMW16_U_OR_I64", "ATOMIC_RMW32_U_OR_I64">; 4260b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 427349cc55cSDimitry Andric atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, 4285ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XOR_I32", "ATOMIC_RMW16_U_XOR_I32", 4295ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XOR_I64", "ATOMIC_RMW16_U_XOR_I64", "ATOMIC_RMW32_U_XOR_I64">; 4300b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 431349cc55cSDimitry Andric atomic_swap_8, atomic_swap_16, atomic_swap_32, 4325ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32", 4335ffd83dbSDimitry Andric "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64", 4345ffd83dbSDimitry Andric "ATOMIC_RMW32_U_XCHG_I64">; 4350b57cec5SDimitry Andric 4360b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 4370b57cec5SDimitry Andric// Atomic ternary read-modify-writes 4380b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 4390b57cec5SDimitry Andric 4400b57cec5SDimitry Andric// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 4410b57cec5SDimitry Andric// flag}. When we use the success flag or both values, we can't make use of i64 4420b57cec5SDimitry Andric// truncate/extend versions of instructions for now, which is suboptimal. 4430b57cec5SDimitry Andric// Consider adding a pass after instruction selection that optimizes this case 4440b57cec5SDimitry Andric// if it is frequent. 4450b57cec5SDimitry Andric 4460b57cec5SDimitry Andricmulticlass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 4470b57cec5SDimitry Andric int atomic_op> { 4485ffd83dbSDimitry Andric defm "_A32" : 4490b57cec5SDimitry Andric ATOMIC_I<(outs rc:$dst), 4500b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 4510b57cec5SDimitry Andric rc:$new_), 4520b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 4530b57cec5SDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 45481ad6265SDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, false>; 4555ffd83dbSDimitry Andric defm "_A64" : 4565ffd83dbSDimitry Andric ATOMIC_I<(outs rc:$dst), 4575ffd83dbSDimitry Andric (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$exp, 4585ffd83dbSDimitry Andric rc:$new_), 4595ffd83dbSDimitry Andric (outs), (ins P2Align:$p2align, offset64_op:$off), [], 4605ffd83dbSDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 46181ad6265SDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op, true>; 4620b57cec5SDimitry Andric} 4630b57cec5SDimitry Andric 4640b57cec5SDimitry Andricdefm ATOMIC_RMW_CMPXCHG_I32 : 4650b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 4660b57cec5SDimitry Andricdefm ATOMIC_RMW_CMPXCHG_I64 : 4670b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 4680b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_CMPXCHG_I32 : 4690b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 4700b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_CMPXCHG_I32 : 4710b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 4720b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_CMPXCHG_I64 : 4730b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 4740b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_CMPXCHG_I64 : 4750b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 4760b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_CMPXCHG_I64 : 4770b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 4780b57cec5SDimitry Andric 479bdd1243dSDimitry Andricmulticlass TerRMWPat<ValueType ty, PatFrag kind, string inst> { 480bdd1243dSDimitry Andric def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$exp, ty:$new)), 481bdd1243dSDimitry Andric (!cast<NI>(inst#_A32) 0, $offset, $addr, $exp, $new)>, 482e8d8bef9SDimitry Andric Requires<[HasAddr32, HasAtomics]>; 483bdd1243dSDimitry Andric def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$exp, ty:$new)), 484bdd1243dSDimitry Andric (!cast<NI>(inst#_A64) 0, $offset, $addr, $exp, $new)>, 485e8d8bef9SDimitry Andric Requires<[HasAddr64, HasAtomics]>; 4865ffd83dbSDimitry Andric} 4870b57cec5SDimitry Andric 488bdd1243dSDimitry Andricdefm : TerRMWPat<i32, atomic_cmp_swap_32, "ATOMIC_RMW_CMPXCHG_I32">; 489bdd1243dSDimitry Andricdefm : TerRMWPat<i64, atomic_cmp_swap_64, "ATOMIC_RMW_CMPXCHG_I64">; 4900b57cec5SDimitry Andric 4910b57cec5SDimitry Andric// Truncating & zero-extending ternary RMW patterns. 4920b57cec5SDimitry Andric// DAG legalization & optimization before instruction selection may introduce 4930b57cec5SDimitry Andric// additional nodes such as anyext or assertzext depending on operand types. 4940b57cec5SDimitry Andricclass zext_ter_rmw_8_32<PatFrag kind> : 4950b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 496fe6060f1SDimitry Andric (i32 (kind node:$addr, node:$exp, node:$new))>; 497fe6060f1SDimitry Andricclass zext_ter_rmw_16_32<PatFrag kind> : zext_ter_rmw_8_32<kind>; 4980b57cec5SDimitry Andricclass zext_ter_rmw_8_64<PatFrag kind> : 4990b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 5000b57cec5SDimitry Andric (zext (i32 (assertzext (i32 (kind node:$addr, 5010b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 5020b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))))>; 5030b57cec5SDimitry Andricclass zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 5040b57cec5SDimitry Andricclass zext_ter_rmw_32_64<PatFrag kind> : 5050b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 5060b57cec5SDimitry Andric (zext (i32 (kind node:$addr, 5070b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 5080b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))>; 5090b57cec5SDimitry Andric 5100b57cec5SDimitry Andric// Truncating & sign-extending ternary RMW patterns. 5110b57cec5SDimitry Andric// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 5120b57cec5SDimitry Andric// zext RMW; the next instruction will be sext_inreg which is selected by 5130b57cec5SDimitry Andric// itself. 5140b57cec5SDimitry Andricclass sext_ter_rmw_8_32<PatFrag kind> : 5150b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 5160b57cec5SDimitry Andric (kind node:$addr, node:$exp, node:$new)>; 5170b57cec5SDimitry Andricclass sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 5180b57cec5SDimitry Andricclass sext_ter_rmw_8_64<PatFrag kind> : 5190b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 5200b57cec5SDimitry Andric (anyext (i32 (assertzext (i32 5210b57cec5SDimitry Andric (kind node:$addr, 5220b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 5230b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))))>; 5240b57cec5SDimitry Andricclass sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 5250b57cec5SDimitry Andric// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 5260b57cec5SDimitry Andric 527bdd1243dSDimitry Andricdefm : TerRMWPat<i32, zext_ter_rmw_8_32<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I32">; 528bdd1243dSDimitry Andricdefm : TerRMWPat<i32, zext_ter_rmw_16_32<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I32">; 529bdd1243dSDimitry Andricdefm : TerRMWPat<i64, zext_ter_rmw_8_64<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I64">; 530bdd1243dSDimitry Andricdefm : TerRMWPat<i64, zext_ter_rmw_16_64<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I64">; 531bdd1243dSDimitry Andricdefm : TerRMWPat<i64, zext_ter_rmw_32_64<atomic_cmp_swap_32>, "ATOMIC_RMW32_U_CMPXCHG_I64">; 5320b57cec5SDimitry Andric 533bdd1243dSDimitry Andricdefm : TerRMWPat<i32, sext_ter_rmw_8_32<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I32">; 534bdd1243dSDimitry Andricdefm : TerRMWPat<i32, sext_ter_rmw_16_32<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I32">; 535bdd1243dSDimitry Andricdefm : TerRMWPat<i64, sext_ter_rmw_8_64<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I64">; 536bdd1243dSDimitry Andricdefm : TerRMWPat<i64, sext_ter_rmw_16_64<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I64">; 537