10b57cec5SDimitry Andric// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 20b57cec5SDimitry Andric// 30b57cec5SDimitry Andric// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric// See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric// 70b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric/// 90b57cec5SDimitry Andric/// \file 100b57cec5SDimitry Andric/// WebAssembly Atomic operand code-gen constructs. 110b57cec5SDimitry Andric/// 120b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 130b57cec5SDimitry Andric 140b57cec5SDimitry Andriclet UseNamedOperandTable = 1 in 150b57cec5SDimitry Andricmulticlass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 160b57cec5SDimitry Andric list<dag> pattern_r, string asmstr_r = "", 170b57cec5SDimitry Andric string asmstr_s = "", bits<32> atomic_op = -1> { 180b57cec5SDimitry Andric defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 190b57cec5SDimitry Andric !or(0xfe00, !and(0xff, atomic_op))>, 200b57cec5SDimitry Andric Requires<[HasAtomics]>; 210b57cec5SDimitry Andric} 220b57cec5SDimitry Andric 230b57cec5SDimitry Andricmulticlass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 240b57cec5SDimitry Andric bits<32> atomic_op = -1> { 250b57cec5SDimitry Andric defm "" : NRI<oops, iops, pattern, asmstr, 260b57cec5SDimitry Andric !or(0xfe00, !and(0xff, atomic_op))>, 270b57cec5SDimitry Andric Requires<[HasAtomics]>; 280b57cec5SDimitry Andric} 290b57cec5SDimitry Andric 300b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 310b57cec5SDimitry Andric// Atomic wait / notify 320b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 330b57cec5SDimitry Andric 340b57cec5SDimitry Andriclet hasSideEffects = 1 in { 350b57cec5SDimitry Andricdefm ATOMIC_NOTIFY : 360b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 370b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 380b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 390b57cec5SDimitry Andric "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 400b57cec5SDimitry Andric "atomic.notify \t${off}${p2align}", 0x00>; 410b57cec5SDimitry Andriclet mayLoad = 1 in { 420b57cec5SDimitry Andricdefm ATOMIC_WAIT_I32 : 430b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 440b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 450b57cec5SDimitry Andric I64:$timeout), 460b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 470b57cec5SDimitry Andric "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 480b57cec5SDimitry Andric "i32.atomic.wait \t${off}${p2align}", 0x01>; 490b57cec5SDimitry Andricdefm ATOMIC_WAIT_I64 : 500b57cec5SDimitry Andric ATOMIC_I<(outs I32:$dst), 510b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 520b57cec5SDimitry Andric I64:$timeout), 530b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 540b57cec5SDimitry Andric "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 550b57cec5SDimitry Andric "i64.atomic.wait \t${off}${p2align}", 0x02>; 560b57cec5SDimitry Andric} // mayLoad = 1 570b57cec5SDimitry Andric} // hasSideEffects = 1 580b57cec5SDimitry Andric 590b57cec5SDimitry Andriclet Predicates = [HasAtomics] in { 600b57cec5SDimitry Andric// Select notifys with no constant offset. 610b57cec5SDimitry Andricdef NotifyPatNoOffset : 620b57cec5SDimitry Andric Pat<(i32 (int_wasm_atomic_notify I32:$addr, I32:$count)), 630b57cec5SDimitry Andric (ATOMIC_NOTIFY 0, 0, I32:$addr, I32:$count)>; 640b57cec5SDimitry Andric 650b57cec5SDimitry Andric// Select notifys with a constant offset. 660b57cec5SDimitry Andric 670b57cec5SDimitry Andric// Pattern with address + immediate offset 680b57cec5SDimitry Andricclass NotifyPatImmOff<PatFrag operand> : 690b57cec5SDimitry Andric Pat<(i32 (int_wasm_atomic_notify (operand I32:$addr, imm:$off), I32:$count)), 700b57cec5SDimitry Andric (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I32:$count)>; 710b57cec5SDimitry Andricdef : NotifyPatImmOff<regPlusImm>; 720b57cec5SDimitry Andricdef : NotifyPatImmOff<or_is_add>; 730b57cec5SDimitry Andric 740b57cec5SDimitry Andricdef NotifyPatGlobalAddr : 750b57cec5SDimitry Andric Pat<(i32 (int_wasm_atomic_notify (regPlusGA I32:$addr, 760b57cec5SDimitry Andric (WebAssemblywrapper tglobaladdr:$off)), 770b57cec5SDimitry Andric I32:$count)), 780b57cec5SDimitry Andric (ATOMIC_NOTIFY 0, tglobaladdr:$off, I32:$addr, I32:$count)>; 790b57cec5SDimitry Andric 800b57cec5SDimitry Andric// Select notifys with just a constant offset. 810b57cec5SDimitry Andricdef NotifyPatOffsetOnly : 820b57cec5SDimitry Andric Pat<(i32 (int_wasm_atomic_notify imm:$off, I32:$count)), 830b57cec5SDimitry Andric (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I32:$count)>; 840b57cec5SDimitry Andric 850b57cec5SDimitry Andricdef NotifyPatGlobalAddrOffOnly : 860b57cec5SDimitry Andric Pat<(i32 (int_wasm_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 870b57cec5SDimitry Andric I32:$count)), 880b57cec5SDimitry Andric (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>; 890b57cec5SDimitry Andric 900b57cec5SDimitry Andric// Select waits with no constant offset. 910b57cec5SDimitry Andricclass WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> : 920b57cec5SDimitry Andric Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 930b57cec5SDimitry Andric (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>; 940b57cec5SDimitry Andricdef : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 950b57cec5SDimitry Andricdef : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 960b57cec5SDimitry Andric 970b57cec5SDimitry Andric// Select waits with a constant offset. 980b57cec5SDimitry Andric 990b57cec5SDimitry Andric// Pattern with address + immediate offset 1000b57cec5SDimitry Andricclass WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> : 1010b57cec5SDimitry Andric Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 1020b57cec5SDimitry Andric (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>; 1030b57cec5SDimitry Andricdef : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>; 1040b57cec5SDimitry Andricdef : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>; 1050b57cec5SDimitry Andricdef : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>; 1060b57cec5SDimitry Andricdef : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>; 1070b57cec5SDimitry Andric 1080b57cec5SDimitry Andricclass WaitPatGlobalAddr<ValueType ty, Intrinsic kind, NI inst> : 1090b57cec5SDimitry Andric Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 1100b57cec5SDimitry Andric ty:$exp, I64:$timeout)), 1110b57cec5SDimitry Andric (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, I64:$timeout)>; 1120b57cec5SDimitry Andricdef : WaitPatGlobalAddr<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1130b57cec5SDimitry Andricdef : WaitPatGlobalAddr<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1140b57cec5SDimitry Andric 1150b57cec5SDimitry Andric// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset. 1160b57cec5SDimitry Andricclass WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> : 1170b57cec5SDimitry Andric Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 1180b57cec5SDimitry Andric (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1190b57cec5SDimitry Andricdef : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1200b57cec5SDimitry Andricdef : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1210b57cec5SDimitry Andric 1220b57cec5SDimitry Andricclass WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> : 1230b57cec5SDimitry Andric Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)), 1240b57cec5SDimitry Andric (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1250b57cec5SDimitry Andricdef : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1260b57cec5SDimitry Andricdef : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1270b57cec5SDimitry Andric} // Predicates = [HasAtomics] 1280b57cec5SDimitry Andric 1290b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 1300b57cec5SDimitry Andric// Atomic loads 1310b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 1320b57cec5SDimitry Andric 1330b57cec5SDimitry Andricmulticlass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 1340b57cec5SDimitry Andric defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op))>, 1350b57cec5SDimitry Andric Requires<[HasAtomics]>; 1360b57cec5SDimitry Andric} 1370b57cec5SDimitry Andric 1380b57cec5SDimitry Andricdefm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 1390b57cec5SDimitry Andricdefm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 1400b57cec5SDimitry Andric 1410b57cec5SDimitry Andric// Select loads with no constant offset. 1420b57cec5SDimitry Andriclet Predicates = [HasAtomics] in { 1430b57cec5SDimitry Andricdef : LoadPatNoOffset<i32, atomic_load_32, ATOMIC_LOAD_I32>; 1440b57cec5SDimitry Andricdef : LoadPatNoOffset<i64, atomic_load_64, ATOMIC_LOAD_I64>; 1450b57cec5SDimitry Andric 1460b57cec5SDimitry Andric// Select loads with a constant offset. 1470b57cec5SDimitry Andric 1480b57cec5SDimitry Andric// Pattern with address + immediate offset 1490b57cec5SDimitry Andricdef : LoadPatImmOff<i32, atomic_load_32, regPlusImm, ATOMIC_LOAD_I32>; 1500b57cec5SDimitry Andricdef : LoadPatImmOff<i64, atomic_load_64, regPlusImm, ATOMIC_LOAD_I64>; 1510b57cec5SDimitry Andricdef : LoadPatImmOff<i32, atomic_load_32, or_is_add, ATOMIC_LOAD_I32>; 1520b57cec5SDimitry Andricdef : LoadPatImmOff<i64, atomic_load_64, or_is_add, ATOMIC_LOAD_I64>; 1530b57cec5SDimitry Andric 1540b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i32, atomic_load_32, ATOMIC_LOAD_I32>; 1550b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i64, atomic_load_64, ATOMIC_LOAD_I64>; 1560b57cec5SDimitry Andric 1570b57cec5SDimitry Andric// Select loads with just a constant offset. 1580b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 1590b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 1600b57cec5SDimitry Andric 1610b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 1620b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 1630b57cec5SDimitry Andric 1640b57cec5SDimitry Andric} // Predicates = [HasAtomics] 1650b57cec5SDimitry Andric 1660b57cec5SDimitry Andric// Extending loads. Note that there are only zero-extending atomic loads, no 1670b57cec5SDimitry Andric// sign-extending loads. 1680b57cec5SDimitry Andricdefm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 1690b57cec5SDimitry Andricdefm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 1700b57cec5SDimitry Andricdefm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 1710b57cec5SDimitry Andricdefm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 1720b57cec5SDimitry Andricdefm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 1730b57cec5SDimitry Andric 1740b57cec5SDimitry Andric// Fragments for extending loads. These are different from regular loads because 1750b57cec5SDimitry Andric// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 1760b57cec5SDimitry Andric// therefore don't have the extension type field. So instead of matching that, 1770b57cec5SDimitry Andric// we match the patterns that the type legalizer expands them to. 1780b57cec5SDimitry Andric 1790b57cec5SDimitry Andric// We directly match zext patterns and select the zext atomic loads. 1800b57cec5SDimitry Andric// i32 (zext (i8 (atomic_load_8))) gets legalized to 1810b57cec5SDimitry Andric// i32 (and (i32 (atomic_load_8)), 255) 1820b57cec5SDimitry Andric// These can be selected to a single zero-extending atomic load instruction. 1830b57cec5SDimitry Andricdef zext_aload_8_32 : 1840b57cec5SDimitry Andric PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 1850b57cec5SDimitry Andricdef zext_aload_16_32 : 1860b57cec5SDimitry Andric PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 1870b57cec5SDimitry Andric// Unlike regular loads, extension to i64 is handled differently than i32. 1880b57cec5SDimitry Andric// i64 (zext (i8 (atomic_load_8))) gets legalized to 1890b57cec5SDimitry Andric// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 1900b57cec5SDimitry Andricdef zext_aload_8_64 : 1910b57cec5SDimitry Andric PatFrag<(ops node:$addr), 1920b57cec5SDimitry Andric (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 1930b57cec5SDimitry Andricdef zext_aload_16_64 : 1940b57cec5SDimitry Andric PatFrag<(ops node:$addr), 1950b57cec5SDimitry Andric (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 1960b57cec5SDimitry Andricdef zext_aload_32_64 : 1970b57cec5SDimitry Andric PatFrag<(ops node:$addr), 1980b57cec5SDimitry Andric (zext (i32 (atomic_load node:$addr)))>; 1990b57cec5SDimitry Andric 2000b57cec5SDimitry Andric// We don't have single sext atomic load instructions. So for sext loads, we 2010b57cec5SDimitry Andric// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 2020b57cec5SDimitry Andric// results) and select a zext load; the next instruction will be sext_inreg 2030b57cec5SDimitry Andric// which is selected by itself. 2040b57cec5SDimitry Andricdef sext_aload_8_64 : 2050b57cec5SDimitry Andric PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 2060b57cec5SDimitry Andricdef sext_aload_16_64 : 2070b57cec5SDimitry Andric PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 2080b57cec5SDimitry Andric 2090b57cec5SDimitry Andriclet Predicates = [HasAtomics] in { 2100b57cec5SDimitry Andric// Select zero-extending loads with no constant offset. 2110b57cec5SDimitry Andricdef : LoadPatNoOffset<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 2120b57cec5SDimitry Andricdef : LoadPatNoOffset<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 2130b57cec5SDimitry Andricdef : LoadPatNoOffset<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 2140b57cec5SDimitry Andricdef : LoadPatNoOffset<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 2150b57cec5SDimitry Andricdef : LoadPatNoOffset<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 2160b57cec5SDimitry Andric 2170b57cec5SDimitry Andric// Select sign-extending loads with no constant offset 2180b57cec5SDimitry Andricdef : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 2190b57cec5SDimitry Andricdef : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 2200b57cec5SDimitry Andricdef : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 2210b57cec5SDimitry Andricdef : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 2220b57cec5SDimitry Andric// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 2230b57cec5SDimitry Andric 2240b57cec5SDimitry Andric// Zero-extending loads with constant offset 2250b57cec5SDimitry Andricdef : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>; 2260b57cec5SDimitry Andricdef : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, ATOMIC_LOAD16_U_I32>; 2270b57cec5SDimitry Andricdef : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, ATOMIC_LOAD8_U_I32>; 2280b57cec5SDimitry Andricdef : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, ATOMIC_LOAD16_U_I32>; 2290b57cec5SDimitry Andricdef : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 2300b57cec5SDimitry Andricdef : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 2310b57cec5SDimitry Andricdef : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, ATOMIC_LOAD32_U_I64>; 2320b57cec5SDimitry Andricdef : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 2330b57cec5SDimitry Andricdef : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 2340b57cec5SDimitry Andricdef : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, ATOMIC_LOAD32_U_I64>; 2350b57cec5SDimitry Andric 2360b57cec5SDimitry Andric// Sign-extending loads with constant offset 2370b57cec5SDimitry Andricdef : LoadPatImmOff<i32, atomic_load_8, regPlusImm, ATOMIC_LOAD8_U_I32>; 2380b57cec5SDimitry Andricdef : LoadPatImmOff<i32, atomic_load_16, regPlusImm, ATOMIC_LOAD16_U_I32>; 2390b57cec5SDimitry Andricdef : LoadPatImmOff<i32, atomic_load_8, or_is_add, ATOMIC_LOAD8_U_I32>; 2400b57cec5SDimitry Andricdef : LoadPatImmOff<i32, atomic_load_16, or_is_add, ATOMIC_LOAD16_U_I32>; 2410b57cec5SDimitry Andricdef : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 2420b57cec5SDimitry Andricdef : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 2430b57cec5SDimitry Andricdef : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 2440b57cec5SDimitry Andricdef : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 2450b57cec5SDimitry Andric// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 2460b57cec5SDimitry Andric 2470b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 2480b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 2490b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 2500b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 2510b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 2520b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 2530b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 2540b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 2550b57cec5SDimitry Andricdef : LoadPatGlobalAddr<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 2560b57cec5SDimitry Andric 2570b57cec5SDimitry Andric// Extending loads with just a constant offset 2580b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 2590b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 2600b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 2610b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 2620b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 2630b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 2640b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 2650b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 2660b57cec5SDimitry Andricdef : LoadPatOffsetOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 2670b57cec5SDimitry Andric 2680b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 2690b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 2700b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 2710b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 2720b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 2730b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 2740b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 2750b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 2760b57cec5SDimitry Andricdef : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 2770b57cec5SDimitry Andric 2780b57cec5SDimitry Andric} // Predicates = [HasAtomics] 2790b57cec5SDimitry Andric 2800b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2810b57cec5SDimitry Andric// Atomic stores 2820b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2830b57cec5SDimitry Andric 2840b57cec5SDimitry Andricmulticlass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 2850b57cec5SDimitry Andric defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op))>, 2860b57cec5SDimitry Andric Requires<[HasAtomics]>; 2870b57cec5SDimitry Andric} 2880b57cec5SDimitry Andric 2890b57cec5SDimitry Andricdefm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 2900b57cec5SDimitry Andricdefm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 2910b57cec5SDimitry Andric 2920b57cec5SDimitry Andric// We need an 'atomic' version of store patterns because store and atomic_store 2930b57cec5SDimitry Andric// nodes have different operand orders: 2940b57cec5SDimitry Andric// store: (store $val, $ptr) 2950b57cec5SDimitry Andric// atomic_store: (store $ptr, $val) 2960b57cec5SDimitry Andric 2970b57cec5SDimitry Andriclet Predicates = [HasAtomics] in { 2980b57cec5SDimitry Andric 2990b57cec5SDimitry Andric// Select stores with no constant offset. 3000b57cec5SDimitry Andricclass AStorePatNoOffset<ValueType ty, PatFrag kind, NI inst> : 3010b57cec5SDimitry Andric Pat<(kind I32:$addr, ty:$val), (inst 0, 0, I32:$addr, ty:$val)>; 3020b57cec5SDimitry Andricdef : AStorePatNoOffset<i32, atomic_store_32, ATOMIC_STORE_I32>; 3030b57cec5SDimitry Andricdef : AStorePatNoOffset<i64, atomic_store_64, ATOMIC_STORE_I64>; 3040b57cec5SDimitry Andric 3050b57cec5SDimitry Andric// Select stores with a constant offset. 3060b57cec5SDimitry Andric 3070b57cec5SDimitry Andric// Pattern with address + immediate offset 3080b57cec5SDimitry Andricclass AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 3090b57cec5SDimitry Andric Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 3100b57cec5SDimitry Andric (inst 0, imm:$off, I32:$addr, ty:$val)>; 3110b57cec5SDimitry Andricdef : AStorePatImmOff<i32, atomic_store_32, regPlusImm, ATOMIC_STORE_I32>; 3120b57cec5SDimitry Andricdef : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>; 3130b57cec5SDimitry Andricdef : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>; 3140b57cec5SDimitry Andricdef : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>; 3150b57cec5SDimitry Andric 3160b57cec5SDimitry Andricclass AStorePatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 3170b57cec5SDimitry Andric Pat<(kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 3180b57cec5SDimitry Andric ty:$val), 3190b57cec5SDimitry Andric (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 3200b57cec5SDimitry Andricdef : AStorePatGlobalAddr<i32, atomic_store_32, ATOMIC_STORE_I32>; 3210b57cec5SDimitry Andricdef : AStorePatGlobalAddr<i64, atomic_store_64, ATOMIC_STORE_I64>; 3220b57cec5SDimitry Andric 3230b57cec5SDimitry Andric// Select stores with just a constant offset. 3240b57cec5SDimitry Andricclass AStorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 3250b57cec5SDimitry Andric Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 3260b57cec5SDimitry Andricdef : AStorePatOffsetOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 3270b57cec5SDimitry Andricdef : AStorePatOffsetOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 3280b57cec5SDimitry Andric 3290b57cec5SDimitry Andricclass AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 3300b57cec5SDimitry Andric Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 3310b57cec5SDimitry Andric (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 3320b57cec5SDimitry Andricdef : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 3330b57cec5SDimitry Andricdef : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 3340b57cec5SDimitry Andric 3350b57cec5SDimitry Andric} // Predicates = [HasAtomics] 3360b57cec5SDimitry Andric 3370b57cec5SDimitry Andric// Truncating stores. 3380b57cec5SDimitry Andricdefm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 3390b57cec5SDimitry Andricdefm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 3400b57cec5SDimitry Andricdefm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 3410b57cec5SDimitry Andricdefm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 3420b57cec5SDimitry Andricdefm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 3430b57cec5SDimitry Andric 3440b57cec5SDimitry Andric// Fragments for truncating stores. 3450b57cec5SDimitry Andric 3460b57cec5SDimitry Andric// We don't have single truncating atomic store instructions. For 32-bit 3470b57cec5SDimitry Andric// instructions, we just need to match bare atomic stores. On the other hand, 3480b57cec5SDimitry Andric// truncating stores from i64 values are once truncated to i32 first. 3490b57cec5SDimitry Andricclass trunc_astore_64<PatFrag kind> : 3500b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 3510b57cec5SDimitry Andric (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 3520b57cec5SDimitry Andricdef trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 3530b57cec5SDimitry Andricdef trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 3540b57cec5SDimitry Andricdef trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 3550b57cec5SDimitry Andric 3560b57cec5SDimitry Andriclet Predicates = [HasAtomics] in { 3570b57cec5SDimitry Andric 3580b57cec5SDimitry Andric// Truncating stores with no constant offset 3590b57cec5SDimitry Andricdef : AStorePatNoOffset<i32, atomic_store_8, ATOMIC_STORE8_I32>; 3600b57cec5SDimitry Andricdef : AStorePatNoOffset<i32, atomic_store_16, ATOMIC_STORE16_I32>; 3610b57cec5SDimitry Andricdef : AStorePatNoOffset<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 3620b57cec5SDimitry Andricdef : AStorePatNoOffset<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 3630b57cec5SDimitry Andricdef : AStorePatNoOffset<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 3640b57cec5SDimitry Andric 3650b57cec5SDimitry Andric// Truncating stores with a constant offset 3660b57cec5SDimitry Andricdef : AStorePatImmOff<i32, atomic_store_8, regPlusImm, ATOMIC_STORE8_I32>; 3670b57cec5SDimitry Andricdef : AStorePatImmOff<i32, atomic_store_16, regPlusImm, ATOMIC_STORE16_I32>; 3680b57cec5SDimitry Andricdef : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, ATOMIC_STORE8_I64>; 3690b57cec5SDimitry Andricdef : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, ATOMIC_STORE16_I64>; 3700b57cec5SDimitry Andricdef : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, ATOMIC_STORE32_I64>; 3710b57cec5SDimitry Andricdef : AStorePatImmOff<i32, atomic_store_8, or_is_add, ATOMIC_STORE8_I32>; 3720b57cec5SDimitry Andricdef : AStorePatImmOff<i32, atomic_store_16, or_is_add, ATOMIC_STORE16_I32>; 3730b57cec5SDimitry Andricdef : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, ATOMIC_STORE8_I64>; 3740b57cec5SDimitry Andricdef : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, ATOMIC_STORE16_I64>; 3750b57cec5SDimitry Andricdef : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, ATOMIC_STORE32_I64>; 3760b57cec5SDimitry Andric 3770b57cec5SDimitry Andricdef : AStorePatGlobalAddr<i32, atomic_store_8, ATOMIC_STORE8_I32>; 3780b57cec5SDimitry Andricdef : AStorePatGlobalAddr<i32, atomic_store_16, ATOMIC_STORE16_I32>; 3790b57cec5SDimitry Andricdef : AStorePatGlobalAddr<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 3800b57cec5SDimitry Andricdef : AStorePatGlobalAddr<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 3810b57cec5SDimitry Andricdef : AStorePatGlobalAddr<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 3820b57cec5SDimitry Andric 3830b57cec5SDimitry Andric// Truncating stores with just a constant offset 3840b57cec5SDimitry Andricdef : AStorePatOffsetOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 3850b57cec5SDimitry Andricdef : AStorePatOffsetOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 3860b57cec5SDimitry Andricdef : AStorePatOffsetOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 3870b57cec5SDimitry Andricdef : AStorePatOffsetOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 3880b57cec5SDimitry Andricdef : AStorePatOffsetOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 3890b57cec5SDimitry Andric 3900b57cec5SDimitry Andricdef : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 3910b57cec5SDimitry Andricdef : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 3920b57cec5SDimitry Andricdef : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 3930b57cec5SDimitry Andricdef : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 3940b57cec5SDimitry Andricdef : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 3950b57cec5SDimitry Andric 3960b57cec5SDimitry Andric} // Predicates = [HasAtomics] 3970b57cec5SDimitry Andric 3980b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 3990b57cec5SDimitry Andric// Atomic binary read-modify-writes 4000b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 4010b57cec5SDimitry Andric 4020b57cec5SDimitry Andricmulticlass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 4030b57cec5SDimitry Andric int atomic_op> { 4040b57cec5SDimitry Andric defm "" : 4050b57cec5SDimitry Andric ATOMIC_I<(outs rc:$dst), 4060b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 4070b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 4080b57cec5SDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 4090b57cec5SDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op>; 4100b57cec5SDimitry Andric} 4110b57cec5SDimitry Andric 4120b57cec5SDimitry Andricdefm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 4130b57cec5SDimitry Andricdefm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 4140b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_ADD_I32 : 4150b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 4160b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_ADD_I32 : 4170b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 4180b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_ADD_I64 : 4190b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 4200b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_ADD_I64 : 4210b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 4220b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_ADD_I64 : 4230b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 4240b57cec5SDimitry Andric 4250b57cec5SDimitry Andricdefm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 4260b57cec5SDimitry Andricdefm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 4270b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_SUB_I32 : 4280b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 4290b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_SUB_I32 : 4300b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 4310b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_SUB_I64 : 4320b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 4330b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_SUB_I64 : 4340b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 4350b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_SUB_I64 : 4360b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 4370b57cec5SDimitry Andric 4380b57cec5SDimitry Andricdefm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 4390b57cec5SDimitry Andricdefm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 4400b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_AND_I32 : 4410b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 4420b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_AND_I32 : 4430b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 4440b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_AND_I64 : 4450b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 4460b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_AND_I64 : 4470b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 4480b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_AND_I64 : 4490b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 4500b57cec5SDimitry Andric 4510b57cec5SDimitry Andricdefm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 4520b57cec5SDimitry Andricdefm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 4530b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_OR_I32 : 4540b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 4550b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_OR_I32 : 4560b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 4570b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_OR_I64 : 4580b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 4590b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_OR_I64 : 4600b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 4610b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_OR_I64 : 4620b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 4630b57cec5SDimitry Andric 4640b57cec5SDimitry Andricdefm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 4650b57cec5SDimitry Andricdefm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 4660b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XOR_I32 : 4670b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 4680b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XOR_I32 : 4690b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 4700b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XOR_I64 : 4710b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 4720b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XOR_I64 : 4730b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 4740b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_XOR_I64 : 4750b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 4760b57cec5SDimitry Andric 4770b57cec5SDimitry Andricdefm ATOMIC_RMW_XCHG_I32 : 4780b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 4790b57cec5SDimitry Andricdefm ATOMIC_RMW_XCHG_I64 : 4800b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 4810b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XCHG_I32 : 4820b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 4830b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XCHG_I32 : 4840b57cec5SDimitry Andric WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 4850b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_XCHG_I64 : 4860b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 4870b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_XCHG_I64 : 4880b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 4890b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_XCHG_I64 : 4900b57cec5SDimitry Andric WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 4910b57cec5SDimitry Andric 4920b57cec5SDimitry Andric// Select binary RMWs with no constant offset. 4930b57cec5SDimitry Andricclass BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 4940b57cec5SDimitry Andric Pat<(ty (kind I32:$addr, ty:$val)), (inst 0, 0, I32:$addr, ty:$val)>; 4950b57cec5SDimitry Andric 4960b57cec5SDimitry Andric// Select binary RMWs with a constant offset. 4970b57cec5SDimitry Andric 4980b57cec5SDimitry Andric// Pattern with address + immediate offset 4990b57cec5SDimitry Andricclass BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 5000b57cec5SDimitry Andric Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 5010b57cec5SDimitry Andric (inst 0, imm:$off, I32:$addr, ty:$val)>; 5020b57cec5SDimitry Andric 5030b57cec5SDimitry Andricclass BinRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 5040b57cec5SDimitry Andric Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 5050b57cec5SDimitry Andric ty:$val)), 5060b57cec5SDimitry Andric (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 5070b57cec5SDimitry Andric 5080b57cec5SDimitry Andric// Select binary RMWs with just a constant offset. 5090b57cec5SDimitry Andricclass BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 5100b57cec5SDimitry Andric Pat<(ty (kind imm:$off, ty:$val)), 5110b57cec5SDimitry Andric (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 5120b57cec5SDimitry Andric 5130b57cec5SDimitry Andricclass BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 5140b57cec5SDimitry Andric Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 5150b57cec5SDimitry Andric (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 5160b57cec5SDimitry Andric 5170b57cec5SDimitry Andric// Patterns for various addressing modes. 5180b57cec5SDimitry Andricmulticlass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 5190b57cec5SDimitry Andric NI inst_64> { 5200b57cec5SDimitry Andric def : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 5210b57cec5SDimitry Andric def : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 5220b57cec5SDimitry Andric 5230b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 5240b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 5250b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 5260b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 5270b57cec5SDimitry Andric 5280b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i32, rmw_32, inst_32>; 5290b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i64, rmw_64, inst_64>; 5300b57cec5SDimitry Andric 5310b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 5320b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 5330b57cec5SDimitry Andric 5340b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 5350b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 5360b57cec5SDimitry Andric} 5370b57cec5SDimitry Andric 5380b57cec5SDimitry Andriclet Predicates = [HasAtomics] in { 5390b57cec5SDimitry Andricdefm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, ATOMIC_RMW_ADD_I32, 5400b57cec5SDimitry Andric ATOMIC_RMW_ADD_I64>; 5410b57cec5SDimitry Andricdefm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, ATOMIC_RMW_SUB_I32, 5420b57cec5SDimitry Andric ATOMIC_RMW_SUB_I64>; 5430b57cec5SDimitry Andricdefm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, ATOMIC_RMW_AND_I32, 5440b57cec5SDimitry Andric ATOMIC_RMW_AND_I64>; 5450b57cec5SDimitry Andricdefm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, ATOMIC_RMW_OR_I32, 5460b57cec5SDimitry Andric ATOMIC_RMW_OR_I64>; 5470b57cec5SDimitry Andricdefm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, ATOMIC_RMW_XOR_I32, 5480b57cec5SDimitry Andric ATOMIC_RMW_XOR_I64>; 5490b57cec5SDimitry Andricdefm : BinRMWPattern<atomic_swap_32, atomic_swap_64, ATOMIC_RMW_XCHG_I32, 5500b57cec5SDimitry Andric ATOMIC_RMW_XCHG_I64>; 5510b57cec5SDimitry Andric} // Predicates = [HasAtomics] 5520b57cec5SDimitry Andric 5530b57cec5SDimitry Andric// Truncating & zero-extending binary RMW patterns. 5540b57cec5SDimitry Andric// These are combined patterns of truncating store patterns and zero-extending 5550b57cec5SDimitry Andric// load patterns above. 5560b57cec5SDimitry Andricclass zext_bin_rmw_8_32<PatFrag kind> : 5570b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 5580b57cec5SDimitry Andric (and (i32 (kind node:$addr, node:$val)), 255)>; 5590b57cec5SDimitry Andricclass zext_bin_rmw_16_32<PatFrag kind> : 5600b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 5610b57cec5SDimitry Andric (and (i32 (kind node:$addr, node:$val)), 65535)>; 5620b57cec5SDimitry Andricclass zext_bin_rmw_8_64<PatFrag kind> : 5630b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 5640b57cec5SDimitry Andric (and (i64 (anyext (i32 (kind node:$addr, 5650b57cec5SDimitry Andric (i32 (trunc (i64 node:$val))))))), 255)>; 5660b57cec5SDimitry Andricclass zext_bin_rmw_16_64<PatFrag kind> : 5670b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 5680b57cec5SDimitry Andric (and (i64 (anyext (i32 (kind node:$addr, 5690b57cec5SDimitry Andric (i32 (trunc (i64 node:$val))))))), 65535)>; 5700b57cec5SDimitry Andricclass zext_bin_rmw_32_64<PatFrag kind> : 5710b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 5720b57cec5SDimitry Andric (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 5730b57cec5SDimitry Andric 5740b57cec5SDimitry Andric// Truncating & sign-extending binary RMW patterns. 5750b57cec5SDimitry Andric// These are combined patterns of truncating store patterns and sign-extending 5760b57cec5SDimitry Andric// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 5770b57cec5SDimitry Andric// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 5780b57cec5SDimitry Andric// is selected by itself. 5790b57cec5SDimitry Andricclass sext_bin_rmw_8_32<PatFrag kind> : 5800b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 5810b57cec5SDimitry Andricclass sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 5820b57cec5SDimitry Andricclass sext_bin_rmw_8_64<PatFrag kind> : 5830b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$val), 5840b57cec5SDimitry Andric (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 5850b57cec5SDimitry Andricclass sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 5860b57cec5SDimitry Andric// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 5870b57cec5SDimitry Andric 5880b57cec5SDimitry Andric// Patterns for various addressing modes for truncating-extending binary RMWs. 5890b57cec5SDimitry Andricmulticlass BinRMWTruncExtPattern< 5900b57cec5SDimitry Andric PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 5910b57cec5SDimitry Andric NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 5920b57cec5SDimitry Andric // Truncating-extending binary RMWs with no constant offset 5930b57cec5SDimitry Andric def : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 5940b57cec5SDimitry Andric def : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 5950b57cec5SDimitry Andric def : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 5960b57cec5SDimitry Andric def : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 5970b57cec5SDimitry Andric def : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 5980b57cec5SDimitry Andric 5990b57cec5SDimitry Andric def : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 6000b57cec5SDimitry Andric def : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 6010b57cec5SDimitry Andric def : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 6020b57cec5SDimitry Andric def : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 6030b57cec5SDimitry Andric 6040b57cec5SDimitry Andric // Truncating-extending binary RMWs with a constant offset 6050b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 6060b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 6070b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 6080b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 6090b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 6100b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 6110b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 6120b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 6130b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 6140b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 6150b57cec5SDimitry Andric 6160b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 6170b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 6180b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 6190b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 6200b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 6210b57cec5SDimitry Andric def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 6220b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 6230b57cec5SDimitry Andric def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 6240b57cec5SDimitry Andric 6250b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 6260b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 6270b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 6280b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 6290b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 6300b57cec5SDimitry Andric 6310b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 6320b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 6330b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 6340b57cec5SDimitry Andric def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 6350b57cec5SDimitry Andric 6360b57cec5SDimitry Andric // Truncating-extending binary RMWs with just a constant offset 6370b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 6380b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 6390b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 6400b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 6410b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 6420b57cec5SDimitry Andric 6430b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 6440b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 6450b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 6460b57cec5SDimitry Andric def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 6470b57cec5SDimitry Andric 6480b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 6490b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 6500b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 6510b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 6520b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 6530b57cec5SDimitry Andric 6540b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 6550b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 6560b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 6570b57cec5SDimitry Andric def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 6580b57cec5SDimitry Andric} 6590b57cec5SDimitry Andric 6600b57cec5SDimitry Andriclet Predicates = [HasAtomics] in { 6610b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 6620b57cec5SDimitry Andric atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 6630b57cec5SDimitry Andric ATOMIC_RMW8_U_ADD_I32, ATOMIC_RMW16_U_ADD_I32, 6640b57cec5SDimitry Andric ATOMIC_RMW8_U_ADD_I64, ATOMIC_RMW16_U_ADD_I64, ATOMIC_RMW32_U_ADD_I64>; 6650b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 6660b57cec5SDimitry Andric atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 6670b57cec5SDimitry Andric ATOMIC_RMW8_U_SUB_I32, ATOMIC_RMW16_U_SUB_I32, 6680b57cec5SDimitry Andric ATOMIC_RMW8_U_SUB_I64, ATOMIC_RMW16_U_SUB_I64, ATOMIC_RMW32_U_SUB_I64>; 6690b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 6700b57cec5SDimitry Andric atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 6710b57cec5SDimitry Andric ATOMIC_RMW8_U_AND_I32, ATOMIC_RMW16_U_AND_I32, 6720b57cec5SDimitry Andric ATOMIC_RMW8_U_AND_I64, ATOMIC_RMW16_U_AND_I64, ATOMIC_RMW32_U_AND_I64>; 6730b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 6740b57cec5SDimitry Andric atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 6750b57cec5SDimitry Andric ATOMIC_RMW8_U_OR_I32, ATOMIC_RMW16_U_OR_I32, 6760b57cec5SDimitry Andric ATOMIC_RMW8_U_OR_I64, ATOMIC_RMW16_U_OR_I64, ATOMIC_RMW32_U_OR_I64>; 6770b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 6780b57cec5SDimitry Andric atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 6790b57cec5SDimitry Andric ATOMIC_RMW8_U_XOR_I32, ATOMIC_RMW16_U_XOR_I32, 6800b57cec5SDimitry Andric ATOMIC_RMW8_U_XOR_I64, ATOMIC_RMW16_U_XOR_I64, ATOMIC_RMW32_U_XOR_I64>; 6810b57cec5SDimitry Andricdefm : BinRMWTruncExtPattern< 6820b57cec5SDimitry Andric atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 6830b57cec5SDimitry Andric ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32, 6840b57cec5SDimitry Andric ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>; 6850b57cec5SDimitry Andric} // Predicates = [HasAtomics] 6860b57cec5SDimitry Andric 6870b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 6880b57cec5SDimitry Andric// Atomic ternary read-modify-writes 6890b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 6900b57cec5SDimitry Andric 6910b57cec5SDimitry Andric// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 6920b57cec5SDimitry Andric// flag}. When we use the success flag or both values, we can't make use of i64 6930b57cec5SDimitry Andric// truncate/extend versions of instructions for now, which is suboptimal. 6940b57cec5SDimitry Andric// Consider adding a pass after instruction selection that optimizes this case 6950b57cec5SDimitry Andric// if it is frequent. 6960b57cec5SDimitry Andric 6970b57cec5SDimitry Andricmulticlass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 6980b57cec5SDimitry Andric int atomic_op> { 6990b57cec5SDimitry Andric defm "" : 7000b57cec5SDimitry Andric ATOMIC_I<(outs rc:$dst), 7010b57cec5SDimitry Andric (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 7020b57cec5SDimitry Andric rc:$new_), 7030b57cec5SDimitry Andric (outs), (ins P2Align:$p2align, offset32_op:$off), [], 7040b57cec5SDimitry Andric !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 7050b57cec5SDimitry Andric !strconcat(name, "\t${off}${p2align}"), atomic_op>; 7060b57cec5SDimitry Andric} 7070b57cec5SDimitry Andric 7080b57cec5SDimitry Andricdefm ATOMIC_RMW_CMPXCHG_I32 : 7090b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 7100b57cec5SDimitry Andricdefm ATOMIC_RMW_CMPXCHG_I64 : 7110b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 7120b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_CMPXCHG_I32 : 7130b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 7140b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_CMPXCHG_I32 : 7150b57cec5SDimitry Andric WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 7160b57cec5SDimitry Andricdefm ATOMIC_RMW8_U_CMPXCHG_I64 : 7170b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 7180b57cec5SDimitry Andricdefm ATOMIC_RMW16_U_CMPXCHG_I64 : 7190b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 7200b57cec5SDimitry Andricdefm ATOMIC_RMW32_U_CMPXCHG_I64 : 7210b57cec5SDimitry Andric WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 7220b57cec5SDimitry Andric 7230b57cec5SDimitry Andric// Select ternary RMWs with no constant offset. 7240b57cec5SDimitry Andricclass TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 7250b57cec5SDimitry Andric Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 7260b57cec5SDimitry Andric (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>; 7270b57cec5SDimitry Andric 7280b57cec5SDimitry Andric// Select ternary RMWs with a constant offset. 7290b57cec5SDimitry Andric 7300b57cec5SDimitry Andric// Pattern with address + immediate offset 7310b57cec5SDimitry Andricclass TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 7320b57cec5SDimitry Andric Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 7330b57cec5SDimitry Andric (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>; 7340b57cec5SDimitry Andric 7350b57cec5SDimitry Andricclass TerRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 7360b57cec5SDimitry Andric Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 7370b57cec5SDimitry Andric ty:$exp, ty:$new)), 7380b57cec5SDimitry Andric (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, ty:$new)>; 7390b57cec5SDimitry Andric 7400b57cec5SDimitry Andric// Select ternary RMWs with just a constant offset. 7410b57cec5SDimitry Andricclass TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 7420b57cec5SDimitry Andric Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 7430b57cec5SDimitry Andric (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 7440b57cec5SDimitry Andric 7450b57cec5SDimitry Andricclass TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 7460b57cec5SDimitry Andric Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 7470b57cec5SDimitry Andric (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 7480b57cec5SDimitry Andric 7490b57cec5SDimitry Andric// Patterns for various addressing modes. 7500b57cec5SDimitry Andricmulticlass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 7510b57cec5SDimitry Andric NI inst_64> { 7520b57cec5SDimitry Andric def : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 7530b57cec5SDimitry Andric def : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 7540b57cec5SDimitry Andric 7550b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 7560b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 7570b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 7580b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 7590b57cec5SDimitry Andric 7600b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i32, rmw_32, inst_32>; 7610b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i64, rmw_64, inst_64>; 7620b57cec5SDimitry Andric 7630b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 7640b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 7650b57cec5SDimitry Andric 7660b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 7670b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 7680b57cec5SDimitry Andric} 7690b57cec5SDimitry Andric 7700b57cec5SDimitry Andriclet Predicates = [HasAtomics] in 7710b57cec5SDimitry Andricdefm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 7720b57cec5SDimitry Andric ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>; 7730b57cec5SDimitry Andric 7740b57cec5SDimitry Andric// Truncating & zero-extending ternary RMW patterns. 7750b57cec5SDimitry Andric// DAG legalization & optimization before instruction selection may introduce 7760b57cec5SDimitry Andric// additional nodes such as anyext or assertzext depending on operand types. 7770b57cec5SDimitry Andricclass zext_ter_rmw_8_32<PatFrag kind> : 7780b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 7790b57cec5SDimitry Andric (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 7800b57cec5SDimitry Andricclass zext_ter_rmw_16_32<PatFrag kind> : 7810b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 7820b57cec5SDimitry Andric (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 7830b57cec5SDimitry Andricclass zext_ter_rmw_8_64<PatFrag kind> : 7840b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 7850b57cec5SDimitry Andric (zext (i32 (assertzext (i32 (kind node:$addr, 7860b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 7870b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))))>; 7880b57cec5SDimitry Andricclass zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 7890b57cec5SDimitry Andricclass zext_ter_rmw_32_64<PatFrag kind> : 7900b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 7910b57cec5SDimitry Andric (zext (i32 (kind node:$addr, 7920b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 7930b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))>; 7940b57cec5SDimitry Andric 7950b57cec5SDimitry Andric// Truncating & sign-extending ternary RMW patterns. 7960b57cec5SDimitry Andric// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 7970b57cec5SDimitry Andric// zext RMW; the next instruction will be sext_inreg which is selected by 7980b57cec5SDimitry Andric// itself. 7990b57cec5SDimitry Andricclass sext_ter_rmw_8_32<PatFrag kind> : 8000b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 8010b57cec5SDimitry Andric (kind node:$addr, node:$exp, node:$new)>; 8020b57cec5SDimitry Andricclass sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 8030b57cec5SDimitry Andricclass sext_ter_rmw_8_64<PatFrag kind> : 8040b57cec5SDimitry Andric PatFrag<(ops node:$addr, node:$exp, node:$new), 8050b57cec5SDimitry Andric (anyext (i32 (assertzext (i32 8060b57cec5SDimitry Andric (kind node:$addr, 8070b57cec5SDimitry Andric (i32 (trunc (i64 node:$exp))), 8080b57cec5SDimitry Andric (i32 (trunc (i64 node:$new))))))))>; 8090b57cec5SDimitry Andricclass sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 8100b57cec5SDimitry Andric// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 8110b57cec5SDimitry Andric 8120b57cec5SDimitry Andric// Patterns for various addressing modes for truncating-extending ternary RMWs. 8130b57cec5SDimitry Andricmulticlass TerRMWTruncExtPattern< 8140b57cec5SDimitry Andric PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 8150b57cec5SDimitry Andric NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 8160b57cec5SDimitry Andric // Truncating-extending ternary RMWs with no constant offset 8170b57cec5SDimitry Andric def : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 8180b57cec5SDimitry Andric def : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 8190b57cec5SDimitry Andric def : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 8200b57cec5SDimitry Andric def : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 8210b57cec5SDimitry Andric def : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 8220b57cec5SDimitry Andric 8230b57cec5SDimitry Andric def : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 8240b57cec5SDimitry Andric def : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 8250b57cec5SDimitry Andric def : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 8260b57cec5SDimitry Andric def : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 8270b57cec5SDimitry Andric 8280b57cec5SDimitry Andric // Truncating-extending ternary RMWs with a constant offset 8290b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 8300b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 8310b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 8320b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 8330b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 8340b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 8350b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 8360b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 8370b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 8380b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 8390b57cec5SDimitry Andric 8400b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 8410b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 8420b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 8430b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 8440b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 8450b57cec5SDimitry Andric def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 8460b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 8470b57cec5SDimitry Andric def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 8480b57cec5SDimitry Andric 8490b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 8500b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 8510b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 8520b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 8530b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 8540b57cec5SDimitry Andric 8550b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 8560b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 8570b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 8580b57cec5SDimitry Andric def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 8590b57cec5SDimitry Andric 8600b57cec5SDimitry Andric // Truncating-extending ternary RMWs with just a constant offset 8610b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 8620b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 8630b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 8640b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 8650b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 8660b57cec5SDimitry Andric 8670b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 8680b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 8690b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 8700b57cec5SDimitry Andric def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 8710b57cec5SDimitry Andric 8720b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 8730b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 8740b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 8750b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 8760b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 8770b57cec5SDimitry Andric 8780b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 8790b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 8800b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 8810b57cec5SDimitry Andric def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 8820b57cec5SDimitry Andric} 8830b57cec5SDimitry Andric 8840b57cec5SDimitry Andriclet Predicates = [HasAtomics] in 8850b57cec5SDimitry Andricdefm : TerRMWTruncExtPattern< 8860b57cec5SDimitry Andric atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 8870b57cec5SDimitry Andric ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32, 8880b57cec5SDimitry Andric ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64, 8890b57cec5SDimitry Andric ATOMIC_RMW32_U_CMPXCHG_I64>; 8900b57cec5SDimitry Andric 8910b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 8920b57cec5SDimitry Andric// Atomic fences 8930b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 8940b57cec5SDimitry Andric 8950b57cec5SDimitry Andric// A compiler fence instruction that prevents reordering of instructions. 8960b57cec5SDimitry Andriclet Defs = [ARGUMENTS] in { 8970b57cec5SDimitry Andriclet isPseudo = 1, hasSideEffects = 1 in 8980b57cec5SDimitry Andricdefm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 8990b57cec5SDimitry Andric} // Defs = [ARGUMENTS] 900