1// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8/// 9/// \file 10/// WebAssembly Atomic operand code-gen constructs. 11/// 12//===----------------------------------------------------------------------===// 13 14let UseNamedOperandTable = 1 in 15multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 16 list<dag> pattern_r, string asmstr_r = "", 17 string asmstr_s = "", bits<32> atomic_op = -1> { 18 defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 19 !or(0xfe00, !and(0xff, atomic_op))>, 20 Requires<[HasAtomics]>; 21} 22 23multiclass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 24 bits<32> atomic_op = -1> { 25 defm "" : NRI<oops, iops, pattern, asmstr, 26 !or(0xfe00, !and(0xff, atomic_op))>, 27 Requires<[HasAtomics]>; 28} 29 30//===----------------------------------------------------------------------===// 31// Atomic wait / notify 32//===----------------------------------------------------------------------===// 33 34let hasSideEffects = 1 in { 35defm ATOMIC_NOTIFY : 36 ATOMIC_I<(outs I32:$dst), 37 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 38 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 39 "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 40 "atomic.notify \t${off}${p2align}", 0x00>; 41let mayLoad = 1 in { 42defm ATOMIC_WAIT_I32 : 43 ATOMIC_I<(outs I32:$dst), 44 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 45 I64:$timeout), 46 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 47 "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 48 "i32.atomic.wait \t${off}${p2align}", 0x01>; 49defm ATOMIC_WAIT_I64 : 50 ATOMIC_I<(outs I32:$dst), 51 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 52 I64:$timeout), 53 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54 "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 55 "i64.atomic.wait \t${off}${p2align}", 0x02>; 56} // mayLoad = 1 57} // hasSideEffects = 1 58 59let Predicates = [HasAtomics] in { 60// Select notifys with no constant offset. 61def NotifyPatNoOffset : 62 Pat<(i32 (int_wasm_atomic_notify I32:$addr, I32:$count)), 63 (ATOMIC_NOTIFY 0, 0, I32:$addr, I32:$count)>; 64 65// Select notifys with a constant offset. 66 67// Pattern with address + immediate offset 68class NotifyPatImmOff<PatFrag operand> : 69 Pat<(i32 (int_wasm_atomic_notify (operand I32:$addr, imm:$off), I32:$count)), 70 (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I32:$count)>; 71def : NotifyPatImmOff<regPlusImm>; 72def : NotifyPatImmOff<or_is_add>; 73 74def NotifyPatGlobalAddr : 75 Pat<(i32 (int_wasm_atomic_notify (regPlusGA I32:$addr, 76 (WebAssemblywrapper tglobaladdr:$off)), 77 I32:$count)), 78 (ATOMIC_NOTIFY 0, tglobaladdr:$off, I32:$addr, I32:$count)>; 79 80// Select notifys with just a constant offset. 81def NotifyPatOffsetOnly : 82 Pat<(i32 (int_wasm_atomic_notify imm:$off, I32:$count)), 83 (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I32:$count)>; 84 85def NotifyPatGlobalAddrOffOnly : 86 Pat<(i32 (int_wasm_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 87 I32:$count)), 88 (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>; 89 90// Select waits with no constant offset. 91class WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> : 92 Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 93 (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>; 94def : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 95def : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 96 97// Select waits with a constant offset. 98 99// Pattern with address + immediate offset 100class WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> : 101 Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 102 (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>; 103def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>; 104def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>; 105def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>; 106def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>; 107 108class WaitPatGlobalAddr<ValueType ty, Intrinsic kind, NI inst> : 109 Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 110 ty:$exp, I64:$timeout)), 111 (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, I64:$timeout)>; 112def : WaitPatGlobalAddr<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 113def : WaitPatGlobalAddr<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 114 115// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset. 116class WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> : 117 Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 118 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 119def : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 120def : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 121 122class WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> : 123 Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)), 124 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 125def : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 126def : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 127} // Predicates = [HasAtomics] 128 129//===----------------------------------------------------------------------===// 130// Atomic loads 131//===----------------------------------------------------------------------===// 132 133multiclass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 134 defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op))>, 135 Requires<[HasAtomics]>; 136} 137 138defm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 139defm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 140 141// Select loads with no constant offset. 142let Predicates = [HasAtomics] in { 143def : LoadPatNoOffset<i32, atomic_load_32, ATOMIC_LOAD_I32>; 144def : LoadPatNoOffset<i64, atomic_load_64, ATOMIC_LOAD_I64>; 145 146// Select loads with a constant offset. 147 148// Pattern with address + immediate offset 149def : LoadPatImmOff<i32, atomic_load_32, regPlusImm, ATOMIC_LOAD_I32>; 150def : LoadPatImmOff<i64, atomic_load_64, regPlusImm, ATOMIC_LOAD_I64>; 151def : LoadPatImmOff<i32, atomic_load_32, or_is_add, ATOMIC_LOAD_I32>; 152def : LoadPatImmOff<i64, atomic_load_64, or_is_add, ATOMIC_LOAD_I64>; 153 154def : LoadPatGlobalAddr<i32, atomic_load_32, ATOMIC_LOAD_I32>; 155def : LoadPatGlobalAddr<i64, atomic_load_64, ATOMIC_LOAD_I64>; 156 157// Select loads with just a constant offset. 158def : LoadPatOffsetOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 159def : LoadPatOffsetOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 160 161def : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 162def : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 163 164} // Predicates = [HasAtomics] 165 166// Extending loads. Note that there are only zero-extending atomic loads, no 167// sign-extending loads. 168defm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 169defm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 170defm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 171defm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 172defm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 173 174// Fragments for extending loads. These are different from regular loads because 175// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 176// therefore don't have the extension type field. So instead of matching that, 177// we match the patterns that the type legalizer expands them to. 178 179// We directly match zext patterns and select the zext atomic loads. 180// i32 (zext (i8 (atomic_load_8))) gets legalized to 181// i32 (and (i32 (atomic_load_8)), 255) 182// These can be selected to a single zero-extending atomic load instruction. 183def zext_aload_8_32 : 184 PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 185def zext_aload_16_32 : 186 PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 187// Unlike regular loads, extension to i64 is handled differently than i32. 188// i64 (zext (i8 (atomic_load_8))) gets legalized to 189// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 190def zext_aload_8_64 : 191 PatFrag<(ops node:$addr), 192 (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 193def zext_aload_16_64 : 194 PatFrag<(ops node:$addr), 195 (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 196def zext_aload_32_64 : 197 PatFrag<(ops node:$addr), 198 (zext (i32 (atomic_load node:$addr)))>; 199 200// We don't have single sext atomic load instructions. So for sext loads, we 201// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 202// results) and select a zext load; the next instruction will be sext_inreg 203// which is selected by itself. 204def sext_aload_8_64 : 205 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 206def sext_aload_16_64 : 207 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 208 209let Predicates = [HasAtomics] in { 210// Select zero-extending loads with no constant offset. 211def : LoadPatNoOffset<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 212def : LoadPatNoOffset<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 213def : LoadPatNoOffset<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 214def : LoadPatNoOffset<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 215def : LoadPatNoOffset<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 216 217// Select sign-extending loads with no constant offset 218def : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 219def : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 220def : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 221def : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 222// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 223 224// Zero-extending loads with constant offset 225def : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>; 226def : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, ATOMIC_LOAD16_U_I32>; 227def : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, ATOMIC_LOAD8_U_I32>; 228def : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, ATOMIC_LOAD16_U_I32>; 229def : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 230def : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 231def : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, ATOMIC_LOAD32_U_I64>; 232def : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 233def : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 234def : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, ATOMIC_LOAD32_U_I64>; 235 236// Sign-extending loads with constant offset 237def : LoadPatImmOff<i32, atomic_load_8, regPlusImm, ATOMIC_LOAD8_U_I32>; 238def : LoadPatImmOff<i32, atomic_load_16, regPlusImm, ATOMIC_LOAD16_U_I32>; 239def : LoadPatImmOff<i32, atomic_load_8, or_is_add, ATOMIC_LOAD8_U_I32>; 240def : LoadPatImmOff<i32, atomic_load_16, or_is_add, ATOMIC_LOAD16_U_I32>; 241def : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 242def : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 243def : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 244def : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 245// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 246 247def : LoadPatGlobalAddr<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 248def : LoadPatGlobalAddr<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 249def : LoadPatGlobalAddr<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 250def : LoadPatGlobalAddr<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 251def : LoadPatGlobalAddr<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 252def : LoadPatGlobalAddr<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 253def : LoadPatGlobalAddr<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 254def : LoadPatGlobalAddr<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 255def : LoadPatGlobalAddr<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 256 257// Extending loads with just a constant offset 258def : LoadPatOffsetOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 259def : LoadPatOffsetOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 260def : LoadPatOffsetOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 261def : LoadPatOffsetOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 262def : LoadPatOffsetOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 263def : LoadPatOffsetOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 264def : LoadPatOffsetOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 265def : LoadPatOffsetOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 266def : LoadPatOffsetOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 267 268def : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 269def : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 270def : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 271def : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 272def : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 273def : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 274def : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 275def : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 276def : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 277 278} // Predicates = [HasAtomics] 279 280//===----------------------------------------------------------------------===// 281// Atomic stores 282//===----------------------------------------------------------------------===// 283 284multiclass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 285 defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op))>, 286 Requires<[HasAtomics]>; 287} 288 289defm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 290defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 291 292// We need an 'atomic' version of store patterns because store and atomic_store 293// nodes have different operand orders: 294// store: (store $val, $ptr) 295// atomic_store: (store $ptr, $val) 296 297let Predicates = [HasAtomics] in { 298 299// Select stores with no constant offset. 300class AStorePatNoOffset<ValueType ty, PatFrag kind, NI inst> : 301 Pat<(kind I32:$addr, ty:$val), (inst 0, 0, I32:$addr, ty:$val)>; 302def : AStorePatNoOffset<i32, atomic_store_32, ATOMIC_STORE_I32>; 303def : AStorePatNoOffset<i64, atomic_store_64, ATOMIC_STORE_I64>; 304 305// Select stores with a constant offset. 306 307// Pattern with address + immediate offset 308class AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 309 Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 310 (inst 0, imm:$off, I32:$addr, ty:$val)>; 311def : AStorePatImmOff<i32, atomic_store_32, regPlusImm, ATOMIC_STORE_I32>; 312def : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>; 313def : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>; 314def : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>; 315 316class AStorePatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 317 Pat<(kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 318 ty:$val), 319 (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 320def : AStorePatGlobalAddr<i32, atomic_store_32, ATOMIC_STORE_I32>; 321def : AStorePatGlobalAddr<i64, atomic_store_64, ATOMIC_STORE_I64>; 322 323// Select stores with just a constant offset. 324class AStorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 325 Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 326def : AStorePatOffsetOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 327def : AStorePatOffsetOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 328 329class AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 330 Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 331 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 332def : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 333def : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 334 335} // Predicates = [HasAtomics] 336 337// Truncating stores. 338defm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 339defm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 340defm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 341defm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 342defm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 343 344// Fragments for truncating stores. 345 346// We don't have single truncating atomic store instructions. For 32-bit 347// instructions, we just need to match bare atomic stores. On the other hand, 348// truncating stores from i64 values are once truncated to i32 first. 349class trunc_astore_64<PatFrag kind> : 350 PatFrag<(ops node:$addr, node:$val), 351 (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 352def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 353def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 354def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 355 356let Predicates = [HasAtomics] in { 357 358// Truncating stores with no constant offset 359def : AStorePatNoOffset<i32, atomic_store_8, ATOMIC_STORE8_I32>; 360def : AStorePatNoOffset<i32, atomic_store_16, ATOMIC_STORE16_I32>; 361def : AStorePatNoOffset<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 362def : AStorePatNoOffset<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 363def : AStorePatNoOffset<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 364 365// Truncating stores with a constant offset 366def : AStorePatImmOff<i32, atomic_store_8, regPlusImm, ATOMIC_STORE8_I32>; 367def : AStorePatImmOff<i32, atomic_store_16, regPlusImm, ATOMIC_STORE16_I32>; 368def : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, ATOMIC_STORE8_I64>; 369def : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, ATOMIC_STORE16_I64>; 370def : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, ATOMIC_STORE32_I64>; 371def : AStorePatImmOff<i32, atomic_store_8, or_is_add, ATOMIC_STORE8_I32>; 372def : AStorePatImmOff<i32, atomic_store_16, or_is_add, ATOMIC_STORE16_I32>; 373def : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, ATOMIC_STORE8_I64>; 374def : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, ATOMIC_STORE16_I64>; 375def : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, ATOMIC_STORE32_I64>; 376 377def : AStorePatGlobalAddr<i32, atomic_store_8, ATOMIC_STORE8_I32>; 378def : AStorePatGlobalAddr<i32, atomic_store_16, ATOMIC_STORE16_I32>; 379def : AStorePatGlobalAddr<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 380def : AStorePatGlobalAddr<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 381def : AStorePatGlobalAddr<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 382 383// Truncating stores with just a constant offset 384def : AStorePatOffsetOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 385def : AStorePatOffsetOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 386def : AStorePatOffsetOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 387def : AStorePatOffsetOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 388def : AStorePatOffsetOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 389 390def : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 391def : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 392def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 393def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 394def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 395 396} // Predicates = [HasAtomics] 397 398//===----------------------------------------------------------------------===// 399// Atomic binary read-modify-writes 400//===----------------------------------------------------------------------===// 401 402multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 403 int atomic_op> { 404 defm "" : 405 ATOMIC_I<(outs rc:$dst), 406 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 407 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 408 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 409 !strconcat(name, "\t${off}${p2align}"), atomic_op>; 410} 411 412defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 413defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 414defm ATOMIC_RMW8_U_ADD_I32 : 415 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 416defm ATOMIC_RMW16_U_ADD_I32 : 417 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 418defm ATOMIC_RMW8_U_ADD_I64 : 419 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 420defm ATOMIC_RMW16_U_ADD_I64 : 421 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 422defm ATOMIC_RMW32_U_ADD_I64 : 423 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 424 425defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 426defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 427defm ATOMIC_RMW8_U_SUB_I32 : 428 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 429defm ATOMIC_RMW16_U_SUB_I32 : 430 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 431defm ATOMIC_RMW8_U_SUB_I64 : 432 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 433defm ATOMIC_RMW16_U_SUB_I64 : 434 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 435defm ATOMIC_RMW32_U_SUB_I64 : 436 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 437 438defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 439defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 440defm ATOMIC_RMW8_U_AND_I32 : 441 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 442defm ATOMIC_RMW16_U_AND_I32 : 443 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 444defm ATOMIC_RMW8_U_AND_I64 : 445 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 446defm ATOMIC_RMW16_U_AND_I64 : 447 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 448defm ATOMIC_RMW32_U_AND_I64 : 449 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 450 451defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 452defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 453defm ATOMIC_RMW8_U_OR_I32 : 454 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 455defm ATOMIC_RMW16_U_OR_I32 : 456 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 457defm ATOMIC_RMW8_U_OR_I64 : 458 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 459defm ATOMIC_RMW16_U_OR_I64 : 460 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 461defm ATOMIC_RMW32_U_OR_I64 : 462 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 463 464defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 465defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 466defm ATOMIC_RMW8_U_XOR_I32 : 467 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 468defm ATOMIC_RMW16_U_XOR_I32 : 469 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 470defm ATOMIC_RMW8_U_XOR_I64 : 471 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 472defm ATOMIC_RMW16_U_XOR_I64 : 473 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 474defm ATOMIC_RMW32_U_XOR_I64 : 475 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 476 477defm ATOMIC_RMW_XCHG_I32 : 478 WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 479defm ATOMIC_RMW_XCHG_I64 : 480 WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 481defm ATOMIC_RMW8_U_XCHG_I32 : 482 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 483defm ATOMIC_RMW16_U_XCHG_I32 : 484 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 485defm ATOMIC_RMW8_U_XCHG_I64 : 486 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 487defm ATOMIC_RMW16_U_XCHG_I64 : 488 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 489defm ATOMIC_RMW32_U_XCHG_I64 : 490 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 491 492// Select binary RMWs with no constant offset. 493class BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 494 Pat<(ty (kind I32:$addr, ty:$val)), (inst 0, 0, I32:$addr, ty:$val)>; 495 496// Select binary RMWs with a constant offset. 497 498// Pattern with address + immediate offset 499class BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 500 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 501 (inst 0, imm:$off, I32:$addr, ty:$val)>; 502 503class BinRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 504 Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 505 ty:$val)), 506 (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 507 508// Select binary RMWs with just a constant offset. 509class BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 510 Pat<(ty (kind imm:$off, ty:$val)), 511 (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 512 513class BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 514 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 515 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 516 517// Patterns for various addressing modes. 518multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 519 NI inst_64> { 520 def : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 521 def : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 522 523 def : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 524 def : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 525 def : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 526 def : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 527 528 def : BinRMWPatGlobalAddr<i32, rmw_32, inst_32>; 529 def : BinRMWPatGlobalAddr<i64, rmw_64, inst_64>; 530 531 def : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 532 def : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 533 534 def : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 535 def : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 536} 537 538let Predicates = [HasAtomics] in { 539defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, ATOMIC_RMW_ADD_I32, 540 ATOMIC_RMW_ADD_I64>; 541defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, ATOMIC_RMW_SUB_I32, 542 ATOMIC_RMW_SUB_I64>; 543defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, ATOMIC_RMW_AND_I32, 544 ATOMIC_RMW_AND_I64>; 545defm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, ATOMIC_RMW_OR_I32, 546 ATOMIC_RMW_OR_I64>; 547defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, ATOMIC_RMW_XOR_I32, 548 ATOMIC_RMW_XOR_I64>; 549defm : BinRMWPattern<atomic_swap_32, atomic_swap_64, ATOMIC_RMW_XCHG_I32, 550 ATOMIC_RMW_XCHG_I64>; 551} // Predicates = [HasAtomics] 552 553// Truncating & zero-extending binary RMW patterns. 554// These are combined patterns of truncating store patterns and zero-extending 555// load patterns above. 556class zext_bin_rmw_8_32<PatFrag kind> : 557 PatFrag<(ops node:$addr, node:$val), 558 (and (i32 (kind node:$addr, node:$val)), 255)>; 559class zext_bin_rmw_16_32<PatFrag kind> : 560 PatFrag<(ops node:$addr, node:$val), 561 (and (i32 (kind node:$addr, node:$val)), 65535)>; 562class zext_bin_rmw_8_64<PatFrag kind> : 563 PatFrag<(ops node:$addr, node:$val), 564 (and (i64 (anyext (i32 (kind node:$addr, 565 (i32 (trunc (i64 node:$val))))))), 255)>; 566class zext_bin_rmw_16_64<PatFrag kind> : 567 PatFrag<(ops node:$addr, node:$val), 568 (and (i64 (anyext (i32 (kind node:$addr, 569 (i32 (trunc (i64 node:$val))))))), 65535)>; 570class zext_bin_rmw_32_64<PatFrag kind> : 571 PatFrag<(ops node:$addr, node:$val), 572 (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 573 574// Truncating & sign-extending binary RMW patterns. 575// These are combined patterns of truncating store patterns and sign-extending 576// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 577// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 578// is selected by itself. 579class sext_bin_rmw_8_32<PatFrag kind> : 580 PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 581class sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 582class sext_bin_rmw_8_64<PatFrag kind> : 583 PatFrag<(ops node:$addr, node:$val), 584 (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 585class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 586// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 587 588// Patterns for various addressing modes for truncating-extending binary RMWs. 589multiclass BinRMWTruncExtPattern< 590 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 591 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 592 // Truncating-extending binary RMWs with no constant offset 593 def : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 594 def : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 595 def : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 596 def : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 597 def : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 598 599 def : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 600 def : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 601 def : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 602 def : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 603 604 // Truncating-extending binary RMWs with a constant offset 605 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 606 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 607 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 608 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 609 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 610 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 611 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 612 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 613 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 614 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 615 616 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 617 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 618 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 619 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 620 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 621 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 622 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 623 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 624 625 def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 626 def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 627 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 628 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 629 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 630 631 def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 632 def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 633 def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 634 def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 635 636 // Truncating-extending binary RMWs with just a constant offset 637 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 638 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 639 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 640 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 641 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 642 643 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 644 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 645 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 646 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 647 648 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 649 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 650 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 651 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 652 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 653 654 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 655 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 656 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 657 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 658} 659 660let Predicates = [HasAtomics] in { 661defm : BinRMWTruncExtPattern< 662 atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 663 ATOMIC_RMW8_U_ADD_I32, ATOMIC_RMW16_U_ADD_I32, 664 ATOMIC_RMW8_U_ADD_I64, ATOMIC_RMW16_U_ADD_I64, ATOMIC_RMW32_U_ADD_I64>; 665defm : BinRMWTruncExtPattern< 666 atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 667 ATOMIC_RMW8_U_SUB_I32, ATOMIC_RMW16_U_SUB_I32, 668 ATOMIC_RMW8_U_SUB_I64, ATOMIC_RMW16_U_SUB_I64, ATOMIC_RMW32_U_SUB_I64>; 669defm : BinRMWTruncExtPattern< 670 atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 671 ATOMIC_RMW8_U_AND_I32, ATOMIC_RMW16_U_AND_I32, 672 ATOMIC_RMW8_U_AND_I64, ATOMIC_RMW16_U_AND_I64, ATOMIC_RMW32_U_AND_I64>; 673defm : BinRMWTruncExtPattern< 674 atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 675 ATOMIC_RMW8_U_OR_I32, ATOMIC_RMW16_U_OR_I32, 676 ATOMIC_RMW8_U_OR_I64, ATOMIC_RMW16_U_OR_I64, ATOMIC_RMW32_U_OR_I64>; 677defm : BinRMWTruncExtPattern< 678 atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 679 ATOMIC_RMW8_U_XOR_I32, ATOMIC_RMW16_U_XOR_I32, 680 ATOMIC_RMW8_U_XOR_I64, ATOMIC_RMW16_U_XOR_I64, ATOMIC_RMW32_U_XOR_I64>; 681defm : BinRMWTruncExtPattern< 682 atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 683 ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32, 684 ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>; 685} // Predicates = [HasAtomics] 686 687//===----------------------------------------------------------------------===// 688// Atomic ternary read-modify-writes 689//===----------------------------------------------------------------------===// 690 691// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 692// flag}. When we use the success flag or both values, we can't make use of i64 693// truncate/extend versions of instructions for now, which is suboptimal. 694// Consider adding a pass after instruction selection that optimizes this case 695// if it is frequent. 696 697multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 698 int atomic_op> { 699 defm "" : 700 ATOMIC_I<(outs rc:$dst), 701 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 702 rc:$new_), 703 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 704 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 705 !strconcat(name, "\t${off}${p2align}"), atomic_op>; 706} 707 708defm ATOMIC_RMW_CMPXCHG_I32 : 709 WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 710defm ATOMIC_RMW_CMPXCHG_I64 : 711 WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 712defm ATOMIC_RMW8_U_CMPXCHG_I32 : 713 WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 714defm ATOMIC_RMW16_U_CMPXCHG_I32 : 715 WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 716defm ATOMIC_RMW8_U_CMPXCHG_I64 : 717 WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 718defm ATOMIC_RMW16_U_CMPXCHG_I64 : 719 WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 720defm ATOMIC_RMW32_U_CMPXCHG_I64 : 721 WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 722 723// Select ternary RMWs with no constant offset. 724class TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 725 Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 726 (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>; 727 728// Select ternary RMWs with a constant offset. 729 730// Pattern with address + immediate offset 731class TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 732 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 733 (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>; 734 735class TerRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 736 Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 737 ty:$exp, ty:$new)), 738 (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, ty:$new)>; 739 740// Select ternary RMWs with just a constant offset. 741class TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 742 Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 743 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 744 745class TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 746 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 747 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 748 749// Patterns for various addressing modes. 750multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 751 NI inst_64> { 752 def : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 753 def : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 754 755 def : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 756 def : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 757 def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 758 def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 759 760 def : TerRMWPatGlobalAddr<i32, rmw_32, inst_32>; 761 def : TerRMWPatGlobalAddr<i64, rmw_64, inst_64>; 762 763 def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 764 def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 765 766 def : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 767 def : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 768} 769 770let Predicates = [HasAtomics] in 771defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 772 ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>; 773 774// Truncating & zero-extending ternary RMW patterns. 775// DAG legalization & optimization before instruction selection may introduce 776// additional nodes such as anyext or assertzext depending on operand types. 777class zext_ter_rmw_8_32<PatFrag kind> : 778 PatFrag<(ops node:$addr, node:$exp, node:$new), 779 (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 780class zext_ter_rmw_16_32<PatFrag kind> : 781 PatFrag<(ops node:$addr, node:$exp, node:$new), 782 (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 783class zext_ter_rmw_8_64<PatFrag kind> : 784 PatFrag<(ops node:$addr, node:$exp, node:$new), 785 (zext (i32 (assertzext (i32 (kind node:$addr, 786 (i32 (trunc (i64 node:$exp))), 787 (i32 (trunc (i64 node:$new))))))))>; 788class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 789class zext_ter_rmw_32_64<PatFrag kind> : 790 PatFrag<(ops node:$addr, node:$exp, node:$new), 791 (zext (i32 (kind node:$addr, 792 (i32 (trunc (i64 node:$exp))), 793 (i32 (trunc (i64 node:$new))))))>; 794 795// Truncating & sign-extending ternary RMW patterns. 796// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 797// zext RMW; the next instruction will be sext_inreg which is selected by 798// itself. 799class sext_ter_rmw_8_32<PatFrag kind> : 800 PatFrag<(ops node:$addr, node:$exp, node:$new), 801 (kind node:$addr, node:$exp, node:$new)>; 802class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 803class sext_ter_rmw_8_64<PatFrag kind> : 804 PatFrag<(ops node:$addr, node:$exp, node:$new), 805 (anyext (i32 (assertzext (i32 806 (kind node:$addr, 807 (i32 (trunc (i64 node:$exp))), 808 (i32 (trunc (i64 node:$new))))))))>; 809class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 810// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 811 812// Patterns for various addressing modes for truncating-extending ternary RMWs. 813multiclass TerRMWTruncExtPattern< 814 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 815 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 816 // Truncating-extending ternary RMWs with no constant offset 817 def : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 818 def : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 819 def : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 820 def : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 821 def : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 822 823 def : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 824 def : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 825 def : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 826 def : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 827 828 // Truncating-extending ternary RMWs with a constant offset 829 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 830 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 831 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 832 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 833 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 834 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 835 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 836 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 837 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 838 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 839 840 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 841 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 842 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 843 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 844 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 845 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 846 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 847 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 848 849 def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 850 def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 851 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 852 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 853 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 854 855 def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 856 def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 857 def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 858 def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 859 860 // Truncating-extending ternary RMWs with just a constant offset 861 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 862 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 863 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 864 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 865 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 866 867 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 868 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 869 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 870 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 871 872 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 873 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 874 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 875 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 876 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 877 878 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 879 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 880 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 881 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 882} 883 884let Predicates = [HasAtomics] in 885defm : TerRMWTruncExtPattern< 886 atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 887 ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32, 888 ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64, 889 ATOMIC_RMW32_U_CMPXCHG_I64>; 890 891//===----------------------------------------------------------------------===// 892// Atomic fences 893//===----------------------------------------------------------------------===// 894 895// A compiler fence instruction that prevents reordering of instructions. 896let Defs = [ARGUMENTS] in { 897let isPseudo = 1, hasSideEffects = 1 in 898defm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 899} // Defs = [ARGUMENTS] 900