1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9def smrd_offset_8 : ImmOperand<i32, "SMRDOffset8", 1>;
10
11let EncoderMethod = "getSMEMOffsetEncoding",
12    DecoderMethod = "decodeSMEMOffset" in {
13def smem_offset : ImmOperand<i32, "SMEMOffset", 1>;
14def smem_offset_mod : NamedIntOperand<i32, "offset", "SMEMOffsetMod">;
15}
16
17//===----------------------------------------------------------------------===//
18// Scalar Memory classes
19//===----------------------------------------------------------------------===//
20
21class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
22  InstSI <outs, ins, "", pattern>,
23  SIMCInstr<opName, SIEncodingFamily.NONE> {
24  let isPseudo = 1;
25  let isCodeGenOnly = 1;
26
27  let LGKM_CNT = 1;
28  let SMRD = 1;
29  let mayStore = 0;
30  let mayLoad = 1;
31  let hasSideEffects = 0;
32  let UseNamedOperandTable = 1;
33  let SchedRW = [WriteSMEM];
34
35  string Mnemonic = opName;
36  string AsmOperands = asmOps;
37
38  bits<1> has_sbase = 1;
39  bits<1> has_sdst = 1;
40  bit has_glc = 0;
41  bit has_dlc = 0;
42  bit has_offset = 0;
43  bit has_soffset = 0;
44  bit is_buffer = 0;
45}
46
47class SM_Real <SM_Pseudo ps, string opName = ps.Mnemonic>
48  : InstSI<ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands> {
49
50  let isPseudo = 0;
51  let isCodeGenOnly = 0;
52
53  Instruction Opcode = !cast<Instruction>(NAME);
54
55  // copy relevant pseudo op flags
56  let LGKM_CNT             = ps.LGKM_CNT;
57  let SMRD                 = ps.SMRD;
58  let mayStore             = ps.mayStore;
59  let mayLoad              = ps.mayLoad;
60  let hasSideEffects       = ps.hasSideEffects;
61  let UseNamedOperandTable = ps.UseNamedOperandTable;
62  let SchedRW              = ps.SchedRW;
63  let SubtargetPredicate   = ps.SubtargetPredicate;
64  let AsmMatchConverter    = ps.AsmMatchConverter;
65  let IsAtomicRet          = ps.IsAtomicRet;
66  let IsAtomicNoRet        = ps.IsAtomicNoRet;
67
68  let TSFlags = ps.TSFlags;
69
70  bit is_buffer = ps.is_buffer;
71
72  // encoding
73  bits<7>  sbase;
74  bits<7>  sdst;
75  bits<32> offset;
76  bits<8>  soffset;
77  bits<5> cpol;
78}
79
80class OffsetMode<bit hasOffset, bit hasSOffset, string variant,
81                 dag ins, string asm> {
82  bit HasOffset = hasOffset;
83  bit HasSOffset = hasSOffset;
84  string Variant = variant;
85  dag Ins = ins;
86  string Asm = asm;
87}
88
89def IMM_Offset : OffsetMode<1, 0, "_IMM", (ins smem_offset:$offset), "$offset">;
90def SGPR_Offset : OffsetMode<0, 1, "_SGPR", (ins SReg_32:$soffset), "$soffset">;
91def SGPR_IMM_Offset : OffsetMode<1, 1, "_SGPR_IMM",
92                                 (ins SReg_32:$soffset, smem_offset_mod:$offset),
93                                 "$soffset$offset">;
94
95class SM_Probe_Pseudo <string opName, RegisterClass baseClass, OffsetMode offsets>
96  : SM_Pseudo<opName, (outs),
97              !con((ins i8imm:$sdata, baseClass:$sbase), offsets.Ins),
98              " $sdata, $sbase, " # offsets.Asm> {
99  let mayLoad = 0;
100  let mayStore = 0;
101  let has_glc = 0;
102  let LGKM_CNT = 0;
103  let ScalarStore = 0;
104  let hasSideEffects = 1;
105  let has_offset = offsets.HasOffset;
106  let has_soffset = offsets.HasSOffset;
107  let PseudoInstr = opName # offsets.Variant;
108}
109
110class SM_Load_Pseudo <string opName, RegisterClass baseClass,
111                      RegisterClass dstClass, OffsetMode offsets>
112  : SM_Pseudo<opName, (outs dstClass:$sdst),
113              !con((ins baseClass:$sbase), offsets.Ins, (ins CPol:$cpol)),
114              " $sdst, $sbase, " # offsets.Asm # "$cpol", []> {
115  RegisterClass BaseClass = baseClass;
116  let mayLoad = 1;
117  let isReMaterializable = 1;
118  let mayStore = 0;
119  let has_glc = 1;
120  let has_dlc = 1;
121  let has_offset = offsets.HasOffset;
122  let has_soffset = offsets.HasSOffset;
123  let PseudoInstr = opName # offsets.Variant;
124}
125
126class SM_Store_Pseudo <string opName, RegisterClass baseClass,
127                       RegisterClass srcClass, OffsetMode offsets>
128  : SM_Pseudo<opName, (outs), !con((ins srcClass:$sdata, baseClass:$sbase),
129                                   offsets.Ins, (ins CPol:$cpol)),
130              " $sdata, $sbase, " # offsets.Asm # "$cpol"> {
131  RegisterClass BaseClass = baseClass;
132  let mayLoad = 0;
133  let mayStore = 1;
134  let has_glc = 1;
135  let has_dlc = 1;
136  let has_offset = offsets.HasOffset;
137  let has_soffset = offsets.HasSOffset;
138  let ScalarStore = 1;
139  let PseudoInstr = opName # offsets.Variant;
140}
141
142class SM_Discard_Pseudo <string opName, OffsetMode offsets>
143  : SM_Pseudo<opName, (outs), !con((ins SReg_64:$sbase), offsets.Ins),
144              " $sbase, " # offsets.Asm> {
145  let mayLoad = 0;
146  let mayStore = 0;
147  let has_glc = 0;
148  let has_sdst = 0;
149  let ScalarStore = 0;
150  let hasSideEffects = 1;
151  let has_offset = offsets.HasOffset;
152  let has_soffset = offsets.HasSOffset;
153  let PseudoInstr = opName # offsets.Variant;
154}
155
156multiclass SM_Pseudo_Loads<RegisterClass baseClass,
157                           RegisterClass dstClass> {
158  defvar opName = !tolower(NAME);
159  def _IMM : SM_Load_Pseudo <opName, baseClass, dstClass, IMM_Offset>;
160  def _SGPR : SM_Load_Pseudo <opName, baseClass, dstClass, SGPR_Offset>;
161  def _SGPR_IMM : SM_Load_Pseudo <opName, baseClass, dstClass, SGPR_IMM_Offset>;
162}
163
164multiclass SM_Pseudo_Stores<RegisterClass baseClass,
165                            RegisterClass srcClass> {
166  defvar opName = !tolower(NAME);
167  def _IMM : SM_Store_Pseudo <opName, baseClass, srcClass, IMM_Offset>;
168  def _SGPR : SM_Store_Pseudo <opName, baseClass, srcClass, SGPR_Offset>;
169  def _SGPR_IMM : SM_Store_Pseudo <opName, baseClass, srcClass, SGPR_IMM_Offset>;
170}
171
172multiclass SM_Pseudo_Discards {
173  defvar opName = !tolower(NAME);
174  def _IMM : SM_Discard_Pseudo <opName, IMM_Offset>;
175  def _SGPR : SM_Discard_Pseudo <opName, SGPR_Offset>;
176  def _SGPR_IMM : SM_Discard_Pseudo <opName, SGPR_IMM_Offset>;
177}
178
179class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo<
180  opName, (outs SReg_64_XEXEC:$sdst), (ins),
181  " $sdst", [(set i64:$sdst, (node))]> {
182  let hasSideEffects = 1;
183
184  let mayStore = 0;
185  let mayLoad = 0;
186  let has_sbase = 0;
187}
188
189class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo<
190  opName, (outs), (ins), "", [(node)]> {
191  let hasSideEffects = 1;
192  let mayLoad = 0;
193  let mayStore = 0;
194  let has_sdst = 0;
195  let has_sbase = 0;
196}
197
198multiclass SM_Pseudo_Probe<RegisterClass baseClass> {
199  defvar opName = !tolower(NAME);
200  def _IMM  : SM_Probe_Pseudo <opName, baseClass, IMM_Offset>;
201  def _SGPR : SM_Probe_Pseudo <opName, baseClass, SGPR_Offset>;
202  def _SGPR_IMM : SM_Probe_Pseudo <opName, baseClass, SGPR_IMM_Offset>;
203}
204
205class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo<
206  opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins),
207  " $sdst", [(set i32:$sdst, (node))]> {
208  let hasSideEffects = 1;
209  let mayStore = 0;
210  let mayLoad = 0;
211  let has_sbase = 0;
212}
213
214//===----------------------------------------------------------------------===//
215// Scalar Atomic Memory Classes
216//===----------------------------------------------------------------------===//
217
218class SM_Atomic_Pseudo <string opName,
219                        dag outs, dag ins, string asmOps, bit isRet>
220  : SM_Pseudo<opName, outs, ins, asmOps, []> {
221
222  bit glc = isRet;
223
224  let mayLoad = 1;
225  let mayStore = 1;
226  let has_glc = 1;
227  let has_dlc = 1;
228  let has_soffset = 1;
229
230  // Should these be set?
231  let ScalarStore = 1;
232  let hasSideEffects = 1;
233  let maybeAtomic = 1;
234
235  let IsAtomicNoRet = !not(isRet);
236  let IsAtomicRet = isRet;
237
238  let AsmMatchConverter = "cvtSMEMAtomic";
239}
240
241class SM_Pseudo_Atomic<string opName,
242                       RegisterClass baseClass,
243                       RegisterClass dataClass,
244                       OffsetMode offsets,
245                       bit isRet,
246                       string opNameWithSuffix =
247                         opName # offsets.Variant # !if(isRet, "_RTN", ""),
248                       Operand CPolTy = !if(isRet, CPol_GLC1, CPol)> :
249  SM_Atomic_Pseudo<opName,
250                   !if(isRet, (outs dataClass:$sdst), (outs)),
251                   !con((ins dataClass:$sdata, baseClass:$sbase), offsets.Ins,
252                        (ins CPolTy:$cpol)),
253                   !if(isRet, " $sdst", " $sdata") #
254                     ", $sbase, " # offsets.Asm # "$cpol",
255                   isRet>,
256  AtomicNoRet <opNameWithSuffix, isRet> {
257  let has_offset = offsets.HasOffset;
258  let has_soffset = offsets.HasSOffset;
259  let PseudoInstr = opNameWithSuffix;
260
261  let Constraints = !if(isRet, "$sdst = $sdata", "");
262  let DisableEncoding = !if(isRet, "$sdata", "");
263}
264
265multiclass SM_Pseudo_Atomics<RegisterClass baseClass,
266                             RegisterClass dataClass> {
267  defvar opName = !tolower(NAME);
268  def _IMM      : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 0>;
269  def _SGPR     : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 0>;
270  def _SGPR_IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 0>;
271  def _IMM_RTN  : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 1>;
272  def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 1>;
273  def _SGPR_IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 1>;
274}
275
276//===----------------------------------------------------------------------===//
277// Scalar Memory Instructions
278//===----------------------------------------------------------------------===//
279
280// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit
281// SMRD instructions, because the SReg_32_XM0 register class does not include M0
282// and writing to M0 from an SMRD instruction will hang the GPU.
283
284// XXX - SMEM instructions do not allow exec for data operand, but
285// does sdst for SMRD on SI/CI?
286defm S_LOAD_DWORD    : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>;
287defm S_LOAD_DWORDX2  : SM_Pseudo_Loads <SReg_64, SReg_64_XEXEC>;
288defm S_LOAD_DWORDX4  : SM_Pseudo_Loads <SReg_64, SReg_128>;
289defm S_LOAD_DWORDX8  : SM_Pseudo_Loads <SReg_64, SReg_256>;
290defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <SReg_64, SReg_512>;
291
292let is_buffer = 1 in {
293defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads <SReg_128, SReg_32_XM0_XEXEC>;
294// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on
295// SI/CI, bit disallowed for SMEM on VI.
296defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_128, SReg_64_XEXEC>;
297defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_128, SReg_128>;
298defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads <SReg_128, SReg_256>;
299defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads <SReg_128, SReg_512>;
300}
301
302let SubtargetPredicate = HasScalarStores in {
303defm S_STORE_DWORD : SM_Pseudo_Stores <SReg_64, SReg_32_XM0_XEXEC>;
304defm S_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_64, SReg_64_XEXEC>;
305defm S_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_64, SReg_128>;
306
307let is_buffer = 1 in {
308defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores <SReg_128, SReg_32_XM0_XEXEC>;
309defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_128, SReg_64_XEXEC>;
310defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_128, SReg_128>;
311}
312} // End SubtargetPredicate = HasScalarStores
313
314let SubtargetPredicate = HasSMemTimeInst in
315def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>;
316def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>;
317
318let SubtargetPredicate = isGFX7GFX8GFX9 in {
319def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>;
320} // let SubtargetPredicate = isGFX7GFX8GFX9
321
322let SubtargetPredicate = isGFX8Plus in {
323let OtherPredicates = [HasScalarStores] in {
324def S_DCACHE_WB     : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>;
325def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>;
326} // End OtherPredicates = [HasScalarStores]
327
328defm S_ATC_PROBE        : SM_Pseudo_Probe <SReg_64>;
329let is_buffer = 1 in {
330defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <SReg_128>;
331}
332} // SubtargetPredicate = isGFX8Plus
333
334let SubtargetPredicate = HasSMemRealTime in
335def S_MEMREALTIME   : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>;
336
337let SubtargetPredicate = isGFX10Plus in
338def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">;
339let SubtargetPredicate = HasGetWaveIdInst in
340def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>;
341
342
343let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in {
344defm S_SCRATCH_LOAD_DWORD    : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>;
345defm S_SCRATCH_LOAD_DWORDX2  : SM_Pseudo_Loads <SReg_64, SReg_64_XEXEC>;
346defm S_SCRATCH_LOAD_DWORDX4  : SM_Pseudo_Loads <SReg_64, SReg_128>;
347
348defm S_SCRATCH_STORE_DWORD   : SM_Pseudo_Stores <SReg_64, SReg_32_XM0_XEXEC>;
349defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_64, SReg_64_XEXEC>;
350defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_64, SReg_128>;
351} // SubtargetPredicate = HasScalarFlatScratchInsts
352
353let SubtargetPredicate = HasScalarAtomics in {
354
355let is_buffer = 1 in {
356defm S_BUFFER_ATOMIC_SWAP         : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
357defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
358defm S_BUFFER_ATOMIC_ADD          : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
359defm S_BUFFER_ATOMIC_SUB          : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
360defm S_BUFFER_ATOMIC_SMIN         : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
361defm S_BUFFER_ATOMIC_UMIN         : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
362defm S_BUFFER_ATOMIC_SMAX         : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
363defm S_BUFFER_ATOMIC_UMAX         : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
364defm S_BUFFER_ATOMIC_AND          : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
365defm S_BUFFER_ATOMIC_OR           : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
366defm S_BUFFER_ATOMIC_XOR          : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
367defm S_BUFFER_ATOMIC_INC          : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
368defm S_BUFFER_ATOMIC_DEC          : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>;
369
370defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
371defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Pseudo_Atomics <SReg_128, SReg_128>;
372defm S_BUFFER_ATOMIC_ADD_X2       : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
373defm S_BUFFER_ATOMIC_SUB_X2       : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
374defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
375defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
376defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
377defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
378defm S_BUFFER_ATOMIC_AND_X2       : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
379defm S_BUFFER_ATOMIC_OR_X2        : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
380defm S_BUFFER_ATOMIC_XOR_X2       : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
381defm S_BUFFER_ATOMIC_INC_X2       : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
382defm S_BUFFER_ATOMIC_DEC_X2       : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>;
383}
384
385defm S_ATOMIC_SWAP                : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
386defm S_ATOMIC_CMPSWAP             : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
387defm S_ATOMIC_ADD                 : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
388defm S_ATOMIC_SUB                 : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
389defm S_ATOMIC_SMIN                : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
390defm S_ATOMIC_UMIN                : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
391defm S_ATOMIC_SMAX                : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
392defm S_ATOMIC_UMAX                : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
393defm S_ATOMIC_AND                 : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
394defm S_ATOMIC_OR                  : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
395defm S_ATOMIC_XOR                 : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
396defm S_ATOMIC_INC                 : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
397defm S_ATOMIC_DEC                 : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;
398
399defm S_ATOMIC_SWAP_X2             : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
400defm S_ATOMIC_CMPSWAP_X2          : SM_Pseudo_Atomics <SReg_64, SReg_128>;
401defm S_ATOMIC_ADD_X2              : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
402defm S_ATOMIC_SUB_X2              : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
403defm S_ATOMIC_SMIN_X2             : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
404defm S_ATOMIC_UMIN_X2             : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
405defm S_ATOMIC_SMAX_X2             : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
406defm S_ATOMIC_UMAX_X2             : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
407defm S_ATOMIC_AND_X2              : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
408defm S_ATOMIC_OR_X2               : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
409defm S_ATOMIC_XOR_X2              : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
410defm S_ATOMIC_INC_X2              : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
411defm S_ATOMIC_DEC_X2              : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;
412
413} // let SubtargetPredicate = HasScalarAtomics
414
415let SubtargetPredicate = HasScalarAtomics in {
416defm S_DCACHE_DISCARD    : SM_Pseudo_Discards;
417defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards;
418}
419
420//===----------------------------------------------------------------------===//
421// Targets
422//===----------------------------------------------------------------------===//
423
424//===----------------------------------------------------------------------===//
425// SI
426//===----------------------------------------------------------------------===//
427
428class SMRD_Real_si <bits<5> op, SM_Pseudo ps>
429  : SM_Real<ps>
430  , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI>
431  , Enc32 {
432
433  let AssemblerPredicate = isGFX6GFX7;
434  let DecoderNamespace = "GFX6GFX7";
435
436  let Inst{7-0}   = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?));
437  let Inst{8}     = ps.has_offset;
438  let Inst{14-9}  = !if(ps.has_sbase, sbase{6-1}, ?);
439  let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
440  let Inst{26-22} = op;
441  let Inst{31-27} = 0x18; //encoding
442}
443
444multiclass SM_Real_Loads_si<bits<5> op> {
445  defvar ps = NAME;
446  defvar immPs = !cast<SM_Load_Pseudo>(ps#_IMM);
447  def _IMM_si : SMRD_Real_si <op, immPs> {
448    let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol);
449  }
450
451  defvar sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR);
452  def _SGPR_si : SMRD_Real_si <op, sgprPs>;
453}
454
455defm S_LOAD_DWORD           : SM_Real_Loads_si <0x00>;
456defm S_LOAD_DWORDX2         : SM_Real_Loads_si <0x01>;
457defm S_LOAD_DWORDX4         : SM_Real_Loads_si <0x02>;
458defm S_LOAD_DWORDX8         : SM_Real_Loads_si <0x03>;
459defm S_LOAD_DWORDX16        : SM_Real_Loads_si <0x04>;
460defm S_BUFFER_LOAD_DWORD    : SM_Real_Loads_si <0x08>;
461defm S_BUFFER_LOAD_DWORDX2  : SM_Real_Loads_si <0x09>;
462defm S_BUFFER_LOAD_DWORDX4  : SM_Real_Loads_si <0x0a>;
463defm S_BUFFER_LOAD_DWORDX8  : SM_Real_Loads_si <0x0b>;
464defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c>;
465
466def S_MEMTIME_si    : SMRD_Real_si <0x1e, S_MEMTIME>;
467def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>;
468
469
470//===----------------------------------------------------------------------===//
471// VI and GFX9.
472//===----------------------------------------------------------------------===//
473
474class SMEM_Real_vi <bits<8> op, SM_Pseudo ps>
475  : SM_Real<ps>
476  , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI>
477  , Enc64 {
478  field bit IsGFX9SpecificEncoding = false;
479  let AssemblerPredicate = !if(IsGFX9SpecificEncoding, isGFX9Only, isGFX8GFX9);
480  let DecoderNamespace = "GFX8";
481
482  let Inst{5-0}   = !if(ps.has_sbase, sbase{6-1}, ?);
483  let Inst{12-6}  = !if(ps.has_sdst, sdst{6-0}, ?);
484
485  // Note that for GFX9 instructions with immediate offsets, soffset_en
486  // must be defined, whereas in GFX8 it's undefined in all cases,
487  // meaning GFX9 is not perfectly backward-compatible with GFX8, despite
488  // documentation suggesting otherwise.
489  field bit SOffsetEn = !if(IsGFX9SpecificEncoding,
490    !if(ps.has_offset, ps.has_soffset, !if(ps.has_soffset, 0, ?)),
491    ?);
492  let Inst{14} = SOffsetEn;
493
494  let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?);
495
496  // imm
497  // TODO: Shall not be defined if the instruction has no offset nor
498  // soffset.
499  let Inst{17} = ps.has_offset;
500
501  let Inst{25-18} = op;
502  let Inst{31-26} = 0x30; //encoding
503
504  // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed.
505  // Offset value is corrected accordingly when offset is encoded/decoded.
506  // TODO: Forbid non-M0 register offsets for GFX8 stores and atomics.
507  field bits<21> Offset;
508  let Offset{6-0} = !if(ps.has_offset, offset{6-0},
509                                       !if(ps.has_soffset, soffset{6-0}, ?));
510  let Offset{20-7} = !if(ps.has_offset, offset{20-7}, ?);
511  let Inst{52-32} = Offset;
512
513  // soffset
514  let Inst{63-57} = !if(!and(IsGFX9SpecificEncoding, ps.has_soffset),
515                        soffset{6-0}, ?);
516}
517
518class SMEM_Real_Load_vi<bits<8> op, string ps>
519    : SMEM_Real_vi<op, !cast<SM_Pseudo>(ps)>;
520
521// The alternative GFX9 SGPR encoding using soffset to encode the
522// offset register. Not available in assembler and goes to the GFX9
523// encoding family to avoid conflicts with the primary SGPR variant.
524class SMEM_Real_SGPR_alt_gfx9 {
525  bit IsGFX9SpecificEncoding = true;
526  bit SOffsetEn = 1;
527  bit Offset = ?;
528  int Subtarget = SIEncodingFamily.GFX9;
529  string AsmVariantName = "NonParsable";
530}
531
532multiclass SM_Real_Loads_vi<bits<8> op> {
533  defvar ps = NAME;
534  def _IMM_vi : SMEM_Real_Load_vi <op, ps#"_IMM">;
535  def _SGPR_vi : SMEM_Real_Load_vi <op, ps#"_SGPR">;
536  def _SGPR_alt_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR">,
537                       SMEM_Real_SGPR_alt_gfx9;
538  let IsGFX9SpecificEncoding = true in
539  def _SGPR_IMM_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR_IMM">;
540}
541
542class SMEM_Real_Store_Base_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> {
543  // encoding
544  bits<7> sdata;
545
546  let sdst = ?;
547  let Inst{12-6}  = !if(ps.has_sdst, sdata{6-0}, ?);
548}
549
550class SMEM_Real_Store_vi <bits<8> op, string ps>
551    : SMEM_Real_Store_Base_vi <op, !cast<SM_Pseudo>(ps)>;
552
553multiclass SM_Real_Stores_vi<bits<8> op> {
554  defvar ps = NAME;
555  def _IMM_vi : SMEM_Real_Store_vi <op, ps#"_IMM">;
556  def _SGPR_vi : SMEM_Real_Store_vi <op, ps#"_SGPR">;
557  def _SGPR_alt_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR">,
558                       SMEM_Real_SGPR_alt_gfx9;
559  let IsGFX9SpecificEncoding = true in
560  def _SGPR_IMM_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR_IMM">;
561}
562
563multiclass SM_Real_Probe_vi<bits<8> op> {
564  defvar ps = NAME;
565  def _IMM_vi  : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;
566  def _SGPR_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>;
567  def _SGPR_alt_gfx9
568    : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>,
569      SMEM_Real_SGPR_alt_gfx9;
570  let IsGFX9SpecificEncoding = true in
571  def _SGPR_IMM_gfx9
572    : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>;
573}
574
575defm S_LOAD_DWORD           : SM_Real_Loads_vi <0x00>;
576defm S_LOAD_DWORDX2         : SM_Real_Loads_vi <0x01>;
577defm S_LOAD_DWORDX4         : SM_Real_Loads_vi <0x02>;
578defm S_LOAD_DWORDX8         : SM_Real_Loads_vi <0x03>;
579defm S_LOAD_DWORDX16        : SM_Real_Loads_vi <0x04>;
580defm S_BUFFER_LOAD_DWORD    : SM_Real_Loads_vi <0x08>;
581defm S_BUFFER_LOAD_DWORDX2  : SM_Real_Loads_vi <0x09>;
582defm S_BUFFER_LOAD_DWORDX4  : SM_Real_Loads_vi <0x0a>;
583defm S_BUFFER_LOAD_DWORDX8  : SM_Real_Loads_vi <0x0b>;
584defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c>;
585
586defm S_STORE_DWORD : SM_Real_Stores_vi <0x10>;
587defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11>;
588defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12>;
589
590defm S_BUFFER_STORE_DWORD    : SM_Real_Stores_vi <0x18>;
591defm S_BUFFER_STORE_DWORDX2  : SM_Real_Stores_vi <0x19>;
592defm S_BUFFER_STORE_DWORDX4  : SM_Real_Stores_vi <0x1a>;
593
594// These instructions use same encoding
595def S_DCACHE_INV_vi         : SMEM_Real_vi <0x20, S_DCACHE_INV>;
596def S_DCACHE_WB_vi          : SMEM_Real_vi <0x21, S_DCACHE_WB>;
597def S_DCACHE_INV_VOL_vi     : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>;
598def S_DCACHE_WB_VOL_vi      : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>;
599def S_MEMTIME_vi            : SMEM_Real_vi <0x24, S_MEMTIME>;
600def S_MEMREALTIME_vi        : SMEM_Real_vi <0x25, S_MEMREALTIME>;
601
602defm S_SCRATCH_LOAD_DWORD    : SM_Real_Loads_vi <0x05>;
603defm S_SCRATCH_LOAD_DWORDX2  : SM_Real_Loads_vi <0x06>;
604defm S_SCRATCH_LOAD_DWORDX4  : SM_Real_Loads_vi <0x07>;
605
606defm S_SCRATCH_STORE_DWORD   : SM_Real_Stores_vi <0x15>;
607defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16>;
608defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17>;
609
610defm S_ATC_PROBE        : SM_Real_Probe_vi <0x26>;
611defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27>;
612
613//===----------------------------------------------------------------------===//
614// GFX9
615//===----------------------------------------------------------------------===//
616
617class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps>
618  : SMEM_Real_vi <op, ps>,
619    AtomicNoRet <!subst("_RTN","",NAME), ps.glc> {
620
621  bits<7> sdata;
622
623  let Constraints = ps.Constraints;
624  let DisableEncoding = ps.DisableEncoding;
625
626  let cpol{CPolBit.GLC} = ps.glc;
627  let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0});
628}
629
630multiclass SM_Real_Atomics_vi<bits<8> op> {
631  defvar ps = NAME;
632  def _IMM_vi       : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>;
633  def _SGPR_vi      : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>;
634  def _SGPR_alt_gfx9
635    : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>,
636      SMEM_Real_SGPR_alt_gfx9;
637  let IsGFX9SpecificEncoding = true in
638  def _SGPR_IMM_gfx9
639    : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>;
640  def _IMM_RTN_vi   : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>;
641  def _SGPR_RTN_vi  : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>;
642  def _SGPR_RTN_alt_gfx9
643    : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>,
644      SMEM_Real_SGPR_alt_gfx9;
645  let IsGFX9SpecificEncoding = true in
646  def _SGPR_IMM_RTN_gfx9
647    : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>;
648}
649
650defm S_BUFFER_ATOMIC_SWAP         : SM_Real_Atomics_vi <0x40>;
651defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Real_Atomics_vi <0x41>;
652defm S_BUFFER_ATOMIC_ADD          : SM_Real_Atomics_vi <0x42>;
653defm S_BUFFER_ATOMIC_SUB          : SM_Real_Atomics_vi <0x43>;
654defm S_BUFFER_ATOMIC_SMIN         : SM_Real_Atomics_vi <0x44>;
655defm S_BUFFER_ATOMIC_UMIN         : SM_Real_Atomics_vi <0x45>;
656defm S_BUFFER_ATOMIC_SMAX         : SM_Real_Atomics_vi <0x46>;
657defm S_BUFFER_ATOMIC_UMAX         : SM_Real_Atomics_vi <0x47>;
658defm S_BUFFER_ATOMIC_AND          : SM_Real_Atomics_vi <0x48>;
659defm S_BUFFER_ATOMIC_OR           : SM_Real_Atomics_vi <0x49>;
660defm S_BUFFER_ATOMIC_XOR          : SM_Real_Atomics_vi <0x4a>;
661defm S_BUFFER_ATOMIC_INC          : SM_Real_Atomics_vi <0x4b>;
662defm S_BUFFER_ATOMIC_DEC          : SM_Real_Atomics_vi <0x4c>;
663
664defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Real_Atomics_vi <0x60>;
665defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Real_Atomics_vi <0x61>;
666defm S_BUFFER_ATOMIC_ADD_X2       : SM_Real_Atomics_vi <0x62>;
667defm S_BUFFER_ATOMIC_SUB_X2       : SM_Real_Atomics_vi <0x63>;
668defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Real_Atomics_vi <0x64>;
669defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Real_Atomics_vi <0x65>;
670defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Real_Atomics_vi <0x66>;
671defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Real_Atomics_vi <0x67>;
672defm S_BUFFER_ATOMIC_AND_X2       : SM_Real_Atomics_vi <0x68>;
673defm S_BUFFER_ATOMIC_OR_X2        : SM_Real_Atomics_vi <0x69>;
674defm S_BUFFER_ATOMIC_XOR_X2       : SM_Real_Atomics_vi <0x6a>;
675defm S_BUFFER_ATOMIC_INC_X2       : SM_Real_Atomics_vi <0x6b>;
676defm S_BUFFER_ATOMIC_DEC_X2       : SM_Real_Atomics_vi <0x6c>;
677
678defm S_ATOMIC_SWAP                : SM_Real_Atomics_vi <0x80>;
679defm S_ATOMIC_CMPSWAP             : SM_Real_Atomics_vi <0x81>;
680defm S_ATOMIC_ADD                 : SM_Real_Atomics_vi <0x82>;
681defm S_ATOMIC_SUB                 : SM_Real_Atomics_vi <0x83>;
682defm S_ATOMIC_SMIN                : SM_Real_Atomics_vi <0x84>;
683defm S_ATOMIC_UMIN                : SM_Real_Atomics_vi <0x85>;
684defm S_ATOMIC_SMAX                : SM_Real_Atomics_vi <0x86>;
685defm S_ATOMIC_UMAX                : SM_Real_Atomics_vi <0x87>;
686defm S_ATOMIC_AND                 : SM_Real_Atomics_vi <0x88>;
687defm S_ATOMIC_OR                  : SM_Real_Atomics_vi <0x89>;
688defm S_ATOMIC_XOR                 : SM_Real_Atomics_vi <0x8a>;
689defm S_ATOMIC_INC                 : SM_Real_Atomics_vi <0x8b>;
690defm S_ATOMIC_DEC                 : SM_Real_Atomics_vi <0x8c>;
691
692defm S_ATOMIC_SWAP_X2             : SM_Real_Atomics_vi <0xa0>;
693defm S_ATOMIC_CMPSWAP_X2          : SM_Real_Atomics_vi <0xa1>;
694defm S_ATOMIC_ADD_X2              : SM_Real_Atomics_vi <0xa2>;
695defm S_ATOMIC_SUB_X2              : SM_Real_Atomics_vi <0xa3>;
696defm S_ATOMIC_SMIN_X2             : SM_Real_Atomics_vi <0xa4>;
697defm S_ATOMIC_UMIN_X2             : SM_Real_Atomics_vi <0xa5>;
698defm S_ATOMIC_SMAX_X2             : SM_Real_Atomics_vi <0xa6>;
699defm S_ATOMIC_UMAX_X2             : SM_Real_Atomics_vi <0xa7>;
700defm S_ATOMIC_AND_X2              : SM_Real_Atomics_vi <0xa8>;
701defm S_ATOMIC_OR_X2               : SM_Real_Atomics_vi <0xa9>;
702defm S_ATOMIC_XOR_X2              : SM_Real_Atomics_vi <0xaa>;
703defm S_ATOMIC_INC_X2              : SM_Real_Atomics_vi <0xab>;
704defm S_ATOMIC_DEC_X2              : SM_Real_Atomics_vi <0xac>;
705
706multiclass SM_Real_Discard_vi<bits<8> op> {
707  defvar ps = NAME;
708  def _IMM_vi  : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>;
709  def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>;
710  def _SGPR_alt_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>,
711                       SMEM_Real_SGPR_alt_gfx9;
712  let IsGFX9SpecificEncoding = true in
713  def _SGPR_IMM_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR_IMM)>;
714}
715
716defm S_DCACHE_DISCARD    : SM_Real_Discard_vi <0x28>;
717defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29>;
718
719//===----------------------------------------------------------------------===//
720// CI
721//===----------------------------------------------------------------------===//
722
723def smrd_literal_offset : ImmOperand<i32, "SMRDLiteralOffset">;
724
725class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> :
726  SM_Real<ps>,
727  Enc64 {
728
729  let AssemblerPredicate = isGFX7Only;
730  let DecoderNamespace = "GFX7";
731  let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, CPol:$cpol);
732
733  let Inst{7-0}   = 0xff;
734  let Inst{8}     = 0;
735  let Inst{14-9}  = sbase{6-1};
736  let Inst{21-15} = sdst{6-0};
737  let Inst{26-22} = op;
738  let Inst{31-27} = 0x18; //encoding
739  let Inst{63-32} = offset{31-0};
740}
741
742def S_LOAD_DWORD_IMM_ci           : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>;
743def S_LOAD_DWORDX2_IMM_ci         : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>;
744def S_LOAD_DWORDX4_IMM_ci         : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>;
745def S_LOAD_DWORDX8_IMM_ci         : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>;
746def S_LOAD_DWORDX16_IMM_ci        : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>;
747def S_BUFFER_LOAD_DWORD_IMM_ci    : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>;
748def S_BUFFER_LOAD_DWORDX2_IMM_ci  : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>;
749def S_BUFFER_LOAD_DWORDX4_IMM_ci  : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>;
750def S_BUFFER_LOAD_DWORDX8_IMM_ci  : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>;
751def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>;
752
753class SMRD_Real_ci <bits<5> op, SM_Pseudo ps>
754  : SM_Real<ps>
755  , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI>
756  , Enc32 {
757
758  let AssemblerPredicate = isGFX7Only;
759  let DecoderNamespace = "GFX7";
760
761  let Inst{7-0}   = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?));
762  let Inst{8}     = ps.has_offset;
763  let Inst{14-9}  = !if(ps.has_sbase, sbase{6-1}, ?);
764  let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
765  let Inst{26-22} = op;
766  let Inst{31-27} = 0x18; //encoding
767}
768
769def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>;
770
771//===----------------------------------------------------------------------===//
772// Scalar Memory Patterns
773//===----------------------------------------------------------------------===//
774
775def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]> {
776  let GISelPredicateCode = [{
777    if (!MI.hasOneMemOperand())
778      return false;
779    if (!isInstrUniform(MI))
780      return false;
781
782    // FIXME: We should probably be caching this.
783    SmallVector<GEPInfo, 4> AddrInfo;
784    getAddrModeInfo(MI, MRI, AddrInfo);
785
786    if (hasVgprParts(AddrInfo))
787      return false;
788    return true;
789  }];
790}
791
792def SMRDImm         : ComplexPattern<iPTR, 2, "SelectSMRDImm">;
793def SMRDImm32       : ComplexPattern<iPTR, 2, "SelectSMRDImm32">;
794def SMRDSgpr        : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">;
795def SMRDSgprImm     : ComplexPattern<iPTR, 3, "SelectSMRDSgprImm">;
796def SMRDBufferImm   : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">;
797def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">;
798def SMRDBufferSgprImm : ComplexPattern<iPTR, 2, "SelectSMRDBufferSgprImm">;
799
800multiclass SMRD_Pattern <string Instr, ValueType vt> {
801
802  // 1. IMM offset
803  def : GCNPat <
804    (smrd_load (SMRDImm i64:$sbase, i32:$offset)),
805    (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))
806  >;
807
808  // 2. 32-bit IMM offset on CI
809  def : GCNPat <
810    (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)),
811    (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> {
812    let OtherPredicates = [isGFX7Only];
813  }
814
815  // 3. SGPR offset
816  def : GCNPat <
817    (smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
818    (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $soffset, 0))> {
819    let OtherPredicates = [isNotGFX9Plus];
820  }
821  def : GCNPat <
822    (smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
823    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, 0, 0))> {
824    let OtherPredicates = [isGFX9Plus];
825  }
826
827  // 4. SGPR+IMM offset
828  def : GCNPat <
829    (smrd_load (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)),
830    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, $offset, 0))> {
831    let OtherPredicates = [isGFX9Plus];
832  }
833
834  // 5. No offset
835  def : GCNPat <
836    (vt (smrd_load (i64 SReg_64:$sbase))),
837    (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0))
838  >;
839}
840
841multiclass SMLoad_Pattern <string Instr, ValueType vt> {
842  // 1. Offset as an immediate
843  def : GCNPat <
844    (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy),
845    (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> {
846    let AddedComplexity = 2;
847  }
848
849  // 2. 32-bit IMM offset on CI
850  def : GCNPat <
851    (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)),
852    (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset,
853                                    (extract_cpol $cachepolicy))> {
854    let OtherPredicates = [isGFX7Only];
855    let AddedComplexity = 1;
856  }
857
858  // 3. Offset loaded in an 32bit SGPR
859  def : GCNPat <
860    (SIsbuffer_load v4i32:$sbase, i32:$soffset, timm:$cachepolicy),
861    (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$soffset, (extract_cpol $cachepolicy)))> {
862    let OtherPredicates = [isNotGFX9Plus];
863  }
864  def : GCNPat <
865    (SIsbuffer_load v4i32:$sbase, i32:$soffset, timm:$cachepolicy),
866    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, 0, (extract_cpol $cachepolicy)))> {
867    let OtherPredicates = [isGFX9Plus];
868  }
869
870  // 4. Offset as an 32-bit SGPR + immediate
871  def : GCNPat <
872    (SIsbuffer_load v4i32:$sbase, (SMRDBufferSgprImm i32:$soffset, i32:$offset),
873                    timm:$cachepolicy),
874    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, i32imm:$offset,
875                                             (extract_cpol $cachepolicy)))> {
876    let OtherPredicates = [isGFX9Plus];
877  }
878}
879
880// Global and constant loads can be selected to either MUBUF or SMRD
881// instructions, but SMRD instructions are faster so we want the instruction
882// selector to prefer those.
883let AddedComplexity = 100 in {
884
885foreach vt = Reg32Types.types in {
886defm : SMRD_Pattern <"S_LOAD_DWORD", vt>;
887}
888
889foreach vt = SReg_64.RegTypes in {
890defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>;
891}
892
893foreach vt = SReg_128.RegTypes in {
894defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>;
895}
896
897foreach vt = SReg_256.RegTypes in {
898defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>;
899}
900
901foreach vt = SReg_512.RegTypes in {
902defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>;
903}
904
905} // End let AddedComplexity = 100
906
907defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD",     i32>;
908defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2",   v2i32>;
909defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4",   v4i32>;
910defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8",   v8i32>;
911defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16",  v16i32>;
912
913defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD",     f32>;
914defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2",   v2f32>;
915defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4",   v4f32>;
916defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8",   v8f32>;
917defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16",  v16f32>;
918
919let OtherPredicates = [HasSMemTimeInst] in {
920def : GCNPat <
921  (i64 (readcyclecounter)),
922  (S_MEMTIME)
923>;
924} // let OtherPredicates = [HasSMemTimeInst]
925
926let OtherPredicates = [HasShaderCyclesRegister] in {
927def : GCNPat <
928  (i64 (readcyclecounter)),
929  (REG_SEQUENCE SReg_64,
930    (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0,
931    (S_MOV_B32 (i32 0)), sub1)> {
932  // Prefer this to s_memtime because it has lower and more predictable latency.
933  let AddedComplexity = 1;
934}
935} // let OtherPredicates = [HasShaderCyclesRegister]
936
937//===----------------------------------------------------------------------===//
938// GFX10.
939//===----------------------------------------------------------------------===//
940
941class SMEM_Real_10Plus_common<bits<8> op, SM_Pseudo ps, string opName,
942                              int subtarget, RegisterWithSubRegs sgpr_null> :
943    SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 {
944  let Inst{5-0}   = !if(ps.has_sbase, sbase{6-1}, ?);
945  let Inst{12-6}  = !if(ps.has_sdst, sdst{6-0}, ?);
946  let Inst{25-18} = op;
947  let Inst{31-26} = 0x3d;
948  // There are SMEM instructions that do not employ any of the offset
949  // fields, in which case we need them to remain undefined.
950  let Inst{52-32} = !if(ps.has_offset, offset{20-0}, !if(ps.has_soffset, 0, ?));
951  let Inst{63-57} = !if(ps.has_soffset, soffset{6-0},
952                        !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?));
953}
954
955class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps>
956    : SMEM_Real_10Plus_common<op, ps, ps.Mnemonic, SIEncodingFamily.GFX10,
957                              SGPR_NULL_gfxpre11> {
958  let AssemblerPredicate = isGFX10Only;
959  let DecoderNamespace = "GFX10";
960  let Inst{14}    = !if(ps.has_dlc, cpol{CPolBit.DLC}, ?);
961  let Inst{16}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ?);
962}
963
964class SMEM_Real_Load_gfx10<bits<8> op, string ps>
965    : SMEM_Real_gfx10<op, !cast<SM_Pseudo>(ps)>;
966
967multiclass SM_Real_Loads_gfx10<bits<8> op> {
968  defvar ps = NAME;
969  def _IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_IMM">;
970  def _SGPR_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_SGPR">;
971  def _SGPR_IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_SGPR_IMM">;
972}
973
974class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> {
975  bits<7> sdata;
976
977  let sdst = ?;
978  let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?);
979}
980
981multiclass SM_Real_Stores_gfx10<bits<8> op> {
982  defvar ps = NAME;
983  defvar immPs = !cast<SM_Store_Pseudo>(ps#_IMM);
984  def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs>;
985
986  defvar sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR);
987  def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs>;
988
989  defvar sgprImmPs = !cast<SM_Store_Pseudo>(ps#_SGPR_IMM);
990  def _SGPR_IMM_gfx10 : SMEM_Real_Store_gfx10 <op, sgprImmPs>;
991}
992
993defm S_LOAD_DWORD            : SM_Real_Loads_gfx10<0x000>;
994defm S_LOAD_DWORDX2          : SM_Real_Loads_gfx10<0x001>;
995defm S_LOAD_DWORDX4          : SM_Real_Loads_gfx10<0x002>;
996defm S_LOAD_DWORDX8          : SM_Real_Loads_gfx10<0x003>;
997defm S_LOAD_DWORDX16         : SM_Real_Loads_gfx10<0x004>;
998
999let SubtargetPredicate = HasScalarFlatScratchInsts in {
1000defm S_SCRATCH_LOAD_DWORD    : SM_Real_Loads_gfx10<0x005>;
1001defm S_SCRATCH_LOAD_DWORDX2  : SM_Real_Loads_gfx10<0x006>;
1002defm S_SCRATCH_LOAD_DWORDX4  : SM_Real_Loads_gfx10<0x007>;
1003} // End SubtargetPredicate = HasScalarFlatScratchInsts
1004
1005defm S_BUFFER_LOAD_DWORD     : SM_Real_Loads_gfx10<0x008>;
1006defm S_BUFFER_LOAD_DWORDX2   : SM_Real_Loads_gfx10<0x009>;
1007defm S_BUFFER_LOAD_DWORDX4   : SM_Real_Loads_gfx10<0x00a>;
1008defm S_BUFFER_LOAD_DWORDX8   : SM_Real_Loads_gfx10<0x00b>;
1009defm S_BUFFER_LOAD_DWORDX16  : SM_Real_Loads_gfx10<0x00c>;
1010
1011let SubtargetPredicate = HasScalarStores in {
1012defm S_STORE_DWORD           : SM_Real_Stores_gfx10<0x010>;
1013defm S_STORE_DWORDX2         : SM_Real_Stores_gfx10<0x011>;
1014defm S_STORE_DWORDX4         : SM_Real_Stores_gfx10<0x012>;
1015let OtherPredicates = [HasScalarFlatScratchInsts] in {
1016defm S_SCRATCH_STORE_DWORD   : SM_Real_Stores_gfx10<0x015>;
1017defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016>;
1018defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017>;
1019} // End OtherPredicates = [HasScalarFlatScratchInsts]
1020defm S_BUFFER_STORE_DWORD    : SM_Real_Stores_gfx10<0x018>;
1021defm S_BUFFER_STORE_DWORDX2  : SM_Real_Stores_gfx10<0x019>;
1022defm S_BUFFER_STORE_DWORDX4  : SM_Real_Stores_gfx10<0x01a>;
1023} // End SubtargetPredicate = HasScalarStores
1024
1025def S_MEMREALTIME_gfx10              : SMEM_Real_gfx10<0x025, S_MEMREALTIME>;
1026def S_MEMTIME_gfx10                  : SMEM_Real_gfx10<0x024, S_MEMTIME>;
1027def S_GL1_INV_gfx10                  : SMEM_Real_gfx10<0x01f, S_GL1_INV>;
1028def S_GET_WAVEID_IN_WORKGROUP_gfx10  : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>;
1029def S_DCACHE_INV_gfx10               : SMEM_Real_gfx10<0x020, S_DCACHE_INV>;
1030
1031let SubtargetPredicate = HasScalarStores in {
1032def S_DCACHE_WB_gfx10                : SMEM_Real_gfx10<0x021, S_DCACHE_WB>;
1033} // End SubtargetPredicate = HasScalarStores
1034
1035multiclass SM_Real_Probe_gfx10<bits<8> op> {
1036  defvar ps = NAME;
1037  def _IMM_gfx10  : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>;
1038  def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>;
1039  def _SGPR_IMM_gfx10
1040    : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>;
1041}
1042
1043defm S_ATC_PROBE        : SM_Real_Probe_gfx10 <0x26>;
1044defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27>;
1045
1046class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps>
1047  : SMEM_Real_gfx10 <op, ps>,
1048    AtomicNoRet <!subst("_RTN","",NAME), ps.glc> {
1049
1050  bits<7> sdata;
1051
1052  let Constraints = ps.Constraints;
1053  let DisableEncoding = ps.DisableEncoding;
1054
1055  let cpol{CPolBit.GLC} = ps.glc;
1056
1057  let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0);
1058  let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0});
1059}
1060
1061multiclass SM_Real_Atomics_gfx10<bits<8> op> {
1062  defvar ps = NAME;
1063  def _IMM_gfx10       : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>;
1064  def _SGPR_gfx10      : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>;
1065  def _SGPR_IMM_gfx10  : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>;
1066  def _IMM_RTN_gfx10   : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>;
1067  def _SGPR_RTN_gfx10  : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>;
1068  def _SGPR_IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>;
1069}
1070
1071let SubtargetPredicate = HasScalarAtomics in {
1072
1073defm S_BUFFER_ATOMIC_SWAP         : SM_Real_Atomics_gfx10 <0x40>;
1074defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Real_Atomics_gfx10 <0x41>;
1075defm S_BUFFER_ATOMIC_ADD          : SM_Real_Atomics_gfx10 <0x42>;
1076defm S_BUFFER_ATOMIC_SUB          : SM_Real_Atomics_gfx10 <0x43>;
1077defm S_BUFFER_ATOMIC_SMIN         : SM_Real_Atomics_gfx10 <0x44>;
1078defm S_BUFFER_ATOMIC_UMIN         : SM_Real_Atomics_gfx10 <0x45>;
1079defm S_BUFFER_ATOMIC_SMAX         : SM_Real_Atomics_gfx10 <0x46>;
1080defm S_BUFFER_ATOMIC_UMAX         : SM_Real_Atomics_gfx10 <0x47>;
1081defm S_BUFFER_ATOMIC_AND          : SM_Real_Atomics_gfx10 <0x48>;
1082defm S_BUFFER_ATOMIC_OR           : SM_Real_Atomics_gfx10 <0x49>;
1083defm S_BUFFER_ATOMIC_XOR          : SM_Real_Atomics_gfx10 <0x4a>;
1084defm S_BUFFER_ATOMIC_INC          : SM_Real_Atomics_gfx10 <0x4b>;
1085defm S_BUFFER_ATOMIC_DEC          : SM_Real_Atomics_gfx10 <0x4c>;
1086
1087defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Real_Atomics_gfx10 <0x60>;
1088defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Real_Atomics_gfx10 <0x61>;
1089defm S_BUFFER_ATOMIC_ADD_X2       : SM_Real_Atomics_gfx10 <0x62>;
1090defm S_BUFFER_ATOMIC_SUB_X2       : SM_Real_Atomics_gfx10 <0x63>;
1091defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Real_Atomics_gfx10 <0x64>;
1092defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Real_Atomics_gfx10 <0x65>;
1093defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Real_Atomics_gfx10 <0x66>;
1094defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Real_Atomics_gfx10 <0x67>;
1095defm S_BUFFER_ATOMIC_AND_X2       : SM_Real_Atomics_gfx10 <0x68>;
1096defm S_BUFFER_ATOMIC_OR_X2        : SM_Real_Atomics_gfx10 <0x69>;
1097defm S_BUFFER_ATOMIC_XOR_X2       : SM_Real_Atomics_gfx10 <0x6a>;
1098defm S_BUFFER_ATOMIC_INC_X2       : SM_Real_Atomics_gfx10 <0x6b>;
1099defm S_BUFFER_ATOMIC_DEC_X2       : SM_Real_Atomics_gfx10 <0x6c>;
1100
1101defm S_ATOMIC_SWAP                : SM_Real_Atomics_gfx10 <0x80>;
1102defm S_ATOMIC_CMPSWAP             : SM_Real_Atomics_gfx10 <0x81>;
1103defm S_ATOMIC_ADD                 : SM_Real_Atomics_gfx10 <0x82>;
1104defm S_ATOMIC_SUB                 : SM_Real_Atomics_gfx10 <0x83>;
1105defm S_ATOMIC_SMIN                : SM_Real_Atomics_gfx10 <0x84>;
1106defm S_ATOMIC_UMIN                : SM_Real_Atomics_gfx10 <0x85>;
1107defm S_ATOMIC_SMAX                : SM_Real_Atomics_gfx10 <0x86>;
1108defm S_ATOMIC_UMAX                : SM_Real_Atomics_gfx10 <0x87>;
1109defm S_ATOMIC_AND                 : SM_Real_Atomics_gfx10 <0x88>;
1110defm S_ATOMIC_OR                  : SM_Real_Atomics_gfx10 <0x89>;
1111defm S_ATOMIC_XOR                 : SM_Real_Atomics_gfx10 <0x8a>;
1112defm S_ATOMIC_INC                 : SM_Real_Atomics_gfx10 <0x8b>;
1113defm S_ATOMIC_DEC                 : SM_Real_Atomics_gfx10 <0x8c>;
1114
1115defm S_ATOMIC_SWAP_X2             : SM_Real_Atomics_gfx10 <0xa0>;
1116defm S_ATOMIC_CMPSWAP_X2          : SM_Real_Atomics_gfx10 <0xa1>;
1117defm S_ATOMIC_ADD_X2              : SM_Real_Atomics_gfx10 <0xa2>;
1118defm S_ATOMIC_SUB_X2              : SM_Real_Atomics_gfx10 <0xa3>;
1119defm S_ATOMIC_SMIN_X2             : SM_Real_Atomics_gfx10 <0xa4>;
1120defm S_ATOMIC_UMIN_X2             : SM_Real_Atomics_gfx10 <0xa5>;
1121defm S_ATOMIC_SMAX_X2             : SM_Real_Atomics_gfx10 <0xa6>;
1122defm S_ATOMIC_UMAX_X2             : SM_Real_Atomics_gfx10 <0xa7>;
1123defm S_ATOMIC_AND_X2              : SM_Real_Atomics_gfx10 <0xa8>;
1124defm S_ATOMIC_OR_X2               : SM_Real_Atomics_gfx10 <0xa9>;
1125defm S_ATOMIC_XOR_X2              : SM_Real_Atomics_gfx10 <0xaa>;
1126defm S_ATOMIC_INC_X2              : SM_Real_Atomics_gfx10 <0xab>;
1127defm S_ATOMIC_DEC_X2              : SM_Real_Atomics_gfx10 <0xac>;
1128
1129multiclass SM_Real_Discard_gfx10<bits<8> op> {
1130  defvar ps = NAME;
1131  def _IMM_gfx10  : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>;
1132  def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>;
1133  def _SGPR_IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>;
1134}
1135
1136defm S_DCACHE_DISCARD    : SM_Real_Discard_gfx10 <0x28>;
1137defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29>;
1138
1139} // End SubtargetPredicate = HasScalarAtomics
1140
1141def SMInfoTable : GenericTable {
1142  let FilterClass = "SM_Real";
1143  let CppTypeName = "SMInfo";
1144  let Fields = ["Opcode", "is_buffer"];
1145
1146  let PrimaryKey = ["Opcode"];
1147  let PrimaryKeyName = "getSMEMOpcodeHelper";
1148}
1149
1150//===----------------------------------------------------------------------===//
1151// GFX11.
1152//===----------------------------------------------------------------------===//
1153
1154class SMEM_Real_gfx11<bits<8> op, SM_Pseudo ps, string opName = ps.Mnemonic> :
1155    SMEM_Real_10Plus_common<op, ps, opName, SIEncodingFamily.GFX11,
1156                            SGPR_NULL_gfx11plus> {
1157  let AssemblerPredicate = isGFX11Plus;
1158  let DecoderNamespace = "GFX11";
1159  let Inst{13}    = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0);
1160  let Inst{14}    = !if(ps.has_glc, cpol{CPolBit.GLC}, 0);
1161}
1162
1163class SMEM_Real_Load_gfx11<bits<8> op, string ps, string opName> :
1164    SMEM_Real_gfx11<op, !cast<SM_Pseudo>(ps), opName>;
1165
1166multiclass SM_Real_Loads_gfx11<bits<8> op, string ps> {
1167  defvar opName = !tolower(NAME);
1168  def _IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_IMM", opName>;
1169  def _SGPR_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR", opName>;
1170  def _SGPR_IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR_IMM", opName>;
1171  def : MnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName>,
1172                      Requires<[isGFX11Plus]>;
1173}
1174
1175defm S_LOAD_B32  : SM_Real_Loads_gfx11<0x000, "S_LOAD_DWORD">;
1176defm S_LOAD_B64  : SM_Real_Loads_gfx11<0x001, "S_LOAD_DWORDX2">;
1177defm S_LOAD_B128 : SM_Real_Loads_gfx11<0x002, "S_LOAD_DWORDX4">;
1178defm S_LOAD_B256 : SM_Real_Loads_gfx11<0x003, "S_LOAD_DWORDX8">;
1179defm S_LOAD_B512 : SM_Real_Loads_gfx11<0x004, "S_LOAD_DWORDX16">;
1180
1181defm S_BUFFER_LOAD_B32  : SM_Real_Loads_gfx11<0x008, "S_BUFFER_LOAD_DWORD">;
1182defm S_BUFFER_LOAD_B64  : SM_Real_Loads_gfx11<0x009, "S_BUFFER_LOAD_DWORDX2">;
1183defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx11<0x00a, "S_BUFFER_LOAD_DWORDX4">;
1184defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx11<0x00b, "S_BUFFER_LOAD_DWORDX8">;
1185defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx11<0x00c, "S_BUFFER_LOAD_DWORDX16">;
1186
1187def S_GL1_INV_gfx11    : SMEM_Real_gfx11<0x020, S_GL1_INV>;
1188def S_DCACHE_INV_gfx11 : SMEM_Real_gfx11<0x021, S_DCACHE_INV>;
1189
1190class SMEM_Real_Store_gfx11 <bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx11<op, ps> {
1191  // encoding
1192  bits<7> sdata;
1193
1194  let sdst = ?;
1195  let Inst{12-6}  = !if(ps.has_sdst, sdata{6-0}, ?);
1196}
1197
1198multiclass SM_Real_Probe_gfx11<bits<8> op> {
1199  defvar ps = NAME;
1200  def _IMM_gfx11  : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;
1201  def _SGPR_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>;
1202  def _SGPR_IMM_gfx11
1203    : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>;
1204}
1205
1206defm S_ATC_PROBE        : SM_Real_Probe_gfx11 <0x22>;
1207defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx11 <0x23>;
1208