1
2// Mask immediates for MMA instructions (2, 4 and 8 bits).
3def Msk2Imm : ImmLeaf<i32, [{ return isUInt<2>(Imm); }]>;
4def Msk4Imm : ImmLeaf<i32, [{ return isUInt<4>(Imm); }]>;
5def Msk8Imm : ImmLeaf<i32, [{ return isUInt<8>(Imm); }]>;
6
7def MMA : Predicate<"Subtarget->hasMMA()">;
8
9
10// Multiclass definitions for MMA accumulator instructions.
11// ----------------------------------------------------------------------------
12
13// Defines 2 unmasked instructions where the xo field for acc/non-acc version
14// is even/odd.
15multiclass ACC_UM_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
16                       string asmstr> {
17  let Predicates = [MMA] in {
18  def NAME :
19    XX3Form_AT3_XAB6<opcode, !or(xo, 0x01), (outs acc:$AT), IOL,
20                     !strconcat(asmbase#" ", asmstr), IIC_VecFP, []>,
21    RegConstraint<"@earlyclobber $AT">;
22  def PP :
23    XX3Form_AT3_XAB6<opcode, xo, (outs acc:$AT), !con((ins acc:$ATi), IOL),
24                     !strconcat(asmbase#"pp ", asmstr), IIC_VecFP, []>,
25    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
26  }
27}
28
29// Defines 4 instructions, masked/unmasked with masks 8, 4, 4 bits.
30// The XO field for acc/non-acc version is even/odd.
31multiclass ACC_UM_M844_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
32                            string asmstr> {
33  defm NAME : ACC_UM_XOEO<opcode, xo, IOL, asmbase, asmstr>;
34  let Predicates = [MMA, PrefixInstrs] in {
35  def PM#NAME :
36    MMIRR_XX3Form_XY4P8_XAB6<
37      opcode, !or(xo, 0x01), (outs acc:$AT),
38      !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u8imm:$PMSK)),
39      !strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK, $PMSK"),
40      IIC_VecFP, []>,
41    RegConstraint<"@earlyclobber $AT">;
42  def PM#NAME#PP :
43    MMIRR_XX3Form_XY4P8_XAB6<
44      opcode, xo, (outs acc:$AT),
45      !con((ins acc:$ATi),
46           !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u8imm:$PMSK))),
47      !strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK, $PMSK"),
48      IIC_VecFP, []>,
49    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
50  }
51}
52
53// Defines 4 instructions, masked/unmasked with masks 4, 4, 4 bits.
54// The XO field for acc/non-acc version is even/odd.
55multiclass ACC_UM_M444_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
56                            string asmstr> {
57  defm NAME : ACC_UM_XOEO<opcode, xo, IOL, asmbase, asmstr>;
58  let Predicates = [MMA, PrefixInstrs] in {
59  def PM#NAME :
60    MMIRR_XX3Form_XYP4_XAB6<
61      opcode, !or(xo, 0x01), (outs acc:$AT),
62      !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u4imm:$PMSK)),
63      !strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK, $PMSK"),
64      IIC_VecFP, []>,
65    RegConstraint<"@earlyclobber $AT">;
66  def PM#NAME#PP :
67    MMIRR_XX3Form_XYP4_XAB6<
68      opcode, xo, (outs acc:$AT),
69      !con((ins acc:$ATi),
70           !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u4imm:$PMSK))),
71      !strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK, $PMSK"),
72      IIC_VecFP, []>,
73    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
74  }
75}
76
77// Defines 4 instructions, masked/unmasked with masks 2, 4, 4 bits.
78// The XO field for acc/non-acc version is even/odd.
79multiclass ACC_UM_M244_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
80                            string asmstr> {
81  defm NAME : ACC_UM_XOEO<opcode, xo, IOL, asmbase, asmstr>;
82  let Predicates = [MMA, PrefixInstrs] in {
83  def PM#NAME :
84    MMIRR_XX3Form_XY4P2_XAB6<
85      opcode, !or(xo, 0x01), (outs acc:$AT),
86      !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK)),
87      !strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK, $PMSK"),
88      IIC_VecFP, []>,
89    RegConstraint<"@earlyclobber $AT">;
90  def PM#NAME#PP :
91    MMIRR_XX3Form_XY4P2_XAB6<
92      opcode, xo, (outs acc:$AT),
93      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
94      !strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK, $PMSK"),
95      IIC_VecFP, []>,
96    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
97  }
98}
99
100// Defines 4 instructions, masked/unmasked with masks 2, 4, 4 bits.
101// Upper nibble of XO field for acc/non-acc version is 0x4/0x6.
102multiclass ACC_UM_M244_XO46<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
103                            string asmstr> {
104  let Predicates = [MMA] in {
105  def NAME :
106    XX3Form_AT3_XAB6<opcode, xo, (outs acc:$AT), IOL,
107                     !strconcat(asmbase#" ", asmstr), IIC_VecFP, []>,
108    RegConstraint<"@earlyclobber $AT">;
109  def PP :
110    XX3Form_AT3_XAB6<
111      opcode, !or(xo, 0x20), (outs acc:$AT), !con((ins acc:$ATi), IOL),
112      !strconcat(asmbase#"pp ", asmstr), IIC_VecFP, []>,
113    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
114  }
115  let Predicates = [MMA, PrefixInstrs] in {
116  def PM#NAME :
117    MMIRR_XX3Form_XY4P2_XAB6<
118      opcode, xo, (outs acc:$AT),
119      !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK)),
120      !strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK, $PMSK"),
121      IIC_VecFP, []>,
122    RegConstraint<"@earlyclobber $AT">;
123  def PM#NAME#PP :
124    MMIRR_XX3Form_XY4P2_XAB6<
125      opcode, !or(xo, 0x20), (outs acc:$AT),
126      !con((ins acc:$ATi),
127           !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
128      !strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK, $PMSK"),
129      IIC_VecFP, []>,
130    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
131  }
132}
133
134// Defines 10 instructions, operand negating, unmasked, masked with 2, 4, 4
135// bits. Upper nibble are masked with 0x8, 0x4, 0xC for negating operands.
136multiclass ACC_NEG_UM_M244_XOM84C<bits<6> opcode, bits<8> xo, dag IOL,
137                                  string asmbase, string asmstr> {
138  defm NAME : ACC_UM_M244_XOEO<opcode, xo, IOL, asmbase, asmstr>;
139  let Predicates = [MMA] in {
140  def PN : XX3Form_AT3_XAB6<
141             opcode, !or(xo, 0x80), (outs acc:$AT), !con((ins acc:$ATi), IOL),
142             !strconcat(asmbase#"pn ", asmstr), IIC_VecFP, []>,
143           RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
144  def NP : XX3Form_AT3_XAB6<
145             opcode, !or(xo, 0x40), (outs acc:$AT), !con((ins acc:$ATi), IOL),
146             !strconcat(asmbase#"np ", asmstr), IIC_VecFP, []>,
147           RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
148  def NN : XX3Form_AT3_XAB6<
149             opcode, !or(xo, 0xC0), (outs acc:$AT), !con((ins acc:$ATi), IOL),
150             !strconcat(asmbase#"nn ", asmstr), IIC_VecFP, []>,
151           RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
152  }
153  let Predicates = [MMA, PrefixInstrs] in {
154  def PM#NAME#PN :
155    MMIRR_XX3Form_XY4P2_XAB6<
156      opcode, !or(xo, 0x80), (outs acc:$AT),
157      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
158      !strconcat("pm"#asmbase#"pn ", asmstr#", $XMSK, $YMSK, $PMSK"),
159      IIC_VecFP, []>,
160    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
161  def PM#NAME#NP :
162    MMIRR_XX3Form_XY4P2_XAB6<
163      opcode, !or(xo, 0x40), (outs acc:$AT),
164      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
165      !strconcat("pm"#asmbase#"np ", asmstr#", $XMSK, $YMSK, $PMSK"),
166      IIC_VecFP, []>,
167    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
168  def PM#NAME#NN :
169    MMIRR_XX3Form_XY4P2_XAB6<
170      opcode, !or(xo, 0xC0), (outs acc:$AT),
171      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
172      !strconcat("pm"#asmbase#"nn ", asmstr#", $XMSK, $YMSK, $PMSK"),
173      IIC_VecFP, []>,
174    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
175  }
176}
177
178// Defines 5 instructions, unmasked, operand negating.
179// Upper nibble are masked with 0x8, 0x4, 0xC for negating operands.
180multiclass ACC_NEG_UM_XOM84C<bits<6> opcode, bits<8> xo, dag IOL,
181                             string asmbase, string asmstr> {
182  defm NAME : ACC_UM_XOEO<opcode, xo, IOL, asmbase, asmstr>;
183  let Predicates = [MMA] in {
184  def PN : XX3Form_AT3_XAB6<opcode, !or(xo, 0x80), (outs acc:$AT),
185                            !con((ins acc:$ATi), IOL),
186                            !strconcat(asmbase#"pn ", asmstr), IIC_VecFP, []>,
187           RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
188  def NP : XX3Form_AT3_XAB6<opcode, !or(xo, 0x40), (outs acc:$AT),
189                            !con((ins acc:$ATi), IOL),
190                            !strconcat(asmbase#"np ", asmstr), IIC_VecFP, []>,
191           RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
192  def NN : XX3Form_AT3_XAB6<opcode, !or(xo, 0xC0), (outs acc:$AT),
193                            !con((ins acc:$ATi), IOL),
194                            !strconcat(asmbase#"nn ", asmstr), IIC_VecFP, []>,
195           RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
196  }
197}
198
199// Defines 10 instructions, operand negating, unmasked, masked with 4, 4 bits.
200// Upper nibble are masked with 0x8, 0x4, 0xC for negating operands.
201multiclass ACC_NEG_UM_M44_XOM84C<bits<6> opcode, bits<8> xo, dag IOL,
202                                 string asmbase, string asmstr> {
203  defm NAME : ACC_NEG_UM_XOM84C<opcode, xo, IOL, asmbase, asmstr>;
204  let Predicates = [MMA, PrefixInstrs] in {
205  def PM#NAME :
206    MMIRR_XX3Form_XY4_XAB6<
207      opcode, !or(xo, 0x01), (outs acc:$AT),
208      !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK)),
209      !strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK"),
210      IIC_VecFP, []>,
211    RegConstraint<"@earlyclobber $AT">;
212  def PM#NAME#PP :
213    MMIRR_XX3Form_XY4_XAB6<
214      opcode, xo, (outs acc:$AT),
215      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK))),
216      !strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK"),
217      IIC_VecFP, []>,
218    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
219  def PM#NAME#PN :
220    MMIRR_XX3Form_XY4_XAB6<
221      opcode, !or(xo, 0x80), (outs acc:$AT),
222      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK))),
223      !strconcat("pm"#asmbase#"pn ", asmstr#", $XMSK, $YMSK"),
224      IIC_VecFP, []>,
225    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
226  def PM#NAME#NP :
227    MMIRR_XX3Form_XY4_XAB6<
228      opcode, !or(xo, 0x40), (outs acc:$AT),
229      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK))),
230      !strconcat("pm"#asmbase#"np ", asmstr#", $XMSK, $YMSK"),
231      IIC_VecFP, []>,
232    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
233  def PM#NAME#NN :
234    MMIRR_XX3Form_XY4_XAB6<
235      opcode, !or(xo, 0xC0), (outs acc:$AT),
236      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK))),
237      !strconcat("pm"#asmbase#"nn ", asmstr#", $XMSK, $YMSK"),
238      IIC_VecFP, []>,
239    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
240  }
241}
242
243// Defines 10 instructions, operand negating, unmasked, masked with 4, 2 bits.
244// Upper nibble are masked with 0x8, 0x4, 0xC for negating operands.
245multiclass ACC_NEG_UM_M42_XOM84C<bits<6> opcode, bits<8> xo, dag IOL,
246                                 string asmbase, string asmstr> {
247  defm NAME : ACC_NEG_UM_XOM84C<opcode, xo, IOL, asmbase, asmstr>;
248  let Predicates = [MMA, PrefixInstrs] in {
249  def PM#NAME :
250    MMIRR_XX3Form_X4Y2_XAB6<
251      opcode, !or(xo, 0x01), (outs acc:$AT),
252      !con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK)),
253      !strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK"),
254      IIC_VecFP, []>,
255    RegConstraint<"@earlyclobber $AT">;
256  def PM#NAME#PP :
257    MMIRR_XX3Form_X4Y2_XAB6<
258      opcode, xo, (outs acc:$AT),
259      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK))),
260      !strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK"),
261      IIC_VecFP, []>,
262    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
263  def PM#NAME#PN :
264    MMIRR_XX3Form_X4Y2_XAB6<
265      opcode, !or(xo, 0x80), (outs acc:$AT),
266      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK))),
267      !strconcat("pm"#asmbase#"pn ", asmstr#", $XMSK, $YMSK"),
268      IIC_VecFP, []>,
269    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
270  def PM#NAME#NP :
271    MMIRR_XX3Form_X4Y2_XAB6<
272      opcode, !or(xo, 0x40), (outs acc:$AT),
273      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK))),
274      !strconcat("pm"#asmbase#"np ", asmstr#", $XMSK, $YMSK"),
275      IIC_VecFP, []>,
276    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
277  def PM#NAME#NN :
278    MMIRR_XX3Form_X4Y2_XAB6<
279      opcode, !or(xo, 0xC0), (outs acc:$AT),
280      !con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK))),
281      !strconcat("pm"#asmbase#"nn ", asmstr#", $XMSK, $YMSK"),
282      IIC_VecFP, []>,
283    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
284  }
285}
286
287// End of class definitions.
288//-----------------------------------------------------------------------------
289
290let Predicates = [MMA] in {
291  def XXMFACC :
292    XForm_AT3<31, 0, 177, (outs acc:$ASo), (ins acc:$AS), "xxmfacc $AS",
293              IIC_VecGeneral,
294              [(set v512i1:$ASo, (int_ppc_mma_xxmfacc v512i1:$AS))]>,
295              RegConstraint<"$ASo = $AS">, NoEncode<"$ASo">;
296  def XXMTACC :
297    XForm_AT3<31, 1, 177, (outs acc:$AT), (ins acc:$ATi), "xxmtacc $AT",
298              IIC_VecGeneral,
299              [(set v512i1:$AT, (int_ppc_mma_xxmtacc v512i1:$ATi))]>,
300              RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
301  def KILL_PAIR : PPCPostRAExpPseudo<(outs vsrprc:$XTp), (ins vsrprc:$XSp),
302                                      "#KILL_PAIR", []>,
303                                      RegConstraint<"$XTp = $XSp">;
304  def BUILD_UACC : PPCPostRAExpPseudo<(outs acc:$AT), (ins uacc:$AS),
305                                      "#BUILD_UACC $AT, $AS", []>;
306  // We define XXSETACCZ as rematerializable to undo CSE of that intrinsic in
307  // the backend. We avoid CSE here because it generates a copy of the acc
308  // register and this copy is more expensive than calling the intrinsic again.
309  let isAsCheapAsAMove = 1, isReMaterializable = 1 in {
310    def XXSETACCZ :
311      XForm_AT3<31, 3, 177, (outs acc:$AT), (ins), "xxsetaccz $AT", IIC_VecGeneral,
312                [(set v512i1:$AT, (int_ppc_mma_xxsetaccz))]>;
313  }
314  def XVI8GER4SPP :
315    XX3Form_AT3_XAB6<59, 99, (outs acc:$AT), (ins acc:$ATi, vsrc:$XA, vsrc:$XB),
316                     "xvi8ger4spp $AT, $XA, $XB", IIC_VecGeneral, []>,
317    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
318  let mayStore = 1 in {
319    def SPILL_ACC: PPCEmitTimePseudo<(outs), (ins acc:$AT, memrix16:$dst),
320                                     "#SPILL_ACC", []>;
321    def SPILL_UACC: PPCEmitTimePseudo<(outs), (ins uacc:$AT, memrix16:$dst),
322                                     "#SPILL_UACC", []>;
323  }
324  let mayLoad = 1, hasSideEffects = 0 in {
325    def RESTORE_ACC: PPCEmitTimePseudo<(outs acc:$AT), (ins memrix16:$src),
326                                       "#RESTORE_ACC", []>;
327    def RESTORE_UACC: PPCEmitTimePseudo<(outs uacc:$AT), (ins memrix16:$src),
328                                       "#RESTORE_UACC", []>;
329  }
330}
331
332let Predicates = [MMA, PrefixInstrs] in {
333  def PMXVI8GER4SPP :
334    MMIRR_XX3Form_XYP4_XAB6<59, 99, (outs acc:$AT),
335                            (ins acc:$ATi, vsrc:$XA,vsrc:$XB, u4imm:$XMSK,
336                             u4imm:$YMSK, u4imm:$PMSK),
337                            "pmxvi8ger4spp $AT, $XA, $XB, $XMSK, $YMSK, $PMSK",
338                            IIC_VecGeneral, []>,
339    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
340}
341
342// MMA accumulating/non-accumulating instructions.
343//------------------------------------------------------------------------------
344
345// XVBF16GER2, XVBF16GER2PP, XVBF16GER2PN, XVBF16GER2NP, XVBF16GER2NN
346// PMXVBF16GER2, PMXVBF16GER2PP, PMXVBF16GER2PN, PMXVBF16GER2NP, PMXVBF16GER2NN
347defm XVBF16GER2 : ACC_NEG_UM_M244_XOM84C<59, 50, (ins vsrc:$XA, vsrc:$XB),
348                                         "xvbf16ger2", "$AT, $XA, $XB">;
349
350// XVI4GER8, XVI4GER8PP, PMXVI4GER8,  PMXVI4GER8PP
351defm XVI4GER8 : ACC_UM_M844_XOEO<59, 34, (ins vsrc:$XA, vsrc:$XB),
352                                 "xvi4ger8", "$AT, $XA, $XB">;
353
354// XVI8GER4, XVI8GER4PP, PMXVI8GER4, PMXVI8GER4PP
355defm XVI8GER4 : ACC_UM_M444_XOEO<59, 2, (ins vsrc:$XA, vsrc:$XB),
356                                 "xvi8ger4", "$AT, $XA, $XB">;
357
358// XVI16GER2, XVI16GER2PP, PMXVI16GER2, PMXVI16GER2PP
359defm XVI16GER2 : ACC_UM_M244_XO46<59, 75, (ins vsrc:$XA, vsrc:$XB),
360                                  "xvi16ger2", "$AT, $XA, $XB">;
361
362// XVI16GER2S, XVI16GER2SPP, PMXVI16GER2S, PMXVI16GER2SPP
363defm XVI16GER2S : ACC_UM_M244_XOEO<59, 42, (ins vsrc:$XA, vsrc:$XB),
364                                   "xvi16ger2s", "$AT, $XA, $XB">;
365
366// XVF16GER2, XVF16GER2PP, XVF16GER2PN, XVF16GER2NP, XVF16GER2NN
367// PMXVF16GER2, PMXVF16GER2PP, PMXVF16GER2PN, PMXVF16GER2NP, PMXVF16GER2NN
368defm XVF16GER2 : ACC_NEG_UM_M244_XOM84C<59, 18, (ins vsrc:$XA, vsrc:$XB),
369                                        "xvf16ger2", "$AT, $XA, $XB">;
370
371// XVF32GER, XVF32GERPP, XVF32GERPN, XVF32GERNP, XVF32GERPP
372// PMXVF32GER, PMXVF32GERPP, PMXVF32GERPN, PMXVF32GERNP, PMXVF32GERPP
373defm XVF32GER : ACC_NEG_UM_M44_XOM84C<59, 26, (ins vsrc:$XA, vsrc:$XB),
374                                      "xvf32ger", "$AT, $XA, $XB">;
375
376// XVF64GER, XVF64GERPP, XVF64GERPN, XVF64GERNP, XVF64GERNN
377// PMXVF64GER, PMXVF64GERPP, PMXVF64GERPN, PMXVF64GERNP, PMXVF64GERNN
378defm XVF64GER : ACC_NEG_UM_M42_XOM84C<59, 58, (ins vsrpevenrc:$XA, vsrc:$XB),
379                                      "xvf64ger", "$AT, $XA, $XB">;
380//------------------------------------------------------------------------------
381
382// MMA Intrinsics
383let Predicates = [MMA] in {
384  def : Pat<(v512i1 (int_ppc_mma_xvi4ger8 v16i8:$XA, v16i8:$XB)),
385            (XVI4GER8 RCCp.AToVSRC, RCCp.BToVSRC)>;
386  def : Pat<(v512i1 (int_ppc_mma_xvi4ger8pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
387            (XVI4GER8PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
388
389  def : Pat<(v512i1 (int_ppc_mma_xvi8ger4 v16i8:$XA, v16i8:$XB)),
390            (XVI8GER4 RCCp.AToVSRC, RCCp.BToVSRC)>;
391  def : Pat<(v512i1 (int_ppc_mma_xvi8ger4pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
392            (XVI8GER4PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
393
394  def : Pat<(v512i1 (int_ppc_mma_xvi16ger2s v16i8:$XA, v16i8:$XB)),
395            (XVI16GER2S RCCp.AToVSRC, RCCp.BToVSRC)>;
396  def : Pat<(v512i1 (int_ppc_mma_xvi16ger2spp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
397            (XVI16GER2SPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
398
399  def : Pat<(v512i1 (int_ppc_mma_xvf16ger2 v16i8:$XA, v16i8:$XB)),
400            (XVF16GER2 RCCp.AToVSRC, RCCp.BToVSRC)>;
401  def : Pat<(v512i1 (int_ppc_mma_xvf16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
402            (XVF16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
403  def : Pat<(v512i1 (int_ppc_mma_xvf16ger2pn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
404            (XVF16GER2PN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
405  def : Pat<(v512i1 (int_ppc_mma_xvf16ger2np v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
406            (XVF16GER2NP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
407  def : Pat<(v512i1 (int_ppc_mma_xvf16ger2nn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
408            (XVF16GER2NN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
409
410  def : Pat<(v512i1 (int_ppc_mma_xvf32ger v16i8:$XA, v16i8:$XB)),
411            (XVF32GER RCCp.AToVSRC, RCCp.BToVSRC)>;
412  def : Pat<(v512i1 (int_ppc_mma_xvf32gerpp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
413            (XVF32GERPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
414  def : Pat<(v512i1 (int_ppc_mma_xvf32gerpn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
415            (XVF32GERPN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
416  def : Pat<(v512i1 (int_ppc_mma_xvf32gernp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
417            (XVF32GERNP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
418  def : Pat<(v512i1 (int_ppc_mma_xvf32gernn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
419            (XVF32GERNN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
420  def : Pat<(v512i1 (int_ppc_mma_xvf64ger v256i1:$XA, v16i8:$XB)),
421            (XVF64GER $XA, RCCp.BToVSRC)>;
422  def : Pat<(v512i1 (int_ppc_mma_xvf64gerpp v512i1:$ATi, v256i1:$XA, v16i8:$XB)),
423            (XVF64GERPP $ATi, $XA, RCCp.BToVSRC)>;
424  def : Pat<(v512i1 (int_ppc_mma_xvf64gerpn v512i1:$ATi, v256i1:$XA, v16i8:$XB)),
425            (XVF64GERPN $ATi, $XA, RCCp.BToVSRC)>;
426  def : Pat<(v512i1 (int_ppc_mma_xvf64gernp v512i1:$ATi, v256i1:$XA, v16i8:$XB)),
427            (XVF64GERNP $ATi, $XA, RCCp.BToVSRC)>;
428  def : Pat<(v512i1 (int_ppc_mma_xvf64gernn v512i1:$ATi, v256i1:$XA, v16i8:$XB)),
429            (XVF64GERNN $ATi, $XA, RCCp.BToVSRC)>;
430
431  def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2 v16i8:$XA, v16i8:$XB)),
432            (XVBF16GER2 RCCp.AToVSRC, RCCp.BToVSRC)>;
433  def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
434            (XVBF16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
435  def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2pn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
436            (XVBF16GER2PN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
437  def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2np v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
438            (XVBF16GER2NP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
439  def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2nn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
440            (XVBF16GER2NN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
441  def : Pat<(v512i1 (int_ppc_mma_xvi16ger2 v16i8:$XA, v16i8:$XB)),
442            (XVI16GER2 RCCp.AToVSRC, RCCp.BToVSRC)>;
443  def : Pat<(v512i1 (int_ppc_mma_xvi16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
444            (XVI16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
445  def : Pat<(v512i1 (int_ppc_mma_xvi8ger4spp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
446            (XVI8GER4SPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
447}
448
449// MMA Intrinsics
450let Predicates = [MMA, PrefixInstrs] in {
451  def : Pat<(v512i1 (int_ppc_mma_pmxvi4ger8 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
452                                            Msk4Imm:$YMSK, Msk8Imm:$PMSK)),
453            (PMXVI4GER8 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
454                        Msk4Imm:$YMSK, Msk8Imm:$PMSK)>;
455  def : Pat<(v512i1 (int_ppc_mma_pmxvi4ger8pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
456                                              Msk4Imm:$XMSK, Msk4Imm:$YMSK,
457                                              Msk8Imm:$PMSK)),
458            (PMXVI4GER8PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
459                          Msk4Imm:$YMSK, Msk8Imm:$PMSK)>;
460
461  def : Pat<(v512i1 (int_ppc_mma_pmxvi8ger4 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
462                                            Msk4Imm:$YMSK, Msk4Imm:$PMSK)),
463            (PMXVI8GER4 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
464                        Msk4Imm:$YMSK, Msk4Imm:$PMSK)>;
465  def : Pat<(v512i1 (int_ppc_mma_pmxvi8ger4pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
466                                              Msk4Imm:$XMSK, Msk4Imm:$YMSK,
467                                              Msk4Imm:$PMSK)),
468            (PMXVI8GER4PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
469                          Msk4Imm:$YMSK, Msk4Imm:$PMSK)>;
470
471  def : Pat<(v512i1 (int_ppc_mma_pmxvi16ger2s v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
472                                              Msk4Imm:$YMSK, Msk2Imm:$PMSK)),
473            (PMXVI16GER2S RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
474                          Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
475  def : Pat<(v512i1 (int_ppc_mma_pmxvi16ger2spp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
476                                                Msk4Imm:$XMSK, Msk4Imm:$YMSK,
477                                                Msk2Imm:$PMSK)),
478            (PMXVI16GER2SPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
479                            Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
480  def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
481                                             Msk4Imm:$YMSK, Msk2Imm:$PMSK)),
482            (PMXVF16GER2 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
483                         Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
484  def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
485                                               Msk4Imm:$XMSK, Msk4Imm:$YMSK,
486                                               Msk2Imm:$PMSK)),
487            (PMXVF16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
488                           Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
489  def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2pn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
490                                               Msk4Imm:$XMSK, Msk4Imm:$YMSK,
491                                               Msk2Imm:$PMSK)),
492            (PMXVF16GER2PN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
493                           Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
494  def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2np v512i1:$ATi, v16i8:$XA, v16i8:$XB,
495                                               Msk4Imm:$XMSK, Msk4Imm:$YMSK,
496                                               Msk2Imm:$PMSK)),
497            (PMXVF16GER2NP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
498                           Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
499  def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2nn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
500                                               Msk4Imm:$XMSK, Msk4Imm:$YMSK,
501                                               Msk2Imm:$PMSK)),
502            (PMXVF16GER2NN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
503                           Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
504
505  def : Pat<(v512i1 (int_ppc_mma_pmxvf32ger v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
506                                            Msk4Imm:$YMSK)),
507            (PMXVF32GER RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
508                        Msk4Imm:$YMSK)>;
509  def : Pat<(v512i1 (int_ppc_mma_pmxvf32gerpp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
510                                              Msk4Imm:$XMSK, Msk4Imm:$YMSK)),
511            (PMXVF32GERPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
512                          Msk4Imm:$YMSK)>;
513  def : Pat<(v512i1 (int_ppc_mma_pmxvf32gerpn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
514                                              Msk4Imm:$XMSK, Msk4Imm:$YMSK)),
515            (PMXVF32GERPN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
516                          Msk4Imm:$YMSK)>;
517  def : Pat<(v512i1 (int_ppc_mma_pmxvf32gernp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
518                                              Msk4Imm:$XMSK, Msk4Imm:$YMSK)),
519            (PMXVF32GERNP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
520                          Msk4Imm:$YMSK)>;
521  def : Pat<(v512i1 (int_ppc_mma_pmxvf32gernn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
522                                              Msk4Imm:$XMSK, Msk4Imm:$YMSK)),
523            (PMXVF32GERNN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
524                          Msk4Imm:$YMSK)>;
525
526  def : Pat<(v512i1 (int_ppc_mma_pmxvf64ger v256i1:$XA, v16i8:$XB, Msk4Imm:$XMSK,
527                                            Msk2Imm:$YMSK)),
528            (PMXVF64GER $XA, RCCp.BToVSRC, Msk4Imm:$XMSK, Msk2Imm:$YMSK)>;
529  def : Pat<(v512i1 (int_ppc_mma_pmxvf64gerpp v512i1:$ATi, v256i1:$XA, v16i8:$XB,
530                                              Msk4Imm:$XMSK, Msk2Imm:$YMSK)),
531            (PMXVF64GERPP $ATi, $XA, RCCp.BToVSRC, Msk4Imm:$XMSK,
532                          Msk2Imm:$YMSK)>;
533  def : Pat<(v512i1 (int_ppc_mma_pmxvf64gerpn v512i1:$ATi, v256i1:$XA, v16i8:$XB,
534                                              Msk4Imm:$XMSK, Msk2Imm:$YMSK)),
535            (PMXVF64GERPN $ATi, $XA, RCCp.BToVSRC, Msk4Imm:$XMSK,
536                          Msk2Imm:$YMSK)>;
537  def : Pat<(v512i1 (int_ppc_mma_pmxvf64gernp v512i1:$ATi, v256i1:$XA, v16i8:$XB,
538                                              Msk4Imm:$XMSK, Msk2Imm:$YMSK)),
539            (PMXVF64GERNP $ATi, $XA, RCCp.BToVSRC, Msk4Imm:$XMSK,
540                          Msk2Imm:$YMSK)>;
541  def : Pat<(v512i1 (int_ppc_mma_pmxvf64gernn v512i1:$ATi, v256i1:$XA, v16i8:$XB,
542                                              Msk4Imm:$XMSK, Msk2Imm:$YMSK)),
543            (PMXVF64GERNN $ATi, $XA, RCCp.BToVSRC, Msk4Imm:$XMSK,
544                          Msk2Imm:$YMSK)>;
545
546  def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
547                                              Msk4Imm:$YMSK, Msk2Imm:$PMSK)),
548            (PMXVBF16GER2 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
549                          Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
550  def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
551                                                Msk4Imm:$XMSK, Msk4Imm:$YMSK,
552                                                Msk2Imm:$PMSK)),
553            (PMXVBF16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
554                            Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
555  def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2pn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
556                                                Msk4Imm:$XMSK, Msk4Imm:$YMSK,
557                                                Msk2Imm:$PMSK)),
558            (PMXVBF16GER2PN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
559                            Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
560  def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2np v512i1:$ATi, v16i8:$XA, v16i8:$XB,
561                                                Msk4Imm:$XMSK, Msk4Imm:$YMSK,
562                                                Msk2Imm:$PMSK)),
563            (PMXVBF16GER2NP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
564                            Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
565  def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2nn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
566                                                Msk4Imm:$XMSK, Msk4Imm:$YMSK,
567                                                Msk2Imm:$PMSK)),
568            (PMXVBF16GER2NN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
569                            Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
570  def : Pat<(v512i1 (int_ppc_mma_pmxvi16ger2 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
571                                             Msk4Imm:$YMSK, Msk2Imm:$PMSK)),
572            (PMXVI16GER2 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
573                         Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
574  def : Pat<(v512i1 (int_ppc_mma_pmxvi8ger4spp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
575                                               Msk4Imm:$XMSK, Msk4Imm:$YMSK,
576                                               Msk2Imm:$PMSK)),
577            (PMXVI8GER4SPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
578                           Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
579  def : Pat<(v512i1 (int_ppc_mma_pmxvi16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
580                                               Msk4Imm:$XMSK, Msk4Imm:$YMSK,
581                                               Msk2Imm:$PMSK)),
582            (PMXVI16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
583                           Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
584}
585
586def ConcatsMMA {
587  dag VecsToVecPair0 =
588    (v256i1 (INSERT_SUBREG
589      (INSERT_SUBREG (IMPLICIT_DEF), $vs0, sub_vsx1),
590      $vs1, sub_vsx0));
591  dag VecsToVecPair1 =
592    (v256i1 (INSERT_SUBREG
593      (INSERT_SUBREG (IMPLICIT_DEF), $vs2, sub_vsx1),
594      $vs3, sub_vsx0));
595  dag VecsToVecQuad =
596    (BUILD_UACC (INSERT_SUBREG
597                  (INSERT_SUBREG (v512i1 (IMPLICIT_DEF)),
598                                 (KILL_PAIR VecsToVecPair0), sub_pair0),
599                  (KILL_PAIR VecsToVecPair1), sub_pair1));
600}
601
602def Extracts {
603  dag Pair0 = (v256i1 (EXTRACT_SUBREG $v, sub_pair0));
604  dag Pair1 = (v256i1 (EXTRACT_SUBREG $v, sub_pair1));
605  dag Vec0 = (v4i32 (EXTRACT_SUBREG Pair0, sub_vsx0));
606  dag Vec1 = (v4i32 (EXTRACT_SUBREG Pair0, sub_vsx1));
607  dag Vec2 = (v4i32 (EXTRACT_SUBREG Pair1, sub_vsx0));
608  dag Vec3 = (v4i32 (EXTRACT_SUBREG Pair1, sub_vsx1));
609}
610
611let Predicates = [MMA] in {
612  def : Pat<(v512i1 (PPCAccBuild v4i32:$vs1, v4i32:$vs0, v4i32:$vs3, v4i32:$vs2)),
613            (XXMTACC ConcatsMMA.VecsToVecQuad)>;
614  def : Pat<(v512i1 (int_ppc_mma_assemble_acc v16i8:$vs1, v16i8:$vs0,
615                                              v16i8:$vs3, v16i8:$vs2)),
616            (XXMTACC ConcatsMMA.VecsToVecQuad)>;
617  def : Pat<(v512i1 (PPCxxmfacc v512i1:$AS)), (XXMFACC acc:$AS)>;
618  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 0)),
619            Extracts.Vec0>;
620  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 1)),
621            Extracts.Vec1>;
622  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 2)),
623            Extracts.Vec2>;
624  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 3)),
625            Extracts.Vec3>;
626}
627
628
629