1//=-- SMEInstrFormats.td -  AArch64 SME Instruction classes -*- tablegen -*--=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// AArch64 Scalable Matrix Extension (SME) Instruction Class Definitions.
10//
11//===----------------------------------------------------------------------===//
12
13def imm_to_tile8   : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAB0>", []>;
14def imm_to_tile16  : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAH0>", []>;
15def imm_to_tile32  : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAS0>", []>;
16def imm_to_tile64  : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAD0>", []>;
17def imm_to_tile128 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAQ0>", []>;
18
19def tileslice8   : ComplexPattern<i32 , 2, "SelectSMETileSlice<15, 1>", []>;
20def tileslice16  : ComplexPattern<i32 , 2, "SelectSMETileSlice<7,  1>", []>;
21def tileslice32  : ComplexPattern<i32 , 2, "SelectSMETileSlice<3,  1>", []>;
22def tileslice64  : ComplexPattern<i32 , 2, "SelectSMETileSlice<1,  1>", []>;
23def tileslice128 : ComplexPattern<i32 , 2, "SelectSMETileSlice<0,  1>", []>; // nop
24
25def tileslicerange3s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<14, 2>", []>;
26def tileslicerange2s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<6,  2>", []>;
27
28def am_sme_indexed_b4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<0,15>", [], [SDNPWantRoot]>;
29
30//===----------------------------------------------------------------------===//
31// SME Pseudo Classes
32//===----------------------------------------------------------------------===//
33
34def getSMEPseudoMap : InstrMapping {
35  let FilterClass = "SMEPseudo2Instr";
36  let RowFields = ["PseudoName"];
37  let ColFields = ["IsInstr"];
38  let KeyCol = ["0"];
39  let ValueCols = [["1"]];
40}
41
42class SMEPseudo2Instr<string name, bit instr> {
43  string PseudoName = name;
44  bit IsInstr = instr;
45}
46
47class sme_outer_product_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag>
48    : Pseudo<(outs), (ins i32imm:$tile, PPR3bAny:$pn, PPR3bAny:$pm,
49                          zpr_ty:$zn, zpr_ty:$zm), []>,
50      Sched<[]> {
51  // Translated to the actual instructions in AArch64ISelLowering.cpp
52  let SMEMatrixType = za_flag;
53  let usesCustomInserter = 1;
54}
55
56class sme2_za_array_2op_multi_single_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty,
57                                            ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag>
58    : SMEPseudo2Instr<name, 0>,
59      Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), []> {
60  let SMEMatrixType = za_flag;
61  let usesCustomInserter = 1;
62}
63
64class sme2_za_array_2op_multi_multi_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty,
65                                           SMEMatrixTypeEnum za_flag>
66    : SMEPseudo2Instr<name, 0>,
67      Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), []> {
68  let SMEMatrixType = za_flag;
69  let usesCustomInserter = 1;
70}
71
72class sme2_za_array_2op_multi_index_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty,
73                                           ZPRRegOp zpr_ty, Operand imm_ty, SMEMatrixTypeEnum za_flag>
74    : SMEPseudo2Instr<name, 0>,
75      Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, imm_ty:$i), []> {
76  let SMEMatrixType = za_flag;
77  let usesCustomInserter = 1;
78}
79
80//===----------------------------------------------------------------------===//
81// SME pattern match helpers.
82//===----------------------------------------------------------------------===//
83
84class SME2_ZA_TwoOp_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty,
85                                     ValueType vt, ComplexPattern tileslice>
86    : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm),
87          (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm)>;
88
89
90class SME2_ZA_TwoOp_VG2_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty,
91                                         ValueType vt, ComplexPattern tileslice>
92    : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm),
93          (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2, vt:$Zn1, zsub0, vt:$Zn2, zsub1),
94                                              zpr_ty:$Zm)>;
95class SME2_ZA_TwoOp_VG4_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty,
96                                         ValueType vt, ComplexPattern tileslice>
97    : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)),
98                     vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm),
99          (!cast<Instruction>(name # _PSEUDO) $base, $offset,
100                                              (REG_SEQUENCE ZPR4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3),
101                                              zpr_ty:$Zm)>;
102
103class SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice>
104    : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2),
105          (!cast<Instruction>(name # _PSEUDO) $base, $offset,
106                                              (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1),
107                                              (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>;
108
109class SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice>
110    : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)),
111                     vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm1, vt:$Zm2, vt:$Zm3, vt:$Zm4),
112          (!cast<Instruction>(name # _PSEUDO) $base, $offset,
113                                              (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3),
114                                              (REG_SEQUENCE ZPR4Mul4, vt:$Zm1, zsub0, vt:$Zm2, zsub1, vt:$Zm3, zsub2, vt:$Zm4, zsub3))>;
115
116class SME2_ZA_TwoOp_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt,
117                                    Operand imm_ty, ComplexPattern tileslice>
118   : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm, (i32 imm_ty:$i)),
119         (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm, (i32 imm_ty:$i))>;
120
121
122class SME2_ZA_TwoOp_VG2_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt,
123                                        Operand imm_ty, ComplexPattern tileslice>
124    : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm, (i32 imm_ty:$i)),
125          (!cast<Instruction>(name # _PSEUDO) $base, $offset,
126                                              (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), zpr_ty:$Zm, imm_ty:$i)>;
127
128class SME2_ZA_TwoOp_VG4_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt,
129                                        Operand imm_ty, ComplexPattern tileslice>
130    : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)),
131                     vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm, (i32 imm_ty:$i)),
132          (!cast<Instruction>(name # _PSEUDO) $base, $offset,
133                                              (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3),
134                                              zpr_ty:$Zm, imm_ty:$i)>;
135
136class SME2_Cvt_VG4_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt>
137    : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, in_vt:$Zn3, in_vt:$Zn4)),
138                  (!cast<Instruction>(name) (REG_SEQUENCE ZPR4Mul4, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1, in_vt:$Zn3, zsub2, in_vt:$Zn4, zsub3))>;
139
140//===----------------------------------------------------------------------===//
141// SME Outer Products
142//===----------------------------------------------------------------------===//
143
144class sme_fp_outer_product_inst<bit S, bits<2> sz, bit op, MatrixTileOperand za_ty,
145                                ZPRRegOp zpr_ty, string mnemonic>
146    : I<(outs za_ty:$ZAda),
147      (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm),
148        mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm",
149        "", []>,
150      Sched<[]> {
151  bits<5> Zm;
152  bits<3> Pm;
153  bits<3> Pn;
154  bits<5> Zn;
155  let Inst{31-25} = 0b1000000;
156  let Inst{24}    = op;
157  let Inst{23}    = 0b1;
158  let Inst{22-21} = sz;
159  let Inst{20-16} = Zm;
160  let Inst{15-13} = Pm;
161  let Inst{12-10} = Pn;
162  let Inst{9-5}   = Zn;
163  let Inst{4}     = S;
164  let Inst{3}     = op;
165
166  let Constraints = "$ZAda = $_ZAda";
167}
168
169multiclass sme_outer_product_fp32<bit S, string mnemonic, SDPatternOperator op> {
170  def NAME : sme_fp_outer_product_inst<S, 0b00, 0b0, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> {
171    bits<2> ZAda;
172    let Inst{1-0} = ZAda;
173    let Inst{2}   = 0b0;
174  }
175
176  def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
177
178  def : Pat<(op timm32_0_3:$tile, (nxv4i1 PPR3bAny:$pn), (nxv4i1 PPR3bAny:$pm),
179                (nxv4f32 ZPR32:$zn), (nxv4f32 ZPR32:$zm)),
180            (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>;
181}
182
183multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op> {
184  def NAME : sme_fp_outer_product_inst<S, 0b10, 0b0, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> {
185    bits<3> ZAda;
186    let Inst{2-0} = ZAda;
187  }
188
189  def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>;
190
191  def : Pat<(op timm32_0_7:$tile, (nxv2i1 PPR3bAny:$pn), (nxv2i1 PPR3bAny:$pm),
192                (nxv2f64 ZPR64:$zn), (nxv2f64 ZPR64:$zm)),
193            (!cast<Instruction>(NAME # _PSEUDO) timm32_0_7:$tile, $pn, $pm, $zn, $zm)>;
194}
195
196multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s>{
197  def NAME : sme_fp_outer_product_inst<s, {0,bf}, 0b1, TileOp16, ZPR16, mnemonic> {
198    bits<1> ZAda;
199    let Inst{2-1} = 0b00;
200    let Inst{0}   = ZAda;
201  }
202}
203
204class sme_int_outer_product_inst<bits<3> opc, bit sz, bit sme2,
205                                 MatrixTileOperand za_ty, ZPRRegOp zpr_ty,
206                                 string mnemonic>
207    : I<(outs za_ty:$ZAda),
208        (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm),
209        mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm",
210        "", []>,
211      Sched<[]> {
212  bits<5> Zm;
213  bits<3> Pm;
214  bits<3> Pn;
215  bits<5> Zn;
216  let Inst{31-25} = 0b1010000;
217  let Inst{24}    = opc{2}; // u0
218  let Inst{23}    = 0b1;
219  let Inst{22}    = sz;
220  let Inst{21}    = opc{1}; // u1
221  let Inst{20-16} = Zm;
222  let Inst{15-13} = Pm;
223  let Inst{12-10} = Pn;
224  let Inst{9-5}   = Zn;
225  let Inst{4}     = opc{0};  //S;
226  let Inst{3}     = sme2;
227
228  let Constraints = "$ZAda = $_ZAda";
229}
230
231multiclass sme_int_outer_product_i32<bits<3> opc, string mnemonic,
232                                     SDPatternOperator op> {
233  def NAME : sme_int_outer_product_inst<opc, 0b0, 0b0,  TileOp32,
234                                        ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> {
235    bits<2> ZAda;
236    let Inst{1-0} = ZAda;
237    let Inst{2}   = 0b0;
238  }
239
240  def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
241
242  def : Pat<(op timm32_0_3:$tile, (nxv16i1 PPR3bAny:$pn), (nxv16i1 PPR3bAny:$pm),
243                (nxv16i8 ZPR8:$zn), (nxv16i8 ZPR8:$zm)),
244            (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>;
245}
246
247multiclass sme_int_outer_product_i64<bits<3> opc, string mnemonic,
248                                     SDPatternOperator op> {
249  def NAME : sme_int_outer_product_inst<opc, 0b1, 0b0, TileOp64,
250                                        ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> {
251    bits<3> ZAda;
252    let Inst{2-0} = ZAda;
253  }
254
255  def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>;
256
257  def : Pat<(op timm32_0_7:$tile, (nxv8i1 PPR3bAny:$pn), (nxv8i1 PPR3bAny:$pm),
258                (nxv8i16 ZPR16:$zn), (nxv8i16 ZPR16:$zm)),
259            (!cast<Instruction>(NAME # _PSEUDO) timm32_0_7:$tile, $pn, $pm, $zn, $zm)>;
260}
261
262class sme_outer_product_widening_inst<bits<3> opc, ZPRRegOp zpr_ty, string mnemonic>
263    : I<(outs TileOp32:$ZAda),
264        (ins  TileOp32:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm),
265        mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm",
266        "", []>,
267      Sched<[]> {
268  bits<5> Zm;
269  bits<3> Pm;
270  bits<3> Pn;
271  bits<5> Zn;
272  bits<2> ZAda;
273  let Inst{31-25} = 0b1000000;
274  let Inst{24}    = !if(opc{2}, 0, 1);
275  let Inst{23-22} = 0b10;
276  let Inst{21}    = opc{1};
277  let Inst{20-16} = Zm;
278  let Inst{15-13} = Pm;
279  let Inst{12-10} = Pn;
280  let Inst{9-5}   = Zn;
281  let Inst{4}     = opc{0};
282  let Inst{3}     = opc{2};
283  let Inst{2}     = 0b0;
284  let Inst{1-0}   = ZAda;
285
286  let Constraints = "$ZAda = $_ZAda";
287}
288
289multiclass sme_bf16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> {
290  def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>;
291
292  def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
293
294  def : Pat<(op timm32_0_3:$tile, (nxv8i1 PPR3bAny:$pn), (nxv8i1 PPR3bAny:$pm),
295                (nxv8bf16 ZPR16:$zn), (nxv8bf16 ZPR16:$zm)),
296            (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>;
297}
298
299multiclass sme_f16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> {
300  def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>;
301
302  def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
303
304  def : Pat<(op timm32_0_3:$tile, (nxv8i1 PPR3bAny:$pn), (nxv8i1 PPR3bAny:$pm),
305                (nxv8f16 ZPR16:$zn), (nxv8f16 ZPR16:$zm)),
306            (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>;
307}
308
309//===----------------------------------------------------------------------===//
310// SME Add Vector to Tile
311//===----------------------------------------------------------------------===//
312
313class sme_add_vector_to_tile_inst<bit op, bit V, MatrixTileOperand tile_ty,
314                                  ZPRRegOp zpr_ty, string mnemonic>
315    : I<(outs tile_ty:$ZAda),
316        (ins tile_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn),
317        mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn",
318        "", []>, Sched<[]> {
319  bits<3> Pm;
320  bits<3> Pn;
321  bits<5> Zn;
322  let Inst{31-23} = 0b110000001;
323  let Inst{22}    = op;
324  let Inst{21-17} = 0b01000;
325  let Inst{16}    = V;
326  let Inst{15-13} = Pm;
327  let Inst{12-10} = Pn;
328  let Inst{9-5}   = Zn;
329  let Inst{4-3}   = 0b00;
330
331  let Constraints = "$ZAda = $_ZAda";
332}
333
334class sme_add_vector_to_tile_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag>
335    : Pseudo<(outs),
336             (ins i32imm:$tile, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), []>,
337      Sched<[]> {
338  // Translated to the actual instructions in AArch64ISelLowering.cpp
339  let SMEMatrixType = za_flag;
340  let usesCustomInserter = 1;
341}
342
343multiclass sme_add_vector_to_tile_u32<bit V, string mnemonic, SDPatternOperator op> {
344    def NAME : sme_add_vector_to_tile_inst<0b0, V, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> {
345  bits<2> ZAda;
346  let Inst{2}   = 0b0;
347  let Inst{1-0} = ZAda;
348  }
349
350  def _PSEUDO_S : sme_add_vector_to_tile_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
351
352  def : Pat<(op timm32_0_3:$tile, (nxv4i1 PPR3bAny:$pn), (nxv4i1 PPR3bAny:$pm),
353            (nxv4i32 ZPR32:$zn)),
354          (!cast<Instruction>(NAME # _PSEUDO_S) timm32_0_3:$tile, $pn, $pm, $zn)>;
355}
356
357multiclass sme_add_vector_to_tile_u64<bit V, string mnemonic, SDPatternOperator op> {
358    def NAME : sme_add_vector_to_tile_inst<0b1, V, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> {
359  bits<3> ZAda;
360  let Inst{2-0} = ZAda;
361  }
362
363  def _PSEUDO_D : sme_add_vector_to_tile_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>;
364
365  let Predicates = [HasSMEI16I64] in {
366  def : Pat<(op timm32_0_7:$tile, (nxv2i1 PPR3bAny:$pn), (nxv2i1 PPR3bAny:$pm),
367                (nxv2i64 ZPR64:$zn)),
368            (!cast<Instruction>(NAME # _PSEUDO_D) timm32_0_7:$tile, $pn, $pm, $zn)>;
369  }
370}
371
372//===----------------------------------------------------------------------===//
373// SME Contiguous Loads
374//===----------------------------------------------------------------------===//
375
376class sme_mem_ld_ss_base<bit Q, bit V, bits<2> msz, dag outs, dag ins,
377                         string mnemonic, string argstr>
378    : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> {
379  bits<5> Rm;
380  bits<2> Rv;
381  bits<3> Pg;
382  bits<5> Rn;
383  let Inst{31-25} = 0b1110000;
384  let Inst{24}    = Q;
385  let Inst{23-22} = msz;
386  let Inst{21}    = 0b0;
387  let Inst{20-16} = Rm;
388  let Inst{15}    = V;
389  let Inst{14-13} = Rv;
390  let Inst{12-10} = Pg;
391  let Inst{9-5}   = Rn;
392  let Inst{4}     = 0b0;
393
394  let mayLoad = 1;
395}
396
397class sme_mem_ld_ss_inst<bit Q, bits<2> msz, string mnemonic,
398                         MatrixTileVectorOperand tile_ty, bit is_col,
399                         Operand imm_ty, RegisterOperand gpr_ty>
400    : sme_mem_ld_ss_base<
401        Q, is_col, msz, (outs tile_ty:$ZAt),
402        (ins MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn,
403             gpr_ty:$Rm),
404        mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg/z, [$Rn, $Rm]">;
405
406multiclass sme_mem_ss_aliases_base<string mnemonic, Instruction inst,
407                                   MatrixTileVectorOperand tile_ty,
408                                   Operand imm_ty, RegisterOperand gpr_ty,
409                                   string pg_suffix=""> {
410  def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn, $Rm]",
411                  (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, gpr_ty:$Rm), 0>;
412  // Default XZR offset aliases
413  def : InstAlias<mnemonic # "\t\\{$ZAt[$Rv, $imm]\\}, $Pg" # pg_suffix # ", [$Rn]",
414                  (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>;
415  def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn]",
416                  (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>;
417}
418
419multiclass sme_mem_ss_aliases<string mnemonic, string inst, bit is_col,
420                              string pg_suffix=""> {
421  defm : sme_mem_ss_aliases_base<mnemonic # "b", !cast<Instruction>(inst # _B),
422                                 !if(is_col, TileVectorOpV8, TileVectorOpH8),
423                                 sme_elm_idx0_15, GPR64shifted8, pg_suffix>;
424  defm : sme_mem_ss_aliases_base<mnemonic # "h", !cast<Instruction>(inst # _H),
425                                 !if(is_col, TileVectorOpV16, TileVectorOpH16),
426                                 sme_elm_idx0_7, GPR64shifted16, pg_suffix>;
427  defm : sme_mem_ss_aliases_base<mnemonic # "w", !cast<Instruction>(inst # _S),
428                                 !if(is_col, TileVectorOpV32, TileVectorOpH32),
429                                 sme_elm_idx0_3, GPR64shifted32, pg_suffix>;
430  defm : sme_mem_ss_aliases_base<mnemonic # "d", !cast<Instruction>(inst # _D),
431                                 !if(is_col, TileVectorOpV64, TileVectorOpH64),
432                                 sme_elm_idx0_1, GPR64shifted64, pg_suffix>;
433  defm : sme_mem_ss_aliases_base<mnemonic # "q", !cast<Instruction>(inst # _Q),
434                                 !if(is_col, TileVectorOpV128, TileVectorOpH128),
435                                 sme_elm_idx0_0, GPR64shifted128, pg_suffix>;
436}
437
438multiclass sme_mem_ld_ss_aliases<string inst, bit is_col> {
439  defm NAME : sme_mem_ss_aliases<"ld1", inst, is_col, "/z">;
440}
441
442multiclass sme_mem_ld_ss_patterns<Instruction Inst, SDPatternOperator Load,
443                                  Operand tile_ty, Operand offset_ty,
444                                  ComplexPattern addr,
445                                  ComplexPattern tileslice> {
446  // base, tileslice
447  def : Pat<(Load PPR3bAny:$pg, GPR64sp:$base, tile_ty:$tile,
448                  (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))),
449            (Inst tile_ty:$tile, $idx, $imm, $pg, $base, XZR)>;
450
451  // reg + reg, tileslice
452  let AddedComplexity = 1 in {
453    def : Pat<(Load PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset),
454                    tile_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx,
455                                              offset_ty:$imm))),
456              (Inst tile_ty:$tile, $idx, $imm, $pg, $base, $offset)>;
457  }
458}
459
460class sme_load_pseudo
461    : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx,
462                          i32imm:$imm, PPR3bAny:$pg, GPR64sp:$base, GPR64:$offset), []>,
463      Sched<[]> {
464  // Translated to the actual instructions in AArch64ISelLowering.cpp
465  let usesCustomInserter = 1;
466  let mayLoad = 1;
467}
468
469multiclass sme_mem_ld_v_ss<string mnemonic, bit is_col> {
470  def _B : sme_mem_ld_ss_inst<0b0, 0b00, mnemonic # "b",
471                              !if(is_col, TileVectorOpV8, TileVectorOpH8),
472                              is_col, sme_elm_idx0_15, GPR64shifted8> {
473    bits<4> imm;
474    let Inst{3-0} = imm;
475  }
476  def _H : sme_mem_ld_ss_inst<0b0, 0b01, mnemonic # "h",
477                              !if(is_col, TileVectorOpV16, TileVectorOpH16),
478                              is_col, sme_elm_idx0_7, GPR64shifted16> {
479    bits<1> ZAt;
480    bits<3> imm;
481    let Inst{3}   = ZAt;
482    let Inst{2-0} = imm;
483  }
484  def _S : sme_mem_ld_ss_inst<0b0, 0b10, mnemonic # "w",
485                              !if(is_col, TileVectorOpV32, TileVectorOpH32),
486                              is_col, sme_elm_idx0_3, GPR64shifted32> {
487    bits<2> ZAt;
488    bits<2> imm;
489    let Inst{3-2} = ZAt;
490    let Inst{1-0} = imm;
491  }
492  def _D : sme_mem_ld_ss_inst<0b0, 0b11, mnemonic # "d",
493                              !if(is_col, TileVectorOpV64, TileVectorOpH64),
494                              is_col, sme_elm_idx0_1, GPR64shifted64> {
495    bits<3> ZAt;
496    bits<1> imm;
497    let Inst{3-1} = ZAt;
498    let Inst{0}   = imm;
499  }
500  def _Q : sme_mem_ld_ss_inst<0b1, 0b11, mnemonic # "q",
501                              !if(is_col, TileVectorOpV128, TileVectorOpH128),
502                              is_col, sme_elm_idx0_0, GPR64shifted128> {
503    bits<4> ZAt;
504    let Inst{3-0} = ZAt;
505  }
506
507  defm : sme_mem_ld_ss_aliases<NAME, is_col>;
508
509  // Pseudo instructions for lowering intrinsics, using immediates instead of
510  // tile registers.
511  def _PSEUDO_B : sme_load_pseudo;
512  def _PSEUDO_H : sme_load_pseudo;
513  def _PSEUDO_S : sme_load_pseudo;
514  def _PSEUDO_D : sme_load_pseudo;
515  def _PSEUDO_Q : sme_load_pseudo;
516
517  defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_B),
518                                !if(is_col, int_aarch64_sme_ld1b_vert,
519                                            int_aarch64_sme_ld1b_horiz),
520                                sme_elm_idx0_0, timm32_0_15, am_sve_regreg_lsl0,
521                                tileslice8>;
522  defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_H),
523                                !if(is_col, int_aarch64_sme_ld1h_vert,
524                                            int_aarch64_sme_ld1h_horiz),
525                                timm32_0_1, timm32_0_7, am_sve_regreg_lsl1,
526                                tileslice16>;
527  defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_S),
528                                !if(is_col, int_aarch64_sme_ld1w_vert,
529                                            int_aarch64_sme_ld1w_horiz),
530                                timm32_0_3, timm32_0_3, am_sve_regreg_lsl2,
531                                tileslice32>;
532  defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_D),
533                                !if(is_col, int_aarch64_sme_ld1d_vert,
534                                            int_aarch64_sme_ld1d_horiz),
535                                timm32_0_7, timm32_0_1, am_sve_regreg_lsl3,
536                                tileslice64>;
537  defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),
538                                !if(is_col, int_aarch64_sme_ld1q_vert,
539                                            int_aarch64_sme_ld1q_horiz),
540                                timm32_0_15, sme_elm_idx0_0, am_sve_regreg_lsl4,
541                                tileslice128>;
542}
543
544multiclass sme_mem_ld_ss<string mnemonic> {
545  defm _H : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b0>;
546  defm _V : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b1>;
547}
548
549//===----------------------------------------------------------------------===//
550// SME Contiguous Stores
551//===----------------------------------------------------------------------===//
552
553class sme_mem_st_ss_base<bit Q, bit V, bits<2> msz, dag ins,
554                         string mnemonic, string argstr>
555    : I<(outs), ins, mnemonic, argstr, "", []>, Sched<[]> {
556  bits<5> Rm;
557  bits<2> Rv;
558  bits<3> Pg;
559  bits<5> Rn;
560  let Inst{31-25} = 0b1110000;
561  let Inst{24}    = Q;
562  let Inst{23-22} = msz;
563  let Inst{21}    = 0b1;
564  let Inst{20-16} = Rm;
565  let Inst{15}    = V;
566  let Inst{14-13} = Rv;
567  let Inst{12-10} = Pg;
568  let Inst{9-5}   = Rn;
569  let Inst{4}     = 0b0;
570
571  let mayStore = 1;
572  let hasSideEffects = 1;
573}
574
575class sme_mem_st_ss_inst<bit Q, bits<2> msz, string mnemonic,
576                         MatrixTileVectorOperand tile_ty, bit is_col,
577                         Operand imm_ty, RegisterOperand gpr_ty>
578    : sme_mem_st_ss_base<
579        Q, is_col, msz,
580        (ins tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg,
581             GPR64sp:$Rn, gpr_ty:$Rm),
582        mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg, [$Rn, $Rm]">;
583
584multiclass sme_mem_st_ss_aliases<string inst, bit is_col> {
585  defm NAME : sme_mem_ss_aliases<"st1", inst, is_col>;
586}
587
588multiclass sme_mem_st_ss_patterns<Instruction Inst, SDPatternOperator Store,
589                                  Operand offset_ty,
590                                  ComplexPattern imm2tile,
591                                  ComplexPattern addr,
592                                  ComplexPattern tileslice> {
593  // base, tileslice
594  def : Pat<(Store PPR3bAny:$pg, GPR64sp:$base, (imm2tile untyped:$tile),
595                   (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))),
596            (Inst $tile, $idx, $imm, $pg, $base, XZR)>;
597
598  // reg + reg, tileslice
599  let AddedComplexity = 1 in {
600    def : Pat<(Store PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset),
601                     (imm2tile untyped:$tile),
602                     (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))),
603              (Inst $tile, $idx, $imm, $pg, $base, $offset)>;
604  }
605}
606
607multiclass sme_mem_st_v_ss<string mnemonic, bit is_col> {
608  def _B : sme_mem_st_ss_inst<0b0, 0b00, mnemonic # "b",
609                              !if(is_col, TileVectorOpV8, TileVectorOpH8),
610                              is_col, sme_elm_idx0_15, GPR64shifted8> {
611    bits<4> imm;
612    let Inst{3-0} = imm;
613  }
614  def _H : sme_mem_st_ss_inst<0b0, 0b01, mnemonic # "h",
615                              !if(is_col, TileVectorOpV16, TileVectorOpH16),
616                              is_col, sme_elm_idx0_7, GPR64shifted16> {
617    bits<1> ZAt;
618    bits<3> imm;
619    let Inst{3}   = ZAt;
620    let Inst{2-0} = imm;
621  }
622  def _S : sme_mem_st_ss_inst<0b0, 0b10, mnemonic # "w",
623                              !if(is_col, TileVectorOpV32, TileVectorOpH32),
624                              is_col, sme_elm_idx0_3, GPR64shifted32> {
625    bits<2> ZAt;
626    bits<2> imm;
627    let Inst{3-2} = ZAt;
628    let Inst{1-0} = imm;
629  }
630  def _D : sme_mem_st_ss_inst<0b0, 0b11, mnemonic # "d",
631                              !if(is_col, TileVectorOpV64, TileVectorOpH64),
632                              is_col, sme_elm_idx0_1, GPR64shifted64> {
633    bits<3> ZAt;
634    bits<1> imm;
635    let Inst{3-1} = ZAt;
636    let Inst{0}   = imm;
637  }
638  def _Q : sme_mem_st_ss_inst<0b1, 0b11, mnemonic # "q",
639                              !if(is_col, TileVectorOpV128, TileVectorOpH128),
640                              is_col, sme_elm_idx0_0, GPR64shifted128> {
641    bits<4> ZAt;
642    let Inst{3-0} = ZAt;
643  }
644
645  defm : sme_mem_st_ss_aliases<NAME, is_col>;
646
647  defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _B),
648                                !if(is_col, int_aarch64_sme_st1b_vert,
649                                            int_aarch64_sme_st1b_horiz),
650                                timm32_0_15, imm_to_tile8, am_sve_regreg_lsl0,
651                                tileslice8>;
652  defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _H),
653                                !if(is_col, int_aarch64_sme_st1h_vert,
654                                            int_aarch64_sme_st1h_horiz),
655                                timm32_0_7, imm_to_tile16, am_sve_regreg_lsl1,
656                                tileslice16>;
657  defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _S),
658                                !if(is_col, int_aarch64_sme_st1w_vert,
659                                            int_aarch64_sme_st1w_horiz),
660                                timm32_0_3, imm_to_tile32, am_sve_regreg_lsl2,
661                                tileslice32>;
662  defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _D),
663                                !if(is_col, int_aarch64_sme_st1d_vert,
664                                            int_aarch64_sme_st1d_horiz),
665                                timm32_0_1, imm_to_tile64, am_sve_regreg_lsl3,
666                                tileslice64>;
667  defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _Q),
668                                !if(is_col, int_aarch64_sme_st1q_vert,
669                                            int_aarch64_sme_st1q_horiz),
670                                sme_elm_idx0_0, imm_to_tile128,
671                                am_sve_regreg_lsl4, tileslice128>;
672}
673
674multiclass sme_mem_st_ss<string mnemonic> {
675  defm _H : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b0>;
676  defm _V : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b1>;
677}
678
679//===----------------------------------------------------------------------===//
680// SME Save and Restore Array
681//===----------------------------------------------------------------------===//
682
683class sme_spill_fill_base<bit isStore, dag outs, dag ins, string opcodestr>
684    : I<outs, ins, opcodestr, "\t$ZAt[$Rv, $imm4], [$Rn, $offset, mul vl]", "",
685        []>,
686      Sched<[]> {
687  bits<2> Rv;
688  bits<5> Rn;
689  bits<4> imm4;
690  let Inst{31-22} = 0b1110000100;
691  let Inst{21}    = isStore;
692  let Inst{20-15} = 0b000000;
693  let Inst{14-13} = Rv;
694  let Inst{12-10} = 0b000;
695  let Inst{9-5}   = Rn;
696  let Inst{4}     = 0b0;
697  let Inst{3-0}   = imm4;
698}
699
700let mayStore = 1 in
701class sme_spill_inst<string opcodestr>
702    : sme_spill_fill_base<0b1, (outs),
703                          (ins MatrixOp:$ZAt, MatrixIndexGPR32Op12_15:$Rv,
704                               sme_elm_idx0_15:$imm4, GPR64sp:$Rn,
705                               imm0_15:$offset),
706                          opcodestr>;
707let mayLoad = 1 in
708class sme_fill_inst<string opcodestr>
709    : sme_spill_fill_base<0b0, (outs MatrixOp:$ZAt),
710                          (ins MatrixIndexGPR32Op12_15:$Rv,
711                               sme_elm_idx0_15:$imm4, GPR64sp:$Rn,
712                               imm0_15:$offset),
713                          opcodestr>;
714multiclass sme_spill<string opcodestr> {
715  def NAME : sme_spill_inst<opcodestr>;
716  def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]",
717                  (!cast<Instruction>(NAME) MatrixOp:$ZAt,
718                   MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>;
719  // base
720  def : Pat<(int_aarch64_sme_str MatrixIndexGPR32Op12_15:$idx, GPR64sp:$base),
721            (!cast<Instruction>(NAME) ZA, $idx, 0, $base, 0)>;
722  // scalar + immediate (mul vl)
723  let AddedComplexity = 2 in {
724    def : Pat<(int_aarch64_sme_str MatrixIndexGPR32Op12_15:$idx,
725                                   (am_sme_indexed_b4 GPR64sp:$base, imm0_15:$imm4)),
726              (!cast<Instruction>(NAME) ZA, $idx, 0, $base, $imm4)>;
727  }
728}
729
730multiclass sme_fill<string opcodestr> {
731  def NAME : sme_fill_inst<opcodestr>;
732  def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]",
733                  (!cast<Instruction>(NAME) MatrixOp:$ZAt,
734                   MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>;
735  def NAME # _PSEUDO
736      : Pseudo<(outs),
737               (ins MatrixIndexGPR32Op12_15:$idx, imm0_15:$imm4,
738                    GPR64sp:$base), []>,
739        Sched<[]> {
740    // Translated to actual instruction in AArch64ISelLowering.cpp
741    let usesCustomInserter = 1;
742    let mayLoad = 1;
743  }
744  // base
745  def : Pat<(int_aarch64_sme_ldr MatrixIndexGPR32Op12_15:$idx, GPR64sp:$base),
746            (!cast<Instruction>(NAME # _PSEUDO) $idx, 0, $base)>;
747  // scalar + immediate (mul vl)
748  let AddedComplexity = 2 in {
749    def : Pat<(int_aarch64_sme_ldr MatrixIndexGPR32Op12_15:$idx,
750                                   (am_sme_indexed_b4 GPR64sp:$base, imm0_15:$imm4)),
751              (!cast<Instruction>(NAME # _PSEUDO) $idx, $imm4, $base)>;
752  }
753}
754
755//===----------------------------------------------------------------------===//
756// Move instructions
757//===----------------------------------------------------------------------===//
758
759class sme_vector_to_tile_base<bit Q, bit V, bits<2> sz, dag outs, dag ins,
760                              string mnemonic, string argstr>
761    : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> {
762  bits<2> Rv;
763  bits<3> Pg;
764  bits<5> Zn;
765  let Inst{31-24} = 0b11000000;
766  let Inst{23-22} = sz;
767  let Inst{21-17} = 0b00000;
768  let Inst{16}    = Q;
769  let Inst{15}    = V;
770  let Inst{14-13} = Rv;
771  let Inst{12-10} = Pg;
772  let Inst{9-5}   = Zn;
773  let Inst{4}     = 0b0;
774}
775
776class sme_vector_to_tile_inst<bit Q, bits<2> sz, MatrixTileVectorOperand tile_ty,
777                              bit is_col, Operand imm_ty, ZPRRegOp zpr_ty,
778                              string mnemonic>
779    : sme_vector_to_tile_base<Q, is_col, sz, (outs tile_ty:$ZAd),
780        (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn),
781        mnemonic, "\t$ZAd[$Rv, $imm], $Pg/m, $Zn">{
782
783  let Constraints = "$ZAd = $_ZAd";
784}
785
786
787multiclass sme_vector_to_tile_aliases<Instruction inst,
788                                      MatrixTileVectorOperand tile_ty,
789                                      ZPRRegOp zpr_ty, Operand imm_ty> {
790  def : InstAlias<"mov\t$ZAd[$Rv, $imm], $Pg/m, $Zn",
791                  (inst tile_ty:$ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), 1>;
792}
793
794multiclass sme_vector_to_tile_patterns<Instruction inst, ValueType zpr_vt,
795                                       ValueType ppr_vt, Operand imm_ty,
796                                       Operand offset_ty,
797                                       SDPatternOperator op,
798                                       ComplexPattern tileslice> {
799  def : Pat<(op imm_ty:$tile, MatrixIndexGPR32Op12_15:$idx,
800                (ppr_vt PPR3bAny:$pg), (zpr_vt ZPRAny:$zn)),
801            (inst imm_ty:$tile, $idx, 0, $pg, $zn)>;
802  let AddedComplexity = 1 in {
803    def : Pat<(op imm_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx,
804                                                offset_ty:$imm)),
805                  (ppr_vt PPR3bAny:$pg), (zpr_vt ZPRAny:$zn)),
806              (inst imm_ty:$tile, $idx, $imm, $pg, $zn)>;
807  }
808}
809
810class sme_mova_insert_pseudo<SMEMatrixTypeEnum za_flag>
811    : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx,
812                          i32imm:$imm, PPR3bAny:$pg, ZPRAny:$zn), []>,
813      Sched<[]> {
814  // Translated to the actual instructions in AArch64ISelLowering.cpp
815  let SMEMatrixType = za_flag;
816  let usesCustomInserter = 1;
817}
818
819multiclass sme_vector_v_to_tile<string mnemonic, bit is_col> {
820  def _B : sme_vector_to_tile_inst<0b0, 0b00, !if(is_col, TileVectorOpV8,
821                                                          TileVectorOpH8),
822                                   is_col, sme_elm_idx0_15, ZPR8, mnemonic>,
823                                   SMEPseudo2Instr<NAME # _B, 1> {
824    bits<4> imm;
825    let Inst{3-0} = imm;
826  }
827  def _H : sme_vector_to_tile_inst<0b0, 0b01, !if(is_col, TileVectorOpV16,
828                                                          TileVectorOpH16),
829                                   is_col, sme_elm_idx0_7, ZPR16, mnemonic>,
830                                   SMEPseudo2Instr<NAME # _H, 1> {
831    bits<1> ZAd;
832    bits<3> imm;
833    let Inst{3}   = ZAd;
834    let Inst{2-0} = imm;
835  }
836  def _S : sme_vector_to_tile_inst<0b0, 0b10, !if(is_col, TileVectorOpV32,
837                                                          TileVectorOpH32),
838                                   is_col, sme_elm_idx0_3, ZPR32, mnemonic>,
839                                   SMEPseudo2Instr<NAME # _S, 1> {
840    bits<2> ZAd;
841    bits<2> imm;
842    let Inst{3-2} = ZAd;
843    let Inst{1-0} = imm;
844  }
845  def _D : sme_vector_to_tile_inst<0b0, 0b11, !if(is_col, TileVectorOpV64,
846                                                          TileVectorOpH64),
847                                   is_col, sme_elm_idx0_1, ZPR64, mnemonic>,
848                                   SMEPseudo2Instr<NAME # _D, 1> {
849    bits<3> ZAd;
850    bits<1> imm;
851    let Inst{3-1} = ZAd;
852    let Inst{0}   = imm;
853  }
854  def _Q : sme_vector_to_tile_inst<0b1, 0b11, !if(is_col, TileVectorOpV128,
855                                                          TileVectorOpH128),
856                                   is_col, sme_elm_idx0_0, ZPR128, mnemonic>,
857                                   SMEPseudo2Instr<NAME # _Q, 1> {
858    bits<4> ZAd;
859    bits<1> imm;
860    let Inst{3-0} = ZAd;
861  }
862
863  // Pseudo instructions for lowering intrinsics, using immediates instead of
864  // tile registers.
865  def _PSEUDO_B : sme_mova_insert_pseudo<SMEMatrixTileB>, SMEPseudo2Instr<NAME # _B, 0>;
866  def _PSEUDO_H : sme_mova_insert_pseudo<SMEMatrixTileH>, SMEPseudo2Instr<NAME # _H, 0>;
867  def _PSEUDO_S : sme_mova_insert_pseudo<SMEMatrixTileS>, SMEPseudo2Instr<NAME # _S, 0>;
868  def _PSEUDO_D : sme_mova_insert_pseudo<SMEMatrixTileD>, SMEPseudo2Instr<NAME # _D, 0>;
869  def _PSEUDO_Q : sme_mova_insert_pseudo<SMEMatrixTileQ>, SMEPseudo2Instr<NAME # _Q, 0>;
870
871  defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _B),
872                                    !if(is_col, TileVectorOpV8,
873                                                TileVectorOpH8),
874                                    ZPR8, sme_elm_idx0_15>;
875  defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _H),
876                                    !if(is_col, TileVectorOpV16,
877                                                TileVectorOpH16),
878                                    ZPR16, sme_elm_idx0_7>;
879  defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _S),
880                                    !if(is_col, TileVectorOpV32,
881                                                TileVectorOpH32),
882                                    ZPR32, sme_elm_idx0_3>;
883  defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _D),
884                                    !if(is_col, TileVectorOpV64,
885                                                TileVectorOpH64),
886                                    ZPR64, sme_elm_idx0_1>;
887  defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _Q),
888                                    !if(is_col, TileVectorOpV128,
889                                                TileVectorOpH128),
890                                    ZPR128, sme_elm_idx0_0>;
891
892  defvar op = !if(is_col, int_aarch64_sme_write_vert,
893                          int_aarch64_sme_write_horiz);
894
895  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_B),
896                                     nxv16i8, nxv16i1, sme_elm_idx0_0, sme_elm_idx0_15,
897                                     op, tileslice8>;
898  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H),
899                                     nxv8i16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7,
900                                     op, tileslice16>;
901  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H),
902                                     nxv8f16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7,
903                                     op, tileslice16>;
904  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H),
905                                     nxv8bf16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7,
906                                     op, tileslice16>;
907  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S),
908                                     nxv4i32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3,
909                                     op, tileslice32>;
910  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S),
911                                     nxv4f32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3,
912                                     op, tileslice32>;
913  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D),
914                                     nxv2i64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1,
915                                     op, tileslice64>;
916  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D),
917                                     nxv2f64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1,
918                                     op, tileslice64>;
919
920  defvar opq = !if(is_col, int_aarch64_sme_writeq_vert,
921                           int_aarch64_sme_writeq_horiz);
922
923  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),
924                                     nxv16i8, nxv16i1, sme_elm_idx0_15,
925                                     sme_elm_idx0_0, opq, tileslice128>;
926  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),
927                                     nxv8i16, nxv8i1, sme_elm_idx0_15,
928                                     sme_elm_idx0_0, opq, tileslice128>;
929  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),
930                                     nxv8f16, nxv8i1, sme_elm_idx0_15,
931                                     sme_elm_idx0_0, opq, tileslice128>;
932  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),
933                                     nxv8bf16, nxv8i1, sme_elm_idx0_15,
934                                     sme_elm_idx0_0, opq, tileslice128>;
935  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),
936                                     nxv4i32, nxv4i1, sme_elm_idx0_15,
937                                     sme_elm_idx0_0, opq, tileslice128>;
938  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),
939                                     nxv4f32, nxv4i1, sme_elm_idx0_15,
940                                     sme_elm_idx0_0, opq, tileslice128>;
941  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),
942                                     nxv2i64, nxv2i1, sme_elm_idx0_15,
943                                     sme_elm_idx0_0, opq, tileslice128>;
944  defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),
945                                     nxv2f64, nxv2i1, sme_elm_idx0_15,
946                                     sme_elm_idx0_0, opq, tileslice128>;
947}
948
949multiclass sme_vector_to_tile<string mnemonic> {
950  defm _H : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b0>;
951  defm _V : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b1>;
952}
953
954class sme_tile_to_vector_base<bit Q, bit V, bits<2> sz, dag outs, dag ins,
955                              string mnemonic, string argstr>
956    : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> {
957  bits<2> Rv;
958  bits<3> Pg;
959  bits<5> Zd;
960  let Inst{31-24} = 0b11000000;
961  let Inst{23-22} = sz;
962  let Inst{21-17} = 0b00001;
963  let Inst{16}    = Q;
964  let Inst{15}    = V;
965  let Inst{14-13} = Rv;
966  let Inst{12-10} = Pg;
967  let Inst{9}     = 0b0;
968  let Inst{4-0}   = Zd;
969}
970
971class sme_tile_to_vector_inst<bit Q, bits<2> sz, ZPRRegOp zpr_ty,
972                              MatrixTileVectorOperand tile_ty,
973                              bit is_col, Operand imm_ty, string mnemonic>
974    : sme_tile_to_vector_base<Q, is_col, sz, (outs zpr_ty:$Zd),
975        (ins zpr_ty:$_Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm),
976        mnemonic, "\t$Zd, $Pg/m, $ZAn[$Rv, $imm]"> {
977
978  let Constraints = "$Zd = $_Zd";
979}
980
981multiclass sme_tile_to_vector_aliases<Instruction inst, ZPRRegOp zpr_ty,
982                                      MatrixTileVectorOperand tile_ty,
983                                      Operand imm_ty > {
984  def : InstAlias<"mov\t$Zd, $Pg/m, $ZAn[$Rv, $imm]",
985                  (inst zpr_ty:$Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1>;
986}
987
988multiclass sme_tile_to_vector_patterns<Instruction inst, ValueType zpr_vt,
989                                       ValueType ppr_vt, Operand offset_ty,
990                                       ComplexPattern imm2tile,
991                                       ComplexPattern tileslice,
992                                       SDPatternOperator op> {
993  def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg),
994                        (imm2tile untyped:$tile), MatrixIndexGPR32Op12_15:$idx)),
995            (inst $passthru, $pg, $tile, $idx, 0)>;
996  let AddedComplexity = 1 in {
997    def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg),
998                          (imm2tile untyped:$tile),
999                          (i32 (tileslice MatrixIndexGPR32Op12_15:$idx,
1000                                          offset_ty:$imm)))),
1001              (inst $passthru, $pg, $tile, $idx, $imm)>;
1002  }
1003}
1004
1005multiclass sme_tile_to_vector_v<string mnemonic, bit is_col> {
1006  def _B : sme_tile_to_vector_inst<0b0, 0b00, ZPR8, !if(is_col, TileVectorOpV8,
1007                                                                TileVectorOpH8),
1008                                   is_col, sme_elm_idx0_15, mnemonic> {
1009    bits<4> imm;
1010    let Inst{8-5} = imm;
1011  }
1012  def _H : sme_tile_to_vector_inst<0b0, 0b01, ZPR16, !if(is_col, TileVectorOpV16,
1013                                                                 TileVectorOpH16),
1014                                   is_col, sme_elm_idx0_7, mnemonic> {
1015    bits<1> ZAn;
1016    bits<3> imm;
1017    let Inst{8}   = ZAn;
1018    let Inst{7-5} = imm;
1019  }
1020  def _S : sme_tile_to_vector_inst<0b0, 0b10, ZPR32, !if(is_col, TileVectorOpV32,
1021                                                                 TileVectorOpH32),
1022                                   is_col, sme_elm_idx0_3, mnemonic> {
1023    bits<2> ZAn;
1024    bits<2> imm;
1025    let Inst{8-7} = ZAn;
1026    let Inst{6-5} = imm;
1027  }
1028  def _D : sme_tile_to_vector_inst<0b0, 0b11, ZPR64, !if(is_col, TileVectorOpV64,
1029                                                                 TileVectorOpH64),
1030                                   is_col, sme_elm_idx0_1, mnemonic> {
1031    bits<3> ZAn;
1032    bits<1> imm;
1033    let Inst{8-6} = ZAn;
1034    let Inst{5}   = imm;
1035  }
1036  def _Q : sme_tile_to_vector_inst<0b1, 0b11, ZPR128, !if(is_col, TileVectorOpV128,
1037                                                                  TileVectorOpH128),
1038                                   is_col, sme_elm_idx0_0, mnemonic> {
1039    bits<4> ZAn;
1040    let Inst{8-5} = ZAn;
1041  }
1042
1043  defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _B), ZPR8,
1044                                    !if(is_col, TileVectorOpV8,
1045                                                TileVectorOpH8), sme_elm_idx0_15>;
1046  defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _H), ZPR16,
1047                                    !if(is_col, TileVectorOpV16,
1048                                                TileVectorOpH16), sme_elm_idx0_7>;
1049  defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _S), ZPR32,
1050                                    !if(is_col, TileVectorOpV32,
1051                                                TileVectorOpH32), sme_elm_idx0_3>;
1052  defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _D), ZPR64,
1053                                    !if(is_col, TileVectorOpV64,
1054                                                TileVectorOpH64), sme_elm_idx0_1>;
1055  defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _Q), ZPR128,
1056                                    !if(is_col, TileVectorOpV128,
1057                                                TileVectorOpH128), sme_elm_idx0_0>;
1058
1059  defvar op = !if(is_col, int_aarch64_sme_read_vert,
1060                          int_aarch64_sme_read_horiz);
1061
1062  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _B),
1063                                     nxv16i8, nxv16i1, sme_elm_idx0_15,
1064                                     imm_to_tile8, tileslice8, op>;
1065  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H),
1066                                     nxv8i16, nxv8i1, sme_elm_idx0_7,
1067                                     imm_to_tile16, tileslice16, op>;
1068  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H),
1069                                     nxv8f16, nxv8i1, sme_elm_idx0_7,
1070                                     imm_to_tile16, tileslice16, op>;
1071  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H),
1072                                     nxv8bf16, nxv8i1, sme_elm_idx0_7,
1073                                     imm_to_tile16, tileslice16, op>;
1074  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S),
1075                                     nxv4i32, nxv4i1, sme_elm_idx0_3,
1076                                     imm_to_tile32, tileslice32, op>;
1077  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S),
1078                                     nxv4f32, nxv4i1, sme_elm_idx0_3,
1079                                     imm_to_tile32, tileslice32, op>;
1080  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D),
1081                                     nxv2i64, nxv2i1, sme_elm_idx0_1,
1082                                     imm_to_tile64, tileslice64, op>;
1083  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D),
1084                                     nxv2f64, nxv2i1, sme_elm_idx0_1,
1085                                     imm_to_tile64, tileslice64, op>;
1086
1087  defvar opq = !if(is_col, int_aarch64_sme_readq_vert,
1088                           int_aarch64_sme_readq_horiz);
1089
1090  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),
1091                                     nxv16i8, nxv16i1, sme_elm_idx0_0,
1092                                     imm_to_tile128, tileslice128, opq>;
1093  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),
1094                                     nxv8i16, nxv8i1, sme_elm_idx0_0,
1095                                     imm_to_tile128, tileslice128, opq>;
1096  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),
1097                                     nxv8f16, nxv8i1, sme_elm_idx0_0,
1098                                     imm_to_tile128, tileslice128, opq>;
1099  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),
1100                                     nxv8bf16, nxv8i1, sme_elm_idx0_0,
1101                                     imm_to_tile128, tileslice128, opq>;
1102  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),
1103                                     nxv4i32, nxv4i1, sme_elm_idx0_0,
1104                                     imm_to_tile128, tileslice128, opq>;
1105  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),
1106                                     nxv4f32, nxv4i1, sme_elm_idx0_0,
1107                                     imm_to_tile128, tileslice128, opq>;
1108  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),
1109                                     nxv2i64, nxv2i1, sme_elm_idx0_0,
1110                                     imm_to_tile128, tileslice128, opq>;
1111  defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),
1112                                     nxv2f64, nxv2i1, sme_elm_idx0_0,
1113                                     imm_to_tile128, tileslice128, opq>;
1114}
1115
1116multiclass sme_tile_to_vector<string mnemonic> {
1117  defm _H : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b0>;
1118  defm _V : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b1>;
1119}
1120
1121//===----------------------------------------------------------------------===//
1122// SME Zero
1123//===----------------------------------------------------------------------===//
1124
1125// NOTE: This definition isn't really correct because there are outputs, i.e.
1126// the tile registers being zeroed. We fix this up in a custom inserter that
1127// marks the appropriate registers as being implicitly defined.
1128class sme_zero_inst<string mnemonic>
1129    : I<(outs), (ins MatrixTileList:$imm),
1130        mnemonic, "\t$imm", "", []>, Sched<[]> {
1131  bits<8> imm;
1132  let Inst{31-8} = 0b110000000000100000000000;
1133  let Inst{7-0}  = imm;
1134}
1135
1136multiclass sme_zero<string mnemonic> {
1137  def NAME : sme_zero_inst<mnemonic>;
1138
1139  def : InstAlias<"zero\t\\{za\\}", (!cast<Instruction>(NAME) 0b11111111), 1>;
1140  def : InstAlias<"zero\t\\{za0.h\\}", (!cast<Instruction>(NAME) 0b01010101), 1>;
1141  def : InstAlias<"zero\t\\{za1.h\\}", (!cast<Instruction>(NAME) 0b10101010), 1>;
1142  def : InstAlias<"zero\t\\{za0.s\\}", (!cast<Instruction>(NAME) 0b00010001), 1>;
1143  def : InstAlias<"zero\t\\{za1.s\\}", (!cast<Instruction>(NAME) 0b00100010), 1>;
1144  def : InstAlias<"zero\t\\{za2.s\\}", (!cast<Instruction>(NAME) 0b01000100), 1>;
1145  def : InstAlias<"zero\t\\{za3.s\\}", (!cast<Instruction>(NAME) 0b10001000), 1>;
1146  def : InstAlias<"zero\t\\{za0.s,za1.s\\}", (!cast<Instruction>(NAME) 0b00110011), 1>;
1147  def : InstAlias<"zero\t\\{za0.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10011001), 1>;
1148  def : InstAlias<"zero\t\\{za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01100110), 1>;
1149  def : InstAlias<"zero\t\\{za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11001100), 1>;
1150  def : InstAlias<"zero\t\\{za0.s,za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01110111), 1>;
1151  def : InstAlias<"zero\t\\{za0.s,za1.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10111011), 1>;
1152  def : InstAlias<"zero\t\\{za0.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11011101), 1>;
1153  def : InstAlias<"zero\t\\{za1.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11101110), 1>;
1154
1155  def NAME # _PSEUDO : Pseudo<(outs), (ins i32imm:$tilelist), []>,
1156      Sched<[]> {
1157    // Translated to the actual instructions in AArch64ISelLowering.cpp
1158    let usesCustomInserter = 1;
1159  }
1160
1161  def : Pat<(int_aarch64_sme_zero timm32_0_255:$imm),
1162            (!cast<Instruction>(NAME # _PSEUDO) timm32_0_255:$imm)>;
1163}
1164
1165//===----------------------------------------------------------------------===//
1166// SVE2 Instructions
1167//===----------------------------------------------------------------------===//
1168
1169class sve2_int_perm_revd<string asm>
1170    : I<(outs ZPR128:$Zd), (ins ZPR128:$_Zd, PPR3bAny:$Pg, ZPR128:$Zn),
1171        asm, "\t$Zd, $Pg/m, $Zn", "", []>,
1172      Sched<[]> {
1173  bits<5> Zd;
1174  bits<3> Pg;
1175  bits<5> Zn;
1176  let Inst{31-24} = 0b00000101;
1177  let Inst{23-22} = 0b00; // size
1178  let Inst{21-13} = 0b101110100;
1179  let Inst{12-10} = Pg;
1180  let Inst{9-5}   = Zn;
1181  let Inst{4-0}   = Zd;
1182
1183  let Constraints = "$Zd = $_Zd";
1184  let DestructiveInstType = DestructiveUnary;
1185  let ElementSize = ZPR128.ElementSize;
1186}
1187
1188multiclass sve2_int_perm_revd<string asm, SDPatternOperator op> {
1189  def NAME : sve2_int_perm_revd<asm>;
1190
1191  def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>;
1192  def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME)>;
1193  def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME)>;
1194  def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME)>;
1195}
1196
1197class sve2_clamp<string asm, bits<2> sz, bit U, ZPRRegOp zpr_ty>
1198    : I<(outs zpr_ty:$Zd), (ins zpr_ty:$Zn, zpr_ty:$Zm, zpr_ty:$_Zd),
1199        asm, "\t$Zd, $Zn, $Zm", "", []>,
1200      Sched<[]> {
1201  bits<5> Zm;
1202  bits<5> Zn;
1203  bits<5> Zd;
1204  let Inst{31-24} = 0b01000100;
1205  let Inst{23-22} = sz;
1206  let Inst{21}    = 0b0;
1207  let Inst{20-16} = Zm;
1208  let Inst{15-11} = 0b11000;
1209  let Inst{10}    = U;
1210  let Inst{9-5}   = Zn;
1211  let Inst{4-0}   = Zd;
1212
1213  let Constraints = "$Zd = $_Zd";
1214  let DestructiveInstType = DestructiveOther;
1215  let ElementSize = zpr_ty.ElementSize;
1216}
1217
1218multiclass sve2_clamp<string asm, bit U, SDPatternOperator op> {
1219  def _B : sve2_clamp<asm, 0b00, U, ZPR8>;
1220  def _H : sve2_clamp<asm, 0b01, U, ZPR16>;
1221  def _S : sve2_clamp<asm, 0b10, U, ZPR32>;
1222  def _D : sve2_clamp<asm, 0b11, U, ZPR64>;
1223
1224  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
1225  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1226  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1227  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1228}
1229
1230class sve2_int_perm_sel_p<string asm, PPRRegOp ppr_ty, Operand imm_ty>
1231    : I<(outs PPRAny:$Pd), (ins PPRAny:$Pn, ppr_ty:$Pm,
1232                            MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm),
1233        asm, "\t$Pd, $Pn, $Pm[$Rv, $imm]", "", []>,
1234      Sched<[]> {
1235  bits<2> Rv;
1236  bits<4> Pn;
1237  bits<4> Pm;
1238  bits<4> Pd;
1239  let Inst{31-24} = 0b00100101;
1240  let Inst{21}    = 0b1;
1241  let Inst{17-16} = Rv;
1242  let Inst{15-14} = 0b01;
1243  let Inst{13-10} = Pn;
1244  let Inst{9}     = 0b0;
1245  let Inst{8-5}   = Pm;
1246  let Inst{4}     = 0b0;
1247  let Inst{3-0}   = Pd;
1248}
1249
1250multiclass sve2_int_perm_sel_p<string asm, SDPatternOperator op> {
1251  def _B : sve2_int_perm_sel_p<asm, PPR8, sme_elm_idx0_15> {
1252    bits<4> imm;
1253    let Inst{23-22} = imm{3-2};
1254    let Inst{20-19} = imm{1-0};
1255    let Inst{18}    = 0b1;
1256  }
1257  def _H : sve2_int_perm_sel_p<asm, PPR16, sme_elm_idx0_7> {
1258    bits<3> imm;
1259    let Inst{23-22} = imm{2-1};
1260    let Inst{20}    = imm{0};
1261    let Inst{19-18} = 0b10;
1262  }
1263  def _S : sve2_int_perm_sel_p<asm, PPR32, sme_elm_idx0_3> {
1264    bits<2> imm;
1265    let Inst{23-22} = imm{1-0};
1266    let Inst{20-18} = 0b100;
1267  }
1268  def _D : sve2_int_perm_sel_p<asm, PPR64, sme_elm_idx0_1> {
1269    bits<1> imm;
1270    let Inst{23}    = imm;
1271    let Inst{22}    = 0b1;
1272    let Inst{20-18} = 0b000;
1273  }
1274
1275  def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]",
1276                  (!cast<Instruction>(NAME # _B) PNRAny:$Pd,
1277                      PNRAny:$Pn, PPR8:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm), 0>;
1278  def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]",
1279                  (!cast<Instruction>(NAME # _H) PNRAny:$Pd,
1280                      PNRAny:$Pn, PPR16:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_7:$imm), 0>;
1281  def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]",
1282                  (!cast<Instruction>(NAME # _S) PNRAny:$Pd,
1283                      PNRAny:$Pn, PPR32:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_3:$imm), 0>;
1284  def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]",
1285                  (!cast<Instruction>(NAME # _D) PNRAny:$Pd,
1286                      PNRAny:$Pn, PPR64:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_1:$imm), 0>;
1287
1288  def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPRAny:$Pm),
1289             MatrixIndexGPR32Op12_15:$idx)),
1290            (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, 0)>;
1291  def : Pat<(nxv8i1 (op (nxv8i1 PPRAny:$Pn), (nxv8i1 PPRAny:$Pm),
1292             MatrixIndexGPR32Op12_15:$idx)),
1293            (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, 0)>;
1294  def : Pat<(nxv4i1 (op (nxv4i1 PPRAny:$Pn), (nxv4i1 PPRAny:$Pm),
1295             MatrixIndexGPR32Op12_15:$idx)),
1296            (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, 0)>;
1297  def : Pat<(nxv2i1 (op (nxv2i1 PPRAny:$Pn), (nxv2i1 PPRAny:$Pm),
1298             MatrixIndexGPR32Op12_15:$idx)),
1299            (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, 0)>;
1300
1301  let AddedComplexity = 1 in {
1302    def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPRAny:$Pm),
1303               (i32 (tileslice8 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_15:$imm)))),
1304              (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, $imm)>;
1305    def : Pat<(nxv8i1 (op (nxv8i1 PPRAny:$Pn), (nxv8i1 PPRAny:$Pm),
1306               (i32 (tileslice16 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_7:$imm)))),
1307              (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, $imm)>;
1308    def : Pat<(nxv4i1 (op (nxv4i1 PPRAny:$Pn), (nxv4i1 PPRAny:$Pm),
1309               (i32 (tileslice32 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_3:$imm)))),
1310              (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, $imm)>;
1311    def : Pat<(nxv2i1 (op (nxv2i1 PPRAny:$Pn), (nxv2i1 PPRAny:$Pm),
1312               (i32 (tileslice64 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_1:$imm)))),
1313              (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, $imm)>;
1314  }
1315}
1316
1317//===----------------------------------------------------------------------===//
1318// SME2 Instructions
1319//===----------------------------------------------------------------------===//
1320
1321//===----------------------------------------------------------------------===//
1322// SME2 single-multi ternary int/fp, two/four registers
1323
1324class sme2_dot_mla_add_sub_array_vg24_single<bits<7> op,
1325                                         MatrixOperand matrix_ty,
1326                                         RegisterOperand multi_vector_ty,
1327                                         ZPRRegOp zpr_ty,
1328                                         string mnemonic>
1329   : I<(outs matrix_ty:$ZAd),
1330       (ins  matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv,
1331       sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm),
1332       mnemonic,"\t$ZAd[$Rv, $imm3, " # !if(op{5}, "vgx4", "vgx2") # "], $Zn, $Zm",
1333       "", []> , Sched<[]> {
1334  bits<4> Zm;
1335  bits<5> Zn;
1336  bits<2> Rv;
1337  bits<3> imm3;
1338  let Inst{31-23} = 0b110000010;
1339  let Inst{22}    = op{6}; //sz
1340  let Inst{21}    = 0b1;
1341  let Inst{20}    = op{5}; //vgx4
1342  let Inst{19-16} = Zm;
1343  let Inst{15}    = 0b0;
1344  let Inst{14-13} = Rv;
1345  let Inst{12-10} = op{4-2};
1346  let Inst{9-5}   = Zn;
1347  let Inst{4-3}   = op{1-0};
1348  let Inst{2-0}   = imm3;
1349  let Constraints = "$ZAd = $_ZAd";
1350}
1351
1352multiclass sme2_dot_mla_add_sub_array_vg24_single<string mnemonic, bits<7> op,
1353                                              MatrixOperand matrix_ty,
1354                                              RegisterOperand multi_vector_ty,
1355                                              ZPRRegOp zpr_ty>{
1356  def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;
1357
1358  def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",
1359                 (!cast<Instruction>(NAME) matrix_ty:$ZAd,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>;
1360}
1361
1362multiclass sme2_dot_mla_add_sub_array_vg2_single<string mnemonic, bits<7> op,
1363                                              MatrixOperand matrix_ty,
1364                                              RegisterOperand multi_vector_ty,
1365                                              ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{
1366  def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;
1367
1368  def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",
1369                 (!cast<Instruction>(NAME) matrix_ty:$ZAd,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>;
1370
1371  def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>;
1372
1373  def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>;
1374}
1375
1376multiclass sme2_dot_mla_add_sub_array_vg4_single<string mnemonic, bits<7> op,
1377                                              MatrixOperand matrix_ty,
1378                                              RegisterOperand multi_vector_ty,
1379                                              ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{
1380  def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;
1381
1382  def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",
1383                 (!cast<Instruction>(NAME) matrix_ty:$ZAd,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>;
1384
1385  def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>;
1386
1387  def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>;
1388}
1389
1390//===----------------------------------------------------------------------===//
1391// SME2 multiple vectors ternary INT/FP  two and four registers
1392class sme2_dot_mla_add_sub_array_vg2_multi<bits<6> op,
1393                                       MatrixOperand matrix_ty,
1394                                       RegisterOperand multi_vector_ty,
1395                                       string mnemonic>
1396   : I<(outs matrix_ty:$ZAd),
1397       (ins  matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv,
1398       sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm),
1399       mnemonic, "\t$ZAd[$Rv, $imm3, vgx2], $Zn, $Zm",
1400       "", []>, Sched<[]>{
1401  bits<4> Zm;
1402  bits<4> Zn;
1403  bits<2> Rv;
1404  bits<3> imm3;
1405  let Inst{31-23} = 0b110000011;
1406  let Inst{22}    = op{5}; //sz
1407  let Inst{21}    = 0b1;
1408  let Inst{20-17} = Zm;
1409  let Inst{16-15} = 0b00;
1410  let Inst{14-13} = Rv;
1411  let Inst{12-10} = op{4-2};
1412  let Inst{9-6}   = Zn;
1413  let Inst{5}     = 0b0;
1414  let Inst{4-3}   = op{1-0};
1415  let Inst{2-0}   = imm3;
1416  let Constraints = "$ZAd = $_ZAd";
1417}
1418
1419multiclass sme2_dot_mla_add_sub_array_vg2_multi<string mnemonic, bits<6> op,
1420                                            MatrixOperand  matrix_ty,
1421                                            RegisterOperand multi_vector_ty, ValueType zpr_ty,
1422                                            SDPatternOperator intrinsic> {
1423  def NAME : sme2_dot_mla_add_sub_array_vg2_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;
1424
1425  def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>;
1426
1427  def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>;
1428
1429  def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",
1430                  (!cast<Instruction>(NAME) matrix_ty:$ZAd,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>;
1431}
1432
1433class sme2_dot_mla_add_sub_array_vg4_multi<bits<6> op,
1434                                            MatrixOperand matrix_ty,
1435                                            RegisterOperand multi_vector_ty,
1436                                            string mnemonic>
1437   : I<(outs matrix_ty:$ZAd),
1438       (ins  matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv,
1439        sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm),
1440        mnemonic, "\t$ZAd[$Rv, $imm3, vgx4], $Zn, $Zm",
1441        "", []>, Sched<[]>{
1442  bits<3> Zm;
1443  bits<3> Zn;
1444  bits<2> Rv;
1445  bits<3> imm3;
1446  let Inst{31-23} = 0b110000011;
1447  let Inst{22}    = op{5}; //sz
1448  let Inst{21}    = 0b1;
1449  let Inst{20-18} = Zm;
1450  let Inst{17-15} = 0b010;
1451  let Inst{14-13} = Rv;
1452  let Inst{12-10} = op{4-2};
1453  let Inst{9-7}   = Zn;
1454  let Inst{6-5}   = 0b00;
1455  let Inst{4-3}   = op{1-0};
1456  let Inst{2-0}   = imm3;
1457  let Constraints = "$ZAd = $_ZAd";
1458}
1459
1460multiclass sme2_dot_mla_add_sub_array_vg4_multi<string mnemonic, bits<6> op,
1461                                            MatrixOperand  matrix_ty,
1462                                            RegisterOperand multi_vector_ty,
1463                                            ValueType zpr_ty, SDPatternOperator intrinsic>{
1464  def NAME : sme2_dot_mla_add_sub_array_vg4_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;
1465
1466  def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>;
1467
1468  def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>;
1469
1470  def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",
1471                 (!cast<Instruction>(NAME) matrix_ty:$ZAd,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>;
1472}
1473
1474//===----------------------------------------------------------------------===//
1475// SME2 multiple vectors binary two or four  registers
1476
1477class sme2_multivec_accum_add_sub<string mnemonic, bit sz, bit vg4, bits<3> op,
1478                                  MatrixOperand matrix_ty,
1479                                  RegisterOperand vector_ty>
1480    : I<(outs matrix_ty:$ZAdn),
1481        (ins matrix_ty:$_ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm),
1482        mnemonic, "\t$ZAdn[$Rv, $imm3, " # !if(vg4, "vgx4", "vgx2") # "], $Zm",
1483        "", []>, Sched<[]> {
1484  bits<2> Rv;
1485  bits<3> imm3;
1486  let Inst{31-23} = 0b110000011;
1487  let Inst{22}    = sz;
1488  let Inst{21-19} = 0b100;
1489  let Inst{18}    = op{2};
1490  let Inst{17}    = 0b0;
1491  let Inst{16}    = vg4;
1492  let Inst{15}    = 0b0;
1493  let Inst{14-13} = Rv;
1494  let Inst{12-10} = 0b111;
1495  let Inst{5}     = 0b0;
1496  let Inst{4-3}   = op{1-0};
1497  let Inst{2-0}   = imm3;
1498
1499  let Constraints = "$ZAdn = $_ZAdn";
1500}
1501
1502class sme2_multivec_accum_add_sub_vg2<string mnemonic, bit sz, bits<3> op,
1503                                      MatrixOperand matrix_ty,
1504                                      RegisterOperand vector_ty>
1505    : sme2_multivec_accum_add_sub<mnemonic, sz, 0b0, op, matrix_ty, vector_ty> {
1506  bits<4> Zm;
1507  let Inst{9-6} = Zm;
1508}
1509
1510
1511multiclass sme2_multivec_accum_add_sub_vg2<string mnemonic, bits<4> op,
1512                                           MatrixOperand matrix_ty,
1513                                           RegisterOperand vector_ty> {
1514  def NAME : sme2_multivec_accum_add_sub_vg2<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>;
1515
1516  def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm",
1517  (!cast<Instruction>(NAME) matrix_ty:$ZAdn,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>;
1518}
1519
1520class sme2_multivec_accum_add_sub_vg4<string mnemonic, bit sz, bits<3> op,
1521                                      MatrixOperand matrix_ty,
1522                                      RegisterOperand vector_ty>
1523    : sme2_multivec_accum_add_sub<mnemonic, sz, 0b1, op, matrix_ty, vector_ty> {
1524  bits<3> Zm;
1525  let Inst{9-7} = Zm;
1526  let Inst{6}   = 0b0;
1527}
1528
1529multiclass sme2_multivec_accum_add_sub_vg4<string mnemonic, bits<4> op,
1530                                           MatrixOperand matrix_ty,
1531                                           RegisterOperand vector_ty> {
1532  def NAME : sme2_multivec_accum_add_sub_vg4<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>;
1533
1534  def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm",
1535  (!cast<Instruction>(NAME) matrix_ty:$ZAdn,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>;
1536}
1537
1538//===----------------------------------------------------------------------===//
1539// SME2 Multi-vector - Multiple and Single SVE Destructive
1540// Two and Four registers
1541
1542class sme2_sve_destructive_vector_vg2_single<bits<2> sz, bits<7> op,
1543                                             RegisterOperand vector_ty,
1544                                             ZPRRegOp zpr_ty,
1545                                             string mnemonic>
1546    : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm),
1547        mnemonic, "\t$Zdn, $_Zdn, $Zm",
1548        "", []>, Sched<[]> {
1549  bits<4> Zm;
1550  bits<4> Zdn;
1551  let Inst{31-24} = 0b11000001;
1552  let Inst{23-22} = sz;
1553  let Inst{21-20} = 0b10;
1554  let Inst{19-16} = Zm;
1555  let Inst{15-11} = 0b10100;
1556  let Inst{10-5}  = op{6-1};
1557  let Inst{4-1}   = Zdn;
1558  let Inst{0}     = op{0};
1559
1560  let Constraints = "$Zdn = $_Zdn";
1561}
1562
1563multiclass sme2_fp_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> {
1564  def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>;
1565  def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>;
1566  def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>;
1567}
1568
1569multiclass sme2_int_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> {
1570  def _B : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_b_mul_r, ZPR4b8, mnemonic>;
1571  def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>;
1572  def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>;
1573  def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>;
1574}
1575
1576// SME2.1 fmax/fmin instructions.
1577multiclass sme2p1_bf_max_min_vector_vg2_single<string mnemonic, bits<7>op> {
1578  def _H : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_h_mul_r,
1579                                                  ZPR4b16, mnemonic>;
1580}
1581
1582class sme2_sve_destructive_vector_vg4_single<bits<2> sz, bits<7> op,
1583                                             RegisterOperand vector_ty,
1584                                             ZPRRegOp zpr_ty,
1585                                             string mnemonic>
1586    : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm),
1587        mnemonic, "\t$Zdn, $_Zdn, $Zm",
1588        "", []>, Sched<[]> {
1589  bits<4> Zm;
1590  bits<3> Zdn;
1591  let Inst{31-24} = 0b11000001;
1592  let Inst{23-22} = sz;
1593  let Inst{21-20} = 0b10;
1594  let Inst{19-16} = Zm;
1595  let Inst{15-11} = 0b10101;
1596  let Inst{10-5}  = op{6-1};
1597  let Inst{4-2}   = Zdn;
1598  let Inst{1}     = 0b0;
1599  let Inst{0}     = op{0};
1600
1601  let Constraints = "$Zdn = $_Zdn";
1602}
1603
1604multiclass sme2_fp_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> {
1605  def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>;
1606  def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>;
1607  def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>;
1608}
1609
1610multiclass sme2_int_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> {
1611  def _B : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_b_mul_r, ZPR4b8, mnemonic>;
1612  def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>;
1613  def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>;
1614  def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>;
1615}
1616
1617// SME2.1 fmax/fmin instructions.
1618multiclass sme2p1_bf_max_min_vector_vg4_single<string mnemonic, bits<7>op> {
1619  def _H : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_h_mul_r,
1620                                                  ZPR4b16, mnemonic>;
1621}
1622
1623class sme2_sve_destructive_vector_vg2_multi<bits<2> sz, bits<7> op,
1624                                            RegisterOperand vector_ty,
1625                                            string mnemonic>
1626    : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm),
1627        mnemonic, "\t$Zdn, $_Zdn, $Zm",
1628        "", []>, Sched<[]> {
1629  bits<4> Zm;
1630  bits<4> Zdn;
1631  let Inst{31-24} = 0b11000001;
1632  let Inst{23-22} = sz;
1633  let Inst{21}    = 0b1;
1634  let Inst{20-17} = Zm;
1635  let Inst{16-11} = 0b010110;
1636  let Inst{10-5}  = op{6-1};
1637  let Inst{4-1}   = Zdn;
1638  let Inst{0}     = op{0};
1639
1640  let Constraints = "$Zdn = $_Zdn";
1641}
1642
1643multiclass sme2_fp_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> {
1644  def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>;
1645  def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>;
1646  def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>;
1647}
1648
1649multiclass sme2_int_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> {
1650  def _B : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_b_mul_r, mnemonic>;
1651  def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>;
1652  def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>;
1653  def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>;
1654}
1655
1656// SME2.1 fmax/fmin instructions.
1657multiclass sme2p1_bf_max_min_vector_vg2_multi<string mnemonic, bits<7>op> {
1658  def _H : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_h_mul_r,
1659                                                 mnemonic>;
1660}
1661
1662class sme2_sve_destructive_vector_vg4_multi<bits<2> sz, bits<7> op,
1663                                            RegisterOperand vector_ty,
1664                                            string mnemonic>
1665    : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm),
1666        mnemonic, "\t$Zdn, $_Zdn, $Zm",
1667        "", []>, Sched<[]> {
1668  bits<3> Zm;
1669  bits<3> Zdn;
1670  let Inst{31-24} = 0b11000001;
1671  let Inst{23-22} = sz;
1672  let Inst{21}    = 0b1;
1673  let Inst{20-18} = Zm;
1674  let Inst{17-11} = 0b0010111;
1675  let Inst{10-5}  = op{6-1};
1676  let Inst{4-2}   = Zdn;
1677  let Inst{1}     = 0b0;
1678  let Inst{0}     = op{0};
1679
1680  let Constraints = "$Zdn = $_Zdn";
1681}
1682
1683multiclass sme2_fp_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> {
1684  def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>;
1685  def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>;
1686  def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>;
1687}
1688
1689multiclass sme2_int_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> {
1690  def _B : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_b_mul_r, mnemonic>;
1691  def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>;
1692  def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>;
1693  def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>;
1694}
1695
1696// SME2.1 fmax/fmin instructions.
1697multiclass sme2p1_bf_max_min_vector_vg4_multi<string mnemonic, bits<7>op> {
1698  def _H : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_h_mul_r,
1699                                                 mnemonic>;
1700}
1701
1702//===----------------------------------------------------------------------===//
1703// SME2 Multi-vector - Index/Single/Multi Array Vectors FMA sources
1704
1705class sme2_mla_long_array_index_base<bits<2> op0, bits<2> op, Operand index_ty,
1706                                     RegisterOperand multi_vector_ty,
1707                                     string mnemonic, string vg_acronym="">
1708    : I<(outs MatrixOp32:$ZAda),
1709        (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm, multi_vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3),
1710        mnemonic, "\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm$i3",
1711        "", []>, Sched<[]> {
1712  bits<4> Zm;
1713  bits<2> Rv;
1714  let Inst{31-24} = 0b11000001;
1715  let Inst{23-22} = op0;
1716  let Inst{21}    = 0b0;
1717  let Inst{20}    = !if(!eq(vg_acronym, ""), 0, 1);
1718  let Inst{19-16} = Zm;
1719  let Inst{14-13} = Rv;
1720  let Inst{12}    = 0b1;
1721  let Inst{4-3}   = op;
1722
1723  let Constraints = "$ZAda = $_ZAda";
1724}
1725
1726multiclass sme2_mla_long_array_index<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
1727  def _S : sme2_mla_long_array_index_base<op0, op, uimm3s2range, ZPR16,
1728                                          mnemonic>, SMEPseudo2Instr<NAME # _S, 1> {
1729    bits<3> i3;
1730    bits<5> Zn;
1731    bits<3> imm;
1732    let Inst{15}    = i3{2};
1733    let Inst{11-10} = i3{1-0};
1734    let Inst{9-5}   = Zn;
1735    let Inst{2-0}   = imm;
1736  }
1737
1738  def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
1739
1740  def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange3s2>;
1741}
1742
1743class sme2_mla_long_array_vg2_index<string mnemonic, bits<2> op0, bits<2> op>
1744    : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZ_h_mul_r,
1745                                     mnemonic, "vgx2"> {
1746  bits<3> i3;
1747  bits<4> Zn;
1748  bits<2> imm;
1749  let Inst{15}    = 0b0;
1750  let Inst{11-10} = i3{2-1};
1751  let Inst{9-6}   = Zn;
1752  let Inst{5}     = 0b0;
1753  let Inst{2}     = i3{0};
1754  let Inst{1-0}   = imm;
1755}
1756
1757multiclass sme2_fp_mla_long_array_vg2_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
1758  def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>;
1759
1760  def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
1761
1762  def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>;
1763
1764  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",
1765                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
1766}
1767
1768multiclass sme2_int_mla_long_array_vg2_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
1769  def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>;
1770
1771  def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
1772
1773  def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>;
1774
1775  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",
1776                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
1777}
1778
1779class sme2_mla_long_array_vg4_index<string mnemonic, bits<2> op0, bits<2> op>
1780    : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZZZ_h_mul_r,
1781                                      mnemonic, "vgx4"> {
1782  bits<3> i3;
1783  bits<3> Zn;
1784  bits<2> imm;
1785  let Inst{15}    = 0b1;
1786  let Inst{11-10} = i3{2-1};
1787  let Inst{9-7}   = Zn;
1788  let Inst{6-5}   = 0b00;
1789  let Inst{2}     = i3{0};
1790  let Inst{1-0}   = imm;
1791}
1792
1793multiclass sme2_fp_mla_long_array_vg4_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
1794  def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>;
1795
1796  def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
1797
1798  def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>;
1799
1800  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",
1801                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
1802}
1803
1804multiclass sme2_int_mla_long_array_vg4_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
1805  def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>;
1806
1807  def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
1808
1809  def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>;
1810
1811  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",
1812                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
1813}
1814
1815class sme2_mla_long_array<bits<2>op0, bits<2> op, Operand index_ty,
1816                          RegisterOperand first_vector_ty,
1817                          RegisterOperand second_vector_ty,
1818                          string mnemonic, string vg_acronym="">
1819   : I<(outs MatrixOp32:$ZAda),
1820       (ins  MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv,
1821       index_ty:$imm, first_vector_ty:$Zn, second_vector_ty:$Zm),
1822       mnemonic,"\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm",
1823       "", []> , Sched<[]> {
1824  bits<2> Rv;
1825  let Inst{31-24} = 0b11000001;
1826  let Inst{23-22} = op0;
1827  let Inst{21}    = 0b1;
1828  let Inst{15}    = 0b0;
1829  let Inst{14-13} = Rv;
1830  let Inst{12-11} = 0b01;
1831  let Inst{10}    = !if(!eq(vg_acronym, ""), 1, 0);
1832  let Inst{4-3}   = op;
1833
1834  let Constraints = "$ZAda = $_ZAda";
1835}
1836
1837multiclass sme2_mla_long_array_single<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
1838  def _S : sme2_mla_long_array<op0, op, uimm3s2range, ZPR16, ZPR4b16,
1839                               mnemonic> , SMEPseudo2Instr<NAME # _S, 1>{
1840    bits<4> Zm;
1841    bits<5> Zn;
1842    bits<3> imm;
1843    let Inst{20}    = 0b0;
1844    let Inst{19-16} = Zm;
1845    let Inst{9-5}   = Zn;
1846    let Inst{2-0}   = imm;
1847  }
1848
1849  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, SMEMatrixArray>;
1850
1851  def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, tileslicerange3s2>;
1852}
1853
1854class sme2_mla_long_array_vg24_single<bits<2> op0, bit vg4, bits<2> op,
1855                                      RegisterOperand first_vector_ty,
1856                                      string mnemonic, string vg_acronym>
1857    : sme2_mla_long_array<op0, op, uimm2s2range, first_vector_ty, ZPR4b16,
1858                          mnemonic, vg_acronym> {
1859  bits<4> Zm;
1860  bits<5> Zn;
1861  bits<2> imm;
1862  let Inst{20}    = vg4;
1863  let Inst{19-16} = Zm;
1864  let Inst{9-5}   = Zn;
1865  let Inst{2}     = 0b0;
1866  let Inst{1-0}   = imm;
1867}
1868
1869multiclass sme2_fp_mla_long_array_vg2_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
1870  def _S : sme2_mla_long_array_vg24_single<0b00, 0b0, op, ZZ_h, mnemonic,
1871                                           "vgx2">, SMEPseudo2Instr<NAME # _S, 1>;
1872
1873  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>;
1874
1875  def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>;
1876
1877  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
1878                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>;
1879}
1880
1881multiclass sme2_int_mla_long_array_vg2_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
1882  def _S : sme2_mla_long_array_vg24_single<0b01, 0b0, op, ZZ_h, mnemonic,
1883                                           "vgx2">, SMEPseudo2Instr<NAME # _S, 1>;
1884
1885  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>;
1886
1887  def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>;
1888
1889  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
1890                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>;
1891}
1892
1893multiclass sme2_fp_mla_long_array_vg4_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
1894  def _S : sme2_mla_long_array_vg24_single<0b00, 0b1, op, ZZZZ_h, mnemonic,
1895                                           "vgx4">, SMEPseudo2Instr<NAME # _S, 1>;
1896
1897  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>;
1898
1899  def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>;
1900
1901  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
1902                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>;
1903}
1904
1905multiclass sme2_int_mla_long_array_vg4_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
1906  def _S : sme2_mla_long_array_vg24_single<0b01, 0b1, op, ZZZZ_h, mnemonic,
1907                                           "vgx4">, SMEPseudo2Instr<NAME # _S, 1>;
1908
1909  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>;
1910
1911  def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>;
1912
1913  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
1914                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>;
1915}
1916class sme2_mla_long_array_vg2_multi<string mnemonic, bits<2> op0, bits<2> op>
1917   : sme2_mla_long_array<op0, op, uimm2s2range, ZZ_h_mul_r, ZZ_h_mul_r,  mnemonic,
1918                         "vgx2"> {
1919  bits<4> Zm;
1920  bits<4> Zn;
1921  bits<2> imm;
1922  let Inst{20-17} = Zm;
1923  let Inst{16}    = 0b0;
1924  let Inst{9-6}   = Zn;
1925  let Inst{5}     = 0b0;
1926  let Inst{2}     = 0b0;
1927  let Inst{1-0}   = imm;
1928}
1929
1930multiclass sme2_fp_mla_long_array_vg2_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
1931  def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>;
1932
1933  def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>;
1934
1935  def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>;
1936
1937  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
1938                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>;
1939}
1940
1941multiclass sme2_int_mla_long_array_vg2_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
1942  def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>;
1943
1944  def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>;
1945
1946  def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>;
1947
1948  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm",
1949                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>;
1950}
1951
1952class sme2_mla_long_array_vg4_multi<string mnemonic, bits<2> op0, bits<2> op>
1953   : sme2_mla_long_array<op0, op, uimm2s2range, ZZZZ_h_mul_r, ZZZZ_h_mul_r, mnemonic,
1954                        "vgx4"> {
1955  bits<3> Zm;
1956  bits<3> Zn;
1957  bits<2> imm;
1958  let Inst{20-18} = Zm;
1959  let Inst{17}    = 0b0;
1960  let Inst{16}    = 0b1;
1961  let Inst{9-7}   = Zn;
1962  let Inst{6-5}   = 0b00;
1963  let Inst{2}     = 0b0;
1964  let Inst{1-0}   = imm;
1965}
1966
1967multiclass sme2_fp_mla_long_array_vg4_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
1968  def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>;
1969
1970  def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>;
1971
1972  def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>;
1973
1974  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
1975                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>;
1976}
1977
1978multiclass sme2_int_mla_long_array_vg4_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
1979  def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>;
1980
1981  def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>;
1982
1983  def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>;
1984
1985  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm",
1986                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>;
1987}
1988
1989//===----------------------------------------------------------------------===//
1990class sme2_frint_cvt_vg2_multi<bits<2>sz, bits<5>op, RegisterOperand first_ty,
1991                               RegisterOperand second_ty, string mnemonic>
1992    : I<(outs first_ty:$Zd), (ins second_ty:$Zn),
1993        mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {
1994  bits<4> Zn;
1995  bits<4> Zd;
1996  let Inst{31-24} = 0b11000001;
1997  let Inst{23-22} = sz;
1998  let Inst{21-20} = 0b10;
1999  let Inst{19-16} = op{4-1};
2000  let Inst{15-10} = 0b111000;
2001  let Inst{9-6}   = Zn;
2002  let Inst{5}     = op{0};
2003  let Inst{4-1}   = Zd;
2004  let Inst{0}     = 0b0;
2005}
2006
2007// SME2 multi-vec FP to int convert two registers
2008// SME2 multi-vec int to FP two registers
2009multiclass sme2_fp_cvt_vg2_multi<string mnemonic, bits<5> op> {
2010  def NAME : sme2_frint_cvt_vg2_multi<0b00, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>;
2011}
2012
2013// SME2 multi-vec FRINT two registers
2014multiclass sme2_frint_vector_vg2_multi<string mnemonic, bits<5> op> {
2015  def _S : sme2_frint_cvt_vg2_multi<0b10, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>;
2016}
2017
2018class sme2_frint_zip_cvt_vg4_multi<bits<2>sz, bits<7>op, RegisterOperand first_ty,
2019                                   RegisterOperand second_ty, string mnemonic>
2020    : I<(outs first_ty:$Zd), (ins second_ty:$Zn),
2021        mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {
2022  bits<3> Zn;
2023  bits<3> Zd;
2024  let Inst{31-24} = 0b11000001;
2025  let Inst{23-22} = sz;
2026  let Inst{21-20} = 0b11;
2027  let Inst{19-16} = op{6-3};
2028  let Inst{15-10} = 0b111000;
2029  let Inst{9-7}   = Zn;
2030  let Inst{6-5}   = op{2-1};
2031  let Inst{4-2}   = Zd;
2032  let Inst{1}     = op{0};
2033  let Inst{0}     = 0b0;
2034}
2035
2036// SME2 multi-vec FP to int convert four registers
2037// SME2 multi-vec int to FP four registers
2038multiclass sme2_fp_cvt_vg4_multi<string mnemonic, bits<7> op> {
2039  def NAME : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, mnemonic>;
2040}
2041
2042// SME2 multi-vec quadwords ZIP four registers
2043multiclass sme2_zip_vector_vg4<string mnemonic, bits<7> op> {
2044  def _B : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_b_mul_r, ZZZZ_b_mul_r,
2045                                        mnemonic>;
2046  def _H : sme2_frint_zip_cvt_vg4_multi<0b01, op, ZZZZ_h_mul_r, ZZZZ_h_mul_r,
2047                                        mnemonic>;
2048  def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r,
2049                                        mnemonic>;
2050  def _D : sme2_frint_zip_cvt_vg4_multi<0b11, op, ZZZZ_d_mul_r, ZZZZ_d_mul_r,
2051                                         mnemonic>;
2052}
2053
2054// SME2 multi-vec quadwords ZIP four registers
2055multiclass sme2_zip_vector_vg4_Q<string mnemonic, bits<7> op> {
2056  def NAME: sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_q_mul_r, ZZZZ_q_mul_r,
2057                                         mnemonic>;
2058}
2059
2060// SME2 multi-vec FRINT four registers
2061multiclass sme2_frint_vector_vg4_multi<string mnemonic, bits<7> op> {
2062  def _S :  sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r,
2063                                         mnemonic>;
2064}
2065
2066class sme2_cvt_vg2_single<string mnemonic, bits<4> op>
2067    : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn),
2068        mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {
2069  bits<4> Zn;
2070  bits<5> Zd;
2071  let Inst{31-23} = 0b110000010;
2072  let Inst{22}    = op{3};
2073  let Inst{21-18} = 0b1000;
2074  let Inst{17-16} = op{2-1};
2075  let Inst{15-10} = 0b111000;
2076  let Inst{9-6}   = Zn;
2077  let Inst{5}     = op{0};
2078  let Inst{4-0}   = Zd;
2079}
2080
2081// SME2 multi-vec FP down convert two registers
2082// SME2 multi-vec int down convert two registers
2083multiclass sme2_cvt_vg2_single<string mnemonic, bits<4> op, ValueType out_vt,
2084                               ValueType in_vt, SDPatternOperator intrinsic> {
2085  def NAME :  sme2_cvt_vg2_single<mnemonic, op>;
2086  def : SVE2p1_Cvt_VG2_Pat<NAME, intrinsic, out_vt, in_vt>;
2087}
2088
2089class sme2_cvt_unpk_vector_vg2<bits<2>sz, bits<3> op, bit u, RegisterOperand first_ty,
2090                           RegisterOperand second_ty, string mnemonic>
2091    : I<(outs first_ty:$Zd), (ins second_ty:$Zn),
2092        mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {
2093  bits<5> Zn;
2094  bits<4> Zd;
2095  let Inst{31-24} = 0b11000001;
2096  let Inst{23-22} = sz;
2097  let Inst{21-19} = 0b100;
2098  let Inst{18-16} = op;
2099  let Inst{15-10} = 0b111000;
2100  let Inst{9-5}   = Zn;
2101  let Inst{4-1}   = Zd;
2102  let Inst{0}     = u;
2103}
2104
2105// SME2 multi-vec unpack two registers
2106multiclass sme2_unpk_vector_vg2<string mnemonic, bit u> {
2107  def _H : sme2_cvt_unpk_vector_vg2<0b01, 0b101, u, ZZ_h_mul_r, ZPR8, mnemonic>;
2108  def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b101, u, ZZ_s_mul_r, ZPR16, mnemonic>;
2109  def _D : sme2_cvt_unpk_vector_vg2<0b11, 0b101, u, ZZ_d_mul_r, ZPR32, mnemonic>;
2110}
2111
2112// SME2.1 multi-vec convert two registers
2113multiclass sme2p1_fp_cvt_vector_vg2_single<string mnemonic, bit l> {
2114  def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b000, l, ZZ_s_mul_r, ZPR16, mnemonic>;
2115}
2116
2117class sme2_cvt_vg4_single<bit sz, bits<3> op, RegisterOperand first_ty,
2118                          RegisterOperand second_ty, string mnemonic>
2119    : I<(outs first_ty:$Zd), (ins second_ty:$Zn),
2120        mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {
2121  bits<3> Zn;
2122  bits<5> Zd;
2123  let Inst{31-24} = 0b11000001;
2124  let Inst{23}    = sz;
2125  let Inst{22}    = op{2};
2126  let Inst{21-10} = 0b110011111000;
2127  let Inst{9-7}   = Zn;
2128  let Inst{6-5}   = op{1-0};
2129  let Inst{4-0}   = Zd;
2130}
2131
2132// SME2 multi-vec int down convert four registers
2133multiclass sme2_int_cvt_vg4_single<string mnemonic, bits<3> op, SDPatternOperator intrinsic> {
2134  def _StoB : sme2_cvt_vg4_single<0, op, ZPR8, ZZZZ_s_mul_r, mnemonic>;
2135  def _DtoH : sme2_cvt_vg4_single<1, op, ZPR16, ZZZZ_d_mul_r, mnemonic>;
2136
2137  def : SME2_Cvt_VG4_Pat<NAME # _StoB, intrinsic, nxv16i8, nxv4i32>;
2138  def : SME2_Cvt_VG4_Pat<NAME # _DtoH, intrinsic, nxv8i16, nxv2i64>;
2139}
2140
2141class sme2_unpk_vector_vg4<bits<2>sz, bit u, RegisterOperand first_ty,
2142                           RegisterOperand second_ty, string mnemonic>
2143    : I<(outs first_ty:$Zd), (ins second_ty:$Zn),
2144        mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {
2145  bits<4> Zn;
2146  bits<3> Zd;
2147  let Inst{31-24} = 0b11000001;
2148  let Inst{23-22} = sz;
2149  let Inst{21-10} = 0b110101111000;
2150  let Inst{9-6}   = Zn;
2151  let Inst{5}     = 0b0;
2152  let Inst{4-2}   = Zd;
2153  let Inst{1}     = 0b0;
2154  let Inst{0}     = u;
2155}
2156
2157// SME2 multi-vec UNPK four registers
2158multiclass sme2_unpk_vector_vg4<string mnemonic, bit u> {
2159  def _H : sme2_unpk_vector_vg4<0b01, u, ZZZZ_h_mul_r, ZZ_b_mul_r, mnemonic>;
2160  def _S : sme2_unpk_vector_vg4<0b10, u, ZZZZ_s_mul_r, ZZ_h_mul_r, mnemonic>;
2161  def _D : sme2_unpk_vector_vg4<0b11, u, ZZZZ_d_mul_r, ZZ_s_mul_r, mnemonic>;
2162}
2163
2164//===----------------------------------------------------------------------===//
2165// SME2 multi-vec CLAMP registers
2166
2167class sme2_clamp_vector_vg24_multi<bits<2> sz, bits<3> op1, bit u,
2168                                   RegisterOperand multi_vector_ty,
2169                                   ZPRRegOp vector_ty, string mnemonic>
2170    : I<(outs multi_vector_ty:$Zd),
2171        (ins  multi_vector_ty:$_Zd, vector_ty:$Zn, vector_ty:$Zm),
2172        mnemonic, "\t$Zd, $Zn, $Zm",
2173        "", []>, Sched<[]>{
2174  bits<5> Zm;
2175  bits<5> Zn;
2176  let Inst{31-24} = 0b11000001;
2177  let Inst{23-22} = sz;
2178  let Inst{21}    = 0b1;
2179  let Inst{20-16} = Zm;
2180  let Inst{15-13} = 0b110;
2181  let Inst{12-10} = op1;
2182  let Inst{9-5}   = Zn;
2183  let Inst{0}     = u;
2184
2185  let Constraints = "$Zd = $_Zd";
2186}
2187
2188class sme2_clamp_vector_vg2_multi<bits<2> sz, bits<3> op1, bit u,
2189                                  RegisterOperand multi_vector_ty,
2190                                  ZPRRegOp vector_ty, string mnemonic>
2191    : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty,
2192                                   mnemonic>{
2193  bits<4> Zd;
2194  let Inst{4-1} = Zd;
2195}
2196
2197multiclass sme2_fp_clamp_vector_vg2_multi<string mnemonic>{
2198  def _H : sme2_clamp_vector_vg2_multi<0b01, 0b000, 0b0, ZZ_h_mul_r, ZPR16, mnemonic>;
2199  def _S : sme2_clamp_vector_vg2_multi<0b10, 0b000, 0b0, ZZ_s_mul_r, ZPR32, mnemonic>;
2200  def _D : sme2_clamp_vector_vg2_multi<0b11, 0b000, 0b0, ZZ_d_mul_r, ZPR64, mnemonic>;
2201}
2202
2203multiclass sme2_int_clamp_vector_vg2_multi<string mnemonic, bit u>{
2204  def _B : sme2_clamp_vector_vg2_multi<0b00, 0b001, u, ZZ_b_mul_r, ZPR8, mnemonic>;
2205  def _H : sme2_clamp_vector_vg2_multi<0b01, 0b001, u, ZZ_h_mul_r, ZPR16, mnemonic>;
2206  def _S : sme2_clamp_vector_vg2_multi<0b10, 0b001, u, ZZ_s_mul_r, ZPR32, mnemonic>;
2207  def _D : sme2_clamp_vector_vg2_multi<0b11, 0b001, u, ZZ_d_mul_r, ZPR64, mnemonic>;
2208}
2209
2210// SME2.1 multi-vec FCLAMP two registers
2211multiclass sme2p1_bfclamp_vector_vg2_multi<string mnemonic> {
2212  def _H : sme2_clamp_vector_vg2_multi<0b00, 0b000, 0b0, ZZ_h_mul_r, ZPR16,
2213                                           mnemonic>;
2214}
2215
2216class sme2_clamp_vector_vg4_multi<bits<2> sz, bits<3> op1, bit u,
2217                                  RegisterOperand multi_vector_ty,
2218                                  ZPRRegOp vector_ty, string mnemonic>
2219    : sme2_clamp_vector_vg24_multi<sz, op1, u,  multi_vector_ty, vector_ty,
2220                                   mnemonic>{
2221  bits<3> Zd;
2222  let Inst{4-2} = Zd;
2223  let Inst{1}   = 0b0;
2224}
2225
2226multiclass sme2_fp_clamp_vector_vg4_multi<string mnemonic>{
2227  def _H : sme2_clamp_vector_vg4_multi<0b01, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, mnemonic>;
2228  def _S : sme2_clamp_vector_vg4_multi<0b10, 0b010, 0b0, ZZZZ_s_mul_r, ZPR32, mnemonic>;
2229  def _D : sme2_clamp_vector_vg4_multi<0b11, 0b010, 0b0, ZZZZ_d_mul_r, ZPR64, mnemonic>;
2230}
2231
2232multiclass sme2_int_clamp_vector_vg4_multi<string mnemonic, bit u>{
2233  def _B : sme2_clamp_vector_vg4_multi<0b00, 0b011, u, ZZZZ_b_mul_r, ZPR8, mnemonic>;
2234  def _H : sme2_clamp_vector_vg4_multi<0b01, 0b011, u, ZZZZ_h_mul_r, ZPR16, mnemonic>;
2235  def _S : sme2_clamp_vector_vg4_multi<0b10, 0b011, u, ZZZZ_s_mul_r, ZPR32, mnemonic>;
2236  def _D : sme2_clamp_vector_vg4_multi<0b11, 0b011, u, ZZZZ_d_mul_r, ZPR64, mnemonic>;
2237}
2238
2239// SME2.1 multi-vec FCLAMP four registers
2240multiclass sme2p1_bfclamp_vector_vg4_multi<string mnemonic> {
2241  def _H : sme2_clamp_vector_vg4_multi<0b00, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16,
2242                                       mnemonic>;
2243}
2244
2245// SME2 multi-vec ZIP two registers
2246class sme2_zip_vector_vg2<bits<2> sz, bit q, bit u,
2247                         RegisterOperand multi_vector_ty,
2248                         ZPRRegOp vector_ty, string mnemonic>
2249    : I<(outs multi_vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm),
2250        mnemonic, "\t$Zd, $Zn, $Zm",
2251        "", []>, Sched<[]>{
2252  bits<4> Zd;
2253  bits<5> Zm;
2254  bits<5> Zn;
2255  let Inst{31-24} = 0b11000001;
2256  let Inst{23-22} = sz;
2257  let Inst{21}    = 0b1;
2258  let Inst{20-16} = Zm;
2259  let Inst{15-11} = 0b11010;
2260  let Inst{10}    = q;
2261  let Inst{9-5}   = Zn;
2262  let Inst{4-1}   = Zd;
2263  let Inst{0}     = u;
2264}
2265
2266multiclass sme2_zip_vector_vg2<string mnemonic, bit op> {
2267  def _B : sme2_zip_vector_vg2<0b00, 0b0, op, ZZ_b_mul_r, ZPR8, mnemonic>;
2268  def _H : sme2_zip_vector_vg2<0b01, 0b0, op, ZZ_h_mul_r, ZPR16, mnemonic>;
2269  def _S : sme2_zip_vector_vg2<0b10, 0b0, op, ZZ_s_mul_r, ZPR32, mnemonic>;
2270  def _D : sme2_zip_vector_vg2<0b11, 0b0, op, ZZ_d_mul_r, ZPR64, mnemonic>;
2271  def _Q : sme2_zip_vector_vg2<0b00, 0b1, op, ZZ_q_mul_r, ZPR128, mnemonic>;
2272}
2273
2274//===----------------------------------------------------------------------===//
2275// SME2 Dot Products and MLA
2276
2277class sme2_multi_vec_array_vg2_index<bit sz, bits<6> op, MatrixOperand matrix_ty,
2278                                     RegisterOperand multi_vector_ty,
2279                                     ZPRRegOp vector_ty, Operand index_ty,
2280                                     string mnemonic>
2281    : I<(outs matrix_ty:$ZAda),
2282        (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
2283         multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i),
2284         mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i",
2285        "", []>, Sched<[]> {
2286  bits<4> Zm;
2287  bits<2> Rv;
2288  bits<4> Zn;
2289  bits<3> imm3;
2290  let Inst{31-23} = 0b110000010;
2291  let Inst{22}    = sz;
2292  let Inst{21-20} = 0b01;
2293  let Inst{19-16} = Zm;
2294  let Inst{15}    = 0b0;
2295  let Inst{14-13} = Rv;
2296  let Inst{12-10} = op{5-3};
2297  let Inst{9-6}   = Zn;
2298  let Inst{5-3}   = op{2-0};
2299  let Inst{2-0}   = imm3;
2300
2301  let Constraints = "$ZAda = $_ZAda";
2302}
2303
2304// SME2 multi-vec ternary indexed two registers 32-bit
2305multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<4> op,
2306                                              RegisterOperand multi_vector_ty,
2307                                              ZPRRegOp vector_ty, ValueType vt,
2308                                              SDPatternOperator intrinsic> {
2309  def NAME : sme2_multi_vec_array_vg2_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, vector_ty,
2310                                             VectorIndexS32b_timm,  mnemonic>, SMEPseudo2Instr<NAME, 1> {
2311    bits<2> i;
2312    let Inst{11-10} = i;
2313  }
2314
2315  def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>;
2316
2317  def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>;
2318
2319  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",
2320        (!cast<Instruction>(NAME) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
2321        multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>;
2322}
2323
2324// SME2.1 multi-vec ternary indexed two registers 16-bit
2325multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> op> {
2326  def NAME : sme2_multi_vec_array_vg2_index<0b0, {0b1,?,?,op,?}, MatrixOp16,
2327                                            ZZ_h_mul_r, ZPR4b16,
2328                                            VectorIndexH, mnemonic> {
2329    bits<3> i;
2330    let Inst{11-10} = i{2-1};
2331    let Inst{3}     = i{0};
2332  }
2333  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",
2334        (!cast<Instruction>(NAME) MatrixOp16:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
2335        ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>;
2336}
2337
2338// SME2 multi-vec ternary indexed two registers 64-bit
2339
2340class sme2_multi_vec_array_vg2_index_64b<bits<2> op,
2341                                         RegisterOperand multi_vector_ty,
2342                                         ZPRRegOp vector_ty,
2343                                         string mnemonic>
2344    : I<(outs MatrixOp64:$ZAda),
2345        (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
2346         multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1),
2347        mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i1",
2348        "", []>, Sched<[]> {
2349  bits<4> Zm;
2350  bits<2> Rv;
2351  bits<1> i1;
2352  bits<4> Zn;
2353  bits<3> imm3;
2354  let Inst{31-20} = 0b110000011101;
2355  let Inst{19-16} = Zm;
2356  let Inst{15}    = 0b0;
2357  let Inst{14-13} = Rv;
2358  let Inst{12-11} = 0b00;
2359  let Inst{10}    = i1;
2360  let Inst{9-6}   = Zn;
2361  let Inst{5}     = 0b0;
2362  let Inst{4-3}   = op;
2363  let Inst{2-0}   = imm3;
2364
2365  let Constraints = "$ZAda = $_ZAda";
2366}
2367
2368multiclass sme2_multi_vec_array_vg2_index_64b<string mnemonic, bits<2> op,
2369                                              RegisterOperand multi_vector_ty,
2370                                              ZPRRegOp vector_ty, ValueType vt,
2371                                              SDPatternOperator intrinsic> {
2372  def NAME : sme2_multi_vec_array_vg2_index_64b<op, multi_vector_ty, vector_ty,
2373                                                mnemonic>, SMEPseudo2Instr<NAME, 1>;
2374
2375  def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>;
2376
2377  def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexD32b_timm, tileslice16>;
2378
2379  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1",
2380        (!cast<Instruction>(NAME) MatrixOp64:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
2381        multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>;
2382}
2383
2384class sme2_multi_vec_array_vg4_index<bit sz, bits<6> op, MatrixOperand matrix_ty,
2385                                     RegisterOperand multi_vector_ty,
2386                                     ZPRRegOp vector_ty, Operand index_ty,
2387                                     string mnemonic>
2388    : I<(outs matrix_ty:$ZAda),
2389        (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
2390         multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i),
2391         mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i",
2392        "", []>, Sched<[]> {
2393  bits<4> Zm;
2394  bits<2> Rv;
2395  bits<3> Zn;
2396  bits<3> imm3;
2397  let Inst{31-23} = 0b110000010;
2398  let Inst{22}    = sz;
2399  let Inst{21-20} = 0b01;
2400  let Inst{19-16} = Zm;
2401  let Inst{15}    = 0b1;
2402  let Inst{14-13} = Rv;
2403  let Inst{12-10} = op{5-3};
2404  let Inst{9-7}   = Zn;
2405  let Inst{6}     = 0b0;
2406  let Inst{5-3}   = op{2-0};
2407  let Inst{2-0}   = imm3;
2408
2409  let Constraints = "$ZAda = $_ZAda";
2410}
2411
2412// SME2 multi-vec ternary indexed four registers 32-bit
2413multiclass sme2_multi_vec_array_vg4_index_32b<string mnemonic, bits<4> op,
2414                                              RegisterOperand multi_vector_ty,
2415                                              ZPRRegOp vector_ty, ValueType vt,
2416                                              SDPatternOperator intrinsic> {
2417  def NAME : sme2_multi_vec_array_vg4_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32,  multi_vector_ty,
2418                                            vector_ty, VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> {
2419   bits<2> i;
2420   let Inst{11-10} = i;
2421  }
2422
2423  def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>;
2424
2425  def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>;
2426
2427  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",
2428        (!cast<Instruction>(NAME) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
2429        multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>;
2430}
2431
2432// SME2.1 multi-vec ternary indexed four registers 16-bit
2433multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<2> op> {
2434  def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16,
2435                                            ZZZZ_h_mul_r, ZPR4b16,
2436                                            VectorIndexH, mnemonic>{
2437    bits<3> i;
2438    let Inst{11-10} = i{2-1};
2439    let Inst{3}     = i{0};
2440  }
2441
2442  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",
2443        (!cast<Instruction>(NAME) MatrixOp16:$ZAda,  MatrixIndexGPR32Op8_11:$Rv,
2444        sme_elm_idx0_7:$imm3, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>;
2445}
2446
2447// SME2 multi-vec ternary indexed four registers 64-bit
2448class sme2_multi_vec_array_vg4_index_64b<bits<3> op,
2449                                         RegisterOperand multi_vector_ty,
2450                                         ZPRRegOp vector_ty,
2451                                         string mnemonic>
2452    : I<(outs MatrixOp64:$ZAda),
2453        (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
2454         multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1),
2455        mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i1",
2456        "", []>, Sched<[]> {
2457  bits<4> Zm;
2458  bits<2> Rv;
2459  bits<1> i1;
2460  bits<3> Zn;
2461  bits<3> imm3;
2462  let Inst{31-20} = 0b110000011101;
2463  let Inst{19-16} = Zm;
2464  let Inst{15}    = 0b1;
2465  let Inst{14-13} = Rv;
2466  let Inst{12}    = 0b0;
2467  let Inst{11}    = op{2};
2468  let Inst{10}    = i1;
2469  let Inst{9-7}   = Zn;
2470  let Inst{6-5}   = 0b00;
2471  let Inst{4-3}   = op{1-0};
2472  let Inst{2-0}   = imm3;
2473
2474  let Constraints = "$ZAda = $_ZAda";
2475}
2476
2477multiclass sme2_multi_vec_array_vg4_index_64b<string mnemonic, bits<3> op,
2478                                              RegisterOperand multi_vector_ty,
2479                                              ZPRRegOp vector_ty, ValueType vty,
2480                                              SDPatternOperator intrinsic> {
2481  def NAME : sme2_multi_vec_array_vg4_index_64b<op, multi_vector_ty, vector_ty,
2482                                                mnemonic>, SMEPseudo2Instr<NAME, 1>;
2483
2484  def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>;
2485
2486  def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vty, VectorIndexD32b_timm, tileslice16>;
2487
2488  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1",
2489        (!cast<Instruction>(NAME) MatrixOp64:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
2490        multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>;
2491}
2492//===----------------------------------------------------------------------===//
2493// SME2 multi-vec indexed long long MLA one source 32-bit
2494class sme2_mla_ll_array_index_32b<string mnemonic, bits<3> op>
2495    : I<(outs MatrixOp32:$ZAda),
2496        (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR8:$Zn, ZPR4b8:$Zm, VectorIndexB:$i),
2497        mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i",
2498        "", []>, Sched<[]> {
2499  bits<4> Zm;
2500  bits<2> Rv;
2501  bits<4> i;
2502  bits<5> Zn;
2503  bits<2> imm2;
2504  let Inst{31-20} = 0b110000010000;
2505  let Inst{19-16} = Zm;
2506  let Inst{15}    = i{3};
2507  let Inst{14-13} = Rv;
2508  let Inst{12-10} = i{2-0};
2509  let Inst{9-5}   = Zn;
2510  let Inst{4-2}   = op;
2511  let Inst{1-0}   = imm2;
2512
2513  let Constraints = "$ZAda = $_ZAda";
2514}
2515
2516// SME2 multi-vec indexed long long MLA one source 64-bit
2517
2518class sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op>
2519    : I<(outs MatrixOp64:$ZAda),
2520        (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR16:$Zn, ZPR4b16:$Zm, VectorIndexH:$i),
2521        mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i",
2522        "", []>, Sched<[]> {
2523  bits<4> Zm;
2524  bits<2> Rv;
2525  bits<3> i;
2526  bits<5> Zn;
2527  bits<2> imm2;
2528  let Inst{31-20} = 0b110000011000;
2529  let Inst{19-16} = Zm;
2530  let Inst{15}    = i{2};
2531  let Inst{14-13} = Rv;
2532  let Inst{12}    = 0b0;
2533  let Inst{11-10} = i{1-0};
2534  let Inst{9-5}   = Zn;
2535  let Inst{4-3}   = op;
2536  let Inst{2}     = 0b0;
2537  let Inst{1-0}   = imm2;
2538
2539  let Constraints = "$ZAda = $_ZAda";
2540}
2541
2542class sme2_mla_ll_array_vg24_index_32b<bit vg4, bits<3> op,
2543                                       RegisterOperand vector_ty,
2544                                       string mnemonic>
2545    : I<(outs MatrixOp32:$ZAda),
2546        (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,
2547             vector_ty:$Zn, ZPR4b8:$Zm, VectorIndexB:$i),
2548        mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i",
2549        "", []>, Sched<[]> {
2550  bits<4> Zm;
2551  bits<2> Rv;
2552  bits<4> i;
2553  bit     imm;
2554  let Inst{31-20} = 0b110000010001;
2555  let Inst{19-16} = Zm;
2556  let Inst{15}    = vg4;
2557  let Inst{14-13} = Rv;
2558  let Inst{12}    = 0b0;
2559  let Inst{11-10} = i{3-2};
2560  let Inst{5-3}   = op;
2561  let Inst{2-1}   = i{1-0};
2562  let Inst{0}     = imm;
2563
2564  let Constraints = "$ZAda = $_ZAda";
2565}
2566
2567//SME2 multi-vec indexed long long MLA two sources 32-bit
2568
2569multiclass sme2_mla_ll_array_vg2_index_32b<string mnemonic, bits<3> op> {
2570  def NAME: sme2_mla_ll_array_vg24_index_32b<0b0, op, ZZ_b_mul_r, mnemonic> {
2571   bits<4> Zn;
2572   let Inst{9-6} = Zn;
2573  }
2574
2575  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i",
2576                 (!cast<Instruction>(NAME) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 0>;
2577}
2578
2579// SME2 multi-vec indexed long long MLA four sources 32-bit
2580
2581multiclass sme2_mla_ll_array_vg4_index_32b<string mnemonic, bits<3> op> {
2582  def NAME: sme2_mla_ll_array_vg24_index_32b<0b1, op, ZZZZ_b_mul_r, mnemonic> {
2583   bits<3> Zn;
2584   let Inst{9-7} = Zn;
2585   let Inst{6}   = 0b0;
2586  }
2587
2588  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i",
2589                 (!cast<Instruction>(NAME) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 0>;
2590}
2591class sme2_mla_ll_array_vg24_index_64b<bit vg4,  bits<2> op,
2592                                       RegisterOperand vector_ty,
2593                                       string mnemonic>
2594    : I<(outs MatrixOp64:$ZAda),
2595        (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,
2596             vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH:$i),
2597        mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i",
2598        "", []>, Sched<[]> {
2599  bits<4> Zm;
2600  bits<2> Rv;
2601  bits<3> i;
2602  bit     imm;
2603  let Inst{31-20} = 0b110000011001;
2604  let Inst{19-16} = Zm;
2605  let Inst{15}    = vg4;
2606  let Inst{14-13} = Rv;
2607  let Inst{12-11} = 0b00;
2608  let Inst{10}    = i{2};
2609  let Inst{5}     = 0b0;
2610  let Inst{4-3}   = op;
2611  let Inst{2-1}   = i{1-0};
2612  let Inst{0}     = imm;
2613
2614  let Constraints = "$ZAda = $_ZAda";
2615}
2616
2617// SME2 multi-vec indexed long long MLA two sources 64-bit
2618
2619multiclass sme2_mla_ll_array_vg2_index_64b<string mnemonic, bits<2> op> {
2620  def NAME: sme2_mla_ll_array_vg24_index_64b<0b0, op, ZZ_h_mul_r, mnemonic>{
2621    bits<4> Zn;
2622    let Inst{9-6} = Zn;
2623  }
2624
2625  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i",
2626                 (!cast<Instruction>(NAME) MatrixOp64:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>;
2627}
2628
2629// SME2 multi-vec indexed long long MLA four sources 64-bit
2630
2631multiclass sme2_mla_ll_array_vg4_index_64b<string mnemonic, bits<2> op> {
2632  def NAME: sme2_mla_ll_array_vg24_index_64b<0b1, op, ZZZZ_h_mul_r,  mnemonic>{
2633    bits<3> Zn;
2634    let Inst{9-7} = Zn;
2635    let Inst{6}   = 0b0;
2636  }
2637
2638  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i",
2639                 (!cast<Instruction>(NAME) MatrixOp64:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>;
2640}
2641
2642
2643//SME2 multiple and single vector long long FMA one source
2644
2645class sme2_mla_ll_array_single<string mnemonic, bits<4> op,
2646                               MatrixOperand matrix_ty, ZPRRegOp vector_ty,
2647                               ZPRRegOp zpr_ty>
2648    : I<(outs matrix_ty:$ZAda),
2649        (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm,
2650             vector_ty:$Zn, zpr_ty:$Zm),
2651        mnemonic, "\t$ZAda[$Rv, $imm], $Zn, $Zm",
2652        "", []>, Sched<[]> {
2653  bits<4> Zm;
2654  bits<2> Rv;
2655  bits<5> Zn;
2656  bits<2> imm;
2657  let Inst{31-23} = 0b110000010;
2658  let Inst{22}    = op{3}; //sz
2659  let Inst{21-20} = 0b10;
2660  let Inst{19-16} = Zm;
2661  let Inst{15}    = 0b0;
2662  let Inst{14-13} = Rv;
2663  let Inst{12-10} = 0b001;
2664  let Inst{9-5}   = Zn;
2665  let Inst{4-2}   = op{2-0};
2666  let Inst{1-0}   = imm;
2667
2668  let Constraints = "$ZAda = $_ZAda";
2669}
2670
2671class sme2_mla_ll_array_vg24_single<bits<5> op, MatrixOperand matrix_ty,
2672                                    RegisterOperand vector_ty, ZPRRegOp zpr_ty,
2673                                    string mnemonic>
2674    : I<(outs matrix_ty:$ZAda),
2675        (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,
2676             vector_ty:$Zn, zpr_ty:$Zm),
2677        mnemonic, "\t$ZAda[$Rv, $imm,  " # !if(op{3}, "vgx4", "vgx2") # "], $Zn, $Zm",
2678        "", []>, Sched<[]> {
2679  bits<4> Zm;
2680  bits<2> Rv;
2681  bits<5> Zn;
2682  bit     imm;
2683  let Inst{31-23} = 0b110000010;
2684  let Inst{22}    = op{4}; //sz
2685  let Inst{21}    = 0b1;
2686  let Inst{20}    = op{3}; //vg4
2687  let Inst{19-16} = Zm;
2688  let Inst{15}    = 0b0;
2689  let Inst{14-13} = Rv;
2690  let Inst{12-10} = 0b000;
2691  let Inst{9-5}   = Zn;
2692  let Inst{4-2}   = op{2-0};
2693  let Inst{1}     = 0b0;
2694  let Inst{0}     = imm;
2695
2696  let Constraints = "$ZAda = $_ZAda";
2697}
2698
2699//SME2 single-multi long long MLA two and four sources
2700
2701multiclass sme2_mla_ll_array_vg24_single<string mnemonic, bits<5> op,
2702                                          MatrixOperand matrix_ty,
2703                                          RegisterOperand multi_vector_ty,
2704                                          ZPRRegOp zpr_ty> {
2705  def NAME: sme2_mla_ll_array_vg24_single<op, matrix_ty, multi_vector_ty,
2706                                          zpr_ty, mnemonic>;
2707
2708  def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm], $Zn, $Zm",
2709                 (!cast<Instruction>(NAME) matrix_ty:$ZAd,  MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>;
2710}
2711
2712// SME2 multiple vectors long long MLA two sources
2713
2714class sme2_mla_ll_array_vg2_multi<bits<4> op, MatrixOperand matrix_ty,
2715                                  RegisterOperand vector_ty,string mnemonic>
2716    : I<(outs matrix_ty:$ZAda),
2717        (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,
2718             vector_ty:$Zn, vector_ty:$Zm),
2719        mnemonic, "\t$ZAda[$Rv, $imm, vgx2], $Zn, $Zm",
2720        "", []>, Sched<[]> {
2721  bits<4> Zm;
2722  bits<2> Rv;
2723  bits<4> Zn;
2724  bit     imm;
2725  let Inst{31-23} = 0b110000011;
2726  let Inst{22}    = op{3};  // sz
2727  let Inst{21}    = 0b1;
2728  let Inst{20-17} = Zm;
2729  let Inst{16-15} = 0b00;
2730  let Inst{14-13} = Rv;
2731  let Inst{12-10} = 0b000;
2732  let Inst{9-6}   = Zn;
2733  let Inst{5}     = 0b0;
2734  let Inst{4-2}   = op{2-0};
2735  let Inst{1}     = 0b0;
2736  let Inst{0}     = imm;
2737
2738  let Constraints = "$ZAda = $_ZAda";
2739}
2740
2741multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<4> op,
2742                                       MatrixOperand matrix_ty,
2743                                       RegisterOperand vector_ty> {
2744  def NAME : sme2_mla_ll_array_vg2_multi<op, matrix_ty, vector_ty, mnemonic>;
2745
2746  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
2747                 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>;
2748}
2749
2750// SME2 multiple vectors long long MLA four sources
2751
2752class sme2_mla_ll_array_vg4_multi<bits<4> op,MatrixOperand matrix_ty,
2753                                  RegisterOperand vector_ty,
2754                                  string mnemonic>
2755    : I<(outs matrix_ty:$ZAda),
2756        (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,
2757             vector_ty:$Zn, vector_ty:$Zm),
2758        mnemonic, "\t$ZAda[$Rv, $imm, vgx4], $Zn, $Zm",
2759        "", []>, Sched<[]> {
2760  bits<3> Zm;
2761  bits<2> Rv;
2762  bits<3> Zn;
2763  bit     imm;
2764  let Inst{31-23} = 0b110000011;
2765  let Inst{22}    = op{3}; // sz
2766  let Inst{21}    = 0b1;
2767  let Inst{20-18} = Zm;
2768  let Inst{17-15} = 0b010;
2769  let Inst{14-13} = Rv;
2770  let Inst{12-10} = 0b000;
2771  let Inst{9-7}   = Zn;
2772  let Inst{6-5}   = 0b00;
2773  let Inst{4-2}   = op{2-0};
2774  let Inst{1}     = 0b0;
2775  let Inst{0}     = imm;
2776
2777  let Constraints = "$ZAda = $_ZAda";
2778}
2779
2780multiclass sme2_mla_ll_array_vg4_multi<string mnemonic, bits<4> op,
2781                                       MatrixOperand matrix_ty,
2782                                       RegisterOperand vector_ty> {
2783  def NAME : sme2_mla_ll_array_vg4_multi<op, matrix_ty, vector_ty, mnemonic>;
2784
2785  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
2786                 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>;
2787}
2788
2789//===----------------------------------------------------------------------===//
2790// SME2 Outer Product and Accumulate
2791
2792multiclass sme2_int_mopx_tile<string mnemonic, bits<3> op> {
2793  def NAME : sme_int_outer_product_inst<op, 0b0, 0b1, TileOp32, ZPR16, mnemonic> {
2794    bits<2> ZAda;
2795    let Inst{1-0} = ZAda;
2796    let Inst{2}   = 0b0;
2797  }
2798}
2799
2800multiclass  sme2_bfp_mopx_tile<string mnemonic, bits<3> op> {
2801  def NAME : sme_outer_product_widening_inst<op, ZPR32, mnemonic>;
2802}
2803
2804//===----------------------------------------------------------------------===///
2805// SME2 Zero Lookup Table.
2806class sme2_zero_zt<string mnemonic, bits<4> opc>
2807    : I<(outs ZTR:$ZT), (ins ),
2808         mnemonic, "\t\\{ $ZT \\}",
2809         "", []>, Sched<[]> {
2810  let Inst{31-4} = 0b1100000001001000000000000000;
2811  let Inst{3-0}  = opc;
2812}
2813
2814//===----------------------------------------------------------------------===//
2815// SME2 lookup table load/store
2816class sme2_spill_fill_vector<string mnemonic, bits<8> opc>
2817    : I<!if(opc{7}, (outs ), (outs ZTR:$ZTt)),
2818        !if(opc{7}, (ins ZTR:$ZTt, GPR64sp:$Rn), (ins GPR64sp:$Rn)),
2819        mnemonic, "\t$ZTt, [$Rn]",
2820        "", []>, Sched<[]> {
2821  bits<5> Rn;
2822  let Inst{31-22} = 0b1110000100;
2823  let Inst{21-16} = opc{7-2};
2824  let Inst{15-10} = 0b100000;
2825  let Inst{9-5}   = Rn;
2826  let Inst{4-2}   = 0b000;
2827  let Inst{1-0}   = opc{1-0};
2828
2829  let mayLoad     = !not(opc{7});
2830  let mayStore    = opc{7};
2831}
2832
2833//===----------------------------------------------------------------------===///
2834// SME2 move to/from lookup table
2835class sme2_movt_zt_to_scalar<string mnemonic, bits<7> opc>
2836    : I<(outs GPR64:$Rt), (ins ZTR:$ZTt, uimm3s8:$imm3),
2837         mnemonic, "\t$Rt, $ZTt$imm3",
2838         "", []>, Sched<[]> {
2839  bits<3> imm3;
2840  bits<5> Rt;
2841  let Inst{31-15} = 0b11000000010011000;
2842  let Inst{14-12} = imm3;
2843  let Inst{11-5}  = opc;
2844  let Inst{4-0}   = Rt;
2845}
2846
2847class sme2_movt_scalar_to_zt<string mnemonic, bits<7> opc>
2848    : I<(outs ZTR:$ZTt), (ins uimm3s8:$imm3, GPR64:$Rt),
2849         mnemonic, "\t$ZTt$imm3, $Rt",
2850         "", []>, Sched<[]> {
2851  bits<3> imm3;
2852  bits<5> Rt;
2853  let Inst{31-15} = 0b11000000010011100;
2854  let Inst{14-12} = imm3;
2855  let Inst{11-5}  = opc;
2856  let Inst{4-0}   = Rt;
2857}
2858
2859//===----------------------------------------------------------------------===//
2860// SME2 lookup table expand one register
2861class sme2_luti_vector_index<bits<2> sz, bits<7> opc, RegisterOperand vector_ty,
2862                             AsmVectorIndexOpnd index_ty, string mnemonic>
2863    : I<(outs vector_ty:$Zd),
2864        (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),
2865        mnemonic, "\t$Zd, $ZTt, $Zn$i",
2866        "", []>, Sched<[]> {
2867  bits<5> Zn;
2868  bits<5> Zd;
2869  let Inst{31-19} = 0b1100000011001;
2870  let Inst{18-14} = opc{6-2};
2871  let Inst{13-12} = sz;
2872  let Inst{11-10} = opc{1-0};
2873  let Inst{9-5}   = Zn;
2874  let Inst{4-0}   = Zd;
2875}
2876
2877class sme2_luti2_vector_index<bits<2> sz, RegisterOperand vector_ty,
2878                              string mnemonic>
2879    : sme2_luti_vector_index<sz, {1,?,?,?,?,0,0}, vector_ty, VectorIndexB, mnemonic> {
2880  bits<4> i;
2881  let Inst{17-14} = i;
2882}
2883
2884multiclass sme2_luti2_vector_index<string mnemonic> {
2885  def _B : sme2_luti2_vector_index<0b00, ZPR8, mnemonic>;
2886  def _H : sme2_luti2_vector_index<0b01, ZPR16, mnemonic>;
2887  def _S : sme2_luti2_vector_index<0b10, ZPR32, mnemonic>;
2888}
2889
2890class sme2_luti4_vector_index<bits<2> sz, RegisterOperand vector_ty,
2891                              string mnemonic>
2892    : sme2_luti_vector_index<sz, {0,1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> {
2893  bits<3> i;
2894  let Inst{16-14} = i;
2895}
2896
2897multiclass sme2_luti4_vector_index<string mnemonic> {
2898  def _B : sme2_luti4_vector_index<0b00, ZPR8, mnemonic>;
2899  def _H : sme2_luti4_vector_index<0b01, ZPR16, mnemonic>;
2900  def _S : sme2_luti4_vector_index<0b10, ZPR32, mnemonic>;
2901}
2902
2903// SME2 lookup table expand two contiguous registers
2904class sme2_luti_vector_vg2_index<bits<2> sz, bits<6> opc, RegisterOperand vector_ty,
2905                                 AsmVectorIndexOpnd index_ty, string mnemonic>
2906    : I<(outs vector_ty:$Zd),
2907        (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),
2908        mnemonic, "\t$Zd, $ZTt, $Zn$i",
2909        "", []>, Sched<[]> {
2910  bits<5> Zn;
2911  bits<4> Zd;
2912  let Inst{31-19} = 0b1100000010001;
2913  let Inst{18-15} = opc{5-2};
2914  let Inst{14}    = 0b1;
2915  let Inst{13-12} = sz;
2916  let Inst{11-10} = opc{1-0};
2917  let Inst{9-5}   = Zn;
2918  let Inst{4-1}   = Zd;
2919  let Inst{0}     = 0b0;
2920}
2921
2922class sme2_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty,
2923                                  string mnemonic>
2924    : sme2_luti_vector_vg2_index<sz, {1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> {
2925  bits<3> i;
2926  let Inst{17-15} = i;
2927}
2928
2929multiclass sme2_luti2_vector_vg2_index<string mnemonic> {
2930  def _B : sme2_luti2_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>;
2931  def _H : sme2_luti2_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>;
2932  def _S : sme2_luti2_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>;
2933}
2934
2935class sme2_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty,
2936                                 string mnemonic>
2937    : sme2_luti_vector_vg2_index<sz, {0,1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> {
2938  bits<2> i;
2939  let Inst{16-15} = i;
2940}
2941
2942multiclass sme2_luti4_vector_vg2_index<string mnemonic> {
2943  def _B : sme2_luti4_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>;
2944  def _H : sme2_luti4_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>;
2945  def _S : sme2_luti4_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>;
2946}
2947
2948// SME2 lookup table expand four contiguous registers
2949class sme2_luti_vector_vg4_index<bits<2> sz, bits<5>opc, RegisterOperand vector_ty,
2950                                 AsmVectorIndexOpnd index_ty, string mnemonic>
2951    : I<(outs vector_ty:$Zd),
2952        (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),
2953        mnemonic, "\t$Zd, $ZTt, $Zn$i",
2954        "", []>, Sched<[]> {
2955  bits<5> Zn;
2956  bits<3> Zd;
2957  let Inst{31-19} = 0b1100000010001;
2958  let Inst{18-16} = opc{4-2};
2959  let Inst{15-14} = 0b10;
2960  let Inst{13-12} = sz;
2961  let Inst{11-10} = opc{1-0};
2962  let Inst{9-5}   = Zn;
2963  let Inst{4-2}   = Zd;
2964  let Inst{1-0}   = 0b00;
2965}
2966
2967class sme2_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty,
2968                                  string mnemonic>
2969    : sme2_luti_vector_vg4_index<sz, {1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> {
2970  bits<2> i;
2971  let Inst{17-16} = i;
2972}
2973
2974multiclass sme2_luti2_vector_vg4_index<string mnemonic> {
2975  def _B : sme2_luti2_vector_vg4_index<0b00, ZZZZ_b_mul_r, mnemonic>;
2976  def _H : sme2_luti2_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>;
2977  def _S : sme2_luti2_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>;
2978}
2979
2980class sme2_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty,
2981                                  string mnemonic>
2982    : sme2_luti_vector_vg4_index<sz, {0,1,?,0,0}, vector_ty, VectorIndexD, mnemonic> {
2983  bits<1> i;
2984  let Inst{16}    = i;
2985}
2986
2987multiclass sme2_luti4_vector_vg4_index<string mnemonic> {
2988  def _H : sme2_luti4_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>;
2989  def _S : sme2_luti4_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>;
2990}
2991
2992//===----------------------------------------------------------------------===//
2993// SME2 MOV
2994class sme2_mova_vec_to_tile_vg2_multi_base<bits<2> sz, bit v,
2995                                           RegisterOperand tile_ty,
2996                                           Operand index_ty,
2997                                           RegisterOperand vector_ty,
2998                                           string mnemonic>
2999   : I<(outs tile_ty:$ZAd),
3000       (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, vector_ty:$Zn),
3001       mnemonic, "\t$ZAd[$Rs, $imm], $Zn",
3002       "", []>, Sched<[]> {
3003  bits<2> Rs;
3004  bits<4> Zn;
3005  let Inst{31-24} = 0b11000000;
3006  let Inst{23-22} = sz;
3007  let Inst{21-16} = 0b000100;
3008  let Inst{15}    = v;
3009  let Inst{14-13} = Rs;
3010  let Inst{12-10} = 0b000;
3011  let Inst{9-6}   = Zn;
3012  let Inst{5-3}   = 0b000;
3013
3014  let Constraints = "$ZAd = $_ZAd";
3015}
3016
3017multiclass sme2_mova_vec_to_tile_or_array_aliases<int prefer, Instruction inst,
3018                                                  RegisterOperand tile_or_array_ty,
3019                                                  RegisterOperand  rv_ty,
3020                                                  Operand index_ty,
3021                                                  RegisterOperand vector_ty,
3022                                                  string mnemonic,
3023                                                  string vg_acronym=""> {
3024  def : InstAlias<mnemonic # "\t$ZAd[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn",
3025                  (inst tile_or_array_ty:$ZAd, rv_ty:$Rs, index_ty:$imm, vector_ty:$Zn), prefer>;
3026
3027}
3028
3029// SME2 move vector to tile, two registers
3030multiclass sme2_mova_vec_to_tile_vg2_multi_base<bit v, string mnemonic> {
3031
3032  def _B : sme2_mova_vec_to_tile_vg2_multi_base<0b00, v,
3033                                                !if(v, TileVectorOpV8,
3034                                                       TileVectorOpH8),
3035                                                uimm3s2range,  ZZ_b_mul_r,
3036                                                mnemonic> {
3037    bits<3> imm;
3038    let Inst{2-0} = imm;
3039  }
3040
3041  def _H : sme2_mova_vec_to_tile_vg2_multi_base<0b01, v,
3042                                                !if(v, TileVectorOpV16,
3043                                                       TileVectorOpH16),
3044                                                uimm2s2range, ZZ_h_mul_r,
3045                                                mnemonic> {
3046    bits<1> ZAd;
3047    bits<2> imm;
3048    let Inst{2}   = ZAd;
3049    let Inst{1-0} = imm;
3050  }
3051
3052  def _S : sme2_mova_vec_to_tile_vg2_multi_base<0b10, v,
3053                                                !if(v, TileVectorOpV32,
3054                                                       TileVectorOpH32),
3055                                                 uimm1s2range, ZZ_s_mul_r,
3056                                                 mnemonic> {
3057    bits<2> ZAd;
3058    bits<1> imm;
3059    let Inst{2-1} = ZAd;
3060    let Inst{0}   = imm;
3061  }
3062
3063  def _D : sme2_mova_vec_to_tile_vg2_multi_base<0b11, v,
3064                                                !if(v, TileVectorOpV64,
3065                                                       TileVectorOpH64),
3066                                                uimm0s2range, ZZ_d_mul_r,
3067                                                mnemonic> {
3068    bits<3> ZAd;
3069    let Inst{2-0} = ZAd;
3070   }
3071
3072  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B),
3073                                                !if(v, TileVectorOpV8,
3074                                                       TileVectorOpH8),
3075                                                MatrixIndexGPR32Op12_15,
3076                                                uimm3s2range,  ZZ_b_mul_r,
3077                                                "mov">;
3078  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H),
3079                                                !if(v, TileVectorOpV16,
3080                                                       TileVectorOpH16),
3081                                                MatrixIndexGPR32Op12_15,
3082                                                uimm2s2range,  ZZ_h_mul_r,
3083                                                "mov">;
3084  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S),
3085                                                !if(v, TileVectorOpV32,
3086                                                       TileVectorOpH32),
3087                                                MatrixIndexGPR32Op12_15,
3088                                                uimm1s2range,  ZZ_s_mul_r,
3089                                                "mov">;
3090  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D),
3091                                                !if(v, TileVectorOpV64,
3092                                                       TileVectorOpH64),
3093                                                MatrixIndexGPR32Op12_15,
3094                                                uimm0s2range,  ZZ_d_mul_r,
3095                                                "mov">;
3096
3097  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B),
3098                                                !if(v, TileVectorOpV8,
3099                                                       TileVectorOpH8),
3100                                                MatrixIndexGPR32Op12_15,
3101                                                uimm3s2range,  ZZ_b_mul_r,
3102                                                "mova">;
3103  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H),
3104                                                !if(v, TileVectorOpV16,
3105                                                       TileVectorOpH16),
3106                                                MatrixIndexGPR32Op12_15,
3107                                                uimm2s2range,  ZZ_h_mul_r,
3108                                                "mova">;
3109  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S),
3110                                                !if(v, TileVectorOpV32,
3111                                                       TileVectorOpH32),
3112                                                MatrixIndexGPR32Op12_15,
3113                                                uimm1s2range,  ZZ_s_mul_r,
3114                                                "mova">;
3115  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D),
3116                                                !if(v, TileVectorOpV64,
3117                                                       TileVectorOpH64),
3118                                                MatrixIndexGPR32Op12_15,
3119                                                uimm0s2range,  ZZ_d_mul_r,
3120                                                "mova">;
3121
3122  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B),
3123                                                !if(v, TileVectorOpV8,
3124                                                       TileVectorOpH8),
3125                                                MatrixIndexGPR32Op12_15,
3126                                                uimm3s2range,  ZZ_b_mul_r,
3127                                                "mova">;
3128  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H),
3129                                                !if(v, TileVectorOpV16,
3130                                                       TileVectorOpH16),
3131                                                MatrixIndexGPR32Op12_15,
3132                                                uimm2s2range,  ZZ_h_mul_r,
3133                                                "mova">;
3134  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S),
3135                                                !if(v, TileVectorOpV32,
3136                                                       TileVectorOpH32),
3137                                                MatrixIndexGPR32Op12_15,
3138                                                uimm1s2range,  ZZ_s_mul_r,
3139                                                "mova">;
3140  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D),
3141                                                !if(v, TileVectorOpV64,
3142                                                       TileVectorOpH64),
3143                                                MatrixIndexGPR32Op12_15,
3144                                                uimm0s2range,  ZZ_d_mul_r,
3145                                                "mova">;
3146}
3147
3148multiclass sme2_mova_vec_to_tile_vg2_multi<string mnemonic>{
3149 defm _H : sme2_mova_vec_to_tile_vg2_multi_base<0b0, mnemonic>;
3150 defm _V : sme2_mova_vec_to_tile_vg2_multi_base<0b1, mnemonic>;
3151}
3152
3153class sme2_mova_vec_to_tile_vg4_multi_base<bits<2> sz, bit v, bits<3> op,
3154                                           RegisterOperand tile_ty,
3155                                           Operand index_ty,
3156                                           RegisterOperand vector_ty,
3157                                           string mnemonic>
3158   : I<(outs tile_ty:$ZAd),
3159       (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm,
3160            vector_ty:$Zn),
3161       mnemonic,
3162       "\t$ZAd[$Rs, $imm], $Zn",
3163       "", []>, Sched<[]> {
3164  bits<2> Rs;
3165  bits<3> Zn;
3166  let Inst{31-24} = 0b11000000;
3167  let Inst{23-22} = sz;
3168  let Inst{21-16} = 0b000100;
3169  let Inst{15}    = v;
3170  let Inst{14-13} = Rs;
3171  let Inst{12-10} = 0b001;
3172  let Inst{9-7}   = Zn;
3173  let Inst{6-3}   = 0b0000;
3174  let Inst{2-0}   = op;
3175  let Constraints = "$ZAd = $_ZAd";
3176}
3177
3178// SME2 move vector to tile, four registers
3179multiclass sme2_mova_vec_to_tile_vg4_multi_base<bit v, string mnemonic> {
3180
3181  def _B : sme2_mova_vec_to_tile_vg4_multi_base<0b00, v, {0,?,?},
3182                                                !if(v, TileVectorOpV8,
3183                                                       TileVectorOpH8),
3184                                                uimm2s4range, ZZZZ_b_mul_r,
3185                                                mnemonic> {
3186    bits<2> imm;
3187    let Inst{1-0} = imm;
3188  }
3189
3190  def _H : sme2_mova_vec_to_tile_vg4_multi_base<0b01, v, {0,?,?},
3191                                                !if(v, TileVectorOpV16,
3192                                                       TileVectorOpH16),
3193                                                uimm1s4range, ZZZZ_h_mul_r,
3194                                                mnemonic> {
3195    bits<1> ZAd;
3196    bits<1> imm;
3197    let Inst{1}   = ZAd;
3198    let Inst{0}   = imm;
3199  }
3200
3201  def _S : sme2_mova_vec_to_tile_vg4_multi_base<0b10, v, {0,?,?},
3202                                                !if(v, TileVectorOpV32,
3203                                                       TileVectorOpH32),
3204                                                 uimm0s4range, ZZZZ_s_mul_r,
3205                                                 mnemonic> {
3206    bits<2> ZAd;
3207    let Inst{1-0} = ZAd;
3208  }
3209
3210  def _D : sme2_mova_vec_to_tile_vg4_multi_base<0b11, v, {?,?,?},
3211                                                !if(v, TileVectorOpV64,
3212                                                       TileVectorOpH64),
3213                                                uimm0s4range, ZZZZ_d_mul_r,
3214                                                mnemonic> {
3215    bits<3> ZAd;
3216    let Inst{2-0} = ZAd;
3217  }
3218
3219  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B),
3220                                                !if(v, TileVectorOpV8,
3221                                                       TileVectorOpH8),
3222                                                MatrixIndexGPR32Op12_15,
3223                                                uimm2s4range, ZZZZ_b_mul_r,
3224                                                "mov">;
3225  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H),
3226                                                !if(v, TileVectorOpV16,
3227                                                       TileVectorOpH16),
3228                                                MatrixIndexGPR32Op12_15,
3229                                                uimm1s4range, ZZZZ_h_mul_r,
3230                                                "mov">;
3231  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S),
3232                                                !if(v, TileVectorOpV32,
3233                                                       TileVectorOpH32),
3234                                                MatrixIndexGPR32Op12_15,
3235                                                uimm0s4range, ZZZZ_s_mul_r,
3236                                                "mov">;
3237  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D),
3238                                                !if(v, TileVectorOpV64,
3239                                                       TileVectorOpH64),
3240                                                MatrixIndexGPR32Op12_15,
3241                                                uimm0s4range, ZZZZ_d_mul_r,
3242                                                "mov">;
3243
3244  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B),
3245                                                !if(v, TileVectorOpV8,
3246                                                       TileVectorOpH8),
3247                                                MatrixIndexGPR32Op12_15,
3248                                                uimm2s4range, ZZZZ_b_mul_r,
3249                                                "mova">;
3250  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H),
3251                                                !if(v, TileVectorOpV16,
3252                                                       TileVectorOpH16),
3253                                                MatrixIndexGPR32Op12_15,
3254                                                uimm1s4range, ZZZZ_h_mul_r,
3255                                                "mova">;
3256  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S),
3257                                                !if(v, TileVectorOpV32,
3258                                                       TileVectorOpH32),
3259                                                MatrixIndexGPR32Op12_15,
3260                                                uimm0s4range, ZZZZ_s_mul_r,
3261                                                "mova">;
3262  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D),
3263                                                !if(v, TileVectorOpV64,
3264                                                       TileVectorOpH64),
3265                                                MatrixIndexGPR32Op12_15,
3266                                                uimm0s4range, ZZZZ_d_mul_r,
3267                                                "mova">;
3268
3269}
3270
3271multiclass sme2_mova_vec_to_tile_vg4_multi<string mnemonic>{
3272 defm _H : sme2_mova_vec_to_tile_vg4_multi_base<0b0, mnemonic>;
3273 defm _V : sme2_mova_vec_to_tile_vg4_multi_base<0b1, mnemonic>;
3274}
3275
3276// SME Move into Array
3277class sme2_mova_vec_to_array_vg24_multi< bits<5> op, RegisterOperand array_ty,
3278                                        RegisterOperand vector_ty,
3279                                        string mnemonic,
3280                                        string vg_acronym="">
3281   : I<(outs array_ty:$ZAd),
3282       (ins array_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm,
3283            vector_ty:$Zn),
3284       mnemonic, "\t$ZAd[$Rs, $imm, " # vg_acronym # "], $Zn",
3285       "", []>, Sched<[]> {
3286  bits<2> Rs;
3287  bits<3> imm;
3288  let Inst{31-15} = 0b11000000000001000;
3289  let Inst{14-13} = Rs;
3290  let Inst{12-11} = 0b01;
3291  let Inst{10-6}  = op;
3292  let Inst{5-3}   = 0b000;
3293  let Inst{2-0}   = imm;
3294
3295  let Constraints = "$ZAd = $_ZAd";
3296}
3297
3298// MOVA (vector to array, two registers)
3299multiclass sme2_mova_vec_to_array_vg2_multi<string mnemonic> {
3300  def NAME : sme2_mova_vec_to_array_vg24_multi<{0,?,?,?,?}, MatrixOp64,
3301                                               ZZ_d_mul_r, mnemonic, "vgx2">{
3302   bits<4> Zn;
3303   let Inst{9-6} = Zn;
3304  }
3305
3306  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3307                                                MatrixOp8,
3308                                                MatrixIndexGPR32Op8_11,
3309                                                sme_elm_idx0_7, ZZ_b_mul_r,
3310                                                "mova">;
3311  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3312                                                MatrixOp16,
3313                                                MatrixIndexGPR32Op8_11,
3314                                                sme_elm_idx0_7, ZZ_h_mul_r,
3315                                                "mova">;
3316  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3317                                                MatrixOp32,
3318                                                MatrixIndexGPR32Op8_11,
3319                                                sme_elm_idx0_7, ZZ_s_mul_r,
3320                                                "mova">;
3321  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3322                                                MatrixOp64,
3323                                                MatrixIndexGPR32Op8_11,
3324                                                sme_elm_idx0_7, ZZ_d_mul_r,
3325                                                "mova">;
3326
3327  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3328                                                MatrixOp8,
3329                                                MatrixIndexGPR32Op8_11,
3330                                                sme_elm_idx0_7, ZZ_b_mul_r,
3331                                                "mova", "vgx2">;
3332  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3333                                                MatrixOp16,
3334                                                MatrixIndexGPR32Op8_11,
3335                                                sme_elm_idx0_7, ZZ_h_mul_r,
3336                                                "mova", "vgx2">;
3337  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3338                                                MatrixOp32,
3339                                                MatrixIndexGPR32Op8_11,
3340                                                sme_elm_idx0_7, ZZ_s_mul_r,
3341                                                "mova", "vgx2">;
3342
3343  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3344                                                MatrixOp8,
3345                                                MatrixIndexGPR32Op8_11,
3346                                                sme_elm_idx0_7, ZZ_b_mul_r,
3347                                                "mov">;
3348  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3349                                                MatrixOp16,
3350                                                MatrixIndexGPR32Op8_11,
3351                                                sme_elm_idx0_7, ZZ_h_mul_r,
3352                                                "mov">;
3353  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3354                                                MatrixOp32,
3355                                                MatrixIndexGPR32Op8_11,
3356                                                sme_elm_idx0_7, ZZ_s_mul_r,
3357                                                "mov">;
3358  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3359                                                MatrixOp64,
3360                                                MatrixIndexGPR32Op8_11,
3361                                                sme_elm_idx0_7, ZZ_d_mul_r,
3362                                                "mov">;
3363
3364  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3365                                                MatrixOp8,
3366                                                MatrixIndexGPR32Op8_11,
3367                                                sme_elm_idx0_7, ZZ_b_mul_r,
3368                                                "mov", "vgx2">;
3369  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3370                                                MatrixOp16,
3371                                                MatrixIndexGPR32Op8_11,
3372                                                sme_elm_idx0_7, ZZ_h_mul_r,
3373                                                "mov", "vgx2">;
3374  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3375                                                MatrixOp32,
3376                                                MatrixIndexGPR32Op8_11,
3377                                                sme_elm_idx0_7, ZZ_s_mul_r,
3378                                                "mov", "vgx2">;
3379  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME),
3380                                                MatrixOp64,
3381                                                MatrixIndexGPR32Op8_11,
3382                                                sme_elm_idx0_7, ZZ_d_mul_r,
3383                                                "mov", "vgx2">;
3384}
3385
3386// MOVA (vector to array, four registers)
3387multiclass sme2_mova_vec_to_array_vg4_multi<string mnemonic> {
3388  def NAME : sme2_mova_vec_to_array_vg24_multi<{1,?,?,?,0}, MatrixOp64,
3389                                               ZZZZ_d_mul_r, mnemonic, "vgx4"> {
3390    bits<3> Zn;
3391    let Inst{9-7} = Zn;
3392  }
3393
3394  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3395                                                MatrixOp8,
3396                                                MatrixIndexGPR32Op8_11,
3397                                                sme_elm_idx0_7, ZZZZ_b_mul_r,
3398                                                "mova">;
3399  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3400                                                MatrixOp16,
3401                                                MatrixIndexGPR32Op8_11,
3402                                                sme_elm_idx0_7, ZZZZ_h_mul_r,
3403                                                "mova">;
3404  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3405                                                MatrixOp32,
3406                                                MatrixIndexGPR32Op8_11,
3407                                                sme_elm_idx0_7, ZZZZ_s_mul_r,
3408                                                "mova">;
3409  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3410                                                MatrixOp64,
3411                                                MatrixIndexGPR32Op8_11,
3412                                                sme_elm_idx0_7, ZZZZ_d_mul_r,
3413                                                "mova">;
3414
3415  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3416                                                MatrixOp8,
3417                                                MatrixIndexGPR32Op8_11,
3418                                                sme_elm_idx0_7, ZZZZ_b_mul_r,
3419                                                "mova", "vgx4">;
3420  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3421                                                MatrixOp16,
3422                                                MatrixIndexGPR32Op8_11,
3423                                                sme_elm_idx0_7, ZZZZ_h_mul_r,
3424                                                "mova", "vgx4">;
3425  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3426                                                MatrixOp32,
3427                                                MatrixIndexGPR32Op8_11,
3428                                                sme_elm_idx0_7, ZZZZ_s_mul_r,
3429                                                "mova", "vgx4">;
3430
3431  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3432                                                MatrixOp8,
3433                                                MatrixIndexGPR32Op8_11,
3434                                                sme_elm_idx0_7, ZZZZ_b_mul_r,
3435                                                "mov">;
3436  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3437                                                MatrixOp16,
3438                                                MatrixIndexGPR32Op8_11,
3439                                                sme_elm_idx0_7, ZZZZ_h_mul_r,
3440                                                "mov">;
3441  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3442                                                MatrixOp32,
3443                                                MatrixIndexGPR32Op8_11,
3444                                                sme_elm_idx0_7, ZZZZ_s_mul_r,
3445                                                "mov">;
3446  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3447                                                MatrixOp64,
3448                                                MatrixIndexGPR32Op8_11,
3449                                                sme_elm_idx0_7, ZZZZ_d_mul_r,
3450                                                "mov">;
3451
3452  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3453                                                MatrixOp8,
3454                                                MatrixIndexGPR32Op8_11,
3455                                                sme_elm_idx0_7, ZZZZ_b_mul_r,
3456                                                "mov", "vgx4">;
3457  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3458                                                MatrixOp16,
3459                                                MatrixIndexGPR32Op8_11,
3460                                                sme_elm_idx0_7, ZZZZ_h_mul_r,
3461                                                "mov", "vgx4">;
3462  defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),
3463                                                MatrixOp32,
3464                                                MatrixIndexGPR32Op8_11,
3465                                                sme_elm_idx0_7, ZZZZ_s_mul_r,
3466                                                "mov", "vgx4">;
3467  defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME),
3468                                                MatrixOp64,
3469                                                MatrixIndexGPR32Op8_11,
3470                                                sme_elm_idx0_7, ZZZZ_d_mul_r,
3471                                                "mov", "vgx4">;
3472
3473}
3474
3475class sme2_mova_tile_to_vec_vg2_multi_base<bits<2> sz, bit v, bits<3> op,
3476                                           RegisterOperand vector_ty,
3477                                           RegisterOperand tile_ty,
3478                                           Operand index_ty,
3479                                           string mnemonic>
3480   : I<!if(op{1}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)),
3481       (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm),
3482       mnemonic,
3483       "\t$Zd, $ZAn[$Rs, $imm]",
3484       "", []>, Sched<[]> {
3485  bits<4> Zd;
3486  bits<2> Rs;
3487  let Inst{31-24} = 0b11000000;
3488  let Inst{23-22} = sz;
3489  let Inst{21-16} = 0b000110;
3490  let Inst{15}    = v;
3491  let Inst{14-13} = Rs;
3492  let Inst{12-11} = 0b00;
3493  let Inst{10-8}  = op;
3494  let Inst{4-1}   = Zd;
3495  let Inst{0}     = 0b0;
3496
3497  let Constraints = !if(op{1}, "$ZAn = $_ZAn", "");
3498}
3499
3500multiclass sme2_mova_tile_or_array_to_vec_aliases<int op, Instruction inst,
3501                                                  RegisterOperand vector_ty,
3502                                                  RegisterOperand tile_or_array_ty,
3503                                                  RegisterOperand rv_ty,
3504                                                  Operand index_ty,
3505                                                  string mnemonic,
3506                                                  string vg_acronym=""> {
3507def : InstAlias<mnemonic # "\t$Zd, $ZAn[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]",
3508                  (inst vector_ty:$Zd, tile_or_array_ty:$ZAn, rv_ty:$Rs, index_ty:$imm), op>;
3509
3510}
3511
3512multiclass sme2_mova_tile_to_vec_vg2_multi_inst<bit v, bits<3> opc, string mnemonic> {
3513
3514  def _B : sme2_mova_tile_to_vec_vg2_multi_base<0b00, v, opc, ZZ_b_mul_r,
3515                                                !if(v, TileVectorOpV8,
3516                                                       TileVectorOpH8),
3517                                                 uimm3s2range, mnemonic> {
3518    bits<3> imm;
3519    let Inst{7-5} = imm;
3520  }
3521
3522  def _H : sme2_mova_tile_to_vec_vg2_multi_base<0b01, v, opc, ZZ_h_mul_r,
3523                                                !if(v, TileVectorOpV16,
3524                                                       TileVectorOpH16),
3525                                                 uimm2s2range, mnemonic> {
3526    bits<1> ZAn;
3527    bits<2> imm;
3528    let Inst{7}   = ZAn;
3529    let Inst{6-5} = imm;
3530  }
3531
3532  def _S : sme2_mova_tile_to_vec_vg2_multi_base<0b10, v, opc, ZZ_s_mul_r,
3533                                                !if(v, TileVectorOpV32,
3534                                                       TileVectorOpH32),
3535                                                 uimm1s2range, mnemonic> {
3536    bits<2> ZAn;
3537    bits<1> imm;
3538    let Inst{7-6} = ZAn;
3539    let Inst{5}   = imm;
3540  }
3541
3542  def _D : sme2_mova_tile_to_vec_vg2_multi_base<0b11, v, opc, ZZ_d_mul_r,
3543                                                !if(v, TileVectorOpV64,
3544                                                       TileVectorOpH64),
3545                                                uimm0s2range, mnemonic> {
3546    bits<3> ZAn;
3547    let Inst{7-5} = ZAn;
3548  }
3549
3550  if !eq(mnemonic, "mova") then {
3551  defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _B),
3552                                                ZZ_b_mul_r,
3553                                               !if(v, TileVectorOpV8,
3554                                                      TileVectorOpH8),
3555                                                MatrixIndexGPR32Op12_15,
3556                                                uimm3s2range, "mov">;
3557  defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _H),
3558                                                ZZ_h_mul_r,
3559                                                !if(v, TileVectorOpV16,
3560                                                       TileVectorOpH16),
3561                                                MatrixIndexGPR32Op12_15,
3562                                                uimm2s2range, "mov">;
3563  defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S),
3564                                                ZZ_s_mul_r,
3565                                                !if(v, TileVectorOpV32,
3566                                                       TileVectorOpH32),
3567                                                MatrixIndexGPR32Op12_15,
3568                                                uimm1s2range, "mov">;
3569  defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D),
3570                                                ZZ_d_mul_r,
3571                                                !if(v, TileVectorOpV64,
3572                                                       TileVectorOpH64),
3573                                                MatrixIndexGPR32Op12_15,
3574                                                uimm0s2range, "mov">;
3575  }
3576
3577  defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _B),
3578                                                ZZ_b_mul_r,
3579                                               !if(v, TileVectorOpV8,
3580                                                      TileVectorOpH8),
3581                                                MatrixIndexGPR32Op12_15,
3582                                                uimm3s2range, mnemonic>;
3583  defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _H),
3584                                                ZZ_h_mul_r,
3585                                                !if(v, TileVectorOpV16,
3586                                                       TileVectorOpH16),
3587                                                MatrixIndexGPR32Op12_15,
3588                                                uimm2s2range, mnemonic>;
3589  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S),
3590                                                ZZ_s_mul_r,
3591                                                !if(v, TileVectorOpV32,
3592                                                       TileVectorOpH32),
3593                                                MatrixIndexGPR32Op12_15,
3594                                                uimm1s2range, mnemonic>;
3595  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D),
3596                                                ZZ_d_mul_r,
3597                                                !if(v, TileVectorOpV64,
3598                                                       TileVectorOpH64),
3599                                                MatrixIndexGPR32Op12_15,
3600                                                uimm0s2range, mnemonic>;
3601
3602}
3603
3604// SME2 move tile to vector, two registers
3605multiclass sme2_mova_tile_to_vec_vg2_multi<string mnemonic>{
3606 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b000, mnemonic>;
3607 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b000, mnemonic>;
3608}
3609
3610// SME2p1 move tile to vector and zero tile, two registers
3611multiclass sme2p1_movaz_tile_to_vec_vg2<string mnemonic>{
3612 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b010, mnemonic>;
3613 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b010, mnemonic>;
3614}
3615
3616class sme2_mova_tile_to_vec_vg4_multi_base<bits<2> sz, bit v, bits<6> op,
3617                                           RegisterOperand vector_ty,
3618                                           RegisterOperand tile_ty,
3619                                           Operand index_ty,
3620                                           string mnemonic>
3621   : I<!if(op{4}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)),
3622       (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm),
3623       mnemonic,
3624       "\t$Zd, $ZAn[$Rs, $imm]",
3625       "", []>, Sched<[]> {
3626  bits<3> Zd;
3627  bits<2> Rs;
3628  let Inst{31-24} = 0b11000000;
3629  let Inst{23-22} = sz;
3630  let Inst{21-16} = 0b000110;
3631  let Inst{15}    = v;
3632  let Inst{14-13} = Rs;
3633  let Inst{12-11} = 0b00;
3634  let Inst{10-5}  = op{5-0};
3635  let Inst{4-2}   = Zd;
3636  let Inst{1-0}   = 0b00;
3637
3638  let Constraints = !if(op{4}, "$ZAn = $_ZAn", "");
3639}
3640
3641multiclass sme2_mova_tile_to_vec_vg4_multi_base<bit v, bits<3> opc, string mnemonic> {
3642
3643  def _B : sme2_mova_tile_to_vec_vg4_multi_base<0b00, v, {opc,0,?,?},
3644                                                ZZZZ_b_mul_r,
3645                                                !if(v, TileVectorOpV8,
3646                                                       TileVectorOpH8),
3647                                                uimm2s4range, mnemonic> {
3648    bits<2> imm;
3649    let Inst{6-5} = imm;
3650  }
3651
3652  def _H : sme2_mova_tile_to_vec_vg4_multi_base<0b01, v, {opc,0,?,?},
3653                                                ZZZZ_h_mul_r,
3654                                                !if(v, TileVectorOpV16,
3655                                                       TileVectorOpH16),
3656                                                uimm1s4range, mnemonic> {
3657    bits<1> ZAn;
3658    bits<1> imm;
3659    let Inst{6}   = ZAn;
3660    let Inst{5}   = imm;
3661  }
3662
3663  def _S : sme2_mova_tile_to_vec_vg4_multi_base<0b10, v, {opc,0,?,?},
3664                                                ZZZZ_s_mul_r,
3665                                                !if(v, TileVectorOpV32,
3666                                                       TileVectorOpH32),
3667                                                 uimm0s4range, mnemonic> {
3668    bits<2> ZAn;
3669    let Inst{6-5} = ZAn;
3670  }
3671
3672  def _D : sme2_mova_tile_to_vec_vg4_multi_base<0b11, v, {opc,?,?,?},
3673                                                ZZZZ_d_mul_r,
3674                                                !if(v, TileVectorOpV64,
3675                                                       TileVectorOpH64),
3676                                                uimm0s4range, mnemonic> {
3677    bits<3> ZAn;
3678    let Inst{7-5} = ZAn;
3679  }
3680
3681  if !eq(mnemonic, "mova") then {
3682  defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _B),
3683                                                ZZZZ_b_mul_r,
3684                                                !if(v, TileVectorOpV8,
3685                                                      TileVectorOpH8),
3686                                                MatrixIndexGPR32Op12_15,
3687                                                uimm2s4range, "mov">;
3688  defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _H),
3689                                                ZZZZ_h_mul_r,
3690                                                !if(v, TileVectorOpV16,
3691                                                       TileVectorOpH16),
3692                                                MatrixIndexGPR32Op12_15,
3693                                                uimm1s4range, "mov">;
3694  defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S),
3695                                                ZZZZ_s_mul_r,
3696                                                !if(v, TileVectorOpV32,
3697                                                      TileVectorOpH32),
3698                                                MatrixIndexGPR32Op12_15,
3699                                                uimm0s4range, "mov">;
3700  defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D),
3701                                                ZZZZ_d_mul_r,
3702                                                !if(v, TileVectorOpV64,
3703                                                       TileVectorOpH64),
3704                                                MatrixIndexGPR32Op12_15,
3705                                                uimm0s4range, "mov">;
3706  }
3707
3708  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _B),
3709                                                ZZZZ_b_mul_r,
3710                                                !if(v, TileVectorOpV8,
3711                                                       TileVectorOpH8),
3712                                                MatrixIndexGPR32Op12_15,
3713                                                uimm2s4range, mnemonic>;
3714  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _H),
3715                                                ZZZZ_h_mul_r,
3716                                                !if(v, TileVectorOpV16,
3717                                                       TileVectorOpH16),
3718                                                MatrixIndexGPR32Op12_15,
3719                                                uimm1s4range, mnemonic>;
3720  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S),
3721                                                ZZZZ_s_mul_r,
3722                                                !if(v, TileVectorOpV32,
3723                                                      TileVectorOpH32),
3724                                                MatrixIndexGPR32Op12_15,
3725                                                uimm0s4range, mnemonic>;
3726  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D),
3727                                                ZZZZ_d_mul_r,
3728                                                !if(v, TileVectorOpV64,
3729                                                       TileVectorOpH64),
3730                                                MatrixIndexGPR32Op12_15,
3731                                                uimm0s4range, mnemonic>;
3732
3733}
3734
3735// SME2 move tile to vector, four registers
3736multiclass sme2_mova_tile_to_vec_vg4_multi<string mnemonic>{
3737 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b100, mnemonic>;
3738 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b100, mnemonic>;
3739}
3740
3741// SME2p1 move tile to vector and zero tile, four registers
3742multiclass sme2p1_movaz_tile_to_vec_vg4<string mnemonic>{
3743 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b110, mnemonic>;
3744 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b110, mnemonic>;
3745}
3746
3747
3748class sme2_mova_array_to_vec_vg24_multi<bits<4>op, RegisterOperand vector_ty,
3749                                        RegisterOperand array_ty,
3750                                        string mnemonic, string vg_acronym>
3751   : I<!if(op{2}, (outs vector_ty:$Zd, array_ty:$_ZAn), (outs vector_ty:$Zd)),
3752       (ins array_ty:$ZAn, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm),
3753       mnemonic,
3754       "\t$Zd, $ZAn[$Rs, $imm, " # vg_acronym # "]",
3755       "", []>, Sched<[]> {
3756  bits<2> Rs;
3757  bits<3> imm;
3758  let Inst{31-15} = 0b11000000000001100;
3759  let Inst{14-13} = Rs;
3760  let Inst{12-11} = 0b01;
3761  let Inst{10-8}  = op{3-1};
3762  let Inst{7-5}   = imm;
3763  let Inst{1}     = op{0};
3764  let Inst{0}     = 0b0;
3765  let Constraints = !if(op{2}, "$ZAn = $_ZAn", "");
3766}
3767
3768// move array to vector, two registers.
3769multiclass sme2_mova_array_to_vec_vg2_multi<bits<3> opc, string mnemonic> {
3770  def NAME : sme2_mova_array_to_vec_vg24_multi<{opc,?}, ZZ_d_mul_r, MatrixOp64,
3771                                               mnemonic, "vgx2"> {
3772    bits<4> Zd;
3773    let Inst{4-1} = Zd;
3774  }
3775
3776  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3777                                                ZZ_b_mul_r, MatrixOp8,
3778                                                MatrixIndexGPR32Op8_11,
3779                                                sme_elm_idx0_7, mnemonic>;
3780  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3781                                                ZZ_h_mul_r, MatrixOp16,
3782                                                MatrixIndexGPR32Op8_11,
3783                                                sme_elm_idx0_7, mnemonic>;
3784  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3785                                                ZZ_s_mul_r, MatrixOp32,
3786                                                MatrixIndexGPR32Op8_11,
3787                                                sme_elm_idx0_7, mnemonic>;
3788  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3789                                                ZZ_d_mul_r,  MatrixOp64,
3790                                                MatrixIndexGPR32Op8_11,
3791                                                sme_elm_idx0_7, mnemonic>;
3792
3793  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3794                                                ZZ_b_mul_r, MatrixOp8,
3795                                                MatrixIndexGPR32Op8_11,
3796                                                sme_elm_idx0_7, mnemonic, "vgx2">;
3797  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3798                                                ZZ_h_mul_r, MatrixOp16,
3799                                                MatrixIndexGPR32Op8_11,
3800                                                sme_elm_idx0_7, mnemonic, "vgx2">;
3801  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3802                                                ZZ_s_mul_r, MatrixOp32,
3803                                                MatrixIndexGPR32Op8_11,
3804                                                sme_elm_idx0_7, mnemonic, "vgx2">;
3805
3806  if !eq(mnemonic, "mova") then {
3807  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3808                                                ZZ_b_mul_r, MatrixOp8,
3809                                                MatrixIndexGPR32Op8_11,
3810                                                sme_elm_idx0_7, "mov">;
3811  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3812                                                ZZ_h_mul_r, MatrixOp16,
3813                                                MatrixIndexGPR32Op8_11,
3814                                                sme_elm_idx0_7, "mov">;
3815  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3816                                                ZZ_s_mul_r, MatrixOp32,
3817                                                MatrixIndexGPR32Op8_11,
3818                                                sme_elm_idx0_7, "mov">;
3819  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3820                                                ZZ_d_mul_r,  MatrixOp64,
3821                                                MatrixIndexGPR32Op8_11,
3822                                                sme_elm_idx0_7, "mov">;
3823
3824  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3825                                                ZZ_b_mul_r, MatrixOp8,
3826                                                MatrixIndexGPR32Op8_11,
3827                                                sme_elm_idx0_7, "mov", "vgx2">;
3828  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3829                                                ZZ_h_mul_r, MatrixOp16,
3830                                                MatrixIndexGPR32Op8_11,
3831                                                sme_elm_idx0_7, "mov", "vgx2">;
3832  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3833                                                ZZ_s_mul_r, MatrixOp32,
3834                                                MatrixIndexGPR32Op8_11,
3835                                                sme_elm_idx0_7, "mov", "vgx2">;
3836  defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME),
3837                                                ZZ_d_mul_r,  MatrixOp64,
3838                                                MatrixIndexGPR32Op8_11,
3839                                                sme_elm_idx0_7, "mov", "vgx2">;
3840  }
3841}
3842
3843// move array to vector, four registers
3844multiclass sme2_mova_array_to_vec_vg4_multi<bits<4> opc, string mnemonic> {
3845  def NAME : sme2_mova_array_to_vec_vg24_multi<opc, ZZZZ_d_mul_r, MatrixOp64,
3846                                               mnemonic, "vgx4"> {
3847    bits<3> Zd;
3848    let Inst{4-2} = Zd;
3849  }
3850
3851  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3852                                                ZZZZ_b_mul_r, MatrixOp8,
3853                                                MatrixIndexGPR32Op8_11,
3854                                                sme_elm_idx0_7, mnemonic>;
3855  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3856                                                ZZZZ_h_mul_r, MatrixOp16,
3857                                                MatrixIndexGPR32Op8_11,
3858                                                sme_elm_idx0_7, mnemonic>;
3859  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3860                                                ZZZZ_s_mul_r, MatrixOp32,
3861                                                MatrixIndexGPR32Op8_11,
3862                                                sme_elm_idx0_7, mnemonic>;
3863  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3864                                                ZZZZ_d_mul_r, MatrixOp64,
3865                                                MatrixIndexGPR32Op8_11,
3866                                                sme_elm_idx0_7, mnemonic>;
3867
3868  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3869                                                ZZZZ_b_mul_r, MatrixOp8,
3870                                                MatrixIndexGPR32Op8_11,
3871                                                sme_elm_idx0_7, mnemonic, "vgx4">;
3872  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3873                                                ZZZZ_h_mul_r, MatrixOp16,
3874                                                MatrixIndexGPR32Op8_11,
3875                                                sme_elm_idx0_7, mnemonic, "vgx4">;
3876  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3877                                                ZZZZ_s_mul_r, MatrixOp32,
3878                                                MatrixIndexGPR32Op8_11,
3879                                                sme_elm_idx0_7, mnemonic, "vgx4">;
3880
3881  if !eq(mnemonic, "mova") then {
3882  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3883                                                ZZZZ_b_mul_r, MatrixOp8,
3884                                                MatrixIndexGPR32Op8_11,
3885                                                sme_elm_idx0_7, "mov">;
3886  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3887                                                ZZZZ_h_mul_r, MatrixOp16,
3888                                                MatrixIndexGPR32Op8_11,
3889                                                sme_elm_idx0_7, "mov">;
3890  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3891                                                ZZZZ_s_mul_r, MatrixOp32,
3892                                                MatrixIndexGPR32Op8_11,
3893                                                sme_elm_idx0_7, "mov">;
3894  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3895                                                ZZZZ_d_mul_r, MatrixOp64,
3896                                                MatrixIndexGPR32Op8_11,
3897                                                sme_elm_idx0_7, "mov">;
3898
3899  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3900                                                ZZZZ_b_mul_r, MatrixOp8,
3901                                                MatrixIndexGPR32Op8_11,
3902                                                sme_elm_idx0_7, "mov", "vgx4">;
3903  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3904                                                ZZZZ_h_mul_r, MatrixOp16,
3905                                                MatrixIndexGPR32Op8_11,
3906                                                sme_elm_idx0_7, "mov", "vgx4">;
3907  defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),
3908                                                ZZZZ_s_mul_r, MatrixOp32,
3909                                                MatrixIndexGPR32Op8_11,
3910                                                sme_elm_idx0_7, "mov", "vgx4">;
3911  defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME),
3912                                                ZZZZ_d_mul_r, MatrixOp64,
3913                                                MatrixIndexGPR32Op8_11,
3914                                                sme_elm_idx0_7, "mov", "vgx4">;
3915  }
3916}
3917
3918//===----------------------------------------------------------------------===//
3919// SME2 multi-vec saturating shift right narrow
3920class sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u>
3921    : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn, vecshiftR16:$imm4),
3922        mnemonic, "\t$Zd, $Zn, $imm4",
3923        "", []>, Sched<[]> {
3924  bits<4> imm4;
3925  bits<4> Zn;
3926  bits<5> Zd;
3927  let Inst{31-21} = 0b11000001111;
3928  let Inst{20}    = op;
3929  let Inst{19-16} = imm4;
3930  let Inst{15-10} = 0b110101;
3931  let Inst{9-6}   = Zn;
3932  let Inst{5}     = u;
3933  let Inst{4-0}   = Zd;
3934}
3935
3936multiclass sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u> {
3937  def _H : sme2_sat_shift_vector_vg2<mnemonic, op, u>;
3938}
3939
3940class sme2_sat_shift_vector_vg4<bits<2> sz, bits<3> op, ZPRRegOp zpr_ty,
3941                                RegisterOperand vector_ty, Operand imm_ty,
3942                                string mnemonic>
3943    : I<(outs zpr_ty:$Zd), (ins vector_ty:$Zn, imm_ty:$imm),
3944        mnemonic, "\t$Zd, $Zn, $imm",
3945        "", []>, Sched<[]> {
3946  bits<3> Zn;
3947  bits<5> Zd;
3948  let Inst{31-24} = 0b11000001;
3949  let Inst{23-22} = sz;
3950  let Inst{21}    = 0b1;
3951  //  Inst{20-16} = imm5;
3952  let Inst{15-11} = 0b11011;
3953  let Inst{10}    = op{2};
3954  let Inst{9-7}   = Zn;
3955  let Inst{6-5}   = op{1-0};
3956  let Inst{4-0}   = Zd;
3957}
3958
3959multiclass sme2_sat_shift_vector_vg4<string mnemonic, bits<3> op> {
3960  def _B : sme2_sat_shift_vector_vg4<{0,1}, op, ZPR8, ZZZZ_s_mul_r, vecshiftR32,
3961                                     mnemonic>{
3962    bits<5> imm;
3963    let Inst{20-16} = imm;
3964  }
3965  def _H : sme2_sat_shift_vector_vg4<{1,?}, op, ZPR16, ZZZZ_d_mul_r, vecshiftR64,
3966                                      mnemonic> {
3967    bits<6> imm;
3968    let Inst{22}    = imm{5};
3969    let Inst{20-16} = imm{4-0};
3970  }
3971}
3972
3973//===----------------------------------------------------------------------===//
3974// SME2 Multi-vector - SVE Select
3975class sme2_sel_vector_vg24<bits<2> sz, bits<4> op, RegisterOperand vector_ty,
3976                           string mnemonic>
3977    : I<(outs vector_ty:$Zd),
3978        (ins PNRAny_p8to15:$PNg, vector_ty:$Zn, vector_ty:$Zm),
3979        mnemonic, "\t$Zd, $PNg, $Zn, $Zm",
3980        "", []>, Sched<[]> {
3981  bits<3> PNg;
3982  let Inst{31-24} = 0b11000001;
3983  let Inst{23-22} = sz;
3984  let Inst{21}    = 0b1;
3985  let Inst{17-16} = op{3-2};
3986  let Inst{15-13} = 0b100;
3987  let Inst{12-10} = PNg;
3988  let Inst{6}     = op{1};
3989  let Inst{5}     = 0b0;
3990  let Inst{1}     = op{0};
3991  let Inst{0}     = 0b0;
3992}
3993
3994class sme2_sel_vector_vg2<bits<2> sz, RegisterOperand vector_ty,
3995                          string mnemonic>
3996     : sme2_sel_vector_vg24<sz, {?,0,?,?}, vector_ty, mnemonic> {
3997  bits<4> Zm;
3998  bits<4> Zn;
3999  bits<4> Zd;
4000  let Inst{20-17} = Zm;
4001  let Inst{9-6}   = Zn;
4002  let Inst{4-1}   = Zd;
4003}
4004
4005multiclass sme2_sel_vector_vg2<string mnemonic>{
4006  def _B : sme2_sel_vector_vg2<0b00, ZZ_b_mul_r, mnemonic>;
4007  def _H : sme2_sel_vector_vg2<0b01, ZZ_h_mul_r, mnemonic>;
4008  def _S : sme2_sel_vector_vg2<0b10, ZZ_s_mul_r, mnemonic>;
4009  def _D : sme2_sel_vector_vg2<0b11, ZZ_d_mul_r, mnemonic>;
4010}
4011class sme2_sel_vector_vg4<bits<2> sz, RegisterOperand vector_ty,
4012                          string mnemonic>
4013     : sme2_sel_vector_vg24<sz, 0b0100, vector_ty, mnemonic> {
4014  bits<3> Zm;
4015  bits<3> Zn;
4016  bits<3> Zd;
4017  let Inst{20-18} = Zm;
4018  let Inst{9-7}   = Zn;
4019  let Inst{4-2}   = Zd;
4020}
4021multiclass sme2_sel_vector_vg4<string mnemonic> {
4022  def _B : sme2_sel_vector_vg4<0b00, ZZZZ_b_mul_r, mnemonic>;
4023  def _H : sme2_sel_vector_vg4<0b01, ZZZZ_h_mul_r, mnemonic>;
4024  def _S : sme2_sel_vector_vg4<0b10, ZZZZ_s_mul_r, mnemonic>;
4025  def _D : sme2_sel_vector_vg4<0b11, ZZZZ_d_mul_r, mnemonic>;
4026}
4027
4028//===----------------------------------------------------------------------===//
4029// Non contiguous Load and Store
4030
4031class sme2_ld_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n,
4032                                             RegisterOperand multi_vector_ty,
4033                                             RegisterOperand gpr_ty,
4034                                             string mnemonic>
4035   : I<(outs multi_vector_ty:$Zt),
4036       (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),
4037       mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]",
4038       "", []>, Sched<[]> {
4039   bits<5> Rm;
4040   bits<3> PNg;
4041   bits<5> Rn;
4042   bits<4> Zt;
4043   let Inst{31-21} = 0b10100001000;
4044   let Inst{20-16} = Rm;
4045   let Inst{15}    = 0b0;
4046   let Inst{14-13} = msz;
4047   let Inst{12-10} = PNg;
4048   let Inst{9-5}   = Rn;
4049   let Inst{4}     = Zt{3};
4050   let Inst{3}     = n;
4051   let Inst{2-0}   = Zt{2-0};
4052
4053   let mayLoad = 1;
4054}
4055
4056class sme2_ld_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n,
4057                                             RegisterOperand multi_vector_ty,
4058                                             RegisterOperand gpr_ty,
4059                                             string mnemonic>
4060   : I<(outs multi_vector_ty:$Zt),
4061       (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),
4062       mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]",
4063       "", []>, Sched<[]> {
4064   bits<5> Rm;
4065   bits<3> PNg;
4066   bits<5> Rn;
4067   bits<3> Zt;
4068   let Inst{31-21} = 0b10100001000;
4069   let Inst{20-16} = Rm;
4070   let Inst{15}    = 0b1;
4071   let Inst{14-13} = msz;
4072   let Inst{12-10} = PNg;
4073   let Inst{9-5}   = Rn;
4074   let Inst{4}     = Zt{2};
4075   let Inst{3}     = n;
4076   let Inst{2}     = 0b0;
4077   let Inst{1-0}   = Zt{1-0};
4078
4079   let mayLoad = 1;
4080}
4081
4082class sme2_ld_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op,
4083                                                 RegisterOperand multi_vector_ty,
4084                                                 Operand index_ty,
4085                                                 string mnemonic>
4086    : I<(outs multi_vector_ty:$Zt),
4087        (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4),
4088        mnemonic,  "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]",
4089        "", []>, Sched<[]> {
4090   bits<4> imm4;
4091   bits<3> PNg;
4092   bits<5> Rn;
4093   let Inst{31-20} = 0b101000010100;
4094   let Inst{19-16} = imm4;
4095   let Inst{15}    = op{1};
4096   let Inst{14-13} = msz;
4097   let Inst{12-10} = PNg;
4098   let Inst{9-5}   = Rn;
4099   let Inst{3}     = n;
4100   let Inst{2}     = op{0};
4101
4102   let mayLoad = 1;
4103}
4104
4105multiclass sme2_ld_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n,
4106                                                     RegisterOperand multi_vector_ty,
4107                                                     Operand index_ty,
4108                                                     string mnemonic>{
4109  def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, {0,?},
4110                                                        multi_vector_ty,
4111                                                        index_ty, mnemonic> {
4112    bits<4> Zt;
4113    let Inst{4} = Zt{3};
4114    let Inst{2-0} = Zt{2-0};
4115  }
4116
4117   def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]",
4118                  (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>;
4119}
4120
4121multiclass sme2_ld_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n,
4122                                                     RegisterOperand multi_vector_ty,
4123                                                     Operand index_ty,
4124                                                     string mnemonic> {
4125  def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, 0b10,
4126                                                        multi_vector_ty,
4127                                                        index_ty, mnemonic> {
4128    bits<3> Zt;
4129    let Inst{4} = Zt{2};
4130    let Inst{1-0} = Zt{1-0};
4131  }
4132
4133   def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]",
4134                   (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>;
4135}
4136
4137//===----------------------------------------------------------------------===//
4138// SME2 Non-Contiguous Store
4139class sme2_st_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n,
4140                                             RegisterOperand multi_vector_ty,
4141                                             RegisterOperand gpr_ty,
4142                                             string mnemonic>
4143   : I<(outs ),
4144       (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),
4145       mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]",
4146       "", []>, Sched<[]> {
4147   bits<5> Rm;
4148   bits<3> PNg;
4149   bits<5> Rn;
4150   bits<4> Zt;
4151   let Inst{31-21} = 0b10100001001;
4152   let Inst{20-16} = Rm;
4153   let Inst{15}    = 0b0;
4154   let Inst{14-13} = msz;
4155   let Inst{12-10} = PNg;
4156   let Inst{9-5}   = Rn;
4157   let Inst{4}     = Zt{3};
4158   let Inst{3}     = n;
4159   let Inst{2-0}   = Zt{2-0};
4160
4161   let mayStore    = 1;
4162}
4163
4164class sme2_st_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n,
4165                                             RegisterOperand multi_vector_ty,
4166                                             RegisterOperand gpr_ty,
4167                                             string mnemonic>
4168   : I<(outs ),
4169       (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),
4170       mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]",
4171       "", []>, Sched<[]> {
4172   bits<5> Rm;
4173   bits<3> PNg;
4174   bits<5> Rn;
4175   bits<3> Zt;
4176   let Inst{31-21} = 0b10100001001;
4177   let Inst{20-16} = Rm;
4178   let Inst{15}     = 0b1;
4179   let Inst{14-13} = msz;
4180   let Inst{12-10} = PNg;
4181   let Inst{9-5}   = Rn;
4182   let Inst{4}     = Zt{2};
4183   let Inst{3}     = n;
4184   let Inst{2}     = 0b0;
4185   let Inst{1-0}   = Zt{1-0};
4186
4187   let mayStore    = 1;
4188}
4189
4190class sme2_st_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op,
4191                                                 RegisterOperand multi_vector_ty,
4192                                                 Operand index_ty,
4193                                                 string mnemonic>
4194    : I<(outs ),
4195        (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4),
4196        mnemonic,  "\t$Zt, $PNg, [$Rn, $imm4, mul vl]",
4197        "", []>, Sched<[]> {
4198   bits<4> imm4;
4199   bits<3> PNg;
4200   bits<5> Rn;
4201   let Inst{31-20} = 0b101000010110;
4202   let Inst{19-16} = imm4;
4203   let Inst{15}    = op{1};
4204   let Inst{14-13} = msz;
4205   let Inst{12-10} = PNg;
4206   let Inst{9-5}   = Rn;
4207   let Inst{3}     = n;
4208   let Inst{2}     = op{0};
4209
4210   let mayStore    = 1;
4211}
4212
4213
4214multiclass sme2_st_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n,
4215                                                     RegisterOperand multi_vector_ty,
4216                                                     Operand index_ty,
4217                                                     string mnemonic> {
4218  def NAME: sme2_st_vector_vg24_multi_scalar_immediate<msz, n, {0,?},
4219                                                       multi_vector_ty,
4220                                                       index_ty, mnemonic> {
4221    bits<4> Zt;
4222    let Inst{4}   = Zt{3};
4223    let Inst{2-0} = Zt{2-0};
4224  }
4225
4226    def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]",
4227                   (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>;
4228}
4229
4230multiclass sme2_st_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n,
4231                                                     RegisterOperand multi_vector_ty,
4232                                                     Operand index_ty,
4233                                                     string mnemonic> {
4234  def NAME : sme2_st_vector_vg24_multi_scalar_immediate<msz, n, 0b10,
4235                                                        multi_vector_ty,
4236                                                        index_ty, mnemonic> {
4237    bits<3> Zt;
4238    let Inst{4}   = Zt{2};
4239    let Inst{1-0} = Zt{1-0};
4240  }
4241
4242    def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]",
4243                   (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>;
4244}
4245
4246//===----------------------------------------------------------------------===//
4247// SME2.1
4248//===----------------------------------------------------------------------===//
4249// SME zeroing move array to vector
4250class sme2p1_movaz_tile_to_vec_base<bits<2> sz, bit q, bit v, ZPRRegOp vector_ty,
4251                                    RegisterOperand tile_ty, Operand index_ty,
4252                                    string mnemonic>
4253    : I<(outs vector_ty:$Zd, tile_ty:$ZAn),
4254        (ins tile_ty:$_ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm),
4255        mnemonic, "\t$Zd, $ZAn[$Rs, $imm]",
4256        "", []>, Sched<[]> {
4257  bits<2> Rs;
4258  bits<5> Zd;
4259  let Inst{31-24} = 0b11000000;
4260  let Inst{23-22} = sz;
4261  let Inst{21-17} = 0b00001;
4262  let Inst{16}    = q;
4263  let Inst{15}    = v;
4264  let Inst{14-13} = Rs;
4265  let Inst{12-9}  = 0b0001;
4266  let Inst{4-0}   = Zd;
4267  let Constraints = "$ZAn = $_ZAn";
4268}
4269
4270multiclass sme2p1_movaz_tile_to_vec_base<bit v, string mnemonic> {
4271  def _B : sme2p1_movaz_tile_to_vec_base<0b00, 0b0, v, ZPR8,
4272                                    !if(v, TileVectorOpV8, TileVectorOpH8),
4273                                    sme_elm_idx0_15, mnemonic> {
4274    bits<4> imm;
4275    let Inst{8-5} = imm;
4276  }
4277
4278  def _H : sme2p1_movaz_tile_to_vec_base<0b01, 0b0, v, ZPR16,
4279                                    !if(v, TileVectorOpV16, TileVectorOpH16),
4280                                    sme_elm_idx0_7, mnemonic> {
4281    bits<1> ZAn;
4282    bits<3> imm;
4283    let Inst{8}   = ZAn;
4284    let Inst{7-5} = imm;
4285  }
4286
4287  def _S : sme2p1_movaz_tile_to_vec_base<0b10, 0b0, v, ZPR32,
4288                                    !if(v, TileVectorOpV32, TileVectorOpH32),
4289                                    sme_elm_idx0_3, mnemonic> {
4290    bits<2> ZAn;
4291    bits<2> imm;
4292    let Inst{8-7} = ZAn;
4293    let Inst{6-5} = imm;
4294  }
4295
4296  def _D : sme2p1_movaz_tile_to_vec_base<0b11, 0b0, v, ZPR64,
4297                                    !if(v, TileVectorOpV64, TileVectorOpH64),
4298                                    sme_elm_idx0_1, mnemonic> {
4299    bits<3> ZAn;
4300    bits<1> imm;
4301    let Inst{8-6} = ZAn;
4302    let Inst{5}   = imm;
4303  }
4304
4305  def _Q : sme2p1_movaz_tile_to_vec_base<0b11, 0b1, v, ZPR128,
4306                                    !if(v, TileVectorOpV128, TileVectorOpH128),
4307                                    sme_elm_idx0_0, mnemonic> {
4308    bits<4> ZAn;
4309    let Inst{8-5} = ZAn;
4310  }
4311}
4312
4313multiclass sme2p1_movaz_tile_to_vec<string mnemonic>{
4314 defm _H : sme2p1_movaz_tile_to_vec_base<0b0, mnemonic>;
4315 defm _V : sme2p1_movaz_tile_to_vec_base<0b1, mnemonic>;
4316}
4317
4318//===----------------------------------------------------------------------===//
4319// SME2.1 multiple vectors zero array
4320
4321class sme2p1_zero_matrix<bits<6> opc, Operand index_ty, string mnemonic,
4322                         string vg_acronym="">
4323    : I<(outs MatrixOp64:$ZAd),
4324        (ins MatrixOp64:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm),
4325        mnemonic, "\t$ZAd[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]",
4326        "", []>, Sched<[]> {
4327  bits <2> Rv;
4328  let Inst{31-18} = 0b11000000000011;
4329  let Inst{17-15} = opc{5-3};
4330  let Inst{14-13} = Rv;
4331  let Inst{12-3} = 0b0000000000;
4332  let Inst{2-0}  = opc{2-0};
4333  let Constraints = "$ZAd = $_ZAd";
4334}
4335
4336multiclass sme2p1_zero_matrix<string mnemonic> {
4337  def _VG2_Z : sme2p1_zero_matrix<{0b000,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx2"> {
4338    bits<3> imm;
4339    let Inst{2-0} = imm;
4340  }
4341  def _2Z : sme2p1_zero_matrix<{0b001,?,?,?}, uimm3s2range, mnemonic> {
4342    bits<3> imm;
4343    let Inst{2-0} = imm;
4344  }
4345  def _VG2_2Z : sme2p1_zero_matrix<{0b0100,?,?}, uimm2s2range, mnemonic, "vgx2"> {
4346    bits<2> imm;
4347    let Inst{1-0} = imm;
4348  }
4349  def _VG4_2Z : sme2p1_zero_matrix<{0b0110,?,?}, uimm2s2range, mnemonic, "vgx4"> {
4350    bits<2> imm;
4351    let Inst{1-0} = imm;
4352  }
4353  def _VG4_Z : sme2p1_zero_matrix<{0b100,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx4"> {
4354    bits<3> imm;
4355    let Inst{2-0} = imm;
4356  }
4357  def _4Z : sme2p1_zero_matrix<{0b1010,?,?}, uimm2s4range, mnemonic> {
4358    bits<2> imm;
4359    let Inst{1-0} = imm;
4360  }
4361  def _VG2_4Z :sme2p1_zero_matrix<{0b11000,?}, uimm1s4range, mnemonic, "vgx2"> {
4362    bits<1> imm;
4363    let Inst{0}   = imm;
4364  }
4365  def _VG4_4Z :sme2p1_zero_matrix<{0b11100,?}, uimm1s4range, mnemonic, "vgx4"> {
4366    bits<1> imm;
4367    let Inst{0}   = imm;
4368  }
4369}
4370
4371//===----------------------------------------------------------------------===//
4372// SME2.1 lookup table expand two non-contiguous registers
4373
4374class sme2p1_luti_vector_vg2_index<bits<4> op, bits<2> sz, RegisterOperand vector_ty,
4375                                   AsmVectorIndexOpnd index_ty,
4376                                   string mnemonic>
4377    :  I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),
4378          mnemonic, "\t$Zd, $ZTt, $Zn$i",
4379          "", []>, Sched<[]> {
4380  bits<5> Zn;
4381  bits<4> Zd;
4382  let Inst{31-19} = 0b1100000010011;
4383  let Inst{18-15} = op;
4384  let Inst{14}    = 0b1;
4385  let Inst{13-12} = sz;
4386  let Inst{11-10} = 0b00;
4387  let Inst{9-5}   = Zn;
4388  let Inst{4}     = Zd{3};
4389  let Inst{3}     = 0b0;
4390  let Inst{2-0}   = Zd{2-0};
4391}
4392
4393class sme2p1_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty,
4394                                    AsmVectorIndexOpnd index_ty,
4395                                    string mnemonic>
4396  : sme2p1_luti_vector_vg2_index<{1,?,?,?}, sz, vector_ty, index_ty, mnemonic> {
4397  bits<3> i;
4398  let Inst{17-15} = i;
4399}
4400
4401multiclass sme2p1_luti2_vector_vg2_index<string mnemonic> {
4402  def _B : sme2p1_luti2_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexH,
4403                                         mnemonic>;
4404  def _H : sme2p1_luti2_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexH,
4405                                         mnemonic>;
4406}
4407
4408class sme2p1_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty,
4409                                    AsmVectorIndexOpnd index_ty,
4410                                    string mnemonic>
4411  : sme2p1_luti_vector_vg2_index<{0b01,?,?}, sz, vector_ty, index_ty, mnemonic> {
4412  bits<2> i;
4413  let Inst{16-15} = i;
4414}
4415multiclass sme2p1_luti4_vector_vg2_index<string mnemonic> {
4416  def _B : sme2p1_luti4_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexS,
4417                                         mnemonic>;
4418  def _H : sme2p1_luti4_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexS,
4419                                         mnemonic>;
4420}
4421
4422// SME2.1 lookup table expand four non-contiguous registers
4423class sme2p1_luti_vector_vg4_index<bits<3> op, bits<2> sz, RegisterOperand vector_ty,
4424                                   AsmVectorIndexOpnd index_ty,
4425                                   string mnemonic>
4426    :  I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),
4427          mnemonic, "\t$Zd, $ZTt, $Zn$i",
4428          "", []>, Sched<[]> {
4429  bits<5> Zn;
4430  bits<3> Zd;
4431  let Inst{31-19} = 0b1100000010011;
4432  let Inst{18-16} = op;
4433  let Inst{15-14} = 0b10;
4434  let Inst{13-12} = sz;
4435  let Inst{11-10} = 0b00;
4436  let Inst{9-5}   = Zn;
4437  let Inst{4}     = Zd{2};
4438  let Inst{3-2}   = 0b00;
4439  let Inst{1-0}   = Zd{1-0};
4440}
4441
4442class sme2p1_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty,
4443                                    AsmVectorIndexOpnd index_ty,
4444                                    string mnemonic>
4445  : sme2p1_luti_vector_vg4_index<{1,?,?}, sz, vector_ty, index_ty, mnemonic> {
4446  bits<2> i;
4447  let Inst{17-16} = i;
4448}
4449
4450multiclass sme2p1_luti2_vector_vg4_index<string mnemonic> {
4451  def _B : sme2p1_luti2_vector_vg4_index<0b00, ZZZZ_b_strided, VectorIndexS,
4452                                         mnemonic>;
4453  def _H : sme2p1_luti2_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexS,
4454                                         mnemonic>;
4455}
4456
4457class sme2p1_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty,
4458                                    AsmVectorIndexOpnd index_ty,
4459                                    string mnemonic>
4460  : sme2p1_luti_vector_vg4_index<{0b01,?}, sz, vector_ty, index_ty, mnemonic> {
4461  bit i;
4462  let Inst{16}    = i;
4463}
4464
4465multiclass sme2p1_luti4_vector_vg4_index<string mnemonic> {
4466  def _H: sme2p1_luti4_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexD, mnemonic>;
4467}
4468