1//=-- SMEInstrFormats.td - AArch64 SME Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Matrix Extension (SME) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def imm_to_tile8 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAB0, 0>", []>; 14def imm_to_tile16 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAH0, 1>", []>; 15def imm_to_tile32 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAS0, 3>", []>; 16def imm_to_tile64 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAD0, 7>", []>; 17def imm_to_tile128 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAQ0, 15>", []>; 18def imm_to_zt : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZT0, 0>", []>; 19 20def tileslice8 : ComplexPattern<i32 , 2, "SelectSMETileSlice<15, 1>", []>; 21def tileslice16 : ComplexPattern<i32 , 2, "SelectSMETileSlice<7, 1>", []>; 22def tileslice32 : ComplexPattern<i32 , 2, "SelectSMETileSlice<3, 1>", []>; 23def tileslice64 : ComplexPattern<i32 , 2, "SelectSMETileSlice<1, 1>", []>; 24def tileslice128 : ComplexPattern<i32 , 2, "SelectSMETileSlice<0, 1>", []>; // nop 25 26def tileslicerange3s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<14, 2>", []>; 27def tileslicerange2s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<6, 2>", []>; 28def tileslicerange1s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<2, 2>", []>; 29def tileslicerange0s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<0, 2>", []>; 30 31def tileslicerange2s4 : ComplexPattern<i32, 2, "SelectSMETileSlice<12, 4>", []>; 32def tileslicerange1s4 : ComplexPattern<i32, 2, "SelectSMETileSlice<4, 4>", []>; 33def tileslicerange0s4 : ComplexPattern<i32, 2, "SelectSMETileSlice<0, 4>", []>; 34 35def am_sme_indexed_b4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<0,15>", [], [SDNPWantRoot]>; 36 37def SDTZALoadStore : SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>]>; 38def AArch64SMELdr : SDNode<"AArch64ISD::SME_ZA_LDR", SDTZALoadStore, 39 [SDNPHasChain, SDNPSideEffect, SDNPMayLoad]>; 40def AArch64SMEStr : SDNode<"AArch64ISD::SME_ZA_STR", SDTZALoadStore, 41 [SDNPHasChain, SDNPSideEffect, SDNPMayStore]>; 42 43//===----------------------------------------------------------------------===// 44// SME Pseudo Classes 45//===----------------------------------------------------------------------===// 46 47def getSMEPseudoMap : InstrMapping { 48 let FilterClass = "SMEPseudo2Instr"; 49 let RowFields = ["PseudoName"]; 50 let ColFields = ["IsInstr"]; 51 let KeyCol = ["0"]; 52 let ValueCols = [["1"]]; 53} 54 55class SMEPseudo2Instr<string name, bit instr> { 56 string PseudoName = name; 57 bit IsInstr = instr; 58} 59 60class sme_outer_product_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> 61 : Pseudo<(outs), (ins i32imm:$tile, PPR3bAny:$pn, PPR3bAny:$pm, 62 zpr_ty:$zn, zpr_ty:$zm), []>, 63 Sched<[]> { 64 // Translated to the actual instructions in AArch64ISelLowering.cpp 65 let SMEMatrixType = za_flag; 66 let usesCustomInserter = 1; 67} 68 69class sme2_za_array_2op_multi_single_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, 70 ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> 71 : SMEPseudo2Instr<name, 0>, 72 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), []> { 73 let SMEMatrixType = za_flag; 74 let usesCustomInserter = 1; 75} 76 77class sme2_za_array_2op_multi_multi_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, 78 SMEMatrixTypeEnum za_flag> 79 : SMEPseudo2Instr<name, 0>, 80 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), []> { 81 let SMEMatrixType = za_flag; 82 let usesCustomInserter = 1; 83} 84 85class sme2_za_array_2op_multi_index_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, 86 ZPRRegOp zpr_ty, Operand imm_ty, SMEMatrixTypeEnum za_flag> 87 : SMEPseudo2Instr<name, 0>, 88 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, imm_ty:$i), []> { 89 let SMEMatrixType = za_flag; 90 let usesCustomInserter = 1; 91} 92 93class sme2_move_to_za_pseudo<string name, Operand imm_ty, RegisterOperand multi_vector_ty, SMEMatrixTypeEnum za_flag> 94 : SMEPseudo2Instr<name, 0>, 95 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rs, imm_ty:$imm, multi_vector_ty:$Zn), []> { 96 let SMEMatrixType = za_flag; 97 let usesCustomInserter = 1; 98} 99 100class sme2_move_to_tile_pseudo<string name, Operand tile_imm, Operand imm_ty, RegisterOperand multi_vector_ty, SMEMatrixTypeEnum za_flag> 101 : SMEPseudo2Instr<name, 0>, 102 Pseudo<(outs), (ins tile_imm:$tile, MatrixIndexGPR32Op12_15:$Rs, imm_ty:$imm, multi_vector_ty:$Zn), []> { 103 let SMEMatrixType = za_flag; 104 let usesCustomInserter = 1; 105} 106 107//===----------------------------------------------------------------------===// 108// SME pattern match helpers. 109//===----------------------------------------------------------------------===// 110 111class SME2_ZA_TwoOp_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, 112 ValueType vt, ComplexPattern tileslice> 113 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm), 114 (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm)>; 115 116 117class SME2_ZA_TwoOp_VG2_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, 118 ValueType vt, ComplexPattern tileslice> 119 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm), 120 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), 121 zpr_ty:$Zm)>; 122class SME2_ZA_TwoOp_VG4_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, 123 ValueType vt, ComplexPattern tileslice> 124 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), 125 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm), 126 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 127 (REG_SEQUENCE ZPR4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), 128 zpr_ty:$Zm)>; 129 130class SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice> 131 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2), 132 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 133 (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), 134 (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>; 135 136class SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice> 137 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), 138 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm1, vt:$Zm2, vt:$Zm3, vt:$Zm4), 139 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 140 (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), 141 (REG_SEQUENCE ZPR4Mul4, vt:$Zm1, zsub0, vt:$Zm2, zsub1, vt:$Zm3, zsub2, vt:$Zm4, zsub3))>; 142 143class SME2_ZA_TwoOp_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, 144 Operand imm_ty, ComplexPattern tileslice> 145 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm, (i32 imm_ty:$i)), 146 (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm, (i32 imm_ty:$i))>; 147 148 149class SME2_ZA_TwoOp_VG2_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, 150 Operand imm_ty, ComplexPattern tileslice> 151 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm, (i32 imm_ty:$i)), 152 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 153 (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), zpr_ty:$Zm, imm_ty:$i)>; 154 155class SME2_ZA_TwoOp_VG4_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, 156 Operand imm_ty, ComplexPattern tileslice> 157 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), 158 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm, (i32 imm_ty:$i)), 159 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 160 (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), 161 zpr_ty:$Zm, imm_ty:$i)>; 162 163class SME2_Sat_Shift_VG2_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt, Operand imm_ty> 164 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, (i32 imm_ty:$i))), 165 (!cast<Instruction>(name) (REG_SEQUENCE ZPR2Mul2, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1), imm_ty:$i)>; 166 167class SME2_Sat_Shift_VG4_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt, Operand imm_ty> 168 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, in_vt:$Zn3, in_vt:$Zn4, (i32 imm_ty:$i))), 169 (!cast<Instruction>(name) (REG_SEQUENCE ZPR4Mul4, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1, in_vt:$Zn3, zsub2, in_vt:$Zn4, zsub3), 170 imm_ty:$i)>; 171 172class SME2_Cvt_VG4_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt> 173 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, in_vt:$Zn3, in_vt:$Zn4)), 174 (!cast<Instruction>(name) (REG_SEQUENCE ZPR4Mul4, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1, in_vt:$Zn3, zsub2, in_vt:$Zn4, zsub3))>; 175 176class SME2_ZA_VG1x2_Multi_Pat<string name, SDPatternOperator intrinsic, ValueType vt, Operand index_ty, ComplexPattern tileslice> 177 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2), 178 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1))>; 179 180class SME2_ZA_VG1x4_Multi_Pat<string name, SDPatternOperator intrinsic, ValueType vt, Operand index_ty, ComplexPattern tileslice> 181 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4), 182 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3))>; 183 184class SME2_Tile_VG2_Multi_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, ValueType vt, Operand index_ty, ComplexPattern tileslice> 185 : Pat<(intrinsic tile_imm:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2), 186 (!cast<Instruction>(name # _PSEUDO) $tile, $base, $offset, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1))>; 187 188class SME2_Tile_VG4_Multi_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, ValueType vt, Operand index_ty, ComplexPattern tileslice> 189 : Pat<(intrinsic tile_imm:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4), 190 (!cast<Instruction>(name # _PSEUDO) $tile, $base, $offset, (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3))>; 191 192//===----------------------------------------------------------------------===// 193// SME pattern match helpers. 194//===----------------------------------------------------------------------===// 195 196class SME_ZA_Tile_TwoPred_TwoVec_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType pg_ty, ValueType vt> 197 : Pat<(intrinsic imm_ty:$tile, (pg_ty PPR3bAny:$Pn), (pg_ty PPR3bAny:$Pm), vt:$Zn, vt:$Zm), 198 (!cast<Instruction>(name # _PSEUDO) $tile, $Pn, $Pm, $Zn, $Zm)>; 199 200 201//===----------------------------------------------------------------------===// 202// SME smstart/smstop 203//===----------------------------------------------------------------------===// 204 205// SME defines three pstate fields to set or clear PSTATE.SM, PSTATE.ZA, or 206// both fields: 207// 208// MSR SVCRSM, #<imm1> 209// MSR SVCRZA, #<imm1> 210// MSR SVCRSMZA, #<imm1> 211// 212// It's tricky to using the existing pstate operand defined in 213// AArch64SystemOperands.td since it only encodes 5 bits including op1;op2, 214// when these fields are also encoded in CRm[3:1]. 215def MSRpstatesvcrImm1 216 : PstateWriteSimple<(ins svcr_op:$pstatefield, timm0_1:$imm), "msr", 217 "\t$pstatefield, $imm">, 218 Sched<[WriteSys]> { 219 bits<3> pstatefield; 220 bit imm; 221 let Inst{18-16} = 0b011; // op1 222 let Inst{11-9} = pstatefield; 223 let Inst{8} = imm; 224 let Inst{7-5} = 0b011; // op2 225 let hasPostISelHook = 1; 226} 227 228def : InstAlias<"smstart", (MSRpstatesvcrImm1 0b011, 0b1)>; 229def : InstAlias<"smstart sm", (MSRpstatesvcrImm1 0b001, 0b1)>; 230def : InstAlias<"smstart za", (MSRpstatesvcrImm1 0b010, 0b1)>; 231 232def : InstAlias<"smstop", (MSRpstatesvcrImm1 0b011, 0b0)>; 233def : InstAlias<"smstop sm", (MSRpstatesvcrImm1 0b001, 0b0)>; 234def : InstAlias<"smstop za", (MSRpstatesvcrImm1 0b010, 0b0)>; 235 236 237//===----------------------------------------------------------------------===// 238// SME Outer Products 239//===----------------------------------------------------------------------===// 240 241class sme_fp_outer_product_inst<bit S, bits<2> sz, bits<2> op, MatrixTileOperand za_ty, 242 ZPRRegOp zpr_ty, string mnemonic> 243 : I<(outs za_ty:$ZAda), 244 (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), 245 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", 246 "", []>, 247 Sched<[]> { 248 bits<5> Zm; 249 bits<3> Pm; 250 bits<3> Pn; 251 bits<5> Zn; 252 let Inst{31-25} = 0b1000000; 253 let Inst{24} = op{1}; 254 let Inst{23} = 0b1; 255 let Inst{22-21} = sz; 256 let Inst{20-16} = Zm; 257 let Inst{15-13} = Pm; 258 let Inst{12-10} = Pn; 259 let Inst{9-5} = Zn; 260 let Inst{4} = S; 261 let Inst{3} = op{0}; 262 263 let Constraints = "$ZAda = $_ZAda"; 264} 265 266multiclass sme_outer_product_fp32<bit S, bits<2> sz, ZPRRegOp zpr_ty, string mnemonic, SDPatternOperator op> { 267 def NAME : sme_fp_outer_product_inst<S, sz, 0b00, TileOp32, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> { 268 bits<2> ZAda; 269 let Inst{1-0} = ZAda; 270 let Inst{2} = 0b0; 271 } 272 273 def NAME # _PSEUDO : sme_outer_product_pseudo<zpr_ty, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 274 275 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv4i1, nxv4f32>; 276} 277 278multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op> { 279 def NAME : sme_fp_outer_product_inst<S, 0b10, 0b00, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> { 280 bits<3> ZAda; 281 let Inst{2-0} = ZAda; 282 } 283 284 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; 285 286 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv2i1, nxv2f64>; 287} 288 289multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s, bits<2> op, ZPRRegOp zpr_ty>{ 290 def NAME : sme_fp_outer_product_inst<s, {0,bf}, op, TileOp16, zpr_ty, mnemonic> { 291 bits<1> ZAda; 292 let Inst{2-1} = 0b00; 293 let Inst{0} = ZAda; 294 } 295} 296 297class sme_int_outer_product_inst<bits<3> opc, bit sz, bit sme2, 298 MatrixTileOperand za_ty, ZPRRegOp zpr_ty, 299 string mnemonic> 300 : I<(outs za_ty:$ZAda), 301 (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), 302 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", 303 "", []>, 304 Sched<[]> { 305 bits<5> Zm; 306 bits<3> Pm; 307 bits<3> Pn; 308 bits<5> Zn; 309 let Inst{31-25} = 0b1010000; 310 let Inst{24} = opc{2}; // u0 311 let Inst{23} = 0b1; 312 let Inst{22} = sz; 313 let Inst{21} = opc{1}; // u1 314 let Inst{20-16} = Zm; 315 let Inst{15-13} = Pm; 316 let Inst{12-10} = Pn; 317 let Inst{9-5} = Zn; 318 let Inst{4} = opc{0}; //S; 319 let Inst{3} = sme2; 320 321 let Constraints = "$ZAda = $_ZAda"; 322} 323 324multiclass sme_int_outer_product_i32<bits<3> opc, string mnemonic, 325 SDPatternOperator op> { 326 def NAME : sme_int_outer_product_inst<opc, 0b0, 0b0, TileOp32, 327 ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> { 328 bits<2> ZAda; 329 let Inst{1-0} = ZAda; 330 let Inst{2} = 0b0; 331 } 332 333 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 334 335 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv16i1, nxv16i8>; 336} 337 338multiclass sme_int_outer_product_i64<bits<3> opc, string mnemonic, 339 SDPatternOperator op> { 340 def NAME : sme_int_outer_product_inst<opc, 0b1, 0b0, TileOp64, 341 ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> { 342 bits<3> ZAda; 343 let Inst{2-0} = ZAda; 344 } 345 346 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; 347 348 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv8i1, nxv8i16>; 349} 350 351class sme_outer_product_widening_inst<bits<3> opc, ZPRRegOp zpr_ty, string mnemonic> 352 : I<(outs TileOp32:$ZAda), 353 (ins TileOp32:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), 354 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", 355 "", []>, 356 Sched<[]> { 357 bits<5> Zm; 358 bits<3> Pm; 359 bits<3> Pn; 360 bits<5> Zn; 361 bits<2> ZAda; 362 let Inst{31-25} = 0b1000000; 363 let Inst{24} = !if(opc{2}, 0, 1); 364 let Inst{23-22} = 0b10; 365 let Inst{21} = opc{1}; 366 let Inst{20-16} = Zm; 367 let Inst{15-13} = Pm; 368 let Inst{12-10} = Pn; 369 let Inst{9-5} = Zn; 370 let Inst{4} = opc{0}; 371 let Inst{3} = opc{2}; 372 let Inst{2} = 0b0; 373 let Inst{1-0} = ZAda; 374 375 let Constraints = "$ZAda = $_ZAda"; 376} 377 378multiclass sme_bf16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> { 379 def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>; 380 381 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 382 383 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv8i1, nxv8bf16>; 384} 385 386multiclass sme_f16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> { 387 def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>; 388 389 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 390 391 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv8i1, nxv8f16>; 392} 393 394//===----------------------------------------------------------------------===// 395// SME Add Vector to Tile 396//===----------------------------------------------------------------------===// 397 398class sme_add_vector_to_tile_inst<bit op, bit V, MatrixTileOperand tile_ty, 399 ZPRRegOp zpr_ty, string mnemonic> 400 : I<(outs tile_ty:$ZAda), 401 (ins tile_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), 402 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn", 403 "", []>, Sched<[]> { 404 bits<3> Pm; 405 bits<3> Pn; 406 bits<5> Zn; 407 let Inst{31-23} = 0b110000001; 408 let Inst{22} = op; 409 let Inst{21-17} = 0b01000; 410 let Inst{16} = V; 411 let Inst{15-13} = Pm; 412 let Inst{12-10} = Pn; 413 let Inst{9-5} = Zn; 414 let Inst{4-3} = 0b00; 415 416 let Constraints = "$ZAda = $_ZAda"; 417} 418 419class sme_add_vector_to_tile_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> 420 : Pseudo<(outs), 421 (ins i32imm:$tile, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), []>, 422 Sched<[]> { 423 // Translated to the actual instructions in AArch64ISelLowering.cpp 424 let SMEMatrixType = za_flag; 425 let usesCustomInserter = 1; 426} 427 428multiclass sme_add_vector_to_tile_u32<bit V, string mnemonic, SDPatternOperator op> { 429 def NAME : sme_add_vector_to_tile_inst<0b0, V, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> { 430 bits<2> ZAda; 431 let Inst{2} = 0b0; 432 let Inst{1-0} = ZAda; 433 } 434 435 def _PSEUDO_S : sme_add_vector_to_tile_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 436 437 def : Pat<(op timm32_0_3:$tile, (nxv4i1 PPR3bAny:$pn), (nxv4i1 PPR3bAny:$pm), 438 (nxv4i32 ZPR32:$zn)), 439 (!cast<Instruction>(NAME # _PSEUDO_S) timm32_0_3:$tile, $pn, $pm, $zn)>; 440} 441 442multiclass sme_add_vector_to_tile_u64<bit V, string mnemonic, SDPatternOperator op> { 443 def NAME : sme_add_vector_to_tile_inst<0b1, V, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> { 444 bits<3> ZAda; 445 let Inst{2-0} = ZAda; 446 } 447 448 def _PSEUDO_D : sme_add_vector_to_tile_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; 449 450 let Predicates = [HasSMEI16I64] in { 451 def : Pat<(op timm32_0_7:$tile, (nxv2i1 PPR3bAny:$pn), (nxv2i1 PPR3bAny:$pm), 452 (nxv2i64 ZPR64:$zn)), 453 (!cast<Instruction>(NAME # _PSEUDO_D) timm32_0_7:$tile, $pn, $pm, $zn)>; 454 } 455} 456 457//===----------------------------------------------------------------------===// 458// SME Contiguous Loads 459//===----------------------------------------------------------------------===// 460 461class sme_mem_ld_ss_base<bit Q, bit V, bits<2> msz, dag outs, dag ins, 462 string mnemonic, string argstr> 463 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { 464 bits<5> Rm; 465 bits<2> Rv; 466 bits<3> Pg; 467 bits<5> Rn; 468 let Inst{31-25} = 0b1110000; 469 let Inst{24} = Q; 470 let Inst{23-22} = msz; 471 let Inst{21} = 0b0; 472 let Inst{20-16} = Rm; 473 let Inst{15} = V; 474 let Inst{14-13} = Rv; 475 let Inst{12-10} = Pg; 476 let Inst{9-5} = Rn; 477 let Inst{4} = 0b0; 478 479 let mayLoad = 1; 480} 481 482class sme_mem_ld_ss_inst<bit Q, bits<2> msz, string mnemonic, 483 MatrixTileVectorOperand tile_ty, bit is_col, 484 Operand imm_ty, RegisterOperand gpr_ty> 485 : sme_mem_ld_ss_base< 486 Q, is_col, msz, (outs tile_ty:$ZAt), 487 (ins MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, 488 gpr_ty:$Rm), 489 mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg/z, [$Rn, $Rm]">; 490 491multiclass sme_mem_ss_aliases_base<string mnemonic, Instruction inst, 492 MatrixTileVectorOperand tile_ty, 493 Operand imm_ty, RegisterOperand gpr_ty, 494 string pg_suffix=""> { 495 def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn, $Rm]", 496 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, gpr_ty:$Rm), 0>; 497 // Default XZR offset aliases 498 def : InstAlias<mnemonic # "\t\\{$ZAt[$Rv, $imm]\\}, $Pg" # pg_suffix # ", [$Rn]", 499 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 500 def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn]", 501 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 502} 503 504multiclass sme_mem_ss_aliases<string mnemonic, string inst, bit is_col, 505 string pg_suffix=""> { 506 defm : sme_mem_ss_aliases_base<mnemonic # "b", !cast<Instruction>(inst # _B), 507 !if(is_col, TileVectorOpV8, TileVectorOpH8), 508 sme_elm_idx0_15, GPR64shifted8, pg_suffix>; 509 defm : sme_mem_ss_aliases_base<mnemonic # "h", !cast<Instruction>(inst # _H), 510 !if(is_col, TileVectorOpV16, TileVectorOpH16), 511 sme_elm_idx0_7, GPR64shifted16, pg_suffix>; 512 defm : sme_mem_ss_aliases_base<mnemonic # "w", !cast<Instruction>(inst # _S), 513 !if(is_col, TileVectorOpV32, TileVectorOpH32), 514 sme_elm_idx0_3, GPR64shifted32, pg_suffix>; 515 defm : sme_mem_ss_aliases_base<mnemonic # "d", !cast<Instruction>(inst # _D), 516 !if(is_col, TileVectorOpV64, TileVectorOpH64), 517 sme_elm_idx0_1, GPR64shifted64, pg_suffix>; 518 defm : sme_mem_ss_aliases_base<mnemonic # "q", !cast<Instruction>(inst # _Q), 519 !if(is_col, TileVectorOpV128, TileVectorOpH128), 520 sme_elm_idx0_0, GPR64shifted128, pg_suffix>; 521} 522 523multiclass sme_mem_ld_ss_aliases<string inst, bit is_col> { 524 defm NAME : sme_mem_ss_aliases<"ld1", inst, is_col, "/z">; 525} 526 527multiclass sme_mem_ld_ss_patterns<Instruction Inst, SDPatternOperator Load, 528 Operand tile_ty, Operand offset_ty, 529 ComplexPattern addr, 530 ComplexPattern tileslice> { 531 // base, tileslice 532 def : Pat<(Load PPR3bAny:$pg, GPR64sp:$base, tile_ty:$tile, 533 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), 534 (Inst tile_ty:$tile, $idx, $imm, $pg, $base, XZR)>; 535 536 // reg + reg, tileslice 537 let AddedComplexity = 1 in { 538 def : Pat<(Load PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset), 539 tile_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, 540 offset_ty:$imm))), 541 (Inst tile_ty:$tile, $idx, $imm, $pg, $base, $offset)>; 542 } 543} 544 545class sme_load_pseudo 546 : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx, 547 i32imm:$imm, PPR3bAny:$pg, GPR64sp:$base, GPR64:$offset), []>, 548 Sched<[]> { 549 // Translated to the actual instructions in AArch64ISelLowering.cpp 550 let usesCustomInserter = 1; 551 let mayLoad = 1; 552} 553 554multiclass sme_mem_ld_v_ss<string mnemonic, bit is_col> { 555 def _B : sme_mem_ld_ss_inst<0b0, 0b00, mnemonic # "b", 556 !if(is_col, TileVectorOpV8, TileVectorOpH8), 557 is_col, sme_elm_idx0_15, GPR64shifted8> { 558 bits<4> imm; 559 let Inst{3-0} = imm; 560 } 561 def _H : sme_mem_ld_ss_inst<0b0, 0b01, mnemonic # "h", 562 !if(is_col, TileVectorOpV16, TileVectorOpH16), 563 is_col, sme_elm_idx0_7, GPR64shifted16> { 564 bits<1> ZAt; 565 bits<3> imm; 566 let Inst{3} = ZAt; 567 let Inst{2-0} = imm; 568 } 569 def _S : sme_mem_ld_ss_inst<0b0, 0b10, mnemonic # "w", 570 !if(is_col, TileVectorOpV32, TileVectorOpH32), 571 is_col, sme_elm_idx0_3, GPR64shifted32> { 572 bits<2> ZAt; 573 bits<2> imm; 574 let Inst{3-2} = ZAt; 575 let Inst{1-0} = imm; 576 } 577 def _D : sme_mem_ld_ss_inst<0b0, 0b11, mnemonic # "d", 578 !if(is_col, TileVectorOpV64, TileVectorOpH64), 579 is_col, sme_elm_idx0_1, GPR64shifted64> { 580 bits<3> ZAt; 581 bits<1> imm; 582 let Inst{3-1} = ZAt; 583 let Inst{0} = imm; 584 } 585 def _Q : sme_mem_ld_ss_inst<0b1, 0b11, mnemonic # "q", 586 !if(is_col, TileVectorOpV128, TileVectorOpH128), 587 is_col, sme_elm_idx0_0, GPR64shifted128> { 588 bits<4> ZAt; 589 let Inst{3-0} = ZAt; 590 } 591 592 defm : sme_mem_ld_ss_aliases<NAME, is_col>; 593 594 // Pseudo instructions for lowering intrinsics, using immediates instead of 595 // tile registers. 596 def _PSEUDO_B : sme_load_pseudo; 597 def _PSEUDO_H : sme_load_pseudo; 598 def _PSEUDO_S : sme_load_pseudo; 599 def _PSEUDO_D : sme_load_pseudo; 600 def _PSEUDO_Q : sme_load_pseudo; 601 602 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_B), 603 !if(is_col, int_aarch64_sme_ld1b_vert, 604 int_aarch64_sme_ld1b_horiz), 605 sme_elm_idx0_0, timm32_0_15, am_sve_regreg_lsl0, 606 tileslice8>; 607 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 608 !if(is_col, int_aarch64_sme_ld1h_vert, 609 int_aarch64_sme_ld1h_horiz), 610 timm32_0_1, timm32_0_7, am_sve_regreg_lsl1, 611 tileslice16>; 612 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_S), 613 !if(is_col, int_aarch64_sme_ld1w_vert, 614 int_aarch64_sme_ld1w_horiz), 615 timm32_0_3, timm32_0_3, am_sve_regreg_lsl2, 616 tileslice32>; 617 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_D), 618 !if(is_col, int_aarch64_sme_ld1d_vert, 619 int_aarch64_sme_ld1d_horiz), 620 timm32_0_7, timm32_0_1, am_sve_regreg_lsl3, 621 tileslice64>; 622 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 623 !if(is_col, int_aarch64_sme_ld1q_vert, 624 int_aarch64_sme_ld1q_horiz), 625 timm32_0_15, sme_elm_idx0_0, am_sve_regreg_lsl4, 626 tileslice128>; 627} 628 629multiclass sme_mem_ld_ss<string mnemonic> { 630 defm _H : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b0>; 631 defm _V : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b1>; 632} 633 634//===----------------------------------------------------------------------===// 635// SME Contiguous Stores 636//===----------------------------------------------------------------------===// 637 638class sme_mem_st_ss_base<bit Q, bit V, bits<2> msz, dag ins, 639 string mnemonic, string argstr> 640 : I<(outs), ins, mnemonic, argstr, "", []>, Sched<[]> { 641 bits<5> Rm; 642 bits<2> Rv; 643 bits<3> Pg; 644 bits<5> Rn; 645 let Inst{31-25} = 0b1110000; 646 let Inst{24} = Q; 647 let Inst{23-22} = msz; 648 let Inst{21} = 0b1; 649 let Inst{20-16} = Rm; 650 let Inst{15} = V; 651 let Inst{14-13} = Rv; 652 let Inst{12-10} = Pg; 653 let Inst{9-5} = Rn; 654 let Inst{4} = 0b0; 655 656 let mayStore = 1; 657 let hasSideEffects = 1; 658} 659 660class sme_mem_st_ss_inst<bit Q, bits<2> msz, string mnemonic, 661 MatrixTileVectorOperand tile_ty, bit is_col, 662 Operand imm_ty, RegisterOperand gpr_ty> 663 : sme_mem_st_ss_base< 664 Q, is_col, msz, 665 (ins tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, 666 GPR64sp:$Rn, gpr_ty:$Rm), 667 mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg, [$Rn, $Rm]">; 668 669multiclass sme_mem_st_ss_aliases<string inst, bit is_col> { 670 defm NAME : sme_mem_ss_aliases<"st1", inst, is_col>; 671} 672 673multiclass sme_mem_st_ss_patterns<Instruction Inst, SDPatternOperator Store, 674 Operand offset_ty, 675 ComplexPattern imm2tile, 676 ComplexPattern addr, 677 ComplexPattern tileslice> { 678 // base, tileslice 679 def : Pat<(Store PPR3bAny:$pg, GPR64sp:$base, (imm2tile untyped:$tile), 680 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), 681 (Inst $tile, $idx, $imm, $pg, $base, XZR)>; 682 683 // reg + reg, tileslice 684 let AddedComplexity = 1 in { 685 def : Pat<(Store PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset), 686 (imm2tile untyped:$tile), 687 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), 688 (Inst $tile, $idx, $imm, $pg, $base, $offset)>; 689 } 690} 691 692multiclass sme_mem_st_v_ss<string mnemonic, bit is_col> { 693 def _B : sme_mem_st_ss_inst<0b0, 0b00, mnemonic # "b", 694 !if(is_col, TileVectorOpV8, TileVectorOpH8), 695 is_col, sme_elm_idx0_15, GPR64shifted8> { 696 bits<4> imm; 697 let Inst{3-0} = imm; 698 } 699 def _H : sme_mem_st_ss_inst<0b0, 0b01, mnemonic # "h", 700 !if(is_col, TileVectorOpV16, TileVectorOpH16), 701 is_col, sme_elm_idx0_7, GPR64shifted16> { 702 bits<1> ZAt; 703 bits<3> imm; 704 let Inst{3} = ZAt; 705 let Inst{2-0} = imm; 706 } 707 def _S : sme_mem_st_ss_inst<0b0, 0b10, mnemonic # "w", 708 !if(is_col, TileVectorOpV32, TileVectorOpH32), 709 is_col, sme_elm_idx0_3, GPR64shifted32> { 710 bits<2> ZAt; 711 bits<2> imm; 712 let Inst{3-2} = ZAt; 713 let Inst{1-0} = imm; 714 } 715 def _D : sme_mem_st_ss_inst<0b0, 0b11, mnemonic # "d", 716 !if(is_col, TileVectorOpV64, TileVectorOpH64), 717 is_col, sme_elm_idx0_1, GPR64shifted64> { 718 bits<3> ZAt; 719 bits<1> imm; 720 let Inst{3-1} = ZAt; 721 let Inst{0} = imm; 722 } 723 def _Q : sme_mem_st_ss_inst<0b1, 0b11, mnemonic # "q", 724 !if(is_col, TileVectorOpV128, TileVectorOpH128), 725 is_col, sme_elm_idx0_0, GPR64shifted128> { 726 bits<4> ZAt; 727 let Inst{3-0} = ZAt; 728 } 729 730 defm : sme_mem_st_ss_aliases<NAME, is_col>; 731 732 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _B), 733 !if(is_col, int_aarch64_sme_st1b_vert, 734 int_aarch64_sme_st1b_horiz), 735 timm32_0_15, imm_to_tile8, am_sve_regreg_lsl0, 736 tileslice8>; 737 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _H), 738 !if(is_col, int_aarch64_sme_st1h_vert, 739 int_aarch64_sme_st1h_horiz), 740 timm32_0_7, imm_to_tile16, am_sve_regreg_lsl1, 741 tileslice16>; 742 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _S), 743 !if(is_col, int_aarch64_sme_st1w_vert, 744 int_aarch64_sme_st1w_horiz), 745 timm32_0_3, imm_to_tile32, am_sve_regreg_lsl2, 746 tileslice32>; 747 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _D), 748 !if(is_col, int_aarch64_sme_st1d_vert, 749 int_aarch64_sme_st1d_horiz), 750 timm32_0_1, imm_to_tile64, am_sve_regreg_lsl3, 751 tileslice64>; 752 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _Q), 753 !if(is_col, int_aarch64_sme_st1q_vert, 754 int_aarch64_sme_st1q_horiz), 755 sme_elm_idx0_0, imm_to_tile128, 756 am_sve_regreg_lsl4, tileslice128>; 757} 758 759multiclass sme_mem_st_ss<string mnemonic> { 760 defm _H : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b0>; 761 defm _V : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b1>; 762} 763 764//===----------------------------------------------------------------------===// 765// SME Save and Restore Array 766//===----------------------------------------------------------------------===// 767 768class sme_spill_fill_base<bit isStore, dag outs, dag ins, string opcodestr> 769 : I<outs, ins, opcodestr, "\t$ZAt[$Rv, $imm4], [$Rn, $offset, mul vl]", "", 770 []>, 771 Sched<[]> { 772 bits<2> Rv; 773 bits<5> Rn; 774 bits<4> imm4; 775 let Inst{31-22} = 0b1110000100; 776 let Inst{21} = isStore; 777 let Inst{20-15} = 0b000000; 778 let Inst{14-13} = Rv; 779 let Inst{12-10} = 0b000; 780 let Inst{9-5} = Rn; 781 let Inst{4} = 0b0; 782 let Inst{3-0} = imm4; 783} 784 785let mayStore = 1 in 786class sme_spill_inst<string opcodestr> 787 : sme_spill_fill_base<0b1, (outs), 788 (ins MatrixOp:$ZAt, MatrixIndexGPR32Op12_15:$Rv, 789 sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 790 imm32_0_15:$offset), 791 opcodestr>; 792let mayLoad = 1 in 793class sme_fill_inst<string opcodestr> 794 : sme_spill_fill_base<0b0, (outs MatrixOp:$ZAt), 795 (ins MatrixIndexGPR32Op12_15:$Rv, 796 sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 797 imm32_0_15:$offset), 798 opcodestr>; 799multiclass sme_spill<string opcodestr> { 800 def NAME : sme_spill_inst<opcodestr>; 801 def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]", 802 (!cast<Instruction>(NAME) MatrixOp:$ZAt, 803 MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>; 804 805 def : Pat<(AArch64SMEStr (i32 MatrixIndexGPR32Op12_15:$slice), (i64 GPR64sp:$base), (i32 sme_elm_idx0_15:$imm)), 806 (!cast<Instruction>(NAME) ZA, MatrixIndexGPR32Op12_15:$slice, sme_elm_idx0_15:$imm, GPR64sp:$base, imm32_0_15:$imm)>; 807} 808 809multiclass sme_fill<string opcodestr> { 810 def NAME : sme_fill_inst<opcodestr>; 811 def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]", 812 (!cast<Instruction>(NAME) MatrixOp:$ZAt, 813 MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>; 814 def NAME # _PSEUDO 815 : Pseudo<(outs), 816 (ins MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_15:$imm4, 817 GPR64sp:$base), []>, 818 Sched<[]> { 819 // Translated to actual instruction in AArch64ISelLowering.cpp 820 let usesCustomInserter = 1; 821 let mayLoad = 1; 822 } 823 def : Pat<(AArch64SMELdr MatrixIndexGPR32Op12_15:$slice, GPR64sp:$base, sme_elm_idx0_15:$imm), 824 (!cast<Instruction>(NAME # _PSEUDO) MatrixIndexGPR32Op12_15:$slice, sme_elm_idx0_15:$imm, GPR64sp:$base)>; 825} 826 827//===----------------------------------------------------------------------===// 828// Move instructions 829//===----------------------------------------------------------------------===// 830 831class sme_vector_to_tile_base<bit Q, bit V, bits<2> sz, dag outs, dag ins, 832 string mnemonic, string argstr> 833 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { 834 bits<2> Rv; 835 bits<3> Pg; 836 bits<5> Zn; 837 let Inst{31-24} = 0b11000000; 838 let Inst{23-22} = sz; 839 let Inst{21-17} = 0b00000; 840 let Inst{16} = Q; 841 let Inst{15} = V; 842 let Inst{14-13} = Rv; 843 let Inst{12-10} = Pg; 844 let Inst{9-5} = Zn; 845 let Inst{4} = 0b0; 846} 847 848class sme_vector_to_tile_inst<bit Q, bits<2> sz, MatrixTileVectorOperand tile_ty, 849 bit is_col, Operand imm_ty, ZPRRegOp zpr_ty, 850 string mnemonic> 851 : sme_vector_to_tile_base<Q, is_col, sz, (outs tile_ty:$ZAd), 852 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), 853 mnemonic, "\t$ZAd[$Rv, $imm], $Pg/m, $Zn">{ 854 855 let Constraints = "$ZAd = $_ZAd"; 856} 857 858 859multiclass sme_vector_to_tile_aliases<Instruction inst, 860 MatrixTileVectorOperand tile_ty, 861 ZPRRegOp zpr_ty, Operand imm_ty> { 862 def : InstAlias<"mov\t$ZAd[$Rv, $imm], $Pg/m, $Zn", 863 (inst tile_ty:$ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), 1>; 864} 865 866multiclass sme_vector_to_tile_patterns<Instruction inst, ValueType zpr_vt, 867 ValueType ppr_vt, Operand imm_ty, 868 Operand offset_ty, 869 SDPatternOperator op, 870 ComplexPattern tileslice> { 871 def : Pat<(op imm_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, 872 offset_ty:$imm)), 873 (ppr_vt PPR3bAny:$pg), (zpr_vt ZPRAny:$zn)), 874 (inst imm_ty:$tile, $idx, $imm, $pg, $zn)>; 875} 876 877class sme_mova_insert_pseudo<SMEMatrixTypeEnum za_flag> 878 : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx, 879 i32imm:$imm, PPR3bAny:$pg, ZPRAny:$zn), []>, 880 Sched<[]> { 881 // Translated to the actual instructions in AArch64ISelLowering.cpp 882 let SMEMatrixType = za_flag; 883 let usesCustomInserter = 1; 884} 885 886multiclass sme_vector_v_to_tile<string mnemonic, bit is_col> { 887 def _B : sme_vector_to_tile_inst<0b0, 0b00, !if(is_col, TileVectorOpV8, 888 TileVectorOpH8), 889 is_col, sme_elm_idx0_15, ZPR8, mnemonic>, 890 SMEPseudo2Instr<NAME # _B, 1> { 891 bits<4> imm; 892 let Inst{3-0} = imm; 893 } 894 def _H : sme_vector_to_tile_inst<0b0, 0b01, !if(is_col, TileVectorOpV16, 895 TileVectorOpH16), 896 is_col, sme_elm_idx0_7, ZPR16, mnemonic>, 897 SMEPseudo2Instr<NAME # _H, 1> { 898 bits<1> ZAd; 899 bits<3> imm; 900 let Inst{3} = ZAd; 901 let Inst{2-0} = imm; 902 } 903 def _S : sme_vector_to_tile_inst<0b0, 0b10, !if(is_col, TileVectorOpV32, 904 TileVectorOpH32), 905 is_col, sme_elm_idx0_3, ZPR32, mnemonic>, 906 SMEPseudo2Instr<NAME # _S, 1> { 907 bits<2> ZAd; 908 bits<2> imm; 909 let Inst{3-2} = ZAd; 910 let Inst{1-0} = imm; 911 } 912 def _D : sme_vector_to_tile_inst<0b0, 0b11, !if(is_col, TileVectorOpV64, 913 TileVectorOpH64), 914 is_col, sme_elm_idx0_1, ZPR64, mnemonic>, 915 SMEPseudo2Instr<NAME # _D, 1> { 916 bits<3> ZAd; 917 bits<1> imm; 918 let Inst{3-1} = ZAd; 919 let Inst{0} = imm; 920 } 921 def _Q : sme_vector_to_tile_inst<0b1, 0b11, !if(is_col, TileVectorOpV128, 922 TileVectorOpH128), 923 is_col, sme_elm_idx0_0, ZPR128, mnemonic>, 924 SMEPseudo2Instr<NAME # _Q, 1> { 925 bits<4> ZAd; 926 bits<1> imm; 927 let Inst{3-0} = ZAd; 928 } 929 930 // Pseudo instructions for lowering intrinsics, using immediates instead of 931 // tile registers. 932 def _PSEUDO_B : sme_mova_insert_pseudo<SMEMatrixTileB>, SMEPseudo2Instr<NAME # _B, 0>; 933 def _PSEUDO_H : sme_mova_insert_pseudo<SMEMatrixTileH>, SMEPseudo2Instr<NAME # _H, 0>; 934 def _PSEUDO_S : sme_mova_insert_pseudo<SMEMatrixTileS>, SMEPseudo2Instr<NAME # _S, 0>; 935 def _PSEUDO_D : sme_mova_insert_pseudo<SMEMatrixTileD>, SMEPseudo2Instr<NAME # _D, 0>; 936 def _PSEUDO_Q : sme_mova_insert_pseudo<SMEMatrixTileQ>, SMEPseudo2Instr<NAME # _Q, 0>; 937 938 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _B), 939 !if(is_col, TileVectorOpV8, 940 TileVectorOpH8), 941 ZPR8, sme_elm_idx0_15>; 942 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _H), 943 !if(is_col, TileVectorOpV16, 944 TileVectorOpH16), 945 ZPR16, sme_elm_idx0_7>; 946 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _S), 947 !if(is_col, TileVectorOpV32, 948 TileVectorOpH32), 949 ZPR32, sme_elm_idx0_3>; 950 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _D), 951 !if(is_col, TileVectorOpV64, 952 TileVectorOpH64), 953 ZPR64, sme_elm_idx0_1>; 954 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _Q), 955 !if(is_col, TileVectorOpV128, 956 TileVectorOpH128), 957 ZPR128, sme_elm_idx0_0>; 958 959 defvar op = !if(is_col, int_aarch64_sme_write_vert, 960 int_aarch64_sme_write_horiz); 961 962 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_B), 963 nxv16i8, nxv16i1, sme_elm_idx0_0, sme_elm_idx0_15, 964 op, tileslice8>; 965 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 966 nxv8i16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, 967 op, tileslice16>; 968 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 969 nxv8f16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, 970 op, tileslice16>; 971 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 972 nxv8bf16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, 973 op, tileslice16>; 974 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S), 975 nxv4i32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3, 976 op, tileslice32>; 977 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S), 978 nxv4f32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3, 979 op, tileslice32>; 980 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D), 981 nxv2i64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1, 982 op, tileslice64>; 983 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D), 984 nxv2f64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1, 985 op, tileslice64>; 986 987 defvar opq = !if(is_col, int_aarch64_sme_writeq_vert, 988 int_aarch64_sme_writeq_horiz); 989 990 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 991 nxv16i8, nxv16i1, sme_elm_idx0_15, 992 sme_elm_idx0_0, opq, tileslice128>; 993 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 994 nxv8i16, nxv8i1, sme_elm_idx0_15, 995 sme_elm_idx0_0, opq, tileslice128>; 996 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 997 nxv8f16, nxv8i1, sme_elm_idx0_15, 998 sme_elm_idx0_0, opq, tileslice128>; 999 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 1000 nxv8bf16, nxv8i1, sme_elm_idx0_15, 1001 sme_elm_idx0_0, opq, tileslice128>; 1002 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 1003 nxv4i32, nxv4i1, sme_elm_idx0_15, 1004 sme_elm_idx0_0, opq, tileslice128>; 1005 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 1006 nxv4f32, nxv4i1, sme_elm_idx0_15, 1007 sme_elm_idx0_0, opq, tileslice128>; 1008 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 1009 nxv2i64, nxv2i1, sme_elm_idx0_15, 1010 sme_elm_idx0_0, opq, tileslice128>; 1011 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 1012 nxv2f64, nxv2i1, sme_elm_idx0_15, 1013 sme_elm_idx0_0, opq, tileslice128>; 1014} 1015 1016multiclass sme_vector_to_tile<string mnemonic> { 1017 defm _H : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b0>; 1018 defm _V : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b1>; 1019} 1020 1021class sme_tile_to_vector_base<bit Q, bit V, bits<2> sz, dag outs, dag ins, 1022 string mnemonic, string argstr> 1023 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { 1024 bits<2> Rv; 1025 bits<3> Pg; 1026 bits<5> Zd; 1027 let Inst{31-24} = 0b11000000; 1028 let Inst{23-22} = sz; 1029 let Inst{21-17} = 0b00001; 1030 let Inst{16} = Q; 1031 let Inst{15} = V; 1032 let Inst{14-13} = Rv; 1033 let Inst{12-10} = Pg; 1034 let Inst{9} = 0b0; 1035 let Inst{4-0} = Zd; 1036} 1037 1038class sme_tile_to_vector_inst<bit Q, bits<2> sz, ZPRRegOp zpr_ty, 1039 MatrixTileVectorOperand tile_ty, 1040 bit is_col, Operand imm_ty, string mnemonic> 1041 : sme_tile_to_vector_base<Q, is_col, sz, (outs zpr_ty:$Zd), 1042 (ins zpr_ty:$_Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1043 mnemonic, "\t$Zd, $Pg/m, $ZAn[$Rv, $imm]"> { 1044 1045 let Constraints = "$Zd = $_Zd"; 1046} 1047 1048multiclass sme_tile_to_vector_aliases<Instruction inst, ZPRRegOp zpr_ty, 1049 MatrixTileVectorOperand tile_ty, 1050 Operand imm_ty > { 1051 def : InstAlias<"mov\t$Zd, $Pg/m, $ZAn[$Rv, $imm]", 1052 (inst zpr_ty:$Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1>; 1053} 1054 1055multiclass sme_tile_to_vector_patterns<Instruction inst, ValueType zpr_vt, 1056 ValueType ppr_vt, Operand offset_ty, 1057 ComplexPattern imm2tile, 1058 ComplexPattern tileslice, 1059 SDPatternOperator op> { 1060 def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg), 1061 (imm2tile untyped:$tile), MatrixIndexGPR32Op12_15:$idx)), 1062 (inst $passthru, $pg, $tile, $idx, 0)>; 1063 let AddedComplexity = 1 in { 1064 def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg), 1065 (imm2tile untyped:$tile), 1066 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, 1067 offset_ty:$imm)))), 1068 (inst $passthru, $pg, $tile, $idx, $imm)>; 1069 } 1070} 1071 1072multiclass sme_tile_to_vector_v<string mnemonic, bit is_col> { 1073 def _B : sme_tile_to_vector_inst<0b0, 0b00, ZPR8, !if(is_col, TileVectorOpV8, 1074 TileVectorOpH8), 1075 is_col, sme_elm_idx0_15, mnemonic> { 1076 bits<4> imm; 1077 let Inst{8-5} = imm; 1078 } 1079 def _H : sme_tile_to_vector_inst<0b0, 0b01, ZPR16, !if(is_col, TileVectorOpV16, 1080 TileVectorOpH16), 1081 is_col, sme_elm_idx0_7, mnemonic> { 1082 bits<1> ZAn; 1083 bits<3> imm; 1084 let Inst{8} = ZAn; 1085 let Inst{7-5} = imm; 1086 } 1087 def _S : sme_tile_to_vector_inst<0b0, 0b10, ZPR32, !if(is_col, TileVectorOpV32, 1088 TileVectorOpH32), 1089 is_col, sme_elm_idx0_3, mnemonic> { 1090 bits<2> ZAn; 1091 bits<2> imm; 1092 let Inst{8-7} = ZAn; 1093 let Inst{6-5} = imm; 1094 } 1095 def _D : sme_tile_to_vector_inst<0b0, 0b11, ZPR64, !if(is_col, TileVectorOpV64, 1096 TileVectorOpH64), 1097 is_col, sme_elm_idx0_1, mnemonic> { 1098 bits<3> ZAn; 1099 bits<1> imm; 1100 let Inst{8-6} = ZAn; 1101 let Inst{5} = imm; 1102 } 1103 def _Q : sme_tile_to_vector_inst<0b1, 0b11, ZPR128, !if(is_col, TileVectorOpV128, 1104 TileVectorOpH128), 1105 is_col, sme_elm_idx0_0, mnemonic> { 1106 bits<4> ZAn; 1107 let Inst{8-5} = ZAn; 1108 } 1109 1110 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _B), ZPR8, 1111 !if(is_col, TileVectorOpV8, 1112 TileVectorOpH8), sme_elm_idx0_15>; 1113 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _H), ZPR16, 1114 !if(is_col, TileVectorOpV16, 1115 TileVectorOpH16), sme_elm_idx0_7>; 1116 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _S), ZPR32, 1117 !if(is_col, TileVectorOpV32, 1118 TileVectorOpH32), sme_elm_idx0_3>; 1119 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _D), ZPR64, 1120 !if(is_col, TileVectorOpV64, 1121 TileVectorOpH64), sme_elm_idx0_1>; 1122 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _Q), ZPR128, 1123 !if(is_col, TileVectorOpV128, 1124 TileVectorOpH128), sme_elm_idx0_0>; 1125 1126 defvar op = !if(is_col, int_aarch64_sme_read_vert, 1127 int_aarch64_sme_read_horiz); 1128 1129 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _B), 1130 nxv16i8, nxv16i1, sme_elm_idx0_15, 1131 imm_to_tile8, tileslice8, op>; 1132 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), 1133 nxv8i16, nxv8i1, sme_elm_idx0_7, 1134 imm_to_tile16, tileslice16, op>; 1135 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), 1136 nxv8f16, nxv8i1, sme_elm_idx0_7, 1137 imm_to_tile16, tileslice16, op>; 1138 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), 1139 nxv8bf16, nxv8i1, sme_elm_idx0_7, 1140 imm_to_tile16, tileslice16, op>; 1141 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S), 1142 nxv4i32, nxv4i1, sme_elm_idx0_3, 1143 imm_to_tile32, tileslice32, op>; 1144 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S), 1145 nxv4f32, nxv4i1, sme_elm_idx0_3, 1146 imm_to_tile32, tileslice32, op>; 1147 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D), 1148 nxv2i64, nxv2i1, sme_elm_idx0_1, 1149 imm_to_tile64, tileslice64, op>; 1150 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D), 1151 nxv2f64, nxv2i1, sme_elm_idx0_1, 1152 imm_to_tile64, tileslice64, op>; 1153 1154 defvar opq = !if(is_col, int_aarch64_sme_readq_vert, 1155 int_aarch64_sme_readq_horiz); 1156 1157 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1158 nxv16i8, nxv16i1, sme_elm_idx0_0, 1159 imm_to_tile128, tileslice128, opq>; 1160 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1161 nxv8i16, nxv8i1, sme_elm_idx0_0, 1162 imm_to_tile128, tileslice128, opq>; 1163 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1164 nxv8f16, nxv8i1, sme_elm_idx0_0, 1165 imm_to_tile128, tileslice128, opq>; 1166 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1167 nxv8bf16, nxv8i1, sme_elm_idx0_0, 1168 imm_to_tile128, tileslice128, opq>; 1169 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1170 nxv4i32, nxv4i1, sme_elm_idx0_0, 1171 imm_to_tile128, tileslice128, opq>; 1172 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1173 nxv4f32, nxv4i1, sme_elm_idx0_0, 1174 imm_to_tile128, tileslice128, opq>; 1175 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1176 nxv2i64, nxv2i1, sme_elm_idx0_0, 1177 imm_to_tile128, tileslice128, opq>; 1178 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1179 nxv2f64, nxv2i1, sme_elm_idx0_0, 1180 imm_to_tile128, tileslice128, opq>; 1181} 1182 1183multiclass sme_tile_to_vector<string mnemonic> { 1184 defm _H : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b0>; 1185 defm _V : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b1>; 1186} 1187 1188//===----------------------------------------------------------------------===// 1189// SME Zero 1190//===----------------------------------------------------------------------===// 1191 1192// NOTE: This definition isn't really correct because there are outputs, i.e. 1193// the tile registers being zeroed. We fix this up in a custom inserter that 1194// marks the appropriate registers as being implicitly defined. 1195class sme_zero_inst<string mnemonic> 1196 : I<(outs), (ins MatrixTileList:$imm), 1197 mnemonic, "\t$imm", "", []>, Sched<[]> { 1198 bits<8> imm; 1199 let Inst{31-8} = 0b110000000000100000000000; 1200 let Inst{7-0} = imm; 1201} 1202 1203multiclass sme_zero<string mnemonic> { 1204 def NAME : sme_zero_inst<mnemonic>; 1205 1206 def : InstAlias<"zero\t\\{za\\}", (!cast<Instruction>(NAME) 0b11111111), 1>; 1207 def : InstAlias<"zero\t\\{za0.h\\}", (!cast<Instruction>(NAME) 0b01010101), 1>; 1208 def : InstAlias<"zero\t\\{za1.h\\}", (!cast<Instruction>(NAME) 0b10101010), 1>; 1209 def : InstAlias<"zero\t\\{za0.s\\}", (!cast<Instruction>(NAME) 0b00010001), 1>; 1210 def : InstAlias<"zero\t\\{za1.s\\}", (!cast<Instruction>(NAME) 0b00100010), 1>; 1211 def : InstAlias<"zero\t\\{za2.s\\}", (!cast<Instruction>(NAME) 0b01000100), 1>; 1212 def : InstAlias<"zero\t\\{za3.s\\}", (!cast<Instruction>(NAME) 0b10001000), 1>; 1213 def : InstAlias<"zero\t\\{za0.s,za1.s\\}", (!cast<Instruction>(NAME) 0b00110011), 1>; 1214 def : InstAlias<"zero\t\\{za0.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10011001), 1>; 1215 def : InstAlias<"zero\t\\{za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01100110), 1>; 1216 def : InstAlias<"zero\t\\{za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11001100), 1>; 1217 def : InstAlias<"zero\t\\{za0.s,za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01110111), 1>; 1218 def : InstAlias<"zero\t\\{za0.s,za1.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10111011), 1>; 1219 def : InstAlias<"zero\t\\{za0.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11011101), 1>; 1220 def : InstAlias<"zero\t\\{za1.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11101110), 1>; 1221 1222 def NAME # _PSEUDO : Pseudo<(outs), (ins i32imm:$tilelist), []>, 1223 Sched<[]> { 1224 // Translated to the actual instructions in AArch64ISelLowering.cpp 1225 let usesCustomInserter = 1; 1226 } 1227 1228 def : Pat<(int_aarch64_sme_zero timm32_0_255:$imm), 1229 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_255:$imm)>; 1230} 1231 1232//===----------------------------------------------------------------------===// 1233// SVE2 Instructions 1234//===----------------------------------------------------------------------===// 1235 1236class sve2_int_perm_revd<string asm> 1237 : I<(outs ZPR128:$Zd), (ins ZPR128:$_Zd, PPR3bAny:$Pg, ZPR128:$Zn), 1238 asm, "\t$Zd, $Pg/m, $Zn", "", []>, 1239 Sched<[]> { 1240 bits<5> Zd; 1241 bits<3> Pg; 1242 bits<5> Zn; 1243 let Inst{31-24} = 0b00000101; 1244 let Inst{23-22} = 0b00; // size 1245 let Inst{21-13} = 0b101110100; 1246 let Inst{12-10} = Pg; 1247 let Inst{9-5} = Zn; 1248 let Inst{4-0} = Zd; 1249 1250 let Constraints = "$Zd = $_Zd"; 1251 let DestructiveInstType = DestructiveUnary; 1252 let ElementSize = ZPR128.ElementSize; 1253} 1254 1255multiclass sve2_int_perm_revd<string asm, SDPatternOperator op> { 1256 def NAME : sve2_int_perm_revd<asm>; 1257 1258 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>; 1259 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME)>; 1260 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME)>; 1261 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME)>; 1262} 1263 1264class sve2_clamp<string asm, bits<2> sz, bit U, ZPRRegOp zpr_ty> 1265 : I<(outs zpr_ty:$Zd), (ins zpr_ty:$Zn, zpr_ty:$Zm, zpr_ty:$_Zd), 1266 asm, "\t$Zd, $Zn, $Zm", "", []>, 1267 Sched<[]> { 1268 bits<5> Zm; 1269 bits<5> Zn; 1270 bits<5> Zd; 1271 let Inst{31-24} = 0b01000100; 1272 let Inst{23-22} = sz; 1273 let Inst{21} = 0b0; 1274 let Inst{20-16} = Zm; 1275 let Inst{15-11} = 0b11000; 1276 let Inst{10} = U; 1277 let Inst{9-5} = Zn; 1278 let Inst{4-0} = Zd; 1279 1280 let Constraints = "$Zd = $_Zd"; 1281 let DestructiveInstType = DestructiveOther; 1282 let ElementSize = zpr_ty.ElementSize; 1283} 1284 1285multiclass sve2_clamp<string asm, bit U, SDPatternOperator op> { 1286 def _B : sve2_clamp<asm, 0b00, U, ZPR8>; 1287 def _H : sve2_clamp<asm, 0b01, U, ZPR16>; 1288 def _S : sve2_clamp<asm, 0b10, U, ZPR32>; 1289 def _D : sve2_clamp<asm, 0b11, U, ZPR64>; 1290 1291 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1292 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1293 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1294 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1295} 1296 1297class sve2_int_perm_sel_p<string asm, PPRRegOp ppr_ty, Operand imm_ty> 1298 : I<(outs PPRAny:$Pd), (ins PPRAny:$Pn, ppr_ty:$Pm, 1299 MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1300 asm, "\t$Pd, $Pn, $Pm[$Rv, $imm]", "", []>, 1301 Sched<[]> { 1302 bits<2> Rv; 1303 bits<4> Pn; 1304 bits<4> Pm; 1305 bits<4> Pd; 1306 let Inst{31-24} = 0b00100101; 1307 let Inst{21} = 0b1; 1308 let Inst{17-16} = Rv; 1309 let Inst{15-14} = 0b01; 1310 let Inst{13-10} = Pn; 1311 let Inst{9} = 0b0; 1312 let Inst{8-5} = Pm; 1313 let Inst{4} = 0b0; 1314 let Inst{3-0} = Pd; 1315} 1316 1317multiclass sve2_int_perm_sel_p<string asm, SDPatternOperator op> { 1318 def _B : sve2_int_perm_sel_p<asm, PPR8, sme_elm_idx0_15> { 1319 bits<4> imm; 1320 let Inst{23-22} = imm{3-2}; 1321 let Inst{20-19} = imm{1-0}; 1322 let Inst{18} = 0b1; 1323 } 1324 def _H : sve2_int_perm_sel_p<asm, PPR16, sme_elm_idx0_7> { 1325 bits<3> imm; 1326 let Inst{23-22} = imm{2-1}; 1327 let Inst{20} = imm{0}; 1328 let Inst{19-18} = 0b10; 1329 } 1330 def _S : sve2_int_perm_sel_p<asm, PPR32, sme_elm_idx0_3> { 1331 bits<2> imm; 1332 let Inst{23-22} = imm{1-0}; 1333 let Inst{20-18} = 0b100; 1334 } 1335 def _D : sve2_int_perm_sel_p<asm, PPR64, sme_elm_idx0_1> { 1336 bits<1> imm; 1337 let Inst{23} = imm; 1338 let Inst{22} = 0b1; 1339 let Inst{20-18} = 0b000; 1340 } 1341 1342 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1343 (!cast<Instruction>(NAME # _B) PNRasPPRAny:$Pd, 1344 PNRasPPRAny:$Pn, PPR8:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm), 0>; 1345 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1346 (!cast<Instruction>(NAME # _H) PNRasPPRAny:$Pd, 1347 PNRasPPRAny:$Pn, PPR16:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_7:$imm), 0>; 1348 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1349 (!cast<Instruction>(NAME # _S) PNRasPPRAny:$Pd, 1350 PNRasPPRAny:$Pn, PPR32:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_3:$imm), 0>; 1351 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1352 (!cast<Instruction>(NAME # _D) PNRasPPRAny:$Pd, 1353 PNRasPPRAny:$Pn, PPR64:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_1:$imm), 0>; 1354 1355 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPR8:$Pm), 1356 MatrixIndexGPR32Op12_15:$idx)), 1357 (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, 0)>; 1358 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv8i1 PPR16:$Pm), 1359 MatrixIndexGPR32Op12_15:$idx)), 1360 (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, 0)>; 1361 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv4i1 PPR32:$Pm), 1362 MatrixIndexGPR32Op12_15:$idx)), 1363 (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, 0)>; 1364 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv2i1 PPR64:$Pm), 1365 MatrixIndexGPR32Op12_15:$idx)), 1366 (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, 0)>; 1367 1368 let AddedComplexity = 1 in { 1369 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPR8:$Pm), 1370 (i32 (tileslice8 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_15:$imm)))), 1371 (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, $imm)>; 1372 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv8i1 PPR16:$Pm), 1373 (i32 (tileslice16 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_7:$imm)))), 1374 (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, $imm)>; 1375 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv4i1 PPR32:$Pm), 1376 (i32 (tileslice32 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_3:$imm)))), 1377 (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, $imm)>; 1378 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv2i1 PPR64:$Pm), 1379 (i32 (tileslice64 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_1:$imm)))), 1380 (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, $imm)>; 1381 } 1382} 1383 1384//===----------------------------------------------------------------------===// 1385// SME2 Instructions 1386//===----------------------------------------------------------------------===// 1387 1388//===----------------------------------------------------------------------===// 1389// SME2 single-multi ternary int/fp, two/four registers 1390 1391class sme2_dot_mla_add_sub_array_vg24_single<bits<7> op, 1392 MatrixOperand matrix_ty, 1393 RegisterOperand multi_vector_ty, 1394 ZPRRegOp zpr_ty, 1395 string mnemonic> 1396 : I<(outs matrix_ty:$ZAd), 1397 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, 1398 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 1399 mnemonic,"\t$ZAd[$Rv, $imm3, " # !if(op{5}, "vgx4", "vgx2") # "], $Zn, $Zm", 1400 "", []> , Sched<[]> { 1401 bits<4> Zm; 1402 bits<5> Zn; 1403 bits<2> Rv; 1404 bits<3> imm3; 1405 let Inst{31-23} = 0b110000010; 1406 let Inst{22} = op{6}; //sz 1407 let Inst{21} = 0b1; 1408 let Inst{20} = op{5}; //vgx4 1409 let Inst{19-16} = Zm; 1410 let Inst{15} = 0b0; 1411 let Inst{14-13} = Rv; 1412 let Inst{12-10} = op{4-2}; 1413 let Inst{9-5} = Zn; 1414 let Inst{4-3} = op{1-0}; 1415 let Inst{2-0} = imm3; 1416 let Constraints = "$ZAd = $_ZAd"; 1417} 1418 1419multiclass sme2_dot_mla_add_sub_array_vg24_single<string mnemonic, bits<7> op, 1420 MatrixOperand matrix_ty, 1421 RegisterOperand multi_vector_ty, 1422 ZPRRegOp zpr_ty>{ 1423 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1424 1425 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1426 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 1427} 1428 1429multiclass sme2_dot_mla_add_sub_array_vg2_single<string mnemonic, bits<7> op, 1430 MatrixOperand matrix_ty, 1431 RegisterOperand multi_vector_ty, 1432 ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{ 1433 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1434 1435 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1436 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 1437 1438 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>; 1439 1440 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>; 1441} 1442 1443multiclass sme2_dot_mla_add_sub_array_vg4_single<string mnemonic, bits<7> op, 1444 MatrixOperand matrix_ty, 1445 RegisterOperand multi_vector_ty, 1446 ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{ 1447 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1448 1449 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1450 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 1451 1452 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>; 1453 1454 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>; 1455} 1456 1457//===----------------------------------------------------------------------===// 1458// SME2 multiple vectors ternary INT/FP two and four registers 1459class sme2_dot_mla_add_sub_array_vg2_multi<bits<7> op, 1460 MatrixOperand matrix_ty, 1461 RegisterOperand multi_vector_ty, 1462 string mnemonic> 1463 : I<(outs matrix_ty:$ZAd), 1464 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, 1465 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 1466 mnemonic, "\t$ZAd[$Rv, $imm3, vgx2], $Zn, $Zm", 1467 "", []>, Sched<[]>{ 1468 bits<4> Zm; 1469 bits<4> Zn; 1470 bits<2> Rv; 1471 bits<3> imm3; 1472 let Inst{31-23} = 0b110000011; 1473 let Inst{22} = op{6}; //sz 1474 let Inst{21} = 0b1; 1475 let Inst{20-17} = Zm; 1476 let Inst{16-15} = 0b00; 1477 let Inst{14-13} = Rv; 1478 let Inst{12-10} = op{5-3}; 1479 let Inst{9-6} = Zn; 1480 let Inst{5-3} = op{2-0}; 1481 let Inst{2-0} = imm3; 1482 let Constraints = "$ZAd = $_ZAd"; 1483} 1484 1485multiclass sme2_dot_mla_add_sub_array_vg2_multi<string mnemonic, bits<7> op, 1486 MatrixOperand matrix_ty, 1487 RegisterOperand multi_vector_ty, ValueType zpr_ty, 1488 SDPatternOperator intrinsic> { 1489 def NAME : sme2_dot_mla_add_sub_array_vg2_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1490 1491 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>; 1492 1493 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>; 1494 1495 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1496 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; 1497} 1498 1499class sme2_dot_mla_add_sub_array_vg4_multi<bits<7> op, 1500 MatrixOperand matrix_ty, 1501 RegisterOperand multi_vector_ty, 1502 string mnemonic> 1503 : I<(outs matrix_ty:$ZAd), 1504 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, 1505 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 1506 mnemonic, "\t$ZAd[$Rv, $imm3, vgx4], $Zn, $Zm", 1507 "", []>, Sched<[]>{ 1508 bits<3> Zm; 1509 bits<3> Zn; 1510 bits<2> Rv; 1511 bits<3> imm3; 1512 let Inst{31-23} = 0b110000011; 1513 let Inst{22} = op{6}; //sz 1514 let Inst{21} = 0b1; 1515 let Inst{20-18} = Zm; 1516 let Inst{17-15} = 0b010; 1517 let Inst{14-13} = Rv; 1518 let Inst{12-10} = op{5-3}; 1519 let Inst{9-7} = Zn; 1520 let Inst{6} = 0b0; 1521 let Inst{5-3} = op{2-0}; 1522 let Inst{2-0} = imm3; 1523 let Constraints = "$ZAd = $_ZAd"; 1524} 1525 1526multiclass sme2_dot_mla_add_sub_array_vg4_multi<string mnemonic, bits<7> op, 1527 MatrixOperand matrix_ty, 1528 RegisterOperand multi_vector_ty, 1529 ValueType zpr_ty, SDPatternOperator intrinsic>{ 1530 def NAME : sme2_dot_mla_add_sub_array_vg4_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1531 1532 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>; 1533 1534 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>; 1535 1536 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1537 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; 1538} 1539 1540//===----------------------------------------------------------------------===// 1541// SME2 multiple vectors binary two or four registers 1542 1543class sme2_multivec_accum_add_sub<string mnemonic, bit sz, bit vg4, bits<3> op, 1544 MatrixOperand matrix_ty, 1545 RegisterOperand vector_ty> 1546 : I<(outs matrix_ty:$ZAdn), 1547 (ins matrix_ty:$_ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 1548 mnemonic, "\t$ZAdn[$Rv, $imm3, " # !if(vg4, "vgx4", "vgx2") # "], $Zm", 1549 "", []>, Sched<[]> { 1550 bits<2> Rv; 1551 bits<3> imm3; 1552 let Inst{31-23} = 0b110000011; 1553 let Inst{22} = sz; 1554 let Inst{21-19} = 0b100; 1555 let Inst{18} = op{2}; 1556 let Inst{17} = 0b0; 1557 let Inst{16} = vg4; 1558 let Inst{15} = 0b0; 1559 let Inst{14-13} = Rv; 1560 let Inst{12-10} = 0b111; 1561 let Inst{5} = 0b0; 1562 let Inst{4-3} = op{1-0}; 1563 let Inst{2-0} = imm3; 1564 1565 let Constraints = "$ZAdn = $_ZAdn"; 1566} 1567 1568class sme2_multivec_accum_add_sub_vg2<string mnemonic, bit sz, bits<3> op, 1569 MatrixOperand matrix_ty, 1570 RegisterOperand vector_ty> 1571 : sme2_multivec_accum_add_sub<mnemonic, sz, 0b0, op, matrix_ty, vector_ty> { 1572 bits<4> Zm; 1573 let Inst{9-6} = Zm; 1574} 1575 1576 1577multiclass sme2_multivec_accum_add_sub_vg2<string mnemonic, bits<4> op, 1578 MatrixOperand matrix_ty, 1579 RegisterOperand vector_ty, 1580 ValueType vty, 1581 SDPatternOperator intrinsic> { 1582 def NAME : sme2_multivec_accum_add_sub_vg2<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>, 1583 SMEPseudo2Instr<NAME, 1>; 1584 def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm", 1585 (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>; 1586 1587 def _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, vector_ty, SMEMatrixArray>; 1588 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, vty, sme_elm_idx0_7, tileslice16>; 1589} 1590 1591class sme2_multivec_accum_add_sub_vg4<string mnemonic, bit sz, bits<3> op, 1592 MatrixOperand matrix_ty, 1593 RegisterOperand vector_ty> 1594 : sme2_multivec_accum_add_sub<mnemonic, sz, 0b1, op, matrix_ty, vector_ty> { 1595 bits<3> Zm; 1596 let Inst{9-7} = Zm; 1597 let Inst{6} = 0b0; 1598} 1599 1600multiclass sme2_multivec_accum_add_sub_vg4<string mnemonic, bits<4> op, 1601 MatrixOperand matrix_ty, 1602 RegisterOperand vector_ty, 1603 ValueType vty, 1604 SDPatternOperator intrinsic> { 1605 def NAME : sme2_multivec_accum_add_sub_vg4<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>, 1606 SMEPseudo2Instr<NAME, 1>; 1607 def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm", 1608 (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>; 1609 1610 def _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, vector_ty, SMEMatrixArray>; 1611 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, vty, sme_elm_idx0_7, tileslice16>; 1612} 1613 1614//===----------------------------------------------------------------------===// 1615// SME2 Multi-vector - Multiple and Single SVE Destructive 1616// Two and Four registers 1617 1618class sme2_sve_destructive_vector_vg2_single<bits<2> sz, bits<7> op, 1619 RegisterOperand vector_ty, 1620 ZPRRegOp zpr_ty, 1621 string mnemonic> 1622 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm), 1623 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1624 "", []>, Sched<[]> { 1625 bits<4> Zm; 1626 bits<4> Zdn; 1627 let Inst{31-24} = 0b11000001; 1628 let Inst{23-22} = sz; 1629 let Inst{21-20} = 0b10; 1630 let Inst{19-16} = Zm; 1631 let Inst{15-11} = 0b10100; 1632 let Inst{10-5} = op{6-1}; 1633 let Inst{4-1} = Zdn; 1634 let Inst{0} = op{0}; 1635 1636 let Constraints = "$Zdn = $_Zdn"; 1637} 1638 1639multiclass sme2_fp_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> { 1640 def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>; 1641 def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>; 1642 def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>; 1643} 1644 1645multiclass sme2_int_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> { 1646 def _B : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_b_mul_r, ZPR4b8, mnemonic>; 1647 def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>; 1648 def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>; 1649 def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>; 1650} 1651 1652// SME2.1 fmax/fmin instructions. 1653multiclass sme2p1_bf_max_min_vector_vg2_single<string mnemonic, bits<7>op> { 1654 def _H : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_h_mul_r, 1655 ZPR4b16, mnemonic>; 1656} 1657 1658class sme2_sve_destructive_vector_vg4_single<bits<2> sz, bits<7> op, 1659 RegisterOperand vector_ty, 1660 ZPRRegOp zpr_ty, 1661 string mnemonic> 1662 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm), 1663 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1664 "", []>, Sched<[]> { 1665 bits<4> Zm; 1666 bits<3> Zdn; 1667 let Inst{31-24} = 0b11000001; 1668 let Inst{23-22} = sz; 1669 let Inst{21-20} = 0b10; 1670 let Inst{19-16} = Zm; 1671 let Inst{15-11} = 0b10101; 1672 let Inst{10-5} = op{6-1}; 1673 let Inst{4-2} = Zdn; 1674 let Inst{1} = 0b0; 1675 let Inst{0} = op{0}; 1676 1677 let Constraints = "$Zdn = $_Zdn"; 1678} 1679 1680multiclass sme2_fp_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> { 1681 def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>; 1682 def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>; 1683 def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>; 1684} 1685 1686multiclass sme2_int_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> { 1687 def _B : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_b_mul_r, ZPR4b8, mnemonic>; 1688 def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>; 1689 def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>; 1690 def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>; 1691} 1692 1693// SME2.1 fmax/fmin instructions. 1694multiclass sme2p1_bf_max_min_vector_vg4_single<string mnemonic, bits<7>op> { 1695 def _H : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_h_mul_r, 1696 ZPR4b16, mnemonic>; 1697} 1698 1699class sme2_sve_destructive_vector_vg2_multi<bits<2> sz, bits<7> op, 1700 RegisterOperand vector_ty, 1701 string mnemonic> 1702 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm), 1703 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1704 "", []>, Sched<[]> { 1705 bits<4> Zm; 1706 bits<4> Zdn; 1707 let Inst{31-24} = 0b11000001; 1708 let Inst{23-22} = sz; 1709 let Inst{21} = 0b1; 1710 let Inst{20-17} = Zm; 1711 let Inst{16-11} = 0b010110; 1712 let Inst{10-5} = op{6-1}; 1713 let Inst{4-1} = Zdn; 1714 let Inst{0} = op{0}; 1715 1716 let Constraints = "$Zdn = $_Zdn"; 1717} 1718 1719multiclass sme2_fp_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> { 1720 def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>; 1721 def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>; 1722 def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>; 1723} 1724 1725multiclass sme2_int_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> { 1726 def _B : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_b_mul_r, mnemonic>; 1727 def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>; 1728 def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>; 1729 def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>; 1730} 1731 1732// SME2.1 fmax/fmin instructions. 1733multiclass sme2p1_bf_max_min_vector_vg2_multi<string mnemonic, bits<7>op> { 1734 def _H : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_h_mul_r, 1735 mnemonic>; 1736} 1737 1738class sme2_sve_destructive_vector_vg4_multi<bits<2> sz, bits<7> op, 1739 RegisterOperand vector_ty, 1740 string mnemonic> 1741 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm), 1742 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1743 "", []>, Sched<[]> { 1744 bits<3> Zm; 1745 bits<3> Zdn; 1746 let Inst{31-24} = 0b11000001; 1747 let Inst{23-22} = sz; 1748 let Inst{21} = 0b1; 1749 let Inst{20-18} = Zm; 1750 let Inst{17-11} = 0b0010111; 1751 let Inst{10-5} = op{6-1}; 1752 let Inst{4-2} = Zdn; 1753 let Inst{1} = 0b0; 1754 let Inst{0} = op{0}; 1755 1756 let Constraints = "$Zdn = $_Zdn"; 1757} 1758 1759multiclass sme2_fp_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> { 1760 def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>; 1761 def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>; 1762 def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>; 1763} 1764 1765multiclass sme2_int_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> { 1766 def _B : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_b_mul_r, mnemonic>; 1767 def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>; 1768 def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>; 1769 def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>; 1770} 1771 1772// SME2.1 fmax/fmin instructions. 1773multiclass sme2p1_bf_max_min_vector_vg4_multi<string mnemonic, bits<7>op> { 1774 def _H : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_h_mul_r, 1775 mnemonic>; 1776} 1777 1778//===----------------------------------------------------------------------===// 1779// SME2 Multi-vector - Index/Single/Multi Array Vectors FMA sources 1780 1781class sme2_mla_long_array_index_base<bits<2> op0, bits<2> op, Operand index_ty, 1782 RegisterOperand multi_vector_ty, 1783 string mnemonic, string vg_acronym=""> 1784 : I<(outs MatrixOp32:$ZAda), 1785 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm, multi_vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 1786 mnemonic, "\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm$i3", 1787 "", []>, Sched<[]> { 1788 bits<4> Zm; 1789 bits<2> Rv; 1790 let Inst{31-24} = 0b11000001; 1791 let Inst{23-22} = op0; 1792 let Inst{21} = 0b0; 1793 let Inst{20} = !if(!eq(vg_acronym, ""), 0, 1); 1794 let Inst{19-16} = Zm; 1795 let Inst{14-13} = Rv; 1796 let Inst{12} = 0b1; 1797 let Inst{4-3} = op; 1798 1799 let Constraints = "$ZAda = $_ZAda"; 1800} 1801 1802multiclass sme2_mla_long_array_index<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1803 def _HtoS : sme2_mla_long_array_index_base<op0, op, uimm3s2range, ZPR16, 1804 mnemonic>, SMEPseudo2Instr<NAME # _HtoS, 1> { 1805 bits<3> i3; 1806 bits<5> Zn; 1807 bits<3> imm; 1808 let Inst{15} = i3{2}; 1809 let Inst{11-10} = i3{1-0}; 1810 let Inst{9-5} = Zn; 1811 let Inst{2-0} = imm; 1812 } 1813 1814 def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm3s2range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1815 1816 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange3s2>; 1817} 1818 1819class sme2_mla_long_array_vg2_index<string mnemonic, bits<2> op0, bits<2> op> 1820 : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZ_h_mul_r, 1821 mnemonic, "vgx2"> { 1822 bits<3> i3; 1823 bits<4> Zn; 1824 bits<2> imm; 1825 let Inst{15} = 0b0; 1826 let Inst{11-10} = i3{2-1}; 1827 let Inst{9-6} = Zn; 1828 let Inst{5} = 0b0; 1829 let Inst{2} = i3{0}; 1830 let Inst{1-0} = imm; 1831} 1832 1833multiclass sme2_fp_mla_long_array_vg2_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1834 def _HtoS : sme2_mla_long_array_vg2_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _HtoS, 1>; 1835 1836 def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1837 1838 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>; 1839 1840 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1841 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1842} 1843 1844multiclass sme2_int_mla_long_array_vg2_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1845 def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; 1846 1847 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1848 1849 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>; 1850 1851 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1852 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1853} 1854 1855class sme2_mla_long_array_vg4_index<string mnemonic, bits<2> op0, bits<2> op> 1856 : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZZZ_h_mul_r, 1857 mnemonic, "vgx4"> { 1858 bits<3> i3; 1859 bits<3> Zn; 1860 bits<2> imm; 1861 let Inst{15} = 0b1; 1862 let Inst{11-10} = i3{2-1}; 1863 let Inst{9-7} = Zn; 1864 let Inst{6-5} = 0b00; 1865 let Inst{2} = i3{0}; 1866 let Inst{1-0} = imm; 1867} 1868 1869multiclass sme2_fp_mla_long_array_vg4_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1870 def _HtoS : sme2_mla_long_array_vg4_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _HtoS, 1>; 1871 1872 def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1873 1874 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>; 1875 1876 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1877 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1878} 1879 1880multiclass sme2_int_mla_long_array_vg4_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1881 def _HtoS : sme2_mla_long_array_vg4_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _HtoS, 1>; 1882 1883 def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1884 1885 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>; 1886 1887 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1888 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1889} 1890 1891class sme2_mla_long_array<bits<2>op0, bits<2> op, 1892 MatrixOperand matrix_ty, 1893 Operand index_ty, 1894 RegisterOperand first_vector_ty, 1895 RegisterOperand second_vector_ty, 1896 string mnemonic, string vg_acronym=""> 1897 : I<(outs matrix_ty:$ZAda), 1898 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, 1899 index_ty:$imm, first_vector_ty:$Zn, second_vector_ty:$Zm), 1900 mnemonic,"\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm", 1901 "", []> , Sched<[]> { 1902 bits<2> Rv; 1903 let Inst{31-24} = 0b11000001; 1904 let Inst{23-22} = op0; 1905 let Inst{21} = 0b1; 1906 let Inst{15} = 0b0; 1907 let Inst{14-13} = Rv; 1908 let Inst{12-11} = 0b01; 1909 let Inst{10} = !if(!eq(vg_acronym, ""), 1, 0); 1910 let Inst{4-3} = op; 1911 1912 let Constraints = "$ZAda = $_ZAda"; 1913} 1914 1915multiclass sme2_mla_long_array_single<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1916 def _HtoS : sme2_mla_long_array<op0, op, MatrixOp32, uimm3s2range, ZPR16, ZPR4b16, 1917 mnemonic> , SMEPseudo2Instr<NAME # _HtoS, 1>{ 1918 bits<4> Zm; 1919 bits<5> Zn; 1920 bits<3> imm; 1921 let Inst{20} = 0b0; 1922 let Inst{19-16} = Zm; 1923 let Inst{9-5} = Zn; 1924 let Inst{2-0} = imm; 1925 } 1926 1927 def _HtoS_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _HtoS, uimm3s2range, ZPR16, ZPR4b16, SMEMatrixArray>; 1928 1929 def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME # _HtoS, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, tileslicerange3s2>; 1930} 1931 1932class sme2_mla_long_array_single_16b<string mnemonic> 1933 : sme2_mla_long_array<0b00, 0b00, MatrixOp16, uimm3s2range, ZPR8, ZPR4b8, mnemonic> { 1934 bits<4> Zm; 1935 bits<5> Zn; 1936 bits<3> imm; 1937 let Inst{20} = 0b1; 1938 let Inst{19-16} = Zm; 1939 let Inst{9-5} = Zn; 1940 let Inst{2-0} = imm; 1941} 1942 1943class sme2_mla_long_array_vg24_single<bits<2> op0, bit vg4, bits<2> op, bit o2, 1944 MatrixOperand matrix_ty, RegisterOperand multi_vector_ty, 1945 ZPRRegOp zpr_ty, string mnemonic, string vg_acronym> 1946 : sme2_mla_long_array<op0, op, matrix_ty, uimm2s2range, multi_vector_ty, zpr_ty, 1947 mnemonic, vg_acronym> { 1948 bits<4> Zm; 1949 bits<5> Zn; 1950 bits<2> imm; 1951 let Inst{20} = vg4; 1952 let Inst{19-16} = Zm; 1953 let Inst{9-5} = Zn; 1954 let Inst{2} = o2; 1955 let Inst{1-0} = imm; 1956} 1957 1958multiclass sme2_fp_mla_long_array_vg2_single<string mnemonic, bits<3> op, MatrixOperand matrix_ty, 1959 RegisterOperand multi_vector_ty, ZPRRegOp vector_ty, 1960 ValueType zpr_ty, SDPatternOperator intrinsic> { 1961 def NAME : sme2_mla_long_array_vg24_single<0b00, 0b0, op{2-1}, op{0}, matrix_ty, multi_vector_ty, 1962 vector_ty, mnemonic, "vgx2">, SMEPseudo2Instr<NAME, 1>; 1963 1964 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm2s2range, multi_vector_ty, 1965 vector_ty, SMEMatrixArray>; 1966 1967 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, uimm2s2range, vector_ty, zpr_ty, 1968 tileslicerange2s2>; 1969 1970 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1971 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, 1972 uimm2s2range:$imm, multi_vector_ty:$Zn, vector_ty:$Zm), 0>; 1973} 1974 1975multiclass sme2_int_mla_long_array_vg2_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1976 def _HtoS : sme2_mla_long_array_vg24_single<0b01, 0b0, op, 0b0, MatrixOp32, ZZ_h, ZPR4b16, mnemonic, 1977 "vgx2">, SMEPseudo2Instr<NAME # _HtoS, 1>; 1978 1979 def _HtoS_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _HtoS, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>; 1980 1981 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>; 1982 1983 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1984 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>; 1985} 1986 1987multiclass sme2_fp_mla_long_array_vg4_single<string mnemonic, bits<3> op, MatrixOperand matrix_ty, 1988 RegisterOperand multi_vector_ty, ZPRRegOp vector_ty, 1989 ValueType zpr_ty, SDPatternOperator intrinsic> { 1990 def NAME : sme2_mla_long_array_vg24_single<0b00, 0b1, op{2-1}, op{0}, matrix_ty, multi_vector_ty, 1991 vector_ty, mnemonic, "vgx4">, 1992 SMEPseudo2Instr<NAME, 1>; 1993 1994 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm2s2range, multi_vector_ty, vector_ty, 1995 SMEMatrixArray>; 1996 1997 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, uimm2s2range, vector_ty, zpr_ty, 1998 tileslicerange2s2>; 1999 2000 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2001 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, 2002 uimm2s2range:$imm, multi_vector_ty:$Zn, vector_ty:$Zm), 0>; 2003} 2004 2005multiclass sme2_int_mla_long_array_vg4_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2006 def _HtoS : sme2_mla_long_array_vg24_single<0b01, 0b1, op, 0b0, MatrixOp32, ZZZZ_h, ZPR4b16, mnemonic, 2007 "vgx4">, SMEPseudo2Instr<NAME # _HtoS, 1>; 2008 2009 def _HtoS_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>; 2010 2011 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>; 2012 2013 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2014 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>; 2015} 2016 2017class sme2_mla_long_array_vg2_multi<string mnemonic, bits<2> op0, bits<3> op, 2018 MatrixOperand matrix_ty, RegisterOperand multi_vector_ty> 2019 : sme2_mla_long_array<op0, op{1-0}, matrix_ty, uimm2s2range, multi_vector_ty, multi_vector_ty, 2020 mnemonic, "vgx2"> { 2021 bits<4> Zm; 2022 bits<4> Zn; 2023 bits<2> imm; 2024 let Inst{20-17} = Zm; 2025 let Inst{16} = 0b0; 2026 let Inst{9-6} = Zn; 2027 let Inst{5} = op{2}; // fp8 2028 let Inst{2} = 0b0; 2029 let Inst{1-0} = imm; 2030} 2031 2032multiclass sme2_fp_mla_long_array_vg2_multi<string mnemonic, bits<3> op, MatrixOperand matrix_ty, 2033 RegisterOperand multi_vector_ty, 2034 ValueType zpr_ty, SDPatternOperator intrinsic> { 2035 2036 def NAME : sme2_mla_long_array_vg2_multi<mnemonic, 0b10, op, matrix_ty, multi_vector_ty>, 2037 SMEPseudo2Instr<NAME, 1>; 2038 2039 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm2s2range, multi_vector_ty, SMEMatrixArray>; 2040 2041 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>; 2042 2043 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2044 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, 2045 uimm2s2range:$imm, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; 2046} 2047 2048multiclass sme2_int_mla_long_array_vg2_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2049 def _HtoS : sme2_mla_long_array_vg2_multi<mnemonic, 0b11, {0b0, op}, MatrixOp32, ZZ_h_mul_r>, 2050 SMEPseudo2Instr<NAME # _HtoS, 1>; 2051 2052 def _HtoS_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _HtoS, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>; 2053 2054 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _HtoS, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>; 2055 2056 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm", 2057 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>; 2058} 2059 2060class sme2_mla_long_array_vg4_multi<string mnemonic, bits<2> op0, bits<3> op, 2061 MatrixOperand matrix_ty, 2062 RegisterOperand multi_vector_ty> 2063 : sme2_mla_long_array<op0, op{1-0}, matrix_ty, uimm2s2range, multi_vector_ty, multi_vector_ty, 2064 mnemonic, "vgx4"> { 2065 bits<3> Zm; 2066 bits<3> Zn; 2067 bits<2> imm; 2068 let Inst{20-18} = Zm; 2069 let Inst{17} = 0b0; 2070 let Inst{16} = 0b1; 2071 let Inst{9-7} = Zn; 2072 let Inst{6} = 0b0; 2073 let Inst{5} = op{2}; //fp8 2074 let Inst{2} = 0b0; 2075 let Inst{1-0} = imm; 2076} 2077 2078multiclass sme2_fp_mla_long_array_vg4_multi<string mnemonic, bits<3> op, MatrixOperand matrix_ty, 2079 RegisterOperand multi_vector_ty, ValueType zpr_ty, 2080 SDPatternOperator intrinsic> { 2081 def NAME : sme2_mla_long_array_vg4_multi<mnemonic, 0b10, op, matrix_ty, multi_vector_ty>, 2082 SMEPseudo2Instr<NAME, 1>; 2083 2084 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm2s2range, multi_vector_ty, SMEMatrixArray>; 2085 2086 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>; 2087 2088 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2089 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, 2090 uimm2s2range:$imm, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; 2091} 2092 2093multiclass sme2_int_mla_long_array_vg4_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2094 def _HtoS : sme2_mla_long_array_vg4_multi<mnemonic, 0b11, {0b0, op}, MatrixOp32, ZZZZ_h_mul_r>, 2095 SMEPseudo2Instr<NAME # _HtoS, 1>; 2096 2097 def _HtoS_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>; 2098 2099 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _HtoS, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>; 2100 2101 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm", 2102 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>; 2103} 2104 2105//===----------------------------------------------------------------------===// 2106class sme2_frint_cvt_vg2_multi<bits<2>sz, bits<5>op, RegisterOperand first_ty, 2107 RegisterOperand second_ty, string mnemonic> 2108 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2109 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2110 bits<4> Zn; 2111 bits<4> Zd; 2112 let Inst{31-24} = 0b11000001; 2113 let Inst{23-22} = sz; 2114 let Inst{21-20} = 0b10; 2115 let Inst{19-16} = op{4-1}; 2116 let Inst{15-10} = 0b111000; 2117 let Inst{9-6} = Zn; 2118 let Inst{5} = op{0}; 2119 let Inst{4-1} = Zd; 2120 let Inst{0} = 0b0; 2121} 2122 2123// SME2 multi-vec FP to int convert two registers 2124// SME2 multi-vec int to FP two registers 2125multiclass sme2_fp_cvt_vg2_multi<string mnemonic, bits<5> op> { 2126 def NAME : sme2_frint_cvt_vg2_multi<0b00, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>; 2127} 2128 2129// SME2 multi-vec FRINT two registers 2130multiclass sme2_frint_vector_vg2_multi<string mnemonic, bits<5> op> { 2131 def _S : sme2_frint_cvt_vg2_multi<0b10, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>; 2132} 2133 2134class sme2_frint_zip_cvt_vg4_multi<bits<2>sz, bits<7>op, RegisterOperand first_ty, 2135 RegisterOperand second_ty, string mnemonic> 2136 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2137 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2138 bits<3> Zn; 2139 bits<3> Zd; 2140 let Inst{31-24} = 0b11000001; 2141 let Inst{23-22} = sz; 2142 let Inst{21-20} = 0b11; 2143 let Inst{19-16} = op{6-3}; 2144 let Inst{15-10} = 0b111000; 2145 let Inst{9-7} = Zn; 2146 let Inst{6-5} = op{2-1}; 2147 let Inst{4-2} = Zd; 2148 let Inst{1} = op{0}; 2149 let Inst{0} = 0b0; 2150} 2151 2152// SME2 multi-vec FP to int convert four registers 2153// SME2 multi-vec int to FP four registers 2154multiclass sme2_fp_cvt_vg4_multi<string mnemonic, bits<7> op> { 2155 def NAME : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, mnemonic>; 2156} 2157 2158// SME2 multi-vec quadwords ZIP four registers 2159multiclass sme2_zip_vector_vg4<string mnemonic, bits<7> op> { 2160 def _B : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_b_mul_r, ZZZZ_b_mul_r, 2161 mnemonic>; 2162 def _H : sme2_frint_zip_cvt_vg4_multi<0b01, op, ZZZZ_h_mul_r, ZZZZ_h_mul_r, 2163 mnemonic>; 2164 def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, 2165 mnemonic>; 2166 def _D : sme2_frint_zip_cvt_vg4_multi<0b11, op, ZZZZ_d_mul_r, ZZZZ_d_mul_r, 2167 mnemonic>; 2168} 2169 2170// SME2 multi-vec quadwords ZIP four registers 2171multiclass sme2_zip_vector_vg4_Q<string mnemonic, bits<7> op> { 2172 def NAME: sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_q_mul_r, ZZZZ_q_mul_r, 2173 mnemonic>; 2174} 2175 2176// SME2 multi-vec FRINT four registers 2177multiclass sme2_frint_vector_vg4_multi<string mnemonic, bits<7> op> { 2178 def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, 2179 mnemonic>; 2180} 2181 2182class sme2_cvt_vg2_single<string mnemonic, bits<5> op, 2183 RegisterOperand first_ty, RegisterOperand second_ty> 2184 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2185 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2186 bits<4> Zn; 2187 bits<5> Zd; 2188 let Inst{31-23} = 0b110000010; 2189 let Inst{22} = op{4}; 2190 let Inst{21-19} = 0b100; 2191 let Inst{18-16} = op{3-1}; 2192 let Inst{15-10} = 0b111000; 2193 let Inst{9-6} = Zn; 2194 let Inst{5} = op{0}; 2195 let Inst{4-0} = Zd; 2196} 2197 2198// SME2 multi-vec FP down convert two registers 2199// SME2 multi-vec int down convert two registers 2200multiclass sme2_cvt_vg2_single<string mnemonic, bits<5> op, ValueType out_vt, 2201 ValueType in_vt, SDPatternOperator intrinsic> { 2202 def NAME : sme2_cvt_vg2_single<mnemonic, op, ZPR16, ZZ_s_mul_r>; 2203 def : SVE2p1_Cvt_VG2_Pat<NAME, intrinsic, out_vt, in_vt>; 2204} 2205 2206// SME2 multi-vec FP8 down convert two registers 2207multiclass sme2_fp8_cvt_vg2_single<string mnemonic, bit op> { 2208 def NAME : sme2_cvt_vg2_single<mnemonic, {op, 0b1000}, ZPR8, ZZ_h_mul_r>; 2209} 2210 2211class sme2_cvt_unpk_vector_vg2<bits<2>sz, bits<3> op, bit u, RegisterOperand first_ty, 2212 RegisterOperand second_ty, string mnemonic> 2213 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2214 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2215 bits<5> Zn; 2216 bits<4> Zd; 2217 let Inst{31-24} = 0b11000001; 2218 let Inst{23-22} = sz; 2219 let Inst{21-19} = 0b100; 2220 let Inst{18-16} = op; 2221 let Inst{15-10} = 0b111000; 2222 let Inst{9-5} = Zn; 2223 let Inst{4-1} = Zd; 2224 let Inst{0} = u; 2225} 2226 2227// SME2 multi-vec unpack two registers 2228multiclass sme2_unpk_vector_vg2<string mnemonic, bit u> { 2229 def _H : sme2_cvt_unpk_vector_vg2<0b01, 0b101, u, ZZ_h_mul_r, ZPR8, mnemonic>; 2230 def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b101, u, ZZ_s_mul_r, ZPR16, mnemonic>; 2231 def _D : sme2_cvt_unpk_vector_vg2<0b11, 0b101, u, ZZ_d_mul_r, ZPR32, mnemonic>; 2232} 2233 2234// SME2.1 multi-vec convert two registers 2235multiclass sme2p1_fp_cvt_vector_vg2_single<string mnemonic, bit l> { 2236 def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b000, l, ZZ_s_mul_r, ZPR16, mnemonic>; 2237} 2238 2239// SME2 multi-vec FP8 up convert two registers 2240multiclass sme2p1_fp8_cvt_vector_vg2_single<string mnemonic, bits<2> opc, bit L> { 2241 def _NAME : sme2_cvt_unpk_vector_vg2<opc, 0b110, L, ZZ_h_mul_r, ZPR8, mnemonic>; 2242} 2243 2244 2245class sme2_cvt_vg4_single<bit sz, bits<3> op, bits<4>op2, RegisterOperand first_ty, 2246 RegisterOperand second_ty, string mnemonic> 2247 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2248 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2249 bits<3> Zn; 2250 bits<5> Zd; 2251 let Inst{31-24} = 0b11000001; 2252 let Inst{23} = sz; 2253 let Inst{22} = op{2}; 2254 let Inst{21-20} = 0b11; 2255 let Inst{19-16} = op2; 2256 let Inst{15-10} = 0b111000; 2257 let Inst{9-7} = Zn; 2258 let Inst{6-5} = op{1-0}; 2259 let Inst{4-0} = Zd; 2260} 2261 2262// SME2 multi-vec int down convert four registers 2263multiclass sme2_int_cvt_vg4_single<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 2264 def _StoB : sme2_cvt_vg4_single<0, op, 0b0011, ZPR8, ZZZZ_s_mul_r, mnemonic>; 2265 def _DtoH : sme2_cvt_vg4_single<1, op, 0b0011, ZPR16, ZZZZ_d_mul_r, mnemonic>; 2266 2267 def : SME2_Cvt_VG4_Pat<NAME # _StoB, intrinsic, nxv16i8, nxv4i32>; 2268 def : SME2_Cvt_VG4_Pat<NAME # _DtoH, intrinsic, nxv8i16, nxv2i64>; 2269} 2270 2271//SME2 multi-vec FP8 down convert four registers 2272multiclass sme2_fp8_cvt_vg4_single<string mnemonic, bit N> { 2273 def _NAME : sme2_cvt_vg4_single<0b0, {0b00, N}, 0b0100, ZPR8, ZZZZ_s_mul_r, mnemonic>; 2274} 2275 2276class sme2_unpk_vector_vg4<bits<2>sz, bit u, RegisterOperand first_ty, 2277 RegisterOperand second_ty, string mnemonic> 2278 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2279 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2280 bits<4> Zn; 2281 bits<3> Zd; 2282 let Inst{31-24} = 0b11000001; 2283 let Inst{23-22} = sz; 2284 let Inst{21-10} = 0b110101111000; 2285 let Inst{9-6} = Zn; 2286 let Inst{5} = 0b0; 2287 let Inst{4-2} = Zd; 2288 let Inst{1} = 0b0; 2289 let Inst{0} = u; 2290} 2291 2292// SME2 multi-vec UNPK four registers 2293multiclass sme2_unpk_vector_vg4<string mnemonic, bit u> { 2294 def _H : sme2_unpk_vector_vg4<0b01, u, ZZZZ_h_mul_r, ZZ_b_mul_r, mnemonic>; 2295 def _S : sme2_unpk_vector_vg4<0b10, u, ZZZZ_s_mul_r, ZZ_h_mul_r, mnemonic>; 2296 def _D : sme2_unpk_vector_vg4<0b11, u, ZZZZ_d_mul_r, ZZ_s_mul_r, mnemonic>; 2297} 2298 2299//===----------------------------------------------------------------------===// 2300// SME2 multi-vec CLAMP registers 2301 2302class sme2_clamp_vector_vg24_multi<bits<2> sz, bits<3> op1, bit u, 2303 RegisterOperand multi_vector_ty, 2304 ZPRRegOp vector_ty, string mnemonic> 2305 : I<(outs multi_vector_ty:$Zd), 2306 (ins multi_vector_ty:$_Zd, vector_ty:$Zn, vector_ty:$Zm), 2307 mnemonic, "\t$Zd, $Zn, $Zm", 2308 "", []>, Sched<[]>{ 2309 bits<5> Zm; 2310 bits<5> Zn; 2311 let Inst{31-24} = 0b11000001; 2312 let Inst{23-22} = sz; 2313 let Inst{21} = 0b1; 2314 let Inst{20-16} = Zm; 2315 let Inst{15-13} = 0b110; 2316 let Inst{12-10} = op1; 2317 let Inst{9-5} = Zn; 2318 let Inst{0} = u; 2319 2320 let Constraints = "$Zd = $_Zd"; 2321} 2322 2323class sme2_clamp_vector_vg2_multi<bits<2> sz, bits<3> op1, bit u, 2324 RegisterOperand multi_vector_ty, 2325 ZPRRegOp vector_ty, string mnemonic> 2326 : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty, 2327 mnemonic>{ 2328 bits<4> Zd; 2329 let Inst{4-1} = Zd; 2330} 2331 2332multiclass sme2_fp_clamp_vector_vg2_multi<string mnemonic>{ 2333 def _H : sme2_clamp_vector_vg2_multi<0b01, 0b000, 0b0, ZZ_h_mul_r, ZPR16, mnemonic>; 2334 def _S : sme2_clamp_vector_vg2_multi<0b10, 0b000, 0b0, ZZ_s_mul_r, ZPR32, mnemonic>; 2335 def _D : sme2_clamp_vector_vg2_multi<0b11, 0b000, 0b0, ZZ_d_mul_r, ZPR64, mnemonic>; 2336} 2337 2338multiclass sme2_int_clamp_vector_vg2_multi<string mnemonic, bit u>{ 2339 def _B : sme2_clamp_vector_vg2_multi<0b00, 0b001, u, ZZ_b_mul_r, ZPR8, mnemonic>; 2340 def _H : sme2_clamp_vector_vg2_multi<0b01, 0b001, u, ZZ_h_mul_r, ZPR16, mnemonic>; 2341 def _S : sme2_clamp_vector_vg2_multi<0b10, 0b001, u, ZZ_s_mul_r, ZPR32, mnemonic>; 2342 def _D : sme2_clamp_vector_vg2_multi<0b11, 0b001, u, ZZ_d_mul_r, ZPR64, mnemonic>; 2343} 2344 2345// SME2.1 multi-vec FCLAMP two registers 2346multiclass sme2p1_bfclamp_vector_vg2_multi<string mnemonic> { 2347 def _H : sme2_clamp_vector_vg2_multi<0b00, 0b000, 0b0, ZZ_h_mul_r, ZPR16, 2348 mnemonic>; 2349} 2350 2351class sme2_clamp_vector_vg4_multi<bits<2> sz, bits<3> op1, bit u, 2352 RegisterOperand multi_vector_ty, 2353 ZPRRegOp vector_ty, string mnemonic> 2354 : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty, 2355 mnemonic>{ 2356 bits<3> Zd; 2357 let Inst{4-2} = Zd; 2358 let Inst{1} = 0b0; 2359} 2360 2361multiclass sme2_fp_clamp_vector_vg4_multi<string mnemonic>{ 2362 def _H : sme2_clamp_vector_vg4_multi<0b01, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, mnemonic>; 2363 def _S : sme2_clamp_vector_vg4_multi<0b10, 0b010, 0b0, ZZZZ_s_mul_r, ZPR32, mnemonic>; 2364 def _D : sme2_clamp_vector_vg4_multi<0b11, 0b010, 0b0, ZZZZ_d_mul_r, ZPR64, mnemonic>; 2365} 2366 2367multiclass sme2_int_clamp_vector_vg4_multi<string mnemonic, bit u>{ 2368 def _B : sme2_clamp_vector_vg4_multi<0b00, 0b011, u, ZZZZ_b_mul_r, ZPR8, mnemonic>; 2369 def _H : sme2_clamp_vector_vg4_multi<0b01, 0b011, u, ZZZZ_h_mul_r, ZPR16, mnemonic>; 2370 def _S : sme2_clamp_vector_vg4_multi<0b10, 0b011, u, ZZZZ_s_mul_r, ZPR32, mnemonic>; 2371 def _D : sme2_clamp_vector_vg4_multi<0b11, 0b011, u, ZZZZ_d_mul_r, ZPR64, mnemonic>; 2372} 2373 2374// SME2.1 multi-vec FCLAMP four registers 2375multiclass sme2p1_bfclamp_vector_vg4_multi<string mnemonic> { 2376 def _H : sme2_clamp_vector_vg4_multi<0b00, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, 2377 mnemonic>; 2378} 2379 2380// SME2 multi-vec ZIP two registers 2381class sme2_zip_vector_vg2<bits<2> sz, bit q, bit u, 2382 RegisterOperand multi_vector_ty, 2383 ZPRRegOp vector_ty, string mnemonic> 2384 : I<(outs multi_vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm), 2385 mnemonic, "\t$Zd, $Zn, $Zm", 2386 "", []>, Sched<[]>{ 2387 bits<4> Zd; 2388 bits<5> Zm; 2389 bits<5> Zn; 2390 let Inst{31-24} = 0b11000001; 2391 let Inst{23-22} = sz; 2392 let Inst{21} = 0b1; 2393 let Inst{20-16} = Zm; 2394 let Inst{15-11} = 0b11010; 2395 let Inst{10} = q; 2396 let Inst{9-5} = Zn; 2397 let Inst{4-1} = Zd; 2398 let Inst{0} = u; 2399} 2400 2401multiclass sme2_zip_vector_vg2<string mnemonic, bit op> { 2402 def _B : sme2_zip_vector_vg2<0b00, 0b0, op, ZZ_b_mul_r, ZPR8, mnemonic>; 2403 def _H : sme2_zip_vector_vg2<0b01, 0b0, op, ZZ_h_mul_r, ZPR16, mnemonic>; 2404 def _S : sme2_zip_vector_vg2<0b10, 0b0, op, ZZ_s_mul_r, ZPR32, mnemonic>; 2405 def _D : sme2_zip_vector_vg2<0b11, 0b0, op, ZZ_d_mul_r, ZPR64, mnemonic>; 2406 def _Q : sme2_zip_vector_vg2<0b00, 0b1, op, ZZ_q_mul_r, ZPR128, mnemonic>; 2407} 2408 2409//===----------------------------------------------------------------------===// 2410// SME2 Dot Products and MLA 2411class sme2_multi_vec_array_vg2_index<bits<2> sz, bits<6> op, MatrixOperand matrix_ty, 2412 RegisterOperand multi_vector_ty, 2413 ZPRRegOp vector_ty, Operand index_ty, 2414 string mnemonic> 2415 : I<(outs matrix_ty:$ZAda), 2416 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2417 multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i), 2418 mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i", 2419 "", []>, Sched<[]> { 2420 bits<4> Zm; 2421 bits<2> Rv; 2422 bits<4> Zn; 2423 bits<3> imm3; 2424 let Inst{31-24} = 0b11000001; 2425 let Inst{23-22} = sz; 2426 let Inst{21-20} = 0b01; 2427 let Inst{19-16} = Zm; 2428 let Inst{15} = 0b0; 2429 let Inst{14-13} = Rv; 2430 let Inst{12-10} = op{5-3}; 2431 let Inst{9-6} = Zn; 2432 let Inst{5-3} = op{2-0}; 2433 let Inst{2-0} = imm3; 2434 2435 let Constraints = "$ZAda = $_ZAda"; 2436} 2437 2438// SME2 multi-vec ternary indexed two registers 32-bit 2439multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<2> sz, bits<4> op, 2440 RegisterOperand multi_vector_ty, 2441 ZPRRegOp vector_ty, ValueType vt, 2442 SDPatternOperator intrinsic> { 2443 def NAME : sme2_multi_vec_array_vg2_index<sz, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, vector_ty, 2444 VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2445 bits<2> i; 2446 let Inst{11-10} = i; 2447 } 2448 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>; 2449 2450 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>; 2451 2452 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2453 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2454 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>; 2455} 2456 2457// SME2.1 multi-vec ternary indexed two registers 16-bit 2458// SME2 multi-vec indexed FP8 two-way dot product to FP16 two registers 2459multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> sz, bits<3> op, 2460 RegisterOperand multi_vector_ty, ZPRRegOp zpr_ty> { 2461 def NAME : sme2_multi_vec_array_vg2_index<sz, {op{2},?,?,op{1-0},?}, MatrixOp16, 2462 multi_vector_ty, zpr_ty, 2463 VectorIndexH, mnemonic> { 2464 bits<3> i; 2465 let Inst{11-10} = i{2-1}; 2466 let Inst{3} = i{0}; 2467 } 2468 2469 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2470 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2471 multi_vector_ty:$Zn, zpr_ty:$Zm, VectorIndexH:$i), 0>; 2472} 2473 2474// SME2 multi-vec indexed FP8 two-way vertical dot product to single precision 2475// two registers 2476class sme2_fp8_multi_vec_array_vg4_index<string mnemonic, bit T> 2477 : sme2_multi_vec_array_vg2_index<0b11, {0b01,?,0b0, T,?}, MatrixOp32, 2478 ZZ_b_mul_r, ZPR4b8, VectorIndexS, mnemonic> { 2479 2480 bits<2> i; 2481 let Inst{10} = i{1}; 2482 let Inst{3} = i{0}; 2483 let AsmString = !strconcat(mnemonic, "{\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i}"); 2484} 2485 2486// SME2 multi-vec ternary indexed two registers 64-bit 2487 2488class sme2_multi_vec_array_vg2_index_64b<bits<2> op, 2489 RegisterOperand multi_vector_ty, 2490 ZPRRegOp vector_ty, 2491 string mnemonic> 2492 : I<(outs MatrixOp64:$ZAda), 2493 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2494 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 2495 mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i1", 2496 "", []>, Sched<[]> { 2497 bits<4> Zm; 2498 bits<2> Rv; 2499 bits<1> i1; 2500 bits<4> Zn; 2501 bits<3> imm3; 2502 let Inst{31-20} = 0b110000011101; 2503 let Inst{19-16} = Zm; 2504 let Inst{15} = 0b0; 2505 let Inst{14-13} = Rv; 2506 let Inst{12-11} = 0b00; 2507 let Inst{10} = i1; 2508 let Inst{9-6} = Zn; 2509 let Inst{5} = 0b0; 2510 let Inst{4-3} = op; 2511 let Inst{2-0} = imm3; 2512 2513 let Constraints = "$ZAda = $_ZAda"; 2514} 2515 2516multiclass sme2_multi_vec_array_vg2_index_64b<string mnemonic, bits<2> op, 2517 RegisterOperand multi_vector_ty, 2518 ZPRRegOp vector_ty, ValueType vt, 2519 SDPatternOperator intrinsic> { 2520 def NAME : sme2_multi_vec_array_vg2_index_64b<op, multi_vector_ty, vector_ty, 2521 mnemonic>, SMEPseudo2Instr<NAME, 1>; 2522 2523 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>; 2524 2525 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexD32b_timm, tileslice16>; 2526 2527 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1", 2528 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2529 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>; 2530} 2531 2532class sme2_multi_vec_array_vg4_index<bit sz, bits<7> op, MatrixOperand matrix_ty, 2533 RegisterOperand multi_vector_ty, 2534 ZPRRegOp vector_ty, Operand index_ty, 2535 string mnemonic> 2536 : I<(outs matrix_ty:$ZAda), 2537 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2538 multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i), 2539 mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i", 2540 "", []>, Sched<[]> { 2541 bits<4> Zm; 2542 bits<2> Rv; 2543 bits<3> Zn; 2544 bits<3> imm3; 2545 let Inst{31-23} = 0b110000010; 2546 let Inst{22} = sz; 2547 let Inst{21-20} = 0b01; 2548 let Inst{19-16} = Zm; 2549 let Inst{15} = 0b1; 2550 let Inst{14-13} = Rv; 2551 let Inst{12-10} = op{6-4}; 2552 let Inst{9-7} = Zn; 2553 let Inst{6-3} = op{3-0}; 2554 let Inst{2-0} = imm3; 2555 2556 let Constraints = "$ZAda = $_ZAda"; 2557} 2558 2559// SME2 multi-vec ternary indexed four registers 32-bit 2560multiclass sme2_multi_vec_array_vg4_index_32b<string mnemonic, bits<4> op, 2561 RegisterOperand multi_vector_ty, 2562 ZPRRegOp vector_ty, ValueType vt, 2563 SDPatternOperator intrinsic> { 2564 def NAME : sme2_multi_vec_array_vg4_index<0b1, {op{3},?,?,0b0, op{2-0}}, MatrixOp32, multi_vector_ty, 2565 vector_ty, VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2566 bits<2> i; 2567 let Inst{11-10} = i; 2568 } 2569 2570 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>; 2571 2572 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>; 2573 2574 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2575 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2576 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>; 2577} 2578 2579// SME2.1 multi-vec ternary indexed four registers 16-bit 2580multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<3> op, 2581 RegisterOperand multi_vector_ty, 2582 ZPRRegOp zpr_ty> { 2583 def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16, 2584 multi_vector_ty, zpr_ty, 2585 VectorIndexH, mnemonic>{ 2586 bits<3> i; 2587 let Inst{11-10} = i{2-1}; 2588 let Inst{3} = i{0}; 2589 } 2590 2591 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2592 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, 2593 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, VectorIndexH:$i), 0>; 2594} 2595 2596// SME2 multi-vec ternary indexed four registers 64-bit 2597class sme2_multi_vec_array_vg4_index_64b<bits<3> op, 2598 RegisterOperand multi_vector_ty, 2599 ZPRRegOp vector_ty, 2600 string mnemonic> 2601 : I<(outs MatrixOp64:$ZAda), 2602 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2603 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 2604 mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i1", 2605 "", []>, Sched<[]> { 2606 bits<4> Zm; 2607 bits<2> Rv; 2608 bits<1> i1; 2609 bits<3> Zn; 2610 bits<3> imm3; 2611 let Inst{31-20} = 0b110000011101; 2612 let Inst{19-16} = Zm; 2613 let Inst{15} = 0b1; 2614 let Inst{14-13} = Rv; 2615 let Inst{12} = 0b0; 2616 let Inst{11} = op{2}; 2617 let Inst{10} = i1; 2618 let Inst{9-7} = Zn; 2619 let Inst{6-5} = 0b00; 2620 let Inst{4-3} = op{1-0}; 2621 let Inst{2-0} = imm3; 2622 2623 let Constraints = "$ZAda = $_ZAda"; 2624} 2625 2626multiclass sme2_multi_vec_array_vg4_index_64b<string mnemonic, bits<3> op, 2627 RegisterOperand multi_vector_ty, 2628 ZPRRegOp vector_ty, ValueType vty, 2629 SDPatternOperator intrinsic> { 2630 def NAME : sme2_multi_vec_array_vg4_index_64b<op, multi_vector_ty, vector_ty, 2631 mnemonic>, SMEPseudo2Instr<NAME, 1>; 2632 2633 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>; 2634 2635 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vty, VectorIndexD32b_timm, tileslice16>; 2636 2637 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1", 2638 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2639 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>; 2640} 2641 2642// FMLAL (multiple and indexed vector, FP8 to FP16) 2643class sme2_multi_vec_array_vg24_index_16b<bits<2> sz, bit vg4, bits<3> op, 2644 RegisterOperand multi_vector_ty, string mnemonic> 2645 : I<(outs MatrixOp16:$ZAda), 2646 (ins MatrixOp16:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, 2647 multi_vector_ty:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 2648 mnemonic, "\t$ZAda[$Rv, $imm2, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i", 2649 "", []>, Sched<[]> { 2650 bits<4> Zm; 2651 bits<2> Rv; 2652 bits<4> i; 2653 bits<2> imm2; 2654 let Inst{31-24} = 0b11000001; 2655 let Inst{23-22} = sz; 2656 let Inst{21-20} = 0b01; 2657 let Inst{19-16} = Zm; 2658 let Inst{15} = vg4; 2659 let Inst{14-13} = Rv; 2660 let Inst{12} = op{2}; 2661 let Inst{11-10} = i{3-2}; 2662 let Inst{5-4} = op{1-0}; 2663 let Inst{3-2} = i{1-0}; 2664 let Inst{1-0} = imm2; 2665 2666 let Constraints = "$ZAda = $_ZAda"; 2667} 2668 2669multiclass sme2_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> sz, bits<3>op> { 2670 def NAME : sme2_multi_vec_array_vg24_index_16b<sz, 0b0, op, ZZ_b_mul_r, mnemonic> { 2671 bits<4> Zn; 2672 let Inst{9-6} = Zn; 2673 } 2674 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", 2675 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, 2676 uimm2s2range:$imm2, ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 0>; 2677} 2678 2679multiclass sme2_multi_vec_array_vg4_index_16b<string mnemonic, bits<2>sz, bits<3>op> { 2680 def NAME: sme2_multi_vec_array_vg24_index_16b<sz, 0b1, op, ZZZZ_b_mul_r, mnemonic> { 2681 bits<3> Zn; 2682 let Inst{9-7} = Zn; 2683 let Inst{6} = 0b0; 2684 } 2685 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", 2686 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, 2687 uimm2s2range:$imm2, ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 0>; 2688} 2689 2690//===----------------------------------------------------------------------===// 2691// SME2 multi-vec indexed long long MLA one source 16-bit 2692class sme2_mla_ll_array_index_16b<string mnemonic, bits<2> sz,bits<2> op> 2693 : I<(outs MatrixOp16:$ZAda), 2694 (ins MatrixOp16:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm3s2range:$imm3, ZPR8:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 2695 mnemonic, "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2696 "", []>, Sched<[]> { 2697 bits<4> Zm; 2698 bits<2> Rv; 2699 bits<4> i; 2700 bits<5> Zn; 2701 bits<3> imm3; 2702 let Inst{31-24} = 0b11000001; 2703 let Inst{23-22} = sz; 2704 let Inst{21-20} = 0b00; 2705 let Inst{19-16} = Zm; 2706 let Inst{15} = i{3}; 2707 let Inst{14-13} = Rv; 2708 let Inst{12} = op{1}; 2709 let Inst{11-10} = i{2-1}; 2710 let Inst{9-5} = Zn; 2711 let Inst{4} = op{0}; 2712 let Inst{3} = i{0}; 2713 let Inst{2-0} = imm3; 2714 2715 let Constraints = "$ZAda = $_ZAda"; 2716} 2717 2718// SME2 multi-vec indexed long long MLA one source 32-bit 2719class sme2_mla_ll_array_index_32b<string mnemonic, bits<2> sz, bits<3> op> 2720 : I<(outs MatrixOp32:$ZAda), 2721 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR8:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 2722 mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", 2723 "", []>, Sched<[]> { 2724 bits<4> Zm; 2725 bits<2> Rv; 2726 bits<4> i; 2727 bits<5> Zn; 2728 bits<2> imm2; 2729 let Inst{31-24} = 0b11000001; 2730 let Inst{23-22} = sz; 2731 let Inst{21-20} = 0b00; 2732 let Inst{19-16} = Zm; 2733 let Inst{15} = i{3}; 2734 let Inst{14-13} = Rv; 2735 let Inst{12-10} = i{2-0}; 2736 let Inst{9-5} = Zn; 2737 let Inst{4-2} = op; 2738 let Inst{1-0} = imm2; 2739 2740 let Constraints = "$ZAda = $_ZAda"; 2741} 2742 2743multiclass sme2_mla_ll_array_index_32b<string mnemonic, bits<2> sz, bits<3> op, SDPatternOperator intrinsic> { 2744 def NAME : sme2_mla_ll_array_index_32b<mnemonic, sz, op>, SMEPseudo2Instr<NAME, 1>; 2745 2746 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm2s4range, ZPR8, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>; 2747 2748 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME, intrinsic, uimm2s4range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange2s4>; 2749} 2750 2751// SME2 multi-vec indexed long long MLA one source 64-bit 2752 2753class sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op> 2754 : I<(outs MatrixOp64:$ZAda), 2755 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR16:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 2756 mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", 2757 "", []>, Sched<[]> { 2758 bits<4> Zm; 2759 bits<2> Rv; 2760 bits<3> i; 2761 bits<5> Zn; 2762 bits<2> imm2; 2763 let Inst{31-20} = 0b110000011000; 2764 let Inst{19-16} = Zm; 2765 let Inst{15} = i{2}; 2766 let Inst{14-13} = Rv; 2767 let Inst{12} = 0b0; 2768 let Inst{11-10} = i{1-0}; 2769 let Inst{9-5} = Zn; 2770 let Inst{4-3} = op; 2771 let Inst{2} = 0b0; 2772 let Inst{1-0} = imm2; 2773 2774 let Constraints = "$ZAda = $_ZAda"; 2775} 2776 2777multiclass sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2778 def NAME : sme2_mla_ll_array_index_64b<mnemonic, op>, SMEPseudo2Instr<NAME, 1>; 2779 2780 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm2s4range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 2781 2782 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME, intrinsic, uimm2s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s4>; 2783} 2784 2785class sme2_mla_ll_array_vg24_index_32b<bits<2> sz, bit vg4, bits<3> op, 2786 RegisterOperand vector_ty, 2787 string mnemonic> 2788 : I<(outs MatrixOp32:$ZAda), 2789 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2790 vector_ty:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 2791 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i", 2792 "", []>, Sched<[]> { 2793 bits<4> Zm; 2794 bits<2> Rv; 2795 bits<4> i; 2796 bit imm; 2797 let Inst{31-24} = 0b11000001; 2798 let Inst{23-22} = sz; 2799 let Inst{21-20} = 0b01; 2800 let Inst{19-16} = Zm; 2801 let Inst{15} = vg4; 2802 let Inst{14-13} = Rv; 2803 let Inst{12} = 0b0; 2804 let Inst{11-10} = i{3-2}; 2805 let Inst{5-3} = op; 2806 let Inst{2-1} = i{1-0}; 2807 let Inst{0} = imm; 2808 2809 let Constraints = "$ZAda = $_ZAda"; 2810} 2811 2812//SME2 multi-vec indexed long long MLA two sources 32-bit 2813 2814multiclass sme2_mla_ll_array_vg2_index_32b<string mnemonic, bits<2> sz, bits<3> op, SDPatternOperator intrinsic> { 2815 def NAME: sme2_mla_ll_array_vg24_index_32b<sz, 0b0, op, ZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2816 bits<4> Zn; 2817 let Inst{9-6} = Zn; 2818 } 2819 2820 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZ_b_mul_r, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>; 2821 2822 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange1s4>; 2823 2824 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2825 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 0>; 2826} 2827 2828// SME2 multi-vec indexed long long MLA four sources 32-bit 2829 2830multiclass sme2_mla_ll_array_vg4_index_32b<string mnemonic, bits<2> sz, bits<4> op, SDPatternOperator intrinsic> { 2831 def NAME: sme2_mla_ll_array_vg24_index_32b<sz, 0b1, op{2-0}, ZZZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2832 bits<3> Zn; 2833 let Inst{9-7} = Zn; 2834 let Inst{6} = op{3}; 2835 } 2836 2837 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZZZ_b_mul_r, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>; 2838 2839 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange1s4>; 2840 2841 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2842 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 0>; 2843} 2844class sme2_mla_ll_array_vg24_index_64b<bit vg4, bits<2> op, 2845 RegisterOperand vector_ty, 2846 string mnemonic> 2847 : I<(outs MatrixOp64:$ZAda), 2848 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2849 vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 2850 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i", 2851 "", []>, Sched<[]> { 2852 bits<4> Zm; 2853 bits<2> Rv; 2854 bits<3> i; 2855 bit imm; 2856 let Inst{31-20} = 0b110000011001; 2857 let Inst{19-16} = Zm; 2858 let Inst{15} = vg4; 2859 let Inst{14-13} = Rv; 2860 let Inst{12-11} = 0b00; 2861 let Inst{10} = i{2}; 2862 let Inst{5} = 0b0; 2863 let Inst{4-3} = op; 2864 let Inst{2-1} = i{1-0}; 2865 let Inst{0} = imm; 2866 2867 let Constraints = "$ZAda = $_ZAda"; 2868} 2869 2870// SME2 multi-vec indexed long long MLA two sources 64-bit 2871 2872multiclass sme2_mla_ll_array_vg2_index_64b<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2873 def NAME: sme2_mla_ll_array_vg24_index_64b<0b0, op, ZZ_h_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2874 bits<4> Zn; 2875 let Inst{9-6} = Zn; 2876 } 2877 2878 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 2879 2880 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange1s4>; 2881 2882 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2883 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 0>; 2884} 2885 2886// SME2 multi-vec indexed long long MLA four sources 64-bit 2887 2888multiclass sme2_mla_ll_array_vg4_index_64b<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2889 def NAME: sme2_mla_ll_array_vg24_index_64b<0b1, op, ZZZZ_h_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2890 bits<3> Zn; 2891 let Inst{9-7} = Zn; 2892 let Inst{6} = 0b0; 2893 } 2894 2895 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 2896 2897 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange1s4>; 2898 2899 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2900 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 0>; 2901} 2902 2903 2904//SME2 multiple and single vector long long FMA one source 2905 2906class sme2_mla_ll_array_single<string mnemonic, bits<5> op, 2907 MatrixOperand matrix_ty, ZPRRegOp vector_ty, 2908 ZPRRegOp zpr_ty> 2909 : I<(outs matrix_ty:$ZAda), 2910 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm, 2911 vector_ty:$Zn, zpr_ty:$Zm), 2912 mnemonic, "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2913 "", []>, Sched<[]> { 2914 bits<4> Zm; 2915 bits<2> Rv; 2916 bits<5> Zn; 2917 bits<2> imm; 2918 let Inst{31-23} = 0b110000010; 2919 let Inst{22} = op{4}; //sz 2920 let Inst{21} = 0b1; 2921 let Inst{20} = op{3}; //fp8 2922 let Inst{19-16} = Zm; 2923 let Inst{15} = 0b0; 2924 let Inst{14-13} = Rv; 2925 let Inst{12-10} = 0b001; 2926 let Inst{9-5} = Zn; 2927 let Inst{4-2} = op{2-0}; 2928 let Inst{1-0} = imm; 2929 2930 let Constraints = "$ZAda = $_ZAda"; 2931} 2932 2933multiclass sme2_mla_ll_array_single<string mnemonic, bits<5> op, 2934 MatrixOperand matrix_ty, ZPRRegOp vector_ty, 2935 ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> { 2936 def NAME : sme2_mla_ll_array_single<mnemonic, op, matrix_ty, vector_ty, zpr_ty>, SMEPseudo2Instr<NAME, 1>; 2937 2938 def NAME # _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm2s4range, vector_ty, zpr_ty, SMEMatrixArray>; 2939 2940 def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME, intrinsic, uimm2s4range, zpr_ty, vt, tileslicerange2s4>; 2941} 2942 2943class sme2_mla_ll_array_vg24_single<bits<6> op, MatrixOperand matrix_ty, 2944 RegisterOperand vector_ty, ZPRRegOp zpr_ty, 2945 string mnemonic> 2946 : I<(outs matrix_ty:$ZAda), 2947 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2948 vector_ty:$Zn, zpr_ty:$Zm), 2949 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(op{4}, "vgx4", "vgx2") # "], $Zn, $Zm", 2950 "", []>, Sched<[]> { 2951 bits<4> Zm; 2952 bits<2> Rv; 2953 bits<5> Zn; 2954 bit imm; 2955 let Inst{31-23} = 0b110000010; 2956 let Inst{22} = op{5}; //sz 2957 let Inst{21} = 0b1; 2958 let Inst{20} = op{4}; //vg4 2959 let Inst{19-16} = Zm; 2960 let Inst{15} = 0b0; 2961 let Inst{14-13} = Rv; 2962 let Inst{12-10} = 0b000; 2963 let Inst{9-5} = Zn; 2964 let Inst{4-1} = op{3-0}; 2965 let Inst{0} = imm; 2966 2967 let Constraints = "$ZAda = $_ZAda"; 2968} 2969 2970//SME2 single-multi long long MLA two and four sources 2971 2972multiclass sme2_mla_ll_array_vg24_single<string mnemonic, bits<6> op, 2973 MatrixOperand matrix_ty, 2974 RegisterOperand multi_vector_ty, 2975 ZPRRegOp zpr_ty> { 2976 def NAME: sme2_mla_ll_array_vg24_single<op, matrix_ty, multi_vector_ty, 2977 zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 2978 2979 def NAME # _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm1s4range, multi_vector_ty, zpr_ty, SMEMatrixArray>; 2980 2981 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm], $Zn, $Zm", 2982 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 2983} 2984 2985multiclass sme2_mla_ll_array_vg2_single<string mnemonic, bits<5> op, 2986 MatrixOperand matrix_ty, 2987 RegisterOperand multi_vector_ty, 2988 ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> { 2989 2990 defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, {op, 0b0}, matrix_ty, multi_vector_ty, zpr_ty>; 2991 2992 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, uimm1s4range, zpr_ty, vt, tileslicerange1s4>; 2993} 2994 2995multiclass sme2_mla_ll_array_vg4_single<string mnemonic, bits<5> op, 2996 MatrixOperand matrix_ty, 2997 RegisterOperand multi_vector_ty, 2998 ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> { 2999 defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, {op, 0b0}, matrix_ty, multi_vector_ty, zpr_ty>; 3000 3001 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, uimm1s4range, zpr_ty, vt, tileslicerange1s4>; 3002} 3003 3004// SME2 multiple vectors long long MLA two sources 3005 3006class sme2_mla_ll_array_vg2_multi<bits<5> op, MatrixOperand matrix_ty, 3007 RegisterOperand vector_ty,string mnemonic> 3008 : I<(outs matrix_ty:$ZAda), 3009 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 3010 vector_ty:$Zn, vector_ty:$Zm), 3011 mnemonic, "\t$ZAda[$Rv, $imm, vgx2], $Zn, $Zm", 3012 "", []>, Sched<[]> { 3013 bits<4> Zm; 3014 bits<2> Rv; 3015 bits<4> Zn; 3016 bit imm; 3017 let Inst{31-23} = 0b110000011; 3018 let Inst{22} = op{4}; // sz 3019 let Inst{21} = 0b1; 3020 let Inst{20-17} = Zm; 3021 let Inst{16-15} = 0b00; 3022 let Inst{14-13} = Rv; 3023 let Inst{12-10} = 0b000; 3024 let Inst{9-6} = Zn; 3025 let Inst{5-2} = op{3-0}; 3026 let Inst{1} = 0b0; 3027 let Inst{0} = imm; 3028 3029 let Constraints = "$ZAda = $_ZAda"; 3030} 3031 3032multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<5> op, 3033 MatrixOperand matrix_ty, 3034 RegisterOperand vector_ty, 3035 ValueType vt, SDPatternOperator intrinsic> { 3036 def NAME : sme2_mla_ll_array_vg2_multi<op, matrix_ty, vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 3037 3038 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm1s4range, vector_ty, SMEMatrixArray>; 3039 3040 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, uimm1s4range, vt, tileslicerange1s4>; 3041 3042 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 3043 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>; 3044} 3045 3046// SME2 multiple vectors long long MLA four sources 3047 3048class sme2_mla_ll_array_vg4_multi<bits<5> op,MatrixOperand matrix_ty, 3049 RegisterOperand vector_ty, 3050 string mnemonic> 3051 : I<(outs matrix_ty:$ZAda), 3052 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 3053 vector_ty:$Zn, vector_ty:$Zm), 3054 mnemonic, "\t$ZAda[$Rv, $imm, vgx4], $Zn, $Zm", 3055 "", []>, Sched<[]> { 3056 bits<3> Zm; 3057 bits<2> Rv; 3058 bits<3> Zn; 3059 bit imm; 3060 let Inst{31-23} = 0b110000011; 3061 let Inst{22} = op{4}; // sz 3062 let Inst{21} = 0b1; 3063 let Inst{20-18} = Zm; 3064 let Inst{17-15} = 0b010; 3065 let Inst{14-13} = Rv; 3066 let Inst{12-10} = 0b000; 3067 let Inst{9-7} = Zn; 3068 let Inst{6} = 0b0; 3069 let Inst{5-2} = op{3-0}; 3070 let Inst{1} = 0b0; 3071 let Inst{0} = imm; 3072 3073 let Constraints = "$ZAda = $_ZAda"; 3074} 3075 3076multiclass sme2_mla_ll_array_vg4_multi<string mnemonic, bits<5> op, 3077 MatrixOperand matrix_ty, 3078 RegisterOperand vector_ty, 3079 ValueType vt, SDPatternOperator intrinsic> { 3080 def NAME : sme2_mla_ll_array_vg4_multi<op, matrix_ty, vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 3081 3082 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm1s4range, vector_ty, SMEMatrixArray>; 3083 3084 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, uimm1s4range, vt, tileslicerange1s4>; 3085 3086 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 3087 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>; 3088} 3089 3090//===----------------------------------------------------------------------===// 3091// SME2 Outer Product and Accumulate 3092 3093multiclass sme2_int_mopx_tile<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 3094 def NAME : sme_int_outer_product_inst<op, 0b0, 0b1, TileOp32, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> { 3095 bits<2> ZAda; 3096 let Inst{1-0} = ZAda; 3097 let Inst{2} = 0b0; 3098 } 3099 3100 def _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 3101 3102 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_3, nxv8i1, nxv8i16>; 3103} 3104 3105multiclass sme2_int_bmopx_tile<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 3106 def NAME : sme_outer_product_widening_inst<op, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1>; 3107 3108 def _PSEUDO : sme_outer_product_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 3109 3110 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_3, nxv4i1, nxv4i32>; 3111} 3112 3113//===----------------------------------------------------------------------===/// 3114// SME2 Zero Lookup Table. 3115class sme2_zero_zt<string mnemonic, bits<4> opc> 3116 : I<(outs ZTR:$ZT), (ins ), 3117 mnemonic, "\t\\{ $ZT \\}", 3118 "", []>, Sched<[]> { 3119 let Inst{31-4} = 0b1100000001001000000000000000; 3120 let Inst{3-0} = opc; 3121} 3122 3123multiclass sme2_zero_zt<string mnemonic, bits<4> opc> { 3124 def NAME : sme2_zero_zt<mnemonic, opc>; 3125 def NAME # _PSEUDO 3126 : Pseudo<(outs), (ins ZTR:$ZT), []>, Sched<[]> { 3127 // Translated to actual instruction in AArch64ISelLowering.cpp 3128 let usesCustomInserter = 1; 3129 } 3130 def : Pat<(int_aarch64_sme_zero_zt (imm_to_zt untyped:$zt)), 3131 (!cast<Instruction>(NAME # _PSEUDO) $zt)>; 3132} 3133 3134//===----------------------------------------------------------------------===// 3135// SME2 lookup table load/store 3136class sme2_spill_fill_vector<string mnemonic, bits<8> opc> 3137 : I<!if(opc{7}, (outs ), (outs ZTR:$ZTt)), 3138 !if(opc{7}, (ins ZTR:$ZTt, GPR64sp:$Rn), (ins GPR64sp:$Rn)), 3139 mnemonic, "\t$ZTt, [$Rn]", 3140 "", []>, Sched<[]> { 3141 bits<5> Rn; 3142 let Inst{31-22} = 0b1110000100; 3143 let Inst{21-16} = opc{7-2}; 3144 let Inst{15-10} = 0b100000; 3145 let Inst{9-5} = Rn; 3146 let Inst{4-2} = 0b000; 3147 let Inst{1-0} = opc{1-0}; 3148 3149 let mayLoad = !not(opc{7}); 3150 let mayStore = opc{7}; 3151} 3152 3153 3154multiclass sme2_spill_fill_vector<string mnemonic, bits<8> opc, SDPatternOperator op> { 3155 def NAME : sme2_spill_fill_vector<mnemonic, opc>; 3156 def NAME # _PSEUDO 3157 : Pseudo<(outs), (ins ZTR:$ZTt, GPR64sp:$base), []>, Sched<[]> { 3158 // Translated to actual instruction in AArch64ISelLowering.cpp 3159 let usesCustomInserter = 1; 3160 } 3161 def : Pat<(op (imm_to_zt untyped:$tile), GPR64sp:$base), 3162 (!cast<Instruction>(NAME # _PSEUDO) $tile, $base)>; 3163} 3164 3165//===----------------------------------------------------------------------===/// 3166// SME2 move to/from lookup table 3167class sme2_movt_zt_to_scalar<string mnemonic, bits<7> opc> 3168 : I<(outs GPR64:$Rt), (ins ZTR:$ZTt, uimm3s8:$imm3), 3169 mnemonic, "\t$Rt, $ZTt[$imm3]", 3170 "", []>, Sched<[]> { 3171 bits<3> imm3; 3172 bits<5> Rt; 3173 let Inst{31-15} = 0b11000000010011000; 3174 let Inst{14-12} = imm3; 3175 let Inst{11-5} = opc; 3176 let Inst{4-0} = Rt; 3177} 3178 3179class sme2_movt_scalar_to_zt<string mnemonic, bits<7> opc> 3180 : I<(outs ZTR:$ZTt), (ins uimm3s8:$imm3, GPR64:$Rt), 3181 mnemonic, "\t$ZTt[$imm3], $Rt", 3182 "", []>, Sched<[]> { 3183 bits<3> imm3; 3184 bits<5> Rt; 3185 let Inst{31-15} = 0b11000000010011100; 3186 let Inst{14-12} = imm3; 3187 let Inst{11-5} = opc; 3188 let Inst{4-0} = Rt; 3189} 3190 3191// SME2 move vector to lookup table 3192class sme2_movt_zt_to_zt<string mnemonic, bits<7> opc> 3193 : I<(outs ZTR:$ZTt), (ins sme_elm_idx0_3:$off2, ZPRAny:$Zt), 3194 mnemonic, "\t$ZTt[$off2, mul vl], $Zt", 3195 "", []>, Sched<[]> { 3196 bits<5> Zt; 3197 bits<2> off2; 3198 let Inst{31-14} = 0b110000000100111100; 3199 let Inst{13-12} = off2; 3200 let Inst{11-5} = opc; 3201 let Inst{4-0} = Zt; 3202} 3203 3204multiclass sme2_movt_zt_to_zt<string mnemonic, bits<7> opc> { 3205 def NAME : sme2_movt_zt_to_zt<mnemonic, opc>; 3206 def : InstAlias<mnemonic # "\t$ZTt, $Zt", 3207 (!cast<Instruction>(NAME) ZTR:$ZTt, 0, ZPRAny:$Zt), 1>; 3208} 3209 3210//===----------------------------------------------------------------------===// 3211// SME2 lookup table expand one register 3212class sme2_luti_vector_index<bits<2> sz, bits<7> opc, RegisterOperand vector_ty, 3213 AsmVectorIndexOpnd index_ty, string mnemonic> 3214 : I<(outs vector_ty:$Zd), 3215 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 3216 mnemonic, "\t$Zd, $ZTt, $Zn$i", 3217 "", []>, Sched<[]> { 3218 bits<5> Zn; 3219 bits<5> Zd; 3220 let Inst{31-19} = 0b1100000011001; 3221 let Inst{18-14} = opc{6-2}; 3222 let Inst{13-12} = sz; 3223 let Inst{11-10} = opc{1-0}; 3224 let Inst{9-5} = Zn; 3225 let Inst{4-0} = Zd; 3226} 3227 3228class sme2_luti2_vector_index<bits<2> sz, RegisterOperand vector_ty, 3229 string mnemonic> 3230 : sme2_luti_vector_index<sz, {1,?,?,?,?,0,0}, vector_ty, VectorIndexB32b_timm, mnemonic> { 3231 bits<4> i; 3232 let Inst{17-14} = i; 3233} 3234 3235multiclass sme2_luti2_vector_index<string mnemonic, SDPatternOperator intrinsic> { 3236 def _B : sme2_luti2_vector_index<0b00, ZPR8, mnemonic>; 3237 def _H : sme2_luti2_vector_index<0b01, ZPR16, mnemonic>; 3238 def _S : sme2_luti2_vector_index<0b10, ZPR32, mnemonic>; 3239 3240 def : Pat<(nxv16i8 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))), 3241 (!cast<Instruction>(NAME # _B) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>; 3242 def : Pat<(nxv8i16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))), 3243 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>; 3244 def : Pat<(nxv4i32 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))), 3245 (!cast<Instruction>(NAME # _S) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>; 3246 def : Pat<(nxv8f16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))), 3247 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>; 3248 def : Pat<(nxv8bf16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))), 3249 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>; 3250 def : Pat<(nxv4f32 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))), 3251 (!cast<Instruction>(NAME # _S) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>; 3252} 3253 3254class sme2_luti4_vector_index<bits<2> sz, RegisterOperand vector_ty, 3255 string mnemonic> 3256 : sme2_luti_vector_index<sz, {0,1,?,?,?,0,0}, vector_ty, VectorIndexH32b_timm, mnemonic> { 3257 bits<3> i; 3258 let Inst{16-14} = i; 3259} 3260 3261multiclass sme2_luti4_vector_index<string mnemonic, SDPatternOperator intrinsic> { 3262 def _B : sme2_luti4_vector_index<0b00, ZPR8, mnemonic>; 3263 def _H : sme2_luti4_vector_index<0b01, ZPR16, mnemonic>; 3264 def _S : sme2_luti4_vector_index<0b10, ZPR32, mnemonic>; 3265 3266 def : Pat<(nxv16i8 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))), 3267 (!cast<Instruction>(NAME # _B) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>; 3268 def : Pat<(nxv8i16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))), 3269 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>; 3270 def : Pat<(nxv4i32 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))), 3271 (!cast<Instruction>(NAME # _S) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>; 3272 def : Pat<(nxv8f16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))), 3273 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>; 3274 def : Pat<(nxv8bf16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))), 3275 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>; 3276 def : Pat<(nxv4f32 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))), 3277 (!cast<Instruction>(NAME # _S) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>; 3278} 3279 3280// SME2 lookup table expand two contiguous registers 3281class sme2_luti_vector_vg2_index<bits<2> sz, bits<6> opc, RegisterOperand vector_ty, 3282 AsmVectorIndexOpnd index_ty, string mnemonic> 3283 : I<(outs vector_ty:$Zd), 3284 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 3285 mnemonic, "\t$Zd, $ZTt, $Zn$i", 3286 "", []>, Sched<[]> { 3287 bits<5> Zn; 3288 bits<4> Zd; 3289 let Inst{31-19} = 0b1100000010001; 3290 let Inst{18-15} = opc{5-2}; 3291 let Inst{14} = 0b1; 3292 let Inst{13-12} = sz; 3293 let Inst{11-10} = opc{1-0}; 3294 let Inst{9-5} = Zn; 3295 let Inst{4-1} = Zd; 3296 let Inst{0} = 0b0; 3297} 3298 3299class sme2_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 3300 string mnemonic> 3301 : sme2_luti_vector_vg2_index<sz, {1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> { 3302 bits<3> i; 3303 let Inst{17-15} = i; 3304} 3305 3306multiclass sme2_luti2_vector_vg2_index<string mnemonic> { 3307 def _B : sme2_luti2_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>; 3308 def _H : sme2_luti2_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>; 3309 def _S : sme2_luti2_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>; 3310} 3311 3312class sme2_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 3313 string mnemonic> 3314 : sme2_luti_vector_vg2_index<sz, {0,1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> { 3315 bits<2> i; 3316 let Inst{16-15} = i; 3317} 3318 3319multiclass sme2_luti4_vector_vg2_index<string mnemonic> { 3320 def _B : sme2_luti4_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>; 3321 def _H : sme2_luti4_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>; 3322 def _S : sme2_luti4_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>; 3323} 3324 3325// SME2 lookup table expand four contiguous registers 3326class sme2_luti_vector_vg4_index<bits<2> sz, bits<5>opc, RegisterOperand vector_ty, 3327 AsmVectorIndexOpnd index_ty, string mnemonic> 3328 : I<(outs vector_ty:$Zd), 3329 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 3330 mnemonic, "\t$Zd, $ZTt, $Zn$i", 3331 "", []>, Sched<[]> { 3332 bits<5> Zn; 3333 bits<3> Zd; 3334 let Inst{31-19} = 0b1100000010001; 3335 let Inst{18-16} = opc{4-2}; 3336 let Inst{15-14} = 0b10; 3337 let Inst{13-12} = sz; 3338 let Inst{11-10} = opc{1-0}; 3339 let Inst{9-5} = Zn; 3340 let Inst{4-2} = Zd; 3341 let Inst{1-0} = 0b00; 3342} 3343 3344class sme2_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 3345 string mnemonic> 3346 : sme2_luti_vector_vg4_index<sz, {1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> { 3347 bits<2> i; 3348 let Inst{17-16} = i; 3349} 3350 3351multiclass sme2_luti2_vector_vg4_index<string mnemonic> { 3352 def _B : sme2_luti2_vector_vg4_index<0b00, ZZZZ_b_mul_r, mnemonic>; 3353 def _H : sme2_luti2_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>; 3354 def _S : sme2_luti2_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>; 3355} 3356 3357class sme2_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 3358 string mnemonic> 3359 : sme2_luti_vector_vg4_index<sz, {0,1,?,0,0}, vector_ty, VectorIndexD, mnemonic> { 3360 bits<1> i; 3361 let Inst{16} = i; 3362} 3363 3364multiclass sme2_luti4_vector_vg4_index<string mnemonic> { 3365 def _H : sme2_luti4_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>; 3366 def _S : sme2_luti4_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>; 3367} 3368 3369//===----------------------------------------------------------------------===// 3370// SME2 MOV 3371class sme2_mova_vec_to_tile_vg2_multi_base<bits<2> sz, bit v, 3372 RegisterOperand tile_ty, 3373 Operand index_ty, 3374 RegisterOperand vector_ty, 3375 string mnemonic> 3376 : I<(outs tile_ty:$ZAd), 3377 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, vector_ty:$Zn), 3378 mnemonic, "\t$ZAd[$Rs, $imm], $Zn", 3379 "", []>, Sched<[]> { 3380 bits<2> Rs; 3381 bits<4> Zn; 3382 let Inst{31-24} = 0b11000000; 3383 let Inst{23-22} = sz; 3384 let Inst{21-16} = 0b000100; 3385 let Inst{15} = v; 3386 let Inst{14-13} = Rs; 3387 let Inst{12-10} = 0b000; 3388 let Inst{9-6} = Zn; 3389 let Inst{5-3} = 0b000; 3390 3391 let Constraints = "$ZAd = $_ZAd"; 3392} 3393 3394multiclass sme2_mova_vec_to_tile_or_array_aliases<int prefer, Instruction inst, 3395 RegisterOperand tile_or_array_ty, 3396 RegisterOperand rv_ty, 3397 Operand index_ty, 3398 RegisterOperand vector_ty, 3399 string mnemonic, 3400 string vg_acronym=""> { 3401 def : InstAlias<mnemonic # "\t$ZAd[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn", 3402 (inst tile_or_array_ty:$ZAd, rv_ty:$Rs, index_ty:$imm, vector_ty:$Zn), prefer>; 3403 3404} 3405 3406// SME2 move vector to tile, two registers 3407multiclass sme2_mova_vec_to_tile_vg2_multi_base<bit v, string mnemonic, SDPatternOperator intrinsic> { 3408 3409 def _B : sme2_mova_vec_to_tile_vg2_multi_base<0b00, v, 3410 !if(v, TileVectorOpV8, 3411 TileVectorOpH8), 3412 uimm3s2range, ZZ_b_mul_r, 3413 mnemonic>, SMEPseudo2Instr<NAME # _B, 1> { 3414 bits<3> imm; 3415 let Inst{2-0} = imm; 3416 } 3417 3418 def _H : sme2_mova_vec_to_tile_vg2_multi_base<0b01, v, 3419 !if(v, TileVectorOpV16, 3420 TileVectorOpH16), 3421 uimm2s2range, ZZ_h_mul_r, 3422 mnemonic>, SMEPseudo2Instr<NAME # _H, 1> { 3423 bits<1> ZAd; 3424 bits<2> imm; 3425 let Inst{2} = ZAd; 3426 let Inst{1-0} = imm; 3427 } 3428 3429 def _S : sme2_mova_vec_to_tile_vg2_multi_base<0b10, v, 3430 !if(v, TileVectorOpV32, 3431 TileVectorOpH32), 3432 uimm1s2range, ZZ_s_mul_r, 3433 mnemonic>, SMEPseudo2Instr<NAME # _S, 1> { 3434 bits<2> ZAd; 3435 bits<1> imm; 3436 let Inst{2-1} = ZAd; 3437 let Inst{0} = imm; 3438 } 3439 3440 def _D : sme2_mova_vec_to_tile_vg2_multi_base<0b11, v, 3441 !if(v, TileVectorOpV64, 3442 TileVectorOpH64), 3443 uimm0s2range, ZZ_d_mul_r, 3444 mnemonic>, SMEPseudo2Instr<NAME # _D, 1> { 3445 bits<3> ZAd; 3446 let Inst{2-0} = ZAd; 3447 } 3448 3449 def NAME # _B_PSEUDO : sme2_move_to_tile_pseudo<NAME # _B, sme_elm_idx0_0, uimm3s2range, ZZ_b_mul_r, SMEMatrixTileB>; 3450 def NAME # _H_PSEUDO : sme2_move_to_tile_pseudo<NAME # _H, sme_elm_idx0_1, uimm2s2range, ZZ_h_mul_r, SMEMatrixTileH>; 3451 def NAME # _S_PSEUDO : sme2_move_to_tile_pseudo<NAME # _S, sme_elm_idx0_3, uimm1s2range, ZZ_s_mul_r, SMEMatrixTileS>; 3452 def NAME # _D_PSEUDO : sme2_move_to_tile_pseudo<NAME # _D, sme_elm_idx0_7, uimm0s2range, ZZ_d_mul_r, SMEMatrixTileD>; 3453 3454 def : SME2_Tile_VG2_Multi_Pat<NAME # _B, intrinsic, sme_elm_idx0_0, nxv16i8, uimm3s2range, tileslicerange3s2>; 3455 def : SME2_Tile_VG2_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8i16, uimm2s2range, tileslicerange2s2>; 3456 def : SME2_Tile_VG2_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8f16, uimm2s2range, tileslicerange2s2>; 3457 def : SME2_Tile_VG2_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8bf16, uimm2s2range, tileslicerange2s2>; 3458 def : SME2_Tile_VG2_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4i32, uimm1s2range, tileslicerange1s2>; 3459 def : SME2_Tile_VG2_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4f32, uimm1s2range, tileslicerange1s2>; 3460 def : SME2_Tile_VG2_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2i64, uimm0s2range, tileslicerange0s2>; 3461 def : SME2_Tile_VG2_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2f64, uimm0s2range, tileslicerange0s2>; 3462 3463 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B), 3464 !if(v, TileVectorOpV8, 3465 TileVectorOpH8), 3466 MatrixIndexGPR32Op12_15, 3467 uimm3s2range, ZZ_b_mul_r, 3468 "mov">; 3469 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H), 3470 !if(v, TileVectorOpV16, 3471 TileVectorOpH16), 3472 MatrixIndexGPR32Op12_15, 3473 uimm2s2range, ZZ_h_mul_r, 3474 "mov">; 3475 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S), 3476 !if(v, TileVectorOpV32, 3477 TileVectorOpH32), 3478 MatrixIndexGPR32Op12_15, 3479 uimm1s2range, ZZ_s_mul_r, 3480 "mov">; 3481 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D), 3482 !if(v, TileVectorOpV64, 3483 TileVectorOpH64), 3484 MatrixIndexGPR32Op12_15, 3485 uimm0s2range, ZZ_d_mul_r, 3486 "mov">; 3487 3488 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), 3489 !if(v, TileVectorOpV8, 3490 TileVectorOpH8), 3491 MatrixIndexGPR32Op12_15, 3492 uimm3s2range, ZZ_b_mul_r, 3493 "mova">; 3494 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), 3495 !if(v, TileVectorOpV16, 3496 TileVectorOpH16), 3497 MatrixIndexGPR32Op12_15, 3498 uimm2s2range, ZZ_h_mul_r, 3499 "mova">; 3500 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), 3501 !if(v, TileVectorOpV32, 3502 TileVectorOpH32), 3503 MatrixIndexGPR32Op12_15, 3504 uimm1s2range, ZZ_s_mul_r, 3505 "mova">; 3506 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), 3507 !if(v, TileVectorOpV64, 3508 TileVectorOpH64), 3509 MatrixIndexGPR32Op12_15, 3510 uimm0s2range, ZZ_d_mul_r, 3511 "mova">; 3512 3513 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), 3514 !if(v, TileVectorOpV8, 3515 TileVectorOpH8), 3516 MatrixIndexGPR32Op12_15, 3517 uimm3s2range, ZZ_b_mul_r, 3518 "mova">; 3519 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), 3520 !if(v, TileVectorOpV16, 3521 TileVectorOpH16), 3522 MatrixIndexGPR32Op12_15, 3523 uimm2s2range, ZZ_h_mul_r, 3524 "mova">; 3525 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), 3526 !if(v, TileVectorOpV32, 3527 TileVectorOpH32), 3528 MatrixIndexGPR32Op12_15, 3529 uimm1s2range, ZZ_s_mul_r, 3530 "mova">; 3531 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), 3532 !if(v, TileVectorOpV64, 3533 TileVectorOpH64), 3534 MatrixIndexGPR32Op12_15, 3535 uimm0s2range, ZZ_d_mul_r, 3536 "mova">; 3537} 3538 3539multiclass sme2_mova_vec_to_tile_vg2_multi<string mnemonic, 3540 SDPatternOperator int_h, SDPatternOperator int_v>{ 3541 defm _H : sme2_mova_vec_to_tile_vg2_multi_base<0b0, mnemonic, int_h>; 3542 defm _V : sme2_mova_vec_to_tile_vg2_multi_base<0b1, mnemonic, int_v>; 3543} 3544 3545class sme2_mova_vec_to_tile_vg4_multi_base<bits<2> sz, bit v, bits<3> op, 3546 RegisterOperand tile_ty, 3547 Operand index_ty, 3548 RegisterOperand vector_ty, 3549 string mnemonic> 3550 : I<(outs tile_ty:$ZAd), 3551 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, 3552 vector_ty:$Zn), 3553 mnemonic, 3554 "\t$ZAd[$Rs, $imm], $Zn", 3555 "", []>, Sched<[]> { 3556 bits<2> Rs; 3557 bits<3> Zn; 3558 let Inst{31-24} = 0b11000000; 3559 let Inst{23-22} = sz; 3560 let Inst{21-16} = 0b000100; 3561 let Inst{15} = v; 3562 let Inst{14-13} = Rs; 3563 let Inst{12-10} = 0b001; 3564 let Inst{9-7} = Zn; 3565 let Inst{6-3} = 0b0000; 3566 let Inst{2-0} = op; 3567 let Constraints = "$ZAd = $_ZAd"; 3568} 3569 3570// SME2 move vector to tile, four registers 3571multiclass sme2_mova_vec_to_tile_vg4_multi_base<bit v, string mnemonic, SDPatternOperator intrinsic> { 3572 3573 def _B : sme2_mova_vec_to_tile_vg4_multi_base<0b00, v, {0,?,?}, 3574 !if(v, TileVectorOpV8, 3575 TileVectorOpH8), 3576 uimm2s4range, ZZZZ_b_mul_r, 3577 mnemonic>, SMEPseudo2Instr<NAME # _B, 1> { 3578 bits<2> imm; 3579 let Inst{1-0} = imm; 3580 } 3581 3582 def _H : sme2_mova_vec_to_tile_vg4_multi_base<0b01, v, {0,?,?}, 3583 !if(v, TileVectorOpV16, 3584 TileVectorOpH16), 3585 uimm1s4range, ZZZZ_h_mul_r, 3586 mnemonic>, SMEPseudo2Instr<NAME # _H, 1> { 3587 bits<1> ZAd; 3588 bits<1> imm; 3589 let Inst{1} = ZAd; 3590 let Inst{0} = imm; 3591 } 3592 3593 def _S : sme2_mova_vec_to_tile_vg4_multi_base<0b10, v, {0,?,?}, 3594 !if(v, TileVectorOpV32, 3595 TileVectorOpH32), 3596 uimm0s4range, ZZZZ_s_mul_r, 3597 mnemonic>, SMEPseudo2Instr<NAME # _S, 1> { 3598 bits<2> ZAd; 3599 let Inst{1-0} = ZAd; 3600 } 3601 3602 def _D : sme2_mova_vec_to_tile_vg4_multi_base<0b11, v, {?,?,?}, 3603 !if(v, TileVectorOpV64, 3604 TileVectorOpH64), 3605 uimm0s4range, ZZZZ_d_mul_r, 3606 mnemonic>, SMEPseudo2Instr<NAME # _D, 1> { 3607 bits<3> ZAd; 3608 let Inst{2-0} = ZAd; 3609 } 3610 3611 def NAME # _B_PSEUDO : sme2_move_to_tile_pseudo<NAME # _B, sme_elm_idx0_0, uimm2s4range, ZZZZ_b_mul_r, SMEMatrixTileB>; 3612 def NAME # _H_PSEUDO : sme2_move_to_tile_pseudo<NAME # _H, sme_elm_idx0_1, uimm1s4range, ZZZZ_h_mul_r, SMEMatrixTileH>; 3613 def NAME # _S_PSEUDO : sme2_move_to_tile_pseudo<NAME # _S, sme_elm_idx0_3, uimm0s4range, ZZZZ_s_mul_r, SMEMatrixTileS>; 3614 def NAME # _D_PSEUDO : sme2_move_to_tile_pseudo<NAME # _D, sme_elm_idx0_7, uimm0s4range, ZZZZ_d_mul_r, SMEMatrixTileD>; 3615 3616 def : SME2_Tile_VG4_Multi_Pat<NAME # _B, intrinsic, sme_elm_idx0_0, nxv16i8, uimm2s4range, tileslicerange2s4>; 3617 def : SME2_Tile_VG4_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8i16, uimm1s4range, tileslicerange1s4>; 3618 def : SME2_Tile_VG4_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8f16, uimm1s4range, tileslicerange1s4>; 3619 def : SME2_Tile_VG4_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8bf16, uimm1s4range, tileslicerange1s4>; 3620 def : SME2_Tile_VG4_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4i32, uimm0s4range, tileslicerange0s4>; 3621 def : SME2_Tile_VG4_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4f32, uimm0s4range, tileslicerange0s4>; 3622 def : SME2_Tile_VG4_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2i64, uimm0s4range, tileslicerange0s4>; 3623 def : SME2_Tile_VG4_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2f64, uimm0s4range, tileslicerange0s4>; 3624 3625 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B), 3626 !if(v, TileVectorOpV8, 3627 TileVectorOpH8), 3628 MatrixIndexGPR32Op12_15, 3629 uimm2s4range, ZZZZ_b_mul_r, 3630 "mov">; 3631 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H), 3632 !if(v, TileVectorOpV16, 3633 TileVectorOpH16), 3634 MatrixIndexGPR32Op12_15, 3635 uimm1s4range, ZZZZ_h_mul_r, 3636 "mov">; 3637 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S), 3638 !if(v, TileVectorOpV32, 3639 TileVectorOpH32), 3640 MatrixIndexGPR32Op12_15, 3641 uimm0s4range, ZZZZ_s_mul_r, 3642 "mov">; 3643 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D), 3644 !if(v, TileVectorOpV64, 3645 TileVectorOpH64), 3646 MatrixIndexGPR32Op12_15, 3647 uimm0s4range, ZZZZ_d_mul_r, 3648 "mov">; 3649 3650 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), 3651 !if(v, TileVectorOpV8, 3652 TileVectorOpH8), 3653 MatrixIndexGPR32Op12_15, 3654 uimm2s4range, ZZZZ_b_mul_r, 3655 "mova">; 3656 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), 3657 !if(v, TileVectorOpV16, 3658 TileVectorOpH16), 3659 MatrixIndexGPR32Op12_15, 3660 uimm1s4range, ZZZZ_h_mul_r, 3661 "mova">; 3662 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), 3663 !if(v, TileVectorOpV32, 3664 TileVectorOpH32), 3665 MatrixIndexGPR32Op12_15, 3666 uimm0s4range, ZZZZ_s_mul_r, 3667 "mova">; 3668 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), 3669 !if(v, TileVectorOpV64, 3670 TileVectorOpH64), 3671 MatrixIndexGPR32Op12_15, 3672 uimm0s4range, ZZZZ_d_mul_r, 3673 "mova">; 3674 3675} 3676 3677multiclass sme2_mova_vec_to_tile_vg4_multi<string mnemonic, 3678 SDPatternOperator int_h, SDPatternOperator int_v>{ 3679 defm _H : sme2_mova_vec_to_tile_vg4_multi_base<0b0, mnemonic, int_h>; 3680 defm _V : sme2_mova_vec_to_tile_vg4_multi_base<0b1, mnemonic, int_v>; 3681} 3682 3683// SME Move into Array 3684class sme2_mova_vec_to_array_vg24_multi< bits<5> op, RegisterOperand array_ty, 3685 RegisterOperand vector_ty, 3686 string mnemonic, 3687 string vg_acronym=""> 3688 : I<(outs array_ty:$ZAd), 3689 (ins array_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm, 3690 vector_ty:$Zn), 3691 mnemonic, "\t$ZAd[$Rs, $imm, " # vg_acronym # "], $Zn", 3692 "", []>, Sched<[]> { 3693 bits<2> Rs; 3694 bits<3> imm; 3695 let Inst{31-15} = 0b11000000000001000; 3696 let Inst{14-13} = Rs; 3697 let Inst{12-11} = 0b01; 3698 let Inst{10-6} = op; 3699 let Inst{5-3} = 0b000; 3700 let Inst{2-0} = imm; 3701 3702 let Constraints = "$ZAd = $_ZAd"; 3703} 3704 3705// MOVA (vector to array, two registers) 3706multiclass sme2_mova_vec_to_array_vg2_multi<string mnemonic, SDPatternOperator intrinsic> { 3707 def NAME : sme2_mova_vec_to_array_vg24_multi<{0,?,?,?,?}, MatrixOp64, 3708 ZZ_d_mul_r, mnemonic, "vgx2">, SMEPseudo2Instr<NAME, 1> { 3709 bits<4> Zn; 3710 let Inst{9-6} = Zn; 3711 } 3712 3713 def NAME # _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, ZZ_d_mul_r, SMEMatrixArray>; 3714 3715 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv2i64, sme_elm_idx0_7, tileslice16>; 3716 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv2f64, sme_elm_idx0_7, tileslice16>; 3717 3718 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3719 MatrixOp8, 3720 MatrixIndexGPR32Op8_11, 3721 sme_elm_idx0_7, ZZ_b_mul_r, 3722 "mova">; 3723 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3724 MatrixOp16, 3725 MatrixIndexGPR32Op8_11, 3726 sme_elm_idx0_7, ZZ_h_mul_r, 3727 "mova">; 3728 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3729 MatrixOp32, 3730 MatrixIndexGPR32Op8_11, 3731 sme_elm_idx0_7, ZZ_s_mul_r, 3732 "mova">; 3733 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3734 MatrixOp64, 3735 MatrixIndexGPR32Op8_11, 3736 sme_elm_idx0_7, ZZ_d_mul_r, 3737 "mova">; 3738 3739 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3740 MatrixOp8, 3741 MatrixIndexGPR32Op8_11, 3742 sme_elm_idx0_7, ZZ_b_mul_r, 3743 "mova", "vgx2">; 3744 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3745 MatrixOp16, 3746 MatrixIndexGPR32Op8_11, 3747 sme_elm_idx0_7, ZZ_h_mul_r, 3748 "mova", "vgx2">; 3749 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3750 MatrixOp32, 3751 MatrixIndexGPR32Op8_11, 3752 sme_elm_idx0_7, ZZ_s_mul_r, 3753 "mova", "vgx2">; 3754 3755 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3756 MatrixOp8, 3757 MatrixIndexGPR32Op8_11, 3758 sme_elm_idx0_7, ZZ_b_mul_r, 3759 "mov">; 3760 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3761 MatrixOp16, 3762 MatrixIndexGPR32Op8_11, 3763 sme_elm_idx0_7, ZZ_h_mul_r, 3764 "mov">; 3765 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3766 MatrixOp32, 3767 MatrixIndexGPR32Op8_11, 3768 sme_elm_idx0_7, ZZ_s_mul_r, 3769 "mov">; 3770 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3771 MatrixOp64, 3772 MatrixIndexGPR32Op8_11, 3773 sme_elm_idx0_7, ZZ_d_mul_r, 3774 "mov">; 3775 3776 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3777 MatrixOp8, 3778 MatrixIndexGPR32Op8_11, 3779 sme_elm_idx0_7, ZZ_b_mul_r, 3780 "mov", "vgx2">; 3781 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3782 MatrixOp16, 3783 MatrixIndexGPR32Op8_11, 3784 sme_elm_idx0_7, ZZ_h_mul_r, 3785 "mov", "vgx2">; 3786 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3787 MatrixOp32, 3788 MatrixIndexGPR32Op8_11, 3789 sme_elm_idx0_7, ZZ_s_mul_r, 3790 "mov", "vgx2">; 3791 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME), 3792 MatrixOp64, 3793 MatrixIndexGPR32Op8_11, 3794 sme_elm_idx0_7, ZZ_d_mul_r, 3795 "mov", "vgx2">; 3796} 3797 3798// MOVA (vector to array, four registers) 3799multiclass sme2_mova_vec_to_array_vg4_multi<string mnemonic, SDPatternOperator intrinsic> { 3800 def NAME : sme2_mova_vec_to_array_vg24_multi<{1,?,?,?,0}, MatrixOp64, 3801 ZZZZ_d_mul_r, mnemonic, "vgx4">, SMEPseudo2Instr<NAME, 1> { 3802 bits<3> Zn; 3803 let Inst{9-7} = Zn; 3804 } 3805 3806 def NAME # _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, ZZZZ_d_mul_r, SMEMatrixArray>; 3807 3808 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv2i64, sme_elm_idx0_7, tileslice16>; 3809 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv2f64, sme_elm_idx0_7, tileslice16>; 3810 3811 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3812 MatrixOp8, 3813 MatrixIndexGPR32Op8_11, 3814 sme_elm_idx0_7, ZZZZ_b_mul_r, 3815 "mova">; 3816 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3817 MatrixOp16, 3818 MatrixIndexGPR32Op8_11, 3819 sme_elm_idx0_7, ZZZZ_h_mul_r, 3820 "mova">; 3821 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3822 MatrixOp32, 3823 MatrixIndexGPR32Op8_11, 3824 sme_elm_idx0_7, ZZZZ_s_mul_r, 3825 "mova">; 3826 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3827 MatrixOp64, 3828 MatrixIndexGPR32Op8_11, 3829 sme_elm_idx0_7, ZZZZ_d_mul_r, 3830 "mova">; 3831 3832 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3833 MatrixOp8, 3834 MatrixIndexGPR32Op8_11, 3835 sme_elm_idx0_7, ZZZZ_b_mul_r, 3836 "mova", "vgx4">; 3837 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3838 MatrixOp16, 3839 MatrixIndexGPR32Op8_11, 3840 sme_elm_idx0_7, ZZZZ_h_mul_r, 3841 "mova", "vgx4">; 3842 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3843 MatrixOp32, 3844 MatrixIndexGPR32Op8_11, 3845 sme_elm_idx0_7, ZZZZ_s_mul_r, 3846 "mova", "vgx4">; 3847 3848 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3849 MatrixOp8, 3850 MatrixIndexGPR32Op8_11, 3851 sme_elm_idx0_7, ZZZZ_b_mul_r, 3852 "mov">; 3853 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3854 MatrixOp16, 3855 MatrixIndexGPR32Op8_11, 3856 sme_elm_idx0_7, ZZZZ_h_mul_r, 3857 "mov">; 3858 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3859 MatrixOp32, 3860 MatrixIndexGPR32Op8_11, 3861 sme_elm_idx0_7, ZZZZ_s_mul_r, 3862 "mov">; 3863 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3864 MatrixOp64, 3865 MatrixIndexGPR32Op8_11, 3866 sme_elm_idx0_7, ZZZZ_d_mul_r, 3867 "mov">; 3868 3869 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3870 MatrixOp8, 3871 MatrixIndexGPR32Op8_11, 3872 sme_elm_idx0_7, ZZZZ_b_mul_r, 3873 "mov", "vgx4">; 3874 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3875 MatrixOp16, 3876 MatrixIndexGPR32Op8_11, 3877 sme_elm_idx0_7, ZZZZ_h_mul_r, 3878 "mov", "vgx4">; 3879 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3880 MatrixOp32, 3881 MatrixIndexGPR32Op8_11, 3882 sme_elm_idx0_7, ZZZZ_s_mul_r, 3883 "mov", "vgx4">; 3884 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME), 3885 MatrixOp64, 3886 MatrixIndexGPR32Op8_11, 3887 sme_elm_idx0_7, ZZZZ_d_mul_r, 3888 "mov", "vgx4">; 3889 3890} 3891 3892class sme2_mova_tile_to_vec_vg2_multi_base<bits<2> sz, bit v, bits<3> op, 3893 RegisterOperand vector_ty, 3894 RegisterOperand tile_ty, 3895 Operand index_ty, 3896 string mnemonic> 3897 : I<!if(op{1}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)), 3898 (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), 3899 mnemonic, 3900 "\t$Zd, $ZAn[$Rs, $imm]", 3901 "", []>, Sched<[]> { 3902 bits<4> Zd; 3903 bits<2> Rs; 3904 let Inst{31-24} = 0b11000000; 3905 let Inst{23-22} = sz; 3906 let Inst{21-16} = 0b000110; 3907 let Inst{15} = v; 3908 let Inst{14-13} = Rs; 3909 let Inst{12-11} = 0b00; 3910 let Inst{10-8} = op; 3911 let Inst{4-1} = Zd; 3912 let Inst{0} = 0b0; 3913 3914 let Constraints = !if(op{1}, "$ZAn = $_ZAn", ""); 3915} 3916 3917multiclass sme2_mova_tile_or_array_to_vec_aliases<int op, Instruction inst, 3918 RegisterOperand vector_ty, 3919 RegisterOperand tile_or_array_ty, 3920 RegisterOperand rv_ty, 3921 Operand index_ty, 3922 string mnemonic, 3923 string vg_acronym=""> { 3924def : InstAlias<mnemonic # "\t$Zd, $ZAn[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]", 3925 (inst vector_ty:$Zd, tile_or_array_ty:$ZAn, rv_ty:$Rs, index_ty:$imm), op>; 3926 3927} 3928 3929multiclass sme2_mova_tile_to_vec_vg2_multi_inst<bit v, bits<3> opc, string mnemonic> { 3930 3931 def _B : sme2_mova_tile_to_vec_vg2_multi_base<0b00, v, opc, ZZ_b_mul_r, 3932 !if(v, TileVectorOpV8, 3933 TileVectorOpH8), 3934 uimm3s2range, mnemonic> { 3935 bits<3> imm; 3936 let Inst{7-5} = imm; 3937 } 3938 3939 def _H : sme2_mova_tile_to_vec_vg2_multi_base<0b01, v, opc, ZZ_h_mul_r, 3940 !if(v, TileVectorOpV16, 3941 TileVectorOpH16), 3942 uimm2s2range, mnemonic> { 3943 bits<1> ZAn; 3944 bits<2> imm; 3945 let Inst{7} = ZAn; 3946 let Inst{6-5} = imm; 3947 } 3948 3949 def _S : sme2_mova_tile_to_vec_vg2_multi_base<0b10, v, opc, ZZ_s_mul_r, 3950 !if(v, TileVectorOpV32, 3951 TileVectorOpH32), 3952 uimm1s2range, mnemonic> { 3953 bits<2> ZAn; 3954 bits<1> imm; 3955 let Inst{7-6} = ZAn; 3956 let Inst{5} = imm; 3957 } 3958 3959 def _D : sme2_mova_tile_to_vec_vg2_multi_base<0b11, v, opc, ZZ_d_mul_r, 3960 !if(v, TileVectorOpV64, 3961 TileVectorOpH64), 3962 uimm0s2range, mnemonic> { 3963 bits<3> ZAn; 3964 let Inst{7-5} = ZAn; 3965 } 3966 3967 if !eq(mnemonic, "mova") then { 3968 defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _B), 3969 ZZ_b_mul_r, 3970 !if(v, TileVectorOpV8, 3971 TileVectorOpH8), 3972 MatrixIndexGPR32Op12_15, 3973 uimm3s2range, "mov">; 3974 defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _H), 3975 ZZ_h_mul_r, 3976 !if(v, TileVectorOpV16, 3977 TileVectorOpH16), 3978 MatrixIndexGPR32Op12_15, 3979 uimm2s2range, "mov">; 3980 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S), 3981 ZZ_s_mul_r, 3982 !if(v, TileVectorOpV32, 3983 TileVectorOpH32), 3984 MatrixIndexGPR32Op12_15, 3985 uimm1s2range, "mov">; 3986 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D), 3987 ZZ_d_mul_r, 3988 !if(v, TileVectorOpV64, 3989 TileVectorOpH64), 3990 MatrixIndexGPR32Op12_15, 3991 uimm0s2range, "mov">; 3992 } 3993 3994 defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _B), 3995 ZZ_b_mul_r, 3996 !if(v, TileVectorOpV8, 3997 TileVectorOpH8), 3998 MatrixIndexGPR32Op12_15, 3999 uimm3s2range, mnemonic>; 4000 defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _H), 4001 ZZ_h_mul_r, 4002 !if(v, TileVectorOpV16, 4003 TileVectorOpH16), 4004 MatrixIndexGPR32Op12_15, 4005 uimm2s2range, mnemonic>; 4006 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S), 4007 ZZ_s_mul_r, 4008 !if(v, TileVectorOpV32, 4009 TileVectorOpH32), 4010 MatrixIndexGPR32Op12_15, 4011 uimm1s2range, mnemonic>; 4012 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D), 4013 ZZ_d_mul_r, 4014 !if(v, TileVectorOpV64, 4015 TileVectorOpH64), 4016 MatrixIndexGPR32Op12_15, 4017 uimm0s2range, mnemonic>; 4018 4019} 4020 4021// SME2 move tile to vector, two registers 4022multiclass sme2_mova_tile_to_vec_vg2_multi<string mnemonic>{ 4023 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b000, mnemonic>; 4024 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b000, mnemonic>; 4025} 4026 4027// SME2p1 move tile to vector and zero tile, two registers 4028multiclass sme2p1_movaz_tile_to_vec_vg2<string mnemonic>{ 4029 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b010, mnemonic>; 4030 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b010, mnemonic>; 4031} 4032 4033class sme2_mova_tile_to_vec_vg4_multi_base<bits<2> sz, bit v, bits<6> op, 4034 RegisterOperand vector_ty, 4035 RegisterOperand tile_ty, 4036 Operand index_ty, 4037 string mnemonic> 4038 : I<!if(op{4}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)), 4039 (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), 4040 mnemonic, 4041 "\t$Zd, $ZAn[$Rs, $imm]", 4042 "", []>, Sched<[]> { 4043 bits<3> Zd; 4044 bits<2> Rs; 4045 let Inst{31-24} = 0b11000000; 4046 let Inst{23-22} = sz; 4047 let Inst{21-16} = 0b000110; 4048 let Inst{15} = v; 4049 let Inst{14-13} = Rs; 4050 let Inst{12-11} = 0b00; 4051 let Inst{10-5} = op{5-0}; 4052 let Inst{4-2} = Zd; 4053 let Inst{1-0} = 0b00; 4054 4055 let Constraints = !if(op{4}, "$ZAn = $_ZAn", ""); 4056} 4057 4058multiclass sme2_mova_tile_to_vec_vg4_multi_base<bit v, bits<3> opc, string mnemonic> { 4059 4060 def _B : sme2_mova_tile_to_vec_vg4_multi_base<0b00, v, {opc,0,?,?}, 4061 ZZZZ_b_mul_r, 4062 !if(v, TileVectorOpV8, 4063 TileVectorOpH8), 4064 uimm2s4range, mnemonic> { 4065 bits<2> imm; 4066 let Inst{6-5} = imm; 4067 } 4068 4069 def _H : sme2_mova_tile_to_vec_vg4_multi_base<0b01, v, {opc,0,?,?}, 4070 ZZZZ_h_mul_r, 4071 !if(v, TileVectorOpV16, 4072 TileVectorOpH16), 4073 uimm1s4range, mnemonic> { 4074 bits<1> ZAn; 4075 bits<1> imm; 4076 let Inst{6} = ZAn; 4077 let Inst{5} = imm; 4078 } 4079 4080 def _S : sme2_mova_tile_to_vec_vg4_multi_base<0b10, v, {opc,0,?,?}, 4081 ZZZZ_s_mul_r, 4082 !if(v, TileVectorOpV32, 4083 TileVectorOpH32), 4084 uimm0s4range, mnemonic> { 4085 bits<2> ZAn; 4086 let Inst{6-5} = ZAn; 4087 } 4088 4089 def _D : sme2_mova_tile_to_vec_vg4_multi_base<0b11, v, {opc,?,?,?}, 4090 ZZZZ_d_mul_r, 4091 !if(v, TileVectorOpV64, 4092 TileVectorOpH64), 4093 uimm0s4range, mnemonic> { 4094 bits<3> ZAn; 4095 let Inst{7-5} = ZAn; 4096 } 4097 4098 if !eq(mnemonic, "mova") then { 4099 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _B), 4100 ZZZZ_b_mul_r, 4101 !if(v, TileVectorOpV8, 4102 TileVectorOpH8), 4103 MatrixIndexGPR32Op12_15, 4104 uimm2s4range, "mov">; 4105 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _H), 4106 ZZZZ_h_mul_r, 4107 !if(v, TileVectorOpV16, 4108 TileVectorOpH16), 4109 MatrixIndexGPR32Op12_15, 4110 uimm1s4range, "mov">; 4111 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S), 4112 ZZZZ_s_mul_r, 4113 !if(v, TileVectorOpV32, 4114 TileVectorOpH32), 4115 MatrixIndexGPR32Op12_15, 4116 uimm0s4range, "mov">; 4117 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D), 4118 ZZZZ_d_mul_r, 4119 !if(v, TileVectorOpV64, 4120 TileVectorOpH64), 4121 MatrixIndexGPR32Op12_15, 4122 uimm0s4range, "mov">; 4123 } 4124 4125 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _B), 4126 ZZZZ_b_mul_r, 4127 !if(v, TileVectorOpV8, 4128 TileVectorOpH8), 4129 MatrixIndexGPR32Op12_15, 4130 uimm2s4range, mnemonic>; 4131 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _H), 4132 ZZZZ_h_mul_r, 4133 !if(v, TileVectorOpV16, 4134 TileVectorOpH16), 4135 MatrixIndexGPR32Op12_15, 4136 uimm1s4range, mnemonic>; 4137 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S), 4138 ZZZZ_s_mul_r, 4139 !if(v, TileVectorOpV32, 4140 TileVectorOpH32), 4141 MatrixIndexGPR32Op12_15, 4142 uimm0s4range, mnemonic>; 4143 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D), 4144 ZZZZ_d_mul_r, 4145 !if(v, TileVectorOpV64, 4146 TileVectorOpH64), 4147 MatrixIndexGPR32Op12_15, 4148 uimm0s4range, mnemonic>; 4149 4150} 4151 4152// SME2 move tile to vector, four registers 4153multiclass sme2_mova_tile_to_vec_vg4_multi<string mnemonic>{ 4154 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b100, mnemonic>; 4155 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b100, mnemonic>; 4156} 4157 4158// SME2p1 move tile to vector and zero tile, four registers 4159multiclass sme2p1_movaz_tile_to_vec_vg4<string mnemonic>{ 4160 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b110, mnemonic>; 4161 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b110, mnemonic>; 4162} 4163 4164 4165class sme2_mova_array_to_vec_vg24_multi<bits<4>op, RegisterOperand vector_ty, 4166 RegisterOperand array_ty, 4167 string mnemonic, string vg_acronym> 4168 : I<!if(op{2}, (outs vector_ty:$Zd, array_ty:$_ZAn), (outs vector_ty:$Zd)), 4169 (ins array_ty:$ZAn, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm), 4170 mnemonic, 4171 "\t$Zd, $ZAn[$Rs, $imm, " # vg_acronym # "]", 4172 "", []>, Sched<[]> { 4173 bits<2> Rs; 4174 bits<3> imm; 4175 let Inst{31-15} = 0b11000000000001100; 4176 let Inst{14-13} = Rs; 4177 let Inst{12-11} = 0b01; 4178 let Inst{10-8} = op{3-1}; 4179 let Inst{7-5} = imm; 4180 let Inst{1} = op{0}; 4181 let Inst{0} = 0b0; 4182 let Constraints = !if(op{2}, "$ZAn = $_ZAn", ""); 4183} 4184 4185// move array to vector, two registers. 4186multiclass sme2_mova_array_to_vec_vg2_multi<bits<3> opc, string mnemonic> { 4187 def NAME : sme2_mova_array_to_vec_vg24_multi<{opc,?}, ZZ_d_mul_r, MatrixOp64, 4188 mnemonic, "vgx2"> { 4189 bits<4> Zd; 4190 let Inst{4-1} = Zd; 4191 } 4192 4193 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4194 ZZ_b_mul_r, MatrixOp8, 4195 MatrixIndexGPR32Op8_11, 4196 sme_elm_idx0_7, mnemonic>; 4197 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4198 ZZ_h_mul_r, MatrixOp16, 4199 MatrixIndexGPR32Op8_11, 4200 sme_elm_idx0_7, mnemonic>; 4201 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4202 ZZ_s_mul_r, MatrixOp32, 4203 MatrixIndexGPR32Op8_11, 4204 sme_elm_idx0_7, mnemonic>; 4205 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4206 ZZ_d_mul_r, MatrixOp64, 4207 MatrixIndexGPR32Op8_11, 4208 sme_elm_idx0_7, mnemonic>; 4209 4210 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4211 ZZ_b_mul_r, MatrixOp8, 4212 MatrixIndexGPR32Op8_11, 4213 sme_elm_idx0_7, mnemonic, "vgx2">; 4214 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4215 ZZ_h_mul_r, MatrixOp16, 4216 MatrixIndexGPR32Op8_11, 4217 sme_elm_idx0_7, mnemonic, "vgx2">; 4218 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4219 ZZ_s_mul_r, MatrixOp32, 4220 MatrixIndexGPR32Op8_11, 4221 sme_elm_idx0_7, mnemonic, "vgx2">; 4222 4223 if !eq(mnemonic, "mova") then { 4224 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4225 ZZ_b_mul_r, MatrixOp8, 4226 MatrixIndexGPR32Op8_11, 4227 sme_elm_idx0_7, "mov">; 4228 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4229 ZZ_h_mul_r, MatrixOp16, 4230 MatrixIndexGPR32Op8_11, 4231 sme_elm_idx0_7, "mov">; 4232 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4233 ZZ_s_mul_r, MatrixOp32, 4234 MatrixIndexGPR32Op8_11, 4235 sme_elm_idx0_7, "mov">; 4236 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4237 ZZ_d_mul_r, MatrixOp64, 4238 MatrixIndexGPR32Op8_11, 4239 sme_elm_idx0_7, "mov">; 4240 4241 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4242 ZZ_b_mul_r, MatrixOp8, 4243 MatrixIndexGPR32Op8_11, 4244 sme_elm_idx0_7, "mov", "vgx2">; 4245 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4246 ZZ_h_mul_r, MatrixOp16, 4247 MatrixIndexGPR32Op8_11, 4248 sme_elm_idx0_7, "mov", "vgx2">; 4249 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4250 ZZ_s_mul_r, MatrixOp32, 4251 MatrixIndexGPR32Op8_11, 4252 sme_elm_idx0_7, "mov", "vgx2">; 4253 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME), 4254 ZZ_d_mul_r, MatrixOp64, 4255 MatrixIndexGPR32Op8_11, 4256 sme_elm_idx0_7, "mov", "vgx2">; 4257 } 4258} 4259 4260// move array to vector, four registers 4261multiclass sme2_mova_array_to_vec_vg4_multi<bits<4> opc, string mnemonic> { 4262 def NAME : sme2_mova_array_to_vec_vg24_multi<opc, ZZZZ_d_mul_r, MatrixOp64, 4263 mnemonic, "vgx4"> { 4264 bits<3> Zd; 4265 let Inst{4-2} = Zd; 4266 } 4267 4268 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4269 ZZZZ_b_mul_r, MatrixOp8, 4270 MatrixIndexGPR32Op8_11, 4271 sme_elm_idx0_7, mnemonic>; 4272 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4273 ZZZZ_h_mul_r, MatrixOp16, 4274 MatrixIndexGPR32Op8_11, 4275 sme_elm_idx0_7, mnemonic>; 4276 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4277 ZZZZ_s_mul_r, MatrixOp32, 4278 MatrixIndexGPR32Op8_11, 4279 sme_elm_idx0_7, mnemonic>; 4280 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4281 ZZZZ_d_mul_r, MatrixOp64, 4282 MatrixIndexGPR32Op8_11, 4283 sme_elm_idx0_7, mnemonic>; 4284 4285 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4286 ZZZZ_b_mul_r, MatrixOp8, 4287 MatrixIndexGPR32Op8_11, 4288 sme_elm_idx0_7, mnemonic, "vgx4">; 4289 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4290 ZZZZ_h_mul_r, MatrixOp16, 4291 MatrixIndexGPR32Op8_11, 4292 sme_elm_idx0_7, mnemonic, "vgx4">; 4293 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4294 ZZZZ_s_mul_r, MatrixOp32, 4295 MatrixIndexGPR32Op8_11, 4296 sme_elm_idx0_7, mnemonic, "vgx4">; 4297 4298 if !eq(mnemonic, "mova") then { 4299 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4300 ZZZZ_b_mul_r, MatrixOp8, 4301 MatrixIndexGPR32Op8_11, 4302 sme_elm_idx0_7, "mov">; 4303 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4304 ZZZZ_h_mul_r, MatrixOp16, 4305 MatrixIndexGPR32Op8_11, 4306 sme_elm_idx0_7, "mov">; 4307 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4308 ZZZZ_s_mul_r, MatrixOp32, 4309 MatrixIndexGPR32Op8_11, 4310 sme_elm_idx0_7, "mov">; 4311 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4312 ZZZZ_d_mul_r, MatrixOp64, 4313 MatrixIndexGPR32Op8_11, 4314 sme_elm_idx0_7, "mov">; 4315 4316 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4317 ZZZZ_b_mul_r, MatrixOp8, 4318 MatrixIndexGPR32Op8_11, 4319 sme_elm_idx0_7, "mov", "vgx4">; 4320 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4321 ZZZZ_h_mul_r, MatrixOp16, 4322 MatrixIndexGPR32Op8_11, 4323 sme_elm_idx0_7, "mov", "vgx4">; 4324 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4325 ZZZZ_s_mul_r, MatrixOp32, 4326 MatrixIndexGPR32Op8_11, 4327 sme_elm_idx0_7, "mov", "vgx4">; 4328 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME), 4329 ZZZZ_d_mul_r, MatrixOp64, 4330 MatrixIndexGPR32Op8_11, 4331 sme_elm_idx0_7, "mov", "vgx4">; 4332 } 4333} 4334 4335//===----------------------------------------------------------------------===// 4336// SME2 multi-vec saturating shift right narrow 4337class sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u> 4338 : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn, tvecshiftR16:$imm4), 4339 mnemonic, "\t$Zd, $Zn, $imm4", 4340 "", []>, Sched<[]> { 4341 bits<4> imm4; 4342 bits<4> Zn; 4343 bits<5> Zd; 4344 let Inst{31-21} = 0b11000001111; 4345 let Inst{20} = op; 4346 let Inst{19-16} = imm4; 4347 let Inst{15-10} = 0b110101; 4348 let Inst{9-6} = Zn; 4349 let Inst{5} = u; 4350 let Inst{4-0} = Zd; 4351} 4352 4353multiclass sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u, SDPatternOperator intrinsic> { 4354 def _H : sme2_sat_shift_vector_vg2<mnemonic, op, u>; 4355 4356 def : SME2_Sat_Shift_VG2_Pat<NAME # _H, intrinsic, nxv8i16, nxv4i32, tvecshiftR16>; 4357} 4358 4359class sme2_sat_shift_vector_vg4<bits<2> sz, bits<3> op, ZPRRegOp zpr_ty, 4360 RegisterOperand vector_ty, Operand imm_ty, 4361 string mnemonic> 4362 : I<(outs zpr_ty:$Zd), (ins vector_ty:$Zn, imm_ty:$imm), 4363 mnemonic, "\t$Zd, $Zn, $imm", 4364 "", []>, Sched<[]> { 4365 bits<3> Zn; 4366 bits<5> Zd; 4367 let Inst{31-24} = 0b11000001; 4368 let Inst{23-22} = sz; 4369 let Inst{21} = 0b1; 4370 // Inst{20-16} = imm5; 4371 let Inst{15-11} = 0b11011; 4372 let Inst{10} = op{2}; 4373 let Inst{9-7} = Zn; 4374 let Inst{6-5} = op{1-0}; 4375 let Inst{4-0} = Zd; 4376} 4377 4378multiclass sme2_sat_shift_vector_vg4<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 4379 def _B : sme2_sat_shift_vector_vg4<{0,1}, op, ZPR8, ZZZZ_s_mul_r, tvecshiftR32, 4380 mnemonic>{ 4381 bits<5> imm; 4382 let Inst{20-16} = imm; 4383 } 4384 def _H : sme2_sat_shift_vector_vg4<{1,?}, op, ZPR16, ZZZZ_d_mul_r, tvecshiftR64, 4385 mnemonic> { 4386 bits<6> imm; 4387 let Inst{22} = imm{5}; 4388 let Inst{20-16} = imm{4-0}; 4389 } 4390 4391 def : SME2_Sat_Shift_VG4_Pat<NAME # _B, intrinsic, nxv16i8, nxv4i32, tvecshiftR32>; 4392 def : SME2_Sat_Shift_VG4_Pat<NAME # _H, intrinsic, nxv8i16, nxv2i64, tvecshiftR64>; 4393} 4394 4395//===----------------------------------------------------------------------===// 4396// SME2 Multi-vector - SVE Select 4397class sme2_sel_vector_vg24<bits<2> sz, bits<4> op, RegisterOperand vector_ty, 4398 string mnemonic> 4399 : I<(outs vector_ty:$Zd), 4400 (ins PNRAny_p8to15:$PNg, vector_ty:$Zn, vector_ty:$Zm), 4401 mnemonic, "\t$Zd, $PNg, $Zn, $Zm", 4402 "", []>, Sched<[]> { 4403 bits<3> PNg; 4404 let Inst{31-24} = 0b11000001; 4405 let Inst{23-22} = sz; 4406 let Inst{21} = 0b1; 4407 let Inst{17-16} = op{3-2}; 4408 let Inst{15-13} = 0b100; 4409 let Inst{12-10} = PNg; 4410 let Inst{6} = op{1}; 4411 let Inst{5} = 0b0; 4412 let Inst{1} = op{0}; 4413 let Inst{0} = 0b0; 4414} 4415 4416class sme2_sel_vector_vg2<bits<2> sz, RegisterOperand vector_ty, 4417 string mnemonic> 4418 : sme2_sel_vector_vg24<sz, {?,0,?,?}, vector_ty, mnemonic> { 4419 bits<4> Zm; 4420 bits<4> Zn; 4421 bits<4> Zd; 4422 let Inst{20-17} = Zm; 4423 let Inst{9-6} = Zn; 4424 let Inst{4-1} = Zd; 4425} 4426 4427multiclass sme2_sel_vector_vg2<string mnemonic>{ 4428 def _B : sme2_sel_vector_vg2<0b00, ZZ_b_mul_r, mnemonic>; 4429 def _H : sme2_sel_vector_vg2<0b01, ZZ_h_mul_r, mnemonic>; 4430 def _S : sme2_sel_vector_vg2<0b10, ZZ_s_mul_r, mnemonic>; 4431 def _D : sme2_sel_vector_vg2<0b11, ZZ_d_mul_r, mnemonic>; 4432} 4433class sme2_sel_vector_vg4<bits<2> sz, RegisterOperand vector_ty, 4434 string mnemonic> 4435 : sme2_sel_vector_vg24<sz, 0b0100, vector_ty, mnemonic> { 4436 bits<3> Zm; 4437 bits<3> Zn; 4438 bits<3> Zd; 4439 let Inst{20-18} = Zm; 4440 let Inst{9-7} = Zn; 4441 let Inst{4-2} = Zd; 4442} 4443multiclass sme2_sel_vector_vg4<string mnemonic> { 4444 def _B : sme2_sel_vector_vg4<0b00, ZZZZ_b_mul_r, mnemonic>; 4445 def _H : sme2_sel_vector_vg4<0b01, ZZZZ_h_mul_r, mnemonic>; 4446 def _S : sme2_sel_vector_vg4<0b10, ZZZZ_s_mul_r, mnemonic>; 4447 def _D : sme2_sel_vector_vg4<0b11, ZZZZ_d_mul_r, mnemonic>; 4448} 4449 4450//===----------------------------------------------------------------------===// 4451// Non contiguous Load and Store 4452 4453class sme2_ld_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n, 4454 RegisterOperand multi_vector_ty, 4455 RegisterOperand gpr_ty, 4456 string mnemonic> 4457 : I<(outs multi_vector_ty:$Zt), 4458 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4459 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", 4460 "", []>, Sched<[]> { 4461 bits<5> Rm; 4462 bits<3> PNg; 4463 bits<5> Rn; 4464 bits<4> Zt; 4465 let Inst{31-21} = 0b10100001000; 4466 let Inst{20-16} = Rm; 4467 let Inst{15} = 0b0; 4468 let Inst{14-13} = msz; 4469 let Inst{12-10} = PNg; 4470 let Inst{9-5} = Rn; 4471 let Inst{4} = Zt{3}; 4472 let Inst{3} = n; 4473 let Inst{2-0} = Zt{2-0}; 4474 4475 let mayLoad = 1; 4476} 4477 4478class sme2_ld_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n, 4479 RegisterOperand multi_vector_ty, 4480 RegisterOperand gpr_ty, 4481 string mnemonic> 4482 : I<(outs multi_vector_ty:$Zt), 4483 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4484 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", 4485 "", []>, Sched<[]> { 4486 bits<5> Rm; 4487 bits<3> PNg; 4488 bits<5> Rn; 4489 bits<3> Zt; 4490 let Inst{31-21} = 0b10100001000; 4491 let Inst{20-16} = Rm; 4492 let Inst{15} = 0b1; 4493 let Inst{14-13} = msz; 4494 let Inst{12-10} = PNg; 4495 let Inst{9-5} = Rn; 4496 let Inst{4} = Zt{2}; 4497 let Inst{3} = n; 4498 let Inst{2} = 0b0; 4499 let Inst{1-0} = Zt{1-0}; 4500 4501 let mayLoad = 1; 4502} 4503 4504class sme2_ld_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op, 4505 RegisterOperand multi_vector_ty, 4506 Operand index_ty, 4507 string mnemonic> 4508 : I<(outs multi_vector_ty:$Zt), 4509 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4), 4510 mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]", 4511 "", []>, Sched<[]> { 4512 bits<4> imm4; 4513 bits<3> PNg; 4514 bits<5> Rn; 4515 let Inst{31-20} = 0b101000010100; 4516 let Inst{19-16} = imm4; 4517 let Inst{15} = op{1}; 4518 let Inst{14-13} = msz; 4519 let Inst{12-10} = PNg; 4520 let Inst{9-5} = Rn; 4521 let Inst{3} = n; 4522 let Inst{2} = op{0}; 4523 4524 let mayLoad = 1; 4525} 4526 4527multiclass sme2_ld_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n, 4528 RegisterOperand multi_vector_ty, 4529 Operand index_ty, 4530 string mnemonic>{ 4531 def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, {0,?}, 4532 multi_vector_ty, 4533 index_ty, mnemonic> { 4534 bits<4> Zt; 4535 let Inst{4} = Zt{3}; 4536 let Inst{2-0} = Zt{2-0}; 4537 } 4538 4539 def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]", 4540 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; 4541} 4542 4543multiclass sme2_ld_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n, 4544 RegisterOperand multi_vector_ty, 4545 Operand index_ty, 4546 string mnemonic> { 4547 def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, 0b10, 4548 multi_vector_ty, 4549 index_ty, mnemonic> { 4550 bits<3> Zt; 4551 let Inst{4} = Zt{2}; 4552 let Inst{1-0} = Zt{1-0}; 4553 } 4554 4555 def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]", 4556 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; 4557} 4558 4559//===----------------------------------------------------------------------===// 4560// SME2 Non-Contiguous Store 4561class sme2_st_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n, 4562 RegisterOperand multi_vector_ty, 4563 RegisterOperand gpr_ty, 4564 string mnemonic> 4565 : I<(outs ), 4566 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4567 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", 4568 "", []>, Sched<[]> { 4569 bits<5> Rm; 4570 bits<3> PNg; 4571 bits<5> Rn; 4572 bits<4> Zt; 4573 let Inst{31-21} = 0b10100001001; 4574 let Inst{20-16} = Rm; 4575 let Inst{15} = 0b0; 4576 let Inst{14-13} = msz; 4577 let Inst{12-10} = PNg; 4578 let Inst{9-5} = Rn; 4579 let Inst{4} = Zt{3}; 4580 let Inst{3} = n; 4581 let Inst{2-0} = Zt{2-0}; 4582 4583 let mayStore = 1; 4584} 4585 4586class sme2_st_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n, 4587 RegisterOperand multi_vector_ty, 4588 RegisterOperand gpr_ty, 4589 string mnemonic> 4590 : I<(outs ), 4591 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4592 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", 4593 "", []>, Sched<[]> { 4594 bits<5> Rm; 4595 bits<3> PNg; 4596 bits<5> Rn; 4597 bits<3> Zt; 4598 let Inst{31-21} = 0b10100001001; 4599 let Inst{20-16} = Rm; 4600 let Inst{15} = 0b1; 4601 let Inst{14-13} = msz; 4602 let Inst{12-10} = PNg; 4603 let Inst{9-5} = Rn; 4604 let Inst{4} = Zt{2}; 4605 let Inst{3} = n; 4606 let Inst{2} = 0b0; 4607 let Inst{1-0} = Zt{1-0}; 4608 4609 let mayStore = 1; 4610} 4611 4612class sme2_st_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op, 4613 RegisterOperand multi_vector_ty, 4614 Operand index_ty, 4615 string mnemonic> 4616 : I<(outs ), 4617 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4), 4618 mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]", 4619 "", []>, Sched<[]> { 4620 bits<4> imm4; 4621 bits<3> PNg; 4622 bits<5> Rn; 4623 let Inst{31-20} = 0b101000010110; 4624 let Inst{19-16} = imm4; 4625 let Inst{15} = op{1}; 4626 let Inst{14-13} = msz; 4627 let Inst{12-10} = PNg; 4628 let Inst{9-5} = Rn; 4629 let Inst{3} = n; 4630 let Inst{2} = op{0}; 4631 4632 let mayStore = 1; 4633} 4634 4635 4636multiclass sme2_st_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n, 4637 RegisterOperand multi_vector_ty, 4638 Operand index_ty, 4639 string mnemonic> { 4640 def NAME: sme2_st_vector_vg24_multi_scalar_immediate<msz, n, {0,?}, 4641 multi_vector_ty, 4642 index_ty, mnemonic> { 4643 bits<4> Zt; 4644 let Inst{4} = Zt{3}; 4645 let Inst{2-0} = Zt{2-0}; 4646 } 4647 4648 def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]", 4649 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>; 4650} 4651 4652multiclass sme2_st_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n, 4653 RegisterOperand multi_vector_ty, 4654 Operand index_ty, 4655 string mnemonic> { 4656 def NAME : sme2_st_vector_vg24_multi_scalar_immediate<msz, n, 0b10, 4657 multi_vector_ty, 4658 index_ty, mnemonic> { 4659 bits<3> Zt; 4660 let Inst{4} = Zt{2}; 4661 let Inst{1-0} = Zt{1-0}; 4662 } 4663 4664 def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]", 4665 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>; 4666} 4667 4668//===----------------------------------------------------------------------===// 4669// SME2.1 4670//===----------------------------------------------------------------------===// 4671// SME zeroing move array to vector 4672class sme2p1_movaz_tile_to_vec_base<bits<2> sz, bit q, bit v, ZPRRegOp vector_ty, 4673 RegisterOperand tile_ty, Operand index_ty, 4674 string mnemonic> 4675 : I<(outs vector_ty:$Zd, tile_ty:$ZAn), 4676 (ins tile_ty:$_ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), 4677 mnemonic, "\t$Zd, $ZAn[$Rs, $imm]", 4678 "", []>, Sched<[]> { 4679 bits<2> Rs; 4680 bits<5> Zd; 4681 let Inst{31-24} = 0b11000000; 4682 let Inst{23-22} = sz; 4683 let Inst{21-17} = 0b00001; 4684 let Inst{16} = q; 4685 let Inst{15} = v; 4686 let Inst{14-13} = Rs; 4687 let Inst{12-9} = 0b0001; 4688 let Inst{4-0} = Zd; 4689 let Constraints = "$ZAn = $_ZAn"; 4690} 4691 4692multiclass sme2p1_movaz_tile_to_vec_base<bit v, string mnemonic> { 4693 def _B : sme2p1_movaz_tile_to_vec_base<0b00, 0b0, v, ZPR8, 4694 !if(v, TileVectorOpV8, TileVectorOpH8), 4695 sme_elm_idx0_15, mnemonic> { 4696 bits<4> imm; 4697 let Inst{8-5} = imm; 4698 } 4699 4700 def _H : sme2p1_movaz_tile_to_vec_base<0b01, 0b0, v, ZPR16, 4701 !if(v, TileVectorOpV16, TileVectorOpH16), 4702 sme_elm_idx0_7, mnemonic> { 4703 bits<1> ZAn; 4704 bits<3> imm; 4705 let Inst{8} = ZAn; 4706 let Inst{7-5} = imm; 4707 } 4708 4709 def _S : sme2p1_movaz_tile_to_vec_base<0b10, 0b0, v, ZPR32, 4710 !if(v, TileVectorOpV32, TileVectorOpH32), 4711 sme_elm_idx0_3, mnemonic> { 4712 bits<2> ZAn; 4713 bits<2> imm; 4714 let Inst{8-7} = ZAn; 4715 let Inst{6-5} = imm; 4716 } 4717 4718 def _D : sme2p1_movaz_tile_to_vec_base<0b11, 0b0, v, ZPR64, 4719 !if(v, TileVectorOpV64, TileVectorOpH64), 4720 sme_elm_idx0_1, mnemonic> { 4721 bits<3> ZAn; 4722 bits<1> imm; 4723 let Inst{8-6} = ZAn; 4724 let Inst{5} = imm; 4725 } 4726 4727 def _Q : sme2p1_movaz_tile_to_vec_base<0b11, 0b1, v, ZPR128, 4728 !if(v, TileVectorOpV128, TileVectorOpH128), 4729 sme_elm_idx0_0, mnemonic> { 4730 bits<4> ZAn; 4731 let Inst{8-5} = ZAn; 4732 } 4733} 4734 4735multiclass sme2p1_movaz_tile_to_vec<string mnemonic>{ 4736 defm _H : sme2p1_movaz_tile_to_vec_base<0b0, mnemonic>; 4737 defm _V : sme2p1_movaz_tile_to_vec_base<0b1, mnemonic>; 4738} 4739 4740//===----------------------------------------------------------------------===// 4741// SME2.1 multiple vectors zero array 4742 4743class sme2p1_zero_matrix<bits<6> opc, Operand index_ty, string mnemonic, 4744 string vg_acronym=""> 4745 : I<(outs MatrixOp64:$ZAd), 4746 (ins MatrixOp64:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm), 4747 mnemonic, "\t$ZAd[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]", 4748 "", []>, Sched<[]> { 4749 bits <2> Rv; 4750 let Inst{31-18} = 0b11000000000011; 4751 let Inst{17-15} = opc{5-3}; 4752 let Inst{14-13} = Rv; 4753 let Inst{12-3} = 0b0000000000; 4754 let Inst{2-0} = opc{2-0}; 4755 let Constraints = "$ZAd = $_ZAd"; 4756} 4757 4758multiclass sme2p1_zero_matrix<string mnemonic> { 4759 def _VG2_Z : sme2p1_zero_matrix<{0b000,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx2"> { 4760 bits<3> imm; 4761 let Inst{2-0} = imm; 4762 } 4763 def _2Z : sme2p1_zero_matrix<{0b001,?,?,?}, uimm3s2range, mnemonic> { 4764 bits<3> imm; 4765 let Inst{2-0} = imm; 4766 } 4767 def _VG2_2Z : sme2p1_zero_matrix<{0b0100,?,?}, uimm2s2range, mnemonic, "vgx2"> { 4768 bits<2> imm; 4769 let Inst{1-0} = imm; 4770 } 4771 def _VG4_2Z : sme2p1_zero_matrix<{0b0110,?,?}, uimm2s2range, mnemonic, "vgx4"> { 4772 bits<2> imm; 4773 let Inst{1-0} = imm; 4774 } 4775 def _VG4_Z : sme2p1_zero_matrix<{0b100,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx4"> { 4776 bits<3> imm; 4777 let Inst{2-0} = imm; 4778 } 4779 def _4Z : sme2p1_zero_matrix<{0b1010,?,?}, uimm2s4range, mnemonic> { 4780 bits<2> imm; 4781 let Inst{1-0} = imm; 4782 } 4783 def _VG2_4Z :sme2p1_zero_matrix<{0b11000,?}, uimm1s4range, mnemonic, "vgx2"> { 4784 bits<1> imm; 4785 let Inst{0} = imm; 4786 } 4787 def _VG4_4Z :sme2p1_zero_matrix<{0b11100,?}, uimm1s4range, mnemonic, "vgx4"> { 4788 bits<1> imm; 4789 let Inst{0} = imm; 4790 } 4791} 4792 4793//===----------------------------------------------------------------------===// 4794// SME2.1 lookup table expand two non-contiguous registers 4795 4796class sme2p1_luti_vector_vg2_index<bits<4> op, bits<2> sz, RegisterOperand vector_ty, 4797 AsmVectorIndexOpnd index_ty, 4798 string mnemonic> 4799 : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 4800 mnemonic, "\t$Zd, $ZTt, $Zn$i", 4801 "", []>, Sched<[]> { 4802 bits<5> Zn; 4803 bits<4> Zd; 4804 let Inst{31-19} = 0b1100000010011; 4805 let Inst{18-15} = op; 4806 let Inst{14} = 0b1; 4807 let Inst{13-12} = sz; 4808 let Inst{11-10} = 0b00; 4809 let Inst{9-5} = Zn; 4810 let Inst{4} = Zd{3}; 4811 let Inst{3} = 0b0; 4812 let Inst{2-0} = Zd{2-0}; 4813} 4814 4815class sme2p1_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 4816 AsmVectorIndexOpnd index_ty, 4817 string mnemonic> 4818 : sme2p1_luti_vector_vg2_index<{1,?,?,?}, sz, vector_ty, index_ty, mnemonic> { 4819 bits<3> i; 4820 let Inst{17-15} = i; 4821} 4822 4823multiclass sme2p1_luti2_vector_vg2_index<string mnemonic> { 4824 def _B : sme2p1_luti2_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexH, 4825 mnemonic>; 4826 def _H : sme2p1_luti2_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexH, 4827 mnemonic>; 4828} 4829 4830class sme2p1_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 4831 AsmVectorIndexOpnd index_ty, 4832 string mnemonic> 4833 : sme2p1_luti_vector_vg2_index<{0b01,?,?}, sz, vector_ty, index_ty, mnemonic> { 4834 bits<2> i; 4835 let Inst{16-15} = i; 4836} 4837multiclass sme2p1_luti4_vector_vg2_index<string mnemonic> { 4838 def _B : sme2p1_luti4_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexS, 4839 mnemonic>; 4840 def _H : sme2p1_luti4_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexS, 4841 mnemonic>; 4842} 4843 4844// SME2.1 lookup table expand four non-contiguous registers 4845class sme2p1_luti_vector_vg4_index<bits<3> op, bits<2> sz, RegisterOperand vector_ty, 4846 AsmVectorIndexOpnd index_ty, 4847 string mnemonic> 4848 : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 4849 mnemonic, "\t$Zd, $ZTt, $Zn$i", 4850 "", []>, Sched<[]> { 4851 bits<5> Zn; 4852 bits<3> Zd; 4853 let Inst{31-19} = 0b1100000010011; 4854 let Inst{18-16} = op; 4855 let Inst{15-14} = 0b10; 4856 let Inst{13-12} = sz; 4857 let Inst{11-10} = 0b00; 4858 let Inst{9-5} = Zn; 4859 let Inst{4} = Zd{2}; 4860 let Inst{3-2} = 0b00; 4861 let Inst{1-0} = Zd{1-0}; 4862} 4863 4864class sme2p1_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 4865 AsmVectorIndexOpnd index_ty, 4866 string mnemonic> 4867 : sme2p1_luti_vector_vg4_index<{1,?,?}, sz, vector_ty, index_ty, mnemonic> { 4868 bits<2> i; 4869 let Inst{17-16} = i; 4870} 4871 4872multiclass sme2p1_luti2_vector_vg4_index<string mnemonic> { 4873 def _B : sme2p1_luti2_vector_vg4_index<0b00, ZZZZ_b_strided, VectorIndexS, 4874 mnemonic>; 4875 def _H : sme2p1_luti2_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexS, 4876 mnemonic>; 4877} 4878 4879class sme2p1_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 4880 AsmVectorIndexOpnd index_ty, 4881 string mnemonic> 4882 : sme2p1_luti_vector_vg4_index<{0b01,?}, sz, vector_ty, index_ty, mnemonic> { 4883 bit i; 4884 let Inst{16} = i; 4885} 4886 4887multiclass sme2p1_luti4_vector_vg4_index<string mnemonic> { 4888 def _H: sme2p1_luti4_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexD, mnemonic>; 4889} 4890 4891// SME2 lookup table two source registers expand to four contiguous destination registers 4892class sme2_luti4_vector_vg4<bits<2> sz, bits<2> op, string mnemonic> 4893 : I<(outs ZZZZ_b_mul_r:$Zd), (ins ZTR:$ZTt, ZZ_mul_r:$Zn), 4894 mnemonic, "\t$Zd, $ZTt, $Zn", 4895 "", []>, Sched<[]> { 4896 bits<4> Zn; 4897 bits<3> Zd; 4898 let Inst{31-14} = 0b110000001000101100; 4899 let Inst{13-12} = sz; 4900 let Inst{11-10} = op; 4901 let Inst{9-6} = Zn; 4902 let Inst{5} = 0b0; 4903 let Inst{4-2} = Zd; 4904 let Inst{1-0} = 0b00; 4905} 4906 4907// SME2 lookup table two source registers expand to four non-contiguous destination registers 4908class sme2_luti4_vector_vg4_strided<bits<2> sz, bits<2> op, string mnemonic> 4909 : I<(outs ZZZZ_b_strided:$Zd), (ins ZTR:$ZTt, ZZ_mul_r:$Zn), 4910 mnemonic, "\t$Zd, $ZTt, $Zn", 4911 "", []>, Sched<[]> { 4912 bits<4> Zn; 4913 bits<3> Zd; 4914 let Inst{31-14} = 0b110000001001101100; 4915 let Inst{13-12} = sz; 4916 let Inst{11-10} = op; 4917 let Inst{9-6} = Zn; 4918 let Inst{5} = 0b0; 4919 let Inst{4} = Zd{2}; 4920 let Inst{3-2} = 0b00; 4921 let Inst{1-0} = Zd{1-0}; 4922} 4923