1//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def SDT_AArch64Setcc : SDTypeProfile<1, 4, [ 14 SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, 15 SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>, 16 SDTCisVT<4, OtherVT> 17]>; 18 19def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>; 20 21def SVEPatternOperand : AsmOperandClass { 22 let Name = "SVEPattern"; 23 let ParserMethod = "tryParseSVEPattern"; 24 let PredicateMethod = "isSVEPattern"; 25 let RenderMethod = "addImmOperands"; 26 let DiagnosticType = "InvalidSVEPattern"; 27} 28 29def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{ 30 return (((uint32_t)Imm) < 32); 31 }]> { 32 33 let PrintMethod = "printSVEPattern"; 34 let ParserMatchClass = SVEPatternOperand; 35} 36 37def SVEPrefetchOperand : AsmOperandClass { 38 let Name = "SVEPrefetch"; 39 let ParserMethod = "tryParsePrefetch<true>"; 40 let PredicateMethod = "isPrefetch"; 41 let RenderMethod = "addPrefetchOperands"; 42} 43 44def sve_prfop : Operand<i32>, TImmLeaf<i32, [{ 45 return (((uint32_t)Imm) <= 15); 46 }]> { 47 let PrintMethod = "printPrefetchOp<true>"; 48 let ParserMatchClass = SVEPrefetchOperand; 49} 50 51class SVELogicalImmOperand<int Width> : AsmOperandClass { 52 let Name = "SVELogicalImm" # Width; 53 let DiagnosticType = "LogicalSecondSource"; 54 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 55 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 56} 57 58def sve_logical_imm8 : Operand<i64> { 59 let ParserMatchClass = SVELogicalImmOperand<8>; 60 let PrintMethod = "printLogicalImm<int8_t>"; 61 62 let MCOperandPredicate = [{ 63 if (!MCOp.isImm()) 64 return false; 65 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 66 return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val); 67 }]; 68} 69 70def sve_logical_imm16 : Operand<i64> { 71 let ParserMatchClass = SVELogicalImmOperand<16>; 72 let PrintMethod = "printLogicalImm<int16_t>"; 73 74 let MCOperandPredicate = [{ 75 if (!MCOp.isImm()) 76 return false; 77 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 78 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val); 79 }]; 80} 81 82def sve_logical_imm32 : Operand<i64> { 83 let ParserMatchClass = SVELogicalImmOperand<32>; 84 let PrintMethod = "printLogicalImm<int32_t>"; 85 86 let MCOperandPredicate = [{ 87 if (!MCOp.isImm()) 88 return false; 89 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 90 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val); 91 }]; 92} 93 94class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass { 95 let Name = "SVEPreferredLogicalImm" # Width; 96 let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>"; 97 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 98} 99 100def sve_preferred_logical_imm16 : Operand<i64> { 101 let ParserMatchClass = SVEPreferredLogicalImmOperand<16>; 102 let PrintMethod = "printSVELogicalImm<int16_t>"; 103 104 let MCOperandPredicate = [{ 105 if (!MCOp.isImm()) 106 return false; 107 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 108 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) && 109 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 110 }]; 111} 112 113def sve_preferred_logical_imm32 : Operand<i64> { 114 let ParserMatchClass = SVEPreferredLogicalImmOperand<32>; 115 let PrintMethod = "printSVELogicalImm<int32_t>"; 116 117 let MCOperandPredicate = [{ 118 if (!MCOp.isImm()) 119 return false; 120 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 121 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) && 122 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 123 }]; 124} 125 126def sve_preferred_logical_imm64 : Operand<i64> { 127 let ParserMatchClass = SVEPreferredLogicalImmOperand<64>; 128 let PrintMethod = "printSVELogicalImm<int64_t>"; 129 130 let MCOperandPredicate = [{ 131 if (!MCOp.isImm()) 132 return false; 133 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 134 return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) && 135 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 136 }]; 137} 138 139class SVELogicalImmNotOperand<int Width> : AsmOperandClass { 140 let Name = "SVELogicalImm" # Width # "Not"; 141 let DiagnosticType = "LogicalSecondSource"; 142 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 143 let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>"; 144} 145 146def sve_logical_imm8_not : Operand<i64> { 147 let ParserMatchClass = SVELogicalImmNotOperand<8>; 148} 149 150def sve_logical_imm16_not : Operand<i64> { 151 let ParserMatchClass = SVELogicalImmNotOperand<16>; 152} 153 154def sve_logical_imm32_not : Operand<i64> { 155 let ParserMatchClass = SVELogicalImmNotOperand<32>; 156} 157 158class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate> 159 : AsmOperandClass { 160 let Name = "SVE" # Infix # "Imm" # ElementWidth; 161 let DiagnosticType = "Invalid" # Name; 162 let RenderMethod = "addImmWithOptionalShiftOperands<8>"; 163 let ParserMethod = "tryParseImmWithOptionalShift"; 164 let PredicateMethod = Predicate; 165} 166 167def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">; 168def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">; 169def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">; 170def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">; 171 172def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">; 173def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">; 174def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">; 175def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">; 176 177class imm8_opt_lsl<int ElementWidth, string printType, 178 AsmOperandClass OpndClass> 179 : Operand<i32> { 180 let EncoderMethod = "getImm8OptLsl"; 181 let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">"; 182 let PrintMethod = "printImm8OptLsl<" # printType # ">"; 183 let ParserMatchClass = OpndClass; 184 let MIOperandInfo = (ops i32imm, i32imm); 185} 186 187def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>; 188def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>; 189def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>; 190def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>; 191 192def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>; 193def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>; 194def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>; 195def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>; 196 197def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; 198def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; 199def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; 200def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64>", []>; 201 202def SVECpyDupImm8Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i8>", []>; 203def SVECpyDupImm16Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i16>", []>; 204def SVECpyDupImm32Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i32>", []>; 205def SVECpyDupImm64Pat : ComplexPattern<i64, 2, "SelectSVECpyDupImm<MVT::i64>", []>; 206 207def SVELogicalImm8Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8>", []>; 208def SVELogicalImm16Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16>", []>; 209def SVELogicalImm32Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32>", []>; 210def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>; 211 212def SVELogicalImm8NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8, true>", []>; 213def SVELogicalImm16NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16, true>", []>; 214def SVELogicalImm32NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32, true>", []>; 215def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>; 216 217def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>; 218def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>; 219def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>; 220def SVEArithUImm64Pat : ComplexPattern<i64, 1, "SelectSVEArithImm<MVT::i64>", []>; 221 222def SVEArithSImmPat32 : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; 223def SVEArithSImmPat64 : ComplexPattern<i64, 1, "SelectSVESignedArithImm", []>; 224 225def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>; 226def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>; 227def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>; 228def SVEShiftImmL64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<0, 63>", []>; 229def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>; 230def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>; 231def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>; 232def SVEShiftImmR64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<1, 64, true>", []>; 233 234def SVEShiftSplatImmR : ComplexPattern<iAny, 1, "SelectSVEShiftSplatImmR", []>; 235 236def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>; 237 238class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass { 239 let Name = "SVEExactFPImmOperand" # Suffix; 240 let DiagnosticType = "Invalid" # Name; 241 let ParserMethod = "tryParseFPImm<false>"; 242 let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">"; 243 let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">"; 244} 245 246class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> { 247 let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">"; 248 let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>; 249} 250 251def sve_fpimm_half_one 252 : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half", 253 "AArch64ExactFPImm::one">; 254def sve_fpimm_half_two 255 : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half", 256 "AArch64ExactFPImm::two">; 257def sve_fpimm_zero_one 258 : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero", 259 "AArch64ExactFPImm::one">; 260 261def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{ 262 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17); 263}]> { 264 let ParserMatchClass = Imm1_16Operand; 265 let EncoderMethod = "getSVEIncDecImm"; 266 let DecoderMethod = "DecodeSVEIncDecImm"; 267} 268 269// This allows i32 immediate extraction from i64 based arithmetic. 270def sve_cnt_mul_imm_i32 : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; 271def sve_cnt_mul_imm_i64 : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, false>">; 272def sve_cnt_shl_imm : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, true>">; 273 274def sve_ext_imm_0_31 : ComplexPattern<i64, 1, "SelectEXTImm<31, 8>">; 275def sve_ext_imm_0_63 : ComplexPattern<i64, 1, "SelectEXTImm<63, 4>">; 276def sve_ext_imm_0_127 : ComplexPattern<i64, 1, "SelectEXTImm<127, 2>">; 277def sve_ext_imm_0_255 : ComplexPattern<i64, 1, "SelectEXTImm<255, 1>">; 278 279def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), 280 (int_aarch64_sve_cntp node:$pred, node:$src2), [{ 281 return N->hasOneUse(); 282}]>; 283 284def step_vector_oneuse : PatFrag<(ops node:$idx), 285 (step_vector node:$idx), [{ 286 return N->hasOneUse(); 287}]>; 288 289 290//===----------------------------------------------------------------------===// 291// SVE PTrue - These are used extensively throughout the pattern matching so 292// it's important we define them first. 293//===----------------------------------------------------------------------===// 294 295class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 296 ValueType vt, SDPatternOperator op> 297: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern), 298 asm, "\t$Pd, $pattern", 299 "", 300 [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> { 301 bits<4> Pd; 302 bits<5> pattern; 303 let Inst{31-24} = 0b00100101; 304 let Inst{23-22} = sz8_64; 305 let Inst{21-19} = 0b011; 306 let Inst{18-17} = opc{2-1}; 307 let Inst{16} = opc{0}; 308 let Inst{15-10} = 0b111000; 309 let Inst{9-5} = pattern; 310 let Inst{4} = 0b0; 311 let Inst{3-0} = Pd; 312 313 let Defs = !if(!eq (opc{0}, 1), [NZCV], []); 314 let ElementSize = pprty.ElementSize; 315 let isReMaterializable = 1; 316} 317 318multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> { 319 def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>; 320 def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>; 321 def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>; 322 def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>; 323 324 def : InstAlias<asm # "\t$Pd", 325 (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>; 326 def : InstAlias<asm # "\t$Pd", 327 (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>; 328 def : InstAlias<asm # "\t$Pd", 329 (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>; 330 def : InstAlias<asm # "\t$Pd", 331 (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>; 332} 333 334def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; 335def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>; 336 337let Predicates = [HasSVEorSME] in { 338 defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>; 339 defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>; 340 341 def : Pat<(nxv16i1 immAllOnesV), (PTRUE_B 31)>; 342 def : Pat<(nxv8i1 immAllOnesV), (PTRUE_H 31)>; 343 def : Pat<(nxv4i1 immAllOnesV), (PTRUE_S 31)>; 344 def : Pat<(nxv2i1 immAllOnesV), (PTRUE_D 31)>; 345} 346 347//===----------------------------------------------------------------------===// 348// SVE pattern match helpers. 349//===----------------------------------------------------------------------===// 350 351class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 352 Instruction inst> 353: Pat<(vtd (op vt1:$Op1)), 354 (inst $Op1)>; 355 356class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 357 ValueType vts, Instruction inst> 358: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)), 359 (inst $Op3, $Op1, $Op2)>; 360 361 362multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 363 ValueType vts, Instruction inst> { 364 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd undef))), 365 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 366 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, vtd:$Op3)), 367 (inst $Op3, $Op1, $Op2)>; 368} 369 370// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the 371// type of rounding. This is matched by timm0_1 in pattern below and ignored. 372class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 373 ValueType vts, Instruction inst> 374: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)), 375 (inst $Op3, $Op1, $Op2)>; 376 377multiclass SVE_1_Op_PassthruUndef_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 378 ValueType vts, Instruction inst>{ 379 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), (vtd undef))), 380 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 381 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, (i64 timm0_1), vtd:$Op3)), 382 (inst $Op3, $Op1, $Op2)>; 383} 384 385class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 386 ValueType it, ComplexPattern cpx, Instruction inst> 387 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm, i32:$shift)))))), 388 (inst $Op1, i32:$imm, i32:$shift)>; 389 390class SVE_1_Op_Imm_Arith_All_Active<ValueType vt, ValueType pt, SDPatternOperator op, 391 ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst> 392 : Pat<(vt (op (pt (SVEAllActive)), (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm)))))), 393 (inst $Op1, i32:$imm)>; 394 395class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 396 ValueType it, ComplexPattern cpx, Instruction inst> 397 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i64:$imm)))))), 398 (inst $Op1, i64:$imm)>; 399 400class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 401 ValueType vt2, Instruction inst> 402: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 403 (inst $Op1, $Op2)>; 404 405class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op, 406 ValueType pt, ValueType vt1, ValueType vt2, 407 Instruction inst> 408: Pat<(vtd (op (pt (SVEAllActive)), vt1:$Op1, vt2:$Op2)), 409 (inst $Op1, $Op2)>; 410 411class SVE_2_Op_Pred_All_Active_Pt<ValueType vtd, SDPatternOperator op, 412 ValueType pt, ValueType vt1, ValueType vt2, 413 Instruction inst> 414: Pat<(vtd (op (pt (SVEAllActive:$Op1)), vt1:$Op2, vt2:$Op3)), 415 (inst $Op1, $Op2, $Op3)>; 416 417class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 418 ValueType vt2, ValueType vt3, Instruction inst> 419: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 420 (inst $Op1, $Op2, $Op3)>; 421 422multiclass SVE_3_Op_Undef_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 423 ValueType vt2, ValueType vt3, Instruction inst> { 424 def : Pat<(vtd (op (vt1 undef), vt2:$Op1, vt3:$Op2)), 425 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 426 def : Pat<(vtd (op vt1:$Op1, (vt2 (SVEAllActive:$Op2)), vt3:$Op3)), 427 (inst $Op1, $Op2, $Op3)>; 428} 429 430class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 431 ValueType vt2, ValueType vt3, ValueType vt4, 432 Instruction inst> 433: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)), 434 (inst $Op1, $Op2, $Op3, $Op4)>; 435 436class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 437 ValueType vt2, Operand ImmTy, Instruction inst> 438: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))), 439 (inst $Op1, ImmTy:$Op2)>; 440 441class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 442 ValueType vt2, ValueType vt3, Operand ImmTy, 443 Instruction inst> 444: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))), 445 (inst $Op1, $Op2, ImmTy:$Op3)>; 446 447class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 448 ValueType vt2, ValueType vt3, ValueType vt4, 449 Operand ImmTy, Instruction inst> 450: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), 451 (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; 452 453def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>; 454def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>; 455 456let AddedComplexity = 1 in { 457class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1, 458 ValueType vt2, ValueType vt3, Instruction inst> 459: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))), 460 (inst $Op1, $Op2, $Op3)>; 461 462class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op, 463 ValueType vt1, ValueType vt2, 464 Operand vt3, Instruction inst> 465: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))), 466 (inst $Op1, $Op2, vt3:$Op3)>; 467} 468 469// 470// Common but less generic patterns. 471// 472 473class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 474 Instruction inst, Instruction ptrue> 475: Pat<(vtd (op vt1:$Op1)), 476 (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>; 477 478class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 479 ValueType vt2, Instruction inst, Instruction ptrue> 480: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 481 (inst (ptrue 31), $Op1, $Op2)>; 482 483class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt, 484 ValueType inreg_vt, Instruction inst> 485: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)), 486 (inst $PassThru, $Pg, $Src)>; 487 488multiclass SVE_InReg_Extend_PassthruUndef<ValueType vt, SDPatternOperator op, ValueType pt, 489 ValueType inreg_vt, Instruction inst> { 490 def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt undef))), 491 (inst (IMPLICIT_DEF), $Pg, $Src)>; 492 def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, vt:$PassThru)), 493 (inst $PassThru, $Pg, $Src)>; 494} 495 496class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op, 497 ValueType pt, ValueType it, 498 ComplexPattern cast, Instruction inst> 499: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))), 500 (inst $Pg, $Rn, i32:$imm)>; 501 502class SVE_Shift_DupImm_All_Active_Pat<ValueType vt, SDPatternOperator op, 503 ValueType pt, ValueType it, 504 ComplexPattern cast, Instruction inst> 505: Pat<(vt (op (pt (SVEAllActive)), vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))), 506 (inst $Rn, i32:$imm)>; 507 508class SVE_2_Op_Fp_Imm_Pat<ValueType vt, SDPatternOperator op, 509 ValueType pt, ValueType it, 510 FPImmLeaf immL, int imm, 511 Instruction inst> 512: Pat<(vt (op (pt PPR_3b:$Pg), (vt ZPR:$Zs1), (vt (splat_vector (it immL))))), 513 (inst $Pg, $Zs1, imm)>; 514 515class SVE_2_Op_Fp_Imm_Pat_Zero<ValueType vt, SDPatternOperator op, 516 ValueType pt, ValueType it, 517 FPImmLeaf immL, int imm, 518 Instruction inst> 519: Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Zs1, (SVEDup0)), 520 (vt (splat_vector (it immL))))), 521 (inst $Pg, $Zs1, imm)>; 522 523// Used to re-order the operands of BSP when lowering to BSL. BSP has the order: 524// mask, in1, in2 whereas BSL for SVE2 has them ordered in1, in2, mask 525class SVE_3_Op_BSP_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 526 ValueType vt2, ValueType vt3, Instruction inst> 527: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 528 (inst $Op2, $Op3, $Op1)>; 529 530class SVE_Shift_Add_All_Active_Pat<ValueType vtd, SDPatternOperator op, ValueType pt, 531 ValueType vt1, ValueType vt2, ValueType vt3, 532 Instruction inst> 533: Pat<(vtd (add vt1:$Op1, (op (pt (SVEAllActive)), vt2:$Op2, vt3:$Op3))), 534 (inst $Op1, $Op2, $Op3)>; 535 536//===----------------------------------------------------------------------===// 537// SVE pattern match helpers. 538//===----------------------------------------------------------------------===// 539 540// Matches either an intrinsic, or a predicated operation with an all active predicate 541class EitherVSelectOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOperator sdnode> 542: PatFrags<(ops node:$Pg, node:$Op1, node:$Op2), [ 543 (intrinsic node:$Pg, node:$Op1, node:$Op2), 544 (vselect node:$Pg, (sdnode (SVEAllActive), node:$Op1, node:$Op2), node:$Op1), 545 ]>; 546 547// 548// Pseudo -> Instruction mappings 549// 550def getSVEPseudoMap : InstrMapping { 551 let FilterClass = "SVEPseudo2Instr"; 552 let RowFields = ["PseudoName"]; 553 let ColFields = ["IsInstr"]; 554 let KeyCol = ["0"]; 555 let ValueCols = [["1"]]; 556} 557 558class SVEPseudo2Instr<string name, bit instr> { 559 string PseudoName = name; 560 bit IsInstr = instr; 561} 562 563// Lookup e.g. DIV -> DIVR 564def getSVERevInstr : InstrMapping { 565 let FilterClass = "SVEInstr2Rev"; 566 let RowFields = ["InstrName"]; 567 let ColFields = ["isReverseInstr"]; 568 let KeyCol = ["0"]; 569 let ValueCols = [["1"]]; 570} 571 572// Lookup e.g. DIVR -> DIV 573def getSVENonRevInstr : InstrMapping { 574 let FilterClass = "SVEInstr2Rev"; 575 let RowFields = ["InstrName"]; 576 let ColFields = ["isReverseInstr"]; 577 let KeyCol = ["1"]; 578 let ValueCols = [["0"]]; 579} 580 581class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> { 582 string InstrName = !if(name1IsReverseInstr, name1, name2); 583 bit isReverseInstr = name1IsReverseInstr; 584} 585 586// 587// Pseudos for destructive operands 588// 589let hasNoSchedulingInfo = 1 in { 590 class PredTwoOpPseudo<string name, ZPRRegOp zprty, 591 FalseLanesEnum flags = FalseLanesNone> 592 : SVEPseudo2Instr<name, 0>, 593 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> { 594 let FalseLanes = flags; 595 } 596 597 class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty, 598 FalseLanesEnum flags = FalseLanesNone> 599 : SVEPseudo2Instr<name, 0>, 600 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> { 601 let FalseLanes = flags; 602 } 603 604 class PredThreeOpPseudo<string name, ZPRRegOp zprty, 605 FalseLanesEnum flags = FalseLanesNone> 606 : SVEPseudo2Instr<name, 0>, 607 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2, zprty:$Zs3), []> { 608 let FalseLanes = flags; 609 } 610} 611 612// 613// Pseudos for passthru operands 614// 615let hasNoSchedulingInfo = 1 in { 616 class PredOneOpPassthruPseudo<string name, ZPRRegOp zprty> 617 : SVEPseudo2Instr<name, 0>, 618 Pseudo<(outs zprty:$Zd), (ins zprty:$Passthru, PPR3bAny:$Pg, zprty:$Zs), []>; 619} 620 621//===----------------------------------------------------------------------===// 622// SVE Predicate Misc Group 623//===----------------------------------------------------------------------===// 624 625class sve_int_pfalse<bits<6> opc, string asm> 626: I<(outs PPR8:$Pd), (ins), 627 asm, "\t$Pd", 628 "", 629 []>, Sched<[]> { 630 bits<4> Pd; 631 let Inst{31-24} = 0b00100101; 632 let Inst{23-22} = opc{5-4}; 633 let Inst{21-19} = 0b011; 634 let Inst{18-16} = opc{3-1}; 635 let Inst{15-10} = 0b111001; 636 let Inst{9} = opc{0}; 637 let Inst{8-4} = 0b00000; 638 let Inst{3-0} = Pd; 639 640 let isReMaterializable = 1; 641} 642 643multiclass sve_int_pfalse<bits<6> opc, string asm> { 644 def NAME : sve_int_pfalse<opc, asm>; 645 646 def : Pat<(nxv16i1 immAllZerosV), (!cast<Instruction>(NAME))>; 647 def : Pat<(nxv8i1 immAllZerosV), (!cast<Instruction>(NAME))>; 648 def : Pat<(nxv4i1 immAllZerosV), (!cast<Instruction>(NAME))>; 649 def : Pat<(nxv2i1 immAllZerosV), (!cast<Instruction>(NAME))>; 650 def : Pat<(nxv1i1 immAllZerosV), (!cast<Instruction>(NAME))>; 651} 652 653class sve_int_ptest<bits<6> opc, string asm> 654: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn), 655 asm, "\t$Pg, $Pn", 656 "", 657 []>, Sched<[]> { 658 bits<4> Pg; 659 bits<4> Pn; 660 let Inst{31-24} = 0b00100101; 661 let Inst{23-22} = opc{5-4}; 662 let Inst{21-19} = 0b010; 663 let Inst{18-16} = opc{3-1}; 664 let Inst{15-14} = 0b11; 665 let Inst{13-10} = Pg; 666 let Inst{9} = opc{0}; 667 let Inst{8-5} = Pn; 668 let Inst{4-0} = 0b00000; 669 670 let Defs = [NZCV]; 671 let isCompare = 1; 672} 673 674class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm, 675 PPRRegOp pprty> 676: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn), 677 asm, "\t$Pdn, $Pg, $_Pdn", 678 "", 679 []>, Sched<[]> { 680 bits<4> Pdn; 681 bits<4> Pg; 682 let Inst{31-24} = 0b00100101; 683 let Inst{23-22} = sz8_64; 684 let Inst{21-19} = 0b011; 685 let Inst{18-16} = opc{4-2}; 686 let Inst{15-11} = 0b11000; 687 let Inst{10-9} = opc{1-0}; 688 let Inst{8-5} = Pg; 689 let Inst{4} = 0; 690 let Inst{3-0} = Pdn; 691 692 let Constraints = "$Pdn = $_Pdn"; 693 let Defs = [NZCV]; 694 let isPTestLike = 1; 695 let ElementSize = pprty.ElementSize; 696} 697 698multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> { 699 def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>; 700 701 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 702} 703 704multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> { 705 def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>; 706 def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>; 707 def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>; 708 def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>; 709 710 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 711 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 712 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 713 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 714} 715 716//===----------------------------------------------------------------------===// 717// SVE Predicate Count Group 718//===----------------------------------------------------------------------===// 719 720class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm, 721 RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty> 722: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn), 723 asm, "\t$Rdn, $Pg", 724 "", 725 []>, Sched<[]> { 726 bits<5> Rdn; 727 bits<4> Pg; 728 let Inst{31-24} = 0b00100101; 729 let Inst{23-22} = sz8_64; 730 let Inst{21-19} = 0b101; 731 let Inst{18-16} = opc{4-2}; 732 let Inst{15-11} = 0b10001; 733 let Inst{10-9} = opc{1-0}; 734 let Inst{8-5} = Pg; 735 let Inst{4-0} = Rdn; 736 737 // Signed 32bit forms require their GPR operand printed. 738 let AsmString = !if(!eq(opc{4,2-0}, 0b0000), 739 !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"), 740 !strconcat(asm, "\t$Rdn, $Pg")); 741 let Constraints = "$Rdn = $_Rdn"; 742} 743 744multiclass sve_int_count_r_s32<bits<5> opc, string asm, 745 SDPatternOperator op> { 746 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>; 747 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>; 748 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>; 749 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>; 750 751 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 752 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 753 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))), 754 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 755 756 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 757 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 758 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))), 759 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 760 761 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 762 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 763 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))), 764 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 765 766 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 767 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 768 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))), 769 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 770} 771 772multiclass sve_int_count_r_u32<bits<5> opc, string asm, 773 SDPatternOperator op> { 774 def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>; 775 def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>; 776 def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>; 777 def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>; 778 779 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 780 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 781 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 782 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 783 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 784 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 785 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 786 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 787} 788 789multiclass sve_int_count_r_x64<bits<5> opc, string asm, 790 SDPatternOperator op, 791 SDPatternOperator combine_op = null_frag> { 792 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>; 793 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>; 794 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>; 795 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>; 796 797 def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))), 798 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 799 def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))), 800 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 801 def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))), 802 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 803 def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))), 804 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 805 806 // combine_op(x, cntp(all_active, p)) ==> inst p, x 807 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 (SVEAllActive)), (nxv16i1 PPRAny:$pred)))), 808 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 809 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 (SVEAllActive)), (nxv8i1 PPRAny:$pred)))), 810 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 811 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 (SVEAllActive)), (nxv4i1 PPRAny:$pred)))), 812 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 813 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 (SVEAllActive)), (nxv2i1 PPRAny:$pred)))), 814 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 815 816 // combine_op(x, cntp(p, p)) ==> inst p, x 817 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 PPRAny:$pred), (nxv16i1 PPRAny:$pred)))), 818 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 819 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 PPRAny:$pred), (nxv8i1 PPRAny:$pred)))), 820 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 821 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 PPRAny:$pred), (nxv4i1 PPRAny:$pred)))), 822 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 823 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 PPRAny:$pred), (nxv2i1 PPRAny:$pred)))), 824 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 825} 826 827class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm, 828 ZPRRegOp zprty, PPRRegOp pprty> 829: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm), 830 asm, "\t$Zdn, $Pm", 831 "", 832 []>, Sched<[]> { 833 bits<4> Pm; 834 bits<5> Zdn; 835 let Inst{31-24} = 0b00100101; 836 let Inst{23-22} = sz8_64; 837 let Inst{21-19} = 0b101; 838 let Inst{18-16} = opc{4-2}; 839 let Inst{15-11} = 0b10000; 840 let Inst{10-9} = opc{1-0}; 841 let Inst{8-5} = Pm; 842 let Inst{4-0} = Zdn; 843 844 let Constraints = "$Zdn = $_Zdn"; 845 let DestructiveInstType = DestructiveOther; 846 let ElementSize = ElementSizeNone; 847} 848 849multiclass sve_int_count_v<bits<5> opc, string asm, 850 SDPatternOperator op = null_frag> { 851 def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>; 852 def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>; 853 def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>; 854 855 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>; 856 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>; 857 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>; 858 859 def : InstAlias<asm # "\t$Zdn, $Pm", 860 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>; 861 def : InstAlias<asm # "\t$Zdn, $Pm", 862 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>; 863 def : InstAlias<asm # "\t$Zdn, $Pm", 864 (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>; 865} 866 867class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm, 868 PPRRegOp pprty> 869: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn), 870 asm, "\t$Rd, $Pg, $Pn", 871 "", 872 []>, Sched<[]> { 873 bits<4> Pg; 874 bits<4> Pn; 875 bits<5> Rd; 876 let Inst{31-24} = 0b00100101; 877 let Inst{23-22} = sz8_64; 878 let Inst{21-19} = 0b100; 879 let Inst{18-16} = opc{3-1}; 880 let Inst{15-14} = 0b10; 881 let Inst{13-10} = Pg; 882 let Inst{9} = opc{0}; 883 let Inst{8-5} = Pn; 884 let Inst{4-0} = Rd; 885} 886 887multiclass sve_int_pcount_pred<bits<4> opc, string asm, 888 SDPatternOperator int_op> { 889 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>; 890 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>; 891 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>; 892 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>; 893 894 def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 895 def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 896 def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 897 def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 898} 899 900//===----------------------------------------------------------------------===// 901// SVE Element Count Group 902//===----------------------------------------------------------------------===// 903 904class sve_int_count<bits<3> opc, string asm> 905: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 906 asm, "\t$Rd, $pattern, mul $imm4", 907 "", 908 []>, Sched<[]> { 909 bits<5> Rd; 910 bits<4> imm4; 911 bits<5> pattern; 912 let Inst{31-24} = 0b00000100; 913 let Inst{23-22} = opc{2-1}; 914 let Inst{21-20} = 0b10; 915 let Inst{19-16} = imm4; 916 let Inst{15-11} = 0b11100; 917 let Inst{10} = opc{0}; 918 let Inst{9-5} = pattern; 919 let Inst{4-0} = Rd; 920 921 let isReMaterializable = 1; 922} 923 924multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> { 925 def NAME : sve_int_count<opc, asm>; 926 927 def : InstAlias<asm # "\t$Rd, $pattern", 928 (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>; 929 def : InstAlias<asm # "\t$Rd", 930 (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>; 931 932 def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm))), 933 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 934 935 def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm))), 936 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 937 938 def : Pat<(i64 (op sve_pred_enum:$pattern)), 939 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>; 940} 941 942class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty> 943: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 944 asm, "\t$Zdn, $pattern, mul $imm4", 945 "", 946 []>, Sched<[]> { 947 bits<5> Zdn; 948 bits<5> pattern; 949 bits<4> imm4; 950 let Inst{31-24} = 0b00000100; 951 let Inst{23-22} = opc{4-3}; 952 let Inst{21} = 0b1; 953 let Inst{20} = opc{2}; 954 let Inst{19-16} = imm4; 955 let Inst{15-12} = 0b1100; 956 let Inst{11-10} = opc{1-0}; 957 let Inst{9-5} = pattern; 958 let Inst{4-0} = Zdn; 959 960 let Constraints = "$Zdn = $_Zdn"; 961 let DestructiveInstType = DestructiveOther; 962 let ElementSize = ElementSizeNone; 963} 964 965multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty, 966 SDPatternOperator op = null_frag, 967 ValueType vt = OtherVT> { 968 def NAME : sve_int_countvlv<opc, asm, zprty>; 969 970 def : InstAlias<asm # "\t$Zdn, $pattern", 971 (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>; 972 def : InstAlias<asm # "\t$Zdn", 973 (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>; 974 975 def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 976 (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 977} 978 979class sve_int_pred_pattern_a<bits<3> opc, string asm> 980: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 981 asm, "\t$Rdn, $pattern, mul $imm4", 982 "", 983 []>, Sched<[]> { 984 bits<5> Rdn; 985 bits<5> pattern; 986 bits<4> imm4; 987 let Inst{31-24} = 0b00000100; 988 let Inst{23-22} = opc{2-1}; 989 let Inst{21-20} = 0b11; 990 let Inst{19-16} = imm4; 991 let Inst{15-11} = 0b11100; 992 let Inst{10} = opc{0}; 993 let Inst{9-5} = pattern; 994 let Inst{4-0} = Rdn; 995 996 let Constraints = "$Rdn = $_Rdn"; 997} 998 999multiclass sve_int_pred_pattern_a<bits<3> opc, string asm, 1000 SDPatternOperator op, 1001 SDPatternOperator opcnt> { 1002 let Predicates = [HasSVEorSME] in { 1003 def NAME : sve_int_pred_pattern_a<opc, asm>; 1004 1005 def : InstAlias<asm # "\t$Rdn, $pattern", 1006 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1007 def : InstAlias<asm # "\t$Rdn", 1008 (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>; 1009 } 1010 1011 let Predicates = [HasSVEorSME, UseScalarIncVL] in { 1012 def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))), 1013 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>; 1014 1015 def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm)))), 1016 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 1017 1018 def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm)))), 1019 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 1020 1021 def : Pat<(i32 (op GPR32:$Rdn, (i32 (trunc (opcnt (sve_pred_enum:$pattern)))))), 1022 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1023 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1), 1024 sub_32))>; 1025 1026 def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm_i32 i32:$imm)))), 1027 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1028 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 1029 sub_32))>; 1030 1031 def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_shl_imm i32:$imm)))), 1032 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1033 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 1034 sub_32))>; 1035 } 1036} 1037 1038class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt, 1039 RegisterOperand st> 1040: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 1041 asm, "\t$Rdn, $pattern, mul $imm4", 1042 "", 1043 []>, Sched<[]> { 1044 bits<5> Rdn; 1045 bits<5> pattern; 1046 bits<4> imm4; 1047 let Inst{31-24} = 0b00000100; 1048 let Inst{23-22} = opc{4-3}; 1049 let Inst{21} = 0b1; 1050 let Inst{20} = opc{2}; 1051 let Inst{19-16} = imm4; 1052 let Inst{15-12} = 0b1111; 1053 let Inst{11-10} = opc{1-0}; 1054 let Inst{9-5} = pattern; 1055 let Inst{4-0} = Rdn; 1056 1057 // Signed 32bit forms require their GPR operand printed. 1058 let AsmString = !if(!eq(opc{2,0}, 0b00), 1059 !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"), 1060 !strconcat(asm, "\t$Rdn, $pattern, mul $imm4")); 1061 1062 let Constraints = "$Rdn = $_Rdn"; 1063} 1064 1065multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm, 1066 SDPatternOperator op> { 1067 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>; 1068 1069 def : InstAlias<asm # "\t$Rd, $Rn, $pattern", 1070 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>; 1071 def : InstAlias<asm # "\t$Rd, $Rn", 1072 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>; 1073 1074 // NOTE: Register allocation doesn't like tied operands of differing register 1075 // class, hence the extra INSERT_SUBREG complication. 1076 1077 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1078 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>; 1079 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))), 1080 (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1081} 1082 1083multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm, 1084 SDPatternOperator op> { 1085 def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>; 1086 1087 def : InstAlias<asm # "\t$Rdn, $pattern", 1088 (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1089 def : InstAlias<asm # "\t$Rdn", 1090 (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>; 1091 1092 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1093 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1094} 1095 1096multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm, 1097 SDPatternOperator op> { 1098 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>; 1099 1100 def : InstAlias<asm # "\t$Rdn, $pattern", 1101 (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1102 def : InstAlias<asm # "\t$Rdn", 1103 (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>; 1104 1105 def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1106 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1107} 1108 1109 1110//===----------------------------------------------------------------------===// 1111// SVE Permute - Cross Lane Group 1112//===----------------------------------------------------------------------===// 1113 1114class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1115 ValueType vt, RegisterClass srcRegType, 1116 SDPatternOperator op> 1117: I<(outs zprty:$Zd), (ins srcRegType:$Rn), 1118 asm, "\t$Zd, $Rn", 1119 "", 1120 [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> { 1121 bits<5> Rn; 1122 bits<5> Zd; 1123 let Inst{31-24} = 0b00000101; 1124 let Inst{23-22} = sz8_64; 1125 let Inst{21-10} = 0b100000001110; 1126 let Inst{9-5} = Rn; 1127 let Inst{4-0} = Zd; 1128} 1129 1130multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> { 1131 def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>; 1132 def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>; 1133 def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>; 1134 def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>; 1135 1136 def : InstAlias<"mov $Zd, $Rn", 1137 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>; 1138 def : InstAlias<"mov $Zd, $Rn", 1139 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>; 1140 def : InstAlias<"mov $Zd, $Rn", 1141 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>; 1142 def : InstAlias<"mov $Zd, $Rn", 1143 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>; 1144} 1145 1146class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm, 1147 ZPRRegOp zprty> 1148: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx), 1149 asm, "\t$Zd, $Zn$idx", 1150 "", 1151 []>, Sched<[]> { 1152 bits<5> Zd; 1153 bits<5> Zn; 1154 bits<7> idx; 1155 let Inst{31-24} = 0b00000101; 1156 let Inst{23-22} = {?,?}; // imm3h 1157 let Inst{21} = 0b1; 1158 let Inst{20-16} = tsz; 1159 let Inst{15-10} = 0b001000; 1160 let Inst{9-5} = Zn; 1161 let Inst{4-0} = Zd; 1162} 1163 1164multiclass sve_int_perm_dup_i<string asm> { 1165 def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> { 1166 let Inst{23-22} = idx{5-4}; 1167 let Inst{20-17} = idx{3-0}; 1168 } 1169 def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> { 1170 let Inst{23-22} = idx{4-3}; 1171 let Inst{20-18} = idx{2-0}; 1172 } 1173 def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> { 1174 let Inst{23-22} = idx{3-2}; 1175 let Inst{20-19} = idx{1-0}; 1176 } 1177 def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> { 1178 let Inst{23-22} = idx{2-1}; 1179 let Inst{20} = idx{0}; 1180 } 1181 def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> { 1182 let Inst{23-22} = idx{1-0}; 1183 } 1184 1185 def : InstAlias<"mov $Zd, $Zn$idx", 1186 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>; 1187 def : InstAlias<"mov $Zd, $Zn$idx", 1188 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>; 1189 def : InstAlias<"mov $Zd, $Zn$idx", 1190 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>; 1191 def : InstAlias<"mov $Zd, $Zn$idx", 1192 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>; 1193 def : InstAlias<"mov $Zd, $Zn$idx", 1194 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>; 1195 def : InstAlias<"mov $Zd, $Bn", 1196 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>; 1197 def : InstAlias<"mov $Zd, $Hn", 1198 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>; 1199 def : InstAlias<"mov $Zd, $Sn", 1200 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>; 1201 def : InstAlias<"mov $Zd, $Dn", 1202 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>; 1203 def : InstAlias<"mov $Zd, $Qn", 1204 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>; 1205 1206 // Duplicate extracted element of vector into all vector elements 1207 def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))), 1208 (!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>; 1209 def : Pat<(nxv8i16 (splat_vector (i32 (vector_extract (nxv8i16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1210 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1211 def : Pat<(nxv4i32 (splat_vector (i32 (vector_extract (nxv4i32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1212 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1213 def : Pat<(nxv2i64 (splat_vector (i64 (vector_extract (nxv2i64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1214 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1215 def : Pat<(nxv8f16 (splat_vector (f16 (vector_extract (nxv8f16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1216 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1217 def : Pat<(nxv8bf16 (splat_vector (bf16 (vector_extract (nxv8bf16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1218 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1219 def : Pat<(nxv4f16 (splat_vector (f16 (vector_extract (nxv4f16 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1220 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1221 def : Pat<(nxv2f16 (splat_vector (f16 (vector_extract (nxv2f16 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1222 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1223 def : Pat<(nxv4f32 (splat_vector (f32 (vector_extract (nxv4f32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1224 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1225 def : Pat<(nxv2f32 (splat_vector (f32 (vector_extract (nxv2f32 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1226 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1227 def : Pat<(nxv2f64 (splat_vector (f64 (vector_extract (nxv2f64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1228 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1229 1230 def : Pat<(nxv16i8 (AArch64duplane128 nxv16i8:$Op1, i64:$imm)), 1231 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1232 def : Pat<(nxv8i16 (AArch64duplane128 nxv8i16:$Op1, i64:$imm)), 1233 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1234 def : Pat<(nxv4i32 (AArch64duplane128 nxv4i32:$Op1, i64:$imm)), 1235 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1236 def : Pat<(nxv2i64 (AArch64duplane128 nxv2i64:$Op1, i64:$imm)), 1237 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1238 def : Pat<(nxv8f16 (AArch64duplane128 nxv8f16:$Op1, i64:$imm)), 1239 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1240 def : Pat<(nxv4f32 (AArch64duplane128 nxv4f32:$Op1, i64:$imm)), 1241 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1242 def : Pat<(nxv2f64 (AArch64duplane128 nxv2f64:$Op1, i64:$imm)), 1243 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1244 def : Pat<(nxv8bf16 (AArch64duplane128 nxv8bf16:$Op1, i64:$imm)), 1245 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1246} 1247 1248class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty, 1249 RegisterOperand VecList> 1250: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm), 1251 asm, "\t$Zd, $Zn, $Zm", 1252 "", 1253 []>, Sched<[]> { 1254 bits<5> Zd; 1255 bits<5> Zm; 1256 bits<5> Zn; 1257 let Inst{31-24} = 0b00000101; 1258 let Inst{23-22} = sz8_64; 1259 let Inst{21} = 0b1; 1260 let Inst{20-16} = Zm; 1261 let Inst{15-13} = 0b001; 1262 let Inst{12-11} = opc; 1263 let Inst{10} = 0b0; 1264 let Inst{9-5} = Zn; 1265 let Inst{4-0} = Zd; 1266} 1267 1268multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> { 1269 def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>; 1270 def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>; 1271 def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>; 1272 def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>; 1273 1274 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1275 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>; 1276 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1277 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>; 1278 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1279 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>; 1280 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1281 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>; 1282 1283 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1284 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1285 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1286 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1287 1288 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1289 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1290 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1291 1292 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1293} 1294 1295multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> { 1296 def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>; 1297 def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>; 1298 def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>; 1299 def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>; 1300 1301 def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)), 1302 (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0, 1303 nxv16i8:$Op2, zsub1), 1304 nxv16i8:$Op3))>; 1305 1306 def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)), 1307 (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0, 1308 nxv8i16:$Op2, zsub1), 1309 nxv8i16:$Op3))>; 1310 1311 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)), 1312 (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0, 1313 nxv4i32:$Op2, zsub1), 1314 nxv4i32:$Op3))>; 1315 1316 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)), 1317 (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0, 1318 nxv2i64:$Op2, zsub1), 1319 nxv2i64:$Op3))>; 1320 1321 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)), 1322 (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0, 1323 nxv8f16:$Op2, zsub1), 1324 nxv8i16:$Op3))>; 1325 1326 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)), 1327 (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0, 1328 nxv4f32:$Op2, zsub1), 1329 nxv4i32:$Op3))>; 1330 1331 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)), 1332 (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0, 1333 nxv2f64:$Op2, zsub1), 1334 nxv2i64:$Op3))>; 1335 1336 def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)), 1337 (nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0, 1338 nxv8bf16:$Op2, zsub1), 1339 nxv8i16:$Op3))>; 1340} 1341 1342class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1343: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm), 1344 asm, "\t$Zd, $Zn, $Zm", 1345 "", 1346 []>, Sched<[]> { 1347 bits<5> Zd; 1348 bits<5> Zm; 1349 bits<5> Zn; 1350 let Inst{31-24} = 0b00000101; 1351 let Inst{23-22} = sz8_64; 1352 let Inst{21} = 0b1; 1353 let Inst{20-16} = Zm; 1354 let Inst{15-10} = 0b001011; 1355 let Inst{9-5} = Zn; 1356 let Inst{4-0} = Zd; 1357 1358 let Constraints = "$Zd = $_Zd"; 1359} 1360 1361multiclass sve2_int_perm_tbx<string asm, SDPatternOperator op> { 1362 def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>; 1363 def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>; 1364 def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>; 1365 def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>; 1366 1367 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1368 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1369 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1370 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1371 1372 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1373 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1374 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1375 1376 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1377} 1378 1379class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1380: I<(outs zprty:$Zd), (ins zprty:$Zn), 1381 asm, "\t$Zd, $Zn", 1382 "", 1383 []>, Sched<[]> { 1384 bits<5> Zd; 1385 bits<5> Zn; 1386 let Inst{31-24} = 0b00000101; 1387 let Inst{23-22} = sz8_64; 1388 let Inst{21-10} = 0b111000001110; 1389 let Inst{9-5} = Zn; 1390 let Inst{4-0} = Zd; 1391} 1392 1393multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> { 1394 def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>; 1395 def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>; 1396 def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>; 1397 def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>; 1398 1399 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>; 1400 def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 1401 def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 1402 def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 1403 1404 def : SVE_1_Op_Pat<nxv2f16, op, nxv2f16, !cast<Instruction>(NAME # _D)>; 1405 def : SVE_1_Op_Pat<nxv4f16, op, nxv4f16, !cast<Instruction>(NAME # _S)>; 1406 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 1407 def : SVE_1_Op_Pat<nxv2f32, op, nxv2f32, !cast<Instruction>(NAME # _D)>; 1408 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 1409 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 1410 1411 def : SVE_1_Op_Pat<nxv2bf16, op, nxv2bf16, !cast<Instruction>(NAME # _D)>; 1412 def : SVE_1_Op_Pat<nxv4bf16, op, nxv4bf16, !cast<Instruction>(NAME # _S)>; 1413 def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1414} 1415 1416class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty> 1417: I<(outs pprty:$Pd), (ins pprty:$Pn), 1418 asm, "\t$Pd, $Pn", 1419 "", 1420 []>, Sched<[]> { 1421 bits<4> Pd; 1422 bits<4> Pn; 1423 let Inst{31-24} = 0b00000101; 1424 let Inst{23-22} = sz8_64; 1425 let Inst{21-9} = 0b1101000100000; 1426 let Inst{8-5} = Pn; 1427 let Inst{4} = 0b0; 1428 let Inst{3-0} = Pd; 1429} 1430 1431multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> { 1432 def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>; 1433 def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>; 1434 def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>; 1435 def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>; 1436 1437 def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>; 1438 def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>; 1439 def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>; 1440 def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>; 1441} 1442 1443class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm, 1444 ZPRRegOp zprty1, ZPRRegOp zprty2> 1445: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 1446 asm, "\t$Zd, $Zn", 1447 "", []>, Sched<[]> { 1448 bits<5> Zd; 1449 bits<5> Zn; 1450 let Inst{31-24} = 0b00000101; 1451 let Inst{23-22} = sz16_64; 1452 let Inst{21-18} = 0b1100; 1453 let Inst{17-16} = opc; 1454 let Inst{15-10} = 0b001110; 1455 let Inst{9-5} = Zn; 1456 let Inst{4-0} = Zd; 1457} 1458 1459multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> { 1460 def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>; 1461 def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>; 1462 def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>; 1463 1464 def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>; 1465 def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>; 1466 def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>; 1467} 1468 1469class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1470 RegisterClass srcRegType> 1471: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm), 1472 asm, "\t$Zdn, $Rm", 1473 "", 1474 []>, Sched<[]> { 1475 bits<5> Rm; 1476 bits<5> Zdn; 1477 let Inst{31-24} = 0b00000101; 1478 let Inst{23-22} = sz8_64; 1479 let Inst{21-10} = 0b100100001110; 1480 let Inst{9-5} = Rm; 1481 let Inst{4-0} = Zdn; 1482 1483 let Constraints = "$Zdn = $_Zdn"; 1484 let DestructiveInstType = DestructiveOther; 1485} 1486 1487multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> { 1488 def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>; 1489 def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>; 1490 def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>; 1491 def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>; 1492 1493 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 1494 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 1495 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 1496 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>; 1497} 1498 1499class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1500 FPRasZPROperand srcOpType> 1501: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcOpType:$Vm), 1502 asm, "\t$Zdn, $Vm", 1503 "", 1504 []>, Sched<[]> { 1505 bits<5> Vm; 1506 bits<5> Zdn; 1507 let Inst{31-24} = 0b00000101; 1508 let Inst{23-22} = sz8_64; 1509 let Inst{21-10} = 0b110100001110; 1510 let Inst{9-5} = Vm; 1511 let Inst{4-0} = Zdn; 1512 1513 let Constraints = "$Zdn = $_Zdn"; 1514 let DestructiveInstType = DestructiveOther; 1515} 1516 1517multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> { 1518 def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8asZPR>; 1519 def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16asZPR>; 1520 def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32asZPR>; 1521 def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64asZPR>; 1522 1523 def : Pat<(nxv8f16 (op nxv8f16:$Zn, f16:$Vm)), 1524 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1525 def : Pat<(nxv4f32 (op nxv4f32:$Zn, f32:$Vm)), 1526 (!cast<Instruction>(NAME # _S) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, ssub))>; 1527 def : Pat<(nxv2f64 (op nxv2f64:$Zn, f64:$Vm)), 1528 (!cast<Instruction>(NAME # _D) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, dsub))>; 1529 1530 def : Pat<(nxv8bf16 (op nxv8bf16:$Zn, bf16:$Vm)), 1531 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1532 1533 // Keep integer insertions within the vector unit. 1534 def : Pat<(nxv16i8 (op (nxv16i8 ZPR:$Zn), (i32 (vector_extract (nxv16i8 ZPR:$Vm), 0)))), 1535 (!cast<Instruction>(NAME # _B) $Zn, ZPR:$Vm)>; 1536 def : Pat<(nxv8i16 (op (nxv8i16 ZPR:$Zn), (i32 (vector_extract (nxv8i16 ZPR:$Vm), 0)))), 1537 (!cast<Instruction>(NAME # _H) $Zn, ZPR:$Vm)>; 1538 def : Pat<(nxv4i32 (op (nxv4i32 ZPR:$Zn), (i32 (vector_extract (nxv4i32 ZPR:$Vm), 0)))), 1539 (!cast<Instruction>(NAME # _S) $Zn, ZPR: $Vm)>; 1540 def : Pat<(nxv2i64 (op (nxv2i64 ZPR:$Zn), (i64 (vector_extract (nxv2i64 ZPR:$Vm), 0)))), 1541 (!cast<Instruction>(NAME # _D) $Zn, ZPR:$Vm)>; 1542 1543} 1544 1545//===----------------------------------------------------------------------===// 1546// SVE Permute - Extract Group 1547//===----------------------------------------------------------------------===// 1548 1549class sve_int_perm_extract_i<string asm> 1550: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8), 1551 asm, "\t$Zdn, $_Zdn, $Zm, $imm8", 1552 "", []>, Sched<[]> { 1553 bits<5> Zdn; 1554 bits<5> Zm; 1555 bits<8> imm8; 1556 let Inst{31-21} = 0b00000101001; 1557 let Inst{20-16} = imm8{7-3}; 1558 let Inst{15-13} = 0b000; 1559 let Inst{12-10} = imm8{2-0}; 1560 let Inst{9-5} = Zm; 1561 let Inst{4-0} = Zdn; 1562 1563 let Constraints = "$Zdn = $_Zdn"; 1564 let DestructiveInstType = DestructiveOther; 1565 let ElementSize = ElementSizeNone; 1566} 1567 1568multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> { 1569 def NAME : sve_int_perm_extract_i<asm>; 1570 1571 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255, 1572 !cast<Instruction>(NAME)>; 1573} 1574 1575class sve2_int_perm_extract_i_cons<string asm> 1576: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8), 1577 asm, "\t$Zd, $Zn, $imm8", 1578 "", []>, Sched<[]> { 1579 bits<5> Zd; 1580 bits<5> Zn; 1581 bits<8> imm8; 1582 let Inst{31-21} = 0b00000101011; 1583 let Inst{20-16} = imm8{7-3}; 1584 let Inst{15-13} = 0b000; 1585 let Inst{12-10} = imm8{2-0}; 1586 let Inst{9-5} = Zn; 1587 let Inst{4-0} = Zd; 1588} 1589 1590//===----------------------------------------------------------------------===// 1591// SVE Vector Select Group 1592//===----------------------------------------------------------------------===// 1593 1594class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1595: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm), 1596 asm, "\t$Zd, $Pg, $Zn, $Zm", 1597 "", 1598 []>, Sched<[]> { 1599 bits<4> Pg; 1600 bits<5> Zd; 1601 bits<5> Zm; 1602 bits<5> Zn; 1603 let Inst{31-24} = 0b00000101; 1604 let Inst{23-22} = sz8_64; 1605 let Inst{21} = 0b1; 1606 let Inst{20-16} = Zm; 1607 let Inst{15-14} = 0b11; 1608 let Inst{13-10} = Pg; 1609 let Inst{9-5} = Zn; 1610 let Inst{4-0} = Zd; 1611} 1612 1613multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> { 1614 def _B : sve_int_sel_vvv<0b00, asm, ZPR8>; 1615 def _H : sve_int_sel_vvv<0b01, asm, ZPR16>; 1616 def _S : sve_int_sel_vvv<0b10, asm, ZPR32>; 1617 def _D : sve_int_sel_vvv<0b11, asm, ZPR64>; 1618 1619 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1620 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1621 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1622 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1623 1624 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1625 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 1626 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1627 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 1628 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 1629 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1630 1631 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1632 1633 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1634 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>; 1635 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1636 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>; 1637 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1638 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>; 1639 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1640 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>; 1641} 1642 1643 1644//===----------------------------------------------------------------------===// 1645// SVE Predicate Logical Operations Group 1646//===----------------------------------------------------------------------===// 1647 1648class sve_int_pred_log<bits<4> opc, string asm> 1649: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 1650 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 1651 "", 1652 []>, Sched<[]> { 1653 bits<4> Pd; 1654 bits<4> Pg; 1655 bits<4> Pm; 1656 bits<4> Pn; 1657 let Inst{31-24} = 0b00100101; 1658 let Inst{23-22} = opc{3-2}; 1659 let Inst{21-20} = 0b00; 1660 let Inst{19-16} = Pm; 1661 let Inst{15-14} = 0b01; 1662 let Inst{13-10} = Pg; 1663 let Inst{9} = opc{1}; 1664 let Inst{8-5} = Pn; 1665 let Inst{4} = opc{0}; 1666 let Inst{3-0} = Pd; 1667 1668 // SEL has no predication qualifier. 1669 let AsmString = !if(!eq(opc, 0b0011), 1670 !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"), 1671 !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm")); 1672 1673 let Defs = !if(!eq (opc{2}, 1), [NZCV], []); 1674 1675} 1676 1677multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op, 1678 SDPatternOperator op_nopred = null_frag> { 1679 def NAME : sve_int_pred_log<opc, asm>; 1680 1681 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 1682 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>; 1683 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>; 1684 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>; 1685 def : SVE_3_Op_Pat<nxv1i1, op, nxv1i1, nxv1i1, nxv1i1, !cast<Instruction>(NAME)>; 1686 def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1, 1687 !cast<Instruction>(NAME), PTRUE_B>; 1688 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1, 1689 !cast<Instruction>(NAME), PTRUE_H>; 1690 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1, 1691 !cast<Instruction>(NAME), PTRUE_S>; 1692 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1, 1693 !cast<Instruction>(NAME), PTRUE_D>; 1694} 1695 1696// An instance of sve_int_pred_log_and but uses op_nopred's first operand as the 1697// general predicate. 1698multiclass sve_int_pred_log_v2<bits<4> opc, string asm, SDPatternOperator op, 1699 SDPatternOperator op_nopred> : 1700 sve_int_pred_log<opc, asm, op> { 1701 def : Pat<(nxv16i1 (op_nopred nxv16i1:$Op1, nxv16i1:$Op2)), 1702 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1703 def : Pat<(nxv8i1 (op_nopred nxv8i1:$Op1, nxv8i1:$Op2)), 1704 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1705 def : Pat<(nxv4i1 (op_nopred nxv4i1:$Op1, nxv4i1:$Op2)), 1706 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1707 def : Pat<(nxv2i1 (op_nopred nxv2i1:$Op1, nxv2i1:$Op2)), 1708 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1709} 1710 1711//===----------------------------------------------------------------------===// 1712// SVE Logical Mask Immediate Group 1713//===----------------------------------------------------------------------===// 1714 1715class sve_int_log_imm<bits<2> opc, string asm> 1716: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13), 1717 asm, "\t$Zdn, $_Zdn, $imms13", 1718 "", []>, Sched<[]> { 1719 bits<5> Zdn; 1720 bits<13> imms13; 1721 let Inst{31-24} = 0b00000101; 1722 let Inst{23-22} = opc; 1723 let Inst{21-18} = 0b0000; 1724 let Inst{17-5} = imms13; 1725 let Inst{4-0} = Zdn; 1726 1727 let Constraints = "$Zdn = $_Zdn"; 1728 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1729 let DestructiveInstType = DestructiveOther; 1730 let ElementSize = ElementSizeNone; 1731} 1732 1733multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> { 1734 def NAME : sve_int_log_imm<opc, asm>; 1735 1736 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>; 1737 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>; 1738 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>; 1739 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>; 1740 1741 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1742 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>; 1743 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1744 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>; 1745 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1746 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>; 1747 1748 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1749 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>; 1750 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1751 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>; 1752 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1753 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>; 1754 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1755 (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>; 1756} 1757 1758multiclass sve_int_log_imm_bic<SDPatternOperator op> { 1759 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI")>; 1760 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI")>; 1761 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI")>; 1762 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI")>; 1763} 1764 1765class sve_int_dup_mask_imm<string asm> 1766: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms), 1767 asm, "\t$Zd, $imms", 1768 "", 1769 []>, Sched<[]> { 1770 bits<5> Zd; 1771 bits<13> imms; 1772 let Inst{31-18} = 0b00000101110000; 1773 let Inst{17-5} = imms; 1774 let Inst{4-0} = Zd; 1775 1776 let isReMaterializable = 1; 1777 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1778} 1779 1780multiclass sve_int_dup_mask_imm<string asm> { 1781 def NAME : sve_int_dup_mask_imm<asm>; 1782 1783 def : InstAlias<"dupm $Zd, $imm", 1784 (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>; 1785 def : InstAlias<"dupm $Zd, $imm", 1786 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>; 1787 def : InstAlias<"dupm $Zd, $imm", 1788 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>; 1789 1790 // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here. 1791 def : InstAlias<"mov $Zd, $imm", 1792 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>; 1793 def : InstAlias<"mov $Zd, $imm", 1794 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>; 1795 def : InstAlias<"mov $Zd, $imm", 1796 (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>; 1797 1798 def : Pat<(nxv2i64 (splat_vector (i64 logical_imm64:$imm))), 1799 (!cast<Instruction>(NAME) logical_imm64:$imm)>; 1800} 1801 1802//===----------------------------------------------------------------------===// 1803// SVE Integer Arithmetic - Unpredicated Group. 1804//===----------------------------------------------------------------------===// 1805 1806class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm, 1807 ZPRRegOp zprty> 1808: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1809 asm, "\t$Zd, $Zn, $Zm", 1810 "", []>, Sched<[]> { 1811 bits<5> Zd; 1812 bits<5> Zm; 1813 bits<5> Zn; 1814 let Inst{31-24} = 0b00000100; 1815 let Inst{23-22} = sz8_64; 1816 let Inst{21} = 0b1; 1817 let Inst{20-16} = Zm; 1818 let Inst{15-13} = 0b000; 1819 let Inst{12-10} = opc; 1820 let Inst{9-5} = Zn; 1821 let Inst{4-0} = Zd; 1822} 1823 1824multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, SDPatternOperator op> { 1825 def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>; 1826 def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>; 1827 def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>; 1828 def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>; 1829 1830 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1831 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1832 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1833 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1834} 1835 1836//===----------------------------------------------------------------------===// 1837// SVE Floating Point Arithmetic - Predicated Group 1838//===----------------------------------------------------------------------===// 1839 1840class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm, 1841 ZPRRegOp zprty, 1842 Operand imm_ty> 1843: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1), 1844 asm, "\t$Zdn, $Pg/m, $_Zdn, $i1", 1845 "", 1846 []>, Sched<[]> { 1847 bits<3> Pg; 1848 bits<5> Zdn; 1849 bit i1; 1850 let Inst{31-24} = 0b01100101; 1851 let Inst{23-22} = sz; 1852 let Inst{21-19} = 0b011; 1853 let Inst{18-16} = opc; 1854 let Inst{15-13} = 0b100; 1855 let Inst{12-10} = Pg; 1856 let Inst{9-6} = 0b0000; 1857 let Inst{5} = i1; 1858 let Inst{4-0} = Zdn; 1859 1860 let Constraints = "$Zdn = $_Zdn"; 1861 let DestructiveInstType = DestructiveOther; 1862 let ElementSize = zprty.ElementSize; 1863} 1864 1865multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, string Ps, Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 1866 let DestructiveInstType = DestructiveBinaryImm in { 1867 def _H : SVEPseudo2Instr<Ps # _H, 1>, sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>; 1868 def _S : SVEPseudo2Instr<Ps # _S, 1>, sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>; 1869 def _D : SVEPseudo2Instr<Ps # _D, 1>, sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>; 1870 } 1871 1872 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_H")>; 1873 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_H")>; 1874 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_S")>; 1875 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_S")>; 1876 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_D")>; 1877 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_D")>; 1878} 1879 1880class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm, 1881 ZPRRegOp zprty> 1882: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 1883 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 1884 "", 1885 []>, Sched<[]> { 1886 bits<3> Pg; 1887 bits<5> Zdn; 1888 bits<5> Zm; 1889 let Inst{31-24} = 0b01100101; 1890 let Inst{23-22} = sz; 1891 let Inst{21-20} = 0b00; 1892 let Inst{19-16} = opc; 1893 let Inst{15-13} = 0b100; 1894 let Inst{12-10} = Pg; 1895 let Inst{9-5} = Zm; 1896 let Inst{4-0} = Zdn; 1897 1898 let Constraints = "$Zdn = $_Zdn"; 1899 let DestructiveInstType = DestructiveOther; 1900 let ElementSize = zprty.ElementSize; 1901} 1902 1903multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps, 1904 SDPatternOperator op, DestructiveInstTypeEnum flags, 1905 string revname="", bit isReverseInstr=0> { 1906 let DestructiveInstType = flags in { 1907 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>, 1908 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1909 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>, 1910 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1911 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>, 1912 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1913 } 1914 1915 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1916 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1917 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1918} 1919 1920multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm, 1921 SDPatternOperator op> { 1922 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>; 1923 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>; 1924 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>; 1925 1926 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1927 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1928 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1929} 1930 1931multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> { 1932 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 1933 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 1934 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 1935 1936 def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>; 1937 def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>; 1938 def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>; 1939} 1940 1941class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty> 1942: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, timm32_0_7:$imm3), 1943 asm, "\t$Zdn, $_Zdn, $Zm, $imm3", 1944 "", 1945 []>, Sched<[]> { 1946 bits<5> Zdn; 1947 bits<5> Zm; 1948 bits<3> imm3; 1949 let Inst{31-24} = 0b01100101; 1950 let Inst{23-22} = sz; 1951 let Inst{21-19} = 0b010; 1952 let Inst{18-16} = imm3; 1953 let Inst{15-10} = 0b100000; 1954 let Inst{9-5} = Zm; 1955 let Inst{4-0} = Zdn; 1956 1957 let Constraints = "$Zdn = $_Zdn"; 1958 let DestructiveInstType = DestructiveOther; 1959 let ElementSize = ElementSizeNone; 1960} 1961 1962multiclass sve_fp_ftmad<string asm, SDPatternOperator op> { 1963 def _H : sve_fp_ftmad<0b01, asm, ZPR16>; 1964 def _S : sve_fp_ftmad<0b10, asm, ZPR32>; 1965 def _D : sve_fp_ftmad<0b11, asm, ZPR64>; 1966 1967 def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 timm32_0_7:$imm))), 1968 (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, timm32_0_7:$imm)>; 1969 def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 timm32_0_7:$imm))), 1970 (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, timm32_0_7:$imm)>; 1971 def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 timm32_0_7:$imm))), 1972 (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, timm32_0_7:$imm)>; 1973} 1974 1975multiclass sve_fp_2op_i_p_zds_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator ir_op = null_frag> { 1976 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesUndef>; 1977 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesUndef>; 1978 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesUndef>; 1979 1980 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1981 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1982 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1983 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1984 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1985 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1986 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 1987 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 1988 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 1989 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 1990 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_UNDEF_D")>; 1991 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_UNDEF_D")>; 1992} 1993 1994multiclass sve_fp_2op_i_p_zds_zeroing_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 1995 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesZero>; 1996 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesZero>; 1997 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesZero>; 1998 1999 let AddedComplexity = 2 in { 2000 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_ZERO_H")>; 2001 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_ZERO_H")>; 2002 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_ZERO_S")>; 2003 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_ZERO_S")>; 2004 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_ZERO_D")>; 2005 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_ZERO_D")>; 2006 } 2007} 2008 2009//===----------------------------------------------------------------------===// 2010// SVE Floating Point Arithmetic - Unpredicated Group 2011//===----------------------------------------------------------------------===// 2012 2013class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 2014: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2015 asm, "\t$Zd, $Zn, $Zm", 2016 "", 2017 []>, Sched<[]> { 2018 bits<5> Zd; 2019 bits<5> Zm; 2020 bits<5> Zn; 2021 let Inst{31-24} = 0b01100101; 2022 let Inst{23-22} = sz; 2023 let Inst{21} = 0b0; 2024 let Inst{20-16} = Zm; 2025 let Inst{15-13} = 0b000; 2026 let Inst{12-10} = opc; 2027 let Inst{9-5} = Zn; 2028 let Inst{4-0} = Zd; 2029} 2030 2031multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op, 2032 SDPatternOperator predicated_op = null_frag> { 2033 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 2034 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 2035 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 2036 2037 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2038 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2039 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2040 2041 def : SVE_2_Op_Pred_All_Active<nxv8f16, predicated_op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2042 def : SVE_2_Op_Pred_All_Active<nxv4f32, predicated_op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2043 def : SVE_2_Op_Pred_All_Active<nxv2f64, predicated_op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2044} 2045 2046multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> { 2047 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 2048 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 2049 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 2050 2051 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2052 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2053 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2054} 2055 2056//===----------------------------------------------------------------------===// 2057// SVE Floating Point Fused Multiply-Add Group 2058//===----------------------------------------------------------------------===// 2059 2060class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty> 2061: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2062 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2063 "", 2064 []>, Sched<[]> { 2065 bits<3> Pg; 2066 bits<5> Zda; 2067 bits<5> Zm; 2068 bits<5> Zn; 2069 let Inst{31-24} = 0b01100101; 2070 let Inst{23-22} = sz; 2071 let Inst{21} = 0b1; 2072 let Inst{20-16} = Zm; 2073 let Inst{15} = 0b0; 2074 let Inst{14-13} = opc; 2075 let Inst{12-10} = Pg; 2076 let Inst{9-5} = Zn; 2077 let Inst{4-0} = Zda; 2078 2079 let Constraints = "$Zda = $_Zda"; 2080 let ElementSize = zprty.ElementSize; 2081} 2082 2083multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, string Ps, 2084 SDPatternOperator op, string revname, 2085 bit isReverseInstr=0> { 2086 let DestructiveInstType = DestructiveTernaryCommWithRev in { 2087 def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>, 2088 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2089 def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>, 2090 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2091 def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>, 2092 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2093 } 2094 2095 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2096 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2097 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2098} 2099 2100class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm, 2101 ZPRRegOp zprty> 2102: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2103 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2104 "", 2105 []>, Sched<[]> { 2106 bits<3> Pg; 2107 bits<5> Za; 2108 bits<5> Zdn; 2109 bits<5> Zm; 2110 let Inst{31-24} = 0b01100101; 2111 let Inst{23-22} = sz; 2112 let Inst{21} = 0b1; 2113 let Inst{20-16} = Za; 2114 let Inst{15} = 0b1; 2115 let Inst{14-13} = opc; 2116 let Inst{12-10} = Pg; 2117 let Inst{9-5} = Zm; 2118 let Inst{4-0} = Zdn; 2119 2120 let Constraints = "$Zdn = $_Zdn"; 2121 let DestructiveInstType = DestructiveOther; 2122 let ElementSize = zprty.ElementSize; 2123} 2124 2125multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op, 2126 string revname, bit isReverseInstr> { 2127 def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>, 2128 SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2129 def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>, 2130 SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2131 def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>, 2132 SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2133 2134 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2135 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2136 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2137} 2138 2139multiclass sve_fp_3op_p_zds_zx { 2140 def _UNDEF_H : PredThreeOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 2141 def _UNDEF_S : PredThreeOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 2142 def _UNDEF_D : PredThreeOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 2143} 2144 2145//===----------------------------------------------------------------------===// 2146// SVE Floating Point Multiply-Add - Indexed Group 2147//===----------------------------------------------------------------------===// 2148 2149class sve_fp_fma_by_indexed_elem<bits<2> sz, bit opc, string asm, 2150 ZPRRegOp zprty1, 2151 ZPRRegOp zprty2, Operand itype> 2152: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop), 2153 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 2154 bits<5> Zda; 2155 bits<5> Zn; 2156 let Inst{31-24} = 0b01100100; 2157 let Inst{23-22} = sz; 2158 let Inst{21} = 0b1; 2159 let Inst{15-11} = 0; 2160 let Inst{10} = opc; 2161 let Inst{9-5} = Zn; 2162 let Inst{4-0} = Zda; 2163 2164 let Constraints = "$Zda = $_Zda"; 2165 let DestructiveInstType = DestructiveOther; 2166 let ElementSize = ElementSizeNone; 2167} 2168 2169multiclass sve_fp_fma_by_indexed_elem<bit opc, string asm, 2170 SDPatternOperator op> { 2171 def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2172 bits<3> Zm; 2173 bits<3> iop; 2174 let Inst{22} = iop{2}; 2175 let Inst{20-19} = iop{1-0}; 2176 let Inst{18-16} = Zm; 2177 } 2178 def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2179 bits<3> Zm; 2180 bits<2> iop; 2181 let Inst{20-19} = iop; 2182 let Inst{18-16} = Zm; 2183 } 2184 def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2185 bits<4> Zm; 2186 bit iop; 2187 let Inst{20} = iop; 2188 let Inst{19-16} = Zm; 2189 } 2190 2191 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))), 2192 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>; 2193 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))), 2194 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 2195 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))), 2196 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 2197} 2198 2199 2200//===----------------------------------------------------------------------===// 2201// SVE Floating Point Multiply - Indexed Group 2202//===----------------------------------------------------------------------===// 2203 2204class sve_fp_fmul_by_indexed_elem<bits<2> sz, string asm, ZPRRegOp zprty, 2205 ZPRRegOp zprty2, Operand itype> 2206: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop), 2207 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 2208 bits<5> Zd; 2209 bits<5> Zn; 2210 let Inst{31-24} = 0b01100100; 2211 let Inst{23-22} = sz; 2212 let Inst{21} = 0b1; 2213 let Inst{15-10} = 0b001000; 2214 let Inst{9-5} = Zn; 2215 let Inst{4-0} = Zd; 2216} 2217 2218multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> { 2219 def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2220 bits<3> Zm; 2221 bits<3> iop; 2222 let Inst{22} = iop{2}; 2223 let Inst{20-19} = iop{1-0}; 2224 let Inst{18-16} = Zm; 2225 } 2226 def _S : sve_fp_fmul_by_indexed_elem<0b10, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2227 bits<3> Zm; 2228 bits<2> iop; 2229 let Inst{20-19} = iop; 2230 let Inst{18-16} = Zm; 2231 } 2232 def _D : sve_fp_fmul_by_indexed_elem<0b11, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2233 bits<4> Zm; 2234 bit iop; 2235 let Inst{20} = iop; 2236 let Inst{19-16} = Zm; 2237 } 2238 2239 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))), 2240 (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>; 2241 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))), 2242 (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>; 2243 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))), 2244 (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>; 2245} 2246 2247//===----------------------------------------------------------------------===// 2248// SVE Floating Point Complex Multiply-Add Group 2249//===----------------------------------------------------------------------===// 2250 2251class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty> 2252: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm, 2253 complexrotateop:$imm), 2254 asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm", 2255 "", []>, Sched<[]> { 2256 bits<5> Zda; 2257 bits<3> Pg; 2258 bits<5> Zn; 2259 bits<5> Zm; 2260 bits<2> imm; 2261 let Inst{31-24} = 0b01100100; 2262 let Inst{23-22} = sz; 2263 let Inst{21} = 0; 2264 let Inst{20-16} = Zm; 2265 let Inst{15} = 0; 2266 let Inst{14-13} = imm; 2267 let Inst{12-10} = Pg; 2268 let Inst{9-5} = Zn; 2269 let Inst{4-0} = Zda; 2270 2271 let Constraints = "$Zda = $_Zda"; 2272 let DestructiveInstType = DestructiveOther; 2273 let ElementSize = zprty.ElementSize; 2274} 2275 2276multiclass sve_fp_fcmla<string asm, SDPatternOperator op> { 2277 def _H : sve_fp_fcmla<0b01, asm, ZPR16>; 2278 def _S : sve_fp_fcmla<0b10, asm, ZPR32>; 2279 def _D : sve_fp_fcmla<0b11, asm, ZPR64>; 2280 2281 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))), 2282 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2283 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))), 2284 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2285 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))), 2286 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2287} 2288 2289//===----------------------------------------------------------------------===// 2290// SVE Floating Point Complex Multiply-Add - Indexed Group 2291//===----------------------------------------------------------------------===// 2292 2293class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm, 2294 ZPRRegOp zprty, 2295 ZPRRegOp zprty2, Operand itype> 2296: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop, 2297 complexrotateop:$imm), 2298 asm, "\t$Zda, $Zn, $Zm$iop, $imm", 2299 "", []>, Sched<[]> { 2300 bits<5> Zda; 2301 bits<5> Zn; 2302 bits<2> imm; 2303 let Inst{31-24} = 0b01100100; 2304 let Inst{23-22} = sz; 2305 let Inst{21} = 0b1; 2306 let Inst{15-12} = 0b0001; 2307 let Inst{11-10} = imm; 2308 let Inst{9-5} = Zn; 2309 let Inst{4-0} = Zda; 2310 2311 let Constraints = "$Zda = $_Zda"; 2312 let DestructiveInstType = DestructiveOther; 2313 let ElementSize = ElementSizeNone; 2314} 2315 2316multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> { 2317 def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> { 2318 bits<3> Zm; 2319 bits<2> iop; 2320 let Inst{20-19} = iop; 2321 let Inst{18-16} = Zm; 2322 } 2323 def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> { 2324 bits<4> Zm; 2325 bits<1> iop; 2326 let Inst{20} = iop; 2327 let Inst{19-16} = Zm; 2328 } 2329 2330 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2331 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2332 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2333 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2334} 2335 2336//===----------------------------------------------------------------------===// 2337// SVE Floating Point Complex Addition Group 2338//===----------------------------------------------------------------------===// 2339 2340class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty> 2341: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, 2342 complexrotateopodd:$imm), 2343 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm", 2344 "", 2345 []>, Sched<[]> { 2346 bits<5> Zdn; 2347 bits<5> Zm; 2348 bits<3> Pg; 2349 bit imm; 2350 let Inst{31-24} = 0b01100100; 2351 let Inst{23-22} = sz; 2352 let Inst{21-17} = 0; 2353 let Inst{16} = imm; 2354 let Inst{15-13} = 0b100; 2355 let Inst{12-10} = Pg; 2356 let Inst{9-5} = Zm; 2357 let Inst{4-0} = Zdn; 2358 2359 let Constraints = "$Zdn = $_Zdn"; 2360 let DestructiveInstType = DestructiveOther; 2361 let ElementSize = zprty.ElementSize; 2362} 2363 2364multiclass sve_fp_fcadd<string asm, SDPatternOperator op> { 2365 def _H : sve_fp_fcadd<0b01, asm, ZPR16>; 2366 def _S : sve_fp_fcadd<0b10, asm, ZPR32>; 2367 def _D : sve_fp_fcadd<0b11, asm, ZPR64>; 2368 2369 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))), 2370 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2371 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))), 2372 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2373 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))), 2374 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2375} 2376 2377//===----------------------------------------------------------------------===// 2378// SVE2 Floating Point Convert Group 2379//===----------------------------------------------------------------------===// 2380 2381class sve2_fp_convert_precision<bits<4> opc, string asm, 2382 ZPRRegOp zprty1, ZPRRegOp zprty2> 2383: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn), 2384 asm, "\t$Zd, $Pg/m, $Zn", 2385 "", 2386 []>, Sched<[]> { 2387 bits<5> Zd; 2388 bits<5> Zn; 2389 bits<3> Pg; 2390 let Inst{31-24} = 0b01100100; 2391 let Inst{23-22} = opc{3-2}; 2392 let Inst{21-18} = 0b0010; 2393 let Inst{17-16} = opc{1-0}; 2394 let Inst{15-13} = 0b101; 2395 let Inst{12-10} = Pg; 2396 let Inst{9-5} = Zn; 2397 let Inst{4-0} = Zd; 2398 2399 let Constraints = "$Zd = $_Zd"; 2400} 2401 2402multiclass sve2_fp_convert_down_narrow<string asm, string op> { 2403 def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>; 2404 def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>; 2405 2406 def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>; 2407 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2408} 2409 2410multiclass sve2_fp_convert_up_long<string asm, string op> { 2411 def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>; 2412 def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>; 2413 2414 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>; 2415 def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>; 2416} 2417 2418multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> { 2419 def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>; 2420 2421 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2422} 2423 2424//===----------------------------------------------------------------------===// 2425// SVE2 Floating Point Pairwise Group 2426//===----------------------------------------------------------------------===// 2427 2428class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm, 2429 ZPRRegOp zprty> 2430: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2431 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 2432 "", 2433 []>, Sched<[]> { 2434 bits<3> Pg; 2435 bits<5> Zm; 2436 bits<5> Zdn; 2437 let Inst{31-24} = 0b01100100; 2438 let Inst{23-22} = sz; 2439 let Inst{21-19} = 0b010; 2440 let Inst{18-16} = opc; 2441 let Inst{15-13} = 0b100; 2442 let Inst{12-10} = Pg; 2443 let Inst{9-5} = Zm; 2444 let Inst{4-0} = Zdn; 2445 2446 let Constraints = "$Zdn = $_Zdn"; 2447 let DestructiveInstType = DestructiveOther; 2448 let ElementSize = zprty.ElementSize; 2449} 2450 2451multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm, 2452 SDPatternOperator op> { 2453 def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>; 2454 def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>; 2455 def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>; 2456 2457 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2458 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2459 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2460} 2461 2462//===----------------------------------------------------------------------===// 2463// SVE2 Floating Point Widening Multiply-Add - Indexed Group 2464//===----------------------------------------------------------------------===// 2465 2466class sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm> 2467: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, 2468 VectorIndexH32b:$iop), 2469 asm, "\t$Zda, $Zn, $Zm$iop", 2470 "", 2471 []>, Sched<[]> { 2472 bits<5> Zda; 2473 bits<5> Zn; 2474 bits<3> Zm; 2475 bits<3> iop; 2476 let Inst{31-21} = 0b01100100101; 2477 let Inst{20-19} = iop{2-1}; 2478 let Inst{18-16} = Zm; 2479 let Inst{15-14} = 0b01; 2480 let Inst{13} = opc{1}; 2481 let Inst{12} = 0b0; 2482 let Inst{11} = iop{0}; 2483 let Inst{10} = opc{0}; 2484 let Inst{9-5} = Zn; 2485 let Inst{4-0} = Zda; 2486 2487 let Constraints = "$Zda = $_Zda"; 2488 let DestructiveInstType = DestructiveOther; 2489 let ElementSize = ElementSizeNone; 2490} 2491 2492multiclass sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm, 2493 SDPatternOperator op> { 2494 def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>; 2495 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>; 2496} 2497 2498//===----------------------------------------------------------------------===// 2499// SVE2 Floating Point Widening Multiply-Add Group 2500//===----------------------------------------------------------------------===// 2501 2502class sve2_fp_mla_long<bits<2> opc, string asm> 2503: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 2504 asm, "\t$Zda, $Zn, $Zm", 2505 "", 2506 []>, Sched<[]> { 2507 bits<5> Zda; 2508 bits<5> Zn; 2509 bits<5> Zm; 2510 let Inst{31-21} = 0b01100100101; 2511 let Inst{20-16} = Zm; 2512 let Inst{15-14} = 0b10; 2513 let Inst{13} = opc{1}; 2514 let Inst{12-11} = 0b00; 2515 let Inst{10} = opc{0}; 2516 let Inst{9-5} = Zn; 2517 let Inst{4-0} = Zda; 2518 2519 let Constraints = "$Zda = $_Zda"; 2520 let DestructiveInstType = DestructiveOther; 2521 let ElementSize = ElementSizeNone; 2522} 2523 2524multiclass sve2_fp_mla_long<bits<2> opc, string asm, SDPatternOperator op> { 2525 def NAME : sve2_fp_mla_long<opc, asm>; 2526 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 2527} 2528 2529//===----------------------------------------------------------------------===// 2530// SVE Stack Allocation Group 2531//===----------------------------------------------------------------------===// 2532 2533class sve_int_arith_vl<bit opc, string asm, bit streaming_sve = 0b0> 2534: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6), 2535 asm, "\t$Rd, $Rn, $imm6", 2536 "", 2537 []>, Sched<[]> { 2538 bits<5> Rd; 2539 bits<5> Rn; 2540 bits<6> imm6; 2541 let Inst{31-23} = 0b000001000; 2542 let Inst{22} = opc; 2543 let Inst{21} = 0b1; 2544 let Inst{20-16} = Rn; 2545 let Inst{15-12} = 0b0101; 2546 let Inst{11} = streaming_sve; 2547 let Inst{10-5} = imm6; 2548 let Inst{4-0} = Rd; 2549} 2550 2551class sve_int_read_vl_a<bit op, bits<5> opc2, string asm, bit streaming_sve = 0b0> 2552: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6), 2553 asm, "\t$Rd, $imm6", 2554 "", 2555 []>, Sched<[]> { 2556 bits<5> Rd; 2557 bits<6> imm6; 2558 let Inst{31-23} = 0b000001001; 2559 let Inst{22} = op; 2560 let Inst{21} = 0b1; 2561 let Inst{20-16} = opc2{4-0}; 2562 let Inst{15-12} = 0b0101; 2563 let Inst{11} = streaming_sve; 2564 let Inst{10-5} = imm6; 2565 let Inst{4-0} = Rd; 2566 2567 let isReMaterializable = 1; 2568} 2569 2570//===----------------------------------------------------------------------===// 2571// SVE Permute - In Lane Group 2572//===----------------------------------------------------------------------===// 2573 2574class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm, 2575 ZPRRegOp zprty> 2576: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2577 asm, "\t$Zd, $Zn, $Zm", 2578 "", 2579 []>, Sched<[]> { 2580 bits<5> Zd; 2581 bits<5> Zm; 2582 bits<5> Zn; 2583 let Inst{31-24} = 0b00000101; 2584 let Inst{23-22} = sz8_64; 2585 let Inst{21} = 0b1; 2586 let Inst{20-16} = Zm; 2587 let Inst{15-13} = 0b011; 2588 let Inst{12-10} = opc; 2589 let Inst{9-5} = Zn; 2590 let Inst{4-0} = Zd; 2591} 2592 2593multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm, 2594 SDPatternOperator op> { 2595 def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>; 2596 def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>; 2597 def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>; 2598 def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>; 2599 2600 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2601 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2602 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2603 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2604 2605 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2606 def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 2607 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2608 def : SVE_2_Op_Pat<nxv2f16, op, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 2609 def : SVE_2_Op_Pat<nxv2f32, op, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 2610 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2611 2612 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 2613} 2614 2615//===----------------------------------------------------------------------===// 2616// SVE Floating Point Unary Operations Group 2617//===----------------------------------------------------------------------===// 2618 2619class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype, 2620 RegisterOperand o_zprtype, ElementSizeEnum Sz> 2621: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn), 2622 asm, "\t$Zd, $Pg/m, $Zn", 2623 "", 2624 []>, Sched<[]> { 2625 bits<3> Pg; 2626 bits<5> Zd; 2627 bits<5> Zn; 2628 let Inst{31-24} = 0b01100101; 2629 let Inst{23-22} = opc{6-5}; 2630 let Inst{21} = 0b0; 2631 let Inst{20-16} = opc{4-0}; 2632 let Inst{15-13} = 0b101; 2633 let Inst{12-10} = Pg; 2634 let Inst{9-5} = Zn; 2635 let Inst{4-0} = Zd; 2636 2637 let Constraints = "$Zd = $_Zd"; 2638 let DestructiveInstType = DestructiveUnaryPassthru; 2639 let ElementSize = Sz; 2640} 2641 2642multiclass sve_fp_2op_p_zd<bits<7> opc, string asm, 2643 RegisterOperand i_zprtype, 2644 RegisterOperand o_zprtype, 2645 SDPatternOperator int_op, 2646 SDPatternOperator ir_op, ValueType vt1, 2647 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2648 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>, 2649 SVEPseudo2Instr<NAME, 1>; 2650 // convert vt1 to a packed type for the intrinsic patterns 2651 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2652 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2653 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2654 1 : vt1); 2655 2656 // convert vt3 to a packed type for the intrinsic patterns 2657 defvar packedvt3 = !cond(!eq(!cast<string>(vt3), "nxv2f16"): nxv8f16, 2658 !eq(!cast<string>(vt3), "nxv4f16"): nxv8f16, 2659 !eq(!cast<string>(vt3), "nxv2f32"): nxv4f32, 2660 1 : vt3); 2661 2662 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, packedvt3, !cast<Instruction>(NAME)>; 2663 def : SVE_1_Op_Passthru_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2664 2665 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>; 2666 2667 defm : SVE_1_Op_PassthruUndef_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>; 2668} 2669 2670multiclass sve_fp_2op_p_zdr<bits<7> opc, string asm, 2671 RegisterOperand i_zprtype, 2672 RegisterOperand o_zprtype, 2673 SDPatternOperator int_op, 2674 SDPatternOperator ir_op, ValueType vt1, 2675 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2676 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>, 2677 SVEPseudo2Instr<NAME, 1>; 2678 2679 // convert vt1 to a packed type for the intrinsic patterns 2680 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2681 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2682 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2683 1 : vt1); 2684 2685 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, vt3, !cast<Instruction>(NAME)>; 2686 def : SVE_1_Op_Passthru_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2687 2688 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>; 2689 2690 defm : SVE_1_Op_PassthruUndef_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>; 2691} 2692 2693multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> { 2694 def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>, 2695 SVEPseudo2Instr<NAME # _H, 1>; 2696 def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>, 2697 SVEPseudo2Instr<NAME # _S, 1>; 2698 def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>, 2699 SVEPseudo2Instr<NAME # _D, 1>; 2700 2701 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2702 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 2703 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 2704 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2705 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 2706 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2707 2708 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 2709 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 2710 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 2711 2712 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2713 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2714 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2715 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2716 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2717 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _UNDEF_D)>; 2718} 2719 2720multiclass sve2_fp_flogb<string asm, SDPatternOperator op> { 2721 def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>; 2722 def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>; 2723 def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>; 2724 2725 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2726 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2727 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2728} 2729 2730multiclass sve2_fp_convert_down_odd_rounding<string asm, string op> { 2731 def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>; 2732 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2733} 2734 2735//===----------------------------------------------------------------------===// 2736// SVE Floating Point Unary Operations - Unpredicated Group 2737//===----------------------------------------------------------------------===// 2738 2739class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm, 2740 ZPRRegOp zprty> 2741: I<(outs zprty:$Zd), (ins zprty:$Zn), 2742 asm, "\t$Zd, $Zn", 2743 "", 2744 []>, Sched<[]> { 2745 bits<5> Zd; 2746 bits<5> Zn; 2747 let Inst{31-24} = 0b01100101; 2748 let Inst{23-22} = sz; 2749 let Inst{21-19} = 0b001; 2750 let Inst{18-16} = opc; 2751 let Inst{15-10} = 0b001100; 2752 let Inst{9-5} = Zn; 2753 let Inst{4-0} = Zd; 2754} 2755 2756multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> { 2757 def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>; 2758 def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>; 2759 def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>; 2760 2761 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 2762 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 2763 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 2764} 2765 2766//===----------------------------------------------------------------------===// 2767// SVE Integer Arithmetic - Binary Predicated Group 2768//===----------------------------------------------------------------------===// 2769 2770class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc, 2771 string asm, ZPRRegOp zprty> 2772: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2773 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 2774 bits<3> Pg; 2775 bits<5> Zdn; 2776 bits<5> Zm; 2777 let Inst{31-24} = 0b00000100; 2778 let Inst{23-22} = sz8_64; 2779 let Inst{21} = 0b0; 2780 let Inst{20-19} = fmt; 2781 let Inst{18-16} = opc; 2782 let Inst{15-13} = 0b000; 2783 let Inst{12-10} = Pg; 2784 let Inst{9-5} = Zm; 2785 let Inst{4-0} = Zdn; 2786 2787 let Constraints = "$Zdn = $_Zdn"; 2788 let DestructiveInstType = DestructiveOther; 2789 let ElementSize = zprty.ElementSize; 2790} 2791 2792multiclass sve_int_bin_pred_log<bits<3> opc, string asm, string Ps, 2793 SDPatternOperator op, 2794 DestructiveInstTypeEnum flags> { 2795 let DestructiveInstType = flags in { 2796 def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>, 2797 SVEPseudo2Instr<Ps # _B, 1>; 2798 def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>, 2799 SVEPseudo2Instr<Ps # _H, 1>; 2800 def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>, 2801 SVEPseudo2Instr<Ps # _S, 1>; 2802 def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>, 2803 SVEPseudo2Instr<Ps # _D, 1>; 2804 } 2805 2806 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2807 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2808 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2809 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2810} 2811 2812multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps, 2813 SDPatternOperator op, 2814 DestructiveInstTypeEnum flags, 2815 string revname="", bit isReverseInstr=0> { 2816 let DestructiveInstType = flags in { 2817 def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>, 2818 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 2819 def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>, 2820 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2821 def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>, 2822 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2823 def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>, 2824 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2825 } 2826 2827 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2828 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2829 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2830 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2831} 2832 2833multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps, 2834 SDPatternOperator op, 2835 DestructiveInstTypeEnum flags> { 2836 let DestructiveInstType = flags in { 2837 def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>, 2838 SVEPseudo2Instr<Ps # _B, 1>; 2839 def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>, 2840 SVEPseudo2Instr<Ps # _H, 1>; 2841 def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>, 2842 SVEPseudo2Instr<Ps # _S, 1>; 2843 def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>, 2844 SVEPseudo2Instr<Ps # _D, 1>; 2845 } 2846 2847 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2848 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2849 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2850 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2851} 2852 2853multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps, 2854 SDPatternOperator op, 2855 DestructiveInstTypeEnum flags> { 2856 let DestructiveInstType = flags in { 2857 def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>, 2858 SVEPseudo2Instr<Ps # _B, 1>; 2859 def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>, 2860 SVEPseudo2Instr<Ps # _H, 1>; 2861 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2862 SVEPseudo2Instr<Ps # _S, 1>; 2863 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2864 SVEPseudo2Instr<Ps # _D, 1>; 2865 } 2866 2867 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2868 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2869 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2870 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2871} 2872 2873// Special case for divides which are not defined for 8b/16b elements. 2874multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps, 2875 SDPatternOperator op, 2876 DestructiveInstTypeEnum flags, 2877 string revname="", bit isReverseInstr=0> { 2878 let DestructiveInstType = flags in { 2879 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2880 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2881 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2882 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2883 } 2884 2885 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2886 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2887} 2888 2889//===----------------------------------------------------------------------===// 2890// SVE Integer Multiply-Add Group 2891//===----------------------------------------------------------------------===// 2892 2893class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2894 ZPRRegOp zprty> 2895: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2896 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2897 "", 2898 []>, Sched<[]> { 2899 bits<3> Pg; 2900 bits<5> Zdn; 2901 bits<5> Za; 2902 bits<5> Zm; 2903 let Inst{31-24} = 0b00000100; 2904 let Inst{23-22} = sz8_64; 2905 let Inst{21} = 0b0; 2906 let Inst{20-16} = Zm; 2907 let Inst{15-14} = 0b11; 2908 let Inst{13} = opc; 2909 let Inst{12-10} = Pg; 2910 let Inst{9-5} = Za; 2911 let Inst{4-0} = Zdn; 2912 2913 let Constraints = "$Zdn = $_Zdn"; 2914 let DestructiveInstType = DestructiveOther; 2915 let ElementSize = zprty.ElementSize; 2916} 2917 2918multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2919 def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>; 2920 def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>; 2921 def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>; 2922 def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>; 2923 2924 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2925 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2926 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2927 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2928} 2929 2930class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2931 ZPRRegOp zprty> 2932: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2933 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2934 "", 2935 []>, Sched<[]> { 2936 bits<3> Pg; 2937 bits<5> Zda; 2938 bits<5> Zm; 2939 bits<5> Zn; 2940 let Inst{31-24} = 0b00000100; 2941 let Inst{23-22} = sz8_64; 2942 let Inst{21} = 0b0; 2943 let Inst{20-16} = Zm; 2944 let Inst{15-14} = 0b01; 2945 let Inst{13} = opc; 2946 let Inst{12-10} = Pg; 2947 let Inst{9-5} = Zn; 2948 let Inst{4-0} = Zda; 2949 2950 let Constraints = "$Zda = $_Zda"; 2951 let DestructiveInstType = DestructiveOther; 2952 let ElementSize = zprty.ElementSize; 2953} 2954 2955multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op, 2956 SDPatternOperator outerop, SDPatternOperator mulop> { 2957 def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>; 2958 def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>; 2959 def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>; 2960 def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>; 2961 2962 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2963 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2964 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2965 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2966 2967 def : Pat<(outerop nxv16i8:$Op1, (mulop nxv16i1:$pred, nxv16i8:$Op2, nxv16i8:$Op3)), 2968 (!cast<Instruction>(NAME # _B) $pred, $Op1, $Op2, $Op3)>; 2969 def : Pat<(outerop nxv8i16:$Op1, (mulop nxv8i1:$pred, nxv8i16:$Op2, nxv8i16:$Op3)), 2970 (!cast<Instruction>(NAME # _H) $pred, $Op1, $Op2, $Op3)>; 2971 def : Pat<(outerop nxv4i32:$Op1, (mulop nxv4i1:$pred, nxv4i32:$Op2, nxv4i32:$Op3)), 2972 (!cast<Instruction>(NAME # _S) $pred, $Op1, $Op2, $Op3)>; 2973 def : Pat<(outerop nxv2i64:$Op1, (mulop nxv2i1:$pred, nxv2i64:$Op2, nxv2i64:$Op3)), 2974 (!cast<Instruction>(NAME # _D) $pred, $Op1, $Op2, $Op3)>; 2975} 2976 2977//===----------------------------------------------------------------------===// 2978// SVE2 Integer Multiply-Add - Unpredicated Group 2979//===----------------------------------------------------------------------===// 2980 2981class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, 2982 ZPRRegOp zprty1, ZPRRegOp zprty2> 2983: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 2984 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2985 bits<5> Zda; 2986 bits<5> Zn; 2987 bits<5> Zm; 2988 let Inst{31-24} = 0b01000100; 2989 let Inst{23-22} = sz; 2990 let Inst{21} = 0b0; 2991 let Inst{20-16} = Zm; 2992 let Inst{15} = 0b0; 2993 let Inst{14-10} = opc; 2994 let Inst{9-5} = Zn; 2995 let Inst{4-0} = Zda; 2996 2997 let Constraints = "$Zda = $_Zda"; 2998 let DestructiveInstType = DestructiveOther; 2999 let ElementSize = ElementSizeNone; 3000} 3001 3002multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> { 3003 def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>; 3004 def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>; 3005 def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>; 3006 def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>; 3007 3008 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3009 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3010 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3011 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3012} 3013 3014multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> { 3015 def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>; 3016 def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>; 3017 def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>; 3018 3019 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3020 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3021 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3022} 3023 3024//===----------------------------------------------------------------------===// 3025// SVE2 Integer Multiply-Add - Indexed Group 3026//===----------------------------------------------------------------------===// 3027 3028class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, 3029 ZPRRegOp zprty1, ZPRRegOp zprty2, 3030 ZPRRegOp zprty3, Operand itype> 3031: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 3032 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 3033 bits<5> Zda; 3034 bits<5> Zn; 3035 let Inst{31-24} = 0b01000100; 3036 let Inst{23-22} = sz; 3037 let Inst{21} = 0b1; 3038 let Inst{15-10} = opc; 3039 let Inst{9-5} = Zn; 3040 let Inst{4-0} = Zda; 3041 3042 let Constraints = "$Zda = $_Zda"; 3043 let DestructiveInstType = DestructiveOther; 3044 let ElementSize = ElementSizeNone; 3045} 3046 3047multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm, 3048 SDPatternOperator op> { 3049 def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3050 bits<3> Zm; 3051 bits<3> iop; 3052 let Inst{22} = iop{2}; 3053 let Inst{20-19} = iop{1-0}; 3054 let Inst{18-16} = Zm; 3055 } 3056 def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3057 bits<3> Zm; 3058 bits<2> iop; 3059 let Inst{20-19} = iop; 3060 let Inst{18-16} = Zm; 3061 } 3062 def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3063 bits<4> Zm; 3064 bit iop; 3065 let Inst{20} = iop; 3066 let Inst{19-16} = Zm; 3067 } 3068 3069 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3070 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3071 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3072} 3073 3074//===----------------------------------------------------------------------===// 3075// SVE2 Integer Multiply-Add Long - Indexed Group 3076//===----------------------------------------------------------------------===// 3077 3078multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm, 3079 SDPatternOperator op> { 3080 def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 3081 asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3082 bits<3> Zm; 3083 bits<3> iop; 3084 let Inst{20-19} = iop{2-1}; 3085 let Inst{18-16} = Zm; 3086 let Inst{11} = iop{0}; 3087 } 3088 def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 3089 asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3090 bits<4> Zm; 3091 bits<2> iop; 3092 let Inst{20} = iop{1}; 3093 let Inst{19-16} = Zm; 3094 let Inst{11} = iop{0}; 3095 } 3096 3097 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3098 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3099} 3100 3101//===----------------------------------------------------------------------===// 3102// SVE Integer Dot Product Group 3103//===----------------------------------------------------------------------===// 3104 3105class sve_intx_dot<bit sz, bit U, string asm, ZPRRegOp zprty1, 3106 ZPRRegOp zprty2> 3107: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm, 3108 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3109 bits<5> Zda; 3110 bits<5> Zn; 3111 bits<5> Zm; 3112 let Inst{31-23} = 0b010001001; 3113 let Inst{22} = sz; 3114 let Inst{21} = 0; 3115 let Inst{20-16} = Zm; 3116 let Inst{15-11} = 0; 3117 let Inst{10} = U; 3118 let Inst{9-5} = Zn; 3119 let Inst{4-0} = Zda; 3120 3121 let Constraints = "$Zda = $_Zda"; 3122 let DestructiveInstType = DestructiveOther; 3123} 3124 3125multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> { 3126 def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>; 3127 def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>; 3128 3129 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>; 3130 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>; 3131} 3132 3133//===----------------------------------------------------------------------===// 3134// SVE Integer Dot Product Group - Indexed Group 3135//===----------------------------------------------------------------------===// 3136 3137class sve_intx_dot_by_indexed_elem<bit sz, bit U, string asm, 3138 ZPRRegOp zprty1, ZPRRegOp zprty2, 3139 ZPRRegOp zprty3, Operand itype> 3140: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 3141 asm, "\t$Zda, $Zn, $Zm$iop", 3142 "", []>, Sched<[]> { 3143 bits<5> Zda; 3144 bits<5> Zn; 3145 let Inst{31-23} = 0b010001001; 3146 let Inst{22} = sz; 3147 let Inst{21} = 0b1; 3148 let Inst{15-11} = 0; 3149 let Inst{10} = U; 3150 let Inst{9-5} = Zn; 3151 let Inst{4-0} = Zda; 3152 3153 let Constraints = "$Zda = $_Zda"; 3154 let DestructiveInstType = DestructiveOther; 3155} 3156 3157multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm, 3158 SDPatternOperator op> { 3159 def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> { 3160 bits<2> iop; 3161 bits<3> Zm; 3162 let Inst{20-19} = iop; 3163 let Inst{18-16} = Zm; 3164 } 3165 def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> { 3166 bits<1> iop; 3167 bits<4> Zm; 3168 let Inst{20} = iop; 3169 let Inst{19-16} = Zm; 3170 } 3171 3172 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3173 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3174} 3175 3176//===----------------------------------------------------------------------===// 3177// SVE2 Complex Integer Dot Product Group 3178//===----------------------------------------------------------------------===// 3179 3180class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm, 3181 ZPRRegOp zprty1, ZPRRegOp zprty2> 3182: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm, 3183 complexrotateop:$rot), 3184 asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> { 3185 bits<5> Zda; 3186 bits<5> Zn; 3187 bits<5> Zm; 3188 bits<2> rot; 3189 let Inst{31-24} = 0b01000100; 3190 let Inst{23-22} = sz; 3191 let Inst{21} = 0b0; 3192 let Inst{20-16} = Zm; 3193 let Inst{15-12} = opc; 3194 let Inst{11-10} = rot; 3195 let Inst{9-5} = Zn; 3196 let Inst{4-0} = Zda; 3197 3198 let Constraints = "$Zda = $_Zda"; 3199 let DestructiveInstType = DestructiveOther; 3200 let ElementSize = ElementSizeNone; 3201} 3202 3203multiclass sve2_cintx_dot<string asm, SDPatternOperator op> { 3204 def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>; 3205 def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>; 3206 3207 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3208 (i32 complexrotateop:$imm))), 3209 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>; 3210 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3211 (i32 complexrotateop:$imm))), 3212 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>; 3213} 3214 3215//===----------------------------------------------------------------------===// 3216// SVE2 Complex Multiply-Add Group 3217//===----------------------------------------------------------------------===// 3218 3219multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> { 3220 def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>; 3221 def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>; 3222 def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>; 3223 def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>; 3224 3225 def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>; 3226 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>; 3227 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>; 3228 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>; 3229} 3230 3231//===----------------------------------------------------------------------===// 3232// SVE2 Complex Integer Dot Product - Indexed Group 3233//===----------------------------------------------------------------------===// 3234 3235class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm, 3236 ZPRRegOp zprty1, ZPRRegOp zprty2, 3237 ZPRRegOp zprty3, Operand itype> 3238: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop, 3239 complexrotateop:$rot), 3240 asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> { 3241 bits<5> Zda; 3242 bits<5> Zn; 3243 bits<2> rot; 3244 let Inst{31-24} = 0b01000100; 3245 let Inst{23-22} = sz; 3246 let Inst{21} = 0b1; 3247 let Inst{15-12} = opc; 3248 let Inst{11-10} = rot; 3249 let Inst{9-5} = Zn; 3250 let Inst{4-0} = Zda; 3251 3252 let Constraints = "$Zda = $_Zda"; 3253 let DestructiveInstType = DestructiveOther; 3254 let ElementSize = ElementSizeNone; 3255} 3256 3257multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> { 3258 def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> { 3259 bits<2> iop; 3260 bits<3> Zm; 3261 let Inst{20-19} = iop; 3262 let Inst{18-16} = Zm; 3263 } 3264 def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> { 3265 bit iop; 3266 bits<4> Zm; 3267 let Inst{20} = iop; 3268 let Inst{19-16} = Zm; 3269 } 3270 3271 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3272 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3273 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3274 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3275 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3276 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3277} 3278 3279//===----------------------------------------------------------------------===// 3280// SVE2 Complex Multiply-Add - Indexed Group 3281//===----------------------------------------------------------------------===// 3282 3283multiclass sve2_cmla_by_indexed_elem<bit opc, string asm, 3284 SDPatternOperator op> { 3285 def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> { 3286 bits<2> iop; 3287 bits<3> Zm; 3288 let Inst{20-19} = iop; 3289 let Inst{18-16} = Zm; 3290 } 3291 def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> { 3292 bit iop; 3293 bits<4> Zm; 3294 let Inst{20} = iop; 3295 let Inst{19-16} = Zm; 3296 } 3297 3298 def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3299 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3300 (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3301 3302 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3), 3303 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3304 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3305} 3306 3307//===----------------------------------------------------------------------===// 3308// SVE2 Integer Multiply - Unpredicated Group 3309//===----------------------------------------------------------------------===// 3310 3311class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 3312: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 3313 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3314 bits<5> Zd; 3315 bits<5> Zm; 3316 bits<5> Zn; 3317 let Inst{31-24} = 0b00000100; 3318 let Inst{23-22} = sz; 3319 let Inst{21} = 0b1; 3320 let Inst{20-16} = Zm; 3321 let Inst{15-13} = 0b011; 3322 let Inst{12-10} = opc; 3323 let Inst{9-5} = Zn; 3324 let Inst{4-0} = Zd; 3325} 3326 3327multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op, 3328 SDPatternOperator op_pred = null_frag> { 3329 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3330 def _H : sve2_int_mul<0b01, opc, asm, ZPR16>; 3331 def _S : sve2_int_mul<0b10, opc, asm, ZPR32>; 3332 def _D : sve2_int_mul<0b11, opc, asm, ZPR64>; 3333 3334 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3335 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3336 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3337 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3338 3339 def : SVE_2_Op_Pred_All_Active<nxv16i8, op_pred, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3340 def : SVE_2_Op_Pred_All_Active<nxv8i16, op_pred, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3341 def : SVE_2_Op_Pred_All_Active<nxv4i32, op_pred, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3342 def : SVE_2_Op_Pred_All_Active<nxv2i64, op_pred, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3343} 3344 3345multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> { 3346 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3347 3348 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3349} 3350 3351//===----------------------------------------------------------------------===// 3352// SVE2 Integer Multiply - Indexed Group 3353//===----------------------------------------------------------------------===// 3354 3355class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, 3356 ZPRRegOp zprty1, ZPRRegOp zprty2, 3357 ZPRRegOp zprty3, Operand itype> 3358: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop), 3359 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 3360 bits<5> Zd; 3361 bits<5> Zn; 3362 let Inst{31-24} = 0b01000100; 3363 let Inst{23-22} = sz; 3364 let Inst{21} = 0b1; 3365 let Inst{15-14} = 0b11; 3366 let Inst{13-10} = opc; 3367 let Inst{9-5} = Zn; 3368 let Inst{4-0} = Zd; 3369} 3370 3371multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm, 3372 SDPatternOperator op> { 3373 def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3374 bits<3> Zm; 3375 bits<3> iop; 3376 let Inst{22} = iop{2}; 3377 let Inst{20-19} = iop{1-0}; 3378 let Inst{18-16} = Zm; 3379 } 3380 def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3381 bits<3> Zm; 3382 bits<2> iop; 3383 let Inst{20-19} = iop; 3384 let Inst{18-16} = Zm; 3385 } 3386 def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3387 bits<4> Zm; 3388 bit iop; 3389 let Inst{20} = iop; 3390 let Inst{19-16} = Zm; 3391 } 3392 3393 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3394 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3395 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3396} 3397 3398multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm, 3399 SDPatternOperator op> { 3400 def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm, 3401 ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3402 bits<3> Zm; 3403 bits<3> iop; 3404 let Inst{20-19} = iop{2-1}; 3405 let Inst{18-16} = Zm; 3406 let Inst{11} = iop{0}; 3407 } 3408 def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm, 3409 ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3410 bits<4> Zm; 3411 bits<2> iop; 3412 let Inst{20} = iop{1}; 3413 let Inst{19-16} = Zm; 3414 let Inst{11} = iop{0}; 3415 } 3416 3417 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3418 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3419} 3420 3421//===----------------------------------------------------------------------===// 3422// SVE2 Integer - Predicated Group 3423//===----------------------------------------------------------------------===// 3424 3425class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm, 3426 ZPRRegOp zprty> 3427: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 3428 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 3429 bits<3> Pg; 3430 bits<5> Zm; 3431 bits<5> Zdn; 3432 let Inst{31-24} = 0b01000100; 3433 let Inst{23-22} = sz; 3434 let Inst{21-20} = 0b01; 3435 let Inst{20-16} = opc{5-1}; 3436 let Inst{15-14} = 0b10; 3437 let Inst{13} = opc{0}; 3438 let Inst{12-10} = Pg; 3439 let Inst{9-5} = Zm; 3440 let Inst{4-0} = Zdn; 3441 3442 let Constraints = "$Zdn = $_Zdn"; 3443 let DestructiveInstType = DestructiveOther; 3444 let ElementSize = zprty.ElementSize; 3445} 3446 3447multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op, 3448 string Ps = "", 3449 DestructiveInstTypeEnum flags=DestructiveOther, 3450 string revname="", bit isReverseInstr=0> { 3451 let DestructiveInstType = flags in { 3452 def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>, 3453 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 3454 def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>, 3455 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 3456 def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>, 3457 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 3458 def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>, 3459 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 3460 } 3461 3462 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3463 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3464 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3465 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3466} 3467 3468class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm, 3469 ZPRRegOp zprty1, ZPRRegOp zprty2> 3470: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn), 3471 asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> { 3472 bits<3> Pg; 3473 bits<5> Zn; 3474 bits<5> Zda; 3475 let Inst{31-24} = 0b01000100; 3476 let Inst{23-22} = sz; 3477 let Inst{21-17} = 0b00010; 3478 let Inst{16} = U; 3479 let Inst{15-13} = 0b101; 3480 let Inst{12-10} = Pg; 3481 let Inst{9-5} = Zn; 3482 let Inst{4-0} = Zda; 3483 3484 let Constraints = "$Zda = $_Zda"; 3485 let DestructiveInstType = DestructiveOther; 3486 let ElementSize = zprty1.ElementSize; 3487} 3488 3489multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> { 3490 def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>; 3491 def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>; 3492 def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>; 3493 3494 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3495 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3496 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3497} 3498 3499class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, 3500 string asm, ZPRRegOp zprty> 3501: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3502 asm, "\t$Zd, $Pg/m, $Zn", 3503 "", 3504 []>, Sched<[]> { 3505 bits<3> Pg; 3506 bits<5> Zd; 3507 bits<5> Zn; 3508 let Inst{31-24} = 0b01000100; 3509 let Inst{23-22} = sz; 3510 let Inst{21-20} = 0b00; 3511 let Inst{19} = Q; 3512 let Inst{18} = 0b0; 3513 let Inst{17-16} = opc; 3514 let Inst{15-13} = 0b101; 3515 let Inst{12-10} = Pg; 3516 let Inst{9-5} = Zn; 3517 let Inst{4-0} = Zd; 3518 3519 let Constraints = "$Zd = $_Zd"; 3520 let DestructiveInstType = DestructiveUnaryPassthru; 3521 let ElementSize = zprty.ElementSize; 3522} 3523 3524multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm, 3525 SDPatternOperator op> { 3526 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3527 SVEPseudo2Instr<NAME # _S, 1>; 3528 3529 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3530 3531 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3532 3533 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3534} 3535 3536multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> { 3537 def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>, 3538 SVEPseudo2Instr<NAME # _B, 1>; 3539 def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>, 3540 SVEPseudo2Instr<NAME # _H, 1>; 3541 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3542 SVEPseudo2Instr<NAME # _S, 1>; 3543 def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>, 3544 SVEPseudo2Instr<NAME # _D, 1>; 3545 3546 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3547 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3548 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3549 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3550 3551 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 3552 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 3553 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3554 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 3555 3556 defm : SVE_3_Op_Undef_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 3557 defm : SVE_3_Op_Undef_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 3558 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3559 defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 3560} 3561 3562//===----------------------------------------------------------------------===// 3563// SVE2 Widening Integer Arithmetic Group 3564//===----------------------------------------------------------------------===// 3565 3566class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, 3567 ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3> 3568: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm), 3569 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3570 bits<5> Zd; 3571 bits<5> Zn; 3572 bits<5> Zm; 3573 let Inst{31-24} = 0b01000101; 3574 let Inst{23-22} = sz; 3575 let Inst{21} = 0b0; 3576 let Inst{20-16} = Zm; 3577 let Inst{15} = 0b0; 3578 let Inst{14-10} = opc; 3579 let Inst{9-5} = Zn; 3580 let Inst{4-0} = Zd; 3581} 3582 3583multiclass sve2_wide_int_arith_long<bits<5> opc, string asm, 3584 SDPatternOperator op> { 3585 def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>; 3586 def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>; 3587 def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>; 3588 3589 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3590 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3591 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3592} 3593 3594multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm, 3595 SDPatternOperator op> { 3596 def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>; 3597 def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>; 3598 def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>; 3599 3600 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3601 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3602 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3603} 3604 3605multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm, 3606 SDPatternOperator op> { 3607 def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>; 3608 3609 // To avoid using 128 bit elements in the IR, the pattern below works with 3610 // llvm intrinsics with the _pair suffix, to reflect that 3611 // _Q is implemented as a pair of _D. 3612 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3613} 3614 3615multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> { 3616 def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>; 3617 def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>; 3618 3619 // To avoid using 128 bit elements in the IR, the patterns below work with 3620 // llvm intrinsics with the _pair suffix, to reflect that 3621 // _H is implemented as a pair of _B and _D is implemented as a pair of _S. 3622 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3623 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3624} 3625 3626//===----------------------------------------------------------------------===// 3627// SVE2 Misc Group 3628//===----------------------------------------------------------------------===// 3629 3630class sve2_misc<bits<2> sz, bits<4> opc, string asm, 3631 ZPRRegOp zprty1, ZPRRegOp zprty2> 3632: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3633 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3634 bits<5> Zd; 3635 bits<5> Zn; 3636 bits<5> Zm; 3637 let Inst{31-24} = 0b01000101; 3638 let Inst{23-22} = sz; 3639 let Inst{21} = 0b0; 3640 let Inst{20-16} = Zm; 3641 let Inst{15-14} = 0b10; 3642 let Inst{13-10} = opc; 3643 let Inst{9-5} = Zn; 3644 let Inst{4-0} = Zd; 3645} 3646 3647multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> { 3648 def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; 3649 def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; 3650 def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; 3651 def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; 3652 3653 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3654 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3655 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3656 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3657} 3658 3659multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm, 3660 SDPatternOperator op> { 3661 def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3662 def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3663 def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3664 3665 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3666 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3667 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3668} 3669 3670class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm, 3671 ZPRRegOp zprty1, ZPRRegOp zprty2> 3672: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3673 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3674 bits<5> Zd; 3675 bits<5> Zn; 3676 bits<5> Zm; 3677 let Inst{31-24} = 0b01000101; 3678 let Inst{23-22} = sz; 3679 let Inst{21} = 0b0; 3680 let Inst{20-16} = Zm; 3681 let Inst{15-11} = 0b10010; 3682 let Inst{10} = opc; 3683 let Inst{9-5} = Zn; 3684 let Inst{4-0} = Zd; 3685 3686 let Constraints = "$Zd = $_Zd"; 3687 let DestructiveInstType = DestructiveOther; 3688 let ElementSize = ElementSizeNone; 3689} 3690 3691multiclass sve2_bitwise_xor_interleaved<bit opc, string asm, 3692 SDPatternOperator op> { 3693 def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8, ZPR8>; 3694 def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>; 3695 def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>; 3696 def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>; 3697 3698 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3699 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3700 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3701 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3702} 3703 3704class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm, 3705 ZPRRegOp zprty1, ZPRRegOp zprty2, 3706 Operand immtype> 3707: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3708 asm, "\t$Zd, $Zn, $imm", 3709 "", []>, Sched<[]> { 3710 bits<5> Zd; 3711 bits<5> Zn; 3712 bits<5> imm; 3713 let Inst{31-23} = 0b010001010; 3714 let Inst{22} = tsz8_64{2}; 3715 let Inst{21} = 0b0; 3716 let Inst{20-19} = tsz8_64{1-0}; 3717 let Inst{18-16} = imm{2-0}; // imm3 3718 let Inst{15-12} = 0b1010; 3719 let Inst{11-10} = opc; 3720 let Inst{9-5} = Zn; 3721 let Inst{4-0} = Zd; 3722} 3723 3724multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm, 3725 SDPatternOperator op> { 3726 def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm, 3727 ZPR16, ZPR8, vecshiftL8>; 3728 def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm, 3729 ZPR32, ZPR16, vecshiftL16> { 3730 let Inst{19} = imm{3}; 3731 } 3732 def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm, 3733 ZPR64, ZPR32, vecshiftL32> { 3734 let Inst{20-19} = imm{4-3}; 3735 } 3736 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _H)>; 3737 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _S)>; 3738 def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _D)>; 3739} 3740 3741//===----------------------------------------------------------------------===// 3742// SVE2 Accumulate Group 3743//===----------------------------------------------------------------------===// 3744 3745class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm, 3746 ZPRRegOp zprty, Operand immtype> 3747: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm), 3748 asm, "\t$Zd, $Zn, $imm", 3749 "", []>, Sched<[]> { 3750 bits<5> Zd; 3751 bits<5> Zn; 3752 bits<6> imm; 3753 let Inst{31-24} = 0b01000101; 3754 let Inst{23-22} = tsz8_64{3-2}; 3755 let Inst{21} = 0b0; 3756 let Inst{20-19} = tsz8_64{1-0}; 3757 let Inst{18-16} = imm{2-0}; // imm3 3758 let Inst{15-11} = 0b11110; 3759 let Inst{10} = opc; 3760 let Inst{9-5} = Zn; 3761 let Inst{4-0} = Zd; 3762 3763 let Constraints = "$Zd = $_Zd"; 3764} 3765 3766multiclass sve2_int_bin_shift_imm_left<bit opc, string asm, 3767 SDPatternOperator op> { 3768 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 3769 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 3770 let Inst{19} = imm{3}; 3771 } 3772 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 3773 let Inst{20-19} = imm{4-3}; 3774 } 3775 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 3776 let Inst{22} = imm{5}; 3777 let Inst{20-19} = imm{4-3}; 3778 } 3779 3780 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 3781 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 3782 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 3783 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 3784} 3785 3786multiclass sve2_int_bin_shift_imm_right<bit opc, string asm, 3787 SDPatternOperator op> { 3788 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3789 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3790 let Inst{19} = imm{3}; 3791 } 3792 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3793 let Inst{20-19} = imm{4-3}; 3794 } 3795 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3796 let Inst{22} = imm{5}; 3797 let Inst{20-19} = imm{4-3}; 3798 } 3799 3800 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3801 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3802 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3803 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3804} 3805 3806class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 3807 ZPRRegOp zprty, Operand immtype> 3808: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm), 3809 asm, "\t$Zda, $Zn, $imm", 3810 "", []>, Sched<[]> { 3811 bits<5> Zda; 3812 bits<5> Zn; 3813 bits<6> imm; 3814 let Inst{31-24} = 0b01000101; 3815 let Inst{23-22} = tsz8_64{3-2}; 3816 let Inst{21} = 0b0; 3817 let Inst{20-19} = tsz8_64{1-0}; 3818 let Inst{18-16} = imm{2-0}; // imm3 3819 let Inst{15-12} = 0b1110; 3820 let Inst{11-10} = opc; 3821 let Inst{9-5} = Zn; 3822 let Inst{4-0} = Zda; 3823 3824 let Constraints = "$Zda = $_Zda"; 3825 let DestructiveInstType = DestructiveOther; 3826 let ElementSize = ElementSizeNone; 3827} 3828 3829multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm, 3830 SDPatternOperator op, 3831 SDPatternOperator shift_op = null_frag> { 3832 def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3833 def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3834 let Inst{19} = imm{3}; 3835 } 3836 def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3837 let Inst{20-19} = imm{4-3}; 3838 } 3839 def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3840 let Inst{22} = imm{5}; 3841 let Inst{20-19} = imm{4-3}; 3842 } 3843 3844 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3845 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3846 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3847 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3848 3849 def : SVE_Shift_Add_All_Active_Pat<nxv16i8, shift_op, nxv16i1, nxv16i8, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 3850 def : SVE_Shift_Add_All_Active_Pat<nxv8i16, shift_op, nxv8i1, nxv8i16, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 3851 def : SVE_Shift_Add_All_Active_Pat<nxv4i32, shift_op, nxv4i1, nxv4i32, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 3852 def : SVE_Shift_Add_All_Active_Pat<nxv2i64, shift_op, nxv2i1, nxv2i64, nxv2i64, i32, !cast<Instruction>(NAME # _D)>; 3853} 3854 3855class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty> 3856: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot), 3857 asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> { 3858 bits<5> Zdn; 3859 bits<5> Zm; 3860 bit rot; 3861 let Inst{31-24} = 0b01000101; 3862 let Inst{23-22} = sz; 3863 let Inst{21-17} = 0b00000; 3864 let Inst{16} = opc; 3865 let Inst{15-11} = 0b11011; 3866 let Inst{10} = rot; 3867 let Inst{9-5} = Zm; 3868 let Inst{4-0} = Zdn; 3869 3870 let Constraints = "$Zdn = $_Zdn"; 3871 let DestructiveInstType = DestructiveOther; 3872 let ElementSize = ElementSizeNone; 3873} 3874 3875multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> { 3876 def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>; 3877 def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>; 3878 def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>; 3879 def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>; 3880 3881 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>; 3882 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>; 3883 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>; 3884 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>; 3885} 3886 3887class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm, 3888 ZPRRegOp zprty1, ZPRRegOp zprty2> 3889: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 3890 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3891 bits<5> Zda; 3892 bits<5> Zn; 3893 bits<5> Zm; 3894 let Inst{31-24} = 0b01000101; 3895 let Inst{23-22} = sz; 3896 let Inst{21} = 0b0; 3897 let Inst{20-16} = Zm; 3898 let Inst{15-14} = 0b11; 3899 let Inst{13-10} = opc; 3900 let Inst{9-5} = Zn; 3901 let Inst{4-0} = Zda; 3902 3903 let Constraints = "$Zda = $_Zda"; 3904 let DestructiveInstType = DestructiveOther; 3905 let ElementSize = ElementSizeNone; 3906} 3907 3908multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> { 3909 def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>; 3910 def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>; 3911 def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>; 3912 def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>; 3913 3914 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3915 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3916 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3917 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3918} 3919 3920multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm, 3921 SDPatternOperator op> { 3922 def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3923 def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3924 def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3925 3926 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3927 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3928 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3929} 3930 3931multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm, 3932 SDPatternOperator op> { 3933 def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm, 3934 ZPR32, ZPR32>; 3935 def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm, 3936 ZPR64, ZPR64>; 3937 3938 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3939 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3940} 3941 3942//===----------------------------------------------------------------------===// 3943// SVE2 Narrowing Group 3944//===----------------------------------------------------------------------===// 3945 3946class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc, 3947 string asm, ZPRRegOp zprty1, 3948 ZPRRegOp zprty2, Operand immtype> 3949: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3950 asm, "\t$Zd, $Zn, $imm", 3951 "", []>, Sched<[]> { 3952 bits<5> Zd; 3953 bits<5> Zn; 3954 bits<5> imm; 3955 let Inst{31-23} = 0b010001010; 3956 let Inst{22} = tsz8_64{2}; 3957 let Inst{21} = 0b1; 3958 let Inst{20-19} = tsz8_64{1-0}; 3959 let Inst{18-16} = imm{2-0}; // imm3 3960 let Inst{15-14} = 0b00; 3961 let Inst{13-11} = opc; 3962 let Inst{10} = 0b0; 3963 let Inst{9-5} = Zn; 3964 let Inst{4-0} = Zd; 3965} 3966 3967multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm, 3968 SDPatternOperator op> { 3969 def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16, 3970 tvecshiftR8>; 3971 def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32, 3972 tvecshiftR16> { 3973 let Inst{19} = imm{3}; 3974 } 3975 def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64, 3976 tvecshiftR32> { 3977 let Inst{20-19} = imm{4-3}; 3978 } 3979 def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3980 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3981 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3982} 3983 3984class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc, 3985 string asm, ZPRRegOp zprty1, 3986 ZPRRegOp zprty2, Operand immtype> 3987: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm), 3988 asm, "\t$Zd, $Zn, $imm", 3989 "", []>, Sched<[]> { 3990 bits<5> Zd; 3991 bits<5> Zn; 3992 bits<5> imm; 3993 let Inst{31-23} = 0b010001010; 3994 let Inst{22} = tsz8_64{2}; 3995 let Inst{21} = 0b1; 3996 let Inst{20-19} = tsz8_64{1-0}; 3997 let Inst{18-16} = imm{2-0}; // imm3 3998 let Inst{15-14} = 0b00; 3999 let Inst{13-11} = opc; 4000 let Inst{10} = 0b1; 4001 let Inst{9-5} = Zn; 4002 let Inst{4-0} = Zd; 4003 4004 let Constraints = "$Zd = $_Zd"; 4005} 4006 4007multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm, 4008 SDPatternOperator op> { 4009 def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16, 4010 tvecshiftR8>; 4011 def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32, 4012 tvecshiftR16> { 4013 let Inst{19} = imm{3}; 4014 } 4015 def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64, 4016 tvecshiftR32> { 4017 let Inst{20-19} = imm{4-3}; 4018 } 4019 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4020 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4021 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4022} 4023 4024class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm, 4025 ZPRRegOp zprty1, ZPRRegOp zprty2> 4026: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 4027 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 4028 bits<5> Zd; 4029 bits<5> Zn; 4030 bits<5> Zm; 4031 let Inst{31-24} = 0b01000101; 4032 let Inst{23-22} = sz; 4033 let Inst{21} = 0b1; 4034 let Inst{20-16} = Zm; 4035 let Inst{15-13} = 0b011; 4036 let Inst{12-11} = opc; // S, R 4037 let Inst{10} = 0b0; // Top 4038 let Inst{9-5} = Zn; 4039 let Inst{4-0} = Zd; 4040} 4041 4042multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm, 4043 SDPatternOperator op> { 4044 def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>; 4045 def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>; 4046 def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>; 4047 4048 def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 4049 def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 4050 def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 4051} 4052 4053class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm, 4054 ZPRRegOp zprty1, ZPRRegOp zprty2> 4055: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 4056 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 4057 bits<5> Zd; 4058 bits<5> Zn; 4059 bits<5> Zm; 4060 let Inst{31-24} = 0b01000101; 4061 let Inst{23-22} = sz; 4062 let Inst{21} = 0b1; 4063 let Inst{20-16} = Zm; 4064 let Inst{15-13} = 0b011; 4065 let Inst{12-11} = opc; // S, R 4066 let Inst{10} = 0b1; // Top 4067 let Inst{9-5} = Zn; 4068 let Inst{4-0} = Zd; 4069 4070 let Constraints = "$Zd = $_Zd"; 4071} 4072 4073multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm, 4074 SDPatternOperator op> { 4075 def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>; 4076 def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>; 4077 def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>; 4078 4079 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 4080 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 4081 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 4082} 4083 4084class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm, 4085 ZPRRegOp zprty1, ZPRRegOp zprty2> 4086: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 4087 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 4088 bits<5> Zd; 4089 bits<5> Zn; 4090 let Inst{31-23} = 0b010001010; 4091 let Inst{22} = tsz8_64{2}; 4092 let Inst{21} = 0b1; 4093 let Inst{20-19} = tsz8_64{1-0}; 4094 let Inst{18-13} = 0b000010; 4095 let Inst{12-11} = opc; 4096 let Inst{10} = 0b0; 4097 let Inst{9-5} = Zn; 4098 let Inst{4-0} = Zd; 4099} 4100 4101multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm, 4102 SDPatternOperator op> { 4103 def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>; 4104 def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>; 4105 def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>; 4106 4107 def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>; 4108 def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>; 4109 def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>; 4110} 4111 4112class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm, 4113 ZPRRegOp zprty1, ZPRRegOp zprty2> 4114: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn), 4115 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 4116 bits<5> Zd; 4117 bits<5> Zn; 4118 let Inst{31-23} = 0b010001010; 4119 let Inst{22} = tsz8_64{2}; 4120 let Inst{21} = 0b1; 4121 let Inst{20-19} = tsz8_64{1-0}; 4122 let Inst{18-13} = 0b000010; 4123 let Inst{12-11} = opc; 4124 let Inst{10} = 0b1; 4125 let Inst{9-5} = Zn; 4126 let Inst{4-0} = Zd; 4127 4128 let Constraints = "$Zd = $_Zd"; 4129} 4130 4131multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm, 4132 SDPatternOperator op> { 4133 def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>; 4134 def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>; 4135 def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>; 4136 4137 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>; 4138 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>; 4139 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4140} 4141 4142//===----------------------------------------------------------------------===// 4143// SVE Integer Arithmetic - Unary Predicated Group 4144//===----------------------------------------------------------------------===// 4145 4146class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc, 4147 string asm, ZPRRegOp zprty> 4148: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 4149 asm, "\t$Zd, $Pg/m, $Zn", 4150 "", 4151 []>, Sched<[]> { 4152 bits<3> Pg; 4153 bits<5> Zd; 4154 bits<5> Zn; 4155 let Inst{31-24} = 0b00000100; 4156 let Inst{23-22} = sz8_64; 4157 let Inst{21-20} = 0b01; 4158 let Inst{19} = opc{0}; 4159 let Inst{18-16} = opc{3-1}; 4160 let Inst{15-13} = 0b101; 4161 let Inst{12-10} = Pg; 4162 let Inst{9-5} = Zn; 4163 let Inst{4-0} = Zd; 4164 4165 let Constraints = "$Zd = $_Zd"; 4166 let DestructiveInstType = DestructiveUnaryPassthru; 4167 let ElementSize = zprty.ElementSize; 4168} 4169 4170multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm, 4171 SDPatternOperator op> { 4172 def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>, 4173 SVEPseudo2Instr<NAME # _B, 1>; 4174 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4175 SVEPseudo2Instr<NAME # _H, 1>; 4176 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4177 SVEPseudo2Instr<NAME # _S, 1>; 4178 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4179 SVEPseudo2Instr<NAME # _D, 1>; 4180 4181 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4182 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4183 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4184 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4185 4186 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4187 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4188 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4189 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4190 4191 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4192 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4193 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4194 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4195} 4196 4197multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm, 4198 SDPatternOperator op> { 4199 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4200 SVEPseudo2Instr<NAME # _H, 1>; 4201 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4202 SVEPseudo2Instr<NAME # _S, 1>; 4203 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4204 SVEPseudo2Instr<NAME # _D, 1>; 4205 4206 def : SVE_InReg_Extend<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>; 4207 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>; 4208 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>; 4209 4210 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4211 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4212 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4213 4214 defm : SVE_InReg_Extend_PassthruUndef<nxv8i16, op, nxv8i1, nxv8i8, !cast<Pseudo>(NAME # _UNDEF_H)>; 4215 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i8, !cast<Pseudo>(NAME # _UNDEF_S)>; 4216 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _UNDEF_D)>; 4217} 4218 4219multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm, 4220 SDPatternOperator op> { 4221 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4222 SVEPseudo2Instr<NAME # _S, 1>; 4223 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4224 SVEPseudo2Instr<NAME # _D, 1>; 4225 4226 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>; 4227 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>; 4228 4229 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4230 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4231 4232 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i16, !cast<Pseudo>(NAME # _UNDEF_S)>; 4233 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _UNDEF_D)>; 4234} 4235 4236multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm, 4237 SDPatternOperator op> { 4238 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4239 SVEPseudo2Instr<NAME # _D, 1>; 4240 4241 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>; 4242 4243 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4244 4245 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _UNDEF_D)>; 4246} 4247 4248multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm, 4249 SDPatternOperator op> { 4250 def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>, 4251 SVEPseudo2Instr<NAME # _B, 1>; 4252 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4253 SVEPseudo2Instr<NAME # _H, 1>; 4254 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4255 SVEPseudo2Instr<NAME # _S, 1>; 4256 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4257 SVEPseudo2Instr<NAME # _D, 1>; 4258 4259 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4260 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4261 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4262 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4263 4264 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4265 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4266 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4267 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4268 4269 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4270 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4271 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4272 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4273} 4274 4275multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> { 4276 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4277 SVEPseudo2Instr<NAME # _H, 1>; 4278 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4279 SVEPseudo2Instr<NAME # _S, 1>; 4280 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4281 SVEPseudo2Instr<NAME # _D, 1>; 4282 4283 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4284 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4285 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4286 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4287 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4288 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4289 4290 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4291 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4292 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4293 4294 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4295 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4296 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4297 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4298 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4299 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4300} 4301 4302//===----------------------------------------------------------------------===// 4303// SVE Integer Wide Immediate - Unpredicated Group 4304//===----------------------------------------------------------------------===// 4305class sve_int_dup_imm<bits<2> sz8_64, string asm, 4306 ZPRRegOp zprty, Operand immtype> 4307: I<(outs zprty:$Zd), (ins immtype:$imm), 4308 asm, "\t$Zd, $imm", 4309 "", 4310 []>, Sched<[]> { 4311 bits<5> Zd; 4312 bits<9> imm; 4313 let Inst{31-24} = 0b00100101; 4314 let Inst{23-22} = sz8_64; 4315 let Inst{21-14} = 0b11100011; 4316 let Inst{13} = imm{8}; // sh 4317 let Inst{12-5} = imm{7-0}; // imm8 4318 let Inst{4-0} = Zd; 4319 4320 let isReMaterializable = 1; 4321} 4322 4323multiclass sve_int_dup_imm<string asm> { 4324 def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>; 4325 def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>; 4326 def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>; 4327 def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>; 4328 4329 def : InstAlias<"mov $Zd, $imm", 4330 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>; 4331 def : InstAlias<"mov $Zd, $imm", 4332 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>; 4333 def : InstAlias<"mov $Zd, $imm", 4334 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>; 4335 def : InstAlias<"mov $Zd, $imm", 4336 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>; 4337 4338 def : InstAlias<"fmov $Zd, #0.0", 4339 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>; 4340 def : InstAlias<"fmov $Zd, #0.0", 4341 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>; 4342 def : InstAlias<"fmov $Zd, #0.0", 4343 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>; 4344} 4345 4346class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype, 4347 string asm, ZPRRegOp zprty> 4348: I<(outs zprty:$Zd), (ins fpimmtype:$imm8), 4349 asm, "\t$Zd, $imm8", 4350 "", 4351 []>, Sched<[]> { 4352 bits<5> Zd; 4353 bits<8> imm8; 4354 let Inst{31-24} = 0b00100101; 4355 let Inst{23-22} = sz8_64; 4356 let Inst{21-14} = 0b11100111; 4357 let Inst{13} = 0b0; 4358 let Inst{12-5} = imm8; 4359 let Inst{4-0} = Zd; 4360 4361 let isReMaterializable = 1; 4362} 4363 4364multiclass sve_int_dup_fpimm<string asm> { 4365 def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>; 4366 def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>; 4367 def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>; 4368 4369 def : InstAlias<"fmov $Zd, $imm8", 4370 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>; 4371 def : InstAlias<"fmov $Zd, $imm8", 4372 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>; 4373 def : InstAlias<"fmov $Zd, $imm8", 4374 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>; 4375} 4376 4377class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm, 4378 ZPRRegOp zprty, Operand immtype> 4379: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4380 asm, "\t$Zdn, $_Zdn, $imm", 4381 "", 4382 []>, Sched<[]> { 4383 bits<5> Zdn; 4384 bits<9> imm; 4385 let Inst{31-24} = 0b00100101; 4386 let Inst{23-22} = sz8_64; 4387 let Inst{21-19} = 0b100; 4388 let Inst{18-16} = opc; 4389 let Inst{15-14} = 0b11; 4390 let Inst{13} = imm{8}; // sh 4391 let Inst{12-5} = imm{7-0}; // imm8 4392 let Inst{4-0} = Zdn; 4393 4394 let Constraints = "$Zdn = $_Zdn"; 4395 let DestructiveInstType = DestructiveOther; 4396 let ElementSize = ElementSizeNone; 4397} 4398 4399multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> { 4400 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 4401 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 4402 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 4403 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 4404 4405 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 4406 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 4407 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 4408 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 4409} 4410 4411class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm, 4412 ZPRRegOp zprty, Operand immtype> 4413: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4414 asm, "\t$Zdn, $_Zdn, $imm", 4415 "", 4416 []>, Sched<[]> { 4417 bits<5> Zdn; 4418 bits<8> imm; 4419 let Inst{31-24} = 0b00100101; 4420 let Inst{23-22} = sz8_64; 4421 let Inst{21-16} = opc; 4422 let Inst{15-13} = 0b110; 4423 let Inst{12-5} = imm; 4424 let Inst{4-0} = Zdn; 4425 4426 let Constraints = "$Zdn = $_Zdn"; 4427 let DestructiveInstType = DestructiveOther; 4428 let ElementSize = ElementSizeNone; 4429} 4430 4431multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> { 4432 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8>; 4433 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8>; 4434 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>; 4435 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; 4436 4437 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; 4438 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; 4439 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; 4440 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; 4441} 4442 4443multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> { 4444 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>; 4445 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>; 4446 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>; 4447 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>; 4448 4449 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>; 4450 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>; 4451 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>; 4452 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>; 4453} 4454 4455multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> { 4456 def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8, simm8>; 4457 def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8>; 4458 def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; 4459 def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; 4460 4461 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; 4462 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; 4463 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; 4464 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; 4465} 4466 4467//===----------------------------------------------------------------------===// 4468// SVE Bitwise Logical - Unpredicated Group 4469//===----------------------------------------------------------------------===// 4470 4471class sve_int_bin_cons_log<bits<2> opc, string asm> 4472: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm), 4473 asm, "\t$Zd, $Zn, $Zm", 4474 "", 4475 []>, Sched<[]> { 4476 bits<5> Zd; 4477 bits<5> Zm; 4478 bits<5> Zn; 4479 let Inst{31-24} = 0b00000100; 4480 let Inst{23-22} = opc{1-0}; 4481 let Inst{21} = 0b1; 4482 let Inst{20-16} = Zm; 4483 let Inst{15-10} = 0b001100; 4484 let Inst{9-5} = Zn; 4485 let Inst{4-0} = Zd; 4486} 4487 4488multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> { 4489 def NAME : sve_int_bin_cons_log<opc, asm>; 4490 4491 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4492 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4493 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4494 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4495 4496 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4497 (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>; 4498 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4499 (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>; 4500 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4501 (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>; 4502} 4503 4504class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm> 4505: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk), 4506 asm, "\t$Zdn, $_Zdn, $Zm, $Zk", 4507 "", 4508 []>, Sched<[]> { 4509 bits<5> Zdn; 4510 bits<5> Zk; 4511 bits<5> Zm; 4512 let Inst{31-24} = 0b00000100; 4513 let Inst{23-22} = opc{2-1}; 4514 let Inst{21} = 0b1; 4515 let Inst{20-16} = Zm; 4516 let Inst{15-11} = 0b00111; 4517 let Inst{10} = opc{0}; 4518 let Inst{9-5} = Zk; 4519 let Inst{4-0} = Zdn; 4520 4521 let Constraints = "$Zdn = $_Zdn"; 4522 let DestructiveInstType = DestructiveOther; 4523 let ElementSize = ElementSizeNone; 4524} 4525 4526multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op, 4527 SDPatternOperator ir_op = null_frag> { 4528 def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>; 4529 4530 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4531 (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>; 4532 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4533 (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>; 4534 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4535 (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>; 4536 4537 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4538 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4539 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4540 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4541 4542 4543 def : SVE_3_Op_BSP_Pat<nxv16i8, ir_op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4544 def : SVE_3_Op_BSP_Pat<nxv8i16, ir_op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4545 def : SVE_3_Op_BSP_Pat<nxv4i32, ir_op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4546 def : SVE_3_Op_BSP_Pat<nxv2i64, ir_op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4547} 4548 4549class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm, 4550 ZPRRegOp zprty, Operand immtype> 4551: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm), 4552 asm, "\t$Zdn, $_Zdn, $Zm, $imm", 4553 "", 4554 []>, Sched<[]> { 4555 bits<5> Zdn; 4556 bits<5> Zm; 4557 bits<6> imm; 4558 let Inst{31-24} = 0b00000100; 4559 let Inst{23-22} = tsz8_64{3-2}; 4560 let Inst{21} = 0b1; 4561 let Inst{20-19} = tsz8_64{1-0}; 4562 let Inst{18-16} = imm{2-0}; // imm3 4563 let Inst{15-10} = 0b001101; 4564 let Inst{9-5} = Zm; 4565 let Inst{4-0} = Zdn; 4566 4567 let Constraints = "$Zdn = $_Zdn"; 4568 let DestructiveInstType = DestructiveOther; 4569 let ElementSize = ElementSizeNone; 4570} 4571 4572multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> { 4573 def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>; 4574 def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> { 4575 let Inst{19} = imm{3}; 4576 } 4577 def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> { 4578 let Inst{20-19} = imm{4-3}; 4579 } 4580 def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> { 4581 let Inst{22} = imm{5}; 4582 let Inst{20-19} = imm{4-3}; 4583 } 4584 4585 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4586 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4587 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4588 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4589} 4590 4591//===----------------------------------------------------------------------===// 4592// SVE Integer Wide Immediate - Predicated Group 4593//===----------------------------------------------------------------------===// 4594 4595class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype, 4596 string asm, ZPRRegOp zprty> 4597: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8), 4598 asm, "\t$Zd, $Pg/m, $imm8", 4599 "", 4600 []>, Sched<[]> { 4601 bits<4> Pg; 4602 bits<5> Zd; 4603 bits<8> imm8; 4604 let Inst{31-24} = 0b00000101; 4605 let Inst{23-22} = sz; 4606 let Inst{21-20} = 0b01; 4607 let Inst{19-16} = Pg; 4608 let Inst{15-13} = 0b110; 4609 let Inst{12-5} = imm8; 4610 let Inst{4-0} = Zd; 4611 4612 let Constraints = "$Zd = $_Zd"; 4613 let DestructiveInstType = DestructiveOther; 4614 let ElementSize = zprty.ElementSize; 4615} 4616 4617multiclass sve_int_dup_fpimm_pred<string asm> { 4618 def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>; 4619 def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>; 4620 def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>; 4621 4622 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4623 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>; 4624 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4625 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>; 4626 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4627 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>; 4628} 4629 4630class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm, 4631 ZPRRegOp zprty, string pred_qual, dag iops> 4632: I<(outs zprty:$Zd), iops, 4633 asm, "\t$Zd, $Pg"#pred_qual#", $imm", 4634 "", []>, Sched<[]> { 4635 bits<5> Zd; 4636 bits<4> Pg; 4637 bits<9> imm; 4638 let Inst{31-24} = 0b00000101; 4639 let Inst{23-22} = sz8_64; 4640 let Inst{21-20} = 0b01; 4641 let Inst{19-16} = Pg; 4642 let Inst{15} = 0b0; 4643 let Inst{14} = m; 4644 let Inst{13} = imm{8}; // sh 4645 let Inst{12-5} = imm{7-0}; // imm8 4646 let Inst{4-0} = Zd; 4647 4648 let DestructiveInstType = DestructiveOther; 4649 let ElementSize = zprty.ElementSize; 4650} 4651 4652multiclass sve_int_dup_imm_pred_merge_inst< 4653 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm, 4654 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> { 4655 let Constraints = "$Zd = $_Zd" in 4656 def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty, "/m", 4657 (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>; 4658 def : InstAlias<"mov $Zd, $Pg/m, $imm", 4659 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4660 def : Pat<(vselect predty:$Pg, 4661 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))), 4662 ZPR:$Zd), 4663 (!cast<Instruction>(NAME) $Zd, $Pg, $imm, $shift)>; 4664} 4665 4666multiclass sve_int_dup_imm_pred_merge<string asm> { 4667 defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8, 4668 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>; 4669 defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16, 4670 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>; 4671 defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32, 4672 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>; 4673 defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64, 4674 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>; 4675 4676 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4677 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>; 4678 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4679 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>; 4680 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4681 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>; 4682 4683 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv8f16 ZPR:$Zd)), 4684 (!cast<Instruction>(NAME # _H) $Zd, $Pg, 0, 0)>; 4685 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f16 ZPR:$Zd)), 4686 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>; 4687 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f16 ZPR:$Zd)), 4688 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4689 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f32 ZPR:$Zd)), 4690 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>; 4691 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f32 ZPR:$Zd)), 4692 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4693 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f64 ZPR:$Zd)), 4694 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4695} 4696 4697multiclass sve_int_dup_imm_pred_zero_inst< 4698 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm, 4699 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> { 4700 def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z", 4701 (ins PPRAny:$Pg, cpyimm:$imm)>; 4702 def : InstAlias<"mov $Zd, $Pg/z, $imm", 4703 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4704 def : Pat<(intty (zext (predty PPRAny:$Ps1))), 4705 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4706 def : Pat<(intty (sext (predty PPRAny:$Ps1))), 4707 (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>; 4708 def : Pat<(intty (anyext (predty PPRAny:$Ps1))), 4709 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4710 def : Pat<(vselect predty:$Pg, 4711 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))), 4712 (intty (splat_vector (scalarty 0)))), 4713 (!cast<Instruction>(NAME) $Pg, $imm, $shift)>; 4714} 4715 4716multiclass sve_int_dup_imm_pred_zero<string asm> { 4717 defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8, 4718 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>; 4719 defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16, 4720 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>; 4721 defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32, 4722 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>; 4723 defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64, 4724 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>; 4725} 4726 4727//===----------------------------------------------------------------------===// 4728// SVE Integer Compare - Vectors Group 4729//===----------------------------------------------------------------------===// 4730 4731class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm, 4732 PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2> 4733: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm), 4734 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4735 "", 4736 []>, Sched<[]> { 4737 bits<4> Pd; 4738 bits<3> Pg; 4739 bits<5> Zm; 4740 bits<5> Zn; 4741 let Inst{31-24} = 0b00100100; 4742 let Inst{23-22} = sz8_64; 4743 let Inst{21} = 0b0; 4744 let Inst{20-16} = Zm; 4745 let Inst{15} = opc{2}; 4746 let Inst{14} = cmp_1; 4747 let Inst{13} = opc{1}; 4748 let Inst{12-10} = Pg; 4749 let Inst{9-5} = Zn; 4750 let Inst{4} = opc{0}; 4751 let Inst{3-0} = Pd; 4752 4753 let Defs = [NZCV]; 4754 let ElementSize = pprty.ElementSize; 4755 let isPTestLike = 1; 4756} 4757 4758multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt, 4759 ValueType intvt, Instruction cmp> { 4760 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)), 4761 (cmp $Op1, $Op2, $Op3)>; 4762 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)), 4763 (cmp $Op1, $Op3, $Op2)>; 4764 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op2, intvt:$Op3, cc))), 4765 (cmp $Pg, $Op2, $Op3)>; 4766 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op2, intvt:$Op3, invcc))), 4767 (cmp $Pg, $Op3, $Op2)>; 4768} 4769 4770multiclass SVE_SETCC_Pat_With_Zero<CondCode cc, CondCode invcc, ValueType predvt, 4771 ValueType intvt, Instruction cmp> { 4772 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, (SVEDup0), cc)), 4773 (cmp $Op1, $Op2)>; 4774 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, (SVEDup0), intvt:$Op2, invcc)), 4775 (cmp $Op1, $Op2)>; 4776 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op1, (SVEDup0), cc))), 4777 (cmp $Pg, $Op1)>; 4778 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), (SVEDup0), intvt:$Op1, invcc))), 4779 (cmp $Pg, $Op1)>; 4780} 4781 4782multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> { 4783 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>; 4784 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>; 4785 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>; 4786 def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>; 4787 4788 defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4789 defm : SVE_SETCC_Pat<cc, invcc, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4790 defm : SVE_SETCC_Pat<cc, invcc, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4791 defm : SVE_SETCC_Pat<cc, invcc, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4792} 4793 4794multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> { 4795 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4796 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4797 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4798 4799 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4800 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4801 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4802} 4803 4804multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> { 4805 def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4806 def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4807 def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4808 4809 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4810 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4811 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4812} 4813 4814 4815//===----------------------------------------------------------------------===// 4816// SVE Integer Compare - Signed Immediate Group 4817//===----------------------------------------------------------------------===// 4818 4819class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 4820 ZPRRegOp zprty, 4821 Operand immtype> 4822: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5), 4823 asm, "\t$Pd, $Pg/z, $Zn, $imm5", 4824 "", 4825 []>, Sched<[]> { 4826 bits<4> Pd; 4827 bits<3> Pg; 4828 bits<5> Zn; 4829 bits<5> imm5; 4830 let Inst{31-24} = 0b00100101; 4831 let Inst{23-22} = sz8_64; 4832 let Inst{21} = 0b0; 4833 let Inst{20-16} = imm5; 4834 let Inst{15} = opc{2}; 4835 let Inst{14} = 0b0; 4836 let Inst{13} = opc{1}; 4837 let Inst{12-10} = Pg; 4838 let Inst{9-5} = Zn; 4839 let Inst{4} = opc{0}; 4840 let Inst{3-0} = Pd; 4841 4842 let Defs = [NZCV]; 4843 let ElementSize = pprty.ElementSize; 4844 let isPTestLike = 1; 4845} 4846 4847multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc, 4848 ValueType predvt, ValueType intvt, 4849 Operand immtype, Instruction cmp> { 4850 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4851 (intvt ZPR:$Zs1), 4852 (intvt (splat_vector (immtype:$imm))), 4853 cc)), 4854 (cmp $Pg, $Zs1, immtype:$imm)>; 4855 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4856 (intvt (splat_vector (immtype:$imm))), 4857 (intvt ZPR:$Zs1), 4858 commuted_cc)), 4859 (cmp $Pg, $Zs1, immtype:$imm)>; 4860 def : Pat<(predvt (and predvt:$Pg, 4861 (AArch64setcc_z (predvt (AArch64ptrue 31)), 4862 (intvt ZPR:$Zs1), 4863 (intvt (splat_vector (immtype:$imm))), 4864 cc))), 4865 (cmp $Pg, $Zs1, immtype:$imm)>; 4866 def : Pat<(predvt (and predvt:$Pg, 4867 (AArch64setcc_z (predvt (AArch64ptrue 31)), 4868 (intvt (splat_vector (immtype:$imm))), 4869 (intvt ZPR:$Zs1), 4870 commuted_cc))), 4871 (cmp $Pg, $Zs1, immtype:$imm)>; 4872} 4873 4874multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> { 4875 def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>; 4876 def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>; 4877 def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>; 4878 def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>; 4879 4880 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b, 4881 !cast<Instruction>(NAME # _B)>; 4882 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, simm5_32b, 4883 !cast<Instruction>(NAME # _H)>; 4884 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, simm5_32b, 4885 !cast<Instruction>(NAME # _S)>; 4886 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, simm5_64b, 4887 !cast<Instruction>(NAME # _D)>; 4888} 4889 4890 4891//===----------------------------------------------------------------------===// 4892// SVE Integer Compare - Unsigned Immediate Group 4893//===----------------------------------------------------------------------===// 4894 4895class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty, 4896 ZPRRegOp zprty, Operand immtype> 4897: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7), 4898 asm, "\t$Pd, $Pg/z, $Zn, $imm7", 4899 "", 4900 []>, Sched<[]> { 4901 bits<4> Pd; 4902 bits<3> Pg; 4903 bits<5> Zn; 4904 bits<7> imm7; 4905 let Inst{31-24} = 0b00100100; 4906 let Inst{23-22} = sz8_64; 4907 let Inst{21} = 1; 4908 let Inst{20-14} = imm7; 4909 let Inst{13} = opc{1}; 4910 let Inst{12-10} = Pg; 4911 let Inst{9-5} = Zn; 4912 let Inst{4} = opc{0}; 4913 let Inst{3-0} = Pd; 4914 4915 let Defs = [NZCV]; 4916 let ElementSize = pprty.ElementSize; 4917 let isPTestLike = 1; 4918} 4919 4920multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc, 4921 CondCode commuted_cc> { 4922 def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>; 4923 def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>; 4924 def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>; 4925 def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>; 4926 4927 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127, 4928 !cast<Instruction>(NAME # _B)>; 4929 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, imm0_127, 4930 !cast<Instruction>(NAME # _H)>; 4931 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, imm0_127, 4932 !cast<Instruction>(NAME # _S)>; 4933 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, imm0_127_64b, 4934 !cast<Instruction>(NAME # _D)>; 4935} 4936 4937 4938//===----------------------------------------------------------------------===// 4939// SVE Integer Compare - Scalars Group 4940//===----------------------------------------------------------------------===// 4941 4942class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt> 4943: I<(outs), (ins rt:$Rn, rt:$Rm), 4944 asm, "\t$Rn, $Rm", 4945 "", 4946 []>, Sched<[]> { 4947 bits<5> Rm; 4948 bits<5> Rn; 4949 let Inst{31-23} = 0b001001011; 4950 let Inst{22} = sz; 4951 let Inst{21} = 0b1; 4952 let Inst{20-16} = Rm; 4953 let Inst{15-10} = 0b001000; 4954 let Inst{9-5} = Rn; 4955 let Inst{4} = opc; 4956 let Inst{3-0} = 0b0000; 4957 4958 let Defs = [NZCV]; 4959} 4960 4961class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm, 4962 RegisterClass gprty, PPRRegOp pprty> 4963: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm), 4964 asm, "\t$Pd, $Rn, $Rm", 4965 "", []>, Sched<[]> { 4966 bits<4> Pd; 4967 bits<5> Rm; 4968 bits<5> Rn; 4969 let Inst{31-24} = 0b00100101; 4970 let Inst{23-22} = sz8_64; 4971 let Inst{21} = 0b1; 4972 let Inst{20-16} = Rm; 4973 let Inst{15-13} = 0b000; 4974 let Inst{12-10} = opc{3-1}; 4975 let Inst{9-5} = Rn; 4976 let Inst{4} = opc{0}; 4977 let Inst{3-0} = Pd; 4978 4979 let Defs = [NZCV]; 4980 let ElementSize = pprty.ElementSize; 4981 let isWhile = 1; 4982} 4983 4984multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> { 4985 def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>; 4986 def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>; 4987 def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>; 4988 def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>; 4989 4990 def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>; 4991 def : SVE_2_Op_Pat<nxv8i1, op, i32, i32, !cast<Instruction>(NAME # _H)>; 4992 def : SVE_2_Op_Pat<nxv4i1, op, i32, i32, !cast<Instruction>(NAME # _S)>; 4993 def : SVE_2_Op_Pat<nxv2i1, op, i32, i32, !cast<Instruction>(NAME # _D)>; 4994} 4995 4996multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op> { 4997 def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8>; 4998 def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16>; 4999 def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32>; 5000 def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>; 5001 5002 def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>; 5003 def : SVE_2_Op_Pat<nxv8i1, op, i64, i64, !cast<Instruction>(NAME # _H)>; 5004 def : SVE_2_Op_Pat<nxv4i1, op, i64, i64, !cast<Instruction>(NAME # _S)>; 5005 def : SVE_2_Op_Pat<nxv2i1, op, i64, i64, !cast<Instruction>(NAME # _D)>; 5006} 5007 5008class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, 5009 PPRRegOp pprty> 5010: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), 5011 asm, "\t$Pd, $Rn, $Rm", 5012 "", []>, Sched<[]> { 5013 bits<4> Pd; 5014 bits<5> Rm; 5015 bits<5> Rn; 5016 let Inst{31-24} = 0b00100101; 5017 let Inst{23-22} = sz8_64; 5018 let Inst{21} = 0b1; 5019 let Inst{20-16} = Rm; 5020 let Inst{15-10} = 0b001100; 5021 let Inst{9-5} = Rn; 5022 let Inst{4} = rw; 5023 let Inst{3-0} = Pd; 5024 5025 let Defs = [NZCV]; 5026 let ElementSize = pprty.ElementSize; 5027 let isWhile = 1; 5028} 5029 5030multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> { 5031 def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; 5032 def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; 5033 def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; 5034 def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; 5035 5036 def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>; 5037 def : SVE_2_Op_Pat<nxv8i1, !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>; 5038 def : SVE_2_Op_Pat<nxv4i1, !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>; 5039 def : SVE_2_Op_Pat<nxv2i1, !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>; 5040} 5041 5042//===----------------------------------------------------------------------===// 5043// SVE Floating Point Fast Reduction Group 5044//===----------------------------------------------------------------------===// 5045 5046class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm, 5047 ZPRRegOp zprty, FPRasZPROperand dstOpType> 5048: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 5049 asm, "\t$Vd, $Pg, $Zn", 5050 "", 5051 []>, Sched<[]> { 5052 bits<5> Zn; 5053 bits<5> Vd; 5054 bits<3> Pg; 5055 let Inst{31-24} = 0b01100101; 5056 let Inst{23-22} = sz; 5057 let Inst{21-19} = 0b000; 5058 let Inst{18-16} = opc; 5059 let Inst{15-13} = 0b001; 5060 let Inst{12-10} = Pg; 5061 let Inst{9-5} = Zn; 5062 let Inst{4-0} = Vd; 5063} 5064 5065multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> { 5066 def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>; 5067 def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>; 5068 def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>; 5069 5070 def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5071 def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5072 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5073 def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5074 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5075 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5076} 5077 5078//===----------------------------------------------------------------------===// 5079// SVE Floating Point Accumulating Reduction Group 5080//===----------------------------------------------------------------------===// 5081 5082class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm, 5083 ZPRRegOp zprty, FPRasZPROperand dstOpType> 5084: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm), 5085 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 5086 "", 5087 []>, 5088 Sched<[]> { 5089 bits<3> Pg; 5090 bits<5> Vdn; 5091 bits<5> Zm; 5092 let Inst{31-24} = 0b01100101; 5093 let Inst{23-22} = sz; 5094 let Inst{21-19} = 0b011; 5095 let Inst{18-16} = opc; 5096 let Inst{15-13} = 0b001; 5097 let Inst{12-10} = Pg; 5098 let Inst{9-5} = Zm; 5099 let Inst{4-0} = Vdn; 5100 5101 let Constraints = "$Vdn = $_Vdn"; 5102} 5103 5104multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> { 5105 def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>; 5106 def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>; 5107 def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>; 5108 5109 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>; 5110 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>; 5111 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5112 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>; 5113 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5114 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5115} 5116 5117//===----------------------------------------------------------------------===// 5118// SVE Floating Point Compare - Vectors Group 5119//===----------------------------------------------------------------------===// 5120 5121class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 5122 ZPRRegOp zprty> 5123: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 5124 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 5125 "", 5126 []>, Sched<[]> { 5127 bits<4> Pd; 5128 bits<3> Pg; 5129 bits<5> Zm; 5130 bits<5> Zn; 5131 let Inst{31-24} = 0b01100101; 5132 let Inst{23-22} = sz; 5133 let Inst{21} = 0b0; 5134 let Inst{20-16} = Zm; 5135 let Inst{15} = opc{2}; 5136 let Inst{14} = 0b1; 5137 let Inst{13} = opc{1}; 5138 let Inst{12-10} = Pg; 5139 let Inst{9-5} = Zn; 5140 let Inst{4} = opc{0}; 5141 let Inst{3-0} = Pd; 5142} 5143 5144multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> { 5145 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5146 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5147 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5148 5149 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5150 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5151 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5152} 5153 5154multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, 5155 CondCode cc1, CondCode cc2, 5156 CondCode invcc1, CondCode invcc2> { 5157 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5158 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5159 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5160 5161 defm : SVE_SETCC_Pat<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5162 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5163 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5164 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5165 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5166 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5167 5168 defm : SVE_SETCC_Pat<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5169 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5170 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5171 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5172 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5173 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5174} 5175 5176//===----------------------------------------------------------------------===// 5177// SVE Floating Point Compare - with Zero Group 5178//===----------------------------------------------------------------------===// 5179 5180class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 5181 ZPRRegOp zprty> 5182: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn), 5183 asm, "\t$Pd, $Pg/z, $Zn, #0.0", 5184 "", 5185 []>, Sched<[]> { 5186 bits<4> Pd; 5187 bits<3> Pg; 5188 bits<5> Zn; 5189 let Inst{31-24} = 0b01100101; 5190 let Inst{23-22} = sz; 5191 let Inst{21-18} = 0b0100; 5192 let Inst{17-16} = opc{2-1}; 5193 let Inst{15-13} = 0b001; 5194 let Inst{12-10} = Pg; 5195 let Inst{9-5} = Zn; 5196 let Inst{4} = opc{0}; 5197 let Inst{3-0} = Pd; 5198} 5199 5200multiclass sve_fp_2op_p_pd<bits<3> opc, string asm, 5201 CondCode cc1, CondCode cc2, 5202 CondCode invcc1, CondCode invcc2> { 5203 def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5204 def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5205 def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5206 5207 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5208 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5209 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5210 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5211 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5212 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5213 5214 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5215 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5216 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5217 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5218 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5219 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5220} 5221 5222 5223//===----------------------------------------------------------------------===// 5224//SVE Index Generation Group 5225//===----------------------------------------------------------------------===// 5226 5227def simm5_8b_tgt : TImmLeaf<i8, [{ return (int8_t)Imm >= -16 && (int8_t)Imm < 16; }]>; 5228def simm5_16b_tgt : TImmLeaf<i16, [{ return (int16_t)Imm >= -16 && (int16_t)Imm < 16; }]>; 5229def simm5_32b_tgt : TImmLeaf<i32, [{ return (int32_t)Imm >= -16 && (int32_t)Imm < 16; }]>; 5230def simm5_64b_tgt : TImmLeaf<i64, [{ return (int64_t)Imm >= -16 && (int64_t)Imm < 16; }]>; 5231def i64imm_32bit_tgt : TImmLeaf<i64, [{ 5232 return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm); 5233}]>; 5234 5235class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5236 Operand imm_ty> 5237: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b), 5238 asm, "\t$Zd, $imm5, $imm5b", 5239 "", []>, Sched<[]> { 5240 bits<5> Zd; 5241 bits<5> imm5; 5242 bits<5> imm5b; 5243 let Inst{31-24} = 0b00000100; 5244 let Inst{23-22} = sz8_64; 5245 let Inst{21} = 0b1; 5246 let Inst{20-16} = imm5b; 5247 let Inst{15-10} = 0b010000; 5248 let Inst{9-5} = imm5; 5249 let Inst{4-0} = Zd; 5250 5251 let isReMaterializable = 1; 5252} 5253 5254multiclass sve_int_index_ii<string asm> { 5255 def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>; 5256 def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>; 5257 def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>; 5258 def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>; 5259 5260 def : Pat<(nxv16i8 (step_vector simm5_8b_tgt:$imm5b)), 5261 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5262 def : Pat<(nxv8i16 (step_vector simm5_16b_tgt:$imm5b)), 5263 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5264 def : Pat<(nxv4i32 (step_vector simm5_32b_tgt:$imm5b)), 5265 (!cast<Instruction>(NAME # "_S") (i32 0), simm5_32b:$imm5b)>; 5266 def : Pat<(nxv2i64 (step_vector simm5_64b_tgt:$imm5b)), 5267 (!cast<Instruction>(NAME # "_D") (i64 0), simm5_64b:$imm5b)>; 5268 5269 // add(step_vector(step), dup(X)) -> index(X, step). 5270 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5b)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5271 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5272 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5b)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5273 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5274 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5b)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5275 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>; 5276 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5b)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5277 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>; 5278} 5279 5280class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5281 RegisterClass srcRegType, Operand imm_ty> 5282: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm), 5283 asm, "\t$Zd, $imm5, $Rm", 5284 "", []>, Sched<[]> { 5285 bits<5> Rm; 5286 bits<5> Zd; 5287 bits<5> imm5; 5288 let Inst{31-24} = 0b00000100; 5289 let Inst{23-22} = sz8_64; 5290 let Inst{21} = 0b1; 5291 let Inst{20-16} = Rm; 5292 let Inst{15-10} = 0b010010; 5293 let Inst{9-5} = imm5; 5294 let Inst{4-0} = Zd; 5295} 5296 5297multiclass sve_int_index_ir<string asm, SDPatternOperator mulop, SDPatternOperator muloneuseop> { 5298 def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>; 5299 def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>; 5300 def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>; 5301 def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>; 5302 5303 def : Pat<(nxv16i8 (step_vector i8:$imm)), 5304 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5305 def : Pat<(nxv8i16 (step_vector i16:$imm)), 5306 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5307 def : Pat<(nxv4i32 (step_vector i32:$imm)), 5308 (!cast<Instruction>(NAME # "_S") (i32 0), (!cast<Instruction>("MOVi32imm") $imm))>; 5309 def : Pat<(nxv2i64 (step_vector i64:$imm)), 5310 (!cast<Instruction>(NAME # "_D") (i64 0), (!cast<Instruction>("MOVi64imm") $imm))>; 5311 def : Pat<(nxv2i64 (step_vector i64imm_32bit_tgt:$imm)), 5312 (!cast<Instruction>(NAME # "_D") (i64 0), (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5313 5314 // add(step_vector(step), dup(X)) -> index(X, step). 5315 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5316 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5317 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5318 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5319 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5320 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, (!cast<Instruction>("MOVi32imm") $imm))>; 5321 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5322 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (!cast<Instruction>("MOVi64imm") $imm))>; 5323 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5324 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5325 5326 // mul(step_vector(1), dup(Y)) -> index(0, Y). 5327 def : Pat<(mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), 5328 (!cast<Instruction>(NAME # "_B") (i32 0), GPR32:$Rm)>; 5329 def : Pat<(mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), 5330 (!cast<Instruction>(NAME # "_H") (i32 0), GPR32:$Rm)>; 5331 def : Pat<(mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), 5332 (!cast<Instruction>(NAME # "_S") (i32 0), GPR32:$Rm)>; 5333 def : Pat<(mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), 5334 (!cast<Instruction>(NAME # "_D") (i64 0), GPR64:$Rm)>; 5335 5336 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5337 def : Pat<(add (muloneuseop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5338 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>; 5339 def : Pat<(add (muloneuseop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5340 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>; 5341 def : Pat<(add (muloneuseop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5342 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>; 5343 def : Pat<(add (muloneuseop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5344 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>; 5345} 5346 5347class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5348 RegisterClass srcRegType, Operand imm_ty> 5349: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5), 5350 asm, "\t$Zd, $Rn, $imm5", 5351 "", []>, Sched<[]> { 5352 bits<5> Rn; 5353 bits<5> Zd; 5354 bits<5> imm5; 5355 let Inst{31-24} = 0b00000100; 5356 let Inst{23-22} = sz8_64; 5357 let Inst{21} = 0b1; 5358 let Inst{20-16} = imm5; 5359 let Inst{15-10} = 0b010001; 5360 let Inst{9-5} = Rn; 5361 let Inst{4-0} = Zd; 5362} 5363 5364multiclass sve_int_index_ri<string asm> { 5365 def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>; 5366 def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>; 5367 def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>; 5368 def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>; 5369 5370 // add(step_vector(step), dup(X)) -> index(X, step). 5371 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5)), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), 5372 (!cast<Instruction>(NAME # "_B") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5373 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5)), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), 5374 (!cast<Instruction>(NAME # "_H") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5375 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5)), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), 5376 (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>; 5377 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5)), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), 5378 (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>; 5379} 5380 5381class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5382 RegisterClass srcRegType> 5383: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm), 5384 asm, "\t$Zd, $Rn, $Rm", 5385 "", []>, Sched<[]> { 5386 bits<5> Zd; 5387 bits<5> Rm; 5388 bits<5> Rn; 5389 let Inst{31-24} = 0b00000100; 5390 let Inst{23-22} = sz8_64; 5391 let Inst{21} = 0b1; 5392 let Inst{20-16} = Rm; 5393 let Inst{15-10} = 0b010011; 5394 let Inst{9-5} = Rn; 5395 let Inst{4-0} = Zd; 5396} 5397 5398multiclass sve_int_index_rr<string asm, SDPatternOperator mulop> { 5399 def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>; 5400 def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>; 5401 def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>; 5402 def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>; 5403 5404 // add(step_vector(step), dup(X)) -> index(X, step). 5405 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))), 5406 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5407 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(i32 GPR32:$Rn)))), 5408 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5409 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(i32 GPR32:$Rn)))), 5410 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") $imm))>; 5411 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5412 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (!cast<Instruction>("MOVi64imm") $imm))>; 5413 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5414 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5415 5416 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5417 def : Pat<(add (mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))), 5418 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, GPR32:$Rm)>; 5419 def : Pat<(add (mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))),(nxv8i16 (splat_vector(i32 GPR32:$Rn)))), 5420 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, GPR32:$Rm)>; 5421 def : Pat<(add (mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))),(nxv4i32 (splat_vector(i32 GPR32:$Rn)))), 5422 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, GPR32:$Rm)>; 5423 def : Pat<(add (mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))),(nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5424 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, GPR64:$Rm)>; 5425} 5426 5427//===----------------------------------------------------------------------===// 5428// SVE Bitwise Shift - Predicated Group 5429//===----------------------------------------------------------------------===// 5430 5431class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm, 5432 ZPRRegOp zprty, Operand immtype> 5433: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm), 5434 asm, "\t$Zdn, $Pg/m, $_Zdn, $imm", 5435 "", 5436 []>, Sched<[]> { 5437 bits<3> Pg; 5438 bits<5> Zdn; 5439 bits<6> imm; 5440 let Inst{31-24} = 0b00000100; 5441 let Inst{23-22} = tsz8_64{3-2}; 5442 let Inst{21-20} = 0b00; 5443 let Inst{19-16} = opc; 5444 let Inst{15-13} = 0b100; 5445 let Inst{12-10} = Pg; 5446 let Inst{9-8} = tsz8_64{1-0}; 5447 let Inst{7-5} = imm{2-0}; // imm3 5448 let Inst{4-0} = Zdn; 5449 5450 let Constraints = "$Zdn = $_Zdn"; 5451 let DestructiveInstType = DestructiveBinaryImm; 5452 let ElementSize = zprty.ElementSize; 5453} 5454 5455multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps, 5456 SDPatternOperator op = null_frag> { 5457 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5458 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5459 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5460 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5461 let Inst{8} = imm{3}; 5462 } 5463 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5464 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5465 let Inst{9-8} = imm{4-3}; 5466 } 5467 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5468 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5469 let Inst{22} = imm{5}; 5470 let Inst{9-8} = imm{4-3}; 5471 } 5472 5473 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 5474 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 5475 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 5476 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 5477} 5478 5479// As above but shift amount takes the form of a "vector immediate". 5480multiclass sve_int_bin_pred_shift_imm_left_dup<bits<4> opc, string asm, 5481 string Ps, SDPatternOperator op> 5482: sve_int_bin_pred_shift_imm_left<opc, asm, Ps, null_frag> { 5483 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5484 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5485 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5486 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5487} 5488 5489multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> { 5490 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, tvecshiftL8, FalseLanesZero>; 5491 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>; 5492 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>; 5493 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>; 5494 5495 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8, !cast<Pseudo>(NAME # _ZERO_B)>; 5496 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>; 5497 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>; 5498 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>; 5499} 5500 5501multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps, 5502 SDPatternOperator op = null_frag> { 5503 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5504 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5505 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5506 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5507 let Inst{8} = imm{3}; 5508 } 5509 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5510 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5511 let Inst{9-8} = imm{4-3}; 5512 } 5513 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5514 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5515 let Inst{22} = imm{5}; 5516 let Inst{9-8} = imm{4-3}; 5517 } 5518 5519 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 5520 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 5521 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 5522 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 5523} 5524 5525// As above but shift amount takes the form of a "vector immediate". 5526multiclass sve_int_bin_pred_shift_imm_right_dup<bits<4> opc, string asm, 5527 string Ps, SDPatternOperator op> 5528: sve_int_bin_pred_shift_imm_right<opc, asm, Ps, null_frag> { 5529 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5530 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5531 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5532 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5533} 5534 5535multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> { 5536 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>; 5537 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>; 5538 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>; 5539 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>; 5540 5541 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftR8, !cast<Pseudo>(NAME # _ZERO_B)>; 5542 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftR16, !cast<Pseudo>(NAME # _ZERO_H)>; 5543 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftR32, !cast<Pseudo>(NAME # _ZERO_S)>; 5544 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftR64, !cast<Pseudo>(NAME # _ZERO_D)>; 5545} 5546 5547class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc, 5548 string asm, ZPRRegOp zprty, ZPRRegOp zprty2> 5549: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm), 5550 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 5551 "", 5552 []>, Sched<[]> { 5553 bits<3> Pg; 5554 bits<5> Zdn; 5555 bits<5> Zm; 5556 let Inst{31-24} = 0b00000100; 5557 let Inst{23-22} = sz8_64; 5558 let Inst{21-20} = 0b01; 5559 let Inst{19} = wide; 5560 let Inst{18-16} = opc; 5561 let Inst{15-13} = 0b100; 5562 let Inst{12-10} = Pg; 5563 let Inst{9-5} = Zm; 5564 let Inst{4-0} = Zdn; 5565 5566 let Constraints = "$Zdn = $_Zdn"; 5567 let DestructiveInstType = DestructiveOther; 5568 let ElementSize = zprty.ElementSize; 5569} 5570 5571multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps, 5572 SDPatternOperator op, string revname, bit isReverseInstr = 0> { 5573 let DestructiveInstType = DestructiveBinaryCommWithRev in { 5574 def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>, 5575 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 5576 def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>, 5577 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 5578 def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>, 5579 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 5580 def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>, 5581 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 5582 } 5583 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5584 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5585 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5586 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5587} 5588 5589multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> { 5590 def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>; 5591 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 5592 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 5593 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 5594 5595 def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _ZERO_B)>; 5596 def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _ZERO_H)>; 5597 def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _ZERO_S)>; 5598 def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _ZERO_D)>; 5599} 5600 5601multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm, 5602 SDPatternOperator op> { 5603 def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>; 5604 def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>; 5605 def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>; 5606 5607 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5608 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5609 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5610} 5611 5612//===----------------------------------------------------------------------===// 5613// SVE Shift - Unpredicated Group 5614//===----------------------------------------------------------------------===// 5615 5616class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm, 5617 ZPRRegOp zprty> 5618: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm), 5619 asm, "\t$Zd, $Zn, $Zm", 5620 "", 5621 []>, Sched<[]> { 5622 bits<5> Zd; 5623 bits<5> Zm; 5624 bits<5> Zn; 5625 let Inst{31-24} = 0b00000100; 5626 let Inst{23-22} = sz8_64; 5627 let Inst{21} = 0b1; 5628 let Inst{20-16} = Zm; 5629 let Inst{15-12} = 0b1000; 5630 let Inst{11-10} = opc; 5631 let Inst{9-5} = Zn; 5632 let Inst{4-0} = Zd; 5633} 5634 5635multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm, SDPatternOperator op> { 5636 def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>; 5637 def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>; 5638 def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>; 5639 5640 def : SVE_2_Op_Pred_All_Active<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5641 def : SVE_2_Op_Pred_All_Active<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5642 def : SVE_2_Op_Pred_All_Active<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5643} 5644 5645class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 5646 ZPRRegOp zprty, Operand immtype> 5647: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm), 5648 asm, "\t$Zd, $Zn, $imm", 5649 "", 5650 []>, Sched<[]> { 5651 bits<5> Zd; 5652 bits<5> Zn; 5653 bits<6> imm; 5654 let Inst{31-24} = 0b00000100; 5655 let Inst{23-22} = tsz8_64{3-2}; 5656 let Inst{21} = 0b1; 5657 let Inst{20-19} = tsz8_64{1-0}; 5658 let Inst{18-16} = imm{2-0}; // imm3 5659 let Inst{15-12} = 0b1001; 5660 let Inst{11-10} = opc; 5661 let Inst{9-5} = Zn; 5662 let Inst{4-0} = Zd; 5663} 5664 5665multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm, 5666 SDPatternOperator op> { 5667 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5668 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5669 let Inst{19} = imm{3}; 5670 } 5671 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5672 let Inst{20-19} = imm{4-3}; 5673 } 5674 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5675 let Inst{22} = imm{5}; 5676 let Inst{20-19} = imm{4-3}; 5677 } 5678 5679 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5680 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5681 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5682 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5683} 5684 5685multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm, 5686 SDPatternOperator op> { 5687 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5688 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5689 let Inst{19} = imm{3}; 5690 } 5691 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5692 let Inst{20-19} = imm{4-3}; 5693 } 5694 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5695 let Inst{22} = imm{5}; 5696 let Inst{20-19} = imm{4-3}; 5697 } 5698 5699 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5700 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5701 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5702 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5703} 5704 5705//===----------------------------------------------------------------------===// 5706// SVE Memory - Store Group 5707//===----------------------------------------------------------------------===// 5708 5709class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5710 RegisterOperand VecList> 5711: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5712 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5713 "", 5714 []>, Sched<[]> { 5715 bits<3> Pg; 5716 bits<5> Rn; 5717 bits<5> Zt; 5718 bits<4> imm4; 5719 let Inst{31-25} = 0b1110010; 5720 let Inst{24-23} = msz; 5721 let Inst{22-21} = esz; 5722 let Inst{20} = 0; 5723 let Inst{19-16} = imm4; 5724 let Inst{15-13} = 0b111; 5725 let Inst{12-10} = Pg; 5726 let Inst{9-5} = Rn; 5727 let Inst{4-0} = Zt; 5728 5729 let mayStore = 1; 5730} 5731 5732multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5733 RegisterOperand listty, ZPRRegOp zprty> 5734{ 5735 def NAME : sve_mem_cst_si<msz, esz, asm, listty>; 5736 5737 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5738 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5739 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5740 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5741 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5742 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5743} 5744 5745class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5746 string asm, Operand immtype> 5747: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 5748 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5749 "", 5750 []>, Sched<[]> { 5751 bits<3> Pg; 5752 bits<5> Rn; 5753 bits<5> Zt; 5754 bits<4> imm4; 5755 let Inst{31-25} = 0b1110010; 5756 let Inst{24-23} = sz; 5757 let Inst{22-21} = nregs; 5758 let Inst{20} = 1; 5759 let Inst{19-16} = imm4; 5760 let Inst{15-13} = 0b111; 5761 let Inst{12-10} = Pg; 5762 let Inst{9-5} = Rn; 5763 let Inst{4-0} = Zt; 5764 5765 let mayStore = 1; 5766} 5767 5768multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5769 string asm, Operand immtype> { 5770 def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>; 5771 5772 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5773 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5774} 5775 5776class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5777 string asm, RegisterOperand gprty> 5778: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5779 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5780 "", 5781 []>, Sched<[]> { 5782 bits<3> Pg; 5783 bits<5> Rm; 5784 bits<5> Rn; 5785 bits<5> Zt; 5786 let Inst{31-25} = 0b1110010; 5787 let Inst{24-23} = sz; 5788 let Inst{22-21} = nregs; 5789 let Inst{20-16} = Rm; 5790 let Inst{15-13} = 0b011; 5791 let Inst{12-10} = Pg; 5792 let Inst{9-5} = Rn; 5793 let Inst{4-0} = Zt; 5794 5795 let mayStore = 1; 5796} 5797 5798class sve_mem_cst_ss_base<bits<4> dtype, string asm, 5799 RegisterOperand listty, RegisterOperand gprty> 5800: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5801 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5802 "", 5803 []>, Sched<[]> { 5804 bits<3> Pg; 5805 bits<5> Rm; 5806 bits<5> Rn; 5807 bits<5> Zt; 5808 let Inst{31-25} = 0b1110010; 5809 let Inst{24-21} = dtype; 5810 let Inst{20-16} = Rm; 5811 let Inst{15-13} = 0b010; 5812 let Inst{12-10} = Pg; 5813 let Inst{9-5} = Rn; 5814 let Inst{4-0} = Zt; 5815 5816 let mayStore = 1; 5817} 5818 5819multiclass sve_mem_cst_ss<bits<4> dtype, string asm, 5820 RegisterOperand listty, ZPRRegOp zprty, 5821 RegisterOperand gprty> { 5822 def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>; 5823 5824 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5825 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5826} 5827 5828class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList> 5829: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5830 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5831 "", 5832 []>, Sched<[]> { 5833 bits<3> Pg; 5834 bits<5> Rn; 5835 bits<5> Zt; 5836 bits<4> imm4; 5837 let Inst{31-25} = 0b1110010; 5838 let Inst{24-23} = msz; 5839 let Inst{22-20} = 0b001; 5840 let Inst{19-16} = imm4; 5841 let Inst{15-13} = 0b111; 5842 let Inst{12-10} = Pg; 5843 let Inst{9-5} = Rn; 5844 let Inst{4-0} = Zt; 5845 5846 let mayStore = 1; 5847} 5848 5849multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty, 5850 ZPRRegOp zprty> { 5851 def NAME : sve_mem_cstnt_si<msz, asm, listty>; 5852 5853 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5854 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5855 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5856 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5857 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5858 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5859} 5860 5861class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty, 5862 RegisterOperand gprty> 5863: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5864 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5865 "", 5866 []>, Sched<[]> { 5867 bits<3> Pg; 5868 bits<5> Rm; 5869 bits<5> Rn; 5870 bits<5> Zt; 5871 let Inst{31-25} = 0b1110010; 5872 let Inst{24-23} = msz; 5873 let Inst{22-21} = 0b00; 5874 let Inst{20-16} = Rm; 5875 let Inst{15-13} = 0b011; 5876 let Inst{12-10} = Pg; 5877 let Inst{9-5} = Rn; 5878 let Inst{4-0} = Zt; 5879 5880 let mayStore = 1; 5881} 5882 5883multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty, 5884 ZPRRegOp zprty, RegisterOperand gprty> { 5885 def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>; 5886 5887 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5888 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5889} 5890 5891class sve2_mem_sstnt_vs_base<bits<3> opc, string asm, 5892 RegisterOperand listty, ZPRRegOp zprty> 5893: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 5894 asm, "\t$Zt, $Pg, [$Zn, $Rm]", 5895 "", 5896 []>, Sched<[]> { 5897 bits<3> Pg; 5898 bits<5> Rm; 5899 bits<5> Zn; 5900 bits<5> Zt; 5901 let Inst{31-25} = 0b1110010; 5902 let Inst{24-22} = opc; 5903 let Inst{21} = 0b0; 5904 let Inst{20-16} = Rm; 5905 let Inst{15-13} = 0b001; 5906 let Inst{12-10} = Pg; 5907 let Inst{9-5} = Zn; 5908 let Inst{4-0} = Zt; 5909 5910 let mayStore = 1; 5911} 5912 5913multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm, 5914 SDPatternOperator op, 5915 ValueType vt> { 5916 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>; 5917 5918 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5919 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 5920 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5921 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 5922 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5923 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 5924 5925 def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt), 5926 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; 5927} 5928 5929multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm, 5930 SDPatternOperator op, 5931 ValueType vt> { 5932 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>; 5933 5934 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5935 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 5936 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5937 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 5938 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5939 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 5940 5941 def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt), 5942 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; 5943} 5944 5945class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm, 5946 RegisterOperand VecList, RegisterOperand zprext> 5947: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5948 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5949 "", 5950 []>, Sched<[]> { 5951 bits<3> Pg; 5952 bits<5> Rn; 5953 bits<5> Zm; 5954 bits<5> Zt; 5955 let Inst{31-25} = 0b1110010; 5956 let Inst{24-22} = opc; 5957 let Inst{21} = scaled; 5958 let Inst{20-16} = Zm; 5959 let Inst{15} = 0b1; 5960 let Inst{14} = xs; 5961 let Inst{13} = 0; 5962 let Inst{12-10} = Pg; 5963 let Inst{9-5} = Rn; 5964 let Inst{4-0} = Zt; 5965 5966 let mayStore = 1; 5967} 5968 5969multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm, 5970 SDPatternOperator sxtw_op, 5971 SDPatternOperator uxtw_op, 5972 RegisterOperand sxtw_opnd, 5973 RegisterOperand uxtw_opnd, 5974 ValueType vt > { 5975 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>; 5976 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>; 5977 5978 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5979 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5980 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5981 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5982 5983 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5984 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5985 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5986 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5987} 5988 5989multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm, 5990 SDPatternOperator sxtw_op, 5991 SDPatternOperator uxtw_op, 5992 RegisterOperand sxtw_opnd, 5993 RegisterOperand uxtw_opnd, 5994 ValueType vt > { 5995 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>; 5996 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>; 5997 5998 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5999 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6000 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6001 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6002 6003 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6004 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6005 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6006 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6007} 6008 6009multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm, 6010 SDPatternOperator sxtw_op, 6011 SDPatternOperator uxtw_op, 6012 RegisterOperand sxtw_opnd, 6013 RegisterOperand uxtw_opnd, 6014 ValueType vt> { 6015 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>; 6016 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>; 6017 6018 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6019 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6020 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6021 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6022 6023 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6024 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6025 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6026 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6027} 6028 6029multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm, 6030 SDPatternOperator sxtw_op, 6031 SDPatternOperator uxtw_op, 6032 RegisterOperand sxtw_opnd, 6033 RegisterOperand uxtw_opnd, 6034 ValueType vt> { 6035 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>; 6036 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>; 6037 6038 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6039 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6040 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6041 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6042 6043 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6044 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6045 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6046 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6047} 6048 6049class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm, 6050 RegisterOperand zprext> 6051: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6052 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 6053 "", 6054 []>, Sched<[]> { 6055 bits<3> Pg; 6056 bits<5> Rn; 6057 bits<5> Zm; 6058 bits<5> Zt; 6059 let Inst{31-25} = 0b1110010; 6060 let Inst{24-23} = msz; 6061 let Inst{22} = 0b0; 6062 let Inst{21} = scaled; 6063 let Inst{20-16} = Zm; 6064 let Inst{15-13} = 0b101; 6065 let Inst{12-10} = Pg; 6066 let Inst{9-5} = Rn; 6067 let Inst{4-0} = Zt; 6068 6069 let mayStore = 1; 6070} 6071 6072multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm, 6073 SDPatternOperator op, 6074 RegisterOperand zprext, 6075 ValueType vt> { 6076 def _SCALED : sve_mem_sst_sv2<msz, 1, asm, zprext>; 6077 6078 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6079 (!cast<Instruction>(NAME # _SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 6080 6081 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt), 6082 (!cast<Instruction>(NAME # _SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6083} 6084 6085multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm, 6086 SDPatternOperator op, 6087 ValueType vt> { 6088 def NAME : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>; 6089 6090 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6091 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 6092 6093 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6094 (!cast<Instruction>(NAME) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6095} 6096 6097class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty, 6098 RegisterOperand VecList, Operand imm_ty> 6099: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5), 6100 asm, "\t$Zt, $Pg, [$Zn, $imm5]", 6101 "", 6102 []>, Sched<[]> { 6103 bits<3> Pg; 6104 bits<5> imm5; 6105 bits<5> Zn; 6106 bits<5> Zt; 6107 let Inst{31-25} = 0b1110010; 6108 let Inst{24-23} = opc{2-1}; 6109 let Inst{22} = 0b1; 6110 let Inst{21} = opc{0}; 6111 let Inst{20-16} = imm5; 6112 let Inst{15-13} = 0b101; 6113 let Inst{12-10} = Pg; 6114 let Inst{9-5} = Zn; 6115 let Inst{4-0} = Zt; 6116 6117 let mayStore = 1; 6118} 6119 6120multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm, 6121 Operand imm_ty, 6122 SDPatternOperator op, 6123 ValueType vt> { 6124 def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>; 6125 6126 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6127 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 6128 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 6129 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 6130 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6131 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 6132 6133 def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt), 6134 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6135} 6136 6137multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm, 6138 Operand imm_ty, 6139 SDPatternOperator op, 6140 ValueType vt> { 6141 def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>; 6142 6143 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6144 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 6145 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 6146 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 6147 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6148 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 6149 6150 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt), 6151 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6152} 6153 6154class sve_mem_z_spill<string asm> 6155: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9), 6156 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 6157 "", 6158 []>, Sched<[]> { 6159 bits<5> Rn; 6160 bits<5> Zt; 6161 bits<9> imm9; 6162 let Inst{31-22} = 0b1110010110; 6163 let Inst{21-16} = imm9{8-3}; 6164 let Inst{15-13} = 0b010; 6165 let Inst{12-10} = imm9{2-0}; 6166 let Inst{9-5} = Rn; 6167 let Inst{4-0} = Zt; 6168 6169 let mayStore = 1; 6170} 6171 6172multiclass sve_mem_z_spill<string asm> { 6173 def NAME : sve_mem_z_spill<asm>; 6174 6175 def : InstAlias<asm # "\t$Zt, [$Rn]", 6176 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 6177} 6178 6179class sve_mem_p_spill<string asm> 6180: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9), 6181 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 6182 "", 6183 []>, Sched<[]> { 6184 bits<4> Pt; 6185 bits<5> Rn; 6186 bits<9> imm9; 6187 let Inst{31-22} = 0b1110010110; 6188 let Inst{21-16} = imm9{8-3}; 6189 let Inst{15-13} = 0b000; 6190 let Inst{12-10} = imm9{2-0}; 6191 let Inst{9-5} = Rn; 6192 let Inst{4} = 0b0; 6193 let Inst{3-0} = Pt; 6194 6195 let mayStore = 1; 6196} 6197 6198multiclass sve_mem_p_spill<string asm> { 6199 def NAME : sve_mem_p_spill<asm>; 6200 6201 def : InstAlias<asm # "\t$Pt, [$Rn]", 6202 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 6203} 6204 6205//===----------------------------------------------------------------------===// 6206// SVE Permute - Predicates Group 6207//===----------------------------------------------------------------------===// 6208 6209class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, 6210 PPRRegOp pprty> 6211: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm), 6212 asm, "\t$Pd, $Pn, $Pm", 6213 "", []>, Sched<[]> { 6214 bits<4> Pd; 6215 bits<4> Pm; 6216 bits<4> Pn; 6217 let Inst{31-24} = 0b00000101; 6218 let Inst{23-22} = sz8_64; 6219 let Inst{21-20} = 0b10; 6220 let Inst{19-16} = Pm; 6221 let Inst{15-13} = 0b010; 6222 let Inst{12-10} = opc; 6223 let Inst{9} = 0b0; 6224 let Inst{8-5} = Pn; 6225 let Inst{4} = 0b0; 6226 let Inst{3-0} = Pd; 6227} 6228 6229multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm, 6230 SDPatternOperator op> { 6231 def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>; 6232 def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>; 6233 def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>; 6234 def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>; 6235 6236 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 6237 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 6238 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 6239 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 6240} 6241 6242class sve_int_perm_punpk<bit opc, string asm> 6243: I<(outs PPR16:$Pd), (ins PPR8:$Pn), 6244 asm, "\t$Pd, $Pn", 6245 "", 6246 []>, Sched<[]> { 6247 bits<4> Pd; 6248 bits<4> Pn; 6249 let Inst{31-17} = 0b000001010011000; 6250 let Inst{16} = opc; 6251 let Inst{15-9} = 0b0100000; 6252 let Inst{8-5} = Pn; 6253 let Inst{4} = 0b0; 6254 let Inst{3-0} = Pd; 6255} 6256 6257multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> { 6258 def NAME : sve_int_perm_punpk<opc, asm>; 6259 6260 def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>; 6261 def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1, !cast<Instruction>(NAME)>; 6262 def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1, !cast<Instruction>(NAME)>; 6263} 6264 6265class sve_int_rdffr_pred<bit s, string asm> 6266: I<(outs PPR8:$Pd), (ins PPRAny:$Pg), 6267 asm, "\t$Pd, $Pg/z", 6268 "", 6269 []>, Sched<[]> { 6270 bits<4> Pd; 6271 bits<4> Pg; 6272 let Inst{31-23} = 0b001001010; 6273 let Inst{22} = s; 6274 let Inst{21-9} = 0b0110001111000; 6275 let Inst{8-5} = Pg; 6276 let Inst{4} = 0; 6277 let Inst{3-0} = Pd; 6278 6279 let Defs = !if(s, [NZCV], []); 6280 let Uses = [FFR]; 6281} 6282 6283multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> { 6284 def _REAL : sve_int_rdffr_pred<s, asm>; 6285 6286 // We need a layer of indirection because early machine code passes balk at 6287 // physical register (i.e. FFR) uses that have no previous definition. 6288 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6289 def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, 6290 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; 6291 } 6292} 6293 6294class sve_int_rdffr_unpred<string asm> : I< 6295 (outs PPR8:$Pd), (ins), 6296 asm, "\t$Pd", 6297 "", 6298 []>, Sched<[]> { 6299 bits<4> Pd; 6300 let Inst{31-4} = 0b0010010100011001111100000000; 6301 let Inst{3-0} = Pd; 6302 6303 let Uses = [FFR]; 6304} 6305 6306multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> { 6307 def _REAL : sve_int_rdffr_unpred<asm>; 6308 6309 // We need a layer of indirection because early machine code passes balk at 6310 // physical register (i.e. FFR) uses that have no previous definition. 6311 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6312 def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, 6313 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>; 6314 } 6315} 6316 6317class sve_int_wrffr<string asm, SDPatternOperator op> 6318: I<(outs), (ins PPR8:$Pn), 6319 asm, "\t$Pn", 6320 "", 6321 [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 6322 bits<4> Pn; 6323 let Inst{31-9} = 0b00100101001010001001000; 6324 let Inst{8-5} = Pn; 6325 let Inst{4-0} = 0b00000; 6326 6327 let hasSideEffects = 1; 6328 let Defs = [FFR]; 6329} 6330 6331class sve_int_setffr<string asm, SDPatternOperator op> 6332: I<(outs), (ins), 6333 asm, "", 6334 "", 6335 [(op)]>, Sched<[]> { 6336 let Inst{31-0} = 0b00100101001011001001000000000000; 6337 6338 let hasSideEffects = 1; 6339 let Defs = [FFR]; 6340} 6341 6342//===----------------------------------------------------------------------===// 6343// SVE Permute Vector - Predicated Group 6344//===----------------------------------------------------------------------===// 6345 6346class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm, 6347 ZPRRegOp zprty, RegisterClass rt> 6348: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm), 6349 asm, "\t$Rdn, $Pg, $_Rdn, $Zm", 6350 "", 6351 []>, Sched<[]> { 6352 bits<3> Pg; 6353 bits<5> Rdn; 6354 bits<5> Zm; 6355 let Inst{31-24} = 0b00000101; 6356 let Inst{23-22} = sz8_64; 6357 let Inst{21-17} = 0b11000; 6358 let Inst{16} = ab; 6359 let Inst{15-13} = 0b101; 6360 let Inst{12-10} = Pg; 6361 let Inst{9-5} = Zm; 6362 let Inst{4-0} = Rdn; 6363 6364 let Constraints = "$Rdn = $_Rdn"; 6365} 6366 6367multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> { 6368 def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>; 6369 def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>; 6370 def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>; 6371 def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>; 6372 6373 def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>; 6374 def : SVE_3_Op_Pat<i32, op, nxv8i1, i32, nxv8i16, !cast<Instruction>(NAME # _H)>; 6375 def : SVE_3_Op_Pat<i32, op, nxv4i1, i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6376 def : SVE_3_Op_Pat<i64, op, nxv2i1, i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6377} 6378 6379class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm, 6380 ZPRRegOp zprty, RegisterClass rt> 6381: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm), 6382 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 6383 "", 6384 []>, Sched<[]> { 6385 bits<3> Pg; 6386 bits<5> Vdn; 6387 bits<5> Zm; 6388 let Inst{31-24} = 0b00000101; 6389 let Inst{23-22} = sz8_64; 6390 let Inst{21-17} = 0b10101; 6391 let Inst{16} = ab; 6392 let Inst{15-13} = 0b100; 6393 let Inst{12-10} = Pg; 6394 let Inst{9-5} = Zm; 6395 let Inst{4-0} = Vdn; 6396 6397 let Constraints = "$Vdn = $_Vdn"; 6398} 6399 6400multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> { 6401 def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>; 6402 def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>; 6403 def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>; 6404 def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>; 6405 6406 def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6407 def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6408 def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6409 6410 def : SVE_3_Op_Pat<bf16, op, nxv8i1, bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6411} 6412 6413class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm, 6414 ZPRRegOp zprty> 6415: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6416 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6417 "", 6418 []>, Sched<[]> { 6419 bits<3> Pg; 6420 bits<5> Zdn; 6421 bits<5> Zm; 6422 let Inst{31-24} = 0b00000101; 6423 let Inst{23-22} = sz8_64; 6424 let Inst{21-17} = 0b10100; 6425 let Inst{16} = ab; 6426 let Inst{15-13} = 0b100; 6427 let Inst{12-10} = Pg; 6428 let Inst{9-5} = Zm; 6429 let Inst{4-0} = Zdn; 6430 6431 let Constraints = "$Zdn = $_Zdn"; 6432 let DestructiveInstType = DestructiveOther; 6433 let ElementSize = ElementSizeNone; 6434} 6435 6436multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> { 6437 def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>; 6438 def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>; 6439 def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>; 6440 def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>; 6441 6442 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6443 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6444 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6445 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6446 6447 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6448 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6449 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6450 6451 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6452} 6453 6454class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm, 6455 ZPRRegOp zprty, RegisterClass resultRegType> 6456: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn), 6457 asm, "\t$Rd, $Pg, $Zn", 6458 "", 6459 []>, Sched<[]> { 6460 bits<3> Pg; 6461 bits<5> Rd; 6462 bits<5> Zn; 6463 let Inst{31-24} = 0b00000101; 6464 let Inst{23-22} = sz8_64; 6465 let Inst{21-17} = 0b10000; 6466 let Inst{16} = ab; 6467 let Inst{15-13} = 0b101; 6468 let Inst{12-10} = Pg; 6469 let Inst{9-5} = Zn; 6470 let Inst{4-0} = Rd; 6471} 6472 6473multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> { 6474 def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>; 6475 def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>; 6476 def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>; 6477 def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>; 6478 6479 def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6480 def : SVE_2_Op_Pat<i32, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6481 def : SVE_2_Op_Pat<i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6482 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6483} 6484 6485class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm, 6486 ZPRRegOp zprty, RegisterClass dstRegtype> 6487: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 6488 asm, "\t$Vd, $Pg, $Zn", 6489 "", 6490 []>, Sched<[]> { 6491 bits<3> Pg; 6492 bits<5> Vd; 6493 bits<5> Zn; 6494 let Inst{31-24} = 0b00000101; 6495 let Inst{23-22} = sz8_64; 6496 let Inst{21-17} = 0b10001; 6497 let Inst{16} = ab; 6498 let Inst{15-13} = 0b100; 6499 let Inst{12-10} = Pg; 6500 let Inst{9-5} = Zn; 6501 let Inst{4-0} = Vd; 6502} 6503 6504multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> { 6505 def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>; 6506 def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>; 6507 def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>; 6508 def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>; 6509 6510 def : SVE_2_Op_Pat<f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 6511 def : SVE_2_Op_Pat<f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6512 def : SVE_2_Op_Pat<f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 6513 def : SVE_2_Op_Pat<f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6514 6515 def : SVE_2_Op_Pat<bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6516} 6517 6518class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty> 6519: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6520 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6521 "", 6522 []>, Sched<[]> { 6523 bits<3> Pg; 6524 bits<5> Zdn; 6525 bits<5> Zm; 6526 let Inst{31-24} = 0b00000101; 6527 let Inst{23-22} = sz8_64; 6528 let Inst{21-13} = 0b101100100; 6529 let Inst{12-10} = Pg; 6530 let Inst{9-5} = Zm; 6531 let Inst{4-0} = Zdn; 6532 6533 let Constraints = "$Zdn = $_Zdn"; 6534 let DestructiveInstType = DestructiveOther; 6535 let ElementSize = ElementSizeNone; 6536} 6537 6538multiclass sve_int_perm_splice<string asm, SDPatternOperator op> { 6539 def _B : sve_int_perm_splice<0b00, asm, ZPR8>; 6540 def _H : sve_int_perm_splice<0b01, asm, ZPR16>; 6541 def _S : sve_int_perm_splice<0b10, asm, ZPR32>; 6542 def _D : sve_int_perm_splice<0b11, asm, ZPR64>; 6543 6544 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6545 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6546 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6547 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6548 6549 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6550 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6551 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6552 6553 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6554} 6555 6556class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm, 6557 ZPRRegOp zprty, RegisterOperand VecList> 6558: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn), 6559 asm, "\t$Zd, $Pg, $Zn", 6560 "", 6561 []>, Sched<[]> { 6562 bits<3> Pg; 6563 bits<5> Zn; 6564 bits<5> Zd; 6565 let Inst{31-24} = 0b00000101; 6566 let Inst{23-22} = sz8_64; 6567 let Inst{21-13} = 0b101101100; 6568 let Inst{12-10} = Pg; 6569 let Inst{9-5} = Zn; 6570 let Inst{4-0} = Zd; 6571} 6572 6573multiclass sve2_int_perm_splice_cons<string asm> { 6574 def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>; 6575 def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>; 6576 def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>; 6577 def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>; 6578} 6579 6580class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm, 6581 ZPRRegOp zprty> 6582: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 6583 asm, "\t$Zd, $Pg/m, $Zn", 6584 "", 6585 []>, Sched<[]> { 6586 bits<5> Zd; 6587 bits<3> Pg; 6588 bits<5> Zn; 6589 let Inst{31-24} = 0b00000101; 6590 let Inst{23-22} = sz8_64; 6591 let Inst{21-18} = 0b1001; 6592 let Inst{17-16} = opc; 6593 let Inst{15-13} = 0b100; 6594 let Inst{12-10} = Pg; 6595 let Inst{9-5} = Zn; 6596 let Inst{4-0} = Zd; 6597 6598 let Constraints = "$Zd = $_Zd"; 6599 let DestructiveInstType = DestructiveOther; 6600 let ElementSize = zprty.ElementSize; 6601} 6602 6603multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> { 6604 def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>; 6605 def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>; 6606 def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>; 6607 def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>; 6608 6609 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6610 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6611 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6612 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6613} 6614 6615multiclass sve_int_perm_rev_revb<string asm, SDPatternOperator op> { 6616 def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>; 6617 def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>; 6618 def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>; 6619 6620 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6621 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6622 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6623} 6624 6625multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> { 6626 def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>; 6627 def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>; 6628 6629 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6630 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6631} 6632 6633multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> { 6634 def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>; 6635 6636 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6637} 6638 6639class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6640 RegisterClass srcRegType> 6641: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn), 6642 asm, "\t$Zd, $Pg/m, $Rn", 6643 "", 6644 []>, Sched<[]> { 6645 bits<3> Pg; 6646 bits<5> Rn; 6647 bits<5> Zd; 6648 let Inst{31-24} = 0b00000101; 6649 let Inst{23-22} = sz8_64; 6650 let Inst{21-13} = 0b101000101; 6651 let Inst{12-10} = Pg; 6652 let Inst{9-5} = Rn; 6653 let Inst{4-0} = Zd; 6654 6655 let Constraints = "$Zd = $_Zd"; 6656 let DestructiveInstType = DestructiveOther; 6657 let ElementSize = zprty.ElementSize; 6658} 6659 6660multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> { 6661 def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>; 6662 def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>; 6663 def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>; 6664 def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>; 6665 6666 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6667 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6668 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6669 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6670 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6671 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6672 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6673 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>; 6674 6675 def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)), 6676 (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>; 6677 def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)), 6678 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6679 def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)), 6680 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6681 def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)), 6682 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6683} 6684 6685class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6686 RegisterClass srcRegtype> 6687: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn), 6688 asm, "\t$Zd, $Pg/m, $Vn", 6689 "", 6690 []>, Sched<[]> { 6691 bits<3> Pg; 6692 bits<5> Vn; 6693 bits<5> Zd; 6694 let Inst{31-24} = 0b00000101; 6695 let Inst{23-22} = sz8_64; 6696 let Inst{21-13} = 0b100000100; 6697 let Inst{12-10} = Pg; 6698 let Inst{9-5} = Vn; 6699 let Inst{4-0} = Zd; 6700 6701 let Constraints = "$Zd = $_Zd"; 6702 let DestructiveInstType = DestructiveOther; 6703 let ElementSize = zprty.ElementSize; 6704} 6705 6706multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> { 6707 def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>; 6708 def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>; 6709 def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>; 6710 def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>; 6711 6712 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6713 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>; 6714 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6715 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>; 6716 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6717 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>; 6718 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6719 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>; 6720 6721 def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)), 6722 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6723 def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)), 6724 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6725 def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)), 6726 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6727 def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)), 6728 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6729 6730 def : Pat<(nxv8bf16 (op nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)), 6731 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6732} 6733 6734class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty> 6735: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn), 6736 asm, "\t$Zd, $Pg, $Zn", 6737 "", 6738 []>, Sched<[]> { 6739 bits<3> Pg; 6740 bits<5> Zd; 6741 bits<5> Zn; 6742 let Inst{31-23} = 0b000001011; 6743 let Inst{22} = sz; 6744 let Inst{21-13} = 0b100001100; 6745 let Inst{12-10} = Pg; 6746 let Inst{9-5} = Zn; 6747 let Inst{4-0} = Zd; 6748} 6749 6750multiclass sve_int_perm_compact<string asm, SDPatternOperator op> { 6751 def _S : sve_int_perm_compact<0b0, asm, ZPR32>; 6752 def _D : sve_int_perm_compact<0b1, asm, ZPR64>; 6753 6754 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6755 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6756 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6757 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6758} 6759 6760//===----------------------------------------------------------------------===// 6761// SVE Memory - Contiguous Load Group 6762//===----------------------------------------------------------------------===// 6763 6764class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6765 RegisterOperand VecList> 6766: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6767 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6768 "", 6769 []>, Sched<[]> { 6770 bits<3> Pg; 6771 bits<5> Rn; 6772 bits<5> Zt; 6773 bits<4> imm4; 6774 let Inst{31-25} = 0b1010010; 6775 let Inst{24-21} = dtype; 6776 let Inst{20} = nf; 6777 let Inst{19-16} = imm4; 6778 let Inst{15-13} = 0b101; 6779 let Inst{12-10} = Pg; 6780 let Inst{9-5} = Rn; 6781 let Inst{4-0} = Zt; 6782 6783 let mayLoad = 1; 6784 let Uses = !if(nf, [FFR], []); 6785 let Defs = !if(nf, [FFR], []); 6786} 6787 6788multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6789 RegisterOperand listty, ZPRRegOp zprty> { 6790 def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>; 6791 6792 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6793 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6794 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6795 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6796 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6797 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6798 6799 // We need a layer of indirection because early machine code passes balk at 6800 // physical register (i.e. FFR) uses that have no previous definition. 6801 let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in { 6802 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>, 6803 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>; 6804 } 6805} 6806 6807multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty, 6808 ZPRRegOp zprty> 6809: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>; 6810 6811class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList> 6812: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6813 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6814 "", 6815 []>, Sched<[]> { 6816 bits<5> Zt; 6817 bits<3> Pg; 6818 bits<5> Rn; 6819 bits<4> imm4; 6820 let Inst{31-25} = 0b1010010; 6821 let Inst{24-23} = msz; 6822 let Inst{22-20} = 0b000; 6823 let Inst{19-16} = imm4; 6824 let Inst{15-13} = 0b111; 6825 let Inst{12-10} = Pg; 6826 let Inst{9-5} = Rn; 6827 let Inst{4-0} = Zt; 6828 6829 let mayLoad = 1; 6830} 6831 6832multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty, 6833 ZPRRegOp zprty> { 6834 def NAME : sve_mem_cldnt_si_base<msz, asm, listty>; 6835 6836 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6837 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6838 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6839 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6840 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6841 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6842} 6843 6844class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList, 6845 RegisterOperand gprty> 6846: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6847 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6848 "", 6849 []>, Sched<[]> { 6850 bits<3> Pg; 6851 bits<5> Rm; 6852 bits<5> Rn; 6853 bits<5> Zt; 6854 let Inst{31-25} = 0b1010010; 6855 let Inst{24-23} = msz; 6856 let Inst{22-21} = 0b00; 6857 let Inst{20-16} = Rm; 6858 let Inst{15-13} = 0b110; 6859 let Inst{12-10} = Pg; 6860 let Inst{9-5} = Rn; 6861 let Inst{4-0} = Zt; 6862 6863 let mayLoad = 1; 6864} 6865 6866multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty, 6867 ZPRRegOp zprty, RegisterOperand gprty> { 6868 def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>; 6869 6870 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6871 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6872} 6873 6874class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList> 6875: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 6876 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 6877 bits<5> Zt; 6878 bits<5> Rn; 6879 bits<3> Pg; 6880 bits<4> imm4; 6881 let Inst{31-25} = 0b1010010; 6882 let Inst{24-23} = sz; 6883 let Inst{22-20} = 0; 6884 let Inst{19-16} = imm4; 6885 let Inst{15-13} = 0b001; 6886 let Inst{12-10} = Pg; 6887 let Inst{9-5} = Rn; 6888 let Inst{4-0} = Zt; 6889 6890 let mayLoad = 1; 6891} 6892 6893multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty, 6894 ZPRRegOp zprty> { 6895 def NAME : sve_mem_ldqr_si<sz, asm, listty>; 6896 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6897 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6898 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6899 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6900 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 6901 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>; 6902} 6903 6904class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList, 6905 RegisterOperand gprty> 6906: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6907 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 6908 bits<5> Zt; 6909 bits<3> Pg; 6910 bits<5> Rn; 6911 bits<5> Rm; 6912 let Inst{31-25} = 0b1010010; 6913 let Inst{24-23} = sz; 6914 let Inst{22-21} = 0; 6915 let Inst{20-16} = Rm; 6916 let Inst{15-13} = 0; 6917 let Inst{12-10} = Pg; 6918 let Inst{9-5} = Rn; 6919 let Inst{4-0} = Zt; 6920 6921 let mayLoad = 1; 6922} 6923 6924multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty, 6925 ZPRRegOp zprty, RegisterOperand gprty> { 6926 def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>; 6927 6928 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6929 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6930} 6931 6932class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6933 RegisterOperand VecList, Operand immtype> 6934: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 6935 asm, "\t$Zt, $Pg/z, [$Rn, $imm6]", 6936 "", 6937 []>, Sched<[]> { 6938 bits<3> Pg; 6939 bits<5> Rn; 6940 bits<5> Zt; 6941 bits<6> imm6; 6942 let Inst{31-25} = 0b1000010; 6943 let Inst{24-23} = dtypeh; 6944 let Inst{22} = 1; 6945 let Inst{21-16} = imm6; 6946 let Inst{15} = 0b1; 6947 let Inst{14-13} = dtypel; 6948 let Inst{12-10} = Pg; 6949 let Inst{9-5} = Rn; 6950 let Inst{4-0} = Zt; 6951 6952 let mayLoad = 1; 6953} 6954 6955multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6956 RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> { 6957 def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>; 6958 6959 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6960 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6961 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]", 6962 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>; 6963 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6964 (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6965} 6966 6967class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm, 6968 RegisterOperand VecList> 6969: I<(outs VecList:$Zt), iops, 6970 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6971 "", 6972 []>, Sched<[]> { 6973 bits<5> Zt; 6974 bits<3> Pg; 6975 bits<5> Rm; 6976 bits<5> Rn; 6977 let Inst{31-25} = 0b1010010; 6978 let Inst{24-21} = dtype; 6979 let Inst{20-16} = Rm; 6980 let Inst{15-14} = 0b01; 6981 let Inst{13} = ff; 6982 let Inst{12-10} = Pg; 6983 let Inst{9-5} = Rn; 6984 let Inst{4-0} = Zt; 6985 6986 let mayLoad = 1; 6987 let Uses = !if(ff, [FFR], []); 6988 let Defs = !if(ff, [FFR], []); 6989} 6990 6991multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty, 6992 ZPRRegOp zprty, RegisterOperand gprty> { 6993 def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6994 asm, listty>; 6995 6996 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6997 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6998} 6999 7000multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty, 7001 ZPRRegOp zprty, RegisterOperand gprty> { 7002 def _REAL : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7003 asm, listty>; 7004 7005 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 7006 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 7007 7008 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7009 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 7010 7011 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7012 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 7013 7014 // We need a layer of indirection because early machine code passes balk at 7015 // physical register (i.e. FFR) uses that have no previous definition. 7016 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7017 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>, 7018 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>; 7019 } 7020} 7021 7022multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty, 7023 ZPRRegOp zprty> 7024: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>; 7025 7026class sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 7027 string asm, Operand immtype> 7028: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 7029 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 7030 "", 7031 []>, Sched<[]> { 7032 bits<5> Zt; 7033 bits<3> Pg; 7034 bits<5> Rn; 7035 bits<4> imm4; 7036 let Inst{31-25} = 0b1010010; 7037 let Inst{24-23} = sz; 7038 let Inst{22-21} = nregs; 7039 let Inst{20} = 0; 7040 let Inst{19-16} = imm4; 7041 let Inst{15-13} = 0b111; 7042 let Inst{12-10} = Pg; 7043 let Inst{9-5} = Rn; 7044 let Inst{4-0} = Zt; 7045 7046 let mayLoad = 1; 7047} 7048 7049multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 7050 string asm, Operand immtype> { 7051 def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>; 7052 7053 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7054 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7055} 7056 7057class sve_mem_eld_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 7058 string asm, RegisterOperand gprty> 7059: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7060 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 7061 "", 7062 []>, Sched<[]> { 7063 bits<3> Pg; 7064 bits<5> Rm; 7065 bits<5> Rn; 7066 bits<5> Zt; 7067 let Inst{31-25} = 0b1010010; 7068 let Inst{24-23} = sz; 7069 let Inst{22-21} = nregs; 7070 let Inst{20-16} = Rm; 7071 let Inst{15-13} = 0b110; 7072 let Inst{12-10} = Pg; 7073 let Inst{9-5} = Rn; 7074 let Inst{4-0} = Zt; 7075 7076 let mayLoad = 1; 7077} 7078 7079//===----------------------------------------------------------------------===// 7080// SVE Memory - 32-bit Gather and Unsized Contiguous Group 7081//===----------------------------------------------------------------------===// 7082 7083// bit xs is '1' if offsets are signed 7084// bit scaled is '1' if the offsets are scaled 7085class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm, 7086 RegisterOperand zprext> 7087: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7088 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7089 "", 7090 []>, Sched<[]> { 7091 bits<3> Pg; 7092 bits<5> Rn; 7093 bits<5> Zm; 7094 bits<5> Zt; 7095 let Inst{31-25} = 0b1000010; 7096 let Inst{24-23} = opc{3-2}; 7097 let Inst{22} = xs; 7098 let Inst{21} = scaled; 7099 let Inst{20-16} = Zm; 7100 let Inst{15} = 0b0; 7101 let Inst{14-13} = opc{1-0}; 7102 let Inst{12-10} = Pg; 7103 let Inst{9-5} = Rn; 7104 let Inst{4-0} = Zt; 7105 7106 let mayLoad = 1; 7107 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7108 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7109} 7110 7111multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm, 7112 SDPatternOperator sxtw_op, 7113 SDPatternOperator uxtw_op, 7114 RegisterOperand sxtw_opnd, 7115 RegisterOperand uxtw_opnd, 7116 ValueType vt> { 7117 def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>; 7118 def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>; 7119 7120 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7121 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7122 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7123 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7124 7125 // We need a layer of indirection because early machine code passes balk at 7126 // physical register (i.e. FFR) uses that have no previous definition. 7127 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7128 def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7129 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7130 def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7131 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7132 } 7133 7134 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 7135 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7136 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 7137 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7138} 7139 7140multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm, 7141 SDPatternOperator sxtw_op, 7142 SDPatternOperator uxtw_op, 7143 RegisterOperand sxtw_opnd, 7144 RegisterOperand uxtw_opnd, 7145 ValueType vt> { 7146 def _UXTW_REAL : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>; 7147 def _SXTW_REAL : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>; 7148 7149 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7150 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7151 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7152 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7153 7154 // We need a layer of indirection because early machine code passes balk at 7155 // physical register (i.e. FFR) uses that have no previous definition. 7156 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7157 def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7158 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7159 def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7160 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7161 } 7162 7163 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7164 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7165 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7166 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7167} 7168 7169 7170class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7171: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7172 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7173 "", 7174 []>, Sched<[]> { 7175 bits<3> Pg; 7176 bits<5> Zn; 7177 bits<5> Zt; 7178 bits<5> imm5; 7179 let Inst{31-25} = 0b1000010; 7180 let Inst{24-23} = opc{3-2}; 7181 let Inst{22-21} = 0b01; 7182 let Inst{20-16} = imm5; 7183 let Inst{15} = 0b1; 7184 let Inst{14-13} = opc{1-0}; 7185 let Inst{12-10} = Pg; 7186 let Inst{9-5} = Zn; 7187 let Inst{4-0} = Zt; 7188 7189 let mayLoad = 1; 7190 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7191 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7192} 7193 7194multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty, 7195 SDPatternOperator op, ValueType vt> { 7196 def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>; 7197 7198 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7199 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 7200 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7201 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 7202 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7203 (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7204 7205 // We need a layer of indirection because early machine code passes balk at 7206 // physical register (i.e. FFR) uses that have no previous definition. 7207 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7208 def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>, 7209 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>; 7210 } 7211 7212 def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), 7213 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7214} 7215 7216class sve_mem_prfm_si<bits<2> msz, string asm> 7217: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6), 7218 asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]", 7219 "", 7220 []>, Sched<[]> { 7221 bits<5> Rn; 7222 bits<3> Pg; 7223 bits<6> imm6; 7224 bits<4> prfop; 7225 let Inst{31-22} = 0b1000010111; 7226 let Inst{21-16} = imm6; 7227 let Inst{15} = 0b0; 7228 let Inst{14-13} = msz; 7229 let Inst{12-10} = Pg; 7230 let Inst{9-5} = Rn; 7231 let Inst{4} = 0b0; 7232 let Inst{3-0} = prfop; 7233 7234 let hasSideEffects = 1; 7235} 7236 7237multiclass sve_mem_prfm_si<bits<2> msz, string asm> { 7238 def NAME : sve_mem_prfm_si<msz, asm>; 7239 7240 def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]", 7241 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7242} 7243 7244class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty> 7245: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7246 asm, "\t$prfop, $Pg, [$Rn, $Rm]", 7247 "", 7248 []>, Sched<[]> { 7249 bits<5> Rm; 7250 bits<5> Rn; 7251 bits<3> Pg; 7252 bits<4> prfop; 7253 let Inst{31-25} = 0b1000010; 7254 let Inst{24-23} = opc{2-1}; 7255 let Inst{22-21} = 0b00; 7256 let Inst{20-16} = Rm; 7257 let Inst{15} = 0b1; 7258 let Inst{14} = opc{0}; 7259 let Inst{13} = 0b0; 7260 let Inst{12-10} = Pg; 7261 let Inst{9-5} = Rn; 7262 let Inst{4} = 0b0; 7263 let Inst{3-0} = prfop; 7264 7265 let hasSideEffects = 1; 7266} 7267 7268class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm, 7269 RegisterOperand zprext> 7270: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7271 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7272 "", 7273 []>, Sched<[]> { 7274 bits<3> Pg; 7275 bits<5> Rn; 7276 bits<5> Zm; 7277 bits<4> prfop; 7278 let Inst{31-23} = 0b100001000; 7279 let Inst{22} = xs; 7280 let Inst{21} = 0b1; 7281 let Inst{20-16} = Zm; 7282 let Inst{15} = 0b0; 7283 let Inst{14-13} = msz; 7284 let Inst{12-10} = Pg; 7285 let Inst{9-5} = Rn; 7286 let Inst{4} = 0b0; 7287 let Inst{3-0} = prfop; 7288 7289 let hasSideEffects = 1; 7290} 7291 7292multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm, 7293 RegisterOperand sxtw_opnd, 7294 RegisterOperand uxtw_opnd, 7295 SDPatternOperator op_sxtw, 7296 SDPatternOperator op_uxtw> { 7297 def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>; 7298 def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>; 7299 7300 def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7301 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7302 7303 def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7304 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7305} 7306 7307class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7308: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7309 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7310 "", 7311 []>, Sched<[]> { 7312 bits<3> Pg; 7313 bits<5> Zn; 7314 bits<5> imm5; 7315 bits<4> prfop; 7316 let Inst{31-25} = 0b1000010; 7317 let Inst{24-23} = msz; 7318 let Inst{22-21} = 0b00; 7319 let Inst{20-16} = imm5; 7320 let Inst{15-13} = 0b111; 7321 let Inst{12-10} = Pg; 7322 let Inst{9-5} = Zn; 7323 let Inst{4} = 0b0; 7324 let Inst{3-0} = prfop; 7325} 7326 7327multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7328 def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>; 7329 7330 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7331 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7332 7333 def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7334 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7335} 7336 7337class sve_mem_z_fill<string asm> 7338: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9), 7339 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 7340 "", 7341 []>, Sched<[]> { 7342 bits<5> Rn; 7343 bits<5> Zt; 7344 bits<9> imm9; 7345 let Inst{31-22} = 0b1000010110; 7346 let Inst{21-16} = imm9{8-3}; 7347 let Inst{15-13} = 0b010; 7348 let Inst{12-10} = imm9{2-0}; 7349 let Inst{9-5} = Rn; 7350 let Inst{4-0} = Zt; 7351 7352 let mayLoad = 1; 7353} 7354 7355multiclass sve_mem_z_fill<string asm> { 7356 def NAME : sve_mem_z_fill<asm>; 7357 7358 def : InstAlias<asm # "\t$Zt, [$Rn]", 7359 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 7360} 7361 7362class sve_mem_p_fill<string asm> 7363: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9), 7364 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 7365 "", 7366 []>, Sched<[]> { 7367 bits<4> Pt; 7368 bits<5> Rn; 7369 bits<9> imm9; 7370 let Inst{31-22} = 0b1000010110; 7371 let Inst{21-16} = imm9{8-3}; 7372 let Inst{15-13} = 0b000; 7373 let Inst{12-10} = imm9{2-0}; 7374 let Inst{9-5} = Rn; 7375 let Inst{4} = 0b0; 7376 let Inst{3-0} = Pt; 7377 7378 let mayLoad = 1; 7379} 7380 7381multiclass sve_mem_p_fill<string asm> { 7382 def NAME : sve_mem_p_fill<asm>; 7383 7384 def : InstAlias<asm # "\t$Pt, [$Rn]", 7385 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 7386} 7387 7388class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm, 7389 RegisterOperand VecList> 7390: I<(outs VecList:$Zt), iops, 7391 asm, "\t$Zt, $Pg/z, [$Zn, $Rm]", 7392 "", 7393 []>, Sched<[]> { 7394 bits<3> Pg; 7395 bits<5> Rm; 7396 bits<5> Zn; 7397 bits<5> Zt; 7398 let Inst{31} = 0b1; 7399 let Inst{30} = opc{4}; 7400 let Inst{29-25} = 0b00010; 7401 let Inst{24-23} = opc{3-2}; 7402 let Inst{22-21} = 0b00; 7403 let Inst{20-16} = Rm; 7404 let Inst{15} = 0b1; 7405 let Inst{14-13} = opc{1-0}; 7406 let Inst{12-10} = Pg; 7407 let Inst{9-5} = Zn; 7408 let Inst{4-0} = Zt; 7409 7410 let mayLoad = 1; 7411} 7412 7413multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm, 7414 SDPatternOperator op, 7415 ValueType vt> { 7416 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 7417 asm, Z_s>; 7418 7419 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7420 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 7421 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7422 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 7423 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7424 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 7425 7426 def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)), 7427 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; 7428} 7429 7430multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm, 7431 SDPatternOperator op, 7432 ValueType vt> { 7433 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 7434 asm, Z_d>; 7435 7436 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7437 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 7438 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7439 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 7440 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7441 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 7442 7443 def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)), 7444 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; 7445} 7446 7447//===----------------------------------------------------------------------===// 7448// SVE Memory - 64-bit Gather Group 7449//===----------------------------------------------------------------------===// 7450 7451// bit xs is '1' if offsets are signed 7452// bit scaled is '1' if the offsets are scaled 7453// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7454class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm, 7455 RegisterOperand zprext> 7456: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7457 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7458 "", 7459 []>, Sched<[]> { 7460 bits<3> Pg; 7461 bits<5> Rn; 7462 bits<5> Zm; 7463 bits<5> Zt; 7464 let Inst{31-25} = 0b1100010; 7465 let Inst{24-23} = opc{3-2}; 7466 let Inst{22} = xs; 7467 let Inst{21} = scaled; 7468 let Inst{20-16} = Zm; 7469 let Inst{15} = lsl; 7470 let Inst{14-13} = opc{1-0}; 7471 let Inst{12-10} = Pg; 7472 let Inst{9-5} = Rn; 7473 let Inst{4-0} = Zt; 7474 7475 let mayLoad = 1; 7476 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7477 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7478} 7479 7480multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm, 7481 SDPatternOperator sxtw_op, 7482 SDPatternOperator uxtw_op, 7483 RegisterOperand sxtw_opnd, 7484 RegisterOperand uxtw_opnd, 7485 ValueType vt> { 7486 def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>; 7487 def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>; 7488 7489 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7490 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7491 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7492 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7493 7494 // We need a layer of indirection because early machine code passes balk at 7495 // physical register (i.e. FFR) uses that have no previous definition. 7496 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7497 def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7498 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7499 def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7500 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7501 } 7502 7503 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7504 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7505 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7506 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7507} 7508 7509multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm, 7510 SDPatternOperator sxtw_op, 7511 SDPatternOperator uxtw_op, 7512 RegisterOperand sxtw_opnd, 7513 RegisterOperand uxtw_opnd, 7514 ValueType vt> { 7515 def _UXTW_REAL : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>; 7516 def _SXTW_REAL : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>; 7517 7518 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7519 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7520 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7521 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7522 7523 // We need a layer of indirection because early machine code passes balk at 7524 // physical register (i.e. FFR) uses that have no previous definition. 7525 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7526 def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7527 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7528 def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7529 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7530 } 7531 7532 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7533 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7534 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7535 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7536} 7537 7538multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm, 7539 SDPatternOperator op, 7540 RegisterOperand zprext, ValueType vt> { 7541 def _SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>; 7542 7543 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7544 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 7545 7546 // We need a layer of indirection because early machine code passes balk at 7547 // physical register (i.e. FFR) uses that have no previous definition. 7548 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7549 def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>, 7550 PseudoInstExpansion<(!cast<Instruction>(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7551 } 7552 7553 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7554 (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7555} 7556 7557multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm, 7558 SDPatternOperator op, ValueType vt> { 7559 def _REAL : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>; 7560 7561 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7562 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 7563 7564 // We need a layer of indirection because early machine code passes balk at 7565 // physical register (i.e. FFR) uses that have no previous definition. 7566 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7567 def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>, 7568 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>; 7569 } 7570 7571 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7572 (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7573} 7574 7575class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7576: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7577 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7578 "", 7579 []>, Sched<[]> { 7580 bits<3> Pg; 7581 bits<5> Zn; 7582 bits<5> Zt; 7583 bits<5> imm5; 7584 let Inst{31-25} = 0b1100010; 7585 let Inst{24-23} = opc{3-2}; 7586 let Inst{22-21} = 0b01; 7587 let Inst{20-16} = imm5; 7588 let Inst{15} = 0b1; 7589 let Inst{14-13} = opc{1-0}; 7590 let Inst{12-10} = Pg; 7591 let Inst{9-5} = Zn; 7592 let Inst{4-0} = Zt; 7593 7594 let mayLoad = 1; 7595 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7596 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7597} 7598 7599multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty, 7600 SDPatternOperator op, ValueType vt> { 7601 def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>; 7602 7603 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7604 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 7605 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7606 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 7607 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7608 (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7609 7610 // We need a layer of indirection because early machine code passes balk at 7611 // physical register (i.e. FFR) uses that have no previous definition. 7612 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7613 def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>, 7614 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>; 7615 } 7616 7617 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), 7618 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7619} 7620 7621// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7622class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm, 7623 RegisterOperand zprext> 7624: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7625 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7626 "", 7627 []>, Sched<[]> { 7628 bits<3> Pg; 7629 bits<5> Rn; 7630 bits<5> Zm; 7631 bits<4> prfop; 7632 let Inst{31-23} = 0b110001000; 7633 let Inst{22} = xs; 7634 let Inst{21} = 0b1; 7635 let Inst{20-16} = Zm; 7636 let Inst{15} = lsl; 7637 let Inst{14-13} = msz; 7638 let Inst{12-10} = Pg; 7639 let Inst{9-5} = Rn; 7640 let Inst{4} = 0b0; 7641 let Inst{3-0} = prfop; 7642 7643 let hasSideEffects = 1; 7644} 7645 7646multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm, 7647 RegisterOperand sxtw_opnd, 7648 RegisterOperand uxtw_opnd, 7649 SDPatternOperator op_sxtw, 7650 SDPatternOperator op_uxtw> { 7651 def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>; 7652 def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>; 7653 7654 def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7655 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7656 7657 def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7658 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7659 7660} 7661 7662multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm, 7663 RegisterOperand zprext, SDPatternOperator frag> { 7664 def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>; 7665 7666 def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)), 7667 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7668 7669} 7670 7671class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7672: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7673 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7674 "", 7675 []>, Sched<[]> { 7676 bits<3> Pg; 7677 bits<5> Zn; 7678 bits<5> imm5; 7679 bits<4> prfop; 7680 let Inst{31-25} = 0b1100010; 7681 let Inst{24-23} = msz; 7682 let Inst{22-21} = 0b00; 7683 let Inst{20-16} = imm5; 7684 let Inst{15-13} = 0b111; 7685 let Inst{12-10} = Pg; 7686 let Inst{9-5} = Zn; 7687 let Inst{4} = 0b0; 7688 let Inst{3-0} = prfop; 7689 7690 let hasSideEffects = 1; 7691} 7692 7693multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7694 def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>; 7695 7696 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7697 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7698 7699 def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7700 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7701} 7702 7703//===----------------------------------------------------------------------===// 7704// SVE Compute Vector Address Group 7705//===----------------------------------------------------------------------===// 7706 7707class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm, 7708 ZPRRegOp zprty, RegisterOperand zprext> 7709: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm), 7710 asm, "\t$Zd, [$Zn, $Zm]", 7711 "", 7712 []>, Sched<[]> { 7713 bits<5> Zd; 7714 bits<5> Zn; 7715 bits<5> Zm; 7716 let Inst{31-24} = 0b00000100; 7717 let Inst{23-22} = opc; 7718 let Inst{21} = 0b1; 7719 let Inst{20-16} = Zm; 7720 let Inst{15-12} = 0b1010; 7721 let Inst{11-10} = msz; 7722 let Inst{9-5} = Zn; 7723 let Inst{4-0} = Zd; 7724} 7725 7726multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> { 7727 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>; 7728 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>; 7729 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>; 7730 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>; 7731} 7732 7733multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> { 7734 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>; 7735 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>; 7736 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>; 7737 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>; 7738} 7739 7740multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> { 7741 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>; 7742 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>; 7743 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>; 7744 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>; 7745} 7746 7747multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> { 7748 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>; 7749 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>; 7750 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>; 7751 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>; 7752} 7753 7754//===----------------------------------------------------------------------===// 7755// SVE Integer Misc - Unpredicated Group 7756//===----------------------------------------------------------------------===// 7757 7758class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty> 7759: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7760 asm, "\t$Zd, $Zn, $Zm", 7761 "", 7762 []>, Sched<[]> { 7763 bits<5> Zd; 7764 bits<5> Zm; 7765 bits<5> Zn; 7766 let Inst{31-24} = 0b00000100; 7767 let Inst{23-22} = sz; 7768 let Inst{21} = 0b1; 7769 let Inst{20-16} = Zm; 7770 let Inst{15-10} = 0b101100; 7771 let Inst{9-5} = Zn; 7772 let Inst{4-0} = Zd; 7773} 7774 7775multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> { 7776 def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>; 7777 def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>; 7778 def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>; 7779 7780 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7781 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7782 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7783} 7784 7785class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty> 7786: I<(outs zprty:$Zd), (ins zprty:$Zn), 7787 asm, "\t$Zd, $Zn", 7788 "", 7789 []>, Sched<[]> { 7790 bits<5> Zd; 7791 bits<5> Zn; 7792 let Inst{31-24} = 0b00000100; 7793 let Inst{23-22} = opc{7-6}; 7794 let Inst{21} = 0b1; 7795 let Inst{20-16} = opc{5-1}; 7796 let Inst{15-11} = 0b10111; 7797 let Inst{10} = opc{0}; 7798 let Inst{9-5} = Zn; 7799 let Inst{4-0} = Zd; 7800} 7801 7802multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> { 7803 def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>; 7804 def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>; 7805 def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>; 7806 7807 def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 7808 def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 7809 def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 7810} 7811 7812//===----------------------------------------------------------------------===// 7813// SVE Integer Reduction Group 7814//===----------------------------------------------------------------------===// 7815 7816class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm, 7817 ZPRRegOp zprty, FPRasZPROperand dstOpType> 7818: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 7819 asm, "\t$Vd, $Pg, $Zn", 7820 "", 7821 []>, Sched<[]> { 7822 bits<3> Pg; 7823 bits<5> Vd; 7824 bits<5> Zn; 7825 let Inst{31-24} = 0b00000100; 7826 let Inst{23-22} = sz8_32; 7827 let Inst{21} = 0b0; 7828 let Inst{20-19} = fmt; 7829 let Inst{18-16} = opc; 7830 let Inst{15-13} = 0b001; 7831 let Inst{12-10} = Pg; 7832 let Inst{9-5} = Zn; 7833 let Inst{4-0} = Vd; 7834} 7835 7836multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm, 7837 SDPatternOperator op> { 7838 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7839 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7840 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7841 7842 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7843 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7844 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7845} 7846 7847multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm, 7848 SDPatternOperator op> { 7849 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7850 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7851 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7852 def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64asZPR>; 7853 7854 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7855 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7856 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7857 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7858} 7859 7860multiclass sve_int_reduce_1<bits<3> opc, string asm, 7861 SDPatternOperator op> { 7862 def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8asZPR>; 7863 def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16asZPR>; 7864 def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32asZPR>; 7865 def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64asZPR>; 7866 7867 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7868 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7869 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7870 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7871} 7872 7873multiclass sve_int_reduce_2<bits<3> opc, string asm, 7874 SDPatternOperator op> { 7875 def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8asZPR>; 7876 def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16asZPR>; 7877 def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32asZPR>; 7878 def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64asZPR>; 7879 7880 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7881 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7882 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7883 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7884} 7885 7886class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm, 7887 ZPRRegOp zprty, string pg_suffix, dag iops> 7888: I<(outs zprty:$Zd), iops, 7889 asm, "\t$Zd, $Pg"#pg_suffix#", $Zn", 7890 "", 7891 []>, Sched<[]> { 7892 bits<3> Pg; 7893 bits<5> Zd; 7894 bits<5> Zn; 7895 let Inst{31-24} = 0b00000100; 7896 let Inst{23-22} = sz8_32; 7897 let Inst{21-19} = 0b010; 7898 let Inst{18-16} = opc; 7899 let Inst{15-13} = 0b001; 7900 let Inst{12-10} = Pg; 7901 let Inst{9-5} = Zn; 7902 let Inst{4-0} = Zd; 7903 7904 let ElementSize = zprty.ElementSize; 7905} 7906 7907multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> { 7908let Constraints = "$Zd = $_Zd" in { 7909 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m", 7910 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>; 7911 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m", 7912 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>; 7913 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m", 7914 (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>; 7915 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m", 7916 (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>; 7917} 7918} 7919 7920multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> { 7921 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z", 7922 (ins PPR3bAny:$Pg, ZPR8:$Zn)>; 7923 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z", 7924 (ins PPR3bAny:$Pg, ZPR16:$Zn)>; 7925 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z", 7926 (ins PPR3bAny:$Pg, ZPR32:$Zn)>; 7927 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z", 7928 (ins PPR3bAny:$Pg, ZPR64:$Zn)>; 7929} 7930 7931//===----------------------------------------------------------------------===// 7932// SVE Propagate Break Group 7933//===----------------------------------------------------------------------===// 7934 7935class sve_int_brkp<bits<2> opc, string asm> 7936: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 7937 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 7938 "", 7939 []>, Sched<[]> { 7940 bits<4> Pd; 7941 bits<4> Pg; 7942 bits<4> Pm; 7943 bits<4> Pn; 7944 let Inst{31-24} = 0b00100101; 7945 let Inst{23} = 0b0; 7946 let Inst{22} = opc{1}; 7947 let Inst{21-20} = 0b00; 7948 let Inst{19-16} = Pm; 7949 let Inst{15-14} = 0b11; 7950 let Inst{13-10} = Pg; 7951 let Inst{9} = 0b0; 7952 let Inst{8-5} = Pn; 7953 let Inst{4} = opc{0}; 7954 let Inst{3-0} = Pd; 7955 7956 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7957} 7958 7959multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> { 7960 def NAME : sve_int_brkp<opc, asm>; 7961 7962 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7963} 7964 7965 7966//===----------------------------------------------------------------------===// 7967// SVE Partition Break Group 7968//===----------------------------------------------------------------------===// 7969 7970class sve_int_brkn<bit S, string asm> 7971: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm), 7972 asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm", 7973 "", 7974 []>, Sched<[]> { 7975 bits<4> Pdm; 7976 bits<4> Pg; 7977 bits<4> Pn; 7978 let Inst{31-23} = 0b001001010; 7979 let Inst{22} = S; 7980 let Inst{21-14} = 0b01100001; 7981 let Inst{13-10} = Pg; 7982 let Inst{9} = 0b0; 7983 let Inst{8-5} = Pn; 7984 let Inst{4} = 0b0; 7985 let Inst{3-0} = Pdm; 7986 7987 let Constraints = "$Pdm = $_Pdm"; 7988 let Defs = !if(S, [NZCV], []); 7989} 7990 7991multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> { 7992 def NAME : sve_int_brkn<opc, asm>; 7993 7994 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7995} 7996 7997class sve_int_break<bits<3> opc, string asm, string suffix, dag iops> 7998: I<(outs PPR8:$Pd), iops, 7999 asm, "\t$Pd, $Pg"#suffix#", $Pn", 8000 "", 8001 []>, Sched<[]> { 8002 bits<4> Pd; 8003 bits<4> Pg; 8004 bits<4> Pn; 8005 let Inst{31-24} = 0b00100101; 8006 let Inst{23-22} = opc{2-1}; 8007 let Inst{21-14} = 0b01000001; 8008 let Inst{13-10} = Pg; 8009 let Inst{9} = 0b0; 8010 let Inst{8-5} = Pn; 8011 let Inst{4} = opc{0}; 8012 let Inst{3-0} = Pd; 8013 8014 let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", ""); 8015 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 8016 8017} 8018 8019multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> { 8020 def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>; 8021 8022 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8023} 8024 8025multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> { 8026 def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>; 8027 8028 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8029} 8030 8031//===----------------------------------------------------------------------===// 8032// SVE2 String Processing Group 8033//===----------------------------------------------------------------------===// 8034 8035class sve2_char_match<bit sz, bit opc, string asm, 8036 PPRRegOp pprty, ZPRRegOp zprty> 8037: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 8038 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 8039 "", 8040 []>, Sched<[]> { 8041 bits<4> Pd; 8042 bits<3> Pg; 8043 bits<5> Zm; 8044 bits<5> Zn; 8045 let Inst{31-23} = 0b010001010; 8046 let Inst{22} = sz; 8047 let Inst{21} = 0b1; 8048 let Inst{20-16} = Zm; 8049 let Inst{15-13} = 0b100; 8050 let Inst{12-10} = Pg; 8051 let Inst{9-5} = Zn; 8052 let Inst{4} = opc; 8053 let Inst{3-0} = Pd; 8054 8055 let Defs = [NZCV]; 8056 let isPTestLike = 1; 8057} 8058 8059multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> { 8060 def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>; 8061 def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>; 8062 8063 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 8064 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 8065} 8066 8067//===----------------------------------------------------------------------===// 8068// SVE2 Histogram Computation - Segment Group 8069//===----------------------------------------------------------------------===// 8070 8071class sve2_hist_gen_segment<string asm, SDPatternOperator op> 8072: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm), 8073 asm, "\t$Zd, $Zn, $Zm", 8074 "", 8075 [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> { 8076 bits<5> Zd; 8077 bits<5> Zn; 8078 bits<5> Zm; 8079 let Inst{31-21} = 0b01000101001; 8080 let Inst{20-16} = Zm; 8081 let Inst{15-10} = 0b101000; 8082 let Inst{9-5} = Zn; 8083 let Inst{4-0} = Zd; 8084} 8085 8086//===----------------------------------------------------------------------===// 8087// SVE2 Histogram Computation - Vector Group 8088//===----------------------------------------------------------------------===// 8089 8090class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty> 8091: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 8092 asm, "\t$Zd, $Pg/z, $Zn, $Zm", 8093 "", 8094 []>, Sched<[]> { 8095 bits<5> Zd; 8096 bits<5> Zn; 8097 bits<3> Pg; 8098 bits<5> Zm; 8099 let Inst{31-23} = 0b010001011; 8100 let Inst{22} = sz; 8101 let Inst{21} = 0b1; 8102 let Inst{20-16} = Zm; 8103 let Inst{15-13} = 0b110; 8104 let Inst{12-10} = Pg; 8105 let Inst{9-5} = Zn; 8106 let Inst{4-0} = Zd; 8107} 8108 8109multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> { 8110 def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>; 8111 def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>; 8112 8113 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 8114 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 8115} 8116 8117//===----------------------------------------------------------------------===// 8118// SVE2 Crypto Extensions Group 8119//===----------------------------------------------------------------------===// 8120 8121class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty> 8122: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 8123 asm, "\t$Zd, $Zn, $Zm", 8124 "", 8125 []>, Sched<[]> { 8126 bits<5> Zd; 8127 bits<5> Zn; 8128 bits<5> Zm; 8129 let Inst{31-21} = 0b01000101001; 8130 let Inst{20-16} = Zm; 8131 let Inst{15-11} = 0b11110; 8132 let Inst{10} = opc; 8133 let Inst{9-5} = Zn; 8134 let Inst{4-0} = Zd; 8135} 8136 8137multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty, 8138 SDPatternOperator op, ValueType vt> { 8139 def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>; 8140 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 8141} 8142 8143class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty> 8144: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm), 8145 asm, "\t$Zdn, $_Zdn, $Zm", 8146 "", 8147 []>, Sched<[]> { 8148 bits<5> Zdn; 8149 bits<5> Zm; 8150 let Inst{31-17} = 0b010001010010001; 8151 let Inst{16} = opc{1}; 8152 let Inst{15-11} = 0b11100; 8153 let Inst{10} = opc{0}; 8154 let Inst{9-5} = Zm; 8155 let Inst{4-0} = Zdn; 8156 8157 let Constraints = "$Zdn = $_Zdn"; 8158} 8159 8160multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty, 8161 SDPatternOperator op, ValueType vt> { 8162 def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>; 8163 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 8164} 8165 8166class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty> 8167: I<(outs zprty:$Zdn), (ins zprty:$_Zdn), 8168 asm, "\t$Zdn, $_Zdn", 8169 "", 8170 []>, Sched<[]> { 8171 bits<5> Zdn; 8172 let Inst{31-11} = 0b010001010010000011100; 8173 let Inst{10} = opc; 8174 let Inst{9-5} = 0b00000; 8175 let Inst{4-0} = Zdn; 8176 8177 let Constraints = "$Zdn = $_Zdn"; 8178} 8179 8180multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> { 8181 def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>; 8182 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>; 8183} 8184 8185//===----------------------------------------------------------------------===// 8186// SVE BFloat16 Group 8187//===----------------------------------------------------------------------===// 8188 8189class sve_bfloat_dot_base<bits<2> opc, string asm, string ops, dag iops> 8190: I<(outs ZPR32:$Zda), iops, asm, ops, "", []>, Sched<[]> { 8191 bits<5> Zda; 8192 bits<5> Zn; 8193 let Inst{31-21} = 0b01100100011; 8194 let Inst{15-14} = opc; 8195 let Inst{13-10} = 0b0000; 8196 let Inst{9-5} = Zn; 8197 let Inst{4-0} = Zda; 8198 8199 let Constraints = "$Zda = $_Zda"; 8200 let DestructiveInstType = DestructiveOther; 8201 let ElementSize = ElementSizeH; 8202} 8203 8204class sve_bfloat_dot<string asm> 8205: sve_bfloat_dot_base<0b10, asm, "\t$Zda, $Zn, $Zm", 8206 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm)> { 8207 bits<5> Zm; 8208 let Inst{20-16} = Zm; 8209} 8210 8211multiclass sve_bfloat_dot<string asm, SDPatternOperator op> { 8212 def NAME : sve_bfloat_dot<asm>; 8213 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8214} 8215 8216class sve_bfloat_dot_indexed<string asm> 8217: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 8218 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> { 8219 bits<2> iop; 8220 bits<3> Zm; 8221 let Inst{20-19} = iop; 8222 let Inst{18-16} = Zm; 8223} 8224 8225multiclass sve_bfloat_dot_indexed<string asm, SDPatternOperator op> { 8226 def NAME : sve_bfloat_dot_indexed<asm>; 8227 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexS_timm, !cast<Instruction>(NAME)>; 8228} 8229 8230class sve_bfloat_matmul<string asm> 8231: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 8232 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8233 bits<5> Zm; 8234 bits<5> Zda; 8235 bits<5> Zn; 8236 let Inst{31-21} = 0b01100100011; 8237 let Inst{20-16} = Zm; 8238 let Inst{15-10} = 0b111001; 8239 let Inst{9-5} = Zn; 8240 let Inst{4-0} = Zda; 8241 8242 let Constraints = "$Zda = $_Zda"; 8243 let DestructiveInstType = DestructiveOther; 8244 let ElementSize = ElementSizeH; 8245} 8246 8247multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> { 8248 def NAME : sve_bfloat_matmul<asm>; 8249 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8250} 8251 8252class sve_bfloat_matmul_longvecl<bit BT, string asm> 8253: sve_bfloat_matmul<asm> { 8254 let Inst{23} = 0b1; 8255 let Inst{14-13} = 0b00; 8256 let Inst{10} = BT; 8257} 8258 8259multiclass sve_bfloat_matmul_longvecl<bit BT, string asm, SDPatternOperator op> { 8260 def NAME : sve_bfloat_matmul_longvecl<BT, asm>; 8261 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8262} 8263 8264class sve_bfloat_matmul_longvecl_idx<bit BT, string asm> 8265: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 8266 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> { 8267 bits<3> iop; 8268 bits<3> Zm; 8269 let Inst{23} = 0b1; 8270 let Inst{20-19} = iop{2-1}; 8271 let Inst{18-16} = Zm; 8272 let Inst{11} = iop{0}; 8273 let Inst{10} = BT; 8274} 8275 8276multiclass sve_bfloat_matmul_longvecl_idx<bit BT, string asm, SDPatternOperator op> { 8277 def NAME : sve_bfloat_matmul_longvecl_idx<BT, asm>; 8278 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexH_timm, !cast<Instruction>(NAME)>; 8279} 8280 8281class sve_bfloat_convert<bit N, string asm> 8282: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), 8283 asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { 8284 bits<5> Zd; 8285 bits<3> Pg; 8286 bits<5> Zn; 8287 let Inst{31-25} = 0b0110010; 8288 let Inst{24} = N; 8289 let Inst{23-13} = 0b10001010101; 8290 let Inst{12-10} = Pg; 8291 let Inst{9-5} = Zn; 8292 let Inst{4-0} = Zd; 8293 8294 let Constraints = "$Zd = $_Zd"; 8295 let DestructiveInstType = DestructiveOther; 8296 let hasSideEffects = 1; 8297 let ElementSize = ElementSizeS; 8298} 8299 8300multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> { 8301 def NAME : sve_bfloat_convert<N, asm>; 8302 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>; 8303} 8304 8305//===----------------------------------------------------------------------===// 8306// SVE Integer Matrix Multiply Group 8307//===----------------------------------------------------------------------===// 8308 8309class sve_int_matmul<bits<2> uns, string asm> 8310: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8311 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8312 bits<5> Zda; 8313 bits<5> Zn; 8314 bits<5> Zm; 8315 let Inst{31-24} = 0b01000101; 8316 let Inst{23-22} = uns; 8317 let Inst{21} = 0; 8318 let Inst{20-16} = Zm; 8319 let Inst{15-10} = 0b100110; 8320 let Inst{9-5} = Zn; 8321 let Inst{4-0} = Zda; 8322 8323 let Constraints = "$Zda = $_Zda"; 8324 let DestructiveInstType = DestructiveOther; 8325 let ElementSize = ZPR32.ElementSize; 8326} 8327 8328multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> { 8329 def NAME : sve_int_matmul<uns, asm>; 8330 8331 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8332} 8333 8334//===----------------------------------------------------------------------===// 8335// SVE Integer Dot Product Mixed Sign Group 8336//===----------------------------------------------------------------------===// 8337 8338class sve_int_dot_mixed<string asm> 8339: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8340 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8341 bits<5> Zda; 8342 bits<5> Zn; 8343 bits<5> Zm; 8344 let Inst{31-21} = 0b01000100100; 8345 let Inst{20-16} = Zm; 8346 let Inst{15-10} = 0b011110; 8347 let Inst{9-5} = Zn; 8348 let Inst{4-0} = Zda; 8349 8350 let Constraints = "$Zda = $_Zda"; 8351 let DestructiveInstType = DestructiveOther; 8352 let ElementSize = ZPR32.ElementSize; 8353} 8354 8355multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> { 8356 def NAME : sve_int_dot_mixed<asm>; 8357 8358 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8359} 8360 8361//===----------------------------------------------------------------------===// 8362// SVE Integer Dot Product Mixed Sign - Indexed Group 8363//===----------------------------------------------------------------------===// 8364 8365class sve_int_dot_mixed_indexed<bit U, string asm> 8366: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx), 8367 asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> { 8368 bits<5> Zda; 8369 bits<5> Zn; 8370 bits<3> Zm; 8371 bits<2> idx; 8372 let Inst{31-21} = 0b01000100101; 8373 let Inst{20-19} = idx; 8374 let Inst{18-16} = Zm; 8375 let Inst{15-11} = 0b00011; 8376 let Inst{10} = U; 8377 let Inst{9-5} = Zn; 8378 let Inst{4-0} = Zda; 8379 8380 let Constraints = "$Zda = $_Zda"; 8381 let DestructiveInstType = DestructiveOther; 8382 let ElementSize = ZPR32.ElementSize; 8383} 8384 8385multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> { 8386 def NAME : sve_int_dot_mixed_indexed<U, asm>; 8387 8388 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>; 8389} 8390 8391//===----------------------------------------------------------------------===// 8392// SVE Floating Point Matrix Multiply Accumulate Group 8393//===----------------------------------------------------------------------===// 8394 8395class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty> 8396: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm), 8397 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8398 bits<5> Zda; 8399 bits<5> Zn; 8400 bits<5> Zm; 8401 let Inst{31-23} = 0b011001001; 8402 let Inst{22} = sz; 8403 let Inst{21} = 1; 8404 let Inst{20-16} = Zm; 8405 let Inst{15-10} = 0b111001; 8406 let Inst{9-5} = Zn; 8407 let Inst{4-0} = Zda; 8408 8409 let Constraints = "$Zda = $_Zda"; 8410 let DestructiveInstType = DestructiveOther; 8411 let ElementSize = zprty.ElementSize; 8412} 8413 8414multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> { 8415 def NAME : sve_fp_matrix_mla<sz, asm, zprty>; 8416 8417 def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>; 8418} 8419 8420//===----------------------------------------------------------------------===// 8421// SVE Memory - Contiguous Load And Replicate 256-bit Group 8422//===----------------------------------------------------------------------===// 8423 8424class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList> 8425: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 8426 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 8427 bits<5> Zt; 8428 bits<5> Rn; 8429 bits<3> Pg; 8430 bits<4> imm4; 8431 let Inst{31-25} = 0b1010010; 8432 let Inst{24-23} = sz; 8433 let Inst{22-20} = 0b010; 8434 let Inst{19-16} = imm4; 8435 let Inst{15-13} = 0b001; 8436 let Inst{12-10} = Pg; 8437 let Inst{9-5} = Rn; 8438 let Inst{4-0} = Zt; 8439 8440 let mayLoad = 1; 8441} 8442 8443multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty, 8444 ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> { 8445 def NAME : sve_mem_ldor_si<sz, asm, listty>; 8446 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8447 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 8448 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8449 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 8450 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 8451 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>; 8452 8453 // Base addressing mode 8454 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)), 8455 (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>; 8456 let AddedComplexity = 2 in { 8457 // Reg + Imm addressing mode 8458 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))), 8459 (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>; 8460 } 8461} 8462 8463class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList, 8464 RegisterOperand gprty> 8465: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 8466 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 8467 bits<5> Zt; 8468 bits<3> Pg; 8469 bits<5> Rn; 8470 bits<5> Rm; 8471 let Inst{31-25} = 0b1010010; 8472 let Inst{24-23} = sz; 8473 let Inst{22-21} = 0b01; 8474 let Inst{20-16} = Rm; 8475 let Inst{15-13} = 0; 8476 let Inst{12-10} = Pg; 8477 let Inst{9-5} = Rn; 8478 let Inst{4-0} = Zt; 8479 8480 let mayLoad = 1; 8481} 8482 8483multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty, 8484 ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty, 8485 ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> { 8486 def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>; 8487 8488 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 8489 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 8490 8491 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))), 8492 (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>; 8493} 8494 8495//===----------------------------------------------------------------------===// 8496// SVE Interleave 128-bit Elements Group 8497//===----------------------------------------------------------------------===// 8498 8499class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm> 8500: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm), 8501 asm, "\t$Zd, $Zn, $Zm", 8502 "", 8503 []>, Sched<[]> { 8504 bits<5> Zd; 8505 bits<5> Zm; 8506 bits<5> Zn; 8507 let Inst{31-21} = 0b00000101101; 8508 let Inst{20-16} = Zm; 8509 let Inst{15-13} = 0b000; 8510 let Inst{12-11} = opc; 8511 let Inst{10} = P; 8512 let Inst{9-5} = Zn; 8513 let Inst{4-0} = Zd; 8514} 8515 8516multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> { 8517 def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>; 8518 8519 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8520 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 8521 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 8522 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 8523 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME)>; 8524 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 8525 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME)>; 8526 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>; 8527} 8528 8529/// Addressing modes 8530def am_sve_indexed_s4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; 8531def am_sve_indexed_s6 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; 8532 8533def am_sve_regreg_lsl0 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<0>", []>; 8534def am_sve_regreg_lsl1 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<1>", []>; 8535def am_sve_regreg_lsl2 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<2>", []>; 8536def am_sve_regreg_lsl3 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<3>", []>; 8537def am_sve_regreg_lsl4 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<4>", []>; 8538 8539// Predicated pseudo floating point two operand instructions. 8540multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> { 8541 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8542 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8543 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8544 8545 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8546 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8547 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8548 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8549 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8550 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8551} 8552 8553// Predicated pseudo integer two operand instructions. 8554multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> { 8555 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8556 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8557 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8558 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8559 8560 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8561 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8562 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8563 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8564} 8565 8566// As sve_int_bin_pred but when only i32 and i64 vector types are required. 8567multiclass sve_int_bin_pred_sd<SDPatternOperator op> { 8568 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8569 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8570 8571 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8572 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8573} 8574 8575// Predicated pseudo integer two operand instructions. Second operand is an 8576// immediate specified by imm_[bhsd]. 8577multiclass sve_int_shift_pred_bhsd<SDPatternOperator op, 8578 ComplexPattern imm_b, ComplexPattern imm_h, 8579 ComplexPattern imm_s, ComplexPattern imm_d> { 8580 def _UNDEF_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, Operand<i32>, FalseLanesUndef>; 8581 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesUndef>; 8582 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesUndef>; 8583 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesUndef>; 8584 8585 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _UNDEF_B)>; 8586 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Instruction>(NAME # _UNDEF_H)>; 8587 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Instruction>(NAME # _UNDEF_S)>; 8588 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Instruction>(NAME # _UNDEF_D)>; 8589} 8590 8591multiclass sve_int_bin_pred_all_active_bhsd<SDPatternOperator op> { 8592 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8593 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8594 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8595 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8596 8597 def : SVE_2_Op_Pred_All_Active_Pt<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8598 def : SVE_2_Op_Pred_All_Active_Pt<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8599 def : SVE_2_Op_Pred_All_Active_Pt<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8600 def : SVE_2_Op_Pred_All_Active_Pt<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8601} 8602 8603