1//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def SDT_AArch64Setcc : SDTypeProfile<1, 4, [ 14 SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, 15 SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>, 16 SDTCisVT<4, OtherVT> 17]>; 18 19def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>; 20 21def SVEPatternOperand : AsmOperandClass { 22 let Name = "SVEPattern"; 23 let ParserMethod = "tryParseSVEPattern"; 24 let PredicateMethod = "isSVEPattern"; 25 let RenderMethod = "addImmOperands"; 26 let DiagnosticType = "InvalidSVEPattern"; 27} 28 29def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{ 30 return (((uint32_t)Imm) < 32); 31 }]> { 32 33 let PrintMethod = "printSVEPattern"; 34 let ParserMatchClass = SVEPatternOperand; 35} 36 37def SVEPrefetchOperand : AsmOperandClass { 38 let Name = "SVEPrefetch"; 39 let ParserMethod = "tryParsePrefetch<true>"; 40 let PredicateMethod = "isPrefetch"; 41 let RenderMethod = "addPrefetchOperands"; 42} 43 44def sve_prfop : Operand<i32>, TImmLeaf<i32, [{ 45 return (((uint32_t)Imm) <= 15); 46 }]> { 47 let PrintMethod = "printPrefetchOp<true>"; 48 let ParserMatchClass = SVEPrefetchOperand; 49} 50 51class SVELogicalImmOperand<int Width> : AsmOperandClass { 52 let Name = "SVELogicalImm" # Width; 53 let DiagnosticType = "LogicalSecondSource"; 54 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 55 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 56} 57 58def sve_logical_imm8 : Operand<i64> { 59 let ParserMatchClass = SVELogicalImmOperand<8>; 60 let PrintMethod = "printLogicalImm<int8_t>"; 61 62 let MCOperandPredicate = [{ 63 if (!MCOp.isImm()) 64 return false; 65 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 66 return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val); 67 }]; 68} 69 70def sve_logical_imm16 : Operand<i64> { 71 let ParserMatchClass = SVELogicalImmOperand<16>; 72 let PrintMethod = "printLogicalImm<int16_t>"; 73 74 let MCOperandPredicate = [{ 75 if (!MCOp.isImm()) 76 return false; 77 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 78 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val); 79 }]; 80} 81 82def sve_logical_imm32 : Operand<i64> { 83 let ParserMatchClass = SVELogicalImmOperand<32>; 84 let PrintMethod = "printLogicalImm<int32_t>"; 85 86 let MCOperandPredicate = [{ 87 if (!MCOp.isImm()) 88 return false; 89 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 90 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val); 91 }]; 92} 93 94class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass { 95 let Name = "SVEPreferredLogicalImm" # Width; 96 let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>"; 97 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 98} 99 100def sve_preferred_logical_imm16 : Operand<i64> { 101 let ParserMatchClass = SVEPreferredLogicalImmOperand<16>; 102 let PrintMethod = "printSVELogicalImm<int16_t>"; 103 104 let MCOperandPredicate = [{ 105 if (!MCOp.isImm()) 106 return false; 107 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 108 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) && 109 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 110 }]; 111} 112 113def sve_preferred_logical_imm32 : Operand<i64> { 114 let ParserMatchClass = SVEPreferredLogicalImmOperand<32>; 115 let PrintMethod = "printSVELogicalImm<int32_t>"; 116 117 let MCOperandPredicate = [{ 118 if (!MCOp.isImm()) 119 return false; 120 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 121 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) && 122 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 123 }]; 124} 125 126def sve_preferred_logical_imm64 : Operand<i64> { 127 let ParserMatchClass = SVEPreferredLogicalImmOperand<64>; 128 let PrintMethod = "printSVELogicalImm<int64_t>"; 129 130 let MCOperandPredicate = [{ 131 if (!MCOp.isImm()) 132 return false; 133 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 134 return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) && 135 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 136 }]; 137} 138 139class SVELogicalImmNotOperand<int Width> : AsmOperandClass { 140 let Name = "SVELogicalImm" # Width # "Not"; 141 let DiagnosticType = "LogicalSecondSource"; 142 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 143 let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>"; 144} 145 146def sve_logical_imm8_not : Operand<i64> { 147 let ParserMatchClass = SVELogicalImmNotOperand<8>; 148} 149 150def sve_logical_imm16_not : Operand<i64> { 151 let ParserMatchClass = SVELogicalImmNotOperand<16>; 152} 153 154def sve_logical_imm32_not : Operand<i64> { 155 let ParserMatchClass = SVELogicalImmNotOperand<32>; 156} 157 158class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate> 159 : AsmOperandClass { 160 let Name = "SVE" # Infix # "Imm" # ElementWidth; 161 let DiagnosticType = "Invalid" # Name; 162 let RenderMethod = "addImmWithOptionalShiftOperands<8>"; 163 let ParserMethod = "tryParseImmWithOptionalShift"; 164 let PredicateMethod = Predicate; 165} 166 167def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">; 168def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">; 169def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">; 170def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">; 171 172def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">; 173def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">; 174def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">; 175def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">; 176 177class imm8_opt_lsl<int ElementWidth, string printType, 178 AsmOperandClass OpndClass> 179 : Operand<i32> { 180 let EncoderMethod = "getImm8OptLsl"; 181 let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">"; 182 let PrintMethod = "printImm8OptLsl<" # printType # ">"; 183 let ParserMatchClass = OpndClass; 184 let MIOperandInfo = (ops i32imm, i32imm); 185} 186 187def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>; 188def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>; 189def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>; 190def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>; 191 192def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>; 193def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>; 194def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>; 195def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>; 196 197def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; 198def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; 199def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; 200def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64>", []>; 201 202def SVECpyDupImm8Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i8>", []>; 203def SVECpyDupImm16Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i16>", []>; 204def SVECpyDupImm32Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i32>", []>; 205def SVECpyDupImm64Pat : ComplexPattern<i64, 2, "SelectSVECpyDupImm<MVT::i64>", []>; 206 207def SVELogicalImm8Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8>", []>; 208def SVELogicalImm16Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16>", []>; 209def SVELogicalImm32Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32>", []>; 210def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>; 211 212def SVELogicalImm8NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8, true>", []>; 213def SVELogicalImm16NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16, true>", []>; 214def SVELogicalImm32NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32, true>", []>; 215def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>; 216 217def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>; 218def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>; 219def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>; 220def SVEArithUImm64Pat : ComplexPattern<i64, 1, "SelectSVEArithImm<MVT::i64>", []>; 221 222def SVEArithSImmPat32 : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; 223def SVEArithSImmPat64 : ComplexPattern<i64, 1, "SelectSVESignedArithImm", []>; 224 225def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>; 226def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>; 227def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>; 228def SVEShiftImmL64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<0, 63>", []>; 229def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>; 230def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>; 231def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>; 232def SVEShiftImmR64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<1, 64, true>", []>; 233 234def SVEShiftSplatImmR : ComplexPattern<iAny, 1, "SelectSVEShiftSplatImmR", []>; 235 236def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>; 237 238class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass { 239 let Name = "SVEExactFPImmOperand" # Suffix; 240 let DiagnosticType = "Invalid" # Name; 241 let ParserMethod = "tryParseFPImm<false>"; 242 let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">"; 243 let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">"; 244} 245 246class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> { 247 let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">"; 248 let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>; 249} 250 251def sve_fpimm_half_one 252 : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half", 253 "AArch64ExactFPImm::one">; 254def sve_fpimm_half_two 255 : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half", 256 "AArch64ExactFPImm::two">; 257def sve_fpimm_zero_one 258 : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero", 259 "AArch64ExactFPImm::one">; 260 261def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{ 262 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17); 263}]> { 264 let ParserMatchClass = Imm1_16Operand; 265 let EncoderMethod = "getSVEIncDecImm"; 266 let DecoderMethod = "DecodeSVEIncDecImm"; 267} 268 269// This allows i32 immediate extraction from i64 based arithmetic. 270def sve_cnt_mul_imm_i32 : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; 271def sve_cnt_mul_imm_i64 : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, false>">; 272def sve_cnt_shl_imm : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, true>">; 273 274def sve_ext_imm_0_31 : ComplexPattern<i64, 1, "SelectEXTImm<31, 8>">; 275def sve_ext_imm_0_63 : ComplexPattern<i64, 1, "SelectEXTImm<63, 4>">; 276def sve_ext_imm_0_127 : ComplexPattern<i64, 1, "SelectEXTImm<127, 2>">; 277def sve_ext_imm_0_255 : ComplexPattern<i64, 1, "SelectEXTImm<255, 1>">; 278 279def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), 280 (int_aarch64_sve_cntp node:$pred, node:$src2), [{ 281 return N->hasOneUse(); 282}]>; 283 284def step_vector_oneuse : PatFrag<(ops node:$idx), 285 (step_vector node:$idx), [{ 286 return N->hasOneUse(); 287}]>; 288 289 290//===----------------------------------------------------------------------===// 291// SVE PTrue - These are used extensively throughout the pattern matching so 292// it's important we define them first. 293//===----------------------------------------------------------------------===// 294 295class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 296 ValueType vt, SDPatternOperator op> 297: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern), 298 asm, "\t$Pd, $pattern", 299 "", 300 [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> { 301 bits<4> Pd; 302 bits<5> pattern; 303 let Inst{31-24} = 0b00100101; 304 let Inst{23-22} = sz8_64; 305 let Inst{21-19} = 0b011; 306 let Inst{18-17} = opc{2-1}; 307 let Inst{16} = opc{0}; 308 let Inst{15-10} = 0b111000; 309 let Inst{9-5} = pattern; 310 let Inst{4} = 0b0; 311 let Inst{3-0} = Pd; 312 313 let Defs = !if(!eq (opc{0}, 1), [NZCV], []); 314 let ElementSize = pprty.ElementSize; 315 let isReMaterializable = 1; 316} 317 318multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> { 319 def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>; 320 def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>; 321 def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>; 322 def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>; 323 324 def : InstAlias<asm # "\t$Pd", 325 (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>; 326 def : InstAlias<asm # "\t$Pd", 327 (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>; 328 def : InstAlias<asm # "\t$Pd", 329 (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>; 330 def : InstAlias<asm # "\t$Pd", 331 (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>; 332} 333 334def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; 335def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>; 336 337let Predicates = [HasSVEorSME] in { 338 defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>; 339 defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>; 340 341 def : Pat<(nxv16i1 immAllOnesV), (PTRUE_B 31)>; 342 def : Pat<(nxv8i1 immAllOnesV), (PTRUE_H 31)>; 343 def : Pat<(nxv4i1 immAllOnesV), (PTRUE_S 31)>; 344 def : Pat<(nxv2i1 immAllOnesV), (PTRUE_D 31)>; 345} 346 347//===----------------------------------------------------------------------===// 348// SVE pattern match helpers. 349//===----------------------------------------------------------------------===// 350 351class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 352 Instruction inst> 353: Pat<(vtd (op vt1:$Op1)), 354 (inst $Op1)>; 355 356class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 357 ValueType vts, Instruction inst> 358: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)), 359 (inst $Op3, $Op1, $Op2)>; 360 361 362multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 363 ValueType vts, Instruction inst> { 364 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd undef))), 365 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 366 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, vtd:$Op3)), 367 (inst $Op3, $Op1, $Op2)>; 368} 369 370// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the 371// type of rounding. This is matched by timm0_1 in pattern below and ignored. 372class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 373 ValueType vts, Instruction inst> 374: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)), 375 (inst $Op3, $Op1, $Op2)>; 376 377multiclass SVE_1_Op_PassthruUndef_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 378 ValueType vts, Instruction inst>{ 379 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), (vtd undef))), 380 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 381 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, (i64 timm0_1), vtd:$Op3)), 382 (inst $Op3, $Op1, $Op2)>; 383} 384 385class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 386 ValueType it, ComplexPattern cpx, Instruction inst> 387 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm, i32:$shift)))))), 388 (inst $Op1, i32:$imm, i32:$shift)>; 389 390class SVE_1_Op_Imm_Arith_All_Active<ValueType vt, ValueType pt, SDPatternOperator op, 391 ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst> 392 : Pat<(vt (op (pt (SVEAllActive)), (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm)))))), 393 (inst $Op1, i32:$imm)>; 394 395class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 396 ValueType it, ComplexPattern cpx, Instruction inst> 397 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i64:$imm)))))), 398 (inst $Op1, i64:$imm)>; 399 400class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 401 ValueType vt2, Instruction inst> 402: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 403 (inst $Op1, $Op2)>; 404 405class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op, 406 ValueType pt, ValueType vt1, ValueType vt2, 407 Instruction inst> 408: Pat<(vtd (op (pt (SVEAllActive)), vt1:$Op1, vt2:$Op2)), 409 (inst $Op1, $Op2)>; 410 411class SVE_2_Op_Pred_All_Active_Pt<ValueType vtd, SDPatternOperator op, 412 ValueType pt, ValueType vt1, ValueType vt2, 413 Instruction inst> 414: Pat<(vtd (op (pt (SVEAllActive:$Op1)), vt1:$Op2, vt2:$Op3)), 415 (inst $Op1, $Op2, $Op3)>; 416 417class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 418 ValueType vt2, ValueType vt3, Instruction inst> 419: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 420 (inst $Op1, $Op2, $Op3)>; 421 422multiclass SVE_3_Op_Undef_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 423 ValueType vt2, ValueType vt3, Instruction inst> { 424 def : Pat<(vtd (op (vt1 undef), vt2:$Op1, vt3:$Op2)), 425 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 426 def : Pat<(vtd (op vt1:$Op1, (vt2 (SVEAllActive:$Op2)), vt3:$Op3)), 427 (inst $Op1, $Op2, $Op3)>; 428} 429 430class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 431 ValueType vt2, ValueType vt3, ValueType vt4, 432 Instruction inst> 433: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)), 434 (inst $Op1, $Op2, $Op3, $Op4)>; 435 436class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 437 ValueType vt2, Operand ImmTy, Instruction inst> 438: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))), 439 (inst $Op1, ImmTy:$Op2)>; 440 441class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 442 ValueType vt2, ValueType vt3, Operand ImmTy, 443 Instruction inst> 444: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))), 445 (inst $Op1, $Op2, ImmTy:$Op3)>; 446 447class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 448 ValueType vt2, ValueType vt3, ValueType vt4, 449 Operand ImmTy, Instruction inst> 450: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), 451 (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; 452 453def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>; 454def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>; 455 456let AddedComplexity = 1 in { 457class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1, 458 ValueType vt2, ValueType vt3, Instruction inst> 459: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))), 460 (inst $Op1, $Op2, $Op3)>; 461 462class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op, 463 ValueType vt1, ValueType vt2, 464 Operand vt3, Instruction inst> 465: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))), 466 (inst $Op1, $Op2, vt3:$Op3)>; 467} 468 469// 470// Common but less generic patterns. 471// 472 473class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 474 Instruction inst, Instruction ptrue> 475: Pat<(vtd (op vt1:$Op1)), 476 (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>; 477 478class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 479 ValueType vt2, Instruction inst, Instruction ptrue> 480: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 481 (inst (ptrue 31), $Op1, $Op2)>; 482 483class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt, 484 ValueType inreg_vt, Instruction inst> 485: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)), 486 (inst $PassThru, $Pg, $Src)>; 487 488multiclass SVE_InReg_Extend_PassthruUndef<ValueType vt, SDPatternOperator op, ValueType pt, 489 ValueType inreg_vt, Instruction inst> { 490 def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt undef))), 491 (inst (IMPLICIT_DEF), $Pg, $Src)>; 492 def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, vt:$PassThru)), 493 (inst $PassThru, $Pg, $Src)>; 494} 495 496class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op, 497 ValueType pt, ValueType it, 498 ComplexPattern cast, Instruction inst> 499: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))), 500 (inst $Pg, $Rn, i32:$imm)>; 501 502class SVE_Shift_DupImm_All_Active_Pat<ValueType vt, SDPatternOperator op, 503 ValueType pt, ValueType it, 504 ComplexPattern cast, Instruction inst> 505: Pat<(vt (op (pt (SVEAllActive)), vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))), 506 (inst $Rn, i32:$imm)>; 507 508class SVE_2_Op_Fp_Imm_Pat<ValueType vt, SDPatternOperator op, 509 ValueType pt, ValueType it, 510 FPImmLeaf immL, int imm, 511 Instruction inst> 512: Pat<(vt (op (pt PPR_3b:$Pg), (vt ZPR:$Zs1), (vt (splat_vector (it immL))))), 513 (inst $Pg, $Zs1, imm)>; 514 515class SVE_2_Op_Fp_Imm_Pat_Zero<ValueType vt, SDPatternOperator op, 516 ValueType pt, ValueType it, 517 FPImmLeaf immL, int imm, 518 Instruction inst> 519: Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Zs1, (SVEDup0)), 520 (vt (splat_vector (it immL))))), 521 (inst $Pg, $Zs1, imm)>; 522 523// Used to re-order the operands of BSP when lowering to BSL. BSP has the order: 524// mask, in1, in2 whereas BSL for SVE2 has them ordered in1, in2, mask 525class SVE_3_Op_BSP_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 526 ValueType vt2, ValueType vt3, Instruction inst> 527: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 528 (inst $Op2, $Op3, $Op1)>; 529 530class SVE_Shift_Add_All_Active_Pat<ValueType vtd, SDPatternOperator op, ValueType pt, 531 ValueType vt1, ValueType vt2, ValueType vt3, 532 Instruction inst> 533: Pat<(vtd (add vt1:$Op1, (op (pt (SVEAllActive)), vt2:$Op2, vt3:$Op3))), 534 (inst $Op1, $Op2, $Op3)>; 535 536//===----------------------------------------------------------------------===// 537// SVE pattern match helpers. 538//===----------------------------------------------------------------------===// 539 540// Matches either an intrinsic, or a predicated operation with an all active predicate 541class EitherVSelectOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOperator sdnode> 542: PatFrags<(ops node:$Pg, node:$Op1, node:$Op2), [ 543 (intrinsic node:$Pg, node:$Op1, node:$Op2), 544 (vselect node:$Pg, (sdnode (SVEAllActive), node:$Op1, node:$Op2), node:$Op1), 545 ]>; 546 547// 548// Pseudo -> Instruction mappings 549// 550def getSVEPseudoMap : InstrMapping { 551 let FilterClass = "SVEPseudo2Instr"; 552 let RowFields = ["PseudoName"]; 553 let ColFields = ["IsInstr"]; 554 let KeyCol = ["0"]; 555 let ValueCols = [["1"]]; 556} 557 558class SVEPseudo2Instr<string name, bit instr> { 559 string PseudoName = name; 560 bit IsInstr = instr; 561} 562 563// Lookup e.g. DIV -> DIVR 564def getSVERevInstr : InstrMapping { 565 let FilterClass = "SVEInstr2Rev"; 566 let RowFields = ["InstrName"]; 567 let ColFields = ["isReverseInstr"]; 568 let KeyCol = ["0"]; 569 let ValueCols = [["1"]]; 570} 571 572// Lookup e.g. DIVR -> DIV 573def getSVENonRevInstr : InstrMapping { 574 let FilterClass = "SVEInstr2Rev"; 575 let RowFields = ["InstrName"]; 576 let ColFields = ["isReverseInstr"]; 577 let KeyCol = ["1"]; 578 let ValueCols = [["0"]]; 579} 580 581class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> { 582 string InstrName = !if(name1IsReverseInstr, name1, name2); 583 bit isReverseInstr = name1IsReverseInstr; 584} 585 586// 587// Pseudos for destructive operands 588// 589let hasNoSchedulingInfo = 1 in { 590 class PredTwoOpPseudo<string name, ZPRRegOp zprty, 591 FalseLanesEnum flags = FalseLanesNone> 592 : SVEPseudo2Instr<name, 0>, 593 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> { 594 let FalseLanes = flags; 595 } 596 597 class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty, 598 FalseLanesEnum flags = FalseLanesNone> 599 : SVEPseudo2Instr<name, 0>, 600 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> { 601 let FalseLanes = flags; 602 } 603 604 class PredThreeOpPseudo<string name, ZPRRegOp zprty, 605 FalseLanesEnum flags = FalseLanesNone> 606 : SVEPseudo2Instr<name, 0>, 607 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2, zprty:$Zs3), []> { 608 let FalseLanes = flags; 609 } 610} 611 612// 613// Pseudos for passthru operands 614// 615let hasNoSchedulingInfo = 1 in { 616 class PredOneOpPassthruPseudo<string name, ZPRRegOp zprty> 617 : SVEPseudo2Instr<name, 0>, 618 Pseudo<(outs zprty:$Zd), (ins zprty:$Passthru, PPR3bAny:$Pg, zprty:$Zs), []>; 619} 620 621//===----------------------------------------------------------------------===// 622// SVE Predicate Misc Group 623//===----------------------------------------------------------------------===// 624 625class sve_int_pfalse<bits<6> opc, string asm> 626: I<(outs PPR8:$Pd), (ins), 627 asm, "\t$Pd", 628 "", 629 []>, Sched<[]> { 630 bits<4> Pd; 631 let Inst{31-24} = 0b00100101; 632 let Inst{23-22} = opc{5-4}; 633 let Inst{21-19} = 0b011; 634 let Inst{18-16} = opc{3-1}; 635 let Inst{15-10} = 0b111001; 636 let Inst{9} = opc{0}; 637 let Inst{8-4} = 0b00000; 638 let Inst{3-0} = Pd; 639 640 let isReMaterializable = 1; 641} 642 643multiclass sve_int_pfalse<bits<6> opc, string asm> { 644 def NAME : sve_int_pfalse<opc, asm>; 645 646 def : Pat<(nxv16i1 immAllZerosV), (!cast<Instruction>(NAME))>; 647 def : Pat<(nxv8i1 immAllZerosV), (!cast<Instruction>(NAME))>; 648 def : Pat<(nxv4i1 immAllZerosV), (!cast<Instruction>(NAME))>; 649 def : Pat<(nxv2i1 immAllZerosV), (!cast<Instruction>(NAME))>; 650 def : Pat<(nxv1i1 immAllZerosV), (!cast<Instruction>(NAME))>; 651} 652 653class sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op> 654: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn), 655 asm, "\t$Pg, $Pn", 656 "", 657 [(op (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 658 bits<4> Pg; 659 bits<4> Pn; 660 let Inst{31-24} = 0b00100101; 661 let Inst{23-22} = opc{5-4}; 662 let Inst{21-19} = 0b010; 663 let Inst{18-16} = opc{3-1}; 664 let Inst{15-14} = 0b11; 665 let Inst{13-10} = Pg; 666 let Inst{9} = opc{0}; 667 let Inst{8-5} = Pn; 668 let Inst{4-0} = 0b00000; 669 670 let Defs = [NZCV]; 671 let isCompare = 1; 672} 673 674class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm, 675 PPRRegOp pprty> 676: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn), 677 asm, "\t$Pdn, $Pg, $_Pdn", 678 "", 679 []>, Sched<[]> { 680 bits<4> Pdn; 681 bits<4> Pg; 682 let Inst{31-24} = 0b00100101; 683 let Inst{23-22} = sz8_64; 684 let Inst{21-19} = 0b011; 685 let Inst{18-16} = opc{4-2}; 686 let Inst{15-11} = 0b11000; 687 let Inst{10-9} = opc{1-0}; 688 let Inst{8-5} = Pg; 689 let Inst{4} = 0; 690 let Inst{3-0} = Pdn; 691 692 let Constraints = "$Pdn = $_Pdn"; 693 let Defs = [NZCV]; 694 let isPTestLike = 1; 695 let ElementSize = pprty.ElementSize; 696} 697 698multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> { 699 def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>; 700 701 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 702} 703 704multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> { 705 def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>; 706 def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>; 707 def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>; 708 def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>; 709 710 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 711 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 712 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 713 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 714} 715 716//===----------------------------------------------------------------------===// 717// SVE Predicate Count Group 718//===----------------------------------------------------------------------===// 719 720class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm, 721 RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty> 722: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn), 723 asm, "\t$Rdn, $Pg", 724 "", 725 []>, Sched<[]> { 726 bits<5> Rdn; 727 bits<4> Pg; 728 let Inst{31-24} = 0b00100101; 729 let Inst{23-22} = sz8_64; 730 let Inst{21-19} = 0b101; 731 let Inst{18-16} = opc{4-2}; 732 let Inst{15-11} = 0b10001; 733 let Inst{10-9} = opc{1-0}; 734 let Inst{8-5} = Pg; 735 let Inst{4-0} = Rdn; 736 737 // Signed 32bit forms require their GPR operand printed. 738 let AsmString = !if(!eq(opc{4,2-0}, 0b0000), 739 !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"), 740 !strconcat(asm, "\t$Rdn, $Pg")); 741 let Constraints = "$Rdn = $_Rdn"; 742} 743 744multiclass sve_int_count_r_s32<bits<5> opc, string asm, 745 SDPatternOperator op> { 746 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>; 747 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>; 748 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>; 749 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>; 750 751 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 752 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 753 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))), 754 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 755 756 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 757 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 758 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))), 759 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 760 761 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 762 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 763 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))), 764 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 765 766 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 767 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 768 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))), 769 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 770} 771 772multiclass sve_int_count_r_u32<bits<5> opc, string asm, 773 SDPatternOperator op> { 774 def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>; 775 def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>; 776 def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>; 777 def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>; 778 779 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 780 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 781 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 782 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 783 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 784 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 785 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 786 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 787} 788 789multiclass sve_int_count_r_x64<bits<5> opc, string asm, 790 SDPatternOperator op, 791 SDPatternOperator combine_op = null_frag> { 792 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>; 793 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>; 794 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>; 795 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>; 796 797 def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))), 798 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 799 def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))), 800 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 801 def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))), 802 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 803 def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))), 804 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 805 806 // combine_op(x, cntp(all_active, p)) ==> inst p, x 807 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 (SVEAllActive)), (nxv16i1 PPRAny:$pred)))), 808 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 809 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 (SVEAllActive)), (nxv8i1 PPRAny:$pred)))), 810 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 811 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 (SVEAllActive)), (nxv4i1 PPRAny:$pred)))), 812 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 813 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 (SVEAllActive)), (nxv2i1 PPRAny:$pred)))), 814 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 815 816 // combine_op(x, cntp(p, p)) ==> inst p, x 817 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 PPRAny:$pred), (nxv16i1 PPRAny:$pred)))), 818 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 819 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 PPRAny:$pred), (nxv8i1 PPRAny:$pred)))), 820 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 821 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 PPRAny:$pred), (nxv4i1 PPRAny:$pred)))), 822 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 823 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 PPRAny:$pred), (nxv2i1 PPRAny:$pred)))), 824 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 825} 826 827class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm, 828 ZPRRegOp zprty, PPRRegOp pprty> 829: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm), 830 asm, "\t$Zdn, $Pm", 831 "", 832 []>, Sched<[]> { 833 bits<4> Pm; 834 bits<5> Zdn; 835 let Inst{31-24} = 0b00100101; 836 let Inst{23-22} = sz8_64; 837 let Inst{21-19} = 0b101; 838 let Inst{18-16} = opc{4-2}; 839 let Inst{15-11} = 0b10000; 840 let Inst{10-9} = opc{1-0}; 841 let Inst{8-5} = Pm; 842 let Inst{4-0} = Zdn; 843 844 let Constraints = "$Zdn = $_Zdn"; 845 let DestructiveInstType = DestructiveOther; 846 let ElementSize = ElementSizeNone; 847} 848 849multiclass sve_int_count_v<bits<5> opc, string asm, 850 SDPatternOperator op = null_frag> { 851 def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>; 852 def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>; 853 def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>; 854 855 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>; 856 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>; 857 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>; 858 859 def : InstAlias<asm # "\t$Zdn, $Pm", 860 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>; 861 def : InstAlias<asm # "\t$Zdn, $Pm", 862 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>; 863 def : InstAlias<asm # "\t$Zdn, $Pm", 864 (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>; 865} 866 867class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm, 868 PPRRegOp pprty> 869: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn), 870 asm, "\t$Rd, $Pg, $Pn", 871 "", 872 []>, Sched<[]> { 873 bits<4> Pg; 874 bits<4> Pn; 875 bits<5> Rd; 876 let Inst{31-24} = 0b00100101; 877 let Inst{23-22} = sz8_64; 878 let Inst{21-19} = 0b100; 879 let Inst{18-16} = opc{3-1}; 880 let Inst{15-14} = 0b10; 881 let Inst{13-10} = Pg; 882 let Inst{9} = opc{0}; 883 let Inst{8-5} = Pn; 884 let Inst{4-0} = Rd; 885} 886 887multiclass sve_int_pcount_pred<bits<4> opc, string asm, 888 SDPatternOperator int_op> { 889 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>; 890 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>; 891 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>; 892 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>; 893 894 def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 895 def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 896 def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 897 def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 898} 899 900//===----------------------------------------------------------------------===// 901// SVE Element Count Group 902//===----------------------------------------------------------------------===// 903 904class sve_int_count<bits<3> opc, string asm> 905: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 906 asm, "\t$Rd, $pattern, mul $imm4", 907 "", 908 []>, Sched<[]> { 909 bits<5> Rd; 910 bits<4> imm4; 911 bits<5> pattern; 912 let Inst{31-24} = 0b00000100; 913 let Inst{23-22} = opc{2-1}; 914 let Inst{21-20} = 0b10; 915 let Inst{19-16} = imm4; 916 let Inst{15-11} = 0b11100; 917 let Inst{10} = opc{0}; 918 let Inst{9-5} = pattern; 919 let Inst{4-0} = Rd; 920 921 let isReMaterializable = 1; 922} 923 924multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> { 925 def NAME : sve_int_count<opc, asm>; 926 927 def : InstAlias<asm # "\t$Rd, $pattern", 928 (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>; 929 def : InstAlias<asm # "\t$Rd", 930 (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>; 931 932 def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm))), 933 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 934 935 def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm))), 936 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 937 938 def : Pat<(i64 (op sve_pred_enum:$pattern)), 939 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>; 940} 941 942class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty> 943: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 944 asm, "\t$Zdn, $pattern, mul $imm4", 945 "", 946 []>, Sched<[]> { 947 bits<5> Zdn; 948 bits<5> pattern; 949 bits<4> imm4; 950 let Inst{31-24} = 0b00000100; 951 let Inst{23-22} = opc{4-3}; 952 let Inst{21} = 0b1; 953 let Inst{20} = opc{2}; 954 let Inst{19-16} = imm4; 955 let Inst{15-12} = 0b1100; 956 let Inst{11-10} = opc{1-0}; 957 let Inst{9-5} = pattern; 958 let Inst{4-0} = Zdn; 959 960 let Constraints = "$Zdn = $_Zdn"; 961 let DestructiveInstType = DestructiveOther; 962 let ElementSize = ElementSizeNone; 963} 964 965multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty, 966 SDPatternOperator op = null_frag, 967 ValueType vt = OtherVT> { 968 def NAME : sve_int_countvlv<opc, asm, zprty>; 969 970 def : InstAlias<asm # "\t$Zdn, $pattern", 971 (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>; 972 def : InstAlias<asm # "\t$Zdn", 973 (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>; 974 975 def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 976 (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 977} 978 979class sve_int_pred_pattern_a<bits<3> opc, string asm> 980: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 981 asm, "\t$Rdn, $pattern, mul $imm4", 982 "", 983 []>, Sched<[]> { 984 bits<5> Rdn; 985 bits<5> pattern; 986 bits<4> imm4; 987 let Inst{31-24} = 0b00000100; 988 let Inst{23-22} = opc{2-1}; 989 let Inst{21-20} = 0b11; 990 let Inst{19-16} = imm4; 991 let Inst{15-11} = 0b11100; 992 let Inst{10} = opc{0}; 993 let Inst{9-5} = pattern; 994 let Inst{4-0} = Rdn; 995 996 let Constraints = "$Rdn = $_Rdn"; 997} 998 999multiclass sve_int_pred_pattern_a<bits<3> opc, string asm, 1000 SDPatternOperator op, 1001 SDPatternOperator opcnt> { 1002 let Predicates = [HasSVEorSME] in { 1003 def NAME : sve_int_pred_pattern_a<opc, asm>; 1004 1005 def : InstAlias<asm # "\t$Rdn, $pattern", 1006 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1007 def : InstAlias<asm # "\t$Rdn", 1008 (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>; 1009 } 1010 1011 let Predicates = [HasSVEorSME, UseScalarIncVL] in { 1012 def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))), 1013 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>; 1014 1015 def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm)))), 1016 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 1017 1018 def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm)))), 1019 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 1020 1021 def : Pat<(i32 (op GPR32:$Rdn, (i32 (trunc (opcnt (sve_pred_enum:$pattern)))))), 1022 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1023 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1), 1024 sub_32))>; 1025 1026 def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm_i32 i32:$imm)))), 1027 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1028 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 1029 sub_32))>; 1030 1031 def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_shl_imm i32:$imm)))), 1032 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1033 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 1034 sub_32))>; 1035 } 1036} 1037 1038class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt, 1039 RegisterOperand st> 1040: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 1041 asm, "\t$Rdn, $pattern, mul $imm4", 1042 "", 1043 []>, Sched<[]> { 1044 bits<5> Rdn; 1045 bits<5> pattern; 1046 bits<4> imm4; 1047 let Inst{31-24} = 0b00000100; 1048 let Inst{23-22} = opc{4-3}; 1049 let Inst{21} = 0b1; 1050 let Inst{20} = opc{2}; 1051 let Inst{19-16} = imm4; 1052 let Inst{15-12} = 0b1111; 1053 let Inst{11-10} = opc{1-0}; 1054 let Inst{9-5} = pattern; 1055 let Inst{4-0} = Rdn; 1056 1057 // Signed 32bit forms require their GPR operand printed. 1058 let AsmString = !if(!eq(opc{2,0}, 0b00), 1059 !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"), 1060 !strconcat(asm, "\t$Rdn, $pattern, mul $imm4")); 1061 1062 let Constraints = "$Rdn = $_Rdn"; 1063} 1064 1065multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm, 1066 SDPatternOperator op> { 1067 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>; 1068 1069 def : InstAlias<asm # "\t$Rd, $Rn, $pattern", 1070 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>; 1071 def : InstAlias<asm # "\t$Rd, $Rn", 1072 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>; 1073 1074 // NOTE: Register allocation doesn't like tied operands of differing register 1075 // class, hence the extra INSERT_SUBREG complication. 1076 1077 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1078 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>; 1079 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))), 1080 (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1081} 1082 1083multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm, 1084 SDPatternOperator op> { 1085 def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>; 1086 1087 def : InstAlias<asm # "\t$Rdn, $pattern", 1088 (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1089 def : InstAlias<asm # "\t$Rdn", 1090 (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>; 1091 1092 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1093 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1094} 1095 1096multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm, 1097 SDPatternOperator op> { 1098 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>; 1099 1100 def : InstAlias<asm # "\t$Rdn, $pattern", 1101 (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1102 def : InstAlias<asm # "\t$Rdn", 1103 (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>; 1104 1105 def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1106 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1107} 1108 1109 1110//===----------------------------------------------------------------------===// 1111// SVE Permute - Cross Lane Group 1112//===----------------------------------------------------------------------===// 1113 1114class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1115 ValueType vt, RegisterClass srcRegType, 1116 SDPatternOperator op> 1117: I<(outs zprty:$Zd), (ins srcRegType:$Rn), 1118 asm, "\t$Zd, $Rn", 1119 "", 1120 [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> { 1121 bits<5> Rn; 1122 bits<5> Zd; 1123 let Inst{31-24} = 0b00000101; 1124 let Inst{23-22} = sz8_64; 1125 let Inst{21-10} = 0b100000001110; 1126 let Inst{9-5} = Rn; 1127 let Inst{4-0} = Zd; 1128} 1129 1130multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> { 1131 def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>; 1132 def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>; 1133 def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>; 1134 def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>; 1135 1136 def : InstAlias<"mov $Zd, $Rn", 1137 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>; 1138 def : InstAlias<"mov $Zd, $Rn", 1139 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>; 1140 def : InstAlias<"mov $Zd, $Rn", 1141 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>; 1142 def : InstAlias<"mov $Zd, $Rn", 1143 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>; 1144} 1145 1146class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm, 1147 ZPRRegOp zprty> 1148: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx), 1149 asm, "\t$Zd, $Zn$idx", 1150 "", 1151 []>, Sched<[]> { 1152 bits<5> Zd; 1153 bits<5> Zn; 1154 bits<7> idx; 1155 let Inst{31-24} = 0b00000101; 1156 let Inst{23-22} = {?,?}; // imm3h 1157 let Inst{21} = 0b1; 1158 let Inst{20-16} = tsz; 1159 let Inst{15-10} = 0b001000; 1160 let Inst{9-5} = Zn; 1161 let Inst{4-0} = Zd; 1162} 1163 1164multiclass sve_int_perm_dup_i<string asm> { 1165 def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> { 1166 let Inst{23-22} = idx{5-4}; 1167 let Inst{20-17} = idx{3-0}; 1168 } 1169 def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> { 1170 let Inst{23-22} = idx{4-3}; 1171 let Inst{20-18} = idx{2-0}; 1172 } 1173 def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> { 1174 let Inst{23-22} = idx{3-2}; 1175 let Inst{20-19} = idx{1-0}; 1176 } 1177 def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> { 1178 let Inst{23-22} = idx{2-1}; 1179 let Inst{20} = idx{0}; 1180 } 1181 def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> { 1182 let Inst{23-22} = idx{1-0}; 1183 } 1184 1185 def : InstAlias<"mov $Zd, $Zn$idx", 1186 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>; 1187 def : InstAlias<"mov $Zd, $Zn$idx", 1188 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>; 1189 def : InstAlias<"mov $Zd, $Zn$idx", 1190 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>; 1191 def : InstAlias<"mov $Zd, $Zn$idx", 1192 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>; 1193 def : InstAlias<"mov $Zd, $Zn$idx", 1194 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>; 1195 def : InstAlias<"mov $Zd, $Bn", 1196 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>; 1197 def : InstAlias<"mov $Zd, $Hn", 1198 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>; 1199 def : InstAlias<"mov $Zd, $Sn", 1200 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>; 1201 def : InstAlias<"mov $Zd, $Dn", 1202 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>; 1203 def : InstAlias<"mov $Zd, $Qn", 1204 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>; 1205 1206 // Duplicate extracted element of vector into all vector elements 1207 def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))), 1208 (!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>; 1209 def : Pat<(nxv8i16 (splat_vector (i32 (vector_extract (nxv8i16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1210 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1211 def : Pat<(nxv4i32 (splat_vector (i32 (vector_extract (nxv4i32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1212 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1213 def : Pat<(nxv2i64 (splat_vector (i64 (vector_extract (nxv2i64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1214 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1215 def : Pat<(nxv8f16 (splat_vector (f16 (vector_extract (nxv8f16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1216 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1217 def : Pat<(nxv8bf16 (splat_vector (bf16 (vector_extract (nxv8bf16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1218 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1219 def : Pat<(nxv4f16 (splat_vector (f16 (vector_extract (nxv4f16 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1220 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1221 def : Pat<(nxv2f16 (splat_vector (f16 (vector_extract (nxv2f16 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1222 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1223 def : Pat<(nxv4f32 (splat_vector (f32 (vector_extract (nxv4f32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1224 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1225 def : Pat<(nxv2f32 (splat_vector (f32 (vector_extract (nxv2f32 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1226 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1227 def : Pat<(nxv2f64 (splat_vector (f64 (vector_extract (nxv2f64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1228 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1229 1230 def : Pat<(nxv16i8 (AArch64duplane128 nxv16i8:$Op1, i64:$imm)), 1231 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1232 def : Pat<(nxv8i16 (AArch64duplane128 nxv8i16:$Op1, i64:$imm)), 1233 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1234 def : Pat<(nxv4i32 (AArch64duplane128 nxv4i32:$Op1, i64:$imm)), 1235 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1236 def : Pat<(nxv2i64 (AArch64duplane128 nxv2i64:$Op1, i64:$imm)), 1237 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1238 def : Pat<(nxv8f16 (AArch64duplane128 nxv8f16:$Op1, i64:$imm)), 1239 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1240 def : Pat<(nxv4f32 (AArch64duplane128 nxv4f32:$Op1, i64:$imm)), 1241 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1242 def : Pat<(nxv2f64 (AArch64duplane128 nxv2f64:$Op1, i64:$imm)), 1243 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1244 def : Pat<(nxv8bf16 (AArch64duplane128 nxv8bf16:$Op1, i64:$imm)), 1245 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1246} 1247 1248class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty, 1249 RegisterOperand VecList> 1250: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm), 1251 asm, "\t$Zd, $Zn, $Zm", 1252 "", 1253 []>, Sched<[]> { 1254 bits<5> Zd; 1255 bits<5> Zm; 1256 bits<5> Zn; 1257 let Inst{31-24} = 0b00000101; 1258 let Inst{23-22} = sz8_64; 1259 let Inst{21} = 0b1; 1260 let Inst{20-16} = Zm; 1261 let Inst{15-13} = 0b001; 1262 let Inst{12-11} = opc; 1263 let Inst{10} = 0b0; 1264 let Inst{9-5} = Zn; 1265 let Inst{4-0} = Zd; 1266} 1267 1268multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> { 1269 def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>; 1270 def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>; 1271 def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>; 1272 def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>; 1273 1274 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1275 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>; 1276 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1277 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>; 1278 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1279 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>; 1280 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1281 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>; 1282 1283 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1284 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1285 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1286 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1287 1288 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1289 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1290 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1291 1292 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1293} 1294 1295multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> { 1296 def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>; 1297 def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>; 1298 def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>; 1299 def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>; 1300 1301 def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)), 1302 (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0, 1303 nxv16i8:$Op2, zsub1), 1304 nxv16i8:$Op3))>; 1305 1306 def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)), 1307 (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0, 1308 nxv8i16:$Op2, zsub1), 1309 nxv8i16:$Op3))>; 1310 1311 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)), 1312 (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0, 1313 nxv4i32:$Op2, zsub1), 1314 nxv4i32:$Op3))>; 1315 1316 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)), 1317 (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0, 1318 nxv2i64:$Op2, zsub1), 1319 nxv2i64:$Op3))>; 1320 1321 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)), 1322 (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0, 1323 nxv8f16:$Op2, zsub1), 1324 nxv8i16:$Op3))>; 1325 1326 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)), 1327 (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0, 1328 nxv4f32:$Op2, zsub1), 1329 nxv4i32:$Op3))>; 1330 1331 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)), 1332 (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0, 1333 nxv2f64:$Op2, zsub1), 1334 nxv2i64:$Op3))>; 1335 1336 def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)), 1337 (nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0, 1338 nxv8bf16:$Op2, zsub1), 1339 nxv8i16:$Op3))>; 1340} 1341 1342class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1343: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm), 1344 asm, "\t$Zd, $Zn, $Zm", 1345 "", 1346 []>, Sched<[]> { 1347 bits<5> Zd; 1348 bits<5> Zm; 1349 bits<5> Zn; 1350 let Inst{31-24} = 0b00000101; 1351 let Inst{23-22} = sz8_64; 1352 let Inst{21} = 0b1; 1353 let Inst{20-16} = Zm; 1354 let Inst{15-10} = 0b001011; 1355 let Inst{9-5} = Zn; 1356 let Inst{4-0} = Zd; 1357 1358 let Constraints = "$Zd = $_Zd"; 1359} 1360 1361multiclass sve2_int_perm_tbx<string asm, SDPatternOperator op> { 1362 def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>; 1363 def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>; 1364 def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>; 1365 def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>; 1366 1367 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1368 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1369 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1370 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1371 1372 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1373 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1374 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1375 1376 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1377} 1378 1379class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1380: I<(outs zprty:$Zd), (ins zprty:$Zn), 1381 asm, "\t$Zd, $Zn", 1382 "", 1383 []>, Sched<[]> { 1384 bits<5> Zd; 1385 bits<5> Zn; 1386 let Inst{31-24} = 0b00000101; 1387 let Inst{23-22} = sz8_64; 1388 let Inst{21-10} = 0b111000001110; 1389 let Inst{9-5} = Zn; 1390 let Inst{4-0} = Zd; 1391} 1392 1393multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> { 1394 def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>; 1395 def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>; 1396 def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>; 1397 def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>; 1398 1399 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>; 1400 def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 1401 def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 1402 def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 1403 1404 def : SVE_1_Op_Pat<nxv2f16, op, nxv2f16, !cast<Instruction>(NAME # _D)>; 1405 def : SVE_1_Op_Pat<nxv4f16, op, nxv4f16, !cast<Instruction>(NAME # _S)>; 1406 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 1407 def : SVE_1_Op_Pat<nxv2f32, op, nxv2f32, !cast<Instruction>(NAME # _D)>; 1408 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 1409 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 1410 1411 def : SVE_1_Op_Pat<nxv2bf16, op, nxv2bf16, !cast<Instruction>(NAME # _D)>; 1412 def : SVE_1_Op_Pat<nxv4bf16, op, nxv4bf16, !cast<Instruction>(NAME # _S)>; 1413 def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1414} 1415 1416class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty> 1417: I<(outs pprty:$Pd), (ins pprty:$Pn), 1418 asm, "\t$Pd, $Pn", 1419 "", 1420 []>, Sched<[]> { 1421 bits<4> Pd; 1422 bits<4> Pn; 1423 let Inst{31-24} = 0b00000101; 1424 let Inst{23-22} = sz8_64; 1425 let Inst{21-9} = 0b1101000100000; 1426 let Inst{8-5} = Pn; 1427 let Inst{4} = 0b0; 1428 let Inst{3-0} = Pd; 1429} 1430 1431multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> { 1432 def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>; 1433 def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>; 1434 def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>; 1435 def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>; 1436 1437 def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>; 1438 def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>; 1439 def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>; 1440 def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>; 1441} 1442 1443class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm, 1444 ZPRRegOp zprty1, ZPRRegOp zprty2> 1445: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 1446 asm, "\t$Zd, $Zn", 1447 "", []>, Sched<[]> { 1448 bits<5> Zd; 1449 bits<5> Zn; 1450 let Inst{31-24} = 0b00000101; 1451 let Inst{23-22} = sz16_64; 1452 let Inst{21-18} = 0b1100; 1453 let Inst{17-16} = opc; 1454 let Inst{15-10} = 0b001110; 1455 let Inst{9-5} = Zn; 1456 let Inst{4-0} = Zd; 1457} 1458 1459multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> { 1460 def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>; 1461 def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>; 1462 def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>; 1463 1464 def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>; 1465 def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>; 1466 def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>; 1467} 1468 1469class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1470 RegisterClass srcRegType> 1471: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm), 1472 asm, "\t$Zdn, $Rm", 1473 "", 1474 []>, Sched<[]> { 1475 bits<5> Rm; 1476 bits<5> Zdn; 1477 let Inst{31-24} = 0b00000101; 1478 let Inst{23-22} = sz8_64; 1479 let Inst{21-10} = 0b100100001110; 1480 let Inst{9-5} = Rm; 1481 let Inst{4-0} = Zdn; 1482 1483 let Constraints = "$Zdn = $_Zdn"; 1484 let DestructiveInstType = DestructiveOther; 1485} 1486 1487multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> { 1488 def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>; 1489 def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>; 1490 def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>; 1491 def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>; 1492 1493 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 1494 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 1495 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 1496 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>; 1497} 1498 1499class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1500 FPRasZPROperand srcOpType> 1501: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcOpType:$Vm), 1502 asm, "\t$Zdn, $Vm", 1503 "", 1504 []>, Sched<[]> { 1505 bits<5> Vm; 1506 bits<5> Zdn; 1507 let Inst{31-24} = 0b00000101; 1508 let Inst{23-22} = sz8_64; 1509 let Inst{21-10} = 0b110100001110; 1510 let Inst{9-5} = Vm; 1511 let Inst{4-0} = Zdn; 1512 1513 let Constraints = "$Zdn = $_Zdn"; 1514 let DestructiveInstType = DestructiveOther; 1515} 1516 1517multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> { 1518 def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8asZPR>; 1519 def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16asZPR>; 1520 def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32asZPR>; 1521 def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64asZPR>; 1522 1523 def : Pat<(nxv8f16 (op nxv8f16:$Zn, f16:$Vm)), 1524 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1525 def : Pat<(nxv4f32 (op nxv4f32:$Zn, f32:$Vm)), 1526 (!cast<Instruction>(NAME # _S) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, ssub))>; 1527 def : Pat<(nxv2f64 (op nxv2f64:$Zn, f64:$Vm)), 1528 (!cast<Instruction>(NAME # _D) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, dsub))>; 1529 1530 def : Pat<(nxv8bf16 (op nxv8bf16:$Zn, bf16:$Vm)), 1531 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1532 1533 // Keep integer insertions within the vector unit. 1534 def : Pat<(nxv16i8 (op (nxv16i8 ZPR:$Zn), (i32 (vector_extract (nxv16i8 ZPR:$Vm), 0)))), 1535 (!cast<Instruction>(NAME # _B) $Zn, ZPR:$Vm)>; 1536 def : Pat<(nxv8i16 (op (nxv8i16 ZPR:$Zn), (i32 (vector_extract (nxv8i16 ZPR:$Vm), 0)))), 1537 (!cast<Instruction>(NAME # _H) $Zn, ZPR:$Vm)>; 1538 def : Pat<(nxv4i32 (op (nxv4i32 ZPR:$Zn), (i32 (vector_extract (nxv4i32 ZPR:$Vm), 0)))), 1539 (!cast<Instruction>(NAME # _S) $Zn, ZPR: $Vm)>; 1540 def : Pat<(nxv2i64 (op (nxv2i64 ZPR:$Zn), (i64 (vector_extract (nxv2i64 ZPR:$Vm), 0)))), 1541 (!cast<Instruction>(NAME # _D) $Zn, ZPR:$Vm)>; 1542 1543} 1544 1545//===----------------------------------------------------------------------===// 1546// SVE Permute - Extract Group 1547//===----------------------------------------------------------------------===// 1548 1549class sve_int_perm_extract_i<string asm> 1550: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8), 1551 asm, "\t$Zdn, $_Zdn, $Zm, $imm8", 1552 "", []>, Sched<[]> { 1553 bits<5> Zdn; 1554 bits<5> Zm; 1555 bits<8> imm8; 1556 let Inst{31-21} = 0b00000101001; 1557 let Inst{20-16} = imm8{7-3}; 1558 let Inst{15-13} = 0b000; 1559 let Inst{12-10} = imm8{2-0}; 1560 let Inst{9-5} = Zm; 1561 let Inst{4-0} = Zdn; 1562 1563 let Constraints = "$Zdn = $_Zdn"; 1564 let DestructiveInstType = DestructiveOther; 1565 let ElementSize = ElementSizeNone; 1566} 1567 1568multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> { 1569 def NAME : sve_int_perm_extract_i<asm>; 1570 1571 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255, 1572 !cast<Instruction>(NAME)>; 1573} 1574 1575class sve2_int_perm_extract_i_cons<string asm> 1576: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8), 1577 asm, "\t$Zd, $Zn, $imm8", 1578 "", []>, Sched<[]> { 1579 bits<5> Zd; 1580 bits<5> Zn; 1581 bits<8> imm8; 1582 let Inst{31-21} = 0b00000101011; 1583 let Inst{20-16} = imm8{7-3}; 1584 let Inst{15-13} = 0b000; 1585 let Inst{12-10} = imm8{2-0}; 1586 let Inst{9-5} = Zn; 1587 let Inst{4-0} = Zd; 1588} 1589 1590//===----------------------------------------------------------------------===// 1591// SVE Vector Select Group 1592//===----------------------------------------------------------------------===// 1593 1594class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1595: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm), 1596 asm, "\t$Zd, $Pg, $Zn, $Zm", 1597 "", 1598 []>, Sched<[]> { 1599 bits<4> Pg; 1600 bits<5> Zd; 1601 bits<5> Zm; 1602 bits<5> Zn; 1603 let Inst{31-24} = 0b00000101; 1604 let Inst{23-22} = sz8_64; 1605 let Inst{21} = 0b1; 1606 let Inst{20-16} = Zm; 1607 let Inst{15-14} = 0b11; 1608 let Inst{13-10} = Pg; 1609 let Inst{9-5} = Zn; 1610 let Inst{4-0} = Zd; 1611} 1612 1613multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> { 1614 def _B : sve_int_sel_vvv<0b00, asm, ZPR8>; 1615 def _H : sve_int_sel_vvv<0b01, asm, ZPR16>; 1616 def _S : sve_int_sel_vvv<0b10, asm, ZPR32>; 1617 def _D : sve_int_sel_vvv<0b11, asm, ZPR64>; 1618 1619 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1620 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1621 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1622 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1623 1624 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1625 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 1626 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1627 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 1628 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 1629 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1630 1631 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1632 1633 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1634 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>; 1635 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1636 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>; 1637 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1638 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>; 1639 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1640 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>; 1641} 1642 1643 1644//===----------------------------------------------------------------------===// 1645// SVE Predicate Logical Operations Group 1646//===----------------------------------------------------------------------===// 1647 1648class sve_int_pred_log<bits<4> opc, string asm> 1649: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 1650 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 1651 "", 1652 []>, Sched<[]> { 1653 bits<4> Pd; 1654 bits<4> Pg; 1655 bits<4> Pm; 1656 bits<4> Pn; 1657 let Inst{31-24} = 0b00100101; 1658 let Inst{23-22} = opc{3-2}; 1659 let Inst{21-20} = 0b00; 1660 let Inst{19-16} = Pm; 1661 let Inst{15-14} = 0b01; 1662 let Inst{13-10} = Pg; 1663 let Inst{9} = opc{1}; 1664 let Inst{8-5} = Pn; 1665 let Inst{4} = opc{0}; 1666 let Inst{3-0} = Pd; 1667 1668 // SEL has no predication qualifier. 1669 let AsmString = !if(!eq(opc, 0b0011), 1670 !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"), 1671 !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm")); 1672 1673 let Defs = !if(!eq (opc{2}, 1), [NZCV], []); 1674 1675} 1676 1677multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op, 1678 SDPatternOperator op_nopred = null_frag> { 1679 def NAME : sve_int_pred_log<opc, asm>; 1680 1681 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 1682 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>; 1683 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>; 1684 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>; 1685 def : SVE_3_Op_Pat<nxv1i1, op, nxv1i1, nxv1i1, nxv1i1, !cast<Instruction>(NAME)>; 1686 def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1, 1687 !cast<Instruction>(NAME), PTRUE_B>; 1688 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1, 1689 !cast<Instruction>(NAME), PTRUE_H>; 1690 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1, 1691 !cast<Instruction>(NAME), PTRUE_S>; 1692 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1, 1693 !cast<Instruction>(NAME), PTRUE_D>; 1694 // Emulate .Q operation using a PTRUE_D when the other lanes don't matter. 1695 def : SVE_2_Op_AllActive_Pat<nxv1i1, op_nopred, nxv1i1, nxv1i1, 1696 !cast<Instruction>(NAME), PTRUE_D>; 1697} 1698 1699// An instance of sve_int_pred_log_and but uses op_nopred's first operand as the 1700// general predicate. 1701multiclass sve_int_pred_log_v2<bits<4> opc, string asm, SDPatternOperator op, 1702 SDPatternOperator op_nopred> : 1703 sve_int_pred_log<opc, asm, op> { 1704 def : Pat<(nxv16i1 (op_nopred nxv16i1:$Op1, nxv16i1:$Op2)), 1705 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1706 def : Pat<(nxv8i1 (op_nopred nxv8i1:$Op1, nxv8i1:$Op2)), 1707 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1708 def : Pat<(nxv4i1 (op_nopred nxv4i1:$Op1, nxv4i1:$Op2)), 1709 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1710 def : Pat<(nxv2i1 (op_nopred nxv2i1:$Op1, nxv2i1:$Op2)), 1711 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1712 // Emulate .Q operation using a PTRUE_D when the other lanes don't matter. 1713 def : Pat<(nxv1i1 (op_nopred nxv1i1:$Op1, nxv1i1:$Op2)), 1714 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1715} 1716 1717//===----------------------------------------------------------------------===// 1718// SVE Logical Mask Immediate Group 1719//===----------------------------------------------------------------------===// 1720 1721class sve_int_log_imm<bits<2> opc, string asm> 1722: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13), 1723 asm, "\t$Zdn, $_Zdn, $imms13", 1724 "", []>, Sched<[]> { 1725 bits<5> Zdn; 1726 bits<13> imms13; 1727 let Inst{31-24} = 0b00000101; 1728 let Inst{23-22} = opc; 1729 let Inst{21-18} = 0b0000; 1730 let Inst{17-5} = imms13; 1731 let Inst{4-0} = Zdn; 1732 1733 let Constraints = "$Zdn = $_Zdn"; 1734 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1735 let DestructiveInstType = DestructiveOther; 1736 let ElementSize = ElementSizeNone; 1737} 1738 1739multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> { 1740 def NAME : sve_int_log_imm<opc, asm>; 1741 1742 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>; 1743 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>; 1744 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>; 1745 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>; 1746 1747 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1748 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>; 1749 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1750 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>; 1751 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1752 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>; 1753 1754 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1755 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>; 1756 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1757 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>; 1758 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1759 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>; 1760 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1761 (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>; 1762} 1763 1764multiclass sve_int_log_imm_bic<SDPatternOperator op> { 1765 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI")>; 1766 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI")>; 1767 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI")>; 1768 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI")>; 1769} 1770 1771class sve_int_dup_mask_imm<string asm> 1772: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms), 1773 asm, "\t$Zd, $imms", 1774 "", 1775 []>, Sched<[]> { 1776 bits<5> Zd; 1777 bits<13> imms; 1778 let Inst{31-18} = 0b00000101110000; 1779 let Inst{17-5} = imms; 1780 let Inst{4-0} = Zd; 1781 1782 let isReMaterializable = 1; 1783 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1784} 1785 1786multiclass sve_int_dup_mask_imm<string asm> { 1787 def NAME : sve_int_dup_mask_imm<asm>; 1788 1789 def : InstAlias<"dupm $Zd, $imm", 1790 (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>; 1791 def : InstAlias<"dupm $Zd, $imm", 1792 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>; 1793 def : InstAlias<"dupm $Zd, $imm", 1794 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>; 1795 1796 // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here. 1797 def : InstAlias<"mov $Zd, $imm", 1798 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>; 1799 def : InstAlias<"mov $Zd, $imm", 1800 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>; 1801 def : InstAlias<"mov $Zd, $imm", 1802 (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>; 1803 1804 def : Pat<(nxv2i64 (splat_vector (i64 logical_imm64:$imm))), 1805 (!cast<Instruction>(NAME) logical_imm64:$imm)>; 1806} 1807 1808//===----------------------------------------------------------------------===// 1809// SVE Integer Arithmetic - Unpredicated Group. 1810//===----------------------------------------------------------------------===// 1811 1812class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm, 1813 ZPRRegOp zprty> 1814: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1815 asm, "\t$Zd, $Zn, $Zm", 1816 "", []>, Sched<[]> { 1817 bits<5> Zd; 1818 bits<5> Zm; 1819 bits<5> Zn; 1820 let Inst{31-24} = 0b00000100; 1821 let Inst{23-22} = sz8_64; 1822 let Inst{21} = 0b1; 1823 let Inst{20-16} = Zm; 1824 let Inst{15-13} = 0b000; 1825 let Inst{12-10} = opc; 1826 let Inst{9-5} = Zn; 1827 let Inst{4-0} = Zd; 1828} 1829 1830multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, SDPatternOperator op> { 1831 def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>; 1832 def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>; 1833 def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>; 1834 def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>; 1835 1836 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1837 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1838 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1839 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1840} 1841 1842//===----------------------------------------------------------------------===// 1843// SVE Floating Point Arithmetic - Predicated Group 1844//===----------------------------------------------------------------------===// 1845 1846class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm, 1847 ZPRRegOp zprty, 1848 Operand imm_ty> 1849: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1), 1850 asm, "\t$Zdn, $Pg/m, $_Zdn, $i1", 1851 "", 1852 []>, Sched<[]> { 1853 bits<3> Pg; 1854 bits<5> Zdn; 1855 bit i1; 1856 let Inst{31-24} = 0b01100101; 1857 let Inst{23-22} = sz; 1858 let Inst{21-19} = 0b011; 1859 let Inst{18-16} = opc; 1860 let Inst{15-13} = 0b100; 1861 let Inst{12-10} = Pg; 1862 let Inst{9-6} = 0b0000; 1863 let Inst{5} = i1; 1864 let Inst{4-0} = Zdn; 1865 1866 let Constraints = "$Zdn = $_Zdn"; 1867 let DestructiveInstType = DestructiveOther; 1868 let ElementSize = zprty.ElementSize; 1869} 1870 1871multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, string Ps, Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 1872 let DestructiveInstType = DestructiveBinaryImm in { 1873 def _H : SVEPseudo2Instr<Ps # _H, 1>, sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>; 1874 def _S : SVEPseudo2Instr<Ps # _S, 1>, sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>; 1875 def _D : SVEPseudo2Instr<Ps # _D, 1>, sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>; 1876 } 1877 1878 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_H")>; 1879 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_H")>; 1880 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_S")>; 1881 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_S")>; 1882 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_D")>; 1883 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_D")>; 1884} 1885 1886class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm, 1887 ZPRRegOp zprty> 1888: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 1889 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 1890 "", 1891 []>, Sched<[]> { 1892 bits<3> Pg; 1893 bits<5> Zdn; 1894 bits<5> Zm; 1895 let Inst{31-24} = 0b01100101; 1896 let Inst{23-22} = sz; 1897 let Inst{21-20} = 0b00; 1898 let Inst{19-16} = opc; 1899 let Inst{15-13} = 0b100; 1900 let Inst{12-10} = Pg; 1901 let Inst{9-5} = Zm; 1902 let Inst{4-0} = Zdn; 1903 1904 let Constraints = "$Zdn = $_Zdn"; 1905 let DestructiveInstType = DestructiveOther; 1906 let ElementSize = zprty.ElementSize; 1907} 1908 1909multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps, 1910 SDPatternOperator op, DestructiveInstTypeEnum flags, 1911 string revname="", bit isReverseInstr=0> { 1912 let DestructiveInstType = flags in { 1913 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>, 1914 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1915 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>, 1916 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1917 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>, 1918 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1919 } 1920 1921 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1922 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1923 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1924} 1925 1926multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm, 1927 SDPatternOperator op> { 1928 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>; 1929 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>; 1930 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>; 1931 1932 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1933 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1934 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1935} 1936 1937multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> { 1938 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 1939 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 1940 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 1941 1942 def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>; 1943 def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>; 1944 def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>; 1945} 1946 1947class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty> 1948: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, timm32_0_7:$imm3), 1949 asm, "\t$Zdn, $_Zdn, $Zm, $imm3", 1950 "", 1951 []>, Sched<[]> { 1952 bits<5> Zdn; 1953 bits<5> Zm; 1954 bits<3> imm3; 1955 let Inst{31-24} = 0b01100101; 1956 let Inst{23-22} = sz; 1957 let Inst{21-19} = 0b010; 1958 let Inst{18-16} = imm3; 1959 let Inst{15-10} = 0b100000; 1960 let Inst{9-5} = Zm; 1961 let Inst{4-0} = Zdn; 1962 1963 let Constraints = "$Zdn = $_Zdn"; 1964 let DestructiveInstType = DestructiveOther; 1965 let ElementSize = ElementSizeNone; 1966} 1967 1968multiclass sve_fp_ftmad<string asm, SDPatternOperator op> { 1969 def _H : sve_fp_ftmad<0b01, asm, ZPR16>; 1970 def _S : sve_fp_ftmad<0b10, asm, ZPR32>; 1971 def _D : sve_fp_ftmad<0b11, asm, ZPR64>; 1972 1973 def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 timm32_0_7:$imm))), 1974 (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, timm32_0_7:$imm)>; 1975 def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 timm32_0_7:$imm))), 1976 (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, timm32_0_7:$imm)>; 1977 def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 timm32_0_7:$imm))), 1978 (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, timm32_0_7:$imm)>; 1979} 1980 1981multiclass sve_fp_2op_i_p_zds_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator ir_op = null_frag> { 1982 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesUndef>; 1983 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesUndef>; 1984 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesUndef>; 1985 1986 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1987 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1988 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1989 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1990 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1991 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1992 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 1993 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 1994 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 1995 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 1996 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_UNDEF_D")>; 1997 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_UNDEF_D")>; 1998} 1999 2000multiclass sve_fp_2op_i_p_zds_zeroing_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 2001 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesZero>; 2002 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesZero>; 2003 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesZero>; 2004 2005 let AddedComplexity = 2 in { 2006 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_ZERO_H")>; 2007 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_ZERO_H")>; 2008 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_ZERO_S")>; 2009 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_ZERO_S")>; 2010 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_ZERO_D")>; 2011 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_ZERO_D")>; 2012 } 2013} 2014 2015//===----------------------------------------------------------------------===// 2016// SVE Floating Point Arithmetic - Unpredicated Group 2017//===----------------------------------------------------------------------===// 2018 2019class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 2020: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2021 asm, "\t$Zd, $Zn, $Zm", 2022 "", 2023 []>, Sched<[]> { 2024 bits<5> Zd; 2025 bits<5> Zm; 2026 bits<5> Zn; 2027 let Inst{31-24} = 0b01100101; 2028 let Inst{23-22} = sz; 2029 let Inst{21} = 0b0; 2030 let Inst{20-16} = Zm; 2031 let Inst{15-13} = 0b000; 2032 let Inst{12-10} = opc; 2033 let Inst{9-5} = Zn; 2034 let Inst{4-0} = Zd; 2035} 2036 2037multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op, 2038 SDPatternOperator predicated_op = null_frag> { 2039 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 2040 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 2041 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 2042 2043 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2044 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2045 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2046 2047 def : SVE_2_Op_Pred_All_Active<nxv8f16, predicated_op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2048 def : SVE_2_Op_Pred_All_Active<nxv4f32, predicated_op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2049 def : SVE_2_Op_Pred_All_Active<nxv2f64, predicated_op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2050} 2051 2052multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> { 2053 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 2054 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 2055 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 2056 2057 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2058 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2059 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2060} 2061 2062//===----------------------------------------------------------------------===// 2063// SVE Floating Point Fused Multiply-Add Group 2064//===----------------------------------------------------------------------===// 2065 2066class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty> 2067: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2068 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2069 "", 2070 []>, Sched<[]> { 2071 bits<3> Pg; 2072 bits<5> Zda; 2073 bits<5> Zm; 2074 bits<5> Zn; 2075 let Inst{31-24} = 0b01100101; 2076 let Inst{23-22} = sz; 2077 let Inst{21} = 0b1; 2078 let Inst{20-16} = Zm; 2079 let Inst{15} = 0b0; 2080 let Inst{14-13} = opc; 2081 let Inst{12-10} = Pg; 2082 let Inst{9-5} = Zn; 2083 let Inst{4-0} = Zda; 2084 2085 let Constraints = "$Zda = $_Zda"; 2086 let ElementSize = zprty.ElementSize; 2087} 2088 2089multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, string Ps, 2090 SDPatternOperator op, string revname, 2091 bit isReverseInstr=0> { 2092 let DestructiveInstType = DestructiveTernaryCommWithRev in { 2093 def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>, 2094 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2095 def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>, 2096 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2097 def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>, 2098 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2099 } 2100 2101 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2102 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2103 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2104} 2105 2106class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm, 2107 ZPRRegOp zprty> 2108: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2109 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2110 "", 2111 []>, Sched<[]> { 2112 bits<3> Pg; 2113 bits<5> Za; 2114 bits<5> Zdn; 2115 bits<5> Zm; 2116 let Inst{31-24} = 0b01100101; 2117 let Inst{23-22} = sz; 2118 let Inst{21} = 0b1; 2119 let Inst{20-16} = Za; 2120 let Inst{15} = 0b1; 2121 let Inst{14-13} = opc; 2122 let Inst{12-10} = Pg; 2123 let Inst{9-5} = Zm; 2124 let Inst{4-0} = Zdn; 2125 2126 let Constraints = "$Zdn = $_Zdn"; 2127 let DestructiveInstType = DestructiveOther; 2128 let ElementSize = zprty.ElementSize; 2129} 2130 2131multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op, 2132 string revname, bit isReverseInstr> { 2133 def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>, 2134 SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2135 def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>, 2136 SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2137 def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>, 2138 SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2139 2140 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2141 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2142 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2143} 2144 2145multiclass sve_fp_3op_p_zds_zx { 2146 def _UNDEF_H : PredThreeOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 2147 def _UNDEF_S : PredThreeOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 2148 def _UNDEF_D : PredThreeOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 2149} 2150 2151//===----------------------------------------------------------------------===// 2152// SVE Floating Point Multiply-Add - Indexed Group 2153//===----------------------------------------------------------------------===// 2154 2155class sve_fp_fma_by_indexed_elem<bits<2> sz, bit opc, string asm, 2156 ZPRRegOp zprty1, 2157 ZPRRegOp zprty2, Operand itype> 2158: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop), 2159 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 2160 bits<5> Zda; 2161 bits<5> Zn; 2162 let Inst{31-24} = 0b01100100; 2163 let Inst{23-22} = sz; 2164 let Inst{21} = 0b1; 2165 let Inst{15-11} = 0; 2166 let Inst{10} = opc; 2167 let Inst{9-5} = Zn; 2168 let Inst{4-0} = Zda; 2169 2170 let Constraints = "$Zda = $_Zda"; 2171 let DestructiveInstType = DestructiveOther; 2172 let ElementSize = ElementSizeNone; 2173} 2174 2175multiclass sve_fp_fma_by_indexed_elem<bit opc, string asm, 2176 SDPatternOperator op> { 2177 def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2178 bits<3> Zm; 2179 bits<3> iop; 2180 let Inst{22} = iop{2}; 2181 let Inst{20-19} = iop{1-0}; 2182 let Inst{18-16} = Zm; 2183 } 2184 def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2185 bits<3> Zm; 2186 bits<2> iop; 2187 let Inst{20-19} = iop; 2188 let Inst{18-16} = Zm; 2189 } 2190 def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2191 bits<4> Zm; 2192 bit iop; 2193 let Inst{20} = iop; 2194 let Inst{19-16} = Zm; 2195 } 2196 2197 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))), 2198 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>; 2199 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))), 2200 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 2201 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))), 2202 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 2203} 2204 2205 2206//===----------------------------------------------------------------------===// 2207// SVE Floating Point Multiply - Indexed Group 2208//===----------------------------------------------------------------------===// 2209 2210class sve_fp_fmul_by_indexed_elem<bits<2> sz, string asm, ZPRRegOp zprty, 2211 ZPRRegOp zprty2, Operand itype> 2212: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop), 2213 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 2214 bits<5> Zd; 2215 bits<5> Zn; 2216 let Inst{31-24} = 0b01100100; 2217 let Inst{23-22} = sz; 2218 let Inst{21} = 0b1; 2219 let Inst{15-10} = 0b001000; 2220 let Inst{9-5} = Zn; 2221 let Inst{4-0} = Zd; 2222} 2223 2224multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> { 2225 def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2226 bits<3> Zm; 2227 bits<3> iop; 2228 let Inst{22} = iop{2}; 2229 let Inst{20-19} = iop{1-0}; 2230 let Inst{18-16} = Zm; 2231 } 2232 def _S : sve_fp_fmul_by_indexed_elem<0b10, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2233 bits<3> Zm; 2234 bits<2> iop; 2235 let Inst{20-19} = iop; 2236 let Inst{18-16} = Zm; 2237 } 2238 def _D : sve_fp_fmul_by_indexed_elem<0b11, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2239 bits<4> Zm; 2240 bit iop; 2241 let Inst{20} = iop; 2242 let Inst{19-16} = Zm; 2243 } 2244 2245 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))), 2246 (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>; 2247 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))), 2248 (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>; 2249 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))), 2250 (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>; 2251} 2252 2253//===----------------------------------------------------------------------===// 2254// SVE Floating Point Complex Multiply-Add Group 2255//===----------------------------------------------------------------------===// 2256 2257class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty> 2258: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm, 2259 complexrotateop:$imm), 2260 asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm", 2261 "", []>, Sched<[]> { 2262 bits<5> Zda; 2263 bits<3> Pg; 2264 bits<5> Zn; 2265 bits<5> Zm; 2266 bits<2> imm; 2267 let Inst{31-24} = 0b01100100; 2268 let Inst{23-22} = sz; 2269 let Inst{21} = 0; 2270 let Inst{20-16} = Zm; 2271 let Inst{15} = 0; 2272 let Inst{14-13} = imm; 2273 let Inst{12-10} = Pg; 2274 let Inst{9-5} = Zn; 2275 let Inst{4-0} = Zda; 2276 2277 let Constraints = "$Zda = $_Zda"; 2278 let DestructiveInstType = DestructiveOther; 2279 let ElementSize = zprty.ElementSize; 2280} 2281 2282multiclass sve_fp_fcmla<string asm, SDPatternOperator op> { 2283 def _H : sve_fp_fcmla<0b01, asm, ZPR16>; 2284 def _S : sve_fp_fcmla<0b10, asm, ZPR32>; 2285 def _D : sve_fp_fcmla<0b11, asm, ZPR64>; 2286 2287 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))), 2288 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2289 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))), 2290 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2291 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))), 2292 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2293} 2294 2295//===----------------------------------------------------------------------===// 2296// SVE Floating Point Complex Multiply-Add - Indexed Group 2297//===----------------------------------------------------------------------===// 2298 2299class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm, 2300 ZPRRegOp zprty, 2301 ZPRRegOp zprty2, Operand itype> 2302: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop, 2303 complexrotateop:$imm), 2304 asm, "\t$Zda, $Zn, $Zm$iop, $imm", 2305 "", []>, Sched<[]> { 2306 bits<5> Zda; 2307 bits<5> Zn; 2308 bits<2> imm; 2309 let Inst{31-24} = 0b01100100; 2310 let Inst{23-22} = sz; 2311 let Inst{21} = 0b1; 2312 let Inst{15-12} = 0b0001; 2313 let Inst{11-10} = imm; 2314 let Inst{9-5} = Zn; 2315 let Inst{4-0} = Zda; 2316 2317 let Constraints = "$Zda = $_Zda"; 2318 let DestructiveInstType = DestructiveOther; 2319 let ElementSize = ElementSizeNone; 2320} 2321 2322multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> { 2323 def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> { 2324 bits<3> Zm; 2325 bits<2> iop; 2326 let Inst{20-19} = iop; 2327 let Inst{18-16} = Zm; 2328 } 2329 def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> { 2330 bits<4> Zm; 2331 bits<1> iop; 2332 let Inst{20} = iop; 2333 let Inst{19-16} = Zm; 2334 } 2335 2336 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2337 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2338 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2339 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2340} 2341 2342//===----------------------------------------------------------------------===// 2343// SVE Floating Point Complex Addition Group 2344//===----------------------------------------------------------------------===// 2345 2346class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty> 2347: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, 2348 complexrotateopodd:$imm), 2349 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm", 2350 "", 2351 []>, Sched<[]> { 2352 bits<5> Zdn; 2353 bits<5> Zm; 2354 bits<3> Pg; 2355 bit imm; 2356 let Inst{31-24} = 0b01100100; 2357 let Inst{23-22} = sz; 2358 let Inst{21-17} = 0; 2359 let Inst{16} = imm; 2360 let Inst{15-13} = 0b100; 2361 let Inst{12-10} = Pg; 2362 let Inst{9-5} = Zm; 2363 let Inst{4-0} = Zdn; 2364 2365 let Constraints = "$Zdn = $_Zdn"; 2366 let DestructiveInstType = DestructiveOther; 2367 let ElementSize = zprty.ElementSize; 2368} 2369 2370multiclass sve_fp_fcadd<string asm, SDPatternOperator op> { 2371 def _H : sve_fp_fcadd<0b01, asm, ZPR16>; 2372 def _S : sve_fp_fcadd<0b10, asm, ZPR32>; 2373 def _D : sve_fp_fcadd<0b11, asm, ZPR64>; 2374 2375 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))), 2376 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2377 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))), 2378 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2379 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))), 2380 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2381} 2382 2383//===----------------------------------------------------------------------===// 2384// SVE2 Floating Point Convert Group 2385//===----------------------------------------------------------------------===// 2386 2387class sve2_fp_convert_precision<bits<4> opc, string asm, 2388 ZPRRegOp zprty1, ZPRRegOp zprty2> 2389: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn), 2390 asm, "\t$Zd, $Pg/m, $Zn", 2391 "", 2392 []>, Sched<[]> { 2393 bits<5> Zd; 2394 bits<5> Zn; 2395 bits<3> Pg; 2396 let Inst{31-24} = 0b01100100; 2397 let Inst{23-22} = opc{3-2}; 2398 let Inst{21-18} = 0b0010; 2399 let Inst{17-16} = opc{1-0}; 2400 let Inst{15-13} = 0b101; 2401 let Inst{12-10} = Pg; 2402 let Inst{9-5} = Zn; 2403 let Inst{4-0} = Zd; 2404 2405 let Constraints = "$Zd = $_Zd"; 2406} 2407 2408multiclass sve2_fp_convert_down_narrow<string asm, string op> { 2409 def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>; 2410 def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>; 2411 2412 def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>; 2413 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2414} 2415 2416multiclass sve2_fp_convert_up_long<string asm, string op> { 2417 def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>; 2418 def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>; 2419 2420 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>; 2421 def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>; 2422} 2423 2424multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> { 2425 def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>; 2426 2427 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2428} 2429 2430//===----------------------------------------------------------------------===// 2431// SVE2 Floating Point Pairwise Group 2432//===----------------------------------------------------------------------===// 2433 2434class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm, 2435 ZPRRegOp zprty> 2436: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2437 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 2438 "", 2439 []>, Sched<[]> { 2440 bits<3> Pg; 2441 bits<5> Zm; 2442 bits<5> Zdn; 2443 let Inst{31-24} = 0b01100100; 2444 let Inst{23-22} = sz; 2445 let Inst{21-19} = 0b010; 2446 let Inst{18-16} = opc; 2447 let Inst{15-13} = 0b100; 2448 let Inst{12-10} = Pg; 2449 let Inst{9-5} = Zm; 2450 let Inst{4-0} = Zdn; 2451 2452 let Constraints = "$Zdn = $_Zdn"; 2453 let DestructiveInstType = DestructiveOther; 2454 let ElementSize = zprty.ElementSize; 2455} 2456 2457multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm, 2458 SDPatternOperator op> { 2459 def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>; 2460 def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>; 2461 def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>; 2462 2463 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2464 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2465 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2466} 2467 2468//===----------------------------------------------------------------------===// 2469// SVE2 Floating Point Widening Multiply-Add - Indexed Group 2470//===----------------------------------------------------------------------===// 2471 2472class sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm> 2473: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, 2474 VectorIndexH32b:$iop), 2475 asm, "\t$Zda, $Zn, $Zm$iop", 2476 "", 2477 []>, Sched<[]> { 2478 bits<5> Zda; 2479 bits<5> Zn; 2480 bits<3> Zm; 2481 bits<3> iop; 2482 let Inst{31-21} = 0b01100100101; 2483 let Inst{20-19} = iop{2-1}; 2484 let Inst{18-16} = Zm; 2485 let Inst{15-14} = 0b01; 2486 let Inst{13} = opc{1}; 2487 let Inst{12} = 0b0; 2488 let Inst{11} = iop{0}; 2489 let Inst{10} = opc{0}; 2490 let Inst{9-5} = Zn; 2491 let Inst{4-0} = Zda; 2492 2493 let Constraints = "$Zda = $_Zda"; 2494 let DestructiveInstType = DestructiveOther; 2495 let ElementSize = ElementSizeNone; 2496} 2497 2498multiclass sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm, 2499 SDPatternOperator op> { 2500 def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>; 2501 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>; 2502} 2503 2504//===----------------------------------------------------------------------===// 2505// SVE2 Floating Point Widening Multiply-Add Group 2506//===----------------------------------------------------------------------===// 2507 2508class sve2_fp_mla_long<bits<2> opc, string asm> 2509: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 2510 asm, "\t$Zda, $Zn, $Zm", 2511 "", 2512 []>, Sched<[]> { 2513 bits<5> Zda; 2514 bits<5> Zn; 2515 bits<5> Zm; 2516 let Inst{31-21} = 0b01100100101; 2517 let Inst{20-16} = Zm; 2518 let Inst{15-14} = 0b10; 2519 let Inst{13} = opc{1}; 2520 let Inst{12-11} = 0b00; 2521 let Inst{10} = opc{0}; 2522 let Inst{9-5} = Zn; 2523 let Inst{4-0} = Zda; 2524 2525 let Constraints = "$Zda = $_Zda"; 2526 let DestructiveInstType = DestructiveOther; 2527 let ElementSize = ElementSizeNone; 2528} 2529 2530multiclass sve2_fp_mla_long<bits<2> opc, string asm, SDPatternOperator op> { 2531 def NAME : sve2_fp_mla_long<opc, asm>; 2532 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 2533} 2534 2535//===----------------------------------------------------------------------===// 2536// SVE Stack Allocation Group 2537//===----------------------------------------------------------------------===// 2538 2539class sve_int_arith_vl<bit opc, string asm, bit streaming_sve = 0b0> 2540: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6), 2541 asm, "\t$Rd, $Rn, $imm6", 2542 "", 2543 []>, Sched<[]> { 2544 bits<5> Rd; 2545 bits<5> Rn; 2546 bits<6> imm6; 2547 let Inst{31-23} = 0b000001000; 2548 let Inst{22} = opc; 2549 let Inst{21} = 0b1; 2550 let Inst{20-16} = Rn; 2551 let Inst{15-12} = 0b0101; 2552 let Inst{11} = streaming_sve; 2553 let Inst{10-5} = imm6; 2554 let Inst{4-0} = Rd; 2555} 2556 2557class sve_int_read_vl_a<bit op, bits<5> opc2, string asm, bit streaming_sve = 0b0> 2558: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6), 2559 asm, "\t$Rd, $imm6", 2560 "", 2561 []>, Sched<[]> { 2562 bits<5> Rd; 2563 bits<6> imm6; 2564 let Inst{31-23} = 0b000001001; 2565 let Inst{22} = op; 2566 let Inst{21} = 0b1; 2567 let Inst{20-16} = opc2{4-0}; 2568 let Inst{15-12} = 0b0101; 2569 let Inst{11} = streaming_sve; 2570 let Inst{10-5} = imm6; 2571 let Inst{4-0} = Rd; 2572 2573 let isReMaterializable = 1; 2574} 2575 2576//===----------------------------------------------------------------------===// 2577// SVE Permute - In Lane Group 2578//===----------------------------------------------------------------------===// 2579 2580class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm, 2581 ZPRRegOp zprty> 2582: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2583 asm, "\t$Zd, $Zn, $Zm", 2584 "", 2585 []>, Sched<[]> { 2586 bits<5> Zd; 2587 bits<5> Zm; 2588 bits<5> Zn; 2589 let Inst{31-24} = 0b00000101; 2590 let Inst{23-22} = sz8_64; 2591 let Inst{21} = 0b1; 2592 let Inst{20-16} = Zm; 2593 let Inst{15-13} = 0b011; 2594 let Inst{12-10} = opc; 2595 let Inst{9-5} = Zn; 2596 let Inst{4-0} = Zd; 2597} 2598 2599multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm, 2600 SDPatternOperator op> { 2601 def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>; 2602 def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>; 2603 def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>; 2604 def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>; 2605 2606 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2607 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2608 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2609 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2610 2611 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2612 def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 2613 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2614 def : SVE_2_Op_Pat<nxv2f16, op, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 2615 def : SVE_2_Op_Pat<nxv2f32, op, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 2616 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2617 2618 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 2619} 2620 2621//===----------------------------------------------------------------------===// 2622// SVE Floating Point Unary Operations Group 2623//===----------------------------------------------------------------------===// 2624 2625class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype, 2626 RegisterOperand o_zprtype, ElementSizeEnum Sz> 2627: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn), 2628 asm, "\t$Zd, $Pg/m, $Zn", 2629 "", 2630 []>, Sched<[]> { 2631 bits<3> Pg; 2632 bits<5> Zd; 2633 bits<5> Zn; 2634 let Inst{31-24} = 0b01100101; 2635 let Inst{23-22} = opc{6-5}; 2636 let Inst{21} = 0b0; 2637 let Inst{20-16} = opc{4-0}; 2638 let Inst{15-13} = 0b101; 2639 let Inst{12-10} = Pg; 2640 let Inst{9-5} = Zn; 2641 let Inst{4-0} = Zd; 2642 2643 let Constraints = "$Zd = $_Zd"; 2644 let DestructiveInstType = DestructiveUnaryPassthru; 2645 let ElementSize = Sz; 2646} 2647 2648multiclass sve_fp_2op_p_zd<bits<7> opc, string asm, 2649 RegisterOperand i_zprtype, 2650 RegisterOperand o_zprtype, 2651 SDPatternOperator int_op, 2652 SDPatternOperator ir_op, ValueType vt1, 2653 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2654 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>, 2655 SVEPseudo2Instr<NAME, 1>; 2656 // convert vt1 to a packed type for the intrinsic patterns 2657 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2658 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2659 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2660 1 : vt1); 2661 2662 // convert vt3 to a packed type for the intrinsic patterns 2663 defvar packedvt3 = !cond(!eq(!cast<string>(vt3), "nxv2f16"): nxv8f16, 2664 !eq(!cast<string>(vt3), "nxv4f16"): nxv8f16, 2665 !eq(!cast<string>(vt3), "nxv2f32"): nxv4f32, 2666 1 : vt3); 2667 2668 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, packedvt3, !cast<Instruction>(NAME)>; 2669 def : SVE_1_Op_Passthru_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2670 2671 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>; 2672 2673 defm : SVE_1_Op_PassthruUndef_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>; 2674} 2675 2676multiclass sve_fp_2op_p_zdr<bits<7> opc, string asm, 2677 RegisterOperand i_zprtype, 2678 RegisterOperand o_zprtype, 2679 SDPatternOperator int_op, 2680 SDPatternOperator ir_op, ValueType vt1, 2681 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2682 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>, 2683 SVEPseudo2Instr<NAME, 1>; 2684 2685 // convert vt1 to a packed type for the intrinsic patterns 2686 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2687 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2688 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2689 1 : vt1); 2690 2691 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, vt3, !cast<Instruction>(NAME)>; 2692 def : SVE_1_Op_Passthru_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2693 2694 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>; 2695 2696 defm : SVE_1_Op_PassthruUndef_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>; 2697} 2698 2699multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> { 2700 def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>, 2701 SVEPseudo2Instr<NAME # _H, 1>; 2702 def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>, 2703 SVEPseudo2Instr<NAME # _S, 1>; 2704 def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>, 2705 SVEPseudo2Instr<NAME # _D, 1>; 2706 2707 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2708 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 2709 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 2710 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2711 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 2712 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2713 2714 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 2715 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 2716 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 2717 2718 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2719 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2720 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2721 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2722 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2723 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _UNDEF_D)>; 2724} 2725 2726multiclass sve2_fp_flogb<string asm, SDPatternOperator op> { 2727 def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>; 2728 def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>; 2729 def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>; 2730 2731 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2732 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2733 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2734} 2735 2736multiclass sve2_fp_convert_down_odd_rounding<string asm, string op> { 2737 def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>; 2738 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2739} 2740 2741//===----------------------------------------------------------------------===// 2742// SVE Floating Point Unary Operations - Unpredicated Group 2743//===----------------------------------------------------------------------===// 2744 2745class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm, 2746 ZPRRegOp zprty> 2747: I<(outs zprty:$Zd), (ins zprty:$Zn), 2748 asm, "\t$Zd, $Zn", 2749 "", 2750 []>, Sched<[]> { 2751 bits<5> Zd; 2752 bits<5> Zn; 2753 let Inst{31-24} = 0b01100101; 2754 let Inst{23-22} = sz; 2755 let Inst{21-19} = 0b001; 2756 let Inst{18-16} = opc; 2757 let Inst{15-10} = 0b001100; 2758 let Inst{9-5} = Zn; 2759 let Inst{4-0} = Zd; 2760} 2761 2762multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> { 2763 def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>; 2764 def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>; 2765 def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>; 2766 2767 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 2768 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 2769 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 2770} 2771 2772//===----------------------------------------------------------------------===// 2773// SVE Integer Arithmetic - Binary Predicated Group 2774//===----------------------------------------------------------------------===// 2775 2776class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc, 2777 string asm, ZPRRegOp zprty> 2778: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2779 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 2780 bits<3> Pg; 2781 bits<5> Zdn; 2782 bits<5> Zm; 2783 let Inst{31-24} = 0b00000100; 2784 let Inst{23-22} = sz8_64; 2785 let Inst{21} = 0b0; 2786 let Inst{20-19} = fmt; 2787 let Inst{18-16} = opc; 2788 let Inst{15-13} = 0b000; 2789 let Inst{12-10} = Pg; 2790 let Inst{9-5} = Zm; 2791 let Inst{4-0} = Zdn; 2792 2793 let Constraints = "$Zdn = $_Zdn"; 2794 let DestructiveInstType = DestructiveOther; 2795 let ElementSize = zprty.ElementSize; 2796} 2797 2798multiclass sve_int_bin_pred_log<bits<3> opc, string asm, string Ps, 2799 SDPatternOperator op, 2800 DestructiveInstTypeEnum flags> { 2801 let DestructiveInstType = flags in { 2802 def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>, 2803 SVEPseudo2Instr<Ps # _B, 1>; 2804 def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>, 2805 SVEPseudo2Instr<Ps # _H, 1>; 2806 def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>, 2807 SVEPseudo2Instr<Ps # _S, 1>; 2808 def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>, 2809 SVEPseudo2Instr<Ps # _D, 1>; 2810 } 2811 2812 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2813 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2814 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2815 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2816} 2817 2818multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps, 2819 SDPatternOperator op, 2820 DestructiveInstTypeEnum flags, 2821 string revname="", bit isReverseInstr=0> { 2822 let DestructiveInstType = flags in { 2823 def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>, 2824 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 2825 def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>, 2826 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2827 def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>, 2828 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2829 def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>, 2830 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2831 } 2832 2833 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2834 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2835 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2836 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2837} 2838 2839multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps, 2840 SDPatternOperator op, 2841 DestructiveInstTypeEnum flags> { 2842 let DestructiveInstType = flags in { 2843 def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>, 2844 SVEPseudo2Instr<Ps # _B, 1>; 2845 def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>, 2846 SVEPseudo2Instr<Ps # _H, 1>; 2847 def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>, 2848 SVEPseudo2Instr<Ps # _S, 1>; 2849 def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>, 2850 SVEPseudo2Instr<Ps # _D, 1>; 2851 } 2852 2853 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2854 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2855 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2856 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2857} 2858 2859multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps, 2860 SDPatternOperator op, 2861 DestructiveInstTypeEnum flags> { 2862 let DestructiveInstType = flags in { 2863 def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>, 2864 SVEPseudo2Instr<Ps # _B, 1>; 2865 def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>, 2866 SVEPseudo2Instr<Ps # _H, 1>; 2867 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2868 SVEPseudo2Instr<Ps # _S, 1>; 2869 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2870 SVEPseudo2Instr<Ps # _D, 1>; 2871 } 2872 2873 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2874 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2875 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2876 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2877} 2878 2879// Special case for divides which are not defined for 8b/16b elements. 2880multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps, 2881 SDPatternOperator op, 2882 DestructiveInstTypeEnum flags, 2883 string revname="", bit isReverseInstr=0> { 2884 let DestructiveInstType = flags in { 2885 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2886 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2887 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2888 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2889 } 2890 2891 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2892 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2893} 2894 2895//===----------------------------------------------------------------------===// 2896// SVE Integer Multiply-Add Group 2897//===----------------------------------------------------------------------===// 2898 2899class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2900 ZPRRegOp zprty> 2901: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2902 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2903 "", 2904 []>, Sched<[]> { 2905 bits<3> Pg; 2906 bits<5> Zdn; 2907 bits<5> Za; 2908 bits<5> Zm; 2909 let Inst{31-24} = 0b00000100; 2910 let Inst{23-22} = sz8_64; 2911 let Inst{21} = 0b0; 2912 let Inst{20-16} = Zm; 2913 let Inst{15-14} = 0b11; 2914 let Inst{13} = opc; 2915 let Inst{12-10} = Pg; 2916 let Inst{9-5} = Za; 2917 let Inst{4-0} = Zdn; 2918 2919 let Constraints = "$Zdn = $_Zdn"; 2920 let DestructiveInstType = DestructiveOther; 2921 let ElementSize = zprty.ElementSize; 2922} 2923 2924multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2925 def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>; 2926 def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>; 2927 def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>; 2928 def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>; 2929 2930 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2931 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2932 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2933 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2934} 2935 2936class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2937 ZPRRegOp zprty> 2938: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2939 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2940 "", 2941 []>, Sched<[]> { 2942 bits<3> Pg; 2943 bits<5> Zda; 2944 bits<5> Zm; 2945 bits<5> Zn; 2946 let Inst{31-24} = 0b00000100; 2947 let Inst{23-22} = sz8_64; 2948 let Inst{21} = 0b0; 2949 let Inst{20-16} = Zm; 2950 let Inst{15-14} = 0b01; 2951 let Inst{13} = opc; 2952 let Inst{12-10} = Pg; 2953 let Inst{9-5} = Zn; 2954 let Inst{4-0} = Zda; 2955 2956 let Constraints = "$Zda = $_Zda"; 2957 let DestructiveInstType = DestructiveOther; 2958 let ElementSize = zprty.ElementSize; 2959} 2960 2961multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op, 2962 SDPatternOperator outerop, SDPatternOperator mulop> { 2963 def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>; 2964 def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>; 2965 def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>; 2966 def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>; 2967 2968 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2969 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2970 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2971 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2972 2973 def : Pat<(outerop nxv16i8:$Op1, (mulop nxv16i1:$pred, nxv16i8:$Op2, nxv16i8:$Op3)), 2974 (!cast<Instruction>(NAME # _B) $pred, $Op1, $Op2, $Op3)>; 2975 def : Pat<(outerop nxv8i16:$Op1, (mulop nxv8i1:$pred, nxv8i16:$Op2, nxv8i16:$Op3)), 2976 (!cast<Instruction>(NAME # _H) $pred, $Op1, $Op2, $Op3)>; 2977 def : Pat<(outerop nxv4i32:$Op1, (mulop nxv4i1:$pred, nxv4i32:$Op2, nxv4i32:$Op3)), 2978 (!cast<Instruction>(NAME # _S) $pred, $Op1, $Op2, $Op3)>; 2979 def : Pat<(outerop nxv2i64:$Op1, (mulop nxv2i1:$pred, nxv2i64:$Op2, nxv2i64:$Op3)), 2980 (!cast<Instruction>(NAME # _D) $pred, $Op1, $Op2, $Op3)>; 2981} 2982 2983//===----------------------------------------------------------------------===// 2984// SVE2 Integer Multiply-Add - Unpredicated Group 2985//===----------------------------------------------------------------------===// 2986 2987class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, 2988 ZPRRegOp zprty1, ZPRRegOp zprty2> 2989: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 2990 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2991 bits<5> Zda; 2992 bits<5> Zn; 2993 bits<5> Zm; 2994 let Inst{31-24} = 0b01000100; 2995 let Inst{23-22} = sz; 2996 let Inst{21} = 0b0; 2997 let Inst{20-16} = Zm; 2998 let Inst{15} = 0b0; 2999 let Inst{14-10} = opc; 3000 let Inst{9-5} = Zn; 3001 let Inst{4-0} = Zda; 3002 3003 let Constraints = "$Zda = $_Zda"; 3004 let DestructiveInstType = DestructiveOther; 3005 let ElementSize = ElementSizeNone; 3006} 3007 3008multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> { 3009 def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>; 3010 def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>; 3011 def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>; 3012 def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>; 3013 3014 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3015 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3016 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3017 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3018} 3019 3020multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> { 3021 def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>; 3022 def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>; 3023 def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>; 3024 3025 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3026 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3027 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3028} 3029 3030//===----------------------------------------------------------------------===// 3031// SVE2 Integer Multiply-Add - Indexed Group 3032//===----------------------------------------------------------------------===// 3033 3034class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, 3035 ZPRRegOp zprty1, ZPRRegOp zprty2, 3036 ZPRRegOp zprty3, Operand itype> 3037: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 3038 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 3039 bits<5> Zda; 3040 bits<5> Zn; 3041 let Inst{31-24} = 0b01000100; 3042 let Inst{23-22} = sz; 3043 let Inst{21} = 0b1; 3044 let Inst{15-10} = opc; 3045 let Inst{9-5} = Zn; 3046 let Inst{4-0} = Zda; 3047 3048 let Constraints = "$Zda = $_Zda"; 3049 let DestructiveInstType = DestructiveOther; 3050 let ElementSize = ElementSizeNone; 3051} 3052 3053multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm, 3054 SDPatternOperator op> { 3055 def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3056 bits<3> Zm; 3057 bits<3> iop; 3058 let Inst{22} = iop{2}; 3059 let Inst{20-19} = iop{1-0}; 3060 let Inst{18-16} = Zm; 3061 } 3062 def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3063 bits<3> Zm; 3064 bits<2> iop; 3065 let Inst{20-19} = iop; 3066 let Inst{18-16} = Zm; 3067 } 3068 def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3069 bits<4> Zm; 3070 bit iop; 3071 let Inst{20} = iop; 3072 let Inst{19-16} = Zm; 3073 } 3074 3075 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3076 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3077 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3078} 3079 3080//===----------------------------------------------------------------------===// 3081// SVE2 Integer Multiply-Add Long - Indexed Group 3082//===----------------------------------------------------------------------===// 3083 3084multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm, 3085 SDPatternOperator op> { 3086 def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 3087 asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3088 bits<3> Zm; 3089 bits<3> iop; 3090 let Inst{20-19} = iop{2-1}; 3091 let Inst{18-16} = Zm; 3092 let Inst{11} = iop{0}; 3093 } 3094 def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 3095 asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3096 bits<4> Zm; 3097 bits<2> iop; 3098 let Inst{20} = iop{1}; 3099 let Inst{19-16} = Zm; 3100 let Inst{11} = iop{0}; 3101 } 3102 3103 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3104 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3105} 3106 3107//===----------------------------------------------------------------------===// 3108// SVE Integer Dot Product Group 3109//===----------------------------------------------------------------------===// 3110 3111class sve_intx_dot<bit sz, bit U, string asm, ZPRRegOp zprty1, 3112 ZPRRegOp zprty2> 3113: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm, 3114 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3115 bits<5> Zda; 3116 bits<5> Zn; 3117 bits<5> Zm; 3118 let Inst{31-23} = 0b010001001; 3119 let Inst{22} = sz; 3120 let Inst{21} = 0; 3121 let Inst{20-16} = Zm; 3122 let Inst{15-11} = 0; 3123 let Inst{10} = U; 3124 let Inst{9-5} = Zn; 3125 let Inst{4-0} = Zda; 3126 3127 let Constraints = "$Zda = $_Zda"; 3128 let DestructiveInstType = DestructiveOther; 3129} 3130 3131multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> { 3132 def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>; 3133 def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>; 3134 3135 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>; 3136 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>; 3137} 3138 3139//===----------------------------------------------------------------------===// 3140// SVE Integer Dot Product Group - Indexed Group 3141//===----------------------------------------------------------------------===// 3142 3143class sve_intx_dot_by_indexed_elem<bit sz, bit U, string asm, 3144 ZPRRegOp zprty1, ZPRRegOp zprty2, 3145 ZPRRegOp zprty3, Operand itype> 3146: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 3147 asm, "\t$Zda, $Zn, $Zm$iop", 3148 "", []>, Sched<[]> { 3149 bits<5> Zda; 3150 bits<5> Zn; 3151 let Inst{31-23} = 0b010001001; 3152 let Inst{22} = sz; 3153 let Inst{21} = 0b1; 3154 let Inst{15-11} = 0; 3155 let Inst{10} = U; 3156 let Inst{9-5} = Zn; 3157 let Inst{4-0} = Zda; 3158 3159 let Constraints = "$Zda = $_Zda"; 3160 let DestructiveInstType = DestructiveOther; 3161} 3162 3163multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm, 3164 SDPatternOperator op> { 3165 def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> { 3166 bits<2> iop; 3167 bits<3> Zm; 3168 let Inst{20-19} = iop; 3169 let Inst{18-16} = Zm; 3170 } 3171 def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> { 3172 bits<1> iop; 3173 bits<4> Zm; 3174 let Inst{20} = iop; 3175 let Inst{19-16} = Zm; 3176 } 3177 3178 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3179 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3180} 3181 3182//===----------------------------------------------------------------------===// 3183// SVE2 Complex Integer Dot Product Group 3184//===----------------------------------------------------------------------===// 3185 3186class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm, 3187 ZPRRegOp zprty1, ZPRRegOp zprty2> 3188: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm, 3189 complexrotateop:$rot), 3190 asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> { 3191 bits<5> Zda; 3192 bits<5> Zn; 3193 bits<5> Zm; 3194 bits<2> rot; 3195 let Inst{31-24} = 0b01000100; 3196 let Inst{23-22} = sz; 3197 let Inst{21} = 0b0; 3198 let Inst{20-16} = Zm; 3199 let Inst{15-12} = opc; 3200 let Inst{11-10} = rot; 3201 let Inst{9-5} = Zn; 3202 let Inst{4-0} = Zda; 3203 3204 let Constraints = "$Zda = $_Zda"; 3205 let DestructiveInstType = DestructiveOther; 3206 let ElementSize = ElementSizeNone; 3207} 3208 3209multiclass sve2_cintx_dot<string asm, SDPatternOperator op> { 3210 def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>; 3211 def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>; 3212 3213 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3214 (i32 complexrotateop:$imm))), 3215 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>; 3216 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3217 (i32 complexrotateop:$imm))), 3218 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>; 3219} 3220 3221//===----------------------------------------------------------------------===// 3222// SVE2 Complex Multiply-Add Group 3223//===----------------------------------------------------------------------===// 3224 3225multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> { 3226 def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>; 3227 def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>; 3228 def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>; 3229 def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>; 3230 3231 def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>; 3232 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>; 3233 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>; 3234 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>; 3235} 3236 3237//===----------------------------------------------------------------------===// 3238// SVE2 Complex Integer Dot Product - Indexed Group 3239//===----------------------------------------------------------------------===// 3240 3241class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm, 3242 ZPRRegOp zprty1, ZPRRegOp zprty2, 3243 ZPRRegOp zprty3, Operand itype> 3244: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop, 3245 complexrotateop:$rot), 3246 asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> { 3247 bits<5> Zda; 3248 bits<5> Zn; 3249 bits<2> rot; 3250 let Inst{31-24} = 0b01000100; 3251 let Inst{23-22} = sz; 3252 let Inst{21} = 0b1; 3253 let Inst{15-12} = opc; 3254 let Inst{11-10} = rot; 3255 let Inst{9-5} = Zn; 3256 let Inst{4-0} = Zda; 3257 3258 let Constraints = "$Zda = $_Zda"; 3259 let DestructiveInstType = DestructiveOther; 3260 let ElementSize = ElementSizeNone; 3261} 3262 3263multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> { 3264 def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> { 3265 bits<2> iop; 3266 bits<3> Zm; 3267 let Inst{20-19} = iop; 3268 let Inst{18-16} = Zm; 3269 } 3270 def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> { 3271 bit iop; 3272 bits<4> Zm; 3273 let Inst{20} = iop; 3274 let Inst{19-16} = Zm; 3275 } 3276 3277 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3278 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3279 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3280 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3281 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3282 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3283} 3284 3285//===----------------------------------------------------------------------===// 3286// SVE2 Complex Multiply-Add - Indexed Group 3287//===----------------------------------------------------------------------===// 3288 3289multiclass sve2_cmla_by_indexed_elem<bit opc, string asm, 3290 SDPatternOperator op> { 3291 def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> { 3292 bits<2> iop; 3293 bits<3> Zm; 3294 let Inst{20-19} = iop; 3295 let Inst{18-16} = Zm; 3296 } 3297 def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> { 3298 bit iop; 3299 bits<4> Zm; 3300 let Inst{20} = iop; 3301 let Inst{19-16} = Zm; 3302 } 3303 3304 def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3305 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3306 (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3307 3308 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3), 3309 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3310 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3311} 3312 3313//===----------------------------------------------------------------------===// 3314// SVE2 Integer Multiply - Unpredicated Group 3315//===----------------------------------------------------------------------===// 3316 3317class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 3318: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 3319 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3320 bits<5> Zd; 3321 bits<5> Zm; 3322 bits<5> Zn; 3323 let Inst{31-24} = 0b00000100; 3324 let Inst{23-22} = sz; 3325 let Inst{21} = 0b1; 3326 let Inst{20-16} = Zm; 3327 let Inst{15-13} = 0b011; 3328 let Inst{12-10} = opc; 3329 let Inst{9-5} = Zn; 3330 let Inst{4-0} = Zd; 3331} 3332 3333multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op, 3334 SDPatternOperator op_pred = null_frag> { 3335 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3336 def _H : sve2_int_mul<0b01, opc, asm, ZPR16>; 3337 def _S : sve2_int_mul<0b10, opc, asm, ZPR32>; 3338 def _D : sve2_int_mul<0b11, opc, asm, ZPR64>; 3339 3340 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3341 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3342 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3343 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3344 3345 def : SVE_2_Op_Pred_All_Active<nxv16i8, op_pred, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3346 def : SVE_2_Op_Pred_All_Active<nxv8i16, op_pred, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3347 def : SVE_2_Op_Pred_All_Active<nxv4i32, op_pred, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3348 def : SVE_2_Op_Pred_All_Active<nxv2i64, op_pred, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3349} 3350 3351multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> { 3352 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3353 3354 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3355} 3356 3357//===----------------------------------------------------------------------===// 3358// SVE2 Integer Multiply - Indexed Group 3359//===----------------------------------------------------------------------===// 3360 3361class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, 3362 ZPRRegOp zprty1, ZPRRegOp zprty2, 3363 ZPRRegOp zprty3, Operand itype> 3364: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop), 3365 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 3366 bits<5> Zd; 3367 bits<5> Zn; 3368 let Inst{31-24} = 0b01000100; 3369 let Inst{23-22} = sz; 3370 let Inst{21} = 0b1; 3371 let Inst{15-14} = 0b11; 3372 let Inst{13-10} = opc; 3373 let Inst{9-5} = Zn; 3374 let Inst{4-0} = Zd; 3375} 3376 3377multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm, 3378 SDPatternOperator op> { 3379 def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3380 bits<3> Zm; 3381 bits<3> iop; 3382 let Inst{22} = iop{2}; 3383 let Inst{20-19} = iop{1-0}; 3384 let Inst{18-16} = Zm; 3385 } 3386 def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3387 bits<3> Zm; 3388 bits<2> iop; 3389 let Inst{20-19} = iop; 3390 let Inst{18-16} = Zm; 3391 } 3392 def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3393 bits<4> Zm; 3394 bit iop; 3395 let Inst{20} = iop; 3396 let Inst{19-16} = Zm; 3397 } 3398 3399 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3400 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3401 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3402} 3403 3404multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm, 3405 SDPatternOperator op> { 3406 def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm, 3407 ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3408 bits<3> Zm; 3409 bits<3> iop; 3410 let Inst{20-19} = iop{2-1}; 3411 let Inst{18-16} = Zm; 3412 let Inst{11} = iop{0}; 3413 } 3414 def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm, 3415 ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3416 bits<4> Zm; 3417 bits<2> iop; 3418 let Inst{20} = iop{1}; 3419 let Inst{19-16} = Zm; 3420 let Inst{11} = iop{0}; 3421 } 3422 3423 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3424 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3425} 3426 3427//===----------------------------------------------------------------------===// 3428// SVE2 Integer - Predicated Group 3429//===----------------------------------------------------------------------===// 3430 3431class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm, 3432 ZPRRegOp zprty> 3433: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 3434 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 3435 bits<3> Pg; 3436 bits<5> Zm; 3437 bits<5> Zdn; 3438 let Inst{31-24} = 0b01000100; 3439 let Inst{23-22} = sz; 3440 let Inst{21-20} = 0b01; 3441 let Inst{20-16} = opc{5-1}; 3442 let Inst{15-14} = 0b10; 3443 let Inst{13} = opc{0}; 3444 let Inst{12-10} = Pg; 3445 let Inst{9-5} = Zm; 3446 let Inst{4-0} = Zdn; 3447 3448 let Constraints = "$Zdn = $_Zdn"; 3449 let DestructiveInstType = DestructiveOther; 3450 let ElementSize = zprty.ElementSize; 3451} 3452 3453multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op, 3454 string Ps = "", 3455 DestructiveInstTypeEnum flags=DestructiveOther, 3456 string revname="", bit isReverseInstr=0> { 3457 let DestructiveInstType = flags in { 3458 def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>, 3459 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 3460 def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>, 3461 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 3462 def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>, 3463 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 3464 def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>, 3465 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 3466 } 3467 3468 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3469 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3470 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3471 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3472} 3473 3474class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm, 3475 ZPRRegOp zprty1, ZPRRegOp zprty2> 3476: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn), 3477 asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> { 3478 bits<3> Pg; 3479 bits<5> Zn; 3480 bits<5> Zda; 3481 let Inst{31-24} = 0b01000100; 3482 let Inst{23-22} = sz; 3483 let Inst{21-17} = 0b00010; 3484 let Inst{16} = U; 3485 let Inst{15-13} = 0b101; 3486 let Inst{12-10} = Pg; 3487 let Inst{9-5} = Zn; 3488 let Inst{4-0} = Zda; 3489 3490 let Constraints = "$Zda = $_Zda"; 3491 let DestructiveInstType = DestructiveOther; 3492 let ElementSize = zprty1.ElementSize; 3493} 3494 3495multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> { 3496 def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>; 3497 def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>; 3498 def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>; 3499 3500 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3501 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3502 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3503} 3504 3505class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, 3506 string asm, ZPRRegOp zprty> 3507: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3508 asm, "\t$Zd, $Pg/m, $Zn", 3509 "", 3510 []>, Sched<[]> { 3511 bits<3> Pg; 3512 bits<5> Zd; 3513 bits<5> Zn; 3514 let Inst{31-24} = 0b01000100; 3515 let Inst{23-22} = sz; 3516 let Inst{21-20} = 0b00; 3517 let Inst{19} = Q; 3518 let Inst{18} = 0b0; 3519 let Inst{17-16} = opc; 3520 let Inst{15-13} = 0b101; 3521 let Inst{12-10} = Pg; 3522 let Inst{9-5} = Zn; 3523 let Inst{4-0} = Zd; 3524 3525 let Constraints = "$Zd = $_Zd"; 3526 let DestructiveInstType = DestructiveUnaryPassthru; 3527 let ElementSize = zprty.ElementSize; 3528} 3529 3530multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm, 3531 SDPatternOperator op> { 3532 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3533 SVEPseudo2Instr<NAME # _S, 1>; 3534 3535 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3536 3537 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3538 3539 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3540} 3541 3542multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> { 3543 def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>, 3544 SVEPseudo2Instr<NAME # _B, 1>; 3545 def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>, 3546 SVEPseudo2Instr<NAME # _H, 1>; 3547 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3548 SVEPseudo2Instr<NAME # _S, 1>; 3549 def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>, 3550 SVEPseudo2Instr<NAME # _D, 1>; 3551 3552 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3553 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3554 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3555 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3556 3557 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 3558 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 3559 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3560 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 3561 3562 defm : SVE_3_Op_Undef_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 3563 defm : SVE_3_Op_Undef_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 3564 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3565 defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 3566} 3567 3568//===----------------------------------------------------------------------===// 3569// SVE2 Widening Integer Arithmetic Group 3570//===----------------------------------------------------------------------===// 3571 3572class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, 3573 ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3> 3574: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm), 3575 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3576 bits<5> Zd; 3577 bits<5> Zn; 3578 bits<5> Zm; 3579 let Inst{31-24} = 0b01000101; 3580 let Inst{23-22} = sz; 3581 let Inst{21} = 0b0; 3582 let Inst{20-16} = Zm; 3583 let Inst{15} = 0b0; 3584 let Inst{14-10} = opc; 3585 let Inst{9-5} = Zn; 3586 let Inst{4-0} = Zd; 3587} 3588 3589multiclass sve2_wide_int_arith_long<bits<5> opc, string asm, 3590 SDPatternOperator op> { 3591 def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>; 3592 def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>; 3593 def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>; 3594 3595 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3596 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3597 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3598} 3599 3600multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm, 3601 SDPatternOperator op> { 3602 def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>; 3603 def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>; 3604 def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>; 3605 3606 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3607 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3608 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3609} 3610 3611multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm, 3612 SDPatternOperator op> { 3613 def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>; 3614 3615 // To avoid using 128 bit elements in the IR, the pattern below works with 3616 // llvm intrinsics with the _pair suffix, to reflect that 3617 // _Q is implemented as a pair of _D. 3618 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3619} 3620 3621multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> { 3622 def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>; 3623 def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>; 3624 3625 // To avoid using 128 bit elements in the IR, the patterns below work with 3626 // llvm intrinsics with the _pair suffix, to reflect that 3627 // _H is implemented as a pair of _B and _D is implemented as a pair of _S. 3628 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3629 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3630} 3631 3632//===----------------------------------------------------------------------===// 3633// SVE2 Misc Group 3634//===----------------------------------------------------------------------===// 3635 3636class sve2_misc<bits<2> sz, bits<4> opc, string asm, 3637 ZPRRegOp zprty1, ZPRRegOp zprty2> 3638: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3639 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3640 bits<5> Zd; 3641 bits<5> Zn; 3642 bits<5> Zm; 3643 let Inst{31-24} = 0b01000101; 3644 let Inst{23-22} = sz; 3645 let Inst{21} = 0b0; 3646 let Inst{20-16} = Zm; 3647 let Inst{15-14} = 0b10; 3648 let Inst{13-10} = opc; 3649 let Inst{9-5} = Zn; 3650 let Inst{4-0} = Zd; 3651} 3652 3653multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> { 3654 def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; 3655 def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; 3656 def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; 3657 def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; 3658 3659 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3660 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3661 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3662 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3663} 3664 3665multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm, 3666 SDPatternOperator op> { 3667 def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3668 def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3669 def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3670 3671 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3672 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3673 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3674} 3675 3676class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm, 3677 ZPRRegOp zprty1, ZPRRegOp zprty2> 3678: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3679 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3680 bits<5> Zd; 3681 bits<5> Zn; 3682 bits<5> Zm; 3683 let Inst{31-24} = 0b01000101; 3684 let Inst{23-22} = sz; 3685 let Inst{21} = 0b0; 3686 let Inst{20-16} = Zm; 3687 let Inst{15-11} = 0b10010; 3688 let Inst{10} = opc; 3689 let Inst{9-5} = Zn; 3690 let Inst{4-0} = Zd; 3691 3692 let Constraints = "$Zd = $_Zd"; 3693 let DestructiveInstType = DestructiveOther; 3694 let ElementSize = ElementSizeNone; 3695} 3696 3697multiclass sve2_bitwise_xor_interleaved<bit opc, string asm, 3698 SDPatternOperator op> { 3699 def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8, ZPR8>; 3700 def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>; 3701 def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>; 3702 def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>; 3703 3704 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3705 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3706 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3707 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3708} 3709 3710class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm, 3711 ZPRRegOp zprty1, ZPRRegOp zprty2, 3712 Operand immtype> 3713: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3714 asm, "\t$Zd, $Zn, $imm", 3715 "", []>, Sched<[]> { 3716 bits<5> Zd; 3717 bits<5> Zn; 3718 bits<5> imm; 3719 let Inst{31-23} = 0b010001010; 3720 let Inst{22} = tsz8_64{2}; 3721 let Inst{21} = 0b0; 3722 let Inst{20-19} = tsz8_64{1-0}; 3723 let Inst{18-16} = imm{2-0}; // imm3 3724 let Inst{15-12} = 0b1010; 3725 let Inst{11-10} = opc; 3726 let Inst{9-5} = Zn; 3727 let Inst{4-0} = Zd; 3728} 3729 3730multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm, 3731 SDPatternOperator op> { 3732 def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm, 3733 ZPR16, ZPR8, vecshiftL8>; 3734 def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm, 3735 ZPR32, ZPR16, vecshiftL16> { 3736 let Inst{19} = imm{3}; 3737 } 3738 def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm, 3739 ZPR64, ZPR32, vecshiftL32> { 3740 let Inst{20-19} = imm{4-3}; 3741 } 3742 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _H)>; 3743 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _S)>; 3744 def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _D)>; 3745} 3746 3747//===----------------------------------------------------------------------===// 3748// SVE2 Accumulate Group 3749//===----------------------------------------------------------------------===// 3750 3751class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm, 3752 ZPRRegOp zprty, Operand immtype> 3753: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm), 3754 asm, "\t$Zd, $Zn, $imm", 3755 "", []>, Sched<[]> { 3756 bits<5> Zd; 3757 bits<5> Zn; 3758 bits<6> imm; 3759 let Inst{31-24} = 0b01000101; 3760 let Inst{23-22} = tsz8_64{3-2}; 3761 let Inst{21} = 0b0; 3762 let Inst{20-19} = tsz8_64{1-0}; 3763 let Inst{18-16} = imm{2-0}; // imm3 3764 let Inst{15-11} = 0b11110; 3765 let Inst{10} = opc; 3766 let Inst{9-5} = Zn; 3767 let Inst{4-0} = Zd; 3768 3769 let Constraints = "$Zd = $_Zd"; 3770} 3771 3772multiclass sve2_int_bin_shift_imm_left<bit opc, string asm, 3773 SDPatternOperator op> { 3774 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 3775 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 3776 let Inst{19} = imm{3}; 3777 } 3778 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 3779 let Inst{20-19} = imm{4-3}; 3780 } 3781 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 3782 let Inst{22} = imm{5}; 3783 let Inst{20-19} = imm{4-3}; 3784 } 3785 3786 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 3787 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 3788 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 3789 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 3790} 3791 3792multiclass sve2_int_bin_shift_imm_right<bit opc, string asm, 3793 SDPatternOperator op> { 3794 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3795 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3796 let Inst{19} = imm{3}; 3797 } 3798 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3799 let Inst{20-19} = imm{4-3}; 3800 } 3801 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3802 let Inst{22} = imm{5}; 3803 let Inst{20-19} = imm{4-3}; 3804 } 3805 3806 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3807 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3808 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3809 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3810} 3811 3812class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 3813 ZPRRegOp zprty, Operand immtype> 3814: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm), 3815 asm, "\t$Zda, $Zn, $imm", 3816 "", []>, Sched<[]> { 3817 bits<5> Zda; 3818 bits<5> Zn; 3819 bits<6> imm; 3820 let Inst{31-24} = 0b01000101; 3821 let Inst{23-22} = tsz8_64{3-2}; 3822 let Inst{21} = 0b0; 3823 let Inst{20-19} = tsz8_64{1-0}; 3824 let Inst{18-16} = imm{2-0}; // imm3 3825 let Inst{15-12} = 0b1110; 3826 let Inst{11-10} = opc; 3827 let Inst{9-5} = Zn; 3828 let Inst{4-0} = Zda; 3829 3830 let Constraints = "$Zda = $_Zda"; 3831 let DestructiveInstType = DestructiveOther; 3832 let ElementSize = ElementSizeNone; 3833} 3834 3835multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm, 3836 SDPatternOperator op, 3837 SDPatternOperator shift_op = null_frag> { 3838 def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3839 def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3840 let Inst{19} = imm{3}; 3841 } 3842 def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3843 let Inst{20-19} = imm{4-3}; 3844 } 3845 def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3846 let Inst{22} = imm{5}; 3847 let Inst{20-19} = imm{4-3}; 3848 } 3849 3850 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3851 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3852 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3853 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3854 3855 def : SVE_Shift_Add_All_Active_Pat<nxv16i8, shift_op, nxv16i1, nxv16i8, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 3856 def : SVE_Shift_Add_All_Active_Pat<nxv8i16, shift_op, nxv8i1, nxv8i16, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 3857 def : SVE_Shift_Add_All_Active_Pat<nxv4i32, shift_op, nxv4i1, nxv4i32, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 3858 def : SVE_Shift_Add_All_Active_Pat<nxv2i64, shift_op, nxv2i1, nxv2i64, nxv2i64, i32, !cast<Instruction>(NAME # _D)>; 3859} 3860 3861class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty> 3862: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot), 3863 asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> { 3864 bits<5> Zdn; 3865 bits<5> Zm; 3866 bit rot; 3867 let Inst{31-24} = 0b01000101; 3868 let Inst{23-22} = sz; 3869 let Inst{21-17} = 0b00000; 3870 let Inst{16} = opc; 3871 let Inst{15-11} = 0b11011; 3872 let Inst{10} = rot; 3873 let Inst{9-5} = Zm; 3874 let Inst{4-0} = Zdn; 3875 3876 let Constraints = "$Zdn = $_Zdn"; 3877 let DestructiveInstType = DestructiveOther; 3878 let ElementSize = ElementSizeNone; 3879} 3880 3881multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> { 3882 def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>; 3883 def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>; 3884 def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>; 3885 def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>; 3886 3887 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>; 3888 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>; 3889 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>; 3890 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>; 3891} 3892 3893class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm, 3894 ZPRRegOp zprty1, ZPRRegOp zprty2> 3895: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 3896 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3897 bits<5> Zda; 3898 bits<5> Zn; 3899 bits<5> Zm; 3900 let Inst{31-24} = 0b01000101; 3901 let Inst{23-22} = sz; 3902 let Inst{21} = 0b0; 3903 let Inst{20-16} = Zm; 3904 let Inst{15-14} = 0b11; 3905 let Inst{13-10} = opc; 3906 let Inst{9-5} = Zn; 3907 let Inst{4-0} = Zda; 3908 3909 let Constraints = "$Zda = $_Zda"; 3910 let DestructiveInstType = DestructiveOther; 3911 let ElementSize = ElementSizeNone; 3912} 3913 3914multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> { 3915 def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>; 3916 def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>; 3917 def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>; 3918 def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>; 3919 3920 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3921 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3922 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3923 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3924} 3925 3926multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm, 3927 SDPatternOperator op> { 3928 def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3929 def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3930 def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3931 3932 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3933 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3934 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3935} 3936 3937multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm, 3938 SDPatternOperator op> { 3939 def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm, 3940 ZPR32, ZPR32>; 3941 def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm, 3942 ZPR64, ZPR64>; 3943 3944 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3945 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3946} 3947 3948//===----------------------------------------------------------------------===// 3949// SVE2 Narrowing Group 3950//===----------------------------------------------------------------------===// 3951 3952class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc, 3953 string asm, ZPRRegOp zprty1, 3954 ZPRRegOp zprty2, Operand immtype> 3955: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3956 asm, "\t$Zd, $Zn, $imm", 3957 "", []>, Sched<[]> { 3958 bits<5> Zd; 3959 bits<5> Zn; 3960 bits<5> imm; 3961 let Inst{31-23} = 0b010001010; 3962 let Inst{22} = tsz8_64{2}; 3963 let Inst{21} = 0b1; 3964 let Inst{20-19} = tsz8_64{1-0}; 3965 let Inst{18-16} = imm{2-0}; // imm3 3966 let Inst{15-14} = 0b00; 3967 let Inst{13-11} = opc; 3968 let Inst{10} = 0b0; 3969 let Inst{9-5} = Zn; 3970 let Inst{4-0} = Zd; 3971} 3972 3973multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm, 3974 SDPatternOperator op> { 3975 def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16, 3976 tvecshiftR8>; 3977 def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32, 3978 tvecshiftR16> { 3979 let Inst{19} = imm{3}; 3980 } 3981 def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64, 3982 tvecshiftR32> { 3983 let Inst{20-19} = imm{4-3}; 3984 } 3985 def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3986 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3987 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3988} 3989 3990class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc, 3991 string asm, ZPRRegOp zprty1, 3992 ZPRRegOp zprty2, Operand immtype> 3993: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm), 3994 asm, "\t$Zd, $Zn, $imm", 3995 "", []>, Sched<[]> { 3996 bits<5> Zd; 3997 bits<5> Zn; 3998 bits<5> imm; 3999 let Inst{31-23} = 0b010001010; 4000 let Inst{22} = tsz8_64{2}; 4001 let Inst{21} = 0b1; 4002 let Inst{20-19} = tsz8_64{1-0}; 4003 let Inst{18-16} = imm{2-0}; // imm3 4004 let Inst{15-14} = 0b00; 4005 let Inst{13-11} = opc; 4006 let Inst{10} = 0b1; 4007 let Inst{9-5} = Zn; 4008 let Inst{4-0} = Zd; 4009 4010 let Constraints = "$Zd = $_Zd"; 4011} 4012 4013multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm, 4014 SDPatternOperator op> { 4015 def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16, 4016 tvecshiftR8>; 4017 def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32, 4018 tvecshiftR16> { 4019 let Inst{19} = imm{3}; 4020 } 4021 def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64, 4022 tvecshiftR32> { 4023 let Inst{20-19} = imm{4-3}; 4024 } 4025 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4026 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4027 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4028} 4029 4030class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm, 4031 ZPRRegOp zprty1, ZPRRegOp zprty2> 4032: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 4033 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 4034 bits<5> Zd; 4035 bits<5> Zn; 4036 bits<5> Zm; 4037 let Inst{31-24} = 0b01000101; 4038 let Inst{23-22} = sz; 4039 let Inst{21} = 0b1; 4040 let Inst{20-16} = Zm; 4041 let Inst{15-13} = 0b011; 4042 let Inst{12-11} = opc; // S, R 4043 let Inst{10} = 0b0; // Top 4044 let Inst{9-5} = Zn; 4045 let Inst{4-0} = Zd; 4046} 4047 4048multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm, 4049 SDPatternOperator op> { 4050 def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>; 4051 def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>; 4052 def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>; 4053 4054 def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 4055 def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 4056 def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 4057} 4058 4059class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm, 4060 ZPRRegOp zprty1, ZPRRegOp zprty2> 4061: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 4062 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 4063 bits<5> Zd; 4064 bits<5> Zn; 4065 bits<5> Zm; 4066 let Inst{31-24} = 0b01000101; 4067 let Inst{23-22} = sz; 4068 let Inst{21} = 0b1; 4069 let Inst{20-16} = Zm; 4070 let Inst{15-13} = 0b011; 4071 let Inst{12-11} = opc; // S, R 4072 let Inst{10} = 0b1; // Top 4073 let Inst{9-5} = Zn; 4074 let Inst{4-0} = Zd; 4075 4076 let Constraints = "$Zd = $_Zd"; 4077} 4078 4079multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm, 4080 SDPatternOperator op> { 4081 def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>; 4082 def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>; 4083 def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>; 4084 4085 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 4086 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 4087 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 4088} 4089 4090class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm, 4091 ZPRRegOp zprty1, ZPRRegOp zprty2> 4092: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 4093 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 4094 bits<5> Zd; 4095 bits<5> Zn; 4096 let Inst{31-23} = 0b010001010; 4097 let Inst{22} = tsz8_64{2}; 4098 let Inst{21} = 0b1; 4099 let Inst{20-19} = tsz8_64{1-0}; 4100 let Inst{18-13} = 0b000010; 4101 let Inst{12-11} = opc; 4102 let Inst{10} = 0b0; 4103 let Inst{9-5} = Zn; 4104 let Inst{4-0} = Zd; 4105} 4106 4107multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm, 4108 SDPatternOperator op> { 4109 def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>; 4110 def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>; 4111 def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>; 4112 4113 def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>; 4114 def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>; 4115 def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>; 4116} 4117 4118class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm, 4119 ZPRRegOp zprty1, ZPRRegOp zprty2> 4120: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn), 4121 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 4122 bits<5> Zd; 4123 bits<5> Zn; 4124 let Inst{31-23} = 0b010001010; 4125 let Inst{22} = tsz8_64{2}; 4126 let Inst{21} = 0b1; 4127 let Inst{20-19} = tsz8_64{1-0}; 4128 let Inst{18-13} = 0b000010; 4129 let Inst{12-11} = opc; 4130 let Inst{10} = 0b1; 4131 let Inst{9-5} = Zn; 4132 let Inst{4-0} = Zd; 4133 4134 let Constraints = "$Zd = $_Zd"; 4135} 4136 4137multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm, 4138 SDPatternOperator op> { 4139 def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>; 4140 def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>; 4141 def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>; 4142 4143 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>; 4144 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>; 4145 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4146} 4147 4148//===----------------------------------------------------------------------===// 4149// SVE Integer Arithmetic - Unary Predicated Group 4150//===----------------------------------------------------------------------===// 4151 4152class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc, 4153 string asm, ZPRRegOp zprty> 4154: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 4155 asm, "\t$Zd, $Pg/m, $Zn", 4156 "", 4157 []>, Sched<[]> { 4158 bits<3> Pg; 4159 bits<5> Zd; 4160 bits<5> Zn; 4161 let Inst{31-24} = 0b00000100; 4162 let Inst{23-22} = sz8_64; 4163 let Inst{21-20} = 0b01; 4164 let Inst{19} = opc{0}; 4165 let Inst{18-16} = opc{3-1}; 4166 let Inst{15-13} = 0b101; 4167 let Inst{12-10} = Pg; 4168 let Inst{9-5} = Zn; 4169 let Inst{4-0} = Zd; 4170 4171 let Constraints = "$Zd = $_Zd"; 4172 let DestructiveInstType = DestructiveUnaryPassthru; 4173 let ElementSize = zprty.ElementSize; 4174} 4175 4176multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm, 4177 SDPatternOperator op> { 4178 def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>, 4179 SVEPseudo2Instr<NAME # _B, 1>; 4180 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4181 SVEPseudo2Instr<NAME # _H, 1>; 4182 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4183 SVEPseudo2Instr<NAME # _S, 1>; 4184 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4185 SVEPseudo2Instr<NAME # _D, 1>; 4186 4187 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4188 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4189 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4190 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4191 4192 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4193 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4194 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4195 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4196 4197 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4198 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4199 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4200 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4201} 4202 4203multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm, 4204 SDPatternOperator op> { 4205 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4206 SVEPseudo2Instr<NAME # _H, 1>; 4207 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4208 SVEPseudo2Instr<NAME # _S, 1>; 4209 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4210 SVEPseudo2Instr<NAME # _D, 1>; 4211 4212 def : SVE_InReg_Extend<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>; 4213 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>; 4214 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>; 4215 4216 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4217 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4218 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4219 4220 defm : SVE_InReg_Extend_PassthruUndef<nxv8i16, op, nxv8i1, nxv8i8, !cast<Pseudo>(NAME # _UNDEF_H)>; 4221 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i8, !cast<Pseudo>(NAME # _UNDEF_S)>; 4222 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _UNDEF_D)>; 4223} 4224 4225multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm, 4226 SDPatternOperator op> { 4227 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4228 SVEPseudo2Instr<NAME # _S, 1>; 4229 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4230 SVEPseudo2Instr<NAME # _D, 1>; 4231 4232 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>; 4233 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>; 4234 4235 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4236 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4237 4238 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i16, !cast<Pseudo>(NAME # _UNDEF_S)>; 4239 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _UNDEF_D)>; 4240} 4241 4242multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm, 4243 SDPatternOperator op> { 4244 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4245 SVEPseudo2Instr<NAME # _D, 1>; 4246 4247 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>; 4248 4249 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4250 4251 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _UNDEF_D)>; 4252} 4253 4254multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm, 4255 SDPatternOperator op> { 4256 def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>, 4257 SVEPseudo2Instr<NAME # _B, 1>; 4258 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4259 SVEPseudo2Instr<NAME # _H, 1>; 4260 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4261 SVEPseudo2Instr<NAME # _S, 1>; 4262 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4263 SVEPseudo2Instr<NAME # _D, 1>; 4264 4265 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4266 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4267 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4268 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4269 4270 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4271 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4272 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4273 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4274 4275 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4276 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4277 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4278 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4279} 4280 4281multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> { 4282 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4283 SVEPseudo2Instr<NAME # _H, 1>; 4284 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4285 SVEPseudo2Instr<NAME # _S, 1>; 4286 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4287 SVEPseudo2Instr<NAME # _D, 1>; 4288 4289 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4290 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4291 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4292 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4293 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4294 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4295 4296 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4297 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4298 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4299 4300 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4301 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4302 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4303 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4304 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4305 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4306} 4307 4308//===----------------------------------------------------------------------===// 4309// SVE Integer Wide Immediate - Unpredicated Group 4310//===----------------------------------------------------------------------===// 4311class sve_int_dup_imm<bits<2> sz8_64, string asm, 4312 ZPRRegOp zprty, Operand immtype> 4313: I<(outs zprty:$Zd), (ins immtype:$imm), 4314 asm, "\t$Zd, $imm", 4315 "", 4316 []>, Sched<[]> { 4317 bits<5> Zd; 4318 bits<9> imm; 4319 let Inst{31-24} = 0b00100101; 4320 let Inst{23-22} = sz8_64; 4321 let Inst{21-14} = 0b11100011; 4322 let Inst{13} = imm{8}; // sh 4323 let Inst{12-5} = imm{7-0}; // imm8 4324 let Inst{4-0} = Zd; 4325 4326 let isReMaterializable = 1; 4327} 4328 4329multiclass sve_int_dup_imm<string asm> { 4330 def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>; 4331 def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>; 4332 def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>; 4333 def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>; 4334 4335 def : InstAlias<"mov $Zd, $imm", 4336 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>; 4337 def : InstAlias<"mov $Zd, $imm", 4338 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>; 4339 def : InstAlias<"mov $Zd, $imm", 4340 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>; 4341 def : InstAlias<"mov $Zd, $imm", 4342 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>; 4343 4344 def : InstAlias<"fmov $Zd, #0.0", 4345 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>; 4346 def : InstAlias<"fmov $Zd, #0.0", 4347 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>; 4348 def : InstAlias<"fmov $Zd, #0.0", 4349 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>; 4350} 4351 4352class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype, 4353 string asm, ZPRRegOp zprty> 4354: I<(outs zprty:$Zd), (ins fpimmtype:$imm8), 4355 asm, "\t$Zd, $imm8", 4356 "", 4357 []>, Sched<[]> { 4358 bits<5> Zd; 4359 bits<8> imm8; 4360 let Inst{31-24} = 0b00100101; 4361 let Inst{23-22} = sz8_64; 4362 let Inst{21-14} = 0b11100111; 4363 let Inst{13} = 0b0; 4364 let Inst{12-5} = imm8; 4365 let Inst{4-0} = Zd; 4366 4367 let isReMaterializable = 1; 4368} 4369 4370multiclass sve_int_dup_fpimm<string asm> { 4371 def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>; 4372 def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>; 4373 def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>; 4374 4375 def : InstAlias<"fmov $Zd, $imm8", 4376 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>; 4377 def : InstAlias<"fmov $Zd, $imm8", 4378 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>; 4379 def : InstAlias<"fmov $Zd, $imm8", 4380 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>; 4381} 4382 4383class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm, 4384 ZPRRegOp zprty, Operand immtype> 4385: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4386 asm, "\t$Zdn, $_Zdn, $imm", 4387 "", 4388 []>, Sched<[]> { 4389 bits<5> Zdn; 4390 bits<9> imm; 4391 let Inst{31-24} = 0b00100101; 4392 let Inst{23-22} = sz8_64; 4393 let Inst{21-19} = 0b100; 4394 let Inst{18-16} = opc; 4395 let Inst{15-14} = 0b11; 4396 let Inst{13} = imm{8}; // sh 4397 let Inst{12-5} = imm{7-0}; // imm8 4398 let Inst{4-0} = Zdn; 4399 4400 let Constraints = "$Zdn = $_Zdn"; 4401 let DestructiveInstType = DestructiveOther; 4402 let ElementSize = ElementSizeNone; 4403} 4404 4405multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> { 4406 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 4407 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 4408 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 4409 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 4410 4411 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 4412 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 4413 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 4414 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 4415} 4416 4417class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm, 4418 ZPRRegOp zprty, Operand immtype> 4419: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4420 asm, "\t$Zdn, $_Zdn, $imm", 4421 "", 4422 []>, Sched<[]> { 4423 bits<5> Zdn; 4424 bits<8> imm; 4425 let Inst{31-24} = 0b00100101; 4426 let Inst{23-22} = sz8_64; 4427 let Inst{21-16} = opc; 4428 let Inst{15-13} = 0b110; 4429 let Inst{12-5} = imm; 4430 let Inst{4-0} = Zdn; 4431 4432 let Constraints = "$Zdn = $_Zdn"; 4433 let DestructiveInstType = DestructiveOther; 4434 let ElementSize = ElementSizeNone; 4435} 4436 4437multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> { 4438 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8>; 4439 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8>; 4440 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>; 4441 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; 4442 4443 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; 4444 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; 4445 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; 4446 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; 4447} 4448 4449multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> { 4450 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>; 4451 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>; 4452 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>; 4453 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>; 4454 4455 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>; 4456 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>; 4457 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>; 4458 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>; 4459} 4460 4461multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> { 4462 def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8, simm8>; 4463 def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8>; 4464 def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; 4465 def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; 4466 4467 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; 4468 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; 4469 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; 4470 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; 4471} 4472 4473//===----------------------------------------------------------------------===// 4474// SVE Bitwise Logical - Unpredicated Group 4475//===----------------------------------------------------------------------===// 4476 4477class sve_int_bin_cons_log<bits<2> opc, string asm> 4478: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm), 4479 asm, "\t$Zd, $Zn, $Zm", 4480 "", 4481 []>, Sched<[]> { 4482 bits<5> Zd; 4483 bits<5> Zm; 4484 bits<5> Zn; 4485 let Inst{31-24} = 0b00000100; 4486 let Inst{23-22} = opc{1-0}; 4487 let Inst{21} = 0b1; 4488 let Inst{20-16} = Zm; 4489 let Inst{15-10} = 0b001100; 4490 let Inst{9-5} = Zn; 4491 let Inst{4-0} = Zd; 4492} 4493 4494multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> { 4495 def NAME : sve_int_bin_cons_log<opc, asm>; 4496 4497 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4498 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4499 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4500 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4501 4502 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4503 (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>; 4504 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4505 (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>; 4506 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4507 (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>; 4508} 4509 4510class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm> 4511: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk), 4512 asm, "\t$Zdn, $_Zdn, $Zm, $Zk", 4513 "", 4514 []>, Sched<[]> { 4515 bits<5> Zdn; 4516 bits<5> Zk; 4517 bits<5> Zm; 4518 let Inst{31-24} = 0b00000100; 4519 let Inst{23-22} = opc{2-1}; 4520 let Inst{21} = 0b1; 4521 let Inst{20-16} = Zm; 4522 let Inst{15-11} = 0b00111; 4523 let Inst{10} = opc{0}; 4524 let Inst{9-5} = Zk; 4525 let Inst{4-0} = Zdn; 4526 4527 let Constraints = "$Zdn = $_Zdn"; 4528 let DestructiveInstType = DestructiveOther; 4529 let ElementSize = ElementSizeNone; 4530} 4531 4532multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op, 4533 SDPatternOperator ir_op = null_frag> { 4534 def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>; 4535 4536 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4537 (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>; 4538 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4539 (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>; 4540 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4541 (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>; 4542 4543 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4544 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4545 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4546 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4547 4548 4549 def : SVE_3_Op_BSP_Pat<nxv16i8, ir_op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4550 def : SVE_3_Op_BSP_Pat<nxv8i16, ir_op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4551 def : SVE_3_Op_BSP_Pat<nxv4i32, ir_op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4552 def : SVE_3_Op_BSP_Pat<nxv2i64, ir_op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4553} 4554 4555class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm, 4556 ZPRRegOp zprty, Operand immtype> 4557: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm), 4558 asm, "\t$Zdn, $_Zdn, $Zm, $imm", 4559 "", 4560 []>, Sched<[]> { 4561 bits<5> Zdn; 4562 bits<5> Zm; 4563 bits<6> imm; 4564 let Inst{31-24} = 0b00000100; 4565 let Inst{23-22} = tsz8_64{3-2}; 4566 let Inst{21} = 0b1; 4567 let Inst{20-19} = tsz8_64{1-0}; 4568 let Inst{18-16} = imm{2-0}; // imm3 4569 let Inst{15-10} = 0b001101; 4570 let Inst{9-5} = Zm; 4571 let Inst{4-0} = Zdn; 4572 4573 let Constraints = "$Zdn = $_Zdn"; 4574 let DestructiveInstType = DestructiveOther; 4575 let ElementSize = ElementSizeNone; 4576} 4577 4578multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> { 4579 def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>; 4580 def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> { 4581 let Inst{19} = imm{3}; 4582 } 4583 def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> { 4584 let Inst{20-19} = imm{4-3}; 4585 } 4586 def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> { 4587 let Inst{22} = imm{5}; 4588 let Inst{20-19} = imm{4-3}; 4589 } 4590 4591 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4592 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4593 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4594 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4595} 4596 4597//===----------------------------------------------------------------------===// 4598// SVE Integer Wide Immediate - Predicated Group 4599//===----------------------------------------------------------------------===// 4600 4601class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype, 4602 string asm, ZPRRegOp zprty> 4603: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8), 4604 asm, "\t$Zd, $Pg/m, $imm8", 4605 "", 4606 []>, Sched<[]> { 4607 bits<4> Pg; 4608 bits<5> Zd; 4609 bits<8> imm8; 4610 let Inst{31-24} = 0b00000101; 4611 let Inst{23-22} = sz; 4612 let Inst{21-20} = 0b01; 4613 let Inst{19-16} = Pg; 4614 let Inst{15-13} = 0b110; 4615 let Inst{12-5} = imm8; 4616 let Inst{4-0} = Zd; 4617 4618 let Constraints = "$Zd = $_Zd"; 4619 let DestructiveInstType = DestructiveOther; 4620 let ElementSize = zprty.ElementSize; 4621} 4622 4623multiclass sve_int_dup_fpimm_pred<string asm> { 4624 def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>; 4625 def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>; 4626 def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>; 4627 4628 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4629 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>; 4630 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4631 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>; 4632 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4633 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>; 4634} 4635 4636class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm, 4637 ZPRRegOp zprty, string pred_qual, dag iops> 4638: I<(outs zprty:$Zd), iops, 4639 asm, "\t$Zd, $Pg"#pred_qual#", $imm", 4640 "", []>, Sched<[]> { 4641 bits<5> Zd; 4642 bits<4> Pg; 4643 bits<9> imm; 4644 let Inst{31-24} = 0b00000101; 4645 let Inst{23-22} = sz8_64; 4646 let Inst{21-20} = 0b01; 4647 let Inst{19-16} = Pg; 4648 let Inst{15} = 0b0; 4649 let Inst{14} = m; 4650 let Inst{13} = imm{8}; // sh 4651 let Inst{12-5} = imm{7-0}; // imm8 4652 let Inst{4-0} = Zd; 4653 4654 let DestructiveInstType = DestructiveOther; 4655 let ElementSize = zprty.ElementSize; 4656} 4657 4658multiclass sve_int_dup_imm_pred_merge_inst< 4659 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm, 4660 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> { 4661 let Constraints = "$Zd = $_Zd" in 4662 def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty, "/m", 4663 (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>; 4664 def : InstAlias<"mov $Zd, $Pg/m, $imm", 4665 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4666 def : Pat<(vselect predty:$Pg, 4667 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))), 4668 ZPR:$Zd), 4669 (!cast<Instruction>(NAME) $Zd, $Pg, $imm, $shift)>; 4670} 4671 4672multiclass sve_int_dup_imm_pred_merge<string asm> { 4673 defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8, 4674 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>; 4675 defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16, 4676 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>; 4677 defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32, 4678 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>; 4679 defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64, 4680 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>; 4681 4682 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4683 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>; 4684 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4685 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>; 4686 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4687 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>; 4688 4689 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv8f16 ZPR:$Zd)), 4690 (!cast<Instruction>(NAME # _H) $Zd, $Pg, 0, 0)>; 4691 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f16 ZPR:$Zd)), 4692 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>; 4693 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f16 ZPR:$Zd)), 4694 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4695 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f32 ZPR:$Zd)), 4696 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>; 4697 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f32 ZPR:$Zd)), 4698 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4699 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f64 ZPR:$Zd)), 4700 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4701} 4702 4703multiclass sve_int_dup_imm_pred_zero_inst< 4704 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm, 4705 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> { 4706 def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z", 4707 (ins PPRAny:$Pg, cpyimm:$imm)>; 4708 def : InstAlias<"mov $Zd, $Pg/z, $imm", 4709 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4710 def : Pat<(intty (zext (predty PPRAny:$Ps1))), 4711 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4712 def : Pat<(intty (sext (predty PPRAny:$Ps1))), 4713 (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>; 4714 def : Pat<(intty (anyext (predty PPRAny:$Ps1))), 4715 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4716 def : Pat<(vselect predty:$Pg, 4717 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))), 4718 (intty (splat_vector (scalarty 0)))), 4719 (!cast<Instruction>(NAME) $Pg, $imm, $shift)>; 4720} 4721 4722multiclass sve_int_dup_imm_pred_zero<string asm> { 4723 defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8, 4724 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>; 4725 defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16, 4726 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>; 4727 defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32, 4728 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>; 4729 defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64, 4730 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>; 4731} 4732 4733//===----------------------------------------------------------------------===// 4734// SVE Integer Compare - Vectors Group 4735//===----------------------------------------------------------------------===// 4736 4737class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm, 4738 PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2> 4739: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm), 4740 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4741 "", 4742 []>, Sched<[]> { 4743 bits<4> Pd; 4744 bits<3> Pg; 4745 bits<5> Zm; 4746 bits<5> Zn; 4747 let Inst{31-24} = 0b00100100; 4748 let Inst{23-22} = sz8_64; 4749 let Inst{21} = 0b0; 4750 let Inst{20-16} = Zm; 4751 let Inst{15} = opc{2}; 4752 let Inst{14} = cmp_1; 4753 let Inst{13} = opc{1}; 4754 let Inst{12-10} = Pg; 4755 let Inst{9-5} = Zn; 4756 let Inst{4} = opc{0}; 4757 let Inst{3-0} = Pd; 4758 4759 let Defs = [NZCV]; 4760 let ElementSize = pprty.ElementSize; 4761 let isPTestLike = 1; 4762} 4763 4764multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt, 4765 ValueType intvt, Instruction cmp> { 4766 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)), 4767 (cmp $Op1, $Op2, $Op3)>; 4768 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)), 4769 (cmp $Op1, $Op3, $Op2)>; 4770 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op2, intvt:$Op3, cc))), 4771 (cmp $Pg, $Op2, $Op3)>; 4772 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op2, intvt:$Op3, invcc))), 4773 (cmp $Pg, $Op3, $Op2)>; 4774} 4775 4776multiclass SVE_SETCC_Pat_With_Zero<CondCode cc, CondCode invcc, ValueType predvt, 4777 ValueType intvt, Instruction cmp> { 4778 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, (SVEDup0), cc)), 4779 (cmp $Op1, $Op2)>; 4780 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, (SVEDup0), intvt:$Op2, invcc)), 4781 (cmp $Op1, $Op2)>; 4782 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op1, (SVEDup0), cc))), 4783 (cmp $Pg, $Op1)>; 4784 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), (SVEDup0), intvt:$Op1, invcc))), 4785 (cmp $Pg, $Op1)>; 4786} 4787 4788multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> { 4789 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>; 4790 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>; 4791 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>; 4792 def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>; 4793 4794 defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4795 defm : SVE_SETCC_Pat<cc, invcc, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4796 defm : SVE_SETCC_Pat<cc, invcc, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4797 defm : SVE_SETCC_Pat<cc, invcc, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4798} 4799 4800multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> { 4801 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4802 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4803 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4804 4805 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4806 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4807 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4808} 4809 4810multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> { 4811 def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4812 def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4813 def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4814 4815 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4816 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4817 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4818} 4819 4820 4821//===----------------------------------------------------------------------===// 4822// SVE Integer Compare - Signed Immediate Group 4823//===----------------------------------------------------------------------===// 4824 4825class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 4826 ZPRRegOp zprty, 4827 Operand immtype> 4828: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5), 4829 asm, "\t$Pd, $Pg/z, $Zn, $imm5", 4830 "", 4831 []>, Sched<[]> { 4832 bits<4> Pd; 4833 bits<3> Pg; 4834 bits<5> Zn; 4835 bits<5> imm5; 4836 let Inst{31-24} = 0b00100101; 4837 let Inst{23-22} = sz8_64; 4838 let Inst{21} = 0b0; 4839 let Inst{20-16} = imm5; 4840 let Inst{15} = opc{2}; 4841 let Inst{14} = 0b0; 4842 let Inst{13} = opc{1}; 4843 let Inst{12-10} = Pg; 4844 let Inst{9-5} = Zn; 4845 let Inst{4} = opc{0}; 4846 let Inst{3-0} = Pd; 4847 4848 let Defs = [NZCV]; 4849 let ElementSize = pprty.ElementSize; 4850 let isPTestLike = 1; 4851} 4852 4853multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc, 4854 ValueType predvt, ValueType intvt, 4855 Operand immtype, Instruction cmp> { 4856 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4857 (intvt ZPR:$Zs1), 4858 (intvt (splat_vector (immtype:$imm))), 4859 cc)), 4860 (cmp $Pg, $Zs1, immtype:$imm)>; 4861 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4862 (intvt (splat_vector (immtype:$imm))), 4863 (intvt ZPR:$Zs1), 4864 commuted_cc)), 4865 (cmp $Pg, $Zs1, immtype:$imm)>; 4866 def : Pat<(predvt (and predvt:$Pg, 4867 (AArch64setcc_z (predvt (AArch64ptrue 31)), 4868 (intvt ZPR:$Zs1), 4869 (intvt (splat_vector (immtype:$imm))), 4870 cc))), 4871 (cmp $Pg, $Zs1, immtype:$imm)>; 4872 def : Pat<(predvt (and predvt:$Pg, 4873 (AArch64setcc_z (predvt (AArch64ptrue 31)), 4874 (intvt (splat_vector (immtype:$imm))), 4875 (intvt ZPR:$Zs1), 4876 commuted_cc))), 4877 (cmp $Pg, $Zs1, immtype:$imm)>; 4878} 4879 4880multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> { 4881 def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>; 4882 def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>; 4883 def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>; 4884 def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>; 4885 4886 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b, 4887 !cast<Instruction>(NAME # _B)>; 4888 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, simm5_32b, 4889 !cast<Instruction>(NAME # _H)>; 4890 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, simm5_32b, 4891 !cast<Instruction>(NAME # _S)>; 4892 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, simm5_64b, 4893 !cast<Instruction>(NAME # _D)>; 4894} 4895 4896 4897//===----------------------------------------------------------------------===// 4898// SVE Integer Compare - Unsigned Immediate Group 4899//===----------------------------------------------------------------------===// 4900 4901class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty, 4902 ZPRRegOp zprty, Operand immtype> 4903: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7), 4904 asm, "\t$Pd, $Pg/z, $Zn, $imm7", 4905 "", 4906 []>, Sched<[]> { 4907 bits<4> Pd; 4908 bits<3> Pg; 4909 bits<5> Zn; 4910 bits<7> imm7; 4911 let Inst{31-24} = 0b00100100; 4912 let Inst{23-22} = sz8_64; 4913 let Inst{21} = 1; 4914 let Inst{20-14} = imm7; 4915 let Inst{13} = opc{1}; 4916 let Inst{12-10} = Pg; 4917 let Inst{9-5} = Zn; 4918 let Inst{4} = opc{0}; 4919 let Inst{3-0} = Pd; 4920 4921 let Defs = [NZCV]; 4922 let ElementSize = pprty.ElementSize; 4923 let isPTestLike = 1; 4924} 4925 4926multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc, 4927 CondCode commuted_cc> { 4928 def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>; 4929 def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>; 4930 def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>; 4931 def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>; 4932 4933 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127, 4934 !cast<Instruction>(NAME # _B)>; 4935 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, imm0_127, 4936 !cast<Instruction>(NAME # _H)>; 4937 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, imm0_127, 4938 !cast<Instruction>(NAME # _S)>; 4939 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, imm0_127_64b, 4940 !cast<Instruction>(NAME # _D)>; 4941} 4942 4943 4944//===----------------------------------------------------------------------===// 4945// SVE Integer Compare - Scalars Group 4946//===----------------------------------------------------------------------===// 4947 4948class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt> 4949: I<(outs), (ins rt:$Rn, rt:$Rm), 4950 asm, "\t$Rn, $Rm", 4951 "", 4952 []>, Sched<[]> { 4953 bits<5> Rm; 4954 bits<5> Rn; 4955 let Inst{31-23} = 0b001001011; 4956 let Inst{22} = sz; 4957 let Inst{21} = 0b1; 4958 let Inst{20-16} = Rm; 4959 let Inst{15-10} = 0b001000; 4960 let Inst{9-5} = Rn; 4961 let Inst{4} = opc; 4962 let Inst{3-0} = 0b0000; 4963 4964 let Defs = [NZCV]; 4965} 4966 4967class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm, 4968 RegisterClass gprty, PPRRegOp pprty> 4969: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm), 4970 asm, "\t$Pd, $Rn, $Rm", 4971 "", []>, Sched<[]> { 4972 bits<4> Pd; 4973 bits<5> Rm; 4974 bits<5> Rn; 4975 let Inst{31-24} = 0b00100101; 4976 let Inst{23-22} = sz8_64; 4977 let Inst{21} = 0b1; 4978 let Inst{20-16} = Rm; 4979 let Inst{15-13} = 0b000; 4980 let Inst{12-10} = opc{3-1}; 4981 let Inst{9-5} = Rn; 4982 let Inst{4} = opc{0}; 4983 let Inst{3-0} = Pd; 4984 4985 let Defs = [NZCV]; 4986 let ElementSize = pprty.ElementSize; 4987 let isWhile = 1; 4988} 4989 4990multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> { 4991 def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>; 4992 def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>; 4993 def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>; 4994 def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>; 4995 4996 def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>; 4997 def : SVE_2_Op_Pat<nxv8i1, op, i32, i32, !cast<Instruction>(NAME # _H)>; 4998 def : SVE_2_Op_Pat<nxv4i1, op, i32, i32, !cast<Instruction>(NAME # _S)>; 4999 def : SVE_2_Op_Pat<nxv2i1, op, i32, i32, !cast<Instruction>(NAME # _D)>; 5000} 5001 5002multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op> { 5003 def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8>; 5004 def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16>; 5005 def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32>; 5006 def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>; 5007 5008 def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>; 5009 def : SVE_2_Op_Pat<nxv8i1, op, i64, i64, !cast<Instruction>(NAME # _H)>; 5010 def : SVE_2_Op_Pat<nxv4i1, op, i64, i64, !cast<Instruction>(NAME # _S)>; 5011 def : SVE_2_Op_Pat<nxv2i1, op, i64, i64, !cast<Instruction>(NAME # _D)>; 5012} 5013 5014class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, 5015 PPRRegOp pprty> 5016: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), 5017 asm, "\t$Pd, $Rn, $Rm", 5018 "", []>, Sched<[]> { 5019 bits<4> Pd; 5020 bits<5> Rm; 5021 bits<5> Rn; 5022 let Inst{31-24} = 0b00100101; 5023 let Inst{23-22} = sz8_64; 5024 let Inst{21} = 0b1; 5025 let Inst{20-16} = Rm; 5026 let Inst{15-10} = 0b001100; 5027 let Inst{9-5} = Rn; 5028 let Inst{4} = rw; 5029 let Inst{3-0} = Pd; 5030 5031 let Defs = [NZCV]; 5032 let ElementSize = pprty.ElementSize; 5033 let isWhile = 1; 5034} 5035 5036multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> { 5037 def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; 5038 def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; 5039 def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; 5040 def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; 5041 5042 def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>; 5043 def : SVE_2_Op_Pat<nxv8i1, !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>; 5044 def : SVE_2_Op_Pat<nxv4i1, !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>; 5045 def : SVE_2_Op_Pat<nxv2i1, !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>; 5046} 5047 5048//===----------------------------------------------------------------------===// 5049// SVE Floating Point Fast Reduction Group 5050//===----------------------------------------------------------------------===// 5051 5052class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm, 5053 ZPRRegOp zprty, FPRasZPROperand dstOpType> 5054: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 5055 asm, "\t$Vd, $Pg, $Zn", 5056 "", 5057 []>, Sched<[]> { 5058 bits<5> Zn; 5059 bits<5> Vd; 5060 bits<3> Pg; 5061 let Inst{31-24} = 0b01100101; 5062 let Inst{23-22} = sz; 5063 let Inst{21-19} = 0b000; 5064 let Inst{18-16} = opc; 5065 let Inst{15-13} = 0b001; 5066 let Inst{12-10} = Pg; 5067 let Inst{9-5} = Zn; 5068 let Inst{4-0} = Vd; 5069} 5070 5071multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> { 5072 def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>; 5073 def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>; 5074 def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>; 5075 5076 def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5077 def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5078 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5079 def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5080 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5081 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5082} 5083 5084//===----------------------------------------------------------------------===// 5085// SVE Floating Point Accumulating Reduction Group 5086//===----------------------------------------------------------------------===// 5087 5088class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm, 5089 ZPRRegOp zprty, FPRasZPROperand dstOpType> 5090: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm), 5091 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 5092 "", 5093 []>, 5094 Sched<[]> { 5095 bits<3> Pg; 5096 bits<5> Vdn; 5097 bits<5> Zm; 5098 let Inst{31-24} = 0b01100101; 5099 let Inst{23-22} = sz; 5100 let Inst{21-19} = 0b011; 5101 let Inst{18-16} = opc; 5102 let Inst{15-13} = 0b001; 5103 let Inst{12-10} = Pg; 5104 let Inst{9-5} = Zm; 5105 let Inst{4-0} = Vdn; 5106 5107 let Constraints = "$Vdn = $_Vdn"; 5108} 5109 5110multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> { 5111 def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>; 5112 def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>; 5113 def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>; 5114 5115 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>; 5116 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>; 5117 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5118 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>; 5119 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5120 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5121} 5122 5123//===----------------------------------------------------------------------===// 5124// SVE Floating Point Compare - Vectors Group 5125//===----------------------------------------------------------------------===// 5126 5127class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 5128 ZPRRegOp zprty> 5129: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 5130 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 5131 "", 5132 []>, Sched<[]> { 5133 bits<4> Pd; 5134 bits<3> Pg; 5135 bits<5> Zm; 5136 bits<5> Zn; 5137 let Inst{31-24} = 0b01100101; 5138 let Inst{23-22} = sz; 5139 let Inst{21} = 0b0; 5140 let Inst{20-16} = Zm; 5141 let Inst{15} = opc{2}; 5142 let Inst{14} = 0b1; 5143 let Inst{13} = opc{1}; 5144 let Inst{12-10} = Pg; 5145 let Inst{9-5} = Zn; 5146 let Inst{4} = opc{0}; 5147 let Inst{3-0} = Pd; 5148} 5149 5150multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> { 5151 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5152 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5153 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5154 5155 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5156 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5157 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5158} 5159 5160multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, 5161 CondCode cc1, CondCode cc2, 5162 CondCode invcc1, CondCode invcc2> { 5163 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5164 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5165 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5166 5167 defm : SVE_SETCC_Pat<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5168 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5169 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5170 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5171 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5172 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5173 5174 defm : SVE_SETCC_Pat<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5175 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5176 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5177 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5178 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5179 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5180} 5181 5182//===----------------------------------------------------------------------===// 5183// SVE Floating Point Compare - with Zero Group 5184//===----------------------------------------------------------------------===// 5185 5186class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 5187 ZPRRegOp zprty> 5188: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn), 5189 asm, "\t$Pd, $Pg/z, $Zn, #0.0", 5190 "", 5191 []>, Sched<[]> { 5192 bits<4> Pd; 5193 bits<3> Pg; 5194 bits<5> Zn; 5195 let Inst{31-24} = 0b01100101; 5196 let Inst{23-22} = sz; 5197 let Inst{21-18} = 0b0100; 5198 let Inst{17-16} = opc{2-1}; 5199 let Inst{15-13} = 0b001; 5200 let Inst{12-10} = Pg; 5201 let Inst{9-5} = Zn; 5202 let Inst{4} = opc{0}; 5203 let Inst{3-0} = Pd; 5204} 5205 5206multiclass sve_fp_2op_p_pd<bits<3> opc, string asm, 5207 CondCode cc1, CondCode cc2, 5208 CondCode invcc1, CondCode invcc2> { 5209 def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5210 def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5211 def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5212 5213 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5214 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5215 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5216 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5217 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5218 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5219 5220 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5221 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5222 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5223 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5224 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5225 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5226} 5227 5228 5229//===----------------------------------------------------------------------===// 5230//SVE Index Generation Group 5231//===----------------------------------------------------------------------===// 5232 5233def simm5_8b_tgt : TImmLeaf<i8, [{ return (int8_t)Imm >= -16 && (int8_t)Imm < 16; }]>; 5234def simm5_16b_tgt : TImmLeaf<i16, [{ return (int16_t)Imm >= -16 && (int16_t)Imm < 16; }]>; 5235def simm5_32b_tgt : TImmLeaf<i32, [{ return (int32_t)Imm >= -16 && (int32_t)Imm < 16; }]>; 5236def simm5_64b_tgt : TImmLeaf<i64, [{ return (int64_t)Imm >= -16 && (int64_t)Imm < 16; }]>; 5237def i64imm_32bit_tgt : TImmLeaf<i64, [{ 5238 return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm); 5239}]>; 5240 5241class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5242 Operand imm_ty> 5243: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b), 5244 asm, "\t$Zd, $imm5, $imm5b", 5245 "", []>, Sched<[]> { 5246 bits<5> Zd; 5247 bits<5> imm5; 5248 bits<5> imm5b; 5249 let Inst{31-24} = 0b00000100; 5250 let Inst{23-22} = sz8_64; 5251 let Inst{21} = 0b1; 5252 let Inst{20-16} = imm5b; 5253 let Inst{15-10} = 0b010000; 5254 let Inst{9-5} = imm5; 5255 let Inst{4-0} = Zd; 5256 5257 let isReMaterializable = 1; 5258} 5259 5260multiclass sve_int_index_ii<string asm> { 5261 def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>; 5262 def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>; 5263 def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>; 5264 def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>; 5265 5266 def : Pat<(nxv16i8 (step_vector simm5_8b_tgt:$imm5b)), 5267 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5268 def : Pat<(nxv8i16 (step_vector simm5_16b_tgt:$imm5b)), 5269 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5270 def : Pat<(nxv4i32 (step_vector simm5_32b_tgt:$imm5b)), 5271 (!cast<Instruction>(NAME # "_S") (i32 0), simm5_32b:$imm5b)>; 5272 def : Pat<(nxv2i64 (step_vector simm5_64b_tgt:$imm5b)), 5273 (!cast<Instruction>(NAME # "_D") (i64 0), simm5_64b:$imm5b)>; 5274 5275 // add(step_vector(step), dup(X)) -> index(X, step). 5276 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5b)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5277 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5278 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5b)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5279 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5280 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5b)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5281 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>; 5282 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5b)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5283 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>; 5284} 5285 5286class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5287 RegisterClass srcRegType, Operand imm_ty> 5288: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm), 5289 asm, "\t$Zd, $imm5, $Rm", 5290 "", []>, Sched<[]> { 5291 bits<5> Rm; 5292 bits<5> Zd; 5293 bits<5> imm5; 5294 let Inst{31-24} = 0b00000100; 5295 let Inst{23-22} = sz8_64; 5296 let Inst{21} = 0b1; 5297 let Inst{20-16} = Rm; 5298 let Inst{15-10} = 0b010010; 5299 let Inst{9-5} = imm5; 5300 let Inst{4-0} = Zd; 5301} 5302 5303multiclass sve_int_index_ir<string asm, SDPatternOperator mulop, SDPatternOperator muloneuseop> { 5304 def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>; 5305 def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>; 5306 def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>; 5307 def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>; 5308 5309 def : Pat<(nxv16i8 (step_vector i8:$imm)), 5310 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5311 def : Pat<(nxv8i16 (step_vector i16:$imm)), 5312 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5313 def : Pat<(nxv4i32 (step_vector i32:$imm)), 5314 (!cast<Instruction>(NAME # "_S") (i32 0), (!cast<Instruction>("MOVi32imm") $imm))>; 5315 def : Pat<(nxv2i64 (step_vector i64:$imm)), 5316 (!cast<Instruction>(NAME # "_D") (i64 0), (!cast<Instruction>("MOVi64imm") $imm))>; 5317 def : Pat<(nxv2i64 (step_vector i64imm_32bit_tgt:$imm)), 5318 (!cast<Instruction>(NAME # "_D") (i64 0), (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5319 5320 // add(step_vector(step), dup(X)) -> index(X, step). 5321 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5322 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5323 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5324 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5325 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5326 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, (!cast<Instruction>("MOVi32imm") $imm))>; 5327 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5328 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (!cast<Instruction>("MOVi64imm") $imm))>; 5329 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5330 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5331 5332 // mul(step_vector(1), dup(Y)) -> index(0, Y). 5333 def : Pat<(mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), 5334 (!cast<Instruction>(NAME # "_B") (i32 0), GPR32:$Rm)>; 5335 def : Pat<(mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), 5336 (!cast<Instruction>(NAME # "_H") (i32 0), GPR32:$Rm)>; 5337 def : Pat<(mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), 5338 (!cast<Instruction>(NAME # "_S") (i32 0), GPR32:$Rm)>; 5339 def : Pat<(mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), 5340 (!cast<Instruction>(NAME # "_D") (i64 0), GPR64:$Rm)>; 5341 5342 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5343 def : Pat<(add (muloneuseop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5344 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>; 5345 def : Pat<(add (muloneuseop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5346 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>; 5347 def : Pat<(add (muloneuseop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5348 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>; 5349 def : Pat<(add (muloneuseop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5350 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>; 5351} 5352 5353class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5354 RegisterClass srcRegType, Operand imm_ty> 5355: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5), 5356 asm, "\t$Zd, $Rn, $imm5", 5357 "", []>, Sched<[]> { 5358 bits<5> Rn; 5359 bits<5> Zd; 5360 bits<5> imm5; 5361 let Inst{31-24} = 0b00000100; 5362 let Inst{23-22} = sz8_64; 5363 let Inst{21} = 0b1; 5364 let Inst{20-16} = imm5; 5365 let Inst{15-10} = 0b010001; 5366 let Inst{9-5} = Rn; 5367 let Inst{4-0} = Zd; 5368} 5369 5370multiclass sve_int_index_ri<string asm> { 5371 def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>; 5372 def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>; 5373 def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>; 5374 def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>; 5375 5376 // add(step_vector(step), dup(X)) -> index(X, step). 5377 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5)), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), 5378 (!cast<Instruction>(NAME # "_B") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5379 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5)), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), 5380 (!cast<Instruction>(NAME # "_H") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5381 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5)), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), 5382 (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>; 5383 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5)), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), 5384 (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>; 5385} 5386 5387class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5388 RegisterClass srcRegType> 5389: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm), 5390 asm, "\t$Zd, $Rn, $Rm", 5391 "", []>, Sched<[]> { 5392 bits<5> Zd; 5393 bits<5> Rm; 5394 bits<5> Rn; 5395 let Inst{31-24} = 0b00000100; 5396 let Inst{23-22} = sz8_64; 5397 let Inst{21} = 0b1; 5398 let Inst{20-16} = Rm; 5399 let Inst{15-10} = 0b010011; 5400 let Inst{9-5} = Rn; 5401 let Inst{4-0} = Zd; 5402} 5403 5404multiclass sve_int_index_rr<string asm, SDPatternOperator mulop> { 5405 def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>; 5406 def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>; 5407 def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>; 5408 def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>; 5409 5410 // add(step_vector(step), dup(X)) -> index(X, step). 5411 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))), 5412 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5413 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(i32 GPR32:$Rn)))), 5414 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5415 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(i32 GPR32:$Rn)))), 5416 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") $imm))>; 5417 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5418 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (!cast<Instruction>("MOVi64imm") $imm))>; 5419 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5420 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5421 5422 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5423 def : Pat<(add (mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))), 5424 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, GPR32:$Rm)>; 5425 def : Pat<(add (mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))),(nxv8i16 (splat_vector(i32 GPR32:$Rn)))), 5426 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, GPR32:$Rm)>; 5427 def : Pat<(add (mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))),(nxv4i32 (splat_vector(i32 GPR32:$Rn)))), 5428 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, GPR32:$Rm)>; 5429 def : Pat<(add (mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))),(nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5430 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, GPR64:$Rm)>; 5431} 5432 5433//===----------------------------------------------------------------------===// 5434// SVE Bitwise Shift - Predicated Group 5435//===----------------------------------------------------------------------===// 5436 5437class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm, 5438 ZPRRegOp zprty, Operand immtype> 5439: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm), 5440 asm, "\t$Zdn, $Pg/m, $_Zdn, $imm", 5441 "", 5442 []>, Sched<[]> { 5443 bits<3> Pg; 5444 bits<5> Zdn; 5445 bits<6> imm; 5446 let Inst{31-24} = 0b00000100; 5447 let Inst{23-22} = tsz8_64{3-2}; 5448 let Inst{21-20} = 0b00; 5449 let Inst{19-16} = opc; 5450 let Inst{15-13} = 0b100; 5451 let Inst{12-10} = Pg; 5452 let Inst{9-8} = tsz8_64{1-0}; 5453 let Inst{7-5} = imm{2-0}; // imm3 5454 let Inst{4-0} = Zdn; 5455 5456 let Constraints = "$Zdn = $_Zdn"; 5457 let DestructiveInstType = DestructiveBinaryImm; 5458 let ElementSize = zprty.ElementSize; 5459} 5460 5461multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps, 5462 SDPatternOperator op = null_frag> { 5463 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5464 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5465 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5466 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5467 let Inst{8} = imm{3}; 5468 } 5469 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5470 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5471 let Inst{9-8} = imm{4-3}; 5472 } 5473 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5474 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5475 let Inst{22} = imm{5}; 5476 let Inst{9-8} = imm{4-3}; 5477 } 5478 5479 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 5480 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 5481 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 5482 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 5483} 5484 5485// As above but shift amount takes the form of a "vector immediate". 5486multiclass sve_int_bin_pred_shift_imm_left_dup<bits<4> opc, string asm, 5487 string Ps, SDPatternOperator op> 5488: sve_int_bin_pred_shift_imm_left<opc, asm, Ps, null_frag> { 5489 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5490 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5491 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5492 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5493} 5494 5495multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> { 5496 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, tvecshiftL8, FalseLanesZero>; 5497 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>; 5498 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>; 5499 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>; 5500 5501 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8, !cast<Pseudo>(NAME # _ZERO_B)>; 5502 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>; 5503 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>; 5504 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>; 5505} 5506 5507multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps, 5508 SDPatternOperator op = null_frag> { 5509 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5510 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5511 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5512 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5513 let Inst{8} = imm{3}; 5514 } 5515 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5516 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5517 let Inst{9-8} = imm{4-3}; 5518 } 5519 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5520 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5521 let Inst{22} = imm{5}; 5522 let Inst{9-8} = imm{4-3}; 5523 } 5524 5525 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 5526 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 5527 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 5528 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 5529} 5530 5531// As above but shift amount takes the form of a "vector immediate". 5532multiclass sve_int_bin_pred_shift_imm_right_dup<bits<4> opc, string asm, 5533 string Ps, SDPatternOperator op> 5534: sve_int_bin_pred_shift_imm_right<opc, asm, Ps, null_frag> { 5535 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5536 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5537 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5538 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5539} 5540 5541multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> { 5542 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>; 5543 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>; 5544 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>; 5545 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>; 5546 5547 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftR8, !cast<Pseudo>(NAME # _ZERO_B)>; 5548 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftR16, !cast<Pseudo>(NAME # _ZERO_H)>; 5549 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftR32, !cast<Pseudo>(NAME # _ZERO_S)>; 5550 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftR64, !cast<Pseudo>(NAME # _ZERO_D)>; 5551} 5552 5553class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc, 5554 string asm, ZPRRegOp zprty, ZPRRegOp zprty2> 5555: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm), 5556 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 5557 "", 5558 []>, Sched<[]> { 5559 bits<3> Pg; 5560 bits<5> Zdn; 5561 bits<5> Zm; 5562 let Inst{31-24} = 0b00000100; 5563 let Inst{23-22} = sz8_64; 5564 let Inst{21-20} = 0b01; 5565 let Inst{19} = wide; 5566 let Inst{18-16} = opc; 5567 let Inst{15-13} = 0b100; 5568 let Inst{12-10} = Pg; 5569 let Inst{9-5} = Zm; 5570 let Inst{4-0} = Zdn; 5571 5572 let Constraints = "$Zdn = $_Zdn"; 5573 let DestructiveInstType = DestructiveOther; 5574 let ElementSize = zprty.ElementSize; 5575} 5576 5577multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps, 5578 SDPatternOperator op, string revname, bit isReverseInstr = 0> { 5579 let DestructiveInstType = DestructiveBinaryCommWithRev in { 5580 def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>, 5581 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 5582 def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>, 5583 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 5584 def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>, 5585 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 5586 def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>, 5587 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 5588 } 5589 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5590 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5591 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5592 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5593} 5594 5595multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> { 5596 def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>; 5597 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 5598 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 5599 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 5600 5601 def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _ZERO_B)>; 5602 def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _ZERO_H)>; 5603 def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _ZERO_S)>; 5604 def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _ZERO_D)>; 5605} 5606 5607multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm, 5608 SDPatternOperator op> { 5609 def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>; 5610 def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>; 5611 def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>; 5612 5613 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5614 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5615 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5616} 5617 5618//===----------------------------------------------------------------------===// 5619// SVE Shift - Unpredicated Group 5620//===----------------------------------------------------------------------===// 5621 5622class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm, 5623 ZPRRegOp zprty> 5624: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm), 5625 asm, "\t$Zd, $Zn, $Zm", 5626 "", 5627 []>, Sched<[]> { 5628 bits<5> Zd; 5629 bits<5> Zm; 5630 bits<5> Zn; 5631 let Inst{31-24} = 0b00000100; 5632 let Inst{23-22} = sz8_64; 5633 let Inst{21} = 0b1; 5634 let Inst{20-16} = Zm; 5635 let Inst{15-12} = 0b1000; 5636 let Inst{11-10} = opc; 5637 let Inst{9-5} = Zn; 5638 let Inst{4-0} = Zd; 5639} 5640 5641multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm, SDPatternOperator op> { 5642 def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>; 5643 def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>; 5644 def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>; 5645 5646 def : SVE_2_Op_Pred_All_Active<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5647 def : SVE_2_Op_Pred_All_Active<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5648 def : SVE_2_Op_Pred_All_Active<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5649} 5650 5651class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 5652 ZPRRegOp zprty, Operand immtype> 5653: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm), 5654 asm, "\t$Zd, $Zn, $imm", 5655 "", 5656 []>, Sched<[]> { 5657 bits<5> Zd; 5658 bits<5> Zn; 5659 bits<6> imm; 5660 let Inst{31-24} = 0b00000100; 5661 let Inst{23-22} = tsz8_64{3-2}; 5662 let Inst{21} = 0b1; 5663 let Inst{20-19} = tsz8_64{1-0}; 5664 let Inst{18-16} = imm{2-0}; // imm3 5665 let Inst{15-12} = 0b1001; 5666 let Inst{11-10} = opc; 5667 let Inst{9-5} = Zn; 5668 let Inst{4-0} = Zd; 5669} 5670 5671multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm, 5672 SDPatternOperator op> { 5673 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5674 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5675 let Inst{19} = imm{3}; 5676 } 5677 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5678 let Inst{20-19} = imm{4-3}; 5679 } 5680 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5681 let Inst{22} = imm{5}; 5682 let Inst{20-19} = imm{4-3}; 5683 } 5684 5685 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5686 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5687 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5688 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5689} 5690 5691multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm, 5692 SDPatternOperator op> { 5693 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5694 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5695 let Inst{19} = imm{3}; 5696 } 5697 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5698 let Inst{20-19} = imm{4-3}; 5699 } 5700 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5701 let Inst{22} = imm{5}; 5702 let Inst{20-19} = imm{4-3}; 5703 } 5704 5705 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5706 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5707 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5708 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5709} 5710 5711//===----------------------------------------------------------------------===// 5712// SVE Memory - Store Group 5713//===----------------------------------------------------------------------===// 5714 5715class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5716 RegisterOperand VecList> 5717: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5718 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5719 "", 5720 []>, Sched<[]> { 5721 bits<3> Pg; 5722 bits<5> Rn; 5723 bits<5> Zt; 5724 bits<4> imm4; 5725 let Inst{31-25} = 0b1110010; 5726 let Inst{24-23} = msz; 5727 let Inst{22-21} = esz; 5728 let Inst{20} = 0; 5729 let Inst{19-16} = imm4; 5730 let Inst{15-13} = 0b111; 5731 let Inst{12-10} = Pg; 5732 let Inst{9-5} = Rn; 5733 let Inst{4-0} = Zt; 5734 5735 let mayStore = 1; 5736} 5737 5738multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5739 RegisterOperand listty, ZPRRegOp zprty> 5740{ 5741 def NAME : sve_mem_cst_si<msz, esz, asm, listty>; 5742 5743 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5744 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5745 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5746 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5747 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5748 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5749} 5750 5751class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5752 string asm, Operand immtype> 5753: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 5754 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5755 "", 5756 []>, Sched<[]> { 5757 bits<3> Pg; 5758 bits<5> Rn; 5759 bits<5> Zt; 5760 bits<4> imm4; 5761 let Inst{31-25} = 0b1110010; 5762 let Inst{24-23} = sz; 5763 let Inst{22-21} = nregs; 5764 let Inst{20} = 1; 5765 let Inst{19-16} = imm4; 5766 let Inst{15-13} = 0b111; 5767 let Inst{12-10} = Pg; 5768 let Inst{9-5} = Rn; 5769 let Inst{4-0} = Zt; 5770 5771 let mayStore = 1; 5772} 5773 5774multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5775 string asm, Operand immtype> { 5776 def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>; 5777 5778 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5779 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5780} 5781 5782class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5783 string asm, RegisterOperand gprty> 5784: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5785 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5786 "", 5787 []>, Sched<[]> { 5788 bits<3> Pg; 5789 bits<5> Rm; 5790 bits<5> Rn; 5791 bits<5> Zt; 5792 let Inst{31-25} = 0b1110010; 5793 let Inst{24-23} = sz; 5794 let Inst{22-21} = nregs; 5795 let Inst{20-16} = Rm; 5796 let Inst{15-13} = 0b011; 5797 let Inst{12-10} = Pg; 5798 let Inst{9-5} = Rn; 5799 let Inst{4-0} = Zt; 5800 5801 let mayStore = 1; 5802} 5803 5804class sve_mem_cst_ss_base<bits<4> dtype, string asm, 5805 RegisterOperand listty, RegisterOperand gprty> 5806: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5807 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5808 "", 5809 []>, Sched<[]> { 5810 bits<3> Pg; 5811 bits<5> Rm; 5812 bits<5> Rn; 5813 bits<5> Zt; 5814 let Inst{31-25} = 0b1110010; 5815 let Inst{24-21} = dtype; 5816 let Inst{20-16} = Rm; 5817 let Inst{15-13} = 0b010; 5818 let Inst{12-10} = Pg; 5819 let Inst{9-5} = Rn; 5820 let Inst{4-0} = Zt; 5821 5822 let mayStore = 1; 5823} 5824 5825multiclass sve_mem_cst_ss<bits<4> dtype, string asm, 5826 RegisterOperand listty, ZPRRegOp zprty, 5827 RegisterOperand gprty> { 5828 def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>; 5829 5830 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5831 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5832} 5833 5834class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList> 5835: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5836 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5837 "", 5838 []>, Sched<[]> { 5839 bits<3> Pg; 5840 bits<5> Rn; 5841 bits<5> Zt; 5842 bits<4> imm4; 5843 let Inst{31-25} = 0b1110010; 5844 let Inst{24-23} = msz; 5845 let Inst{22-20} = 0b001; 5846 let Inst{19-16} = imm4; 5847 let Inst{15-13} = 0b111; 5848 let Inst{12-10} = Pg; 5849 let Inst{9-5} = Rn; 5850 let Inst{4-0} = Zt; 5851 5852 let mayStore = 1; 5853} 5854 5855multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty, 5856 ZPRRegOp zprty> { 5857 def NAME : sve_mem_cstnt_si<msz, asm, listty>; 5858 5859 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5860 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5861 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5862 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5863 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5864 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5865} 5866 5867class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty, 5868 RegisterOperand gprty> 5869: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5870 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5871 "", 5872 []>, Sched<[]> { 5873 bits<3> Pg; 5874 bits<5> Rm; 5875 bits<5> Rn; 5876 bits<5> Zt; 5877 let Inst{31-25} = 0b1110010; 5878 let Inst{24-23} = msz; 5879 let Inst{22-21} = 0b00; 5880 let Inst{20-16} = Rm; 5881 let Inst{15-13} = 0b011; 5882 let Inst{12-10} = Pg; 5883 let Inst{9-5} = Rn; 5884 let Inst{4-0} = Zt; 5885 5886 let mayStore = 1; 5887} 5888 5889multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty, 5890 ZPRRegOp zprty, RegisterOperand gprty> { 5891 def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>; 5892 5893 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5894 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5895} 5896 5897class sve2_mem_sstnt_vs_base<bits<3> opc, string asm, 5898 RegisterOperand listty, ZPRRegOp zprty> 5899: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 5900 asm, "\t$Zt, $Pg, [$Zn, $Rm]", 5901 "", 5902 []>, Sched<[]> { 5903 bits<3> Pg; 5904 bits<5> Rm; 5905 bits<5> Zn; 5906 bits<5> Zt; 5907 let Inst{31-25} = 0b1110010; 5908 let Inst{24-22} = opc; 5909 let Inst{21} = 0b0; 5910 let Inst{20-16} = Rm; 5911 let Inst{15-13} = 0b001; 5912 let Inst{12-10} = Pg; 5913 let Inst{9-5} = Zn; 5914 let Inst{4-0} = Zt; 5915 5916 let mayStore = 1; 5917} 5918 5919multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm, 5920 SDPatternOperator op, 5921 ValueType vt> { 5922 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>; 5923 5924 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5925 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 5926 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5927 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 5928 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5929 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 5930 5931 def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt), 5932 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; 5933} 5934 5935multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm, 5936 SDPatternOperator op, 5937 ValueType vt> { 5938 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>; 5939 5940 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5941 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 5942 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5943 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 5944 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5945 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 5946 5947 def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt), 5948 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; 5949} 5950 5951class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm, 5952 RegisterOperand VecList, RegisterOperand zprext> 5953: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5954 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5955 "", 5956 []>, Sched<[]> { 5957 bits<3> Pg; 5958 bits<5> Rn; 5959 bits<5> Zm; 5960 bits<5> Zt; 5961 let Inst{31-25} = 0b1110010; 5962 let Inst{24-22} = opc; 5963 let Inst{21} = scaled; 5964 let Inst{20-16} = Zm; 5965 let Inst{15} = 0b1; 5966 let Inst{14} = xs; 5967 let Inst{13} = 0; 5968 let Inst{12-10} = Pg; 5969 let Inst{9-5} = Rn; 5970 let Inst{4-0} = Zt; 5971 5972 let mayStore = 1; 5973} 5974 5975multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm, 5976 SDPatternOperator sxtw_op, 5977 SDPatternOperator uxtw_op, 5978 RegisterOperand sxtw_opnd, 5979 RegisterOperand uxtw_opnd, 5980 ValueType vt > { 5981 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>; 5982 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>; 5983 5984 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5985 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5986 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5987 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5988 5989 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5990 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5991 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5992 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5993} 5994 5995multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm, 5996 SDPatternOperator sxtw_op, 5997 SDPatternOperator uxtw_op, 5998 RegisterOperand sxtw_opnd, 5999 RegisterOperand uxtw_opnd, 6000 ValueType vt > { 6001 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>; 6002 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>; 6003 6004 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6005 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6006 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6007 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6008 6009 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6010 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6011 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6012 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6013} 6014 6015multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm, 6016 SDPatternOperator sxtw_op, 6017 SDPatternOperator uxtw_op, 6018 RegisterOperand sxtw_opnd, 6019 RegisterOperand uxtw_opnd, 6020 ValueType vt> { 6021 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>; 6022 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>; 6023 6024 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6025 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6026 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6027 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6028 6029 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6030 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6031 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6032 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6033} 6034 6035multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm, 6036 SDPatternOperator sxtw_op, 6037 SDPatternOperator uxtw_op, 6038 RegisterOperand sxtw_opnd, 6039 RegisterOperand uxtw_opnd, 6040 ValueType vt> { 6041 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>; 6042 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>; 6043 6044 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6045 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6046 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6047 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6048 6049 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6050 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6051 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6052 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6053} 6054 6055class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm, 6056 RegisterOperand zprext> 6057: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6058 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 6059 "", 6060 []>, Sched<[]> { 6061 bits<3> Pg; 6062 bits<5> Rn; 6063 bits<5> Zm; 6064 bits<5> Zt; 6065 let Inst{31-25} = 0b1110010; 6066 let Inst{24-23} = msz; 6067 let Inst{22} = 0b0; 6068 let Inst{21} = scaled; 6069 let Inst{20-16} = Zm; 6070 let Inst{15-13} = 0b101; 6071 let Inst{12-10} = Pg; 6072 let Inst{9-5} = Rn; 6073 let Inst{4-0} = Zt; 6074 6075 let mayStore = 1; 6076} 6077 6078multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm, 6079 SDPatternOperator op, 6080 RegisterOperand zprext, 6081 ValueType vt> { 6082 def _SCALED : sve_mem_sst_sv2<msz, 1, asm, zprext>; 6083 6084 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6085 (!cast<Instruction>(NAME # _SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 6086 6087 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt), 6088 (!cast<Instruction>(NAME # _SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6089} 6090 6091multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm, 6092 SDPatternOperator op, 6093 ValueType vt> { 6094 def NAME : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>; 6095 6096 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6097 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 6098 6099 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6100 (!cast<Instruction>(NAME) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6101} 6102 6103class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty, 6104 RegisterOperand VecList, Operand imm_ty> 6105: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5), 6106 asm, "\t$Zt, $Pg, [$Zn, $imm5]", 6107 "", 6108 []>, Sched<[]> { 6109 bits<3> Pg; 6110 bits<5> imm5; 6111 bits<5> Zn; 6112 bits<5> Zt; 6113 let Inst{31-25} = 0b1110010; 6114 let Inst{24-23} = opc{2-1}; 6115 let Inst{22} = 0b1; 6116 let Inst{21} = opc{0}; 6117 let Inst{20-16} = imm5; 6118 let Inst{15-13} = 0b101; 6119 let Inst{12-10} = Pg; 6120 let Inst{9-5} = Zn; 6121 let Inst{4-0} = Zt; 6122 6123 let mayStore = 1; 6124} 6125 6126multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm, 6127 Operand imm_ty, 6128 SDPatternOperator op, 6129 ValueType vt> { 6130 def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>; 6131 6132 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6133 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 6134 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 6135 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 6136 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6137 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 6138 6139 def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt), 6140 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6141} 6142 6143multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm, 6144 Operand imm_ty, 6145 SDPatternOperator op, 6146 ValueType vt> { 6147 def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>; 6148 6149 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6150 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 6151 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 6152 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 6153 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6154 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 6155 6156 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt), 6157 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6158} 6159 6160class sve_mem_z_spill<string asm> 6161: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9), 6162 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 6163 "", 6164 []>, Sched<[]> { 6165 bits<5> Rn; 6166 bits<5> Zt; 6167 bits<9> imm9; 6168 let Inst{31-22} = 0b1110010110; 6169 let Inst{21-16} = imm9{8-3}; 6170 let Inst{15-13} = 0b010; 6171 let Inst{12-10} = imm9{2-0}; 6172 let Inst{9-5} = Rn; 6173 let Inst{4-0} = Zt; 6174 6175 let mayStore = 1; 6176} 6177 6178multiclass sve_mem_z_spill<string asm> { 6179 def NAME : sve_mem_z_spill<asm>; 6180 6181 def : InstAlias<asm # "\t$Zt, [$Rn]", 6182 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 6183} 6184 6185class sve_mem_p_spill<string asm> 6186: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9), 6187 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 6188 "", 6189 []>, Sched<[]> { 6190 bits<4> Pt; 6191 bits<5> Rn; 6192 bits<9> imm9; 6193 let Inst{31-22} = 0b1110010110; 6194 let Inst{21-16} = imm9{8-3}; 6195 let Inst{15-13} = 0b000; 6196 let Inst{12-10} = imm9{2-0}; 6197 let Inst{9-5} = Rn; 6198 let Inst{4} = 0b0; 6199 let Inst{3-0} = Pt; 6200 6201 let mayStore = 1; 6202} 6203 6204multiclass sve_mem_p_spill<string asm> { 6205 def NAME : sve_mem_p_spill<asm>; 6206 6207 def : InstAlias<asm # "\t$Pt, [$Rn]", 6208 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 6209} 6210 6211//===----------------------------------------------------------------------===// 6212// SVE Permute - Predicates Group 6213//===----------------------------------------------------------------------===// 6214 6215class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, 6216 PPRRegOp pprty> 6217: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm), 6218 asm, "\t$Pd, $Pn, $Pm", 6219 "", []>, Sched<[]> { 6220 bits<4> Pd; 6221 bits<4> Pm; 6222 bits<4> Pn; 6223 let Inst{31-24} = 0b00000101; 6224 let Inst{23-22} = sz8_64; 6225 let Inst{21-20} = 0b10; 6226 let Inst{19-16} = Pm; 6227 let Inst{15-13} = 0b010; 6228 let Inst{12-10} = opc; 6229 let Inst{9} = 0b0; 6230 let Inst{8-5} = Pn; 6231 let Inst{4} = 0b0; 6232 let Inst{3-0} = Pd; 6233} 6234 6235multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm, 6236 SDPatternOperator op> { 6237 def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>; 6238 def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>; 6239 def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>; 6240 def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>; 6241 6242 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 6243 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 6244 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 6245 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 6246} 6247 6248class sve_int_perm_punpk<bit opc, string asm> 6249: I<(outs PPR16:$Pd), (ins PPR8:$Pn), 6250 asm, "\t$Pd, $Pn", 6251 "", 6252 []>, Sched<[]> { 6253 bits<4> Pd; 6254 bits<4> Pn; 6255 let Inst{31-17} = 0b000001010011000; 6256 let Inst{16} = opc; 6257 let Inst{15-9} = 0b0100000; 6258 let Inst{8-5} = Pn; 6259 let Inst{4} = 0b0; 6260 let Inst{3-0} = Pd; 6261} 6262 6263multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> { 6264 def NAME : sve_int_perm_punpk<opc, asm>; 6265 6266 def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>; 6267 def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1, !cast<Instruction>(NAME)>; 6268 def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1, !cast<Instruction>(NAME)>; 6269} 6270 6271class sve_int_rdffr_pred<bit s, string asm> 6272: I<(outs PPR8:$Pd), (ins PPRAny:$Pg), 6273 asm, "\t$Pd, $Pg/z", 6274 "", 6275 []>, Sched<[]> { 6276 bits<4> Pd; 6277 bits<4> Pg; 6278 let Inst{31-23} = 0b001001010; 6279 let Inst{22} = s; 6280 let Inst{21-9} = 0b0110001111000; 6281 let Inst{8-5} = Pg; 6282 let Inst{4} = 0; 6283 let Inst{3-0} = Pd; 6284 6285 let Defs = !if(s, [NZCV], []); 6286 let Uses = [FFR]; 6287} 6288 6289multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> { 6290 def _REAL : sve_int_rdffr_pred<s, asm>; 6291 6292 // We need a layer of indirection because early machine code passes balk at 6293 // physical register (i.e. FFR) uses that have no previous definition. 6294 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6295 def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, 6296 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; 6297 } 6298} 6299 6300class sve_int_rdffr_unpred<string asm> : I< 6301 (outs PPR8:$Pd), (ins), 6302 asm, "\t$Pd", 6303 "", 6304 []>, Sched<[]> { 6305 bits<4> Pd; 6306 let Inst{31-4} = 0b0010010100011001111100000000; 6307 let Inst{3-0} = Pd; 6308 6309 let Uses = [FFR]; 6310} 6311 6312multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> { 6313 def _REAL : sve_int_rdffr_unpred<asm>; 6314 6315 // We need a layer of indirection because early machine code passes balk at 6316 // physical register (i.e. FFR) uses that have no previous definition. 6317 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6318 def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, 6319 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>; 6320 } 6321} 6322 6323class sve_int_wrffr<string asm, SDPatternOperator op> 6324: I<(outs), (ins PPR8:$Pn), 6325 asm, "\t$Pn", 6326 "", 6327 [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 6328 bits<4> Pn; 6329 let Inst{31-9} = 0b00100101001010001001000; 6330 let Inst{8-5} = Pn; 6331 let Inst{4-0} = 0b00000; 6332 6333 let hasSideEffects = 1; 6334 let Defs = [FFR]; 6335} 6336 6337class sve_int_setffr<string asm, SDPatternOperator op> 6338: I<(outs), (ins), 6339 asm, "", 6340 "", 6341 [(op)]>, Sched<[]> { 6342 let Inst{31-0} = 0b00100101001011001001000000000000; 6343 6344 let hasSideEffects = 1; 6345 let Defs = [FFR]; 6346} 6347 6348//===----------------------------------------------------------------------===// 6349// SVE Permute Vector - Predicated Group 6350//===----------------------------------------------------------------------===// 6351 6352class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm, 6353 ZPRRegOp zprty, RegisterClass rt> 6354: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm), 6355 asm, "\t$Rdn, $Pg, $_Rdn, $Zm", 6356 "", 6357 []>, Sched<[]> { 6358 bits<3> Pg; 6359 bits<5> Rdn; 6360 bits<5> Zm; 6361 let Inst{31-24} = 0b00000101; 6362 let Inst{23-22} = sz8_64; 6363 let Inst{21-17} = 0b11000; 6364 let Inst{16} = ab; 6365 let Inst{15-13} = 0b101; 6366 let Inst{12-10} = Pg; 6367 let Inst{9-5} = Zm; 6368 let Inst{4-0} = Rdn; 6369 6370 let Constraints = "$Rdn = $_Rdn"; 6371} 6372 6373multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> { 6374 def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>; 6375 def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>; 6376 def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>; 6377 def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>; 6378 6379 def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>; 6380 def : SVE_3_Op_Pat<i32, op, nxv8i1, i32, nxv8i16, !cast<Instruction>(NAME # _H)>; 6381 def : SVE_3_Op_Pat<i32, op, nxv4i1, i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6382 def : SVE_3_Op_Pat<i64, op, nxv2i1, i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6383} 6384 6385class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm, 6386 ZPRRegOp zprty, RegisterClass rt> 6387: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm), 6388 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 6389 "", 6390 []>, Sched<[]> { 6391 bits<3> Pg; 6392 bits<5> Vdn; 6393 bits<5> Zm; 6394 let Inst{31-24} = 0b00000101; 6395 let Inst{23-22} = sz8_64; 6396 let Inst{21-17} = 0b10101; 6397 let Inst{16} = ab; 6398 let Inst{15-13} = 0b100; 6399 let Inst{12-10} = Pg; 6400 let Inst{9-5} = Zm; 6401 let Inst{4-0} = Vdn; 6402 6403 let Constraints = "$Vdn = $_Vdn"; 6404} 6405 6406multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> { 6407 def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>; 6408 def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>; 6409 def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>; 6410 def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>; 6411 6412 def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6413 def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6414 def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6415 6416 def : SVE_3_Op_Pat<bf16, op, nxv8i1, bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6417} 6418 6419class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm, 6420 ZPRRegOp zprty> 6421: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6422 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6423 "", 6424 []>, Sched<[]> { 6425 bits<3> Pg; 6426 bits<5> Zdn; 6427 bits<5> Zm; 6428 let Inst{31-24} = 0b00000101; 6429 let Inst{23-22} = sz8_64; 6430 let Inst{21-17} = 0b10100; 6431 let Inst{16} = ab; 6432 let Inst{15-13} = 0b100; 6433 let Inst{12-10} = Pg; 6434 let Inst{9-5} = Zm; 6435 let Inst{4-0} = Zdn; 6436 6437 let Constraints = "$Zdn = $_Zdn"; 6438 let DestructiveInstType = DestructiveOther; 6439 let ElementSize = ElementSizeNone; 6440} 6441 6442multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> { 6443 def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>; 6444 def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>; 6445 def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>; 6446 def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>; 6447 6448 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6449 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6450 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6451 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6452 6453 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6454 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6455 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6456 6457 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6458} 6459 6460class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm, 6461 ZPRRegOp zprty, RegisterClass resultRegType> 6462: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn), 6463 asm, "\t$Rd, $Pg, $Zn", 6464 "", 6465 []>, Sched<[]> { 6466 bits<3> Pg; 6467 bits<5> Rd; 6468 bits<5> Zn; 6469 let Inst{31-24} = 0b00000101; 6470 let Inst{23-22} = sz8_64; 6471 let Inst{21-17} = 0b10000; 6472 let Inst{16} = ab; 6473 let Inst{15-13} = 0b101; 6474 let Inst{12-10} = Pg; 6475 let Inst{9-5} = Zn; 6476 let Inst{4-0} = Rd; 6477} 6478 6479multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> { 6480 def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>; 6481 def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>; 6482 def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>; 6483 def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>; 6484 6485 def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6486 def : SVE_2_Op_Pat<i32, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6487 def : SVE_2_Op_Pat<i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6488 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6489} 6490 6491class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm, 6492 ZPRRegOp zprty, RegisterClass dstRegtype> 6493: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 6494 asm, "\t$Vd, $Pg, $Zn", 6495 "", 6496 []>, Sched<[]> { 6497 bits<3> Pg; 6498 bits<5> Vd; 6499 bits<5> Zn; 6500 let Inst{31-24} = 0b00000101; 6501 let Inst{23-22} = sz8_64; 6502 let Inst{21-17} = 0b10001; 6503 let Inst{16} = ab; 6504 let Inst{15-13} = 0b100; 6505 let Inst{12-10} = Pg; 6506 let Inst{9-5} = Zn; 6507 let Inst{4-0} = Vd; 6508} 6509 6510multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> { 6511 def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>; 6512 def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>; 6513 def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>; 6514 def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>; 6515 6516 def : SVE_2_Op_Pat<f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 6517 def : SVE_2_Op_Pat<f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6518 def : SVE_2_Op_Pat<f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 6519 def : SVE_2_Op_Pat<f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6520 6521 def : SVE_2_Op_Pat<bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6522} 6523 6524class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty> 6525: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6526 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6527 "", 6528 []>, Sched<[]> { 6529 bits<3> Pg; 6530 bits<5> Zdn; 6531 bits<5> Zm; 6532 let Inst{31-24} = 0b00000101; 6533 let Inst{23-22} = sz8_64; 6534 let Inst{21-13} = 0b101100100; 6535 let Inst{12-10} = Pg; 6536 let Inst{9-5} = Zm; 6537 let Inst{4-0} = Zdn; 6538 6539 let Constraints = "$Zdn = $_Zdn"; 6540 let DestructiveInstType = DestructiveOther; 6541 let ElementSize = ElementSizeNone; 6542} 6543 6544multiclass sve_int_perm_splice<string asm, SDPatternOperator op> { 6545 def _B : sve_int_perm_splice<0b00, asm, ZPR8>; 6546 def _H : sve_int_perm_splice<0b01, asm, ZPR16>; 6547 def _S : sve_int_perm_splice<0b10, asm, ZPR32>; 6548 def _D : sve_int_perm_splice<0b11, asm, ZPR64>; 6549 6550 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6551 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6552 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6553 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6554 6555 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6556 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6557 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6558 6559 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6560} 6561 6562class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm, 6563 ZPRRegOp zprty, RegisterOperand VecList> 6564: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn), 6565 asm, "\t$Zd, $Pg, $Zn", 6566 "", 6567 []>, Sched<[]> { 6568 bits<3> Pg; 6569 bits<5> Zn; 6570 bits<5> Zd; 6571 let Inst{31-24} = 0b00000101; 6572 let Inst{23-22} = sz8_64; 6573 let Inst{21-13} = 0b101101100; 6574 let Inst{12-10} = Pg; 6575 let Inst{9-5} = Zn; 6576 let Inst{4-0} = Zd; 6577} 6578 6579multiclass sve2_int_perm_splice_cons<string asm> { 6580 def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>; 6581 def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>; 6582 def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>; 6583 def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>; 6584} 6585 6586class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm, 6587 ZPRRegOp zprty> 6588: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 6589 asm, "\t$Zd, $Pg/m, $Zn", 6590 "", 6591 []>, Sched<[]> { 6592 bits<5> Zd; 6593 bits<3> Pg; 6594 bits<5> Zn; 6595 let Inst{31-24} = 0b00000101; 6596 let Inst{23-22} = sz8_64; 6597 let Inst{21-18} = 0b1001; 6598 let Inst{17-16} = opc; 6599 let Inst{15-13} = 0b100; 6600 let Inst{12-10} = Pg; 6601 let Inst{9-5} = Zn; 6602 let Inst{4-0} = Zd; 6603 6604 let Constraints = "$Zd = $_Zd"; 6605 let DestructiveInstType = DestructiveOther; 6606 let ElementSize = zprty.ElementSize; 6607} 6608 6609multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> { 6610 def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>; 6611 def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>; 6612 def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>; 6613 def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>; 6614 6615 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6616 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6617 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6618 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6619} 6620 6621multiclass sve_int_perm_rev_revb<string asm, SDPatternOperator op> { 6622 def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>; 6623 def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>; 6624 def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>; 6625 6626 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6627 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6628 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6629} 6630 6631multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> { 6632 def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>; 6633 def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>; 6634 6635 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6636 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6637} 6638 6639multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> { 6640 def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>; 6641 6642 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6643} 6644 6645class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6646 RegisterClass srcRegType> 6647: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn), 6648 asm, "\t$Zd, $Pg/m, $Rn", 6649 "", 6650 []>, Sched<[]> { 6651 bits<3> Pg; 6652 bits<5> Rn; 6653 bits<5> Zd; 6654 let Inst{31-24} = 0b00000101; 6655 let Inst{23-22} = sz8_64; 6656 let Inst{21-13} = 0b101000101; 6657 let Inst{12-10} = Pg; 6658 let Inst{9-5} = Rn; 6659 let Inst{4-0} = Zd; 6660 6661 let Constraints = "$Zd = $_Zd"; 6662 let DestructiveInstType = DestructiveOther; 6663 let ElementSize = zprty.ElementSize; 6664} 6665 6666multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> { 6667 def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>; 6668 def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>; 6669 def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>; 6670 def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>; 6671 6672 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6673 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6674 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6675 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6676 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6677 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6678 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6679 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>; 6680 6681 def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)), 6682 (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>; 6683 def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)), 6684 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6685 def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)), 6686 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6687 def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)), 6688 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6689} 6690 6691class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6692 RegisterClass srcRegtype> 6693: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn), 6694 asm, "\t$Zd, $Pg/m, $Vn", 6695 "", 6696 []>, Sched<[]> { 6697 bits<3> Pg; 6698 bits<5> Vn; 6699 bits<5> Zd; 6700 let Inst{31-24} = 0b00000101; 6701 let Inst{23-22} = sz8_64; 6702 let Inst{21-13} = 0b100000100; 6703 let Inst{12-10} = Pg; 6704 let Inst{9-5} = Vn; 6705 let Inst{4-0} = Zd; 6706 6707 let Constraints = "$Zd = $_Zd"; 6708 let DestructiveInstType = DestructiveOther; 6709 let ElementSize = zprty.ElementSize; 6710} 6711 6712multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> { 6713 def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>; 6714 def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>; 6715 def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>; 6716 def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>; 6717 6718 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6719 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>; 6720 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6721 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>; 6722 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6723 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>; 6724 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6725 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>; 6726 6727 def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)), 6728 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6729 def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)), 6730 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6731 def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)), 6732 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6733 def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)), 6734 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6735 6736 def : Pat<(nxv8bf16 (op nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)), 6737 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6738} 6739 6740class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty> 6741: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn), 6742 asm, "\t$Zd, $Pg, $Zn", 6743 "", 6744 []>, Sched<[]> { 6745 bits<3> Pg; 6746 bits<5> Zd; 6747 bits<5> Zn; 6748 let Inst{31-23} = 0b000001011; 6749 let Inst{22} = sz; 6750 let Inst{21-13} = 0b100001100; 6751 let Inst{12-10} = Pg; 6752 let Inst{9-5} = Zn; 6753 let Inst{4-0} = Zd; 6754} 6755 6756multiclass sve_int_perm_compact<string asm, SDPatternOperator op> { 6757 def _S : sve_int_perm_compact<0b0, asm, ZPR32>; 6758 def _D : sve_int_perm_compact<0b1, asm, ZPR64>; 6759 6760 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6761 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6762 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6763 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6764} 6765 6766//===----------------------------------------------------------------------===// 6767// SVE Memory - Contiguous Load Group 6768//===----------------------------------------------------------------------===// 6769 6770class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6771 RegisterOperand VecList> 6772: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6773 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6774 "", 6775 []>, Sched<[]> { 6776 bits<3> Pg; 6777 bits<5> Rn; 6778 bits<5> Zt; 6779 bits<4> imm4; 6780 let Inst{31-25} = 0b1010010; 6781 let Inst{24-21} = dtype; 6782 let Inst{20} = nf; 6783 let Inst{19-16} = imm4; 6784 let Inst{15-13} = 0b101; 6785 let Inst{12-10} = Pg; 6786 let Inst{9-5} = Rn; 6787 let Inst{4-0} = Zt; 6788 6789 let mayLoad = 1; 6790 let Uses = !if(nf, [FFR], []); 6791 let Defs = !if(nf, [FFR], []); 6792} 6793 6794multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6795 RegisterOperand listty, ZPRRegOp zprty> { 6796 def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>; 6797 6798 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6799 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6800 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6801 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6802 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6803 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6804 6805 // We need a layer of indirection because early machine code passes balk at 6806 // physical register (i.e. FFR) uses that have no previous definition. 6807 let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in { 6808 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>, 6809 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>; 6810 } 6811} 6812 6813multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty, 6814 ZPRRegOp zprty> 6815: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>; 6816 6817class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList> 6818: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6819 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6820 "", 6821 []>, Sched<[]> { 6822 bits<5> Zt; 6823 bits<3> Pg; 6824 bits<5> Rn; 6825 bits<4> imm4; 6826 let Inst{31-25} = 0b1010010; 6827 let Inst{24-23} = msz; 6828 let Inst{22-20} = 0b000; 6829 let Inst{19-16} = imm4; 6830 let Inst{15-13} = 0b111; 6831 let Inst{12-10} = Pg; 6832 let Inst{9-5} = Rn; 6833 let Inst{4-0} = Zt; 6834 6835 let mayLoad = 1; 6836} 6837 6838multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty, 6839 ZPRRegOp zprty> { 6840 def NAME : sve_mem_cldnt_si_base<msz, asm, listty>; 6841 6842 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6843 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6844 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6845 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6846 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6847 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6848} 6849 6850class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList, 6851 RegisterOperand gprty> 6852: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6853 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6854 "", 6855 []>, Sched<[]> { 6856 bits<3> Pg; 6857 bits<5> Rm; 6858 bits<5> Rn; 6859 bits<5> Zt; 6860 let Inst{31-25} = 0b1010010; 6861 let Inst{24-23} = msz; 6862 let Inst{22-21} = 0b00; 6863 let Inst{20-16} = Rm; 6864 let Inst{15-13} = 0b110; 6865 let Inst{12-10} = Pg; 6866 let Inst{9-5} = Rn; 6867 let Inst{4-0} = Zt; 6868 6869 let mayLoad = 1; 6870} 6871 6872multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty, 6873 ZPRRegOp zprty, RegisterOperand gprty> { 6874 def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>; 6875 6876 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6877 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6878} 6879 6880class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList> 6881: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 6882 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 6883 bits<5> Zt; 6884 bits<5> Rn; 6885 bits<3> Pg; 6886 bits<4> imm4; 6887 let Inst{31-25} = 0b1010010; 6888 let Inst{24-23} = sz; 6889 let Inst{22-20} = 0; 6890 let Inst{19-16} = imm4; 6891 let Inst{15-13} = 0b001; 6892 let Inst{12-10} = Pg; 6893 let Inst{9-5} = Rn; 6894 let Inst{4-0} = Zt; 6895 6896 let mayLoad = 1; 6897} 6898 6899multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty, 6900 ZPRRegOp zprty> { 6901 def NAME : sve_mem_ldqr_si<sz, asm, listty>; 6902 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6903 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6904 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6905 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6906 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 6907 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>; 6908} 6909 6910class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList, 6911 RegisterOperand gprty> 6912: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6913 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 6914 bits<5> Zt; 6915 bits<3> Pg; 6916 bits<5> Rn; 6917 bits<5> Rm; 6918 let Inst{31-25} = 0b1010010; 6919 let Inst{24-23} = sz; 6920 let Inst{22-21} = 0; 6921 let Inst{20-16} = Rm; 6922 let Inst{15-13} = 0; 6923 let Inst{12-10} = Pg; 6924 let Inst{9-5} = Rn; 6925 let Inst{4-0} = Zt; 6926 6927 let mayLoad = 1; 6928} 6929 6930multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty, 6931 ZPRRegOp zprty, RegisterOperand gprty> { 6932 def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>; 6933 6934 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6935 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6936} 6937 6938class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6939 RegisterOperand VecList, Operand immtype> 6940: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 6941 asm, "\t$Zt, $Pg/z, [$Rn, $imm6]", 6942 "", 6943 []>, Sched<[]> { 6944 bits<3> Pg; 6945 bits<5> Rn; 6946 bits<5> Zt; 6947 bits<6> imm6; 6948 let Inst{31-25} = 0b1000010; 6949 let Inst{24-23} = dtypeh; 6950 let Inst{22} = 1; 6951 let Inst{21-16} = imm6; 6952 let Inst{15} = 0b1; 6953 let Inst{14-13} = dtypel; 6954 let Inst{12-10} = Pg; 6955 let Inst{9-5} = Rn; 6956 let Inst{4-0} = Zt; 6957 6958 let mayLoad = 1; 6959} 6960 6961multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6962 RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> { 6963 def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>; 6964 6965 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6966 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6967 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]", 6968 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>; 6969 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6970 (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6971} 6972 6973class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm, 6974 RegisterOperand VecList> 6975: I<(outs VecList:$Zt), iops, 6976 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6977 "", 6978 []>, Sched<[]> { 6979 bits<5> Zt; 6980 bits<3> Pg; 6981 bits<5> Rm; 6982 bits<5> Rn; 6983 let Inst{31-25} = 0b1010010; 6984 let Inst{24-21} = dtype; 6985 let Inst{20-16} = Rm; 6986 let Inst{15-14} = 0b01; 6987 let Inst{13} = ff; 6988 let Inst{12-10} = Pg; 6989 let Inst{9-5} = Rn; 6990 let Inst{4-0} = Zt; 6991 6992 let mayLoad = 1; 6993 let Uses = !if(ff, [FFR], []); 6994 let Defs = !if(ff, [FFR], []); 6995} 6996 6997multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty, 6998 ZPRRegOp zprty, RegisterOperand gprty> { 6999 def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7000 asm, listty>; 7001 7002 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 7003 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 7004} 7005 7006multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty, 7007 ZPRRegOp zprty, RegisterOperand gprty> { 7008 def _REAL : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7009 asm, listty>; 7010 7011 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 7012 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 7013 7014 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7015 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 7016 7017 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7018 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 7019 7020 // We need a layer of indirection because early machine code passes balk at 7021 // physical register (i.e. FFR) uses that have no previous definition. 7022 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7023 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>, 7024 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>; 7025 } 7026} 7027 7028multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty, 7029 ZPRRegOp zprty> 7030: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>; 7031 7032class sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 7033 string asm, Operand immtype> 7034: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 7035 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 7036 "", 7037 []>, Sched<[]> { 7038 bits<5> Zt; 7039 bits<3> Pg; 7040 bits<5> Rn; 7041 bits<4> imm4; 7042 let Inst{31-25} = 0b1010010; 7043 let Inst{24-23} = sz; 7044 let Inst{22-21} = nregs; 7045 let Inst{20} = 0; 7046 let Inst{19-16} = imm4; 7047 let Inst{15-13} = 0b111; 7048 let Inst{12-10} = Pg; 7049 let Inst{9-5} = Rn; 7050 let Inst{4-0} = Zt; 7051 7052 let mayLoad = 1; 7053} 7054 7055multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 7056 string asm, Operand immtype> { 7057 def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>; 7058 7059 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7060 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7061} 7062 7063class sve_mem_eld_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 7064 string asm, RegisterOperand gprty> 7065: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7066 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 7067 "", 7068 []>, Sched<[]> { 7069 bits<3> Pg; 7070 bits<5> Rm; 7071 bits<5> Rn; 7072 bits<5> Zt; 7073 let Inst{31-25} = 0b1010010; 7074 let Inst{24-23} = sz; 7075 let Inst{22-21} = nregs; 7076 let Inst{20-16} = Rm; 7077 let Inst{15-13} = 0b110; 7078 let Inst{12-10} = Pg; 7079 let Inst{9-5} = Rn; 7080 let Inst{4-0} = Zt; 7081 7082 let mayLoad = 1; 7083} 7084 7085//===----------------------------------------------------------------------===// 7086// SVE Memory - 32-bit Gather and Unsized Contiguous Group 7087//===----------------------------------------------------------------------===// 7088 7089// bit xs is '1' if offsets are signed 7090// bit scaled is '1' if the offsets are scaled 7091class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm, 7092 RegisterOperand zprext> 7093: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7094 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7095 "", 7096 []>, Sched<[]> { 7097 bits<3> Pg; 7098 bits<5> Rn; 7099 bits<5> Zm; 7100 bits<5> Zt; 7101 let Inst{31-25} = 0b1000010; 7102 let Inst{24-23} = opc{3-2}; 7103 let Inst{22} = xs; 7104 let Inst{21} = scaled; 7105 let Inst{20-16} = Zm; 7106 let Inst{15} = 0b0; 7107 let Inst{14-13} = opc{1-0}; 7108 let Inst{12-10} = Pg; 7109 let Inst{9-5} = Rn; 7110 let Inst{4-0} = Zt; 7111 7112 let mayLoad = 1; 7113 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7114 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7115} 7116 7117multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm, 7118 SDPatternOperator sxtw_op, 7119 SDPatternOperator uxtw_op, 7120 RegisterOperand sxtw_opnd, 7121 RegisterOperand uxtw_opnd, 7122 ValueType vt> { 7123 def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>; 7124 def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>; 7125 7126 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7127 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7128 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7129 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7130 7131 // We need a layer of indirection because early machine code passes balk at 7132 // physical register (i.e. FFR) uses that have no previous definition. 7133 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7134 def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7135 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7136 def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7137 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7138 } 7139 7140 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 7141 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7142 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 7143 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7144} 7145 7146multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm, 7147 SDPatternOperator sxtw_op, 7148 SDPatternOperator uxtw_op, 7149 RegisterOperand sxtw_opnd, 7150 RegisterOperand uxtw_opnd, 7151 ValueType vt> { 7152 def _UXTW_REAL : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>; 7153 def _SXTW_REAL : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>; 7154 7155 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7156 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7157 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7158 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7159 7160 // We need a layer of indirection because early machine code passes balk at 7161 // physical register (i.e. FFR) uses that have no previous definition. 7162 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7163 def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7164 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7165 def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7166 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7167 } 7168 7169 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7170 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7171 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7172 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7173} 7174 7175 7176class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7177: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7178 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7179 "", 7180 []>, Sched<[]> { 7181 bits<3> Pg; 7182 bits<5> Zn; 7183 bits<5> Zt; 7184 bits<5> imm5; 7185 let Inst{31-25} = 0b1000010; 7186 let Inst{24-23} = opc{3-2}; 7187 let Inst{22-21} = 0b01; 7188 let Inst{20-16} = imm5; 7189 let Inst{15} = 0b1; 7190 let Inst{14-13} = opc{1-0}; 7191 let Inst{12-10} = Pg; 7192 let Inst{9-5} = Zn; 7193 let Inst{4-0} = Zt; 7194 7195 let mayLoad = 1; 7196 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7197 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7198} 7199 7200multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty, 7201 SDPatternOperator op, ValueType vt> { 7202 def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>; 7203 7204 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7205 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 7206 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7207 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 7208 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7209 (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7210 7211 // We need a layer of indirection because early machine code passes balk at 7212 // physical register (i.e. FFR) uses that have no previous definition. 7213 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7214 def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>, 7215 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>; 7216 } 7217 7218 def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), 7219 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7220} 7221 7222class sve_mem_prfm_si<bits<2> msz, string asm> 7223: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6), 7224 asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]", 7225 "", 7226 []>, Sched<[]> { 7227 bits<5> Rn; 7228 bits<3> Pg; 7229 bits<6> imm6; 7230 bits<4> prfop; 7231 let Inst{31-22} = 0b1000010111; 7232 let Inst{21-16} = imm6; 7233 let Inst{15} = 0b0; 7234 let Inst{14-13} = msz; 7235 let Inst{12-10} = Pg; 7236 let Inst{9-5} = Rn; 7237 let Inst{4} = 0b0; 7238 let Inst{3-0} = prfop; 7239 7240 let hasSideEffects = 1; 7241} 7242 7243multiclass sve_mem_prfm_si<bits<2> msz, string asm> { 7244 def NAME : sve_mem_prfm_si<msz, asm>; 7245 7246 def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]", 7247 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7248} 7249 7250class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty> 7251: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7252 asm, "\t$prfop, $Pg, [$Rn, $Rm]", 7253 "", 7254 []>, Sched<[]> { 7255 bits<5> Rm; 7256 bits<5> Rn; 7257 bits<3> Pg; 7258 bits<4> prfop; 7259 let Inst{31-25} = 0b1000010; 7260 let Inst{24-23} = opc{2-1}; 7261 let Inst{22-21} = 0b00; 7262 let Inst{20-16} = Rm; 7263 let Inst{15} = 0b1; 7264 let Inst{14} = opc{0}; 7265 let Inst{13} = 0b0; 7266 let Inst{12-10} = Pg; 7267 let Inst{9-5} = Rn; 7268 let Inst{4} = 0b0; 7269 let Inst{3-0} = prfop; 7270 7271 let hasSideEffects = 1; 7272} 7273 7274class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm, 7275 RegisterOperand zprext> 7276: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7277 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7278 "", 7279 []>, Sched<[]> { 7280 bits<3> Pg; 7281 bits<5> Rn; 7282 bits<5> Zm; 7283 bits<4> prfop; 7284 let Inst{31-23} = 0b100001000; 7285 let Inst{22} = xs; 7286 let Inst{21} = 0b1; 7287 let Inst{20-16} = Zm; 7288 let Inst{15} = 0b0; 7289 let Inst{14-13} = msz; 7290 let Inst{12-10} = Pg; 7291 let Inst{9-5} = Rn; 7292 let Inst{4} = 0b0; 7293 let Inst{3-0} = prfop; 7294 7295 let hasSideEffects = 1; 7296} 7297 7298multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm, 7299 RegisterOperand sxtw_opnd, 7300 RegisterOperand uxtw_opnd, 7301 SDPatternOperator op_sxtw, 7302 SDPatternOperator op_uxtw> { 7303 def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>; 7304 def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>; 7305 7306 def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7307 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7308 7309 def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7310 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7311} 7312 7313class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7314: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7315 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7316 "", 7317 []>, Sched<[]> { 7318 bits<3> Pg; 7319 bits<5> Zn; 7320 bits<5> imm5; 7321 bits<4> prfop; 7322 let Inst{31-25} = 0b1000010; 7323 let Inst{24-23} = msz; 7324 let Inst{22-21} = 0b00; 7325 let Inst{20-16} = imm5; 7326 let Inst{15-13} = 0b111; 7327 let Inst{12-10} = Pg; 7328 let Inst{9-5} = Zn; 7329 let Inst{4} = 0b0; 7330 let Inst{3-0} = prfop; 7331} 7332 7333multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7334 def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>; 7335 7336 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7337 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7338 7339 def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7340 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7341} 7342 7343class sve_mem_z_fill<string asm> 7344: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9), 7345 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 7346 "", 7347 []>, Sched<[]> { 7348 bits<5> Rn; 7349 bits<5> Zt; 7350 bits<9> imm9; 7351 let Inst{31-22} = 0b1000010110; 7352 let Inst{21-16} = imm9{8-3}; 7353 let Inst{15-13} = 0b010; 7354 let Inst{12-10} = imm9{2-0}; 7355 let Inst{9-5} = Rn; 7356 let Inst{4-0} = Zt; 7357 7358 let mayLoad = 1; 7359} 7360 7361multiclass sve_mem_z_fill<string asm> { 7362 def NAME : sve_mem_z_fill<asm>; 7363 7364 def : InstAlias<asm # "\t$Zt, [$Rn]", 7365 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 7366} 7367 7368class sve_mem_p_fill<string asm> 7369: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9), 7370 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 7371 "", 7372 []>, Sched<[]> { 7373 bits<4> Pt; 7374 bits<5> Rn; 7375 bits<9> imm9; 7376 let Inst{31-22} = 0b1000010110; 7377 let Inst{21-16} = imm9{8-3}; 7378 let Inst{15-13} = 0b000; 7379 let Inst{12-10} = imm9{2-0}; 7380 let Inst{9-5} = Rn; 7381 let Inst{4} = 0b0; 7382 let Inst{3-0} = Pt; 7383 7384 let mayLoad = 1; 7385} 7386 7387multiclass sve_mem_p_fill<string asm> { 7388 def NAME : sve_mem_p_fill<asm>; 7389 7390 def : InstAlias<asm # "\t$Pt, [$Rn]", 7391 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 7392} 7393 7394class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm, 7395 RegisterOperand VecList> 7396: I<(outs VecList:$Zt), iops, 7397 asm, "\t$Zt, $Pg/z, [$Zn, $Rm]", 7398 "", 7399 []>, Sched<[]> { 7400 bits<3> Pg; 7401 bits<5> Rm; 7402 bits<5> Zn; 7403 bits<5> Zt; 7404 let Inst{31} = 0b1; 7405 let Inst{30} = opc{4}; 7406 let Inst{29-25} = 0b00010; 7407 let Inst{24-23} = opc{3-2}; 7408 let Inst{22-21} = 0b00; 7409 let Inst{20-16} = Rm; 7410 let Inst{15} = 0b1; 7411 let Inst{14-13} = opc{1-0}; 7412 let Inst{12-10} = Pg; 7413 let Inst{9-5} = Zn; 7414 let Inst{4-0} = Zt; 7415 7416 let mayLoad = 1; 7417} 7418 7419multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm, 7420 SDPatternOperator op, 7421 ValueType vt> { 7422 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 7423 asm, Z_s>; 7424 7425 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7426 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 7427 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7428 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 7429 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7430 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 7431 7432 def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)), 7433 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; 7434} 7435 7436multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm, 7437 SDPatternOperator op, 7438 ValueType vt> { 7439 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 7440 asm, Z_d>; 7441 7442 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7443 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 7444 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7445 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 7446 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7447 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 7448 7449 def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)), 7450 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; 7451} 7452 7453//===----------------------------------------------------------------------===// 7454// SVE Memory - 64-bit Gather Group 7455//===----------------------------------------------------------------------===// 7456 7457// bit xs is '1' if offsets are signed 7458// bit scaled is '1' if the offsets are scaled 7459// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7460class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm, 7461 RegisterOperand zprext> 7462: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7463 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7464 "", 7465 []>, Sched<[]> { 7466 bits<3> Pg; 7467 bits<5> Rn; 7468 bits<5> Zm; 7469 bits<5> Zt; 7470 let Inst{31-25} = 0b1100010; 7471 let Inst{24-23} = opc{3-2}; 7472 let Inst{22} = xs; 7473 let Inst{21} = scaled; 7474 let Inst{20-16} = Zm; 7475 let Inst{15} = lsl; 7476 let Inst{14-13} = opc{1-0}; 7477 let Inst{12-10} = Pg; 7478 let Inst{9-5} = Rn; 7479 let Inst{4-0} = Zt; 7480 7481 let mayLoad = 1; 7482 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7483 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7484} 7485 7486multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm, 7487 SDPatternOperator sxtw_op, 7488 SDPatternOperator uxtw_op, 7489 RegisterOperand sxtw_opnd, 7490 RegisterOperand uxtw_opnd, 7491 ValueType vt> { 7492 def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>; 7493 def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>; 7494 7495 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7496 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7497 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7498 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7499 7500 // We need a layer of indirection because early machine code passes balk at 7501 // physical register (i.e. FFR) uses that have no previous definition. 7502 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7503 def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7504 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7505 def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7506 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7507 } 7508 7509 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7510 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7511 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7512 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7513} 7514 7515multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm, 7516 SDPatternOperator sxtw_op, 7517 SDPatternOperator uxtw_op, 7518 RegisterOperand sxtw_opnd, 7519 RegisterOperand uxtw_opnd, 7520 ValueType vt> { 7521 def _UXTW_REAL : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>; 7522 def _SXTW_REAL : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>; 7523 7524 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7525 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7526 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7527 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7528 7529 // We need a layer of indirection because early machine code passes balk at 7530 // physical register (i.e. FFR) uses that have no previous definition. 7531 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7532 def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7533 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7534 def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7535 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7536 } 7537 7538 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7539 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7540 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7541 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7542} 7543 7544multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm, 7545 SDPatternOperator op, 7546 RegisterOperand zprext, ValueType vt> { 7547 def _SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>; 7548 7549 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7550 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 7551 7552 // We need a layer of indirection because early machine code passes balk at 7553 // physical register (i.e. FFR) uses that have no previous definition. 7554 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7555 def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>, 7556 PseudoInstExpansion<(!cast<Instruction>(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7557 } 7558 7559 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7560 (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7561} 7562 7563multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm, 7564 SDPatternOperator op, ValueType vt> { 7565 def _REAL : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>; 7566 7567 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7568 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 7569 7570 // We need a layer of indirection because early machine code passes balk at 7571 // physical register (i.e. FFR) uses that have no previous definition. 7572 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7573 def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>, 7574 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>; 7575 } 7576 7577 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7578 (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7579} 7580 7581class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7582: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7583 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7584 "", 7585 []>, Sched<[]> { 7586 bits<3> Pg; 7587 bits<5> Zn; 7588 bits<5> Zt; 7589 bits<5> imm5; 7590 let Inst{31-25} = 0b1100010; 7591 let Inst{24-23} = opc{3-2}; 7592 let Inst{22-21} = 0b01; 7593 let Inst{20-16} = imm5; 7594 let Inst{15} = 0b1; 7595 let Inst{14-13} = opc{1-0}; 7596 let Inst{12-10} = Pg; 7597 let Inst{9-5} = Zn; 7598 let Inst{4-0} = Zt; 7599 7600 let mayLoad = 1; 7601 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7602 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7603} 7604 7605multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty, 7606 SDPatternOperator op, ValueType vt> { 7607 def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>; 7608 7609 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7610 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 7611 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7612 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 7613 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7614 (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7615 7616 // We need a layer of indirection because early machine code passes balk at 7617 // physical register (i.e. FFR) uses that have no previous definition. 7618 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7619 def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>, 7620 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>; 7621 } 7622 7623 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), 7624 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7625} 7626 7627// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7628class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm, 7629 RegisterOperand zprext> 7630: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7631 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7632 "", 7633 []>, Sched<[]> { 7634 bits<3> Pg; 7635 bits<5> Rn; 7636 bits<5> Zm; 7637 bits<4> prfop; 7638 let Inst{31-23} = 0b110001000; 7639 let Inst{22} = xs; 7640 let Inst{21} = 0b1; 7641 let Inst{20-16} = Zm; 7642 let Inst{15} = lsl; 7643 let Inst{14-13} = msz; 7644 let Inst{12-10} = Pg; 7645 let Inst{9-5} = Rn; 7646 let Inst{4} = 0b0; 7647 let Inst{3-0} = prfop; 7648 7649 let hasSideEffects = 1; 7650} 7651 7652multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm, 7653 RegisterOperand sxtw_opnd, 7654 RegisterOperand uxtw_opnd, 7655 SDPatternOperator op_sxtw, 7656 SDPatternOperator op_uxtw> { 7657 def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>; 7658 def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>; 7659 7660 def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7661 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7662 7663 def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7664 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7665 7666} 7667 7668multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm, 7669 RegisterOperand zprext, SDPatternOperator frag> { 7670 def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>; 7671 7672 def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)), 7673 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7674 7675} 7676 7677class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7678: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7679 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7680 "", 7681 []>, Sched<[]> { 7682 bits<3> Pg; 7683 bits<5> Zn; 7684 bits<5> imm5; 7685 bits<4> prfop; 7686 let Inst{31-25} = 0b1100010; 7687 let Inst{24-23} = msz; 7688 let Inst{22-21} = 0b00; 7689 let Inst{20-16} = imm5; 7690 let Inst{15-13} = 0b111; 7691 let Inst{12-10} = Pg; 7692 let Inst{9-5} = Zn; 7693 let Inst{4} = 0b0; 7694 let Inst{3-0} = prfop; 7695 7696 let hasSideEffects = 1; 7697} 7698 7699multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7700 def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>; 7701 7702 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7703 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7704 7705 def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7706 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7707} 7708 7709//===----------------------------------------------------------------------===// 7710// SVE Compute Vector Address Group 7711//===----------------------------------------------------------------------===// 7712 7713class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm, 7714 ZPRRegOp zprty, RegisterOperand zprext> 7715: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm), 7716 asm, "\t$Zd, [$Zn, $Zm]", 7717 "", 7718 []>, Sched<[]> { 7719 bits<5> Zd; 7720 bits<5> Zn; 7721 bits<5> Zm; 7722 let Inst{31-24} = 0b00000100; 7723 let Inst{23-22} = opc; 7724 let Inst{21} = 0b1; 7725 let Inst{20-16} = Zm; 7726 let Inst{15-12} = 0b1010; 7727 let Inst{11-10} = msz; 7728 let Inst{9-5} = Zn; 7729 let Inst{4-0} = Zd; 7730} 7731 7732multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> { 7733 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>; 7734 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>; 7735 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>; 7736 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>; 7737} 7738 7739multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> { 7740 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>; 7741 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>; 7742 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>; 7743 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>; 7744} 7745 7746multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> { 7747 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>; 7748 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>; 7749 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>; 7750 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>; 7751} 7752 7753multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> { 7754 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>; 7755 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>; 7756 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>; 7757 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>; 7758} 7759 7760//===----------------------------------------------------------------------===// 7761// SVE Integer Misc - Unpredicated Group 7762//===----------------------------------------------------------------------===// 7763 7764class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty> 7765: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7766 asm, "\t$Zd, $Zn, $Zm", 7767 "", 7768 []>, Sched<[]> { 7769 bits<5> Zd; 7770 bits<5> Zm; 7771 bits<5> Zn; 7772 let Inst{31-24} = 0b00000100; 7773 let Inst{23-22} = sz; 7774 let Inst{21} = 0b1; 7775 let Inst{20-16} = Zm; 7776 let Inst{15-10} = 0b101100; 7777 let Inst{9-5} = Zn; 7778 let Inst{4-0} = Zd; 7779} 7780 7781multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> { 7782 def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>; 7783 def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>; 7784 def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>; 7785 7786 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7787 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7788 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7789} 7790 7791class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty> 7792: I<(outs zprty:$Zd), (ins zprty:$Zn), 7793 asm, "\t$Zd, $Zn", 7794 "", 7795 []>, Sched<[]> { 7796 bits<5> Zd; 7797 bits<5> Zn; 7798 let Inst{31-24} = 0b00000100; 7799 let Inst{23-22} = opc{7-6}; 7800 let Inst{21} = 0b1; 7801 let Inst{20-16} = opc{5-1}; 7802 let Inst{15-11} = 0b10111; 7803 let Inst{10} = opc{0}; 7804 let Inst{9-5} = Zn; 7805 let Inst{4-0} = Zd; 7806} 7807 7808multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> { 7809 def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>; 7810 def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>; 7811 def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>; 7812 7813 def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 7814 def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 7815 def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 7816} 7817 7818//===----------------------------------------------------------------------===// 7819// SVE Integer Reduction Group 7820//===----------------------------------------------------------------------===// 7821 7822class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm, 7823 ZPRRegOp zprty, FPRasZPROperand dstOpType> 7824: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 7825 asm, "\t$Vd, $Pg, $Zn", 7826 "", 7827 []>, Sched<[]> { 7828 bits<3> Pg; 7829 bits<5> Vd; 7830 bits<5> Zn; 7831 let Inst{31-24} = 0b00000100; 7832 let Inst{23-22} = sz8_32; 7833 let Inst{21} = 0b0; 7834 let Inst{20-19} = fmt; 7835 let Inst{18-16} = opc; 7836 let Inst{15-13} = 0b001; 7837 let Inst{12-10} = Pg; 7838 let Inst{9-5} = Zn; 7839 let Inst{4-0} = Vd; 7840} 7841 7842multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm, 7843 SDPatternOperator op> { 7844 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7845 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7846 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7847 7848 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7849 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7850 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7851} 7852 7853multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm, 7854 SDPatternOperator op> { 7855 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7856 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7857 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7858 def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64asZPR>; 7859 7860 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7861 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7862 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7863 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7864} 7865 7866multiclass sve_int_reduce_1<bits<3> opc, string asm, 7867 SDPatternOperator op> { 7868 def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8asZPR>; 7869 def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16asZPR>; 7870 def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32asZPR>; 7871 def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64asZPR>; 7872 7873 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7874 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7875 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7876 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7877} 7878 7879multiclass sve_int_reduce_2<bits<3> opc, string asm, 7880 SDPatternOperator op> { 7881 def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8asZPR>; 7882 def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16asZPR>; 7883 def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32asZPR>; 7884 def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64asZPR>; 7885 7886 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7887 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7888 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7889 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7890} 7891 7892class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm, 7893 ZPRRegOp zprty, string pg_suffix, dag iops> 7894: I<(outs zprty:$Zd), iops, 7895 asm, "\t$Zd, $Pg"#pg_suffix#", $Zn", 7896 "", 7897 []>, Sched<[]> { 7898 bits<3> Pg; 7899 bits<5> Zd; 7900 bits<5> Zn; 7901 let Inst{31-24} = 0b00000100; 7902 let Inst{23-22} = sz8_32; 7903 let Inst{21-19} = 0b010; 7904 let Inst{18-16} = opc; 7905 let Inst{15-13} = 0b001; 7906 let Inst{12-10} = Pg; 7907 let Inst{9-5} = Zn; 7908 let Inst{4-0} = Zd; 7909 7910 let ElementSize = zprty.ElementSize; 7911} 7912 7913multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> { 7914let Constraints = "$Zd = $_Zd" in { 7915 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m", 7916 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>; 7917 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m", 7918 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>; 7919 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m", 7920 (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>; 7921 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m", 7922 (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>; 7923} 7924} 7925 7926multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> { 7927 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z", 7928 (ins PPR3bAny:$Pg, ZPR8:$Zn)>; 7929 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z", 7930 (ins PPR3bAny:$Pg, ZPR16:$Zn)>; 7931 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z", 7932 (ins PPR3bAny:$Pg, ZPR32:$Zn)>; 7933 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z", 7934 (ins PPR3bAny:$Pg, ZPR64:$Zn)>; 7935} 7936 7937//===----------------------------------------------------------------------===// 7938// SVE Propagate Break Group 7939//===----------------------------------------------------------------------===// 7940 7941class sve_int_brkp<bits<2> opc, string asm> 7942: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 7943 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 7944 "", 7945 []>, Sched<[]> { 7946 bits<4> Pd; 7947 bits<4> Pg; 7948 bits<4> Pm; 7949 bits<4> Pn; 7950 let Inst{31-24} = 0b00100101; 7951 let Inst{23} = 0b0; 7952 let Inst{22} = opc{1}; 7953 let Inst{21-20} = 0b00; 7954 let Inst{19-16} = Pm; 7955 let Inst{15-14} = 0b11; 7956 let Inst{13-10} = Pg; 7957 let Inst{9} = 0b0; 7958 let Inst{8-5} = Pn; 7959 let Inst{4} = opc{0}; 7960 let Inst{3-0} = Pd; 7961 7962 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7963} 7964 7965multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> { 7966 def NAME : sve_int_brkp<opc, asm>; 7967 7968 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7969} 7970 7971 7972//===----------------------------------------------------------------------===// 7973// SVE Partition Break Group 7974//===----------------------------------------------------------------------===// 7975 7976class sve_int_brkn<bit S, string asm> 7977: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm), 7978 asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm", 7979 "", 7980 []>, Sched<[]> { 7981 bits<4> Pdm; 7982 bits<4> Pg; 7983 bits<4> Pn; 7984 let Inst{31-23} = 0b001001010; 7985 let Inst{22} = S; 7986 let Inst{21-14} = 0b01100001; 7987 let Inst{13-10} = Pg; 7988 let Inst{9} = 0b0; 7989 let Inst{8-5} = Pn; 7990 let Inst{4} = 0b0; 7991 let Inst{3-0} = Pdm; 7992 7993 let Constraints = "$Pdm = $_Pdm"; 7994 let Defs = !if(S, [NZCV], []); 7995} 7996 7997multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> { 7998 def NAME : sve_int_brkn<opc, asm>; 7999 8000 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8001} 8002 8003class sve_int_break<bits<3> opc, string asm, string suffix, dag iops> 8004: I<(outs PPR8:$Pd), iops, 8005 asm, "\t$Pd, $Pg"#suffix#", $Pn", 8006 "", 8007 []>, Sched<[]> { 8008 bits<4> Pd; 8009 bits<4> Pg; 8010 bits<4> Pn; 8011 let Inst{31-24} = 0b00100101; 8012 let Inst{23-22} = opc{2-1}; 8013 let Inst{21-14} = 0b01000001; 8014 let Inst{13-10} = Pg; 8015 let Inst{9} = 0b0; 8016 let Inst{8-5} = Pn; 8017 let Inst{4} = opc{0}; 8018 let Inst{3-0} = Pd; 8019 8020 let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", ""); 8021 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 8022 8023} 8024 8025multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> { 8026 def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>; 8027 8028 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8029} 8030 8031multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> { 8032 def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>; 8033 8034 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8035} 8036 8037//===----------------------------------------------------------------------===// 8038// SVE2 String Processing Group 8039//===----------------------------------------------------------------------===// 8040 8041class sve2_char_match<bit sz, bit opc, string asm, 8042 PPRRegOp pprty, ZPRRegOp zprty> 8043: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 8044 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 8045 "", 8046 []>, Sched<[]> { 8047 bits<4> Pd; 8048 bits<3> Pg; 8049 bits<5> Zm; 8050 bits<5> Zn; 8051 let Inst{31-23} = 0b010001010; 8052 let Inst{22} = sz; 8053 let Inst{21} = 0b1; 8054 let Inst{20-16} = Zm; 8055 let Inst{15-13} = 0b100; 8056 let Inst{12-10} = Pg; 8057 let Inst{9-5} = Zn; 8058 let Inst{4} = opc; 8059 let Inst{3-0} = Pd; 8060 8061 let Defs = [NZCV]; 8062 let isPTestLike = 1; 8063} 8064 8065multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> { 8066 def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>; 8067 def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>; 8068 8069 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 8070 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 8071} 8072 8073//===----------------------------------------------------------------------===// 8074// SVE2 Histogram Computation - Segment Group 8075//===----------------------------------------------------------------------===// 8076 8077class sve2_hist_gen_segment<string asm, SDPatternOperator op> 8078: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm), 8079 asm, "\t$Zd, $Zn, $Zm", 8080 "", 8081 [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> { 8082 bits<5> Zd; 8083 bits<5> Zn; 8084 bits<5> Zm; 8085 let Inst{31-21} = 0b01000101001; 8086 let Inst{20-16} = Zm; 8087 let Inst{15-10} = 0b101000; 8088 let Inst{9-5} = Zn; 8089 let Inst{4-0} = Zd; 8090} 8091 8092//===----------------------------------------------------------------------===// 8093// SVE2 Histogram Computation - Vector Group 8094//===----------------------------------------------------------------------===// 8095 8096class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty> 8097: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 8098 asm, "\t$Zd, $Pg/z, $Zn, $Zm", 8099 "", 8100 []>, Sched<[]> { 8101 bits<5> Zd; 8102 bits<5> Zn; 8103 bits<3> Pg; 8104 bits<5> Zm; 8105 let Inst{31-23} = 0b010001011; 8106 let Inst{22} = sz; 8107 let Inst{21} = 0b1; 8108 let Inst{20-16} = Zm; 8109 let Inst{15-13} = 0b110; 8110 let Inst{12-10} = Pg; 8111 let Inst{9-5} = Zn; 8112 let Inst{4-0} = Zd; 8113} 8114 8115multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> { 8116 def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>; 8117 def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>; 8118 8119 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 8120 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 8121} 8122 8123//===----------------------------------------------------------------------===// 8124// SVE2 Crypto Extensions Group 8125//===----------------------------------------------------------------------===// 8126 8127class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty> 8128: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 8129 asm, "\t$Zd, $Zn, $Zm", 8130 "", 8131 []>, Sched<[]> { 8132 bits<5> Zd; 8133 bits<5> Zn; 8134 bits<5> Zm; 8135 let Inst{31-21} = 0b01000101001; 8136 let Inst{20-16} = Zm; 8137 let Inst{15-11} = 0b11110; 8138 let Inst{10} = opc; 8139 let Inst{9-5} = Zn; 8140 let Inst{4-0} = Zd; 8141} 8142 8143multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty, 8144 SDPatternOperator op, ValueType vt> { 8145 def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>; 8146 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 8147} 8148 8149class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty> 8150: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm), 8151 asm, "\t$Zdn, $_Zdn, $Zm", 8152 "", 8153 []>, Sched<[]> { 8154 bits<5> Zdn; 8155 bits<5> Zm; 8156 let Inst{31-17} = 0b010001010010001; 8157 let Inst{16} = opc{1}; 8158 let Inst{15-11} = 0b11100; 8159 let Inst{10} = opc{0}; 8160 let Inst{9-5} = Zm; 8161 let Inst{4-0} = Zdn; 8162 8163 let Constraints = "$Zdn = $_Zdn"; 8164} 8165 8166multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty, 8167 SDPatternOperator op, ValueType vt> { 8168 def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>; 8169 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 8170} 8171 8172class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty> 8173: I<(outs zprty:$Zdn), (ins zprty:$_Zdn), 8174 asm, "\t$Zdn, $_Zdn", 8175 "", 8176 []>, Sched<[]> { 8177 bits<5> Zdn; 8178 let Inst{31-11} = 0b010001010010000011100; 8179 let Inst{10} = opc; 8180 let Inst{9-5} = 0b00000; 8181 let Inst{4-0} = Zdn; 8182 8183 let Constraints = "$Zdn = $_Zdn"; 8184} 8185 8186multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> { 8187 def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>; 8188 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>; 8189} 8190 8191//===----------------------------------------------------------------------===// 8192// SVE BFloat16 Group 8193//===----------------------------------------------------------------------===// 8194 8195class sve_bfloat_dot_base<bits<2> opc, string asm, string ops, dag iops> 8196: I<(outs ZPR32:$Zda), iops, asm, ops, "", []>, Sched<[]> { 8197 bits<5> Zda; 8198 bits<5> Zn; 8199 let Inst{31-21} = 0b01100100011; 8200 let Inst{15-14} = opc; 8201 let Inst{13-10} = 0b0000; 8202 let Inst{9-5} = Zn; 8203 let Inst{4-0} = Zda; 8204 8205 let Constraints = "$Zda = $_Zda"; 8206 let DestructiveInstType = DestructiveOther; 8207 let ElementSize = ElementSizeH; 8208} 8209 8210class sve_bfloat_dot<string asm> 8211: sve_bfloat_dot_base<0b10, asm, "\t$Zda, $Zn, $Zm", 8212 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm)> { 8213 bits<5> Zm; 8214 let Inst{20-16} = Zm; 8215} 8216 8217multiclass sve_bfloat_dot<string asm, SDPatternOperator op> { 8218 def NAME : sve_bfloat_dot<asm>; 8219 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8220} 8221 8222class sve_bfloat_dot_indexed<string asm> 8223: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 8224 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> { 8225 bits<2> iop; 8226 bits<3> Zm; 8227 let Inst{20-19} = iop; 8228 let Inst{18-16} = Zm; 8229} 8230 8231multiclass sve_bfloat_dot_indexed<string asm, SDPatternOperator op> { 8232 def NAME : sve_bfloat_dot_indexed<asm>; 8233 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexS_timm, !cast<Instruction>(NAME)>; 8234} 8235 8236class sve_bfloat_matmul<string asm> 8237: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 8238 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8239 bits<5> Zm; 8240 bits<5> Zda; 8241 bits<5> Zn; 8242 let Inst{31-21} = 0b01100100011; 8243 let Inst{20-16} = Zm; 8244 let Inst{15-10} = 0b111001; 8245 let Inst{9-5} = Zn; 8246 let Inst{4-0} = Zda; 8247 8248 let Constraints = "$Zda = $_Zda"; 8249 let DestructiveInstType = DestructiveOther; 8250 let ElementSize = ElementSizeH; 8251} 8252 8253multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> { 8254 def NAME : sve_bfloat_matmul<asm>; 8255 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8256} 8257 8258class sve_bfloat_matmul_longvecl<bit BT, string asm> 8259: sve_bfloat_matmul<asm> { 8260 let Inst{23} = 0b1; 8261 let Inst{14-13} = 0b00; 8262 let Inst{10} = BT; 8263} 8264 8265multiclass sve_bfloat_matmul_longvecl<bit BT, string asm, SDPatternOperator op> { 8266 def NAME : sve_bfloat_matmul_longvecl<BT, asm>; 8267 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8268} 8269 8270class sve_bfloat_matmul_longvecl_idx<bit BT, string asm> 8271: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 8272 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> { 8273 bits<3> iop; 8274 bits<3> Zm; 8275 let Inst{23} = 0b1; 8276 let Inst{20-19} = iop{2-1}; 8277 let Inst{18-16} = Zm; 8278 let Inst{11} = iop{0}; 8279 let Inst{10} = BT; 8280} 8281 8282multiclass sve_bfloat_matmul_longvecl_idx<bit BT, string asm, SDPatternOperator op> { 8283 def NAME : sve_bfloat_matmul_longvecl_idx<BT, asm>; 8284 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexH_timm, !cast<Instruction>(NAME)>; 8285} 8286 8287class sve_bfloat_convert<bit N, string asm> 8288: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), 8289 asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { 8290 bits<5> Zd; 8291 bits<3> Pg; 8292 bits<5> Zn; 8293 let Inst{31-25} = 0b0110010; 8294 let Inst{24} = N; 8295 let Inst{23-13} = 0b10001010101; 8296 let Inst{12-10} = Pg; 8297 let Inst{9-5} = Zn; 8298 let Inst{4-0} = Zd; 8299 8300 let Constraints = "$Zd = $_Zd"; 8301 let DestructiveInstType = DestructiveOther; 8302 let hasSideEffects = 1; 8303 let ElementSize = ElementSizeS; 8304} 8305 8306multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> { 8307 def NAME : sve_bfloat_convert<N, asm>; 8308 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>; 8309} 8310 8311//===----------------------------------------------------------------------===// 8312// SVE Integer Matrix Multiply Group 8313//===----------------------------------------------------------------------===// 8314 8315class sve_int_matmul<bits<2> uns, string asm> 8316: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8317 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8318 bits<5> Zda; 8319 bits<5> Zn; 8320 bits<5> Zm; 8321 let Inst{31-24} = 0b01000101; 8322 let Inst{23-22} = uns; 8323 let Inst{21} = 0; 8324 let Inst{20-16} = Zm; 8325 let Inst{15-10} = 0b100110; 8326 let Inst{9-5} = Zn; 8327 let Inst{4-0} = Zda; 8328 8329 let Constraints = "$Zda = $_Zda"; 8330 let DestructiveInstType = DestructiveOther; 8331 let ElementSize = ZPR32.ElementSize; 8332} 8333 8334multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> { 8335 def NAME : sve_int_matmul<uns, asm>; 8336 8337 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8338} 8339 8340//===----------------------------------------------------------------------===// 8341// SVE Integer Dot Product Mixed Sign Group 8342//===----------------------------------------------------------------------===// 8343 8344class sve_int_dot_mixed<string asm> 8345: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8346 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8347 bits<5> Zda; 8348 bits<5> Zn; 8349 bits<5> Zm; 8350 let Inst{31-21} = 0b01000100100; 8351 let Inst{20-16} = Zm; 8352 let Inst{15-10} = 0b011110; 8353 let Inst{9-5} = Zn; 8354 let Inst{4-0} = Zda; 8355 8356 let Constraints = "$Zda = $_Zda"; 8357 let DestructiveInstType = DestructiveOther; 8358 let ElementSize = ZPR32.ElementSize; 8359} 8360 8361multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> { 8362 def NAME : sve_int_dot_mixed<asm>; 8363 8364 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8365} 8366 8367//===----------------------------------------------------------------------===// 8368// SVE Integer Dot Product Mixed Sign - Indexed Group 8369//===----------------------------------------------------------------------===// 8370 8371class sve_int_dot_mixed_indexed<bit U, string asm> 8372: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx), 8373 asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> { 8374 bits<5> Zda; 8375 bits<5> Zn; 8376 bits<3> Zm; 8377 bits<2> idx; 8378 let Inst{31-21} = 0b01000100101; 8379 let Inst{20-19} = idx; 8380 let Inst{18-16} = Zm; 8381 let Inst{15-11} = 0b00011; 8382 let Inst{10} = U; 8383 let Inst{9-5} = Zn; 8384 let Inst{4-0} = Zda; 8385 8386 let Constraints = "$Zda = $_Zda"; 8387 let DestructiveInstType = DestructiveOther; 8388 let ElementSize = ZPR32.ElementSize; 8389} 8390 8391multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> { 8392 def NAME : sve_int_dot_mixed_indexed<U, asm>; 8393 8394 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>; 8395} 8396 8397//===----------------------------------------------------------------------===// 8398// SVE Floating Point Matrix Multiply Accumulate Group 8399//===----------------------------------------------------------------------===// 8400 8401class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty> 8402: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm), 8403 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8404 bits<5> Zda; 8405 bits<5> Zn; 8406 bits<5> Zm; 8407 let Inst{31-23} = 0b011001001; 8408 let Inst{22} = sz; 8409 let Inst{21} = 1; 8410 let Inst{20-16} = Zm; 8411 let Inst{15-10} = 0b111001; 8412 let Inst{9-5} = Zn; 8413 let Inst{4-0} = Zda; 8414 8415 let Constraints = "$Zda = $_Zda"; 8416 let DestructiveInstType = DestructiveOther; 8417 let ElementSize = zprty.ElementSize; 8418} 8419 8420multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> { 8421 def NAME : sve_fp_matrix_mla<sz, asm, zprty>; 8422 8423 def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>; 8424} 8425 8426//===----------------------------------------------------------------------===// 8427// SVE Memory - Contiguous Load And Replicate 256-bit Group 8428//===----------------------------------------------------------------------===// 8429 8430class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList> 8431: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 8432 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 8433 bits<5> Zt; 8434 bits<5> Rn; 8435 bits<3> Pg; 8436 bits<4> imm4; 8437 let Inst{31-25} = 0b1010010; 8438 let Inst{24-23} = sz; 8439 let Inst{22-20} = 0b010; 8440 let Inst{19-16} = imm4; 8441 let Inst{15-13} = 0b001; 8442 let Inst{12-10} = Pg; 8443 let Inst{9-5} = Rn; 8444 let Inst{4-0} = Zt; 8445 8446 let mayLoad = 1; 8447} 8448 8449multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty, 8450 ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> { 8451 def NAME : sve_mem_ldor_si<sz, asm, listty>; 8452 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8453 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 8454 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8455 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 8456 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 8457 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>; 8458 8459 // Base addressing mode 8460 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)), 8461 (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>; 8462 let AddedComplexity = 2 in { 8463 // Reg + Imm addressing mode 8464 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))), 8465 (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>; 8466 } 8467} 8468 8469class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList, 8470 RegisterOperand gprty> 8471: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 8472 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 8473 bits<5> Zt; 8474 bits<3> Pg; 8475 bits<5> Rn; 8476 bits<5> Rm; 8477 let Inst{31-25} = 0b1010010; 8478 let Inst{24-23} = sz; 8479 let Inst{22-21} = 0b01; 8480 let Inst{20-16} = Rm; 8481 let Inst{15-13} = 0; 8482 let Inst{12-10} = Pg; 8483 let Inst{9-5} = Rn; 8484 let Inst{4-0} = Zt; 8485 8486 let mayLoad = 1; 8487} 8488 8489multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty, 8490 ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty, 8491 ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> { 8492 def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>; 8493 8494 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 8495 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 8496 8497 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))), 8498 (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>; 8499} 8500 8501//===----------------------------------------------------------------------===// 8502// SVE Interleave 128-bit Elements Group 8503//===----------------------------------------------------------------------===// 8504 8505class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm> 8506: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm), 8507 asm, "\t$Zd, $Zn, $Zm", 8508 "", 8509 []>, Sched<[]> { 8510 bits<5> Zd; 8511 bits<5> Zm; 8512 bits<5> Zn; 8513 let Inst{31-21} = 0b00000101101; 8514 let Inst{20-16} = Zm; 8515 let Inst{15-13} = 0b000; 8516 let Inst{12-11} = opc; 8517 let Inst{10} = P; 8518 let Inst{9-5} = Zn; 8519 let Inst{4-0} = Zd; 8520} 8521 8522multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> { 8523 def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>; 8524 8525 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8526 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 8527 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 8528 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 8529 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME)>; 8530 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 8531 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME)>; 8532 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>; 8533} 8534 8535/// Addressing modes 8536def am_sve_indexed_s4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; 8537def am_sve_indexed_s6 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; 8538 8539def am_sve_regreg_lsl0 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<0>", []>; 8540def am_sve_regreg_lsl1 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<1>", []>; 8541def am_sve_regreg_lsl2 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<2>", []>; 8542def am_sve_regreg_lsl3 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<3>", []>; 8543def am_sve_regreg_lsl4 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<4>", []>; 8544 8545// Predicated pseudo floating point two operand instructions. 8546multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> { 8547 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8548 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8549 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8550 8551 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8552 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8553 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8554 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8555 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8556 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8557} 8558 8559// Predicated pseudo integer two operand instructions. 8560multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> { 8561 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8562 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8563 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8564 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8565 8566 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8567 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8568 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8569 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8570} 8571 8572// As sve_int_bin_pred but when only i32 and i64 vector types are required. 8573multiclass sve_int_bin_pred_sd<SDPatternOperator op> { 8574 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8575 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8576 8577 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8578 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8579} 8580 8581// Predicated pseudo integer two operand instructions. Second operand is an 8582// immediate specified by imm_[bhsd]. 8583multiclass sve_int_shift_pred_bhsd<SDPatternOperator op, 8584 ComplexPattern imm_b, ComplexPattern imm_h, 8585 ComplexPattern imm_s, ComplexPattern imm_d> { 8586 def _UNDEF_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, Operand<i32>, FalseLanesUndef>; 8587 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesUndef>; 8588 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesUndef>; 8589 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesUndef>; 8590 8591 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _UNDEF_B)>; 8592 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Instruction>(NAME # _UNDEF_H)>; 8593 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Instruction>(NAME # _UNDEF_S)>; 8594 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Instruction>(NAME # _UNDEF_D)>; 8595} 8596 8597multiclass sve_int_bin_pred_all_active_bhsd<SDPatternOperator op> { 8598 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8599 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8600 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8601 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8602 8603 def : SVE_2_Op_Pred_All_Active_Pt<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8604 def : SVE_2_Op_Pred_All_Active_Pt<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8605 def : SVE_2_Op_Pred_All_Active_Pt<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8606 def : SVE_2_Op_Pred_All_Active_Pt<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8607} 8608 8609