1//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def SDT_AArch64Setcc : SDTypeProfile<1, 4, [ 14 SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, 15 SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>, 16 SDTCisVT<4, OtherVT> 17]>; 18 19def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>; 20 21def SVEPatternOperand : AsmOperandClass { 22 let Name = "SVEPattern"; 23 let ParserMethod = "tryParseSVEPattern"; 24 let PredicateMethod = "isSVEPattern"; 25 let RenderMethod = "addImmOperands"; 26 let DiagnosticType = "InvalidSVEPattern"; 27} 28 29def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{ 30 return (((uint32_t)Imm) < 32); 31 }]> { 32 33 let PrintMethod = "printSVEPattern"; 34 let ParserMatchClass = SVEPatternOperand; 35} 36 37def SVEPrefetchOperand : AsmOperandClass { 38 let Name = "SVEPrefetch"; 39 let ParserMethod = "tryParsePrefetch<true>"; 40 let PredicateMethod = "isPrefetch"; 41 let RenderMethod = "addPrefetchOperands"; 42} 43 44def sve_prfop : Operand<i32>, TImmLeaf<i32, [{ 45 return (((uint32_t)Imm) <= 15); 46 }]> { 47 let PrintMethod = "printPrefetchOp<true>"; 48 let ParserMatchClass = SVEPrefetchOperand; 49} 50 51class SVELogicalImmOperand<int Width> : AsmOperandClass { 52 let Name = "SVELogicalImm" # Width; 53 let DiagnosticType = "LogicalSecondSource"; 54 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 55 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 56} 57 58def sve_logical_imm8 : Operand<i64> { 59 let ParserMatchClass = SVELogicalImmOperand<8>; 60 let PrintMethod = "printLogicalImm<int8_t>"; 61 62 let MCOperandPredicate = [{ 63 if (!MCOp.isImm()) 64 return false; 65 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 66 return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val); 67 }]; 68} 69 70def sve_logical_imm16 : Operand<i64> { 71 let ParserMatchClass = SVELogicalImmOperand<16>; 72 let PrintMethod = "printLogicalImm<int16_t>"; 73 74 let MCOperandPredicate = [{ 75 if (!MCOp.isImm()) 76 return false; 77 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 78 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val); 79 }]; 80} 81 82def sve_logical_imm32 : Operand<i64> { 83 let ParserMatchClass = SVELogicalImmOperand<32>; 84 let PrintMethod = "printLogicalImm<int32_t>"; 85 86 let MCOperandPredicate = [{ 87 if (!MCOp.isImm()) 88 return false; 89 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 90 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val); 91 }]; 92} 93 94class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass { 95 let Name = "SVEPreferredLogicalImm" # Width; 96 let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>"; 97 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 98} 99 100def sve_preferred_logical_imm16 : Operand<i64> { 101 let ParserMatchClass = SVEPreferredLogicalImmOperand<16>; 102 let PrintMethod = "printSVELogicalImm<int16_t>"; 103 104 let MCOperandPredicate = [{ 105 if (!MCOp.isImm()) 106 return false; 107 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 108 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) && 109 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 110 }]; 111} 112 113def sve_preferred_logical_imm32 : Operand<i64> { 114 let ParserMatchClass = SVEPreferredLogicalImmOperand<32>; 115 let PrintMethod = "printSVELogicalImm<int32_t>"; 116 117 let MCOperandPredicate = [{ 118 if (!MCOp.isImm()) 119 return false; 120 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 121 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) && 122 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 123 }]; 124} 125 126def sve_preferred_logical_imm64 : Operand<i64> { 127 let ParserMatchClass = SVEPreferredLogicalImmOperand<64>; 128 let PrintMethod = "printSVELogicalImm<int64_t>"; 129 130 let MCOperandPredicate = [{ 131 if (!MCOp.isImm()) 132 return false; 133 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 134 return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) && 135 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 136 }]; 137} 138 139class SVELogicalImmNotOperand<int Width> : AsmOperandClass { 140 let Name = "SVELogicalImm" # Width # "Not"; 141 let DiagnosticType = "LogicalSecondSource"; 142 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 143 let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>"; 144} 145 146def sve_logical_imm8_not : Operand<i64> { 147 let ParserMatchClass = SVELogicalImmNotOperand<8>; 148} 149 150def sve_logical_imm16_not : Operand<i64> { 151 let ParserMatchClass = SVELogicalImmNotOperand<16>; 152} 153 154def sve_logical_imm32_not : Operand<i64> { 155 let ParserMatchClass = SVELogicalImmNotOperand<32>; 156} 157 158class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate> 159 : AsmOperandClass { 160 let Name = "SVE" # Infix # "Imm" # ElementWidth; 161 let DiagnosticType = "Invalid" # Name; 162 let RenderMethod = "addImmWithOptionalShiftOperands<8>"; 163 let ParserMethod = "tryParseImmWithOptionalShift"; 164 let PredicateMethod = Predicate; 165} 166 167def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">; 168def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">; 169def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">; 170def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">; 171 172def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">; 173def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">; 174def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">; 175def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">; 176 177class imm8_opt_lsl<int ElementWidth, string printType, 178 AsmOperandClass OpndClass> 179 : Operand<i32> { 180 let EncoderMethod = "getImm8OptLsl"; 181 let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">"; 182 let PrintMethod = "printImm8OptLsl<" # printType # ">"; 183 let ParserMatchClass = OpndClass; 184 let MIOperandInfo = (ops i32imm, i32imm); 185} 186 187def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>; 188def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>; 189def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>; 190def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>; 191 192def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>; 193def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>; 194def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>; 195def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>; 196 197def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; 198def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; 199def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; 200def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64>", []>; 201 202def SVECpyDupImm8Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i8>", []>; 203def SVECpyDupImm16Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i16>", []>; 204def SVECpyDupImm32Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i32>", []>; 205def SVECpyDupImm64Pat : ComplexPattern<i64, 2, "SelectSVECpyDupImm<MVT::i64>", []>; 206 207def SVELogicalImm8Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8>", []>; 208def SVELogicalImm16Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16>", []>; 209def SVELogicalImm32Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32>", []>; 210def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>; 211 212def SVELogicalImm8NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8, true>", []>; 213def SVELogicalImm16NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16, true>", []>; 214def SVELogicalImm32NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32, true>", []>; 215def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>; 216 217def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>; 218def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>; 219def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>; 220def SVEArithUImm64Pat : ComplexPattern<i64, 1, "SelectSVEArithImm<MVT::i64>", []>; 221 222def SVEArithSImmPat32 : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; 223def SVEArithSImmPat64 : ComplexPattern<i64, 1, "SelectSVESignedArithImm", []>; 224 225def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>; 226def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>; 227def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>; 228def SVEShiftImmL64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<0, 63>", []>; 229def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>; 230def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>; 231def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>; 232def SVEShiftImmR64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<1, 64, true>", []>; 233 234def SVEShiftSplatImmR : ComplexPattern<iAny, 1, "SelectSVEShiftSplatImmR", []>; 235 236def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>; 237 238class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass { 239 let Name = "SVEExactFPImmOperand" # Suffix; 240 let DiagnosticType = "Invalid" # Name; 241 let ParserMethod = "tryParseFPImm<false>"; 242 let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">"; 243 let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">"; 244} 245 246class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> { 247 let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">"; 248 let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>; 249} 250 251def sve_fpimm_half_one 252 : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half", 253 "AArch64ExactFPImm::one">; 254def sve_fpimm_half_two 255 : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half", 256 "AArch64ExactFPImm::two">; 257def sve_fpimm_zero_one 258 : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero", 259 "AArch64ExactFPImm::one">; 260 261def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{ 262 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17); 263}]> { 264 let ParserMatchClass = Imm1_16Operand; 265 let EncoderMethod = "getSVEIncDecImm"; 266 let DecoderMethod = "DecodeSVEIncDecImm"; 267} 268 269// This allows i32 immediate extraction from i64 based arithmetic. 270def sve_cnt_mul_imm_i32 : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; 271def sve_cnt_mul_imm_i64 : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, false>">; 272def sve_cnt_shl_imm : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, true>">; 273 274def sve_ext_imm_0_31 : ComplexPattern<i64, 1, "SelectEXTImm<31, 8>">; 275def sve_ext_imm_0_63 : ComplexPattern<i64, 1, "SelectEXTImm<63, 4>">; 276def sve_ext_imm_0_127 : ComplexPattern<i64, 1, "SelectEXTImm<127, 2>">; 277def sve_ext_imm_0_255 : ComplexPattern<i64, 1, "SelectEXTImm<255, 1>">; 278 279def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), 280 (int_aarch64_sve_cntp node:$pred, node:$src2), [{ 281 return N->hasOneUse(); 282}]>; 283 284def step_vector_oneuse : PatFrag<(ops node:$idx), 285 (step_vector node:$idx), [{ 286 return N->hasOneUse(); 287}]>; 288 289 290//===----------------------------------------------------------------------===// 291// SVE PTrue - These are used extensively throughout the pattern matching so 292// it's important we define them first. 293//===----------------------------------------------------------------------===// 294 295class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 296 ValueType vt, SDPatternOperator op> 297: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern), 298 asm, "\t$Pd, $pattern", 299 "", 300 [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> { 301 bits<4> Pd; 302 bits<5> pattern; 303 let Inst{31-24} = 0b00100101; 304 let Inst{23-22} = sz8_64; 305 let Inst{21-19} = 0b011; 306 let Inst{18-17} = opc{2-1}; 307 let Inst{16} = opc{0}; 308 let Inst{15-10} = 0b111000; 309 let Inst{9-5} = pattern; 310 let Inst{4} = 0b0; 311 let Inst{3-0} = Pd; 312 313 let Defs = !if(!eq (opc{0}, 1), [NZCV], []); 314 let ElementSize = pprty.ElementSize; 315 let isReMaterializable = 1; 316} 317 318multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> { 319 def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>; 320 def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>; 321 def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>; 322 def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>; 323 324 def : InstAlias<asm # "\t$Pd", 325 (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>; 326 def : InstAlias<asm # "\t$Pd", 327 (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>; 328 def : InstAlias<asm # "\t$Pd", 329 (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>; 330 def : InstAlias<asm # "\t$Pd", 331 (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>; 332} 333 334def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; 335def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>; 336 337let Predicates = [HasSVEorSME] in { 338 defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>; 339 defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>; 340 341 def : Pat<(nxv16i1 immAllOnesV), (PTRUE_B 31)>; 342 def : Pat<(nxv8i1 immAllOnesV), (PTRUE_H 31)>; 343 def : Pat<(nxv4i1 immAllOnesV), (PTRUE_S 31)>; 344 def : Pat<(nxv2i1 immAllOnesV), (PTRUE_D 31)>; 345} 346 347//===----------------------------------------------------------------------===// 348// SVE pattern match helpers. 349//===----------------------------------------------------------------------===// 350 351class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 352 Instruction inst> 353: Pat<(vtd (op vt1:$Op1)), 354 (inst $Op1)>; 355 356class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 357 ValueType vts, Instruction inst> 358: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)), 359 (inst $Op3, $Op1, $Op2)>; 360 361 362multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 363 ValueType vts, Instruction inst> { 364 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd undef))), 365 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 366 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, vtd:$Op3)), 367 (inst $Op3, $Op1, $Op2)>; 368} 369 370// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the 371// type of rounding. This is matched by timm0_1 in pattern below and ignored. 372class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 373 ValueType vts, Instruction inst> 374: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)), 375 (inst $Op3, $Op1, $Op2)>; 376 377multiclass SVE_1_Op_PassthruUndef_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 378 ValueType vts, Instruction inst>{ 379 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), (vtd undef))), 380 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 381 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, (i64 timm0_1), vtd:$Op3)), 382 (inst $Op3, $Op1, $Op2)>; 383} 384 385class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 386 ValueType it, ComplexPattern cpx, Instruction inst> 387 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm, i32:$shift)))))), 388 (inst $Op1, i32:$imm, i32:$shift)>; 389 390class SVE_1_Op_Imm_Arith_All_Active<ValueType vt, ValueType pt, SDPatternOperator op, 391 ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst> 392 : Pat<(vt (op (pt (SVEAllActive)), (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm)))))), 393 (inst $Op1, i32:$imm)>; 394 395class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 396 ValueType it, ComplexPattern cpx, Instruction inst> 397 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i64:$imm)))))), 398 (inst $Op1, i64:$imm)>; 399 400class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 401 ValueType vt2, Instruction inst> 402: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 403 (inst $Op1, $Op2)>; 404 405class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op, 406 ValueType pt, ValueType vt1, ValueType vt2, 407 Instruction inst> 408: Pat<(vtd (op (pt (SVEAllActive)), vt1:$Op1, vt2:$Op2)), 409 (inst $Op1, $Op2)>; 410 411class SVE_2_Op_Pred_All_Active_Pt<ValueType vtd, SDPatternOperator op, 412 ValueType pt, ValueType vt1, ValueType vt2, 413 Instruction inst> 414: Pat<(vtd (op (pt (SVEAllActive:$Op1)), vt1:$Op2, vt2:$Op3)), 415 (inst $Op1, $Op2, $Op3)>; 416 417class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 418 ValueType vt2, ValueType vt3, Instruction inst> 419: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 420 (inst $Op1, $Op2, $Op3)>; 421 422multiclass SVE_3_Op_Undef_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 423 ValueType vt2, ValueType vt3, Instruction inst> { 424 def : Pat<(vtd (op (vt1 undef), vt2:$Op1, vt3:$Op2)), 425 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 426 def : Pat<(vtd (op vt1:$Op1, (vt2 (SVEAllActive:$Op2)), vt3:$Op3)), 427 (inst $Op1, $Op2, $Op3)>; 428} 429 430class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 431 ValueType vt2, ValueType vt3, ValueType vt4, 432 Instruction inst> 433: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)), 434 (inst $Op1, $Op2, $Op3, $Op4)>; 435 436class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 437 ValueType vt2, Operand ImmTy, Instruction inst> 438: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))), 439 (inst $Op1, ImmTy:$Op2)>; 440 441class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 442 ValueType vt2, ValueType vt3, Operand ImmTy, 443 Instruction inst> 444: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))), 445 (inst $Op1, $Op2, ImmTy:$Op3)>; 446 447class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 448 ValueType vt2, ValueType vt3, ValueType vt4, 449 Operand ImmTy, Instruction inst> 450: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), 451 (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; 452 453def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>; 454def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>; 455 456let AddedComplexity = 1 in { 457class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1, 458 ValueType vt2, ValueType vt3, Instruction inst> 459: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))), 460 (inst $Op1, $Op2, $Op3)>; 461 462class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op, 463 ValueType vt1, ValueType vt2, 464 Operand vt3, Instruction inst> 465: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))), 466 (inst $Op1, $Op2, vt3:$Op3)>; 467} 468 469// 470// Common but less generic patterns. 471// 472 473class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 474 Instruction inst, Instruction ptrue> 475: Pat<(vtd (op vt1:$Op1)), 476 (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>; 477 478class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 479 ValueType vt2, Instruction inst, Instruction ptrue> 480: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 481 (inst (ptrue 31), $Op1, $Op2)>; 482 483class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt, 484 ValueType inreg_vt, Instruction inst> 485: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)), 486 (inst $PassThru, $Pg, $Src)>; 487 488multiclass SVE_InReg_Extend_PassthruUndef<ValueType vt, SDPatternOperator op, ValueType pt, 489 ValueType inreg_vt, Instruction inst> { 490 def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt undef))), 491 (inst (IMPLICIT_DEF), $Pg, $Src)>; 492 def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, vt:$PassThru)), 493 (inst $PassThru, $Pg, $Src)>; 494} 495 496class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op, 497 ValueType pt, ValueType it, 498 ComplexPattern cast, Instruction inst> 499: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))), 500 (inst $Pg, $Rn, i32:$imm)>; 501 502class SVE_Shift_DupImm_All_Active_Pat<ValueType vt, SDPatternOperator op, 503 ValueType pt, ValueType it, 504 ComplexPattern cast, Instruction inst> 505: Pat<(vt (op (pt (SVEAllActive)), vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))), 506 (inst $Rn, i32:$imm)>; 507 508class SVE_2_Op_Fp_Imm_Pat<ValueType vt, SDPatternOperator op, 509 ValueType pt, ValueType it, 510 FPImmLeaf immL, int imm, 511 Instruction inst> 512: Pat<(vt (op (pt PPR_3b:$Pg), (vt ZPR:$Zs1), (vt (splat_vector (it immL))))), 513 (inst $Pg, $Zs1, imm)>; 514 515class SVE_2_Op_Fp_Imm_Pat_Zero<ValueType vt, SDPatternOperator op, 516 ValueType pt, ValueType it, 517 FPImmLeaf immL, int imm, 518 Instruction inst> 519: Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Zs1, (SVEDup0)), 520 (vt (splat_vector (it immL))))), 521 (inst $Pg, $Zs1, imm)>; 522 523// Used to re-order the operands of BSP when lowering to BSL. BSP has the order: 524// mask, in1, in2 whereas BSL for SVE2 has them ordered in1, in2, mask 525class SVE_3_Op_BSP_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 526 ValueType vt2, ValueType vt3, Instruction inst> 527: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 528 (inst $Op2, $Op3, $Op1)>; 529 530class SVE_Shift_Add_All_Active_Pat<ValueType vtd, SDPatternOperator op, ValueType pt, 531 ValueType vt1, ValueType vt2, ValueType vt3, 532 Instruction inst> 533: Pat<(vtd (add vt1:$Op1, (op (pt (SVEAllActive)), vt2:$Op2, vt3:$Op3))), 534 (inst $Op1, $Op2, $Op3)>; 535 536//===----------------------------------------------------------------------===// 537// SVE pattern match helpers. 538//===----------------------------------------------------------------------===// 539 540// Matches either an intrinsic, or a predicated operation with an all active predicate 541class EitherVSelectOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOperator sdnode> 542: PatFrags<(ops node:$Pg, node:$Op1, node:$Op2), [ 543 (intrinsic node:$Pg, node:$Op1, node:$Op2), 544 (vselect node:$Pg, (sdnode (SVEAllActive), node:$Op1, node:$Op2), node:$Op1), 545 ]>; 546 547// 548// Pseudo -> Instruction mappings 549// 550def getSVEPseudoMap : InstrMapping { 551 let FilterClass = "SVEPseudo2Instr"; 552 let RowFields = ["PseudoName"]; 553 let ColFields = ["IsInstr"]; 554 let KeyCol = ["0"]; 555 let ValueCols = [["1"]]; 556} 557 558class SVEPseudo2Instr<string name, bit instr> { 559 string PseudoName = name; 560 bit IsInstr = instr; 561} 562 563// Lookup e.g. DIV -> DIVR 564def getSVERevInstr : InstrMapping { 565 let FilterClass = "SVEInstr2Rev"; 566 let RowFields = ["InstrName"]; 567 let ColFields = ["isReverseInstr"]; 568 let KeyCol = ["0"]; 569 let ValueCols = [["1"]]; 570} 571 572// Lookup e.g. DIVR -> DIV 573def getSVENonRevInstr : InstrMapping { 574 let FilterClass = "SVEInstr2Rev"; 575 let RowFields = ["InstrName"]; 576 let ColFields = ["isReverseInstr"]; 577 let KeyCol = ["1"]; 578 let ValueCols = [["0"]]; 579} 580 581class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> { 582 string InstrName = !if(name1IsReverseInstr, name1, name2); 583 bit isReverseInstr = name1IsReverseInstr; 584} 585 586// 587// Pseudos for destructive operands 588// 589let hasNoSchedulingInfo = 1 in { 590 class PredTwoOpPseudo<string name, ZPRRegOp zprty, 591 FalseLanesEnum flags = FalseLanesNone> 592 : SVEPseudo2Instr<name, 0>, 593 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> { 594 let FalseLanes = flags; 595 } 596 597 class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty, 598 FalseLanesEnum flags = FalseLanesNone> 599 : SVEPseudo2Instr<name, 0>, 600 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> { 601 let FalseLanes = flags; 602 } 603 604 class PredThreeOpPseudo<string name, ZPRRegOp zprty, 605 FalseLanesEnum flags = FalseLanesNone> 606 : SVEPseudo2Instr<name, 0>, 607 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2, zprty:$Zs3), []> { 608 let FalseLanes = flags; 609 } 610} 611 612// 613// Pseudos for passthru operands 614// 615let hasNoSchedulingInfo = 1 in { 616 class PredOneOpPassthruPseudo<string name, ZPRRegOp zprty> 617 : SVEPseudo2Instr<name, 0>, 618 Pseudo<(outs zprty:$Zd), (ins zprty:$Passthru, PPR3bAny:$Pg, zprty:$Zs), []>; 619} 620 621//===----------------------------------------------------------------------===// 622// SVE Predicate Misc Group 623//===----------------------------------------------------------------------===// 624 625class sve_int_pfalse<bits<6> opc, string asm> 626: I<(outs PPR8:$Pd), (ins), 627 asm, "\t$Pd", 628 "", 629 []>, Sched<[]> { 630 bits<4> Pd; 631 let Inst{31-24} = 0b00100101; 632 let Inst{23-22} = opc{5-4}; 633 let Inst{21-19} = 0b011; 634 let Inst{18-16} = opc{3-1}; 635 let Inst{15-10} = 0b111001; 636 let Inst{9} = opc{0}; 637 let Inst{8-4} = 0b00000; 638 let Inst{3-0} = Pd; 639 640 let isReMaterializable = 1; 641} 642 643multiclass sve_int_pfalse<bits<6> opc, string asm> { 644 def NAME : sve_int_pfalse<opc, asm>; 645 646 def : Pat<(nxv16i1 immAllZerosV), (!cast<Instruction>(NAME))>; 647 def : Pat<(nxv8i1 immAllZerosV), (!cast<Instruction>(NAME))>; 648 def : Pat<(nxv4i1 immAllZerosV), (!cast<Instruction>(NAME))>; 649 def : Pat<(nxv2i1 immAllZerosV), (!cast<Instruction>(NAME))>; 650 def : Pat<(nxv1i1 immAllZerosV), (!cast<Instruction>(NAME))>; 651} 652 653class sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op> 654: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn), 655 asm, "\t$Pg, $Pn", 656 "", 657 [(op (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 658 bits<4> Pg; 659 bits<4> Pn; 660 let Inst{31-24} = 0b00100101; 661 let Inst{23-22} = opc{5-4}; 662 let Inst{21-19} = 0b010; 663 let Inst{18-16} = opc{3-1}; 664 let Inst{15-14} = 0b11; 665 let Inst{13-10} = Pg; 666 let Inst{9} = opc{0}; 667 let Inst{8-5} = Pn; 668 let Inst{4-0} = 0b00000; 669 670 let Defs = [NZCV]; 671 let isCompare = 1; 672} 673 674class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm, 675 PPRRegOp pprty> 676: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn), 677 asm, "\t$Pdn, $Pg, $_Pdn", 678 "", 679 []>, Sched<[]> { 680 bits<4> Pdn; 681 bits<4> Pg; 682 let Inst{31-24} = 0b00100101; 683 let Inst{23-22} = sz8_64; 684 let Inst{21-19} = 0b011; 685 let Inst{18-16} = opc{4-2}; 686 let Inst{15-11} = 0b11000; 687 let Inst{10-9} = opc{1-0}; 688 let Inst{8-5} = Pg; 689 let Inst{4} = 0; 690 let Inst{3-0} = Pdn; 691 692 let Constraints = "$Pdn = $_Pdn"; 693 let Defs = [NZCV]; 694 let isPTestLike = 1; 695 let ElementSize = pprty.ElementSize; 696} 697 698multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> { 699 def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>; 700 701 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 702} 703 704multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> { 705 def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>; 706 def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>; 707 def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>; 708 def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>; 709 710 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 711 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 712 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 713 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 714} 715 716//===----------------------------------------------------------------------===// 717// SVE Predicate Count Group 718//===----------------------------------------------------------------------===// 719 720class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm, 721 RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty> 722: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn), 723 asm, "\t$Rdn, $Pg", 724 "", 725 []>, Sched<[]> { 726 bits<5> Rdn; 727 bits<4> Pg; 728 let Inst{31-24} = 0b00100101; 729 let Inst{23-22} = sz8_64; 730 let Inst{21-19} = 0b101; 731 let Inst{18-16} = opc{4-2}; 732 let Inst{15-11} = 0b10001; 733 let Inst{10-9} = opc{1-0}; 734 let Inst{8-5} = Pg; 735 let Inst{4-0} = Rdn; 736 737 // Signed 32bit forms require their GPR operand printed. 738 let AsmString = !if(!eq(opc{4,2-0}, 0b0000), 739 !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"), 740 !strconcat(asm, "\t$Rdn, $Pg")); 741 let Constraints = "$Rdn = $_Rdn"; 742} 743 744multiclass sve_int_count_r_s32<bits<5> opc, string asm, 745 SDPatternOperator op> { 746 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>; 747 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>; 748 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>; 749 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>; 750 751 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 752 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 753 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))), 754 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 755 756 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 757 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 758 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))), 759 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 760 761 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 762 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 763 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))), 764 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 765 766 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 767 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 768 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))), 769 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 770} 771 772multiclass sve_int_count_r_u32<bits<5> opc, string asm, 773 SDPatternOperator op> { 774 def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>; 775 def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>; 776 def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>; 777 def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>; 778 779 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 780 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 781 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 782 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 783 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 784 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 785 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 786 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 787} 788 789multiclass sve_int_count_r_x64<bits<5> opc, string asm, 790 SDPatternOperator op, 791 SDPatternOperator combine_op = null_frag> { 792 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>; 793 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>; 794 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>; 795 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>; 796 797 def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))), 798 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 799 def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))), 800 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 801 def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))), 802 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 803 def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))), 804 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 805 806 // combine_op(x, cntp(all_active, p)) ==> inst p, x 807 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 (SVEAllActive)), (nxv16i1 PPRAny:$pred)))), 808 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 809 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 (SVEAllActive)), (nxv8i1 PPRAny:$pred)))), 810 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 811 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 (SVEAllActive)), (nxv4i1 PPRAny:$pred)))), 812 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 813 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 (SVEAllActive)), (nxv2i1 PPRAny:$pred)))), 814 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 815 816 // combine_op(x, cntp(p, p)) ==> inst p, x 817 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 PPRAny:$pred), (nxv16i1 PPRAny:$pred)))), 818 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 819 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 PPRAny:$pred), (nxv8i1 PPRAny:$pred)))), 820 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 821 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 PPRAny:$pred), (nxv4i1 PPRAny:$pred)))), 822 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 823 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 PPRAny:$pred), (nxv2i1 PPRAny:$pred)))), 824 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 825} 826 827class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm, 828 ZPRRegOp zprty, PPRRegOp pprty> 829: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm), 830 asm, "\t$Zdn, $Pm", 831 "", 832 []>, Sched<[]> { 833 bits<4> Pm; 834 bits<5> Zdn; 835 let Inst{31-24} = 0b00100101; 836 let Inst{23-22} = sz8_64; 837 let Inst{21-19} = 0b101; 838 let Inst{18-16} = opc{4-2}; 839 let Inst{15-11} = 0b10000; 840 let Inst{10-9} = opc{1-0}; 841 let Inst{8-5} = Pm; 842 let Inst{4-0} = Zdn; 843 844 let Constraints = "$Zdn = $_Zdn"; 845 let DestructiveInstType = DestructiveOther; 846 let ElementSize = ElementSizeNone; 847} 848 849multiclass sve_int_count_v<bits<5> opc, string asm, 850 SDPatternOperator op = null_frag> { 851 def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>; 852 def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>; 853 def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>; 854 855 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>; 856 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>; 857 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>; 858 859 def : InstAlias<asm # "\t$Zdn, $Pm", 860 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>; 861 def : InstAlias<asm # "\t$Zdn, $Pm", 862 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>; 863 def : InstAlias<asm # "\t$Zdn, $Pm", 864 (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>; 865} 866 867class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm, 868 PPRRegOp pprty> 869: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn), 870 asm, "\t$Rd, $Pg, $Pn", 871 "", 872 []>, Sched<[]> { 873 bits<4> Pg; 874 bits<4> Pn; 875 bits<5> Rd; 876 let Inst{31-24} = 0b00100101; 877 let Inst{23-22} = sz8_64; 878 let Inst{21-19} = 0b100; 879 let Inst{18-16} = opc{3-1}; 880 let Inst{15-14} = 0b10; 881 let Inst{13-10} = Pg; 882 let Inst{9} = opc{0}; 883 let Inst{8-5} = Pn; 884 let Inst{4-0} = Rd; 885} 886 887multiclass sve_int_pcount_pred<bits<4> opc, string asm, 888 SDPatternOperator int_op> { 889 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>; 890 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>; 891 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>; 892 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>; 893 894 def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 895 def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 896 def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 897 def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 898} 899 900//===----------------------------------------------------------------------===// 901// SVE Element Count Group 902//===----------------------------------------------------------------------===// 903 904class sve_int_count<bits<3> opc, string asm> 905: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 906 asm, "\t$Rd, $pattern, mul $imm4", 907 "", 908 []>, Sched<[]> { 909 bits<5> Rd; 910 bits<4> imm4; 911 bits<5> pattern; 912 let Inst{31-24} = 0b00000100; 913 let Inst{23-22} = opc{2-1}; 914 let Inst{21-20} = 0b10; 915 let Inst{19-16} = imm4; 916 let Inst{15-11} = 0b11100; 917 let Inst{10} = opc{0}; 918 let Inst{9-5} = pattern; 919 let Inst{4-0} = Rd; 920 921 let isReMaterializable = 1; 922} 923 924multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> { 925 def NAME : sve_int_count<opc, asm>; 926 927 def : InstAlias<asm # "\t$Rd, $pattern", 928 (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>; 929 def : InstAlias<asm # "\t$Rd", 930 (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>; 931 932 def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm))), 933 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 934 935 def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm))), 936 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 937 938 def : Pat<(i64 (op sve_pred_enum:$pattern)), 939 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>; 940} 941 942class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty> 943: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 944 asm, "\t$Zdn, $pattern, mul $imm4", 945 "", 946 []>, Sched<[]> { 947 bits<5> Zdn; 948 bits<5> pattern; 949 bits<4> imm4; 950 let Inst{31-24} = 0b00000100; 951 let Inst{23-22} = opc{4-3}; 952 let Inst{21} = 0b1; 953 let Inst{20} = opc{2}; 954 let Inst{19-16} = imm4; 955 let Inst{15-12} = 0b1100; 956 let Inst{11-10} = opc{1-0}; 957 let Inst{9-5} = pattern; 958 let Inst{4-0} = Zdn; 959 960 let Constraints = "$Zdn = $_Zdn"; 961 let DestructiveInstType = DestructiveOther; 962 let ElementSize = ElementSizeNone; 963} 964 965multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty, 966 SDPatternOperator op = null_frag, 967 ValueType vt = OtherVT> { 968 def NAME : sve_int_countvlv<opc, asm, zprty>; 969 970 def : InstAlias<asm # "\t$Zdn, $pattern", 971 (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>; 972 def : InstAlias<asm # "\t$Zdn", 973 (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>; 974 975 def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 976 (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 977} 978 979class sve_int_pred_pattern_a<bits<3> opc, string asm> 980: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 981 asm, "\t$Rdn, $pattern, mul $imm4", 982 "", 983 []>, Sched<[]> { 984 bits<5> Rdn; 985 bits<5> pattern; 986 bits<4> imm4; 987 let Inst{31-24} = 0b00000100; 988 let Inst{23-22} = opc{2-1}; 989 let Inst{21-20} = 0b11; 990 let Inst{19-16} = imm4; 991 let Inst{15-11} = 0b11100; 992 let Inst{10} = opc{0}; 993 let Inst{9-5} = pattern; 994 let Inst{4-0} = Rdn; 995 996 let Constraints = "$Rdn = $_Rdn"; 997} 998 999multiclass sve_int_pred_pattern_a<bits<3> opc, string asm, 1000 SDPatternOperator op, 1001 SDPatternOperator opcnt> { 1002 let Predicates = [HasSVEorSME] in { 1003 def NAME : sve_int_pred_pattern_a<opc, asm>; 1004 1005 def : InstAlias<asm # "\t$Rdn, $pattern", 1006 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1007 def : InstAlias<asm # "\t$Rdn", 1008 (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>; 1009 } 1010 1011 let Predicates = [HasSVEorSME, UseScalarIncVL] in { 1012 def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))), 1013 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>; 1014 1015 def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm)))), 1016 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 1017 1018 def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm)))), 1019 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 1020 1021 def : Pat<(i32 (op GPR32:$Rdn, (i32 (trunc (opcnt (sve_pred_enum:$pattern)))))), 1022 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1023 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1), 1024 sub_32))>; 1025 1026 def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm_i32 i32:$imm)))), 1027 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1028 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 1029 sub_32))>; 1030 1031 def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_shl_imm i32:$imm)))), 1032 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1033 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 1034 sub_32))>; 1035 } 1036} 1037 1038class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt, 1039 RegisterOperand st> 1040: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 1041 asm, "\t$Rdn, $pattern, mul $imm4", 1042 "", 1043 []>, Sched<[]> { 1044 bits<5> Rdn; 1045 bits<5> pattern; 1046 bits<4> imm4; 1047 let Inst{31-24} = 0b00000100; 1048 let Inst{23-22} = opc{4-3}; 1049 let Inst{21} = 0b1; 1050 let Inst{20} = opc{2}; 1051 let Inst{19-16} = imm4; 1052 let Inst{15-12} = 0b1111; 1053 let Inst{11-10} = opc{1-0}; 1054 let Inst{9-5} = pattern; 1055 let Inst{4-0} = Rdn; 1056 1057 // Signed 32bit forms require their GPR operand printed. 1058 let AsmString = !if(!eq(opc{2,0}, 0b00), 1059 !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"), 1060 !strconcat(asm, "\t$Rdn, $pattern, mul $imm4")); 1061 1062 let Constraints = "$Rdn = $_Rdn"; 1063} 1064 1065multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm, 1066 SDPatternOperator op> { 1067 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>; 1068 1069 def : InstAlias<asm # "\t$Rd, $Rn, $pattern", 1070 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>; 1071 def : InstAlias<asm # "\t$Rd, $Rn", 1072 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>; 1073 1074 // NOTE: Register allocation doesn't like tied operands of differing register 1075 // class, hence the extra INSERT_SUBREG complication. 1076 1077 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1078 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>; 1079 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))), 1080 (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1081} 1082 1083multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm, 1084 SDPatternOperator op> { 1085 def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>; 1086 1087 def : InstAlias<asm # "\t$Rdn, $pattern", 1088 (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1089 def : InstAlias<asm # "\t$Rdn", 1090 (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>; 1091 1092 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1093 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1094} 1095 1096multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm, 1097 SDPatternOperator op> { 1098 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>; 1099 1100 def : InstAlias<asm # "\t$Rdn, $pattern", 1101 (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1102 def : InstAlias<asm # "\t$Rdn", 1103 (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>; 1104 1105 def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1106 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1107} 1108 1109 1110//===----------------------------------------------------------------------===// 1111// SVE Permute - Cross Lane Group 1112//===----------------------------------------------------------------------===// 1113 1114class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1115 ValueType vt, RegisterClass srcRegType, 1116 SDPatternOperator op> 1117: I<(outs zprty:$Zd), (ins srcRegType:$Rn), 1118 asm, "\t$Zd, $Rn", 1119 "", 1120 [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> { 1121 bits<5> Rn; 1122 bits<5> Zd; 1123 let Inst{31-24} = 0b00000101; 1124 let Inst{23-22} = sz8_64; 1125 let Inst{21-10} = 0b100000001110; 1126 let Inst{9-5} = Rn; 1127 let Inst{4-0} = Zd; 1128} 1129 1130multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> { 1131 def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>; 1132 def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>; 1133 def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>; 1134 def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>; 1135 1136 def : InstAlias<"mov $Zd, $Rn", 1137 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>; 1138 def : InstAlias<"mov $Zd, $Rn", 1139 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>; 1140 def : InstAlias<"mov $Zd, $Rn", 1141 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>; 1142 def : InstAlias<"mov $Zd, $Rn", 1143 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>; 1144} 1145 1146class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm, 1147 ZPRRegOp zprty> 1148: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx), 1149 asm, "\t$Zd, $Zn$idx", 1150 "", 1151 []>, Sched<[]> { 1152 bits<5> Zd; 1153 bits<5> Zn; 1154 bits<7> idx; 1155 let Inst{31-24} = 0b00000101; 1156 let Inst{23-22} = {?,?}; // imm3h 1157 let Inst{21} = 0b1; 1158 let Inst{20-16} = tsz; 1159 let Inst{15-10} = 0b001000; 1160 let Inst{9-5} = Zn; 1161 let Inst{4-0} = Zd; 1162} 1163 1164multiclass sve_int_perm_dup_i<string asm> { 1165 def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> { 1166 let Inst{23-22} = idx{5-4}; 1167 let Inst{20-17} = idx{3-0}; 1168 } 1169 def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> { 1170 let Inst{23-22} = idx{4-3}; 1171 let Inst{20-18} = idx{2-0}; 1172 } 1173 def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> { 1174 let Inst{23-22} = idx{3-2}; 1175 let Inst{20-19} = idx{1-0}; 1176 } 1177 def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> { 1178 let Inst{23-22} = idx{2-1}; 1179 let Inst{20} = idx{0}; 1180 } 1181 def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> { 1182 let Inst{23-22} = idx{1-0}; 1183 } 1184 1185 def : InstAlias<"mov $Zd, $Zn$idx", 1186 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>; 1187 def : InstAlias<"mov $Zd, $Zn$idx", 1188 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>; 1189 def : InstAlias<"mov $Zd, $Zn$idx", 1190 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>; 1191 def : InstAlias<"mov $Zd, $Zn$idx", 1192 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>; 1193 def : InstAlias<"mov $Zd, $Zn$idx", 1194 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>; 1195 def : InstAlias<"mov $Zd, $Bn", 1196 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>; 1197 def : InstAlias<"mov $Zd, $Hn", 1198 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>; 1199 def : InstAlias<"mov $Zd, $Sn", 1200 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>; 1201 def : InstAlias<"mov $Zd, $Dn", 1202 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>; 1203 def : InstAlias<"mov $Zd, $Qn", 1204 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>; 1205 1206 // Duplicate extracted element of vector into all vector elements 1207 def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))), 1208 (!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>; 1209 def : Pat<(nxv8i16 (splat_vector (i32 (vector_extract (nxv8i16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1210 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1211 def : Pat<(nxv4i32 (splat_vector (i32 (vector_extract (nxv4i32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1212 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1213 def : Pat<(nxv2i64 (splat_vector (i64 (vector_extract (nxv2i64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1214 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1215 def : Pat<(nxv8f16 (splat_vector (f16 (vector_extract (nxv8f16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1216 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1217 def : Pat<(nxv8bf16 (splat_vector (bf16 (vector_extract (nxv8bf16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1218 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1219 def : Pat<(nxv4f16 (splat_vector (f16 (vector_extract (nxv4f16 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1220 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1221 def : Pat<(nxv2f16 (splat_vector (f16 (vector_extract (nxv2f16 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1222 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1223 def : Pat<(nxv4f32 (splat_vector (f32 (vector_extract (nxv4f32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1224 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1225 def : Pat<(nxv2f32 (splat_vector (f32 (vector_extract (nxv2f32 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1226 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1227 def : Pat<(nxv2f64 (splat_vector (f64 (vector_extract (nxv2f64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1228 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1229 1230 def : Pat<(nxv16i8 (AArch64duplane128 nxv16i8:$Op1, i64:$imm)), 1231 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1232 def : Pat<(nxv8i16 (AArch64duplane128 nxv8i16:$Op1, i64:$imm)), 1233 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1234 def : Pat<(nxv4i32 (AArch64duplane128 nxv4i32:$Op1, i64:$imm)), 1235 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1236 def : Pat<(nxv2i64 (AArch64duplane128 nxv2i64:$Op1, i64:$imm)), 1237 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1238 def : Pat<(nxv8f16 (AArch64duplane128 nxv8f16:$Op1, i64:$imm)), 1239 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1240 def : Pat<(nxv4f32 (AArch64duplane128 nxv4f32:$Op1, i64:$imm)), 1241 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1242 def : Pat<(nxv2f64 (AArch64duplane128 nxv2f64:$Op1, i64:$imm)), 1243 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1244 def : Pat<(nxv8bf16 (AArch64duplane128 nxv8bf16:$Op1, i64:$imm)), 1245 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1246} 1247 1248class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty, 1249 RegisterOperand VecList> 1250: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm), 1251 asm, "\t$Zd, $Zn, $Zm", 1252 "", 1253 []>, Sched<[]> { 1254 bits<5> Zd; 1255 bits<5> Zm; 1256 bits<5> Zn; 1257 let Inst{31-24} = 0b00000101; 1258 let Inst{23-22} = sz8_64; 1259 let Inst{21} = 0b1; 1260 let Inst{20-16} = Zm; 1261 let Inst{15-13} = 0b001; 1262 let Inst{12-11} = opc; 1263 let Inst{10} = 0b0; 1264 let Inst{9-5} = Zn; 1265 let Inst{4-0} = Zd; 1266} 1267 1268multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> { 1269 def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>; 1270 def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>; 1271 def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>; 1272 def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>; 1273 1274 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1275 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>; 1276 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1277 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>; 1278 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1279 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>; 1280 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1281 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>; 1282 1283 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1284 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1285 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1286 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1287 1288 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1289 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1290 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1291 1292 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1293} 1294 1295multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> { 1296 def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>; 1297 def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>; 1298 def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>; 1299 def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>; 1300 1301 def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)), 1302 (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0, 1303 nxv16i8:$Op2, zsub1), 1304 nxv16i8:$Op3))>; 1305 1306 def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)), 1307 (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0, 1308 nxv8i16:$Op2, zsub1), 1309 nxv8i16:$Op3))>; 1310 1311 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)), 1312 (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0, 1313 nxv4i32:$Op2, zsub1), 1314 nxv4i32:$Op3))>; 1315 1316 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)), 1317 (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0, 1318 nxv2i64:$Op2, zsub1), 1319 nxv2i64:$Op3))>; 1320 1321 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)), 1322 (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0, 1323 nxv8f16:$Op2, zsub1), 1324 nxv8i16:$Op3))>; 1325 1326 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)), 1327 (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0, 1328 nxv4f32:$Op2, zsub1), 1329 nxv4i32:$Op3))>; 1330 1331 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)), 1332 (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0, 1333 nxv2f64:$Op2, zsub1), 1334 nxv2i64:$Op3))>; 1335 1336 def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)), 1337 (nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0, 1338 nxv8bf16:$Op2, zsub1), 1339 nxv8i16:$Op3))>; 1340} 1341 1342class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1343: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm), 1344 asm, "\t$Zd, $Zn, $Zm", 1345 "", 1346 []>, Sched<[]> { 1347 bits<5> Zd; 1348 bits<5> Zm; 1349 bits<5> Zn; 1350 let Inst{31-24} = 0b00000101; 1351 let Inst{23-22} = sz8_64; 1352 let Inst{21} = 0b1; 1353 let Inst{20-16} = Zm; 1354 let Inst{15-10} = 0b001011; 1355 let Inst{9-5} = Zn; 1356 let Inst{4-0} = Zd; 1357 1358 let Constraints = "$Zd = $_Zd"; 1359} 1360 1361multiclass sve2_int_perm_tbx<string asm, SDPatternOperator op> { 1362 def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>; 1363 def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>; 1364 def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>; 1365 def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>; 1366 1367 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1368 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1369 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1370 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1371 1372 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1373 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1374 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1375 1376 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1377} 1378 1379class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1380: I<(outs zprty:$Zd), (ins zprty:$Zn), 1381 asm, "\t$Zd, $Zn", 1382 "", 1383 []>, Sched<[]> { 1384 bits<5> Zd; 1385 bits<5> Zn; 1386 let Inst{31-24} = 0b00000101; 1387 let Inst{23-22} = sz8_64; 1388 let Inst{21-10} = 0b111000001110; 1389 let Inst{9-5} = Zn; 1390 let Inst{4-0} = Zd; 1391} 1392 1393multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> { 1394 def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>; 1395 def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>; 1396 def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>; 1397 def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>; 1398 1399 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>; 1400 def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 1401 def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 1402 def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 1403 1404 def : SVE_1_Op_Pat<nxv2f16, op, nxv2f16, !cast<Instruction>(NAME # _D)>; 1405 def : SVE_1_Op_Pat<nxv4f16, op, nxv4f16, !cast<Instruction>(NAME # _S)>; 1406 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 1407 def : SVE_1_Op_Pat<nxv2f32, op, nxv2f32, !cast<Instruction>(NAME # _D)>; 1408 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 1409 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 1410 1411 def : SVE_1_Op_Pat<nxv2bf16, op, nxv2bf16, !cast<Instruction>(NAME # _D)>; 1412 def : SVE_1_Op_Pat<nxv4bf16, op, nxv4bf16, !cast<Instruction>(NAME # _S)>; 1413 def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1414} 1415 1416class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty> 1417: I<(outs pprty:$Pd), (ins pprty:$Pn), 1418 asm, "\t$Pd, $Pn", 1419 "", 1420 []>, Sched<[]> { 1421 bits<4> Pd; 1422 bits<4> Pn; 1423 let Inst{31-24} = 0b00000101; 1424 let Inst{23-22} = sz8_64; 1425 let Inst{21-9} = 0b1101000100000; 1426 let Inst{8-5} = Pn; 1427 let Inst{4} = 0b0; 1428 let Inst{3-0} = Pd; 1429} 1430 1431multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> { 1432 def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>; 1433 def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>; 1434 def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>; 1435 def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>; 1436 1437 def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>; 1438 def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>; 1439 def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>; 1440 def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>; 1441} 1442 1443class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm, 1444 ZPRRegOp zprty1, ZPRRegOp zprty2> 1445: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 1446 asm, "\t$Zd, $Zn", 1447 "", []>, Sched<[]> { 1448 bits<5> Zd; 1449 bits<5> Zn; 1450 let Inst{31-24} = 0b00000101; 1451 let Inst{23-22} = sz16_64; 1452 let Inst{21-18} = 0b1100; 1453 let Inst{17-16} = opc; 1454 let Inst{15-10} = 0b001110; 1455 let Inst{9-5} = Zn; 1456 let Inst{4-0} = Zd; 1457} 1458 1459multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> { 1460 def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>; 1461 def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>; 1462 def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>; 1463 1464 def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>; 1465 def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>; 1466 def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>; 1467} 1468 1469class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1470 RegisterClass srcRegType> 1471: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm), 1472 asm, "\t$Zdn, $Rm", 1473 "", 1474 []>, Sched<[]> { 1475 bits<5> Rm; 1476 bits<5> Zdn; 1477 let Inst{31-24} = 0b00000101; 1478 let Inst{23-22} = sz8_64; 1479 let Inst{21-10} = 0b100100001110; 1480 let Inst{9-5} = Rm; 1481 let Inst{4-0} = Zdn; 1482 1483 let Constraints = "$Zdn = $_Zdn"; 1484 let DestructiveInstType = DestructiveOther; 1485} 1486 1487multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> { 1488 def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>; 1489 def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>; 1490 def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>; 1491 def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>; 1492 1493 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 1494 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 1495 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 1496 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>; 1497} 1498 1499class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1500 FPRasZPROperand srcOpType> 1501: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcOpType:$Vm), 1502 asm, "\t$Zdn, $Vm", 1503 "", 1504 []>, Sched<[]> { 1505 bits<5> Vm; 1506 bits<5> Zdn; 1507 let Inst{31-24} = 0b00000101; 1508 let Inst{23-22} = sz8_64; 1509 let Inst{21-10} = 0b110100001110; 1510 let Inst{9-5} = Vm; 1511 let Inst{4-0} = Zdn; 1512 1513 let Constraints = "$Zdn = $_Zdn"; 1514 let DestructiveInstType = DestructiveOther; 1515} 1516 1517multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> { 1518 def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8asZPR>; 1519 def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16asZPR>; 1520 def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32asZPR>; 1521 def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64asZPR>; 1522 1523 def : Pat<(nxv8f16 (op nxv8f16:$Zn, f16:$Vm)), 1524 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1525 def : Pat<(nxv4f32 (op nxv4f32:$Zn, f32:$Vm)), 1526 (!cast<Instruction>(NAME # _S) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, ssub))>; 1527 def : Pat<(nxv2f64 (op nxv2f64:$Zn, f64:$Vm)), 1528 (!cast<Instruction>(NAME # _D) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, dsub))>; 1529 1530 def : Pat<(nxv8bf16 (op nxv8bf16:$Zn, bf16:$Vm)), 1531 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1532 1533 // Keep integer insertions within the vector unit. 1534 def : Pat<(nxv16i8 (op (nxv16i8 ZPR:$Zn), (i32 (vector_extract (nxv16i8 ZPR:$Vm), 0)))), 1535 (!cast<Instruction>(NAME # _B) $Zn, ZPR:$Vm)>; 1536 def : Pat<(nxv8i16 (op (nxv8i16 ZPR:$Zn), (i32 (vector_extract (nxv8i16 ZPR:$Vm), 0)))), 1537 (!cast<Instruction>(NAME # _H) $Zn, ZPR:$Vm)>; 1538 def : Pat<(nxv4i32 (op (nxv4i32 ZPR:$Zn), (i32 (vector_extract (nxv4i32 ZPR:$Vm), 0)))), 1539 (!cast<Instruction>(NAME # _S) $Zn, ZPR: $Vm)>; 1540 def : Pat<(nxv2i64 (op (nxv2i64 ZPR:$Zn), (i64 (vector_extract (nxv2i64 ZPR:$Vm), 0)))), 1541 (!cast<Instruction>(NAME # _D) $Zn, ZPR:$Vm)>; 1542 1543} 1544 1545//===----------------------------------------------------------------------===// 1546// SVE Permute - Extract Group 1547//===----------------------------------------------------------------------===// 1548 1549class sve_int_perm_extract_i<string asm> 1550: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8), 1551 asm, "\t$Zdn, $_Zdn, $Zm, $imm8", 1552 "", []>, Sched<[]> { 1553 bits<5> Zdn; 1554 bits<5> Zm; 1555 bits<8> imm8; 1556 let Inst{31-21} = 0b00000101001; 1557 let Inst{20-16} = imm8{7-3}; 1558 let Inst{15-13} = 0b000; 1559 let Inst{12-10} = imm8{2-0}; 1560 let Inst{9-5} = Zm; 1561 let Inst{4-0} = Zdn; 1562 1563 let Constraints = "$Zdn = $_Zdn"; 1564 let DestructiveInstType = DestructiveOther; 1565 let ElementSize = ElementSizeNone; 1566} 1567 1568multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> { 1569 def NAME : sve_int_perm_extract_i<asm>; 1570 1571 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255, 1572 !cast<Instruction>(NAME)>; 1573} 1574 1575class sve2_int_perm_extract_i_cons<string asm> 1576: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8), 1577 asm, "\t$Zd, $Zn, $imm8", 1578 "", []>, Sched<[]> { 1579 bits<5> Zd; 1580 bits<5> Zn; 1581 bits<8> imm8; 1582 let Inst{31-21} = 0b00000101011; 1583 let Inst{20-16} = imm8{7-3}; 1584 let Inst{15-13} = 0b000; 1585 let Inst{12-10} = imm8{2-0}; 1586 let Inst{9-5} = Zn; 1587 let Inst{4-0} = Zd; 1588} 1589 1590//===----------------------------------------------------------------------===// 1591// SVE Vector Select Group 1592//===----------------------------------------------------------------------===// 1593 1594class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1595: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm), 1596 asm, "\t$Zd, $Pg, $Zn, $Zm", 1597 "", 1598 []>, Sched<[]> { 1599 bits<4> Pg; 1600 bits<5> Zd; 1601 bits<5> Zm; 1602 bits<5> Zn; 1603 let Inst{31-24} = 0b00000101; 1604 let Inst{23-22} = sz8_64; 1605 let Inst{21} = 0b1; 1606 let Inst{20-16} = Zm; 1607 let Inst{15-14} = 0b11; 1608 let Inst{13-10} = Pg; 1609 let Inst{9-5} = Zn; 1610 let Inst{4-0} = Zd; 1611} 1612 1613multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> { 1614 def _B : sve_int_sel_vvv<0b00, asm, ZPR8>; 1615 def _H : sve_int_sel_vvv<0b01, asm, ZPR16>; 1616 def _S : sve_int_sel_vvv<0b10, asm, ZPR32>; 1617 def _D : sve_int_sel_vvv<0b11, asm, ZPR64>; 1618 1619 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1620 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1621 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1622 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1623 1624 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1625 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 1626 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1627 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 1628 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 1629 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1630 1631 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1632 1633 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1634 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>; 1635 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1636 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>; 1637 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1638 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>; 1639 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1640 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>; 1641} 1642 1643 1644//===----------------------------------------------------------------------===// 1645// SVE Predicate Logical Operations Group 1646//===----------------------------------------------------------------------===// 1647 1648class sve_int_pred_log<bits<4> opc, string asm> 1649: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 1650 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 1651 "", 1652 []>, Sched<[]> { 1653 bits<4> Pd; 1654 bits<4> Pg; 1655 bits<4> Pm; 1656 bits<4> Pn; 1657 let Inst{31-24} = 0b00100101; 1658 let Inst{23-22} = opc{3-2}; 1659 let Inst{21-20} = 0b00; 1660 let Inst{19-16} = Pm; 1661 let Inst{15-14} = 0b01; 1662 let Inst{13-10} = Pg; 1663 let Inst{9} = opc{1}; 1664 let Inst{8-5} = Pn; 1665 let Inst{4} = opc{0}; 1666 let Inst{3-0} = Pd; 1667 1668 // SEL has no predication qualifier. 1669 let AsmString = !if(!eq(opc, 0b0011), 1670 !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"), 1671 !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm")); 1672 1673 let Defs = !if(!eq (opc{2}, 1), [NZCV], []); 1674 1675} 1676 1677multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op, 1678 SDPatternOperator op_nopred = null_frag> { 1679 def NAME : sve_int_pred_log<opc, asm>; 1680 1681 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 1682 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>; 1683 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>; 1684 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>; 1685 def : SVE_3_Op_Pat<nxv1i1, op, nxv1i1, nxv1i1, nxv1i1, !cast<Instruction>(NAME)>; 1686 def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1, 1687 !cast<Instruction>(NAME), PTRUE_B>; 1688 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1, 1689 !cast<Instruction>(NAME), PTRUE_H>; 1690 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1, 1691 !cast<Instruction>(NAME), PTRUE_S>; 1692 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1, 1693 !cast<Instruction>(NAME), PTRUE_D>; 1694 // Emulate .Q operation using a PTRUE_D when the other lanes don't matter. 1695 def : SVE_2_Op_AllActive_Pat<nxv1i1, op_nopred, nxv1i1, nxv1i1, 1696 !cast<Instruction>(NAME), PTRUE_D>; 1697} 1698 1699// An instance of sve_int_pred_log_and but uses op_nopred's first operand as the 1700// general predicate. 1701multiclass sve_int_pred_log_v2<bits<4> opc, string asm, SDPatternOperator op, 1702 SDPatternOperator op_nopred> : 1703 sve_int_pred_log<opc, asm, op> { 1704 def : Pat<(nxv16i1 (op_nopred nxv16i1:$Op1, nxv16i1:$Op2)), 1705 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1706 def : Pat<(nxv8i1 (op_nopred nxv8i1:$Op1, nxv8i1:$Op2)), 1707 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1708 def : Pat<(nxv4i1 (op_nopred nxv4i1:$Op1, nxv4i1:$Op2)), 1709 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1710 def : Pat<(nxv2i1 (op_nopred nxv2i1:$Op1, nxv2i1:$Op2)), 1711 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1712 // Emulate .Q operation using a PTRUE_D when the other lanes don't matter. 1713 def : Pat<(nxv1i1 (op_nopred nxv1i1:$Op1, nxv1i1:$Op2)), 1714 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1715} 1716 1717//===----------------------------------------------------------------------===// 1718// SVE Logical Mask Immediate Group 1719//===----------------------------------------------------------------------===// 1720 1721class sve_int_log_imm<bits<2> opc, string asm> 1722: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13), 1723 asm, "\t$Zdn, $_Zdn, $imms13", 1724 "", []>, Sched<[]> { 1725 bits<5> Zdn; 1726 bits<13> imms13; 1727 let Inst{31-24} = 0b00000101; 1728 let Inst{23-22} = opc; 1729 let Inst{21-18} = 0b0000; 1730 let Inst{17-5} = imms13; 1731 let Inst{4-0} = Zdn; 1732 1733 let Constraints = "$Zdn = $_Zdn"; 1734 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1735 let DestructiveInstType = DestructiveOther; 1736 let ElementSize = ElementSizeNone; 1737} 1738 1739multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> { 1740 def NAME : sve_int_log_imm<opc, asm>; 1741 1742 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>; 1743 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>; 1744 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>; 1745 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>; 1746 1747 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1748 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>; 1749 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1750 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>; 1751 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1752 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>; 1753 1754 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1755 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>; 1756 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1757 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>; 1758 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1759 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>; 1760 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1761 (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>; 1762} 1763 1764multiclass sve_int_log_imm_bic<SDPatternOperator op> { 1765 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI")>; 1766 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI")>; 1767 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI")>; 1768 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI")>; 1769} 1770 1771class sve_int_dup_mask_imm<string asm> 1772: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms), 1773 asm, "\t$Zd, $imms", 1774 "", 1775 []>, Sched<[]> { 1776 bits<5> Zd; 1777 bits<13> imms; 1778 let Inst{31-18} = 0b00000101110000; 1779 let Inst{17-5} = imms; 1780 let Inst{4-0} = Zd; 1781 1782 let isReMaterializable = 1; 1783 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1784} 1785 1786multiclass sve_int_dup_mask_imm<string asm> { 1787 def NAME : sve_int_dup_mask_imm<asm>; 1788 1789 def : InstAlias<"dupm $Zd, $imm", 1790 (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>; 1791 def : InstAlias<"dupm $Zd, $imm", 1792 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>; 1793 def : InstAlias<"dupm $Zd, $imm", 1794 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>; 1795 1796 // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here. 1797 def : InstAlias<"mov $Zd, $imm", 1798 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>; 1799 def : InstAlias<"mov $Zd, $imm", 1800 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>; 1801 def : InstAlias<"mov $Zd, $imm", 1802 (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>; 1803 1804 def : Pat<(nxv2i64 (splat_vector (i64 logical_imm64:$imm))), 1805 (!cast<Instruction>(NAME) logical_imm64:$imm)>; 1806} 1807 1808//===----------------------------------------------------------------------===// 1809// SVE Integer Arithmetic - Unpredicated Group. 1810//===----------------------------------------------------------------------===// 1811 1812class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm, 1813 ZPRRegOp zprty> 1814: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1815 asm, "\t$Zd, $Zn, $Zm", 1816 "", []>, Sched<[]> { 1817 bits<5> Zd; 1818 bits<5> Zm; 1819 bits<5> Zn; 1820 let Inst{31-24} = 0b00000100; 1821 let Inst{23-22} = sz8_64; 1822 let Inst{21} = 0b1; 1823 let Inst{20-16} = Zm; 1824 let Inst{15-13} = 0b000; 1825 let Inst{12-10} = opc; 1826 let Inst{9-5} = Zn; 1827 let Inst{4-0} = Zd; 1828} 1829 1830multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, SDPatternOperator op> { 1831 def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>; 1832 def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>; 1833 def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>; 1834 def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>; 1835 1836 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1837 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1838 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1839 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1840} 1841 1842//===----------------------------------------------------------------------===// 1843// SVE Floating Point Arithmetic - Predicated Group 1844//===----------------------------------------------------------------------===// 1845 1846class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm, 1847 ZPRRegOp zprty, 1848 Operand imm_ty> 1849: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1), 1850 asm, "\t$Zdn, $Pg/m, $_Zdn, $i1", 1851 "", 1852 []>, Sched<[]> { 1853 bits<3> Pg; 1854 bits<5> Zdn; 1855 bit i1; 1856 let Inst{31-24} = 0b01100101; 1857 let Inst{23-22} = sz; 1858 let Inst{21-19} = 0b011; 1859 let Inst{18-16} = opc; 1860 let Inst{15-13} = 0b100; 1861 let Inst{12-10} = Pg; 1862 let Inst{9-6} = 0b0000; 1863 let Inst{5} = i1; 1864 let Inst{4-0} = Zdn; 1865 1866 let Constraints = "$Zdn = $_Zdn"; 1867 let DestructiveInstType = DestructiveOther; 1868 let ElementSize = zprty.ElementSize; 1869} 1870 1871multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, string Ps, Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 1872 let DestructiveInstType = DestructiveBinaryImm in { 1873 def _H : SVEPseudo2Instr<Ps # _H, 1>, sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>; 1874 def _S : SVEPseudo2Instr<Ps # _S, 1>, sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>; 1875 def _D : SVEPseudo2Instr<Ps # _D, 1>, sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>; 1876 } 1877 1878 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_H")>; 1879 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_H")>; 1880 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_S")>; 1881 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_S")>; 1882 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_D")>; 1883 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_D")>; 1884} 1885 1886class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm, 1887 ZPRRegOp zprty> 1888: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 1889 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 1890 "", 1891 []>, Sched<[]> { 1892 bits<3> Pg; 1893 bits<5> Zdn; 1894 bits<5> Zm; 1895 let Inst{31-24} = 0b01100101; 1896 let Inst{23-22} = sz; 1897 let Inst{21-20} = 0b00; 1898 let Inst{19-16} = opc; 1899 let Inst{15-13} = 0b100; 1900 let Inst{12-10} = Pg; 1901 let Inst{9-5} = Zm; 1902 let Inst{4-0} = Zdn; 1903 1904 let Constraints = "$Zdn = $_Zdn"; 1905 let DestructiveInstType = DestructiveOther; 1906 let ElementSize = zprty.ElementSize; 1907} 1908 1909multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps, 1910 SDPatternOperator op, DestructiveInstTypeEnum flags, 1911 string revname="", bit isReverseInstr=0> { 1912 let DestructiveInstType = flags in { 1913 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>, 1914 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1915 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>, 1916 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1917 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>, 1918 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1919 } 1920 1921 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1922 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1923 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1924} 1925 1926multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm, 1927 SDPatternOperator op> { 1928 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>; 1929 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>; 1930 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>; 1931 1932 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1933 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1934 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1935} 1936 1937multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> { 1938 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 1939 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 1940 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 1941 1942 def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>; 1943 def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>; 1944 def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>; 1945} 1946 1947class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty> 1948: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, timm32_0_7:$imm3), 1949 asm, "\t$Zdn, $_Zdn, $Zm, $imm3", 1950 "", 1951 []>, Sched<[]> { 1952 bits<5> Zdn; 1953 bits<5> Zm; 1954 bits<3> imm3; 1955 let Inst{31-24} = 0b01100101; 1956 let Inst{23-22} = sz; 1957 let Inst{21-19} = 0b010; 1958 let Inst{18-16} = imm3; 1959 let Inst{15-10} = 0b100000; 1960 let Inst{9-5} = Zm; 1961 let Inst{4-0} = Zdn; 1962 1963 let Constraints = "$Zdn = $_Zdn"; 1964 let DestructiveInstType = DestructiveOther; 1965 let ElementSize = ElementSizeNone; 1966} 1967 1968multiclass sve_fp_ftmad<string asm, SDPatternOperator op> { 1969 def _H : sve_fp_ftmad<0b01, asm, ZPR16>; 1970 def _S : sve_fp_ftmad<0b10, asm, ZPR32>; 1971 def _D : sve_fp_ftmad<0b11, asm, ZPR64>; 1972 1973 def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 timm32_0_7:$imm))), 1974 (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, timm32_0_7:$imm)>; 1975 def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 timm32_0_7:$imm))), 1976 (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, timm32_0_7:$imm)>; 1977 def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 timm32_0_7:$imm))), 1978 (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, timm32_0_7:$imm)>; 1979} 1980 1981multiclass sve_fp_2op_i_p_zds_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator ir_op = null_frag> { 1982 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesUndef>; 1983 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesUndef>; 1984 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesUndef>; 1985 1986 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1987 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1988 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1989 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1990 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1991 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1992 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 1993 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 1994 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 1995 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 1996 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_UNDEF_D")>; 1997 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_UNDEF_D")>; 1998} 1999 2000multiclass sve_fp_2op_i_p_zds_zeroing_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 2001 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesZero>; 2002 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesZero>; 2003 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesZero>; 2004 2005 let AddedComplexity = 2 in { 2006 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_ZERO_H")>; 2007 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_ZERO_H")>; 2008 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_ZERO_S")>; 2009 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_ZERO_S")>; 2010 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_ZERO_D")>; 2011 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_ZERO_D")>; 2012 } 2013} 2014 2015//===----------------------------------------------------------------------===// 2016// SVE Floating Point Arithmetic - Unpredicated Group 2017//===----------------------------------------------------------------------===// 2018 2019class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 2020: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2021 asm, "\t$Zd, $Zn, $Zm", 2022 "", 2023 []>, Sched<[]> { 2024 bits<5> Zd; 2025 bits<5> Zm; 2026 bits<5> Zn; 2027 let Inst{31-24} = 0b01100101; 2028 let Inst{23-22} = sz; 2029 let Inst{21} = 0b0; 2030 let Inst{20-16} = Zm; 2031 let Inst{15-13} = 0b000; 2032 let Inst{12-10} = opc; 2033 let Inst{9-5} = Zn; 2034 let Inst{4-0} = Zd; 2035} 2036 2037multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op, 2038 SDPatternOperator predicated_op = null_frag> { 2039 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 2040 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 2041 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 2042 2043 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2044 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2045 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2046 2047 def : SVE_2_Op_Pred_All_Active<nxv8f16, predicated_op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2048 def : SVE_2_Op_Pred_All_Active<nxv4f32, predicated_op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2049 def : SVE_2_Op_Pred_All_Active<nxv2f64, predicated_op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2050} 2051 2052multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> { 2053 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 2054 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 2055 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 2056 2057 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2058 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2059 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2060} 2061 2062//===----------------------------------------------------------------------===// 2063// SVE Floating Point Fused Multiply-Add Group 2064//===----------------------------------------------------------------------===// 2065 2066class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty> 2067: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2068 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2069 "", 2070 []>, Sched<[]> { 2071 bits<3> Pg; 2072 bits<5> Zda; 2073 bits<5> Zm; 2074 bits<5> Zn; 2075 let Inst{31-24} = 0b01100101; 2076 let Inst{23-22} = sz; 2077 let Inst{21} = 0b1; 2078 let Inst{20-16} = Zm; 2079 let Inst{15} = 0b0; 2080 let Inst{14-13} = opc; 2081 let Inst{12-10} = Pg; 2082 let Inst{9-5} = Zn; 2083 let Inst{4-0} = Zda; 2084 2085 let Constraints = "$Zda = $_Zda"; 2086 let ElementSize = zprty.ElementSize; 2087} 2088 2089multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, string Ps, 2090 SDPatternOperator op, string revname, 2091 bit isReverseInstr=0> { 2092 let DestructiveInstType = DestructiveTernaryCommWithRev in { 2093 def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>, 2094 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2095 def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>, 2096 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2097 def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>, 2098 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2099 } 2100 2101 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2102 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2103 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2104} 2105 2106class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm, 2107 ZPRRegOp zprty> 2108: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2109 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2110 "", 2111 []>, Sched<[]> { 2112 bits<3> Pg; 2113 bits<5> Za; 2114 bits<5> Zdn; 2115 bits<5> Zm; 2116 let Inst{31-24} = 0b01100101; 2117 let Inst{23-22} = sz; 2118 let Inst{21} = 0b1; 2119 let Inst{20-16} = Za; 2120 let Inst{15} = 0b1; 2121 let Inst{14-13} = opc; 2122 let Inst{12-10} = Pg; 2123 let Inst{9-5} = Zm; 2124 let Inst{4-0} = Zdn; 2125 2126 let Constraints = "$Zdn = $_Zdn"; 2127 let DestructiveInstType = DestructiveOther; 2128 let ElementSize = zprty.ElementSize; 2129} 2130 2131multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op, 2132 string revname, bit isReverseInstr> { 2133 def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>, 2134 SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2135 def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>, 2136 SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2137 def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>, 2138 SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2139 2140 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2141 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2142 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2143} 2144 2145multiclass sve_fp_3op_p_zds_zx { 2146 def _UNDEF_H : PredThreeOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 2147 def _UNDEF_S : PredThreeOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 2148 def _UNDEF_D : PredThreeOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 2149} 2150 2151//===----------------------------------------------------------------------===// 2152// SVE Floating Point Multiply-Add - Indexed Group 2153//===----------------------------------------------------------------------===// 2154 2155class sve_fp_fma_by_indexed_elem<bits<2> sz, bit opc, string asm, 2156 ZPRRegOp zprty1, 2157 ZPRRegOp zprty2, Operand itype> 2158: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop), 2159 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 2160 bits<5> Zda; 2161 bits<5> Zn; 2162 let Inst{31-24} = 0b01100100; 2163 let Inst{23-22} = sz; 2164 let Inst{21} = 0b1; 2165 let Inst{15-11} = 0; 2166 let Inst{10} = opc; 2167 let Inst{9-5} = Zn; 2168 let Inst{4-0} = Zda; 2169 2170 let Constraints = "$Zda = $_Zda"; 2171 let DestructiveInstType = DestructiveOther; 2172 let ElementSize = ElementSizeNone; 2173} 2174 2175multiclass sve_fp_fma_by_indexed_elem<bit opc, string asm, 2176 SDPatternOperator op> { 2177 def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2178 bits<3> Zm; 2179 bits<3> iop; 2180 let Inst{22} = iop{2}; 2181 let Inst{20-19} = iop{1-0}; 2182 let Inst{18-16} = Zm; 2183 } 2184 def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2185 bits<3> Zm; 2186 bits<2> iop; 2187 let Inst{20-19} = iop; 2188 let Inst{18-16} = Zm; 2189 } 2190 def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2191 bits<4> Zm; 2192 bit iop; 2193 let Inst{20} = iop; 2194 let Inst{19-16} = Zm; 2195 } 2196 2197 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))), 2198 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>; 2199 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))), 2200 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 2201 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))), 2202 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 2203} 2204 2205 2206//===----------------------------------------------------------------------===// 2207// SVE Floating Point Multiply - Indexed Group 2208//===----------------------------------------------------------------------===// 2209 2210class sve_fp_fmul_by_indexed_elem<bits<2> sz, string asm, ZPRRegOp zprty, 2211 ZPRRegOp zprty2, Operand itype> 2212: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop), 2213 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 2214 bits<5> Zd; 2215 bits<5> Zn; 2216 let Inst{31-24} = 0b01100100; 2217 let Inst{23-22} = sz; 2218 let Inst{21} = 0b1; 2219 let Inst{15-10} = 0b001000; 2220 let Inst{9-5} = Zn; 2221 let Inst{4-0} = Zd; 2222} 2223 2224multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> { 2225 def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2226 bits<3> Zm; 2227 bits<3> iop; 2228 let Inst{22} = iop{2}; 2229 let Inst{20-19} = iop{1-0}; 2230 let Inst{18-16} = Zm; 2231 } 2232 def _S : sve_fp_fmul_by_indexed_elem<0b10, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2233 bits<3> Zm; 2234 bits<2> iop; 2235 let Inst{20-19} = iop; 2236 let Inst{18-16} = Zm; 2237 } 2238 def _D : sve_fp_fmul_by_indexed_elem<0b11, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2239 bits<4> Zm; 2240 bit iop; 2241 let Inst{20} = iop; 2242 let Inst{19-16} = Zm; 2243 } 2244 2245 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))), 2246 (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>; 2247 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))), 2248 (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>; 2249 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))), 2250 (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>; 2251} 2252 2253//===----------------------------------------------------------------------===// 2254// SVE Floating Point Complex Multiply-Add Group 2255//===----------------------------------------------------------------------===// 2256 2257class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty> 2258: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm, 2259 complexrotateop:$imm), 2260 asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm", 2261 "", []>, Sched<[]> { 2262 bits<5> Zda; 2263 bits<3> Pg; 2264 bits<5> Zn; 2265 bits<5> Zm; 2266 bits<2> imm; 2267 let Inst{31-24} = 0b01100100; 2268 let Inst{23-22} = sz; 2269 let Inst{21} = 0; 2270 let Inst{20-16} = Zm; 2271 let Inst{15} = 0; 2272 let Inst{14-13} = imm; 2273 let Inst{12-10} = Pg; 2274 let Inst{9-5} = Zn; 2275 let Inst{4-0} = Zda; 2276 2277 let Constraints = "$Zda = $_Zda"; 2278 let DestructiveInstType = DestructiveOther; 2279 let ElementSize = zprty.ElementSize; 2280} 2281 2282multiclass sve_fp_fcmla<string asm, SDPatternOperator op> { 2283 def _H : sve_fp_fcmla<0b01, asm, ZPR16>; 2284 def _S : sve_fp_fcmla<0b10, asm, ZPR32>; 2285 def _D : sve_fp_fcmla<0b11, asm, ZPR64>; 2286 2287 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))), 2288 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2289 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))), 2290 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2291 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))), 2292 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2293} 2294 2295//===----------------------------------------------------------------------===// 2296// SVE Floating Point Complex Multiply-Add - Indexed Group 2297//===----------------------------------------------------------------------===// 2298 2299class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm, 2300 ZPRRegOp zprty, 2301 ZPRRegOp zprty2, Operand itype> 2302: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop, 2303 complexrotateop:$imm), 2304 asm, "\t$Zda, $Zn, $Zm$iop, $imm", 2305 "", []>, Sched<[]> { 2306 bits<5> Zda; 2307 bits<5> Zn; 2308 bits<2> imm; 2309 let Inst{31-24} = 0b01100100; 2310 let Inst{23-22} = sz; 2311 let Inst{21} = 0b1; 2312 let Inst{15-12} = 0b0001; 2313 let Inst{11-10} = imm; 2314 let Inst{9-5} = Zn; 2315 let Inst{4-0} = Zda; 2316 2317 let Constraints = "$Zda = $_Zda"; 2318 let DestructiveInstType = DestructiveOther; 2319 let ElementSize = ElementSizeNone; 2320} 2321 2322multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> { 2323 def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> { 2324 bits<3> Zm; 2325 bits<2> iop; 2326 let Inst{20-19} = iop; 2327 let Inst{18-16} = Zm; 2328 } 2329 def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> { 2330 bits<4> Zm; 2331 bits<1> iop; 2332 let Inst{20} = iop; 2333 let Inst{19-16} = Zm; 2334 } 2335 2336 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2337 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2338 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2339 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2340} 2341 2342//===----------------------------------------------------------------------===// 2343// SVE Floating Point Complex Addition Group 2344//===----------------------------------------------------------------------===// 2345 2346class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty> 2347: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, 2348 complexrotateopodd:$imm), 2349 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm", 2350 "", 2351 []>, Sched<[]> { 2352 bits<5> Zdn; 2353 bits<5> Zm; 2354 bits<3> Pg; 2355 bit imm; 2356 let Inst{31-24} = 0b01100100; 2357 let Inst{23-22} = sz; 2358 let Inst{21-17} = 0; 2359 let Inst{16} = imm; 2360 let Inst{15-13} = 0b100; 2361 let Inst{12-10} = Pg; 2362 let Inst{9-5} = Zm; 2363 let Inst{4-0} = Zdn; 2364 2365 let Constraints = "$Zdn = $_Zdn"; 2366 let DestructiveInstType = DestructiveOther; 2367 let ElementSize = zprty.ElementSize; 2368} 2369 2370multiclass sve_fp_fcadd<string asm, SDPatternOperator op> { 2371 def _H : sve_fp_fcadd<0b01, asm, ZPR16>; 2372 def _S : sve_fp_fcadd<0b10, asm, ZPR32>; 2373 def _D : sve_fp_fcadd<0b11, asm, ZPR64>; 2374 2375 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))), 2376 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2377 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))), 2378 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2379 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))), 2380 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2381} 2382 2383//===----------------------------------------------------------------------===// 2384// SVE2 Floating Point Convert Group 2385//===----------------------------------------------------------------------===// 2386 2387class sve2_fp_convert_precision<bits<4> opc, string asm, 2388 ZPRRegOp zprty1, ZPRRegOp zprty2> 2389: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn), 2390 asm, "\t$Zd, $Pg/m, $Zn", 2391 "", 2392 []>, Sched<[]> { 2393 bits<5> Zd; 2394 bits<5> Zn; 2395 bits<3> Pg; 2396 let Inst{31-24} = 0b01100100; 2397 let Inst{23-22} = opc{3-2}; 2398 let Inst{21-18} = 0b0010; 2399 let Inst{17-16} = opc{1-0}; 2400 let Inst{15-13} = 0b101; 2401 let Inst{12-10} = Pg; 2402 let Inst{9-5} = Zn; 2403 let Inst{4-0} = Zd; 2404 2405 let Constraints = "$Zd = $_Zd"; 2406} 2407 2408multiclass sve2_fp_convert_down_narrow<string asm, string op> { 2409 def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>; 2410 def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>; 2411 2412 def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>; 2413 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2414} 2415 2416multiclass sve2_fp_convert_up_long<string asm, string op> { 2417 def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>; 2418 def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>; 2419 2420 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>; 2421 def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>; 2422} 2423 2424multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> { 2425 def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>; 2426 2427 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2428} 2429 2430//===----------------------------------------------------------------------===// 2431// SVE2 Floating Point Pairwise Group 2432//===----------------------------------------------------------------------===// 2433 2434class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm, 2435 ZPRRegOp zprty> 2436: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2437 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 2438 "", 2439 []>, Sched<[]> { 2440 bits<3> Pg; 2441 bits<5> Zm; 2442 bits<5> Zdn; 2443 let Inst{31-24} = 0b01100100; 2444 let Inst{23-22} = sz; 2445 let Inst{21-19} = 0b010; 2446 let Inst{18-16} = opc; 2447 let Inst{15-13} = 0b100; 2448 let Inst{12-10} = Pg; 2449 let Inst{9-5} = Zm; 2450 let Inst{4-0} = Zdn; 2451 2452 let Constraints = "$Zdn = $_Zdn"; 2453 let DestructiveInstType = DestructiveOther; 2454 let ElementSize = zprty.ElementSize; 2455} 2456 2457multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm, 2458 SDPatternOperator op> { 2459 def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>; 2460 def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>; 2461 def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>; 2462 2463 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2464 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2465 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2466} 2467 2468//===----------------------------------------------------------------------===// 2469// SVE2 Floating Point Widening Multiply-Add - Indexed Group 2470//===----------------------------------------------------------------------===// 2471 2472class sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm> 2473: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, 2474 VectorIndexH32b:$iop), 2475 asm, "\t$Zda, $Zn, $Zm$iop", 2476 "", 2477 []>, Sched<[]> { 2478 bits<5> Zda; 2479 bits<5> Zn; 2480 bits<3> Zm; 2481 bits<3> iop; 2482 let Inst{31-21} = 0b01100100101; 2483 let Inst{20-19} = iop{2-1}; 2484 let Inst{18-16} = Zm; 2485 let Inst{15-14} = 0b01; 2486 let Inst{13} = opc{1}; 2487 let Inst{12} = 0b0; 2488 let Inst{11} = iop{0}; 2489 let Inst{10} = opc{0}; 2490 let Inst{9-5} = Zn; 2491 let Inst{4-0} = Zda; 2492 2493 let Constraints = "$Zda = $_Zda"; 2494 let DestructiveInstType = DestructiveOther; 2495 let ElementSize = ElementSizeNone; 2496} 2497 2498multiclass sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm, 2499 SDPatternOperator op> { 2500 def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>; 2501 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>; 2502} 2503 2504//===----------------------------------------------------------------------===// 2505// SVE2 Floating Point Widening Multiply-Add Group 2506//===----------------------------------------------------------------------===// 2507 2508class sve2_fp_mla_long<bits<2> opc, string asm> 2509: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 2510 asm, "\t$Zda, $Zn, $Zm", 2511 "", 2512 []>, Sched<[]> { 2513 bits<5> Zda; 2514 bits<5> Zn; 2515 bits<5> Zm; 2516 let Inst{31-21} = 0b01100100101; 2517 let Inst{20-16} = Zm; 2518 let Inst{15-14} = 0b10; 2519 let Inst{13} = opc{1}; 2520 let Inst{12-11} = 0b00; 2521 let Inst{10} = opc{0}; 2522 let Inst{9-5} = Zn; 2523 let Inst{4-0} = Zda; 2524 2525 let Constraints = "$Zda = $_Zda"; 2526 let DestructiveInstType = DestructiveOther; 2527 let ElementSize = ElementSizeNone; 2528} 2529 2530multiclass sve2_fp_mla_long<bits<2> opc, string asm, SDPatternOperator op> { 2531 def NAME : sve2_fp_mla_long<opc, asm>; 2532 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 2533} 2534 2535//===----------------------------------------------------------------------===// 2536// SVE Stack Allocation Group 2537//===----------------------------------------------------------------------===// 2538 2539class sve_int_arith_vl<bit opc, string asm, bit streaming_sve = 0b0> 2540: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6), 2541 asm, "\t$Rd, $Rn, $imm6", 2542 "", 2543 []>, Sched<[]> { 2544 bits<5> Rd; 2545 bits<5> Rn; 2546 bits<6> imm6; 2547 let Inst{31-23} = 0b000001000; 2548 let Inst{22} = opc; 2549 let Inst{21} = 0b1; 2550 let Inst{20-16} = Rn; 2551 let Inst{15-12} = 0b0101; 2552 let Inst{11} = streaming_sve; 2553 let Inst{10-5} = imm6; 2554 let Inst{4-0} = Rd; 2555} 2556 2557class sve_int_read_vl_a<bit op, bits<5> opc2, string asm, bit streaming_sve = 0b0> 2558: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6), 2559 asm, "\t$Rd, $imm6", 2560 "", 2561 []>, Sched<[]> { 2562 bits<5> Rd; 2563 bits<6> imm6; 2564 let Inst{31-23} = 0b000001001; 2565 let Inst{22} = op; 2566 let Inst{21} = 0b1; 2567 let Inst{20-16} = opc2{4-0}; 2568 let Inst{15-12} = 0b0101; 2569 let Inst{11} = streaming_sve; 2570 let Inst{10-5} = imm6; 2571 let Inst{4-0} = Rd; 2572 2573 let isReMaterializable = 1; 2574} 2575 2576//===----------------------------------------------------------------------===// 2577// SVE Permute - In Lane Group 2578//===----------------------------------------------------------------------===// 2579 2580class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm, 2581 ZPRRegOp zprty> 2582: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2583 asm, "\t$Zd, $Zn, $Zm", 2584 "", 2585 []>, Sched<[]> { 2586 bits<5> Zd; 2587 bits<5> Zm; 2588 bits<5> Zn; 2589 let Inst{31-24} = 0b00000101; 2590 let Inst{23-22} = sz8_64; 2591 let Inst{21} = 0b1; 2592 let Inst{20-16} = Zm; 2593 let Inst{15-13} = 0b011; 2594 let Inst{12-10} = opc; 2595 let Inst{9-5} = Zn; 2596 let Inst{4-0} = Zd; 2597} 2598 2599multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm, 2600 SDPatternOperator op> { 2601 def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>; 2602 def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>; 2603 def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>; 2604 def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>; 2605 2606 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2607 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2608 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2609 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2610 2611 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2612 def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 2613 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2614 def : SVE_2_Op_Pat<nxv2f16, op, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 2615 def : SVE_2_Op_Pat<nxv2f32, op, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 2616 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2617 2618 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 2619} 2620 2621//===----------------------------------------------------------------------===// 2622// SVE Floating Point Unary Operations Group 2623//===----------------------------------------------------------------------===// 2624 2625class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype, 2626 RegisterOperand o_zprtype, ElementSizeEnum Sz> 2627: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn), 2628 asm, "\t$Zd, $Pg/m, $Zn", 2629 "", 2630 []>, Sched<[]> { 2631 bits<3> Pg; 2632 bits<5> Zd; 2633 bits<5> Zn; 2634 let Inst{31-24} = 0b01100101; 2635 let Inst{23-22} = opc{6-5}; 2636 let Inst{21} = 0b0; 2637 let Inst{20-16} = opc{4-0}; 2638 let Inst{15-13} = 0b101; 2639 let Inst{12-10} = Pg; 2640 let Inst{9-5} = Zn; 2641 let Inst{4-0} = Zd; 2642 2643 let Constraints = "$Zd = $_Zd"; 2644 let DestructiveInstType = DestructiveUnaryPassthru; 2645 let ElementSize = Sz; 2646} 2647 2648multiclass sve_fp_2op_p_zd<bits<7> opc, string asm, 2649 RegisterOperand i_zprtype, 2650 RegisterOperand o_zprtype, 2651 SDPatternOperator int_op, 2652 SDPatternOperator ir_op, ValueType vt1, 2653 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2654 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>, 2655 SVEPseudo2Instr<NAME, 1>; 2656 // convert vt1 to a packed type for the intrinsic patterns 2657 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2658 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2659 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2660 1 : vt1); 2661 2662 // convert vt3 to a packed type for the intrinsic patterns 2663 defvar packedvt3 = !cond(!eq(!cast<string>(vt3), "nxv2f16"): nxv8f16, 2664 !eq(!cast<string>(vt3), "nxv4f16"): nxv8f16, 2665 !eq(!cast<string>(vt3), "nxv2f32"): nxv4f32, 2666 1 : vt3); 2667 2668 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, packedvt3, !cast<Instruction>(NAME)>; 2669 def : SVE_1_Op_Passthru_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2670 2671 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>; 2672 2673 defm : SVE_1_Op_PassthruUndef_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>; 2674} 2675 2676multiclass sve_fp_2op_p_zdr<bits<7> opc, string asm, 2677 RegisterOperand i_zprtype, 2678 RegisterOperand o_zprtype, 2679 SDPatternOperator int_op, 2680 SDPatternOperator ir_op, ValueType vt1, 2681 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2682 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>, 2683 SVEPseudo2Instr<NAME, 1>; 2684 2685 // convert vt1 to a packed type for the intrinsic patterns 2686 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2687 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2688 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2689 1 : vt1); 2690 2691 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, vt3, !cast<Instruction>(NAME)>; 2692 def : SVE_1_Op_Passthru_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2693 2694 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>; 2695 2696 defm : SVE_1_Op_PassthruUndef_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>; 2697} 2698 2699multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> { 2700 def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>, 2701 SVEPseudo2Instr<NAME # _H, 1>; 2702 def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>, 2703 SVEPseudo2Instr<NAME # _S, 1>; 2704 def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>, 2705 SVEPseudo2Instr<NAME # _D, 1>; 2706 2707 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2708 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 2709 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 2710 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2711 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 2712 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2713 2714 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 2715 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 2716 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 2717 2718 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2719 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2720 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2721 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2722 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2723 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _UNDEF_D)>; 2724} 2725 2726multiclass sve2_fp_flogb<string asm, SDPatternOperator op> { 2727 def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>; 2728 def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>; 2729 def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>; 2730 2731 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2732 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2733 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2734} 2735 2736multiclass sve2_fp_convert_down_odd_rounding<string asm, string op> { 2737 def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>; 2738 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2739} 2740 2741//===----------------------------------------------------------------------===// 2742// SVE Floating Point Unary Operations - Unpredicated Group 2743//===----------------------------------------------------------------------===// 2744 2745class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm, 2746 ZPRRegOp zprty> 2747: I<(outs zprty:$Zd), (ins zprty:$Zn), 2748 asm, "\t$Zd, $Zn", 2749 "", 2750 []>, Sched<[]> { 2751 bits<5> Zd; 2752 bits<5> Zn; 2753 let Inst{31-24} = 0b01100101; 2754 let Inst{23-22} = sz; 2755 let Inst{21-19} = 0b001; 2756 let Inst{18-16} = opc; 2757 let Inst{15-10} = 0b001100; 2758 let Inst{9-5} = Zn; 2759 let Inst{4-0} = Zd; 2760} 2761 2762multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> { 2763 def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>; 2764 def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>; 2765 def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>; 2766 2767 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 2768 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 2769 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 2770} 2771 2772//===----------------------------------------------------------------------===// 2773// SVE Integer Arithmetic - Binary Predicated Group 2774//===----------------------------------------------------------------------===// 2775 2776class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc, 2777 string asm, ZPRRegOp zprty> 2778: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2779 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 2780 bits<3> Pg; 2781 bits<5> Zdn; 2782 bits<5> Zm; 2783 let Inst{31-24} = 0b00000100; 2784 let Inst{23-22} = sz8_64; 2785 let Inst{21} = 0b0; 2786 let Inst{20-19} = fmt; 2787 let Inst{18-16} = opc; 2788 let Inst{15-13} = 0b000; 2789 let Inst{12-10} = Pg; 2790 let Inst{9-5} = Zm; 2791 let Inst{4-0} = Zdn; 2792 2793 let Constraints = "$Zdn = $_Zdn"; 2794 let DestructiveInstType = DestructiveOther; 2795 let ElementSize = zprty.ElementSize; 2796} 2797 2798multiclass sve_int_bin_pred_log<bits<3> opc, string asm, string Ps, 2799 SDPatternOperator op, 2800 DestructiveInstTypeEnum flags> { 2801 let DestructiveInstType = flags in { 2802 def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>, 2803 SVEPseudo2Instr<Ps # _B, 1>; 2804 def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>, 2805 SVEPseudo2Instr<Ps # _H, 1>; 2806 def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>, 2807 SVEPseudo2Instr<Ps # _S, 1>; 2808 def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>, 2809 SVEPseudo2Instr<Ps # _D, 1>; 2810 } 2811 2812 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2813 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2814 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2815 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2816} 2817 2818multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps, 2819 SDPatternOperator op, 2820 DestructiveInstTypeEnum flags, 2821 string revname="", bit isReverseInstr=0> { 2822 let DestructiveInstType = flags in { 2823 def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>, 2824 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 2825 def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>, 2826 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2827 def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>, 2828 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2829 def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>, 2830 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2831 } 2832 2833 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2834 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2835 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2836 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2837} 2838 2839multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps, 2840 SDPatternOperator op, 2841 DestructiveInstTypeEnum flags> { 2842 let DestructiveInstType = flags in { 2843 def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>, 2844 SVEPseudo2Instr<Ps # _B, 1>; 2845 def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>, 2846 SVEPseudo2Instr<Ps # _H, 1>; 2847 def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>, 2848 SVEPseudo2Instr<Ps # _S, 1>; 2849 def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>, 2850 SVEPseudo2Instr<Ps # _D, 1>; 2851 } 2852 2853 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2854 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2855 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2856 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2857} 2858 2859multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps, 2860 SDPatternOperator op, 2861 DestructiveInstTypeEnum flags> { 2862 let DestructiveInstType = flags in { 2863 def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>, 2864 SVEPseudo2Instr<Ps # _B, 1>; 2865 def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>, 2866 SVEPseudo2Instr<Ps # _H, 1>; 2867 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2868 SVEPseudo2Instr<Ps # _S, 1>; 2869 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2870 SVEPseudo2Instr<Ps # _D, 1>; 2871 } 2872 2873 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2874 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2875 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2876 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2877} 2878 2879// Special case for divides which are not defined for 8b/16b elements. 2880multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps, 2881 SDPatternOperator op, 2882 DestructiveInstTypeEnum flags, 2883 string revname="", bit isReverseInstr=0> { 2884 let DestructiveInstType = flags in { 2885 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2886 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2887 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2888 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2889 } 2890 2891 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2892 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2893} 2894 2895//===----------------------------------------------------------------------===// 2896// SVE Integer Multiply-Add Group 2897//===----------------------------------------------------------------------===// 2898 2899class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2900 ZPRRegOp zprty> 2901: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2902 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2903 "", 2904 []>, Sched<[]> { 2905 bits<3> Pg; 2906 bits<5> Zdn; 2907 bits<5> Za; 2908 bits<5> Zm; 2909 let Inst{31-24} = 0b00000100; 2910 let Inst{23-22} = sz8_64; 2911 let Inst{21} = 0b0; 2912 let Inst{20-16} = Zm; 2913 let Inst{15-14} = 0b11; 2914 let Inst{13} = opc; 2915 let Inst{12-10} = Pg; 2916 let Inst{9-5} = Za; 2917 let Inst{4-0} = Zdn; 2918 2919 let Constraints = "$Zdn = $_Zdn"; 2920 let DestructiveInstType = DestructiveOther; 2921 let ElementSize = zprty.ElementSize; 2922} 2923 2924multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2925 def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>; 2926 def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>; 2927 def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>; 2928 def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>; 2929 2930 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2931 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2932 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2933 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2934} 2935 2936class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2937 ZPRRegOp zprty> 2938: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2939 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2940 "", 2941 []>, Sched<[]> { 2942 bits<3> Pg; 2943 bits<5> Zda; 2944 bits<5> Zm; 2945 bits<5> Zn; 2946 let Inst{31-24} = 0b00000100; 2947 let Inst{23-22} = sz8_64; 2948 let Inst{21} = 0b0; 2949 let Inst{20-16} = Zm; 2950 let Inst{15-14} = 0b01; 2951 let Inst{13} = opc; 2952 let Inst{12-10} = Pg; 2953 let Inst{9-5} = Zn; 2954 let Inst{4-0} = Zda; 2955 2956 let Constraints = "$Zda = $_Zda"; 2957 let DestructiveInstType = DestructiveOther; 2958 let ElementSize = zprty.ElementSize; 2959} 2960 2961multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2962 def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>; 2963 def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>; 2964 def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>; 2965 def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>; 2966 2967 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2968 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2969 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2970 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2971} 2972 2973//===----------------------------------------------------------------------===// 2974// SVE2 Integer Multiply-Add - Unpredicated Group 2975//===----------------------------------------------------------------------===// 2976 2977class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, 2978 ZPRRegOp zprty1, ZPRRegOp zprty2> 2979: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 2980 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2981 bits<5> Zda; 2982 bits<5> Zn; 2983 bits<5> Zm; 2984 let Inst{31-24} = 0b01000100; 2985 let Inst{23-22} = sz; 2986 let Inst{21} = 0b0; 2987 let Inst{20-16} = Zm; 2988 let Inst{15} = 0b0; 2989 let Inst{14-10} = opc; 2990 let Inst{9-5} = Zn; 2991 let Inst{4-0} = Zda; 2992 2993 let Constraints = "$Zda = $_Zda"; 2994 let DestructiveInstType = DestructiveOther; 2995 let ElementSize = ElementSizeNone; 2996} 2997 2998multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> { 2999 def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>; 3000 def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>; 3001 def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>; 3002 def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>; 3003 3004 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3005 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3006 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3007 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3008} 3009 3010multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> { 3011 def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>; 3012 def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>; 3013 def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>; 3014 3015 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3016 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3017 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3018} 3019 3020//===----------------------------------------------------------------------===// 3021// SVE2 Integer Multiply-Add - Indexed Group 3022//===----------------------------------------------------------------------===// 3023 3024class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, 3025 ZPRRegOp zprty1, ZPRRegOp zprty2, 3026 ZPRRegOp zprty3, Operand itype> 3027: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 3028 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 3029 bits<5> Zda; 3030 bits<5> Zn; 3031 let Inst{31-24} = 0b01000100; 3032 let Inst{23-22} = sz; 3033 let Inst{21} = 0b1; 3034 let Inst{15-10} = opc; 3035 let Inst{9-5} = Zn; 3036 let Inst{4-0} = Zda; 3037 3038 let Constraints = "$Zda = $_Zda"; 3039 let DestructiveInstType = DestructiveOther; 3040 let ElementSize = ElementSizeNone; 3041} 3042 3043multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm, 3044 SDPatternOperator op> { 3045 def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3046 bits<3> Zm; 3047 bits<3> iop; 3048 let Inst{22} = iop{2}; 3049 let Inst{20-19} = iop{1-0}; 3050 let Inst{18-16} = Zm; 3051 } 3052 def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3053 bits<3> Zm; 3054 bits<2> iop; 3055 let Inst{20-19} = iop; 3056 let Inst{18-16} = Zm; 3057 } 3058 def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3059 bits<4> Zm; 3060 bit iop; 3061 let Inst{20} = iop; 3062 let Inst{19-16} = Zm; 3063 } 3064 3065 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3066 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3067 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3068} 3069 3070//===----------------------------------------------------------------------===// 3071// SVE2 Integer Multiply-Add Long - Indexed Group 3072//===----------------------------------------------------------------------===// 3073 3074multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm, 3075 SDPatternOperator op> { 3076 def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 3077 asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3078 bits<3> Zm; 3079 bits<3> iop; 3080 let Inst{20-19} = iop{2-1}; 3081 let Inst{18-16} = Zm; 3082 let Inst{11} = iop{0}; 3083 } 3084 def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 3085 asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3086 bits<4> Zm; 3087 bits<2> iop; 3088 let Inst{20} = iop{1}; 3089 let Inst{19-16} = Zm; 3090 let Inst{11} = iop{0}; 3091 } 3092 3093 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3094 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3095} 3096 3097//===----------------------------------------------------------------------===// 3098// SVE Integer Dot Product Group 3099//===----------------------------------------------------------------------===// 3100 3101class sve_intx_dot<bit sz, bit U, string asm, ZPRRegOp zprty1, 3102 ZPRRegOp zprty2> 3103: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm, 3104 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3105 bits<5> Zda; 3106 bits<5> Zn; 3107 bits<5> Zm; 3108 let Inst{31-23} = 0b010001001; 3109 let Inst{22} = sz; 3110 let Inst{21} = 0; 3111 let Inst{20-16} = Zm; 3112 let Inst{15-11} = 0; 3113 let Inst{10} = U; 3114 let Inst{9-5} = Zn; 3115 let Inst{4-0} = Zda; 3116 3117 let Constraints = "$Zda = $_Zda"; 3118 let DestructiveInstType = DestructiveOther; 3119} 3120 3121multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> { 3122 def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>; 3123 def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>; 3124 3125 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>; 3126 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>; 3127} 3128 3129//===----------------------------------------------------------------------===// 3130// SVE Integer Dot Product Group - Indexed Group 3131//===----------------------------------------------------------------------===// 3132 3133class sve_intx_dot_by_indexed_elem<bit sz, bit U, string asm, 3134 ZPRRegOp zprty1, ZPRRegOp zprty2, 3135 ZPRRegOp zprty3, Operand itype> 3136: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 3137 asm, "\t$Zda, $Zn, $Zm$iop", 3138 "", []>, Sched<[]> { 3139 bits<5> Zda; 3140 bits<5> Zn; 3141 let Inst{31-23} = 0b010001001; 3142 let Inst{22} = sz; 3143 let Inst{21} = 0b1; 3144 let Inst{15-11} = 0; 3145 let Inst{10} = U; 3146 let Inst{9-5} = Zn; 3147 let Inst{4-0} = Zda; 3148 3149 let Constraints = "$Zda = $_Zda"; 3150 let DestructiveInstType = DestructiveOther; 3151} 3152 3153multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm, 3154 SDPatternOperator op> { 3155 def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> { 3156 bits<2> iop; 3157 bits<3> Zm; 3158 let Inst{20-19} = iop; 3159 let Inst{18-16} = Zm; 3160 } 3161 def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> { 3162 bits<1> iop; 3163 bits<4> Zm; 3164 let Inst{20} = iop; 3165 let Inst{19-16} = Zm; 3166 } 3167 3168 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3169 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3170} 3171 3172//===----------------------------------------------------------------------===// 3173// SVE2 Complex Integer Dot Product Group 3174//===----------------------------------------------------------------------===// 3175 3176class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm, 3177 ZPRRegOp zprty1, ZPRRegOp zprty2> 3178: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm, 3179 complexrotateop:$rot), 3180 asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> { 3181 bits<5> Zda; 3182 bits<5> Zn; 3183 bits<5> Zm; 3184 bits<2> rot; 3185 let Inst{31-24} = 0b01000100; 3186 let Inst{23-22} = sz; 3187 let Inst{21} = 0b0; 3188 let Inst{20-16} = Zm; 3189 let Inst{15-12} = opc; 3190 let Inst{11-10} = rot; 3191 let Inst{9-5} = Zn; 3192 let Inst{4-0} = Zda; 3193 3194 let Constraints = "$Zda = $_Zda"; 3195 let DestructiveInstType = DestructiveOther; 3196 let ElementSize = ElementSizeNone; 3197} 3198 3199multiclass sve2_cintx_dot<string asm, SDPatternOperator op> { 3200 def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>; 3201 def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>; 3202 3203 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3204 (i32 complexrotateop:$imm))), 3205 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>; 3206 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3207 (i32 complexrotateop:$imm))), 3208 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>; 3209} 3210 3211//===----------------------------------------------------------------------===// 3212// SVE2 Complex Multiply-Add Group 3213//===----------------------------------------------------------------------===// 3214 3215multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> { 3216 def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>; 3217 def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>; 3218 def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>; 3219 def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>; 3220 3221 def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>; 3222 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>; 3223 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>; 3224 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>; 3225} 3226 3227//===----------------------------------------------------------------------===// 3228// SVE2 Complex Integer Dot Product - Indexed Group 3229//===----------------------------------------------------------------------===// 3230 3231class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm, 3232 ZPRRegOp zprty1, ZPRRegOp zprty2, 3233 ZPRRegOp zprty3, Operand itype> 3234: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop, 3235 complexrotateop:$rot), 3236 asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> { 3237 bits<5> Zda; 3238 bits<5> Zn; 3239 bits<2> rot; 3240 let Inst{31-24} = 0b01000100; 3241 let Inst{23-22} = sz; 3242 let Inst{21} = 0b1; 3243 let Inst{15-12} = opc; 3244 let Inst{11-10} = rot; 3245 let Inst{9-5} = Zn; 3246 let Inst{4-0} = Zda; 3247 3248 let Constraints = "$Zda = $_Zda"; 3249 let DestructiveInstType = DestructiveOther; 3250 let ElementSize = ElementSizeNone; 3251} 3252 3253multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> { 3254 def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> { 3255 bits<2> iop; 3256 bits<3> Zm; 3257 let Inst{20-19} = iop; 3258 let Inst{18-16} = Zm; 3259 } 3260 def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> { 3261 bit iop; 3262 bits<4> Zm; 3263 let Inst{20} = iop; 3264 let Inst{19-16} = Zm; 3265 } 3266 3267 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3268 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3269 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3270 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3271 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3272 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3273} 3274 3275//===----------------------------------------------------------------------===// 3276// SVE2 Complex Multiply-Add - Indexed Group 3277//===----------------------------------------------------------------------===// 3278 3279multiclass sve2_cmla_by_indexed_elem<bit opc, string asm, 3280 SDPatternOperator op> { 3281 def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> { 3282 bits<2> iop; 3283 bits<3> Zm; 3284 let Inst{20-19} = iop; 3285 let Inst{18-16} = Zm; 3286 } 3287 def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> { 3288 bit iop; 3289 bits<4> Zm; 3290 let Inst{20} = iop; 3291 let Inst{19-16} = Zm; 3292 } 3293 3294 def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3295 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3296 (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3297 3298 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3), 3299 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3300 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3301} 3302 3303//===----------------------------------------------------------------------===// 3304// SVE2 Integer Multiply - Unpredicated Group 3305//===----------------------------------------------------------------------===// 3306 3307class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 3308: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 3309 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3310 bits<5> Zd; 3311 bits<5> Zm; 3312 bits<5> Zn; 3313 let Inst{31-24} = 0b00000100; 3314 let Inst{23-22} = sz; 3315 let Inst{21} = 0b1; 3316 let Inst{20-16} = Zm; 3317 let Inst{15-13} = 0b011; 3318 let Inst{12-10} = opc; 3319 let Inst{9-5} = Zn; 3320 let Inst{4-0} = Zd; 3321} 3322 3323multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op, 3324 SDPatternOperator op_pred = null_frag> { 3325 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3326 def _H : sve2_int_mul<0b01, opc, asm, ZPR16>; 3327 def _S : sve2_int_mul<0b10, opc, asm, ZPR32>; 3328 def _D : sve2_int_mul<0b11, opc, asm, ZPR64>; 3329 3330 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3331 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3332 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3333 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3334 3335 def : SVE_2_Op_Pred_All_Active<nxv16i8, op_pred, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3336 def : SVE_2_Op_Pred_All_Active<nxv8i16, op_pred, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3337 def : SVE_2_Op_Pred_All_Active<nxv4i32, op_pred, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3338 def : SVE_2_Op_Pred_All_Active<nxv2i64, op_pred, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3339} 3340 3341multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> { 3342 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3343 3344 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3345} 3346 3347//===----------------------------------------------------------------------===// 3348// SVE2 Integer Multiply - Indexed Group 3349//===----------------------------------------------------------------------===// 3350 3351class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, 3352 ZPRRegOp zprty1, ZPRRegOp zprty2, 3353 ZPRRegOp zprty3, Operand itype> 3354: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop), 3355 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 3356 bits<5> Zd; 3357 bits<5> Zn; 3358 let Inst{31-24} = 0b01000100; 3359 let Inst{23-22} = sz; 3360 let Inst{21} = 0b1; 3361 let Inst{15-14} = 0b11; 3362 let Inst{13-10} = opc; 3363 let Inst{9-5} = Zn; 3364 let Inst{4-0} = Zd; 3365} 3366 3367multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm, 3368 SDPatternOperator op> { 3369 def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3370 bits<3> Zm; 3371 bits<3> iop; 3372 let Inst{22} = iop{2}; 3373 let Inst{20-19} = iop{1-0}; 3374 let Inst{18-16} = Zm; 3375 } 3376 def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3377 bits<3> Zm; 3378 bits<2> iop; 3379 let Inst{20-19} = iop; 3380 let Inst{18-16} = Zm; 3381 } 3382 def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3383 bits<4> Zm; 3384 bit iop; 3385 let Inst{20} = iop; 3386 let Inst{19-16} = Zm; 3387 } 3388 3389 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3390 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3391 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3392} 3393 3394multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm, 3395 SDPatternOperator op> { 3396 def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm, 3397 ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3398 bits<3> Zm; 3399 bits<3> iop; 3400 let Inst{20-19} = iop{2-1}; 3401 let Inst{18-16} = Zm; 3402 let Inst{11} = iop{0}; 3403 } 3404 def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm, 3405 ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3406 bits<4> Zm; 3407 bits<2> iop; 3408 let Inst{20} = iop{1}; 3409 let Inst{19-16} = Zm; 3410 let Inst{11} = iop{0}; 3411 } 3412 3413 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3414 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3415} 3416 3417//===----------------------------------------------------------------------===// 3418// SVE2 Integer - Predicated Group 3419//===----------------------------------------------------------------------===// 3420 3421class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm, 3422 ZPRRegOp zprty> 3423: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 3424 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 3425 bits<3> Pg; 3426 bits<5> Zm; 3427 bits<5> Zdn; 3428 let Inst{31-24} = 0b01000100; 3429 let Inst{23-22} = sz; 3430 let Inst{21-20} = 0b01; 3431 let Inst{20-16} = opc{5-1}; 3432 let Inst{15-14} = 0b10; 3433 let Inst{13} = opc{0}; 3434 let Inst{12-10} = Pg; 3435 let Inst{9-5} = Zm; 3436 let Inst{4-0} = Zdn; 3437 3438 let Constraints = "$Zdn = $_Zdn"; 3439 let DestructiveInstType = DestructiveOther; 3440 let ElementSize = zprty.ElementSize; 3441} 3442 3443multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op, 3444 string Ps = "", 3445 DestructiveInstTypeEnum flags=DestructiveOther, 3446 string revname="", bit isReverseInstr=0> { 3447 let DestructiveInstType = flags in { 3448 def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>, 3449 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 3450 def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>, 3451 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 3452 def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>, 3453 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 3454 def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>, 3455 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 3456 } 3457 3458 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3459 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3460 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3461 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3462} 3463 3464class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm, 3465 ZPRRegOp zprty1, ZPRRegOp zprty2> 3466: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn), 3467 asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> { 3468 bits<3> Pg; 3469 bits<5> Zn; 3470 bits<5> Zda; 3471 let Inst{31-24} = 0b01000100; 3472 let Inst{23-22} = sz; 3473 let Inst{21-17} = 0b00010; 3474 let Inst{16} = U; 3475 let Inst{15-13} = 0b101; 3476 let Inst{12-10} = Pg; 3477 let Inst{9-5} = Zn; 3478 let Inst{4-0} = Zda; 3479 3480 let Constraints = "$Zda = $_Zda"; 3481 let DestructiveInstType = DestructiveOther; 3482 let ElementSize = zprty1.ElementSize; 3483} 3484 3485multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> { 3486 def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>; 3487 def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>; 3488 def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>; 3489 3490 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3491 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3492 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3493} 3494 3495class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, 3496 string asm, ZPRRegOp zprty> 3497: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3498 asm, "\t$Zd, $Pg/m, $Zn", 3499 "", 3500 []>, Sched<[]> { 3501 bits<3> Pg; 3502 bits<5> Zd; 3503 bits<5> Zn; 3504 let Inst{31-24} = 0b01000100; 3505 let Inst{23-22} = sz; 3506 let Inst{21-20} = 0b00; 3507 let Inst{19} = Q; 3508 let Inst{18} = 0b0; 3509 let Inst{17-16} = opc; 3510 let Inst{15-13} = 0b101; 3511 let Inst{12-10} = Pg; 3512 let Inst{9-5} = Zn; 3513 let Inst{4-0} = Zd; 3514 3515 let Constraints = "$Zd = $_Zd"; 3516 let DestructiveInstType = DestructiveUnaryPassthru; 3517 let ElementSize = zprty.ElementSize; 3518} 3519 3520multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm, 3521 SDPatternOperator op> { 3522 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3523 SVEPseudo2Instr<NAME # _S, 1>; 3524 3525 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3526 3527 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3528 3529 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3530} 3531 3532multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> { 3533 def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>, 3534 SVEPseudo2Instr<NAME # _B, 1>; 3535 def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>, 3536 SVEPseudo2Instr<NAME # _H, 1>; 3537 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3538 SVEPseudo2Instr<NAME # _S, 1>; 3539 def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>, 3540 SVEPseudo2Instr<NAME # _D, 1>; 3541 3542 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3543 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3544 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3545 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3546 3547 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 3548 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 3549 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3550 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 3551 3552 defm : SVE_3_Op_Undef_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 3553 defm : SVE_3_Op_Undef_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 3554 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3555 defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 3556} 3557 3558//===----------------------------------------------------------------------===// 3559// SVE2 Widening Integer Arithmetic Group 3560//===----------------------------------------------------------------------===// 3561 3562class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, 3563 ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3> 3564: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm), 3565 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3566 bits<5> Zd; 3567 bits<5> Zn; 3568 bits<5> Zm; 3569 let Inst{31-24} = 0b01000101; 3570 let Inst{23-22} = sz; 3571 let Inst{21} = 0b0; 3572 let Inst{20-16} = Zm; 3573 let Inst{15} = 0b0; 3574 let Inst{14-10} = opc; 3575 let Inst{9-5} = Zn; 3576 let Inst{4-0} = Zd; 3577} 3578 3579multiclass sve2_wide_int_arith_long<bits<5> opc, string asm, 3580 SDPatternOperator op> { 3581 def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>; 3582 def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>; 3583 def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>; 3584 3585 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3586 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3587 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3588} 3589 3590multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm, 3591 SDPatternOperator op> { 3592 def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>; 3593 def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>; 3594 def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>; 3595 3596 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3597 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3598 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3599} 3600 3601multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm, 3602 SDPatternOperator op> { 3603 def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>; 3604 3605 // To avoid using 128 bit elements in the IR, the pattern below works with 3606 // llvm intrinsics with the _pair suffix, to reflect that 3607 // _Q is implemented as a pair of _D. 3608 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3609} 3610 3611multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> { 3612 def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>; 3613 def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>; 3614 3615 // To avoid using 128 bit elements in the IR, the patterns below work with 3616 // llvm intrinsics with the _pair suffix, to reflect that 3617 // _H is implemented as a pair of _B and _D is implemented as a pair of _S. 3618 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3619 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3620} 3621 3622//===----------------------------------------------------------------------===// 3623// SVE2 Misc Group 3624//===----------------------------------------------------------------------===// 3625 3626class sve2_misc<bits<2> sz, bits<4> opc, string asm, 3627 ZPRRegOp zprty1, ZPRRegOp zprty2> 3628: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3629 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3630 bits<5> Zd; 3631 bits<5> Zn; 3632 bits<5> Zm; 3633 let Inst{31-24} = 0b01000101; 3634 let Inst{23-22} = sz; 3635 let Inst{21} = 0b0; 3636 let Inst{20-16} = Zm; 3637 let Inst{15-14} = 0b10; 3638 let Inst{13-10} = opc; 3639 let Inst{9-5} = Zn; 3640 let Inst{4-0} = Zd; 3641} 3642 3643multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> { 3644 def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; 3645 def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; 3646 def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; 3647 def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; 3648 3649 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3650 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3651 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3652 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3653} 3654 3655multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm, 3656 SDPatternOperator op> { 3657 def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3658 def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3659 def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3660 3661 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3662 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3663 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3664} 3665 3666class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm, 3667 ZPRRegOp zprty1, ZPRRegOp zprty2> 3668: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3669 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3670 bits<5> Zd; 3671 bits<5> Zn; 3672 bits<5> Zm; 3673 let Inst{31-24} = 0b01000101; 3674 let Inst{23-22} = sz; 3675 let Inst{21} = 0b0; 3676 let Inst{20-16} = Zm; 3677 let Inst{15-11} = 0b10010; 3678 let Inst{10} = opc; 3679 let Inst{9-5} = Zn; 3680 let Inst{4-0} = Zd; 3681 3682 let Constraints = "$Zd = $_Zd"; 3683 let DestructiveInstType = DestructiveOther; 3684 let ElementSize = ElementSizeNone; 3685} 3686 3687multiclass sve2_bitwise_xor_interleaved<bit opc, string asm, 3688 SDPatternOperator op> { 3689 def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8, ZPR8>; 3690 def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>; 3691 def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>; 3692 def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>; 3693 3694 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3695 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3696 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3697 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3698} 3699 3700class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm, 3701 ZPRRegOp zprty1, ZPRRegOp zprty2, 3702 Operand immtype> 3703: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3704 asm, "\t$Zd, $Zn, $imm", 3705 "", []>, Sched<[]> { 3706 bits<5> Zd; 3707 bits<5> Zn; 3708 bits<5> imm; 3709 let Inst{31-23} = 0b010001010; 3710 let Inst{22} = tsz8_64{2}; 3711 let Inst{21} = 0b0; 3712 let Inst{20-19} = tsz8_64{1-0}; 3713 let Inst{18-16} = imm{2-0}; // imm3 3714 let Inst{15-12} = 0b1010; 3715 let Inst{11-10} = opc; 3716 let Inst{9-5} = Zn; 3717 let Inst{4-0} = Zd; 3718} 3719 3720multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm, 3721 SDPatternOperator op> { 3722 def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm, 3723 ZPR16, ZPR8, vecshiftL8>; 3724 def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm, 3725 ZPR32, ZPR16, vecshiftL16> { 3726 let Inst{19} = imm{3}; 3727 } 3728 def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm, 3729 ZPR64, ZPR32, vecshiftL32> { 3730 let Inst{20-19} = imm{4-3}; 3731 } 3732 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _H)>; 3733 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _S)>; 3734 def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _D)>; 3735} 3736 3737//===----------------------------------------------------------------------===// 3738// SVE2 Accumulate Group 3739//===----------------------------------------------------------------------===// 3740 3741class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm, 3742 ZPRRegOp zprty, Operand immtype> 3743: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm), 3744 asm, "\t$Zd, $Zn, $imm", 3745 "", []>, Sched<[]> { 3746 bits<5> Zd; 3747 bits<5> Zn; 3748 bits<6> imm; 3749 let Inst{31-24} = 0b01000101; 3750 let Inst{23-22} = tsz8_64{3-2}; 3751 let Inst{21} = 0b0; 3752 let Inst{20-19} = tsz8_64{1-0}; 3753 let Inst{18-16} = imm{2-0}; // imm3 3754 let Inst{15-11} = 0b11110; 3755 let Inst{10} = opc; 3756 let Inst{9-5} = Zn; 3757 let Inst{4-0} = Zd; 3758 3759 let Constraints = "$Zd = $_Zd"; 3760} 3761 3762multiclass sve2_int_bin_shift_imm_left<bit opc, string asm, 3763 SDPatternOperator op> { 3764 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 3765 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 3766 let Inst{19} = imm{3}; 3767 } 3768 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 3769 let Inst{20-19} = imm{4-3}; 3770 } 3771 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 3772 let Inst{22} = imm{5}; 3773 let Inst{20-19} = imm{4-3}; 3774 } 3775 3776 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 3777 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 3778 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 3779 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 3780} 3781 3782multiclass sve2_int_bin_shift_imm_right<bit opc, string asm, 3783 SDPatternOperator op> { 3784 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3785 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3786 let Inst{19} = imm{3}; 3787 } 3788 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3789 let Inst{20-19} = imm{4-3}; 3790 } 3791 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3792 let Inst{22} = imm{5}; 3793 let Inst{20-19} = imm{4-3}; 3794 } 3795 3796 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3797 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3798 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3799 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3800} 3801 3802class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 3803 ZPRRegOp zprty, Operand immtype> 3804: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm), 3805 asm, "\t$Zda, $Zn, $imm", 3806 "", []>, Sched<[]> { 3807 bits<5> Zda; 3808 bits<5> Zn; 3809 bits<6> imm; 3810 let Inst{31-24} = 0b01000101; 3811 let Inst{23-22} = tsz8_64{3-2}; 3812 let Inst{21} = 0b0; 3813 let Inst{20-19} = tsz8_64{1-0}; 3814 let Inst{18-16} = imm{2-0}; // imm3 3815 let Inst{15-12} = 0b1110; 3816 let Inst{11-10} = opc; 3817 let Inst{9-5} = Zn; 3818 let Inst{4-0} = Zda; 3819 3820 let Constraints = "$Zda = $_Zda"; 3821 let DestructiveInstType = DestructiveOther; 3822 let ElementSize = ElementSizeNone; 3823} 3824 3825multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm, 3826 SDPatternOperator op, 3827 SDPatternOperator shift_op = null_frag> { 3828 def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3829 def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3830 let Inst{19} = imm{3}; 3831 } 3832 def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3833 let Inst{20-19} = imm{4-3}; 3834 } 3835 def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3836 let Inst{22} = imm{5}; 3837 let Inst{20-19} = imm{4-3}; 3838 } 3839 3840 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3841 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3842 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3843 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3844 3845 def : SVE_Shift_Add_All_Active_Pat<nxv16i8, shift_op, nxv16i1, nxv16i8, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 3846 def : SVE_Shift_Add_All_Active_Pat<nxv8i16, shift_op, nxv8i1, nxv8i16, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 3847 def : SVE_Shift_Add_All_Active_Pat<nxv4i32, shift_op, nxv4i1, nxv4i32, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 3848 def : SVE_Shift_Add_All_Active_Pat<nxv2i64, shift_op, nxv2i1, nxv2i64, nxv2i64, i32, !cast<Instruction>(NAME # _D)>; 3849} 3850 3851class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty> 3852: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot), 3853 asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> { 3854 bits<5> Zdn; 3855 bits<5> Zm; 3856 bit rot; 3857 let Inst{31-24} = 0b01000101; 3858 let Inst{23-22} = sz; 3859 let Inst{21-17} = 0b00000; 3860 let Inst{16} = opc; 3861 let Inst{15-11} = 0b11011; 3862 let Inst{10} = rot; 3863 let Inst{9-5} = Zm; 3864 let Inst{4-0} = Zdn; 3865 3866 let Constraints = "$Zdn = $_Zdn"; 3867 let DestructiveInstType = DestructiveOther; 3868 let ElementSize = ElementSizeNone; 3869} 3870 3871multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> { 3872 def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>; 3873 def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>; 3874 def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>; 3875 def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>; 3876 3877 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>; 3878 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>; 3879 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>; 3880 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>; 3881} 3882 3883class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm, 3884 ZPRRegOp zprty1, ZPRRegOp zprty2> 3885: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 3886 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3887 bits<5> Zda; 3888 bits<5> Zn; 3889 bits<5> Zm; 3890 let Inst{31-24} = 0b01000101; 3891 let Inst{23-22} = sz; 3892 let Inst{21} = 0b0; 3893 let Inst{20-16} = Zm; 3894 let Inst{15-14} = 0b11; 3895 let Inst{13-10} = opc; 3896 let Inst{9-5} = Zn; 3897 let Inst{4-0} = Zda; 3898 3899 let Constraints = "$Zda = $_Zda"; 3900 let DestructiveInstType = DestructiveOther; 3901 let ElementSize = ElementSizeNone; 3902} 3903 3904multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> { 3905 def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>; 3906 def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>; 3907 def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>; 3908 def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>; 3909 3910 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3911 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3912 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3913 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3914} 3915 3916multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm, 3917 SDPatternOperator op> { 3918 def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3919 def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3920 def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3921 3922 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3923 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3924 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3925} 3926 3927multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm, 3928 SDPatternOperator op> { 3929 def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm, 3930 ZPR32, ZPR32>; 3931 def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm, 3932 ZPR64, ZPR64>; 3933 3934 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3935 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3936} 3937 3938//===----------------------------------------------------------------------===// 3939// SVE2 Narrowing Group 3940//===----------------------------------------------------------------------===// 3941 3942class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc, 3943 string asm, ZPRRegOp zprty1, 3944 ZPRRegOp zprty2, Operand immtype> 3945: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3946 asm, "\t$Zd, $Zn, $imm", 3947 "", []>, Sched<[]> { 3948 bits<5> Zd; 3949 bits<5> Zn; 3950 bits<5> imm; 3951 let Inst{31-23} = 0b010001010; 3952 let Inst{22} = tsz8_64{2}; 3953 let Inst{21} = 0b1; 3954 let Inst{20-19} = tsz8_64{1-0}; 3955 let Inst{18-16} = imm{2-0}; // imm3 3956 let Inst{15-14} = 0b00; 3957 let Inst{13-11} = opc; 3958 let Inst{10} = 0b0; 3959 let Inst{9-5} = Zn; 3960 let Inst{4-0} = Zd; 3961} 3962 3963multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm, 3964 SDPatternOperator op> { 3965 def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16, 3966 tvecshiftR8>; 3967 def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32, 3968 tvecshiftR16> { 3969 let Inst{19} = imm{3}; 3970 } 3971 def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64, 3972 tvecshiftR32> { 3973 let Inst{20-19} = imm{4-3}; 3974 } 3975 def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3976 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3977 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3978} 3979 3980class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc, 3981 string asm, ZPRRegOp zprty1, 3982 ZPRRegOp zprty2, Operand immtype> 3983: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm), 3984 asm, "\t$Zd, $Zn, $imm", 3985 "", []>, Sched<[]> { 3986 bits<5> Zd; 3987 bits<5> Zn; 3988 bits<5> imm; 3989 let Inst{31-23} = 0b010001010; 3990 let Inst{22} = tsz8_64{2}; 3991 let Inst{21} = 0b1; 3992 let Inst{20-19} = tsz8_64{1-0}; 3993 let Inst{18-16} = imm{2-0}; // imm3 3994 let Inst{15-14} = 0b00; 3995 let Inst{13-11} = opc; 3996 let Inst{10} = 0b1; 3997 let Inst{9-5} = Zn; 3998 let Inst{4-0} = Zd; 3999 4000 let Constraints = "$Zd = $_Zd"; 4001} 4002 4003multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm, 4004 SDPatternOperator op> { 4005 def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16, 4006 tvecshiftR8>; 4007 def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32, 4008 tvecshiftR16> { 4009 let Inst{19} = imm{3}; 4010 } 4011 def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64, 4012 tvecshiftR32> { 4013 let Inst{20-19} = imm{4-3}; 4014 } 4015 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4016 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4017 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4018} 4019 4020class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm, 4021 ZPRRegOp zprty1, ZPRRegOp zprty2> 4022: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 4023 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 4024 bits<5> Zd; 4025 bits<5> Zn; 4026 bits<5> Zm; 4027 let Inst{31-24} = 0b01000101; 4028 let Inst{23-22} = sz; 4029 let Inst{21} = 0b1; 4030 let Inst{20-16} = Zm; 4031 let Inst{15-13} = 0b011; 4032 let Inst{12-11} = opc; // S, R 4033 let Inst{10} = 0b0; // Top 4034 let Inst{9-5} = Zn; 4035 let Inst{4-0} = Zd; 4036} 4037 4038multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm, 4039 SDPatternOperator op> { 4040 def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>; 4041 def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>; 4042 def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>; 4043 4044 def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 4045 def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 4046 def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 4047} 4048 4049class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm, 4050 ZPRRegOp zprty1, ZPRRegOp zprty2> 4051: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 4052 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 4053 bits<5> Zd; 4054 bits<5> Zn; 4055 bits<5> Zm; 4056 let Inst{31-24} = 0b01000101; 4057 let Inst{23-22} = sz; 4058 let Inst{21} = 0b1; 4059 let Inst{20-16} = Zm; 4060 let Inst{15-13} = 0b011; 4061 let Inst{12-11} = opc; // S, R 4062 let Inst{10} = 0b1; // Top 4063 let Inst{9-5} = Zn; 4064 let Inst{4-0} = Zd; 4065 4066 let Constraints = "$Zd = $_Zd"; 4067} 4068 4069multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm, 4070 SDPatternOperator op> { 4071 def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>; 4072 def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>; 4073 def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>; 4074 4075 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 4076 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 4077 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 4078} 4079 4080class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm, 4081 ZPRRegOp zprty1, ZPRRegOp zprty2> 4082: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 4083 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 4084 bits<5> Zd; 4085 bits<5> Zn; 4086 let Inst{31-23} = 0b010001010; 4087 let Inst{22} = tsz8_64{2}; 4088 let Inst{21} = 0b1; 4089 let Inst{20-19} = tsz8_64{1-0}; 4090 let Inst{18-13} = 0b000010; 4091 let Inst{12-11} = opc; 4092 let Inst{10} = 0b0; 4093 let Inst{9-5} = Zn; 4094 let Inst{4-0} = Zd; 4095} 4096 4097multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm, 4098 SDPatternOperator op> { 4099 def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>; 4100 def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>; 4101 def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>; 4102 4103 def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>; 4104 def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>; 4105 def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>; 4106} 4107 4108class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm, 4109 ZPRRegOp zprty1, ZPRRegOp zprty2> 4110: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn), 4111 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 4112 bits<5> Zd; 4113 bits<5> Zn; 4114 let Inst{31-23} = 0b010001010; 4115 let Inst{22} = tsz8_64{2}; 4116 let Inst{21} = 0b1; 4117 let Inst{20-19} = tsz8_64{1-0}; 4118 let Inst{18-13} = 0b000010; 4119 let Inst{12-11} = opc; 4120 let Inst{10} = 0b1; 4121 let Inst{9-5} = Zn; 4122 let Inst{4-0} = Zd; 4123 4124 let Constraints = "$Zd = $_Zd"; 4125} 4126 4127multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm, 4128 SDPatternOperator op> { 4129 def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>; 4130 def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>; 4131 def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>; 4132 4133 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>; 4134 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>; 4135 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4136} 4137 4138//===----------------------------------------------------------------------===// 4139// SVE Integer Arithmetic - Unary Predicated Group 4140//===----------------------------------------------------------------------===// 4141 4142class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc, 4143 string asm, ZPRRegOp zprty> 4144: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 4145 asm, "\t$Zd, $Pg/m, $Zn", 4146 "", 4147 []>, Sched<[]> { 4148 bits<3> Pg; 4149 bits<5> Zd; 4150 bits<5> Zn; 4151 let Inst{31-24} = 0b00000100; 4152 let Inst{23-22} = sz8_64; 4153 let Inst{21-20} = 0b01; 4154 let Inst{19} = opc{0}; 4155 let Inst{18-16} = opc{3-1}; 4156 let Inst{15-13} = 0b101; 4157 let Inst{12-10} = Pg; 4158 let Inst{9-5} = Zn; 4159 let Inst{4-0} = Zd; 4160 4161 let Constraints = "$Zd = $_Zd"; 4162 let DestructiveInstType = DestructiveUnaryPassthru; 4163 let ElementSize = zprty.ElementSize; 4164} 4165 4166multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm, 4167 SDPatternOperator op> { 4168 def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>, 4169 SVEPseudo2Instr<NAME # _B, 1>; 4170 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4171 SVEPseudo2Instr<NAME # _H, 1>; 4172 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4173 SVEPseudo2Instr<NAME # _S, 1>; 4174 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4175 SVEPseudo2Instr<NAME # _D, 1>; 4176 4177 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4178 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4179 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4180 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4181 4182 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4183 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4184 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4185 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4186 4187 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4188 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4189 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4190 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4191} 4192 4193multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm, 4194 SDPatternOperator op> { 4195 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4196 SVEPseudo2Instr<NAME # _H, 1>; 4197 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4198 SVEPseudo2Instr<NAME # _S, 1>; 4199 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4200 SVEPseudo2Instr<NAME # _D, 1>; 4201 4202 def : SVE_InReg_Extend<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>; 4203 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>; 4204 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>; 4205 4206 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4207 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4208 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4209 4210 defm : SVE_InReg_Extend_PassthruUndef<nxv8i16, op, nxv8i1, nxv8i8, !cast<Pseudo>(NAME # _UNDEF_H)>; 4211 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i8, !cast<Pseudo>(NAME # _UNDEF_S)>; 4212 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _UNDEF_D)>; 4213} 4214 4215multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm, 4216 SDPatternOperator op> { 4217 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4218 SVEPseudo2Instr<NAME # _S, 1>; 4219 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4220 SVEPseudo2Instr<NAME # _D, 1>; 4221 4222 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>; 4223 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>; 4224 4225 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4226 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4227 4228 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i16, !cast<Pseudo>(NAME # _UNDEF_S)>; 4229 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _UNDEF_D)>; 4230} 4231 4232multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm, 4233 SDPatternOperator op> { 4234 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4235 SVEPseudo2Instr<NAME # _D, 1>; 4236 4237 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>; 4238 4239 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4240 4241 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _UNDEF_D)>; 4242} 4243 4244multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm, 4245 SDPatternOperator op> { 4246 def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>, 4247 SVEPseudo2Instr<NAME # _B, 1>; 4248 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4249 SVEPseudo2Instr<NAME # _H, 1>; 4250 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4251 SVEPseudo2Instr<NAME # _S, 1>; 4252 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4253 SVEPseudo2Instr<NAME # _D, 1>; 4254 4255 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4256 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4257 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4258 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4259 4260 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4261 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4262 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4263 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4264 4265 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4266 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4267 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4268 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4269} 4270 4271multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> { 4272 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4273 SVEPseudo2Instr<NAME # _H, 1>; 4274 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4275 SVEPseudo2Instr<NAME # _S, 1>; 4276 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4277 SVEPseudo2Instr<NAME # _D, 1>; 4278 4279 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4280 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4281 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4282 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4283 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4284 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4285 4286 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4287 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4288 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4289 4290 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4291 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4292 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4293 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4294 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4295 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4296} 4297 4298//===----------------------------------------------------------------------===// 4299// SVE Integer Wide Immediate - Unpredicated Group 4300//===----------------------------------------------------------------------===// 4301class sve_int_dup_imm<bits<2> sz8_64, string asm, 4302 ZPRRegOp zprty, Operand immtype> 4303: I<(outs zprty:$Zd), (ins immtype:$imm), 4304 asm, "\t$Zd, $imm", 4305 "", 4306 []>, Sched<[]> { 4307 bits<5> Zd; 4308 bits<9> imm; 4309 let Inst{31-24} = 0b00100101; 4310 let Inst{23-22} = sz8_64; 4311 let Inst{21-14} = 0b11100011; 4312 let Inst{13} = imm{8}; // sh 4313 let Inst{12-5} = imm{7-0}; // imm8 4314 let Inst{4-0} = Zd; 4315 4316 let isReMaterializable = 1; 4317} 4318 4319multiclass sve_int_dup_imm<string asm> { 4320 def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>; 4321 def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>; 4322 def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>; 4323 def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>; 4324 4325 def : InstAlias<"mov $Zd, $imm", 4326 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>; 4327 def : InstAlias<"mov $Zd, $imm", 4328 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>; 4329 def : InstAlias<"mov $Zd, $imm", 4330 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>; 4331 def : InstAlias<"mov $Zd, $imm", 4332 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>; 4333 4334 def : InstAlias<"fmov $Zd, #0.0", 4335 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>; 4336 def : InstAlias<"fmov $Zd, #0.0", 4337 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>; 4338 def : InstAlias<"fmov $Zd, #0.0", 4339 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>; 4340} 4341 4342class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype, 4343 string asm, ZPRRegOp zprty> 4344: I<(outs zprty:$Zd), (ins fpimmtype:$imm8), 4345 asm, "\t$Zd, $imm8", 4346 "", 4347 []>, Sched<[]> { 4348 bits<5> Zd; 4349 bits<8> imm8; 4350 let Inst{31-24} = 0b00100101; 4351 let Inst{23-22} = sz8_64; 4352 let Inst{21-14} = 0b11100111; 4353 let Inst{13} = 0b0; 4354 let Inst{12-5} = imm8; 4355 let Inst{4-0} = Zd; 4356 4357 let isReMaterializable = 1; 4358} 4359 4360multiclass sve_int_dup_fpimm<string asm> { 4361 def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>; 4362 def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>; 4363 def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>; 4364 4365 def : InstAlias<"fmov $Zd, $imm8", 4366 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>; 4367 def : InstAlias<"fmov $Zd, $imm8", 4368 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>; 4369 def : InstAlias<"fmov $Zd, $imm8", 4370 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>; 4371} 4372 4373class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm, 4374 ZPRRegOp zprty, Operand immtype> 4375: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4376 asm, "\t$Zdn, $_Zdn, $imm", 4377 "", 4378 []>, Sched<[]> { 4379 bits<5> Zdn; 4380 bits<9> imm; 4381 let Inst{31-24} = 0b00100101; 4382 let Inst{23-22} = sz8_64; 4383 let Inst{21-19} = 0b100; 4384 let Inst{18-16} = opc; 4385 let Inst{15-14} = 0b11; 4386 let Inst{13} = imm{8}; // sh 4387 let Inst{12-5} = imm{7-0}; // imm8 4388 let Inst{4-0} = Zdn; 4389 4390 let Constraints = "$Zdn = $_Zdn"; 4391 let DestructiveInstType = DestructiveOther; 4392 let ElementSize = ElementSizeNone; 4393} 4394 4395multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> { 4396 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 4397 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 4398 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 4399 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 4400 4401 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 4402 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 4403 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 4404 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 4405} 4406 4407class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm, 4408 ZPRRegOp zprty, Operand immtype> 4409: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4410 asm, "\t$Zdn, $_Zdn, $imm", 4411 "", 4412 []>, Sched<[]> { 4413 bits<5> Zdn; 4414 bits<8> imm; 4415 let Inst{31-24} = 0b00100101; 4416 let Inst{23-22} = sz8_64; 4417 let Inst{21-16} = opc; 4418 let Inst{15-13} = 0b110; 4419 let Inst{12-5} = imm; 4420 let Inst{4-0} = Zdn; 4421 4422 let Constraints = "$Zdn = $_Zdn"; 4423 let DestructiveInstType = DestructiveOther; 4424 let ElementSize = ElementSizeNone; 4425} 4426 4427multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> { 4428 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8>; 4429 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8>; 4430 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>; 4431 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; 4432 4433 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; 4434 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; 4435 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; 4436 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; 4437} 4438 4439multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> { 4440 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>; 4441 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>; 4442 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>; 4443 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>; 4444 4445 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>; 4446 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>; 4447 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>; 4448 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>; 4449} 4450 4451multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> { 4452 def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8, simm8>; 4453 def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8>; 4454 def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; 4455 def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; 4456 4457 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; 4458 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; 4459 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; 4460 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; 4461} 4462 4463//===----------------------------------------------------------------------===// 4464// SVE Bitwise Logical - Unpredicated Group 4465//===----------------------------------------------------------------------===// 4466 4467class sve_int_bin_cons_log<bits<2> opc, string asm> 4468: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm), 4469 asm, "\t$Zd, $Zn, $Zm", 4470 "", 4471 []>, Sched<[]> { 4472 bits<5> Zd; 4473 bits<5> Zm; 4474 bits<5> Zn; 4475 let Inst{31-24} = 0b00000100; 4476 let Inst{23-22} = opc{1-0}; 4477 let Inst{21} = 0b1; 4478 let Inst{20-16} = Zm; 4479 let Inst{15-10} = 0b001100; 4480 let Inst{9-5} = Zn; 4481 let Inst{4-0} = Zd; 4482} 4483 4484multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> { 4485 def NAME : sve_int_bin_cons_log<opc, asm>; 4486 4487 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4488 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4489 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4490 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4491 4492 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4493 (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>; 4494 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4495 (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>; 4496 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4497 (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>; 4498} 4499 4500class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm> 4501: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk), 4502 asm, "\t$Zdn, $_Zdn, $Zm, $Zk", 4503 "", 4504 []>, Sched<[]> { 4505 bits<5> Zdn; 4506 bits<5> Zk; 4507 bits<5> Zm; 4508 let Inst{31-24} = 0b00000100; 4509 let Inst{23-22} = opc{2-1}; 4510 let Inst{21} = 0b1; 4511 let Inst{20-16} = Zm; 4512 let Inst{15-11} = 0b00111; 4513 let Inst{10} = opc{0}; 4514 let Inst{9-5} = Zk; 4515 let Inst{4-0} = Zdn; 4516 4517 let Constraints = "$Zdn = $_Zdn"; 4518 let DestructiveInstType = DestructiveOther; 4519 let ElementSize = ElementSizeNone; 4520} 4521 4522multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op, 4523 SDPatternOperator ir_op = null_frag> { 4524 def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>; 4525 4526 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4527 (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>; 4528 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4529 (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>; 4530 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4531 (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>; 4532 4533 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4534 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4535 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4536 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4537 4538 4539 def : SVE_3_Op_BSP_Pat<nxv16i8, ir_op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4540 def : SVE_3_Op_BSP_Pat<nxv8i16, ir_op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4541 def : SVE_3_Op_BSP_Pat<nxv4i32, ir_op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4542 def : SVE_3_Op_BSP_Pat<nxv2i64, ir_op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4543} 4544 4545class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm, 4546 ZPRRegOp zprty, Operand immtype> 4547: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm), 4548 asm, "\t$Zdn, $_Zdn, $Zm, $imm", 4549 "", 4550 []>, Sched<[]> { 4551 bits<5> Zdn; 4552 bits<5> Zm; 4553 bits<6> imm; 4554 let Inst{31-24} = 0b00000100; 4555 let Inst{23-22} = tsz8_64{3-2}; 4556 let Inst{21} = 0b1; 4557 let Inst{20-19} = tsz8_64{1-0}; 4558 let Inst{18-16} = imm{2-0}; // imm3 4559 let Inst{15-10} = 0b001101; 4560 let Inst{9-5} = Zm; 4561 let Inst{4-0} = Zdn; 4562 4563 let Constraints = "$Zdn = $_Zdn"; 4564 let DestructiveInstType = DestructiveOther; 4565 let ElementSize = ElementSizeNone; 4566} 4567 4568multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> { 4569 def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>; 4570 def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> { 4571 let Inst{19} = imm{3}; 4572 } 4573 def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> { 4574 let Inst{20-19} = imm{4-3}; 4575 } 4576 def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> { 4577 let Inst{22} = imm{5}; 4578 let Inst{20-19} = imm{4-3}; 4579 } 4580 4581 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4582 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4583 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4584 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4585} 4586 4587//===----------------------------------------------------------------------===// 4588// SVE Integer Wide Immediate - Predicated Group 4589//===----------------------------------------------------------------------===// 4590 4591class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype, 4592 string asm, ZPRRegOp zprty> 4593: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8), 4594 asm, "\t$Zd, $Pg/m, $imm8", 4595 "", 4596 []>, Sched<[]> { 4597 bits<4> Pg; 4598 bits<5> Zd; 4599 bits<8> imm8; 4600 let Inst{31-24} = 0b00000101; 4601 let Inst{23-22} = sz; 4602 let Inst{21-20} = 0b01; 4603 let Inst{19-16} = Pg; 4604 let Inst{15-13} = 0b110; 4605 let Inst{12-5} = imm8; 4606 let Inst{4-0} = Zd; 4607 4608 let Constraints = "$Zd = $_Zd"; 4609 let DestructiveInstType = DestructiveOther; 4610 let ElementSize = zprty.ElementSize; 4611} 4612 4613multiclass sve_int_dup_fpimm_pred<string asm> { 4614 def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>; 4615 def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>; 4616 def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>; 4617 4618 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4619 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>; 4620 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4621 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>; 4622 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4623 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>; 4624} 4625 4626class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm, 4627 ZPRRegOp zprty, string pred_qual, dag iops> 4628: I<(outs zprty:$Zd), iops, 4629 asm, "\t$Zd, $Pg"#pred_qual#", $imm", 4630 "", []>, Sched<[]> { 4631 bits<5> Zd; 4632 bits<4> Pg; 4633 bits<9> imm; 4634 let Inst{31-24} = 0b00000101; 4635 let Inst{23-22} = sz8_64; 4636 let Inst{21-20} = 0b01; 4637 let Inst{19-16} = Pg; 4638 let Inst{15} = 0b0; 4639 let Inst{14} = m; 4640 let Inst{13} = imm{8}; // sh 4641 let Inst{12-5} = imm{7-0}; // imm8 4642 let Inst{4-0} = Zd; 4643 4644 let DestructiveInstType = DestructiveOther; 4645 let ElementSize = zprty.ElementSize; 4646} 4647 4648multiclass sve_int_dup_imm_pred_merge_inst< 4649 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm, 4650 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> { 4651 let Constraints = "$Zd = $_Zd" in 4652 def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty, "/m", 4653 (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>; 4654 def : InstAlias<"mov $Zd, $Pg/m, $imm", 4655 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4656 def : Pat<(vselect predty:$Pg, 4657 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))), 4658 ZPR:$Zd), 4659 (!cast<Instruction>(NAME) $Zd, $Pg, $imm, $shift)>; 4660} 4661 4662multiclass sve_int_dup_imm_pred_merge<string asm> { 4663 defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8, 4664 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>; 4665 defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16, 4666 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>; 4667 defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32, 4668 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>; 4669 defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64, 4670 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>; 4671 4672 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4673 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>; 4674 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4675 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>; 4676 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4677 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>; 4678 4679 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv8f16 ZPR:$Zd)), 4680 (!cast<Instruction>(NAME # _H) $Zd, $Pg, 0, 0)>; 4681 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f16 ZPR:$Zd)), 4682 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>; 4683 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f16 ZPR:$Zd)), 4684 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4685 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f32 ZPR:$Zd)), 4686 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>; 4687 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f32 ZPR:$Zd)), 4688 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4689 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f64 ZPR:$Zd)), 4690 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4691} 4692 4693multiclass sve_int_dup_imm_pred_zero_inst< 4694 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm, 4695 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> { 4696 def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z", 4697 (ins PPRAny:$Pg, cpyimm:$imm)>; 4698 def : InstAlias<"mov $Zd, $Pg/z, $imm", 4699 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4700 def : Pat<(intty (zext (predty PPRAny:$Ps1))), 4701 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4702 def : Pat<(intty (sext (predty PPRAny:$Ps1))), 4703 (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>; 4704 def : Pat<(intty (anyext (predty PPRAny:$Ps1))), 4705 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4706 def : Pat<(vselect predty:$Pg, 4707 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))), 4708 (intty (splat_vector (scalarty 0)))), 4709 (!cast<Instruction>(NAME) $Pg, $imm, $shift)>; 4710} 4711 4712multiclass sve_int_dup_imm_pred_zero<string asm> { 4713 defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8, 4714 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>; 4715 defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16, 4716 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>; 4717 defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32, 4718 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>; 4719 defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64, 4720 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>; 4721} 4722 4723//===----------------------------------------------------------------------===// 4724// SVE Integer Compare - Vectors Group 4725//===----------------------------------------------------------------------===// 4726 4727class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm, 4728 PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2> 4729: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm), 4730 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4731 "", 4732 []>, Sched<[]> { 4733 bits<4> Pd; 4734 bits<3> Pg; 4735 bits<5> Zm; 4736 bits<5> Zn; 4737 let Inst{31-24} = 0b00100100; 4738 let Inst{23-22} = sz8_64; 4739 let Inst{21} = 0b0; 4740 let Inst{20-16} = Zm; 4741 let Inst{15} = opc{2}; 4742 let Inst{14} = cmp_1; 4743 let Inst{13} = opc{1}; 4744 let Inst{12-10} = Pg; 4745 let Inst{9-5} = Zn; 4746 let Inst{4} = opc{0}; 4747 let Inst{3-0} = Pd; 4748 4749 let Defs = [NZCV]; 4750 let ElementSize = pprty.ElementSize; 4751 let isPTestLike = 1; 4752} 4753 4754multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt, 4755 ValueType intvt, Instruction cmp> { 4756 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)), 4757 (cmp $Op1, $Op2, $Op3)>; 4758 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)), 4759 (cmp $Op1, $Op3, $Op2)>; 4760 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op2, intvt:$Op3, cc))), 4761 (cmp $Pg, $Op2, $Op3)>; 4762 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op2, intvt:$Op3, invcc))), 4763 (cmp $Pg, $Op3, $Op2)>; 4764} 4765 4766multiclass SVE_SETCC_Pat_With_Zero<CondCode cc, CondCode invcc, ValueType predvt, 4767 ValueType intvt, Instruction cmp> { 4768 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, (SVEDup0), cc)), 4769 (cmp $Op1, $Op2)>; 4770 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, (SVEDup0), intvt:$Op2, invcc)), 4771 (cmp $Op1, $Op2)>; 4772 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op1, (SVEDup0), cc))), 4773 (cmp $Pg, $Op1)>; 4774 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), (SVEDup0), intvt:$Op1, invcc))), 4775 (cmp $Pg, $Op1)>; 4776} 4777 4778multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> { 4779 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>; 4780 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>; 4781 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>; 4782 def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>; 4783 4784 defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4785 defm : SVE_SETCC_Pat<cc, invcc, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4786 defm : SVE_SETCC_Pat<cc, invcc, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4787 defm : SVE_SETCC_Pat<cc, invcc, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4788} 4789 4790multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> { 4791 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4792 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4793 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4794 4795 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4796 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4797 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4798} 4799 4800multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> { 4801 def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4802 def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4803 def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4804 4805 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4806 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4807 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4808} 4809 4810 4811//===----------------------------------------------------------------------===// 4812// SVE Integer Compare - Signed Immediate Group 4813//===----------------------------------------------------------------------===// 4814 4815class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 4816 ZPRRegOp zprty, 4817 Operand immtype> 4818: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5), 4819 asm, "\t$Pd, $Pg/z, $Zn, $imm5", 4820 "", 4821 []>, Sched<[]> { 4822 bits<4> Pd; 4823 bits<3> Pg; 4824 bits<5> Zn; 4825 bits<5> imm5; 4826 let Inst{31-24} = 0b00100101; 4827 let Inst{23-22} = sz8_64; 4828 let Inst{21} = 0b0; 4829 let Inst{20-16} = imm5; 4830 let Inst{15} = opc{2}; 4831 let Inst{14} = 0b0; 4832 let Inst{13} = opc{1}; 4833 let Inst{12-10} = Pg; 4834 let Inst{9-5} = Zn; 4835 let Inst{4} = opc{0}; 4836 let Inst{3-0} = Pd; 4837 4838 let Defs = [NZCV]; 4839 let ElementSize = pprty.ElementSize; 4840 let isPTestLike = 1; 4841} 4842 4843multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc, 4844 ValueType predvt, ValueType intvt, 4845 Operand immtype, Instruction cmp> { 4846 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4847 (intvt ZPR:$Zs1), 4848 (intvt (splat_vector (immtype:$imm))), 4849 cc)), 4850 (cmp $Pg, $Zs1, immtype:$imm)>; 4851 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4852 (intvt (splat_vector (immtype:$imm))), 4853 (intvt ZPR:$Zs1), 4854 commuted_cc)), 4855 (cmp $Pg, $Zs1, immtype:$imm)>; 4856 def : Pat<(predvt (and predvt:$Pg, 4857 (AArch64setcc_z (predvt (AArch64ptrue 31)), 4858 (intvt ZPR:$Zs1), 4859 (intvt (splat_vector (immtype:$imm))), 4860 cc))), 4861 (cmp $Pg, $Zs1, immtype:$imm)>; 4862 def : Pat<(predvt (and predvt:$Pg, 4863 (AArch64setcc_z (predvt (AArch64ptrue 31)), 4864 (intvt (splat_vector (immtype:$imm))), 4865 (intvt ZPR:$Zs1), 4866 commuted_cc))), 4867 (cmp $Pg, $Zs1, immtype:$imm)>; 4868} 4869 4870multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> { 4871 def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>; 4872 def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>; 4873 def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>; 4874 def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>; 4875 4876 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b, 4877 !cast<Instruction>(NAME # _B)>; 4878 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, simm5_32b, 4879 !cast<Instruction>(NAME # _H)>; 4880 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, simm5_32b, 4881 !cast<Instruction>(NAME # _S)>; 4882 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, simm5_64b, 4883 !cast<Instruction>(NAME # _D)>; 4884} 4885 4886 4887//===----------------------------------------------------------------------===// 4888// SVE Integer Compare - Unsigned Immediate Group 4889//===----------------------------------------------------------------------===// 4890 4891class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty, 4892 ZPRRegOp zprty, Operand immtype> 4893: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7), 4894 asm, "\t$Pd, $Pg/z, $Zn, $imm7", 4895 "", 4896 []>, Sched<[]> { 4897 bits<4> Pd; 4898 bits<3> Pg; 4899 bits<5> Zn; 4900 bits<7> imm7; 4901 let Inst{31-24} = 0b00100100; 4902 let Inst{23-22} = sz8_64; 4903 let Inst{21} = 1; 4904 let Inst{20-14} = imm7; 4905 let Inst{13} = opc{1}; 4906 let Inst{12-10} = Pg; 4907 let Inst{9-5} = Zn; 4908 let Inst{4} = opc{0}; 4909 let Inst{3-0} = Pd; 4910 4911 let Defs = [NZCV]; 4912 let ElementSize = pprty.ElementSize; 4913 let isPTestLike = 1; 4914} 4915 4916multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc, 4917 CondCode commuted_cc> { 4918 def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>; 4919 def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>; 4920 def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>; 4921 def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>; 4922 4923 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127, 4924 !cast<Instruction>(NAME # _B)>; 4925 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, imm0_127, 4926 !cast<Instruction>(NAME # _H)>; 4927 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, imm0_127, 4928 !cast<Instruction>(NAME # _S)>; 4929 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, imm0_127_64b, 4930 !cast<Instruction>(NAME # _D)>; 4931} 4932 4933 4934//===----------------------------------------------------------------------===// 4935// SVE Integer Compare - Scalars Group 4936//===----------------------------------------------------------------------===// 4937 4938class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt> 4939: I<(outs), (ins rt:$Rn, rt:$Rm), 4940 asm, "\t$Rn, $Rm", 4941 "", 4942 []>, Sched<[]> { 4943 bits<5> Rm; 4944 bits<5> Rn; 4945 let Inst{31-23} = 0b001001011; 4946 let Inst{22} = sz; 4947 let Inst{21} = 0b1; 4948 let Inst{20-16} = Rm; 4949 let Inst{15-10} = 0b001000; 4950 let Inst{9-5} = Rn; 4951 let Inst{4} = opc; 4952 let Inst{3-0} = 0b0000; 4953 4954 let Defs = [NZCV]; 4955} 4956 4957class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm, 4958 RegisterClass gprty, PPRRegOp pprty> 4959: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm), 4960 asm, "\t$Pd, $Rn, $Rm", 4961 "", []>, Sched<[]> { 4962 bits<4> Pd; 4963 bits<5> Rm; 4964 bits<5> Rn; 4965 let Inst{31-24} = 0b00100101; 4966 let Inst{23-22} = sz8_64; 4967 let Inst{21} = 0b1; 4968 let Inst{20-16} = Rm; 4969 let Inst{15-13} = 0b000; 4970 let Inst{12-10} = opc{3-1}; 4971 let Inst{9-5} = Rn; 4972 let Inst{4} = opc{0}; 4973 let Inst{3-0} = Pd; 4974 4975 let Defs = [NZCV]; 4976 let ElementSize = pprty.ElementSize; 4977 let isWhile = 1; 4978} 4979 4980multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> { 4981 def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>; 4982 def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>; 4983 def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>; 4984 def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>; 4985 4986 def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>; 4987 def : SVE_2_Op_Pat<nxv8i1, op, i32, i32, !cast<Instruction>(NAME # _H)>; 4988 def : SVE_2_Op_Pat<nxv4i1, op, i32, i32, !cast<Instruction>(NAME # _S)>; 4989 def : SVE_2_Op_Pat<nxv2i1, op, i32, i32, !cast<Instruction>(NAME # _D)>; 4990} 4991 4992multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op> { 4993 def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8>; 4994 def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16>; 4995 def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32>; 4996 def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>; 4997 4998 def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>; 4999 def : SVE_2_Op_Pat<nxv8i1, op, i64, i64, !cast<Instruction>(NAME # _H)>; 5000 def : SVE_2_Op_Pat<nxv4i1, op, i64, i64, !cast<Instruction>(NAME # _S)>; 5001 def : SVE_2_Op_Pat<nxv2i1, op, i64, i64, !cast<Instruction>(NAME # _D)>; 5002} 5003 5004class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, 5005 PPRRegOp pprty> 5006: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), 5007 asm, "\t$Pd, $Rn, $Rm", 5008 "", []>, Sched<[]> { 5009 bits<4> Pd; 5010 bits<5> Rm; 5011 bits<5> Rn; 5012 let Inst{31-24} = 0b00100101; 5013 let Inst{23-22} = sz8_64; 5014 let Inst{21} = 0b1; 5015 let Inst{20-16} = Rm; 5016 let Inst{15-10} = 0b001100; 5017 let Inst{9-5} = Rn; 5018 let Inst{4} = rw; 5019 let Inst{3-0} = Pd; 5020 5021 let Defs = [NZCV]; 5022 let ElementSize = pprty.ElementSize; 5023 let isWhile = 1; 5024} 5025 5026multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> { 5027 def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; 5028 def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; 5029 def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; 5030 def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; 5031 5032 def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>; 5033 def : SVE_2_Op_Pat<nxv8i1, !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>; 5034 def : SVE_2_Op_Pat<nxv4i1, !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>; 5035 def : SVE_2_Op_Pat<nxv2i1, !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>; 5036} 5037 5038//===----------------------------------------------------------------------===// 5039// SVE Floating Point Fast Reduction Group 5040//===----------------------------------------------------------------------===// 5041 5042class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm, 5043 ZPRRegOp zprty, FPRasZPROperand dstOpType> 5044: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 5045 asm, "\t$Vd, $Pg, $Zn", 5046 "", 5047 []>, Sched<[]> { 5048 bits<5> Zn; 5049 bits<5> Vd; 5050 bits<3> Pg; 5051 let Inst{31-24} = 0b01100101; 5052 let Inst{23-22} = sz; 5053 let Inst{21-19} = 0b000; 5054 let Inst{18-16} = opc; 5055 let Inst{15-13} = 0b001; 5056 let Inst{12-10} = Pg; 5057 let Inst{9-5} = Zn; 5058 let Inst{4-0} = Vd; 5059} 5060 5061multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> { 5062 def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>; 5063 def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>; 5064 def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>; 5065 5066 def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5067 def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5068 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5069 def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5070 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5071 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5072} 5073 5074//===----------------------------------------------------------------------===// 5075// SVE Floating Point Accumulating Reduction Group 5076//===----------------------------------------------------------------------===// 5077 5078class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm, 5079 ZPRRegOp zprty, FPRasZPROperand dstOpType> 5080: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm), 5081 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 5082 "", 5083 []>, 5084 Sched<[]> { 5085 bits<3> Pg; 5086 bits<5> Vdn; 5087 bits<5> Zm; 5088 let Inst{31-24} = 0b01100101; 5089 let Inst{23-22} = sz; 5090 let Inst{21-19} = 0b011; 5091 let Inst{18-16} = opc; 5092 let Inst{15-13} = 0b001; 5093 let Inst{12-10} = Pg; 5094 let Inst{9-5} = Zm; 5095 let Inst{4-0} = Vdn; 5096 5097 let Constraints = "$Vdn = $_Vdn"; 5098} 5099 5100multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> { 5101 def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>; 5102 def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>; 5103 def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>; 5104 5105 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>; 5106 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>; 5107 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5108 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>; 5109 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5110 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5111} 5112 5113//===----------------------------------------------------------------------===// 5114// SVE Floating Point Compare - Vectors Group 5115//===----------------------------------------------------------------------===// 5116 5117class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 5118 ZPRRegOp zprty> 5119: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 5120 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 5121 "", 5122 []>, Sched<[]> { 5123 bits<4> Pd; 5124 bits<3> Pg; 5125 bits<5> Zm; 5126 bits<5> Zn; 5127 let Inst{31-24} = 0b01100101; 5128 let Inst{23-22} = sz; 5129 let Inst{21} = 0b0; 5130 let Inst{20-16} = Zm; 5131 let Inst{15} = opc{2}; 5132 let Inst{14} = 0b1; 5133 let Inst{13} = opc{1}; 5134 let Inst{12-10} = Pg; 5135 let Inst{9-5} = Zn; 5136 let Inst{4} = opc{0}; 5137 let Inst{3-0} = Pd; 5138} 5139 5140multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> { 5141 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5142 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5143 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5144 5145 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5146 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5147 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5148} 5149 5150multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, 5151 CondCode cc1, CondCode cc2, 5152 CondCode invcc1, CondCode invcc2> { 5153 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5154 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5155 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5156 5157 defm : SVE_SETCC_Pat<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5158 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5159 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5160 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5161 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5162 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5163 5164 defm : SVE_SETCC_Pat<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5165 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5166 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5167 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5168 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5169 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5170} 5171 5172//===----------------------------------------------------------------------===// 5173// SVE Floating Point Compare - with Zero Group 5174//===----------------------------------------------------------------------===// 5175 5176class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 5177 ZPRRegOp zprty> 5178: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn), 5179 asm, "\t$Pd, $Pg/z, $Zn, #0.0", 5180 "", 5181 []>, Sched<[]> { 5182 bits<4> Pd; 5183 bits<3> Pg; 5184 bits<5> Zn; 5185 let Inst{31-24} = 0b01100101; 5186 let Inst{23-22} = sz; 5187 let Inst{21-18} = 0b0100; 5188 let Inst{17-16} = opc{2-1}; 5189 let Inst{15-13} = 0b001; 5190 let Inst{12-10} = Pg; 5191 let Inst{9-5} = Zn; 5192 let Inst{4} = opc{0}; 5193 let Inst{3-0} = Pd; 5194} 5195 5196multiclass sve_fp_2op_p_pd<bits<3> opc, string asm, 5197 CondCode cc1, CondCode cc2, 5198 CondCode invcc1, CondCode invcc2> { 5199 def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5200 def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5201 def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5202 5203 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5204 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5205 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5206 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5207 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5208 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5209 5210 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5211 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5212 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5213 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5214 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5215 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5216} 5217 5218 5219//===----------------------------------------------------------------------===// 5220//SVE Index Generation Group 5221//===----------------------------------------------------------------------===// 5222 5223def simm5_8b_tgt : TImmLeaf<i8, [{ return (int8_t)Imm >= -16 && (int8_t)Imm < 16; }]>; 5224def simm5_16b_tgt : TImmLeaf<i16, [{ return (int16_t)Imm >= -16 && (int16_t)Imm < 16; }]>; 5225def simm5_32b_tgt : TImmLeaf<i32, [{ return (int32_t)Imm >= -16 && (int32_t)Imm < 16; }]>; 5226def simm5_64b_tgt : TImmLeaf<i64, [{ return (int64_t)Imm >= -16 && (int64_t)Imm < 16; }]>; 5227def i64imm_32bit_tgt : TImmLeaf<i64, [{ 5228 return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm); 5229}]>; 5230 5231class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5232 Operand imm_ty> 5233: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b), 5234 asm, "\t$Zd, $imm5, $imm5b", 5235 "", []>, Sched<[]> { 5236 bits<5> Zd; 5237 bits<5> imm5; 5238 bits<5> imm5b; 5239 let Inst{31-24} = 0b00000100; 5240 let Inst{23-22} = sz8_64; 5241 let Inst{21} = 0b1; 5242 let Inst{20-16} = imm5b; 5243 let Inst{15-10} = 0b010000; 5244 let Inst{9-5} = imm5; 5245 let Inst{4-0} = Zd; 5246 5247 let isReMaterializable = 1; 5248} 5249 5250multiclass sve_int_index_ii<string asm> { 5251 def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>; 5252 def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>; 5253 def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>; 5254 def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>; 5255 5256 def : Pat<(nxv16i8 (step_vector simm5_8b_tgt:$imm5b)), 5257 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5258 def : Pat<(nxv8i16 (step_vector simm5_16b_tgt:$imm5b)), 5259 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5260 def : Pat<(nxv4i32 (step_vector simm5_32b_tgt:$imm5b)), 5261 (!cast<Instruction>(NAME # "_S") (i32 0), simm5_32b:$imm5b)>; 5262 def : Pat<(nxv2i64 (step_vector simm5_64b_tgt:$imm5b)), 5263 (!cast<Instruction>(NAME # "_D") (i64 0), simm5_64b:$imm5b)>; 5264 5265 // add(step_vector(step), dup(X)) -> index(X, step). 5266 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5b)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5267 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5268 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5b)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5269 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5270 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5b)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5271 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>; 5272 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5b)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5273 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>; 5274} 5275 5276class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5277 RegisterClass srcRegType, Operand imm_ty> 5278: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm), 5279 asm, "\t$Zd, $imm5, $Rm", 5280 "", []>, Sched<[]> { 5281 bits<5> Rm; 5282 bits<5> Zd; 5283 bits<5> imm5; 5284 let Inst{31-24} = 0b00000100; 5285 let Inst{23-22} = sz8_64; 5286 let Inst{21} = 0b1; 5287 let Inst{20-16} = Rm; 5288 let Inst{15-10} = 0b010010; 5289 let Inst{9-5} = imm5; 5290 let Inst{4-0} = Zd; 5291} 5292 5293multiclass sve_int_index_ir<string asm, SDPatternOperator mulop, SDPatternOperator muloneuseop> { 5294 def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>; 5295 def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>; 5296 def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>; 5297 def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>; 5298 5299 def : Pat<(nxv16i8 (step_vector i8:$imm)), 5300 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5301 def : Pat<(nxv8i16 (step_vector i16:$imm)), 5302 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5303 def : Pat<(nxv4i32 (step_vector i32:$imm)), 5304 (!cast<Instruction>(NAME # "_S") (i32 0), (!cast<Instruction>("MOVi32imm") $imm))>; 5305 def : Pat<(nxv2i64 (step_vector i64:$imm)), 5306 (!cast<Instruction>(NAME # "_D") (i64 0), (!cast<Instruction>("MOVi64imm") $imm))>; 5307 def : Pat<(nxv2i64 (step_vector i64imm_32bit_tgt:$imm)), 5308 (!cast<Instruction>(NAME # "_D") (i64 0), (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5309 5310 // add(step_vector(step), dup(X)) -> index(X, step). 5311 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5312 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5313 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5314 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5315 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5316 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, (!cast<Instruction>("MOVi32imm") $imm))>; 5317 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5318 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (!cast<Instruction>("MOVi64imm") $imm))>; 5319 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5320 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5321 5322 // mul(step_vector(1), dup(Y)) -> index(0, Y). 5323 def : Pat<(mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), 5324 (!cast<Instruction>(NAME # "_B") (i32 0), GPR32:$Rm)>; 5325 def : Pat<(mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), 5326 (!cast<Instruction>(NAME # "_H") (i32 0), GPR32:$Rm)>; 5327 def : Pat<(mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), 5328 (!cast<Instruction>(NAME # "_S") (i32 0), GPR32:$Rm)>; 5329 def : Pat<(mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), 5330 (!cast<Instruction>(NAME # "_D") (i64 0), GPR64:$Rm)>; 5331 5332 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5333 def : Pat<(add (muloneuseop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5334 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>; 5335 def : Pat<(add (muloneuseop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5336 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>; 5337 def : Pat<(add (muloneuseop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5338 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>; 5339 def : Pat<(add (muloneuseop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5340 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>; 5341} 5342 5343class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5344 RegisterClass srcRegType, Operand imm_ty> 5345: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5), 5346 asm, "\t$Zd, $Rn, $imm5", 5347 "", []>, Sched<[]> { 5348 bits<5> Rn; 5349 bits<5> Zd; 5350 bits<5> imm5; 5351 let Inst{31-24} = 0b00000100; 5352 let Inst{23-22} = sz8_64; 5353 let Inst{21} = 0b1; 5354 let Inst{20-16} = imm5; 5355 let Inst{15-10} = 0b010001; 5356 let Inst{9-5} = Rn; 5357 let Inst{4-0} = Zd; 5358} 5359 5360multiclass sve_int_index_ri<string asm> { 5361 def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>; 5362 def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>; 5363 def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>; 5364 def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>; 5365 5366 // add(step_vector(step), dup(X)) -> index(X, step). 5367 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5)), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), 5368 (!cast<Instruction>(NAME # "_B") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5369 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5)), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), 5370 (!cast<Instruction>(NAME # "_H") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5371 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5)), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), 5372 (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>; 5373 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5)), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), 5374 (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>; 5375} 5376 5377class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5378 RegisterClass srcRegType> 5379: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm), 5380 asm, "\t$Zd, $Rn, $Rm", 5381 "", []>, Sched<[]> { 5382 bits<5> Zd; 5383 bits<5> Rm; 5384 bits<5> Rn; 5385 let Inst{31-24} = 0b00000100; 5386 let Inst{23-22} = sz8_64; 5387 let Inst{21} = 0b1; 5388 let Inst{20-16} = Rm; 5389 let Inst{15-10} = 0b010011; 5390 let Inst{9-5} = Rn; 5391 let Inst{4-0} = Zd; 5392} 5393 5394multiclass sve_int_index_rr<string asm, SDPatternOperator mulop> { 5395 def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>; 5396 def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>; 5397 def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>; 5398 def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>; 5399 5400 // add(step_vector(step), dup(X)) -> index(X, step). 5401 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))), 5402 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5403 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(i32 GPR32:$Rn)))), 5404 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5405 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(i32 GPR32:$Rn)))), 5406 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") $imm))>; 5407 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5408 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (!cast<Instruction>("MOVi64imm") $imm))>; 5409 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5410 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5411 5412 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5413 def : Pat<(add (mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))), 5414 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, GPR32:$Rm)>; 5415 def : Pat<(add (mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))),(nxv8i16 (splat_vector(i32 GPR32:$Rn)))), 5416 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, GPR32:$Rm)>; 5417 def : Pat<(add (mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))),(nxv4i32 (splat_vector(i32 GPR32:$Rn)))), 5418 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, GPR32:$Rm)>; 5419 def : Pat<(add (mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))),(nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5420 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, GPR64:$Rm)>; 5421} 5422 5423//===----------------------------------------------------------------------===// 5424// SVE Bitwise Shift - Predicated Group 5425//===----------------------------------------------------------------------===// 5426 5427class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm, 5428 ZPRRegOp zprty, Operand immtype> 5429: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm), 5430 asm, "\t$Zdn, $Pg/m, $_Zdn, $imm", 5431 "", 5432 []>, Sched<[]> { 5433 bits<3> Pg; 5434 bits<5> Zdn; 5435 bits<6> imm; 5436 let Inst{31-24} = 0b00000100; 5437 let Inst{23-22} = tsz8_64{3-2}; 5438 let Inst{21-20} = 0b00; 5439 let Inst{19-16} = opc; 5440 let Inst{15-13} = 0b100; 5441 let Inst{12-10} = Pg; 5442 let Inst{9-8} = tsz8_64{1-0}; 5443 let Inst{7-5} = imm{2-0}; // imm3 5444 let Inst{4-0} = Zdn; 5445 5446 let Constraints = "$Zdn = $_Zdn"; 5447 let DestructiveInstType = DestructiveBinaryImm; 5448 let ElementSize = zprty.ElementSize; 5449} 5450 5451multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps, 5452 SDPatternOperator op = null_frag> { 5453 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5454 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5455 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5456 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5457 let Inst{8} = imm{3}; 5458 } 5459 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5460 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5461 let Inst{9-8} = imm{4-3}; 5462 } 5463 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5464 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5465 let Inst{22} = imm{5}; 5466 let Inst{9-8} = imm{4-3}; 5467 } 5468 5469 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 5470 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 5471 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 5472 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 5473} 5474 5475// As above but shift amount takes the form of a "vector immediate". 5476multiclass sve_int_bin_pred_shift_imm_left_dup<bits<4> opc, string asm, 5477 string Ps, SDPatternOperator op> 5478: sve_int_bin_pred_shift_imm_left<opc, asm, Ps, null_frag> { 5479 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5480 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5481 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5482 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5483} 5484 5485multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> { 5486 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, tvecshiftL8, FalseLanesZero>; 5487 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>; 5488 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>; 5489 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>; 5490 5491 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8, !cast<Pseudo>(NAME # _ZERO_B)>; 5492 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>; 5493 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>; 5494 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>; 5495} 5496 5497multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps, 5498 SDPatternOperator op = null_frag> { 5499 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5500 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5501 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5502 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5503 let Inst{8} = imm{3}; 5504 } 5505 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5506 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5507 let Inst{9-8} = imm{4-3}; 5508 } 5509 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5510 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5511 let Inst{22} = imm{5}; 5512 let Inst{9-8} = imm{4-3}; 5513 } 5514 5515 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 5516 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 5517 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 5518 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 5519} 5520 5521// As above but shift amount takes the form of a "vector immediate". 5522multiclass sve_int_bin_pred_shift_imm_right_dup<bits<4> opc, string asm, 5523 string Ps, SDPatternOperator op> 5524: sve_int_bin_pred_shift_imm_right<opc, asm, Ps, null_frag> { 5525 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5526 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5527 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5528 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5529} 5530 5531multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> { 5532 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>; 5533 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>; 5534 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>; 5535 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>; 5536 5537 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftR8, !cast<Pseudo>(NAME # _ZERO_B)>; 5538 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftR16, !cast<Pseudo>(NAME # _ZERO_H)>; 5539 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftR32, !cast<Pseudo>(NAME # _ZERO_S)>; 5540 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftR64, !cast<Pseudo>(NAME # _ZERO_D)>; 5541} 5542 5543class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc, 5544 string asm, ZPRRegOp zprty, ZPRRegOp zprty2> 5545: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm), 5546 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 5547 "", 5548 []>, Sched<[]> { 5549 bits<3> Pg; 5550 bits<5> Zdn; 5551 bits<5> Zm; 5552 let Inst{31-24} = 0b00000100; 5553 let Inst{23-22} = sz8_64; 5554 let Inst{21-20} = 0b01; 5555 let Inst{19} = wide; 5556 let Inst{18-16} = opc; 5557 let Inst{15-13} = 0b100; 5558 let Inst{12-10} = Pg; 5559 let Inst{9-5} = Zm; 5560 let Inst{4-0} = Zdn; 5561 5562 let Constraints = "$Zdn = $_Zdn"; 5563 let DestructiveInstType = DestructiveOther; 5564 let ElementSize = zprty.ElementSize; 5565} 5566 5567multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps, 5568 SDPatternOperator op, string revname, bit isReverseInstr = 0> { 5569 let DestructiveInstType = DestructiveBinaryCommWithRev in { 5570 def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>, 5571 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 5572 def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>, 5573 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 5574 def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>, 5575 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 5576 def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>, 5577 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 5578 } 5579 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5580 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5581 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5582 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5583} 5584 5585multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> { 5586 def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>; 5587 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 5588 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 5589 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 5590 5591 def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _ZERO_B)>; 5592 def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _ZERO_H)>; 5593 def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _ZERO_S)>; 5594 def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _ZERO_D)>; 5595} 5596 5597multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm, 5598 SDPatternOperator op> { 5599 def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>; 5600 def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>; 5601 def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>; 5602 5603 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5604 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5605 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5606} 5607 5608//===----------------------------------------------------------------------===// 5609// SVE Shift - Unpredicated Group 5610//===----------------------------------------------------------------------===// 5611 5612class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm, 5613 ZPRRegOp zprty> 5614: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm), 5615 asm, "\t$Zd, $Zn, $Zm", 5616 "", 5617 []>, Sched<[]> { 5618 bits<5> Zd; 5619 bits<5> Zm; 5620 bits<5> Zn; 5621 let Inst{31-24} = 0b00000100; 5622 let Inst{23-22} = sz8_64; 5623 let Inst{21} = 0b1; 5624 let Inst{20-16} = Zm; 5625 let Inst{15-12} = 0b1000; 5626 let Inst{11-10} = opc; 5627 let Inst{9-5} = Zn; 5628 let Inst{4-0} = Zd; 5629} 5630 5631multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm, SDPatternOperator op> { 5632 def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>; 5633 def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>; 5634 def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>; 5635 5636 def : SVE_2_Op_Pred_All_Active<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5637 def : SVE_2_Op_Pred_All_Active<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5638 def : SVE_2_Op_Pred_All_Active<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5639} 5640 5641class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 5642 ZPRRegOp zprty, Operand immtype> 5643: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm), 5644 asm, "\t$Zd, $Zn, $imm", 5645 "", 5646 []>, Sched<[]> { 5647 bits<5> Zd; 5648 bits<5> Zn; 5649 bits<6> imm; 5650 let Inst{31-24} = 0b00000100; 5651 let Inst{23-22} = tsz8_64{3-2}; 5652 let Inst{21} = 0b1; 5653 let Inst{20-19} = tsz8_64{1-0}; 5654 let Inst{18-16} = imm{2-0}; // imm3 5655 let Inst{15-12} = 0b1001; 5656 let Inst{11-10} = opc; 5657 let Inst{9-5} = Zn; 5658 let Inst{4-0} = Zd; 5659} 5660 5661multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm, 5662 SDPatternOperator op> { 5663 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5664 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5665 let Inst{19} = imm{3}; 5666 } 5667 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5668 let Inst{20-19} = imm{4-3}; 5669 } 5670 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5671 let Inst{22} = imm{5}; 5672 let Inst{20-19} = imm{4-3}; 5673 } 5674 5675 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5676 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5677 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5678 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5679} 5680 5681multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm, 5682 SDPatternOperator op> { 5683 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5684 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5685 let Inst{19} = imm{3}; 5686 } 5687 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5688 let Inst{20-19} = imm{4-3}; 5689 } 5690 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5691 let Inst{22} = imm{5}; 5692 let Inst{20-19} = imm{4-3}; 5693 } 5694 5695 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5696 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5697 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5698 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5699} 5700 5701//===----------------------------------------------------------------------===// 5702// SVE Memory - Store Group 5703//===----------------------------------------------------------------------===// 5704 5705class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5706 RegisterOperand VecList> 5707: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5708 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5709 "", 5710 []>, Sched<[]> { 5711 bits<3> Pg; 5712 bits<5> Rn; 5713 bits<5> Zt; 5714 bits<4> imm4; 5715 let Inst{31-25} = 0b1110010; 5716 let Inst{24-23} = msz; 5717 let Inst{22-21} = esz; 5718 let Inst{20} = 0; 5719 let Inst{19-16} = imm4; 5720 let Inst{15-13} = 0b111; 5721 let Inst{12-10} = Pg; 5722 let Inst{9-5} = Rn; 5723 let Inst{4-0} = Zt; 5724 5725 let mayStore = 1; 5726} 5727 5728multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5729 RegisterOperand listty, ZPRRegOp zprty> 5730{ 5731 def NAME : sve_mem_cst_si<msz, esz, asm, listty>; 5732 5733 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5734 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5735 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5736 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5737 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5738 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5739} 5740 5741class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5742 string asm, Operand immtype> 5743: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 5744 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5745 "", 5746 []>, Sched<[]> { 5747 bits<3> Pg; 5748 bits<5> Rn; 5749 bits<5> Zt; 5750 bits<4> imm4; 5751 let Inst{31-25} = 0b1110010; 5752 let Inst{24-23} = sz; 5753 let Inst{22-21} = nregs; 5754 let Inst{20} = 1; 5755 let Inst{19-16} = imm4; 5756 let Inst{15-13} = 0b111; 5757 let Inst{12-10} = Pg; 5758 let Inst{9-5} = Rn; 5759 let Inst{4-0} = Zt; 5760 5761 let mayStore = 1; 5762} 5763 5764multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5765 string asm, Operand immtype> { 5766 def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>; 5767 5768 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5769 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5770} 5771 5772class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5773 string asm, RegisterOperand gprty> 5774: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5775 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5776 "", 5777 []>, Sched<[]> { 5778 bits<3> Pg; 5779 bits<5> Rm; 5780 bits<5> Rn; 5781 bits<5> Zt; 5782 let Inst{31-25} = 0b1110010; 5783 let Inst{24-23} = sz; 5784 let Inst{22-21} = nregs; 5785 let Inst{20-16} = Rm; 5786 let Inst{15-13} = 0b011; 5787 let Inst{12-10} = Pg; 5788 let Inst{9-5} = Rn; 5789 let Inst{4-0} = Zt; 5790 5791 let mayStore = 1; 5792} 5793 5794class sve_mem_cst_ss_base<bits<4> dtype, string asm, 5795 RegisterOperand listty, RegisterOperand gprty> 5796: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5797 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5798 "", 5799 []>, Sched<[]> { 5800 bits<3> Pg; 5801 bits<5> Rm; 5802 bits<5> Rn; 5803 bits<5> Zt; 5804 let Inst{31-25} = 0b1110010; 5805 let Inst{24-21} = dtype; 5806 let Inst{20-16} = Rm; 5807 let Inst{15-13} = 0b010; 5808 let Inst{12-10} = Pg; 5809 let Inst{9-5} = Rn; 5810 let Inst{4-0} = Zt; 5811 5812 let mayStore = 1; 5813} 5814 5815multiclass sve_mem_cst_ss<bits<4> dtype, string asm, 5816 RegisterOperand listty, ZPRRegOp zprty, 5817 RegisterOperand gprty> { 5818 def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>; 5819 5820 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5821 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5822} 5823 5824class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList> 5825: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5826 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5827 "", 5828 []>, Sched<[]> { 5829 bits<3> Pg; 5830 bits<5> Rn; 5831 bits<5> Zt; 5832 bits<4> imm4; 5833 let Inst{31-25} = 0b1110010; 5834 let Inst{24-23} = msz; 5835 let Inst{22-20} = 0b001; 5836 let Inst{19-16} = imm4; 5837 let Inst{15-13} = 0b111; 5838 let Inst{12-10} = Pg; 5839 let Inst{9-5} = Rn; 5840 let Inst{4-0} = Zt; 5841 5842 let mayStore = 1; 5843} 5844 5845multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty, 5846 ZPRRegOp zprty> { 5847 def NAME : sve_mem_cstnt_si<msz, asm, listty>; 5848 5849 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5850 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5851 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5852 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5853 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5854 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5855} 5856 5857class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty, 5858 RegisterOperand gprty> 5859: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5860 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5861 "", 5862 []>, Sched<[]> { 5863 bits<3> Pg; 5864 bits<5> Rm; 5865 bits<5> Rn; 5866 bits<5> Zt; 5867 let Inst{31-25} = 0b1110010; 5868 let Inst{24-23} = msz; 5869 let Inst{22-21} = 0b00; 5870 let Inst{20-16} = Rm; 5871 let Inst{15-13} = 0b011; 5872 let Inst{12-10} = Pg; 5873 let Inst{9-5} = Rn; 5874 let Inst{4-0} = Zt; 5875 5876 let mayStore = 1; 5877} 5878 5879multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty, 5880 ZPRRegOp zprty, RegisterOperand gprty> { 5881 def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>; 5882 5883 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5884 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5885} 5886 5887class sve2_mem_sstnt_vs_base<bits<3> opc, string asm, 5888 RegisterOperand listty, ZPRRegOp zprty> 5889: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 5890 asm, "\t$Zt, $Pg, [$Zn, $Rm]", 5891 "", 5892 []>, Sched<[]> { 5893 bits<3> Pg; 5894 bits<5> Rm; 5895 bits<5> Zn; 5896 bits<5> Zt; 5897 let Inst{31-25} = 0b1110010; 5898 let Inst{24-22} = opc; 5899 let Inst{21} = 0b0; 5900 let Inst{20-16} = Rm; 5901 let Inst{15-13} = 0b001; 5902 let Inst{12-10} = Pg; 5903 let Inst{9-5} = Zn; 5904 let Inst{4-0} = Zt; 5905 5906 let mayStore = 1; 5907} 5908 5909multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm, 5910 SDPatternOperator op, 5911 ValueType vt> { 5912 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>; 5913 5914 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5915 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 5916 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5917 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 5918 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5919 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 5920 5921 def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt), 5922 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; 5923} 5924 5925multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm, 5926 SDPatternOperator op, 5927 ValueType vt> { 5928 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>; 5929 5930 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5931 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 5932 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5933 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 5934 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5935 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 5936 5937 def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt), 5938 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; 5939} 5940 5941class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm, 5942 RegisterOperand VecList, RegisterOperand zprext> 5943: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5944 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5945 "", 5946 []>, Sched<[]> { 5947 bits<3> Pg; 5948 bits<5> Rn; 5949 bits<5> Zm; 5950 bits<5> Zt; 5951 let Inst{31-25} = 0b1110010; 5952 let Inst{24-22} = opc; 5953 let Inst{21} = scaled; 5954 let Inst{20-16} = Zm; 5955 let Inst{15} = 0b1; 5956 let Inst{14} = xs; 5957 let Inst{13} = 0; 5958 let Inst{12-10} = Pg; 5959 let Inst{9-5} = Rn; 5960 let Inst{4-0} = Zt; 5961 5962 let mayStore = 1; 5963} 5964 5965multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm, 5966 SDPatternOperator sxtw_op, 5967 SDPatternOperator uxtw_op, 5968 RegisterOperand sxtw_opnd, 5969 RegisterOperand uxtw_opnd, 5970 ValueType vt > { 5971 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>; 5972 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>; 5973 5974 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5975 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5976 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5977 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5978 5979 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5980 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5981 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5982 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5983} 5984 5985multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm, 5986 SDPatternOperator sxtw_op, 5987 SDPatternOperator uxtw_op, 5988 RegisterOperand sxtw_opnd, 5989 RegisterOperand uxtw_opnd, 5990 ValueType vt > { 5991 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>; 5992 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>; 5993 5994 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5995 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5996 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5997 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5998 5999 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6000 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6001 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6002 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6003} 6004 6005multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm, 6006 SDPatternOperator sxtw_op, 6007 SDPatternOperator uxtw_op, 6008 RegisterOperand sxtw_opnd, 6009 RegisterOperand uxtw_opnd, 6010 ValueType vt> { 6011 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>; 6012 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>; 6013 6014 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6015 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6016 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6017 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6018 6019 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6020 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6021 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6022 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6023} 6024 6025multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm, 6026 SDPatternOperator sxtw_op, 6027 SDPatternOperator uxtw_op, 6028 RegisterOperand sxtw_opnd, 6029 RegisterOperand uxtw_opnd, 6030 ValueType vt> { 6031 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>; 6032 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>; 6033 6034 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6035 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6036 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6037 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6038 6039 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6040 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6041 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6042 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6043} 6044 6045class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm, 6046 RegisterOperand zprext> 6047: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6048 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 6049 "", 6050 []>, Sched<[]> { 6051 bits<3> Pg; 6052 bits<5> Rn; 6053 bits<5> Zm; 6054 bits<5> Zt; 6055 let Inst{31-25} = 0b1110010; 6056 let Inst{24-23} = msz; 6057 let Inst{22} = 0b0; 6058 let Inst{21} = scaled; 6059 let Inst{20-16} = Zm; 6060 let Inst{15-13} = 0b101; 6061 let Inst{12-10} = Pg; 6062 let Inst{9-5} = Rn; 6063 let Inst{4-0} = Zt; 6064 6065 let mayStore = 1; 6066} 6067 6068multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm, 6069 SDPatternOperator op, 6070 RegisterOperand zprext, 6071 ValueType vt> { 6072 def _SCALED : sve_mem_sst_sv2<msz, 1, asm, zprext>; 6073 6074 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6075 (!cast<Instruction>(NAME # _SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 6076 6077 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt), 6078 (!cast<Instruction>(NAME # _SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6079} 6080 6081multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm, 6082 SDPatternOperator op, 6083 ValueType vt> { 6084 def NAME : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>; 6085 6086 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6087 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 6088 6089 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6090 (!cast<Instruction>(NAME) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6091} 6092 6093class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty, 6094 RegisterOperand VecList, Operand imm_ty> 6095: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5), 6096 asm, "\t$Zt, $Pg, [$Zn, $imm5]", 6097 "", 6098 []>, Sched<[]> { 6099 bits<3> Pg; 6100 bits<5> imm5; 6101 bits<5> Zn; 6102 bits<5> Zt; 6103 let Inst{31-25} = 0b1110010; 6104 let Inst{24-23} = opc{2-1}; 6105 let Inst{22} = 0b1; 6106 let Inst{21} = opc{0}; 6107 let Inst{20-16} = imm5; 6108 let Inst{15-13} = 0b101; 6109 let Inst{12-10} = Pg; 6110 let Inst{9-5} = Zn; 6111 let Inst{4-0} = Zt; 6112 6113 let mayStore = 1; 6114} 6115 6116multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm, 6117 Operand imm_ty, 6118 SDPatternOperator op, 6119 ValueType vt> { 6120 def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>; 6121 6122 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6123 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 6124 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 6125 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 6126 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6127 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 6128 6129 def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt), 6130 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6131} 6132 6133multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm, 6134 Operand imm_ty, 6135 SDPatternOperator op, 6136 ValueType vt> { 6137 def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>; 6138 6139 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6140 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 6141 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 6142 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 6143 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6144 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 6145 6146 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt), 6147 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6148} 6149 6150class sve_mem_z_spill<string asm> 6151: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9), 6152 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 6153 "", 6154 []>, Sched<[]> { 6155 bits<5> Rn; 6156 bits<5> Zt; 6157 bits<9> imm9; 6158 let Inst{31-22} = 0b1110010110; 6159 let Inst{21-16} = imm9{8-3}; 6160 let Inst{15-13} = 0b010; 6161 let Inst{12-10} = imm9{2-0}; 6162 let Inst{9-5} = Rn; 6163 let Inst{4-0} = Zt; 6164 6165 let mayStore = 1; 6166} 6167 6168multiclass sve_mem_z_spill<string asm> { 6169 def NAME : sve_mem_z_spill<asm>; 6170 6171 def : InstAlias<asm # "\t$Zt, [$Rn]", 6172 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 6173} 6174 6175class sve_mem_p_spill<string asm> 6176: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9), 6177 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 6178 "", 6179 []>, Sched<[]> { 6180 bits<4> Pt; 6181 bits<5> Rn; 6182 bits<9> imm9; 6183 let Inst{31-22} = 0b1110010110; 6184 let Inst{21-16} = imm9{8-3}; 6185 let Inst{15-13} = 0b000; 6186 let Inst{12-10} = imm9{2-0}; 6187 let Inst{9-5} = Rn; 6188 let Inst{4} = 0b0; 6189 let Inst{3-0} = Pt; 6190 6191 let mayStore = 1; 6192} 6193 6194multiclass sve_mem_p_spill<string asm> { 6195 def NAME : sve_mem_p_spill<asm>; 6196 6197 def : InstAlias<asm # "\t$Pt, [$Rn]", 6198 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 6199} 6200 6201//===----------------------------------------------------------------------===// 6202// SVE Permute - Predicates Group 6203//===----------------------------------------------------------------------===// 6204 6205class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, 6206 PPRRegOp pprty> 6207: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm), 6208 asm, "\t$Pd, $Pn, $Pm", 6209 "", []>, Sched<[]> { 6210 bits<4> Pd; 6211 bits<4> Pm; 6212 bits<4> Pn; 6213 let Inst{31-24} = 0b00000101; 6214 let Inst{23-22} = sz8_64; 6215 let Inst{21-20} = 0b10; 6216 let Inst{19-16} = Pm; 6217 let Inst{15-13} = 0b010; 6218 let Inst{12-10} = opc; 6219 let Inst{9} = 0b0; 6220 let Inst{8-5} = Pn; 6221 let Inst{4} = 0b0; 6222 let Inst{3-0} = Pd; 6223} 6224 6225multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm, 6226 SDPatternOperator op> { 6227 def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>; 6228 def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>; 6229 def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>; 6230 def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>; 6231 6232 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 6233 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 6234 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 6235 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 6236} 6237 6238class sve_int_perm_punpk<bit opc, string asm> 6239: I<(outs PPR16:$Pd), (ins PPR8:$Pn), 6240 asm, "\t$Pd, $Pn", 6241 "", 6242 []>, Sched<[]> { 6243 bits<4> Pd; 6244 bits<4> Pn; 6245 let Inst{31-17} = 0b000001010011000; 6246 let Inst{16} = opc; 6247 let Inst{15-9} = 0b0100000; 6248 let Inst{8-5} = Pn; 6249 let Inst{4} = 0b0; 6250 let Inst{3-0} = Pd; 6251} 6252 6253multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> { 6254 def NAME : sve_int_perm_punpk<opc, asm>; 6255 6256 def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>; 6257 def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1, !cast<Instruction>(NAME)>; 6258 def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1, !cast<Instruction>(NAME)>; 6259} 6260 6261class sve_int_rdffr_pred<bit s, string asm> 6262: I<(outs PPR8:$Pd), (ins PPRAny:$Pg), 6263 asm, "\t$Pd, $Pg/z", 6264 "", 6265 []>, Sched<[]> { 6266 bits<4> Pd; 6267 bits<4> Pg; 6268 let Inst{31-23} = 0b001001010; 6269 let Inst{22} = s; 6270 let Inst{21-9} = 0b0110001111000; 6271 let Inst{8-5} = Pg; 6272 let Inst{4} = 0; 6273 let Inst{3-0} = Pd; 6274 6275 let Defs = !if(s, [NZCV], []); 6276 let Uses = [FFR]; 6277} 6278 6279multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> { 6280 def _REAL : sve_int_rdffr_pred<s, asm>; 6281 6282 // We need a layer of indirection because early machine code passes balk at 6283 // physical register (i.e. FFR) uses that have no previous definition. 6284 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6285 def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, 6286 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; 6287 } 6288} 6289 6290class sve_int_rdffr_unpred<string asm> : I< 6291 (outs PPR8:$Pd), (ins), 6292 asm, "\t$Pd", 6293 "", 6294 []>, Sched<[]> { 6295 bits<4> Pd; 6296 let Inst{31-4} = 0b0010010100011001111100000000; 6297 let Inst{3-0} = Pd; 6298 6299 let Uses = [FFR]; 6300} 6301 6302multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> { 6303 def _REAL : sve_int_rdffr_unpred<asm>; 6304 6305 // We need a layer of indirection because early machine code passes balk at 6306 // physical register (i.e. FFR) uses that have no previous definition. 6307 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6308 def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, 6309 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>; 6310 } 6311} 6312 6313class sve_int_wrffr<string asm, SDPatternOperator op> 6314: I<(outs), (ins PPR8:$Pn), 6315 asm, "\t$Pn", 6316 "", 6317 [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 6318 bits<4> Pn; 6319 let Inst{31-9} = 0b00100101001010001001000; 6320 let Inst{8-5} = Pn; 6321 let Inst{4-0} = 0b00000; 6322 6323 let hasSideEffects = 1; 6324 let Defs = [FFR]; 6325} 6326 6327class sve_int_setffr<string asm, SDPatternOperator op> 6328: I<(outs), (ins), 6329 asm, "", 6330 "", 6331 [(op)]>, Sched<[]> { 6332 let Inst{31-0} = 0b00100101001011001001000000000000; 6333 6334 let hasSideEffects = 1; 6335 let Defs = [FFR]; 6336} 6337 6338//===----------------------------------------------------------------------===// 6339// SVE Permute Vector - Predicated Group 6340//===----------------------------------------------------------------------===// 6341 6342class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm, 6343 ZPRRegOp zprty, RegisterClass rt> 6344: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm), 6345 asm, "\t$Rdn, $Pg, $_Rdn, $Zm", 6346 "", 6347 []>, Sched<[]> { 6348 bits<3> Pg; 6349 bits<5> Rdn; 6350 bits<5> Zm; 6351 let Inst{31-24} = 0b00000101; 6352 let Inst{23-22} = sz8_64; 6353 let Inst{21-17} = 0b11000; 6354 let Inst{16} = ab; 6355 let Inst{15-13} = 0b101; 6356 let Inst{12-10} = Pg; 6357 let Inst{9-5} = Zm; 6358 let Inst{4-0} = Rdn; 6359 6360 let Constraints = "$Rdn = $_Rdn"; 6361} 6362 6363multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> { 6364 def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>; 6365 def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>; 6366 def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>; 6367 def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>; 6368 6369 def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>; 6370 def : SVE_3_Op_Pat<i32, op, nxv8i1, i32, nxv8i16, !cast<Instruction>(NAME # _H)>; 6371 def : SVE_3_Op_Pat<i32, op, nxv4i1, i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6372 def : SVE_3_Op_Pat<i64, op, nxv2i1, i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6373} 6374 6375class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm, 6376 ZPRRegOp zprty, RegisterClass rt> 6377: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm), 6378 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 6379 "", 6380 []>, Sched<[]> { 6381 bits<3> Pg; 6382 bits<5> Vdn; 6383 bits<5> Zm; 6384 let Inst{31-24} = 0b00000101; 6385 let Inst{23-22} = sz8_64; 6386 let Inst{21-17} = 0b10101; 6387 let Inst{16} = ab; 6388 let Inst{15-13} = 0b100; 6389 let Inst{12-10} = Pg; 6390 let Inst{9-5} = Zm; 6391 let Inst{4-0} = Vdn; 6392 6393 let Constraints = "$Vdn = $_Vdn"; 6394} 6395 6396multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> { 6397 def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>; 6398 def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>; 6399 def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>; 6400 def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>; 6401 6402 def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6403 def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6404 def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6405 6406 def : SVE_3_Op_Pat<bf16, op, nxv8i1, bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6407} 6408 6409class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm, 6410 ZPRRegOp zprty> 6411: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6412 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6413 "", 6414 []>, Sched<[]> { 6415 bits<3> Pg; 6416 bits<5> Zdn; 6417 bits<5> Zm; 6418 let Inst{31-24} = 0b00000101; 6419 let Inst{23-22} = sz8_64; 6420 let Inst{21-17} = 0b10100; 6421 let Inst{16} = ab; 6422 let Inst{15-13} = 0b100; 6423 let Inst{12-10} = Pg; 6424 let Inst{9-5} = Zm; 6425 let Inst{4-0} = Zdn; 6426 6427 let Constraints = "$Zdn = $_Zdn"; 6428 let DestructiveInstType = DestructiveOther; 6429 let ElementSize = ElementSizeNone; 6430} 6431 6432multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> { 6433 def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>; 6434 def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>; 6435 def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>; 6436 def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>; 6437 6438 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6439 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6440 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6441 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6442 6443 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6444 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6445 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6446 6447 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6448} 6449 6450class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm, 6451 ZPRRegOp zprty, RegisterClass resultRegType> 6452: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn), 6453 asm, "\t$Rd, $Pg, $Zn", 6454 "", 6455 []>, Sched<[]> { 6456 bits<3> Pg; 6457 bits<5> Rd; 6458 bits<5> Zn; 6459 let Inst{31-24} = 0b00000101; 6460 let Inst{23-22} = sz8_64; 6461 let Inst{21-17} = 0b10000; 6462 let Inst{16} = ab; 6463 let Inst{15-13} = 0b101; 6464 let Inst{12-10} = Pg; 6465 let Inst{9-5} = Zn; 6466 let Inst{4-0} = Rd; 6467} 6468 6469multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> { 6470 def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>; 6471 def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>; 6472 def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>; 6473 def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>; 6474 6475 def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6476 def : SVE_2_Op_Pat<i32, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6477 def : SVE_2_Op_Pat<i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6478 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6479} 6480 6481class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm, 6482 ZPRRegOp zprty, RegisterClass dstRegtype> 6483: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 6484 asm, "\t$Vd, $Pg, $Zn", 6485 "", 6486 []>, Sched<[]> { 6487 bits<3> Pg; 6488 bits<5> Vd; 6489 bits<5> Zn; 6490 let Inst{31-24} = 0b00000101; 6491 let Inst{23-22} = sz8_64; 6492 let Inst{21-17} = 0b10001; 6493 let Inst{16} = ab; 6494 let Inst{15-13} = 0b100; 6495 let Inst{12-10} = Pg; 6496 let Inst{9-5} = Zn; 6497 let Inst{4-0} = Vd; 6498} 6499 6500multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> { 6501 def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>; 6502 def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>; 6503 def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>; 6504 def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>; 6505 6506 def : SVE_2_Op_Pat<f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 6507 def : SVE_2_Op_Pat<f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6508 def : SVE_2_Op_Pat<f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 6509 def : SVE_2_Op_Pat<f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6510 6511 def : SVE_2_Op_Pat<bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6512} 6513 6514class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty> 6515: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6516 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6517 "", 6518 []>, Sched<[]> { 6519 bits<3> Pg; 6520 bits<5> Zdn; 6521 bits<5> Zm; 6522 let Inst{31-24} = 0b00000101; 6523 let Inst{23-22} = sz8_64; 6524 let Inst{21-13} = 0b101100100; 6525 let Inst{12-10} = Pg; 6526 let Inst{9-5} = Zm; 6527 let Inst{4-0} = Zdn; 6528 6529 let Constraints = "$Zdn = $_Zdn"; 6530 let DestructiveInstType = DestructiveOther; 6531 let ElementSize = ElementSizeNone; 6532} 6533 6534multiclass sve_int_perm_splice<string asm, SDPatternOperator op> { 6535 def _B : sve_int_perm_splice<0b00, asm, ZPR8>; 6536 def _H : sve_int_perm_splice<0b01, asm, ZPR16>; 6537 def _S : sve_int_perm_splice<0b10, asm, ZPR32>; 6538 def _D : sve_int_perm_splice<0b11, asm, ZPR64>; 6539 6540 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6541 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6542 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6543 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6544 6545 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6546 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6547 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6548 6549 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6550} 6551 6552class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm, 6553 ZPRRegOp zprty, RegisterOperand VecList> 6554: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn), 6555 asm, "\t$Zd, $Pg, $Zn", 6556 "", 6557 []>, Sched<[]> { 6558 bits<3> Pg; 6559 bits<5> Zn; 6560 bits<5> Zd; 6561 let Inst{31-24} = 0b00000101; 6562 let Inst{23-22} = sz8_64; 6563 let Inst{21-13} = 0b101101100; 6564 let Inst{12-10} = Pg; 6565 let Inst{9-5} = Zn; 6566 let Inst{4-0} = Zd; 6567} 6568 6569multiclass sve2_int_perm_splice_cons<string asm> { 6570 def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>; 6571 def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>; 6572 def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>; 6573 def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>; 6574} 6575 6576class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm, 6577 ZPRRegOp zprty> 6578: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 6579 asm, "\t$Zd, $Pg/m, $Zn", 6580 "", 6581 []>, Sched<[]> { 6582 bits<5> Zd; 6583 bits<3> Pg; 6584 bits<5> Zn; 6585 let Inst{31-24} = 0b00000101; 6586 let Inst{23-22} = sz8_64; 6587 let Inst{21-18} = 0b1001; 6588 let Inst{17-16} = opc; 6589 let Inst{15-13} = 0b100; 6590 let Inst{12-10} = Pg; 6591 let Inst{9-5} = Zn; 6592 let Inst{4-0} = Zd; 6593 6594 let Constraints = "$Zd = $_Zd"; 6595 let DestructiveInstType = DestructiveOther; 6596 let ElementSize = zprty.ElementSize; 6597} 6598 6599multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> { 6600 def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>; 6601 def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>; 6602 def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>; 6603 def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>; 6604 6605 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6606 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6607 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6608 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6609} 6610 6611multiclass sve_int_perm_rev_revb<string asm, SDPatternOperator op> { 6612 def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>; 6613 def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>; 6614 def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>; 6615 6616 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6617 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6618 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6619} 6620 6621multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> { 6622 def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>; 6623 def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>; 6624 6625 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6626 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6627} 6628 6629multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> { 6630 def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>; 6631 6632 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6633} 6634 6635class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6636 RegisterClass srcRegType> 6637: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn), 6638 asm, "\t$Zd, $Pg/m, $Rn", 6639 "", 6640 []>, Sched<[]> { 6641 bits<3> Pg; 6642 bits<5> Rn; 6643 bits<5> Zd; 6644 let Inst{31-24} = 0b00000101; 6645 let Inst{23-22} = sz8_64; 6646 let Inst{21-13} = 0b101000101; 6647 let Inst{12-10} = Pg; 6648 let Inst{9-5} = Rn; 6649 let Inst{4-0} = Zd; 6650 6651 let Constraints = "$Zd = $_Zd"; 6652 let DestructiveInstType = DestructiveOther; 6653 let ElementSize = zprty.ElementSize; 6654} 6655 6656multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> { 6657 def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>; 6658 def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>; 6659 def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>; 6660 def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>; 6661 6662 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6663 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6664 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6665 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6666 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6667 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6668 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6669 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>; 6670 6671 def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)), 6672 (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>; 6673 def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)), 6674 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6675 def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)), 6676 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6677 def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)), 6678 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6679} 6680 6681class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6682 RegisterClass srcRegtype> 6683: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn), 6684 asm, "\t$Zd, $Pg/m, $Vn", 6685 "", 6686 []>, Sched<[]> { 6687 bits<3> Pg; 6688 bits<5> Vn; 6689 bits<5> Zd; 6690 let Inst{31-24} = 0b00000101; 6691 let Inst{23-22} = sz8_64; 6692 let Inst{21-13} = 0b100000100; 6693 let Inst{12-10} = Pg; 6694 let Inst{9-5} = Vn; 6695 let Inst{4-0} = Zd; 6696 6697 let Constraints = "$Zd = $_Zd"; 6698 let DestructiveInstType = DestructiveOther; 6699 let ElementSize = zprty.ElementSize; 6700} 6701 6702multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> { 6703 def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>; 6704 def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>; 6705 def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>; 6706 def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>; 6707 6708 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6709 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>; 6710 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6711 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>; 6712 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6713 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>; 6714 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6715 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>; 6716 6717 def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)), 6718 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6719 def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)), 6720 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6721 def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)), 6722 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6723 def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)), 6724 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6725 6726 def : Pat<(nxv8bf16 (op nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)), 6727 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6728} 6729 6730class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty> 6731: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn), 6732 asm, "\t$Zd, $Pg, $Zn", 6733 "", 6734 []>, Sched<[]> { 6735 bits<3> Pg; 6736 bits<5> Zd; 6737 bits<5> Zn; 6738 let Inst{31-23} = 0b000001011; 6739 let Inst{22} = sz; 6740 let Inst{21-13} = 0b100001100; 6741 let Inst{12-10} = Pg; 6742 let Inst{9-5} = Zn; 6743 let Inst{4-0} = Zd; 6744} 6745 6746multiclass sve_int_perm_compact<string asm, SDPatternOperator op> { 6747 def _S : sve_int_perm_compact<0b0, asm, ZPR32>; 6748 def _D : sve_int_perm_compact<0b1, asm, ZPR64>; 6749 6750 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6751 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6752 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6753 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6754} 6755 6756//===----------------------------------------------------------------------===// 6757// SVE Memory - Contiguous Load Group 6758//===----------------------------------------------------------------------===// 6759 6760class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6761 RegisterOperand VecList> 6762: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6763 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6764 "", 6765 []>, Sched<[]> { 6766 bits<3> Pg; 6767 bits<5> Rn; 6768 bits<5> Zt; 6769 bits<4> imm4; 6770 let Inst{31-25} = 0b1010010; 6771 let Inst{24-21} = dtype; 6772 let Inst{20} = nf; 6773 let Inst{19-16} = imm4; 6774 let Inst{15-13} = 0b101; 6775 let Inst{12-10} = Pg; 6776 let Inst{9-5} = Rn; 6777 let Inst{4-0} = Zt; 6778 6779 let mayLoad = 1; 6780 let Uses = !if(nf, [FFR], []); 6781 let Defs = !if(nf, [FFR], []); 6782} 6783 6784multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6785 RegisterOperand listty, ZPRRegOp zprty> { 6786 def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>; 6787 6788 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6789 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6790 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6791 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6792 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6793 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6794 6795 // We need a layer of indirection because early machine code passes balk at 6796 // physical register (i.e. FFR) uses that have no previous definition. 6797 let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in { 6798 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>, 6799 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>; 6800 } 6801} 6802 6803multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty, 6804 ZPRRegOp zprty> 6805: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>; 6806 6807class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList> 6808: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6809 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6810 "", 6811 []>, Sched<[]> { 6812 bits<5> Zt; 6813 bits<3> Pg; 6814 bits<5> Rn; 6815 bits<4> imm4; 6816 let Inst{31-25} = 0b1010010; 6817 let Inst{24-23} = msz; 6818 let Inst{22-20} = 0b000; 6819 let Inst{19-16} = imm4; 6820 let Inst{15-13} = 0b111; 6821 let Inst{12-10} = Pg; 6822 let Inst{9-5} = Rn; 6823 let Inst{4-0} = Zt; 6824 6825 let mayLoad = 1; 6826} 6827 6828multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty, 6829 ZPRRegOp zprty> { 6830 def NAME : sve_mem_cldnt_si_base<msz, asm, listty>; 6831 6832 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6833 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6834 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6835 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6836 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6837 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6838} 6839 6840class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList, 6841 RegisterOperand gprty> 6842: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6843 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6844 "", 6845 []>, Sched<[]> { 6846 bits<3> Pg; 6847 bits<5> Rm; 6848 bits<5> Rn; 6849 bits<5> Zt; 6850 let Inst{31-25} = 0b1010010; 6851 let Inst{24-23} = msz; 6852 let Inst{22-21} = 0b00; 6853 let Inst{20-16} = Rm; 6854 let Inst{15-13} = 0b110; 6855 let Inst{12-10} = Pg; 6856 let Inst{9-5} = Rn; 6857 let Inst{4-0} = Zt; 6858 6859 let mayLoad = 1; 6860} 6861 6862multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty, 6863 ZPRRegOp zprty, RegisterOperand gprty> { 6864 def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>; 6865 6866 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6867 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6868} 6869 6870class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList> 6871: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 6872 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 6873 bits<5> Zt; 6874 bits<5> Rn; 6875 bits<3> Pg; 6876 bits<4> imm4; 6877 let Inst{31-25} = 0b1010010; 6878 let Inst{24-23} = sz; 6879 let Inst{22-20} = 0; 6880 let Inst{19-16} = imm4; 6881 let Inst{15-13} = 0b001; 6882 let Inst{12-10} = Pg; 6883 let Inst{9-5} = Rn; 6884 let Inst{4-0} = Zt; 6885 6886 let mayLoad = 1; 6887} 6888 6889multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty, 6890 ZPRRegOp zprty> { 6891 def NAME : sve_mem_ldqr_si<sz, asm, listty>; 6892 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6893 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6894 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6895 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6896 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 6897 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>; 6898} 6899 6900class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList, 6901 RegisterOperand gprty> 6902: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6903 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 6904 bits<5> Zt; 6905 bits<3> Pg; 6906 bits<5> Rn; 6907 bits<5> Rm; 6908 let Inst{31-25} = 0b1010010; 6909 let Inst{24-23} = sz; 6910 let Inst{22-21} = 0; 6911 let Inst{20-16} = Rm; 6912 let Inst{15-13} = 0; 6913 let Inst{12-10} = Pg; 6914 let Inst{9-5} = Rn; 6915 let Inst{4-0} = Zt; 6916 6917 let mayLoad = 1; 6918} 6919 6920multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty, 6921 ZPRRegOp zprty, RegisterOperand gprty> { 6922 def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>; 6923 6924 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6925 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6926} 6927 6928class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6929 RegisterOperand VecList, Operand immtype> 6930: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 6931 asm, "\t$Zt, $Pg/z, [$Rn, $imm6]", 6932 "", 6933 []>, Sched<[]> { 6934 bits<3> Pg; 6935 bits<5> Rn; 6936 bits<5> Zt; 6937 bits<6> imm6; 6938 let Inst{31-25} = 0b1000010; 6939 let Inst{24-23} = dtypeh; 6940 let Inst{22} = 1; 6941 let Inst{21-16} = imm6; 6942 let Inst{15} = 0b1; 6943 let Inst{14-13} = dtypel; 6944 let Inst{12-10} = Pg; 6945 let Inst{9-5} = Rn; 6946 let Inst{4-0} = Zt; 6947 6948 let mayLoad = 1; 6949} 6950 6951multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6952 RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> { 6953 def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>; 6954 6955 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6956 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6957 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]", 6958 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>; 6959 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6960 (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6961} 6962 6963class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm, 6964 RegisterOperand VecList> 6965: I<(outs VecList:$Zt), iops, 6966 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6967 "", 6968 []>, Sched<[]> { 6969 bits<5> Zt; 6970 bits<3> Pg; 6971 bits<5> Rm; 6972 bits<5> Rn; 6973 let Inst{31-25} = 0b1010010; 6974 let Inst{24-21} = dtype; 6975 let Inst{20-16} = Rm; 6976 let Inst{15-14} = 0b01; 6977 let Inst{13} = ff; 6978 let Inst{12-10} = Pg; 6979 let Inst{9-5} = Rn; 6980 let Inst{4-0} = Zt; 6981 6982 let mayLoad = 1; 6983 let Uses = !if(ff, [FFR], []); 6984 let Defs = !if(ff, [FFR], []); 6985} 6986 6987multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty, 6988 ZPRRegOp zprty, RegisterOperand gprty> { 6989 def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6990 asm, listty>; 6991 6992 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6993 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6994} 6995 6996multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty, 6997 ZPRRegOp zprty, RegisterOperand gprty> { 6998 def _REAL : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6999 asm, listty>; 7000 7001 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 7002 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 7003 7004 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7005 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 7006 7007 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7008 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 7009 7010 // We need a layer of indirection because early machine code passes balk at 7011 // physical register (i.e. FFR) uses that have no previous definition. 7012 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7013 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>, 7014 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>; 7015 } 7016} 7017 7018multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty, 7019 ZPRRegOp zprty> 7020: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>; 7021 7022class sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 7023 string asm, Operand immtype> 7024: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 7025 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 7026 "", 7027 []>, Sched<[]> { 7028 bits<5> Zt; 7029 bits<3> Pg; 7030 bits<5> Rn; 7031 bits<4> imm4; 7032 let Inst{31-25} = 0b1010010; 7033 let Inst{24-23} = sz; 7034 let Inst{22-21} = nregs; 7035 let Inst{20} = 0; 7036 let Inst{19-16} = imm4; 7037 let Inst{15-13} = 0b111; 7038 let Inst{12-10} = Pg; 7039 let Inst{9-5} = Rn; 7040 let Inst{4-0} = Zt; 7041 7042 let mayLoad = 1; 7043} 7044 7045multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 7046 string asm, Operand immtype> { 7047 def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>; 7048 7049 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7050 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7051} 7052 7053class sve_mem_eld_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 7054 string asm, RegisterOperand gprty> 7055: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7056 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 7057 "", 7058 []>, Sched<[]> { 7059 bits<3> Pg; 7060 bits<5> Rm; 7061 bits<5> Rn; 7062 bits<5> Zt; 7063 let Inst{31-25} = 0b1010010; 7064 let Inst{24-23} = sz; 7065 let Inst{22-21} = nregs; 7066 let Inst{20-16} = Rm; 7067 let Inst{15-13} = 0b110; 7068 let Inst{12-10} = Pg; 7069 let Inst{9-5} = Rn; 7070 let Inst{4-0} = Zt; 7071 7072 let mayLoad = 1; 7073} 7074 7075//===----------------------------------------------------------------------===// 7076// SVE Memory - 32-bit Gather and Unsized Contiguous Group 7077//===----------------------------------------------------------------------===// 7078 7079// bit xs is '1' if offsets are signed 7080// bit scaled is '1' if the offsets are scaled 7081class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm, 7082 RegisterOperand zprext> 7083: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7084 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7085 "", 7086 []>, Sched<[]> { 7087 bits<3> Pg; 7088 bits<5> Rn; 7089 bits<5> Zm; 7090 bits<5> Zt; 7091 let Inst{31-25} = 0b1000010; 7092 let Inst{24-23} = opc{3-2}; 7093 let Inst{22} = xs; 7094 let Inst{21} = scaled; 7095 let Inst{20-16} = Zm; 7096 let Inst{15} = 0b0; 7097 let Inst{14-13} = opc{1-0}; 7098 let Inst{12-10} = Pg; 7099 let Inst{9-5} = Rn; 7100 let Inst{4-0} = Zt; 7101 7102 let mayLoad = 1; 7103 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7104 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7105} 7106 7107multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm, 7108 SDPatternOperator sxtw_op, 7109 SDPatternOperator uxtw_op, 7110 RegisterOperand sxtw_opnd, 7111 RegisterOperand uxtw_opnd, 7112 ValueType vt> { 7113 def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>; 7114 def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>; 7115 7116 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7117 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7118 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7119 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7120 7121 // We need a layer of indirection because early machine code passes balk at 7122 // physical register (i.e. FFR) uses that have no previous definition. 7123 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7124 def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7125 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7126 def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7127 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7128 } 7129 7130 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 7131 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7132 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 7133 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7134} 7135 7136multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm, 7137 SDPatternOperator sxtw_op, 7138 SDPatternOperator uxtw_op, 7139 RegisterOperand sxtw_opnd, 7140 RegisterOperand uxtw_opnd, 7141 ValueType vt> { 7142 def _UXTW_REAL : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>; 7143 def _SXTW_REAL : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>; 7144 7145 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7146 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7147 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7148 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7149 7150 // We need a layer of indirection because early machine code passes balk at 7151 // physical register (i.e. FFR) uses that have no previous definition. 7152 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7153 def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7154 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7155 def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7156 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7157 } 7158 7159 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7160 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7161 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7162 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7163} 7164 7165 7166class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7167: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7168 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7169 "", 7170 []>, Sched<[]> { 7171 bits<3> Pg; 7172 bits<5> Zn; 7173 bits<5> Zt; 7174 bits<5> imm5; 7175 let Inst{31-25} = 0b1000010; 7176 let Inst{24-23} = opc{3-2}; 7177 let Inst{22-21} = 0b01; 7178 let Inst{20-16} = imm5; 7179 let Inst{15} = 0b1; 7180 let Inst{14-13} = opc{1-0}; 7181 let Inst{12-10} = Pg; 7182 let Inst{9-5} = Zn; 7183 let Inst{4-0} = Zt; 7184 7185 let mayLoad = 1; 7186 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7187 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7188} 7189 7190multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty, 7191 SDPatternOperator op, ValueType vt> { 7192 def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>; 7193 7194 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7195 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 7196 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7197 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 7198 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7199 (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7200 7201 // We need a layer of indirection because early machine code passes balk at 7202 // physical register (i.e. FFR) uses that have no previous definition. 7203 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7204 def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>, 7205 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>; 7206 } 7207 7208 def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), 7209 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7210} 7211 7212class sve_mem_prfm_si<bits<2> msz, string asm> 7213: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6), 7214 asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]", 7215 "", 7216 []>, Sched<[]> { 7217 bits<5> Rn; 7218 bits<3> Pg; 7219 bits<6> imm6; 7220 bits<4> prfop; 7221 let Inst{31-22} = 0b1000010111; 7222 let Inst{21-16} = imm6; 7223 let Inst{15} = 0b0; 7224 let Inst{14-13} = msz; 7225 let Inst{12-10} = Pg; 7226 let Inst{9-5} = Rn; 7227 let Inst{4} = 0b0; 7228 let Inst{3-0} = prfop; 7229 7230 let hasSideEffects = 1; 7231} 7232 7233multiclass sve_mem_prfm_si<bits<2> msz, string asm> { 7234 def NAME : sve_mem_prfm_si<msz, asm>; 7235 7236 def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]", 7237 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7238} 7239 7240class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty> 7241: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7242 asm, "\t$prfop, $Pg, [$Rn, $Rm]", 7243 "", 7244 []>, Sched<[]> { 7245 bits<5> Rm; 7246 bits<5> Rn; 7247 bits<3> Pg; 7248 bits<4> prfop; 7249 let Inst{31-25} = 0b1000010; 7250 let Inst{24-23} = opc{2-1}; 7251 let Inst{22-21} = 0b00; 7252 let Inst{20-16} = Rm; 7253 let Inst{15} = 0b1; 7254 let Inst{14} = opc{0}; 7255 let Inst{13} = 0b0; 7256 let Inst{12-10} = Pg; 7257 let Inst{9-5} = Rn; 7258 let Inst{4} = 0b0; 7259 let Inst{3-0} = prfop; 7260 7261 let hasSideEffects = 1; 7262} 7263 7264class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm, 7265 RegisterOperand zprext> 7266: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7267 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7268 "", 7269 []>, Sched<[]> { 7270 bits<3> Pg; 7271 bits<5> Rn; 7272 bits<5> Zm; 7273 bits<4> prfop; 7274 let Inst{31-23} = 0b100001000; 7275 let Inst{22} = xs; 7276 let Inst{21} = 0b1; 7277 let Inst{20-16} = Zm; 7278 let Inst{15} = 0b0; 7279 let Inst{14-13} = msz; 7280 let Inst{12-10} = Pg; 7281 let Inst{9-5} = Rn; 7282 let Inst{4} = 0b0; 7283 let Inst{3-0} = prfop; 7284 7285 let hasSideEffects = 1; 7286} 7287 7288multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm, 7289 RegisterOperand sxtw_opnd, 7290 RegisterOperand uxtw_opnd, 7291 SDPatternOperator op_sxtw, 7292 SDPatternOperator op_uxtw> { 7293 def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>; 7294 def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>; 7295 7296 def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7297 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7298 7299 def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7300 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7301} 7302 7303class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7304: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7305 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7306 "", 7307 []>, Sched<[]> { 7308 bits<3> Pg; 7309 bits<5> Zn; 7310 bits<5> imm5; 7311 bits<4> prfop; 7312 let Inst{31-25} = 0b1000010; 7313 let Inst{24-23} = msz; 7314 let Inst{22-21} = 0b00; 7315 let Inst{20-16} = imm5; 7316 let Inst{15-13} = 0b111; 7317 let Inst{12-10} = Pg; 7318 let Inst{9-5} = Zn; 7319 let Inst{4} = 0b0; 7320 let Inst{3-0} = prfop; 7321} 7322 7323multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7324 def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>; 7325 7326 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7327 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7328 7329 def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7330 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7331} 7332 7333class sve_mem_z_fill<string asm> 7334: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9), 7335 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 7336 "", 7337 []>, Sched<[]> { 7338 bits<5> Rn; 7339 bits<5> Zt; 7340 bits<9> imm9; 7341 let Inst{31-22} = 0b1000010110; 7342 let Inst{21-16} = imm9{8-3}; 7343 let Inst{15-13} = 0b010; 7344 let Inst{12-10} = imm9{2-0}; 7345 let Inst{9-5} = Rn; 7346 let Inst{4-0} = Zt; 7347 7348 let mayLoad = 1; 7349} 7350 7351multiclass sve_mem_z_fill<string asm> { 7352 def NAME : sve_mem_z_fill<asm>; 7353 7354 def : InstAlias<asm # "\t$Zt, [$Rn]", 7355 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 7356} 7357 7358class sve_mem_p_fill<string asm> 7359: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9), 7360 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 7361 "", 7362 []>, Sched<[]> { 7363 bits<4> Pt; 7364 bits<5> Rn; 7365 bits<9> imm9; 7366 let Inst{31-22} = 0b1000010110; 7367 let Inst{21-16} = imm9{8-3}; 7368 let Inst{15-13} = 0b000; 7369 let Inst{12-10} = imm9{2-0}; 7370 let Inst{9-5} = Rn; 7371 let Inst{4} = 0b0; 7372 let Inst{3-0} = Pt; 7373 7374 let mayLoad = 1; 7375} 7376 7377multiclass sve_mem_p_fill<string asm> { 7378 def NAME : sve_mem_p_fill<asm>; 7379 7380 def : InstAlias<asm # "\t$Pt, [$Rn]", 7381 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 7382} 7383 7384class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm, 7385 RegisterOperand VecList> 7386: I<(outs VecList:$Zt), iops, 7387 asm, "\t$Zt, $Pg/z, [$Zn, $Rm]", 7388 "", 7389 []>, Sched<[]> { 7390 bits<3> Pg; 7391 bits<5> Rm; 7392 bits<5> Zn; 7393 bits<5> Zt; 7394 let Inst{31} = 0b1; 7395 let Inst{30} = opc{4}; 7396 let Inst{29-25} = 0b00010; 7397 let Inst{24-23} = opc{3-2}; 7398 let Inst{22-21} = 0b00; 7399 let Inst{20-16} = Rm; 7400 let Inst{15} = 0b1; 7401 let Inst{14-13} = opc{1-0}; 7402 let Inst{12-10} = Pg; 7403 let Inst{9-5} = Zn; 7404 let Inst{4-0} = Zt; 7405 7406 let mayLoad = 1; 7407} 7408 7409multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm, 7410 SDPatternOperator op, 7411 ValueType vt> { 7412 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 7413 asm, Z_s>; 7414 7415 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7416 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 7417 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7418 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 7419 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7420 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 7421 7422 def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)), 7423 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; 7424} 7425 7426multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm, 7427 SDPatternOperator op, 7428 ValueType vt> { 7429 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 7430 asm, Z_d>; 7431 7432 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7433 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 7434 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7435 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 7436 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7437 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 7438 7439 def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)), 7440 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; 7441} 7442 7443//===----------------------------------------------------------------------===// 7444// SVE Memory - 64-bit Gather Group 7445//===----------------------------------------------------------------------===// 7446 7447// bit xs is '1' if offsets are signed 7448// bit scaled is '1' if the offsets are scaled 7449// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7450class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm, 7451 RegisterOperand zprext> 7452: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7453 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7454 "", 7455 []>, Sched<[]> { 7456 bits<3> Pg; 7457 bits<5> Rn; 7458 bits<5> Zm; 7459 bits<5> Zt; 7460 let Inst{31-25} = 0b1100010; 7461 let Inst{24-23} = opc{3-2}; 7462 let Inst{22} = xs; 7463 let Inst{21} = scaled; 7464 let Inst{20-16} = Zm; 7465 let Inst{15} = lsl; 7466 let Inst{14-13} = opc{1-0}; 7467 let Inst{12-10} = Pg; 7468 let Inst{9-5} = Rn; 7469 let Inst{4-0} = Zt; 7470 7471 let mayLoad = 1; 7472 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7473 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7474} 7475 7476multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm, 7477 SDPatternOperator sxtw_op, 7478 SDPatternOperator uxtw_op, 7479 RegisterOperand sxtw_opnd, 7480 RegisterOperand uxtw_opnd, 7481 ValueType vt> { 7482 def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>; 7483 def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>; 7484 7485 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7486 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7487 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7488 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7489 7490 // We need a layer of indirection because early machine code passes balk at 7491 // physical register (i.e. FFR) uses that have no previous definition. 7492 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7493 def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7494 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7495 def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7496 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7497 } 7498 7499 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7500 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7501 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7502 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7503} 7504 7505multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm, 7506 SDPatternOperator sxtw_op, 7507 SDPatternOperator uxtw_op, 7508 RegisterOperand sxtw_opnd, 7509 RegisterOperand uxtw_opnd, 7510 ValueType vt> { 7511 def _UXTW_REAL : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>; 7512 def _SXTW_REAL : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>; 7513 7514 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7515 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7516 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7517 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7518 7519 // We need a layer of indirection because early machine code passes balk at 7520 // physical register (i.e. FFR) uses that have no previous definition. 7521 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7522 def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7523 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7524 def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7525 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7526 } 7527 7528 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7529 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7530 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7531 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7532} 7533 7534multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm, 7535 SDPatternOperator op, 7536 RegisterOperand zprext, ValueType vt> { 7537 def _SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>; 7538 7539 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7540 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 7541 7542 // We need a layer of indirection because early machine code passes balk at 7543 // physical register (i.e. FFR) uses that have no previous definition. 7544 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7545 def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>, 7546 PseudoInstExpansion<(!cast<Instruction>(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7547 } 7548 7549 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7550 (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7551} 7552 7553multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm, 7554 SDPatternOperator op, ValueType vt> { 7555 def _REAL : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>; 7556 7557 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7558 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 7559 7560 // We need a layer of indirection because early machine code passes balk at 7561 // physical register (i.e. FFR) uses that have no previous definition. 7562 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7563 def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>, 7564 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>; 7565 } 7566 7567 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7568 (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7569} 7570 7571class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7572: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7573 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7574 "", 7575 []>, Sched<[]> { 7576 bits<3> Pg; 7577 bits<5> Zn; 7578 bits<5> Zt; 7579 bits<5> imm5; 7580 let Inst{31-25} = 0b1100010; 7581 let Inst{24-23} = opc{3-2}; 7582 let Inst{22-21} = 0b01; 7583 let Inst{20-16} = imm5; 7584 let Inst{15} = 0b1; 7585 let Inst{14-13} = opc{1-0}; 7586 let Inst{12-10} = Pg; 7587 let Inst{9-5} = Zn; 7588 let Inst{4-0} = Zt; 7589 7590 let mayLoad = 1; 7591 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7592 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7593} 7594 7595multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty, 7596 SDPatternOperator op, ValueType vt> { 7597 def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>; 7598 7599 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7600 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 7601 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7602 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 7603 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7604 (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7605 7606 // We need a layer of indirection because early machine code passes balk at 7607 // physical register (i.e. FFR) uses that have no previous definition. 7608 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7609 def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>, 7610 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>; 7611 } 7612 7613 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), 7614 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7615} 7616 7617// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7618class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm, 7619 RegisterOperand zprext> 7620: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7621 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7622 "", 7623 []>, Sched<[]> { 7624 bits<3> Pg; 7625 bits<5> Rn; 7626 bits<5> Zm; 7627 bits<4> prfop; 7628 let Inst{31-23} = 0b110001000; 7629 let Inst{22} = xs; 7630 let Inst{21} = 0b1; 7631 let Inst{20-16} = Zm; 7632 let Inst{15} = lsl; 7633 let Inst{14-13} = msz; 7634 let Inst{12-10} = Pg; 7635 let Inst{9-5} = Rn; 7636 let Inst{4} = 0b0; 7637 let Inst{3-0} = prfop; 7638 7639 let hasSideEffects = 1; 7640} 7641 7642multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm, 7643 RegisterOperand sxtw_opnd, 7644 RegisterOperand uxtw_opnd, 7645 SDPatternOperator op_sxtw, 7646 SDPatternOperator op_uxtw> { 7647 def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>; 7648 def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>; 7649 7650 def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7651 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7652 7653 def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7654 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7655 7656} 7657 7658multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm, 7659 RegisterOperand zprext, SDPatternOperator frag> { 7660 def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>; 7661 7662 def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)), 7663 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7664 7665} 7666 7667class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7668: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7669 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7670 "", 7671 []>, Sched<[]> { 7672 bits<3> Pg; 7673 bits<5> Zn; 7674 bits<5> imm5; 7675 bits<4> prfop; 7676 let Inst{31-25} = 0b1100010; 7677 let Inst{24-23} = msz; 7678 let Inst{22-21} = 0b00; 7679 let Inst{20-16} = imm5; 7680 let Inst{15-13} = 0b111; 7681 let Inst{12-10} = Pg; 7682 let Inst{9-5} = Zn; 7683 let Inst{4} = 0b0; 7684 let Inst{3-0} = prfop; 7685 7686 let hasSideEffects = 1; 7687} 7688 7689multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7690 def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>; 7691 7692 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7693 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7694 7695 def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7696 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7697} 7698 7699//===----------------------------------------------------------------------===// 7700// SVE Compute Vector Address Group 7701//===----------------------------------------------------------------------===// 7702 7703class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm, 7704 ZPRRegOp zprty, RegisterOperand zprext> 7705: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm), 7706 asm, "\t$Zd, [$Zn, $Zm]", 7707 "", 7708 []>, Sched<[]> { 7709 bits<5> Zd; 7710 bits<5> Zn; 7711 bits<5> Zm; 7712 let Inst{31-24} = 0b00000100; 7713 let Inst{23-22} = opc; 7714 let Inst{21} = 0b1; 7715 let Inst{20-16} = Zm; 7716 let Inst{15-12} = 0b1010; 7717 let Inst{11-10} = msz; 7718 let Inst{9-5} = Zn; 7719 let Inst{4-0} = Zd; 7720} 7721 7722multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> { 7723 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>; 7724 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>; 7725 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>; 7726 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>; 7727} 7728 7729multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> { 7730 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>; 7731 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>; 7732 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>; 7733 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>; 7734} 7735 7736multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> { 7737 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>; 7738 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>; 7739 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>; 7740 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>; 7741} 7742 7743multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> { 7744 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>; 7745 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>; 7746 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>; 7747 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>; 7748} 7749 7750//===----------------------------------------------------------------------===// 7751// SVE Integer Misc - Unpredicated Group 7752//===----------------------------------------------------------------------===// 7753 7754class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty> 7755: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7756 asm, "\t$Zd, $Zn, $Zm", 7757 "", 7758 []>, Sched<[]> { 7759 bits<5> Zd; 7760 bits<5> Zm; 7761 bits<5> Zn; 7762 let Inst{31-24} = 0b00000100; 7763 let Inst{23-22} = sz; 7764 let Inst{21} = 0b1; 7765 let Inst{20-16} = Zm; 7766 let Inst{15-10} = 0b101100; 7767 let Inst{9-5} = Zn; 7768 let Inst{4-0} = Zd; 7769} 7770 7771multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> { 7772 def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>; 7773 def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>; 7774 def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>; 7775 7776 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7777 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7778 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7779} 7780 7781class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty> 7782: I<(outs zprty:$Zd), (ins zprty:$Zn), 7783 asm, "\t$Zd, $Zn", 7784 "", 7785 []>, Sched<[]> { 7786 bits<5> Zd; 7787 bits<5> Zn; 7788 let Inst{31-24} = 0b00000100; 7789 let Inst{23-22} = opc{7-6}; 7790 let Inst{21} = 0b1; 7791 let Inst{20-16} = opc{5-1}; 7792 let Inst{15-11} = 0b10111; 7793 let Inst{10} = opc{0}; 7794 let Inst{9-5} = Zn; 7795 let Inst{4-0} = Zd; 7796} 7797 7798multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> { 7799 def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>; 7800 def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>; 7801 def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>; 7802 7803 def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 7804 def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 7805 def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 7806} 7807 7808//===----------------------------------------------------------------------===// 7809// SVE Integer Reduction Group 7810//===----------------------------------------------------------------------===// 7811 7812class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm, 7813 ZPRRegOp zprty, FPRasZPROperand dstOpType> 7814: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 7815 asm, "\t$Vd, $Pg, $Zn", 7816 "", 7817 []>, Sched<[]> { 7818 bits<3> Pg; 7819 bits<5> Vd; 7820 bits<5> Zn; 7821 let Inst{31-24} = 0b00000100; 7822 let Inst{23-22} = sz8_32; 7823 let Inst{21} = 0b0; 7824 let Inst{20-19} = fmt; 7825 let Inst{18-16} = opc; 7826 let Inst{15-13} = 0b001; 7827 let Inst{12-10} = Pg; 7828 let Inst{9-5} = Zn; 7829 let Inst{4-0} = Vd; 7830} 7831 7832multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm, 7833 SDPatternOperator op> { 7834 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7835 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7836 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7837 7838 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7839 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7840 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7841} 7842 7843multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm, 7844 SDPatternOperator op> { 7845 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7846 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7847 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7848 def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64asZPR>; 7849 7850 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7851 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7852 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7853 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7854} 7855 7856multiclass sve_int_reduce_1<bits<3> opc, string asm, 7857 SDPatternOperator op> { 7858 def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8asZPR>; 7859 def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16asZPR>; 7860 def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32asZPR>; 7861 def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64asZPR>; 7862 7863 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7864 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7865 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7866 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7867} 7868 7869multiclass sve_int_reduce_2<bits<3> opc, string asm, 7870 SDPatternOperator op> { 7871 def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8asZPR>; 7872 def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16asZPR>; 7873 def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32asZPR>; 7874 def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64asZPR>; 7875 7876 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7877 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7878 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7879 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7880} 7881 7882class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm, 7883 ZPRRegOp zprty, string pg_suffix, dag iops> 7884: I<(outs zprty:$Zd), iops, 7885 asm, "\t$Zd, $Pg"#pg_suffix#", $Zn", 7886 "", 7887 []>, Sched<[]> { 7888 bits<3> Pg; 7889 bits<5> Zd; 7890 bits<5> Zn; 7891 let Inst{31-24} = 0b00000100; 7892 let Inst{23-22} = sz8_32; 7893 let Inst{21-19} = 0b010; 7894 let Inst{18-16} = opc; 7895 let Inst{15-13} = 0b001; 7896 let Inst{12-10} = Pg; 7897 let Inst{9-5} = Zn; 7898 let Inst{4-0} = Zd; 7899 7900 let ElementSize = zprty.ElementSize; 7901} 7902 7903multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> { 7904let Constraints = "$Zd = $_Zd" in { 7905 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m", 7906 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>; 7907 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m", 7908 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>; 7909 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m", 7910 (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>; 7911 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m", 7912 (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>; 7913} 7914} 7915 7916multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> { 7917 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z", 7918 (ins PPR3bAny:$Pg, ZPR8:$Zn)>; 7919 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z", 7920 (ins PPR3bAny:$Pg, ZPR16:$Zn)>; 7921 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z", 7922 (ins PPR3bAny:$Pg, ZPR32:$Zn)>; 7923 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z", 7924 (ins PPR3bAny:$Pg, ZPR64:$Zn)>; 7925} 7926 7927//===----------------------------------------------------------------------===// 7928// SVE Propagate Break Group 7929//===----------------------------------------------------------------------===// 7930 7931class sve_int_brkp<bits<2> opc, string asm> 7932: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 7933 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 7934 "", 7935 []>, Sched<[]> { 7936 bits<4> Pd; 7937 bits<4> Pg; 7938 bits<4> Pm; 7939 bits<4> Pn; 7940 let Inst{31-24} = 0b00100101; 7941 let Inst{23} = 0b0; 7942 let Inst{22} = opc{1}; 7943 let Inst{21-20} = 0b00; 7944 let Inst{19-16} = Pm; 7945 let Inst{15-14} = 0b11; 7946 let Inst{13-10} = Pg; 7947 let Inst{9} = 0b0; 7948 let Inst{8-5} = Pn; 7949 let Inst{4} = opc{0}; 7950 let Inst{3-0} = Pd; 7951 7952 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7953} 7954 7955multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> { 7956 def NAME : sve_int_brkp<opc, asm>; 7957 7958 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7959} 7960 7961 7962//===----------------------------------------------------------------------===// 7963// SVE Partition Break Group 7964//===----------------------------------------------------------------------===// 7965 7966class sve_int_brkn<bit S, string asm> 7967: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm), 7968 asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm", 7969 "", 7970 []>, Sched<[]> { 7971 bits<4> Pdm; 7972 bits<4> Pg; 7973 bits<4> Pn; 7974 let Inst{31-23} = 0b001001010; 7975 let Inst{22} = S; 7976 let Inst{21-14} = 0b01100001; 7977 let Inst{13-10} = Pg; 7978 let Inst{9} = 0b0; 7979 let Inst{8-5} = Pn; 7980 let Inst{4} = 0b0; 7981 let Inst{3-0} = Pdm; 7982 7983 let Constraints = "$Pdm = $_Pdm"; 7984 let Defs = !if(S, [NZCV], []); 7985} 7986 7987multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> { 7988 def NAME : sve_int_brkn<opc, asm>; 7989 7990 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7991} 7992 7993class sve_int_break<bits<3> opc, string asm, string suffix, dag iops> 7994: I<(outs PPR8:$Pd), iops, 7995 asm, "\t$Pd, $Pg"#suffix#", $Pn", 7996 "", 7997 []>, Sched<[]> { 7998 bits<4> Pd; 7999 bits<4> Pg; 8000 bits<4> Pn; 8001 let Inst{31-24} = 0b00100101; 8002 let Inst{23-22} = opc{2-1}; 8003 let Inst{21-14} = 0b01000001; 8004 let Inst{13-10} = Pg; 8005 let Inst{9} = 0b0; 8006 let Inst{8-5} = Pn; 8007 let Inst{4} = opc{0}; 8008 let Inst{3-0} = Pd; 8009 8010 let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", ""); 8011 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 8012 8013} 8014 8015multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> { 8016 def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>; 8017 8018 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8019} 8020 8021multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> { 8022 def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>; 8023 8024 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8025} 8026 8027//===----------------------------------------------------------------------===// 8028// SVE2 String Processing Group 8029//===----------------------------------------------------------------------===// 8030 8031class sve2_char_match<bit sz, bit opc, string asm, 8032 PPRRegOp pprty, ZPRRegOp zprty> 8033: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 8034 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 8035 "", 8036 []>, Sched<[]> { 8037 bits<4> Pd; 8038 bits<3> Pg; 8039 bits<5> Zm; 8040 bits<5> Zn; 8041 let Inst{31-23} = 0b010001010; 8042 let Inst{22} = sz; 8043 let Inst{21} = 0b1; 8044 let Inst{20-16} = Zm; 8045 let Inst{15-13} = 0b100; 8046 let Inst{12-10} = Pg; 8047 let Inst{9-5} = Zn; 8048 let Inst{4} = opc; 8049 let Inst{3-0} = Pd; 8050 8051 let Defs = [NZCV]; 8052 let isPTestLike = 1; 8053} 8054 8055multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> { 8056 def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>; 8057 def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>; 8058 8059 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 8060 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 8061} 8062 8063//===----------------------------------------------------------------------===// 8064// SVE2 Histogram Computation - Segment Group 8065//===----------------------------------------------------------------------===// 8066 8067class sve2_hist_gen_segment<string asm, SDPatternOperator op> 8068: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm), 8069 asm, "\t$Zd, $Zn, $Zm", 8070 "", 8071 [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> { 8072 bits<5> Zd; 8073 bits<5> Zn; 8074 bits<5> Zm; 8075 let Inst{31-21} = 0b01000101001; 8076 let Inst{20-16} = Zm; 8077 let Inst{15-10} = 0b101000; 8078 let Inst{9-5} = Zn; 8079 let Inst{4-0} = Zd; 8080} 8081 8082//===----------------------------------------------------------------------===// 8083// SVE2 Histogram Computation - Vector Group 8084//===----------------------------------------------------------------------===// 8085 8086class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty> 8087: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 8088 asm, "\t$Zd, $Pg/z, $Zn, $Zm", 8089 "", 8090 []>, Sched<[]> { 8091 bits<5> Zd; 8092 bits<5> Zn; 8093 bits<3> Pg; 8094 bits<5> Zm; 8095 let Inst{31-23} = 0b010001011; 8096 let Inst{22} = sz; 8097 let Inst{21} = 0b1; 8098 let Inst{20-16} = Zm; 8099 let Inst{15-13} = 0b110; 8100 let Inst{12-10} = Pg; 8101 let Inst{9-5} = Zn; 8102 let Inst{4-0} = Zd; 8103} 8104 8105multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> { 8106 def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>; 8107 def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>; 8108 8109 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 8110 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 8111} 8112 8113//===----------------------------------------------------------------------===// 8114// SVE2 Crypto Extensions Group 8115//===----------------------------------------------------------------------===// 8116 8117class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty> 8118: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 8119 asm, "\t$Zd, $Zn, $Zm", 8120 "", 8121 []>, Sched<[]> { 8122 bits<5> Zd; 8123 bits<5> Zn; 8124 bits<5> Zm; 8125 let Inst{31-21} = 0b01000101001; 8126 let Inst{20-16} = Zm; 8127 let Inst{15-11} = 0b11110; 8128 let Inst{10} = opc; 8129 let Inst{9-5} = Zn; 8130 let Inst{4-0} = Zd; 8131} 8132 8133multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty, 8134 SDPatternOperator op, ValueType vt> { 8135 def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>; 8136 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 8137} 8138 8139class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty> 8140: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm), 8141 asm, "\t$Zdn, $_Zdn, $Zm", 8142 "", 8143 []>, Sched<[]> { 8144 bits<5> Zdn; 8145 bits<5> Zm; 8146 let Inst{31-17} = 0b010001010010001; 8147 let Inst{16} = opc{1}; 8148 let Inst{15-11} = 0b11100; 8149 let Inst{10} = opc{0}; 8150 let Inst{9-5} = Zm; 8151 let Inst{4-0} = Zdn; 8152 8153 let Constraints = "$Zdn = $_Zdn"; 8154} 8155 8156multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty, 8157 SDPatternOperator op, ValueType vt> { 8158 def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>; 8159 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 8160} 8161 8162class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty> 8163: I<(outs zprty:$Zdn), (ins zprty:$_Zdn), 8164 asm, "\t$Zdn, $_Zdn", 8165 "", 8166 []>, Sched<[]> { 8167 bits<5> Zdn; 8168 let Inst{31-11} = 0b010001010010000011100; 8169 let Inst{10} = opc; 8170 let Inst{9-5} = 0b00000; 8171 let Inst{4-0} = Zdn; 8172 8173 let Constraints = "$Zdn = $_Zdn"; 8174} 8175 8176multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> { 8177 def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>; 8178 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>; 8179} 8180 8181//===----------------------------------------------------------------------===// 8182// SVE BFloat16 Group 8183//===----------------------------------------------------------------------===// 8184 8185class sve_bfloat_dot_base<bits<2> opc, string asm, string ops, dag iops> 8186: I<(outs ZPR32:$Zda), iops, asm, ops, "", []>, Sched<[]> { 8187 bits<5> Zda; 8188 bits<5> Zn; 8189 let Inst{31-21} = 0b01100100011; 8190 let Inst{15-14} = opc; 8191 let Inst{13-10} = 0b0000; 8192 let Inst{9-5} = Zn; 8193 let Inst{4-0} = Zda; 8194 8195 let Constraints = "$Zda = $_Zda"; 8196 let DestructiveInstType = DestructiveOther; 8197 let ElementSize = ElementSizeH; 8198} 8199 8200class sve_bfloat_dot<string asm> 8201: sve_bfloat_dot_base<0b10, asm, "\t$Zda, $Zn, $Zm", 8202 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm)> { 8203 bits<5> Zm; 8204 let Inst{20-16} = Zm; 8205} 8206 8207multiclass sve_bfloat_dot<string asm, SDPatternOperator op> { 8208 def NAME : sve_bfloat_dot<asm>; 8209 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8210} 8211 8212class sve_bfloat_dot_indexed<string asm> 8213: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 8214 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> { 8215 bits<2> iop; 8216 bits<3> Zm; 8217 let Inst{20-19} = iop; 8218 let Inst{18-16} = Zm; 8219} 8220 8221multiclass sve_bfloat_dot_indexed<string asm, SDPatternOperator op> { 8222 def NAME : sve_bfloat_dot_indexed<asm>; 8223 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexS_timm, !cast<Instruction>(NAME)>; 8224} 8225 8226class sve_bfloat_matmul<string asm> 8227: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 8228 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8229 bits<5> Zm; 8230 bits<5> Zda; 8231 bits<5> Zn; 8232 let Inst{31-21} = 0b01100100011; 8233 let Inst{20-16} = Zm; 8234 let Inst{15-10} = 0b111001; 8235 let Inst{9-5} = Zn; 8236 let Inst{4-0} = Zda; 8237 8238 let Constraints = "$Zda = $_Zda"; 8239 let DestructiveInstType = DestructiveOther; 8240 let ElementSize = ElementSizeH; 8241} 8242 8243multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> { 8244 def NAME : sve_bfloat_matmul<asm>; 8245 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8246} 8247 8248class sve_bfloat_matmul_longvecl<bit BT, string asm> 8249: sve_bfloat_matmul<asm> { 8250 let Inst{23} = 0b1; 8251 let Inst{14-13} = 0b00; 8252 let Inst{10} = BT; 8253} 8254 8255multiclass sve_bfloat_matmul_longvecl<bit BT, string asm, SDPatternOperator op> { 8256 def NAME : sve_bfloat_matmul_longvecl<BT, asm>; 8257 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8258} 8259 8260class sve_bfloat_matmul_longvecl_idx<bit BT, string asm> 8261: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 8262 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> { 8263 bits<3> iop; 8264 bits<3> Zm; 8265 let Inst{23} = 0b1; 8266 let Inst{20-19} = iop{2-1}; 8267 let Inst{18-16} = Zm; 8268 let Inst{11} = iop{0}; 8269 let Inst{10} = BT; 8270} 8271 8272multiclass sve_bfloat_matmul_longvecl_idx<bit BT, string asm, SDPatternOperator op> { 8273 def NAME : sve_bfloat_matmul_longvecl_idx<BT, asm>; 8274 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexH_timm, !cast<Instruction>(NAME)>; 8275} 8276 8277class sve_bfloat_convert<bit N, string asm> 8278: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), 8279 asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { 8280 bits<5> Zd; 8281 bits<3> Pg; 8282 bits<5> Zn; 8283 let Inst{31-25} = 0b0110010; 8284 let Inst{24} = N; 8285 let Inst{23-13} = 0b10001010101; 8286 let Inst{12-10} = Pg; 8287 let Inst{9-5} = Zn; 8288 let Inst{4-0} = Zd; 8289 8290 let Constraints = "$Zd = $_Zd"; 8291 let DestructiveInstType = DestructiveOther; 8292 let hasSideEffects = 1; 8293 let ElementSize = ElementSizeS; 8294} 8295 8296multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> { 8297 def NAME : sve_bfloat_convert<N, asm>; 8298 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>; 8299} 8300 8301//===----------------------------------------------------------------------===// 8302// SVE Integer Matrix Multiply Group 8303//===----------------------------------------------------------------------===// 8304 8305class sve_int_matmul<bits<2> uns, string asm> 8306: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8307 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8308 bits<5> Zda; 8309 bits<5> Zn; 8310 bits<5> Zm; 8311 let Inst{31-24} = 0b01000101; 8312 let Inst{23-22} = uns; 8313 let Inst{21} = 0; 8314 let Inst{20-16} = Zm; 8315 let Inst{15-10} = 0b100110; 8316 let Inst{9-5} = Zn; 8317 let Inst{4-0} = Zda; 8318 8319 let Constraints = "$Zda = $_Zda"; 8320 let DestructiveInstType = DestructiveOther; 8321 let ElementSize = ZPR32.ElementSize; 8322} 8323 8324multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> { 8325 def NAME : sve_int_matmul<uns, asm>; 8326 8327 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8328} 8329 8330//===----------------------------------------------------------------------===// 8331// SVE Integer Dot Product Mixed Sign Group 8332//===----------------------------------------------------------------------===// 8333 8334class sve_int_dot_mixed<string asm> 8335: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8336 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8337 bits<5> Zda; 8338 bits<5> Zn; 8339 bits<5> Zm; 8340 let Inst{31-21} = 0b01000100100; 8341 let Inst{20-16} = Zm; 8342 let Inst{15-10} = 0b011110; 8343 let Inst{9-5} = Zn; 8344 let Inst{4-0} = Zda; 8345 8346 let Constraints = "$Zda = $_Zda"; 8347 let DestructiveInstType = DestructiveOther; 8348 let ElementSize = ZPR32.ElementSize; 8349} 8350 8351multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> { 8352 def NAME : sve_int_dot_mixed<asm>; 8353 8354 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8355} 8356 8357//===----------------------------------------------------------------------===// 8358// SVE Integer Dot Product Mixed Sign - Indexed Group 8359//===----------------------------------------------------------------------===// 8360 8361class sve_int_dot_mixed_indexed<bit U, string asm> 8362: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx), 8363 asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> { 8364 bits<5> Zda; 8365 bits<5> Zn; 8366 bits<3> Zm; 8367 bits<2> idx; 8368 let Inst{31-21} = 0b01000100101; 8369 let Inst{20-19} = idx; 8370 let Inst{18-16} = Zm; 8371 let Inst{15-11} = 0b00011; 8372 let Inst{10} = U; 8373 let Inst{9-5} = Zn; 8374 let Inst{4-0} = Zda; 8375 8376 let Constraints = "$Zda = $_Zda"; 8377 let DestructiveInstType = DestructiveOther; 8378 let ElementSize = ZPR32.ElementSize; 8379} 8380 8381multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> { 8382 def NAME : sve_int_dot_mixed_indexed<U, asm>; 8383 8384 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>; 8385} 8386 8387//===----------------------------------------------------------------------===// 8388// SVE Floating Point Matrix Multiply Accumulate Group 8389//===----------------------------------------------------------------------===// 8390 8391class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty> 8392: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm), 8393 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8394 bits<5> Zda; 8395 bits<5> Zn; 8396 bits<5> Zm; 8397 let Inst{31-23} = 0b011001001; 8398 let Inst{22} = sz; 8399 let Inst{21} = 1; 8400 let Inst{20-16} = Zm; 8401 let Inst{15-10} = 0b111001; 8402 let Inst{9-5} = Zn; 8403 let Inst{4-0} = Zda; 8404 8405 let Constraints = "$Zda = $_Zda"; 8406 let DestructiveInstType = DestructiveOther; 8407 let ElementSize = zprty.ElementSize; 8408} 8409 8410multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> { 8411 def NAME : sve_fp_matrix_mla<sz, asm, zprty>; 8412 8413 def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>; 8414} 8415 8416//===----------------------------------------------------------------------===// 8417// SVE Memory - Contiguous Load And Replicate 256-bit Group 8418//===----------------------------------------------------------------------===// 8419 8420class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList> 8421: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 8422 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 8423 bits<5> Zt; 8424 bits<5> Rn; 8425 bits<3> Pg; 8426 bits<4> imm4; 8427 let Inst{31-25} = 0b1010010; 8428 let Inst{24-23} = sz; 8429 let Inst{22-20} = 0b010; 8430 let Inst{19-16} = imm4; 8431 let Inst{15-13} = 0b001; 8432 let Inst{12-10} = Pg; 8433 let Inst{9-5} = Rn; 8434 let Inst{4-0} = Zt; 8435 8436 let mayLoad = 1; 8437} 8438 8439multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty, 8440 ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> { 8441 def NAME : sve_mem_ldor_si<sz, asm, listty>; 8442 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8443 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 8444 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8445 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 8446 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 8447 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>; 8448 8449 // Base addressing mode 8450 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)), 8451 (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>; 8452 let AddedComplexity = 2 in { 8453 // Reg + Imm addressing mode 8454 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))), 8455 (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>; 8456 } 8457} 8458 8459class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList, 8460 RegisterOperand gprty> 8461: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 8462 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 8463 bits<5> Zt; 8464 bits<3> Pg; 8465 bits<5> Rn; 8466 bits<5> Rm; 8467 let Inst{31-25} = 0b1010010; 8468 let Inst{24-23} = sz; 8469 let Inst{22-21} = 0b01; 8470 let Inst{20-16} = Rm; 8471 let Inst{15-13} = 0; 8472 let Inst{12-10} = Pg; 8473 let Inst{9-5} = Rn; 8474 let Inst{4-0} = Zt; 8475 8476 let mayLoad = 1; 8477} 8478 8479multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty, 8480 ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty, 8481 ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> { 8482 def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>; 8483 8484 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 8485 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 8486 8487 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))), 8488 (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>; 8489} 8490 8491//===----------------------------------------------------------------------===// 8492// SVE Interleave 128-bit Elements Group 8493//===----------------------------------------------------------------------===// 8494 8495class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm> 8496: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm), 8497 asm, "\t$Zd, $Zn, $Zm", 8498 "", 8499 []>, Sched<[]> { 8500 bits<5> Zd; 8501 bits<5> Zm; 8502 bits<5> Zn; 8503 let Inst{31-21} = 0b00000101101; 8504 let Inst{20-16} = Zm; 8505 let Inst{15-13} = 0b000; 8506 let Inst{12-11} = opc; 8507 let Inst{10} = P; 8508 let Inst{9-5} = Zn; 8509 let Inst{4-0} = Zd; 8510} 8511 8512multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> { 8513 def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>; 8514 8515 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8516 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 8517 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 8518 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 8519 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME)>; 8520 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 8521 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME)>; 8522 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>; 8523} 8524 8525/// Addressing modes 8526def am_sve_indexed_s4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; 8527def am_sve_indexed_s6 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; 8528 8529def am_sve_regreg_lsl0 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<0>", []>; 8530def am_sve_regreg_lsl1 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<1>", []>; 8531def am_sve_regreg_lsl2 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<2>", []>; 8532def am_sve_regreg_lsl3 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<3>", []>; 8533def am_sve_regreg_lsl4 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<4>", []>; 8534 8535// Predicated pseudo floating point two operand instructions. 8536multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> { 8537 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8538 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8539 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8540 8541 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8542 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8543 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8544 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8545 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8546 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8547} 8548 8549// Predicated pseudo integer two operand instructions. 8550multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> { 8551 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8552 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8553 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8554 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8555 8556 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8557 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8558 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8559 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8560} 8561 8562// As sve_int_bin_pred but when only i32 and i64 vector types are required. 8563multiclass sve_int_bin_pred_sd<SDPatternOperator op> { 8564 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8565 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8566 8567 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8568 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8569} 8570 8571// Predicated pseudo integer two operand instructions. Second operand is an 8572// immediate specified by imm_[bhsd]. 8573multiclass sve_int_shift_pred_bhsd<SDPatternOperator op, 8574 ComplexPattern imm_b, ComplexPattern imm_h, 8575 ComplexPattern imm_s, ComplexPattern imm_d> { 8576 def _UNDEF_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, Operand<i32>, FalseLanesUndef>; 8577 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesUndef>; 8578 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesUndef>; 8579 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesUndef>; 8580 8581 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _UNDEF_B)>; 8582 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Instruction>(NAME # _UNDEF_H)>; 8583 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Instruction>(NAME # _UNDEF_S)>; 8584 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Instruction>(NAME # _UNDEF_D)>; 8585} 8586 8587multiclass sve_int_bin_pred_all_active_bhsd<SDPatternOperator op> { 8588 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8589 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8590 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8591 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8592 8593 def : SVE_2_Op_Pred_All_Active_Pt<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8594 def : SVE_2_Op_Pred_All_Active_Pt<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8595 def : SVE_2_Op_Pred_All_Active_Pt<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8596 def : SVE_2_Op_Pred_All_Active_Pt<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8597} 8598 8599