1//==--- riscv_vector.td - RISC-V V-ext Builtin function list --------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the builtins for RISC-V V-extension. See: 10// 11// https://github.com/riscv/rvv-intrinsic-doc 12// 13//===----------------------------------------------------------------------===// 14 15include "riscv_vector_common.td" 16 17//===----------------------------------------------------------------------===// 18// Basic classes with automatic codegen. 19//===----------------------------------------------------------------------===// 20 21class RVVOutBuiltin<string suffix, string prototype, string type_range> 22 : RVVBuiltin<suffix, prototype, type_range> { 23 let IntrinsicTypes = [-1]; 24} 25 26class RVVOp0Builtin<string suffix, string prototype, string type_range> 27 : RVVBuiltin<suffix, prototype, type_range> { 28 let IntrinsicTypes = [0]; 29} 30 31class RVVOutOp1Builtin<string suffix, string prototype, string type_range> 32 : RVVBuiltin<suffix, prototype, type_range> { 33 let IntrinsicTypes = [-1, 1]; 34} 35 36class RVVOutOp0Op1Builtin<string suffix, string prototype, string type_range> 37 : RVVBuiltin<suffix, prototype, type_range> { 38 let IntrinsicTypes = [-1, 0, 1]; 39} 40 41multiclass RVVBuiltinSet<string intrinsic_name, string type_range, 42 list<list<string>> suffixes_prototypes, 43 list<int> intrinsic_types> { 44 let IRName = intrinsic_name, MaskedIRName = intrinsic_name # "_mask", 45 IntrinsicTypes = intrinsic_types in { 46 foreach s_p = suffixes_prototypes in { 47 let Name = NAME # "_" # s_p[0] in { 48 defvar suffix = s_p[1]; 49 defvar prototype = s_p[2]; 50 def : RVVBuiltin<suffix, prototype, type_range>; 51 } 52 } 53 } 54} 55 56// IntrinsicTypes is output, op0, op1 [-1, 0, 1] 57multiclass RVVOutOp0Op1BuiltinSet<string intrinsic_name, string type_range, 58 list<list<string>> suffixes_prototypes> 59 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, 60 [-1, 0, 1]>; 61 62multiclass RVVOutBuiltinSet<string intrinsic_name, string type_range, 63 list<list<string>> suffixes_prototypes> 64 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1]>; 65 66multiclass RVVOp0BuiltinSet<string intrinsic_name, string type_range, 67 list<list<string>> suffixes_prototypes> 68 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0]>; 69 70// IntrinsicTypes is output, op1 [-1, 0] 71multiclass RVVOutOp0BuiltinSet<string intrinsic_name, string type_range, 72 list<list<string>> suffixes_prototypes> 73 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 0]>; 74 75// IntrinsicTypes is output, op1 [-1, 1] 76multiclass RVVOutOp1BuiltinSet<string intrinsic_name, string type_range, 77 list<list<string>> suffixes_prototypes> 78 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1]>; 79 80multiclass RVVOp0Op1BuiltinSet<string intrinsic_name, string type_range, 81 list<list<string>> suffixes_prototypes> 82 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0, 1]>; 83 84multiclass RVVOutOp1Op2BuiltinSet<string intrinsic_name, string type_range, 85 list<list<string>> suffixes_prototypes> 86 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1, 2]>; 87 88multiclass RVVSignedBinBuiltinSet 89 : RVVOutOp1BuiltinSet<NAME, "csil", 90 [["vv", "v", "vvv"], 91 ["vx", "v", "vve"]]>; 92 93multiclass RVVSignedBinBuiltinSetRoundingMode 94 : RVVOutOp1BuiltinSet<NAME, "csil", 95 [["vv", "v", "vvvu"], 96 ["vx", "v", "vveu"]]>; 97 98multiclass RVVUnsignedBinBuiltinSet 99 : RVVOutOp1BuiltinSet<NAME, "csil", 100 [["vv", "Uv", "UvUvUv"], 101 ["vx", "Uv", "UvUvUe"]]>; 102 103multiclass RVVUnsignedBinBuiltinSetRoundingMode 104 : RVVOutOp1BuiltinSet<NAME, "csil", 105 [["vv", "Uv", "UvUvUvu"], 106 ["vx", "Uv", "UvUvUeu"]]>; 107 108multiclass RVVIntBinBuiltinSet 109 : RVVSignedBinBuiltinSet, 110 RVVUnsignedBinBuiltinSet; 111 112multiclass RVVSlideOneBuiltinSet 113 : RVVOutOp1BuiltinSet<NAME, "csil", 114 [["vx", "v", "vve"], 115 ["vx", "Uv", "UvUvUe"]]>; 116 117multiclass RVVSignedShiftBuiltinSet 118 : RVVOutOp1BuiltinSet<NAME, "csil", 119 [["vv", "v", "vvUv"], 120 ["vx", "v", "vvz"]]>; 121 122multiclass RVVSignedShiftBuiltinSetRoundingMode 123 : RVVOutOp1BuiltinSet<NAME, "csil", 124 [["vv", "v", "vvUvu"], 125 ["vx", "v", "vvzu"]]>; 126 127multiclass RVVUnsignedShiftBuiltinSet 128 : RVVOutOp1BuiltinSet<NAME, "csil", 129 [["vv", "Uv", "UvUvUv"], 130 ["vx", "Uv", "UvUvz"]]>; 131 132multiclass RVVUnsignedShiftBuiltinSetRoundingMode 133 : RVVOutOp1BuiltinSet<NAME, "csil", 134 [["vv", "Uv", "UvUvUvu"], 135 ["vx", "Uv", "UvUvzu"]]>; 136 137multiclass RVVShiftBuiltinSet 138 : RVVSignedShiftBuiltinSet, 139 RVVUnsignedShiftBuiltinSet; 140 141let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 142 multiclass RVVSignedNShiftBuiltinSet 143 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 144 [["wv", "v", "vwUv"], 145 ["wx", "v", "vwz"]]>; 146 147 multiclass RVVSignedNShiftBuiltinSetRoundingMode 148 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 149 [["wv", "v", "vwUvu"], 150 ["wx", "v", "vwzu"]]>; 151 152 multiclass RVVUnsignedNShiftBuiltinSet 153 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 154 [["wv", "Uv", "UvUwUv"], 155 ["wx", "Uv", "UvUwz"]]>; 156 157 multiclass RVVUnsignedNShiftBuiltinSetRoundingMode 158 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 159 [["wv", "Uv", "UvUwUvu"], 160 ["wx", "Uv", "UvUwzu"]]>; 161 162} 163 164multiclass RVVCarryinBuiltinSet 165 : RVVOutOp1BuiltinSet<NAME, "csil", 166 [["vvm", "v", "vvvm"], 167 ["vxm", "v", "vvem"], 168 ["vvm", "Uv", "UvUvUvm"], 169 ["vxm", "Uv", "UvUvUem"]]>; 170 171multiclass RVVCarryOutInBuiltinSet<string intrinsic_name> 172 : RVVOp0Op1BuiltinSet<intrinsic_name, "csil", 173 [["vvm", "vm", "mvvm"], 174 ["vxm", "vm", "mvem"], 175 ["vvm", "Uvm", "mUvUvm"], 176 ["vxm", "Uvm", "mUvUem"]]>; 177 178multiclass RVVSignedMaskOutBuiltinSet 179 : RVVOp0Op1BuiltinSet<NAME, "csil", 180 [["vv", "vm", "mvv"], 181 ["vx", "vm", "mve"]]>; 182 183multiclass RVVUnsignedMaskOutBuiltinSet 184 : RVVOp0Op1BuiltinSet<NAME, "csil", 185 [["vv", "Uvm", "mUvUv"], 186 ["vx", "Uvm", "mUvUe"]]>; 187 188multiclass RVVIntMaskOutBuiltinSet 189 : RVVSignedMaskOutBuiltinSet, 190 RVVUnsignedMaskOutBuiltinSet; 191 192class RVVIntExt<string intrinsic_name, string suffix, string prototype, 193 string type_range> 194 : RVVBuiltin<suffix, prototype, type_range> { 195 let IRName = intrinsic_name; 196 let MaskedIRName = intrinsic_name # "_mask"; 197 let OverloadedName = NAME; 198 let IntrinsicTypes = [-1, 0]; 199} 200 201let HasMaskedOffOperand = false in { 202 multiclass RVVIntTerBuiltinSet { 203 defm "" : RVVOutOp1BuiltinSet<NAME, "csil", 204 [["vv", "v", "vvvv"], 205 ["vx", "v", "vvev"], 206 ["vv", "Uv", "UvUvUvUv"], 207 ["vx", "Uv", "UvUvUeUv"]]>; 208 } 209 multiclass RVVFloatingTerBuiltinSet { 210 defm "" : RVVOutOp1BuiltinSet<NAME, "xfd", 211 [["vv", "v", "vvvv"], 212 ["vf", "v", "vvev"]]>; 213 } 214 multiclass RVVFloatingTerBuiltinSetRoundingMode { 215 defm "" : RVVOutOp1BuiltinSet<NAME, "xfd", 216 [["vv", "v", "vvvvu"], 217 ["vf", "v", "vvevu"]]>; 218 } 219} 220 221let HasMaskedOffOperand = false, Log2LMUL = [-2, -1, 0, 1, 2] in { 222 multiclass RVVFloatingWidenTerBuiltinSet { 223 defm "" : RVVOutOp1Op2BuiltinSet<NAME, "xf", 224 [["vv", "w", "wwvv"], 225 ["vf", "w", "wwev"]]>; 226 } 227 multiclass RVVFloatingWidenTerBuiltinSetRoundingMode { 228 defm "" : RVVOutOp1Op2BuiltinSet<NAME, "xf", 229 [["vv", "w", "wwvvu"], 230 ["vf", "w", "wwevu"]]>; 231 } 232} 233 234multiclass RVVFloatingBinBuiltinSet 235 : RVVOutOp1BuiltinSet<NAME, "xfd", 236 [["vv", "v", "vvv"], 237 ["vf", "v", "vve"]]>; 238 239multiclass RVVFloatingBinBuiltinSetRoundingMode 240 : RVVOutOp1BuiltinSet<NAME, "xfd", 241 [["vv", "v", "vvvu"], 242 ["vf", "v", "vveu"]]>; 243 244multiclass RVVFloatingBinVFBuiltinSet 245 : RVVOutOp1BuiltinSet<NAME, "xfd", 246 [["vf", "v", "vve"]]>; 247 248multiclass RVVFloatingBinVFBuiltinSetRoundingMode 249 : RVVOutOp1BuiltinSet<NAME, "xfd", 250 [["vf", "v", "vveu"]]>; 251 252multiclass RVVFloatingMaskOutBuiltinSet 253 : RVVOp0Op1BuiltinSet<NAME, "xfd", 254 [["vv", "vm", "mvv"], 255 ["vf", "vm", "mve"]]>; 256 257multiclass RVVFloatingMaskOutVFBuiltinSet 258 : RVVOp0Op1BuiltinSet<NAME, "fd", 259 [["vf", "vm", "mve"]]>; 260 261multiclass RVVConvBuiltinSet<string intrinsic_name, string type_range, 262 list<list<string>> suffixes_prototypes> { 263let Name = intrinsic_name, 264 IRName = intrinsic_name, 265 MaskedIRName = intrinsic_name # "_mask", 266 IntrinsicTypes = [-1, 0] in { 267 foreach s_p = suffixes_prototypes in { 268 defvar suffix = s_p[0]; 269 defvar prototype = s_p[1]; 270 def : RVVBuiltin<suffix, prototype, type_range>; 271 } 272 } 273} 274 275 276class RVVMaskBinBuiltin : RVVOutBuiltin<"m", "mmm", "c"> { 277 let Name = NAME # "_mm"; 278 let HasMasked = false; 279} 280 281class RVVMaskUnaryBuiltin : RVVOutBuiltin<"m", "mm", "c"> { 282 let Name = NAME # "_m"; 283} 284 285class RVVMaskNullaryBuiltin : RVVOutBuiltin<"m", "m", "c"> { 286 let Name = NAME # "_m"; 287 let HasMasked = false; 288 let SupportOverloading = false; 289} 290 291class RVVMaskOp0Builtin<string prototype> : RVVOp0Builtin<"m", prototype, "c"> { 292 let Name = NAME # "_m"; 293 let HasMaskedOffOperand = false; 294} 295 296let UnMaskedPolicyScheme = HasPolicyOperand, 297 HasMaskedOffOperand = false in { 298 multiclass RVVSlideUpBuiltinSet { 299 defm "" : RVVOutBuiltinSet<NAME, "csilxfd", 300 [["vx","v", "vvvz"]]>; 301 defm "" : RVVOutBuiltinSet<NAME, "csil", 302 [["vx","Uv", "UvUvUvz"]]>; 303 } 304} 305 306let UnMaskedPolicyScheme = HasPassthruOperand, 307 ManualCodegen = [{ 308 if (IsMasked) { 309 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 310 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 311 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 312 } else { 313 if (PolicyAttrs & RVV_VTA) 314 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 315 } 316 317 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 318 IntrinsicTypes = {ResultType, Ops.back()->getType()}; 319 }] in { 320 multiclass RVVSlideDownBuiltinSet { 321 defm "" : RVVOutBuiltinSet<NAME, "csilxfd", 322 [["vx","v", "vvz"]]>; 323 defm "" : RVVOutBuiltinSet<NAME, "csil", 324 [["vx","Uv", "UvUvz"]]>; 325 } 326} 327 328class RVVFloatingUnaryBuiltin<string builtin_suffix, string ir_suffix, 329 string prototype> 330 : RVVOutBuiltin<ir_suffix, prototype, "xfd"> { 331 let Name = NAME # "_" # builtin_suffix; 332} 333 334class RVVFloatingUnaryVVBuiltin : RVVFloatingUnaryBuiltin<"v", "v", "vv">; 335 336class RVVConvBuiltin<string suffix, string prototype, string type_range, 337 string overloaded_name> 338 : RVVBuiltin<suffix, prototype, type_range> { 339 let IntrinsicTypes = [-1, 0]; 340 let OverloadedName = overloaded_name; 341} 342 343class RVVConvToSignedBuiltin<string overloaded_name> 344 : RVVConvBuiltin<"Iv", "Ivv", "xfd", overloaded_name>; 345 346class RVVConvToUnsignedBuiltin<string overloaded_name> 347 : RVVConvBuiltin<"Uv", "Uvv", "xfd", overloaded_name>; 348 349class RVVConvToWidenSignedBuiltin<string overloaded_name> 350 : RVVConvBuiltin<"Iw", "Iwv", "xf", overloaded_name>; 351 352class RVVConvToWidenUnsignedBuiltin<string overloaded_name> 353 : RVVConvBuiltin<"Uw", "Uwv", "xf", overloaded_name>; 354 355class RVVConvToNarrowingSignedBuiltin<string overloaded_name> 356 : RVVConvBuiltin<"Iv", "IvFw", "csi", overloaded_name>; 357 358class RVVConvToNarrowingUnsignedBuiltin<string overloaded_name> 359 : RVVConvBuiltin<"Uv", "UvFw", "csi", overloaded_name>; 360 361let HasMaskedOffOperand = true in { 362 multiclass RVVSignedReductionBuiltin { 363 defm "" : RVVOutOp0BuiltinSet<NAME, "csil", 364 [["vs", "vSv", "SvvSv"]]>; 365 } 366 multiclass RVVUnsignedReductionBuiltin { 367 defm "" : RVVOutOp0BuiltinSet<NAME, "csil", 368 [["vs", "UvUSv", "USvUvUSv"]]>; 369 } 370 multiclass RVVFloatingReductionBuiltin { 371 defm "" : RVVOutOp0BuiltinSet<NAME, "xfd", 372 [["vs", "vSv", "SvvSv"]]>; 373 } 374 multiclass RVVFloatingReductionBuiltinRoundingMode { 375 defm "" : RVVOutOp0BuiltinSet<NAME, "xfd", 376 [["vs", "vSv", "SvvSvu"]]>; 377 } 378 multiclass RVVFloatingWidenReductionBuiltin { 379 defm "" : RVVOutOp0BuiltinSet<NAME, "xf", 380 [["vs", "vSw", "SwvSw"]]>; 381 } 382 multiclass RVVFloatingWidenReductionBuiltinRoundingMode { 383 defm "" : RVVOutOp0BuiltinSet<NAME, "xf", 384 [["vs", "vSw", "SwvSwu"]]>; 385 } 386} 387 388multiclass RVVIntReductionBuiltinSet 389 : RVVSignedReductionBuiltin, 390 RVVUnsignedReductionBuiltin; 391 392// For widen operation which has different mangling name. 393multiclass RVVWidenBuiltinSet<string intrinsic_name, string type_range, 394 list<list<string>> suffixes_prototypes> { 395 let Log2LMUL = [-3, -2, -1, 0, 1, 2], 396 IRName = intrinsic_name, MaskedIRName = intrinsic_name # "_mask" in { 397 foreach s_p = suffixes_prototypes in { 398 let Name = NAME # "_" # s_p[0], 399 OverloadedName = NAME # "_" # s_p[0] in { 400 defvar suffix = s_p[1]; 401 defvar prototype = s_p[2]; 402 def : RVVOutOp0Op1Builtin<suffix, prototype, type_range>; 403 } 404 } 405 } 406} 407 408// For widen operation with widen operand which has different mangling name. 409multiclass RVVWidenWOp0BuiltinSet<string intrinsic_name, string type_range, 410 list<list<string>> suffixes_prototypes> { 411 let Log2LMUL = [-3, -2, -1, 0, 1, 2], 412 IRName = intrinsic_name, MaskedIRName = intrinsic_name # "_mask" in { 413 foreach s_p = suffixes_prototypes in { 414 let Name = NAME # "_" # s_p[0], 415 OverloadedName = NAME # "_" # s_p[0] in { 416 defvar suffix = s_p[1]; 417 defvar prototype = s_p[2]; 418 def : RVVOutOp1Builtin<suffix, prototype, type_range>; 419 } 420 } 421 } 422} 423 424multiclass RVVSignedWidenBinBuiltinSet 425 : RVVWidenBuiltinSet<NAME, "csi", 426 [["vv", "w", "wvv"], 427 ["vx", "w", "wve"]]>; 428 429multiclass RVVSignedWidenOp0BinBuiltinSet 430 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", 431 [["wv", "w", "wwv"], 432 ["wx", "w", "wwe"]]>; 433 434multiclass RVVUnsignedWidenBinBuiltinSet 435 : RVVWidenBuiltinSet<NAME, "csi", 436 [["vv", "Uw", "UwUvUv"], 437 ["vx", "Uw", "UwUvUe"]]>; 438 439multiclass RVVUnsignedWidenOp0BinBuiltinSet 440 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", 441 [["wv", "Uw", "UwUwUv"], 442 ["wx", "Uw", "UwUwUe"]]>; 443 444multiclass RVVFloatingWidenBinBuiltinSet 445 : RVVWidenBuiltinSet<NAME, "xf", 446 [["vv", "w", "wvv"], 447 ["vf", "w", "wve"]]>; 448 449multiclass RVVFloatingWidenBinBuiltinSetRoundingMode 450 : RVVWidenBuiltinSet<NAME, "xf", 451 [["vv", "w", "wvvu"], 452 ["vf", "w", "wveu"]]>; 453 454multiclass RVVFloatingWidenOp0BinBuiltinSet 455 : RVVWidenWOp0BuiltinSet<NAME # "_w", "xf", 456 [["wv", "w", "wwv"], 457 ["wf", "w", "wwe"]]>; 458 459multiclass RVVFloatingWidenOp0BinBuiltinSetRoundingMode 460 : RVVWidenWOp0BuiltinSet<NAME # "_w", "xf", 461 [["wv", "w", "wwvu"], 462 ["wf", "w", "wweu"]]>; 463 464defvar TypeList = ["c","s","i","l","x","f","d"]; 465defvar EEWList = [["8", "(Log2EEW:3)"], 466 ["16", "(Log2EEW:4)"], 467 ["32", "(Log2EEW:5)"], 468 ["64", "(Log2EEW:6)"]]; 469 470class IsFloat<string type> { 471 bit val = !or(!eq(type, "x"), !eq(type, "f"), !eq(type, "d")); 472} 473 474let SupportOverloading = false, 475 MaskedPolicyScheme = NonePolicy in { 476 class RVVVLEMaskBuiltin : RVVOutBuiltin<"m", "mPCUe", "c"> { 477 let Name = "vlm_v"; 478 let IRName = "vlm"; 479 let HasMasked = false; 480 } 481} 482 483let SupportOverloading = false, 484 UnMaskedPolicyScheme = HasPassthruOperand in { 485 multiclass RVVVLEBuiltin<list<string> types> { 486 let Name = NAME # "_v", 487 IRName = "vle", 488 MaskedIRName ="vle_mask" in { 489 foreach type = types in { 490 def : RVVOutBuiltin<"v", "vPCe", type>; 491 if !not(IsFloat<type>.val) then { 492 def : RVVOutBuiltin<"Uv", "UvPCUe", type>; 493 } 494 } 495 } 496 } 497} 498 499multiclass RVVVLEFFBuiltin<list<string> types> { 500 let Name = NAME # "_v", 501 IRName = "vleff", 502 MaskedIRName = "vleff_mask", 503 SupportOverloading = false, 504 UnMaskedPolicyScheme = HasPassthruOperand, 505 ManualCodegen = [{ 506 { 507 if (IsMasked) { 508 // Move mask to right before vl. 509 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 510 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 511 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 512 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 513 IntrinsicTypes = {ResultType, Ops[4]->getType()}; 514 } else { 515 if (PolicyAttrs & RVV_VTA) 516 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 517 IntrinsicTypes = {ResultType, Ops[3]->getType()}; 518 } 519 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); 520 Value *NewVL = Ops[2]; 521 Ops.erase(Ops.begin() + 2); 522 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 523 llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); 524 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); 525 // Store new_vl. 526 clang::CharUnits Align; 527 if (IsMasked) 528 Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(E->getNumArgs()-2)->getType()); 529 else 530 Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); 531 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); 532 Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); 533 return V; 534 } 535 }] in { 536 foreach type = types in { 537 def : RVVBuiltin<"v", "vPCePz", type>; 538 // Skip floating types for unsigned versions. 539 if !not(IsFloat<type>.val) then { 540 def : RVVBuiltin<"Uv", "UvPCUePz", type>; 541 } 542 } 543 } 544} 545 546multiclass RVVVLSEBuiltin<list<string> types> { 547 let Name = NAME # "_v", 548 IRName = "vlse", 549 MaskedIRName ="vlse_mask", 550 SupportOverloading = false, 551 UnMaskedPolicyScheme = HasPassthruOperand in { 552 foreach type = types in { 553 def : RVVOutBuiltin<"v", "vPCet", type>; 554 if !not(IsFloat<type>.val) then { 555 def : RVVOutBuiltin<"Uv", "UvPCUet", type>; 556 } 557 } 558 } 559} 560 561multiclass RVVIndexedLoad<string op> { 562 let UnMaskedPolicyScheme = HasPassthruOperand in { 563 foreach type = TypeList in { 564 foreach eew_list = EEWList[0-2] in { 565 defvar eew = eew_list[0]; 566 defvar eew_type = eew_list[1]; 567 let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask" in { 568 def: RVVOutOp1Builtin<"v", "vPCe" # eew_type # "Uv", type>; 569 if !not(IsFloat<type>.val) then { 570 def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew_type # "Uv", type>; 571 } 572 } 573 } 574 defvar eew64 = "64"; 575 defvar eew64_type = "(Log2EEW:6)"; 576 let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", 577 RequiredFeatures = ["RV64"] in { 578 def: RVVOutOp1Builtin<"v", "vPCe" # eew64_type # "Uv", type>; 579 if !not(IsFloat<type>.val) then { 580 def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew64_type # "Uv", type>; 581 } 582 } 583 } 584 } 585} 586 587let HasMaskedOffOperand = false, 588 MaskedPolicyScheme = NonePolicy, 589 ManualCodegen = [{ 590 if (IsMasked) { 591 // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) 592 std::swap(Ops[0], Ops[2]); 593 } else { 594 // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) 595 std::swap(Ops[0], Ops[1]); 596 } 597 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 598 if (IsMasked) 599 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 600 else 601 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()}; 602 }] in { 603 class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { 604 let Name = "vsm_v"; 605 let IRName = "vsm"; 606 let HasMasked = false; 607 } 608 multiclass RVVVSEBuiltin<list<string> types> { 609 let Name = NAME # "_v", 610 IRName = "vse", 611 MaskedIRName = "vse_mask" in { 612 foreach type = types in { 613 def : RVVBuiltin<"v", "0Pev", type>; 614 if !not(IsFloat<type>.val) then { 615 def : RVVBuiltin<"Uv", "0PUeUv", type>; 616 } 617 } 618 } 619 } 620} 621 622multiclass RVVVSSEBuiltin<list<string> types> { 623 let Name = NAME # "_v", 624 IRName = "vsse", 625 MaskedIRName = "vsse_mask", 626 HasMaskedOffOperand = false, 627 MaskedPolicyScheme = NonePolicy, 628 ManualCodegen = [{ 629 if (IsMasked) { 630 // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) 631 std::swap(Ops[0], Ops[3]); 632 } else { 633 // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) 634 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 635 } 636 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 637 if (IsMasked) 638 IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()}; 639 else 640 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 641 }] in { 642 foreach type = types in { 643 def : RVVBuiltin<"v", "0Petv", type>; 644 if !not(IsFloat<type>.val) then { 645 def : RVVBuiltin<"Uv", "0PUetUv", type>; 646 } 647 } 648 } 649} 650 651multiclass RVVIndexedStore<string op> { 652 let HasMaskedOffOperand = false, 653 MaskedPolicyScheme = NonePolicy, 654 ManualCodegen = [{ 655 if (IsMasked) { 656 // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) 657 std::swap(Ops[0], Ops[3]); 658 } else { 659 // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) 660 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 661 } 662 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 663 if (IsMasked) 664 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()}; 665 else 666 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()}; 667 }] in { 668 foreach type = TypeList in { 669 foreach eew_list = EEWList[0-2] in { 670 defvar eew = eew_list[0]; 671 defvar eew_type = eew_list[1]; 672 let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask" in { 673 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>; 674 if !not(IsFloat<type>.val) then { 675 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "UvUv", type>; 676 } 677 } 678 } 679 defvar eew64 = "64"; 680 defvar eew64_type = "(Log2EEW:6)"; 681 let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", 682 RequiredFeatures = ["RV64"] in { 683 def : RVVBuiltin<"v", "0Pe" # eew64_type # "Uvv", type>; 684 if !not(IsFloat<type>.val) then { 685 def : RVVBuiltin<"Uv", "0PUe" # eew64_type # "UvUv", type>; 686 } 687 } 688 } 689 } 690} 691 692defvar NFList = [2, 3, 4, 5, 6, 7, 8]; 693/* 694A segment load builtin has different variants. 695 696Therefore a segment unit-stride load builtin can have 4 variants, 6971. When unmasked and the policies are all specified as agnostic: 698(Address0, ..., Address{NF - 1}, Ptr, VL) 6992. When masked and the policies are all specified as agnostic: 700(Address0, ..., Address{NF - 1}, Mask, Ptr, VL) 7013. When unmasked and one of the policies is specified as undisturbed: 702(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 703 Ptr, VL) 7044. When masked and one of the policies is specified as undisturbed: 705(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 706 Ptr, VL) 707 708Other variants of segment load builtin share the same structure, but they 709have their own extra parameter. 710 711The segment unit-stride fault-only-first load builtin has a 'NewVL' 712operand after the 'Ptr' operand. 7131. When unmasked and the policies are all specified as agnostic: 714(Address0, ..., Address{NF - 1}, Ptr, NewVL, VL) 7152. When masked and the policies are all specified as agnostic: 716(Address0, ..., Address{NF - 1}, Mask, Ptr, NewVL, VL) 7173. When unmasked and one of the policies is specified as undisturbed: 718(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 719 Ptr, NewVL, VL) 7204. When masked and one of the policies is specified as undisturbed: 721(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 722 Ptr, NewVL, VL) 723 724The segment strided load builtin has a 'Stride' operand after the 'Ptr' 725operand. 7261. When unmasked and the policies are all specified as agnostic: 727(Address0, ..., Address{NF - 1}, Ptr, Stride, VL) 7282. When masked and the policies are all specified as agnostic: 729(Address0, ..., Address{NF - 1}, Mask, Ptr, Stride, VL) 7303. When unmasked and one of the policies is specified as undisturbed: 731(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 732 Ptr, Stride, VL) 7334. When masked and one of the policies is specified as undisturbed: 734(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 735 Ptr, Stride, VL) 736 737The segment indexed load builtin has a 'Idx' operand after the 'Ptr' operand. 7381. When unmasked and the policies are all specified as agnostic: 739(Address0, ..., Address{NF - 1}, Ptr, Idx, VL) 7402. When masked and the policies are all specified as agnostic: 741(Address0, ..., Address{NF - 1}, Mask, Ptr, Idx, VL) 7423. When unmasked and one of the policies is specified as undisturbed: 743(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 744 Ptr, Idx, VL) 7454. When masked and one of the policies is specified as undisturbed: 746(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 747 Ptr, Idx, VL) 748 749Segment load intrinsics has different variants similar to their builtins. 750 751Segment unit-stride load intrinsic, 752 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) 753 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) 754Segment unit-stride fault-only-first load intrinsic, 755 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) 756 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) 757Segment strided load intrinsic, 758 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, Mask, VL, Policy) 759 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, VL) 760Segment indexed load intrinsic, 761 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, Mask, VL, Policy) 762 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, VL) 763 764The Vector(s) is poison when the policy behavior allows us to not care 765about any masked-off elements. 766*/ 767 768class PVString<int nf, bit signed> { 769 string S = 770 !cond(!eq(nf, 2): !if(signed, "PvPv", "PUvPUv"), 771 !eq(nf, 3): !if(signed, "PvPvPv", "PUvPUvPUv"), 772 !eq(nf, 4): !if(signed, "PvPvPvPv", "PUvPUvPUvPUv"), 773 !eq(nf, 5): !if(signed, "PvPvPvPvPv", "PUvPUvPUvPUvPUv"), 774 !eq(nf, 6): !if(signed, "PvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUv"), 775 !eq(nf, 7): !if(signed, "PvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUv"), 776 !eq(nf, 8): !if(signed, "PvPvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUvPUv")); 777} 778 779class VString<int nf, bit signed> { 780 string S = !cond(!eq(nf, 2): !if(signed, "vv", "UvUv"), 781 !eq(nf, 3): !if(signed, "vvv", "UvUvUv"), 782 !eq(nf, 4): !if(signed, "vvvv", "UvUvUvUv"), 783 !eq(nf, 5): !if(signed, "vvvvv", "UvUvUvUvUv"), 784 !eq(nf, 6): !if(signed, "vvvvvv", "UvUvUvUvUvUv"), 785 !eq(nf, 7): !if(signed, "vvvvvvv", "UvUvUvUvUvUvUv"), 786 !eq(nf, 8): !if(signed, "vvvvvvvv", "UvUvUvUvUvUvUvUv")); 787} 788 789multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { 790 let Name = NAME, 791 IRName = IR, 792 MaskedIRName = IR # "_mask", 793 UnMaskedPolicyScheme = HasPassthruOperand, 794 ManualCodegen = [{ 795 { 796 if (IsMasked) { 797 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 798 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 799 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 800 } else { 801 if (PolicyAttrs & RVV_VTA) 802 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 803 } 804 auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); 805 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); 806 807 if (IsMasked) { 808 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 809 // maskedoff, op1, op2, mask, vl, policy 810 IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; 811 } else { 812 // passthru, op1, op2, vl 813 IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; 814 } 815 break; 816 } 817 }] in { 818 def : RVVBuiltin<"v", "vv", type_range>; 819 } 820} 821 822multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { 823 let Name = NAME, 824 IRName = IR, 825 MaskedIRName = IR # "_mask", 826 UnMaskedPolicyScheme = HasPassthruOperand, 827 ManualCodegen = [{ 828 { 829 if (IsMasked) { 830 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 831 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 832 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 833 } else { 834 if (PolicyAttrs & RVV_VTA) 835 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 836 } 837 auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); 838 Ops.insert(Ops.begin() + 2, 839 llvm::Constant::getAllOnesValue(ElemTy)); 840 if (IsMasked) { 841 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 842 // maskedoff, op1, po2, mask, vl, policy 843 IntrinsicTypes = {ResultType, 844 ElemTy, 845 Ops[4]->getType()}; 846 } else { 847 // passthru, op1, op2, vl 848 IntrinsicTypes = {ResultType, 849 ElemTy, 850 Ops[3]->getType()}; 851 } 852 break; 853 } 854 }] in { 855 def : RVVBuiltin<"v", "vv", type_range>; 856 def : RVVBuiltin<"Uv", "UvUv", type_range>; 857 } 858} 859 860multiclass RVVPseudoMaskBuiltin<string IR, string type_range> { 861 let Name = NAME, 862 IRName = IR, 863 HasMasked = false, 864 ManualCodegen = [{ 865 { 866 // op1, vl 867 IntrinsicTypes = {ResultType, 868 Ops[1]->getType()}; 869 Ops.insert(Ops.begin() + 1, Ops[0]); 870 break; 871 } 872 }] in { 873 def : RVVBuiltin<"m", "mm", type_range>; 874 } 875} 876 877multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { 878 let Name = NAME, 879 IRName = IR, 880 MaskedIRName = IR # "_mask", 881 UnMaskedPolicyScheme = HasPassthruOperand, 882 ManualCodegen = [{ 883 { 884 if (IsMasked) { 885 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 886 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 887 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 888 Ops.insert(Ops.begin() + 2, Ops[1]); 889 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 890 // maskedoff, op1, op2, mask, vl 891 IntrinsicTypes = {ResultType, 892 Ops[2]->getType(), 893 Ops.back()->getType()}; 894 } else { 895 if (PolicyAttrs & RVV_VTA) 896 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 897 // op1, po2, vl 898 IntrinsicTypes = {ResultType, 899 Ops[1]->getType(), Ops[2]->getType()}; 900 Ops.insert(Ops.begin() + 2, Ops[1]); 901 break; 902 } 903 break; 904 } 905 }] in { 906 def : RVVBuiltin<"v", "vv", type_range>; 907 } 908} 909 910multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, 911 list<list<string>> suffixes_prototypes> { 912 let Name = NAME, 913 OverloadedName = MName, 914 IRName = IR, 915 MaskedIRName = IR # "_mask", 916 UnMaskedPolicyScheme = HasPassthruOperand, 917 ManualCodegen = [{ 918 { 919 if (IsMasked) { 920 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 921 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 922 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 923 } else { 924 if (PolicyAttrs & RVV_VTA) 925 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 926 } 927 auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); 928 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); 929 if (IsMasked) { 930 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 931 // maskedoff, op1, op2, mask, vl, policy 932 IntrinsicTypes = {ResultType, 933 Ops[1]->getType(), 934 ElemTy, 935 Ops[4]->getType()}; 936 } else { 937 // passtru, op1, op2, vl 938 IntrinsicTypes = {ResultType, 939 Ops[1]->getType(), 940 ElemTy, 941 Ops[3]->getType()}; 942 } 943 break; 944 } 945 }] in { 946 foreach s_p = suffixes_prototypes in { 947 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 948 } 949 } 950} 951 952multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, 953 list<list<string>> suffixes_prototypes> { 954 let Name = NAME, 955 OverloadedName = MName, 956 IRName = IR, 957 MaskedIRName = IR # "_mask", 958 UnMaskedPolicyScheme = HasPassthruOperand, 959 ManualCodegen = [{ 960 { 961 if (IsMasked) { 962 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 963 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 964 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 965 } else { 966 if (PolicyAttrs & RVV_VTA) 967 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 968 } 969 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType())); 970 if (IsMasked) { 971 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 972 // maskedoff, op1, xlen, mask, vl 973 IntrinsicTypes = {ResultType, 974 Ops[1]->getType(), 975 Ops[4]->getType(), 976 Ops[4]->getType()}; 977 } else { 978 // passthru, op1, xlen, vl 979 IntrinsicTypes = {ResultType, 980 Ops[1]->getType(), 981 Ops[3]->getType(), 982 Ops[3]->getType()}; 983 } 984 break; 985 } 986 }] in { 987 foreach s_p = suffixes_prototypes in { 988 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 989 } 990 } 991} 992 993let HeaderCode = 994[{ 995#define __riscv_vlenb() __builtin_rvv_vlenb() 996}] in 997def vlenb_macro: RVVHeader; 998 999let HasBuiltinAlias = false, HasVL = false, HasMasked = false, 1000 UnMaskedPolicyScheme = NonePolicy, MaskedPolicyScheme = NonePolicy, 1001 Log2LMUL = [0], IRName = "", 1002 ManualCodegen = [{ 1003 { 1004 LLVMContext &Context = CGM.getLLVMContext(); 1005 llvm::MDBuilder MDHelper(Context); 1006 1007 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "vlenb")}; 1008 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); 1009 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); 1010 llvm::Function *F = 1011 CGM.getIntrinsic(llvm::Intrinsic::read_register, {SizeTy}); 1012 return Builder.CreateCall(F, Metadata); 1013 } 1014 }] in 1015{ 1016 def vlenb : RVVBuiltin<"", "u", "i">; 1017} 1018 1019// 6. Configuration-Setting Instructions 1020// 6.1. vsetvli/vsetvl instructions 1021 1022// vsetvl/vsetvlmax are a macro because they require constant integers in SEW 1023// and LMUL. 1024let HeaderCode = 1025[{ 1026#define __riscv_vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6) 1027#define __riscv_vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7) 1028#define __riscv_vsetvl_e8m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 0) 1029#define __riscv_vsetvl_e8m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 1) 1030#define __riscv_vsetvl_e8m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 2) 1031#define __riscv_vsetvl_e8m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 3) 1032 1033#define __riscv_vsetvl_e16mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 7) 1034#define __riscv_vsetvl_e16m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 0) 1035#define __riscv_vsetvl_e16m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 1) 1036#define __riscv_vsetvl_e16m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 2) 1037#define __riscv_vsetvl_e16m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 3) 1038 1039#define __riscv_vsetvl_e32m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 0) 1040#define __riscv_vsetvl_e32m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 1) 1041#define __riscv_vsetvl_e32m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 2) 1042#define __riscv_vsetvl_e32m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 3) 1043 1044#if __riscv_v_elen >= 64 1045#define __riscv_vsetvl_e8mf8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 5) 1046#define __riscv_vsetvl_e16mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 6) 1047#define __riscv_vsetvl_e32mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 7) 1048 1049#define __riscv_vsetvl_e64m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 0) 1050#define __riscv_vsetvl_e64m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 1) 1051#define __riscv_vsetvl_e64m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 2) 1052#define __riscv_vsetvl_e64m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 3) 1053#endif 1054 1055#define __riscv_vsetvlmax_e8mf4() __builtin_rvv_vsetvlimax(0, 6) 1056#define __riscv_vsetvlmax_e8mf2() __builtin_rvv_vsetvlimax(0, 7) 1057#define __riscv_vsetvlmax_e8m1() __builtin_rvv_vsetvlimax(0, 0) 1058#define __riscv_vsetvlmax_e8m2() __builtin_rvv_vsetvlimax(0, 1) 1059#define __riscv_vsetvlmax_e8m4() __builtin_rvv_vsetvlimax(0, 2) 1060#define __riscv_vsetvlmax_e8m8() __builtin_rvv_vsetvlimax(0, 3) 1061 1062#define __riscv_vsetvlmax_e16mf2() __builtin_rvv_vsetvlimax(1, 7) 1063#define __riscv_vsetvlmax_e16m1() __builtin_rvv_vsetvlimax(1, 0) 1064#define __riscv_vsetvlmax_e16m2() __builtin_rvv_vsetvlimax(1, 1) 1065#define __riscv_vsetvlmax_e16m4() __builtin_rvv_vsetvlimax(1, 2) 1066#define __riscv_vsetvlmax_e16m8() __builtin_rvv_vsetvlimax(1, 3) 1067 1068#define __riscv_vsetvlmax_e32m1() __builtin_rvv_vsetvlimax(2, 0) 1069#define __riscv_vsetvlmax_e32m2() __builtin_rvv_vsetvlimax(2, 1) 1070#define __riscv_vsetvlmax_e32m4() __builtin_rvv_vsetvlimax(2, 2) 1071#define __riscv_vsetvlmax_e32m8() __builtin_rvv_vsetvlimax(2, 3) 1072 1073#if __riscv_v_elen >= 64 1074#define __riscv_vsetvlmax_e8mf8() __builtin_rvv_vsetvlimax(0, 5) 1075#define __riscv_vsetvlmax_e16mf4() __builtin_rvv_vsetvlimax(1, 6) 1076#define __riscv_vsetvlmax_e32mf2() __builtin_rvv_vsetvlimax(2, 7) 1077 1078#define __riscv_vsetvlmax_e64m1() __builtin_rvv_vsetvlimax(3, 0) 1079#define __riscv_vsetvlmax_e64m2() __builtin_rvv_vsetvlimax(3, 1) 1080#define __riscv_vsetvlmax_e64m4() __builtin_rvv_vsetvlimax(3, 2) 1081#define __riscv_vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3) 1082#endif 1083 1084}] in 1085def vsetvl_macro: RVVHeader; 1086 1087let HasBuiltinAlias = false, 1088 HasVL = false, 1089 HasMasked = false, 1090 MaskedPolicyScheme = NonePolicy, 1091 Log2LMUL = [0], 1092 ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type 1093{ 1094 def vsetvli : RVVBuiltin<"", "zzKzKz", "i">; 1095 def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">; 1096} 1097 1098// 7. Vector Loads and Stores 1099// 7.4. Vector Unit-Stride Instructions 1100def vlm: RVVVLEMaskBuiltin; 1101defm vle8: RVVVLEBuiltin<["c"]>; 1102defm vle16: RVVVLEBuiltin<["s","x"]>; 1103defm vle32: RVVVLEBuiltin<["i","f"]>; 1104defm vle64: RVVVLEBuiltin<["l","d"]>; 1105 1106def vsm : RVVVSEMaskBuiltin; 1107defm vse8 : RVVVSEBuiltin<["c"]>; 1108defm vse16: RVVVSEBuiltin<["s","x"]>; 1109defm vse32: RVVVSEBuiltin<["i","f"]>; 1110defm vse64: RVVVSEBuiltin<["l","d"]>; 1111 1112// 7.5. Vector Strided Instructions 1113defm vlse8: RVVVLSEBuiltin<["c"]>; 1114defm vlse16: RVVVLSEBuiltin<["s","x"]>; 1115defm vlse32: RVVVLSEBuiltin<["i","f"]>; 1116defm vlse64: RVVVLSEBuiltin<["l","d"]>; 1117 1118defm vsse8 : RVVVSSEBuiltin<["c"]>; 1119defm vsse16: RVVVSSEBuiltin<["s","x"]>; 1120defm vsse32: RVVVSSEBuiltin<["i","f"]>; 1121defm vsse64: RVVVSSEBuiltin<["l","d"]>; 1122 1123// 7.6. Vector Indexed Instructions 1124defm : RVVIndexedLoad<"vluxei">; 1125defm : RVVIndexedLoad<"vloxei">; 1126 1127defm : RVVIndexedStore<"vsuxei">; 1128defm : RVVIndexedStore<"vsoxei">; 1129 1130// 7.7. Unit-stride Fault-Only-First Loads 1131defm vle8ff: RVVVLEFFBuiltin<["c"]>; 1132defm vle16ff: RVVVLEFFBuiltin<["s","x"]>; 1133defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>; 1134defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>; 1135 1136multiclass RVVUnitStridedSegLoadTuple<string op> { 1137 foreach type = TypeList in { 1138 defvar eew = !cond(!eq(type, "c") : "8", 1139 !eq(type, "s") : "16", 1140 !eq(type, "i") : "32", 1141 !eq(type, "l") : "64", 1142 !eq(type, "x") : "16", 1143 !eq(type, "f") : "32", 1144 !eq(type, "d") : "64"); 1145 foreach nf = NFList in { 1146 let Name = op # nf # "e" # eew # "_v", 1147 IRName = op # nf, 1148 MaskedIRName = op # nf # "_mask", 1149 NF = nf, 1150 ManualCodegen = [{ 1151 { 1152 llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; 1153 IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; 1154 SmallVector<llvm::Value*, 12> Operands; 1155 1156 bool NoPassthru = 1157 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | 1158 (!IsMasked && (PolicyAttrs & RVV_VTA)); 1159 unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; 1160 1161 if (NoPassthru) { // Push poison into passthru 1162 Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); 1163 } else { // Push intrinsics operands into passthru 1164 llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; 1165 for (unsigned I = 0; I < NF; ++I) 1166 Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); 1167 } 1168 1169 Operands.push_back(Ops[Offset]); // Ptr 1170 if (IsMasked) 1171 Operands.push_back(Ops[0]); 1172 Operands.push_back(Ops[Offset + 1]); // VL 1173 if (IsMasked) 1174 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1175 1176 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1177 1178 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1179 if (ReturnValue.isNull()) 1180 return LoadValue; 1181 else 1182 return Builder.CreateStore(LoadValue, ReturnValue.getValue()); 1183 } 1184 }] in { 1185 defvar T = "(Tuple:" # nf # ")"; 1186 def : RVVBuiltin<T # "v", T # "vPCe", type>; 1187 if !not(IsFloat<type>.val) then { 1188 def : RVVBuiltin<T # "Uv", T # "UvPCUe", type>; 1189 } 1190 } 1191 } 1192 } 1193} 1194 1195multiclass RVVUnitStridedSegStoreTuple<string op> { 1196 foreach type = TypeList in { 1197 defvar eew = !cond(!eq(type, "c") : "8", 1198 !eq(type, "s") : "16", 1199 !eq(type, "i") : "32", 1200 !eq(type, "l") : "64", 1201 !eq(type, "x") : "16", 1202 !eq(type, "f") : "32", 1203 !eq(type, "d") : "64"); 1204 foreach nf = NFList in { 1205 let Name = op # nf # "e" # eew # "_v", 1206 IRName = op # nf, 1207 MaskedIRName = op # nf # "_mask", 1208 NF = nf, 1209 HasMaskedOffOperand = false, 1210 ManualCodegen = [{ 1211 { 1212 // Masked 1213 // Builtin: (mask, ptr, v_tuple, vl) 1214 // Intrinsic: (val0, val1, ..., ptr, mask, vl) 1215 // Unmasked 1216 // Builtin: (ptr, v_tuple, vl) 1217 // Intrinsic: (val0, val1, ..., ptr, vl) 1218 unsigned Offset = IsMasked ? 1 : 0; 1219 llvm::Value *VTupleOperand = Ops[Offset + 1]; 1220 1221 SmallVector<llvm::Value*, 12> Operands; 1222 for (unsigned I = 0; I < NF; ++I) { 1223 llvm::Value *V = Builder.CreateExtractValue(VTupleOperand, {I}); 1224 Operands.push_back(V); 1225 } 1226 Operands.push_back(Ops[Offset]); // Ptr 1227 if (IsMasked) 1228 Operands.push_back(Ops[0]); 1229 Operands.push_back(Ops[Offset + 2]); // VL 1230 1231 IntrinsicTypes = {Operands[0]->getType(), Operands.back()->getType()}; 1232 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1233 return Builder.CreateCall(F, Operands, ""); 1234 } 1235 }] in { 1236 defvar T = "(Tuple:" # nf # ")"; 1237 def : RVVBuiltin<T # "v", "0Pe" # T # "v", type>; 1238 if !not(IsFloat<type>.val) then { 1239 def : RVVBuiltin<T # "Uv", "0PUe" # T # "Uv", type>; 1240 } 1241 } 1242 } 1243 } 1244} 1245 1246multiclass RVVUnitStridedSegLoadFFTuple<string op> { 1247 foreach type = TypeList in { 1248 defvar eew = !cond(!eq(type, "c") : "8", 1249 !eq(type, "s") : "16", 1250 !eq(type, "i") : "32", 1251 !eq(type, "l") : "64", 1252 !eq(type, "x") : "16", 1253 !eq(type, "f") : "32", 1254 !eq(type, "d") : "64"); 1255 foreach nf = NFList in { 1256 let Name = op # nf # "e" # eew # "ff_v", 1257 IRName = op # nf # "ff", 1258 MaskedIRName = op # nf # "ff_mask", 1259 NF = nf, 1260 ManualCodegen = [{ 1261 { 1262 llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; 1263 IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; 1264 SmallVector<llvm::Value*, 12> Operands; 1265 1266 bool NoPassthru = 1267 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | 1268 (!IsMasked && (PolicyAttrs & RVV_VTA)); 1269 unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; 1270 1271 if (NoPassthru) { // Push poison into passthru 1272 Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); 1273 } else { // Push intrinsics operands into passthru 1274 llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; 1275 for (unsigned I = 0; I < NF; ++I) 1276 Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); 1277 } 1278 1279 Operands.push_back(Ops[Offset]); // Ptr 1280 if (IsMasked) 1281 Operands.push_back(Ops[0]); 1282 Operands.push_back(Ops[Offset + 2]); // vl 1283 if (IsMasked) 1284 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1285 1286 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1287 1288 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1289 // Get alignment from the new vl operand 1290 clang::CharUnits Align = 1291 CGM.getNaturalPointeeTypeAlignment(E->getArg(Offset + 1)->getType()); 1292 1293 llvm::Value *ReturnTuple = llvm::PoisonValue::get(ResultType); 1294 for (unsigned I = 0; I < NF; ++I) { 1295 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {I}); 1296 ReturnTuple = Builder.CreateInsertValue(ReturnTuple, V, {I}); 1297 } 1298 1299 // Store new_vl 1300 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {NF}); 1301 Builder.CreateStore(V, Address(Ops[Offset + 1], V->getType(), Align)); 1302 1303 if (ReturnValue.isNull()) 1304 return ReturnTuple; 1305 else 1306 return Builder.CreateStore(ReturnTuple, ReturnValue.getValue()); 1307 } 1308 }] in { 1309 defvar T = "(Tuple:" # nf # ")"; 1310 def : RVVBuiltin<T # "v", T # "vPCePz", type>; 1311 if !not(IsFloat<type>.val) then { 1312 def : RVVBuiltin<T # "Uv", T # "UvPCUePz", type>; 1313 } 1314 } 1315 } 1316 } 1317} 1318 1319multiclass RVVStridedSegLoadTuple<string op> { 1320 foreach type = TypeList in { 1321 defvar eew = !cond(!eq(type, "c") : "8", 1322 !eq(type, "s") : "16", 1323 !eq(type, "i") : "32", 1324 !eq(type, "l") : "64", 1325 !eq(type, "x") : "16", 1326 !eq(type, "f") : "32", 1327 !eq(type, "d") : "64"); 1328 foreach nf = NFList in { 1329 let Name = op # nf # "e" # eew # "_v", 1330 IRName = op # nf, 1331 MaskedIRName = op # nf # "_mask", 1332 NF = nf, 1333 ManualCodegen = [{ 1334 { 1335 llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; 1336 IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; 1337 SmallVector<llvm::Value*, 12> Operands; 1338 1339 bool NoPassthru = 1340 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | 1341 (!IsMasked && (PolicyAttrs & RVV_VTA)); 1342 unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; 1343 1344 if (NoPassthru) { // Push poison into passthru 1345 Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); 1346 } else { // Push intrinsics operands into passthru 1347 llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; 1348 for (unsigned I = 0; I < NF; ++I) 1349 Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); 1350 } 1351 1352 Operands.push_back(Ops[Offset]); // Ptr 1353 Operands.push_back(Ops[Offset + 1]); // Stride 1354 if (IsMasked) 1355 Operands.push_back(Ops[0]); 1356 Operands.push_back(Ops[Offset + 2]); // VL 1357 if (IsMasked) 1358 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1359 1360 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1361 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1362 1363 if (ReturnValue.isNull()) 1364 return LoadValue; 1365 else 1366 return Builder.CreateStore(LoadValue, ReturnValue.getValue()); 1367 } 1368 }] in { 1369 defvar T = "(Tuple:" # nf # ")"; 1370 def : RVVBuiltin<T # "v", T # "vPCet", type>; 1371 if !not(IsFloat<type>.val) then { 1372 def : RVVBuiltin<T # "Uv", T # "UvPCUet", type>; 1373 } 1374 } 1375 } 1376 } 1377} 1378 1379multiclass RVVStridedSegStoreTuple<string op> { 1380 foreach type = TypeList in { 1381 defvar eew = !cond(!eq(type, "c") : "8", 1382 !eq(type, "s") : "16", 1383 !eq(type, "i") : "32", 1384 !eq(type, "l") : "64", 1385 !eq(type, "x") : "16", 1386 !eq(type, "f") : "32", 1387 !eq(type, "d") : "64"); 1388 foreach nf = NFList in { 1389 let Name = op # nf # "e" # eew # "_v", 1390 IRName = op # nf, 1391 MaskedIRName = op # nf # "_mask", 1392 NF = nf, 1393 HasMaskedOffOperand = false, 1394 MaskedPolicyScheme = NonePolicy, 1395 ManualCodegen = [{ 1396 { 1397 // Masked 1398 // Builtin: (mask, ptr, stride, v_tuple, vl) 1399 // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl) 1400 // Unmasked 1401 // Builtin: (ptr, stride, v_tuple, vl) 1402 // Intrinsic: (val0, val1, ..., ptr, stride, vl) 1403 unsigned Offset = IsMasked ? 1 : 0; 1404 llvm::Value *VTupleOperand = Ops[Offset + 2]; 1405 1406 SmallVector<llvm::Value*, 12> Operands; 1407 for (unsigned I = 0; I < NF; ++I) { 1408 llvm::Value *V = Builder.CreateExtractValue(VTupleOperand, {I}); 1409 Operands.push_back(V); 1410 } 1411 Operands.push_back(Ops[Offset]); // Ptr 1412 Operands.push_back(Ops[Offset + 1]); // Stride 1413 if (IsMasked) 1414 Operands.push_back(Ops[0]); 1415 Operands.push_back(Ops[Offset + 3]); // VL 1416 1417 IntrinsicTypes = {Operands[0]->getType(), Operands.back()->getType()}; 1418 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1419 return Builder.CreateCall(F, Operands, ""); 1420 } 1421 }] in { 1422 defvar T = "(Tuple:" # nf # ")"; 1423 def : RVVBuiltin<T # "v", "0Pet" # T # "v", type>; 1424 if !not(IsFloat<type>.val) then { 1425 def : RVVBuiltin<T # "Uv", "0PUet" # T # "Uv", type>; 1426 } 1427 } 1428 } 1429 } 1430} 1431 1432multiclass RVVIndexedSegLoadTuple<string op> { 1433 foreach type = TypeList in { 1434 foreach eew_info = EEWList in { 1435 defvar eew = eew_info[0]; 1436 defvar eew_type = eew_info[1]; 1437 foreach nf = NFList in { 1438 let Name = op # nf # "ei" # eew # "_v", 1439 IRName = op # nf, 1440 MaskedIRName = op # nf # "_mask", 1441 NF = nf, 1442 ManualCodegen = [{ 1443 { 1444 llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; 1445 IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; 1446 SmallVector<llvm::Value*, 12> Operands; 1447 1448 bool NoPassthru = 1449 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | 1450 (!IsMasked && (PolicyAttrs & RVV_VTA)); 1451 unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; 1452 1453 if (NoPassthru) { // Push poison into passthru 1454 Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); 1455 } else { // Push intrinsics operands into passthru 1456 llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; 1457 for (unsigned I = 0; I < NF; ++I) 1458 Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); 1459 } 1460 1461 Operands.push_back(Ops[Offset]); // Ptr 1462 Operands.push_back(Ops[Offset + 1]); // Idx 1463 if (IsMasked) 1464 Operands.push_back(Ops[0]); 1465 Operands.push_back(Ops[Offset + 2]); // VL 1466 if (IsMasked) 1467 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1468 1469 IntrinsicTypes = {ElementVectorType, Ops[Offset + 1]->getType(), 1470 Ops.back()->getType()}; 1471 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1472 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1473 1474 if (ReturnValue.isNull()) 1475 return LoadValue; 1476 else 1477 return Builder.CreateStore(LoadValue, ReturnValue.getValue()); 1478 } 1479 }] in { 1480 defvar T = "(Tuple:" # nf # ")"; 1481 def : RVVBuiltin<T # "v", T # "vPCe" # eew_type # "Uv", type>; 1482 if !not(IsFloat<type>.val) then { 1483 def : RVVBuiltin<T # "Uv", T # "UvPCUe" # eew_type # "Uv", type>; 1484 } 1485 } 1486 } 1487 } 1488 } 1489} 1490 1491multiclass RVVIndexedSegStoreTuple<string op> { 1492 foreach type = TypeList in { 1493 foreach eew_info = EEWList in { 1494 defvar eew = eew_info[0]; 1495 defvar eew_type = eew_info[1]; 1496 foreach nf = NFList in { 1497 let Name = op # nf # "ei" # eew # "_v", 1498 IRName = op # nf, 1499 MaskedIRName = op # nf # "_mask", 1500 NF = nf, 1501 HasMaskedOffOperand = false, 1502 MaskedPolicyScheme = NonePolicy, 1503 ManualCodegen = [{ 1504 { 1505 // Masked 1506 // Builtin: (mask, ptr, index, v_tuple, vl) 1507 // Intrinsic: (val0, val1, ..., ptr, index, mask, vl) 1508 // Unmasked 1509 // Builtin: (ptr, index, v_tuple, vl) 1510 // Intrinsic: (val0, val1, ..., ptr, index, vl) 1511 unsigned Offset = IsMasked ? 1 : 0; 1512 llvm::Value *VTupleOperand = Ops[Offset + 2]; 1513 1514 SmallVector<llvm::Value*, 12> Operands; 1515 for (unsigned I = 0; I < NF; ++I) { 1516 llvm::Value *V = Builder.CreateExtractValue(VTupleOperand, {I}); 1517 Operands.push_back(V); 1518 } 1519 Operands.push_back(Ops[Offset]); // Ptr 1520 Operands.push_back(Ops[Offset + 1]); // Idx 1521 if (IsMasked) 1522 Operands.push_back(Ops[0]); 1523 Operands.push_back(Ops[Offset + 3]); // VL 1524 1525 IntrinsicTypes = {Operands[0]->getType(), Ops[Offset + 1]->getType(), 1526 Operands.back()->getType()}; 1527 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1528 return Builder.CreateCall(F, Operands, ""); 1529 } 1530 }] in { 1531 defvar T = "(Tuple:" # nf # ")"; 1532 def : RVVBuiltin<T # "v", "0Pe" # eew_type # "Uv" # T # "v", type>; 1533 if !not(IsFloat<type>.val) then { 1534 def : RVVBuiltin<T # "Uv", "0PUe" # eew_type # "Uv" # T # "Uv", type>; 1535 } 1536 } 1537 } 1538 } 1539 } 1540} 1541 1542// 7.8 Vector Load/Store Segment Instructions 1543let UnMaskedPolicyScheme = HasPassthruOperand, 1544 IsTuple = true in { 1545 defm : RVVUnitStridedSegLoadTuple<"vlseg">; 1546 defm : RVVUnitStridedSegLoadFFTuple<"vlseg">; 1547 defm : RVVStridedSegLoadTuple<"vlsseg">; 1548 defm : RVVIndexedSegLoadTuple<"vluxseg">; 1549 defm : RVVIndexedSegLoadTuple<"vloxseg">; 1550} 1551 1552let UnMaskedPolicyScheme = NonePolicy, 1553 MaskedPolicyScheme = NonePolicy, 1554 IsTuple = true in { 1555defm : RVVUnitStridedSegStoreTuple<"vsseg">; 1556defm : RVVStridedSegStoreTuple<"vssseg">; 1557defm : RVVIndexedSegStoreTuple<"vsuxseg">; 1558defm : RVVIndexedSegStoreTuple<"vsoxseg">; 1559} 1560 1561// 12. Vector Integer Arithmetic Instructions 1562// 12.1. Vector Single-Width Integer Add and Subtract 1563let UnMaskedPolicyScheme = HasPassthruOperand in { 1564defm vadd : RVVIntBinBuiltinSet; 1565defm vsub : RVVIntBinBuiltinSet; 1566defm vrsub : RVVOutOp1BuiltinSet<"vrsub", "csil", 1567 [["vx", "v", "vve"], 1568 ["vx", "Uv", "UvUvUe"]]>; 1569} 1570defm vneg_v : RVVPseudoUnaryBuiltin<"vrsub", "csil">; 1571 1572// 12.2. Vector Widening Integer Add/Subtract 1573// Widening unsigned integer add/subtract, 2*SEW = SEW +/- SEW 1574let UnMaskedPolicyScheme = HasPassthruOperand in { 1575defm vwaddu : RVVUnsignedWidenBinBuiltinSet; 1576defm vwsubu : RVVUnsignedWidenBinBuiltinSet; 1577// Widening signed integer add/subtract, 2*SEW = SEW +/- SEW 1578defm vwadd : RVVSignedWidenBinBuiltinSet; 1579defm vwsub : RVVSignedWidenBinBuiltinSet; 1580// Widening unsigned integer add/subtract, 2*SEW = 2*SEW +/- SEW 1581defm vwaddu : RVVUnsignedWidenOp0BinBuiltinSet; 1582defm vwsubu : RVVUnsignedWidenOp0BinBuiltinSet; 1583// Widening signed integer add/subtract, 2*SEW = 2*SEW +/- SEW 1584defm vwadd : RVVSignedWidenOp0BinBuiltinSet; 1585defm vwsub : RVVSignedWidenOp0BinBuiltinSet; 1586} 1587defm vwcvtu_x_x_v : RVVPseudoVWCVTBuiltin<"vwaddu", "vwcvtu_x", "csi", 1588 [["Uw", "UwUv"]]>; 1589defm vwcvt_x_x_v : RVVPseudoVWCVTBuiltin<"vwadd", "vwcvt_x", "csi", 1590 [["w", "wv"]]>; 1591 1592// 12.3. Vector Integer Extension 1593let UnMaskedPolicyScheme = HasPassthruOperand in { 1594let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1595 def vsext_vf2 : RVVIntExt<"vsext", "w", "wv", "csi">; 1596 def vzext_vf2 : RVVIntExt<"vzext", "Uw", "UwUv", "csi">; 1597} 1598let Log2LMUL = [-3, -2, -1, 0, 1] in { 1599 def vsext_vf4 : RVVIntExt<"vsext", "q", "qv", "cs">; 1600 def vzext_vf4 : RVVIntExt<"vzext", "Uq", "UqUv", "cs">; 1601} 1602let Log2LMUL = [-3, -2, -1, 0] in { 1603 def vsext_vf8 : RVVIntExt<"vsext", "o", "ov", "c">; 1604 def vzext_vf8 : RVVIntExt<"vzext", "Uo", "UoUv", "c">; 1605} 1606} 1607 1608// 12.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions 1609let HasMasked = false, MaskedPolicyScheme = NonePolicy in { 1610 let UnMaskedPolicyScheme = HasPassthruOperand in { 1611 defm vadc : RVVCarryinBuiltinSet; 1612 defm vsbc : RVVCarryinBuiltinSet; 1613 } 1614 defm vmadc : RVVCarryOutInBuiltinSet<"vmadc_carry_in">; 1615 defm vmadc : RVVIntMaskOutBuiltinSet; 1616 defm vmsbc : RVVCarryOutInBuiltinSet<"vmsbc_borrow_in">; 1617 defm vmsbc : RVVIntMaskOutBuiltinSet; 1618} 1619 1620// 12.5. Vector Bitwise Logical Instructions 1621let UnMaskedPolicyScheme = HasPassthruOperand in { 1622defm vand : RVVIntBinBuiltinSet; 1623defm vxor : RVVIntBinBuiltinSet; 1624defm vor : RVVIntBinBuiltinSet; 1625} 1626defm vnot_v : RVVPseudoVNotBuiltin<"vxor", "csil">; 1627 1628// 12.6. Vector Single-Width Bit Shift Instructions 1629let UnMaskedPolicyScheme = HasPassthruOperand in { 1630defm vsll : RVVShiftBuiltinSet; 1631defm vsrl : RVVUnsignedShiftBuiltinSet; 1632defm vsra : RVVSignedShiftBuiltinSet; 1633 1634// 12.7. Vector Narrowing Integer Right Shift Instructions 1635defm vnsrl : RVVUnsignedNShiftBuiltinSet; 1636defm vnsra : RVVSignedNShiftBuiltinSet; 1637} 1638defm vncvt_x_x_w : RVVPseudoVNCVTBuiltin<"vnsrl", "vncvt_x", "csi", 1639 [["v", "vw"], 1640 ["Uv", "UvUw"]]>; 1641 1642// 12.8. Vector Integer Comparison Instructions 1643let MaskedPolicyScheme = HasPassthruOperand, 1644 HasTailPolicy = false in { 1645defm vmseq : RVVIntMaskOutBuiltinSet; 1646defm vmsne : RVVIntMaskOutBuiltinSet; 1647defm vmsltu : RVVUnsignedMaskOutBuiltinSet; 1648defm vmslt : RVVSignedMaskOutBuiltinSet; 1649defm vmsleu : RVVUnsignedMaskOutBuiltinSet; 1650defm vmsle : RVVSignedMaskOutBuiltinSet; 1651defm vmsgtu : RVVUnsignedMaskOutBuiltinSet; 1652defm vmsgt : RVVSignedMaskOutBuiltinSet; 1653defm vmsgeu : RVVUnsignedMaskOutBuiltinSet; 1654defm vmsge : RVVSignedMaskOutBuiltinSet; 1655} 1656 1657// 12.9. Vector Integer Min/Max Instructions 1658let UnMaskedPolicyScheme = HasPassthruOperand in { 1659defm vminu : RVVUnsignedBinBuiltinSet; 1660defm vmin : RVVSignedBinBuiltinSet; 1661defm vmaxu : RVVUnsignedBinBuiltinSet; 1662defm vmax : RVVSignedBinBuiltinSet; 1663 1664// 12.10. Vector Single-Width Integer Multiply Instructions 1665defm vmul : RVVIntBinBuiltinSet; 1666defm vmulh : RVVSignedBinBuiltinSet; 1667defm vmulhu : RVVUnsignedBinBuiltinSet; 1668defm vmulhsu : RVVOutOp1BuiltinSet<"vmulhsu", "csil", 1669 [["vv", "v", "vvUv"], 1670 ["vx", "v", "vvUe"]]>; 1671 1672// 12.11. Vector Integer Divide Instructions 1673defm vdivu : RVVUnsignedBinBuiltinSet; 1674defm vdiv : RVVSignedBinBuiltinSet; 1675defm vremu : RVVUnsignedBinBuiltinSet; 1676defm vrem : RVVSignedBinBuiltinSet; 1677} 1678 1679// 12.12. Vector Widening Integer Multiply Instructions 1680let Log2LMUL = [-3, -2, -1, 0, 1, 2], UnMaskedPolicyScheme = HasPassthruOperand in { 1681defm vwmul : RVVOutOp0Op1BuiltinSet<"vwmul", "csi", 1682 [["vv", "w", "wvv"], 1683 ["vx", "w", "wve"]]>; 1684defm vwmulu : RVVOutOp0Op1BuiltinSet<"vwmulu", "csi", 1685 [["vv", "Uw", "UwUvUv"], 1686 ["vx", "Uw", "UwUvUe"]]>; 1687defm vwmulsu : RVVOutOp0Op1BuiltinSet<"vwmulsu", "csi", 1688 [["vv", "w", "wvUv"], 1689 ["vx", "w", "wvUe"]]>; 1690} 1691 1692// 12.13. Vector Single-Width Integer Multiply-Add Instructions 1693let UnMaskedPolicyScheme = HasPolicyOperand in { 1694defm vmacc : RVVIntTerBuiltinSet; 1695defm vnmsac : RVVIntTerBuiltinSet; 1696defm vmadd : RVVIntTerBuiltinSet; 1697defm vnmsub : RVVIntTerBuiltinSet; 1698 1699// 12.14. Vector Widening Integer Multiply-Add Instructions 1700let HasMaskedOffOperand = false, 1701 Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1702defm vwmaccu : RVVOutOp1Op2BuiltinSet<"vwmaccu", "csi", 1703 [["vv", "Uw", "UwUwUvUv"], 1704 ["vx", "Uw", "UwUwUeUv"]]>; 1705defm vwmacc : RVVOutOp1Op2BuiltinSet<"vwmacc", "csi", 1706 [["vv", "w", "wwvv"], 1707 ["vx", "w", "wwev"]]>; 1708defm vwmaccsu : RVVOutOp1Op2BuiltinSet<"vwmaccsu", "csi", 1709 [["vv", "w", "wwvUv"], 1710 ["vx", "w", "wweUv"]]>; 1711defm vwmaccus : RVVOutOp1Op2BuiltinSet<"vwmaccus", "csi", 1712 [["vx", "w", "wwUev"]]>; 1713} 1714} 1715 1716// 12.15. Vector Integer Merge Instructions 1717// C/C++ Operand: (mask, op1, op2, vl), Intrinsic: (passthru, op1, op2, mask, vl) 1718let HasMasked = false, 1719 UnMaskedPolicyScheme = HasPassthruOperand, 1720 MaskedPolicyScheme = NonePolicy, 1721 ManualCodegen = [{ 1722 // insert poison passthru 1723 if (PolicyAttrs & RVV_VTA) 1724 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1725 IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; 1726 }] in { 1727 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "csil", 1728 [["vvm", "v", "vvvm"], 1729 ["vxm", "v", "vvem"], 1730 ["vvm", "Uv", "UvUvUvm"], 1731 ["vxm", "Uv", "UvUvUem"]]>; 1732} 1733 1734// 12.16. Vector Integer Move Instructions 1735let HasMasked = false, 1736 UnMaskedPolicyScheme = HasPassthruOperand, 1737 MaskedPolicyScheme = NonePolicy, 1738 OverloadedName = "vmv_v" in { 1739 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csil", 1740 [["v", "Uv", "UvUv"]]>; 1741 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilxfd", 1742 [["v", "v", "vv"]]>; 1743 let SupportOverloading = false in 1744 defm vmv_v : RVVOutBuiltinSet<"vmv_v_x", "csil", 1745 [["x", "v", "ve"], 1746 ["x", "Uv", "UvUe"]]>; 1747} 1748 1749// 13. Vector Fixed-Point Arithmetic Instructions 1750let HeaderCode = 1751[{ 1752enum __RISCV_VXRM { 1753 __RISCV_VXRM_RNU = 0, 1754 __RISCV_VXRM_RNE = 1, 1755 __RISCV_VXRM_RDN = 2, 1756 __RISCV_VXRM_ROD = 3, 1757}; 1758}] in 1759def vxrm_enum : RVVHeader; 1760 1761// 13.1. Vector Single-Width Saturating Add and Subtract 1762let UnMaskedPolicyScheme = HasPassthruOperand in { 1763defm vsaddu : RVVUnsignedBinBuiltinSet; 1764defm vsadd : RVVSignedBinBuiltinSet; 1765defm vssubu : RVVUnsignedBinBuiltinSet; 1766defm vssub : RVVSignedBinBuiltinSet; 1767 1768let ManualCodegen = [{ 1769 { 1770 // LLVM intrinsic 1771 // Unmasked: (passthru, op0, op1, round_mode, vl) 1772 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy) 1773 1774 SmallVector<llvm::Value*, 7> Operands; 1775 bool HasMaskedOff = !( 1776 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1777 (!IsMasked && PolicyAttrs & RVV_VTA)); 1778 unsigned Offset = IsMasked ? 1779 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1780 1781 if (!HasMaskedOff) 1782 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1783 else 1784 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1785 1786 Operands.push_back(Ops[Offset]); // op0 1787 Operands.push_back(Ops[Offset + 1]); // op1 1788 1789 if (IsMasked) 1790 Operands.push_back(Ops[0]); // mask 1791 1792 Operands.push_back(Ops[Offset + 2]); // vxrm 1793 Operands.push_back(Ops[Offset + 3]); // vl 1794 1795 if (IsMasked) 1796 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1797 1798 IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(), Ops.back()->getType()}; 1799 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1800 return Builder.CreateCall(F, Operands, ""); 1801 } 1802}] in { 1803 // 13.2. Vector Single-Width Averaging Add and Subtract 1804 defm vaaddu : RVVUnsignedBinBuiltinSetRoundingMode; 1805 defm vaadd : RVVSignedBinBuiltinSetRoundingMode; 1806 defm vasubu : RVVUnsignedBinBuiltinSetRoundingMode; 1807 defm vasub : RVVSignedBinBuiltinSetRoundingMode; 1808 1809 // 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation 1810 defm vsmul : RVVSignedBinBuiltinSetRoundingMode; 1811 1812 // 13.4. Vector Single-Width Scaling Shift Instructions 1813 defm vssrl : RVVUnsignedShiftBuiltinSetRoundingMode; 1814 defm vssra : RVVSignedShiftBuiltinSetRoundingMode; 1815} 1816 1817let ManualCodegen = [{ 1818 { 1819 // LLVM intrinsic 1820 // Unmasked: (passthru, op0, op1, round_mode, vl) 1821 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy) 1822 1823 SmallVector<llvm::Value*, 7> Operands; 1824 bool HasMaskedOff = !( 1825 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1826 (!IsMasked && PolicyAttrs & RVV_VTA)); 1827 unsigned Offset = IsMasked ? 1828 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1829 1830 if (!HasMaskedOff) 1831 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1832 else 1833 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1834 1835 Operands.push_back(Ops[Offset]); // op0 1836 Operands.push_back(Ops[Offset + 1]); // op1 1837 1838 if (IsMasked) 1839 Operands.push_back(Ops[0]); // mask 1840 1841 Operands.push_back(Ops[Offset + 2]); // vxrm 1842 Operands.push_back(Ops[Offset + 3]); // vl 1843 1844 if (IsMasked) 1845 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1846 1847 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), 1848 Ops.back()->getType()}; 1849 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1850 return Builder.CreateCall(F, Operands, ""); 1851 } 1852}] in { 1853 // 13.5. Vector Narrowing Fixed-Point Clip Instructions 1854 defm vnclipu : RVVUnsignedNShiftBuiltinSetRoundingMode; 1855 defm vnclip : RVVSignedNShiftBuiltinSetRoundingMode; 1856} 1857} 1858 1859// 14. Vector Floating-Point Instructions 1860let HeaderCode = 1861[{ 1862enum __RISCV_FRM { 1863 __RISCV_FRM_RNE = 0, 1864 __RISCV_FRM_RTZ = 1, 1865 __RISCV_FRM_RDN = 2, 1866 __RISCV_FRM_RUP = 3, 1867 __RISCV_FRM_RMM = 4, 1868}; 1869}] in def frm_enum : RVVHeader; 1870 1871let UnMaskedPolicyScheme = HasPassthruOperand in { 1872let ManualCodegen = [{ 1873 { 1874 // LLVM intrinsic 1875 // Unmasked: (passthru, op0, op1, round_mode, vl) 1876 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 1877 1878 SmallVector<llvm::Value*, 7> Operands; 1879 bool HasMaskedOff = !( 1880 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1881 (!IsMasked && PolicyAttrs & RVV_VTA)); 1882 bool HasRoundModeOp = IsMasked ? 1883 (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : 1884 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); 1885 1886 unsigned Offset = IsMasked ? 1887 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1888 1889 if (!HasMaskedOff) 1890 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1891 else 1892 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1893 1894 Operands.push_back(Ops[Offset]); // op0 1895 Operands.push_back(Ops[Offset + 1]); // op1 1896 1897 if (IsMasked) 1898 Operands.push_back(Ops[0]); // mask 1899 1900 if (HasRoundModeOp) { 1901 Operands.push_back(Ops[Offset + 2]); // frm 1902 Operands.push_back(Ops[Offset + 3]); // vl 1903 } else { 1904 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 1905 Operands.push_back(Ops[Offset + 2]); // vl 1906 } 1907 1908 if (IsMasked) 1909 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1910 1911 IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(), 1912 Operands.back()->getType()}; 1913 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1914 return Builder.CreateCall(F, Operands, ""); 1915 } 1916}] in { 1917 let HasFRMRoundModeOp = true in { 1918 // 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions 1919 defm vfadd : RVVFloatingBinBuiltinSetRoundingMode; 1920 defm vfsub : RVVFloatingBinBuiltinSetRoundingMode; 1921 defm vfrsub : RVVFloatingBinVFBuiltinSetRoundingMode; 1922 1923 // 14.3. Vector Widening Floating-Point Add/Subtract Instructions 1924 // Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW 1925 defm vfwadd : RVVFloatingWidenOp0BinBuiltinSetRoundingMode; 1926 defm vfwsub : RVVFloatingWidenOp0BinBuiltinSetRoundingMode; 1927 1928 // 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions 1929 defm vfmul : RVVFloatingBinBuiltinSetRoundingMode; 1930 defm vfdiv : RVVFloatingBinBuiltinSetRoundingMode; 1931 defm vfrdiv : RVVFloatingBinVFBuiltinSetRoundingMode; 1932 } 1933 // 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions 1934 defm vfadd : RVVFloatingBinBuiltinSet; 1935 defm vfsub : RVVFloatingBinBuiltinSet; 1936 defm vfrsub : RVVFloatingBinVFBuiltinSet; 1937 1938 // 14.3. Vector Widening Floating-Point Add/Subtract Instructions 1939 // Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW 1940 defm vfwadd : RVVFloatingWidenOp0BinBuiltinSet; 1941 defm vfwsub : RVVFloatingWidenOp0BinBuiltinSet; 1942 1943 // 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions 1944 defm vfmul : RVVFloatingBinBuiltinSet; 1945 defm vfdiv : RVVFloatingBinBuiltinSet; 1946 defm vfrdiv : RVVFloatingBinVFBuiltinSet; 1947} 1948 1949let ManualCodegen = [{ 1950 { 1951 // LLVM intrinsic 1952 // Unmasked: (passthru, op0, op1, round_mode, vl) 1953 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 1954 1955 SmallVector<llvm::Value*, 7> Operands; 1956 bool HasMaskedOff = !( 1957 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1958 (!IsMasked && PolicyAttrs & RVV_VTA)); 1959 bool HasRoundModeOp = IsMasked ? 1960 (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : 1961 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); 1962 1963 unsigned Offset = IsMasked ? 1964 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1965 1966 if (!HasMaskedOff) 1967 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1968 else 1969 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1970 1971 Operands.push_back(Ops[Offset]); // op0 1972 Operands.push_back(Ops[Offset + 1]); // op1 1973 1974 if (IsMasked) 1975 Operands.push_back(Ops[0]); // mask 1976 1977 if (HasRoundModeOp) { 1978 Operands.push_back(Ops[Offset + 2]); // frm 1979 Operands.push_back(Ops[Offset + 3]); // vl 1980 } else { 1981 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 1982 Operands.push_back(Ops[Offset + 2]); // vl 1983 } 1984 1985 if (IsMasked) 1986 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1987 1988 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), 1989 Ops.back()->getType()}; 1990 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1991 return Builder.CreateCall(F, Operands, ""); 1992 } 1993}] in { 1994 let HasFRMRoundModeOp = true in { 1995 // 14.3. Vector Widening Floating-Point Add/Subtract Instructions 1996 // Widening FP add/subtract, 2*SEW = SEW +/- SEW 1997 defm vfwadd : RVVFloatingWidenBinBuiltinSetRoundingMode; 1998 defm vfwsub : RVVFloatingWidenBinBuiltinSetRoundingMode; 1999 2000 // 14.5. Vector Widening Floating-Point Multiply 2001 let Log2LMUL = [-2, -1, 0, 1, 2] in { 2002 defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", 2003 [["vv", "w", "wvvu"], 2004 ["vf", "w", "wveu"]]>; 2005 } 2006 } 2007 // 14.3. Vector Widening Floating-Point Add/Subtract Instructions 2008 // Widening FP add/subtract, 2*SEW = SEW +/- SEW 2009 defm vfwadd : RVVFloatingWidenBinBuiltinSet; 2010 defm vfwsub : RVVFloatingWidenBinBuiltinSet; 2011 2012 // 14.5. Vector Widening Floating-Point Multiply 2013 let Log2LMUL = [-2, -1, 0, 1, 2] in { 2014 defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", 2015 [["vv", "w", "wvv"], 2016 ["vf", "w", "wve"]]>; 2017 } 2018} 2019} 2020 2021 2022let UnMaskedPolicyScheme = HasPolicyOperand in { 2023let ManualCodegen = [{ 2024 { 2025 // LLVM intrinsic 2026 // Unmasked: (passthru, op0, op1, round_mode, vl) 2027 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 2028 2029 SmallVector<llvm::Value*, 7> Operands; 2030 bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; 2031 2032 unsigned Offset = IsMasked ? 2 : 1; 2033 2034 Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough 2035 2036 Operands.push_back(Ops[Offset]); // op0 2037 Operands.push_back(Ops[Offset + 1]); // op1 2038 2039 if (IsMasked) 2040 Operands.push_back(Ops[0]); // mask 2041 2042 if (HasRoundModeOp) { 2043 Operands.push_back(Ops[Offset + 2]); // frm 2044 Operands.push_back(Ops[Offset + 3]); // vl 2045 } else { 2046 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 2047 Operands.push_back(Ops[Offset + 2]); // vl 2048 } 2049 2050 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 2051 2052 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), 2053 Operands.back()->getType()}; 2054 2055 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 2056 2057 return Builder.CreateCall(F, Operands, ""); 2058 } 2059}] in { 2060 let HasFRMRoundModeOp = 1 in { 2061 // 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions 2062 defm vfmacc : RVVFloatingTerBuiltinSetRoundingMode; 2063 defm vfnmacc : RVVFloatingTerBuiltinSetRoundingMode; 2064 defm vfmsac : RVVFloatingTerBuiltinSetRoundingMode; 2065 defm vfnmsac : RVVFloatingTerBuiltinSetRoundingMode; 2066 defm vfmadd : RVVFloatingTerBuiltinSetRoundingMode; 2067 defm vfnmadd : RVVFloatingTerBuiltinSetRoundingMode; 2068 defm vfmsub : RVVFloatingTerBuiltinSetRoundingMode; 2069 defm vfnmsub : RVVFloatingTerBuiltinSetRoundingMode; 2070 } 2071 // 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions 2072 defm vfmacc : RVVFloatingTerBuiltinSet; 2073 defm vfnmacc : RVVFloatingTerBuiltinSet; 2074 defm vfmsac : RVVFloatingTerBuiltinSet; 2075 defm vfnmsac : RVVFloatingTerBuiltinSet; 2076 defm vfmadd : RVVFloatingTerBuiltinSet; 2077 defm vfnmadd : RVVFloatingTerBuiltinSet; 2078 defm vfmsub : RVVFloatingTerBuiltinSet; 2079 defm vfnmsub : RVVFloatingTerBuiltinSet; 2080} 2081 2082let ManualCodegen = [{ 2083 { 2084 // LLVM intrinsic 2085 // Unmasked: (passthru, op0, op1, round_mode, vl) 2086 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 2087 2088 SmallVector<llvm::Value*, 7> Operands; 2089 bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; 2090 2091 unsigned Offset = IsMasked ? 2 : 1; 2092 2093 Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough 2094 2095 Operands.push_back(Ops[Offset]); // op0 2096 Operands.push_back(Ops[Offset + 1]); // op1 2097 2098 if (IsMasked) 2099 Operands.push_back(Ops[0]); // mask 2100 2101 if (HasRoundModeOp) { 2102 Operands.push_back(Ops[Offset + 2]); // frm 2103 Operands.push_back(Ops[Offset + 3]); // vl 2104 } else { 2105 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 2106 Operands.push_back(Ops[Offset + 2]); // vl 2107 } 2108 2109 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 2110 2111 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), 2112 Operands.back()->getType()}; 2113 2114 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 2115 2116 return Builder.CreateCall(F, Operands, ""); 2117 } 2118}] in { 2119 let HasFRMRoundModeOp = 1 in { 2120 // 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions 2121 defm vfwmacc : RVVFloatingWidenTerBuiltinSetRoundingMode; 2122 defm vfwnmacc : RVVFloatingWidenTerBuiltinSetRoundingMode; 2123 defm vfwmsac : RVVFloatingWidenTerBuiltinSetRoundingMode; 2124 defm vfwnmsac : RVVFloatingWidenTerBuiltinSetRoundingMode; 2125 } 2126 // 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions 2127 defm vfwmacc : RVVFloatingWidenTerBuiltinSet; 2128 defm vfwnmacc : RVVFloatingWidenTerBuiltinSet; 2129 defm vfwmsac : RVVFloatingWidenTerBuiltinSet; 2130 defm vfwnmsac : RVVFloatingWidenTerBuiltinSet; 2131} 2132 2133} 2134 2135let UnMaskedPolicyScheme = HasPassthruOperand in { 2136let ManualCodegen = [{ 2137 { 2138 // LLVM intrinsic 2139 // Unmasked: (passthru, op0, round_mode, vl) 2140 // Masked: (passthru, op0, mask, frm, vl, policy) 2141 2142 SmallVector<llvm::Value*, 7> Operands; 2143 bool HasMaskedOff = !( 2144 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 2145 (!IsMasked && PolicyAttrs & RVV_VTA)); 2146 bool HasRoundModeOp = IsMasked ? 2147 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) : 2148 (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); 2149 2150 unsigned Offset = IsMasked ? 2151 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 2152 2153 if (!HasMaskedOff) 2154 Operands.push_back(llvm::PoisonValue::get(ResultType)); 2155 else 2156 Operands.push_back(Ops[IsMasked ? 1 : 0]); 2157 2158 Operands.push_back(Ops[Offset]); // op0 2159 2160 if (IsMasked) 2161 Operands.push_back(Ops[0]); // mask 2162 2163 if (HasRoundModeOp) { 2164 Operands.push_back(Ops[Offset + 1]); // frm 2165 Operands.push_back(Ops[Offset + 2]); // vl 2166 } else { 2167 Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm 2168 Operands.push_back(Ops[Offset + 1]); // vl 2169 } 2170 2171 if (IsMasked) 2172 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 2173 2174 IntrinsicTypes = {ResultType, Operands.back()->getType()}; 2175 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 2176 return Builder.CreateCall(F, Operands, ""); 2177 } 2178}] in { 2179 let HasFRMRoundModeOp = 1 in { 2180 // 14.8. Vector Floating-Point Square-Root Instruction 2181 defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "xfd", [["v", "v", "vvu"]]>; 2182 2183 // 14.10. Vector Floating-Point Reciprocal Estimate Instruction 2184 defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "xfd", [["v", "v", "vvu"]]>; 2185 } 2186 // 14.8. Vector Floating-Point Square-Root Instruction 2187 defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "xfd", [["v", "v", "vv"]]>; 2188 2189 // 14.10. Vector Floating-Point Reciprocal Estimate Instruction 2190 defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "xfd", [["v", "v", "vv"]]>; 2191} 2192 2193// 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction 2194def vfrsqrt7 : RVVFloatingUnaryVVBuiltin; 2195 2196// 14.11. Vector Floating-Point MIN/MAX Instructions 2197defm vfmin : RVVFloatingBinBuiltinSet; 2198defm vfmax : RVVFloatingBinBuiltinSet; 2199 2200// 14.12. Vector Floating-Point Sign-Injection Instructions 2201defm vfsgnj : RVVFloatingBinBuiltinSet; 2202defm vfsgnjn : RVVFloatingBinBuiltinSet; 2203defm vfsgnjx : RVVFloatingBinBuiltinSet; 2204} 2205defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "xfd">; 2206defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "xfd">; 2207 2208// 14.13. Vector Floating-Point Compare Instructions 2209let MaskedPolicyScheme = HasPassthruOperand, 2210 HasTailPolicy = false in { 2211defm vmfeq : RVVFloatingMaskOutBuiltinSet; 2212defm vmfne : RVVFloatingMaskOutBuiltinSet; 2213defm vmflt : RVVFloatingMaskOutBuiltinSet; 2214defm vmfle : RVVFloatingMaskOutBuiltinSet; 2215defm vmfgt : RVVFloatingMaskOutBuiltinSet; 2216defm vmfge : RVVFloatingMaskOutBuiltinSet; 2217} 2218 2219// 14.14. Vector Floating-Point Classify Instruction 2220let Name = "vfclass_v", UnMaskedPolicyScheme = HasPassthruOperand in 2221 def vfclass : RVVOp0Builtin<"Uv", "Uvv", "xfd">; 2222 2223// 14.15. Vector Floating-Point Merge Instructio 2224// C/C++ Operand: (mask, op1, op2, vl), Builtin: (op1, op2, mask, vl) 2225let HasMasked = false, 2226 UnMaskedPolicyScheme = HasPassthruOperand, 2227 MaskedPolicyScheme = NonePolicy, 2228 ManualCodegen = [{ 2229 // insert poison passthru 2230 if (PolicyAttrs & RVV_VTA) 2231 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 2232 IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; 2233 }] in { 2234 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "xfd", 2235 [["vvm", "v", "vvvm"]]>; 2236 defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "xfd", 2237 [["vfm", "v", "vvem"]]>; 2238} 2239 2240// 14.16. Vector Floating-Point Move Instruction 2241let HasMasked = false, 2242 UnMaskedPolicyScheme = HasPassthruOperand, 2243 SupportOverloading = false, 2244 MaskedPolicyScheme = NonePolicy, 2245 OverloadedName = "vfmv_v" in 2246 defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "xfd", 2247 [["f", "v", "ve"]]>; 2248 2249// 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions 2250let UnMaskedPolicyScheme = HasPassthruOperand in { 2251def vfcvt_rtz_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_rtz_xu">; 2252def vfcvt_rtz_x_f_v : RVVConvToSignedBuiltin<"vfcvt_rtz_x">; 2253 2254// 14.18. Widening Floating-Point/Integer Type-Convert Instructions 2255let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2256 def vfwcvt_rtz_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_rtz_xu">; 2257 def vfwcvt_rtz_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_rtz_x">; 2258 def vfwcvt_f_xu_v : RVVConvBuiltin<"Fw", "FwUv", "csi", "vfwcvt_f">; 2259 def vfwcvt_f_x_v : RVVConvBuiltin<"Fw", "Fwv", "csi", "vfwcvt_f">; 2260 def vfwcvt_f_f_v : RVVConvBuiltin<"w", "wv", "xf", "vfwcvt_f">; 2261} 2262 2263// 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions 2264let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2265 def vfncvt_rtz_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_rtz_xu">; 2266 def vfncvt_rtz_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_rtz_x">; 2267 def vfncvt_rod_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_rod_f">; 2268} 2269let ManualCodegen = [{ 2270 { 2271 // LLVM intrinsic 2272 // Unmasked: (passthru, op0, frm, vl) 2273 // Masked: (passthru, op0, mask, frm, vl, policy) 2274 SmallVector<llvm::Value*, 7> Operands; 2275 bool HasMaskedOff = !( 2276 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 2277 (!IsMasked && PolicyAttrs & RVV_VTA)); 2278 bool HasRoundModeOp = IsMasked ? 2279 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) : 2280 (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); 2281 2282 unsigned Offset = IsMasked ? 2283 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 2284 2285 if (!HasMaskedOff) 2286 Operands.push_back(llvm::PoisonValue::get(ResultType)); 2287 else 2288 Operands.push_back(Ops[IsMasked ? 1 : 0]); 2289 2290 Operands.push_back(Ops[Offset]); // op0 2291 2292 if (IsMasked) 2293 Operands.push_back(Ops[0]); // mask 2294 2295 if (HasRoundModeOp) { 2296 Operands.push_back(Ops[Offset + 1]); // frm 2297 Operands.push_back(Ops[Offset + 2]); // vl 2298 } else { 2299 Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm 2300 Operands.push_back(Ops[Offset + 1]); // vl 2301 } 2302 2303 if (IsMasked) 2304 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 2305 2306 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), 2307 Operands.back()->getType()}; 2308 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 2309 return Builder.CreateCall(F, Operands, ""); 2310 } 2311}] in { 2312 let HasFRMRoundModeOp = 1 in { 2313 // 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions 2314 let OverloadedName = "vfcvt_x" in 2315 defm : 2316 RVVConvBuiltinSet<"vfcvt_x_f_v", "xfd", [["Iv", "Ivvu"]]>; 2317 let OverloadedName = "vfcvt_xu" in 2318 defm : 2319 RVVConvBuiltinSet<"vfcvt_xu_f_v", "xfd", [["Uv", "Uvvu"]]>; 2320 let OverloadedName = "vfcvt_f" in { 2321 defm : 2322 RVVConvBuiltinSet<"vfcvt_f_x_v", "sil", [["Fv", "Fvvu"]]>; 2323 defm : 2324 RVVConvBuiltinSet<"vfcvt_f_xu_v", "sil", [["Fv", "FvUvu"]]>; 2325 } 2326 2327 // 14.18. Widening Floating-Point/Integer Type-Convert Instructions 2328 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2329 let OverloadedName = "vfwcvt_x" in 2330 defm : 2331 RVVConvBuiltinSet<"vfwcvt_x_f_v", "xf", [["Iw", "Iwvu"]]>; 2332 let OverloadedName = "vfwcvt_xu" in 2333 defm : 2334 RVVConvBuiltinSet<"vfwcvt_xu_f_v", "xf", [["Uw", "Uwvu"]]>; 2335 } 2336 // 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions 2337 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2338 let OverloadedName = "vfncvt_x" in 2339 defm : 2340 RVVConvBuiltinSet<"vfncvt_x_f_w", "csi", [["Iv", "IvFwu"]]>; 2341 let OverloadedName = "vfncvt_xu" in 2342 defm : 2343 RVVConvBuiltinSet<"vfncvt_xu_f_w", "csi", [["Uv", "UvFwu"]]>; 2344 let OverloadedName = "vfncvt_f" in { 2345 defm : 2346 RVVConvBuiltinSet<"vfncvt_f_x_w", "csi", [["Fv", "Fvwu"]]>; 2347 defm : 2348 RVVConvBuiltinSet<"vfncvt_f_xu_w", "csi", [["Fv", "FvUwu"]]>; 2349 } 2350 let OverloadedName = "vfncvt_f" in 2351 defm : 2352 RVVConvBuiltinSet<"vfncvt_f_f_w", "xf", [["v", "vwu"]]>; 2353 } 2354 } 2355 2356 // 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions 2357 let OverloadedName = "vfcvt_x" in 2358 defm : 2359 RVVConvBuiltinSet<"vfcvt_x_f_v", "xfd", [["Iv", "Ivv"]]>; 2360 let OverloadedName = "vfcvt_xu" in 2361 defm : 2362 RVVConvBuiltinSet<"vfcvt_xu_f_v", "xfd", [["Uv", "Uvv"]]>; 2363 let OverloadedName = "vfcvt_f" in { 2364 defm : 2365 RVVConvBuiltinSet<"vfcvt_f_x_v", "sil", [["Fv", "Fvv"]]>; 2366 defm : 2367 RVVConvBuiltinSet<"vfcvt_f_xu_v", "sil", [["Fv", "FvUv"]]>; 2368 } 2369 2370 // 14.18. Widening Floating-Point/Integer Type-Convert Instructions 2371 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2372 let OverloadedName = "vfwcvt_x" in 2373 defm : 2374 RVVConvBuiltinSet<"vfwcvt_x_f_v", "xf", [["Iw", "Iwv"]]>; 2375 let OverloadedName = "vfwcvt_xu" in 2376 defm : 2377 RVVConvBuiltinSet<"vfwcvt_xu_f_v", "xf", [["Uw", "Uwv"]]>; 2378 } 2379 // 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions 2380 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2381 let OverloadedName = "vfncvt_x" in 2382 defm : 2383 RVVConvBuiltinSet<"vfncvt_x_f_w", "csi", [["Iv", "IvFw"]]>; 2384 let OverloadedName = "vfncvt_xu" in 2385 defm : 2386 RVVConvBuiltinSet<"vfncvt_xu_f_w", "csi", [["Uv", "UvFw"]]>; 2387 let OverloadedName = "vfncvt_f" in { 2388 defm : 2389 RVVConvBuiltinSet<"vfncvt_f_x_w", "csi", [["Fv", "Fvw"]]>; 2390 defm : 2391 RVVConvBuiltinSet<"vfncvt_f_xu_w", "csi", [["Fv", "FvUw"]]>; 2392 } 2393 let OverloadedName = "vfncvt_f" in 2394 defm : 2395 RVVConvBuiltinSet<"vfncvt_f_f_w", "xf", [["v", "vw"]]>; 2396 } 2397} 2398} 2399 2400// 15. Vector Reduction Operations 2401// 15.1. Vector Single-Width Integer Reduction Instructions 2402let UnMaskedPolicyScheme = HasPassthruOperand, 2403 MaskedPolicyScheme = HasPassthruOperand, 2404 HasMaskPolicy = false in { 2405defm vredsum : RVVIntReductionBuiltinSet; 2406defm vredmaxu : RVVUnsignedReductionBuiltin; 2407defm vredmax : RVVSignedReductionBuiltin; 2408defm vredminu : RVVUnsignedReductionBuiltin; 2409defm vredmin : RVVSignedReductionBuiltin; 2410defm vredand : RVVIntReductionBuiltinSet; 2411defm vredor : RVVIntReductionBuiltinSet; 2412defm vredxor : RVVIntReductionBuiltinSet; 2413 2414// 15.2. Vector Widening Integer Reduction Instructions 2415// Vector Widening Integer Reduction Operations 2416let HasMaskedOffOperand = true in { 2417 defm vwredsum : RVVOutOp0BuiltinSet<"vwredsum", "csi", 2418 [["vs", "vSw", "SwvSw"]]>; 2419 defm vwredsumu : RVVOutOp0BuiltinSet<"vwredsumu", "csi", 2420 [["vs", "UvUSw", "USwUvUSw"]]>; 2421} 2422 2423// 15.3. Vector Single-Width Floating-Point Reduction Instructions 2424defm vfredmax : RVVFloatingReductionBuiltin; 2425defm vfredmin : RVVFloatingReductionBuiltin; 2426let ManualCodegen = [{ 2427 { 2428 // LLVM intrinsic 2429 // Unmasked: (passthru, op0, op1, round_mode, vl) 2430 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 2431 2432 SmallVector<llvm::Value*, 7> Operands; 2433 bool HasMaskedOff = !( 2434 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 2435 (!IsMasked && PolicyAttrs & RVV_VTA)); 2436 bool HasRoundModeOp = IsMasked ? 2437 (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : 2438 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); 2439 2440 unsigned Offset = IsMasked ? 2441 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 2442 2443 if (!HasMaskedOff) 2444 Operands.push_back(llvm::PoisonValue::get(ResultType)); 2445 else 2446 Operands.push_back(Ops[IsMasked ? 1 : 0]); 2447 2448 Operands.push_back(Ops[Offset]); // op0 2449 Operands.push_back(Ops[Offset + 1]); // op1 2450 2451 if (IsMasked) 2452 Operands.push_back(Ops[0]); // mask 2453 2454 if (HasRoundModeOp) { 2455 Operands.push_back(Ops[Offset + 2]); // frm 2456 Operands.push_back(Ops[Offset + 3]); // vl 2457 } else { 2458 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 2459 Operands.push_back(Ops[Offset + 2]); // vl 2460 } 2461 2462 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), 2463 Ops.back()->getType()}; 2464 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 2465 return Builder.CreateCall(F, Operands, ""); 2466 } 2467}] in { 2468 let HasFRMRoundModeOp = 1 in { 2469 // 15.3. Vector Single-Width Floating-Point Reduction Instructions 2470 defm vfredusum : RVVFloatingReductionBuiltinRoundingMode; 2471 defm vfredosum : RVVFloatingReductionBuiltinRoundingMode; 2472 2473 // 15.4. Vector Widening Floating-Point Reduction Instructions 2474 defm vfwredusum : RVVFloatingWidenReductionBuiltinRoundingMode; 2475 defm vfwredosum : RVVFloatingWidenReductionBuiltinRoundingMode; 2476 } 2477 // 15.3. Vector Single-Width Floating-Point Reduction Instructions 2478 defm vfredusum : RVVFloatingReductionBuiltin; 2479 defm vfredosum : RVVFloatingReductionBuiltin; 2480 2481 // 15.4. Vector Widening Floating-Point Reduction Instructions 2482 defm vfwredusum : RVVFloatingWidenReductionBuiltin; 2483 defm vfwredosum : RVVFloatingWidenReductionBuiltin; 2484} 2485} 2486 2487// 16. Vector Mask Instructions 2488// 16.1. Vector Mask-Register Logical Instructions 2489def vmand : RVVMaskBinBuiltin; 2490def vmnand : RVVMaskBinBuiltin; 2491def vmandn : RVVMaskBinBuiltin; 2492def vmxor : RVVMaskBinBuiltin; 2493def vmor : RVVMaskBinBuiltin; 2494def vmnor : RVVMaskBinBuiltin; 2495def vmorn : RVVMaskBinBuiltin; 2496def vmxnor : RVVMaskBinBuiltin; 2497// pseudoinstructions 2498def vmclr : RVVMaskNullaryBuiltin; 2499def vmset : RVVMaskNullaryBuiltin; 2500defm vmmv_m : RVVPseudoMaskBuiltin<"vmand", "c">; 2501defm vmnot_m : RVVPseudoMaskBuiltin<"vmnand", "c">; 2502 2503let MaskedPolicyScheme = NonePolicy in { 2504// 16.2. Vector count population in mask vcpop.m 2505def vcpop : RVVMaskOp0Builtin<"um">; 2506 2507// 16.3. vfirst find-first-set mask bit 2508def vfirst : RVVMaskOp0Builtin<"lm">; 2509} 2510 2511let MaskedPolicyScheme = HasPassthruOperand, 2512 HasTailPolicy = false in { 2513// 16.4. vmsbf.m set-before-first mask bit 2514def vmsbf : RVVMaskUnaryBuiltin; 2515 2516// 16.5. vmsif.m set-including-first mask bit 2517def vmsif : RVVMaskUnaryBuiltin; 2518 2519// 16.6. vmsof.m set-only-first mask bit 2520def vmsof : RVVMaskUnaryBuiltin; 2521} 2522 2523let UnMaskedPolicyScheme = HasPassthruOperand, SupportOverloading = false in { 2524 // 16.8. Vector Iota Instruction 2525 defm viota : RVVOutBuiltinSet<"viota", "csil", [["m", "Uv", "Uvm"]]>; 2526 2527 // 16.9. Vector Element Index Instruction 2528 defm vid : RVVOutBuiltinSet<"vid", "csil", [["v", "v", "v"], 2529 ["v", "Uv", "Uv"]]>; 2530} 2531 2532// 17. Vector Permutation Instructions 2533// 17.1. Integer Scalar Move Instructions 2534let HasMasked = false, MaskedPolicyScheme = NonePolicy in { 2535 let HasVL = false, OverloadedName = "vmv_x" in 2536 defm vmv_x : RVVOp0BuiltinSet<"vmv_x_s", "csil", 2537 [["s", "ve", "ev"], 2538 ["s", "UvUe", "UeUv"]]>; 2539 let OverloadedName = "vmv_s", 2540 UnMaskedPolicyScheme = HasPassthruOperand, 2541 SupportOverloading = false in 2542 defm vmv_s : RVVOutBuiltinSet<"vmv_s_x", "csil", 2543 [["x", "v", "ve"], 2544 ["x", "Uv", "UvUe"]]>; 2545} 2546 2547// 17.2. Floating-Point Scalar Move Instructions 2548let HasMasked = false, MaskedPolicyScheme = NonePolicy in { 2549 let HasVL = false, OverloadedName = "vfmv_f" in 2550 defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "xfd", 2551 [["s", "ve", "ev"]]>; 2552 let OverloadedName = "vfmv_s", 2553 UnMaskedPolicyScheme = HasPassthruOperand, 2554 SupportOverloading = false in 2555 defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "xfd", 2556 [["f", "v", "ve"], 2557 ["x", "Uv", "UvUe"]]>; 2558} 2559 2560// 17.3. Vector Slide Instructions 2561// 17.3.1. Vector Slideup Instructions 2562defm vslideup : RVVSlideUpBuiltinSet; 2563// 17.3.2. Vector Slidedown Instructions 2564defm vslidedown : RVVSlideDownBuiltinSet; 2565 2566// 17.3.3. Vector Slide1up Instructions 2567let UnMaskedPolicyScheme = HasPassthruOperand in { 2568defm vslide1up : RVVSlideOneBuiltinSet; 2569defm vfslide1up : RVVFloatingBinVFBuiltinSet; 2570 2571// 17.3.4. Vector Slide1down Instruction 2572defm vslide1down : RVVSlideOneBuiltinSet; 2573defm vfslide1down : RVVFloatingBinVFBuiltinSet; 2574 2575// 17.4. Vector Register Gather Instructions 2576// signed and floating type 2577defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfd", 2578 [["vv", "v", "vvUv"]]>; 2579defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfd", 2580 [["vx", "v", "vvz"]]>; 2581defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd", 2582 [["vv", "v", "vv(Log2EEW:4)Uv"]]>; 2583// unsigned type 2584defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil", 2585 [["vv", "Uv", "UvUvUv"]]>; 2586defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csil", 2587 [["vx", "Uv", "UvUvz"]]>; 2588defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csil", 2589 [["vv", "Uv", "UvUv(Log2EEW:4)Uv"]]>; 2590} 2591 2592// 17.5. Vector Compress Instruction 2593let HasMasked = false, 2594 UnMaskedPolicyScheme = HasPassthruOperand, 2595 MaskedPolicyScheme = NonePolicy, 2596 ManualCodegen = [{ 2597 // insert poison passthru 2598 if (PolicyAttrs & RVV_VTA) 2599 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 2600 IntrinsicTypes = {ResultType, Ops.back()->getType()}; 2601 }] in { 2602 // signed and floating type 2603 defm vcompress : RVVOutBuiltinSet<"vcompress", "csilxfd", 2604 [["vm", "v", "vvm"]]>; 2605 // unsigned type 2606 defm vcompress : RVVOutBuiltinSet<"vcompress", "csil", 2607 [["vm", "Uv", "UvUvm"]]>; 2608} 2609 2610// Miscellaneous 2611let HasMasked = false, HasVL = false, IRName = "" in { 2612 let Name = "vreinterpret_v", MaskedPolicyScheme = NonePolicy, 2613 ManualCodegen = [{ 2614 if (ResultType->isIntOrIntVectorTy(1) || 2615 Ops[0]->getType()->isIntOrIntVectorTy(1)) { 2616 assert(isa<ScalableVectorType>(ResultType) && 2617 isa<ScalableVectorType>(Ops[0]->getType())); 2618 2619 LLVMContext &Context = CGM.getLLVMContext(); 2620 ScalableVectorType *Boolean64Ty = 2621 ScalableVectorType::get(llvm::Type::getInt1Ty(Context), 64); 2622 2623 if (ResultType->isIntOrIntVectorTy(1)) { 2624 // Casting from m1 vector integer -> vector boolean 2625 // Ex: <vscale x 8 x i8> 2626 // --(bitcast)--------> <vscale x 64 x i1> 2627 // --(vector_extract)-> <vscale x 8 x i1> 2628 llvm::Value *BitCast = Builder.CreateBitCast(Ops[0], Boolean64Ty); 2629 return Builder.CreateExtractVector(ResultType, BitCast, 2630 ConstantInt::get(Int64Ty, 0)); 2631 } else { 2632 // Casting from vector boolean -> m1 vector integer 2633 // Ex: <vscale x 1 x i1> 2634 // --(vector_insert)-> <vscale x 64 x i1> 2635 // --(bitcast)-------> <vscale x 8 x i8> 2636 llvm::Value *Boolean64Val = 2637 Builder.CreateInsertVector(Boolean64Ty, 2638 llvm::PoisonValue::get(Boolean64Ty), 2639 Ops[0], 2640 ConstantInt::get(Int64Ty, 0)); 2641 return Builder.CreateBitCast(Boolean64Val, ResultType); 2642 } 2643 } 2644 return Builder.CreateBitCast(Ops[0], ResultType); 2645 }] in { 2646 // Reinterpret between different type under the same SEW and LMUL 2647 def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">; 2648 def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "sil", "v">; 2649 def vreinterpret_u_i : RVVBuiltin<"vUv", "Uvv", "csil", "Uv">; 2650 def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "sil", "Uv">; 2651 def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "sil", "Fv">; 2652 def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "sil", "Fv">; 2653 2654 // Reinterpret between different SEW under the same LMUL 2655 foreach dst_sew = ["(FixedSEW:8)", "(FixedSEW:16)", "(FixedSEW:32)", 2656 "(FixedSEW:64)"] in { 2657 def vreinterpret_i_ # dst_sew : RVVBuiltin<"v" # dst_sew # "v", 2658 dst_sew # "vv", "csil", dst_sew # "v">; 2659 def vreinterpret_u_ # dst_sew : RVVBuiltin<"Uv" # dst_sew # "Uv", 2660 dst_sew # "UvUv", "csil", dst_sew # "Uv">; 2661 } 2662 2663 // Existing users of FixedSEW - the reinterpretation between different SEW 2664 // and same LMUL has the implicit assumption that if FixedSEW is set to the 2665 // given element width, then the type will be identified as invalid, thus 2666 // skipping definition of reinterpret of SEW=8 to SEW=8. However this blocks 2667 // our usage here of defining all possible combinations of a fixed SEW to 2668 // any boolean. So we need to separately define SEW=8 here. 2669 // Reinterpret from LMUL=1 integer type to vector boolean type 2670 def vreintrepret_m1_b8_signed : 2671 RVVBuiltin<"Svm", 2672 "mSv", 2673 "c", "m">; 2674 def vreintrepret_m1_b8_usigned : 2675 RVVBuiltin<"USvm", 2676 "mUSv", 2677 "c", "m">; 2678 2679 // Reinterpret from vector boolean type to LMUL=1 integer type 2680 def vreintrepret_b8_m1_signed : 2681 RVVBuiltin<"mSv", 2682 "Svm", 2683 "c", "Sv">; 2684 def vreintrepret_b8_m1_usigned : 2685 RVVBuiltin<"mUSv", 2686 "USvm", 2687 "c", "USv">; 2688 2689 foreach dst_sew = ["16", "32", "64"] in { 2690 // Reinterpret from LMUL=1 integer type to vector boolean type 2691 def vreinterpret_m1_b # dst_sew # _signed: 2692 RVVBuiltin<"(FixedSEW:" # dst_sew # ")Svm", 2693 "m(FixedSEW:" # dst_sew # ")Sv", 2694 "c", "m">; 2695 def vreinterpret_m1_b # dst_sew # _unsigned: 2696 RVVBuiltin<"(FixedSEW:" # dst_sew # ")USvm", 2697 "m(FixedSEW:" # dst_sew # ")USv", 2698 "c", "m">; 2699 // Reinterpret from vector boolean type to LMUL=1 integer type 2700 def vreinterpret_b # dst_sew # _m1_signed: 2701 RVVBuiltin<"m(FixedSEW:" # dst_sew # ")Sv", 2702 "(FixedSEW:" # dst_sew # ")Svm", 2703 "c", "(FixedSEW:" # dst_sew # ")Sv">; 2704 def vreinterpret_b # dst_sew # _m1_unsigned: 2705 RVVBuiltin<"m(FixedSEW:" # dst_sew # ")USv", 2706 "(FixedSEW:" # dst_sew # ")USvm", 2707 "c", "(FixedSEW:" # dst_sew # ")USv">; 2708 } 2709 } 2710 2711 let Name = "vundefined", SupportOverloading = false, 2712 MaskedPolicyScheme = NonePolicy, 2713 ManualCodegen = [{ 2714 return llvm::PoisonValue::get(ResultType); 2715 }] in { 2716 def vundefined : RVVBuiltin<"v", "v", "csilxfd">; 2717 def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">; 2718 } 2719 2720 // LMUL truncation 2721 // C/C++ Operand: VecTy, IR Operand: VecTy, Index 2722 let Name = "vlmul_trunc_v", OverloadedName = "vlmul_trunc", 2723 MaskedPolicyScheme = NonePolicy, 2724 ManualCodegen = [{ { 2725 return Builder.CreateExtractVector(ResultType, Ops[0], 2726 ConstantInt::get(Int64Ty, 0)); 2727 } }] in { 2728 foreach dst_lmul = ["(SFixedLog2LMUL:-3)", "(SFixedLog2LMUL:-2)", "(SFixedLog2LMUL:-1)", 2729 "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2730 def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2731 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2732 def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2733 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2734 } 2735 } 2736 2737 // LMUL extension 2738 // C/C++ Operand: SubVecTy, IR Operand: VecTy, SubVecTy, Index 2739 let Name = "vlmul_ext_v", OverloadedName = "vlmul_ext", 2740 MaskedPolicyScheme = NonePolicy, 2741 ManualCodegen = [{ 2742 return Builder.CreateInsertVector(ResultType, 2743 llvm::PoisonValue::get(ResultType), 2744 Ops[0], ConstantInt::get(Int64Ty, 0)); 2745 }] in { 2746 foreach dst_lmul = ["(LFixedLog2LMUL:-2)", "(LFixedLog2LMUL:-1)", "(LFixedLog2LMUL:-0)", 2747 "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2748 def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2749 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2750 def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2751 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2752 } 2753 } 2754 2755 let Name = "vget_v", MaskedPolicyScheme = NonePolicy, 2756 ManualCodegen = [{ 2757 { 2758 if (isa<StructType>(Ops[0]->getType())) // For tuple type 2759 // Extract value from index (operand 1) of vtuple (operand 0) 2760 return Builder.CreateExtractValue( 2761 Ops[0], 2762 {(unsigned)cast<ConstantInt>(Ops[1])->getZExtValue()}); 2763 auto *VecTy = cast<ScalableVectorType>(ResultType); 2764 auto *OpVecTy = cast<ScalableVectorType>(Ops[0]->getType()); 2765 // Mask to only valid indices. 2766 unsigned MaxIndex = OpVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2767 assert(isPowerOf2_32(MaxIndex)); 2768 Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); 2769 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2770 Ops[1] = Builder.CreateMul(Ops[1], 2771 ConstantInt::get(Ops[1]->getType(), 2772 VecTy->getMinNumElements())); 2773 return Builder.CreateExtractVector(ResultType, Ops[0], Ops[1]); 2774 } 2775 }] in { 2776 foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2777 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfd", dst_lmul # "v">; 2778 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">; 2779 } 2780 foreach nf = NFList in { 2781 defvar T = "(Tuple:" # nf # ")"; 2782 def : RVVBuiltin<T # "vv", "v" # T # "vKz", "csilxfd", "v">; 2783 def : RVVBuiltin<T # "UvUv", "Uv" # T # "UvKz", "csil", "Uv">; 2784 } 2785 } 2786 2787 let Name = "vset_v", MaskedPolicyScheme = NonePolicy, 2788 ManualCodegen = [{ 2789 { 2790 if (isa<StructType>(ResultType)) // For tuple type 2791 // Insert value (operand 2) into index (operand 1) of vtuple (operand 0) 2792 return Builder.CreateInsertValue( 2793 Ops[0], Ops[2], 2794 {(unsigned)cast<ConstantInt>(Ops[1])->getZExtValue()}); 2795 auto *ResVecTy = cast<ScalableVectorType>(ResultType); 2796 auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType()); 2797 // Mask to only valid indices. 2798 unsigned MaxIndex = ResVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2799 assert(isPowerOf2_32(MaxIndex)); 2800 Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); 2801 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2802 Ops[1] = Builder.CreateMul(Ops[1], 2803 ConstantInt::get(Ops[1]->getType(), 2804 VecTy->getMinNumElements())); 2805 return Builder.CreateInsertVector(ResultType, Ops[0], Ops[2], Ops[1]); 2806 } 2807 }] in { 2808 let Log2LMUL = [0, 1, 2] in { 2809 foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2810 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfd">; 2811 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">; 2812 } 2813 } 2814 foreach nf = NFList in { 2815 defvar T = "(Tuple:" # nf # ")"; 2816 def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "csilxfd">; 2817 def : RVVBuiltin<"Uv" # T # "Uv", T # "Uv" # T # "UvKzUv", "csil">; 2818 } 2819 } 2820} 2821