1//===-- IR/VPIntrinsics.def - Describes llvm.vp.* Intrinsics -*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file contains descriptions of the various Vector Predication intrinsics. 10// This is used as a central place for enumerating the different instructions 11// and should eventually be the place to put comments about the instructions. 12// 13//===----------------------------------------------------------------------===// 14 15// NOTE: NO INCLUDE GUARD DESIRED! 16 17// Provide definitions of macros so that users of this file do not have to 18// define everything to use it... 19// 20// Register a VP intrinsic and begin its property scope. 21// All VP intrinsic scopes are top level, ie it is illegal to place a 22// BEGIN_REGISTER_VP_INTRINSIC within a VP intrinsic scope. 23// \p VPID The VP intrinsic id. 24// \p MASKPOS The mask operand position. 25// \p EVLPOS The explicit vector length operand position. 26#ifndef BEGIN_REGISTER_VP_INTRINSIC 27#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) 28#endif 29 30// End the property scope of a VP intrinsic. 31#ifndef END_REGISTER_VP_INTRINSIC 32#define END_REGISTER_VP_INTRINSIC(VPID) 33#endif 34 35// Register a new VP SDNode and begin its property scope. 36// When the SDNode scope is nested within a VP intrinsic scope, it is 37// implicitly registered as the canonical SDNode for this VP intrinsic. There 38// is one VP intrinsic that maps directly to one SDNode that goes by the 39// same name. Since the operands are also the same, we open the property 40// scopes for both the VPIntrinsic and the SDNode at once. 41// \p VPSD The SelectionDAG Node id (eg VP_ADD). 42// \p LEGALPOS The operand position of the SDNode that is used for legalizing. 43// If LEGALPOS < 0, then the return type given by 44// TheNode->getValueType(-1-LEGALPOS) is used. 45// \p TDNAME The name of the TableGen definition of this SDNode. 46// \p MASKPOS The mask operand position. 47// \p EVLPOS The explicit vector length operand position. 48#ifndef BEGIN_REGISTER_VP_SDNODE 49#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) 50#endif 51 52// End the property scope of a new VP SDNode. 53#ifndef END_REGISTER_VP_SDNODE 54#define END_REGISTER_VP_SDNODE(VPSD) 55#endif 56 57// Helper macro to set up the mapping from VP intrinsic to ISD opcode. 58// Note: More than one VP intrinsic may map to one ISD opcode. 59#ifndef HELPER_MAP_VPID_TO_VPSD 60#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) 61#endif 62 63// Helper macros for the common "1:1 - Intrinsic : SDNode" case. 64// 65// There is one VP intrinsic that maps directly to one SDNode that goes by the 66// same name. Since the operands are also the same, we open the property 67// scopes for both the VPIntrinsic and the SDNode at once. 68// 69// \p VPID The canonical name (eg `vp_add`, which at the same time is the 70// name of the intrinsic and the TableGen def of the SDNode). 71// \p MASKPOS The mask operand position. 72// \p EVLPOS The explicit vector length operand position. 73// \p VPSD The SelectionDAG Node id (eg VP_ADD). 74// \p LEGALPOS The operand position of the SDNode that is used for legalizing 75// this SDNode. This can be `-1`, in which case the return type of 76// the SDNode is used. 77#define BEGIN_REGISTER_VP(VPID, MASKPOS, EVLPOS, VPSD, LEGALPOS) \ 78 BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) \ 79 BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, VPID, MASKPOS, EVLPOS) \ 80 HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) 81 82#define END_REGISTER_VP(VPID, VPSD) \ 83 END_REGISTER_VP_INTRINSIC(VPID) \ 84 END_REGISTER_VP_SDNODE(VPSD) 85 86// The following macros attach properties to the scope they are placed in. This 87// assigns the property to the VP Intrinsic and/or SDNode that belongs to the 88// scope. 89// 90// Property Macros { 91 92// The intrinsic and/or SDNode has the same function as this LLVM IR Opcode. 93// \p OPC The opcode of the instruction with the same function. 94#ifndef VP_PROPERTY_FUNCTIONAL_OPC 95#define VP_PROPERTY_FUNCTIONAL_OPC(OPC) 96#endif 97 98// Whether the intrinsic may have a rounding mode or exception behavior operand 99// bundle. 100// \p HASROUND '1' if the intrinsic can have a rounding mode operand bundle, 101// '0' otherwise. 102// \p HASEXCEPT '1' if the intrinsic can have an exception behavior operand 103// bundle, '0' otherwise. 104// \p INTRINID The constrained fp intrinsic this VP intrinsic corresponds to. 105#ifndef VP_PROPERTY_CONSTRAINEDFP 106#define VP_PROPERTY_CONSTRAINEDFP(HASROUND, HASEXCEPT, INTRINID) 107#endif 108 109// The intrinsic and/or SDNode has the same function as this ISD Opcode. 110// \p SDOPC The opcode of the instruction with the same function. 111#ifndef VP_PROPERTY_FUNCTIONAL_SDOPC 112#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) 113#endif 114 115// Map this VP intrinsic to its canonical functional intrinsic. 116// \p INTRIN The non-VP intrinsics with the same function. 117#ifndef VP_PROPERTY_FUNCTIONAL_INTRINSIC 118#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) 119#endif 120 121// This VP Intrinsic is a memory operation 122// The pointer arg is at POINTERPOS and the data arg is at DATAPOS. 123#ifndef VP_PROPERTY_MEMOP 124#define VP_PROPERTY_MEMOP(POINTERPOS, DATAPOS) 125#endif 126 127// Map this VP reduction intrinsic to its reduction operand positions. 128#ifndef VP_PROPERTY_REDUCTION 129#define VP_PROPERTY_REDUCTION(STARTPOS, VECTORPOS) 130#endif 131 132// A property to infer VP binary-op SDNode opcodes automatically. 133#ifndef VP_PROPERTY_BINARYOP 134#define VP_PROPERTY_BINARYOP 135#endif 136 137// A property to infer VP type casts automatically. 138#ifndef VP_PROPERTY_CASTOP 139#define VP_PROPERTY_CASTOP 140#endif 141 142// This VP Intrinsic is a comparison operation 143// The condition code arg is at CCPOS and accepts floating-point condition 144// codes if ISFP is set, else it accepts integer condition codes. 145#ifndef VP_PROPERTY_CMP 146#define VP_PROPERTY_CMP(CCPOS, ISFP) 147#endif 148 149/// } Property Macros 150 151///// Integer Arithmetic { 152 153// Specialized helper macro for integer binary operators (%x, %y, %mask, %evl). 154#ifdef HELPER_REGISTER_BINARY_INT_VP 155#error \ 156 "The internal helper macro HELPER_REGISTER_BINARY_INT_VP is already defined!" 157#endif 158#define HELPER_REGISTER_BINARY_INT_VP(VPID, VPSD, IROPC, SDOPC) \ 159 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, -1) \ 160 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 161 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 162 VP_PROPERTY_BINARYOP \ 163 END_REGISTER_VP(VPID, VPSD) 164 165// llvm.vp.add(x,y,mask,vlen) 166HELPER_REGISTER_BINARY_INT_VP(vp_add, VP_ADD, Add, ADD) 167 168// llvm.vp.and(x,y,mask,vlen) 169HELPER_REGISTER_BINARY_INT_VP(vp_and, VP_AND, And, AND) 170 171// llvm.vp.ashr(x,y,mask,vlen) 172HELPER_REGISTER_BINARY_INT_VP(vp_ashr, VP_ASHR, AShr, SRA) 173 174// llvm.vp.lshr(x,y,mask,vlen) 175HELPER_REGISTER_BINARY_INT_VP(vp_lshr, VP_LSHR, LShr, SRL) 176 177// llvm.vp.mul(x,y,mask,vlen) 178HELPER_REGISTER_BINARY_INT_VP(vp_mul, VP_MUL, Mul, MUL) 179 180// llvm.vp.or(x,y,mask,vlen) 181HELPER_REGISTER_BINARY_INT_VP(vp_or, VP_OR, Or, OR) 182 183// llvm.vp.sdiv(x,y,mask,vlen) 184HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv, SDIV) 185 186// llvm.vp.shl(x,y,mask,vlen) 187HELPER_REGISTER_BINARY_INT_VP(vp_shl, VP_SHL, Shl, SHL) 188 189// llvm.vp.srem(x,y,mask,vlen) 190HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem, SREM) 191 192// llvm.vp.sub(x,y,mask,vlen) 193HELPER_REGISTER_BINARY_INT_VP(vp_sub, VP_SUB, Sub, SUB) 194 195// llvm.vp.udiv(x,y,mask,vlen) 196HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv, UDIV) 197 198// llvm.vp.urem(x,y,mask,vlen) 199HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem, UREM) 200 201// llvm.vp.xor(x,y,mask,vlen) 202HELPER_REGISTER_BINARY_INT_VP(vp_xor, VP_XOR, Xor, XOR) 203 204#undef HELPER_REGISTER_BINARY_INT_VP 205 206// llvm.vp.smin(x,y,mask,vlen) 207BEGIN_REGISTER_VP(vp_smin, 2, 3, VP_SMIN, -1) 208VP_PROPERTY_BINARYOP 209VP_PROPERTY_FUNCTIONAL_SDOPC(SMIN) 210END_REGISTER_VP(vp_smin, VP_SMIN) 211 212// llvm.vp.smax(x,y,mask,vlen) 213BEGIN_REGISTER_VP(vp_smax, 2, 3, VP_SMAX, -1) 214VP_PROPERTY_BINARYOP 215VP_PROPERTY_FUNCTIONAL_SDOPC(SMAX) 216END_REGISTER_VP(vp_smax, VP_SMAX) 217 218// llvm.vp.umin(x,y,mask,vlen) 219BEGIN_REGISTER_VP(vp_umin, 2, 3, VP_UMIN, -1) 220VP_PROPERTY_BINARYOP 221VP_PROPERTY_FUNCTIONAL_SDOPC(UMIN) 222END_REGISTER_VP(vp_umin, VP_UMIN) 223 224// llvm.vp.umax(x,y,mask,vlen) 225BEGIN_REGISTER_VP(vp_umax, 2, 3, VP_UMAX, -1) 226VP_PROPERTY_BINARYOP 227VP_PROPERTY_FUNCTIONAL_SDOPC(UMAX) 228END_REGISTER_VP(vp_umax, VP_UMAX) 229 230// llvm.vp.abs(x,is_int_min_poison,mask,vlen) 231BEGIN_REGISTER_VP_INTRINSIC(vp_abs, 2, 3) 232BEGIN_REGISTER_VP_SDNODE(VP_ABS, -1, vp_abs, 1, 2) 233HELPER_MAP_VPID_TO_VPSD(vp_abs, VP_ABS) 234VP_PROPERTY_FUNCTIONAL_SDOPC(ABS) 235END_REGISTER_VP(vp_abs, VP_ABS) 236 237// llvm.vp.bswap(x,mask,vlen) 238BEGIN_REGISTER_VP(vp_bswap, 1, 2, VP_BSWAP, -1) 239VP_PROPERTY_FUNCTIONAL_SDOPC(BSWAP) 240END_REGISTER_VP(vp_bswap, VP_BSWAP) 241 242// llvm.vp.bitreverse(x,mask,vlen) 243BEGIN_REGISTER_VP(vp_bitreverse, 1, 2, VP_BITREVERSE, -1) 244VP_PROPERTY_FUNCTIONAL_SDOPC(BITREVERSE) 245END_REGISTER_VP(vp_bitreverse, VP_BITREVERSE) 246 247// llvm.vp.ctpop(x,mask,vlen) 248BEGIN_REGISTER_VP(vp_ctpop, 1, 2, VP_CTPOP, -1) 249VP_PROPERTY_FUNCTIONAL_SDOPC(CTPOP) 250END_REGISTER_VP(vp_ctpop, VP_CTPOP) 251 252// llvm.vp.ctlz(x,is_zero_poison,mask,vlen) 253BEGIN_REGISTER_VP_INTRINSIC(vp_ctlz, 2, 3) 254BEGIN_REGISTER_VP_SDNODE(VP_CTLZ, -1, vp_ctlz, 1, 2) 255VP_PROPERTY_FUNCTIONAL_SDOPC(CTLZ) 256END_REGISTER_VP_SDNODE(VP_CTLZ) 257BEGIN_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF, -1, vp_ctlz_zero_undef, 1, 2) 258END_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF) 259END_REGISTER_VP_INTRINSIC(vp_ctlz) 260 261// llvm.vp.cttz(x,is_zero_poison,mask,vlen) 262BEGIN_REGISTER_VP_INTRINSIC(vp_cttz, 2, 3) 263BEGIN_REGISTER_VP_SDNODE(VP_CTTZ, -1, vp_cttz, 1, 2) 264VP_PROPERTY_FUNCTIONAL_SDOPC(CTTZ) 265END_REGISTER_VP_SDNODE(VP_CTTZ) 266BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF, -1, vp_cttz_zero_undef, 1, 2) 267END_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF) 268END_REGISTER_VP_INTRINSIC(vp_cttz) 269 270// llvm.vp.fshl(x,y,z,mask,vlen) 271BEGIN_REGISTER_VP(vp_fshl, 3, 4, VP_FSHL, -1) 272VP_PROPERTY_FUNCTIONAL_SDOPC(FSHL) 273END_REGISTER_VP(vp_fshl, VP_FSHL) 274 275// llvm.vp.fshr(x,y,z,mask,vlen) 276BEGIN_REGISTER_VP(vp_fshr, 3, 4, VP_FSHR, -1) 277VP_PROPERTY_FUNCTIONAL_SDOPC(FSHR) 278END_REGISTER_VP(vp_fshr, VP_FSHR) 279///// } Integer Arithmetic 280 281///// Floating-Point Arithmetic { 282 283// Specialized helper macro for floating-point binary operators 284// <operation>(%x, %y, %mask, %evl). 285#ifdef HELPER_REGISTER_BINARY_FP_VP 286#error \ 287 "The internal helper macro HELPER_REGISTER_BINARY_FP_VP is already defined!" 288#endif 289#define HELPER_REGISTER_BINARY_FP_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \ 290 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 2, 3, VPSD, -1) \ 291 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 292 VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_##OPSUFFIX) \ 293 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 294 VP_PROPERTY_BINARYOP \ 295 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 296 297// llvm.vp.fadd(x,y,mask,vlen) 298HELPER_REGISTER_BINARY_FP_VP(fadd, VP_FADD, FAdd, FADD) 299 300// llvm.vp.fsub(x,y,mask,vlen) 301HELPER_REGISTER_BINARY_FP_VP(fsub, VP_FSUB, FSub, FSUB) 302 303// llvm.vp.fmul(x,y,mask,vlen) 304HELPER_REGISTER_BINARY_FP_VP(fmul, VP_FMUL, FMul, FMUL) 305 306// llvm.vp.fdiv(x,y,mask,vlen) 307HELPER_REGISTER_BINARY_FP_VP(fdiv, VP_FDIV, FDiv, FDIV) 308 309// llvm.vp.frem(x,y,mask,vlen) 310HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem, FREM) 311 312#undef HELPER_REGISTER_BINARY_FP_VP 313 314// llvm.vp.fneg(x,mask,vlen) 315BEGIN_REGISTER_VP(vp_fneg, 1, 2, VP_FNEG, -1) 316VP_PROPERTY_FUNCTIONAL_OPC(FNeg) 317VP_PROPERTY_FUNCTIONAL_SDOPC(FNEG) 318END_REGISTER_VP(vp_fneg, VP_FNEG) 319 320// llvm.vp.fabs(x,mask,vlen) 321BEGIN_REGISTER_VP(vp_fabs, 1, 2, VP_FABS, -1) 322VP_PROPERTY_FUNCTIONAL_SDOPC(FABS) 323END_REGISTER_VP(vp_fabs, VP_FABS) 324 325// llvm.vp.sqrt(x,mask,vlen) 326BEGIN_REGISTER_VP(vp_sqrt, 1, 2, VP_SQRT, -1) 327VP_PROPERTY_FUNCTIONAL_SDOPC(FSQRT) 328END_REGISTER_VP(vp_sqrt, VP_SQRT) 329 330// llvm.vp.fma(x,y,z,mask,vlen) 331BEGIN_REGISTER_VP(vp_fma, 3, 4, VP_FMA, -1) 332VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_fma) 333VP_PROPERTY_FUNCTIONAL_SDOPC(FMA) 334END_REGISTER_VP(vp_fma, VP_FMA) 335 336// llvm.vp.fmuladd(x,y,z,mask,vlen) 337BEGIN_REGISTER_VP(vp_fmuladd, 3, 4, VP_FMULADD, -1) 338VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_fmuladd) 339VP_PROPERTY_FUNCTIONAL_SDOPC(FMAD) 340END_REGISTER_VP(vp_fmuladd, VP_FMULADD) 341 342// llvm.vp.copysign(x,y,mask,vlen) 343BEGIN_REGISTER_VP(vp_copysign, 2, 3, VP_FCOPYSIGN, -1) 344VP_PROPERTY_BINARYOP 345VP_PROPERTY_FUNCTIONAL_SDOPC(FCOPYSIGN) 346END_REGISTER_VP(vp_copysign, VP_FCOPYSIGN) 347 348// llvm.vp.minnum(x, y, mask,vlen) 349BEGIN_REGISTER_VP(vp_minnum, 2, 3, VP_FMINNUM, -1) 350VP_PROPERTY_BINARYOP 351VP_PROPERTY_FUNCTIONAL_SDOPC(FMINNUM) 352END_REGISTER_VP(vp_minnum, VP_FMINNUM) 353 354// llvm.vp.maxnum(x, y, mask,vlen) 355BEGIN_REGISTER_VP(vp_maxnum, 2, 3, VP_FMAXNUM, -1) 356VP_PROPERTY_BINARYOP 357VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXNUM) 358END_REGISTER_VP(vp_maxnum, VP_FMAXNUM) 359 360// llvm.vp.ceil(x,mask,vlen) 361BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1) 362VP_PROPERTY_FUNCTIONAL_SDOPC(FCEIL) 363END_REGISTER_VP(vp_ceil, VP_FCEIL) 364 365// llvm.vp.floor(x,mask,vlen) 366BEGIN_REGISTER_VP(vp_floor, 1, 2, VP_FFLOOR, -1) 367VP_PROPERTY_FUNCTIONAL_SDOPC(FFLOOR) 368END_REGISTER_VP(vp_floor, VP_FFLOOR) 369 370// llvm.vp.round(x,mask,vlen) 371BEGIN_REGISTER_VP(vp_round, 1, 2, VP_FROUND, -1) 372VP_PROPERTY_FUNCTIONAL_SDOPC(FROUND) 373END_REGISTER_VP(vp_round, VP_FROUND) 374 375// llvm.vp.roundeven(x,mask,vlen) 376BEGIN_REGISTER_VP(vp_roundeven, 1, 2, VP_FROUNDEVEN, -1) 377VP_PROPERTY_FUNCTIONAL_SDOPC(FROUNDEVEN) 378END_REGISTER_VP(vp_roundeven, VP_FROUNDEVEN) 379 380// llvm.vp.roundtozero(x,mask,vlen) 381BEGIN_REGISTER_VP(vp_roundtozero, 1, 2, VP_FROUNDTOZERO, -1) 382VP_PROPERTY_FUNCTIONAL_SDOPC(FTRUNC) 383END_REGISTER_VP(vp_roundtozero, VP_FROUNDTOZERO) 384 385// llvm.vp.rint(x,mask,vlen) 386BEGIN_REGISTER_VP(vp_rint, 1, 2, VP_FRINT, -1) 387VP_PROPERTY_FUNCTIONAL_SDOPC(FRINT) 388END_REGISTER_VP(vp_rint, VP_FRINT) 389 390// llvm.vp.nearbyint(x,mask,vlen) 391BEGIN_REGISTER_VP(vp_nearbyint, 1, 2, VP_FNEARBYINT, -1) 392VP_PROPERTY_FUNCTIONAL_SDOPC(FNEARBYINT) 393END_REGISTER_VP(vp_nearbyint, VP_FNEARBYINT) 394 395///// } Floating-Point Arithmetic 396 397///// Type Casts { 398// Specialized helper macro for type conversions. 399// <operation>(%x, %mask, %evl). 400#ifdef HELPER_REGISTER_FP_CAST_VP 401#error \ 402 "The internal helper macro HELPER_REGISTER_FP_CAST_VP is already defined!" 403#endif 404#define HELPER_REGISTER_FP_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC, HASROUND) \ 405 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \ 406 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 407 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 408 VP_PROPERTY_CONSTRAINEDFP(HASROUND, 1, experimental_constrained_##OPSUFFIX) \ 409 VP_PROPERTY_CASTOP \ 410 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 411 412// llvm.vp.fptoui(x,mask,vlen) 413HELPER_REGISTER_FP_CAST_VP(fptoui, VP_FP_TO_UINT, FPToUI, FP_TO_UINT, 0) 414 415// llvm.vp.fptosi(x,mask,vlen) 416HELPER_REGISTER_FP_CAST_VP(fptosi, VP_FP_TO_SINT, FPToSI, FP_TO_SINT, 0) 417 418// llvm.vp.uitofp(x,mask,vlen) 419HELPER_REGISTER_FP_CAST_VP(uitofp, VP_UINT_TO_FP, UIToFP, UINT_TO_FP, 1) 420 421// llvm.vp.sitofp(x,mask,vlen) 422HELPER_REGISTER_FP_CAST_VP(sitofp, VP_SINT_TO_FP, SIToFP, SINT_TO_FP, 1) 423 424// llvm.vp.fptrunc(x,mask,vlen) 425HELPER_REGISTER_FP_CAST_VP(fptrunc, VP_FP_ROUND, FPTrunc, FP_ROUND, 1) 426 427// llvm.vp.fpext(x,mask,vlen) 428HELPER_REGISTER_FP_CAST_VP(fpext, VP_FP_EXTEND, FPExt, FP_EXTEND, 0) 429 430#undef HELPER_REGISTER_FP_CAST_VP 431 432// Specialized helper macro for integer type conversions. 433// <operation>(%x, %mask, %evl). 434#ifdef HELPER_REGISTER_INT_CAST_VP 435#error \ 436 "The internal helper macro HELPER_REGISTER_INT_CAST_VP is already defined!" 437#endif 438#define HELPER_REGISTER_INT_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \ 439 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \ 440 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 441 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 442 VP_PROPERTY_CASTOP \ 443 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 444 445// llvm.vp.trunc(x,mask,vlen) 446HELPER_REGISTER_INT_CAST_VP(trunc, VP_TRUNCATE, Trunc, TRUNCATE) 447 448// llvm.vp.zext(x,mask,vlen) 449HELPER_REGISTER_INT_CAST_VP(zext, VP_ZERO_EXTEND, ZExt, ZERO_EXTEND) 450 451// llvm.vp.sext(x,mask,vlen) 452HELPER_REGISTER_INT_CAST_VP(sext, VP_SIGN_EXTEND, SExt, SIGN_EXTEND) 453 454// llvm.vp.ptrtoint(x,mask,vlen) 455BEGIN_REGISTER_VP(vp_ptrtoint, 1, 2, VP_PTRTOINT, -1) 456VP_PROPERTY_FUNCTIONAL_OPC(PtrToInt) 457VP_PROPERTY_CASTOP 458END_REGISTER_VP(vp_ptrtoint, VP_PTRTOINT) 459 460// llvm.vp.inttoptr(x,mask,vlen) 461BEGIN_REGISTER_VP(vp_inttoptr, 1, 2, VP_INTTOPTR, -1) 462VP_PROPERTY_FUNCTIONAL_OPC(IntToPtr) 463VP_PROPERTY_CASTOP 464END_REGISTER_VP(vp_inttoptr, VP_INTTOPTR) 465 466#undef HELPER_REGISTER_INT_CAST_VP 467 468///// } Type Casts 469 470///// Comparisons { 471 472// VP_SETCC (ISel only) 473BEGIN_REGISTER_VP_SDNODE(VP_SETCC, 0, vp_setcc, 3, 4) 474END_REGISTER_VP_SDNODE(VP_SETCC) 475 476// llvm.vp.fcmp(x,y,cc,mask,vlen) 477BEGIN_REGISTER_VP_INTRINSIC(vp_fcmp, 3, 4) 478HELPER_MAP_VPID_TO_VPSD(vp_fcmp, VP_SETCC) 479VP_PROPERTY_FUNCTIONAL_OPC(FCmp) 480VP_PROPERTY_CMP(2, true) 481VP_PROPERTY_CONSTRAINEDFP(0, 1, experimental_constrained_fcmp) 482END_REGISTER_VP_INTRINSIC(vp_fcmp) 483 484// llvm.vp.icmp(x,y,cc,mask,vlen) 485BEGIN_REGISTER_VP_INTRINSIC(vp_icmp, 3, 4) 486HELPER_MAP_VPID_TO_VPSD(vp_icmp, VP_SETCC) 487VP_PROPERTY_FUNCTIONAL_OPC(ICmp) 488VP_PROPERTY_CMP(2, false) 489END_REGISTER_VP_INTRINSIC(vp_icmp) 490 491///// } Comparisons 492 493///// Memory Operations { 494// llvm.vp.store(val,ptr,mask,vlen) 495BEGIN_REGISTER_VP_INTRINSIC(vp_store, 2, 3) 496// chain = VP_STORE chain,val,base,offset,mask,evl 497BEGIN_REGISTER_VP_SDNODE(VP_STORE, 1, vp_store, 4, 5) 498HELPER_MAP_VPID_TO_VPSD(vp_store, VP_STORE) 499VP_PROPERTY_FUNCTIONAL_OPC(Store) 500VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_store) 501VP_PROPERTY_MEMOP(1, 0) 502END_REGISTER_VP(vp_store, VP_STORE) 503 504// llvm.experimental.vp.strided.store(val,ptr,stride,mask,vlen) 505BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_store, 3, 4) 506// chain = EXPERIMENTAL_VP_STRIDED_STORE chain,val,base,offset,stride,mask,evl 507BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_STORE, 1, experimental_vp_strided_store, 5, 6) 508HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE) 509VP_PROPERTY_MEMOP(1, 0) 510END_REGISTER_VP(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE) 511 512// llvm.vp.scatter(ptr,val,mask,vlen) 513BEGIN_REGISTER_VP_INTRINSIC(vp_scatter, 2, 3) 514// chain = VP_SCATTER chain,val,base,indices,scale,mask,evl 515BEGIN_REGISTER_VP_SDNODE(VP_SCATTER, 1, vp_scatter, 5, 6) 516HELPER_MAP_VPID_TO_VPSD(vp_scatter, VP_SCATTER) 517VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_scatter) 518VP_PROPERTY_MEMOP(1, 0) 519END_REGISTER_VP(vp_scatter, VP_SCATTER) 520 521// llvm.vp.load(ptr,mask,vlen) 522BEGIN_REGISTER_VP_INTRINSIC(vp_load, 1, 2) 523// val,chain = VP_LOAD chain,base,offset,mask,evl 524BEGIN_REGISTER_VP_SDNODE(VP_LOAD, -1, vp_load, 3, 4) 525HELPER_MAP_VPID_TO_VPSD(vp_load, VP_LOAD) 526VP_PROPERTY_FUNCTIONAL_OPC(Load) 527VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_load) 528VP_PROPERTY_MEMOP(0, std::nullopt) 529END_REGISTER_VP(vp_load, VP_LOAD) 530 531// llvm.experimental.vp.strided.load(ptr,stride,mask,vlen) 532BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_load, 2, 3) 533// chain = EXPERIMENTAL_VP_STRIDED_LOAD chain,base,offset,stride,mask,evl 534BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_LOAD, -1, experimental_vp_strided_load, 4, 5) 535HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD) 536VP_PROPERTY_MEMOP(0, std::nullopt) 537END_REGISTER_VP(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD) 538 539// llvm.vp.gather(ptr,mask,vlen) 540BEGIN_REGISTER_VP_INTRINSIC(vp_gather, 1, 2) 541// val,chain = VP_GATHER chain,base,indices,scale,mask,evl 542BEGIN_REGISTER_VP_SDNODE(VP_GATHER, -1, vp_gather, 4, 5) 543HELPER_MAP_VPID_TO_VPSD(vp_gather, VP_GATHER) 544VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_gather) 545VP_PROPERTY_MEMOP(0, std::nullopt) 546END_REGISTER_VP(vp_gather, VP_GATHER) 547 548///// } Memory Operations 549 550///// Reductions { 551 552// Specialized helper macro for VP reductions (%start, %x, %mask, %evl). 553#ifdef HELPER_REGISTER_REDUCTION_VP 554#error \ 555 "The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!" 556#endif 557#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN) \ 558 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, 1) \ 559 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \ 560 VP_PROPERTY_REDUCTION(0, 1) \ 561 END_REGISTER_VP(VPID, VPSD) 562 563// llvm.vp.reduce.add(start,x,mask,vlen) 564HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD, 565 experimental_vector_reduce_add) 566 567// llvm.vp.reduce.mul(start,x,mask,vlen) 568HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL, 569 experimental_vector_reduce_mul) 570 571// llvm.vp.reduce.and(start,x,mask,vlen) 572HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND, 573 experimental_vector_reduce_and) 574 575// llvm.vp.reduce.or(start,x,mask,vlen) 576HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR, 577 experimental_vector_reduce_or) 578 579// llvm.vp.reduce.xor(start,x,mask,vlen) 580HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR, 581 experimental_vector_reduce_xor) 582 583// llvm.vp.reduce.smax(start,x,mask,vlen) 584HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX, 585 experimental_vector_reduce_smax) 586 587// llvm.vp.reduce.smin(start,x,mask,vlen) 588HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN, 589 experimental_vector_reduce_smin) 590 591// llvm.vp.reduce.umax(start,x,mask,vlen) 592HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX, 593 experimental_vector_reduce_umax) 594 595// llvm.vp.reduce.umin(start,x,mask,vlen) 596HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN, 597 experimental_vector_reduce_umin) 598 599// llvm.vp.reduce.fmax(start,x,mask,vlen) 600HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX, 601 experimental_vector_reduce_fmax) 602 603// llvm.vp.reduce.fmin(start,x,mask,vlen) 604HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN, 605 experimental_vector_reduce_fmin) 606 607#undef HELPER_REGISTER_REDUCTION_VP 608 609// Specialized helper macro for VP reductions as above but with two forms: 610// sequential and reassociative. These manifest as the presence of 'reassoc' 611// fast-math flags in the IR and as two distinct ISD opcodes in the 612// SelectionDAG. 613// Note we by default map from the VP intrinsic to the SEQ ISD opcode, which 614// can then be relaxed to the non-SEQ ISD opcode if the 'reassoc' flag is set. 615#ifdef HELPER_REGISTER_REDUCTION_SEQ_VP 616#error \ 617 "The internal helper macro HELPER_REGISTER_REDUCTION_SEQ_VP is already defined!" 618#endif 619#define HELPER_REGISTER_REDUCTION_SEQ_VP(VPID, VPSD, SEQ_VPSD, INTRIN) \ 620 BEGIN_REGISTER_VP_INTRINSIC(VPID, 2, 3) \ 621 BEGIN_REGISTER_VP_SDNODE(VPSD, 1, VPID, 2, 3) \ 622 VP_PROPERTY_REDUCTION(0, 1) \ 623 END_REGISTER_VP_SDNODE(VPSD) \ 624 BEGIN_REGISTER_VP_SDNODE(SEQ_VPSD, 1, VPID, 2, 3) \ 625 HELPER_MAP_VPID_TO_VPSD(VPID, SEQ_VPSD) \ 626 VP_PROPERTY_REDUCTION(0, 1) \ 627 END_REGISTER_VP_SDNODE(SEQ_VPSD) \ 628 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \ 629 END_REGISTER_VP_INTRINSIC(VPID) 630 631// llvm.vp.reduce.fadd(start,x,mask,vlen) 632HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fadd, VP_REDUCE_FADD, 633 VP_REDUCE_SEQ_FADD, 634 experimental_vector_reduce_fadd) 635 636// llvm.vp.reduce.fmul(start,x,mask,vlen) 637HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL, 638 VP_REDUCE_SEQ_FMUL, 639 experimental_vector_reduce_fmul) 640 641#undef HELPER_REGISTER_REDUCTION_SEQ_VP 642 643///// } Reduction 644 645///// Shuffles { 646 647// The mask 'cond' operand of llvm.vp.select and llvm.vp.merge are not reported 648// as masks with the BEGIN_REGISTER_VP_* macros. This is because, unlike other 649// VP intrinsics, these two have a defined result on lanes where the mask is 650// false. 651// 652// llvm.vp.select(cond,on_true,on_false,vlen) 653BEGIN_REGISTER_VP(vp_select, std::nullopt, 3, VP_SELECT, -1) 654VP_PROPERTY_FUNCTIONAL_OPC(Select) 655VP_PROPERTY_FUNCTIONAL_SDOPC(VSELECT) 656END_REGISTER_VP(vp_select, VP_SELECT) 657 658// llvm.vp.merge(cond,on_true,on_false,pivot) 659BEGIN_REGISTER_VP(vp_merge, std::nullopt, 3, VP_MERGE, -1) 660END_REGISTER_VP(vp_merge, VP_MERGE) 661 662BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1) 663END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE) 664 665///// } Shuffles 666 667#undef BEGIN_REGISTER_VP 668#undef BEGIN_REGISTER_VP_INTRINSIC 669#undef BEGIN_REGISTER_VP_SDNODE 670#undef END_REGISTER_VP 671#undef END_REGISTER_VP_INTRINSIC 672#undef END_REGISTER_VP_SDNODE 673#undef HELPER_MAP_VPID_TO_VPSD 674#undef VP_PROPERTY_BINARYOP 675#undef VP_PROPERTY_CASTOP 676#undef VP_PROPERTY_CMP 677#undef VP_PROPERTY_CONSTRAINEDFP 678#undef VP_PROPERTY_FUNCTIONAL_INTRINSIC 679#undef VP_PROPERTY_FUNCTIONAL_OPC 680#undef VP_PROPERTY_FUNCTIONAL_SDOPC 681#undef VP_PROPERTY_MEMOP 682#undef VP_PROPERTY_REDUCTION 683