1//===-- IR/VPIntrinsics.def - Describes llvm.vp.* Intrinsics -*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file contains descriptions of the various Vector Predication intrinsics. 10// This is used as a central place for enumerating the different instructions 11// and should eventually be the place to put comments about the instructions. 12// 13//===----------------------------------------------------------------------===// 14 15// NOTE: NO INCLUDE GUARD DESIRED! 16 17// Provide definitions of macros so that users of this file do not have to 18// define everything to use it... 19// 20// Register a VP intrinsic and begin its property scope. 21// All VP intrinsic scopes are top level, ie it is illegal to place a 22// BEGIN_REGISTER_VP_INTRINSIC within a VP intrinsic scope. 23// \p VPID The VP intrinsic id. 24// \p MASKPOS The mask operand position. 25// \p EVLPOS The explicit vector length operand position. 26#ifndef BEGIN_REGISTER_VP_INTRINSIC 27#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) 28#endif 29 30// End the property scope of a VP intrinsic. 31#ifndef END_REGISTER_VP_INTRINSIC 32#define END_REGISTER_VP_INTRINSIC(VPID) 33#endif 34 35// Register a new VP SDNode and begin its property scope. 36// When the SDNode scope is nested within a VP intrinsic scope, it is 37// implicitly registered as the canonical SDNode for this VP intrinsic. There 38// is one VP intrinsic that maps directly to one SDNode that goes by the 39// same name. Since the operands are also the same, we open the property 40// scopes for both the VPIntrinsic and the SDNode at once. 41// \p VPSD The SelectionDAG Node id (eg VP_ADD). 42// \p LEGALPOS The operand position of the SDNode that is used for legalizing. 43// If LEGALPOS < 0, then the return type given by 44// TheNode->getValueType(-1-LEGALPOS) is used. 45// \p TDNAME The name of the TableGen definition of this SDNode. 46// \p MASKPOS The mask operand position. 47// \p EVLPOS The explicit vector length operand position. 48#ifndef BEGIN_REGISTER_VP_SDNODE 49#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) 50#endif 51 52// End the property scope of a new VP SDNode. 53#ifndef END_REGISTER_VP_SDNODE 54#define END_REGISTER_VP_SDNODE(VPSD) 55#endif 56 57// Helper macros for the common "1:1 - Intrinsic : SDNode" case. 58// 59// There is one VP intrinsic that maps directly to one SDNode that goes by the 60// same name. Since the operands are also the same, we open the property 61// scopes for both the VPIntrinsic and the SDNode at once. 62// 63// \p VPID The canonical name (eg `vp_add`, which at the same time is the 64// name of the intrinsic and the TableGen def of the SDNode). 65// \p MASKPOS The mask operand position. 66// \p EVLPOS The explicit vector length operand position. 67// \p VPSD The SelectionDAG Node id (eg VP_ADD). 68// \p LEGALPOS The operand position of the SDNode that is used for legalizing 69// this SDNode. This can be `-1`, in which case the return type of 70// the SDNode is used. 71#define BEGIN_REGISTER_VP(VPID, MASKPOS, EVLPOS, VPSD, LEGALPOS) \ 72 BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) \ 73 BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, VPID, MASKPOS, EVLPOS) 74 75#define END_REGISTER_VP(VPID, VPSD) \ 76 END_REGISTER_VP_INTRINSIC(VPID) \ 77 END_REGISTER_VP_SDNODE(VPSD) 78 79// The following macros attach properties to the scope they are placed in. This 80// assigns the property to the VP Intrinsic and/or SDNode that belongs to the 81// scope. 82// 83// Property Macros { 84 85// The intrinsic and/or SDNode has the same function as this LLVM IR Opcode. 86// \p OPC The opcode of the instruction with the same function. 87#ifndef VP_PROPERTY_FUNCTIONAL_OPC 88#define VP_PROPERTY_FUNCTIONAL_OPC(OPC) 89#endif 90 91// Whether the intrinsic may have a rounding mode or exception behavior operand 92// bundle. 93// \p HASROUND '1' if the intrinsic can have a rounding mode operand bundle, 94// '0' otherwise. 95// \p HASEXCEPT '1' if the intrinsic can have an exception behavior operand 96// bundle, '0' otherwise. 97// \p INTRINID The constrained fp intrinsic this VP intrinsic corresponds to. 98#ifndef VP_PROPERTY_CONSTRAINEDFP 99#define VP_PROPERTY_CONSTRAINEDFP(HASROUND, HASEXCEPT, INTRINID) 100#endif 101 102// Map this VP intrinsic to its canonical functional intrinsic. 103// \p INTRIN The non-VP intrinsics with the same function. 104#ifndef VP_PROPERTY_FUNCTIONAL_INTRINSIC 105#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) 106#endif 107 108// This VP Intrinsic is a memory operation 109// The pointer arg is at POINTERPOS and the data arg is at DATAPOS. 110#ifndef VP_PROPERTY_MEMOP 111#define VP_PROPERTY_MEMOP(POINTERPOS, DATAPOS) 112#endif 113 114// Map this VP reduction intrinsic to its reduction operand positions. 115#ifndef VP_PROPERTY_REDUCTION 116#define VP_PROPERTY_REDUCTION(STARTPOS, VECTORPOS) 117#endif 118 119// A property to infer VP binary-op SDNode opcodes automatically. 120#ifndef VP_PROPERTY_BINARYOP 121#define VP_PROPERTY_BINARYOP 122#endif 123 124/// } Property Macros 125 126///// Integer Arithmetic { 127 128// Specialized helper macro for integer binary operators (%x, %y, %mask, %evl). 129#ifdef HELPER_REGISTER_BINARY_INT_VP 130#error \ 131 "The internal helper macro HELPER_REGISTER_BINARY_INT_VP is already defined!" 132#endif 133#define HELPER_REGISTER_BINARY_INT_VP(VPID, VPSD, IROPC) \ 134 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, -1) \ 135 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 136 VP_PROPERTY_BINARYOP \ 137 END_REGISTER_VP(VPID, VPSD) 138 139// llvm.vp.add(x,y,mask,vlen) 140HELPER_REGISTER_BINARY_INT_VP(vp_add, VP_ADD, Add) 141 142// llvm.vp.and(x,y,mask,vlen) 143HELPER_REGISTER_BINARY_INT_VP(vp_and, VP_AND, And) 144 145// llvm.vp.ashr(x,y,mask,vlen) 146HELPER_REGISTER_BINARY_INT_VP(vp_ashr, VP_ASHR, AShr) 147 148// llvm.vp.lshr(x,y,mask,vlen) 149HELPER_REGISTER_BINARY_INT_VP(vp_lshr, VP_LSHR, LShr) 150 151// llvm.vp.mul(x,y,mask,vlen) 152HELPER_REGISTER_BINARY_INT_VP(vp_mul, VP_MUL, Mul) 153 154// llvm.vp.or(x,y,mask,vlen) 155HELPER_REGISTER_BINARY_INT_VP(vp_or, VP_OR, Or) 156 157// llvm.vp.sdiv(x,y,mask,vlen) 158HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv) 159 160// llvm.vp.shl(x,y,mask,vlen) 161HELPER_REGISTER_BINARY_INT_VP(vp_shl, VP_SHL, Shl) 162 163// llvm.vp.srem(x,y,mask,vlen) 164HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem) 165 166// llvm.vp.sub(x,y,mask,vlen) 167HELPER_REGISTER_BINARY_INT_VP(vp_sub, VP_SUB, Sub) 168 169// llvm.vp.udiv(x,y,mask,vlen) 170HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv) 171 172// llvm.vp.urem(x,y,mask,vlen) 173HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem) 174 175// llvm.vp.xor(x,y,mask,vlen) 176HELPER_REGISTER_BINARY_INT_VP(vp_xor, VP_XOR, Xor) 177 178#undef HELPER_REGISTER_BINARY_INT_VP 179 180///// } Integer Arithmetic 181 182///// Floating-Point Arithmetic { 183 184// Specialized helper macro for floating-point binary operators 185// <operation>(%x, %y, %mask, %evl). 186#ifdef HELPER_REGISTER_BINARY_FP_VP 187#error \ 188 "The internal helper macro HELPER_REGISTER_BINARY_FP_VP is already defined!" 189#endif 190#define HELPER_REGISTER_BINARY_FP_VP(OPSUFFIX, VPSD, IROPC) \ 191 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 2, 3, VPSD, -1) \ 192 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 193 VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_##OPSUFFIX) \ 194 VP_PROPERTY_BINARYOP \ 195 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 196 197// llvm.vp.fadd(x,y,mask,vlen) 198HELPER_REGISTER_BINARY_FP_VP(fadd, VP_FADD, FAdd) 199 200// llvm.vp.fsub(x,y,mask,vlen) 201HELPER_REGISTER_BINARY_FP_VP(fsub, VP_FSUB, FSub) 202 203// llvm.vp.fmul(x,y,mask,vlen) 204HELPER_REGISTER_BINARY_FP_VP(fmul, VP_FMUL, FMul) 205 206// llvm.vp.fdiv(x,y,mask,vlen) 207HELPER_REGISTER_BINARY_FP_VP(fdiv, VP_FDIV, FDiv) 208 209// llvm.vp.frem(x,y,mask,vlen) 210HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem) 211 212#undef HELPER_REGISTER_BINARY_FP_VP 213 214///// } Floating-Point Arithmetic 215 216///// Memory Operations { 217// llvm.vp.store(val,ptr,mask,vlen) 218BEGIN_REGISTER_VP_INTRINSIC(vp_store, 2, 3) 219// chain = VP_STORE chain,val,base,offset,mask,evl 220BEGIN_REGISTER_VP_SDNODE(VP_STORE, 0, vp_store, 4, 5) 221VP_PROPERTY_FUNCTIONAL_OPC(Store) 222VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_store) 223VP_PROPERTY_MEMOP(1, 0) 224END_REGISTER_VP(vp_store, VP_STORE) 225 226// llvm.vp.scatter(ptr,val,mask,vlen) 227BEGIN_REGISTER_VP_INTRINSIC(vp_scatter, 2, 3) 228// chain = VP_SCATTER chain,val,base,indices,scale,mask,evl 229BEGIN_REGISTER_VP_SDNODE(VP_SCATTER, -1, vp_scatter, 5, 6) 230VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_scatter) 231VP_PROPERTY_MEMOP(1, 0) 232END_REGISTER_VP(vp_scatter, VP_SCATTER) 233 234// llvm.vp.load(ptr,mask,vlen) 235BEGIN_REGISTER_VP_INTRINSIC(vp_load, 1, 2) 236// val,chain = VP_LOAD chain,base,offset,mask,evl 237BEGIN_REGISTER_VP_SDNODE(VP_LOAD, -1, vp_load, 3, 4) 238VP_PROPERTY_FUNCTIONAL_OPC(Load) 239VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_load) 240VP_PROPERTY_MEMOP(0, None) 241END_REGISTER_VP(vp_load, VP_LOAD) 242 243// llvm.vp.gather(ptr,mask,vlen) 244BEGIN_REGISTER_VP_INTRINSIC(vp_gather, 1, 2) 245// val,chain = VP_GATHER chain,base,indices,scale,mask,evl 246BEGIN_REGISTER_VP_SDNODE(VP_GATHER, -1, vp_gather, 4, 5) 247VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_gather) 248VP_PROPERTY_MEMOP(0, None) 249END_REGISTER_VP(vp_gather, VP_GATHER) 250 251///// } Memory Operations 252 253///// Reductions { 254 255// Specialized helper macro for VP reductions (%start, %x, %mask, %evl). 256#ifdef HELPER_REGISTER_REDUCTION_VP 257#error \ 258 "The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!" 259#endif 260#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN) \ 261 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, -1) \ 262 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \ 263 VP_PROPERTY_REDUCTION(0, 1) \ 264 END_REGISTER_VP(VPID, VPSD) 265 266// llvm.vp.reduce.add(start,x,mask,vlen) 267HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD, 268 experimental_vector_reduce_add) 269 270// llvm.vp.reduce.mul(start,x,mask,vlen) 271HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL, 272 experimental_vector_reduce_mul) 273 274// llvm.vp.reduce.and(start,x,mask,vlen) 275HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND, 276 experimental_vector_reduce_and) 277 278// llvm.vp.reduce.or(start,x,mask,vlen) 279HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR, 280 experimental_vector_reduce_or) 281 282// llvm.vp.reduce.xor(start,x,mask,vlen) 283HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR, 284 experimental_vector_reduce_xor) 285 286// llvm.vp.reduce.smax(start,x,mask,vlen) 287HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX, 288 experimental_vector_reduce_smax) 289 290// llvm.vp.reduce.smin(start,x,mask,vlen) 291HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN, 292 experimental_vector_reduce_smin) 293 294// llvm.vp.reduce.umax(start,x,mask,vlen) 295HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX, 296 experimental_vector_reduce_umax) 297 298// llvm.vp.reduce.umin(start,x,mask,vlen) 299HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN, 300 experimental_vector_reduce_umin) 301 302// llvm.vp.reduce.fmax(start,x,mask,vlen) 303HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX, 304 experimental_vector_reduce_fmax) 305 306// llvm.vp.reduce.fmin(start,x,mask,vlen) 307HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN, 308 experimental_vector_reduce_fmin) 309 310#undef HELPER_REGISTER_REDUCTION_VP 311 312// Specialized helper macro for VP reductions as above but with two forms: 313// sequential and reassociative. These manifest as the presence of 'reassoc' 314// fast-math flags in the IR and as two distinct ISD opcodes in the 315// SelectionDAG. 316#ifdef HELPER_REGISTER_REDUCTION_SEQ_VP 317#error \ 318 "The internal helper macro HELPER_REGISTER_REDUCTION_SEQ_VP is already defined!" 319#endif 320#define HELPER_REGISTER_REDUCTION_SEQ_VP(VPID, VPSD, SEQ_VPSD, INTRIN) \ 321 BEGIN_REGISTER_VP_INTRINSIC(VPID, 2, 3) \ 322 BEGIN_REGISTER_VP_SDNODE(VPSD, -1, VPID, 2, 3) \ 323 VP_PROPERTY_REDUCTION(0, 1) \ 324 END_REGISTER_VP_SDNODE(VPSD) \ 325 BEGIN_REGISTER_VP_SDNODE(SEQ_VPSD, -1, VPID, 2, 3) \ 326 VP_PROPERTY_REDUCTION(0, 1) \ 327 END_REGISTER_VP_SDNODE(SEQ_VPSD) \ 328 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \ 329 END_REGISTER_VP_INTRINSIC(VPID) 330 331// llvm.vp.reduce.fadd(start,x,mask,vlen) 332HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fadd, VP_REDUCE_FADD, 333 VP_REDUCE_SEQ_FADD, 334 experimental_vector_reduce_fadd) 335 336// llvm.vp.reduce.fmul(start,x,mask,vlen) 337HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL, 338 VP_REDUCE_SEQ_FMUL, 339 experimental_vector_reduce_fmul) 340 341#undef HELPER_REGISTER_REDUCTION_SEQ_VP 342 343///// } Reduction 344 345///// Shuffles { 346 347// llvm.vp.select(mask,on_true,on_false,vlen) 348BEGIN_REGISTER_VP(vp_select, 0, 3, VP_SELECT, -1) 349VP_PROPERTY_FUNCTIONAL_OPC(Select) 350END_REGISTER_VP(vp_select, VP_SELECT) 351 352// llvm.vp.merge(mask,on_true,on_false,pivot) 353BEGIN_REGISTER_VP(vp_merge, 0, 3, VP_MERGE, -1) 354END_REGISTER_VP(vp_merge, VP_MERGE) 355 356BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1) 357END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE) 358 359///// } Shuffles 360 361#undef BEGIN_REGISTER_VP 362#undef BEGIN_REGISTER_VP_INTRINSIC 363#undef BEGIN_REGISTER_VP_SDNODE 364#undef END_REGISTER_VP 365#undef END_REGISTER_VP_INTRINSIC 366#undef END_REGISTER_VP_SDNODE 367#undef VP_PROPERTY_BINARYOP 368#undef VP_PROPERTY_CONSTRAINEDFP 369#undef VP_PROPERTY_FUNCTIONAL_INTRINSIC 370#undef VP_PROPERTY_FUNCTIONAL_OPC 371#undef VP_PROPERTY_MEMOP 372#undef VP_PROPERTY_REDUCTION 373