1//==--- riscv_vector.td - RISC-V V-ext Builtin function list --------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the builtins for RISC-V V-extension. See: 10// 11// https://github.com/riscv/rvv-intrinsic-doc 12// 13//===----------------------------------------------------------------------===// 14 15//===----------------------------------------------------------------------===// 16// Instruction definitions 17//===----------------------------------------------------------------------===// 18// Each record of the class RVVBuiltin defines a collection of builtins (i.e. 19// "def vadd : RVVBuiltin" will be used to define things like "vadd_vv_i32m1", 20// "vadd_vv_i32m2", etc). 21// 22// The elements of this collection are defined by an instantiation process the 23// range of which is specified by the cross product of the LMUL attribute and 24// every element in the attribute TypeRange. By default builtins have LMUL = [1, 25// 2, 4, 8, 1/2, 1/4, 1/8] so the process is repeated 7 times. In tablegen we 26// use the Log2LMUL [0, 1, 2, 3, -1, -2, -3] to represent the LMUL. 27// 28// LMUL represents the fact that the types of values used by that builtin are 29// values generated by instructions that are executed under that LMUL. However, 30// this does not mean the builtin is necessarily lowered into an instruction 31// that executes under the specified LMUL. An example where this happens are 32// loads and stores of masks. A mask like `vbool8_t` can be generated, for 33// instance, by comparing two `__rvv_int8m1_t` (this is LMUL=1) or comparing two 34// `__rvv_int16m2_t` (this is LMUL=2). The actual load or store, however, will 35// be performed under LMUL=1 because mask registers are not grouped. 36// 37// TypeRange is a non-empty sequence of basic types: 38// 39// c: int8_t (i8) 40// s: int16_t (i16) 41// i: int32_t (i32) 42// l: int64_t (i64) 43// x: float16_t (half) 44// f: float32_t (float) 45// d: float64_t (double) 46// 47// This way, given an LMUL, a record with a TypeRange "sil" will cause the 48// definition of 3 builtins. Each type "t" in the TypeRange (in this example 49// they are int16_t, int32_t, int64_t) is used as a parameter that drives the 50// definition of that particular builtin (for the given LMUL). 51// 52// During the instantiation, types can be transformed or modified using type 53// transformers. Given a type "t" the following primitive type transformers can 54// be applied to it to yield another type. 55// 56// e: type of "t" as is (identity) 57// v: computes a vector type whose element type is "t" for the current LMUL 58// w: computes a vector type identical to what 'v' computes except for the 59// element type which is twice as wide as the element type of 'v' 60// q: computes a vector type identical to what 'v' computes except for the 61// element type which is four times as wide as the element type of 'v' 62// o: computes a vector type identical to what 'v' computes except for the 63// element type which is eight times as wide as the element type of 'v' 64// m: computes a vector type identical to what 'v' computes except for the 65// element type which is bool 66// 0: void type, ignores "t" 67// z: size_t, ignores "t" 68// t: ptrdiff_t, ignores "t" 69// u: unsigned long, ignores "t" 70// l: long, ignores "t" 71// 72// So for instance if t is "i", i.e. int, then "e" will yield int again. "v" 73// will yield an RVV vector type (assume LMUL=1), so __rvv_int32m1_t. 74// Accordingly "w" would yield __rvv_int64m2_t. 75// 76// A type transformer can be prefixed by other non-primitive type transformers. 77// 78// P: constructs a pointer to the current type 79// C: adds const to the type 80// K: requires the integer type to be a constant expression 81// U: given an integer type or vector type, computes its unsigned variant 82// I: given a vector type, compute the vector type with integer type 83// elements of the same width 84// F: given a vector type, compute the vector type with floating-point type 85// elements of the same width 86// S: given a vector type, computes its equivalent one for LMUL=1. This is a 87// no-op if the vector was already LMUL=1 88// (Log2EEW:Value): Log2EEW value could be 3/4/5/6 (8/16/32/64), given a 89// vector type (SEW and LMUL) and EEW (8/16/32/64), computes its 90// equivalent integer vector type with EEW and corresponding ELMUL (elmul = 91// (eew/sew) * lmul). For example, vector type is __rvv_float16m4 92// (SEW=16, LMUL=4) and Log2EEW is 3 (EEW=8), and then equivalent vector 93// type is __rvv_uint8m2_t (elmul=(8/16)*4 = 2). Ignore to define a new 94// builtins if its equivalent type has illegal lmul. 95// (FixedSEW:Value): Given a vector type (SEW and LMUL), and computes another 96// vector type which only changed SEW as given value. Ignore to define a new 97// builtin if its equivalent type has illegal lmul or the SEW does not changed. 98// (SFixedLog2LMUL:Value): Smaller Fixed Log2LMUL. Given a vector type (SEW 99// and LMUL), and computes another vector type which only changed LMUL as 100// given value. The new LMUL should be smaller than the old one. Ignore to 101// define a new builtin if its equivalent type has illegal lmul. 102// (LFixedLog2LMUL:Value): Larger Fixed Log2LMUL. Given a vector type (SEW 103// and LMUL), and computes another vector type which only changed LMUL as 104// given value. The new LMUL should be larger than the old one. Ignore to 105// define a new builtin if its equivalent type has illegal lmul. 106// 107// Following with the example above, if t is "i", then "Ue" will yield unsigned 108// int and "Fv" will yield __rvv_float32m1_t (again assuming LMUL=1), Fw would 109// yield __rvv_float64m2_t, etc. 110// 111// Each builtin is then defined by applying each type in TypeRange against the 112// sequence of type transformers described in Suffix and Prototype. 113// 114// The name of the builtin is defined by the Name attribute (which defaults to 115// the name of the class) appended (separated with an underscore) the Suffix 116// attribute. For instance with Name="foo", Suffix = "v" and TypeRange = "il", 117// the builtin generated will be __builtin_rvv_foo_i32m1 and 118// __builtin_rvv_foo_i64m1 (under LMUL=1). If Suffix contains more than one 119// type transformer (say "vv") each of the types is separated with an 120// underscore as in "__builtin_rvv_foo_i32m1_i32m1". 121// 122// The C/C++ prototype of the builtin is defined by the Prototype attribute. 123// Prototype is a non-empty sequence of type transformers, the first of which 124// is the return type of the builtin and the rest are the parameters of the 125// builtin, in order. For instance if Prototype is "wvv" and TypeRange is "si" 126// a first builtin will have type 127// __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t) and the second builtin 128// will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t) (again 129// under LMUL=1). 130// 131// There are a number of attributes that are used to constraint the number and 132// shape of the builtins generated. Refer to the comments below for them. 133class RVVBuiltin<string suffix, string prototype, string type_range, 134 string mangled_suffix = ""> { 135 // Base name that will be prepended in __builtin_rvv_ and appended the 136 // computed Suffix. 137 string Name = NAME; 138 139 // If not empty, each instantiated builtin will have this appended after an 140 // underscore (_). It is instantiated like Prototype. 141 string Suffix = suffix; 142 143 // If empty, default MangledName is sub string of `Name` which end of first 144 // '_'. For example, the default mangled name is `vadd` for Name `vadd_vv`. 145 // It's used for describe some special naming cases. 146 string MangledName = ""; 147 148 // If not empty, each MangledName will have this appended after an 149 // underscore (_). It is instantiated like Prototype. 150 string MangledSuffix = mangled_suffix; 151 152 // The different variants of the builtin, parameterised with a type. 153 string TypeRange = type_range; 154 155 // We use each type described in TypeRange and LMUL with prototype to 156 // instantiate a specific element of the set of builtins being defined. 157 // Prototype attribute defines the C/C++ prototype of the builtin. It is a 158 // non-empty sequence of type transformers, the first of which is the return 159 // type of the builtin and the rest are the parameters of the builtin, in 160 // order. For instance if Prototype is "wvv", TypeRange is "si" and LMUL=1, a 161 // first builtin will have type 162 // __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t), and the second builtin 163 // will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t). 164 string Prototype = prototype; 165 166 // This builtin has a masked form. 167 bit HasMask = true; 168 169 // If HasMask, this flag states that this builtin has a maskedoff operand. It 170 // is always the first operand in builtin and IR intrinsic. 171 bit HasMaskedOffOperand = true; 172 173 // This builtin has a granted vector length parameter. 174 bit HasVL = true; 175 176 // There are several cases for specifying tail policy. 177 // 1. Add tail policy argument to masked intrinsics. It may have the maskedoff 178 // argument or not. 179 // * Have the maskedoff argument: (HasPolicy = true, HasMaskedOffOperand = true) 180 // Ex: vadd_vv_i8m1_mt(mask, maskedoff, op1, op2, vl, ta); 181 // * Do not have the maskedoff argument: (HasPolicy = true, HasMaskedOffOperand = false) 182 // Ex: vmacc_vv_i8m1_mt(mask, vd, vs1, vs2, vl, ta); 183 // 2. Add dest argument for no mask intrinsics. (TODO) 184 // Ex: vmv_v_x_i8m1_t(dest, src, vl); 185 // 3. Always tail agnostic. (HasPolicy = false) 186 // Ex: vmseq_vv_i8m1_b8_m(mask, maskedoff, op1, op2, vl); 187 // The tail policy argument is located at the last position. 188 bit HasPolicy = true; 189 190 // This builtin supports non-masked function overloading api. 191 // All masked operations support overloading api. 192 bit HasNoMaskedOverloaded = true; 193 194 // This builtin is valid for the given Log2LMULs. 195 list<int> Log2LMUL = [0, 1, 2, 3, -1, -2, -3]; 196 197 // Manual code in clang codegen riscv_vector_builtin_cg.inc 198 code ManualCodegen = [{}]; 199 code ManualCodegenMask = [{}]; 200 201 // When emit the automatic clang codegen, it describes what types we have to use 202 // to obtain the specific LLVM intrinsic. -1 means the return type, otherwise, 203 // k >= 0 meaning the k-th operand (counting from zero) of the codegen'd 204 // parameter of the unmasked version. k can't be the mask operand's position. 205 list<int> IntrinsicTypes = []; 206 207 // If these names are not empty, this is the ID of the LLVM intrinsic 208 // we want to lower to. 209 string IRName = NAME; 210 211 // If HasMask, this is the ID of the LLVM intrinsic we want to lower to. 212 string IRNameMask = NAME #"_mask"; 213 214 // If non empty, this is the code emitted in the header, otherwise 215 // an automatic definition in header is emitted. 216 string HeaderCode = ""; 217 218 // Features required to enable for this builtin. 219 list<string> RequiredFeatures = []; 220 221 // Number of fields for Load/Store Segment instructions. 222 int NF = 1; 223} 224 225class RVVHeader 226{ 227 code HeaderCode; 228} 229 230//===----------------------------------------------------------------------===// 231// Basic classes with automatic codegen. 232//===----------------------------------------------------------------------===// 233 234class RVVOutBuiltin<string suffix, string prototype, string type_range> 235 : RVVBuiltin<suffix, prototype, type_range> { 236 let IntrinsicTypes = [-1]; 237} 238 239class RVVOp0Builtin<string suffix, string prototype, string type_range> 240 : RVVBuiltin<suffix, prototype, type_range> { 241 let IntrinsicTypes = [0]; 242} 243 244class RVVOutOp1Builtin<string suffix, string prototype, string type_range> 245 : RVVBuiltin<suffix, prototype, type_range> { 246 let IntrinsicTypes = [-1, 1]; 247} 248 249class RVVOutOp0Op1Builtin<string suffix, string prototype, string type_range> 250 : RVVBuiltin<suffix, prototype, type_range> { 251 let IntrinsicTypes = [-1, 0, 1]; 252} 253 254multiclass RVVBuiltinSet<string intrinsic_name, string type_range, 255 list<list<string>> suffixes_prototypes, 256 list<int> intrinsic_types> { 257 let IRName = intrinsic_name, IRNameMask = intrinsic_name # "_mask", 258 IntrinsicTypes = intrinsic_types in { 259 foreach s_p = suffixes_prototypes in { 260 let Name = NAME # "_" # s_p[0] in { 261 defvar suffix = s_p[1]; 262 defvar prototype = s_p[2]; 263 def : RVVBuiltin<suffix, prototype, type_range>; 264 } 265 } 266 } 267} 268 269// IntrinsicTypes is output, op0, op1 [-1, 0, 1] 270multiclass RVVOutOp0Op1BuiltinSet<string intrinsic_name, string type_range, 271 list<list<string>> suffixes_prototypes> 272 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, 273 [-1, 0, 1]>; 274 275multiclass RVVOutBuiltinSet<string intrinsic_name, string type_range, 276 list<list<string>> suffixes_prototypes> 277 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1]>; 278 279multiclass RVVOp0BuiltinSet<string intrinsic_name, string type_range, 280 list<list<string>> suffixes_prototypes> 281 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0]>; 282 283// IntrinsicTypes is output, op1 [-1, 1] 284multiclass RVVOutOp1BuiltinSet<string intrinsic_name, string type_range, 285 list<list<string>> suffixes_prototypes> 286 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1]>; 287 288multiclass RVVOp0Op1BuiltinSet<string intrinsic_name, string type_range, 289 list<list<string>> suffixes_prototypes> 290 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0, 1]>; 291 292multiclass RVVOutOp1Op2BuiltinSet<string intrinsic_name, string type_range, 293 list<list<string>> suffixes_prototypes> 294 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1, 2]>; 295 296multiclass RVVSignedBinBuiltinSet 297 : RVVOutOp1BuiltinSet<NAME, "csil", 298 [["vv", "v", "vvv"], 299 ["vx", "v", "vve"]]>; 300 301multiclass RVVUnsignedBinBuiltinSet 302 : RVVOutOp1BuiltinSet<NAME, "csil", 303 [["vv", "Uv", "UvUvUv"], 304 ["vx", "Uv", "UvUvUe"]]>; 305 306multiclass RVVIntBinBuiltinSet 307 : RVVSignedBinBuiltinSet, 308 RVVUnsignedBinBuiltinSet; 309 310multiclass RVVSlideOneBuiltinSet 311 : RVVOutOp1BuiltinSet<NAME, "csil", 312 [["vx", "v", "vve"], 313 ["vx", "Uv", "UvUve"]]>; 314 315multiclass RVVSignedShiftBuiltinSet 316 : RVVOutOp1BuiltinSet<NAME, "csil", 317 [["vv", "v", "vvUv"], 318 ["vx", "v", "vvz"]]>; 319 320multiclass RVVUnsignedShiftBuiltinSet 321 : RVVOutOp1BuiltinSet<NAME, "csil", 322 [["vv", "Uv", "UvUvUv"], 323 ["vx", "Uv", "UvUvz"]]>; 324 325multiclass RVVShiftBuiltinSet 326 : RVVSignedShiftBuiltinSet, 327 RVVUnsignedShiftBuiltinSet; 328 329let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 330 multiclass RVVSignedNShiftBuiltinSet 331 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 332 [["wv", "v", "vwUv"], 333 ["wx", "v", "vwz"]]>; 334 multiclass RVVUnsignedNShiftBuiltinSet 335 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 336 [["wv", "Uv", "UvUwUv"], 337 ["wx", "Uv", "UvUwz"]]>; 338} 339 340multiclass RVVCarryinBuiltinSet 341 : RVVOutOp1BuiltinSet<NAME, "csil", 342 [["vvm", "v", "vvvm"], 343 ["vxm", "v", "vvem"], 344 ["vvm", "Uv", "UvUvUvm"], 345 ["vxm", "Uv", "UvUvUem"]]>; 346 347multiclass RVVCarryOutInBuiltinSet<string intrinsic_name> 348 : RVVOp0Op1BuiltinSet<intrinsic_name, "csil", 349 [["vvm", "vm", "mvvm"], 350 ["vxm", "vm", "mvem"], 351 ["vvm", "Uvm", "mUvUvm"], 352 ["vxm", "Uvm", "mUvUem"]]>; 353 354multiclass RVVSignedMaskOutBuiltinSet 355 : RVVOp0Op1BuiltinSet<NAME, "csil", 356 [["vv", "vm", "mvv"], 357 ["vx", "vm", "mve"]]>; 358 359multiclass RVVUnsignedMaskOutBuiltinSet 360 : RVVOp0Op1BuiltinSet<NAME, "csil", 361 [["vv", "Uvm", "mUvUv"], 362 ["vx", "Uvm", "mUvUe"]]>; 363 364multiclass RVVIntMaskOutBuiltinSet 365 : RVVSignedMaskOutBuiltinSet, 366 RVVUnsignedMaskOutBuiltinSet; 367 368class RVVIntExt<string intrinsic_name, string suffix, string prototype, 369 string type_range> 370 : RVVBuiltin<suffix, prototype, type_range> { 371 let IRName = intrinsic_name; 372 let IRNameMask = intrinsic_name # "_mask"; 373 let MangledName = NAME; 374 let IntrinsicTypes = [-1, 0]; 375} 376 377let HasMaskedOffOperand = false in { 378 multiclass RVVIntTerBuiltinSet { 379 defm "" : RVVOutOp1BuiltinSet<NAME, "csil", 380 [["vv", "v", "vvvv"], 381 ["vx", "v", "vvev"], 382 ["vv", "Uv", "UvUvUvUv"], 383 ["vx", "Uv", "UvUvUeUv"]]>; 384 } 385 multiclass RVVFloatingTerBuiltinSet { 386 defm "" : RVVOutOp1BuiltinSet<NAME, "xfd", 387 [["vv", "v", "vvvv"], 388 ["vf", "v", "vvev"]]>; 389 } 390} 391 392let HasMaskedOffOperand = false, Log2LMUL = [-2, -1, 0, 1, 2] in { 393 multiclass RVVFloatingWidenTerBuiltinSet { 394 defm "" : RVVOutOp1Op2BuiltinSet<NAME, "xf", 395 [["vv", "w", "wwvv"], 396 ["vf", "w", "wwev"]]>; 397 } 398} 399 400multiclass RVVFloatingBinBuiltinSet 401 : RVVOutOp1BuiltinSet<NAME, "xfd", 402 [["vv", "v", "vvv"], 403 ["vf", "v", "vve"]]>; 404 405multiclass RVVFloatingBinVFBuiltinSet 406 : RVVOutOp1BuiltinSet<NAME, "xfd", 407 [["vf", "v", "vve"]]>; 408 409multiclass RVVFloatingMaskOutBuiltinSet 410 : RVVOp0Op1BuiltinSet<NAME, "xfd", 411 [["vv", "vm", "mvv"], 412 ["vf", "vm", "mve"]]>; 413 414multiclass RVVFloatingMaskOutVFBuiltinSet 415 : RVVOp0Op1BuiltinSet<NAME, "fd", 416 [["vf", "vm", "mve"]]>; 417 418class RVVMaskBinBuiltin : RVVOutBuiltin<"m", "mmm", "c"> { 419 let Name = NAME # "_mm"; 420 let HasMask = false; 421} 422 423class RVVMaskUnaryBuiltin : RVVOutBuiltin<"m", "mm", "c"> { 424 let Name = NAME # "_m"; 425} 426 427class RVVMaskNullaryBuiltin : RVVOutBuiltin<"m", "m", "c"> { 428 let Name = NAME # "_m"; 429 let HasMask = false; 430 let HasNoMaskedOverloaded = false; 431} 432 433class RVVMaskOp0Builtin<string prototype> : RVVOp0Builtin<"m", prototype, "c"> { 434 let Name = NAME # "_m"; 435 let HasMaskedOffOperand = false; 436} 437 438let HasMaskedOffOperand = false in { 439 multiclass RVVSlideBuiltinSet { 440 defm "" : RVVOutBuiltinSet<NAME, "csilxfd", 441 [["vx","v", "vvvz"]]>; 442 defm "" : RVVOutBuiltinSet<NAME, "csil", 443 [["vx","Uv", "UvUvUvz"]]>; 444 } 445} 446 447class RVVFloatingUnaryBuiltin<string builtin_suffix, string ir_suffix, 448 string prototype> 449 : RVVOutBuiltin<ir_suffix, prototype, "xfd"> { 450 let Name = NAME # "_" # builtin_suffix; 451} 452 453class RVVFloatingUnaryVVBuiltin : RVVFloatingUnaryBuiltin<"v", "v", "vv">; 454 455class RVVConvBuiltin<string suffix, string prototype, string type_range, 456 string mangled_name> 457 : RVVBuiltin<suffix, prototype, type_range> { 458 let IntrinsicTypes = [-1, 0]; 459 let MangledName = mangled_name; 460} 461 462class RVVConvToSignedBuiltin<string mangled_name> 463 : RVVConvBuiltin<"Iv", "Ivv", "xfd", mangled_name>; 464 465class RVVConvToUnsignedBuiltin<string mangled_name> 466 : RVVConvBuiltin<"Uv", "Uvv", "xfd", mangled_name>; 467 468class RVVConvToWidenSignedBuiltin<string mangled_name> 469 : RVVConvBuiltin<"Iw", "Iwv", "xf", mangled_name>; 470 471class RVVConvToWidenUnsignedBuiltin<string mangled_name> 472 : RVVConvBuiltin<"Uw", "Uwv", "xf", mangled_name>; 473 474class RVVConvToNarrowingSignedBuiltin<string mangled_name> 475 : RVVConvBuiltin<"Iv", "IvFw", "csi", mangled_name>; 476 477class RVVConvToNarrowingUnsignedBuiltin<string mangled_name> 478 : RVVConvBuiltin<"Uv", "UvFw", "csi", mangled_name>; 479 480let HasMaskedOffOperand = false in { 481 multiclass RVVSignedReductionBuiltin { 482 defm "" : RVVOutOp1BuiltinSet<NAME, "csil", 483 [["vs", "vSv", "SvSvvSv"]]>; 484 } 485 multiclass RVVUnsignedReductionBuiltin { 486 defm "" : RVVOutOp1BuiltinSet<NAME, "csil", 487 [["vs", "UvUSv", "USvUSvUvUSv"]]>; 488 } 489 multiclass RVVFloatingReductionBuiltin { 490 defm "" : RVVOutOp1BuiltinSet<NAME, "xfd", 491 [["vs", "vSv", "SvSvvSv"]]>; 492 } 493 multiclass RVVFloatingWidenReductionBuiltin { 494 defm "" : RVVOutOp1BuiltinSet<NAME, "xf", 495 [["vs", "vSw", "SwSwvSw"]]>; 496 } 497} 498 499multiclass RVVIntReductionBuiltinSet 500 : RVVSignedReductionBuiltin, 501 RVVUnsignedReductionBuiltin; 502 503// For widen operation which has different mangling name. 504multiclass RVVWidenBuiltinSet<string intrinsic_name, string type_range, 505 list<list<string>> suffixes_prototypes> { 506 let Log2LMUL = [-3, -2, -1, 0, 1, 2], 507 IRName = intrinsic_name, IRNameMask = intrinsic_name # "_mask" in { 508 foreach s_p = suffixes_prototypes in { 509 let Name = NAME # "_" # s_p[0], 510 MangledName = NAME # "_" # s_p[0] in { 511 defvar suffix = s_p[1]; 512 defvar prototype = s_p[2]; 513 def : RVVOutOp0Op1Builtin<suffix, prototype, type_range>; 514 } 515 } 516 } 517} 518 519// For widen operation with widen operand which has different mangling name. 520multiclass RVVWidenWOp0BuiltinSet<string intrinsic_name, string type_range, 521 list<list<string>> suffixes_prototypes> { 522 let Log2LMUL = [-3, -2, -1, 0, 1, 2], 523 IRName = intrinsic_name, IRNameMask = intrinsic_name # "_mask" in { 524 foreach s_p = suffixes_prototypes in { 525 let Name = NAME # "_" # s_p[0], 526 MangledName = NAME # "_" # s_p[0] in { 527 defvar suffix = s_p[1]; 528 defvar prototype = s_p[2]; 529 def : RVVOutOp1Builtin<suffix, prototype, type_range>; 530 } 531 } 532 } 533} 534 535multiclass RVVSignedWidenBinBuiltinSet 536 : RVVWidenBuiltinSet<NAME, "csi", 537 [["vv", "w", "wvv"], 538 ["vx", "w", "wve"]]>; 539 540multiclass RVVSignedWidenOp0BinBuiltinSet 541 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", 542 [["wv", "w", "wwv"], 543 ["wx", "w", "wwe"]]>; 544 545multiclass RVVUnsignedWidenBinBuiltinSet 546 : RVVWidenBuiltinSet<NAME, "csi", 547 [["vv", "Uw", "UwUvUv"], 548 ["vx", "Uw", "UwUvUe"]]>; 549 550multiclass RVVUnsignedWidenOp0BinBuiltinSet 551 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", 552 [["wv", "Uw", "UwUwUv"], 553 ["wx", "Uw", "UwUwUe"]]>; 554 555multiclass RVVFloatingWidenBinBuiltinSet 556 : RVVWidenBuiltinSet<NAME, "xf", 557 [["vv", "w", "wvv"], 558 ["vf", "w", "wve"]]>; 559 560multiclass RVVFloatingWidenOp0BinBuiltinSet 561 : RVVWidenWOp0BuiltinSet<NAME # "_w", "xf", 562 [["wv", "w", "wwv"], 563 ["wf", "w", "wwe"]]>; 564 565defvar TypeList = ["c","s","i","l","x","f","d"]; 566defvar EEWList = [["8", "(Log2EEW:3)"], 567 ["16", "(Log2EEW:4)"], 568 ["32", "(Log2EEW:5)"], 569 ["64", "(Log2EEW:6)"]]; 570 571class IsFloat<string type> { 572 bit val = !or(!eq(type, "x"), !eq(type, "f"), !eq(type, "d")); 573} 574 575let HasNoMaskedOverloaded = false, 576 HasPolicy = false, 577 ManualCodegen = [{ 578 IntrinsicTypes = {ResultType, Ops[1]->getType()}; 579 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); 580 }], 581 ManualCodegenMask= [{ 582 // Move mask to right before vl. 583 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 584 IntrinsicTypes = {ResultType, Ops[3]->getType()}; 585 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); 586 }] in { 587 class RVVVLEMaskBuiltin : RVVBuiltin<"m", "mPCUe", "c"> { 588 let Name = "vlm_v"; 589 let IRName = "vlm"; 590 let HasMask = false; 591 } 592} 593 594let HasNoMaskedOverloaded = false, 595 ManualCodegen = [{ 596 IntrinsicTypes = {ResultType, Ops[1]->getType()}; 597 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); 598 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 599 }], 600 ManualCodegenMask= [{ 601 // Move mask to right before vl. 602 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 603 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 604 IntrinsicTypes = {ResultType, Ops[3]->getType()}; 605 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); 606 }] in { 607 multiclass RVVVLEBuiltin<list<string> types> { 608 let Name = NAME # "_v", 609 IRName = "vle", 610 IRNameMask ="vle_mask" in { 611 foreach type = types in { 612 def : RVVBuiltin<"v", "vPCe", type>; 613 if !not(IsFloat<type>.val) then { 614 def : RVVBuiltin<"Uv", "UvPCUe", type>; 615 } 616 } 617 } 618 } 619} 620 621multiclass RVVVLEFFBuiltin<list<string> types> { 622 let Name = NAME # "_v", 623 IRName = "vleff", 624 IRNameMask = "vleff_mask", 625 HasNoMaskedOverloaded = false, 626 ManualCodegen = [{ 627 { 628 IntrinsicTypes = {ResultType, Ops[2]->getType()}; 629 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); 630 Value *NewVL = Ops[1]; 631 Ops.erase(Ops.begin() + 1); 632 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 633 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 634 llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); 635 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); 636 // Store new_vl. 637 clang::CharUnits Align = 638 CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); 639 Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {1}), 640 Address(NewVL, Align)); 641 return V; 642 } 643 }], 644 ManualCodegenMask = [{ 645 { 646 // Move mask to right before vl. 647 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 648 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 649 IntrinsicTypes = {ResultType, Ops[4]->getType()}; 650 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); 651 Value *NewVL = Ops[2]; 652 Ops.erase(Ops.begin() + 2); 653 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 654 llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); 655 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); 656 // Store new_vl. 657 clang::CharUnits Align = 658 CGM.getNaturalPointeeTypeAlignment(E->getArg(3)->getType()); 659 Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {1}), 660 Address(NewVL, Align)); 661 return V; 662 } 663 }] in { 664 foreach type = types in { 665 def : RVVBuiltin<"v", "vPCePz", type>; 666 // Skip floating types for unsigned versions. 667 if !not(IsFloat<type>.val) then { 668 def : RVVBuiltin<"Uv", "UvPCUePz", type>; 669 } 670 } 671 } 672} 673 674multiclass RVVVLSEBuiltin<list<string> types> { 675 let Name = NAME # "_v", 676 IRName = "vlse", 677 IRNameMask ="vlse_mask", 678 HasNoMaskedOverloaded = false, 679 ManualCodegen = [{ 680 IntrinsicTypes = {ResultType, Ops[2]->getType()}; 681 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); 682 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 683 }], 684 ManualCodegenMask= [{ 685 // Move mask to right before vl. 686 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 687 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 688 IntrinsicTypes = {ResultType, Ops[4]->getType()}; 689 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); 690 }] in { 691 foreach type = types in { 692 def : RVVBuiltin<"v", "vPCet", type>; 693 if !not(IsFloat<type>.val) then { 694 def : RVVBuiltin<"Uv", "UvPCUet", type>; 695 } 696 } 697 } 698} 699 700multiclass RVVIndexedLoad<string op> { 701 let ManualCodegen = [{ 702 IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType()}; 703 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); 704 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 705 }], 706 ManualCodegenMask = [{ 707 // Move mask to right before vl. 708 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 709 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 710 IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops[4]->getType()}; 711 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); 712 }] in { 713 foreach type = TypeList in { 714 foreach eew_list = EEWList[0-2] in { 715 defvar eew = eew_list[0]; 716 defvar eew_type = eew_list[1]; 717 let Name = op # eew # "_v", IRName = op, IRNameMask = op # "_mask" in { 718 def: RVVBuiltin<"v", "vPCe" # eew_type # "Uv", type>; 719 if !not(IsFloat<type>.val) then { 720 def: RVVBuiltin<"Uv", "UvPCUe" # eew_type # "Uv", type>; 721 } 722 } 723 } 724 defvar eew64 = "64"; 725 defvar eew64_type = "(Log2EEW:6)"; 726 let Name = op # eew64 # "_v", IRName = op, IRNameMask = op # "_mask", 727 RequiredFeatures = ["RV64"] in { 728 def: RVVBuiltin<"v", "vPCe" # eew64_type # "Uv", type>; 729 if !not(IsFloat<type>.val) then { 730 def: RVVBuiltin<"Uv", "UvPCUe" # eew64_type # "Uv", type>; 731 } 732 } 733 } 734 } 735} 736 737let HasMaskedOffOperand = false, 738 HasPolicy = false, 739 ManualCodegen = [{ 740 // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) 741 std::swap(Ops[0], Ops[1]); 742 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 743 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()}; 744 }], 745 ManualCodegenMask= [{ 746 // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) 747 std::swap(Ops[0], Ops[2]); 748 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 749 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 750 }] in { 751 class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { 752 let Name = "vsm_v"; 753 let IRName = "vsm"; 754 let HasMask = false; 755 } 756 multiclass RVVVSEBuiltin<list<string> types> { 757 let Name = NAME # "_v", 758 IRName = "vse", 759 IRNameMask = "vse_mask" in { 760 foreach type = types in { 761 def : RVVBuiltin<"v", "0Pev", type>; 762 if !not(IsFloat<type>.val) then { 763 def : RVVBuiltin<"Uv", "0PUeUv", type>; 764 } 765 } 766 } 767 } 768} 769 770multiclass RVVVSSEBuiltin<list<string> types> { 771 let Name = NAME # "_v", 772 IRName = "vsse", 773 IRNameMask = "vsse_mask", 774 HasMaskedOffOperand = false, 775 HasPolicy = false, 776 ManualCodegen = [{ 777 // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) 778 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 779 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 780 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 781 }], 782 ManualCodegenMask= [{ 783 // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) 784 std::swap(Ops[0], Ops[3]); 785 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 786 IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()}; 787 }] in { 788 foreach type = types in { 789 def : RVVBuiltin<"v", "0Petv", type>; 790 if !not(IsFloat<type>.val) then { 791 def : RVVBuiltin<"Uv", "0PUetUv", type>; 792 } 793 } 794 } 795} 796 797multiclass RVVIndexedStore<string op> { 798 let HasMaskedOffOperand = false, 799 HasPolicy = false, 800 ManualCodegen = [{ 801 // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) 802 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 803 Ops[1] = Builder.CreateBitCast(Ops[1],Ops[0]->getType()->getPointerTo()); 804 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()}; 805 }], 806 ManualCodegenMask= [{ 807 // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) 808 std::swap(Ops[0], Ops[3]); 809 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 810 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()}; 811 }] in { 812 foreach type = TypeList in { 813 foreach eew_list = EEWList[0-2] in { 814 defvar eew = eew_list[0]; 815 defvar eew_type = eew_list[1]; 816 let Name = op # eew # "_v", IRName = op, IRNameMask = op # "_mask" in { 817 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>; 818 if !not(IsFloat<type>.val) then { 819 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "UvUv", type>; 820 } 821 } 822 } 823 defvar eew64 = "64"; 824 defvar eew64_type = "(Log2EEW:6)"; 825 let Name = op # eew64 # "_v", IRName = op, IRNameMask = op # "_mask", 826 RequiredFeatures = ["RV64"] in { 827 def : RVVBuiltin<"v", "0Pe" # eew64_type # "Uvv", type>; 828 if !not(IsFloat<type>.val) then { 829 def : RVVBuiltin<"Uv", "0PUe" # eew64_type # "UvUv", type>; 830 } 831 } 832 } 833 } 834} 835 836defvar NFList = [2, 3, 4, 5, 6, 7, 8]; 837 838class PVString<int nf, bit signed> { 839 string S = 840 !cond(!eq(nf, 2): !if(signed, "PvPv", "PUvPUv"), 841 !eq(nf, 3): !if(signed, "PvPvPv", "PUvPUvPUv"), 842 !eq(nf, 4): !if(signed, "PvPvPvPv", "PUvPUvPUvPUv"), 843 !eq(nf, 5): !if(signed, "PvPvPvPvPv", "PUvPUvPUvPUvPUv"), 844 !eq(nf, 6): !if(signed, "PvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUv"), 845 !eq(nf, 7): !if(signed, "PvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUv"), 846 !eq(nf, 8): !if(signed, "PvPvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUvPUv")); 847} 848 849multiclass RVVUnitStridedSegLoad<string op> { 850 foreach type = TypeList in { 851 defvar eew = !cond(!eq(type, "c") : "8", 852 !eq(type, "s") : "16", 853 !eq(type, "i") : "32", 854 !eq(type, "l") : "64", 855 !eq(type, "x") : "16", 856 !eq(type, "f") : "32", 857 !eq(type, "d") : "64"); 858 foreach nf = NFList in { 859 let Name = op # nf # "e" # eew # "_v", 860 IRName = op # nf, 861 IRNameMask = op # nf # "_mask", 862 NF = nf, 863 HasNoMaskedOverloaded = false, 864 ManualCodegen = [{ 865 { 866 // builtin: (val0 address, val1 address, ..., ptr, vl) 867 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 868 Ops[NF + 1]->getType()}; 869 // intrinsic: (ptr, vl) 870 llvm::Value *Operands[] = {Ops[NF], Ops[NF + 1]}; 871 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 872 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 873 clang::CharUnits Align = 874 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 875 llvm::Value *V; 876 for (unsigned I = 0; I < NF; ++I) { 877 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), 878 Address(Ops[I], Align)); 879 } 880 return V; 881 } 882 }], 883 ManualCodegenMask = [{ 884 { 885 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl) 886 // intrinsic: (maskedoff0, ..., ptr, mask, vl) 887 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 888 Ops[2 * NF + 2]->getType()}; 889 SmallVector<llvm::Value*, 12> Operands; 890 for (unsigned I = 0; I < NF; ++I) 891 Operands.push_back(Ops[NF + I + 1]); 892 Operands.push_back(Ops[2 * NF + 1]); 893 Operands.push_back(Ops[NF]); 894 Operands.push_back(Ops[2 * NF + 2]); 895 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 896 assert(Operands.size() == NF + 4); 897 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 898 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 899 clang::CharUnits Align = 900 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 901 llvm::Value *V; 902 for (unsigned I = 0; I < NF; ++I) { 903 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), 904 Address(Ops[I], Align)); 905 } 906 return V; 907 } 908 }] in { 909 defvar PV = PVString<nf, /*signed=*/true>.S; 910 defvar PUV = PVString<nf, /*signed=*/false>.S; 911 def : RVVBuiltin<"v", "0" # PV # "PCe", type>; 912 if !not(IsFloat<type>.val) then { 913 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe", type>; 914 } 915 } 916 } 917 } 918} 919 920multiclass RVVUnitStridedSegLoadFF<string op> { 921 foreach type = TypeList in { 922 defvar eew = !cond(!eq(type, "c") : "8", 923 !eq(type, "s") : "16", 924 !eq(type, "i") : "32", 925 !eq(type, "l") : "64", 926 !eq(type, "x") : "16", 927 !eq(type, "f") : "32", 928 !eq(type, "d") : "64"); 929 foreach nf = NFList in { 930 let Name = op # nf # "e" # eew # "ff_v", 931 IRName = op # nf # "ff", 932 IRNameMask = op # nf # "ff_mask", 933 NF = nf, 934 HasNoMaskedOverloaded = false, 935 ManualCodegen = [{ 936 { 937 // builtin: (val0 address, val1 address, ..., ptr, new_vl, vl) 938 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 939 Ops[NF + 2]->getType()}; 940 // intrinsic: (ptr, vl) 941 llvm::Value *Operands[] = {Ops[NF], Ops[NF + 2]}; 942 Value *NewVL = Ops[NF + 1]; 943 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 944 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 945 clang::CharUnits Align = 946 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 947 for (unsigned I = 0; I < NF; ++I) { 948 Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), 949 Address(Ops[I], Align)); 950 } 951 // Store new_vl. 952 return Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {NF}), 953 Address(NewVL, Align)); 954 } 955 }], 956 ManualCodegenMask = [{ 957 { 958 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl) 959 // intrinsic: (maskedoff0, ..., ptr, mask, vl) 960 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 961 Ops[2 * NF + 3]->getType()}; 962 SmallVector<llvm::Value*, 12> Operands; 963 for (unsigned I = 0; I < NF; ++I) 964 Operands.push_back(Ops[NF + I + 1]); 965 Operands.push_back(Ops[2 * NF + 1]); 966 Operands.push_back(Ops[NF]); 967 Operands.push_back(Ops[2 * NF + 3]); 968 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 969 Value *NewVL = Ops[2 * NF + 2]; 970 assert(Operands.size() == NF + 4); 971 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 972 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 973 clang::CharUnits Align = 974 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 975 for (unsigned I = 0; I < NF; ++I) { 976 Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), 977 Address(Ops[I], Align)); 978 } 979 // Store new_vl. 980 return Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {NF}), 981 Address(NewVL, Align)); 982 } 983 }] in { 984 defvar PV = PVString<nf, /*signed=*/true>.S; 985 defvar PUV = PVString<nf, /*signed=*/false>.S; 986 def : RVVBuiltin<"v", "0" # PV # "PCe" # "Pz", type>; 987 if !not(IsFloat<type>.val) then { 988 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "Pz", type>; 989 } 990 } 991 } 992 } 993} 994 995multiclass RVVStridedSegLoad<string op> { 996 foreach type = TypeList in { 997 defvar eew = !cond(!eq(type, "c") : "8", 998 !eq(type, "s") : "16", 999 !eq(type, "i") : "32", 1000 !eq(type, "l") : "64", 1001 !eq(type, "x") : "16", 1002 !eq(type, "f") : "32", 1003 !eq(type, "d") : "64"); 1004 foreach nf = NFList in { 1005 let Name = op # nf # "e" # eew # "_v", 1006 IRName = op # nf, 1007 IRNameMask = op # nf # "_mask", 1008 NF = nf, 1009 HasNoMaskedOverloaded = false, 1010 ManualCodegen = [{ 1011 { 1012 // builtin: (val0 address, val1 address, ..., ptr, stride, vl) 1013 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 1014 Ops[NF + 2]->getType()}; 1015 // intrinsic: (ptr, stride, vl) 1016 llvm::Value *Operands[] = {Ops[NF], Ops[NF + 1], Ops[NF + 2]}; 1017 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1018 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1019 clang::CharUnits Align = 1020 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1021 llvm::Value *V; 1022 for (unsigned I = 0; I < NF; ++I) { 1023 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), 1024 Address(Ops[I], Align)); 1025 } 1026 return V; 1027 } 1028 }], 1029 ManualCodegenMask = [{ 1030 { 1031 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl) 1032 // intrinsic: (maskedoff0, ..., ptr, stride, mask, vl) 1033 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 1034 Ops[2 * NF + 3]->getType()}; 1035 SmallVector<llvm::Value*, 12> Operands; 1036 for (unsigned I = 0; I < NF; ++I) 1037 Operands.push_back(Ops[NF + I + 1]); 1038 Operands.push_back(Ops[2 * NF + 1]); 1039 Operands.push_back(Ops[2 * NF + 2]); 1040 Operands.push_back(Ops[NF]); 1041 Operands.push_back(Ops[2 * NF + 3]); 1042 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1043 assert(Operands.size() == NF + 5); 1044 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1045 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1046 clang::CharUnits Align = 1047 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1048 llvm::Value *V; 1049 for (unsigned I = 0; I < NF; ++I) { 1050 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), 1051 Address(Ops[I], Align)); 1052 } 1053 return V; 1054 } 1055 }] in { 1056 defvar PV = PVString<nf, /*signed=*/true>.S; 1057 defvar PUV = PVString<nf, /*signed=*/false>.S; 1058 def : RVVBuiltin<"v", "0" # PV # "PCe" # "t", type>; 1059 if !not(IsFloat<type>.val) then { 1060 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "t", type>; 1061 } 1062 } 1063 } 1064 } 1065} 1066 1067multiclass RVVIndexedSegLoad<string op> { 1068 foreach type = TypeList in { 1069 foreach eew_info = EEWList in { 1070 defvar eew = eew_info[0]; 1071 defvar eew_type = eew_info[1]; 1072 foreach nf = NFList in { 1073 let Name = op # nf # "ei" # eew # "_v", 1074 IRName = op # nf, 1075 IRNameMask = op # nf # "_mask", 1076 NF = nf, 1077 ManualCodegen = [{ 1078 { 1079 // builtin: (val0 address, val1 address, ..., ptr, index, vl) 1080 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 1081 Ops[NF + 1]->getType(), Ops[NF + 2]->getType()}; 1082 // intrinsic: (ptr, index, vl) 1083 llvm::Value *Operands[] = {Ops[NF], Ops[NF + 1], Ops[NF + 2]}; 1084 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1085 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1086 clang::CharUnits Align = 1087 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1088 llvm::Value *V; 1089 for (unsigned I = 0; I < NF; ++I) { 1090 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), 1091 Address(Ops[I], Align)); 1092 } 1093 return V; 1094 } 1095 }], 1096 ManualCodegenMask = [{ 1097 { 1098 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl) 1099 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 1100 Ops[2 * NF + 2]->getType(), Ops[2 * NF + 3]->getType()}; 1101 // intrinsic: (maskedoff0, ..., ptr, index, mask, vl) 1102 SmallVector<llvm::Value*, 12> Operands; 1103 for (unsigned I = 0; I < NF; ++I) 1104 Operands.push_back(Ops[NF + I + 1]); 1105 Operands.push_back(Ops[2 * NF + 1]); 1106 Operands.push_back(Ops[2 * NF + 2]); 1107 Operands.push_back(Ops[NF]); 1108 Operands.push_back(Ops[2 * NF + 3]); 1109 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1110 assert(Operands.size() == NF + 5); 1111 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1112 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1113 clang::CharUnits Align = 1114 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1115 llvm::Value *V; 1116 for (unsigned I = 0; I < NF; ++I) { 1117 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), 1118 Address(Ops[I], Align)); 1119 } 1120 return V; 1121 } 1122 }] in { 1123 defvar PV = PVString<nf, /*signed=*/true>.S; 1124 defvar PUV = PVString<nf, /*signed=*/false>.S; 1125 def : RVVBuiltin<"v", "0" # PV # "PCe" # eew_type # "Uv", type>; 1126 if !not(IsFloat<type>.val) then { 1127 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # eew_type # "Uv", type>; 1128 } 1129 } 1130 } 1131 } 1132 } 1133} 1134 1135class VString<int nf, bit signed> { 1136 string S = !cond(!eq(nf, 2): !if(signed, "vv", "UvUv"), 1137 !eq(nf, 3): !if(signed, "vvv", "UvUvUv"), 1138 !eq(nf, 4): !if(signed, "vvvv", "UvUvUvUv"), 1139 !eq(nf, 5): !if(signed, "vvvvv", "UvUvUvUvUv"), 1140 !eq(nf, 6): !if(signed, "vvvvvv", "UvUvUvUvUvUv"), 1141 !eq(nf, 7): !if(signed, "vvvvvvv", "UvUvUvUvUvUvUv"), 1142 !eq(nf, 8): !if(signed, "vvvvvvvv", "UvUvUvUvUvUvUvUv")); 1143} 1144 1145multiclass RVVUnitStridedSegStore<string op> { 1146 foreach type = TypeList in { 1147 defvar eew = !cond(!eq(type, "c") : "8", 1148 !eq(type, "s") : "16", 1149 !eq(type, "i") : "32", 1150 !eq(type, "l") : "64", 1151 !eq(type, "x") : "16", 1152 !eq(type, "f") : "32", 1153 !eq(type, "d") : "64"); 1154 foreach nf = NFList in { 1155 let Name = op # nf # "e" # eew # "_v", 1156 IRName = op # nf, 1157 IRNameMask = op # nf # "_mask", 1158 NF = nf, 1159 HasMaskedOffOperand = false, 1160 HasPolicy = false, 1161 ManualCodegen = [{ 1162 { 1163 // Builtin: (ptr, val0, val1, ..., vl) 1164 // Intrinsic: (val0, val1, ..., ptr, vl) 1165 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1166 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; 1167 assert(Ops.size() == NF + 2); 1168 } 1169 }], 1170 ManualCodegenMask = [{ 1171 { 1172 // Builtin: (mask, ptr, val0, val1, ..., vl) 1173 // Intrinsic: (val0, val1, ..., ptr, mask, vl) 1174 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); 1175 std::swap(Ops[NF], Ops[NF + 1]); 1176 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 2]->getType()}; 1177 assert(Ops.size() == NF + 3); 1178 } 1179 }] in { 1180 defvar V = VString<nf, /*signed=*/true>.S; 1181 defvar UV = VString<nf, /*signed=*/false>.S; 1182 def : RVVBuiltin<"v", "0Pe" # V, type>; 1183 if !not(IsFloat<type>.val) then { 1184 def : RVVBuiltin<"Uv", "0PUe" # UV, type>; 1185 } 1186 } 1187 } 1188 } 1189} 1190 1191multiclass RVVStridedSegStore<string op> { 1192 foreach type = TypeList in { 1193 defvar eew = !cond(!eq(type, "c") : "8", 1194 !eq(type, "s") : "16", 1195 !eq(type, "i") : "32", 1196 !eq(type, "l") : "64", 1197 !eq(type, "x") : "16", 1198 !eq(type, "f") : "32", 1199 !eq(type, "d") : "64"); 1200 foreach nf = NFList in { 1201 let Name = op # nf # "e" # eew # "_v", 1202 IRName = op # nf, 1203 IRNameMask = op # nf # "_mask", 1204 NF = nf, 1205 HasMaskedOffOperand = false, 1206 HasPolicy = false, 1207 ManualCodegen = [{ 1208 { 1209 // Builtin: (ptr, stride, val0, val1, ..., vl). 1210 // Intrinsic: (val0, val1, ..., ptr, stride, vl) 1211 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); 1212 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; 1213 assert(Ops.size() == NF + 3); 1214 } 1215 }], 1216 ManualCodegenMask = [{ 1217 { 1218 // Builtin: (mask, ptr, stride, val0, val1, ..., vl). 1219 // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl) 1220 std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); 1221 std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); 1222 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; 1223 assert(Ops.size() == NF + 4); 1224 } 1225 }] in { 1226 defvar V = VString<nf, /*signed=*/true>.S; 1227 defvar UV = VString<nf, /*signed=*/false>.S; 1228 def : RVVBuiltin<"v", "0Pet" # V, type>; 1229 if !not(IsFloat<type>.val) then { 1230 def : RVVBuiltin<"Uv", "0PUet" # UV, type>; 1231 } 1232 } 1233 } 1234 } 1235} 1236 1237multiclass RVVIndexedSegStore<string op> { 1238 foreach type = TypeList in { 1239 foreach eew_info = EEWList in { 1240 defvar eew = eew_info[0]; 1241 defvar eew_type = eew_info[1]; 1242 foreach nf = NFList in { 1243 let Name = op # nf # "ei" # eew # "_v", 1244 IRName = op # nf, 1245 IRNameMask = op # nf # "_mask", 1246 NF = nf, 1247 HasMaskedOffOperand = false, 1248 HasPolicy = false, 1249 ManualCodegen = [{ 1250 { 1251 // Builtin: (ptr, index, val0, val1, ..., vl) 1252 // Intrinsic: (val0, val1, ..., ptr, index, vl) 1253 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); 1254 IntrinsicTypes = {Ops[0]->getType(), 1255 Ops[NF + 1]->getType(), Ops[NF + 2]->getType()}; 1256 assert(Ops.size() == NF + 3); 1257 } 1258 }], 1259 ManualCodegenMask = [{ 1260 { 1261 // Builtin: (mask, ptr, index, val0, val1, ..., vl) 1262 // Intrinsic: (val0, val1, ..., ptr, index, mask, vl) 1263 std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); 1264 std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); 1265 IntrinsicTypes = {Ops[0]->getType(), 1266 Ops[NF + 1]->getType(), Ops[NF + 3]->getType()}; 1267 assert(Ops.size() == NF + 4); 1268 } 1269 }] in { 1270 defvar V = VString<nf, /*signed=*/true>.S; 1271 defvar UV = VString<nf, /*signed=*/false>.S; 1272 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uv" # V, type>; 1273 if !not(IsFloat<type>.val) then { 1274 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "Uv" # UV, type>; 1275 } 1276 } 1277 } 1278 } 1279 } 1280} 1281 1282multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { 1283 let Name = NAME, 1284 IRName = IR, 1285 IRNameMask = IR # "_mask", 1286 ManualCodegen = [{ 1287 { 1288 // op1, vl 1289 IntrinsicTypes = {ResultType, 1290 cast<llvm::VectorType>(ResultType)->getElementType(), 1291 Ops[1]->getType()}; 1292 Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[1])); 1293 break; 1294 } 1295 }], 1296 ManualCodegenMask = [{ 1297 { 1298 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1299 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1300 // maskedoff, op1, mask, vl 1301 IntrinsicTypes = {ResultType, 1302 cast<llvm::VectorType>(ResultType)->getElementType(), 1303 Ops[3]->getType()}; 1304 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[1])); 1305 break; 1306 } 1307 }] in { 1308 def : RVVBuiltin<"v", "vv", type_range>; 1309 } 1310} 1311 1312multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { 1313 let Name = NAME, 1314 IRName = IR, 1315 IRNameMask = IR # "_mask", 1316 ManualCodegen = [{ 1317 { 1318 // op1, vl 1319 IntrinsicTypes = {ResultType, 1320 cast<llvm::VectorType>(ResultType)->getElementType(), 1321 Ops[1]->getType()}; 1322 Ops.insert(Ops.begin() + 1, 1323 llvm::Constant::getAllOnesValue(IntrinsicTypes[1])); 1324 break; 1325 } 1326 }], 1327 ManualCodegenMask = [{ 1328 { 1329 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1330 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1331 // maskedoff, op1, mask, vl 1332 IntrinsicTypes = {ResultType, 1333 cast<llvm::VectorType>(ResultType)->getElementType(), 1334 Ops[3]->getType()}; 1335 Ops.insert(Ops.begin() + 2, 1336 llvm::Constant::getAllOnesValue(IntrinsicTypes[1])); 1337 break; 1338 } 1339 }] in { 1340 def : RVVBuiltin<"v", "vv", type_range>; 1341 def : RVVBuiltin<"Uv", "UvUv", type_range>; 1342 } 1343} 1344 1345multiclass RVVPseudoMaskBuiltin<string IR, string type_range> { 1346 let Name = NAME, 1347 IRName = IR, 1348 HasMask = false, 1349 ManualCodegen = [{ 1350 { 1351 // op1, vl 1352 IntrinsicTypes = {ResultType, 1353 Ops[1]->getType()}; 1354 Ops.insert(Ops.begin() + 1, Ops[0]); 1355 break; 1356 } 1357 }] in { 1358 def : RVVBuiltin<"m", "mm", type_range>; 1359 } 1360} 1361 1362multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { 1363 let Name = NAME, 1364 IRName = IR, 1365 IRNameMask = IR # "_mask", 1366 ManualCodegen = [{ 1367 { 1368 // op1, vl 1369 IntrinsicTypes = {ResultType, 1370 Ops[0]->getType(), Ops[1]->getType()}; 1371 Ops.insert(Ops.begin() + 1, Ops[0]); 1372 break; 1373 } 1374 }], 1375 ManualCodegenMask = [{ 1376 { 1377 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1378 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1379 // maskedoff, op1, mask, vl 1380 IntrinsicTypes = {ResultType, 1381 Ops[1]->getType(), 1382 Ops[3]->getType()}; 1383 Ops.insert(Ops.begin() + 2, Ops[1]); 1384 break; 1385 } 1386 }] in { 1387 def : RVVBuiltin<"v", "vv", type_range>; 1388 } 1389} 1390 1391multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, 1392 list<list<string>> suffixes_prototypes> { 1393 let Name = NAME, 1394 MangledName = MName, 1395 IRName = IR, 1396 IRNameMask = IR # "_mask", 1397 ManualCodegen = [{ 1398 { 1399 // op1, vl 1400 IntrinsicTypes = {ResultType, 1401 Ops[0]->getType(), 1402 cast<llvm::VectorType>(Ops[0]->getType())->getElementType(), 1403 Ops[1]->getType()}; 1404 Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[2])); 1405 break; 1406 } 1407 }], 1408 ManualCodegenMask = [{ 1409 { 1410 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1411 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1412 // maskedoff, op1, mask, vl 1413 IntrinsicTypes = {ResultType, 1414 Ops[1]->getType(), 1415 cast<llvm::VectorType>(Ops[1]->getType())->getElementType(), 1416 Ops[3]->getType()}; 1417 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[2])); 1418 break; 1419 } 1420 }] in { 1421 foreach s_p = suffixes_prototypes in { 1422 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 1423 } 1424 } 1425} 1426 1427multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, 1428 list<list<string>> suffixes_prototypes> { 1429 let Name = NAME, 1430 MangledName = MName, 1431 IRName = IR, 1432 IRNameMask = IR # "_mask", 1433 ManualCodegen = [{ 1434 { 1435 // op1, vl 1436 IntrinsicTypes = {ResultType, 1437 Ops[0]->getType(), 1438 Ops[1]->getType(), 1439 Ops[1]->getType()}; 1440 Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[2])); 1441 break; 1442 } 1443 }], 1444 ManualCodegenMask = [{ 1445 { 1446 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1447 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1448 // maskedoff, op1, mask, vl 1449 IntrinsicTypes = {ResultType, 1450 Ops[1]->getType(), 1451 Ops[3]->getType(), 1452 Ops[3]->getType()}; 1453 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[2])); 1454 break; 1455 } 1456 }] in { 1457 foreach s_p = suffixes_prototypes in { 1458 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 1459 } 1460 } 1461} 1462 1463// 6. Configuration-Setting Instructions 1464// 6.1. vsetvli/vsetvl instructions 1465let HasVL = false, 1466 HasMask = false, 1467 HasPolicy = false, 1468 Log2LMUL = [0], 1469 ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type 1470{ 1471 // vsetvl is a macro because for it require constant integers in SEW and LMUL. 1472 let HeaderCode = 1473[{ 1474#define vsetvl_e8mf8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 5) 1475#define vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6) 1476#define vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7) 1477#define vsetvl_e8m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 0) 1478#define vsetvl_e8m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 1) 1479#define vsetvl_e8m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 2) 1480#define vsetvl_e8m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 3) 1481 1482#define vsetvl_e16mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 6) 1483#define vsetvl_e16mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 7) 1484#define vsetvl_e16m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 0) 1485#define vsetvl_e16m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 1) 1486#define vsetvl_e16m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 2) 1487#define vsetvl_e16m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 3) 1488 1489#define vsetvl_e32mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 7) 1490#define vsetvl_e32m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 0) 1491#define vsetvl_e32m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 1) 1492#define vsetvl_e32m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 2) 1493#define vsetvl_e32m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 3) 1494 1495#define vsetvl_e64m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 0) 1496#define vsetvl_e64m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 1) 1497#define vsetvl_e64m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 2) 1498#define vsetvl_e64m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 3) 1499 1500}] in 1501 def vsetvli : RVVBuiltin<"", "zzKzKz", "i">; 1502 1503 let HeaderCode = 1504[{ 1505#define vsetvlmax_e8mf8() __builtin_rvv_vsetvlimax(0, 5) 1506#define vsetvlmax_e8mf4() __builtin_rvv_vsetvlimax(0, 6) 1507#define vsetvlmax_e8mf2() __builtin_rvv_vsetvlimax(0, 7) 1508#define vsetvlmax_e8m1() __builtin_rvv_vsetvlimax(0, 0) 1509#define vsetvlmax_e8m2() __builtin_rvv_vsetvlimax(0, 1) 1510#define vsetvlmax_e8m4() __builtin_rvv_vsetvlimax(0, 2) 1511#define vsetvlmax_e8m8() __builtin_rvv_vsetvlimax(0, 3) 1512 1513#define vsetvlmax_e16mf4() __builtin_rvv_vsetvlimax(1, 6) 1514#define vsetvlmax_e16mf2() __builtin_rvv_vsetvlimax(1, 7) 1515#define vsetvlmax_e16m1() __builtin_rvv_vsetvlimax(1, 0) 1516#define vsetvlmax_e16m2() __builtin_rvv_vsetvlimax(1, 1) 1517#define vsetvlmax_e16m4() __builtin_rvv_vsetvlimax(1, 2) 1518#define vsetvlmax_e16m8() __builtin_rvv_vsetvlimax(1, 3) 1519 1520#define vsetvlmax_e32mf2() __builtin_rvv_vsetvlimax(2, 7) 1521#define vsetvlmax_e32m1() __builtin_rvv_vsetvlimax(2, 0) 1522#define vsetvlmax_e32m2() __builtin_rvv_vsetvlimax(2, 1) 1523#define vsetvlmax_e32m4() __builtin_rvv_vsetvlimax(2, 2) 1524#define vsetvlmax_e32m8() __builtin_rvv_vsetvlimax(2, 3) 1525 1526#define vsetvlmax_e64m1() __builtin_rvv_vsetvlimax(3, 0) 1527#define vsetvlmax_e64m2() __builtin_rvv_vsetvlimax(3, 1) 1528#define vsetvlmax_e64m4() __builtin_rvv_vsetvlimax(3, 2) 1529#define vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3) 1530 1531}] in 1532 def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">; 1533} 1534 1535// 7. Vector Loads and Stores 1536// 7.4. Vector Unit-Stride Instructions 1537def vlm: RVVVLEMaskBuiltin; 1538defm vle8: RVVVLEBuiltin<["c"]>; 1539defm vle16: RVVVLEBuiltin<["s","x"]>; 1540defm vle32: RVVVLEBuiltin<["i","f"]>; 1541defm vle64: RVVVLEBuiltin<["l","d"]>; 1542 1543def vsm : RVVVSEMaskBuiltin; 1544defm vse8 : RVVVSEBuiltin<["c"]>; 1545defm vse16: RVVVSEBuiltin<["s","x"]>; 1546defm vse32: RVVVSEBuiltin<["i","f"]>; 1547defm vse64: RVVVSEBuiltin<["l","d"]>; 1548 1549// 7.5. Vector Strided Instructions 1550defm vlse8: RVVVLSEBuiltin<["c"]>; 1551defm vlse16: RVVVLSEBuiltin<["s","x"]>; 1552defm vlse32: RVVVLSEBuiltin<["i","f"]>; 1553defm vlse64: RVVVLSEBuiltin<["l","d"]>; 1554 1555defm vsse8 : RVVVSSEBuiltin<["c"]>; 1556defm vsse16: RVVVSSEBuiltin<["s","x"]>; 1557defm vsse32: RVVVSSEBuiltin<["i","f"]>; 1558defm vsse64: RVVVSSEBuiltin<["l","d"]>; 1559 1560// 7.6. Vector Indexed Instructions 1561defm : RVVIndexedLoad<"vluxei">; 1562defm : RVVIndexedLoad<"vloxei">; 1563 1564defm : RVVIndexedStore<"vsuxei">; 1565defm : RVVIndexedStore<"vsoxei">; 1566 1567// 7.7. Unit-stride Fault-Only-First Loads 1568defm vle8ff: RVVVLEFFBuiltin<["c"]>; 1569defm vle16ff: RVVVLEFFBuiltin<["s","x"]>; 1570defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>; 1571defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>; 1572 1573// 7.8 Vector Load/Store Segment Instructions 1574defm : RVVUnitStridedSegLoad<"vlseg">; 1575defm : RVVUnitStridedSegLoadFF<"vlseg">; 1576defm : RVVStridedSegLoad<"vlsseg">; 1577defm : RVVIndexedSegLoad<"vluxseg">; 1578defm : RVVIndexedSegLoad<"vloxseg">; 1579defm : RVVUnitStridedSegStore<"vsseg">; 1580defm : RVVStridedSegStore<"vssseg">; 1581defm : RVVIndexedSegStore<"vsuxseg">; 1582defm : RVVIndexedSegStore<"vsoxseg">; 1583 1584// 12. Vector Integer Arithmetic Instructions 1585// 12.1. Vector Single-Width Integer Add and Subtract 1586defm vadd : RVVIntBinBuiltinSet; 1587defm vsub : RVVIntBinBuiltinSet; 1588defm vrsub : RVVOutOp1BuiltinSet<"vrsub", "csil", 1589 [["vx", "v", "vve"], 1590 ["vx", "Uv", "UvUvUe"]]>; 1591defm vneg_v : RVVPseudoUnaryBuiltin<"vrsub", "csil">; 1592 1593// 12.2. Vector Widening Integer Add/Subtract 1594// Widening unsigned integer add/subtract, 2*SEW = SEW +/- SEW 1595defm vwaddu : RVVUnsignedWidenBinBuiltinSet; 1596defm vwsubu : RVVUnsignedWidenBinBuiltinSet; 1597// Widening signed integer add/subtract, 2*SEW = SEW +/- SEW 1598defm vwadd : RVVSignedWidenBinBuiltinSet; 1599defm vwsub : RVVSignedWidenBinBuiltinSet; 1600// Widening unsigned integer add/subtract, 2*SEW = 2*SEW +/- SEW 1601defm vwaddu : RVVUnsignedWidenOp0BinBuiltinSet; 1602defm vwsubu : RVVUnsignedWidenOp0BinBuiltinSet; 1603// Widening signed integer add/subtract, 2*SEW = 2*SEW +/- SEW 1604defm vwadd : RVVSignedWidenOp0BinBuiltinSet; 1605defm vwsub : RVVSignedWidenOp0BinBuiltinSet; 1606defm vwcvtu_x_x_v : RVVPseudoVWCVTBuiltin<"vwaddu", "vwcvtu_x", "csi", 1607 [["Uw", "UwUv"]]>; 1608defm vwcvt_x_x_v : RVVPseudoVWCVTBuiltin<"vwadd", "vwcvt_x", "csi", 1609 [["w", "wv"]]>; 1610 1611// 12.3. Vector Integer Extension 1612let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1613 def vsext_vf2 : RVVIntExt<"vsext", "w", "wv", "csi">; 1614 def vzext_vf2 : RVVIntExt<"vzext", "Uw", "UwUv", "csi">; 1615} 1616let Log2LMUL = [-3, -2, -1, 0, 1] in { 1617 def vsext_vf4 : RVVIntExt<"vsext", "q", "qv", "cs">; 1618 def vzext_vf4 : RVVIntExt<"vzext", "Uq", "UqUv", "cs">; 1619} 1620let Log2LMUL = [-3, -2, -1, 0] in { 1621 def vsext_vf8 : RVVIntExt<"vsext", "o", "ov", "c">; 1622 def vzext_vf8 : RVVIntExt<"vzext", "Uo", "UoUv", "c">; 1623} 1624 1625// 12.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions 1626let HasMask = false, HasPolicy = false in { 1627 defm vadc : RVVCarryinBuiltinSet; 1628 defm vmadc : RVVCarryOutInBuiltinSet<"vmadc_carry_in">; 1629 defm vmadc : RVVIntMaskOutBuiltinSet; 1630 defm vsbc : RVVCarryinBuiltinSet; 1631 defm vmsbc : RVVCarryOutInBuiltinSet<"vmsbc_borrow_in">; 1632 defm vmsbc : RVVIntMaskOutBuiltinSet; 1633} 1634 1635// 12.5. Vector Bitwise Logical Instructions 1636defm vand : RVVIntBinBuiltinSet; 1637defm vxor : RVVIntBinBuiltinSet; 1638defm vor : RVVIntBinBuiltinSet; 1639defm vnot_v : RVVPseudoVNotBuiltin<"vxor", "csil">; 1640 1641// 12.6. Vector Single-Width Bit Shift Instructions 1642defm vsll : RVVShiftBuiltinSet; 1643defm vsrl : RVVUnsignedShiftBuiltinSet; 1644defm vsra : RVVSignedShiftBuiltinSet; 1645 1646// 12.7. Vector Narrowing Integer Right Shift Instructions 1647defm vnsrl : RVVUnsignedNShiftBuiltinSet; 1648defm vnsra : RVVSignedNShiftBuiltinSet; 1649defm vncvt_x_x_w : RVVPseudoVNCVTBuiltin<"vnsrl", "vncvt_x", "csi", 1650 [["v", "vw"], 1651 ["Uv", "UvUw"]]>; 1652 1653// 12.8. Vector Integer Comparison Instructions 1654let HasPolicy = false in { 1655defm vmseq : RVVIntMaskOutBuiltinSet; 1656defm vmsne : RVVIntMaskOutBuiltinSet; 1657defm vmsltu : RVVUnsignedMaskOutBuiltinSet; 1658defm vmslt : RVVSignedMaskOutBuiltinSet; 1659defm vmsleu : RVVUnsignedMaskOutBuiltinSet; 1660defm vmsle : RVVSignedMaskOutBuiltinSet; 1661defm vmsgtu : RVVUnsignedMaskOutBuiltinSet; 1662defm vmsgt : RVVSignedMaskOutBuiltinSet; 1663defm vmsgeu : RVVUnsignedMaskOutBuiltinSet; 1664defm vmsge : RVVSignedMaskOutBuiltinSet; 1665} 1666 1667// 12.9. Vector Integer Min/Max Instructions 1668defm vminu : RVVUnsignedBinBuiltinSet; 1669defm vmin : RVVSignedBinBuiltinSet; 1670defm vmaxu : RVVUnsignedBinBuiltinSet; 1671defm vmax : RVVSignedBinBuiltinSet; 1672 1673// 12.10. Vector Single-Width Integer Multiply Instructions 1674defm vmul : RVVIntBinBuiltinSet; 1675let RequiredFeatures = ["FullMultiply"] in { 1676defm vmulh : RVVSignedBinBuiltinSet; 1677defm vmulhu : RVVUnsignedBinBuiltinSet; 1678defm vmulhsu : RVVOutOp1BuiltinSet<"vmulhsu", "csil", 1679 [["vv", "v", "vvUv"], 1680 ["vx", "v", "vvUe"]]>; 1681} 1682 1683// 12.11. Vector Integer Divide Instructions 1684defm vdivu : RVVUnsignedBinBuiltinSet; 1685defm vdiv : RVVSignedBinBuiltinSet; 1686defm vremu : RVVUnsignedBinBuiltinSet; 1687defm vrem : RVVSignedBinBuiltinSet; 1688 1689// 12.12. Vector Widening Integer Multiply Instructions 1690let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1691defm vwmul : RVVOutOp0Op1BuiltinSet<"vwmul", "csi", 1692 [["vv", "w", "wvv"], 1693 ["vx", "w", "wve"]]>; 1694defm vwmulu : RVVOutOp0Op1BuiltinSet<"vwmulu", "csi", 1695 [["vv", "Uw", "UwUvUv"], 1696 ["vx", "Uw", "UwUvUe"]]>; 1697defm vwmulsu : RVVOutOp0Op1BuiltinSet<"vwmulsu", "csi", 1698 [["vv", "w", "wvUv"], 1699 ["vx", "w", "wvUe"]]>; 1700} 1701 1702// 12.13. Vector Single-Width Integer Multiply-Add Instructions 1703let HasPolicy = false in { 1704defm vmacc : RVVIntTerBuiltinSet; 1705defm vnmsac : RVVIntTerBuiltinSet; 1706defm vmadd : RVVIntTerBuiltinSet; 1707defm vnmsub : RVVIntTerBuiltinSet; 1708 1709// 12.14. Vector Widening Integer Multiply-Add Instructions 1710let HasMaskedOffOperand = false, 1711 Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1712defm vwmaccu : RVVOutOp1Op2BuiltinSet<"vwmaccu", "csi", 1713 [["vv", "Uw", "UwUwUvUv"], 1714 ["vx", "Uw", "UwUwUeUv"]]>; 1715defm vwmacc : RVVOutOp1Op2BuiltinSet<"vwmacc", "csi", 1716 [["vv", "w", "wwvv"], 1717 ["vx", "w", "wwev"]]>; 1718defm vwmaccsu : RVVOutOp1Op2BuiltinSet<"vwmaccsu", "csi", 1719 [["vv", "w", "wwvUv"], 1720 ["vx", "w", "wweUv"]]>; 1721defm vwmaccus : RVVOutOp1Op2BuiltinSet<"vwmaccus", "csi", 1722 [["vx", "w", "wwUev"]]>; 1723} 1724} 1725 1726// 12.15. Vector Integer Merge Instructions 1727// C/C++ Operand: (mask, op1, op2, vl), Intrinsic: (op1, op2, mask, vl) 1728let HasMask = false, HasPolicy = false, 1729 ManualCodegen = [{ 1730 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3); 1731 IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType()}; 1732 }] in { 1733 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "csil", 1734 [["vvm", "v", "vmvv"], 1735 ["vxm", "v", "vmve"], 1736 ["vvm", "Uv", "UvmUvUv"], 1737 ["vxm", "Uv", "UvmUvUe"]]>; 1738} 1739 1740// 12.16. Vector Integer Move Instructions 1741let HasMask = false, HasPolicy = false in { 1742 let MangledName = "vmv_v" in { 1743 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csil", 1744 [["v", "Uv", "UvUv"]]>; 1745 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilxfd", 1746 [["v", "v", "vv"]]>; 1747 } 1748 let HasNoMaskedOverloaded = false in 1749 defm vmv_v : RVVOutBuiltinSet<"vmv_v_x", "csil", 1750 [["x", "v", "ve"], 1751 ["x", "Uv", "UvUe"]]>; 1752} 1753 1754// 13. Vector Fixed-Point Arithmetic Instructions 1755// 13.1. Vector Single-Width Saturating Add and Subtract 1756defm vsaddu : RVVUnsignedBinBuiltinSet; 1757defm vsadd : RVVSignedBinBuiltinSet; 1758defm vssubu : RVVUnsignedBinBuiltinSet; 1759defm vssub : RVVSignedBinBuiltinSet; 1760 1761// 13.2. Vector Single-Width Averaging Add and Subtract 1762defm vaaddu : RVVUnsignedBinBuiltinSet; 1763defm vaadd : RVVSignedBinBuiltinSet; 1764defm vasubu : RVVUnsignedBinBuiltinSet; 1765defm vasub : RVVSignedBinBuiltinSet; 1766 1767// 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation 1768let RequiredFeatures = ["FullMultiply"] in { 1769defm vsmul : RVVSignedBinBuiltinSet; 1770} 1771 1772// 13.4. Vector Single-Width Scaling Shift Instructions 1773defm vssrl : RVVUnsignedShiftBuiltinSet; 1774defm vssra : RVVSignedShiftBuiltinSet; 1775 1776// 13.5. Vector Narrowing Fixed-Point Clip Instructions 1777defm vnclipu : RVVUnsignedNShiftBuiltinSet; 1778defm vnclip : RVVSignedNShiftBuiltinSet; 1779 1780// 14. Vector Floating-Point Instructions 1781// 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions 1782defm vfadd : RVVFloatingBinBuiltinSet; 1783defm vfsub : RVVFloatingBinBuiltinSet; 1784defm vfrsub : RVVFloatingBinVFBuiltinSet; 1785 1786// 14.3. Vector Widening Floating-Point Add/Subtract Instructions 1787// Widening FP add/subtract, 2*SEW = SEW +/- SEW 1788defm vfwadd : RVVFloatingWidenBinBuiltinSet; 1789defm vfwsub : RVVFloatingWidenBinBuiltinSet; 1790// Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW 1791defm vfwadd : RVVFloatingWidenOp0BinBuiltinSet; 1792defm vfwsub : RVVFloatingWidenOp0BinBuiltinSet; 1793 1794// 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions 1795defm vfmul : RVVFloatingBinBuiltinSet; 1796defm vfdiv : RVVFloatingBinBuiltinSet; 1797defm vfrdiv : RVVFloatingBinVFBuiltinSet; 1798 1799// 14.5. Vector Widening Floating-Point Multiply 1800let Log2LMUL = [-2, -1, 0, 1, 2] in { 1801 defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", 1802 [["vv", "w", "wvv"], 1803 ["vf", "w", "wve"]]>; 1804} 1805 1806// 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions 1807let HasPolicy = false in { 1808defm vfmacc : RVVFloatingTerBuiltinSet; 1809defm vfnmacc : RVVFloatingTerBuiltinSet; 1810defm vfmsac : RVVFloatingTerBuiltinSet; 1811defm vfnmsac : RVVFloatingTerBuiltinSet; 1812defm vfmadd : RVVFloatingTerBuiltinSet; 1813defm vfnmadd : RVVFloatingTerBuiltinSet; 1814defm vfmsub : RVVFloatingTerBuiltinSet; 1815defm vfnmsub : RVVFloatingTerBuiltinSet; 1816 1817// 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions 1818defm vfwmacc : RVVFloatingWidenTerBuiltinSet; 1819defm vfwnmacc : RVVFloatingWidenTerBuiltinSet; 1820defm vfwmsac : RVVFloatingWidenTerBuiltinSet; 1821defm vfwnmsac : RVVFloatingWidenTerBuiltinSet; 1822} 1823 1824// 14.8. Vector Floating-Point Square-Root Instruction 1825def vfsqrt : RVVFloatingUnaryVVBuiltin; 1826 1827// 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction 1828def vfrsqrt7 : RVVFloatingUnaryVVBuiltin; 1829 1830// 14.10. Vector Floating-Point Reciprocal Estimate Instruction 1831def vfrec7 : RVVFloatingUnaryVVBuiltin; 1832 1833// 14.11. Vector Floating-Point MIN/MAX Instructions 1834defm vfmin : RVVFloatingBinBuiltinSet; 1835defm vfmax : RVVFloatingBinBuiltinSet; 1836 1837// 14.12. Vector Floating-Point Sign-Injection Instructions 1838defm vfsgnj : RVVFloatingBinBuiltinSet; 1839defm vfsgnjn : RVVFloatingBinBuiltinSet; 1840defm vfsgnjx : RVVFloatingBinBuiltinSet; 1841defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "xfd">; 1842defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "xfd">; 1843 1844// 14.13. Vector Floating-Point Compare Instructions 1845let HasPolicy = false in { 1846defm vmfeq : RVVFloatingMaskOutBuiltinSet; 1847defm vmfne : RVVFloatingMaskOutBuiltinSet; 1848defm vmflt : RVVFloatingMaskOutBuiltinSet; 1849defm vmfle : RVVFloatingMaskOutBuiltinSet; 1850defm vmfgt : RVVFloatingMaskOutBuiltinSet; 1851defm vmfge : RVVFloatingMaskOutBuiltinSet; 1852} 1853 1854// 14.14. Vector Floating-Point Classify Instruction 1855let Name = "vfclass_v", HasPolicy = false in 1856 def vfclass : RVVOp0Builtin<"Uv", "Uvv", "xfd">; 1857 1858// 14.15. Vector Floating-Point Merge Instructio 1859// C/C++ Operand: (mask, op1, op2, vl), Builtin: (op1, op2, mask, vl) 1860let HasMask = false, HasPolicy = false, 1861 ManualCodegen = [{ 1862 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3); 1863 IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType()}; 1864 }] in { 1865 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "xfd", 1866 [["vvm", "v", "vmvv"]]>; 1867 defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "xfd", 1868 [["vfm", "v", "vmve"]]>; 1869} 1870 1871// 14.16. Vector Floating-Point Move Instruction 1872let HasMask = false, HasNoMaskedOverloaded = false, HasPolicy = false in 1873 defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "xfd", 1874 [["f", "v", "ve"]]>; 1875 1876// 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions 1877def vfcvt_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_xu">; 1878def vfcvt_x_f_v : RVVConvToSignedBuiltin<"vfcvt_x">; 1879def vfcvt_rtz_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_rtz_xu">; 1880def vfcvt_rtz_x_f_v : RVVConvToSignedBuiltin<"vfcvt_rtz_x">; 1881def vfcvt_f_xu_v : RVVConvBuiltin<"Fv", "FvUv", "sil", "vfcvt_f">; 1882def vfcvt_f_x_v : RVVConvBuiltin<"Fv", "Fvv", "sil", "vfcvt_f">; 1883 1884// 14.18. Widening Floating-Point/Integer Type-Convert Instructions 1885let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1886 def vfwcvt_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_xu">; 1887 def vfwcvt_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_x">; 1888 def vfwcvt_rtz_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_rtz_xu">; 1889 def vfwcvt_rtz_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_rtz_x">; 1890 def vfwcvt_f_xu_v : RVVConvBuiltin<"Fw", "FwUv", "csi", "vfwcvt_f">; 1891 def vfwcvt_f_x_v : RVVConvBuiltin<"Fw", "Fwv", "csi", "vfwcvt_f">; 1892 def vfwcvt_f_f_v : RVVConvBuiltin<"w", "wv", "xf", "vfwcvt_f">; 1893} 1894 1895// 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions 1896let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1897 def vfncvt_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_xu">; 1898 def vfncvt_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_x">; 1899 def vfncvt_rtz_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_rtz_xu">; 1900 def vfncvt_rtz_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_rtz_x">; 1901 def vfncvt_f_xu_w : RVVConvBuiltin<"Fv", "FvUw", "csi", "vfncvt_f">; 1902 def vfncvt_f_x_w : RVVConvBuiltin<"Fv", "Fvw", "csi", "vfncvt_f">; 1903 def vfncvt_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_f">; 1904 def vfncvt_rod_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_rod_f">; 1905} 1906 1907// 15. Vector Reduction Operations 1908// 15.1. Vector Single-Width Integer Reduction Instructions 1909let HasPolicy = false in { 1910defm vredsum : RVVIntReductionBuiltinSet; 1911defm vredmaxu : RVVUnsignedReductionBuiltin; 1912defm vredmax : RVVSignedReductionBuiltin; 1913defm vredminu : RVVUnsignedReductionBuiltin; 1914defm vredmin : RVVSignedReductionBuiltin; 1915defm vredand : RVVIntReductionBuiltinSet; 1916defm vredor : RVVIntReductionBuiltinSet; 1917defm vredxor : RVVIntReductionBuiltinSet; 1918 1919// 15.2. Vector Widening Integer Reduction Instructions 1920// Vector Widening Integer Reduction Operations 1921let HasMaskedOffOperand = false in { 1922 defm vwredsum : RVVOutOp1BuiltinSet<"vwredsum", "csi", 1923 [["vs", "vSw", "SwSwvSw"]]>; 1924 defm vwredsumu : RVVOutOp1BuiltinSet<"vwredsumu", "csi", 1925 [["vs", "UvUSw", "USwUSwUvUSw"]]>; 1926} 1927 1928// 15.3. Vector Single-Width Floating-Point Reduction Instructions 1929defm vfredmax : RVVFloatingReductionBuiltin; 1930defm vfredmin : RVVFloatingReductionBuiltin; 1931defm vfredusum : RVVFloatingReductionBuiltin; 1932defm vfredosum : RVVFloatingReductionBuiltin; 1933 1934// 15.4. Vector Widening Floating-Point Reduction Instructions 1935defm vfwredusum : RVVFloatingWidenReductionBuiltin; 1936defm vfwredosum : RVVFloatingWidenReductionBuiltin; 1937} 1938 1939// 16. Vector Mask Instructions 1940// 16.1. Vector Mask-Register Logical Instructions 1941def vmand : RVVMaskBinBuiltin; 1942def vmnand : RVVMaskBinBuiltin; 1943def vmandn : RVVMaskBinBuiltin; 1944def vmxor : RVVMaskBinBuiltin; 1945def vmor : RVVMaskBinBuiltin; 1946def vmnor : RVVMaskBinBuiltin; 1947def vmorn : RVVMaskBinBuiltin; 1948def vmxnor : RVVMaskBinBuiltin; 1949// pseudoinstructions 1950def vmclr : RVVMaskNullaryBuiltin; 1951def vmset : RVVMaskNullaryBuiltin; 1952defm vmmv_m : RVVPseudoMaskBuiltin<"vmand", "c">; 1953defm vmnot_m : RVVPseudoMaskBuiltin<"vmnand", "c">; 1954 1955let HasPolicy = false in { 1956// 16.2. Vector count population in mask vcpop.m 1957def vcpop : RVVMaskOp0Builtin<"um">; 1958 1959// 16.3. vfirst find-first-set mask bit 1960def vfirst : RVVMaskOp0Builtin<"lm">; 1961 1962// 16.4. vmsbf.m set-before-first mask bit 1963def vmsbf : RVVMaskUnaryBuiltin; 1964 1965// 16.5. vmsif.m set-including-first mask bit 1966def vmsif : RVVMaskUnaryBuiltin; 1967 1968// 16.6. vmsof.m set-only-first mask bit 1969def vmsof : RVVMaskUnaryBuiltin; 1970 1971let HasNoMaskedOverloaded = false in { 1972 // 16.8. Vector Iota Instruction 1973 defm viota : RVVOutBuiltinSet<"viota", "csil", [["m", "Uv", "Uvm"]]>; 1974 1975 // 16.9. Vector Element Index Instruction 1976 defm vid : RVVOutBuiltinSet<"vid", "csil", [["v", "v", "v"], 1977 ["v", "Uv", "Uv"]]>; 1978} 1979} 1980 1981// 17. Vector Permutation Instructions 1982// 17.1. Integer Scalar Move Instructions 1983let HasMask = false, HasPolicy = false in { 1984 let HasVL = false, MangledName = "vmv_x" in 1985 defm vmv_x : RVVOp0BuiltinSet<"vmv_x_s", "csil", 1986 [["s", "ve", "ev"], 1987 ["s", "UvUe", "UeUv"]]>; 1988 let MangledName = "vmv_s" in 1989 defm vmv_s : RVVOutBuiltinSet<"vmv_s_x", "csil", 1990 [["x", "v", "vve"], 1991 ["x", "Uv", "UvUvUe"]]>; 1992} 1993 1994// 17.2. Floating-Point Scalar Move Instructions 1995let HasMask = false, HasPolicy = false in { 1996 let HasVL = false, MangledName = "vfmv_f" in 1997 defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "xfd", 1998 [["s", "ve", "ev"]]>; 1999 let MangledName = "vfmv_s" in 2000 defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "xfd", 2001 [["f", "v", "vve"], 2002 ["x", "Uv", "UvUvUe"]]>; 2003} 2004 2005// 17.3. Vector Slide Instructions 2006let HasPolicy = false in { 2007// 17.3.1. Vector Slideup Instructions 2008defm vslideup : RVVSlideBuiltinSet; 2009// 17.3.2. Vector Slidedown Instructions 2010defm vslidedown : RVVSlideBuiltinSet; 2011} 2012 2013// 17.3.3. Vector Slide1up Instructions 2014defm vslide1up : RVVSlideOneBuiltinSet; 2015defm vfslide1up : RVVFloatingBinVFBuiltinSet; 2016 2017// 17.3.4. Vector Slide1down Instruction 2018defm vslide1down : RVVSlideOneBuiltinSet; 2019defm vfslide1down : RVVFloatingBinVFBuiltinSet; 2020 2021// 17.4. Vector Register Gather Instructions 2022// signed and floating type 2023defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfd", 2024 [["vv", "v", "vvUv"]]>; 2025defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfd", 2026 [["vx", "v", "vvz"]]>; 2027defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd", 2028 [["vv", "v", "vv(Log2EEW:4)Uv"]]>; 2029// unsigned type 2030defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil", 2031 [["vv", "Uv", "UvUvUv"]]>; 2032defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csil", 2033 [["vx", "Uv", "UvUvz"]]>; 2034defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csil", 2035 [["vv", "Uv", "UvUv(Log2EEW:4)Uv"]]>; 2036 2037// 17.5. Vector Compress Instruction 2038let HasMask = false, HasPolicy = false, 2039 ManualCodegen = [{ 2040 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3); 2041 IntrinsicTypes = {ResultType, Ops[3]->getType()}; 2042 }] in { 2043 // signed and floating type 2044 defm vcompress : RVVOutBuiltinSet<"vcompress", "csilxfd", 2045 [["vm", "v", "vmvv"]]>; 2046 // unsigned type 2047 defm vcompress : RVVOutBuiltinSet<"vcompress", "csil", 2048 [["vm", "Uv", "UvmUvUv"]]>; 2049} 2050 2051// Miscellaneous 2052let HasMask = false, HasVL = false, IRName = "" in { 2053 let Name = "vreinterpret_v", HasPolicy = false, 2054 ManualCodegen = [{ 2055 return Builder.CreateBitCast(Ops[0], ResultType); 2056 }] in { 2057 // Reinterpret between different type under the same SEW and LMUL 2058 def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">; 2059 def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "sil", "v">; 2060 def vreinterpret_u_i : RVVBuiltin<"vUv", "Uvv", "csil", "Uv">; 2061 def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "sil", "Uv">; 2062 def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "sil", "Fv">; 2063 def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "sil", "Fv">; 2064 2065 // Reinterpret between different SEW under the same LMUL 2066 foreach dst_sew = ["(FixedSEW:8)", "(FixedSEW:16)", "(FixedSEW:32)", 2067 "(FixedSEW:64)"] in { 2068 def vreinterpret_i_ # dst_sew : RVVBuiltin<"v" # dst_sew # "v", 2069 dst_sew # "vv", "csil", dst_sew # "v">; 2070 def vreinterpret_u_ # dst_sew : RVVBuiltin<"Uv" # dst_sew # "Uv", 2071 dst_sew # "UvUv", "csil", dst_sew # "Uv">; 2072 } 2073 } 2074 2075 let Name = "vundefined", HasNoMaskedOverloaded = false, HasPolicy = false, 2076 ManualCodegen = [{ 2077 return llvm::UndefValue::get(ResultType); 2078 }] in { 2079 def vundefined : RVVBuiltin<"v", "v", "csilxfd">; 2080 def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">; 2081 } 2082 2083 // LMUL truncation 2084 // C/C++ Operand: VecTy, IR Operand: VecTy, Index 2085 let Name = "vlmul_trunc_v", MangledName = "vlmul_trunc", HasPolicy = false, 2086 ManualCodegen = [{ { 2087 ID = Intrinsic::experimental_vector_extract; 2088 IntrinsicTypes = {ResultType, Ops[0]->getType()}; 2089 Ops.push_back(ConstantInt::get(Int64Ty, 0)); 2090 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2091 } }] in { 2092 foreach dst_lmul = ["(SFixedLog2LMUL:-3)", "(SFixedLog2LMUL:-2)", "(SFixedLog2LMUL:-1)", 2093 "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2094 def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2095 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2096 def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2097 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2098 } 2099 } 2100 2101 // LMUL extension 2102 // C/C++ Operand: SubVecTy, IR Operand: VecTy, SubVecTy, Index 2103 let Name = "vlmul_ext_v", MangledName = "vlmul_ext", HasPolicy = false, 2104 ManualCodegen = [{ 2105 ID = Intrinsic::experimental_vector_insert; 2106 IntrinsicTypes = {ResultType, Ops[0]->getType()}; 2107 Ops.push_back(llvm::UndefValue::get(ResultType)); 2108 std::swap(Ops[0], Ops[1]); 2109 Ops.push_back(ConstantInt::get(Int64Ty, 0)); 2110 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2111 }] in { 2112 foreach dst_lmul = ["(LFixedLog2LMUL:-2)", "(LFixedLog2LMUL:-1)", "(LFixedLog2LMUL:-0)", 2113 "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2114 def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2115 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2116 def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2117 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2118 } 2119 } 2120 2121 let Name = "vget_v", HasPolicy = false, 2122 ManualCodegen = [{ 2123 { 2124 ID = Intrinsic::experimental_vector_extract; 2125 auto *VecTy = cast<ScalableVectorType>(ResultType); 2126 auto *OpVecTy = cast<ScalableVectorType>(Ops[0]->getType()); 2127 // Mask to only valid indices. 2128 unsigned MaxIndex = OpVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2129 assert(isPowerOf2_32(MaxIndex)); 2130 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2131 Ops[1] = Builder.CreateMul(Ops[1], 2132 ConstantInt::get(Ops[1]->getType(), 2133 VecTy->getMinNumElements())); 2134 IntrinsicTypes = {ResultType, Ops[0]->getType()}; 2135 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2136 } 2137 }] in { 2138 foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2139 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfd", dst_lmul # "v">; 2140 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">; 2141 } 2142 } 2143 2144 let Name = "vset_v", Log2LMUL = [0, 1, 2], HasPolicy = false, 2145 ManualCodegen = [{ 2146 { 2147 ID = Intrinsic::experimental_vector_insert; 2148 IntrinsicTypes = {ResultType, Ops[2]->getType()}; 2149 auto *ResVecTy = cast<ScalableVectorType>(ResultType); 2150 auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType()); 2151 // Mask to only valid indices. 2152 unsigned MaxIndex = ResVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2153 assert(isPowerOf2_32(MaxIndex)); 2154 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2155 Ops[1] = Builder.CreateMul(Ops[1], 2156 ConstantInt::get(Ops[1]->getType(), 2157 VecTy->getMinNumElements())); 2158 std::swap(Ops[1], Ops[2]); 2159 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2160 } 2161 }] in { 2162 foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2163 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfd">; 2164 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">; 2165 } 2166 } 2167} 2168 2169let HeaderCode = [{ 2170#define VE_TAIL_UNDISTURBED 0 2171#define VE_TAIL_AGNOSTIC 1 2172}] in 2173def policy : RVVHeader; 2174