1//==--- riscv_vector.td - RISC-V V-ext Builtin function list --------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the builtins for RISC-V V-extension. See: 10// 11// https://github.com/riscv/rvv-intrinsic-doc 12// 13//===----------------------------------------------------------------------===// 14 15//===----------------------------------------------------------------------===// 16// Instruction definitions 17//===----------------------------------------------------------------------===// 18// Each record of the class RVVBuiltin defines a collection of builtins (i.e. 19// "def vadd : RVVBuiltin" will be used to define things like "vadd_vv_i32m1", 20// "vadd_vv_i32m2", etc). 21// 22// The elements of this collection are defined by an instantiation process the 23// range of which is specified by the cross product of the LMUL attribute and 24// every element in the attribute TypeRange. By default builtins have LMUL = [1, 25// 2, 4, 8, 1/2, 1/4, 1/8] so the process is repeated 7 times. In tablegen we 26// use the Log2LMUL [0, 1, 2, 3, -1, -2, -3] to represent the LMUL. 27// 28// LMUL represents the fact that the types of values used by that builtin are 29// values generated by instructions that are executed under that LMUL. However, 30// this does not mean the builtin is necessarily lowered into an instruction 31// that executes under the specified LMUL. An example where this happens are 32// loads and stores of masks. A mask like `vbool8_t` can be generated, for 33// instance, by comparing two `__rvv_int8m1_t` (this is LMUL=1) or comparing two 34// `__rvv_int16m2_t` (this is LMUL=2). The actual load or store, however, will 35// be performed under LMUL=1 because mask registers are not grouped. 36// 37// TypeRange is a non-empty sequence of basic types: 38// 39// c: int8_t (i8) 40// s: int16_t (i16) 41// i: int32_t (i32) 42// l: int64_t (i64) 43// x: float16_t (half) 44// f: float32_t (float) 45// d: float64_t (double) 46// 47// This way, given an LMUL, a record with a TypeRange "sil" will cause the 48// definition of 3 builtins. Each type "t" in the TypeRange (in this example 49// they are int16_t, int32_t, int64_t) is used as a parameter that drives the 50// definition of that particular builtin (for the given LMUL). 51// 52// During the instantiation, types can be transformed or modified using type 53// transformers. Given a type "t" the following primitive type transformers can 54// be applied to it to yield another type. 55// 56// e: type of "t" as is (identity) 57// v: computes a vector type whose element type is "t" for the current LMUL 58// w: computes a vector type identical to what 'v' computes except for the 59// element type which is twice as wide as the element type of 'v' 60// q: computes a vector type identical to what 'v' computes except for the 61// element type which is four times as wide as the element type of 'v' 62// o: computes a vector type identical to what 'v' computes except for the 63// element type which is eight times as wide as the element type of 'v' 64// m: computes a vector type identical to what 'v' computes except for the 65// element type which is bool 66// 0: void type, ignores "t" 67// z: size_t, ignores "t" 68// t: ptrdiff_t, ignores "t" 69// u: unsigned long, ignores "t" 70// l: long, ignores "t" 71// 72// So for instance if t is "i", i.e. int, then "e" will yield int again. "v" 73// will yield an RVV vector type (assume LMUL=1), so __rvv_int32m1_t. 74// Accordingly "w" would yield __rvv_int64m2_t. 75// 76// A type transformer can be prefixed by other non-primitive type transformers. 77// 78// P: constructs a pointer to the current type 79// C: adds const to the type 80// K: requires the integer type to be a constant expression 81// U: given an integer type or vector type, computes its unsigned variant 82// I: given a vector type, compute the vector type with integer type 83// elements of the same width 84// F: given a vector type, compute the vector type with floating-point type 85// elements of the same width 86// S: given a vector type, computes its equivalent one for LMUL=1. This is a 87// no-op if the vector was already LMUL=1 88// (Log2EEW:Value): Log2EEW value could be 3/4/5/6 (8/16/32/64), given a 89// vector type (SEW and LMUL) and EEW (8/16/32/64), computes its 90// equivalent integer vector type with EEW and corresponding ELMUL (elmul = 91// (eew/sew) * lmul). For example, vector type is __rvv_float16m4 92// (SEW=16, LMUL=4) and Log2EEW is 3 (EEW=8), and then equivalent vector 93// type is __rvv_uint8m2_t (elmul=(8/16)*4 = 2). Ignore to define a new 94// builtins if its equivalent type has illegal lmul. 95// (FixedSEW:Value): Given a vector type (SEW and LMUL), and computes another 96// vector type which only changed SEW as given value. Ignore to define a new 97// builtin if its equivalent type has illegal lmul or the SEW does not changed. 98// (SFixedLog2LMUL:Value): Smaller Fixed Log2LMUL. Given a vector type (SEW 99// and LMUL), and computes another vector type which only changed LMUL as 100// given value. The new LMUL should be smaller than the old one. Ignore to 101// define a new builtin if its equivalent type has illegal lmul. 102// (LFixedLog2LMUL:Value): Larger Fixed Log2LMUL. Given a vector type (SEW 103// and LMUL), and computes another vector type which only changed LMUL as 104// given value. The new LMUL should be larger than the old one. Ignore to 105// define a new builtin if its equivalent type has illegal lmul. 106// 107// Following with the example above, if t is "i", then "Ue" will yield unsigned 108// int and "Fv" will yield __rvv_float32m1_t (again assuming LMUL=1), Fw would 109// yield __rvv_float64m2_t, etc. 110// 111// Each builtin is then defined by applying each type in TypeRange against the 112// sequence of type transformers described in Suffix and Prototype. 113// 114// The name of the builtin is defined by the Name attribute (which defaults to 115// the name of the class) appended (separated with an underscore) the Suffix 116// attribute. For instance with Name="foo", Suffix = "v" and TypeRange = "il", 117// the builtin generated will be __builtin_rvv_foo_i32m1 and 118// __builtin_rvv_foo_i64m1 (under LMUL=1). If Suffix contains more than one 119// type transformer (say "vv") each of the types is separated with an 120// underscore as in "__builtin_rvv_foo_i32m1_i32m1". 121// 122// The C/C++ prototype of the builtin is defined by the Prototype attribute. 123// Prototype is a non-empty sequence of type transformers, the first of which 124// is the return type of the builtin and the rest are the parameters of the 125// builtin, in order. For instance if Prototype is "wvv" and TypeRange is "si" 126// a first builtin will have type 127// __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t) and the second builtin 128// will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t) (again 129// under LMUL=1). 130// 131// There are a number of attributes that are used to constraint the number and 132// shape of the builtins generated. Refer to the comments below for them. 133 134class Policy<int val>{ 135 int Value = val; 136} 137def NonePolicy : Policy<0>; 138def HasPassthruOperand : Policy<1>; 139def HasPolicyOperand : Policy<2>; 140 141class RVVBuiltin<string suffix, string prototype, string type_range, 142 string overloaded_suffix = ""> { 143 // Base name that will be prepended in __builtin_rvv_ and appended the 144 // computed Suffix. 145 string Name = NAME; 146 147 // If not empty, each instantiated builtin will have this appended after an 148 // underscore (_). It is instantiated like Prototype. 149 string Suffix = suffix; 150 151 // If empty, default OverloadedName is sub string of `Name` which end of first 152 // '_'. For example, the default overloaded name is `vadd` for Name `vadd_vv`. 153 // It's used for describe some special naming cases. 154 string OverloadedName = ""; 155 156 // If not empty, each OverloadedName will have this appended after an 157 // underscore (_). It is instantiated like Prototype. 158 string OverloadedSuffix = overloaded_suffix; 159 160 // The different variants of the builtin, parameterised with a type. 161 string TypeRange = type_range; 162 163 // We use each type described in TypeRange and LMUL with prototype to 164 // instantiate a specific element of the set of builtins being defined. 165 // Prototype attribute defines the C/C++ prototype of the builtin. It is a 166 // non-empty sequence of type transformers, the first of which is the return 167 // type of the builtin and the rest are the parameters of the builtin, in 168 // order. For instance if Prototype is "wvv", TypeRange is "si" and LMUL=1, a 169 // first builtin will have type 170 // __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t), and the second builtin 171 // will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t). 172 string Prototype = prototype; 173 174 // This builtin has a masked form. 175 bit HasMasked = true; 176 177 // If HasMasked, this flag states that this builtin has a maskedoff operand. It 178 // is always the first operand in builtin and IR intrinsic. 179 bit HasMaskedOffOperand = true; 180 181 // This builtin has a granted vector length parameter. 182 bit HasVL = true; 183 184 // The policy scheme for masked intrinsic IR. 185 // It could be NonePolicy or HasPolicyOperand. 186 // HasPolicyOperand: Has a policy operand. 0 is tail and mask undisturbed, 1 is 187 // tail agnostic, 2 is mask undisturbed, and 3 is tail and mask agnostic. The 188 // policy operand is located at the last position. 189 Policy MaskedPolicy = HasPolicyOperand; 190 191 // The policy scheme for unmasked intrinsic IR. 192 // It could be NonePolicy, HasPassthruOperand or HasPolicyOperand. 193 // HasPassthruOperand: Has a passthru operand to decide tail policy. If it is 194 // undef, tail policy is tail agnostic, otherwise policy is tail undisturbed. 195 // HasPolicyOperand: Has a policy operand. 1 is tail agnostic and 0 is tail 196 // undisturbed. 197 Policy UnMaskedPolicy = NonePolicy; 198 199 // This builtin supports non-masked function overloading api. 200 // All masked operations support overloading api. 201 bit HasUnMaskedOverloaded = true; 202 203 // This builtin is valid for the given Log2LMULs. 204 list<int> Log2LMUL = [0, 1, 2, 3, -1, -2, -3]; 205 206 // Manual code in clang codegen riscv_vector_builtin_cg.inc 207 code ManualCodegen = [{}]; 208 code MaskedManualCodegen = [{}]; 209 210 // When emit the automatic clang codegen, it describes what types we have to use 211 // to obtain the specific LLVM intrinsic. -1 means the return type, otherwise, 212 // k >= 0 meaning the k-th operand (counting from zero) of the codegen'd 213 // parameter of the unmasked version. k can't be the mask operand's position. 214 list<int> IntrinsicTypes = []; 215 216 // If these names are not empty, this is the ID of the LLVM intrinsic 217 // we want to lower to. 218 string IRName = NAME; 219 220 // If HasMasked, this is the ID of the LLVM intrinsic we want to lower to. 221 string MaskedIRName = NAME #"_mask"; 222 223 // Use clang_builtin_alias to save the number of builtins. 224 bit HasBuiltinAlias = true; 225 226 // Features required to enable for this builtin. 227 list<string> RequiredFeatures = []; 228 229 // Number of fields for Load/Store Segment instructions. 230 int NF = 1; 231} 232 233// This is the code emitted in the header. 234class RVVHeader { 235 code HeaderCode; 236} 237 238//===----------------------------------------------------------------------===// 239// Basic classes with automatic codegen. 240//===----------------------------------------------------------------------===// 241 242class RVVOutBuiltin<string suffix, string prototype, string type_range> 243 : RVVBuiltin<suffix, prototype, type_range> { 244 let IntrinsicTypes = [-1]; 245} 246 247class RVVOp0Builtin<string suffix, string prototype, string type_range> 248 : RVVBuiltin<suffix, prototype, type_range> { 249 let IntrinsicTypes = [0]; 250} 251 252class RVVOutOp1Builtin<string suffix, string prototype, string type_range> 253 : RVVBuiltin<suffix, prototype, type_range> { 254 let IntrinsicTypes = [-1, 1]; 255} 256 257class RVVOutOp0Op1Builtin<string suffix, string prototype, string type_range> 258 : RVVBuiltin<suffix, prototype, type_range> { 259 let IntrinsicTypes = [-1, 0, 1]; 260} 261 262multiclass RVVBuiltinSet<string intrinsic_name, string type_range, 263 list<list<string>> suffixes_prototypes, 264 list<int> intrinsic_types> { 265 let IRName = intrinsic_name, MaskedIRName = intrinsic_name # "_mask", 266 IntrinsicTypes = intrinsic_types in { 267 foreach s_p = suffixes_prototypes in { 268 let Name = NAME # "_" # s_p[0] in { 269 defvar suffix = s_p[1]; 270 defvar prototype = s_p[2]; 271 def : RVVBuiltin<suffix, prototype, type_range>; 272 } 273 } 274 } 275} 276 277// IntrinsicTypes is output, op0, op1 [-1, 0, 1] 278multiclass RVVOutOp0Op1BuiltinSet<string intrinsic_name, string type_range, 279 list<list<string>> suffixes_prototypes> 280 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, 281 [-1, 0, 1]>; 282 283multiclass RVVOutBuiltinSet<string intrinsic_name, string type_range, 284 list<list<string>> suffixes_prototypes> 285 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1]>; 286 287multiclass RVVOp0BuiltinSet<string intrinsic_name, string type_range, 288 list<list<string>> suffixes_prototypes> 289 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0]>; 290 291// IntrinsicTypes is output, op1 [-1, 1] 292multiclass RVVOutOp1BuiltinSet<string intrinsic_name, string type_range, 293 list<list<string>> suffixes_prototypes> 294 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1]>; 295 296multiclass RVVOp0Op1BuiltinSet<string intrinsic_name, string type_range, 297 list<list<string>> suffixes_prototypes> 298 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0, 1]>; 299 300multiclass RVVOutOp1Op2BuiltinSet<string intrinsic_name, string type_range, 301 list<list<string>> suffixes_prototypes> 302 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1, 2]>; 303 304multiclass RVVSignedBinBuiltinSet 305 : RVVOutOp1BuiltinSet<NAME, "csil", 306 [["vv", "v", "vvv"], 307 ["vx", "v", "vve"]]>; 308 309multiclass RVVUnsignedBinBuiltinSet 310 : RVVOutOp1BuiltinSet<NAME, "csil", 311 [["vv", "Uv", "UvUvUv"], 312 ["vx", "Uv", "UvUvUe"]]>; 313 314multiclass RVVIntBinBuiltinSet 315 : RVVSignedBinBuiltinSet, 316 RVVUnsignedBinBuiltinSet; 317 318multiclass RVVSlideOneBuiltinSet 319 : RVVOutOp1BuiltinSet<NAME, "csil", 320 [["vx", "v", "vve"], 321 ["vx", "Uv", "UvUve"]]>; 322 323multiclass RVVSignedShiftBuiltinSet 324 : RVVOutOp1BuiltinSet<NAME, "csil", 325 [["vv", "v", "vvUv"], 326 ["vx", "v", "vvz"]]>; 327 328multiclass RVVUnsignedShiftBuiltinSet 329 : RVVOutOp1BuiltinSet<NAME, "csil", 330 [["vv", "Uv", "UvUvUv"], 331 ["vx", "Uv", "UvUvz"]]>; 332 333multiclass RVVShiftBuiltinSet 334 : RVVSignedShiftBuiltinSet, 335 RVVUnsignedShiftBuiltinSet; 336 337let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 338 multiclass RVVSignedNShiftBuiltinSet 339 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 340 [["wv", "v", "vwUv"], 341 ["wx", "v", "vwz"]]>; 342 multiclass RVVUnsignedNShiftBuiltinSet 343 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 344 [["wv", "Uv", "UvUwUv"], 345 ["wx", "Uv", "UvUwz"]]>; 346} 347 348multiclass RVVCarryinBuiltinSet 349 : RVVOutOp1BuiltinSet<NAME, "csil", 350 [["vvm", "v", "vvvm"], 351 ["vxm", "v", "vvem"], 352 ["vvm", "Uv", "UvUvUvm"], 353 ["vxm", "Uv", "UvUvUem"]]>; 354 355multiclass RVVCarryOutInBuiltinSet<string intrinsic_name> 356 : RVVOp0Op1BuiltinSet<intrinsic_name, "csil", 357 [["vvm", "vm", "mvvm"], 358 ["vxm", "vm", "mvem"], 359 ["vvm", "Uvm", "mUvUvm"], 360 ["vxm", "Uvm", "mUvUem"]]>; 361 362multiclass RVVSignedMaskOutBuiltinSet 363 : RVVOp0Op1BuiltinSet<NAME, "csil", 364 [["vv", "vm", "mvv"], 365 ["vx", "vm", "mve"]]>; 366 367multiclass RVVUnsignedMaskOutBuiltinSet 368 : RVVOp0Op1BuiltinSet<NAME, "csil", 369 [["vv", "Uvm", "mUvUv"], 370 ["vx", "Uvm", "mUvUe"]]>; 371 372multiclass RVVIntMaskOutBuiltinSet 373 : RVVSignedMaskOutBuiltinSet, 374 RVVUnsignedMaskOutBuiltinSet; 375 376class RVVIntExt<string intrinsic_name, string suffix, string prototype, 377 string type_range> 378 : RVVBuiltin<suffix, prototype, type_range> { 379 let IRName = intrinsic_name; 380 let MaskedIRName = intrinsic_name # "_mask"; 381 let OverloadedName = NAME; 382 let IntrinsicTypes = [-1, 0]; 383} 384 385let HasMaskedOffOperand = false in { 386 multiclass RVVIntTerBuiltinSet { 387 defm "" : RVVOutOp1BuiltinSet<NAME, "csil", 388 [["vv", "v", "vvvv"], 389 ["vx", "v", "vvev"], 390 ["vv", "Uv", "UvUvUvUv"], 391 ["vx", "Uv", "UvUvUeUv"]]>; 392 } 393 multiclass RVVFloatingTerBuiltinSet { 394 defm "" : RVVOutOp1BuiltinSet<NAME, "xfd", 395 [["vv", "v", "vvvv"], 396 ["vf", "v", "vvev"]]>; 397 } 398} 399 400let HasMaskedOffOperand = false, Log2LMUL = [-2, -1, 0, 1, 2] in { 401 multiclass RVVFloatingWidenTerBuiltinSet { 402 defm "" : RVVOutOp1Op2BuiltinSet<NAME, "xf", 403 [["vv", "w", "wwvv"], 404 ["vf", "w", "wwev"]]>; 405 } 406} 407 408multiclass RVVFloatingBinBuiltinSet 409 : RVVOutOp1BuiltinSet<NAME, "xfd", 410 [["vv", "v", "vvv"], 411 ["vf", "v", "vve"]]>; 412 413multiclass RVVFloatingBinVFBuiltinSet 414 : RVVOutOp1BuiltinSet<NAME, "xfd", 415 [["vf", "v", "vve"]]>; 416 417multiclass RVVFloatingMaskOutBuiltinSet 418 : RVVOp0Op1BuiltinSet<NAME, "xfd", 419 [["vv", "vm", "mvv"], 420 ["vf", "vm", "mve"]]>; 421 422multiclass RVVFloatingMaskOutVFBuiltinSet 423 : RVVOp0Op1BuiltinSet<NAME, "fd", 424 [["vf", "vm", "mve"]]>; 425 426class RVVMaskBinBuiltin : RVVOutBuiltin<"m", "mmm", "c"> { 427 let Name = NAME # "_mm"; 428 let HasMasked = false; 429} 430 431class RVVMaskUnaryBuiltin : RVVOutBuiltin<"m", "mm", "c"> { 432 let Name = NAME # "_m"; 433} 434 435class RVVMaskNullaryBuiltin : RVVOutBuiltin<"m", "m", "c"> { 436 let Name = NAME # "_m"; 437 let HasMasked = false; 438 let HasUnMaskedOverloaded = false; 439} 440 441class RVVMaskOp0Builtin<string prototype> : RVVOp0Builtin<"m", prototype, "c"> { 442 let Name = NAME # "_m"; 443 let HasMaskedOffOperand = false; 444} 445 446let UnMaskedPolicy = HasPolicyOperand, 447 HasMaskedOffOperand = false in { 448 multiclass RVVSlideBuiltinSet { 449 defm "" : RVVOutBuiltinSet<NAME, "csilxfd", 450 [["vx","v", "vvvz"]]>; 451 defm "" : RVVOutBuiltinSet<NAME, "csil", 452 [["vx","Uv", "UvUvUvz"]]>; 453 } 454} 455 456class RVVFloatingUnaryBuiltin<string builtin_suffix, string ir_suffix, 457 string prototype> 458 : RVVOutBuiltin<ir_suffix, prototype, "xfd"> { 459 let Name = NAME # "_" # builtin_suffix; 460} 461 462class RVVFloatingUnaryVVBuiltin : RVVFloatingUnaryBuiltin<"v", "v", "vv">; 463 464class RVVConvBuiltin<string suffix, string prototype, string type_range, 465 string overloaded_name> 466 : RVVBuiltin<suffix, prototype, type_range> { 467 let IntrinsicTypes = [-1, 0]; 468 let OverloadedName = overloaded_name; 469} 470 471class RVVConvToSignedBuiltin<string overloaded_name> 472 : RVVConvBuiltin<"Iv", "Ivv", "xfd", overloaded_name>; 473 474class RVVConvToUnsignedBuiltin<string overloaded_name> 475 : RVVConvBuiltin<"Uv", "Uvv", "xfd", overloaded_name>; 476 477class RVVConvToWidenSignedBuiltin<string overloaded_name> 478 : RVVConvBuiltin<"Iw", "Iwv", "xf", overloaded_name>; 479 480class RVVConvToWidenUnsignedBuiltin<string overloaded_name> 481 : RVVConvBuiltin<"Uw", "Uwv", "xf", overloaded_name>; 482 483class RVVConvToNarrowingSignedBuiltin<string overloaded_name> 484 : RVVConvBuiltin<"Iv", "IvFw", "csi", overloaded_name>; 485 486class RVVConvToNarrowingUnsignedBuiltin<string overloaded_name> 487 : RVVConvBuiltin<"Uv", "UvFw", "csi", overloaded_name>; 488 489let HasMaskedOffOperand = false in { 490 multiclass RVVSignedReductionBuiltin { 491 defm "" : RVVOutOp1BuiltinSet<NAME, "csil", 492 [["vs", "vSv", "SvSvvSv"]]>; 493 } 494 multiclass RVVUnsignedReductionBuiltin { 495 defm "" : RVVOutOp1BuiltinSet<NAME, "csil", 496 [["vs", "UvUSv", "USvUSvUvUSv"]]>; 497 } 498 multiclass RVVFloatingReductionBuiltin { 499 defm "" : RVVOutOp1BuiltinSet<NAME, "xfd", 500 [["vs", "vSv", "SvSvvSv"]]>; 501 } 502 multiclass RVVFloatingWidenReductionBuiltin { 503 defm "" : RVVOutOp1BuiltinSet<NAME, "xf", 504 [["vs", "vSw", "SwSwvSw"]]>; 505 } 506} 507 508multiclass RVVIntReductionBuiltinSet 509 : RVVSignedReductionBuiltin, 510 RVVUnsignedReductionBuiltin; 511 512// For widen operation which has different mangling name. 513multiclass RVVWidenBuiltinSet<string intrinsic_name, string type_range, 514 list<list<string>> suffixes_prototypes> { 515 let Log2LMUL = [-3, -2, -1, 0, 1, 2], 516 IRName = intrinsic_name, MaskedIRName = intrinsic_name # "_mask" in { 517 foreach s_p = suffixes_prototypes in { 518 let Name = NAME # "_" # s_p[0], 519 OverloadedName = NAME # "_" # s_p[0] in { 520 defvar suffix = s_p[1]; 521 defvar prototype = s_p[2]; 522 def : RVVOutOp0Op1Builtin<suffix, prototype, type_range>; 523 } 524 } 525 } 526} 527 528// For widen operation with widen operand which has different mangling name. 529multiclass RVVWidenWOp0BuiltinSet<string intrinsic_name, string type_range, 530 list<list<string>> suffixes_prototypes> { 531 let Log2LMUL = [-3, -2, -1, 0, 1, 2], 532 IRName = intrinsic_name, MaskedIRName = intrinsic_name # "_mask" in { 533 foreach s_p = suffixes_prototypes in { 534 let Name = NAME # "_" # s_p[0], 535 OverloadedName = NAME # "_" # s_p[0] in { 536 defvar suffix = s_p[1]; 537 defvar prototype = s_p[2]; 538 def : RVVOutOp1Builtin<suffix, prototype, type_range>; 539 } 540 } 541 } 542} 543 544multiclass RVVSignedWidenBinBuiltinSet 545 : RVVWidenBuiltinSet<NAME, "csi", 546 [["vv", "w", "wvv"], 547 ["vx", "w", "wve"]]>; 548 549multiclass RVVSignedWidenOp0BinBuiltinSet 550 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", 551 [["wv", "w", "wwv"], 552 ["wx", "w", "wwe"]]>; 553 554multiclass RVVUnsignedWidenBinBuiltinSet 555 : RVVWidenBuiltinSet<NAME, "csi", 556 [["vv", "Uw", "UwUvUv"], 557 ["vx", "Uw", "UwUvUe"]]>; 558 559multiclass RVVUnsignedWidenOp0BinBuiltinSet 560 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", 561 [["wv", "Uw", "UwUwUv"], 562 ["wx", "Uw", "UwUwUe"]]>; 563 564multiclass RVVFloatingWidenBinBuiltinSet 565 : RVVWidenBuiltinSet<NAME, "xf", 566 [["vv", "w", "wvv"], 567 ["vf", "w", "wve"]]>; 568 569multiclass RVVFloatingWidenOp0BinBuiltinSet 570 : RVVWidenWOp0BuiltinSet<NAME # "_w", "xf", 571 [["wv", "w", "wwv"], 572 ["wf", "w", "wwe"]]>; 573 574defvar TypeList = ["c","s","i","l","x","f","d"]; 575defvar EEWList = [["8", "(Log2EEW:3)"], 576 ["16", "(Log2EEW:4)"], 577 ["32", "(Log2EEW:5)"], 578 ["64", "(Log2EEW:6)"]]; 579 580class IsFloat<string type> { 581 bit val = !or(!eq(type, "x"), !eq(type, "f"), !eq(type, "d")); 582} 583 584let HasUnMaskedOverloaded = false, 585 MaskedPolicy = NonePolicy in { 586 class RVVVLEMaskBuiltin : RVVOutBuiltin<"m", "mPCUe", "c"> { 587 let Name = "vlm_v"; 588 let IRName = "vlm"; 589 let HasMasked = false; 590 } 591} 592 593let HasUnMaskedOverloaded = false, 594 UnMaskedPolicy = HasPassthruOperand in { 595 multiclass RVVVLEBuiltin<list<string> types> { 596 let Name = NAME # "_v", 597 IRName = "vle", 598 MaskedIRName ="vle_mask" in { 599 foreach type = types in { 600 def : RVVOutBuiltin<"v", "vPCe", type>; 601 if !not(IsFloat<type>.val) then { 602 def : RVVOutBuiltin<"Uv", "UvPCUe", type>; 603 } 604 } 605 } 606 } 607} 608 609multiclass RVVVLEFFBuiltin<list<string> types> { 610 let Name = NAME # "_v", 611 IRName = "vleff", 612 MaskedIRName = "vleff_mask", 613 HasUnMaskedOverloaded = false, 614 ManualCodegen = [{ 615 { 616 IntrinsicTypes = {ResultType, Ops[2]->getType()}; 617 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); 618 Value *NewVL = Ops[1]; 619 Ops.erase(Ops.begin() + 1); 620 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 621 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 622 llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); 623 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); 624 // Store new_vl. 625 clang::CharUnits Align = 626 CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); 627 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); 628 Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); 629 return V; 630 } 631 }], 632 MaskedManualCodegen = [{ 633 { 634 // Move mask to right before vl. 635 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 636 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 637 IntrinsicTypes = {ResultType, Ops[4]->getType()}; 638 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); 639 Value *NewVL = Ops[2]; 640 Ops.erase(Ops.begin() + 2); 641 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 642 llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); 643 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); 644 // Store new_vl. 645 clang::CharUnits Align = 646 CGM.getNaturalPointeeTypeAlignment(E->getArg(3)->getType()); 647 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); 648 Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); 649 return V; 650 } 651 }] in { 652 foreach type = types in { 653 def : RVVBuiltin<"v", "vPCePz", type>; 654 // Skip floating types for unsigned versions. 655 if !not(IsFloat<type>.val) then { 656 def : RVVBuiltin<"Uv", "UvPCUePz", type>; 657 } 658 } 659 } 660} 661 662multiclass RVVVLSEBuiltin<list<string> types> { 663 let Name = NAME # "_v", 664 IRName = "vlse", 665 MaskedIRName ="vlse_mask", 666 HasUnMaskedOverloaded = false, 667 UnMaskedPolicy = HasPassthruOperand in { 668 foreach type = types in { 669 def : RVVOutBuiltin<"v", "vPCet", type>; 670 if !not(IsFloat<type>.val) then { 671 def : RVVOutBuiltin<"Uv", "UvPCUet", type>; 672 } 673 } 674 } 675} 676 677multiclass RVVIndexedLoad<string op> { 678 let UnMaskedPolicy = HasPassthruOperand in { 679 foreach type = TypeList in { 680 foreach eew_list = EEWList[0-2] in { 681 defvar eew = eew_list[0]; 682 defvar eew_type = eew_list[1]; 683 let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask" in { 684 def: RVVOutOp1Builtin<"v", "vPCe" # eew_type # "Uv", type>; 685 if !not(IsFloat<type>.val) then { 686 def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew_type # "Uv", type>; 687 } 688 } 689 } 690 defvar eew64 = "64"; 691 defvar eew64_type = "(Log2EEW:6)"; 692 let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", 693 RequiredFeatures = ["RV64"] in { 694 def: RVVOutOp1Builtin<"v", "vPCe" # eew64_type # "Uv", type>; 695 if !not(IsFloat<type>.val) then { 696 def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew64_type # "Uv", type>; 697 } 698 } 699 } 700 } 701} 702 703let HasMaskedOffOperand = false, 704 MaskedPolicy = NonePolicy, 705 ManualCodegen = [{ 706 // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) 707 std::swap(Ops[0], Ops[1]); 708 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 709 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()}; 710 }], 711 MaskedManualCodegen= [{ 712 // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) 713 std::swap(Ops[0], Ops[2]); 714 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 715 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 716 }] in { 717 class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { 718 let Name = "vsm_v"; 719 let IRName = "vsm"; 720 let HasMasked = false; 721 } 722 multiclass RVVVSEBuiltin<list<string> types> { 723 let Name = NAME # "_v", 724 IRName = "vse", 725 MaskedIRName = "vse_mask" in { 726 foreach type = types in { 727 def : RVVBuiltin<"v", "0Pev", type>; 728 if !not(IsFloat<type>.val) then { 729 def : RVVBuiltin<"Uv", "0PUeUv", type>; 730 } 731 } 732 } 733 } 734} 735 736multiclass RVVVSSEBuiltin<list<string> types> { 737 let Name = NAME # "_v", 738 IRName = "vsse", 739 MaskedIRName = "vsse_mask", 740 HasMaskedOffOperand = false, 741 MaskedPolicy = NonePolicy, 742 ManualCodegen = [{ 743 // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) 744 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 745 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 746 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 747 }], 748 MaskedManualCodegen= [{ 749 // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) 750 std::swap(Ops[0], Ops[3]); 751 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 752 IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()}; 753 }] in { 754 foreach type = types in { 755 def : RVVBuiltin<"v", "0Petv", type>; 756 if !not(IsFloat<type>.val) then { 757 def : RVVBuiltin<"Uv", "0PUetUv", type>; 758 } 759 } 760 } 761} 762 763multiclass RVVIndexedStore<string op> { 764 let HasMaskedOffOperand = false, 765 MaskedPolicy = NonePolicy, 766 ManualCodegen = [{ 767 // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) 768 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 769 Ops[1] = Builder.CreateBitCast(Ops[1],Ops[0]->getType()->getPointerTo()); 770 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()}; 771 }], 772 MaskedManualCodegen= [{ 773 // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) 774 std::swap(Ops[0], Ops[3]); 775 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 776 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()}; 777 }] in { 778 foreach type = TypeList in { 779 foreach eew_list = EEWList[0-2] in { 780 defvar eew = eew_list[0]; 781 defvar eew_type = eew_list[1]; 782 let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask" in { 783 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>; 784 if !not(IsFloat<type>.val) then { 785 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "UvUv", type>; 786 } 787 } 788 } 789 defvar eew64 = "64"; 790 defvar eew64_type = "(Log2EEW:6)"; 791 let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", 792 RequiredFeatures = ["RV64"] in { 793 def : RVVBuiltin<"v", "0Pe" # eew64_type # "Uvv", type>; 794 if !not(IsFloat<type>.val) then { 795 def : RVVBuiltin<"Uv", "0PUe" # eew64_type # "UvUv", type>; 796 } 797 } 798 } 799 } 800} 801 802defvar NFList = [2, 3, 4, 5, 6, 7, 8]; 803 804class PVString<int nf, bit signed> { 805 string S = 806 !cond(!eq(nf, 2): !if(signed, "PvPv", "PUvPUv"), 807 !eq(nf, 3): !if(signed, "PvPvPv", "PUvPUvPUv"), 808 !eq(nf, 4): !if(signed, "PvPvPvPv", "PUvPUvPUvPUv"), 809 !eq(nf, 5): !if(signed, "PvPvPvPvPv", "PUvPUvPUvPUvPUv"), 810 !eq(nf, 6): !if(signed, "PvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUv"), 811 !eq(nf, 7): !if(signed, "PvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUv"), 812 !eq(nf, 8): !if(signed, "PvPvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUvPUv")); 813} 814 815multiclass RVVUnitStridedSegLoad<string op> { 816 foreach type = TypeList in { 817 defvar eew = !cond(!eq(type, "c") : "8", 818 !eq(type, "s") : "16", 819 !eq(type, "i") : "32", 820 !eq(type, "l") : "64", 821 !eq(type, "x") : "16", 822 !eq(type, "f") : "32", 823 !eq(type, "d") : "64"); 824 foreach nf = NFList in { 825 let Name = op # nf # "e" # eew # "_v", 826 IRName = op # nf, 827 MaskedIRName = op # nf # "_mask", 828 NF = nf, 829 HasUnMaskedOverloaded = false, 830 ManualCodegen = [{ 831 { 832 // builtin: (val0 address, val1 address, ..., ptr, vl) 833 ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); 834 IntrinsicTypes = {ResultType, Ops[NF + 1]->getType()}; 835 // intrinsic: (passthru0, passthru1, ..., ptr, vl) 836 SmallVector<llvm::Value*, 10> Operands; 837 for (unsigned I = 0; I < NF; ++I) 838 Operands.push_back(llvm::UndefValue::get(ResultType)); 839 Operands.push_back(Ops[NF]); 840 Operands.push_back(Ops[NF + 1]); 841 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 842 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 843 clang::CharUnits Align = 844 CGM.getNaturalPointeeTypeAlignment(E->getArg(NF)->getType()); 845 llvm::Value *V; 846 for (unsigned I = 0; I < NF; ++I) { 847 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 848 V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 849 } 850 return V; 851 } 852 }], 853 MaskedManualCodegen = [{ 854 { 855 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl) 856 // intrinsic: (maskedoff0, ..., ptr, mask, vl) 857 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 858 Ops[2 * NF + 2]->getType()}; 859 SmallVector<llvm::Value*, 12> Operands; 860 for (unsigned I = 0; I < NF; ++I) 861 Operands.push_back(Ops[NF + I + 1]); 862 Operands.push_back(Ops[2 * NF + 1]); 863 Operands.push_back(Ops[NF]); 864 Operands.push_back(Ops[2 * NF + 2]); 865 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 866 assert(Operands.size() == NF + 4); 867 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 868 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 869 clang::CharUnits Align = 870 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 871 llvm::Value *V; 872 for (unsigned I = 0; I < NF; ++I) { 873 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 874 V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 875 } 876 return V; 877 } 878 }] in { 879 defvar PV = PVString<nf, /*signed=*/true>.S; 880 defvar PUV = PVString<nf, /*signed=*/false>.S; 881 def : RVVBuiltin<"v", "0" # PV # "PCe", type>; 882 if !not(IsFloat<type>.val) then { 883 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe", type>; 884 } 885 } 886 } 887 } 888} 889 890multiclass RVVUnitStridedSegLoadFF<string op> { 891 foreach type = TypeList in { 892 defvar eew = !cond(!eq(type, "c") : "8", 893 !eq(type, "s") : "16", 894 !eq(type, "i") : "32", 895 !eq(type, "l") : "64", 896 !eq(type, "x") : "16", 897 !eq(type, "f") : "32", 898 !eq(type, "d") : "64"); 899 foreach nf = NFList in { 900 let Name = op # nf # "e" # eew # "ff_v", 901 IRName = op # nf # "ff", 902 MaskedIRName = op # nf # "ff_mask", 903 NF = nf, 904 HasUnMaskedOverloaded = false, 905 ManualCodegen = [{ 906 { 907 // builtin: (val0 address, val1 address, ..., ptr, new_vl, vl) 908 ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); 909 IntrinsicTypes = {ResultType, Ops[NF + 2]->getType()}; 910 // intrinsic: (passthru0, passthru1, ..., ptr, vl) 911 SmallVector<llvm::Value*, 12> Operands; 912 for (unsigned I = 0; I < NF; ++I) 913 Operands.push_back(llvm::UndefValue::get(ResultType)); 914 Operands.push_back(Ops[NF]); 915 Operands.push_back(Ops[NF + 2]); 916 Value *NewVL = Ops[NF + 1]; 917 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 918 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 919 clang::CharUnits Align = 920 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 921 for (unsigned I = 0; I < NF; ++I) { 922 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 923 Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 924 } 925 // Store new_vl. 926 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {NF}); 927 return Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); 928 } 929 }], 930 MaskedManualCodegen = [{ 931 { 932 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl) 933 // intrinsic: (maskedoff0, ..., ptr, mask, vl) 934 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 935 Ops[2 * NF + 3]->getType()}; 936 SmallVector<llvm::Value*, 12> Operands; 937 for (unsigned I = 0; I < NF; ++I) 938 Operands.push_back(Ops[NF + I + 1]); 939 Operands.push_back(Ops[2 * NF + 1]); 940 Operands.push_back(Ops[NF]); 941 Operands.push_back(Ops[2 * NF + 3]); 942 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 943 Value *NewVL = Ops[2 * NF + 2]; 944 assert(Operands.size() == NF + 4); 945 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 946 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 947 clang::CharUnits Align = 948 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 949 for (unsigned I = 0; I < NF; ++I) { 950 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 951 Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 952 } 953 // Store new_vl. 954 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {NF}); 955 return Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); 956 } 957 }] in { 958 defvar PV = PVString<nf, /*signed=*/true>.S; 959 defvar PUV = PVString<nf, /*signed=*/false>.S; 960 def : RVVBuiltin<"v", "0" # PV # "PCe" # "Pz", type>; 961 if !not(IsFloat<type>.val) then { 962 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "Pz", type>; 963 } 964 } 965 } 966 } 967} 968 969multiclass RVVStridedSegLoad<string op> { 970 foreach type = TypeList in { 971 defvar eew = !cond(!eq(type, "c") : "8", 972 !eq(type, "s") : "16", 973 !eq(type, "i") : "32", 974 !eq(type, "l") : "64", 975 !eq(type, "x") : "16", 976 !eq(type, "f") : "32", 977 !eq(type, "d") : "64"); 978 foreach nf = NFList in { 979 let Name = op # nf # "e" # eew # "_v", 980 IRName = op # nf, 981 MaskedIRName = op # nf # "_mask", 982 NF = nf, 983 HasUnMaskedOverloaded = false, 984 ManualCodegen = [{ 985 { 986 // builtin: (val0 address, val1 address, ..., ptr, stride, vl) 987 ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); 988 IntrinsicTypes = {ResultType, Ops[NF + 2]->getType()}; 989 // intrinsic: (passthru0, passthru1, ..., ptr, stride, vl) 990 SmallVector<llvm::Value*, 12> Operands; 991 for (unsigned I = 0; I < NF; ++I) 992 Operands.push_back(llvm::UndefValue::get(ResultType)); 993 Operands.push_back(Ops[NF]); 994 Operands.push_back(Ops[NF + 1]); 995 Operands.push_back(Ops[NF + 2]); 996 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 997 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 998 clang::CharUnits Align = 999 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1000 llvm::Value *V; 1001 for (unsigned I = 0; I < NF; ++I) { 1002 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 1003 V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 1004 } 1005 return V; 1006 } 1007 }], 1008 MaskedManualCodegen = [{ 1009 { 1010 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl) 1011 // intrinsic: (maskedoff0, ..., ptr, stride, mask, vl) 1012 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 1013 Ops[2 * NF + 3]->getType()}; 1014 SmallVector<llvm::Value*, 12> Operands; 1015 for (unsigned I = 0; I < NF; ++I) 1016 Operands.push_back(Ops[NF + I + 1]); 1017 Operands.push_back(Ops[2 * NF + 1]); 1018 Operands.push_back(Ops[2 * NF + 2]); 1019 Operands.push_back(Ops[NF]); 1020 Operands.push_back(Ops[2 * NF + 3]); 1021 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1022 assert(Operands.size() == NF + 5); 1023 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1024 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1025 clang::CharUnits Align = 1026 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1027 llvm::Value *V; 1028 for (unsigned I = 0; I < NF; ++I) { 1029 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 1030 V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 1031 } 1032 return V; 1033 } 1034 }] in { 1035 defvar PV = PVString<nf, /*signed=*/true>.S; 1036 defvar PUV = PVString<nf, /*signed=*/false>.S; 1037 def : RVVBuiltin<"v", "0" # PV # "PCe" # "t", type>; 1038 if !not(IsFloat<type>.val) then { 1039 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "t", type>; 1040 } 1041 } 1042 } 1043 } 1044} 1045 1046multiclass RVVIndexedSegLoad<string op> { 1047 foreach type = TypeList in { 1048 foreach eew_info = EEWList in { 1049 defvar eew = eew_info[0]; 1050 defvar eew_type = eew_info[1]; 1051 foreach nf = NFList in { 1052 let Name = op # nf # "ei" # eew # "_v", 1053 IRName = op # nf, 1054 MaskedIRName = op # nf # "_mask", 1055 NF = nf, 1056 ManualCodegen = [{ 1057 { 1058 // builtin: (val0 address, val1 address, ..., ptr, index, vl) 1059 ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); 1060 IntrinsicTypes = {ResultType, Ops[NF + 1]->getType(), Ops[NF + 2]->getType()}; 1061 // intrinsic: (passthru0, passthru1, ..., ptr, index, vl) 1062 SmallVector<llvm::Value*, 12> Operands; 1063 for (unsigned I = 0; I < NF; ++I) 1064 Operands.push_back(llvm::UndefValue::get(ResultType)); 1065 Operands.push_back(Ops[NF]); 1066 Operands.push_back(Ops[NF + 1]); 1067 Operands.push_back(Ops[NF + 2]); 1068 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1069 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1070 clang::CharUnits Align = 1071 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1072 llvm::Value *V; 1073 for (unsigned I = 0; I < NF; ++I) { 1074 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 1075 V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 1076 } 1077 return V; 1078 } 1079 }], 1080 MaskedManualCodegen = [{ 1081 { 1082 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl) 1083 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), 1084 Ops[2 * NF + 2]->getType(), Ops[2 * NF + 3]->getType()}; 1085 // intrinsic: (maskedoff0, ..., ptr, index, mask, vl) 1086 SmallVector<llvm::Value*, 12> Operands; 1087 for (unsigned I = 0; I < NF; ++I) 1088 Operands.push_back(Ops[NF + I + 1]); 1089 Operands.push_back(Ops[2 * NF + 1]); 1090 Operands.push_back(Ops[2 * NF + 2]); 1091 Operands.push_back(Ops[NF]); 1092 Operands.push_back(Ops[2 * NF + 3]); 1093 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1094 assert(Operands.size() == NF + 5); 1095 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1096 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1097 clang::CharUnits Align = 1098 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1099 llvm::Value *V; 1100 for (unsigned I = 0; I < NF; ++I) { 1101 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 1102 V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 1103 } 1104 return V; 1105 } 1106 }] in { 1107 defvar PV = PVString<nf, /*signed=*/true>.S; 1108 defvar PUV = PVString<nf, /*signed=*/false>.S; 1109 def : RVVBuiltin<"v", "0" # PV # "PCe" # eew_type # "Uv", type>; 1110 if !not(IsFloat<type>.val) then { 1111 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # eew_type # "Uv", type>; 1112 } 1113 } 1114 } 1115 } 1116 } 1117} 1118 1119class VString<int nf, bit signed> { 1120 string S = !cond(!eq(nf, 2): !if(signed, "vv", "UvUv"), 1121 !eq(nf, 3): !if(signed, "vvv", "UvUvUv"), 1122 !eq(nf, 4): !if(signed, "vvvv", "UvUvUvUv"), 1123 !eq(nf, 5): !if(signed, "vvvvv", "UvUvUvUvUv"), 1124 !eq(nf, 6): !if(signed, "vvvvvv", "UvUvUvUvUvUv"), 1125 !eq(nf, 7): !if(signed, "vvvvvvv", "UvUvUvUvUvUvUv"), 1126 !eq(nf, 8): !if(signed, "vvvvvvvv", "UvUvUvUvUvUvUvUv")); 1127} 1128 1129multiclass RVVUnitStridedSegStore<string op> { 1130 foreach type = TypeList in { 1131 defvar eew = !cond(!eq(type, "c") : "8", 1132 !eq(type, "s") : "16", 1133 !eq(type, "i") : "32", 1134 !eq(type, "l") : "64", 1135 !eq(type, "x") : "16", 1136 !eq(type, "f") : "32", 1137 !eq(type, "d") : "64"); 1138 foreach nf = NFList in { 1139 let Name = op # nf # "e" # eew # "_v", 1140 IRName = op # nf, 1141 MaskedIRName = op # nf # "_mask", 1142 NF = nf, 1143 HasMaskedOffOperand = false, 1144 MaskedPolicy = NonePolicy, 1145 ManualCodegen = [{ 1146 { 1147 // Builtin: (ptr, val0, val1, ..., vl) 1148 // Intrinsic: (val0, val1, ..., ptr, vl) 1149 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1150 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; 1151 assert(Ops.size() == NF + 2); 1152 } 1153 }], 1154 MaskedManualCodegen = [{ 1155 { 1156 // Builtin: (mask, ptr, val0, val1, ..., vl) 1157 // Intrinsic: (val0, val1, ..., ptr, mask, vl) 1158 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); 1159 std::swap(Ops[NF], Ops[NF + 1]); 1160 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 2]->getType()}; 1161 assert(Ops.size() == NF + 3); 1162 } 1163 }] in { 1164 defvar V = VString<nf, /*signed=*/true>.S; 1165 defvar UV = VString<nf, /*signed=*/false>.S; 1166 def : RVVBuiltin<"v", "0Pe" # V, type>; 1167 if !not(IsFloat<type>.val) then { 1168 def : RVVBuiltin<"Uv", "0PUe" # UV, type>; 1169 } 1170 } 1171 } 1172 } 1173} 1174 1175multiclass RVVStridedSegStore<string op> { 1176 foreach type = TypeList in { 1177 defvar eew = !cond(!eq(type, "c") : "8", 1178 !eq(type, "s") : "16", 1179 !eq(type, "i") : "32", 1180 !eq(type, "l") : "64", 1181 !eq(type, "x") : "16", 1182 !eq(type, "f") : "32", 1183 !eq(type, "d") : "64"); 1184 foreach nf = NFList in { 1185 let Name = op # nf # "e" # eew # "_v", 1186 IRName = op # nf, 1187 MaskedIRName = op # nf # "_mask", 1188 NF = nf, 1189 HasMaskedOffOperand = false, 1190 MaskedPolicy = NonePolicy, 1191 ManualCodegen = [{ 1192 { 1193 // Builtin: (ptr, stride, val0, val1, ..., vl). 1194 // Intrinsic: (val0, val1, ..., ptr, stride, vl) 1195 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); 1196 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; 1197 assert(Ops.size() == NF + 3); 1198 } 1199 }], 1200 MaskedManualCodegen = [{ 1201 { 1202 // Builtin: (mask, ptr, stride, val0, val1, ..., vl). 1203 // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl) 1204 std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); 1205 std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); 1206 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; 1207 assert(Ops.size() == NF + 4); 1208 } 1209 }] in { 1210 defvar V = VString<nf, /*signed=*/true>.S; 1211 defvar UV = VString<nf, /*signed=*/false>.S; 1212 def : RVVBuiltin<"v", "0Pet" # V, type>; 1213 if !not(IsFloat<type>.val) then { 1214 def : RVVBuiltin<"Uv", "0PUet" # UV, type>; 1215 } 1216 } 1217 } 1218 } 1219} 1220 1221multiclass RVVIndexedSegStore<string op> { 1222 foreach type = TypeList in { 1223 foreach eew_info = EEWList in { 1224 defvar eew = eew_info[0]; 1225 defvar eew_type = eew_info[1]; 1226 foreach nf = NFList in { 1227 let Name = op # nf # "ei" # eew # "_v", 1228 IRName = op # nf, 1229 MaskedIRName = op # nf # "_mask", 1230 NF = nf, 1231 HasMaskedOffOperand = false, 1232 MaskedPolicy = NonePolicy, 1233 ManualCodegen = [{ 1234 { 1235 // Builtin: (ptr, index, val0, val1, ..., vl) 1236 // Intrinsic: (val0, val1, ..., ptr, index, vl) 1237 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); 1238 IntrinsicTypes = {Ops[0]->getType(), 1239 Ops[NF + 1]->getType(), Ops[NF + 2]->getType()}; 1240 assert(Ops.size() == NF + 3); 1241 } 1242 }], 1243 MaskedManualCodegen = [{ 1244 { 1245 // Builtin: (mask, ptr, index, val0, val1, ..., vl) 1246 // Intrinsic: (val0, val1, ..., ptr, index, mask, vl) 1247 std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); 1248 std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); 1249 IntrinsicTypes = {Ops[0]->getType(), 1250 Ops[NF + 1]->getType(), Ops[NF + 3]->getType()}; 1251 assert(Ops.size() == NF + 4); 1252 } 1253 }] in { 1254 defvar V = VString<nf, /*signed=*/true>.S; 1255 defvar UV = VString<nf, /*signed=*/false>.S; 1256 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uv" # V, type>; 1257 if !not(IsFloat<type>.val) then { 1258 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "Uv" # UV, type>; 1259 } 1260 } 1261 } 1262 } 1263 } 1264} 1265 1266multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { 1267 let Name = NAME, 1268 IRName = IR, 1269 MaskedIRName = IR # "_mask", 1270 ManualCodegen = [{ 1271 { 1272 // op1, vl 1273 IntrinsicTypes = {ResultType, 1274 cast<llvm::VectorType>(ResultType)->getElementType(), 1275 Ops[1]->getType()}; 1276 Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[1])); 1277 // insert undef passthru 1278 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 1279 break; 1280 } 1281 }], 1282 MaskedManualCodegen = [{ 1283 { 1284 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1285 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1286 // maskedoff, op1, mask, vl 1287 IntrinsicTypes = {ResultType, 1288 cast<llvm::VectorType>(ResultType)->getElementType(), 1289 Ops[3]->getType()}; 1290 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[1])); 1291 break; 1292 } 1293 }] in { 1294 def : RVVBuiltin<"v", "vv", type_range>; 1295 } 1296} 1297 1298multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { 1299 let Name = NAME, 1300 IRName = IR, 1301 MaskedIRName = IR # "_mask", 1302 ManualCodegen = [{ 1303 { 1304 // op1, vl 1305 IntrinsicTypes = {ResultType, 1306 cast<llvm::VectorType>(ResultType)->getElementType(), 1307 Ops[1]->getType()}; 1308 Ops.insert(Ops.begin() + 1, 1309 llvm::Constant::getAllOnesValue(IntrinsicTypes[1])); 1310 // insert undef passthru 1311 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 1312 break; 1313 } 1314 }], 1315 MaskedManualCodegen = [{ 1316 { 1317 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1318 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1319 // maskedoff, op1, mask, vl 1320 IntrinsicTypes = {ResultType, 1321 cast<llvm::VectorType>(ResultType)->getElementType(), 1322 Ops[3]->getType()}; 1323 Ops.insert(Ops.begin() + 2, 1324 llvm::Constant::getAllOnesValue(IntrinsicTypes[1])); 1325 break; 1326 } 1327 }] in { 1328 def : RVVBuiltin<"v", "vv", type_range>; 1329 def : RVVBuiltin<"Uv", "UvUv", type_range>; 1330 } 1331} 1332 1333multiclass RVVPseudoMaskBuiltin<string IR, string type_range> { 1334 let Name = NAME, 1335 IRName = IR, 1336 HasMasked = false, 1337 ManualCodegen = [{ 1338 { 1339 // op1, vl 1340 IntrinsicTypes = {ResultType, 1341 Ops[1]->getType()}; 1342 Ops.insert(Ops.begin() + 1, Ops[0]); 1343 break; 1344 } 1345 }] in { 1346 def : RVVBuiltin<"m", "mm", type_range>; 1347 } 1348} 1349 1350multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { 1351 let Name = NAME, 1352 IRName = IR, 1353 MaskedIRName = IR # "_mask", 1354 ManualCodegen = [{ 1355 { 1356 // op1, vl 1357 IntrinsicTypes = {ResultType, 1358 Ops[0]->getType(), Ops[1]->getType()}; 1359 Ops.insert(Ops.begin() + 1, Ops[0]); 1360 // insert undef passthru 1361 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 1362 break; 1363 } 1364 }], 1365 MaskedManualCodegen = [{ 1366 { 1367 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1368 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1369 // maskedoff, op1, mask, vl 1370 IntrinsicTypes = {ResultType, 1371 Ops[1]->getType(), 1372 Ops[3]->getType()}; 1373 Ops.insert(Ops.begin() + 2, Ops[1]); 1374 break; 1375 } 1376 }] in { 1377 def : RVVBuiltin<"v", "vv", type_range>; 1378 } 1379} 1380 1381multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, 1382 list<list<string>> suffixes_prototypes> { 1383 let Name = NAME, 1384 OverloadedName = MName, 1385 IRName = IR, 1386 MaskedIRName = IR # "_mask", 1387 ManualCodegen = [{ 1388 { 1389 // op1, vl 1390 IntrinsicTypes = {ResultType, 1391 Ops[0]->getType(), 1392 cast<llvm::VectorType>(Ops[0]->getType())->getElementType(), 1393 Ops[1]->getType()}; 1394 Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[2])); 1395 // insert undef passthru 1396 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 1397 break; 1398 } 1399 }], 1400 MaskedManualCodegen = [{ 1401 { 1402 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1403 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1404 // maskedoff, op1, mask, vl 1405 IntrinsicTypes = {ResultType, 1406 Ops[1]->getType(), 1407 cast<llvm::VectorType>(Ops[1]->getType())->getElementType(), 1408 Ops[3]->getType()}; 1409 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[2])); 1410 break; 1411 } 1412 }] in { 1413 foreach s_p = suffixes_prototypes in { 1414 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 1415 } 1416 } 1417} 1418 1419multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, 1420 list<list<string>> suffixes_prototypes> { 1421 let Name = NAME, 1422 OverloadedName = MName, 1423 IRName = IR, 1424 MaskedIRName = IR # "_mask", 1425 ManualCodegen = [{ 1426 { 1427 // op1, vl 1428 IntrinsicTypes = {ResultType, 1429 Ops[0]->getType(), 1430 Ops[1]->getType(), 1431 Ops[1]->getType()}; 1432 Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[2])); 1433 // insert undef passthru 1434 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 1435 break; 1436 } 1437 }], 1438 MaskedManualCodegen = [{ 1439 { 1440 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1441 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); 1442 // maskedoff, op1, mask, vl 1443 IntrinsicTypes = {ResultType, 1444 Ops[1]->getType(), 1445 Ops[3]->getType(), 1446 Ops[3]->getType()}; 1447 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[2])); 1448 break; 1449 } 1450 }] in { 1451 foreach s_p = suffixes_prototypes in { 1452 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 1453 } 1454 } 1455} 1456 1457// Define vread_csr&vwrite_csr described in RVV intrinsics doc. 1458let HeaderCode = 1459[{ 1460enum RVV_CSR { 1461 RVV_VSTART = 0, 1462 RVV_VXSAT, 1463 RVV_VXRM, 1464 RVV_VCSR, 1465}; 1466 1467static __inline__ __attribute__((__always_inline__, __nodebug__)) 1468unsigned long vread_csr(enum RVV_CSR __csr) { 1469 unsigned long __rv = 0; 1470 switch (__csr) { 1471 case RVV_VSTART: 1472 __asm__ __volatile__ ("csrr\t%0, vstart" : "=r"(__rv) : : "memory"); 1473 break; 1474 case RVV_VXSAT: 1475 __asm__ __volatile__ ("csrr\t%0, vxsat" : "=r"(__rv) : : "memory"); 1476 break; 1477 case RVV_VXRM: 1478 __asm__ __volatile__ ("csrr\t%0, vxrm" : "=r"(__rv) : : "memory"); 1479 break; 1480 case RVV_VCSR: 1481 __asm__ __volatile__ ("csrr\t%0, vcsr" : "=r"(__rv) : : "memory"); 1482 break; 1483 } 1484 return __rv; 1485} 1486 1487static __inline__ __attribute__((__always_inline__, __nodebug__)) 1488void vwrite_csr(enum RVV_CSR __csr, unsigned long __value) { 1489 switch (__csr) { 1490 case RVV_VSTART: 1491 __asm__ __volatile__ ("csrw\tvstart, %z0" : : "rJ"(__value) : "memory"); 1492 break; 1493 case RVV_VXSAT: 1494 __asm__ __volatile__ ("csrw\tvxsat, %z0" : : "rJ"(__value) : "memory"); 1495 break; 1496 case RVV_VXRM: 1497 __asm__ __volatile__ ("csrw\tvxrm, %z0" : : "rJ"(__value) : "memory"); 1498 break; 1499 case RVV_VCSR: 1500 __asm__ __volatile__ ("csrw\tvcsr, %z0" : : "rJ"(__value) : "memory"); 1501 break; 1502 } 1503} 1504}] in 1505def vread_vwrite_csr: RVVHeader; 1506 1507// 6. Configuration-Setting Instructions 1508// 6.1. vsetvli/vsetvl instructions 1509 1510// vsetvl/vsetvlmax are a macro because they require constant integers in SEW 1511// and LMUL. 1512let HeaderCode = 1513[{ 1514#define vsetvl_e8mf8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 5) 1515#define vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6) 1516#define vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7) 1517#define vsetvl_e8m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 0) 1518#define vsetvl_e8m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 1) 1519#define vsetvl_e8m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 2) 1520#define vsetvl_e8m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 3) 1521 1522#define vsetvl_e16mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 6) 1523#define vsetvl_e16mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 7) 1524#define vsetvl_e16m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 0) 1525#define vsetvl_e16m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 1) 1526#define vsetvl_e16m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 2) 1527#define vsetvl_e16m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 3) 1528 1529#define vsetvl_e32mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 7) 1530#define vsetvl_e32m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 0) 1531#define vsetvl_e32m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 1) 1532#define vsetvl_e32m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 2) 1533#define vsetvl_e32m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 3) 1534 1535#define vsetvl_e64m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 0) 1536#define vsetvl_e64m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 1) 1537#define vsetvl_e64m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 2) 1538#define vsetvl_e64m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 3) 1539 1540#define vsetvlmax_e8mf8() __builtin_rvv_vsetvlimax(0, 5) 1541#define vsetvlmax_e8mf4() __builtin_rvv_vsetvlimax(0, 6) 1542#define vsetvlmax_e8mf2() __builtin_rvv_vsetvlimax(0, 7) 1543#define vsetvlmax_e8m1() __builtin_rvv_vsetvlimax(0, 0) 1544#define vsetvlmax_e8m2() __builtin_rvv_vsetvlimax(0, 1) 1545#define vsetvlmax_e8m4() __builtin_rvv_vsetvlimax(0, 2) 1546#define vsetvlmax_e8m8() __builtin_rvv_vsetvlimax(0, 3) 1547 1548#define vsetvlmax_e16mf4() __builtin_rvv_vsetvlimax(1, 6) 1549#define vsetvlmax_e16mf2() __builtin_rvv_vsetvlimax(1, 7) 1550#define vsetvlmax_e16m1() __builtin_rvv_vsetvlimax(1, 0) 1551#define vsetvlmax_e16m2() __builtin_rvv_vsetvlimax(1, 1) 1552#define vsetvlmax_e16m4() __builtin_rvv_vsetvlimax(1, 2) 1553#define vsetvlmax_e16m8() __builtin_rvv_vsetvlimax(1, 3) 1554 1555#define vsetvlmax_e32mf2() __builtin_rvv_vsetvlimax(2, 7) 1556#define vsetvlmax_e32m1() __builtin_rvv_vsetvlimax(2, 0) 1557#define vsetvlmax_e32m2() __builtin_rvv_vsetvlimax(2, 1) 1558#define vsetvlmax_e32m4() __builtin_rvv_vsetvlimax(2, 2) 1559#define vsetvlmax_e32m8() __builtin_rvv_vsetvlimax(2, 3) 1560 1561#define vsetvlmax_e64m1() __builtin_rvv_vsetvlimax(3, 0) 1562#define vsetvlmax_e64m2() __builtin_rvv_vsetvlimax(3, 1) 1563#define vsetvlmax_e64m4() __builtin_rvv_vsetvlimax(3, 2) 1564#define vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3) 1565}] in 1566def vsetvl_macro: RVVHeader; 1567 1568let HasBuiltinAlias = false, 1569 HasVL = false, 1570 HasMasked = false, 1571 MaskedPolicy = NonePolicy, 1572 Log2LMUL = [0], 1573 ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type 1574{ 1575 def vsetvli : RVVBuiltin<"", "zzKzKz", "i">; 1576 def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">; 1577} 1578 1579// 7. Vector Loads and Stores 1580// 7.4. Vector Unit-Stride Instructions 1581def vlm: RVVVLEMaskBuiltin; 1582defm vle8: RVVVLEBuiltin<["c"]>; 1583defm vle16: RVVVLEBuiltin<["s","x"]>; 1584defm vle32: RVVVLEBuiltin<["i","f"]>; 1585defm vle64: RVVVLEBuiltin<["l","d"]>; 1586 1587def vsm : RVVVSEMaskBuiltin; 1588defm vse8 : RVVVSEBuiltin<["c"]>; 1589defm vse16: RVVVSEBuiltin<["s","x"]>; 1590defm vse32: RVVVSEBuiltin<["i","f"]>; 1591defm vse64: RVVVSEBuiltin<["l","d"]>; 1592 1593// 7.5. Vector Strided Instructions 1594defm vlse8: RVVVLSEBuiltin<["c"]>; 1595defm vlse16: RVVVLSEBuiltin<["s","x"]>; 1596defm vlse32: RVVVLSEBuiltin<["i","f"]>; 1597defm vlse64: RVVVLSEBuiltin<["l","d"]>; 1598 1599defm vsse8 : RVVVSSEBuiltin<["c"]>; 1600defm vsse16: RVVVSSEBuiltin<["s","x"]>; 1601defm vsse32: RVVVSSEBuiltin<["i","f"]>; 1602defm vsse64: RVVVSSEBuiltin<["l","d"]>; 1603 1604// 7.6. Vector Indexed Instructions 1605defm : RVVIndexedLoad<"vluxei">; 1606defm : RVVIndexedLoad<"vloxei">; 1607 1608defm : RVVIndexedStore<"vsuxei">; 1609defm : RVVIndexedStore<"vsoxei">; 1610 1611// 7.7. Unit-stride Fault-Only-First Loads 1612defm vle8ff: RVVVLEFFBuiltin<["c"]>; 1613defm vle16ff: RVVVLEFFBuiltin<["s","x"]>; 1614defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>; 1615defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>; 1616 1617// 7.8 Vector Load/Store Segment Instructions 1618defm : RVVUnitStridedSegLoad<"vlseg">; 1619defm : RVVUnitStridedSegLoadFF<"vlseg">; 1620defm : RVVStridedSegLoad<"vlsseg">; 1621defm : RVVIndexedSegLoad<"vluxseg">; 1622defm : RVVIndexedSegLoad<"vloxseg">; 1623defm : RVVUnitStridedSegStore<"vsseg">; 1624defm : RVVStridedSegStore<"vssseg">; 1625defm : RVVIndexedSegStore<"vsuxseg">; 1626defm : RVVIndexedSegStore<"vsoxseg">; 1627 1628// 12. Vector Integer Arithmetic Instructions 1629// 12.1. Vector Single-Width Integer Add and Subtract 1630let UnMaskedPolicy = HasPassthruOperand in { 1631defm vadd : RVVIntBinBuiltinSet; 1632defm vsub : RVVIntBinBuiltinSet; 1633defm vrsub : RVVOutOp1BuiltinSet<"vrsub", "csil", 1634 [["vx", "v", "vve"], 1635 ["vx", "Uv", "UvUvUe"]]>; 1636} 1637defm vneg_v : RVVPseudoUnaryBuiltin<"vrsub", "csil">; 1638 1639// 12.2. Vector Widening Integer Add/Subtract 1640// Widening unsigned integer add/subtract, 2*SEW = SEW +/- SEW 1641let UnMaskedPolicy = HasPassthruOperand in { 1642defm vwaddu : RVVUnsignedWidenBinBuiltinSet; 1643defm vwsubu : RVVUnsignedWidenBinBuiltinSet; 1644// Widening signed integer add/subtract, 2*SEW = SEW +/- SEW 1645defm vwadd : RVVSignedWidenBinBuiltinSet; 1646defm vwsub : RVVSignedWidenBinBuiltinSet; 1647// Widening unsigned integer add/subtract, 2*SEW = 2*SEW +/- SEW 1648defm vwaddu : RVVUnsignedWidenOp0BinBuiltinSet; 1649defm vwsubu : RVVUnsignedWidenOp0BinBuiltinSet; 1650// Widening signed integer add/subtract, 2*SEW = 2*SEW +/- SEW 1651defm vwadd : RVVSignedWidenOp0BinBuiltinSet; 1652defm vwsub : RVVSignedWidenOp0BinBuiltinSet; 1653} 1654defm vwcvtu_x_x_v : RVVPseudoVWCVTBuiltin<"vwaddu", "vwcvtu_x", "csi", 1655 [["Uw", "UwUv"]]>; 1656defm vwcvt_x_x_v : RVVPseudoVWCVTBuiltin<"vwadd", "vwcvt_x", "csi", 1657 [["w", "wv"]]>; 1658 1659// 12.3. Vector Integer Extension 1660let UnMaskedPolicy = HasPassthruOperand in { 1661let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1662 def vsext_vf2 : RVVIntExt<"vsext", "w", "wv", "csi">; 1663 def vzext_vf2 : RVVIntExt<"vzext", "Uw", "UwUv", "csi">; 1664} 1665let Log2LMUL = [-3, -2, -1, 0, 1] in { 1666 def vsext_vf4 : RVVIntExt<"vsext", "q", "qv", "cs">; 1667 def vzext_vf4 : RVVIntExt<"vzext", "Uq", "UqUv", "cs">; 1668} 1669let Log2LMUL = [-3, -2, -1, 0] in { 1670 def vsext_vf8 : RVVIntExt<"vsext", "o", "ov", "c">; 1671 def vzext_vf8 : RVVIntExt<"vzext", "Uo", "UoUv", "c">; 1672} 1673} 1674 1675// 12.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions 1676let HasMasked = false, MaskedPolicy = NonePolicy in { 1677 let UnMaskedPolicy = HasPassthruOperand in { 1678 defm vadc : RVVCarryinBuiltinSet; 1679 defm vsbc : RVVCarryinBuiltinSet; 1680 } 1681 defm vmadc : RVVCarryOutInBuiltinSet<"vmadc_carry_in">; 1682 defm vmadc : RVVIntMaskOutBuiltinSet; 1683 defm vmsbc : RVVCarryOutInBuiltinSet<"vmsbc_borrow_in">; 1684 defm vmsbc : RVVIntMaskOutBuiltinSet; 1685} 1686 1687// 12.5. Vector Bitwise Logical Instructions 1688let UnMaskedPolicy = HasPassthruOperand in { 1689defm vand : RVVIntBinBuiltinSet; 1690defm vxor : RVVIntBinBuiltinSet; 1691defm vor : RVVIntBinBuiltinSet; 1692} 1693defm vnot_v : RVVPseudoVNotBuiltin<"vxor", "csil">; 1694 1695// 12.6. Vector Single-Width Bit Shift Instructions 1696let UnMaskedPolicy = HasPassthruOperand in { 1697defm vsll : RVVShiftBuiltinSet; 1698defm vsrl : RVVUnsignedShiftBuiltinSet; 1699defm vsra : RVVSignedShiftBuiltinSet; 1700 1701// 12.7. Vector Narrowing Integer Right Shift Instructions 1702defm vnsrl : RVVUnsignedNShiftBuiltinSet; 1703defm vnsra : RVVSignedNShiftBuiltinSet; 1704} 1705defm vncvt_x_x_w : RVVPseudoVNCVTBuiltin<"vnsrl", "vncvt_x", "csi", 1706 [["v", "vw"], 1707 ["Uv", "UvUw"]]>; 1708 1709// 12.8. Vector Integer Comparison Instructions 1710let MaskedPolicy = NonePolicy in { 1711defm vmseq : RVVIntMaskOutBuiltinSet; 1712defm vmsne : RVVIntMaskOutBuiltinSet; 1713defm vmsltu : RVVUnsignedMaskOutBuiltinSet; 1714defm vmslt : RVVSignedMaskOutBuiltinSet; 1715defm vmsleu : RVVUnsignedMaskOutBuiltinSet; 1716defm vmsle : RVVSignedMaskOutBuiltinSet; 1717defm vmsgtu : RVVUnsignedMaskOutBuiltinSet; 1718defm vmsgt : RVVSignedMaskOutBuiltinSet; 1719defm vmsgeu : RVVUnsignedMaskOutBuiltinSet; 1720defm vmsge : RVVSignedMaskOutBuiltinSet; 1721} 1722 1723// 12.9. Vector Integer Min/Max Instructions 1724let UnMaskedPolicy = HasPassthruOperand in { 1725defm vminu : RVVUnsignedBinBuiltinSet; 1726defm vmin : RVVSignedBinBuiltinSet; 1727defm vmaxu : RVVUnsignedBinBuiltinSet; 1728defm vmax : RVVSignedBinBuiltinSet; 1729 1730// 12.10. Vector Single-Width Integer Multiply Instructions 1731defm vmul : RVVIntBinBuiltinSet; 1732let RequiredFeatures = ["FullMultiply"] in { 1733defm vmulh : RVVSignedBinBuiltinSet; 1734defm vmulhu : RVVUnsignedBinBuiltinSet; 1735defm vmulhsu : RVVOutOp1BuiltinSet<"vmulhsu", "csil", 1736 [["vv", "v", "vvUv"], 1737 ["vx", "v", "vvUe"]]>; 1738} 1739 1740// 12.11. Vector Integer Divide Instructions 1741defm vdivu : RVVUnsignedBinBuiltinSet; 1742defm vdiv : RVVSignedBinBuiltinSet; 1743defm vremu : RVVUnsignedBinBuiltinSet; 1744defm vrem : RVVSignedBinBuiltinSet; 1745} 1746 1747// 12.12. Vector Widening Integer Multiply Instructions 1748let Log2LMUL = [-3, -2, -1, 0, 1, 2], UnMaskedPolicy = HasPassthruOperand in { 1749defm vwmul : RVVOutOp0Op1BuiltinSet<"vwmul", "csi", 1750 [["vv", "w", "wvv"], 1751 ["vx", "w", "wve"]]>; 1752defm vwmulu : RVVOutOp0Op1BuiltinSet<"vwmulu", "csi", 1753 [["vv", "Uw", "UwUvUv"], 1754 ["vx", "Uw", "UwUvUe"]]>; 1755defm vwmulsu : RVVOutOp0Op1BuiltinSet<"vwmulsu", "csi", 1756 [["vv", "w", "wvUv"], 1757 ["vx", "w", "wvUe"]]>; 1758} 1759 1760// 12.13. Vector Single-Width Integer Multiply-Add Instructions 1761let UnMaskedPolicy = HasPolicyOperand in { 1762defm vmacc : RVVIntTerBuiltinSet; 1763defm vnmsac : RVVIntTerBuiltinSet; 1764defm vmadd : RVVIntTerBuiltinSet; 1765defm vnmsub : RVVIntTerBuiltinSet; 1766 1767// 12.14. Vector Widening Integer Multiply-Add Instructions 1768let HasMaskedOffOperand = false, 1769 Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1770defm vwmaccu : RVVOutOp1Op2BuiltinSet<"vwmaccu", "csi", 1771 [["vv", "Uw", "UwUwUvUv"], 1772 ["vx", "Uw", "UwUwUeUv"]]>; 1773defm vwmacc : RVVOutOp1Op2BuiltinSet<"vwmacc", "csi", 1774 [["vv", "w", "wwvv"], 1775 ["vx", "w", "wwev"]]>; 1776defm vwmaccsu : RVVOutOp1Op2BuiltinSet<"vwmaccsu", "csi", 1777 [["vv", "w", "wwvUv"], 1778 ["vx", "w", "wweUv"]]>; 1779defm vwmaccus : RVVOutOp1Op2BuiltinSet<"vwmaccus", "csi", 1780 [["vx", "w", "wwUev"]]>; 1781} 1782} 1783 1784// 12.15. Vector Integer Merge Instructions 1785// C/C++ Operand: (mask, op1, op2, vl), Intrinsic: (op1, op2, mask, vl) 1786let HasMasked = false, MaskedPolicy = NonePolicy, 1787 ManualCodegen = [{ 1788 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3); 1789 IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType()}; 1790 // insert undef passthru 1791 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 1792 }] in { 1793 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "csil", 1794 [["vvm", "v", "vmvv"], 1795 ["vxm", "v", "vmve"], 1796 ["vvm", "Uv", "UvmUvUv"], 1797 ["vxm", "Uv", "UvmUvUe"]]>; 1798} 1799 1800// 12.16. Vector Integer Move Instructions 1801let HasMasked = false, UnMaskedPolicy = HasPassthruOperand, MaskedPolicy = NonePolicy in { 1802 let OverloadedName = "vmv_v" in { 1803 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csil", 1804 [["v", "Uv", "UvUv"]]>; 1805 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilxfd", 1806 [["v", "v", "vv"]]>; 1807 } 1808 let HasUnMaskedOverloaded = false in 1809 defm vmv_v : RVVOutBuiltinSet<"vmv_v_x", "csil", 1810 [["x", "v", "ve"], 1811 ["x", "Uv", "UvUe"]]>; 1812} 1813 1814// 13. Vector Fixed-Point Arithmetic Instructions 1815// 13.1. Vector Single-Width Saturating Add and Subtract 1816let UnMaskedPolicy = HasPassthruOperand in { 1817defm vsaddu : RVVUnsignedBinBuiltinSet; 1818defm vsadd : RVVSignedBinBuiltinSet; 1819defm vssubu : RVVUnsignedBinBuiltinSet; 1820defm vssub : RVVSignedBinBuiltinSet; 1821 1822// 13.2. Vector Single-Width Averaging Add and Subtract 1823defm vaaddu : RVVUnsignedBinBuiltinSet; 1824defm vaadd : RVVSignedBinBuiltinSet; 1825defm vasubu : RVVUnsignedBinBuiltinSet; 1826defm vasub : RVVSignedBinBuiltinSet; 1827 1828// 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation 1829let RequiredFeatures = ["FullMultiply"] in { 1830defm vsmul : RVVSignedBinBuiltinSet; 1831} 1832 1833// 13.4. Vector Single-Width Scaling Shift Instructions 1834defm vssrl : RVVUnsignedShiftBuiltinSet; 1835defm vssra : RVVSignedShiftBuiltinSet; 1836 1837// 13.5. Vector Narrowing Fixed-Point Clip Instructions 1838defm vnclipu : RVVUnsignedNShiftBuiltinSet; 1839defm vnclip : RVVSignedNShiftBuiltinSet; 1840 1841// 14. Vector Floating-Point Instructions 1842// 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions 1843defm vfadd : RVVFloatingBinBuiltinSet; 1844defm vfsub : RVVFloatingBinBuiltinSet; 1845defm vfrsub : RVVFloatingBinVFBuiltinSet; 1846 1847// 14.3. Vector Widening Floating-Point Add/Subtract Instructions 1848// Widening FP add/subtract, 2*SEW = SEW +/- SEW 1849defm vfwadd : RVVFloatingWidenBinBuiltinSet; 1850defm vfwsub : RVVFloatingWidenBinBuiltinSet; 1851// Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW 1852defm vfwadd : RVVFloatingWidenOp0BinBuiltinSet; 1853defm vfwsub : RVVFloatingWidenOp0BinBuiltinSet; 1854 1855// 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions 1856defm vfmul : RVVFloatingBinBuiltinSet; 1857defm vfdiv : RVVFloatingBinBuiltinSet; 1858defm vfrdiv : RVVFloatingBinVFBuiltinSet; 1859 1860// 14.5. Vector Widening Floating-Point Multiply 1861let Log2LMUL = [-2, -1, 0, 1, 2] in { 1862 defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", 1863 [["vv", "w", "wvv"], 1864 ["vf", "w", "wve"]]>; 1865} 1866} 1867 1868// 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions 1869let UnMaskedPolicy = HasPolicyOperand in { 1870defm vfmacc : RVVFloatingTerBuiltinSet; 1871defm vfnmacc : RVVFloatingTerBuiltinSet; 1872defm vfmsac : RVVFloatingTerBuiltinSet; 1873defm vfnmsac : RVVFloatingTerBuiltinSet; 1874defm vfmadd : RVVFloatingTerBuiltinSet; 1875defm vfnmadd : RVVFloatingTerBuiltinSet; 1876defm vfmsub : RVVFloatingTerBuiltinSet; 1877defm vfnmsub : RVVFloatingTerBuiltinSet; 1878 1879// 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions 1880defm vfwmacc : RVVFloatingWidenTerBuiltinSet; 1881defm vfwnmacc : RVVFloatingWidenTerBuiltinSet; 1882defm vfwmsac : RVVFloatingWidenTerBuiltinSet; 1883defm vfwnmsac : RVVFloatingWidenTerBuiltinSet; 1884} 1885 1886// 14.8. Vector Floating-Point Square-Root Instruction 1887let UnMaskedPolicy = HasPassthruOperand in { 1888def vfsqrt : RVVFloatingUnaryVVBuiltin; 1889 1890// 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction 1891def vfrsqrt7 : RVVFloatingUnaryVVBuiltin; 1892 1893// 14.10. Vector Floating-Point Reciprocal Estimate Instruction 1894def vfrec7 : RVVFloatingUnaryVVBuiltin; 1895 1896// 14.11. Vector Floating-Point MIN/MAX Instructions 1897defm vfmin : RVVFloatingBinBuiltinSet; 1898defm vfmax : RVVFloatingBinBuiltinSet; 1899 1900// 14.12. Vector Floating-Point Sign-Injection Instructions 1901defm vfsgnj : RVVFloatingBinBuiltinSet; 1902defm vfsgnjn : RVVFloatingBinBuiltinSet; 1903defm vfsgnjx : RVVFloatingBinBuiltinSet; 1904} 1905defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "xfd">; 1906defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "xfd">; 1907 1908// 14.13. Vector Floating-Point Compare Instructions 1909let MaskedPolicy = NonePolicy in { 1910defm vmfeq : RVVFloatingMaskOutBuiltinSet; 1911defm vmfne : RVVFloatingMaskOutBuiltinSet; 1912defm vmflt : RVVFloatingMaskOutBuiltinSet; 1913defm vmfle : RVVFloatingMaskOutBuiltinSet; 1914defm vmfgt : RVVFloatingMaskOutBuiltinSet; 1915defm vmfge : RVVFloatingMaskOutBuiltinSet; 1916} 1917 1918// 14.14. Vector Floating-Point Classify Instruction 1919let Name = "vfclass_v", UnMaskedPolicy = HasPassthruOperand in 1920 def vfclass : RVVOp0Builtin<"Uv", "Uvv", "xfd">; 1921 1922// 14.15. Vector Floating-Point Merge Instructio 1923// C/C++ Operand: (mask, op1, op2, vl), Builtin: (op1, op2, mask, vl) 1924let HasMasked = false, MaskedPolicy = NonePolicy, 1925 ManualCodegen = [{ 1926 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3); 1927 IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType()}; 1928 // insert undef passthru 1929 Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType)); 1930 }] in { 1931 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "xfd", 1932 [["vvm", "v", "vmvv"]]>; 1933 defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "xfd", 1934 [["vfm", "v", "vmve"]]>; 1935} 1936 1937// 14.16. Vector Floating-Point Move Instruction 1938let HasMasked = false, UnMaskedPolicy = HasPassthruOperand, 1939 HasUnMaskedOverloaded = false, MaskedPolicy = NonePolicy in 1940 defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "xfd", 1941 [["f", "v", "ve"]]>; 1942 1943// 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions 1944let UnMaskedPolicy = HasPassthruOperand in { 1945def vfcvt_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_xu">; 1946def vfcvt_x_f_v : RVVConvToSignedBuiltin<"vfcvt_x">; 1947def vfcvt_rtz_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_rtz_xu">; 1948def vfcvt_rtz_x_f_v : RVVConvToSignedBuiltin<"vfcvt_rtz_x">; 1949def vfcvt_f_xu_v : RVVConvBuiltin<"Fv", "FvUv", "sil", "vfcvt_f">; 1950def vfcvt_f_x_v : RVVConvBuiltin<"Fv", "Fvv", "sil", "vfcvt_f">; 1951 1952// 14.18. Widening Floating-Point/Integer Type-Convert Instructions 1953let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1954 def vfwcvt_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_xu">; 1955 def vfwcvt_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_x">; 1956 def vfwcvt_rtz_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_rtz_xu">; 1957 def vfwcvt_rtz_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_rtz_x">; 1958 def vfwcvt_f_xu_v : RVVConvBuiltin<"Fw", "FwUv", "csi", "vfwcvt_f">; 1959 def vfwcvt_f_x_v : RVVConvBuiltin<"Fw", "Fwv", "csi", "vfwcvt_f">; 1960 def vfwcvt_f_f_v : RVVConvBuiltin<"w", "wv", "xf", "vfwcvt_f">; 1961} 1962 1963// 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions 1964let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1965 def vfncvt_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_xu">; 1966 def vfncvt_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_x">; 1967 def vfncvt_rtz_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_rtz_xu">; 1968 def vfncvt_rtz_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_rtz_x">; 1969 def vfncvt_f_xu_w : RVVConvBuiltin<"Fv", "FvUw", "csi", "vfncvt_f">; 1970 def vfncvt_f_x_w : RVVConvBuiltin<"Fv", "Fvw", "csi", "vfncvt_f">; 1971 def vfncvt_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_f">; 1972 def vfncvt_rod_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_rod_f">; 1973} 1974} 1975 1976// 15. Vector Reduction Operations 1977// 15.1. Vector Single-Width Integer Reduction Instructions 1978let MaskedPolicy = NonePolicy in { 1979defm vredsum : RVVIntReductionBuiltinSet; 1980defm vredmaxu : RVVUnsignedReductionBuiltin; 1981defm vredmax : RVVSignedReductionBuiltin; 1982defm vredminu : RVVUnsignedReductionBuiltin; 1983defm vredmin : RVVSignedReductionBuiltin; 1984defm vredand : RVVIntReductionBuiltinSet; 1985defm vredor : RVVIntReductionBuiltinSet; 1986defm vredxor : RVVIntReductionBuiltinSet; 1987 1988// 15.2. Vector Widening Integer Reduction Instructions 1989// Vector Widening Integer Reduction Operations 1990let HasMaskedOffOperand = false in { 1991 defm vwredsum : RVVOutOp1BuiltinSet<"vwredsum", "csi", 1992 [["vs", "vSw", "SwSwvSw"]]>; 1993 defm vwredsumu : RVVOutOp1BuiltinSet<"vwredsumu", "csi", 1994 [["vs", "UvUSw", "USwUSwUvUSw"]]>; 1995} 1996 1997// 15.3. Vector Single-Width Floating-Point Reduction Instructions 1998defm vfredmax : RVVFloatingReductionBuiltin; 1999defm vfredmin : RVVFloatingReductionBuiltin; 2000defm vfredusum : RVVFloatingReductionBuiltin; 2001defm vfredosum : RVVFloatingReductionBuiltin; 2002 2003// 15.4. Vector Widening Floating-Point Reduction Instructions 2004defm vfwredusum : RVVFloatingWidenReductionBuiltin; 2005defm vfwredosum : RVVFloatingWidenReductionBuiltin; 2006} 2007 2008// 16. Vector Mask Instructions 2009// 16.1. Vector Mask-Register Logical Instructions 2010def vmand : RVVMaskBinBuiltin; 2011def vmnand : RVVMaskBinBuiltin; 2012def vmandn : RVVMaskBinBuiltin; 2013def vmxor : RVVMaskBinBuiltin; 2014def vmor : RVVMaskBinBuiltin; 2015def vmnor : RVVMaskBinBuiltin; 2016def vmorn : RVVMaskBinBuiltin; 2017def vmxnor : RVVMaskBinBuiltin; 2018// pseudoinstructions 2019def vmclr : RVVMaskNullaryBuiltin; 2020def vmset : RVVMaskNullaryBuiltin; 2021defm vmmv_m : RVVPseudoMaskBuiltin<"vmand", "c">; 2022defm vmnot_m : RVVPseudoMaskBuiltin<"vmnand", "c">; 2023 2024let MaskedPolicy = NonePolicy in { 2025// 16.2. Vector count population in mask vcpop.m 2026def vcpop : RVVMaskOp0Builtin<"um">; 2027 2028// 16.3. vfirst find-first-set mask bit 2029def vfirst : RVVMaskOp0Builtin<"lm">; 2030 2031// 16.4. vmsbf.m set-before-first mask bit 2032def vmsbf : RVVMaskUnaryBuiltin; 2033 2034// 16.5. vmsif.m set-including-first mask bit 2035def vmsif : RVVMaskUnaryBuiltin; 2036 2037// 16.6. vmsof.m set-only-first mask bit 2038def vmsof : RVVMaskUnaryBuiltin; 2039} 2040 2041let UnMaskedPolicy = HasPassthruOperand, HasUnMaskedOverloaded = false in { 2042 // 16.8. Vector Iota Instruction 2043 defm viota : RVVOutBuiltinSet<"viota", "csil", [["m", "Uv", "Uvm"]]>; 2044 2045 // 16.9. Vector Element Index Instruction 2046 defm vid : RVVOutBuiltinSet<"vid", "csil", [["v", "v", "v"], 2047 ["v", "Uv", "Uv"]]>; 2048} 2049 2050// 17. Vector Permutation Instructions 2051// 17.1. Integer Scalar Move Instructions 2052let HasMasked = false, MaskedPolicy = NonePolicy in { 2053 let HasVL = false, OverloadedName = "vmv_x" in 2054 defm vmv_x : RVVOp0BuiltinSet<"vmv_x_s", "csil", 2055 [["s", "ve", "ev"], 2056 ["s", "UvUe", "UeUv"]]>; 2057 let OverloadedName = "vmv_s" in 2058 defm vmv_s : RVVOutBuiltinSet<"vmv_s_x", "csil", 2059 [["x", "v", "vve"], 2060 ["x", "Uv", "UvUvUe"]]>; 2061} 2062 2063// 17.2. Floating-Point Scalar Move Instructions 2064let HasMasked = false, MaskedPolicy = NonePolicy in { 2065 let HasVL = false, OverloadedName = "vfmv_f" in 2066 defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "xfd", 2067 [["s", "ve", "ev"]]>; 2068 let OverloadedName = "vfmv_s" in 2069 defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "xfd", 2070 [["f", "v", "vve"], 2071 ["x", "Uv", "UvUvUe"]]>; 2072} 2073 2074// 17.3. Vector Slide Instructions 2075// 17.3.1. Vector Slideup Instructions 2076defm vslideup : RVVSlideBuiltinSet; 2077// 17.3.2. Vector Slidedown Instructions 2078defm vslidedown : RVVSlideBuiltinSet; 2079 2080// 17.3.3. Vector Slide1up Instructions 2081let UnMaskedPolicy = HasPassthruOperand in { 2082defm vslide1up : RVVSlideOneBuiltinSet; 2083defm vfslide1up : RVVFloatingBinVFBuiltinSet; 2084 2085// 17.3.4. Vector Slide1down Instruction 2086defm vslide1down : RVVSlideOneBuiltinSet; 2087defm vfslide1down : RVVFloatingBinVFBuiltinSet; 2088 2089// 17.4. Vector Register Gather Instructions 2090// signed and floating type 2091defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfd", 2092 [["vv", "v", "vvUv"]]>; 2093defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfd", 2094 [["vx", "v", "vvz"]]>; 2095defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd", 2096 [["vv", "v", "vv(Log2EEW:4)Uv"]]>; 2097// unsigned type 2098defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil", 2099 [["vv", "Uv", "UvUvUv"]]>; 2100defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csil", 2101 [["vx", "Uv", "UvUvz"]]>; 2102defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csil", 2103 [["vv", "Uv", "UvUv(Log2EEW:4)Uv"]]>; 2104} 2105 2106// 17.5. Vector Compress Instruction 2107let HasMasked = false, MaskedPolicy = NonePolicy, 2108 ManualCodegen = [{ 2109 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3); 2110 IntrinsicTypes = {ResultType, Ops[3]->getType()}; 2111 }] in { 2112 // signed and floating type 2113 defm vcompress : RVVOutBuiltinSet<"vcompress", "csilxfd", 2114 [["vm", "v", "vmvv"]]>; 2115 // unsigned type 2116 defm vcompress : RVVOutBuiltinSet<"vcompress", "csil", 2117 [["vm", "Uv", "UvmUvUv"]]>; 2118} 2119 2120// Miscellaneous 2121let HasMasked = false, HasVL = false, IRName = "" in { 2122 let Name = "vreinterpret_v", MaskedPolicy = NonePolicy, 2123 ManualCodegen = [{ 2124 return Builder.CreateBitCast(Ops[0], ResultType); 2125 }] in { 2126 // Reinterpret between different type under the same SEW and LMUL 2127 def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">; 2128 def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "sil", "v">; 2129 def vreinterpret_u_i : RVVBuiltin<"vUv", "Uvv", "csil", "Uv">; 2130 def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "sil", "Uv">; 2131 def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "sil", "Fv">; 2132 def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "sil", "Fv">; 2133 2134 // Reinterpret between different SEW under the same LMUL 2135 foreach dst_sew = ["(FixedSEW:8)", "(FixedSEW:16)", "(FixedSEW:32)", 2136 "(FixedSEW:64)"] in { 2137 def vreinterpret_i_ # dst_sew : RVVBuiltin<"v" # dst_sew # "v", 2138 dst_sew # "vv", "csil", dst_sew # "v">; 2139 def vreinterpret_u_ # dst_sew : RVVBuiltin<"Uv" # dst_sew # "Uv", 2140 dst_sew # "UvUv", "csil", dst_sew # "Uv">; 2141 } 2142 } 2143 2144 let Name = "vundefined", HasUnMaskedOverloaded = false, MaskedPolicy = NonePolicy, 2145 ManualCodegen = [{ 2146 return llvm::UndefValue::get(ResultType); 2147 }] in { 2148 def vundefined : RVVBuiltin<"v", "v", "csilxfd">; 2149 def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">; 2150 } 2151 2152 // LMUL truncation 2153 // C/C++ Operand: VecTy, IR Operand: VecTy, Index 2154 let Name = "vlmul_trunc_v", OverloadedName = "vlmul_trunc", MaskedPolicy = NonePolicy, 2155 ManualCodegen = [{ { 2156 ID = Intrinsic::vector_extract; 2157 IntrinsicTypes = {ResultType, Ops[0]->getType()}; 2158 Ops.push_back(ConstantInt::get(Int64Ty, 0)); 2159 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2160 } }] in { 2161 foreach dst_lmul = ["(SFixedLog2LMUL:-3)", "(SFixedLog2LMUL:-2)", "(SFixedLog2LMUL:-1)", 2162 "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2163 def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2164 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2165 def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2166 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2167 } 2168 } 2169 2170 // LMUL extension 2171 // C/C++ Operand: SubVecTy, IR Operand: VecTy, SubVecTy, Index 2172 let Name = "vlmul_ext_v", OverloadedName = "vlmul_ext", MaskedPolicy = NonePolicy, 2173 ManualCodegen = [{ 2174 ID = Intrinsic::vector_insert; 2175 IntrinsicTypes = {ResultType, Ops[0]->getType()}; 2176 Ops.push_back(llvm::UndefValue::get(ResultType)); 2177 std::swap(Ops[0], Ops[1]); 2178 Ops.push_back(ConstantInt::get(Int64Ty, 0)); 2179 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2180 }] in { 2181 foreach dst_lmul = ["(LFixedLog2LMUL:-2)", "(LFixedLog2LMUL:-1)", "(LFixedLog2LMUL:-0)", 2182 "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2183 def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2184 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2185 def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2186 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2187 } 2188 } 2189 2190 let Name = "vget_v", MaskedPolicy = NonePolicy, 2191 ManualCodegen = [{ 2192 { 2193 ID = Intrinsic::vector_extract; 2194 auto *VecTy = cast<ScalableVectorType>(ResultType); 2195 auto *OpVecTy = cast<ScalableVectorType>(Ops[0]->getType()); 2196 // Mask to only valid indices. 2197 unsigned MaxIndex = OpVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2198 assert(isPowerOf2_32(MaxIndex)); 2199 Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); 2200 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2201 Ops[1] = Builder.CreateMul(Ops[1], 2202 ConstantInt::get(Ops[1]->getType(), 2203 VecTy->getMinNumElements())); 2204 IntrinsicTypes = {ResultType, Ops[0]->getType()}; 2205 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2206 } 2207 }] in { 2208 foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2209 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfd", dst_lmul # "v">; 2210 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">; 2211 } 2212 } 2213 2214 let Name = "vset_v", Log2LMUL = [0, 1, 2], MaskedPolicy = NonePolicy, 2215 ManualCodegen = [{ 2216 { 2217 ID = Intrinsic::vector_insert; 2218 IntrinsicTypes = {ResultType, Ops[2]->getType()}; 2219 auto *ResVecTy = cast<ScalableVectorType>(ResultType); 2220 auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType()); 2221 // Mask to only valid indices. 2222 unsigned MaxIndex = ResVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2223 assert(isPowerOf2_32(MaxIndex)); 2224 Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); 2225 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2226 Ops[1] = Builder.CreateMul(Ops[1], 2227 ConstantInt::get(Ops[1]->getType(), 2228 VecTy->getMinNumElements())); 2229 std::swap(Ops[1], Ops[2]); 2230 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2231 } 2232 }] in { 2233 foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2234 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfd">; 2235 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">; 2236 } 2237 } 2238} 2239