1//==--- riscv_vector.td - RISC-V V-ext Builtin function list --------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the builtins for RISC-V V-extension. See: 10// 11// https://github.com/riscv/rvv-intrinsic-doc 12// 13//===----------------------------------------------------------------------===// 14 15//===----------------------------------------------------------------------===// 16// Instruction definitions 17//===----------------------------------------------------------------------===// 18// Each record of the class RVVBuiltin defines a collection of builtins (i.e. 19// "def vadd : RVVBuiltin" will be used to define things like "vadd_vv_i32m1", 20// "vadd_vv_i32m2", etc). 21// 22// The elements of this collection are defined by an instantiation process the 23// range of which is specified by the cross product of the LMUL attribute and 24// every element in the attribute TypeRange. By default builtins have LMUL = [1, 25// 2, 4, 8, 1/2, 1/4, 1/8] so the process is repeated 7 times. In tablegen we 26// use the Log2LMUL [0, 1, 2, 3, -1, -2, -3] to represent the LMUL. 27// 28// LMUL represents the fact that the types of values used by that builtin are 29// values generated by instructions that are executed under that LMUL. However, 30// this does not mean the builtin is necessarily lowered into an instruction 31// that executes under the specified LMUL. An example where this happens are 32// loads and stores of masks. A mask like `vbool8_t` can be generated, for 33// instance, by comparing two `__rvv_int8m1_t` (this is LMUL=1) or comparing two 34// `__rvv_int16m2_t` (this is LMUL=2). The actual load or store, however, will 35// be performed under LMUL=1 because mask registers are not grouped. 36// 37// TypeRange is a non-empty sequence of basic types: 38// 39// c: int8_t (i8) 40// s: int16_t (i16) 41// i: int32_t (i32) 42// l: int64_t (i64) 43// x: float16_t (half) 44// f: float32_t (float) 45// d: float64_t (double) 46// 47// This way, given an LMUL, a record with a TypeRange "sil" will cause the 48// definition of 3 builtins. Each type "t" in the TypeRange (in this example 49// they are int16_t, int32_t, int64_t) is used as a parameter that drives the 50// definition of that particular builtin (for the given LMUL). 51// 52// During the instantiation, types can be transformed or modified using type 53// transformers. Given a type "t" the following primitive type transformers can 54// be applied to it to yield another type. 55// 56// e: type of "t" as is (identity) 57// v: computes a vector type whose element type is "t" for the current LMUL 58// w: computes a vector type identical to what 'v' computes except for the 59// element type which is twice as wide as the element type of 'v' 60// q: computes a vector type identical to what 'v' computes except for the 61// element type which is four times as wide as the element type of 'v' 62// o: computes a vector type identical to what 'v' computes except for the 63// element type which is eight times as wide as the element type of 'v' 64// m: computes a vector type identical to what 'v' computes except for the 65// element type which is bool 66// 0: void type, ignores "t" 67// z: size_t, ignores "t" 68// t: ptrdiff_t, ignores "t" 69// u: unsigned long, ignores "t" 70// l: long, ignores "t" 71// 72// So for instance if t is "i", i.e. int, then "e" will yield int again. "v" 73// will yield an RVV vector type (assume LMUL=1), so __rvv_int32m1_t. 74// Accordingly "w" would yield __rvv_int64m2_t. 75// 76// A type transformer can be prefixed by other non-primitive type transformers. 77// 78// P: constructs a pointer to the current type 79// C: adds const to the type 80// K: requires the integer type to be a constant expression 81// U: given an integer type or vector type, computes its unsigned variant 82// I: given a vector type, compute the vector type with integer type 83// elements of the same width 84// F: given a vector type, compute the vector type with floating-point type 85// elements of the same width 86// S: given a vector type, computes its equivalent one for LMUL=1. This is a 87// no-op if the vector was already LMUL=1 88// (Log2EEW:Value): Log2EEW value could be 3/4/5/6 (8/16/32/64), given a 89// vector type (SEW and LMUL) and EEW (8/16/32/64), computes its 90// equivalent integer vector type with EEW and corresponding ELMUL (elmul = 91// (eew/sew) * lmul). For example, vector type is __rvv_float16m4 92// (SEW=16, LMUL=4) and Log2EEW is 3 (EEW=8), and then equivalent vector 93// type is __rvv_uint8m2_t (elmul=(8/16)*4 = 2). Ignore to define a new 94// builtins if its equivalent type has illegal lmul. 95// (FixedSEW:Value): Given a vector type (SEW and LMUL), and computes another 96// vector type which only changed SEW as given value. Ignore to define a new 97// builtin if its equivalent type has illegal lmul or the SEW does not changed. 98// (SFixedLog2LMUL:Value): Smaller Fixed Log2LMUL. Given a vector type (SEW 99// and LMUL), and computes another vector type which only changed LMUL as 100// given value. The new LMUL should be smaller than the old one. Ignore to 101// define a new builtin if its equivalent type has illegal lmul. 102// (LFixedLog2LMUL:Value): Larger Fixed Log2LMUL. Given a vector type (SEW 103// and LMUL), and computes another vector type which only changed LMUL as 104// given value. The new LMUL should be larger than the old one. Ignore to 105// define a new builtin if its equivalent type has illegal lmul. 106// 107// Following with the example above, if t is "i", then "Ue" will yield unsigned 108// int and "Fv" will yield __rvv_float32m1_t (again assuming LMUL=1), Fw would 109// yield __rvv_float64m2_t, etc. 110// 111// Each builtin is then defined by applying each type in TypeRange against the 112// sequence of type transformers described in Suffix and Prototype. 113// 114// The name of the builtin is defined by the Name attribute (which defaults to 115// the name of the class) appended (separated with an underscore) the Suffix 116// attribute. For instance with Name="foo", Suffix = "v" and TypeRange = "il", 117// the builtin generated will be __builtin_rvv_foo_i32m1 and 118// __builtin_rvv_foo_i64m1 (under LMUL=1). If Suffix contains more than one 119// type transformer (say "vv") each of the types is separated with an 120// underscore as in "__builtin_rvv_foo_i32m1_i32m1". 121// 122// The C/C++ prototype of the builtin is defined by the Prototype attribute. 123// Prototype is a non-empty sequence of type transformers, the first of which 124// is the return type of the builtin and the rest are the parameters of the 125// builtin, in order. For instance if Prototype is "wvv" and TypeRange is "si" 126// a first builtin will have type 127// __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t) and the second builtin 128// will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t) (again 129// under LMUL=1). 130// 131// There are a number of attributes that are used to constraint the number and 132// shape of the builtins generated. Refer to the comments below for them. 133 134class PolicyScheme<int val>{ 135 int Value = val; 136} 137def NonePolicy : PolicyScheme<0>; 138def HasPassthruOperand : PolicyScheme<1>; 139def HasPolicyOperand : PolicyScheme<2>; 140 141class RVVBuiltin<string suffix, string prototype, string type_range, 142 string overloaded_suffix = ""> { 143 // Base name that will be prepended in __builtin_rvv_ and appended the 144 // computed Suffix. 145 string Name = NAME; 146 147 // If not empty, each instantiated builtin will have this appended after an 148 // underscore (_). It is instantiated like Prototype. 149 string Suffix = suffix; 150 151 // If empty, default OverloadedName is sub string of `Name` which end of first 152 // '_'. For example, the default overloaded name is `vadd` for Name `vadd_vv`. 153 // It's used for describe some special naming cases. 154 string OverloadedName = ""; 155 156 // If not empty, each OverloadedName will have this appended after an 157 // underscore (_). It is instantiated like Prototype. 158 string OverloadedSuffix = overloaded_suffix; 159 160 // The different variants of the builtin, parameterised with a type. 161 string TypeRange = type_range; 162 163 // We use each type described in TypeRange and LMUL with prototype to 164 // instantiate a specific element of the set of builtins being defined. 165 // Prototype attribute defines the C/C++ prototype of the builtin. It is a 166 // non-empty sequence of type transformers, the first of which is the return 167 // type of the builtin and the rest are the parameters of the builtin, in 168 // order. For instance if Prototype is "wvv", TypeRange is "si" and LMUL=1, a 169 // first builtin will have type 170 // __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t), and the second builtin 171 // will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t). 172 string Prototype = prototype; 173 174 // This builtin has a masked form. 175 bit HasMasked = true; 176 177 // If HasMasked, this flag states that this builtin has a maskedoff operand. It 178 // is always the first operand in builtin and IR intrinsic. 179 bit HasMaskedOffOperand = true; 180 181 // This builtin has a granted vector length parameter. 182 bit HasVL = true; 183 184 // The policy scheme for masked intrinsic IR. 185 // It could be NonePolicy or HasPolicyOperand. 186 // HasPolicyOperand: Has a policy operand. 0 is tail and mask undisturbed, 1 is 187 // tail agnostic, 2 is mask undisturbed, and 3 is tail and mask agnostic. The 188 // policy operand is located at the last position. 189 PolicyScheme MaskedPolicyScheme = HasPolicyOperand; 190 191 // The policy scheme for unmasked intrinsic IR. 192 // It could be NonePolicy, HasPassthruOperand or HasPolicyOperand. 193 // HasPassthruOperand: Has a passthru operand to decide tail policy. If it is 194 // poison, tail policy is tail agnostic, otherwise policy is tail undisturbed. 195 // HasPolicyOperand: Has a policy operand. 1 is tail agnostic and 0 is tail 196 // undisturbed. 197 PolicyScheme UnMaskedPolicyScheme = NonePolicy; 198 199 // This builtin support tail agnostic and undisturbed policy. 200 bit HasTailPolicy = true; 201 // This builtin support mask agnostic and undisturbed policy. 202 bit HasMaskPolicy = true; 203 204 // This builtin prototype with TA or TAMA policy could not support overloading 205 // API. Other policy intrinsic functions would support overloading API with 206 // suffix `_tu`, `tumu`, `tuma`, `tamu` and `tama`. 207 bit SupportOverloading = true; 208 209 // This builtin is valid for the given Log2LMULs. 210 list<int> Log2LMUL = [0, 1, 2, 3, -1, -2, -3]; 211 212 // Manual code in clang codegen riscv_vector_builtin_cg.inc 213 code ManualCodegen = [{}]; 214 215 // When emit the automatic clang codegen, it describes what types we have to use 216 // to obtain the specific LLVM intrinsic. -1 means the return type, otherwise, 217 // k >= 0 meaning the k-th operand (counting from zero) of the codegen'd 218 // parameter of the unmasked version. k can't be the mask operand's position. 219 list<int> IntrinsicTypes = []; 220 221 // If these names are not empty, this is the ID of the LLVM intrinsic 222 // we want to lower to. 223 string IRName = NAME; 224 225 // If HasMasked, this is the ID of the LLVM intrinsic we want to lower to. 226 string MaskedIRName = NAME #"_mask"; 227 228 // Use clang_builtin_alias to save the number of builtins. 229 bit HasBuiltinAlias = true; 230 231 // Features required to enable for this builtin. 232 list<string> RequiredFeatures = []; 233 234 // Number of fields for Load/Store Segment instructions. 235 int NF = 1; 236} 237 238// This is the code emitted in the header. 239class RVVHeader { 240 code HeaderCode; 241} 242 243//===----------------------------------------------------------------------===// 244// Basic classes with automatic codegen. 245//===----------------------------------------------------------------------===// 246 247class RVVOutBuiltin<string suffix, string prototype, string type_range> 248 : RVVBuiltin<suffix, prototype, type_range> { 249 let IntrinsicTypes = [-1]; 250} 251 252class RVVOp0Builtin<string suffix, string prototype, string type_range> 253 : RVVBuiltin<suffix, prototype, type_range> { 254 let IntrinsicTypes = [0]; 255} 256 257class RVVOutOp1Builtin<string suffix, string prototype, string type_range> 258 : RVVBuiltin<suffix, prototype, type_range> { 259 let IntrinsicTypes = [-1, 1]; 260} 261 262class RVVOutOp0Op1Builtin<string suffix, string prototype, string type_range> 263 : RVVBuiltin<suffix, prototype, type_range> { 264 let IntrinsicTypes = [-1, 0, 1]; 265} 266 267multiclass RVVBuiltinSet<string intrinsic_name, string type_range, 268 list<list<string>> suffixes_prototypes, 269 list<int> intrinsic_types> { 270 let IRName = intrinsic_name, MaskedIRName = intrinsic_name # "_mask", 271 IntrinsicTypes = intrinsic_types in { 272 foreach s_p = suffixes_prototypes in { 273 let Name = NAME # "_" # s_p[0] in { 274 defvar suffix = s_p[1]; 275 defvar prototype = s_p[2]; 276 def : RVVBuiltin<suffix, prototype, type_range>; 277 } 278 } 279 } 280} 281 282// IntrinsicTypes is output, op0, op1 [-1, 0, 1] 283multiclass RVVOutOp0Op1BuiltinSet<string intrinsic_name, string type_range, 284 list<list<string>> suffixes_prototypes> 285 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, 286 [-1, 0, 1]>; 287 288multiclass RVVOutBuiltinSet<string intrinsic_name, string type_range, 289 list<list<string>> suffixes_prototypes> 290 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1]>; 291 292multiclass RVVOp0BuiltinSet<string intrinsic_name, string type_range, 293 list<list<string>> suffixes_prototypes> 294 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0]>; 295 296// IntrinsicTypes is output, op1 [-1, 0] 297multiclass RVVOutOp0BuiltinSet<string intrinsic_name, string type_range, 298 list<list<string>> suffixes_prototypes> 299 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 0]>; 300 301// IntrinsicTypes is output, op1 [-1, 1] 302multiclass RVVOutOp1BuiltinSet<string intrinsic_name, string type_range, 303 list<list<string>> suffixes_prototypes> 304 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1]>; 305 306multiclass RVVOp0Op1BuiltinSet<string intrinsic_name, string type_range, 307 list<list<string>> suffixes_prototypes> 308 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0, 1]>; 309 310multiclass RVVOutOp1Op2BuiltinSet<string intrinsic_name, string type_range, 311 list<list<string>> suffixes_prototypes> 312 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1, 2]>; 313 314multiclass RVVSignedBinBuiltinSet 315 : RVVOutOp1BuiltinSet<NAME, "csil", 316 [["vv", "v", "vvv"], 317 ["vx", "v", "vve"]]>; 318 319multiclass RVVUnsignedBinBuiltinSet 320 : RVVOutOp1BuiltinSet<NAME, "csil", 321 [["vv", "Uv", "UvUvUv"], 322 ["vx", "Uv", "UvUvUe"]]>; 323 324multiclass RVVIntBinBuiltinSet 325 : RVVSignedBinBuiltinSet, 326 RVVUnsignedBinBuiltinSet; 327 328multiclass RVVSlideOneBuiltinSet 329 : RVVOutOp1BuiltinSet<NAME, "csil", 330 [["vx", "v", "vve"], 331 ["vx", "Uv", "UvUve"]]>; 332 333multiclass RVVSignedShiftBuiltinSet 334 : RVVOutOp1BuiltinSet<NAME, "csil", 335 [["vv", "v", "vvUv"], 336 ["vx", "v", "vvz"]]>; 337 338multiclass RVVUnsignedShiftBuiltinSet 339 : RVVOutOp1BuiltinSet<NAME, "csil", 340 [["vv", "Uv", "UvUvUv"], 341 ["vx", "Uv", "UvUvz"]]>; 342 343multiclass RVVShiftBuiltinSet 344 : RVVSignedShiftBuiltinSet, 345 RVVUnsignedShiftBuiltinSet; 346 347let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 348 multiclass RVVSignedNShiftBuiltinSet 349 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 350 [["wv", "v", "vwUv"], 351 ["wx", "v", "vwz"]]>; 352 multiclass RVVUnsignedNShiftBuiltinSet 353 : RVVOutOp0Op1BuiltinSet<NAME, "csil", 354 [["wv", "Uv", "UvUwUv"], 355 ["wx", "Uv", "UvUwz"]]>; 356} 357 358multiclass RVVCarryinBuiltinSet 359 : RVVOutOp1BuiltinSet<NAME, "csil", 360 [["vvm", "v", "vvvm"], 361 ["vxm", "v", "vvem"], 362 ["vvm", "Uv", "UvUvUvm"], 363 ["vxm", "Uv", "UvUvUem"]]>; 364 365multiclass RVVCarryOutInBuiltinSet<string intrinsic_name> 366 : RVVOp0Op1BuiltinSet<intrinsic_name, "csil", 367 [["vvm", "vm", "mvvm"], 368 ["vxm", "vm", "mvem"], 369 ["vvm", "Uvm", "mUvUvm"], 370 ["vxm", "Uvm", "mUvUem"]]>; 371 372multiclass RVVSignedMaskOutBuiltinSet 373 : RVVOp0Op1BuiltinSet<NAME, "csil", 374 [["vv", "vm", "mvv"], 375 ["vx", "vm", "mve"]]>; 376 377multiclass RVVUnsignedMaskOutBuiltinSet 378 : RVVOp0Op1BuiltinSet<NAME, "csil", 379 [["vv", "Uvm", "mUvUv"], 380 ["vx", "Uvm", "mUvUe"]]>; 381 382multiclass RVVIntMaskOutBuiltinSet 383 : RVVSignedMaskOutBuiltinSet, 384 RVVUnsignedMaskOutBuiltinSet; 385 386class RVVIntExt<string intrinsic_name, string suffix, string prototype, 387 string type_range> 388 : RVVBuiltin<suffix, prototype, type_range> { 389 let IRName = intrinsic_name; 390 let MaskedIRName = intrinsic_name # "_mask"; 391 let OverloadedName = NAME; 392 let IntrinsicTypes = [-1, 0]; 393} 394 395let HasMaskedOffOperand = false in { 396 multiclass RVVIntTerBuiltinSet { 397 defm "" : RVVOutOp1BuiltinSet<NAME, "csil", 398 [["vv", "v", "vvvv"], 399 ["vx", "v", "vvev"], 400 ["vv", "Uv", "UvUvUvUv"], 401 ["vx", "Uv", "UvUvUeUv"]]>; 402 } 403 multiclass RVVFloatingTerBuiltinSet { 404 defm "" : RVVOutOp1BuiltinSet<NAME, "xfd", 405 [["vv", "v", "vvvv"], 406 ["vf", "v", "vvev"]]>; 407 } 408} 409 410let HasMaskedOffOperand = false, Log2LMUL = [-2, -1, 0, 1, 2] in { 411 multiclass RVVFloatingWidenTerBuiltinSet { 412 defm "" : RVVOutOp1Op2BuiltinSet<NAME, "xf", 413 [["vv", "w", "wwvv"], 414 ["vf", "w", "wwev"]]>; 415 } 416} 417 418multiclass RVVFloatingBinBuiltinSet 419 : RVVOutOp1BuiltinSet<NAME, "xfd", 420 [["vv", "v", "vvv"], 421 ["vf", "v", "vve"]]>; 422 423multiclass RVVFloatingBinVFBuiltinSet 424 : RVVOutOp1BuiltinSet<NAME, "xfd", 425 [["vf", "v", "vve"]]>; 426 427multiclass RVVFloatingMaskOutBuiltinSet 428 : RVVOp0Op1BuiltinSet<NAME, "xfd", 429 [["vv", "vm", "mvv"], 430 ["vf", "vm", "mve"]]>; 431 432multiclass RVVFloatingMaskOutVFBuiltinSet 433 : RVVOp0Op1BuiltinSet<NAME, "fd", 434 [["vf", "vm", "mve"]]>; 435 436class RVVMaskBinBuiltin : RVVOutBuiltin<"m", "mmm", "c"> { 437 let Name = NAME # "_mm"; 438 let HasMasked = false; 439} 440 441class RVVMaskUnaryBuiltin : RVVOutBuiltin<"m", "mm", "c"> { 442 let Name = NAME # "_m"; 443} 444 445class RVVMaskNullaryBuiltin : RVVOutBuiltin<"m", "m", "c"> { 446 let Name = NAME # "_m"; 447 let HasMasked = false; 448 let SupportOverloading = false; 449} 450 451class RVVMaskOp0Builtin<string prototype> : RVVOp0Builtin<"m", prototype, "c"> { 452 let Name = NAME # "_m"; 453 let HasMaskedOffOperand = false; 454} 455 456let UnMaskedPolicyScheme = HasPolicyOperand, 457 HasMaskedOffOperand = false in { 458 multiclass RVVSlideUpBuiltinSet { 459 defm "" : RVVOutBuiltinSet<NAME, "csilxfd", 460 [["vx","v", "vvvz"]]>; 461 defm "" : RVVOutBuiltinSet<NAME, "csil", 462 [["vx","Uv", "UvUvUvz"]]>; 463 } 464} 465 466let UnMaskedPolicyScheme = HasPassthruOperand, 467 ManualCodegen = [{ 468 if (IsMasked) { 469 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 470 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 471 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 472 } else { 473 if (PolicyAttrs & RVV_VTA) 474 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 475 } 476 477 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 478 IntrinsicTypes = {ResultType, Ops.back()->getType()}; 479 }] in { 480 multiclass RVVSlideDownBuiltinSet { 481 defm "" : RVVOutBuiltinSet<NAME, "csilxfd", 482 [["vx","v", "vvz"]]>; 483 defm "" : RVVOutBuiltinSet<NAME, "csil", 484 [["vx","Uv", "UvUvz"]]>; 485 } 486} 487 488class RVVFloatingUnaryBuiltin<string builtin_suffix, string ir_suffix, 489 string prototype> 490 : RVVOutBuiltin<ir_suffix, prototype, "xfd"> { 491 let Name = NAME # "_" # builtin_suffix; 492} 493 494class RVVFloatingUnaryVVBuiltin : RVVFloatingUnaryBuiltin<"v", "v", "vv">; 495 496class RVVConvBuiltin<string suffix, string prototype, string type_range, 497 string overloaded_name> 498 : RVVBuiltin<suffix, prototype, type_range> { 499 let IntrinsicTypes = [-1, 0]; 500 let OverloadedName = overloaded_name; 501} 502 503class RVVConvToSignedBuiltin<string overloaded_name> 504 : RVVConvBuiltin<"Iv", "Ivv", "xfd", overloaded_name>; 505 506class RVVConvToUnsignedBuiltin<string overloaded_name> 507 : RVVConvBuiltin<"Uv", "Uvv", "xfd", overloaded_name>; 508 509class RVVConvToWidenSignedBuiltin<string overloaded_name> 510 : RVVConvBuiltin<"Iw", "Iwv", "xf", overloaded_name>; 511 512class RVVConvToWidenUnsignedBuiltin<string overloaded_name> 513 : RVVConvBuiltin<"Uw", "Uwv", "xf", overloaded_name>; 514 515class RVVConvToNarrowingSignedBuiltin<string overloaded_name> 516 : RVVConvBuiltin<"Iv", "IvFw", "csi", overloaded_name>; 517 518class RVVConvToNarrowingUnsignedBuiltin<string overloaded_name> 519 : RVVConvBuiltin<"Uv", "UvFw", "csi", overloaded_name>; 520 521let HasMaskedOffOperand = true in { 522 multiclass RVVSignedReductionBuiltin { 523 defm "" : RVVOutOp0BuiltinSet<NAME, "csil", 524 [["vs", "vSv", "SvvSv"]]>; 525 } 526 multiclass RVVUnsignedReductionBuiltin { 527 defm "" : RVVOutOp0BuiltinSet<NAME, "csil", 528 [["vs", "UvUSv", "USvUvUSv"]]>; 529 } 530 multiclass RVVFloatingReductionBuiltin { 531 defm "" : RVVOutOp0BuiltinSet<NAME, "xfd", 532 [["vs", "vSv", "SvvSv"]]>; 533 } 534 multiclass RVVFloatingWidenReductionBuiltin { 535 defm "" : RVVOutOp0BuiltinSet<NAME, "xf", 536 [["vs", "vSw", "SwvSw"]]>; 537 } 538} 539 540multiclass RVVIntReductionBuiltinSet 541 : RVVSignedReductionBuiltin, 542 RVVUnsignedReductionBuiltin; 543 544// For widen operation which has different mangling name. 545multiclass RVVWidenBuiltinSet<string intrinsic_name, string type_range, 546 list<list<string>> suffixes_prototypes> { 547 let Log2LMUL = [-3, -2, -1, 0, 1, 2], 548 IRName = intrinsic_name, MaskedIRName = intrinsic_name # "_mask" in { 549 foreach s_p = suffixes_prototypes in { 550 let Name = NAME # "_" # s_p[0], 551 OverloadedName = NAME # "_" # s_p[0] in { 552 defvar suffix = s_p[1]; 553 defvar prototype = s_p[2]; 554 def : RVVOutOp0Op1Builtin<suffix, prototype, type_range>; 555 } 556 } 557 } 558} 559 560// For widen operation with widen operand which has different mangling name. 561multiclass RVVWidenWOp0BuiltinSet<string intrinsic_name, string type_range, 562 list<list<string>> suffixes_prototypes> { 563 let Log2LMUL = [-3, -2, -1, 0, 1, 2], 564 IRName = intrinsic_name, MaskedIRName = intrinsic_name # "_mask" in { 565 foreach s_p = suffixes_prototypes in { 566 let Name = NAME # "_" # s_p[0], 567 OverloadedName = NAME # "_" # s_p[0] in { 568 defvar suffix = s_p[1]; 569 defvar prototype = s_p[2]; 570 def : RVVOutOp1Builtin<suffix, prototype, type_range>; 571 } 572 } 573 } 574} 575 576multiclass RVVSignedWidenBinBuiltinSet 577 : RVVWidenBuiltinSet<NAME, "csi", 578 [["vv", "w", "wvv"], 579 ["vx", "w", "wve"]]>; 580 581multiclass RVVSignedWidenOp0BinBuiltinSet 582 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", 583 [["wv", "w", "wwv"], 584 ["wx", "w", "wwe"]]>; 585 586multiclass RVVUnsignedWidenBinBuiltinSet 587 : RVVWidenBuiltinSet<NAME, "csi", 588 [["vv", "Uw", "UwUvUv"], 589 ["vx", "Uw", "UwUvUe"]]>; 590 591multiclass RVVUnsignedWidenOp0BinBuiltinSet 592 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", 593 [["wv", "Uw", "UwUwUv"], 594 ["wx", "Uw", "UwUwUe"]]>; 595 596multiclass RVVFloatingWidenBinBuiltinSet 597 : RVVWidenBuiltinSet<NAME, "xf", 598 [["vv", "w", "wvv"], 599 ["vf", "w", "wve"]]>; 600 601multiclass RVVFloatingWidenOp0BinBuiltinSet 602 : RVVWidenWOp0BuiltinSet<NAME # "_w", "xf", 603 [["wv", "w", "wwv"], 604 ["wf", "w", "wwe"]]>; 605 606defvar TypeList = ["c","s","i","l","x","f","d"]; 607defvar EEWList = [["8", "(Log2EEW:3)"], 608 ["16", "(Log2EEW:4)"], 609 ["32", "(Log2EEW:5)"], 610 ["64", "(Log2EEW:6)"]]; 611 612class IsFloat<string type> { 613 bit val = !or(!eq(type, "x"), !eq(type, "f"), !eq(type, "d")); 614} 615 616let SupportOverloading = false, 617 MaskedPolicyScheme = NonePolicy in { 618 class RVVVLEMaskBuiltin : RVVOutBuiltin<"m", "mPCUe", "c"> { 619 let Name = "vlm_v"; 620 let IRName = "vlm"; 621 let HasMasked = false; 622 } 623} 624 625let SupportOverloading = false, 626 UnMaskedPolicyScheme = HasPassthruOperand in { 627 multiclass RVVVLEBuiltin<list<string> types> { 628 let Name = NAME # "_v", 629 IRName = "vle", 630 MaskedIRName ="vle_mask" in { 631 foreach type = types in { 632 def : RVVOutBuiltin<"v", "vPCe", type>; 633 if !not(IsFloat<type>.val) then { 634 def : RVVOutBuiltin<"Uv", "UvPCUe", type>; 635 } 636 } 637 } 638 } 639} 640 641multiclass RVVVLEFFBuiltin<list<string> types> { 642 let Name = NAME # "_v", 643 IRName = "vleff", 644 MaskedIRName = "vleff_mask", 645 SupportOverloading = false, 646 UnMaskedPolicyScheme = HasPassthruOperand, 647 ManualCodegen = [{ 648 { 649 if (IsMasked) { 650 // Move mask to right before vl. 651 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 652 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 653 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 654 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 655 IntrinsicTypes = {ResultType, Ops[4]->getType()}; 656 } else { 657 if (PolicyAttrs & RVV_VTA) 658 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 659 IntrinsicTypes = {ResultType, Ops[3]->getType()}; 660 } 661 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); 662 Value *NewVL = Ops[2]; 663 Ops.erase(Ops.begin() + 2); 664 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 665 llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); 666 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); 667 // Store new_vl. 668 clang::CharUnits Align; 669 if (IsMasked) 670 Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(E->getNumArgs()-2)->getType()); 671 else 672 Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); 673 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); 674 Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); 675 return V; 676 } 677 }] in { 678 foreach type = types in { 679 def : RVVBuiltin<"v", "vPCePz", type>; 680 // Skip floating types for unsigned versions. 681 if !not(IsFloat<type>.val) then { 682 def : RVVBuiltin<"Uv", "UvPCUePz", type>; 683 } 684 } 685 } 686} 687 688multiclass RVVVLSEBuiltin<list<string> types> { 689 let Name = NAME # "_v", 690 IRName = "vlse", 691 MaskedIRName ="vlse_mask", 692 SupportOverloading = false, 693 UnMaskedPolicyScheme = HasPassthruOperand in { 694 foreach type = types in { 695 def : RVVOutBuiltin<"v", "vPCet", type>; 696 if !not(IsFloat<type>.val) then { 697 def : RVVOutBuiltin<"Uv", "UvPCUet", type>; 698 } 699 } 700 } 701} 702 703multiclass RVVIndexedLoad<string op> { 704 let UnMaskedPolicyScheme = HasPassthruOperand in { 705 foreach type = TypeList in { 706 foreach eew_list = EEWList[0-2] in { 707 defvar eew = eew_list[0]; 708 defvar eew_type = eew_list[1]; 709 let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask" in { 710 def: RVVOutOp1Builtin<"v", "vPCe" # eew_type # "Uv", type>; 711 if !not(IsFloat<type>.val) then { 712 def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew_type # "Uv", type>; 713 } 714 } 715 } 716 defvar eew64 = "64"; 717 defvar eew64_type = "(Log2EEW:6)"; 718 let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", 719 RequiredFeatures = ["RV64"] in { 720 def: RVVOutOp1Builtin<"v", "vPCe" # eew64_type # "Uv", type>; 721 if !not(IsFloat<type>.val) then { 722 def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew64_type # "Uv", type>; 723 } 724 } 725 } 726 } 727} 728 729let HasMaskedOffOperand = false, 730 MaskedPolicyScheme = NonePolicy, 731 ManualCodegen = [{ 732 if (IsMasked) { 733 // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) 734 std::swap(Ops[0], Ops[2]); 735 } else { 736 // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) 737 std::swap(Ops[0], Ops[1]); 738 } 739 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 740 if (IsMasked) 741 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 742 else 743 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()}; 744 }] in { 745 class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { 746 let Name = "vsm_v"; 747 let IRName = "vsm"; 748 let HasMasked = false; 749 } 750 multiclass RVVVSEBuiltin<list<string> types> { 751 let Name = NAME # "_v", 752 IRName = "vse", 753 MaskedIRName = "vse_mask" in { 754 foreach type = types in { 755 def : RVVBuiltin<"v", "0Pev", type>; 756 if !not(IsFloat<type>.val) then { 757 def : RVVBuiltin<"Uv", "0PUeUv", type>; 758 } 759 } 760 } 761 } 762} 763 764multiclass RVVVSSEBuiltin<list<string> types> { 765 let Name = NAME # "_v", 766 IRName = "vsse", 767 MaskedIRName = "vsse_mask", 768 HasMaskedOffOperand = false, 769 MaskedPolicyScheme = NonePolicy, 770 ManualCodegen = [{ 771 if (IsMasked) { 772 // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) 773 std::swap(Ops[0], Ops[3]); 774 } else { 775 // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) 776 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 777 } 778 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 779 if (IsMasked) 780 IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()}; 781 else 782 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 783 }] in { 784 foreach type = types in { 785 def : RVVBuiltin<"v", "0Petv", type>; 786 if !not(IsFloat<type>.val) then { 787 def : RVVBuiltin<"Uv", "0PUetUv", type>; 788 } 789 } 790 } 791} 792 793multiclass RVVIndexedStore<string op> { 794 let HasMaskedOffOperand = false, 795 MaskedPolicyScheme = NonePolicy, 796 ManualCodegen = [{ 797 if (IsMasked) { 798 // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) 799 std::swap(Ops[0], Ops[3]); 800 } else { 801 // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) 802 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 803 } 804 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); 805 if (IsMasked) 806 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()}; 807 else 808 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()}; 809 }] in { 810 foreach type = TypeList in { 811 foreach eew_list = EEWList[0-2] in { 812 defvar eew = eew_list[0]; 813 defvar eew_type = eew_list[1]; 814 let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask" in { 815 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>; 816 if !not(IsFloat<type>.val) then { 817 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "UvUv", type>; 818 } 819 } 820 } 821 defvar eew64 = "64"; 822 defvar eew64_type = "(Log2EEW:6)"; 823 let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", 824 RequiredFeatures = ["RV64"] in { 825 def : RVVBuiltin<"v", "0Pe" # eew64_type # "Uvv", type>; 826 if !not(IsFloat<type>.val) then { 827 def : RVVBuiltin<"Uv", "0PUe" # eew64_type # "UvUv", type>; 828 } 829 } 830 } 831 } 832} 833 834defvar NFList = [2, 3, 4, 5, 6, 7, 8]; 835/* 836A segment load builtin has different variants. 837 838Therefore a segment unit-stride load builtin can have 4 variants, 8391. When unmasked and the policies are all specified as agnostic: 840(Address0, ..., Address{NF - 1}, Ptr, VL) 8412. When masked and the policies are all specified as agnostic: 842(Address0, ..., Address{NF - 1}, Mask, Ptr, VL) 8433. When unmasked and one of the policies is specified as undisturbed: 844(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 845 Ptr, VL) 8464. When masked and one of the policies is specified as undisturbed: 847(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 848 Ptr, VL) 849 850Other variants of segment load builtin share the same structure, but they 851have their own extra parameter. 852 853The segment unit-stride fault-only-first load builtin has a 'NewVL' 854operand after the 'Ptr' operand. 8551. When unmasked and the policies are all specified as agnostic: 856(Address0, ..., Address{NF - 1}, Ptr, NewVL, VL) 8572. When masked and the policies are all specified as agnostic: 858(Address0, ..., Address{NF - 1}, Mask, Ptr, NewVL, VL) 8593. When unmasked and one of the policies is specified as undisturbed: 860(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 861 Ptr, NewVL, VL) 8624. When masked and one of the policies is specified as undisturbed: 863(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 864 Ptr, NewVL, VL) 865 866The segment strided load builtin has a 'Stride' operand after the 'Ptr' 867operand. 8681. When unmasked and the policies are all specified as agnostic: 869(Address0, ..., Address{NF - 1}, Ptr, Stride, VL) 8702. When masked and the policies are all specified as agnostic: 871(Address0, ..., Address{NF - 1}, Mask, Ptr, Stride, VL) 8723. When unmasked and one of the policies is specified as undisturbed: 873(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 874 Ptr, Stride, VL) 8754. When masked and one of the policies is specified as undisturbed: 876(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 877 Ptr, Stride, VL) 878 879The segment indexed load builtin has a 'Idx' operand after the 'Ptr' operand. 8801. When unmasked and the policies are all specified as agnostic: 881(Address0, ..., Address{NF - 1}, Ptr, Idx, VL) 8822. When masked and the policies are all specified as agnostic: 883(Address0, ..., Address{NF - 1}, Mask, Ptr, Idx, VL) 8843. When unmasked and one of the policies is specified as undisturbed: 885(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 886 Ptr, Idx, VL) 8874. When masked and one of the policies is specified as undisturbed: 888(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 889 Ptr, Idx, VL) 890 891Segment load intrinsics has different variants similar to their builtins. 892 893Segment unit-stride load intrinsic, 894 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) 895 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) 896Segment unit-stride fault-only-first load intrinsic, 897 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) 898 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) 899Segment strided load intrinsic, 900 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, Mask, VL, Policy) 901 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, VL) 902Segment indexed load intrinsic, 903 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, Mask, VL, Policy) 904 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, VL) 905 906The Vector(s) is poison when the policy behavior allows us to not care 907about any masked-off elements. 908*/ 909 910class PVString<int nf, bit signed> { 911 string S = 912 !cond(!eq(nf, 2): !if(signed, "PvPv", "PUvPUv"), 913 !eq(nf, 3): !if(signed, "PvPvPv", "PUvPUvPUv"), 914 !eq(nf, 4): !if(signed, "PvPvPvPv", "PUvPUvPUvPUv"), 915 !eq(nf, 5): !if(signed, "PvPvPvPvPv", "PUvPUvPUvPUvPUv"), 916 !eq(nf, 6): !if(signed, "PvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUv"), 917 !eq(nf, 7): !if(signed, "PvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUv"), 918 !eq(nf, 8): !if(signed, "PvPvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUvPUv")); 919} 920 921multiclass RVVUnitStridedSegLoad<string op> { 922 foreach type = TypeList in { 923 defvar eew = !cond(!eq(type, "c") : "8", 924 !eq(type, "s") : "16", 925 !eq(type, "i") : "32", 926 !eq(type, "l") : "64", 927 !eq(type, "x") : "16", 928 !eq(type, "f") : "32", 929 !eq(type, "d") : "64"); 930 foreach nf = NFList in { 931 let Name = op # nf # "e" # eew # "_v", 932 IRName = op # nf, 933 MaskedIRName = op # nf # "_mask", 934 NF = nf, 935 ManualCodegen = [{ 936 { 937 ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); 938 IntrinsicTypes = {ResultType, Ops.back()->getType()}; 939 SmallVector<llvm::Value*, 12> Operands; 940 941 // Please refer to comment under 'defvar NFList' in this file 942 if ((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 943 (!IsMasked && PolicyAttrs & RVV_VTA)) 944 Operands.append(NF, llvm::PoisonValue::get(ResultType)); 945 else { 946 if (IsMasked) 947 Operands.append(Ops.begin() + NF + 1, Ops.begin() + 2 * NF + 1); 948 else // Unmasked 949 Operands.append(Ops.begin() + NF, Ops.begin() + 2 * NF); 950 } 951 unsigned PtrOperandIdx = IsMasked ? 952 ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ? NF + 1 : 2 * NF + 1 : 953 (PolicyAttrs & RVV_VTA) ? NF : 2 * NF; 954 Value *PtrOperand = Ops[PtrOperandIdx]; 955 Value *VLOperand = Ops[PtrOperandIdx + 1]; 956 Operands.push_back(PtrOperand); 957 if (IsMasked) 958 Operands.push_back(Ops[NF]); 959 Operands.push_back(VLOperand); 960 if (IsMasked) 961 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 962 963 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 964 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 965 clang::CharUnits Align = 966 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 967 llvm::Value *V; 968 for (unsigned I = 0; I < NF; ++I) { 969 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 970 V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 971 } 972 return V; 973 } 974 }] in { 975 defvar PV = PVString<nf, /*signed=*/true>.S; 976 defvar PUV = PVString<nf, /*signed=*/false>.S; 977 def : RVVBuiltin<"v", "0" # PV # "PCe", type>; 978 if !not(IsFloat<type>.val) then { 979 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe", type>; 980 } 981 } 982 } 983 } 984} 985 986multiclass RVVUnitStridedSegLoadFF<string op> { 987 foreach type = TypeList in { 988 defvar eew = !cond(!eq(type, "c") : "8", 989 !eq(type, "s") : "16", 990 !eq(type, "i") : "32", 991 !eq(type, "l") : "64", 992 !eq(type, "x") : "16", 993 !eq(type, "f") : "32", 994 !eq(type, "d") : "64"); 995 foreach nf = NFList in { 996 let Name = op # nf # "e" # eew # "ff_v", 997 IRName = op # nf # "ff", 998 MaskedIRName = op # nf # "ff_mask", 999 NF = nf, 1000 ManualCodegen = [{ 1001 { 1002 ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); 1003 IntrinsicTypes = {ResultType, Ops.back()->getType()}; 1004 SmallVector<llvm::Value*, 12> Operands; 1005 1006 // Please refer to comment under 'defvar NFList' in this file 1007 if ((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1008 (!IsMasked && PolicyAttrs & RVV_VTA)) 1009 Operands.append(NF, llvm::PoisonValue::get(ResultType)); 1010 else { 1011 if (IsMasked) 1012 Operands.append(Ops.begin() + NF + 1, Ops.begin() + 2 * NF + 1); 1013 else // Unmasked 1014 Operands.append(Ops.begin() + NF, Ops.begin() + 2 * NF); 1015 } 1016 unsigned PtrOperandIdx = IsMasked ? 1017 ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ? NF + 1 : 2 * NF + 1 : 1018 (PolicyAttrs & RVV_VTA) ? NF : 2 * NF; 1019 Value *PtrOperand = Ops[PtrOperandIdx]; 1020 Value *NewVLOperand = Ops[PtrOperandIdx + 1]; 1021 Value *VLOperand = Ops[PtrOperandIdx + 2]; 1022 Operands.push_back(PtrOperand); 1023 if (IsMasked) 1024 Operands.push_back(Ops[NF]); 1025 Operands.push_back(VLOperand); 1026 if (IsMasked) 1027 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1028 1029 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1030 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1031 clang::CharUnits Align = 1032 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1033 for (unsigned I = 0; I < NF; ++I) { 1034 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 1035 Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 1036 } 1037 // Store new_vl. 1038 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {NF}); 1039 return Builder.CreateStore(Val, Address(NewVLOperand, Val->getType(), Align)); 1040 } 1041 }] in { 1042 defvar PV = PVString<nf, /*signed=*/true>.S; 1043 defvar PUV = PVString<nf, /*signed=*/false>.S; 1044 def : RVVBuiltin<"v", "0" # PV # "PCe" # "Pz", type>; 1045 if !not(IsFloat<type>.val) then { 1046 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "Pz", type>; 1047 } 1048 } 1049 } 1050 } 1051} 1052 1053multiclass RVVStridedSegLoad<string op> { 1054 foreach type = TypeList in { 1055 defvar eew = !cond(!eq(type, "c") : "8", 1056 !eq(type, "s") : "16", 1057 !eq(type, "i") : "32", 1058 !eq(type, "l") : "64", 1059 !eq(type, "x") : "16", 1060 !eq(type, "f") : "32", 1061 !eq(type, "d") : "64"); 1062 foreach nf = NFList in { 1063 let Name = op # nf # "e" # eew # "_v", 1064 IRName = op # nf, 1065 MaskedIRName = op # nf # "_mask", 1066 NF = nf, 1067 ManualCodegen = [{ 1068 { 1069 ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); 1070 IntrinsicTypes = {ResultType, Ops.back()->getType()}; 1071 SmallVector<llvm::Value*, 12> Operands; 1072 1073 // Please refer to comment under 'defvar NFList' in this file 1074 if ((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1075 (!IsMasked && PolicyAttrs & RVV_VTA)) 1076 Operands.append(NF, llvm::PoisonValue::get(ResultType)); 1077 else { 1078 if (IsMasked) 1079 Operands.append(Ops.begin() + NF + 1, Ops.begin() + 2 * NF + 1); 1080 else // Unmasked 1081 Operands.append(Ops.begin() + NF, Ops.begin() + 2 * NF); 1082 } 1083 unsigned PtrOperandIdx = IsMasked ? 1084 ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ? NF + 1 : 2 * NF + 1 : 1085 (PolicyAttrs & RVV_VTA) ? NF : 2 * NF; 1086 Value *PtrOperand = Ops[PtrOperandIdx]; 1087 Value *StrideOperand = Ops[PtrOperandIdx + 1]; 1088 Value *VLOperand = Ops[PtrOperandIdx + 2]; 1089 Operands.push_back(PtrOperand); 1090 Operands.push_back(StrideOperand); 1091 if (IsMasked) 1092 Operands.push_back(Ops[NF]); 1093 Operands.push_back(VLOperand); 1094 if (IsMasked) 1095 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1096 1097 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1098 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1099 clang::CharUnits Align = 1100 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1101 llvm::Value *V; 1102 for (unsigned I = 0; I < NF; ++I) { 1103 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 1104 V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 1105 } 1106 return V; 1107 } 1108 }] in { 1109 defvar PV = PVString<nf, /*signed=*/true>.S; 1110 defvar PUV = PVString<nf, /*signed=*/false>.S; 1111 def : RVVBuiltin<"v", "0" # PV # "PCe" # "t", type>; 1112 if !not(IsFloat<type>.val) then { 1113 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "t", type>; 1114 } 1115 } 1116 } 1117 } 1118} 1119 1120multiclass RVVIndexedSegLoad<string op> { 1121 foreach type = TypeList in { 1122 foreach eew_info = EEWList in { 1123 defvar eew = eew_info[0]; 1124 defvar eew_type = eew_info[1]; 1125 foreach nf = NFList in { 1126 let Name = op # nf # "ei" # eew # "_v", 1127 IRName = op # nf, 1128 MaskedIRName = op # nf # "_mask", 1129 NF = nf, 1130 ManualCodegen = [{ 1131 { 1132 ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); 1133 SmallVector<llvm::Value*, 12> Operands; 1134 1135 // Please refer to comment under 'defvar NFList' in this file 1136 if ((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1137 (!IsMasked && PolicyAttrs & RVV_VTA)) 1138 Operands.append(NF, llvm::PoisonValue::get(ResultType)); 1139 else { 1140 if (IsMasked) 1141 Operands.append(Ops.begin() + NF + 1, Ops.begin() + 2 * NF + 1); 1142 else // Unmasked 1143 Operands.append(Ops.begin() + NF, Ops.begin() + 2 * NF); 1144 } 1145 unsigned PtrOperandIdx = IsMasked ? 1146 ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ? NF + 1 : 2 * NF + 1 : 1147 (PolicyAttrs & RVV_VTA) ? NF : 2 * NF; 1148 Value *PtrOperand = Ops[PtrOperandIdx]; 1149 Value *IndexOperand = Ops[PtrOperandIdx + 1]; 1150 Value *VLOperand = Ops[PtrOperandIdx + 2]; 1151 Operands.push_back(PtrOperand); 1152 Operands.push_back(IndexOperand); 1153 if (IsMasked) 1154 Operands.push_back(Ops[NF]); 1155 Operands.push_back(VLOperand); 1156 if (IsMasked) 1157 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1158 IntrinsicTypes = {ResultType, IndexOperand->getType(), Ops.back()->getType()}; 1159 1160 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1161 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1162 clang::CharUnits Align = 1163 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); 1164 llvm::Value *V; 1165 for (unsigned I = 0; I < NF; ++I) { 1166 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); 1167 V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); 1168 } 1169 return V; 1170 } 1171 }] in { 1172 defvar PV = PVString<nf, /*signed=*/true>.S; 1173 defvar PUV = PVString<nf, /*signed=*/false>.S; 1174 def : RVVBuiltin<"v", "0" # PV # "PCe" # eew_type # "Uv", type>; 1175 if !not(IsFloat<type>.val) then { 1176 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # eew_type # "Uv", type>; 1177 } 1178 } 1179 } 1180 } 1181 } 1182} 1183 1184class VString<int nf, bit signed> { 1185 string S = !cond(!eq(nf, 2): !if(signed, "vv", "UvUv"), 1186 !eq(nf, 3): !if(signed, "vvv", "UvUvUv"), 1187 !eq(nf, 4): !if(signed, "vvvv", "UvUvUvUv"), 1188 !eq(nf, 5): !if(signed, "vvvvv", "UvUvUvUvUv"), 1189 !eq(nf, 6): !if(signed, "vvvvvv", "UvUvUvUvUvUv"), 1190 !eq(nf, 7): !if(signed, "vvvvvvv", "UvUvUvUvUvUvUv"), 1191 !eq(nf, 8): !if(signed, "vvvvvvvv", "UvUvUvUvUvUvUvUv")); 1192} 1193 1194multiclass RVVUnitStridedSegStore<string op> { 1195 foreach type = TypeList in { 1196 defvar eew = !cond(!eq(type, "c") : "8", 1197 !eq(type, "s") : "16", 1198 !eq(type, "i") : "32", 1199 !eq(type, "l") : "64", 1200 !eq(type, "x") : "16", 1201 !eq(type, "f") : "32", 1202 !eq(type, "d") : "64"); 1203 foreach nf = NFList in { 1204 let Name = op # nf # "e" # eew # "_v", 1205 IRName = op # nf, 1206 MaskedIRName = op # nf # "_mask", 1207 NF = nf, 1208 HasMaskedOffOperand = false, 1209 MaskedPolicyScheme = NonePolicy, 1210 ManualCodegen = [{ 1211 { 1212 if (IsMasked) { 1213 // Builtin: (mask, ptr, val0, val1, ..., vl) 1214 // Intrinsic: (val0, val1, ..., ptr, mask, vl) 1215 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); 1216 std::swap(Ops[NF], Ops[NF + 1]); 1217 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 2]->getType()}; 1218 assert(Ops.size() == NF + 3); 1219 } else { 1220 // Builtin: (ptr, val0, val1, ..., vl) 1221 // Intrinsic: (val0, val1, ..., ptr, vl) 1222 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1223 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; 1224 assert(Ops.size() == NF + 2); 1225 } 1226 } 1227 }] in { 1228 defvar V = VString<nf, /*signed=*/true>.S; 1229 defvar UV = VString<nf, /*signed=*/false>.S; 1230 def : RVVBuiltin<"v", "0Pe" # V, type>; 1231 if !not(IsFloat<type>.val) then { 1232 def : RVVBuiltin<"Uv", "0PUe" # UV, type>; 1233 } 1234 } 1235 } 1236 } 1237} 1238 1239multiclass RVVStridedSegStore<string op> { 1240 foreach type = TypeList in { 1241 defvar eew = !cond(!eq(type, "c") : "8", 1242 !eq(type, "s") : "16", 1243 !eq(type, "i") : "32", 1244 !eq(type, "l") : "64", 1245 !eq(type, "x") : "16", 1246 !eq(type, "f") : "32", 1247 !eq(type, "d") : "64"); 1248 foreach nf = NFList in { 1249 let Name = op # nf # "e" # eew # "_v", 1250 IRName = op # nf, 1251 MaskedIRName = op # nf # "_mask", 1252 NF = nf, 1253 HasMaskedOffOperand = false, 1254 MaskedPolicyScheme = NonePolicy, 1255 ManualCodegen = [{ 1256 { 1257 if (IsMasked) { 1258 // Builtin: (mask, ptr, stride, val0, val1, ..., vl). 1259 // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl) 1260 std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); 1261 std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); 1262 assert(Ops.size() == NF + 4); 1263 } else { 1264 // Builtin: (ptr, stride, val0, val1, ..., vl). 1265 // Intrinsic: (val0, val1, ..., ptr, stride, vl) 1266 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); 1267 assert(Ops.size() == NF + 3); 1268 } 1269 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; 1270 } 1271 }] in { 1272 defvar V = VString<nf, /*signed=*/true>.S; 1273 defvar UV = VString<nf, /*signed=*/false>.S; 1274 def : RVVBuiltin<"v", "0Pet" # V, type>; 1275 if !not(IsFloat<type>.val) then { 1276 def : RVVBuiltin<"Uv", "0PUet" # UV, type>; 1277 } 1278 } 1279 } 1280 } 1281} 1282 1283multiclass RVVIndexedSegStore<string op> { 1284 foreach type = TypeList in { 1285 foreach eew_info = EEWList in { 1286 defvar eew = eew_info[0]; 1287 defvar eew_type = eew_info[1]; 1288 foreach nf = NFList in { 1289 let Name = op # nf # "ei" # eew # "_v", 1290 IRName = op # nf, 1291 MaskedIRName = op # nf # "_mask", 1292 NF = nf, 1293 HasMaskedOffOperand = false, 1294 MaskedPolicyScheme = NonePolicy, 1295 ManualCodegen = [{ 1296 { 1297 if (IsMasked) { 1298 // Builtin: (mask, ptr, index, val0, val1, ..., vl) 1299 // Intrinsic: (val0, val1, ..., ptr, index, mask, vl) 1300 std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); 1301 std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); 1302 IntrinsicTypes = {Ops[0]->getType(), 1303 Ops[NF + 1]->getType(), Ops[NF + 3]->getType()}; 1304 assert(Ops.size() == NF + 4); 1305 } else { 1306 // Builtin: (ptr, index, val0, val1, ..., vl) 1307 // Intrinsic: (val0, val1, ..., ptr, index, vl) 1308 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); 1309 IntrinsicTypes = {Ops[0]->getType(), 1310 Ops[NF + 1]->getType(), Ops[NF + 2]->getType()}; 1311 assert(Ops.size() == NF + 3); 1312 } 1313 } 1314 }] in { 1315 defvar V = VString<nf, /*signed=*/true>.S; 1316 defvar UV = VString<nf, /*signed=*/false>.S; 1317 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uv" # V, type>; 1318 if !not(IsFloat<type>.val) then { 1319 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "Uv" # UV, type>; 1320 } 1321 } 1322 } 1323 } 1324 } 1325} 1326 1327multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { 1328 let Name = NAME, 1329 IRName = IR, 1330 MaskedIRName = IR # "_mask", 1331 UnMaskedPolicyScheme = HasPassthruOperand, 1332 ManualCodegen = [{ 1333 { 1334 if (IsMasked) { 1335 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1336 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 1337 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1338 } else { 1339 if (PolicyAttrs & RVV_VTA) 1340 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1341 } 1342 auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); 1343 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); 1344 1345 if (IsMasked) { 1346 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1347 // maskedoff, op1, op2, mask, vl, policy 1348 IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; 1349 } else { 1350 // passthru, op1, op2, vl 1351 IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; 1352 } 1353 break; 1354 } 1355 }] in { 1356 def : RVVBuiltin<"v", "vv", type_range>; 1357 } 1358} 1359 1360multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { 1361 let Name = NAME, 1362 IRName = IR, 1363 MaskedIRName = IR # "_mask", 1364 UnMaskedPolicyScheme = HasPassthruOperand, 1365 ManualCodegen = [{ 1366 { 1367 if (IsMasked) { 1368 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1369 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 1370 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1371 } else { 1372 if (PolicyAttrs & RVV_VTA) 1373 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1374 } 1375 auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); 1376 Ops.insert(Ops.begin() + 2, 1377 llvm::Constant::getAllOnesValue(ElemTy)); 1378 if (IsMasked) { 1379 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1380 // maskedoff, op1, po2, mask, vl, policy 1381 IntrinsicTypes = {ResultType, 1382 ElemTy, 1383 Ops[4]->getType()}; 1384 } else { 1385 // passthru, op1, op2, vl 1386 IntrinsicTypes = {ResultType, 1387 ElemTy, 1388 Ops[3]->getType()}; 1389 } 1390 break; 1391 } 1392 }] in { 1393 def : RVVBuiltin<"v", "vv", type_range>; 1394 def : RVVBuiltin<"Uv", "UvUv", type_range>; 1395 } 1396} 1397 1398multiclass RVVPseudoMaskBuiltin<string IR, string type_range> { 1399 let Name = NAME, 1400 IRName = IR, 1401 HasMasked = false, 1402 ManualCodegen = [{ 1403 { 1404 // op1, vl 1405 IntrinsicTypes = {ResultType, 1406 Ops[1]->getType()}; 1407 Ops.insert(Ops.begin() + 1, Ops[0]); 1408 break; 1409 } 1410 }] in { 1411 def : RVVBuiltin<"m", "mm", type_range>; 1412 } 1413} 1414 1415multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { 1416 let Name = NAME, 1417 IRName = IR, 1418 MaskedIRName = IR # "_mask", 1419 UnMaskedPolicyScheme = HasPassthruOperand, 1420 ManualCodegen = [{ 1421 { 1422 if (IsMasked) { 1423 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1424 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 1425 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1426 Ops.insert(Ops.begin() + 2, Ops[1]); 1427 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1428 // maskedoff, op1, op2, mask, vl 1429 IntrinsicTypes = {ResultType, 1430 Ops[2]->getType(), 1431 Ops.back()->getType()}; 1432 } else { 1433 if (PolicyAttrs & RVV_VTA) 1434 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1435 // op1, po2, vl 1436 IntrinsicTypes = {ResultType, 1437 Ops[1]->getType(), Ops[2]->getType()}; 1438 Ops.insert(Ops.begin() + 2, Ops[1]); 1439 break; 1440 } 1441 break; 1442 } 1443 }] in { 1444 def : RVVBuiltin<"v", "vv", type_range>; 1445 } 1446} 1447 1448multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, 1449 list<list<string>> suffixes_prototypes> { 1450 let Name = NAME, 1451 OverloadedName = MName, 1452 IRName = IR, 1453 MaskedIRName = IR # "_mask", 1454 UnMaskedPolicyScheme = HasPassthruOperand, 1455 ManualCodegen = [{ 1456 { 1457 if (IsMasked) { 1458 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1459 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 1460 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1461 } else { 1462 if (PolicyAttrs & RVV_VTA) 1463 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1464 } 1465 auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); 1466 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); 1467 if (IsMasked) { 1468 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1469 // maskedoff, op1, op2, mask, vl, policy 1470 IntrinsicTypes = {ResultType, 1471 Ops[1]->getType(), 1472 ElemTy, 1473 Ops[4]->getType()}; 1474 } else { 1475 // passtru, op1, op2, vl 1476 IntrinsicTypes = {ResultType, 1477 Ops[1]->getType(), 1478 ElemTy, 1479 Ops[3]->getType()}; 1480 } 1481 break; 1482 } 1483 }] in { 1484 foreach s_p = suffixes_prototypes in { 1485 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 1486 } 1487 } 1488} 1489 1490multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, 1491 list<list<string>> suffixes_prototypes> { 1492 let Name = NAME, 1493 OverloadedName = MName, 1494 IRName = IR, 1495 MaskedIRName = IR # "_mask", 1496 UnMaskedPolicyScheme = HasPassthruOperand, 1497 ManualCodegen = [{ 1498 { 1499 if (IsMasked) { 1500 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 1501 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 1502 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1503 } else { 1504 if (PolicyAttrs & RVV_VTA) 1505 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1506 } 1507 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType())); 1508 if (IsMasked) { 1509 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1510 // maskedoff, op1, xlen, mask, vl 1511 IntrinsicTypes = {ResultType, 1512 Ops[1]->getType(), 1513 Ops[4]->getType(), 1514 Ops[4]->getType()}; 1515 } else { 1516 // passthru, op1, xlen, vl 1517 IntrinsicTypes = {ResultType, 1518 Ops[1]->getType(), 1519 Ops[3]->getType(), 1520 Ops[3]->getType()}; 1521 } 1522 break; 1523 } 1524 }] in { 1525 foreach s_p = suffixes_prototypes in { 1526 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 1527 } 1528 } 1529} 1530 1531// Define vread_csr&vwrite_csr described in RVV intrinsics doc. 1532let HeaderCode = 1533[{ 1534enum RVV_CSR { 1535 RVV_VSTART = 0, 1536 RVV_VXSAT, 1537 RVV_VXRM, 1538 RVV_VCSR, 1539}; 1540 1541static __inline__ __attribute__((__always_inline__, __nodebug__)) 1542unsigned long __riscv_vread_csr(enum RVV_CSR __csr) { 1543 unsigned long __rv = 0; 1544 switch (__csr) { 1545 case RVV_VSTART: 1546 __asm__ __volatile__ ("csrr\t%0, vstart" : "=r"(__rv) : : "memory"); 1547 break; 1548 case RVV_VXSAT: 1549 __asm__ __volatile__ ("csrr\t%0, vxsat" : "=r"(__rv) : : "memory"); 1550 break; 1551 case RVV_VXRM: 1552 __asm__ __volatile__ ("csrr\t%0, vxrm" : "=r"(__rv) : : "memory"); 1553 break; 1554 case RVV_VCSR: 1555 __asm__ __volatile__ ("csrr\t%0, vcsr" : "=r"(__rv) : : "memory"); 1556 break; 1557 } 1558 return __rv; 1559} 1560 1561static __inline__ __attribute__((__always_inline__, __nodebug__)) 1562void __riscv_vwrite_csr(enum RVV_CSR __csr, unsigned long __value) { 1563 switch (__csr) { 1564 case RVV_VSTART: 1565 __asm__ __volatile__ ("csrw\tvstart, %z0" : : "rJ"(__value) : "memory"); 1566 break; 1567 case RVV_VXSAT: 1568 __asm__ __volatile__ ("csrw\tvxsat, %z0" : : "rJ"(__value) : "memory"); 1569 break; 1570 case RVV_VXRM: 1571 __asm__ __volatile__ ("csrw\tvxrm, %z0" : : "rJ"(__value) : "memory"); 1572 break; 1573 case RVV_VCSR: 1574 __asm__ __volatile__ ("csrw\tvcsr, %z0" : : "rJ"(__value) : "memory"); 1575 break; 1576 } 1577} 1578}] in 1579def vread_vwrite_csr: RVVHeader; 1580 1581let HeaderCode = 1582[{ 1583#define __riscv_vlenb() __builtin_rvv_vlenb() 1584}] in 1585def vlenb_macro: RVVHeader; 1586 1587let HasBuiltinAlias = false, HasVL = false, HasMasked = false, 1588 UnMaskedPolicyScheme = NonePolicy, MaskedPolicyScheme = NonePolicy, 1589 Log2LMUL = [0], IRName = "", 1590 ManualCodegen = [{ 1591 { 1592 LLVMContext &Context = CGM.getLLVMContext(); 1593 llvm::MDBuilder MDHelper(Context); 1594 1595 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "vlenb")}; 1596 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); 1597 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); 1598 llvm::Function *F = 1599 CGM.getIntrinsic(llvm::Intrinsic::read_register, {SizeTy}); 1600 return Builder.CreateCall(F, Metadata); 1601 } 1602 }] in 1603{ 1604 def vlenb : RVVBuiltin<"", "u", "i">; 1605} 1606 1607// 6. Configuration-Setting Instructions 1608// 6.1. vsetvli/vsetvl instructions 1609 1610// vsetvl/vsetvlmax are a macro because they require constant integers in SEW 1611// and LMUL. 1612let HeaderCode = 1613[{ 1614#define __riscv_vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6) 1615#define __riscv_vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7) 1616#define __riscv_vsetvl_e8m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 0) 1617#define __riscv_vsetvl_e8m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 1) 1618#define __riscv_vsetvl_e8m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 2) 1619#define __riscv_vsetvl_e8m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 3) 1620 1621#define __riscv_vsetvl_e16mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 7) 1622#define __riscv_vsetvl_e16m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 0) 1623#define __riscv_vsetvl_e16m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 1) 1624#define __riscv_vsetvl_e16m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 2) 1625#define __riscv_vsetvl_e16m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 3) 1626 1627#define __riscv_vsetvl_e32m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 0) 1628#define __riscv_vsetvl_e32m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 1) 1629#define __riscv_vsetvl_e32m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 2) 1630#define __riscv_vsetvl_e32m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 3) 1631 1632#if __riscv_v_elen >= 64 1633#define __riscv_vsetvl_e8mf8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 5) 1634#define __riscv_vsetvl_e16mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 6) 1635#define __riscv_vsetvl_e32mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 7) 1636 1637#define __riscv_vsetvl_e64m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 0) 1638#define __riscv_vsetvl_e64m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 1) 1639#define __riscv_vsetvl_e64m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 2) 1640#define __riscv_vsetvl_e64m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 3) 1641#endif 1642 1643#define __riscv_vsetvlmax_e8mf4() __builtin_rvv_vsetvlimax(0, 6) 1644#define __riscv_vsetvlmax_e8mf2() __builtin_rvv_vsetvlimax(0, 7) 1645#define __riscv_vsetvlmax_e8m1() __builtin_rvv_vsetvlimax(0, 0) 1646#define __riscv_vsetvlmax_e8m2() __builtin_rvv_vsetvlimax(0, 1) 1647#define __riscv_vsetvlmax_e8m4() __builtin_rvv_vsetvlimax(0, 2) 1648#define __riscv_vsetvlmax_e8m8() __builtin_rvv_vsetvlimax(0, 3) 1649 1650#define __riscv_vsetvlmax_e16mf2() __builtin_rvv_vsetvlimax(1, 7) 1651#define __riscv_vsetvlmax_e16m1() __builtin_rvv_vsetvlimax(1, 0) 1652#define __riscv_vsetvlmax_e16m2() __builtin_rvv_vsetvlimax(1, 1) 1653#define __riscv_vsetvlmax_e16m4() __builtin_rvv_vsetvlimax(1, 2) 1654#define __riscv_vsetvlmax_e16m8() __builtin_rvv_vsetvlimax(1, 3) 1655 1656#define __riscv_vsetvlmax_e32m1() __builtin_rvv_vsetvlimax(2, 0) 1657#define __riscv_vsetvlmax_e32m2() __builtin_rvv_vsetvlimax(2, 1) 1658#define __riscv_vsetvlmax_e32m4() __builtin_rvv_vsetvlimax(2, 2) 1659#define __riscv_vsetvlmax_e32m8() __builtin_rvv_vsetvlimax(2, 3) 1660 1661#if __riscv_v_elen >= 64 1662#define __riscv_vsetvlmax_e8mf8() __builtin_rvv_vsetvlimax(0, 5) 1663#define __riscv_vsetvlmax_e16mf4() __builtin_rvv_vsetvlimax(1, 6) 1664#define __riscv_vsetvlmax_e32mf2() __builtin_rvv_vsetvlimax(2, 7) 1665 1666#define __riscv_vsetvlmax_e64m1() __builtin_rvv_vsetvlimax(3, 0) 1667#define __riscv_vsetvlmax_e64m2() __builtin_rvv_vsetvlimax(3, 1) 1668#define __riscv_vsetvlmax_e64m4() __builtin_rvv_vsetvlimax(3, 2) 1669#define __riscv_vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3) 1670#endif 1671 1672}] in 1673def vsetvl_macro: RVVHeader; 1674 1675let HasBuiltinAlias = false, 1676 HasVL = false, 1677 HasMasked = false, 1678 MaskedPolicyScheme = NonePolicy, 1679 Log2LMUL = [0], 1680 ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type 1681{ 1682 def vsetvli : RVVBuiltin<"", "zzKzKz", "i">; 1683 def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">; 1684} 1685 1686// 7. Vector Loads and Stores 1687// 7.4. Vector Unit-Stride Instructions 1688def vlm: RVVVLEMaskBuiltin; 1689defm vle8: RVVVLEBuiltin<["c"]>; 1690defm vle16: RVVVLEBuiltin<["s","x"]>; 1691defm vle32: RVVVLEBuiltin<["i","f"]>; 1692defm vle64: RVVVLEBuiltin<["l","d"]>; 1693 1694def vsm : RVVVSEMaskBuiltin; 1695defm vse8 : RVVVSEBuiltin<["c"]>; 1696defm vse16: RVVVSEBuiltin<["s","x"]>; 1697defm vse32: RVVVSEBuiltin<["i","f"]>; 1698defm vse64: RVVVSEBuiltin<["l","d"]>; 1699 1700// 7.5. Vector Strided Instructions 1701defm vlse8: RVVVLSEBuiltin<["c"]>; 1702defm vlse16: RVVVLSEBuiltin<["s","x"]>; 1703defm vlse32: RVVVLSEBuiltin<["i","f"]>; 1704defm vlse64: RVVVLSEBuiltin<["l","d"]>; 1705 1706defm vsse8 : RVVVSSEBuiltin<["c"]>; 1707defm vsse16: RVVVSSEBuiltin<["s","x"]>; 1708defm vsse32: RVVVSSEBuiltin<["i","f"]>; 1709defm vsse64: RVVVSSEBuiltin<["l","d"]>; 1710 1711// 7.6. Vector Indexed Instructions 1712defm : RVVIndexedLoad<"vluxei">; 1713defm : RVVIndexedLoad<"vloxei">; 1714 1715defm : RVVIndexedStore<"vsuxei">; 1716defm : RVVIndexedStore<"vsoxei">; 1717 1718// 7.7. Unit-stride Fault-Only-First Loads 1719defm vle8ff: RVVVLEFFBuiltin<["c"]>; 1720defm vle16ff: RVVVLEFFBuiltin<["s","x"]>; 1721defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>; 1722defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>; 1723 1724// 7.8 Vector Load/Store Segment Instructions 1725let UnMaskedPolicyScheme = HasPassthruOperand in { 1726defm : RVVUnitStridedSegLoad<"vlseg">; 1727defm : RVVUnitStridedSegLoadFF<"vlseg">; 1728defm : RVVStridedSegLoad<"vlsseg">; 1729defm : RVVIndexedSegLoad<"vluxseg">; 1730defm : RVVIndexedSegLoad<"vloxseg">; 1731} 1732let UnMaskedPolicyScheme = NonePolicy, 1733 MaskedPolicyScheme = NonePolicy in { 1734defm : RVVUnitStridedSegStore<"vsseg">; 1735defm : RVVStridedSegStore<"vssseg">; 1736defm : RVVIndexedSegStore<"vsuxseg">; 1737defm : RVVIndexedSegStore<"vsoxseg">; 1738} 1739 1740// 12. Vector Integer Arithmetic Instructions 1741// 12.1. Vector Single-Width Integer Add and Subtract 1742let UnMaskedPolicyScheme = HasPassthruOperand in { 1743defm vadd : RVVIntBinBuiltinSet; 1744defm vsub : RVVIntBinBuiltinSet; 1745defm vrsub : RVVOutOp1BuiltinSet<"vrsub", "csil", 1746 [["vx", "v", "vve"], 1747 ["vx", "Uv", "UvUvUe"]]>; 1748} 1749defm vneg_v : RVVPseudoUnaryBuiltin<"vrsub", "csil">; 1750 1751// 12.2. Vector Widening Integer Add/Subtract 1752// Widening unsigned integer add/subtract, 2*SEW = SEW +/- SEW 1753let UnMaskedPolicyScheme = HasPassthruOperand in { 1754defm vwaddu : RVVUnsignedWidenBinBuiltinSet; 1755defm vwsubu : RVVUnsignedWidenBinBuiltinSet; 1756// Widening signed integer add/subtract, 2*SEW = SEW +/- SEW 1757defm vwadd : RVVSignedWidenBinBuiltinSet; 1758defm vwsub : RVVSignedWidenBinBuiltinSet; 1759// Widening unsigned integer add/subtract, 2*SEW = 2*SEW +/- SEW 1760defm vwaddu : RVVUnsignedWidenOp0BinBuiltinSet; 1761defm vwsubu : RVVUnsignedWidenOp0BinBuiltinSet; 1762// Widening signed integer add/subtract, 2*SEW = 2*SEW +/- SEW 1763defm vwadd : RVVSignedWidenOp0BinBuiltinSet; 1764defm vwsub : RVVSignedWidenOp0BinBuiltinSet; 1765} 1766defm vwcvtu_x_x_v : RVVPseudoVWCVTBuiltin<"vwaddu", "vwcvtu_x", "csi", 1767 [["Uw", "UwUv"]]>; 1768defm vwcvt_x_x_v : RVVPseudoVWCVTBuiltin<"vwadd", "vwcvt_x", "csi", 1769 [["w", "wv"]]>; 1770 1771// 12.3. Vector Integer Extension 1772let UnMaskedPolicyScheme = HasPassthruOperand in { 1773let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1774 def vsext_vf2 : RVVIntExt<"vsext", "w", "wv", "csi">; 1775 def vzext_vf2 : RVVIntExt<"vzext", "Uw", "UwUv", "csi">; 1776} 1777let Log2LMUL = [-3, -2, -1, 0, 1] in { 1778 def vsext_vf4 : RVVIntExt<"vsext", "q", "qv", "cs">; 1779 def vzext_vf4 : RVVIntExt<"vzext", "Uq", "UqUv", "cs">; 1780} 1781let Log2LMUL = [-3, -2, -1, 0] in { 1782 def vsext_vf8 : RVVIntExt<"vsext", "o", "ov", "c">; 1783 def vzext_vf8 : RVVIntExt<"vzext", "Uo", "UoUv", "c">; 1784} 1785} 1786 1787// 12.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions 1788let HasMasked = false, MaskedPolicyScheme = NonePolicy in { 1789 let UnMaskedPolicyScheme = HasPassthruOperand in { 1790 defm vadc : RVVCarryinBuiltinSet; 1791 defm vsbc : RVVCarryinBuiltinSet; 1792 } 1793 defm vmadc : RVVCarryOutInBuiltinSet<"vmadc_carry_in">; 1794 defm vmadc : RVVIntMaskOutBuiltinSet; 1795 defm vmsbc : RVVCarryOutInBuiltinSet<"vmsbc_borrow_in">; 1796 defm vmsbc : RVVIntMaskOutBuiltinSet; 1797} 1798 1799// 12.5. Vector Bitwise Logical Instructions 1800let UnMaskedPolicyScheme = HasPassthruOperand in { 1801defm vand : RVVIntBinBuiltinSet; 1802defm vxor : RVVIntBinBuiltinSet; 1803defm vor : RVVIntBinBuiltinSet; 1804} 1805defm vnot_v : RVVPseudoVNotBuiltin<"vxor", "csil">; 1806 1807// 12.6. Vector Single-Width Bit Shift Instructions 1808let UnMaskedPolicyScheme = HasPassthruOperand in { 1809defm vsll : RVVShiftBuiltinSet; 1810defm vsrl : RVVUnsignedShiftBuiltinSet; 1811defm vsra : RVVSignedShiftBuiltinSet; 1812 1813// 12.7. Vector Narrowing Integer Right Shift Instructions 1814defm vnsrl : RVVUnsignedNShiftBuiltinSet; 1815defm vnsra : RVVSignedNShiftBuiltinSet; 1816} 1817defm vncvt_x_x_w : RVVPseudoVNCVTBuiltin<"vnsrl", "vncvt_x", "csi", 1818 [["v", "vw"], 1819 ["Uv", "UvUw"]]>; 1820 1821// 12.8. Vector Integer Comparison Instructions 1822let MaskedPolicyScheme = HasPassthruOperand, 1823 HasTailPolicy = false in { 1824defm vmseq : RVVIntMaskOutBuiltinSet; 1825defm vmsne : RVVIntMaskOutBuiltinSet; 1826defm vmsltu : RVVUnsignedMaskOutBuiltinSet; 1827defm vmslt : RVVSignedMaskOutBuiltinSet; 1828defm vmsleu : RVVUnsignedMaskOutBuiltinSet; 1829defm vmsle : RVVSignedMaskOutBuiltinSet; 1830defm vmsgtu : RVVUnsignedMaskOutBuiltinSet; 1831defm vmsgt : RVVSignedMaskOutBuiltinSet; 1832defm vmsgeu : RVVUnsignedMaskOutBuiltinSet; 1833defm vmsge : RVVSignedMaskOutBuiltinSet; 1834} 1835 1836// 12.9. Vector Integer Min/Max Instructions 1837let UnMaskedPolicyScheme = HasPassthruOperand in { 1838defm vminu : RVVUnsignedBinBuiltinSet; 1839defm vmin : RVVSignedBinBuiltinSet; 1840defm vmaxu : RVVUnsignedBinBuiltinSet; 1841defm vmax : RVVSignedBinBuiltinSet; 1842 1843// 12.10. Vector Single-Width Integer Multiply Instructions 1844defm vmul : RVVIntBinBuiltinSet; 1845let RequiredFeatures = ["FullMultiply"] in { 1846defm vmulh : RVVSignedBinBuiltinSet; 1847defm vmulhu : RVVUnsignedBinBuiltinSet; 1848defm vmulhsu : RVVOutOp1BuiltinSet<"vmulhsu", "csil", 1849 [["vv", "v", "vvUv"], 1850 ["vx", "v", "vvUe"]]>; 1851} 1852 1853// 12.11. Vector Integer Divide Instructions 1854defm vdivu : RVVUnsignedBinBuiltinSet; 1855defm vdiv : RVVSignedBinBuiltinSet; 1856defm vremu : RVVUnsignedBinBuiltinSet; 1857defm vrem : RVVSignedBinBuiltinSet; 1858} 1859 1860// 12.12. Vector Widening Integer Multiply Instructions 1861let Log2LMUL = [-3, -2, -1, 0, 1, 2], UnMaskedPolicyScheme = HasPassthruOperand in { 1862defm vwmul : RVVOutOp0Op1BuiltinSet<"vwmul", "csi", 1863 [["vv", "w", "wvv"], 1864 ["vx", "w", "wve"]]>; 1865defm vwmulu : RVVOutOp0Op1BuiltinSet<"vwmulu", "csi", 1866 [["vv", "Uw", "UwUvUv"], 1867 ["vx", "Uw", "UwUvUe"]]>; 1868defm vwmulsu : RVVOutOp0Op1BuiltinSet<"vwmulsu", "csi", 1869 [["vv", "w", "wvUv"], 1870 ["vx", "w", "wvUe"]]>; 1871} 1872 1873// 12.13. Vector Single-Width Integer Multiply-Add Instructions 1874let UnMaskedPolicyScheme = HasPolicyOperand in { 1875defm vmacc : RVVIntTerBuiltinSet; 1876defm vnmsac : RVVIntTerBuiltinSet; 1877defm vmadd : RVVIntTerBuiltinSet; 1878defm vnmsub : RVVIntTerBuiltinSet; 1879 1880// 12.14. Vector Widening Integer Multiply-Add Instructions 1881let HasMaskedOffOperand = false, 1882 Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1883defm vwmaccu : RVVOutOp1Op2BuiltinSet<"vwmaccu", "csi", 1884 [["vv", "Uw", "UwUwUvUv"], 1885 ["vx", "Uw", "UwUwUeUv"]]>; 1886defm vwmacc : RVVOutOp1Op2BuiltinSet<"vwmacc", "csi", 1887 [["vv", "w", "wwvv"], 1888 ["vx", "w", "wwev"]]>; 1889defm vwmaccsu : RVVOutOp1Op2BuiltinSet<"vwmaccsu", "csi", 1890 [["vv", "w", "wwvUv"], 1891 ["vx", "w", "wweUv"]]>; 1892defm vwmaccus : RVVOutOp1Op2BuiltinSet<"vwmaccus", "csi", 1893 [["vx", "w", "wwUev"]]>; 1894} 1895} 1896 1897// 12.15. Vector Integer Merge Instructions 1898// C/C++ Operand: (mask, op1, op2, vl), Intrinsic: (passthru, op1, op2, mask, vl) 1899let HasMasked = false, 1900 UnMaskedPolicyScheme = HasPassthruOperand, 1901 MaskedPolicyScheme = NonePolicy, 1902 ManualCodegen = [{ 1903 // insert poison passthru 1904 if (PolicyAttrs & RVV_VTA) 1905 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1906 IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; 1907 }] in { 1908 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "csil", 1909 [["vvm", "v", "vvvm"], 1910 ["vxm", "v", "vvem"], 1911 ["vvm", "Uv", "UvUvUvm"], 1912 ["vxm", "Uv", "UvUvUem"]]>; 1913} 1914 1915// 12.16. Vector Integer Move Instructions 1916let HasMasked = false, 1917 UnMaskedPolicyScheme = HasPassthruOperand, 1918 MaskedPolicyScheme = NonePolicy, 1919 OverloadedName = "vmv_v" in { 1920 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csil", 1921 [["v", "Uv", "UvUv"]]>; 1922 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilxfd", 1923 [["v", "v", "vv"]]>; 1924 let SupportOverloading = false in 1925 defm vmv_v : RVVOutBuiltinSet<"vmv_v_x", "csil", 1926 [["x", "v", "ve"], 1927 ["x", "Uv", "UvUe"]]>; 1928} 1929 1930// 13. Vector Fixed-Point Arithmetic Instructions 1931// 13.1. Vector Single-Width Saturating Add and Subtract 1932let UnMaskedPolicyScheme = HasPassthruOperand in { 1933defm vsaddu : RVVUnsignedBinBuiltinSet; 1934defm vsadd : RVVSignedBinBuiltinSet; 1935defm vssubu : RVVUnsignedBinBuiltinSet; 1936defm vssub : RVVSignedBinBuiltinSet; 1937 1938// 13.2. Vector Single-Width Averaging Add and Subtract 1939defm vaaddu : RVVUnsignedBinBuiltinSet; 1940defm vaadd : RVVSignedBinBuiltinSet; 1941defm vasubu : RVVUnsignedBinBuiltinSet; 1942defm vasub : RVVSignedBinBuiltinSet; 1943 1944// 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation 1945let RequiredFeatures = ["FullMultiply"] in { 1946defm vsmul : RVVSignedBinBuiltinSet; 1947} 1948 1949// 13.4. Vector Single-Width Scaling Shift Instructions 1950defm vssrl : RVVUnsignedShiftBuiltinSet; 1951defm vssra : RVVSignedShiftBuiltinSet; 1952 1953// 13.5. Vector Narrowing Fixed-Point Clip Instructions 1954defm vnclipu : RVVUnsignedNShiftBuiltinSet; 1955defm vnclip : RVVSignedNShiftBuiltinSet; 1956 1957// 14. Vector Floating-Point Instructions 1958// 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions 1959defm vfadd : RVVFloatingBinBuiltinSet; 1960defm vfsub : RVVFloatingBinBuiltinSet; 1961defm vfrsub : RVVFloatingBinVFBuiltinSet; 1962 1963// 14.3. Vector Widening Floating-Point Add/Subtract Instructions 1964// Widening FP add/subtract, 2*SEW = SEW +/- SEW 1965defm vfwadd : RVVFloatingWidenBinBuiltinSet; 1966defm vfwsub : RVVFloatingWidenBinBuiltinSet; 1967// Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW 1968defm vfwadd : RVVFloatingWidenOp0BinBuiltinSet; 1969defm vfwsub : RVVFloatingWidenOp0BinBuiltinSet; 1970 1971// 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions 1972defm vfmul : RVVFloatingBinBuiltinSet; 1973defm vfdiv : RVVFloatingBinBuiltinSet; 1974defm vfrdiv : RVVFloatingBinVFBuiltinSet; 1975 1976// 14.5. Vector Widening Floating-Point Multiply 1977let Log2LMUL = [-2, -1, 0, 1, 2] in { 1978 defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", 1979 [["vv", "w", "wvv"], 1980 ["vf", "w", "wve"]]>; 1981} 1982} 1983 1984// 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions 1985let UnMaskedPolicyScheme = HasPolicyOperand in { 1986defm vfmacc : RVVFloatingTerBuiltinSet; 1987defm vfnmacc : RVVFloatingTerBuiltinSet; 1988defm vfmsac : RVVFloatingTerBuiltinSet; 1989defm vfnmsac : RVVFloatingTerBuiltinSet; 1990defm vfmadd : RVVFloatingTerBuiltinSet; 1991defm vfnmadd : RVVFloatingTerBuiltinSet; 1992defm vfmsub : RVVFloatingTerBuiltinSet; 1993defm vfnmsub : RVVFloatingTerBuiltinSet; 1994 1995// 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions 1996defm vfwmacc : RVVFloatingWidenTerBuiltinSet; 1997defm vfwnmacc : RVVFloatingWidenTerBuiltinSet; 1998defm vfwmsac : RVVFloatingWidenTerBuiltinSet; 1999defm vfwnmsac : RVVFloatingWidenTerBuiltinSet; 2000} 2001 2002// 14.8. Vector Floating-Point Square-Root Instruction 2003let UnMaskedPolicyScheme = HasPassthruOperand in { 2004def vfsqrt : RVVFloatingUnaryVVBuiltin; 2005 2006// 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction 2007def vfrsqrt7 : RVVFloatingUnaryVVBuiltin; 2008 2009// 14.10. Vector Floating-Point Reciprocal Estimate Instruction 2010def vfrec7 : RVVFloatingUnaryVVBuiltin; 2011 2012// 14.11. Vector Floating-Point MIN/MAX Instructions 2013defm vfmin : RVVFloatingBinBuiltinSet; 2014defm vfmax : RVVFloatingBinBuiltinSet; 2015 2016// 14.12. Vector Floating-Point Sign-Injection Instructions 2017defm vfsgnj : RVVFloatingBinBuiltinSet; 2018defm vfsgnjn : RVVFloatingBinBuiltinSet; 2019defm vfsgnjx : RVVFloatingBinBuiltinSet; 2020} 2021defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "xfd">; 2022defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "xfd">; 2023 2024// 14.13. Vector Floating-Point Compare Instructions 2025let MaskedPolicyScheme = HasPassthruOperand, 2026 HasTailPolicy = false in { 2027defm vmfeq : RVVFloatingMaskOutBuiltinSet; 2028defm vmfne : RVVFloatingMaskOutBuiltinSet; 2029defm vmflt : RVVFloatingMaskOutBuiltinSet; 2030defm vmfle : RVVFloatingMaskOutBuiltinSet; 2031defm vmfgt : RVVFloatingMaskOutBuiltinSet; 2032defm vmfge : RVVFloatingMaskOutBuiltinSet; 2033} 2034 2035// 14.14. Vector Floating-Point Classify Instruction 2036let Name = "vfclass_v", UnMaskedPolicyScheme = HasPassthruOperand in 2037 def vfclass : RVVOp0Builtin<"Uv", "Uvv", "xfd">; 2038 2039// 14.15. Vector Floating-Point Merge Instructio 2040// C/C++ Operand: (mask, op1, op2, vl), Builtin: (op1, op2, mask, vl) 2041let HasMasked = false, 2042 UnMaskedPolicyScheme = HasPassthruOperand, 2043 MaskedPolicyScheme = NonePolicy, 2044 ManualCodegen = [{ 2045 // insert poison passthru 2046 if (PolicyAttrs & RVV_VTA) 2047 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 2048 IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; 2049 }] in { 2050 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "xfd", 2051 [["vvm", "v", "vvvm"]]>; 2052 defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "xfd", 2053 [["vfm", "v", "vvem"]]>; 2054} 2055 2056// 14.16. Vector Floating-Point Move Instruction 2057let HasMasked = false, 2058 UnMaskedPolicyScheme = HasPassthruOperand, 2059 SupportOverloading = false, 2060 MaskedPolicyScheme = NonePolicy, 2061 OverloadedName = "vfmv_v" in 2062 defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "xfd", 2063 [["f", "v", "ve"]]>; 2064 2065// 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions 2066let UnMaskedPolicyScheme = HasPassthruOperand in { 2067def vfcvt_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_xu">; 2068def vfcvt_x_f_v : RVVConvToSignedBuiltin<"vfcvt_x">; 2069def vfcvt_rtz_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_rtz_xu">; 2070def vfcvt_rtz_x_f_v : RVVConvToSignedBuiltin<"vfcvt_rtz_x">; 2071def vfcvt_f_xu_v : RVVConvBuiltin<"Fv", "FvUv", "sil", "vfcvt_f">; 2072def vfcvt_f_x_v : RVVConvBuiltin<"Fv", "Fvv", "sil", "vfcvt_f">; 2073 2074// 14.18. Widening Floating-Point/Integer Type-Convert Instructions 2075let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2076 def vfwcvt_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_xu">; 2077 def vfwcvt_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_x">; 2078 def vfwcvt_rtz_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_rtz_xu">; 2079 def vfwcvt_rtz_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_rtz_x">; 2080 def vfwcvt_f_xu_v : RVVConvBuiltin<"Fw", "FwUv", "csi", "vfwcvt_f">; 2081 def vfwcvt_f_x_v : RVVConvBuiltin<"Fw", "Fwv", "csi", "vfwcvt_f">; 2082 def vfwcvt_f_f_v : RVVConvBuiltin<"w", "wv", "xf", "vfwcvt_f">; 2083} 2084 2085// 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions 2086let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2087 def vfncvt_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_xu">; 2088 def vfncvt_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_x">; 2089 def vfncvt_rtz_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_rtz_xu">; 2090 def vfncvt_rtz_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_rtz_x">; 2091 def vfncvt_f_xu_w : RVVConvBuiltin<"Fv", "FvUw", "csi", "vfncvt_f">; 2092 def vfncvt_f_x_w : RVVConvBuiltin<"Fv", "Fvw", "csi", "vfncvt_f">; 2093 def vfncvt_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_f">; 2094 def vfncvt_rod_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_rod_f">; 2095} 2096} 2097 2098// 15. Vector Reduction Operations 2099// 15.1. Vector Single-Width Integer Reduction Instructions 2100let UnMaskedPolicyScheme = HasPassthruOperand, 2101 MaskedPolicyScheme = HasPassthruOperand, 2102 HasMaskPolicy = false in { 2103defm vredsum : RVVIntReductionBuiltinSet; 2104defm vredmaxu : RVVUnsignedReductionBuiltin; 2105defm vredmax : RVVSignedReductionBuiltin; 2106defm vredminu : RVVUnsignedReductionBuiltin; 2107defm vredmin : RVVSignedReductionBuiltin; 2108defm vredand : RVVIntReductionBuiltinSet; 2109defm vredor : RVVIntReductionBuiltinSet; 2110defm vredxor : RVVIntReductionBuiltinSet; 2111 2112// 15.2. Vector Widening Integer Reduction Instructions 2113// Vector Widening Integer Reduction Operations 2114let HasMaskedOffOperand = true in { 2115 defm vwredsum : RVVOutOp0BuiltinSet<"vwredsum", "csi", 2116 [["vs", "vSw", "SwvSw"]]>; 2117 defm vwredsumu : RVVOutOp0BuiltinSet<"vwredsumu", "csi", 2118 [["vs", "UvUSw", "USwUvUSw"]]>; 2119} 2120 2121// 15.3. Vector Single-Width Floating-Point Reduction Instructions 2122defm vfredmax : RVVFloatingReductionBuiltin; 2123defm vfredmin : RVVFloatingReductionBuiltin; 2124defm vfredusum : RVVFloatingReductionBuiltin; 2125defm vfredosum : RVVFloatingReductionBuiltin; 2126 2127// 15.4. Vector Widening Floating-Point Reduction Instructions 2128defm vfwredusum : RVVFloatingWidenReductionBuiltin; 2129defm vfwredosum : RVVFloatingWidenReductionBuiltin; 2130} 2131 2132// 16. Vector Mask Instructions 2133// 16.1. Vector Mask-Register Logical Instructions 2134def vmand : RVVMaskBinBuiltin; 2135def vmnand : RVVMaskBinBuiltin; 2136def vmandn : RVVMaskBinBuiltin; 2137def vmxor : RVVMaskBinBuiltin; 2138def vmor : RVVMaskBinBuiltin; 2139def vmnor : RVVMaskBinBuiltin; 2140def vmorn : RVVMaskBinBuiltin; 2141def vmxnor : RVVMaskBinBuiltin; 2142// pseudoinstructions 2143def vmclr : RVVMaskNullaryBuiltin; 2144def vmset : RVVMaskNullaryBuiltin; 2145defm vmmv_m : RVVPseudoMaskBuiltin<"vmand", "c">; 2146defm vmnot_m : RVVPseudoMaskBuiltin<"vmnand", "c">; 2147 2148let MaskedPolicyScheme = NonePolicy in { 2149// 16.2. Vector count population in mask vcpop.m 2150def vcpop : RVVMaskOp0Builtin<"um">; 2151 2152// 16.3. vfirst find-first-set mask bit 2153def vfirst : RVVMaskOp0Builtin<"lm">; 2154} 2155 2156let MaskedPolicyScheme = HasPassthruOperand, 2157 HasTailPolicy = false in { 2158// 16.4. vmsbf.m set-before-first mask bit 2159def vmsbf : RVVMaskUnaryBuiltin; 2160 2161// 16.5. vmsif.m set-including-first mask bit 2162def vmsif : RVVMaskUnaryBuiltin; 2163 2164// 16.6. vmsof.m set-only-first mask bit 2165def vmsof : RVVMaskUnaryBuiltin; 2166} 2167 2168let UnMaskedPolicyScheme = HasPassthruOperand, SupportOverloading = false in { 2169 // 16.8. Vector Iota Instruction 2170 defm viota : RVVOutBuiltinSet<"viota", "csil", [["m", "Uv", "Uvm"]]>; 2171 2172 // 16.9. Vector Element Index Instruction 2173 defm vid : RVVOutBuiltinSet<"vid", "csil", [["v", "v", "v"], 2174 ["v", "Uv", "Uv"]]>; 2175} 2176 2177// 17. Vector Permutation Instructions 2178// 17.1. Integer Scalar Move Instructions 2179let HasMasked = false, MaskedPolicyScheme = NonePolicy in { 2180 let HasVL = false, OverloadedName = "vmv_x" in 2181 defm vmv_x : RVVOp0BuiltinSet<"vmv_x_s", "csil", 2182 [["s", "ve", "ev"], 2183 ["s", "UvUe", "UeUv"]]>; 2184 let OverloadedName = "vmv_s", 2185 UnMaskedPolicyScheme = HasPassthruOperand, 2186 SupportOverloading = false in 2187 defm vmv_s : RVVOutBuiltinSet<"vmv_s_x", "csil", 2188 [["x", "v", "ve"], 2189 ["x", "Uv", "UvUe"]]>; 2190} 2191 2192// 17.2. Floating-Point Scalar Move Instructions 2193let HasMasked = false, MaskedPolicyScheme = NonePolicy in { 2194 let HasVL = false, OverloadedName = "vfmv_f" in 2195 defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "xfd", 2196 [["s", "ve", "ev"]]>; 2197 let OverloadedName = "vfmv_s", 2198 UnMaskedPolicyScheme = HasPassthruOperand, 2199 SupportOverloading = false in 2200 defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "xfd", 2201 [["f", "v", "ve"], 2202 ["x", "Uv", "UvUe"]]>; 2203} 2204 2205// 17.3. Vector Slide Instructions 2206// 17.3.1. Vector Slideup Instructions 2207defm vslideup : RVVSlideUpBuiltinSet; 2208// 17.3.2. Vector Slidedown Instructions 2209defm vslidedown : RVVSlideDownBuiltinSet; 2210 2211// 17.3.3. Vector Slide1up Instructions 2212let UnMaskedPolicyScheme = HasPassthruOperand in { 2213defm vslide1up : RVVSlideOneBuiltinSet; 2214defm vfslide1up : RVVFloatingBinVFBuiltinSet; 2215 2216// 17.3.4. Vector Slide1down Instruction 2217defm vslide1down : RVVSlideOneBuiltinSet; 2218defm vfslide1down : RVVFloatingBinVFBuiltinSet; 2219 2220// 17.4. Vector Register Gather Instructions 2221// signed and floating type 2222defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfd", 2223 [["vv", "v", "vvUv"]]>; 2224defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfd", 2225 [["vx", "v", "vvz"]]>; 2226defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd", 2227 [["vv", "v", "vv(Log2EEW:4)Uv"]]>; 2228// unsigned type 2229defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil", 2230 [["vv", "Uv", "UvUvUv"]]>; 2231defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csil", 2232 [["vx", "Uv", "UvUvz"]]>; 2233defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csil", 2234 [["vv", "Uv", "UvUv(Log2EEW:4)Uv"]]>; 2235} 2236 2237// 17.5. Vector Compress Instruction 2238let HasMasked = false, 2239 UnMaskedPolicyScheme = HasPassthruOperand, 2240 MaskedPolicyScheme = NonePolicy, 2241 ManualCodegen = [{ 2242 // insert poison passthru 2243 if (PolicyAttrs & RVV_VTA) 2244 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 2245 IntrinsicTypes = {ResultType, Ops.back()->getType()}; 2246 }] in { 2247 // signed and floating type 2248 defm vcompress : RVVOutBuiltinSet<"vcompress", "csilxfd", 2249 [["vm", "v", "vvm"]]>; 2250 // unsigned type 2251 defm vcompress : RVVOutBuiltinSet<"vcompress", "csil", 2252 [["vm", "Uv", "UvUvm"]]>; 2253} 2254 2255// Miscellaneous 2256let HasMasked = false, HasVL = false, IRName = "" in { 2257 let Name = "vreinterpret_v", MaskedPolicyScheme = NonePolicy, 2258 ManualCodegen = [{ 2259 return Builder.CreateBitCast(Ops[0], ResultType); 2260 }] in { 2261 // Reinterpret between different type under the same SEW and LMUL 2262 def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">; 2263 def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "sil", "v">; 2264 def vreinterpret_u_i : RVVBuiltin<"vUv", "Uvv", "csil", "Uv">; 2265 def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "sil", "Uv">; 2266 def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "sil", "Fv">; 2267 def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "sil", "Fv">; 2268 2269 // Reinterpret between different SEW under the same LMUL 2270 foreach dst_sew = ["(FixedSEW:8)", "(FixedSEW:16)", "(FixedSEW:32)", 2271 "(FixedSEW:64)"] in { 2272 def vreinterpret_i_ # dst_sew : RVVBuiltin<"v" # dst_sew # "v", 2273 dst_sew # "vv", "csil", dst_sew # "v">; 2274 def vreinterpret_u_ # dst_sew : RVVBuiltin<"Uv" # dst_sew # "Uv", 2275 dst_sew # "UvUv", "csil", dst_sew # "Uv">; 2276 } 2277 } 2278 2279 let Name = "vundefined", SupportOverloading = false, 2280 MaskedPolicyScheme = NonePolicy, 2281 ManualCodegen = [{ 2282 return llvm::PoisonValue::get(ResultType); 2283 }] in { 2284 def vundefined : RVVBuiltin<"v", "v", "csilxfd">; 2285 def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">; 2286 } 2287 2288 // LMUL truncation 2289 // C/C++ Operand: VecTy, IR Operand: VecTy, Index 2290 let Name = "vlmul_trunc_v", OverloadedName = "vlmul_trunc", 2291 MaskedPolicyScheme = NonePolicy, 2292 ManualCodegen = [{ { 2293 ID = Intrinsic::vector_extract; 2294 IntrinsicTypes = {ResultType, Ops[0]->getType()}; 2295 Ops.push_back(ConstantInt::get(Int64Ty, 0)); 2296 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2297 } }] in { 2298 foreach dst_lmul = ["(SFixedLog2LMUL:-3)", "(SFixedLog2LMUL:-2)", "(SFixedLog2LMUL:-1)", 2299 "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2300 def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2301 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2302 def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2303 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2304 } 2305 } 2306 2307 // LMUL extension 2308 // C/C++ Operand: SubVecTy, IR Operand: VecTy, SubVecTy, Index 2309 let Name = "vlmul_ext_v", OverloadedName = "vlmul_ext", 2310 MaskedPolicyScheme = NonePolicy, 2311 ManualCodegen = [{ 2312 ID = Intrinsic::vector_insert; 2313 IntrinsicTypes = {ResultType, Ops[0]->getType()}; 2314 Ops.push_back(llvm::PoisonValue::get(ResultType)); 2315 std::swap(Ops[0], Ops[1]); 2316 Ops.push_back(ConstantInt::get(Int64Ty, 0)); 2317 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2318 }] in { 2319 foreach dst_lmul = ["(LFixedLog2LMUL:-2)", "(LFixedLog2LMUL:-1)", "(LFixedLog2LMUL:-0)", 2320 "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2321 def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2322 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2323 def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2324 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2325 } 2326 } 2327 2328 let Name = "vget_v", MaskedPolicyScheme = NonePolicy, 2329 ManualCodegen = [{ 2330 { 2331 ID = Intrinsic::vector_extract; 2332 auto *VecTy = cast<ScalableVectorType>(ResultType); 2333 auto *OpVecTy = cast<ScalableVectorType>(Ops[0]->getType()); 2334 // Mask to only valid indices. 2335 unsigned MaxIndex = OpVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2336 assert(isPowerOf2_32(MaxIndex)); 2337 Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); 2338 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2339 Ops[1] = Builder.CreateMul(Ops[1], 2340 ConstantInt::get(Ops[1]->getType(), 2341 VecTy->getMinNumElements())); 2342 IntrinsicTypes = {ResultType, Ops[0]->getType()}; 2343 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2344 } 2345 }] in { 2346 foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2347 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfd", dst_lmul # "v">; 2348 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">; 2349 } 2350 } 2351 2352 let Name = "vset_v", Log2LMUL = [0, 1, 2], MaskedPolicyScheme = NonePolicy, 2353 ManualCodegen = [{ 2354 { 2355 ID = Intrinsic::vector_insert; 2356 IntrinsicTypes = {ResultType, Ops[2]->getType()}; 2357 auto *ResVecTy = cast<ScalableVectorType>(ResultType); 2358 auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType()); 2359 // Mask to only valid indices. 2360 unsigned MaxIndex = ResVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2361 assert(isPowerOf2_32(MaxIndex)); 2362 Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); 2363 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2364 Ops[1] = Builder.CreateMul(Ops[1], 2365 ConstantInt::get(Ops[1]->getType(), 2366 VecTy->getMinNumElements())); 2367 std::swap(Ops[1], Ops[2]); 2368 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); 2369 } 2370 }] in { 2371 foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2372 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfd">; 2373 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">; 2374 } 2375 } 2376} 2377