1 //===--- RISCVVIntrinsicUtils.h - RISC-V Vector Intrinsic Utils -*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef CLANG_SUPPORT_RISCVVINTRINSICUTILS_H 10 #define CLANG_SUPPORT_RISCVVINTRINSICUTILS_H 11 12 #include "llvm/ADT/ArrayRef.h" 13 #include "llvm/ADT/BitmaskEnum.h" 14 #include "llvm/ADT/SmallVector.h" 15 #include "llvm/ADT/StringRef.h" 16 #include <cstdint> 17 #include <optional> 18 #include <set> 19 #include <string> 20 #include <unordered_map> 21 #include <vector> 22 23 namespace llvm { 24 class raw_ostream; 25 } // end namespace llvm 26 27 namespace clang { 28 namespace RISCV { 29 30 using VScaleVal = std::optional<unsigned>; 31 32 // Modifier for vector type. 33 enum class VectorTypeModifier : uint8_t { 34 NoModifier, 35 Widening2XVector, 36 Widening4XVector, 37 Widening8XVector, 38 MaskVector, 39 Log2EEW3, 40 Log2EEW4, 41 Log2EEW5, 42 Log2EEW6, 43 FixedSEW8, 44 FixedSEW16, 45 FixedSEW32, 46 FixedSEW64, 47 LFixedLog2LMULN3, 48 LFixedLog2LMULN2, 49 LFixedLog2LMULN1, 50 LFixedLog2LMUL0, 51 LFixedLog2LMUL1, 52 LFixedLog2LMUL2, 53 LFixedLog2LMUL3, 54 SFixedLog2LMULN3, 55 SFixedLog2LMULN2, 56 SFixedLog2LMULN1, 57 SFixedLog2LMUL0, 58 SFixedLog2LMUL1, 59 SFixedLog2LMUL2, 60 SFixedLog2LMUL3, 61 Tuple2, 62 Tuple3, 63 Tuple4, 64 Tuple5, 65 Tuple6, 66 Tuple7, 67 Tuple8, 68 }; 69 70 // Similar to basic type but used to describe what's kind of type related to 71 // basic vector type, used to compute type info of arguments. 72 enum class BaseTypeModifier : uint8_t { 73 Invalid, 74 Scalar, 75 Vector, 76 Void, 77 SizeT, 78 Ptrdiff, 79 UnsignedLong, 80 SignedLong, 81 }; 82 83 // Modifier for type, used for both scalar and vector types. 84 enum class TypeModifier : uint8_t { 85 NoModifier = 0, 86 Pointer = 1 << 0, 87 Const = 1 << 1, 88 Immediate = 1 << 2, 89 UnsignedInteger = 1 << 3, 90 SignedInteger = 1 << 4, 91 Float = 1 << 5, 92 // LMUL1 should be kind of VectorTypeModifier, but that might come with 93 // Widening2XVector for widening reduction. 94 // However that might require VectorTypeModifier become bitmask rather than 95 // simple enum, so we decide keek LMUL1 in TypeModifier for code size 96 // optimization of clang binary size. 97 LMUL1 = 1 << 6, 98 MaxOffset = 6, 99 LLVM_MARK_AS_BITMASK_ENUM(LMUL1), 100 }; 101 102 class Policy { 103 public: 104 enum PolicyType { 105 Undisturbed, 106 Agnostic, 107 }; 108 109 private: 110 // The default assumption for an RVV instruction is TAMA, as an undisturbed 111 // policy generally will affect the performance of an out-of-order core. 112 const PolicyType TailPolicy = Agnostic; 113 const PolicyType MaskPolicy = Agnostic; 114 115 public: 116 Policy() = default; 117 Policy(PolicyType TailPolicy) : TailPolicy(TailPolicy) {} 118 Policy(PolicyType TailPolicy, PolicyType MaskPolicy) 119 : TailPolicy(TailPolicy), MaskPolicy(MaskPolicy) {} 120 121 bool isTAMAPolicy() const { 122 return TailPolicy == Agnostic && MaskPolicy == Agnostic; 123 } 124 125 bool isTAMUPolicy() const { 126 return TailPolicy == Agnostic && MaskPolicy == Undisturbed; 127 } 128 129 bool isTUMAPolicy() const { 130 return TailPolicy == Undisturbed && MaskPolicy == Agnostic; 131 } 132 133 bool isTUMUPolicy() const { 134 return TailPolicy == Undisturbed && MaskPolicy == Undisturbed; 135 } 136 137 bool isTAPolicy() const { return TailPolicy == Agnostic; } 138 139 bool isTUPolicy() const { return TailPolicy == Undisturbed; } 140 141 bool isMAPolicy() const { return MaskPolicy == Agnostic; } 142 143 bool isMUPolicy() const { return MaskPolicy == Undisturbed; } 144 145 bool operator==(const Policy &Other) const { 146 return TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy; 147 } 148 149 bool operator!=(const Policy &Other) const { return !(*this == Other); } 150 151 bool operator<(const Policy &Other) const { 152 // Just for maintain the old order for quick test. 153 if (MaskPolicy != Other.MaskPolicy) 154 return Other.MaskPolicy < MaskPolicy; 155 return TailPolicy < Other.TailPolicy; 156 } 157 }; 158 159 // PrototypeDescriptor is used to compute type info of arguments or return 160 // value. 161 struct PrototypeDescriptor { 162 constexpr PrototypeDescriptor() = default; 163 constexpr PrototypeDescriptor( 164 BaseTypeModifier PT, 165 VectorTypeModifier VTM = VectorTypeModifier::NoModifier, 166 TypeModifier TM = TypeModifier::NoModifier) 167 : PT(static_cast<uint8_t>(PT)), VTM(static_cast<uint8_t>(VTM)), 168 TM(static_cast<uint8_t>(TM)) {} 169 constexpr PrototypeDescriptor(uint8_t PT, uint8_t VTM, uint8_t TM) 170 : PT(PT), VTM(VTM), TM(TM) {} 171 172 uint8_t PT = static_cast<uint8_t>(BaseTypeModifier::Invalid); 173 uint8_t VTM = static_cast<uint8_t>(VectorTypeModifier::NoModifier); 174 uint8_t TM = static_cast<uint8_t>(TypeModifier::NoModifier); 175 176 bool operator!=(const PrototypeDescriptor &PD) const { 177 return !(*this == PD); 178 } 179 bool operator==(const PrototypeDescriptor &PD) const { 180 return PD.PT == PT && PD.VTM == VTM && PD.TM == TM; 181 } 182 bool operator<(const PrototypeDescriptor &PD) const { 183 return std::tie(PT, VTM, TM) < std::tie(PD.PT, PD.VTM, PD.TM); 184 } 185 static const PrototypeDescriptor Mask; 186 static const PrototypeDescriptor Vector; 187 static const PrototypeDescriptor VL; 188 static std::optional<PrototypeDescriptor> 189 parsePrototypeDescriptor(llvm::StringRef PrototypeStr); 190 }; 191 192 llvm::SmallVector<PrototypeDescriptor> 193 parsePrototypes(llvm::StringRef Prototypes); 194 195 // Basic type of vector type. 196 enum class BasicType : uint8_t { 197 Unknown = 0, 198 Int8 = 1 << 0, 199 Int16 = 1 << 1, 200 Int32 = 1 << 2, 201 Int64 = 1 << 3, 202 Float16 = 1 << 4, 203 Float32 = 1 << 5, 204 Float64 = 1 << 6, 205 MaxOffset = 6, 206 LLVM_MARK_AS_BITMASK_ENUM(Float64), 207 }; 208 209 // Type of vector type. 210 enum ScalarTypeKind : uint8_t { 211 Void, 212 Size_t, 213 Ptrdiff_t, 214 UnsignedLong, 215 SignedLong, 216 Boolean, 217 SignedInteger, 218 UnsignedInteger, 219 Float, 220 Invalid, 221 }; 222 223 // Exponential LMUL 224 struct LMULType { 225 int Log2LMUL; 226 LMULType(int Log2LMUL); 227 // Return the C/C++ string representation of LMUL 228 std::string str() const; 229 std::optional<unsigned> getScale(unsigned ElementBitwidth) const; 230 void MulLog2LMUL(int Log2LMUL); 231 }; 232 233 class RVVType; 234 using RVVTypePtr = RVVType *; 235 using RVVTypes = std::vector<RVVTypePtr>; 236 class RVVTypeCache; 237 238 // This class is compact representation of a valid and invalid RVVType. 239 class RVVType { 240 friend class RVVTypeCache; 241 242 BasicType BT; 243 ScalarTypeKind ScalarType = Invalid; 244 LMULType LMUL; 245 bool IsPointer = false; 246 // IsConstant indices are "int", but have the constant expression. 247 bool IsImmediate = false; 248 // Const qualifier for pointer to const object or object of const type. 249 bool IsConstant = false; 250 unsigned ElementBitwidth = 0; 251 VScaleVal Scale = 0; 252 bool Valid; 253 bool IsTuple = false; 254 unsigned NF = 0; 255 256 std::string BuiltinStr; 257 std::string ClangBuiltinStr; 258 std::string Str; 259 std::string ShortStr; 260 261 enum class FixedLMULType { LargerThan, SmallerThan }; 262 263 RVVType(BasicType BT, int Log2LMUL, const PrototypeDescriptor &Profile); 264 265 public: 266 // Return the string representation of a type, which is an encoded string for 267 // passing to the BUILTIN() macro in Builtins.def. 268 const std::string &getBuiltinStr() const { return BuiltinStr; } 269 270 // Return the clang builtin type for RVV vector type which are used in the 271 // riscv_vector.h header file. 272 const std::string &getClangBuiltinStr() const { return ClangBuiltinStr; } 273 274 // Return the C/C++ string representation of a type for use in the 275 // riscv_vector.h header file. 276 const std::string &getTypeStr() const { return Str; } 277 278 // Return the short name of a type for C/C++ name suffix. 279 const std::string &getShortStr() { 280 // Not all types are used in short name, so compute the short name by 281 // demanded. 282 if (ShortStr.empty()) 283 initShortStr(); 284 return ShortStr; 285 } 286 287 bool isValid() const { return Valid; } 288 bool isScalar() const { return Scale && *Scale == 0; } 289 bool isVector() const { return Scale && *Scale != 0; } 290 bool isVector(unsigned Width) const { 291 return isVector() && ElementBitwidth == Width; 292 } 293 bool isFloat() const { return ScalarType == ScalarTypeKind::Float; } 294 bool isSignedInteger() const { 295 return ScalarType == ScalarTypeKind::SignedInteger; 296 } 297 bool isFloatVector(unsigned Width) const { 298 return isVector() && isFloat() && ElementBitwidth == Width; 299 } 300 bool isFloat(unsigned Width) const { 301 return isFloat() && ElementBitwidth == Width; 302 } 303 bool isConstant() const { return IsConstant; } 304 bool isPointer() const { return IsPointer; } 305 bool isTuple() const { return IsTuple; } 306 unsigned getElementBitwidth() const { return ElementBitwidth; } 307 308 ScalarTypeKind getScalarType() const { return ScalarType; } 309 VScaleVal getScale() const { return Scale; } 310 unsigned getNF() const { 311 assert(NF > 1 && NF <= 8 && "Only legal NF should be fetched"); 312 return NF; 313 } 314 315 private: 316 // Verify RVV vector type and set Valid. 317 bool verifyType() const; 318 319 // Creates a type based on basic types of TypeRange 320 void applyBasicType(); 321 322 // Applies a prototype modifier to the current type. The result maybe an 323 // invalid type. 324 void applyModifier(const PrototypeDescriptor &prototype); 325 326 void applyLog2EEW(unsigned Log2EEW); 327 void applyFixedSEW(unsigned NewSEW); 328 void applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type); 329 330 // Compute and record a string for legal type. 331 void initBuiltinStr(); 332 // Compute and record a builtin RVV vector type string. 333 void initClangBuiltinStr(); 334 // Compute and record a type string for used in the header. 335 void initTypeStr(); 336 // Compute and record a short name of a type for C/C++ name suffix. 337 void initShortStr(); 338 }; 339 340 // This class is used to manage RVVType, RVVType should only created by this 341 // class, also provided thread-safe cache capability. 342 class RVVTypeCache { 343 private: 344 std::unordered_map<uint64_t, RVVType> LegalTypes; 345 std::set<uint64_t> IllegalTypes; 346 347 public: 348 /// Compute output and input types by applying different config (basic type 349 /// and LMUL with type transformers). It also record result of type in legal 350 /// or illegal set to avoid compute the same config again. The result maybe 351 /// have illegal RVVType. 352 std::optional<RVVTypes> 353 computeTypes(BasicType BT, int Log2LMUL, unsigned NF, 354 llvm::ArrayRef<PrototypeDescriptor> Prototype); 355 std::optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL, 356 PrototypeDescriptor Proto); 357 }; 358 359 enum PolicyScheme : uint8_t { 360 SchemeNone, 361 // Passthru operand is at first parameter in C builtin. 362 HasPassthruOperand, 363 HasPolicyOperand, 364 }; 365 366 // TODO refactor RVVIntrinsic class design after support all intrinsic 367 // combination. This represents an instantiation of an intrinsic with a 368 // particular type and prototype 369 class RVVIntrinsic { 370 371 private: 372 std::string BuiltinName; // Builtin name 373 std::string Name; // C intrinsic name. 374 std::string OverloadedName; 375 std::string IRName; 376 bool IsMasked; 377 bool HasMaskedOffOperand; 378 bool HasVL; 379 PolicyScheme Scheme; 380 bool SupportOverloading; 381 bool HasBuiltinAlias; 382 std::string ManualCodegen; 383 RVVTypePtr OutputType; // Builtin output type 384 RVVTypes InputTypes; // Builtin input types 385 // The types we use to obtain the specific LLVM intrinsic. They are index of 386 // InputTypes. -1 means the return type. 387 std::vector<int64_t> IntrinsicTypes; 388 unsigned NF = 1; 389 Policy PolicyAttrs; 390 391 public: 392 RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix, 393 llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix, 394 llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand, 395 bool HasVL, PolicyScheme Scheme, bool SupportOverloading, 396 bool HasBuiltinAlias, llvm::StringRef ManualCodegen, 397 const RVVTypes &Types, 398 const std::vector<int64_t> &IntrinsicTypes, 399 const std::vector<llvm::StringRef> &RequiredFeatures, 400 unsigned NF, Policy PolicyAttrs, bool HasFRMRoundModeOp); 401 ~RVVIntrinsic() = default; 402 403 RVVTypePtr getOutputType() const { return OutputType; } 404 const RVVTypes &getInputTypes() const { return InputTypes; } 405 llvm::StringRef getBuiltinName() const { return BuiltinName; } 406 llvm::StringRef getName() const { return Name; } 407 llvm::StringRef getOverloadedName() const { return OverloadedName; } 408 bool hasMaskedOffOperand() const { return HasMaskedOffOperand; } 409 bool hasVL() const { return HasVL; } 410 bool hasPolicy() const { return Scheme != PolicyScheme::SchemeNone; } 411 bool hasPassthruOperand() const { 412 return Scheme == PolicyScheme::HasPassthruOperand; 413 } 414 bool hasPolicyOperand() const { 415 return Scheme == PolicyScheme::HasPolicyOperand; 416 } 417 bool supportOverloading() const { return SupportOverloading; } 418 bool hasBuiltinAlias() const { return HasBuiltinAlias; } 419 bool hasManualCodegen() const { return !ManualCodegen.empty(); } 420 bool isMasked() const { return IsMasked; } 421 llvm::StringRef getIRName() const { return IRName; } 422 llvm::StringRef getManualCodegen() const { return ManualCodegen; } 423 PolicyScheme getPolicyScheme() const { return Scheme; } 424 unsigned getNF() const { return NF; } 425 const std::vector<int64_t> &getIntrinsicTypes() const { 426 return IntrinsicTypes; 427 } 428 Policy getPolicyAttrs() const { 429 return PolicyAttrs; 430 } 431 unsigned getPolicyAttrsBits() const { 432 // CGBuiltin.cpp 433 // The 0th bit simulates the `vta` of RVV 434 // The 1st bit simulates the `vma` of RVV 435 // int PolicyAttrs = 0; 436 437 if (PolicyAttrs.isTUMAPolicy()) 438 return 2; 439 if (PolicyAttrs.isTAMAPolicy()) 440 return 3; 441 if (PolicyAttrs.isTUMUPolicy()) 442 return 0; 443 if (PolicyAttrs.isTAMUPolicy()) 444 return 1; 445 446 llvm_unreachable("unsupport policy"); 447 return 0; 448 } 449 450 // Return the type string for a BUILTIN() macro in Builtins.def. 451 std::string getBuiltinTypeStr() const; 452 453 static std::string 454 getSuffixStr(RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL, 455 llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors); 456 457 static llvm::SmallVector<PrototypeDescriptor> 458 computeBuiltinTypes(llvm::ArrayRef<PrototypeDescriptor> Prototype, 459 bool IsMasked, bool HasMaskedOffOperand, bool HasVL, 460 unsigned NF, PolicyScheme DefaultScheme, 461 Policy PolicyAttrs, bool IsTuple); 462 463 static llvm::SmallVector<Policy> getSupportedUnMaskedPolicies(); 464 static llvm::SmallVector<Policy> 465 getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy); 466 467 static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy, 468 std::string &Name, std::string &BuiltinName, 469 std::string &OverloadedName, 470 Policy &PolicyAttrs, bool HasFRMRoundModeOp); 471 }; 472 473 // RVVRequire should be sync'ed with target features, but only 474 // required features used in riscv_vector.td. 475 enum RVVRequire : uint8_t { 476 RVV_REQ_None = 0, 477 RVV_REQ_RV64 = 1 << 0, 478 RVV_REQ_Xsfvcp = 1 << 1, 479 480 LLVM_MARK_AS_BITMASK_ENUM(RVV_REQ_Xsfvcp) 481 }; 482 483 // Raw RVV intrinsic info, used to expand later. 484 // This struct is highly compact for minimized code size. 485 struct RVVIntrinsicRecord { 486 // Intrinsic name, e.g. vadd_vv 487 const char *Name; 488 489 // Overloaded intrinsic name, could be empty if it can be computed from Name. 490 // e.g. vadd 491 const char *OverloadedName; 492 493 // Prototype for this intrinsic, index of RVVSignatureTable. 494 uint16_t PrototypeIndex; 495 496 // Suffix of intrinsic name, index of RVVSignatureTable. 497 uint16_t SuffixIndex; 498 499 // Suffix of overloaded intrinsic name, index of RVVSignatureTable. 500 uint16_t OverloadedSuffixIndex; 501 502 // Length of the prototype. 503 uint8_t PrototypeLength; 504 505 // Length of intrinsic name suffix. 506 uint8_t SuffixLength; 507 508 // Length of overloaded intrinsic suffix. 509 uint8_t OverloadedSuffixSize; 510 511 // Required target features for this intrinsic. 512 uint8_t RequiredExtensions; 513 514 // Supported type, mask of BasicType. 515 uint8_t TypeRangeMask; 516 517 // Supported LMUL. 518 uint8_t Log2LMULMask; 519 520 // Number of fields, greater than 1 if it's segment load/store. 521 uint8_t NF; 522 523 bool HasMasked : 1; 524 bool HasVL : 1; 525 bool HasMaskedOffOperand : 1; 526 bool HasTailPolicy : 1; 527 bool HasMaskPolicy : 1; 528 bool HasFRMRoundModeOp : 1; 529 bool IsTuple : 1; 530 uint8_t UnMaskedPolicyScheme : 2; 531 uint8_t MaskedPolicyScheme : 2; 532 }; 533 534 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, 535 const RVVIntrinsicRecord &RVVInstrRecord); 536 537 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); 538 } // end namespace RISCV 539 540 } // end namespace clang 541 542 #endif // CLANG_SUPPORT_RISCVVINTRINSICUTILS_H 543