1 //===--- RISCVVIntrinsicUtils.h - RISC-V Vector Intrinsic Utils -*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
10 #define CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
11 
12 #include "llvm/ADT/ArrayRef.h"
13 #include "llvm/ADT/BitmaskEnum.h"
14 #include "llvm/ADT/SmallVector.h"
15 #include "llvm/ADT/StringRef.h"
16 #include <cstdint>
17 #include <optional>
18 #include <set>
19 #include <string>
20 #include <unordered_map>
21 #include <vector>
22 
23 namespace llvm {
24 class raw_ostream;
25 } // end namespace llvm
26 
27 namespace clang {
28 namespace RISCV {
29 
30 using VScaleVal = std::optional<unsigned>;
31 
32 // Modifier for vector type.
33 enum class VectorTypeModifier : uint8_t {
34   NoModifier,
35   Widening2XVector,
36   Widening4XVector,
37   Widening8XVector,
38   MaskVector,
39   Log2EEW3,
40   Log2EEW4,
41   Log2EEW5,
42   Log2EEW6,
43   FixedSEW8,
44   FixedSEW16,
45   FixedSEW32,
46   FixedSEW64,
47   LFixedLog2LMULN3,
48   LFixedLog2LMULN2,
49   LFixedLog2LMULN1,
50   LFixedLog2LMUL0,
51   LFixedLog2LMUL1,
52   LFixedLog2LMUL2,
53   LFixedLog2LMUL3,
54   SFixedLog2LMULN3,
55   SFixedLog2LMULN2,
56   SFixedLog2LMULN1,
57   SFixedLog2LMUL0,
58   SFixedLog2LMUL1,
59   SFixedLog2LMUL2,
60   SFixedLog2LMUL3,
61   SEFixedLog2LMULN3,
62   SEFixedLog2LMULN2,
63   SEFixedLog2LMULN1,
64   SEFixedLog2LMUL0,
65   SEFixedLog2LMUL1,
66   SEFixedLog2LMUL2,
67   SEFixedLog2LMUL3,
68   Tuple2,
69   Tuple3,
70   Tuple4,
71   Tuple5,
72   Tuple6,
73   Tuple7,
74   Tuple8,
75 };
76 
77 // Similar to basic type but used to describe what's kind of type related to
78 // basic vector type, used to compute type info of arguments.
79 enum class BaseTypeModifier : uint8_t {
80   Invalid,
81   Scalar,
82   Vector,
83   Void,
84   SizeT,
85   Ptrdiff,
86   UnsignedLong,
87   SignedLong,
88   Float32
89 };
90 
91 // Modifier for type, used for both scalar and vector types.
92 enum class TypeModifier : uint8_t {
93   NoModifier = 0,
94   Pointer = 1 << 0,
95   Const = 1 << 1,
96   Immediate = 1 << 2,
97   UnsignedInteger = 1 << 3,
98   SignedInteger = 1 << 4,
99   Float = 1 << 5,
100   BFloat = 1 << 6,
101   // LMUL1 should be kind of VectorTypeModifier, but that might come with
102   // Widening2XVector for widening reduction.
103   // However that might require VectorTypeModifier become bitmask rather than
104   // simple enum, so we decide keek LMUL1 in TypeModifier for code size
105   // optimization of clang binary size.
106   LMUL1 = 1 << 7,
107   MaxOffset = 7,
108   LLVM_MARK_AS_BITMASK_ENUM(LMUL1),
109 };
110 
111 class Policy {
112 public:
113   enum PolicyType {
114     Undisturbed,
115     Agnostic,
116   };
117 
118 private:
119   // The default assumption for an RVV instruction is TAMA, as an undisturbed
120   // policy generally will affect the performance of an out-of-order core.
121   const PolicyType TailPolicy = Agnostic;
122   const PolicyType MaskPolicy = Agnostic;
123 
124 public:
125   Policy() = default;
Policy(PolicyType TailPolicy)126   Policy(PolicyType TailPolicy) : TailPolicy(TailPolicy) {}
Policy(PolicyType TailPolicy,PolicyType MaskPolicy)127   Policy(PolicyType TailPolicy, PolicyType MaskPolicy)
128       : TailPolicy(TailPolicy), MaskPolicy(MaskPolicy) {}
129 
isTAMAPolicy()130   bool isTAMAPolicy() const {
131     return TailPolicy == Agnostic && MaskPolicy == Agnostic;
132   }
133 
isTAMUPolicy()134   bool isTAMUPolicy() const {
135     return TailPolicy == Agnostic && MaskPolicy == Undisturbed;
136   }
137 
isTUMAPolicy()138   bool isTUMAPolicy() const {
139     return TailPolicy == Undisturbed && MaskPolicy == Agnostic;
140   }
141 
isTUMUPolicy()142   bool isTUMUPolicy() const {
143     return TailPolicy == Undisturbed && MaskPolicy == Undisturbed;
144   }
145 
isTAPolicy()146   bool isTAPolicy() const { return TailPolicy == Agnostic; }
147 
isTUPolicy()148   bool isTUPolicy() const { return TailPolicy == Undisturbed; }
149 
isMAPolicy()150   bool isMAPolicy() const { return MaskPolicy == Agnostic; }
151 
isMUPolicy()152   bool isMUPolicy() const { return MaskPolicy == Undisturbed; }
153 
154   bool operator==(const Policy &Other) const {
155     return TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy;
156   }
157 
158   bool operator!=(const Policy &Other) const { return !(*this == Other); }
159 
160   bool operator<(const Policy &Other) const {
161     // Just for maintain the old order for quick test.
162     if (MaskPolicy != Other.MaskPolicy)
163       return Other.MaskPolicy < MaskPolicy;
164     return TailPolicy < Other.TailPolicy;
165   }
166 };
167 
168 // PrototypeDescriptor is used to compute type info of arguments or return
169 // value.
170 struct PrototypeDescriptor {
171   constexpr PrototypeDescriptor() = default;
172   constexpr PrototypeDescriptor(
173       BaseTypeModifier PT,
174       VectorTypeModifier VTM = VectorTypeModifier::NoModifier,
175       TypeModifier TM = TypeModifier::NoModifier)
PTPrototypeDescriptor176       : PT(static_cast<uint8_t>(PT)), VTM(static_cast<uint8_t>(VTM)),
177         TM(static_cast<uint8_t>(TM)) {}
PrototypeDescriptorPrototypeDescriptor178   constexpr PrototypeDescriptor(uint8_t PT, uint8_t VTM, uint8_t TM)
179       : PT(PT), VTM(VTM), TM(TM) {}
180 
181   uint8_t PT = static_cast<uint8_t>(BaseTypeModifier::Invalid);
182   uint8_t VTM = static_cast<uint8_t>(VectorTypeModifier::NoModifier);
183   uint8_t TM = static_cast<uint8_t>(TypeModifier::NoModifier);
184 
185   bool operator!=(const PrototypeDescriptor &PD) const {
186     return !(*this == PD);
187   }
188   bool operator==(const PrototypeDescriptor &PD) const {
189     return PD.PT == PT && PD.VTM == VTM && PD.TM == TM;
190   }
191   bool operator<(const PrototypeDescriptor &PD) const {
192     return std::tie(PT, VTM, TM) < std::tie(PD.PT, PD.VTM, PD.TM);
193   }
194   static const PrototypeDescriptor Mask;
195   static const PrototypeDescriptor Vector;
196   static const PrototypeDescriptor VL;
197   static std::optional<PrototypeDescriptor>
198   parsePrototypeDescriptor(llvm::StringRef PrototypeStr);
199 };
200 
201 llvm::SmallVector<PrototypeDescriptor>
202 parsePrototypes(llvm::StringRef Prototypes);
203 
204 // Basic type of vector type.
205 enum class BasicType : uint8_t {
206   Unknown = 0,
207   Int8 = 1 << 0,
208   Int16 = 1 << 1,
209   Int32 = 1 << 2,
210   Int64 = 1 << 3,
211   BFloat16 = 1 << 4,
212   Float16 = 1 << 5,
213   Float32 = 1 << 6,
214   Float64 = 1 << 7,
215   MaxOffset = 7,
216   LLVM_MARK_AS_BITMASK_ENUM(Float64),
217 };
218 
219 // Type of vector type.
220 enum ScalarTypeKind : uint8_t {
221   Void,
222   Size_t,
223   Ptrdiff_t,
224   UnsignedLong,
225   SignedLong,
226   Boolean,
227   SignedInteger,
228   UnsignedInteger,
229   Float,
230   BFloat,
231   Invalid,
232   Undefined,
233 };
234 
235 // Exponential LMUL
236 struct LMULType {
237   int Log2LMUL;
238   LMULType(int Log2LMUL);
239   // Return the C/C++ string representation of LMUL
240   std::string str() const;
241   std::optional<unsigned> getScale(unsigned ElementBitwidth) const;
242   void MulLog2LMUL(int Log2LMUL);
243 };
244 
245 class RVVType;
246 using RVVTypePtr = RVVType *;
247 using RVVTypes = std::vector<RVVTypePtr>;
248 class RVVTypeCache;
249 
250 // This class is compact representation of a valid and invalid RVVType.
251 class RVVType {
252   friend class RVVTypeCache;
253 
254   BasicType BT;
255   ScalarTypeKind ScalarType = Undefined;
256   LMULType LMUL;
257   bool IsPointer = false;
258   // IsConstant indices are "int", but have the constant expression.
259   bool IsImmediate = false;
260   // Const qualifier for pointer to const object or object of const type.
261   bool IsConstant = false;
262   unsigned ElementBitwidth = 0;
263   VScaleVal Scale = 0;
264   bool Valid;
265   bool IsTuple = false;
266   unsigned NF = 0;
267 
268   std::string BuiltinStr;
269   std::string ClangBuiltinStr;
270   std::string Str;
271   std::string ShortStr;
272 
273   enum class FixedLMULType { LargerThan, SmallerThan, SmallerOrEqual };
274 
275   RVVType(BasicType BT, int Log2LMUL, const PrototypeDescriptor &Profile);
276 
277 public:
278   // Return the string representation of a type, which is an encoded string for
279   // passing to the BUILTIN() macro in Builtins.def.
getBuiltinStr()280   const std::string &getBuiltinStr() const { return BuiltinStr; }
281 
282   // Return the clang builtin type for RVV vector type which are used in the
283   // riscv_vector.h header file.
getClangBuiltinStr()284   const std::string &getClangBuiltinStr() const { return ClangBuiltinStr; }
285 
286   // Return the C/C++ string representation of a type for use in the
287   // riscv_vector.h header file.
getTypeStr()288   const std::string &getTypeStr() const { return Str; }
289 
290   // Return the short name of a type for C/C++ name suffix.
getShortStr()291   const std::string &getShortStr() {
292     // Not all types are used in short name, so compute the short name by
293     // demanded.
294     if (ShortStr.empty())
295       initShortStr();
296     return ShortStr;
297   }
298 
isValid()299   bool isValid() const { return Valid; }
isScalar()300   bool isScalar() const { return Scale && *Scale == 0; }
isVector()301   bool isVector() const { return Scale && *Scale != 0; }
isVector(unsigned Width)302   bool isVector(unsigned Width) const {
303     return isVector() && ElementBitwidth == Width;
304   }
isFloat()305   bool isFloat() const { return ScalarType == ScalarTypeKind::Float; }
isBFloat()306   bool isBFloat() const { return ScalarType == ScalarTypeKind::BFloat; }
isSignedInteger()307   bool isSignedInteger() const {
308     return ScalarType == ScalarTypeKind::SignedInteger;
309   }
isFloatVector(unsigned Width)310   bool isFloatVector(unsigned Width) const {
311     return isVector() && isFloat() && ElementBitwidth == Width;
312   }
isFloat(unsigned Width)313   bool isFloat(unsigned Width) const {
314     return isFloat() && ElementBitwidth == Width;
315   }
isConstant()316   bool isConstant() const { return IsConstant; }
isPointer()317   bool isPointer() const { return IsPointer; }
isTuple()318   bool isTuple() const { return IsTuple; }
getElementBitwidth()319   unsigned getElementBitwidth() const { return ElementBitwidth; }
320 
getScalarType()321   ScalarTypeKind getScalarType() const { return ScalarType; }
getScale()322   VScaleVal getScale() const { return Scale; }
getNF()323   unsigned getNF() const {
324     assert(NF > 1 && NF <= 8 && "Only legal NF should be fetched");
325     return NF;
326   }
327 
328 private:
329   // Verify RVV vector type and set Valid.
330   bool verifyType() const;
331 
332   // Creates a type based on basic types of TypeRange
333   void applyBasicType();
334 
335   // Applies a prototype modifier to the current type. The result maybe an
336   // invalid type.
337   void applyModifier(const PrototypeDescriptor &prototype);
338 
339   void applyLog2EEW(unsigned Log2EEW);
340   void applyFixedSEW(unsigned NewSEW);
341   void applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type);
342 
343   // Compute and record a string for legal type.
344   void initBuiltinStr();
345   // Compute and record a builtin RVV vector type string.
346   void initClangBuiltinStr();
347   // Compute and record a type string for used in the header.
348   void initTypeStr();
349   // Compute and record a short name of a type for C/C++ name suffix.
350   void initShortStr();
351 };
352 
353 // This class is used to manage RVVType, RVVType should only created by this
354 // class, also provided thread-safe cache capability.
355 class RVVTypeCache {
356 private:
357   std::unordered_map<uint64_t, RVVType> LegalTypes;
358   std::set<uint64_t> IllegalTypes;
359 
360 public:
361   /// Compute output and input types by applying different config (basic type
362   /// and LMUL with type transformers). It also record result of type in legal
363   /// or illegal set to avoid compute the same config again. The result maybe
364   /// have illegal RVVType.
365   std::optional<RVVTypes>
366   computeTypes(BasicType BT, int Log2LMUL, unsigned NF,
367                llvm::ArrayRef<PrototypeDescriptor> Prototype);
368   std::optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL,
369                                         PrototypeDescriptor Proto);
370 };
371 
372 enum PolicyScheme : uint8_t {
373   SchemeNone,
374   // Passthru operand is at first parameter in C builtin.
375   HasPassthruOperand,
376   HasPolicyOperand,
377 };
378 
379 // TODO refactor RVVIntrinsic class design after support all intrinsic
380 // combination. This represents an instantiation of an intrinsic with a
381 // particular type and prototype
382 class RVVIntrinsic {
383 
384 private:
385   std::string BuiltinName; // Builtin name
386   std::string Name;        // C intrinsic name.
387   std::string OverloadedName;
388   std::string IRName;
389   bool IsMasked;
390   bool HasMaskedOffOperand;
391   bool HasVL;
392   PolicyScheme Scheme;
393   bool SupportOverloading;
394   bool HasBuiltinAlias;
395   std::string ManualCodegen;
396   RVVTypePtr OutputType; // Builtin output type
397   RVVTypes InputTypes;   // Builtin input types
398   // The types we use to obtain the specific LLVM intrinsic. They are index of
399   // InputTypes. -1 means the return type.
400   std::vector<int64_t> IntrinsicTypes;
401   unsigned NF = 1;
402   Policy PolicyAttrs;
403 
404 public:
405   RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix,
406                llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix,
407                llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand,
408                bool HasVL, PolicyScheme Scheme, bool SupportOverloading,
409                bool HasBuiltinAlias, llvm::StringRef ManualCodegen,
410                const RVVTypes &Types,
411                const std::vector<int64_t> &IntrinsicTypes,
412                const std::vector<llvm::StringRef> &RequiredFeatures,
413                unsigned NF, Policy PolicyAttrs, bool HasFRMRoundModeOp);
414   ~RVVIntrinsic() = default;
415 
getOutputType()416   RVVTypePtr getOutputType() const { return OutputType; }
getInputTypes()417   const RVVTypes &getInputTypes() const { return InputTypes; }
getBuiltinName()418   llvm::StringRef getBuiltinName() const { return BuiltinName; }
getName()419   llvm::StringRef getName() const { return Name; }
getOverloadedName()420   llvm::StringRef getOverloadedName() const { return OverloadedName; }
hasMaskedOffOperand()421   bool hasMaskedOffOperand() const { return HasMaskedOffOperand; }
hasVL()422   bool hasVL() const { return HasVL; }
hasPolicy()423   bool hasPolicy() const { return Scheme != PolicyScheme::SchemeNone; }
hasPassthruOperand()424   bool hasPassthruOperand() const {
425     return Scheme == PolicyScheme::HasPassthruOperand;
426   }
hasPolicyOperand()427   bool hasPolicyOperand() const {
428     return Scheme == PolicyScheme::HasPolicyOperand;
429   }
supportOverloading()430   bool supportOverloading() const { return SupportOverloading; }
hasBuiltinAlias()431   bool hasBuiltinAlias() const { return HasBuiltinAlias; }
hasManualCodegen()432   bool hasManualCodegen() const { return !ManualCodegen.empty(); }
isMasked()433   bool isMasked() const { return IsMasked; }
getIRName()434   llvm::StringRef getIRName() const { return IRName; }
getManualCodegen()435   llvm::StringRef getManualCodegen() const { return ManualCodegen; }
getPolicyScheme()436   PolicyScheme getPolicyScheme() const { return Scheme; }
getNF()437   unsigned getNF() const { return NF; }
getIntrinsicTypes()438   const std::vector<int64_t> &getIntrinsicTypes() const {
439     return IntrinsicTypes;
440   }
getPolicyAttrs()441   Policy getPolicyAttrs() const {
442     return PolicyAttrs;
443   }
getPolicyAttrsBits()444   unsigned getPolicyAttrsBits() const {
445     // CGBuiltin.cpp
446     // The 0th bit simulates the `vta` of RVV
447     // The 1st bit simulates the `vma` of RVV
448     // int PolicyAttrs = 0;
449 
450     if (PolicyAttrs.isTUMAPolicy())
451       return 2;
452     if (PolicyAttrs.isTAMAPolicy())
453       return 3;
454     if (PolicyAttrs.isTUMUPolicy())
455       return 0;
456     if (PolicyAttrs.isTAMUPolicy())
457       return 1;
458 
459     llvm_unreachable("unsupport policy");
460     return 0;
461   }
462 
463   // Return the type string for a BUILTIN() macro in Builtins.def.
464   std::string getBuiltinTypeStr() const;
465 
466   static std::string
467   getSuffixStr(RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL,
468                llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors);
469 
470   static llvm::SmallVector<PrototypeDescriptor>
471   computeBuiltinTypes(llvm::ArrayRef<PrototypeDescriptor> Prototype,
472                       bool IsMasked, bool HasMaskedOffOperand, bool HasVL,
473                       unsigned NF, PolicyScheme DefaultScheme,
474                       Policy PolicyAttrs, bool IsTuple);
475 
476   static llvm::SmallVector<Policy> getSupportedUnMaskedPolicies();
477   static llvm::SmallVector<Policy>
478       getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy);
479 
480   static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy,
481                                    std::string &Name, std::string &BuiltinName,
482                                    std::string &OverloadedName,
483                                    Policy &PolicyAttrs, bool HasFRMRoundModeOp);
484 };
485 
486 // RVVRequire should be sync'ed with target features, but only
487 // required features used in riscv_vector.td.
488 enum RVVRequire : uint32_t {
489   RVV_REQ_None = 0,
490   RVV_REQ_RV64 = 1 << 0,
491   RVV_REQ_Zvfhmin = 1 << 1,
492   RVV_REQ_Xsfvcp = 1 << 2,
493   RVV_REQ_Xsfvfnrclipxfqf = 1 << 3,
494   RVV_REQ_Xsfvfwmaccqqq = 1 << 4,
495   RVV_REQ_Xsfvqmaccdod = 1 << 5,
496   RVV_REQ_Xsfvqmaccqoq = 1 << 6,
497   RVV_REQ_Zvbb = 1 << 7,
498   RVV_REQ_Zvbc = 1 << 8,
499   RVV_REQ_Zvkb = 1 << 9,
500   RVV_REQ_Zvkg = 1 << 10,
501   RVV_REQ_Zvkned = 1 << 11,
502   RVV_REQ_Zvknha = 1 << 12,
503   RVV_REQ_Zvknhb = 1 << 13,
504   RVV_REQ_Zvksed = 1 << 14,
505   RVV_REQ_Zvksh = 1 << 15,
506   RVV_REQ_Experimental = 1 << 16,
507 
508   LLVM_MARK_AS_BITMASK_ENUM(RVV_REQ_Experimental)
509 };
510 
511 // Raw RVV intrinsic info, used to expand later.
512 // This struct is highly compact for minimized code size.
513 struct RVVIntrinsicRecord {
514   // Intrinsic name, e.g. vadd_vv
515   const char *Name;
516 
517   // Overloaded intrinsic name, could be empty if it can be computed from Name.
518   // e.g. vadd
519   const char *OverloadedName;
520 
521   // Prototype for this intrinsic, index of RVVSignatureTable.
522   uint16_t PrototypeIndex;
523 
524   // Suffix of intrinsic name, index of RVVSignatureTable.
525   uint16_t SuffixIndex;
526 
527   // Suffix of overloaded intrinsic name, index of RVVSignatureTable.
528   uint16_t OverloadedSuffixIndex;
529 
530   // Length of the prototype.
531   uint8_t PrototypeLength;
532 
533   // Length of intrinsic name suffix.
534   uint8_t SuffixLength;
535 
536   // Length of overloaded intrinsic suffix.
537   uint8_t OverloadedSuffixSize;
538 
539   // Required target features for this intrinsic.
540   uint32_t RequiredExtensions;
541 
542   // Supported type, mask of BasicType.
543   uint8_t TypeRangeMask;
544 
545   // Supported LMUL.
546   uint8_t Log2LMULMask;
547 
548   // Number of fields, greater than 1 if it's segment load/store.
549   uint8_t NF;
550 
551   bool HasMasked : 1;
552   bool HasVL : 1;
553   bool HasMaskedOffOperand : 1;
554   bool HasTailPolicy : 1;
555   bool HasMaskPolicy : 1;
556   bool HasFRMRoundModeOp : 1;
557   bool IsTuple : 1;
558   uint8_t UnMaskedPolicyScheme : 2;
559   uint8_t MaskedPolicyScheme : 2;
560 };
561 
562 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
563                               const RVVIntrinsicRecord &RVVInstrRecord);
564 
565 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
566 } // end namespace RISCV
567 
568 } // end namespace clang
569 
570 #endif // CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
571