1 //===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===//
2 //
3 //  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 //  See https://llvm.org/LICENSE.txt for license information.
5 //  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This tablegen backend is responsible for emitting arm_sve.h, which includes
10 // a declaration and definition of each function specified by the ARM C/C++
11 // Language Extensions (ACLE).
12 //
13 // For details, visit:
14 //  https://developer.arm.com/architectures/system-architectures/software-standards/acle
15 //
16 // Each SVE instruction is implemented in terms of 1 or more functions which
17 // are suffixed with the element type of the input vectors.  Functions may be
18 // implemented in terms of generic vector operations such as +, *, -, etc. or
19 // by calling a __builtin_-prefixed function which will be handled by clang's
20 // CodeGen library.
21 //
22 // See also the documentation in include/clang/Basic/arm_sve.td.
23 //
24 //===----------------------------------------------------------------------===//
25 
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringMap.h"
30 #include "llvm/TableGen/Error.h"
31 #include "llvm/TableGen/Record.h"
32 #include <array>
33 #include <cctype>
34 #include <set>
35 #include <sstream>
36 #include <string>
37 #include <tuple>
38 
39 using namespace llvm;
40 
41 enum ClassKind {
42   ClassNone,
43   ClassS,     // signed/unsigned, e.g., "_s8", "_u8" suffix
44   ClassG,     // Overloaded name without type suffix
45 };
46 
47 enum class ACLEKind { SVE, SME };
48 
49 using TypeSpec = std::string;
50 
51 namespace {
52 
53 class ImmCheck {
54   unsigned Arg;
55   unsigned Kind;
56   unsigned ElementSizeInBits;
57 
58 public:
ImmCheck(unsigned Arg,unsigned Kind,unsigned ElementSizeInBits=0)59   ImmCheck(unsigned Arg, unsigned Kind, unsigned ElementSizeInBits = 0)
60       : Arg(Arg), Kind(Kind), ElementSizeInBits(ElementSizeInBits) {}
61   ImmCheck(const ImmCheck &Other) = default;
62   ~ImmCheck() = default;
63 
getArg() const64   unsigned getArg() const { return Arg; }
getKind() const65   unsigned getKind() const { return Kind; }
getElementSizeInBits() const66   unsigned getElementSizeInBits() const { return ElementSizeInBits; }
67 };
68 
69 class SVEType {
70   bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat;
71   bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
72       Svcount;
73   unsigned Bitwidth, ElementBitwidth, NumVectors;
74 
75 public:
SVEType()76   SVEType() : SVEType("", 'v') {}
77 
SVEType(StringRef TS,char CharMod,unsigned NumVectors=1)78   SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
79       : Float(false), Signed(true), Immediate(false), Void(false),
80         Constant(false), Pointer(false), BFloat(false), DefaultType(false),
81         IsScalable(true), Predicate(false), PredicatePattern(false),
82         PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U),
83         NumVectors(NumVectors) {
84     if (!TS.empty())
85       applyTypespec(TS);
86     applyModifier(CharMod);
87   }
88 
SVEType(const SVEType & Base,unsigned NumV)89   SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
90     NumVectors = NumV;
91   }
92 
isPointer() const93   bool isPointer() const { return Pointer; }
isVoidPointer() const94   bool isVoidPointer() const { return Pointer && Void; }
isSigned() const95   bool isSigned() const { return Signed; }
isImmediate() const96   bool isImmediate() const { return Immediate; }
isScalar() const97   bool isScalar() const { return NumVectors == 0; }
isVector() const98   bool isVector() const { return NumVectors > 0; }
isScalableVector() const99   bool isScalableVector() const { return isVector() && IsScalable; }
isFixedLengthVector() const100   bool isFixedLengthVector() const { return isVector() && !IsScalable; }
isChar() const101   bool isChar() const { return ElementBitwidth == 8; }
isVoid() const102   bool isVoid() const { return Void & !Pointer; }
isDefault() const103   bool isDefault() const { return DefaultType; }
isFloat() const104   bool isFloat() const { return Float && !BFloat; }
isBFloat() const105   bool isBFloat() const { return BFloat && !Float; }
isFloatingPoint() const106   bool isFloatingPoint() const { return Float || BFloat; }
isInteger() const107   bool isInteger() const {
108     return !isFloatingPoint() && !Predicate && !Svcount;
109   }
isScalarPredicate() const110   bool isScalarPredicate() const {
111     return !isFloatingPoint() && Predicate && NumVectors == 0;
112   }
isPredicateVector() const113   bool isPredicateVector() const { return Predicate; }
isPredicatePattern() const114   bool isPredicatePattern() const { return PredicatePattern; }
isPrefetchOp() const115   bool isPrefetchOp() const { return PrefetchOp; }
isSvcount() const116   bool isSvcount() const { return Svcount; }
isConstant() const117   bool isConstant() const { return Constant; }
getElementSizeInBits() const118   unsigned getElementSizeInBits() const { return ElementBitwidth; }
getNumVectors() const119   unsigned getNumVectors() const { return NumVectors; }
120 
getNumElements() const121   unsigned getNumElements() const {
122     assert(ElementBitwidth != ~0U);
123     return Bitwidth / ElementBitwidth;
124   }
getSizeInBits() const125   unsigned getSizeInBits() const {
126     return Bitwidth;
127   }
128 
129   /// Return the string representation of a type, which is an encoded
130   /// string for passing to the BUILTIN() macro in Builtins.def.
131   std::string builtin_str() const;
132 
133   /// Return the C/C++ string representation of a type for use in the
134   /// arm_sve.h header file.
135   std::string str() const;
136 
137 private:
138   /// Creates the type based on the typespec string in TS.
139   void applyTypespec(StringRef TS);
140 
141   /// Applies a prototype modifier to the type.
142   void applyModifier(char Mod);
143 };
144 
145 class SVEEmitter;
146 
147 /// The main grunt class. This represents an instantiation of an intrinsic with
148 /// a particular typespec and prototype.
149 class Intrinsic {
150   /// The unmangled name.
151   std::string Name;
152 
153   /// The name of the corresponding LLVM IR intrinsic.
154   std::string LLVMName;
155 
156   /// Intrinsic prototype.
157   std::string Proto;
158 
159   /// The base type spec for this intrinsic.
160   TypeSpec BaseTypeSpec;
161 
162   /// The base class kind. Most intrinsics use ClassS, which has full type
163   /// info for integers (_s32/_u32), or ClassG which is used for overloaded
164   /// intrinsics.
165   ClassKind Class;
166 
167   /// The architectural #ifdef guard.
168   std::string Guard;
169 
170   // The merge suffix such as _m, _x or _z.
171   std::string MergeSuffix;
172 
173   /// The types of return value [0] and parameters [1..].
174   std::vector<SVEType> Types;
175 
176   /// The "base type", which is VarType('d', BaseTypeSpec).
177   SVEType BaseType;
178 
179   uint64_t Flags;
180 
181   SmallVector<ImmCheck, 2> ImmChecks;
182 
183 public:
184   Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
185             StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
186             uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
187             ClassKind Class, SVEEmitter &Emitter, StringRef Guard);
188 
189   ~Intrinsic()=default;
190 
getName() const191   std::string getName() const { return Name; }
getLLVMName() const192   std::string getLLVMName() const { return LLVMName; }
getProto() const193   std::string getProto() const { return Proto; }
getBaseTypeSpec() const194   TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
getBaseType() const195   SVEType getBaseType() const { return BaseType; }
196 
getGuard() const197   StringRef getGuard() const { return Guard; }
getClassKind() const198   ClassKind getClassKind() const { return Class; }
199 
getReturnType() const200   SVEType getReturnType() const { return Types[0]; }
getTypes() const201   ArrayRef<SVEType> getTypes() const { return Types; }
getParamType(unsigned I) const202   SVEType getParamType(unsigned I) const { return Types[I + 1]; }
getNumParams() const203   unsigned getNumParams() const {
204     return Proto.size() - (2 * llvm::count(Proto, '.')) - 1;
205   }
206 
getFlags() const207   uint64_t getFlags() const { return Flags; }
isFlagSet(uint64_t Flag) const208   bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
209 
getImmChecks() const210   ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
211 
212   /// Return the type string for a BUILTIN() macro in Builtins.def.
213   std::string getBuiltinTypeStr();
214 
215   /// Return the name, mangled with type information. The name is mangled for
216   /// ClassS, so will add type suffixes such as _u32/_s32.
getMangledName() const217   std::string getMangledName() const { return mangleName(ClassS); }
218 
219   /// As above, but mangles the LLVM name instead.
getMangledLLVMName() const220   std::string getMangledLLVMName() const { return mangleLLVMName(); }
221 
222   /// Returns true if the intrinsic is overloaded, in that it should also generate
223   /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
224   /// 'svld1_u32(..)'.
isOverloadedIntrinsic(StringRef Name)225   static bool isOverloadedIntrinsic(StringRef Name) {
226     auto BrOpen = Name.find('[');
227     auto BrClose = Name.find(']');
228     return BrOpen != std::string::npos && BrClose != std::string::npos;
229   }
230 
231   /// Return true if the intrinsic takes a splat operand.
hasSplat() const232   bool hasSplat() const {
233     // These prototype modifiers are described in arm_sve.td.
234     return Proto.find_first_of("ajfrKLR@") != std::string::npos;
235   }
236 
237   /// Return the parameter index of the splat operand.
getSplatIdx() const238   unsigned getSplatIdx() const {
239     unsigned I = 1, Param = 0;
240     for (; I < Proto.size(); ++I, ++Param) {
241       if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
242           Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
243           Proto[I] == 'R' || Proto[I] == '@')
244         break;
245 
246       // Multivector modifier can be skipped
247       if (Proto[I] == '.')
248         I += 2;
249     }
250     assert(I != Proto.size() && "Prototype has no splat operand");
251     return Param;
252   }
253 
254   /// Emits the intrinsic declaration to the ostream.
255   void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
256 
257 private:
getMergeSuffix() const258   std::string getMergeSuffix() const { return MergeSuffix; }
259   std::string mangleName(ClassKind LocalCK) const;
260   std::string mangleLLVMName() const;
261   std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
262                                    std::string Proto) const;
263 };
264 
265 class SVEEmitter {
266 private:
267   // The reinterpret builtins are generated separately because they
268   // need the cross product of all types (121 functions in total),
269   // which is inconvenient to specify in the arm_sve.td file or
270   // generate in CGBuiltin.cpp.
271   struct ReinterpretTypeInfo {
272     SVEType BaseType;
273     const char *Suffix;
274   };
275 
276   static const std::array<ReinterpretTypeInfo, 12> Reinterprets;
277 
278   RecordKeeper &Records;
279   llvm::StringMap<uint64_t> EltTypes;
280   llvm::StringMap<uint64_t> MemEltTypes;
281   llvm::StringMap<uint64_t> FlagTypes;
282   llvm::StringMap<uint64_t> MergeTypes;
283   llvm::StringMap<uint64_t> ImmCheckTypes;
284 
285 public:
SVEEmitter(RecordKeeper & R)286   SVEEmitter(RecordKeeper &R) : Records(R) {
287     for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
288       EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
289     for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
290       MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
291     for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
292       FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
293     for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
294       MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
295     for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
296       ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
297   }
298 
299   /// Returns the enum value for the immcheck type
getEnumValueForImmCheck(StringRef C) const300   unsigned getEnumValueForImmCheck(StringRef C) const {
301     auto It = ImmCheckTypes.find(C);
302     if (It != ImmCheckTypes.end())
303       return It->getValue();
304     llvm_unreachable("Unsupported imm check");
305   }
306 
307   /// Returns the enum value for the flag type
getEnumValueForFlag(StringRef C) const308   uint64_t getEnumValueForFlag(StringRef C) const {
309     auto Res = FlagTypes.find(C);
310     if (Res != FlagTypes.end())
311       return Res->getValue();
312     llvm_unreachable("Unsupported flag");
313   }
314 
315   // Returns the SVETypeFlags for a given value and mask.
encodeFlag(uint64_t V,StringRef MaskName) const316   uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
317     auto It = FlagTypes.find(MaskName);
318     if (It != FlagTypes.end()) {
319       uint64_t Mask = It->getValue();
320       unsigned Shift = llvm::countr_zero(Mask);
321       assert(Shift < 64 && "Mask value produced an invalid shift value");
322       return (V << Shift) & Mask;
323     }
324     llvm_unreachable("Unsupported flag");
325   }
326 
327   // Returns the SVETypeFlags for the given element type.
encodeEltType(StringRef EltName)328   uint64_t encodeEltType(StringRef EltName) {
329     auto It = EltTypes.find(EltName);
330     if (It != EltTypes.end())
331       return encodeFlag(It->getValue(), "EltTypeMask");
332     llvm_unreachable("Unsupported EltType");
333   }
334 
335   // Returns the SVETypeFlags for the given memory element type.
encodeMemoryElementType(uint64_t MT)336   uint64_t encodeMemoryElementType(uint64_t MT) {
337     return encodeFlag(MT, "MemEltTypeMask");
338   }
339 
340   // Returns the SVETypeFlags for the given merge type.
encodeMergeType(uint64_t MT)341   uint64_t encodeMergeType(uint64_t MT) {
342     return encodeFlag(MT, "MergeTypeMask");
343   }
344 
345   // Returns the SVETypeFlags for the given splat operand.
encodeSplatOperand(unsigned SplatIdx)346   unsigned encodeSplatOperand(unsigned SplatIdx) {
347     assert(SplatIdx < 7 && "SplatIdx out of encodable range");
348     return encodeFlag(SplatIdx + 1, "SplatOperandMask");
349   }
350 
351   // Returns the SVETypeFlags value for the given SVEType.
352   uint64_t encodeTypeFlags(const SVEType &T);
353 
354   /// Emit arm_sve.h.
355   void createHeader(raw_ostream &o);
356 
357   // Emits core intrinsics in both arm_sme.h and arm_sve.h
358   void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
359                                   ACLEKind Kind);
360 
361   /// Emit all the __builtin prototypes and code needed by Sema.
362   void createBuiltins(raw_ostream &o);
363 
364   /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
365   void createCodeGenMap(raw_ostream &o);
366 
367   /// Emit all the range checks for the immediates.
368   void createRangeChecks(raw_ostream &o);
369 
370   /// Create the SVETypeFlags used in CGBuiltins
371   void createTypeFlags(raw_ostream &o);
372 
373   /// Emit arm_sme.h.
374   void createSMEHeader(raw_ostream &o);
375 
376   /// Emit all the SME __builtin prototypes and code needed by Sema.
377   void createSMEBuiltins(raw_ostream &o);
378 
379   /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
380   void createSMECodeGenMap(raw_ostream &o);
381 
382   /// Create a table for a builtin's requirement for PSTATE.SM.
383   void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
384 
385   /// Emit all the range checks for the immediates.
386   void createSMERangeChecks(raw_ostream &o);
387 
388   /// Create a table for a builtin's requirement for PSTATE.ZA.
389   void createBuiltinZAState(raw_ostream &OS);
390 
391   /// Create intrinsic and add it to \p Out
392   void createIntrinsic(Record *R,
393                        SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
394 };
395 
396 const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
397     {{{SVEType("c", 'd'), "s8"},
398       {SVEType("Uc", 'd'), "u8"},
399       {SVEType("s", 'd'), "s16"},
400       {SVEType("Us", 'd'), "u16"},
401       {SVEType("i", 'd'), "s32"},
402       {SVEType("Ui", 'd'), "u32"},
403       {SVEType("l", 'd'), "s64"},
404       {SVEType("Ul", 'd'), "u64"},
405       {SVEType("h", 'd'), "f16"},
406       {SVEType("b", 'd'), "bf16"},
407       {SVEType("f", 'd'), "f32"},
408       {SVEType("d", 'd'), "f64"}}};
409 
410 } // end anonymous namespace
411 
412 
413 //===----------------------------------------------------------------------===//
414 // Type implementation
415 //===----------------------------------------------------------------------===//
416 
builtin_str() const417 std::string SVEType::builtin_str() const {
418   std::string S;
419   if (isVoid())
420     return "v";
421 
422   if (isScalarPredicate())
423     return "b";
424 
425   if (isSvcount())
426     return "Qa";
427 
428   if (isVoidPointer())
429     S += "v";
430   else if (!isFloatingPoint())
431     switch (ElementBitwidth) {
432     case 1: S += "b"; break;
433     case 8: S += "c"; break;
434     case 16: S += "s"; break;
435     case 32: S += "i"; break;
436     case 64: S += "Wi"; break;
437     case 128: S += "LLLi"; break;
438     default: llvm_unreachable("Unhandled case!");
439     }
440   else if (isFloat())
441     switch (ElementBitwidth) {
442     case 16: S += "h"; break;
443     case 32: S += "f"; break;
444     case 64: S += "d"; break;
445     default: llvm_unreachable("Unhandled case!");
446     }
447   else if (isBFloat()) {
448     assert(ElementBitwidth == 16 && "Not a valid BFloat.");
449     S += "y";
450   }
451 
452   if (!isFloatingPoint()) {
453     if ((isChar() || isPointer()) && !isVoidPointer()) {
454       // Make chars and typed pointers explicitly signed.
455       if (Signed)
456         S = "S" + S;
457       else if (!Signed)
458         S = "U" + S;
459     } else if (!isVoidPointer() && !Signed) {
460       S = "U" + S;
461     }
462   }
463 
464   // Constant indices are "int", but have the "constant expression" modifier.
465   if (isImmediate()) {
466     assert(!isFloat() && "fp immediates are not supported");
467     S = "I" + S;
468   }
469 
470   if (isScalar()) {
471     if (Constant) S += "C";
472     if (Pointer) S += "*";
473     return S;
474   }
475 
476   if (isFixedLengthVector())
477     return "V" + utostr(getNumElements() * NumVectors) + S;
478   return "q" + utostr(getNumElements() * NumVectors) + S;
479 }
480 
str() const481 std::string SVEType::str() const {
482   if (isPredicatePattern())
483     return "enum svpattern";
484 
485   if (isPrefetchOp())
486     return "enum svprfop";
487 
488   std::string S;
489   if (Void)
490     S += "void";
491   else {
492     if (isScalableVector() || isSvcount())
493       S += "sv";
494     if (!Signed && !isFloatingPoint())
495       S += "u";
496 
497     if (Float)
498       S += "float";
499     else if (isSvcount())
500       S += "count";
501     else if (isScalarPredicate() || isPredicateVector())
502       S += "bool";
503     else if (isBFloat())
504       S += "bfloat";
505     else
506       S += "int";
507 
508     if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
509       S += utostr(ElementBitwidth);
510     if (isFixedLengthVector())
511       S += "x" + utostr(getNumElements());
512     if (NumVectors > 1)
513       S += "x" + utostr(NumVectors);
514     if (!isScalarPredicate())
515       S += "_t";
516   }
517 
518   if (Constant)
519     S += " const";
520   if (Pointer)
521     S += " *";
522 
523   return S;
524 }
525 
applyTypespec(StringRef TS)526 void SVEType::applyTypespec(StringRef TS) {
527   for (char I : TS) {
528     switch (I) {
529     case 'Q':
530       Svcount = true;
531       break;
532     case 'P':
533       Predicate = true;
534       break;
535     case 'U':
536       Signed = false;
537       break;
538     case 'c':
539       ElementBitwidth = 8;
540       break;
541     case 's':
542       ElementBitwidth = 16;
543       break;
544     case 'i':
545       ElementBitwidth = 32;
546       break;
547     case 'l':
548       ElementBitwidth = 64;
549       break;
550     case 'q':
551       ElementBitwidth = 128;
552       break;
553     case 'h':
554       Float = true;
555       ElementBitwidth = 16;
556       break;
557     case 'f':
558       Float = true;
559       ElementBitwidth = 32;
560       break;
561     case 'd':
562       Float = true;
563       ElementBitwidth = 64;
564       break;
565     case 'b':
566       BFloat = true;
567       Float = false;
568       ElementBitwidth = 16;
569       break;
570     default:
571       llvm_unreachable("Unhandled type code!");
572     }
573   }
574   assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
575 }
576 
applyModifier(char Mod)577 void SVEType::applyModifier(char Mod) {
578   switch (Mod) {
579   case 'v':
580     Void = true;
581     break;
582   case 'd':
583     DefaultType = true;
584     break;
585   case 'c':
586     Constant = true;
587     [[fallthrough]];
588   case 'p':
589     Pointer = true;
590     Bitwidth = ElementBitwidth;
591     NumVectors = 0;
592     break;
593   case 'e':
594     Signed = false;
595     ElementBitwidth /= 2;
596     break;
597   case 'h':
598     ElementBitwidth /= 2;
599     break;
600   case 'q':
601     ElementBitwidth /= 4;
602     break;
603   case 'b':
604     Signed = false;
605     Float = false;
606     BFloat = false;
607     ElementBitwidth /= 4;
608     break;
609   case 'o':
610     ElementBitwidth *= 4;
611     break;
612   case 'P':
613     Signed = true;
614     Float = false;
615     BFloat = false;
616     Predicate = true;
617     Svcount = false;
618     Bitwidth = 16;
619     ElementBitwidth = 1;
620     break;
621   case '{':
622     IsScalable = false;
623     Bitwidth = 128;
624     NumVectors = 1;
625     break;
626   case 's':
627   case 'a':
628     Bitwidth = ElementBitwidth;
629     NumVectors = 0;
630     break;
631   case 'R':
632     ElementBitwidth /= 2;
633     NumVectors = 0;
634     break;
635   case 'r':
636     ElementBitwidth /= 4;
637     NumVectors = 0;
638     break;
639   case '@':
640     Signed = false;
641     Float = false;
642     BFloat = false;
643     ElementBitwidth /= 4;
644     NumVectors = 0;
645     break;
646   case 'K':
647     Signed = true;
648     Float = false;
649     BFloat = false;
650     Bitwidth = ElementBitwidth;
651     NumVectors = 0;
652     break;
653   case 'L':
654     Signed = false;
655     Float = false;
656     BFloat = false;
657     Bitwidth = ElementBitwidth;
658     NumVectors = 0;
659     break;
660   case 'u':
661     Predicate = false;
662     Svcount = false;
663     Signed = false;
664     Float = false;
665     BFloat = false;
666     break;
667   case 'x':
668     Predicate = false;
669     Svcount = false;
670     Signed = true;
671     Float = false;
672     BFloat = false;
673     break;
674   case 'i':
675     Predicate = false;
676     Svcount = false;
677     Float = false;
678     BFloat = false;
679     ElementBitwidth = Bitwidth = 64;
680     NumVectors = 0;
681     Signed = false;
682     Immediate = true;
683     break;
684   case 'I':
685     Predicate = false;
686     Svcount = false;
687     Float = false;
688     BFloat = false;
689     ElementBitwidth = Bitwidth = 32;
690     NumVectors = 0;
691     Signed = true;
692     Immediate = true;
693     PredicatePattern = true;
694     break;
695   case 'J':
696     Predicate = false;
697     Svcount = false;
698     Float = false;
699     BFloat = false;
700     ElementBitwidth = Bitwidth = 32;
701     NumVectors = 0;
702     Signed = true;
703     Immediate = true;
704     PrefetchOp = true;
705     break;
706   case 'k':
707     Predicate = false;
708     Svcount = false;
709     Signed = true;
710     Float = false;
711     BFloat = false;
712     ElementBitwidth = Bitwidth = 32;
713     NumVectors = 0;
714     break;
715   case 'l':
716     Predicate = false;
717     Svcount = false;
718     Signed = true;
719     Float = false;
720     BFloat = false;
721     ElementBitwidth = Bitwidth = 64;
722     NumVectors = 0;
723     break;
724   case 'm':
725     Predicate = false;
726     Svcount = false;
727     Signed = false;
728     Float = false;
729     BFloat = false;
730     ElementBitwidth = Bitwidth = 32;
731     NumVectors = 0;
732     break;
733   case 'n':
734     Predicate = false;
735     Svcount = false;
736     Signed = false;
737     Float = false;
738     BFloat = false;
739     ElementBitwidth = Bitwidth = 64;
740     NumVectors = 0;
741     break;
742   case 'w':
743     ElementBitwidth = 64;
744     break;
745   case 'j':
746     ElementBitwidth = Bitwidth = 64;
747     NumVectors = 0;
748     break;
749   case 'f':
750     Signed = false;
751     ElementBitwidth = Bitwidth = 64;
752     NumVectors = 0;
753     break;
754   case 'g':
755     Signed = false;
756     Float = false;
757     BFloat = false;
758     ElementBitwidth = 64;
759     break;
760   case '[':
761     Signed = false;
762     Float = false;
763     BFloat = false;
764     ElementBitwidth = 8;
765     break;
766   case 't':
767     Signed = true;
768     Float = false;
769     BFloat = false;
770     ElementBitwidth = 32;
771     break;
772   case 'z':
773     Signed = false;
774     Float = false;
775     BFloat = false;
776     ElementBitwidth = 32;
777     break;
778   case 'O':
779     Predicate = false;
780     Svcount = false;
781     Float = true;
782     ElementBitwidth = 16;
783     break;
784   case 'M':
785     Predicate = false;
786     Svcount = false;
787     Float = true;
788     BFloat = false;
789     ElementBitwidth = 32;
790     break;
791   case 'N':
792     Predicate = false;
793     Svcount = false;
794     Float = true;
795     ElementBitwidth = 64;
796     break;
797   case 'Q':
798     Constant = true;
799     Pointer = true;
800     Void = true;
801     NumVectors = 0;
802     break;
803   case 'S':
804     Constant = true;
805     Pointer = true;
806     ElementBitwidth = Bitwidth = 8;
807     NumVectors = 0;
808     Signed = true;
809     break;
810   case 'W':
811     Constant = true;
812     Pointer = true;
813     ElementBitwidth = Bitwidth = 8;
814     NumVectors = 0;
815     Signed = false;
816     break;
817   case 'T':
818     Constant = true;
819     Pointer = true;
820     ElementBitwidth = Bitwidth = 16;
821     NumVectors = 0;
822     Signed = true;
823     break;
824   case 'X':
825     Constant = true;
826     Pointer = true;
827     ElementBitwidth = Bitwidth = 16;
828     NumVectors = 0;
829     Signed = false;
830     break;
831   case 'Y':
832     Constant = true;
833     Pointer = true;
834     ElementBitwidth = Bitwidth = 32;
835     NumVectors = 0;
836     Signed = false;
837     break;
838   case 'U':
839     Constant = true;
840     Pointer = true;
841     ElementBitwidth = Bitwidth = 32;
842     NumVectors = 0;
843     Signed = true;
844     break;
845   case '%':
846     Pointer = true;
847     Void = true;
848     NumVectors = 0;
849     break;
850   case 'A':
851     Pointer = true;
852     ElementBitwidth = Bitwidth = 8;
853     NumVectors = 0;
854     Signed = true;
855     break;
856   case 'B':
857     Pointer = true;
858     ElementBitwidth = Bitwidth = 16;
859     NumVectors = 0;
860     Signed = true;
861     break;
862   case 'C':
863     Pointer = true;
864     ElementBitwidth = Bitwidth = 32;
865     NumVectors = 0;
866     Signed = true;
867     break;
868   case 'D':
869     Pointer = true;
870     ElementBitwidth = Bitwidth = 64;
871     NumVectors = 0;
872     Signed = true;
873     break;
874   case 'E':
875     Pointer = true;
876     ElementBitwidth = Bitwidth = 8;
877     NumVectors = 0;
878     Signed = false;
879     break;
880   case 'F':
881     Pointer = true;
882     ElementBitwidth = Bitwidth = 16;
883     NumVectors = 0;
884     Signed = false;
885     break;
886   case 'G':
887     Pointer = true;
888     ElementBitwidth = Bitwidth = 32;
889     NumVectors = 0;
890     Signed = false;
891     break;
892   case '$':
893     Predicate = false;
894     Svcount = false;
895     Float = false;
896     BFloat = true;
897     ElementBitwidth = 16;
898     break;
899   case '}':
900     Predicate = false;
901     Signed = true;
902     Svcount = true;
903     NumVectors = 0;
904     Float = false;
905     BFloat = false;
906     break;
907   case '.':
908     llvm_unreachable(". is never a type in itself");
909     break;
910   default:
911     llvm_unreachable("Unhandled character!");
912   }
913 }
914 
915 /// Returns the modifier and number of vectors for the given operand \p Op.
getProtoModifier(StringRef Proto,unsigned Op)916 std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
917   for (unsigned P = 0; !Proto.empty(); ++P) {
918     unsigned NumVectors = 1;
919     unsigned CharsToSkip = 1;
920     char Mod = Proto[0];
921     if (Mod == '2' || Mod == '3' || Mod == '4') {
922       NumVectors = Mod - '0';
923       Mod = 'd';
924       if (Proto.size() > 1 && Proto[1] == '.') {
925         Mod = Proto[2];
926         CharsToSkip = 3;
927       }
928     }
929 
930     if (P == Op)
931       return {Mod, NumVectors};
932 
933     Proto = Proto.drop_front(CharsToSkip);
934   }
935   llvm_unreachable("Unexpected Op");
936 }
937 
938 //===----------------------------------------------------------------------===//
939 // Intrinsic implementation
940 //===----------------------------------------------------------------------===//
941 
Intrinsic(StringRef Name,StringRef Proto,uint64_t MergeTy,StringRef MergeSuffix,uint64_t MemoryElementTy,StringRef LLVMName,uint64_t Flags,ArrayRef<ImmCheck> Checks,TypeSpec BT,ClassKind Class,SVEEmitter & Emitter,StringRef Guard)942 Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
943                      StringRef MergeSuffix, uint64_t MemoryElementTy,
944                      StringRef LLVMName, uint64_t Flags,
945                      ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
946                      SVEEmitter &Emitter, StringRef Guard)
947     : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
948       BaseTypeSpec(BT), Class(Class), Guard(Guard.str()),
949       MergeSuffix(MergeSuffix.str()), BaseType(BT, 'd'), Flags(Flags),
950       ImmChecks(Checks.begin(), Checks.end()) {
951   // Types[0] is the return value.
952   for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
953     char Mod;
954     unsigned NumVectors;
955     std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
956     SVEType T(BaseTypeSpec, Mod, NumVectors);
957     Types.push_back(T);
958 
959     // Add range checks for immediates
960     if (I > 0) {
961       if (T.isPredicatePattern())
962         ImmChecks.emplace_back(
963             I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
964       else if (T.isPrefetchOp())
965         ImmChecks.emplace_back(
966             I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
967     }
968   }
969 
970   // Set flags based on properties
971   this->Flags |= Emitter.encodeTypeFlags(BaseType);
972   this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
973   this->Flags |= Emitter.encodeMergeType(MergeTy);
974   if (hasSplat())
975     this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
976 }
977 
getBuiltinTypeStr()978 std::string Intrinsic::getBuiltinTypeStr() {
979   std::string S = getReturnType().builtin_str();
980   for (unsigned I = 0; I < getNumParams(); ++I)
981     S += getParamType(I).builtin_str();
982 
983   return S;
984 }
985 
replaceTemplatedArgs(std::string Name,TypeSpec TS,std::string Proto) const986 std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
987                                             std::string Proto) const {
988   std::string Ret = Name;
989   while (Ret.find('{') != std::string::npos) {
990     size_t Pos = Ret.find('{');
991     size_t End = Ret.find('}');
992     unsigned NumChars = End - Pos + 1;
993     assert(NumChars == 3 && "Unexpected template argument");
994 
995     SVEType T;
996     char C = Ret[Pos+1];
997     switch(C) {
998     default:
999       llvm_unreachable("Unknown predication specifier");
1000     case 'd':
1001       T = SVEType(TS, 'd');
1002       break;
1003     case '0':
1004     case '1':
1005     case '2':
1006     case '3':
1007       T = SVEType(TS, Proto[C - '0']);
1008       break;
1009     }
1010 
1011     // Replace templated arg with the right suffix (e.g. u32)
1012     std::string TypeCode;
1013     if (T.isInteger())
1014       TypeCode = T.isSigned() ? 's' : 'u';
1015     else if (T.isSvcount())
1016       TypeCode = 'c';
1017     else if (T.isPredicateVector())
1018       TypeCode = 'b';
1019     else if (T.isBFloat())
1020       TypeCode = "bf";
1021     else
1022       TypeCode = 'f';
1023     Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
1024   }
1025 
1026   return Ret;
1027 }
1028 
mangleLLVMName() const1029 std::string Intrinsic::mangleLLVMName() const {
1030   std::string S = getLLVMName();
1031 
1032   // Replace all {d} like expressions with e.g. 'u32'
1033   return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
1034 }
1035 
mangleName(ClassKind LocalCK) const1036 std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1037   std::string S = getName();
1038 
1039   if (LocalCK == ClassG) {
1040     // Remove the square brackets and everything in between.
1041     while (S.find('[') != std::string::npos) {
1042       auto Start = S.find('[');
1043       auto End = S.find(']');
1044       S.erase(Start, (End-Start)+1);
1045     }
1046   } else {
1047     // Remove the square brackets.
1048     while (S.find('[') != std::string::npos) {
1049       auto BrPos = S.find('[');
1050       if (BrPos != std::string::npos)
1051         S.erase(BrPos, 1);
1052       BrPos = S.find(']');
1053       if (BrPos != std::string::npos)
1054         S.erase(BrPos, 1);
1055     }
1056   }
1057 
1058   // Replace all {d} like expressions with e.g. 'u32'
1059   return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
1060          getMergeSuffix();
1061 }
1062 
emitIntrinsic(raw_ostream & OS,SVEEmitter & Emitter,ACLEKind Kind) const1063 void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
1064                               ACLEKind Kind) const {
1065   bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1066 
1067   std::string FullName = mangleName(ClassS);
1068   std::string ProtoName = mangleName(getClassKind());
1069   OS << (IsOverloaded ? "__aio " : "__ai ")
1070      << "__attribute__((__clang_arm_builtin_alias(";
1071 
1072   switch (Kind) {
1073   case ACLEKind::SME:
1074     OS << "__builtin_sme_" << FullName << ")";
1075     break;
1076   case ACLEKind::SVE:
1077     OS << "__builtin_sve_" << FullName << ")";
1078     break;
1079   }
1080 
1081   OS << "))\n";
1082 
1083   OS << getTypes()[0].str() << " " << ProtoName << "(";
1084   for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1085     if (I != 0)
1086       OS << ", ";
1087     OS << getTypes()[I + 1].str();
1088   }
1089   OS << ");\n";
1090 }
1091 
1092 //===----------------------------------------------------------------------===//
1093 // SVEEmitter implementation
1094 //===----------------------------------------------------------------------===//
encodeTypeFlags(const SVEType & T)1095 uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1096   if (T.isFloat()) {
1097     switch (T.getElementSizeInBits()) {
1098     case 16:
1099       return encodeEltType("EltTyFloat16");
1100     case 32:
1101       return encodeEltType("EltTyFloat32");
1102     case 64:
1103       return encodeEltType("EltTyFloat64");
1104     default:
1105       llvm_unreachable("Unhandled float element bitwidth!");
1106     }
1107   }
1108 
1109   if (T.isBFloat()) {
1110     assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1111     return encodeEltType("EltTyBFloat16");
1112   }
1113 
1114   if (T.isPredicateVector() || T.isSvcount()) {
1115     switch (T.getElementSizeInBits()) {
1116     case 8:
1117       return encodeEltType("EltTyBool8");
1118     case 16:
1119       return encodeEltType("EltTyBool16");
1120     case 32:
1121       return encodeEltType("EltTyBool32");
1122     case 64:
1123       return encodeEltType("EltTyBool64");
1124     default:
1125       llvm_unreachable("Unhandled predicate element bitwidth!");
1126     }
1127   }
1128 
1129   switch (T.getElementSizeInBits()) {
1130   case 8:
1131     return encodeEltType("EltTyInt8");
1132   case 16:
1133     return encodeEltType("EltTyInt16");
1134   case 32:
1135     return encodeEltType("EltTyInt32");
1136   case 64:
1137     return encodeEltType("EltTyInt64");
1138   case 128:
1139     return encodeEltType("EltTyInt128");
1140   default:
1141     llvm_unreachable("Unhandled integer element bitwidth!");
1142   }
1143 }
1144 
createIntrinsic(Record * R,SmallVectorImpl<std::unique_ptr<Intrinsic>> & Out)1145 void SVEEmitter::createIntrinsic(
1146     Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1147   StringRef Name = R->getValueAsString("Name");
1148   StringRef Proto = R->getValueAsString("Prototype");
1149   StringRef Types = R->getValueAsString("Types");
1150   StringRef Guard = R->getValueAsString("TargetGuard");
1151   StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
1152   uint64_t Merge = R->getValueAsInt("Merge");
1153   StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
1154   uint64_t MemEltType = R->getValueAsInt("MemEltType");
1155   std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags");
1156   std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");
1157 
1158   int64_t Flags = 0;
1159   for (auto FlagRec : FlagsList)
1160     Flags |= FlagRec->getValueAsInt("Value");
1161 
1162   // Create a dummy TypeSpec for non-overloaded builtins.
1163   if (Types.empty()) {
1164     assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1165            "Expect TypeSpec for overloaded builtin!");
1166     Types = "i";
1167   }
1168 
1169   // Extract type specs from string
1170   SmallVector<TypeSpec, 8> TypeSpecs;
1171   TypeSpec Acc;
1172   for (char I : Types) {
1173     Acc.push_back(I);
1174     if (islower(I)) {
1175       TypeSpecs.push_back(TypeSpec(Acc));
1176       Acc.clear();
1177     }
1178   }
1179 
1180   // Remove duplicate type specs.
1181   llvm::sort(TypeSpecs);
1182   TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
1183                   TypeSpecs.end());
1184 
1185   // Create an Intrinsic for each type spec.
1186   for (auto TS : TypeSpecs) {
1187     // Collate a list of range/option checks for the immediates.
1188     SmallVector<ImmCheck, 2> ImmChecks;
1189     for (auto *R : ImmCheckList) {
1190       int64_t Arg = R->getValueAsInt("Arg");
1191       int64_t EltSizeArg = R->getValueAsInt("EltSizeArg");
1192       int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value");
1193       assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative");
1194 
1195       unsigned ElementSizeInBits = 0;
1196       char Mod;
1197       unsigned NumVectors;
1198       std::tie(Mod, NumVectors) = getProtoModifier(Proto, EltSizeArg + 1);
1199       if (EltSizeArg >= 0)
1200         ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1201       ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits));
1202     }
1203 
1204     Out.push_back(std::make_unique<Intrinsic>(
1205         Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
1206         TS, ClassS, *this, Guard));
1207 
1208     // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1209     if (Intrinsic::isOverloadedIntrinsic(Name))
1210       Out.push_back(std::make_unique<Intrinsic>(
1211           Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
1212           ImmChecks, TS, ClassG, *this, Guard));
1213   }
1214 }
1215 
createCoreHeaderIntrinsics(raw_ostream & OS,SVEEmitter & Emitter,ACLEKind Kind)1216 void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
1217                                             SVEEmitter &Emitter,
1218                                             ACLEKind Kind) {
1219   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1220   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1221   for (auto *R : RV)
1222     createIntrinsic(R, Defs);
1223 
1224   // Sort intrinsics in header file by following order/priority:
1225   // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1226   // - Class (is intrinsic overloaded or not)
1227   // - Intrinsic name
1228   std::stable_sort(Defs.begin(), Defs.end(),
1229                    [](const std::unique_ptr<Intrinsic> &A,
1230                       const std::unique_ptr<Intrinsic> &B) {
1231                      auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1232                        return std::make_tuple(I->getGuard(),
1233                                               (unsigned)I->getClassKind(),
1234                                               I->getName());
1235                      };
1236                      return ToTuple(A) < ToTuple(B);
1237                    });
1238 
1239   // Actually emit the intrinsic declarations.
1240   for (auto &I : Defs)
1241     I->emitIntrinsic(OS, Emitter, Kind);
1242 }
1243 
createHeader(raw_ostream & OS)1244 void SVEEmitter::createHeader(raw_ostream &OS) {
1245   OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1246         "-----------------------------------===\n"
1247         " *\n"
1248         " *\n"
1249         " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1250         "Exceptions.\n"
1251         " * See https://llvm.org/LICENSE.txt for license information.\n"
1252         " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1253         " *\n"
1254         " *===-----------------------------------------------------------------"
1255         "------===\n"
1256         " */\n\n";
1257 
1258   OS << "#ifndef __ARM_SVE_H\n";
1259   OS << "#define __ARM_SVE_H\n\n";
1260 
1261   OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1262   OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1263   OS << "#endif\n";
1264 
1265   OS << "#include <stdint.h>\n\n";
1266   OS << "#ifdef  __cplusplus\n";
1267   OS << "extern \"C\" {\n";
1268   OS << "#else\n";
1269   OS << "#include <stdbool.h>\n";
1270   OS << "#endif\n\n";
1271 
1272   OS << "typedef __fp16 float16_t;\n";
1273   OS << "typedef float float32_t;\n";
1274   OS << "typedef double float64_t;\n";
1275 
1276   OS << "typedef __SVInt8_t svint8_t;\n";
1277   OS << "typedef __SVInt16_t svint16_t;\n";
1278   OS << "typedef __SVInt32_t svint32_t;\n";
1279   OS << "typedef __SVInt64_t svint64_t;\n";
1280   OS << "typedef __SVUint8_t svuint8_t;\n";
1281   OS << "typedef __SVUint16_t svuint16_t;\n";
1282   OS << "typedef __SVUint32_t svuint32_t;\n";
1283   OS << "typedef __SVUint64_t svuint64_t;\n";
1284   OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1285 
1286   OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1287 
1288   OS << "#include <arm_bf16.h>\n";
1289   OS << "#include <arm_vector_types.h>\n";
1290 
1291   OS << "typedef __SVFloat32_t svfloat32_t;\n";
1292   OS << "typedef __SVFloat64_t svfloat64_t;\n";
1293   OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1294   OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1295   OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1296   OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1297   OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1298   OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1299   OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1300   OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1301   OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1302   OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1303   OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1304   OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1305   OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1306   OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1307   OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1308   OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1309   OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1310   OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1311   OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1312   OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1313   OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1314   OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1315   OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1316   OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1317   OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1318   OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1319   OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1320   OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1321   OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1322   OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1323   OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1324   OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1325   OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1326   OS << "typedef __SVBool_t  svbool_t;\n";
1327   OS << "typedef __clang_svboolx2_t  svboolx2_t;\n";
1328   OS << "typedef __clang_svboolx4_t  svboolx4_t;\n\n";
1329 
1330   OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1331   OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1332   OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1333 
1334   OS << "typedef __SVCount_t svcount_t;\n\n";
1335 
1336   OS << "enum svpattern\n";
1337   OS << "{\n";
1338   OS << "  SV_POW2 = 0,\n";
1339   OS << "  SV_VL1 = 1,\n";
1340   OS << "  SV_VL2 = 2,\n";
1341   OS << "  SV_VL3 = 3,\n";
1342   OS << "  SV_VL4 = 4,\n";
1343   OS << "  SV_VL5 = 5,\n";
1344   OS << "  SV_VL6 = 6,\n";
1345   OS << "  SV_VL7 = 7,\n";
1346   OS << "  SV_VL8 = 8,\n";
1347   OS << "  SV_VL16 = 9,\n";
1348   OS << "  SV_VL32 = 10,\n";
1349   OS << "  SV_VL64 = 11,\n";
1350   OS << "  SV_VL128 = 12,\n";
1351   OS << "  SV_VL256 = 13,\n";
1352   OS << "  SV_MUL4 = 29,\n";
1353   OS << "  SV_MUL3 = 30,\n";
1354   OS << "  SV_ALL = 31\n";
1355   OS << "};\n\n";
1356 
1357   OS << "enum svprfop\n";
1358   OS << "{\n";
1359   OS << "  SV_PLDL1KEEP = 0,\n";
1360   OS << "  SV_PLDL1STRM = 1,\n";
1361   OS << "  SV_PLDL2KEEP = 2,\n";
1362   OS << "  SV_PLDL2STRM = 3,\n";
1363   OS << "  SV_PLDL3KEEP = 4,\n";
1364   OS << "  SV_PLDL3STRM = 5,\n";
1365   OS << "  SV_PSTL1KEEP = 8,\n";
1366   OS << "  SV_PSTL1STRM = 9,\n";
1367   OS << "  SV_PSTL2KEEP = 10,\n";
1368   OS << "  SV_PSTL2STRM = 11,\n";
1369   OS << "  SV_PSTL3KEEP = 12,\n";
1370   OS << "  SV_PSTL3STRM = 13\n";
1371   OS << "};\n\n";
1372 
1373   OS << "/* Function attributes */\n";
1374   OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1375         "__nodebug__))\n\n";
1376   OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1377         "__nodebug__, __overloadable__))\n\n";
1378 
1379   // Add reinterpret functions.
1380   for (auto [N, Suffix] :
1381        std::initializer_list<std::pair<unsigned, const char *>>{
1382            {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1383     for (auto ShortForm : {false, true})
1384       for (const ReinterpretTypeInfo &To : Reinterprets) {
1385         SVEType ToV(To.BaseType, N);
1386         for (const ReinterpretTypeInfo &From : Reinterprets) {
1387           SVEType FromV(From.BaseType, N);
1388           if (ShortForm) {
1389             OS << "__aio __attribute__((target(\"sve\"))) " << ToV.str()
1390                << " svreinterpret_" << To.Suffix;
1391             OS << "(" << FromV.str() << " op) __arm_streaming_compatible {\n";
1392             OS << "  return __builtin_sve_reinterpret_" << To.Suffix << "_"
1393                << From.Suffix << Suffix << "(op);\n";
1394             OS << "}\n\n";
1395           } else
1396             OS << "#define svreinterpret_" << To.Suffix << "_" << From.Suffix
1397                << Suffix << "(...) __builtin_sve_reinterpret_" << To.Suffix
1398                << "_" << From.Suffix << Suffix << "(__VA_ARGS__)\n";
1399         }
1400       }
1401   }
1402 
1403   createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE);
1404 
1405   OS << "#define svcvtnt_bf16_x      svcvtnt_bf16_m\n";
1406   OS << "#define svcvtnt_bf16_f32_x  svcvtnt_bf16_f32_m\n";
1407 
1408   OS << "#define svcvtnt_f16_x      svcvtnt_f16_m\n";
1409   OS << "#define svcvtnt_f16_f32_x  svcvtnt_f16_f32_m\n";
1410   OS << "#define svcvtnt_f32_x      svcvtnt_f32_m\n";
1411   OS << "#define svcvtnt_f32_f64_x  svcvtnt_f32_f64_m\n\n";
1412 
1413   OS << "#define svcvtxnt_f32_x     svcvtxnt_f32_m\n";
1414   OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1415 
1416   OS << "#ifdef __cplusplus\n";
1417   OS << "} // extern \"C\"\n";
1418   OS << "#endif\n\n";
1419   OS << "#undef __ai\n\n";
1420   OS << "#undef __aio\n\n";
1421   OS << "#endif /* __ARM_SVE_H */\n";
1422 }
1423 
createBuiltins(raw_ostream & OS)1424 void SVEEmitter::createBuiltins(raw_ostream &OS) {
1425   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1426   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1427   for (auto *R : RV)
1428     createIntrinsic(R, Defs);
1429 
1430   // The mappings must be sorted based on BuiltinID.
1431   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1432                       const std::unique_ptr<Intrinsic> &B) {
1433     return A->getMangledName() < B->getMangledName();
1434   });
1435 
1436   OS << "#ifdef GET_SVE_BUILTINS\n";
1437   for (auto &Def : Defs) {
1438     // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1439     // declarations only live in the header file.
1440     if (Def->getClassKind() != ClassG)
1441       OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
1442          << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard()
1443          << "\")\n";
1444   }
1445 
1446   // Add reinterpret functions.
1447   for (auto [N, Suffix] :
1448        std::initializer_list<std::pair<unsigned, const char *>>{
1449            {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1450     for (const ReinterpretTypeInfo &To : Reinterprets) {
1451       SVEType ToV(To.BaseType, N);
1452       for (const ReinterpretTypeInfo &From : Reinterprets) {
1453         SVEType FromV(From.BaseType, N);
1454         OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << To.Suffix << "_"
1455            << From.Suffix << Suffix << +", \"" << ToV.builtin_str()
1456            << FromV.builtin_str() << "\", \"n\", \"sve\")\n";
1457       }
1458     }
1459   }
1460 
1461   OS << "#endif\n\n";
1462 }
1463 
createCodeGenMap(raw_ostream & OS)1464 void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1465   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1466   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1467   for (auto *R : RV)
1468     createIntrinsic(R, Defs);
1469 
1470   // The mappings must be sorted based on BuiltinID.
1471   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1472                       const std::unique_ptr<Intrinsic> &B) {
1473     return A->getMangledName() < B->getMangledName();
1474   });
1475 
1476   OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1477   for (auto &Def : Defs) {
1478     // Builtins only exist for non-overloaded intrinsics, overloaded
1479     // declarations only live in the header file.
1480     if (Def->getClassKind() == ClassG)
1481       continue;
1482 
1483     uint64_t Flags = Def->getFlags();
1484     auto FlagString = std::to_string(Flags);
1485 
1486     std::string LLVMName = Def->getMangledLLVMName();
1487     std::string Builtin = Def->getMangledName();
1488     if (!LLVMName.empty())
1489       OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1490          << "),\n";
1491     else
1492       OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1493   }
1494   OS << "#endif\n\n";
1495 }
1496 
createRangeChecks(raw_ostream & OS)1497 void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1498   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1499   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1500   for (auto *R : RV)
1501     createIntrinsic(R, Defs);
1502 
1503   // The mappings must be sorted based on BuiltinID.
1504   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1505                       const std::unique_ptr<Intrinsic> &B) {
1506     return A->getMangledName() < B->getMangledName();
1507   });
1508 
1509 
1510   OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1511 
1512   // Ensure these are only emitted once.
1513   std::set<std::string> Emitted;
1514 
1515   for (auto &Def : Defs) {
1516     if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1517         Def->getImmChecks().empty())
1518       continue;
1519 
1520     OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1521     for (auto &Check : Def->getImmChecks())
1522       OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
1523          << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
1524     OS << "  break;\n";
1525 
1526     Emitted.insert(Def->getMangledName());
1527   }
1528 
1529   OS << "#endif\n\n";
1530 }
1531 
1532 /// Create the SVETypeFlags used in CGBuiltins
createTypeFlags(raw_ostream & OS)1533 void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1534   OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1535   for (auto &KV : FlagTypes)
1536     OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1537   OS << "#endif\n\n";
1538 
1539   OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1540   for (auto &KV : EltTypes)
1541     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1542   OS << "#endif\n\n";
1543 
1544   OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1545   for (auto &KV : MemEltTypes)
1546     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1547   OS << "#endif\n\n";
1548 
1549   OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1550   for (auto &KV : MergeTypes)
1551     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1552   OS << "#endif\n\n";
1553 
1554   OS << "#ifdef LLVM_GET_SVE_IMMCHECKTYPES\n";
1555   for (auto &KV : ImmCheckTypes)
1556     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1557   OS << "#endif\n\n";
1558 }
1559 
createSMEHeader(raw_ostream & OS)1560 void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1561   OS << "/*===---- arm_sme.h - ARM SME intrinsics "
1562         "------===\n"
1563         " *\n"
1564         " *\n"
1565         " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1566         "Exceptions.\n"
1567         " * See https://llvm.org/LICENSE.txt for license information.\n"
1568         " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1569         " *\n"
1570         " *===-----------------------------------------------------------------"
1571         "------===\n"
1572         " */\n\n";
1573 
1574   OS << "#ifndef __ARM_SME_H\n";
1575   OS << "#define __ARM_SME_H\n\n";
1576 
1577   OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1578   OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
1579   OS << "#endif\n";
1580 
1581   OS << "#include <arm_sve.h>\n\n";
1582 
1583   OS << "/* Function attributes */\n";
1584   OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1585         "__nodebug__))\n\n";
1586   OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1587         "__nodebug__, __overloadable__))\n\n";
1588 
1589   OS << "#ifdef  __cplusplus\n";
1590   OS << "extern \"C\" {\n";
1591   OS << "#endif\n\n";
1592 
1593   OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
1594 
1595   OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
1596   OS << "  uint64_t x0, x1;\n";
1597   OS << "  __builtin_arm_get_sme_state(&x0, &x1);\n";
1598   OS << "  return x0 & (1ULL << 63);\n";
1599   OS << "}\n\n";
1600 
1601   OS << "__ai bool __arm_in_streaming_mode(void) __arm_streaming_compatible "
1602         "{\n";
1603   OS << "  uint64_t x0, x1;\n";
1604   OS << "  __builtin_arm_get_sme_state(&x0, &x1);\n";
1605   OS << "  return x0 & 1;\n";
1606   OS << "}\n\n";
1607 
1608   OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
1609         "__arm_streaming_compatible __arm_out(\"za\") "
1610         "{ }\n\n";
1611 
1612   createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME);
1613 
1614   OS << "#ifdef __cplusplus\n";
1615   OS << "} // extern \"C\"\n";
1616   OS << "#endif\n\n";
1617   OS << "#undef __ai\n\n";
1618   OS << "#endif /* __ARM_SME_H */\n";
1619 }
1620 
createSMEBuiltins(raw_ostream & OS)1621 void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1622   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1623   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1624   for (auto *R : RV) {
1625     createIntrinsic(R, Defs);
1626   }
1627 
1628   // The mappings must be sorted based on BuiltinID.
1629   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1630                       const std::unique_ptr<Intrinsic> &B) {
1631     return A->getMangledName() < B->getMangledName();
1632   });
1633 
1634   OS << "#ifdef GET_SME_BUILTINS\n";
1635   for (auto &Def : Defs) {
1636     // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1637     // declarations only live in the header file.
1638     if (Def->getClassKind() != ClassG)
1639       OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \""
1640          << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard()
1641          << "\")\n";
1642   }
1643 
1644   OS << "#endif\n\n";
1645 }
1646 
createSMECodeGenMap(raw_ostream & OS)1647 void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1648   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1649   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1650   for (auto *R : RV) {
1651     createIntrinsic(R, Defs);
1652   }
1653 
1654   // The mappings must be sorted based on BuiltinID.
1655   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1656                       const std::unique_ptr<Intrinsic> &B) {
1657     return A->getMangledName() < B->getMangledName();
1658   });
1659 
1660   OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1661   for (auto &Def : Defs) {
1662     // Builtins only exist for non-overloaded intrinsics, overloaded
1663     // declarations only live in the header file.
1664     if (Def->getClassKind() == ClassG)
1665       continue;
1666 
1667     uint64_t Flags = Def->getFlags();
1668     auto FlagString = std::to_string(Flags);
1669 
1670     std::string LLVMName = Def->getLLVMName();
1671     std::string Builtin = Def->getMangledName();
1672     if (!LLVMName.empty())
1673       OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1674          << "),\n";
1675     else
1676       OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1677   }
1678   OS << "#endif\n\n";
1679 }
1680 
createSMERangeChecks(raw_ostream & OS)1681 void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1682   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1683   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1684   for (auto *R : RV) {
1685     createIntrinsic(R, Defs);
1686   }
1687 
1688   // The mappings must be sorted based on BuiltinID.
1689   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1690                       const std::unique_ptr<Intrinsic> &B) {
1691     return A->getMangledName() < B->getMangledName();
1692   });
1693 
1694 
1695   OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1696 
1697   // Ensure these are only emitted once.
1698   std::set<std::string> Emitted;
1699 
1700   for (auto &Def : Defs) {
1701     if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1702         Def->getImmChecks().empty())
1703       continue;
1704 
1705     OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1706     for (auto &Check : Def->getImmChecks())
1707       OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
1708          << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
1709     OS << "  break;\n";
1710 
1711     Emitted.insert(Def->getMangledName());
1712   }
1713 
1714   OS << "#endif\n\n";
1715 }
1716 
createBuiltinZAState(raw_ostream & OS)1717 void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
1718   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1719   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1720   for (auto *R : RV)
1721     createIntrinsic(R, Defs);
1722 
1723   std::map<std::string, std::set<std::string>> IntrinsicsPerState;
1724   for (auto &Def : Defs) {
1725     std::string Key;
1726     auto AddToKey = [&Key](const std::string &S) -> void {
1727       Key = Key.empty() ? S : (Key + " | " + S);
1728     };
1729 
1730     if (Def->isFlagSet(getEnumValueForFlag("IsInZA")))
1731       AddToKey("ArmInZA");
1732     else if (Def->isFlagSet(getEnumValueForFlag("IsOutZA")))
1733       AddToKey("ArmOutZA");
1734     else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZA")))
1735       AddToKey("ArmInOutZA");
1736 
1737     if (Def->isFlagSet(getEnumValueForFlag("IsInZT0")))
1738       AddToKey("ArmInZT0");
1739     else if (Def->isFlagSet(getEnumValueForFlag("IsOutZT0")))
1740       AddToKey("ArmOutZT0");
1741     else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZT0")))
1742       AddToKey("ArmInOutZT0");
1743 
1744     if (!Key.empty())
1745       IntrinsicsPerState[Key].insert(Def->getMangledName());
1746   }
1747 
1748   OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
1749   for (auto &KV : IntrinsicsPerState) {
1750     for (StringRef Name : KV.second)
1751       OS << "case SME::BI__builtin_sme_" << Name << ":\n";
1752     OS << "  return " << KV.first << ";\n";
1753   }
1754   OS << "#endif\n\n";
1755 }
1756 
createStreamingAttrs(raw_ostream & OS,ACLEKind Kind)1757 void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
1758   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1759   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1760   for (auto *R : RV)
1761     createIntrinsic(R, Defs);
1762 
1763   StringRef ExtensionKind;
1764   switch (Kind) {
1765   case ACLEKind::SME:
1766     ExtensionKind = "SME";
1767     break;
1768   case ACLEKind::SVE:
1769     ExtensionKind = "SVE";
1770     break;
1771   }
1772 
1773   OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
1774 
1775   llvm::StringMap<std::set<std::string>> StreamingMap;
1776 
1777   uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming");
1778   uint64_t IsStreamingOrSVE2p1Flag = getEnumValueForFlag("IsStreamingOrSVE2p1");
1779   uint64_t IsStreamingCompatibleFlag =
1780       getEnumValueForFlag("IsStreamingCompatible");
1781   for (auto &Def : Defs) {
1782     if (Def->isFlagSet(IsStreamingFlag))
1783       StreamingMap["ArmStreaming"].insert(Def->getMangledName());
1784     else if (Def->isFlagSet(IsStreamingOrSVE2p1Flag))
1785       StreamingMap["ArmStreamingOrSVE2p1"].insert(Def->getMangledName());
1786     else if (Def->isFlagSet(IsStreamingCompatibleFlag))
1787       StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName());
1788     else
1789       StreamingMap["ArmNonStreaming"].insert(Def->getMangledName());
1790   }
1791 
1792   for (auto BuiltinType : StreamingMap.keys()) {
1793     for (auto Name : StreamingMap[BuiltinType]) {
1794       OS << "case " << ExtensionKind << "::BI__builtin_"
1795          << ExtensionKind.lower() << "_";
1796       OS << Name << ":\n";
1797     }
1798     OS << "  BuiltinType = " << BuiltinType << ";\n";
1799     OS << "  break;\n";
1800   }
1801 
1802   OS << "#endif\n\n";
1803 }
1804 
1805 namespace clang {
EmitSveHeader(RecordKeeper & Records,raw_ostream & OS)1806 void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) {
1807   SVEEmitter(Records).createHeader(OS);
1808 }
1809 
EmitSveBuiltins(RecordKeeper & Records,raw_ostream & OS)1810 void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) {
1811   SVEEmitter(Records).createBuiltins(OS);
1812 }
1813 
EmitSveBuiltinCG(RecordKeeper & Records,raw_ostream & OS)1814 void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1815   SVEEmitter(Records).createCodeGenMap(OS);
1816 }
1817 
EmitSveRangeChecks(RecordKeeper & Records,raw_ostream & OS)1818 void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
1819   SVEEmitter(Records).createRangeChecks(OS);
1820 }
1821 
EmitSveTypeFlags(RecordKeeper & Records,raw_ostream & OS)1822 void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) {
1823   SVEEmitter(Records).createTypeFlags(OS);
1824 }
1825 
EmitSveStreamingAttrs(RecordKeeper & Records,raw_ostream & OS)1826 void EmitSveStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
1827   SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE);
1828 }
1829 
EmitSmeHeader(RecordKeeper & Records,raw_ostream & OS)1830 void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) {
1831   SVEEmitter(Records).createSMEHeader(OS);
1832 }
1833 
EmitSmeBuiltins(RecordKeeper & Records,raw_ostream & OS)1834 void EmitSmeBuiltins(RecordKeeper &Records, raw_ostream &OS) {
1835   SVEEmitter(Records).createSMEBuiltins(OS);
1836 }
1837 
EmitSmeBuiltinCG(RecordKeeper & Records,raw_ostream & OS)1838 void EmitSmeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1839   SVEEmitter(Records).createSMECodeGenMap(OS);
1840 }
1841 
EmitSmeRangeChecks(RecordKeeper & Records,raw_ostream & OS)1842 void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
1843   SVEEmitter(Records).createSMERangeChecks(OS);
1844 }
1845 
EmitSmeStreamingAttrs(RecordKeeper & Records,raw_ostream & OS)1846 void EmitSmeStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
1847   SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME);
1848 }
1849 
EmitSmeBuiltinZAState(RecordKeeper & Records,raw_ostream & OS)1850 void EmitSmeBuiltinZAState(RecordKeeper &Records, raw_ostream &OS) {
1851   SVEEmitter(Records).createBuiltinZAState(OS);
1852 }
1853 } // End namespace clang
1854