1 //===- NeonEmitter.cpp - Generate arm_neon.h for use with clang -*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This tablegen backend is responsible for emitting arm_neon.h, which includes
10 // a declaration and definition of each function specified by the ARM NEON
11 // compiler interface. See ARM document DUI0348B.
12 //
13 // Each NEON instruction is implemented in terms of 1 or more functions which
14 // are suffixed with the element type of the input vectors. Functions may be
15 // implemented in terms of generic vector operations such as +, *, -, etc. or
16 // by calling a __builtin_-prefixed function which will be handled by clang's
17 // CodeGen library.
18 //
19 // Additional validation code can be generated by this file when runHeader() is
20 // called, rather than the normal run() entry point.
21 //
22 // See also the documentation in include/clang/Basic/arm_neon.td.
23 //
24 //===----------------------------------------------------------------------===//
25
26 #include "TableGenBackends.h"
27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/DenseMap.h"
29 #include "llvm/ADT/None.h"
30 #include "llvm/ADT/Optional.h"
31 #include "llvm/ADT/STLExtras.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/StringExtras.h"
34 #include "llvm/ADT/StringRef.h"
35 #include "llvm/Support/Casting.h"
36 #include "llvm/Support/ErrorHandling.h"
37 #include "llvm/Support/raw_ostream.h"
38 #include "llvm/TableGen/Error.h"
39 #include "llvm/TableGen/Record.h"
40 #include "llvm/TableGen/SetTheory.h"
41 #include <algorithm>
42 #include <cassert>
43 #include <cctype>
44 #include <cstddef>
45 #include <cstdint>
46 #include <deque>
47 #include <map>
48 #include <set>
49 #include <sstream>
50 #include <string>
51 #include <utility>
52 #include <vector>
53
54 using namespace llvm;
55
56 namespace {
57
58 // While globals are generally bad, this one allows us to perform assertions
59 // liberally and somehow still trace them back to the def they indirectly
60 // came from.
61 static Record *CurrentRecord = nullptr;
assert_with_loc(bool Assertion,const std::string & Str)62 static void assert_with_loc(bool Assertion, const std::string &Str) {
63 if (!Assertion) {
64 if (CurrentRecord)
65 PrintFatalError(CurrentRecord->getLoc(), Str);
66 else
67 PrintFatalError(Str);
68 }
69 }
70
71 enum ClassKind {
72 ClassNone,
73 ClassI, // generic integer instruction, e.g., "i8" suffix
74 ClassS, // signed/unsigned/poly, e.g., "s8", "u8" or "p8" suffix
75 ClassW, // width-specific instruction, e.g., "8" suffix
76 ClassB, // bitcast arguments with enum argument to specify type
77 ClassL, // Logical instructions which are op instructions
78 // but we need to not emit any suffix for in our
79 // tests.
80 ClassNoTest // Instructions which we do not test since they are
81 // not TRUE instructions.
82 };
83
84 /// NeonTypeFlags - Flags to identify the types for overloaded Neon
85 /// builtins. These must be kept in sync with the flags in
86 /// include/clang/Basic/TargetBuiltins.h.
87 namespace NeonTypeFlags {
88
89 enum { EltTypeMask = 0xf, UnsignedFlag = 0x10, QuadFlag = 0x20 };
90
91 enum EltType {
92 Int8,
93 Int16,
94 Int32,
95 Int64,
96 Poly8,
97 Poly16,
98 Poly64,
99 Poly128,
100 Float16,
101 Float32,
102 Float64,
103 BFloat16
104 };
105
106 } // end namespace NeonTypeFlags
107
108 class NeonEmitter;
109
110 //===----------------------------------------------------------------------===//
111 // TypeSpec
112 //===----------------------------------------------------------------------===//
113
114 /// A TypeSpec is just a simple wrapper around a string, but gets its own type
115 /// for strong typing purposes.
116 ///
117 /// A TypeSpec can be used to create a type.
118 class TypeSpec : public std::string {
119 public:
fromTypeSpecs(StringRef Str)120 static std::vector<TypeSpec> fromTypeSpecs(StringRef Str) {
121 std::vector<TypeSpec> Ret;
122 TypeSpec Acc;
123 for (char I : Str.str()) {
124 if (islower(I)) {
125 Acc.push_back(I);
126 Ret.push_back(TypeSpec(Acc));
127 Acc.clear();
128 } else {
129 Acc.push_back(I);
130 }
131 }
132 return Ret;
133 }
134 };
135
136 //===----------------------------------------------------------------------===//
137 // Type
138 //===----------------------------------------------------------------------===//
139
140 /// A Type. Not much more to say here.
141 class Type {
142 private:
143 TypeSpec TS;
144
145 enum TypeKind {
146 Void,
147 Float,
148 SInt,
149 UInt,
150 Poly,
151 BFloat16,
152 };
153 TypeKind Kind;
154 bool Immediate, Constant, Pointer;
155 // ScalarForMangling and NoManglingQ are really not suited to live here as
156 // they are not related to the type. But they live in the TypeSpec (not the
157 // prototype), so this is really the only place to store them.
158 bool ScalarForMangling, NoManglingQ;
159 unsigned Bitwidth, ElementBitwidth, NumVectors;
160
161 public:
Type()162 Type()
163 : Kind(Void), Immediate(false), Constant(false),
164 Pointer(false), ScalarForMangling(false), NoManglingQ(false),
165 Bitwidth(0), ElementBitwidth(0), NumVectors(0) {}
166
Type(TypeSpec TS,StringRef CharMods)167 Type(TypeSpec TS, StringRef CharMods)
168 : TS(std::move(TS)), Kind(Void), Immediate(false),
169 Constant(false), Pointer(false), ScalarForMangling(false),
170 NoManglingQ(false), Bitwidth(0), ElementBitwidth(0), NumVectors(0) {
171 applyModifiers(CharMods);
172 }
173
174 /// Returns a type representing "void".
getVoid()175 static Type getVoid() { return Type(); }
176
operator ==(const Type & Other) const177 bool operator==(const Type &Other) const { return str() == Other.str(); }
operator !=(const Type & Other) const178 bool operator!=(const Type &Other) const { return !operator==(Other); }
179
180 //
181 // Query functions
182 //
isScalarForMangling() const183 bool isScalarForMangling() const { return ScalarForMangling; }
noManglingQ() const184 bool noManglingQ() const { return NoManglingQ; }
185
isPointer() const186 bool isPointer() const { return Pointer; }
isValue() const187 bool isValue() const { return !isVoid() && !isPointer(); }
isScalar() const188 bool isScalar() const { return isValue() && NumVectors == 0; }
isVector() const189 bool isVector() const { return isValue() && NumVectors > 0; }
isConstPointer() const190 bool isConstPointer() const { return Constant; }
isFloating() const191 bool isFloating() const { return Kind == Float; }
isInteger() const192 bool isInteger() const { return Kind == SInt || Kind == UInt; }
isPoly() const193 bool isPoly() const { return Kind == Poly; }
isSigned() const194 bool isSigned() const { return Kind == SInt; }
isImmediate() const195 bool isImmediate() const { return Immediate; }
isFloat() const196 bool isFloat() const { return isFloating() && ElementBitwidth == 32; }
isDouble() const197 bool isDouble() const { return isFloating() && ElementBitwidth == 64; }
isHalf() const198 bool isHalf() const { return isFloating() && ElementBitwidth == 16; }
isChar() const199 bool isChar() const { return ElementBitwidth == 8; }
isShort() const200 bool isShort() const { return isInteger() && ElementBitwidth == 16; }
isInt() const201 bool isInt() const { return isInteger() && ElementBitwidth == 32; }
isLong() const202 bool isLong() const { return isInteger() && ElementBitwidth == 64; }
isVoid() const203 bool isVoid() const { return Kind == Void; }
isBFloat16() const204 bool isBFloat16() const { return Kind == BFloat16; }
getNumElements() const205 unsigned getNumElements() const { return Bitwidth / ElementBitwidth; }
getSizeInBits() const206 unsigned getSizeInBits() const { return Bitwidth; }
getElementSizeInBits() const207 unsigned getElementSizeInBits() const { return ElementBitwidth; }
getNumVectors() const208 unsigned getNumVectors() const { return NumVectors; }
209
210 //
211 // Mutator functions
212 //
makeUnsigned()213 void makeUnsigned() {
214 assert(!isVoid() && "not a potentially signed type");
215 Kind = UInt;
216 }
makeSigned()217 void makeSigned() {
218 assert(!isVoid() && "not a potentially signed type");
219 Kind = SInt;
220 }
221
makeInteger(unsigned ElemWidth,bool Sign)222 void makeInteger(unsigned ElemWidth, bool Sign) {
223 assert(!isVoid() && "converting void to int probably not useful");
224 Kind = Sign ? SInt : UInt;
225 Immediate = false;
226 ElementBitwidth = ElemWidth;
227 }
228
makeImmediate(unsigned ElemWidth)229 void makeImmediate(unsigned ElemWidth) {
230 Kind = SInt;
231 Immediate = true;
232 ElementBitwidth = ElemWidth;
233 }
234
makeScalar()235 void makeScalar() {
236 Bitwidth = ElementBitwidth;
237 NumVectors = 0;
238 }
239
makeOneVector()240 void makeOneVector() {
241 assert(isVector());
242 NumVectors = 1;
243 }
244
make32BitElement()245 void make32BitElement() {
246 assert_with_loc(Bitwidth > 32, "Not enough bits to make it 32!");
247 ElementBitwidth = 32;
248 }
249
doubleLanes()250 void doubleLanes() {
251 assert_with_loc(Bitwidth != 128, "Can't get bigger than 128!");
252 Bitwidth = 128;
253 }
254
halveLanes()255 void halveLanes() {
256 assert_with_loc(Bitwidth != 64, "Can't get smaller than 64!");
257 Bitwidth = 64;
258 }
259
260 /// Return the C string representation of a type, which is the typename
261 /// defined in stdint.h or arm_neon.h.
262 std::string str() const;
263
264 /// Return the string representation of a type, which is an encoded
265 /// string for passing to the BUILTIN() macro in Builtins.def.
266 std::string builtin_str() const;
267
268 /// Return the value in NeonTypeFlags for this type.
269 unsigned getNeonEnum() const;
270
271 /// Parse a type from a stdint.h or arm_neon.h typedef name,
272 /// for example uint32x2_t or int64_t.
273 static Type fromTypedefName(StringRef Name);
274
275 private:
276 /// Creates the type based on the typespec string in TS.
277 /// Sets "Quad" to true if the "Q" or "H" modifiers were
278 /// seen. This is needed by applyModifier as some modifiers
279 /// only take effect if the type size was changed by "Q" or "H".
280 void applyTypespec(bool &Quad);
281 /// Applies prototype modifiers to the type.
282 void applyModifiers(StringRef Mods);
283 };
284
285 //===----------------------------------------------------------------------===//
286 // Variable
287 //===----------------------------------------------------------------------===//
288
289 /// A variable is a simple class that just has a type and a name.
290 class Variable {
291 Type T;
292 std::string N;
293
294 public:
Variable()295 Variable() : T(Type::getVoid()), N("") {}
Variable(Type T,std::string N)296 Variable(Type T, std::string N) : T(std::move(T)), N(std::move(N)) {}
297
getType() const298 Type getType() const { return T; }
getName() const299 std::string getName() const { return "__" + N; }
300 };
301
302 //===----------------------------------------------------------------------===//
303 // Intrinsic
304 //===----------------------------------------------------------------------===//
305
306 /// The main grunt class. This represents an instantiation of an intrinsic with
307 /// a particular typespec and prototype.
308 class Intrinsic {
309 /// The Record this intrinsic was created from.
310 Record *R;
311 /// The unmangled name.
312 std::string Name;
313 /// The input and output typespecs. InTS == OutTS except when
314 /// CartesianProductWith is non-empty - this is the case for vreinterpret.
315 TypeSpec OutTS, InTS;
316 /// The base class kind. Most intrinsics use ClassS, which has full type
317 /// info for integers (s32/u32). Some use ClassI, which doesn't care about
318 /// signedness (i32), while some (ClassB) have no type at all, only a width
319 /// (32).
320 ClassKind CK;
321 /// The list of DAGs for the body. May be empty, in which case we should
322 /// emit a builtin call.
323 ListInit *Body;
324 /// The architectural #ifdef guard.
325 std::string Guard;
326 /// Set if the Unavailable bit is 1. This means we don't generate a body,
327 /// just an "unavailable" attribute on a declaration.
328 bool IsUnavailable;
329 /// Is this intrinsic safe for big-endian? or does it need its arguments
330 /// reversing?
331 bool BigEndianSafe;
332
333 /// The types of return value [0] and parameters [1..].
334 std::vector<Type> Types;
335 /// The index of the key type passed to CGBuiltin.cpp for polymorphic calls.
336 int PolymorphicKeyType;
337 /// The local variables defined.
338 std::map<std::string, Variable> Variables;
339 /// NeededEarly - set if any other intrinsic depends on this intrinsic.
340 bool NeededEarly;
341 /// UseMacro - set if we should implement using a macro or unset for a
342 /// function.
343 bool UseMacro;
344 /// The set of intrinsics that this intrinsic uses/requires.
345 std::set<Intrinsic *> Dependencies;
346 /// The "base type", which is Type('d', OutTS). InBaseType is only
347 /// different if CartesianProductWith is non-empty (for vreinterpret).
348 Type BaseType, InBaseType;
349 /// The return variable.
350 Variable RetVar;
351 /// A postfix to apply to every variable. Defaults to "".
352 std::string VariablePostfix;
353
354 NeonEmitter &Emitter;
355 std::stringstream OS;
356
isBigEndianSafe() const357 bool isBigEndianSafe() const {
358 if (BigEndianSafe)
359 return true;
360
361 for (const auto &T : Types){
362 if (T.isVector() && T.getNumElements() > 1)
363 return false;
364 }
365 return true;
366 }
367
368 public:
Intrinsic(Record * R,StringRef Name,StringRef Proto,TypeSpec OutTS,TypeSpec InTS,ClassKind CK,ListInit * Body,NeonEmitter & Emitter,StringRef Guard,bool IsUnavailable,bool BigEndianSafe)369 Intrinsic(Record *R, StringRef Name, StringRef Proto, TypeSpec OutTS,
370 TypeSpec InTS, ClassKind CK, ListInit *Body, NeonEmitter &Emitter,
371 StringRef Guard, bool IsUnavailable, bool BigEndianSafe)
372 : R(R), Name(Name.str()), OutTS(OutTS), InTS(InTS), CK(CK), Body(Body),
373 Guard(Guard.str()), IsUnavailable(IsUnavailable),
374 BigEndianSafe(BigEndianSafe), PolymorphicKeyType(0), NeededEarly(false),
375 UseMacro(false), BaseType(OutTS, "."), InBaseType(InTS, "."),
376 Emitter(Emitter) {
377 // Modify the TypeSpec per-argument to get a concrete Type, and create
378 // known variables for each.
379 // Types[0] is the return value.
380 unsigned Pos = 0;
381 Types.emplace_back(OutTS, getNextModifiers(Proto, Pos));
382 StringRef Mods = getNextModifiers(Proto, Pos);
383 while (!Mods.empty()) {
384 Types.emplace_back(InTS, Mods);
385 if (Mods.find('!') != StringRef::npos)
386 PolymorphicKeyType = Types.size() - 1;
387
388 Mods = getNextModifiers(Proto, Pos);
389 }
390
391 for (auto Type : Types) {
392 // If this builtin takes an immediate argument, we need to #define it rather
393 // than use a standard declaration, so that SemaChecking can range check
394 // the immediate passed by the user.
395
396 // Pointer arguments need to use macros to avoid hiding aligned attributes
397 // from the pointer type.
398
399 // It is not permitted to pass or return an __fp16 by value, so intrinsics
400 // taking a scalar float16_t must be implemented as macros.
401 if (Type.isImmediate() || Type.isPointer() ||
402 (Type.isScalar() && Type.isHalf()))
403 UseMacro = true;
404 }
405 }
406
407 /// Get the Record that this intrinsic is based off.
getRecord() const408 Record *getRecord() const { return R; }
409 /// Get the set of Intrinsics that this intrinsic calls.
410 /// this is the set of immediate dependencies, NOT the
411 /// transitive closure.
getDependencies() const412 const std::set<Intrinsic *> &getDependencies() const { return Dependencies; }
413 /// Get the architectural guard string (#ifdef).
getGuard() const414 std::string getGuard() const { return Guard; }
415 /// Get the non-mangled name.
getName() const416 std::string getName() const { return Name; }
417
418 /// Return true if the intrinsic takes an immediate operand.
hasImmediate() const419 bool hasImmediate() const {
420 return std::any_of(Types.begin(), Types.end(),
421 [](const Type &T) { return T.isImmediate(); });
422 }
423
424 /// Return the parameter index of the immediate operand.
getImmediateIdx() const425 unsigned getImmediateIdx() const {
426 for (unsigned Idx = 0; Idx < Types.size(); ++Idx)
427 if (Types[Idx].isImmediate())
428 return Idx - 1;
429 llvm_unreachable("Intrinsic has no immediate");
430 }
431
432
getNumParams() const433 unsigned getNumParams() const { return Types.size() - 1; }
getReturnType() const434 Type getReturnType() const { return Types[0]; }
getParamType(unsigned I) const435 Type getParamType(unsigned I) const { return Types[I + 1]; }
getBaseType() const436 Type getBaseType() const { return BaseType; }
getPolymorphicKeyType() const437 Type getPolymorphicKeyType() const { return Types[PolymorphicKeyType]; }
438
439 /// Return true if the prototype has a scalar argument.
440 bool protoHasScalar() const;
441
442 /// Return the index that parameter PIndex will sit at
443 /// in a generated function call. This is often just PIndex,
444 /// but may not be as things such as multiple-vector operands
445 /// and sret parameters need to be taken into accont.
getGeneratedParamIdx(unsigned PIndex)446 unsigned getGeneratedParamIdx(unsigned PIndex) {
447 unsigned Idx = 0;
448 if (getReturnType().getNumVectors() > 1)
449 // Multiple vectors are passed as sret.
450 ++Idx;
451
452 for (unsigned I = 0; I < PIndex; ++I)
453 Idx += std::max(1U, getParamType(I).getNumVectors());
454
455 return Idx;
456 }
457
hasBody() const458 bool hasBody() const { return Body && !Body->getValues().empty(); }
459
setNeededEarly()460 void setNeededEarly() { NeededEarly = true; }
461
operator <(const Intrinsic & Other) const462 bool operator<(const Intrinsic &Other) const {
463 // Sort lexicographically on a two-tuple (Guard, Name)
464 if (Guard != Other.Guard)
465 return Guard < Other.Guard;
466 return Name < Other.Name;
467 }
468
getClassKind(bool UseClassBIfScalar=false)469 ClassKind getClassKind(bool UseClassBIfScalar = false) {
470 if (UseClassBIfScalar && !protoHasScalar())
471 return ClassB;
472 return CK;
473 }
474
475 /// Return the name, mangled with type information.
476 /// If ForceClassS is true, use ClassS (u32/s32) instead
477 /// of the intrinsic's own type class.
478 std::string getMangledName(bool ForceClassS = false) const;
479 /// Return the type code for a builtin function call.
480 std::string getInstTypeCode(Type T, ClassKind CK) const;
481 /// Return the type string for a BUILTIN() macro in Builtins.def.
482 std::string getBuiltinTypeStr();
483
484 /// Generate the intrinsic, returning code.
485 std::string generate();
486 /// Perform type checking and populate the dependency graph, but
487 /// don't generate code yet.
488 void indexBody();
489
490 private:
491 StringRef getNextModifiers(StringRef Proto, unsigned &Pos) const;
492
493 std::string mangleName(std::string Name, ClassKind CK) const;
494
495 void initVariables();
496 std::string replaceParamsIn(std::string S);
497
498 void emitBodyAsBuiltinCall();
499
500 void generateImpl(bool ReverseArguments,
501 StringRef NamePrefix, StringRef CallPrefix);
502 void emitReturn();
503 void emitBody(StringRef CallPrefix);
504 void emitShadowedArgs();
505 void emitArgumentReversal();
506 void emitReturnReversal();
507 void emitReverseVariable(Variable &Dest, Variable &Src);
508 void emitNewLine();
509 void emitClosingBrace();
510 void emitOpeningBrace();
511 void emitPrototype(StringRef NamePrefix);
512
513 class DagEmitter {
514 Intrinsic &Intr;
515 StringRef CallPrefix;
516
517 public:
DagEmitter(Intrinsic & Intr,StringRef CallPrefix)518 DagEmitter(Intrinsic &Intr, StringRef CallPrefix) :
519 Intr(Intr), CallPrefix(CallPrefix) {
520 }
521 std::pair<Type, std::string> emitDagArg(Init *Arg, std::string ArgName);
522 std::pair<Type, std::string> emitDagSaveTemp(DagInit *DI);
523 std::pair<Type, std::string> emitDagSplat(DagInit *DI);
524 std::pair<Type, std::string> emitDagDup(DagInit *DI);
525 std::pair<Type, std::string> emitDagDupTyped(DagInit *DI);
526 std::pair<Type, std::string> emitDagShuffle(DagInit *DI);
527 std::pair<Type, std::string> emitDagCast(DagInit *DI, bool IsBitCast);
528 std::pair<Type, std::string> emitDagCall(DagInit *DI,
529 bool MatchMangledName);
530 std::pair<Type, std::string> emitDagNameReplace(DagInit *DI);
531 std::pair<Type, std::string> emitDagLiteral(DagInit *DI);
532 std::pair<Type, std::string> emitDagOp(DagInit *DI);
533 std::pair<Type, std::string> emitDag(DagInit *DI);
534 };
535 };
536
537 //===----------------------------------------------------------------------===//
538 // NeonEmitter
539 //===----------------------------------------------------------------------===//
540
541 class NeonEmitter {
542 RecordKeeper &Records;
543 DenseMap<Record *, ClassKind> ClassMap;
544 std::map<std::string, std::deque<Intrinsic>> IntrinsicMap;
545 unsigned UniqueNumber;
546
547 void createIntrinsic(Record *R, SmallVectorImpl<Intrinsic *> &Out);
548 void genBuiltinsDef(raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs);
549 void genOverloadTypeCheckCode(raw_ostream &OS,
550 SmallVectorImpl<Intrinsic *> &Defs);
551 void genIntrinsicRangeCheckCode(raw_ostream &OS,
552 SmallVectorImpl<Intrinsic *> &Defs);
553
554 public:
555 /// Called by Intrinsic - this attempts to get an intrinsic that takes
556 /// the given types as arguments.
557 Intrinsic &getIntrinsic(StringRef Name, ArrayRef<Type> Types,
558 Optional<std::string> MangledName);
559
560 /// Called by Intrinsic - returns a globally-unique number.
getUniqueNumber()561 unsigned getUniqueNumber() { return UniqueNumber++; }
562
NeonEmitter(RecordKeeper & R)563 NeonEmitter(RecordKeeper &R) : Records(R), UniqueNumber(0) {
564 Record *SI = R.getClass("SInst");
565 Record *II = R.getClass("IInst");
566 Record *WI = R.getClass("WInst");
567 Record *SOpI = R.getClass("SOpInst");
568 Record *IOpI = R.getClass("IOpInst");
569 Record *WOpI = R.getClass("WOpInst");
570 Record *LOpI = R.getClass("LOpInst");
571 Record *NoTestOpI = R.getClass("NoTestOpInst");
572
573 ClassMap[SI] = ClassS;
574 ClassMap[II] = ClassI;
575 ClassMap[WI] = ClassW;
576 ClassMap[SOpI] = ClassS;
577 ClassMap[IOpI] = ClassI;
578 ClassMap[WOpI] = ClassW;
579 ClassMap[LOpI] = ClassL;
580 ClassMap[NoTestOpI] = ClassNoTest;
581 }
582
583 // Emit arm_neon.h.inc
584 void run(raw_ostream &o);
585
586 // Emit arm_fp16.h.inc
587 void runFP16(raw_ostream &o);
588
589 // Emit arm_bf16.h.inc
590 void runBF16(raw_ostream &o);
591
592 // Emit all the __builtin prototypes used in arm_neon.h, arm_fp16.h and
593 // arm_bf16.h
594 void runHeader(raw_ostream &o);
595 };
596
597 } // end anonymous namespace
598
599 //===----------------------------------------------------------------------===//
600 // Type implementation
601 //===----------------------------------------------------------------------===//
602
str() const603 std::string Type::str() const {
604 if (isVoid())
605 return "void";
606 std::string S;
607
608 if (isInteger() && !isSigned())
609 S += "u";
610
611 if (isPoly())
612 S += "poly";
613 else if (isFloating())
614 S += "float";
615 else if (isBFloat16())
616 S += "bfloat";
617 else
618 S += "int";
619
620 S += utostr(ElementBitwidth);
621 if (isVector())
622 S += "x" + utostr(getNumElements());
623 if (NumVectors > 1)
624 S += "x" + utostr(NumVectors);
625 S += "_t";
626
627 if (Constant)
628 S += " const";
629 if (Pointer)
630 S += " *";
631
632 return S;
633 }
634
builtin_str() const635 std::string Type::builtin_str() const {
636 std::string S;
637 if (isVoid())
638 return "v";
639
640 if (isPointer()) {
641 // All pointers are void pointers.
642 S = "v";
643 if (isConstPointer())
644 S += "C";
645 S += "*";
646 return S;
647 } else if (isInteger())
648 switch (ElementBitwidth) {
649 case 8: S += "c"; break;
650 case 16: S += "s"; break;
651 case 32: S += "i"; break;
652 case 64: S += "Wi"; break;
653 case 128: S += "LLLi"; break;
654 default: llvm_unreachable("Unhandled case!");
655 }
656 else if (isBFloat16()) {
657 assert(ElementBitwidth == 16 && "BFloat16 can only be 16 bits");
658 S += "y";
659 } else
660 switch (ElementBitwidth) {
661 case 16: S += "h"; break;
662 case 32: S += "f"; break;
663 case 64: S += "d"; break;
664 default: llvm_unreachable("Unhandled case!");
665 }
666
667 // FIXME: NECESSARY???????????????????????????????????????????????????????????????????????
668 if (isChar() && !isPointer() && isSigned())
669 // Make chars explicitly signed.
670 S = "S" + S;
671 else if (isInteger() && !isSigned())
672 S = "U" + S;
673
674 // Constant indices are "int", but have the "constant expression" modifier.
675 if (isImmediate()) {
676 assert(isInteger() && isSigned());
677 S = "I" + S;
678 }
679
680 if (isScalar())
681 return S;
682
683 std::string Ret;
684 for (unsigned I = 0; I < NumVectors; ++I)
685 Ret += "V" + utostr(getNumElements()) + S;
686
687 return Ret;
688 }
689
getNeonEnum() const690 unsigned Type::getNeonEnum() const {
691 unsigned Addend;
692 switch (ElementBitwidth) {
693 case 8: Addend = 0; break;
694 case 16: Addend = 1; break;
695 case 32: Addend = 2; break;
696 case 64: Addend = 3; break;
697 case 128: Addend = 4; break;
698 default: llvm_unreachable("Unhandled element bitwidth!");
699 }
700
701 unsigned Base = (unsigned)NeonTypeFlags::Int8 + Addend;
702 if (isPoly()) {
703 // Adjustment needed because Poly32 doesn't exist.
704 if (Addend >= 2)
705 --Addend;
706 Base = (unsigned)NeonTypeFlags::Poly8 + Addend;
707 }
708 if (isFloating()) {
709 assert(Addend != 0 && "Float8 doesn't exist!");
710 Base = (unsigned)NeonTypeFlags::Float16 + (Addend - 1);
711 }
712
713 if (isBFloat16()) {
714 assert(Addend == 1 && "BFloat16 is only 16 bit");
715 Base = (unsigned)NeonTypeFlags::BFloat16;
716 }
717
718 if (Bitwidth == 128)
719 Base |= (unsigned)NeonTypeFlags::QuadFlag;
720 if (isInteger() && !isSigned())
721 Base |= (unsigned)NeonTypeFlags::UnsignedFlag;
722
723 return Base;
724 }
725
fromTypedefName(StringRef Name)726 Type Type::fromTypedefName(StringRef Name) {
727 Type T;
728 T.Kind = SInt;
729
730 if (Name.front() == 'u') {
731 T.Kind = UInt;
732 Name = Name.drop_front();
733 }
734
735 if (Name.startswith("float")) {
736 T.Kind = Float;
737 Name = Name.drop_front(5);
738 } else if (Name.startswith("poly")) {
739 T.Kind = Poly;
740 Name = Name.drop_front(4);
741 } else if (Name.startswith("bfloat")) {
742 T.Kind = BFloat16;
743 Name = Name.drop_front(6);
744 } else {
745 assert(Name.startswith("int"));
746 Name = Name.drop_front(3);
747 }
748
749 unsigned I = 0;
750 for (I = 0; I < Name.size(); ++I) {
751 if (!isdigit(Name[I]))
752 break;
753 }
754 Name.substr(0, I).getAsInteger(10, T.ElementBitwidth);
755 Name = Name.drop_front(I);
756
757 T.Bitwidth = T.ElementBitwidth;
758 T.NumVectors = 1;
759
760 if (Name.front() == 'x') {
761 Name = Name.drop_front();
762 unsigned I = 0;
763 for (I = 0; I < Name.size(); ++I) {
764 if (!isdigit(Name[I]))
765 break;
766 }
767 unsigned NumLanes;
768 Name.substr(0, I).getAsInteger(10, NumLanes);
769 Name = Name.drop_front(I);
770 T.Bitwidth = T.ElementBitwidth * NumLanes;
771 } else {
772 // Was scalar.
773 T.NumVectors = 0;
774 }
775 if (Name.front() == 'x') {
776 Name = Name.drop_front();
777 unsigned I = 0;
778 for (I = 0; I < Name.size(); ++I) {
779 if (!isdigit(Name[I]))
780 break;
781 }
782 Name.substr(0, I).getAsInteger(10, T.NumVectors);
783 Name = Name.drop_front(I);
784 }
785
786 assert(Name.startswith("_t") && "Malformed typedef!");
787 return T;
788 }
789
applyTypespec(bool & Quad)790 void Type::applyTypespec(bool &Quad) {
791 std::string S = TS;
792 ScalarForMangling = false;
793 Kind = SInt;
794 ElementBitwidth = ~0U;
795 NumVectors = 1;
796
797 for (char I : S) {
798 switch (I) {
799 case 'S':
800 ScalarForMangling = true;
801 break;
802 case 'H':
803 NoManglingQ = true;
804 Quad = true;
805 break;
806 case 'Q':
807 Quad = true;
808 break;
809 case 'P':
810 Kind = Poly;
811 break;
812 case 'U':
813 Kind = UInt;
814 break;
815 case 'c':
816 ElementBitwidth = 8;
817 break;
818 case 'h':
819 Kind = Float;
820 LLVM_FALLTHROUGH;
821 case 's':
822 ElementBitwidth = 16;
823 break;
824 case 'f':
825 Kind = Float;
826 LLVM_FALLTHROUGH;
827 case 'i':
828 ElementBitwidth = 32;
829 break;
830 case 'd':
831 Kind = Float;
832 LLVM_FALLTHROUGH;
833 case 'l':
834 ElementBitwidth = 64;
835 break;
836 case 'k':
837 ElementBitwidth = 128;
838 // Poly doesn't have a 128x1 type.
839 if (isPoly())
840 NumVectors = 0;
841 break;
842 case 'b':
843 Kind = BFloat16;
844 ElementBitwidth = 16;
845 break;
846 default:
847 llvm_unreachable("Unhandled type code!");
848 }
849 }
850 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
851
852 Bitwidth = Quad ? 128 : 64;
853 }
854
applyModifiers(StringRef Mods)855 void Type::applyModifiers(StringRef Mods) {
856 bool AppliedQuad = false;
857 applyTypespec(AppliedQuad);
858
859 for (char Mod : Mods) {
860 switch (Mod) {
861 case '.':
862 break;
863 case 'v':
864 Kind = Void;
865 break;
866 case 'S':
867 Kind = SInt;
868 break;
869 case 'U':
870 Kind = UInt;
871 break;
872 case 'B':
873 Kind = BFloat16;
874 ElementBitwidth = 16;
875 break;
876 case 'F':
877 Kind = Float;
878 break;
879 case 'P':
880 Kind = Poly;
881 break;
882 case '>':
883 assert(ElementBitwidth < 128);
884 ElementBitwidth *= 2;
885 break;
886 case '<':
887 assert(ElementBitwidth > 8);
888 ElementBitwidth /= 2;
889 break;
890 case '1':
891 NumVectors = 0;
892 break;
893 case '2':
894 NumVectors = 2;
895 break;
896 case '3':
897 NumVectors = 3;
898 break;
899 case '4':
900 NumVectors = 4;
901 break;
902 case '*':
903 Pointer = true;
904 break;
905 case 'c':
906 Constant = true;
907 break;
908 case 'Q':
909 Bitwidth = 128;
910 break;
911 case 'q':
912 Bitwidth = 64;
913 break;
914 case 'I':
915 Kind = SInt;
916 ElementBitwidth = Bitwidth = 32;
917 NumVectors = 0;
918 Immediate = true;
919 break;
920 case 'p':
921 if (isPoly())
922 Kind = UInt;
923 break;
924 case '!':
925 // Key type, handled elsewhere.
926 break;
927 default:
928 llvm_unreachable("Unhandled character!");
929 }
930 }
931 }
932
933 //===----------------------------------------------------------------------===//
934 // Intrinsic implementation
935 //===----------------------------------------------------------------------===//
936
getNextModifiers(StringRef Proto,unsigned & Pos) const937 StringRef Intrinsic::getNextModifiers(StringRef Proto, unsigned &Pos) const {
938 if (Proto.size() == Pos)
939 return StringRef();
940 else if (Proto[Pos] != '(')
941 return Proto.substr(Pos++, 1);
942
943 size_t Start = Pos + 1;
944 size_t End = Proto.find(')', Start);
945 assert_with_loc(End != StringRef::npos, "unmatched modifier group paren");
946 Pos = End + 1;
947 return Proto.slice(Start, End);
948 }
949
getInstTypeCode(Type T,ClassKind CK) const950 std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
951 char typeCode = '\0';
952 bool printNumber = true;
953
954 if (CK == ClassB)
955 return "";
956
957 if (T.isBFloat16())
958 return "bf16";
959
960 if (T.isPoly())
961 typeCode = 'p';
962 else if (T.isInteger())
963 typeCode = T.isSigned() ? 's' : 'u';
964 else
965 typeCode = 'f';
966
967 if (CK == ClassI) {
968 switch (typeCode) {
969 default:
970 break;
971 case 's':
972 case 'u':
973 case 'p':
974 typeCode = 'i';
975 break;
976 }
977 }
978 if (CK == ClassB) {
979 typeCode = '\0';
980 }
981
982 std::string S;
983 if (typeCode != '\0')
984 S.push_back(typeCode);
985 if (printNumber)
986 S += utostr(T.getElementSizeInBits());
987
988 return S;
989 }
990
getBuiltinTypeStr()991 std::string Intrinsic::getBuiltinTypeStr() {
992 ClassKind LocalCK = getClassKind(true);
993 std::string S;
994
995 Type RetT = getReturnType();
996 if ((LocalCK == ClassI || LocalCK == ClassW) && RetT.isScalar() &&
997 !RetT.isFloating() && !RetT.isBFloat16())
998 RetT.makeInteger(RetT.getElementSizeInBits(), false);
999
1000 // Since the return value must be one type, return a vector type of the
1001 // appropriate width which we will bitcast. An exception is made for
1002 // returning structs of 2, 3, or 4 vectors which are returned in a sret-like
1003 // fashion, storing them to a pointer arg.
1004 if (RetT.getNumVectors() > 1) {
1005 S += "vv*"; // void result with void* first argument
1006 } else {
1007 if (RetT.isPoly())
1008 RetT.makeInteger(RetT.getElementSizeInBits(), false);
1009 if (!RetT.isScalar() && RetT.isInteger() && !RetT.isSigned())
1010 RetT.makeSigned();
1011
1012 if (LocalCK == ClassB && RetT.isValue() && !RetT.isScalar())
1013 // Cast to vector of 8-bit elements.
1014 RetT.makeInteger(8, true);
1015
1016 S += RetT.builtin_str();
1017 }
1018
1019 for (unsigned I = 0; I < getNumParams(); ++I) {
1020 Type T = getParamType(I);
1021 if (T.isPoly())
1022 T.makeInteger(T.getElementSizeInBits(), false);
1023
1024 if (LocalCK == ClassB && !T.isScalar())
1025 T.makeInteger(8, true);
1026 // Halves always get converted to 8-bit elements.
1027 if (T.isHalf() && T.isVector() && !T.isScalarForMangling())
1028 T.makeInteger(8, true);
1029
1030 if (LocalCK == ClassI && T.isInteger())
1031 T.makeSigned();
1032
1033 if (hasImmediate() && getImmediateIdx() == I)
1034 T.makeImmediate(32);
1035
1036 S += T.builtin_str();
1037 }
1038
1039 // Extra constant integer to hold type class enum for this function, e.g. s8
1040 if (LocalCK == ClassB)
1041 S += "i";
1042
1043 return S;
1044 }
1045
getMangledName(bool ForceClassS) const1046 std::string Intrinsic::getMangledName(bool ForceClassS) const {
1047 // Check if the prototype has a scalar operand with the type of the vector
1048 // elements. If not, bitcasting the args will take care of arg checking.
1049 // The actual signedness etc. will be taken care of with special enums.
1050 ClassKind LocalCK = CK;
1051 if (!protoHasScalar())
1052 LocalCK = ClassB;
1053
1054 return mangleName(Name, ForceClassS ? ClassS : LocalCK);
1055 }
1056
mangleName(std::string Name,ClassKind LocalCK) const1057 std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
1058 std::string typeCode = getInstTypeCode(BaseType, LocalCK);
1059 std::string S = Name;
1060
1061 if (Name == "vcvt_f16_f32" || Name == "vcvt_f32_f16" ||
1062 Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32" ||
1063 Name == "vcvt_f32_bf16")
1064 return Name;
1065
1066 if (!typeCode.empty()) {
1067 // If the name ends with _xN (N = 2,3,4), insert the typeCode before _xN.
1068 if (Name.size() >= 3 && isdigit(Name.back()) &&
1069 Name[Name.length() - 2] == 'x' && Name[Name.length() - 3] == '_')
1070 S.insert(S.length() - 3, "_" + typeCode);
1071 else
1072 S += "_" + typeCode;
1073 }
1074
1075 if (BaseType != InBaseType) {
1076 // A reinterpret - out the input base type at the end.
1077 S += "_" + getInstTypeCode(InBaseType, LocalCK);
1078 }
1079
1080 if (LocalCK == ClassB)
1081 S += "_v";
1082
1083 // Insert a 'q' before the first '_' character so that it ends up before
1084 // _lane or _n on vector-scalar operations.
1085 if (BaseType.getSizeInBits() == 128 && !BaseType.noManglingQ()) {
1086 size_t Pos = S.find('_');
1087 S.insert(Pos, "q");
1088 }
1089
1090 char Suffix = '\0';
1091 if (BaseType.isScalarForMangling()) {
1092 switch (BaseType.getElementSizeInBits()) {
1093 case 8: Suffix = 'b'; break;
1094 case 16: Suffix = 'h'; break;
1095 case 32: Suffix = 's'; break;
1096 case 64: Suffix = 'd'; break;
1097 default: llvm_unreachable("Bad suffix!");
1098 }
1099 }
1100 if (Suffix != '\0') {
1101 size_t Pos = S.find('_');
1102 S.insert(Pos, &Suffix, 1);
1103 }
1104
1105 return S;
1106 }
1107
replaceParamsIn(std::string S)1108 std::string Intrinsic::replaceParamsIn(std::string S) {
1109 while (S.find('$') != std::string::npos) {
1110 size_t Pos = S.find('$');
1111 size_t End = Pos + 1;
1112 while (isalpha(S[End]))
1113 ++End;
1114
1115 std::string VarName = S.substr(Pos + 1, End - Pos - 1);
1116 assert_with_loc(Variables.find(VarName) != Variables.end(),
1117 "Variable not defined!");
1118 S.replace(Pos, End - Pos, Variables.find(VarName)->second.getName());
1119 }
1120
1121 return S;
1122 }
1123
initVariables()1124 void Intrinsic::initVariables() {
1125 Variables.clear();
1126
1127 // Modify the TypeSpec per-argument to get a concrete Type, and create
1128 // known variables for each.
1129 for (unsigned I = 1; I < Types.size(); ++I) {
1130 char NameC = '0' + (I - 1);
1131 std::string Name = "p";
1132 Name.push_back(NameC);
1133
1134 Variables[Name] = Variable(Types[I], Name + VariablePostfix);
1135 }
1136 RetVar = Variable(Types[0], "ret" + VariablePostfix);
1137 }
1138
emitPrototype(StringRef NamePrefix)1139 void Intrinsic::emitPrototype(StringRef NamePrefix) {
1140 if (UseMacro)
1141 OS << "#define ";
1142 else
1143 OS << "__ai " << Types[0].str() << " ";
1144
1145 OS << NamePrefix.str() << mangleName(Name, ClassS) << "(";
1146
1147 for (unsigned I = 0; I < getNumParams(); ++I) {
1148 if (I != 0)
1149 OS << ", ";
1150
1151 char NameC = '0' + I;
1152 std::string Name = "p";
1153 Name.push_back(NameC);
1154 assert(Variables.find(Name) != Variables.end());
1155 Variable &V = Variables[Name];
1156
1157 if (!UseMacro)
1158 OS << V.getType().str() << " ";
1159 OS << V.getName();
1160 }
1161
1162 OS << ")";
1163 }
1164
emitOpeningBrace()1165 void Intrinsic::emitOpeningBrace() {
1166 if (UseMacro)
1167 OS << " __extension__ ({";
1168 else
1169 OS << " {";
1170 emitNewLine();
1171 }
1172
emitClosingBrace()1173 void Intrinsic::emitClosingBrace() {
1174 if (UseMacro)
1175 OS << "})";
1176 else
1177 OS << "}";
1178 }
1179
emitNewLine()1180 void Intrinsic::emitNewLine() {
1181 if (UseMacro)
1182 OS << " \\\n";
1183 else
1184 OS << "\n";
1185 }
1186
emitReverseVariable(Variable & Dest,Variable & Src)1187 void Intrinsic::emitReverseVariable(Variable &Dest, Variable &Src) {
1188 if (Dest.getType().getNumVectors() > 1) {
1189 emitNewLine();
1190
1191 for (unsigned K = 0; K < Dest.getType().getNumVectors(); ++K) {
1192 OS << " " << Dest.getName() << ".val[" << K << "] = "
1193 << "__builtin_shufflevector("
1194 << Src.getName() << ".val[" << K << "], "
1195 << Src.getName() << ".val[" << K << "]";
1196 for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1197 OS << ", " << J;
1198 OS << ");";
1199 emitNewLine();
1200 }
1201 } else {
1202 OS << " " << Dest.getName()
1203 << " = __builtin_shufflevector(" << Src.getName() << ", " << Src.getName();
1204 for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1205 OS << ", " << J;
1206 OS << ");";
1207 emitNewLine();
1208 }
1209 }
1210
emitArgumentReversal()1211 void Intrinsic::emitArgumentReversal() {
1212 if (isBigEndianSafe())
1213 return;
1214
1215 // Reverse all vector arguments.
1216 for (unsigned I = 0; I < getNumParams(); ++I) {
1217 std::string Name = "p" + utostr(I);
1218 std::string NewName = "rev" + utostr(I);
1219
1220 Variable &V = Variables[Name];
1221 Variable NewV(V.getType(), NewName + VariablePostfix);
1222
1223 if (!NewV.getType().isVector() || NewV.getType().getNumElements() == 1)
1224 continue;
1225
1226 OS << " " << NewV.getType().str() << " " << NewV.getName() << ";";
1227 emitReverseVariable(NewV, V);
1228 V = NewV;
1229 }
1230 }
1231
emitReturnReversal()1232 void Intrinsic::emitReturnReversal() {
1233 if (isBigEndianSafe())
1234 return;
1235 if (!getReturnType().isVector() || getReturnType().isVoid() ||
1236 getReturnType().getNumElements() == 1)
1237 return;
1238 emitReverseVariable(RetVar, RetVar);
1239 }
1240
emitShadowedArgs()1241 void Intrinsic::emitShadowedArgs() {
1242 // Macro arguments are not type-checked like inline function arguments,
1243 // so assign them to local temporaries to get the right type checking.
1244 if (!UseMacro)
1245 return;
1246
1247 for (unsigned I = 0; I < getNumParams(); ++I) {
1248 // Do not create a temporary for an immediate argument.
1249 // That would defeat the whole point of using a macro!
1250 if (getParamType(I).isImmediate())
1251 continue;
1252 // Do not create a temporary for pointer arguments. The input
1253 // pointer may have an alignment hint.
1254 if (getParamType(I).isPointer())
1255 continue;
1256
1257 std::string Name = "p" + utostr(I);
1258
1259 assert(Variables.find(Name) != Variables.end());
1260 Variable &V = Variables[Name];
1261
1262 std::string NewName = "s" + utostr(I);
1263 Variable V2(V.getType(), NewName + VariablePostfix);
1264
1265 OS << " " << V2.getType().str() << " " << V2.getName() << " = "
1266 << V.getName() << ";";
1267 emitNewLine();
1268
1269 V = V2;
1270 }
1271 }
1272
protoHasScalar() const1273 bool Intrinsic::protoHasScalar() const {
1274 return std::any_of(Types.begin(), Types.end(), [](const Type &T) {
1275 return T.isScalar() && !T.isImmediate();
1276 });
1277 }
1278
emitBodyAsBuiltinCall()1279 void Intrinsic::emitBodyAsBuiltinCall() {
1280 std::string S;
1281
1282 // If this builtin returns a struct 2, 3, or 4 vectors, pass it as an implicit
1283 // sret-like argument.
1284 bool SRet = getReturnType().getNumVectors() >= 2;
1285
1286 StringRef N = Name;
1287 ClassKind LocalCK = CK;
1288 if (!protoHasScalar())
1289 LocalCK = ClassB;
1290
1291 if (!getReturnType().isVoid() && !SRet)
1292 S += "(" + RetVar.getType().str() + ") ";
1293
1294 S += "__builtin_neon_" + mangleName(std::string(N), LocalCK) + "(";
1295
1296 if (SRet)
1297 S += "&" + RetVar.getName() + ", ";
1298
1299 for (unsigned I = 0; I < getNumParams(); ++I) {
1300 Variable &V = Variables["p" + utostr(I)];
1301 Type T = V.getType();
1302
1303 // Handle multiple-vector values specially, emitting each subvector as an
1304 // argument to the builtin.
1305 if (T.getNumVectors() > 1) {
1306 // Check if an explicit cast is needed.
1307 std::string Cast;
1308 if (LocalCK == ClassB) {
1309 Type T2 = T;
1310 T2.makeOneVector();
1311 T2.makeInteger(8, /*Signed=*/true);
1312 Cast = "(" + T2.str() + ")";
1313 }
1314
1315 for (unsigned J = 0; J < T.getNumVectors(); ++J)
1316 S += Cast + V.getName() + ".val[" + utostr(J) + "], ";
1317 continue;
1318 }
1319
1320 std::string Arg = V.getName();
1321 Type CastToType = T;
1322
1323 // Check if an explicit cast is needed.
1324 if (CastToType.isVector() &&
1325 (LocalCK == ClassB || (T.isHalf() && !T.isScalarForMangling()))) {
1326 CastToType.makeInteger(8, true);
1327 Arg = "(" + CastToType.str() + ")" + Arg;
1328 } else if (CastToType.isVector() && LocalCK == ClassI) {
1329 if (CastToType.isInteger())
1330 CastToType.makeSigned();
1331 Arg = "(" + CastToType.str() + ")" + Arg;
1332 }
1333
1334 S += Arg + ", ";
1335 }
1336
1337 // Extra constant integer to hold type class enum for this function, e.g. s8
1338 if (getClassKind(true) == ClassB) {
1339 S += utostr(getPolymorphicKeyType().getNeonEnum());
1340 } else {
1341 // Remove extraneous ", ".
1342 S.pop_back();
1343 S.pop_back();
1344 }
1345 S += ");";
1346
1347 std::string RetExpr;
1348 if (!SRet && !RetVar.getType().isVoid())
1349 RetExpr = RetVar.getName() + " = ";
1350
1351 OS << " " << RetExpr << S;
1352 emitNewLine();
1353 }
1354
emitBody(StringRef CallPrefix)1355 void Intrinsic::emitBody(StringRef CallPrefix) {
1356 std::vector<std::string> Lines;
1357
1358 assert(RetVar.getType() == Types[0]);
1359 // Create a return variable, if we're not void.
1360 if (!RetVar.getType().isVoid()) {
1361 OS << " " << RetVar.getType().str() << " " << RetVar.getName() << ";";
1362 emitNewLine();
1363 }
1364
1365 if (!Body || Body->getValues().empty()) {
1366 // Nothing specific to output - must output a builtin.
1367 emitBodyAsBuiltinCall();
1368 return;
1369 }
1370
1371 // We have a list of "things to output". The last should be returned.
1372 for (auto *I : Body->getValues()) {
1373 if (StringInit *SI = dyn_cast<StringInit>(I)) {
1374 Lines.push_back(replaceParamsIn(SI->getAsString()));
1375 } else if (DagInit *DI = dyn_cast<DagInit>(I)) {
1376 DagEmitter DE(*this, CallPrefix);
1377 Lines.push_back(DE.emitDag(DI).second + ";");
1378 }
1379 }
1380
1381 assert(!Lines.empty() && "Empty def?");
1382 if (!RetVar.getType().isVoid())
1383 Lines.back().insert(0, RetVar.getName() + " = ");
1384
1385 for (auto &L : Lines) {
1386 OS << " " << L;
1387 emitNewLine();
1388 }
1389 }
1390
emitReturn()1391 void Intrinsic::emitReturn() {
1392 if (RetVar.getType().isVoid())
1393 return;
1394 if (UseMacro)
1395 OS << " " << RetVar.getName() << ";";
1396 else
1397 OS << " return " << RetVar.getName() << ";";
1398 emitNewLine();
1399 }
1400
emitDag(DagInit * DI)1401 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDag(DagInit *DI) {
1402 // At this point we should only be seeing a def.
1403 DefInit *DefI = cast<DefInit>(DI->getOperator());
1404 std::string Op = DefI->getAsString();
1405
1406 if (Op == "cast" || Op == "bitcast")
1407 return emitDagCast(DI, Op == "bitcast");
1408 if (Op == "shuffle")
1409 return emitDagShuffle(DI);
1410 if (Op == "dup")
1411 return emitDagDup(DI);
1412 if (Op == "dup_typed")
1413 return emitDagDupTyped(DI);
1414 if (Op == "splat")
1415 return emitDagSplat(DI);
1416 if (Op == "save_temp")
1417 return emitDagSaveTemp(DI);
1418 if (Op == "op")
1419 return emitDagOp(DI);
1420 if (Op == "call" || Op == "call_mangled")
1421 return emitDagCall(DI, Op == "call_mangled");
1422 if (Op == "name_replace")
1423 return emitDagNameReplace(DI);
1424 if (Op == "literal")
1425 return emitDagLiteral(DI);
1426 assert_with_loc(false, "Unknown operation!");
1427 return std::make_pair(Type::getVoid(), "");
1428 }
1429
emitDagOp(DagInit * DI)1430 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagOp(DagInit *DI) {
1431 std::string Op = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1432 if (DI->getNumArgs() == 2) {
1433 // Unary op.
1434 std::pair<Type, std::string> R =
1435 emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1436 return std::make_pair(R.first, Op + R.second);
1437 } else {
1438 assert(DI->getNumArgs() == 3 && "Can only handle unary and binary ops!");
1439 std::pair<Type, std::string> R1 =
1440 emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1441 std::pair<Type, std::string> R2 =
1442 emitDagArg(DI->getArg(2), std::string(DI->getArgNameStr(2)));
1443 assert_with_loc(R1.first == R2.first, "Argument type mismatch!");
1444 return std::make_pair(R1.first, R1.second + " " + Op + " " + R2.second);
1445 }
1446 }
1447
1448 std::pair<Type, std::string>
emitDagCall(DagInit * DI,bool MatchMangledName)1449 Intrinsic::DagEmitter::emitDagCall(DagInit *DI, bool MatchMangledName) {
1450 std::vector<Type> Types;
1451 std::vector<std::string> Values;
1452 for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1453 std::pair<Type, std::string> R =
1454 emitDagArg(DI->getArg(I + 1), std::string(DI->getArgNameStr(I + 1)));
1455 Types.push_back(R.first);
1456 Values.push_back(R.second);
1457 }
1458
1459 // Look up the called intrinsic.
1460 std::string N;
1461 if (StringInit *SI = dyn_cast<StringInit>(DI->getArg(0)))
1462 N = SI->getAsUnquotedString();
1463 else
1464 N = emitDagArg(DI->getArg(0), "").second;
1465 Optional<std::string> MangledName;
1466 if (MatchMangledName) {
1467 if (Intr.getRecord()->getValueAsBit("isLaneQ"))
1468 N += "q";
1469 MangledName = Intr.mangleName(N, ClassS);
1470 }
1471 Intrinsic &Callee = Intr.Emitter.getIntrinsic(N, Types, MangledName);
1472
1473 // Make sure the callee is known as an early def.
1474 Callee.setNeededEarly();
1475 Intr.Dependencies.insert(&Callee);
1476
1477 // Now create the call itself.
1478 std::string S = "";
1479 if (!Callee.isBigEndianSafe())
1480 S += CallPrefix.str();
1481 S += Callee.getMangledName(true) + "(";
1482 for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1483 if (I != 0)
1484 S += ", ";
1485 S += Values[I];
1486 }
1487 S += ")";
1488
1489 return std::make_pair(Callee.getReturnType(), S);
1490 }
1491
emitDagCast(DagInit * DI,bool IsBitCast)1492 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCast(DagInit *DI,
1493 bool IsBitCast){
1494 // (cast MOD* VAL) -> cast VAL to type given by MOD.
1495 std::pair<Type, std::string> R =
1496 emitDagArg(DI->getArg(DI->getNumArgs() - 1),
1497 std::string(DI->getArgNameStr(DI->getNumArgs() - 1)));
1498 Type castToType = R.first;
1499 for (unsigned ArgIdx = 0; ArgIdx < DI->getNumArgs() - 1; ++ArgIdx) {
1500
1501 // MOD can take several forms:
1502 // 1. $X - take the type of parameter / variable X.
1503 // 2. The value "R" - take the type of the return type.
1504 // 3. a type string
1505 // 4. The value "U" or "S" to switch the signedness.
1506 // 5. The value "H" or "D" to half or double the bitwidth.
1507 // 6. The value "8" to convert to 8-bit (signed) integer lanes.
1508 if (!DI->getArgNameStr(ArgIdx).empty()) {
1509 assert_with_loc(Intr.Variables.find(std::string(
1510 DI->getArgNameStr(ArgIdx))) != Intr.Variables.end(),
1511 "Variable not found");
1512 castToType =
1513 Intr.Variables[std::string(DI->getArgNameStr(ArgIdx))].getType();
1514 } else {
1515 StringInit *SI = dyn_cast<StringInit>(DI->getArg(ArgIdx));
1516 assert_with_loc(SI, "Expected string type or $Name for cast type");
1517
1518 if (SI->getAsUnquotedString() == "R") {
1519 castToType = Intr.getReturnType();
1520 } else if (SI->getAsUnquotedString() == "U") {
1521 castToType.makeUnsigned();
1522 } else if (SI->getAsUnquotedString() == "S") {
1523 castToType.makeSigned();
1524 } else if (SI->getAsUnquotedString() == "H") {
1525 castToType.halveLanes();
1526 } else if (SI->getAsUnquotedString() == "D") {
1527 castToType.doubleLanes();
1528 } else if (SI->getAsUnquotedString() == "8") {
1529 castToType.makeInteger(8, true);
1530 } else if (SI->getAsUnquotedString() == "32") {
1531 castToType.make32BitElement();
1532 } else {
1533 castToType = Type::fromTypedefName(SI->getAsUnquotedString());
1534 assert_with_loc(!castToType.isVoid(), "Unknown typedef");
1535 }
1536 }
1537 }
1538
1539 std::string S;
1540 if (IsBitCast) {
1541 // Emit a reinterpret cast. The second operand must be an lvalue, so create
1542 // a temporary.
1543 std::string N = "reint";
1544 unsigned I = 0;
1545 while (Intr.Variables.find(N) != Intr.Variables.end())
1546 N = "reint" + utostr(++I);
1547 Intr.Variables[N] = Variable(R.first, N + Intr.VariablePostfix);
1548
1549 Intr.OS << R.first.str() << " " << Intr.Variables[N].getName() << " = "
1550 << R.second << ";";
1551 Intr.emitNewLine();
1552
1553 S = "*(" + castToType.str() + " *) &" + Intr.Variables[N].getName() + "";
1554 } else {
1555 // Emit a normal (static) cast.
1556 S = "(" + castToType.str() + ")(" + R.second + ")";
1557 }
1558
1559 return std::make_pair(castToType, S);
1560 }
1561
emitDagShuffle(DagInit * DI)1562 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagShuffle(DagInit *DI){
1563 // See the documentation in arm_neon.td for a description of these operators.
1564 class LowHalf : public SetTheory::Operator {
1565 public:
1566 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1567 ArrayRef<SMLoc> Loc) override {
1568 SetTheory::RecSet Elts2;
1569 ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc);
1570 Elts.insert(Elts2.begin(), Elts2.begin() + (Elts2.size() / 2));
1571 }
1572 };
1573
1574 class HighHalf : public SetTheory::Operator {
1575 public:
1576 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1577 ArrayRef<SMLoc> Loc) override {
1578 SetTheory::RecSet Elts2;
1579 ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc);
1580 Elts.insert(Elts2.begin() + (Elts2.size() / 2), Elts2.end());
1581 }
1582 };
1583
1584 class Rev : public SetTheory::Operator {
1585 unsigned ElementSize;
1586
1587 public:
1588 Rev(unsigned ElementSize) : ElementSize(ElementSize) {}
1589
1590 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1591 ArrayRef<SMLoc> Loc) override {
1592 SetTheory::RecSet Elts2;
1593 ST.evaluate(Expr->arg_begin() + 1, Expr->arg_end(), Elts2, Loc);
1594
1595 int64_t VectorSize = cast<IntInit>(Expr->getArg(0))->getValue();
1596 VectorSize /= ElementSize;
1597
1598 std::vector<Record *> Revved;
1599 for (unsigned VI = 0; VI < Elts2.size(); VI += VectorSize) {
1600 for (int LI = VectorSize - 1; LI >= 0; --LI) {
1601 Revved.push_back(Elts2[VI + LI]);
1602 }
1603 }
1604
1605 Elts.insert(Revved.begin(), Revved.end());
1606 }
1607 };
1608
1609 class MaskExpander : public SetTheory::Expander {
1610 unsigned N;
1611
1612 public:
1613 MaskExpander(unsigned N) : N(N) {}
1614
1615 void expand(SetTheory &ST, Record *R, SetTheory::RecSet &Elts) override {
1616 unsigned Addend = 0;
1617 if (R->getName() == "mask0")
1618 Addend = 0;
1619 else if (R->getName() == "mask1")
1620 Addend = N;
1621 else
1622 return;
1623 for (unsigned I = 0; I < N; ++I)
1624 Elts.insert(R->getRecords().getDef("sv" + utostr(I + Addend)));
1625 }
1626 };
1627
1628 // (shuffle arg1, arg2, sequence)
1629 std::pair<Type, std::string> Arg1 =
1630 emitDagArg(DI->getArg(0), std::string(DI->getArgNameStr(0)));
1631 std::pair<Type, std::string> Arg2 =
1632 emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1633 assert_with_loc(Arg1.first == Arg2.first,
1634 "Different types in arguments to shuffle!");
1635
1636 SetTheory ST;
1637 SetTheory::RecSet Elts;
1638 ST.addOperator("lowhalf", std::make_unique<LowHalf>());
1639 ST.addOperator("highhalf", std::make_unique<HighHalf>());
1640 ST.addOperator("rev",
1641 std::make_unique<Rev>(Arg1.first.getElementSizeInBits()));
1642 ST.addExpander("MaskExpand",
1643 std::make_unique<MaskExpander>(Arg1.first.getNumElements()));
1644 ST.evaluate(DI->getArg(2), Elts, None);
1645
1646 std::string S = "__builtin_shufflevector(" + Arg1.second + ", " + Arg2.second;
1647 for (auto &E : Elts) {
1648 StringRef Name = E->getName();
1649 assert_with_loc(Name.startswith("sv"),
1650 "Incorrect element kind in shuffle mask!");
1651 S += ", " + Name.drop_front(2).str();
1652 }
1653 S += ")";
1654
1655 // Recalculate the return type - the shuffle may have halved or doubled it.
1656 Type T(Arg1.first);
1657 if (Elts.size() > T.getNumElements()) {
1658 assert_with_loc(
1659 Elts.size() == T.getNumElements() * 2,
1660 "Can only double or half the number of elements in a shuffle!");
1661 T.doubleLanes();
1662 } else if (Elts.size() < T.getNumElements()) {
1663 assert_with_loc(
1664 Elts.size() == T.getNumElements() / 2,
1665 "Can only double or half the number of elements in a shuffle!");
1666 T.halveLanes();
1667 }
1668
1669 return std::make_pair(T, S);
1670 }
1671
emitDagDup(DagInit * DI)1672 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDup(DagInit *DI) {
1673 assert_with_loc(DI->getNumArgs() == 1, "dup() expects one argument");
1674 std::pair<Type, std::string> A =
1675 emitDagArg(DI->getArg(0), std::string(DI->getArgNameStr(0)));
1676 assert_with_loc(A.first.isScalar(), "dup() expects a scalar argument");
1677
1678 Type T = Intr.getBaseType();
1679 assert_with_loc(T.isVector(), "dup() used but default type is scalar!");
1680 std::string S = "(" + T.str() + ") {";
1681 for (unsigned I = 0; I < T.getNumElements(); ++I) {
1682 if (I != 0)
1683 S += ", ";
1684 S += A.second;
1685 }
1686 S += "}";
1687
1688 return std::make_pair(T, S);
1689 }
1690
emitDagDupTyped(DagInit * DI)1691 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDupTyped(DagInit *DI) {
1692 assert_with_loc(DI->getNumArgs() == 2, "dup_typed() expects two arguments");
1693 std::pair<Type, std::string> B =
1694 emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1695 assert_with_loc(B.first.isScalar(),
1696 "dup_typed() requires a scalar as the second argument");
1697 Type T;
1698 // If the type argument is a constant string, construct the type directly.
1699 if (StringInit *SI = dyn_cast<StringInit>(DI->getArg(0))) {
1700 T = Type::fromTypedefName(SI->getAsUnquotedString());
1701 assert_with_loc(!T.isVoid(), "Unknown typedef");
1702 } else
1703 T = emitDagArg(DI->getArg(0), std::string(DI->getArgNameStr(0))).first;
1704
1705 assert_with_loc(T.isVector(), "dup_typed() used but target type is scalar!");
1706 std::string S = "(" + T.str() + ") {";
1707 for (unsigned I = 0; I < T.getNumElements(); ++I) {
1708 if (I != 0)
1709 S += ", ";
1710 S += B.second;
1711 }
1712 S += "}";
1713
1714 return std::make_pair(T, S);
1715 }
1716
emitDagSplat(DagInit * DI)1717 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSplat(DagInit *DI) {
1718 assert_with_loc(DI->getNumArgs() == 2, "splat() expects two arguments");
1719 std::pair<Type, std::string> A =
1720 emitDagArg(DI->getArg(0), std::string(DI->getArgNameStr(0)));
1721 std::pair<Type, std::string> B =
1722 emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1723
1724 assert_with_loc(B.first.isScalar(),
1725 "splat() requires a scalar int as the second argument");
1726
1727 std::string S = "__builtin_shufflevector(" + A.second + ", " + A.second;
1728 for (unsigned I = 0; I < Intr.getBaseType().getNumElements(); ++I) {
1729 S += ", " + B.second;
1730 }
1731 S += ")";
1732
1733 return std::make_pair(Intr.getBaseType(), S);
1734 }
1735
emitDagSaveTemp(DagInit * DI)1736 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSaveTemp(DagInit *DI) {
1737 assert_with_loc(DI->getNumArgs() == 2, "save_temp() expects two arguments");
1738 std::pair<Type, std::string> A =
1739 emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1740
1741 assert_with_loc(!A.first.isVoid(),
1742 "Argument to save_temp() must have non-void type!");
1743
1744 std::string N = std::string(DI->getArgNameStr(0));
1745 assert_with_loc(!N.empty(),
1746 "save_temp() expects a name as the first argument");
1747
1748 assert_with_loc(Intr.Variables.find(N) == Intr.Variables.end(),
1749 "Variable already defined!");
1750 Intr.Variables[N] = Variable(A.first, N + Intr.VariablePostfix);
1751
1752 std::string S =
1753 A.first.str() + " " + Intr.Variables[N].getName() + " = " + A.second;
1754
1755 return std::make_pair(Type::getVoid(), S);
1756 }
1757
1758 std::pair<Type, std::string>
emitDagNameReplace(DagInit * DI)1759 Intrinsic::DagEmitter::emitDagNameReplace(DagInit *DI) {
1760 std::string S = Intr.Name;
1761
1762 assert_with_loc(DI->getNumArgs() == 2, "name_replace requires 2 arguments!");
1763 std::string ToReplace = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1764 std::string ReplaceWith = cast<StringInit>(DI->getArg(1))->getAsUnquotedString();
1765
1766 size_t Idx = S.find(ToReplace);
1767
1768 assert_with_loc(Idx != std::string::npos, "name should contain '" + ToReplace + "'!");
1769 S.replace(Idx, ToReplace.size(), ReplaceWith);
1770
1771 return std::make_pair(Type::getVoid(), S);
1772 }
1773
emitDagLiteral(DagInit * DI)1774 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagLiteral(DagInit *DI){
1775 std::string Ty = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1776 std::string Value = cast<StringInit>(DI->getArg(1))->getAsUnquotedString();
1777 return std::make_pair(Type::fromTypedefName(Ty), Value);
1778 }
1779
1780 std::pair<Type, std::string>
emitDagArg(Init * Arg,std::string ArgName)1781 Intrinsic::DagEmitter::emitDagArg(Init *Arg, std::string ArgName) {
1782 if (!ArgName.empty()) {
1783 assert_with_loc(!Arg->isComplete(),
1784 "Arguments must either be DAGs or names, not both!");
1785 assert_with_loc(Intr.Variables.find(ArgName) != Intr.Variables.end(),
1786 "Variable not defined!");
1787 Variable &V = Intr.Variables[ArgName];
1788 return std::make_pair(V.getType(), V.getName());
1789 }
1790
1791 assert(Arg && "Neither ArgName nor Arg?!");
1792 DagInit *DI = dyn_cast<DagInit>(Arg);
1793 assert_with_loc(DI, "Arguments must either be DAGs or names!");
1794
1795 return emitDag(DI);
1796 }
1797
generate()1798 std::string Intrinsic::generate() {
1799 // Avoid duplicated code for big and little endian
1800 if (isBigEndianSafe()) {
1801 generateImpl(false, "", "");
1802 return OS.str();
1803 }
1804 // Little endian intrinsics are simple and don't require any argument
1805 // swapping.
1806 OS << "#ifdef __LITTLE_ENDIAN__\n";
1807
1808 generateImpl(false, "", "");
1809
1810 OS << "#else\n";
1811
1812 // Big endian intrinsics are more complex. The user intended these
1813 // intrinsics to operate on a vector "as-if" loaded by (V)LDR,
1814 // but we load as-if (V)LD1. So we should swap all arguments and
1815 // swap the return value too.
1816 //
1817 // If we call sub-intrinsics, we should call a version that does
1818 // not re-swap the arguments!
1819 generateImpl(true, "", "__noswap_");
1820
1821 // If we're needed early, create a non-swapping variant for
1822 // big-endian.
1823 if (NeededEarly) {
1824 generateImpl(false, "__noswap_", "__noswap_");
1825 }
1826 OS << "#endif\n\n";
1827
1828 return OS.str();
1829 }
1830
generateImpl(bool ReverseArguments,StringRef NamePrefix,StringRef CallPrefix)1831 void Intrinsic::generateImpl(bool ReverseArguments,
1832 StringRef NamePrefix, StringRef CallPrefix) {
1833 CurrentRecord = R;
1834
1835 // If we call a macro, our local variables may be corrupted due to
1836 // lack of proper lexical scoping. So, add a globally unique postfix
1837 // to every variable.
1838 //
1839 // indexBody() should have set up the Dependencies set by now.
1840 for (auto *I : Dependencies)
1841 if (I->UseMacro) {
1842 VariablePostfix = "_" + utostr(Emitter.getUniqueNumber());
1843 break;
1844 }
1845
1846 initVariables();
1847
1848 emitPrototype(NamePrefix);
1849
1850 if (IsUnavailable) {
1851 OS << " __attribute__((unavailable));";
1852 } else {
1853 emitOpeningBrace();
1854 emitShadowedArgs();
1855 if (ReverseArguments)
1856 emitArgumentReversal();
1857 emitBody(CallPrefix);
1858 if (ReverseArguments)
1859 emitReturnReversal();
1860 emitReturn();
1861 emitClosingBrace();
1862 }
1863 OS << "\n";
1864
1865 CurrentRecord = nullptr;
1866 }
1867
indexBody()1868 void Intrinsic::indexBody() {
1869 CurrentRecord = R;
1870
1871 initVariables();
1872 emitBody("");
1873 OS.str("");
1874
1875 CurrentRecord = nullptr;
1876 }
1877
1878 //===----------------------------------------------------------------------===//
1879 // NeonEmitter implementation
1880 //===----------------------------------------------------------------------===//
1881
getIntrinsic(StringRef Name,ArrayRef<Type> Types,Optional<std::string> MangledName)1882 Intrinsic &NeonEmitter::getIntrinsic(StringRef Name, ArrayRef<Type> Types,
1883 Optional<std::string> MangledName) {
1884 // First, look up the name in the intrinsic map.
1885 assert_with_loc(IntrinsicMap.find(Name.str()) != IntrinsicMap.end(),
1886 ("Intrinsic '" + Name + "' not found!").str());
1887 auto &V = IntrinsicMap.find(Name.str())->second;
1888 std::vector<Intrinsic *> GoodVec;
1889
1890 // Create a string to print if we end up failing.
1891 std::string ErrMsg = "looking up intrinsic '" + Name.str() + "(";
1892 for (unsigned I = 0; I < Types.size(); ++I) {
1893 if (I != 0)
1894 ErrMsg += ", ";
1895 ErrMsg += Types[I].str();
1896 }
1897 ErrMsg += ")'\n";
1898 ErrMsg += "Available overloads:\n";
1899
1900 // Now, look through each intrinsic implementation and see if the types are
1901 // compatible.
1902 for (auto &I : V) {
1903 ErrMsg += " - " + I.getReturnType().str() + " " + I.getMangledName();
1904 ErrMsg += "(";
1905 for (unsigned A = 0; A < I.getNumParams(); ++A) {
1906 if (A != 0)
1907 ErrMsg += ", ";
1908 ErrMsg += I.getParamType(A).str();
1909 }
1910 ErrMsg += ")\n";
1911
1912 if (MangledName && MangledName != I.getMangledName(true))
1913 continue;
1914
1915 if (I.getNumParams() != Types.size())
1916 continue;
1917
1918 unsigned ArgNum = 0;
1919 bool MatchingArgumentTypes =
1920 std::all_of(Types.begin(), Types.end(), [&](const auto &Type) {
1921 return Type == I.getParamType(ArgNum++);
1922 });
1923
1924 if (MatchingArgumentTypes)
1925 GoodVec.push_back(&I);
1926 }
1927
1928 assert_with_loc(!GoodVec.empty(),
1929 "No compatible intrinsic found - " + ErrMsg);
1930 assert_with_loc(GoodVec.size() == 1, "Multiple overloads found - " + ErrMsg);
1931
1932 return *GoodVec.front();
1933 }
1934
createIntrinsic(Record * R,SmallVectorImpl<Intrinsic * > & Out)1935 void NeonEmitter::createIntrinsic(Record *R,
1936 SmallVectorImpl<Intrinsic *> &Out) {
1937 std::string Name = std::string(R->getValueAsString("Name"));
1938 std::string Proto = std::string(R->getValueAsString("Prototype"));
1939 std::string Types = std::string(R->getValueAsString("Types"));
1940 Record *OperationRec = R->getValueAsDef("Operation");
1941 bool BigEndianSafe = R->getValueAsBit("BigEndianSafe");
1942 std::string Guard = std::string(R->getValueAsString("ArchGuard"));
1943 bool IsUnavailable = OperationRec->getValueAsBit("Unavailable");
1944 std::string CartesianProductWith = std::string(R->getValueAsString("CartesianProductWith"));
1945
1946 // Set the global current record. This allows assert_with_loc to produce
1947 // decent location information even when highly nested.
1948 CurrentRecord = R;
1949
1950 ListInit *Body = OperationRec->getValueAsListInit("Ops");
1951
1952 std::vector<TypeSpec> TypeSpecs = TypeSpec::fromTypeSpecs(Types);
1953
1954 ClassKind CK = ClassNone;
1955 if (R->getSuperClasses().size() >= 2)
1956 CK = ClassMap[R->getSuperClasses()[1].first];
1957
1958 std::vector<std::pair<TypeSpec, TypeSpec>> NewTypeSpecs;
1959 if (!CartesianProductWith.empty()) {
1960 std::vector<TypeSpec> ProductTypeSpecs = TypeSpec::fromTypeSpecs(CartesianProductWith);
1961 for (auto TS : TypeSpecs) {
1962 Type DefaultT(TS, ".");
1963 for (auto SrcTS : ProductTypeSpecs) {
1964 Type DefaultSrcT(SrcTS, ".");
1965 if (TS == SrcTS ||
1966 DefaultSrcT.getSizeInBits() != DefaultT.getSizeInBits())
1967 continue;
1968 NewTypeSpecs.push_back(std::make_pair(TS, SrcTS));
1969 }
1970 }
1971 } else {
1972 for (auto TS : TypeSpecs) {
1973 NewTypeSpecs.push_back(std::make_pair(TS, TS));
1974 }
1975 }
1976
1977 llvm::sort(NewTypeSpecs);
1978 NewTypeSpecs.erase(std::unique(NewTypeSpecs.begin(), NewTypeSpecs.end()),
1979 NewTypeSpecs.end());
1980 auto &Entry = IntrinsicMap[Name];
1981
1982 for (auto &I : NewTypeSpecs) {
1983 Entry.emplace_back(R, Name, Proto, I.first, I.second, CK, Body, *this,
1984 Guard, IsUnavailable, BigEndianSafe);
1985 Out.push_back(&Entry.back());
1986 }
1987
1988 CurrentRecord = nullptr;
1989 }
1990
1991 /// genBuiltinsDef: Generate the BuiltinsARM.def and BuiltinsAArch64.def
1992 /// declaration of builtins, checking for unique builtin declarations.
genBuiltinsDef(raw_ostream & OS,SmallVectorImpl<Intrinsic * > & Defs)1993 void NeonEmitter::genBuiltinsDef(raw_ostream &OS,
1994 SmallVectorImpl<Intrinsic *> &Defs) {
1995 OS << "#ifdef GET_NEON_BUILTINS\n";
1996
1997 // We only want to emit a builtin once, and we want to emit them in
1998 // alphabetical order, so use a std::set.
1999 std::set<std::string> Builtins;
2000
2001 for (auto *Def : Defs) {
2002 if (Def->hasBody())
2003 continue;
2004
2005 std::string S = "BUILTIN(__builtin_neon_" + Def->getMangledName() + ", \"";
2006
2007 S += Def->getBuiltinTypeStr();
2008 S += "\", \"n\")";
2009
2010 Builtins.insert(S);
2011 }
2012
2013 for (auto &S : Builtins)
2014 OS << S << "\n";
2015 OS << "#endif\n\n";
2016 }
2017
2018 /// Generate the ARM and AArch64 overloaded type checking code for
2019 /// SemaChecking.cpp, checking for unique builtin declarations.
genOverloadTypeCheckCode(raw_ostream & OS,SmallVectorImpl<Intrinsic * > & Defs)2020 void NeonEmitter::genOverloadTypeCheckCode(raw_ostream &OS,
2021 SmallVectorImpl<Intrinsic *> &Defs) {
2022 OS << "#ifdef GET_NEON_OVERLOAD_CHECK\n";
2023
2024 // We record each overload check line before emitting because subsequent Inst
2025 // definitions may extend the number of permitted types (i.e. augment the
2026 // Mask). Use std::map to avoid sorting the table by hash number.
2027 struct OverloadInfo {
2028 uint64_t Mask;
2029 int PtrArgNum;
2030 bool HasConstPtr;
2031 OverloadInfo() : Mask(0ULL), PtrArgNum(0), HasConstPtr(false) {}
2032 };
2033 std::map<std::string, OverloadInfo> OverloadMap;
2034
2035 for (auto *Def : Defs) {
2036 // If the def has a body (that is, it has Operation DAGs), it won't call
2037 // __builtin_neon_* so we don't need to generate a definition for it.
2038 if (Def->hasBody())
2039 continue;
2040 // Functions which have a scalar argument cannot be overloaded, no need to
2041 // check them if we are emitting the type checking code.
2042 if (Def->protoHasScalar())
2043 continue;
2044
2045 uint64_t Mask = 0ULL;
2046 Mask |= 1ULL << Def->getPolymorphicKeyType().getNeonEnum();
2047
2048 // Check if the function has a pointer or const pointer argument.
2049 int PtrArgNum = -1;
2050 bool HasConstPtr = false;
2051 for (unsigned I = 0; I < Def->getNumParams(); ++I) {
2052 const auto &Type = Def->getParamType(I);
2053 if (Type.isPointer()) {
2054 PtrArgNum = I;
2055 HasConstPtr = Type.isConstPointer();
2056 }
2057 }
2058
2059 // For sret builtins, adjust the pointer argument index.
2060 if (PtrArgNum >= 0 && Def->getReturnType().getNumVectors() > 1)
2061 PtrArgNum += 1;
2062
2063 std::string Name = Def->getName();
2064 // Omit type checking for the pointer arguments of vld1_lane, vld1_dup,
2065 // and vst1_lane intrinsics. Using a pointer to the vector element
2066 // type with one of those operations causes codegen to select an aligned
2067 // load/store instruction. If you want an unaligned operation,
2068 // the pointer argument needs to have less alignment than element type,
2069 // so just accept any pointer type.
2070 if (Name == "vld1_lane" || Name == "vld1_dup" || Name == "vst1_lane") {
2071 PtrArgNum = -1;
2072 HasConstPtr = false;
2073 }
2074
2075 if (Mask) {
2076 std::string Name = Def->getMangledName();
2077 OverloadMap.insert(std::make_pair(Name, OverloadInfo()));
2078 OverloadInfo &OI = OverloadMap[Name];
2079 OI.Mask |= Mask;
2080 OI.PtrArgNum |= PtrArgNum;
2081 OI.HasConstPtr = HasConstPtr;
2082 }
2083 }
2084
2085 for (auto &I : OverloadMap) {
2086 OverloadInfo &OI = I.second;
2087
2088 OS << "case NEON::BI__builtin_neon_" << I.first << ": ";
2089 OS << "mask = 0x" << Twine::utohexstr(OI.Mask) << "ULL";
2090 if (OI.PtrArgNum >= 0)
2091 OS << "; PtrArgNum = " << OI.PtrArgNum;
2092 if (OI.HasConstPtr)
2093 OS << "; HasConstPtr = true";
2094 OS << "; break;\n";
2095 }
2096 OS << "#endif\n\n";
2097 }
2098
genIntrinsicRangeCheckCode(raw_ostream & OS,SmallVectorImpl<Intrinsic * > & Defs)2099 void NeonEmitter::genIntrinsicRangeCheckCode(raw_ostream &OS,
2100 SmallVectorImpl<Intrinsic *> &Defs) {
2101 OS << "#ifdef GET_NEON_IMMEDIATE_CHECK\n";
2102
2103 std::set<std::string> Emitted;
2104
2105 for (auto *Def : Defs) {
2106 if (Def->hasBody())
2107 continue;
2108 // Functions which do not have an immediate do not need to have range
2109 // checking code emitted.
2110 if (!Def->hasImmediate())
2111 continue;
2112 if (Emitted.find(Def->getMangledName()) != Emitted.end())
2113 continue;
2114
2115 std::string LowerBound, UpperBound;
2116
2117 Record *R = Def->getRecord();
2118 if (R->getValueAsBit("isVXAR")) {
2119 //VXAR takes an immediate in the range [0, 63]
2120 LowerBound = "0";
2121 UpperBound = "63";
2122 } else if (R->getValueAsBit("isVCVT_N")) {
2123 // VCVT between floating- and fixed-point values takes an immediate
2124 // in the range [1, 32) for f32 or [1, 64) for f64 or [1, 16) for f16.
2125 LowerBound = "1";
2126 if (Def->getBaseType().getElementSizeInBits() == 16 ||
2127 Def->getName().find('h') != std::string::npos)
2128 // VCVTh operating on FP16 intrinsics in range [1, 16)
2129 UpperBound = "15";
2130 else if (Def->getBaseType().getElementSizeInBits() == 32)
2131 UpperBound = "31";
2132 else
2133 UpperBound = "63";
2134 } else if (R->getValueAsBit("isScalarShift")) {
2135 // Right shifts have an 'r' in the name, left shifts do not. Convert
2136 // instructions have the same bounds and right shifts.
2137 if (Def->getName().find('r') != std::string::npos ||
2138 Def->getName().find("cvt") != std::string::npos)
2139 LowerBound = "1";
2140
2141 UpperBound = utostr(Def->getReturnType().getElementSizeInBits() - 1);
2142 } else if (R->getValueAsBit("isShift")) {
2143 // Builtins which are overloaded by type will need to have their upper
2144 // bound computed at Sema time based on the type constant.
2145
2146 // Right shifts have an 'r' in the name, left shifts do not.
2147 if (Def->getName().find('r') != std::string::npos)
2148 LowerBound = "1";
2149 UpperBound = "RFT(TV, true)";
2150 } else if (Def->getClassKind(true) == ClassB) {
2151 // ClassB intrinsics have a type (and hence lane number) that is only
2152 // known at runtime.
2153 if (R->getValueAsBit("isLaneQ"))
2154 UpperBound = "RFT(TV, false, true)";
2155 else
2156 UpperBound = "RFT(TV, false, false)";
2157 } else {
2158 // The immediate generally refers to a lane in the preceding argument.
2159 assert(Def->getImmediateIdx() > 0);
2160 Type T = Def->getParamType(Def->getImmediateIdx() - 1);
2161 UpperBound = utostr(T.getNumElements() - 1);
2162 }
2163
2164 // Calculate the index of the immediate that should be range checked.
2165 unsigned Idx = Def->getNumParams();
2166 if (Def->hasImmediate())
2167 Idx = Def->getGeneratedParamIdx(Def->getImmediateIdx());
2168
2169 OS << "case NEON::BI__builtin_neon_" << Def->getMangledName() << ": "
2170 << "i = " << Idx << ";";
2171 if (!LowerBound.empty())
2172 OS << " l = " << LowerBound << ";";
2173 if (!UpperBound.empty())
2174 OS << " u = " << UpperBound << ";";
2175 OS << " break;\n";
2176
2177 Emitted.insert(Def->getMangledName());
2178 }
2179
2180 OS << "#endif\n\n";
2181 }
2182
2183 /// runHeader - Emit a file with sections defining:
2184 /// 1. the NEON section of BuiltinsARM.def and BuiltinsAArch64.def.
2185 /// 2. the SemaChecking code for the type overload checking.
2186 /// 3. the SemaChecking code for validation of intrinsic immediate arguments.
runHeader(raw_ostream & OS)2187 void NeonEmitter::runHeader(raw_ostream &OS) {
2188 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2189
2190 SmallVector<Intrinsic *, 128> Defs;
2191 for (auto *R : RV)
2192 createIntrinsic(R, Defs);
2193
2194 // Generate shared BuiltinsXXX.def
2195 genBuiltinsDef(OS, Defs);
2196
2197 // Generate ARM overloaded type checking code for SemaChecking.cpp
2198 genOverloadTypeCheckCode(OS, Defs);
2199
2200 // Generate ARM range checking code for shift/lane immediates.
2201 genIntrinsicRangeCheckCode(OS, Defs);
2202 }
2203
emitNeonTypeDefs(const std::string & types,raw_ostream & OS)2204 static void emitNeonTypeDefs(const std::string& types, raw_ostream &OS) {
2205 std::string TypedefTypes(types);
2206 std::vector<TypeSpec> TDTypeVec = TypeSpec::fromTypeSpecs(TypedefTypes);
2207
2208 // Emit vector typedefs.
2209 bool InIfdef = false;
2210 for (auto &TS : TDTypeVec) {
2211 bool IsA64 = false;
2212 Type T(TS, ".");
2213 if (T.isDouble())
2214 IsA64 = true;
2215
2216 if (InIfdef && !IsA64) {
2217 OS << "#endif\n";
2218 InIfdef = false;
2219 }
2220 if (!InIfdef && IsA64) {
2221 OS << "#ifdef __aarch64__\n";
2222 InIfdef = true;
2223 }
2224
2225 if (T.isPoly())
2226 OS << "typedef __attribute__((neon_polyvector_type(";
2227 else
2228 OS << "typedef __attribute__((neon_vector_type(";
2229
2230 Type T2 = T;
2231 T2.makeScalar();
2232 OS << T.getNumElements() << "))) ";
2233 OS << T2.str();
2234 OS << " " << T.str() << ";\n";
2235 }
2236 if (InIfdef)
2237 OS << "#endif\n";
2238 OS << "\n";
2239
2240 // Emit struct typedefs.
2241 InIfdef = false;
2242 for (unsigned NumMembers = 2; NumMembers <= 4; ++NumMembers) {
2243 for (auto &TS : TDTypeVec) {
2244 bool IsA64 = false;
2245 Type T(TS, ".");
2246 if (T.isDouble())
2247 IsA64 = true;
2248
2249 if (InIfdef && !IsA64) {
2250 OS << "#endif\n";
2251 InIfdef = false;
2252 }
2253 if (!InIfdef && IsA64) {
2254 OS << "#ifdef __aarch64__\n";
2255 InIfdef = true;
2256 }
2257
2258 const char Mods[] = { static_cast<char>('2' + (NumMembers - 2)), 0};
2259 Type VT(TS, Mods);
2260 OS << "typedef struct " << VT.str() << " {\n";
2261 OS << " " << T.str() << " val";
2262 OS << "[" << NumMembers << "]";
2263 OS << ";\n} ";
2264 OS << VT.str() << ";\n";
2265 OS << "\n";
2266 }
2267 }
2268 if (InIfdef)
2269 OS << "#endif\n";
2270 }
2271
2272 /// run - Read the records in arm_neon.td and output arm_neon.h. arm_neon.h
2273 /// is comprised of type definitions and function declarations.
run(raw_ostream & OS)2274 void NeonEmitter::run(raw_ostream &OS) {
2275 OS << "/*===---- arm_neon.h - ARM Neon intrinsics "
2276 "------------------------------"
2277 "---===\n"
2278 " *\n"
2279 " * Permission is hereby granted, free of charge, to any person "
2280 "obtaining "
2281 "a copy\n"
2282 " * of this software and associated documentation files (the "
2283 "\"Software\"),"
2284 " to deal\n"
2285 " * in the Software without restriction, including without limitation "
2286 "the "
2287 "rights\n"
2288 " * to use, copy, modify, merge, publish, distribute, sublicense, "
2289 "and/or sell\n"
2290 " * copies of the Software, and to permit persons to whom the Software "
2291 "is\n"
2292 " * furnished to do so, subject to the following conditions:\n"
2293 " *\n"
2294 " * The above copyright notice and this permission notice shall be "
2295 "included in\n"
2296 " * all copies or substantial portions of the Software.\n"
2297 " *\n"
2298 " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, "
2299 "EXPRESS OR\n"
2300 " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF "
2301 "MERCHANTABILITY,\n"
2302 " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT "
2303 "SHALL THE\n"
2304 " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR "
2305 "OTHER\n"
2306 " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, "
2307 "ARISING FROM,\n"
2308 " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER "
2309 "DEALINGS IN\n"
2310 " * THE SOFTWARE.\n"
2311 " *\n"
2312 " *===-----------------------------------------------------------------"
2313 "---"
2314 "---===\n"
2315 " */\n\n";
2316
2317 OS << "#ifndef __ARM_NEON_H\n";
2318 OS << "#define __ARM_NEON_H\n\n";
2319
2320 OS << "#ifndef __ARM_FP\n";
2321 OS << "#error \"NEON intrinsics not available with the soft-float ABI. "
2322 "Please use -mfloat-abi=softfp or -mfloat-abi=hard\"\n";
2323 OS << "#else\n\n";
2324
2325 OS << "#if !defined(__ARM_NEON)\n";
2326 OS << "#error \"NEON support not enabled\"\n";
2327 OS << "#else\n\n";
2328
2329 OS << "#include <stdint.h>\n\n";
2330
2331 OS << "#ifdef __ARM_FEATURE_BF16\n";
2332 OS << "#include <arm_bf16.h>\n";
2333 OS << "typedef __bf16 bfloat16_t;\n";
2334 OS << "#endif\n\n";
2335
2336 // Emit NEON-specific scalar typedefs.
2337 OS << "typedef float float32_t;\n";
2338 OS << "typedef __fp16 float16_t;\n";
2339
2340 OS << "#ifdef __aarch64__\n";
2341 OS << "typedef double float64_t;\n";
2342 OS << "#endif\n\n";
2343
2344 // For now, signedness of polynomial types depends on target
2345 OS << "#ifdef __aarch64__\n";
2346 OS << "typedef uint8_t poly8_t;\n";
2347 OS << "typedef uint16_t poly16_t;\n";
2348 OS << "typedef uint64_t poly64_t;\n";
2349 OS << "typedef __uint128_t poly128_t;\n";
2350 OS << "#else\n";
2351 OS << "typedef int8_t poly8_t;\n";
2352 OS << "typedef int16_t poly16_t;\n";
2353 OS << "typedef int64_t poly64_t;\n";
2354 OS << "#endif\n";
2355
2356 emitNeonTypeDefs("cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQdPcQPcPsQPsPlQPl", OS);
2357
2358 OS << "#ifdef __ARM_FEATURE_BF16\n";
2359 emitNeonTypeDefs("bQb", OS);
2360 OS << "#endif\n\n";
2361
2362 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
2363 "__nodebug__))\n\n";
2364
2365 SmallVector<Intrinsic *, 128> Defs;
2366 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2367 for (auto *R : RV)
2368 createIntrinsic(R, Defs);
2369
2370 for (auto *I : Defs)
2371 I->indexBody();
2372
2373 llvm::stable_sort(Defs, llvm::deref<std::less<>>());
2374
2375 // Only emit a def when its requirements have been met.
2376 // FIXME: This loop could be made faster, but it's fast enough for now.
2377 bool MadeProgress = true;
2378 std::string InGuard;
2379 while (!Defs.empty() && MadeProgress) {
2380 MadeProgress = false;
2381
2382 for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2383 I != Defs.end(); /*No step*/) {
2384 bool DependenciesSatisfied = true;
2385 for (auto *II : (*I)->getDependencies()) {
2386 if (llvm::is_contained(Defs, II))
2387 DependenciesSatisfied = false;
2388 }
2389 if (!DependenciesSatisfied) {
2390 // Try the next one.
2391 ++I;
2392 continue;
2393 }
2394
2395 // Emit #endif/#if pair if needed.
2396 if ((*I)->getGuard() != InGuard) {
2397 if (!InGuard.empty())
2398 OS << "#endif\n";
2399 InGuard = (*I)->getGuard();
2400 if (!InGuard.empty())
2401 OS << "#if " << InGuard << "\n";
2402 }
2403
2404 // Actually generate the intrinsic code.
2405 OS << (*I)->generate();
2406
2407 MadeProgress = true;
2408 I = Defs.erase(I);
2409 }
2410 }
2411 assert(Defs.empty() && "Some requirements were not satisfied!");
2412 if (!InGuard.empty())
2413 OS << "#endif\n";
2414
2415 OS << "\n";
2416 OS << "#undef __ai\n\n";
2417 OS << "#endif /* if !defined(__ARM_NEON) */\n";
2418 OS << "#endif /* ifndef __ARM_FP */\n";
2419 OS << "#endif /* __ARM_NEON_H */\n";
2420 }
2421
2422 /// run - Read the records in arm_fp16.td and output arm_fp16.h. arm_fp16.h
2423 /// is comprised of type definitions and function declarations.
runFP16(raw_ostream & OS)2424 void NeonEmitter::runFP16(raw_ostream &OS) {
2425 OS << "/*===---- arm_fp16.h - ARM FP16 intrinsics "
2426 "------------------------------"
2427 "---===\n"
2428 " *\n"
2429 " * Permission is hereby granted, free of charge, to any person "
2430 "obtaining a copy\n"
2431 " * of this software and associated documentation files (the "
2432 "\"Software\"), to deal\n"
2433 " * in the Software without restriction, including without limitation "
2434 "the rights\n"
2435 " * to use, copy, modify, merge, publish, distribute, sublicense, "
2436 "and/or sell\n"
2437 " * copies of the Software, and to permit persons to whom the Software "
2438 "is\n"
2439 " * furnished to do so, subject to the following conditions:\n"
2440 " *\n"
2441 " * The above copyright notice and this permission notice shall be "
2442 "included in\n"
2443 " * all copies or substantial portions of the Software.\n"
2444 " *\n"
2445 " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, "
2446 "EXPRESS OR\n"
2447 " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF "
2448 "MERCHANTABILITY,\n"
2449 " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT "
2450 "SHALL THE\n"
2451 " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR "
2452 "OTHER\n"
2453 " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, "
2454 "ARISING FROM,\n"
2455 " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER "
2456 "DEALINGS IN\n"
2457 " * THE SOFTWARE.\n"
2458 " *\n"
2459 " *===-----------------------------------------------------------------"
2460 "---"
2461 "---===\n"
2462 " */\n\n";
2463
2464 OS << "#ifndef __ARM_FP16_H\n";
2465 OS << "#define __ARM_FP16_H\n\n";
2466
2467 OS << "#include <stdint.h>\n\n";
2468
2469 OS << "typedef __fp16 float16_t;\n";
2470
2471 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
2472 "__nodebug__))\n\n";
2473
2474 SmallVector<Intrinsic *, 128> Defs;
2475 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2476 for (auto *R : RV)
2477 createIntrinsic(R, Defs);
2478
2479 for (auto *I : Defs)
2480 I->indexBody();
2481
2482 llvm::stable_sort(Defs, llvm::deref<std::less<>>());
2483
2484 // Only emit a def when its requirements have been met.
2485 // FIXME: This loop could be made faster, but it's fast enough for now.
2486 bool MadeProgress = true;
2487 std::string InGuard;
2488 while (!Defs.empty() && MadeProgress) {
2489 MadeProgress = false;
2490
2491 for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2492 I != Defs.end(); /*No step*/) {
2493 bool DependenciesSatisfied = true;
2494 for (auto *II : (*I)->getDependencies()) {
2495 if (llvm::is_contained(Defs, II))
2496 DependenciesSatisfied = false;
2497 }
2498 if (!DependenciesSatisfied) {
2499 // Try the next one.
2500 ++I;
2501 continue;
2502 }
2503
2504 // Emit #endif/#if pair if needed.
2505 if ((*I)->getGuard() != InGuard) {
2506 if (!InGuard.empty())
2507 OS << "#endif\n";
2508 InGuard = (*I)->getGuard();
2509 if (!InGuard.empty())
2510 OS << "#if " << InGuard << "\n";
2511 }
2512
2513 // Actually generate the intrinsic code.
2514 OS << (*I)->generate();
2515
2516 MadeProgress = true;
2517 I = Defs.erase(I);
2518 }
2519 }
2520 assert(Defs.empty() && "Some requirements were not satisfied!");
2521 if (!InGuard.empty())
2522 OS << "#endif\n";
2523
2524 OS << "\n";
2525 OS << "#undef __ai\n\n";
2526 OS << "#endif /* __ARM_FP16_H */\n";
2527 }
2528
runBF16(raw_ostream & OS)2529 void NeonEmitter::runBF16(raw_ostream &OS) {
2530 OS << "/*===---- arm_bf16.h - ARM BF16 intrinsics "
2531 "-----------------------------------===\n"
2532 " *\n"
2533 " *\n"
2534 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
2535 "Exceptions.\n"
2536 " * See https://llvm.org/LICENSE.txt for license information.\n"
2537 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
2538 " *\n"
2539 " *===-----------------------------------------------------------------"
2540 "------===\n"
2541 " */\n\n";
2542
2543 OS << "#ifndef __ARM_BF16_H\n";
2544 OS << "#define __ARM_BF16_H\n\n";
2545
2546 OS << "typedef __bf16 bfloat16_t;\n";
2547
2548 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
2549 "__nodebug__))\n\n";
2550
2551 SmallVector<Intrinsic *, 128> Defs;
2552 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2553 for (auto *R : RV)
2554 createIntrinsic(R, Defs);
2555
2556 for (auto *I : Defs)
2557 I->indexBody();
2558
2559 llvm::stable_sort(Defs, llvm::deref<std::less<>>());
2560
2561 // Only emit a def when its requirements have been met.
2562 // FIXME: This loop could be made faster, but it's fast enough for now.
2563 bool MadeProgress = true;
2564 std::string InGuard;
2565 while (!Defs.empty() && MadeProgress) {
2566 MadeProgress = false;
2567
2568 for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2569 I != Defs.end(); /*No step*/) {
2570 bool DependenciesSatisfied = true;
2571 for (auto *II : (*I)->getDependencies()) {
2572 if (llvm::is_contained(Defs, II))
2573 DependenciesSatisfied = false;
2574 }
2575 if (!DependenciesSatisfied) {
2576 // Try the next one.
2577 ++I;
2578 continue;
2579 }
2580
2581 // Emit #endif/#if pair if needed.
2582 if ((*I)->getGuard() != InGuard) {
2583 if (!InGuard.empty())
2584 OS << "#endif\n";
2585 InGuard = (*I)->getGuard();
2586 if (!InGuard.empty())
2587 OS << "#if " << InGuard << "\n";
2588 }
2589
2590 // Actually generate the intrinsic code.
2591 OS << (*I)->generate();
2592
2593 MadeProgress = true;
2594 I = Defs.erase(I);
2595 }
2596 }
2597 assert(Defs.empty() && "Some requirements were not satisfied!");
2598 if (!InGuard.empty())
2599 OS << "#endif\n";
2600
2601 OS << "\n";
2602 OS << "#undef __ai\n\n";
2603
2604 OS << "#endif\n";
2605 }
2606
EmitNeon(RecordKeeper & Records,raw_ostream & OS)2607 void clang::EmitNeon(RecordKeeper &Records, raw_ostream &OS) {
2608 NeonEmitter(Records).run(OS);
2609 }
2610
EmitFP16(RecordKeeper & Records,raw_ostream & OS)2611 void clang::EmitFP16(RecordKeeper &Records, raw_ostream &OS) {
2612 NeonEmitter(Records).runFP16(OS);
2613 }
2614
EmitBF16(RecordKeeper & Records,raw_ostream & OS)2615 void clang::EmitBF16(RecordKeeper &Records, raw_ostream &OS) {
2616 NeonEmitter(Records).runBF16(OS);
2617 }
2618
EmitNeonSema(RecordKeeper & Records,raw_ostream & OS)2619 void clang::EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) {
2620 NeonEmitter(Records).runHeader(OS);
2621 }
2622
EmitNeonTest(RecordKeeper & Records,raw_ostream & OS)2623 void clang::EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) {
2624 llvm_unreachable("Neon test generation no longer implemented!");
2625 }
2626