1 //===- NeonEmitter.cpp - Generate arm_neon.h for use with clang -*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This tablegen backend is responsible for emitting arm_neon.h, which includes
11 // a declaration and definition of each function specified by the ARM NEON
12 // compiler interface. See ARM document DUI0348B.
13 //
14 // Each NEON instruction is implemented in terms of 1 or more functions which
15 // are suffixed with the element type of the input vectors. Functions may be
16 // implemented in terms of generic vector operations such as +, *, -, etc. or
17 // by calling a __builtin_-prefixed function which will be handled by clang's
18 // CodeGen library.
19 //
20 // Additional validation code can be generated by this file when runHeader() is
21 // called, rather than the normal run() entry point.
22 //
23 // See also the documentation in include/clang/Basic/arm_neon.td.
24 //
25 //===----------------------------------------------------------------------===//
26
27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/DenseMap.h"
29 #include "llvm/ADT/None.h"
30 #include "llvm/ADT/SmallVector.h"
31 #include "llvm/ADT/STLExtras.h"
32 #include "llvm/ADT/StringExtras.h"
33 #include "llvm/ADT/StringRef.h"
34 #include "llvm/Support/Casting.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/raw_ostream.h"
37 #include "llvm/TableGen/Error.h"
38 #include "llvm/TableGen/Record.h"
39 #include "llvm/TableGen/SetTheory.h"
40 #include <algorithm>
41 #include <cassert>
42 #include <cctype>
43 #include <cstddef>
44 #include <cstdint>
45 #include <deque>
46 #include <map>
47 #include <set>
48 #include <sstream>
49 #include <string>
50 #include <utility>
51 #include <vector>
52
53 using namespace llvm;
54
55 namespace {
56
57 // While globals are generally bad, this one allows us to perform assertions
58 // liberally and somehow still trace them back to the def they indirectly
59 // came from.
60 static Record *CurrentRecord = nullptr;
assert_with_loc(bool Assertion,const std::string & Str)61 static void assert_with_loc(bool Assertion, const std::string &Str) {
62 if (!Assertion) {
63 if (CurrentRecord)
64 PrintFatalError(CurrentRecord->getLoc(), Str);
65 else
66 PrintFatalError(Str);
67 }
68 }
69
70 enum ClassKind {
71 ClassNone,
72 ClassI, // generic integer instruction, e.g., "i8" suffix
73 ClassS, // signed/unsigned/poly, e.g., "s8", "u8" or "p8" suffix
74 ClassW, // width-specific instruction, e.g., "8" suffix
75 ClassB, // bitcast arguments with enum argument to specify type
76 ClassL, // Logical instructions which are op instructions
77 // but we need to not emit any suffix for in our
78 // tests.
79 ClassNoTest // Instructions which we do not test since they are
80 // not TRUE instructions.
81 };
82
83 /// NeonTypeFlags - Flags to identify the types for overloaded Neon
84 /// builtins. These must be kept in sync with the flags in
85 /// include/clang/Basic/TargetBuiltins.h.
86 namespace NeonTypeFlags {
87
88 enum { EltTypeMask = 0xf, UnsignedFlag = 0x10, QuadFlag = 0x20 };
89
90 enum EltType {
91 Int8,
92 Int16,
93 Int32,
94 Int64,
95 Poly8,
96 Poly16,
97 Poly64,
98 Poly128,
99 Float16,
100 Float32,
101 Float64
102 };
103
104 } // end namespace NeonTypeFlags
105
106 class NeonEmitter;
107
108 //===----------------------------------------------------------------------===//
109 // TypeSpec
110 //===----------------------------------------------------------------------===//
111
112 /// A TypeSpec is just a simple wrapper around a string, but gets its own type
113 /// for strong typing purposes.
114 ///
115 /// A TypeSpec can be used to create a type.
116 class TypeSpec : public std::string {
117 public:
fromTypeSpecs(StringRef Str)118 static std::vector<TypeSpec> fromTypeSpecs(StringRef Str) {
119 std::vector<TypeSpec> Ret;
120 TypeSpec Acc;
121 for (char I : Str.str()) {
122 if (islower(I)) {
123 Acc.push_back(I);
124 Ret.push_back(TypeSpec(Acc));
125 Acc.clear();
126 } else {
127 Acc.push_back(I);
128 }
129 }
130 return Ret;
131 }
132 };
133
134 //===----------------------------------------------------------------------===//
135 // Type
136 //===----------------------------------------------------------------------===//
137
138 /// A Type. Not much more to say here.
139 class Type {
140 private:
141 TypeSpec TS;
142
143 bool Float, Signed, Immediate, Void, Poly, Constant, Pointer;
144 // ScalarForMangling and NoManglingQ are really not suited to live here as
145 // they are not related to the type. But they live in the TypeSpec (not the
146 // prototype), so this is really the only place to store them.
147 bool ScalarForMangling, NoManglingQ;
148 unsigned Bitwidth, ElementBitwidth, NumVectors;
149
150 public:
Type()151 Type()
152 : Float(false), Signed(false), Immediate(false), Void(true), Poly(false),
153 Constant(false), Pointer(false), ScalarForMangling(false),
154 NoManglingQ(false), Bitwidth(0), ElementBitwidth(0), NumVectors(0) {}
155
Type(TypeSpec TS,char CharMod)156 Type(TypeSpec TS, char CharMod)
157 : TS(std::move(TS)), Float(false), Signed(false), Immediate(false),
158 Void(false), Poly(false), Constant(false), Pointer(false),
159 ScalarForMangling(false), NoManglingQ(false), Bitwidth(0),
160 ElementBitwidth(0), NumVectors(0) {
161 applyModifier(CharMod);
162 }
163
164 /// Returns a type representing "void".
getVoid()165 static Type getVoid() { return Type(); }
166
operator ==(const Type & Other) const167 bool operator==(const Type &Other) const { return str() == Other.str(); }
operator !=(const Type & Other) const168 bool operator!=(const Type &Other) const { return !operator==(Other); }
169
170 //
171 // Query functions
172 //
isScalarForMangling() const173 bool isScalarForMangling() const { return ScalarForMangling; }
noManglingQ() const174 bool noManglingQ() const { return NoManglingQ; }
175
isPointer() const176 bool isPointer() const { return Pointer; }
isFloating() const177 bool isFloating() const { return Float; }
isInteger() const178 bool isInteger() const { return !Float && !Poly; }
isSigned() const179 bool isSigned() const { return Signed; }
isImmediate() const180 bool isImmediate() const { return Immediate; }
isScalar() const181 bool isScalar() const { return NumVectors == 0; }
isVector() const182 bool isVector() const { return NumVectors > 0; }
isFloat() const183 bool isFloat() const { return Float && ElementBitwidth == 32; }
isDouble() const184 bool isDouble() const { return Float && ElementBitwidth == 64; }
isHalf() const185 bool isHalf() const { return Float && ElementBitwidth == 16; }
isPoly() const186 bool isPoly() const { return Poly; }
isChar() const187 bool isChar() const { return ElementBitwidth == 8; }
isShort() const188 bool isShort() const { return !Float && ElementBitwidth == 16; }
isInt() const189 bool isInt() const { return !Float && ElementBitwidth == 32; }
isLong() const190 bool isLong() const { return !Float && ElementBitwidth == 64; }
isVoid() const191 bool isVoid() const { return Void; }
getNumElements() const192 unsigned getNumElements() const { return Bitwidth / ElementBitwidth; }
getSizeInBits() const193 unsigned getSizeInBits() const { return Bitwidth; }
getElementSizeInBits() const194 unsigned getElementSizeInBits() const { return ElementBitwidth; }
getNumVectors() const195 unsigned getNumVectors() const { return NumVectors; }
196
197 //
198 // Mutator functions
199 //
makeUnsigned()200 void makeUnsigned() { Signed = false; }
makeSigned()201 void makeSigned() { Signed = true; }
202
makeInteger(unsigned ElemWidth,bool Sign)203 void makeInteger(unsigned ElemWidth, bool Sign) {
204 Float = false;
205 Poly = false;
206 Signed = Sign;
207 Immediate = false;
208 ElementBitwidth = ElemWidth;
209 }
210
makeImmediate(unsigned ElemWidth)211 void makeImmediate(unsigned ElemWidth) {
212 Float = false;
213 Poly = false;
214 Signed = true;
215 Immediate = true;
216 ElementBitwidth = ElemWidth;
217 }
218
makeScalar()219 void makeScalar() {
220 Bitwidth = ElementBitwidth;
221 NumVectors = 0;
222 }
223
makeOneVector()224 void makeOneVector() {
225 assert(isVector());
226 NumVectors = 1;
227 }
228
doubleLanes()229 void doubleLanes() {
230 assert_with_loc(Bitwidth != 128, "Can't get bigger than 128!");
231 Bitwidth = 128;
232 }
233
halveLanes()234 void halveLanes() {
235 assert_with_loc(Bitwidth != 64, "Can't get smaller than 64!");
236 Bitwidth = 64;
237 }
238
239 /// Return the C string representation of a type, which is the typename
240 /// defined in stdint.h or arm_neon.h.
241 std::string str() const;
242
243 /// Return the string representation of a type, which is an encoded
244 /// string for passing to the BUILTIN() macro in Builtins.def.
245 std::string builtin_str() const;
246
247 /// Return the value in NeonTypeFlags for this type.
248 unsigned getNeonEnum() const;
249
250 /// Parse a type from a stdint.h or arm_neon.h typedef name,
251 /// for example uint32x2_t or int64_t.
252 static Type fromTypedefName(StringRef Name);
253
254 private:
255 /// Creates the type based on the typespec string in TS.
256 /// Sets "Quad" to true if the "Q" or "H" modifiers were
257 /// seen. This is needed by applyModifier as some modifiers
258 /// only take effect if the type size was changed by "Q" or "H".
259 void applyTypespec(bool &Quad);
260 /// Applies a prototype modifier to the type.
261 void applyModifier(char Mod);
262 };
263
264 //===----------------------------------------------------------------------===//
265 // Variable
266 //===----------------------------------------------------------------------===//
267
268 /// A variable is a simple class that just has a type and a name.
269 class Variable {
270 Type T;
271 std::string N;
272
273 public:
Variable()274 Variable() : T(Type::getVoid()), N("") {}
Variable(Type T,std::string N)275 Variable(Type T, std::string N) : T(std::move(T)), N(std::move(N)) {}
276
getType() const277 Type getType() const { return T; }
getName() const278 std::string getName() const { return "__" + N; }
279 };
280
281 //===----------------------------------------------------------------------===//
282 // Intrinsic
283 //===----------------------------------------------------------------------===//
284
285 /// The main grunt class. This represents an instantiation of an intrinsic with
286 /// a particular typespec and prototype.
287 class Intrinsic {
288 friend class DagEmitter;
289
290 /// The Record this intrinsic was created from.
291 Record *R;
292 /// The unmangled name and prototype.
293 std::string Name, Proto;
294 /// The input and output typespecs. InTS == OutTS except when
295 /// CartesianProductOfTypes is 1 - this is the case for vreinterpret.
296 TypeSpec OutTS, InTS;
297 /// The base class kind. Most intrinsics use ClassS, which has full type
298 /// info for integers (s32/u32). Some use ClassI, which doesn't care about
299 /// signedness (i32), while some (ClassB) have no type at all, only a width
300 /// (32).
301 ClassKind CK;
302 /// The list of DAGs for the body. May be empty, in which case we should
303 /// emit a builtin call.
304 ListInit *Body;
305 /// The architectural #ifdef guard.
306 std::string Guard;
307 /// Set if the Unavailable bit is 1. This means we don't generate a body,
308 /// just an "unavailable" attribute on a declaration.
309 bool IsUnavailable;
310 /// Is this intrinsic safe for big-endian? or does it need its arguments
311 /// reversing?
312 bool BigEndianSafe;
313
314 /// The types of return value [0] and parameters [1..].
315 std::vector<Type> Types;
316 /// The local variables defined.
317 std::map<std::string, Variable> Variables;
318 /// NeededEarly - set if any other intrinsic depends on this intrinsic.
319 bool NeededEarly;
320 /// UseMacro - set if we should implement using a macro or unset for a
321 /// function.
322 bool UseMacro;
323 /// The set of intrinsics that this intrinsic uses/requires.
324 std::set<Intrinsic *> Dependencies;
325 /// The "base type", which is Type('d', OutTS). InBaseType is only
326 /// different if CartesianProductOfTypes = 1 (for vreinterpret).
327 Type BaseType, InBaseType;
328 /// The return variable.
329 Variable RetVar;
330 /// A postfix to apply to every variable. Defaults to "".
331 std::string VariablePostfix;
332
333 NeonEmitter &Emitter;
334 std::stringstream OS;
335
336 public:
Intrinsic(Record * R,StringRef Name,StringRef Proto,TypeSpec OutTS,TypeSpec InTS,ClassKind CK,ListInit * Body,NeonEmitter & Emitter,StringRef Guard,bool IsUnavailable,bool BigEndianSafe)337 Intrinsic(Record *R, StringRef Name, StringRef Proto, TypeSpec OutTS,
338 TypeSpec InTS, ClassKind CK, ListInit *Body, NeonEmitter &Emitter,
339 StringRef Guard, bool IsUnavailable, bool BigEndianSafe)
340 : R(R), Name(Name.str()), Proto(Proto.str()), OutTS(OutTS), InTS(InTS),
341 CK(CK), Body(Body), Guard(Guard.str()), IsUnavailable(IsUnavailable),
342 BigEndianSafe(BigEndianSafe), NeededEarly(false), UseMacro(false),
343 BaseType(OutTS, 'd'), InBaseType(InTS, 'd'), Emitter(Emitter) {
344 // If this builtin takes an immediate argument, we need to #define it rather
345 // than use a standard declaration, so that SemaChecking can range check
346 // the immediate passed by the user.
347 if (Proto.find('i') != std::string::npos)
348 UseMacro = true;
349
350 // Pointer arguments need to use macros to avoid hiding aligned attributes
351 // from the pointer type.
352 if (Proto.find('p') != std::string::npos ||
353 Proto.find('c') != std::string::npos)
354 UseMacro = true;
355
356 // It is not permitted to pass or return an __fp16 by value, so intrinsics
357 // taking a scalar float16_t must be implemented as macros.
358 if (OutTS.find('h') != std::string::npos &&
359 Proto.find('s') != std::string::npos)
360 UseMacro = true;
361
362 // Modify the TypeSpec per-argument to get a concrete Type, and create
363 // known variables for each.
364 // Types[0] is the return value.
365 Types.emplace_back(OutTS, Proto[0]);
366 for (unsigned I = 1; I < Proto.size(); ++I)
367 Types.emplace_back(InTS, Proto[I]);
368 }
369
370 /// Get the Record that this intrinsic is based off.
getRecord() const371 Record *getRecord() const { return R; }
372 /// Get the set of Intrinsics that this intrinsic calls.
373 /// this is the set of immediate dependencies, NOT the
374 /// transitive closure.
getDependencies() const375 const std::set<Intrinsic *> &getDependencies() const { return Dependencies; }
376 /// Get the architectural guard string (#ifdef).
getGuard() const377 std::string getGuard() const { return Guard; }
378 /// Get the non-mangled name.
getName() const379 std::string getName() const { return Name; }
380
381 /// Return true if the intrinsic takes an immediate operand.
hasImmediate() const382 bool hasImmediate() const {
383 return Proto.find('i') != std::string::npos;
384 }
385
386 /// Return the parameter index of the immediate operand.
getImmediateIdx() const387 unsigned getImmediateIdx() const {
388 assert(hasImmediate());
389 unsigned Idx = Proto.find('i');
390 assert(Idx > 0 && "Can't return an immediate!");
391 return Idx - 1;
392 }
393
394 /// Return true if the intrinsic takes an splat operand.
hasSplat() const395 bool hasSplat() const { return Proto.find('a') != std::string::npos; }
396
397 /// Return the parameter index of the splat operand.
getSplatIdx() const398 unsigned getSplatIdx() const {
399 assert(hasSplat());
400 unsigned Idx = Proto.find('a');
401 assert(Idx > 0 && "Can't return a splat!");
402 return Idx - 1;
403 }
404
getNumParams() const405 unsigned getNumParams() const { return Proto.size() - 1; }
getReturnType() const406 Type getReturnType() const { return Types[0]; }
getParamType(unsigned I) const407 Type getParamType(unsigned I) const { return Types[I + 1]; }
getBaseType() const408 Type getBaseType() const { return BaseType; }
409 /// Return the raw prototype string.
getProto() const410 std::string getProto() const { return Proto; }
411
412 /// Return true if the prototype has a scalar argument.
413 /// This does not return true for the "splat" code ('a').
414 bool protoHasScalar() const;
415
416 /// Return the index that parameter PIndex will sit at
417 /// in a generated function call. This is often just PIndex,
418 /// but may not be as things such as multiple-vector operands
419 /// and sret parameters need to be taken into accont.
getGeneratedParamIdx(unsigned PIndex)420 unsigned getGeneratedParamIdx(unsigned PIndex) {
421 unsigned Idx = 0;
422 if (getReturnType().getNumVectors() > 1)
423 // Multiple vectors are passed as sret.
424 ++Idx;
425
426 for (unsigned I = 0; I < PIndex; ++I)
427 Idx += std::max(1U, getParamType(I).getNumVectors());
428
429 return Idx;
430 }
431
hasBody() const432 bool hasBody() const { return Body && !Body->getValues().empty(); }
433
setNeededEarly()434 void setNeededEarly() { NeededEarly = true; }
435
operator <(const Intrinsic & Other) const436 bool operator<(const Intrinsic &Other) const {
437 // Sort lexicographically on a two-tuple (Guard, Name)
438 if (Guard != Other.Guard)
439 return Guard < Other.Guard;
440 return Name < Other.Name;
441 }
442
getClassKind(bool UseClassBIfScalar=false)443 ClassKind getClassKind(bool UseClassBIfScalar = false) {
444 if (UseClassBIfScalar && !protoHasScalar())
445 return ClassB;
446 return CK;
447 }
448
449 /// Return the name, mangled with type information.
450 /// If ForceClassS is true, use ClassS (u32/s32) instead
451 /// of the intrinsic's own type class.
452 std::string getMangledName(bool ForceClassS = false) const;
453 /// Return the type code for a builtin function call.
454 std::string getInstTypeCode(Type T, ClassKind CK) const;
455 /// Return the type string for a BUILTIN() macro in Builtins.def.
456 std::string getBuiltinTypeStr();
457
458 /// Generate the intrinsic, returning code.
459 std::string generate();
460 /// Perform type checking and populate the dependency graph, but
461 /// don't generate code yet.
462 void indexBody();
463
464 private:
465 std::string mangleName(std::string Name, ClassKind CK) const;
466
467 void initVariables();
468 std::string replaceParamsIn(std::string S);
469
470 void emitBodyAsBuiltinCall();
471
472 void generateImpl(bool ReverseArguments,
473 StringRef NamePrefix, StringRef CallPrefix);
474 void emitReturn();
475 void emitBody(StringRef CallPrefix);
476 void emitShadowedArgs();
477 void emitArgumentReversal();
478 void emitReturnReversal();
479 void emitReverseVariable(Variable &Dest, Variable &Src);
480 void emitNewLine();
481 void emitClosingBrace();
482 void emitOpeningBrace();
483 void emitPrototype(StringRef NamePrefix);
484
485 class DagEmitter {
486 Intrinsic &Intr;
487 StringRef CallPrefix;
488
489 public:
DagEmitter(Intrinsic & Intr,StringRef CallPrefix)490 DagEmitter(Intrinsic &Intr, StringRef CallPrefix) :
491 Intr(Intr), CallPrefix(CallPrefix) {
492 }
493 std::pair<Type, std::string> emitDagArg(Init *Arg, std::string ArgName);
494 std::pair<Type, std::string> emitDagSaveTemp(DagInit *DI);
495 std::pair<Type, std::string> emitDagSplat(DagInit *DI);
496 std::pair<Type, std::string> emitDagDup(DagInit *DI);
497 std::pair<Type, std::string> emitDagShuffle(DagInit *DI);
498 std::pair<Type, std::string> emitDagCast(DagInit *DI, bool IsBitCast);
499 std::pair<Type, std::string> emitDagCall(DagInit *DI);
500 std::pair<Type, std::string> emitDagNameReplace(DagInit *DI);
501 std::pair<Type, std::string> emitDagLiteral(DagInit *DI);
502 std::pair<Type, std::string> emitDagOp(DagInit *DI);
503 std::pair<Type, std::string> emitDag(DagInit *DI);
504 };
505 };
506
507 //===----------------------------------------------------------------------===//
508 // NeonEmitter
509 //===----------------------------------------------------------------------===//
510
511 class NeonEmitter {
512 RecordKeeper &Records;
513 DenseMap<Record *, ClassKind> ClassMap;
514 std::map<std::string, std::deque<Intrinsic>> IntrinsicMap;
515 unsigned UniqueNumber;
516
517 void createIntrinsic(Record *R, SmallVectorImpl<Intrinsic *> &Out);
518 void genBuiltinsDef(raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs);
519 void genOverloadTypeCheckCode(raw_ostream &OS,
520 SmallVectorImpl<Intrinsic *> &Defs);
521 void genIntrinsicRangeCheckCode(raw_ostream &OS,
522 SmallVectorImpl<Intrinsic *> &Defs);
523
524 public:
525 /// Called by Intrinsic - this attempts to get an intrinsic that takes
526 /// the given types as arguments.
527 Intrinsic &getIntrinsic(StringRef Name, ArrayRef<Type> Types);
528
529 /// Called by Intrinsic - returns a globally-unique number.
getUniqueNumber()530 unsigned getUniqueNumber() { return UniqueNumber++; }
531
NeonEmitter(RecordKeeper & R)532 NeonEmitter(RecordKeeper &R) : Records(R), UniqueNumber(0) {
533 Record *SI = R.getClass("SInst");
534 Record *II = R.getClass("IInst");
535 Record *WI = R.getClass("WInst");
536 Record *SOpI = R.getClass("SOpInst");
537 Record *IOpI = R.getClass("IOpInst");
538 Record *WOpI = R.getClass("WOpInst");
539 Record *LOpI = R.getClass("LOpInst");
540 Record *NoTestOpI = R.getClass("NoTestOpInst");
541
542 ClassMap[SI] = ClassS;
543 ClassMap[II] = ClassI;
544 ClassMap[WI] = ClassW;
545 ClassMap[SOpI] = ClassS;
546 ClassMap[IOpI] = ClassI;
547 ClassMap[WOpI] = ClassW;
548 ClassMap[LOpI] = ClassL;
549 ClassMap[NoTestOpI] = ClassNoTest;
550 }
551
552 // run - Emit arm_neon.h.inc
553 void run(raw_ostream &o);
554
555 // runFP16 - Emit arm_fp16.h.inc
556 void runFP16(raw_ostream &o);
557
558 // runHeader - Emit all the __builtin prototypes used in arm_neon.h
559 // and arm_fp16.h
560 void runHeader(raw_ostream &o);
561
562 // runTests - Emit tests for all the Neon intrinsics.
563 void runTests(raw_ostream &o);
564 };
565
566 } // end anonymous namespace
567
568 //===----------------------------------------------------------------------===//
569 // Type implementation
570 //===----------------------------------------------------------------------===//
571
str() const572 std::string Type::str() const {
573 if (Void)
574 return "void";
575 std::string S;
576
577 if (!Signed && isInteger())
578 S += "u";
579
580 if (Poly)
581 S += "poly";
582 else if (Float)
583 S += "float";
584 else
585 S += "int";
586
587 S += utostr(ElementBitwidth);
588 if (isVector())
589 S += "x" + utostr(getNumElements());
590 if (NumVectors > 1)
591 S += "x" + utostr(NumVectors);
592 S += "_t";
593
594 if (Constant)
595 S += " const";
596 if (Pointer)
597 S += " *";
598
599 return S;
600 }
601
builtin_str() const602 std::string Type::builtin_str() const {
603 std::string S;
604 if (isVoid())
605 return "v";
606
607 if (Pointer)
608 // All pointers are void pointers.
609 S += "v";
610 else if (isInteger())
611 switch (ElementBitwidth) {
612 case 8: S += "c"; break;
613 case 16: S += "s"; break;
614 case 32: S += "i"; break;
615 case 64: S += "Wi"; break;
616 case 128: S += "LLLi"; break;
617 default: llvm_unreachable("Unhandled case!");
618 }
619 else
620 switch (ElementBitwidth) {
621 case 16: S += "h"; break;
622 case 32: S += "f"; break;
623 case 64: S += "d"; break;
624 default: llvm_unreachable("Unhandled case!");
625 }
626
627 if (isChar() && !Pointer)
628 // Make chars explicitly signed.
629 S = "S" + S;
630 else if (isInteger() && !Pointer && !Signed)
631 S = "U" + S;
632
633 // Constant indices are "int", but have the "constant expression" modifier.
634 if (isImmediate()) {
635 assert(isInteger() && isSigned());
636 S = "I" + S;
637 }
638
639 if (isScalar()) {
640 if (Constant) S += "C";
641 if (Pointer) S += "*";
642 return S;
643 }
644
645 std::string Ret;
646 for (unsigned I = 0; I < NumVectors; ++I)
647 Ret += "V" + utostr(getNumElements()) + S;
648
649 return Ret;
650 }
651
getNeonEnum() const652 unsigned Type::getNeonEnum() const {
653 unsigned Addend;
654 switch (ElementBitwidth) {
655 case 8: Addend = 0; break;
656 case 16: Addend = 1; break;
657 case 32: Addend = 2; break;
658 case 64: Addend = 3; break;
659 case 128: Addend = 4; break;
660 default: llvm_unreachable("Unhandled element bitwidth!");
661 }
662
663 unsigned Base = (unsigned)NeonTypeFlags::Int8 + Addend;
664 if (Poly) {
665 // Adjustment needed because Poly32 doesn't exist.
666 if (Addend >= 2)
667 --Addend;
668 Base = (unsigned)NeonTypeFlags::Poly8 + Addend;
669 }
670 if (Float) {
671 assert(Addend != 0 && "Float8 doesn't exist!");
672 Base = (unsigned)NeonTypeFlags::Float16 + (Addend - 1);
673 }
674
675 if (Bitwidth == 128)
676 Base |= (unsigned)NeonTypeFlags::QuadFlag;
677 if (isInteger() && !Signed)
678 Base |= (unsigned)NeonTypeFlags::UnsignedFlag;
679
680 return Base;
681 }
682
fromTypedefName(StringRef Name)683 Type Type::fromTypedefName(StringRef Name) {
684 Type T;
685 T.Void = false;
686 T.Float = false;
687 T.Poly = false;
688
689 if (Name.front() == 'u') {
690 T.Signed = false;
691 Name = Name.drop_front();
692 } else {
693 T.Signed = true;
694 }
695
696 if (Name.startswith("float")) {
697 T.Float = true;
698 Name = Name.drop_front(5);
699 } else if (Name.startswith("poly")) {
700 T.Poly = true;
701 Name = Name.drop_front(4);
702 } else {
703 assert(Name.startswith("int"));
704 Name = Name.drop_front(3);
705 }
706
707 unsigned I = 0;
708 for (I = 0; I < Name.size(); ++I) {
709 if (!isdigit(Name[I]))
710 break;
711 }
712 Name.substr(0, I).getAsInteger(10, T.ElementBitwidth);
713 Name = Name.drop_front(I);
714
715 T.Bitwidth = T.ElementBitwidth;
716 T.NumVectors = 1;
717
718 if (Name.front() == 'x') {
719 Name = Name.drop_front();
720 unsigned I = 0;
721 for (I = 0; I < Name.size(); ++I) {
722 if (!isdigit(Name[I]))
723 break;
724 }
725 unsigned NumLanes;
726 Name.substr(0, I).getAsInteger(10, NumLanes);
727 Name = Name.drop_front(I);
728 T.Bitwidth = T.ElementBitwidth * NumLanes;
729 } else {
730 // Was scalar.
731 T.NumVectors = 0;
732 }
733 if (Name.front() == 'x') {
734 Name = Name.drop_front();
735 unsigned I = 0;
736 for (I = 0; I < Name.size(); ++I) {
737 if (!isdigit(Name[I]))
738 break;
739 }
740 Name.substr(0, I).getAsInteger(10, T.NumVectors);
741 Name = Name.drop_front(I);
742 }
743
744 assert(Name.startswith("_t") && "Malformed typedef!");
745 return T;
746 }
747
applyTypespec(bool & Quad)748 void Type::applyTypespec(bool &Quad) {
749 std::string S = TS;
750 ScalarForMangling = false;
751 Void = false;
752 Poly = Float = false;
753 ElementBitwidth = ~0U;
754 Signed = true;
755 NumVectors = 1;
756
757 for (char I : S) {
758 switch (I) {
759 case 'S':
760 ScalarForMangling = true;
761 break;
762 case 'H':
763 NoManglingQ = true;
764 Quad = true;
765 break;
766 case 'Q':
767 Quad = true;
768 break;
769 case 'P':
770 Poly = true;
771 break;
772 case 'U':
773 Signed = false;
774 break;
775 case 'c':
776 ElementBitwidth = 8;
777 break;
778 case 'h':
779 Float = true;
780 LLVM_FALLTHROUGH;
781 case 's':
782 ElementBitwidth = 16;
783 break;
784 case 'f':
785 Float = true;
786 LLVM_FALLTHROUGH;
787 case 'i':
788 ElementBitwidth = 32;
789 break;
790 case 'd':
791 Float = true;
792 LLVM_FALLTHROUGH;
793 case 'l':
794 ElementBitwidth = 64;
795 break;
796 case 'k':
797 ElementBitwidth = 128;
798 // Poly doesn't have a 128x1 type.
799 if (Poly)
800 NumVectors = 0;
801 break;
802 default:
803 llvm_unreachable("Unhandled type code!");
804 }
805 }
806 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
807
808 Bitwidth = Quad ? 128 : 64;
809 }
810
applyModifier(char Mod)811 void Type::applyModifier(char Mod) {
812 bool AppliedQuad = false;
813 applyTypespec(AppliedQuad);
814
815 switch (Mod) {
816 case 'v':
817 Void = true;
818 break;
819 case 't':
820 if (Poly) {
821 Poly = false;
822 Signed = false;
823 }
824 break;
825 case 'b':
826 Signed = false;
827 Float = false;
828 Poly = false;
829 NumVectors = 0;
830 Bitwidth = ElementBitwidth;
831 break;
832 case '$':
833 Signed = true;
834 Float = false;
835 Poly = false;
836 NumVectors = 0;
837 Bitwidth = ElementBitwidth;
838 break;
839 case 'u':
840 Signed = false;
841 Poly = false;
842 Float = false;
843 break;
844 case 'x':
845 Signed = true;
846 assert(!Poly && "'u' can't be used with poly types!");
847 Float = false;
848 break;
849 case 'o':
850 Bitwidth = ElementBitwidth = 64;
851 NumVectors = 0;
852 Float = true;
853 break;
854 case 'y':
855 Bitwidth = ElementBitwidth = 32;
856 NumVectors = 0;
857 Float = true;
858 break;
859 case 'Y':
860 Bitwidth = ElementBitwidth = 16;
861 NumVectors = 0;
862 Float = true;
863 break;
864 case 'I':
865 Bitwidth = ElementBitwidth = 32;
866 NumVectors = 0;
867 Float = false;
868 Signed = true;
869 break;
870 case 'L':
871 Bitwidth = ElementBitwidth = 64;
872 NumVectors = 0;
873 Float = false;
874 Signed = true;
875 break;
876 case 'U':
877 Bitwidth = ElementBitwidth = 32;
878 NumVectors = 0;
879 Float = false;
880 Signed = false;
881 break;
882 case 'O':
883 Bitwidth = ElementBitwidth = 64;
884 NumVectors = 0;
885 Float = false;
886 Signed = false;
887 break;
888 case 'f':
889 Float = true;
890 ElementBitwidth = 32;
891 break;
892 case 'F':
893 Float = true;
894 ElementBitwidth = 64;
895 break;
896 case 'H':
897 Float = true;
898 ElementBitwidth = 16;
899 break;
900 case 'g':
901 if (AppliedQuad)
902 Bitwidth /= 2;
903 break;
904 case 'j':
905 if (!AppliedQuad)
906 Bitwidth *= 2;
907 break;
908 case 'w':
909 ElementBitwidth *= 2;
910 Bitwidth *= 2;
911 break;
912 case 'n':
913 ElementBitwidth *= 2;
914 break;
915 case 'i':
916 Float = false;
917 Poly = false;
918 ElementBitwidth = Bitwidth = 32;
919 NumVectors = 0;
920 Signed = true;
921 Immediate = true;
922 break;
923 case 'l':
924 Float = false;
925 Poly = false;
926 ElementBitwidth = Bitwidth = 64;
927 NumVectors = 0;
928 Signed = false;
929 Immediate = true;
930 break;
931 case 'z':
932 ElementBitwidth /= 2;
933 Bitwidth = ElementBitwidth;
934 NumVectors = 0;
935 break;
936 case 'r':
937 ElementBitwidth *= 2;
938 Bitwidth = ElementBitwidth;
939 NumVectors = 0;
940 break;
941 case 's':
942 case 'a':
943 Bitwidth = ElementBitwidth;
944 NumVectors = 0;
945 break;
946 case 'k':
947 Bitwidth *= 2;
948 break;
949 case 'c':
950 Constant = true;
951 LLVM_FALLTHROUGH;
952 case 'p':
953 Pointer = true;
954 Bitwidth = ElementBitwidth;
955 NumVectors = 0;
956 break;
957 case 'h':
958 ElementBitwidth /= 2;
959 break;
960 case 'q':
961 ElementBitwidth /= 2;
962 Bitwidth *= 2;
963 break;
964 case 'e':
965 ElementBitwidth /= 2;
966 Signed = false;
967 break;
968 case 'm':
969 ElementBitwidth /= 2;
970 Bitwidth /= 2;
971 break;
972 case 'd':
973 break;
974 case '2':
975 NumVectors = 2;
976 break;
977 case '3':
978 NumVectors = 3;
979 break;
980 case '4':
981 NumVectors = 4;
982 break;
983 case 'B':
984 NumVectors = 2;
985 if (!AppliedQuad)
986 Bitwidth *= 2;
987 break;
988 case 'C':
989 NumVectors = 3;
990 if (!AppliedQuad)
991 Bitwidth *= 2;
992 break;
993 case 'D':
994 NumVectors = 4;
995 if (!AppliedQuad)
996 Bitwidth *= 2;
997 break;
998 case '7':
999 if (AppliedQuad)
1000 Bitwidth /= 2;
1001 ElementBitwidth = 8;
1002 break;
1003 case '8':
1004 ElementBitwidth = 8;
1005 break;
1006 case '9':
1007 if (!AppliedQuad)
1008 Bitwidth *= 2;
1009 ElementBitwidth = 8;
1010 break;
1011 default:
1012 llvm_unreachable("Unhandled character!");
1013 }
1014 }
1015
1016 //===----------------------------------------------------------------------===//
1017 // Intrinsic implementation
1018 //===----------------------------------------------------------------------===//
1019
getInstTypeCode(Type T,ClassKind CK) const1020 std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
1021 char typeCode = '\0';
1022 bool printNumber = true;
1023
1024 if (CK == ClassB)
1025 return "";
1026
1027 if (T.isPoly())
1028 typeCode = 'p';
1029 else if (T.isInteger())
1030 typeCode = T.isSigned() ? 's' : 'u';
1031 else
1032 typeCode = 'f';
1033
1034 if (CK == ClassI) {
1035 switch (typeCode) {
1036 default:
1037 break;
1038 case 's':
1039 case 'u':
1040 case 'p':
1041 typeCode = 'i';
1042 break;
1043 }
1044 }
1045 if (CK == ClassB) {
1046 typeCode = '\0';
1047 }
1048
1049 std::string S;
1050 if (typeCode != '\0')
1051 S.push_back(typeCode);
1052 if (printNumber)
1053 S += utostr(T.getElementSizeInBits());
1054
1055 return S;
1056 }
1057
isFloatingPointProtoModifier(char Mod)1058 static bool isFloatingPointProtoModifier(char Mod) {
1059 return Mod == 'F' || Mod == 'f' || Mod == 'H' || Mod == 'Y' || Mod == 'I';
1060 }
1061
getBuiltinTypeStr()1062 std::string Intrinsic::getBuiltinTypeStr() {
1063 ClassKind LocalCK = getClassKind(true);
1064 std::string S;
1065
1066 Type RetT = getReturnType();
1067 if ((LocalCK == ClassI || LocalCK == ClassW) && RetT.isScalar() &&
1068 !RetT.isFloating())
1069 RetT.makeInteger(RetT.getElementSizeInBits(), false);
1070
1071 // Since the return value must be one type, return a vector type of the
1072 // appropriate width which we will bitcast. An exception is made for
1073 // returning structs of 2, 3, or 4 vectors which are returned in a sret-like
1074 // fashion, storing them to a pointer arg.
1075 if (RetT.getNumVectors() > 1) {
1076 S += "vv*"; // void result with void* first argument
1077 } else {
1078 if (RetT.isPoly())
1079 RetT.makeInteger(RetT.getElementSizeInBits(), false);
1080 if (!RetT.isScalar() && !RetT.isSigned())
1081 RetT.makeSigned();
1082
1083 bool ForcedVectorFloatingType = isFloatingPointProtoModifier(Proto[0]);
1084 if (LocalCK == ClassB && !RetT.isScalar() && !ForcedVectorFloatingType)
1085 // Cast to vector of 8-bit elements.
1086 RetT.makeInteger(8, true);
1087
1088 S += RetT.builtin_str();
1089 }
1090
1091 for (unsigned I = 0; I < getNumParams(); ++I) {
1092 Type T = getParamType(I);
1093 if (T.isPoly())
1094 T.makeInteger(T.getElementSizeInBits(), false);
1095
1096 bool ForcedFloatingType = isFloatingPointProtoModifier(Proto[I + 1]);
1097 if (LocalCK == ClassB && !T.isScalar() && !ForcedFloatingType)
1098 T.makeInteger(8, true);
1099 // Halves always get converted to 8-bit elements.
1100 if (T.isHalf() && T.isVector() && !T.isScalarForMangling())
1101 T.makeInteger(8, true);
1102
1103 if (LocalCK == ClassI)
1104 T.makeSigned();
1105
1106 if (hasImmediate() && getImmediateIdx() == I)
1107 T.makeImmediate(32);
1108
1109 S += T.builtin_str();
1110 }
1111
1112 // Extra constant integer to hold type class enum for this function, e.g. s8
1113 if (LocalCK == ClassB)
1114 S += "i";
1115
1116 return S;
1117 }
1118
getMangledName(bool ForceClassS) const1119 std::string Intrinsic::getMangledName(bool ForceClassS) const {
1120 // Check if the prototype has a scalar operand with the type of the vector
1121 // elements. If not, bitcasting the args will take care of arg checking.
1122 // The actual signedness etc. will be taken care of with special enums.
1123 ClassKind LocalCK = CK;
1124 if (!protoHasScalar())
1125 LocalCK = ClassB;
1126
1127 return mangleName(Name, ForceClassS ? ClassS : LocalCK);
1128 }
1129
mangleName(std::string Name,ClassKind LocalCK) const1130 std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
1131 std::string typeCode = getInstTypeCode(BaseType, LocalCK);
1132 std::string S = Name;
1133
1134 if (Name == "vcvt_f16_f32" || Name == "vcvt_f32_f16" ||
1135 Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32")
1136 return Name;
1137
1138 if (!typeCode.empty()) {
1139 // If the name ends with _xN (N = 2,3,4), insert the typeCode before _xN.
1140 if (Name.size() >= 3 && isdigit(Name.back()) &&
1141 Name[Name.length() - 2] == 'x' && Name[Name.length() - 3] == '_')
1142 S.insert(S.length() - 3, "_" + typeCode);
1143 else
1144 S += "_" + typeCode;
1145 }
1146
1147 if (BaseType != InBaseType) {
1148 // A reinterpret - out the input base type at the end.
1149 S += "_" + getInstTypeCode(InBaseType, LocalCK);
1150 }
1151
1152 if (LocalCK == ClassB)
1153 S += "_v";
1154
1155 // Insert a 'q' before the first '_' character so that it ends up before
1156 // _lane or _n on vector-scalar operations.
1157 if (BaseType.getSizeInBits() == 128 && !BaseType.noManglingQ()) {
1158 size_t Pos = S.find('_');
1159 S.insert(Pos, "q");
1160 }
1161
1162 char Suffix = '\0';
1163 if (BaseType.isScalarForMangling()) {
1164 switch (BaseType.getElementSizeInBits()) {
1165 case 8: Suffix = 'b'; break;
1166 case 16: Suffix = 'h'; break;
1167 case 32: Suffix = 's'; break;
1168 case 64: Suffix = 'd'; break;
1169 default: llvm_unreachable("Bad suffix!");
1170 }
1171 }
1172 if (Suffix != '\0') {
1173 size_t Pos = S.find('_');
1174 S.insert(Pos, &Suffix, 1);
1175 }
1176
1177 return S;
1178 }
1179
replaceParamsIn(std::string S)1180 std::string Intrinsic::replaceParamsIn(std::string S) {
1181 while (S.find('$') != std::string::npos) {
1182 size_t Pos = S.find('$');
1183 size_t End = Pos + 1;
1184 while (isalpha(S[End]))
1185 ++End;
1186
1187 std::string VarName = S.substr(Pos + 1, End - Pos - 1);
1188 assert_with_loc(Variables.find(VarName) != Variables.end(),
1189 "Variable not defined!");
1190 S.replace(Pos, End - Pos, Variables.find(VarName)->second.getName());
1191 }
1192
1193 return S;
1194 }
1195
initVariables()1196 void Intrinsic::initVariables() {
1197 Variables.clear();
1198
1199 // Modify the TypeSpec per-argument to get a concrete Type, and create
1200 // known variables for each.
1201 for (unsigned I = 1; I < Proto.size(); ++I) {
1202 char NameC = '0' + (I - 1);
1203 std::string Name = "p";
1204 Name.push_back(NameC);
1205
1206 Variables[Name] = Variable(Types[I], Name + VariablePostfix);
1207 }
1208 RetVar = Variable(Types[0], "ret" + VariablePostfix);
1209 }
1210
emitPrototype(StringRef NamePrefix)1211 void Intrinsic::emitPrototype(StringRef NamePrefix) {
1212 if (UseMacro)
1213 OS << "#define ";
1214 else
1215 OS << "__ai " << Types[0].str() << " ";
1216
1217 OS << NamePrefix.str() << mangleName(Name, ClassS) << "(";
1218
1219 for (unsigned I = 0; I < getNumParams(); ++I) {
1220 if (I != 0)
1221 OS << ", ";
1222
1223 char NameC = '0' + I;
1224 std::string Name = "p";
1225 Name.push_back(NameC);
1226 assert(Variables.find(Name) != Variables.end());
1227 Variable &V = Variables[Name];
1228
1229 if (!UseMacro)
1230 OS << V.getType().str() << " ";
1231 OS << V.getName();
1232 }
1233
1234 OS << ")";
1235 }
1236
emitOpeningBrace()1237 void Intrinsic::emitOpeningBrace() {
1238 if (UseMacro)
1239 OS << " __extension__ ({";
1240 else
1241 OS << " {";
1242 emitNewLine();
1243 }
1244
emitClosingBrace()1245 void Intrinsic::emitClosingBrace() {
1246 if (UseMacro)
1247 OS << "})";
1248 else
1249 OS << "}";
1250 }
1251
emitNewLine()1252 void Intrinsic::emitNewLine() {
1253 if (UseMacro)
1254 OS << " \\\n";
1255 else
1256 OS << "\n";
1257 }
1258
emitReverseVariable(Variable & Dest,Variable & Src)1259 void Intrinsic::emitReverseVariable(Variable &Dest, Variable &Src) {
1260 if (Dest.getType().getNumVectors() > 1) {
1261 emitNewLine();
1262
1263 for (unsigned K = 0; K < Dest.getType().getNumVectors(); ++K) {
1264 OS << " " << Dest.getName() << ".val[" << K << "] = "
1265 << "__builtin_shufflevector("
1266 << Src.getName() << ".val[" << K << "], "
1267 << Src.getName() << ".val[" << K << "]";
1268 for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1269 OS << ", " << J;
1270 OS << ");";
1271 emitNewLine();
1272 }
1273 } else {
1274 OS << " " << Dest.getName()
1275 << " = __builtin_shufflevector(" << Src.getName() << ", " << Src.getName();
1276 for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1277 OS << ", " << J;
1278 OS << ");";
1279 emitNewLine();
1280 }
1281 }
1282
emitArgumentReversal()1283 void Intrinsic::emitArgumentReversal() {
1284 if (BigEndianSafe)
1285 return;
1286
1287 // Reverse all vector arguments.
1288 for (unsigned I = 0; I < getNumParams(); ++I) {
1289 std::string Name = "p" + utostr(I);
1290 std::string NewName = "rev" + utostr(I);
1291
1292 Variable &V = Variables[Name];
1293 Variable NewV(V.getType(), NewName + VariablePostfix);
1294
1295 if (!NewV.getType().isVector() || NewV.getType().getNumElements() == 1)
1296 continue;
1297
1298 OS << " " << NewV.getType().str() << " " << NewV.getName() << ";";
1299 emitReverseVariable(NewV, V);
1300 V = NewV;
1301 }
1302 }
1303
emitReturnReversal()1304 void Intrinsic::emitReturnReversal() {
1305 if (BigEndianSafe)
1306 return;
1307 if (!getReturnType().isVector() || getReturnType().isVoid() ||
1308 getReturnType().getNumElements() == 1)
1309 return;
1310 emitReverseVariable(RetVar, RetVar);
1311 }
1312
emitShadowedArgs()1313 void Intrinsic::emitShadowedArgs() {
1314 // Macro arguments are not type-checked like inline function arguments,
1315 // so assign them to local temporaries to get the right type checking.
1316 if (!UseMacro)
1317 return;
1318
1319 for (unsigned I = 0; I < getNumParams(); ++I) {
1320 // Do not create a temporary for an immediate argument.
1321 // That would defeat the whole point of using a macro!
1322 if (hasImmediate() && Proto[I+1] == 'i')
1323 continue;
1324 // Do not create a temporary for pointer arguments. The input
1325 // pointer may have an alignment hint.
1326 if (getParamType(I).isPointer())
1327 continue;
1328
1329 std::string Name = "p" + utostr(I);
1330
1331 assert(Variables.find(Name) != Variables.end());
1332 Variable &V = Variables[Name];
1333
1334 std::string NewName = "s" + utostr(I);
1335 Variable V2(V.getType(), NewName + VariablePostfix);
1336
1337 OS << " " << V2.getType().str() << " " << V2.getName() << " = "
1338 << V.getName() << ";";
1339 emitNewLine();
1340
1341 V = V2;
1342 }
1343 }
1344
1345 // We don't check 'a' in this function, because for builtin function the
1346 // argument matching to 'a' uses a vector type splatted from a scalar type.
protoHasScalar() const1347 bool Intrinsic::protoHasScalar() const {
1348 return (Proto.find('s') != std::string::npos ||
1349 Proto.find('z') != std::string::npos ||
1350 Proto.find('r') != std::string::npos ||
1351 Proto.find('b') != std::string::npos ||
1352 Proto.find('$') != std::string::npos ||
1353 Proto.find('y') != std::string::npos ||
1354 Proto.find('o') != std::string::npos);
1355 }
1356
emitBodyAsBuiltinCall()1357 void Intrinsic::emitBodyAsBuiltinCall() {
1358 std::string S;
1359
1360 // If this builtin returns a struct 2, 3, or 4 vectors, pass it as an implicit
1361 // sret-like argument.
1362 bool SRet = getReturnType().getNumVectors() >= 2;
1363
1364 StringRef N = Name;
1365 if (hasSplat()) {
1366 // Call the non-splat builtin: chop off the "_n" suffix from the name.
1367 assert(N.endswith("_n"));
1368 N = N.drop_back(2);
1369 }
1370
1371 ClassKind LocalCK = CK;
1372 if (!protoHasScalar())
1373 LocalCK = ClassB;
1374
1375 if (!getReturnType().isVoid() && !SRet)
1376 S += "(" + RetVar.getType().str() + ") ";
1377
1378 S += "__builtin_neon_" + mangleName(N, LocalCK) + "(";
1379
1380 if (SRet)
1381 S += "&" + RetVar.getName() + ", ";
1382
1383 for (unsigned I = 0; I < getNumParams(); ++I) {
1384 Variable &V = Variables["p" + utostr(I)];
1385 Type T = V.getType();
1386
1387 // Handle multiple-vector values specially, emitting each subvector as an
1388 // argument to the builtin.
1389 if (T.getNumVectors() > 1) {
1390 // Check if an explicit cast is needed.
1391 std::string Cast;
1392 if (T.isChar() || T.isPoly() || !T.isSigned()) {
1393 Type T2 = T;
1394 T2.makeOneVector();
1395 T2.makeInteger(8, /*Signed=*/true);
1396 Cast = "(" + T2.str() + ")";
1397 }
1398
1399 for (unsigned J = 0; J < T.getNumVectors(); ++J)
1400 S += Cast + V.getName() + ".val[" + utostr(J) + "], ";
1401 continue;
1402 }
1403
1404 std::string Arg;
1405 Type CastToType = T;
1406 if (hasSplat() && I == getSplatIdx()) {
1407 Arg = "(" + BaseType.str() + ") {";
1408 for (unsigned J = 0; J < BaseType.getNumElements(); ++J) {
1409 if (J != 0)
1410 Arg += ", ";
1411 Arg += V.getName();
1412 }
1413 Arg += "}";
1414
1415 CastToType = BaseType;
1416 } else {
1417 Arg = V.getName();
1418 }
1419
1420 // Check if an explicit cast is needed.
1421 if (CastToType.isVector()) {
1422 CastToType.makeInteger(8, true);
1423 Arg = "(" + CastToType.str() + ")" + Arg;
1424 }
1425
1426 S += Arg + ", ";
1427 }
1428
1429 // Extra constant integer to hold type class enum for this function, e.g. s8
1430 if (getClassKind(true) == ClassB) {
1431 Type ThisTy = getReturnType();
1432 if (Proto[0] == 'v' || isFloatingPointProtoModifier(Proto[0]))
1433 ThisTy = getParamType(0);
1434 if (ThisTy.isPointer())
1435 ThisTy = getParamType(1);
1436
1437 S += utostr(ThisTy.getNeonEnum());
1438 } else {
1439 // Remove extraneous ", ".
1440 S.pop_back();
1441 S.pop_back();
1442 }
1443 S += ");";
1444
1445 std::string RetExpr;
1446 if (!SRet && !RetVar.getType().isVoid())
1447 RetExpr = RetVar.getName() + " = ";
1448
1449 OS << " " << RetExpr << S;
1450 emitNewLine();
1451 }
1452
emitBody(StringRef CallPrefix)1453 void Intrinsic::emitBody(StringRef CallPrefix) {
1454 std::vector<std::string> Lines;
1455
1456 assert(RetVar.getType() == Types[0]);
1457 // Create a return variable, if we're not void.
1458 if (!RetVar.getType().isVoid()) {
1459 OS << " " << RetVar.getType().str() << " " << RetVar.getName() << ";";
1460 emitNewLine();
1461 }
1462
1463 if (!Body || Body->getValues().empty()) {
1464 // Nothing specific to output - must output a builtin.
1465 emitBodyAsBuiltinCall();
1466 return;
1467 }
1468
1469 // We have a list of "things to output". The last should be returned.
1470 for (auto *I : Body->getValues()) {
1471 if (StringInit *SI = dyn_cast<StringInit>(I)) {
1472 Lines.push_back(replaceParamsIn(SI->getAsString()));
1473 } else if (DagInit *DI = dyn_cast<DagInit>(I)) {
1474 DagEmitter DE(*this, CallPrefix);
1475 Lines.push_back(DE.emitDag(DI).second + ";");
1476 }
1477 }
1478
1479 assert(!Lines.empty() && "Empty def?");
1480 if (!RetVar.getType().isVoid())
1481 Lines.back().insert(0, RetVar.getName() + " = ");
1482
1483 for (auto &L : Lines) {
1484 OS << " " << L;
1485 emitNewLine();
1486 }
1487 }
1488
emitReturn()1489 void Intrinsic::emitReturn() {
1490 if (RetVar.getType().isVoid())
1491 return;
1492 if (UseMacro)
1493 OS << " " << RetVar.getName() << ";";
1494 else
1495 OS << " return " << RetVar.getName() << ";";
1496 emitNewLine();
1497 }
1498
emitDag(DagInit * DI)1499 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDag(DagInit *DI) {
1500 // At this point we should only be seeing a def.
1501 DefInit *DefI = cast<DefInit>(DI->getOperator());
1502 std::string Op = DefI->getAsString();
1503
1504 if (Op == "cast" || Op == "bitcast")
1505 return emitDagCast(DI, Op == "bitcast");
1506 if (Op == "shuffle")
1507 return emitDagShuffle(DI);
1508 if (Op == "dup")
1509 return emitDagDup(DI);
1510 if (Op == "splat")
1511 return emitDagSplat(DI);
1512 if (Op == "save_temp")
1513 return emitDagSaveTemp(DI);
1514 if (Op == "op")
1515 return emitDagOp(DI);
1516 if (Op == "call")
1517 return emitDagCall(DI);
1518 if (Op == "name_replace")
1519 return emitDagNameReplace(DI);
1520 if (Op == "literal")
1521 return emitDagLiteral(DI);
1522 assert_with_loc(false, "Unknown operation!");
1523 return std::make_pair(Type::getVoid(), "");
1524 }
1525
emitDagOp(DagInit * DI)1526 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagOp(DagInit *DI) {
1527 std::string Op = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1528 if (DI->getNumArgs() == 2) {
1529 // Unary op.
1530 std::pair<Type, std::string> R =
1531 emitDagArg(DI->getArg(1), DI->getArgNameStr(1));
1532 return std::make_pair(R.first, Op + R.second);
1533 } else {
1534 assert(DI->getNumArgs() == 3 && "Can only handle unary and binary ops!");
1535 std::pair<Type, std::string> R1 =
1536 emitDagArg(DI->getArg(1), DI->getArgNameStr(1));
1537 std::pair<Type, std::string> R2 =
1538 emitDagArg(DI->getArg(2), DI->getArgNameStr(2));
1539 assert_with_loc(R1.first == R2.first, "Argument type mismatch!");
1540 return std::make_pair(R1.first, R1.second + " " + Op + " " + R2.second);
1541 }
1542 }
1543
emitDagCall(DagInit * DI)1544 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCall(DagInit *DI) {
1545 std::vector<Type> Types;
1546 std::vector<std::string> Values;
1547 for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1548 std::pair<Type, std::string> R =
1549 emitDagArg(DI->getArg(I + 1), DI->getArgNameStr(I + 1));
1550 Types.push_back(R.first);
1551 Values.push_back(R.second);
1552 }
1553
1554 // Look up the called intrinsic.
1555 std::string N;
1556 if (StringInit *SI = dyn_cast<StringInit>(DI->getArg(0)))
1557 N = SI->getAsUnquotedString();
1558 else
1559 N = emitDagArg(DI->getArg(0), "").second;
1560 Intrinsic &Callee = Intr.Emitter.getIntrinsic(N, Types);
1561
1562 // Make sure the callee is known as an early def.
1563 Callee.setNeededEarly();
1564 Intr.Dependencies.insert(&Callee);
1565
1566 // Now create the call itself.
1567 std::string S = CallPrefix.str() + Callee.getMangledName(true) + "(";
1568 for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1569 if (I != 0)
1570 S += ", ";
1571 S += Values[I];
1572 }
1573 S += ")";
1574
1575 return std::make_pair(Callee.getReturnType(), S);
1576 }
1577
emitDagCast(DagInit * DI,bool IsBitCast)1578 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCast(DagInit *DI,
1579 bool IsBitCast){
1580 // (cast MOD* VAL) -> cast VAL to type given by MOD.
1581 std::pair<Type, std::string> R = emitDagArg(
1582 DI->getArg(DI->getNumArgs() - 1),
1583 DI->getArgNameStr(DI->getNumArgs() - 1));
1584 Type castToType = R.first;
1585 for (unsigned ArgIdx = 0; ArgIdx < DI->getNumArgs() - 1; ++ArgIdx) {
1586
1587 // MOD can take several forms:
1588 // 1. $X - take the type of parameter / variable X.
1589 // 2. The value "R" - take the type of the return type.
1590 // 3. a type string
1591 // 4. The value "U" or "S" to switch the signedness.
1592 // 5. The value "H" or "D" to half or double the bitwidth.
1593 // 6. The value "8" to convert to 8-bit (signed) integer lanes.
1594 if (!DI->getArgNameStr(ArgIdx).empty()) {
1595 assert_with_loc(Intr.Variables.find(DI->getArgNameStr(ArgIdx)) !=
1596 Intr.Variables.end(),
1597 "Variable not found");
1598 castToType = Intr.Variables[DI->getArgNameStr(ArgIdx)].getType();
1599 } else {
1600 StringInit *SI = dyn_cast<StringInit>(DI->getArg(ArgIdx));
1601 assert_with_loc(SI, "Expected string type or $Name for cast type");
1602
1603 if (SI->getAsUnquotedString() == "R") {
1604 castToType = Intr.getReturnType();
1605 } else if (SI->getAsUnquotedString() == "U") {
1606 castToType.makeUnsigned();
1607 } else if (SI->getAsUnquotedString() == "S") {
1608 castToType.makeSigned();
1609 } else if (SI->getAsUnquotedString() == "H") {
1610 castToType.halveLanes();
1611 } else if (SI->getAsUnquotedString() == "D") {
1612 castToType.doubleLanes();
1613 } else if (SI->getAsUnquotedString() == "8") {
1614 castToType.makeInteger(8, true);
1615 } else {
1616 castToType = Type::fromTypedefName(SI->getAsUnquotedString());
1617 assert_with_loc(!castToType.isVoid(), "Unknown typedef");
1618 }
1619 }
1620 }
1621
1622 std::string S;
1623 if (IsBitCast) {
1624 // Emit a reinterpret cast. The second operand must be an lvalue, so create
1625 // a temporary.
1626 std::string N = "reint";
1627 unsigned I = 0;
1628 while (Intr.Variables.find(N) != Intr.Variables.end())
1629 N = "reint" + utostr(++I);
1630 Intr.Variables[N] = Variable(R.first, N + Intr.VariablePostfix);
1631
1632 Intr.OS << R.first.str() << " " << Intr.Variables[N].getName() << " = "
1633 << R.second << ";";
1634 Intr.emitNewLine();
1635
1636 S = "*(" + castToType.str() + " *) &" + Intr.Variables[N].getName() + "";
1637 } else {
1638 // Emit a normal (static) cast.
1639 S = "(" + castToType.str() + ")(" + R.second + ")";
1640 }
1641
1642 return std::make_pair(castToType, S);
1643 }
1644
emitDagShuffle(DagInit * DI)1645 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagShuffle(DagInit *DI){
1646 // See the documentation in arm_neon.td for a description of these operators.
1647 class LowHalf : public SetTheory::Operator {
1648 public:
1649 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1650 ArrayRef<SMLoc> Loc) override {
1651 SetTheory::RecSet Elts2;
1652 ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc);
1653 Elts.insert(Elts2.begin(), Elts2.begin() + (Elts2.size() / 2));
1654 }
1655 };
1656
1657 class HighHalf : public SetTheory::Operator {
1658 public:
1659 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1660 ArrayRef<SMLoc> Loc) override {
1661 SetTheory::RecSet Elts2;
1662 ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc);
1663 Elts.insert(Elts2.begin() + (Elts2.size() / 2), Elts2.end());
1664 }
1665 };
1666
1667 class Rev : public SetTheory::Operator {
1668 unsigned ElementSize;
1669
1670 public:
1671 Rev(unsigned ElementSize) : ElementSize(ElementSize) {}
1672
1673 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1674 ArrayRef<SMLoc> Loc) override {
1675 SetTheory::RecSet Elts2;
1676 ST.evaluate(Expr->arg_begin() + 1, Expr->arg_end(), Elts2, Loc);
1677
1678 int64_t VectorSize = cast<IntInit>(Expr->getArg(0))->getValue();
1679 VectorSize /= ElementSize;
1680
1681 std::vector<Record *> Revved;
1682 for (unsigned VI = 0; VI < Elts2.size(); VI += VectorSize) {
1683 for (int LI = VectorSize - 1; LI >= 0; --LI) {
1684 Revved.push_back(Elts2[VI + LI]);
1685 }
1686 }
1687
1688 Elts.insert(Revved.begin(), Revved.end());
1689 }
1690 };
1691
1692 class MaskExpander : public SetTheory::Expander {
1693 unsigned N;
1694
1695 public:
1696 MaskExpander(unsigned N) : N(N) {}
1697
1698 void expand(SetTheory &ST, Record *R, SetTheory::RecSet &Elts) override {
1699 unsigned Addend = 0;
1700 if (R->getName() == "mask0")
1701 Addend = 0;
1702 else if (R->getName() == "mask1")
1703 Addend = N;
1704 else
1705 return;
1706 for (unsigned I = 0; I < N; ++I)
1707 Elts.insert(R->getRecords().getDef("sv" + utostr(I + Addend)));
1708 }
1709 };
1710
1711 // (shuffle arg1, arg2, sequence)
1712 std::pair<Type, std::string> Arg1 =
1713 emitDagArg(DI->getArg(0), DI->getArgNameStr(0));
1714 std::pair<Type, std::string> Arg2 =
1715 emitDagArg(DI->getArg(1), DI->getArgNameStr(1));
1716 assert_with_loc(Arg1.first == Arg2.first,
1717 "Different types in arguments to shuffle!");
1718
1719 SetTheory ST;
1720 SetTheory::RecSet Elts;
1721 ST.addOperator("lowhalf", llvm::make_unique<LowHalf>());
1722 ST.addOperator("highhalf", llvm::make_unique<HighHalf>());
1723 ST.addOperator("rev",
1724 llvm::make_unique<Rev>(Arg1.first.getElementSizeInBits()));
1725 ST.addExpander("MaskExpand",
1726 llvm::make_unique<MaskExpander>(Arg1.first.getNumElements()));
1727 ST.evaluate(DI->getArg(2), Elts, None);
1728
1729 std::string S = "__builtin_shufflevector(" + Arg1.second + ", " + Arg2.second;
1730 for (auto &E : Elts) {
1731 StringRef Name = E->getName();
1732 assert_with_loc(Name.startswith("sv"),
1733 "Incorrect element kind in shuffle mask!");
1734 S += ", " + Name.drop_front(2).str();
1735 }
1736 S += ")";
1737
1738 // Recalculate the return type - the shuffle may have halved or doubled it.
1739 Type T(Arg1.first);
1740 if (Elts.size() > T.getNumElements()) {
1741 assert_with_loc(
1742 Elts.size() == T.getNumElements() * 2,
1743 "Can only double or half the number of elements in a shuffle!");
1744 T.doubleLanes();
1745 } else if (Elts.size() < T.getNumElements()) {
1746 assert_with_loc(
1747 Elts.size() == T.getNumElements() / 2,
1748 "Can only double or half the number of elements in a shuffle!");
1749 T.halveLanes();
1750 }
1751
1752 return std::make_pair(T, S);
1753 }
1754
emitDagDup(DagInit * DI)1755 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDup(DagInit *DI) {
1756 assert_with_loc(DI->getNumArgs() == 1, "dup() expects one argument");
1757 std::pair<Type, std::string> A = emitDagArg(DI->getArg(0),
1758 DI->getArgNameStr(0));
1759 assert_with_loc(A.first.isScalar(), "dup() expects a scalar argument");
1760
1761 Type T = Intr.getBaseType();
1762 assert_with_loc(T.isVector(), "dup() used but default type is scalar!");
1763 std::string S = "(" + T.str() + ") {";
1764 for (unsigned I = 0; I < T.getNumElements(); ++I) {
1765 if (I != 0)
1766 S += ", ";
1767 S += A.second;
1768 }
1769 S += "}";
1770
1771 return std::make_pair(T, S);
1772 }
1773
emitDagSplat(DagInit * DI)1774 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSplat(DagInit *DI) {
1775 assert_with_loc(DI->getNumArgs() == 2, "splat() expects two arguments");
1776 std::pair<Type, std::string> A = emitDagArg(DI->getArg(0),
1777 DI->getArgNameStr(0));
1778 std::pair<Type, std::string> B = emitDagArg(DI->getArg(1),
1779 DI->getArgNameStr(1));
1780
1781 assert_with_loc(B.first.isScalar(),
1782 "splat() requires a scalar int as the second argument");
1783
1784 std::string S = "__builtin_shufflevector(" + A.second + ", " + A.second;
1785 for (unsigned I = 0; I < Intr.getBaseType().getNumElements(); ++I) {
1786 S += ", " + B.second;
1787 }
1788 S += ")";
1789
1790 return std::make_pair(Intr.getBaseType(), S);
1791 }
1792
emitDagSaveTemp(DagInit * DI)1793 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSaveTemp(DagInit *DI) {
1794 assert_with_loc(DI->getNumArgs() == 2, "save_temp() expects two arguments");
1795 std::pair<Type, std::string> A = emitDagArg(DI->getArg(1),
1796 DI->getArgNameStr(1));
1797
1798 assert_with_loc(!A.first.isVoid(),
1799 "Argument to save_temp() must have non-void type!");
1800
1801 std::string N = DI->getArgNameStr(0);
1802 assert_with_loc(!N.empty(),
1803 "save_temp() expects a name as the first argument");
1804
1805 assert_with_loc(Intr.Variables.find(N) == Intr.Variables.end(),
1806 "Variable already defined!");
1807 Intr.Variables[N] = Variable(A.first, N + Intr.VariablePostfix);
1808
1809 std::string S =
1810 A.first.str() + " " + Intr.Variables[N].getName() + " = " + A.second;
1811
1812 return std::make_pair(Type::getVoid(), S);
1813 }
1814
1815 std::pair<Type, std::string>
emitDagNameReplace(DagInit * DI)1816 Intrinsic::DagEmitter::emitDagNameReplace(DagInit *DI) {
1817 std::string S = Intr.Name;
1818
1819 assert_with_loc(DI->getNumArgs() == 2, "name_replace requires 2 arguments!");
1820 std::string ToReplace = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1821 std::string ReplaceWith = cast<StringInit>(DI->getArg(1))->getAsUnquotedString();
1822
1823 size_t Idx = S.find(ToReplace);
1824
1825 assert_with_loc(Idx != std::string::npos, "name should contain '" + ToReplace + "'!");
1826 S.replace(Idx, ToReplace.size(), ReplaceWith);
1827
1828 return std::make_pair(Type::getVoid(), S);
1829 }
1830
emitDagLiteral(DagInit * DI)1831 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagLiteral(DagInit *DI){
1832 std::string Ty = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1833 std::string Value = cast<StringInit>(DI->getArg(1))->getAsUnquotedString();
1834 return std::make_pair(Type::fromTypedefName(Ty), Value);
1835 }
1836
1837 std::pair<Type, std::string>
emitDagArg(Init * Arg,std::string ArgName)1838 Intrinsic::DagEmitter::emitDagArg(Init *Arg, std::string ArgName) {
1839 if (!ArgName.empty()) {
1840 assert_with_loc(!Arg->isComplete(),
1841 "Arguments must either be DAGs or names, not both!");
1842 assert_with_loc(Intr.Variables.find(ArgName) != Intr.Variables.end(),
1843 "Variable not defined!");
1844 Variable &V = Intr.Variables[ArgName];
1845 return std::make_pair(V.getType(), V.getName());
1846 }
1847
1848 assert(Arg && "Neither ArgName nor Arg?!");
1849 DagInit *DI = dyn_cast<DagInit>(Arg);
1850 assert_with_loc(DI, "Arguments must either be DAGs or names!");
1851
1852 return emitDag(DI);
1853 }
1854
generate()1855 std::string Intrinsic::generate() {
1856 // Little endian intrinsics are simple and don't require any argument
1857 // swapping.
1858 OS << "#ifdef __LITTLE_ENDIAN__\n";
1859
1860 generateImpl(false, "", "");
1861
1862 OS << "#else\n";
1863
1864 // Big endian intrinsics are more complex. The user intended these
1865 // intrinsics to operate on a vector "as-if" loaded by (V)LDR,
1866 // but we load as-if (V)LD1. So we should swap all arguments and
1867 // swap the return value too.
1868 //
1869 // If we call sub-intrinsics, we should call a version that does
1870 // not re-swap the arguments!
1871 generateImpl(true, "", "__noswap_");
1872
1873 // If we're needed early, create a non-swapping variant for
1874 // big-endian.
1875 if (NeededEarly) {
1876 generateImpl(false, "__noswap_", "__noswap_");
1877 }
1878 OS << "#endif\n\n";
1879
1880 return OS.str();
1881 }
1882
generateImpl(bool ReverseArguments,StringRef NamePrefix,StringRef CallPrefix)1883 void Intrinsic::generateImpl(bool ReverseArguments,
1884 StringRef NamePrefix, StringRef CallPrefix) {
1885 CurrentRecord = R;
1886
1887 // If we call a macro, our local variables may be corrupted due to
1888 // lack of proper lexical scoping. So, add a globally unique postfix
1889 // to every variable.
1890 //
1891 // indexBody() should have set up the Dependencies set by now.
1892 for (auto *I : Dependencies)
1893 if (I->UseMacro) {
1894 VariablePostfix = "_" + utostr(Emitter.getUniqueNumber());
1895 break;
1896 }
1897
1898 initVariables();
1899
1900 emitPrototype(NamePrefix);
1901
1902 if (IsUnavailable) {
1903 OS << " __attribute__((unavailable));";
1904 } else {
1905 emitOpeningBrace();
1906 emitShadowedArgs();
1907 if (ReverseArguments)
1908 emitArgumentReversal();
1909 emitBody(CallPrefix);
1910 if (ReverseArguments)
1911 emitReturnReversal();
1912 emitReturn();
1913 emitClosingBrace();
1914 }
1915 OS << "\n";
1916
1917 CurrentRecord = nullptr;
1918 }
1919
indexBody()1920 void Intrinsic::indexBody() {
1921 CurrentRecord = R;
1922
1923 initVariables();
1924 emitBody("");
1925 OS.str("");
1926
1927 CurrentRecord = nullptr;
1928 }
1929
1930 //===----------------------------------------------------------------------===//
1931 // NeonEmitter implementation
1932 //===----------------------------------------------------------------------===//
1933
getIntrinsic(StringRef Name,ArrayRef<Type> Types)1934 Intrinsic &NeonEmitter::getIntrinsic(StringRef Name, ArrayRef<Type> Types) {
1935 // First, look up the name in the intrinsic map.
1936 assert_with_loc(IntrinsicMap.find(Name.str()) != IntrinsicMap.end(),
1937 ("Intrinsic '" + Name + "' not found!").str());
1938 auto &V = IntrinsicMap.find(Name.str())->second;
1939 std::vector<Intrinsic *> GoodVec;
1940
1941 // Create a string to print if we end up failing.
1942 std::string ErrMsg = "looking up intrinsic '" + Name.str() + "(";
1943 for (unsigned I = 0; I < Types.size(); ++I) {
1944 if (I != 0)
1945 ErrMsg += ", ";
1946 ErrMsg += Types[I].str();
1947 }
1948 ErrMsg += ")'\n";
1949 ErrMsg += "Available overloads:\n";
1950
1951 // Now, look through each intrinsic implementation and see if the types are
1952 // compatible.
1953 for (auto &I : V) {
1954 ErrMsg += " - " + I.getReturnType().str() + " " + I.getMangledName();
1955 ErrMsg += "(";
1956 for (unsigned A = 0; A < I.getNumParams(); ++A) {
1957 if (A != 0)
1958 ErrMsg += ", ";
1959 ErrMsg += I.getParamType(A).str();
1960 }
1961 ErrMsg += ")\n";
1962
1963 if (I.getNumParams() != Types.size())
1964 continue;
1965
1966 bool Good = true;
1967 for (unsigned Arg = 0; Arg < Types.size(); ++Arg) {
1968 if (I.getParamType(Arg) != Types[Arg]) {
1969 Good = false;
1970 break;
1971 }
1972 }
1973 if (Good)
1974 GoodVec.push_back(&I);
1975 }
1976
1977 assert_with_loc(!GoodVec.empty(),
1978 "No compatible intrinsic found - " + ErrMsg);
1979 assert_with_loc(GoodVec.size() == 1, "Multiple overloads found - " + ErrMsg);
1980
1981 return *GoodVec.front();
1982 }
1983
createIntrinsic(Record * R,SmallVectorImpl<Intrinsic * > & Out)1984 void NeonEmitter::createIntrinsic(Record *R,
1985 SmallVectorImpl<Intrinsic *> &Out) {
1986 std::string Name = R->getValueAsString("Name");
1987 std::string Proto = R->getValueAsString("Prototype");
1988 std::string Types = R->getValueAsString("Types");
1989 Record *OperationRec = R->getValueAsDef("Operation");
1990 bool CartesianProductOfTypes = R->getValueAsBit("CartesianProductOfTypes");
1991 bool BigEndianSafe = R->getValueAsBit("BigEndianSafe");
1992 std::string Guard = R->getValueAsString("ArchGuard");
1993 bool IsUnavailable = OperationRec->getValueAsBit("Unavailable");
1994
1995 // Set the global current record. This allows assert_with_loc to produce
1996 // decent location information even when highly nested.
1997 CurrentRecord = R;
1998
1999 ListInit *Body = OperationRec->getValueAsListInit("Ops");
2000
2001 std::vector<TypeSpec> TypeSpecs = TypeSpec::fromTypeSpecs(Types);
2002
2003 ClassKind CK = ClassNone;
2004 if (R->getSuperClasses().size() >= 2)
2005 CK = ClassMap[R->getSuperClasses()[1].first];
2006
2007 std::vector<std::pair<TypeSpec, TypeSpec>> NewTypeSpecs;
2008 for (auto TS : TypeSpecs) {
2009 if (CartesianProductOfTypes) {
2010 Type DefaultT(TS, 'd');
2011 for (auto SrcTS : TypeSpecs) {
2012 Type DefaultSrcT(SrcTS, 'd');
2013 if (TS == SrcTS ||
2014 DefaultSrcT.getSizeInBits() != DefaultT.getSizeInBits())
2015 continue;
2016 NewTypeSpecs.push_back(std::make_pair(TS, SrcTS));
2017 }
2018 } else {
2019 NewTypeSpecs.push_back(std::make_pair(TS, TS));
2020 }
2021 }
2022
2023 llvm::sort(NewTypeSpecs.begin(), NewTypeSpecs.end());
2024 NewTypeSpecs.erase(std::unique(NewTypeSpecs.begin(), NewTypeSpecs.end()),
2025 NewTypeSpecs.end());
2026 auto &Entry = IntrinsicMap[Name];
2027
2028 for (auto &I : NewTypeSpecs) {
2029 Entry.emplace_back(R, Name, Proto, I.first, I.second, CK, Body, *this,
2030 Guard, IsUnavailable, BigEndianSafe);
2031 Out.push_back(&Entry.back());
2032 }
2033
2034 CurrentRecord = nullptr;
2035 }
2036
2037 /// genBuiltinsDef: Generate the BuiltinsARM.def and BuiltinsAArch64.def
2038 /// declaration of builtins, checking for unique builtin declarations.
genBuiltinsDef(raw_ostream & OS,SmallVectorImpl<Intrinsic * > & Defs)2039 void NeonEmitter::genBuiltinsDef(raw_ostream &OS,
2040 SmallVectorImpl<Intrinsic *> &Defs) {
2041 OS << "#ifdef GET_NEON_BUILTINS\n";
2042
2043 // We only want to emit a builtin once, and we want to emit them in
2044 // alphabetical order, so use a std::set.
2045 std::set<std::string> Builtins;
2046
2047 for (auto *Def : Defs) {
2048 if (Def->hasBody())
2049 continue;
2050 // Functions with 'a' (the splat code) in the type prototype should not get
2051 // their own builtin as they use the non-splat variant.
2052 if (Def->hasSplat())
2053 continue;
2054
2055 std::string S = "BUILTIN(__builtin_neon_" + Def->getMangledName() + ", \"";
2056
2057 S += Def->getBuiltinTypeStr();
2058 S += "\", \"n\")";
2059
2060 Builtins.insert(S);
2061 }
2062
2063 for (auto &S : Builtins)
2064 OS << S << "\n";
2065 OS << "#endif\n\n";
2066 }
2067
2068 /// Generate the ARM and AArch64 overloaded type checking code for
2069 /// SemaChecking.cpp, checking for unique builtin declarations.
genOverloadTypeCheckCode(raw_ostream & OS,SmallVectorImpl<Intrinsic * > & Defs)2070 void NeonEmitter::genOverloadTypeCheckCode(raw_ostream &OS,
2071 SmallVectorImpl<Intrinsic *> &Defs) {
2072 OS << "#ifdef GET_NEON_OVERLOAD_CHECK\n";
2073
2074 // We record each overload check line before emitting because subsequent Inst
2075 // definitions may extend the number of permitted types (i.e. augment the
2076 // Mask). Use std::map to avoid sorting the table by hash number.
2077 struct OverloadInfo {
2078 uint64_t Mask;
2079 int PtrArgNum;
2080 bool HasConstPtr;
2081 OverloadInfo() : Mask(0ULL), PtrArgNum(0), HasConstPtr(false) {}
2082 };
2083 std::map<std::string, OverloadInfo> OverloadMap;
2084
2085 for (auto *Def : Defs) {
2086 // If the def has a body (that is, it has Operation DAGs), it won't call
2087 // __builtin_neon_* so we don't need to generate a definition for it.
2088 if (Def->hasBody())
2089 continue;
2090 // Functions with 'a' (the splat code) in the type prototype should not get
2091 // their own builtin as they use the non-splat variant.
2092 if (Def->hasSplat())
2093 continue;
2094 // Functions which have a scalar argument cannot be overloaded, no need to
2095 // check them if we are emitting the type checking code.
2096 if (Def->protoHasScalar())
2097 continue;
2098
2099 uint64_t Mask = 0ULL;
2100 Type Ty = Def->getReturnType();
2101 if (Def->getProto()[0] == 'v' ||
2102 isFloatingPointProtoModifier(Def->getProto()[0]))
2103 Ty = Def->getParamType(0);
2104 if (Ty.isPointer())
2105 Ty = Def->getParamType(1);
2106
2107 Mask |= 1ULL << Ty.getNeonEnum();
2108
2109 // Check if the function has a pointer or const pointer argument.
2110 std::string Proto = Def->getProto();
2111 int PtrArgNum = -1;
2112 bool HasConstPtr = false;
2113 for (unsigned I = 0; I < Def->getNumParams(); ++I) {
2114 char ArgType = Proto[I + 1];
2115 if (ArgType == 'c') {
2116 HasConstPtr = true;
2117 PtrArgNum = I;
2118 break;
2119 }
2120 if (ArgType == 'p') {
2121 PtrArgNum = I;
2122 break;
2123 }
2124 }
2125 // For sret builtins, adjust the pointer argument index.
2126 if (PtrArgNum >= 0 && Def->getReturnType().getNumVectors() > 1)
2127 PtrArgNum += 1;
2128
2129 std::string Name = Def->getName();
2130 // Omit type checking for the pointer arguments of vld1_lane, vld1_dup,
2131 // and vst1_lane intrinsics. Using a pointer to the vector element
2132 // type with one of those operations causes codegen to select an aligned
2133 // load/store instruction. If you want an unaligned operation,
2134 // the pointer argument needs to have less alignment than element type,
2135 // so just accept any pointer type.
2136 if (Name == "vld1_lane" || Name == "vld1_dup" || Name == "vst1_lane") {
2137 PtrArgNum = -1;
2138 HasConstPtr = false;
2139 }
2140
2141 if (Mask) {
2142 std::string Name = Def->getMangledName();
2143 OverloadMap.insert(std::make_pair(Name, OverloadInfo()));
2144 OverloadInfo &OI = OverloadMap[Name];
2145 OI.Mask |= Mask;
2146 OI.PtrArgNum |= PtrArgNum;
2147 OI.HasConstPtr = HasConstPtr;
2148 }
2149 }
2150
2151 for (auto &I : OverloadMap) {
2152 OverloadInfo &OI = I.second;
2153
2154 OS << "case NEON::BI__builtin_neon_" << I.first << ": ";
2155 OS << "mask = 0x" << Twine::utohexstr(OI.Mask) << "ULL";
2156 if (OI.PtrArgNum >= 0)
2157 OS << "; PtrArgNum = " << OI.PtrArgNum;
2158 if (OI.HasConstPtr)
2159 OS << "; HasConstPtr = true";
2160 OS << "; break;\n";
2161 }
2162 OS << "#endif\n\n";
2163 }
2164
genIntrinsicRangeCheckCode(raw_ostream & OS,SmallVectorImpl<Intrinsic * > & Defs)2165 void NeonEmitter::genIntrinsicRangeCheckCode(raw_ostream &OS,
2166 SmallVectorImpl<Intrinsic *> &Defs) {
2167 OS << "#ifdef GET_NEON_IMMEDIATE_CHECK\n";
2168
2169 std::set<std::string> Emitted;
2170
2171 for (auto *Def : Defs) {
2172 if (Def->hasBody())
2173 continue;
2174 // Functions with 'a' (the splat code) in the type prototype should not get
2175 // their own builtin as they use the non-splat variant.
2176 if (Def->hasSplat())
2177 continue;
2178 // Functions which do not have an immediate do not need to have range
2179 // checking code emitted.
2180 if (!Def->hasImmediate())
2181 continue;
2182 if (Emitted.find(Def->getMangledName()) != Emitted.end())
2183 continue;
2184
2185 std::string LowerBound, UpperBound;
2186
2187 Record *R = Def->getRecord();
2188 if (R->getValueAsBit("isVCVT_N")) {
2189 // VCVT between floating- and fixed-point values takes an immediate
2190 // in the range [1, 32) for f32 or [1, 64) for f64 or [1, 16) for f16.
2191 LowerBound = "1";
2192 if (Def->getBaseType().getElementSizeInBits() == 16 ||
2193 Def->getName().find('h') != std::string::npos)
2194 // VCVTh operating on FP16 intrinsics in range [1, 16)
2195 UpperBound = "15";
2196 else if (Def->getBaseType().getElementSizeInBits() == 32)
2197 UpperBound = "31";
2198 else
2199 UpperBound = "63";
2200 } else if (R->getValueAsBit("isScalarShift")) {
2201 // Right shifts have an 'r' in the name, left shifts do not. Convert
2202 // instructions have the same bounds and right shifts.
2203 if (Def->getName().find('r') != std::string::npos ||
2204 Def->getName().find("cvt") != std::string::npos)
2205 LowerBound = "1";
2206
2207 UpperBound = utostr(Def->getReturnType().getElementSizeInBits() - 1);
2208 } else if (R->getValueAsBit("isShift")) {
2209 // Builtins which are overloaded by type will need to have their upper
2210 // bound computed at Sema time based on the type constant.
2211
2212 // Right shifts have an 'r' in the name, left shifts do not.
2213 if (Def->getName().find('r') != std::string::npos)
2214 LowerBound = "1";
2215 UpperBound = "RFT(TV, true)";
2216 } else if (Def->getClassKind(true) == ClassB) {
2217 // ClassB intrinsics have a type (and hence lane number) that is only
2218 // known at runtime.
2219 if (R->getValueAsBit("isLaneQ"))
2220 UpperBound = "RFT(TV, false, true)";
2221 else
2222 UpperBound = "RFT(TV, false, false)";
2223 } else {
2224 // The immediate generally refers to a lane in the preceding argument.
2225 assert(Def->getImmediateIdx() > 0);
2226 Type T = Def->getParamType(Def->getImmediateIdx() - 1);
2227 UpperBound = utostr(T.getNumElements() - 1);
2228 }
2229
2230 // Calculate the index of the immediate that should be range checked.
2231 unsigned Idx = Def->getNumParams();
2232 if (Def->hasImmediate())
2233 Idx = Def->getGeneratedParamIdx(Def->getImmediateIdx());
2234
2235 OS << "case NEON::BI__builtin_neon_" << Def->getMangledName() << ": "
2236 << "i = " << Idx << ";";
2237 if (!LowerBound.empty())
2238 OS << " l = " << LowerBound << ";";
2239 if (!UpperBound.empty())
2240 OS << " u = " << UpperBound << ";";
2241 OS << " break;\n";
2242
2243 Emitted.insert(Def->getMangledName());
2244 }
2245
2246 OS << "#endif\n\n";
2247 }
2248
2249 /// runHeader - Emit a file with sections defining:
2250 /// 1. the NEON section of BuiltinsARM.def and BuiltinsAArch64.def.
2251 /// 2. the SemaChecking code for the type overload checking.
2252 /// 3. the SemaChecking code for validation of intrinsic immediate arguments.
runHeader(raw_ostream & OS)2253 void NeonEmitter::runHeader(raw_ostream &OS) {
2254 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2255
2256 SmallVector<Intrinsic *, 128> Defs;
2257 for (auto *R : RV)
2258 createIntrinsic(R, Defs);
2259
2260 // Generate shared BuiltinsXXX.def
2261 genBuiltinsDef(OS, Defs);
2262
2263 // Generate ARM overloaded type checking code for SemaChecking.cpp
2264 genOverloadTypeCheckCode(OS, Defs);
2265
2266 // Generate ARM range checking code for shift/lane immediates.
2267 genIntrinsicRangeCheckCode(OS, Defs);
2268 }
2269
2270 /// run - Read the records in arm_neon.td and output arm_neon.h. arm_neon.h
2271 /// is comprised of type definitions and function declarations.
run(raw_ostream & OS)2272 void NeonEmitter::run(raw_ostream &OS) {
2273 OS << "/*===---- arm_neon.h - ARM Neon intrinsics "
2274 "------------------------------"
2275 "---===\n"
2276 " *\n"
2277 " * Permission is hereby granted, free of charge, to any person "
2278 "obtaining "
2279 "a copy\n"
2280 " * of this software and associated documentation files (the "
2281 "\"Software\"),"
2282 " to deal\n"
2283 " * in the Software without restriction, including without limitation "
2284 "the "
2285 "rights\n"
2286 " * to use, copy, modify, merge, publish, distribute, sublicense, "
2287 "and/or sell\n"
2288 " * copies of the Software, and to permit persons to whom the Software "
2289 "is\n"
2290 " * furnished to do so, subject to the following conditions:\n"
2291 " *\n"
2292 " * The above copyright notice and this permission notice shall be "
2293 "included in\n"
2294 " * all copies or substantial portions of the Software.\n"
2295 " *\n"
2296 " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, "
2297 "EXPRESS OR\n"
2298 " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF "
2299 "MERCHANTABILITY,\n"
2300 " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT "
2301 "SHALL THE\n"
2302 " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR "
2303 "OTHER\n"
2304 " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, "
2305 "ARISING FROM,\n"
2306 " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER "
2307 "DEALINGS IN\n"
2308 " * THE SOFTWARE.\n"
2309 " *\n"
2310 " *===-----------------------------------------------------------------"
2311 "---"
2312 "---===\n"
2313 " */\n\n";
2314
2315 OS << "#ifndef __ARM_NEON_H\n";
2316 OS << "#define __ARM_NEON_H\n\n";
2317
2318 OS << "#if !defined(__ARM_NEON)\n";
2319 OS << "#error \"NEON support not enabled\"\n";
2320 OS << "#endif\n\n";
2321
2322 OS << "#include <stdint.h>\n\n";
2323
2324 // Emit NEON-specific scalar typedefs.
2325 OS << "typedef float float32_t;\n";
2326 OS << "typedef __fp16 float16_t;\n";
2327
2328 OS << "#ifdef __aarch64__\n";
2329 OS << "typedef double float64_t;\n";
2330 OS << "#endif\n\n";
2331
2332 // For now, signedness of polynomial types depends on target
2333 OS << "#ifdef __aarch64__\n";
2334 OS << "typedef uint8_t poly8_t;\n";
2335 OS << "typedef uint16_t poly16_t;\n";
2336 OS << "typedef uint64_t poly64_t;\n";
2337 OS << "typedef __uint128_t poly128_t;\n";
2338 OS << "#else\n";
2339 OS << "typedef int8_t poly8_t;\n";
2340 OS << "typedef int16_t poly16_t;\n";
2341 OS << "#endif\n";
2342
2343 // Emit Neon vector typedefs.
2344 std::string TypedefTypes(
2345 "cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQdPcQPcPsQPsPlQPl");
2346 std::vector<TypeSpec> TDTypeVec = TypeSpec::fromTypeSpecs(TypedefTypes);
2347
2348 // Emit vector typedefs.
2349 bool InIfdef = false;
2350 for (auto &TS : TDTypeVec) {
2351 bool IsA64 = false;
2352 Type T(TS, 'd');
2353 if (T.isDouble() || (T.isPoly() && T.isLong()))
2354 IsA64 = true;
2355
2356 if (InIfdef && !IsA64) {
2357 OS << "#endif\n";
2358 InIfdef = false;
2359 }
2360 if (!InIfdef && IsA64) {
2361 OS << "#ifdef __aarch64__\n";
2362 InIfdef = true;
2363 }
2364
2365 if (T.isPoly())
2366 OS << "typedef __attribute__((neon_polyvector_type(";
2367 else
2368 OS << "typedef __attribute__((neon_vector_type(";
2369
2370 Type T2 = T;
2371 T2.makeScalar();
2372 OS << T.getNumElements() << "))) ";
2373 OS << T2.str();
2374 OS << " " << T.str() << ";\n";
2375 }
2376 if (InIfdef)
2377 OS << "#endif\n";
2378 OS << "\n";
2379
2380 // Emit struct typedefs.
2381 InIfdef = false;
2382 for (unsigned NumMembers = 2; NumMembers <= 4; ++NumMembers) {
2383 for (auto &TS : TDTypeVec) {
2384 bool IsA64 = false;
2385 Type T(TS, 'd');
2386 if (T.isDouble() || (T.isPoly() && T.isLong()))
2387 IsA64 = true;
2388
2389 if (InIfdef && !IsA64) {
2390 OS << "#endif\n";
2391 InIfdef = false;
2392 }
2393 if (!InIfdef && IsA64) {
2394 OS << "#ifdef __aarch64__\n";
2395 InIfdef = true;
2396 }
2397
2398 char M = '2' + (NumMembers - 2);
2399 Type VT(TS, M);
2400 OS << "typedef struct " << VT.str() << " {\n";
2401 OS << " " << T.str() << " val";
2402 OS << "[" << NumMembers << "]";
2403 OS << ";\n} ";
2404 OS << VT.str() << ";\n";
2405 OS << "\n";
2406 }
2407 }
2408 if (InIfdef)
2409 OS << "#endif\n";
2410 OS << "\n";
2411
2412 OS << "#define __ai static inline __attribute__((__always_inline__, "
2413 "__nodebug__))\n\n";
2414
2415 SmallVector<Intrinsic *, 128> Defs;
2416 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2417 for (auto *R : RV)
2418 createIntrinsic(R, Defs);
2419
2420 for (auto *I : Defs)
2421 I->indexBody();
2422
2423 std::stable_sort(
2424 Defs.begin(), Defs.end(),
2425 [](const Intrinsic *A, const Intrinsic *B) { return *A < *B; });
2426
2427 // Only emit a def when its requirements have been met.
2428 // FIXME: This loop could be made faster, but it's fast enough for now.
2429 bool MadeProgress = true;
2430 std::string InGuard;
2431 while (!Defs.empty() && MadeProgress) {
2432 MadeProgress = false;
2433
2434 for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2435 I != Defs.end(); /*No step*/) {
2436 bool DependenciesSatisfied = true;
2437 for (auto *II : (*I)->getDependencies()) {
2438 if (std::find(Defs.begin(), Defs.end(), II) != Defs.end())
2439 DependenciesSatisfied = false;
2440 }
2441 if (!DependenciesSatisfied) {
2442 // Try the next one.
2443 ++I;
2444 continue;
2445 }
2446
2447 // Emit #endif/#if pair if needed.
2448 if ((*I)->getGuard() != InGuard) {
2449 if (!InGuard.empty())
2450 OS << "#endif\n";
2451 InGuard = (*I)->getGuard();
2452 if (!InGuard.empty())
2453 OS << "#if " << InGuard << "\n";
2454 }
2455
2456 // Actually generate the intrinsic code.
2457 OS << (*I)->generate();
2458
2459 MadeProgress = true;
2460 I = Defs.erase(I);
2461 }
2462 }
2463 assert(Defs.empty() && "Some requirements were not satisfied!");
2464 if (!InGuard.empty())
2465 OS << "#endif\n";
2466
2467 OS << "\n";
2468 OS << "#undef __ai\n\n";
2469 OS << "#endif /* __ARM_NEON_H */\n";
2470 }
2471
2472 /// run - Read the records in arm_fp16.td and output arm_fp16.h. arm_fp16.h
2473 /// is comprised of type definitions and function declarations.
runFP16(raw_ostream & OS)2474 void NeonEmitter::runFP16(raw_ostream &OS) {
2475 OS << "/*===---- arm_fp16.h - ARM FP16 intrinsics "
2476 "------------------------------"
2477 "---===\n"
2478 " *\n"
2479 " * Permission is hereby granted, free of charge, to any person "
2480 "obtaining a copy\n"
2481 " * of this software and associated documentation files (the "
2482 "\"Software\"), to deal\n"
2483 " * in the Software without restriction, including without limitation "
2484 "the rights\n"
2485 " * to use, copy, modify, merge, publish, distribute, sublicense, "
2486 "and/or sell\n"
2487 " * copies of the Software, and to permit persons to whom the Software "
2488 "is\n"
2489 " * furnished to do so, subject to the following conditions:\n"
2490 " *\n"
2491 " * The above copyright notice and this permission notice shall be "
2492 "included in\n"
2493 " * all copies or substantial portions of the Software.\n"
2494 " *\n"
2495 " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, "
2496 "EXPRESS OR\n"
2497 " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF "
2498 "MERCHANTABILITY,\n"
2499 " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT "
2500 "SHALL THE\n"
2501 " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR "
2502 "OTHER\n"
2503 " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, "
2504 "ARISING FROM,\n"
2505 " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER "
2506 "DEALINGS IN\n"
2507 " * THE SOFTWARE.\n"
2508 " *\n"
2509 " *===-----------------------------------------------------------------"
2510 "---"
2511 "---===\n"
2512 " */\n\n";
2513
2514 OS << "#ifndef __ARM_FP16_H\n";
2515 OS << "#define __ARM_FP16_H\n\n";
2516
2517 OS << "#include <stdint.h>\n\n";
2518
2519 OS << "typedef __fp16 float16_t;\n";
2520
2521 OS << "#define __ai static inline __attribute__((__always_inline__, "
2522 "__nodebug__))\n\n";
2523
2524 SmallVector<Intrinsic *, 128> Defs;
2525 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2526 for (auto *R : RV)
2527 createIntrinsic(R, Defs);
2528
2529 for (auto *I : Defs)
2530 I->indexBody();
2531
2532 std::stable_sort(
2533 Defs.begin(), Defs.end(),
2534 [](const Intrinsic *A, const Intrinsic *B) { return *A < *B; });
2535
2536 // Only emit a def when its requirements have been met.
2537 // FIXME: This loop could be made faster, but it's fast enough for now.
2538 bool MadeProgress = true;
2539 std::string InGuard;
2540 while (!Defs.empty() && MadeProgress) {
2541 MadeProgress = false;
2542
2543 for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2544 I != Defs.end(); /*No step*/) {
2545 bool DependenciesSatisfied = true;
2546 for (auto *II : (*I)->getDependencies()) {
2547 if (std::find(Defs.begin(), Defs.end(), II) != Defs.end())
2548 DependenciesSatisfied = false;
2549 }
2550 if (!DependenciesSatisfied) {
2551 // Try the next one.
2552 ++I;
2553 continue;
2554 }
2555
2556 // Emit #endif/#if pair if needed.
2557 if ((*I)->getGuard() != InGuard) {
2558 if (!InGuard.empty())
2559 OS << "#endif\n";
2560 InGuard = (*I)->getGuard();
2561 if (!InGuard.empty())
2562 OS << "#if " << InGuard << "\n";
2563 }
2564
2565 // Actually generate the intrinsic code.
2566 OS << (*I)->generate();
2567
2568 MadeProgress = true;
2569 I = Defs.erase(I);
2570 }
2571 }
2572 assert(Defs.empty() && "Some requirements were not satisfied!");
2573 if (!InGuard.empty())
2574 OS << "#endif\n";
2575
2576 OS << "\n";
2577 OS << "#undef __ai\n\n";
2578 OS << "#endif /* __ARM_FP16_H */\n";
2579 }
2580
2581 namespace clang {
2582
EmitNeon(RecordKeeper & Records,raw_ostream & OS)2583 void EmitNeon(RecordKeeper &Records, raw_ostream &OS) {
2584 NeonEmitter(Records).run(OS);
2585 }
2586
EmitFP16(RecordKeeper & Records,raw_ostream & OS)2587 void EmitFP16(RecordKeeper &Records, raw_ostream &OS) {
2588 NeonEmitter(Records).runFP16(OS);
2589 }
2590
EmitNeonSema(RecordKeeper & Records,raw_ostream & OS)2591 void EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) {
2592 NeonEmitter(Records).runHeader(OS);
2593 }
2594
EmitNeonTest(RecordKeeper & Records,raw_ostream & OS)2595 void EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) {
2596 llvm_unreachable("Neon test generation no longer implemented!");
2597 }
2598
2599 } // end namespace clang
2600