1 //===- MveEmitter.cpp - Generate arm_mve.h for use with clang -*- C++ -*-=====//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This set of linked tablegen backends is responsible for emitting the bits
10 // and pieces that implement <arm_mve.h>, which is defined by the ACLE standard
11 // and provides a set of types and functions for (more or less) direct access
12 // to the MVE instruction set, including the scalar shifts as well as the
13 // vector instructions.
14 //
15 // MVE's standard intrinsic functions are unusual in that they have a system of
16 // polymorphism. For example, the function vaddq() can behave like vaddq_u16(),
17 // vaddq_f32(), vaddq_s8(), etc., depending on the types of the vector
18 // arguments you give it.
19 //
20 // This constrains the implementation strategies. The usual approach to making
21 // the user-facing functions polymorphic would be to either use
22 // __attribute__((overloadable)) to make a set of vaddq() functions that are
23 // all inline wrappers on the underlying clang builtins, or to define a single
24 // vaddq() macro which expands to an instance of _Generic.
25 //
26 // The inline-wrappers approach would work fine for most intrinsics, except for
27 // the ones that take an argument required to be a compile-time constant,
28 // because if you wrap an inline function around a call to a builtin, the
29 // constant nature of the argument is not passed through.
30 //
31 // The _Generic approach can be made to work with enough effort, but it takes a
32 // lot of machinery, because of the design feature of _Generic that even the
33 // untaken branches are required to pass all front-end validity checks such as
34 // type-correctness. You can work around that by nesting further _Generics all
35 // over the place to coerce things to the right type in untaken branches, but
36 // what you get out is complicated, hard to guarantee its correctness, and
37 // worst of all, gives _completely unreadable_ error messages if the user gets
38 // the types wrong for an intrinsic call.
39 //
40 // Therefore, my strategy is to introduce a new __attribute__ that allows a
41 // function to be mapped to a clang builtin even though it doesn't have the
42 // same name, and then declare all the user-facing MVE function names with that
43 // attribute, mapping each one directly to the clang builtin. And the
44 // polymorphic ones have __attribute__((overloadable)) as well. So once the
45 // compiler has resolved the overload, it knows the internal builtin ID of the
46 // selected function, and can check the immediate arguments against that; and
47 // if the user gets the types wrong in a call to a polymorphic intrinsic, they
48 // get a completely clear error message showing all the declarations of that
49 // function in the header file and explaining why each one doesn't fit their
50 // call.
51 //
52 // The downside of this is that if every clang builtin has to correspond
53 // exactly to a user-facing ACLE intrinsic, then you can't save work in the
54 // frontend by doing it in the header file: CGBuiltin.cpp has to do the entire
55 // job of converting an ACLE intrinsic call into LLVM IR. So the Tablegen
56 // description for an MVE intrinsic has to contain a full description of the
57 // sequence of IRBuilder calls that clang will need to make.
58 //
59 //===----------------------------------------------------------------------===//
60
61 #include "llvm/ADT/APInt.h"
62 #include "llvm/ADT/StringRef.h"
63 #include "llvm/Support/Casting.h"
64 #include "llvm/Support/raw_ostream.h"
65 #include "llvm/TableGen/Error.h"
66 #include "llvm/TableGen/Record.h"
67 #include <cassert>
68 #include <cstddef>
69 #include <cstdint>
70 #include <list>
71 #include <map>
72 #include <memory>
73 #include <set>
74 #include <string>
75 #include <vector>
76
77 using namespace llvm;
78
79 namespace {
80
81 class MveEmitter;
82 class Result;
83
84 // -----------------------------------------------------------------------------
85 // A system of classes to represent all the types we'll need to deal with in
86 // the prototypes of intrinsics.
87 //
88 // Query methods include finding out the C name of a type; the "LLVM name" in
89 // the sense of a C++ code snippet that can be used in the codegen function;
90 // the suffix that represents the type in the ACLE intrinsic naming scheme
91 // (e.g. 's32' represents int32_t in intrinsics such as vaddq_s32); whether the
92 // type is floating-point related (hence should be under #ifdef in the MVE
93 // header so that it isn't included in integer-only MVE mode); and the type's
94 // size in bits. Not all subtypes support all these queries.
95
96 class Type {
97 public:
98 enum class TypeKind {
99 // Void appears as a return type (for store intrinsics, which are pure
100 // side-effect). It's also used as the parameter type in the Tablegen
101 // when an intrinsic doesn't need to come in various suffixed forms like
102 // vfooq_s8,vfooq_u16,vfooq_f32.
103 Void,
104
105 // Scalar is used for ordinary int and float types of all sizes.
106 Scalar,
107
108 // Vector is used for anything that occupies exactly one MVE vector
109 // register, i.e. {uint,int,float}NxM_t.
110 Vector,
111
112 // MultiVector is used for the {uint,int,float}NxMxK_t types used by the
113 // interleaving load/store intrinsics v{ld,st}{2,4}q.
114 MultiVector,
115
116 // Predicate is used by all the predicated intrinsics. Its C
117 // representation is mve_pred16_t (which is just an alias for uint16_t).
118 // But we give more detail here, by indicating that a given predicate
119 // instruction is logically regarded as a vector of i1 containing the
120 // same number of lanes as the input vector type. So our Predicate type
121 // comes with a lane count, which we use to decide which kind of <n x i1>
122 // we'll invoke the pred_i2v IR intrinsic to translate it into.
123 Predicate,
124
125 // Pointer is used for pointer types (obviously), and comes with a flag
126 // indicating whether it's a pointer to a const or mutable instance of
127 // the pointee type.
128 Pointer,
129 };
130
131 private:
132 const TypeKind TKind;
133
134 protected:
Type(TypeKind K)135 Type(TypeKind K) : TKind(K) {}
136
137 public:
typeKind() const138 TypeKind typeKind() const { return TKind; }
139 virtual ~Type() = default;
140 virtual bool requiresFloat() const = 0;
141 virtual unsigned sizeInBits() const = 0;
142 virtual std::string cName() const = 0;
llvmName() const143 virtual std::string llvmName() const {
144 PrintFatalError("no LLVM type name available for type " + cName());
145 }
acleSuffix(std::string) const146 virtual std::string acleSuffix(std::string) const {
147 PrintFatalError("no ACLE suffix available for this type");
148 }
149 };
150
151 enum class ScalarTypeKind { SignedInt, UnsignedInt, Float };
toLetter(ScalarTypeKind kind)152 inline std::string toLetter(ScalarTypeKind kind) {
153 switch (kind) {
154 case ScalarTypeKind::SignedInt:
155 return "s";
156 case ScalarTypeKind::UnsignedInt:
157 return "u";
158 case ScalarTypeKind::Float:
159 return "f";
160 }
161 llvm_unreachable("Unhandled ScalarTypeKind enum");
162 }
toCPrefix(ScalarTypeKind kind)163 inline std::string toCPrefix(ScalarTypeKind kind) {
164 switch (kind) {
165 case ScalarTypeKind::SignedInt:
166 return "int";
167 case ScalarTypeKind::UnsignedInt:
168 return "uint";
169 case ScalarTypeKind::Float:
170 return "float";
171 }
172 llvm_unreachable("Unhandled ScalarTypeKind enum");
173 }
174
175 class VoidType : public Type {
176 public:
VoidType()177 VoidType() : Type(TypeKind::Void) {}
sizeInBits() const178 unsigned sizeInBits() const override { return 0; }
requiresFloat() const179 bool requiresFloat() const override { return false; }
cName() const180 std::string cName() const override { return "void"; }
181
classof(const Type * T)182 static bool classof(const Type *T) { return T->typeKind() == TypeKind::Void; }
acleSuffix(std::string) const183 std::string acleSuffix(std::string) const override { return ""; }
184 };
185
186 class PointerType : public Type {
187 const Type *Pointee;
188 bool Const;
189
190 public:
PointerType(const Type * Pointee,bool Const)191 PointerType(const Type *Pointee, bool Const)
192 : Type(TypeKind::Pointer), Pointee(Pointee), Const(Const) {}
sizeInBits() const193 unsigned sizeInBits() const override { return 32; }
requiresFloat() const194 bool requiresFloat() const override { return Pointee->requiresFloat(); }
cName() const195 std::string cName() const override {
196 std::string Name = Pointee->cName();
197
198 // The syntax for a pointer in C is different when the pointee is
199 // itself a pointer. The MVE intrinsics don't contain any double
200 // pointers, so we don't need to worry about that wrinkle.
201 assert(!isa<PointerType>(Pointee) && "Pointer to pointer not supported");
202
203 if (Const)
204 Name = "const " + Name;
205 return Name + " *";
206 }
llvmName() const207 std::string llvmName() const override {
208 return "llvm::PointerType::getUnqual(" + Pointee->llvmName() + ")";
209 }
210
classof(const Type * T)211 static bool classof(const Type *T) {
212 return T->typeKind() == TypeKind::Pointer;
213 }
214 };
215
216 // Base class for all the types that have a name of the form
217 // [prefix][numbers]_t, like int32_t, uint16x8_t, float32x4x2_t.
218 //
219 // For this sub-hierarchy we invent a cNameBase() method which returns the
220 // whole name except for the trailing "_t", so that Vector and MultiVector can
221 // append an extra "x2" or whatever to their element type's cNameBase(). Then
222 // the main cName() query method puts "_t" on the end for the final type name.
223
224 class CRegularNamedType : public Type {
225 using Type::Type;
226 virtual std::string cNameBase() const = 0;
227
228 public:
cName() const229 std::string cName() const override { return cNameBase() + "_t"; }
230 };
231
232 class ScalarType : public CRegularNamedType {
233 ScalarTypeKind Kind;
234 unsigned Bits;
235 std::string NameOverride;
236
237 public:
ScalarType(const Record * Record)238 ScalarType(const Record *Record) : CRegularNamedType(TypeKind::Scalar) {
239 Kind = StringSwitch<ScalarTypeKind>(Record->getValueAsString("kind"))
240 .Case("s", ScalarTypeKind::SignedInt)
241 .Case("u", ScalarTypeKind::UnsignedInt)
242 .Case("f", ScalarTypeKind::Float);
243 Bits = Record->getValueAsInt("size");
244 NameOverride = Record->getValueAsString("nameOverride");
245 }
sizeInBits() const246 unsigned sizeInBits() const override { return Bits; }
kind() const247 ScalarTypeKind kind() const { return Kind; }
suffix() const248 std::string suffix() const { return toLetter(Kind) + utostr(Bits); }
cNameBase() const249 std::string cNameBase() const override {
250 return toCPrefix(Kind) + utostr(Bits);
251 }
cName() const252 std::string cName() const override {
253 if (NameOverride.empty())
254 return CRegularNamedType::cName();
255 return NameOverride;
256 }
llvmName() const257 std::string llvmName() const override {
258 if (Kind == ScalarTypeKind::Float) {
259 if (Bits == 16)
260 return "HalfTy";
261 if (Bits == 32)
262 return "FloatTy";
263 if (Bits == 64)
264 return "DoubleTy";
265 PrintFatalError("bad size for floating type");
266 }
267 return "Int" + utostr(Bits) + "Ty";
268 }
acleSuffix(std::string overrideLetter) const269 std::string acleSuffix(std::string overrideLetter) const override {
270 return "_" + (overrideLetter.size() ? overrideLetter : toLetter(Kind))
271 + utostr(Bits);
272 }
isInteger() const273 bool isInteger() const { return Kind != ScalarTypeKind::Float; }
requiresFloat() const274 bool requiresFloat() const override { return !isInteger(); }
hasNonstandardName() const275 bool hasNonstandardName() const { return !NameOverride.empty(); }
276
classof(const Type * T)277 static bool classof(const Type *T) {
278 return T->typeKind() == TypeKind::Scalar;
279 }
280 };
281
282 class VectorType : public CRegularNamedType {
283 const ScalarType *Element;
284 unsigned Lanes;
285
286 public:
VectorType(const ScalarType * Element,unsigned Lanes)287 VectorType(const ScalarType *Element, unsigned Lanes)
288 : CRegularNamedType(TypeKind::Vector), Element(Element), Lanes(Lanes) {}
sizeInBits() const289 unsigned sizeInBits() const override { return Lanes * Element->sizeInBits(); }
lanes() const290 unsigned lanes() const { return Lanes; }
requiresFloat() const291 bool requiresFloat() const override { return Element->requiresFloat(); }
cNameBase() const292 std::string cNameBase() const override {
293 return Element->cNameBase() + "x" + utostr(Lanes);
294 }
llvmName() const295 std::string llvmName() const override {
296 return "llvm::VectorType::get(" + Element->llvmName() + ", " +
297 utostr(Lanes) + ")";
298 }
299
classof(const Type * T)300 static bool classof(const Type *T) {
301 return T->typeKind() == TypeKind::Vector;
302 }
303 };
304
305 class MultiVectorType : public CRegularNamedType {
306 const VectorType *Element;
307 unsigned Registers;
308
309 public:
MultiVectorType(unsigned Registers,const VectorType * Element)310 MultiVectorType(unsigned Registers, const VectorType *Element)
311 : CRegularNamedType(TypeKind::MultiVector), Element(Element),
312 Registers(Registers) {}
sizeInBits() const313 unsigned sizeInBits() const override {
314 return Registers * Element->sizeInBits();
315 }
registers() const316 unsigned registers() const { return Registers; }
requiresFloat() const317 bool requiresFloat() const override { return Element->requiresFloat(); }
cNameBase() const318 std::string cNameBase() const override {
319 return Element->cNameBase() + "x" + utostr(Registers);
320 }
321
322 // MultiVectorType doesn't override llvmName, because we don't expect to do
323 // automatic code generation for the MVE intrinsics that use it: the {vld2,
324 // vld4, vst2, vst4} family are the only ones that use these types, so it was
325 // easier to hand-write the codegen for dealing with these structs than to
326 // build in lots of extra automatic machinery that would only be used once.
327
classof(const Type * T)328 static bool classof(const Type *T) {
329 return T->typeKind() == TypeKind::MultiVector;
330 }
331 };
332
333 class PredicateType : public CRegularNamedType {
334 unsigned Lanes;
335
336 public:
PredicateType(unsigned Lanes)337 PredicateType(unsigned Lanes)
338 : CRegularNamedType(TypeKind::Predicate), Lanes(Lanes) {}
sizeInBits() const339 unsigned sizeInBits() const override { return 16; }
cNameBase() const340 std::string cNameBase() const override { return "mve_pred16"; }
requiresFloat() const341 bool requiresFloat() const override { return false; };
llvmName() const342 std::string llvmName() const override {
343 // Use <4 x i1> instead of <2 x i1> for two-lane vector types. See
344 // the comment in llvm/lib/Target/ARM/ARMInstrMVE.td for further
345 // explanation.
346 unsigned ModifiedLanes = (Lanes == 2 ? 4 : Lanes);
347
348 return "llvm::VectorType::get(Builder.getInt1Ty(), " +
349 utostr(ModifiedLanes) + ")";
350 }
351
classof(const Type * T)352 static bool classof(const Type *T) {
353 return T->typeKind() == TypeKind::Predicate;
354 }
355 };
356
357 // -----------------------------------------------------------------------------
358 // Class to facilitate merging together the code generation for many intrinsics
359 // by means of varying a few constant or type parameters.
360 //
361 // Most obviously, the intrinsics in a single parametrised family will have
362 // code generation sequences that only differ in a type or two, e.g. vaddq_s8
363 // and vaddq_u16 will look the same apart from putting a different vector type
364 // in the call to CGM.getIntrinsic(). But also, completely different intrinsics
365 // will often code-generate in the same way, with only a different choice of
366 // _which_ IR intrinsic they lower to (e.g. vaddq_m_s8 and vmulq_m_s8), but
367 // marshalling the arguments and return values of the IR intrinsic in exactly
368 // the same way. And others might differ only in some other kind of constant,
369 // such as a lane index.
370 //
371 // So, when we generate the IR-building code for all these intrinsics, we keep
372 // track of every value that could possibly be pulled out of the code and
373 // stored ahead of time in a local variable. Then we group together intrinsics
374 // by textual equivalence of the code that would result if _all_ those
375 // parameters were stored in local variables. That gives us maximal sets that
376 // can be implemented by a single piece of IR-building code by changing
377 // parameter values ahead of time.
378 //
379 // After we've done that, we do a second pass in which we only allocate _some_
380 // of the parameters into local variables, by tracking which ones have the same
381 // values as each other (so that a single variable can be reused) and which
382 // ones are the same across the whole set (so that no variable is needed at
383 // all).
384 //
385 // Hence the class below. Its allocParam method is invoked during code
386 // generation by every method of a Result subclass (see below) that wants to
387 // give it the opportunity to pull something out into a switchable parameter.
388 // It returns a variable name for the parameter, or (if it's being used in the
389 // second pass once we've decided that some parameters don't need to be stored
390 // in variables after all) it might just return the input expression unchanged.
391
392 struct CodeGenParamAllocator {
393 // Accumulated during code generation
394 std::vector<std::string> *ParamTypes = nullptr;
395 std::vector<std::string> *ParamValues = nullptr;
396
397 // Provided ahead of time in pass 2, to indicate which parameters are being
398 // assigned to what. This vector contains an entry for each call to
399 // allocParam expected during code gen (which we counted up in pass 1), and
400 // indicates the number of the parameter variable that should be returned, or
401 // -1 if this call shouldn't allocate a parameter variable at all.
402 //
403 // We rely on the recursive code generation working identically in passes 1
404 // and 2, so that the same list of calls to allocParam happen in the same
405 // order. That guarantees that the parameter numbers recorded in pass 1 will
406 // match the entries in this vector that store what MveEmitter::EmitBuiltinCG
407 // decided to do about each one in pass 2.
408 std::vector<int> *ParamNumberMap = nullptr;
409
410 // Internally track how many things we've allocated
411 unsigned nparams = 0;
412
allocParam__anondc5b1bb70111::CodeGenParamAllocator413 std::string allocParam(StringRef Type, StringRef Value) {
414 unsigned ParamNumber;
415
416 if (!ParamNumberMap) {
417 // In pass 1, unconditionally assign a new parameter variable to every
418 // value we're asked to process.
419 ParamNumber = nparams++;
420 } else {
421 // In pass 2, consult the map provided by the caller to find out which
422 // variable we should be keeping things in.
423 int MapValue = (*ParamNumberMap)[nparams++];
424 if (MapValue < 0)
425 return Value;
426 ParamNumber = MapValue;
427 }
428
429 // If we've allocated a new parameter variable for the first time, store
430 // its type and value to be retrieved after codegen.
431 if (ParamTypes && ParamTypes->size() == ParamNumber)
432 ParamTypes->push_back(Type);
433 if (ParamValues && ParamValues->size() == ParamNumber)
434 ParamValues->push_back(Value);
435
436 // Unimaginative naming scheme for parameter variables.
437 return "Param" + utostr(ParamNumber);
438 }
439 };
440
441 // -----------------------------------------------------------------------------
442 // System of classes that represent all the intermediate values used during
443 // code-generation for an intrinsic.
444 //
445 // The base class 'Result' can represent a value of the LLVM type 'Value', or
446 // sometimes 'Address' (for loads/stores, including an alignment requirement).
447 //
448 // In the case where the Tablegen provides a value in the codegen dag as a
449 // plain integer literal, the Result object we construct here will be one that
450 // returns true from hasIntegerConstantValue(). This allows the generated C++
451 // code to use the constant directly in contexts which can take a literal
452 // integer, such as Builder.CreateExtractValue(thing, 1), without going to the
453 // effort of calling llvm::ConstantInt::get() and then pulling the constant
454 // back out of the resulting llvm:Value later.
455
456 class Result {
457 public:
458 // Convenient shorthand for the pointer type we'll be using everywhere.
459 using Ptr = std::shared_ptr<Result>;
460
461 private:
462 Ptr Predecessor;
463 std::string VarName;
464 bool VarNameUsed = false;
465 unsigned Visited = 0;
466
467 public:
468 virtual ~Result() = default;
469 using Scope = std::map<std::string, Ptr>;
470 virtual void genCode(raw_ostream &OS, CodeGenParamAllocator &) const = 0;
hasIntegerConstantValue() const471 virtual bool hasIntegerConstantValue() const { return false; }
integerConstantValue() const472 virtual uint32_t integerConstantValue() const { return 0; }
hasIntegerValue() const473 virtual bool hasIntegerValue() const { return false; }
getIntegerValue(const std::string &)474 virtual std::string getIntegerValue(const std::string &) {
475 llvm_unreachable("non-working Result::getIntegerValue called");
476 }
typeName() const477 virtual std::string typeName() const { return "Value *"; }
478
479 // Mostly, when a code-generation operation has a dependency on prior
480 // operations, it's because it uses the output values of those operations as
481 // inputs. But there's one exception, which is the use of 'seq' in Tablegen
482 // to indicate that operations have to be performed in sequence regardless of
483 // whether they use each others' output values.
484 //
485 // So, the actual generation of code is done by depth-first search, using the
486 // prerequisites() method to get a list of all the other Results that have to
487 // be computed before this one. That method divides into the 'predecessor',
488 // set by setPredecessor() while processing a 'seq' dag node, and the list
489 // returned by 'morePrerequisites', which each subclass implements to return
490 // a list of the Results it uses as input to whatever its own computation is
491 // doing.
492
morePrerequisites(std::vector<Ptr> & output) const493 virtual void morePrerequisites(std::vector<Ptr> &output) const {}
prerequisites() const494 std::vector<Ptr> prerequisites() const {
495 std::vector<Ptr> ToRet;
496 if (Predecessor)
497 ToRet.push_back(Predecessor);
498 morePrerequisites(ToRet);
499 return ToRet;
500 }
501
setPredecessor(Ptr p)502 void setPredecessor(Ptr p) {
503 assert(!Predecessor);
504 Predecessor = p;
505 }
506
507 // Each Result will be assigned a variable name in the output code, but not
508 // all those variable names will actually be used (e.g. the return value of
509 // Builder.CreateStore has void type, so nobody will want to refer to it). To
510 // prevent annoying compiler warnings, we track whether each Result's
511 // variable name was ever actually mentioned in subsequent statements, so
512 // that it can be left out of the final generated code.
varname()513 std::string varname() {
514 VarNameUsed = true;
515 return VarName;
516 }
setVarname(const StringRef s)517 void setVarname(const StringRef s) { VarName = s; }
varnameUsed() const518 bool varnameUsed() const { return VarNameUsed; }
519
520 // Emit code to generate this result as a Value *.
asValue()521 virtual std::string asValue() {
522 return varname();
523 }
524
525 // Code generation happens in multiple passes. This method tracks whether a
526 // Result has yet been visited in a given pass, without the need for a
527 // tedious loop in between passes that goes through and resets a 'visited'
528 // flag back to false: you just set Pass=1 the first time round, and Pass=2
529 // the second time.
needsVisiting(unsigned Pass)530 bool needsVisiting(unsigned Pass) {
531 bool ToRet = Visited < Pass;
532 Visited = Pass;
533 return ToRet;
534 }
535 };
536
537 // Result subclass that retrieves one of the arguments to the clang builtin
538 // function. In cases where the argument has pointer type, we call
539 // EmitPointerWithAlignment and store the result in a variable of type Address,
540 // so that load and store IR nodes can know the right alignment. Otherwise, we
541 // call EmitScalarExpr.
542 //
543 // There are aggregate parameters in the MVE intrinsics API, but we don't deal
544 // with them in this Tablegen back end: they only arise in the vld2q/vld4q and
545 // vst2q/vst4q family, which is few enough that we just write the code by hand
546 // for those in CGBuiltin.cpp.
547 class BuiltinArgResult : public Result {
548 public:
549 unsigned ArgNum;
550 bool AddressType;
551 bool Immediate;
BuiltinArgResult(unsigned ArgNum,bool AddressType,bool Immediate)552 BuiltinArgResult(unsigned ArgNum, bool AddressType, bool Immediate)
553 : ArgNum(ArgNum), AddressType(AddressType), Immediate(Immediate) {}
genCode(raw_ostream & OS,CodeGenParamAllocator &) const554 void genCode(raw_ostream &OS, CodeGenParamAllocator &) const override {
555 OS << (AddressType ? "EmitPointerWithAlignment" : "EmitScalarExpr")
556 << "(E->getArg(" << ArgNum << "))";
557 }
typeName() const558 std::string typeName() const override {
559 return AddressType ? "Address" : Result::typeName();
560 }
561 // Emit code to generate this result as a Value *.
asValue()562 std::string asValue() override {
563 if (AddressType)
564 return "(" + varname() + ".getPointer())";
565 return Result::asValue();
566 }
hasIntegerValue() const567 bool hasIntegerValue() const override { return Immediate; }
getIntegerValue(const std::string & IntType)568 std::string getIntegerValue(const std::string &IntType) override {
569 return "GetIntegerConstantValue<" + IntType + ">(E->getArg(" +
570 utostr(ArgNum) + "), getContext())";
571 }
572 };
573
574 // Result subclass for an integer literal appearing in Tablegen. This may need
575 // to be turned into an llvm::Result by means of llvm::ConstantInt::get(), or
576 // it may be used directly as an integer, depending on which IRBuilder method
577 // it's being passed to.
578 class IntLiteralResult : public Result {
579 public:
580 const ScalarType *IntegerType;
581 uint32_t IntegerValue;
IntLiteralResult(const ScalarType * IntegerType,uint32_t IntegerValue)582 IntLiteralResult(const ScalarType *IntegerType, uint32_t IntegerValue)
583 : IntegerType(IntegerType), IntegerValue(IntegerValue) {}
genCode(raw_ostream & OS,CodeGenParamAllocator & ParamAlloc) const584 void genCode(raw_ostream &OS,
585 CodeGenParamAllocator &ParamAlloc) const override {
586 OS << "llvm::ConstantInt::get("
587 << ParamAlloc.allocParam("llvm::Type *", IntegerType->llvmName())
588 << ", ";
589 OS << ParamAlloc.allocParam(IntegerType->cName(), utostr(IntegerValue))
590 << ")";
591 }
hasIntegerConstantValue() const592 bool hasIntegerConstantValue() const override { return true; }
integerConstantValue() const593 uint32_t integerConstantValue() const override { return IntegerValue; }
594 };
595
596 // Result subclass representing a cast between different integer types. We use
597 // our own ScalarType abstraction as the representation of the target type,
598 // which gives both size and signedness.
599 class IntCastResult : public Result {
600 public:
601 const ScalarType *IntegerType;
602 Ptr V;
IntCastResult(const ScalarType * IntegerType,Ptr V)603 IntCastResult(const ScalarType *IntegerType, Ptr V)
604 : IntegerType(IntegerType), V(V) {}
genCode(raw_ostream & OS,CodeGenParamAllocator & ParamAlloc) const605 void genCode(raw_ostream &OS,
606 CodeGenParamAllocator &ParamAlloc) const override {
607 OS << "Builder.CreateIntCast(" << V->varname() << ", "
608 << ParamAlloc.allocParam("llvm::Type *", IntegerType->llvmName()) << ", "
609 << ParamAlloc.allocParam("bool",
610 IntegerType->kind() == ScalarTypeKind::SignedInt
611 ? "true"
612 : "false")
613 << ")";
614 }
morePrerequisites(std::vector<Ptr> & output) const615 void morePrerequisites(std::vector<Ptr> &output) const override {
616 output.push_back(V);
617 }
618 };
619
620 // Result subclass representing a cast between different pointer types.
621 class PointerCastResult : public Result {
622 public:
623 const PointerType *PtrType;
624 Ptr V;
PointerCastResult(const PointerType * PtrType,Ptr V)625 PointerCastResult(const PointerType *PtrType, Ptr V)
626 : PtrType(PtrType), V(V) {}
genCode(raw_ostream & OS,CodeGenParamAllocator & ParamAlloc) const627 void genCode(raw_ostream &OS,
628 CodeGenParamAllocator &ParamAlloc) const override {
629 OS << "Builder.CreatePointerCast(" << V->asValue() << ", "
630 << ParamAlloc.allocParam("llvm::Type *", PtrType->llvmName()) << ")";
631 }
morePrerequisites(std::vector<Ptr> & output) const632 void morePrerequisites(std::vector<Ptr> &output) const override {
633 output.push_back(V);
634 }
635 };
636
637 // Result subclass representing a call to an IRBuilder method. Each IRBuilder
638 // method we want to use will have a Tablegen record giving the method name and
639 // describing any important details of how to call it, such as whether a
640 // particular argument should be an integer constant instead of an llvm::Value.
641 class IRBuilderResult : public Result {
642 public:
643 StringRef CallPrefix;
644 std::vector<Ptr> Args;
645 std::set<unsigned> AddressArgs;
646 std::map<unsigned, std::string> IntegerArgs;
IRBuilderResult(StringRef CallPrefix,std::vector<Ptr> Args,std::set<unsigned> AddressArgs,std::map<unsigned,std::string> IntegerArgs)647 IRBuilderResult(StringRef CallPrefix, std::vector<Ptr> Args,
648 std::set<unsigned> AddressArgs,
649 std::map<unsigned, std::string> IntegerArgs)
650 : CallPrefix(CallPrefix), Args(Args), AddressArgs(AddressArgs),
651 IntegerArgs(IntegerArgs) {}
genCode(raw_ostream & OS,CodeGenParamAllocator & ParamAlloc) const652 void genCode(raw_ostream &OS,
653 CodeGenParamAllocator &ParamAlloc) const override {
654 OS << CallPrefix;
655 const char *Sep = "";
656 for (unsigned i = 0, e = Args.size(); i < e; ++i) {
657 Ptr Arg = Args[i];
658 auto it = IntegerArgs.find(i);
659
660 OS << Sep;
661 Sep = ", ";
662
663 if (it != IntegerArgs.end()) {
664 if (Arg->hasIntegerConstantValue())
665 OS << "static_cast<" << it->second << ">("
666 << ParamAlloc.allocParam(it->second,
667 utostr(Arg->integerConstantValue()))
668 << ")";
669 else if (Arg->hasIntegerValue())
670 OS << ParamAlloc.allocParam(it->second,
671 Arg->getIntegerValue(it->second));
672 } else {
673 OS << Arg->varname();
674 }
675 }
676 OS << ")";
677 }
morePrerequisites(std::vector<Ptr> & output) const678 void morePrerequisites(std::vector<Ptr> &output) const override {
679 for (unsigned i = 0, e = Args.size(); i < e; ++i) {
680 Ptr Arg = Args[i];
681 if (IntegerArgs.find(i) != IntegerArgs.end())
682 continue;
683 output.push_back(Arg);
684 }
685 }
686 };
687
688 // Result subclass representing making an Address out of a Value.
689 class AddressResult : public Result {
690 public:
691 Ptr Arg;
692 unsigned Align;
AddressResult(Ptr Arg,unsigned Align)693 AddressResult(Ptr Arg, unsigned Align) : Arg(Arg), Align(Align) {}
genCode(raw_ostream & OS,CodeGenParamAllocator & ParamAlloc) const694 void genCode(raw_ostream &OS,
695 CodeGenParamAllocator &ParamAlloc) const override {
696 OS << "Address(" << Arg->varname() << ", CharUnits::fromQuantity("
697 << Align << "))";
698 }
typeName() const699 std::string typeName() const override {
700 return "Address";
701 }
morePrerequisites(std::vector<Ptr> & output) const702 void morePrerequisites(std::vector<Ptr> &output) const override {
703 output.push_back(Arg);
704 }
705 };
706
707 // Result subclass representing a call to an IR intrinsic, which we first have
708 // to look up using an Intrinsic::ID constant and an array of types.
709 class IRIntrinsicResult : public Result {
710 public:
711 std::string IntrinsicID;
712 std::vector<const Type *> ParamTypes;
713 std::vector<Ptr> Args;
IRIntrinsicResult(StringRef IntrinsicID,std::vector<const Type * > ParamTypes,std::vector<Ptr> Args)714 IRIntrinsicResult(StringRef IntrinsicID, std::vector<const Type *> ParamTypes,
715 std::vector<Ptr> Args)
716 : IntrinsicID(IntrinsicID), ParamTypes(ParamTypes), Args(Args) {}
genCode(raw_ostream & OS,CodeGenParamAllocator & ParamAlloc) const717 void genCode(raw_ostream &OS,
718 CodeGenParamAllocator &ParamAlloc) const override {
719 std::string IntNo = ParamAlloc.allocParam(
720 "Intrinsic::ID", "Intrinsic::" + IntrinsicID);
721 OS << "Builder.CreateCall(CGM.getIntrinsic(" << IntNo;
722 if (!ParamTypes.empty()) {
723 OS << ", llvm::SmallVector<llvm::Type *, " << ParamTypes.size() << "> {";
724 const char *Sep = "";
725 for (auto T : ParamTypes) {
726 OS << Sep << ParamAlloc.allocParam("llvm::Type *", T->llvmName());
727 Sep = ", ";
728 }
729 OS << "}";
730 }
731 OS << "), llvm::SmallVector<Value *, " << Args.size() << "> {";
732 const char *Sep = "";
733 for (auto Arg : Args) {
734 OS << Sep << Arg->asValue();
735 Sep = ", ";
736 }
737 OS << "})";
738 }
morePrerequisites(std::vector<Ptr> & output) const739 void morePrerequisites(std::vector<Ptr> &output) const override {
740 output.insert(output.end(), Args.begin(), Args.end());
741 }
742 };
743
744 // Result subclass that specifies a type, for use in IRBuilder operations such
745 // as CreateBitCast that take a type argument.
746 class TypeResult : public Result {
747 public:
748 const Type *T;
TypeResult(const Type * T)749 TypeResult(const Type *T) : T(T) {}
genCode(raw_ostream & OS,CodeGenParamAllocator &) const750 void genCode(raw_ostream &OS, CodeGenParamAllocator &) const override {
751 OS << T->llvmName();
752 }
typeName() const753 std::string typeName() const override {
754 return "llvm::Type *";
755 }
756 };
757
758 // -----------------------------------------------------------------------------
759 // Class that describes a single ACLE intrinsic.
760 //
761 // A Tablegen record will typically describe more than one ACLE intrinsic, by
762 // means of setting the 'list<Type> Params' field to a list of multiple
763 // parameter types, so as to define vaddq_{s8,u8,...,f16,f32} all in one go.
764 // We'll end up with one instance of ACLEIntrinsic for *each* parameter type,
765 // rather than a single one for all of them. Hence, the constructor takes both
766 // a Tablegen record and the current value of the parameter type.
767
768 class ACLEIntrinsic {
769 // Structure documenting that one of the intrinsic's arguments is required to
770 // be a compile-time constant integer, and what constraints there are on its
771 // value. Used when generating Sema checking code.
772 struct ImmediateArg {
773 enum class BoundsType { ExplicitRange, UInt };
774 BoundsType boundsType;
775 int64_t i1, i2;
776 StringRef ExtraCheckType, ExtraCheckArgs;
777 const Type *ArgType;
778 };
779
780 // For polymorphic intrinsics, FullName is the explicit name that uniquely
781 // identifies this variant of the intrinsic, and ShortName is the name it
782 // shares with at least one other intrinsic.
783 std::string ShortName, FullName;
784
785 // A very small number of intrinsics _only_ have a polymorphic
786 // variant (vuninitializedq taking an unevaluated argument).
787 bool PolymorphicOnly;
788
789 // Another rarely-used flag indicating that the builtin doesn't
790 // evaluate its argument(s) at all.
791 bool NonEvaluating;
792
793 const Type *ReturnType;
794 std::vector<const Type *> ArgTypes;
795 std::map<unsigned, ImmediateArg> ImmediateArgs;
796 Result::Ptr Code;
797
798 std::map<std::string, std::string> CustomCodeGenArgs;
799
800 // Recursive function that does the internals of code generation.
genCodeDfs(Result::Ptr V,std::list<Result::Ptr> & Used,unsigned Pass) const801 void genCodeDfs(Result::Ptr V, std::list<Result::Ptr> &Used,
802 unsigned Pass) const {
803 if (!V->needsVisiting(Pass))
804 return;
805
806 for (Result::Ptr W : V->prerequisites())
807 genCodeDfs(W, Used, Pass);
808
809 Used.push_back(V);
810 }
811
812 public:
shortName() const813 const std::string &shortName() const { return ShortName; }
fullName() const814 const std::string &fullName() const { return FullName; }
returnType() const815 const Type *returnType() const { return ReturnType; }
argTypes() const816 const std::vector<const Type *> &argTypes() const { return ArgTypes; }
requiresFloat() const817 bool requiresFloat() const {
818 if (ReturnType->requiresFloat())
819 return true;
820 for (const Type *T : ArgTypes)
821 if (T->requiresFloat())
822 return true;
823 return false;
824 }
polymorphic() const825 bool polymorphic() const { return ShortName != FullName; }
polymorphicOnly() const826 bool polymorphicOnly() const { return PolymorphicOnly; }
nonEvaluating() const827 bool nonEvaluating() const { return NonEvaluating; }
828
829 // External entry point for code generation, called from MveEmitter.
genCode(raw_ostream & OS,CodeGenParamAllocator & ParamAlloc,unsigned Pass) const830 void genCode(raw_ostream &OS, CodeGenParamAllocator &ParamAlloc,
831 unsigned Pass) const {
832 if (!hasCode()) {
833 for (auto kv : CustomCodeGenArgs)
834 OS << " " << kv.first << " = " << kv.second << ";\n";
835 OS << " break; // custom code gen\n";
836 return;
837 }
838 std::list<Result::Ptr> Used;
839 genCodeDfs(Code, Used, Pass);
840
841 unsigned varindex = 0;
842 for (Result::Ptr V : Used)
843 if (V->varnameUsed())
844 V->setVarname("Val" + utostr(varindex++));
845
846 for (Result::Ptr V : Used) {
847 OS << " ";
848 if (V == Used.back()) {
849 assert(!V->varnameUsed());
850 OS << "return "; // FIXME: what if the top-level thing is void?
851 } else if (V->varnameUsed()) {
852 std::string Type = V->typeName();
853 OS << V->typeName();
854 if (!StringRef(Type).endswith("*"))
855 OS << " ";
856 OS << V->varname() << " = ";
857 }
858 V->genCode(OS, ParamAlloc);
859 OS << ";\n";
860 }
861 }
hasCode() const862 bool hasCode() const { return Code != nullptr; }
863
signedHexLiteral(const llvm::APInt & iOrig)864 static std::string signedHexLiteral(const llvm::APInt &iOrig) {
865 llvm::APInt i = iOrig.trunc(64);
866 SmallString<40> s;
867 i.toString(s, 16, true, true);
868 return s.str();
869 }
870
genSema() const871 std::string genSema() const {
872 std::vector<std::string> SemaChecks;
873
874 for (const auto &kv : ImmediateArgs) {
875 const ImmediateArg &IA = kv.second;
876
877 llvm::APInt lo(128, 0), hi(128, 0);
878 switch (IA.boundsType) {
879 case ImmediateArg::BoundsType::ExplicitRange:
880 lo = IA.i1;
881 hi = IA.i2;
882 break;
883 case ImmediateArg::BoundsType::UInt:
884 lo = 0;
885 hi = IA.i1;
886 break;
887 }
888
889 llvm::APInt typelo, typehi;
890 unsigned Bits = IA.ArgType->sizeInBits();
891 if (cast<ScalarType>(IA.ArgType)->kind() == ScalarTypeKind::SignedInt) {
892 typelo = llvm::APInt::getSignedMinValue(Bits).sext(128);
893 typehi = llvm::APInt::getSignedMaxValue(Bits).sext(128);
894 } else {
895 typelo = llvm::APInt::getMinValue(Bits).zext(128);
896 typehi = llvm::APInt::getMaxValue(Bits).zext(128);
897 }
898
899 std::string Index = utostr(kv.first);
900
901 if (lo.sle(typelo) && hi.sge(typehi))
902 SemaChecks.push_back("SemaBuiltinConstantArg(TheCall, " + Index + ")");
903 else
904 SemaChecks.push_back("SemaBuiltinConstantArgRange(TheCall, " + Index +
905 ", " + signedHexLiteral(lo) + ", " +
906 signedHexLiteral(hi) + ")");
907
908 if (!IA.ExtraCheckType.empty()) {
909 std::string Suffix;
910 if (!IA.ExtraCheckArgs.empty())
911 Suffix = (Twine(", ") + IA.ExtraCheckArgs).str();
912 SemaChecks.push_back((Twine("SemaBuiltinConstantArg") +
913 IA.ExtraCheckType + "(TheCall, " + Index +
914 Suffix + ")")
915 .str());
916 }
917 }
918 if (SemaChecks.empty())
919 return "";
920 return (Twine(" return ") +
921 join(std::begin(SemaChecks), std::end(SemaChecks),
922 " ||\n ") +
923 ";\n")
924 .str();
925 }
926
927 ACLEIntrinsic(MveEmitter &ME, Record *R, const Type *Param);
928 };
929
930 // -----------------------------------------------------------------------------
931 // The top-level class that holds all the state from analyzing the entire
932 // Tablegen input.
933
934 class MveEmitter {
935 // MveEmitter holds a collection of all the types we've instantiated.
936 VoidType Void;
937 std::map<std::string, std::unique_ptr<ScalarType>> ScalarTypes;
938 std::map<std::tuple<ScalarTypeKind, unsigned, unsigned>,
939 std::unique_ptr<VectorType>>
940 VectorTypes;
941 std::map<std::pair<std::string, unsigned>, std::unique_ptr<MultiVectorType>>
942 MultiVectorTypes;
943 std::map<unsigned, std::unique_ptr<PredicateType>> PredicateTypes;
944 std::map<std::string, std::unique_ptr<PointerType>> PointerTypes;
945
946 // And all the ACLEIntrinsic instances we've created.
947 std::map<std::string, std::unique_ptr<ACLEIntrinsic>> ACLEIntrinsics;
948
949 public:
950 // Methods to create a Type object, or return the right existing one from the
951 // maps stored in this object.
getVoidType()952 const VoidType *getVoidType() { return &Void; }
getScalarType(StringRef Name)953 const ScalarType *getScalarType(StringRef Name) {
954 return ScalarTypes[Name].get();
955 }
getScalarType(Record * R)956 const ScalarType *getScalarType(Record *R) {
957 return getScalarType(R->getName());
958 }
getVectorType(const ScalarType * ST,unsigned Lanes)959 const VectorType *getVectorType(const ScalarType *ST, unsigned Lanes) {
960 std::tuple<ScalarTypeKind, unsigned, unsigned> key(ST->kind(),
961 ST->sizeInBits(), Lanes);
962 if (VectorTypes.find(key) == VectorTypes.end())
963 VectorTypes[key] = std::make_unique<VectorType>(ST, Lanes);
964 return VectorTypes[key].get();
965 }
getVectorType(const ScalarType * ST)966 const VectorType *getVectorType(const ScalarType *ST) {
967 return getVectorType(ST, 128 / ST->sizeInBits());
968 }
getMultiVectorType(unsigned Registers,const VectorType * VT)969 const MultiVectorType *getMultiVectorType(unsigned Registers,
970 const VectorType *VT) {
971 std::pair<std::string, unsigned> key(VT->cNameBase(), Registers);
972 if (MultiVectorTypes.find(key) == MultiVectorTypes.end())
973 MultiVectorTypes[key] = std::make_unique<MultiVectorType>(Registers, VT);
974 return MultiVectorTypes[key].get();
975 }
getPredicateType(unsigned Lanes)976 const PredicateType *getPredicateType(unsigned Lanes) {
977 unsigned key = Lanes;
978 if (PredicateTypes.find(key) == PredicateTypes.end())
979 PredicateTypes[key] = std::make_unique<PredicateType>(Lanes);
980 return PredicateTypes[key].get();
981 }
getPointerType(const Type * T,bool Const)982 const PointerType *getPointerType(const Type *T, bool Const) {
983 PointerType PT(T, Const);
984 std::string key = PT.cName();
985 if (PointerTypes.find(key) == PointerTypes.end())
986 PointerTypes[key] = std::make_unique<PointerType>(PT);
987 return PointerTypes[key].get();
988 }
989
990 // Methods to construct a type from various pieces of Tablegen. These are
991 // always called in the context of setting up a particular ACLEIntrinsic, so
992 // there's always an ambient parameter type (because we're iterating through
993 // the Params list in the Tablegen record for the intrinsic), which is used
994 // to expand Tablegen classes like 'Vector' which mean something different in
995 // each member of a parametric family.
996 const Type *getType(Record *R, const Type *Param);
997 const Type *getType(DagInit *D, const Type *Param);
998 const Type *getType(Init *I, const Type *Param);
999
1000 // Functions that translate the Tablegen representation of an intrinsic's
1001 // code generation into a collection of Value objects (which will then be
1002 // reprocessed to read out the actual C++ code included by CGBuiltin.cpp).
1003 Result::Ptr getCodeForDag(DagInit *D, const Result::Scope &Scope,
1004 const Type *Param);
1005 Result::Ptr getCodeForDagArg(DagInit *D, unsigned ArgNum,
1006 const Result::Scope &Scope, const Type *Param);
1007 Result::Ptr getCodeForArg(unsigned ArgNum, const Type *ArgType, bool Promote,
1008 bool Immediate);
1009
1010 // Constructor and top-level functions.
1011
1012 MveEmitter(RecordKeeper &Records);
1013
1014 void EmitHeader(raw_ostream &OS);
1015 void EmitBuiltinDef(raw_ostream &OS);
1016 void EmitBuiltinSema(raw_ostream &OS);
1017 void EmitBuiltinCG(raw_ostream &OS);
1018 void EmitBuiltinAliases(raw_ostream &OS);
1019 };
1020
getType(Init * I,const Type * Param)1021 const Type *MveEmitter::getType(Init *I, const Type *Param) {
1022 if (auto Dag = dyn_cast<DagInit>(I))
1023 return getType(Dag, Param);
1024 if (auto Def = dyn_cast<DefInit>(I))
1025 return getType(Def->getDef(), Param);
1026
1027 PrintFatalError("Could not convert this value into a type");
1028 }
1029
getType(Record * R,const Type * Param)1030 const Type *MveEmitter::getType(Record *R, const Type *Param) {
1031 // Pass to a subfield of any wrapper records. We don't expect more than one
1032 // of these: immediate operands are used as plain numbers rather than as
1033 // llvm::Value, so it's meaningless to promote their type anyway.
1034 if (R->isSubClassOf("Immediate"))
1035 R = R->getValueAsDef("type");
1036 else if (R->isSubClassOf("unpromoted"))
1037 R = R->getValueAsDef("underlying_type");
1038
1039 if (R->getName() == "Void")
1040 return getVoidType();
1041 if (R->isSubClassOf("PrimitiveType"))
1042 return getScalarType(R);
1043 if (R->isSubClassOf("ComplexType"))
1044 return getType(R->getValueAsDag("spec"), Param);
1045
1046 PrintFatalError(R->getLoc(), "Could not convert this record into a type");
1047 }
1048
getType(DagInit * D,const Type * Param)1049 const Type *MveEmitter::getType(DagInit *D, const Type *Param) {
1050 // The meat of the getType system: types in the Tablegen are represented by a
1051 // dag whose operators select sub-cases of this function.
1052
1053 Record *Op = cast<DefInit>(D->getOperator())->getDef();
1054 if (!Op->isSubClassOf("ComplexTypeOp"))
1055 PrintFatalError(
1056 "Expected ComplexTypeOp as dag operator in type expression");
1057
1058 if (Op->getName() == "CTO_Parameter") {
1059 if (isa<VoidType>(Param))
1060 PrintFatalError("Parametric type in unparametrised context");
1061 return Param;
1062 }
1063
1064 if (Op->getName() == "CTO_Vec") {
1065 const Type *Element = getType(D->getArg(0), Param);
1066 if (D->getNumArgs() == 1) {
1067 return getVectorType(cast<ScalarType>(Element));
1068 } else {
1069 const Type *ExistingVector = getType(D->getArg(1), Param);
1070 return getVectorType(cast<ScalarType>(Element),
1071 cast<VectorType>(ExistingVector)->lanes());
1072 }
1073 }
1074
1075 if (Op->getName() == "CTO_Pred") {
1076 const Type *Element = getType(D->getArg(0), Param);
1077 return getPredicateType(128 / Element->sizeInBits());
1078 }
1079
1080 if (Op->isSubClassOf("CTO_Tuple")) {
1081 unsigned Registers = Op->getValueAsInt("n");
1082 const Type *Element = getType(D->getArg(0), Param);
1083 return getMultiVectorType(Registers, cast<VectorType>(Element));
1084 }
1085
1086 if (Op->isSubClassOf("CTO_Pointer")) {
1087 const Type *Pointee = getType(D->getArg(0), Param);
1088 return getPointerType(Pointee, Op->getValueAsBit("const"));
1089 }
1090
1091 if (Op->getName() == "CTO_CopyKind") {
1092 const ScalarType *STSize = cast<ScalarType>(getType(D->getArg(0), Param));
1093 const ScalarType *STKind = cast<ScalarType>(getType(D->getArg(1), Param));
1094 for (const auto &kv : ScalarTypes) {
1095 const ScalarType *RT = kv.second.get();
1096 if (RT->kind() == STKind->kind() && RT->sizeInBits() == STSize->sizeInBits())
1097 return RT;
1098 }
1099 PrintFatalError("Cannot find a type to satisfy CopyKind");
1100 }
1101
1102 if (Op->isSubClassOf("CTO_ScaleSize")) {
1103 const ScalarType *STKind = cast<ScalarType>(getType(D->getArg(0), Param));
1104 int Num = Op->getValueAsInt("num"), Denom = Op->getValueAsInt("denom");
1105 unsigned DesiredSize = STKind->sizeInBits() * Num / Denom;
1106 for (const auto &kv : ScalarTypes) {
1107 const ScalarType *RT = kv.second.get();
1108 if (RT->kind() == STKind->kind() && RT->sizeInBits() == DesiredSize)
1109 return RT;
1110 }
1111 PrintFatalError("Cannot find a type to satisfy ScaleSize");
1112 }
1113
1114 PrintFatalError("Bad operator in type dag expression");
1115 }
1116
getCodeForDag(DagInit * D,const Result::Scope & Scope,const Type * Param)1117 Result::Ptr MveEmitter::getCodeForDag(DagInit *D, const Result::Scope &Scope,
1118 const Type *Param) {
1119 Record *Op = cast<DefInit>(D->getOperator())->getDef();
1120
1121 if (Op->getName() == "seq") {
1122 Result::Scope SubScope = Scope;
1123 Result::Ptr PrevV = nullptr;
1124 for (unsigned i = 0, e = D->getNumArgs(); i < e; ++i) {
1125 // We don't use getCodeForDagArg here, because the argument name
1126 // has different semantics in a seq
1127 Result::Ptr V =
1128 getCodeForDag(cast<DagInit>(D->getArg(i)), SubScope, Param);
1129 StringRef ArgName = D->getArgNameStr(i);
1130 if (!ArgName.empty())
1131 SubScope[ArgName] = V;
1132 if (PrevV)
1133 V->setPredecessor(PrevV);
1134 PrevV = V;
1135 }
1136 return PrevV;
1137 } else if (Op->isSubClassOf("Type")) {
1138 if (D->getNumArgs() != 1)
1139 PrintFatalError("Type casts should have exactly one argument");
1140 const Type *CastType = getType(Op, Param);
1141 Result::Ptr Arg = getCodeForDagArg(D, 0, Scope, Param);
1142 if (const auto *ST = dyn_cast<ScalarType>(CastType)) {
1143 if (!ST->requiresFloat()) {
1144 if (Arg->hasIntegerConstantValue())
1145 return std::make_shared<IntLiteralResult>(
1146 ST, Arg->integerConstantValue());
1147 else
1148 return std::make_shared<IntCastResult>(ST, Arg);
1149 }
1150 } else if (const auto *PT = dyn_cast<PointerType>(CastType)) {
1151 return std::make_shared<PointerCastResult>(PT, Arg);
1152 }
1153 PrintFatalError("Unsupported type cast");
1154 } else if (Op->getName() == "address") {
1155 if (D->getNumArgs() != 2)
1156 PrintFatalError("'address' should have two arguments");
1157 Result::Ptr Arg = getCodeForDagArg(D, 0, Scope, Param);
1158 unsigned Alignment;
1159 if (auto *II = dyn_cast<IntInit>(D->getArg(1))) {
1160 Alignment = II->getValue();
1161 } else {
1162 PrintFatalError("'address' alignment argument should be an integer");
1163 }
1164 return std::make_shared<AddressResult>(Arg, Alignment);
1165 } else if (Op->getName() == "unsignedflag") {
1166 if (D->getNumArgs() != 1)
1167 PrintFatalError("unsignedflag should have exactly one argument");
1168 Record *TypeRec = cast<DefInit>(D->getArg(0))->getDef();
1169 if (!TypeRec->isSubClassOf("Type"))
1170 PrintFatalError("unsignedflag's argument should be a type");
1171 if (const auto *ST = dyn_cast<ScalarType>(getType(TypeRec, Param))) {
1172 return std::make_shared<IntLiteralResult>(
1173 getScalarType("u32"), ST->kind() == ScalarTypeKind::UnsignedInt);
1174 } else {
1175 PrintFatalError("unsignedflag's argument should be a scalar type");
1176 }
1177 } else {
1178 std::vector<Result::Ptr> Args;
1179 for (unsigned i = 0, e = D->getNumArgs(); i < e; ++i)
1180 Args.push_back(getCodeForDagArg(D, i, Scope, Param));
1181 if (Op->isSubClassOf("IRBuilderBase")) {
1182 std::set<unsigned> AddressArgs;
1183 std::map<unsigned, std::string> IntegerArgs;
1184 for (Record *sp : Op->getValueAsListOfDefs("special_params")) {
1185 unsigned Index = sp->getValueAsInt("index");
1186 if (sp->isSubClassOf("IRBuilderAddrParam")) {
1187 AddressArgs.insert(Index);
1188 } else if (sp->isSubClassOf("IRBuilderIntParam")) {
1189 IntegerArgs[Index] = sp->getValueAsString("type");
1190 }
1191 }
1192 return std::make_shared<IRBuilderResult>(Op->getValueAsString("prefix"),
1193 Args, AddressArgs, IntegerArgs);
1194 } else if (Op->isSubClassOf("IRIntBase")) {
1195 std::vector<const Type *> ParamTypes;
1196 for (Record *RParam : Op->getValueAsListOfDefs("params"))
1197 ParamTypes.push_back(getType(RParam, Param));
1198 std::string IntName = Op->getValueAsString("intname");
1199 if (Op->getValueAsBit("appendKind"))
1200 IntName += "_" + toLetter(cast<ScalarType>(Param)->kind());
1201 return std::make_shared<IRIntrinsicResult>(IntName, ParamTypes, Args);
1202 } else {
1203 PrintFatalError("Unsupported dag node " + Op->getName());
1204 }
1205 }
1206 }
1207
getCodeForDagArg(DagInit * D,unsigned ArgNum,const Result::Scope & Scope,const Type * Param)1208 Result::Ptr MveEmitter::getCodeForDagArg(DagInit *D, unsigned ArgNum,
1209 const Result::Scope &Scope,
1210 const Type *Param) {
1211 Init *Arg = D->getArg(ArgNum);
1212 StringRef Name = D->getArgNameStr(ArgNum);
1213
1214 if (!Name.empty()) {
1215 if (!isa<UnsetInit>(Arg))
1216 PrintFatalError(
1217 "dag operator argument should not have both a value and a name");
1218 auto it = Scope.find(Name);
1219 if (it == Scope.end())
1220 PrintFatalError("unrecognized variable name '" + Name + "'");
1221 return it->second;
1222 }
1223
1224 if (auto *II = dyn_cast<IntInit>(Arg))
1225 return std::make_shared<IntLiteralResult>(getScalarType("u32"),
1226 II->getValue());
1227
1228 if (auto *DI = dyn_cast<DagInit>(Arg))
1229 return getCodeForDag(DI, Scope, Param);
1230
1231 if (auto *DI = dyn_cast<DefInit>(Arg)) {
1232 Record *Rec = DI->getDef();
1233 if (Rec->isSubClassOf("Type")) {
1234 const Type *T = getType(Rec, Param);
1235 return std::make_shared<TypeResult>(T);
1236 }
1237 }
1238
1239 PrintFatalError("bad dag argument type for code generation");
1240 }
1241
getCodeForArg(unsigned ArgNum,const Type * ArgType,bool Promote,bool Immediate)1242 Result::Ptr MveEmitter::getCodeForArg(unsigned ArgNum, const Type *ArgType,
1243 bool Promote, bool Immediate) {
1244 Result::Ptr V = std::make_shared<BuiltinArgResult>(
1245 ArgNum, isa<PointerType>(ArgType), Immediate);
1246
1247 if (Promote) {
1248 if (const auto *ST = dyn_cast<ScalarType>(ArgType)) {
1249 if (ST->isInteger() && ST->sizeInBits() < 32)
1250 V = std::make_shared<IntCastResult>(getScalarType("u32"), V);
1251 } else if (const auto *PT = dyn_cast<PredicateType>(ArgType)) {
1252 V = std::make_shared<IntCastResult>(getScalarType("u32"), V);
1253 V = std::make_shared<IRIntrinsicResult>("arm_mve_pred_i2v",
1254 std::vector<const Type *>{PT},
1255 std::vector<Result::Ptr>{V});
1256 }
1257 }
1258
1259 return V;
1260 }
1261
ACLEIntrinsic(MveEmitter & ME,Record * R,const Type * Param)1262 ACLEIntrinsic::ACLEIntrinsic(MveEmitter &ME, Record *R, const Type *Param)
1263 : ReturnType(ME.getType(R->getValueAsDef("ret"), Param)) {
1264 // Derive the intrinsic's full name, by taking the name of the
1265 // Tablegen record (or override) and appending the suffix from its
1266 // parameter type. (If the intrinsic is unparametrised, its
1267 // parameter type will be given as Void, which returns the empty
1268 // string for acleSuffix.)
1269 StringRef BaseName =
1270 (R->isSubClassOf("NameOverride") ? R->getValueAsString("basename")
1271 : R->getName());
1272 StringRef overrideLetter = R->getValueAsString("overrideKindLetter");
1273 FullName = (Twine(BaseName) + Param->acleSuffix(overrideLetter)).str();
1274
1275 // Derive the intrinsic's polymorphic name, by removing components from the
1276 // full name as specified by its 'pnt' member ('polymorphic name type'),
1277 // which indicates how many type suffixes to remove, and any other piece of
1278 // the name that should be removed.
1279 Record *PolymorphicNameType = R->getValueAsDef("pnt");
1280 SmallVector<StringRef, 8> NameParts;
1281 StringRef(FullName).split(NameParts, '_');
1282 for (unsigned i = 0, e = PolymorphicNameType->getValueAsInt(
1283 "NumTypeSuffixesToDiscard");
1284 i < e; ++i)
1285 NameParts.pop_back();
1286 if (!PolymorphicNameType->isValueUnset("ExtraSuffixToDiscard")) {
1287 StringRef ExtraSuffix =
1288 PolymorphicNameType->getValueAsString("ExtraSuffixToDiscard");
1289 auto it = NameParts.end();
1290 while (it != NameParts.begin()) {
1291 --it;
1292 if (*it == ExtraSuffix) {
1293 NameParts.erase(it);
1294 break;
1295 }
1296 }
1297 }
1298 ShortName = join(std::begin(NameParts), std::end(NameParts), "_");
1299
1300 PolymorphicOnly = R->getValueAsBit("polymorphicOnly");
1301 NonEvaluating = R->getValueAsBit("nonEvaluating");
1302
1303 // Process the intrinsic's argument list.
1304 DagInit *ArgsDag = R->getValueAsDag("args");
1305 Result::Scope Scope;
1306 for (unsigned i = 0, e = ArgsDag->getNumArgs(); i < e; ++i) {
1307 Init *TypeInit = ArgsDag->getArg(i);
1308
1309 bool Promote = true;
1310 if (auto TypeDI = dyn_cast<DefInit>(TypeInit))
1311 if (TypeDI->getDef()->isSubClassOf("unpromoted"))
1312 Promote = false;
1313
1314 // Work out the type of the argument, for use in the function prototype in
1315 // the header file.
1316 const Type *ArgType = ME.getType(TypeInit, Param);
1317 ArgTypes.push_back(ArgType);
1318
1319 // If the argument is a subclass of Immediate, record the details about
1320 // what values it can take, for Sema checking.
1321 bool Immediate = false;
1322 if (auto TypeDI = dyn_cast<DefInit>(TypeInit)) {
1323 Record *TypeRec = TypeDI->getDef();
1324 if (TypeRec->isSubClassOf("Immediate")) {
1325 Immediate = true;
1326
1327 Record *Bounds = TypeRec->getValueAsDef("bounds");
1328 ImmediateArg &IA = ImmediateArgs[i];
1329 if (Bounds->isSubClassOf("IB_ConstRange")) {
1330 IA.boundsType = ImmediateArg::BoundsType::ExplicitRange;
1331 IA.i1 = Bounds->getValueAsInt("lo");
1332 IA.i2 = Bounds->getValueAsInt("hi");
1333 } else if (Bounds->getName() == "IB_UEltValue") {
1334 IA.boundsType = ImmediateArg::BoundsType::UInt;
1335 IA.i1 = Param->sizeInBits();
1336 } else if (Bounds->getName() == "IB_LaneIndex") {
1337 IA.boundsType = ImmediateArg::BoundsType::ExplicitRange;
1338 IA.i1 = 0;
1339 IA.i2 = 128 / Param->sizeInBits() - 1;
1340 } else if (Bounds->isSubClassOf("IB_EltBit")) {
1341 IA.boundsType = ImmediateArg::BoundsType::ExplicitRange;
1342 IA.i1 = Bounds->getValueAsInt("base");
1343 const Type *T = ME.getType(Bounds->getValueAsDef("type"), Param);
1344 IA.i2 = IA.i1 + T->sizeInBits() - 1;
1345 } else {
1346 PrintFatalError("unrecognised ImmediateBounds subclass");
1347 }
1348
1349 IA.ArgType = ArgType;
1350
1351 if (!TypeRec->isValueUnset("extra")) {
1352 IA.ExtraCheckType = TypeRec->getValueAsString("extra");
1353 if (!TypeRec->isValueUnset("extraarg"))
1354 IA.ExtraCheckArgs = TypeRec->getValueAsString("extraarg");
1355 }
1356 }
1357 }
1358
1359 // The argument will usually have a name in the arguments dag, which goes
1360 // into the variable-name scope that the code gen will refer to.
1361 StringRef ArgName = ArgsDag->getArgNameStr(i);
1362 if (!ArgName.empty())
1363 Scope[ArgName] = ME.getCodeForArg(i, ArgType, Promote, Immediate);
1364 }
1365
1366 // Finally, go through the codegen dag and translate it into a Result object
1367 // (with an arbitrary DAG of depended-on Results hanging off it).
1368 DagInit *CodeDag = R->getValueAsDag("codegen");
1369 Record *MainOp = cast<DefInit>(CodeDag->getOperator())->getDef();
1370 if (MainOp->isSubClassOf("CustomCodegen")) {
1371 // Or, if it's the special case of CustomCodegen, just accumulate
1372 // a list of parameters we're going to assign to variables before
1373 // breaking from the loop.
1374 CustomCodeGenArgs["CustomCodeGenType"] =
1375 (Twine("CustomCodeGen::") + MainOp->getValueAsString("type")).str();
1376 for (unsigned i = 0, e = CodeDag->getNumArgs(); i < e; ++i) {
1377 StringRef Name = CodeDag->getArgNameStr(i);
1378 if (Name.empty()) {
1379 PrintFatalError("Operands to CustomCodegen should have names");
1380 } else if (auto *II = dyn_cast<IntInit>(CodeDag->getArg(i))) {
1381 CustomCodeGenArgs[Name] = itostr(II->getValue());
1382 } else if (auto *SI = dyn_cast<StringInit>(CodeDag->getArg(i))) {
1383 CustomCodeGenArgs[Name] = SI->getValue();
1384 } else {
1385 PrintFatalError("Operands to CustomCodegen should be integers");
1386 }
1387 }
1388 } else {
1389 Code = ME.getCodeForDag(CodeDag, Scope, Param);
1390 }
1391 }
1392
MveEmitter(RecordKeeper & Records)1393 MveEmitter::MveEmitter(RecordKeeper &Records) {
1394 // Construct the whole MveEmitter.
1395
1396 // First, look up all the instances of PrimitiveType. This gives us the list
1397 // of vector typedefs we have to put in arm_mve.h, and also allows us to
1398 // collect all the useful ScalarType instances into a big list so that we can
1399 // use it for operations such as 'find the unsigned version of this signed
1400 // integer type'.
1401 for (Record *R : Records.getAllDerivedDefinitions("PrimitiveType"))
1402 ScalarTypes[R->getName()] = std::make_unique<ScalarType>(R);
1403
1404 // Now go through the instances of Intrinsic, and for each one, iterate
1405 // through its list of type parameters making an ACLEIntrinsic for each one.
1406 for (Record *R : Records.getAllDerivedDefinitions("Intrinsic")) {
1407 for (Record *RParam : R->getValueAsListOfDefs("params")) {
1408 const Type *Param = getType(RParam, getVoidType());
1409 auto Intrinsic = std::make_unique<ACLEIntrinsic>(*this, R, Param);
1410 ACLEIntrinsics[Intrinsic->fullName()] = std::move(Intrinsic);
1411 }
1412 }
1413 }
1414
1415 /// A wrapper on raw_string_ostream that contains its own buffer rather than
1416 /// having to point it at one elsewhere. (In other words, it works just like
1417 /// std::ostringstream; also, this makes it convenient to declare a whole array
1418 /// of them at once.)
1419 ///
1420 /// We have to set this up using multiple inheritance, to ensure that the
1421 /// string member has been constructed before raw_string_ostream's constructor
1422 /// is given a pointer to it.
1423 class string_holder {
1424 protected:
1425 std::string S;
1426 };
1427 class raw_self_contained_string_ostream : private string_holder,
1428 public raw_string_ostream {
1429 public:
raw_self_contained_string_ostream()1430 raw_self_contained_string_ostream()
1431 : string_holder(), raw_string_ostream(S) {}
1432 };
1433
EmitHeader(raw_ostream & OS)1434 void MveEmitter::EmitHeader(raw_ostream &OS) {
1435 // Accumulate pieces of the header file that will be enabled under various
1436 // different combinations of #ifdef. The index into parts[] is made up of
1437 // the following bit flags.
1438 constexpr unsigned Float = 1;
1439 constexpr unsigned UseUserNamespace = 2;
1440
1441 constexpr unsigned NumParts = 4;
1442 raw_self_contained_string_ostream parts[NumParts];
1443
1444 // Write typedefs for all the required vector types, and a few scalar
1445 // types that don't already have the name we want them to have.
1446
1447 parts[0] << "typedef uint16_t mve_pred16_t;\n";
1448 parts[Float] << "typedef __fp16 float16_t;\n"
1449 "typedef float float32_t;\n";
1450 for (const auto &kv : ScalarTypes) {
1451 const ScalarType *ST = kv.second.get();
1452 if (ST->hasNonstandardName())
1453 continue;
1454 raw_ostream &OS = parts[ST->requiresFloat() ? Float : 0];
1455 const VectorType *VT = getVectorType(ST);
1456
1457 OS << "typedef __attribute__((neon_vector_type(" << VT->lanes() << "))) "
1458 << ST->cName() << " " << VT->cName() << ";\n";
1459
1460 // Every vector type also comes with a pair of multi-vector types for
1461 // the VLD2 and VLD4 instructions.
1462 for (unsigned n = 2; n <= 4; n += 2) {
1463 const MultiVectorType *MT = getMultiVectorType(n, VT);
1464 OS << "typedef struct { " << VT->cName() << " val[" << n << "]; } "
1465 << MT->cName() << ";\n";
1466 }
1467 }
1468 parts[0] << "\n";
1469 parts[Float] << "\n";
1470
1471 // Write declarations for all the intrinsics.
1472
1473 for (const auto &kv : ACLEIntrinsics) {
1474 const ACLEIntrinsic &Int = *kv.second;
1475
1476 // We generate each intrinsic twice, under its full unambiguous
1477 // name and its shorter polymorphic name (if the latter exists).
1478 for (bool Polymorphic : {false, true}) {
1479 if (Polymorphic && !Int.polymorphic())
1480 continue;
1481 if (!Polymorphic && Int.polymorphicOnly())
1482 continue;
1483
1484 // We also generate each intrinsic under a name like __arm_vfooq
1485 // (which is in C language implementation namespace, so it's
1486 // safe to define in any conforming user program) and a shorter
1487 // one like vfooq (which is in user namespace, so a user might
1488 // reasonably have used it for something already). If so, they
1489 // can #define __ARM_MVE_PRESERVE_USER_NAMESPACE before
1490 // including the header, which will suppress the shorter names
1491 // and leave only the implementation-namespace ones. Then they
1492 // have to write __arm_vfooq everywhere, of course.
1493
1494 for (bool UserNamespace : {false, true}) {
1495 raw_ostream &OS = parts[(Int.requiresFloat() ? Float : 0) |
1496 (UserNamespace ? UseUserNamespace : 0)];
1497
1498 // Make the name of the function in this declaration.
1499
1500 std::string FunctionName =
1501 Polymorphic ? Int.shortName() : Int.fullName();
1502 if (!UserNamespace)
1503 FunctionName = "__arm_" + FunctionName;
1504
1505 // Make strings for the types involved in the function's
1506 // prototype.
1507
1508 std::string RetTypeName = Int.returnType()->cName();
1509 if (!StringRef(RetTypeName).endswith("*"))
1510 RetTypeName += " ";
1511
1512 std::vector<std::string> ArgTypeNames;
1513 for (const Type *ArgTypePtr : Int.argTypes())
1514 ArgTypeNames.push_back(ArgTypePtr->cName());
1515 std::string ArgTypesString =
1516 join(std::begin(ArgTypeNames), std::end(ArgTypeNames), ", ");
1517
1518 // Emit the actual declaration. All these functions are
1519 // declared 'static inline' without a body, which is fine
1520 // provided clang recognizes them as builtins, and has the
1521 // effect that this type signature is used in place of the one
1522 // that Builtins.def didn't provide. That's how we can get
1523 // structure types that weren't defined until this header was
1524 // included to be part of the type signature of a builtin that
1525 // was known to clang already.
1526 //
1527 // The declarations use __attribute__(__clang_arm_mve_alias),
1528 // so that each function declared will be recognized as the
1529 // appropriate MVE builtin in spite of its user-facing name.
1530 //
1531 // (That's better than making them all wrapper functions,
1532 // partly because it avoids any compiler error message citing
1533 // the wrapper function definition instead of the user's code,
1534 // and mostly because some MVE intrinsics have arguments
1535 // required to be compile-time constants, and that property
1536 // can't be propagated through a wrapper function. It can be
1537 // propagated through a macro, but macros can't be overloaded
1538 // on argument types very easily - you have to use _Generic,
1539 // which makes error messages very confusing when the user
1540 // gets it wrong.)
1541 //
1542 // Finally, the polymorphic versions of the intrinsics are
1543 // also defined with __attribute__(overloadable), so that when
1544 // the same name is defined with several type signatures, the
1545 // right thing happens. Each one of the overloaded
1546 // declarations is given a different builtin id, which
1547 // has exactly the effect we want: first clang resolves the
1548 // overload to the right function, then it knows which builtin
1549 // it's referring to, and then the Sema checking for that
1550 // builtin can check further things like the constant
1551 // arguments.
1552 //
1553 // One more subtlety is the newline just before the return
1554 // type name. That's a cosmetic tweak to make the error
1555 // messages legible if the user gets the types wrong in a call
1556 // to a polymorphic function: this way, clang will print just
1557 // the _final_ line of each declaration in the header, to show
1558 // the type signatures that would have been legal. So all the
1559 // confusing machinery with __attribute__ is left out of the
1560 // error message, and the user sees something that's more or
1561 // less self-documenting: "here's a list of actually readable
1562 // type signatures for vfooq(), and here's why each one didn't
1563 // match your call".
1564
1565 OS << "static __inline__ __attribute__(("
1566 << (Polymorphic ? "overloadable, " : "")
1567 << "__clang_arm_mve_alias(__builtin_arm_mve_" << Int.fullName()
1568 << ")))\n"
1569 << RetTypeName << FunctionName << "(" << ArgTypesString << ");\n";
1570 }
1571 }
1572 }
1573 for (auto &part : parts)
1574 part << "\n";
1575
1576 // Now we've finished accumulating bits and pieces into the parts[] array.
1577 // Put it all together to write the final output file.
1578
1579 OS << "/*===---- arm_mve.h - ARM MVE intrinsics "
1580 "-----------------------------------===\n"
1581 " *\n"
1582 " *\n"
1583 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1584 "Exceptions.\n"
1585 " * See https://llvm.org/LICENSE.txt for license information.\n"
1586 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1587 " *\n"
1588 " *===-------------------------------------------------------------"
1589 "----"
1590 "------===\n"
1591 " */\n"
1592 "\n"
1593 "#ifndef __ARM_MVE_H\n"
1594 "#define __ARM_MVE_H\n"
1595 "\n"
1596 "#if !__ARM_FEATURE_MVE\n"
1597 "#error \"MVE support not enabled\"\n"
1598 "#endif\n"
1599 "\n"
1600 "#include <stdint.h>\n"
1601 "\n";
1602
1603 for (size_t i = 0; i < NumParts; ++i) {
1604 std::vector<std::string> conditions;
1605 if (i & Float)
1606 conditions.push_back("(__ARM_FEATURE_MVE & 2)");
1607 if (i & UseUserNamespace)
1608 conditions.push_back("(!defined __ARM_MVE_PRESERVE_USER_NAMESPACE)");
1609
1610 std::string condition =
1611 join(std::begin(conditions), std::end(conditions), " && ");
1612 if (!condition.empty())
1613 OS << "#if " << condition << "\n\n";
1614 OS << parts[i].str();
1615 if (!condition.empty())
1616 OS << "#endif /* " << condition << " */\n\n";
1617 }
1618
1619 OS << "#endif /* __ARM_MVE_H */\n";
1620 }
1621
EmitBuiltinDef(raw_ostream & OS)1622 void MveEmitter::EmitBuiltinDef(raw_ostream &OS) {
1623 for (const auto &kv : ACLEIntrinsics) {
1624 const ACLEIntrinsic &Int = *kv.second;
1625 OS << "TARGET_HEADER_BUILTIN(__builtin_arm_mve_" << Int.fullName()
1626 << ", \"\", \"n\", \"arm_mve.h\", ALL_LANGUAGES, \"\")\n";
1627 }
1628
1629 std::set<std::string> ShortNamesSeen;
1630
1631 for (const auto &kv : ACLEIntrinsics) {
1632 const ACLEIntrinsic &Int = *kv.second;
1633 if (Int.polymorphic()) {
1634 StringRef Name = Int.shortName();
1635 if (ShortNamesSeen.find(Name) == ShortNamesSeen.end()) {
1636 OS << "BUILTIN(__builtin_arm_mve_" << Name << ", \"vi.\", \"nt";
1637 if (Int.nonEvaluating())
1638 OS << "u"; // indicate that this builtin doesn't evaluate its args
1639 OS << "\")\n";
1640 ShortNamesSeen.insert(Name);
1641 }
1642 }
1643 }
1644 }
1645
EmitBuiltinSema(raw_ostream & OS)1646 void MveEmitter::EmitBuiltinSema(raw_ostream &OS) {
1647 std::map<std::string, std::set<std::string>> Checks;
1648
1649 for (const auto &kv : ACLEIntrinsics) {
1650 const ACLEIntrinsic &Int = *kv.second;
1651 std::string Check = Int.genSema();
1652 if (!Check.empty())
1653 Checks[Check].insert(Int.fullName());
1654 }
1655
1656 for (const auto &kv : Checks) {
1657 for (StringRef Name : kv.second)
1658 OS << "case ARM::BI__builtin_arm_mve_" << Name << ":\n";
1659 OS << kv.first;
1660 }
1661 }
1662
1663 // Machinery for the grouping of intrinsics by similar codegen.
1664 //
1665 // The general setup is that 'MergeableGroup' stores the things that a set of
1666 // similarly shaped intrinsics have in common: the text of their code
1667 // generation, and the number and type of their parameter variables.
1668 // MergeableGroup is the key in a std::map whose value is a set of
1669 // OutputIntrinsic, which stores the ways in which a particular intrinsic
1670 // specializes the MergeableGroup's generic description: the function name and
1671 // the _values_ of the parameter variables.
1672
1673 struct ComparableStringVector : std::vector<std::string> {
1674 // Infrastructure: a derived class of vector<string> which comes with an
1675 // ordering, so that it can be used as a key in maps and an element in sets.
1676 // There's no requirement on the ordering beyond being deterministic.
operator <__anondc5b1bb70111::ComparableStringVector1677 bool operator<(const ComparableStringVector &rhs) const {
1678 if (size() != rhs.size())
1679 return size() < rhs.size();
1680 for (size_t i = 0, e = size(); i < e; ++i)
1681 if ((*this)[i] != rhs[i])
1682 return (*this)[i] < rhs[i];
1683 return false;
1684 }
1685 };
1686
1687 struct OutputIntrinsic {
1688 const ACLEIntrinsic *Int;
1689 std::string Name;
1690 ComparableStringVector ParamValues;
operator <__anondc5b1bb70111::OutputIntrinsic1691 bool operator<(const OutputIntrinsic &rhs) const {
1692 if (Name != rhs.Name)
1693 return Name < rhs.Name;
1694 return ParamValues < rhs.ParamValues;
1695 }
1696 };
1697 struct MergeableGroup {
1698 std::string Code;
1699 ComparableStringVector ParamTypes;
operator <__anondc5b1bb70111::MergeableGroup1700 bool operator<(const MergeableGroup &rhs) const {
1701 if (Code != rhs.Code)
1702 return Code < rhs.Code;
1703 return ParamTypes < rhs.ParamTypes;
1704 }
1705 };
1706
EmitBuiltinCG(raw_ostream & OS)1707 void MveEmitter::EmitBuiltinCG(raw_ostream &OS) {
1708 // Pass 1: generate code for all the intrinsics as if every type or constant
1709 // that can possibly be abstracted out into a parameter variable will be.
1710 // This identifies the sets of intrinsics we'll group together into a single
1711 // piece of code generation.
1712
1713 std::map<MergeableGroup, std::set<OutputIntrinsic>> MergeableGroupsPrelim;
1714
1715 for (const auto &kv : ACLEIntrinsics) {
1716 const ACLEIntrinsic &Int = *kv.second;
1717
1718 MergeableGroup MG;
1719 OutputIntrinsic OI;
1720
1721 OI.Int = ∬
1722 OI.Name = Int.fullName();
1723 CodeGenParamAllocator ParamAllocPrelim{&MG.ParamTypes, &OI.ParamValues};
1724 raw_string_ostream OS(MG.Code);
1725 Int.genCode(OS, ParamAllocPrelim, 1);
1726 OS.flush();
1727
1728 MergeableGroupsPrelim[MG].insert(OI);
1729 }
1730
1731 // Pass 2: for each of those groups, optimize the parameter variable set by
1732 // eliminating 'parameters' that are the same for all intrinsics in the
1733 // group, and merging together pairs of parameter variables that take the
1734 // same values as each other for all intrinsics in the group.
1735
1736 std::map<MergeableGroup, std::set<OutputIntrinsic>> MergeableGroups;
1737
1738 for (const auto &kv : MergeableGroupsPrelim) {
1739 const MergeableGroup &MG = kv.first;
1740 std::vector<int> ParamNumbers;
1741 std::map<ComparableStringVector, int> ParamNumberMap;
1742
1743 // Loop over the parameters for this group.
1744 for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i) {
1745 // Is this parameter the same for all intrinsics in the group?
1746 const OutputIntrinsic &OI_first = *kv.second.begin();
1747 bool Constant = all_of(kv.second, [&](const OutputIntrinsic &OI) {
1748 return OI.ParamValues[i] == OI_first.ParamValues[i];
1749 });
1750
1751 // If so, record it as -1, meaning 'no parameter variable needed'. Then
1752 // the corresponding call to allocParam in pass 2 will not generate a
1753 // variable at all, and just use the value inline.
1754 if (Constant) {
1755 ParamNumbers.push_back(-1);
1756 continue;
1757 }
1758
1759 // Otherwise, make a list of the values this parameter takes for each
1760 // intrinsic, and see if that value vector matches anything we already
1761 // have. We also record the parameter type, so that we don't accidentally
1762 // match up two parameter variables with different types. (Not that
1763 // there's much chance of them having textually equivalent values, but in
1764 // _principle_ it could happen.)
1765 ComparableStringVector key;
1766 key.push_back(MG.ParamTypes[i]);
1767 for (const auto &OI : kv.second)
1768 key.push_back(OI.ParamValues[i]);
1769
1770 auto Found = ParamNumberMap.find(key);
1771 if (Found != ParamNumberMap.end()) {
1772 // Yes, an existing parameter variable can be reused for this.
1773 ParamNumbers.push_back(Found->second);
1774 continue;
1775 }
1776
1777 // No, we need a new parameter variable.
1778 int ExistingIndex = ParamNumberMap.size();
1779 ParamNumberMap[key] = ExistingIndex;
1780 ParamNumbers.push_back(ExistingIndex);
1781 }
1782
1783 // Now we're ready to do the pass 2 code generation, which will emit the
1784 // reduced set of parameter variables we've just worked out.
1785
1786 for (const auto &OI_prelim : kv.second) {
1787 const ACLEIntrinsic *Int = OI_prelim.Int;
1788
1789 MergeableGroup MG;
1790 OutputIntrinsic OI;
1791
1792 OI.Int = OI_prelim.Int;
1793 OI.Name = OI_prelim.Name;
1794 CodeGenParamAllocator ParamAlloc{&MG.ParamTypes, &OI.ParamValues,
1795 &ParamNumbers};
1796 raw_string_ostream OS(MG.Code);
1797 Int->genCode(OS, ParamAlloc, 2);
1798 OS.flush();
1799
1800 MergeableGroups[MG].insert(OI);
1801 }
1802 }
1803
1804 // Output the actual C++ code.
1805
1806 for (const auto &kv : MergeableGroups) {
1807 const MergeableGroup &MG = kv.first;
1808
1809 // List of case statements in the main switch on BuiltinID, and an open
1810 // brace.
1811 const char *prefix = "";
1812 for (const auto &OI : kv.second) {
1813 OS << prefix << "case ARM::BI__builtin_arm_mve_" << OI.Name << ":";
1814 prefix = "\n";
1815 }
1816 OS << " {\n";
1817
1818 if (!MG.ParamTypes.empty()) {
1819 // If we've got some parameter variables, then emit their declarations...
1820 for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i) {
1821 StringRef Type = MG.ParamTypes[i];
1822 OS << " " << Type;
1823 if (!Type.endswith("*"))
1824 OS << " ";
1825 OS << " Param" << utostr(i) << ";\n";
1826 }
1827
1828 // ... and an inner switch on BuiltinID that will fill them in with each
1829 // individual intrinsic's values.
1830 OS << " switch (BuiltinID) {\n";
1831 for (const auto &OI : kv.second) {
1832 OS << " case ARM::BI__builtin_arm_mve_" << OI.Name << ":\n";
1833 for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i)
1834 OS << " Param" << utostr(i) << " = " << OI.ParamValues[i] << ";\n";
1835 OS << " break;\n";
1836 }
1837 OS << " }\n";
1838 }
1839
1840 // And finally, output the code, and close the outer pair of braces. (The
1841 // code will always end with a 'return' statement, so we need not insert a
1842 // 'break' here.)
1843 OS << MG.Code << "}\n";
1844 }
1845 }
1846
EmitBuiltinAliases(raw_ostream & OS)1847 void MveEmitter::EmitBuiltinAliases(raw_ostream &OS) {
1848 for (const auto &kv : ACLEIntrinsics) {
1849 const ACLEIntrinsic &Int = *kv.second;
1850 OS << "case ARM::BI__builtin_arm_mve_" << Int.fullName() << ":\n"
1851 << " return AliasName == \"" << Int.fullName() << "\"";
1852 if (Int.polymorphic())
1853 OS << " || AliasName == \"" << Int.shortName() << "\"";
1854 OS << ";\n";
1855 }
1856 }
1857
1858 } // namespace
1859
1860 namespace clang {
1861
EmitMveHeader(RecordKeeper & Records,raw_ostream & OS)1862 void EmitMveHeader(RecordKeeper &Records, raw_ostream &OS) {
1863 MveEmitter(Records).EmitHeader(OS);
1864 }
1865
EmitMveBuiltinDef(RecordKeeper & Records,raw_ostream & OS)1866 void EmitMveBuiltinDef(RecordKeeper &Records, raw_ostream &OS) {
1867 MveEmitter(Records).EmitBuiltinDef(OS);
1868 }
1869
EmitMveBuiltinSema(RecordKeeper & Records,raw_ostream & OS)1870 void EmitMveBuiltinSema(RecordKeeper &Records, raw_ostream &OS) {
1871 MveEmitter(Records).EmitBuiltinSema(OS);
1872 }
1873
EmitMveBuiltinCG(RecordKeeper & Records,raw_ostream & OS)1874 void EmitMveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1875 MveEmitter(Records).EmitBuiltinCG(OS);
1876 }
1877
EmitMveBuiltinAliases(RecordKeeper & Records,raw_ostream & OS)1878 void EmitMveBuiltinAliases(RecordKeeper &Records, raw_ostream &OS) {
1879 MveEmitter(Records).EmitBuiltinAliases(OS);
1880 }
1881
1882 } // end namespace clang
1883