1 //===---- TargetInfo.cpp - Encapsulate target details -----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // These classes wrap the information about a call or function
10 // definition used to handle ABI compliancy.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "TargetInfo.h"
15 #include "ABIInfo.h"
16 #include "CGBlocks.h"
17 #include "CGCXXABI.h"
18 #include "CGValue.h"
19 #include "CodeGenFunction.h"
20 #include "clang/AST/Attr.h"
21 #include "clang/AST/RecordLayout.h"
22 #include "clang/Basic/Builtins.h"
23 #include "clang/Basic/CodeGenOptions.h"
24 #include "clang/Basic/DiagnosticFrontend.h"
25 #include "clang/CodeGen/CGFunctionInfo.h"
26 #include "llvm/ADT/SmallBitVector.h"
27 #include "llvm/ADT/StringExtras.h"
28 #include "llvm/ADT/StringSwitch.h"
29 #include "llvm/ADT/Triple.h"
30 #include "llvm/ADT/Twine.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/IntrinsicsNVPTX.h"
33 #include "llvm/IR/IntrinsicsS390.h"
34 #include "llvm/IR/Type.h"
35 #include "llvm/Support/MathExtras.h"
36 #include "llvm/Support/raw_ostream.h"
37 #include <algorithm>
38 
39 using namespace clang;
40 using namespace CodeGen;
41 
42 // Helper for coercing an aggregate argument or return value into an integer
43 // array of the same size (including padding) and alignment.  This alternate
44 // coercion happens only for the RenderScript ABI and can be removed after
45 // runtimes that rely on it are no longer supported.
46 //
47 // RenderScript assumes that the size of the argument / return value in the IR
48 // is the same as the size of the corresponding qualified type. This helper
49 // coerces the aggregate type into an array of the same size (including
50 // padding).  This coercion is used in lieu of expansion of struct members or
51 // other canonical coercions that return a coerced-type of larger size.
52 //
53 // Ty          - The argument / return value type
54 // Context     - The associated ASTContext
55 // LLVMContext - The associated LLVMContext
coerceToIntArray(QualType Ty,ASTContext & Context,llvm::LLVMContext & LLVMContext)56 static ABIArgInfo coerceToIntArray(QualType Ty,
57                                    ASTContext &Context,
58                                    llvm::LLVMContext &LLVMContext) {
59   // Alignment and Size are measured in bits.
60   const uint64_t Size = Context.getTypeSize(Ty);
61   const uint64_t Alignment = Context.getTypeAlign(Ty);
62   llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
63   const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
64   return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
65 }
66 
AssignToArrayRange(CodeGen::CGBuilderTy & Builder,llvm::Value * Array,llvm::Value * Value,unsigned FirstIndex,unsigned LastIndex)67 static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder,
68                                llvm::Value *Array,
69                                llvm::Value *Value,
70                                unsigned FirstIndex,
71                                unsigned LastIndex) {
72   // Alternatively, we could emit this as a loop in the source.
73   for (unsigned I = FirstIndex; I <= LastIndex; ++I) {
74     llvm::Value *Cell =
75         Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
76     Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
77   }
78 }
79 
isAggregateTypeForABI(QualType T)80 static bool isAggregateTypeForABI(QualType T) {
81   return !CodeGenFunction::hasScalarEvaluationKind(T) ||
82          T->isMemberFunctionPointerType();
83 }
84 
getNaturalAlignIndirect(QualType Ty,bool ByVal,bool Realign,llvm::Type * Padding) const85 ABIArgInfo ABIInfo::getNaturalAlignIndirect(QualType Ty, bool ByVal,
86                                             bool Realign,
87                                             llvm::Type *Padding) const {
88   return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), ByVal,
89                                  Realign, Padding);
90 }
91 
92 ABIArgInfo
getNaturalAlignIndirectInReg(QualType Ty,bool Realign) const93 ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, bool Realign) const {
94   return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
95                                       /*ByVal*/ false, Realign);
96 }
97 
EmitMSVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const98 Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
99                              QualType Ty) const {
100   return Address::invalid();
101 }
102 
getVAListElementType(CodeGenFunction & CGF)103 static llvm::Type *getVAListElementType(CodeGenFunction &CGF) {
104   return CGF.ConvertTypeForMem(
105       CGF.getContext().getBuiltinVaListType()->getPointeeType());
106 }
107 
isPromotableIntegerTypeForABI(QualType Ty) const108 bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
109   if (getContext().isPromotableIntegerType(Ty))
110     return true;
111 
112   if (const auto *EIT = Ty->getAs<BitIntType>())
113     if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
114       return true;
115 
116   return false;
117 }
118 
119 ABIInfo::~ABIInfo() = default;
120 
121 SwiftABIInfo::~SwiftABIInfo() = default;
122 
123 /// Does the given lowering require more than the given number of
124 /// registers when expanded?
125 ///
126 /// This is intended to be the basis of a reasonable basic implementation
127 /// of should{Pass,Return}IndirectlyForSwift.
128 ///
129 /// For most targets, a limit of four total registers is reasonable; this
130 /// limits the amount of code required in order to move around the value
131 /// in case it wasn't produced immediately prior to the call by the caller
132 /// (or wasn't produced in exactly the right registers) or isn't used
133 /// immediately within the callee.  But some targets may need to further
134 /// limit the register count due to an inability to support that many
135 /// return registers.
occupiesMoreThan(CodeGenTypes & cgt,ArrayRef<llvm::Type * > scalarTypes,unsigned maxAllRegisters)136 static bool occupiesMoreThan(CodeGenTypes &cgt,
137                              ArrayRef<llvm::Type*> scalarTypes,
138                              unsigned maxAllRegisters) {
139   unsigned intCount = 0, fpCount = 0;
140   for (llvm::Type *type : scalarTypes) {
141     if (type->isPointerTy()) {
142       intCount++;
143     } else if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
144       auto ptrWidth = cgt.getTarget().getPointerWidth(LangAS::Default);
145       intCount += (intTy->getBitWidth() + ptrWidth - 1) / ptrWidth;
146     } else {
147       assert(type->isVectorTy() || type->isFloatingPointTy());
148       fpCount++;
149     }
150   }
151 
152   return (intCount + fpCount > maxAllRegisters);
153 }
154 
shouldPassIndirectly(ArrayRef<llvm::Type * > ComponentTys,bool AsReturnValue) const155 bool SwiftABIInfo::shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,
156                                         bool AsReturnValue) const {
157   return occupiesMoreThan(CGT, ComponentTys, /*total=*/4);
158 }
159 
isLegalVectorType(CharUnits VectorSize,llvm::Type * EltTy,unsigned NumElts) const160 bool SwiftABIInfo::isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
161                                      unsigned NumElts) const {
162   // The default implementation of this assumes that the target guarantees
163   // 128-bit SIMD support but nothing more.
164   return (VectorSize.getQuantity() > 8 && VectorSize.getQuantity() <= 16);
165 }
166 
getRecordArgABI(const RecordType * RT,CGCXXABI & CXXABI)167 static CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT,
168                                               CGCXXABI &CXXABI) {
169   const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
170   if (!RD) {
171     if (!RT->getDecl()->canPassInRegisters())
172       return CGCXXABI::RAA_Indirect;
173     return CGCXXABI::RAA_Default;
174   }
175   return CXXABI.getRecordArgABI(RD);
176 }
177 
getRecordArgABI(QualType T,CGCXXABI & CXXABI)178 static CGCXXABI::RecordArgABI getRecordArgABI(QualType T,
179                                               CGCXXABI &CXXABI) {
180   const RecordType *RT = T->getAs<RecordType>();
181   if (!RT)
182     return CGCXXABI::RAA_Default;
183   return getRecordArgABI(RT, CXXABI);
184 }
185 
classifyReturnType(const CGCXXABI & CXXABI,CGFunctionInfo & FI,const ABIInfo & Info)186 static bool classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI,
187                                const ABIInfo &Info) {
188   QualType Ty = FI.getReturnType();
189 
190   if (const auto *RT = Ty->getAs<RecordType>())
191     if (!isa<CXXRecordDecl>(RT->getDecl()) &&
192         !RT->getDecl()->canPassInRegisters()) {
193       FI.getReturnInfo() = Info.getNaturalAlignIndirect(Ty);
194       return true;
195     }
196 
197   return CXXABI.classifyReturnType(FI);
198 }
199 
200 /// Pass transparent unions as if they were the type of the first element. Sema
201 /// should ensure that all elements of the union have the same "machine type".
useFirstFieldIfTransparentUnion(QualType Ty)202 static QualType useFirstFieldIfTransparentUnion(QualType Ty) {
203   if (const RecordType *UT = Ty->getAsUnionType()) {
204     const RecordDecl *UD = UT->getDecl();
205     if (UD->hasAttr<TransparentUnionAttr>()) {
206       assert(!UD->field_empty() && "sema created an empty transparent union");
207       return UD->field_begin()->getType();
208     }
209   }
210   return Ty;
211 }
212 
getCXXABI() const213 CGCXXABI &ABIInfo::getCXXABI() const {
214   return CGT.getCXXABI();
215 }
216 
getContext() const217 ASTContext &ABIInfo::getContext() const {
218   return CGT.getContext();
219 }
220 
getVMContext() const221 llvm::LLVMContext &ABIInfo::getVMContext() const {
222   return CGT.getLLVMContext();
223 }
224 
getDataLayout() const225 const llvm::DataLayout &ABIInfo::getDataLayout() const {
226   return CGT.getDataLayout();
227 }
228 
getTarget() const229 const TargetInfo &ABIInfo::getTarget() const {
230   return CGT.getTarget();
231 }
232 
getCodeGenOpts() const233 const CodeGenOptions &ABIInfo::getCodeGenOpts() const {
234   return CGT.getCodeGenOpts();
235 }
236 
isAndroid() const237 bool ABIInfo::isAndroid() const { return getTarget().getTriple().isAndroid(); }
238 
isHomogeneousAggregateBaseType(QualType Ty) const239 bool ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
240   return false;
241 }
242 
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const243 bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
244                                                 uint64_t Members) const {
245   return false;
246 }
247 
isZeroLengthBitfieldPermittedInHomogeneousAggregate() const248 bool ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() const {
249   // For compatibility with GCC, ignore empty bitfields in C++ mode.
250   return getContext().getLangOpts().CPlusPlus;
251 }
252 
dump() const253 LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
254   raw_ostream &OS = llvm::errs();
255   OS << "(ABIArgInfo Kind=";
256   switch (TheKind) {
257   case Direct:
258     OS << "Direct Type=";
259     if (llvm::Type *Ty = getCoerceToType())
260       Ty->print(OS);
261     else
262       OS << "null";
263     break;
264   case Extend:
265     OS << "Extend";
266     break;
267   case Ignore:
268     OS << "Ignore";
269     break;
270   case InAlloca:
271     OS << "InAlloca Offset=" << getInAllocaFieldIndex();
272     break;
273   case Indirect:
274     OS << "Indirect Align=" << getIndirectAlign().getQuantity()
275        << " ByVal=" << getIndirectByVal()
276        << " Realign=" << getIndirectRealign();
277     break;
278   case IndirectAliased:
279     OS << "Indirect Align=" << getIndirectAlign().getQuantity()
280        << " AadrSpace=" << getIndirectAddrSpace()
281        << " Realign=" << getIndirectRealign();
282     break;
283   case Expand:
284     OS << "Expand";
285     break;
286   case CoerceAndExpand:
287     OS << "CoerceAndExpand Type=";
288     getCoerceAndExpandType()->print(OS);
289     break;
290   }
291   OS << ")\n";
292 }
293 
294 // Dynamically round a pointer up to a multiple of the given alignment.
emitRoundPointerUpToAlignment(CodeGenFunction & CGF,llvm::Value * Ptr,CharUnits Align)295 static llvm::Value *emitRoundPointerUpToAlignment(CodeGenFunction &CGF,
296                                                   llvm::Value *Ptr,
297                                                   CharUnits Align) {
298   llvm::Value *PtrAsInt = Ptr;
299   // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
300   PtrAsInt = CGF.Builder.CreatePtrToInt(PtrAsInt, CGF.IntPtrTy);
301   PtrAsInt = CGF.Builder.CreateAdd(PtrAsInt,
302         llvm::ConstantInt::get(CGF.IntPtrTy, Align.getQuantity() - 1));
303   PtrAsInt = CGF.Builder.CreateAnd(PtrAsInt,
304            llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity()));
305   PtrAsInt = CGF.Builder.CreateIntToPtr(PtrAsInt,
306                                         Ptr->getType(),
307                                         Ptr->getName() + ".aligned");
308   return PtrAsInt;
309 }
310 
311 /// Emit va_arg for a platform using the common void* representation,
312 /// where arguments are simply emitted in an array of slots on the stack.
313 ///
314 /// This version implements the core direct-value passing rules.
315 ///
316 /// \param SlotSize - The size and alignment of a stack slot.
317 ///   Each argument will be allocated to a multiple of this number of
318 ///   slots, and all the slots will be aligned to this value.
319 /// \param AllowHigherAlign - The slot alignment is not a cap;
320 ///   an argument type with an alignment greater than the slot size
321 ///   will be emitted on a higher-alignment address, potentially
322 ///   leaving one or more empty slots behind as padding.  If this
323 ///   is false, the returned address might be less-aligned than
324 ///   DirectAlign.
325 /// \param ForceRightAdjust - Default is false. On big-endian platform and
326 ///   if the argument is smaller than a slot, set this flag will force
327 ///   right-adjust the argument in its slot irrespective of the type.
emitVoidPtrDirectVAArg(CodeGenFunction & CGF,Address VAListAddr,llvm::Type * DirectTy,CharUnits DirectSize,CharUnits DirectAlign,CharUnits SlotSize,bool AllowHigherAlign,bool ForceRightAdjust=false)328 static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF,
329                                       Address VAListAddr,
330                                       llvm::Type *DirectTy,
331                                       CharUnits DirectSize,
332                                       CharUnits DirectAlign,
333                                       CharUnits SlotSize,
334                                       bool AllowHigherAlign,
335                                       bool ForceRightAdjust = false) {
336   // Cast the element type to i8* if necessary.  Some platforms define
337   // va_list as a struct containing an i8* instead of just an i8*.
338   if (VAListAddr.getElementType() != CGF.Int8PtrTy)
339     VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
340 
341   llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
342 
343   // If the CC aligns values higher than the slot size, do so if needed.
344   Address Addr = Address::invalid();
345   if (AllowHigherAlign && DirectAlign > SlotSize) {
346     Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
347                    CGF.Int8Ty, DirectAlign);
348   } else {
349     Addr = Address(Ptr, CGF.Int8Ty, SlotSize);
350   }
351 
352   // Advance the pointer past the argument, then store that back.
353   CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
354   Address NextPtr =
355       CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
356   CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
357 
358   // If the argument is smaller than a slot, and this is a big-endian
359   // target, the argument will be right-adjusted in its slot.
360   if (DirectSize < SlotSize && CGF.CGM.getDataLayout().isBigEndian() &&
361       (!DirectTy->isStructTy() || ForceRightAdjust)) {
362     Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
363   }
364 
365   Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
366   return Addr;
367 }
368 
369 /// Emit va_arg for a platform using the common void* representation,
370 /// where arguments are simply emitted in an array of slots on the stack.
371 ///
372 /// \param IsIndirect - Values of this type are passed indirectly.
373 /// \param ValueInfo - The size and alignment of this type, generally
374 ///   computed with getContext().getTypeInfoInChars(ValueTy).
375 /// \param SlotSizeAndAlign - The size and alignment of a stack slot.
376 ///   Each argument will be allocated to a multiple of this number of
377 ///   slots, and all the slots will be aligned to this value.
378 /// \param AllowHigherAlign - The slot alignment is not a cap;
379 ///   an argument type with an alignment greater than the slot size
380 ///   will be emitted on a higher-alignment address, potentially
381 ///   leaving one or more empty slots behind as padding.
382 /// \param ForceRightAdjust - Default is false. On big-endian platform and
383 ///   if the argument is smaller than a slot, set this flag will force
384 ///   right-adjust the argument in its slot irrespective of the type.
emitVoidPtrVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType ValueTy,bool IsIndirect,TypeInfoChars ValueInfo,CharUnits SlotSizeAndAlign,bool AllowHigherAlign,bool ForceRightAdjust=false)385 static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
386                                 QualType ValueTy, bool IsIndirect,
387                                 TypeInfoChars ValueInfo,
388                                 CharUnits SlotSizeAndAlign,
389                                 bool AllowHigherAlign,
390                                 bool ForceRightAdjust = false) {
391   // The size and alignment of the value that was passed directly.
392   CharUnits DirectSize, DirectAlign;
393   if (IsIndirect) {
394     DirectSize = CGF.getPointerSize();
395     DirectAlign = CGF.getPointerAlign();
396   } else {
397     DirectSize = ValueInfo.Width;
398     DirectAlign = ValueInfo.Align;
399   }
400 
401   // Cast the address we've calculated to the right type.
402   llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy), *ElementTy = DirectTy;
403   if (IsIndirect)
404     DirectTy = DirectTy->getPointerTo(0);
405 
406   Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy, DirectSize,
407                                         DirectAlign, SlotSizeAndAlign,
408                                         AllowHigherAlign, ForceRightAdjust);
409 
410   if (IsIndirect) {
411     Addr = Address(CGF.Builder.CreateLoad(Addr), ElementTy, ValueInfo.Align);
412   }
413 
414   return Addr;
415 }
416 
complexTempStructure(CodeGenFunction & CGF,Address VAListAddr,QualType Ty,CharUnits SlotSize,CharUnits EltSize,const ComplexType * CTy)417 static Address complexTempStructure(CodeGenFunction &CGF, Address VAListAddr,
418                                     QualType Ty, CharUnits SlotSize,
419                                     CharUnits EltSize, const ComplexType *CTy) {
420   Address Addr =
421       emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty, SlotSize * 2,
422                              SlotSize, SlotSize, /*AllowHigher*/ true);
423 
424   Address RealAddr = Addr;
425   Address ImagAddr = RealAddr;
426   if (CGF.CGM.getDataLayout().isBigEndian()) {
427     RealAddr =
428         CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize - EltSize);
429     ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(ImagAddr,
430                                                       2 * SlotSize - EltSize);
431   } else {
432     ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize);
433   }
434 
435   llvm::Type *EltTy = CGF.ConvertTypeForMem(CTy->getElementType());
436   RealAddr = CGF.Builder.CreateElementBitCast(RealAddr, EltTy);
437   ImagAddr = CGF.Builder.CreateElementBitCast(ImagAddr, EltTy);
438   llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal");
439   llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag");
440 
441   Address Temp = CGF.CreateMemTemp(Ty, "vacplx");
442   CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty),
443                          /*init*/ true);
444   return Temp;
445 }
446 
emitMergePHI(CodeGenFunction & CGF,Address Addr1,llvm::BasicBlock * Block1,Address Addr2,llvm::BasicBlock * Block2,const llvm::Twine & Name="")447 static Address emitMergePHI(CodeGenFunction &CGF,
448                             Address Addr1, llvm::BasicBlock *Block1,
449                             Address Addr2, llvm::BasicBlock *Block2,
450                             const llvm::Twine &Name = "") {
451   assert(Addr1.getType() == Addr2.getType());
452   llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
453   PHI->addIncoming(Addr1.getPointer(), Block1);
454   PHI->addIncoming(Addr2.getPointer(), Block2);
455   CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
456   return Address(PHI, Addr1.getElementType(), Align);
457 }
458 
TargetCodeGenInfo(std::unique_ptr<ABIInfo> Info)459 TargetCodeGenInfo::TargetCodeGenInfo(std::unique_ptr<ABIInfo> Info)
460     : Info(std::move(Info)) {}
461 
462 TargetCodeGenInfo::~TargetCodeGenInfo() = default;
463 
464 // If someone can figure out a general rule for this, that would be great.
465 // It's probably just doomed to be platform-dependent, though.
getSizeOfUnwindException() const466 unsigned TargetCodeGenInfo::getSizeOfUnwindException() const {
467   // Verified for:
468   //   x86-64     FreeBSD, Linux, Darwin
469   //   x86-32     FreeBSD, Linux, Darwin
470   //   PowerPC    Linux, Darwin
471   //   ARM        Darwin (*not* EABI)
472   //   AArch64    Linux
473   return 32;
474 }
475 
isNoProtoCallVariadic(const CallArgList & args,const FunctionNoProtoType * fnType) const476 bool TargetCodeGenInfo::isNoProtoCallVariadic(const CallArgList &args,
477                                      const FunctionNoProtoType *fnType) const {
478   // The following conventions are known to require this to be false:
479   //   x86_stdcall
480   //   MIPS
481   // For everything else, we just prefer false unless we opt out.
482   return false;
483 }
484 
485 void
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const486 TargetCodeGenInfo::getDependentLibraryOption(llvm::StringRef Lib,
487                                              llvm::SmallString<24> &Opt) const {
488   // This assumes the user is passing a library name like "rt" instead of a
489   // filename like "librt.a/so", and that they don't care whether it's static or
490   // dynamic.
491   Opt = "-l";
492   Opt += Lib;
493 }
494 
getOpenCLKernelCallingConv() const495 unsigned TargetCodeGenInfo::getOpenCLKernelCallingConv() const {
496   // OpenCL kernels are called via an explicit runtime API with arguments
497   // set with clSetKernelArg(), not as normal sub-functions.
498   // Return SPIR_KERNEL by default as the kernel calling convention to
499   // ensure the fingerprint is fixed such way that each OpenCL argument
500   // gets one matching argument in the produced kernel function argument
501   // list to enable feasible implementation of clSetKernelArg() with
502   // aggregates etc. In case we would use the default C calling conv here,
503   // clSetKernelArg() might break depending on the target-specific
504   // conventions; different targets might split structs passed as values
505   // to multiple function arguments etc.
506   return llvm::CallingConv::SPIR_KERNEL;
507 }
508 
getNullPointer(const CodeGen::CodeGenModule & CGM,llvm::PointerType * T,QualType QT) const509 llvm::Constant *TargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
510     llvm::PointerType *T, QualType QT) const {
511   return llvm::ConstantPointerNull::get(T);
512 }
513 
getGlobalVarAddressSpace(CodeGenModule & CGM,const VarDecl * D) const514 LangAS TargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
515                                                    const VarDecl *D) const {
516   assert(!CGM.getLangOpts().OpenCL &&
517          !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
518          "Address space agnostic languages only");
519   return D ? D->getType().getAddressSpace() : LangAS::Default;
520 }
521 
performAddrSpaceCast(CodeGen::CodeGenFunction & CGF,llvm::Value * Src,LangAS SrcAddr,LangAS DestAddr,llvm::Type * DestTy,bool isNonNull) const522 llvm::Value *TargetCodeGenInfo::performAddrSpaceCast(
523     CodeGen::CodeGenFunction &CGF, llvm::Value *Src, LangAS SrcAddr,
524     LangAS DestAddr, llvm::Type *DestTy, bool isNonNull) const {
525   // Since target may map different address spaces in AST to the same address
526   // space, an address space conversion may end up as a bitcast.
527   if (auto *C = dyn_cast<llvm::Constant>(Src))
528     return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
529   // Try to preserve the source's name to make IR more readable.
530   return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
531       Src, DestTy, Src->hasName() ? Src->getName() + ".ascast" : "");
532 }
533 
534 llvm::Constant *
performAddrSpaceCast(CodeGenModule & CGM,llvm::Constant * Src,LangAS SrcAddr,LangAS DestAddr,llvm::Type * DestTy) const535 TargetCodeGenInfo::performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *Src,
536                                         LangAS SrcAddr, LangAS DestAddr,
537                                         llvm::Type *DestTy) const {
538   // Since target may map different address spaces in AST to the same address
539   // space, an address space conversion may end up as a bitcast.
540   return llvm::ConstantExpr::getPointerCast(Src, DestTy);
541 }
542 
543 llvm::SyncScope::ID
getLLVMSyncScopeID(const LangOptions & LangOpts,SyncScope Scope,llvm::AtomicOrdering Ordering,llvm::LLVMContext & Ctx) const544 TargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
545                                       SyncScope Scope,
546                                       llvm::AtomicOrdering Ordering,
547                                       llvm::LLVMContext &Ctx) const {
548   return Ctx.getOrInsertSyncScopeID(""); /* default sync scope */
549 }
550 
551 static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
552 
553 /// isEmptyField - Return true iff a the field is "empty", that is it
554 /// is an unnamed bit-field or an (array of) empty record(s).
isEmptyField(ASTContext & Context,const FieldDecl * FD,bool AllowArrays)555 static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
556                          bool AllowArrays) {
557   if (FD->isUnnamedBitfield())
558     return true;
559 
560   QualType FT = FD->getType();
561 
562   // Constant arrays of empty records count as empty, strip them off.
563   // Constant arrays of zero length always count as empty.
564   bool WasArray = false;
565   if (AllowArrays)
566     while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
567       if (AT->getSize() == 0)
568         return true;
569       FT = AT->getElementType();
570       // The [[no_unique_address]] special case below does not apply to
571       // arrays of C++ empty records, so we need to remember this fact.
572       WasArray = true;
573     }
574 
575   const RecordType *RT = FT->getAs<RecordType>();
576   if (!RT)
577     return false;
578 
579   // C++ record fields are never empty, at least in the Itanium ABI.
580   //
581   // FIXME: We should use a predicate for whether this behavior is true in the
582   // current ABI.
583   //
584   // The exception to the above rule are fields marked with the
585   // [[no_unique_address]] attribute (since C++20).  Those do count as empty
586   // according to the Itanium ABI.  The exception applies only to records,
587   // not arrays of records, so we must also check whether we stripped off an
588   // array type above.
589   if (isa<CXXRecordDecl>(RT->getDecl()) &&
590       (WasArray || !FD->hasAttr<NoUniqueAddressAttr>()))
591     return false;
592 
593   return isEmptyRecord(Context, FT, AllowArrays);
594 }
595 
596 /// isEmptyRecord - Return true iff a structure contains only empty
597 /// fields. Note that a structure with a flexible array member is not
598 /// considered empty.
isEmptyRecord(ASTContext & Context,QualType T,bool AllowArrays)599 static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) {
600   const RecordType *RT = T->getAs<RecordType>();
601   if (!RT)
602     return false;
603   const RecordDecl *RD = RT->getDecl();
604   if (RD->hasFlexibleArrayMember())
605     return false;
606 
607   // If this is a C++ record, check the bases first.
608   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
609     for (const auto &I : CXXRD->bases())
610       if (!isEmptyRecord(Context, I.getType(), true))
611         return false;
612 
613   for (const auto *I : RD->fields())
614     if (!isEmptyField(Context, I, AllowArrays))
615       return false;
616   return true;
617 }
618 
619 /// isSingleElementStruct - Determine if a structure is a "single
620 /// element struct", i.e. it has exactly one non-empty field or
621 /// exactly one field which is itself a single element
622 /// struct. Structures with flexible array members are never
623 /// considered single element structs.
624 ///
625 /// \return The field declaration for the single non-empty field, if
626 /// it exists.
isSingleElementStruct(QualType T,ASTContext & Context)627 static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
628   const RecordType *RT = T->getAs<RecordType>();
629   if (!RT)
630     return nullptr;
631 
632   const RecordDecl *RD = RT->getDecl();
633   if (RD->hasFlexibleArrayMember())
634     return nullptr;
635 
636   const Type *Found = nullptr;
637 
638   // If this is a C++ record, check the bases first.
639   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
640     for (const auto &I : CXXRD->bases()) {
641       // Ignore empty records.
642       if (isEmptyRecord(Context, I.getType(), true))
643         continue;
644 
645       // If we already found an element then this isn't a single-element struct.
646       if (Found)
647         return nullptr;
648 
649       // If this is non-empty and not a single element struct, the composite
650       // cannot be a single element struct.
651       Found = isSingleElementStruct(I.getType(), Context);
652       if (!Found)
653         return nullptr;
654     }
655   }
656 
657   // Check for single element.
658   for (const auto *FD : RD->fields()) {
659     QualType FT = FD->getType();
660 
661     // Ignore empty fields.
662     if (isEmptyField(Context, FD, true))
663       continue;
664 
665     // If we already found an element then this isn't a single-element
666     // struct.
667     if (Found)
668       return nullptr;
669 
670     // Treat single element arrays as the element.
671     while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
672       if (AT->getSize().getZExtValue() != 1)
673         break;
674       FT = AT->getElementType();
675     }
676 
677     if (!isAggregateTypeForABI(FT)) {
678       Found = FT.getTypePtr();
679     } else {
680       Found = isSingleElementStruct(FT, Context);
681       if (!Found)
682         return nullptr;
683     }
684   }
685 
686   // We don't consider a struct a single-element struct if it has
687   // padding beyond the element type.
688   if (Found && Context.getTypeSize(Found) != Context.getTypeSize(T))
689     return nullptr;
690 
691   return Found;
692 }
693 
694 namespace {
EmitVAArgInstr(CodeGenFunction & CGF,Address VAListAddr,QualType Ty,const ABIArgInfo & AI)695 Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
696                        const ABIArgInfo &AI) {
697   // This default implementation defers to the llvm backend's va_arg
698   // instruction. It can handle only passing arguments directly
699   // (typically only handled in the backend for primitive types), or
700   // aggregates passed indirectly by pointer (NOTE: if the "byval"
701   // flag has ABI impact in the callee, this implementation cannot
702   // work.)
703 
704   // Only a few cases are covered here at the moment -- those needed
705   // by the default abi.
706   llvm::Value *Val;
707 
708   if (AI.isIndirect()) {
709     assert(!AI.getPaddingType() &&
710            "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
711     assert(
712         !AI.getIndirectRealign() &&
713         "Unexpected IndirectRealign seen in arginfo in generic VAArg emitter!");
714 
715     auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
716     CharUnits TyAlignForABI = TyInfo.Align;
717 
718     llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty);
719     llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy);
720     llvm::Value *Addr =
721         CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
722     return Address(Addr, ElementTy, TyAlignForABI);
723   } else {
724     assert((AI.isDirect() || AI.isExtend()) &&
725            "Unexpected ArgInfo Kind in generic VAArg emitter!");
726 
727     assert(!AI.getInReg() &&
728            "Unexpected InReg seen in arginfo in generic VAArg emitter!");
729     assert(!AI.getPaddingType() &&
730            "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
731     assert(!AI.getDirectOffset() &&
732            "Unexpected DirectOffset seen in arginfo in generic VAArg emitter!");
733     assert(!AI.getCoerceToType() &&
734            "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
735 
736     Address Temp = CGF.CreateMemTemp(Ty, "varet");
737     Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(),
738                                   CGF.ConvertTypeForMem(Ty));
739     CGF.Builder.CreateStore(Val, Temp);
740     return Temp;
741   }
742 }
743 
744 /// DefaultABIInfo - The default implementation for ABI specific
745 /// details. This implementation provides information which results in
746 /// self-consistent and sensible LLVM IR generation, but does not
747 /// conform to any particular ABI.
748 class DefaultABIInfo : public ABIInfo {
749 public:
DefaultABIInfo(CodeGen::CodeGenTypes & CGT)750   DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
751 
752   ABIArgInfo classifyReturnType(QualType RetTy) const;
753   ABIArgInfo classifyArgumentType(QualType RetTy) const;
754 
computeInfo(CGFunctionInfo & FI) const755   void computeInfo(CGFunctionInfo &FI) const override {
756     if (!getCXXABI().classifyReturnType(FI))
757       FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
758     for (auto &I : FI.arguments())
759       I.info = classifyArgumentType(I.type);
760   }
761 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const762   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
763                     QualType Ty) const override {
764     return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
765   }
766 };
767 
768 class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
769 public:
DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT)770   DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
771       : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
772 };
773 
classifyArgumentType(QualType Ty) const774 ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
775   Ty = useFirstFieldIfTransparentUnion(Ty);
776 
777   if (isAggregateTypeForABI(Ty)) {
778     // Records with non-trivial destructors/copy-constructors should not be
779     // passed by value.
780     if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
781       return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
782 
783     return getNaturalAlignIndirect(Ty);
784   }
785 
786   // Treat an enum type as its underlying type.
787   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
788     Ty = EnumTy->getDecl()->getIntegerType();
789 
790   ASTContext &Context = getContext();
791   if (const auto *EIT = Ty->getAs<BitIntType>())
792     if (EIT->getNumBits() >
793         Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
794                                 ? Context.Int128Ty
795                                 : Context.LongLongTy))
796       return getNaturalAlignIndirect(Ty);
797 
798   return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
799                                             : ABIArgInfo::getDirect());
800 }
801 
classifyReturnType(QualType RetTy) const802 ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
803   if (RetTy->isVoidType())
804     return ABIArgInfo::getIgnore();
805 
806   if (isAggregateTypeForABI(RetTy))
807     return getNaturalAlignIndirect(RetTy);
808 
809   // Treat an enum type as its underlying type.
810   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
811     RetTy = EnumTy->getDecl()->getIntegerType();
812 
813   if (const auto *EIT = RetTy->getAs<BitIntType>())
814     if (EIT->getNumBits() >
815         getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
816                                      ? getContext().Int128Ty
817                                      : getContext().LongLongTy))
818       return getNaturalAlignIndirect(RetTy);
819 
820   return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
821                                                : ABIArgInfo::getDirect());
822 }
823 
824 //===----------------------------------------------------------------------===//
825 // WebAssembly ABI Implementation
826 //
827 // This is a very simple ABI that relies a lot on DefaultABIInfo.
828 //===----------------------------------------------------------------------===//
829 
830 class WebAssemblyABIInfo final : public ABIInfo {
831 public:
832   enum ABIKind {
833     MVP = 0,
834     ExperimentalMV = 1,
835   };
836 
837 private:
838   DefaultABIInfo defaultInfo;
839   ABIKind Kind;
840 
841 public:
WebAssemblyABIInfo(CodeGen::CodeGenTypes & CGT,ABIKind Kind)842   explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind)
843       : ABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {}
844 
845 private:
846   ABIArgInfo classifyReturnType(QualType RetTy) const;
847   ABIArgInfo classifyArgumentType(QualType Ty) const;
848 
849   // DefaultABIInfo's classifyReturnType and classifyArgumentType are
850   // non-virtual, but computeInfo and EmitVAArg are virtual, so we
851   // overload them.
computeInfo(CGFunctionInfo & FI) const852   void computeInfo(CGFunctionInfo &FI) const override {
853     if (!getCXXABI().classifyReturnType(FI))
854       FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
855     for (auto &Arg : FI.arguments())
856       Arg.info = classifyArgumentType(Arg.type);
857   }
858 
859   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
860                     QualType Ty) const override;
861 };
862 
863 class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
864 public:
WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,WebAssemblyABIInfo::ABIKind K)865   explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
866                                         WebAssemblyABIInfo::ABIKind K)
867       : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {
868     SwiftInfo =
869         std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/false);
870   }
871 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const872   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
873                            CodeGen::CodeGenModule &CGM) const override {
874     TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
875     if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
876       if (const auto *Attr = FD->getAttr<WebAssemblyImportModuleAttr>()) {
877         llvm::Function *Fn = cast<llvm::Function>(GV);
878         llvm::AttrBuilder B(GV->getContext());
879         B.addAttribute("wasm-import-module", Attr->getImportModule());
880         Fn->addFnAttrs(B);
881       }
882       if (const auto *Attr = FD->getAttr<WebAssemblyImportNameAttr>()) {
883         llvm::Function *Fn = cast<llvm::Function>(GV);
884         llvm::AttrBuilder B(GV->getContext());
885         B.addAttribute("wasm-import-name", Attr->getImportName());
886         Fn->addFnAttrs(B);
887       }
888       if (const auto *Attr = FD->getAttr<WebAssemblyExportNameAttr>()) {
889         llvm::Function *Fn = cast<llvm::Function>(GV);
890         llvm::AttrBuilder B(GV->getContext());
891         B.addAttribute("wasm-export-name", Attr->getExportName());
892         Fn->addFnAttrs(B);
893       }
894     }
895 
896     if (auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
897       llvm::Function *Fn = cast<llvm::Function>(GV);
898       if (!FD->doesThisDeclarationHaveABody() && !FD->hasPrototype())
899         Fn->addFnAttr("no-prototype");
900     }
901   }
902 };
903 
904 /// Classify argument of given type \p Ty.
classifyArgumentType(QualType Ty) const905 ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
906   Ty = useFirstFieldIfTransparentUnion(Ty);
907 
908   if (isAggregateTypeForABI(Ty)) {
909     // Records with non-trivial destructors/copy-constructors should not be
910     // passed by value.
911     if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
912       return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
913     // Ignore empty structs/unions.
914     if (isEmptyRecord(getContext(), Ty, true))
915       return ABIArgInfo::getIgnore();
916     // Lower single-element structs to just pass a regular value. TODO: We
917     // could do reasonable-size multiple-element structs too, using getExpand(),
918     // though watch out for things like bitfields.
919     if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
920       return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
921     // For the experimental multivalue ABI, fully expand all other aggregates
922     if (Kind == ABIKind::ExperimentalMV) {
923       const RecordType *RT = Ty->getAs<RecordType>();
924       assert(RT);
925       bool HasBitField = false;
926       for (auto *Field : RT->getDecl()->fields()) {
927         if (Field->isBitField()) {
928           HasBitField = true;
929           break;
930         }
931       }
932       if (!HasBitField)
933         return ABIArgInfo::getExpand();
934     }
935   }
936 
937   // Otherwise just do the default thing.
938   return defaultInfo.classifyArgumentType(Ty);
939 }
940 
classifyReturnType(QualType RetTy) const941 ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const {
942   if (isAggregateTypeForABI(RetTy)) {
943     // Records with non-trivial destructors/copy-constructors should not be
944     // returned by value.
945     if (!getRecordArgABI(RetTy, getCXXABI())) {
946       // Ignore empty structs/unions.
947       if (isEmptyRecord(getContext(), RetTy, true))
948         return ABIArgInfo::getIgnore();
949       // Lower single-element structs to just return a regular value. TODO: We
950       // could do reasonable-size multiple-element structs too, using
951       // ABIArgInfo::getDirect().
952       if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
953         return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
954       // For the experimental multivalue ABI, return all other aggregates
955       if (Kind == ABIKind::ExperimentalMV)
956         return ABIArgInfo::getDirect();
957     }
958   }
959 
960   // Otherwise just do the default thing.
961   return defaultInfo.classifyReturnType(RetTy);
962 }
963 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const964 Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
965                                       QualType Ty) const {
966   bool IsIndirect = isAggregateTypeForABI(Ty) &&
967                     !isEmptyRecord(getContext(), Ty, true) &&
968                     !isSingleElementStruct(Ty, getContext());
969   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
970                           getContext().getTypeInfoInChars(Ty),
971                           CharUnits::fromQuantity(4),
972                           /*AllowHigherAlign=*/true);
973 }
974 
975 //===----------------------------------------------------------------------===//
976 // le32/PNaCl bitcode ABI Implementation
977 //
978 // This is a simplified version of the x86_32 ABI.  Arguments and return values
979 // are always passed on the stack.
980 //===----------------------------------------------------------------------===//
981 
982 class PNaClABIInfo : public ABIInfo {
983  public:
PNaClABIInfo(CodeGen::CodeGenTypes & CGT)984   PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
985 
986   ABIArgInfo classifyReturnType(QualType RetTy) const;
987   ABIArgInfo classifyArgumentType(QualType RetTy) const;
988 
989   void computeInfo(CGFunctionInfo &FI) const override;
990   Address EmitVAArg(CodeGenFunction &CGF,
991                     Address VAListAddr, QualType Ty) const override;
992 };
993 
994 class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
995  public:
PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT)996    PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
997        : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {}
998 };
999 
computeInfo(CGFunctionInfo & FI) const1000 void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
1001   if (!getCXXABI().classifyReturnType(FI))
1002     FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
1003 
1004   for (auto &I : FI.arguments())
1005     I.info = classifyArgumentType(I.type);
1006 }
1007 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const1008 Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1009                                 QualType Ty) const {
1010   // The PNaCL ABI is a bit odd, in that varargs don't use normal
1011   // function classification. Structs get passed directly for varargs
1012   // functions, through a rewriting transform in
1013   // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows
1014   // this target to actually support a va_arg instructions with an
1015   // aggregate type, unlike other targets.
1016   return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
1017 }
1018 
1019 /// Classify argument of given type \p Ty.
classifyArgumentType(QualType Ty) const1020 ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
1021   if (isAggregateTypeForABI(Ty)) {
1022     if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
1023       return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
1024     return getNaturalAlignIndirect(Ty);
1025   } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
1026     // Treat an enum type as its underlying type.
1027     Ty = EnumTy->getDecl()->getIntegerType();
1028   } else if (Ty->isFloatingType()) {
1029     // Floating-point types don't go inreg.
1030     return ABIArgInfo::getDirect();
1031   } else if (const auto *EIT = Ty->getAs<BitIntType>()) {
1032     // Treat bit-precise integers as integers if <= 64, otherwise pass
1033     // indirectly.
1034     if (EIT->getNumBits() > 64)
1035       return getNaturalAlignIndirect(Ty);
1036     return ABIArgInfo::getDirect();
1037   }
1038 
1039   return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
1040                                             : ABIArgInfo::getDirect());
1041 }
1042 
classifyReturnType(QualType RetTy) const1043 ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
1044   if (RetTy->isVoidType())
1045     return ABIArgInfo::getIgnore();
1046 
1047   // In the PNaCl ABI we always return records/structures on the stack.
1048   if (isAggregateTypeForABI(RetTy))
1049     return getNaturalAlignIndirect(RetTy);
1050 
1051   // Treat bit-precise integers as integers if <= 64, otherwise pass indirectly.
1052   if (const auto *EIT = RetTy->getAs<BitIntType>()) {
1053     if (EIT->getNumBits() > 64)
1054       return getNaturalAlignIndirect(RetTy);
1055     return ABIArgInfo::getDirect();
1056   }
1057 
1058   // Treat an enum type as its underlying type.
1059   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1060     RetTy = EnumTy->getDecl()->getIntegerType();
1061 
1062   return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1063                                                : ABIArgInfo::getDirect());
1064 }
1065 
1066 /// IsX86_MMXType - Return true if this is an MMX type.
IsX86_MMXType(llvm::Type * IRType)1067 bool IsX86_MMXType(llvm::Type *IRType) {
1068   // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
1069   return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
1070     cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy() &&
1071     IRType->getScalarSizeInBits() != 64;
1072 }
1073 
X86AdjustInlineAsmType(CodeGen::CodeGenFunction & CGF,StringRef Constraint,llvm::Type * Ty)1074 static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1075                                           StringRef Constraint,
1076                                           llvm::Type* Ty) {
1077   bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
1078                      .Cases("y", "&y", "^Ym", true)
1079                      .Default(false);
1080   if (IsMMXCons && Ty->isVectorTy()) {
1081     if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedValue() !=
1082         64) {
1083       // Invalid MMX constraint
1084       return nullptr;
1085     }
1086 
1087     return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
1088   }
1089 
1090   // No operation needed
1091   return Ty;
1092 }
1093 
1094 /// Returns true if this type can be passed in SSE registers with the
1095 /// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
isX86VectorTypeForVectorCall(ASTContext & Context,QualType Ty)1096 static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
1097   if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1098     if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {
1099       if (BT->getKind() == BuiltinType::LongDouble) {
1100         if (&Context.getTargetInfo().getLongDoubleFormat() ==
1101             &llvm::APFloat::x87DoubleExtended())
1102           return false;
1103       }
1104       return true;
1105     }
1106   } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
1107     // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
1108     // registers specially.
1109     unsigned VecSize = Context.getTypeSize(VT);
1110     if (VecSize == 128 || VecSize == 256 || VecSize == 512)
1111       return true;
1112   }
1113   return false;
1114 }
1115 
1116 /// Returns true if this aggregate is small enough to be passed in SSE registers
1117 /// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
isX86VectorCallAggregateSmallEnough(uint64_t NumMembers)1118 static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
1119   return NumMembers <= 4;
1120 }
1121 
1122 /// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
getDirectX86Hva(llvm::Type * T=nullptr)1123 static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
1124   auto AI = ABIArgInfo::getDirect(T);
1125   AI.setInReg(true);
1126   AI.setCanBeFlattened(false);
1127   return AI;
1128 }
1129 
1130 //===----------------------------------------------------------------------===//
1131 // X86-32 ABI Implementation
1132 //===----------------------------------------------------------------------===//
1133 
1134 /// Similar to llvm::CCState, but for Clang.
1135 struct CCState {
CCState__anondfff523e0111::CCState1136   CCState(CGFunctionInfo &FI)
1137       : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()) {}
1138 
1139   llvm::SmallBitVector IsPreassigned;
1140   unsigned CC = CallingConv::CC_C;
1141   unsigned FreeRegs = 0;
1142   unsigned FreeSSERegs = 0;
1143 };
1144 
1145 /// X86_32ABIInfo - The X86-32 ABI information.
1146 class X86_32ABIInfo : public ABIInfo {
1147   enum Class {
1148     Integer,
1149     Float
1150   };
1151 
1152   static const unsigned MinABIStackAlignInBytes = 4;
1153 
1154   bool IsDarwinVectorABI;
1155   bool IsRetSmallStructInRegABI;
1156   bool IsWin32StructABI;
1157   bool IsSoftFloatABI;
1158   bool IsMCUABI;
1159   bool IsLinuxABI;
1160   unsigned DefaultNumRegisterParameters;
1161 
isRegisterSize(unsigned Size)1162   static bool isRegisterSize(unsigned Size) {
1163     return (Size == 8 || Size == 16 || Size == 32 || Size == 64);
1164   }
1165 
isHomogeneousAggregateBaseType(QualType Ty) const1166   bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1167     // FIXME: Assumes vectorcall is in use.
1168     return isX86VectorTypeForVectorCall(getContext(), Ty);
1169   }
1170 
isHomogeneousAggregateSmallEnough(const Type * Ty,uint64_t NumMembers) const1171   bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1172                                          uint64_t NumMembers) const override {
1173     // FIXME: Assumes vectorcall is in use.
1174     return isX86VectorCallAggregateSmallEnough(NumMembers);
1175   }
1176 
1177   bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
1178 
1179   /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1180   /// such that the argument will be passed in memory.
1181   ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
1182 
1183   ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
1184 
1185   /// Return the alignment to use for the given type on the stack.
1186   unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
1187 
1188   Class classify(QualType Ty) const;
1189   ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
1190   ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
1191 
1192   /// Updates the number of available free registers, returns
1193   /// true if any registers were allocated.
1194   bool updateFreeRegs(QualType Ty, CCState &State) const;
1195 
1196   bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
1197                                 bool &NeedsPadding) const;
1198   bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
1199 
1200   bool canExpandIndirectArgument(QualType Ty) const;
1201 
1202   /// Rewrite the function info so that all memory arguments use
1203   /// inalloca.
1204   void rewriteWithInAlloca(CGFunctionInfo &FI) const;
1205 
1206   void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1207                            CharUnits &StackOffset, ABIArgInfo &Info,
1208                            QualType Type) const;
1209   void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
1210 
1211 public:
1212 
1213   void computeInfo(CGFunctionInfo &FI) const override;
1214   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1215                     QualType Ty) const override;
1216 
X86_32ABIInfo(CodeGen::CodeGenTypes & CGT,bool DarwinVectorABI,bool RetSmallStructInRegABI,bool Win32StructABI,unsigned NumRegisterParameters,bool SoftFloatABI)1217   X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1218                 bool RetSmallStructInRegABI, bool Win32StructABI,
1219                 unsigned NumRegisterParameters, bool SoftFloatABI)
1220       : ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
1221         IsRetSmallStructInRegABI(RetSmallStructInRegABI),
1222         IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
1223         IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
1224         IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
1225                    CGT.getTarget().getTriple().isOSCygMing()),
1226         DefaultNumRegisterParameters(NumRegisterParameters) {}
1227 };
1228 
1229 class X86_32SwiftABIInfo : public SwiftABIInfo {
1230 public:
X86_32SwiftABIInfo(CodeGenTypes & CGT)1231   explicit X86_32SwiftABIInfo(CodeGenTypes &CGT)
1232       : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/false) {}
1233 
shouldPassIndirectly(ArrayRef<llvm::Type * > ComponentTys,bool AsReturnValue) const1234   bool shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,
1235                             bool AsReturnValue) const override {
1236     // LLVM's x86-32 lowering currently only assigns up to three
1237     // integer registers and three fp registers.  Oddly, it'll use up to
1238     // four vector registers for vectors, but those can overlap with the
1239     // scalar registers.
1240     return occupiesMoreThan(CGT, ComponentTys, /*total=*/3);
1241   }
1242 };
1243 
1244 class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
1245 public:
X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,bool DarwinVectorABI,bool RetSmallStructInRegABI,bool Win32StructABI,unsigned NumRegisterParameters,bool SoftFloatABI)1246   X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1247                           bool RetSmallStructInRegABI, bool Win32StructABI,
1248                           unsigned NumRegisterParameters, bool SoftFloatABI)
1249       : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
1250             CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
1251             NumRegisterParameters, SoftFloatABI)) {
1252     SwiftInfo = std::make_unique<X86_32SwiftABIInfo>(CGT);
1253   }
1254 
1255   static bool isStructReturnInRegABI(
1256       const llvm::Triple &Triple, const CodeGenOptions &Opts);
1257 
1258   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1259                            CodeGen::CodeGenModule &CGM) const override;
1260 
getDwarfEHStackPointer(CodeGen::CodeGenModule & CGM) const1261   int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1262     // Darwin uses different dwarf register numbers for EH.
1263     if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
1264     return 4;
1265   }
1266 
1267   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1268                                llvm::Value *Address) const override;
1269 
adjustInlineAsmType(CodeGen::CodeGenFunction & CGF,StringRef Constraint,llvm::Type * Ty) const1270   llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1271                                   StringRef Constraint,
1272                                   llvm::Type* Ty) const override {
1273     return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1274   }
1275 
1276   void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
1277                                 std::string &Constraints,
1278                                 std::vector<llvm::Type *> &ResultRegTypes,
1279                                 std::vector<llvm::Type *> &ResultTruncRegTypes,
1280                                 std::vector<LValue> &ResultRegDests,
1281                                 std::string &AsmString,
1282                                 unsigned NumOutputs) const override;
1283 
1284   llvm::Constant *
getUBSanFunctionSignature(CodeGen::CodeGenModule & CGM) const1285   getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
1286     unsigned Sig = (0xeb << 0) |  // jmp rel8
1287                    (0x06 << 8) |  //           .+0x08
1288                    ('v' << 16) |
1289                    ('2' << 24);
1290     return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
1291   }
1292 
getARCRetainAutoreleasedReturnValueMarker() const1293   StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
1294     return "movl\t%ebp, %ebp"
1295            "\t\t// marker for objc_retainAutoreleaseReturnValue";
1296   }
1297 };
1298 
1299 }
1300 
1301 /// Rewrite input constraint references after adding some output constraints.
1302 /// In the case where there is one output and one input and we add one output,
1303 /// we need to replace all operand references greater than or equal to 1:
1304 ///     mov $0, $1
1305 ///     mov eax, $1
1306 /// The result will be:
1307 ///     mov $0, $2
1308 ///     mov eax, $2
rewriteInputConstraintReferences(unsigned FirstIn,unsigned NumNewOuts,std::string & AsmString)1309 static void rewriteInputConstraintReferences(unsigned FirstIn,
1310                                              unsigned NumNewOuts,
1311                                              std::string &AsmString) {
1312   std::string Buf;
1313   llvm::raw_string_ostream OS(Buf);
1314   size_t Pos = 0;
1315   while (Pos < AsmString.size()) {
1316     size_t DollarStart = AsmString.find('$', Pos);
1317     if (DollarStart == std::string::npos)
1318       DollarStart = AsmString.size();
1319     size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);
1320     if (DollarEnd == std::string::npos)
1321       DollarEnd = AsmString.size();
1322     OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
1323     Pos = DollarEnd;
1324     size_t NumDollars = DollarEnd - DollarStart;
1325     if (NumDollars % 2 != 0 && Pos < AsmString.size()) {
1326       // We have an operand reference.
1327       size_t DigitStart = Pos;
1328       if (AsmString[DigitStart] == '{') {
1329         OS << '{';
1330         ++DigitStart;
1331       }
1332       size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);
1333       if (DigitEnd == std::string::npos)
1334         DigitEnd = AsmString.size();
1335       StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
1336       unsigned OperandIndex;
1337       if (!OperandStr.getAsInteger(10, OperandIndex)) {
1338         if (OperandIndex >= FirstIn)
1339           OperandIndex += NumNewOuts;
1340         OS << OperandIndex;
1341       } else {
1342         OS << OperandStr;
1343       }
1344       Pos = DigitEnd;
1345     }
1346   }
1347   AsmString = std::move(OS.str());
1348 }
1349 
1350 /// Add output constraints for EAX:EDX because they are return registers.
addReturnRegisterOutputs(CodeGenFunction & CGF,LValue ReturnSlot,std::string & Constraints,std::vector<llvm::Type * > & ResultRegTypes,std::vector<llvm::Type * > & ResultTruncRegTypes,std::vector<LValue> & ResultRegDests,std::string & AsmString,unsigned NumOutputs) const1351 void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
1352     CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
1353     std::vector<llvm::Type *> &ResultRegTypes,
1354     std::vector<llvm::Type *> &ResultTruncRegTypes,
1355     std::vector<LValue> &ResultRegDests, std::string &AsmString,
1356     unsigned NumOutputs) const {
1357   uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());
1358 
1359   // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
1360   // larger.
1361   if (!Constraints.empty())
1362     Constraints += ',';
1363   if (RetWidth <= 32) {
1364     Constraints += "={eax}";
1365     ResultRegTypes.push_back(CGF.Int32Ty);
1366   } else {
1367     // Use the 'A' constraint for EAX:EDX.
1368     Constraints += "=A";
1369     ResultRegTypes.push_back(CGF.Int64Ty);
1370   }
1371 
1372   // Truncate EAX or EAX:EDX to an integer of the appropriate size.
1373   llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);
1374   ResultTruncRegTypes.push_back(CoerceTy);
1375 
1376   // Coerce the integer by bitcasting the return slot pointer.
1377   ReturnSlot.setAddress(
1378       CGF.Builder.CreateElementBitCast(ReturnSlot.getAddress(CGF), CoerceTy));
1379   ResultRegDests.push_back(ReturnSlot);
1380 
1381   rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
1382 }
1383 
1384 /// shouldReturnTypeInRegister - Determine if the given type should be
1385 /// returned in a register (for the Darwin and MCU ABI).
shouldReturnTypeInRegister(QualType Ty,ASTContext & Context) const1386 bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
1387                                                ASTContext &Context) const {
1388   uint64_t Size = Context.getTypeSize(Ty);
1389 
1390   // For i386, type must be register sized.
1391   // For the MCU ABI, it only needs to be <= 8-byte
1392   if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))
1393    return false;
1394 
1395   if (Ty->isVectorType()) {
1396     // 64- and 128- bit vectors inside structures are not returned in
1397     // registers.
1398     if (Size == 64 || Size == 128)
1399       return false;
1400 
1401     return true;
1402   }
1403 
1404   // If this is a builtin, pointer, enum, complex type, member pointer, or
1405   // member function pointer it is ok.
1406   if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||
1407       Ty->isAnyComplexType() || Ty->isEnumeralType() ||
1408       Ty->isBlockPointerType() || Ty->isMemberPointerType())
1409     return true;
1410 
1411   // Arrays are treated like records.
1412   if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))
1413     return shouldReturnTypeInRegister(AT->getElementType(), Context);
1414 
1415   // Otherwise, it must be a record type.
1416   const RecordType *RT = Ty->getAs<RecordType>();
1417   if (!RT) return false;
1418 
1419   // FIXME: Traverse bases here too.
1420 
1421   // Structure types are passed in register if all fields would be
1422   // passed in a register.
1423   for (const auto *FD : RT->getDecl()->fields()) {
1424     // Empty fields are ignored.
1425     if (isEmptyField(Context, FD, true))
1426       continue;
1427 
1428     // Check fields recursively.
1429     if (!shouldReturnTypeInRegister(FD->getType(), Context))
1430       return false;
1431   }
1432   return true;
1433 }
1434 
is32Or64BitBasicType(QualType Ty,ASTContext & Context)1435 static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
1436   // Treat complex types as the element type.
1437   if (const ComplexType *CTy = Ty->getAs<ComplexType>())
1438     Ty = CTy->getElementType();
1439 
1440   // Check for a type which we know has a simple scalar argument-passing
1441   // convention without any padding.  (We're specifically looking for 32
1442   // and 64-bit integer and integer-equivalents, float, and double.)
1443   if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&
1444       !Ty->isEnumeralType() && !Ty->isBlockPointerType())
1445     return false;
1446 
1447   uint64_t Size = Context.getTypeSize(Ty);
1448   return Size == 32 || Size == 64;
1449 }
1450 
addFieldSizes(ASTContext & Context,const RecordDecl * RD,uint64_t & Size)1451 static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
1452                           uint64_t &Size) {
1453   for (const auto *FD : RD->fields()) {
1454     // Scalar arguments on the stack get 4 byte alignment on x86. If the
1455     // argument is smaller than 32-bits, expanding the struct will create
1456     // alignment padding.
1457     if (!is32Or64BitBasicType(FD->getType(), Context))
1458       return false;
1459 
1460     // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
1461     // how to expand them yet, and the predicate for telling if a bitfield still
1462     // counts as "basic" is more complicated than what we were doing previously.
1463     if (FD->isBitField())
1464       return false;
1465 
1466     Size += Context.getTypeSize(FD->getType());
1467   }
1468   return true;
1469 }
1470 
addBaseAndFieldSizes(ASTContext & Context,const CXXRecordDecl * RD,uint64_t & Size)1471 static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
1472                                  uint64_t &Size) {
1473   // Don't do this if there are any non-empty bases.
1474   for (const CXXBaseSpecifier &Base : RD->bases()) {
1475     if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),
1476                               Size))
1477       return false;
1478   }
1479   if (!addFieldSizes(Context, RD, Size))
1480     return false;
1481   return true;
1482 }
1483 
1484 /// Test whether an argument type which is to be passed indirectly (on the
1485 /// stack) would have the equivalent layout if it was expanded into separate
1486 /// arguments. If so, we prefer to do the latter to avoid inhibiting
1487 /// optimizations.
canExpandIndirectArgument(QualType Ty) const1488 bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
1489   // We can only expand structure types.
1490   const RecordType *RT = Ty->getAs<RecordType>();
1491   if (!RT)
1492     return false;
1493   const RecordDecl *RD = RT->getDecl();
1494   uint64_t Size = 0;
1495   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1496     if (!IsWin32StructABI) {
1497       // On non-Windows, we have to conservatively match our old bitcode
1498       // prototypes in order to be ABI-compatible at the bitcode level.
1499       if (!CXXRD->isCLike())
1500         return false;
1501     } else {
1502       // Don't do this for dynamic classes.
1503       if (CXXRD->isDynamicClass())
1504         return false;
1505     }
1506     if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))
1507       return false;
1508   } else {
1509     if (!addFieldSizes(getContext(), RD, Size))
1510       return false;
1511   }
1512 
1513   // We can do this if there was no alignment padding.
1514   return Size == getContext().getTypeSize(Ty);
1515 }
1516 
getIndirectReturnResult(QualType RetTy,CCState & State) const1517 ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
1518   // If the return value is indirect, then the hidden argument is consuming one
1519   // integer register.
1520   if (State.FreeRegs) {
1521     --State.FreeRegs;
1522     if (!IsMCUABI)
1523       return getNaturalAlignIndirectInReg(RetTy);
1524   }
1525   return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
1526 }
1527 
classifyReturnType(QualType RetTy,CCState & State) const1528 ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
1529                                              CCState &State) const {
1530   if (RetTy->isVoidType())
1531     return ABIArgInfo::getIgnore();
1532 
1533   const Type *Base = nullptr;
1534   uint64_t NumElts = 0;
1535   if ((State.CC == llvm::CallingConv::X86_VectorCall ||
1536        State.CC == llvm::CallingConv::X86_RegCall) &&
1537       isHomogeneousAggregate(RetTy, Base, NumElts)) {
1538     // The LLVM struct type for such an aggregate should lower properly.
1539     return ABIArgInfo::getDirect();
1540   }
1541 
1542   if (const VectorType *VT = RetTy->getAs<VectorType>()) {
1543     // On Darwin, some vectors are returned in registers.
1544     if (IsDarwinVectorABI) {
1545       uint64_t Size = getContext().getTypeSize(RetTy);
1546 
1547       // 128-bit vectors are a special case; they are returned in
1548       // registers and we need to make sure to pick a type the LLVM
1549       // backend will like.
1550       if (Size == 128)
1551         return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1552             llvm::Type::getInt64Ty(getVMContext()), 2));
1553 
1554       // Always return in register if it fits in a general purpose
1555       // register, or if it is 64 bits and has a single element.
1556       if ((Size == 8 || Size == 16 || Size == 32) ||
1557           (Size == 64 && VT->getNumElements() == 1))
1558         return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
1559                                                             Size));
1560 
1561       return getIndirectReturnResult(RetTy, State);
1562     }
1563 
1564     return ABIArgInfo::getDirect();
1565   }
1566 
1567   if (isAggregateTypeForABI(RetTy)) {
1568     if (const RecordType *RT = RetTy->getAs<RecordType>()) {
1569       // Structures with flexible arrays are always indirect.
1570       if (RT->getDecl()->hasFlexibleArrayMember())
1571         return getIndirectReturnResult(RetTy, State);
1572     }
1573 
1574     // If specified, structs and unions are always indirect.
1575     if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
1576       return getIndirectReturnResult(RetTy, State);
1577 
1578     // Ignore empty structs/unions.
1579     if (isEmptyRecord(getContext(), RetTy, true))
1580       return ABIArgInfo::getIgnore();
1581 
1582     // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
1583     if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
1584       QualType ET = getContext().getCanonicalType(CT->getElementType());
1585       if (ET->isFloat16Type())
1586         return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1587             llvm::Type::getHalfTy(getVMContext()), 2));
1588     }
1589 
1590     // Small structures which are register sized are generally returned
1591     // in a register.
1592     if (shouldReturnTypeInRegister(RetTy, getContext())) {
1593       uint64_t Size = getContext().getTypeSize(RetTy);
1594 
1595       // As a special-case, if the struct is a "single-element" struct, and
1596       // the field is of type "float" or "double", return it in a
1597       // floating-point register. (MSVC does not apply this special case.)
1598       // We apply a similar transformation for pointer types to improve the
1599       // quality of the generated IR.
1600       if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
1601         if ((!IsWin32StructABI && SeltTy->isRealFloatingType())
1602             || SeltTy->hasPointerRepresentation())
1603           return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
1604 
1605       // FIXME: We should be able to narrow this integer in cases with dead
1606       // padding.
1607       return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
1608     }
1609 
1610     return getIndirectReturnResult(RetTy, State);
1611   }
1612 
1613   // Treat an enum type as its underlying type.
1614   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1615     RetTy = EnumTy->getDecl()->getIntegerType();
1616 
1617   if (const auto *EIT = RetTy->getAs<BitIntType>())
1618     if (EIT->getNumBits() > 64)
1619       return getIndirectReturnResult(RetTy, State);
1620 
1621   return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1622                                                : ABIArgInfo::getDirect());
1623 }
1624 
isSIMDVectorType(ASTContext & Context,QualType Ty)1625 static bool isSIMDVectorType(ASTContext &Context, QualType Ty) {
1626   return Ty->getAs<VectorType>() && Context.getTypeSize(Ty) == 128;
1627 }
1628 
isRecordWithSIMDVectorType(ASTContext & Context,QualType Ty)1629 static bool isRecordWithSIMDVectorType(ASTContext &Context, QualType Ty) {
1630   const RecordType *RT = Ty->getAs<RecordType>();
1631   if (!RT)
1632     return false;
1633   const RecordDecl *RD = RT->getDecl();
1634 
1635   // If this is a C++ record, check the bases first.
1636   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
1637     for (const auto &I : CXXRD->bases())
1638       if (!isRecordWithSIMDVectorType(Context, I.getType()))
1639         return false;
1640 
1641   for (const auto *i : RD->fields()) {
1642     QualType FT = i->getType();
1643 
1644     if (isSIMDVectorType(Context, FT))
1645       return true;
1646 
1647     if (isRecordWithSIMDVectorType(Context, FT))
1648       return true;
1649   }
1650 
1651   return false;
1652 }
1653 
getTypeStackAlignInBytes(QualType Ty,unsigned Align) const1654 unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
1655                                                  unsigned Align) const {
1656   // Otherwise, if the alignment is less than or equal to the minimum ABI
1657   // alignment, just use the default; the backend will handle this.
1658   if (Align <= MinABIStackAlignInBytes)
1659     return 0; // Use default alignment.
1660 
1661   if (IsLinuxABI) {
1662     // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
1663     // want to spend any effort dealing with the ramifications of ABI breaks.
1664     //
1665     // If the vector type is __m128/__m256/__m512, return the default alignment.
1666     if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))
1667       return Align;
1668   }
1669   // On non-Darwin, the stack type alignment is always 4.
1670   if (!IsDarwinVectorABI) {
1671     // Set explicit alignment, since we may need to realign the top.
1672     return MinABIStackAlignInBytes;
1673   }
1674 
1675   // Otherwise, if the type contains an SSE vector type, the alignment is 16.
1676   if (Align >= 16 && (isSIMDVectorType(getContext(), Ty) ||
1677                       isRecordWithSIMDVectorType(getContext(), Ty)))
1678     return 16;
1679 
1680   return MinABIStackAlignInBytes;
1681 }
1682 
getIndirectResult(QualType Ty,bool ByVal,CCState & State) const1683 ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
1684                                             CCState &State) const {
1685   if (!ByVal) {
1686     if (State.FreeRegs) {
1687       --State.FreeRegs; // Non-byval indirects just use one pointer.
1688       if (!IsMCUABI)
1689         return getNaturalAlignIndirectInReg(Ty);
1690     }
1691     return getNaturalAlignIndirect(Ty, false);
1692   }
1693 
1694   // Compute the byval alignment.
1695   unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
1696   unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
1697   if (StackAlign == 0)
1698     return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
1699 
1700   // If the stack alignment is less than the type alignment, realign the
1701   // argument.
1702   bool Realign = TypeAlign > StackAlign;
1703   return ABIArgInfo::getIndirect(CharUnits::fromQuantity(StackAlign),
1704                                  /*ByVal=*/true, Realign);
1705 }
1706 
classify(QualType Ty) const1707 X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
1708   const Type *T = isSingleElementStruct(Ty, getContext());
1709   if (!T)
1710     T = Ty.getTypePtr();
1711 
1712   if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
1713     BuiltinType::Kind K = BT->getKind();
1714     if (K == BuiltinType::Float || K == BuiltinType::Double)
1715       return Float;
1716   }
1717   return Integer;
1718 }
1719 
updateFreeRegs(QualType Ty,CCState & State) const1720 bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
1721   if (!IsSoftFloatABI) {
1722     Class C = classify(Ty);
1723     if (C == Float)
1724       return false;
1725   }
1726 
1727   unsigned Size = getContext().getTypeSize(Ty);
1728   unsigned SizeInRegs = (Size + 31) / 32;
1729 
1730   if (SizeInRegs == 0)
1731     return false;
1732 
1733   if (!IsMCUABI) {
1734     if (SizeInRegs > State.FreeRegs) {
1735       State.FreeRegs = 0;
1736       return false;
1737     }
1738   } else {
1739     // The MCU psABI allows passing parameters in-reg even if there are
1740     // earlier parameters that are passed on the stack. Also,
1741     // it does not allow passing >8-byte structs in-register,
1742     // even if there are 3 free registers available.
1743     if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
1744       return false;
1745   }
1746 
1747   State.FreeRegs -= SizeInRegs;
1748   return true;
1749 }
1750 
shouldAggregateUseDirect(QualType Ty,CCState & State,bool & InReg,bool & NeedsPadding) const1751 bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
1752                                              bool &InReg,
1753                                              bool &NeedsPadding) const {
1754   // On Windows, aggregates other than HFAs are never passed in registers, and
1755   // they do not consume register slots. Homogenous floating-point aggregates
1756   // (HFAs) have already been dealt with at this point.
1757   if (IsWin32StructABI && isAggregateTypeForABI(Ty))
1758     return false;
1759 
1760   NeedsPadding = false;
1761   InReg = !IsMCUABI;
1762 
1763   if (!updateFreeRegs(Ty, State))
1764     return false;
1765 
1766   if (IsMCUABI)
1767     return true;
1768 
1769   if (State.CC == llvm::CallingConv::X86_FastCall ||
1770       State.CC == llvm::CallingConv::X86_VectorCall ||
1771       State.CC == llvm::CallingConv::X86_RegCall) {
1772     if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)
1773       NeedsPadding = true;
1774 
1775     return false;
1776   }
1777 
1778   return true;
1779 }
1780 
shouldPrimitiveUseInReg(QualType Ty,CCState & State) const1781 bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
1782   bool IsPtrOrInt = (getContext().getTypeSize(Ty) <= 32) &&
1783                     (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
1784                      Ty->isReferenceType());
1785 
1786   if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||
1787                       State.CC == llvm::CallingConv::X86_VectorCall))
1788     return false;
1789 
1790   if (!updateFreeRegs(Ty, State))
1791     return false;
1792 
1793   if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)
1794     return false;
1795 
1796   // Return true to apply inreg to all legal parameters except for MCU targets.
1797   return !IsMCUABI;
1798 }
1799 
runVectorCallFirstPass(CGFunctionInfo & FI,CCState & State) const1800 void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
1801   // Vectorcall x86 works subtly different than in x64, so the format is
1802   // a bit different than the x64 version.  First, all vector types (not HVAs)
1803   // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
1804   // This differs from the x64 implementation, where the first 6 by INDEX get
1805   // registers.
1806   // In the second pass over the arguments, HVAs are passed in the remaining
1807   // vector registers if possible, or indirectly by address. The address will be
1808   // passed in ECX/EDX if available. Any other arguments are passed according to
1809   // the usual fastcall rules.
1810   MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1811   for (int I = 0, E = Args.size(); I < E; ++I) {
1812     const Type *Base = nullptr;
1813     uint64_t NumElts = 0;
1814     const QualType &Ty = Args[I].type;
1815     if ((Ty->isVectorType() || Ty->isBuiltinType()) &&
1816         isHomogeneousAggregate(Ty, Base, NumElts)) {
1817       if (State.FreeSSERegs >= NumElts) {
1818         State.FreeSSERegs -= NumElts;
1819         Args[I].info = ABIArgInfo::getDirectInReg();
1820         State.IsPreassigned.set(I);
1821       }
1822     }
1823   }
1824 }
1825 
classifyArgumentType(QualType Ty,CCState & State) const1826 ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
1827                                                CCState &State) const {
1828   // FIXME: Set alignment on indirect arguments.
1829   bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
1830   bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
1831   bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
1832 
1833   Ty = useFirstFieldIfTransparentUnion(Ty);
1834   TypeInfo TI = getContext().getTypeInfo(Ty);
1835 
1836   // Check with the C++ ABI first.
1837   const RecordType *RT = Ty->getAs<RecordType>();
1838   if (RT) {
1839     CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
1840     if (RAA == CGCXXABI::RAA_Indirect) {
1841       return getIndirectResult(Ty, false, State);
1842     } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
1843       // The field index doesn't matter, we'll fix it up later.
1844       return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
1845     }
1846   }
1847 
1848   // Regcall uses the concept of a homogenous vector aggregate, similar
1849   // to other targets.
1850   const Type *Base = nullptr;
1851   uint64_t NumElts = 0;
1852   if ((IsRegCall || IsVectorCall) &&
1853       isHomogeneousAggregate(Ty, Base, NumElts)) {
1854     if (State.FreeSSERegs >= NumElts) {
1855       State.FreeSSERegs -= NumElts;
1856 
1857       // Vectorcall passes HVAs directly and does not flatten them, but regcall
1858       // does.
1859       if (IsVectorCall)
1860         return getDirectX86Hva();
1861 
1862       if (Ty->isBuiltinType() || Ty->isVectorType())
1863         return ABIArgInfo::getDirect();
1864       return ABIArgInfo::getExpand();
1865     }
1866     return getIndirectResult(Ty, /*ByVal=*/false, State);
1867   }
1868 
1869   if (isAggregateTypeForABI(Ty)) {
1870     // Structures with flexible arrays are always indirect.
1871     // FIXME: This should not be byval!
1872     if (RT && RT->getDecl()->hasFlexibleArrayMember())
1873       return getIndirectResult(Ty, true, State);
1874 
1875     // Ignore empty structs/unions on non-Windows.
1876     if (!IsWin32StructABI && isEmptyRecord(getContext(), Ty, true))
1877       return ABIArgInfo::getIgnore();
1878 
1879     llvm::LLVMContext &LLVMContext = getVMContext();
1880     llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
1881     bool NeedsPadding = false;
1882     bool InReg;
1883     if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
1884       unsigned SizeInRegs = (TI.Width + 31) / 32;
1885       SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
1886       llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
1887       if (InReg)
1888         return ABIArgInfo::getDirectInReg(Result);
1889       else
1890         return ABIArgInfo::getDirect(Result);
1891     }
1892     llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
1893 
1894     // Pass over-aligned aggregates on Windows indirectly. This behavior was
1895     // added in MSVC 2015.
1896     if (IsWin32StructABI && TI.isAlignRequired() && TI.Align > 32)
1897       return getIndirectResult(Ty, /*ByVal=*/false, State);
1898 
1899     // Expand small (<= 128-bit) record types when we know that the stack layout
1900     // of those arguments will match the struct. This is important because the
1901     // LLVM backend isn't smart enough to remove byval, which inhibits many
1902     // optimizations.
1903     // Don't do this for the MCU if there are still free integer registers
1904     // (see X86_64 ABI for full explanation).
1905     if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
1906         canExpandIndirectArgument(Ty))
1907       return ABIArgInfo::getExpandWithPadding(
1908           IsFastCall || IsVectorCall || IsRegCall, PaddingType);
1909 
1910     return getIndirectResult(Ty, true, State);
1911   }
1912 
1913   if (const VectorType *VT = Ty->getAs<VectorType>()) {
1914     // On Windows, vectors are passed directly if registers are available, or
1915     // indirectly if not. This avoids the need to align argument memory. Pass
1916     // user-defined vector types larger than 512 bits indirectly for simplicity.
1917     if (IsWin32StructABI) {
1918       if (TI.Width <= 512 && State.FreeSSERegs > 0) {
1919         --State.FreeSSERegs;
1920         return ABIArgInfo::getDirectInReg();
1921       }
1922       return getIndirectResult(Ty, /*ByVal=*/false, State);
1923     }
1924 
1925     // On Darwin, some vectors are passed in memory, we handle this by passing
1926     // it as an i8/i16/i32/i64.
1927     if (IsDarwinVectorABI) {
1928       if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
1929           (TI.Width == 64 && VT->getNumElements() == 1))
1930         return ABIArgInfo::getDirect(
1931             llvm::IntegerType::get(getVMContext(), TI.Width));
1932     }
1933 
1934     if (IsX86_MMXType(CGT.ConvertType(Ty)))
1935       return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));
1936 
1937     return ABIArgInfo::getDirect();
1938   }
1939 
1940 
1941   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
1942     Ty = EnumTy->getDecl()->getIntegerType();
1943 
1944   bool InReg = shouldPrimitiveUseInReg(Ty, State);
1945 
1946   if (isPromotableIntegerTypeForABI(Ty)) {
1947     if (InReg)
1948       return ABIArgInfo::getExtendInReg(Ty);
1949     return ABIArgInfo::getExtend(Ty);
1950   }
1951 
1952   if (const auto *EIT = Ty->getAs<BitIntType>()) {
1953     if (EIT->getNumBits() <= 64) {
1954       if (InReg)
1955         return ABIArgInfo::getDirectInReg();
1956       return ABIArgInfo::getDirect();
1957     }
1958     return getIndirectResult(Ty, /*ByVal=*/false, State);
1959   }
1960 
1961   if (InReg)
1962     return ABIArgInfo::getDirectInReg();
1963   return ABIArgInfo::getDirect();
1964 }
1965 
computeInfo(CGFunctionInfo & FI) const1966 void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1967   CCState State(FI);
1968   if (IsMCUABI)
1969     State.FreeRegs = 3;
1970   else if (State.CC == llvm::CallingConv::X86_FastCall) {
1971     State.FreeRegs = 2;
1972     State.FreeSSERegs = 3;
1973   } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1974     State.FreeRegs = 2;
1975     State.FreeSSERegs = 6;
1976   } else if (FI.getHasRegParm())
1977     State.FreeRegs = FI.getRegParm();
1978   else if (State.CC == llvm::CallingConv::X86_RegCall) {
1979     State.FreeRegs = 5;
1980     State.FreeSSERegs = 8;
1981   } else if (IsWin32StructABI) {
1982     // Since MSVC 2015, the first three SSE vectors have been passed in
1983     // registers. The rest are passed indirectly.
1984     State.FreeRegs = DefaultNumRegisterParameters;
1985     State.FreeSSERegs = 3;
1986   } else
1987     State.FreeRegs = DefaultNumRegisterParameters;
1988 
1989   if (!::classifyReturnType(getCXXABI(), FI, *this)) {
1990     FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), State);
1991   } else if (FI.getReturnInfo().isIndirect()) {
1992     // The C++ ABI is not aware of register usage, so we have to check if the
1993     // return value was sret and put it in a register ourselves if appropriate.
1994     if (State.FreeRegs) {
1995       --State.FreeRegs;  // The sret parameter consumes a register.
1996       if (!IsMCUABI)
1997         FI.getReturnInfo().setInReg(true);
1998     }
1999   }
2000 
2001   // The chain argument effectively gives us another free register.
2002   if (FI.isChainCall())
2003     ++State.FreeRegs;
2004 
2005   // For vectorcall, do a first pass over the arguments, assigning FP and vector
2006   // arguments to XMM registers as available.
2007   if (State.CC == llvm::CallingConv::X86_VectorCall)
2008     runVectorCallFirstPass(FI, State);
2009 
2010   bool UsedInAlloca = false;
2011   MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
2012   for (int I = 0, E = Args.size(); I < E; ++I) {
2013     // Skip arguments that have already been assigned.
2014     if (State.IsPreassigned.test(I))
2015       continue;
2016 
2017     Args[I].info = classifyArgumentType(Args[I].type, State);
2018     UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
2019   }
2020 
2021   // If we needed to use inalloca for any argument, do a second pass and rewrite
2022   // all the memory arguments to use inalloca.
2023   if (UsedInAlloca)
2024     rewriteWithInAlloca(FI);
2025 }
2026 
2027 void
addFieldToArgStruct(SmallVector<llvm::Type *,6> & FrameFields,CharUnits & StackOffset,ABIArgInfo & Info,QualType Type) const2028 X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
2029                                    CharUnits &StackOffset, ABIArgInfo &Info,
2030                                    QualType Type) const {
2031   // Arguments are always 4-byte-aligned.
2032   CharUnits WordSize = CharUnits::fromQuantity(4);
2033   assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
2034 
2035   // sret pointers and indirect things will require an extra pointer
2036   // indirection, unless they are byval. Most things are byval, and will not
2037   // require this indirection.
2038   bool IsIndirect = false;
2039   if (Info.isIndirect() && !Info.getIndirectByVal())
2040     IsIndirect = true;
2041   Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
2042   llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
2043   if (IsIndirect)
2044     LLTy = LLTy->getPointerTo(0);
2045   FrameFields.push_back(LLTy);
2046   StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(Type);
2047 
2048   // Insert padding bytes to respect alignment.
2049   CharUnits FieldEnd = StackOffset;
2050   StackOffset = FieldEnd.alignTo(WordSize);
2051   if (StackOffset != FieldEnd) {
2052     CharUnits NumBytes = StackOffset - FieldEnd;
2053     llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
2054     Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
2055     FrameFields.push_back(Ty);
2056   }
2057 }
2058 
isArgInAlloca(const ABIArgInfo & Info)2059 static bool isArgInAlloca(const ABIArgInfo &Info) {
2060   // Leave ignored and inreg arguments alone.
2061   switch (Info.getKind()) {
2062   case ABIArgInfo::InAlloca:
2063     return true;
2064   case ABIArgInfo::Ignore:
2065   case ABIArgInfo::IndirectAliased:
2066     return false;
2067   case ABIArgInfo::Indirect:
2068   case ABIArgInfo::Direct:
2069   case ABIArgInfo::Extend:
2070     return !Info.getInReg();
2071   case ABIArgInfo::Expand:
2072   case ABIArgInfo::CoerceAndExpand:
2073     // These are aggregate types which are never passed in registers when
2074     // inalloca is involved.
2075     return true;
2076   }
2077   llvm_unreachable("invalid enum");
2078 }
2079 
rewriteWithInAlloca(CGFunctionInfo & FI) const2080 void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
2081   assert(IsWin32StructABI && "inalloca only supported on win32");
2082 
2083   // Build a packed struct type for all of the arguments in memory.
2084   SmallVector<llvm::Type *, 6> FrameFields;
2085 
2086   // The stack alignment is always 4.
2087   CharUnits StackAlign = CharUnits::fromQuantity(4);
2088 
2089   CharUnits StackOffset;
2090   CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2091 
2092   // Put 'this' into the struct before 'sret', if necessary.
2093   bool IsThisCall =
2094       FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
2095   ABIArgInfo &Ret = FI.getReturnInfo();
2096   if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&
2097       isArgInAlloca(I->info)) {
2098     addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2099     ++I;
2100   }
2101 
2102   // Put the sret parameter into the inalloca struct if it's in memory.
2103   if (Ret.isIndirect() && !Ret.getInReg()) {
2104     addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
2105     // On Windows, the hidden sret parameter is always returned in eax.
2106     Ret.setInAllocaSRet(IsWin32StructABI);
2107   }
2108 
2109   // Skip the 'this' parameter in ecx.
2110   if (IsThisCall)
2111     ++I;
2112 
2113   // Put arguments passed in memory into the struct.
2114   for (; I != E; ++I) {
2115     if (isArgInAlloca(I->info))
2116       addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2117   }
2118 
2119   FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
2120                                         /*isPacked=*/true),
2121                   StackAlign);
2122 }
2123 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const2124 Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
2125                                  Address VAListAddr, QualType Ty) const {
2126 
2127   auto TypeInfo = getContext().getTypeInfoInChars(Ty);
2128 
2129   // x86-32 changes the alignment of certain arguments on the stack.
2130   //
2131   // Just messing with TypeInfo like this works because we never pass
2132   // anything indirectly.
2133   TypeInfo.Align = CharUnits::fromQuantity(
2134                 getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));
2135 
2136   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
2137                           TypeInfo, CharUnits::fromQuantity(4),
2138                           /*AllowHigherAlign*/ true);
2139 }
2140 
isStructReturnInRegABI(const llvm::Triple & Triple,const CodeGenOptions & Opts)2141 bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
2142     const llvm::Triple &Triple, const CodeGenOptions &Opts) {
2143   assert(Triple.getArch() == llvm::Triple::x86);
2144 
2145   switch (Opts.getStructReturnConvention()) {
2146   case CodeGenOptions::SRCK_Default:
2147     break;
2148   case CodeGenOptions::SRCK_OnStack:  // -fpcc-struct-return
2149     return false;
2150   case CodeGenOptions::SRCK_InRegs:  // -freg-struct-return
2151     return true;
2152   }
2153 
2154   if (Triple.isOSDarwin() || Triple.isOSIAMCU())
2155     return true;
2156 
2157   switch (Triple.getOS()) {
2158   case llvm::Triple::DragonFly:
2159   case llvm::Triple::FreeBSD:
2160   case llvm::Triple::OpenBSD:
2161   case llvm::Triple::Win32:
2162     return true;
2163   default:
2164     return false;
2165   }
2166 }
2167 
addX86InterruptAttrs(const FunctionDecl * FD,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM)2168 static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
2169                                  CodeGen::CodeGenModule &CGM) {
2170   if (!FD->hasAttr<AnyX86InterruptAttr>())
2171     return;
2172 
2173   llvm::Function *Fn = cast<llvm::Function>(GV);
2174   Fn->setCallingConv(llvm::CallingConv::X86_INTR);
2175   if (FD->getNumParams() == 0)
2176     return;
2177 
2178   auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());
2179   llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());
2180   llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
2181     Fn->getContext(), ByValTy);
2182   Fn->addParamAttr(0, NewAttr);
2183 }
2184 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const2185 void X86_32TargetCodeGenInfo::setTargetAttributes(
2186     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2187   if (GV->isDeclaration())
2188     return;
2189   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2190     if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2191       llvm::Function *Fn = cast<llvm::Function>(GV);
2192       Fn->addFnAttr("stackrealign");
2193     }
2194 
2195     addX86InterruptAttrs(FD, GV, CGM);
2196   }
2197 }
2198 
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const2199 bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
2200                                                CodeGen::CodeGenFunction &CGF,
2201                                                llvm::Value *Address) const {
2202   CodeGen::CGBuilderTy &Builder = CGF.Builder;
2203 
2204   llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
2205 
2206   // 0-7 are the eight integer registers;  the order is different
2207   //   on Darwin (for EH), but the range is the same.
2208   // 8 is %eip.
2209   AssignToArrayRange(Builder, Address, Four8, 0, 8);
2210 
2211   if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
2212     // 12-16 are st(0..4).  Not sure why we stop at 4.
2213     // These have size 16, which is sizeof(long double) on
2214     // platforms with 8-byte alignment for that type.
2215     llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);
2216     AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
2217 
2218   } else {
2219     // 9 is %eflags, which doesn't get a size on Darwin for some
2220     // reason.
2221     Builder.CreateAlignedStore(
2222         Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
2223                                CharUnits::One());
2224 
2225     // 11-16 are st(0..5).  Not sure why we stop at 5.
2226     // These have size 12, which is sizeof(long double) on
2227     // platforms with 4-byte alignment for that type.
2228     llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);
2229     AssignToArrayRange(Builder, Address, Twelve8, 11, 16);
2230   }
2231 
2232   return false;
2233 }
2234 
2235 //===----------------------------------------------------------------------===//
2236 // X86-64 ABI Implementation
2237 //===----------------------------------------------------------------------===//
2238 
2239 
2240 namespace {
2241 /// The AVX ABI level for X86 targets.
2242 enum class X86AVXABILevel {
2243   None,
2244   AVX,
2245   AVX512
2246 };
2247 
2248 /// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel)2249 static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
2250   switch (AVXLevel) {
2251   case X86AVXABILevel::AVX512:
2252     return 512;
2253   case X86AVXABILevel::AVX:
2254     return 256;
2255   case X86AVXABILevel::None:
2256     return 128;
2257   }
2258   llvm_unreachable("Unknown AVXLevel");
2259 }
2260 
2261 /// X86_64ABIInfo - The X86_64 ABI information.
2262 class X86_64ABIInfo : public ABIInfo {
2263   enum Class {
2264     Integer = 0,
2265     SSE,
2266     SSEUp,
2267     X87,
2268     X87Up,
2269     ComplexX87,
2270     NoClass,
2271     Memory
2272   };
2273 
2274   /// merge - Implement the X86_64 ABI merging algorithm.
2275   ///
2276   /// Merge an accumulating classification \arg Accum with a field
2277   /// classification \arg Field.
2278   ///
2279   /// \param Accum - The accumulating classification. This should
2280   /// always be either NoClass or the result of a previous merge
2281   /// call. In addition, this should never be Memory (the caller
2282   /// should just return Memory for the aggregate).
2283   static Class merge(Class Accum, Class Field);
2284 
2285   /// postMerge - Implement the X86_64 ABI post merging algorithm.
2286   ///
2287   /// Post merger cleanup, reduces a malformed Hi and Lo pair to
2288   /// final MEMORY or SSE classes when necessary.
2289   ///
2290   /// \param AggregateSize - The size of the current aggregate in
2291   /// the classification process.
2292   ///
2293   /// \param Lo - The classification for the parts of the type
2294   /// residing in the low word of the containing object.
2295   ///
2296   /// \param Hi - The classification for the parts of the type
2297   /// residing in the higher words of the containing object.
2298   ///
2299   void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
2300 
2301   /// classify - Determine the x86_64 register classes in which the
2302   /// given type T should be passed.
2303   ///
2304   /// \param Lo - The classification for the parts of the type
2305   /// residing in the low word of the containing object.
2306   ///
2307   /// \param Hi - The classification for the parts of the type
2308   /// residing in the high word of the containing object.
2309   ///
2310   /// \param OffsetBase - The bit offset of this type in the
2311   /// containing object.  Some parameters are classified different
2312   /// depending on whether they straddle an eightbyte boundary.
2313   ///
2314   /// \param isNamedArg - Whether the argument in question is a "named"
2315   /// argument, as used in AMD64-ABI 3.5.7.
2316   ///
2317   /// \param IsRegCall - Whether the calling conversion is regcall.
2318   ///
2319   /// If a word is unused its result will be NoClass; if a type should
2320   /// be passed in Memory then at least the classification of \arg Lo
2321   /// will be Memory.
2322   ///
2323   /// The \arg Lo class will be NoClass iff the argument is ignored.
2324   ///
2325   /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
2326   /// also be ComplexX87.
2327   void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
2328                 bool isNamedArg, bool IsRegCall = false) const;
2329 
2330   llvm::Type *GetByteVectorType(QualType Ty) const;
2331   llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
2332                                  unsigned IROffset, QualType SourceTy,
2333                                  unsigned SourceOffset) const;
2334   llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
2335                                      unsigned IROffset, QualType SourceTy,
2336                                      unsigned SourceOffset) const;
2337 
2338   /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2339   /// such that the argument will be returned in memory.
2340   ABIArgInfo getIndirectReturnResult(QualType Ty) const;
2341 
2342   /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2343   /// such that the argument will be passed in memory.
2344   ///
2345   /// \param freeIntRegs - The number of free integer registers remaining
2346   /// available.
2347   ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
2348 
2349   ABIArgInfo classifyReturnType(QualType RetTy) const;
2350 
2351   ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2352                                   unsigned &neededInt, unsigned &neededSSE,
2353                                   bool isNamedArg,
2354                                   bool IsRegCall = false) const;
2355 
2356   ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2357                                        unsigned &NeededSSE,
2358                                        unsigned &MaxVectorWidth) const;
2359 
2360   ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2361                                            unsigned &NeededSSE,
2362                                            unsigned &MaxVectorWidth) const;
2363 
2364   bool IsIllegalVectorType(QualType Ty) const;
2365 
2366   /// The 0.98 ABI revision clarified a lot of ambiguities,
2367   /// unfortunately in ways that were not always consistent with
2368   /// certain previous compilers.  In particular, platforms which
2369   /// required strict binary compatibility with older versions of GCC
2370   /// may need to exempt themselves.
honorsRevision0_98() const2371   bool honorsRevision0_98() const {
2372     return !getTarget().getTriple().isOSDarwin();
2373   }
2374 
2375   /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
2376   /// classify it as INTEGER (for compatibility with older clang compilers).
classifyIntegerMMXAsSSE() const2377   bool classifyIntegerMMXAsSSE() const {
2378     // Clang <= 3.8 did not do this.
2379     if (getContext().getLangOpts().getClangABICompat() <=
2380         LangOptions::ClangABI::Ver3_8)
2381       return false;
2382 
2383     const llvm::Triple &Triple = getTarget().getTriple();
2384     if (Triple.isOSDarwin() || Triple.isPS() || Triple.isOSFreeBSD())
2385       return false;
2386     return true;
2387   }
2388 
2389   // GCC classifies vectors of __int128 as memory.
passInt128VectorsInMem() const2390   bool passInt128VectorsInMem() const {
2391     // Clang <= 9.0 did not do this.
2392     if (getContext().getLangOpts().getClangABICompat() <=
2393         LangOptions::ClangABI::Ver9)
2394       return false;
2395 
2396     const llvm::Triple &T = getTarget().getTriple();
2397     return T.isOSLinux() || T.isOSNetBSD();
2398   }
2399 
2400   X86AVXABILevel AVXLevel;
2401   // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
2402   // 64-bit hardware.
2403   bool Has64BitPointers;
2404 
2405 public:
X86_64ABIInfo(CodeGen::CodeGenTypes & CGT,X86AVXABILevel AVXLevel)2406   X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2407       : ABIInfo(CGT), AVXLevel(AVXLevel),
2408         Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {}
2409 
isPassedUsingAVXType(QualType type) const2410   bool isPassedUsingAVXType(QualType type) const {
2411     unsigned neededInt, neededSSE;
2412     // The freeIntRegs argument doesn't matter here.
2413     ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,
2414                                            /*isNamedArg*/true);
2415     if (info.isDirect()) {
2416       llvm::Type *ty = info.getCoerceToType();
2417       if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))
2418         return vectorTy->getPrimitiveSizeInBits().getFixedValue() > 128;
2419     }
2420     return false;
2421   }
2422 
2423   void computeInfo(CGFunctionInfo &FI) const override;
2424 
2425   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2426                     QualType Ty) const override;
2427   Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
2428                       QualType Ty) const override;
2429 
has64BitPointers() const2430   bool has64BitPointers() const {
2431     return Has64BitPointers;
2432   }
2433 };
2434 
2435 /// WinX86_64ABIInfo - The Windows X86_64 ABI information.
2436 class WinX86_64ABIInfo : public ABIInfo {
2437 public:
WinX86_64ABIInfo(CodeGen::CodeGenTypes & CGT,X86AVXABILevel AVXLevel)2438   WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2439       : ABIInfo(CGT), AVXLevel(AVXLevel),
2440         IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
2441 
2442   void computeInfo(CGFunctionInfo &FI) const override;
2443 
2444   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2445                     QualType Ty) const override;
2446 
isHomogeneousAggregateBaseType(QualType Ty) const2447   bool isHomogeneousAggregateBaseType(QualType Ty) const override {
2448     // FIXME: Assumes vectorcall is in use.
2449     return isX86VectorTypeForVectorCall(getContext(), Ty);
2450   }
2451 
isHomogeneousAggregateSmallEnough(const Type * Ty,uint64_t NumMembers) const2452   bool isHomogeneousAggregateSmallEnough(const Type *Ty,
2453                                          uint64_t NumMembers) const override {
2454     // FIXME: Assumes vectorcall is in use.
2455     return isX86VectorCallAggregateSmallEnough(NumMembers);
2456   }
2457 
2458 private:
2459   ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
2460                       bool IsVectorCall, bool IsRegCall) const;
2461   ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
2462                                            const ABIArgInfo &current) const;
2463 
2464   X86AVXABILevel AVXLevel;
2465 
2466   bool IsMingw64;
2467 };
2468 
2469 class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2470 public:
X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,X86AVXABILevel AVXLevel)2471   X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2472       : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {
2473     SwiftInfo =
2474         std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/true);
2475   }
2476 
getABIInfo() const2477   const X86_64ABIInfo &getABIInfo() const {
2478     return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
2479   }
2480 
2481   /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
2482   /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
markARCOptimizedReturnCallsAsNoTail() const2483   bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
2484 
getDwarfEHStackPointer(CodeGen::CodeGenModule & CGM) const2485   int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2486     return 7;
2487   }
2488 
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const2489   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2490                                llvm::Value *Address) const override {
2491     llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2492 
2493     // 0-15 are the 16 integer registers.
2494     // 16 is %rip.
2495     AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2496     return false;
2497   }
2498 
adjustInlineAsmType(CodeGen::CodeGenFunction & CGF,StringRef Constraint,llvm::Type * Ty) const2499   llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
2500                                   StringRef Constraint,
2501                                   llvm::Type* Ty) const override {
2502     return X86AdjustInlineAsmType(CGF, Constraint, Ty);
2503   }
2504 
isNoProtoCallVariadic(const CallArgList & args,const FunctionNoProtoType * fnType) const2505   bool isNoProtoCallVariadic(const CallArgList &args,
2506                              const FunctionNoProtoType *fnType) const override {
2507     // The default CC on x86-64 sets %al to the number of SSA
2508     // registers used, and GCC sets this when calling an unprototyped
2509     // function, so we override the default behavior.  However, don't do
2510     // that when AVX types are involved: the ABI explicitly states it is
2511     // undefined, and it doesn't work in practice because of how the ABI
2512     // defines varargs anyway.
2513     if (fnType->getCallConv() == CC_C) {
2514       bool HasAVXType = false;
2515       for (CallArgList::const_iterator
2516              it = args.begin(), ie = args.end(); it != ie; ++it) {
2517         if (getABIInfo().isPassedUsingAVXType(it->Ty)) {
2518           HasAVXType = true;
2519           break;
2520         }
2521       }
2522 
2523       if (!HasAVXType)
2524         return true;
2525     }
2526 
2527     return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
2528   }
2529 
2530   llvm::Constant *
getUBSanFunctionSignature(CodeGen::CodeGenModule & CGM) const2531   getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
2532     unsigned Sig = (0xeb << 0) | // jmp rel8
2533                    (0x06 << 8) | //           .+0x08
2534                    ('v' << 16) |
2535                    ('2' << 24);
2536     return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
2537   }
2538 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const2539   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2540                            CodeGen::CodeGenModule &CGM) const override {
2541     if (GV->isDeclaration())
2542       return;
2543     if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2544       if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2545         llvm::Function *Fn = cast<llvm::Function>(GV);
2546         Fn->addFnAttr("stackrealign");
2547       }
2548 
2549       addX86InterruptAttrs(FD, GV, CGM);
2550     }
2551   }
2552 
2553   void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
2554                             const FunctionDecl *Caller,
2555                             const FunctionDecl *Callee,
2556                             const CallArgList &Args) const override;
2557 };
2558 
initFeatureMaps(const ASTContext & Ctx,llvm::StringMap<bool> & CallerMap,const FunctionDecl * Caller,llvm::StringMap<bool> & CalleeMap,const FunctionDecl * Callee)2559 static void initFeatureMaps(const ASTContext &Ctx,
2560                             llvm::StringMap<bool> &CallerMap,
2561                             const FunctionDecl *Caller,
2562                             llvm::StringMap<bool> &CalleeMap,
2563                             const FunctionDecl *Callee) {
2564   if (CalleeMap.empty() && CallerMap.empty()) {
2565     // The caller is potentially nullptr in the case where the call isn't in a
2566     // function.  In this case, the getFunctionFeatureMap ensures we just get
2567     // the TU level setting (since it cannot be modified by 'target'..
2568     Ctx.getFunctionFeatureMap(CallerMap, Caller);
2569     Ctx.getFunctionFeatureMap(CalleeMap, Callee);
2570   }
2571 }
2572 
checkAVXParamFeature(DiagnosticsEngine & Diag,SourceLocation CallLoc,const llvm::StringMap<bool> & CallerMap,const llvm::StringMap<bool> & CalleeMap,QualType Ty,StringRef Feature,bool IsArgument)2573 static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
2574                                  SourceLocation CallLoc,
2575                                  const llvm::StringMap<bool> &CallerMap,
2576                                  const llvm::StringMap<bool> &CalleeMap,
2577                                  QualType Ty, StringRef Feature,
2578                                  bool IsArgument) {
2579   bool CallerHasFeat = CallerMap.lookup(Feature);
2580   bool CalleeHasFeat = CalleeMap.lookup(Feature);
2581   if (!CallerHasFeat && !CalleeHasFeat)
2582     return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
2583            << IsArgument << Ty << Feature;
2584 
2585   // Mixing calling conventions here is very clearly an error.
2586   if (!CallerHasFeat || !CalleeHasFeat)
2587     return Diag.Report(CallLoc, diag::err_avx_calling_convention)
2588            << IsArgument << Ty << Feature;
2589 
2590   // Else, both caller and callee have the required feature, so there is no need
2591   // to diagnose.
2592   return false;
2593 }
2594 
checkAVXParam(DiagnosticsEngine & Diag,ASTContext & Ctx,SourceLocation CallLoc,const llvm::StringMap<bool> & CallerMap,const llvm::StringMap<bool> & CalleeMap,QualType Ty,bool IsArgument)2595 static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
2596                           SourceLocation CallLoc,
2597                           const llvm::StringMap<bool> &CallerMap,
2598                           const llvm::StringMap<bool> &CalleeMap, QualType Ty,
2599                           bool IsArgument) {
2600   uint64_t Size = Ctx.getTypeSize(Ty);
2601   if (Size > 256)
2602     return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
2603                                 "avx512f", IsArgument);
2604 
2605   if (Size > 128)
2606     return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
2607                                 IsArgument);
2608 
2609   return false;
2610 }
2611 
checkFunctionCallABI(CodeGenModule & CGM,SourceLocation CallLoc,const FunctionDecl * Caller,const FunctionDecl * Callee,const CallArgList & Args) const2612 void X86_64TargetCodeGenInfo::checkFunctionCallABI(
2613     CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
2614     const FunctionDecl *Callee, const CallArgList &Args) const {
2615   llvm::StringMap<bool> CallerMap;
2616   llvm::StringMap<bool> CalleeMap;
2617   unsigned ArgIndex = 0;
2618 
2619   // We need to loop through the actual call arguments rather than the
2620   // function's parameters, in case this variadic.
2621   for (const CallArg &Arg : Args) {
2622     // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
2623     // additionally changes how vectors >256 in size are passed. Like GCC, we
2624     // warn when a function is called with an argument where this will change.
2625     // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
2626     // the caller and callee features are mismatched.
2627     // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
2628     // change its ABI with attribute-target after this call.
2629     if (Arg.getType()->isVectorType() &&
2630         CGM.getContext().getTypeSize(Arg.getType()) > 128) {
2631       initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2632       QualType Ty = Arg.getType();
2633       // The CallArg seems to have desugared the type already, so for clearer
2634       // diagnostics, replace it with the type in the FunctionDecl if possible.
2635       if (ArgIndex < Callee->getNumParams())
2636         Ty = Callee->getParamDecl(ArgIndex)->getType();
2637 
2638       if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2639                         CalleeMap, Ty, /*IsArgument*/ true))
2640         return;
2641     }
2642     ++ArgIndex;
2643   }
2644 
2645   // Check return always, as we don't have a good way of knowing in codegen
2646   // whether this value is used, tail-called, etc.
2647   if (Callee->getReturnType()->isVectorType() &&
2648       CGM.getContext().getTypeSize(Callee->getReturnType()) > 128) {
2649     initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2650     checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2651                   CalleeMap, Callee->getReturnType(),
2652                   /*IsArgument*/ false);
2653   }
2654 }
2655 
qualifyWindowsLibrary(llvm::StringRef Lib)2656 static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
2657   // If the argument does not end in .lib, automatically add the suffix.
2658   // If the argument contains a space, enclose it in quotes.
2659   // This matches the behavior of MSVC.
2660   bool Quote = Lib.contains(' ');
2661   std::string ArgStr = Quote ? "\"" : "";
2662   ArgStr += Lib;
2663   if (!Lib.endswith_insensitive(".lib") && !Lib.endswith_insensitive(".a"))
2664     ArgStr += ".lib";
2665   ArgStr += Quote ? "\"" : "";
2666   return ArgStr;
2667 }
2668 
2669 class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
2670 public:
WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,bool DarwinVectorABI,bool RetSmallStructInRegABI,bool Win32StructABI,unsigned NumRegisterParameters)2671   WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2672         bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
2673         unsigned NumRegisterParameters)
2674     : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
2675         Win32StructABI, NumRegisterParameters, false) {}
2676 
2677   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2678                            CodeGen::CodeGenModule &CGM) const override;
2679 
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const2680   void getDependentLibraryOption(llvm::StringRef Lib,
2681                                  llvm::SmallString<24> &Opt) const override {
2682     Opt = "/DEFAULTLIB:";
2683     Opt += qualifyWindowsLibrary(Lib);
2684   }
2685 
getDetectMismatchOption(llvm::StringRef Name,llvm::StringRef Value,llvm::SmallString<32> & Opt) const2686   void getDetectMismatchOption(llvm::StringRef Name,
2687                                llvm::StringRef Value,
2688                                llvm::SmallString<32> &Opt) const override {
2689     Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2690   }
2691 };
2692 
addStackProbeTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM)2693 static void addStackProbeTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2694                                           CodeGen::CodeGenModule &CGM) {
2695   if (llvm::Function *Fn = dyn_cast_or_null<llvm::Function>(GV)) {
2696 
2697     if (CGM.getCodeGenOpts().StackProbeSize != 4096)
2698       Fn->addFnAttr("stack-probe-size",
2699                     llvm::utostr(CGM.getCodeGenOpts().StackProbeSize));
2700     if (CGM.getCodeGenOpts().NoStackArgProbe)
2701       Fn->addFnAttr("no-stack-arg-probe");
2702   }
2703 }
2704 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const2705 void WinX86_32TargetCodeGenInfo::setTargetAttributes(
2706     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2707   X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2708   if (GV->isDeclaration())
2709     return;
2710   addStackProbeTargetAttributes(D, GV, CGM);
2711 }
2712 
2713 class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2714 public:
WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,X86AVXABILevel AVXLevel)2715   WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2716                              X86AVXABILevel AVXLevel)
2717       : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {
2718     SwiftInfo =
2719         std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/true);
2720   }
2721 
2722   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2723                            CodeGen::CodeGenModule &CGM) const override;
2724 
getDwarfEHStackPointer(CodeGen::CodeGenModule & CGM) const2725   int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2726     return 7;
2727   }
2728 
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const2729   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2730                                llvm::Value *Address) const override {
2731     llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2732 
2733     // 0-15 are the 16 integer registers.
2734     // 16 is %rip.
2735     AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2736     return false;
2737   }
2738 
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const2739   void getDependentLibraryOption(llvm::StringRef Lib,
2740                                  llvm::SmallString<24> &Opt) const override {
2741     Opt = "/DEFAULTLIB:";
2742     Opt += qualifyWindowsLibrary(Lib);
2743   }
2744 
getDetectMismatchOption(llvm::StringRef Name,llvm::StringRef Value,llvm::SmallString<32> & Opt) const2745   void getDetectMismatchOption(llvm::StringRef Name,
2746                                llvm::StringRef Value,
2747                                llvm::SmallString<32> &Opt) const override {
2748     Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2749   }
2750 };
2751 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const2752 void WinX86_64TargetCodeGenInfo::setTargetAttributes(
2753     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2754   TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2755   if (GV->isDeclaration())
2756     return;
2757   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2758     if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2759       llvm::Function *Fn = cast<llvm::Function>(GV);
2760       Fn->addFnAttr("stackrealign");
2761     }
2762 
2763     addX86InterruptAttrs(FD, GV, CGM);
2764   }
2765 
2766   addStackProbeTargetAttributes(D, GV, CGM);
2767 }
2768 }
2769 
postMerge(unsigned AggregateSize,Class & Lo,Class & Hi) const2770 void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
2771                               Class &Hi) const {
2772   // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
2773   //
2774   // (a) If one of the classes is Memory, the whole argument is passed in
2775   //     memory.
2776   //
2777   // (b) If X87UP is not preceded by X87, the whole argument is passed in
2778   //     memory.
2779   //
2780   // (c) If the size of the aggregate exceeds two eightbytes and the first
2781   //     eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
2782   //     argument is passed in memory. NOTE: This is necessary to keep the
2783   //     ABI working for processors that don't support the __m256 type.
2784   //
2785   // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
2786   //
2787   // Some of these are enforced by the merging logic.  Others can arise
2788   // only with unions; for example:
2789   //   union { _Complex double; unsigned; }
2790   //
2791   // Note that clauses (b) and (c) were added in 0.98.
2792   //
2793   if (Hi == Memory)
2794     Lo = Memory;
2795   if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
2796     Lo = Memory;
2797   if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
2798     Lo = Memory;
2799   if (Hi == SSEUp && Lo != SSE)
2800     Hi = SSE;
2801 }
2802 
merge(Class Accum,Class Field)2803 X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
2804   // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
2805   // classified recursively so that always two fields are
2806   // considered. The resulting class is calculated according to
2807   // the classes of the fields in the eightbyte:
2808   //
2809   // (a) If both classes are equal, this is the resulting class.
2810   //
2811   // (b) If one of the classes is NO_CLASS, the resulting class is
2812   // the other class.
2813   //
2814   // (c) If one of the classes is MEMORY, the result is the MEMORY
2815   // class.
2816   //
2817   // (d) If one of the classes is INTEGER, the result is the
2818   // INTEGER.
2819   //
2820   // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2821   // MEMORY is used as class.
2822   //
2823   // (f) Otherwise class SSE is used.
2824 
2825   // Accum should never be memory (we should have returned) or
2826   // ComplexX87 (because this cannot be passed in a structure).
2827   assert((Accum != Memory && Accum != ComplexX87) &&
2828          "Invalid accumulated classification during merge.");
2829   if (Accum == Field || Field == NoClass)
2830     return Accum;
2831   if (Field == Memory)
2832     return Memory;
2833   if (Accum == NoClass)
2834     return Field;
2835   if (Accum == Integer || Field == Integer)
2836     return Integer;
2837   if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2838       Accum == X87 || Accum == X87Up)
2839     return Memory;
2840   return SSE;
2841 }
2842 
classify(QualType Ty,uint64_t OffsetBase,Class & Lo,Class & Hi,bool isNamedArg,bool IsRegCall) const2843 void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
2844                              Class &Hi, bool isNamedArg, bool IsRegCall) const {
2845   // FIXME: This code can be simplified by introducing a simple value class for
2846   // Class pairs with appropriate constructor methods for the various
2847   // situations.
2848 
2849   // FIXME: Some of the split computations are wrong; unaligned vectors
2850   // shouldn't be passed in registers for example, so there is no chance they
2851   // can straddle an eightbyte. Verify & simplify.
2852 
2853   Lo = Hi = NoClass;
2854 
2855   Class &Current = OffsetBase < 64 ? Lo : Hi;
2856   Current = Memory;
2857 
2858   if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2859     BuiltinType::Kind k = BT->getKind();
2860 
2861     if (k == BuiltinType::Void) {
2862       Current = NoClass;
2863     } else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
2864       Lo = Integer;
2865       Hi = Integer;
2866     } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
2867       Current = Integer;
2868     } else if (k == BuiltinType::Float || k == BuiltinType::Double ||
2869                k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {
2870       Current = SSE;
2871     } else if (k == BuiltinType::LongDouble) {
2872       const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2873       if (LDF == &llvm::APFloat::IEEEquad()) {
2874         Lo = SSE;
2875         Hi = SSEUp;
2876       } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2877         Lo = X87;
2878         Hi = X87Up;
2879       } else if (LDF == &llvm::APFloat::IEEEdouble()) {
2880         Current = SSE;
2881       } else
2882         llvm_unreachable("unexpected long double representation!");
2883     }
2884     // FIXME: _Decimal32 and _Decimal64 are SSE.
2885     // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2886     return;
2887   }
2888 
2889   if (const EnumType *ET = Ty->getAs<EnumType>()) {
2890     // Classify the underlying integer type.
2891     classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2892     return;
2893   }
2894 
2895   if (Ty->hasPointerRepresentation()) {
2896     Current = Integer;
2897     return;
2898   }
2899 
2900   if (Ty->isMemberPointerType()) {
2901     if (Ty->isMemberFunctionPointerType()) {
2902       if (Has64BitPointers) {
2903         // If Has64BitPointers, this is an {i64, i64}, so classify both
2904         // Lo and Hi now.
2905         Lo = Hi = Integer;
2906       } else {
2907         // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2908         // straddles an eightbyte boundary, Hi should be classified as well.
2909         uint64_t EB_FuncPtr = (OffsetBase) / 64;
2910         uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2911         if (EB_FuncPtr != EB_ThisAdj) {
2912           Lo = Hi = Integer;
2913         } else {
2914           Current = Integer;
2915         }
2916       }
2917     } else {
2918       Current = Integer;
2919     }
2920     return;
2921   }
2922 
2923   if (const VectorType *VT = Ty->getAs<VectorType>()) {
2924     uint64_t Size = getContext().getTypeSize(VT);
2925     if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
2926       // gcc passes the following as integer:
2927       // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2928       // 2 bytes - <2 x char>, <1 x short>
2929       // 1 byte  - <1 x char>
2930       Current = Integer;
2931 
2932       // If this type crosses an eightbyte boundary, it should be
2933       // split.
2934       uint64_t EB_Lo = (OffsetBase) / 64;
2935       uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2936       if (EB_Lo != EB_Hi)
2937         Hi = Lo;
2938     } else if (Size == 64) {
2939       QualType ElementType = VT->getElementType();
2940 
2941       // gcc passes <1 x double> in memory. :(
2942       if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
2943         return;
2944 
2945       // gcc passes <1 x long long> as SSE but clang used to unconditionally
2946       // pass them as integer.  For platforms where clang is the de facto
2947       // platform compiler, we must continue to use integer.
2948       if (!classifyIntegerMMXAsSSE() &&
2949           (ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||
2950            ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||
2951            ElementType->isSpecificBuiltinType(BuiltinType::Long) ||
2952            ElementType->isSpecificBuiltinType(BuiltinType::ULong)))
2953         Current = Integer;
2954       else
2955         Current = SSE;
2956 
2957       // If this type crosses an eightbyte boundary, it should be
2958       // split.
2959       if (OffsetBase && OffsetBase != 64)
2960         Hi = Lo;
2961     } else if (Size == 128 ||
2962                (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
2963       QualType ElementType = VT->getElementType();
2964 
2965       // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2966       if (passInt128VectorsInMem() && Size != 128 &&
2967           (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
2968            ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
2969         return;
2970 
2971       // Arguments of 256-bits are split into four eightbyte chunks. The
2972       // least significant one belongs to class SSE and all the others to class
2973       // SSEUP. The original Lo and Hi design considers that types can't be
2974       // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2975       // This design isn't correct for 256-bits, but since there're no cases
2976       // where the upper parts would need to be inspected, avoid adding
2977       // complexity and just consider Hi to match the 64-256 part.
2978       //
2979       // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2980       // registers if they are "named", i.e. not part of the "..." of a
2981       // variadic function.
2982       //
2983       // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2984       // split into eight eightbyte chunks, one SSE and seven SSEUP.
2985       Lo = SSE;
2986       Hi = SSEUp;
2987     }
2988     return;
2989   }
2990 
2991   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2992     QualType ET = getContext().getCanonicalType(CT->getElementType());
2993 
2994     uint64_t Size = getContext().getTypeSize(Ty);
2995     if (ET->isIntegralOrEnumerationType()) {
2996       if (Size <= 64)
2997         Current = Integer;
2998       else if (Size <= 128)
2999         Lo = Hi = Integer;
3000     } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
3001                ET->isBFloat16Type()) {
3002       Current = SSE;
3003     } else if (ET == getContext().DoubleTy) {
3004       Lo = Hi = SSE;
3005     } else if (ET == getContext().LongDoubleTy) {
3006       const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
3007       if (LDF == &llvm::APFloat::IEEEquad())
3008         Current = Memory;
3009       else if (LDF == &llvm::APFloat::x87DoubleExtended())
3010         Current = ComplexX87;
3011       else if (LDF == &llvm::APFloat::IEEEdouble())
3012         Lo = Hi = SSE;
3013       else
3014         llvm_unreachable("unexpected long double representation!");
3015     }
3016 
3017     // If this complex type crosses an eightbyte boundary then it
3018     // should be split.
3019     uint64_t EB_Real = (OffsetBase) / 64;
3020     uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
3021     if (Hi == NoClass && EB_Real != EB_Imag)
3022       Hi = Lo;
3023 
3024     return;
3025   }
3026 
3027   if (const auto *EITy = Ty->getAs<BitIntType>()) {
3028     if (EITy->getNumBits() <= 64)
3029       Current = Integer;
3030     else if (EITy->getNumBits() <= 128)
3031       Lo = Hi = Integer;
3032     // Larger values need to get passed in memory.
3033     return;
3034   }
3035 
3036   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
3037     // Arrays are treated like structures.
3038 
3039     uint64_t Size = getContext().getTypeSize(Ty);
3040 
3041     // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3042     // than eight eightbytes, ..., it has class MEMORY.
3043     // regcall ABI doesn't have limitation to an object. The only limitation
3044     // is the free registers, which will be checked in computeInfo.
3045     if (!IsRegCall && Size > 512)
3046       return;
3047 
3048     // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
3049     // fields, it has class MEMORY.
3050     //
3051     // Only need to check alignment of array base.
3052     if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
3053       return;
3054 
3055     // Otherwise implement simplified merge. We could be smarter about
3056     // this, but it isn't worth it and would be harder to verify.
3057     Current = NoClass;
3058     uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
3059     uint64_t ArraySize = AT->getSize().getZExtValue();
3060 
3061     // The only case a 256-bit wide vector could be used is when the array
3062     // contains a single 256-bit element. Since Lo and Hi logic isn't extended
3063     // to work for sizes wider than 128, early check and fallback to memory.
3064     //
3065     if (Size > 128 &&
3066         (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
3067       return;
3068 
3069     for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
3070       Class FieldLo, FieldHi;
3071       classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
3072       Lo = merge(Lo, FieldLo);
3073       Hi = merge(Hi, FieldHi);
3074       if (Lo == Memory || Hi == Memory)
3075         break;
3076     }
3077 
3078     postMerge(Size, Lo, Hi);
3079     assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
3080     return;
3081   }
3082 
3083   if (const RecordType *RT = Ty->getAs<RecordType>()) {
3084     uint64_t Size = getContext().getTypeSize(Ty);
3085 
3086     // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3087     // than eight eightbytes, ..., it has class MEMORY.
3088     if (Size > 512)
3089       return;
3090 
3091     // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
3092     // copy constructor or a non-trivial destructor, it is passed by invisible
3093     // reference.
3094     if (getRecordArgABI(RT, getCXXABI()))
3095       return;
3096 
3097     const RecordDecl *RD = RT->getDecl();
3098 
3099     // Assume variable sized types are passed in memory.
3100     if (RD->hasFlexibleArrayMember())
3101       return;
3102 
3103     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
3104 
3105     // Reset Lo class, this will be recomputed.
3106     Current = NoClass;
3107 
3108     // If this is a C++ record, classify the bases first.
3109     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3110       for (const auto &I : CXXRD->bases()) {
3111         assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3112                "Unexpected base class!");
3113         const auto *Base =
3114             cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3115 
3116         // Classify this field.
3117         //
3118         // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
3119         // single eightbyte, each is classified separately. Each eightbyte gets
3120         // initialized to class NO_CLASS.
3121         Class FieldLo, FieldHi;
3122         uint64_t Offset =
3123           OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
3124         classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
3125         Lo = merge(Lo, FieldLo);
3126         Hi = merge(Hi, FieldHi);
3127         if (Lo == Memory || Hi == Memory) {
3128           postMerge(Size, Lo, Hi);
3129           return;
3130         }
3131       }
3132     }
3133 
3134     // Classify the fields one at a time, merging the results.
3135     unsigned idx = 0;
3136     bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
3137                                 LangOptions::ClangABI::Ver11 ||
3138                             getContext().getTargetInfo().getTriple().isPS();
3139     bool IsUnion = RT->isUnionType() && !UseClang11Compat;
3140 
3141     for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3142            i != e; ++i, ++idx) {
3143       uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3144       bool BitField = i->isBitField();
3145 
3146       // Ignore padding bit-fields.
3147       if (BitField && i->isUnnamedBitfield())
3148         continue;
3149 
3150       // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
3151       // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
3152       //
3153       // The only case a 256-bit or a 512-bit wide vector could be used is when
3154       // the struct contains a single 256-bit or 512-bit element. Early check
3155       // and fallback to memory.
3156       //
3157       // FIXME: Extended the Lo and Hi logic properly to work for size wider
3158       // than 128.
3159       if (Size > 128 &&
3160           ((!IsUnion && Size != getContext().getTypeSize(i->getType())) ||
3161            Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
3162         Lo = Memory;
3163         postMerge(Size, Lo, Hi);
3164         return;
3165       }
3166       // Note, skip this test for bit-fields, see below.
3167       if (!BitField && Offset % getContext().getTypeAlign(i->getType())) {
3168         Lo = Memory;
3169         postMerge(Size, Lo, Hi);
3170         return;
3171       }
3172 
3173       // Classify this field.
3174       //
3175       // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
3176       // exceeds a single eightbyte, each is classified
3177       // separately. Each eightbyte gets initialized to class
3178       // NO_CLASS.
3179       Class FieldLo, FieldHi;
3180 
3181       // Bit-fields require special handling, they do not force the
3182       // structure to be passed in memory even if unaligned, and
3183       // therefore they can straddle an eightbyte.
3184       if (BitField) {
3185         assert(!i->isUnnamedBitfield());
3186         uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3187         uint64_t Size = i->getBitWidthValue(getContext());
3188 
3189         uint64_t EB_Lo = Offset / 64;
3190         uint64_t EB_Hi = (Offset + Size - 1) / 64;
3191 
3192         if (EB_Lo) {
3193           assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
3194           FieldLo = NoClass;
3195           FieldHi = Integer;
3196         } else {
3197           FieldLo = Integer;
3198           FieldHi = EB_Hi ? Integer : NoClass;
3199         }
3200       } else
3201         classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
3202       Lo = merge(Lo, FieldLo);
3203       Hi = merge(Hi, FieldHi);
3204       if (Lo == Memory || Hi == Memory)
3205         break;
3206     }
3207 
3208     postMerge(Size, Lo, Hi);
3209   }
3210 }
3211 
getIndirectReturnResult(QualType Ty) const3212 ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
3213   // If this is a scalar LLVM value then assume LLVM will pass it in the right
3214   // place naturally.
3215   if (!isAggregateTypeForABI(Ty)) {
3216     // Treat an enum type as its underlying type.
3217     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3218       Ty = EnumTy->getDecl()->getIntegerType();
3219 
3220     if (Ty->isBitIntType())
3221       return getNaturalAlignIndirect(Ty);
3222 
3223     return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
3224                                               : ABIArgInfo::getDirect());
3225   }
3226 
3227   return getNaturalAlignIndirect(Ty);
3228 }
3229 
IsIllegalVectorType(QualType Ty) const3230 bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
3231   if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
3232     uint64_t Size = getContext().getTypeSize(VecTy);
3233     unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
3234     if (Size <= 64 || Size > LargestVector)
3235       return true;
3236     QualType EltTy = VecTy->getElementType();
3237     if (passInt128VectorsInMem() &&
3238         (EltTy->isSpecificBuiltinType(BuiltinType::Int128) ||
3239          EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))
3240       return true;
3241   }
3242 
3243   return false;
3244 }
3245 
getIndirectResult(QualType Ty,unsigned freeIntRegs) const3246 ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
3247                                             unsigned freeIntRegs) const {
3248   // If this is a scalar LLVM value then assume LLVM will pass it in the right
3249   // place naturally.
3250   //
3251   // This assumption is optimistic, as there could be free registers available
3252   // when we need to pass this argument in memory, and LLVM could try to pass
3253   // the argument in the free register. This does not seem to happen currently,
3254   // but this code would be much safer if we could mark the argument with
3255   // 'onstack'. See PR12193.
3256   if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty) &&
3257       !Ty->isBitIntType()) {
3258     // Treat an enum type as its underlying type.
3259     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3260       Ty = EnumTy->getDecl()->getIntegerType();
3261 
3262     return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
3263                                               : ABIArgInfo::getDirect());
3264   }
3265 
3266   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
3267     return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
3268 
3269   // Compute the byval alignment. We specify the alignment of the byval in all
3270   // cases so that the mid-level optimizer knows the alignment of the byval.
3271   unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);
3272 
3273   // Attempt to avoid passing indirect results using byval when possible. This
3274   // is important for good codegen.
3275   //
3276   // We do this by coercing the value into a scalar type which the backend can
3277   // handle naturally (i.e., without using byval).
3278   //
3279   // For simplicity, we currently only do this when we have exhausted all of the
3280   // free integer registers. Doing this when there are free integer registers
3281   // would require more care, as we would have to ensure that the coerced value
3282   // did not claim the unused register. That would require either reording the
3283   // arguments to the function (so that any subsequent inreg values came first),
3284   // or only doing this optimization when there were no following arguments that
3285   // might be inreg.
3286   //
3287   // We currently expect it to be rare (particularly in well written code) for
3288   // arguments to be passed on the stack when there are still free integer
3289   // registers available (this would typically imply large structs being passed
3290   // by value), so this seems like a fair tradeoff for now.
3291   //
3292   // We can revisit this if the backend grows support for 'onstack' parameter
3293   // attributes. See PR12193.
3294   if (freeIntRegs == 0) {
3295     uint64_t Size = getContext().getTypeSize(Ty);
3296 
3297     // If this type fits in an eightbyte, coerce it into the matching integral
3298     // type, which will end up on the stack (with alignment 8).
3299     if (Align == 8 && Size <= 64)
3300       return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
3301                                                           Size));
3302   }
3303 
3304   return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align));
3305 }
3306 
3307 /// The ABI specifies that a value should be passed in a full vector XMM/YMM
3308 /// register. Pick an LLVM IR type that will be passed as a vector register.
GetByteVectorType(QualType Ty) const3309 llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
3310   // Wrapper structs/arrays that only contain vectors are passed just like
3311   // vectors; strip them off if present.
3312   if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))
3313     Ty = QualType(InnerTy, 0);
3314 
3315   llvm::Type *IRType = CGT.ConvertType(Ty);
3316   if (isa<llvm::VectorType>(IRType)) {
3317     // Don't pass vXi128 vectors in their native type, the backend can't
3318     // legalize them.
3319     if (passInt128VectorsInMem() &&
3320         cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)) {
3321       // Use a vXi64 vector.
3322       uint64_t Size = getContext().getTypeSize(Ty);
3323       return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),
3324                                         Size / 64);
3325     }
3326 
3327     return IRType;
3328   }
3329 
3330   if (IRType->getTypeID() == llvm::Type::FP128TyID)
3331     return IRType;
3332 
3333   // We couldn't find the preferred IR vector type for 'Ty'.
3334   uint64_t Size = getContext().getTypeSize(Ty);
3335   assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
3336 
3337 
3338   // Return a LLVM IR vector type based on the size of 'Ty'.
3339   return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
3340                                     Size / 64);
3341 }
3342 
3343 /// BitsContainNoUserData - Return true if the specified [start,end) bit range
3344 /// is known to either be off the end of the specified type or being in
3345 /// alignment padding.  The user type specified is known to be at most 128 bits
3346 /// in size, and have passed through X86_64ABIInfo::classify with a successful
3347 /// classification that put one of the two halves in the INTEGER class.
3348 ///
3349 /// It is conservatively correct to return false.
BitsContainNoUserData(QualType Ty,unsigned StartBit,unsigned EndBit,ASTContext & Context)3350 static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
3351                                   unsigned EndBit, ASTContext &Context) {
3352   // If the bytes being queried are off the end of the type, there is no user
3353   // data hiding here.  This handles analysis of builtins, vectors and other
3354   // types that don't contain interesting padding.
3355   unsigned TySize = (unsigned)Context.getTypeSize(Ty);
3356   if (TySize <= StartBit)
3357     return true;
3358 
3359   if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
3360     unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
3361     unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
3362 
3363     // Check each element to see if the element overlaps with the queried range.
3364     for (unsigned i = 0; i != NumElts; ++i) {
3365       // If the element is after the span we care about, then we're done..
3366       unsigned EltOffset = i*EltSize;
3367       if (EltOffset >= EndBit) break;
3368 
3369       unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
3370       if (!BitsContainNoUserData(AT->getElementType(), EltStart,
3371                                  EndBit-EltOffset, Context))
3372         return false;
3373     }
3374     // If it overlaps no elements, then it is safe to process as padding.
3375     return true;
3376   }
3377 
3378   if (const RecordType *RT = Ty->getAs<RecordType>()) {
3379     const RecordDecl *RD = RT->getDecl();
3380     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
3381 
3382     // If this is a C++ record, check the bases first.
3383     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3384       for (const auto &I : CXXRD->bases()) {
3385         assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3386                "Unexpected base class!");
3387         const auto *Base =
3388             cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3389 
3390         // If the base is after the span we care about, ignore it.
3391         unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
3392         if (BaseOffset >= EndBit) continue;
3393 
3394         unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
3395         if (!BitsContainNoUserData(I.getType(), BaseStart,
3396                                    EndBit-BaseOffset, Context))
3397           return false;
3398       }
3399     }
3400 
3401     // Verify that no field has data that overlaps the region of interest.  Yes
3402     // this could be sped up a lot by being smarter about queried fields,
3403     // however we're only looking at structs up to 16 bytes, so we don't care
3404     // much.
3405     unsigned idx = 0;
3406     for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3407          i != e; ++i, ++idx) {
3408       unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
3409 
3410       // If we found a field after the region we care about, then we're done.
3411       if (FieldOffset >= EndBit) break;
3412 
3413       unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
3414       if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
3415                                  Context))
3416         return false;
3417     }
3418 
3419     // If nothing in this record overlapped the area of interest, then we're
3420     // clean.
3421     return true;
3422   }
3423 
3424   return false;
3425 }
3426 
3427 /// getFPTypeAtOffset - Return a floating point type at the specified offset.
getFPTypeAtOffset(llvm::Type * IRType,unsigned IROffset,const llvm::DataLayout & TD)3428 static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3429                                      const llvm::DataLayout &TD) {
3430   if (IROffset == 0 && IRType->isFloatingPointTy())
3431     return IRType;
3432 
3433   // If this is a struct, recurse into the field at the specified offset.
3434   if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3435     if (!STy->getNumContainedTypes())
3436       return nullptr;
3437 
3438     const llvm::StructLayout *SL = TD.getStructLayout(STy);
3439     unsigned Elt = SL->getElementContainingOffset(IROffset);
3440     IROffset -= SL->getElementOffset(Elt);
3441     return getFPTypeAtOffset(STy->getElementType(Elt), IROffset, TD);
3442   }
3443 
3444   // If this is an array, recurse into the field at the specified offset.
3445   if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3446     llvm::Type *EltTy = ATy->getElementType();
3447     unsigned EltSize = TD.getTypeAllocSize(EltTy);
3448     IROffset -= IROffset / EltSize * EltSize;
3449     return getFPTypeAtOffset(EltTy, IROffset, TD);
3450   }
3451 
3452   return nullptr;
3453 }
3454 
3455 /// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
3456 /// low 8 bytes of an XMM register, corresponding to the SSE class.
3457 llvm::Type *X86_64ABIInfo::
GetSSETypeAtOffset(llvm::Type * IRType,unsigned IROffset,QualType SourceTy,unsigned SourceOffset) const3458 GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3459                    QualType SourceTy, unsigned SourceOffset) const {
3460   const llvm::DataLayout &TD = getDataLayout();
3461   unsigned SourceSize =
3462       (unsigned)getContext().getTypeSize(SourceTy) / 8 - SourceOffset;
3463   llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
3464   if (!T0 || T0->isDoubleTy())
3465     return llvm::Type::getDoubleTy(getVMContext());
3466 
3467   // Get the adjacent FP type.
3468   llvm::Type *T1 = nullptr;
3469   unsigned T0Size = TD.getTypeAllocSize(T0);
3470   if (SourceSize > T0Size)
3471       T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);
3472   if (T1 == nullptr) {
3473     // Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due
3474     // to its alignment.
3475     if (T0->is16bitFPTy() && SourceSize > 4)
3476       T1 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3477     // If we can't get a second FP type, return a simple half or float.
3478     // avx512fp16-abi.c:pr51813_2 shows it works to return float for
3479     // {float, i8} too.
3480     if (T1 == nullptr)
3481       return T0;
3482   }
3483 
3484   if (T0->isFloatTy() && T1->isFloatTy())
3485     return llvm::FixedVectorType::get(T0, 2);
3486 
3487   if (T0->is16bitFPTy() && T1->is16bitFPTy()) {
3488     llvm::Type *T2 = nullptr;
3489     if (SourceSize > 4)
3490       T2 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3491     if (T2 == nullptr)
3492       return llvm::FixedVectorType::get(T0, 2);
3493     return llvm::FixedVectorType::get(T0, 4);
3494   }
3495 
3496   if (T0->is16bitFPTy() || T1->is16bitFPTy())
3497     return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);
3498 
3499   return llvm::Type::getDoubleTy(getVMContext());
3500 }
3501 
3502 
3503 /// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
3504 /// an 8-byte GPR.  This means that we either have a scalar or we are talking
3505 /// about the high or low part of an up-to-16-byte struct.  This routine picks
3506 /// the best LLVM IR type to represent this, which may be i64 or may be anything
3507 /// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
3508 /// etc).
3509 ///
3510 /// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
3511 /// the source type.  IROffset is an offset in bytes into the LLVM IR type that
3512 /// the 8-byte value references.  PrefType may be null.
3513 ///
3514 /// SourceTy is the source-level type for the entire argument.  SourceOffset is
3515 /// an offset into this that we're processing (which is always either 0 or 8).
3516 ///
3517 llvm::Type *X86_64ABIInfo::
GetINTEGERTypeAtOffset(llvm::Type * IRType,unsigned IROffset,QualType SourceTy,unsigned SourceOffset) const3518 GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3519                        QualType SourceTy, unsigned SourceOffset) const {
3520   // If we're dealing with an un-offset LLVM IR type, then it means that we're
3521   // returning an 8-byte unit starting with it.  See if we can safely use it.
3522   if (IROffset == 0) {
3523     // Pointers and int64's always fill the 8-byte unit.
3524     if ((isa<llvm::PointerType>(IRType) && Has64BitPointers) ||
3525         IRType->isIntegerTy(64))
3526       return IRType;
3527 
3528     // If we have a 1/2/4-byte integer, we can use it only if the rest of the
3529     // goodness in the source type is just tail padding.  This is allowed to
3530     // kick in for struct {double,int} on the int, but not on
3531     // struct{double,int,int} because we wouldn't return the second int.  We
3532     // have to do this analysis on the source type because we can't depend on
3533     // unions being lowered a specific way etc.
3534     if (IRType->isIntegerTy(8) || IRType->isIntegerTy(16) ||
3535         IRType->isIntegerTy(32) ||
3536         (isa<llvm::PointerType>(IRType) && !Has64BitPointers)) {
3537       unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 32 :
3538           cast<llvm::IntegerType>(IRType)->getBitWidth();
3539 
3540       if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
3541                                 SourceOffset*8+64, getContext()))
3542         return IRType;
3543     }
3544   }
3545 
3546   if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3547     // If this is a struct, recurse into the field at the specified offset.
3548     const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
3549     if (IROffset < SL->getSizeInBytes()) {
3550       unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
3551       IROffset -= SL->getElementOffset(FieldIdx);
3552 
3553       return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
3554                                     SourceTy, SourceOffset);
3555     }
3556   }
3557 
3558   if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3559     llvm::Type *EltTy = ATy->getElementType();
3560     unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
3561     unsigned EltOffset = IROffset/EltSize*EltSize;
3562     return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
3563                                   SourceOffset);
3564   }
3565 
3566   // Okay, we don't have any better idea of what to pass, so we pass this in an
3567   // integer register that isn't too big to fit the rest of the struct.
3568   unsigned TySizeInBytes =
3569     (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
3570 
3571   assert(TySizeInBytes != SourceOffset && "Empty field?");
3572 
3573   // It is always safe to classify this as an integer type up to i64 that
3574   // isn't larger than the structure.
3575   return llvm::IntegerType::get(getVMContext(),
3576                                 std::min(TySizeInBytes-SourceOffset, 8U)*8);
3577 }
3578 
3579 
3580 /// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
3581 /// be used as elements of a two register pair to pass or return, return a
3582 /// first class aggregate to represent them.  For example, if the low part of
3583 /// a by-value argument should be passed as i32* and the high part as float,
3584 /// return {i32*, float}.
3585 static llvm::Type *
GetX86_64ByValArgumentPair(llvm::Type * Lo,llvm::Type * Hi,const llvm::DataLayout & TD)3586 GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
3587                            const llvm::DataLayout &TD) {
3588   // In order to correctly satisfy the ABI, we need to the high part to start
3589   // at offset 8.  If the high and low parts we inferred are both 4-byte types
3590   // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
3591   // the second element at offset 8.  Check for this:
3592   unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
3593   llvm::Align HiAlign = TD.getABITypeAlign(Hi);
3594   unsigned HiStart = llvm::alignTo(LoSize, HiAlign);
3595   assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
3596 
3597   // To handle this, we have to increase the size of the low part so that the
3598   // second element will start at an 8 byte offset.  We can't increase the size
3599   // of the second element because it might make us access off the end of the
3600   // struct.
3601   if (HiStart != 8) {
3602     // There are usually two sorts of types the ABI generation code can produce
3603     // for the low part of a pair that aren't 8 bytes in size: half, float or
3604     // i8/i16/i32.  This can also include pointers when they are 32-bit (X32 and
3605     // NaCl).
3606     // Promote these to a larger type.
3607     if (Lo->isHalfTy() || Lo->isFloatTy())
3608       Lo = llvm::Type::getDoubleTy(Lo->getContext());
3609     else {
3610       assert((Lo->isIntegerTy() || Lo->isPointerTy())
3611              && "Invalid/unknown lo type");
3612       Lo = llvm::Type::getInt64Ty(Lo->getContext());
3613     }
3614   }
3615 
3616   llvm::StructType *Result = llvm::StructType::get(Lo, Hi);
3617 
3618   // Verify that the second element is at an 8-byte offset.
3619   assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
3620          "Invalid x86-64 argument pair!");
3621   return Result;
3622 }
3623 
3624 ABIArgInfo X86_64ABIInfo::
classifyReturnType(QualType RetTy) const3625 classifyReturnType(QualType RetTy) const {
3626   // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
3627   // classification algorithm.
3628   X86_64ABIInfo::Class Lo, Hi;
3629   classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
3630 
3631   // Check some invariants.
3632   assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3633   assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3634 
3635   llvm::Type *ResType = nullptr;
3636   switch (Lo) {
3637   case NoClass:
3638     if (Hi == NoClass)
3639       return ABIArgInfo::getIgnore();
3640     // If the low part is just padding, it takes no register, leave ResType
3641     // null.
3642     assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3643            "Unknown missing lo part");
3644     break;
3645 
3646   case SSEUp:
3647   case X87Up:
3648     llvm_unreachable("Invalid classification for lo word.");
3649 
3650     // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
3651     // hidden argument.
3652   case Memory:
3653     return getIndirectReturnResult(RetTy);
3654 
3655     // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
3656     // available register of the sequence %rax, %rdx is used.
3657   case Integer:
3658     ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3659 
3660     // If we have a sign or zero extended integer, make sure to return Extend
3661     // so that the parameter gets the right LLVM IR attributes.
3662     if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
3663       // Treat an enum type as its underlying type.
3664       if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
3665         RetTy = EnumTy->getDecl()->getIntegerType();
3666 
3667       if (RetTy->isIntegralOrEnumerationType() &&
3668           isPromotableIntegerTypeForABI(RetTy))
3669         return ABIArgInfo::getExtend(RetTy);
3670     }
3671     break;
3672 
3673     // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
3674     // available SSE register of the sequence %xmm0, %xmm1 is used.
3675   case SSE:
3676     ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3677     break;
3678 
3679     // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
3680     // returned on the X87 stack in %st0 as 80-bit x87 number.
3681   case X87:
3682     ResType = llvm::Type::getX86_FP80Ty(getVMContext());
3683     break;
3684 
3685     // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
3686     // part of the value is returned in %st0 and the imaginary part in
3687     // %st1.
3688   case ComplexX87:
3689     assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
3690     ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),
3691                                     llvm::Type::getX86_FP80Ty(getVMContext()));
3692     break;
3693   }
3694 
3695   llvm::Type *HighPart = nullptr;
3696   switch (Hi) {
3697     // Memory was handled previously and X87 should
3698     // never occur as a hi class.
3699   case Memory:
3700   case X87:
3701     llvm_unreachable("Invalid classification for hi word.");
3702 
3703   case ComplexX87: // Previously handled.
3704   case NoClass:
3705     break;
3706 
3707   case Integer:
3708     HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3709     if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3710       return ABIArgInfo::getDirect(HighPart, 8);
3711     break;
3712   case SSE:
3713     HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3714     if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3715       return ABIArgInfo::getDirect(HighPart, 8);
3716     break;
3717 
3718     // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
3719     // is passed in the next available eightbyte chunk if the last used
3720     // vector register.
3721     //
3722     // SSEUP should always be preceded by SSE, just widen.
3723   case SSEUp:
3724     assert(Lo == SSE && "Unexpected SSEUp classification.");
3725     ResType = GetByteVectorType(RetTy);
3726     break;
3727 
3728     // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
3729     // returned together with the previous X87 value in %st0.
3730   case X87Up:
3731     // If X87Up is preceded by X87, we don't need to do
3732     // anything. However, in some cases with unions it may not be
3733     // preceded by X87. In such situations we follow gcc and pass the
3734     // extra bits in an SSE reg.
3735     if (Lo != X87) {
3736       HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3737       if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3738         return ABIArgInfo::getDirect(HighPart, 8);
3739     }
3740     break;
3741   }
3742 
3743   // If a high part was specified, merge it together with the low part.  It is
3744   // known to pass in the high eightbyte of the result.  We do this by forming a
3745   // first class struct aggregate with the high and low part: {low, high}
3746   if (HighPart)
3747     ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3748 
3749   return ABIArgInfo::getDirect(ResType);
3750 }
3751 
3752 ABIArgInfo
classifyArgumentType(QualType Ty,unsigned freeIntRegs,unsigned & neededInt,unsigned & neededSSE,bool isNamedArg,bool IsRegCall) const3753 X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
3754                                     unsigned &neededInt, unsigned &neededSSE,
3755                                     bool isNamedArg, bool IsRegCall) const {
3756   Ty = useFirstFieldIfTransparentUnion(Ty);
3757 
3758   X86_64ABIInfo::Class Lo, Hi;
3759   classify(Ty, 0, Lo, Hi, isNamedArg, IsRegCall);
3760 
3761   // Check some invariants.
3762   // FIXME: Enforce these by construction.
3763   assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3764   assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3765 
3766   neededInt = 0;
3767   neededSSE = 0;
3768   llvm::Type *ResType = nullptr;
3769   switch (Lo) {
3770   case NoClass:
3771     if (Hi == NoClass)
3772       return ABIArgInfo::getIgnore();
3773     // If the low part is just padding, it takes no register, leave ResType
3774     // null.
3775     assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3776            "Unknown missing lo part");
3777     break;
3778 
3779     // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
3780     // on the stack.
3781   case Memory:
3782 
3783     // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
3784     // COMPLEX_X87, it is passed in memory.
3785   case X87:
3786   case ComplexX87:
3787     if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)
3788       ++neededInt;
3789     return getIndirectResult(Ty, freeIntRegs);
3790 
3791   case SSEUp:
3792   case X87Up:
3793     llvm_unreachable("Invalid classification for lo word.");
3794 
3795     // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3796     // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3797     // and %r9 is used.
3798   case Integer:
3799     ++neededInt;
3800 
3801     // Pick an 8-byte type based on the preferred type.
3802     ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);
3803 
3804     // If we have a sign or zero extended integer, make sure to return Extend
3805     // so that the parameter gets the right LLVM IR attributes.
3806     if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
3807       // Treat an enum type as its underlying type.
3808       if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3809         Ty = EnumTy->getDecl()->getIntegerType();
3810 
3811       if (Ty->isIntegralOrEnumerationType() &&
3812           isPromotableIntegerTypeForABI(Ty))
3813         return ABIArgInfo::getExtend(Ty);
3814     }
3815 
3816     break;
3817 
3818     // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3819     // available SSE register is used, the registers are taken in the
3820     // order from %xmm0 to %xmm7.
3821   case SSE: {
3822     llvm::Type *IRType = CGT.ConvertType(Ty);
3823     ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
3824     ++neededSSE;
3825     break;
3826   }
3827   }
3828 
3829   llvm::Type *HighPart = nullptr;
3830   switch (Hi) {
3831     // Memory was handled previously, ComplexX87 and X87 should
3832     // never occur as hi classes, and X87Up must be preceded by X87,
3833     // which is passed in memory.
3834   case Memory:
3835   case X87:
3836   case ComplexX87:
3837     llvm_unreachable("Invalid classification for hi word.");
3838 
3839   case NoClass: break;
3840 
3841   case Integer:
3842     ++neededInt;
3843     // Pick an 8-byte type based on the preferred type.
3844     HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3845 
3846     if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3847       return ABIArgInfo::getDirect(HighPart, 8);
3848     break;
3849 
3850     // X87Up generally doesn't occur here (long double is passed in
3851     // memory), except in situations involving unions.
3852   case X87Up:
3853   case SSE:
3854     HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3855 
3856     if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3857       return ABIArgInfo::getDirect(HighPart, 8);
3858 
3859     ++neededSSE;
3860     break;
3861 
3862     // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3863     // eightbyte is passed in the upper half of the last used SSE
3864     // register.  This only happens when 128-bit vectors are passed.
3865   case SSEUp:
3866     assert(Lo == SSE && "Unexpected SSEUp classification");
3867     ResType = GetByteVectorType(Ty);
3868     break;
3869   }
3870 
3871   // If a high part was specified, merge it together with the low part.  It is
3872   // known to pass in the high eightbyte of the result.  We do this by forming a
3873   // first class struct aggregate with the high and low part: {low, high}
3874   if (HighPart)
3875     ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3876 
3877   return ABIArgInfo::getDirect(ResType);
3878 }
3879 
3880 ABIArgInfo
classifyRegCallStructTypeImpl(QualType Ty,unsigned & NeededInt,unsigned & NeededSSE,unsigned & MaxVectorWidth) const3881 X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
3882                                              unsigned &NeededSSE,
3883                                              unsigned &MaxVectorWidth) const {
3884   auto RT = Ty->getAs<RecordType>();
3885   assert(RT && "classifyRegCallStructType only valid with struct types");
3886 
3887   if (RT->getDecl()->hasFlexibleArrayMember())
3888     return getIndirectReturnResult(Ty);
3889 
3890   // Sum up bases
3891   if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
3892     if (CXXRD->isDynamicClass()) {
3893       NeededInt = NeededSSE = 0;
3894       return getIndirectReturnResult(Ty);
3895     }
3896 
3897     for (const auto &I : CXXRD->bases())
3898       if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE,
3899                                         MaxVectorWidth)
3900               .isIndirect()) {
3901         NeededInt = NeededSSE = 0;
3902         return getIndirectReturnResult(Ty);
3903       }
3904   }
3905 
3906   // Sum up members
3907   for (const auto *FD : RT->getDecl()->fields()) {
3908     QualType MTy = FD->getType();
3909     if (MTy->isRecordType() && !MTy->isUnionType()) {
3910       if (classifyRegCallStructTypeImpl(MTy, NeededInt, NeededSSE,
3911                                         MaxVectorWidth)
3912               .isIndirect()) {
3913         NeededInt = NeededSSE = 0;
3914         return getIndirectReturnResult(Ty);
3915       }
3916     } else {
3917       unsigned LocalNeededInt, LocalNeededSSE;
3918       if (classifyArgumentType(MTy, UINT_MAX, LocalNeededInt, LocalNeededSSE,
3919                                true, true)
3920               .isIndirect()) {
3921         NeededInt = NeededSSE = 0;
3922         return getIndirectReturnResult(Ty);
3923       }
3924       if (const auto *AT = getContext().getAsConstantArrayType(MTy))
3925         MTy = AT->getElementType();
3926       if (const auto *VT = MTy->getAs<VectorType>())
3927         if (getContext().getTypeSize(VT) > MaxVectorWidth)
3928           MaxVectorWidth = getContext().getTypeSize(VT);
3929       NeededInt += LocalNeededInt;
3930       NeededSSE += LocalNeededSSE;
3931     }
3932   }
3933 
3934   return ABIArgInfo::getDirect();
3935 }
3936 
3937 ABIArgInfo
classifyRegCallStructType(QualType Ty,unsigned & NeededInt,unsigned & NeededSSE,unsigned & MaxVectorWidth) const3938 X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
3939                                          unsigned &NeededSSE,
3940                                          unsigned &MaxVectorWidth) const {
3941 
3942   NeededInt = 0;
3943   NeededSSE = 0;
3944   MaxVectorWidth = 0;
3945 
3946   return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE,
3947                                        MaxVectorWidth);
3948 }
3949 
computeInfo(CGFunctionInfo & FI) const3950 void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3951 
3952   const unsigned CallingConv = FI.getCallingConvention();
3953   // It is possible to force Win64 calling convention on any x86_64 target by
3954   // using __attribute__((ms_abi)). In such case to correctly emit Win64
3955   // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3956   if (CallingConv == llvm::CallingConv::Win64) {
3957     WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3958     Win64ABIInfo.computeInfo(FI);
3959     return;
3960   }
3961 
3962   bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3963 
3964   // Keep track of the number of assigned registers.
3965   unsigned FreeIntRegs = IsRegCall ? 11 : 6;
3966   unsigned FreeSSERegs = IsRegCall ? 16 : 8;
3967   unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
3968 
3969   if (!::classifyReturnType(getCXXABI(), FI, *this)) {
3970     if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
3971         !FI.getReturnType()->getTypePtr()->isUnionType()) {
3972       FI.getReturnInfo() = classifyRegCallStructType(
3973           FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
3974       if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3975         FreeIntRegs -= NeededInt;
3976         FreeSSERegs -= NeededSSE;
3977       } else {
3978         FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3979       }
3980     } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&
3981                getContext().getCanonicalType(FI.getReturnType()
3982                                                  ->getAs<ComplexType>()
3983                                                  ->getElementType()) ==
3984                    getContext().LongDoubleTy)
3985       // Complex Long Double Type is passed in Memory when Regcall
3986       // calling convention is used.
3987       FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3988     else
3989       FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
3990   }
3991 
3992   // If the return value is indirect, then the hidden argument is consuming one
3993   // integer register.
3994   if (FI.getReturnInfo().isIndirect())
3995     --FreeIntRegs;
3996   else if (NeededSSE && MaxVectorWidth > 0)
3997     FI.setMaxVectorWidth(MaxVectorWidth);
3998 
3999   // The chain argument effectively gives us another free register.
4000   if (FI.isChainCall())
4001     ++FreeIntRegs;
4002 
4003   unsigned NumRequiredArgs = FI.getNumRequiredArgs();
4004   // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
4005   // get assigned (in left-to-right order) for passing as follows...
4006   unsigned ArgNo = 0;
4007   for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
4008        it != ie; ++it, ++ArgNo) {
4009     bool IsNamedArg = ArgNo < NumRequiredArgs;
4010 
4011     if (IsRegCall && it->type->isStructureOrClassType())
4012       it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE,
4013                                            MaxVectorWidth);
4014     else
4015       it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
4016                                       NeededSSE, IsNamedArg);
4017 
4018     // AMD64-ABI 3.2.3p3: If there are no registers available for any
4019     // eightbyte of an argument, the whole argument is passed on the
4020     // stack. If registers have already been assigned for some
4021     // eightbytes of such an argument, the assignments get reverted.
4022     if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
4023       FreeIntRegs -= NeededInt;
4024       FreeSSERegs -= NeededSSE;
4025       if (MaxVectorWidth > FI.getMaxVectorWidth())
4026         FI.setMaxVectorWidth(MaxVectorWidth);
4027     } else {
4028       it->info = getIndirectResult(it->type, FreeIntRegs);
4029     }
4030   }
4031 }
4032 
EmitX86_64VAArgFromMemory(CodeGenFunction & CGF,Address VAListAddr,QualType Ty)4033 static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
4034                                          Address VAListAddr, QualType Ty) {
4035   Address overflow_arg_area_p =
4036       CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
4037   llvm::Value *overflow_arg_area =
4038     CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
4039 
4040   // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
4041   // byte boundary if alignment needed by type exceeds 8 byte boundary.
4042   // It isn't stated explicitly in the standard, but in practice we use
4043   // alignment greater than 16 where necessary.
4044   CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4045   if (Align > CharUnits::fromQuantity(8)) {
4046     overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
4047                                                       Align);
4048   }
4049 
4050   // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
4051   llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4052   llvm::Value *Res =
4053     CGF.Builder.CreateBitCast(overflow_arg_area,
4054                               llvm::PointerType::getUnqual(LTy));
4055 
4056   // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
4057   // l->overflow_arg_area + sizeof(type).
4058   // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
4059   // an 8 byte boundary.
4060 
4061   uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
4062   llvm::Value *Offset =
4063       llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7)  & ~7);
4064   overflow_arg_area = CGF.Builder.CreateGEP(CGF.Int8Ty, overflow_arg_area,
4065                                             Offset, "overflow_arg_area.next");
4066   CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
4067 
4068   // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
4069   return Address(Res, LTy, Align);
4070 }
4071 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const4072 Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4073                                  QualType Ty) const {
4074   // Assume that va_list type is correct; should be pointer to LLVM type:
4075   // struct {
4076   //   i32 gp_offset;
4077   //   i32 fp_offset;
4078   //   i8* overflow_arg_area;
4079   //   i8* reg_save_area;
4080   // };
4081   unsigned neededInt, neededSSE;
4082 
4083   Ty = getContext().getCanonicalType(Ty);
4084   ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
4085                                        /*isNamedArg*/false);
4086 
4087   // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
4088   // in the registers. If not go to step 7.
4089   if (!neededInt && !neededSSE)
4090     return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4091 
4092   // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
4093   // general purpose registers needed to pass type and num_fp to hold
4094   // the number of floating point registers needed.
4095 
4096   // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
4097   // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
4098   // l->fp_offset > 304 - num_fp * 16 go to step 7.
4099   //
4100   // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
4101   // register save space).
4102 
4103   llvm::Value *InRegs = nullptr;
4104   Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
4105   llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
4106   if (neededInt) {
4107     gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
4108     gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
4109     InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
4110     InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
4111   }
4112 
4113   if (neededSSE) {
4114     fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
4115     fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
4116     llvm::Value *FitsInFP =
4117       llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
4118     FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");
4119     InRegs = InRegs ? CGF.Builder.CreateAnd(InRegs, FitsInFP) : FitsInFP;
4120   }
4121 
4122   llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
4123   llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
4124   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
4125   CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
4126 
4127   // Emit code to load the value if it was passed in registers.
4128 
4129   CGF.EmitBlock(InRegBlock);
4130 
4131   // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
4132   // an offset of l->gp_offset and/or l->fp_offset. This may require
4133   // copying to a temporary location in case the parameter is passed
4134   // in different register classes or requires an alignment greater
4135   // than 8 for general purpose registers and 16 for XMM registers.
4136   //
4137   // FIXME: This really results in shameful code when we end up needing to
4138   // collect arguments from different places; often what should result in a
4139   // simple assembling of a structure from scattered addresses has many more
4140   // loads than necessary. Can we clean this up?
4141   llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4142   llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
4143       CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");
4144 
4145   Address RegAddr = Address::invalid();
4146   if (neededInt && neededSSE) {
4147     // FIXME: Cleanup.
4148     assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
4149     llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
4150     Address Tmp = CGF.CreateMemTemp(Ty);
4151     Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4152     assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
4153     llvm::Type *TyLo = ST->getElementType(0);
4154     llvm::Type *TyHi = ST->getElementType(1);
4155     assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
4156            "Unexpected ABI info for mixed regs");
4157     llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
4158     llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
4159     llvm::Value *GPAddr =
4160         CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset);
4161     llvm::Value *FPAddr =
4162         CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset);
4163     llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
4164     llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
4165 
4166     // Copy the first element.
4167     // FIXME: Our choice of alignment here and below is probably pessimistic.
4168     llvm::Value *V = CGF.Builder.CreateAlignedLoad(
4169         TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo),
4170         CharUnits::fromQuantity(getDataLayout().getABITypeAlign(TyLo)));
4171     CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4172 
4173     // Copy the second element.
4174     V = CGF.Builder.CreateAlignedLoad(
4175         TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi),
4176         CharUnits::fromQuantity(getDataLayout().getABITypeAlign(TyHi)));
4177     CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4178 
4179     RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4180   } else if (neededInt) {
4181     RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset),
4182                       CGF.Int8Ty, CharUnits::fromQuantity(8));
4183     RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4184 
4185     // Copy to a temporary if necessary to ensure the appropriate alignment.
4186     auto TInfo = getContext().getTypeInfoInChars(Ty);
4187     uint64_t TySize = TInfo.Width.getQuantity();
4188     CharUnits TyAlign = TInfo.Align;
4189 
4190     // Copy into a temporary if the type is more aligned than the
4191     // register save area.
4192     if (TyAlign.getQuantity() > 8) {
4193       Address Tmp = CGF.CreateMemTemp(Ty);
4194       CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
4195       RegAddr = Tmp;
4196     }
4197 
4198   } else if (neededSSE == 1) {
4199     RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset),
4200                       CGF.Int8Ty, CharUnits::fromQuantity(16));
4201     RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4202   } else {
4203     assert(neededSSE == 2 && "Invalid number of needed registers!");
4204     // SSE registers are spaced 16 bytes apart in the register save
4205     // area, we need to collect the two eightbytes together.
4206     // The ABI isn't explicit about this, but it seems reasonable
4207     // to assume that the slots are 16-byte aligned, since the stack is
4208     // naturally 16-byte aligned and the prologue is expected to store
4209     // all the SSE registers to the RSA.
4210     Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,
4211                                                       fp_offset),
4212                                 CGF.Int8Ty, CharUnits::fromQuantity(16));
4213     Address RegAddrHi =
4214       CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
4215                                              CharUnits::fromQuantity(16));
4216     llvm::Type *ST = AI.canHaveCoerceToType()
4217                          ? AI.getCoerceToType()
4218                          : llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy);
4219     llvm::Value *V;
4220     Address Tmp = CGF.CreateMemTemp(Ty);
4221     Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4222     V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4223         RegAddrLo, ST->getStructElementType(0)));
4224     CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4225     V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4226         RegAddrHi, ST->getStructElementType(1)));
4227     CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4228 
4229     RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4230   }
4231 
4232   // AMD64-ABI 3.5.7p5: Step 5. Set:
4233   // l->gp_offset = l->gp_offset + num_gp * 8
4234   // l->fp_offset = l->fp_offset + num_fp * 16.
4235   if (neededInt) {
4236     llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);
4237     CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
4238                             gp_offset_p);
4239   }
4240   if (neededSSE) {
4241     llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);
4242     CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
4243                             fp_offset_p);
4244   }
4245   CGF.EmitBranch(ContBlock);
4246 
4247   // Emit code to load the value if it was passed in memory.
4248 
4249   CGF.EmitBlock(InMemBlock);
4250   Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4251 
4252   // Return the appropriate result.
4253 
4254   CGF.EmitBlock(ContBlock);
4255   Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
4256                                  "vaarg.addr");
4257   return ResAddr;
4258 }
4259 
EmitMSVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const4260 Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
4261                                    QualType Ty) const {
4262   // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4263   // not 1, 2, 4, or 8 bytes, must be passed by reference."
4264   uint64_t Width = getContext().getTypeSize(Ty);
4265   bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);
4266 
4267   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4268                           CGF.getContext().getTypeInfoInChars(Ty),
4269                           CharUnits::fromQuantity(8),
4270                           /*allowHigherAlign*/ false);
4271 }
4272 
reclassifyHvaArgForVectorCall(QualType Ty,unsigned & FreeSSERegs,const ABIArgInfo & current) const4273 ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
4274     QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
4275   const Type *Base = nullptr;
4276   uint64_t NumElts = 0;
4277 
4278   if (!Ty->isBuiltinType() && !Ty->isVectorType() &&
4279       isHomogeneousAggregate(Ty, Base, NumElts) && FreeSSERegs >= NumElts) {
4280     FreeSSERegs -= NumElts;
4281     return getDirectX86Hva();
4282   }
4283   return current;
4284 }
4285 
classify(QualType Ty,unsigned & FreeSSERegs,bool IsReturnType,bool IsVectorCall,bool IsRegCall) const4286 ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
4287                                       bool IsReturnType, bool IsVectorCall,
4288                                       bool IsRegCall) const {
4289 
4290   if (Ty->isVoidType())
4291     return ABIArgInfo::getIgnore();
4292 
4293   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4294     Ty = EnumTy->getDecl()->getIntegerType();
4295 
4296   TypeInfo Info = getContext().getTypeInfo(Ty);
4297   uint64_t Width = Info.Width;
4298   CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
4299 
4300   const RecordType *RT = Ty->getAs<RecordType>();
4301   if (RT) {
4302     if (!IsReturnType) {
4303       if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
4304         return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4305     }
4306 
4307     if (RT->getDecl()->hasFlexibleArrayMember())
4308       return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4309 
4310   }
4311 
4312   const Type *Base = nullptr;
4313   uint64_t NumElts = 0;
4314   // vectorcall adds the concept of a homogenous vector aggregate, similar to
4315   // other targets.
4316   if ((IsVectorCall || IsRegCall) &&
4317       isHomogeneousAggregate(Ty, Base, NumElts)) {
4318     if (IsRegCall) {
4319       if (FreeSSERegs >= NumElts) {
4320         FreeSSERegs -= NumElts;
4321         if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())
4322           return ABIArgInfo::getDirect();
4323         return ABIArgInfo::getExpand();
4324       }
4325       return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4326     } else if (IsVectorCall) {
4327       if (FreeSSERegs >= NumElts &&
4328           (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())) {
4329         FreeSSERegs -= NumElts;
4330         return ABIArgInfo::getDirect();
4331       } else if (IsReturnType) {
4332         return ABIArgInfo::getExpand();
4333       } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {
4334         // HVAs are delayed and reclassified in the 2nd step.
4335         return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4336       }
4337     }
4338   }
4339 
4340   if (Ty->isMemberPointerType()) {
4341     // If the member pointer is represented by an LLVM int or ptr, pass it
4342     // directly.
4343     llvm::Type *LLTy = CGT.ConvertType(Ty);
4344     if (LLTy->isPointerTy() || LLTy->isIntegerTy())
4345       return ABIArgInfo::getDirect();
4346   }
4347 
4348   if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {
4349     // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4350     // not 1, 2, 4, or 8 bytes, must be passed by reference."
4351     if (Width > 64 || !llvm::isPowerOf2_64(Width))
4352       return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4353 
4354     // Otherwise, coerce it to a small integer.
4355     return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
4356   }
4357 
4358   if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
4359     switch (BT->getKind()) {
4360     case BuiltinType::Bool:
4361       // Bool type is always extended to the ABI, other builtin types are not
4362       // extended.
4363       return ABIArgInfo::getExtend(Ty);
4364 
4365     case BuiltinType::LongDouble:
4366       // Mingw64 GCC uses the old 80 bit extended precision floating point
4367       // unit. It passes them indirectly through memory.
4368       if (IsMingw64) {
4369         const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
4370         if (LDF == &llvm::APFloat::x87DoubleExtended())
4371           return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4372       }
4373       break;
4374 
4375     case BuiltinType::Int128:
4376     case BuiltinType::UInt128:
4377       // If it's a parameter type, the normal ABI rule is that arguments larger
4378       // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
4379       // even though it isn't particularly efficient.
4380       if (!IsReturnType)
4381         return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4382 
4383       // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
4384       // Clang matches them for compatibility.
4385       return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
4386           llvm::Type::getInt64Ty(getVMContext()), 2));
4387 
4388     default:
4389       break;
4390     }
4391   }
4392 
4393   if (Ty->isBitIntType()) {
4394     // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4395     // not 1, 2, 4, or 8 bytes, must be passed by reference."
4396     // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
4397     // or 8 bytes anyway as long is it fits in them, so we don't have to check
4398     // the power of 2.
4399     if (Width <= 64)
4400       return ABIArgInfo::getDirect();
4401     return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4402   }
4403 
4404   return ABIArgInfo::getDirect();
4405 }
4406 
computeInfo(CGFunctionInfo & FI) const4407 void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
4408   const unsigned CC = FI.getCallingConvention();
4409   bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
4410   bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
4411 
4412   // If __attribute__((sysv_abi)) is in use, use the SysV argument
4413   // classification rules.
4414   if (CC == llvm::CallingConv::X86_64_SysV) {
4415     X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
4416     SysVABIInfo.computeInfo(FI);
4417     return;
4418   }
4419 
4420   unsigned FreeSSERegs = 0;
4421   if (IsVectorCall) {
4422     // We can use up to 4 SSE return registers with vectorcall.
4423     FreeSSERegs = 4;
4424   } else if (IsRegCall) {
4425     // RegCall gives us 16 SSE registers.
4426     FreeSSERegs = 16;
4427   }
4428 
4429   if (!getCXXABI().classifyReturnType(FI))
4430     FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,
4431                                   IsVectorCall, IsRegCall);
4432 
4433   if (IsVectorCall) {
4434     // We can use up to 6 SSE register parameters with vectorcall.
4435     FreeSSERegs = 6;
4436   } else if (IsRegCall) {
4437     // RegCall gives us 16 SSE registers, we can reuse the return registers.
4438     FreeSSERegs = 16;
4439   }
4440 
4441   unsigned ArgNum = 0;
4442   unsigned ZeroSSERegs = 0;
4443   for (auto &I : FI.arguments()) {
4444     // Vectorcall in x64 only permits the first 6 arguments to be passed as
4445     // XMM/YMM registers. After the sixth argument, pretend no vector
4446     // registers are left.
4447     unsigned *MaybeFreeSSERegs =
4448         (IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;
4449     I.info =
4450         classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);
4451     ++ArgNum;
4452   }
4453 
4454   if (IsVectorCall) {
4455     // For vectorcall, assign aggregate HVAs to any free vector registers in a
4456     // second pass.
4457     for (auto &I : FI.arguments())
4458       I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);
4459   }
4460 }
4461 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const4462 Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4463                                     QualType Ty) const {
4464   // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4465   // not 1, 2, 4, or 8 bytes, must be passed by reference."
4466   uint64_t Width = getContext().getTypeSize(Ty);
4467   bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);
4468 
4469   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4470                           CGF.getContext().getTypeInfoInChars(Ty),
4471                           CharUnits::fromQuantity(8),
4472                           /*allowHigherAlign*/ false);
4473 }
4474 
PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address,bool Is64Bit,bool IsAIX)4475 static bool PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4476                                         llvm::Value *Address, bool Is64Bit,
4477                                         bool IsAIX) {
4478   // This is calculated from the LLVM and GCC tables and verified
4479   // against gcc output.  AFAIK all PPC ABIs use the same encoding.
4480 
4481   CodeGen::CGBuilderTy &Builder = CGF.Builder;
4482 
4483   llvm::IntegerType *i8 = CGF.Int8Ty;
4484   llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
4485   llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
4486   llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
4487 
4488   // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers
4489   AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 0, 31);
4490 
4491   // 32-63: fp0-31, the 8-byte floating-point registers
4492   AssignToArrayRange(Builder, Address, Eight8, 32, 63);
4493 
4494   // 64-67 are various 4-byte or 8-byte special-purpose registers:
4495   // 64: mq
4496   // 65: lr
4497   // 66: ctr
4498   // 67: ap
4499   AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 64, 67);
4500 
4501   // 68-76 are various 4-byte special-purpose registers:
4502   // 68-75 cr0-7
4503   // 76: xer
4504   AssignToArrayRange(Builder, Address, Four8, 68, 76);
4505 
4506   // 77-108: v0-31, the 16-byte vector registers
4507   AssignToArrayRange(Builder, Address, Sixteen8, 77, 108);
4508 
4509   // 109: vrsave
4510   // 110: vscr
4511   AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 109, 110);
4512 
4513   // AIX does not utilize the rest of the registers.
4514   if (IsAIX)
4515     return false;
4516 
4517   // 111: spe_acc
4518   // 112: spefscr
4519   // 113: sfp
4520   AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 111, 113);
4521 
4522   if (!Is64Bit)
4523     return false;
4524 
4525   // TODO: Need to verify if these registers are used on 64 bit AIX with Power8
4526   // or above CPU.
4527   // 64-bit only registers:
4528   // 114: tfhar
4529   // 115: tfiar
4530   // 116: texasr
4531   AssignToArrayRange(Builder, Address, Eight8, 114, 116);
4532 
4533   return false;
4534 }
4535 
4536 // AIX
4537 namespace {
4538 /// AIXABIInfo - The AIX XCOFF ABI information.
4539 class AIXABIInfo : public ABIInfo {
4540   const bool Is64Bit;
4541   const unsigned PtrByteSize;
4542   CharUnits getParamTypeAlignment(QualType Ty) const;
4543 
4544 public:
AIXABIInfo(CodeGen::CodeGenTypes & CGT,bool Is64Bit)4545   AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4546       : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {}
4547 
4548   bool isPromotableTypeForABI(QualType Ty) const;
4549 
4550   ABIArgInfo classifyReturnType(QualType RetTy) const;
4551   ABIArgInfo classifyArgumentType(QualType Ty) const;
4552 
computeInfo(CGFunctionInfo & FI) const4553   void computeInfo(CGFunctionInfo &FI) const override {
4554     if (!getCXXABI().classifyReturnType(FI))
4555       FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4556 
4557     for (auto &I : FI.arguments())
4558       I.info = classifyArgumentType(I.type);
4559   }
4560 
4561   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4562                     QualType Ty) const override;
4563 };
4564 
4565 class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
4566   const bool Is64Bit;
4567 
4568 public:
AIXTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,bool Is64Bit)4569   AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4570       : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)),
4571         Is64Bit(Is64Bit) {}
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const4572   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4573     return 1; // r1 is the dedicated stack pointer
4574   }
4575 
4576   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4577                                llvm::Value *Address) const override;
4578 };
4579 } // namespace
4580 
4581 // Return true if the ABI requires Ty to be passed sign- or zero-
4582 // extended to 32/64 bits.
isPromotableTypeForABI(QualType Ty) const4583 bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
4584   // Treat an enum type as its underlying type.
4585   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4586     Ty = EnumTy->getDecl()->getIntegerType();
4587 
4588   // Promotable integer types are required to be promoted by the ABI.
4589   if (getContext().isPromotableIntegerType(Ty))
4590     return true;
4591 
4592   if (!Is64Bit)
4593     return false;
4594 
4595   // For 64 bit mode, in addition to the usual promotable integer types, we also
4596   // need to extend all 32-bit types, since the ABI requires promotion to 64
4597   // bits.
4598   if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4599     switch (BT->getKind()) {
4600     case BuiltinType::Int:
4601     case BuiltinType::UInt:
4602       return true;
4603     default:
4604       break;
4605     }
4606 
4607   return false;
4608 }
4609 
classifyReturnType(QualType RetTy) const4610 ABIArgInfo AIXABIInfo::classifyReturnType(QualType RetTy) const {
4611   if (RetTy->isAnyComplexType())
4612     return ABIArgInfo::getDirect();
4613 
4614   if (RetTy->isVectorType())
4615     return ABIArgInfo::getDirect();
4616 
4617   if (RetTy->isVoidType())
4618     return ABIArgInfo::getIgnore();
4619 
4620   if (isAggregateTypeForABI(RetTy))
4621     return getNaturalAlignIndirect(RetTy);
4622 
4623   return (isPromotableTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
4624                                         : ABIArgInfo::getDirect());
4625 }
4626 
classifyArgumentType(QualType Ty) const4627 ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
4628   Ty = useFirstFieldIfTransparentUnion(Ty);
4629 
4630   if (Ty->isAnyComplexType())
4631     return ABIArgInfo::getDirect();
4632 
4633   if (Ty->isVectorType())
4634     return ABIArgInfo::getDirect();
4635 
4636   if (isAggregateTypeForABI(Ty)) {
4637     // Records with non-trivial destructors/copy-constructors should not be
4638     // passed by value.
4639     if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
4640       return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4641 
4642     CharUnits CCAlign = getParamTypeAlignment(Ty);
4643     CharUnits TyAlign = getContext().getTypeAlignInChars(Ty);
4644 
4645     return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true,
4646                                    /*Realign*/ TyAlign > CCAlign);
4647   }
4648 
4649   return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
4650                                      : ABIArgInfo::getDirect());
4651 }
4652 
getParamTypeAlignment(QualType Ty) const4653 CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
4654   // Complex types are passed just like their elements.
4655   if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4656     Ty = CTy->getElementType();
4657 
4658   if (Ty->isVectorType())
4659     return CharUnits::fromQuantity(16);
4660 
4661   // If the structure contains a vector type, the alignment is 16.
4662   if (isRecordWithSIMDVectorType(getContext(), Ty))
4663     return CharUnits::fromQuantity(16);
4664 
4665   return CharUnits::fromQuantity(PtrByteSize);
4666 }
4667 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const4668 Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4669                               QualType Ty) const {
4670 
4671   auto TypeInfo = getContext().getTypeInfoInChars(Ty);
4672   TypeInfo.Align = getParamTypeAlignment(Ty);
4673 
4674   CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize);
4675 
4676   // If we have a complex type and the base type is smaller than the register
4677   // size, the ABI calls for the real and imaginary parts to be right-adjusted
4678   // in separate words in 32bit mode or doublewords in 64bit mode. However,
4679   // Clang expects us to produce a pointer to a structure with the two parts
4680   // packed tightly. So generate loads of the real and imaginary parts relative
4681   // to the va_list pointer, and store them to a temporary structure. We do the
4682   // same as the PPC64ABI here.
4683   if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4684     CharUnits EltSize = TypeInfo.Width / 2;
4685     if (EltSize < SlotSize)
4686       return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
4687   }
4688 
4689   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
4690                           SlotSize, /*AllowHigher*/ true);
4691 }
4692 
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const4693 bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable(
4694     CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const {
4695   return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true);
4696 }
4697 
4698 // PowerPC-32
4699 namespace {
4700 /// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
4701 class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
4702   bool IsSoftFloatABI;
4703   bool IsRetSmallStructInRegABI;
4704 
4705   CharUnits getParamTypeAlignment(QualType Ty) const;
4706 
4707 public:
PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes & CGT,bool SoftFloatABI,bool RetSmallStructInRegABI)4708   PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI,
4709                      bool RetSmallStructInRegABI)
4710       : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI),
4711         IsRetSmallStructInRegABI(RetSmallStructInRegABI) {}
4712 
4713   ABIArgInfo classifyReturnType(QualType RetTy) const;
4714 
computeInfo(CGFunctionInfo & FI) const4715   void computeInfo(CGFunctionInfo &FI) const override {
4716     if (!getCXXABI().classifyReturnType(FI))
4717       FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4718     for (auto &I : FI.arguments())
4719       I.info = classifyArgumentType(I.type);
4720   }
4721 
4722   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4723                     QualType Ty) const override;
4724 };
4725 
4726 class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
4727 public:
PPC32TargetCodeGenInfo(CodeGenTypes & CGT,bool SoftFloatABI,bool RetSmallStructInRegABI)4728   PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI,
4729                          bool RetSmallStructInRegABI)
4730       : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>(
4731             CGT, SoftFloatABI, RetSmallStructInRegABI)) {}
4732 
4733   static bool isStructReturnInRegABI(const llvm::Triple &Triple,
4734                                      const CodeGenOptions &Opts);
4735 
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const4736   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4737     // This is recovered from gcc output.
4738     return 1; // r1 is the dedicated stack pointer
4739   }
4740 
4741   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4742                                llvm::Value *Address) const override;
4743 };
4744 }
4745 
getParamTypeAlignment(QualType Ty) const4746 CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
4747   // Complex types are passed just like their elements.
4748   if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4749     Ty = CTy->getElementType();
4750 
4751   if (Ty->isVectorType())
4752     return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16
4753                                                                        : 4);
4754 
4755   // For single-element float/vector structs, we consider the whole type
4756   // to have the same alignment requirements as its single element.
4757   const Type *AlignTy = nullptr;
4758   if (const Type *EltType = isSingleElementStruct(Ty, getContext())) {
4759     const BuiltinType *BT = EltType->getAs<BuiltinType>();
4760     if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
4761         (BT && BT->isFloatingPoint()))
4762       AlignTy = EltType;
4763   }
4764 
4765   if (AlignTy)
4766     return CharUnits::fromQuantity(AlignTy->isVectorType() ? 16 : 4);
4767   return CharUnits::fromQuantity(4);
4768 }
4769 
classifyReturnType(QualType RetTy) const4770 ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
4771   uint64_t Size;
4772 
4773   // -msvr4-struct-return puts small aggregates in GPR3 and GPR4.
4774   if (isAggregateTypeForABI(RetTy) && IsRetSmallStructInRegABI &&
4775       (Size = getContext().getTypeSize(RetTy)) <= 64) {
4776     // System V ABI (1995), page 3-22, specified:
4777     // > A structure or union whose size is less than or equal to 8 bytes
4778     // > shall be returned in r3 and r4, as if it were first stored in the
4779     // > 8-byte aligned memory area and then the low addressed word were
4780     // > loaded into r3 and the high-addressed word into r4.  Bits beyond
4781     // > the last member of the structure or union are not defined.
4782     //
4783     // GCC for big-endian PPC32 inserts the pad before the first member,
4784     // not "beyond the last member" of the struct.  To stay compatible
4785     // with GCC, we coerce the struct to an integer of the same size.
4786     // LLVM will extend it and return i32 in r3, or i64 in r3:r4.
4787     if (Size == 0)
4788       return ABIArgInfo::getIgnore();
4789     else {
4790       llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size);
4791       return ABIArgInfo::getDirect(CoerceTy);
4792     }
4793   }
4794 
4795   return DefaultABIInfo::classifyReturnType(RetTy);
4796 }
4797 
4798 // TODO: this implementation is now likely redundant with
4799 // DefaultABIInfo::EmitVAArg.
EmitVAArg(CodeGenFunction & CGF,Address VAList,QualType Ty) const4800 Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
4801                                       QualType Ty) const {
4802   if (getTarget().getTriple().isOSDarwin()) {
4803     auto TI = getContext().getTypeInfoInChars(Ty);
4804     TI.Align = getParamTypeAlignment(Ty);
4805 
4806     CharUnits SlotSize = CharUnits::fromQuantity(4);
4807     return emitVoidPtrVAArg(CGF, VAList, Ty,
4808                             classifyArgumentType(Ty).isIndirect(), TI, SlotSize,
4809                             /*AllowHigherAlign=*/true);
4810   }
4811 
4812   const unsigned OverflowLimit = 8;
4813   if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4814     // TODO: Implement this. For now ignore.
4815     (void)CTy;
4816     return Address::invalid(); // FIXME?
4817   }
4818 
4819   // struct __va_list_tag {
4820   //   unsigned char gpr;
4821   //   unsigned char fpr;
4822   //   unsigned short reserved;
4823   //   void *overflow_arg_area;
4824   //   void *reg_save_area;
4825   // };
4826 
4827   bool isI64 = Ty->isIntegerType() && getContext().getTypeSize(Ty) == 64;
4828   bool isInt = !Ty->isFloatingType();
4829   bool isF64 = Ty->isFloatingType() && getContext().getTypeSize(Ty) == 64;
4830 
4831   // All aggregates are passed indirectly?  That doesn't seem consistent
4832   // with the argument-lowering code.
4833   bool isIndirect = isAggregateTypeForABI(Ty);
4834 
4835   CGBuilderTy &Builder = CGF.Builder;
4836 
4837   // The calling convention either uses 1-2 GPRs or 1 FPR.
4838   Address NumRegsAddr = Address::invalid();
4839   if (isInt || IsSoftFloatABI) {
4840     NumRegsAddr = Builder.CreateStructGEP(VAList, 0, "gpr");
4841   } else {
4842     NumRegsAddr = Builder.CreateStructGEP(VAList, 1, "fpr");
4843   }
4844 
4845   llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
4846 
4847   // "Align" the register count when TY is i64.
4848   if (isI64 || (isF64 && IsSoftFloatABI)) {
4849     NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
4850     NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
4851   }
4852 
4853   llvm::Value *CC =
4854       Builder.CreateICmpULT(NumRegs, Builder.getInt8(OverflowLimit), "cond");
4855 
4856   llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
4857   llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
4858   llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
4859 
4860   Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
4861 
4862   llvm::Type *DirectTy = CGF.ConvertType(Ty), *ElementTy = DirectTy;
4863   if (isIndirect) DirectTy = DirectTy->getPointerTo(0);
4864 
4865   // Case 1: consume registers.
4866   Address RegAddr = Address::invalid();
4867   {
4868     CGF.EmitBlock(UsingRegs);
4869 
4870     Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
4871     RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr), CGF.Int8Ty,
4872                       CharUnits::fromQuantity(8));
4873     assert(RegAddr.getElementType() == CGF.Int8Ty);
4874 
4875     // Floating-point registers start after the general-purpose registers.
4876     if (!(isInt || IsSoftFloatABI)) {
4877       RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
4878                                                    CharUnits::fromQuantity(32));
4879     }
4880 
4881     // Get the address of the saved value by scaling the number of
4882     // registers we've used by the number of
4883     CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8);
4884     llvm::Value *RegOffset =
4885         Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
4886     RegAddr = Address(
4887         Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset),
4888         CGF.Int8Ty, RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
4889     RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
4890 
4891     // Increase the used-register count.
4892     NumRegs =
4893       Builder.CreateAdd(NumRegs,
4894                         Builder.getInt8((isI64 || (isF64 && IsSoftFloatABI)) ? 2 : 1));
4895     Builder.CreateStore(NumRegs, NumRegsAddr);
4896 
4897     CGF.EmitBranch(Cont);
4898   }
4899 
4900   // Case 2: consume space in the overflow area.
4901   Address MemAddr = Address::invalid();
4902   {
4903     CGF.EmitBlock(UsingOverflow);
4904 
4905     Builder.CreateStore(Builder.getInt8(OverflowLimit), NumRegsAddr);
4906 
4907     // Everything in the overflow area is rounded up to a size of at least 4.
4908     CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
4909 
4910     CharUnits Size;
4911     if (!isIndirect) {
4912       auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
4913       Size = TypeInfo.Width.alignTo(OverflowAreaAlign);
4914     } else {
4915       Size = CGF.getPointerSize();
4916     }
4917 
4918     Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
4919     Address OverflowArea =
4920         Address(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"), CGF.Int8Ty,
4921                 OverflowAreaAlign);
4922     // Round up address of argument to alignment
4923     CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4924     if (Align > OverflowAreaAlign) {
4925       llvm::Value *Ptr = OverflowArea.getPointer();
4926       OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
4927                              OverflowArea.getElementType(), Align);
4928     }
4929 
4930     MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
4931 
4932     // Increase the overflow area.
4933     OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
4934     Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
4935     CGF.EmitBranch(Cont);
4936   }
4937 
4938   CGF.EmitBlock(Cont);
4939 
4940   // Merge the cases with a phi.
4941   Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
4942                                 "vaarg.addr");
4943 
4944   // Load the pointer if the argument was passed indirectly.
4945   if (isIndirect) {
4946     Result = Address(Builder.CreateLoad(Result, "aggr"), ElementTy,
4947                      getContext().getTypeAlignInChars(Ty));
4948   }
4949 
4950   return Result;
4951 }
4952 
isStructReturnInRegABI(const llvm::Triple & Triple,const CodeGenOptions & Opts)4953 bool PPC32TargetCodeGenInfo::isStructReturnInRegABI(
4954     const llvm::Triple &Triple, const CodeGenOptions &Opts) {
4955   assert(Triple.isPPC32());
4956 
4957   switch (Opts.getStructReturnConvention()) {
4958   case CodeGenOptions::SRCK_Default:
4959     break;
4960   case CodeGenOptions::SRCK_OnStack: // -maix-struct-return
4961     return false;
4962   case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return
4963     return true;
4964   }
4965 
4966   if (Triple.isOSBinFormatELF() && !Triple.isOSLinux())
4967     return true;
4968 
4969   return false;
4970 }
4971 
4972 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const4973 PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4974                                                 llvm::Value *Address) const {
4975   return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false,
4976                                      /*IsAIX*/ false);
4977 }
4978 
4979 // PowerPC-64
4980 
4981 namespace {
4982 /// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
4983 class PPC64_SVR4_ABIInfo : public ABIInfo {
4984 public:
4985   enum ABIKind {
4986     ELFv1 = 0,
4987     ELFv2
4988   };
4989 
4990 private:
4991   static const unsigned GPRBits = 64;
4992   ABIKind Kind;
4993   bool IsSoftFloatABI;
4994 
4995 public:
PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes & CGT,ABIKind Kind,bool SoftFloatABI)4996   PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind,
4997                      bool SoftFloatABI)
4998       : ABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
4999 
5000   bool isPromotableTypeForABI(QualType Ty) const;
5001   CharUnits getParamTypeAlignment(QualType Ty) const;
5002 
5003   ABIArgInfo classifyReturnType(QualType RetTy) const;
5004   ABIArgInfo classifyArgumentType(QualType Ty) const;
5005 
5006   bool isHomogeneousAggregateBaseType(QualType Ty) const override;
5007   bool isHomogeneousAggregateSmallEnough(const Type *Ty,
5008                                          uint64_t Members) const override;
5009 
5010   // TODO: We can add more logic to computeInfo to improve performance.
5011   // Example: For aggregate arguments that fit in a register, we could
5012   // use getDirectInReg (as is done below for structs containing a single
5013   // floating-point value) to avoid pushing them to memory on function
5014   // entry.  This would require changing the logic in PPCISelLowering
5015   // when lowering the parameters in the caller and args in the callee.
computeInfo(CGFunctionInfo & FI) const5016   void computeInfo(CGFunctionInfo &FI) const override {
5017     if (!getCXXABI().classifyReturnType(FI))
5018       FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
5019     for (auto &I : FI.arguments()) {
5020       // We rely on the default argument classification for the most part.
5021       // One exception:  An aggregate containing a single floating-point
5022       // or vector item must be passed in a register if one is available.
5023       const Type *T = isSingleElementStruct(I.type, getContext());
5024       if (T) {
5025         const BuiltinType *BT = T->getAs<BuiltinType>();
5026         if ((T->isVectorType() && getContext().getTypeSize(T) == 128) ||
5027             (BT && BT->isFloatingPoint())) {
5028           QualType QT(T, 0);
5029           I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
5030           continue;
5031         }
5032       }
5033       I.info = classifyArgumentType(I.type);
5034     }
5035   }
5036 
5037   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5038                     QualType Ty) const override;
5039 };
5040 
5041 class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
5042 
5043 public:
PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes & CGT,PPC64_SVR4_ABIInfo::ABIKind Kind,bool SoftFloatABI)5044   PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
5045                                PPC64_SVR4_ABIInfo::ABIKind Kind,
5046                                bool SoftFloatABI)
5047       : TargetCodeGenInfo(
5048             std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {
5049     SwiftInfo =
5050         std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/false);
5051   }
5052 
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const5053   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5054     // This is recovered from gcc output.
5055     return 1; // r1 is the dedicated stack pointer
5056   }
5057 
5058   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5059                                llvm::Value *Address) const override;
5060 };
5061 
5062 class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
5063 public:
PPC64TargetCodeGenInfo(CodeGenTypes & CGT)5064   PPC64TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
5065 
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const5066   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5067     // This is recovered from gcc output.
5068     return 1; // r1 is the dedicated stack pointer
5069   }
5070 
5071   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5072                                llvm::Value *Address) const override;
5073 };
5074 
5075 }
5076 
5077 // Return true if the ABI requires Ty to be passed sign- or zero-
5078 // extended to 64 bits.
5079 bool
isPromotableTypeForABI(QualType Ty) const5080 PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
5081   // Treat an enum type as its underlying type.
5082   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5083     Ty = EnumTy->getDecl()->getIntegerType();
5084 
5085   // Promotable integer types are required to be promoted by the ABI.
5086   if (isPromotableIntegerTypeForABI(Ty))
5087     return true;
5088 
5089   // In addition to the usual promotable integer types, we also need to
5090   // extend all 32-bit types, since the ABI requires promotion to 64 bits.
5091   if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
5092     switch (BT->getKind()) {
5093     case BuiltinType::Int:
5094     case BuiltinType::UInt:
5095       return true;
5096     default:
5097       break;
5098     }
5099 
5100   if (const auto *EIT = Ty->getAs<BitIntType>())
5101     if (EIT->getNumBits() < 64)
5102       return true;
5103 
5104   return false;
5105 }
5106 
5107 /// isAlignedParamType - Determine whether a type requires 16-byte or
5108 /// higher alignment in the parameter area.  Always returns at least 8.
getParamTypeAlignment(QualType Ty) const5109 CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
5110   // Complex types are passed just like their elements.
5111   if (const ComplexType *CTy = Ty->getAs<ComplexType>())
5112     Ty = CTy->getElementType();
5113 
5114   auto FloatUsesVector = [this](QualType Ty){
5115     return Ty->isRealFloatingType() && &getContext().getFloatTypeSemantics(
5116                                            Ty) == &llvm::APFloat::IEEEquad();
5117   };
5118 
5119   // Only vector types of size 16 bytes need alignment (larger types are
5120   // passed via reference, smaller types are not aligned).
5121   if (Ty->isVectorType()) {
5122     return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
5123   } else if (FloatUsesVector(Ty)) {
5124     // According to ABI document section 'Optional Save Areas': If extended
5125     // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
5126     // format are supported, map them to a single quadword, quadword aligned.
5127     return CharUnits::fromQuantity(16);
5128   }
5129 
5130   // For single-element float/vector structs, we consider the whole type
5131   // to have the same alignment requirements as its single element.
5132   const Type *AlignAsType = nullptr;
5133   const Type *EltType = isSingleElementStruct(Ty, getContext());
5134   if (EltType) {
5135     const BuiltinType *BT = EltType->getAs<BuiltinType>();
5136     if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
5137         (BT && BT->isFloatingPoint()))
5138       AlignAsType = EltType;
5139   }
5140 
5141   // Likewise for ELFv2 homogeneous aggregates.
5142   const Type *Base = nullptr;
5143   uint64_t Members = 0;
5144   if (!AlignAsType && Kind == ELFv2 &&
5145       isAggregateTypeForABI(Ty) && isHomogeneousAggregate(Ty, Base, Members))
5146     AlignAsType = Base;
5147 
5148   // With special case aggregates, only vector base types need alignment.
5149   if (AlignAsType) {
5150     bool UsesVector = AlignAsType->isVectorType() ||
5151                       FloatUsesVector(QualType(AlignAsType, 0));
5152     return CharUnits::fromQuantity(UsesVector ? 16 : 8);
5153   }
5154 
5155   // Otherwise, we only need alignment for any aggregate type that
5156   // has an alignment requirement of >= 16 bytes.
5157   if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) {
5158     return CharUnits::fromQuantity(16);
5159   }
5160 
5161   return CharUnits::fromQuantity(8);
5162 }
5163 
5164 /// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous
5165 /// aggregate.  Base is set to the base element type, and Members is set
5166 /// to the number of base elements.
isHomogeneousAggregate(QualType Ty,const Type * & Base,uint64_t & Members) const5167 bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
5168                                      uint64_t &Members) const {
5169   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
5170     uint64_t NElements = AT->getSize().getZExtValue();
5171     if (NElements == 0)
5172       return false;
5173     if (!isHomogeneousAggregate(AT->getElementType(), Base, Members))
5174       return false;
5175     Members *= NElements;
5176   } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
5177     const RecordDecl *RD = RT->getDecl();
5178     if (RD->hasFlexibleArrayMember())
5179       return false;
5180 
5181     Members = 0;
5182 
5183     // If this is a C++ record, check the properties of the record such as
5184     // bases and ABI specific restrictions
5185     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
5186       if (!getCXXABI().isPermittedToBeHomogeneousAggregate(CXXRD))
5187         return false;
5188 
5189       for (const auto &I : CXXRD->bases()) {
5190         // Ignore empty records.
5191         if (isEmptyRecord(getContext(), I.getType(), true))
5192           continue;
5193 
5194         uint64_t FldMembers;
5195         if (!isHomogeneousAggregate(I.getType(), Base, FldMembers))
5196           return false;
5197 
5198         Members += FldMembers;
5199       }
5200     }
5201 
5202     for (const auto *FD : RD->fields()) {
5203       // Ignore (non-zero arrays of) empty records.
5204       QualType FT = FD->getType();
5205       while (const ConstantArrayType *AT =
5206              getContext().getAsConstantArrayType(FT)) {
5207         if (AT->getSize().getZExtValue() == 0)
5208           return false;
5209         FT = AT->getElementType();
5210       }
5211       if (isEmptyRecord(getContext(), FT, true))
5212         continue;
5213 
5214       if (isZeroLengthBitfieldPermittedInHomogeneousAggregate() &&
5215           FD->isZeroLengthBitField(getContext()))
5216         continue;
5217 
5218       uint64_t FldMembers;
5219       if (!isHomogeneousAggregate(FD->getType(), Base, FldMembers))
5220         return false;
5221 
5222       Members = (RD->isUnion() ?
5223                  std::max(Members, FldMembers) : Members + FldMembers);
5224     }
5225 
5226     if (!Base)
5227       return false;
5228 
5229     // Ensure there is no padding.
5230     if (getContext().getTypeSize(Base) * Members !=
5231         getContext().getTypeSize(Ty))
5232       return false;
5233   } else {
5234     Members = 1;
5235     if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
5236       Members = 2;
5237       Ty = CT->getElementType();
5238     }
5239 
5240     // Most ABIs only support float, double, and some vector type widths.
5241     if (!isHomogeneousAggregateBaseType(Ty))
5242       return false;
5243 
5244     // The base type must be the same for all members.  Types that
5245     // agree in both total size and mode (float vs. vector) are
5246     // treated as being equivalent here.
5247     const Type *TyPtr = Ty.getTypePtr();
5248     if (!Base) {
5249       Base = TyPtr;
5250       // If it's a non-power-of-2 vector, its size is already a power-of-2,
5251       // so make sure to widen it explicitly.
5252       if (const VectorType *VT = Base->getAs<VectorType>()) {
5253         QualType EltTy = VT->getElementType();
5254         unsigned NumElements =
5255             getContext().getTypeSize(VT) / getContext().getTypeSize(EltTy);
5256         Base = getContext()
5257                    .getVectorType(EltTy, NumElements, VT->getVectorKind())
5258                    .getTypePtr();
5259       }
5260     }
5261 
5262     if (Base->isVectorType() != TyPtr->isVectorType() ||
5263         getContext().getTypeSize(Base) != getContext().getTypeSize(TyPtr))
5264       return false;
5265   }
5266   return Members > 0 && isHomogeneousAggregateSmallEnough(Base, Members);
5267 }
5268 
isHomogeneousAggregateBaseType(QualType Ty) const5269 bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
5270   // Homogeneous aggregates for ELFv2 must have base types of float,
5271   // double, long double, or 128-bit vectors.
5272   if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
5273     if (BT->getKind() == BuiltinType::Float ||
5274         BT->getKind() == BuiltinType::Double ||
5275         BT->getKind() == BuiltinType::LongDouble ||
5276         BT->getKind() == BuiltinType::Ibm128 ||
5277         (getContext().getTargetInfo().hasFloat128Type() &&
5278          (BT->getKind() == BuiltinType::Float128))) {
5279       if (IsSoftFloatABI)
5280         return false;
5281       return true;
5282     }
5283   }
5284   if (const VectorType *VT = Ty->getAs<VectorType>()) {
5285     if (getContext().getTypeSize(VT) == 128)
5286       return true;
5287   }
5288   return false;
5289 }
5290 
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const5291 bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateSmallEnough(
5292     const Type *Base, uint64_t Members) const {
5293   // Vector and fp128 types require one register, other floating point types
5294   // require one or two registers depending on their size.
5295   uint32_t NumRegs =
5296       ((getContext().getTargetInfo().hasFloat128Type() &&
5297           Base->isFloat128Type()) ||
5298         Base->isVectorType()) ? 1
5299                               : (getContext().getTypeSize(Base) + 63) / 64;
5300 
5301   // Homogeneous Aggregates may occupy at most 8 registers.
5302   return Members * NumRegs <= 8;
5303 }
5304 
5305 ABIArgInfo
classifyArgumentType(QualType Ty) const5306 PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const {
5307   Ty = useFirstFieldIfTransparentUnion(Ty);
5308 
5309   if (Ty->isAnyComplexType())
5310     return ABIArgInfo::getDirect();
5311 
5312   // Non-Altivec vector types are passed in GPRs (smaller than 16 bytes)
5313   // or via reference (larger than 16 bytes).
5314   if (Ty->isVectorType()) {
5315     uint64_t Size = getContext().getTypeSize(Ty);
5316     if (Size > 128)
5317       return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5318     else if (Size < 128) {
5319       llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
5320       return ABIArgInfo::getDirect(CoerceTy);
5321     }
5322   }
5323 
5324   if (const auto *EIT = Ty->getAs<BitIntType>())
5325     if (EIT->getNumBits() > 128)
5326       return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
5327 
5328   if (isAggregateTypeForABI(Ty)) {
5329     if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
5330       return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
5331 
5332     uint64_t ABIAlign = getParamTypeAlignment(Ty).getQuantity();
5333     uint64_t TyAlign = getContext().getTypeAlignInChars(Ty).getQuantity();
5334 
5335     // ELFv2 homogeneous aggregates are passed as array types.
5336     const Type *Base = nullptr;
5337     uint64_t Members = 0;
5338     if (Kind == ELFv2 &&
5339         isHomogeneousAggregate(Ty, Base, Members)) {
5340       llvm::Type *BaseTy = CGT.ConvertType(QualType(Base, 0));
5341       llvm::Type *CoerceTy = llvm::ArrayType::get(BaseTy, Members);
5342       return ABIArgInfo::getDirect(CoerceTy);
5343     }
5344 
5345     // If an aggregate may end up fully in registers, we do not
5346     // use the ByVal method, but pass the aggregate as array.
5347     // This is usually beneficial since we avoid forcing the
5348     // back-end to store the argument to memory.
5349     uint64_t Bits = getContext().getTypeSize(Ty);
5350     if (Bits > 0 && Bits <= 8 * GPRBits) {
5351       llvm::Type *CoerceTy;
5352 
5353       // Types up to 8 bytes are passed as integer type (which will be
5354       // properly aligned in the argument save area doubleword).
5355       if (Bits <= GPRBits)
5356         CoerceTy =
5357             llvm::IntegerType::get(getVMContext(), llvm::alignTo(Bits, 8));
5358       // Larger types are passed as arrays, with the base type selected
5359       // according to the required alignment in the save area.
5360       else {
5361         uint64_t RegBits = ABIAlign * 8;
5362         uint64_t NumRegs = llvm::alignTo(Bits, RegBits) / RegBits;
5363         llvm::Type *RegTy = llvm::IntegerType::get(getVMContext(), RegBits);
5364         CoerceTy = llvm::ArrayType::get(RegTy, NumRegs);
5365       }
5366 
5367       return ABIArgInfo::getDirect(CoerceTy);
5368     }
5369 
5370     // All other aggregates are passed ByVal.
5371     return ABIArgInfo::getIndirect(CharUnits::fromQuantity(ABIAlign),
5372                                    /*ByVal=*/true,
5373                                    /*Realign=*/TyAlign > ABIAlign);
5374   }
5375 
5376   return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
5377                                      : ABIArgInfo::getDirect());
5378 }
5379 
5380 ABIArgInfo
classifyReturnType(QualType RetTy) const5381 PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
5382   if (RetTy->isVoidType())
5383     return ABIArgInfo::getIgnore();
5384 
5385   if (RetTy->isAnyComplexType())
5386     return ABIArgInfo::getDirect();
5387 
5388   // Non-Altivec vector types are returned in GPRs (smaller than 16 bytes)
5389   // or via reference (larger than 16 bytes).
5390   if (RetTy->isVectorType()) {
5391     uint64_t Size = getContext().getTypeSize(RetTy);
5392     if (Size > 128)
5393       return getNaturalAlignIndirect(RetTy);
5394     else if (Size < 128) {
5395       llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
5396       return ABIArgInfo::getDirect(CoerceTy);
5397     }
5398   }
5399 
5400   if (const auto *EIT = RetTy->getAs<BitIntType>())
5401     if (EIT->getNumBits() > 128)
5402       return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
5403 
5404   if (isAggregateTypeForABI(RetTy)) {
5405     // ELFv2 homogeneous aggregates are returned as array types.
5406     const Type *Base = nullptr;
5407     uint64_t Members = 0;
5408     if (Kind == ELFv2 &&
5409         isHomogeneousAggregate(RetTy, Base, Members)) {
5410       llvm::Type *BaseTy = CGT.ConvertType(QualType(Base, 0));
5411       llvm::Type *CoerceTy = llvm::ArrayType::get(BaseTy, Members);
5412       return ABIArgInfo::getDirect(CoerceTy);
5413     }
5414 
5415     // ELFv2 small aggregates are returned in up to two registers.
5416     uint64_t Bits = getContext().getTypeSize(RetTy);
5417     if (Kind == ELFv2 && Bits <= 2 * GPRBits) {
5418       if (Bits == 0)
5419         return ABIArgInfo::getIgnore();
5420 
5421       llvm::Type *CoerceTy;
5422       if (Bits > GPRBits) {
5423         CoerceTy = llvm::IntegerType::get(getVMContext(), GPRBits);
5424         CoerceTy = llvm::StructType::get(CoerceTy, CoerceTy);
5425       } else
5426         CoerceTy =
5427             llvm::IntegerType::get(getVMContext(), llvm::alignTo(Bits, 8));
5428       return ABIArgInfo::getDirect(CoerceTy);
5429     }
5430 
5431     // All other aggregates are returned indirectly.
5432     return getNaturalAlignIndirect(RetTy);
5433   }
5434 
5435   return (isPromotableTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
5436                                         : ABIArgInfo::getDirect());
5437 }
5438 
5439 // Based on ARMABIInfo::EmitVAArg, adjusted for 64-bit machine.
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const5440 Address PPC64_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5441                                       QualType Ty) const {
5442   auto TypeInfo = getContext().getTypeInfoInChars(Ty);
5443   TypeInfo.Align = getParamTypeAlignment(Ty);
5444 
5445   CharUnits SlotSize = CharUnits::fromQuantity(8);
5446 
5447   // If we have a complex type and the base type is smaller than 8 bytes,
5448   // the ABI calls for the real and imaginary parts to be right-adjusted
5449   // in separate doublewords.  However, Clang expects us to produce a
5450   // pointer to a structure with the two parts packed tightly.  So generate
5451   // loads of the real and imaginary parts relative to the va_list pointer,
5452   // and store them to a temporary structure.
5453   if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
5454     CharUnits EltSize = TypeInfo.Width / 2;
5455     if (EltSize < SlotSize)
5456       return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
5457   }
5458 
5459   // Otherwise, just use the general rule.
5460   //
5461   // The PPC64 ABI passes some arguments in integer registers, even to variadic
5462   // functions. To allow va_list to use the simple "void*" representation,
5463   // variadic calls allocate space in the argument area for the integer argument
5464   // registers, and variadic functions spill their integer argument registers to
5465   // this area in their prologues. When aggregates smaller than a register are
5466   // passed this way, they are passed in the least significant bits of the
5467   // register, which means that after spilling on big-endian targets they will
5468   // be right-aligned in their argument slot. This is uncommon; for a variety of
5469   // reasons, other big-endian targets don't end up right-aligning aggregate
5470   // types this way, and so right-alignment only applies to fundamental types.
5471   // So on PPC64, we must force the use of right-alignment even for aggregates.
5472   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
5473                           SlotSize, /*AllowHigher*/ true,
5474                           /*ForceRightAdjust*/ true);
5475 }
5476 
5477 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const5478 PPC64_SVR4_TargetCodeGenInfo::initDwarfEHRegSizeTable(
5479   CodeGen::CodeGenFunction &CGF,
5480   llvm::Value *Address) const {
5481   return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ true,
5482                                      /*IsAIX*/ false);
5483 }
5484 
5485 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const5486 PPC64TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5487                                                 llvm::Value *Address) const {
5488   return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ true,
5489                                      /*IsAIX*/ false);
5490 }
5491 
5492 //===----------------------------------------------------------------------===//
5493 // AArch64 ABI Implementation
5494 //===----------------------------------------------------------------------===//
5495 
5496 namespace {
5497 
5498 class AArch64ABIInfo : public ABIInfo {
5499 public:
5500   enum ABIKind {
5501     AAPCS = 0,
5502     DarwinPCS,
5503     Win64
5504   };
5505 
5506 private:
5507   ABIKind Kind;
5508 
5509 public:
AArch64ABIInfo(CodeGenTypes & CGT,ABIKind Kind)5510   AArch64ABIInfo(CodeGenTypes &CGT, ABIKind Kind) : ABIInfo(CGT), Kind(Kind) {}
5511 
5512 private:
getABIKind() const5513   ABIKind getABIKind() const { return Kind; }
isDarwinPCS() const5514   bool isDarwinPCS() const { return Kind == DarwinPCS; }
5515 
5516   ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadic) const;
5517   ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadic,
5518                                   unsigned CallingConvention) const;
5519   ABIArgInfo coerceIllegalVector(QualType Ty) const;
5520   bool isHomogeneousAggregateBaseType(QualType Ty) const override;
5521   bool isHomogeneousAggregateSmallEnough(const Type *Ty,
5522                                          uint64_t Members) const override;
5523   bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
5524 
5525   bool isIllegalVectorType(QualType Ty) const;
5526 
computeInfo(CGFunctionInfo & FI) const5527   void computeInfo(CGFunctionInfo &FI) const override {
5528     if (!::classifyReturnType(getCXXABI(), FI, *this))
5529       FI.getReturnInfo() =
5530           classifyReturnType(FI.getReturnType(), FI.isVariadic());
5531 
5532     for (auto &it : FI.arguments())
5533       it.info = classifyArgumentType(it.type, FI.isVariadic(),
5534                                      FI.getCallingConvention());
5535   }
5536 
5537   Address EmitDarwinVAArg(Address VAListAddr, QualType Ty,
5538                           CodeGenFunction &CGF) const;
5539 
5540   Address EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
5541                          CodeGenFunction &CGF) const;
5542 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const5543   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5544                     QualType Ty) const override {
5545     llvm::Type *BaseTy = CGF.ConvertType(Ty);
5546     if (isa<llvm::ScalableVectorType>(BaseTy))
5547       llvm::report_fatal_error("Passing SVE types to variadic functions is "
5548                                "currently not supported");
5549 
5550     return Kind == Win64 ? EmitMSVAArg(CGF, VAListAddr, Ty)
5551                          : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF)
5552                                          : EmitAAPCSVAArg(VAListAddr, Ty, CGF);
5553   }
5554 
5555   Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
5556                       QualType Ty) const override;
5557 
allowBFloatArgsAndRet() const5558   bool allowBFloatArgsAndRet() const override {
5559     return getTarget().hasBFloat16Type();
5560   }
5561 };
5562 
5563 class AArch64SwiftABIInfo : public SwiftABIInfo {
5564 public:
AArch64SwiftABIInfo(CodeGenTypes & CGT)5565   explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)
5566       : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
5567 
5568   bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
5569                          unsigned NumElts) const override;
5570 };
5571 
5572 class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
5573 public:
AArch64TargetCodeGenInfo(CodeGenTypes & CGT,AArch64ABIInfo::ABIKind Kind)5574   AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIInfo::ABIKind Kind)
5575       : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(CGT, Kind)) {
5576     SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(CGT);
5577   }
5578 
getARCRetainAutoreleasedReturnValueMarker() const5579   StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
5580     return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
5581   }
5582 
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const5583   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5584     return 31;
5585   }
5586 
doesReturnSlotInterfereWithArgs() const5587   bool doesReturnSlotInterfereWithArgs() const override { return false; }
5588 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const5589   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
5590                            CodeGen::CodeGenModule &CGM) const override {
5591     const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
5592     if (!FD)
5593       return;
5594 
5595     const auto *TA = FD->getAttr<TargetAttr>();
5596     if (TA == nullptr)
5597       return;
5598 
5599     ParsedTargetAttr Attr =
5600         CGM.getTarget().parseTargetAttr(TA->getFeaturesStr());
5601     if (Attr.BranchProtection.empty())
5602       return;
5603 
5604     TargetInfo::BranchProtectionInfo BPI;
5605     StringRef Error;
5606     (void)CGM.getTarget().validateBranchProtection(Attr.BranchProtection,
5607                                                    Attr.CPU, BPI, Error);
5608     assert(Error.empty());
5609 
5610     auto *Fn = cast<llvm::Function>(GV);
5611     static const char *SignReturnAddrStr[] = {"none", "non-leaf", "all"};
5612     Fn->addFnAttr("sign-return-address", SignReturnAddrStr[static_cast<int>(BPI.SignReturnAddr)]);
5613 
5614     if (BPI.SignReturnAddr != LangOptions::SignReturnAddressScopeKind::None) {
5615       Fn->addFnAttr("sign-return-address-key",
5616                     BPI.SignKey == LangOptions::SignReturnAddressKeyKind::AKey
5617                         ? "a_key"
5618                         : "b_key");
5619     }
5620 
5621     Fn->addFnAttr("branch-target-enforcement",
5622                   BPI.BranchTargetEnforcement ? "true" : "false");
5623   }
5624 
isScalarizableAsmOperand(CodeGen::CodeGenFunction & CGF,llvm::Type * Ty) const5625   bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
5626                                 llvm::Type *Ty) const override {
5627     if (CGF.getTarget().hasFeature("ls64")) {
5628       auto *ST = dyn_cast<llvm::StructType>(Ty);
5629       if (ST && ST->getNumElements() == 1) {
5630         auto *AT = dyn_cast<llvm::ArrayType>(ST->getElementType(0));
5631         if (AT && AT->getNumElements() == 8 &&
5632             AT->getElementType()->isIntegerTy(64))
5633           return true;
5634       }
5635     }
5636     return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);
5637   }
5638 };
5639 
5640 class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
5641 public:
WindowsAArch64TargetCodeGenInfo(CodeGenTypes & CGT,AArch64ABIInfo::ABIKind K)5642   WindowsAArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIInfo::ABIKind K)
5643       : AArch64TargetCodeGenInfo(CGT, K) {}
5644 
5645   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
5646                            CodeGen::CodeGenModule &CGM) const override;
5647 
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const5648   void getDependentLibraryOption(llvm::StringRef Lib,
5649                                  llvm::SmallString<24> &Opt) const override {
5650     Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
5651   }
5652 
getDetectMismatchOption(llvm::StringRef Name,llvm::StringRef Value,llvm::SmallString<32> & Opt) const5653   void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
5654                                llvm::SmallString<32> &Opt) const override {
5655     Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
5656   }
5657 };
5658 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const5659 void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
5660     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
5661   AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
5662   if (GV->isDeclaration())
5663     return;
5664   addStackProbeTargetAttributes(D, GV, CGM);
5665 }
5666 }
5667 
coerceIllegalVector(QualType Ty) const5668 ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty) const {
5669   assert(Ty->isVectorType() && "expected vector type!");
5670 
5671   const auto *VT = Ty->castAs<VectorType>();
5672   if (VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector) {
5673     assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
5674     assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
5675                BuiltinType::UChar &&
5676            "unexpected builtin type for SVE predicate!");
5677     return ABIArgInfo::getDirect(llvm::ScalableVectorType::get(
5678         llvm::Type::getInt1Ty(getVMContext()), 16));
5679   }
5680 
5681   if (VT->getVectorKind() == VectorType::SveFixedLengthDataVector) {
5682     assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
5683 
5684     const auto *BT = VT->getElementType()->castAs<BuiltinType>();
5685     llvm::ScalableVectorType *ResType = nullptr;
5686     switch (BT->getKind()) {
5687     default:
5688       llvm_unreachable("unexpected builtin type for SVE vector!");
5689     case BuiltinType::SChar:
5690     case BuiltinType::UChar:
5691       ResType = llvm::ScalableVectorType::get(
5692           llvm::Type::getInt8Ty(getVMContext()), 16);
5693       break;
5694     case BuiltinType::Short:
5695     case BuiltinType::UShort:
5696       ResType = llvm::ScalableVectorType::get(
5697           llvm::Type::getInt16Ty(getVMContext()), 8);
5698       break;
5699     case BuiltinType::Int:
5700     case BuiltinType::UInt:
5701       ResType = llvm::ScalableVectorType::get(
5702           llvm::Type::getInt32Ty(getVMContext()), 4);
5703       break;
5704     case BuiltinType::Long:
5705     case BuiltinType::ULong:
5706       ResType = llvm::ScalableVectorType::get(
5707           llvm::Type::getInt64Ty(getVMContext()), 2);
5708       break;
5709     case BuiltinType::Half:
5710       ResType = llvm::ScalableVectorType::get(
5711           llvm::Type::getHalfTy(getVMContext()), 8);
5712       break;
5713     case BuiltinType::Float:
5714       ResType = llvm::ScalableVectorType::get(
5715           llvm::Type::getFloatTy(getVMContext()), 4);
5716       break;
5717     case BuiltinType::Double:
5718       ResType = llvm::ScalableVectorType::get(
5719           llvm::Type::getDoubleTy(getVMContext()), 2);
5720       break;
5721     case BuiltinType::BFloat16:
5722       ResType = llvm::ScalableVectorType::get(
5723           llvm::Type::getBFloatTy(getVMContext()), 8);
5724       break;
5725     }
5726     return ABIArgInfo::getDirect(ResType);
5727   }
5728 
5729   uint64_t Size = getContext().getTypeSize(Ty);
5730   // Android promotes <2 x i8> to i16, not i32
5731   if (isAndroid() && (Size <= 16)) {
5732     llvm::Type *ResType = llvm::Type::getInt16Ty(getVMContext());
5733     return ABIArgInfo::getDirect(ResType);
5734   }
5735   if (Size <= 32) {
5736     llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext());
5737     return ABIArgInfo::getDirect(ResType);
5738   }
5739   if (Size == 64) {
5740     auto *ResType =
5741         llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2);
5742     return ABIArgInfo::getDirect(ResType);
5743   }
5744   if (Size == 128) {
5745     auto *ResType =
5746         llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4);
5747     return ABIArgInfo::getDirect(ResType);
5748   }
5749   return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5750 }
5751 
5752 ABIArgInfo
classifyArgumentType(QualType Ty,bool IsVariadic,unsigned CallingConvention) const5753 AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
5754                                      unsigned CallingConvention) const {
5755   Ty = useFirstFieldIfTransparentUnion(Ty);
5756 
5757   // Handle illegal vector types here.
5758   if (isIllegalVectorType(Ty))
5759     return coerceIllegalVector(Ty);
5760 
5761   if (!isAggregateTypeForABI(Ty)) {
5762     // Treat an enum type as its underlying type.
5763     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5764       Ty = EnumTy->getDecl()->getIntegerType();
5765 
5766     if (const auto *EIT = Ty->getAs<BitIntType>())
5767       if (EIT->getNumBits() > 128)
5768         return getNaturalAlignIndirect(Ty);
5769 
5770     return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
5771                 ? ABIArgInfo::getExtend(Ty)
5772                 : ABIArgInfo::getDirect());
5773   }
5774 
5775   // Structures with either a non-trivial destructor or a non-trivial
5776   // copy constructor are always indirect.
5777   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
5778     return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
5779                                      CGCXXABI::RAA_DirectInMemory);
5780   }
5781 
5782   // Empty records are always ignored on Darwin, but actually passed in C++ mode
5783   // elsewhere for GNU compatibility.
5784   uint64_t Size = getContext().getTypeSize(Ty);
5785   bool IsEmpty = isEmptyRecord(getContext(), Ty, true);
5786   if (IsEmpty || Size == 0) {
5787     if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
5788       return ABIArgInfo::getIgnore();
5789 
5790     // GNU C mode. The only argument that gets ignored is an empty one with size
5791     // 0.
5792     if (IsEmpty && Size == 0)
5793       return ABIArgInfo::getIgnore();
5794     return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
5795   }
5796 
5797   // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
5798   const Type *Base = nullptr;
5799   uint64_t Members = 0;
5800   bool IsWin64 = Kind == Win64 || CallingConvention == llvm::CallingConv::Win64;
5801   bool IsWinVariadic = IsWin64 && IsVariadic;
5802   // In variadic functions on Windows, all composite types are treated alike,
5803   // no special handling of HFAs/HVAs.
5804   if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
5805     if (Kind != AArch64ABIInfo::AAPCS)
5806       return ABIArgInfo::getDirect(
5807           llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members));
5808 
5809     // For alignment adjusted HFAs, cap the argument alignment to 16, leave it
5810     // default otherwise.
5811     unsigned Align =
5812         getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();
5813     unsigned BaseAlign = getContext().getTypeAlignInChars(Base).getQuantity();
5814     Align = (Align > BaseAlign && Align >= 16) ? 16 : 0;
5815     return ABIArgInfo::getDirect(
5816         llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members), 0,
5817         nullptr, true, Align);
5818   }
5819 
5820   // Aggregates <= 16 bytes are passed directly in registers or on the stack.
5821   if (Size <= 128) {
5822     // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
5823     // same size and alignment.
5824     if (getTarget().isRenderScriptTarget()) {
5825       return coerceToIntArray(Ty, getContext(), getVMContext());
5826     }
5827     unsigned Alignment;
5828     if (Kind == AArch64ABIInfo::AAPCS) {
5829       Alignment = getContext().getTypeUnadjustedAlign(Ty);
5830       Alignment = Alignment < 128 ? 64 : 128;
5831     } else {
5832       Alignment =
5833           std::max(getContext().getTypeAlign(Ty),
5834                    (unsigned)getTarget().getPointerWidth(LangAS::Default));
5835     }
5836     Size = llvm::alignTo(Size, Alignment);
5837 
5838     // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
5839     // For aggregates with 16-byte alignment, we use i128.
5840     llvm::Type *BaseTy = llvm::Type::getIntNTy(getVMContext(), Alignment);
5841     return ABIArgInfo::getDirect(
5842         Size == Alignment ? BaseTy
5843                           : llvm::ArrayType::get(BaseTy, Size / Alignment));
5844   }
5845 
5846   return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5847 }
5848 
classifyReturnType(QualType RetTy,bool IsVariadic) const5849 ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
5850                                               bool IsVariadic) const {
5851   if (RetTy->isVoidType())
5852     return ABIArgInfo::getIgnore();
5853 
5854   if (const auto *VT = RetTy->getAs<VectorType>()) {
5855     if (VT->getVectorKind() == VectorType::SveFixedLengthDataVector ||
5856         VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
5857       return coerceIllegalVector(RetTy);
5858   }
5859 
5860   // Large vector types should be returned via memory.
5861   if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128)
5862     return getNaturalAlignIndirect(RetTy);
5863 
5864   if (!isAggregateTypeForABI(RetTy)) {
5865     // Treat an enum type as its underlying type.
5866     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
5867       RetTy = EnumTy->getDecl()->getIntegerType();
5868 
5869     if (const auto *EIT = RetTy->getAs<BitIntType>())
5870       if (EIT->getNumBits() > 128)
5871         return getNaturalAlignIndirect(RetTy);
5872 
5873     return (isPromotableIntegerTypeForABI(RetTy) && isDarwinPCS()
5874                 ? ABIArgInfo::getExtend(RetTy)
5875                 : ABIArgInfo::getDirect());
5876   }
5877 
5878   uint64_t Size = getContext().getTypeSize(RetTy);
5879   if (isEmptyRecord(getContext(), RetTy, true) || Size == 0)
5880     return ABIArgInfo::getIgnore();
5881 
5882   const Type *Base = nullptr;
5883   uint64_t Members = 0;
5884   if (isHomogeneousAggregate(RetTy, Base, Members) &&
5885       !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
5886         IsVariadic))
5887     // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
5888     return ABIArgInfo::getDirect();
5889 
5890   // Aggregates <= 16 bytes are returned directly in registers or on the stack.
5891   if (Size <= 128) {
5892     // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
5893     // same size and alignment.
5894     if (getTarget().isRenderScriptTarget()) {
5895       return coerceToIntArray(RetTy, getContext(), getVMContext());
5896     }
5897 
5898     if (Size <= 64 && getDataLayout().isLittleEndian()) {
5899       // Composite types are returned in lower bits of a 64-bit register for LE,
5900       // and in higher bits for BE. However, integer types are always returned
5901       // in lower bits for both LE and BE, and they are not rounded up to
5902       // 64-bits. We can skip rounding up of composite types for LE, but not for
5903       // BE, otherwise composite types will be indistinguishable from integer
5904       // types.
5905       return ABIArgInfo::getDirect(
5906           llvm::IntegerType::get(getVMContext(), Size));
5907     }
5908 
5909     unsigned Alignment = getContext().getTypeAlign(RetTy);
5910     Size = llvm::alignTo(Size, 64); // round up to multiple of 8 bytes
5911 
5912     // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
5913     // For aggregates with 16-byte alignment, we use i128.
5914     if (Alignment < 128 && Size == 128) {
5915       llvm::Type *BaseTy = llvm::Type::getInt64Ty(getVMContext());
5916       return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64));
5917     }
5918     return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size));
5919   }
5920 
5921   return getNaturalAlignIndirect(RetTy);
5922 }
5923 
5924 /// isIllegalVectorType - check whether the vector type is legal for AArch64.
isIllegalVectorType(QualType Ty) const5925 bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
5926   if (const VectorType *VT = Ty->getAs<VectorType>()) {
5927     // Check whether VT is a fixed-length SVE vector. These types are
5928     // represented as scalable vectors in function args/return and must be
5929     // coerced from fixed vectors.
5930     if (VT->getVectorKind() == VectorType::SveFixedLengthDataVector ||
5931         VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
5932       return true;
5933 
5934     // Check whether VT is legal.
5935     unsigned NumElements = VT->getNumElements();
5936     uint64_t Size = getContext().getTypeSize(VT);
5937     // NumElements should be power of 2.
5938     if (!llvm::isPowerOf2_32(NumElements))
5939       return true;
5940 
5941     // arm64_32 has to be compatible with the ARM logic here, which allows huge
5942     // vectors for some reason.
5943     llvm::Triple Triple = getTarget().getTriple();
5944     if (Triple.getArch() == llvm::Triple::aarch64_32 &&
5945         Triple.isOSBinFormatMachO())
5946       return Size <= 32;
5947 
5948     return Size != 64 && (Size != 128 || NumElements == 1);
5949   }
5950   return false;
5951 }
5952 
isLegalVectorType(CharUnits VectorSize,llvm::Type * EltTy,unsigned NumElts) const5953 bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,
5954                                             llvm::Type *EltTy,
5955                                             unsigned NumElts) const {
5956   if (!llvm::isPowerOf2_32(NumElts))
5957     return false;
5958   if (VectorSize.getQuantity() != 8 &&
5959       (VectorSize.getQuantity() != 16 || NumElts == 1))
5960     return false;
5961   return true;
5962 }
5963 
isHomogeneousAggregateBaseType(QualType Ty) const5964 bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
5965   // Homogeneous aggregates for AAPCS64 must have base types of a floating
5966   // point type or a short-vector type. This is the same as the 32-bit ABI,
5967   // but with the difference that any floating-point type is allowed,
5968   // including __fp16.
5969   if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
5970     if (BT->isFloatingPoint())
5971       return true;
5972   } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
5973     unsigned VecSize = getContext().getTypeSize(VT);
5974     if (VecSize == 64 || VecSize == 128)
5975       return true;
5976   }
5977   return false;
5978 }
5979 
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const5980 bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
5981                                                        uint64_t Members) const {
5982   return Members <= 4;
5983 }
5984 
isZeroLengthBitfieldPermittedInHomogeneousAggregate() const5985 bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
5986     const {
5987   // AAPCS64 says that the rule for whether something is a homogeneous
5988   // aggregate is applied to the output of the data layout decision. So
5989   // anything that doesn't affect the data layout also does not affect
5990   // homogeneity. In particular, zero-length bitfields don't stop a struct
5991   // being homogeneous.
5992   return true;
5993 }
5994 
EmitAAPCSVAArg(Address VAListAddr,QualType Ty,CodeGenFunction & CGF) const5995 Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
5996                                        CodeGenFunction &CGF) const {
5997   ABIArgInfo AI = classifyArgumentType(Ty, /*IsVariadic=*/true,
5998                                        CGF.CurFnInfo->getCallingConvention());
5999   // Empty records are ignored for parameter passing purposes.
6000   if (AI.isIgnore()) {
6001     uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8;
6002     CharUnits SlotSize = CharUnits::fromQuantity(PointerSize);
6003     VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
6004     auto *Load = CGF.Builder.CreateLoad(VAListAddr);
6005     Address Addr = Address(Load, CGF.Int8Ty, SlotSize);
6006     return CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
6007   }
6008 
6009   bool IsIndirect = AI.isIndirect();
6010 
6011   llvm::Type *BaseTy = CGF.ConvertType(Ty);
6012   if (IsIndirect)
6013     BaseTy = llvm::PointerType::getUnqual(BaseTy);
6014   else if (AI.getCoerceToType())
6015     BaseTy = AI.getCoerceToType();
6016 
6017   unsigned NumRegs = 1;
6018   if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(BaseTy)) {
6019     BaseTy = ArrTy->getElementType();
6020     NumRegs = ArrTy->getNumElements();
6021   }
6022   bool IsFPR = BaseTy->isFloatingPointTy() || BaseTy->isVectorTy();
6023 
6024   // The AArch64 va_list type and handling is specified in the Procedure Call
6025   // Standard, section B.4:
6026   //
6027   // struct {
6028   //   void *__stack;
6029   //   void *__gr_top;
6030   //   void *__vr_top;
6031   //   int __gr_offs;
6032   //   int __vr_offs;
6033   // };
6034 
6035   llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg");
6036   llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
6037   llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");
6038   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
6039 
6040   CharUnits TySize = getContext().getTypeSizeInChars(Ty);
6041   CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(Ty);
6042 
6043   Address reg_offs_p = Address::invalid();
6044   llvm::Value *reg_offs = nullptr;
6045   int reg_top_index;
6046   int RegSize = IsIndirect ? 8 : TySize.getQuantity();
6047   if (!IsFPR) {
6048     // 3 is the field number of __gr_offs
6049     reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p");
6050     reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs");
6051     reg_top_index = 1; // field number for __gr_top
6052     RegSize = llvm::alignTo(RegSize, 8);
6053   } else {
6054     // 4 is the field number of __vr_offs.
6055     reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p");
6056     reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs");
6057     reg_top_index = 2; // field number for __vr_top
6058     RegSize = 16 * NumRegs;
6059   }
6060 
6061   //=======================================
6062   // Find out where argument was passed
6063   //=======================================
6064 
6065   // If reg_offs >= 0 we're already using the stack for this type of
6066   // argument. We don't want to keep updating reg_offs (in case it overflows,
6067   // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
6068   // whatever they get).
6069   llvm::Value *UsingStack = nullptr;
6070   UsingStack = CGF.Builder.CreateICmpSGE(
6071       reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, 0));
6072 
6073   CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, MaybeRegBlock);
6074 
6075   // Otherwise, at least some kind of argument could go in these registers, the
6076   // question is whether this particular type is too big.
6077   CGF.EmitBlock(MaybeRegBlock);
6078 
6079   // Integer arguments may need to correct register alignment (for example a
6080   // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
6081   // align __gr_offs to calculate the potential address.
6082   if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
6083     int Align = TyAlign.getQuantity();
6084 
6085     reg_offs = CGF.Builder.CreateAdd(
6086         reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, Align - 1),
6087         "align_regoffs");
6088     reg_offs = CGF.Builder.CreateAnd(
6089         reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, -Align),
6090         "aligned_regoffs");
6091   }
6092 
6093   // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
6094   // The fact that this is done unconditionally reflects the fact that
6095   // allocating an argument to the stack also uses up all the remaining
6096   // registers of the appropriate kind.
6097   llvm::Value *NewOffset = nullptr;
6098   NewOffset = CGF.Builder.CreateAdd(
6099       reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, RegSize), "new_reg_offs");
6100   CGF.Builder.CreateStore(NewOffset, reg_offs_p);
6101 
6102   // Now we're in a position to decide whether this argument really was in
6103   // registers or not.
6104   llvm::Value *InRegs = nullptr;
6105   InRegs = CGF.Builder.CreateICmpSLE(
6106       NewOffset, llvm::ConstantInt::get(CGF.Int32Ty, 0), "inreg");
6107 
6108   CGF.Builder.CreateCondBr(InRegs, InRegBlock, OnStackBlock);
6109 
6110   //=======================================
6111   // Argument was in registers
6112   //=======================================
6113 
6114   // Now we emit the code for if the argument was originally passed in
6115   // registers. First start the appropriate block:
6116   CGF.EmitBlock(InRegBlock);
6117 
6118   llvm::Value *reg_top = nullptr;
6119   Address reg_top_p =
6120       CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p");
6121   reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top");
6122   Address BaseAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, reg_top, reg_offs),
6123                    CGF.Int8Ty, CharUnits::fromQuantity(IsFPR ? 16 : 8));
6124   Address RegAddr = Address::invalid();
6125   llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty), *ElementTy = MemTy;
6126 
6127   if (IsIndirect) {
6128     // If it's been passed indirectly (actually a struct), whatever we find from
6129     // stored registers or on the stack will actually be a struct **.
6130     MemTy = llvm::PointerType::getUnqual(MemTy);
6131   }
6132 
6133   const Type *Base = nullptr;
6134   uint64_t NumMembers = 0;
6135   bool IsHFA = isHomogeneousAggregate(Ty, Base, NumMembers);
6136   if (IsHFA && NumMembers > 1) {
6137     // Homogeneous aggregates passed in registers will have their elements split
6138     // and stored 16-bytes apart regardless of size (they're notionally in qN,
6139     // qN+1, ...). We reload and store into a temporary local variable
6140     // contiguously.
6141     assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
6142     auto BaseTyInfo = getContext().getTypeInfoInChars(QualType(Base, 0));
6143     llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0));
6144     llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers);
6145     Address Tmp = CGF.CreateTempAlloca(HFATy,
6146                                        std::max(TyAlign, BaseTyInfo.Align));
6147 
6148     // On big-endian platforms, the value will be right-aligned in its slot.
6149     int Offset = 0;
6150     if (CGF.CGM.getDataLayout().isBigEndian() &&
6151         BaseTyInfo.Width.getQuantity() < 16)
6152       Offset = 16 - BaseTyInfo.Width.getQuantity();
6153 
6154     for (unsigned i = 0; i < NumMembers; ++i) {
6155       CharUnits BaseOffset = CharUnits::fromQuantity(16 * i + Offset);
6156       Address LoadAddr =
6157         CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, BaseOffset);
6158       LoadAddr = CGF.Builder.CreateElementBitCast(LoadAddr, BaseTy);
6159 
6160       Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Tmp, i);
6161 
6162       llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr);
6163       CGF.Builder.CreateStore(Elem, StoreAddr);
6164     }
6165 
6166     RegAddr = CGF.Builder.CreateElementBitCast(Tmp, MemTy);
6167   } else {
6168     // Otherwise the object is contiguous in memory.
6169 
6170     // It might be right-aligned in its slot.
6171     CharUnits SlotSize = BaseAddr.getAlignment();
6172     if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
6173         (IsHFA || !isAggregateTypeForABI(Ty)) &&
6174         TySize < SlotSize) {
6175       CharUnits Offset = SlotSize - TySize;
6176       BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, Offset);
6177     }
6178 
6179     RegAddr = CGF.Builder.CreateElementBitCast(BaseAddr, MemTy);
6180   }
6181 
6182   CGF.EmitBranch(ContBlock);
6183 
6184   //=======================================
6185   // Argument was on the stack
6186   //=======================================
6187   CGF.EmitBlock(OnStackBlock);
6188 
6189   Address stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "stack_p");
6190   llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(stack_p, "stack");
6191 
6192   // Again, stack arguments may need realignment. In this case both integer and
6193   // floating-point ones might be affected.
6194   if (!IsIndirect && TyAlign.getQuantity() > 8) {
6195     int Align = TyAlign.getQuantity();
6196 
6197     OnStackPtr = CGF.Builder.CreatePtrToInt(OnStackPtr, CGF.Int64Ty);
6198 
6199     OnStackPtr = CGF.Builder.CreateAdd(
6200         OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, Align - 1),
6201         "align_stack");
6202     OnStackPtr = CGF.Builder.CreateAnd(
6203         OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, -Align),
6204         "align_stack");
6205 
6206     OnStackPtr = CGF.Builder.CreateIntToPtr(OnStackPtr, CGF.Int8PtrTy);
6207   }
6208   Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
6209                                 std::max(CharUnits::fromQuantity(8), TyAlign));
6210 
6211   // All stack slots are multiples of 8 bytes.
6212   CharUnits StackSlotSize = CharUnits::fromQuantity(8);
6213   CharUnits StackSize;
6214   if (IsIndirect)
6215     StackSize = StackSlotSize;
6216   else
6217     StackSize = TySize.alignTo(StackSlotSize);
6218 
6219   llvm::Value *StackSizeC = CGF.Builder.getSize(StackSize);
6220   llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
6221       CGF.Int8Ty, OnStackPtr, StackSizeC, "new_stack");
6222 
6223   // Write the new value of __stack for the next call to va_arg
6224   CGF.Builder.CreateStore(NewStack, stack_p);
6225 
6226   if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(Ty) &&
6227       TySize < StackSlotSize) {
6228     CharUnits Offset = StackSlotSize - TySize;
6229     OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(OnStackAddr, Offset);
6230   }
6231 
6232   OnStackAddr = CGF.Builder.CreateElementBitCast(OnStackAddr, MemTy);
6233 
6234   CGF.EmitBranch(ContBlock);
6235 
6236   //=======================================
6237   // Tidy up
6238   //=======================================
6239   CGF.EmitBlock(ContBlock);
6240 
6241   Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, OnStackAddr,
6242                                  OnStackBlock, "vaargs.addr");
6243 
6244   if (IsIndirect)
6245     return Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), ElementTy,
6246                    TyAlign);
6247 
6248   return ResAddr;
6249 }
6250 
EmitDarwinVAArg(Address VAListAddr,QualType Ty,CodeGenFunction & CGF) const6251 Address AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
6252                                         CodeGenFunction &CGF) const {
6253   // The backend's lowering doesn't support va_arg for aggregates or
6254   // illegal vector types.  Lower VAArg here for these cases and use
6255   // the LLVM va_arg instruction for everything else.
6256   if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty))
6257     return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
6258 
6259   uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8;
6260   CharUnits SlotSize = CharUnits::fromQuantity(PointerSize);
6261 
6262   // Empty records are ignored for parameter passing purposes.
6263   if (isEmptyRecord(getContext(), Ty, true)) {
6264     Address Addr = Address(CGF.Builder.CreateLoad(VAListAddr, "ap.cur"),
6265                            getVAListElementType(CGF), SlotSize);
6266     Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
6267     return Addr;
6268   }
6269 
6270   // The size of the actual thing passed, which might end up just
6271   // being a pointer for indirect types.
6272   auto TyInfo = getContext().getTypeInfoInChars(Ty);
6273 
6274   // Arguments bigger than 16 bytes which aren't homogeneous
6275   // aggregates should be passed indirectly.
6276   bool IsIndirect = false;
6277   if (TyInfo.Width.getQuantity() > 16) {
6278     const Type *Base = nullptr;
6279     uint64_t Members = 0;
6280     IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
6281   }
6282 
6283   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
6284                           TyInfo, SlotSize, /*AllowHigherAlign*/ true);
6285 }
6286 
EmitMSVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const6287 Address AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
6288                                     QualType Ty) const {
6289   bool IsIndirect = false;
6290 
6291   // Composites larger than 16 bytes are passed by reference.
6292   if (isAggregateTypeForABI(Ty) && getContext().getTypeSize(Ty) > 128)
6293     IsIndirect = true;
6294 
6295   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
6296                           CGF.getContext().getTypeInfoInChars(Ty),
6297                           CharUnits::fromQuantity(8),
6298                           /*allowHigherAlign*/ false);
6299 }
6300 
6301 //===----------------------------------------------------------------------===//
6302 // ARM ABI Implementation
6303 //===----------------------------------------------------------------------===//
6304 
6305 namespace {
6306 
6307 class ARMABIInfo : public ABIInfo {
6308 public:
6309   enum ABIKind {
6310     APCS = 0,
6311     AAPCS = 1,
6312     AAPCS_VFP = 2,
6313     AAPCS16_VFP = 3,
6314   };
6315 
6316 private:
6317   ABIKind Kind;
6318   bool IsFloatABISoftFP;
6319 
6320 public:
ARMABIInfo(CodeGenTypes & CGT,ABIKind Kind)6321   ARMABIInfo(CodeGenTypes &CGT, ABIKind Kind) : ABIInfo(CGT), Kind(Kind) {
6322     setCCs();
6323     IsFloatABISoftFP = CGT.getCodeGenOpts().FloatABI == "softfp" ||
6324         CGT.getCodeGenOpts().FloatABI == ""; // default
6325   }
6326 
isEABI() const6327   bool isEABI() const {
6328     switch (getTarget().getTriple().getEnvironment()) {
6329     case llvm::Triple::Android:
6330     case llvm::Triple::EABI:
6331     case llvm::Triple::EABIHF:
6332     case llvm::Triple::GNUEABI:
6333     case llvm::Triple::GNUEABIHF:
6334     case llvm::Triple::MuslEABI:
6335     case llvm::Triple::MuslEABIHF:
6336       return true;
6337     default:
6338       return false;
6339     }
6340   }
6341 
isEABIHF() const6342   bool isEABIHF() const {
6343     switch (getTarget().getTriple().getEnvironment()) {
6344     case llvm::Triple::EABIHF:
6345     case llvm::Triple::GNUEABIHF:
6346     case llvm::Triple::MuslEABIHF:
6347       return true;
6348     default:
6349       return false;
6350     }
6351   }
6352 
getABIKind() const6353   ABIKind getABIKind() const { return Kind; }
6354 
allowBFloatArgsAndRet() const6355   bool allowBFloatArgsAndRet() const override {
6356     return !IsFloatABISoftFP && getTarget().hasBFloat16Type();
6357   }
6358 
6359 private:
6360   ABIArgInfo classifyReturnType(QualType RetTy, bool isVariadic,
6361                                 unsigned functionCallConv) const;
6362   ABIArgInfo classifyArgumentType(QualType RetTy, bool isVariadic,
6363                                   unsigned functionCallConv) const;
6364   ABIArgInfo classifyHomogeneousAggregate(QualType Ty, const Type *Base,
6365                                           uint64_t Members) const;
6366   ABIArgInfo coerceIllegalVector(QualType Ty) const;
6367   bool isIllegalVectorType(QualType Ty) const;
6368   bool containsAnyFP16Vectors(QualType Ty) const;
6369 
6370   bool isHomogeneousAggregateBaseType(QualType Ty) const override;
6371   bool isHomogeneousAggregateSmallEnough(const Type *Ty,
6372                                          uint64_t Members) const override;
6373   bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
6374 
6375   bool isEffectivelyAAPCS_VFP(unsigned callConvention, bool acceptHalf) const;
6376 
6377   void computeInfo(CGFunctionInfo &FI) const override;
6378 
6379   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
6380                     QualType Ty) const override;
6381 
6382   llvm::CallingConv::ID getLLVMDefaultCC() const;
6383   llvm::CallingConv::ID getABIDefaultCC() const;
6384   void setCCs();
6385 };
6386 
6387 class ARMSwiftABIInfo : public SwiftABIInfo {
6388 public:
ARMSwiftABIInfo(CodeGenTypes & CGT)6389   explicit ARMSwiftABIInfo(CodeGenTypes &CGT)
6390       : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
6391 
6392   bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
6393                          unsigned NumElts) const override;
6394 };
6395 
6396 class ARMTargetCodeGenInfo : public TargetCodeGenInfo {
6397 public:
ARMTargetCodeGenInfo(CodeGenTypes & CGT,ARMABIInfo::ABIKind K)6398   ARMTargetCodeGenInfo(CodeGenTypes &CGT, ARMABIInfo::ABIKind K)
6399       : TargetCodeGenInfo(std::make_unique<ARMABIInfo>(CGT, K)) {
6400     SwiftInfo = std::make_unique<ARMSwiftABIInfo>(CGT);
6401   }
6402 
getABIInfo() const6403   const ARMABIInfo &getABIInfo() const {
6404     return static_cast<const ARMABIInfo&>(TargetCodeGenInfo::getABIInfo());
6405   }
6406 
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const6407   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
6408     return 13;
6409   }
6410 
getARCRetainAutoreleasedReturnValueMarker() const6411   StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
6412     return "mov\tr7, r7\t\t// marker for objc_retainAutoreleaseReturnValue";
6413   }
6414 
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const6415   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
6416                                llvm::Value *Address) const override {
6417     llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
6418 
6419     // 0-15 are the 16 integer registers.
6420     AssignToArrayRange(CGF.Builder, Address, Four8, 0, 15);
6421     return false;
6422   }
6423 
getSizeOfUnwindException() const6424   unsigned getSizeOfUnwindException() const override {
6425     if (getABIInfo().isEABI()) return 88;
6426     return TargetCodeGenInfo::getSizeOfUnwindException();
6427   }
6428 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const6429   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
6430                            CodeGen::CodeGenModule &CGM) const override {
6431     if (GV->isDeclaration())
6432       return;
6433     const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
6434     if (!FD)
6435       return;
6436     auto *Fn = cast<llvm::Function>(GV);
6437 
6438     if (const auto *TA = FD->getAttr<TargetAttr>()) {
6439       ParsedTargetAttr Attr =
6440           CGM.getTarget().parseTargetAttr(TA->getFeaturesStr());
6441       if (!Attr.BranchProtection.empty()) {
6442         TargetInfo::BranchProtectionInfo BPI;
6443         StringRef DiagMsg;
6444         StringRef Arch =
6445             Attr.CPU.empty() ? CGM.getTarget().getTargetOpts().CPU : Attr.CPU;
6446         if (!CGM.getTarget().validateBranchProtection(Attr.BranchProtection,
6447                                                       Arch, BPI, DiagMsg)) {
6448           CGM.getDiags().Report(
6449               D->getLocation(),
6450               diag::warn_target_unsupported_branch_protection_attribute)
6451               << Arch;
6452         } else {
6453           static const char *SignReturnAddrStr[] = {"none", "non-leaf", "all"};
6454           assert(static_cast<unsigned>(BPI.SignReturnAddr) <= 2 &&
6455                  "Unexpected SignReturnAddressScopeKind");
6456           Fn->addFnAttr(
6457               "sign-return-address",
6458               SignReturnAddrStr[static_cast<int>(BPI.SignReturnAddr)]);
6459 
6460           Fn->addFnAttr("branch-target-enforcement",
6461                         BPI.BranchTargetEnforcement ? "true" : "false");
6462         }
6463       } else if (CGM.getLangOpts().BranchTargetEnforcement ||
6464                  CGM.getLangOpts().hasSignReturnAddress()) {
6465         // If the Branch Protection attribute is missing, validate the target
6466         // Architecture attribute against Branch Protection command line
6467         // settings.
6468         if (!CGM.getTarget().isBranchProtectionSupportedArch(Attr.CPU))
6469           CGM.getDiags().Report(
6470               D->getLocation(),
6471               diag::warn_target_unsupported_branch_protection_attribute)
6472               << Attr.CPU;
6473       }
6474     }
6475 
6476     const ARMInterruptAttr *Attr = FD->getAttr<ARMInterruptAttr>();
6477     if (!Attr)
6478       return;
6479 
6480     const char *Kind;
6481     switch (Attr->getInterrupt()) {
6482     case ARMInterruptAttr::Generic: Kind = ""; break;
6483     case ARMInterruptAttr::IRQ:     Kind = "IRQ"; break;
6484     case ARMInterruptAttr::FIQ:     Kind = "FIQ"; break;
6485     case ARMInterruptAttr::SWI:     Kind = "SWI"; break;
6486     case ARMInterruptAttr::ABORT:   Kind = "ABORT"; break;
6487     case ARMInterruptAttr::UNDEF:   Kind = "UNDEF"; break;
6488     }
6489 
6490     Fn->addFnAttr("interrupt", Kind);
6491 
6492     ARMABIInfo::ABIKind ABI = cast<ARMABIInfo>(getABIInfo()).getABIKind();
6493     if (ABI == ARMABIInfo::APCS)
6494       return;
6495 
6496     // AAPCS guarantees that sp will be 8-byte aligned on any public interface,
6497     // however this is not necessarily true on taking any interrupt. Instruct
6498     // the backend to perform a realignment as part of the function prologue.
6499     llvm::AttrBuilder B(Fn->getContext());
6500     B.addStackAlignmentAttr(8);
6501     Fn->addFnAttrs(B);
6502   }
6503 };
6504 
6505 class WindowsARMTargetCodeGenInfo : public ARMTargetCodeGenInfo {
6506 public:
WindowsARMTargetCodeGenInfo(CodeGenTypes & CGT,ARMABIInfo::ABIKind K)6507   WindowsARMTargetCodeGenInfo(CodeGenTypes &CGT, ARMABIInfo::ABIKind K)
6508       : ARMTargetCodeGenInfo(CGT, K) {}
6509 
6510   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
6511                            CodeGen::CodeGenModule &CGM) const override;
6512 
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const6513   void getDependentLibraryOption(llvm::StringRef Lib,
6514                                  llvm::SmallString<24> &Opt) const override {
6515     Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
6516   }
6517 
getDetectMismatchOption(llvm::StringRef Name,llvm::StringRef Value,llvm::SmallString<32> & Opt) const6518   void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
6519                                llvm::SmallString<32> &Opt) const override {
6520     Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
6521   }
6522 };
6523 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const6524 void WindowsARMTargetCodeGenInfo::setTargetAttributes(
6525     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
6526   ARMTargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
6527   if (GV->isDeclaration())
6528     return;
6529   addStackProbeTargetAttributes(D, GV, CGM);
6530 }
6531 }
6532 
computeInfo(CGFunctionInfo & FI) const6533 void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const {
6534   if (!::classifyReturnType(getCXXABI(), FI, *this))
6535     FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), FI.isVariadic(),
6536                                             FI.getCallingConvention());
6537 
6538   for (auto &I : FI.arguments())
6539     I.info = classifyArgumentType(I.type, FI.isVariadic(),
6540                                   FI.getCallingConvention());
6541 
6542 
6543   // Always honor user-specified calling convention.
6544   if (FI.getCallingConvention() != llvm::CallingConv::C)
6545     return;
6546 
6547   llvm::CallingConv::ID cc = getRuntimeCC();
6548   if (cc != llvm::CallingConv::C)
6549     FI.setEffectiveCallingConvention(cc);
6550 }
6551 
6552 /// Return the default calling convention that LLVM will use.
getLLVMDefaultCC() const6553 llvm::CallingConv::ID ARMABIInfo::getLLVMDefaultCC() const {
6554   // The default calling convention that LLVM will infer.
6555   if (isEABIHF() || getTarget().getTriple().isWatchABI())
6556     return llvm::CallingConv::ARM_AAPCS_VFP;
6557   else if (isEABI())
6558     return llvm::CallingConv::ARM_AAPCS;
6559   else
6560     return llvm::CallingConv::ARM_APCS;
6561 }
6562 
6563 /// Return the calling convention that our ABI would like us to use
6564 /// as the C calling convention.
getABIDefaultCC() const6565 llvm::CallingConv::ID ARMABIInfo::getABIDefaultCC() const {
6566   switch (getABIKind()) {
6567   case APCS: return llvm::CallingConv::ARM_APCS;
6568   case AAPCS: return llvm::CallingConv::ARM_AAPCS;
6569   case AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
6570   case AAPCS16_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
6571   }
6572   llvm_unreachable("bad ABI kind");
6573 }
6574 
setCCs()6575 void ARMABIInfo::setCCs() {
6576   assert(getRuntimeCC() == llvm::CallingConv::C);
6577 
6578   // Don't muddy up the IR with a ton of explicit annotations if
6579   // they'd just match what LLVM will infer from the triple.
6580   llvm::CallingConv::ID abiCC = getABIDefaultCC();
6581   if (abiCC != getLLVMDefaultCC())
6582     RuntimeCC = abiCC;
6583 }
6584 
coerceIllegalVector(QualType Ty) const6585 ABIArgInfo ARMABIInfo::coerceIllegalVector(QualType Ty) const {
6586   uint64_t Size = getContext().getTypeSize(Ty);
6587   if (Size <= 32) {
6588     llvm::Type *ResType =
6589         llvm::Type::getInt32Ty(getVMContext());
6590     return ABIArgInfo::getDirect(ResType);
6591   }
6592   if (Size == 64 || Size == 128) {
6593     auto *ResType = llvm::FixedVectorType::get(
6594         llvm::Type::getInt32Ty(getVMContext()), Size / 32);
6595     return ABIArgInfo::getDirect(ResType);
6596   }
6597   return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
6598 }
6599 
classifyHomogeneousAggregate(QualType Ty,const Type * Base,uint64_t Members) const6600 ABIArgInfo ARMABIInfo::classifyHomogeneousAggregate(QualType Ty,
6601                                                     const Type *Base,
6602                                                     uint64_t Members) const {
6603   assert(Base && "Base class should be set for homogeneous aggregate");
6604   // Base can be a floating-point or a vector.
6605   if (const VectorType *VT = Base->getAs<VectorType>()) {
6606     // FP16 vectors should be converted to integer vectors
6607     if (!getTarget().hasLegalHalfType() && containsAnyFP16Vectors(Ty)) {
6608       uint64_t Size = getContext().getTypeSize(VT);
6609       auto *NewVecTy = llvm::FixedVectorType::get(
6610           llvm::Type::getInt32Ty(getVMContext()), Size / 32);
6611       llvm::Type *Ty = llvm::ArrayType::get(NewVecTy, Members);
6612       return ABIArgInfo::getDirect(Ty, 0, nullptr, false);
6613     }
6614   }
6615   unsigned Align = 0;
6616   if (getABIKind() == ARMABIInfo::AAPCS ||
6617       getABIKind() == ARMABIInfo::AAPCS_VFP) {
6618     // For alignment adjusted HFAs, cap the argument alignment to 8, leave it
6619     // default otherwise.
6620     Align = getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();
6621     unsigned BaseAlign = getContext().getTypeAlignInChars(Base).getQuantity();
6622     Align = (Align > BaseAlign && Align >= 8) ? 8 : 0;
6623   }
6624   return ABIArgInfo::getDirect(nullptr, 0, nullptr, false, Align);
6625 }
6626 
classifyArgumentType(QualType Ty,bool isVariadic,unsigned functionCallConv) const6627 ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic,
6628                                             unsigned functionCallConv) const {
6629   // 6.1.2.1 The following argument types are VFP CPRCs:
6630   //   A single-precision floating-point type (including promoted
6631   //   half-precision types); A double-precision floating-point type;
6632   //   A 64-bit or 128-bit containerized vector type; Homogeneous Aggregate
6633   //   with a Base Type of a single- or double-precision floating-point type,
6634   //   64-bit containerized vectors or 128-bit containerized vectors with one
6635   //   to four Elements.
6636   // Variadic functions should always marshal to the base standard.
6637   bool IsAAPCS_VFP =
6638       !isVariadic && isEffectivelyAAPCS_VFP(functionCallConv, /* AAPCS16 */ false);
6639 
6640   Ty = useFirstFieldIfTransparentUnion(Ty);
6641 
6642   // Handle illegal vector types here.
6643   if (isIllegalVectorType(Ty))
6644     return coerceIllegalVector(Ty);
6645 
6646   if (!isAggregateTypeForABI(Ty)) {
6647     // Treat an enum type as its underlying type.
6648     if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
6649       Ty = EnumTy->getDecl()->getIntegerType();
6650     }
6651 
6652     if (const auto *EIT = Ty->getAs<BitIntType>())
6653       if (EIT->getNumBits() > 64)
6654         return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
6655 
6656     return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
6657                                               : ABIArgInfo::getDirect());
6658   }
6659 
6660   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
6661     return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
6662   }
6663 
6664   // Ignore empty records.
6665   if (isEmptyRecord(getContext(), Ty, true))
6666     return ABIArgInfo::getIgnore();
6667 
6668   if (IsAAPCS_VFP) {
6669     // Homogeneous Aggregates need to be expanded when we can fit the aggregate
6670     // into VFP registers.
6671     const Type *Base = nullptr;
6672     uint64_t Members = 0;
6673     if (isHomogeneousAggregate(Ty, Base, Members))
6674       return classifyHomogeneousAggregate(Ty, Base, Members);
6675   } else if (getABIKind() == ARMABIInfo::AAPCS16_VFP) {
6676     // WatchOS does have homogeneous aggregates. Note that we intentionally use
6677     // this convention even for a variadic function: the backend will use GPRs
6678     // if needed.
6679     const Type *Base = nullptr;
6680     uint64_t Members = 0;
6681     if (isHomogeneousAggregate(Ty, Base, Members)) {
6682       assert(Base && Members <= 4 && "unexpected homogeneous aggregate");
6683       llvm::Type *Ty =
6684         llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members);
6685       return ABIArgInfo::getDirect(Ty, 0, nullptr, false);
6686     }
6687   }
6688 
6689   if (getABIKind() == ARMABIInfo::AAPCS16_VFP &&
6690       getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(16)) {
6691     // WatchOS is adopting the 64-bit AAPCS rule on composite types: if they're
6692     // bigger than 128-bits, they get placed in space allocated by the caller,
6693     // and a pointer is passed.
6694     return ABIArgInfo::getIndirect(
6695         CharUnits::fromQuantity(getContext().getTypeAlign(Ty) / 8), false);
6696   }
6697 
6698   // Support byval for ARM.
6699   // The ABI alignment for APCS is 4-byte and for AAPCS at least 4-byte and at
6700   // most 8-byte. We realign the indirect argument if type alignment is bigger
6701   // than ABI alignment.
6702   uint64_t ABIAlign = 4;
6703   uint64_t TyAlign;
6704   if (getABIKind() == ARMABIInfo::AAPCS_VFP ||
6705       getABIKind() == ARMABIInfo::AAPCS) {
6706     TyAlign = getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();
6707     ABIAlign = std::clamp(TyAlign, (uint64_t)4, (uint64_t)8);
6708   } else {
6709     TyAlign = getContext().getTypeAlignInChars(Ty).getQuantity();
6710   }
6711   if (getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(64)) {
6712     assert(getABIKind() != ARMABIInfo::AAPCS16_VFP && "unexpected byval");
6713     return ABIArgInfo::getIndirect(CharUnits::fromQuantity(ABIAlign),
6714                                    /*ByVal=*/true,
6715                                    /*Realign=*/TyAlign > ABIAlign);
6716   }
6717 
6718   // On RenderScript, coerce Aggregates <= 64 bytes to an integer array of
6719   // same size and alignment.
6720   if (getTarget().isRenderScriptTarget()) {
6721     return coerceToIntArray(Ty, getContext(), getVMContext());
6722   }
6723 
6724   // Otherwise, pass by coercing to a structure of the appropriate size.
6725   llvm::Type* ElemTy;
6726   unsigned SizeRegs;
6727   // FIXME: Try to match the types of the arguments more accurately where
6728   // we can.
6729   if (TyAlign <= 4) {
6730     ElemTy = llvm::Type::getInt32Ty(getVMContext());
6731     SizeRegs = (getContext().getTypeSize(Ty) + 31) / 32;
6732   } else {
6733     ElemTy = llvm::Type::getInt64Ty(getVMContext());
6734     SizeRegs = (getContext().getTypeSize(Ty) + 63) / 64;
6735   }
6736 
6737   return ABIArgInfo::getDirect(llvm::ArrayType::get(ElemTy, SizeRegs));
6738 }
6739 
isIntegerLikeType(QualType Ty,ASTContext & Context,llvm::LLVMContext & VMContext)6740 static bool isIntegerLikeType(QualType Ty, ASTContext &Context,
6741                               llvm::LLVMContext &VMContext) {
6742   // APCS, C Language Calling Conventions, Non-Simple Return Values: A structure
6743   // is called integer-like if its size is less than or equal to one word, and
6744   // the offset of each of its addressable sub-fields is zero.
6745 
6746   uint64_t Size = Context.getTypeSize(Ty);
6747 
6748   // Check that the type fits in a word.
6749   if (Size > 32)
6750     return false;
6751 
6752   // FIXME: Handle vector types!
6753   if (Ty->isVectorType())
6754     return false;
6755 
6756   // Float types are never treated as "integer like".
6757   if (Ty->isRealFloatingType())
6758     return false;
6759 
6760   // If this is a builtin or pointer type then it is ok.
6761   if (Ty->getAs<BuiltinType>() || Ty->isPointerType())
6762     return true;
6763 
6764   // Small complex integer types are "integer like".
6765   if (const ComplexType *CT = Ty->getAs<ComplexType>())
6766     return isIntegerLikeType(CT->getElementType(), Context, VMContext);
6767 
6768   // Single element and zero sized arrays should be allowed, by the definition
6769   // above, but they are not.
6770 
6771   // Otherwise, it must be a record type.
6772   const RecordType *RT = Ty->getAs<RecordType>();
6773   if (!RT) return false;
6774 
6775   // Ignore records with flexible arrays.
6776   const RecordDecl *RD = RT->getDecl();
6777   if (RD->hasFlexibleArrayMember())
6778     return false;
6779 
6780   // Check that all sub-fields are at offset 0, and are themselves "integer
6781   // like".
6782   const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
6783 
6784   bool HadField = false;
6785   unsigned idx = 0;
6786   for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
6787        i != e; ++i, ++idx) {
6788     const FieldDecl *FD = *i;
6789 
6790     // Bit-fields are not addressable, we only need to verify they are "integer
6791     // like". We still have to disallow a subsequent non-bitfield, for example:
6792     //   struct { int : 0; int x }
6793     // is non-integer like according to gcc.
6794     if (FD->isBitField()) {
6795       if (!RD->isUnion())
6796         HadField = true;
6797 
6798       if (!isIntegerLikeType(FD->getType(), Context, VMContext))
6799         return false;
6800 
6801       continue;
6802     }
6803 
6804     // Check if this field is at offset 0.
6805     if (Layout.getFieldOffset(idx) != 0)
6806       return false;
6807 
6808     if (!isIntegerLikeType(FD->getType(), Context, VMContext))
6809       return false;
6810 
6811     // Only allow at most one field in a structure. This doesn't match the
6812     // wording above, but follows gcc in situations with a field following an
6813     // empty structure.
6814     if (!RD->isUnion()) {
6815       if (HadField)
6816         return false;
6817 
6818       HadField = true;
6819     }
6820   }
6821 
6822   return true;
6823 }
6824 
classifyReturnType(QualType RetTy,bool isVariadic,unsigned functionCallConv) const6825 ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic,
6826                                           unsigned functionCallConv) const {
6827 
6828   // Variadic functions should always marshal to the base standard.
6829   bool IsAAPCS_VFP =
6830       !isVariadic && isEffectivelyAAPCS_VFP(functionCallConv, /* AAPCS16 */ true);
6831 
6832   if (RetTy->isVoidType())
6833     return ABIArgInfo::getIgnore();
6834 
6835   if (const VectorType *VT = RetTy->getAs<VectorType>()) {
6836     // Large vector types should be returned via memory.
6837     if (getContext().getTypeSize(RetTy) > 128)
6838       return getNaturalAlignIndirect(RetTy);
6839     // TODO: FP16/BF16 vectors should be converted to integer vectors
6840     // This check is similar  to isIllegalVectorType - refactor?
6841     if ((!getTarget().hasLegalHalfType() &&
6842         (VT->getElementType()->isFloat16Type() ||
6843          VT->getElementType()->isHalfType())) ||
6844         (IsFloatABISoftFP &&
6845          VT->getElementType()->isBFloat16Type()))
6846       return coerceIllegalVector(RetTy);
6847   }
6848 
6849   if (!isAggregateTypeForABI(RetTy)) {
6850     // Treat an enum type as its underlying type.
6851     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
6852       RetTy = EnumTy->getDecl()->getIntegerType();
6853 
6854     if (const auto *EIT = RetTy->getAs<BitIntType>())
6855       if (EIT->getNumBits() > 64)
6856         return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
6857 
6858     return isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
6859                                                 : ABIArgInfo::getDirect();
6860   }
6861 
6862   // Are we following APCS?
6863   if (getABIKind() == APCS) {
6864     if (isEmptyRecord(getContext(), RetTy, false))
6865       return ABIArgInfo::getIgnore();
6866 
6867     // Complex types are all returned as packed integers.
6868     //
6869     // FIXME: Consider using 2 x vector types if the back end handles them
6870     // correctly.
6871     if (RetTy->isAnyComplexType())
6872       return ABIArgInfo::getDirect(llvm::IntegerType::get(
6873           getVMContext(), getContext().getTypeSize(RetTy)));
6874 
6875     // Integer like structures are returned in r0.
6876     if (isIntegerLikeType(RetTy, getContext(), getVMContext())) {
6877       // Return in the smallest viable integer type.
6878       uint64_t Size = getContext().getTypeSize(RetTy);
6879       if (Size <= 8)
6880         return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
6881       if (Size <= 16)
6882         return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
6883       return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
6884     }
6885 
6886     // Otherwise return in memory.
6887     return getNaturalAlignIndirect(RetTy);
6888   }
6889 
6890   // Otherwise this is an AAPCS variant.
6891 
6892   if (isEmptyRecord(getContext(), RetTy, true))
6893     return ABIArgInfo::getIgnore();
6894 
6895   // Check for homogeneous aggregates with AAPCS-VFP.
6896   if (IsAAPCS_VFP) {
6897     const Type *Base = nullptr;
6898     uint64_t Members = 0;
6899     if (isHomogeneousAggregate(RetTy, Base, Members))
6900       return classifyHomogeneousAggregate(RetTy, Base, Members);
6901   }
6902 
6903   // Aggregates <= 4 bytes are returned in r0; other aggregates
6904   // are returned indirectly.
6905   uint64_t Size = getContext().getTypeSize(RetTy);
6906   if (Size <= 32) {
6907     // On RenderScript, coerce Aggregates <= 4 bytes to an integer array of
6908     // same size and alignment.
6909     if (getTarget().isRenderScriptTarget()) {
6910       return coerceToIntArray(RetTy, getContext(), getVMContext());
6911     }
6912     if (getDataLayout().isBigEndian())
6913       // Return in 32 bit integer integer type (as if loaded by LDR, AAPCS 5.4)
6914       return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
6915 
6916     // Return in the smallest viable integer type.
6917     if (Size <= 8)
6918       return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
6919     if (Size <= 16)
6920       return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
6921     return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
6922   } else if (Size <= 128 && getABIKind() == AAPCS16_VFP) {
6923     llvm::Type *Int32Ty = llvm::Type::getInt32Ty(getVMContext());
6924     llvm::Type *CoerceTy =
6925         llvm::ArrayType::get(Int32Ty, llvm::alignTo(Size, 32) / 32);
6926     return ABIArgInfo::getDirect(CoerceTy);
6927   }
6928 
6929   return getNaturalAlignIndirect(RetTy);
6930 }
6931 
6932 /// isIllegalVector - check whether Ty is an illegal vector type.
isIllegalVectorType(QualType Ty) const6933 bool ARMABIInfo::isIllegalVectorType(QualType Ty) const {
6934   if (const VectorType *VT = Ty->getAs<VectorType> ()) {
6935     // On targets that don't support half, fp16 or bfloat, they are expanded
6936     // into float, and we don't want the ABI to depend on whether or not they
6937     // are supported in hardware. Thus return false to coerce vectors of these
6938     // types into integer vectors.
6939     // We do not depend on hasLegalHalfType for bfloat as it is a
6940     // separate IR type.
6941     if ((!getTarget().hasLegalHalfType() &&
6942         (VT->getElementType()->isFloat16Type() ||
6943          VT->getElementType()->isHalfType())) ||
6944         (IsFloatABISoftFP &&
6945          VT->getElementType()->isBFloat16Type()))
6946       return true;
6947     if (isAndroid()) {
6948       // Android shipped using Clang 3.1, which supported a slightly different
6949       // vector ABI. The primary differences were that 3-element vector types
6950       // were legal, and so were sub 32-bit vectors (i.e. <2 x i8>). This path
6951       // accepts that legacy behavior for Android only.
6952       // Check whether VT is legal.
6953       unsigned NumElements = VT->getNumElements();
6954       // NumElements should be power of 2 or equal to 3.
6955       if (!llvm::isPowerOf2_32(NumElements) && NumElements != 3)
6956         return true;
6957     } else {
6958       // Check whether VT is legal.
6959       unsigned NumElements = VT->getNumElements();
6960       uint64_t Size = getContext().getTypeSize(VT);
6961       // NumElements should be power of 2.
6962       if (!llvm::isPowerOf2_32(NumElements))
6963         return true;
6964       // Size should be greater than 32 bits.
6965       return Size <= 32;
6966     }
6967   }
6968   return false;
6969 }
6970 
6971 /// Return true if a type contains any 16-bit floating point vectors
containsAnyFP16Vectors(QualType Ty) const6972 bool ARMABIInfo::containsAnyFP16Vectors(QualType Ty) const {
6973   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
6974     uint64_t NElements = AT->getSize().getZExtValue();
6975     if (NElements == 0)
6976       return false;
6977     return containsAnyFP16Vectors(AT->getElementType());
6978   } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
6979     const RecordDecl *RD = RT->getDecl();
6980 
6981     // If this is a C++ record, check the bases first.
6982     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
6983       if (llvm::any_of(CXXRD->bases(), [this](const CXXBaseSpecifier &B) {
6984             return containsAnyFP16Vectors(B.getType());
6985           }))
6986         return true;
6987 
6988     if (llvm::any_of(RD->fields(), [this](FieldDecl *FD) {
6989           return FD && containsAnyFP16Vectors(FD->getType());
6990         }))
6991       return true;
6992 
6993     return false;
6994   } else {
6995     if (const VectorType *VT = Ty->getAs<VectorType>())
6996       return (VT->getElementType()->isFloat16Type() ||
6997               VT->getElementType()->isBFloat16Type() ||
6998               VT->getElementType()->isHalfType());
6999     return false;
7000   }
7001 }
7002 
isLegalVectorType(CharUnits VectorSize,llvm::Type * EltTy,unsigned NumElts) const7003 bool ARMSwiftABIInfo::isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
7004                                         unsigned NumElts) const {
7005   if (!llvm::isPowerOf2_32(NumElts))
7006     return false;
7007   unsigned size = CGT.getDataLayout().getTypeStoreSizeInBits(EltTy);
7008   if (size > 64)
7009     return false;
7010   if (VectorSize.getQuantity() != 8 &&
7011       (VectorSize.getQuantity() != 16 || NumElts == 1))
7012     return false;
7013   return true;
7014 }
7015 
isHomogeneousAggregateBaseType(QualType Ty) const7016 bool ARMABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
7017   // Homogeneous aggregates for AAPCS-VFP must have base types of float,
7018   // double, or 64-bit or 128-bit vectors.
7019   if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
7020     if (BT->getKind() == BuiltinType::Float ||
7021         BT->getKind() == BuiltinType::Double ||
7022         BT->getKind() == BuiltinType::LongDouble)
7023       return true;
7024   } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
7025     unsigned VecSize = getContext().getTypeSize(VT);
7026     if (VecSize == 64 || VecSize == 128)
7027       return true;
7028   }
7029   return false;
7030 }
7031 
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const7032 bool ARMABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
7033                                                    uint64_t Members) const {
7034   return Members <= 4;
7035 }
7036 
isZeroLengthBitfieldPermittedInHomogeneousAggregate() const7037 bool ARMABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() const {
7038   // AAPCS32 says that the rule for whether something is a homogeneous
7039   // aggregate is applied to the output of the data layout decision. So
7040   // anything that doesn't affect the data layout also does not affect
7041   // homogeneity. In particular, zero-length bitfields don't stop a struct
7042   // being homogeneous.
7043   return true;
7044 }
7045 
isEffectivelyAAPCS_VFP(unsigned callConvention,bool acceptHalf) const7046 bool ARMABIInfo::isEffectivelyAAPCS_VFP(unsigned callConvention,
7047                                         bool acceptHalf) const {
7048   // Give precedence to user-specified calling conventions.
7049   if (callConvention != llvm::CallingConv::C)
7050     return (callConvention == llvm::CallingConv::ARM_AAPCS_VFP);
7051   else
7052     return (getABIKind() == AAPCS_VFP) ||
7053            (acceptHalf && (getABIKind() == AAPCS16_VFP));
7054 }
7055 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const7056 Address ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7057                               QualType Ty) const {
7058   CharUnits SlotSize = CharUnits::fromQuantity(4);
7059 
7060   // Empty records are ignored for parameter passing purposes.
7061   if (isEmptyRecord(getContext(), Ty, true)) {
7062     VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
7063     auto *Load = CGF.Builder.CreateLoad(VAListAddr);
7064     Address Addr = Address(Load, CGF.Int8Ty, SlotSize);
7065     return CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
7066   }
7067 
7068   CharUnits TySize = getContext().getTypeSizeInChars(Ty);
7069   CharUnits TyAlignForABI = getContext().getTypeUnadjustedAlignInChars(Ty);
7070 
7071   // Use indirect if size of the illegal vector is bigger than 16 bytes.
7072   bool IsIndirect = false;
7073   const Type *Base = nullptr;
7074   uint64_t Members = 0;
7075   if (TySize > CharUnits::fromQuantity(16) && isIllegalVectorType(Ty)) {
7076     IsIndirect = true;
7077 
7078   // ARMv7k passes structs bigger than 16 bytes indirectly, in space
7079   // allocated by the caller.
7080   } else if (TySize > CharUnits::fromQuantity(16) &&
7081              getABIKind() == ARMABIInfo::AAPCS16_VFP &&
7082              !isHomogeneousAggregate(Ty, Base, Members)) {
7083     IsIndirect = true;
7084 
7085   // Otherwise, bound the type's ABI alignment.
7086   // The ABI alignment for 64-bit or 128-bit vectors is 8 for AAPCS and 4 for
7087   // APCS. For AAPCS, the ABI alignment is at least 4-byte and at most 8-byte.
7088   // Our callers should be prepared to handle an under-aligned address.
7089   } else if (getABIKind() == ARMABIInfo::AAPCS_VFP ||
7090              getABIKind() == ARMABIInfo::AAPCS) {
7091     TyAlignForABI = std::max(TyAlignForABI, CharUnits::fromQuantity(4));
7092     TyAlignForABI = std::min(TyAlignForABI, CharUnits::fromQuantity(8));
7093   } else if (getABIKind() == ARMABIInfo::AAPCS16_VFP) {
7094     // ARMv7k allows type alignment up to 16 bytes.
7095     TyAlignForABI = std::max(TyAlignForABI, CharUnits::fromQuantity(4));
7096     TyAlignForABI = std::min(TyAlignForABI, CharUnits::fromQuantity(16));
7097   } else {
7098     TyAlignForABI = CharUnits::fromQuantity(4);
7099   }
7100 
7101   TypeInfoChars TyInfo(TySize, TyAlignForABI, AlignRequirementKind::None);
7102   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo,
7103                           SlotSize, /*AllowHigherAlign*/ true);
7104 }
7105 
7106 //===----------------------------------------------------------------------===//
7107 // NVPTX ABI Implementation
7108 //===----------------------------------------------------------------------===//
7109 
7110 namespace {
7111 
7112 class NVPTXTargetCodeGenInfo;
7113 
7114 class NVPTXABIInfo : public ABIInfo {
7115   NVPTXTargetCodeGenInfo &CGInfo;
7116 
7117 public:
NVPTXABIInfo(CodeGenTypes & CGT,NVPTXTargetCodeGenInfo & Info)7118   NVPTXABIInfo(CodeGenTypes &CGT, NVPTXTargetCodeGenInfo &Info)
7119       : ABIInfo(CGT), CGInfo(Info) {}
7120 
7121   ABIArgInfo classifyReturnType(QualType RetTy) const;
7122   ABIArgInfo classifyArgumentType(QualType Ty) const;
7123 
7124   void computeInfo(CGFunctionInfo &FI) const override;
7125   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7126                     QualType Ty) const override;
7127   bool isUnsupportedType(QualType T) const;
7128   ABIArgInfo coerceToIntArrayWithLimit(QualType Ty, unsigned MaxSize) const;
7129 };
7130 
7131 class NVPTXTargetCodeGenInfo : public TargetCodeGenInfo {
7132 public:
NVPTXTargetCodeGenInfo(CodeGenTypes & CGT)7133   NVPTXTargetCodeGenInfo(CodeGenTypes &CGT)
7134       : TargetCodeGenInfo(std::make_unique<NVPTXABIInfo>(CGT, *this)) {}
7135 
7136   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
7137                            CodeGen::CodeGenModule &M) const override;
7138   bool shouldEmitStaticExternCAliases() const override;
7139 
getCUDADeviceBuiltinSurfaceDeviceType() const7140   llvm::Type *getCUDADeviceBuiltinSurfaceDeviceType() const override {
7141     // On the device side, surface reference is represented as an object handle
7142     // in 64-bit integer.
7143     return llvm::Type::getInt64Ty(getABIInfo().getVMContext());
7144   }
7145 
getCUDADeviceBuiltinTextureDeviceType() const7146   llvm::Type *getCUDADeviceBuiltinTextureDeviceType() const override {
7147     // On the device side, texture reference is represented as an object handle
7148     // in 64-bit integer.
7149     return llvm::Type::getInt64Ty(getABIInfo().getVMContext());
7150   }
7151 
emitCUDADeviceBuiltinSurfaceDeviceCopy(CodeGenFunction & CGF,LValue Dst,LValue Src) const7152   bool emitCUDADeviceBuiltinSurfaceDeviceCopy(CodeGenFunction &CGF, LValue Dst,
7153                                               LValue Src) const override {
7154     emitBuiltinSurfTexDeviceCopy(CGF, Dst, Src);
7155     return true;
7156   }
7157 
emitCUDADeviceBuiltinTextureDeviceCopy(CodeGenFunction & CGF,LValue Dst,LValue Src) const7158   bool emitCUDADeviceBuiltinTextureDeviceCopy(CodeGenFunction &CGF, LValue Dst,
7159                                               LValue Src) const override {
7160     emitBuiltinSurfTexDeviceCopy(CGF, Dst, Src);
7161     return true;
7162   }
7163 
7164 private:
7165   // Adds a NamedMDNode with GV, Name, and Operand as operands, and adds the
7166   // resulting MDNode to the nvvm.annotations MDNode.
7167   static void addNVVMMetadata(llvm::GlobalValue *GV, StringRef Name,
7168                               int Operand);
7169 
emitBuiltinSurfTexDeviceCopy(CodeGenFunction & CGF,LValue Dst,LValue Src)7170   static void emitBuiltinSurfTexDeviceCopy(CodeGenFunction &CGF, LValue Dst,
7171                                            LValue Src) {
7172     llvm::Value *Handle = nullptr;
7173     llvm::Constant *C =
7174         llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).getPointer());
7175     // Lookup `addrspacecast` through the constant pointer if any.
7176     if (auto *ASC = llvm::dyn_cast_or_null<llvm::AddrSpaceCastOperator>(C))
7177       C = llvm::cast<llvm::Constant>(ASC->getPointerOperand());
7178     if (auto *GV = llvm::dyn_cast_or_null<llvm::GlobalVariable>(C)) {
7179       // Load the handle from the specific global variable using
7180       // `nvvm.texsurf.handle.internal` intrinsic.
7181       Handle = CGF.EmitRuntimeCall(
7182           CGF.CGM.getIntrinsic(llvm::Intrinsic::nvvm_texsurf_handle_internal,
7183                                {GV->getType()}),
7184           {GV}, "texsurf_handle");
7185     } else
7186       Handle = CGF.EmitLoadOfScalar(Src, SourceLocation());
7187     CGF.EmitStoreOfScalar(Handle, Dst);
7188   }
7189 };
7190 
7191 /// Checks if the type is unsupported directly by the current target.
isUnsupportedType(QualType T) const7192 bool NVPTXABIInfo::isUnsupportedType(QualType T) const {
7193   ASTContext &Context = getContext();
7194   if (!Context.getTargetInfo().hasFloat16Type() && T->isFloat16Type())
7195     return true;
7196   if (!Context.getTargetInfo().hasFloat128Type() &&
7197       (T->isFloat128Type() ||
7198        (T->isRealFloatingType() && Context.getTypeSize(T) == 128)))
7199     return true;
7200   if (const auto *EIT = T->getAs<BitIntType>())
7201     return EIT->getNumBits() >
7202            (Context.getTargetInfo().hasInt128Type() ? 128U : 64U);
7203   if (!Context.getTargetInfo().hasInt128Type() && T->isIntegerType() &&
7204       Context.getTypeSize(T) > 64U)
7205     return true;
7206   if (const auto *AT = T->getAsArrayTypeUnsafe())
7207     return isUnsupportedType(AT->getElementType());
7208   const auto *RT = T->getAs<RecordType>();
7209   if (!RT)
7210     return false;
7211   const RecordDecl *RD = RT->getDecl();
7212 
7213   // If this is a C++ record, check the bases first.
7214   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
7215     for (const CXXBaseSpecifier &I : CXXRD->bases())
7216       if (isUnsupportedType(I.getType()))
7217         return true;
7218 
7219   for (const FieldDecl *I : RD->fields())
7220     if (isUnsupportedType(I->getType()))
7221       return true;
7222   return false;
7223 }
7224 
7225 /// Coerce the given type into an array with maximum allowed size of elements.
coerceToIntArrayWithLimit(QualType Ty,unsigned MaxSize) const7226 ABIArgInfo NVPTXABIInfo::coerceToIntArrayWithLimit(QualType Ty,
7227                                                    unsigned MaxSize) const {
7228   // Alignment and Size are measured in bits.
7229   const uint64_t Size = getContext().getTypeSize(Ty);
7230   const uint64_t Alignment = getContext().getTypeAlign(Ty);
7231   const unsigned Div = std::min<unsigned>(MaxSize, Alignment);
7232   llvm::Type *IntType = llvm::Type::getIntNTy(getVMContext(), Div);
7233   const uint64_t NumElements = (Size + Div - 1) / Div;
7234   return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
7235 }
7236 
classifyReturnType(QualType RetTy) const7237 ABIArgInfo NVPTXABIInfo::classifyReturnType(QualType RetTy) const {
7238   if (RetTy->isVoidType())
7239     return ABIArgInfo::getIgnore();
7240 
7241   if (getContext().getLangOpts().OpenMP &&
7242       getContext().getLangOpts().OpenMPIsDevice && isUnsupportedType(RetTy))
7243     return coerceToIntArrayWithLimit(RetTy, 64);
7244 
7245   // note: this is different from default ABI
7246   if (!RetTy->isScalarType())
7247     return ABIArgInfo::getDirect();
7248 
7249   // Treat an enum type as its underlying type.
7250   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
7251     RetTy = EnumTy->getDecl()->getIntegerType();
7252 
7253   return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
7254                                                : ABIArgInfo::getDirect());
7255 }
7256 
classifyArgumentType(QualType Ty) const7257 ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const {
7258   // Treat an enum type as its underlying type.
7259   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
7260     Ty = EnumTy->getDecl()->getIntegerType();
7261 
7262   // Return aggregates type as indirect by value
7263   if (isAggregateTypeForABI(Ty)) {
7264     // Under CUDA device compilation, tex/surf builtin types are replaced with
7265     // object types and passed directly.
7266     if (getContext().getLangOpts().CUDAIsDevice) {
7267       if (Ty->isCUDADeviceBuiltinSurfaceType())
7268         return ABIArgInfo::getDirect(
7269             CGInfo.getCUDADeviceBuiltinSurfaceDeviceType());
7270       if (Ty->isCUDADeviceBuiltinTextureType())
7271         return ABIArgInfo::getDirect(
7272             CGInfo.getCUDADeviceBuiltinTextureDeviceType());
7273     }
7274     return getNaturalAlignIndirect(Ty, /* byval */ true);
7275   }
7276 
7277   if (const auto *EIT = Ty->getAs<BitIntType>()) {
7278     if ((EIT->getNumBits() > 128) ||
7279         (!getContext().getTargetInfo().hasInt128Type() &&
7280          EIT->getNumBits() > 64))
7281       return getNaturalAlignIndirect(Ty, /* byval */ true);
7282   }
7283 
7284   return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
7285                                             : ABIArgInfo::getDirect());
7286 }
7287 
computeInfo(CGFunctionInfo & FI) const7288 void NVPTXABIInfo::computeInfo(CGFunctionInfo &FI) const {
7289   if (!getCXXABI().classifyReturnType(FI))
7290     FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
7291   for (auto &I : FI.arguments())
7292     I.info = classifyArgumentType(I.type);
7293 
7294   // Always honor user-specified calling convention.
7295   if (FI.getCallingConvention() != llvm::CallingConv::C)
7296     return;
7297 
7298   FI.setEffectiveCallingConvention(getRuntimeCC());
7299 }
7300 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const7301 Address NVPTXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7302                                 QualType Ty) const {
7303   llvm_unreachable("NVPTX does not support varargs");
7304 }
7305 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const7306 void NVPTXTargetCodeGenInfo::setTargetAttributes(
7307     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
7308   if (GV->isDeclaration())
7309     return;
7310   const VarDecl *VD = dyn_cast_or_null<VarDecl>(D);
7311   if (VD) {
7312     if (M.getLangOpts().CUDA) {
7313       if (VD->getType()->isCUDADeviceBuiltinSurfaceType())
7314         addNVVMMetadata(GV, "surface", 1);
7315       else if (VD->getType()->isCUDADeviceBuiltinTextureType())
7316         addNVVMMetadata(GV, "texture", 1);
7317       return;
7318     }
7319   }
7320 
7321   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
7322   if (!FD) return;
7323 
7324   llvm::Function *F = cast<llvm::Function>(GV);
7325 
7326   // Perform special handling in OpenCL mode
7327   if (M.getLangOpts().OpenCL) {
7328     // Use OpenCL function attributes to check for kernel functions
7329     // By default, all functions are device functions
7330     if (FD->hasAttr<OpenCLKernelAttr>()) {
7331       // OpenCL __kernel functions get kernel metadata
7332       // Create !{<func-ref>, metadata !"kernel", i32 1} node
7333       addNVVMMetadata(F, "kernel", 1);
7334       // And kernel functions are not subject to inlining
7335       F->addFnAttr(llvm::Attribute::NoInline);
7336     }
7337   }
7338 
7339   // Perform special handling in CUDA mode.
7340   if (M.getLangOpts().CUDA) {
7341     // CUDA __global__ functions get a kernel metadata entry.  Since
7342     // __global__ functions cannot be called from the device, we do not
7343     // need to set the noinline attribute.
7344     if (FD->hasAttr<CUDAGlobalAttr>()) {
7345       // Create !{<func-ref>, metadata !"kernel", i32 1} node
7346       addNVVMMetadata(F, "kernel", 1);
7347     }
7348     if (CUDALaunchBoundsAttr *Attr = FD->getAttr<CUDALaunchBoundsAttr>()) {
7349       // Create !{<func-ref>, metadata !"maxntidx", i32 <val>} node
7350       llvm::APSInt MaxThreads(32);
7351       MaxThreads = Attr->getMaxThreads()->EvaluateKnownConstInt(M.getContext());
7352       if (MaxThreads > 0)
7353         addNVVMMetadata(F, "maxntidx", MaxThreads.getExtValue());
7354 
7355       // min blocks is an optional argument for CUDALaunchBoundsAttr. If it was
7356       // not specified in __launch_bounds__ or if the user specified a 0 value,
7357       // we don't have to add a PTX directive.
7358       if (Attr->getMinBlocks()) {
7359         llvm::APSInt MinBlocks(32);
7360         MinBlocks = Attr->getMinBlocks()->EvaluateKnownConstInt(M.getContext());
7361         if (MinBlocks > 0)
7362           // Create !{<func-ref>, metadata !"minctasm", i32 <val>} node
7363           addNVVMMetadata(F, "minctasm", MinBlocks.getExtValue());
7364       }
7365     }
7366   }
7367 }
7368 
addNVVMMetadata(llvm::GlobalValue * GV,StringRef Name,int Operand)7369 void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::GlobalValue *GV,
7370                                              StringRef Name, int Operand) {
7371   llvm::Module *M = GV->getParent();
7372   llvm::LLVMContext &Ctx = M->getContext();
7373 
7374   // Get "nvvm.annotations" metadata node
7375   llvm::NamedMDNode *MD = M->getOrInsertNamedMetadata("nvvm.annotations");
7376 
7377   llvm::Metadata *MDVals[] = {
7378       llvm::ConstantAsMetadata::get(GV), llvm::MDString::get(Ctx, Name),
7379       llvm::ConstantAsMetadata::get(
7380           llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), Operand))};
7381   // Append metadata to nvvm.annotations
7382   MD->addOperand(llvm::MDNode::get(Ctx, MDVals));
7383 }
7384 
shouldEmitStaticExternCAliases() const7385 bool NVPTXTargetCodeGenInfo::shouldEmitStaticExternCAliases() const {
7386   return false;
7387 }
7388 }
7389 
7390 //===----------------------------------------------------------------------===//
7391 // SystemZ ABI Implementation
7392 //===----------------------------------------------------------------------===//
7393 
7394 namespace {
7395 
7396 class SystemZABIInfo : public ABIInfo {
7397   bool HasVector;
7398   bool IsSoftFloatABI;
7399 
7400 public:
SystemZABIInfo(CodeGenTypes & CGT,bool HV,bool SF)7401   SystemZABIInfo(CodeGenTypes &CGT, bool HV, bool SF)
7402       : ABIInfo(CGT), HasVector(HV), IsSoftFloatABI(SF) {}
7403 
7404   bool isPromotableIntegerTypeForABI(QualType Ty) const;
7405   bool isCompoundType(QualType Ty) const;
7406   bool isVectorArgumentType(QualType Ty) const;
7407   bool isFPArgumentType(QualType Ty) const;
7408   QualType GetSingleElementType(QualType Ty) const;
7409 
7410   ABIArgInfo classifyReturnType(QualType RetTy) const;
7411   ABIArgInfo classifyArgumentType(QualType ArgTy) const;
7412 
7413   void computeInfo(CGFunctionInfo &FI) const override;
7414   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7415                     QualType Ty) const override;
7416 };
7417 
7418 class SystemZTargetCodeGenInfo : public TargetCodeGenInfo {
7419   ASTContext &Ctx;
7420 
getABIInfo() const7421   const SystemZABIInfo &getABIInfo() const {
7422     return static_cast<const SystemZABIInfo&>(TargetCodeGenInfo::getABIInfo());
7423   }
7424 
7425   // These are used for speeding up the search for a visible vector ABI.
7426   mutable bool HasVisibleVecABIFlag = false;
7427   mutable std::set<const Type *> SeenTypes;
7428 
7429   // Returns true (the first time) if Ty is, or is found to include, a vector
7430   // type that exposes the vector ABI. This is any vector >=16 bytes which
7431   // with vector support are aligned to only 8 bytes. When IsParam is true,
7432   // the type belongs to a value as passed between functions. If it is a
7433   // vector <=16 bytes it will be passed in a vector register (if supported).
7434   bool isVectorTypeBased(const Type *Ty, bool IsParam) const;
7435 
7436 public:
SystemZTargetCodeGenInfo(CodeGenTypes & CGT,bool HasVector,bool SoftFloatABI)7437   SystemZTargetCodeGenInfo(CodeGenTypes &CGT, bool HasVector, bool SoftFloatABI)
7438       : TargetCodeGenInfo(
7439             std::make_unique<SystemZABIInfo>(CGT, HasVector, SoftFloatABI)),
7440             Ctx(CGT.getContext()) {
7441     SwiftInfo =
7442         std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/false);
7443   }
7444 
7445   // The vector ABI is different when the vector facility is present and when
7446   // a module e.g. defines an externally visible vector variable, a flag
7447   // indicating a visible vector ABI is added. Eventually this will result in
7448   // a GNU attribute indicating the vector ABI of the module.  Ty is the type
7449   // of a variable or function parameter that is globally visible.
handleExternallyVisibleObjABI(const Type * Ty,CodeGen::CodeGenModule & M,bool IsParam) const7450   void handleExternallyVisibleObjABI(const Type *Ty, CodeGen::CodeGenModule &M,
7451                                      bool IsParam) const {
7452     if (!HasVisibleVecABIFlag && isVectorTypeBased(Ty, IsParam)) {
7453       M.getModule().addModuleFlag(llvm::Module::Warning,
7454                                   "s390x-visible-vector-ABI", 1);
7455       HasVisibleVecABIFlag = true;
7456     }
7457   }
7458 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const7459   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
7460                            CodeGen::CodeGenModule &M) const override {
7461     if (!D)
7462       return;
7463 
7464     // Check if the vector ABI becomes visible by an externally visible
7465     // variable or function.
7466     if (const auto *VD = dyn_cast<VarDecl>(D)) {
7467       if (VD->isExternallyVisible())
7468         handleExternallyVisibleObjABI(VD->getType().getTypePtr(), M,
7469                                       /*IsParam*/false);
7470     }
7471     else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
7472       if (FD->isExternallyVisible())
7473         handleExternallyVisibleObjABI(FD->getType().getTypePtr(), M,
7474                                       /*IsParam*/false);
7475     }
7476   }
7477 
testFPKind(llvm::Value * V,unsigned BuiltinID,CGBuilderTy & Builder,CodeGenModule & CGM) const7478   llvm::Value *testFPKind(llvm::Value *V, unsigned BuiltinID,
7479                           CGBuilderTy &Builder,
7480                           CodeGenModule &CGM) const override {
7481     assert(V->getType()->isFloatingPointTy() && "V should have an FP type.");
7482     // Only use TDC in constrained FP mode.
7483     if (!Builder.getIsFPConstrained())
7484       return nullptr;
7485 
7486     llvm::Type *Ty = V->getType();
7487     if (Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isFP128Ty()) {
7488       llvm::Module &M = CGM.getModule();
7489       auto &Ctx = M.getContext();
7490       llvm::Function *TDCFunc =
7491           llvm::Intrinsic::getDeclaration(&M, llvm::Intrinsic::s390_tdc, Ty);
7492       unsigned TDCBits = 0;
7493       switch (BuiltinID) {
7494       case Builtin::BI__builtin_isnan:
7495         TDCBits = 0xf;
7496         break;
7497       case Builtin::BIfinite:
7498       case Builtin::BI__finite:
7499       case Builtin::BIfinitef:
7500       case Builtin::BI__finitef:
7501       case Builtin::BIfinitel:
7502       case Builtin::BI__finitel:
7503       case Builtin::BI__builtin_isfinite:
7504         TDCBits = 0xfc0;
7505         break;
7506       case Builtin::BI__builtin_isinf:
7507         TDCBits = 0x30;
7508         break;
7509       default:
7510         break;
7511       }
7512       if (TDCBits)
7513         return Builder.CreateCall(
7514             TDCFunc,
7515             {V, llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), TDCBits)});
7516     }
7517     return nullptr;
7518   }
7519 };
7520 }
7521 
isPromotableIntegerTypeForABI(QualType Ty) const7522 bool SystemZABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
7523   // Treat an enum type as its underlying type.
7524   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
7525     Ty = EnumTy->getDecl()->getIntegerType();
7526 
7527   // Promotable integer types are required to be promoted by the ABI.
7528   if (ABIInfo::isPromotableIntegerTypeForABI(Ty))
7529     return true;
7530 
7531   if (const auto *EIT = Ty->getAs<BitIntType>())
7532     if (EIT->getNumBits() < 64)
7533       return true;
7534 
7535   // 32-bit values must also be promoted.
7536   if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
7537     switch (BT->getKind()) {
7538     case BuiltinType::Int:
7539     case BuiltinType::UInt:
7540       return true;
7541     default:
7542       return false;
7543     }
7544   return false;
7545 }
7546 
isCompoundType(QualType Ty) const7547 bool SystemZABIInfo::isCompoundType(QualType Ty) const {
7548   return (Ty->isAnyComplexType() ||
7549           Ty->isVectorType() ||
7550           isAggregateTypeForABI(Ty));
7551 }
7552 
isVectorArgumentType(QualType Ty) const7553 bool SystemZABIInfo::isVectorArgumentType(QualType Ty) const {
7554   return (HasVector &&
7555           Ty->isVectorType() &&
7556           getContext().getTypeSize(Ty) <= 128);
7557 }
7558 
isFPArgumentType(QualType Ty) const7559 bool SystemZABIInfo::isFPArgumentType(QualType Ty) const {
7560   if (IsSoftFloatABI)
7561     return false;
7562 
7563   if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
7564     switch (BT->getKind()) {
7565     case BuiltinType::Float:
7566     case BuiltinType::Double:
7567       return true;
7568     default:
7569       return false;
7570     }
7571 
7572   return false;
7573 }
7574 
GetSingleElementType(QualType Ty) const7575 QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const {
7576   const RecordType *RT = Ty->getAs<RecordType>();
7577 
7578   if (RT && RT->isStructureOrClassType()) {
7579     const RecordDecl *RD = RT->getDecl();
7580     QualType Found;
7581 
7582     // If this is a C++ record, check the bases first.
7583     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
7584       if (CXXRD->hasDefinition())
7585         for (const auto &I : CXXRD->bases()) {
7586           QualType Base = I.getType();
7587 
7588           // Empty bases don't affect things either way.
7589           if (isEmptyRecord(getContext(), Base, true))
7590             continue;
7591 
7592           if (!Found.isNull())
7593             return Ty;
7594           Found = GetSingleElementType(Base);
7595         }
7596 
7597     // Check the fields.
7598     for (const auto *FD : RD->fields()) {
7599       // Unlike isSingleElementStruct(), empty structure and array fields
7600       // do count.  So do anonymous bitfields that aren't zero-sized.
7601 
7602       // Like isSingleElementStruct(), ignore C++20 empty data members.
7603       if (FD->hasAttr<NoUniqueAddressAttr>() &&
7604           isEmptyRecord(getContext(), FD->getType(), true))
7605         continue;
7606 
7607       // Unlike isSingleElementStruct(), arrays do not count.
7608       // Nested structures still do though.
7609       if (!Found.isNull())
7610         return Ty;
7611       Found = GetSingleElementType(FD->getType());
7612     }
7613 
7614     // Unlike isSingleElementStruct(), trailing padding is allowed.
7615     // An 8-byte aligned struct s { float f; } is passed as a double.
7616     if (!Found.isNull())
7617       return Found;
7618   }
7619 
7620   return Ty;
7621 }
7622 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const7623 Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7624                                   QualType Ty) const {
7625   // Assume that va_list type is correct; should be pointer to LLVM type:
7626   // struct {
7627   //   i64 __gpr;
7628   //   i64 __fpr;
7629   //   i8 *__overflow_arg_area;
7630   //   i8 *__reg_save_area;
7631   // };
7632 
7633   // Every non-vector argument occupies 8 bytes and is passed by preference
7634   // in either GPRs or FPRs.  Vector arguments occupy 8 or 16 bytes and are
7635   // always passed on the stack.
7636   const SystemZTargetCodeGenInfo &SZCGI =
7637       static_cast<const SystemZTargetCodeGenInfo &>(
7638           CGT.getCGM().getTargetCodeGenInfo());
7639   Ty = getContext().getCanonicalType(Ty);
7640   auto TyInfo = getContext().getTypeInfoInChars(Ty);
7641   llvm::Type *ArgTy = CGF.ConvertTypeForMem(Ty);
7642   llvm::Type *DirectTy = ArgTy;
7643   ABIArgInfo AI = classifyArgumentType(Ty);
7644   bool IsIndirect = AI.isIndirect();
7645   bool InFPRs = false;
7646   bool IsVector = false;
7647   CharUnits UnpaddedSize;
7648   CharUnits DirectAlign;
7649   SZCGI.handleExternallyVisibleObjABI(Ty.getTypePtr(), CGT.getCGM(),
7650                                       /*IsParam*/true);
7651   if (IsIndirect) {
7652     DirectTy = llvm::PointerType::getUnqual(DirectTy);
7653     UnpaddedSize = DirectAlign = CharUnits::fromQuantity(8);
7654   } else {
7655     if (AI.getCoerceToType())
7656       ArgTy = AI.getCoerceToType();
7657     InFPRs = (!IsSoftFloatABI && (ArgTy->isFloatTy() || ArgTy->isDoubleTy()));
7658     IsVector = ArgTy->isVectorTy();
7659     UnpaddedSize = TyInfo.Width;
7660     DirectAlign = TyInfo.Align;
7661   }
7662   CharUnits PaddedSize = CharUnits::fromQuantity(8);
7663   if (IsVector && UnpaddedSize > PaddedSize)
7664     PaddedSize = CharUnits::fromQuantity(16);
7665   assert((UnpaddedSize <= PaddedSize) && "Invalid argument size.");
7666 
7667   CharUnits Padding = (PaddedSize - UnpaddedSize);
7668 
7669   llvm::Type *IndexTy = CGF.Int64Ty;
7670   llvm::Value *PaddedSizeV =
7671     llvm::ConstantInt::get(IndexTy, PaddedSize.getQuantity());
7672 
7673   if (IsVector) {
7674     // Work out the address of a vector argument on the stack.
7675     // Vector arguments are always passed in the high bits of a
7676     // single (8 byte) or double (16 byte) stack slot.
7677     Address OverflowArgAreaPtr =
7678         CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_ptr");
7679     Address OverflowArgArea =
7680         Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
7681                 CGF.Int8Ty, TyInfo.Align);
7682     Address MemAddr =
7683         CGF.Builder.CreateElementBitCast(OverflowArgArea, DirectTy, "mem_addr");
7684 
7685     // Update overflow_arg_area_ptr pointer
7686     llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP(
7687         OverflowArgArea.getElementType(), OverflowArgArea.getPointer(),
7688         PaddedSizeV, "overflow_arg_area");
7689     CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
7690 
7691     return MemAddr;
7692   }
7693 
7694   assert(PaddedSize.getQuantity() == 8);
7695 
7696   unsigned MaxRegs, RegCountField, RegSaveIndex;
7697   CharUnits RegPadding;
7698   if (InFPRs) {
7699     MaxRegs = 4; // Maximum of 4 FPR arguments
7700     RegCountField = 1; // __fpr
7701     RegSaveIndex = 16; // save offset for f0
7702     RegPadding = CharUnits(); // floats are passed in the high bits of an FPR
7703   } else {
7704     MaxRegs = 5; // Maximum of 5 GPR arguments
7705     RegCountField = 0; // __gpr
7706     RegSaveIndex = 2; // save offset for r2
7707     RegPadding = Padding; // values are passed in the low bits of a GPR
7708   }
7709 
7710   Address RegCountPtr =
7711       CGF.Builder.CreateStructGEP(VAListAddr, RegCountField, "reg_count_ptr");
7712   llvm::Value *RegCount = CGF.Builder.CreateLoad(RegCountPtr, "reg_count");
7713   llvm::Value *MaxRegsV = llvm::ConstantInt::get(IndexTy, MaxRegs);
7714   llvm::Value *InRegs = CGF.Builder.CreateICmpULT(RegCount, MaxRegsV,
7715                                                  "fits_in_regs");
7716 
7717   llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
7718   llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
7719   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
7720   CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
7721 
7722   // Emit code to load the value if it was passed in registers.
7723   CGF.EmitBlock(InRegBlock);
7724 
7725   // Work out the address of an argument register.
7726   llvm::Value *ScaledRegCount =
7727     CGF.Builder.CreateMul(RegCount, PaddedSizeV, "scaled_reg_count");
7728   llvm::Value *RegBase =
7729     llvm::ConstantInt::get(IndexTy, RegSaveIndex * PaddedSize.getQuantity()
7730                                       + RegPadding.getQuantity());
7731   llvm::Value *RegOffset =
7732     CGF.Builder.CreateAdd(ScaledRegCount, RegBase, "reg_offset");
7733   Address RegSaveAreaPtr =
7734       CGF.Builder.CreateStructGEP(VAListAddr, 3, "reg_save_area_ptr");
7735   llvm::Value *RegSaveArea =
7736       CGF.Builder.CreateLoad(RegSaveAreaPtr, "reg_save_area");
7737   Address RawRegAddr(
7738       CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, RegOffset, "raw_reg_addr"),
7739       CGF.Int8Ty, PaddedSize);
7740   Address RegAddr =
7741       CGF.Builder.CreateElementBitCast(RawRegAddr, DirectTy, "reg_addr");
7742 
7743   // Update the register count
7744   llvm::Value *One = llvm::ConstantInt::get(IndexTy, 1);
7745   llvm::Value *NewRegCount =
7746     CGF.Builder.CreateAdd(RegCount, One, "reg_count");
7747   CGF.Builder.CreateStore(NewRegCount, RegCountPtr);
7748   CGF.EmitBranch(ContBlock);
7749 
7750   // Emit code to load the value if it was passed in memory.
7751   CGF.EmitBlock(InMemBlock);
7752 
7753   // Work out the address of a stack argument.
7754   Address OverflowArgAreaPtr =
7755       CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_ptr");
7756   Address OverflowArgArea =
7757       Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
7758               CGF.Int8Ty, PaddedSize);
7759   Address RawMemAddr =
7760       CGF.Builder.CreateConstByteGEP(OverflowArgArea, Padding, "raw_mem_addr");
7761   Address MemAddr =
7762     CGF.Builder.CreateElementBitCast(RawMemAddr, DirectTy, "mem_addr");
7763 
7764   // Update overflow_arg_area_ptr pointer
7765   llvm::Value *NewOverflowArgArea =
7766     CGF.Builder.CreateGEP(OverflowArgArea.getElementType(),
7767                           OverflowArgArea.getPointer(), PaddedSizeV,
7768                           "overflow_arg_area");
7769   CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
7770   CGF.EmitBranch(ContBlock);
7771 
7772   // Return the appropriate result.
7773   CGF.EmitBlock(ContBlock);
7774   Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
7775                                  "va_arg.addr");
7776 
7777   if (IsIndirect)
7778     ResAddr = Address(CGF.Builder.CreateLoad(ResAddr, "indirect_arg"), ArgTy,
7779                       TyInfo.Align);
7780 
7781   return ResAddr;
7782 }
7783 
classifyReturnType(QualType RetTy) const7784 ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const {
7785   if (RetTy->isVoidType())
7786     return ABIArgInfo::getIgnore();
7787   if (isVectorArgumentType(RetTy))
7788     return ABIArgInfo::getDirect();
7789   if (isCompoundType(RetTy) || getContext().getTypeSize(RetTy) > 64)
7790     return getNaturalAlignIndirect(RetTy);
7791   return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
7792                                                : ABIArgInfo::getDirect());
7793 }
7794 
classifyArgumentType(QualType Ty) const7795 ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const {
7796   // Handle the generic C++ ABI.
7797   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
7798     return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
7799 
7800   // Integers and enums are extended to full register width.
7801   if (isPromotableIntegerTypeForABI(Ty))
7802     return ABIArgInfo::getExtend(Ty);
7803 
7804   // Handle vector types and vector-like structure types.  Note that
7805   // as opposed to float-like structure types, we do not allow any
7806   // padding for vector-like structures, so verify the sizes match.
7807   uint64_t Size = getContext().getTypeSize(Ty);
7808   QualType SingleElementTy = GetSingleElementType(Ty);
7809   if (isVectorArgumentType(SingleElementTy) &&
7810       getContext().getTypeSize(SingleElementTy) == Size)
7811     return ABIArgInfo::getDirect(CGT.ConvertType(SingleElementTy));
7812 
7813   // Values that are not 1, 2, 4 or 8 bytes in size are passed indirectly.
7814   if (Size != 8 && Size != 16 && Size != 32 && Size != 64)
7815     return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
7816 
7817   // Handle small structures.
7818   if (const RecordType *RT = Ty->getAs<RecordType>()) {
7819     // Structures with flexible arrays have variable length, so really
7820     // fail the size test above.
7821     const RecordDecl *RD = RT->getDecl();
7822     if (RD->hasFlexibleArrayMember())
7823       return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
7824 
7825     // The structure is passed as an unextended integer, a float, or a double.
7826     llvm::Type *PassTy;
7827     if (isFPArgumentType(SingleElementTy)) {
7828       assert(Size == 32 || Size == 64);
7829       if (Size == 32)
7830         PassTy = llvm::Type::getFloatTy(getVMContext());
7831       else
7832         PassTy = llvm::Type::getDoubleTy(getVMContext());
7833     } else
7834       PassTy = llvm::IntegerType::get(getVMContext(), Size);
7835     return ABIArgInfo::getDirect(PassTy);
7836   }
7837 
7838   // Non-structure compounds are passed indirectly.
7839   if (isCompoundType(Ty))
7840     return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
7841 
7842   return ABIArgInfo::getDirect(nullptr);
7843 }
7844 
computeInfo(CGFunctionInfo & FI) const7845 void SystemZABIInfo::computeInfo(CGFunctionInfo &FI) const {
7846   const SystemZTargetCodeGenInfo &SZCGI =
7847       static_cast<const SystemZTargetCodeGenInfo &>(
7848           CGT.getCGM().getTargetCodeGenInfo());
7849   if (!getCXXABI().classifyReturnType(FI))
7850     FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
7851   unsigned Idx = 0;
7852   for (auto &I : FI.arguments()) {
7853     I.info = classifyArgumentType(I.type);
7854     if (FI.isVariadic() && Idx++ >= FI.getNumRequiredArgs())
7855       // Check if a vararg vector argument is passed, in which case the
7856       // vector ABI becomes visible as the va_list could be passed on to
7857       // other functions.
7858       SZCGI.handleExternallyVisibleObjABI(I.type.getTypePtr(), CGT.getCGM(),
7859                                           /*IsParam*/true);
7860   }
7861 }
7862 
isVectorTypeBased(const Type * Ty,bool IsParam) const7863 bool SystemZTargetCodeGenInfo::isVectorTypeBased(const Type *Ty,
7864                                                  bool IsParam) const {
7865   if (!SeenTypes.insert(Ty).second)
7866     return false;
7867 
7868   if (IsParam) {
7869     // A narrow (<16 bytes) vector will as a parameter also expose the ABI as
7870     // it will be passed in a vector register. A wide (>16 bytes) vector will
7871     // be passed via "hidden" pointer where any extra alignment is not
7872     // required (per GCC).
7873     const Type *SingleEltTy =
7874       getABIInfo().GetSingleElementType(QualType(Ty, 0)).getTypePtr();
7875     bool SingleVecEltStruct = SingleEltTy != Ty && SingleEltTy->isVectorType() &&
7876       Ctx.getTypeSize(SingleEltTy) == Ctx.getTypeSize(Ty);
7877     if (Ty->isVectorType() || SingleVecEltStruct)
7878       return Ctx.getTypeSize(Ty) / 8 <= 16;
7879   }
7880 
7881   // Assume pointers are dereferenced.
7882   while (Ty->isPointerType() || Ty->isArrayType())
7883     Ty = Ty->getPointeeOrArrayElementType();
7884 
7885   // Vectors >= 16 bytes expose the ABI through alignment requirements.
7886   if (Ty->isVectorType() && Ctx.getTypeSize(Ty) / 8 >= 16)
7887       return true;
7888 
7889   if (const auto *RecordTy = Ty->getAs<RecordType>()) {
7890     const RecordDecl *RD = RecordTy->getDecl();
7891     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
7892       if (CXXRD->hasDefinition())
7893         for (const auto &I : CXXRD->bases())
7894           if (isVectorTypeBased(I.getType().getTypePtr(), /*IsParam*/false))
7895             return true;
7896     for (const auto *FD : RD->fields())
7897       if (isVectorTypeBased(FD->getType().getTypePtr(), /*IsParam*/false))
7898         return true;
7899   }
7900 
7901   if (const auto *FT = Ty->getAs<FunctionType>())
7902     if (isVectorTypeBased(FT->getReturnType().getTypePtr(), /*IsParam*/true))
7903       return true;
7904   if (const FunctionProtoType *Proto = Ty->getAs<FunctionProtoType>())
7905     for (auto ParamType : Proto->getParamTypes())
7906       if (isVectorTypeBased(ParamType.getTypePtr(), /*IsParam*/true))
7907         return true;
7908 
7909   return false;
7910 }
7911 
7912 //===----------------------------------------------------------------------===//
7913 // MSP430 ABI Implementation
7914 //===----------------------------------------------------------------------===//
7915 
7916 namespace {
7917 
7918 class MSP430ABIInfo : public DefaultABIInfo {
complexArgInfo()7919   static ABIArgInfo complexArgInfo() {
7920     ABIArgInfo Info = ABIArgInfo::getDirect();
7921     Info.setCanBeFlattened(false);
7922     return Info;
7923   }
7924 
7925 public:
MSP430ABIInfo(CodeGenTypes & CGT)7926   MSP430ABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
7927 
classifyReturnType(QualType RetTy) const7928   ABIArgInfo classifyReturnType(QualType RetTy) const {
7929     if (RetTy->isAnyComplexType())
7930       return complexArgInfo();
7931 
7932     return DefaultABIInfo::classifyReturnType(RetTy);
7933   }
7934 
classifyArgumentType(QualType RetTy) const7935   ABIArgInfo classifyArgumentType(QualType RetTy) const {
7936     if (RetTy->isAnyComplexType())
7937       return complexArgInfo();
7938 
7939     return DefaultABIInfo::classifyArgumentType(RetTy);
7940   }
7941 
7942   // Just copy the original implementations because
7943   // DefaultABIInfo::classify{Return,Argument}Type() are not virtual
computeInfo(CGFunctionInfo & FI) const7944   void computeInfo(CGFunctionInfo &FI) const override {
7945     if (!getCXXABI().classifyReturnType(FI))
7946       FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
7947     for (auto &I : FI.arguments())
7948       I.info = classifyArgumentType(I.type);
7949   }
7950 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const7951   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7952                     QualType Ty) const override {
7953     return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
7954   }
7955 };
7956 
7957 class MSP430TargetCodeGenInfo : public TargetCodeGenInfo {
7958 public:
MSP430TargetCodeGenInfo(CodeGenTypes & CGT)7959   MSP430TargetCodeGenInfo(CodeGenTypes &CGT)
7960       : TargetCodeGenInfo(std::make_unique<MSP430ABIInfo>(CGT)) {}
7961   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
7962                            CodeGen::CodeGenModule &M) const override;
7963 };
7964 
7965 }
7966 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const7967 void MSP430TargetCodeGenInfo::setTargetAttributes(
7968     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
7969   if (GV->isDeclaration())
7970     return;
7971   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
7972     const auto *InterruptAttr = FD->getAttr<MSP430InterruptAttr>();
7973     if (!InterruptAttr)
7974       return;
7975 
7976     // Handle 'interrupt' attribute:
7977     llvm::Function *F = cast<llvm::Function>(GV);
7978 
7979     // Step 1: Set ISR calling convention.
7980     F->setCallingConv(llvm::CallingConv::MSP430_INTR);
7981 
7982     // Step 2: Add attributes goodness.
7983     F->addFnAttr(llvm::Attribute::NoInline);
7984     F->addFnAttr("interrupt", llvm::utostr(InterruptAttr->getNumber()));
7985   }
7986 }
7987 
7988 //===----------------------------------------------------------------------===//
7989 // MIPS ABI Implementation.  This works for both little-endian and
7990 // big-endian variants.
7991 //===----------------------------------------------------------------------===//
7992 
7993 namespace {
7994 class MipsABIInfo : public ABIInfo {
7995   bool IsO32;
7996   const unsigned MinABIStackAlignInBytes, StackAlignInBytes;
7997   void CoerceToIntArgs(uint64_t TySize,
7998                        SmallVectorImpl<llvm::Type *> &ArgList) const;
7999   llvm::Type* HandleAggregates(QualType Ty, uint64_t TySize) const;
8000   llvm::Type* returnAggregateInRegs(QualType RetTy, uint64_t Size) const;
8001   llvm::Type* getPaddingType(uint64_t Align, uint64_t Offset) const;
8002 public:
MipsABIInfo(CodeGenTypes & CGT,bool _IsO32)8003   MipsABIInfo(CodeGenTypes &CGT, bool _IsO32) :
8004     ABIInfo(CGT), IsO32(_IsO32), MinABIStackAlignInBytes(IsO32 ? 4 : 8),
8005     StackAlignInBytes(IsO32 ? 8 : 16) {}
8006 
8007   ABIArgInfo classifyReturnType(QualType RetTy) const;
8008   ABIArgInfo classifyArgumentType(QualType RetTy, uint64_t &Offset) const;
8009   void computeInfo(CGFunctionInfo &FI) const override;
8010   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
8011                     QualType Ty) const override;
8012   ABIArgInfo extendType(QualType Ty) const;
8013 };
8014 
8015 class MIPSTargetCodeGenInfo : public TargetCodeGenInfo {
8016   unsigned SizeOfUnwindException;
8017 public:
MIPSTargetCodeGenInfo(CodeGenTypes & CGT,bool IsO32)8018   MIPSTargetCodeGenInfo(CodeGenTypes &CGT, bool IsO32)
8019       : TargetCodeGenInfo(std::make_unique<MipsABIInfo>(CGT, IsO32)),
8020         SizeOfUnwindException(IsO32 ? 24 : 32) {}
8021 
getDwarfEHStackPointer(CodeGen::CodeGenModule & CGM) const8022   int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
8023     return 29;
8024   }
8025 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const8026   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
8027                            CodeGen::CodeGenModule &CGM) const override {
8028     const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
8029     if (!FD) return;
8030     llvm::Function *Fn = cast<llvm::Function>(GV);
8031 
8032     if (FD->hasAttr<MipsLongCallAttr>())
8033       Fn->addFnAttr("long-call");
8034     else if (FD->hasAttr<MipsShortCallAttr>())
8035       Fn->addFnAttr("short-call");
8036 
8037     // Other attributes do not have a meaning for declarations.
8038     if (GV->isDeclaration())
8039       return;
8040 
8041     if (FD->hasAttr<Mips16Attr>()) {
8042       Fn->addFnAttr("mips16");
8043     }
8044     else if (FD->hasAttr<NoMips16Attr>()) {
8045       Fn->addFnAttr("nomips16");
8046     }
8047 
8048     if (FD->hasAttr<MicroMipsAttr>())
8049       Fn->addFnAttr("micromips");
8050     else if (FD->hasAttr<NoMicroMipsAttr>())
8051       Fn->addFnAttr("nomicromips");
8052 
8053     const MipsInterruptAttr *Attr = FD->getAttr<MipsInterruptAttr>();
8054     if (!Attr)
8055       return;
8056 
8057     const char *Kind;
8058     switch (Attr->getInterrupt()) {
8059     case MipsInterruptAttr::eic:     Kind = "eic"; break;
8060     case MipsInterruptAttr::sw0:     Kind = "sw0"; break;
8061     case MipsInterruptAttr::sw1:     Kind = "sw1"; break;
8062     case MipsInterruptAttr::hw0:     Kind = "hw0"; break;
8063     case MipsInterruptAttr::hw1:     Kind = "hw1"; break;
8064     case MipsInterruptAttr::hw2:     Kind = "hw2"; break;
8065     case MipsInterruptAttr::hw3:     Kind = "hw3"; break;
8066     case MipsInterruptAttr::hw4:     Kind = "hw4"; break;
8067     case MipsInterruptAttr::hw5:     Kind = "hw5"; break;
8068     }
8069 
8070     Fn->addFnAttr("interrupt", Kind);
8071 
8072   }
8073 
8074   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
8075                                llvm::Value *Address) const override;
8076 
getSizeOfUnwindException() const8077   unsigned getSizeOfUnwindException() const override {
8078     return SizeOfUnwindException;
8079   }
8080 };
8081 }
8082 
CoerceToIntArgs(uint64_t TySize,SmallVectorImpl<llvm::Type * > & ArgList) const8083 void MipsABIInfo::CoerceToIntArgs(
8084     uint64_t TySize, SmallVectorImpl<llvm::Type *> &ArgList) const {
8085   llvm::IntegerType *IntTy =
8086     llvm::IntegerType::get(getVMContext(), MinABIStackAlignInBytes * 8);
8087 
8088   // Add (TySize / MinABIStackAlignInBytes) args of IntTy.
8089   for (unsigned N = TySize / (MinABIStackAlignInBytes * 8); N; --N)
8090     ArgList.push_back(IntTy);
8091 
8092   // If necessary, add one more integer type to ArgList.
8093   unsigned R = TySize % (MinABIStackAlignInBytes * 8);
8094 
8095   if (R)
8096     ArgList.push_back(llvm::IntegerType::get(getVMContext(), R));
8097 }
8098 
8099 // In N32/64, an aligned double precision floating point field is passed in
8100 // a register.
HandleAggregates(QualType Ty,uint64_t TySize) const8101 llvm::Type* MipsABIInfo::HandleAggregates(QualType Ty, uint64_t TySize) const {
8102   SmallVector<llvm::Type*, 8> ArgList, IntArgList;
8103 
8104   if (IsO32) {
8105     CoerceToIntArgs(TySize, ArgList);
8106     return llvm::StructType::get(getVMContext(), ArgList);
8107   }
8108 
8109   if (Ty->isComplexType())
8110     return CGT.ConvertType(Ty);
8111 
8112   const RecordType *RT = Ty->getAs<RecordType>();
8113 
8114   // Unions/vectors are passed in integer registers.
8115   if (!RT || !RT->isStructureOrClassType()) {
8116     CoerceToIntArgs(TySize, ArgList);
8117     return llvm::StructType::get(getVMContext(), ArgList);
8118   }
8119 
8120   const RecordDecl *RD = RT->getDecl();
8121   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
8122   assert(!(TySize % 8) && "Size of structure must be multiple of 8.");
8123 
8124   uint64_t LastOffset = 0;
8125   unsigned idx = 0;
8126   llvm::IntegerType *I64 = llvm::IntegerType::get(getVMContext(), 64);
8127 
8128   // Iterate over fields in the struct/class and check if there are any aligned
8129   // double fields.
8130   for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
8131        i != e; ++i, ++idx) {
8132     const QualType Ty = i->getType();
8133     const BuiltinType *BT = Ty->getAs<BuiltinType>();
8134 
8135     if (!BT || BT->getKind() != BuiltinType::Double)
8136       continue;
8137 
8138     uint64_t Offset = Layout.getFieldOffset(idx);
8139     if (Offset % 64) // Ignore doubles that are not aligned.
8140       continue;
8141 
8142     // Add ((Offset - LastOffset) / 64) args of type i64.
8143     for (unsigned j = (Offset - LastOffset) / 64; j > 0; --j)
8144       ArgList.push_back(I64);
8145 
8146     // Add double type.
8147     ArgList.push_back(llvm::Type::getDoubleTy(getVMContext()));
8148     LastOffset = Offset + 64;
8149   }
8150 
8151   CoerceToIntArgs(TySize - LastOffset, IntArgList);
8152   ArgList.append(IntArgList.begin(), IntArgList.end());
8153 
8154   return llvm::StructType::get(getVMContext(), ArgList);
8155 }
8156 
getPaddingType(uint64_t OrigOffset,uint64_t Offset) const8157 llvm::Type *MipsABIInfo::getPaddingType(uint64_t OrigOffset,
8158                                         uint64_t Offset) const {
8159   if (OrigOffset + MinABIStackAlignInBytes > Offset)
8160     return nullptr;
8161 
8162   return llvm::IntegerType::get(getVMContext(), (Offset - OrigOffset) * 8);
8163 }
8164 
8165 ABIArgInfo
classifyArgumentType(QualType Ty,uint64_t & Offset) const8166 MipsABIInfo::classifyArgumentType(QualType Ty, uint64_t &Offset) const {
8167   Ty = useFirstFieldIfTransparentUnion(Ty);
8168 
8169   uint64_t OrigOffset = Offset;
8170   uint64_t TySize = getContext().getTypeSize(Ty);
8171   uint64_t Align = getContext().getTypeAlign(Ty) / 8;
8172 
8173   Align = std::clamp(Align, (uint64_t)MinABIStackAlignInBytes,
8174                      (uint64_t)StackAlignInBytes);
8175   unsigned CurrOffset = llvm::alignTo(Offset, Align);
8176   Offset = CurrOffset + llvm::alignTo(TySize, Align * 8) / 8;
8177 
8178   if (isAggregateTypeForABI(Ty) || Ty->isVectorType()) {
8179     // Ignore empty aggregates.
8180     if (TySize == 0)
8181       return ABIArgInfo::getIgnore();
8182 
8183     if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
8184       Offset = OrigOffset + MinABIStackAlignInBytes;
8185       return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
8186     }
8187 
8188     // If we have reached here, aggregates are passed directly by coercing to
8189     // another structure type. Padding is inserted if the offset of the
8190     // aggregate is unaligned.
8191     ABIArgInfo ArgInfo =
8192         ABIArgInfo::getDirect(HandleAggregates(Ty, TySize), 0,
8193                               getPaddingType(OrigOffset, CurrOffset));
8194     ArgInfo.setInReg(true);
8195     return ArgInfo;
8196   }
8197 
8198   // Treat an enum type as its underlying type.
8199   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
8200     Ty = EnumTy->getDecl()->getIntegerType();
8201 
8202   // Make sure we pass indirectly things that are too large.
8203   if (const auto *EIT = Ty->getAs<BitIntType>())
8204     if (EIT->getNumBits() > 128 ||
8205         (EIT->getNumBits() > 64 &&
8206          !getContext().getTargetInfo().hasInt128Type()))
8207       return getNaturalAlignIndirect(Ty);
8208 
8209   // All integral types are promoted to the GPR width.
8210   if (Ty->isIntegralOrEnumerationType())
8211     return extendType(Ty);
8212 
8213   return ABIArgInfo::getDirect(
8214       nullptr, 0, IsO32 ? nullptr : getPaddingType(OrigOffset, CurrOffset));
8215 }
8216 
8217 llvm::Type*
returnAggregateInRegs(QualType RetTy,uint64_t Size) const8218 MipsABIInfo::returnAggregateInRegs(QualType RetTy, uint64_t Size) const {
8219   const RecordType *RT = RetTy->getAs<RecordType>();
8220   SmallVector<llvm::Type*, 8> RTList;
8221 
8222   if (RT && RT->isStructureOrClassType()) {
8223     const RecordDecl *RD = RT->getDecl();
8224     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
8225     unsigned FieldCnt = Layout.getFieldCount();
8226 
8227     // N32/64 returns struct/classes in floating point registers if the
8228     // following conditions are met:
8229     // 1. The size of the struct/class is no larger than 128-bit.
8230     // 2. The struct/class has one or two fields all of which are floating
8231     //    point types.
8232     // 3. The offset of the first field is zero (this follows what gcc does).
8233     //
8234     // Any other composite results are returned in integer registers.
8235     //
8236     if (FieldCnt && (FieldCnt <= 2) && !Layout.getFieldOffset(0)) {
8237       RecordDecl::field_iterator b = RD->field_begin(), e = RD->field_end();
8238       for (; b != e; ++b) {
8239         const BuiltinType *BT = b->getType()->getAs<BuiltinType>();
8240 
8241         if (!BT || !BT->isFloatingPoint())
8242           break;
8243 
8244         RTList.push_back(CGT.ConvertType(b->getType()));
8245       }
8246 
8247       if (b == e)
8248         return llvm::StructType::get(getVMContext(), RTList,
8249                                      RD->hasAttr<PackedAttr>());
8250 
8251       RTList.clear();
8252     }
8253   }
8254 
8255   CoerceToIntArgs(Size, RTList);
8256   return llvm::StructType::get(getVMContext(), RTList);
8257 }
8258 
classifyReturnType(QualType RetTy) const8259 ABIArgInfo MipsABIInfo::classifyReturnType(QualType RetTy) const {
8260   uint64_t Size = getContext().getTypeSize(RetTy);
8261 
8262   if (RetTy->isVoidType())
8263     return ABIArgInfo::getIgnore();
8264 
8265   // O32 doesn't treat zero-sized structs differently from other structs.
8266   // However, N32/N64 ignores zero sized return values.
8267   if (!IsO32 && Size == 0)
8268     return ABIArgInfo::getIgnore();
8269 
8270   if (isAggregateTypeForABI(RetTy) || RetTy->isVectorType()) {
8271     if (Size <= 128) {
8272       if (RetTy->isAnyComplexType())
8273         return ABIArgInfo::getDirect();
8274 
8275       // O32 returns integer vectors in registers and N32/N64 returns all small
8276       // aggregates in registers.
8277       if (!IsO32 ||
8278           (RetTy->isVectorType() && !RetTy->hasFloatingRepresentation())) {
8279         ABIArgInfo ArgInfo =
8280             ABIArgInfo::getDirect(returnAggregateInRegs(RetTy, Size));
8281         ArgInfo.setInReg(true);
8282         return ArgInfo;
8283       }
8284     }
8285 
8286     return getNaturalAlignIndirect(RetTy);
8287   }
8288 
8289   // Treat an enum type as its underlying type.
8290   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
8291     RetTy = EnumTy->getDecl()->getIntegerType();
8292 
8293   // Make sure we pass indirectly things that are too large.
8294   if (const auto *EIT = RetTy->getAs<BitIntType>())
8295     if (EIT->getNumBits() > 128 ||
8296         (EIT->getNumBits() > 64 &&
8297          !getContext().getTargetInfo().hasInt128Type()))
8298       return getNaturalAlignIndirect(RetTy);
8299 
8300   if (isPromotableIntegerTypeForABI(RetTy))
8301     return ABIArgInfo::getExtend(RetTy);
8302 
8303   if ((RetTy->isUnsignedIntegerOrEnumerationType() ||
8304       RetTy->isSignedIntegerOrEnumerationType()) && Size == 32 && !IsO32)
8305     return ABIArgInfo::getSignExtend(RetTy);
8306 
8307   return ABIArgInfo::getDirect();
8308 }
8309 
computeInfo(CGFunctionInfo & FI) const8310 void MipsABIInfo::computeInfo(CGFunctionInfo &FI) const {
8311   ABIArgInfo &RetInfo = FI.getReturnInfo();
8312   if (!getCXXABI().classifyReturnType(FI))
8313     RetInfo = classifyReturnType(FI.getReturnType());
8314 
8315   // Check if a pointer to an aggregate is passed as a hidden argument.
8316   uint64_t Offset = RetInfo.isIndirect() ? MinABIStackAlignInBytes : 0;
8317 
8318   for (auto &I : FI.arguments())
8319     I.info = classifyArgumentType(I.type, Offset);
8320 }
8321 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType OrigTy) const8322 Address MipsABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
8323                                QualType OrigTy) const {
8324   QualType Ty = OrigTy;
8325 
8326   // Integer arguments are promoted to 32-bit on O32 and 64-bit on N32/N64.
8327   // Pointers are also promoted in the same way but this only matters for N32.
8328   unsigned SlotSizeInBits = IsO32 ? 32 : 64;
8329   unsigned PtrWidth = getTarget().getPointerWidth(LangAS::Default);
8330   bool DidPromote = false;
8331   if ((Ty->isIntegerType() &&
8332           getContext().getIntWidth(Ty) < SlotSizeInBits) ||
8333       (Ty->isPointerType() && PtrWidth < SlotSizeInBits)) {
8334     DidPromote = true;
8335     Ty = getContext().getIntTypeForBitwidth(SlotSizeInBits,
8336                                             Ty->isSignedIntegerType());
8337   }
8338 
8339   auto TyInfo = getContext().getTypeInfoInChars(Ty);
8340 
8341   // The alignment of things in the argument area is never larger than
8342   // StackAlignInBytes.
8343   TyInfo.Align =
8344     std::min(TyInfo.Align, CharUnits::fromQuantity(StackAlignInBytes));
8345 
8346   // MinABIStackAlignInBytes is the size of argument slots on the stack.
8347   CharUnits ArgSlotSize = CharUnits::fromQuantity(MinABIStackAlignInBytes);
8348 
8349   Address Addr = emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
8350                           TyInfo, ArgSlotSize, /*AllowHigherAlign*/ true);
8351 
8352 
8353   // If there was a promotion, "unpromote" into a temporary.
8354   // TODO: can we just use a pointer into a subset of the original slot?
8355   if (DidPromote) {
8356     Address Temp = CGF.CreateMemTemp(OrigTy, "vaarg.promotion-temp");
8357     llvm::Value *Promoted = CGF.Builder.CreateLoad(Addr);
8358 
8359     // Truncate down to the right width.
8360     llvm::Type *IntTy = (OrigTy->isIntegerType() ? Temp.getElementType()
8361                                                  : CGF.IntPtrTy);
8362     llvm::Value *V = CGF.Builder.CreateTrunc(Promoted, IntTy);
8363     if (OrigTy->isPointerType())
8364       V = CGF.Builder.CreateIntToPtr(V, Temp.getElementType());
8365 
8366     CGF.Builder.CreateStore(V, Temp);
8367     Addr = Temp;
8368   }
8369 
8370   return Addr;
8371 }
8372 
extendType(QualType Ty) const8373 ABIArgInfo MipsABIInfo::extendType(QualType Ty) const {
8374   int TySize = getContext().getTypeSize(Ty);
8375 
8376   // MIPS64 ABI requires unsigned 32 bit integers to be sign extended.
8377   if (Ty->isUnsignedIntegerOrEnumerationType() && TySize == 32)
8378     return ABIArgInfo::getSignExtend(Ty);
8379 
8380   return ABIArgInfo::getExtend(Ty);
8381 }
8382 
8383 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const8384 MIPSTargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
8385                                                llvm::Value *Address) const {
8386   // This information comes from gcc's implementation, which seems to
8387   // as canonical as it gets.
8388 
8389   // Everything on MIPS is 4 bytes.  Double-precision FP registers
8390   // are aliased to pairs of single-precision FP registers.
8391   llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
8392 
8393   // 0-31 are the general purpose registers, $0 - $31.
8394   // 32-63 are the floating-point registers, $f0 - $f31.
8395   // 64 and 65 are the multiply/divide registers, $hi and $lo.
8396   // 66 is the (notional, I think) register for signal-handler return.
8397   AssignToArrayRange(CGF.Builder, Address, Four8, 0, 65);
8398 
8399   // 67-74 are the floating-point status registers, $fcc0 - $fcc7.
8400   // They are one bit wide and ignored here.
8401 
8402   // 80-111 are the coprocessor 0 registers, $c0r0 - $c0r31.
8403   // (coprocessor 1 is the FP unit)
8404   // 112-143 are the coprocessor 2 registers, $c2r0 - $c2r31.
8405   // 144-175 are the coprocessor 3 registers, $c3r0 - $c3r31.
8406   // 176-181 are the DSP accumulator registers.
8407   AssignToArrayRange(CGF.Builder, Address, Four8, 80, 181);
8408   return false;
8409 }
8410 
8411 //===----------------------------------------------------------------------===//
8412 // M68k ABI Implementation
8413 //===----------------------------------------------------------------------===//
8414 
8415 namespace {
8416 
8417 class M68kTargetCodeGenInfo : public TargetCodeGenInfo {
8418 public:
M68kTargetCodeGenInfo(CodeGenTypes & CGT)8419   M68kTargetCodeGenInfo(CodeGenTypes &CGT)
8420       : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
8421   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
8422                            CodeGen::CodeGenModule &M) const override;
8423 };
8424 
8425 } // namespace
8426 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const8427 void M68kTargetCodeGenInfo::setTargetAttributes(
8428     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
8429   if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
8430     if (const auto *attr = FD->getAttr<M68kInterruptAttr>()) {
8431       // Handle 'interrupt' attribute:
8432       llvm::Function *F = cast<llvm::Function>(GV);
8433 
8434       // Step 1: Set ISR calling convention.
8435       F->setCallingConv(llvm::CallingConv::M68k_INTR);
8436 
8437       // Step 2: Add attributes goodness.
8438       F->addFnAttr(llvm::Attribute::NoInline);
8439 
8440       // Step 3: Emit ISR vector alias.
8441       unsigned Num = attr->getNumber() / 2;
8442       llvm::GlobalAlias::create(llvm::Function::ExternalLinkage,
8443                                 "__isr_" + Twine(Num), F);
8444     }
8445   }
8446 }
8447 
8448 //===----------------------------------------------------------------------===//
8449 // AVR ABI Implementation. Documented at
8450 // https://gcc.gnu.org/wiki/avr-gcc#Calling_Convention
8451 // https://gcc.gnu.org/wiki/avr-gcc#Reduced_Tiny
8452 //===----------------------------------------------------------------------===//
8453 
8454 namespace {
8455 class AVRABIInfo : public DefaultABIInfo {
8456 private:
8457   // The total amount of registers can be used to pass parameters. It is 18 on
8458   // AVR, or 6 on AVRTiny.
8459   const unsigned ParamRegs;
8460   // The total amount of registers can be used to pass return value. It is 8 on
8461   // AVR, or 4 on AVRTiny.
8462   const unsigned RetRegs;
8463 
8464 public:
AVRABIInfo(CodeGenTypes & CGT,unsigned NPR,unsigned NRR)8465   AVRABIInfo(CodeGenTypes &CGT, unsigned NPR, unsigned NRR)
8466       : DefaultABIInfo(CGT), ParamRegs(NPR), RetRegs(NRR) {}
8467 
classifyReturnType(QualType Ty,bool & LargeRet) const8468   ABIArgInfo classifyReturnType(QualType Ty, bool &LargeRet) const {
8469     // On AVR, a return struct with size less than or equals to 8 bytes is
8470     // returned directly via registers R18-R25. On AVRTiny, a return struct
8471     // with size less than or equals to 4 bytes is returned directly via
8472     // registers R22-R25.
8473     if (isAggregateTypeForABI(Ty) &&
8474         getContext().getTypeSize(Ty) <= RetRegs * 8)
8475       return ABIArgInfo::getDirect();
8476     // A return value (struct or scalar) with larger size is returned via a
8477     // stack slot, along with a pointer as the function's implicit argument.
8478     if (getContext().getTypeSize(Ty) > RetRegs * 8) {
8479       LargeRet = true;
8480       return getNaturalAlignIndirect(Ty);
8481     }
8482     // An i8 return value should not be extended to i16, since AVR has 8-bit
8483     // registers.
8484     if (Ty->isIntegralOrEnumerationType() && getContext().getTypeSize(Ty) <= 8)
8485       return ABIArgInfo::getDirect();
8486     // Otherwise we follow the default way which is compatible.
8487     return DefaultABIInfo::classifyReturnType(Ty);
8488   }
8489 
classifyArgumentType(QualType Ty,unsigned & NumRegs) const8490   ABIArgInfo classifyArgumentType(QualType Ty, unsigned &NumRegs) const {
8491     unsigned TySize = getContext().getTypeSize(Ty);
8492 
8493     // An int8 type argument always costs two registers like an int16.
8494     if (TySize == 8 && NumRegs >= 2) {
8495       NumRegs -= 2;
8496       return ABIArgInfo::getExtend(Ty);
8497     }
8498 
8499     // If the argument size is an odd number of bytes, round up the size
8500     // to the next even number.
8501     TySize = llvm::alignTo(TySize, 16);
8502 
8503     // Any type including an array/struct type can be passed in rgisters,
8504     // if there are enough registers left.
8505     if (TySize <= NumRegs * 8) {
8506       NumRegs -= TySize / 8;
8507       return ABIArgInfo::getDirect();
8508     }
8509 
8510     // An argument is passed either completely in registers or completely in
8511     // memory. Since there are not enough registers left, current argument
8512     // and all other unprocessed arguments should be passed in memory.
8513     // However we still need to return `ABIArgInfo::getDirect()` other than
8514     // `ABIInfo::getNaturalAlignIndirect(Ty)`, otherwise an extra stack slot
8515     // will be allocated, so the stack frame layout will be incompatible with
8516     // avr-gcc.
8517     NumRegs = 0;
8518     return ABIArgInfo::getDirect();
8519   }
8520 
computeInfo(CGFunctionInfo & FI) const8521   void computeInfo(CGFunctionInfo &FI) const override {
8522     // Decide the return type.
8523     bool LargeRet = false;
8524     if (!getCXXABI().classifyReturnType(FI))
8525       FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), LargeRet);
8526 
8527     // Decide each argument type. The total number of registers can be used for
8528     // arguments depends on several factors:
8529     // 1. Arguments of varargs functions are passed on the stack. This applies
8530     //    even to the named arguments. So no register can be used.
8531     // 2. Total 18 registers can be used on avr and 6 ones on avrtiny.
8532     // 3. If the return type is a struct with too large size, two registers
8533     //    (out of 18/6) will be cost as an implicit pointer argument.
8534     unsigned NumRegs = ParamRegs;
8535     if (FI.isVariadic())
8536       NumRegs = 0;
8537     else if (LargeRet)
8538       NumRegs -= 2;
8539     for (auto &I : FI.arguments())
8540       I.info = classifyArgumentType(I.type, NumRegs);
8541   }
8542 };
8543 
8544 class AVRTargetCodeGenInfo : public TargetCodeGenInfo {
8545 public:
AVRTargetCodeGenInfo(CodeGenTypes & CGT,unsigned NPR,unsigned NRR)8546   AVRTargetCodeGenInfo(CodeGenTypes &CGT, unsigned NPR, unsigned NRR)
8547       : TargetCodeGenInfo(std::make_unique<AVRABIInfo>(CGT, NPR, NRR)) {}
8548 
getGlobalVarAddressSpace(CodeGenModule & CGM,const VarDecl * D) const8549   LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
8550                                   const VarDecl *D) const override {
8551     // Check if global/static variable is defined in address space
8552     // 1~6 (__flash, __flash1, __flash2, __flash3, __flash4, __flash5)
8553     // but not constant.
8554     if (D) {
8555       LangAS AS = D->getType().getAddressSpace();
8556       if (isTargetAddressSpace(AS) && 1 <= toTargetAddressSpace(AS) &&
8557           toTargetAddressSpace(AS) <= 6 && !D->getType().isConstQualified())
8558         CGM.getDiags().Report(D->getLocation(),
8559                               diag::err_verify_nonconst_addrspace)
8560             << "__flash*";
8561     }
8562     return TargetCodeGenInfo::getGlobalVarAddressSpace(CGM, D);
8563   }
8564 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const8565   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
8566                            CodeGen::CodeGenModule &CGM) const override {
8567     if (GV->isDeclaration())
8568       return;
8569     const auto *FD = dyn_cast_or_null<FunctionDecl>(D);
8570     if (!FD) return;
8571     auto *Fn = cast<llvm::Function>(GV);
8572 
8573     if (FD->getAttr<AVRInterruptAttr>())
8574       Fn->addFnAttr("interrupt");
8575 
8576     if (FD->getAttr<AVRSignalAttr>())
8577       Fn->addFnAttr("signal");
8578   }
8579 };
8580 }
8581 
8582 //===----------------------------------------------------------------------===//
8583 // TCE ABI Implementation (see http://tce.cs.tut.fi). Uses mostly the defaults.
8584 // Currently subclassed only to implement custom OpenCL C function attribute
8585 // handling.
8586 //===----------------------------------------------------------------------===//
8587 
8588 namespace {
8589 
8590 class TCETargetCodeGenInfo : public DefaultTargetCodeGenInfo {
8591 public:
TCETargetCodeGenInfo(CodeGenTypes & CGT)8592   TCETargetCodeGenInfo(CodeGenTypes &CGT)
8593     : DefaultTargetCodeGenInfo(CGT) {}
8594 
8595   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
8596                            CodeGen::CodeGenModule &M) const override;
8597 };
8598 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const8599 void TCETargetCodeGenInfo::setTargetAttributes(
8600     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
8601   if (GV->isDeclaration())
8602     return;
8603   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
8604   if (!FD) return;
8605 
8606   llvm::Function *F = cast<llvm::Function>(GV);
8607 
8608   if (M.getLangOpts().OpenCL) {
8609     if (FD->hasAttr<OpenCLKernelAttr>()) {
8610       // OpenCL C Kernel functions are not subject to inlining
8611       F->addFnAttr(llvm::Attribute::NoInline);
8612       const ReqdWorkGroupSizeAttr *Attr = FD->getAttr<ReqdWorkGroupSizeAttr>();
8613       if (Attr) {
8614         // Convert the reqd_work_group_size() attributes to metadata.
8615         llvm::LLVMContext &Context = F->getContext();
8616         llvm::NamedMDNode *OpenCLMetadata =
8617             M.getModule().getOrInsertNamedMetadata(
8618                 "opencl.kernel_wg_size_info");
8619 
8620         SmallVector<llvm::Metadata *, 5> Operands;
8621         Operands.push_back(llvm::ConstantAsMetadata::get(F));
8622 
8623         Operands.push_back(
8624             llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue(
8625                 M.Int32Ty, llvm::APInt(32, Attr->getXDim()))));
8626         Operands.push_back(
8627             llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue(
8628                 M.Int32Ty, llvm::APInt(32, Attr->getYDim()))));
8629         Operands.push_back(
8630             llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue(
8631                 M.Int32Ty, llvm::APInt(32, Attr->getZDim()))));
8632 
8633         // Add a boolean constant operand for "required" (true) or "hint"
8634         // (false) for implementing the work_group_size_hint attr later.
8635         // Currently always true as the hint is not yet implemented.
8636         Operands.push_back(
8637             llvm::ConstantAsMetadata::get(llvm::ConstantInt::getTrue(Context)));
8638         OpenCLMetadata->addOperand(llvm::MDNode::get(Context, Operands));
8639       }
8640     }
8641   }
8642 }
8643 
8644 }
8645 
8646 //===----------------------------------------------------------------------===//
8647 // Hexagon ABI Implementation
8648 //===----------------------------------------------------------------------===//
8649 
8650 namespace {
8651 
8652 class HexagonABIInfo : public DefaultABIInfo {
8653 public:
HexagonABIInfo(CodeGenTypes & CGT)8654   HexagonABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
8655 
8656 private:
8657   ABIArgInfo classifyReturnType(QualType RetTy) const;
8658   ABIArgInfo classifyArgumentType(QualType RetTy) const;
8659   ABIArgInfo classifyArgumentType(QualType RetTy, unsigned *RegsLeft) const;
8660 
8661   void computeInfo(CGFunctionInfo &FI) const override;
8662 
8663   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
8664                     QualType Ty) const override;
8665   Address EmitVAArgFromMemory(CodeGenFunction &CFG, Address VAListAddr,
8666                               QualType Ty) const;
8667   Address EmitVAArgForHexagon(CodeGenFunction &CFG, Address VAListAddr,
8668                               QualType Ty) const;
8669   Address EmitVAArgForHexagonLinux(CodeGenFunction &CFG, Address VAListAddr,
8670                                    QualType Ty) const;
8671 };
8672 
8673 class HexagonTargetCodeGenInfo : public TargetCodeGenInfo {
8674 public:
HexagonTargetCodeGenInfo(CodeGenTypes & CGT)8675   HexagonTargetCodeGenInfo(CodeGenTypes &CGT)
8676       : TargetCodeGenInfo(std::make_unique<HexagonABIInfo>(CGT)) {}
8677 
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const8678   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
8679     return 29;
8680   }
8681 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & GCM) const8682   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
8683                            CodeGen::CodeGenModule &GCM) const override {
8684     if (GV->isDeclaration())
8685       return;
8686     const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
8687     if (!FD)
8688       return;
8689   }
8690 };
8691 
8692 } // namespace
8693 
computeInfo(CGFunctionInfo & FI) const8694 void HexagonABIInfo::computeInfo(CGFunctionInfo &FI) const {
8695   unsigned RegsLeft = 6;
8696   if (!getCXXABI().classifyReturnType(FI))
8697     FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
8698   for (auto &I : FI.arguments())
8699     I.info = classifyArgumentType(I.type, &RegsLeft);
8700 }
8701 
HexagonAdjustRegsLeft(uint64_t Size,unsigned * RegsLeft)8702 static bool HexagonAdjustRegsLeft(uint64_t Size, unsigned *RegsLeft) {
8703   assert(Size <= 64 && "Not expecting to pass arguments larger than 64 bits"
8704                        " through registers");
8705 
8706   if (*RegsLeft == 0)
8707     return false;
8708 
8709   if (Size <= 32) {
8710     (*RegsLeft)--;
8711     return true;
8712   }
8713 
8714   if (2 <= (*RegsLeft & (~1U))) {
8715     *RegsLeft = (*RegsLeft & (~1U)) - 2;
8716     return true;
8717   }
8718 
8719   // Next available register was r5 but candidate was greater than 32-bits so it
8720   // has to go on the stack. However we still consume r5
8721   if (*RegsLeft == 1)
8722     *RegsLeft = 0;
8723 
8724   return false;
8725 }
8726 
classifyArgumentType(QualType Ty,unsigned * RegsLeft) const8727 ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty,
8728                                                 unsigned *RegsLeft) const {
8729   if (!isAggregateTypeForABI(Ty)) {
8730     // Treat an enum type as its underlying type.
8731     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
8732       Ty = EnumTy->getDecl()->getIntegerType();
8733 
8734     uint64_t Size = getContext().getTypeSize(Ty);
8735     if (Size <= 64)
8736       HexagonAdjustRegsLeft(Size, RegsLeft);
8737 
8738     if (Size > 64 && Ty->isBitIntType())
8739       return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
8740 
8741     return isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
8742                                              : ABIArgInfo::getDirect();
8743   }
8744 
8745   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
8746     return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
8747 
8748   // Ignore empty records.
8749   if (isEmptyRecord(getContext(), Ty, true))
8750     return ABIArgInfo::getIgnore();
8751 
8752   uint64_t Size = getContext().getTypeSize(Ty);
8753   unsigned Align = getContext().getTypeAlign(Ty);
8754 
8755   if (Size > 64)
8756     return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
8757 
8758   if (HexagonAdjustRegsLeft(Size, RegsLeft))
8759     Align = Size <= 32 ? 32 : 64;
8760   if (Size <= Align) {
8761     // Pass in the smallest viable integer type.
8762     if (!llvm::isPowerOf2_64(Size))
8763       Size = llvm::NextPowerOf2(Size);
8764     return ABIArgInfo::getDirect(llvm::Type::getIntNTy(getVMContext(), Size));
8765   }
8766   return DefaultABIInfo::classifyArgumentType(Ty);
8767 }
8768 
classifyReturnType(QualType RetTy) const8769 ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const {
8770   if (RetTy->isVoidType())
8771     return ABIArgInfo::getIgnore();
8772 
8773   const TargetInfo &T = CGT.getTarget();
8774   uint64_t Size = getContext().getTypeSize(RetTy);
8775 
8776   if (RetTy->getAs<VectorType>()) {
8777     // HVX vectors are returned in vector registers or register pairs.
8778     if (T.hasFeature("hvx")) {
8779       assert(T.hasFeature("hvx-length64b") || T.hasFeature("hvx-length128b"));
8780       uint64_t VecSize = T.hasFeature("hvx-length64b") ? 64*8 : 128*8;
8781       if (Size == VecSize || Size == 2*VecSize)
8782         return ABIArgInfo::getDirectInReg();
8783     }
8784     // Large vector types should be returned via memory.
8785     if (Size > 64)
8786       return getNaturalAlignIndirect(RetTy);
8787   }
8788 
8789   if (!isAggregateTypeForABI(RetTy)) {
8790     // Treat an enum type as its underlying type.
8791     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
8792       RetTy = EnumTy->getDecl()->getIntegerType();
8793 
8794     if (Size > 64 && RetTy->isBitIntType())
8795       return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
8796 
8797     return isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
8798                                                 : ABIArgInfo::getDirect();
8799   }
8800 
8801   if (isEmptyRecord(getContext(), RetTy, true))
8802     return ABIArgInfo::getIgnore();
8803 
8804   // Aggregates <= 8 bytes are returned in registers, other aggregates
8805   // are returned indirectly.
8806   if (Size <= 64) {
8807     // Return in the smallest viable integer type.
8808     if (!llvm::isPowerOf2_64(Size))
8809       Size = llvm::NextPowerOf2(Size);
8810     return ABIArgInfo::getDirect(llvm::Type::getIntNTy(getVMContext(), Size));
8811   }
8812   return getNaturalAlignIndirect(RetTy, /*ByVal=*/true);
8813 }
8814 
EmitVAArgFromMemory(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const8815 Address HexagonABIInfo::EmitVAArgFromMemory(CodeGenFunction &CGF,
8816                                             Address VAListAddr,
8817                                             QualType Ty) const {
8818   // Load the overflow area pointer.
8819   Address __overflow_area_pointer_p =
8820       CGF.Builder.CreateStructGEP(VAListAddr, 2, "__overflow_area_pointer_p");
8821   llvm::Value *__overflow_area_pointer = CGF.Builder.CreateLoad(
8822       __overflow_area_pointer_p, "__overflow_area_pointer");
8823 
8824   uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8;
8825   if (Align > 4) {
8826     // Alignment should be a power of 2.
8827     assert((Align & (Align - 1)) == 0 && "Alignment is not power of 2!");
8828 
8829     // overflow_arg_area = (overflow_arg_area + align - 1) & -align;
8830     llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int64Ty, Align - 1);
8831 
8832     // Add offset to the current pointer to access the argument.
8833     __overflow_area_pointer =
8834         CGF.Builder.CreateGEP(CGF.Int8Ty, __overflow_area_pointer, Offset);
8835     llvm::Value *AsInt =
8836         CGF.Builder.CreatePtrToInt(__overflow_area_pointer, CGF.Int32Ty);
8837 
8838     // Create a mask which should be "AND"ed
8839     // with (overflow_arg_area + align - 1)
8840     llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int32Ty, -(int)Align);
8841     __overflow_area_pointer = CGF.Builder.CreateIntToPtr(
8842         CGF.Builder.CreateAnd(AsInt, Mask), __overflow_area_pointer->getType(),
8843         "__overflow_area_pointer.align");
8844   }
8845 
8846   // Get the type of the argument from memory and bitcast
8847   // overflow area pointer to the argument type.
8848   llvm::Type *PTy = CGF.ConvertTypeForMem(Ty);
8849   Address AddrTyped = CGF.Builder.CreateElementBitCast(
8850       Address(__overflow_area_pointer, CGF.Int8Ty,
8851               CharUnits::fromQuantity(Align)),
8852       PTy);
8853 
8854   // Round up to the minimum stack alignment for varargs which is 4 bytes.
8855   uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4);
8856 
8857   __overflow_area_pointer = CGF.Builder.CreateGEP(
8858       CGF.Int8Ty, __overflow_area_pointer,
8859       llvm::ConstantInt::get(CGF.Int32Ty, Offset),
8860       "__overflow_area_pointer.next");
8861   CGF.Builder.CreateStore(__overflow_area_pointer, __overflow_area_pointer_p);
8862 
8863   return AddrTyped;
8864 }
8865 
EmitVAArgForHexagon(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const8866 Address HexagonABIInfo::EmitVAArgForHexagon(CodeGenFunction &CGF,
8867                                             Address VAListAddr,
8868                                             QualType Ty) const {
8869   // FIXME: Need to handle alignment
8870   llvm::Type *BP = CGF.Int8PtrTy;
8871   CGBuilderTy &Builder = CGF.Builder;
8872   Address VAListAddrAsBPP = Builder.CreateElementBitCast(VAListAddr, BP, "ap");
8873   llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
8874   // Handle address alignment for type alignment > 32 bits
8875   uint64_t TyAlign = CGF.getContext().getTypeAlign(Ty) / 8;
8876   if (TyAlign > 4) {
8877     assert((TyAlign & (TyAlign - 1)) == 0 && "Alignment is not power of 2!");
8878     llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int32Ty);
8879     AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt32(TyAlign - 1));
8880     AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt32(~(TyAlign - 1)));
8881     Addr = Builder.CreateIntToPtr(AddrAsInt, BP);
8882   }
8883   Address AddrTyped = Builder.CreateElementBitCast(
8884       Address(Addr, CGF.Int8Ty, CharUnits::fromQuantity(TyAlign)),
8885       CGF.ConvertType(Ty));
8886 
8887   uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4);
8888   llvm::Value *NextAddr = Builder.CreateGEP(
8889       CGF.Int8Ty, Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), "ap.next");
8890   Builder.CreateStore(NextAddr, VAListAddrAsBPP);
8891 
8892   return AddrTyped;
8893 }
8894 
EmitVAArgForHexagonLinux(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const8895 Address HexagonABIInfo::EmitVAArgForHexagonLinux(CodeGenFunction &CGF,
8896                                                  Address VAListAddr,
8897                                                  QualType Ty) const {
8898   int ArgSize = CGF.getContext().getTypeSize(Ty) / 8;
8899 
8900   if (ArgSize > 8)
8901     return EmitVAArgFromMemory(CGF, VAListAddr, Ty);
8902 
8903   // Here we have check if the argument is in register area or
8904   // in overflow area.
8905   // If the saved register area pointer + argsize rounded up to alignment >
8906   // saved register area end pointer, argument is in overflow area.
8907   unsigned RegsLeft = 6;
8908   Ty = CGF.getContext().getCanonicalType(Ty);
8909   (void)classifyArgumentType(Ty, &RegsLeft);
8910 
8911   llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg");
8912   llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
8913   llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");
8914   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
8915 
8916   // Get rounded size of the argument.GCC does not allow vararg of
8917   // size < 4 bytes. We follow the same logic here.
8918   ArgSize = (CGF.getContext().getTypeSize(Ty) <= 32) ? 4 : 8;
8919   int ArgAlign = (CGF.getContext().getTypeSize(Ty) <= 32) ? 4 : 8;
8920 
8921   // Argument may be in saved register area
8922   CGF.EmitBlock(MaybeRegBlock);
8923 
8924   // Load the current saved register area pointer.
8925   Address __current_saved_reg_area_pointer_p = CGF.Builder.CreateStructGEP(
8926       VAListAddr, 0, "__current_saved_reg_area_pointer_p");
8927   llvm::Value *__current_saved_reg_area_pointer = CGF.Builder.CreateLoad(
8928       __current_saved_reg_area_pointer_p, "__current_saved_reg_area_pointer");
8929 
8930   // Load the saved register area end pointer.
8931   Address __saved_reg_area_end_pointer_p = CGF.Builder.CreateStructGEP(
8932       VAListAddr, 1, "__saved_reg_area_end_pointer_p");
8933   llvm::Value *__saved_reg_area_end_pointer = CGF.Builder.CreateLoad(
8934       __saved_reg_area_end_pointer_p, "__saved_reg_area_end_pointer");
8935 
8936   // If the size of argument is > 4 bytes, check if the stack
8937   // location is aligned to 8 bytes
8938   if (ArgAlign > 4) {
8939 
8940     llvm::Value *__current_saved_reg_area_pointer_int =
8941         CGF.Builder.CreatePtrToInt(__current_saved_reg_area_pointer,
8942                                    CGF.Int32Ty);
8943 
8944     __current_saved_reg_area_pointer_int = CGF.Builder.CreateAdd(
8945         __current_saved_reg_area_pointer_int,
8946         llvm::ConstantInt::get(CGF.Int32Ty, (ArgAlign - 1)),
8947         "align_current_saved_reg_area_pointer");
8948 
8949     __current_saved_reg_area_pointer_int =
8950         CGF.Builder.CreateAnd(__current_saved_reg_area_pointer_int,
8951                               llvm::ConstantInt::get(CGF.Int32Ty, -ArgAlign),
8952                               "align_current_saved_reg_area_pointer");
8953 
8954     __current_saved_reg_area_pointer =
8955         CGF.Builder.CreateIntToPtr(__current_saved_reg_area_pointer_int,
8956                                    __current_saved_reg_area_pointer->getType(),
8957                                    "align_current_saved_reg_area_pointer");
8958   }
8959 
8960   llvm::Value *__new_saved_reg_area_pointer =
8961       CGF.Builder.CreateGEP(CGF.Int8Ty, __current_saved_reg_area_pointer,
8962                             llvm::ConstantInt::get(CGF.Int32Ty, ArgSize),
8963                             "__new_saved_reg_area_pointer");
8964 
8965   llvm::Value *UsingStack = nullptr;
8966   UsingStack = CGF.Builder.CreateICmpSGT(__new_saved_reg_area_pointer,
8967                                          __saved_reg_area_end_pointer);
8968 
8969   CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, InRegBlock);
8970 
8971   // Argument in saved register area
8972   // Implement the block where argument is in register saved area
8973   CGF.EmitBlock(InRegBlock);
8974 
8975   llvm::Type *PTy = CGF.ConvertType(Ty);
8976   llvm::Value *__saved_reg_area_p = CGF.Builder.CreateBitCast(
8977       __current_saved_reg_area_pointer, llvm::PointerType::getUnqual(PTy));
8978 
8979   CGF.Builder.CreateStore(__new_saved_reg_area_pointer,
8980                           __current_saved_reg_area_pointer_p);
8981 
8982   CGF.EmitBranch(ContBlock);
8983 
8984   // Argument in overflow area
8985   // Implement the block where the argument is in overflow area.
8986   CGF.EmitBlock(OnStackBlock);
8987 
8988   // Load the overflow area pointer
8989   Address __overflow_area_pointer_p =
8990       CGF.Builder.CreateStructGEP(VAListAddr, 2, "__overflow_area_pointer_p");
8991   llvm::Value *__overflow_area_pointer = CGF.Builder.CreateLoad(
8992       __overflow_area_pointer_p, "__overflow_area_pointer");
8993 
8994   // Align the overflow area pointer according to the alignment of the argument
8995   if (ArgAlign > 4) {
8996     llvm::Value *__overflow_area_pointer_int =
8997         CGF.Builder.CreatePtrToInt(__overflow_area_pointer, CGF.Int32Ty);
8998 
8999     __overflow_area_pointer_int =
9000         CGF.Builder.CreateAdd(__overflow_area_pointer_int,
9001                               llvm::ConstantInt::get(CGF.Int32Ty, ArgAlign - 1),
9002                               "align_overflow_area_pointer");
9003 
9004     __overflow_area_pointer_int =
9005         CGF.Builder.CreateAnd(__overflow_area_pointer_int,
9006                               llvm::ConstantInt::get(CGF.Int32Ty, -ArgAlign),
9007                               "align_overflow_area_pointer");
9008 
9009     __overflow_area_pointer = CGF.Builder.CreateIntToPtr(
9010         __overflow_area_pointer_int, __overflow_area_pointer->getType(),
9011         "align_overflow_area_pointer");
9012   }
9013 
9014   // Get the pointer for next argument in overflow area and store it
9015   // to overflow area pointer.
9016   llvm::Value *__new_overflow_area_pointer = CGF.Builder.CreateGEP(
9017       CGF.Int8Ty, __overflow_area_pointer,
9018       llvm::ConstantInt::get(CGF.Int32Ty, ArgSize),
9019       "__overflow_area_pointer.next");
9020 
9021   CGF.Builder.CreateStore(__new_overflow_area_pointer,
9022                           __overflow_area_pointer_p);
9023 
9024   CGF.Builder.CreateStore(__new_overflow_area_pointer,
9025                           __current_saved_reg_area_pointer_p);
9026 
9027   // Bitcast the overflow area pointer to the type of argument.
9028   llvm::Type *OverflowPTy = CGF.ConvertTypeForMem(Ty);
9029   llvm::Value *__overflow_area_p = CGF.Builder.CreateBitCast(
9030       __overflow_area_pointer, llvm::PointerType::getUnqual(OverflowPTy));
9031 
9032   CGF.EmitBranch(ContBlock);
9033 
9034   // Get the correct pointer to load the variable argument
9035   // Implement the ContBlock
9036   CGF.EmitBlock(ContBlock);
9037 
9038   llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty);
9039   llvm::Type *MemPTy = llvm::PointerType::getUnqual(MemTy);
9040   llvm::PHINode *ArgAddr = CGF.Builder.CreatePHI(MemPTy, 2, "vaarg.addr");
9041   ArgAddr->addIncoming(__saved_reg_area_p, InRegBlock);
9042   ArgAddr->addIncoming(__overflow_area_p, OnStackBlock);
9043 
9044   return Address(ArgAddr, MemTy, CharUnits::fromQuantity(ArgAlign));
9045 }
9046 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const9047 Address HexagonABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9048                                   QualType Ty) const {
9049 
9050   if (getTarget().getTriple().isMusl())
9051     return EmitVAArgForHexagonLinux(CGF, VAListAddr, Ty);
9052 
9053   return EmitVAArgForHexagon(CGF, VAListAddr, Ty);
9054 }
9055 
9056 //===----------------------------------------------------------------------===//
9057 // Lanai ABI Implementation
9058 //===----------------------------------------------------------------------===//
9059 
9060 namespace {
9061 class LanaiABIInfo : public DefaultABIInfo {
9062 public:
LanaiABIInfo(CodeGen::CodeGenTypes & CGT)9063   LanaiABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
9064 
9065   bool shouldUseInReg(QualType Ty, CCState &State) const;
9066 
computeInfo(CGFunctionInfo & FI) const9067   void computeInfo(CGFunctionInfo &FI) const override {
9068     CCState State(FI);
9069     // Lanai uses 4 registers to pass arguments unless the function has the
9070     // regparm attribute set.
9071     if (FI.getHasRegParm()) {
9072       State.FreeRegs = FI.getRegParm();
9073     } else {
9074       State.FreeRegs = 4;
9075     }
9076 
9077     if (!getCXXABI().classifyReturnType(FI))
9078       FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
9079     for (auto &I : FI.arguments())
9080       I.info = classifyArgumentType(I.type, State);
9081   }
9082 
9083   ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
9084   ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
9085 };
9086 } // end anonymous namespace
9087 
shouldUseInReg(QualType Ty,CCState & State) const9088 bool LanaiABIInfo::shouldUseInReg(QualType Ty, CCState &State) const {
9089   unsigned Size = getContext().getTypeSize(Ty);
9090   unsigned SizeInRegs = llvm::alignTo(Size, 32U) / 32U;
9091 
9092   if (SizeInRegs == 0)
9093     return false;
9094 
9095   if (SizeInRegs > State.FreeRegs) {
9096     State.FreeRegs = 0;
9097     return false;
9098   }
9099 
9100   State.FreeRegs -= SizeInRegs;
9101 
9102   return true;
9103 }
9104 
getIndirectResult(QualType Ty,bool ByVal,CCState & State) const9105 ABIArgInfo LanaiABIInfo::getIndirectResult(QualType Ty, bool ByVal,
9106                                            CCState &State) const {
9107   if (!ByVal) {
9108     if (State.FreeRegs) {
9109       --State.FreeRegs; // Non-byval indirects just use one pointer.
9110       return getNaturalAlignIndirectInReg(Ty);
9111     }
9112     return getNaturalAlignIndirect(Ty, false);
9113   }
9114 
9115   // Compute the byval alignment.
9116   const unsigned MinABIStackAlignInBytes = 4;
9117   unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
9118   return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true,
9119                                  /*Realign=*/TypeAlign >
9120                                      MinABIStackAlignInBytes);
9121 }
9122 
classifyArgumentType(QualType Ty,CCState & State) const9123 ABIArgInfo LanaiABIInfo::classifyArgumentType(QualType Ty,
9124                                               CCState &State) const {
9125   // Check with the C++ ABI first.
9126   const RecordType *RT = Ty->getAs<RecordType>();
9127   if (RT) {
9128     CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
9129     if (RAA == CGCXXABI::RAA_Indirect) {
9130       return getIndirectResult(Ty, /*ByVal=*/false, State);
9131     } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
9132       return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
9133     }
9134   }
9135 
9136   if (isAggregateTypeForABI(Ty)) {
9137     // Structures with flexible arrays are always indirect.
9138     if (RT && RT->getDecl()->hasFlexibleArrayMember())
9139       return getIndirectResult(Ty, /*ByVal=*/true, State);
9140 
9141     // Ignore empty structs/unions.
9142     if (isEmptyRecord(getContext(), Ty, true))
9143       return ABIArgInfo::getIgnore();
9144 
9145     llvm::LLVMContext &LLVMContext = getVMContext();
9146     unsigned SizeInRegs = (getContext().getTypeSize(Ty) + 31) / 32;
9147     if (SizeInRegs <= State.FreeRegs) {
9148       llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
9149       SmallVector<llvm::Type *, 3> Elements(SizeInRegs, Int32);
9150       llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
9151       State.FreeRegs -= SizeInRegs;
9152       return ABIArgInfo::getDirectInReg(Result);
9153     } else {
9154       State.FreeRegs = 0;
9155     }
9156     return getIndirectResult(Ty, true, State);
9157   }
9158 
9159   // Treat an enum type as its underlying type.
9160   if (const auto *EnumTy = Ty->getAs<EnumType>())
9161     Ty = EnumTy->getDecl()->getIntegerType();
9162 
9163   bool InReg = shouldUseInReg(Ty, State);
9164 
9165   // Don't pass >64 bit integers in registers.
9166   if (const auto *EIT = Ty->getAs<BitIntType>())
9167     if (EIT->getNumBits() > 64)
9168       return getIndirectResult(Ty, /*ByVal=*/true, State);
9169 
9170   if (isPromotableIntegerTypeForABI(Ty)) {
9171     if (InReg)
9172       return ABIArgInfo::getDirectInReg();
9173     return ABIArgInfo::getExtend(Ty);
9174   }
9175   if (InReg)
9176     return ABIArgInfo::getDirectInReg();
9177   return ABIArgInfo::getDirect();
9178 }
9179 
9180 namespace {
9181 class LanaiTargetCodeGenInfo : public TargetCodeGenInfo {
9182 public:
LanaiTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT)9183   LanaiTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
9184       : TargetCodeGenInfo(std::make_unique<LanaiABIInfo>(CGT)) {}
9185 };
9186 }
9187 
9188 //===----------------------------------------------------------------------===//
9189 // AMDGPU ABI Implementation
9190 //===----------------------------------------------------------------------===//
9191 
9192 namespace {
9193 
9194 class AMDGPUABIInfo final : public DefaultABIInfo {
9195 private:
9196   static const unsigned MaxNumRegsForArgsRet = 16;
9197 
9198   unsigned numRegsForType(QualType Ty) const;
9199 
9200   bool isHomogeneousAggregateBaseType(QualType Ty) const override;
9201   bool isHomogeneousAggregateSmallEnough(const Type *Base,
9202                                          uint64_t Members) const override;
9203 
9204   // Coerce HIP scalar pointer arguments from generic pointers to global ones.
coerceKernelArgumentType(llvm::Type * Ty,unsigned FromAS,unsigned ToAS) const9205   llvm::Type *coerceKernelArgumentType(llvm::Type *Ty, unsigned FromAS,
9206                                        unsigned ToAS) const {
9207     // Single value types.
9208     auto *PtrTy = llvm::dyn_cast<llvm::PointerType>(Ty);
9209     if (PtrTy && PtrTy->getAddressSpace() == FromAS)
9210       return llvm::PointerType::getWithSamePointeeType(PtrTy, ToAS);
9211     return Ty;
9212   }
9213 
9214 public:
AMDGPUABIInfo(CodeGen::CodeGenTypes & CGT)9215   explicit AMDGPUABIInfo(CodeGen::CodeGenTypes &CGT) :
9216     DefaultABIInfo(CGT) {}
9217 
9218   ABIArgInfo classifyReturnType(QualType RetTy) const;
9219   ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
9220   ABIArgInfo classifyArgumentType(QualType Ty, unsigned &NumRegsLeft) const;
9221 
9222   void computeInfo(CGFunctionInfo &FI) const override;
9223   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9224                     QualType Ty) const override;
9225 };
9226 
isHomogeneousAggregateBaseType(QualType Ty) const9227 bool AMDGPUABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
9228   return true;
9229 }
9230 
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const9231 bool AMDGPUABIInfo::isHomogeneousAggregateSmallEnough(
9232   const Type *Base, uint64_t Members) const {
9233   uint32_t NumRegs = (getContext().getTypeSize(Base) + 31) / 32;
9234 
9235   // Homogeneous Aggregates may occupy at most 16 registers.
9236   return Members * NumRegs <= MaxNumRegsForArgsRet;
9237 }
9238 
9239 /// Estimate number of registers the type will use when passed in registers.
numRegsForType(QualType Ty) const9240 unsigned AMDGPUABIInfo::numRegsForType(QualType Ty) const {
9241   unsigned NumRegs = 0;
9242 
9243   if (const VectorType *VT = Ty->getAs<VectorType>()) {
9244     // Compute from the number of elements. The reported size is based on the
9245     // in-memory size, which includes the padding 4th element for 3-vectors.
9246     QualType EltTy = VT->getElementType();
9247     unsigned EltSize = getContext().getTypeSize(EltTy);
9248 
9249     // 16-bit element vectors should be passed as packed.
9250     if (EltSize == 16)
9251       return (VT->getNumElements() + 1) / 2;
9252 
9253     unsigned EltNumRegs = (EltSize + 31) / 32;
9254     return EltNumRegs * VT->getNumElements();
9255   }
9256 
9257   if (const RecordType *RT = Ty->getAs<RecordType>()) {
9258     const RecordDecl *RD = RT->getDecl();
9259     assert(!RD->hasFlexibleArrayMember());
9260 
9261     for (const FieldDecl *Field : RD->fields()) {
9262       QualType FieldTy = Field->getType();
9263       NumRegs += numRegsForType(FieldTy);
9264     }
9265 
9266     return NumRegs;
9267   }
9268 
9269   return (getContext().getTypeSize(Ty) + 31) / 32;
9270 }
9271 
computeInfo(CGFunctionInfo & FI) const9272 void AMDGPUABIInfo::computeInfo(CGFunctionInfo &FI) const {
9273   llvm::CallingConv::ID CC = FI.getCallingConvention();
9274 
9275   if (!getCXXABI().classifyReturnType(FI))
9276     FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
9277 
9278   unsigned NumRegsLeft = MaxNumRegsForArgsRet;
9279   for (auto &Arg : FI.arguments()) {
9280     if (CC == llvm::CallingConv::AMDGPU_KERNEL) {
9281       Arg.info = classifyKernelArgumentType(Arg.type);
9282     } else {
9283       Arg.info = classifyArgumentType(Arg.type, NumRegsLeft);
9284     }
9285   }
9286 }
9287 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const9288 Address AMDGPUABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9289                                  QualType Ty) const {
9290   llvm_unreachable("AMDGPU does not support varargs");
9291 }
9292 
classifyReturnType(QualType RetTy) const9293 ABIArgInfo AMDGPUABIInfo::classifyReturnType(QualType RetTy) const {
9294   if (isAggregateTypeForABI(RetTy)) {
9295     // Records with non-trivial destructors/copy-constructors should not be
9296     // returned by value.
9297     if (!getRecordArgABI(RetTy, getCXXABI())) {
9298       // Ignore empty structs/unions.
9299       if (isEmptyRecord(getContext(), RetTy, true))
9300         return ABIArgInfo::getIgnore();
9301 
9302       // Lower single-element structs to just return a regular value.
9303       if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
9304         return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
9305 
9306       if (const RecordType *RT = RetTy->getAs<RecordType>()) {
9307         const RecordDecl *RD = RT->getDecl();
9308         if (RD->hasFlexibleArrayMember())
9309           return DefaultABIInfo::classifyReturnType(RetTy);
9310       }
9311 
9312       // Pack aggregates <= 4 bytes into single VGPR or pair.
9313       uint64_t Size = getContext().getTypeSize(RetTy);
9314       if (Size <= 16)
9315         return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
9316 
9317       if (Size <= 32)
9318         return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
9319 
9320       if (Size <= 64) {
9321         llvm::Type *I32Ty = llvm::Type::getInt32Ty(getVMContext());
9322         return ABIArgInfo::getDirect(llvm::ArrayType::get(I32Ty, 2));
9323       }
9324 
9325       if (numRegsForType(RetTy) <= MaxNumRegsForArgsRet)
9326         return ABIArgInfo::getDirect();
9327     }
9328   }
9329 
9330   // Otherwise just do the default thing.
9331   return DefaultABIInfo::classifyReturnType(RetTy);
9332 }
9333 
9334 /// For kernels all parameters are really passed in a special buffer. It doesn't
9335 /// make sense to pass anything byval, so everything must be direct.
classifyKernelArgumentType(QualType Ty) const9336 ABIArgInfo AMDGPUABIInfo::classifyKernelArgumentType(QualType Ty) const {
9337   Ty = useFirstFieldIfTransparentUnion(Ty);
9338 
9339   // TODO: Can we omit empty structs?
9340 
9341   if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
9342     Ty = QualType(SeltTy, 0);
9343 
9344   llvm::Type *OrigLTy = CGT.ConvertType(Ty);
9345   llvm::Type *LTy = OrigLTy;
9346   if (getContext().getLangOpts().HIP) {
9347     LTy = coerceKernelArgumentType(
9348         OrigLTy, /*FromAS=*/getContext().getTargetAddressSpace(LangAS::Default),
9349         /*ToAS=*/getContext().getTargetAddressSpace(LangAS::cuda_device));
9350   }
9351 
9352   // FIXME: Should also use this for OpenCL, but it requires addressing the
9353   // problem of kernels being called.
9354   //
9355   // FIXME: This doesn't apply the optimization of coercing pointers in structs
9356   // to global address space when using byref. This would require implementing a
9357   // new kind of coercion of the in-memory type when for indirect arguments.
9358   if (!getContext().getLangOpts().OpenCL && LTy == OrigLTy &&
9359       isAggregateTypeForABI(Ty)) {
9360     return ABIArgInfo::getIndirectAliased(
9361         getContext().getTypeAlignInChars(Ty),
9362         getContext().getTargetAddressSpace(LangAS::opencl_constant),
9363         false /*Realign*/, nullptr /*Padding*/);
9364   }
9365 
9366   // If we set CanBeFlattened to true, CodeGen will expand the struct to its
9367   // individual elements, which confuses the Clover OpenCL backend; therefore we
9368   // have to set it to false here. Other args of getDirect() are just defaults.
9369   return ABIArgInfo::getDirect(LTy, 0, nullptr, false);
9370 }
9371 
classifyArgumentType(QualType Ty,unsigned & NumRegsLeft) const9372 ABIArgInfo AMDGPUABIInfo::classifyArgumentType(QualType Ty,
9373                                                unsigned &NumRegsLeft) const {
9374   assert(NumRegsLeft <= MaxNumRegsForArgsRet && "register estimate underflow");
9375 
9376   Ty = useFirstFieldIfTransparentUnion(Ty);
9377 
9378   if (isAggregateTypeForABI(Ty)) {
9379     // Records with non-trivial destructors/copy-constructors should not be
9380     // passed by value.
9381     if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
9382       return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
9383 
9384     // Ignore empty structs/unions.
9385     if (isEmptyRecord(getContext(), Ty, true))
9386       return ABIArgInfo::getIgnore();
9387 
9388     // Lower single-element structs to just pass a regular value. TODO: We
9389     // could do reasonable-size multiple-element structs too, using getExpand(),
9390     // though watch out for things like bitfields.
9391     if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
9392       return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
9393 
9394     if (const RecordType *RT = Ty->getAs<RecordType>()) {
9395       const RecordDecl *RD = RT->getDecl();
9396       if (RD->hasFlexibleArrayMember())
9397         return DefaultABIInfo::classifyArgumentType(Ty);
9398     }
9399 
9400     // Pack aggregates <= 8 bytes into single VGPR or pair.
9401     uint64_t Size = getContext().getTypeSize(Ty);
9402     if (Size <= 64) {
9403       unsigned NumRegs = (Size + 31) / 32;
9404       NumRegsLeft -= std::min(NumRegsLeft, NumRegs);
9405 
9406       if (Size <= 16)
9407         return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
9408 
9409       if (Size <= 32)
9410         return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
9411 
9412       // XXX: Should this be i64 instead, and should the limit increase?
9413       llvm::Type *I32Ty = llvm::Type::getInt32Ty(getVMContext());
9414       return ABIArgInfo::getDirect(llvm::ArrayType::get(I32Ty, 2));
9415     }
9416 
9417     if (NumRegsLeft > 0) {
9418       unsigned NumRegs = numRegsForType(Ty);
9419       if (NumRegsLeft >= NumRegs) {
9420         NumRegsLeft -= NumRegs;
9421         return ABIArgInfo::getDirect();
9422       }
9423     }
9424   }
9425 
9426   // Otherwise just do the default thing.
9427   ABIArgInfo ArgInfo = DefaultABIInfo::classifyArgumentType(Ty);
9428   if (!ArgInfo.isIndirect()) {
9429     unsigned NumRegs = numRegsForType(Ty);
9430     NumRegsLeft -= std::min(NumRegs, NumRegsLeft);
9431   }
9432 
9433   return ArgInfo;
9434 }
9435 
9436 class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo {
9437 public:
AMDGPUTargetCodeGenInfo(CodeGenTypes & CGT)9438   AMDGPUTargetCodeGenInfo(CodeGenTypes &CGT)
9439       : TargetCodeGenInfo(std::make_unique<AMDGPUABIInfo>(CGT)) {}
9440 
9441   void setFunctionDeclAttributes(const FunctionDecl *FD, llvm::Function *F,
9442                                  CodeGenModule &CGM) const;
9443 
9444   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
9445                            CodeGen::CodeGenModule &M) const override;
9446   unsigned getOpenCLKernelCallingConv() const override;
9447 
9448   llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM,
9449       llvm::PointerType *T, QualType QT) const override;
9450 
getASTAllocaAddressSpace() const9451   LangAS getASTAllocaAddressSpace() const override {
9452     return getLangASFromTargetAS(
9453         getABIInfo().getDataLayout().getAllocaAddrSpace());
9454   }
9455   LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
9456                                   const VarDecl *D) const override;
9457   llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts,
9458                                          SyncScope Scope,
9459                                          llvm::AtomicOrdering Ordering,
9460                                          llvm::LLVMContext &Ctx) const override;
9461   llvm::Function *
9462   createEnqueuedBlockKernel(CodeGenFunction &CGF,
9463                             llvm::Function *BlockInvokeFunc,
9464                             llvm::Type *BlockTy) const override;
9465   bool shouldEmitStaticExternCAliases() const override;
9466   void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
9467 };
9468 }
9469 
requiresAMDGPUProtectedVisibility(const Decl * D,llvm::GlobalValue * GV)9470 static bool requiresAMDGPUProtectedVisibility(const Decl *D,
9471                                               llvm::GlobalValue *GV) {
9472   if (GV->getVisibility() != llvm::GlobalValue::HiddenVisibility)
9473     return false;
9474 
9475   return D->hasAttr<OpenCLKernelAttr>() ||
9476          (isa<FunctionDecl>(D) && D->hasAttr<CUDAGlobalAttr>()) ||
9477          (isa<VarDecl>(D) &&
9478           (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
9479            cast<VarDecl>(D)->getType()->isCUDADeviceBuiltinSurfaceType() ||
9480            cast<VarDecl>(D)->getType()->isCUDADeviceBuiltinTextureType()));
9481 }
9482 
setFunctionDeclAttributes(const FunctionDecl * FD,llvm::Function * F,CodeGenModule & M) const9483 void AMDGPUTargetCodeGenInfo::setFunctionDeclAttributes(
9484     const FunctionDecl *FD, llvm::Function *F, CodeGenModule &M) const {
9485   const auto *ReqdWGS =
9486       M.getLangOpts().OpenCL ? FD->getAttr<ReqdWorkGroupSizeAttr>() : nullptr;
9487   const bool IsOpenCLKernel =
9488       M.getLangOpts().OpenCL && FD->hasAttr<OpenCLKernelAttr>();
9489   const bool IsHIPKernel = M.getLangOpts().HIP && FD->hasAttr<CUDAGlobalAttr>();
9490 
9491   const auto *FlatWGS = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>();
9492   if (ReqdWGS || FlatWGS) {
9493     unsigned Min = 0;
9494     unsigned Max = 0;
9495     if (FlatWGS) {
9496       Min = FlatWGS->getMin()
9497                 ->EvaluateKnownConstInt(M.getContext())
9498                 .getExtValue();
9499       Max = FlatWGS->getMax()
9500                 ->EvaluateKnownConstInt(M.getContext())
9501                 .getExtValue();
9502     }
9503     if (ReqdWGS && Min == 0 && Max == 0)
9504       Min = Max = ReqdWGS->getXDim() * ReqdWGS->getYDim() * ReqdWGS->getZDim();
9505 
9506     if (Min != 0) {
9507       assert(Min <= Max && "Min must be less than or equal Max");
9508 
9509       std::string AttrVal = llvm::utostr(Min) + "," + llvm::utostr(Max);
9510       F->addFnAttr("amdgpu-flat-work-group-size", AttrVal);
9511     } else
9512       assert(Max == 0 && "Max must be zero");
9513   } else if (IsOpenCLKernel || IsHIPKernel) {
9514     // By default, restrict the maximum size to a value specified by
9515     // --gpu-max-threads-per-block=n or its default value for HIP.
9516     const unsigned OpenCLDefaultMaxWorkGroupSize = 256;
9517     const unsigned DefaultMaxWorkGroupSize =
9518         IsOpenCLKernel ? OpenCLDefaultMaxWorkGroupSize
9519                        : M.getLangOpts().GPUMaxThreadsPerBlock;
9520     std::string AttrVal =
9521         std::string("1,") + llvm::utostr(DefaultMaxWorkGroupSize);
9522     F->addFnAttr("amdgpu-flat-work-group-size", AttrVal);
9523   }
9524 
9525   if (const auto *Attr = FD->getAttr<AMDGPUWavesPerEUAttr>()) {
9526     unsigned Min =
9527         Attr->getMin()->EvaluateKnownConstInt(M.getContext()).getExtValue();
9528     unsigned Max = Attr->getMax() ? Attr->getMax()
9529                                         ->EvaluateKnownConstInt(M.getContext())
9530                                         .getExtValue()
9531                                   : 0;
9532 
9533     if (Min != 0) {
9534       assert((Max == 0 || Min <= Max) && "Min must be less than or equal Max");
9535 
9536       std::string AttrVal = llvm::utostr(Min);
9537       if (Max != 0)
9538         AttrVal = AttrVal + "," + llvm::utostr(Max);
9539       F->addFnAttr("amdgpu-waves-per-eu", AttrVal);
9540     } else
9541       assert(Max == 0 && "Max must be zero");
9542   }
9543 
9544   if (const auto *Attr = FD->getAttr<AMDGPUNumSGPRAttr>()) {
9545     unsigned NumSGPR = Attr->getNumSGPR();
9546 
9547     if (NumSGPR != 0)
9548       F->addFnAttr("amdgpu-num-sgpr", llvm::utostr(NumSGPR));
9549   }
9550 
9551   if (const auto *Attr = FD->getAttr<AMDGPUNumVGPRAttr>()) {
9552     uint32_t NumVGPR = Attr->getNumVGPR();
9553 
9554     if (NumVGPR != 0)
9555       F->addFnAttr("amdgpu-num-vgpr", llvm::utostr(NumVGPR));
9556   }
9557 }
9558 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const9559 void AMDGPUTargetCodeGenInfo::setTargetAttributes(
9560     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
9561   if (requiresAMDGPUProtectedVisibility(D, GV)) {
9562     GV->setVisibility(llvm::GlobalValue::ProtectedVisibility);
9563     GV->setDSOLocal(true);
9564   }
9565 
9566   if (GV->isDeclaration())
9567     return;
9568 
9569   llvm::Function *F = dyn_cast<llvm::Function>(GV);
9570   if (!F)
9571     return;
9572 
9573   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
9574   if (FD)
9575     setFunctionDeclAttributes(FD, F, M);
9576 
9577   const bool IsHIPKernel =
9578       M.getLangOpts().HIP && FD && FD->hasAttr<CUDAGlobalAttr>();
9579   const bool IsOpenMPkernel =
9580       M.getLangOpts().OpenMPIsDevice &&
9581       (F->getCallingConv() == llvm::CallingConv::AMDGPU_KERNEL);
9582 
9583   // TODO: This should be moved to language specific attributes instead.
9584   if (IsHIPKernel || IsOpenMPkernel)
9585     F->addFnAttr("uniform-work-group-size", "true");
9586 
9587   if (M.getContext().getTargetInfo().allowAMDGPUUnsafeFPAtomics())
9588     F->addFnAttr("amdgpu-unsafe-fp-atomics", "true");
9589 
9590   if (!getABIInfo().getCodeGenOpts().EmitIEEENaNCompliantInsts)
9591     F->addFnAttr("amdgpu-ieee", "false");
9592 }
9593 
getOpenCLKernelCallingConv() const9594 unsigned AMDGPUTargetCodeGenInfo::getOpenCLKernelCallingConv() const {
9595   return llvm::CallingConv::AMDGPU_KERNEL;
9596 }
9597 
9598 // Currently LLVM assumes null pointers always have value 0,
9599 // which results in incorrectly transformed IR. Therefore, instead of
9600 // emitting null pointers in private and local address spaces, a null
9601 // pointer in generic address space is emitted which is casted to a
9602 // pointer in local or private address space.
getNullPointer(const CodeGen::CodeGenModule & CGM,llvm::PointerType * PT,QualType QT) const9603 llvm::Constant *AMDGPUTargetCodeGenInfo::getNullPointer(
9604     const CodeGen::CodeGenModule &CGM, llvm::PointerType *PT,
9605     QualType QT) const {
9606   if (CGM.getContext().getTargetNullPointerValue(QT) == 0)
9607     return llvm::ConstantPointerNull::get(PT);
9608 
9609   auto &Ctx = CGM.getContext();
9610   auto NPT = llvm::PointerType::getWithSamePointeeType(
9611       PT, Ctx.getTargetAddressSpace(LangAS::opencl_generic));
9612   return llvm::ConstantExpr::getAddrSpaceCast(
9613       llvm::ConstantPointerNull::get(NPT), PT);
9614 }
9615 
9616 LangAS
getGlobalVarAddressSpace(CodeGenModule & CGM,const VarDecl * D) const9617 AMDGPUTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
9618                                                   const VarDecl *D) const {
9619   assert(!CGM.getLangOpts().OpenCL &&
9620          !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
9621          "Address space agnostic languages only");
9622   LangAS DefaultGlobalAS = getLangASFromTargetAS(
9623       CGM.getContext().getTargetAddressSpace(LangAS::opencl_global));
9624   if (!D)
9625     return DefaultGlobalAS;
9626 
9627   LangAS AddrSpace = D->getType().getAddressSpace();
9628   assert(AddrSpace == LangAS::Default || isTargetAddressSpace(AddrSpace));
9629   if (AddrSpace != LangAS::Default)
9630     return AddrSpace;
9631 
9632   // Only promote to address space 4 if VarDecl has constant initialization.
9633   if (CGM.isTypeConstant(D->getType(), false) &&
9634       D->hasConstantInitialization()) {
9635     if (auto ConstAS = CGM.getTarget().getConstantAddressSpace())
9636       return *ConstAS;
9637   }
9638   return DefaultGlobalAS;
9639 }
9640 
9641 llvm::SyncScope::ID
getLLVMSyncScopeID(const LangOptions & LangOpts,SyncScope Scope,llvm::AtomicOrdering Ordering,llvm::LLVMContext & Ctx) const9642 AMDGPUTargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
9643                                             SyncScope Scope,
9644                                             llvm::AtomicOrdering Ordering,
9645                                             llvm::LLVMContext &Ctx) const {
9646   std::string Name;
9647   switch (Scope) {
9648   case SyncScope::HIPSingleThread:
9649     Name = "singlethread";
9650     break;
9651   case SyncScope::HIPWavefront:
9652   case SyncScope::OpenCLSubGroup:
9653     Name = "wavefront";
9654     break;
9655   case SyncScope::HIPWorkgroup:
9656   case SyncScope::OpenCLWorkGroup:
9657     Name = "workgroup";
9658     break;
9659   case SyncScope::HIPAgent:
9660   case SyncScope::OpenCLDevice:
9661     Name = "agent";
9662     break;
9663   case SyncScope::HIPSystem:
9664   case SyncScope::OpenCLAllSVMDevices:
9665     Name = "";
9666     break;
9667   }
9668 
9669   if (Ordering != llvm::AtomicOrdering::SequentiallyConsistent) {
9670     if (!Name.empty())
9671       Name = Twine(Twine(Name) + Twine("-")).str();
9672 
9673     Name = Twine(Twine(Name) + Twine("one-as")).str();
9674   }
9675 
9676   return Ctx.getOrInsertSyncScopeID(Name);
9677 }
9678 
shouldEmitStaticExternCAliases() const9679 bool AMDGPUTargetCodeGenInfo::shouldEmitStaticExternCAliases() const {
9680   return false;
9681 }
9682 
setCUDAKernelCallingConvention(const FunctionType * & FT) const9683 void AMDGPUTargetCodeGenInfo::setCUDAKernelCallingConvention(
9684     const FunctionType *&FT) const {
9685   FT = getABIInfo().getContext().adjustFunctionType(
9686       FT, FT->getExtInfo().withCallingConv(CC_OpenCLKernel));
9687 }
9688 
9689 //===----------------------------------------------------------------------===//
9690 // SPARC v8 ABI Implementation.
9691 // Based on the SPARC Compliance Definition version 2.4.1.
9692 //
9693 // Ensures that complex values are passed in registers.
9694 //
9695 namespace {
9696 class SparcV8ABIInfo : public DefaultABIInfo {
9697 public:
SparcV8ABIInfo(CodeGenTypes & CGT)9698   SparcV8ABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
9699 
9700 private:
9701   ABIArgInfo classifyReturnType(QualType RetTy) const;
9702   void computeInfo(CGFunctionInfo &FI) const override;
9703 };
9704 } // end anonymous namespace
9705 
9706 
9707 ABIArgInfo
classifyReturnType(QualType Ty) const9708 SparcV8ABIInfo::classifyReturnType(QualType Ty) const {
9709   if (Ty->isAnyComplexType()) {
9710     return ABIArgInfo::getDirect();
9711   }
9712   else {
9713     return DefaultABIInfo::classifyReturnType(Ty);
9714   }
9715 }
9716 
computeInfo(CGFunctionInfo & FI) const9717 void SparcV8ABIInfo::computeInfo(CGFunctionInfo &FI) const {
9718 
9719   FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
9720   for (auto &Arg : FI.arguments())
9721     Arg.info = classifyArgumentType(Arg.type);
9722 }
9723 
9724 namespace {
9725 class SparcV8TargetCodeGenInfo : public TargetCodeGenInfo {
9726 public:
SparcV8TargetCodeGenInfo(CodeGenTypes & CGT)9727   SparcV8TargetCodeGenInfo(CodeGenTypes &CGT)
9728       : TargetCodeGenInfo(std::make_unique<SparcV8ABIInfo>(CGT)) {}
9729 
decodeReturnAddress(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const9730   llvm::Value *decodeReturnAddress(CodeGen::CodeGenFunction &CGF,
9731                                    llvm::Value *Address) const override {
9732     int Offset;
9733     if (isAggregateTypeForABI(CGF.CurFnInfo->getReturnType()))
9734       Offset = 12;
9735     else
9736       Offset = 8;
9737     return CGF.Builder.CreateGEP(CGF.Int8Ty, Address,
9738                                  llvm::ConstantInt::get(CGF.Int32Ty, Offset));
9739   }
9740 
encodeReturnAddress(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const9741   llvm::Value *encodeReturnAddress(CodeGen::CodeGenFunction &CGF,
9742                                    llvm::Value *Address) const override {
9743     int Offset;
9744     if (isAggregateTypeForABI(CGF.CurFnInfo->getReturnType()))
9745       Offset = -12;
9746     else
9747       Offset = -8;
9748     return CGF.Builder.CreateGEP(CGF.Int8Ty, Address,
9749                                  llvm::ConstantInt::get(CGF.Int32Ty, Offset));
9750   }
9751 };
9752 } // end anonymous namespace
9753 
9754 //===----------------------------------------------------------------------===//
9755 // SPARC v9 ABI Implementation.
9756 // Based on the SPARC Compliance Definition version 2.4.1.
9757 //
9758 // Function arguments a mapped to a nominal "parameter array" and promoted to
9759 // registers depending on their type. Each argument occupies 8 or 16 bytes in
9760 // the array, structs larger than 16 bytes are passed indirectly.
9761 //
9762 // One case requires special care:
9763 //
9764 //   struct mixed {
9765 //     int i;
9766 //     float f;
9767 //   };
9768 //
9769 // When a struct mixed is passed by value, it only occupies 8 bytes in the
9770 // parameter array, but the int is passed in an integer register, and the float
9771 // is passed in a floating point register. This is represented as two arguments
9772 // with the LLVM IR inreg attribute:
9773 //
9774 //   declare void f(i32 inreg %i, float inreg %f)
9775 //
9776 // The code generator will only allocate 4 bytes from the parameter array for
9777 // the inreg arguments. All other arguments are allocated a multiple of 8
9778 // bytes.
9779 //
9780 namespace {
9781 class SparcV9ABIInfo : public ABIInfo {
9782 public:
SparcV9ABIInfo(CodeGenTypes & CGT)9783   SparcV9ABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
9784 
9785 private:
9786   ABIArgInfo classifyType(QualType RetTy, unsigned SizeLimit) const;
9787   void computeInfo(CGFunctionInfo &FI) const override;
9788   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9789                     QualType Ty) const override;
9790 
9791   // Coercion type builder for structs passed in registers. The coercion type
9792   // serves two purposes:
9793   //
9794   // 1. Pad structs to a multiple of 64 bits, so they are passed 'left-aligned'
9795   //    in registers.
9796   // 2. Expose aligned floating point elements as first-level elements, so the
9797   //    code generator knows to pass them in floating point registers.
9798   //
9799   // We also compute the InReg flag which indicates that the struct contains
9800   // aligned 32-bit floats.
9801   //
9802   struct CoerceBuilder {
9803     llvm::LLVMContext &Context;
9804     const llvm::DataLayout &DL;
9805     SmallVector<llvm::Type*, 8> Elems;
9806     uint64_t Size;
9807     bool InReg;
9808 
CoerceBuilder__anondfff523e1811::SparcV9ABIInfo::CoerceBuilder9809     CoerceBuilder(llvm::LLVMContext &c, const llvm::DataLayout &dl)
9810       : Context(c), DL(dl), Size(0), InReg(false) {}
9811 
9812     // Pad Elems with integers until Size is ToSize.
pad__anondfff523e1811::SparcV9ABIInfo::CoerceBuilder9813     void pad(uint64_t ToSize) {
9814       assert(ToSize >= Size && "Cannot remove elements");
9815       if (ToSize == Size)
9816         return;
9817 
9818       // Finish the current 64-bit word.
9819       uint64_t Aligned = llvm::alignTo(Size, 64);
9820       if (Aligned > Size && Aligned <= ToSize) {
9821         Elems.push_back(llvm::IntegerType::get(Context, Aligned - Size));
9822         Size = Aligned;
9823       }
9824 
9825       // Add whole 64-bit words.
9826       while (Size + 64 <= ToSize) {
9827         Elems.push_back(llvm::Type::getInt64Ty(Context));
9828         Size += 64;
9829       }
9830 
9831       // Final in-word padding.
9832       if (Size < ToSize) {
9833         Elems.push_back(llvm::IntegerType::get(Context, ToSize - Size));
9834         Size = ToSize;
9835       }
9836     }
9837 
9838     // Add a floating point element at Offset.
addFloat__anondfff523e1811::SparcV9ABIInfo::CoerceBuilder9839     void addFloat(uint64_t Offset, llvm::Type *Ty, unsigned Bits) {
9840       // Unaligned floats are treated as integers.
9841       if (Offset % Bits)
9842         return;
9843       // The InReg flag is only required if there are any floats < 64 bits.
9844       if (Bits < 64)
9845         InReg = true;
9846       pad(Offset);
9847       Elems.push_back(Ty);
9848       Size = Offset + Bits;
9849     }
9850 
9851     // Add a struct type to the coercion type, starting at Offset (in bits).
addStruct__anondfff523e1811::SparcV9ABIInfo::CoerceBuilder9852     void addStruct(uint64_t Offset, llvm::StructType *StrTy) {
9853       const llvm::StructLayout *Layout = DL.getStructLayout(StrTy);
9854       for (unsigned i = 0, e = StrTy->getNumElements(); i != e; ++i) {
9855         llvm::Type *ElemTy = StrTy->getElementType(i);
9856         uint64_t ElemOffset = Offset + Layout->getElementOffsetInBits(i);
9857         switch (ElemTy->getTypeID()) {
9858         case llvm::Type::StructTyID:
9859           addStruct(ElemOffset, cast<llvm::StructType>(ElemTy));
9860           break;
9861         case llvm::Type::FloatTyID:
9862           addFloat(ElemOffset, ElemTy, 32);
9863           break;
9864         case llvm::Type::DoubleTyID:
9865           addFloat(ElemOffset, ElemTy, 64);
9866           break;
9867         case llvm::Type::FP128TyID:
9868           addFloat(ElemOffset, ElemTy, 128);
9869           break;
9870         case llvm::Type::PointerTyID:
9871           if (ElemOffset % 64 == 0) {
9872             pad(ElemOffset);
9873             Elems.push_back(ElemTy);
9874             Size += 64;
9875           }
9876           break;
9877         default:
9878           break;
9879         }
9880       }
9881     }
9882 
9883     // Check if Ty is a usable substitute for the coercion type.
isUsableType__anondfff523e1811::SparcV9ABIInfo::CoerceBuilder9884     bool isUsableType(llvm::StructType *Ty) const {
9885       return llvm::ArrayRef(Elems) == Ty->elements();
9886     }
9887 
9888     // Get the coercion type as a literal struct type.
getType__anondfff523e1811::SparcV9ABIInfo::CoerceBuilder9889     llvm::Type *getType() const {
9890       if (Elems.size() == 1)
9891         return Elems.front();
9892       else
9893         return llvm::StructType::get(Context, Elems);
9894     }
9895   };
9896 };
9897 } // end anonymous namespace
9898 
9899 ABIArgInfo
classifyType(QualType Ty,unsigned SizeLimit) const9900 SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const {
9901   if (Ty->isVoidType())
9902     return ABIArgInfo::getIgnore();
9903 
9904   uint64_t Size = getContext().getTypeSize(Ty);
9905 
9906   // Anything too big to fit in registers is passed with an explicit indirect
9907   // pointer / sret pointer.
9908   if (Size > SizeLimit)
9909     return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
9910 
9911   // Treat an enum type as its underlying type.
9912   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
9913     Ty = EnumTy->getDecl()->getIntegerType();
9914 
9915   // Integer types smaller than a register are extended.
9916   if (Size < 64 && Ty->isIntegerType())
9917     return ABIArgInfo::getExtend(Ty);
9918 
9919   if (const auto *EIT = Ty->getAs<BitIntType>())
9920     if (EIT->getNumBits() < 64)
9921       return ABIArgInfo::getExtend(Ty);
9922 
9923   // Other non-aggregates go in registers.
9924   if (!isAggregateTypeForABI(Ty))
9925     return ABIArgInfo::getDirect();
9926 
9927   // If a C++ object has either a non-trivial copy constructor or a non-trivial
9928   // destructor, it is passed with an explicit indirect pointer / sret pointer.
9929   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
9930     return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
9931 
9932   // This is a small aggregate type that should be passed in registers.
9933   // Build a coercion type from the LLVM struct type.
9934   llvm::StructType *StrTy = dyn_cast<llvm::StructType>(CGT.ConvertType(Ty));
9935   if (!StrTy)
9936     return ABIArgInfo::getDirect();
9937 
9938   CoerceBuilder CB(getVMContext(), getDataLayout());
9939   CB.addStruct(0, StrTy);
9940   CB.pad(llvm::alignTo(CB.DL.getTypeSizeInBits(StrTy), 64));
9941 
9942   // Try to use the original type for coercion.
9943   llvm::Type *CoerceTy = CB.isUsableType(StrTy) ? StrTy : CB.getType();
9944 
9945   if (CB.InReg)
9946     return ABIArgInfo::getDirectInReg(CoerceTy);
9947   else
9948     return ABIArgInfo::getDirect(CoerceTy);
9949 }
9950 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const9951 Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9952                                   QualType Ty) const {
9953   ABIArgInfo AI = classifyType(Ty, 16 * 8);
9954   llvm::Type *ArgTy = CGT.ConvertType(Ty);
9955   if (AI.canHaveCoerceToType() && !AI.getCoerceToType())
9956     AI.setCoerceToType(ArgTy);
9957 
9958   CharUnits SlotSize = CharUnits::fromQuantity(8);
9959 
9960   CGBuilderTy &Builder = CGF.Builder;
9961   Address Addr = Address(Builder.CreateLoad(VAListAddr, "ap.cur"),
9962                          getVAListElementType(CGF), SlotSize);
9963   llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy);
9964 
9965   auto TypeInfo = getContext().getTypeInfoInChars(Ty);
9966 
9967   Address ArgAddr = Address::invalid();
9968   CharUnits Stride;
9969   switch (AI.getKind()) {
9970   case ABIArgInfo::Expand:
9971   case ABIArgInfo::CoerceAndExpand:
9972   case ABIArgInfo::InAlloca:
9973     llvm_unreachable("Unsupported ABI kind for va_arg");
9974 
9975   case ABIArgInfo::Extend: {
9976     Stride = SlotSize;
9977     CharUnits Offset = SlotSize - TypeInfo.Width;
9978     ArgAddr = Builder.CreateConstInBoundsByteGEP(Addr, Offset, "extend");
9979     break;
9980   }
9981 
9982   case ABIArgInfo::Direct: {
9983     auto AllocSize = getDataLayout().getTypeAllocSize(AI.getCoerceToType());
9984     Stride = CharUnits::fromQuantity(AllocSize).alignTo(SlotSize);
9985     ArgAddr = Addr;
9986     break;
9987   }
9988 
9989   case ABIArgInfo::Indirect:
9990   case ABIArgInfo::IndirectAliased:
9991     Stride = SlotSize;
9992     ArgAddr = Builder.CreateElementBitCast(Addr, ArgPtrTy, "indirect");
9993     ArgAddr = Address(Builder.CreateLoad(ArgAddr, "indirect.arg"), ArgTy,
9994                       TypeInfo.Align);
9995     break;
9996 
9997   case ABIArgInfo::Ignore:
9998     return Address(llvm::UndefValue::get(ArgPtrTy), ArgTy, TypeInfo.Align);
9999   }
10000 
10001   // Update VAList.
10002   Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next");
10003   Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
10004 
10005   return Builder.CreateElementBitCast(ArgAddr, ArgTy, "arg.addr");
10006 }
10007 
computeInfo(CGFunctionInfo & FI) const10008 void SparcV9ABIInfo::computeInfo(CGFunctionInfo &FI) const {
10009   FI.getReturnInfo() = classifyType(FI.getReturnType(), 32 * 8);
10010   for (auto &I : FI.arguments())
10011     I.info = classifyType(I.type, 16 * 8);
10012 }
10013 
10014 namespace {
10015 class SparcV9TargetCodeGenInfo : public TargetCodeGenInfo {
10016 public:
SparcV9TargetCodeGenInfo(CodeGenTypes & CGT)10017   SparcV9TargetCodeGenInfo(CodeGenTypes &CGT)
10018       : TargetCodeGenInfo(std::make_unique<SparcV9ABIInfo>(CGT)) {}
10019 
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const10020   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
10021     return 14;
10022   }
10023 
10024   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
10025                                llvm::Value *Address) const override;
10026 
decodeReturnAddress(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const10027   llvm::Value *decodeReturnAddress(CodeGen::CodeGenFunction &CGF,
10028                                    llvm::Value *Address) const override {
10029     return CGF.Builder.CreateGEP(CGF.Int8Ty, Address,
10030                                  llvm::ConstantInt::get(CGF.Int32Ty, 8));
10031   }
10032 
encodeReturnAddress(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const10033   llvm::Value *encodeReturnAddress(CodeGen::CodeGenFunction &CGF,
10034                                    llvm::Value *Address) const override {
10035     return CGF.Builder.CreateGEP(CGF.Int8Ty, Address,
10036                                  llvm::ConstantInt::get(CGF.Int32Ty, -8));
10037   }
10038 };
10039 } // end anonymous namespace
10040 
10041 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const10042 SparcV9TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
10043                                                 llvm::Value *Address) const {
10044   // This is calculated from the LLVM and GCC tables and verified
10045   // against gcc output.  AFAIK all ABIs use the same encoding.
10046 
10047   CodeGen::CGBuilderTy &Builder = CGF.Builder;
10048 
10049   llvm::IntegerType *i8 = CGF.Int8Ty;
10050   llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
10051   llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
10052 
10053   // 0-31: the 8-byte general-purpose registers
10054   AssignToArrayRange(Builder, Address, Eight8, 0, 31);
10055 
10056   // 32-63: f0-31, the 4-byte floating-point registers
10057   AssignToArrayRange(Builder, Address, Four8, 32, 63);
10058 
10059   //   Y   = 64
10060   //   PSR = 65
10061   //   WIM = 66
10062   //   TBR = 67
10063   //   PC  = 68
10064   //   NPC = 69
10065   //   FSR = 70
10066   //   CSR = 71
10067   AssignToArrayRange(Builder, Address, Eight8, 64, 71);
10068 
10069   // 72-87: d0-15, the 8-byte floating-point registers
10070   AssignToArrayRange(Builder, Address, Eight8, 72, 87);
10071 
10072   return false;
10073 }
10074 
10075 // ARC ABI implementation.
10076 namespace {
10077 
10078 class ARCABIInfo : public DefaultABIInfo {
10079 public:
10080   using DefaultABIInfo::DefaultABIInfo;
10081 
10082 private:
10083   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
10084                     QualType Ty) const override;
10085 
updateState(const ABIArgInfo & Info,QualType Ty,CCState & State) const10086   void updateState(const ABIArgInfo &Info, QualType Ty, CCState &State) const {
10087     if (!State.FreeRegs)
10088       return;
10089     if (Info.isIndirect() && Info.getInReg())
10090       State.FreeRegs--;
10091     else if (Info.isDirect() && Info.getInReg()) {
10092       unsigned sz = (getContext().getTypeSize(Ty) + 31) / 32;
10093       if (sz < State.FreeRegs)
10094         State.FreeRegs -= sz;
10095       else
10096         State.FreeRegs = 0;
10097     }
10098   }
10099 
computeInfo(CGFunctionInfo & FI) const10100   void computeInfo(CGFunctionInfo &FI) const override {
10101     CCState State(FI);
10102     // ARC uses 8 registers to pass arguments.
10103     State.FreeRegs = 8;
10104 
10105     if (!getCXXABI().classifyReturnType(FI))
10106       FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
10107     updateState(FI.getReturnInfo(), FI.getReturnType(), State);
10108     for (auto &I : FI.arguments()) {
10109       I.info = classifyArgumentType(I.type, State.FreeRegs);
10110       updateState(I.info, I.type, State);
10111     }
10112   }
10113 
10114   ABIArgInfo getIndirectByRef(QualType Ty, bool HasFreeRegs) const;
10115   ABIArgInfo getIndirectByValue(QualType Ty) const;
10116   ABIArgInfo classifyArgumentType(QualType Ty, uint8_t FreeRegs) const;
10117   ABIArgInfo classifyReturnType(QualType RetTy) const;
10118 };
10119 
10120 class ARCTargetCodeGenInfo : public TargetCodeGenInfo {
10121 public:
ARCTargetCodeGenInfo(CodeGenTypes & CGT)10122   ARCTargetCodeGenInfo(CodeGenTypes &CGT)
10123       : TargetCodeGenInfo(std::make_unique<ARCABIInfo>(CGT)) {}
10124 };
10125 
10126 
getIndirectByRef(QualType Ty,bool HasFreeRegs) const10127 ABIArgInfo ARCABIInfo::getIndirectByRef(QualType Ty, bool HasFreeRegs) const {
10128   return HasFreeRegs ? getNaturalAlignIndirectInReg(Ty) :
10129                        getNaturalAlignIndirect(Ty, false);
10130 }
10131 
getIndirectByValue(QualType Ty) const10132 ABIArgInfo ARCABIInfo::getIndirectByValue(QualType Ty) const {
10133   // Compute the byval alignment.
10134   const unsigned MinABIStackAlignInBytes = 4;
10135   unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
10136   return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true,
10137                                  TypeAlign > MinABIStackAlignInBytes);
10138 }
10139 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const10140 Address ARCABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
10141                               QualType Ty) const {
10142   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
10143                           getContext().getTypeInfoInChars(Ty),
10144                           CharUnits::fromQuantity(4), true);
10145 }
10146 
classifyArgumentType(QualType Ty,uint8_t FreeRegs) const10147 ABIArgInfo ARCABIInfo::classifyArgumentType(QualType Ty,
10148                                             uint8_t FreeRegs) const {
10149   // Handle the generic C++ ABI.
10150   const RecordType *RT = Ty->getAs<RecordType>();
10151   if (RT) {
10152     CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
10153     if (RAA == CGCXXABI::RAA_Indirect)
10154       return getIndirectByRef(Ty, FreeRegs > 0);
10155 
10156     if (RAA == CGCXXABI::RAA_DirectInMemory)
10157       return getIndirectByValue(Ty);
10158   }
10159 
10160   // Treat an enum type as its underlying type.
10161   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
10162     Ty = EnumTy->getDecl()->getIntegerType();
10163 
10164   auto SizeInRegs = llvm::alignTo(getContext().getTypeSize(Ty), 32) / 32;
10165 
10166   if (isAggregateTypeForABI(Ty)) {
10167     // Structures with flexible arrays are always indirect.
10168     if (RT && RT->getDecl()->hasFlexibleArrayMember())
10169       return getIndirectByValue(Ty);
10170 
10171     // Ignore empty structs/unions.
10172     if (isEmptyRecord(getContext(), Ty, true))
10173       return ABIArgInfo::getIgnore();
10174 
10175     llvm::LLVMContext &LLVMContext = getVMContext();
10176 
10177     llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
10178     SmallVector<llvm::Type *, 3> Elements(SizeInRegs, Int32);
10179     llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
10180 
10181     return FreeRegs >= SizeInRegs ?
10182         ABIArgInfo::getDirectInReg(Result) :
10183         ABIArgInfo::getDirect(Result, 0, nullptr, false);
10184   }
10185 
10186   if (const auto *EIT = Ty->getAs<BitIntType>())
10187     if (EIT->getNumBits() > 64)
10188       return getIndirectByValue(Ty);
10189 
10190   return isPromotableIntegerTypeForABI(Ty)
10191              ? (FreeRegs >= SizeInRegs ? ABIArgInfo::getExtendInReg(Ty)
10192                                        : ABIArgInfo::getExtend(Ty))
10193              : (FreeRegs >= SizeInRegs ? ABIArgInfo::getDirectInReg()
10194                                        : ABIArgInfo::getDirect());
10195 }
10196 
classifyReturnType(QualType RetTy) const10197 ABIArgInfo ARCABIInfo::classifyReturnType(QualType RetTy) const {
10198   if (RetTy->isAnyComplexType())
10199     return ABIArgInfo::getDirectInReg();
10200 
10201   // Arguments of size > 4 registers are indirect.
10202   auto RetSize = llvm::alignTo(getContext().getTypeSize(RetTy), 32) / 32;
10203   if (RetSize > 4)
10204     return getIndirectByRef(RetTy, /*HasFreeRegs*/ true);
10205 
10206   return DefaultABIInfo::classifyReturnType(RetTy);
10207 }
10208 
10209 } // End anonymous namespace.
10210 
10211 //===----------------------------------------------------------------------===//
10212 // XCore ABI Implementation
10213 //===----------------------------------------------------------------------===//
10214 
10215 namespace {
10216 
10217 /// A SmallStringEnc instance is used to build up the TypeString by passing
10218 /// it by reference between functions that append to it.
10219 typedef llvm::SmallString<128> SmallStringEnc;
10220 
10221 /// TypeStringCache caches the meta encodings of Types.
10222 ///
10223 /// The reason for caching TypeStrings is two fold:
10224 ///   1. To cache a type's encoding for later uses;
10225 ///   2. As a means to break recursive member type inclusion.
10226 ///
10227 /// A cache Entry can have a Status of:
10228 ///   NonRecursive:   The type encoding is not recursive;
10229 ///   Recursive:      The type encoding is recursive;
10230 ///   Incomplete:     An incomplete TypeString;
10231 ///   IncompleteUsed: An incomplete TypeString that has been used in a
10232 ///                   Recursive type encoding.
10233 ///
10234 /// A NonRecursive entry will have all of its sub-members expanded as fully
10235 /// as possible. Whilst it may contain types which are recursive, the type
10236 /// itself is not recursive and thus its encoding may be safely used whenever
10237 /// the type is encountered.
10238 ///
10239 /// A Recursive entry will have all of its sub-members expanded as fully as
10240 /// possible. The type itself is recursive and it may contain other types which
10241 /// are recursive. The Recursive encoding must not be used during the expansion
10242 /// of a recursive type's recursive branch. For simplicity the code uses
10243 /// IncompleteCount to reject all usage of Recursive encodings for member types.
10244 ///
10245 /// An Incomplete entry is always a RecordType and only encodes its
10246 /// identifier e.g. "s(S){}". Incomplete 'StubEnc' entries are ephemeral and
10247 /// are placed into the cache during type expansion as a means to identify and
10248 /// handle recursive inclusion of types as sub-members. If there is recursion
10249 /// the entry becomes IncompleteUsed.
10250 ///
10251 /// During the expansion of a RecordType's members:
10252 ///
10253 ///   If the cache contains a NonRecursive encoding for the member type, the
10254 ///   cached encoding is used;
10255 ///
10256 ///   If the cache contains a Recursive encoding for the member type, the
10257 ///   cached encoding is 'Swapped' out, as it may be incorrect, and...
10258 ///
10259 ///   If the member is a RecordType, an Incomplete encoding is placed into the
10260 ///   cache to break potential recursive inclusion of itself as a sub-member;
10261 ///
10262 ///   Once a member RecordType has been expanded, its temporary incomplete
10263 ///   entry is removed from the cache. If a Recursive encoding was swapped out
10264 ///   it is swapped back in;
10265 ///
10266 ///   If an incomplete entry is used to expand a sub-member, the incomplete
10267 ///   entry is marked as IncompleteUsed. The cache keeps count of how many
10268 ///   IncompleteUsed entries it currently contains in IncompleteUsedCount;
10269 ///
10270 ///   If a member's encoding is found to be a NonRecursive or Recursive viz:
10271 ///   IncompleteUsedCount==0, the member's encoding is added to the cache.
10272 ///   Else the member is part of a recursive type and thus the recursion has
10273 ///   been exited too soon for the encoding to be correct for the member.
10274 ///
10275 class TypeStringCache {
10276   enum Status {NonRecursive, Recursive, Incomplete, IncompleteUsed};
10277   struct Entry {
10278     std::string Str;     // The encoded TypeString for the type.
10279     enum Status State;   // Information about the encoding in 'Str'.
10280     std::string Swapped; // A temporary place holder for a Recursive encoding
10281                          // during the expansion of RecordType's members.
10282   };
10283   std::map<const IdentifierInfo *, struct Entry> Map;
10284   unsigned IncompleteCount;     // Number of Incomplete entries in the Map.
10285   unsigned IncompleteUsedCount; // Number of IncompleteUsed entries in the Map.
10286 public:
TypeStringCache()10287   TypeStringCache() : IncompleteCount(0), IncompleteUsedCount(0) {}
10288   void addIncomplete(const IdentifierInfo *ID, std::string StubEnc);
10289   bool removeIncomplete(const IdentifierInfo *ID);
10290   void addIfComplete(const IdentifierInfo *ID, StringRef Str,
10291                      bool IsRecursive);
10292   StringRef lookupStr(const IdentifierInfo *ID);
10293 };
10294 
10295 /// TypeString encodings for enum & union fields must be order.
10296 /// FieldEncoding is a helper for this ordering process.
10297 class FieldEncoding {
10298   bool HasName;
10299   std::string Enc;
10300 public:
FieldEncoding(bool b,SmallStringEnc & e)10301   FieldEncoding(bool b, SmallStringEnc &e) : HasName(b), Enc(e.c_str()) {}
str()10302   StringRef str() { return Enc; }
operator <(const FieldEncoding & rhs) const10303   bool operator<(const FieldEncoding &rhs) const {
10304     if (HasName != rhs.HasName) return HasName;
10305     return Enc < rhs.Enc;
10306   }
10307 };
10308 
10309 class XCoreABIInfo : public DefaultABIInfo {
10310 public:
XCoreABIInfo(CodeGen::CodeGenTypes & CGT)10311   XCoreABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
10312   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
10313                     QualType Ty) const override;
10314 };
10315 
10316 class XCoreTargetCodeGenInfo : public TargetCodeGenInfo {
10317   mutable TypeStringCache TSC;
10318   void emitTargetMD(const Decl *D, llvm::GlobalValue *GV,
10319                     const CodeGen::CodeGenModule &M) const;
10320 
10321 public:
XCoreTargetCodeGenInfo(CodeGenTypes & CGT)10322   XCoreTargetCodeGenInfo(CodeGenTypes &CGT)
10323       : TargetCodeGenInfo(std::make_unique<XCoreABIInfo>(CGT)) {}
10324   void emitTargetMetadata(CodeGen::CodeGenModule &CGM,
10325                           const llvm::MapVector<GlobalDecl, StringRef>
10326                               &MangledDeclNames) const override;
10327 };
10328 
10329 } // End anonymous namespace.
10330 
10331 // TODO: this implementation is likely now redundant with the default
10332 // EmitVAArg.
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const10333 Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
10334                                 QualType Ty) const {
10335   CGBuilderTy &Builder = CGF.Builder;
10336 
10337   // Get the VAList.
10338   CharUnits SlotSize = CharUnits::fromQuantity(4);
10339   Address AP = Address(Builder.CreateLoad(VAListAddr),
10340                        getVAListElementType(CGF), SlotSize);
10341 
10342   // Handle the argument.
10343   ABIArgInfo AI = classifyArgumentType(Ty);
10344   CharUnits TypeAlign = getContext().getTypeAlignInChars(Ty);
10345   llvm::Type *ArgTy = CGT.ConvertType(Ty);
10346   if (AI.canHaveCoerceToType() && !AI.getCoerceToType())
10347     AI.setCoerceToType(ArgTy);
10348   llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy);
10349 
10350   Address Val = Address::invalid();
10351   CharUnits ArgSize = CharUnits::Zero();
10352   switch (AI.getKind()) {
10353   case ABIArgInfo::Expand:
10354   case ABIArgInfo::CoerceAndExpand:
10355   case ABIArgInfo::InAlloca:
10356     llvm_unreachable("Unsupported ABI kind for va_arg");
10357   case ABIArgInfo::Ignore:
10358     Val = Address(llvm::UndefValue::get(ArgPtrTy), ArgTy, TypeAlign);
10359     ArgSize = CharUnits::Zero();
10360     break;
10361   case ABIArgInfo::Extend:
10362   case ABIArgInfo::Direct:
10363     Val = Builder.CreateElementBitCast(AP, ArgTy);
10364     ArgSize = CharUnits::fromQuantity(
10365         getDataLayout().getTypeAllocSize(AI.getCoerceToType()));
10366     ArgSize = ArgSize.alignTo(SlotSize);
10367     break;
10368   case ABIArgInfo::Indirect:
10369   case ABIArgInfo::IndirectAliased:
10370     Val = Builder.CreateElementBitCast(AP, ArgPtrTy);
10371     Val = Address(Builder.CreateLoad(Val), ArgTy, TypeAlign);
10372     ArgSize = SlotSize;
10373     break;
10374   }
10375 
10376   // Increment the VAList.
10377   if (!ArgSize.isZero()) {
10378     Address APN = Builder.CreateConstInBoundsByteGEP(AP, ArgSize);
10379     Builder.CreateStore(APN.getPointer(), VAListAddr);
10380   }
10381 
10382   return Val;
10383 }
10384 
10385 /// During the expansion of a RecordType, an incomplete TypeString is placed
10386 /// into the cache as a means to identify and break recursion.
10387 /// If there is a Recursive encoding in the cache, it is swapped out and will
10388 /// be reinserted by removeIncomplete().
10389 /// All other types of encoding should have been used rather than arriving here.
addIncomplete(const IdentifierInfo * ID,std::string StubEnc)10390 void TypeStringCache::addIncomplete(const IdentifierInfo *ID,
10391                                     std::string StubEnc) {
10392   if (!ID)
10393     return;
10394   Entry &E = Map[ID];
10395   assert( (E.Str.empty() || E.State == Recursive) &&
10396          "Incorrectly use of addIncomplete");
10397   assert(!StubEnc.empty() && "Passing an empty string to addIncomplete()");
10398   E.Swapped.swap(E.Str); // swap out the Recursive
10399   E.Str.swap(StubEnc);
10400   E.State = Incomplete;
10401   ++IncompleteCount;
10402 }
10403 
10404 /// Once the RecordType has been expanded, the temporary incomplete TypeString
10405 /// must be removed from the cache.
10406 /// If a Recursive was swapped out by addIncomplete(), it will be replaced.
10407 /// Returns true if the RecordType was defined recursively.
removeIncomplete(const IdentifierInfo * ID)10408 bool TypeStringCache::removeIncomplete(const IdentifierInfo *ID) {
10409   if (!ID)
10410     return false;
10411   auto I = Map.find(ID);
10412   assert(I != Map.end() && "Entry not present");
10413   Entry &E = I->second;
10414   assert( (E.State == Incomplete ||
10415            E.State == IncompleteUsed) &&
10416          "Entry must be an incomplete type");
10417   bool IsRecursive = false;
10418   if (E.State == IncompleteUsed) {
10419     // We made use of our Incomplete encoding, thus we are recursive.
10420     IsRecursive = true;
10421     --IncompleteUsedCount;
10422   }
10423   if (E.Swapped.empty())
10424     Map.erase(I);
10425   else {
10426     // Swap the Recursive back.
10427     E.Swapped.swap(E.Str);
10428     E.Swapped.clear();
10429     E.State = Recursive;
10430   }
10431   --IncompleteCount;
10432   return IsRecursive;
10433 }
10434 
10435 /// Add the encoded TypeString to the cache only if it is NonRecursive or
10436 /// Recursive (viz: all sub-members were expanded as fully as possible).
addIfComplete(const IdentifierInfo * ID,StringRef Str,bool IsRecursive)10437 void TypeStringCache::addIfComplete(const IdentifierInfo *ID, StringRef Str,
10438                                     bool IsRecursive) {
10439   if (!ID || IncompleteUsedCount)
10440     return; // No key or it is an incomplete sub-type so don't add.
10441   Entry &E = Map[ID];
10442   if (IsRecursive && !E.Str.empty()) {
10443     assert(E.State==Recursive && E.Str.size() == Str.size() &&
10444            "This is not the same Recursive entry");
10445     // The parent container was not recursive after all, so we could have used
10446     // this Recursive sub-member entry after all, but we assumed the worse when
10447     // we started viz: IncompleteCount!=0.
10448     return;
10449   }
10450   assert(E.Str.empty() && "Entry already present");
10451   E.Str = Str.str();
10452   E.State = IsRecursive? Recursive : NonRecursive;
10453 }
10454 
10455 /// Return a cached TypeString encoding for the ID. If there isn't one, or we
10456 /// are recursively expanding a type (IncompleteCount != 0) and the cached
10457 /// encoding is Recursive, return an empty StringRef.
lookupStr(const IdentifierInfo * ID)10458 StringRef TypeStringCache::lookupStr(const IdentifierInfo *ID) {
10459   if (!ID)
10460     return StringRef();   // We have no key.
10461   auto I = Map.find(ID);
10462   if (I == Map.end())
10463     return StringRef();   // We have no encoding.
10464   Entry &E = I->second;
10465   if (E.State == Recursive && IncompleteCount)
10466     return StringRef();   // We don't use Recursive encodings for member types.
10467 
10468   if (E.State == Incomplete) {
10469     // The incomplete type is being used to break out of recursion.
10470     E.State = IncompleteUsed;
10471     ++IncompleteUsedCount;
10472   }
10473   return E.Str;
10474 }
10475 
10476 /// The XCore ABI includes a type information section that communicates symbol
10477 /// type information to the linker. The linker uses this information to verify
10478 /// safety/correctness of things such as array bound and pointers et al.
10479 /// The ABI only requires C (and XC) language modules to emit TypeStrings.
10480 /// This type information (TypeString) is emitted into meta data for all global
10481 /// symbols: definitions, declarations, functions & variables.
10482 ///
10483 /// The TypeString carries type, qualifier, name, size & value details.
10484 /// Please see 'Tools Development Guide' section 2.16.2 for format details:
10485 /// https://www.xmos.com/download/public/Tools-Development-Guide%28X9114A%29.pdf
10486 /// The output is tested by test/CodeGen/xcore-stringtype.c.
10487 ///
10488 static bool getTypeString(SmallStringEnc &Enc, const Decl *D,
10489                           const CodeGen::CodeGenModule &CGM,
10490                           TypeStringCache &TSC);
10491 
10492 /// XCore uses emitTargetMD to emit TypeString metadata for global symbols.
emitTargetMD(const Decl * D,llvm::GlobalValue * GV,const CodeGen::CodeGenModule & CGM) const10493 void XCoreTargetCodeGenInfo::emitTargetMD(
10494     const Decl *D, llvm::GlobalValue *GV,
10495     const CodeGen::CodeGenModule &CGM) const {
10496   SmallStringEnc Enc;
10497   if (getTypeString(Enc, D, CGM, TSC)) {
10498     llvm::LLVMContext &Ctx = CGM.getModule().getContext();
10499     llvm::Metadata *MDVals[] = {llvm::ConstantAsMetadata::get(GV),
10500                                 llvm::MDString::get(Ctx, Enc.str())};
10501     llvm::NamedMDNode *MD =
10502       CGM.getModule().getOrInsertNamedMetadata("xcore.typestrings");
10503     MD->addOperand(llvm::MDNode::get(Ctx, MDVals));
10504   }
10505 }
10506 
emitTargetMetadata(CodeGen::CodeGenModule & CGM,const llvm::MapVector<GlobalDecl,StringRef> & MangledDeclNames) const10507 void XCoreTargetCodeGenInfo::emitTargetMetadata(
10508     CodeGen::CodeGenModule &CGM,
10509     const llvm::MapVector<GlobalDecl, StringRef> &MangledDeclNames) const {
10510   // Warning, new MangledDeclNames may be appended within this loop.
10511   // We rely on MapVector insertions adding new elements to the end
10512   // of the container.
10513   for (unsigned I = 0; I != MangledDeclNames.size(); ++I) {
10514     auto Val = *(MangledDeclNames.begin() + I);
10515     llvm::GlobalValue *GV = CGM.GetGlobalValue(Val.second);
10516     if (GV) {
10517       const Decl *D = Val.first.getDecl()->getMostRecentDecl();
10518       emitTargetMD(D, GV, CGM);
10519     }
10520   }
10521 }
10522 
10523 //===----------------------------------------------------------------------===//
10524 // Base ABI and target codegen info implementation common between SPIR and
10525 // SPIR-V.
10526 //===----------------------------------------------------------------------===//
10527 
10528 namespace {
10529 class CommonSPIRABIInfo : public DefaultABIInfo {
10530 public:
CommonSPIRABIInfo(CodeGenTypes & CGT)10531   CommonSPIRABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) { setCCs(); }
10532 
10533 private:
10534   void setCCs();
10535 };
10536 
10537 class SPIRVABIInfo : public CommonSPIRABIInfo {
10538 public:
SPIRVABIInfo(CodeGenTypes & CGT)10539   SPIRVABIInfo(CodeGenTypes &CGT) : CommonSPIRABIInfo(CGT) {}
10540   void computeInfo(CGFunctionInfo &FI) const override;
10541 
10542 private:
10543   ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
10544 };
10545 } // end anonymous namespace
10546 namespace {
10547 class CommonSPIRTargetCodeGenInfo : public TargetCodeGenInfo {
10548 public:
CommonSPIRTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT)10549   CommonSPIRTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
10550       : TargetCodeGenInfo(std::make_unique<CommonSPIRABIInfo>(CGT)) {}
CommonSPIRTargetCodeGenInfo(std::unique_ptr<ABIInfo> ABIInfo)10551   CommonSPIRTargetCodeGenInfo(std::unique_ptr<ABIInfo> ABIInfo)
10552       : TargetCodeGenInfo(std::move(ABIInfo)) {}
10553 
getASTAllocaAddressSpace() const10554   LangAS getASTAllocaAddressSpace() const override {
10555     return getLangASFromTargetAS(
10556         getABIInfo().getDataLayout().getAllocaAddrSpace());
10557   }
10558 
10559   unsigned getOpenCLKernelCallingConv() const override;
10560 };
10561 class SPIRVTargetCodeGenInfo : public CommonSPIRTargetCodeGenInfo {
10562 public:
SPIRVTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT)10563   SPIRVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
10564       : CommonSPIRTargetCodeGenInfo(std::make_unique<SPIRVABIInfo>(CGT)) {}
10565   void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
10566 };
10567 } // End anonymous namespace.
10568 
setCCs()10569 void CommonSPIRABIInfo::setCCs() {
10570   assert(getRuntimeCC() == llvm::CallingConv::C);
10571   RuntimeCC = llvm::CallingConv::SPIR_FUNC;
10572 }
10573 
classifyKernelArgumentType(QualType Ty) const10574 ABIArgInfo SPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const {
10575   if (getContext().getLangOpts().CUDAIsDevice) {
10576     // Coerce pointer arguments with default address space to CrossWorkGroup
10577     // pointers for HIPSPV/CUDASPV. When the language mode is HIP/CUDA, the
10578     // SPIRTargetInfo maps cuda_device to SPIR-V's CrossWorkGroup address space.
10579     llvm::Type *LTy = CGT.ConvertType(Ty);
10580     auto DefaultAS = getContext().getTargetAddressSpace(LangAS::Default);
10581     auto GlobalAS = getContext().getTargetAddressSpace(LangAS::cuda_device);
10582     auto *PtrTy = llvm::dyn_cast<llvm::PointerType>(LTy);
10583     if (PtrTy && PtrTy->getAddressSpace() == DefaultAS) {
10584       LTy = llvm::PointerType::getWithSamePointeeType(PtrTy, GlobalAS);
10585       return ABIArgInfo::getDirect(LTy, 0, nullptr, false);
10586     }
10587 
10588     // Force copying aggregate type in kernel arguments by value when
10589     // compiling CUDA targeting SPIR-V. This is required for the object
10590     // copied to be valid on the device.
10591     // This behavior follows the CUDA spec
10592     // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#global-function-argument-processing,
10593     // and matches the NVPTX implementation.
10594     if (isAggregateTypeForABI(Ty))
10595       return getNaturalAlignIndirect(Ty, /* byval */ true);
10596   }
10597   return classifyArgumentType(Ty);
10598 }
10599 
computeInfo(CGFunctionInfo & FI) const10600 void SPIRVABIInfo::computeInfo(CGFunctionInfo &FI) const {
10601   // The logic is same as in DefaultABIInfo with an exception on the kernel
10602   // arguments handling.
10603   llvm::CallingConv::ID CC = FI.getCallingConvention();
10604 
10605   if (!getCXXABI().classifyReturnType(FI))
10606     FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
10607 
10608   for (auto &I : FI.arguments()) {
10609     if (CC == llvm::CallingConv::SPIR_KERNEL) {
10610       I.info = classifyKernelArgumentType(I.type);
10611     } else {
10612       I.info = classifyArgumentType(I.type);
10613     }
10614   }
10615 }
10616 
10617 namespace clang {
10618 namespace CodeGen {
computeSPIRKernelABIInfo(CodeGenModule & CGM,CGFunctionInfo & FI)10619 void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) {
10620   if (CGM.getTarget().getTriple().isSPIRV())
10621     SPIRVABIInfo(CGM.getTypes()).computeInfo(FI);
10622   else
10623     CommonSPIRABIInfo(CGM.getTypes()).computeInfo(FI);
10624 }
10625 }
10626 }
10627 
getOpenCLKernelCallingConv() const10628 unsigned CommonSPIRTargetCodeGenInfo::getOpenCLKernelCallingConv() const {
10629   return llvm::CallingConv::SPIR_KERNEL;
10630 }
10631 
setCUDAKernelCallingConvention(const FunctionType * & FT) const10632 void SPIRVTargetCodeGenInfo::setCUDAKernelCallingConvention(
10633     const FunctionType *&FT) const {
10634   // Convert HIP kernels to SPIR-V kernels.
10635   if (getABIInfo().getContext().getLangOpts().HIP) {
10636     FT = getABIInfo().getContext().adjustFunctionType(
10637         FT, FT->getExtInfo().withCallingConv(CC_OpenCLKernel));
10638     return;
10639   }
10640 }
10641 
10642 static bool appendType(SmallStringEnc &Enc, QualType QType,
10643                        const CodeGen::CodeGenModule &CGM,
10644                        TypeStringCache &TSC);
10645 
10646 /// Helper function for appendRecordType().
10647 /// Builds a SmallVector containing the encoded field types in declaration
10648 /// order.
extractFieldType(SmallVectorImpl<FieldEncoding> & FE,const RecordDecl * RD,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10649 static bool extractFieldType(SmallVectorImpl<FieldEncoding> &FE,
10650                              const RecordDecl *RD,
10651                              const CodeGen::CodeGenModule &CGM,
10652                              TypeStringCache &TSC) {
10653   for (const auto *Field : RD->fields()) {
10654     SmallStringEnc Enc;
10655     Enc += "m(";
10656     Enc += Field->getName();
10657     Enc += "){";
10658     if (Field->isBitField()) {
10659       Enc += "b(";
10660       llvm::raw_svector_ostream OS(Enc);
10661       OS << Field->getBitWidthValue(CGM.getContext());
10662       Enc += ':';
10663     }
10664     if (!appendType(Enc, Field->getType(), CGM, TSC))
10665       return false;
10666     if (Field->isBitField())
10667       Enc += ')';
10668     Enc += '}';
10669     FE.emplace_back(!Field->getName().empty(), Enc);
10670   }
10671   return true;
10672 }
10673 
10674 /// Appends structure and union types to Enc and adds encoding to cache.
10675 /// Recursively calls appendType (via extractFieldType) for each field.
10676 /// Union types have their fields ordered according to the ABI.
appendRecordType(SmallStringEnc & Enc,const RecordType * RT,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC,const IdentifierInfo * ID)10677 static bool appendRecordType(SmallStringEnc &Enc, const RecordType *RT,
10678                              const CodeGen::CodeGenModule &CGM,
10679                              TypeStringCache &TSC, const IdentifierInfo *ID) {
10680   // Append the cached TypeString if we have one.
10681   StringRef TypeString = TSC.lookupStr(ID);
10682   if (!TypeString.empty()) {
10683     Enc += TypeString;
10684     return true;
10685   }
10686 
10687   // Start to emit an incomplete TypeString.
10688   size_t Start = Enc.size();
10689   Enc += (RT->isUnionType()? 'u' : 's');
10690   Enc += '(';
10691   if (ID)
10692     Enc += ID->getName();
10693   Enc += "){";
10694 
10695   // We collect all encoded fields and order as necessary.
10696   bool IsRecursive = false;
10697   const RecordDecl *RD = RT->getDecl()->getDefinition();
10698   if (RD && !RD->field_empty()) {
10699     // An incomplete TypeString stub is placed in the cache for this RecordType
10700     // so that recursive calls to this RecordType will use it whilst building a
10701     // complete TypeString for this RecordType.
10702     SmallVector<FieldEncoding, 16> FE;
10703     std::string StubEnc(Enc.substr(Start).str());
10704     StubEnc += '}';  // StubEnc now holds a valid incomplete TypeString.
10705     TSC.addIncomplete(ID, std::move(StubEnc));
10706     if (!extractFieldType(FE, RD, CGM, TSC)) {
10707       (void) TSC.removeIncomplete(ID);
10708       return false;
10709     }
10710     IsRecursive = TSC.removeIncomplete(ID);
10711     // The ABI requires unions to be sorted but not structures.
10712     // See FieldEncoding::operator< for sort algorithm.
10713     if (RT->isUnionType())
10714       llvm::sort(FE);
10715     // We can now complete the TypeString.
10716     unsigned E = FE.size();
10717     for (unsigned I = 0; I != E; ++I) {
10718       if (I)
10719         Enc += ',';
10720       Enc += FE[I].str();
10721     }
10722   }
10723   Enc += '}';
10724   TSC.addIfComplete(ID, Enc.substr(Start), IsRecursive);
10725   return true;
10726 }
10727 
10728 /// Appends enum types to Enc and adds the encoding to the cache.
appendEnumType(SmallStringEnc & Enc,const EnumType * ET,TypeStringCache & TSC,const IdentifierInfo * ID)10729 static bool appendEnumType(SmallStringEnc &Enc, const EnumType *ET,
10730                            TypeStringCache &TSC,
10731                            const IdentifierInfo *ID) {
10732   // Append the cached TypeString if we have one.
10733   StringRef TypeString = TSC.lookupStr(ID);
10734   if (!TypeString.empty()) {
10735     Enc += TypeString;
10736     return true;
10737   }
10738 
10739   size_t Start = Enc.size();
10740   Enc += "e(";
10741   if (ID)
10742     Enc += ID->getName();
10743   Enc += "){";
10744 
10745   // We collect all encoded enumerations and order them alphanumerically.
10746   if (const EnumDecl *ED = ET->getDecl()->getDefinition()) {
10747     SmallVector<FieldEncoding, 16> FE;
10748     for (auto I = ED->enumerator_begin(), E = ED->enumerator_end(); I != E;
10749          ++I) {
10750       SmallStringEnc EnumEnc;
10751       EnumEnc += "m(";
10752       EnumEnc += I->getName();
10753       EnumEnc += "){";
10754       I->getInitVal().toString(EnumEnc);
10755       EnumEnc += '}';
10756       FE.push_back(FieldEncoding(!I->getName().empty(), EnumEnc));
10757     }
10758     llvm::sort(FE);
10759     unsigned E = FE.size();
10760     for (unsigned I = 0; I != E; ++I) {
10761       if (I)
10762         Enc += ',';
10763       Enc += FE[I].str();
10764     }
10765   }
10766   Enc += '}';
10767   TSC.addIfComplete(ID, Enc.substr(Start), false);
10768   return true;
10769 }
10770 
10771 /// Appends type's qualifier to Enc.
10772 /// This is done prior to appending the type's encoding.
appendQualifier(SmallStringEnc & Enc,QualType QT)10773 static void appendQualifier(SmallStringEnc &Enc, QualType QT) {
10774   // Qualifiers are emitted in alphabetical order.
10775   static const char *const Table[]={"","c:","r:","cr:","v:","cv:","rv:","crv:"};
10776   int Lookup = 0;
10777   if (QT.isConstQualified())
10778     Lookup += 1<<0;
10779   if (QT.isRestrictQualified())
10780     Lookup += 1<<1;
10781   if (QT.isVolatileQualified())
10782     Lookup += 1<<2;
10783   Enc += Table[Lookup];
10784 }
10785 
10786 /// Appends built-in types to Enc.
appendBuiltinType(SmallStringEnc & Enc,const BuiltinType * BT)10787 static bool appendBuiltinType(SmallStringEnc &Enc, const BuiltinType *BT) {
10788   const char *EncType;
10789   switch (BT->getKind()) {
10790     case BuiltinType::Void:
10791       EncType = "0";
10792       break;
10793     case BuiltinType::Bool:
10794       EncType = "b";
10795       break;
10796     case BuiltinType::Char_U:
10797       EncType = "uc";
10798       break;
10799     case BuiltinType::UChar:
10800       EncType = "uc";
10801       break;
10802     case BuiltinType::SChar:
10803       EncType = "sc";
10804       break;
10805     case BuiltinType::UShort:
10806       EncType = "us";
10807       break;
10808     case BuiltinType::Short:
10809       EncType = "ss";
10810       break;
10811     case BuiltinType::UInt:
10812       EncType = "ui";
10813       break;
10814     case BuiltinType::Int:
10815       EncType = "si";
10816       break;
10817     case BuiltinType::ULong:
10818       EncType = "ul";
10819       break;
10820     case BuiltinType::Long:
10821       EncType = "sl";
10822       break;
10823     case BuiltinType::ULongLong:
10824       EncType = "ull";
10825       break;
10826     case BuiltinType::LongLong:
10827       EncType = "sll";
10828       break;
10829     case BuiltinType::Float:
10830       EncType = "ft";
10831       break;
10832     case BuiltinType::Double:
10833       EncType = "d";
10834       break;
10835     case BuiltinType::LongDouble:
10836       EncType = "ld";
10837       break;
10838     default:
10839       return false;
10840   }
10841   Enc += EncType;
10842   return true;
10843 }
10844 
10845 /// Appends a pointer encoding to Enc before calling appendType for the pointee.
appendPointerType(SmallStringEnc & Enc,const PointerType * PT,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10846 static bool appendPointerType(SmallStringEnc &Enc, const PointerType *PT,
10847                               const CodeGen::CodeGenModule &CGM,
10848                               TypeStringCache &TSC) {
10849   Enc += "p(";
10850   if (!appendType(Enc, PT->getPointeeType(), CGM, TSC))
10851     return false;
10852   Enc += ')';
10853   return true;
10854 }
10855 
10856 /// Appends array encoding to Enc before calling appendType for the element.
appendArrayType(SmallStringEnc & Enc,QualType QT,const ArrayType * AT,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC,StringRef NoSizeEnc)10857 static bool appendArrayType(SmallStringEnc &Enc, QualType QT,
10858                             const ArrayType *AT,
10859                             const CodeGen::CodeGenModule &CGM,
10860                             TypeStringCache &TSC, StringRef NoSizeEnc) {
10861   if (AT->getSizeModifier() != ArrayType::Normal)
10862     return false;
10863   Enc += "a(";
10864   if (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(AT))
10865     CAT->getSize().toStringUnsigned(Enc);
10866   else
10867     Enc += NoSizeEnc; // Global arrays use "*", otherwise it is "".
10868   Enc += ':';
10869   // The Qualifiers should be attached to the type rather than the array.
10870   appendQualifier(Enc, QT);
10871   if (!appendType(Enc, AT->getElementType(), CGM, TSC))
10872     return false;
10873   Enc += ')';
10874   return true;
10875 }
10876 
10877 /// Appends a function encoding to Enc, calling appendType for the return type
10878 /// and the arguments.
appendFunctionType(SmallStringEnc & Enc,const FunctionType * FT,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10879 static bool appendFunctionType(SmallStringEnc &Enc, const FunctionType *FT,
10880                              const CodeGen::CodeGenModule &CGM,
10881                              TypeStringCache &TSC) {
10882   Enc += "f{";
10883   if (!appendType(Enc, FT->getReturnType(), CGM, TSC))
10884     return false;
10885   Enc += "}(";
10886   if (const FunctionProtoType *FPT = FT->getAs<FunctionProtoType>()) {
10887     // N.B. we are only interested in the adjusted param types.
10888     auto I = FPT->param_type_begin();
10889     auto E = FPT->param_type_end();
10890     if (I != E) {
10891       do {
10892         if (!appendType(Enc, *I, CGM, TSC))
10893           return false;
10894         ++I;
10895         if (I != E)
10896           Enc += ',';
10897       } while (I != E);
10898       if (FPT->isVariadic())
10899         Enc += ",va";
10900     } else {
10901       if (FPT->isVariadic())
10902         Enc += "va";
10903       else
10904         Enc += '0';
10905     }
10906   }
10907   Enc += ')';
10908   return true;
10909 }
10910 
10911 /// Handles the type's qualifier before dispatching a call to handle specific
10912 /// type encodings.
appendType(SmallStringEnc & Enc,QualType QType,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10913 static bool appendType(SmallStringEnc &Enc, QualType QType,
10914                        const CodeGen::CodeGenModule &CGM,
10915                        TypeStringCache &TSC) {
10916 
10917   QualType QT = QType.getCanonicalType();
10918 
10919   if (const ArrayType *AT = QT->getAsArrayTypeUnsafe())
10920     // The Qualifiers should be attached to the type rather than the array.
10921     // Thus we don't call appendQualifier() here.
10922     return appendArrayType(Enc, QT, AT, CGM, TSC, "");
10923 
10924   appendQualifier(Enc, QT);
10925 
10926   if (const BuiltinType *BT = QT->getAs<BuiltinType>())
10927     return appendBuiltinType(Enc, BT);
10928 
10929   if (const PointerType *PT = QT->getAs<PointerType>())
10930     return appendPointerType(Enc, PT, CGM, TSC);
10931 
10932   if (const EnumType *ET = QT->getAs<EnumType>())
10933     return appendEnumType(Enc, ET, TSC, QT.getBaseTypeIdentifier());
10934 
10935   if (const RecordType *RT = QT->getAsStructureType())
10936     return appendRecordType(Enc, RT, CGM, TSC, QT.getBaseTypeIdentifier());
10937 
10938   if (const RecordType *RT = QT->getAsUnionType())
10939     return appendRecordType(Enc, RT, CGM, TSC, QT.getBaseTypeIdentifier());
10940 
10941   if (const FunctionType *FT = QT->getAs<FunctionType>())
10942     return appendFunctionType(Enc, FT, CGM, TSC);
10943 
10944   return false;
10945 }
10946 
getTypeString(SmallStringEnc & Enc,const Decl * D,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10947 static bool getTypeString(SmallStringEnc &Enc, const Decl *D,
10948                           const CodeGen::CodeGenModule &CGM,
10949                           TypeStringCache &TSC) {
10950   if (!D)
10951     return false;
10952 
10953   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
10954     if (FD->getLanguageLinkage() != CLanguageLinkage)
10955       return false;
10956     return appendType(Enc, FD->getType(), CGM, TSC);
10957   }
10958 
10959   if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
10960     if (VD->getLanguageLinkage() != CLanguageLinkage)
10961       return false;
10962     QualType QT = VD->getType().getCanonicalType();
10963     if (const ArrayType *AT = QT->getAsArrayTypeUnsafe()) {
10964       // Global ArrayTypes are given a size of '*' if the size is unknown.
10965       // The Qualifiers should be attached to the type rather than the array.
10966       // Thus we don't call appendQualifier() here.
10967       return appendArrayType(Enc, QT, AT, CGM, TSC, "*");
10968     }
10969     return appendType(Enc, QT, CGM, TSC);
10970   }
10971   return false;
10972 }
10973 
10974 //===----------------------------------------------------------------------===//
10975 // RISCV ABI Implementation
10976 //===----------------------------------------------------------------------===//
10977 
10978 namespace {
10979 class RISCVABIInfo : public DefaultABIInfo {
10980 private:
10981   // Size of the integer ('x') registers in bits.
10982   unsigned XLen;
10983   // Size of the floating point ('f') registers in bits. Note that the target
10984   // ISA might have a wider FLen than the selected ABI (e.g. an RV32IF target
10985   // with soft float ABI has FLen==0).
10986   unsigned FLen;
10987   static const int NumArgGPRs = 8;
10988   static const int NumArgFPRs = 8;
10989   bool detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
10990                                       llvm::Type *&Field1Ty,
10991                                       CharUnits &Field1Off,
10992                                       llvm::Type *&Field2Ty,
10993                                       CharUnits &Field2Off) const;
10994 
10995 public:
RISCVABIInfo(CodeGen::CodeGenTypes & CGT,unsigned XLen,unsigned FLen)10996   RISCVABIInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen, unsigned FLen)
10997       : DefaultABIInfo(CGT), XLen(XLen), FLen(FLen) {}
10998 
10999   // DefaultABIInfo's classifyReturnType and classifyArgumentType are
11000   // non-virtual, but computeInfo is virtual, so we overload it.
11001   void computeInfo(CGFunctionInfo &FI) const override;
11002 
11003   ABIArgInfo classifyArgumentType(QualType Ty, bool IsFixed, int &ArgGPRsLeft,
11004                                   int &ArgFPRsLeft) const;
11005   ABIArgInfo classifyReturnType(QualType RetTy) const;
11006 
11007   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
11008                     QualType Ty) const override;
11009 
11010   ABIArgInfo extendType(QualType Ty) const;
11011 
11012   bool detectFPCCEligibleStruct(QualType Ty, llvm::Type *&Field1Ty,
11013                                 CharUnits &Field1Off, llvm::Type *&Field2Ty,
11014                                 CharUnits &Field2Off, int &NeededArgGPRs,
11015                                 int &NeededArgFPRs) const;
11016   ABIArgInfo coerceAndExpandFPCCEligibleStruct(llvm::Type *Field1Ty,
11017                                                CharUnits Field1Off,
11018                                                llvm::Type *Field2Ty,
11019                                                CharUnits Field2Off) const;
11020 };
11021 } // end anonymous namespace
11022 
computeInfo(CGFunctionInfo & FI) const11023 void RISCVABIInfo::computeInfo(CGFunctionInfo &FI) const {
11024   QualType RetTy = FI.getReturnType();
11025   if (!getCXXABI().classifyReturnType(FI))
11026     FI.getReturnInfo() = classifyReturnType(RetTy);
11027 
11028   // IsRetIndirect is true if classifyArgumentType indicated the value should
11029   // be passed indirect, or if the type size is a scalar greater than 2*XLen
11030   // and not a complex type with elements <= FLen. e.g. fp128 is passed direct
11031   // in LLVM IR, relying on the backend lowering code to rewrite the argument
11032   // list and pass indirectly on RV32.
11033   bool IsRetIndirect = FI.getReturnInfo().getKind() == ABIArgInfo::Indirect;
11034   if (!IsRetIndirect && RetTy->isScalarType() &&
11035       getContext().getTypeSize(RetTy) > (2 * XLen)) {
11036     if (RetTy->isComplexType() && FLen) {
11037       QualType EltTy = RetTy->castAs<ComplexType>()->getElementType();
11038       IsRetIndirect = getContext().getTypeSize(EltTy) > FLen;
11039     } else {
11040       // This is a normal scalar > 2*XLen, such as fp128 on RV32.
11041       IsRetIndirect = true;
11042     }
11043   }
11044 
11045   int ArgGPRsLeft = IsRetIndirect ? NumArgGPRs - 1 : NumArgGPRs;
11046   int ArgFPRsLeft = FLen ? NumArgFPRs : 0;
11047   int NumFixedArgs = FI.getNumRequiredArgs();
11048 
11049   int ArgNum = 0;
11050   for (auto &ArgInfo : FI.arguments()) {
11051     bool IsFixed = ArgNum < NumFixedArgs;
11052     ArgInfo.info =
11053         classifyArgumentType(ArgInfo.type, IsFixed, ArgGPRsLeft, ArgFPRsLeft);
11054     ArgNum++;
11055   }
11056 }
11057 
11058 // Returns true if the struct is a potential candidate for the floating point
11059 // calling convention. If this function returns true, the caller is
11060 // responsible for checking that if there is only a single field then that
11061 // field is a float.
detectFPCCEligibleStructHelper(QualType Ty,CharUnits CurOff,llvm::Type * & Field1Ty,CharUnits & Field1Off,llvm::Type * & Field2Ty,CharUnits & Field2Off) const11062 bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
11063                                                   llvm::Type *&Field1Ty,
11064                                                   CharUnits &Field1Off,
11065                                                   llvm::Type *&Field2Ty,
11066                                                   CharUnits &Field2Off) const {
11067   bool IsInt = Ty->isIntegralOrEnumerationType();
11068   bool IsFloat = Ty->isRealFloatingType();
11069 
11070   if (IsInt || IsFloat) {
11071     uint64_t Size = getContext().getTypeSize(Ty);
11072     if (IsInt && Size > XLen)
11073       return false;
11074     // Can't be eligible if larger than the FP registers. Half precision isn't
11075     // currently supported on RISC-V and the ABI hasn't been confirmed, so
11076     // default to the integer ABI in that case.
11077     if (IsFloat && (Size > FLen || Size < 32))
11078       return false;
11079     // Can't be eligible if an integer type was already found (int+int pairs
11080     // are not eligible).
11081     if (IsInt && Field1Ty && Field1Ty->isIntegerTy())
11082       return false;
11083     if (!Field1Ty) {
11084       Field1Ty = CGT.ConvertType(Ty);
11085       Field1Off = CurOff;
11086       return true;
11087     }
11088     if (!Field2Ty) {
11089       Field2Ty = CGT.ConvertType(Ty);
11090       Field2Off = CurOff;
11091       return true;
11092     }
11093     return false;
11094   }
11095 
11096   if (auto CTy = Ty->getAs<ComplexType>()) {
11097     if (Field1Ty)
11098       return false;
11099     QualType EltTy = CTy->getElementType();
11100     if (getContext().getTypeSize(EltTy) > FLen)
11101       return false;
11102     Field1Ty = CGT.ConvertType(EltTy);
11103     Field1Off = CurOff;
11104     Field2Ty = Field1Ty;
11105     Field2Off = Field1Off + getContext().getTypeSizeInChars(EltTy);
11106     return true;
11107   }
11108 
11109   if (const ConstantArrayType *ATy = getContext().getAsConstantArrayType(Ty)) {
11110     uint64_t ArraySize = ATy->getSize().getZExtValue();
11111     QualType EltTy = ATy->getElementType();
11112     CharUnits EltSize = getContext().getTypeSizeInChars(EltTy);
11113     for (uint64_t i = 0; i < ArraySize; ++i) {
11114       bool Ret = detectFPCCEligibleStructHelper(EltTy, CurOff, Field1Ty,
11115                                                 Field1Off, Field2Ty, Field2Off);
11116       if (!Ret)
11117         return false;
11118       CurOff += EltSize;
11119     }
11120     return true;
11121   }
11122 
11123   if (const auto *RTy = Ty->getAs<RecordType>()) {
11124     // Structures with either a non-trivial destructor or a non-trivial
11125     // copy constructor are not eligible for the FP calling convention.
11126     if (getRecordArgABI(Ty, CGT.getCXXABI()))
11127       return false;
11128     if (isEmptyRecord(getContext(), Ty, true))
11129       return true;
11130     const RecordDecl *RD = RTy->getDecl();
11131     // Unions aren't eligible unless they're empty (which is caught above).
11132     if (RD->isUnion())
11133       return false;
11134     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
11135     // If this is a C++ record, check the bases first.
11136     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
11137       for (const CXXBaseSpecifier &B : CXXRD->bases()) {
11138         const auto *BDecl =
11139             cast<CXXRecordDecl>(B.getType()->castAs<RecordType>()->getDecl());
11140         CharUnits BaseOff = Layout.getBaseClassOffset(BDecl);
11141         bool Ret = detectFPCCEligibleStructHelper(B.getType(), CurOff + BaseOff,
11142                                                   Field1Ty, Field1Off, Field2Ty,
11143                                                   Field2Off);
11144         if (!Ret)
11145           return false;
11146       }
11147     }
11148     int ZeroWidthBitFieldCount = 0;
11149     for (const FieldDecl *FD : RD->fields()) {
11150       uint64_t FieldOffInBits = Layout.getFieldOffset(FD->getFieldIndex());
11151       QualType QTy = FD->getType();
11152       if (FD->isBitField()) {
11153         unsigned BitWidth = FD->getBitWidthValue(getContext());
11154         // Allow a bitfield with a type greater than XLen as long as the
11155         // bitwidth is XLen or less.
11156         if (getContext().getTypeSize(QTy) > XLen && BitWidth <= XLen)
11157           QTy = getContext().getIntTypeForBitwidth(XLen, false);
11158         if (BitWidth == 0) {
11159           ZeroWidthBitFieldCount++;
11160           continue;
11161         }
11162       }
11163 
11164       bool Ret = detectFPCCEligibleStructHelper(
11165           QTy, CurOff + getContext().toCharUnitsFromBits(FieldOffInBits),
11166           Field1Ty, Field1Off, Field2Ty, Field2Off);
11167       if (!Ret)
11168         return false;
11169 
11170       // As a quirk of the ABI, zero-width bitfields aren't ignored for fp+fp
11171       // or int+fp structs, but are ignored for a struct with an fp field and
11172       // any number of zero-width bitfields.
11173       if (Field2Ty && ZeroWidthBitFieldCount > 0)
11174         return false;
11175     }
11176     return Field1Ty != nullptr;
11177   }
11178 
11179   return false;
11180 }
11181 
11182 // Determine if a struct is eligible for passing according to the floating
11183 // point calling convention (i.e., when flattened it contains a single fp
11184 // value, fp+fp, or int+fp of appropriate size). If so, NeededArgFPRs and
11185 // NeededArgGPRs are incremented appropriately.
detectFPCCEligibleStruct(QualType Ty,llvm::Type * & Field1Ty,CharUnits & Field1Off,llvm::Type * & Field2Ty,CharUnits & Field2Off,int & NeededArgGPRs,int & NeededArgFPRs) const11186 bool RISCVABIInfo::detectFPCCEligibleStruct(QualType Ty, llvm::Type *&Field1Ty,
11187                                             CharUnits &Field1Off,
11188                                             llvm::Type *&Field2Ty,
11189                                             CharUnits &Field2Off,
11190                                             int &NeededArgGPRs,
11191                                             int &NeededArgFPRs) const {
11192   Field1Ty = nullptr;
11193   Field2Ty = nullptr;
11194   NeededArgGPRs = 0;
11195   NeededArgFPRs = 0;
11196   bool IsCandidate = detectFPCCEligibleStructHelper(
11197       Ty, CharUnits::Zero(), Field1Ty, Field1Off, Field2Ty, Field2Off);
11198   // Not really a candidate if we have a single int but no float.
11199   if (Field1Ty && !Field2Ty && !Field1Ty->isFloatingPointTy())
11200     return false;
11201   if (!IsCandidate)
11202     return false;
11203   if (Field1Ty && Field1Ty->isFloatingPointTy())
11204     NeededArgFPRs++;
11205   else if (Field1Ty)
11206     NeededArgGPRs++;
11207   if (Field2Ty && Field2Ty->isFloatingPointTy())
11208     NeededArgFPRs++;
11209   else if (Field2Ty)
11210     NeededArgGPRs++;
11211   return true;
11212 }
11213 
11214 // Call getCoerceAndExpand for the two-element flattened struct described by
11215 // Field1Ty, Field1Off, Field2Ty, Field2Off. This method will create an
11216 // appropriate coerceToType and unpaddedCoerceToType.
coerceAndExpandFPCCEligibleStruct(llvm::Type * Field1Ty,CharUnits Field1Off,llvm::Type * Field2Ty,CharUnits Field2Off) const11217 ABIArgInfo RISCVABIInfo::coerceAndExpandFPCCEligibleStruct(
11218     llvm::Type *Field1Ty, CharUnits Field1Off, llvm::Type *Field2Ty,
11219     CharUnits Field2Off) const {
11220   SmallVector<llvm::Type *, 3> CoerceElts;
11221   SmallVector<llvm::Type *, 2> UnpaddedCoerceElts;
11222   if (!Field1Off.isZero())
11223     CoerceElts.push_back(llvm::ArrayType::get(
11224         llvm::Type::getInt8Ty(getVMContext()), Field1Off.getQuantity()));
11225 
11226   CoerceElts.push_back(Field1Ty);
11227   UnpaddedCoerceElts.push_back(Field1Ty);
11228 
11229   if (!Field2Ty) {
11230     return ABIArgInfo::getCoerceAndExpand(
11231         llvm::StructType::get(getVMContext(), CoerceElts, !Field1Off.isZero()),
11232         UnpaddedCoerceElts[0]);
11233   }
11234 
11235   CharUnits Field2Align =
11236       CharUnits::fromQuantity(getDataLayout().getABITypeAlign(Field2Ty));
11237   CharUnits Field1End = Field1Off +
11238       CharUnits::fromQuantity(getDataLayout().getTypeStoreSize(Field1Ty));
11239   CharUnits Field2OffNoPadNoPack = Field1End.alignTo(Field2Align);
11240 
11241   CharUnits Padding = CharUnits::Zero();
11242   if (Field2Off > Field2OffNoPadNoPack)
11243     Padding = Field2Off - Field2OffNoPadNoPack;
11244   else if (Field2Off != Field2Align && Field2Off > Field1End)
11245     Padding = Field2Off - Field1End;
11246 
11247   bool IsPacked = !Field2Off.isMultipleOf(Field2Align);
11248 
11249   if (!Padding.isZero())
11250     CoerceElts.push_back(llvm::ArrayType::get(
11251         llvm::Type::getInt8Ty(getVMContext()), Padding.getQuantity()));
11252 
11253   CoerceElts.push_back(Field2Ty);
11254   UnpaddedCoerceElts.push_back(Field2Ty);
11255 
11256   auto CoerceToType =
11257       llvm::StructType::get(getVMContext(), CoerceElts, IsPacked);
11258   auto UnpaddedCoerceToType =
11259       llvm::StructType::get(getVMContext(), UnpaddedCoerceElts, IsPacked);
11260 
11261   return ABIArgInfo::getCoerceAndExpand(CoerceToType, UnpaddedCoerceToType);
11262 }
11263 
classifyArgumentType(QualType Ty,bool IsFixed,int & ArgGPRsLeft,int & ArgFPRsLeft) const11264 ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
11265                                               int &ArgGPRsLeft,
11266                                               int &ArgFPRsLeft) const {
11267   assert(ArgGPRsLeft <= NumArgGPRs && "Arg GPR tracking underflow");
11268   Ty = useFirstFieldIfTransparentUnion(Ty);
11269 
11270   // Structures with either a non-trivial destructor or a non-trivial
11271   // copy constructor are always passed indirectly.
11272   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
11273     if (ArgGPRsLeft)
11274       ArgGPRsLeft -= 1;
11275     return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
11276                                            CGCXXABI::RAA_DirectInMemory);
11277   }
11278 
11279   // Ignore empty structs/unions.
11280   if (isEmptyRecord(getContext(), Ty, true))
11281     return ABIArgInfo::getIgnore();
11282 
11283   uint64_t Size = getContext().getTypeSize(Ty);
11284 
11285   // Pass floating point values via FPRs if possible.
11286   if (IsFixed && Ty->isFloatingType() && !Ty->isComplexType() &&
11287       FLen >= Size && ArgFPRsLeft) {
11288     ArgFPRsLeft--;
11289     return ABIArgInfo::getDirect();
11290   }
11291 
11292   // Complex types for the hard float ABI must be passed direct rather than
11293   // using CoerceAndExpand.
11294   if (IsFixed && Ty->isComplexType() && FLen && ArgFPRsLeft >= 2) {
11295     QualType EltTy = Ty->castAs<ComplexType>()->getElementType();
11296     if (getContext().getTypeSize(EltTy) <= FLen) {
11297       ArgFPRsLeft -= 2;
11298       return ABIArgInfo::getDirect();
11299     }
11300   }
11301 
11302   if (IsFixed && FLen && Ty->isStructureOrClassType()) {
11303     llvm::Type *Field1Ty = nullptr;
11304     llvm::Type *Field2Ty = nullptr;
11305     CharUnits Field1Off = CharUnits::Zero();
11306     CharUnits Field2Off = CharUnits::Zero();
11307     int NeededArgGPRs = 0;
11308     int NeededArgFPRs = 0;
11309     bool IsCandidate =
11310         detectFPCCEligibleStruct(Ty, Field1Ty, Field1Off, Field2Ty, Field2Off,
11311                                  NeededArgGPRs, NeededArgFPRs);
11312     if (IsCandidate && NeededArgGPRs <= ArgGPRsLeft &&
11313         NeededArgFPRs <= ArgFPRsLeft) {
11314       ArgGPRsLeft -= NeededArgGPRs;
11315       ArgFPRsLeft -= NeededArgFPRs;
11316       return coerceAndExpandFPCCEligibleStruct(Field1Ty, Field1Off, Field2Ty,
11317                                                Field2Off);
11318     }
11319   }
11320 
11321   uint64_t NeededAlign = getContext().getTypeAlign(Ty);
11322   // Determine the number of GPRs needed to pass the current argument
11323   // according to the ABI. 2*XLen-aligned varargs are passed in "aligned"
11324   // register pairs, so may consume 3 registers.
11325   int NeededArgGPRs = 1;
11326   if (!IsFixed && NeededAlign == 2 * XLen)
11327     NeededArgGPRs = 2 + (ArgGPRsLeft % 2);
11328   else if (Size > XLen && Size <= 2 * XLen)
11329     NeededArgGPRs = 2;
11330 
11331   if (NeededArgGPRs > ArgGPRsLeft) {
11332     NeededArgGPRs = ArgGPRsLeft;
11333   }
11334 
11335   ArgGPRsLeft -= NeededArgGPRs;
11336 
11337   if (!isAggregateTypeForABI(Ty) && !Ty->isVectorType()) {
11338     // Treat an enum type as its underlying type.
11339     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
11340       Ty = EnumTy->getDecl()->getIntegerType();
11341 
11342     // All integral types are promoted to XLen width
11343     if (Size < XLen && Ty->isIntegralOrEnumerationType()) {
11344       return extendType(Ty);
11345     }
11346 
11347     if (const auto *EIT = Ty->getAs<BitIntType>()) {
11348       if (EIT->getNumBits() < XLen)
11349         return extendType(Ty);
11350       if (EIT->getNumBits() > 128 ||
11351           (!getContext().getTargetInfo().hasInt128Type() &&
11352            EIT->getNumBits() > 64))
11353         return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
11354     }
11355 
11356     return ABIArgInfo::getDirect();
11357   }
11358 
11359   // Aggregates which are <= 2*XLen will be passed in registers if possible,
11360   // so coerce to integers.
11361   if (Size <= 2 * XLen) {
11362     unsigned Alignment = getContext().getTypeAlign(Ty);
11363 
11364     // Use a single XLen int if possible, 2*XLen if 2*XLen alignment is
11365     // required, and a 2-element XLen array if only XLen alignment is required.
11366     if (Size <= XLen) {
11367       return ABIArgInfo::getDirect(
11368           llvm::IntegerType::get(getVMContext(), XLen));
11369     } else if (Alignment == 2 * XLen) {
11370       return ABIArgInfo::getDirect(
11371           llvm::IntegerType::get(getVMContext(), 2 * XLen));
11372     } else {
11373       return ABIArgInfo::getDirect(llvm::ArrayType::get(
11374           llvm::IntegerType::get(getVMContext(), XLen), 2));
11375     }
11376   }
11377   return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
11378 }
11379 
classifyReturnType(QualType RetTy) const11380 ABIArgInfo RISCVABIInfo::classifyReturnType(QualType RetTy) const {
11381   if (RetTy->isVoidType())
11382     return ABIArgInfo::getIgnore();
11383 
11384   int ArgGPRsLeft = 2;
11385   int ArgFPRsLeft = FLen ? 2 : 0;
11386 
11387   // The rules for return and argument types are the same, so defer to
11388   // classifyArgumentType.
11389   return classifyArgumentType(RetTy, /*IsFixed=*/true, ArgGPRsLeft,
11390                               ArgFPRsLeft);
11391 }
11392 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const11393 Address RISCVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
11394                                 QualType Ty) const {
11395   CharUnits SlotSize = CharUnits::fromQuantity(XLen / 8);
11396 
11397   // Empty records are ignored for parameter passing purposes.
11398   if (isEmptyRecord(getContext(), Ty, true)) {
11399     Address Addr = Address(CGF.Builder.CreateLoad(VAListAddr),
11400                            getVAListElementType(CGF), SlotSize);
11401     Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
11402     return Addr;
11403   }
11404 
11405   auto TInfo = getContext().getTypeInfoInChars(Ty);
11406 
11407   // Arguments bigger than 2*Xlen bytes are passed indirectly.
11408   bool IsIndirect = TInfo.Width > 2 * SlotSize;
11409 
11410   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TInfo,
11411                           SlotSize, /*AllowHigherAlign=*/true);
11412 }
11413 
extendType(QualType Ty) const11414 ABIArgInfo RISCVABIInfo::extendType(QualType Ty) const {
11415   int TySize = getContext().getTypeSize(Ty);
11416   // RV64 ABI requires unsigned 32 bit integers to be sign extended.
11417   if (XLen == 64 && Ty->isUnsignedIntegerOrEnumerationType() && TySize == 32)
11418     return ABIArgInfo::getSignExtend(Ty);
11419   return ABIArgInfo::getExtend(Ty);
11420 }
11421 
11422 namespace {
11423 class RISCVTargetCodeGenInfo : public TargetCodeGenInfo {
11424 public:
RISCVTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,unsigned XLen,unsigned FLen)11425   RISCVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen,
11426                          unsigned FLen)
11427       : TargetCodeGenInfo(std::make_unique<RISCVABIInfo>(CGT, XLen, FLen)) {}
11428 
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const11429   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
11430                            CodeGen::CodeGenModule &CGM) const override {
11431     const auto *FD = dyn_cast_or_null<FunctionDecl>(D);
11432     if (!FD) return;
11433 
11434     const auto *Attr = FD->getAttr<RISCVInterruptAttr>();
11435     if (!Attr)
11436       return;
11437 
11438     const char *Kind;
11439     switch (Attr->getInterrupt()) {
11440     case RISCVInterruptAttr::user: Kind = "user"; break;
11441     case RISCVInterruptAttr::supervisor: Kind = "supervisor"; break;
11442     case RISCVInterruptAttr::machine: Kind = "machine"; break;
11443     }
11444 
11445     auto *Fn = cast<llvm::Function>(GV);
11446 
11447     Fn->addFnAttr("interrupt", Kind);
11448   }
11449 };
11450 } // namespace
11451 
11452 //===----------------------------------------------------------------------===//
11453 // VE ABI Implementation.
11454 //
11455 namespace {
11456 class VEABIInfo : public DefaultABIInfo {
11457 public:
VEABIInfo(CodeGenTypes & CGT)11458   VEABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
11459 
11460 private:
11461   ABIArgInfo classifyReturnType(QualType RetTy) const;
11462   ABIArgInfo classifyArgumentType(QualType RetTy) const;
11463   void computeInfo(CGFunctionInfo &FI) const override;
11464 };
11465 } // end anonymous namespace
11466 
classifyReturnType(QualType Ty) const11467 ABIArgInfo VEABIInfo::classifyReturnType(QualType Ty) const {
11468   if (Ty->isAnyComplexType())
11469     return ABIArgInfo::getDirect();
11470   uint64_t Size = getContext().getTypeSize(Ty);
11471   if (Size < 64 && Ty->isIntegerType())
11472     return ABIArgInfo::getExtend(Ty);
11473   return DefaultABIInfo::classifyReturnType(Ty);
11474 }
11475 
classifyArgumentType(QualType Ty) const11476 ABIArgInfo VEABIInfo::classifyArgumentType(QualType Ty) const {
11477   if (Ty->isAnyComplexType())
11478     return ABIArgInfo::getDirect();
11479   uint64_t Size = getContext().getTypeSize(Ty);
11480   if (Size < 64 && Ty->isIntegerType())
11481     return ABIArgInfo::getExtend(Ty);
11482   return DefaultABIInfo::classifyArgumentType(Ty);
11483 }
11484 
computeInfo(CGFunctionInfo & FI) const11485 void VEABIInfo::computeInfo(CGFunctionInfo &FI) const {
11486   FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
11487   for (auto &Arg : FI.arguments())
11488     Arg.info = classifyArgumentType(Arg.type);
11489 }
11490 
11491 namespace {
11492 class VETargetCodeGenInfo : public TargetCodeGenInfo {
11493 public:
VETargetCodeGenInfo(CodeGenTypes & CGT)11494   VETargetCodeGenInfo(CodeGenTypes &CGT)
11495       : TargetCodeGenInfo(std::make_unique<VEABIInfo>(CGT)) {}
11496   // VE ABI requires the arguments of variadic and prototype-less functions
11497   // are passed in both registers and memory.
isNoProtoCallVariadic(const CallArgList & args,const FunctionNoProtoType * fnType) const11498   bool isNoProtoCallVariadic(const CallArgList &args,
11499                              const FunctionNoProtoType *fnType) const override {
11500     return true;
11501   }
11502 };
11503 } // end anonymous namespace
11504 
11505 //===----------------------------------------------------------------------===//
11506 // CSKY ABI Implementation
11507 //===----------------------------------------------------------------------===//
11508 namespace {
11509 class CSKYABIInfo : public DefaultABIInfo {
11510   static const int NumArgGPRs = 4;
11511   static const int NumArgFPRs = 4;
11512 
11513   static const unsigned XLen = 32;
11514   unsigned FLen;
11515 
11516 public:
CSKYABIInfo(CodeGen::CodeGenTypes & CGT,unsigned FLen)11517   CSKYABIInfo(CodeGen::CodeGenTypes &CGT, unsigned FLen)
11518       : DefaultABIInfo(CGT), FLen(FLen) {}
11519 
11520   void computeInfo(CGFunctionInfo &FI) const override;
11521   ABIArgInfo classifyArgumentType(QualType Ty, int &ArgGPRsLeft,
11522                                   int &ArgFPRsLeft,
11523                                   bool isReturnType = false) const;
11524   ABIArgInfo classifyReturnType(QualType RetTy) const;
11525 
11526   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
11527                     QualType Ty) const override;
11528 };
11529 
11530 } // end anonymous namespace
11531 
computeInfo(CGFunctionInfo & FI) const11532 void CSKYABIInfo::computeInfo(CGFunctionInfo &FI) const {
11533   QualType RetTy = FI.getReturnType();
11534   if (!getCXXABI().classifyReturnType(FI))
11535     FI.getReturnInfo() = classifyReturnType(RetTy);
11536 
11537   bool IsRetIndirect = FI.getReturnInfo().getKind() == ABIArgInfo::Indirect;
11538 
11539   // We must track the number of GPRs used in order to conform to the CSKY
11540   // ABI, as integer scalars passed in registers should have signext/zeroext
11541   // when promoted.
11542   int ArgGPRsLeft = IsRetIndirect ? NumArgGPRs - 1 : NumArgGPRs;
11543   int ArgFPRsLeft = FLen ? NumArgFPRs : 0;
11544 
11545   for (auto &ArgInfo : FI.arguments()) {
11546     ArgInfo.info = classifyArgumentType(ArgInfo.type, ArgGPRsLeft, ArgFPRsLeft);
11547   }
11548 }
11549 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const11550 Address CSKYABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
11551                                QualType Ty) const {
11552   CharUnits SlotSize = CharUnits::fromQuantity(XLen / 8);
11553 
11554   // Empty records are ignored for parameter passing purposes.
11555   if (isEmptyRecord(getContext(), Ty, true)) {
11556     Address Addr = Address(CGF.Builder.CreateLoad(VAListAddr),
11557                            getVAListElementType(CGF), SlotSize);
11558     Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
11559     return Addr;
11560   }
11561 
11562   auto TInfo = getContext().getTypeInfoInChars(Ty);
11563 
11564   return emitVoidPtrVAArg(CGF, VAListAddr, Ty, false, TInfo, SlotSize,
11565                           /*AllowHigherAlign=*/true);
11566 }
11567 
classifyArgumentType(QualType Ty,int & ArgGPRsLeft,int & ArgFPRsLeft,bool isReturnType) const11568 ABIArgInfo CSKYABIInfo::classifyArgumentType(QualType Ty, int &ArgGPRsLeft,
11569                                              int &ArgFPRsLeft,
11570                                              bool isReturnType) const {
11571   assert(ArgGPRsLeft <= NumArgGPRs && "Arg GPR tracking underflow");
11572   Ty = useFirstFieldIfTransparentUnion(Ty);
11573 
11574   // Structures with either a non-trivial destructor or a non-trivial
11575   // copy constructor are always passed indirectly.
11576   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
11577     if (ArgGPRsLeft)
11578       ArgGPRsLeft -= 1;
11579     return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
11580                                            CGCXXABI::RAA_DirectInMemory);
11581   }
11582 
11583   // Ignore empty structs/unions.
11584   if (isEmptyRecord(getContext(), Ty, true))
11585     return ABIArgInfo::getIgnore();
11586 
11587   if (!Ty->getAsUnionType())
11588     if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
11589       return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
11590 
11591   uint64_t Size = getContext().getTypeSize(Ty);
11592   // Pass floating point values via FPRs if possible.
11593   if (Ty->isFloatingType() && !Ty->isComplexType() && FLen >= Size &&
11594       ArgFPRsLeft) {
11595     ArgFPRsLeft--;
11596     return ABIArgInfo::getDirect();
11597   }
11598 
11599   // Complex types for the hard float ABI must be passed direct rather than
11600   // using CoerceAndExpand.
11601   if (Ty->isComplexType() && FLen && !isReturnType) {
11602     QualType EltTy = Ty->castAs<ComplexType>()->getElementType();
11603     if (getContext().getTypeSize(EltTy) <= FLen) {
11604       ArgFPRsLeft -= 2;
11605       return ABIArgInfo::getDirect();
11606     }
11607   }
11608 
11609   if (!isAggregateTypeForABI(Ty)) {
11610     // Treat an enum type as its underlying type.
11611     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
11612       Ty = EnumTy->getDecl()->getIntegerType();
11613 
11614     // All integral types are promoted to XLen width, unless passed on the
11615     // stack.
11616     if (Size < XLen && Ty->isIntegralOrEnumerationType())
11617       return ABIArgInfo::getExtend(Ty);
11618 
11619     if (const auto *EIT = Ty->getAs<BitIntType>()) {
11620       if (EIT->getNumBits() < XLen)
11621         return ABIArgInfo::getExtend(Ty);
11622     }
11623 
11624     return ABIArgInfo::getDirect();
11625   }
11626 
11627   // For argument type, the first 4*XLen parts of aggregate will be passed
11628   // in registers, and the rest will be passed in stack.
11629   // So we can coerce to integers directly and let backend handle it correctly.
11630   // For return type, aggregate which <= 2*XLen will be returned in registers.
11631   // Otherwise, aggregate will be returned indirectly.
11632   if (!isReturnType || (isReturnType && Size <= 2 * XLen)) {
11633     if (Size <= XLen) {
11634       return ABIArgInfo::getDirect(
11635           llvm::IntegerType::get(getVMContext(), XLen));
11636     } else {
11637       return ABIArgInfo::getDirect(llvm::ArrayType::get(
11638           llvm::IntegerType::get(getVMContext(), XLen), (Size + 31) / XLen));
11639     }
11640   }
11641   return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
11642 }
11643 
classifyReturnType(QualType RetTy) const11644 ABIArgInfo CSKYABIInfo::classifyReturnType(QualType RetTy) const {
11645   if (RetTy->isVoidType())
11646     return ABIArgInfo::getIgnore();
11647 
11648   int ArgGPRsLeft = 2;
11649   int ArgFPRsLeft = FLen ? 1 : 0;
11650 
11651   // The rules for return and argument types are the same, so defer to
11652   // classifyArgumentType.
11653   return classifyArgumentType(RetTy, ArgGPRsLeft, ArgFPRsLeft, true);
11654 }
11655 
11656 namespace {
11657 class CSKYTargetCodeGenInfo : public TargetCodeGenInfo {
11658 public:
CSKYTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,unsigned FLen)11659   CSKYTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned FLen)
11660       : TargetCodeGenInfo(std::make_unique<CSKYABIInfo>(CGT, FLen)) {}
11661 };
11662 } // end anonymous namespace
11663 
11664 //===----------------------------------------------------------------------===//
11665 // BPF ABI Implementation
11666 //===----------------------------------------------------------------------===//
11667 
11668 namespace {
11669 
11670 class BPFABIInfo : public DefaultABIInfo {
11671 public:
BPFABIInfo(CodeGenTypes & CGT)11672   BPFABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
11673 
classifyArgumentType(QualType Ty) const11674   ABIArgInfo classifyArgumentType(QualType Ty) const {
11675     Ty = useFirstFieldIfTransparentUnion(Ty);
11676 
11677     if (isAggregateTypeForABI(Ty)) {
11678       uint64_t Bits = getContext().getTypeSize(Ty);
11679       if (Bits == 0)
11680         return ABIArgInfo::getIgnore();
11681 
11682       // If the aggregate needs 1 or 2 registers, do not use reference.
11683       if (Bits <= 128) {
11684         llvm::Type *CoerceTy;
11685         if (Bits <= 64) {
11686           CoerceTy =
11687               llvm::IntegerType::get(getVMContext(), llvm::alignTo(Bits, 8));
11688         } else {
11689           llvm::Type *RegTy = llvm::IntegerType::get(getVMContext(), 64);
11690           CoerceTy = llvm::ArrayType::get(RegTy, 2);
11691         }
11692         return ABIArgInfo::getDirect(CoerceTy);
11693       } else {
11694         return getNaturalAlignIndirect(Ty);
11695       }
11696     }
11697 
11698     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
11699       Ty = EnumTy->getDecl()->getIntegerType();
11700 
11701     ASTContext &Context = getContext();
11702     if (const auto *EIT = Ty->getAs<BitIntType>())
11703       if (EIT->getNumBits() > Context.getTypeSize(Context.Int128Ty))
11704         return getNaturalAlignIndirect(Ty);
11705 
11706     return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
11707                                               : ABIArgInfo::getDirect());
11708   }
11709 
classifyReturnType(QualType RetTy) const11710   ABIArgInfo classifyReturnType(QualType RetTy) const {
11711     if (RetTy->isVoidType())
11712       return ABIArgInfo::getIgnore();
11713 
11714     if (isAggregateTypeForABI(RetTy))
11715       return getNaturalAlignIndirect(RetTy);
11716 
11717     // Treat an enum type as its underlying type.
11718     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
11719       RetTy = EnumTy->getDecl()->getIntegerType();
11720 
11721     ASTContext &Context = getContext();
11722     if (const auto *EIT = RetTy->getAs<BitIntType>())
11723       if (EIT->getNumBits() > Context.getTypeSize(Context.Int128Ty))
11724         return getNaturalAlignIndirect(RetTy);
11725 
11726     // Caller will do necessary sign/zero extension.
11727     return ABIArgInfo::getDirect();
11728   }
11729 
computeInfo(CGFunctionInfo & FI) const11730   void computeInfo(CGFunctionInfo &FI) const override {
11731     FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
11732     for (auto &I : FI.arguments())
11733       I.info = classifyArgumentType(I.type);
11734   }
11735 
11736 };
11737 
11738 class BPFTargetCodeGenInfo : public TargetCodeGenInfo {
11739 public:
BPFTargetCodeGenInfo(CodeGenTypes & CGT)11740   BPFTargetCodeGenInfo(CodeGenTypes &CGT)
11741       : TargetCodeGenInfo(std::make_unique<BPFABIInfo>(CGT)) {}
11742 
getABIInfo() const11743   const BPFABIInfo &getABIInfo() const {
11744     return static_cast<const BPFABIInfo&>(TargetCodeGenInfo::getABIInfo());
11745   }
11746 };
11747 
11748 }
11749 
11750 // LoongArch ABI Implementation. Documented at
11751 // https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html
11752 //
11753 //===----------------------------------------------------------------------===//
11754 
11755 namespace {
11756 class LoongArchABIInfo : public DefaultABIInfo {
11757 private:
11758   // Size of the integer ('r') registers in bits.
11759   unsigned GRLen;
11760   // Size of the floating point ('f') registers in bits.
11761   unsigned FRLen;
11762   // Number of general-purpose argument registers.
11763   static const int NumGARs = 8;
11764   // Number of floating-point argument registers.
11765   static const int NumFARs = 8;
11766   bool detectFARsEligibleStructHelper(QualType Ty, CharUnits CurOff,
11767                                       llvm::Type *&Field1Ty,
11768                                       CharUnits &Field1Off,
11769                                       llvm::Type *&Field2Ty,
11770                                       CharUnits &Field2Off) const;
11771 
11772 public:
LoongArchABIInfo(CodeGen::CodeGenTypes & CGT,unsigned GRLen,unsigned FRLen)11773   LoongArchABIInfo(CodeGen::CodeGenTypes &CGT, unsigned GRLen, unsigned FRLen)
11774       : DefaultABIInfo(CGT), GRLen(GRLen), FRLen(FRLen) {}
11775 
11776   void computeInfo(CGFunctionInfo &FI) const override;
11777 
11778   ABIArgInfo classifyArgumentType(QualType Ty, bool IsFixed, int &GARsLeft,
11779                                   int &FARsLeft) const;
11780   ABIArgInfo classifyReturnType(QualType RetTy) const;
11781 
11782   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
11783                     QualType Ty) const override;
11784 
11785   ABIArgInfo extendType(QualType Ty) const;
11786 
11787   bool detectFARsEligibleStruct(QualType Ty, llvm::Type *&Field1Ty,
11788                                 CharUnits &Field1Off, llvm::Type *&Field2Ty,
11789                                 CharUnits &Field2Off, int &NeededArgGPRs,
11790                                 int &NeededArgFPRs) const;
11791   ABIArgInfo coerceAndExpandFARsEligibleStruct(llvm::Type *Field1Ty,
11792                                                CharUnits Field1Off,
11793                                                llvm::Type *Field2Ty,
11794                                                CharUnits Field2Off) const;
11795 };
11796 } // end anonymous namespace
11797 
computeInfo(CGFunctionInfo & FI) const11798 void LoongArchABIInfo::computeInfo(CGFunctionInfo &FI) const {
11799   QualType RetTy = FI.getReturnType();
11800   if (!getCXXABI().classifyReturnType(FI))
11801     FI.getReturnInfo() = classifyReturnType(RetTy);
11802 
11803   // IsRetIndirect is true if classifyArgumentType indicated the value should
11804   // be passed indirect, or if the type size is a scalar greater than 2*GRLen
11805   // and not a complex type with elements <= FRLen. e.g. fp128 is passed direct
11806   // in LLVM IR, relying on the backend lowering code to rewrite the argument
11807   // list and pass indirectly on LA32.
11808   bool IsRetIndirect = FI.getReturnInfo().getKind() == ABIArgInfo::Indirect;
11809   if (!IsRetIndirect && RetTy->isScalarType() &&
11810       getContext().getTypeSize(RetTy) > (2 * GRLen)) {
11811     if (RetTy->isComplexType() && FRLen) {
11812       QualType EltTy = RetTy->castAs<ComplexType>()->getElementType();
11813       IsRetIndirect = getContext().getTypeSize(EltTy) > FRLen;
11814     } else {
11815       // This is a normal scalar > 2*GRLen, such as fp128 on LA32.
11816       IsRetIndirect = true;
11817     }
11818   }
11819 
11820   // We must track the number of GARs and FARs used in order to conform to the
11821   // LoongArch ABI. As GAR usage is different for variadic arguments, we must
11822   // also track whether we are examining a vararg or not.
11823   int GARsLeft = IsRetIndirect ? NumGARs - 1 : NumGARs;
11824   int FARsLeft = FRLen ? NumFARs : 0;
11825   int NumFixedArgs = FI.getNumRequiredArgs();
11826 
11827   int ArgNum = 0;
11828   for (auto &ArgInfo : FI.arguments()) {
11829     ArgInfo.info = classifyArgumentType(
11830         ArgInfo.type, /*IsFixed=*/ArgNum < NumFixedArgs, GARsLeft, FARsLeft);
11831     ArgNum++;
11832   }
11833 }
11834 
11835 // Returns true if the struct is a potential candidate to be passed in FARs (and
11836 // GARs). If this function returns true, the caller is responsible for checking
11837 // that if there is only a single field then that field is a float.
detectFARsEligibleStructHelper(QualType Ty,CharUnits CurOff,llvm::Type * & Field1Ty,CharUnits & Field1Off,llvm::Type * & Field2Ty,CharUnits & Field2Off) const11838 bool LoongArchABIInfo::detectFARsEligibleStructHelper(
11839     QualType Ty, CharUnits CurOff, llvm::Type *&Field1Ty, CharUnits &Field1Off,
11840     llvm::Type *&Field2Ty, CharUnits &Field2Off) const {
11841   bool IsInt = Ty->isIntegralOrEnumerationType();
11842   bool IsFloat = Ty->isRealFloatingType();
11843 
11844   if (IsInt || IsFloat) {
11845     uint64_t Size = getContext().getTypeSize(Ty);
11846     if (IsInt && Size > GRLen)
11847       return false;
11848     // Can't be eligible if larger than the FP registers. Half precision isn't
11849     // currently supported on LoongArch and the ABI hasn't been confirmed, so
11850     // default to the integer ABI in that case.
11851     if (IsFloat && (Size > FRLen || Size < 32))
11852       return false;
11853     // Can't be eligible if an integer type was already found (int+int pairs
11854     // are not eligible).
11855     if (IsInt && Field1Ty && Field1Ty->isIntegerTy())
11856       return false;
11857     if (!Field1Ty) {
11858       Field1Ty = CGT.ConvertType(Ty);
11859       Field1Off = CurOff;
11860       return true;
11861     }
11862     if (!Field2Ty) {
11863       Field2Ty = CGT.ConvertType(Ty);
11864       Field2Off = CurOff;
11865       return true;
11866     }
11867     return false;
11868   }
11869 
11870   if (auto CTy = Ty->getAs<ComplexType>()) {
11871     if (Field1Ty)
11872       return false;
11873     QualType EltTy = CTy->getElementType();
11874     if (getContext().getTypeSize(EltTy) > FRLen)
11875       return false;
11876     Field1Ty = CGT.ConvertType(EltTy);
11877     Field1Off = CurOff;
11878     Field2Ty = Field1Ty;
11879     Field2Off = Field1Off + getContext().getTypeSizeInChars(EltTy);
11880     return true;
11881   }
11882 
11883   if (const ConstantArrayType *ATy = getContext().getAsConstantArrayType(Ty)) {
11884     uint64_t ArraySize = ATy->getSize().getZExtValue();
11885     QualType EltTy = ATy->getElementType();
11886     CharUnits EltSize = getContext().getTypeSizeInChars(EltTy);
11887     for (uint64_t i = 0; i < ArraySize; ++i) {
11888       if (!detectFARsEligibleStructHelper(EltTy, CurOff, Field1Ty, Field1Off,
11889                                           Field2Ty, Field2Off))
11890         return false;
11891       CurOff += EltSize;
11892     }
11893     return true;
11894   }
11895 
11896   if (const auto *RTy = Ty->getAs<RecordType>()) {
11897     // Structures with either a non-trivial destructor or a non-trivial
11898     // copy constructor are not eligible for the FP calling convention.
11899     if (getRecordArgABI(Ty, CGT.getCXXABI()))
11900       return false;
11901     if (isEmptyRecord(getContext(), Ty, true))
11902       return true;
11903     const RecordDecl *RD = RTy->getDecl();
11904     // Unions aren't eligible unless they're empty (which is caught above).
11905     if (RD->isUnion())
11906       return false;
11907     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
11908     // If this is a C++ record, check the bases first.
11909     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
11910       for (const CXXBaseSpecifier &B : CXXRD->bases()) {
11911         const auto *BDecl =
11912             cast<CXXRecordDecl>(B.getType()->castAs<RecordType>()->getDecl());
11913         if (!detectFARsEligibleStructHelper(
11914                 B.getType(), CurOff + Layout.getBaseClassOffset(BDecl),
11915                 Field1Ty, Field1Off, Field2Ty, Field2Off))
11916           return false;
11917       }
11918     }
11919     for (const FieldDecl *FD : RD->fields()) {
11920       QualType QTy = FD->getType();
11921       if (FD->isBitField()) {
11922         unsigned BitWidth = FD->getBitWidthValue(getContext());
11923         // Zero-width bitfields are ignored.
11924         if (BitWidth == 0)
11925           continue;
11926         // Allow a bitfield with a type greater than GRLen as long as the
11927         // bitwidth is GRLen or less.
11928         if (getContext().getTypeSize(QTy) > GRLen && BitWidth <= GRLen) {
11929           QTy = getContext().getIntTypeForBitwidth(GRLen, false);
11930         }
11931       }
11932 
11933       if (!detectFARsEligibleStructHelper(
11934               QTy,
11935               CurOff + getContext().toCharUnitsFromBits(
11936                            Layout.getFieldOffset(FD->getFieldIndex())),
11937               Field1Ty, Field1Off, Field2Ty, Field2Off))
11938         return false;
11939     }
11940     return Field1Ty != nullptr;
11941   }
11942 
11943   return false;
11944 }
11945 
11946 // Determine if a struct is eligible to be passed in FARs (and GARs) (i.e., when
11947 // flattened it contains a single fp value, fp+fp, or int+fp of appropriate
11948 // size). If so, NeededFARs and NeededGARs are incremented appropriately.
detectFARsEligibleStruct(QualType Ty,llvm::Type * & Field1Ty,CharUnits & Field1Off,llvm::Type * & Field2Ty,CharUnits & Field2Off,int & NeededGARs,int & NeededFARs) const11949 bool LoongArchABIInfo::detectFARsEligibleStruct(
11950     QualType Ty, llvm::Type *&Field1Ty, CharUnits &Field1Off,
11951     llvm::Type *&Field2Ty, CharUnits &Field2Off, int &NeededGARs,
11952     int &NeededFARs) const {
11953   Field1Ty = nullptr;
11954   Field2Ty = nullptr;
11955   NeededGARs = 0;
11956   NeededFARs = 0;
11957   if (!detectFARsEligibleStructHelper(Ty, CharUnits::Zero(), Field1Ty,
11958                                       Field1Off, Field2Ty, Field2Off))
11959     return false;
11960   // Not really a candidate if we have a single int but no float.
11961   if (Field1Ty && !Field2Ty && !Field1Ty->isFloatingPointTy())
11962     return false;
11963   if (Field1Ty && Field1Ty->isFloatingPointTy())
11964     NeededFARs++;
11965   else if (Field1Ty)
11966     NeededGARs++;
11967   if (Field2Ty && Field2Ty->isFloatingPointTy())
11968     NeededFARs++;
11969   else if (Field2Ty)
11970     NeededGARs++;
11971   return true;
11972 }
11973 
11974 // Call getCoerceAndExpand for the two-element flattened struct described by
11975 // Field1Ty, Field1Off, Field2Ty, Field2Off. This method will create an
11976 // appropriate coerceToType and unpaddedCoerceToType.
coerceAndExpandFARsEligibleStruct(llvm::Type * Field1Ty,CharUnits Field1Off,llvm::Type * Field2Ty,CharUnits Field2Off) const11977 ABIArgInfo LoongArchABIInfo::coerceAndExpandFARsEligibleStruct(
11978     llvm::Type *Field1Ty, CharUnits Field1Off, llvm::Type *Field2Ty,
11979     CharUnits Field2Off) const {
11980   SmallVector<llvm::Type *, 3> CoerceElts;
11981   SmallVector<llvm::Type *, 2> UnpaddedCoerceElts;
11982   if (!Field1Off.isZero())
11983     CoerceElts.push_back(llvm::ArrayType::get(
11984         llvm::Type::getInt8Ty(getVMContext()), Field1Off.getQuantity()));
11985 
11986   CoerceElts.push_back(Field1Ty);
11987   UnpaddedCoerceElts.push_back(Field1Ty);
11988 
11989   if (!Field2Ty) {
11990     return ABIArgInfo::getCoerceAndExpand(
11991         llvm::StructType::get(getVMContext(), CoerceElts, !Field1Off.isZero()),
11992         UnpaddedCoerceElts[0]);
11993   }
11994 
11995   CharUnits Field2Align =
11996       CharUnits::fromQuantity(getDataLayout().getABITypeAlign(Field2Ty));
11997   CharUnits Field1End =
11998       Field1Off +
11999       CharUnits::fromQuantity(getDataLayout().getTypeStoreSize(Field1Ty));
12000   CharUnits Field2OffNoPadNoPack = Field1End.alignTo(Field2Align);
12001 
12002   CharUnits Padding = CharUnits::Zero();
12003   if (Field2Off > Field2OffNoPadNoPack)
12004     Padding = Field2Off - Field2OffNoPadNoPack;
12005   else if (Field2Off != Field2Align && Field2Off > Field1End)
12006     Padding = Field2Off - Field1End;
12007 
12008   bool IsPacked = !Field2Off.isMultipleOf(Field2Align);
12009 
12010   if (!Padding.isZero())
12011     CoerceElts.push_back(llvm::ArrayType::get(
12012         llvm::Type::getInt8Ty(getVMContext()), Padding.getQuantity()));
12013 
12014   CoerceElts.push_back(Field2Ty);
12015   UnpaddedCoerceElts.push_back(Field2Ty);
12016 
12017   return ABIArgInfo::getCoerceAndExpand(
12018       llvm::StructType::get(getVMContext(), CoerceElts, IsPacked),
12019       llvm::StructType::get(getVMContext(), UnpaddedCoerceElts, IsPacked));
12020 }
12021 
classifyArgumentType(QualType Ty,bool IsFixed,int & GARsLeft,int & FARsLeft) const12022 ABIArgInfo LoongArchABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
12023                                                   int &GARsLeft,
12024                                                   int &FARsLeft) const {
12025   assert(GARsLeft <= NumGARs && "GAR tracking underflow");
12026   Ty = useFirstFieldIfTransparentUnion(Ty);
12027 
12028   // Structures with either a non-trivial destructor or a non-trivial
12029   // copy constructor are always passed indirectly.
12030   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
12031     if (GARsLeft)
12032       GARsLeft -= 1;
12033     return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
12034                                            CGCXXABI::RAA_DirectInMemory);
12035   }
12036 
12037   // Ignore empty structs/unions.
12038   if (isEmptyRecord(getContext(), Ty, true))
12039     return ABIArgInfo::getIgnore();
12040 
12041   uint64_t Size = getContext().getTypeSize(Ty);
12042 
12043   // Pass floating point values via FARs if possible.
12044   if (IsFixed && Ty->isFloatingType() && !Ty->isComplexType() &&
12045       FRLen >= Size && FARsLeft) {
12046     FARsLeft--;
12047     return ABIArgInfo::getDirect();
12048   }
12049 
12050   // Complex types for the *f or *d ABI must be passed directly rather than
12051   // using CoerceAndExpand.
12052   if (IsFixed && Ty->isComplexType() && FRLen && FARsLeft >= 2) {
12053     QualType EltTy = Ty->castAs<ComplexType>()->getElementType();
12054     if (getContext().getTypeSize(EltTy) <= FRLen) {
12055       FARsLeft -= 2;
12056       return ABIArgInfo::getDirect();
12057     }
12058   }
12059 
12060   if (IsFixed && FRLen && Ty->isStructureOrClassType()) {
12061     llvm::Type *Field1Ty = nullptr;
12062     llvm::Type *Field2Ty = nullptr;
12063     CharUnits Field1Off = CharUnits::Zero();
12064     CharUnits Field2Off = CharUnits::Zero();
12065     int NeededGARs = 0;
12066     int NeededFARs = 0;
12067     bool IsCandidate = detectFARsEligibleStruct(
12068         Ty, Field1Ty, Field1Off, Field2Ty, Field2Off, NeededGARs, NeededFARs);
12069     if (IsCandidate && NeededGARs <= GARsLeft && NeededFARs <= FARsLeft) {
12070       GARsLeft -= NeededGARs;
12071       FARsLeft -= NeededFARs;
12072       return coerceAndExpandFARsEligibleStruct(Field1Ty, Field1Off, Field2Ty,
12073                                                Field2Off);
12074     }
12075   }
12076 
12077   uint64_t NeededAlign = getContext().getTypeAlign(Ty);
12078   // Determine the number of GARs needed to pass the current argument
12079   // according to the ABI. 2*GRLen-aligned varargs are passed in "aligned"
12080   // register pairs, so may consume 3 registers.
12081   int NeededGARs = 1;
12082   if (!IsFixed && NeededAlign == 2 * GRLen)
12083     NeededGARs = 2 + (GARsLeft % 2);
12084   else if (Size > GRLen && Size <= 2 * GRLen)
12085     NeededGARs = 2;
12086 
12087   if (NeededGARs > GARsLeft)
12088     NeededGARs = GARsLeft;
12089 
12090   GARsLeft -= NeededGARs;
12091 
12092   if (!isAggregateTypeForABI(Ty) && !Ty->isVectorType()) {
12093     // Treat an enum type as its underlying type.
12094     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
12095       Ty = EnumTy->getDecl()->getIntegerType();
12096 
12097     // All integral types are promoted to GRLen width.
12098     if (Size < GRLen && Ty->isIntegralOrEnumerationType())
12099       return extendType(Ty);
12100 
12101     if (const auto *EIT = Ty->getAs<BitIntType>()) {
12102       if (EIT->getNumBits() < GRLen)
12103         return extendType(Ty);
12104       if (EIT->getNumBits() > 128 ||
12105           (!getContext().getTargetInfo().hasInt128Type() &&
12106            EIT->getNumBits() > 64))
12107         return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
12108     }
12109 
12110     return ABIArgInfo::getDirect();
12111   }
12112 
12113   // Aggregates which are <= 2*GRLen will be passed in registers if possible,
12114   // so coerce to integers.
12115   if (Size <= 2 * GRLen) {
12116     // Use a single GRLen int if possible, 2*GRLen if 2*GRLen alignment is
12117     // required, and a 2-element GRLen array if only GRLen alignment is
12118     // required.
12119     if (Size <= GRLen) {
12120       return ABIArgInfo::getDirect(
12121           llvm::IntegerType::get(getVMContext(), GRLen));
12122     }
12123     if (getContext().getTypeAlign(Ty) == 2 * GRLen) {
12124       return ABIArgInfo::getDirect(
12125           llvm::IntegerType::get(getVMContext(), 2 * GRLen));
12126     }
12127     return ABIArgInfo::getDirect(
12128         llvm::ArrayType::get(llvm::IntegerType::get(getVMContext(), GRLen), 2));
12129   }
12130   return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
12131 }
12132 
classifyReturnType(QualType RetTy) const12133 ABIArgInfo LoongArchABIInfo::classifyReturnType(QualType RetTy) const {
12134   if (RetTy->isVoidType())
12135     return ABIArgInfo::getIgnore();
12136   // The rules for return and argument types are the same, so defer to
12137   // classifyArgumentType.
12138   int GARsLeft = 2;
12139   int FARsLeft = FRLen ? 2 : 0;
12140   return classifyArgumentType(RetTy, /*IsFixed=*/true, GARsLeft, FARsLeft);
12141 }
12142 
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const12143 Address LoongArchABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
12144                                     QualType Ty) const {
12145   CharUnits SlotSize = CharUnits::fromQuantity(GRLen / 8);
12146 
12147   // Empty records are ignored for parameter passing purposes.
12148   if (isEmptyRecord(getContext(), Ty, true)) {
12149     Address Addr = Address(CGF.Builder.CreateLoad(VAListAddr),
12150                            getVAListElementType(CGF), SlotSize);
12151     Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
12152     return Addr;
12153   }
12154 
12155   auto TInfo = getContext().getTypeInfoInChars(Ty);
12156 
12157   // Arguments bigger than 2*GRLen bytes are passed indirectly.
12158   return emitVoidPtrVAArg(CGF, VAListAddr, Ty,
12159                           /*IsIndirect=*/TInfo.Width > 2 * SlotSize, TInfo,
12160                           SlotSize,
12161                           /*AllowHigherAlign=*/true);
12162 }
12163 
extendType(QualType Ty) const12164 ABIArgInfo LoongArchABIInfo::extendType(QualType Ty) const {
12165   int TySize = getContext().getTypeSize(Ty);
12166   // LA64 ABI requires unsigned 32 bit integers to be sign extended.
12167   if (GRLen == 64 && Ty->isUnsignedIntegerOrEnumerationType() && TySize == 32)
12168     return ABIArgInfo::getSignExtend(Ty);
12169   return ABIArgInfo::getExtend(Ty);
12170 }
12171 
12172 namespace {
12173 class LoongArchTargetCodeGenInfo : public TargetCodeGenInfo {
12174 public:
LoongArchTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,unsigned GRLen,unsigned FRLen)12175   LoongArchTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned GRLen,
12176                              unsigned FRLen)
12177       : TargetCodeGenInfo(
12178             std::make_unique<LoongArchABIInfo>(CGT, GRLen, FRLen)) {}
12179 };
12180 } // namespace
12181 
12182 //===----------------------------------------------------------------------===//
12183 // Driver code
12184 //===----------------------------------------------------------------------===//
12185 
supportsCOMDAT() const12186 bool CodeGenModule::supportsCOMDAT() const {
12187   return getTriple().supportsCOMDAT();
12188 }
12189 
getTargetCodeGenInfo()12190 const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
12191   if (TheTargetCodeGenInfo)
12192     return *TheTargetCodeGenInfo;
12193 
12194   // Helper to set the unique_ptr while still keeping the return value.
12195   auto SetCGInfo = [&](TargetCodeGenInfo *P) -> const TargetCodeGenInfo & {
12196     this->TheTargetCodeGenInfo.reset(P);
12197     return *P;
12198   };
12199 
12200   const llvm::Triple &Triple = getTarget().getTriple();
12201   switch (Triple.getArch()) {
12202   default:
12203     return SetCGInfo(new DefaultTargetCodeGenInfo(Types));
12204 
12205   case llvm::Triple::le32:
12206     return SetCGInfo(new PNaClTargetCodeGenInfo(Types));
12207   case llvm::Triple::m68k:
12208     return SetCGInfo(new M68kTargetCodeGenInfo(Types));
12209   case llvm::Triple::mips:
12210   case llvm::Triple::mipsel:
12211     if (Triple.getOS() == llvm::Triple::NaCl)
12212       return SetCGInfo(new PNaClTargetCodeGenInfo(Types));
12213     return SetCGInfo(new MIPSTargetCodeGenInfo(Types, true));
12214 
12215   case llvm::Triple::mips64:
12216   case llvm::Triple::mips64el:
12217     return SetCGInfo(new MIPSTargetCodeGenInfo(Types, false));
12218 
12219   case llvm::Triple::avr: {
12220     // For passing parameters, R8~R25 are used on avr, and R18~R25 are used
12221     // on avrtiny. For passing return value, R18~R25 are used on avr, and
12222     // R22~R25 are used on avrtiny.
12223     unsigned NPR = getTarget().getABI() == "avrtiny" ? 6 : 18;
12224     unsigned NRR = getTarget().getABI() == "avrtiny" ? 4 : 8;
12225     return SetCGInfo(new AVRTargetCodeGenInfo(Types, NPR, NRR));
12226   }
12227 
12228   case llvm::Triple::aarch64:
12229   case llvm::Triple::aarch64_32:
12230   case llvm::Triple::aarch64_be: {
12231     AArch64ABIInfo::ABIKind Kind = AArch64ABIInfo::AAPCS;
12232     if (getTarget().getABI() == "darwinpcs")
12233       Kind = AArch64ABIInfo::DarwinPCS;
12234     else if (Triple.isOSWindows())
12235       return SetCGInfo(
12236           new WindowsAArch64TargetCodeGenInfo(Types, AArch64ABIInfo::Win64));
12237 
12238     return SetCGInfo(new AArch64TargetCodeGenInfo(Types, Kind));
12239   }
12240 
12241   case llvm::Triple::wasm32:
12242   case llvm::Triple::wasm64: {
12243     WebAssemblyABIInfo::ABIKind Kind = WebAssemblyABIInfo::MVP;
12244     if (getTarget().getABI() == "experimental-mv")
12245       Kind = WebAssemblyABIInfo::ExperimentalMV;
12246     return SetCGInfo(new WebAssemblyTargetCodeGenInfo(Types, Kind));
12247   }
12248 
12249   case llvm::Triple::arm:
12250   case llvm::Triple::armeb:
12251   case llvm::Triple::thumb:
12252   case llvm::Triple::thumbeb: {
12253     if (Triple.getOS() == llvm::Triple::Win32) {
12254       return SetCGInfo(
12255           new WindowsARMTargetCodeGenInfo(Types, ARMABIInfo::AAPCS_VFP));
12256     }
12257 
12258     ARMABIInfo::ABIKind Kind = ARMABIInfo::AAPCS;
12259     StringRef ABIStr = getTarget().getABI();
12260     if (ABIStr == "apcs-gnu")
12261       Kind = ARMABIInfo::APCS;
12262     else if (ABIStr == "aapcs16")
12263       Kind = ARMABIInfo::AAPCS16_VFP;
12264     else if (CodeGenOpts.FloatABI == "hard" ||
12265              (CodeGenOpts.FloatABI != "soft" &&
12266               (Triple.getEnvironment() == llvm::Triple::GNUEABIHF ||
12267                Triple.getEnvironment() == llvm::Triple::MuslEABIHF ||
12268                Triple.getEnvironment() == llvm::Triple::EABIHF)))
12269       Kind = ARMABIInfo::AAPCS_VFP;
12270 
12271     return SetCGInfo(new ARMTargetCodeGenInfo(Types, Kind));
12272   }
12273 
12274   case llvm::Triple::ppc: {
12275     if (Triple.isOSAIX())
12276       return SetCGInfo(new AIXTargetCodeGenInfo(Types, /*Is64Bit*/ false));
12277 
12278     bool IsSoftFloat =
12279         CodeGenOpts.FloatABI == "soft" || getTarget().hasFeature("spe");
12280     bool RetSmallStructInRegABI =
12281         PPC32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts);
12282     return SetCGInfo(
12283         new PPC32TargetCodeGenInfo(Types, IsSoftFloat, RetSmallStructInRegABI));
12284   }
12285   case llvm::Triple::ppcle: {
12286     bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
12287     bool RetSmallStructInRegABI =
12288         PPC32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts);
12289     return SetCGInfo(
12290         new PPC32TargetCodeGenInfo(Types, IsSoftFloat, RetSmallStructInRegABI));
12291   }
12292   case llvm::Triple::ppc64:
12293     if (Triple.isOSAIX())
12294       return SetCGInfo(new AIXTargetCodeGenInfo(Types, /*Is64Bit*/ true));
12295 
12296     if (Triple.isOSBinFormatELF()) {
12297       PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv1;
12298       if (getTarget().getABI() == "elfv2")
12299         Kind = PPC64_SVR4_ABIInfo::ELFv2;
12300       bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
12301 
12302       return SetCGInfo(
12303           new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, IsSoftFloat));
12304     }
12305     return SetCGInfo(new PPC64TargetCodeGenInfo(Types));
12306   case llvm::Triple::ppc64le: {
12307     assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!");
12308     PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv2;
12309     if (getTarget().getABI() == "elfv1")
12310       Kind = PPC64_SVR4_ABIInfo::ELFv1;
12311     bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
12312 
12313     return SetCGInfo(
12314         new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, IsSoftFloat));
12315   }
12316 
12317   case llvm::Triple::nvptx:
12318   case llvm::Triple::nvptx64:
12319     return SetCGInfo(new NVPTXTargetCodeGenInfo(Types));
12320 
12321   case llvm::Triple::msp430:
12322     return SetCGInfo(new MSP430TargetCodeGenInfo(Types));
12323 
12324   case llvm::Triple::riscv32:
12325   case llvm::Triple::riscv64: {
12326     StringRef ABIStr = getTarget().getABI();
12327     unsigned XLen = getTarget().getPointerWidth(LangAS::Default);
12328     unsigned ABIFLen = 0;
12329     if (ABIStr.endswith("f"))
12330       ABIFLen = 32;
12331     else if (ABIStr.endswith("d"))
12332       ABIFLen = 64;
12333     return SetCGInfo(new RISCVTargetCodeGenInfo(Types, XLen, ABIFLen));
12334   }
12335 
12336   case llvm::Triple::systemz: {
12337     bool SoftFloat = CodeGenOpts.FloatABI == "soft";
12338     bool HasVector = !SoftFloat && getTarget().getABI() == "vector";
12339     return SetCGInfo(new SystemZTargetCodeGenInfo(Types, HasVector, SoftFloat));
12340   }
12341 
12342   case llvm::Triple::tce:
12343   case llvm::Triple::tcele:
12344     return SetCGInfo(new TCETargetCodeGenInfo(Types));
12345 
12346   case llvm::Triple::x86: {
12347     bool IsDarwinVectorABI = Triple.isOSDarwin();
12348     bool RetSmallStructInRegABI =
12349         X86_32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts);
12350     bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing();
12351 
12352     if (Triple.getOS() == llvm::Triple::Win32) {
12353       return SetCGInfo(new WinX86_32TargetCodeGenInfo(
12354           Types, IsDarwinVectorABI, RetSmallStructInRegABI,
12355           IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters));
12356     } else {
12357       return SetCGInfo(new X86_32TargetCodeGenInfo(
12358           Types, IsDarwinVectorABI, RetSmallStructInRegABI,
12359           IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters,
12360           CodeGenOpts.FloatABI == "soft"));
12361     }
12362   }
12363 
12364   case llvm::Triple::x86_64: {
12365     StringRef ABI = getTarget().getABI();
12366     X86AVXABILevel AVXLevel =
12367         (ABI == "avx512"
12368              ? X86AVXABILevel::AVX512
12369              : ABI == "avx" ? X86AVXABILevel::AVX : X86AVXABILevel::None);
12370 
12371     switch (Triple.getOS()) {
12372     case llvm::Triple::Win32:
12373       return SetCGInfo(new WinX86_64TargetCodeGenInfo(Types, AVXLevel));
12374     default:
12375       return SetCGInfo(new X86_64TargetCodeGenInfo(Types, AVXLevel));
12376     }
12377   }
12378   case llvm::Triple::hexagon:
12379     return SetCGInfo(new HexagonTargetCodeGenInfo(Types));
12380   case llvm::Triple::lanai:
12381     return SetCGInfo(new LanaiTargetCodeGenInfo(Types));
12382   case llvm::Triple::r600:
12383     return SetCGInfo(new AMDGPUTargetCodeGenInfo(Types));
12384   case llvm::Triple::amdgcn:
12385     return SetCGInfo(new AMDGPUTargetCodeGenInfo(Types));
12386   case llvm::Triple::sparc:
12387     return SetCGInfo(new SparcV8TargetCodeGenInfo(Types));
12388   case llvm::Triple::sparcv9:
12389     return SetCGInfo(new SparcV9TargetCodeGenInfo(Types));
12390   case llvm::Triple::xcore:
12391     return SetCGInfo(new XCoreTargetCodeGenInfo(Types));
12392   case llvm::Triple::arc:
12393     return SetCGInfo(new ARCTargetCodeGenInfo(Types));
12394   case llvm::Triple::spir:
12395   case llvm::Triple::spir64:
12396     return SetCGInfo(new CommonSPIRTargetCodeGenInfo(Types));
12397   case llvm::Triple::spirv32:
12398   case llvm::Triple::spirv64:
12399     return SetCGInfo(new SPIRVTargetCodeGenInfo(Types));
12400   case llvm::Triple::ve:
12401     return SetCGInfo(new VETargetCodeGenInfo(Types));
12402   case llvm::Triple::csky: {
12403     bool IsSoftFloat = !getTarget().hasFeature("hard-float-abi");
12404     bool hasFP64 = getTarget().hasFeature("fpuv2_df") ||
12405                    getTarget().hasFeature("fpuv3_df");
12406     return SetCGInfo(new CSKYTargetCodeGenInfo(Types, IsSoftFloat ? 0
12407                                                       : hasFP64   ? 64
12408                                                                   : 32));
12409   }
12410   case llvm::Triple::bpfeb:
12411   case llvm::Triple::bpfel:
12412     return SetCGInfo(new BPFTargetCodeGenInfo(Types));
12413   case llvm::Triple::loongarch32:
12414   case llvm::Triple::loongarch64: {
12415     StringRef ABIStr = getTarget().getABI();
12416     unsigned ABIFRLen = 0;
12417     if (ABIStr.endswith("f"))
12418       ABIFRLen = 32;
12419     else if (ABIStr.endswith("d"))
12420       ABIFRLen = 64;
12421     return SetCGInfo(new LoongArchTargetCodeGenInfo(
12422         Types, getTarget().getPointerWidth(LangAS::Default), ABIFRLen));
12423   }
12424   }
12425 }
12426 
12427 /// Create an OpenCL kernel for an enqueued block.
12428 ///
12429 /// The kernel has the same function type as the block invoke function. Its
12430 /// name is the name of the block invoke function postfixed with "_kernel".
12431 /// It simply calls the block invoke function then returns.
12432 llvm::Function *
createEnqueuedBlockKernel(CodeGenFunction & CGF,llvm::Function * Invoke,llvm::Type * BlockTy) const12433 TargetCodeGenInfo::createEnqueuedBlockKernel(CodeGenFunction &CGF,
12434                                              llvm::Function *Invoke,
12435                                              llvm::Type *BlockTy) const {
12436   auto *InvokeFT = Invoke->getFunctionType();
12437   auto &C = CGF.getLLVMContext();
12438   std::string Name = Invoke->getName().str() + "_kernel";
12439   auto *FT = llvm::FunctionType::get(llvm::Type::getVoidTy(C),
12440                                      InvokeFT->params(), false);
12441   auto *F = llvm::Function::Create(FT, llvm::GlobalValue::ExternalLinkage, Name,
12442                                    &CGF.CGM.getModule());
12443   auto IP = CGF.Builder.saveIP();
12444   auto *BB = llvm::BasicBlock::Create(C, "entry", F);
12445   auto &Builder = CGF.Builder;
12446   Builder.SetInsertPoint(BB);
12447   llvm::SmallVector<llvm::Value *, 2> Args(llvm::make_pointer_range(F->args()));
12448   llvm::CallInst *call = Builder.CreateCall(Invoke, Args);
12449   call->setCallingConv(Invoke->getCallingConv());
12450   Builder.CreateRetVoid();
12451   Builder.restoreIP(IP);
12452   return F;
12453 }
12454 
12455 /// Create an OpenCL kernel for an enqueued block.
12456 ///
12457 /// The type of the first argument (the block literal) is the struct type
12458 /// of the block literal instead of a pointer type. The first argument
12459 /// (block literal) is passed directly by value to the kernel. The kernel
12460 /// allocates the same type of struct on stack and stores the block literal
12461 /// to it and passes its pointer to the block invoke function. The kernel
12462 /// has "enqueued-block" function attribute and kernel argument metadata.
createEnqueuedBlockKernel(CodeGenFunction & CGF,llvm::Function * Invoke,llvm::Type * BlockTy) const12463 llvm::Function *AMDGPUTargetCodeGenInfo::createEnqueuedBlockKernel(
12464     CodeGenFunction &CGF, llvm::Function *Invoke,
12465     llvm::Type *BlockTy) const {
12466   auto &Builder = CGF.Builder;
12467   auto &C = CGF.getLLVMContext();
12468 
12469   auto *InvokeFT = Invoke->getFunctionType();
12470   llvm::SmallVector<llvm::Type *, 2> ArgTys;
12471   llvm::SmallVector<llvm::Metadata *, 8> AddressQuals;
12472   llvm::SmallVector<llvm::Metadata *, 8> AccessQuals;
12473   llvm::SmallVector<llvm::Metadata *, 8> ArgTypeNames;
12474   llvm::SmallVector<llvm::Metadata *, 8> ArgBaseTypeNames;
12475   llvm::SmallVector<llvm::Metadata *, 8> ArgTypeQuals;
12476   llvm::SmallVector<llvm::Metadata *, 8> ArgNames;
12477 
12478   ArgTys.push_back(BlockTy);
12479   ArgTypeNames.push_back(llvm::MDString::get(C, "__block_literal"));
12480   AddressQuals.push_back(llvm::ConstantAsMetadata::get(Builder.getInt32(0)));
12481   ArgBaseTypeNames.push_back(llvm::MDString::get(C, "__block_literal"));
12482   ArgTypeQuals.push_back(llvm::MDString::get(C, ""));
12483   AccessQuals.push_back(llvm::MDString::get(C, "none"));
12484   ArgNames.push_back(llvm::MDString::get(C, "block_literal"));
12485   for (unsigned I = 1, E = InvokeFT->getNumParams(); I < E; ++I) {
12486     ArgTys.push_back(InvokeFT->getParamType(I));
12487     ArgTypeNames.push_back(llvm::MDString::get(C, "void*"));
12488     AddressQuals.push_back(llvm::ConstantAsMetadata::get(Builder.getInt32(3)));
12489     AccessQuals.push_back(llvm::MDString::get(C, "none"));
12490     ArgBaseTypeNames.push_back(llvm::MDString::get(C, "void*"));
12491     ArgTypeQuals.push_back(llvm::MDString::get(C, ""));
12492     ArgNames.push_back(
12493         llvm::MDString::get(C, (Twine("local_arg") + Twine(I)).str()));
12494   }
12495   std::string Name = Invoke->getName().str() + "_kernel";
12496   auto *FT = llvm::FunctionType::get(llvm::Type::getVoidTy(C), ArgTys, false);
12497   auto *F = llvm::Function::Create(FT, llvm::GlobalValue::InternalLinkage, Name,
12498                                    &CGF.CGM.getModule());
12499   F->addFnAttr("enqueued-block");
12500   auto IP = CGF.Builder.saveIP();
12501   auto *BB = llvm::BasicBlock::Create(C, "entry", F);
12502   Builder.SetInsertPoint(BB);
12503   const auto BlockAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(BlockTy);
12504   auto *BlockPtr = Builder.CreateAlloca(BlockTy, nullptr);
12505   BlockPtr->setAlignment(BlockAlign);
12506   Builder.CreateAlignedStore(F->arg_begin(), BlockPtr, BlockAlign);
12507   auto *Cast = Builder.CreatePointerCast(BlockPtr, InvokeFT->getParamType(0));
12508   llvm::SmallVector<llvm::Value *, 2> Args;
12509   Args.push_back(Cast);
12510   for (llvm::Argument &A : llvm::drop_begin(F->args()))
12511     Args.push_back(&A);
12512   llvm::CallInst *call = Builder.CreateCall(Invoke, Args);
12513   call->setCallingConv(Invoke->getCallingConv());
12514   Builder.CreateRetVoid();
12515   Builder.restoreIP(IP);
12516 
12517   F->setMetadata("kernel_arg_addr_space", llvm::MDNode::get(C, AddressQuals));
12518   F->setMetadata("kernel_arg_access_qual", llvm::MDNode::get(C, AccessQuals));
12519   F->setMetadata("kernel_arg_type", llvm::MDNode::get(C, ArgTypeNames));
12520   F->setMetadata("kernel_arg_base_type",
12521                  llvm::MDNode::get(C, ArgBaseTypeNames));
12522   F->setMetadata("kernel_arg_type_qual", llvm::MDNode::get(C, ArgTypeQuals));
12523   if (CGF.CGM.getCodeGenOpts().EmitOpenCLArgMetadata)
12524     F->setMetadata("kernel_arg_name", llvm::MDNode::get(C, ArgNames));
12525 
12526   return F;
12527 }
12528