1 //===---- TargetInfo.cpp - Encapsulate target details -----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // These classes wrap the information about a call or function
10 // definition used to handle ABI compliancy.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "TargetInfo.h"
15 #include "ABIInfo.h"
16 #include "CGBlocks.h"
17 #include "CGCXXABI.h"
18 #include "CGValue.h"
19 #include "CodeGenFunction.h"
20 #include "clang/AST/Attr.h"
21 #include "clang/AST/RecordLayout.h"
22 #include "clang/Basic/CodeGenOptions.h"
23 #include "clang/Basic/DiagnosticFrontend.h"
24 #include "clang/Basic/Builtins.h"
25 #include "clang/CodeGen/CGFunctionInfo.h"
26 #include "clang/CodeGen/SwiftCallingConv.h"
27 #include "llvm/ADT/SmallBitVector.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringSwitch.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/IR/DataLayout.h"
33 #include "llvm/IR/IntrinsicsNVPTX.h"
34 #include "llvm/IR/IntrinsicsS390.h"
35 #include "llvm/IR/Type.h"
36 #include "llvm/Support/raw_ostream.h"
37 #include <algorithm> // std::sort
38
39 using namespace clang;
40 using namespace CodeGen;
41
42 // Helper for coercing an aggregate argument or return value into an integer
43 // array of the same size (including padding) and alignment. This alternate
44 // coercion happens only for the RenderScript ABI and can be removed after
45 // runtimes that rely on it are no longer supported.
46 //
47 // RenderScript assumes that the size of the argument / return value in the IR
48 // is the same as the size of the corresponding qualified type. This helper
49 // coerces the aggregate type into an array of the same size (including
50 // padding). This coercion is used in lieu of expansion of struct members or
51 // other canonical coercions that return a coerced-type of larger size.
52 //
53 // Ty - The argument / return value type
54 // Context - The associated ASTContext
55 // LLVMContext - The associated LLVMContext
coerceToIntArray(QualType Ty,ASTContext & Context,llvm::LLVMContext & LLVMContext)56 static ABIArgInfo coerceToIntArray(QualType Ty,
57 ASTContext &Context,
58 llvm::LLVMContext &LLVMContext) {
59 // Alignment and Size are measured in bits.
60 const uint64_t Size = Context.getTypeSize(Ty);
61 const uint64_t Alignment = Context.getTypeAlign(Ty);
62 llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
63 const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
64 return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
65 }
66
AssignToArrayRange(CodeGen::CGBuilderTy & Builder,llvm::Value * Array,llvm::Value * Value,unsigned FirstIndex,unsigned LastIndex)67 static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder,
68 llvm::Value *Array,
69 llvm::Value *Value,
70 unsigned FirstIndex,
71 unsigned LastIndex) {
72 // Alternatively, we could emit this as a loop in the source.
73 for (unsigned I = FirstIndex; I <= LastIndex; ++I) {
74 llvm::Value *Cell =
75 Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
76 Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
77 }
78 }
79
isAggregateTypeForABI(QualType T)80 static bool isAggregateTypeForABI(QualType T) {
81 return !CodeGenFunction::hasScalarEvaluationKind(T) ||
82 T->isMemberFunctionPointerType();
83 }
84
getNaturalAlignIndirect(QualType Ty,bool ByVal,bool Realign,llvm::Type * Padding) const85 ABIArgInfo ABIInfo::getNaturalAlignIndirect(QualType Ty, bool ByVal,
86 bool Realign,
87 llvm::Type *Padding) const {
88 return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), ByVal,
89 Realign, Padding);
90 }
91
92 ABIArgInfo
getNaturalAlignIndirectInReg(QualType Ty,bool Realign) const93 ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, bool Realign) const {
94 return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
95 /*ByVal*/ false, Realign);
96 }
97
EmitMSVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const98 Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
99 QualType Ty) const {
100 return Address::invalid();
101 }
102
isPromotableIntegerTypeForABI(QualType Ty) const103 bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
104 if (Ty->isPromotableIntegerType())
105 return true;
106
107 if (const auto *EIT = Ty->getAs<ExtIntType>())
108 if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
109 return true;
110
111 return false;
112 }
113
~ABIInfo()114 ABIInfo::~ABIInfo() {}
115
116 /// Does the given lowering require more than the given number of
117 /// registers when expanded?
118 ///
119 /// This is intended to be the basis of a reasonable basic implementation
120 /// of should{Pass,Return}IndirectlyForSwift.
121 ///
122 /// For most targets, a limit of four total registers is reasonable; this
123 /// limits the amount of code required in order to move around the value
124 /// in case it wasn't produced immediately prior to the call by the caller
125 /// (or wasn't produced in exactly the right registers) or isn't used
126 /// immediately within the callee. But some targets may need to further
127 /// limit the register count due to an inability to support that many
128 /// return registers.
occupiesMoreThan(CodeGenTypes & cgt,ArrayRef<llvm::Type * > scalarTypes,unsigned maxAllRegisters)129 static bool occupiesMoreThan(CodeGenTypes &cgt,
130 ArrayRef<llvm::Type*> scalarTypes,
131 unsigned maxAllRegisters) {
132 unsigned intCount = 0, fpCount = 0;
133 for (llvm::Type *type : scalarTypes) {
134 if (type->isPointerTy()) {
135 intCount++;
136 } else if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
137 auto ptrWidth = cgt.getTarget().getPointerWidth(0);
138 intCount += (intTy->getBitWidth() + ptrWidth - 1) / ptrWidth;
139 } else {
140 assert(type->isVectorTy() || type->isFloatingPointTy());
141 fpCount++;
142 }
143 }
144
145 return (intCount + fpCount > maxAllRegisters);
146 }
147
isLegalVectorTypeForSwift(CharUnits vectorSize,llvm::Type * eltTy,unsigned numElts) const148 bool SwiftABIInfo::isLegalVectorTypeForSwift(CharUnits vectorSize,
149 llvm::Type *eltTy,
150 unsigned numElts) const {
151 // The default implementation of this assumes that the target guarantees
152 // 128-bit SIMD support but nothing more.
153 return (vectorSize.getQuantity() > 8 && vectorSize.getQuantity() <= 16);
154 }
155
getRecordArgABI(const RecordType * RT,CGCXXABI & CXXABI)156 static CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT,
157 CGCXXABI &CXXABI) {
158 const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
159 if (!RD) {
160 if (!RT->getDecl()->canPassInRegisters())
161 return CGCXXABI::RAA_Indirect;
162 return CGCXXABI::RAA_Default;
163 }
164 return CXXABI.getRecordArgABI(RD);
165 }
166
getRecordArgABI(QualType T,CGCXXABI & CXXABI)167 static CGCXXABI::RecordArgABI getRecordArgABI(QualType T,
168 CGCXXABI &CXXABI) {
169 const RecordType *RT = T->getAs<RecordType>();
170 if (!RT)
171 return CGCXXABI::RAA_Default;
172 return getRecordArgABI(RT, CXXABI);
173 }
174
classifyReturnType(const CGCXXABI & CXXABI,CGFunctionInfo & FI,const ABIInfo & Info)175 static bool classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI,
176 const ABIInfo &Info) {
177 QualType Ty = FI.getReturnType();
178
179 if (const auto *RT = Ty->getAs<RecordType>())
180 if (!isa<CXXRecordDecl>(RT->getDecl()) &&
181 !RT->getDecl()->canPassInRegisters()) {
182 FI.getReturnInfo() = Info.getNaturalAlignIndirect(Ty);
183 return true;
184 }
185
186 return CXXABI.classifyReturnType(FI);
187 }
188
189 /// Pass transparent unions as if they were the type of the first element. Sema
190 /// should ensure that all elements of the union have the same "machine type".
useFirstFieldIfTransparentUnion(QualType Ty)191 static QualType useFirstFieldIfTransparentUnion(QualType Ty) {
192 if (const RecordType *UT = Ty->getAsUnionType()) {
193 const RecordDecl *UD = UT->getDecl();
194 if (UD->hasAttr<TransparentUnionAttr>()) {
195 assert(!UD->field_empty() && "sema created an empty transparent union");
196 return UD->field_begin()->getType();
197 }
198 }
199 return Ty;
200 }
201
getCXXABI() const202 CGCXXABI &ABIInfo::getCXXABI() const {
203 return CGT.getCXXABI();
204 }
205
getContext() const206 ASTContext &ABIInfo::getContext() const {
207 return CGT.getContext();
208 }
209
getVMContext() const210 llvm::LLVMContext &ABIInfo::getVMContext() const {
211 return CGT.getLLVMContext();
212 }
213
getDataLayout() const214 const llvm::DataLayout &ABIInfo::getDataLayout() const {
215 return CGT.getDataLayout();
216 }
217
getTarget() const218 const TargetInfo &ABIInfo::getTarget() const {
219 return CGT.getTarget();
220 }
221
getCodeGenOpts() const222 const CodeGenOptions &ABIInfo::getCodeGenOpts() const {
223 return CGT.getCodeGenOpts();
224 }
225
isAndroid() const226 bool ABIInfo::isAndroid() const { return getTarget().getTriple().isAndroid(); }
227
isHomogeneousAggregateBaseType(QualType Ty) const228 bool ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
229 return false;
230 }
231
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const232 bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
233 uint64_t Members) const {
234 return false;
235 }
236
dump() const237 LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
238 raw_ostream &OS = llvm::errs();
239 OS << "(ABIArgInfo Kind=";
240 switch (TheKind) {
241 case Direct:
242 OS << "Direct Type=";
243 if (llvm::Type *Ty = getCoerceToType())
244 Ty->print(OS);
245 else
246 OS << "null";
247 break;
248 case Extend:
249 OS << "Extend";
250 break;
251 case Ignore:
252 OS << "Ignore";
253 break;
254 case InAlloca:
255 OS << "InAlloca Offset=" << getInAllocaFieldIndex();
256 break;
257 case Indirect:
258 OS << "Indirect Align=" << getIndirectAlign().getQuantity()
259 << " ByVal=" << getIndirectByVal()
260 << " Realign=" << getIndirectRealign();
261 break;
262 case IndirectAliased:
263 OS << "Indirect Align=" << getIndirectAlign().getQuantity()
264 << " AadrSpace=" << getIndirectAddrSpace()
265 << " Realign=" << getIndirectRealign();
266 break;
267 case Expand:
268 OS << "Expand";
269 break;
270 case CoerceAndExpand:
271 OS << "CoerceAndExpand Type=";
272 getCoerceAndExpandType()->print(OS);
273 break;
274 }
275 OS << ")\n";
276 }
277
278 // Dynamically round a pointer up to a multiple of the given alignment.
emitRoundPointerUpToAlignment(CodeGenFunction & CGF,llvm::Value * Ptr,CharUnits Align)279 static llvm::Value *emitRoundPointerUpToAlignment(CodeGenFunction &CGF,
280 llvm::Value *Ptr,
281 CharUnits Align) {
282 llvm::Value *PtrAsInt = Ptr;
283 // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
284 PtrAsInt = CGF.Builder.CreatePtrToInt(PtrAsInt, CGF.IntPtrTy);
285 PtrAsInt = CGF.Builder.CreateAdd(PtrAsInt,
286 llvm::ConstantInt::get(CGF.IntPtrTy, Align.getQuantity() - 1));
287 PtrAsInt = CGF.Builder.CreateAnd(PtrAsInt,
288 llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity()));
289 PtrAsInt = CGF.Builder.CreateIntToPtr(PtrAsInt,
290 Ptr->getType(),
291 Ptr->getName() + ".aligned");
292 return PtrAsInt;
293 }
294
295 /// Emit va_arg for a platform using the common void* representation,
296 /// where arguments are simply emitted in an array of slots on the stack.
297 ///
298 /// This version implements the core direct-value passing rules.
299 ///
300 /// \param SlotSize - The size and alignment of a stack slot.
301 /// Each argument will be allocated to a multiple of this number of
302 /// slots, and all the slots will be aligned to this value.
303 /// \param AllowHigherAlign - The slot alignment is not a cap;
304 /// an argument type with an alignment greater than the slot size
305 /// will be emitted on a higher-alignment address, potentially
306 /// leaving one or more empty slots behind as padding. If this
307 /// is false, the returned address might be less-aligned than
308 /// DirectAlign.
emitVoidPtrDirectVAArg(CodeGenFunction & CGF,Address VAListAddr,llvm::Type * DirectTy,CharUnits DirectSize,CharUnits DirectAlign,CharUnits SlotSize,bool AllowHigherAlign)309 static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF,
310 Address VAListAddr,
311 llvm::Type *DirectTy,
312 CharUnits DirectSize,
313 CharUnits DirectAlign,
314 CharUnits SlotSize,
315 bool AllowHigherAlign) {
316 // Cast the element type to i8* if necessary. Some platforms define
317 // va_list as a struct containing an i8* instead of just an i8*.
318 if (VAListAddr.getElementType() != CGF.Int8PtrTy)
319 VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
320
321 llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
322
323 // If the CC aligns values higher than the slot size, do so if needed.
324 Address Addr = Address::invalid();
325 if (AllowHigherAlign && DirectAlign > SlotSize) {
326 Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
327 DirectAlign);
328 } else {
329 Addr = Address(Ptr, SlotSize);
330 }
331
332 // Advance the pointer past the argument, then store that back.
333 CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
334 Address NextPtr =
335 CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
336 CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
337
338 // If the argument is smaller than a slot, and this is a big-endian
339 // target, the argument will be right-adjusted in its slot.
340 if (DirectSize < SlotSize && CGF.CGM.getDataLayout().isBigEndian() &&
341 !DirectTy->isStructTy()) {
342 Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
343 }
344
345 Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
346 return Addr;
347 }
348
349 /// Emit va_arg for a platform using the common void* representation,
350 /// where arguments are simply emitted in an array of slots on the stack.
351 ///
352 /// \param IsIndirect - Values of this type are passed indirectly.
353 /// \param ValueInfo - The size and alignment of this type, generally
354 /// computed with getContext().getTypeInfoInChars(ValueTy).
355 /// \param SlotSizeAndAlign - The size and alignment of a stack slot.
356 /// Each argument will be allocated to a multiple of this number of
357 /// slots, and all the slots will be aligned to this value.
358 /// \param AllowHigherAlign - The slot alignment is not a cap;
359 /// an argument type with an alignment greater than the slot size
360 /// will be emitted on a higher-alignment address, potentially
361 /// leaving one or more empty slots behind as padding.
emitVoidPtrVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType ValueTy,bool IsIndirect,TypeInfoChars ValueInfo,CharUnits SlotSizeAndAlign,bool AllowHigherAlign)362 static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
363 QualType ValueTy, bool IsIndirect,
364 TypeInfoChars ValueInfo,
365 CharUnits SlotSizeAndAlign,
366 bool AllowHigherAlign) {
367 // The size and alignment of the value that was passed directly.
368 CharUnits DirectSize, DirectAlign;
369 if (IsIndirect) {
370 DirectSize = CGF.getPointerSize();
371 DirectAlign = CGF.getPointerAlign();
372 } else {
373 DirectSize = ValueInfo.Width;
374 DirectAlign = ValueInfo.Align;
375 }
376
377 // Cast the address we've calculated to the right type.
378 llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy);
379 if (IsIndirect)
380 DirectTy = DirectTy->getPointerTo(0);
381
382 Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy,
383 DirectSize, DirectAlign,
384 SlotSizeAndAlign,
385 AllowHigherAlign);
386
387 if (IsIndirect) {
388 Addr = Address(CGF.Builder.CreateLoad(Addr), ValueInfo.Align);
389 }
390
391 return Addr;
392
393 }
394
complexTempStructure(CodeGenFunction & CGF,Address VAListAddr,QualType Ty,CharUnits SlotSize,CharUnits EltSize,const ComplexType * CTy)395 static Address complexTempStructure(CodeGenFunction &CGF, Address VAListAddr,
396 QualType Ty, CharUnits SlotSize,
397 CharUnits EltSize, const ComplexType *CTy) {
398 Address Addr =
399 emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty, SlotSize * 2,
400 SlotSize, SlotSize, /*AllowHigher*/ true);
401
402 Address RealAddr = Addr;
403 Address ImagAddr = RealAddr;
404 if (CGF.CGM.getDataLayout().isBigEndian()) {
405 RealAddr =
406 CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize - EltSize);
407 ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(ImagAddr,
408 2 * SlotSize - EltSize);
409 } else {
410 ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize);
411 }
412
413 llvm::Type *EltTy = CGF.ConvertTypeForMem(CTy->getElementType());
414 RealAddr = CGF.Builder.CreateElementBitCast(RealAddr, EltTy);
415 ImagAddr = CGF.Builder.CreateElementBitCast(ImagAddr, EltTy);
416 llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal");
417 llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag");
418
419 Address Temp = CGF.CreateMemTemp(Ty, "vacplx");
420 CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty),
421 /*init*/ true);
422 return Temp;
423 }
424
emitMergePHI(CodeGenFunction & CGF,Address Addr1,llvm::BasicBlock * Block1,Address Addr2,llvm::BasicBlock * Block2,const llvm::Twine & Name="")425 static Address emitMergePHI(CodeGenFunction &CGF,
426 Address Addr1, llvm::BasicBlock *Block1,
427 Address Addr2, llvm::BasicBlock *Block2,
428 const llvm::Twine &Name = "") {
429 assert(Addr1.getType() == Addr2.getType());
430 llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
431 PHI->addIncoming(Addr1.getPointer(), Block1);
432 PHI->addIncoming(Addr2.getPointer(), Block2);
433 CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
434 return Address(PHI, Align);
435 }
436
437 TargetCodeGenInfo::~TargetCodeGenInfo() = default;
438
439 // If someone can figure out a general rule for this, that would be great.
440 // It's probably just doomed to be platform-dependent, though.
getSizeOfUnwindException() const441 unsigned TargetCodeGenInfo::getSizeOfUnwindException() const {
442 // Verified for:
443 // x86-64 FreeBSD, Linux, Darwin
444 // x86-32 FreeBSD, Linux, Darwin
445 // PowerPC Linux, Darwin
446 // ARM Darwin (*not* EABI)
447 // AArch64 Linux
448 return 32;
449 }
450
isNoProtoCallVariadic(const CallArgList & args,const FunctionNoProtoType * fnType) const451 bool TargetCodeGenInfo::isNoProtoCallVariadic(const CallArgList &args,
452 const FunctionNoProtoType *fnType) const {
453 // The following conventions are known to require this to be false:
454 // x86_stdcall
455 // MIPS
456 // For everything else, we just prefer false unless we opt out.
457 return false;
458 }
459
460 void
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const461 TargetCodeGenInfo::getDependentLibraryOption(llvm::StringRef Lib,
462 llvm::SmallString<24> &Opt) const {
463 // This assumes the user is passing a library name like "rt" instead of a
464 // filename like "librt.a/so", and that they don't care whether it's static or
465 // dynamic.
466 Opt = "-l";
467 Opt += Lib;
468 }
469
getOpenCLKernelCallingConv() const470 unsigned TargetCodeGenInfo::getOpenCLKernelCallingConv() const {
471 // OpenCL kernels are called via an explicit runtime API with arguments
472 // set with clSetKernelArg(), not as normal sub-functions.
473 // Return SPIR_KERNEL by default as the kernel calling convention to
474 // ensure the fingerprint is fixed such way that each OpenCL argument
475 // gets one matching argument in the produced kernel function argument
476 // list to enable feasible implementation of clSetKernelArg() with
477 // aggregates etc. In case we would use the default C calling conv here,
478 // clSetKernelArg() might break depending on the target-specific
479 // conventions; different targets might split structs passed as values
480 // to multiple function arguments etc.
481 return llvm::CallingConv::SPIR_KERNEL;
482 }
483
getNullPointer(const CodeGen::CodeGenModule & CGM,llvm::PointerType * T,QualType QT) const484 llvm::Constant *TargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
485 llvm::PointerType *T, QualType QT) const {
486 return llvm::ConstantPointerNull::get(T);
487 }
488
getGlobalVarAddressSpace(CodeGenModule & CGM,const VarDecl * D) const489 LangAS TargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
490 const VarDecl *D) const {
491 assert(!CGM.getLangOpts().OpenCL &&
492 !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
493 "Address space agnostic languages only");
494 return D ? D->getType().getAddressSpace() : LangAS::Default;
495 }
496
performAddrSpaceCast(CodeGen::CodeGenFunction & CGF,llvm::Value * Src,LangAS SrcAddr,LangAS DestAddr,llvm::Type * DestTy,bool isNonNull) const497 llvm::Value *TargetCodeGenInfo::performAddrSpaceCast(
498 CodeGen::CodeGenFunction &CGF, llvm::Value *Src, LangAS SrcAddr,
499 LangAS DestAddr, llvm::Type *DestTy, bool isNonNull) const {
500 // Since target may map different address spaces in AST to the same address
501 // space, an address space conversion may end up as a bitcast.
502 if (auto *C = dyn_cast<llvm::Constant>(Src))
503 return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
504 // Try to preserve the source's name to make IR more readable.
505 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
506 Src, DestTy, Src->hasName() ? Src->getName() + ".ascast" : "");
507 }
508
509 llvm::Constant *
performAddrSpaceCast(CodeGenModule & CGM,llvm::Constant * Src,LangAS SrcAddr,LangAS DestAddr,llvm::Type * DestTy) const510 TargetCodeGenInfo::performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *Src,
511 LangAS SrcAddr, LangAS DestAddr,
512 llvm::Type *DestTy) const {
513 // Since target may map different address spaces in AST to the same address
514 // space, an address space conversion may end up as a bitcast.
515 return llvm::ConstantExpr::getPointerCast(Src, DestTy);
516 }
517
518 llvm::SyncScope::ID
getLLVMSyncScopeID(const LangOptions & LangOpts,SyncScope Scope,llvm::AtomicOrdering Ordering,llvm::LLVMContext & Ctx) const519 TargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
520 SyncScope Scope,
521 llvm::AtomicOrdering Ordering,
522 llvm::LLVMContext &Ctx) const {
523 return Ctx.getOrInsertSyncScopeID(""); /* default sync scope */
524 }
525
526 static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
527
528 /// isEmptyField - Return true iff a the field is "empty", that is it
529 /// is an unnamed bit-field or an (array of) empty record(s).
isEmptyField(ASTContext & Context,const FieldDecl * FD,bool AllowArrays)530 static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
531 bool AllowArrays) {
532 if (FD->isUnnamedBitfield())
533 return true;
534
535 QualType FT = FD->getType();
536
537 // Constant arrays of empty records count as empty, strip them off.
538 // Constant arrays of zero length always count as empty.
539 bool WasArray = false;
540 if (AllowArrays)
541 while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
542 if (AT->getSize() == 0)
543 return true;
544 FT = AT->getElementType();
545 // The [[no_unique_address]] special case below does not apply to
546 // arrays of C++ empty records, so we need to remember this fact.
547 WasArray = true;
548 }
549
550 const RecordType *RT = FT->getAs<RecordType>();
551 if (!RT)
552 return false;
553
554 // C++ record fields are never empty, at least in the Itanium ABI.
555 //
556 // FIXME: We should use a predicate for whether this behavior is true in the
557 // current ABI.
558 //
559 // The exception to the above rule are fields marked with the
560 // [[no_unique_address]] attribute (since C++20). Those do count as empty
561 // according to the Itanium ABI. The exception applies only to records,
562 // not arrays of records, so we must also check whether we stripped off an
563 // array type above.
564 if (isa<CXXRecordDecl>(RT->getDecl()) &&
565 (WasArray || !FD->hasAttr<NoUniqueAddressAttr>()))
566 return false;
567
568 return isEmptyRecord(Context, FT, AllowArrays);
569 }
570
571 /// isEmptyRecord - Return true iff a structure contains only empty
572 /// fields. Note that a structure with a flexible array member is not
573 /// considered empty.
isEmptyRecord(ASTContext & Context,QualType T,bool AllowArrays)574 static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) {
575 const RecordType *RT = T->getAs<RecordType>();
576 if (!RT)
577 return false;
578 const RecordDecl *RD = RT->getDecl();
579 if (RD->hasFlexibleArrayMember())
580 return false;
581
582 // If this is a C++ record, check the bases first.
583 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
584 for (const auto &I : CXXRD->bases())
585 if (!isEmptyRecord(Context, I.getType(), true))
586 return false;
587
588 for (const auto *I : RD->fields())
589 if (!isEmptyField(Context, I, AllowArrays))
590 return false;
591 return true;
592 }
593
594 /// isSingleElementStruct - Determine if a structure is a "single
595 /// element struct", i.e. it has exactly one non-empty field or
596 /// exactly one field which is itself a single element
597 /// struct. Structures with flexible array members are never
598 /// considered single element structs.
599 ///
600 /// \return The field declaration for the single non-empty field, if
601 /// it exists.
isSingleElementStruct(QualType T,ASTContext & Context)602 static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
603 const RecordType *RT = T->getAs<RecordType>();
604 if (!RT)
605 return nullptr;
606
607 const RecordDecl *RD = RT->getDecl();
608 if (RD->hasFlexibleArrayMember())
609 return nullptr;
610
611 const Type *Found = nullptr;
612
613 // If this is a C++ record, check the bases first.
614 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
615 for (const auto &I : CXXRD->bases()) {
616 // Ignore empty records.
617 if (isEmptyRecord(Context, I.getType(), true))
618 continue;
619
620 // If we already found an element then this isn't a single-element struct.
621 if (Found)
622 return nullptr;
623
624 // If this is non-empty and not a single element struct, the composite
625 // cannot be a single element struct.
626 Found = isSingleElementStruct(I.getType(), Context);
627 if (!Found)
628 return nullptr;
629 }
630 }
631
632 // Check for single element.
633 for (const auto *FD : RD->fields()) {
634 QualType FT = FD->getType();
635
636 // Ignore empty fields.
637 if (isEmptyField(Context, FD, true))
638 continue;
639
640 // If we already found an element then this isn't a single-element
641 // struct.
642 if (Found)
643 return nullptr;
644
645 // Treat single element arrays as the element.
646 while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
647 if (AT->getSize().getZExtValue() != 1)
648 break;
649 FT = AT->getElementType();
650 }
651
652 if (!isAggregateTypeForABI(FT)) {
653 Found = FT.getTypePtr();
654 } else {
655 Found = isSingleElementStruct(FT, Context);
656 if (!Found)
657 return nullptr;
658 }
659 }
660
661 // We don't consider a struct a single-element struct if it has
662 // padding beyond the element type.
663 if (Found && Context.getTypeSize(Found) != Context.getTypeSize(T))
664 return nullptr;
665
666 return Found;
667 }
668
669 namespace {
EmitVAArgInstr(CodeGenFunction & CGF,Address VAListAddr,QualType Ty,const ABIArgInfo & AI)670 Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
671 const ABIArgInfo &AI) {
672 // This default implementation defers to the llvm backend's va_arg
673 // instruction. It can handle only passing arguments directly
674 // (typically only handled in the backend for primitive types), or
675 // aggregates passed indirectly by pointer (NOTE: if the "byval"
676 // flag has ABI impact in the callee, this implementation cannot
677 // work.)
678
679 // Only a few cases are covered here at the moment -- those needed
680 // by the default abi.
681 llvm::Value *Val;
682
683 if (AI.isIndirect()) {
684 assert(!AI.getPaddingType() &&
685 "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
686 assert(
687 !AI.getIndirectRealign() &&
688 "Unexpected IndirectRealign seen in arginfo in generic VAArg emitter!");
689
690 auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
691 CharUnits TyAlignForABI = TyInfo.Align;
692
693 llvm::Type *BaseTy =
694 llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
695 llvm::Value *Addr =
696 CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
697 return Address(Addr, TyAlignForABI);
698 } else {
699 assert((AI.isDirect() || AI.isExtend()) &&
700 "Unexpected ArgInfo Kind in generic VAArg emitter!");
701
702 assert(!AI.getInReg() &&
703 "Unexpected InReg seen in arginfo in generic VAArg emitter!");
704 assert(!AI.getPaddingType() &&
705 "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
706 assert(!AI.getDirectOffset() &&
707 "Unexpected DirectOffset seen in arginfo in generic VAArg emitter!");
708 assert(!AI.getCoerceToType() &&
709 "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
710
711 Address Temp = CGF.CreateMemTemp(Ty, "varet");
712 Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), CGF.ConvertType(Ty));
713 CGF.Builder.CreateStore(Val, Temp);
714 return Temp;
715 }
716 }
717
718 /// DefaultABIInfo - The default implementation for ABI specific
719 /// details. This implementation provides information which results in
720 /// self-consistent and sensible LLVM IR generation, but does not
721 /// conform to any particular ABI.
722 class DefaultABIInfo : public ABIInfo {
723 public:
DefaultABIInfo(CodeGen::CodeGenTypes & CGT)724 DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
725
726 ABIArgInfo classifyReturnType(QualType RetTy) const;
727 ABIArgInfo classifyArgumentType(QualType RetTy) const;
728
computeInfo(CGFunctionInfo & FI) const729 void computeInfo(CGFunctionInfo &FI) const override {
730 if (!getCXXABI().classifyReturnType(FI))
731 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
732 for (auto &I : FI.arguments())
733 I.info = classifyArgumentType(I.type);
734 }
735
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const736 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
737 QualType Ty) const override {
738 return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
739 }
740 };
741
742 class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
743 public:
DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT)744 DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
745 : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
746 };
747
classifyArgumentType(QualType Ty) const748 ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
749 Ty = useFirstFieldIfTransparentUnion(Ty);
750
751 if (isAggregateTypeForABI(Ty)) {
752 // Records with non-trivial destructors/copy-constructors should not be
753 // passed by value.
754 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
755 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
756
757 return getNaturalAlignIndirect(Ty);
758 }
759
760 // Treat an enum type as its underlying type.
761 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
762 Ty = EnumTy->getDecl()->getIntegerType();
763
764 ASTContext &Context = getContext();
765 if (const auto *EIT = Ty->getAs<ExtIntType>())
766 if (EIT->getNumBits() >
767 Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
768 ? Context.Int128Ty
769 : Context.LongLongTy))
770 return getNaturalAlignIndirect(Ty);
771
772 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
773 : ABIArgInfo::getDirect());
774 }
775
classifyReturnType(QualType RetTy) const776 ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
777 if (RetTy->isVoidType())
778 return ABIArgInfo::getIgnore();
779
780 if (isAggregateTypeForABI(RetTy))
781 return getNaturalAlignIndirect(RetTy);
782
783 // Treat an enum type as its underlying type.
784 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
785 RetTy = EnumTy->getDecl()->getIntegerType();
786
787 if (const auto *EIT = RetTy->getAs<ExtIntType>())
788 if (EIT->getNumBits() >
789 getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
790 ? getContext().Int128Ty
791 : getContext().LongLongTy))
792 return getNaturalAlignIndirect(RetTy);
793
794 return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
795 : ABIArgInfo::getDirect());
796 }
797
798 //===----------------------------------------------------------------------===//
799 // WebAssembly ABI Implementation
800 //
801 // This is a very simple ABI that relies a lot on DefaultABIInfo.
802 //===----------------------------------------------------------------------===//
803
804 class WebAssemblyABIInfo final : public SwiftABIInfo {
805 public:
806 enum ABIKind {
807 MVP = 0,
808 ExperimentalMV = 1,
809 };
810
811 private:
812 DefaultABIInfo defaultInfo;
813 ABIKind Kind;
814
815 public:
WebAssemblyABIInfo(CodeGen::CodeGenTypes & CGT,ABIKind Kind)816 explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind)
817 : SwiftABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {}
818
819 private:
820 ABIArgInfo classifyReturnType(QualType RetTy) const;
821 ABIArgInfo classifyArgumentType(QualType Ty) const;
822
823 // DefaultABIInfo's classifyReturnType and classifyArgumentType are
824 // non-virtual, but computeInfo and EmitVAArg are virtual, so we
825 // overload them.
computeInfo(CGFunctionInfo & FI) const826 void computeInfo(CGFunctionInfo &FI) const override {
827 if (!getCXXABI().classifyReturnType(FI))
828 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
829 for (auto &Arg : FI.arguments())
830 Arg.info = classifyArgumentType(Arg.type);
831 }
832
833 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
834 QualType Ty) const override;
835
shouldPassIndirectlyForSwift(ArrayRef<llvm::Type * > scalars,bool asReturnValue) const836 bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
837 bool asReturnValue) const override {
838 return occupiesMoreThan(CGT, scalars, /*total*/ 4);
839 }
840
isSwiftErrorInRegister() const841 bool isSwiftErrorInRegister() const override {
842 return false;
843 }
844 };
845
846 class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
847 public:
WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,WebAssemblyABIInfo::ABIKind K)848 explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
849 WebAssemblyABIInfo::ABIKind K)
850 : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {}
851
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const852 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
853 CodeGen::CodeGenModule &CGM) const override {
854 TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
855 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
856 if (const auto *Attr = FD->getAttr<WebAssemblyImportModuleAttr>()) {
857 llvm::Function *Fn = cast<llvm::Function>(GV);
858 llvm::AttrBuilder B;
859 B.addAttribute("wasm-import-module", Attr->getImportModule());
860 Fn->addFnAttrs(B);
861 }
862 if (const auto *Attr = FD->getAttr<WebAssemblyImportNameAttr>()) {
863 llvm::Function *Fn = cast<llvm::Function>(GV);
864 llvm::AttrBuilder B;
865 B.addAttribute("wasm-import-name", Attr->getImportName());
866 Fn->addFnAttrs(B);
867 }
868 if (const auto *Attr = FD->getAttr<WebAssemblyExportNameAttr>()) {
869 llvm::Function *Fn = cast<llvm::Function>(GV);
870 llvm::AttrBuilder B;
871 B.addAttribute("wasm-export-name", Attr->getExportName());
872 Fn->addFnAttrs(B);
873 }
874 }
875
876 if (auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
877 llvm::Function *Fn = cast<llvm::Function>(GV);
878 if (!FD->doesThisDeclarationHaveABody() && !FD->hasPrototype())
879 Fn->addFnAttr("no-prototype");
880 }
881 }
882 };
883
884 /// Classify argument of given type \p Ty.
classifyArgumentType(QualType Ty) const885 ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
886 Ty = useFirstFieldIfTransparentUnion(Ty);
887
888 if (isAggregateTypeForABI(Ty)) {
889 // Records with non-trivial destructors/copy-constructors should not be
890 // passed by value.
891 if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
892 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
893 // Ignore empty structs/unions.
894 if (isEmptyRecord(getContext(), Ty, true))
895 return ABIArgInfo::getIgnore();
896 // Lower single-element structs to just pass a regular value. TODO: We
897 // could do reasonable-size multiple-element structs too, using getExpand(),
898 // though watch out for things like bitfields.
899 if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
900 return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
901 // For the experimental multivalue ABI, fully expand all other aggregates
902 if (Kind == ABIKind::ExperimentalMV) {
903 const RecordType *RT = Ty->getAs<RecordType>();
904 assert(RT);
905 bool HasBitField = false;
906 for (auto *Field : RT->getDecl()->fields()) {
907 if (Field->isBitField()) {
908 HasBitField = true;
909 break;
910 }
911 }
912 if (!HasBitField)
913 return ABIArgInfo::getExpand();
914 }
915 }
916
917 // Otherwise just do the default thing.
918 return defaultInfo.classifyArgumentType(Ty);
919 }
920
classifyReturnType(QualType RetTy) const921 ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const {
922 if (isAggregateTypeForABI(RetTy)) {
923 // Records with non-trivial destructors/copy-constructors should not be
924 // returned by value.
925 if (!getRecordArgABI(RetTy, getCXXABI())) {
926 // Ignore empty structs/unions.
927 if (isEmptyRecord(getContext(), RetTy, true))
928 return ABIArgInfo::getIgnore();
929 // Lower single-element structs to just return a regular value. TODO: We
930 // could do reasonable-size multiple-element structs too, using
931 // ABIArgInfo::getDirect().
932 if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
933 return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
934 // For the experimental multivalue ABI, return all other aggregates
935 if (Kind == ABIKind::ExperimentalMV)
936 return ABIArgInfo::getDirect();
937 }
938 }
939
940 // Otherwise just do the default thing.
941 return defaultInfo.classifyReturnType(RetTy);
942 }
943
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const944 Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
945 QualType Ty) const {
946 bool IsIndirect = isAggregateTypeForABI(Ty) &&
947 !isEmptyRecord(getContext(), Ty, true) &&
948 !isSingleElementStruct(Ty, getContext());
949 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
950 getContext().getTypeInfoInChars(Ty),
951 CharUnits::fromQuantity(4),
952 /*AllowHigherAlign=*/true);
953 }
954
955 //===----------------------------------------------------------------------===//
956 // le32/PNaCl bitcode ABI Implementation
957 //
958 // This is a simplified version of the x86_32 ABI. Arguments and return values
959 // are always passed on the stack.
960 //===----------------------------------------------------------------------===//
961
962 class PNaClABIInfo : public ABIInfo {
963 public:
PNaClABIInfo(CodeGen::CodeGenTypes & CGT)964 PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
965
966 ABIArgInfo classifyReturnType(QualType RetTy) const;
967 ABIArgInfo classifyArgumentType(QualType RetTy) const;
968
969 void computeInfo(CGFunctionInfo &FI) const override;
970 Address EmitVAArg(CodeGenFunction &CGF,
971 Address VAListAddr, QualType Ty) const override;
972 };
973
974 class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
975 public:
PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT)976 PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
977 : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {}
978 };
979
computeInfo(CGFunctionInfo & FI) const980 void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
981 if (!getCXXABI().classifyReturnType(FI))
982 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
983
984 for (auto &I : FI.arguments())
985 I.info = classifyArgumentType(I.type);
986 }
987
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const988 Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
989 QualType Ty) const {
990 // The PNaCL ABI is a bit odd, in that varargs don't use normal
991 // function classification. Structs get passed directly for varargs
992 // functions, through a rewriting transform in
993 // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows
994 // this target to actually support a va_arg instructions with an
995 // aggregate type, unlike other targets.
996 return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
997 }
998
999 /// Classify argument of given type \p Ty.
classifyArgumentType(QualType Ty) const1000 ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
1001 if (isAggregateTypeForABI(Ty)) {
1002 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
1003 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
1004 return getNaturalAlignIndirect(Ty);
1005 } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
1006 // Treat an enum type as its underlying type.
1007 Ty = EnumTy->getDecl()->getIntegerType();
1008 } else if (Ty->isFloatingType()) {
1009 // Floating-point types don't go inreg.
1010 return ABIArgInfo::getDirect();
1011 } else if (const auto *EIT = Ty->getAs<ExtIntType>()) {
1012 // Treat extended integers as integers if <=64, otherwise pass indirectly.
1013 if (EIT->getNumBits() > 64)
1014 return getNaturalAlignIndirect(Ty);
1015 return ABIArgInfo::getDirect();
1016 }
1017
1018 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
1019 : ABIArgInfo::getDirect());
1020 }
1021
classifyReturnType(QualType RetTy) const1022 ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
1023 if (RetTy->isVoidType())
1024 return ABIArgInfo::getIgnore();
1025
1026 // In the PNaCl ABI we always return records/structures on the stack.
1027 if (isAggregateTypeForABI(RetTy))
1028 return getNaturalAlignIndirect(RetTy);
1029
1030 // Treat extended integers as integers if <=64, otherwise pass indirectly.
1031 if (const auto *EIT = RetTy->getAs<ExtIntType>()) {
1032 if (EIT->getNumBits() > 64)
1033 return getNaturalAlignIndirect(RetTy);
1034 return ABIArgInfo::getDirect();
1035 }
1036
1037 // Treat an enum type as its underlying type.
1038 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1039 RetTy = EnumTy->getDecl()->getIntegerType();
1040
1041 return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1042 : ABIArgInfo::getDirect());
1043 }
1044
1045 /// IsX86_MMXType - Return true if this is an MMX type.
IsX86_MMXType(llvm::Type * IRType)1046 bool IsX86_MMXType(llvm::Type *IRType) {
1047 // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
1048 return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
1049 cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy() &&
1050 IRType->getScalarSizeInBits() != 64;
1051 }
1052
X86AdjustInlineAsmType(CodeGen::CodeGenFunction & CGF,StringRef Constraint,llvm::Type * Ty)1053 static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1054 StringRef Constraint,
1055 llvm::Type* Ty) {
1056 bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
1057 .Cases("y", "&y", "^Ym", true)
1058 .Default(false);
1059 if (IsMMXCons && Ty->isVectorTy()) {
1060 if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedSize() !=
1061 64) {
1062 // Invalid MMX constraint
1063 return nullptr;
1064 }
1065
1066 return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
1067 }
1068
1069 // No operation needed
1070 return Ty;
1071 }
1072
1073 /// Returns true if this type can be passed in SSE registers with the
1074 /// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
isX86VectorTypeForVectorCall(ASTContext & Context,QualType Ty)1075 static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
1076 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1077 if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {
1078 if (BT->getKind() == BuiltinType::LongDouble) {
1079 if (&Context.getTargetInfo().getLongDoubleFormat() ==
1080 &llvm::APFloat::x87DoubleExtended())
1081 return false;
1082 }
1083 return true;
1084 }
1085 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
1086 // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
1087 // registers specially.
1088 unsigned VecSize = Context.getTypeSize(VT);
1089 if (VecSize == 128 || VecSize == 256 || VecSize == 512)
1090 return true;
1091 }
1092 return false;
1093 }
1094
1095 /// Returns true if this aggregate is small enough to be passed in SSE registers
1096 /// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
isX86VectorCallAggregateSmallEnough(uint64_t NumMembers)1097 static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
1098 return NumMembers <= 4;
1099 }
1100
1101 /// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
getDirectX86Hva(llvm::Type * T=nullptr)1102 static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
1103 auto AI = ABIArgInfo::getDirect(T);
1104 AI.setInReg(true);
1105 AI.setCanBeFlattened(false);
1106 return AI;
1107 }
1108
1109 //===----------------------------------------------------------------------===//
1110 // X86-32 ABI Implementation
1111 //===----------------------------------------------------------------------===//
1112
1113 /// Similar to llvm::CCState, but for Clang.
1114 struct CCState {
CCState__anon1dc9252a0111::CCState1115 CCState(CGFunctionInfo &FI)
1116 : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()) {}
1117
1118 llvm::SmallBitVector IsPreassigned;
1119 unsigned CC = CallingConv::CC_C;
1120 unsigned FreeRegs = 0;
1121 unsigned FreeSSERegs = 0;
1122 };
1123
1124 /// X86_32ABIInfo - The X86-32 ABI information.
1125 class X86_32ABIInfo : public SwiftABIInfo {
1126 enum Class {
1127 Integer,
1128 Float
1129 };
1130
1131 static const unsigned MinABIStackAlignInBytes = 4;
1132
1133 bool IsDarwinVectorABI;
1134 bool IsRetSmallStructInRegABI;
1135 bool IsWin32StructABI;
1136 bool IsSoftFloatABI;
1137 bool IsMCUABI;
1138 bool IsLinuxABI;
1139 unsigned DefaultNumRegisterParameters;
1140
isRegisterSize(unsigned Size)1141 static bool isRegisterSize(unsigned Size) {
1142 return (Size == 8 || Size == 16 || Size == 32 || Size == 64);
1143 }
1144
isHomogeneousAggregateBaseType(QualType Ty) const1145 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1146 // FIXME: Assumes vectorcall is in use.
1147 return isX86VectorTypeForVectorCall(getContext(), Ty);
1148 }
1149
isHomogeneousAggregateSmallEnough(const Type * Ty,uint64_t NumMembers) const1150 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1151 uint64_t NumMembers) const override {
1152 // FIXME: Assumes vectorcall is in use.
1153 return isX86VectorCallAggregateSmallEnough(NumMembers);
1154 }
1155
1156 bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
1157
1158 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1159 /// such that the argument will be passed in memory.
1160 ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
1161
1162 ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
1163
1164 /// Return the alignment to use for the given type on the stack.
1165 unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
1166
1167 Class classify(QualType Ty) const;
1168 ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
1169 ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
1170
1171 /// Updates the number of available free registers, returns
1172 /// true if any registers were allocated.
1173 bool updateFreeRegs(QualType Ty, CCState &State) const;
1174
1175 bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
1176 bool &NeedsPadding) const;
1177 bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
1178
1179 bool canExpandIndirectArgument(QualType Ty) const;
1180
1181 /// Rewrite the function info so that all memory arguments use
1182 /// inalloca.
1183 void rewriteWithInAlloca(CGFunctionInfo &FI) const;
1184
1185 void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1186 CharUnits &StackOffset, ABIArgInfo &Info,
1187 QualType Type) const;
1188 void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
1189
1190 public:
1191
1192 void computeInfo(CGFunctionInfo &FI) const override;
1193 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1194 QualType Ty) const override;
1195
X86_32ABIInfo(CodeGen::CodeGenTypes & CGT,bool DarwinVectorABI,bool RetSmallStructInRegABI,bool Win32StructABI,unsigned NumRegisterParameters,bool SoftFloatABI)1196 X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1197 bool RetSmallStructInRegABI, bool Win32StructABI,
1198 unsigned NumRegisterParameters, bool SoftFloatABI)
1199 : SwiftABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
1200 IsRetSmallStructInRegABI(RetSmallStructInRegABI),
1201 IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
1202 IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
1203 IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
1204 CGT.getTarget().getTriple().isOSCygMing()),
1205 DefaultNumRegisterParameters(NumRegisterParameters) {}
1206
shouldPassIndirectlyForSwift(ArrayRef<llvm::Type * > scalars,bool asReturnValue) const1207 bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
1208 bool asReturnValue) const override {
1209 // LLVM's x86-32 lowering currently only assigns up to three
1210 // integer registers and three fp registers. Oddly, it'll use up to
1211 // four vector registers for vectors, but those can overlap with the
1212 // scalar registers.
1213 return occupiesMoreThan(CGT, scalars, /*total*/ 3);
1214 }
1215
isSwiftErrorInRegister() const1216 bool isSwiftErrorInRegister() const override {
1217 // x86-32 lowering does not support passing swifterror in a register.
1218 return false;
1219 }
1220 };
1221
1222 class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
1223 public:
X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,bool DarwinVectorABI,bool RetSmallStructInRegABI,bool Win32StructABI,unsigned NumRegisterParameters,bool SoftFloatABI)1224 X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1225 bool RetSmallStructInRegABI, bool Win32StructABI,
1226 unsigned NumRegisterParameters, bool SoftFloatABI)
1227 : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
1228 CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
1229 NumRegisterParameters, SoftFloatABI)) {}
1230
1231 static bool isStructReturnInRegABI(
1232 const llvm::Triple &Triple, const CodeGenOptions &Opts);
1233
1234 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1235 CodeGen::CodeGenModule &CGM) const override;
1236
getDwarfEHStackPointer(CodeGen::CodeGenModule & CGM) const1237 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1238 // Darwin uses different dwarf register numbers for EH.
1239 if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
1240 return 4;
1241 }
1242
1243 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1244 llvm::Value *Address) const override;
1245
adjustInlineAsmType(CodeGen::CodeGenFunction & CGF,StringRef Constraint,llvm::Type * Ty) const1246 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1247 StringRef Constraint,
1248 llvm::Type* Ty) const override {
1249 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1250 }
1251
1252 void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
1253 std::string &Constraints,
1254 std::vector<llvm::Type *> &ResultRegTypes,
1255 std::vector<llvm::Type *> &ResultTruncRegTypes,
1256 std::vector<LValue> &ResultRegDests,
1257 std::string &AsmString,
1258 unsigned NumOutputs) const override;
1259
1260 llvm::Constant *
getUBSanFunctionSignature(CodeGen::CodeGenModule & CGM) const1261 getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
1262 unsigned Sig = (0xeb << 0) | // jmp rel8
1263 (0x06 << 8) | // .+0x08
1264 ('v' << 16) |
1265 ('2' << 24);
1266 return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
1267 }
1268
getARCRetainAutoreleasedReturnValueMarker() const1269 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
1270 return "movl\t%ebp, %ebp"
1271 "\t\t// marker for objc_retainAutoreleaseReturnValue";
1272 }
1273 };
1274
1275 }
1276
1277 /// Rewrite input constraint references after adding some output constraints.
1278 /// In the case where there is one output and one input and we add one output,
1279 /// we need to replace all operand references greater than or equal to 1:
1280 /// mov $0, $1
1281 /// mov eax, $1
1282 /// The result will be:
1283 /// mov $0, $2
1284 /// mov eax, $2
rewriteInputConstraintReferences(unsigned FirstIn,unsigned NumNewOuts,std::string & AsmString)1285 static void rewriteInputConstraintReferences(unsigned FirstIn,
1286 unsigned NumNewOuts,
1287 std::string &AsmString) {
1288 std::string Buf;
1289 llvm::raw_string_ostream OS(Buf);
1290 size_t Pos = 0;
1291 while (Pos < AsmString.size()) {
1292 size_t DollarStart = AsmString.find('$', Pos);
1293 if (DollarStart == std::string::npos)
1294 DollarStart = AsmString.size();
1295 size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);
1296 if (DollarEnd == std::string::npos)
1297 DollarEnd = AsmString.size();
1298 OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
1299 Pos = DollarEnd;
1300 size_t NumDollars = DollarEnd - DollarStart;
1301 if (NumDollars % 2 != 0 && Pos < AsmString.size()) {
1302 // We have an operand reference.
1303 size_t DigitStart = Pos;
1304 if (AsmString[DigitStart] == '{') {
1305 OS << '{';
1306 ++DigitStart;
1307 }
1308 size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);
1309 if (DigitEnd == std::string::npos)
1310 DigitEnd = AsmString.size();
1311 StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
1312 unsigned OperandIndex;
1313 if (!OperandStr.getAsInteger(10, OperandIndex)) {
1314 if (OperandIndex >= FirstIn)
1315 OperandIndex += NumNewOuts;
1316 OS << OperandIndex;
1317 } else {
1318 OS << OperandStr;
1319 }
1320 Pos = DigitEnd;
1321 }
1322 }
1323 AsmString = std::move(OS.str());
1324 }
1325
1326 /// Add output constraints for EAX:EDX because they are return registers.
addReturnRegisterOutputs(CodeGenFunction & CGF,LValue ReturnSlot,std::string & Constraints,std::vector<llvm::Type * > & ResultRegTypes,std::vector<llvm::Type * > & ResultTruncRegTypes,std::vector<LValue> & ResultRegDests,std::string & AsmString,unsigned NumOutputs) const1327 void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
1328 CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
1329 std::vector<llvm::Type *> &ResultRegTypes,
1330 std::vector<llvm::Type *> &ResultTruncRegTypes,
1331 std::vector<LValue> &ResultRegDests, std::string &AsmString,
1332 unsigned NumOutputs) const {
1333 uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());
1334
1335 // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
1336 // larger.
1337 if (!Constraints.empty())
1338 Constraints += ',';
1339 if (RetWidth <= 32) {
1340 Constraints += "={eax}";
1341 ResultRegTypes.push_back(CGF.Int32Ty);
1342 } else {
1343 // Use the 'A' constraint for EAX:EDX.
1344 Constraints += "=A";
1345 ResultRegTypes.push_back(CGF.Int64Ty);
1346 }
1347
1348 // Truncate EAX or EAX:EDX to an integer of the appropriate size.
1349 llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);
1350 ResultTruncRegTypes.push_back(CoerceTy);
1351
1352 // Coerce the integer by bitcasting the return slot pointer.
1353 ReturnSlot.setAddress(CGF.Builder.CreateBitCast(ReturnSlot.getAddress(CGF),
1354 CoerceTy->getPointerTo()));
1355 ResultRegDests.push_back(ReturnSlot);
1356
1357 rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
1358 }
1359
1360 /// shouldReturnTypeInRegister - Determine if the given type should be
1361 /// returned in a register (for the Darwin and MCU ABI).
shouldReturnTypeInRegister(QualType Ty,ASTContext & Context) const1362 bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
1363 ASTContext &Context) const {
1364 uint64_t Size = Context.getTypeSize(Ty);
1365
1366 // For i386, type must be register sized.
1367 // For the MCU ABI, it only needs to be <= 8-byte
1368 if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))
1369 return false;
1370
1371 if (Ty->isVectorType()) {
1372 // 64- and 128- bit vectors inside structures are not returned in
1373 // registers.
1374 if (Size == 64 || Size == 128)
1375 return false;
1376
1377 return true;
1378 }
1379
1380 // If this is a builtin, pointer, enum, complex type, member pointer, or
1381 // member function pointer it is ok.
1382 if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||
1383 Ty->isAnyComplexType() || Ty->isEnumeralType() ||
1384 Ty->isBlockPointerType() || Ty->isMemberPointerType())
1385 return true;
1386
1387 // Arrays are treated like records.
1388 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))
1389 return shouldReturnTypeInRegister(AT->getElementType(), Context);
1390
1391 // Otherwise, it must be a record type.
1392 const RecordType *RT = Ty->getAs<RecordType>();
1393 if (!RT) return false;
1394
1395 // FIXME: Traverse bases here too.
1396
1397 // Structure types are passed in register if all fields would be
1398 // passed in a register.
1399 for (const auto *FD : RT->getDecl()->fields()) {
1400 // Empty fields are ignored.
1401 if (isEmptyField(Context, FD, true))
1402 continue;
1403
1404 // Check fields recursively.
1405 if (!shouldReturnTypeInRegister(FD->getType(), Context))
1406 return false;
1407 }
1408 return true;
1409 }
1410
is32Or64BitBasicType(QualType Ty,ASTContext & Context)1411 static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
1412 // Treat complex types as the element type.
1413 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
1414 Ty = CTy->getElementType();
1415
1416 // Check for a type which we know has a simple scalar argument-passing
1417 // convention without any padding. (We're specifically looking for 32
1418 // and 64-bit integer and integer-equivalents, float, and double.)
1419 if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&
1420 !Ty->isEnumeralType() && !Ty->isBlockPointerType())
1421 return false;
1422
1423 uint64_t Size = Context.getTypeSize(Ty);
1424 return Size == 32 || Size == 64;
1425 }
1426
addFieldSizes(ASTContext & Context,const RecordDecl * RD,uint64_t & Size)1427 static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
1428 uint64_t &Size) {
1429 for (const auto *FD : RD->fields()) {
1430 // Scalar arguments on the stack get 4 byte alignment on x86. If the
1431 // argument is smaller than 32-bits, expanding the struct will create
1432 // alignment padding.
1433 if (!is32Or64BitBasicType(FD->getType(), Context))
1434 return false;
1435
1436 // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
1437 // how to expand them yet, and the predicate for telling if a bitfield still
1438 // counts as "basic" is more complicated than what we were doing previously.
1439 if (FD->isBitField())
1440 return false;
1441
1442 Size += Context.getTypeSize(FD->getType());
1443 }
1444 return true;
1445 }
1446
addBaseAndFieldSizes(ASTContext & Context,const CXXRecordDecl * RD,uint64_t & Size)1447 static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
1448 uint64_t &Size) {
1449 // Don't do this if there are any non-empty bases.
1450 for (const CXXBaseSpecifier &Base : RD->bases()) {
1451 if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),
1452 Size))
1453 return false;
1454 }
1455 if (!addFieldSizes(Context, RD, Size))
1456 return false;
1457 return true;
1458 }
1459
1460 /// Test whether an argument type which is to be passed indirectly (on the
1461 /// stack) would have the equivalent layout if it was expanded into separate
1462 /// arguments. If so, we prefer to do the latter to avoid inhibiting
1463 /// optimizations.
canExpandIndirectArgument(QualType Ty) const1464 bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
1465 // We can only expand structure types.
1466 const RecordType *RT = Ty->getAs<RecordType>();
1467 if (!RT)
1468 return false;
1469 const RecordDecl *RD = RT->getDecl();
1470 uint64_t Size = 0;
1471 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1472 if (!IsWin32StructABI) {
1473 // On non-Windows, we have to conservatively match our old bitcode
1474 // prototypes in order to be ABI-compatible at the bitcode level.
1475 if (!CXXRD->isCLike())
1476 return false;
1477 } else {
1478 // Don't do this for dynamic classes.
1479 if (CXXRD->isDynamicClass())
1480 return false;
1481 }
1482 if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))
1483 return false;
1484 } else {
1485 if (!addFieldSizes(getContext(), RD, Size))
1486 return false;
1487 }
1488
1489 // We can do this if there was no alignment padding.
1490 return Size == getContext().getTypeSize(Ty);
1491 }
1492
getIndirectReturnResult(QualType RetTy,CCState & State) const1493 ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
1494 // If the return value is indirect, then the hidden argument is consuming one
1495 // integer register.
1496 if (State.FreeRegs) {
1497 --State.FreeRegs;
1498 if (!IsMCUABI)
1499 return getNaturalAlignIndirectInReg(RetTy);
1500 }
1501 return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
1502 }
1503
classifyReturnType(QualType RetTy,CCState & State) const1504 ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
1505 CCState &State) const {
1506 if (RetTy->isVoidType())
1507 return ABIArgInfo::getIgnore();
1508
1509 const Type *Base = nullptr;
1510 uint64_t NumElts = 0;
1511 if ((State.CC == llvm::CallingConv::X86_VectorCall ||
1512 State.CC == llvm::CallingConv::X86_RegCall) &&
1513 isHomogeneousAggregate(RetTy, Base, NumElts)) {
1514 // The LLVM struct type for such an aggregate should lower properly.
1515 return ABIArgInfo::getDirect();
1516 }
1517
1518 if (const VectorType *VT = RetTy->getAs<VectorType>()) {
1519 // On Darwin, some vectors are returned in registers.
1520 if (IsDarwinVectorABI) {
1521 uint64_t Size = getContext().getTypeSize(RetTy);
1522
1523 // 128-bit vectors are a special case; they are returned in
1524 // registers and we need to make sure to pick a type the LLVM
1525 // backend will like.
1526 if (Size == 128)
1527 return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1528 llvm::Type::getInt64Ty(getVMContext()), 2));
1529
1530 // Always return in register if it fits in a general purpose
1531 // register, or if it is 64 bits and has a single element.
1532 if ((Size == 8 || Size == 16 || Size == 32) ||
1533 (Size == 64 && VT->getNumElements() == 1))
1534 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
1535 Size));
1536
1537 return getIndirectReturnResult(RetTy, State);
1538 }
1539
1540 return ABIArgInfo::getDirect();
1541 }
1542
1543 if (isAggregateTypeForABI(RetTy)) {
1544 if (const RecordType *RT = RetTy->getAs<RecordType>()) {
1545 // Structures with flexible arrays are always indirect.
1546 if (RT->getDecl()->hasFlexibleArrayMember())
1547 return getIndirectReturnResult(RetTy, State);
1548 }
1549
1550 // If specified, structs and unions are always indirect.
1551 if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
1552 return getIndirectReturnResult(RetTy, State);
1553
1554 // Ignore empty structs/unions.
1555 if (isEmptyRecord(getContext(), RetTy, true))
1556 return ABIArgInfo::getIgnore();
1557
1558 // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
1559 if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
1560 QualType ET = getContext().getCanonicalType(CT->getElementType());
1561 if (ET->isFloat16Type())
1562 return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1563 llvm::Type::getHalfTy(getVMContext()), 2));
1564 }
1565
1566 // Small structures which are register sized are generally returned
1567 // in a register.
1568 if (shouldReturnTypeInRegister(RetTy, getContext())) {
1569 uint64_t Size = getContext().getTypeSize(RetTy);
1570
1571 // As a special-case, if the struct is a "single-element" struct, and
1572 // the field is of type "float" or "double", return it in a
1573 // floating-point register. (MSVC does not apply this special case.)
1574 // We apply a similar transformation for pointer types to improve the
1575 // quality of the generated IR.
1576 if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
1577 if ((!IsWin32StructABI && SeltTy->isRealFloatingType())
1578 || SeltTy->hasPointerRepresentation())
1579 return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
1580
1581 // FIXME: We should be able to narrow this integer in cases with dead
1582 // padding.
1583 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
1584 }
1585
1586 return getIndirectReturnResult(RetTy, State);
1587 }
1588
1589 // Treat an enum type as its underlying type.
1590 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1591 RetTy = EnumTy->getDecl()->getIntegerType();
1592
1593 if (const auto *EIT = RetTy->getAs<ExtIntType>())
1594 if (EIT->getNumBits() > 64)
1595 return getIndirectReturnResult(RetTy, State);
1596
1597 return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1598 : ABIArgInfo::getDirect());
1599 }
1600
isSIMDVectorType(ASTContext & Context,QualType Ty)1601 static bool isSIMDVectorType(ASTContext &Context, QualType Ty) {
1602 return Ty->getAs<VectorType>() && Context.getTypeSize(Ty) == 128;
1603 }
1604
isRecordWithSIMDVectorType(ASTContext & Context,QualType Ty)1605 static bool isRecordWithSIMDVectorType(ASTContext &Context, QualType Ty) {
1606 const RecordType *RT = Ty->getAs<RecordType>();
1607 if (!RT)
1608 return 0;
1609 const RecordDecl *RD = RT->getDecl();
1610
1611 // If this is a C++ record, check the bases first.
1612 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
1613 for (const auto &I : CXXRD->bases())
1614 if (!isRecordWithSIMDVectorType(Context, I.getType()))
1615 return false;
1616
1617 for (const auto *i : RD->fields()) {
1618 QualType FT = i->getType();
1619
1620 if (isSIMDVectorType(Context, FT))
1621 return true;
1622
1623 if (isRecordWithSIMDVectorType(Context, FT))
1624 return true;
1625 }
1626
1627 return false;
1628 }
1629
getTypeStackAlignInBytes(QualType Ty,unsigned Align) const1630 unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
1631 unsigned Align) const {
1632 // Otherwise, if the alignment is less than or equal to the minimum ABI
1633 // alignment, just use the default; the backend will handle this.
1634 if (Align <= MinABIStackAlignInBytes)
1635 return 0; // Use default alignment.
1636
1637 if (IsLinuxABI) {
1638 // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
1639 // want to spend any effort dealing with the ramifications of ABI breaks.
1640 //
1641 // If the vector type is __m128/__m256/__m512, return the default alignment.
1642 if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))
1643 return Align;
1644 }
1645 // On non-Darwin, the stack type alignment is always 4.
1646 if (!IsDarwinVectorABI) {
1647 // Set explicit alignment, since we may need to realign the top.
1648 return MinABIStackAlignInBytes;
1649 }
1650
1651 // Otherwise, if the type contains an SSE vector type, the alignment is 16.
1652 if (Align >= 16 && (isSIMDVectorType(getContext(), Ty) ||
1653 isRecordWithSIMDVectorType(getContext(), Ty)))
1654 return 16;
1655
1656 return MinABIStackAlignInBytes;
1657 }
1658
getIndirectResult(QualType Ty,bool ByVal,CCState & State) const1659 ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
1660 CCState &State) const {
1661 if (!ByVal) {
1662 if (State.FreeRegs) {
1663 --State.FreeRegs; // Non-byval indirects just use one pointer.
1664 if (!IsMCUABI)
1665 return getNaturalAlignIndirectInReg(Ty);
1666 }
1667 return getNaturalAlignIndirect(Ty, false);
1668 }
1669
1670 // Compute the byval alignment.
1671 unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
1672 unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
1673 if (StackAlign == 0)
1674 return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
1675
1676 // If the stack alignment is less than the type alignment, realign the
1677 // argument.
1678 bool Realign = TypeAlign > StackAlign;
1679 return ABIArgInfo::getIndirect(CharUnits::fromQuantity(StackAlign),
1680 /*ByVal=*/true, Realign);
1681 }
1682
classify(QualType Ty) const1683 X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
1684 const Type *T = isSingleElementStruct(Ty, getContext());
1685 if (!T)
1686 T = Ty.getTypePtr();
1687
1688 if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
1689 BuiltinType::Kind K = BT->getKind();
1690 if (K == BuiltinType::Float || K == BuiltinType::Double)
1691 return Float;
1692 }
1693 return Integer;
1694 }
1695
updateFreeRegs(QualType Ty,CCState & State) const1696 bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
1697 if (!IsSoftFloatABI) {
1698 Class C = classify(Ty);
1699 if (C == Float)
1700 return false;
1701 }
1702
1703 unsigned Size = getContext().getTypeSize(Ty);
1704 unsigned SizeInRegs = (Size + 31) / 32;
1705
1706 if (SizeInRegs == 0)
1707 return false;
1708
1709 if (!IsMCUABI) {
1710 if (SizeInRegs > State.FreeRegs) {
1711 State.FreeRegs = 0;
1712 return false;
1713 }
1714 } else {
1715 // The MCU psABI allows passing parameters in-reg even if there are
1716 // earlier parameters that are passed on the stack. Also,
1717 // it does not allow passing >8-byte structs in-register,
1718 // even if there are 3 free registers available.
1719 if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
1720 return false;
1721 }
1722
1723 State.FreeRegs -= SizeInRegs;
1724 return true;
1725 }
1726
shouldAggregateUseDirect(QualType Ty,CCState & State,bool & InReg,bool & NeedsPadding) const1727 bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
1728 bool &InReg,
1729 bool &NeedsPadding) const {
1730 // On Windows, aggregates other than HFAs are never passed in registers, and
1731 // they do not consume register slots. Homogenous floating-point aggregates
1732 // (HFAs) have already been dealt with at this point.
1733 if (IsWin32StructABI && isAggregateTypeForABI(Ty))
1734 return false;
1735
1736 NeedsPadding = false;
1737 InReg = !IsMCUABI;
1738
1739 if (!updateFreeRegs(Ty, State))
1740 return false;
1741
1742 if (IsMCUABI)
1743 return true;
1744
1745 if (State.CC == llvm::CallingConv::X86_FastCall ||
1746 State.CC == llvm::CallingConv::X86_VectorCall ||
1747 State.CC == llvm::CallingConv::X86_RegCall) {
1748 if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)
1749 NeedsPadding = true;
1750
1751 return false;
1752 }
1753
1754 return true;
1755 }
1756
shouldPrimitiveUseInReg(QualType Ty,CCState & State) const1757 bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
1758 if (!updateFreeRegs(Ty, State))
1759 return false;
1760
1761 if (IsMCUABI)
1762 return false;
1763
1764 if (State.CC == llvm::CallingConv::X86_FastCall ||
1765 State.CC == llvm::CallingConv::X86_VectorCall ||
1766 State.CC == llvm::CallingConv::X86_RegCall) {
1767 if (getContext().getTypeSize(Ty) > 32)
1768 return false;
1769
1770 return (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
1771 Ty->isReferenceType());
1772 }
1773
1774 return true;
1775 }
1776
runVectorCallFirstPass(CGFunctionInfo & FI,CCState & State) const1777 void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
1778 // Vectorcall x86 works subtly different than in x64, so the format is
1779 // a bit different than the x64 version. First, all vector types (not HVAs)
1780 // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
1781 // This differs from the x64 implementation, where the first 6 by INDEX get
1782 // registers.
1783 // In the second pass over the arguments, HVAs are passed in the remaining
1784 // vector registers if possible, or indirectly by address. The address will be
1785 // passed in ECX/EDX if available. Any other arguments are passed according to
1786 // the usual fastcall rules.
1787 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1788 for (int I = 0, E = Args.size(); I < E; ++I) {
1789 const Type *Base = nullptr;
1790 uint64_t NumElts = 0;
1791 const QualType &Ty = Args[I].type;
1792 if ((Ty->isVectorType() || Ty->isBuiltinType()) &&
1793 isHomogeneousAggregate(Ty, Base, NumElts)) {
1794 if (State.FreeSSERegs >= NumElts) {
1795 State.FreeSSERegs -= NumElts;
1796 Args[I].info = ABIArgInfo::getDirectInReg();
1797 State.IsPreassigned.set(I);
1798 }
1799 }
1800 }
1801 }
1802
classifyArgumentType(QualType Ty,CCState & State) const1803 ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
1804 CCState &State) const {
1805 // FIXME: Set alignment on indirect arguments.
1806 bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
1807 bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
1808 bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
1809
1810 Ty = useFirstFieldIfTransparentUnion(Ty);
1811 TypeInfo TI = getContext().getTypeInfo(Ty);
1812
1813 // Check with the C++ ABI first.
1814 const RecordType *RT = Ty->getAs<RecordType>();
1815 if (RT) {
1816 CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
1817 if (RAA == CGCXXABI::RAA_Indirect) {
1818 return getIndirectResult(Ty, false, State);
1819 } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
1820 // The field index doesn't matter, we'll fix it up later.
1821 return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
1822 }
1823 }
1824
1825 // Regcall uses the concept of a homogenous vector aggregate, similar
1826 // to other targets.
1827 const Type *Base = nullptr;
1828 uint64_t NumElts = 0;
1829 if ((IsRegCall || IsVectorCall) &&
1830 isHomogeneousAggregate(Ty, Base, NumElts)) {
1831 if (State.FreeSSERegs >= NumElts) {
1832 State.FreeSSERegs -= NumElts;
1833
1834 // Vectorcall passes HVAs directly and does not flatten them, but regcall
1835 // does.
1836 if (IsVectorCall)
1837 return getDirectX86Hva();
1838
1839 if (Ty->isBuiltinType() || Ty->isVectorType())
1840 return ABIArgInfo::getDirect();
1841 return ABIArgInfo::getExpand();
1842 }
1843 return getIndirectResult(Ty, /*ByVal=*/false, State);
1844 }
1845
1846 if (isAggregateTypeForABI(Ty)) {
1847 // Structures with flexible arrays are always indirect.
1848 // FIXME: This should not be byval!
1849 if (RT && RT->getDecl()->hasFlexibleArrayMember())
1850 return getIndirectResult(Ty, true, State);
1851
1852 // Ignore empty structs/unions on non-Windows.
1853 if (!IsWin32StructABI && isEmptyRecord(getContext(), Ty, true))
1854 return ABIArgInfo::getIgnore();
1855
1856 llvm::LLVMContext &LLVMContext = getVMContext();
1857 llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
1858 bool NeedsPadding = false;
1859 bool InReg;
1860 if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
1861 unsigned SizeInRegs = (TI.Width + 31) / 32;
1862 SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
1863 llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
1864 if (InReg)
1865 return ABIArgInfo::getDirectInReg(Result);
1866 else
1867 return ABIArgInfo::getDirect(Result);
1868 }
1869 llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
1870
1871 // Pass over-aligned aggregates on Windows indirectly. This behavior was
1872 // added in MSVC 2015.
1873 if (IsWin32StructABI && TI.isAlignRequired() && TI.Align > 32)
1874 return getIndirectResult(Ty, /*ByVal=*/false, State);
1875
1876 // Expand small (<= 128-bit) record types when we know that the stack layout
1877 // of those arguments will match the struct. This is important because the
1878 // LLVM backend isn't smart enough to remove byval, which inhibits many
1879 // optimizations.
1880 // Don't do this for the MCU if there are still free integer registers
1881 // (see X86_64 ABI for full explanation).
1882 if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
1883 canExpandIndirectArgument(Ty))
1884 return ABIArgInfo::getExpandWithPadding(
1885 IsFastCall || IsVectorCall || IsRegCall, PaddingType);
1886
1887 return getIndirectResult(Ty, true, State);
1888 }
1889
1890 if (const VectorType *VT = Ty->getAs<VectorType>()) {
1891 // On Windows, vectors are passed directly if registers are available, or
1892 // indirectly if not. This avoids the need to align argument memory. Pass
1893 // user-defined vector types larger than 512 bits indirectly for simplicity.
1894 if (IsWin32StructABI) {
1895 if (TI.Width <= 512 && State.FreeSSERegs > 0) {
1896 --State.FreeSSERegs;
1897 return ABIArgInfo::getDirectInReg();
1898 }
1899 return getIndirectResult(Ty, /*ByVal=*/false, State);
1900 }
1901
1902 // On Darwin, some vectors are passed in memory, we handle this by passing
1903 // it as an i8/i16/i32/i64.
1904 if (IsDarwinVectorABI) {
1905 if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
1906 (TI.Width == 64 && VT->getNumElements() == 1))
1907 return ABIArgInfo::getDirect(
1908 llvm::IntegerType::get(getVMContext(), TI.Width));
1909 }
1910
1911 if (IsX86_MMXType(CGT.ConvertType(Ty)))
1912 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));
1913
1914 return ABIArgInfo::getDirect();
1915 }
1916
1917
1918 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
1919 Ty = EnumTy->getDecl()->getIntegerType();
1920
1921 bool InReg = shouldPrimitiveUseInReg(Ty, State);
1922
1923 if (isPromotableIntegerTypeForABI(Ty)) {
1924 if (InReg)
1925 return ABIArgInfo::getExtendInReg(Ty);
1926 return ABIArgInfo::getExtend(Ty);
1927 }
1928
1929 if (const auto * EIT = Ty->getAs<ExtIntType>()) {
1930 if (EIT->getNumBits() <= 64) {
1931 if (InReg)
1932 return ABIArgInfo::getDirectInReg();
1933 return ABIArgInfo::getDirect();
1934 }
1935 return getIndirectResult(Ty, /*ByVal=*/false, State);
1936 }
1937
1938 if (InReg)
1939 return ABIArgInfo::getDirectInReg();
1940 return ABIArgInfo::getDirect();
1941 }
1942
computeInfo(CGFunctionInfo & FI) const1943 void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1944 CCState State(FI);
1945 if (IsMCUABI)
1946 State.FreeRegs = 3;
1947 else if (State.CC == llvm::CallingConv::X86_FastCall) {
1948 State.FreeRegs = 2;
1949 State.FreeSSERegs = 3;
1950 } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1951 State.FreeRegs = 2;
1952 State.FreeSSERegs = 6;
1953 } else if (FI.getHasRegParm())
1954 State.FreeRegs = FI.getRegParm();
1955 else if (State.CC == llvm::CallingConv::X86_RegCall) {
1956 State.FreeRegs = 5;
1957 State.FreeSSERegs = 8;
1958 } else if (IsWin32StructABI) {
1959 // Since MSVC 2015, the first three SSE vectors have been passed in
1960 // registers. The rest are passed indirectly.
1961 State.FreeRegs = DefaultNumRegisterParameters;
1962 State.FreeSSERegs = 3;
1963 } else
1964 State.FreeRegs = DefaultNumRegisterParameters;
1965
1966 if (!::classifyReturnType(getCXXABI(), FI, *this)) {
1967 FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), State);
1968 } else if (FI.getReturnInfo().isIndirect()) {
1969 // The C++ ABI is not aware of register usage, so we have to check if the
1970 // return value was sret and put it in a register ourselves if appropriate.
1971 if (State.FreeRegs) {
1972 --State.FreeRegs; // The sret parameter consumes a register.
1973 if (!IsMCUABI)
1974 FI.getReturnInfo().setInReg(true);
1975 }
1976 }
1977
1978 // The chain argument effectively gives us another free register.
1979 if (FI.isChainCall())
1980 ++State.FreeRegs;
1981
1982 // For vectorcall, do a first pass over the arguments, assigning FP and vector
1983 // arguments to XMM registers as available.
1984 if (State.CC == llvm::CallingConv::X86_VectorCall)
1985 runVectorCallFirstPass(FI, State);
1986
1987 bool UsedInAlloca = false;
1988 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1989 for (int I = 0, E = Args.size(); I < E; ++I) {
1990 // Skip arguments that have already been assigned.
1991 if (State.IsPreassigned.test(I))
1992 continue;
1993
1994 Args[I].info = classifyArgumentType(Args[I].type, State);
1995 UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
1996 }
1997
1998 // If we needed to use inalloca for any argument, do a second pass and rewrite
1999 // all the memory arguments to use inalloca.
2000 if (UsedInAlloca)
2001 rewriteWithInAlloca(FI);
2002 }
2003
2004 void
addFieldToArgStruct(SmallVector<llvm::Type *,6> & FrameFields,CharUnits & StackOffset,ABIArgInfo & Info,QualType Type) const2005 X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
2006 CharUnits &StackOffset, ABIArgInfo &Info,
2007 QualType Type) const {
2008 // Arguments are always 4-byte-aligned.
2009 CharUnits WordSize = CharUnits::fromQuantity(4);
2010 assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
2011
2012 // sret pointers and indirect things will require an extra pointer
2013 // indirection, unless they are byval. Most things are byval, and will not
2014 // require this indirection.
2015 bool IsIndirect = false;
2016 if (Info.isIndirect() && !Info.getIndirectByVal())
2017 IsIndirect = true;
2018 Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
2019 llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
2020 if (IsIndirect)
2021 LLTy = LLTy->getPointerTo(0);
2022 FrameFields.push_back(LLTy);
2023 StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(Type);
2024
2025 // Insert padding bytes to respect alignment.
2026 CharUnits FieldEnd = StackOffset;
2027 StackOffset = FieldEnd.alignTo(WordSize);
2028 if (StackOffset != FieldEnd) {
2029 CharUnits NumBytes = StackOffset - FieldEnd;
2030 llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
2031 Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
2032 FrameFields.push_back(Ty);
2033 }
2034 }
2035
isArgInAlloca(const ABIArgInfo & Info)2036 static bool isArgInAlloca(const ABIArgInfo &Info) {
2037 // Leave ignored and inreg arguments alone.
2038 switch (Info.getKind()) {
2039 case ABIArgInfo::InAlloca:
2040 return true;
2041 case ABIArgInfo::Ignore:
2042 case ABIArgInfo::IndirectAliased:
2043 return false;
2044 case ABIArgInfo::Indirect:
2045 case ABIArgInfo::Direct:
2046 case ABIArgInfo::Extend:
2047 return !Info.getInReg();
2048 case ABIArgInfo::Expand:
2049 case ABIArgInfo::CoerceAndExpand:
2050 // These are aggregate types which are never passed in registers when
2051 // inalloca is involved.
2052 return true;
2053 }
2054 llvm_unreachable("invalid enum");
2055 }
2056
rewriteWithInAlloca(CGFunctionInfo & FI) const2057 void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
2058 assert(IsWin32StructABI && "inalloca only supported on win32");
2059
2060 // Build a packed struct type for all of the arguments in memory.
2061 SmallVector<llvm::Type *, 6> FrameFields;
2062
2063 // The stack alignment is always 4.
2064 CharUnits StackAlign = CharUnits::fromQuantity(4);
2065
2066 CharUnits StackOffset;
2067 CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2068
2069 // Put 'this' into the struct before 'sret', if necessary.
2070 bool IsThisCall =
2071 FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
2072 ABIArgInfo &Ret = FI.getReturnInfo();
2073 if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&
2074 isArgInAlloca(I->info)) {
2075 addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2076 ++I;
2077 }
2078
2079 // Put the sret parameter into the inalloca struct if it's in memory.
2080 if (Ret.isIndirect() && !Ret.getInReg()) {
2081 addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
2082 // On Windows, the hidden sret parameter is always returned in eax.
2083 Ret.setInAllocaSRet(IsWin32StructABI);
2084 }
2085
2086 // Skip the 'this' parameter in ecx.
2087 if (IsThisCall)
2088 ++I;
2089
2090 // Put arguments passed in memory into the struct.
2091 for (; I != E; ++I) {
2092 if (isArgInAlloca(I->info))
2093 addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2094 }
2095
2096 FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
2097 /*isPacked=*/true),
2098 StackAlign);
2099 }
2100
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const2101 Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
2102 Address VAListAddr, QualType Ty) const {
2103
2104 auto TypeInfo = getContext().getTypeInfoInChars(Ty);
2105
2106 // x86-32 changes the alignment of certain arguments on the stack.
2107 //
2108 // Just messing with TypeInfo like this works because we never pass
2109 // anything indirectly.
2110 TypeInfo.Align = CharUnits::fromQuantity(
2111 getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));
2112
2113 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
2114 TypeInfo, CharUnits::fromQuantity(4),
2115 /*AllowHigherAlign*/ true);
2116 }
2117
isStructReturnInRegABI(const llvm::Triple & Triple,const CodeGenOptions & Opts)2118 bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
2119 const llvm::Triple &Triple, const CodeGenOptions &Opts) {
2120 assert(Triple.getArch() == llvm::Triple::x86);
2121
2122 switch (Opts.getStructReturnConvention()) {
2123 case CodeGenOptions::SRCK_Default:
2124 break;
2125 case CodeGenOptions::SRCK_OnStack: // -fpcc-struct-return
2126 return false;
2127 case CodeGenOptions::SRCK_InRegs: // -freg-struct-return
2128 return true;
2129 }
2130
2131 if (Triple.isOSDarwin() || Triple.isOSIAMCU())
2132 return true;
2133
2134 switch (Triple.getOS()) {
2135 case llvm::Triple::DragonFly:
2136 case llvm::Triple::FreeBSD:
2137 case llvm::Triple::OpenBSD:
2138 case llvm::Triple::Win32:
2139 return true;
2140 default:
2141 return false;
2142 }
2143 }
2144
addX86InterruptAttrs(const FunctionDecl * FD,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM)2145 static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
2146 CodeGen::CodeGenModule &CGM) {
2147 if (!FD->hasAttr<AnyX86InterruptAttr>())
2148 return;
2149
2150 llvm::Function *Fn = cast<llvm::Function>(GV);
2151 Fn->setCallingConv(llvm::CallingConv::X86_INTR);
2152 if (FD->getNumParams() == 0)
2153 return;
2154
2155 auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());
2156 llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());
2157 llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
2158 Fn->getContext(), ByValTy);
2159 Fn->addParamAttr(0, NewAttr);
2160 }
2161
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const2162 void X86_32TargetCodeGenInfo::setTargetAttributes(
2163 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2164 if (GV->isDeclaration())
2165 return;
2166 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2167 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2168 llvm::Function *Fn = cast<llvm::Function>(GV);
2169 Fn->addFnAttr("stackrealign");
2170 }
2171
2172 addX86InterruptAttrs(FD, GV, CGM);
2173 }
2174 }
2175
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const2176 bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
2177 CodeGen::CodeGenFunction &CGF,
2178 llvm::Value *Address) const {
2179 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2180
2181 llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
2182
2183 // 0-7 are the eight integer registers; the order is different
2184 // on Darwin (for EH), but the range is the same.
2185 // 8 is %eip.
2186 AssignToArrayRange(Builder, Address, Four8, 0, 8);
2187
2188 if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
2189 // 12-16 are st(0..4). Not sure why we stop at 4.
2190 // These have size 16, which is sizeof(long double) on
2191 // platforms with 8-byte alignment for that type.
2192 llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);
2193 AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
2194
2195 } else {
2196 // 9 is %eflags, which doesn't get a size on Darwin for some
2197 // reason.
2198 Builder.CreateAlignedStore(
2199 Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
2200 CharUnits::One());
2201
2202 // 11-16 are st(0..5). Not sure why we stop at 5.
2203 // These have size 12, which is sizeof(long double) on
2204 // platforms with 4-byte alignment for that type.
2205 llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);
2206 AssignToArrayRange(Builder, Address, Twelve8, 11, 16);
2207 }
2208
2209 return false;
2210 }
2211
2212 //===----------------------------------------------------------------------===//
2213 // X86-64 ABI Implementation
2214 //===----------------------------------------------------------------------===//
2215
2216
2217 namespace {
2218 /// The AVX ABI level for X86 targets.
2219 enum class X86AVXABILevel {
2220 None,
2221 AVX,
2222 AVX512
2223 };
2224
2225 /// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel)2226 static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
2227 switch (AVXLevel) {
2228 case X86AVXABILevel::AVX512:
2229 return 512;
2230 case X86AVXABILevel::AVX:
2231 return 256;
2232 case X86AVXABILevel::None:
2233 return 128;
2234 }
2235 llvm_unreachable("Unknown AVXLevel");
2236 }
2237
2238 /// X86_64ABIInfo - The X86_64 ABI information.
2239 class X86_64ABIInfo : public SwiftABIInfo {
2240 enum Class {
2241 Integer = 0,
2242 SSE,
2243 SSEUp,
2244 X87,
2245 X87Up,
2246 ComplexX87,
2247 NoClass,
2248 Memory
2249 };
2250
2251 /// merge - Implement the X86_64 ABI merging algorithm.
2252 ///
2253 /// Merge an accumulating classification \arg Accum with a field
2254 /// classification \arg Field.
2255 ///
2256 /// \param Accum - The accumulating classification. This should
2257 /// always be either NoClass or the result of a previous merge
2258 /// call. In addition, this should never be Memory (the caller
2259 /// should just return Memory for the aggregate).
2260 static Class merge(Class Accum, Class Field);
2261
2262 /// postMerge - Implement the X86_64 ABI post merging algorithm.
2263 ///
2264 /// Post merger cleanup, reduces a malformed Hi and Lo pair to
2265 /// final MEMORY or SSE classes when necessary.
2266 ///
2267 /// \param AggregateSize - The size of the current aggregate in
2268 /// the classification process.
2269 ///
2270 /// \param Lo - The classification for the parts of the type
2271 /// residing in the low word of the containing object.
2272 ///
2273 /// \param Hi - The classification for the parts of the type
2274 /// residing in the higher words of the containing object.
2275 ///
2276 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
2277
2278 /// classify - Determine the x86_64 register classes in which the
2279 /// given type T should be passed.
2280 ///
2281 /// \param Lo - The classification for the parts of the type
2282 /// residing in the low word of the containing object.
2283 ///
2284 /// \param Hi - The classification for the parts of the type
2285 /// residing in the high word of the containing object.
2286 ///
2287 /// \param OffsetBase - The bit offset of this type in the
2288 /// containing object. Some parameters are classified different
2289 /// depending on whether they straddle an eightbyte boundary.
2290 ///
2291 /// \param isNamedArg - Whether the argument in question is a "named"
2292 /// argument, as used in AMD64-ABI 3.5.7.
2293 ///
2294 /// If a word is unused its result will be NoClass; if a type should
2295 /// be passed in Memory then at least the classification of \arg Lo
2296 /// will be Memory.
2297 ///
2298 /// The \arg Lo class will be NoClass iff the argument is ignored.
2299 ///
2300 /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
2301 /// also be ComplexX87.
2302 void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
2303 bool isNamedArg) const;
2304
2305 llvm::Type *GetByteVectorType(QualType Ty) const;
2306 llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
2307 unsigned IROffset, QualType SourceTy,
2308 unsigned SourceOffset) const;
2309 llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
2310 unsigned IROffset, QualType SourceTy,
2311 unsigned SourceOffset) const;
2312
2313 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2314 /// such that the argument will be returned in memory.
2315 ABIArgInfo getIndirectReturnResult(QualType Ty) const;
2316
2317 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2318 /// such that the argument will be passed in memory.
2319 ///
2320 /// \param freeIntRegs - The number of free integer registers remaining
2321 /// available.
2322 ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
2323
2324 ABIArgInfo classifyReturnType(QualType RetTy) const;
2325
2326 ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2327 unsigned &neededInt, unsigned &neededSSE,
2328 bool isNamedArg) const;
2329
2330 ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2331 unsigned &NeededSSE) const;
2332
2333 ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2334 unsigned &NeededSSE) const;
2335
2336 bool IsIllegalVectorType(QualType Ty) const;
2337
2338 /// The 0.98 ABI revision clarified a lot of ambiguities,
2339 /// unfortunately in ways that were not always consistent with
2340 /// certain previous compilers. In particular, platforms which
2341 /// required strict binary compatibility with older versions of GCC
2342 /// may need to exempt themselves.
honorsRevision0_98() const2343 bool honorsRevision0_98() const {
2344 return !getTarget().getTriple().isOSDarwin();
2345 }
2346
2347 /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
2348 /// classify it as INTEGER (for compatibility with older clang compilers).
classifyIntegerMMXAsSSE() const2349 bool classifyIntegerMMXAsSSE() const {
2350 // Clang <= 3.8 did not do this.
2351 if (getContext().getLangOpts().getClangABICompat() <=
2352 LangOptions::ClangABI::Ver3_8)
2353 return false;
2354
2355 const llvm::Triple &Triple = getTarget().getTriple();
2356 if (Triple.isOSDarwin() || Triple.getOS() == llvm::Triple::PS4)
2357 return false;
2358 if (Triple.isOSFreeBSD() && Triple.getOSMajorVersion() >= 10)
2359 return false;
2360 return true;
2361 }
2362
2363 // GCC classifies vectors of __int128 as memory.
passInt128VectorsInMem() const2364 bool passInt128VectorsInMem() const {
2365 // Clang <= 9.0 did not do this.
2366 if (getContext().getLangOpts().getClangABICompat() <=
2367 LangOptions::ClangABI::Ver9)
2368 return false;
2369
2370 const llvm::Triple &T = getTarget().getTriple();
2371 return T.isOSLinux() || T.isOSNetBSD();
2372 }
2373
2374 X86AVXABILevel AVXLevel;
2375 // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
2376 // 64-bit hardware.
2377 bool Has64BitPointers;
2378
2379 public:
X86_64ABIInfo(CodeGen::CodeGenTypes & CGT,X86AVXABILevel AVXLevel)2380 X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) :
2381 SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2382 Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {
2383 }
2384
isPassedUsingAVXType(QualType type) const2385 bool isPassedUsingAVXType(QualType type) const {
2386 unsigned neededInt, neededSSE;
2387 // The freeIntRegs argument doesn't matter here.
2388 ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,
2389 /*isNamedArg*/true);
2390 if (info.isDirect()) {
2391 llvm::Type *ty = info.getCoerceToType();
2392 if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))
2393 return vectorTy->getPrimitiveSizeInBits().getFixedSize() > 128;
2394 }
2395 return false;
2396 }
2397
2398 void computeInfo(CGFunctionInfo &FI) const override;
2399
2400 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2401 QualType Ty) const override;
2402 Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
2403 QualType Ty) const override;
2404
has64BitPointers() const2405 bool has64BitPointers() const {
2406 return Has64BitPointers;
2407 }
2408
shouldPassIndirectlyForSwift(ArrayRef<llvm::Type * > scalars,bool asReturnValue) const2409 bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
2410 bool asReturnValue) const override {
2411 return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2412 }
isSwiftErrorInRegister() const2413 bool isSwiftErrorInRegister() const override {
2414 return true;
2415 }
2416 };
2417
2418 /// WinX86_64ABIInfo - The Windows X86_64 ABI information.
2419 class WinX86_64ABIInfo : public SwiftABIInfo {
2420 public:
WinX86_64ABIInfo(CodeGen::CodeGenTypes & CGT,X86AVXABILevel AVXLevel)2421 WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2422 : SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2423 IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
2424
2425 void computeInfo(CGFunctionInfo &FI) const override;
2426
2427 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2428 QualType Ty) const override;
2429
isHomogeneousAggregateBaseType(QualType Ty) const2430 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
2431 // FIXME: Assumes vectorcall is in use.
2432 return isX86VectorTypeForVectorCall(getContext(), Ty);
2433 }
2434
isHomogeneousAggregateSmallEnough(const Type * Ty,uint64_t NumMembers) const2435 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
2436 uint64_t NumMembers) const override {
2437 // FIXME: Assumes vectorcall is in use.
2438 return isX86VectorCallAggregateSmallEnough(NumMembers);
2439 }
2440
shouldPassIndirectlyForSwift(ArrayRef<llvm::Type * > scalars,bool asReturnValue) const2441 bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type *> scalars,
2442 bool asReturnValue) const override {
2443 return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2444 }
2445
isSwiftErrorInRegister() const2446 bool isSwiftErrorInRegister() const override {
2447 return true;
2448 }
2449
2450 private:
2451 ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
2452 bool IsVectorCall, bool IsRegCall) const;
2453 ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
2454 const ABIArgInfo ¤t) const;
2455
2456 X86AVXABILevel AVXLevel;
2457
2458 bool IsMingw64;
2459 };
2460
2461 class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2462 public:
X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,X86AVXABILevel AVXLevel)2463 X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2464 : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {}
2465
getABIInfo() const2466 const X86_64ABIInfo &getABIInfo() const {
2467 return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
2468 }
2469
2470 /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
2471 /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
markARCOptimizedReturnCallsAsNoTail() const2472 bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
2473
getDwarfEHStackPointer(CodeGen::CodeGenModule & CGM) const2474 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2475 return 7;
2476 }
2477
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const2478 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2479 llvm::Value *Address) const override {
2480 llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2481
2482 // 0-15 are the 16 integer registers.
2483 // 16 is %rip.
2484 AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2485 return false;
2486 }
2487
adjustInlineAsmType(CodeGen::CodeGenFunction & CGF,StringRef Constraint,llvm::Type * Ty) const2488 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
2489 StringRef Constraint,
2490 llvm::Type* Ty) const override {
2491 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
2492 }
2493
isNoProtoCallVariadic(const CallArgList & args,const FunctionNoProtoType * fnType) const2494 bool isNoProtoCallVariadic(const CallArgList &args,
2495 const FunctionNoProtoType *fnType) const override {
2496 // The default CC on x86-64 sets %al to the number of SSA
2497 // registers used, and GCC sets this when calling an unprototyped
2498 // function, so we override the default behavior. However, don't do
2499 // that when AVX types are involved: the ABI explicitly states it is
2500 // undefined, and it doesn't work in practice because of how the ABI
2501 // defines varargs anyway.
2502 if (fnType->getCallConv() == CC_C) {
2503 bool HasAVXType = false;
2504 for (CallArgList::const_iterator
2505 it = args.begin(), ie = args.end(); it != ie; ++it) {
2506 if (getABIInfo().isPassedUsingAVXType(it->Ty)) {
2507 HasAVXType = true;
2508 break;
2509 }
2510 }
2511
2512 if (!HasAVXType)
2513 return true;
2514 }
2515
2516 return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
2517 }
2518
2519 llvm::Constant *
getUBSanFunctionSignature(CodeGen::CodeGenModule & CGM) const2520 getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
2521 unsigned Sig = (0xeb << 0) | // jmp rel8
2522 (0x06 << 8) | // .+0x08
2523 ('v' << 16) |
2524 ('2' << 24);
2525 return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
2526 }
2527
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const2528 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2529 CodeGen::CodeGenModule &CGM) const override {
2530 if (GV->isDeclaration())
2531 return;
2532 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2533 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2534 llvm::Function *Fn = cast<llvm::Function>(GV);
2535 Fn->addFnAttr("stackrealign");
2536 }
2537
2538 addX86InterruptAttrs(FD, GV, CGM);
2539 }
2540 }
2541
2542 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
2543 const FunctionDecl *Caller,
2544 const FunctionDecl *Callee,
2545 const CallArgList &Args) const override;
2546 };
2547
initFeatureMaps(const ASTContext & Ctx,llvm::StringMap<bool> & CallerMap,const FunctionDecl * Caller,llvm::StringMap<bool> & CalleeMap,const FunctionDecl * Callee)2548 static void initFeatureMaps(const ASTContext &Ctx,
2549 llvm::StringMap<bool> &CallerMap,
2550 const FunctionDecl *Caller,
2551 llvm::StringMap<bool> &CalleeMap,
2552 const FunctionDecl *Callee) {
2553 if (CalleeMap.empty() && CallerMap.empty()) {
2554 // The caller is potentially nullptr in the case where the call isn't in a
2555 // function. In this case, the getFunctionFeatureMap ensures we just get
2556 // the TU level setting (since it cannot be modified by 'target'..
2557 Ctx.getFunctionFeatureMap(CallerMap, Caller);
2558 Ctx.getFunctionFeatureMap(CalleeMap, Callee);
2559 }
2560 }
2561
checkAVXParamFeature(DiagnosticsEngine & Diag,SourceLocation CallLoc,const llvm::StringMap<bool> & CallerMap,const llvm::StringMap<bool> & CalleeMap,QualType Ty,StringRef Feature,bool IsArgument)2562 static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
2563 SourceLocation CallLoc,
2564 const llvm::StringMap<bool> &CallerMap,
2565 const llvm::StringMap<bool> &CalleeMap,
2566 QualType Ty, StringRef Feature,
2567 bool IsArgument) {
2568 bool CallerHasFeat = CallerMap.lookup(Feature);
2569 bool CalleeHasFeat = CalleeMap.lookup(Feature);
2570 if (!CallerHasFeat && !CalleeHasFeat)
2571 return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
2572 << IsArgument << Ty << Feature;
2573
2574 // Mixing calling conventions here is very clearly an error.
2575 if (!CallerHasFeat || !CalleeHasFeat)
2576 return Diag.Report(CallLoc, diag::err_avx_calling_convention)
2577 << IsArgument << Ty << Feature;
2578
2579 // Else, both caller and callee have the required feature, so there is no need
2580 // to diagnose.
2581 return false;
2582 }
2583
checkAVXParam(DiagnosticsEngine & Diag,ASTContext & Ctx,SourceLocation CallLoc,const llvm::StringMap<bool> & CallerMap,const llvm::StringMap<bool> & CalleeMap,QualType Ty,bool IsArgument)2584 static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
2585 SourceLocation CallLoc,
2586 const llvm::StringMap<bool> &CallerMap,
2587 const llvm::StringMap<bool> &CalleeMap, QualType Ty,
2588 bool IsArgument) {
2589 uint64_t Size = Ctx.getTypeSize(Ty);
2590 if (Size > 256)
2591 return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
2592 "avx512f", IsArgument);
2593
2594 if (Size > 128)
2595 return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
2596 IsArgument);
2597
2598 return false;
2599 }
2600
checkFunctionCallABI(CodeGenModule & CGM,SourceLocation CallLoc,const FunctionDecl * Caller,const FunctionDecl * Callee,const CallArgList & Args) const2601 void X86_64TargetCodeGenInfo::checkFunctionCallABI(
2602 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
2603 const FunctionDecl *Callee, const CallArgList &Args) const {
2604 llvm::StringMap<bool> CallerMap;
2605 llvm::StringMap<bool> CalleeMap;
2606 unsigned ArgIndex = 0;
2607
2608 // We need to loop through the actual call arguments rather than the the
2609 // function's parameters, in case this variadic.
2610 for (const CallArg &Arg : Args) {
2611 // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
2612 // additionally changes how vectors >256 in size are passed. Like GCC, we
2613 // warn when a function is called with an argument where this will change.
2614 // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
2615 // the caller and callee features are mismatched.
2616 // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
2617 // change its ABI with attribute-target after this call.
2618 if (Arg.getType()->isVectorType() &&
2619 CGM.getContext().getTypeSize(Arg.getType()) > 128) {
2620 initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2621 QualType Ty = Arg.getType();
2622 // The CallArg seems to have desugared the type already, so for clearer
2623 // diagnostics, replace it with the type in the FunctionDecl if possible.
2624 if (ArgIndex < Callee->getNumParams())
2625 Ty = Callee->getParamDecl(ArgIndex)->getType();
2626
2627 if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2628 CalleeMap, Ty, /*IsArgument*/ true))
2629 return;
2630 }
2631 ++ArgIndex;
2632 }
2633
2634 // Check return always, as we don't have a good way of knowing in codegen
2635 // whether this value is used, tail-called, etc.
2636 if (Callee->getReturnType()->isVectorType() &&
2637 CGM.getContext().getTypeSize(Callee->getReturnType()) > 128) {
2638 initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2639 checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2640 CalleeMap, Callee->getReturnType(),
2641 /*IsArgument*/ false);
2642 }
2643 }
2644
qualifyWindowsLibrary(llvm::StringRef Lib)2645 static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
2646 // If the argument does not end in .lib, automatically add the suffix.
2647 // If the argument contains a space, enclose it in quotes.
2648 // This matches the behavior of MSVC.
2649 bool Quote = (Lib.find(' ') != StringRef::npos);
2650 std::string ArgStr = Quote ? "\"" : "";
2651 ArgStr += Lib;
2652 if (!Lib.endswith_insensitive(".lib") && !Lib.endswith_insensitive(".a"))
2653 ArgStr += ".lib";
2654 ArgStr += Quote ? "\"" : "";
2655 return ArgStr;
2656 }
2657
2658 class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
2659 public:
WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,bool DarwinVectorABI,bool RetSmallStructInRegABI,bool Win32StructABI,unsigned NumRegisterParameters)2660 WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2661 bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
2662 unsigned NumRegisterParameters)
2663 : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
2664 Win32StructABI, NumRegisterParameters, false) {}
2665
2666 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2667 CodeGen::CodeGenModule &CGM) const override;
2668
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const2669 void getDependentLibraryOption(llvm::StringRef Lib,
2670 llvm::SmallString<24> &Opt) const override {
2671 Opt = "/DEFAULTLIB:";
2672 Opt += qualifyWindowsLibrary(Lib);
2673 }
2674
getDetectMismatchOption(llvm::StringRef Name,llvm::StringRef Value,llvm::SmallString<32> & Opt) const2675 void getDetectMismatchOption(llvm::StringRef Name,
2676 llvm::StringRef Value,
2677 llvm::SmallString<32> &Opt) const override {
2678 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2679 }
2680 };
2681
addStackProbeTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM)2682 static void addStackProbeTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2683 CodeGen::CodeGenModule &CGM) {
2684 if (llvm::Function *Fn = dyn_cast_or_null<llvm::Function>(GV)) {
2685
2686 if (CGM.getCodeGenOpts().StackProbeSize != 4096)
2687 Fn->addFnAttr("stack-probe-size",
2688 llvm::utostr(CGM.getCodeGenOpts().StackProbeSize));
2689 if (CGM.getCodeGenOpts().NoStackArgProbe)
2690 Fn->addFnAttr("no-stack-arg-probe");
2691 }
2692 }
2693
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const2694 void WinX86_32TargetCodeGenInfo::setTargetAttributes(
2695 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2696 X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2697 if (GV->isDeclaration())
2698 return;
2699 addStackProbeTargetAttributes(D, GV, CGM);
2700 }
2701
2702 class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2703 public:
WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,X86AVXABILevel AVXLevel)2704 WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2705 X86AVXABILevel AVXLevel)
2706 : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {}
2707
2708 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2709 CodeGen::CodeGenModule &CGM) const override;
2710
getDwarfEHStackPointer(CodeGen::CodeGenModule & CGM) const2711 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2712 return 7;
2713 }
2714
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const2715 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2716 llvm::Value *Address) const override {
2717 llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2718
2719 // 0-15 are the 16 integer registers.
2720 // 16 is %rip.
2721 AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2722 return false;
2723 }
2724
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const2725 void getDependentLibraryOption(llvm::StringRef Lib,
2726 llvm::SmallString<24> &Opt) const override {
2727 Opt = "/DEFAULTLIB:";
2728 Opt += qualifyWindowsLibrary(Lib);
2729 }
2730
getDetectMismatchOption(llvm::StringRef Name,llvm::StringRef Value,llvm::SmallString<32> & Opt) const2731 void getDetectMismatchOption(llvm::StringRef Name,
2732 llvm::StringRef Value,
2733 llvm::SmallString<32> &Opt) const override {
2734 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2735 }
2736 };
2737
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const2738 void WinX86_64TargetCodeGenInfo::setTargetAttributes(
2739 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2740 TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2741 if (GV->isDeclaration())
2742 return;
2743 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2744 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2745 llvm::Function *Fn = cast<llvm::Function>(GV);
2746 Fn->addFnAttr("stackrealign");
2747 }
2748
2749 addX86InterruptAttrs(FD, GV, CGM);
2750 }
2751
2752 addStackProbeTargetAttributes(D, GV, CGM);
2753 }
2754 }
2755
postMerge(unsigned AggregateSize,Class & Lo,Class & Hi) const2756 void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
2757 Class &Hi) const {
2758 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
2759 //
2760 // (a) If one of the classes is Memory, the whole argument is passed in
2761 // memory.
2762 //
2763 // (b) If X87UP is not preceded by X87, the whole argument is passed in
2764 // memory.
2765 //
2766 // (c) If the size of the aggregate exceeds two eightbytes and the first
2767 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
2768 // argument is passed in memory. NOTE: This is necessary to keep the
2769 // ABI working for processors that don't support the __m256 type.
2770 //
2771 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
2772 //
2773 // Some of these are enforced by the merging logic. Others can arise
2774 // only with unions; for example:
2775 // union { _Complex double; unsigned; }
2776 //
2777 // Note that clauses (b) and (c) were added in 0.98.
2778 //
2779 if (Hi == Memory)
2780 Lo = Memory;
2781 if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
2782 Lo = Memory;
2783 if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
2784 Lo = Memory;
2785 if (Hi == SSEUp && Lo != SSE)
2786 Hi = SSE;
2787 }
2788
merge(Class Accum,Class Field)2789 X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
2790 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
2791 // classified recursively so that always two fields are
2792 // considered. The resulting class is calculated according to
2793 // the classes of the fields in the eightbyte:
2794 //
2795 // (a) If both classes are equal, this is the resulting class.
2796 //
2797 // (b) If one of the classes is NO_CLASS, the resulting class is
2798 // the other class.
2799 //
2800 // (c) If one of the classes is MEMORY, the result is the MEMORY
2801 // class.
2802 //
2803 // (d) If one of the classes is INTEGER, the result is the
2804 // INTEGER.
2805 //
2806 // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2807 // MEMORY is used as class.
2808 //
2809 // (f) Otherwise class SSE is used.
2810
2811 // Accum should never be memory (we should have returned) or
2812 // ComplexX87 (because this cannot be passed in a structure).
2813 assert((Accum != Memory && Accum != ComplexX87) &&
2814 "Invalid accumulated classification during merge.");
2815 if (Accum == Field || Field == NoClass)
2816 return Accum;
2817 if (Field == Memory)
2818 return Memory;
2819 if (Accum == NoClass)
2820 return Field;
2821 if (Accum == Integer || Field == Integer)
2822 return Integer;
2823 if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2824 Accum == X87 || Accum == X87Up)
2825 return Memory;
2826 return SSE;
2827 }
2828
classify(QualType Ty,uint64_t OffsetBase,Class & Lo,Class & Hi,bool isNamedArg) const2829 void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
2830 Class &Lo, Class &Hi, bool isNamedArg) const {
2831 // FIXME: This code can be simplified by introducing a simple value class for
2832 // Class pairs with appropriate constructor methods for the various
2833 // situations.
2834
2835 // FIXME: Some of the split computations are wrong; unaligned vectors
2836 // shouldn't be passed in registers for example, so there is no chance they
2837 // can straddle an eightbyte. Verify & simplify.
2838
2839 Lo = Hi = NoClass;
2840
2841 Class &Current = OffsetBase < 64 ? Lo : Hi;
2842 Current = Memory;
2843
2844 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2845 BuiltinType::Kind k = BT->getKind();
2846
2847 if (k == BuiltinType::Void) {
2848 Current = NoClass;
2849 } else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
2850 Lo = Integer;
2851 Hi = Integer;
2852 } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
2853 Current = Integer;
2854 } else if (k == BuiltinType::Float || k == BuiltinType::Double ||
2855 k == BuiltinType::Float16) {
2856 Current = SSE;
2857 } else if (k == BuiltinType::LongDouble) {
2858 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2859 if (LDF == &llvm::APFloat::IEEEquad()) {
2860 Lo = SSE;
2861 Hi = SSEUp;
2862 } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2863 Lo = X87;
2864 Hi = X87Up;
2865 } else if (LDF == &llvm::APFloat::IEEEdouble()) {
2866 Current = SSE;
2867 } else
2868 llvm_unreachable("unexpected long double representation!");
2869 }
2870 // FIXME: _Decimal32 and _Decimal64 are SSE.
2871 // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2872 return;
2873 }
2874
2875 if (const EnumType *ET = Ty->getAs<EnumType>()) {
2876 // Classify the underlying integer type.
2877 classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2878 return;
2879 }
2880
2881 if (Ty->hasPointerRepresentation()) {
2882 Current = Integer;
2883 return;
2884 }
2885
2886 if (Ty->isMemberPointerType()) {
2887 if (Ty->isMemberFunctionPointerType()) {
2888 if (Has64BitPointers) {
2889 // If Has64BitPointers, this is an {i64, i64}, so classify both
2890 // Lo and Hi now.
2891 Lo = Hi = Integer;
2892 } else {
2893 // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2894 // straddles an eightbyte boundary, Hi should be classified as well.
2895 uint64_t EB_FuncPtr = (OffsetBase) / 64;
2896 uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2897 if (EB_FuncPtr != EB_ThisAdj) {
2898 Lo = Hi = Integer;
2899 } else {
2900 Current = Integer;
2901 }
2902 }
2903 } else {
2904 Current = Integer;
2905 }
2906 return;
2907 }
2908
2909 if (const VectorType *VT = Ty->getAs<VectorType>()) {
2910 uint64_t Size = getContext().getTypeSize(VT);
2911 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
2912 // gcc passes the following as integer:
2913 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2914 // 2 bytes - <2 x char>, <1 x short>
2915 // 1 byte - <1 x char>
2916 Current = Integer;
2917
2918 // If this type crosses an eightbyte boundary, it should be
2919 // split.
2920 uint64_t EB_Lo = (OffsetBase) / 64;
2921 uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2922 if (EB_Lo != EB_Hi)
2923 Hi = Lo;
2924 } else if (Size == 64) {
2925 QualType ElementType = VT->getElementType();
2926
2927 // gcc passes <1 x double> in memory. :(
2928 if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
2929 return;
2930
2931 // gcc passes <1 x long long> as SSE but clang used to unconditionally
2932 // pass them as integer. For platforms where clang is the de facto
2933 // platform compiler, we must continue to use integer.
2934 if (!classifyIntegerMMXAsSSE() &&
2935 (ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||
2936 ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||
2937 ElementType->isSpecificBuiltinType(BuiltinType::Long) ||
2938 ElementType->isSpecificBuiltinType(BuiltinType::ULong)))
2939 Current = Integer;
2940 else
2941 Current = SSE;
2942
2943 // If this type crosses an eightbyte boundary, it should be
2944 // split.
2945 if (OffsetBase && OffsetBase != 64)
2946 Hi = Lo;
2947 } else if (Size == 128 ||
2948 (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
2949 QualType ElementType = VT->getElementType();
2950
2951 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2952 if (passInt128VectorsInMem() && Size != 128 &&
2953 (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
2954 ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
2955 return;
2956
2957 // Arguments of 256-bits are split into four eightbyte chunks. The
2958 // least significant one belongs to class SSE and all the others to class
2959 // SSEUP. The original Lo and Hi design considers that types can't be
2960 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2961 // This design isn't correct for 256-bits, but since there're no cases
2962 // where the upper parts would need to be inspected, avoid adding
2963 // complexity and just consider Hi to match the 64-256 part.
2964 //
2965 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2966 // registers if they are "named", i.e. not part of the "..." of a
2967 // variadic function.
2968 //
2969 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2970 // split into eight eightbyte chunks, one SSE and seven SSEUP.
2971 Lo = SSE;
2972 Hi = SSEUp;
2973 }
2974 return;
2975 }
2976
2977 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2978 QualType ET = getContext().getCanonicalType(CT->getElementType());
2979
2980 uint64_t Size = getContext().getTypeSize(Ty);
2981 if (ET->isIntegralOrEnumerationType()) {
2982 if (Size <= 64)
2983 Current = Integer;
2984 else if (Size <= 128)
2985 Lo = Hi = Integer;
2986 } else if (ET->isFloat16Type() || ET == getContext().FloatTy) {
2987 Current = SSE;
2988 } else if (ET == getContext().DoubleTy) {
2989 Lo = Hi = SSE;
2990 } else if (ET == getContext().LongDoubleTy) {
2991 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2992 if (LDF == &llvm::APFloat::IEEEquad())
2993 Current = Memory;
2994 else if (LDF == &llvm::APFloat::x87DoubleExtended())
2995 Current = ComplexX87;
2996 else if (LDF == &llvm::APFloat::IEEEdouble())
2997 Lo = Hi = SSE;
2998 else
2999 llvm_unreachable("unexpected long double representation!");
3000 }
3001
3002 // If this complex type crosses an eightbyte boundary then it
3003 // should be split.
3004 uint64_t EB_Real = (OffsetBase) / 64;
3005 uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
3006 if (Hi == NoClass && EB_Real != EB_Imag)
3007 Hi = Lo;
3008
3009 return;
3010 }
3011
3012 if (const auto *EITy = Ty->getAs<ExtIntType>()) {
3013 if (EITy->getNumBits() <= 64)
3014 Current = Integer;
3015 else if (EITy->getNumBits() <= 128)
3016 Lo = Hi = Integer;
3017 // Larger values need to get passed in memory.
3018 return;
3019 }
3020
3021 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
3022 // Arrays are treated like structures.
3023
3024 uint64_t Size = getContext().getTypeSize(Ty);
3025
3026 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3027 // than eight eightbytes, ..., it has class MEMORY.
3028 if (Size > 512)
3029 return;
3030
3031 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
3032 // fields, it has class MEMORY.
3033 //
3034 // Only need to check alignment of array base.
3035 if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
3036 return;
3037
3038 // Otherwise implement simplified merge. We could be smarter about
3039 // this, but it isn't worth it and would be harder to verify.
3040 Current = NoClass;
3041 uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
3042 uint64_t ArraySize = AT->getSize().getZExtValue();
3043
3044 // The only case a 256-bit wide vector could be used is when the array
3045 // contains a single 256-bit element. Since Lo and Hi logic isn't extended
3046 // to work for sizes wider than 128, early check and fallback to memory.
3047 //
3048 if (Size > 128 &&
3049 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
3050 return;
3051
3052 for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
3053 Class FieldLo, FieldHi;
3054 classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
3055 Lo = merge(Lo, FieldLo);
3056 Hi = merge(Hi, FieldHi);
3057 if (Lo == Memory || Hi == Memory)
3058 break;
3059 }
3060
3061 postMerge(Size, Lo, Hi);
3062 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
3063 return;
3064 }
3065
3066 if (const RecordType *RT = Ty->getAs<RecordType>()) {
3067 uint64_t Size = getContext().getTypeSize(Ty);
3068
3069 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3070 // than eight eightbytes, ..., it has class MEMORY.
3071 if (Size > 512)
3072 return;
3073
3074 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
3075 // copy constructor or a non-trivial destructor, it is passed by invisible
3076 // reference.
3077 if (getRecordArgABI(RT, getCXXABI()))
3078 return;
3079
3080 const RecordDecl *RD = RT->getDecl();
3081
3082 // Assume variable sized types are passed in memory.
3083 if (RD->hasFlexibleArrayMember())
3084 return;
3085
3086 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
3087
3088 // Reset Lo class, this will be recomputed.
3089 Current = NoClass;
3090
3091 // If this is a C++ record, classify the bases first.
3092 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3093 for (const auto &I : CXXRD->bases()) {
3094 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3095 "Unexpected base class!");
3096 const auto *Base =
3097 cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3098
3099 // Classify this field.
3100 //
3101 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
3102 // single eightbyte, each is classified separately. Each eightbyte gets
3103 // initialized to class NO_CLASS.
3104 Class FieldLo, FieldHi;
3105 uint64_t Offset =
3106 OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
3107 classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
3108 Lo = merge(Lo, FieldLo);
3109 Hi = merge(Hi, FieldHi);
3110 if (Lo == Memory || Hi == Memory) {
3111 postMerge(Size, Lo, Hi);
3112 return;
3113 }
3114 }
3115 }
3116
3117 // Classify the fields one at a time, merging the results.
3118 unsigned idx = 0;
3119 bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
3120 LangOptions::ClangABI::Ver11 ||
3121 getContext().getTargetInfo().getTriple().isPS4();
3122 bool IsUnion = RT->isUnionType() && !UseClang11Compat;
3123
3124 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3125 i != e; ++i, ++idx) {
3126 uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3127 bool BitField = i->isBitField();
3128
3129 // Ignore padding bit-fields.
3130 if (BitField && i->isUnnamedBitfield())
3131 continue;
3132
3133 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
3134 // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
3135 //
3136 // The only case a 256-bit or a 512-bit wide vector could be used is when
3137 // the struct contains a single 256-bit or 512-bit element. Early check
3138 // and fallback to memory.
3139 //
3140 // FIXME: Extended the Lo and Hi logic properly to work for size wider
3141 // than 128.
3142 if (Size > 128 &&
3143 ((!IsUnion && Size != getContext().getTypeSize(i->getType())) ||
3144 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
3145 Lo = Memory;
3146 postMerge(Size, Lo, Hi);
3147 return;
3148 }
3149 // Note, skip this test for bit-fields, see below.
3150 if (!BitField && Offset % getContext().getTypeAlign(i->getType())) {
3151 Lo = Memory;
3152 postMerge(Size, Lo, Hi);
3153 return;
3154 }
3155
3156 // Classify this field.
3157 //
3158 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
3159 // exceeds a single eightbyte, each is classified
3160 // separately. Each eightbyte gets initialized to class
3161 // NO_CLASS.
3162 Class FieldLo, FieldHi;
3163
3164 // Bit-fields require special handling, they do not force the
3165 // structure to be passed in memory even if unaligned, and
3166 // therefore they can straddle an eightbyte.
3167 if (BitField) {
3168 assert(!i->isUnnamedBitfield());
3169 uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3170 uint64_t Size = i->getBitWidthValue(getContext());
3171
3172 uint64_t EB_Lo = Offset / 64;
3173 uint64_t EB_Hi = (Offset + Size - 1) / 64;
3174
3175 if (EB_Lo) {
3176 assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
3177 FieldLo = NoClass;
3178 FieldHi = Integer;
3179 } else {
3180 FieldLo = Integer;
3181 FieldHi = EB_Hi ? Integer : NoClass;
3182 }
3183 } else
3184 classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
3185 Lo = merge(Lo, FieldLo);
3186 Hi = merge(Hi, FieldHi);
3187 if (Lo == Memory || Hi == Memory)
3188 break;
3189 }
3190
3191 postMerge(Size, Lo, Hi);
3192 }
3193 }
3194
getIndirectReturnResult(QualType Ty) const3195 ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
3196 // If this is a scalar LLVM value then assume LLVM will pass it in the right
3197 // place naturally.
3198 if (!isAggregateTypeForABI(Ty)) {
3199 // Treat an enum type as its underlying type.
3200 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3201 Ty = EnumTy->getDecl()->getIntegerType();
3202
3203 if (Ty->isExtIntType())
3204 return getNaturalAlignIndirect(Ty);
3205
3206 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
3207 : ABIArgInfo::getDirect());
3208 }
3209
3210 return getNaturalAlignIndirect(Ty);
3211 }
3212
IsIllegalVectorType(QualType Ty) const3213 bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
3214 if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
3215 uint64_t Size = getContext().getTypeSize(VecTy);
3216 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
3217 if (Size <= 64 || Size > LargestVector)
3218 return true;
3219 QualType EltTy = VecTy->getElementType();
3220 if (passInt128VectorsInMem() &&
3221 (EltTy->isSpecificBuiltinType(BuiltinType::Int128) ||
3222 EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))
3223 return true;
3224 }
3225
3226 return false;
3227 }
3228
getIndirectResult(QualType Ty,unsigned freeIntRegs) const3229 ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
3230 unsigned freeIntRegs) const {
3231 // If this is a scalar LLVM value then assume LLVM will pass it in the right
3232 // place naturally.
3233 //
3234 // This assumption is optimistic, as there could be free registers available
3235 // when we need to pass this argument in memory, and LLVM could try to pass
3236 // the argument in the free register. This does not seem to happen currently,
3237 // but this code would be much safer if we could mark the argument with
3238 // 'onstack'. See PR12193.
3239 if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty) &&
3240 !Ty->isExtIntType()) {
3241 // Treat an enum type as its underlying type.
3242 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3243 Ty = EnumTy->getDecl()->getIntegerType();
3244
3245 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
3246 : ABIArgInfo::getDirect());
3247 }
3248
3249 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
3250 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
3251
3252 // Compute the byval alignment. We specify the alignment of the byval in all
3253 // cases so that the mid-level optimizer knows the alignment of the byval.
3254 unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);
3255
3256 // Attempt to avoid passing indirect results using byval when possible. This
3257 // is important for good codegen.
3258 //
3259 // We do this by coercing the value into a scalar type which the backend can
3260 // handle naturally (i.e., without using byval).
3261 //
3262 // For simplicity, we currently only do this when we have exhausted all of the
3263 // free integer registers. Doing this when there are free integer registers
3264 // would require more care, as we would have to ensure that the coerced value
3265 // did not claim the unused register. That would require either reording the
3266 // arguments to the function (so that any subsequent inreg values came first),
3267 // or only doing this optimization when there were no following arguments that
3268 // might be inreg.
3269 //
3270 // We currently expect it to be rare (particularly in well written code) for
3271 // arguments to be passed on the stack when there are still free integer
3272 // registers available (this would typically imply large structs being passed
3273 // by value), so this seems like a fair tradeoff for now.
3274 //
3275 // We can revisit this if the backend grows support for 'onstack' parameter
3276 // attributes. See PR12193.
3277 if (freeIntRegs == 0) {
3278 uint64_t Size = getContext().getTypeSize(Ty);
3279
3280 // If this type fits in an eightbyte, coerce it into the matching integral
3281 // type, which will end up on the stack (with alignment 8).
3282 if (Align == 8 && Size <= 64)
3283 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
3284 Size));
3285 }
3286
3287 return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align));
3288 }
3289
3290 /// The ABI specifies that a value should be passed in a full vector XMM/YMM
3291 /// register. Pick an LLVM IR type that will be passed as a vector register.
GetByteVectorType(QualType Ty) const3292 llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
3293 // Wrapper structs/arrays that only contain vectors are passed just like
3294 // vectors; strip them off if present.
3295 if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))
3296 Ty = QualType(InnerTy, 0);
3297
3298 llvm::Type *IRType = CGT.ConvertType(Ty);
3299 if (isa<llvm::VectorType>(IRType)) {
3300 // Don't pass vXi128 vectors in their native type, the backend can't
3301 // legalize them.
3302 if (passInt128VectorsInMem() &&
3303 cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)) {
3304 // Use a vXi64 vector.
3305 uint64_t Size = getContext().getTypeSize(Ty);
3306 return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),
3307 Size / 64);
3308 }
3309
3310 return IRType;
3311 }
3312
3313 if (IRType->getTypeID() == llvm::Type::FP128TyID)
3314 return IRType;
3315
3316 // We couldn't find the preferred IR vector type for 'Ty'.
3317 uint64_t Size = getContext().getTypeSize(Ty);
3318 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
3319
3320
3321 // Return a LLVM IR vector type based on the size of 'Ty'.
3322 return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
3323 Size / 64);
3324 }
3325
3326 /// BitsContainNoUserData - Return true if the specified [start,end) bit range
3327 /// is known to either be off the end of the specified type or being in
3328 /// alignment padding. The user type specified is known to be at most 128 bits
3329 /// in size, and have passed through X86_64ABIInfo::classify with a successful
3330 /// classification that put one of the two halves in the INTEGER class.
3331 ///
3332 /// It is conservatively correct to return false.
BitsContainNoUserData(QualType Ty,unsigned StartBit,unsigned EndBit,ASTContext & Context)3333 static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
3334 unsigned EndBit, ASTContext &Context) {
3335 // If the bytes being queried are off the end of the type, there is no user
3336 // data hiding here. This handles analysis of builtins, vectors and other
3337 // types that don't contain interesting padding.
3338 unsigned TySize = (unsigned)Context.getTypeSize(Ty);
3339 if (TySize <= StartBit)
3340 return true;
3341
3342 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
3343 unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
3344 unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
3345
3346 // Check each element to see if the element overlaps with the queried range.
3347 for (unsigned i = 0; i != NumElts; ++i) {
3348 // If the element is after the span we care about, then we're done..
3349 unsigned EltOffset = i*EltSize;
3350 if (EltOffset >= EndBit) break;
3351
3352 unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
3353 if (!BitsContainNoUserData(AT->getElementType(), EltStart,
3354 EndBit-EltOffset, Context))
3355 return false;
3356 }
3357 // If it overlaps no elements, then it is safe to process as padding.
3358 return true;
3359 }
3360
3361 if (const RecordType *RT = Ty->getAs<RecordType>()) {
3362 const RecordDecl *RD = RT->getDecl();
3363 const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
3364
3365 // If this is a C++ record, check the bases first.
3366 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3367 for (const auto &I : CXXRD->bases()) {
3368 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3369 "Unexpected base class!");
3370 const auto *Base =
3371 cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3372
3373 // If the base is after the span we care about, ignore it.
3374 unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
3375 if (BaseOffset >= EndBit) continue;
3376
3377 unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
3378 if (!BitsContainNoUserData(I.getType(), BaseStart,
3379 EndBit-BaseOffset, Context))
3380 return false;
3381 }
3382 }
3383
3384 // Verify that no field has data that overlaps the region of interest. Yes
3385 // this could be sped up a lot by being smarter about queried fields,
3386 // however we're only looking at structs up to 16 bytes, so we don't care
3387 // much.
3388 unsigned idx = 0;
3389 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3390 i != e; ++i, ++idx) {
3391 unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
3392
3393 // If we found a field after the region we care about, then we're done.
3394 if (FieldOffset >= EndBit) break;
3395
3396 unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
3397 if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
3398 Context))
3399 return false;
3400 }
3401
3402 // If nothing in this record overlapped the area of interest, then we're
3403 // clean.
3404 return true;
3405 }
3406
3407 return false;
3408 }
3409
3410 /// getFPTypeAtOffset - Return a floating point type at the specified offset.
getFPTypeAtOffset(llvm::Type * IRType,unsigned IROffset,const llvm::DataLayout & TD)3411 static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3412 const llvm::DataLayout &TD) {
3413 if (IROffset == 0 && IRType->isFloatingPointTy())
3414 return IRType;
3415
3416 // If this is a struct, recurse into the field at the specified offset.
3417 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3418 if (!STy->getNumContainedTypes())
3419 return nullptr;
3420
3421 const llvm::StructLayout *SL = TD.getStructLayout(STy);
3422 unsigned Elt = SL->getElementContainingOffset(IROffset);
3423 IROffset -= SL->getElementOffset(Elt);
3424 return getFPTypeAtOffset(STy->getElementType(Elt), IROffset, TD);
3425 }
3426
3427 // If this is an array, recurse into the field at the specified offset.
3428 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3429 llvm::Type *EltTy = ATy->getElementType();
3430 unsigned EltSize = TD.getTypeAllocSize(EltTy);
3431 IROffset -= IROffset / EltSize * EltSize;
3432 return getFPTypeAtOffset(EltTy, IROffset, TD);
3433 }
3434
3435 return nullptr;
3436 }
3437
3438 /// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
3439 /// low 8 bytes of an XMM register, corresponding to the SSE class.
3440 llvm::Type *X86_64ABIInfo::
GetSSETypeAtOffset(llvm::Type * IRType,unsigned IROffset,QualType SourceTy,unsigned SourceOffset) const3441 GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3442 QualType SourceTy, unsigned SourceOffset) const {
3443 const llvm::DataLayout &TD = getDataLayout();
3444 unsigned SourceSize =
3445 (unsigned)getContext().getTypeSize(SourceTy) / 8 - SourceOffset;
3446 llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
3447 if (!T0 || T0->isDoubleTy())
3448 return llvm::Type::getDoubleTy(getVMContext());
3449
3450 // Get the adjacent FP type.
3451 llvm::Type *T1 = nullptr;
3452 unsigned T0Size = TD.getTypeAllocSize(T0);
3453 if (SourceSize > T0Size)
3454 T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);
3455 if (T1 == nullptr) {
3456 // Check if IRType is a half + float. float type will be in IROffset+4 due
3457 // to its alignment.
3458 if (T0->isHalfTy() && SourceSize > 4)
3459 T1 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3460 // If we can't get a second FP type, return a simple half or float.
3461 // avx512fp16-abi.c:pr51813_2 shows it works to return float for
3462 // {float, i8} too.
3463 if (T1 == nullptr)
3464 return T0;
3465 }
3466
3467 if (T0->isFloatTy() && T1->isFloatTy())
3468 return llvm::FixedVectorType::get(T0, 2);
3469
3470 if (T0->isHalfTy() && T1->isHalfTy()) {
3471 llvm::Type *T2 = nullptr;
3472 if (SourceSize > 4)
3473 T2 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3474 if (T2 == nullptr)
3475 return llvm::FixedVectorType::get(T0, 2);
3476 return llvm::FixedVectorType::get(T0, 4);
3477 }
3478
3479 if (T0->isHalfTy() || T1->isHalfTy())
3480 return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);
3481
3482 return llvm::Type::getDoubleTy(getVMContext());
3483 }
3484
3485
3486 /// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
3487 /// an 8-byte GPR. This means that we either have a scalar or we are talking
3488 /// about the high or low part of an up-to-16-byte struct. This routine picks
3489 /// the best LLVM IR type to represent this, which may be i64 or may be anything
3490 /// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
3491 /// etc).
3492 ///
3493 /// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
3494 /// the source type. IROffset is an offset in bytes into the LLVM IR type that
3495 /// the 8-byte value references. PrefType may be null.
3496 ///
3497 /// SourceTy is the source-level type for the entire argument. SourceOffset is
3498 /// an offset into this that we're processing (which is always either 0 or 8).
3499 ///
3500 llvm::Type *X86_64ABIInfo::
GetINTEGERTypeAtOffset(llvm::Type * IRType,unsigned IROffset,QualType SourceTy,unsigned SourceOffset) const3501 GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3502 QualType SourceTy, unsigned SourceOffset) const {
3503 // If we're dealing with an un-offset LLVM IR type, then it means that we're
3504 // returning an 8-byte unit starting with it. See if we can safely use it.
3505 if (IROffset == 0) {
3506 // Pointers and int64's always fill the 8-byte unit.
3507 if ((isa<llvm::PointerType>(IRType) && Has64BitPointers) ||
3508 IRType->isIntegerTy(64))
3509 return IRType;
3510
3511 // If we have a 1/2/4-byte integer, we can use it only if the rest of the
3512 // goodness in the source type is just tail padding. This is allowed to
3513 // kick in for struct {double,int} on the int, but not on
3514 // struct{double,int,int} because we wouldn't return the second int. We
3515 // have to do this analysis on the source type because we can't depend on
3516 // unions being lowered a specific way etc.
3517 if (IRType->isIntegerTy(8) || IRType->isIntegerTy(16) ||
3518 IRType->isIntegerTy(32) ||
3519 (isa<llvm::PointerType>(IRType) && !Has64BitPointers)) {
3520 unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 32 :
3521 cast<llvm::IntegerType>(IRType)->getBitWidth();
3522
3523 if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
3524 SourceOffset*8+64, getContext()))
3525 return IRType;
3526 }
3527 }
3528
3529 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3530 // If this is a struct, recurse into the field at the specified offset.
3531 const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
3532 if (IROffset < SL->getSizeInBytes()) {
3533 unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
3534 IROffset -= SL->getElementOffset(FieldIdx);
3535
3536 return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
3537 SourceTy, SourceOffset);
3538 }
3539 }
3540
3541 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3542 llvm::Type *EltTy = ATy->getElementType();
3543 unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
3544 unsigned EltOffset = IROffset/EltSize*EltSize;
3545 return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
3546 SourceOffset);
3547 }
3548
3549 // Okay, we don't have any better idea of what to pass, so we pass this in an
3550 // integer register that isn't too big to fit the rest of the struct.
3551 unsigned TySizeInBytes =
3552 (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
3553
3554 assert(TySizeInBytes != SourceOffset && "Empty field?");
3555
3556 // It is always safe to classify this as an integer type up to i64 that
3557 // isn't larger than the structure.
3558 return llvm::IntegerType::get(getVMContext(),
3559 std::min(TySizeInBytes-SourceOffset, 8U)*8);
3560 }
3561
3562
3563 /// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
3564 /// be used as elements of a two register pair to pass or return, return a
3565 /// first class aggregate to represent them. For example, if the low part of
3566 /// a by-value argument should be passed as i32* and the high part as float,
3567 /// return {i32*, float}.
3568 static llvm::Type *
GetX86_64ByValArgumentPair(llvm::Type * Lo,llvm::Type * Hi,const llvm::DataLayout & TD)3569 GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
3570 const llvm::DataLayout &TD) {
3571 // In order to correctly satisfy the ABI, we need to the high part to start
3572 // at offset 8. If the high and low parts we inferred are both 4-byte types
3573 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
3574 // the second element at offset 8. Check for this:
3575 unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
3576 unsigned HiAlign = TD.getABITypeAlignment(Hi);
3577 unsigned HiStart = llvm::alignTo(LoSize, HiAlign);
3578 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
3579
3580 // To handle this, we have to increase the size of the low part so that the
3581 // second element will start at an 8 byte offset. We can't increase the size
3582 // of the second element because it might make us access off the end of the
3583 // struct.
3584 if (HiStart != 8) {
3585 // There are usually two sorts of types the ABI generation code can produce
3586 // for the low part of a pair that aren't 8 bytes in size: half, float or
3587 // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
3588 // NaCl).
3589 // Promote these to a larger type.
3590 if (Lo->isHalfTy() || Lo->isFloatTy())
3591 Lo = llvm::Type::getDoubleTy(Lo->getContext());
3592 else {
3593 assert((Lo->isIntegerTy() || Lo->isPointerTy())
3594 && "Invalid/unknown lo type");
3595 Lo = llvm::Type::getInt64Ty(Lo->getContext());
3596 }
3597 }
3598
3599 llvm::StructType *Result = llvm::StructType::get(Lo, Hi);
3600
3601 // Verify that the second element is at an 8-byte offset.
3602 assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
3603 "Invalid x86-64 argument pair!");
3604 return Result;
3605 }
3606
3607 ABIArgInfo X86_64ABIInfo::
classifyReturnType(QualType RetTy) const3608 classifyReturnType(QualType RetTy) const {
3609 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
3610 // classification algorithm.
3611 X86_64ABIInfo::Class Lo, Hi;
3612 classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
3613
3614 // Check some invariants.
3615 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3616 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3617
3618 llvm::Type *ResType = nullptr;
3619 switch (Lo) {
3620 case NoClass:
3621 if (Hi == NoClass)
3622 return ABIArgInfo::getIgnore();
3623 // If the low part is just padding, it takes no register, leave ResType
3624 // null.
3625 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3626 "Unknown missing lo part");
3627 break;
3628
3629 case SSEUp:
3630 case X87Up:
3631 llvm_unreachable("Invalid classification for lo word.");
3632
3633 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
3634 // hidden argument.
3635 case Memory:
3636 return getIndirectReturnResult(RetTy);
3637
3638 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
3639 // available register of the sequence %rax, %rdx is used.
3640 case Integer:
3641 ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3642
3643 // If we have a sign or zero extended integer, make sure to return Extend
3644 // so that the parameter gets the right LLVM IR attributes.
3645 if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
3646 // Treat an enum type as its underlying type.
3647 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
3648 RetTy = EnumTy->getDecl()->getIntegerType();
3649
3650 if (RetTy->isIntegralOrEnumerationType() &&
3651 isPromotableIntegerTypeForABI(RetTy))
3652 return ABIArgInfo::getExtend(RetTy);
3653 }
3654 break;
3655
3656 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
3657 // available SSE register of the sequence %xmm0, %xmm1 is used.
3658 case SSE:
3659 ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3660 break;
3661
3662 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
3663 // returned on the X87 stack in %st0 as 80-bit x87 number.
3664 case X87:
3665 ResType = llvm::Type::getX86_FP80Ty(getVMContext());
3666 break;
3667
3668 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
3669 // part of the value is returned in %st0 and the imaginary part in
3670 // %st1.
3671 case ComplexX87:
3672 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
3673 ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),
3674 llvm::Type::getX86_FP80Ty(getVMContext()));
3675 break;
3676 }
3677
3678 llvm::Type *HighPart = nullptr;
3679 switch (Hi) {
3680 // Memory was handled previously and X87 should
3681 // never occur as a hi class.
3682 case Memory:
3683 case X87:
3684 llvm_unreachable("Invalid classification for hi word.");
3685
3686 case ComplexX87: // Previously handled.
3687 case NoClass:
3688 break;
3689
3690 case Integer:
3691 HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3692 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
3693 return ABIArgInfo::getDirect(HighPart, 8);
3694 break;
3695 case SSE:
3696 HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3697 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
3698 return ABIArgInfo::getDirect(HighPart, 8);
3699 break;
3700
3701 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
3702 // is passed in the next available eightbyte chunk if the last used
3703 // vector register.
3704 //
3705 // SSEUP should always be preceded by SSE, just widen.
3706 case SSEUp:
3707 assert(Lo == SSE && "Unexpected SSEUp classification.");
3708 ResType = GetByteVectorType(RetTy);
3709 break;
3710
3711 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
3712 // returned together with the previous X87 value in %st0.
3713 case X87Up:
3714 // If X87Up is preceded by X87, we don't need to do
3715 // anything. However, in some cases with unions it may not be
3716 // preceded by X87. In such situations we follow gcc and pass the
3717 // extra bits in an SSE reg.
3718 if (Lo != X87) {
3719 HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3720 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
3721 return ABIArgInfo::getDirect(HighPart, 8);
3722 }
3723 break;
3724 }
3725
3726 // If a high part was specified, merge it together with the low part. It is
3727 // known to pass in the high eightbyte of the result. We do this by forming a
3728 // first class struct aggregate with the high and low part: {low, high}
3729 if (HighPart)
3730 ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3731
3732 return ABIArgInfo::getDirect(ResType);
3733 }
3734
classifyArgumentType(QualType Ty,unsigned freeIntRegs,unsigned & neededInt,unsigned & neededSSE,bool isNamedArg) const3735 ABIArgInfo X86_64ABIInfo::classifyArgumentType(
3736 QualType Ty, unsigned freeIntRegs, unsigned &neededInt, unsigned &neededSSE,
3737 bool isNamedArg)
3738 const
3739 {
3740 Ty = useFirstFieldIfTransparentUnion(Ty);
3741
3742 X86_64ABIInfo::Class Lo, Hi;
3743 classify(Ty, 0, Lo, Hi, isNamedArg);
3744
3745 // Check some invariants.
3746 // FIXME: Enforce these by construction.
3747 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3748 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3749
3750 neededInt = 0;
3751 neededSSE = 0;
3752 llvm::Type *ResType = nullptr;
3753 switch (Lo) {
3754 case NoClass:
3755 if (Hi == NoClass)
3756 return ABIArgInfo::getIgnore();
3757 // If the low part is just padding, it takes no register, leave ResType
3758 // null.
3759 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3760 "Unknown missing lo part");
3761 break;
3762
3763 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
3764 // on the stack.
3765 case Memory:
3766
3767 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
3768 // COMPLEX_X87, it is passed in memory.
3769 case X87:
3770 case ComplexX87:
3771 if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)
3772 ++neededInt;
3773 return getIndirectResult(Ty, freeIntRegs);
3774
3775 case SSEUp:
3776 case X87Up:
3777 llvm_unreachable("Invalid classification for lo word.");
3778
3779 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3780 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3781 // and %r9 is used.
3782 case Integer:
3783 ++neededInt;
3784
3785 // Pick an 8-byte type based on the preferred type.
3786 ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);
3787
3788 // If we have a sign or zero extended integer, make sure to return Extend
3789 // so that the parameter gets the right LLVM IR attributes.
3790 if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
3791 // Treat an enum type as its underlying type.
3792 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3793 Ty = EnumTy->getDecl()->getIntegerType();
3794
3795 if (Ty->isIntegralOrEnumerationType() &&
3796 isPromotableIntegerTypeForABI(Ty))
3797 return ABIArgInfo::getExtend(Ty);
3798 }
3799
3800 break;
3801
3802 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3803 // available SSE register is used, the registers are taken in the
3804 // order from %xmm0 to %xmm7.
3805 case SSE: {
3806 llvm::Type *IRType = CGT.ConvertType(Ty);
3807 ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
3808 ++neededSSE;
3809 break;
3810 }
3811 }
3812
3813 llvm::Type *HighPart = nullptr;
3814 switch (Hi) {
3815 // Memory was handled previously, ComplexX87 and X87 should
3816 // never occur as hi classes, and X87Up must be preceded by X87,
3817 // which is passed in memory.
3818 case Memory:
3819 case X87:
3820 case ComplexX87:
3821 llvm_unreachable("Invalid classification for hi word.");
3822
3823 case NoClass: break;
3824
3825 case Integer:
3826 ++neededInt;
3827 // Pick an 8-byte type based on the preferred type.
3828 HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3829
3830 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3831 return ABIArgInfo::getDirect(HighPart, 8);
3832 break;
3833
3834 // X87Up generally doesn't occur here (long double is passed in
3835 // memory), except in situations involving unions.
3836 case X87Up:
3837 case SSE:
3838 HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3839
3840 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3841 return ABIArgInfo::getDirect(HighPart, 8);
3842
3843 ++neededSSE;
3844 break;
3845
3846 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3847 // eightbyte is passed in the upper half of the last used SSE
3848 // register. This only happens when 128-bit vectors are passed.
3849 case SSEUp:
3850 assert(Lo == SSE && "Unexpected SSEUp classification");
3851 ResType = GetByteVectorType(Ty);
3852 break;
3853 }
3854
3855 // If a high part was specified, merge it together with the low part. It is
3856 // known to pass in the high eightbyte of the result. We do this by forming a
3857 // first class struct aggregate with the high and low part: {low, high}
3858 if (HighPart)
3859 ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3860
3861 return ABIArgInfo::getDirect(ResType);
3862 }
3863
3864 ABIArgInfo
classifyRegCallStructTypeImpl(QualType Ty,unsigned & NeededInt,unsigned & NeededSSE) const3865 X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
3866 unsigned &NeededSSE) const {
3867 auto RT = Ty->getAs<RecordType>();
3868 assert(RT && "classifyRegCallStructType only valid with struct types");
3869
3870 if (RT->getDecl()->hasFlexibleArrayMember())
3871 return getIndirectReturnResult(Ty);
3872
3873 // Sum up bases
3874 if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
3875 if (CXXRD->isDynamicClass()) {
3876 NeededInt = NeededSSE = 0;
3877 return getIndirectReturnResult(Ty);
3878 }
3879
3880 for (const auto &I : CXXRD->bases())
3881 if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE)
3882 .isIndirect()) {
3883 NeededInt = NeededSSE = 0;
3884 return getIndirectReturnResult(Ty);
3885 }
3886 }
3887
3888 // Sum up members
3889 for (const auto *FD : RT->getDecl()->fields()) {
3890 if (FD->getType()->isRecordType() && !FD->getType()->isUnionType()) {
3891 if (classifyRegCallStructTypeImpl(FD->getType(), NeededInt, NeededSSE)
3892 .isIndirect()) {
3893 NeededInt = NeededSSE = 0;
3894 return getIndirectReturnResult(Ty);
3895 }
3896 } else {
3897 unsigned LocalNeededInt, LocalNeededSSE;
3898 if (classifyArgumentType(FD->getType(), UINT_MAX, LocalNeededInt,
3899 LocalNeededSSE, true)
3900 .isIndirect()) {
3901 NeededInt = NeededSSE = 0;
3902 return getIndirectReturnResult(Ty);
3903 }
3904 NeededInt += LocalNeededInt;
3905 NeededSSE += LocalNeededSSE;
3906 }
3907 }
3908
3909 return ABIArgInfo::getDirect();
3910 }
3911
classifyRegCallStructType(QualType Ty,unsigned & NeededInt,unsigned & NeededSSE) const3912 ABIArgInfo X86_64ABIInfo::classifyRegCallStructType(QualType Ty,
3913 unsigned &NeededInt,
3914 unsigned &NeededSSE) const {
3915
3916 NeededInt = 0;
3917 NeededSSE = 0;
3918
3919 return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE);
3920 }
3921
computeInfo(CGFunctionInfo & FI) const3922 void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3923
3924 const unsigned CallingConv = FI.getCallingConvention();
3925 // It is possible to force Win64 calling convention on any x86_64 target by
3926 // using __attribute__((ms_abi)). In such case to correctly emit Win64
3927 // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3928 if (CallingConv == llvm::CallingConv::Win64) {
3929 WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3930 Win64ABIInfo.computeInfo(FI);
3931 return;
3932 }
3933
3934 bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3935
3936 // Keep track of the number of assigned registers.
3937 unsigned FreeIntRegs = IsRegCall ? 11 : 6;
3938 unsigned FreeSSERegs = IsRegCall ? 16 : 8;
3939 unsigned NeededInt, NeededSSE;
3940
3941 if (!::classifyReturnType(getCXXABI(), FI, *this)) {
3942 if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
3943 !FI.getReturnType()->getTypePtr()->isUnionType()) {
3944 FI.getReturnInfo() =
3945 classifyRegCallStructType(FI.getReturnType(), NeededInt, NeededSSE);
3946 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3947 FreeIntRegs -= NeededInt;
3948 FreeSSERegs -= NeededSSE;
3949 } else {
3950 FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3951 }
3952 } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&
3953 getContext().getCanonicalType(FI.getReturnType()
3954 ->getAs<ComplexType>()
3955 ->getElementType()) ==
3956 getContext().LongDoubleTy)
3957 // Complex Long Double Type is passed in Memory when Regcall
3958 // calling convention is used.
3959 FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3960 else
3961 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
3962 }
3963
3964 // If the return value is indirect, then the hidden argument is consuming one
3965 // integer register.
3966 if (FI.getReturnInfo().isIndirect())
3967 --FreeIntRegs;
3968
3969 // The chain argument effectively gives us another free register.
3970 if (FI.isChainCall())
3971 ++FreeIntRegs;
3972
3973 unsigned NumRequiredArgs = FI.getNumRequiredArgs();
3974 // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
3975 // get assigned (in left-to-right order) for passing as follows...
3976 unsigned ArgNo = 0;
3977 for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
3978 it != ie; ++it, ++ArgNo) {
3979 bool IsNamedArg = ArgNo < NumRequiredArgs;
3980
3981 if (IsRegCall && it->type->isStructureOrClassType())
3982 it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE);
3983 else
3984 it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
3985 NeededSSE, IsNamedArg);
3986
3987 // AMD64-ABI 3.2.3p3: If there are no registers available for any
3988 // eightbyte of an argument, the whole argument is passed on the
3989 // stack. If registers have already been assigned for some
3990 // eightbytes of such an argument, the assignments get reverted.
3991 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3992 FreeIntRegs -= NeededInt;
3993 FreeSSERegs -= NeededSSE;
3994 } else {
3995 it->info = getIndirectResult(it->type, FreeIntRegs);
3996 }
3997 }
3998 }
3999
EmitX86_64VAArgFromMemory(CodeGenFunction & CGF,Address VAListAddr,QualType Ty)4000 static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
4001 Address VAListAddr, QualType Ty) {
4002 Address overflow_arg_area_p =
4003 CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
4004 llvm::Value *overflow_arg_area =
4005 CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
4006
4007 // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
4008 // byte boundary if alignment needed by type exceeds 8 byte boundary.
4009 // It isn't stated explicitly in the standard, but in practice we use
4010 // alignment greater than 16 where necessary.
4011 CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4012 if (Align > CharUnits::fromQuantity(8)) {
4013 overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
4014 Align);
4015 }
4016
4017 // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
4018 llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4019 llvm::Value *Res =
4020 CGF.Builder.CreateBitCast(overflow_arg_area,
4021 llvm::PointerType::getUnqual(LTy));
4022
4023 // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
4024 // l->overflow_arg_area + sizeof(type).
4025 // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
4026 // an 8 byte boundary.
4027
4028 uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
4029 llvm::Value *Offset =
4030 llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7) & ~7);
4031 overflow_arg_area = CGF.Builder.CreateGEP(CGF.Int8Ty, overflow_arg_area,
4032 Offset, "overflow_arg_area.next");
4033 CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
4034
4035 // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
4036 return Address(Res, Align);
4037 }
4038
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const4039 Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4040 QualType Ty) const {
4041 // Assume that va_list type is correct; should be pointer to LLVM type:
4042 // struct {
4043 // i32 gp_offset;
4044 // i32 fp_offset;
4045 // i8* overflow_arg_area;
4046 // i8* reg_save_area;
4047 // };
4048 unsigned neededInt, neededSSE;
4049
4050 Ty = getContext().getCanonicalType(Ty);
4051 ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
4052 /*isNamedArg*/false);
4053
4054 // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
4055 // in the registers. If not go to step 7.
4056 if (!neededInt && !neededSSE)
4057 return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4058
4059 // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
4060 // general purpose registers needed to pass type and num_fp to hold
4061 // the number of floating point registers needed.
4062
4063 // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
4064 // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
4065 // l->fp_offset > 304 - num_fp * 16 go to step 7.
4066 //
4067 // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
4068 // register save space).
4069
4070 llvm::Value *InRegs = nullptr;
4071 Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
4072 llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
4073 if (neededInt) {
4074 gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
4075 gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
4076 InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
4077 InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
4078 }
4079
4080 if (neededSSE) {
4081 fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
4082 fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
4083 llvm::Value *FitsInFP =
4084 llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
4085 FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");
4086 InRegs = InRegs ? CGF.Builder.CreateAnd(InRegs, FitsInFP) : FitsInFP;
4087 }
4088
4089 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
4090 llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
4091 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
4092 CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
4093
4094 // Emit code to load the value if it was passed in registers.
4095
4096 CGF.EmitBlock(InRegBlock);
4097
4098 // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
4099 // an offset of l->gp_offset and/or l->fp_offset. This may require
4100 // copying to a temporary location in case the parameter is passed
4101 // in different register classes or requires an alignment greater
4102 // than 8 for general purpose registers and 16 for XMM registers.
4103 //
4104 // FIXME: This really results in shameful code when we end up needing to
4105 // collect arguments from different places; often what should result in a
4106 // simple assembling of a structure from scattered addresses has many more
4107 // loads than necessary. Can we clean this up?
4108 llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4109 llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
4110 CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");
4111
4112 Address RegAddr = Address::invalid();
4113 if (neededInt && neededSSE) {
4114 // FIXME: Cleanup.
4115 assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
4116 llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
4117 Address Tmp = CGF.CreateMemTemp(Ty);
4118 Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4119 assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
4120 llvm::Type *TyLo = ST->getElementType(0);
4121 llvm::Type *TyHi = ST->getElementType(1);
4122 assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
4123 "Unexpected ABI info for mixed regs");
4124 llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
4125 llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
4126 llvm::Value *GPAddr =
4127 CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset);
4128 llvm::Value *FPAddr =
4129 CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset);
4130 llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
4131 llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
4132
4133 // Copy the first element.
4134 // FIXME: Our choice of alignment here and below is probably pessimistic.
4135 llvm::Value *V = CGF.Builder.CreateAlignedLoad(
4136 TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo),
4137 CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyLo)));
4138 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4139
4140 // Copy the second element.
4141 V = CGF.Builder.CreateAlignedLoad(
4142 TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi),
4143 CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyHi)));
4144 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4145
4146 RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4147 } else if (neededInt) {
4148 RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset),
4149 CharUnits::fromQuantity(8));
4150 RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4151
4152 // Copy to a temporary if necessary to ensure the appropriate alignment.
4153 auto TInfo = getContext().getTypeInfoInChars(Ty);
4154 uint64_t TySize = TInfo.Width.getQuantity();
4155 CharUnits TyAlign = TInfo.Align;
4156
4157 // Copy into a temporary if the type is more aligned than the
4158 // register save area.
4159 if (TyAlign.getQuantity() > 8) {
4160 Address Tmp = CGF.CreateMemTemp(Ty);
4161 CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
4162 RegAddr = Tmp;
4163 }
4164
4165 } else if (neededSSE == 1) {
4166 RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset),
4167 CharUnits::fromQuantity(16));
4168 RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4169 } else {
4170 assert(neededSSE == 2 && "Invalid number of needed registers!");
4171 // SSE registers are spaced 16 bytes apart in the register save
4172 // area, we need to collect the two eightbytes together.
4173 // The ABI isn't explicit about this, but it seems reasonable
4174 // to assume that the slots are 16-byte aligned, since the stack is
4175 // naturally 16-byte aligned and the prologue is expected to store
4176 // all the SSE registers to the RSA.
4177 Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,
4178 fp_offset),
4179 CharUnits::fromQuantity(16));
4180 Address RegAddrHi =
4181 CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
4182 CharUnits::fromQuantity(16));
4183 llvm::Type *ST = AI.canHaveCoerceToType()
4184 ? AI.getCoerceToType()
4185 : llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy);
4186 llvm::Value *V;
4187 Address Tmp = CGF.CreateMemTemp(Ty);
4188 Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4189 V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4190 RegAddrLo, ST->getStructElementType(0)));
4191 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4192 V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4193 RegAddrHi, ST->getStructElementType(1)));
4194 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4195
4196 RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4197 }
4198
4199 // AMD64-ABI 3.5.7p5: Step 5. Set:
4200 // l->gp_offset = l->gp_offset + num_gp * 8
4201 // l->fp_offset = l->fp_offset + num_fp * 16.
4202 if (neededInt) {
4203 llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);
4204 CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
4205 gp_offset_p);
4206 }
4207 if (neededSSE) {
4208 llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);
4209 CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
4210 fp_offset_p);
4211 }
4212 CGF.EmitBranch(ContBlock);
4213
4214 // Emit code to load the value if it was passed in memory.
4215
4216 CGF.EmitBlock(InMemBlock);
4217 Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4218
4219 // Return the appropriate result.
4220
4221 CGF.EmitBlock(ContBlock);
4222 Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
4223 "vaarg.addr");
4224 return ResAddr;
4225 }
4226
EmitMSVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const4227 Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
4228 QualType Ty) const {
4229 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4230 // not 1, 2, 4, or 8 bytes, must be passed by reference."
4231 uint64_t Width = getContext().getTypeSize(Ty);
4232 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);
4233
4234 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4235 CGF.getContext().getTypeInfoInChars(Ty),
4236 CharUnits::fromQuantity(8),
4237 /*allowHigherAlign*/ false);
4238 }
4239
reclassifyHvaArgForVectorCall(QualType Ty,unsigned & FreeSSERegs,const ABIArgInfo & current) const4240 ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
4241 QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo ¤t) const {
4242 const Type *Base = nullptr;
4243 uint64_t NumElts = 0;
4244
4245 if (!Ty->isBuiltinType() && !Ty->isVectorType() &&
4246 isHomogeneousAggregate(Ty, Base, NumElts) && FreeSSERegs >= NumElts) {
4247 FreeSSERegs -= NumElts;
4248 return getDirectX86Hva();
4249 }
4250 return current;
4251 }
4252
classify(QualType Ty,unsigned & FreeSSERegs,bool IsReturnType,bool IsVectorCall,bool IsRegCall) const4253 ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
4254 bool IsReturnType, bool IsVectorCall,
4255 bool IsRegCall) const {
4256
4257 if (Ty->isVoidType())
4258 return ABIArgInfo::getIgnore();
4259
4260 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4261 Ty = EnumTy->getDecl()->getIntegerType();
4262
4263 TypeInfo Info = getContext().getTypeInfo(Ty);
4264 uint64_t Width = Info.Width;
4265 CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
4266
4267 const RecordType *RT = Ty->getAs<RecordType>();
4268 if (RT) {
4269 if (!IsReturnType) {
4270 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
4271 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4272 }
4273
4274 if (RT->getDecl()->hasFlexibleArrayMember())
4275 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4276
4277 }
4278
4279 const Type *Base = nullptr;
4280 uint64_t NumElts = 0;
4281 // vectorcall adds the concept of a homogenous vector aggregate, similar to
4282 // other targets.
4283 if ((IsVectorCall || IsRegCall) &&
4284 isHomogeneousAggregate(Ty, Base, NumElts)) {
4285 if (IsRegCall) {
4286 if (FreeSSERegs >= NumElts) {
4287 FreeSSERegs -= NumElts;
4288 if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())
4289 return ABIArgInfo::getDirect();
4290 return ABIArgInfo::getExpand();
4291 }
4292 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4293 } else if (IsVectorCall) {
4294 if (FreeSSERegs >= NumElts &&
4295 (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())) {
4296 FreeSSERegs -= NumElts;
4297 return ABIArgInfo::getDirect();
4298 } else if (IsReturnType) {
4299 return ABIArgInfo::getExpand();
4300 } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {
4301 // HVAs are delayed and reclassified in the 2nd step.
4302 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4303 }
4304 }
4305 }
4306
4307 if (Ty->isMemberPointerType()) {
4308 // If the member pointer is represented by an LLVM int or ptr, pass it
4309 // directly.
4310 llvm::Type *LLTy = CGT.ConvertType(Ty);
4311 if (LLTy->isPointerTy() || LLTy->isIntegerTy())
4312 return ABIArgInfo::getDirect();
4313 }
4314
4315 if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {
4316 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4317 // not 1, 2, 4, or 8 bytes, must be passed by reference."
4318 if (Width > 64 || !llvm::isPowerOf2_64(Width))
4319 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4320
4321 // Otherwise, coerce it to a small integer.
4322 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
4323 }
4324
4325 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
4326 switch (BT->getKind()) {
4327 case BuiltinType::Bool:
4328 // Bool type is always extended to the ABI, other builtin types are not
4329 // extended.
4330 return ABIArgInfo::getExtend(Ty);
4331
4332 case BuiltinType::LongDouble:
4333 // Mingw64 GCC uses the old 80 bit extended precision floating point
4334 // unit. It passes them indirectly through memory.
4335 if (IsMingw64) {
4336 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
4337 if (LDF == &llvm::APFloat::x87DoubleExtended())
4338 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4339 }
4340 break;
4341
4342 case BuiltinType::Int128:
4343 case BuiltinType::UInt128:
4344 // If it's a parameter type, the normal ABI rule is that arguments larger
4345 // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
4346 // even though it isn't particularly efficient.
4347 if (!IsReturnType)
4348 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4349
4350 // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
4351 // Clang matches them for compatibility.
4352 return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
4353 llvm::Type::getInt64Ty(getVMContext()), 2));
4354
4355 default:
4356 break;
4357 }
4358 }
4359
4360 if (Ty->isExtIntType()) {
4361 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4362 // not 1, 2, 4, or 8 bytes, must be passed by reference."
4363 // However, non-power-of-two _ExtInts will be passed as 1,2,4 or 8 bytes
4364 // anyway as long is it fits in them, so we don't have to check the power of
4365 // 2.
4366 if (Width <= 64)
4367 return ABIArgInfo::getDirect();
4368 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4369 }
4370
4371 return ABIArgInfo::getDirect();
4372 }
4373
computeInfo(CGFunctionInfo & FI) const4374 void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
4375 const unsigned CC = FI.getCallingConvention();
4376 bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
4377 bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
4378
4379 // If __attribute__((sysv_abi)) is in use, use the SysV argument
4380 // classification rules.
4381 if (CC == llvm::CallingConv::X86_64_SysV) {
4382 X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
4383 SysVABIInfo.computeInfo(FI);
4384 return;
4385 }
4386
4387 unsigned FreeSSERegs = 0;
4388 if (IsVectorCall) {
4389 // We can use up to 4 SSE return registers with vectorcall.
4390 FreeSSERegs = 4;
4391 } else if (IsRegCall) {
4392 // RegCall gives us 16 SSE registers.
4393 FreeSSERegs = 16;
4394 }
4395
4396 if (!getCXXABI().classifyReturnType(FI))
4397 FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,
4398 IsVectorCall, IsRegCall);
4399
4400 if (IsVectorCall) {
4401 // We can use up to 6 SSE register parameters with vectorcall.
4402 FreeSSERegs = 6;
4403 } else if (IsRegCall) {
4404 // RegCall gives us 16 SSE registers, we can reuse the return registers.
4405 FreeSSERegs = 16;
4406 }
4407
4408 unsigned ArgNum = 0;
4409 unsigned ZeroSSERegs = 0;
4410 for (auto &I : FI.arguments()) {
4411 // Vectorcall in x64 only permits the first 6 arguments to be passed as
4412 // XMM/YMM registers. After the sixth argument, pretend no vector
4413 // registers are left.
4414 unsigned *MaybeFreeSSERegs =
4415 (IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;
4416 I.info =
4417 classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);
4418 ++ArgNum;
4419 }
4420
4421 if (IsVectorCall) {
4422 // For vectorcall, assign aggregate HVAs to any free vector registers in a
4423 // second pass.
4424 for (auto &I : FI.arguments())
4425 I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);
4426 }
4427 }
4428
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const4429 Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4430 QualType Ty) const {
4431 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4432 // not 1, 2, 4, or 8 bytes, must be passed by reference."
4433 uint64_t Width = getContext().getTypeSize(Ty);
4434 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);
4435
4436 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4437 CGF.getContext().getTypeInfoInChars(Ty),
4438 CharUnits::fromQuantity(8),
4439 /*allowHigherAlign*/ false);
4440 }
4441
PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address,bool Is64Bit,bool IsAIX)4442 static bool PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4443 llvm::Value *Address, bool Is64Bit,
4444 bool IsAIX) {
4445 // This is calculated from the LLVM and GCC tables and verified
4446 // against gcc output. AFAIK all PPC ABIs use the same encoding.
4447
4448 CodeGen::CGBuilderTy &Builder = CGF.Builder;
4449
4450 llvm::IntegerType *i8 = CGF.Int8Ty;
4451 llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
4452 llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
4453 llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
4454
4455 // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers
4456 AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 0, 31);
4457
4458 // 32-63: fp0-31, the 8-byte floating-point registers
4459 AssignToArrayRange(Builder, Address, Eight8, 32, 63);
4460
4461 // 64-67 are various 4-byte or 8-byte special-purpose registers:
4462 // 64: mq
4463 // 65: lr
4464 // 66: ctr
4465 // 67: ap
4466 AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 64, 67);
4467
4468 // 68-76 are various 4-byte special-purpose registers:
4469 // 68-75 cr0-7
4470 // 76: xer
4471 AssignToArrayRange(Builder, Address, Four8, 68, 76);
4472
4473 // 77-108: v0-31, the 16-byte vector registers
4474 AssignToArrayRange(Builder, Address, Sixteen8, 77, 108);
4475
4476 // 109: vrsave
4477 // 110: vscr
4478 AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 109, 110);
4479
4480 // AIX does not utilize the rest of the registers.
4481 if (IsAIX)
4482 return false;
4483
4484 // 111: spe_acc
4485 // 112: spefscr
4486 // 113: sfp
4487 AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 111, 113);
4488
4489 if (!Is64Bit)
4490 return false;
4491
4492 // TODO: Need to verify if these registers are used on 64 bit AIX with Power8
4493 // or above CPU.
4494 // 64-bit only registers:
4495 // 114: tfhar
4496 // 115: tfiar
4497 // 116: texasr
4498 AssignToArrayRange(Builder, Address, Eight8, 114, 116);
4499
4500 return false;
4501 }
4502
4503 // AIX
4504 namespace {
4505 /// AIXABIInfo - The AIX XCOFF ABI information.
4506 class AIXABIInfo : public ABIInfo {
4507 const bool Is64Bit;
4508 const unsigned PtrByteSize;
4509 CharUnits getParamTypeAlignment(QualType Ty) const;
4510
4511 public:
AIXABIInfo(CodeGen::CodeGenTypes & CGT,bool Is64Bit)4512 AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4513 : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {}
4514
4515 bool isPromotableTypeForABI(QualType Ty) const;
4516
4517 ABIArgInfo classifyReturnType(QualType RetTy) const;
4518 ABIArgInfo classifyArgumentType(QualType Ty) const;
4519
computeInfo(CGFunctionInfo & FI) const4520 void computeInfo(CGFunctionInfo &FI) const override {
4521 if (!getCXXABI().classifyReturnType(FI))
4522 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4523
4524 for (auto &I : FI.arguments())
4525 I.info = classifyArgumentType(I.type);
4526 }
4527
4528 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4529 QualType Ty) const override;
4530 };
4531
4532 class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
4533 const bool Is64Bit;
4534
4535 public:
AIXTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,bool Is64Bit)4536 AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4537 : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)),
4538 Is64Bit(Is64Bit) {}
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const4539 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4540 return 1; // r1 is the dedicated stack pointer
4541 }
4542
4543 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4544 llvm::Value *Address) const override;
4545 };
4546 } // namespace
4547
4548 // Return true if the ABI requires Ty to be passed sign- or zero-
4549 // extended to 32/64 bits.
isPromotableTypeForABI(QualType Ty) const4550 bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
4551 // Treat an enum type as its underlying type.
4552 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4553 Ty = EnumTy->getDecl()->getIntegerType();
4554
4555 // Promotable integer types are required to be promoted by the ABI.
4556 if (Ty->isPromotableIntegerType())
4557 return true;
4558
4559 if (!Is64Bit)
4560 return false;
4561
4562 // For 64 bit mode, in addition to the usual promotable integer types, we also
4563 // need to extend all 32-bit types, since the ABI requires promotion to 64
4564 // bits.
4565 if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4566 switch (BT->getKind()) {
4567 case BuiltinType::Int:
4568 case BuiltinType::UInt:
4569 return true;
4570 default:
4571 break;
4572 }
4573
4574 return false;
4575 }
4576
classifyReturnType(QualType RetTy) const4577 ABIArgInfo AIXABIInfo::classifyReturnType(QualType RetTy) const {
4578 if (RetTy->isAnyComplexType())
4579 return ABIArgInfo::getDirect();
4580
4581 if (RetTy->isVectorType())
4582 return ABIArgInfo::getDirect();
4583
4584 if (RetTy->isVoidType())
4585 return ABIArgInfo::getIgnore();
4586
4587 if (isAggregateTypeForABI(RetTy))
4588 return getNaturalAlignIndirect(RetTy);
4589
4590 return (isPromotableTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
4591 : ABIArgInfo::getDirect());
4592 }
4593
classifyArgumentType(QualType Ty) const4594 ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
4595 Ty = useFirstFieldIfTransparentUnion(Ty);
4596
4597 if (Ty->isAnyComplexType())
4598 return ABIArgInfo::getDirect();
4599
4600 if (Ty->isVectorType())
4601 return ABIArgInfo::getDirect();
4602
4603 if (isAggregateTypeForABI(Ty)) {
4604 // Records with non-trivial destructors/copy-constructors should not be
4605 // passed by value.
4606 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
4607 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4608
4609 CharUnits CCAlign = getParamTypeAlignment(Ty);
4610 CharUnits TyAlign = getContext().getTypeAlignInChars(Ty);
4611
4612 return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true,
4613 /*Realign*/ TyAlign > CCAlign);
4614 }
4615
4616 return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
4617 : ABIArgInfo::getDirect());
4618 }
4619
getParamTypeAlignment(QualType Ty) const4620 CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
4621 // Complex types are passed just like their elements.
4622 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4623 Ty = CTy->getElementType();
4624
4625 if (Ty->isVectorType())
4626 return CharUnits::fromQuantity(16);
4627
4628 // If the structure contains a vector type, the alignment is 16.
4629 if (isRecordWithSIMDVectorType(getContext(), Ty))
4630 return CharUnits::fromQuantity(16);
4631
4632 return CharUnits::fromQuantity(PtrByteSize);
4633 }
4634
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const4635 Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4636 QualType Ty) const {
4637
4638 auto TypeInfo = getContext().getTypeInfoInChars(Ty);
4639 TypeInfo.Align = getParamTypeAlignment(Ty);
4640
4641 CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize);
4642
4643 // If we have a complex type and the base type is smaller than the register
4644 // size, the ABI calls for the real and imaginary parts to be right-adjusted
4645 // in separate words in 32bit mode or doublewords in 64bit mode. However,
4646 // Clang expects us to produce a pointer to a structure with the two parts
4647 // packed tightly. So generate loads of the real and imaginary parts relative
4648 // to the va_list pointer, and store them to a temporary structure. We do the
4649 // same as the PPC64ABI here.
4650 if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4651 CharUnits EltSize = TypeInfo.Width / 2;
4652 if (EltSize < SlotSize)
4653 return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
4654 }
4655
4656 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
4657 SlotSize, /*AllowHigher*/ true);
4658 }
4659
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const4660 bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable(
4661 CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const {
4662 return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true);
4663 }
4664
4665 // PowerPC-32
4666 namespace {
4667 /// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
4668 class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
4669 bool IsSoftFloatABI;
4670 bool IsRetSmallStructInRegABI;
4671
4672 CharUnits getParamTypeAlignment(QualType Ty) const;
4673
4674 public:
PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes & CGT,bool SoftFloatABI,bool RetSmallStructInRegABI)4675 PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI,
4676 bool RetSmallStructInRegABI)
4677 : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI),
4678 IsRetSmallStructInRegABI(RetSmallStructInRegABI) {}
4679
4680 ABIArgInfo classifyReturnType(QualType RetTy) const;
4681
computeInfo(CGFunctionInfo & FI) const4682 void computeInfo(CGFunctionInfo &FI) const override {
4683 if (!getCXXABI().classifyReturnType(FI))
4684 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4685 for (auto &I : FI.arguments())
4686 I.info = classifyArgumentType(I.type);
4687 }
4688
4689 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4690 QualType Ty) const override;
4691 };
4692
4693 class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
4694 public:
PPC32TargetCodeGenInfo(CodeGenTypes & CGT,bool SoftFloatABI,bool RetSmallStructInRegABI)4695 PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI,
4696 bool RetSmallStructInRegABI)
4697 : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>(
4698 CGT, SoftFloatABI, RetSmallStructInRegABI)) {}
4699
4700 static bool isStructReturnInRegABI(const llvm::Triple &Triple,
4701 const CodeGenOptions &Opts);
4702
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const4703 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4704 // This is recovered from gcc output.
4705 return 1; // r1 is the dedicated stack pointer
4706 }
4707
4708 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4709 llvm::Value *Address) const override;
4710 };
4711 }
4712
getParamTypeAlignment(QualType Ty) const4713 CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
4714 // Complex types are passed just like their elements.
4715 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4716 Ty = CTy->getElementType();
4717
4718 if (Ty->isVectorType())
4719 return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16
4720 : 4);
4721
4722 // For single-element float/vector structs, we consider the whole type
4723 // to have the same alignment requirements as its single element.
4724 const Type *AlignTy = nullptr;
4725 if (const Type *EltType = isSingleElementStruct(Ty, getContext())) {
4726 const BuiltinType *BT = EltType->getAs<BuiltinType>();
4727 if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
4728 (BT && BT->isFloatingPoint()))
4729 AlignTy = EltType;
4730 }
4731
4732 if (AlignTy)
4733 return CharUnits::fromQuantity(AlignTy->isVectorType() ? 16 : 4);
4734 return CharUnits::fromQuantity(4);
4735 }
4736
classifyReturnType(QualType RetTy) const4737 ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
4738 uint64_t Size;
4739
4740 // -msvr4-struct-return puts small aggregates in GPR3 and GPR4.
4741 if (isAggregateTypeForABI(RetTy) && IsRetSmallStructInRegABI &&
4742 (Size = getContext().getTypeSize(RetTy)) <= 64) {
4743 // System V ABI (1995), page 3-22, specified:
4744 // > A structure or union whose size is less than or equal to 8 bytes
4745 // > shall be returned in r3 and r4, as if it were first stored in the
4746 // > 8-byte aligned memory area and then the low addressed word were
4747 // > loaded into r3 and the high-addressed word into r4. Bits beyond
4748 // > the last member of the structure or union are not defined.
4749 //
4750 // GCC for big-endian PPC32 inserts the pad before the first member,
4751 // not "beyond the last member" of the struct. To stay compatible
4752 // with GCC, we coerce the struct to an integer of the same size.
4753 // LLVM will extend it and return i32 in r3, or i64 in r3:r4.
4754 if (Size == 0)
4755 return ABIArgInfo::getIgnore();
4756 else {
4757 llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size);
4758 return ABIArgInfo::getDirect(CoerceTy);
4759 }
4760 }
4761
4762 return DefaultABIInfo::classifyReturnType(RetTy);
4763 }
4764
4765 // TODO: this implementation is now likely redundant with
4766 // DefaultABIInfo::EmitVAArg.
EmitVAArg(CodeGenFunction & CGF,Address VAList,QualType Ty) const4767 Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
4768 QualType Ty) const {
4769 if (getTarget().getTriple().isOSDarwin()) {
4770 auto TI = getContext().getTypeInfoInChars(Ty);
4771 TI.Align = getParamTypeAlignment(Ty);
4772
4773 CharUnits SlotSize = CharUnits::fromQuantity(4);
4774 return emitVoidPtrVAArg(CGF, VAList, Ty,
4775 classifyArgumentType(Ty).isIndirect(), TI, SlotSize,
4776 /*AllowHigherAlign=*/true);
4777 }
4778
4779 const unsigned OverflowLimit = 8;
4780 if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4781 // TODO: Implement this. For now ignore.
4782 (void)CTy;
4783 return Address::invalid(); // FIXME?
4784 }
4785
4786 // struct __va_list_tag {
4787 // unsigned char gpr;
4788 // unsigned char fpr;
4789 // unsigned short reserved;
4790 // void *overflow_arg_area;
4791 // void *reg_save_area;
4792 // };
4793
4794 bool isI64 = Ty->isIntegerType() && getContext().getTypeSize(Ty) == 64;
4795 bool isInt = !Ty->isFloatingType();
4796 bool isF64 = Ty->isFloatingType() && getContext().getTypeSize(Ty) == 64;
4797
4798 // All aggregates are passed indirectly? That doesn't seem consistent
4799 // with the argument-lowering code.
4800 bool isIndirect = isAggregateTypeForABI(Ty);
4801
4802 CGBuilderTy &Builder = CGF.Builder;
4803
4804 // The calling convention either uses 1-2 GPRs or 1 FPR.
4805 Address NumRegsAddr = Address::invalid();
4806 if (isInt || IsSoftFloatABI) {
4807 NumRegsAddr = Builder.CreateStructGEP(VAList, 0, "gpr");
4808 } else {
4809 NumRegsAddr = Builder.CreateStructGEP(VAList, 1, "fpr");
4810 }
4811
4812 llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
4813
4814 // "Align" the register count when TY is i64.
4815 if (isI64 || (isF64 && IsSoftFloatABI)) {
4816 NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
4817 NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
4818 }
4819
4820 llvm::Value *CC =
4821 Builder.CreateICmpULT(NumRegs, Builder.getInt8(OverflowLimit), "cond");
4822
4823 llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
4824 llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
4825 llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
4826
4827 Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
4828
4829 llvm::Type *DirectTy = CGF.ConvertType(Ty);
4830 if (isIndirect) DirectTy = DirectTy->getPointerTo(0);
4831
4832 // Case 1: consume registers.
4833 Address RegAddr = Address::invalid();
4834 {
4835 CGF.EmitBlock(UsingRegs);
4836
4837 Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
4838 RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr),
4839 CharUnits::fromQuantity(8));
4840 assert(RegAddr.getElementType() == CGF.Int8Ty);
4841
4842 // Floating-point registers start after the general-purpose registers.
4843 if (!(isInt || IsSoftFloatABI)) {
4844 RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
4845 CharUnits::fromQuantity(32));
4846 }
4847
4848 // Get the address of the saved value by scaling the number of
4849 // registers we've used by the number of
4850 CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8);
4851 llvm::Value *RegOffset =
4852 Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
4853 RegAddr = Address(Builder.CreateInBoundsGEP(CGF.Int8Ty,
4854 RegAddr.getPointer(), RegOffset),
4855 RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
4856 RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
4857
4858 // Increase the used-register count.
4859 NumRegs =
4860 Builder.CreateAdd(NumRegs,
4861 Builder.getInt8((isI64 || (isF64 && IsSoftFloatABI)) ? 2 : 1));
4862 Builder.CreateStore(NumRegs, NumRegsAddr);
4863
4864 CGF.EmitBranch(Cont);
4865 }
4866
4867 // Case 2: consume space in the overflow area.
4868 Address MemAddr = Address::invalid();
4869 {
4870 CGF.EmitBlock(UsingOverflow);
4871
4872 Builder.CreateStore(Builder.getInt8(OverflowLimit), NumRegsAddr);
4873
4874 // Everything in the overflow area is rounded up to a size of at least 4.
4875 CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
4876
4877 CharUnits Size;
4878 if (!isIndirect) {
4879 auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
4880 Size = TypeInfo.Width.alignTo(OverflowAreaAlign);
4881 } else {
4882 Size = CGF.getPointerSize();
4883 }
4884
4885 Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
4886 Address OverflowArea(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"),
4887 OverflowAreaAlign);
4888 // Round up address of argument to alignment
4889 CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4890 if (Align > OverflowAreaAlign) {
4891 llvm::Value *Ptr = OverflowArea.getPointer();
4892 OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
4893 Align);
4894 }
4895
4896 MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
4897
4898 // Increase the overflow area.
4899 OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
4900 Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
4901 CGF.EmitBranch(Cont);
4902 }
4903
4904 CGF.EmitBlock(Cont);
4905
4906 // Merge the cases with a phi.
4907 Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
4908 "vaarg.addr");
4909
4910 // Load the pointer if the argument was passed indirectly.
4911 if (isIndirect) {
4912 Result = Address(Builder.CreateLoad(Result, "aggr"),
4913 getContext().getTypeAlignInChars(Ty));
4914 }
4915
4916 return Result;
4917 }
4918
isStructReturnInRegABI(const llvm::Triple & Triple,const CodeGenOptions & Opts)4919 bool PPC32TargetCodeGenInfo::isStructReturnInRegABI(
4920 const llvm::Triple &Triple, const CodeGenOptions &Opts) {
4921 assert(Triple.isPPC32());
4922
4923 switch (Opts.getStructReturnConvention()) {
4924 case CodeGenOptions::SRCK_Default:
4925 break;
4926 case CodeGenOptions::SRCK_OnStack: // -maix-struct-return
4927 return false;
4928 case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return
4929 return true;
4930 }
4931
4932 if (Triple.isOSBinFormatELF() && !Triple.isOSLinux())
4933 return true;
4934
4935 return false;
4936 }
4937
4938 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const4939 PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4940 llvm::Value *Address) const {
4941 return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false,
4942 /*IsAIX*/ false);
4943 }
4944
4945 // PowerPC-64
4946
4947 namespace {
4948 /// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
4949 class PPC64_SVR4_ABIInfo : public SwiftABIInfo {
4950 public:
4951 enum ABIKind {
4952 ELFv1 = 0,
4953 ELFv2
4954 };
4955
4956 private:
4957 static const unsigned GPRBits = 64;
4958 ABIKind Kind;
4959 bool IsSoftFloatABI;
4960
4961 public:
PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes & CGT,ABIKind Kind,bool SoftFloatABI)4962 PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind,
4963 bool SoftFloatABI)
4964 : SwiftABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
4965
4966 bool isPromotableTypeForABI(QualType Ty) const;
4967 CharUnits getParamTypeAlignment(QualType Ty) const;
4968
4969 ABIArgInfo classifyReturnType(QualType RetTy) const;
4970 ABIArgInfo classifyArgumentType(QualType Ty) const;
4971
4972 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
4973 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
4974 uint64_t Members) const override;
4975
4976 // TODO: We can add more logic to computeInfo to improve performance.
4977 // Example: For aggregate arguments that fit in a register, we could
4978 // use getDirectInReg (as is done below for structs containing a single
4979 // floating-point value) to avoid pushing them to memory on function
4980 // entry. This would require changing the logic in PPCISelLowering
4981 // when lowering the parameters in the caller and args in the callee.
computeInfo(CGFunctionInfo & FI) const4982 void computeInfo(CGFunctionInfo &FI) const override {
4983 if (!getCXXABI().classifyReturnType(FI))
4984 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4985 for (auto &I : FI.arguments()) {
4986 // We rely on the default argument classification for the most part.
4987 // One exception: An aggregate containing a single floating-point
4988 // or vector item must be passed in a register if one is available.
4989 const Type *T = isSingleElementStruct(I.type, getContext());
4990 if (T) {
4991 const BuiltinType *BT = T->getAs<BuiltinType>();
4992 if ((T->isVectorType() && getContext().getTypeSize(T) == 128) ||
4993 (BT && BT->isFloatingPoint())) {
4994 QualType QT(T, 0);
4995 I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
4996 continue;
4997 }
4998 }
4999 I.info = classifyArgumentType(I.type);
5000 }
5001 }
5002
5003 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5004 QualType Ty) const override;
5005
shouldPassIndirectlyForSwift(ArrayRef<llvm::Type * > scalars,bool asReturnValue) const5006 bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
5007 bool asReturnValue) const override {
5008 return occupiesMoreThan(CGT, scalars, /*total*/ 4);
5009 }
5010
isSwiftErrorInRegister() const5011 bool isSwiftErrorInRegister() const override {
5012 return false;
5013 }
5014 };
5015
5016 class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
5017
5018 public:
PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes & CGT,PPC64_SVR4_ABIInfo::ABIKind Kind,bool SoftFloatABI)5019 PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
5020 PPC64_SVR4_ABIInfo::ABIKind Kind,
5021 bool SoftFloatABI)
5022 : TargetCodeGenInfo(
5023 std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {}
5024
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const5025 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5026 // This is recovered from gcc output.
5027 return 1; // r1 is the dedicated stack pointer
5028 }
5029
5030 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5031 llvm::Value *Address) const override;
5032 };
5033
5034 class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
5035 public:
PPC64TargetCodeGenInfo(CodeGenTypes & CGT)5036 PPC64TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
5037
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const5038 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5039 // This is recovered from gcc output.
5040 return 1; // r1 is the dedicated stack pointer
5041 }
5042
5043 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5044 llvm::Value *Address) const override;
5045 };
5046
5047 }
5048
5049 // Return true if the ABI requires Ty to be passed sign- or zero-
5050 // extended to 64 bits.
5051 bool
isPromotableTypeForABI(QualType Ty) const5052 PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
5053 // Treat an enum type as its underlying type.
5054 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5055 Ty = EnumTy->getDecl()->getIntegerType();
5056
5057 // Promotable integer types are required to be promoted by the ABI.
5058 if (isPromotableIntegerTypeForABI(Ty))
5059 return true;
5060
5061 // In addition to the usual promotable integer types, we also need to
5062 // extend all 32-bit types, since the ABI requires promotion to 64 bits.
5063 if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
5064 switch (BT->getKind()) {
5065 case BuiltinType::Int:
5066 case BuiltinType::UInt:
5067 return true;
5068 default:
5069 break;
5070 }
5071
5072 if (const auto *EIT = Ty->getAs<ExtIntType>())
5073 if (EIT->getNumBits() < 64)
5074 return true;
5075
5076 return false;
5077 }
5078
5079 /// isAlignedParamType - Determine whether a type requires 16-byte or
5080 /// higher alignment in the parameter area. Always returns at least 8.
getParamTypeAlignment(QualType Ty) const5081 CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
5082 // Complex types are passed just like their elements.
5083 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
5084 Ty = CTy->getElementType();
5085
5086 // Only vector types of size 16 bytes need alignment (larger types are
5087 // passed via reference, smaller types are not aligned).
5088 if (Ty->isVectorType()) {
5089 return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
5090 } else if (Ty->isRealFloatingType() &&
5091 &getContext().getFloatTypeSemantics(Ty) ==
5092 &llvm::APFloat::IEEEquad()) {
5093 // According to ABI document section 'Optional Save Areas': If extended
5094 // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
5095 // format are supported, map them to a single quadword, quadword aligned.
5096 return CharUnits::fromQuantity(16);
5097 }
5098
5099 // For single-element float/vector structs, we consider the whole type
5100 // to have the same alignment requirements as its single element.
5101 const Type *AlignAsType = nullptr;
5102 const Type *EltType = isSingleElementStruct(Ty, getContext());
5103 if (EltType) {
5104 const BuiltinType *BT = EltType->getAs<BuiltinType>();
5105 if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
5106 (BT && BT->isFloatingPoint()))
5107 AlignAsType = EltType;
5108 }
5109
5110 // Likewise for ELFv2 homogeneous aggregates.
5111 const Type *Base = nullptr;
5112 uint64_t Members = 0;
5113 if (!AlignAsType && Kind == ELFv2 &&
5114 isAggregateTypeForABI(Ty) && isHomogeneousAggregate(Ty, Base, Members))
5115 AlignAsType = Base;
5116
5117 // With special case aggregates, only vector base types need alignment.
5118 if (AlignAsType) {
5119 return CharUnits::fromQuantity(AlignAsType->isVectorType() ? 16 : 8);
5120 }
5121
5122 // Otherwise, we only need alignment for any aggregate type that
5123 // has an alignment requirement of >= 16 bytes.
5124 if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) {
5125 return CharUnits::fromQuantity(16);
5126 }
5127
5128 return CharUnits::fromQuantity(8);
5129 }
5130
5131 /// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous
5132 /// aggregate. Base is set to the base element type, and Members is set
5133 /// to the number of base elements.
isHomogeneousAggregate(QualType Ty,const Type * & Base,uint64_t & Members) const5134 bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
5135 uint64_t &Members) const {
5136 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
5137 uint64_t NElements = AT->getSize().getZExtValue();
5138 if (NElements == 0)
5139 return false;
5140 if (!isHomogeneousAggregate(AT->getElementType(), Base, Members))
5141 return false;
5142 Members *= NElements;
5143 } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
5144 const RecordDecl *RD = RT->getDecl();
5145 if (RD->hasFlexibleArrayMember())
5146 return false;
5147
5148 Members = 0;
5149
5150 // If this is a C++ record, check the properties of the record such as
5151 // bases and ABI specific restrictions
5152 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
5153 if (!getCXXABI().isPermittedToBeHomogeneousAggregate(CXXRD))
5154 return false;
5155
5156 for (const auto &I : CXXRD->bases()) {
5157 // Ignore empty records.
5158 if (isEmptyRecord(getContext(), I.getType(), true))
5159 continue;
5160
5161 uint64_t FldMembers;
5162 if (!isHomogeneousAggregate(I.getType(), Base, FldMembers))
5163 return false;
5164
5165 Members += FldMembers;
5166 }
5167 }
5168
5169 for (const auto *FD : RD->fields()) {
5170 // Ignore (non-zero arrays of) empty records.
5171 QualType FT = FD->getType();
5172 while (const ConstantArrayType *AT =
5173 getContext().getAsConstantArrayType(FT)) {
5174 if (AT->getSize().getZExtValue() == 0)
5175 return false;
5176 FT = AT->getElementType();
5177 }
5178 if (isEmptyRecord(getContext(), FT, true))
5179 continue;
5180
5181 // For compatibility with GCC, ignore empty bitfields in C++ mode.
5182 if (getContext().getLangOpts().CPlusPlus &&
5183 FD->isZeroLengthBitField(getContext()))
5184 continue;
5185
5186 uint64_t FldMembers;
5187 if (!isHomogeneousAggregate(FD->getType(), Base, FldMembers))
5188 return false;
5189
5190 Members = (RD->isUnion() ?
5191 std::max(Members, FldMembers) : Members + FldMembers);
5192 }
5193
5194 if (!Base)
5195 return false;
5196
5197 // Ensure there is no padding.
5198 if (getContext().getTypeSize(Base) * Members !=
5199 getContext().getTypeSize(Ty))
5200 return false;
5201 } else {
5202 Members = 1;
5203 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
5204 Members = 2;
5205 Ty = CT->getElementType();
5206 }
5207
5208 // Most ABIs only support float, double, and some vector type widths.
5209 if (!isHomogeneousAggregateBaseType(Ty))
5210 return false;
5211
5212 // The base type must be the same for all members. Types that
5213 // agree in both total size and mode (float vs. vector) are
5214 // treated as being equivalent here.
5215 const Type *TyPtr = Ty.getTypePtr();
5216 if (!Base) {
5217 Base = TyPtr;
5218 // If it's a non-power-of-2 vector, its size is already a power-of-2,
5219 // so make sure to widen it explicitly.
5220 if (const VectorType *VT = Base->getAs<VectorType>()) {
5221 QualType EltTy = VT->getElementType();
5222 unsigned NumElements =
5223 getContext().getTypeSize(VT) / getContext().getTypeSize(EltTy);
5224 Base = getContext()
5225 .getVectorType(EltTy, NumElements, VT->getVectorKind())
5226 .getTypePtr();
5227 }
5228 }
5229
5230 if (Base->isVectorType() != TyPtr->isVectorType() ||
5231 getContext().getTypeSize(Base) != getContext().getTypeSize(TyPtr))
5232 return false;
5233 }
5234 return Members > 0 && isHomogeneousAggregateSmallEnough(Base, Members);
5235 }
5236
isHomogeneousAggregateBaseType(QualType Ty) const5237 bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
5238 // Homogeneous aggregates for ELFv2 must have base types of float,
5239 // double, long double, or 128-bit vectors.
5240 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
5241 if (BT->getKind() == BuiltinType::Float ||
5242 BT->getKind() == BuiltinType::Double ||
5243 BT->getKind() == BuiltinType::LongDouble ||
5244 BT->getKind() == BuiltinType::Ibm128 ||
5245 (getContext().getTargetInfo().hasFloat128Type() &&
5246 (BT->getKind() == BuiltinType::Float128))) {
5247 if (IsSoftFloatABI)
5248 return false;
5249 return true;
5250 }
5251 }
5252 if (const VectorType *VT = Ty->getAs<VectorType>()) {
5253 if (getContext().getTypeSize(VT) == 128)
5254 return true;
5255 }
5256 return false;
5257 }
5258
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const5259 bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateSmallEnough(
5260 const Type *Base, uint64_t Members) const {
5261 // Vector and fp128 types require one register, other floating point types
5262 // require one or two registers depending on their size.
5263 uint32_t NumRegs =
5264 ((getContext().getTargetInfo().hasFloat128Type() &&
5265 Base->isFloat128Type()) ||
5266 Base->isVectorType()) ? 1
5267 : (getContext().getTypeSize(Base) + 63) / 64;
5268
5269 // Homogeneous Aggregates may occupy at most 8 registers.
5270 return Members * NumRegs <= 8;
5271 }
5272
5273 ABIArgInfo
classifyArgumentType(QualType Ty) const5274 PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const {
5275 Ty = useFirstFieldIfTransparentUnion(Ty);
5276
5277 if (Ty->isAnyComplexType())
5278 return ABIArgInfo::getDirect();
5279
5280 // Non-Altivec vector types are passed in GPRs (smaller than 16 bytes)
5281 // or via reference (larger than 16 bytes).
5282 if (Ty->isVectorType()) {
5283 uint64_t Size = getContext().getTypeSize(Ty);
5284 if (Size > 128)
5285 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5286 else if (Size < 128) {
5287 llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
5288 return ABIArgInfo::getDirect(CoerceTy);
5289 }
5290 }
5291
5292 if (const auto *EIT = Ty->getAs<ExtIntType>())
5293 if (EIT->getNumBits() > 128)
5294 return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
5295
5296 if (isAggregateTypeForABI(Ty)) {
5297 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
5298 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
5299
5300 uint64_t ABIAlign = getParamTypeAlignment(Ty).getQuantity();
5301 uint64_t TyAlign = getContext().getTypeAlignInChars(Ty).getQuantity();
5302
5303 // ELFv2 homogeneous aggregates are passed as array types.
5304 const Type *Base = nullptr;
5305 uint64_t Members = 0;
5306 if (Kind == ELFv2 &&
5307 isHomogeneousAggregate(Ty, Base, Members)) {
5308 llvm::Type *BaseTy = CGT.ConvertType(QualType(Base, 0));
5309 llvm::Type *CoerceTy = llvm::ArrayType::get(BaseTy, Members);
5310 return ABIArgInfo::getDirect(CoerceTy);
5311 }
5312
5313 // If an aggregate may end up fully in registers, we do not
5314 // use the ByVal method, but pass the aggregate as array.
5315 // This is usually beneficial since we avoid forcing the
5316 // back-end to store the argument to memory.
5317 uint64_t Bits = getContext().getTypeSize(Ty);
5318 if (Bits > 0 && Bits <= 8 * GPRBits) {
5319 llvm::Type *CoerceTy;
5320
5321 // Types up to 8 bytes are passed as integer type (which will be
5322 // properly aligned in the argument save area doubleword).
5323 if (Bits <= GPRBits)
5324 CoerceTy =
5325 llvm::IntegerType::get(getVMContext(), llvm::alignTo(Bits, 8));
5326 // Larger types are passed as arrays, with the base type selected
5327 // according to the required alignment in the save area.
5328 else {
5329 uint64_t RegBits = ABIAlign * 8;
5330 uint64_t NumRegs = llvm::alignTo(Bits, RegBits) / RegBits;
5331 llvm::Type *RegTy = llvm::IntegerType::get(getVMContext(), RegBits);
5332 CoerceTy = llvm::ArrayType::get(RegTy, NumRegs);
5333 }
5334
5335 return ABIArgInfo::getDirect(CoerceTy);
5336 }
5337
5338 // All other aggregates are passed ByVal.
5339 return ABIArgInfo::getIndirect(CharUnits::fromQuantity(ABIAlign),
5340 /*ByVal=*/true,
5341 /*Realign=*/TyAlign > ABIAlign);
5342 }
5343
5344 return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
5345 : ABIArgInfo::getDirect());
5346 }
5347
5348 ABIArgInfo
classifyReturnType(QualType RetTy) const5349 PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
5350 if (RetTy->isVoidType())
5351 return ABIArgInfo::getIgnore();
5352
5353 if (RetTy->isAnyComplexType())
5354 return ABIArgInfo::getDirect();
5355
5356 // Non-Altivec vector types are returned in GPRs (smaller than 16 bytes)
5357 // or via reference (larger than 16 bytes).
5358 if (RetTy->isVectorType()) {
5359 uint64_t Size = getContext().getTypeSize(RetTy);
5360 if (Size > 128)
5361 return getNaturalAlignIndirect(RetTy);
5362 else if (Size < 128) {
5363 llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
5364 return ABIArgInfo::getDirect(CoerceTy);
5365 }
5366 }
5367
5368 if (const auto *EIT = RetTy->getAs<ExtIntType>())
5369 if (EIT->getNumBits() > 128)
5370 return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
5371
5372 if (isAggregateTypeForABI(RetTy)) {
5373 // ELFv2 homogeneous aggregates are returned as array types.
5374 const Type *Base = nullptr;
5375 uint64_t Members = 0;
5376 if (Kind == ELFv2 &&
5377 isHomogeneousAggregate(RetTy, Base, Members)) {
5378 llvm::Type *BaseTy = CGT.ConvertType(QualType(Base, 0));
5379 llvm::Type *CoerceTy = llvm::ArrayType::get(BaseTy, Members);
5380 return ABIArgInfo::getDirect(CoerceTy);
5381 }
5382
5383 // ELFv2 small aggregates are returned in up to two registers.
5384 uint64_t Bits = getContext().getTypeSize(RetTy);
5385 if (Kind == ELFv2 && Bits <= 2 * GPRBits) {
5386 if (Bits == 0)
5387 return ABIArgInfo::getIgnore();
5388
5389 llvm::Type *CoerceTy;
5390 if (Bits > GPRBits) {
5391 CoerceTy = llvm::IntegerType::get(getVMContext(), GPRBits);
5392 CoerceTy = llvm::StructType::get(CoerceTy, CoerceTy);
5393 } else
5394 CoerceTy =
5395 llvm::IntegerType::get(getVMContext(), llvm::alignTo(Bits, 8));
5396 return ABIArgInfo::getDirect(CoerceTy);
5397 }
5398
5399 // All other aggregates are returned indirectly.
5400 return getNaturalAlignIndirect(RetTy);
5401 }
5402
5403 return (isPromotableTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
5404 : ABIArgInfo::getDirect());
5405 }
5406
5407 // Based on ARMABIInfo::EmitVAArg, adjusted for 64-bit machine.
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const5408 Address PPC64_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5409 QualType Ty) const {
5410 auto TypeInfo = getContext().getTypeInfoInChars(Ty);
5411 TypeInfo.Align = getParamTypeAlignment(Ty);
5412
5413 CharUnits SlotSize = CharUnits::fromQuantity(8);
5414
5415 // If we have a complex type and the base type is smaller than 8 bytes,
5416 // the ABI calls for the real and imaginary parts to be right-adjusted
5417 // in separate doublewords. However, Clang expects us to produce a
5418 // pointer to a structure with the two parts packed tightly. So generate
5419 // loads of the real and imaginary parts relative to the va_list pointer,
5420 // and store them to a temporary structure.
5421 if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
5422 CharUnits EltSize = TypeInfo.Width / 2;
5423 if (EltSize < SlotSize)
5424 return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
5425 }
5426
5427 // Otherwise, just use the general rule.
5428 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
5429 TypeInfo, SlotSize, /*AllowHigher*/ true);
5430 }
5431
5432 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const5433 PPC64_SVR4_TargetCodeGenInfo::initDwarfEHRegSizeTable(
5434 CodeGen::CodeGenFunction &CGF,
5435 llvm::Value *Address) const {
5436 return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ true,
5437 /*IsAIX*/ false);
5438 }
5439
5440 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const5441 PPC64TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5442 llvm::Value *Address) const {
5443 return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ true,
5444 /*IsAIX*/ false);
5445 }
5446
5447 //===----------------------------------------------------------------------===//
5448 // AArch64 ABI Implementation
5449 //===----------------------------------------------------------------------===//
5450
5451 namespace {
5452
5453 class AArch64ABIInfo : public SwiftABIInfo {
5454 public:
5455 enum ABIKind {
5456 AAPCS = 0,
5457 DarwinPCS,
5458 Win64
5459 };
5460
5461 private:
5462 ABIKind Kind;
5463
5464 public:
AArch64ABIInfo(CodeGenTypes & CGT,ABIKind Kind)5465 AArch64ABIInfo(CodeGenTypes &CGT, ABIKind Kind)
5466 : SwiftABIInfo(CGT), Kind(Kind) {}
5467
5468 private:
getABIKind() const5469 ABIKind getABIKind() const { return Kind; }
isDarwinPCS() const5470 bool isDarwinPCS() const { return Kind == DarwinPCS; }
5471
5472 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadic) const;
5473 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadic,
5474 unsigned CallingConvention) const;
5475 ABIArgInfo coerceIllegalVector(QualType Ty) const;
5476 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
5477 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
5478 uint64_t Members) const override;
5479
5480 bool isIllegalVectorType(QualType Ty) const;
5481
computeInfo(CGFunctionInfo & FI) const5482 void computeInfo(CGFunctionInfo &FI) const override {
5483 if (!::classifyReturnType(getCXXABI(), FI, *this))
5484 FI.getReturnInfo() =
5485 classifyReturnType(FI.getReturnType(), FI.isVariadic());
5486
5487 for (auto &it : FI.arguments())
5488 it.info = classifyArgumentType(it.type, FI.isVariadic(),
5489 FI.getCallingConvention());
5490 }
5491
5492 Address EmitDarwinVAArg(Address VAListAddr, QualType Ty,
5493 CodeGenFunction &CGF) const;
5494
5495 Address EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
5496 CodeGenFunction &CGF) const;
5497
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const5498 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5499 QualType Ty) const override {
5500 llvm::Type *BaseTy = CGF.ConvertType(Ty);
5501 if (isa<llvm::ScalableVectorType>(BaseTy))
5502 llvm::report_fatal_error("Passing SVE types to variadic functions is "
5503 "currently not supported");
5504
5505 return Kind == Win64 ? EmitMSVAArg(CGF, VAListAddr, Ty)
5506 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF)
5507 : EmitAAPCSVAArg(VAListAddr, Ty, CGF);
5508 }
5509
5510 Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
5511 QualType Ty) const override;
5512
shouldPassIndirectlyForSwift(ArrayRef<llvm::Type * > scalars,bool asReturnValue) const5513 bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
5514 bool asReturnValue) const override {
5515 return occupiesMoreThan(CGT, scalars, /*total*/ 4);
5516 }
isSwiftErrorInRegister() const5517 bool isSwiftErrorInRegister() const override {
5518 return true;
5519 }
5520
5521 bool isLegalVectorTypeForSwift(CharUnits totalSize, llvm::Type *eltTy,
5522 unsigned elts) const override;
5523
allowBFloatArgsAndRet() const5524 bool allowBFloatArgsAndRet() const override {
5525 return getTarget().hasBFloat16Type();
5526 }
5527 };
5528
5529 class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
5530 public:
AArch64TargetCodeGenInfo(CodeGenTypes & CGT,AArch64ABIInfo::ABIKind Kind)5531 AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIInfo::ABIKind Kind)
5532 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(CGT, Kind)) {}
5533
getARCRetainAutoreleasedReturnValueMarker() const5534 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
5535 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
5536 }
5537
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const5538 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5539 return 31;
5540 }
5541
doesReturnSlotInterfereWithArgs() const5542 bool doesReturnSlotInterfereWithArgs() const override { return false; }
5543
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const5544 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
5545 CodeGen::CodeGenModule &CGM) const override {
5546 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
5547 if (!FD)
5548 return;
5549
5550 const auto *TA = FD->getAttr<TargetAttr>();
5551 if (TA == nullptr)
5552 return;
5553
5554 ParsedTargetAttr Attr = TA->parse();
5555 if (Attr.BranchProtection.empty())
5556 return;
5557
5558 TargetInfo::BranchProtectionInfo BPI;
5559 StringRef Error;
5560 (void)CGM.getTarget().validateBranchProtection(Attr.BranchProtection,
5561 BPI, Error);
5562 assert(Error.empty());
5563
5564 auto *Fn = cast<llvm::Function>(GV);
5565 static const char *SignReturnAddrStr[] = {"none", "non-leaf", "all"};
5566 Fn->addFnAttr("sign-return-address", SignReturnAddrStr[static_cast<int>(BPI.SignReturnAddr)]);
5567
5568 if (BPI.SignReturnAddr != LangOptions::SignReturnAddressScopeKind::None) {
5569 Fn->addFnAttr("sign-return-address-key",
5570 BPI.SignKey == LangOptions::SignReturnAddressKeyKind::AKey
5571 ? "a_key"
5572 : "b_key");
5573 }
5574
5575 Fn->addFnAttr("branch-target-enforcement",
5576 BPI.BranchTargetEnforcement ? "true" : "false");
5577 }
5578
isScalarizableAsmOperand(CodeGen::CodeGenFunction & CGF,llvm::Type * Ty) const5579 bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
5580 llvm::Type *Ty) const override {
5581 if (CGF.getTarget().hasFeature("ls64")) {
5582 auto *ST = dyn_cast<llvm::StructType>(Ty);
5583 if (ST && ST->getNumElements() == 1) {
5584 auto *AT = dyn_cast<llvm::ArrayType>(ST->getElementType(0));
5585 if (AT && AT->getNumElements() == 8 &&
5586 AT->getElementType()->isIntegerTy(64))
5587 return true;
5588 }
5589 }
5590 return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);
5591 }
5592 };
5593
5594 class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
5595 public:
WindowsAArch64TargetCodeGenInfo(CodeGenTypes & CGT,AArch64ABIInfo::ABIKind K)5596 WindowsAArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIInfo::ABIKind K)
5597 : AArch64TargetCodeGenInfo(CGT, K) {}
5598
5599 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
5600 CodeGen::CodeGenModule &CGM) const override;
5601
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const5602 void getDependentLibraryOption(llvm::StringRef Lib,
5603 llvm::SmallString<24> &Opt) const override {
5604 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
5605 }
5606
getDetectMismatchOption(llvm::StringRef Name,llvm::StringRef Value,llvm::SmallString<32> & Opt) const5607 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
5608 llvm::SmallString<32> &Opt) const override {
5609 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
5610 }
5611 };
5612
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const5613 void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
5614 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
5615 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
5616 if (GV->isDeclaration())
5617 return;
5618 addStackProbeTargetAttributes(D, GV, CGM);
5619 }
5620 }
5621
coerceIllegalVector(QualType Ty) const5622 ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty) const {
5623 assert(Ty->isVectorType() && "expected vector type!");
5624
5625 const auto *VT = Ty->castAs<VectorType>();
5626 if (VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector) {
5627 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
5628 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
5629 BuiltinType::UChar &&
5630 "unexpected builtin type for SVE predicate!");
5631 return ABIArgInfo::getDirect(llvm::ScalableVectorType::get(
5632 llvm::Type::getInt1Ty(getVMContext()), 16));
5633 }
5634
5635 if (VT->getVectorKind() == VectorType::SveFixedLengthDataVector) {
5636 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
5637
5638 const auto *BT = VT->getElementType()->castAs<BuiltinType>();
5639 llvm::ScalableVectorType *ResType = nullptr;
5640 switch (BT->getKind()) {
5641 default:
5642 llvm_unreachable("unexpected builtin type for SVE vector!");
5643 case BuiltinType::SChar:
5644 case BuiltinType::UChar:
5645 ResType = llvm::ScalableVectorType::get(
5646 llvm::Type::getInt8Ty(getVMContext()), 16);
5647 break;
5648 case BuiltinType::Short:
5649 case BuiltinType::UShort:
5650 ResType = llvm::ScalableVectorType::get(
5651 llvm::Type::getInt16Ty(getVMContext()), 8);
5652 break;
5653 case BuiltinType::Int:
5654 case BuiltinType::UInt:
5655 ResType = llvm::ScalableVectorType::get(
5656 llvm::Type::getInt32Ty(getVMContext()), 4);
5657 break;
5658 case BuiltinType::Long:
5659 case BuiltinType::ULong:
5660 ResType = llvm::ScalableVectorType::get(
5661 llvm::Type::getInt64Ty(getVMContext()), 2);
5662 break;
5663 case BuiltinType::Half:
5664 ResType = llvm::ScalableVectorType::get(
5665 llvm::Type::getHalfTy(getVMContext()), 8);
5666 break;
5667 case BuiltinType::Float:
5668 ResType = llvm::ScalableVectorType::get(
5669 llvm::Type::getFloatTy(getVMContext()), 4);
5670 break;
5671 case BuiltinType::Double:
5672 ResType = llvm::ScalableVectorType::get(
5673 llvm::Type::getDoubleTy(getVMContext()), 2);
5674 break;
5675 case BuiltinType::BFloat16:
5676 ResType = llvm::ScalableVectorType::get(
5677 llvm::Type::getBFloatTy(getVMContext()), 8);
5678 break;
5679 }
5680 return ABIArgInfo::getDirect(ResType);
5681 }
5682
5683 uint64_t Size = getContext().getTypeSize(Ty);
5684 // Android promotes <2 x i8> to i16, not i32
5685 if (isAndroid() && (Size <= 16)) {
5686 llvm::Type *ResType = llvm::Type::getInt16Ty(getVMContext());
5687 return ABIArgInfo::getDirect(ResType);
5688 }
5689 if (Size <= 32) {
5690 llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext());
5691 return ABIArgInfo::getDirect(ResType);
5692 }
5693 if (Size == 64) {
5694 auto *ResType =
5695 llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2);
5696 return ABIArgInfo::getDirect(ResType);
5697 }
5698 if (Size == 128) {
5699 auto *ResType =
5700 llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4);
5701 return ABIArgInfo::getDirect(ResType);
5702 }
5703 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5704 }
5705
5706 ABIArgInfo
classifyArgumentType(QualType Ty,bool IsVariadic,unsigned CallingConvention) const5707 AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
5708 unsigned CallingConvention) const {
5709 Ty = useFirstFieldIfTransparentUnion(Ty);
5710
5711 // Handle illegal vector types here.
5712 if (isIllegalVectorType(Ty))
5713 return coerceIllegalVector(Ty);
5714
5715 if (!isAggregateTypeForABI(Ty)) {
5716 // Treat an enum type as its underlying type.
5717 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5718 Ty = EnumTy->getDecl()->getIntegerType();
5719
5720 if (const auto *EIT = Ty->getAs<ExtIntType>())
5721 if (EIT->getNumBits() > 128)
5722 return getNaturalAlignIndirect(Ty);
5723
5724 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
5725 ? ABIArgInfo::getExtend(Ty)
5726 : ABIArgInfo::getDirect());
5727 }
5728
5729 // Structures with either a non-trivial destructor or a non-trivial
5730 // copy constructor are always indirect.
5731 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
5732 return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
5733 CGCXXABI::RAA_DirectInMemory);
5734 }
5735
5736 // Empty records are always ignored on Darwin, but actually passed in C++ mode
5737 // elsewhere for GNU compatibility.
5738 uint64_t Size = getContext().getTypeSize(Ty);
5739 bool IsEmpty = isEmptyRecord(getContext(), Ty, true);
5740 if (IsEmpty || Size == 0) {
5741 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
5742 return ABIArgInfo::getIgnore();
5743
5744 // GNU C mode. The only argument that gets ignored is an empty one with size
5745 // 0.
5746 if (IsEmpty && Size == 0)
5747 return ABIArgInfo::getIgnore();
5748 return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
5749 }
5750
5751 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
5752 const Type *Base = nullptr;
5753 uint64_t Members = 0;
5754 bool IsWin64 = Kind == Win64 || CallingConvention == llvm::CallingConv::Win64;
5755 bool IsWinVariadic = IsWin64 && IsVariadic;
5756 // In variadic functions on Windows, all composite types are treated alike,
5757 // no special handling of HFAs/HVAs.
5758 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
5759 if (Kind != AArch64ABIInfo::AAPCS)
5760 return ABIArgInfo::getDirect(
5761 llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members));
5762
5763 // For alignment adjusted HFAs, cap the argument alignment to 16, leave it
5764 // default otherwise.
5765 unsigned Align =
5766 getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();
5767 unsigned BaseAlign = getContext().getTypeAlignInChars(Base).getQuantity();
5768 Align = (Align > BaseAlign && Align >= 16) ? 16 : 0;
5769 return ABIArgInfo::getDirect(
5770 llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members), 0,
5771 nullptr, true, Align);
5772 }
5773
5774 // Aggregates <= 16 bytes are passed directly in registers or on the stack.
5775 if (Size <= 128) {
5776 // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
5777 // same size and alignment.
5778 if (getTarget().isRenderScriptTarget()) {
5779 return coerceToIntArray(Ty, getContext(), getVMContext());
5780 }
5781 unsigned Alignment;
5782 if (Kind == AArch64ABIInfo::AAPCS) {
5783 Alignment = getContext().getTypeUnadjustedAlign(Ty);
5784 Alignment = Alignment < 128 ? 64 : 128;
5785 } else {
5786 Alignment = std::max(getContext().getTypeAlign(Ty),
5787 (unsigned)getTarget().getPointerWidth(0));
5788 }
5789 Size = llvm::alignTo(Size, Alignment);
5790
5791 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
5792 // For aggregates with 16-byte alignment, we use i128.
5793 llvm::Type *BaseTy = llvm::Type::getIntNTy(getVMContext(), Alignment);
5794 return ABIArgInfo::getDirect(
5795 Size == Alignment ? BaseTy
5796 : llvm::ArrayType::get(BaseTy, Size / Alignment));
5797 }
5798
5799 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5800 }
5801
classifyReturnType(QualType RetTy,bool IsVariadic) const5802 ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
5803 bool IsVariadic) const {
5804 if (RetTy->isVoidType())
5805 return ABIArgInfo::getIgnore();
5806
5807 if (const auto *VT = RetTy->getAs<VectorType>()) {
5808 if (VT->getVectorKind() == VectorType::SveFixedLengthDataVector ||
5809 VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
5810 return coerceIllegalVector(RetTy);
5811 }
5812
5813 // Large vector types should be returned via memory.
5814 if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128)
5815 return getNaturalAlignIndirect(RetTy);
5816
5817 if (!isAggregateTypeForABI(RetTy)) {
5818 // Treat an enum type as its underlying type.
5819 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
5820 RetTy = EnumTy->getDecl()->getIntegerType();
5821
5822 if (const auto *EIT = RetTy->getAs<ExtIntType>())
5823 if (EIT->getNumBits() > 128)
5824 return getNaturalAlignIndirect(RetTy);
5825
5826 return (isPromotableIntegerTypeForABI(RetTy) && isDarwinPCS()
5827 ? ABIArgInfo::getExtend(RetTy)
5828 : ABIArgInfo::getDirect());
5829 }
5830
5831 uint64_t Size = getContext().getTypeSize(RetTy);
5832 if (isEmptyRecord(getContext(), RetTy, true) || Size == 0)
5833 return ABIArgInfo::getIgnore();
5834
5835 const Type *Base = nullptr;
5836 uint64_t Members = 0;
5837 if (isHomogeneousAggregate(RetTy, Base, Members) &&
5838 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
5839 IsVariadic))
5840 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
5841 return ABIArgInfo::getDirect();
5842
5843 // Aggregates <= 16 bytes are returned directly in registers or on the stack.
5844 if (Size <= 128) {
5845 // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
5846 // same size and alignment.
5847 if (getTarget().isRenderScriptTarget()) {
5848 return coerceToIntArray(RetTy, getContext(), getVMContext());
5849 }
5850
5851 if (Size <= 64 && getDataLayout().isLittleEndian()) {
5852 // Composite types are returned in lower bits of a 64-bit register for LE,
5853 // and in higher bits for BE. However, integer types are always returned
5854 // in lower bits for both LE and BE, and they are not rounded up to
5855 // 64-bits. We can skip rounding up of composite types for LE, but not for
5856 // BE, otherwise composite types will be indistinguishable from integer
5857 // types.
5858 return ABIArgInfo::getDirect(
5859 llvm::IntegerType::get(getVMContext(), Size));
5860 }
5861
5862 unsigned Alignment = getContext().getTypeAlign(RetTy);
5863 Size = llvm::alignTo(Size, 64); // round up to multiple of 8 bytes
5864
5865 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
5866 // For aggregates with 16-byte alignment, we use i128.
5867 if (Alignment < 128 && Size == 128) {
5868 llvm::Type *BaseTy = llvm::Type::getInt64Ty(getVMContext());
5869 return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64));
5870 }
5871 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size));
5872 }
5873
5874 return getNaturalAlignIndirect(RetTy);
5875 }
5876
5877 /// isIllegalVectorType - check whether the vector type is legal for AArch64.
isIllegalVectorType(QualType Ty) const5878 bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
5879 if (const VectorType *VT = Ty->getAs<VectorType>()) {
5880 // Check whether VT is a fixed-length SVE vector. These types are
5881 // represented as scalable vectors in function args/return and must be
5882 // coerced from fixed vectors.
5883 if (VT->getVectorKind() == VectorType::SveFixedLengthDataVector ||
5884 VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
5885 return true;
5886
5887 // Check whether VT is legal.
5888 unsigned NumElements = VT->getNumElements();
5889 uint64_t Size = getContext().getTypeSize(VT);
5890 // NumElements should be power of 2.
5891 if (!llvm::isPowerOf2_32(NumElements))
5892 return true;
5893
5894 // arm64_32 has to be compatible with the ARM logic here, which allows huge
5895 // vectors for some reason.
5896 llvm::Triple Triple = getTarget().getTriple();
5897 if (Triple.getArch() == llvm::Triple::aarch64_32 &&
5898 Triple.isOSBinFormatMachO())
5899 return Size <= 32;
5900
5901 return Size != 64 && (Size != 128 || NumElements == 1);
5902 }
5903 return false;
5904 }
5905
isLegalVectorTypeForSwift(CharUnits totalSize,llvm::Type * eltTy,unsigned elts) const5906 bool AArch64ABIInfo::isLegalVectorTypeForSwift(CharUnits totalSize,
5907 llvm::Type *eltTy,
5908 unsigned elts) const {
5909 if (!llvm::isPowerOf2_32(elts))
5910 return false;
5911 if (totalSize.getQuantity() != 8 &&
5912 (totalSize.getQuantity() != 16 || elts == 1))
5913 return false;
5914 return true;
5915 }
5916
isHomogeneousAggregateBaseType(QualType Ty) const5917 bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
5918 // Homogeneous aggregates for AAPCS64 must have base types of a floating
5919 // point type or a short-vector type. This is the same as the 32-bit ABI,
5920 // but with the difference that any floating-point type is allowed,
5921 // including __fp16.
5922 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
5923 if (BT->isFloatingPoint())
5924 return true;
5925 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
5926 unsigned VecSize = getContext().getTypeSize(VT);
5927 if (VecSize == 64 || VecSize == 128)
5928 return true;
5929 }
5930 return false;
5931 }
5932
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const5933 bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
5934 uint64_t Members) const {
5935 return Members <= 4;
5936 }
5937
EmitAAPCSVAArg(Address VAListAddr,QualType Ty,CodeGenFunction & CGF) const5938 Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
5939 CodeGenFunction &CGF) const {
5940 ABIArgInfo AI = classifyArgumentType(Ty, /*IsVariadic=*/true,
5941 CGF.CurFnInfo->getCallingConvention());
5942 bool IsIndirect = AI.isIndirect();
5943
5944 llvm::Type *BaseTy = CGF.ConvertType(Ty);
5945 if (IsIndirect)
5946 BaseTy = llvm::PointerType::getUnqual(BaseTy);
5947 else if (AI.getCoerceToType())
5948 BaseTy = AI.getCoerceToType();
5949
5950 unsigned NumRegs = 1;
5951 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(BaseTy)) {
5952 BaseTy = ArrTy->getElementType();
5953 NumRegs = ArrTy->getNumElements();
5954 }
5955 bool IsFPR = BaseTy->isFloatingPointTy() || BaseTy->isVectorTy();
5956
5957 // The AArch64 va_list type and handling is specified in the Procedure Call
5958 // Standard, section B.4:
5959 //
5960 // struct {
5961 // void *__stack;
5962 // void *__gr_top;
5963 // void *__vr_top;
5964 // int __gr_offs;
5965 // int __vr_offs;
5966 // };
5967
5968 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg");
5969 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
5970 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");
5971 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
5972
5973 CharUnits TySize = getContext().getTypeSizeInChars(Ty);
5974 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(Ty);
5975
5976 Address reg_offs_p = Address::invalid();
5977 llvm::Value *reg_offs = nullptr;
5978 int reg_top_index;
5979 int RegSize = IsIndirect ? 8 : TySize.getQuantity();
5980 if (!IsFPR) {
5981 // 3 is the field number of __gr_offs
5982 reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p");
5983 reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs");
5984 reg_top_index = 1; // field number for __gr_top
5985 RegSize = llvm::alignTo(RegSize, 8);
5986 } else {
5987 // 4 is the field number of __vr_offs.
5988 reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p");
5989 reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs");
5990 reg_top_index = 2; // field number for __vr_top
5991 RegSize = 16 * NumRegs;
5992 }
5993
5994 //=======================================
5995 // Find out where argument was passed
5996 //=======================================
5997
5998 // If reg_offs >= 0 we're already using the stack for this type of
5999 // argument. We don't want to keep updating reg_offs (in case it overflows,
6000 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
6001 // whatever they get).
6002 llvm::Value *UsingStack = nullptr;
6003 UsingStack = CGF.Builder.CreateICmpSGE(
6004 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, 0));
6005
6006 CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, MaybeRegBlock);
6007
6008 // Otherwise, at least some kind of argument could go in these registers, the
6009 // question is whether this particular type is too big.
6010 CGF.EmitBlock(MaybeRegBlock);
6011
6012 // Integer arguments may need to correct register alignment (for example a
6013 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
6014 // align __gr_offs to calculate the potential address.
6015 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
6016 int Align = TyAlign.getQuantity();
6017
6018 reg_offs = CGF.Builder.CreateAdd(
6019 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, Align - 1),
6020 "align_regoffs");
6021 reg_offs = CGF.Builder.CreateAnd(
6022 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, -Align),
6023 "aligned_regoffs");
6024 }
6025
6026 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
6027 // The fact that this is done unconditionally reflects the fact that
6028 // allocating an argument to the stack also uses up all the remaining
6029 // registers of the appropriate kind.
6030 llvm::Value *NewOffset = nullptr;
6031 NewOffset = CGF.Builder.CreateAdd(
6032 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, RegSize), "new_reg_offs");
6033 CGF.Builder.CreateStore(NewOffset, reg_offs_p);
6034
6035 // Now we're in a position to decide whether this argument really was in
6036 // registers or not.
6037 llvm::Value *InRegs = nullptr;
6038 InRegs = CGF.Builder.CreateICmpSLE(
6039 NewOffset, llvm::ConstantInt::get(CGF.Int32Ty, 0), "inreg");
6040
6041 CGF.Builder.CreateCondBr(InRegs, InRegBlock, OnStackBlock);
6042
6043 //=======================================
6044 // Argument was in registers
6045 //=======================================
6046
6047 // Now we emit the code for if the argument was originally passed in
6048 // registers. First start the appropriate block:
6049 CGF.EmitBlock(InRegBlock);
6050
6051 llvm::Value *reg_top = nullptr;
6052 Address reg_top_p =
6053 CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p");
6054 reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top");
6055 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, reg_top, reg_offs),
6056 CharUnits::fromQuantity(IsFPR ? 16 : 8));
6057 Address RegAddr = Address::invalid();
6058 llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty);
6059
6060 if (IsIndirect) {
6061 // If it's been passed indirectly (actually a struct), whatever we find from
6062 // stored registers or on the stack will actually be a struct **.
6063 MemTy = llvm::PointerType::getUnqual(MemTy);
6064 }
6065
6066 const Type *Base = nullptr;
6067 uint64_t NumMembers = 0;
6068 bool IsHFA = isHomogeneousAggregate(Ty, Base, NumMembers);
6069 if (IsHFA && NumMembers > 1) {
6070 // Homogeneous aggregates passed in registers will have their elements split
6071 // and stored 16-bytes apart regardless of size (they're notionally in qN,
6072 // qN+1, ...). We reload and store into a temporary local variable
6073 // contiguously.
6074 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
6075 auto BaseTyInfo = getContext().getTypeInfoInChars(QualType(Base, 0));
6076 llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0));
6077 llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers);
6078 Address Tmp = CGF.CreateTempAlloca(HFATy,
6079 std::max(TyAlign, BaseTyInfo.Align));
6080
6081 // On big-endian platforms, the value will be right-aligned in its slot.
6082 int Offset = 0;
6083 if (CGF.CGM.getDataLayout().isBigEndian() &&
6084 BaseTyInfo.Width.getQuantity() < 16)
6085 Offset = 16 - BaseTyInfo.Width.getQuantity();
6086
6087 for (unsigned i = 0; i < NumMembers; ++i) {
6088 CharUnits BaseOffset = CharUnits::fromQuantity(16 * i + Offset);
6089 Address LoadAddr =
6090 CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, BaseOffset);
6091 LoadAddr = CGF.Builder.CreateElementBitCast(LoadAddr, BaseTy);
6092
6093 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Tmp, i);
6094
6095 llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr);
6096 CGF.Builder.CreateStore(Elem, StoreAddr);
6097 }
6098
6099 RegAddr = CGF.Builder.CreateElementBitCast(Tmp, MemTy);
6100 } else {
6101 // Otherwise the object is contiguous in memory.
6102
6103 // It might be right-aligned in its slot.
6104 CharUnits SlotSize = BaseAddr.getAlignment();
6105 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
6106 (IsHFA || !isAggregateTypeForABI(Ty)) &&
6107 TySize < SlotSize) {
6108 CharUnits Offset = SlotSize - TySize;
6109 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, Offset);
6110 }
6111
6112 RegAddr = CGF.Builder.CreateElementBitCast(BaseAddr, MemTy);
6113 }
6114
6115 CGF.EmitBranch(ContBlock);
6116
6117 //=======================================
6118 // Argument was on the stack
6119 //=======================================
6120 CGF.EmitBlock(OnStackBlock);
6121
6122 Address stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "stack_p");
6123 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(stack_p, "stack");
6124
6125 // Again, stack arguments may need realignment. In this case both integer and
6126 // floating-point ones might be affected.
6127 if (!IsIndirect && TyAlign.getQuantity() > 8) {
6128 int Align = TyAlign.getQuantity();
6129
6130 OnStackPtr = CGF.Builder.CreatePtrToInt(OnStackPtr, CGF.Int64Ty);
6131
6132 OnStackPtr = CGF.Builder.CreateAdd(
6133 OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, Align - 1),
6134 "align_stack");
6135 OnStackPtr = CGF.Builder.CreateAnd(
6136 OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, -Align),
6137 "align_stack");
6138
6139 OnStackPtr = CGF.Builder.CreateIntToPtr(OnStackPtr, CGF.Int8PtrTy);
6140 }
6141 Address OnStackAddr(OnStackPtr,
6142 std::max(CharUnits::fromQuantity(8), TyAlign));
6143
6144 // All stack slots are multiples of 8 bytes.
6145 CharUnits StackSlotSize = CharUnits::fromQuantity(8);
6146 CharUnits StackSize;
6147 if (IsIndirect)
6148 StackSize = StackSlotSize;
6149 else
6150 StackSize = TySize.alignTo(StackSlotSize);
6151
6152 llvm::Value *StackSizeC = CGF.Builder.getSize(StackSize);
6153 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
6154 CGF.Int8Ty, OnStackPtr, StackSizeC, "new_stack");
6155
6156 // Write the new value of __stack for the next call to va_arg
6157 CGF.Builder.CreateStore(NewStack, stack_p);
6158
6159 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(Ty) &&
6160 TySize < StackSlotSize) {
6161 CharUnits Offset = StackSlotSize - TySize;
6162 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(OnStackAddr, Offset);
6163 }
6164
6165 OnStackAddr = CGF.Builder.CreateElementBitCast(OnStackAddr, MemTy);
6166
6167 CGF.EmitBranch(ContBlock);
6168
6169 //=======================================
6170 // Tidy up
6171 //=======================================
6172 CGF.EmitBlock(ContBlock);
6173
6174 Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock,
6175 OnStackAddr, OnStackBlock, "vaargs.addr");
6176
6177 if (IsIndirect)
6178 return Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"),
6179 TyAlign);
6180
6181 return ResAddr;
6182 }
6183
EmitDarwinVAArg(Address VAListAddr,QualType Ty,CodeGenFunction & CGF) const6184 Address AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
6185 CodeGenFunction &CGF) const {
6186 // The backend's lowering doesn't support va_arg for aggregates or
6187 // illegal vector types. Lower VAArg here for these cases and use
6188 // the LLVM va_arg instruction for everything else.
6189 if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty))
6190 return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
6191
6192 uint64_t PointerSize = getTarget().getPointerWidth(0) / 8;
6193 CharUnits SlotSize = CharUnits::fromQuantity(PointerSize);
6194
6195 // Empty records are ignored for parameter passing purposes.
6196 if (isEmptyRecord(getContext(), Ty, true)) {
6197 Address Addr(CGF.Builder.CreateLoad(VAListAddr, "ap.cur"), SlotSize);
6198 Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
6199 return Addr;
6200 }
6201
6202 // The size of the actual thing passed, which might end up just
6203 // being a pointer for indirect types.
6204 auto TyInfo = getContext().getTypeInfoInChars(Ty);
6205
6206 // Arguments bigger than 16 bytes which aren't homogeneous
6207 // aggregates should be passed indirectly.
6208 bool IsIndirect = false;
6209 if (TyInfo.Width.getQuantity() > 16) {
6210 const Type *Base = nullptr;
6211 uint64_t Members = 0;
6212 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
6213 }
6214
6215 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
6216 TyInfo, SlotSize, /*AllowHigherAlign*/ true);
6217 }
6218
EmitMSVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const6219 Address AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
6220 QualType Ty) const {
6221 bool IsIndirect = false;
6222
6223 // Composites larger than 16 bytes are passed by reference.
6224 if (isAggregateTypeForABI(Ty) && getContext().getTypeSize(Ty) > 128)
6225 IsIndirect = true;
6226
6227 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
6228 CGF.getContext().getTypeInfoInChars(Ty),
6229 CharUnits::fromQuantity(8),
6230 /*allowHigherAlign*/ false);
6231 }
6232
6233 //===----------------------------------------------------------------------===//
6234 // ARM ABI Implementation
6235 //===----------------------------------------------------------------------===//
6236
6237 namespace {
6238
6239 class ARMABIInfo : public SwiftABIInfo {
6240 public:
6241 enum ABIKind {
6242 APCS = 0,
6243 AAPCS = 1,
6244 AAPCS_VFP = 2,
6245 AAPCS16_VFP = 3,
6246 };
6247
6248 private:
6249 ABIKind Kind;
6250 bool IsFloatABISoftFP;
6251
6252 public:
ARMABIInfo(CodeGenTypes & CGT,ABIKind _Kind)6253 ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind)
6254 : SwiftABIInfo(CGT), Kind(_Kind) {
6255 setCCs();
6256 IsFloatABISoftFP = CGT.getCodeGenOpts().FloatABI == "softfp" ||
6257 CGT.getCodeGenOpts().FloatABI == ""; // default
6258 }
6259
isEABI() const6260 bool isEABI() const {
6261 switch (getTarget().getTriple().getEnvironment()) {
6262 case llvm::Triple::Android:
6263 case llvm::Triple::EABI:
6264 case llvm::Triple::EABIHF:
6265 case llvm::Triple::GNUEABI:
6266 case llvm::Triple::GNUEABIHF:
6267 case llvm::Triple::MuslEABI:
6268 case llvm::Triple::MuslEABIHF:
6269 return true;
6270 default:
6271 return false;
6272 }
6273 }
6274
isEABIHF() const6275 bool isEABIHF() const {
6276 switch (getTarget().getTriple().getEnvironment()) {
6277 case llvm::Triple::EABIHF:
6278 case llvm::Triple::GNUEABIHF:
6279 case llvm::Triple::MuslEABIHF:
6280 return true;
6281 default:
6282 return false;
6283 }
6284 }
6285
getABIKind() const6286 ABIKind getABIKind() const { return Kind; }
6287
allowBFloatArgsAndRet() const6288 bool allowBFloatArgsAndRet() const override {
6289 return !IsFloatABISoftFP && getTarget().hasBFloat16Type();
6290 }
6291
6292 private:
6293 ABIArgInfo classifyReturnType(QualType RetTy, bool isVariadic,
6294 unsigned functionCallConv) const;
6295 ABIArgInfo classifyArgumentType(QualType RetTy, bool isVariadic,
6296 unsigned functionCallConv) const;
6297 ABIArgInfo classifyHomogeneousAggregate(QualType Ty, const Type *Base,
6298 uint64_t Members) const;
6299 ABIArgInfo coerceIllegalVector(QualType Ty) const;
6300 bool isIllegalVectorType(QualType Ty) const;
6301 bool containsAnyFP16Vectors(QualType Ty) const;
6302
6303 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
6304 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
6305 uint64_t Members) const override;
6306
6307 bool isEffectivelyAAPCS_VFP(unsigned callConvention, bool acceptHalf) const;
6308
6309 void computeInfo(CGFunctionInfo &FI) const override;
6310
6311 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
6312 QualType Ty) const override;
6313
6314 llvm::CallingConv::ID getLLVMDefaultCC() const;
6315 llvm::CallingConv::ID getABIDefaultCC() const;
6316 void setCCs();
6317
shouldPassIndirectlyForSwift(ArrayRef<llvm::Type * > scalars,bool asReturnValue) const6318 bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
6319 bool asReturnValue) const override {
6320 return occupiesMoreThan(CGT, scalars, /*total*/ 4);
6321 }
isSwiftErrorInRegister() const6322 bool isSwiftErrorInRegister() const override {
6323 return true;
6324 }
6325 bool isLegalVectorTypeForSwift(CharUnits totalSize, llvm::Type *eltTy,
6326 unsigned elts) const override;
6327 };
6328
6329 class ARMTargetCodeGenInfo : public TargetCodeGenInfo {
6330 public:
ARMTargetCodeGenInfo(CodeGenTypes & CGT,ARMABIInfo::ABIKind K)6331 ARMTargetCodeGenInfo(CodeGenTypes &CGT, ARMABIInfo::ABIKind K)
6332 : TargetCodeGenInfo(std::make_unique<ARMABIInfo>(CGT, K)) {}
6333
getABIInfo() const6334 const ARMABIInfo &getABIInfo() const {
6335 return static_cast<const ARMABIInfo&>(TargetCodeGenInfo::getABIInfo());
6336 }
6337
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const6338 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
6339 return 13;
6340 }
6341
getARCRetainAutoreleasedReturnValueMarker() const6342 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
6343 return "mov\tr7, r7\t\t// marker for objc_retainAutoreleaseReturnValue";
6344 }
6345
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const6346 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
6347 llvm::Value *Address) const override {
6348 llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
6349
6350 // 0-15 are the 16 integer registers.
6351 AssignToArrayRange(CGF.Builder, Address, Four8, 0, 15);
6352 return false;
6353 }
6354
getSizeOfUnwindException() const6355 unsigned getSizeOfUnwindException() const override {
6356 if (getABIInfo().isEABI()) return 88;
6357 return TargetCodeGenInfo::getSizeOfUnwindException();
6358 }
6359
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const6360 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
6361 CodeGen::CodeGenModule &CGM) const override {
6362 if (GV->isDeclaration())
6363 return;
6364 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
6365 if (!FD)
6366 return;
6367
6368 const ARMInterruptAttr *Attr = FD->getAttr<ARMInterruptAttr>();
6369 if (!Attr)
6370 return;
6371
6372 const char *Kind;
6373 switch (Attr->getInterrupt()) {
6374 case ARMInterruptAttr::Generic: Kind = ""; break;
6375 case ARMInterruptAttr::IRQ: Kind = "IRQ"; break;
6376 case ARMInterruptAttr::FIQ: Kind = "FIQ"; break;
6377 case ARMInterruptAttr::SWI: Kind = "SWI"; break;
6378 case ARMInterruptAttr::ABORT: Kind = "ABORT"; break;
6379 case ARMInterruptAttr::UNDEF: Kind = "UNDEF"; break;
6380 }
6381
6382 llvm::Function *Fn = cast<llvm::Function>(GV);
6383
6384 Fn->addFnAttr("interrupt", Kind);
6385
6386 ARMABIInfo::ABIKind ABI = cast<ARMABIInfo>(getABIInfo()).getABIKind();
6387 if (ABI == ARMABIInfo::APCS)
6388 return;
6389
6390 // AAPCS guarantees that sp will be 8-byte aligned on any public interface,
6391 // however this is not necessarily true on taking any interrupt. Instruct
6392 // the backend to perform a realignment as part of the function prologue.
6393 llvm::AttrBuilder B;
6394 B.addStackAlignmentAttr(8);
6395 Fn->addFnAttrs(B);
6396 }
6397 };
6398
6399 class WindowsARMTargetCodeGenInfo : public ARMTargetCodeGenInfo {
6400 public:
WindowsARMTargetCodeGenInfo(CodeGenTypes & CGT,ARMABIInfo::ABIKind K)6401 WindowsARMTargetCodeGenInfo(CodeGenTypes &CGT, ARMABIInfo::ABIKind K)
6402 : ARMTargetCodeGenInfo(CGT, K) {}
6403
6404 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
6405 CodeGen::CodeGenModule &CGM) const override;
6406
getDependentLibraryOption(llvm::StringRef Lib,llvm::SmallString<24> & Opt) const6407 void getDependentLibraryOption(llvm::StringRef Lib,
6408 llvm::SmallString<24> &Opt) const override {
6409 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
6410 }
6411
getDetectMismatchOption(llvm::StringRef Name,llvm::StringRef Value,llvm::SmallString<32> & Opt) const6412 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
6413 llvm::SmallString<32> &Opt) const override {
6414 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
6415 }
6416 };
6417
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const6418 void WindowsARMTargetCodeGenInfo::setTargetAttributes(
6419 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
6420 ARMTargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
6421 if (GV->isDeclaration())
6422 return;
6423 addStackProbeTargetAttributes(D, GV, CGM);
6424 }
6425 }
6426
computeInfo(CGFunctionInfo & FI) const6427 void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const {
6428 if (!::classifyReturnType(getCXXABI(), FI, *this))
6429 FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), FI.isVariadic(),
6430 FI.getCallingConvention());
6431
6432 for (auto &I : FI.arguments())
6433 I.info = classifyArgumentType(I.type, FI.isVariadic(),
6434 FI.getCallingConvention());
6435
6436
6437 // Always honor user-specified calling convention.
6438 if (FI.getCallingConvention() != llvm::CallingConv::C)
6439 return;
6440
6441 llvm::CallingConv::ID cc = getRuntimeCC();
6442 if (cc != llvm::CallingConv::C)
6443 FI.setEffectiveCallingConvention(cc);
6444 }
6445
6446 /// Return the default calling convention that LLVM will use.
getLLVMDefaultCC() const6447 llvm::CallingConv::ID ARMABIInfo::getLLVMDefaultCC() const {
6448 // The default calling convention that LLVM will infer.
6449 if (isEABIHF() || getTarget().getTriple().isWatchABI())
6450 return llvm::CallingConv::ARM_AAPCS_VFP;
6451 else if (isEABI())
6452 return llvm::CallingConv::ARM_AAPCS;
6453 else
6454 return llvm::CallingConv::ARM_APCS;
6455 }
6456
6457 /// Return the calling convention that our ABI would like us to use
6458 /// as the C calling convention.
getABIDefaultCC() const6459 llvm::CallingConv::ID ARMABIInfo::getABIDefaultCC() const {
6460 switch (getABIKind()) {
6461 case APCS: return llvm::CallingConv::ARM_APCS;
6462 case AAPCS: return llvm::CallingConv::ARM_AAPCS;
6463 case AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
6464 case AAPCS16_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
6465 }
6466 llvm_unreachable("bad ABI kind");
6467 }
6468
setCCs()6469 void ARMABIInfo::setCCs() {
6470 assert(getRuntimeCC() == llvm::CallingConv::C);
6471
6472 // Don't muddy up the IR with a ton of explicit annotations if
6473 // they'd just match what LLVM will infer from the triple.
6474 llvm::CallingConv::ID abiCC = getABIDefaultCC();
6475 if (abiCC != getLLVMDefaultCC())
6476 RuntimeCC = abiCC;
6477 }
6478
coerceIllegalVector(QualType Ty) const6479 ABIArgInfo ARMABIInfo::coerceIllegalVector(QualType Ty) const {
6480 uint64_t Size = getContext().getTypeSize(Ty);
6481 if (Size <= 32) {
6482 llvm::Type *ResType =
6483 llvm::Type::getInt32Ty(getVMContext());
6484 return ABIArgInfo::getDirect(ResType);
6485 }
6486 if (Size == 64 || Size == 128) {
6487 auto *ResType = llvm::FixedVectorType::get(
6488 llvm::Type::getInt32Ty(getVMContext()), Size / 32);
6489 return ABIArgInfo::getDirect(ResType);
6490 }
6491 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
6492 }
6493
classifyHomogeneousAggregate(QualType Ty,const Type * Base,uint64_t Members) const6494 ABIArgInfo ARMABIInfo::classifyHomogeneousAggregate(QualType Ty,
6495 const Type *Base,
6496 uint64_t Members) const {
6497 assert(Base && "Base class should be set for homogeneous aggregate");
6498 // Base can be a floating-point or a vector.
6499 if (const VectorType *VT = Base->getAs<VectorType>()) {
6500 // FP16 vectors should be converted to integer vectors
6501 if (!getTarget().hasLegalHalfType() && containsAnyFP16Vectors(Ty)) {
6502 uint64_t Size = getContext().getTypeSize(VT);
6503 auto *NewVecTy = llvm::FixedVectorType::get(
6504 llvm::Type::getInt32Ty(getVMContext()), Size / 32);
6505 llvm::Type *Ty = llvm::ArrayType::get(NewVecTy, Members);
6506 return ABIArgInfo::getDirect(Ty, 0, nullptr, false);
6507 }
6508 }
6509 unsigned Align = 0;
6510 if (getABIKind() == ARMABIInfo::AAPCS ||
6511 getABIKind() == ARMABIInfo::AAPCS_VFP) {
6512 // For alignment adjusted HFAs, cap the argument alignment to 8, leave it
6513 // default otherwise.
6514 Align = getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();
6515 unsigned BaseAlign = getContext().getTypeAlignInChars(Base).getQuantity();
6516 Align = (Align > BaseAlign && Align >= 8) ? 8 : 0;
6517 }
6518 return ABIArgInfo::getDirect(nullptr, 0, nullptr, false, Align);
6519 }
6520
classifyArgumentType(QualType Ty,bool isVariadic,unsigned functionCallConv) const6521 ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic,
6522 unsigned functionCallConv) const {
6523 // 6.1.2.1 The following argument types are VFP CPRCs:
6524 // A single-precision floating-point type (including promoted
6525 // half-precision types); A double-precision floating-point type;
6526 // A 64-bit or 128-bit containerized vector type; Homogeneous Aggregate
6527 // with a Base Type of a single- or double-precision floating-point type,
6528 // 64-bit containerized vectors or 128-bit containerized vectors with one
6529 // to four Elements.
6530 // Variadic functions should always marshal to the base standard.
6531 bool IsAAPCS_VFP =
6532 !isVariadic && isEffectivelyAAPCS_VFP(functionCallConv, /* AAPCS16 */ false);
6533
6534 Ty = useFirstFieldIfTransparentUnion(Ty);
6535
6536 // Handle illegal vector types here.
6537 if (isIllegalVectorType(Ty))
6538 return coerceIllegalVector(Ty);
6539
6540 if (!isAggregateTypeForABI(Ty)) {
6541 // Treat an enum type as its underlying type.
6542 if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
6543 Ty = EnumTy->getDecl()->getIntegerType();
6544 }
6545
6546 if (const auto *EIT = Ty->getAs<ExtIntType>())
6547 if (EIT->getNumBits() > 64)
6548 return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
6549
6550 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
6551 : ABIArgInfo::getDirect());
6552 }
6553
6554 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
6555 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
6556 }
6557
6558 // Ignore empty records.
6559 if (isEmptyRecord(getContext(), Ty, true))
6560 return ABIArgInfo::getIgnore();
6561
6562 if (IsAAPCS_VFP) {
6563 // Homogeneous Aggregates need to be expanded when we can fit the aggregate
6564 // into VFP registers.
6565 const Type *Base = nullptr;
6566 uint64_t Members = 0;
6567 if (isHomogeneousAggregate(Ty, Base, Members))
6568 return classifyHomogeneousAggregate(Ty, Base, Members);
6569 } else if (getABIKind() == ARMABIInfo::AAPCS16_VFP) {
6570 // WatchOS does have homogeneous aggregates. Note that we intentionally use
6571 // this convention even for a variadic function: the backend will use GPRs
6572 // if needed.
6573 const Type *Base = nullptr;
6574 uint64_t Members = 0;
6575 if (isHomogeneousAggregate(Ty, Base, Members)) {
6576 assert(Base && Members <= 4 && "unexpected homogeneous aggregate");
6577 llvm::Type *Ty =
6578 llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members);
6579 return ABIArgInfo::getDirect(Ty, 0, nullptr, false);
6580 }
6581 }
6582
6583 if (getABIKind() == ARMABIInfo::AAPCS16_VFP &&
6584 getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(16)) {
6585 // WatchOS is adopting the 64-bit AAPCS rule on composite types: if they're
6586 // bigger than 128-bits, they get placed in space allocated by the caller,
6587 // and a pointer is passed.
6588 return ABIArgInfo::getIndirect(
6589 CharUnits::fromQuantity(getContext().getTypeAlign(Ty) / 8), false);
6590 }
6591
6592 // Support byval for ARM.
6593 // The ABI alignment for APCS is 4-byte and for AAPCS at least 4-byte and at
6594 // most 8-byte. We realign the indirect argument if type alignment is bigger
6595 // than ABI alignment.
6596 uint64_t ABIAlign = 4;
6597 uint64_t TyAlign;
6598 if (getABIKind() == ARMABIInfo::AAPCS_VFP ||
6599 getABIKind() == ARMABIInfo::AAPCS) {
6600 TyAlign = getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();
6601 ABIAlign = std::min(std::max(TyAlign, (uint64_t)4), (uint64_t)8);
6602 } else {
6603 TyAlign = getContext().getTypeAlignInChars(Ty).getQuantity();
6604 }
6605 if (getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(64)) {
6606 assert(getABIKind() != ARMABIInfo::AAPCS16_VFP && "unexpected byval");
6607 return ABIArgInfo::getIndirect(CharUnits::fromQuantity(ABIAlign),
6608 /*ByVal=*/true,
6609 /*Realign=*/TyAlign > ABIAlign);
6610 }
6611
6612 // On RenderScript, coerce Aggregates <= 64 bytes to an integer array of
6613 // same size and alignment.
6614 if (getTarget().isRenderScriptTarget()) {
6615 return coerceToIntArray(Ty, getContext(), getVMContext());
6616 }
6617
6618 // Otherwise, pass by coercing to a structure of the appropriate size.
6619 llvm::Type* ElemTy;
6620 unsigned SizeRegs;
6621 // FIXME: Try to match the types of the arguments more accurately where
6622 // we can.
6623 if (TyAlign <= 4) {
6624 ElemTy = llvm::Type::getInt32Ty(getVMContext());
6625 SizeRegs = (getContext().getTypeSize(Ty) + 31) / 32;
6626 } else {
6627 ElemTy = llvm::Type::getInt64Ty(getVMContext());
6628 SizeRegs = (getContext().getTypeSize(Ty) + 63) / 64;
6629 }
6630
6631 return ABIArgInfo::getDirect(llvm::ArrayType::get(ElemTy, SizeRegs));
6632 }
6633
isIntegerLikeType(QualType Ty,ASTContext & Context,llvm::LLVMContext & VMContext)6634 static bool isIntegerLikeType(QualType Ty, ASTContext &Context,
6635 llvm::LLVMContext &VMContext) {
6636 // APCS, C Language Calling Conventions, Non-Simple Return Values: A structure
6637 // is called integer-like if its size is less than or equal to one word, and
6638 // the offset of each of its addressable sub-fields is zero.
6639
6640 uint64_t Size = Context.getTypeSize(Ty);
6641
6642 // Check that the type fits in a word.
6643 if (Size > 32)
6644 return false;
6645
6646 // FIXME: Handle vector types!
6647 if (Ty->isVectorType())
6648 return false;
6649
6650 // Float types are never treated as "integer like".
6651 if (Ty->isRealFloatingType())
6652 return false;
6653
6654 // If this is a builtin or pointer type then it is ok.
6655 if (Ty->getAs<BuiltinType>() || Ty->isPointerType())
6656 return true;
6657
6658 // Small complex integer types are "integer like".
6659 if (const ComplexType *CT = Ty->getAs<ComplexType>())
6660 return isIntegerLikeType(CT->getElementType(), Context, VMContext);
6661
6662 // Single element and zero sized arrays should be allowed, by the definition
6663 // above, but they are not.
6664
6665 // Otherwise, it must be a record type.
6666 const RecordType *RT = Ty->getAs<RecordType>();
6667 if (!RT) return false;
6668
6669 // Ignore records with flexible arrays.
6670 const RecordDecl *RD = RT->getDecl();
6671 if (RD->hasFlexibleArrayMember())
6672 return false;
6673
6674 // Check that all sub-fields are at offset 0, and are themselves "integer
6675 // like".
6676 const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
6677
6678 bool HadField = false;
6679 unsigned idx = 0;
6680 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
6681 i != e; ++i, ++idx) {
6682 const FieldDecl *FD = *i;
6683
6684 // Bit-fields are not addressable, we only need to verify they are "integer
6685 // like". We still have to disallow a subsequent non-bitfield, for example:
6686 // struct { int : 0; int x }
6687 // is non-integer like according to gcc.
6688 if (FD->isBitField()) {
6689 if (!RD->isUnion())
6690 HadField = true;
6691
6692 if (!isIntegerLikeType(FD->getType(), Context, VMContext))
6693 return false;
6694
6695 continue;
6696 }
6697
6698 // Check if this field is at offset 0.
6699 if (Layout.getFieldOffset(idx) != 0)
6700 return false;
6701
6702 if (!isIntegerLikeType(FD->getType(), Context, VMContext))
6703 return false;
6704
6705 // Only allow at most one field in a structure. This doesn't match the
6706 // wording above, but follows gcc in situations with a field following an
6707 // empty structure.
6708 if (!RD->isUnion()) {
6709 if (HadField)
6710 return false;
6711
6712 HadField = true;
6713 }
6714 }
6715
6716 return true;
6717 }
6718
classifyReturnType(QualType RetTy,bool isVariadic,unsigned functionCallConv) const6719 ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic,
6720 unsigned functionCallConv) const {
6721
6722 // Variadic functions should always marshal to the base standard.
6723 bool IsAAPCS_VFP =
6724 !isVariadic && isEffectivelyAAPCS_VFP(functionCallConv, /* AAPCS16 */ true);
6725
6726 if (RetTy->isVoidType())
6727 return ABIArgInfo::getIgnore();
6728
6729 if (const VectorType *VT = RetTy->getAs<VectorType>()) {
6730 // Large vector types should be returned via memory.
6731 if (getContext().getTypeSize(RetTy) > 128)
6732 return getNaturalAlignIndirect(RetTy);
6733 // TODO: FP16/BF16 vectors should be converted to integer vectors
6734 // This check is similar to isIllegalVectorType - refactor?
6735 if ((!getTarget().hasLegalHalfType() &&
6736 (VT->getElementType()->isFloat16Type() ||
6737 VT->getElementType()->isHalfType())) ||
6738 (IsFloatABISoftFP &&
6739 VT->getElementType()->isBFloat16Type()))
6740 return coerceIllegalVector(RetTy);
6741 }
6742
6743 if (!isAggregateTypeForABI(RetTy)) {
6744 // Treat an enum type as its underlying type.
6745 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
6746 RetTy = EnumTy->getDecl()->getIntegerType();
6747
6748 if (const auto *EIT = RetTy->getAs<ExtIntType>())
6749 if (EIT->getNumBits() > 64)
6750 return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
6751
6752 return isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
6753 : ABIArgInfo::getDirect();
6754 }
6755
6756 // Are we following APCS?
6757 if (getABIKind() == APCS) {
6758 if (isEmptyRecord(getContext(), RetTy, false))
6759 return ABIArgInfo::getIgnore();
6760
6761 // Complex types are all returned as packed integers.
6762 //
6763 // FIXME: Consider using 2 x vector types if the back end handles them
6764 // correctly.
6765 if (RetTy->isAnyComplexType())
6766 return ABIArgInfo::getDirect(llvm::IntegerType::get(
6767 getVMContext(), getContext().getTypeSize(RetTy)));
6768
6769 // Integer like structures are returned in r0.
6770 if (isIntegerLikeType(RetTy, getContext(), getVMContext())) {
6771 // Return in the smallest viable integer type.
6772 uint64_t Size = getContext().getTypeSize(RetTy);
6773 if (Size <= 8)
6774 return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
6775 if (Size <= 16)
6776 return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
6777 return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
6778 }
6779
6780 // Otherwise return in memory.
6781 return getNaturalAlignIndirect(RetTy);
6782 }
6783
6784 // Otherwise this is an AAPCS variant.
6785
6786 if (isEmptyRecord(getContext(), RetTy, true))
6787 return ABIArgInfo::getIgnore();
6788
6789 // Check for homogeneous aggregates with AAPCS-VFP.
6790 if (IsAAPCS_VFP) {
6791 const Type *Base = nullptr;
6792 uint64_t Members = 0;
6793 if (isHomogeneousAggregate(RetTy, Base, Members))
6794 return classifyHomogeneousAggregate(RetTy, Base, Members);
6795 }
6796
6797 // Aggregates <= 4 bytes are returned in r0; other aggregates
6798 // are returned indirectly.
6799 uint64_t Size = getContext().getTypeSize(RetTy);
6800 if (Size <= 32) {
6801 // On RenderScript, coerce Aggregates <= 4 bytes to an integer array of
6802 // same size and alignment.
6803 if (getTarget().isRenderScriptTarget()) {
6804 return coerceToIntArray(RetTy, getContext(), getVMContext());
6805 }
6806 if (getDataLayout().isBigEndian())
6807 // Return in 32 bit integer integer type (as if loaded by LDR, AAPCS 5.4)
6808 return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
6809
6810 // Return in the smallest viable integer type.
6811 if (Size <= 8)
6812 return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
6813 if (Size <= 16)
6814 return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
6815 return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
6816 } else if (Size <= 128 && getABIKind() == AAPCS16_VFP) {
6817 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(getVMContext());
6818 llvm::Type *CoerceTy =
6819 llvm::ArrayType::get(Int32Ty, llvm::alignTo(Size, 32) / 32);
6820 return ABIArgInfo::getDirect(CoerceTy);
6821 }
6822
6823 return getNaturalAlignIndirect(RetTy);
6824 }
6825
6826 /// isIllegalVector - check whether Ty is an illegal vector type.
isIllegalVectorType(QualType Ty) const6827 bool ARMABIInfo::isIllegalVectorType(QualType Ty) const {
6828 if (const VectorType *VT = Ty->getAs<VectorType> ()) {
6829 // On targets that don't support half, fp16 or bfloat, they are expanded
6830 // into float, and we don't want the ABI to depend on whether or not they
6831 // are supported in hardware. Thus return false to coerce vectors of these
6832 // types into integer vectors.
6833 // We do not depend on hasLegalHalfType for bfloat as it is a
6834 // separate IR type.
6835 if ((!getTarget().hasLegalHalfType() &&
6836 (VT->getElementType()->isFloat16Type() ||
6837 VT->getElementType()->isHalfType())) ||
6838 (IsFloatABISoftFP &&
6839 VT->getElementType()->isBFloat16Type()))
6840 return true;
6841 if (isAndroid()) {
6842 // Android shipped using Clang 3.1, which supported a slightly different
6843 // vector ABI. The primary differences were that 3-element vector types
6844 // were legal, and so were sub 32-bit vectors (i.e. <2 x i8>). This path
6845 // accepts that legacy behavior for Android only.
6846 // Check whether VT is legal.
6847 unsigned NumElements = VT->getNumElements();
6848 // NumElements should be power of 2 or equal to 3.
6849 if (!llvm::isPowerOf2_32(NumElements) && NumElements != 3)
6850 return true;
6851 } else {
6852 // Check whether VT is legal.
6853 unsigned NumElements = VT->getNumElements();
6854 uint64_t Size = getContext().getTypeSize(VT);
6855 // NumElements should be power of 2.
6856 if (!llvm::isPowerOf2_32(NumElements))
6857 return true;
6858 // Size should be greater than 32 bits.
6859 return Size <= 32;
6860 }
6861 }
6862 return false;
6863 }
6864
6865 /// Return true if a type contains any 16-bit floating point vectors
containsAnyFP16Vectors(QualType Ty) const6866 bool ARMABIInfo::containsAnyFP16Vectors(QualType Ty) const {
6867 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
6868 uint64_t NElements = AT->getSize().getZExtValue();
6869 if (NElements == 0)
6870 return false;
6871 return containsAnyFP16Vectors(AT->getElementType());
6872 } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
6873 const RecordDecl *RD = RT->getDecl();
6874
6875 // If this is a C++ record, check the bases first.
6876 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
6877 if (llvm::any_of(CXXRD->bases(), [this](const CXXBaseSpecifier &B) {
6878 return containsAnyFP16Vectors(B.getType());
6879 }))
6880 return true;
6881
6882 if (llvm::any_of(RD->fields(), [this](FieldDecl *FD) {
6883 return FD && containsAnyFP16Vectors(FD->getType());
6884 }))
6885 return true;
6886
6887 return false;
6888 } else {
6889 if (const VectorType *VT = Ty->getAs<VectorType>())
6890 return (VT->getElementType()->isFloat16Type() ||
6891 VT->getElementType()->isBFloat16Type() ||
6892 VT->getElementType()->isHalfType());
6893 return false;
6894 }
6895 }
6896
isLegalVectorTypeForSwift(CharUnits vectorSize,llvm::Type * eltTy,unsigned numElts) const6897 bool ARMABIInfo::isLegalVectorTypeForSwift(CharUnits vectorSize,
6898 llvm::Type *eltTy,
6899 unsigned numElts) const {
6900 if (!llvm::isPowerOf2_32(numElts))
6901 return false;
6902 unsigned size = getDataLayout().getTypeStoreSizeInBits(eltTy);
6903 if (size > 64)
6904 return false;
6905 if (vectorSize.getQuantity() != 8 &&
6906 (vectorSize.getQuantity() != 16 || numElts == 1))
6907 return false;
6908 return true;
6909 }
6910
isHomogeneousAggregateBaseType(QualType Ty) const6911 bool ARMABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
6912 // Homogeneous aggregates for AAPCS-VFP must have base types of float,
6913 // double, or 64-bit or 128-bit vectors.
6914 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
6915 if (BT->getKind() == BuiltinType::Float ||
6916 BT->getKind() == BuiltinType::Double ||
6917 BT->getKind() == BuiltinType::LongDouble)
6918 return true;
6919 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
6920 unsigned VecSize = getContext().getTypeSize(VT);
6921 if (VecSize == 64 || VecSize == 128)
6922 return true;
6923 }
6924 return false;
6925 }
6926
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const6927 bool ARMABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
6928 uint64_t Members) const {
6929 return Members <= 4;
6930 }
6931
isEffectivelyAAPCS_VFP(unsigned callConvention,bool acceptHalf) const6932 bool ARMABIInfo::isEffectivelyAAPCS_VFP(unsigned callConvention,
6933 bool acceptHalf) const {
6934 // Give precedence to user-specified calling conventions.
6935 if (callConvention != llvm::CallingConv::C)
6936 return (callConvention == llvm::CallingConv::ARM_AAPCS_VFP);
6937 else
6938 return (getABIKind() == AAPCS_VFP) ||
6939 (acceptHalf && (getABIKind() == AAPCS16_VFP));
6940 }
6941
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const6942 Address ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
6943 QualType Ty) const {
6944 CharUnits SlotSize = CharUnits::fromQuantity(4);
6945
6946 // Empty records are ignored for parameter passing purposes.
6947 if (isEmptyRecord(getContext(), Ty, true)) {
6948 Address Addr(CGF.Builder.CreateLoad(VAListAddr), SlotSize);
6949 Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
6950 return Addr;
6951 }
6952
6953 CharUnits TySize = getContext().getTypeSizeInChars(Ty);
6954 CharUnits TyAlignForABI = getContext().getTypeUnadjustedAlignInChars(Ty);
6955
6956 // Use indirect if size of the illegal vector is bigger than 16 bytes.
6957 bool IsIndirect = false;
6958 const Type *Base = nullptr;
6959 uint64_t Members = 0;
6960 if (TySize > CharUnits::fromQuantity(16) && isIllegalVectorType(Ty)) {
6961 IsIndirect = true;
6962
6963 // ARMv7k passes structs bigger than 16 bytes indirectly, in space
6964 // allocated by the caller.
6965 } else if (TySize > CharUnits::fromQuantity(16) &&
6966 getABIKind() == ARMABIInfo::AAPCS16_VFP &&
6967 !isHomogeneousAggregate(Ty, Base, Members)) {
6968 IsIndirect = true;
6969
6970 // Otherwise, bound the type's ABI alignment.
6971 // The ABI alignment for 64-bit or 128-bit vectors is 8 for AAPCS and 4 for
6972 // APCS. For AAPCS, the ABI alignment is at least 4-byte and at most 8-byte.
6973 // Our callers should be prepared to handle an under-aligned address.
6974 } else if (getABIKind() == ARMABIInfo::AAPCS_VFP ||
6975 getABIKind() == ARMABIInfo::AAPCS) {
6976 TyAlignForABI = std::max(TyAlignForABI, CharUnits::fromQuantity(4));
6977 TyAlignForABI = std::min(TyAlignForABI, CharUnits::fromQuantity(8));
6978 } else if (getABIKind() == ARMABIInfo::AAPCS16_VFP) {
6979 // ARMv7k allows type alignment up to 16 bytes.
6980 TyAlignForABI = std::max(TyAlignForABI, CharUnits::fromQuantity(4));
6981 TyAlignForABI = std::min(TyAlignForABI, CharUnits::fromQuantity(16));
6982 } else {
6983 TyAlignForABI = CharUnits::fromQuantity(4);
6984 }
6985
6986 TypeInfoChars TyInfo(TySize, TyAlignForABI, AlignRequirementKind::None);
6987 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo,
6988 SlotSize, /*AllowHigherAlign*/ true);
6989 }
6990
6991 //===----------------------------------------------------------------------===//
6992 // NVPTX ABI Implementation
6993 //===----------------------------------------------------------------------===//
6994
6995 namespace {
6996
6997 class NVPTXTargetCodeGenInfo;
6998
6999 class NVPTXABIInfo : public ABIInfo {
7000 NVPTXTargetCodeGenInfo &CGInfo;
7001
7002 public:
NVPTXABIInfo(CodeGenTypes & CGT,NVPTXTargetCodeGenInfo & Info)7003 NVPTXABIInfo(CodeGenTypes &CGT, NVPTXTargetCodeGenInfo &Info)
7004 : ABIInfo(CGT), CGInfo(Info) {}
7005
7006 ABIArgInfo classifyReturnType(QualType RetTy) const;
7007 ABIArgInfo classifyArgumentType(QualType Ty) const;
7008
7009 void computeInfo(CGFunctionInfo &FI) const override;
7010 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7011 QualType Ty) const override;
7012 bool isUnsupportedType(QualType T) const;
7013 ABIArgInfo coerceToIntArrayWithLimit(QualType Ty, unsigned MaxSize) const;
7014 };
7015
7016 class NVPTXTargetCodeGenInfo : public TargetCodeGenInfo {
7017 public:
NVPTXTargetCodeGenInfo(CodeGenTypes & CGT)7018 NVPTXTargetCodeGenInfo(CodeGenTypes &CGT)
7019 : TargetCodeGenInfo(std::make_unique<NVPTXABIInfo>(CGT, *this)) {}
7020
7021 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
7022 CodeGen::CodeGenModule &M) const override;
7023 bool shouldEmitStaticExternCAliases() const override;
7024
getCUDADeviceBuiltinSurfaceDeviceType() const7025 llvm::Type *getCUDADeviceBuiltinSurfaceDeviceType() const override {
7026 // On the device side, surface reference is represented as an object handle
7027 // in 64-bit integer.
7028 return llvm::Type::getInt64Ty(getABIInfo().getVMContext());
7029 }
7030
getCUDADeviceBuiltinTextureDeviceType() const7031 llvm::Type *getCUDADeviceBuiltinTextureDeviceType() const override {
7032 // On the device side, texture reference is represented as an object handle
7033 // in 64-bit integer.
7034 return llvm::Type::getInt64Ty(getABIInfo().getVMContext());
7035 }
7036
emitCUDADeviceBuiltinSurfaceDeviceCopy(CodeGenFunction & CGF,LValue Dst,LValue Src) const7037 bool emitCUDADeviceBuiltinSurfaceDeviceCopy(CodeGenFunction &CGF, LValue Dst,
7038 LValue Src) const override {
7039 emitBuiltinSurfTexDeviceCopy(CGF, Dst, Src);
7040 return true;
7041 }
7042
emitCUDADeviceBuiltinTextureDeviceCopy(CodeGenFunction & CGF,LValue Dst,LValue Src) const7043 bool emitCUDADeviceBuiltinTextureDeviceCopy(CodeGenFunction &CGF, LValue Dst,
7044 LValue Src) const override {
7045 emitBuiltinSurfTexDeviceCopy(CGF, Dst, Src);
7046 return true;
7047 }
7048
7049 private:
7050 // Adds a NamedMDNode with GV, Name, and Operand as operands, and adds the
7051 // resulting MDNode to the nvvm.annotations MDNode.
7052 static void addNVVMMetadata(llvm::GlobalValue *GV, StringRef Name,
7053 int Operand);
7054
emitBuiltinSurfTexDeviceCopy(CodeGenFunction & CGF,LValue Dst,LValue Src)7055 static void emitBuiltinSurfTexDeviceCopy(CodeGenFunction &CGF, LValue Dst,
7056 LValue Src) {
7057 llvm::Value *Handle = nullptr;
7058 llvm::Constant *C =
7059 llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).getPointer());
7060 // Lookup `addrspacecast` through the constant pointer if any.
7061 if (auto *ASC = llvm::dyn_cast_or_null<llvm::AddrSpaceCastOperator>(C))
7062 C = llvm::cast<llvm::Constant>(ASC->getPointerOperand());
7063 if (auto *GV = llvm::dyn_cast_or_null<llvm::GlobalVariable>(C)) {
7064 // Load the handle from the specific global variable using
7065 // `nvvm.texsurf.handle.internal` intrinsic.
7066 Handle = CGF.EmitRuntimeCall(
7067 CGF.CGM.getIntrinsic(llvm::Intrinsic::nvvm_texsurf_handle_internal,
7068 {GV->getType()}),
7069 {GV}, "texsurf_handle");
7070 } else
7071 Handle = CGF.EmitLoadOfScalar(Src, SourceLocation());
7072 CGF.EmitStoreOfScalar(Handle, Dst);
7073 }
7074 };
7075
7076 /// Checks if the type is unsupported directly by the current target.
isUnsupportedType(QualType T) const7077 bool NVPTXABIInfo::isUnsupportedType(QualType T) const {
7078 ASTContext &Context = getContext();
7079 if (!Context.getTargetInfo().hasFloat16Type() && T->isFloat16Type())
7080 return true;
7081 if (!Context.getTargetInfo().hasFloat128Type() &&
7082 (T->isFloat128Type() ||
7083 (T->isRealFloatingType() && Context.getTypeSize(T) == 128)))
7084 return true;
7085 if (const auto *EIT = T->getAs<ExtIntType>())
7086 return EIT->getNumBits() >
7087 (Context.getTargetInfo().hasInt128Type() ? 128U : 64U);
7088 if (!Context.getTargetInfo().hasInt128Type() && T->isIntegerType() &&
7089 Context.getTypeSize(T) > 64U)
7090 return true;
7091 if (const auto *AT = T->getAsArrayTypeUnsafe())
7092 return isUnsupportedType(AT->getElementType());
7093 const auto *RT = T->getAs<RecordType>();
7094 if (!RT)
7095 return false;
7096 const RecordDecl *RD = RT->getDecl();
7097
7098 // If this is a C++ record, check the bases first.
7099 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
7100 for (const CXXBaseSpecifier &I : CXXRD->bases())
7101 if (isUnsupportedType(I.getType()))
7102 return true;
7103
7104 for (const FieldDecl *I : RD->fields())
7105 if (isUnsupportedType(I->getType()))
7106 return true;
7107 return false;
7108 }
7109
7110 /// Coerce the given type into an array with maximum allowed size of elements.
coerceToIntArrayWithLimit(QualType Ty,unsigned MaxSize) const7111 ABIArgInfo NVPTXABIInfo::coerceToIntArrayWithLimit(QualType Ty,
7112 unsigned MaxSize) const {
7113 // Alignment and Size are measured in bits.
7114 const uint64_t Size = getContext().getTypeSize(Ty);
7115 const uint64_t Alignment = getContext().getTypeAlign(Ty);
7116 const unsigned Div = std::min<unsigned>(MaxSize, Alignment);
7117 llvm::Type *IntType = llvm::Type::getIntNTy(getVMContext(), Div);
7118 const uint64_t NumElements = (Size + Div - 1) / Div;
7119 return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
7120 }
7121
classifyReturnType(QualType RetTy) const7122 ABIArgInfo NVPTXABIInfo::classifyReturnType(QualType RetTy) const {
7123 if (RetTy->isVoidType())
7124 return ABIArgInfo::getIgnore();
7125
7126 if (getContext().getLangOpts().OpenMP &&
7127 getContext().getLangOpts().OpenMPIsDevice && isUnsupportedType(RetTy))
7128 return coerceToIntArrayWithLimit(RetTy, 64);
7129
7130 // note: this is different from default ABI
7131 if (!RetTy->isScalarType())
7132 return ABIArgInfo::getDirect();
7133
7134 // Treat an enum type as its underlying type.
7135 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
7136 RetTy = EnumTy->getDecl()->getIntegerType();
7137
7138 return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
7139 : ABIArgInfo::getDirect());
7140 }
7141
classifyArgumentType(QualType Ty) const7142 ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const {
7143 // Treat an enum type as its underlying type.
7144 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
7145 Ty = EnumTy->getDecl()->getIntegerType();
7146
7147 // Return aggregates type as indirect by value
7148 if (isAggregateTypeForABI(Ty)) {
7149 // Under CUDA device compilation, tex/surf builtin types are replaced with
7150 // object types and passed directly.
7151 if (getContext().getLangOpts().CUDAIsDevice) {
7152 if (Ty->isCUDADeviceBuiltinSurfaceType())
7153 return ABIArgInfo::getDirect(
7154 CGInfo.getCUDADeviceBuiltinSurfaceDeviceType());
7155 if (Ty->isCUDADeviceBuiltinTextureType())
7156 return ABIArgInfo::getDirect(
7157 CGInfo.getCUDADeviceBuiltinTextureDeviceType());
7158 }
7159 return getNaturalAlignIndirect(Ty, /* byval */ true);
7160 }
7161
7162 if (const auto *EIT = Ty->getAs<ExtIntType>()) {
7163 if ((EIT->getNumBits() > 128) ||
7164 (!getContext().getTargetInfo().hasInt128Type() &&
7165 EIT->getNumBits() > 64))
7166 return getNaturalAlignIndirect(Ty, /* byval */ true);
7167 }
7168
7169 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
7170 : ABIArgInfo::getDirect());
7171 }
7172
computeInfo(CGFunctionInfo & FI) const7173 void NVPTXABIInfo::computeInfo(CGFunctionInfo &FI) const {
7174 if (!getCXXABI().classifyReturnType(FI))
7175 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
7176 for (auto &I : FI.arguments())
7177 I.info = classifyArgumentType(I.type);
7178
7179 // Always honor user-specified calling convention.
7180 if (FI.getCallingConvention() != llvm::CallingConv::C)
7181 return;
7182
7183 FI.setEffectiveCallingConvention(getRuntimeCC());
7184 }
7185
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const7186 Address NVPTXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7187 QualType Ty) const {
7188 llvm_unreachable("NVPTX does not support varargs");
7189 }
7190
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const7191 void NVPTXTargetCodeGenInfo::setTargetAttributes(
7192 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
7193 if (GV->isDeclaration())
7194 return;
7195 const VarDecl *VD = dyn_cast_or_null<VarDecl>(D);
7196 if (VD) {
7197 if (M.getLangOpts().CUDA) {
7198 if (VD->getType()->isCUDADeviceBuiltinSurfaceType())
7199 addNVVMMetadata(GV, "surface", 1);
7200 else if (VD->getType()->isCUDADeviceBuiltinTextureType())
7201 addNVVMMetadata(GV, "texture", 1);
7202 return;
7203 }
7204 }
7205
7206 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
7207 if (!FD) return;
7208
7209 llvm::Function *F = cast<llvm::Function>(GV);
7210
7211 // Perform special handling in OpenCL mode
7212 if (M.getLangOpts().OpenCL) {
7213 // Use OpenCL function attributes to check for kernel functions
7214 // By default, all functions are device functions
7215 if (FD->hasAttr<OpenCLKernelAttr>()) {
7216 // OpenCL __kernel functions get kernel metadata
7217 // Create !{<func-ref>, metadata !"kernel", i32 1} node
7218 addNVVMMetadata(F, "kernel", 1);
7219 // And kernel functions are not subject to inlining
7220 F->addFnAttr(llvm::Attribute::NoInline);
7221 }
7222 }
7223
7224 // Perform special handling in CUDA mode.
7225 if (M.getLangOpts().CUDA) {
7226 // CUDA __global__ functions get a kernel metadata entry. Since
7227 // __global__ functions cannot be called from the device, we do not
7228 // need to set the noinline attribute.
7229 if (FD->hasAttr<CUDAGlobalAttr>()) {
7230 // Create !{<func-ref>, metadata !"kernel", i32 1} node
7231 addNVVMMetadata(F, "kernel", 1);
7232 }
7233 if (CUDALaunchBoundsAttr *Attr = FD->getAttr<CUDALaunchBoundsAttr>()) {
7234 // Create !{<func-ref>, metadata !"maxntidx", i32 <val>} node
7235 llvm::APSInt MaxThreads(32);
7236 MaxThreads = Attr->getMaxThreads()->EvaluateKnownConstInt(M.getContext());
7237 if (MaxThreads > 0)
7238 addNVVMMetadata(F, "maxntidx", MaxThreads.getExtValue());
7239
7240 // min blocks is an optional argument for CUDALaunchBoundsAttr. If it was
7241 // not specified in __launch_bounds__ or if the user specified a 0 value,
7242 // we don't have to add a PTX directive.
7243 if (Attr->getMinBlocks()) {
7244 llvm::APSInt MinBlocks(32);
7245 MinBlocks = Attr->getMinBlocks()->EvaluateKnownConstInt(M.getContext());
7246 if (MinBlocks > 0)
7247 // Create !{<func-ref>, metadata !"minctasm", i32 <val>} node
7248 addNVVMMetadata(F, "minctasm", MinBlocks.getExtValue());
7249 }
7250 }
7251 }
7252 }
7253
addNVVMMetadata(llvm::GlobalValue * GV,StringRef Name,int Operand)7254 void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::GlobalValue *GV,
7255 StringRef Name, int Operand) {
7256 llvm::Module *M = GV->getParent();
7257 llvm::LLVMContext &Ctx = M->getContext();
7258
7259 // Get "nvvm.annotations" metadata node
7260 llvm::NamedMDNode *MD = M->getOrInsertNamedMetadata("nvvm.annotations");
7261
7262 llvm::Metadata *MDVals[] = {
7263 llvm::ConstantAsMetadata::get(GV), llvm::MDString::get(Ctx, Name),
7264 llvm::ConstantAsMetadata::get(
7265 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), Operand))};
7266 // Append metadata to nvvm.annotations
7267 MD->addOperand(llvm::MDNode::get(Ctx, MDVals));
7268 }
7269
shouldEmitStaticExternCAliases() const7270 bool NVPTXTargetCodeGenInfo::shouldEmitStaticExternCAliases() const {
7271 return false;
7272 }
7273 }
7274
7275 //===----------------------------------------------------------------------===//
7276 // SystemZ ABI Implementation
7277 //===----------------------------------------------------------------------===//
7278
7279 namespace {
7280
7281 class SystemZABIInfo : public SwiftABIInfo {
7282 bool HasVector;
7283 bool IsSoftFloatABI;
7284
7285 public:
SystemZABIInfo(CodeGenTypes & CGT,bool HV,bool SF)7286 SystemZABIInfo(CodeGenTypes &CGT, bool HV, bool SF)
7287 : SwiftABIInfo(CGT), HasVector(HV), IsSoftFloatABI(SF) {}
7288
7289 bool isPromotableIntegerTypeForABI(QualType Ty) const;
7290 bool isCompoundType(QualType Ty) const;
7291 bool isVectorArgumentType(QualType Ty) const;
7292 bool isFPArgumentType(QualType Ty) const;
7293 QualType GetSingleElementType(QualType Ty) const;
7294
7295 ABIArgInfo classifyReturnType(QualType RetTy) const;
7296 ABIArgInfo classifyArgumentType(QualType ArgTy) const;
7297
computeInfo(CGFunctionInfo & FI) const7298 void computeInfo(CGFunctionInfo &FI) const override {
7299 if (!getCXXABI().classifyReturnType(FI))
7300 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
7301 for (auto &I : FI.arguments())
7302 I.info = classifyArgumentType(I.type);
7303 }
7304
7305 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7306 QualType Ty) const override;
7307
shouldPassIndirectlyForSwift(ArrayRef<llvm::Type * > scalars,bool asReturnValue) const7308 bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
7309 bool asReturnValue) const override {
7310 return occupiesMoreThan(CGT, scalars, /*total*/ 4);
7311 }
isSwiftErrorInRegister() const7312 bool isSwiftErrorInRegister() const override {
7313 return false;
7314 }
7315 };
7316
7317 class SystemZTargetCodeGenInfo : public TargetCodeGenInfo {
7318 public:
SystemZTargetCodeGenInfo(CodeGenTypes & CGT,bool HasVector,bool SoftFloatABI)7319 SystemZTargetCodeGenInfo(CodeGenTypes &CGT, bool HasVector, bool SoftFloatABI)
7320 : TargetCodeGenInfo(
7321 std::make_unique<SystemZABIInfo>(CGT, HasVector, SoftFloatABI)) {}
7322
testFPKind(llvm::Value * V,unsigned BuiltinID,CGBuilderTy & Builder,CodeGenModule & CGM) const7323 llvm::Value *testFPKind(llvm::Value *V, unsigned BuiltinID,
7324 CGBuilderTy &Builder,
7325 CodeGenModule &CGM) const override {
7326 assert(V->getType()->isFloatingPointTy() && "V should have an FP type.");
7327 // Only use TDC in constrained FP mode.
7328 if (!Builder.getIsFPConstrained())
7329 return nullptr;
7330
7331 llvm::Type *Ty = V->getType();
7332 if (Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isFP128Ty()) {
7333 llvm::Module &M = CGM.getModule();
7334 auto &Ctx = M.getContext();
7335 llvm::Function *TDCFunc =
7336 llvm::Intrinsic::getDeclaration(&M, llvm::Intrinsic::s390_tdc, Ty);
7337 unsigned TDCBits = 0;
7338 switch (BuiltinID) {
7339 case Builtin::BI__builtin_isnan:
7340 TDCBits = 0xf;
7341 break;
7342 case Builtin::BIfinite:
7343 case Builtin::BI__finite:
7344 case Builtin::BIfinitef:
7345 case Builtin::BI__finitef:
7346 case Builtin::BIfinitel:
7347 case Builtin::BI__finitel:
7348 case Builtin::BI__builtin_isfinite:
7349 TDCBits = 0xfc0;
7350 break;
7351 case Builtin::BI__builtin_isinf:
7352 TDCBits = 0x30;
7353 break;
7354 default:
7355 break;
7356 }
7357 if (TDCBits)
7358 return Builder.CreateCall(
7359 TDCFunc,
7360 {V, llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), TDCBits)});
7361 }
7362 return nullptr;
7363 }
7364 };
7365 }
7366
isPromotableIntegerTypeForABI(QualType Ty) const7367 bool SystemZABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
7368 // Treat an enum type as its underlying type.
7369 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
7370 Ty = EnumTy->getDecl()->getIntegerType();
7371
7372 // Promotable integer types are required to be promoted by the ABI.
7373 if (ABIInfo::isPromotableIntegerTypeForABI(Ty))
7374 return true;
7375
7376 if (const auto *EIT = Ty->getAs<ExtIntType>())
7377 if (EIT->getNumBits() < 64)
7378 return true;
7379
7380 // 32-bit values must also be promoted.
7381 if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
7382 switch (BT->getKind()) {
7383 case BuiltinType::Int:
7384 case BuiltinType::UInt:
7385 return true;
7386 default:
7387 return false;
7388 }
7389 return false;
7390 }
7391
isCompoundType(QualType Ty) const7392 bool SystemZABIInfo::isCompoundType(QualType Ty) const {
7393 return (Ty->isAnyComplexType() ||
7394 Ty->isVectorType() ||
7395 isAggregateTypeForABI(Ty));
7396 }
7397
isVectorArgumentType(QualType Ty) const7398 bool SystemZABIInfo::isVectorArgumentType(QualType Ty) const {
7399 return (HasVector &&
7400 Ty->isVectorType() &&
7401 getContext().getTypeSize(Ty) <= 128);
7402 }
7403
isFPArgumentType(QualType Ty) const7404 bool SystemZABIInfo::isFPArgumentType(QualType Ty) const {
7405 if (IsSoftFloatABI)
7406 return false;
7407
7408 if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
7409 switch (BT->getKind()) {
7410 case BuiltinType::Float:
7411 case BuiltinType::Double:
7412 return true;
7413 default:
7414 return false;
7415 }
7416
7417 return false;
7418 }
7419
GetSingleElementType(QualType Ty) const7420 QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const {
7421 const RecordType *RT = Ty->getAs<RecordType>();
7422
7423 if (RT && RT->isStructureOrClassType()) {
7424 const RecordDecl *RD = RT->getDecl();
7425 QualType Found;
7426
7427 // If this is a C++ record, check the bases first.
7428 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
7429 for (const auto &I : CXXRD->bases()) {
7430 QualType Base = I.getType();
7431
7432 // Empty bases don't affect things either way.
7433 if (isEmptyRecord(getContext(), Base, true))
7434 continue;
7435
7436 if (!Found.isNull())
7437 return Ty;
7438 Found = GetSingleElementType(Base);
7439 }
7440
7441 // Check the fields.
7442 for (const auto *FD : RD->fields()) {
7443 // For compatibility with GCC, ignore empty bitfields in C++ mode.
7444 // Unlike isSingleElementStruct(), empty structure and array fields
7445 // do count. So do anonymous bitfields that aren't zero-sized.
7446 if (getContext().getLangOpts().CPlusPlus &&
7447 FD->isZeroLengthBitField(getContext()))
7448 continue;
7449 // Like isSingleElementStruct(), ignore C++20 empty data members.
7450 if (FD->hasAttr<NoUniqueAddressAttr>() &&
7451 isEmptyRecord(getContext(), FD->getType(), true))
7452 continue;
7453
7454 // Unlike isSingleElementStruct(), arrays do not count.
7455 // Nested structures still do though.
7456 if (!Found.isNull())
7457 return Ty;
7458 Found = GetSingleElementType(FD->getType());
7459 }
7460
7461 // Unlike isSingleElementStruct(), trailing padding is allowed.
7462 // An 8-byte aligned struct s { float f; } is passed as a double.
7463 if (!Found.isNull())
7464 return Found;
7465 }
7466
7467 return Ty;
7468 }
7469
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const7470 Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7471 QualType Ty) const {
7472 // Assume that va_list type is correct; should be pointer to LLVM type:
7473 // struct {
7474 // i64 __gpr;
7475 // i64 __fpr;
7476 // i8 *__overflow_arg_area;
7477 // i8 *__reg_save_area;
7478 // };
7479
7480 // Every non-vector argument occupies 8 bytes and is passed by preference
7481 // in either GPRs or FPRs. Vector arguments occupy 8 or 16 bytes and are
7482 // always passed on the stack.
7483 Ty = getContext().getCanonicalType(Ty);
7484 auto TyInfo = getContext().getTypeInfoInChars(Ty);
7485 llvm::Type *ArgTy = CGF.ConvertTypeForMem(Ty);
7486 llvm::Type *DirectTy = ArgTy;
7487 ABIArgInfo AI = classifyArgumentType(Ty);
7488 bool IsIndirect = AI.isIndirect();
7489 bool InFPRs = false;
7490 bool IsVector = false;
7491 CharUnits UnpaddedSize;
7492 CharUnits DirectAlign;
7493 if (IsIndirect) {
7494 DirectTy = llvm::PointerType::getUnqual(DirectTy);
7495 UnpaddedSize = DirectAlign = CharUnits::fromQuantity(8);
7496 } else {
7497 if (AI.getCoerceToType())
7498 ArgTy = AI.getCoerceToType();
7499 InFPRs = (!IsSoftFloatABI && (ArgTy->isFloatTy() || ArgTy->isDoubleTy()));
7500 IsVector = ArgTy->isVectorTy();
7501 UnpaddedSize = TyInfo.Width;
7502 DirectAlign = TyInfo.Align;
7503 }
7504 CharUnits PaddedSize = CharUnits::fromQuantity(8);
7505 if (IsVector && UnpaddedSize > PaddedSize)
7506 PaddedSize = CharUnits::fromQuantity(16);
7507 assert((UnpaddedSize <= PaddedSize) && "Invalid argument size.");
7508
7509 CharUnits Padding = (PaddedSize - UnpaddedSize);
7510
7511 llvm::Type *IndexTy = CGF.Int64Ty;
7512 llvm::Value *PaddedSizeV =
7513 llvm::ConstantInt::get(IndexTy, PaddedSize.getQuantity());
7514
7515 if (IsVector) {
7516 // Work out the address of a vector argument on the stack.
7517 // Vector arguments are always passed in the high bits of a
7518 // single (8 byte) or double (16 byte) stack slot.
7519 Address OverflowArgAreaPtr =
7520 CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_ptr");
7521 Address OverflowArgArea =
7522 Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
7523 TyInfo.Align);
7524 Address MemAddr =
7525 CGF.Builder.CreateElementBitCast(OverflowArgArea, DirectTy, "mem_addr");
7526
7527 // Update overflow_arg_area_ptr pointer
7528 llvm::Value *NewOverflowArgArea =
7529 CGF.Builder.CreateGEP(OverflowArgArea.getElementType(),
7530 OverflowArgArea.getPointer(), PaddedSizeV,
7531 "overflow_arg_area");
7532 CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
7533
7534 return MemAddr;
7535 }
7536
7537 assert(PaddedSize.getQuantity() == 8);
7538
7539 unsigned MaxRegs, RegCountField, RegSaveIndex;
7540 CharUnits RegPadding;
7541 if (InFPRs) {
7542 MaxRegs = 4; // Maximum of 4 FPR arguments
7543 RegCountField = 1; // __fpr
7544 RegSaveIndex = 16; // save offset for f0
7545 RegPadding = CharUnits(); // floats are passed in the high bits of an FPR
7546 } else {
7547 MaxRegs = 5; // Maximum of 5 GPR arguments
7548 RegCountField = 0; // __gpr
7549 RegSaveIndex = 2; // save offset for r2
7550 RegPadding = Padding; // values are passed in the low bits of a GPR
7551 }
7552
7553 Address RegCountPtr =
7554 CGF.Builder.CreateStructGEP(VAListAddr, RegCountField, "reg_count_ptr");
7555 llvm::Value *RegCount = CGF.Builder.CreateLoad(RegCountPtr, "reg_count");
7556 llvm::Value *MaxRegsV = llvm::ConstantInt::get(IndexTy, MaxRegs);
7557 llvm::Value *InRegs = CGF.Builder.CreateICmpULT(RegCount, MaxRegsV,
7558 "fits_in_regs");
7559
7560 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
7561 llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
7562 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
7563 CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
7564
7565 // Emit code to load the value if it was passed in registers.
7566 CGF.EmitBlock(InRegBlock);
7567
7568 // Work out the address of an argument register.
7569 llvm::Value *ScaledRegCount =
7570 CGF.Builder.CreateMul(RegCount, PaddedSizeV, "scaled_reg_count");
7571 llvm::Value *RegBase =
7572 llvm::ConstantInt::get(IndexTy, RegSaveIndex * PaddedSize.getQuantity()
7573 + RegPadding.getQuantity());
7574 llvm::Value *RegOffset =
7575 CGF.Builder.CreateAdd(ScaledRegCount, RegBase, "reg_offset");
7576 Address RegSaveAreaPtr =
7577 CGF.Builder.CreateStructGEP(VAListAddr, 3, "reg_save_area_ptr");
7578 llvm::Value *RegSaveArea =
7579 CGF.Builder.CreateLoad(RegSaveAreaPtr, "reg_save_area");
7580 Address RawRegAddr(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, RegOffset,
7581 "raw_reg_addr"),
7582 PaddedSize);
7583 Address RegAddr =
7584 CGF.Builder.CreateElementBitCast(RawRegAddr, DirectTy, "reg_addr");
7585
7586 // Update the register count
7587 llvm::Value *One = llvm::ConstantInt::get(IndexTy, 1);
7588 llvm::Value *NewRegCount =
7589 CGF.Builder.CreateAdd(RegCount, One, "reg_count");
7590 CGF.Builder.CreateStore(NewRegCount, RegCountPtr);
7591 CGF.EmitBranch(ContBlock);
7592
7593 // Emit code to load the value if it was passed in memory.
7594 CGF.EmitBlock(InMemBlock);
7595
7596 // Work out the address of a stack argument.
7597 Address OverflowArgAreaPtr =
7598 CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_ptr");
7599 Address OverflowArgArea =
7600 Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
7601 PaddedSize);
7602 Address RawMemAddr =
7603 CGF.Builder.CreateConstByteGEP(OverflowArgArea, Padding, "raw_mem_addr");
7604 Address MemAddr =
7605 CGF.Builder.CreateElementBitCast(RawMemAddr, DirectTy, "mem_addr");
7606
7607 // Update overflow_arg_area_ptr pointer
7608 llvm::Value *NewOverflowArgArea =
7609 CGF.Builder.CreateGEP(OverflowArgArea.getElementType(),
7610 OverflowArgArea.getPointer(), PaddedSizeV,
7611 "overflow_arg_area");
7612 CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
7613 CGF.EmitBranch(ContBlock);
7614
7615 // Return the appropriate result.
7616 CGF.EmitBlock(ContBlock);
7617 Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock,
7618 MemAddr, InMemBlock, "va_arg.addr");
7619
7620 if (IsIndirect)
7621 ResAddr = Address(CGF.Builder.CreateLoad(ResAddr, "indirect_arg"),
7622 TyInfo.Align);
7623
7624 return ResAddr;
7625 }
7626
classifyReturnType(QualType RetTy) const7627 ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const {
7628 if (RetTy->isVoidType())
7629 return ABIArgInfo::getIgnore();
7630 if (isVectorArgumentType(RetTy))
7631 return ABIArgInfo::getDirect();
7632 if (isCompoundType(RetTy) || getContext().getTypeSize(RetTy) > 64)
7633 return getNaturalAlignIndirect(RetTy);
7634 return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
7635 : ABIArgInfo::getDirect());
7636 }
7637
classifyArgumentType(QualType Ty) const7638 ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const {
7639 // Handle the generic C++ ABI.
7640 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
7641 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
7642
7643 // Integers and enums are extended to full register width.
7644 if (isPromotableIntegerTypeForABI(Ty))
7645 return ABIArgInfo::getExtend(Ty);
7646
7647 // Handle vector types and vector-like structure types. Note that
7648 // as opposed to float-like structure types, we do not allow any
7649 // padding for vector-like structures, so verify the sizes match.
7650 uint64_t Size = getContext().getTypeSize(Ty);
7651 QualType SingleElementTy = GetSingleElementType(Ty);
7652 if (isVectorArgumentType(SingleElementTy) &&
7653 getContext().getTypeSize(SingleElementTy) == Size)
7654 return ABIArgInfo::getDirect(CGT.ConvertType(SingleElementTy));
7655
7656 // Values that are not 1, 2, 4 or 8 bytes in size are passed indirectly.
7657 if (Size != 8 && Size != 16 && Size != 32 && Size != 64)
7658 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
7659
7660 // Handle small structures.
7661 if (const RecordType *RT = Ty->getAs<RecordType>()) {
7662 // Structures with flexible arrays have variable length, so really
7663 // fail the size test above.
7664 const RecordDecl *RD = RT->getDecl();
7665 if (RD->hasFlexibleArrayMember())
7666 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
7667
7668 // The structure is passed as an unextended integer, a float, or a double.
7669 llvm::Type *PassTy;
7670 if (isFPArgumentType(SingleElementTy)) {
7671 assert(Size == 32 || Size == 64);
7672 if (Size == 32)
7673 PassTy = llvm::Type::getFloatTy(getVMContext());
7674 else
7675 PassTy = llvm::Type::getDoubleTy(getVMContext());
7676 } else
7677 PassTy = llvm::IntegerType::get(getVMContext(), Size);
7678 return ABIArgInfo::getDirect(PassTy);
7679 }
7680
7681 // Non-structure compounds are passed indirectly.
7682 if (isCompoundType(Ty))
7683 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
7684
7685 return ABIArgInfo::getDirect(nullptr);
7686 }
7687
7688 //===----------------------------------------------------------------------===//
7689 // MSP430 ABI Implementation
7690 //===----------------------------------------------------------------------===//
7691
7692 namespace {
7693
7694 class MSP430ABIInfo : public DefaultABIInfo {
complexArgInfo()7695 static ABIArgInfo complexArgInfo() {
7696 ABIArgInfo Info = ABIArgInfo::getDirect();
7697 Info.setCanBeFlattened(false);
7698 return Info;
7699 }
7700
7701 public:
MSP430ABIInfo(CodeGenTypes & CGT)7702 MSP430ABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
7703
classifyReturnType(QualType RetTy) const7704 ABIArgInfo classifyReturnType(QualType RetTy) const {
7705 if (RetTy->isAnyComplexType())
7706 return complexArgInfo();
7707
7708 return DefaultABIInfo::classifyReturnType(RetTy);
7709 }
7710
classifyArgumentType(QualType RetTy) const7711 ABIArgInfo classifyArgumentType(QualType RetTy) const {
7712 if (RetTy->isAnyComplexType())
7713 return complexArgInfo();
7714
7715 return DefaultABIInfo::classifyArgumentType(RetTy);
7716 }
7717
7718 // Just copy the original implementations because
7719 // DefaultABIInfo::classify{Return,Argument}Type() are not virtual
computeInfo(CGFunctionInfo & FI) const7720 void computeInfo(CGFunctionInfo &FI) const override {
7721 if (!getCXXABI().classifyReturnType(FI))
7722 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
7723 for (auto &I : FI.arguments())
7724 I.info = classifyArgumentType(I.type);
7725 }
7726
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const7727 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7728 QualType Ty) const override {
7729 return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
7730 }
7731 };
7732
7733 class MSP430TargetCodeGenInfo : public TargetCodeGenInfo {
7734 public:
MSP430TargetCodeGenInfo(CodeGenTypes & CGT)7735 MSP430TargetCodeGenInfo(CodeGenTypes &CGT)
7736 : TargetCodeGenInfo(std::make_unique<MSP430ABIInfo>(CGT)) {}
7737 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
7738 CodeGen::CodeGenModule &M) const override;
7739 };
7740
7741 }
7742
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const7743 void MSP430TargetCodeGenInfo::setTargetAttributes(
7744 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
7745 if (GV->isDeclaration())
7746 return;
7747 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
7748 const auto *InterruptAttr = FD->getAttr<MSP430InterruptAttr>();
7749 if (!InterruptAttr)
7750 return;
7751
7752 // Handle 'interrupt' attribute:
7753 llvm::Function *F = cast<llvm::Function>(GV);
7754
7755 // Step 1: Set ISR calling convention.
7756 F->setCallingConv(llvm::CallingConv::MSP430_INTR);
7757
7758 // Step 2: Add attributes goodness.
7759 F->addFnAttr(llvm::Attribute::NoInline);
7760 F->addFnAttr("interrupt", llvm::utostr(InterruptAttr->getNumber()));
7761 }
7762 }
7763
7764 //===----------------------------------------------------------------------===//
7765 // MIPS ABI Implementation. This works for both little-endian and
7766 // big-endian variants.
7767 //===----------------------------------------------------------------------===//
7768
7769 namespace {
7770 class MipsABIInfo : public ABIInfo {
7771 bool IsO32;
7772 unsigned MinABIStackAlignInBytes, StackAlignInBytes;
7773 void CoerceToIntArgs(uint64_t TySize,
7774 SmallVectorImpl<llvm::Type *> &ArgList) const;
7775 llvm::Type* HandleAggregates(QualType Ty, uint64_t TySize) const;
7776 llvm::Type* returnAggregateInRegs(QualType RetTy, uint64_t Size) const;
7777 llvm::Type* getPaddingType(uint64_t Align, uint64_t Offset) const;
7778 public:
MipsABIInfo(CodeGenTypes & CGT,bool _IsO32)7779 MipsABIInfo(CodeGenTypes &CGT, bool _IsO32) :
7780 ABIInfo(CGT), IsO32(_IsO32), MinABIStackAlignInBytes(IsO32 ? 4 : 8),
7781 StackAlignInBytes(IsO32 ? 8 : 16) {}
7782
7783 ABIArgInfo classifyReturnType(QualType RetTy) const;
7784 ABIArgInfo classifyArgumentType(QualType RetTy, uint64_t &Offset) const;
7785 void computeInfo(CGFunctionInfo &FI) const override;
7786 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
7787 QualType Ty) const override;
7788 ABIArgInfo extendType(QualType Ty) const;
7789 };
7790
7791 class MIPSTargetCodeGenInfo : public TargetCodeGenInfo {
7792 unsigned SizeOfUnwindException;
7793 public:
MIPSTargetCodeGenInfo(CodeGenTypes & CGT,bool IsO32)7794 MIPSTargetCodeGenInfo(CodeGenTypes &CGT, bool IsO32)
7795 : TargetCodeGenInfo(std::make_unique<MipsABIInfo>(CGT, IsO32)),
7796 SizeOfUnwindException(IsO32 ? 24 : 32) {}
7797
getDwarfEHStackPointer(CodeGen::CodeGenModule & CGM) const7798 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
7799 return 29;
7800 }
7801
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const7802 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
7803 CodeGen::CodeGenModule &CGM) const override {
7804 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
7805 if (!FD) return;
7806 llvm::Function *Fn = cast<llvm::Function>(GV);
7807
7808 if (FD->hasAttr<MipsLongCallAttr>())
7809 Fn->addFnAttr("long-call");
7810 else if (FD->hasAttr<MipsShortCallAttr>())
7811 Fn->addFnAttr("short-call");
7812
7813 // Other attributes do not have a meaning for declarations.
7814 if (GV->isDeclaration())
7815 return;
7816
7817 if (FD->hasAttr<Mips16Attr>()) {
7818 Fn->addFnAttr("mips16");
7819 }
7820 else if (FD->hasAttr<NoMips16Attr>()) {
7821 Fn->addFnAttr("nomips16");
7822 }
7823
7824 if (FD->hasAttr<MicroMipsAttr>())
7825 Fn->addFnAttr("micromips");
7826 else if (FD->hasAttr<NoMicroMipsAttr>())
7827 Fn->addFnAttr("nomicromips");
7828
7829 const MipsInterruptAttr *Attr = FD->getAttr<MipsInterruptAttr>();
7830 if (!Attr)
7831 return;
7832
7833 const char *Kind;
7834 switch (Attr->getInterrupt()) {
7835 case MipsInterruptAttr::eic: Kind = "eic"; break;
7836 case MipsInterruptAttr::sw0: Kind = "sw0"; break;
7837 case MipsInterruptAttr::sw1: Kind = "sw1"; break;
7838 case MipsInterruptAttr::hw0: Kind = "hw0"; break;
7839 case MipsInterruptAttr::hw1: Kind = "hw1"; break;
7840 case MipsInterruptAttr::hw2: Kind = "hw2"; break;
7841 case MipsInterruptAttr::hw3: Kind = "hw3"; break;
7842 case MipsInterruptAttr::hw4: Kind = "hw4"; break;
7843 case MipsInterruptAttr::hw5: Kind = "hw5"; break;
7844 }
7845
7846 Fn->addFnAttr("interrupt", Kind);
7847
7848 }
7849
7850 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
7851 llvm::Value *Address) const override;
7852
getSizeOfUnwindException() const7853 unsigned getSizeOfUnwindException() const override {
7854 return SizeOfUnwindException;
7855 }
7856 };
7857 }
7858
CoerceToIntArgs(uint64_t TySize,SmallVectorImpl<llvm::Type * > & ArgList) const7859 void MipsABIInfo::CoerceToIntArgs(
7860 uint64_t TySize, SmallVectorImpl<llvm::Type *> &ArgList) const {
7861 llvm::IntegerType *IntTy =
7862 llvm::IntegerType::get(getVMContext(), MinABIStackAlignInBytes * 8);
7863
7864 // Add (TySize / MinABIStackAlignInBytes) args of IntTy.
7865 for (unsigned N = TySize / (MinABIStackAlignInBytes * 8); N; --N)
7866 ArgList.push_back(IntTy);
7867
7868 // If necessary, add one more integer type to ArgList.
7869 unsigned R = TySize % (MinABIStackAlignInBytes * 8);
7870
7871 if (R)
7872 ArgList.push_back(llvm::IntegerType::get(getVMContext(), R));
7873 }
7874
7875 // In N32/64, an aligned double precision floating point field is passed in
7876 // a register.
HandleAggregates(QualType Ty,uint64_t TySize) const7877 llvm::Type* MipsABIInfo::HandleAggregates(QualType Ty, uint64_t TySize) const {
7878 SmallVector<llvm::Type*, 8> ArgList, IntArgList;
7879
7880 if (IsO32) {
7881 CoerceToIntArgs(TySize, ArgList);
7882 return llvm::StructType::get(getVMContext(), ArgList);
7883 }
7884
7885 if (Ty->isComplexType())
7886 return CGT.ConvertType(Ty);
7887
7888 const RecordType *RT = Ty->getAs<RecordType>();
7889
7890 // Unions/vectors are passed in integer registers.
7891 if (!RT || !RT->isStructureOrClassType()) {
7892 CoerceToIntArgs(TySize, ArgList);
7893 return llvm::StructType::get(getVMContext(), ArgList);
7894 }
7895
7896 const RecordDecl *RD = RT->getDecl();
7897 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
7898 assert(!(TySize % 8) && "Size of structure must be multiple of 8.");
7899
7900 uint64_t LastOffset = 0;
7901 unsigned idx = 0;
7902 llvm::IntegerType *I64 = llvm::IntegerType::get(getVMContext(), 64);
7903
7904 // Iterate over fields in the struct/class and check if there are any aligned
7905 // double fields.
7906 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
7907 i != e; ++i, ++idx) {
7908 const QualType Ty = i->getType();
7909 const BuiltinType *BT = Ty->getAs<BuiltinType>();
7910
7911 if (!BT || BT->getKind() != BuiltinType::Double)
7912 continue;
7913
7914 uint64_t Offset = Layout.getFieldOffset(idx);
7915 if (Offset % 64) // Ignore doubles that are not aligned.
7916 continue;
7917
7918 // Add ((Offset - LastOffset) / 64) args of type i64.
7919 for (unsigned j = (Offset - LastOffset) / 64; j > 0; --j)
7920 ArgList.push_back(I64);
7921
7922 // Add double type.
7923 ArgList.push_back(llvm::Type::getDoubleTy(getVMContext()));
7924 LastOffset = Offset + 64;
7925 }
7926
7927 CoerceToIntArgs(TySize - LastOffset, IntArgList);
7928 ArgList.append(IntArgList.begin(), IntArgList.end());
7929
7930 return llvm::StructType::get(getVMContext(), ArgList);
7931 }
7932
getPaddingType(uint64_t OrigOffset,uint64_t Offset) const7933 llvm::Type *MipsABIInfo::getPaddingType(uint64_t OrigOffset,
7934 uint64_t Offset) const {
7935 if (OrigOffset + MinABIStackAlignInBytes > Offset)
7936 return nullptr;
7937
7938 return llvm::IntegerType::get(getVMContext(), (Offset - OrigOffset) * 8);
7939 }
7940
7941 ABIArgInfo
classifyArgumentType(QualType Ty,uint64_t & Offset) const7942 MipsABIInfo::classifyArgumentType(QualType Ty, uint64_t &Offset) const {
7943 Ty = useFirstFieldIfTransparentUnion(Ty);
7944
7945 uint64_t OrigOffset = Offset;
7946 uint64_t TySize = getContext().getTypeSize(Ty);
7947 uint64_t Align = getContext().getTypeAlign(Ty) / 8;
7948
7949 Align = std::min(std::max(Align, (uint64_t)MinABIStackAlignInBytes),
7950 (uint64_t)StackAlignInBytes);
7951 unsigned CurrOffset = llvm::alignTo(Offset, Align);
7952 Offset = CurrOffset + llvm::alignTo(TySize, Align * 8) / 8;
7953
7954 if (isAggregateTypeForABI(Ty) || Ty->isVectorType()) {
7955 // Ignore empty aggregates.
7956 if (TySize == 0)
7957 return ABIArgInfo::getIgnore();
7958
7959 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
7960 Offset = OrigOffset + MinABIStackAlignInBytes;
7961 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
7962 }
7963
7964 // If we have reached here, aggregates are passed directly by coercing to
7965 // another structure type. Padding is inserted if the offset of the
7966 // aggregate is unaligned.
7967 ABIArgInfo ArgInfo =
7968 ABIArgInfo::getDirect(HandleAggregates(Ty, TySize), 0,
7969 getPaddingType(OrigOffset, CurrOffset));
7970 ArgInfo.setInReg(true);
7971 return ArgInfo;
7972 }
7973
7974 // Treat an enum type as its underlying type.
7975 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
7976 Ty = EnumTy->getDecl()->getIntegerType();
7977
7978 // Make sure we pass indirectly things that are too large.
7979 if (const auto *EIT = Ty->getAs<ExtIntType>())
7980 if (EIT->getNumBits() > 128 ||
7981 (EIT->getNumBits() > 64 &&
7982 !getContext().getTargetInfo().hasInt128Type()))
7983 return getNaturalAlignIndirect(Ty);
7984
7985 // All integral types are promoted to the GPR width.
7986 if (Ty->isIntegralOrEnumerationType())
7987 return extendType(Ty);
7988
7989 return ABIArgInfo::getDirect(
7990 nullptr, 0, IsO32 ? nullptr : getPaddingType(OrigOffset, CurrOffset));
7991 }
7992
7993 llvm::Type*
returnAggregateInRegs(QualType RetTy,uint64_t Size) const7994 MipsABIInfo::returnAggregateInRegs(QualType RetTy, uint64_t Size) const {
7995 const RecordType *RT = RetTy->getAs<RecordType>();
7996 SmallVector<llvm::Type*, 8> RTList;
7997
7998 if (RT && RT->isStructureOrClassType()) {
7999 const RecordDecl *RD = RT->getDecl();
8000 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
8001 unsigned FieldCnt = Layout.getFieldCount();
8002
8003 // N32/64 returns struct/classes in floating point registers if the
8004 // following conditions are met:
8005 // 1. The size of the struct/class is no larger than 128-bit.
8006 // 2. The struct/class has one or two fields all of which are floating
8007 // point types.
8008 // 3. The offset of the first field is zero (this follows what gcc does).
8009 //
8010 // Any other composite results are returned in integer registers.
8011 //
8012 if (FieldCnt && (FieldCnt <= 2) && !Layout.getFieldOffset(0)) {
8013 RecordDecl::field_iterator b = RD->field_begin(), e = RD->field_end();
8014 for (; b != e; ++b) {
8015 const BuiltinType *BT = b->getType()->getAs<BuiltinType>();
8016
8017 if (!BT || !BT->isFloatingPoint())
8018 break;
8019
8020 RTList.push_back(CGT.ConvertType(b->getType()));
8021 }
8022
8023 if (b == e)
8024 return llvm::StructType::get(getVMContext(), RTList,
8025 RD->hasAttr<PackedAttr>());
8026
8027 RTList.clear();
8028 }
8029 }
8030
8031 CoerceToIntArgs(Size, RTList);
8032 return llvm::StructType::get(getVMContext(), RTList);
8033 }
8034
classifyReturnType(QualType RetTy) const8035 ABIArgInfo MipsABIInfo::classifyReturnType(QualType RetTy) const {
8036 uint64_t Size = getContext().getTypeSize(RetTy);
8037
8038 if (RetTy->isVoidType())
8039 return ABIArgInfo::getIgnore();
8040
8041 // O32 doesn't treat zero-sized structs differently from other structs.
8042 // However, N32/N64 ignores zero sized return values.
8043 if (!IsO32 && Size == 0)
8044 return ABIArgInfo::getIgnore();
8045
8046 if (isAggregateTypeForABI(RetTy) || RetTy->isVectorType()) {
8047 if (Size <= 128) {
8048 if (RetTy->isAnyComplexType())
8049 return ABIArgInfo::getDirect();
8050
8051 // O32 returns integer vectors in registers and N32/N64 returns all small
8052 // aggregates in registers.
8053 if (!IsO32 ||
8054 (RetTy->isVectorType() && !RetTy->hasFloatingRepresentation())) {
8055 ABIArgInfo ArgInfo =
8056 ABIArgInfo::getDirect(returnAggregateInRegs(RetTy, Size));
8057 ArgInfo.setInReg(true);
8058 return ArgInfo;
8059 }
8060 }
8061
8062 return getNaturalAlignIndirect(RetTy);
8063 }
8064
8065 // Treat an enum type as its underlying type.
8066 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
8067 RetTy = EnumTy->getDecl()->getIntegerType();
8068
8069 // Make sure we pass indirectly things that are too large.
8070 if (const auto *EIT = RetTy->getAs<ExtIntType>())
8071 if (EIT->getNumBits() > 128 ||
8072 (EIT->getNumBits() > 64 &&
8073 !getContext().getTargetInfo().hasInt128Type()))
8074 return getNaturalAlignIndirect(RetTy);
8075
8076 if (isPromotableIntegerTypeForABI(RetTy))
8077 return ABIArgInfo::getExtend(RetTy);
8078
8079 if ((RetTy->isUnsignedIntegerOrEnumerationType() ||
8080 RetTy->isSignedIntegerOrEnumerationType()) && Size == 32 && !IsO32)
8081 return ABIArgInfo::getSignExtend(RetTy);
8082
8083 return ABIArgInfo::getDirect();
8084 }
8085
computeInfo(CGFunctionInfo & FI) const8086 void MipsABIInfo::computeInfo(CGFunctionInfo &FI) const {
8087 ABIArgInfo &RetInfo = FI.getReturnInfo();
8088 if (!getCXXABI().classifyReturnType(FI))
8089 RetInfo = classifyReturnType(FI.getReturnType());
8090
8091 // Check if a pointer to an aggregate is passed as a hidden argument.
8092 uint64_t Offset = RetInfo.isIndirect() ? MinABIStackAlignInBytes : 0;
8093
8094 for (auto &I : FI.arguments())
8095 I.info = classifyArgumentType(I.type, Offset);
8096 }
8097
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType OrigTy) const8098 Address MipsABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
8099 QualType OrigTy) const {
8100 QualType Ty = OrigTy;
8101
8102 // Integer arguments are promoted to 32-bit on O32 and 64-bit on N32/N64.
8103 // Pointers are also promoted in the same way but this only matters for N32.
8104 unsigned SlotSizeInBits = IsO32 ? 32 : 64;
8105 unsigned PtrWidth = getTarget().getPointerWidth(0);
8106 bool DidPromote = false;
8107 if ((Ty->isIntegerType() &&
8108 getContext().getIntWidth(Ty) < SlotSizeInBits) ||
8109 (Ty->isPointerType() && PtrWidth < SlotSizeInBits)) {
8110 DidPromote = true;
8111 Ty = getContext().getIntTypeForBitwidth(SlotSizeInBits,
8112 Ty->isSignedIntegerType());
8113 }
8114
8115 auto TyInfo = getContext().getTypeInfoInChars(Ty);
8116
8117 // The alignment of things in the argument area is never larger than
8118 // StackAlignInBytes.
8119 TyInfo.Align =
8120 std::min(TyInfo.Align, CharUnits::fromQuantity(StackAlignInBytes));
8121
8122 // MinABIStackAlignInBytes is the size of argument slots on the stack.
8123 CharUnits ArgSlotSize = CharUnits::fromQuantity(MinABIStackAlignInBytes);
8124
8125 Address Addr = emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
8126 TyInfo, ArgSlotSize, /*AllowHigherAlign*/ true);
8127
8128
8129 // If there was a promotion, "unpromote" into a temporary.
8130 // TODO: can we just use a pointer into a subset of the original slot?
8131 if (DidPromote) {
8132 Address Temp = CGF.CreateMemTemp(OrigTy, "vaarg.promotion-temp");
8133 llvm::Value *Promoted = CGF.Builder.CreateLoad(Addr);
8134
8135 // Truncate down to the right width.
8136 llvm::Type *IntTy = (OrigTy->isIntegerType() ? Temp.getElementType()
8137 : CGF.IntPtrTy);
8138 llvm::Value *V = CGF.Builder.CreateTrunc(Promoted, IntTy);
8139 if (OrigTy->isPointerType())
8140 V = CGF.Builder.CreateIntToPtr(V, Temp.getElementType());
8141
8142 CGF.Builder.CreateStore(V, Temp);
8143 Addr = Temp;
8144 }
8145
8146 return Addr;
8147 }
8148
extendType(QualType Ty) const8149 ABIArgInfo MipsABIInfo::extendType(QualType Ty) const {
8150 int TySize = getContext().getTypeSize(Ty);
8151
8152 // MIPS64 ABI requires unsigned 32 bit integers to be sign extended.
8153 if (Ty->isUnsignedIntegerOrEnumerationType() && TySize == 32)
8154 return ABIArgInfo::getSignExtend(Ty);
8155
8156 return ABIArgInfo::getExtend(Ty);
8157 }
8158
8159 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const8160 MIPSTargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
8161 llvm::Value *Address) const {
8162 // This information comes from gcc's implementation, which seems to
8163 // as canonical as it gets.
8164
8165 // Everything on MIPS is 4 bytes. Double-precision FP registers
8166 // are aliased to pairs of single-precision FP registers.
8167 llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
8168
8169 // 0-31 are the general purpose registers, $0 - $31.
8170 // 32-63 are the floating-point registers, $f0 - $f31.
8171 // 64 and 65 are the multiply/divide registers, $hi and $lo.
8172 // 66 is the (notional, I think) register for signal-handler return.
8173 AssignToArrayRange(CGF.Builder, Address, Four8, 0, 65);
8174
8175 // 67-74 are the floating-point status registers, $fcc0 - $fcc7.
8176 // They are one bit wide and ignored here.
8177
8178 // 80-111 are the coprocessor 0 registers, $c0r0 - $c0r31.
8179 // (coprocessor 1 is the FP unit)
8180 // 112-143 are the coprocessor 2 registers, $c2r0 - $c2r31.
8181 // 144-175 are the coprocessor 3 registers, $c3r0 - $c3r31.
8182 // 176-181 are the DSP accumulator registers.
8183 AssignToArrayRange(CGF.Builder, Address, Four8, 80, 181);
8184 return false;
8185 }
8186
8187 //===----------------------------------------------------------------------===//
8188 // M68k ABI Implementation
8189 //===----------------------------------------------------------------------===//
8190
8191 namespace {
8192
8193 class M68kTargetCodeGenInfo : public TargetCodeGenInfo {
8194 public:
M68kTargetCodeGenInfo(CodeGenTypes & CGT)8195 M68kTargetCodeGenInfo(CodeGenTypes &CGT)
8196 : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
8197 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
8198 CodeGen::CodeGenModule &M) const override;
8199 };
8200
8201 } // namespace
8202
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const8203 void M68kTargetCodeGenInfo::setTargetAttributes(
8204 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
8205 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
8206 if (const auto *attr = FD->getAttr<M68kInterruptAttr>()) {
8207 // Handle 'interrupt' attribute:
8208 llvm::Function *F = cast<llvm::Function>(GV);
8209
8210 // Step 1: Set ISR calling convention.
8211 F->setCallingConv(llvm::CallingConv::M68k_INTR);
8212
8213 // Step 2: Add attributes goodness.
8214 F->addFnAttr(llvm::Attribute::NoInline);
8215
8216 // Step 3: Emit ISR vector alias.
8217 unsigned Num = attr->getNumber() / 2;
8218 llvm::GlobalAlias::create(llvm::Function::ExternalLinkage,
8219 "__isr_" + Twine(Num), F);
8220 }
8221 }
8222 }
8223
8224 //===----------------------------------------------------------------------===//
8225 // AVR ABI Implementation. Documented at
8226 // https://gcc.gnu.org/wiki/avr-gcc#Calling_Convention
8227 // https://gcc.gnu.org/wiki/avr-gcc#Reduced_Tiny
8228 //===----------------------------------------------------------------------===//
8229
8230 namespace {
8231 class AVRABIInfo : public DefaultABIInfo {
8232 public:
AVRABIInfo(CodeGenTypes & CGT)8233 AVRABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
8234
classifyReturnType(QualType Ty) const8235 ABIArgInfo classifyReturnType(QualType Ty) const {
8236 // A return struct with size less than or equal to 8 bytes is returned
8237 // directly via registers R18-R25.
8238 if (isAggregateTypeForABI(Ty) && getContext().getTypeSize(Ty) <= 64)
8239 return ABIArgInfo::getDirect();
8240 else
8241 return DefaultABIInfo::classifyReturnType(Ty);
8242 }
8243
8244 // Just copy the original implementation of DefaultABIInfo::computeInfo(),
8245 // since DefaultABIInfo::classify{Return,Argument}Type() are not virtual.
computeInfo(CGFunctionInfo & FI) const8246 void computeInfo(CGFunctionInfo &FI) const override {
8247 if (!getCXXABI().classifyReturnType(FI))
8248 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
8249 for (auto &I : FI.arguments())
8250 I.info = classifyArgumentType(I.type);
8251 }
8252 };
8253
8254 class AVRTargetCodeGenInfo : public TargetCodeGenInfo {
8255 public:
AVRTargetCodeGenInfo(CodeGenTypes & CGT)8256 AVRTargetCodeGenInfo(CodeGenTypes &CGT)
8257 : TargetCodeGenInfo(std::make_unique<AVRABIInfo>(CGT)) {}
8258
getGlobalVarAddressSpace(CodeGenModule & CGM,const VarDecl * D) const8259 LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
8260 const VarDecl *D) const override {
8261 // Check if a global/static variable is defined within address space 1
8262 // but not constant.
8263 LangAS AS = D->getType().getAddressSpace();
8264 if (isTargetAddressSpace(AS) && toTargetAddressSpace(AS) == 1 &&
8265 !D->getType().isConstQualified())
8266 CGM.getDiags().Report(D->getLocation(),
8267 diag::err_verify_nonconst_addrspace)
8268 << "__flash";
8269 return TargetCodeGenInfo::getGlobalVarAddressSpace(CGM, D);
8270 }
8271
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const8272 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
8273 CodeGen::CodeGenModule &CGM) const override {
8274 if (GV->isDeclaration())
8275 return;
8276 const auto *FD = dyn_cast_or_null<FunctionDecl>(D);
8277 if (!FD) return;
8278 auto *Fn = cast<llvm::Function>(GV);
8279
8280 if (FD->getAttr<AVRInterruptAttr>())
8281 Fn->addFnAttr("interrupt");
8282
8283 if (FD->getAttr<AVRSignalAttr>())
8284 Fn->addFnAttr("signal");
8285 }
8286 };
8287 }
8288
8289 //===----------------------------------------------------------------------===//
8290 // TCE ABI Implementation (see http://tce.cs.tut.fi). Uses mostly the defaults.
8291 // Currently subclassed only to implement custom OpenCL C function attribute
8292 // handling.
8293 //===----------------------------------------------------------------------===//
8294
8295 namespace {
8296
8297 class TCETargetCodeGenInfo : public DefaultTargetCodeGenInfo {
8298 public:
TCETargetCodeGenInfo(CodeGenTypes & CGT)8299 TCETargetCodeGenInfo(CodeGenTypes &CGT)
8300 : DefaultTargetCodeGenInfo(CGT) {}
8301
8302 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
8303 CodeGen::CodeGenModule &M) const override;
8304 };
8305
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const8306 void TCETargetCodeGenInfo::setTargetAttributes(
8307 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
8308 if (GV->isDeclaration())
8309 return;
8310 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
8311 if (!FD) return;
8312
8313 llvm::Function *F = cast<llvm::Function>(GV);
8314
8315 if (M.getLangOpts().OpenCL) {
8316 if (FD->hasAttr<OpenCLKernelAttr>()) {
8317 // OpenCL C Kernel functions are not subject to inlining
8318 F->addFnAttr(llvm::Attribute::NoInline);
8319 const ReqdWorkGroupSizeAttr *Attr = FD->getAttr<ReqdWorkGroupSizeAttr>();
8320 if (Attr) {
8321 // Convert the reqd_work_group_size() attributes to metadata.
8322 llvm::LLVMContext &Context = F->getContext();
8323 llvm::NamedMDNode *OpenCLMetadata =
8324 M.getModule().getOrInsertNamedMetadata(
8325 "opencl.kernel_wg_size_info");
8326
8327 SmallVector<llvm::Metadata *, 5> Operands;
8328 Operands.push_back(llvm::ConstantAsMetadata::get(F));
8329
8330 Operands.push_back(
8331 llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue(
8332 M.Int32Ty, llvm::APInt(32, Attr->getXDim()))));
8333 Operands.push_back(
8334 llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue(
8335 M.Int32Ty, llvm::APInt(32, Attr->getYDim()))));
8336 Operands.push_back(
8337 llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue(
8338 M.Int32Ty, llvm::APInt(32, Attr->getZDim()))));
8339
8340 // Add a boolean constant operand for "required" (true) or "hint"
8341 // (false) for implementing the work_group_size_hint attr later.
8342 // Currently always true as the hint is not yet implemented.
8343 Operands.push_back(
8344 llvm::ConstantAsMetadata::get(llvm::ConstantInt::getTrue(Context)));
8345 OpenCLMetadata->addOperand(llvm::MDNode::get(Context, Operands));
8346 }
8347 }
8348 }
8349 }
8350
8351 }
8352
8353 //===----------------------------------------------------------------------===//
8354 // Hexagon ABI Implementation
8355 //===----------------------------------------------------------------------===//
8356
8357 namespace {
8358
8359 class HexagonABIInfo : public DefaultABIInfo {
8360 public:
HexagonABIInfo(CodeGenTypes & CGT)8361 HexagonABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
8362
8363 private:
8364 ABIArgInfo classifyReturnType(QualType RetTy) const;
8365 ABIArgInfo classifyArgumentType(QualType RetTy) const;
8366 ABIArgInfo classifyArgumentType(QualType RetTy, unsigned *RegsLeft) const;
8367
8368 void computeInfo(CGFunctionInfo &FI) const override;
8369
8370 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
8371 QualType Ty) const override;
8372 Address EmitVAArgFromMemory(CodeGenFunction &CFG, Address VAListAddr,
8373 QualType Ty) const;
8374 Address EmitVAArgForHexagon(CodeGenFunction &CFG, Address VAListAddr,
8375 QualType Ty) const;
8376 Address EmitVAArgForHexagonLinux(CodeGenFunction &CFG, Address VAListAddr,
8377 QualType Ty) const;
8378 };
8379
8380 class HexagonTargetCodeGenInfo : public TargetCodeGenInfo {
8381 public:
HexagonTargetCodeGenInfo(CodeGenTypes & CGT)8382 HexagonTargetCodeGenInfo(CodeGenTypes &CGT)
8383 : TargetCodeGenInfo(std::make_unique<HexagonABIInfo>(CGT)) {}
8384
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const8385 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
8386 return 29;
8387 }
8388
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & GCM) const8389 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
8390 CodeGen::CodeGenModule &GCM) const override {
8391 if (GV->isDeclaration())
8392 return;
8393 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
8394 if (!FD)
8395 return;
8396 }
8397 };
8398
8399 } // namespace
8400
computeInfo(CGFunctionInfo & FI) const8401 void HexagonABIInfo::computeInfo(CGFunctionInfo &FI) const {
8402 unsigned RegsLeft = 6;
8403 if (!getCXXABI().classifyReturnType(FI))
8404 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
8405 for (auto &I : FI.arguments())
8406 I.info = classifyArgumentType(I.type, &RegsLeft);
8407 }
8408
HexagonAdjustRegsLeft(uint64_t Size,unsigned * RegsLeft)8409 static bool HexagonAdjustRegsLeft(uint64_t Size, unsigned *RegsLeft) {
8410 assert(Size <= 64 && "Not expecting to pass arguments larger than 64 bits"
8411 " through registers");
8412
8413 if (*RegsLeft == 0)
8414 return false;
8415
8416 if (Size <= 32) {
8417 (*RegsLeft)--;
8418 return true;
8419 }
8420
8421 if (2 <= (*RegsLeft & (~1U))) {
8422 *RegsLeft = (*RegsLeft & (~1U)) - 2;
8423 return true;
8424 }
8425
8426 // Next available register was r5 but candidate was greater than 32-bits so it
8427 // has to go on the stack. However we still consume r5
8428 if (*RegsLeft == 1)
8429 *RegsLeft = 0;
8430
8431 return false;
8432 }
8433
classifyArgumentType(QualType Ty,unsigned * RegsLeft) const8434 ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty,
8435 unsigned *RegsLeft) const {
8436 if (!isAggregateTypeForABI(Ty)) {
8437 // Treat an enum type as its underlying type.
8438 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
8439 Ty = EnumTy->getDecl()->getIntegerType();
8440
8441 uint64_t Size = getContext().getTypeSize(Ty);
8442 if (Size <= 64)
8443 HexagonAdjustRegsLeft(Size, RegsLeft);
8444
8445 if (Size > 64 && Ty->isExtIntType())
8446 return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
8447
8448 return isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
8449 : ABIArgInfo::getDirect();
8450 }
8451
8452 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
8453 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
8454
8455 // Ignore empty records.
8456 if (isEmptyRecord(getContext(), Ty, true))
8457 return ABIArgInfo::getIgnore();
8458
8459 uint64_t Size = getContext().getTypeSize(Ty);
8460 unsigned Align = getContext().getTypeAlign(Ty);
8461
8462 if (Size > 64)
8463 return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
8464
8465 if (HexagonAdjustRegsLeft(Size, RegsLeft))
8466 Align = Size <= 32 ? 32 : 64;
8467 if (Size <= Align) {
8468 // Pass in the smallest viable integer type.
8469 if (!llvm::isPowerOf2_64(Size))
8470 Size = llvm::NextPowerOf2(Size);
8471 return ABIArgInfo::getDirect(llvm::Type::getIntNTy(getVMContext(), Size));
8472 }
8473 return DefaultABIInfo::classifyArgumentType(Ty);
8474 }
8475
classifyReturnType(QualType RetTy) const8476 ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const {
8477 if (RetTy->isVoidType())
8478 return ABIArgInfo::getIgnore();
8479
8480 const TargetInfo &T = CGT.getTarget();
8481 uint64_t Size = getContext().getTypeSize(RetTy);
8482
8483 if (RetTy->getAs<VectorType>()) {
8484 // HVX vectors are returned in vector registers or register pairs.
8485 if (T.hasFeature("hvx")) {
8486 assert(T.hasFeature("hvx-length64b") || T.hasFeature("hvx-length128b"));
8487 uint64_t VecSize = T.hasFeature("hvx-length64b") ? 64*8 : 128*8;
8488 if (Size == VecSize || Size == 2*VecSize)
8489 return ABIArgInfo::getDirectInReg();
8490 }
8491 // Large vector types should be returned via memory.
8492 if (Size > 64)
8493 return getNaturalAlignIndirect(RetTy);
8494 }
8495
8496 if (!isAggregateTypeForABI(RetTy)) {
8497 // Treat an enum type as its underlying type.
8498 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
8499 RetTy = EnumTy->getDecl()->getIntegerType();
8500
8501 if (Size > 64 && RetTy->isExtIntType())
8502 return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
8503
8504 return isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
8505 : ABIArgInfo::getDirect();
8506 }
8507
8508 if (isEmptyRecord(getContext(), RetTy, true))
8509 return ABIArgInfo::getIgnore();
8510
8511 // Aggregates <= 8 bytes are returned in registers, other aggregates
8512 // are returned indirectly.
8513 if (Size <= 64) {
8514 // Return in the smallest viable integer type.
8515 if (!llvm::isPowerOf2_64(Size))
8516 Size = llvm::NextPowerOf2(Size);
8517 return ABIArgInfo::getDirect(llvm::Type::getIntNTy(getVMContext(), Size));
8518 }
8519 return getNaturalAlignIndirect(RetTy, /*ByVal=*/true);
8520 }
8521
EmitVAArgFromMemory(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const8522 Address HexagonABIInfo::EmitVAArgFromMemory(CodeGenFunction &CGF,
8523 Address VAListAddr,
8524 QualType Ty) const {
8525 // Load the overflow area pointer.
8526 Address __overflow_area_pointer_p =
8527 CGF.Builder.CreateStructGEP(VAListAddr, 2, "__overflow_area_pointer_p");
8528 llvm::Value *__overflow_area_pointer = CGF.Builder.CreateLoad(
8529 __overflow_area_pointer_p, "__overflow_area_pointer");
8530
8531 uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8;
8532 if (Align > 4) {
8533 // Alignment should be a power of 2.
8534 assert((Align & (Align - 1)) == 0 && "Alignment is not power of 2!");
8535
8536 // overflow_arg_area = (overflow_arg_area + align - 1) & -align;
8537 llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int64Ty, Align - 1);
8538
8539 // Add offset to the current pointer to access the argument.
8540 __overflow_area_pointer =
8541 CGF.Builder.CreateGEP(CGF.Int8Ty, __overflow_area_pointer, Offset);
8542 llvm::Value *AsInt =
8543 CGF.Builder.CreatePtrToInt(__overflow_area_pointer, CGF.Int32Ty);
8544
8545 // Create a mask which should be "AND"ed
8546 // with (overflow_arg_area + align - 1)
8547 llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int32Ty, -(int)Align);
8548 __overflow_area_pointer = CGF.Builder.CreateIntToPtr(
8549 CGF.Builder.CreateAnd(AsInt, Mask), __overflow_area_pointer->getType(),
8550 "__overflow_area_pointer.align");
8551 }
8552
8553 // Get the type of the argument from memory and bitcast
8554 // overflow area pointer to the argument type.
8555 llvm::Type *PTy = CGF.ConvertTypeForMem(Ty);
8556 Address AddrTyped = CGF.Builder.CreateBitCast(
8557 Address(__overflow_area_pointer, CharUnits::fromQuantity(Align)),
8558 llvm::PointerType::getUnqual(PTy));
8559
8560 // Round up to the minimum stack alignment for varargs which is 4 bytes.
8561 uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4);
8562
8563 __overflow_area_pointer = CGF.Builder.CreateGEP(
8564 CGF.Int8Ty, __overflow_area_pointer,
8565 llvm::ConstantInt::get(CGF.Int32Ty, Offset),
8566 "__overflow_area_pointer.next");
8567 CGF.Builder.CreateStore(__overflow_area_pointer, __overflow_area_pointer_p);
8568
8569 return AddrTyped;
8570 }
8571
EmitVAArgForHexagon(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const8572 Address HexagonABIInfo::EmitVAArgForHexagon(CodeGenFunction &CGF,
8573 Address VAListAddr,
8574 QualType Ty) const {
8575 // FIXME: Need to handle alignment
8576 llvm::Type *BP = CGF.Int8PtrTy;
8577 llvm::Type *BPP = CGF.Int8PtrPtrTy;
8578 CGBuilderTy &Builder = CGF.Builder;
8579 Address VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
8580 llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
8581 // Handle address alignment for type alignment > 32 bits
8582 uint64_t TyAlign = CGF.getContext().getTypeAlign(Ty) / 8;
8583 if (TyAlign > 4) {
8584 assert((TyAlign & (TyAlign - 1)) == 0 && "Alignment is not power of 2!");
8585 llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int32Ty);
8586 AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt32(TyAlign - 1));
8587 AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt32(~(TyAlign - 1)));
8588 Addr = Builder.CreateIntToPtr(AddrAsInt, BP);
8589 }
8590 llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
8591 Address AddrTyped = Builder.CreateBitCast(
8592 Address(Addr, CharUnits::fromQuantity(TyAlign)), PTy);
8593
8594 uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4);
8595 llvm::Value *NextAddr = Builder.CreateGEP(
8596 CGF.Int8Ty, Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), "ap.next");
8597 Builder.CreateStore(NextAddr, VAListAddrAsBPP);
8598
8599 return AddrTyped;
8600 }
8601
EmitVAArgForHexagonLinux(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const8602 Address HexagonABIInfo::EmitVAArgForHexagonLinux(CodeGenFunction &CGF,
8603 Address VAListAddr,
8604 QualType Ty) const {
8605 int ArgSize = CGF.getContext().getTypeSize(Ty) / 8;
8606
8607 if (ArgSize > 8)
8608 return EmitVAArgFromMemory(CGF, VAListAddr, Ty);
8609
8610 // Here we have check if the argument is in register area or
8611 // in overflow area.
8612 // If the saved register area pointer + argsize rounded up to alignment >
8613 // saved register area end pointer, argument is in overflow area.
8614 unsigned RegsLeft = 6;
8615 Ty = CGF.getContext().getCanonicalType(Ty);
8616 (void)classifyArgumentType(Ty, &RegsLeft);
8617
8618 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg");
8619 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
8620 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");
8621 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
8622
8623 // Get rounded size of the argument.GCC does not allow vararg of
8624 // size < 4 bytes. We follow the same logic here.
8625 ArgSize = (CGF.getContext().getTypeSize(Ty) <= 32) ? 4 : 8;
8626 int ArgAlign = (CGF.getContext().getTypeSize(Ty) <= 32) ? 4 : 8;
8627
8628 // Argument may be in saved register area
8629 CGF.EmitBlock(MaybeRegBlock);
8630
8631 // Load the current saved register area pointer.
8632 Address __current_saved_reg_area_pointer_p = CGF.Builder.CreateStructGEP(
8633 VAListAddr, 0, "__current_saved_reg_area_pointer_p");
8634 llvm::Value *__current_saved_reg_area_pointer = CGF.Builder.CreateLoad(
8635 __current_saved_reg_area_pointer_p, "__current_saved_reg_area_pointer");
8636
8637 // Load the saved register area end pointer.
8638 Address __saved_reg_area_end_pointer_p = CGF.Builder.CreateStructGEP(
8639 VAListAddr, 1, "__saved_reg_area_end_pointer_p");
8640 llvm::Value *__saved_reg_area_end_pointer = CGF.Builder.CreateLoad(
8641 __saved_reg_area_end_pointer_p, "__saved_reg_area_end_pointer");
8642
8643 // If the size of argument is > 4 bytes, check if the stack
8644 // location is aligned to 8 bytes
8645 if (ArgAlign > 4) {
8646
8647 llvm::Value *__current_saved_reg_area_pointer_int =
8648 CGF.Builder.CreatePtrToInt(__current_saved_reg_area_pointer,
8649 CGF.Int32Ty);
8650
8651 __current_saved_reg_area_pointer_int = CGF.Builder.CreateAdd(
8652 __current_saved_reg_area_pointer_int,
8653 llvm::ConstantInt::get(CGF.Int32Ty, (ArgAlign - 1)),
8654 "align_current_saved_reg_area_pointer");
8655
8656 __current_saved_reg_area_pointer_int =
8657 CGF.Builder.CreateAnd(__current_saved_reg_area_pointer_int,
8658 llvm::ConstantInt::get(CGF.Int32Ty, -ArgAlign),
8659 "align_current_saved_reg_area_pointer");
8660
8661 __current_saved_reg_area_pointer =
8662 CGF.Builder.CreateIntToPtr(__current_saved_reg_area_pointer_int,
8663 __current_saved_reg_area_pointer->getType(),
8664 "align_current_saved_reg_area_pointer");
8665 }
8666
8667 llvm::Value *__new_saved_reg_area_pointer =
8668 CGF.Builder.CreateGEP(CGF.Int8Ty, __current_saved_reg_area_pointer,
8669 llvm::ConstantInt::get(CGF.Int32Ty, ArgSize),
8670 "__new_saved_reg_area_pointer");
8671
8672 llvm::Value *UsingStack = 0;
8673 UsingStack = CGF.Builder.CreateICmpSGT(__new_saved_reg_area_pointer,
8674 __saved_reg_area_end_pointer);
8675
8676 CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, InRegBlock);
8677
8678 // Argument in saved register area
8679 // Implement the block where argument is in register saved area
8680 CGF.EmitBlock(InRegBlock);
8681
8682 llvm::Type *PTy = CGF.ConvertType(Ty);
8683 llvm::Value *__saved_reg_area_p = CGF.Builder.CreateBitCast(
8684 __current_saved_reg_area_pointer, llvm::PointerType::getUnqual(PTy));
8685
8686 CGF.Builder.CreateStore(__new_saved_reg_area_pointer,
8687 __current_saved_reg_area_pointer_p);
8688
8689 CGF.EmitBranch(ContBlock);
8690
8691 // Argument in overflow area
8692 // Implement the block where the argument is in overflow area.
8693 CGF.EmitBlock(OnStackBlock);
8694
8695 // Load the overflow area pointer
8696 Address __overflow_area_pointer_p =
8697 CGF.Builder.CreateStructGEP(VAListAddr, 2, "__overflow_area_pointer_p");
8698 llvm::Value *__overflow_area_pointer = CGF.Builder.CreateLoad(
8699 __overflow_area_pointer_p, "__overflow_area_pointer");
8700
8701 // Align the overflow area pointer according to the alignment of the argument
8702 if (ArgAlign > 4) {
8703 llvm::Value *__overflow_area_pointer_int =
8704 CGF.Builder.CreatePtrToInt(__overflow_area_pointer, CGF.Int32Ty);
8705
8706 __overflow_area_pointer_int =
8707 CGF.Builder.CreateAdd(__overflow_area_pointer_int,
8708 llvm::ConstantInt::get(CGF.Int32Ty, ArgAlign - 1),
8709 "align_overflow_area_pointer");
8710
8711 __overflow_area_pointer_int =
8712 CGF.Builder.CreateAnd(__overflow_area_pointer_int,
8713 llvm::ConstantInt::get(CGF.Int32Ty, -ArgAlign),
8714 "align_overflow_area_pointer");
8715
8716 __overflow_area_pointer = CGF.Builder.CreateIntToPtr(
8717 __overflow_area_pointer_int, __overflow_area_pointer->getType(),
8718 "align_overflow_area_pointer");
8719 }
8720
8721 // Get the pointer for next argument in overflow area and store it
8722 // to overflow area pointer.
8723 llvm::Value *__new_overflow_area_pointer = CGF.Builder.CreateGEP(
8724 CGF.Int8Ty, __overflow_area_pointer,
8725 llvm::ConstantInt::get(CGF.Int32Ty, ArgSize),
8726 "__overflow_area_pointer.next");
8727
8728 CGF.Builder.CreateStore(__new_overflow_area_pointer,
8729 __overflow_area_pointer_p);
8730
8731 CGF.Builder.CreateStore(__new_overflow_area_pointer,
8732 __current_saved_reg_area_pointer_p);
8733
8734 // Bitcast the overflow area pointer to the type of argument.
8735 llvm::Type *OverflowPTy = CGF.ConvertTypeForMem(Ty);
8736 llvm::Value *__overflow_area_p = CGF.Builder.CreateBitCast(
8737 __overflow_area_pointer, llvm::PointerType::getUnqual(OverflowPTy));
8738
8739 CGF.EmitBranch(ContBlock);
8740
8741 // Get the correct pointer to load the variable argument
8742 // Implement the ContBlock
8743 CGF.EmitBlock(ContBlock);
8744
8745 llvm::Type *MemPTy = llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
8746 llvm::PHINode *ArgAddr = CGF.Builder.CreatePHI(MemPTy, 2, "vaarg.addr");
8747 ArgAddr->addIncoming(__saved_reg_area_p, InRegBlock);
8748 ArgAddr->addIncoming(__overflow_area_p, OnStackBlock);
8749
8750 return Address(ArgAddr, CharUnits::fromQuantity(ArgAlign));
8751 }
8752
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const8753 Address HexagonABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
8754 QualType Ty) const {
8755
8756 if (getTarget().getTriple().isMusl())
8757 return EmitVAArgForHexagonLinux(CGF, VAListAddr, Ty);
8758
8759 return EmitVAArgForHexagon(CGF, VAListAddr, Ty);
8760 }
8761
8762 //===----------------------------------------------------------------------===//
8763 // Lanai ABI Implementation
8764 //===----------------------------------------------------------------------===//
8765
8766 namespace {
8767 class LanaiABIInfo : public DefaultABIInfo {
8768 public:
LanaiABIInfo(CodeGen::CodeGenTypes & CGT)8769 LanaiABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
8770
8771 bool shouldUseInReg(QualType Ty, CCState &State) const;
8772
computeInfo(CGFunctionInfo & FI) const8773 void computeInfo(CGFunctionInfo &FI) const override {
8774 CCState State(FI);
8775 // Lanai uses 4 registers to pass arguments unless the function has the
8776 // regparm attribute set.
8777 if (FI.getHasRegParm()) {
8778 State.FreeRegs = FI.getRegParm();
8779 } else {
8780 State.FreeRegs = 4;
8781 }
8782
8783 if (!getCXXABI().classifyReturnType(FI))
8784 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
8785 for (auto &I : FI.arguments())
8786 I.info = classifyArgumentType(I.type, State);
8787 }
8788
8789 ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
8790 ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
8791 };
8792 } // end anonymous namespace
8793
shouldUseInReg(QualType Ty,CCState & State) const8794 bool LanaiABIInfo::shouldUseInReg(QualType Ty, CCState &State) const {
8795 unsigned Size = getContext().getTypeSize(Ty);
8796 unsigned SizeInRegs = llvm::alignTo(Size, 32U) / 32U;
8797
8798 if (SizeInRegs == 0)
8799 return false;
8800
8801 if (SizeInRegs > State.FreeRegs) {
8802 State.FreeRegs = 0;
8803 return false;
8804 }
8805
8806 State.FreeRegs -= SizeInRegs;
8807
8808 return true;
8809 }
8810
getIndirectResult(QualType Ty,bool ByVal,CCState & State) const8811 ABIArgInfo LanaiABIInfo::getIndirectResult(QualType Ty, bool ByVal,
8812 CCState &State) const {
8813 if (!ByVal) {
8814 if (State.FreeRegs) {
8815 --State.FreeRegs; // Non-byval indirects just use one pointer.
8816 return getNaturalAlignIndirectInReg(Ty);
8817 }
8818 return getNaturalAlignIndirect(Ty, false);
8819 }
8820
8821 // Compute the byval alignment.
8822 const unsigned MinABIStackAlignInBytes = 4;
8823 unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
8824 return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true,
8825 /*Realign=*/TypeAlign >
8826 MinABIStackAlignInBytes);
8827 }
8828
classifyArgumentType(QualType Ty,CCState & State) const8829 ABIArgInfo LanaiABIInfo::classifyArgumentType(QualType Ty,
8830 CCState &State) const {
8831 // Check with the C++ ABI first.
8832 const RecordType *RT = Ty->getAs<RecordType>();
8833 if (RT) {
8834 CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
8835 if (RAA == CGCXXABI::RAA_Indirect) {
8836 return getIndirectResult(Ty, /*ByVal=*/false, State);
8837 } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
8838 return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
8839 }
8840 }
8841
8842 if (isAggregateTypeForABI(Ty)) {
8843 // Structures with flexible arrays are always indirect.
8844 if (RT && RT->getDecl()->hasFlexibleArrayMember())
8845 return getIndirectResult(Ty, /*ByVal=*/true, State);
8846
8847 // Ignore empty structs/unions.
8848 if (isEmptyRecord(getContext(), Ty, true))
8849 return ABIArgInfo::getIgnore();
8850
8851 llvm::LLVMContext &LLVMContext = getVMContext();
8852 unsigned SizeInRegs = (getContext().getTypeSize(Ty) + 31) / 32;
8853 if (SizeInRegs <= State.FreeRegs) {
8854 llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
8855 SmallVector<llvm::Type *, 3> Elements(SizeInRegs, Int32);
8856 llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
8857 State.FreeRegs -= SizeInRegs;
8858 return ABIArgInfo::getDirectInReg(Result);
8859 } else {
8860 State.FreeRegs = 0;
8861 }
8862 return getIndirectResult(Ty, true, State);
8863 }
8864
8865 // Treat an enum type as its underlying type.
8866 if (const auto *EnumTy = Ty->getAs<EnumType>())
8867 Ty = EnumTy->getDecl()->getIntegerType();
8868
8869 bool InReg = shouldUseInReg(Ty, State);
8870
8871 // Don't pass >64 bit integers in registers.
8872 if (const auto *EIT = Ty->getAs<ExtIntType>())
8873 if (EIT->getNumBits() > 64)
8874 return getIndirectResult(Ty, /*ByVal=*/true, State);
8875
8876 if (isPromotableIntegerTypeForABI(Ty)) {
8877 if (InReg)
8878 return ABIArgInfo::getDirectInReg();
8879 return ABIArgInfo::getExtend(Ty);
8880 }
8881 if (InReg)
8882 return ABIArgInfo::getDirectInReg();
8883 return ABIArgInfo::getDirect();
8884 }
8885
8886 namespace {
8887 class LanaiTargetCodeGenInfo : public TargetCodeGenInfo {
8888 public:
LanaiTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT)8889 LanaiTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
8890 : TargetCodeGenInfo(std::make_unique<LanaiABIInfo>(CGT)) {}
8891 };
8892 }
8893
8894 //===----------------------------------------------------------------------===//
8895 // AMDGPU ABI Implementation
8896 //===----------------------------------------------------------------------===//
8897
8898 namespace {
8899
8900 class AMDGPUABIInfo final : public DefaultABIInfo {
8901 private:
8902 static const unsigned MaxNumRegsForArgsRet = 16;
8903
8904 unsigned numRegsForType(QualType Ty) const;
8905
8906 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
8907 bool isHomogeneousAggregateSmallEnough(const Type *Base,
8908 uint64_t Members) const override;
8909
8910 // Coerce HIP scalar pointer arguments from generic pointers to global ones.
coerceKernelArgumentType(llvm::Type * Ty,unsigned FromAS,unsigned ToAS) const8911 llvm::Type *coerceKernelArgumentType(llvm::Type *Ty, unsigned FromAS,
8912 unsigned ToAS) const {
8913 // Single value types.
8914 if (Ty->isPointerTy() && Ty->getPointerAddressSpace() == FromAS)
8915 return llvm::PointerType::get(
8916 cast<llvm::PointerType>(Ty)->getElementType(), ToAS);
8917 return Ty;
8918 }
8919
8920 public:
AMDGPUABIInfo(CodeGen::CodeGenTypes & CGT)8921 explicit AMDGPUABIInfo(CodeGen::CodeGenTypes &CGT) :
8922 DefaultABIInfo(CGT) {}
8923
8924 ABIArgInfo classifyReturnType(QualType RetTy) const;
8925 ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
8926 ABIArgInfo classifyArgumentType(QualType Ty, unsigned &NumRegsLeft) const;
8927
8928 void computeInfo(CGFunctionInfo &FI) const override;
8929 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
8930 QualType Ty) const override;
8931 };
8932
isHomogeneousAggregateBaseType(QualType Ty) const8933 bool AMDGPUABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
8934 return true;
8935 }
8936
isHomogeneousAggregateSmallEnough(const Type * Base,uint64_t Members) const8937 bool AMDGPUABIInfo::isHomogeneousAggregateSmallEnough(
8938 const Type *Base, uint64_t Members) const {
8939 uint32_t NumRegs = (getContext().getTypeSize(Base) + 31) / 32;
8940
8941 // Homogeneous Aggregates may occupy at most 16 registers.
8942 return Members * NumRegs <= MaxNumRegsForArgsRet;
8943 }
8944
8945 /// Estimate number of registers the type will use when passed in registers.
numRegsForType(QualType Ty) const8946 unsigned AMDGPUABIInfo::numRegsForType(QualType Ty) const {
8947 unsigned NumRegs = 0;
8948
8949 if (const VectorType *VT = Ty->getAs<VectorType>()) {
8950 // Compute from the number of elements. The reported size is based on the
8951 // in-memory size, which includes the padding 4th element for 3-vectors.
8952 QualType EltTy = VT->getElementType();
8953 unsigned EltSize = getContext().getTypeSize(EltTy);
8954
8955 // 16-bit element vectors should be passed as packed.
8956 if (EltSize == 16)
8957 return (VT->getNumElements() + 1) / 2;
8958
8959 unsigned EltNumRegs = (EltSize + 31) / 32;
8960 return EltNumRegs * VT->getNumElements();
8961 }
8962
8963 if (const RecordType *RT = Ty->getAs<RecordType>()) {
8964 const RecordDecl *RD = RT->getDecl();
8965 assert(!RD->hasFlexibleArrayMember());
8966
8967 for (const FieldDecl *Field : RD->fields()) {
8968 QualType FieldTy = Field->getType();
8969 NumRegs += numRegsForType(FieldTy);
8970 }
8971
8972 return NumRegs;
8973 }
8974
8975 return (getContext().getTypeSize(Ty) + 31) / 32;
8976 }
8977
computeInfo(CGFunctionInfo & FI) const8978 void AMDGPUABIInfo::computeInfo(CGFunctionInfo &FI) const {
8979 llvm::CallingConv::ID CC = FI.getCallingConvention();
8980
8981 if (!getCXXABI().classifyReturnType(FI))
8982 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
8983
8984 unsigned NumRegsLeft = MaxNumRegsForArgsRet;
8985 for (auto &Arg : FI.arguments()) {
8986 if (CC == llvm::CallingConv::AMDGPU_KERNEL) {
8987 Arg.info = classifyKernelArgumentType(Arg.type);
8988 } else {
8989 Arg.info = classifyArgumentType(Arg.type, NumRegsLeft);
8990 }
8991 }
8992 }
8993
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const8994 Address AMDGPUABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
8995 QualType Ty) const {
8996 llvm_unreachable("AMDGPU does not support varargs");
8997 }
8998
classifyReturnType(QualType RetTy) const8999 ABIArgInfo AMDGPUABIInfo::classifyReturnType(QualType RetTy) const {
9000 if (isAggregateTypeForABI(RetTy)) {
9001 // Records with non-trivial destructors/copy-constructors should not be
9002 // returned by value.
9003 if (!getRecordArgABI(RetTy, getCXXABI())) {
9004 // Ignore empty structs/unions.
9005 if (isEmptyRecord(getContext(), RetTy, true))
9006 return ABIArgInfo::getIgnore();
9007
9008 // Lower single-element structs to just return a regular value.
9009 if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
9010 return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
9011
9012 if (const RecordType *RT = RetTy->getAs<RecordType>()) {
9013 const RecordDecl *RD = RT->getDecl();
9014 if (RD->hasFlexibleArrayMember())
9015 return DefaultABIInfo::classifyReturnType(RetTy);
9016 }
9017
9018 // Pack aggregates <= 4 bytes into single VGPR or pair.
9019 uint64_t Size = getContext().getTypeSize(RetTy);
9020 if (Size <= 16)
9021 return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
9022
9023 if (Size <= 32)
9024 return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
9025
9026 if (Size <= 64) {
9027 llvm::Type *I32Ty = llvm::Type::getInt32Ty(getVMContext());
9028 return ABIArgInfo::getDirect(llvm::ArrayType::get(I32Ty, 2));
9029 }
9030
9031 if (numRegsForType(RetTy) <= MaxNumRegsForArgsRet)
9032 return ABIArgInfo::getDirect();
9033 }
9034 }
9035
9036 // Otherwise just do the default thing.
9037 return DefaultABIInfo::classifyReturnType(RetTy);
9038 }
9039
9040 /// For kernels all parameters are really passed in a special buffer. It doesn't
9041 /// make sense to pass anything byval, so everything must be direct.
classifyKernelArgumentType(QualType Ty) const9042 ABIArgInfo AMDGPUABIInfo::classifyKernelArgumentType(QualType Ty) const {
9043 Ty = useFirstFieldIfTransparentUnion(Ty);
9044
9045 // TODO: Can we omit empty structs?
9046
9047 if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
9048 Ty = QualType(SeltTy, 0);
9049
9050 llvm::Type *OrigLTy = CGT.ConvertType(Ty);
9051 llvm::Type *LTy = OrigLTy;
9052 if (getContext().getLangOpts().HIP) {
9053 LTy = coerceKernelArgumentType(
9054 OrigLTy, /*FromAS=*/getContext().getTargetAddressSpace(LangAS::Default),
9055 /*ToAS=*/getContext().getTargetAddressSpace(LangAS::cuda_device));
9056 }
9057
9058 // FIXME: Should also use this for OpenCL, but it requires addressing the
9059 // problem of kernels being called.
9060 //
9061 // FIXME: This doesn't apply the optimization of coercing pointers in structs
9062 // to global address space when using byref. This would require implementing a
9063 // new kind of coercion of the in-memory type when for indirect arguments.
9064 if (!getContext().getLangOpts().OpenCL && LTy == OrigLTy &&
9065 isAggregateTypeForABI(Ty)) {
9066 return ABIArgInfo::getIndirectAliased(
9067 getContext().getTypeAlignInChars(Ty),
9068 getContext().getTargetAddressSpace(LangAS::opencl_constant),
9069 false /*Realign*/, nullptr /*Padding*/);
9070 }
9071
9072 // If we set CanBeFlattened to true, CodeGen will expand the struct to its
9073 // individual elements, which confuses the Clover OpenCL backend; therefore we
9074 // have to set it to false here. Other args of getDirect() are just defaults.
9075 return ABIArgInfo::getDirect(LTy, 0, nullptr, false);
9076 }
9077
classifyArgumentType(QualType Ty,unsigned & NumRegsLeft) const9078 ABIArgInfo AMDGPUABIInfo::classifyArgumentType(QualType Ty,
9079 unsigned &NumRegsLeft) const {
9080 assert(NumRegsLeft <= MaxNumRegsForArgsRet && "register estimate underflow");
9081
9082 Ty = useFirstFieldIfTransparentUnion(Ty);
9083
9084 if (isAggregateTypeForABI(Ty)) {
9085 // Records with non-trivial destructors/copy-constructors should not be
9086 // passed by value.
9087 if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
9088 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
9089
9090 // Ignore empty structs/unions.
9091 if (isEmptyRecord(getContext(), Ty, true))
9092 return ABIArgInfo::getIgnore();
9093
9094 // Lower single-element structs to just pass a regular value. TODO: We
9095 // could do reasonable-size multiple-element structs too, using getExpand(),
9096 // though watch out for things like bitfields.
9097 if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
9098 return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
9099
9100 if (const RecordType *RT = Ty->getAs<RecordType>()) {
9101 const RecordDecl *RD = RT->getDecl();
9102 if (RD->hasFlexibleArrayMember())
9103 return DefaultABIInfo::classifyArgumentType(Ty);
9104 }
9105
9106 // Pack aggregates <= 8 bytes into single VGPR or pair.
9107 uint64_t Size = getContext().getTypeSize(Ty);
9108 if (Size <= 64) {
9109 unsigned NumRegs = (Size + 31) / 32;
9110 NumRegsLeft -= std::min(NumRegsLeft, NumRegs);
9111
9112 if (Size <= 16)
9113 return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
9114
9115 if (Size <= 32)
9116 return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
9117
9118 // XXX: Should this be i64 instead, and should the limit increase?
9119 llvm::Type *I32Ty = llvm::Type::getInt32Ty(getVMContext());
9120 return ABIArgInfo::getDirect(llvm::ArrayType::get(I32Ty, 2));
9121 }
9122
9123 if (NumRegsLeft > 0) {
9124 unsigned NumRegs = numRegsForType(Ty);
9125 if (NumRegsLeft >= NumRegs) {
9126 NumRegsLeft -= NumRegs;
9127 return ABIArgInfo::getDirect();
9128 }
9129 }
9130 }
9131
9132 // Otherwise just do the default thing.
9133 ABIArgInfo ArgInfo = DefaultABIInfo::classifyArgumentType(Ty);
9134 if (!ArgInfo.isIndirect()) {
9135 unsigned NumRegs = numRegsForType(Ty);
9136 NumRegsLeft -= std::min(NumRegs, NumRegsLeft);
9137 }
9138
9139 return ArgInfo;
9140 }
9141
9142 class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo {
9143 public:
AMDGPUTargetCodeGenInfo(CodeGenTypes & CGT)9144 AMDGPUTargetCodeGenInfo(CodeGenTypes &CGT)
9145 : TargetCodeGenInfo(std::make_unique<AMDGPUABIInfo>(CGT)) {}
9146 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
9147 CodeGen::CodeGenModule &M) const override;
9148 unsigned getOpenCLKernelCallingConv() const override;
9149
9150 llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM,
9151 llvm::PointerType *T, QualType QT) const override;
9152
getASTAllocaAddressSpace() const9153 LangAS getASTAllocaAddressSpace() const override {
9154 return getLangASFromTargetAS(
9155 getABIInfo().getDataLayout().getAllocaAddrSpace());
9156 }
9157 LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
9158 const VarDecl *D) const override;
9159 llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts,
9160 SyncScope Scope,
9161 llvm::AtomicOrdering Ordering,
9162 llvm::LLVMContext &Ctx) const override;
9163 llvm::Function *
9164 createEnqueuedBlockKernel(CodeGenFunction &CGF,
9165 llvm::Function *BlockInvokeFunc,
9166 llvm::Value *BlockLiteral) const override;
9167 bool shouldEmitStaticExternCAliases() const override;
9168 void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
9169 };
9170 }
9171
requiresAMDGPUProtectedVisibility(const Decl * D,llvm::GlobalValue * GV)9172 static bool requiresAMDGPUProtectedVisibility(const Decl *D,
9173 llvm::GlobalValue *GV) {
9174 if (GV->getVisibility() != llvm::GlobalValue::HiddenVisibility)
9175 return false;
9176
9177 return D->hasAttr<OpenCLKernelAttr>() ||
9178 (isa<FunctionDecl>(D) && D->hasAttr<CUDAGlobalAttr>()) ||
9179 (isa<VarDecl>(D) &&
9180 (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
9181 cast<VarDecl>(D)->getType()->isCUDADeviceBuiltinSurfaceType() ||
9182 cast<VarDecl>(D)->getType()->isCUDADeviceBuiltinTextureType()));
9183 }
9184
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & M) const9185 void AMDGPUTargetCodeGenInfo::setTargetAttributes(
9186 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
9187 if (requiresAMDGPUProtectedVisibility(D, GV)) {
9188 GV->setVisibility(llvm::GlobalValue::ProtectedVisibility);
9189 GV->setDSOLocal(true);
9190 }
9191
9192 if (GV->isDeclaration())
9193 return;
9194 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
9195 if (!FD)
9196 return;
9197
9198 llvm::Function *F = cast<llvm::Function>(GV);
9199
9200 const auto *ReqdWGS = M.getLangOpts().OpenCL ?
9201 FD->getAttr<ReqdWorkGroupSizeAttr>() : nullptr;
9202
9203
9204 const bool IsOpenCLKernel = M.getLangOpts().OpenCL &&
9205 FD->hasAttr<OpenCLKernelAttr>();
9206 const bool IsHIPKernel = M.getLangOpts().HIP &&
9207 FD->hasAttr<CUDAGlobalAttr>();
9208 if ((IsOpenCLKernel || IsHIPKernel) &&
9209 (M.getTriple().getOS() == llvm::Triple::AMDHSA))
9210 F->addFnAttr("amdgpu-implicitarg-num-bytes", "56");
9211
9212 if (IsHIPKernel)
9213 F->addFnAttr("uniform-work-group-size", "true");
9214
9215
9216 const auto *FlatWGS = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>();
9217 if (ReqdWGS || FlatWGS) {
9218 unsigned Min = 0;
9219 unsigned Max = 0;
9220 if (FlatWGS) {
9221 Min = FlatWGS->getMin()
9222 ->EvaluateKnownConstInt(M.getContext())
9223 .getExtValue();
9224 Max = FlatWGS->getMax()
9225 ->EvaluateKnownConstInt(M.getContext())
9226 .getExtValue();
9227 }
9228 if (ReqdWGS && Min == 0 && Max == 0)
9229 Min = Max = ReqdWGS->getXDim() * ReqdWGS->getYDim() * ReqdWGS->getZDim();
9230
9231 if (Min != 0) {
9232 assert(Min <= Max && "Min must be less than or equal Max");
9233
9234 std::string AttrVal = llvm::utostr(Min) + "," + llvm::utostr(Max);
9235 F->addFnAttr("amdgpu-flat-work-group-size", AttrVal);
9236 } else
9237 assert(Max == 0 && "Max must be zero");
9238 } else if (IsOpenCLKernel || IsHIPKernel) {
9239 // By default, restrict the maximum size to a value specified by
9240 // --gpu-max-threads-per-block=n or its default value for HIP.
9241 const unsigned OpenCLDefaultMaxWorkGroupSize = 256;
9242 const unsigned DefaultMaxWorkGroupSize =
9243 IsOpenCLKernel ? OpenCLDefaultMaxWorkGroupSize
9244 : M.getLangOpts().GPUMaxThreadsPerBlock;
9245 std::string AttrVal =
9246 std::string("1,") + llvm::utostr(DefaultMaxWorkGroupSize);
9247 F->addFnAttr("amdgpu-flat-work-group-size", AttrVal);
9248 }
9249
9250 if (const auto *Attr = FD->getAttr<AMDGPUWavesPerEUAttr>()) {
9251 unsigned Min =
9252 Attr->getMin()->EvaluateKnownConstInt(M.getContext()).getExtValue();
9253 unsigned Max = Attr->getMax() ? Attr->getMax()
9254 ->EvaluateKnownConstInt(M.getContext())
9255 .getExtValue()
9256 : 0;
9257
9258 if (Min != 0) {
9259 assert((Max == 0 || Min <= Max) && "Min must be less than or equal Max");
9260
9261 std::string AttrVal = llvm::utostr(Min);
9262 if (Max != 0)
9263 AttrVal = AttrVal + "," + llvm::utostr(Max);
9264 F->addFnAttr("amdgpu-waves-per-eu", AttrVal);
9265 } else
9266 assert(Max == 0 && "Max must be zero");
9267 }
9268
9269 if (const auto *Attr = FD->getAttr<AMDGPUNumSGPRAttr>()) {
9270 unsigned NumSGPR = Attr->getNumSGPR();
9271
9272 if (NumSGPR != 0)
9273 F->addFnAttr("amdgpu-num-sgpr", llvm::utostr(NumSGPR));
9274 }
9275
9276 if (const auto *Attr = FD->getAttr<AMDGPUNumVGPRAttr>()) {
9277 uint32_t NumVGPR = Attr->getNumVGPR();
9278
9279 if (NumVGPR != 0)
9280 F->addFnAttr("amdgpu-num-vgpr", llvm::utostr(NumVGPR));
9281 }
9282
9283 if (M.getContext().getTargetInfo().allowAMDGPUUnsafeFPAtomics())
9284 F->addFnAttr("amdgpu-unsafe-fp-atomics", "true");
9285
9286 if (!getABIInfo().getCodeGenOpts().EmitIEEENaNCompliantInsts)
9287 F->addFnAttr("amdgpu-ieee", "false");
9288 }
9289
getOpenCLKernelCallingConv() const9290 unsigned AMDGPUTargetCodeGenInfo::getOpenCLKernelCallingConv() const {
9291 return llvm::CallingConv::AMDGPU_KERNEL;
9292 }
9293
9294 // Currently LLVM assumes null pointers always have value 0,
9295 // which results in incorrectly transformed IR. Therefore, instead of
9296 // emitting null pointers in private and local address spaces, a null
9297 // pointer in generic address space is emitted which is casted to a
9298 // pointer in local or private address space.
getNullPointer(const CodeGen::CodeGenModule & CGM,llvm::PointerType * PT,QualType QT) const9299 llvm::Constant *AMDGPUTargetCodeGenInfo::getNullPointer(
9300 const CodeGen::CodeGenModule &CGM, llvm::PointerType *PT,
9301 QualType QT) const {
9302 if (CGM.getContext().getTargetNullPointerValue(QT) == 0)
9303 return llvm::ConstantPointerNull::get(PT);
9304
9305 auto &Ctx = CGM.getContext();
9306 auto NPT = llvm::PointerType::get(PT->getElementType(),
9307 Ctx.getTargetAddressSpace(LangAS::opencl_generic));
9308 return llvm::ConstantExpr::getAddrSpaceCast(
9309 llvm::ConstantPointerNull::get(NPT), PT);
9310 }
9311
9312 LangAS
getGlobalVarAddressSpace(CodeGenModule & CGM,const VarDecl * D) const9313 AMDGPUTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
9314 const VarDecl *D) const {
9315 assert(!CGM.getLangOpts().OpenCL &&
9316 !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
9317 "Address space agnostic languages only");
9318 LangAS DefaultGlobalAS = getLangASFromTargetAS(
9319 CGM.getContext().getTargetAddressSpace(LangAS::opencl_global));
9320 if (!D)
9321 return DefaultGlobalAS;
9322
9323 LangAS AddrSpace = D->getType().getAddressSpace();
9324 assert(AddrSpace == LangAS::Default || isTargetAddressSpace(AddrSpace));
9325 if (AddrSpace != LangAS::Default)
9326 return AddrSpace;
9327
9328 if (CGM.isTypeConstant(D->getType(), false)) {
9329 if (auto ConstAS = CGM.getTarget().getConstantAddressSpace())
9330 return ConstAS.getValue();
9331 }
9332 return DefaultGlobalAS;
9333 }
9334
9335 llvm::SyncScope::ID
getLLVMSyncScopeID(const LangOptions & LangOpts,SyncScope Scope,llvm::AtomicOrdering Ordering,llvm::LLVMContext & Ctx) const9336 AMDGPUTargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
9337 SyncScope Scope,
9338 llvm::AtomicOrdering Ordering,
9339 llvm::LLVMContext &Ctx) const {
9340 std::string Name;
9341 switch (Scope) {
9342 case SyncScope::OpenCLWorkGroup:
9343 Name = "workgroup";
9344 break;
9345 case SyncScope::OpenCLDevice:
9346 Name = "agent";
9347 break;
9348 case SyncScope::OpenCLAllSVMDevices:
9349 Name = "";
9350 break;
9351 case SyncScope::OpenCLSubGroup:
9352 Name = "wavefront";
9353 }
9354
9355 if (Ordering != llvm::AtomicOrdering::SequentiallyConsistent) {
9356 if (!Name.empty())
9357 Name = Twine(Twine(Name) + Twine("-")).str();
9358
9359 Name = Twine(Twine(Name) + Twine("one-as")).str();
9360 }
9361
9362 return Ctx.getOrInsertSyncScopeID(Name);
9363 }
9364
shouldEmitStaticExternCAliases() const9365 bool AMDGPUTargetCodeGenInfo::shouldEmitStaticExternCAliases() const {
9366 return false;
9367 }
9368
setCUDAKernelCallingConvention(const FunctionType * & FT) const9369 void AMDGPUTargetCodeGenInfo::setCUDAKernelCallingConvention(
9370 const FunctionType *&FT) const {
9371 FT = getABIInfo().getContext().adjustFunctionType(
9372 FT, FT->getExtInfo().withCallingConv(CC_OpenCLKernel));
9373 }
9374
9375 //===----------------------------------------------------------------------===//
9376 // SPARC v8 ABI Implementation.
9377 // Based on the SPARC Compliance Definition version 2.4.1.
9378 //
9379 // Ensures that complex values are passed in registers.
9380 //
9381 namespace {
9382 class SparcV8ABIInfo : public DefaultABIInfo {
9383 public:
SparcV8ABIInfo(CodeGenTypes & CGT)9384 SparcV8ABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
9385
9386 private:
9387 ABIArgInfo classifyReturnType(QualType RetTy) const;
9388 void computeInfo(CGFunctionInfo &FI) const override;
9389 };
9390 } // end anonymous namespace
9391
9392
9393 ABIArgInfo
classifyReturnType(QualType Ty) const9394 SparcV8ABIInfo::classifyReturnType(QualType Ty) const {
9395 if (Ty->isAnyComplexType()) {
9396 return ABIArgInfo::getDirect();
9397 }
9398 else {
9399 return DefaultABIInfo::classifyReturnType(Ty);
9400 }
9401 }
9402
computeInfo(CGFunctionInfo & FI) const9403 void SparcV8ABIInfo::computeInfo(CGFunctionInfo &FI) const {
9404
9405 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
9406 for (auto &Arg : FI.arguments())
9407 Arg.info = classifyArgumentType(Arg.type);
9408 }
9409
9410 namespace {
9411 class SparcV8TargetCodeGenInfo : public TargetCodeGenInfo {
9412 public:
SparcV8TargetCodeGenInfo(CodeGenTypes & CGT)9413 SparcV8TargetCodeGenInfo(CodeGenTypes &CGT)
9414 : TargetCodeGenInfo(std::make_unique<SparcV8ABIInfo>(CGT)) {}
9415 };
9416 } // end anonymous namespace
9417
9418 //===----------------------------------------------------------------------===//
9419 // SPARC v9 ABI Implementation.
9420 // Based on the SPARC Compliance Definition version 2.4.1.
9421 //
9422 // Function arguments a mapped to a nominal "parameter array" and promoted to
9423 // registers depending on their type. Each argument occupies 8 or 16 bytes in
9424 // the array, structs larger than 16 bytes are passed indirectly.
9425 //
9426 // One case requires special care:
9427 //
9428 // struct mixed {
9429 // int i;
9430 // float f;
9431 // };
9432 //
9433 // When a struct mixed is passed by value, it only occupies 8 bytes in the
9434 // parameter array, but the int is passed in an integer register, and the float
9435 // is passed in a floating point register. This is represented as two arguments
9436 // with the LLVM IR inreg attribute:
9437 //
9438 // declare void f(i32 inreg %i, float inreg %f)
9439 //
9440 // The code generator will only allocate 4 bytes from the parameter array for
9441 // the inreg arguments. All other arguments are allocated a multiple of 8
9442 // bytes.
9443 //
9444 namespace {
9445 class SparcV9ABIInfo : public ABIInfo {
9446 public:
SparcV9ABIInfo(CodeGenTypes & CGT)9447 SparcV9ABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
9448
9449 private:
9450 ABIArgInfo classifyType(QualType RetTy, unsigned SizeLimit) const;
9451 void computeInfo(CGFunctionInfo &FI) const override;
9452 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9453 QualType Ty) const override;
9454
9455 // Coercion type builder for structs passed in registers. The coercion type
9456 // serves two purposes:
9457 //
9458 // 1. Pad structs to a multiple of 64 bits, so they are passed 'left-aligned'
9459 // in registers.
9460 // 2. Expose aligned floating point elements as first-level elements, so the
9461 // code generator knows to pass them in floating point registers.
9462 //
9463 // We also compute the InReg flag which indicates that the struct contains
9464 // aligned 32-bit floats.
9465 //
9466 struct CoerceBuilder {
9467 llvm::LLVMContext &Context;
9468 const llvm::DataLayout &DL;
9469 SmallVector<llvm::Type*, 8> Elems;
9470 uint64_t Size;
9471 bool InReg;
9472
CoerceBuilder__anon1dc9252a1711::SparcV9ABIInfo::CoerceBuilder9473 CoerceBuilder(llvm::LLVMContext &c, const llvm::DataLayout &dl)
9474 : Context(c), DL(dl), Size(0), InReg(false) {}
9475
9476 // Pad Elems with integers until Size is ToSize.
pad__anon1dc9252a1711::SparcV9ABIInfo::CoerceBuilder9477 void pad(uint64_t ToSize) {
9478 assert(ToSize >= Size && "Cannot remove elements");
9479 if (ToSize == Size)
9480 return;
9481
9482 // Finish the current 64-bit word.
9483 uint64_t Aligned = llvm::alignTo(Size, 64);
9484 if (Aligned > Size && Aligned <= ToSize) {
9485 Elems.push_back(llvm::IntegerType::get(Context, Aligned - Size));
9486 Size = Aligned;
9487 }
9488
9489 // Add whole 64-bit words.
9490 while (Size + 64 <= ToSize) {
9491 Elems.push_back(llvm::Type::getInt64Ty(Context));
9492 Size += 64;
9493 }
9494
9495 // Final in-word padding.
9496 if (Size < ToSize) {
9497 Elems.push_back(llvm::IntegerType::get(Context, ToSize - Size));
9498 Size = ToSize;
9499 }
9500 }
9501
9502 // Add a floating point element at Offset.
addFloat__anon1dc9252a1711::SparcV9ABIInfo::CoerceBuilder9503 void addFloat(uint64_t Offset, llvm::Type *Ty, unsigned Bits) {
9504 // Unaligned floats are treated as integers.
9505 if (Offset % Bits)
9506 return;
9507 // The InReg flag is only required if there are any floats < 64 bits.
9508 if (Bits < 64)
9509 InReg = true;
9510 pad(Offset);
9511 Elems.push_back(Ty);
9512 Size = Offset + Bits;
9513 }
9514
9515 // Add a struct type to the coercion type, starting at Offset (in bits).
addStruct__anon1dc9252a1711::SparcV9ABIInfo::CoerceBuilder9516 void addStruct(uint64_t Offset, llvm::StructType *StrTy) {
9517 const llvm::StructLayout *Layout = DL.getStructLayout(StrTy);
9518 for (unsigned i = 0, e = StrTy->getNumElements(); i != e; ++i) {
9519 llvm::Type *ElemTy = StrTy->getElementType(i);
9520 uint64_t ElemOffset = Offset + Layout->getElementOffsetInBits(i);
9521 switch (ElemTy->getTypeID()) {
9522 case llvm::Type::StructTyID:
9523 addStruct(ElemOffset, cast<llvm::StructType>(ElemTy));
9524 break;
9525 case llvm::Type::FloatTyID:
9526 addFloat(ElemOffset, ElemTy, 32);
9527 break;
9528 case llvm::Type::DoubleTyID:
9529 addFloat(ElemOffset, ElemTy, 64);
9530 break;
9531 case llvm::Type::FP128TyID:
9532 addFloat(ElemOffset, ElemTy, 128);
9533 break;
9534 case llvm::Type::PointerTyID:
9535 if (ElemOffset % 64 == 0) {
9536 pad(ElemOffset);
9537 Elems.push_back(ElemTy);
9538 Size += 64;
9539 }
9540 break;
9541 default:
9542 break;
9543 }
9544 }
9545 }
9546
9547 // Check if Ty is a usable substitute for the coercion type.
isUsableType__anon1dc9252a1711::SparcV9ABIInfo::CoerceBuilder9548 bool isUsableType(llvm::StructType *Ty) const {
9549 return llvm::makeArrayRef(Elems) == Ty->elements();
9550 }
9551
9552 // Get the coercion type as a literal struct type.
getType__anon1dc9252a1711::SparcV9ABIInfo::CoerceBuilder9553 llvm::Type *getType() const {
9554 if (Elems.size() == 1)
9555 return Elems.front();
9556 else
9557 return llvm::StructType::get(Context, Elems);
9558 }
9559 };
9560 };
9561 } // end anonymous namespace
9562
9563 ABIArgInfo
classifyType(QualType Ty,unsigned SizeLimit) const9564 SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const {
9565 if (Ty->isVoidType())
9566 return ABIArgInfo::getIgnore();
9567
9568 uint64_t Size = getContext().getTypeSize(Ty);
9569
9570 // Anything too big to fit in registers is passed with an explicit indirect
9571 // pointer / sret pointer.
9572 if (Size > SizeLimit)
9573 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
9574
9575 // Treat an enum type as its underlying type.
9576 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
9577 Ty = EnumTy->getDecl()->getIntegerType();
9578
9579 // Integer types smaller than a register are extended.
9580 if (Size < 64 && Ty->isIntegerType())
9581 return ABIArgInfo::getExtend(Ty);
9582
9583 if (const auto *EIT = Ty->getAs<ExtIntType>())
9584 if (EIT->getNumBits() < 64)
9585 return ABIArgInfo::getExtend(Ty);
9586
9587 // Other non-aggregates go in registers.
9588 if (!isAggregateTypeForABI(Ty))
9589 return ABIArgInfo::getDirect();
9590
9591 // If a C++ object has either a non-trivial copy constructor or a non-trivial
9592 // destructor, it is passed with an explicit indirect pointer / sret pointer.
9593 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
9594 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
9595
9596 // This is a small aggregate type that should be passed in registers.
9597 // Build a coercion type from the LLVM struct type.
9598 llvm::StructType *StrTy = dyn_cast<llvm::StructType>(CGT.ConvertType(Ty));
9599 if (!StrTy)
9600 return ABIArgInfo::getDirect();
9601
9602 CoerceBuilder CB(getVMContext(), getDataLayout());
9603 CB.addStruct(0, StrTy);
9604 CB.pad(llvm::alignTo(CB.DL.getTypeSizeInBits(StrTy), 64));
9605
9606 // Try to use the original type for coercion.
9607 llvm::Type *CoerceTy = CB.isUsableType(StrTy) ? StrTy : CB.getType();
9608
9609 if (CB.InReg)
9610 return ABIArgInfo::getDirectInReg(CoerceTy);
9611 else
9612 return ABIArgInfo::getDirect(CoerceTy);
9613 }
9614
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const9615 Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9616 QualType Ty) const {
9617 ABIArgInfo AI = classifyType(Ty, 16 * 8);
9618 llvm::Type *ArgTy = CGT.ConvertType(Ty);
9619 if (AI.canHaveCoerceToType() && !AI.getCoerceToType())
9620 AI.setCoerceToType(ArgTy);
9621
9622 CharUnits SlotSize = CharUnits::fromQuantity(8);
9623
9624 CGBuilderTy &Builder = CGF.Builder;
9625 Address Addr(Builder.CreateLoad(VAListAddr, "ap.cur"), SlotSize);
9626 llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy);
9627
9628 auto TypeInfo = getContext().getTypeInfoInChars(Ty);
9629
9630 Address ArgAddr = Address::invalid();
9631 CharUnits Stride;
9632 switch (AI.getKind()) {
9633 case ABIArgInfo::Expand:
9634 case ABIArgInfo::CoerceAndExpand:
9635 case ABIArgInfo::InAlloca:
9636 llvm_unreachable("Unsupported ABI kind for va_arg");
9637
9638 case ABIArgInfo::Extend: {
9639 Stride = SlotSize;
9640 CharUnits Offset = SlotSize - TypeInfo.Width;
9641 ArgAddr = Builder.CreateConstInBoundsByteGEP(Addr, Offset, "extend");
9642 break;
9643 }
9644
9645 case ABIArgInfo::Direct: {
9646 auto AllocSize = getDataLayout().getTypeAllocSize(AI.getCoerceToType());
9647 Stride = CharUnits::fromQuantity(AllocSize).alignTo(SlotSize);
9648 ArgAddr = Addr;
9649 break;
9650 }
9651
9652 case ABIArgInfo::Indirect:
9653 case ABIArgInfo::IndirectAliased:
9654 Stride = SlotSize;
9655 ArgAddr = Builder.CreateElementBitCast(Addr, ArgPtrTy, "indirect");
9656 ArgAddr = Address(Builder.CreateLoad(ArgAddr, "indirect.arg"),
9657 TypeInfo.Align);
9658 break;
9659
9660 case ABIArgInfo::Ignore:
9661 return Address(llvm::UndefValue::get(ArgPtrTy), TypeInfo.Align);
9662 }
9663
9664 // Update VAList.
9665 Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next");
9666 Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
9667
9668 return Builder.CreateBitCast(ArgAddr, ArgPtrTy, "arg.addr");
9669 }
9670
computeInfo(CGFunctionInfo & FI) const9671 void SparcV9ABIInfo::computeInfo(CGFunctionInfo &FI) const {
9672 FI.getReturnInfo() = classifyType(FI.getReturnType(), 32 * 8);
9673 for (auto &I : FI.arguments())
9674 I.info = classifyType(I.type, 16 * 8);
9675 }
9676
9677 namespace {
9678 class SparcV9TargetCodeGenInfo : public TargetCodeGenInfo {
9679 public:
SparcV9TargetCodeGenInfo(CodeGenTypes & CGT)9680 SparcV9TargetCodeGenInfo(CodeGenTypes &CGT)
9681 : TargetCodeGenInfo(std::make_unique<SparcV9ABIInfo>(CGT)) {}
9682
getDwarfEHStackPointer(CodeGen::CodeGenModule & M) const9683 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
9684 return 14;
9685 }
9686
9687 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
9688 llvm::Value *Address) const override;
9689 };
9690 } // end anonymous namespace
9691
9692 bool
initDwarfEHRegSizeTable(CodeGen::CodeGenFunction & CGF,llvm::Value * Address) const9693 SparcV9TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
9694 llvm::Value *Address) const {
9695 // This is calculated from the LLVM and GCC tables and verified
9696 // against gcc output. AFAIK all ABIs use the same encoding.
9697
9698 CodeGen::CGBuilderTy &Builder = CGF.Builder;
9699
9700 llvm::IntegerType *i8 = CGF.Int8Ty;
9701 llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
9702 llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
9703
9704 // 0-31: the 8-byte general-purpose registers
9705 AssignToArrayRange(Builder, Address, Eight8, 0, 31);
9706
9707 // 32-63: f0-31, the 4-byte floating-point registers
9708 AssignToArrayRange(Builder, Address, Four8, 32, 63);
9709
9710 // Y = 64
9711 // PSR = 65
9712 // WIM = 66
9713 // TBR = 67
9714 // PC = 68
9715 // NPC = 69
9716 // FSR = 70
9717 // CSR = 71
9718 AssignToArrayRange(Builder, Address, Eight8, 64, 71);
9719
9720 // 72-87: d0-15, the 8-byte floating-point registers
9721 AssignToArrayRange(Builder, Address, Eight8, 72, 87);
9722
9723 return false;
9724 }
9725
9726 // ARC ABI implementation.
9727 namespace {
9728
9729 class ARCABIInfo : public DefaultABIInfo {
9730 public:
9731 using DefaultABIInfo::DefaultABIInfo;
9732
9733 private:
9734 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9735 QualType Ty) const override;
9736
updateState(const ABIArgInfo & Info,QualType Ty,CCState & State) const9737 void updateState(const ABIArgInfo &Info, QualType Ty, CCState &State) const {
9738 if (!State.FreeRegs)
9739 return;
9740 if (Info.isIndirect() && Info.getInReg())
9741 State.FreeRegs--;
9742 else if (Info.isDirect() && Info.getInReg()) {
9743 unsigned sz = (getContext().getTypeSize(Ty) + 31) / 32;
9744 if (sz < State.FreeRegs)
9745 State.FreeRegs -= sz;
9746 else
9747 State.FreeRegs = 0;
9748 }
9749 }
9750
computeInfo(CGFunctionInfo & FI) const9751 void computeInfo(CGFunctionInfo &FI) const override {
9752 CCState State(FI);
9753 // ARC uses 8 registers to pass arguments.
9754 State.FreeRegs = 8;
9755
9756 if (!getCXXABI().classifyReturnType(FI))
9757 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
9758 updateState(FI.getReturnInfo(), FI.getReturnType(), State);
9759 for (auto &I : FI.arguments()) {
9760 I.info = classifyArgumentType(I.type, State.FreeRegs);
9761 updateState(I.info, I.type, State);
9762 }
9763 }
9764
9765 ABIArgInfo getIndirectByRef(QualType Ty, bool HasFreeRegs) const;
9766 ABIArgInfo getIndirectByValue(QualType Ty) const;
9767 ABIArgInfo classifyArgumentType(QualType Ty, uint8_t FreeRegs) const;
9768 ABIArgInfo classifyReturnType(QualType RetTy) const;
9769 };
9770
9771 class ARCTargetCodeGenInfo : public TargetCodeGenInfo {
9772 public:
ARCTargetCodeGenInfo(CodeGenTypes & CGT)9773 ARCTargetCodeGenInfo(CodeGenTypes &CGT)
9774 : TargetCodeGenInfo(std::make_unique<ARCABIInfo>(CGT)) {}
9775 };
9776
9777
getIndirectByRef(QualType Ty,bool HasFreeRegs) const9778 ABIArgInfo ARCABIInfo::getIndirectByRef(QualType Ty, bool HasFreeRegs) const {
9779 return HasFreeRegs ? getNaturalAlignIndirectInReg(Ty) :
9780 getNaturalAlignIndirect(Ty, false);
9781 }
9782
getIndirectByValue(QualType Ty) const9783 ABIArgInfo ARCABIInfo::getIndirectByValue(QualType Ty) const {
9784 // Compute the byval alignment.
9785 const unsigned MinABIStackAlignInBytes = 4;
9786 unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
9787 return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true,
9788 TypeAlign > MinABIStackAlignInBytes);
9789 }
9790
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const9791 Address ARCABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9792 QualType Ty) const {
9793 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
9794 getContext().getTypeInfoInChars(Ty),
9795 CharUnits::fromQuantity(4), true);
9796 }
9797
classifyArgumentType(QualType Ty,uint8_t FreeRegs) const9798 ABIArgInfo ARCABIInfo::classifyArgumentType(QualType Ty,
9799 uint8_t FreeRegs) const {
9800 // Handle the generic C++ ABI.
9801 const RecordType *RT = Ty->getAs<RecordType>();
9802 if (RT) {
9803 CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
9804 if (RAA == CGCXXABI::RAA_Indirect)
9805 return getIndirectByRef(Ty, FreeRegs > 0);
9806
9807 if (RAA == CGCXXABI::RAA_DirectInMemory)
9808 return getIndirectByValue(Ty);
9809 }
9810
9811 // Treat an enum type as its underlying type.
9812 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
9813 Ty = EnumTy->getDecl()->getIntegerType();
9814
9815 auto SizeInRegs = llvm::alignTo(getContext().getTypeSize(Ty), 32) / 32;
9816
9817 if (isAggregateTypeForABI(Ty)) {
9818 // Structures with flexible arrays are always indirect.
9819 if (RT && RT->getDecl()->hasFlexibleArrayMember())
9820 return getIndirectByValue(Ty);
9821
9822 // Ignore empty structs/unions.
9823 if (isEmptyRecord(getContext(), Ty, true))
9824 return ABIArgInfo::getIgnore();
9825
9826 llvm::LLVMContext &LLVMContext = getVMContext();
9827
9828 llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
9829 SmallVector<llvm::Type *, 3> Elements(SizeInRegs, Int32);
9830 llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
9831
9832 return FreeRegs >= SizeInRegs ?
9833 ABIArgInfo::getDirectInReg(Result) :
9834 ABIArgInfo::getDirect(Result, 0, nullptr, false);
9835 }
9836
9837 if (const auto *EIT = Ty->getAs<ExtIntType>())
9838 if (EIT->getNumBits() > 64)
9839 return getIndirectByValue(Ty);
9840
9841 return isPromotableIntegerTypeForABI(Ty)
9842 ? (FreeRegs >= SizeInRegs ? ABIArgInfo::getExtendInReg(Ty)
9843 : ABIArgInfo::getExtend(Ty))
9844 : (FreeRegs >= SizeInRegs ? ABIArgInfo::getDirectInReg()
9845 : ABIArgInfo::getDirect());
9846 }
9847
classifyReturnType(QualType RetTy) const9848 ABIArgInfo ARCABIInfo::classifyReturnType(QualType RetTy) const {
9849 if (RetTy->isAnyComplexType())
9850 return ABIArgInfo::getDirectInReg();
9851
9852 // Arguments of size > 4 registers are indirect.
9853 auto RetSize = llvm::alignTo(getContext().getTypeSize(RetTy), 32) / 32;
9854 if (RetSize > 4)
9855 return getIndirectByRef(RetTy, /*HasFreeRegs*/ true);
9856
9857 return DefaultABIInfo::classifyReturnType(RetTy);
9858 }
9859
9860 } // End anonymous namespace.
9861
9862 //===----------------------------------------------------------------------===//
9863 // XCore ABI Implementation
9864 //===----------------------------------------------------------------------===//
9865
9866 namespace {
9867
9868 /// A SmallStringEnc instance is used to build up the TypeString by passing
9869 /// it by reference between functions that append to it.
9870 typedef llvm::SmallString<128> SmallStringEnc;
9871
9872 /// TypeStringCache caches the meta encodings of Types.
9873 ///
9874 /// The reason for caching TypeStrings is two fold:
9875 /// 1. To cache a type's encoding for later uses;
9876 /// 2. As a means to break recursive member type inclusion.
9877 ///
9878 /// A cache Entry can have a Status of:
9879 /// NonRecursive: The type encoding is not recursive;
9880 /// Recursive: The type encoding is recursive;
9881 /// Incomplete: An incomplete TypeString;
9882 /// IncompleteUsed: An incomplete TypeString that has been used in a
9883 /// Recursive type encoding.
9884 ///
9885 /// A NonRecursive entry will have all of its sub-members expanded as fully
9886 /// as possible. Whilst it may contain types which are recursive, the type
9887 /// itself is not recursive and thus its encoding may be safely used whenever
9888 /// the type is encountered.
9889 ///
9890 /// A Recursive entry will have all of its sub-members expanded as fully as
9891 /// possible. The type itself is recursive and it may contain other types which
9892 /// are recursive. The Recursive encoding must not be used during the expansion
9893 /// of a recursive type's recursive branch. For simplicity the code uses
9894 /// IncompleteCount to reject all usage of Recursive encodings for member types.
9895 ///
9896 /// An Incomplete entry is always a RecordType and only encodes its
9897 /// identifier e.g. "s(S){}". Incomplete 'StubEnc' entries are ephemeral and
9898 /// are placed into the cache during type expansion as a means to identify and
9899 /// handle recursive inclusion of types as sub-members. If there is recursion
9900 /// the entry becomes IncompleteUsed.
9901 ///
9902 /// During the expansion of a RecordType's members:
9903 ///
9904 /// If the cache contains a NonRecursive encoding for the member type, the
9905 /// cached encoding is used;
9906 ///
9907 /// If the cache contains a Recursive encoding for the member type, the
9908 /// cached encoding is 'Swapped' out, as it may be incorrect, and...
9909 ///
9910 /// If the member is a RecordType, an Incomplete encoding is placed into the
9911 /// cache to break potential recursive inclusion of itself as a sub-member;
9912 ///
9913 /// Once a member RecordType has been expanded, its temporary incomplete
9914 /// entry is removed from the cache. If a Recursive encoding was swapped out
9915 /// it is swapped back in;
9916 ///
9917 /// If an incomplete entry is used to expand a sub-member, the incomplete
9918 /// entry is marked as IncompleteUsed. The cache keeps count of how many
9919 /// IncompleteUsed entries it currently contains in IncompleteUsedCount;
9920 ///
9921 /// If a member's encoding is found to be a NonRecursive or Recursive viz:
9922 /// IncompleteUsedCount==0, the member's encoding is added to the cache.
9923 /// Else the member is part of a recursive type and thus the recursion has
9924 /// been exited too soon for the encoding to be correct for the member.
9925 ///
9926 class TypeStringCache {
9927 enum Status {NonRecursive, Recursive, Incomplete, IncompleteUsed};
9928 struct Entry {
9929 std::string Str; // The encoded TypeString for the type.
9930 enum Status State; // Information about the encoding in 'Str'.
9931 std::string Swapped; // A temporary place holder for a Recursive encoding
9932 // during the expansion of RecordType's members.
9933 };
9934 std::map<const IdentifierInfo *, struct Entry> Map;
9935 unsigned IncompleteCount; // Number of Incomplete entries in the Map.
9936 unsigned IncompleteUsedCount; // Number of IncompleteUsed entries in the Map.
9937 public:
TypeStringCache()9938 TypeStringCache() : IncompleteCount(0), IncompleteUsedCount(0) {}
9939 void addIncomplete(const IdentifierInfo *ID, std::string StubEnc);
9940 bool removeIncomplete(const IdentifierInfo *ID);
9941 void addIfComplete(const IdentifierInfo *ID, StringRef Str,
9942 bool IsRecursive);
9943 StringRef lookupStr(const IdentifierInfo *ID);
9944 };
9945
9946 /// TypeString encodings for enum & union fields must be order.
9947 /// FieldEncoding is a helper for this ordering process.
9948 class FieldEncoding {
9949 bool HasName;
9950 std::string Enc;
9951 public:
FieldEncoding(bool b,SmallStringEnc & e)9952 FieldEncoding(bool b, SmallStringEnc &e) : HasName(b), Enc(e.c_str()) {}
str()9953 StringRef str() { return Enc; }
operator <(const FieldEncoding & rhs) const9954 bool operator<(const FieldEncoding &rhs) const {
9955 if (HasName != rhs.HasName) return HasName;
9956 return Enc < rhs.Enc;
9957 }
9958 };
9959
9960 class XCoreABIInfo : public DefaultABIInfo {
9961 public:
XCoreABIInfo(CodeGen::CodeGenTypes & CGT)9962 XCoreABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
9963 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9964 QualType Ty) const override;
9965 };
9966
9967 class XCoreTargetCodeGenInfo : public TargetCodeGenInfo {
9968 mutable TypeStringCache TSC;
9969 void emitTargetMD(const Decl *D, llvm::GlobalValue *GV,
9970 const CodeGen::CodeGenModule &M) const;
9971
9972 public:
XCoreTargetCodeGenInfo(CodeGenTypes & CGT)9973 XCoreTargetCodeGenInfo(CodeGenTypes &CGT)
9974 : TargetCodeGenInfo(std::make_unique<XCoreABIInfo>(CGT)) {}
9975 void emitTargetMetadata(CodeGen::CodeGenModule &CGM,
9976 const llvm::MapVector<GlobalDecl, StringRef>
9977 &MangledDeclNames) const override;
9978 };
9979
9980 } // End anonymous namespace.
9981
9982 // TODO: this implementation is likely now redundant with the default
9983 // EmitVAArg.
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const9984 Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
9985 QualType Ty) const {
9986 CGBuilderTy &Builder = CGF.Builder;
9987
9988 // Get the VAList.
9989 CharUnits SlotSize = CharUnits::fromQuantity(4);
9990 Address AP(Builder.CreateLoad(VAListAddr), SlotSize);
9991
9992 // Handle the argument.
9993 ABIArgInfo AI = classifyArgumentType(Ty);
9994 CharUnits TypeAlign = getContext().getTypeAlignInChars(Ty);
9995 llvm::Type *ArgTy = CGT.ConvertType(Ty);
9996 if (AI.canHaveCoerceToType() && !AI.getCoerceToType())
9997 AI.setCoerceToType(ArgTy);
9998 llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy);
9999
10000 Address Val = Address::invalid();
10001 CharUnits ArgSize = CharUnits::Zero();
10002 switch (AI.getKind()) {
10003 case ABIArgInfo::Expand:
10004 case ABIArgInfo::CoerceAndExpand:
10005 case ABIArgInfo::InAlloca:
10006 llvm_unreachable("Unsupported ABI kind for va_arg");
10007 case ABIArgInfo::Ignore:
10008 Val = Address(llvm::UndefValue::get(ArgPtrTy), TypeAlign);
10009 ArgSize = CharUnits::Zero();
10010 break;
10011 case ABIArgInfo::Extend:
10012 case ABIArgInfo::Direct:
10013 Val = Builder.CreateBitCast(AP, ArgPtrTy);
10014 ArgSize = CharUnits::fromQuantity(
10015 getDataLayout().getTypeAllocSize(AI.getCoerceToType()));
10016 ArgSize = ArgSize.alignTo(SlotSize);
10017 break;
10018 case ABIArgInfo::Indirect:
10019 case ABIArgInfo::IndirectAliased:
10020 Val = Builder.CreateElementBitCast(AP, ArgPtrTy);
10021 Val = Address(Builder.CreateLoad(Val), TypeAlign);
10022 ArgSize = SlotSize;
10023 break;
10024 }
10025
10026 // Increment the VAList.
10027 if (!ArgSize.isZero()) {
10028 Address APN = Builder.CreateConstInBoundsByteGEP(AP, ArgSize);
10029 Builder.CreateStore(APN.getPointer(), VAListAddr);
10030 }
10031
10032 return Val;
10033 }
10034
10035 /// During the expansion of a RecordType, an incomplete TypeString is placed
10036 /// into the cache as a means to identify and break recursion.
10037 /// If there is a Recursive encoding in the cache, it is swapped out and will
10038 /// be reinserted by removeIncomplete().
10039 /// All other types of encoding should have been used rather than arriving here.
addIncomplete(const IdentifierInfo * ID,std::string StubEnc)10040 void TypeStringCache::addIncomplete(const IdentifierInfo *ID,
10041 std::string StubEnc) {
10042 if (!ID)
10043 return;
10044 Entry &E = Map[ID];
10045 assert( (E.Str.empty() || E.State == Recursive) &&
10046 "Incorrectly use of addIncomplete");
10047 assert(!StubEnc.empty() && "Passing an empty string to addIncomplete()");
10048 E.Swapped.swap(E.Str); // swap out the Recursive
10049 E.Str.swap(StubEnc);
10050 E.State = Incomplete;
10051 ++IncompleteCount;
10052 }
10053
10054 /// Once the RecordType has been expanded, the temporary incomplete TypeString
10055 /// must be removed from the cache.
10056 /// If a Recursive was swapped out by addIncomplete(), it will be replaced.
10057 /// Returns true if the RecordType was defined recursively.
removeIncomplete(const IdentifierInfo * ID)10058 bool TypeStringCache::removeIncomplete(const IdentifierInfo *ID) {
10059 if (!ID)
10060 return false;
10061 auto I = Map.find(ID);
10062 assert(I != Map.end() && "Entry not present");
10063 Entry &E = I->second;
10064 assert( (E.State == Incomplete ||
10065 E.State == IncompleteUsed) &&
10066 "Entry must be an incomplete type");
10067 bool IsRecursive = false;
10068 if (E.State == IncompleteUsed) {
10069 // We made use of our Incomplete encoding, thus we are recursive.
10070 IsRecursive = true;
10071 --IncompleteUsedCount;
10072 }
10073 if (E.Swapped.empty())
10074 Map.erase(I);
10075 else {
10076 // Swap the Recursive back.
10077 E.Swapped.swap(E.Str);
10078 E.Swapped.clear();
10079 E.State = Recursive;
10080 }
10081 --IncompleteCount;
10082 return IsRecursive;
10083 }
10084
10085 /// Add the encoded TypeString to the cache only if it is NonRecursive or
10086 /// Recursive (viz: all sub-members were expanded as fully as possible).
addIfComplete(const IdentifierInfo * ID,StringRef Str,bool IsRecursive)10087 void TypeStringCache::addIfComplete(const IdentifierInfo *ID, StringRef Str,
10088 bool IsRecursive) {
10089 if (!ID || IncompleteUsedCount)
10090 return; // No key or it is is an incomplete sub-type so don't add.
10091 Entry &E = Map[ID];
10092 if (IsRecursive && !E.Str.empty()) {
10093 assert(E.State==Recursive && E.Str.size() == Str.size() &&
10094 "This is not the same Recursive entry");
10095 // The parent container was not recursive after all, so we could have used
10096 // this Recursive sub-member entry after all, but we assumed the worse when
10097 // we started viz: IncompleteCount!=0.
10098 return;
10099 }
10100 assert(E.Str.empty() && "Entry already present");
10101 E.Str = Str.str();
10102 E.State = IsRecursive? Recursive : NonRecursive;
10103 }
10104
10105 /// Return a cached TypeString encoding for the ID. If there isn't one, or we
10106 /// are recursively expanding a type (IncompleteCount != 0) and the cached
10107 /// encoding is Recursive, return an empty StringRef.
lookupStr(const IdentifierInfo * ID)10108 StringRef TypeStringCache::lookupStr(const IdentifierInfo *ID) {
10109 if (!ID)
10110 return StringRef(); // We have no key.
10111 auto I = Map.find(ID);
10112 if (I == Map.end())
10113 return StringRef(); // We have no encoding.
10114 Entry &E = I->second;
10115 if (E.State == Recursive && IncompleteCount)
10116 return StringRef(); // We don't use Recursive encodings for member types.
10117
10118 if (E.State == Incomplete) {
10119 // The incomplete type is being used to break out of recursion.
10120 E.State = IncompleteUsed;
10121 ++IncompleteUsedCount;
10122 }
10123 return E.Str;
10124 }
10125
10126 /// The XCore ABI includes a type information section that communicates symbol
10127 /// type information to the linker. The linker uses this information to verify
10128 /// safety/correctness of things such as array bound and pointers et al.
10129 /// The ABI only requires C (and XC) language modules to emit TypeStrings.
10130 /// This type information (TypeString) is emitted into meta data for all global
10131 /// symbols: definitions, declarations, functions & variables.
10132 ///
10133 /// The TypeString carries type, qualifier, name, size & value details.
10134 /// Please see 'Tools Development Guide' section 2.16.2 for format details:
10135 /// https://www.xmos.com/download/public/Tools-Development-Guide%28X9114A%29.pdf
10136 /// The output is tested by test/CodeGen/xcore-stringtype.c.
10137 ///
10138 static bool getTypeString(SmallStringEnc &Enc, const Decl *D,
10139 const CodeGen::CodeGenModule &CGM,
10140 TypeStringCache &TSC);
10141
10142 /// XCore uses emitTargetMD to emit TypeString metadata for global symbols.
emitTargetMD(const Decl * D,llvm::GlobalValue * GV,const CodeGen::CodeGenModule & CGM) const10143 void XCoreTargetCodeGenInfo::emitTargetMD(
10144 const Decl *D, llvm::GlobalValue *GV,
10145 const CodeGen::CodeGenModule &CGM) const {
10146 SmallStringEnc Enc;
10147 if (getTypeString(Enc, D, CGM, TSC)) {
10148 llvm::LLVMContext &Ctx = CGM.getModule().getContext();
10149 llvm::Metadata *MDVals[] = {llvm::ConstantAsMetadata::get(GV),
10150 llvm::MDString::get(Ctx, Enc.str())};
10151 llvm::NamedMDNode *MD =
10152 CGM.getModule().getOrInsertNamedMetadata("xcore.typestrings");
10153 MD->addOperand(llvm::MDNode::get(Ctx, MDVals));
10154 }
10155 }
10156
emitTargetMetadata(CodeGen::CodeGenModule & CGM,const llvm::MapVector<GlobalDecl,StringRef> & MangledDeclNames) const10157 void XCoreTargetCodeGenInfo::emitTargetMetadata(
10158 CodeGen::CodeGenModule &CGM,
10159 const llvm::MapVector<GlobalDecl, StringRef> &MangledDeclNames) const {
10160 // Warning, new MangledDeclNames may be appended within this loop.
10161 // We rely on MapVector insertions adding new elements to the end
10162 // of the container.
10163 for (unsigned I = 0; I != MangledDeclNames.size(); ++I) {
10164 auto Val = *(MangledDeclNames.begin() + I);
10165 llvm::GlobalValue *GV = CGM.GetGlobalValue(Val.second);
10166 if (GV) {
10167 const Decl *D = Val.first.getDecl()->getMostRecentDecl();
10168 emitTargetMD(D, GV, CGM);
10169 }
10170 }
10171 }
10172 //===----------------------------------------------------------------------===//
10173 // SPIR ABI Implementation
10174 //===----------------------------------------------------------------------===//
10175
10176 namespace {
10177 class SPIRABIInfo : public DefaultABIInfo {
10178 public:
SPIRABIInfo(CodeGenTypes & CGT)10179 SPIRABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) { setCCs(); }
10180
10181 private:
10182 void setCCs();
10183 };
10184 } // end anonymous namespace
10185 namespace {
10186 class SPIRTargetCodeGenInfo : public TargetCodeGenInfo {
10187 public:
SPIRTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT)10188 SPIRTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
10189 : TargetCodeGenInfo(std::make_unique<SPIRABIInfo>(CGT)) {}
10190
getASTAllocaAddressSpace() const10191 LangAS getASTAllocaAddressSpace() const override {
10192 return getLangASFromTargetAS(
10193 getABIInfo().getDataLayout().getAllocaAddrSpace());
10194 }
10195
10196 unsigned getOpenCLKernelCallingConv() const override;
10197 };
10198
10199 } // End anonymous namespace.
setCCs()10200 void SPIRABIInfo::setCCs() {
10201 assert(getRuntimeCC() == llvm::CallingConv::C);
10202 RuntimeCC = llvm::CallingConv::SPIR_FUNC;
10203 }
10204
10205 namespace clang {
10206 namespace CodeGen {
computeSPIRKernelABIInfo(CodeGenModule & CGM,CGFunctionInfo & FI)10207 void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) {
10208 DefaultABIInfo SPIRABI(CGM.getTypes());
10209 SPIRABI.computeInfo(FI);
10210 }
10211 }
10212 }
10213
getOpenCLKernelCallingConv() const10214 unsigned SPIRTargetCodeGenInfo::getOpenCLKernelCallingConv() const {
10215 return llvm::CallingConv::SPIR_KERNEL;
10216 }
10217
10218 static bool appendType(SmallStringEnc &Enc, QualType QType,
10219 const CodeGen::CodeGenModule &CGM,
10220 TypeStringCache &TSC);
10221
10222 /// Helper function for appendRecordType().
10223 /// Builds a SmallVector containing the encoded field types in declaration
10224 /// order.
extractFieldType(SmallVectorImpl<FieldEncoding> & FE,const RecordDecl * RD,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10225 static bool extractFieldType(SmallVectorImpl<FieldEncoding> &FE,
10226 const RecordDecl *RD,
10227 const CodeGen::CodeGenModule &CGM,
10228 TypeStringCache &TSC) {
10229 for (const auto *Field : RD->fields()) {
10230 SmallStringEnc Enc;
10231 Enc += "m(";
10232 Enc += Field->getName();
10233 Enc += "){";
10234 if (Field->isBitField()) {
10235 Enc += "b(";
10236 llvm::raw_svector_ostream OS(Enc);
10237 OS << Field->getBitWidthValue(CGM.getContext());
10238 Enc += ':';
10239 }
10240 if (!appendType(Enc, Field->getType(), CGM, TSC))
10241 return false;
10242 if (Field->isBitField())
10243 Enc += ')';
10244 Enc += '}';
10245 FE.emplace_back(!Field->getName().empty(), Enc);
10246 }
10247 return true;
10248 }
10249
10250 /// Appends structure and union types to Enc and adds encoding to cache.
10251 /// Recursively calls appendType (via extractFieldType) for each field.
10252 /// Union types have their fields ordered according to the ABI.
appendRecordType(SmallStringEnc & Enc,const RecordType * RT,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC,const IdentifierInfo * ID)10253 static bool appendRecordType(SmallStringEnc &Enc, const RecordType *RT,
10254 const CodeGen::CodeGenModule &CGM,
10255 TypeStringCache &TSC, const IdentifierInfo *ID) {
10256 // Append the cached TypeString if we have one.
10257 StringRef TypeString = TSC.lookupStr(ID);
10258 if (!TypeString.empty()) {
10259 Enc += TypeString;
10260 return true;
10261 }
10262
10263 // Start to emit an incomplete TypeString.
10264 size_t Start = Enc.size();
10265 Enc += (RT->isUnionType()? 'u' : 's');
10266 Enc += '(';
10267 if (ID)
10268 Enc += ID->getName();
10269 Enc += "){";
10270
10271 // We collect all encoded fields and order as necessary.
10272 bool IsRecursive = false;
10273 const RecordDecl *RD = RT->getDecl()->getDefinition();
10274 if (RD && !RD->field_empty()) {
10275 // An incomplete TypeString stub is placed in the cache for this RecordType
10276 // so that recursive calls to this RecordType will use it whilst building a
10277 // complete TypeString for this RecordType.
10278 SmallVector<FieldEncoding, 16> FE;
10279 std::string StubEnc(Enc.substr(Start).str());
10280 StubEnc += '}'; // StubEnc now holds a valid incomplete TypeString.
10281 TSC.addIncomplete(ID, std::move(StubEnc));
10282 if (!extractFieldType(FE, RD, CGM, TSC)) {
10283 (void) TSC.removeIncomplete(ID);
10284 return false;
10285 }
10286 IsRecursive = TSC.removeIncomplete(ID);
10287 // The ABI requires unions to be sorted but not structures.
10288 // See FieldEncoding::operator< for sort algorithm.
10289 if (RT->isUnionType())
10290 llvm::sort(FE);
10291 // We can now complete the TypeString.
10292 unsigned E = FE.size();
10293 for (unsigned I = 0; I != E; ++I) {
10294 if (I)
10295 Enc += ',';
10296 Enc += FE[I].str();
10297 }
10298 }
10299 Enc += '}';
10300 TSC.addIfComplete(ID, Enc.substr(Start), IsRecursive);
10301 return true;
10302 }
10303
10304 /// Appends enum types to Enc and adds the encoding to the cache.
appendEnumType(SmallStringEnc & Enc,const EnumType * ET,TypeStringCache & TSC,const IdentifierInfo * ID)10305 static bool appendEnumType(SmallStringEnc &Enc, const EnumType *ET,
10306 TypeStringCache &TSC,
10307 const IdentifierInfo *ID) {
10308 // Append the cached TypeString if we have one.
10309 StringRef TypeString = TSC.lookupStr(ID);
10310 if (!TypeString.empty()) {
10311 Enc += TypeString;
10312 return true;
10313 }
10314
10315 size_t Start = Enc.size();
10316 Enc += "e(";
10317 if (ID)
10318 Enc += ID->getName();
10319 Enc += "){";
10320
10321 // We collect all encoded enumerations and order them alphanumerically.
10322 if (const EnumDecl *ED = ET->getDecl()->getDefinition()) {
10323 SmallVector<FieldEncoding, 16> FE;
10324 for (auto I = ED->enumerator_begin(), E = ED->enumerator_end(); I != E;
10325 ++I) {
10326 SmallStringEnc EnumEnc;
10327 EnumEnc += "m(";
10328 EnumEnc += I->getName();
10329 EnumEnc += "){";
10330 I->getInitVal().toString(EnumEnc);
10331 EnumEnc += '}';
10332 FE.push_back(FieldEncoding(!I->getName().empty(), EnumEnc));
10333 }
10334 llvm::sort(FE);
10335 unsigned E = FE.size();
10336 for (unsigned I = 0; I != E; ++I) {
10337 if (I)
10338 Enc += ',';
10339 Enc += FE[I].str();
10340 }
10341 }
10342 Enc += '}';
10343 TSC.addIfComplete(ID, Enc.substr(Start), false);
10344 return true;
10345 }
10346
10347 /// Appends type's qualifier to Enc.
10348 /// This is done prior to appending the type's encoding.
appendQualifier(SmallStringEnc & Enc,QualType QT)10349 static void appendQualifier(SmallStringEnc &Enc, QualType QT) {
10350 // Qualifiers are emitted in alphabetical order.
10351 static const char *const Table[]={"","c:","r:","cr:","v:","cv:","rv:","crv:"};
10352 int Lookup = 0;
10353 if (QT.isConstQualified())
10354 Lookup += 1<<0;
10355 if (QT.isRestrictQualified())
10356 Lookup += 1<<1;
10357 if (QT.isVolatileQualified())
10358 Lookup += 1<<2;
10359 Enc += Table[Lookup];
10360 }
10361
10362 /// Appends built-in types to Enc.
appendBuiltinType(SmallStringEnc & Enc,const BuiltinType * BT)10363 static bool appendBuiltinType(SmallStringEnc &Enc, const BuiltinType *BT) {
10364 const char *EncType;
10365 switch (BT->getKind()) {
10366 case BuiltinType::Void:
10367 EncType = "0";
10368 break;
10369 case BuiltinType::Bool:
10370 EncType = "b";
10371 break;
10372 case BuiltinType::Char_U:
10373 EncType = "uc";
10374 break;
10375 case BuiltinType::UChar:
10376 EncType = "uc";
10377 break;
10378 case BuiltinType::SChar:
10379 EncType = "sc";
10380 break;
10381 case BuiltinType::UShort:
10382 EncType = "us";
10383 break;
10384 case BuiltinType::Short:
10385 EncType = "ss";
10386 break;
10387 case BuiltinType::UInt:
10388 EncType = "ui";
10389 break;
10390 case BuiltinType::Int:
10391 EncType = "si";
10392 break;
10393 case BuiltinType::ULong:
10394 EncType = "ul";
10395 break;
10396 case BuiltinType::Long:
10397 EncType = "sl";
10398 break;
10399 case BuiltinType::ULongLong:
10400 EncType = "ull";
10401 break;
10402 case BuiltinType::LongLong:
10403 EncType = "sll";
10404 break;
10405 case BuiltinType::Float:
10406 EncType = "ft";
10407 break;
10408 case BuiltinType::Double:
10409 EncType = "d";
10410 break;
10411 case BuiltinType::LongDouble:
10412 EncType = "ld";
10413 break;
10414 default:
10415 return false;
10416 }
10417 Enc += EncType;
10418 return true;
10419 }
10420
10421 /// Appends a pointer encoding to Enc before calling appendType for the pointee.
appendPointerType(SmallStringEnc & Enc,const PointerType * PT,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10422 static bool appendPointerType(SmallStringEnc &Enc, const PointerType *PT,
10423 const CodeGen::CodeGenModule &CGM,
10424 TypeStringCache &TSC) {
10425 Enc += "p(";
10426 if (!appendType(Enc, PT->getPointeeType(), CGM, TSC))
10427 return false;
10428 Enc += ')';
10429 return true;
10430 }
10431
10432 /// Appends array encoding to Enc before calling appendType for the element.
appendArrayType(SmallStringEnc & Enc,QualType QT,const ArrayType * AT,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC,StringRef NoSizeEnc)10433 static bool appendArrayType(SmallStringEnc &Enc, QualType QT,
10434 const ArrayType *AT,
10435 const CodeGen::CodeGenModule &CGM,
10436 TypeStringCache &TSC, StringRef NoSizeEnc) {
10437 if (AT->getSizeModifier() != ArrayType::Normal)
10438 return false;
10439 Enc += "a(";
10440 if (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(AT))
10441 CAT->getSize().toStringUnsigned(Enc);
10442 else
10443 Enc += NoSizeEnc; // Global arrays use "*", otherwise it is "".
10444 Enc += ':';
10445 // The Qualifiers should be attached to the type rather than the array.
10446 appendQualifier(Enc, QT);
10447 if (!appendType(Enc, AT->getElementType(), CGM, TSC))
10448 return false;
10449 Enc += ')';
10450 return true;
10451 }
10452
10453 /// Appends a function encoding to Enc, calling appendType for the return type
10454 /// and the arguments.
appendFunctionType(SmallStringEnc & Enc,const FunctionType * FT,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10455 static bool appendFunctionType(SmallStringEnc &Enc, const FunctionType *FT,
10456 const CodeGen::CodeGenModule &CGM,
10457 TypeStringCache &TSC) {
10458 Enc += "f{";
10459 if (!appendType(Enc, FT->getReturnType(), CGM, TSC))
10460 return false;
10461 Enc += "}(";
10462 if (const FunctionProtoType *FPT = FT->getAs<FunctionProtoType>()) {
10463 // N.B. we are only interested in the adjusted param types.
10464 auto I = FPT->param_type_begin();
10465 auto E = FPT->param_type_end();
10466 if (I != E) {
10467 do {
10468 if (!appendType(Enc, *I, CGM, TSC))
10469 return false;
10470 ++I;
10471 if (I != E)
10472 Enc += ',';
10473 } while (I != E);
10474 if (FPT->isVariadic())
10475 Enc += ",va";
10476 } else {
10477 if (FPT->isVariadic())
10478 Enc += "va";
10479 else
10480 Enc += '0';
10481 }
10482 }
10483 Enc += ')';
10484 return true;
10485 }
10486
10487 /// Handles the type's qualifier before dispatching a call to handle specific
10488 /// type encodings.
appendType(SmallStringEnc & Enc,QualType QType,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10489 static bool appendType(SmallStringEnc &Enc, QualType QType,
10490 const CodeGen::CodeGenModule &CGM,
10491 TypeStringCache &TSC) {
10492
10493 QualType QT = QType.getCanonicalType();
10494
10495 if (const ArrayType *AT = QT->getAsArrayTypeUnsafe())
10496 // The Qualifiers should be attached to the type rather than the array.
10497 // Thus we don't call appendQualifier() here.
10498 return appendArrayType(Enc, QT, AT, CGM, TSC, "");
10499
10500 appendQualifier(Enc, QT);
10501
10502 if (const BuiltinType *BT = QT->getAs<BuiltinType>())
10503 return appendBuiltinType(Enc, BT);
10504
10505 if (const PointerType *PT = QT->getAs<PointerType>())
10506 return appendPointerType(Enc, PT, CGM, TSC);
10507
10508 if (const EnumType *ET = QT->getAs<EnumType>())
10509 return appendEnumType(Enc, ET, TSC, QT.getBaseTypeIdentifier());
10510
10511 if (const RecordType *RT = QT->getAsStructureType())
10512 return appendRecordType(Enc, RT, CGM, TSC, QT.getBaseTypeIdentifier());
10513
10514 if (const RecordType *RT = QT->getAsUnionType())
10515 return appendRecordType(Enc, RT, CGM, TSC, QT.getBaseTypeIdentifier());
10516
10517 if (const FunctionType *FT = QT->getAs<FunctionType>())
10518 return appendFunctionType(Enc, FT, CGM, TSC);
10519
10520 return false;
10521 }
10522
getTypeString(SmallStringEnc & Enc,const Decl * D,const CodeGen::CodeGenModule & CGM,TypeStringCache & TSC)10523 static bool getTypeString(SmallStringEnc &Enc, const Decl *D,
10524 const CodeGen::CodeGenModule &CGM,
10525 TypeStringCache &TSC) {
10526 if (!D)
10527 return false;
10528
10529 if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
10530 if (FD->getLanguageLinkage() != CLanguageLinkage)
10531 return false;
10532 return appendType(Enc, FD->getType(), CGM, TSC);
10533 }
10534
10535 if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
10536 if (VD->getLanguageLinkage() != CLanguageLinkage)
10537 return false;
10538 QualType QT = VD->getType().getCanonicalType();
10539 if (const ArrayType *AT = QT->getAsArrayTypeUnsafe()) {
10540 // Global ArrayTypes are given a size of '*' if the size is unknown.
10541 // The Qualifiers should be attached to the type rather than the array.
10542 // Thus we don't call appendQualifier() here.
10543 return appendArrayType(Enc, QT, AT, CGM, TSC, "*");
10544 }
10545 return appendType(Enc, QT, CGM, TSC);
10546 }
10547 return false;
10548 }
10549
10550 //===----------------------------------------------------------------------===//
10551 // RISCV ABI Implementation
10552 //===----------------------------------------------------------------------===//
10553
10554 namespace {
10555 class RISCVABIInfo : public DefaultABIInfo {
10556 private:
10557 // Size of the integer ('x') registers in bits.
10558 unsigned XLen;
10559 // Size of the floating point ('f') registers in bits. Note that the target
10560 // ISA might have a wider FLen than the selected ABI (e.g. an RV32IF target
10561 // with soft float ABI has FLen==0).
10562 unsigned FLen;
10563 static const int NumArgGPRs = 8;
10564 static const int NumArgFPRs = 8;
10565 bool detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
10566 llvm::Type *&Field1Ty,
10567 CharUnits &Field1Off,
10568 llvm::Type *&Field2Ty,
10569 CharUnits &Field2Off) const;
10570
10571 public:
RISCVABIInfo(CodeGen::CodeGenTypes & CGT,unsigned XLen,unsigned FLen)10572 RISCVABIInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen, unsigned FLen)
10573 : DefaultABIInfo(CGT), XLen(XLen), FLen(FLen) {}
10574
10575 // DefaultABIInfo's classifyReturnType and classifyArgumentType are
10576 // non-virtual, but computeInfo is virtual, so we overload it.
10577 void computeInfo(CGFunctionInfo &FI) const override;
10578
10579 ABIArgInfo classifyArgumentType(QualType Ty, bool IsFixed, int &ArgGPRsLeft,
10580 int &ArgFPRsLeft) const;
10581 ABIArgInfo classifyReturnType(QualType RetTy) const;
10582
10583 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
10584 QualType Ty) const override;
10585
10586 ABIArgInfo extendType(QualType Ty) const;
10587
10588 bool detectFPCCEligibleStruct(QualType Ty, llvm::Type *&Field1Ty,
10589 CharUnits &Field1Off, llvm::Type *&Field2Ty,
10590 CharUnits &Field2Off, int &NeededArgGPRs,
10591 int &NeededArgFPRs) const;
10592 ABIArgInfo coerceAndExpandFPCCEligibleStruct(llvm::Type *Field1Ty,
10593 CharUnits Field1Off,
10594 llvm::Type *Field2Ty,
10595 CharUnits Field2Off) const;
10596 };
10597 } // end anonymous namespace
10598
computeInfo(CGFunctionInfo & FI) const10599 void RISCVABIInfo::computeInfo(CGFunctionInfo &FI) const {
10600 QualType RetTy = FI.getReturnType();
10601 if (!getCXXABI().classifyReturnType(FI))
10602 FI.getReturnInfo() = classifyReturnType(RetTy);
10603
10604 // IsRetIndirect is true if classifyArgumentType indicated the value should
10605 // be passed indirect, or if the type size is a scalar greater than 2*XLen
10606 // and not a complex type with elements <= FLen. e.g. fp128 is passed direct
10607 // in LLVM IR, relying on the backend lowering code to rewrite the argument
10608 // list and pass indirectly on RV32.
10609 bool IsRetIndirect = FI.getReturnInfo().getKind() == ABIArgInfo::Indirect;
10610 if (!IsRetIndirect && RetTy->isScalarType() &&
10611 getContext().getTypeSize(RetTy) > (2 * XLen)) {
10612 if (RetTy->isComplexType() && FLen) {
10613 QualType EltTy = RetTy->castAs<ComplexType>()->getElementType();
10614 IsRetIndirect = getContext().getTypeSize(EltTy) > FLen;
10615 } else {
10616 // This is a normal scalar > 2*XLen, such as fp128 on RV32.
10617 IsRetIndirect = true;
10618 }
10619 }
10620
10621 // We must track the number of GPRs used in order to conform to the RISC-V
10622 // ABI, as integer scalars passed in registers should have signext/zeroext
10623 // when promoted, but are anyext if passed on the stack. As GPR usage is
10624 // different for variadic arguments, we must also track whether we are
10625 // examining a vararg or not.
10626 int ArgGPRsLeft = IsRetIndirect ? NumArgGPRs - 1 : NumArgGPRs;
10627 int ArgFPRsLeft = FLen ? NumArgFPRs : 0;
10628 int NumFixedArgs = FI.getNumRequiredArgs();
10629
10630 int ArgNum = 0;
10631 for (auto &ArgInfo : FI.arguments()) {
10632 bool IsFixed = ArgNum < NumFixedArgs;
10633 ArgInfo.info =
10634 classifyArgumentType(ArgInfo.type, IsFixed, ArgGPRsLeft, ArgFPRsLeft);
10635 ArgNum++;
10636 }
10637 }
10638
10639 // Returns true if the struct is a potential candidate for the floating point
10640 // calling convention. If this function returns true, the caller is
10641 // responsible for checking that if there is only a single field then that
10642 // field is a float.
detectFPCCEligibleStructHelper(QualType Ty,CharUnits CurOff,llvm::Type * & Field1Ty,CharUnits & Field1Off,llvm::Type * & Field2Ty,CharUnits & Field2Off) const10643 bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
10644 llvm::Type *&Field1Ty,
10645 CharUnits &Field1Off,
10646 llvm::Type *&Field2Ty,
10647 CharUnits &Field2Off) const {
10648 bool IsInt = Ty->isIntegralOrEnumerationType();
10649 bool IsFloat = Ty->isRealFloatingType();
10650
10651 if (IsInt || IsFloat) {
10652 uint64_t Size = getContext().getTypeSize(Ty);
10653 if (IsInt && Size > XLen)
10654 return false;
10655 // Can't be eligible if larger than the FP registers. Half precision isn't
10656 // currently supported on RISC-V and the ABI hasn't been confirmed, so
10657 // default to the integer ABI in that case.
10658 if (IsFloat && (Size > FLen || Size < 32))
10659 return false;
10660 // Can't be eligible if an integer type was already found (int+int pairs
10661 // are not eligible).
10662 if (IsInt && Field1Ty && Field1Ty->isIntegerTy())
10663 return false;
10664 if (!Field1Ty) {
10665 Field1Ty = CGT.ConvertType(Ty);
10666 Field1Off = CurOff;
10667 return true;
10668 }
10669 if (!Field2Ty) {
10670 Field2Ty = CGT.ConvertType(Ty);
10671 Field2Off = CurOff;
10672 return true;
10673 }
10674 return false;
10675 }
10676
10677 if (auto CTy = Ty->getAs<ComplexType>()) {
10678 if (Field1Ty)
10679 return false;
10680 QualType EltTy = CTy->getElementType();
10681 if (getContext().getTypeSize(EltTy) > FLen)
10682 return false;
10683 Field1Ty = CGT.ConvertType(EltTy);
10684 Field1Off = CurOff;
10685 Field2Ty = Field1Ty;
10686 Field2Off = Field1Off + getContext().getTypeSizeInChars(EltTy);
10687 return true;
10688 }
10689
10690 if (const ConstantArrayType *ATy = getContext().getAsConstantArrayType(Ty)) {
10691 uint64_t ArraySize = ATy->getSize().getZExtValue();
10692 QualType EltTy = ATy->getElementType();
10693 CharUnits EltSize = getContext().getTypeSizeInChars(EltTy);
10694 for (uint64_t i = 0; i < ArraySize; ++i) {
10695 bool Ret = detectFPCCEligibleStructHelper(EltTy, CurOff, Field1Ty,
10696 Field1Off, Field2Ty, Field2Off);
10697 if (!Ret)
10698 return false;
10699 CurOff += EltSize;
10700 }
10701 return true;
10702 }
10703
10704 if (const auto *RTy = Ty->getAs<RecordType>()) {
10705 // Structures with either a non-trivial destructor or a non-trivial
10706 // copy constructor are not eligible for the FP calling convention.
10707 if (getRecordArgABI(Ty, CGT.getCXXABI()))
10708 return false;
10709 if (isEmptyRecord(getContext(), Ty, true))
10710 return true;
10711 const RecordDecl *RD = RTy->getDecl();
10712 // Unions aren't eligible unless they're empty (which is caught above).
10713 if (RD->isUnion())
10714 return false;
10715 int ZeroWidthBitFieldCount = 0;
10716 for (const FieldDecl *FD : RD->fields()) {
10717 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
10718 uint64_t FieldOffInBits = Layout.getFieldOffset(FD->getFieldIndex());
10719 QualType QTy = FD->getType();
10720 if (FD->isBitField()) {
10721 unsigned BitWidth = FD->getBitWidthValue(getContext());
10722 // Allow a bitfield with a type greater than XLen as long as the
10723 // bitwidth is XLen or less.
10724 if (getContext().getTypeSize(QTy) > XLen && BitWidth <= XLen)
10725 QTy = getContext().getIntTypeForBitwidth(XLen, false);
10726 if (BitWidth == 0) {
10727 ZeroWidthBitFieldCount++;
10728 continue;
10729 }
10730 }
10731
10732 bool Ret = detectFPCCEligibleStructHelper(
10733 QTy, CurOff + getContext().toCharUnitsFromBits(FieldOffInBits),
10734 Field1Ty, Field1Off, Field2Ty, Field2Off);
10735 if (!Ret)
10736 return false;
10737
10738 // As a quirk of the ABI, zero-width bitfields aren't ignored for fp+fp
10739 // or int+fp structs, but are ignored for a struct with an fp field and
10740 // any number of zero-width bitfields.
10741 if (Field2Ty && ZeroWidthBitFieldCount > 0)
10742 return false;
10743 }
10744 return Field1Ty != nullptr;
10745 }
10746
10747 return false;
10748 }
10749
10750 // Determine if a struct is eligible for passing according to the floating
10751 // point calling convention (i.e., when flattened it contains a single fp
10752 // value, fp+fp, or int+fp of appropriate size). If so, NeededArgFPRs and
10753 // NeededArgGPRs are incremented appropriately.
detectFPCCEligibleStruct(QualType Ty,llvm::Type * & Field1Ty,CharUnits & Field1Off,llvm::Type * & Field2Ty,CharUnits & Field2Off,int & NeededArgGPRs,int & NeededArgFPRs) const10754 bool RISCVABIInfo::detectFPCCEligibleStruct(QualType Ty, llvm::Type *&Field1Ty,
10755 CharUnits &Field1Off,
10756 llvm::Type *&Field2Ty,
10757 CharUnits &Field2Off,
10758 int &NeededArgGPRs,
10759 int &NeededArgFPRs) const {
10760 Field1Ty = nullptr;
10761 Field2Ty = nullptr;
10762 NeededArgGPRs = 0;
10763 NeededArgFPRs = 0;
10764 bool IsCandidate = detectFPCCEligibleStructHelper(
10765 Ty, CharUnits::Zero(), Field1Ty, Field1Off, Field2Ty, Field2Off);
10766 // Not really a candidate if we have a single int but no float.
10767 if (Field1Ty && !Field2Ty && !Field1Ty->isFloatingPointTy())
10768 return false;
10769 if (!IsCandidate)
10770 return false;
10771 if (Field1Ty && Field1Ty->isFloatingPointTy())
10772 NeededArgFPRs++;
10773 else if (Field1Ty)
10774 NeededArgGPRs++;
10775 if (Field2Ty && Field2Ty->isFloatingPointTy())
10776 NeededArgFPRs++;
10777 else if (Field2Ty)
10778 NeededArgGPRs++;
10779 return true;
10780 }
10781
10782 // Call getCoerceAndExpand for the two-element flattened struct described by
10783 // Field1Ty, Field1Off, Field2Ty, Field2Off. This method will create an
10784 // appropriate coerceToType and unpaddedCoerceToType.
coerceAndExpandFPCCEligibleStruct(llvm::Type * Field1Ty,CharUnits Field1Off,llvm::Type * Field2Ty,CharUnits Field2Off) const10785 ABIArgInfo RISCVABIInfo::coerceAndExpandFPCCEligibleStruct(
10786 llvm::Type *Field1Ty, CharUnits Field1Off, llvm::Type *Field2Ty,
10787 CharUnits Field2Off) const {
10788 SmallVector<llvm::Type *, 3> CoerceElts;
10789 SmallVector<llvm::Type *, 2> UnpaddedCoerceElts;
10790 if (!Field1Off.isZero())
10791 CoerceElts.push_back(llvm::ArrayType::get(
10792 llvm::Type::getInt8Ty(getVMContext()), Field1Off.getQuantity()));
10793
10794 CoerceElts.push_back(Field1Ty);
10795 UnpaddedCoerceElts.push_back(Field1Ty);
10796
10797 if (!Field2Ty) {
10798 return ABIArgInfo::getCoerceAndExpand(
10799 llvm::StructType::get(getVMContext(), CoerceElts, !Field1Off.isZero()),
10800 UnpaddedCoerceElts[0]);
10801 }
10802
10803 CharUnits Field2Align =
10804 CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(Field2Ty));
10805 CharUnits Field1End = Field1Off +
10806 CharUnits::fromQuantity(getDataLayout().getTypeStoreSize(Field1Ty));
10807 CharUnits Field2OffNoPadNoPack = Field1End.alignTo(Field2Align);
10808
10809 CharUnits Padding = CharUnits::Zero();
10810 if (Field2Off > Field2OffNoPadNoPack)
10811 Padding = Field2Off - Field2OffNoPadNoPack;
10812 else if (Field2Off != Field2Align && Field2Off > Field1End)
10813 Padding = Field2Off - Field1End;
10814
10815 bool IsPacked = !Field2Off.isMultipleOf(Field2Align);
10816
10817 if (!Padding.isZero())
10818 CoerceElts.push_back(llvm::ArrayType::get(
10819 llvm::Type::getInt8Ty(getVMContext()), Padding.getQuantity()));
10820
10821 CoerceElts.push_back(Field2Ty);
10822 UnpaddedCoerceElts.push_back(Field2Ty);
10823
10824 auto CoerceToType =
10825 llvm::StructType::get(getVMContext(), CoerceElts, IsPacked);
10826 auto UnpaddedCoerceToType =
10827 llvm::StructType::get(getVMContext(), UnpaddedCoerceElts, IsPacked);
10828
10829 return ABIArgInfo::getCoerceAndExpand(CoerceToType, UnpaddedCoerceToType);
10830 }
10831
classifyArgumentType(QualType Ty,bool IsFixed,int & ArgGPRsLeft,int & ArgFPRsLeft) const10832 ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
10833 int &ArgGPRsLeft,
10834 int &ArgFPRsLeft) const {
10835 assert(ArgGPRsLeft <= NumArgGPRs && "Arg GPR tracking underflow");
10836 Ty = useFirstFieldIfTransparentUnion(Ty);
10837
10838 // Structures with either a non-trivial destructor or a non-trivial
10839 // copy constructor are always passed indirectly.
10840 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
10841 if (ArgGPRsLeft)
10842 ArgGPRsLeft -= 1;
10843 return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
10844 CGCXXABI::RAA_DirectInMemory);
10845 }
10846
10847 // Ignore empty structs/unions.
10848 if (isEmptyRecord(getContext(), Ty, true))
10849 return ABIArgInfo::getIgnore();
10850
10851 uint64_t Size = getContext().getTypeSize(Ty);
10852
10853 // Pass floating point values via FPRs if possible.
10854 if (IsFixed && Ty->isFloatingType() && !Ty->isComplexType() &&
10855 FLen >= Size && ArgFPRsLeft) {
10856 ArgFPRsLeft--;
10857 return ABIArgInfo::getDirect();
10858 }
10859
10860 // Complex types for the hard float ABI must be passed direct rather than
10861 // using CoerceAndExpand.
10862 if (IsFixed && Ty->isComplexType() && FLen && ArgFPRsLeft >= 2) {
10863 QualType EltTy = Ty->castAs<ComplexType>()->getElementType();
10864 if (getContext().getTypeSize(EltTy) <= FLen) {
10865 ArgFPRsLeft -= 2;
10866 return ABIArgInfo::getDirect();
10867 }
10868 }
10869
10870 if (IsFixed && FLen && Ty->isStructureOrClassType()) {
10871 llvm::Type *Field1Ty = nullptr;
10872 llvm::Type *Field2Ty = nullptr;
10873 CharUnits Field1Off = CharUnits::Zero();
10874 CharUnits Field2Off = CharUnits::Zero();
10875 int NeededArgGPRs = 0;
10876 int NeededArgFPRs = 0;
10877 bool IsCandidate =
10878 detectFPCCEligibleStruct(Ty, Field1Ty, Field1Off, Field2Ty, Field2Off,
10879 NeededArgGPRs, NeededArgFPRs);
10880 if (IsCandidate && NeededArgGPRs <= ArgGPRsLeft &&
10881 NeededArgFPRs <= ArgFPRsLeft) {
10882 ArgGPRsLeft -= NeededArgGPRs;
10883 ArgFPRsLeft -= NeededArgFPRs;
10884 return coerceAndExpandFPCCEligibleStruct(Field1Ty, Field1Off, Field2Ty,
10885 Field2Off);
10886 }
10887 }
10888
10889 uint64_t NeededAlign = getContext().getTypeAlign(Ty);
10890 bool MustUseStack = false;
10891 // Determine the number of GPRs needed to pass the current argument
10892 // according to the ABI. 2*XLen-aligned varargs are passed in "aligned"
10893 // register pairs, so may consume 3 registers.
10894 int NeededArgGPRs = 1;
10895 if (!IsFixed && NeededAlign == 2 * XLen)
10896 NeededArgGPRs = 2 + (ArgGPRsLeft % 2);
10897 else if (Size > XLen && Size <= 2 * XLen)
10898 NeededArgGPRs = 2;
10899
10900 if (NeededArgGPRs > ArgGPRsLeft) {
10901 MustUseStack = true;
10902 NeededArgGPRs = ArgGPRsLeft;
10903 }
10904
10905 ArgGPRsLeft -= NeededArgGPRs;
10906
10907 if (!isAggregateTypeForABI(Ty) && !Ty->isVectorType()) {
10908 // Treat an enum type as its underlying type.
10909 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
10910 Ty = EnumTy->getDecl()->getIntegerType();
10911
10912 // All integral types are promoted to XLen width, unless passed on the
10913 // stack.
10914 if (Size < XLen && Ty->isIntegralOrEnumerationType() && !MustUseStack) {
10915 return extendType(Ty);
10916 }
10917
10918 if (const auto *EIT = Ty->getAs<ExtIntType>()) {
10919 if (EIT->getNumBits() < XLen && !MustUseStack)
10920 return extendType(Ty);
10921 if (EIT->getNumBits() > 128 ||
10922 (!getContext().getTargetInfo().hasInt128Type() &&
10923 EIT->getNumBits() > 64))
10924 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
10925 }
10926
10927 return ABIArgInfo::getDirect();
10928 }
10929
10930 // Aggregates which are <= 2*XLen will be passed in registers if possible,
10931 // so coerce to integers.
10932 if (Size <= 2 * XLen) {
10933 unsigned Alignment = getContext().getTypeAlign(Ty);
10934
10935 // Use a single XLen int if possible, 2*XLen if 2*XLen alignment is
10936 // required, and a 2-element XLen array if only XLen alignment is required.
10937 if (Size <= XLen) {
10938 return ABIArgInfo::getDirect(
10939 llvm::IntegerType::get(getVMContext(), XLen));
10940 } else if (Alignment == 2 * XLen) {
10941 return ABIArgInfo::getDirect(
10942 llvm::IntegerType::get(getVMContext(), 2 * XLen));
10943 } else {
10944 return ABIArgInfo::getDirect(llvm::ArrayType::get(
10945 llvm::IntegerType::get(getVMContext(), XLen), 2));
10946 }
10947 }
10948 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
10949 }
10950
classifyReturnType(QualType RetTy) const10951 ABIArgInfo RISCVABIInfo::classifyReturnType(QualType RetTy) const {
10952 if (RetTy->isVoidType())
10953 return ABIArgInfo::getIgnore();
10954
10955 int ArgGPRsLeft = 2;
10956 int ArgFPRsLeft = FLen ? 2 : 0;
10957
10958 // The rules for return and argument types are the same, so defer to
10959 // classifyArgumentType.
10960 return classifyArgumentType(RetTy, /*IsFixed=*/true, ArgGPRsLeft,
10961 ArgFPRsLeft);
10962 }
10963
EmitVAArg(CodeGenFunction & CGF,Address VAListAddr,QualType Ty) const10964 Address RISCVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
10965 QualType Ty) const {
10966 CharUnits SlotSize = CharUnits::fromQuantity(XLen / 8);
10967
10968 // Empty records are ignored for parameter passing purposes.
10969 if (isEmptyRecord(getContext(), Ty, true)) {
10970 Address Addr(CGF.Builder.CreateLoad(VAListAddr), SlotSize);
10971 Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
10972 return Addr;
10973 }
10974
10975 auto TInfo = getContext().getTypeInfoInChars(Ty);
10976
10977 // Arguments bigger than 2*Xlen bytes are passed indirectly.
10978 bool IsIndirect = TInfo.Width > 2 * SlotSize;
10979
10980 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TInfo,
10981 SlotSize, /*AllowHigherAlign=*/true);
10982 }
10983
extendType(QualType Ty) const10984 ABIArgInfo RISCVABIInfo::extendType(QualType Ty) const {
10985 int TySize = getContext().getTypeSize(Ty);
10986 // RV64 ABI requires unsigned 32 bit integers to be sign extended.
10987 if (XLen == 64 && Ty->isUnsignedIntegerOrEnumerationType() && TySize == 32)
10988 return ABIArgInfo::getSignExtend(Ty);
10989 return ABIArgInfo::getExtend(Ty);
10990 }
10991
10992 namespace {
10993 class RISCVTargetCodeGenInfo : public TargetCodeGenInfo {
10994 public:
RISCVTargetCodeGenInfo(CodeGen::CodeGenTypes & CGT,unsigned XLen,unsigned FLen)10995 RISCVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen,
10996 unsigned FLen)
10997 : TargetCodeGenInfo(std::make_unique<RISCVABIInfo>(CGT, XLen, FLen)) {}
10998
setTargetAttributes(const Decl * D,llvm::GlobalValue * GV,CodeGen::CodeGenModule & CGM) const10999 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
11000 CodeGen::CodeGenModule &CGM) const override {
11001 const auto *FD = dyn_cast_or_null<FunctionDecl>(D);
11002 if (!FD) return;
11003
11004 const auto *Attr = FD->getAttr<RISCVInterruptAttr>();
11005 if (!Attr)
11006 return;
11007
11008 const char *Kind;
11009 switch (Attr->getInterrupt()) {
11010 case RISCVInterruptAttr::user: Kind = "user"; break;
11011 case RISCVInterruptAttr::supervisor: Kind = "supervisor"; break;
11012 case RISCVInterruptAttr::machine: Kind = "machine"; break;
11013 }
11014
11015 auto *Fn = cast<llvm::Function>(GV);
11016
11017 Fn->addFnAttr("interrupt", Kind);
11018 }
11019 };
11020 } // namespace
11021
11022 //===----------------------------------------------------------------------===//
11023 // VE ABI Implementation.
11024 //
11025 namespace {
11026 class VEABIInfo : public DefaultABIInfo {
11027 public:
VEABIInfo(CodeGenTypes & CGT)11028 VEABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
11029
11030 private:
11031 ABIArgInfo classifyReturnType(QualType RetTy) const;
11032 ABIArgInfo classifyArgumentType(QualType RetTy) const;
11033 void computeInfo(CGFunctionInfo &FI) const override;
11034 };
11035 } // end anonymous namespace
11036
classifyReturnType(QualType Ty) const11037 ABIArgInfo VEABIInfo::classifyReturnType(QualType Ty) const {
11038 if (Ty->isAnyComplexType())
11039 return ABIArgInfo::getDirect();
11040 uint64_t Size = getContext().getTypeSize(Ty);
11041 if (Size < 64 && Ty->isIntegerType())
11042 return ABIArgInfo::getExtend(Ty);
11043 return DefaultABIInfo::classifyReturnType(Ty);
11044 }
11045
classifyArgumentType(QualType Ty) const11046 ABIArgInfo VEABIInfo::classifyArgumentType(QualType Ty) const {
11047 if (Ty->isAnyComplexType())
11048 return ABIArgInfo::getDirect();
11049 uint64_t Size = getContext().getTypeSize(Ty);
11050 if (Size < 64 && Ty->isIntegerType())
11051 return ABIArgInfo::getExtend(Ty);
11052 return DefaultABIInfo::classifyArgumentType(Ty);
11053 }
11054
computeInfo(CGFunctionInfo & FI) const11055 void VEABIInfo::computeInfo(CGFunctionInfo &FI) const {
11056 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
11057 for (auto &Arg : FI.arguments())
11058 Arg.info = classifyArgumentType(Arg.type);
11059 }
11060
11061 namespace {
11062 class VETargetCodeGenInfo : public TargetCodeGenInfo {
11063 public:
VETargetCodeGenInfo(CodeGenTypes & CGT)11064 VETargetCodeGenInfo(CodeGenTypes &CGT)
11065 : TargetCodeGenInfo(std::make_unique<VEABIInfo>(CGT)) {}
11066 // VE ABI requires the arguments of variadic and prototype-less functions
11067 // are passed in both registers and memory.
isNoProtoCallVariadic(const CallArgList & args,const FunctionNoProtoType * fnType) const11068 bool isNoProtoCallVariadic(const CallArgList &args,
11069 const FunctionNoProtoType *fnType) const override {
11070 return true;
11071 }
11072 };
11073 } // end anonymous namespace
11074
11075 //===----------------------------------------------------------------------===//
11076 // Driver code
11077 //===----------------------------------------------------------------------===//
11078
supportsCOMDAT() const11079 bool CodeGenModule::supportsCOMDAT() const {
11080 return getTriple().supportsCOMDAT();
11081 }
11082
getTargetCodeGenInfo()11083 const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
11084 if (TheTargetCodeGenInfo)
11085 return *TheTargetCodeGenInfo;
11086
11087 // Helper to set the unique_ptr while still keeping the return value.
11088 auto SetCGInfo = [&](TargetCodeGenInfo *P) -> const TargetCodeGenInfo & {
11089 this->TheTargetCodeGenInfo.reset(P);
11090 return *P;
11091 };
11092
11093 const llvm::Triple &Triple = getTarget().getTriple();
11094 switch (Triple.getArch()) {
11095 default:
11096 return SetCGInfo(new DefaultTargetCodeGenInfo(Types));
11097
11098 case llvm::Triple::le32:
11099 return SetCGInfo(new PNaClTargetCodeGenInfo(Types));
11100 case llvm::Triple::m68k:
11101 return SetCGInfo(new M68kTargetCodeGenInfo(Types));
11102 case llvm::Triple::mips:
11103 case llvm::Triple::mipsel:
11104 if (Triple.getOS() == llvm::Triple::NaCl)
11105 return SetCGInfo(new PNaClTargetCodeGenInfo(Types));
11106 return SetCGInfo(new MIPSTargetCodeGenInfo(Types, true));
11107
11108 case llvm::Triple::mips64:
11109 case llvm::Triple::mips64el:
11110 return SetCGInfo(new MIPSTargetCodeGenInfo(Types, false));
11111
11112 case llvm::Triple::avr:
11113 return SetCGInfo(new AVRTargetCodeGenInfo(Types));
11114
11115 case llvm::Triple::aarch64:
11116 case llvm::Triple::aarch64_32:
11117 case llvm::Triple::aarch64_be: {
11118 AArch64ABIInfo::ABIKind Kind = AArch64ABIInfo::AAPCS;
11119 if (getTarget().getABI() == "darwinpcs")
11120 Kind = AArch64ABIInfo::DarwinPCS;
11121 else if (Triple.isOSWindows())
11122 return SetCGInfo(
11123 new WindowsAArch64TargetCodeGenInfo(Types, AArch64ABIInfo::Win64));
11124
11125 return SetCGInfo(new AArch64TargetCodeGenInfo(Types, Kind));
11126 }
11127
11128 case llvm::Triple::wasm32:
11129 case llvm::Triple::wasm64: {
11130 WebAssemblyABIInfo::ABIKind Kind = WebAssemblyABIInfo::MVP;
11131 if (getTarget().getABI() == "experimental-mv")
11132 Kind = WebAssemblyABIInfo::ExperimentalMV;
11133 return SetCGInfo(new WebAssemblyTargetCodeGenInfo(Types, Kind));
11134 }
11135
11136 case llvm::Triple::arm:
11137 case llvm::Triple::armeb:
11138 case llvm::Triple::thumb:
11139 case llvm::Triple::thumbeb: {
11140 if (Triple.getOS() == llvm::Triple::Win32) {
11141 return SetCGInfo(
11142 new WindowsARMTargetCodeGenInfo(Types, ARMABIInfo::AAPCS_VFP));
11143 }
11144
11145 ARMABIInfo::ABIKind Kind = ARMABIInfo::AAPCS;
11146 StringRef ABIStr = getTarget().getABI();
11147 if (ABIStr == "apcs-gnu")
11148 Kind = ARMABIInfo::APCS;
11149 else if (ABIStr == "aapcs16")
11150 Kind = ARMABIInfo::AAPCS16_VFP;
11151 else if (CodeGenOpts.FloatABI == "hard" ||
11152 (CodeGenOpts.FloatABI != "soft" &&
11153 (Triple.getEnvironment() == llvm::Triple::GNUEABIHF ||
11154 Triple.getEnvironment() == llvm::Triple::MuslEABIHF ||
11155 Triple.getEnvironment() == llvm::Triple::EABIHF)))
11156 Kind = ARMABIInfo::AAPCS_VFP;
11157
11158 return SetCGInfo(new ARMTargetCodeGenInfo(Types, Kind));
11159 }
11160
11161 case llvm::Triple::ppc: {
11162 if (Triple.isOSAIX())
11163 return SetCGInfo(new AIXTargetCodeGenInfo(Types, /*Is64Bit*/ false));
11164
11165 bool IsSoftFloat =
11166 CodeGenOpts.FloatABI == "soft" || getTarget().hasFeature("spe");
11167 bool RetSmallStructInRegABI =
11168 PPC32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts);
11169 return SetCGInfo(
11170 new PPC32TargetCodeGenInfo(Types, IsSoftFloat, RetSmallStructInRegABI));
11171 }
11172 case llvm::Triple::ppcle: {
11173 bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
11174 bool RetSmallStructInRegABI =
11175 PPC32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts);
11176 return SetCGInfo(
11177 new PPC32TargetCodeGenInfo(Types, IsSoftFloat, RetSmallStructInRegABI));
11178 }
11179 case llvm::Triple::ppc64:
11180 if (Triple.isOSAIX())
11181 return SetCGInfo(new AIXTargetCodeGenInfo(Types, /*Is64Bit*/ true));
11182
11183 if (Triple.isOSBinFormatELF()) {
11184 PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv1;
11185 if (getTarget().getABI() == "elfv2")
11186 Kind = PPC64_SVR4_ABIInfo::ELFv2;
11187 bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
11188
11189 return SetCGInfo(
11190 new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, IsSoftFloat));
11191 }
11192 return SetCGInfo(new PPC64TargetCodeGenInfo(Types));
11193 case llvm::Triple::ppc64le: {
11194 assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!");
11195 PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv2;
11196 if (getTarget().getABI() == "elfv1")
11197 Kind = PPC64_SVR4_ABIInfo::ELFv1;
11198 bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
11199
11200 return SetCGInfo(
11201 new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, IsSoftFloat));
11202 }
11203
11204 case llvm::Triple::nvptx:
11205 case llvm::Triple::nvptx64:
11206 return SetCGInfo(new NVPTXTargetCodeGenInfo(Types));
11207
11208 case llvm::Triple::msp430:
11209 return SetCGInfo(new MSP430TargetCodeGenInfo(Types));
11210
11211 case llvm::Triple::riscv32:
11212 case llvm::Triple::riscv64: {
11213 StringRef ABIStr = getTarget().getABI();
11214 unsigned XLen = getTarget().getPointerWidth(0);
11215 unsigned ABIFLen = 0;
11216 if (ABIStr.endswith("f"))
11217 ABIFLen = 32;
11218 else if (ABIStr.endswith("d"))
11219 ABIFLen = 64;
11220 return SetCGInfo(new RISCVTargetCodeGenInfo(Types, XLen, ABIFLen));
11221 }
11222
11223 case llvm::Triple::systemz: {
11224 bool SoftFloat = CodeGenOpts.FloatABI == "soft";
11225 bool HasVector = !SoftFloat && getTarget().getABI() == "vector";
11226 return SetCGInfo(new SystemZTargetCodeGenInfo(Types, HasVector, SoftFloat));
11227 }
11228
11229 case llvm::Triple::tce:
11230 case llvm::Triple::tcele:
11231 return SetCGInfo(new TCETargetCodeGenInfo(Types));
11232
11233 case llvm::Triple::x86: {
11234 bool IsDarwinVectorABI = Triple.isOSDarwin();
11235 bool RetSmallStructInRegABI =
11236 X86_32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts);
11237 bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing();
11238
11239 if (Triple.getOS() == llvm::Triple::Win32) {
11240 return SetCGInfo(new WinX86_32TargetCodeGenInfo(
11241 Types, IsDarwinVectorABI, RetSmallStructInRegABI,
11242 IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters));
11243 } else {
11244 return SetCGInfo(new X86_32TargetCodeGenInfo(
11245 Types, IsDarwinVectorABI, RetSmallStructInRegABI,
11246 IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters,
11247 CodeGenOpts.FloatABI == "soft"));
11248 }
11249 }
11250
11251 case llvm::Triple::x86_64: {
11252 StringRef ABI = getTarget().getABI();
11253 X86AVXABILevel AVXLevel =
11254 (ABI == "avx512"
11255 ? X86AVXABILevel::AVX512
11256 : ABI == "avx" ? X86AVXABILevel::AVX : X86AVXABILevel::None);
11257
11258 switch (Triple.getOS()) {
11259 case llvm::Triple::Win32:
11260 return SetCGInfo(new WinX86_64TargetCodeGenInfo(Types, AVXLevel));
11261 default:
11262 return SetCGInfo(new X86_64TargetCodeGenInfo(Types, AVXLevel));
11263 }
11264 }
11265 case llvm::Triple::hexagon:
11266 return SetCGInfo(new HexagonTargetCodeGenInfo(Types));
11267 case llvm::Triple::lanai:
11268 return SetCGInfo(new LanaiTargetCodeGenInfo(Types));
11269 case llvm::Triple::r600:
11270 return SetCGInfo(new AMDGPUTargetCodeGenInfo(Types));
11271 case llvm::Triple::amdgcn:
11272 return SetCGInfo(new AMDGPUTargetCodeGenInfo(Types));
11273 case llvm::Triple::sparc:
11274 return SetCGInfo(new SparcV8TargetCodeGenInfo(Types));
11275 case llvm::Triple::sparcv9:
11276 return SetCGInfo(new SparcV9TargetCodeGenInfo(Types));
11277 case llvm::Triple::xcore:
11278 return SetCGInfo(new XCoreTargetCodeGenInfo(Types));
11279 case llvm::Triple::arc:
11280 return SetCGInfo(new ARCTargetCodeGenInfo(Types));
11281 case llvm::Triple::spir:
11282 case llvm::Triple::spir64:
11283 return SetCGInfo(new SPIRTargetCodeGenInfo(Types));
11284 case llvm::Triple::ve:
11285 return SetCGInfo(new VETargetCodeGenInfo(Types));
11286 }
11287 }
11288
11289 /// Create an OpenCL kernel for an enqueued block.
11290 ///
11291 /// The kernel has the same function type as the block invoke function. Its
11292 /// name is the name of the block invoke function postfixed with "_kernel".
11293 /// It simply calls the block invoke function then returns.
11294 llvm::Function *
createEnqueuedBlockKernel(CodeGenFunction & CGF,llvm::Function * Invoke,llvm::Value * BlockLiteral) const11295 TargetCodeGenInfo::createEnqueuedBlockKernel(CodeGenFunction &CGF,
11296 llvm::Function *Invoke,
11297 llvm::Value *BlockLiteral) const {
11298 auto *InvokeFT = Invoke->getFunctionType();
11299 llvm::SmallVector<llvm::Type *, 2> ArgTys;
11300 for (auto &P : InvokeFT->params())
11301 ArgTys.push_back(P);
11302 auto &C = CGF.getLLVMContext();
11303 std::string Name = Invoke->getName().str() + "_kernel";
11304 auto *FT = llvm::FunctionType::get(llvm::Type::getVoidTy(C), ArgTys, false);
11305 auto *F = llvm::Function::Create(FT, llvm::GlobalValue::InternalLinkage, Name,
11306 &CGF.CGM.getModule());
11307 auto IP = CGF.Builder.saveIP();
11308 auto *BB = llvm::BasicBlock::Create(C, "entry", F);
11309 auto &Builder = CGF.Builder;
11310 Builder.SetInsertPoint(BB);
11311 llvm::SmallVector<llvm::Value *, 2> Args;
11312 for (auto &A : F->args())
11313 Args.push_back(&A);
11314 llvm::CallInst *call = Builder.CreateCall(Invoke, Args);
11315 call->setCallingConv(Invoke->getCallingConv());
11316 Builder.CreateRetVoid();
11317 Builder.restoreIP(IP);
11318 return F;
11319 }
11320
11321 /// Create an OpenCL kernel for an enqueued block.
11322 ///
11323 /// The type of the first argument (the block literal) is the struct type
11324 /// of the block literal instead of a pointer type. The first argument
11325 /// (block literal) is passed directly by value to the kernel. The kernel
11326 /// allocates the same type of struct on stack and stores the block literal
11327 /// to it and passes its pointer to the block invoke function. The kernel
11328 /// has "enqueued-block" function attribute and kernel argument metadata.
createEnqueuedBlockKernel(CodeGenFunction & CGF,llvm::Function * Invoke,llvm::Value * BlockLiteral) const11329 llvm::Function *AMDGPUTargetCodeGenInfo::createEnqueuedBlockKernel(
11330 CodeGenFunction &CGF, llvm::Function *Invoke,
11331 llvm::Value *BlockLiteral) const {
11332 auto &Builder = CGF.Builder;
11333 auto &C = CGF.getLLVMContext();
11334
11335 auto *BlockTy = BlockLiteral->getType()->getPointerElementType();
11336 auto *InvokeFT = Invoke->getFunctionType();
11337 llvm::SmallVector<llvm::Type *, 2> ArgTys;
11338 llvm::SmallVector<llvm::Metadata *, 8> AddressQuals;
11339 llvm::SmallVector<llvm::Metadata *, 8> AccessQuals;
11340 llvm::SmallVector<llvm::Metadata *, 8> ArgTypeNames;
11341 llvm::SmallVector<llvm::Metadata *, 8> ArgBaseTypeNames;
11342 llvm::SmallVector<llvm::Metadata *, 8> ArgTypeQuals;
11343 llvm::SmallVector<llvm::Metadata *, 8> ArgNames;
11344
11345 ArgTys.push_back(BlockTy);
11346 ArgTypeNames.push_back(llvm::MDString::get(C, "__block_literal"));
11347 AddressQuals.push_back(llvm::ConstantAsMetadata::get(Builder.getInt32(0)));
11348 ArgBaseTypeNames.push_back(llvm::MDString::get(C, "__block_literal"));
11349 ArgTypeQuals.push_back(llvm::MDString::get(C, ""));
11350 AccessQuals.push_back(llvm::MDString::get(C, "none"));
11351 ArgNames.push_back(llvm::MDString::get(C, "block_literal"));
11352 for (unsigned I = 1, E = InvokeFT->getNumParams(); I < E; ++I) {
11353 ArgTys.push_back(InvokeFT->getParamType(I));
11354 ArgTypeNames.push_back(llvm::MDString::get(C, "void*"));
11355 AddressQuals.push_back(llvm::ConstantAsMetadata::get(Builder.getInt32(3)));
11356 AccessQuals.push_back(llvm::MDString::get(C, "none"));
11357 ArgBaseTypeNames.push_back(llvm::MDString::get(C, "void*"));
11358 ArgTypeQuals.push_back(llvm::MDString::get(C, ""));
11359 ArgNames.push_back(
11360 llvm::MDString::get(C, (Twine("local_arg") + Twine(I)).str()));
11361 }
11362 std::string Name = Invoke->getName().str() + "_kernel";
11363 auto *FT = llvm::FunctionType::get(llvm::Type::getVoidTy(C), ArgTys, false);
11364 auto *F = llvm::Function::Create(FT, llvm::GlobalValue::InternalLinkage, Name,
11365 &CGF.CGM.getModule());
11366 F->addFnAttr("enqueued-block");
11367 auto IP = CGF.Builder.saveIP();
11368 auto *BB = llvm::BasicBlock::Create(C, "entry", F);
11369 Builder.SetInsertPoint(BB);
11370 const auto BlockAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(BlockTy);
11371 auto *BlockPtr = Builder.CreateAlloca(BlockTy, nullptr);
11372 BlockPtr->setAlignment(BlockAlign);
11373 Builder.CreateAlignedStore(F->arg_begin(), BlockPtr, BlockAlign);
11374 auto *Cast = Builder.CreatePointerCast(BlockPtr, InvokeFT->getParamType(0));
11375 llvm::SmallVector<llvm::Value *, 2> Args;
11376 Args.push_back(Cast);
11377 for (auto I = F->arg_begin() + 1, E = F->arg_end(); I != E; ++I)
11378 Args.push_back(I);
11379 llvm::CallInst *call = Builder.CreateCall(Invoke, Args);
11380 call->setCallingConv(Invoke->getCallingConv());
11381 Builder.CreateRetVoid();
11382 Builder.restoreIP(IP);
11383
11384 F->setMetadata("kernel_arg_addr_space", llvm::MDNode::get(C, AddressQuals));
11385 F->setMetadata("kernel_arg_access_qual", llvm::MDNode::get(C, AccessQuals));
11386 F->setMetadata("kernel_arg_type", llvm::MDNode::get(C, ArgTypeNames));
11387 F->setMetadata("kernel_arg_base_type",
11388 llvm::MDNode::get(C, ArgBaseTypeNames));
11389 F->setMetadata("kernel_arg_type_qual", llvm::MDNode::get(C, ArgTypeQuals));
11390 if (CGF.CGM.getCodeGenOpts().EmitOpenCLArgMetadata)
11391 F->setMetadata("kernel_arg_name", llvm::MDNode::get(C, ArgNames));
11392
11393 return F;
11394 }
11395