1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements some simple delegations needed for call lowering.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/Analysis.h"
15 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
16 #include "llvm/CodeGen/GlobalISel/Utils.h"
17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18 #include "llvm/CodeGen/MachineOperand.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/TargetLowering.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Instructions.h"
23 #include "llvm/IR/LLVMContext.h"
24 #include "llvm/IR/Module.h"
25 
26 #define DEBUG_TYPE "call-lowering"
27 
28 using namespace llvm;
29 
anchor()30 void CallLowering::anchor() {}
31 
lowerCall(MachineIRBuilder & MIRBuilder,ImmutableCallSite CS,ArrayRef<Register> ResRegs,ArrayRef<ArrayRef<Register>> ArgRegs,Register SwiftErrorVReg,std::function<unsigned ()> GetCalleeReg) const32 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
33                              ArrayRef<Register> ResRegs,
34                              ArrayRef<ArrayRef<Register>> ArgRegs,
35                              Register SwiftErrorVReg,
36                              std::function<unsigned()> GetCalleeReg) const {
37   CallLoweringInfo Info;
38   auto &DL = CS.getParent()->getParent()->getParent()->getDataLayout();
39 
40   // First step is to marshall all the function's parameters into the correct
41   // physregs and memory locations. Gather the sequence of argument types that
42   // we'll pass to the assigner function.
43   unsigned i = 0;
44   unsigned NumFixedArgs = CS.getFunctionType()->getNumParams();
45   for (auto &Arg : CS.args()) {
46     ArgInfo OrigArg{ArgRegs[i], Arg->getType(), ISD::ArgFlagsTy{},
47                     i < NumFixedArgs};
48     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CS);
49     Info.OrigArgs.push_back(OrigArg);
50     ++i;
51   }
52 
53   if (const Function *F = CS.getCalledFunction())
54     Info.Callee = MachineOperand::CreateGA(F, 0);
55   else
56     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
57 
58   Info.OrigRet = ArgInfo{ResRegs, CS.getType(), ISD::ArgFlagsTy{}};
59   if (!Info.OrigRet.Ty->isVoidTy())
60     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CS);
61 
62   Info.KnownCallees =
63       CS.getInstruction()->getMetadata(LLVMContext::MD_callees);
64   Info.CallConv = CS.getCallingConv();
65   Info.SwiftErrorVReg = SwiftErrorVReg;
66   Info.IsMustTailCall = CS.isMustTailCall();
67   Info.IsTailCall = CS.isTailCall() &&
68                     isInTailCallPosition(CS, MIRBuilder.getMF().getTarget()) &&
69                     (MIRBuilder.getMF()
70                          .getFunction()
71                          .getFnAttribute("disable-tail-calls")
72                          .getValueAsString() != "true");
73   Info.IsVarArg = CS.getFunctionType()->isVarArg();
74   return lowerCall(MIRBuilder, Info);
75 }
76 
77 template <typename FuncInfoTy>
setArgFlags(CallLowering::ArgInfo & Arg,unsigned OpIdx,const DataLayout & DL,const FuncInfoTy & FuncInfo) const78 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
79                                const DataLayout &DL,
80                                const FuncInfoTy &FuncInfo) const {
81   auto &Flags = Arg.Flags[0];
82   const AttributeList &Attrs = FuncInfo.getAttributes();
83   if (Attrs.hasAttribute(OpIdx, Attribute::ZExt))
84     Flags.setZExt();
85   if (Attrs.hasAttribute(OpIdx, Attribute::SExt))
86     Flags.setSExt();
87   if (Attrs.hasAttribute(OpIdx, Attribute::InReg))
88     Flags.setInReg();
89   if (Attrs.hasAttribute(OpIdx, Attribute::StructRet))
90     Flags.setSRet();
91   if (Attrs.hasAttribute(OpIdx, Attribute::SwiftSelf))
92     Flags.setSwiftSelf();
93   if (Attrs.hasAttribute(OpIdx, Attribute::SwiftError))
94     Flags.setSwiftError();
95   if (Attrs.hasAttribute(OpIdx, Attribute::ByVal))
96     Flags.setByVal();
97   if (Attrs.hasAttribute(OpIdx, Attribute::InAlloca))
98     Flags.setInAlloca();
99 
100   if (Flags.isByVal() || Flags.isInAlloca()) {
101     Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType();
102 
103     auto Ty = Attrs.getAttribute(OpIdx, Attribute::ByVal).getValueAsType();
104     Flags.setByValSize(DL.getTypeAllocSize(Ty ? Ty : ElementTy));
105 
106     // For ByVal, alignment should be passed from FE.  BE will guess if
107     // this info is not there but there are cases it cannot get right.
108     unsigned FrameAlign;
109     if (FuncInfo.getParamAlignment(OpIdx - 2))
110       FrameAlign = FuncInfo.getParamAlignment(OpIdx - 2);
111     else
112       FrameAlign = getTLI()->getByValTypeAlignment(ElementTy, DL);
113     Flags.setByValAlign(Align(FrameAlign));
114   }
115   if (Attrs.hasAttribute(OpIdx, Attribute::Nest))
116     Flags.setNest();
117   Flags.setOrigAlign(Align(DL.getABITypeAlignment(Arg.Ty)));
118 }
119 
120 template void
121 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
122                                     const DataLayout &DL,
123                                     const Function &FuncInfo) const;
124 
125 template void
126 CallLowering::setArgFlags<CallInst>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
127                                     const DataLayout &DL,
128                                     const CallInst &FuncInfo) const;
129 
packRegs(ArrayRef<Register> SrcRegs,Type * PackedTy,MachineIRBuilder & MIRBuilder) const130 Register CallLowering::packRegs(ArrayRef<Register> SrcRegs, Type *PackedTy,
131                                 MachineIRBuilder &MIRBuilder) const {
132   assert(SrcRegs.size() > 1 && "Nothing to pack");
133 
134   const DataLayout &DL = MIRBuilder.getMF().getDataLayout();
135   MachineRegisterInfo *MRI = MIRBuilder.getMRI();
136 
137   LLT PackedLLT = getLLTForType(*PackedTy, DL);
138 
139   SmallVector<LLT, 8> LLTs;
140   SmallVector<uint64_t, 8> Offsets;
141   computeValueLLTs(DL, *PackedTy, LLTs, &Offsets);
142   assert(LLTs.size() == SrcRegs.size() && "Regs / types mismatch");
143 
144   Register Dst = MRI->createGenericVirtualRegister(PackedLLT);
145   MIRBuilder.buildUndef(Dst);
146   for (unsigned i = 0; i < SrcRegs.size(); ++i) {
147     Register NewDst = MRI->createGenericVirtualRegister(PackedLLT);
148     MIRBuilder.buildInsert(NewDst, Dst, SrcRegs[i], Offsets[i]);
149     Dst = NewDst;
150   }
151 
152   return Dst;
153 }
154 
unpackRegs(ArrayRef<Register> DstRegs,Register SrcReg,Type * PackedTy,MachineIRBuilder & MIRBuilder) const155 void CallLowering::unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg,
156                               Type *PackedTy,
157                               MachineIRBuilder &MIRBuilder) const {
158   assert(DstRegs.size() > 1 && "Nothing to unpack");
159 
160   const DataLayout &DL = MIRBuilder.getMF().getDataLayout();
161 
162   SmallVector<LLT, 8> LLTs;
163   SmallVector<uint64_t, 8> Offsets;
164   computeValueLLTs(DL, *PackedTy, LLTs, &Offsets);
165   assert(LLTs.size() == DstRegs.size() && "Regs / types mismatch");
166 
167   for (unsigned i = 0; i < DstRegs.size(); ++i)
168     MIRBuilder.buildExtract(DstRegs[i], SrcReg, Offsets[i]);
169 }
170 
handleAssignments(MachineIRBuilder & MIRBuilder,SmallVectorImpl<ArgInfo> & Args,ValueHandler & Handler) const171 bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder,
172                                      SmallVectorImpl<ArgInfo> &Args,
173                                      ValueHandler &Handler) const {
174   MachineFunction &MF = MIRBuilder.getMF();
175   const Function &F = MF.getFunction();
176   SmallVector<CCValAssign, 16> ArgLocs;
177   CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
178   return handleAssignments(CCInfo, ArgLocs, MIRBuilder, Args, Handler);
179 }
180 
handleAssignments(CCState & CCInfo,SmallVectorImpl<CCValAssign> & ArgLocs,MachineIRBuilder & MIRBuilder,SmallVectorImpl<ArgInfo> & Args,ValueHandler & Handler) const181 bool CallLowering::handleAssignments(CCState &CCInfo,
182                                      SmallVectorImpl<CCValAssign> &ArgLocs,
183                                      MachineIRBuilder &MIRBuilder,
184                                      SmallVectorImpl<ArgInfo> &Args,
185                                      ValueHandler &Handler) const {
186   MachineFunction &MF = MIRBuilder.getMF();
187   const Function &F = MF.getFunction();
188   const DataLayout &DL = F.getParent()->getDataLayout();
189 
190   unsigned NumArgs = Args.size();
191   for (unsigned i = 0; i != NumArgs; ++i) {
192     MVT CurVT = MVT::getVT(Args[i].Ty);
193     if (Handler.assignArg(i, CurVT, CurVT, CCValAssign::Full, Args[i],
194                           Args[i].Flags[0], CCInfo)) {
195       if (!CurVT.isValid())
196         return false;
197       MVT NewVT = TLI->getRegisterTypeForCallingConv(
198           F.getContext(), F.getCallingConv(), EVT(CurVT));
199 
200       // If we need to split the type over multiple regs, check it's a scenario
201       // we currently support.
202       unsigned NumParts = TLI->getNumRegistersForCallingConv(
203           F.getContext(), F.getCallingConv(), CurVT);
204       if (NumParts > 1) {
205         // For now only handle exact splits.
206         if (NewVT.getSizeInBits() * NumParts != CurVT.getSizeInBits())
207           return false;
208       }
209 
210       // For incoming arguments (physregs to vregs), we could have values in
211       // physregs (or memlocs) which we want to extract and copy to vregs.
212       // During this, we might have to deal with the LLT being split across
213       // multiple regs, so we have to record this information for later.
214       //
215       // If we have outgoing args, then we have the opposite case. We have a
216       // vreg with an LLT which we want to assign to a physical location, and
217       // we might have to record that the value has to be split later.
218       if (Handler.isIncomingArgumentHandler()) {
219         if (NumParts == 1) {
220           // Try to use the register type if we couldn't assign the VT.
221           if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i],
222                                 Args[i].Flags[0], CCInfo))
223             return false;
224         } else {
225           // We're handling an incoming arg which is split over multiple regs.
226           // E.g. passing an s128 on AArch64.
227           ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
228           Args[i].OrigRegs.push_back(Args[i].Regs[0]);
229           Args[i].Regs.clear();
230           Args[i].Flags.clear();
231           LLT NewLLT = getLLTForMVT(NewVT);
232           // For each split register, create and assign a vreg that will store
233           // the incoming component of the larger value. These will later be
234           // merged to form the final vreg.
235           for (unsigned Part = 0; Part < NumParts; ++Part) {
236             Register Reg =
237                 MIRBuilder.getMRI()->createGenericVirtualRegister(NewLLT);
238             ISD::ArgFlagsTy Flags = OrigFlags;
239             if (Part == 0) {
240               Flags.setSplit();
241             } else {
242               Flags.setOrigAlign(Align::None());
243               if (Part == NumParts - 1)
244                 Flags.setSplitEnd();
245             }
246             Args[i].Regs.push_back(Reg);
247             Args[i].Flags.push_back(Flags);
248             if (Handler.assignArg(i + Part, NewVT, NewVT, CCValAssign::Full,
249                                   Args[i], Args[i].Flags[Part], CCInfo)) {
250               // Still couldn't assign this smaller part type for some reason.
251               return false;
252             }
253           }
254         }
255       } else {
256         // Handling an outgoing arg that might need to be split.
257         if (NumParts < 2)
258           return false; // Don't know how to deal with this type combination.
259 
260         // This type is passed via multiple registers in the calling convention.
261         // We need to extract the individual parts.
262         Register LargeReg = Args[i].Regs[0];
263         LLT SmallTy = LLT::scalar(NewVT.getSizeInBits());
264         auto Unmerge = MIRBuilder.buildUnmerge(SmallTy, LargeReg);
265         assert(Unmerge->getNumOperands() == NumParts + 1);
266         ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
267         // We're going to replace the regs and flags with the split ones.
268         Args[i].Regs.clear();
269         Args[i].Flags.clear();
270         for (unsigned PartIdx = 0; PartIdx < NumParts; ++PartIdx) {
271           ISD::ArgFlagsTy Flags = OrigFlags;
272           if (PartIdx == 0) {
273             Flags.setSplit();
274           } else {
275             Flags.setOrigAlign(Align::None());
276             if (PartIdx == NumParts - 1)
277               Flags.setSplitEnd();
278           }
279           Args[i].Regs.push_back(Unmerge.getReg(PartIdx));
280           Args[i].Flags.push_back(Flags);
281           if (Handler.assignArg(i + PartIdx, NewVT, NewVT, CCValAssign::Full,
282                                 Args[i], Args[i].Flags[PartIdx], CCInfo))
283             return false;
284         }
285       }
286     }
287   }
288 
289   for (unsigned i = 0, e = Args.size(), j = 0; i != e; ++i, ++j) {
290     assert(j < ArgLocs.size() && "Skipped too many arg locs");
291 
292     CCValAssign &VA = ArgLocs[j];
293     assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
294 
295     if (VA.needsCustom()) {
296       j += Handler.assignCustomValue(Args[i], makeArrayRef(ArgLocs).slice(j));
297       continue;
298     }
299 
300     // FIXME: Pack registers if we have more than one.
301     Register ArgReg = Args[i].Regs[0];
302 
303     MVT OrigVT = MVT::getVT(Args[i].Ty);
304     MVT VAVT = VA.getValVT();
305     if (VA.isRegLoc()) {
306       if (Handler.isIncomingArgumentHandler() && VAVT != OrigVT) {
307         if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) {
308           // Expected to be multiple regs for a single incoming arg.
309           unsigned NumArgRegs = Args[i].Regs.size();
310           if (NumArgRegs < 2)
311             return false;
312 
313           assert((j + (NumArgRegs - 1)) < ArgLocs.size() &&
314                  "Too many regs for number of args");
315           for (unsigned Part = 0; Part < NumArgRegs; ++Part) {
316             // There should be Regs.size() ArgLocs per argument.
317             VA = ArgLocs[j + Part];
318             Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
319           }
320           j += NumArgRegs - 1;
321           // Merge the split registers into the expected larger result vreg
322           // of the original call.
323           MIRBuilder.buildMerge(Args[i].OrigRegs[0], Args[i].Regs);
324           continue;
325         }
326         const LLT VATy(VAVT);
327         Register NewReg =
328             MIRBuilder.getMRI()->createGenericVirtualRegister(VATy);
329         Handler.assignValueToReg(NewReg, VA.getLocReg(), VA);
330         // If it's a vector type, we either need to truncate the elements
331         // or do an unmerge to get the lower block of elements.
332         if (VATy.isVector() &&
333             VATy.getNumElements() > OrigVT.getVectorNumElements()) {
334           const LLT OrigTy(OrigVT);
335           // Just handle the case where the VA type is 2 * original type.
336           if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) {
337             LLVM_DEBUG(dbgs()
338                        << "Incoming promoted vector arg has too many elts");
339             return false;
340           }
341           auto Unmerge = MIRBuilder.buildUnmerge({OrigTy, OrigTy}, {NewReg});
342           MIRBuilder.buildCopy(ArgReg, Unmerge.getReg(0));
343         } else {
344           MIRBuilder.buildTrunc(ArgReg, {NewReg}).getReg(0);
345         }
346       } else if (!Handler.isIncomingArgumentHandler()) {
347         assert((j + (Args[i].Regs.size() - 1)) < ArgLocs.size() &&
348                "Too many regs for number of args");
349         // This is an outgoing argument that might have been split.
350         for (unsigned Part = 0; Part < Args[i].Regs.size(); ++Part) {
351           // There should be Regs.size() ArgLocs per argument.
352           VA = ArgLocs[j + Part];
353           Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
354         }
355         j += Args[i].Regs.size() - 1;
356       } else {
357         Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
358       }
359     } else if (VA.isMemLoc()) {
360       // Don't currently support loading/storing a type that needs to be split
361       // to the stack. Should be easy, just not implemented yet.
362       if (Args[i].Regs.size() > 1) {
363         LLVM_DEBUG(
364             dbgs()
365             << "Load/store a split arg to/from the stack not implemented yet");
366         return false;
367       }
368       MVT VT = MVT::getVT(Args[i].Ty);
369       unsigned Size = VT == MVT::iPTR ? DL.getPointerSize()
370                                       : alignTo(VT.getSizeInBits(), 8) / 8;
371       unsigned Offset = VA.getLocMemOffset();
372       MachinePointerInfo MPO;
373       Register StackAddr = Handler.getStackAddress(Size, Offset, MPO);
374       Handler.assignValueToAddress(ArgReg, StackAddr, Size, MPO, VA);
375     } else {
376       // FIXME: Support byvals and other weirdness
377       return false;
378     }
379   }
380   return true;
381 }
382 
analyzeArgInfo(CCState & CCState,SmallVectorImpl<ArgInfo> & Args,CCAssignFn & AssignFnFixed,CCAssignFn & AssignFnVarArg) const383 bool CallLowering::analyzeArgInfo(CCState &CCState,
384                                   SmallVectorImpl<ArgInfo> &Args,
385                                   CCAssignFn &AssignFnFixed,
386                                   CCAssignFn &AssignFnVarArg) const {
387   for (unsigned i = 0, e = Args.size(); i < e; ++i) {
388     MVT VT = MVT::getVT(Args[i].Ty);
389     CCAssignFn &Fn = Args[i].IsFixed ? AssignFnFixed : AssignFnVarArg;
390     if (Fn(i, VT, VT, CCValAssign::Full, Args[i].Flags[0], CCState)) {
391       // Bail out on anything we can't handle.
392       LLVM_DEBUG(dbgs() << "Cannot analyze " << EVT(VT).getEVTString()
393                         << " (arg number = " << i << "\n");
394       return false;
395     }
396   }
397   return true;
398 }
399 
resultsCompatible(CallLoweringInfo & Info,MachineFunction & MF,SmallVectorImpl<ArgInfo> & InArgs,CCAssignFn & CalleeAssignFnFixed,CCAssignFn & CalleeAssignFnVarArg,CCAssignFn & CallerAssignFnFixed,CCAssignFn & CallerAssignFnVarArg) const400 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
401                                      MachineFunction &MF,
402                                      SmallVectorImpl<ArgInfo> &InArgs,
403                                      CCAssignFn &CalleeAssignFnFixed,
404                                      CCAssignFn &CalleeAssignFnVarArg,
405                                      CCAssignFn &CallerAssignFnFixed,
406                                      CCAssignFn &CallerAssignFnVarArg) const {
407   const Function &F = MF.getFunction();
408   CallingConv::ID CalleeCC = Info.CallConv;
409   CallingConv::ID CallerCC = F.getCallingConv();
410 
411   if (CallerCC == CalleeCC)
412     return true;
413 
414   SmallVector<CCValAssign, 16> ArgLocs1;
415   CCState CCInfo1(CalleeCC, false, MF, ArgLocs1, F.getContext());
416   if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFnFixed,
417                       CalleeAssignFnVarArg))
418     return false;
419 
420   SmallVector<CCValAssign, 16> ArgLocs2;
421   CCState CCInfo2(CallerCC, false, MF, ArgLocs2, F.getContext());
422   if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFnFixed,
423                       CalleeAssignFnVarArg))
424     return false;
425 
426   // We need the argument locations to match up exactly. If there's more in
427   // one than the other, then we are done.
428   if (ArgLocs1.size() != ArgLocs2.size())
429     return false;
430 
431   // Make sure that each location is passed in exactly the same way.
432   for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
433     const CCValAssign &Loc1 = ArgLocs1[i];
434     const CCValAssign &Loc2 = ArgLocs2[i];
435 
436     // We need both of them to be the same. So if one is a register and one
437     // isn't, we're done.
438     if (Loc1.isRegLoc() != Loc2.isRegLoc())
439       return false;
440 
441     if (Loc1.isRegLoc()) {
442       // If they don't have the same register location, we're done.
443       if (Loc1.getLocReg() != Loc2.getLocReg())
444         return false;
445 
446       // They matched, so we can move to the next ArgLoc.
447       continue;
448     }
449 
450     // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
451     if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
452       return false;
453   }
454 
455   return true;
456 }
457 
extendRegister(Register ValReg,CCValAssign & VA)458 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
459                                                     CCValAssign &VA) {
460   LLT LocTy{VA.getLocVT()};
461   if (LocTy.getSizeInBits() == MRI.getType(ValReg).getSizeInBits())
462     return ValReg;
463   switch (VA.getLocInfo()) {
464   default: break;
465   case CCValAssign::Full:
466   case CCValAssign::BCvt:
467     // FIXME: bitconverting between vector types may or may not be a
468     // nop in big-endian situations.
469     return ValReg;
470   case CCValAssign::AExt: {
471     auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
472     return MIB->getOperand(0).getReg();
473   }
474   case CCValAssign::SExt: {
475     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
476     MIRBuilder.buildSExt(NewReg, ValReg);
477     return NewReg;
478   }
479   case CCValAssign::ZExt: {
480     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
481     MIRBuilder.buildZExt(NewReg, ValReg);
482     return NewReg;
483   }
484   }
485   llvm_unreachable("unable to extend register");
486 }
487 
anchor()488 void CallLowering::ValueHandler::anchor() {}
489