1 //=== ARMCallingConv.cpp - ARM Custom CC Routines ---------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the custom routines for the ARM Calling Convention that
10 // aren't done by tablegen, and includes the table generated implementations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "ARM.h"
15 #include "ARMCallingConv.h"
16 #include "ARMSubtarget.h"
17 #include "ARMRegisterInfo.h"
18 using namespace llvm;
19 
20 // APCS f64 is in register pairs, possibly split to stack
21 static bool f64AssignAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
22                           CCValAssign::LocInfo LocInfo,
23                           CCState &State, bool CanFail) {
24   static const MCPhysReg RegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
25 
26   // Try to get the first register.
27   if (unsigned Reg = State.AllocateReg(RegList))
28     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
29   else {
30     // For the 2nd half of a v2f64, do not fail.
31     if (CanFail)
32       return false;
33 
34     // Put the whole thing on the stack.
35     State.addLoc(CCValAssign::getCustomMem(
36         ValNo, ValVT, State.AllocateStack(8, Align(4)), LocVT, LocInfo));
37     return true;
38   }
39 
40   // Try to get the second register.
41   if (unsigned Reg = State.AllocateReg(RegList))
42     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
43   else
44     State.addLoc(CCValAssign::getCustomMem(
45         ValNo, ValVT, State.AllocateStack(4, Align(4)), LocVT, LocInfo));
46   return true;
47 }
48 
49 static bool CC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
50                                    CCValAssign::LocInfo LocInfo,
51                                    ISD::ArgFlagsTy ArgFlags,
52                                    CCState &State) {
53   if (!f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
54     return false;
55   if (LocVT == MVT::v2f64 &&
56       !f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
57     return false;
58   return true;  // we handled it
59 }
60 
61 // AAPCS f64 is in aligned register pairs
62 static bool f64AssignAAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
63                            CCValAssign::LocInfo LocInfo,
64                            CCState &State, bool CanFail) {
65   static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
66   static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
67   static const MCPhysReg ShadowRegList[] = { ARM::R0, ARM::R1 };
68   static const MCPhysReg GPRArgRegs[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
69 
70   unsigned Reg = State.AllocateReg(HiRegList, ShadowRegList);
71   if (Reg == 0) {
72 
73     // If we had R3 unallocated only, now we still must to waste it.
74     Reg = State.AllocateReg(GPRArgRegs);
75     assert((!Reg || Reg == ARM::R3) && "Wrong GPRs usage for f64");
76 
77     // For the 2nd half of a v2f64, do not just fail.
78     if (CanFail)
79       return false;
80 
81     // Put the whole thing on the stack.
82     State.addLoc(CCValAssign::getCustomMem(
83         ValNo, ValVT, State.AllocateStack(8, Align(8)), LocVT, LocInfo));
84     return true;
85   }
86 
87   unsigned i;
88   for (i = 0; i < 2; ++i)
89     if (HiRegList[i] == Reg)
90       break;
91 
92   unsigned T = State.AllocateReg(LoRegList[i]);
93   (void)T;
94   assert(T == LoRegList[i] && "Could not allocate register");
95 
96   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
97   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
98                                          LocVT, LocInfo));
99   return true;
100 }
101 
102 static bool CC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
103                                     CCValAssign::LocInfo LocInfo,
104                                     ISD::ArgFlagsTy ArgFlags,
105                                     CCState &State) {
106   if (!f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
107     return false;
108   if (LocVT == MVT::v2f64 &&
109       !f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
110     return false;
111   return true;  // we handled it
112 }
113 
114 static bool f64RetAssign(unsigned ValNo, MVT ValVT, MVT LocVT,
115                          CCValAssign::LocInfo LocInfo, CCState &State) {
116   static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
117   static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
118 
119   unsigned Reg = State.AllocateReg(HiRegList, LoRegList);
120   if (Reg == 0)
121     return false; // we didn't handle it
122 
123   unsigned i;
124   for (i = 0; i < 2; ++i)
125     if (HiRegList[i] == Reg)
126       break;
127 
128   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
129   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
130                                          LocVT, LocInfo));
131   return true;
132 }
133 
134 static bool RetCC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
135                                       CCValAssign::LocInfo LocInfo,
136                                       ISD::ArgFlagsTy ArgFlags,
137                                       CCState &State) {
138   if (!f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
139     return false;
140   if (LocVT == MVT::v2f64 && !f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
141     return false;
142   return true;  // we handled it
143 }
144 
145 static bool RetCC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
146                                        CCValAssign::LocInfo LocInfo,
147                                        ISD::ArgFlagsTy ArgFlags,
148                                        CCState &State) {
149   return RetCC_ARM_APCS_Custom_f64(ValNo, ValVT, LocVT, LocInfo, ArgFlags,
150                                    State);
151 }
152 
153 static const MCPhysReg RRegList[] = { ARM::R0,  ARM::R1,  ARM::R2,  ARM::R3 };
154 
155 static const MCPhysReg SRegList[] = { ARM::S0,  ARM::S1,  ARM::S2,  ARM::S3,
156                                       ARM::S4,  ARM::S5,  ARM::S6,  ARM::S7,
157                                       ARM::S8,  ARM::S9,  ARM::S10, ARM::S11,
158                                       ARM::S12, ARM::S13, ARM::S14,  ARM::S15 };
159 static const MCPhysReg DRegList[] = { ARM::D0, ARM::D1, ARM::D2, ARM::D3,
160                                       ARM::D4, ARM::D5, ARM::D6, ARM::D7 };
161 static const MCPhysReg QRegList[] = { ARM::Q0, ARM::Q1, ARM::Q2, ARM::Q3 };
162 
163 
164 // Allocate part of an AAPCS HFA or HVA. We assume that each member of the HA
165 // has InConsecutiveRegs set, and that the last member also has
166 // InConsecutiveRegsLast set. We must process all members of the HA before
167 // we can allocate it, as we need to know the total number of registers that
168 // will be needed in order to (attempt to) allocate a contiguous block.
169 static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned ValNo, MVT ValVT,
170                                           MVT LocVT,
171                                           CCValAssign::LocInfo LocInfo,
172                                           ISD::ArgFlagsTy ArgFlags,
173                                           CCState &State) {
174   SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
175 
176   // AAPCS HFAs must have 1-4 elements, all of the same type
177   if (PendingMembers.size() > 0)
178     assert(PendingMembers[0].getLocVT() == LocVT);
179 
180   // Add the argument to the list to be allocated once we know the size of the
181   // aggregate. Store the type's required alignment as extra info for later: in
182   // the [N x i64] case all trace has been removed by the time we actually get
183   // to do allocation.
184   PendingMembers.push_back(CCValAssign::getPending(
185       ValNo, ValVT, LocVT, LocInfo, ArgFlags.getNonZeroOrigAlign().value()));
186 
187   if (!ArgFlags.isInConsecutiveRegsLast())
188     return true;
189 
190   // Try to allocate a contiguous block of registers, each of the correct
191   // size to hold one member.
192   auto &DL = State.getMachineFunction().getDataLayout();
193   const Align StackAlign = DL.getStackAlignment();
194   const Align FirstMemberAlign(PendingMembers[0].getExtraInfo());
195   Align Alignment = std::min(FirstMemberAlign, StackAlign);
196 
197   ArrayRef<MCPhysReg> RegList;
198   switch (LocVT.SimpleTy) {
199   case MVT::i32: {
200     RegList = RRegList;
201     unsigned RegIdx = State.getFirstUnallocated(RegList);
202 
203     // First consume all registers that would give an unaligned object. Whether
204     // we go on stack or in regs, no-one will be using them in future.
205     unsigned RegAlign = alignTo(Alignment.value(), 4) / 4;
206     while (RegIdx % RegAlign != 0 && RegIdx < RegList.size())
207       State.AllocateReg(RegList[RegIdx++]);
208 
209     break;
210   }
211   case MVT::f16:
212   case MVT::bf16:
213   case MVT::f32:
214     RegList = SRegList;
215     break;
216   case MVT::v4f16:
217   case MVT::v4bf16:
218   case MVT::f64:
219     RegList = DRegList;
220     break;
221   case MVT::v8f16:
222   case MVT::v8bf16:
223   case MVT::v2f64:
224     RegList = QRegList;
225     break;
226   default:
227     llvm_unreachable("Unexpected member type for block aggregate");
228     break;
229   }
230 
231   unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size());
232   if (RegResult) {
233     for (SmallVectorImpl<CCValAssign>::iterator It = PendingMembers.begin();
234          It != PendingMembers.end(); ++It) {
235       It->convertToReg(RegResult);
236       State.addLoc(*It);
237       ++RegResult;
238     }
239     PendingMembers.clear();
240     return true;
241   }
242 
243   // Register allocation failed, we'll be needing the stack
244   unsigned Size = LocVT.getSizeInBits() / 8;
245   if (LocVT == MVT::i32 && State.getNextStackOffset() == 0) {
246     // If nothing else has used the stack until this point, a non-HFA aggregate
247     // can be split between regs and stack.
248     unsigned RegIdx = State.getFirstUnallocated(RegList);
249     for (auto &It : PendingMembers) {
250       if (RegIdx >= RegList.size())
251         It.convertToMem(State.AllocateStack(Size, Align(Size)));
252       else
253         It.convertToReg(State.AllocateReg(RegList[RegIdx++]));
254 
255       State.addLoc(It);
256     }
257     PendingMembers.clear();
258     return true;
259   }
260 
261   if (LocVT != MVT::i32)
262     RegList = SRegList;
263 
264   // Mark all regs as unavailable (AAPCS rule C.2.vfp for VFP, C.6 for core)
265   for (auto Reg : RegList)
266     State.AllocateReg(Reg);
267 
268   // Clamp the alignment between 4 and 8.
269   if (State.getMachineFunction().getSubtarget<ARMSubtarget>().isTargetAEABI())
270     Alignment = ArgFlags.getNonZeroMemAlign() <= 4 ? Align(4) : Align(8);
271 
272   // After the first item has been allocated, the rest are packed as tightly as
273   // possible. (E.g. an incoming i64 would have starting Align of 8, but we'll
274   // be allocating a bunch of i32 slots).
275   for (auto &It : PendingMembers) {
276     It.convertToMem(State.AllocateStack(Size, Alignment));
277     State.addLoc(It);
278     Alignment = Align(1);
279   }
280 
281   // All pending members have now been allocated
282   PendingMembers.clear();
283 
284   // This will be allocated by the last member of the aggregate
285   return true;
286 }
287 
288 static bool CustomAssignInRegList(unsigned ValNo, MVT ValVT, MVT LocVT,
289                                   CCValAssign::LocInfo LocInfo, CCState &State,
290                                   ArrayRef<MCPhysReg> RegList) {
291   unsigned Reg = State.AllocateReg(RegList);
292   if (Reg) {
293     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
294     return true;
295   }
296   return false;
297 }
298 
299 static bool CC_ARM_AAPCS_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,
300                                     CCValAssign::LocInfo LocInfo,
301                                     ISD::ArgFlagsTy ArgFlags, CCState &State) {
302   // f16 arguments are extended to i32 and assigned to a register in [r0, r3]
303   return CustomAssignInRegList(ValNo, ValVT, MVT::i32, LocInfo, State,
304                                RRegList);
305 }
306 
307 static bool CC_ARM_AAPCS_VFP_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,
308                                         CCValAssign::LocInfo LocInfo,
309                                         ISD::ArgFlagsTy ArgFlags,
310                                         CCState &State) {
311   // f16 arguments are extended to f32 and assigned to a register in [s0, s15]
312   return CustomAssignInRegList(ValNo, ValVT, MVT::f32, LocInfo, State,
313                                SRegList);
314 }
315 
316 // Include the table generated calling convention implementations.
317 #include "ARMGenCallingConv.inc"
318