1 //===-- FixupStatepointCallerSaved.cpp - Fixup caller saved registers ----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// Statepoint instruction in deopt parameters contains values which are
12 /// meaningful to the runtime and should be able to be read at the moment the
13 /// call returns. So we can say that we need to encode the fact that these
14 /// values are "late read" by runtime. If we could express this notion for
15 /// register allocator it would produce the right form for us.
16 /// The need to fixup (i.e this pass) is specifically handling the fact that
17 /// we cannot describe such a late read for the register allocator.
18 /// Register allocator may put the value on a register clobbered by the call.
19 /// This pass forces the spill of such registers and replaces corresponding
20 /// statepoint operands to added spill slots.
21 ///
22 //===----------------------------------------------------------------------===//
23
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/Statistic.h"
26 #include "llvm/CodeGen/MachineFrameInfo.h"
27 #include "llvm/CodeGen/MachineFunctionPass.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/Passes.h"
30 #include "llvm/CodeGen/StackMaps.h"
31 #include "llvm/CodeGen/TargetFrameLowering.h"
32 #include "llvm/CodeGen/TargetInstrInfo.h"
33 #include "llvm/IR/Statepoint.h"
34 #include "llvm/InitializePasses.h"
35 #include "llvm/Support/Debug.h"
36
37 using namespace llvm;
38
39 #define DEBUG_TYPE "fixup-statepoint-caller-saved"
40 STATISTIC(NumSpilledRegisters, "Number of spilled register");
41 STATISTIC(NumSpillSlotsAllocated, "Number of spill slots allocated");
42 STATISTIC(NumSpillSlotsExtended, "Number of spill slots extended");
43
44 static cl::opt<bool> FixupSCSExtendSlotSize(
45 "fixup-scs-extend-slot-size", cl::Hidden, cl::init(false),
46 cl::desc("Allow spill in spill slot of greater size than register size"),
47 cl::Hidden);
48
49 static cl::opt<bool> PassGCPtrInCSR(
50 "fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false),
51 cl::desc("Allow passing GC Pointer arguments in callee saved registers"));
52
53 static cl::opt<bool> EnableCopyProp(
54 "fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true),
55 cl::desc("Enable simple copy propagation during register reloading"));
56
57 // This is purely debugging option.
58 // It may be handy for investigating statepoint spilling issues.
59 static cl::opt<unsigned> MaxStatepointsWithRegs(
60 "fixup-max-csr-statepoints", cl::Hidden,
61 cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"));
62
63 namespace {
64
65 class FixupStatepointCallerSaved : public MachineFunctionPass {
66 public:
67 static char ID;
68
FixupStatepointCallerSaved()69 FixupStatepointCallerSaved() : MachineFunctionPass(ID) {
70 initializeFixupStatepointCallerSavedPass(*PassRegistry::getPassRegistry());
71 }
72
getAnalysisUsage(AnalysisUsage & AU) const73 void getAnalysisUsage(AnalysisUsage &AU) const override {
74 AU.setPreservesCFG();
75 MachineFunctionPass::getAnalysisUsage(AU);
76 }
77
getPassName() const78 StringRef getPassName() const override {
79 return "Fixup Statepoint Caller Saved";
80 }
81
82 bool runOnMachineFunction(MachineFunction &MF) override;
83 };
84
85 } // End anonymous namespace.
86
87 char FixupStatepointCallerSaved::ID = 0;
88 char &llvm::FixupStatepointCallerSavedID = FixupStatepointCallerSaved::ID;
89
90 INITIALIZE_PASS_BEGIN(FixupStatepointCallerSaved, DEBUG_TYPE,
91 "Fixup Statepoint Caller Saved", false, false)
92 INITIALIZE_PASS_END(FixupStatepointCallerSaved, DEBUG_TYPE,
93 "Fixup Statepoint Caller Saved", false, false)
94
95 // Utility function to get size of the register.
getRegisterSize(const TargetRegisterInfo & TRI,Register Reg)96 static unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg) {
97 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
98 return TRI.getSpillSize(*RC);
99 }
100
101 // Try to eliminate redundant copy to register which we're going to
102 // spill, i.e. try to change:
103 // X = COPY Y
104 // SPILL X
105 // to
106 // SPILL Y
107 // If there are no uses of X between copy and STATEPOINT, that COPY
108 // may be eliminated.
109 // Reg - register we're about to spill
110 // RI - On entry points to statepoint.
111 // On successful copy propagation set to new spill point.
112 // IsKill - set to true if COPY is Kill (there are no uses of Y)
113 // Returns either found source copy register or original one.
performCopyPropagation(Register Reg,MachineBasicBlock::iterator & RI,bool & IsKill,const TargetInstrInfo & TII,const TargetRegisterInfo & TRI)114 static Register performCopyPropagation(Register Reg,
115 MachineBasicBlock::iterator &RI,
116 bool &IsKill, const TargetInstrInfo &TII,
117 const TargetRegisterInfo &TRI) {
118 // First check if statepoint itself uses Reg in non-meta operands.
119 int Idx = RI->findRegisterUseOperandIdx(Reg, false, &TRI);
120 if (Idx >= 0 && (unsigned)Idx < StatepointOpers(&*RI).getNumDeoptArgsIdx()) {
121 IsKill = false;
122 return Reg;
123 }
124
125 if (!EnableCopyProp)
126 return Reg;
127
128 MachineBasicBlock *MBB = RI->getParent();
129 MachineBasicBlock::reverse_iterator E = MBB->rend();
130 MachineInstr *Def = nullptr, *Use = nullptr;
131 for (auto It = ++(RI.getReverse()); It != E; ++It) {
132 if (It->readsRegister(Reg, &TRI) && !Use)
133 Use = &*It;
134 if (It->modifiesRegister(Reg, &TRI)) {
135 Def = &*It;
136 break;
137 }
138 }
139
140 if (!Def)
141 return Reg;
142
143 auto DestSrc = TII.isCopyInstr(*Def);
144 if (!DestSrc || DestSrc->Destination->getReg() != Reg)
145 return Reg;
146
147 Register SrcReg = DestSrc->Source->getReg();
148
149 if (getRegisterSize(TRI, Reg) != getRegisterSize(TRI, SrcReg))
150 return Reg;
151
152 LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation "
153 << printReg(Reg, &TRI) << " -> " << printReg(SrcReg, &TRI)
154 << "\n");
155
156 // Insert spill immediately after Def
157 RI = ++MachineBasicBlock::iterator(Def);
158 IsKill = DestSrc->Source->isKill();
159
160 // There are no uses of original register between COPY and STATEPOINT.
161 // There can't be any after STATEPOINT, so we can eliminate Def.
162 if (!Use) {
163 LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def);
164 Def->eraseFromParent();
165 }
166 return SrcReg;
167 }
168
169 namespace {
170 // Pair {Register, FrameIndex}
171 using RegSlotPair = std::pair<Register, int>;
172
173 // Keeps track of what reloads were inserted in MBB.
174 class RegReloadCache {
175 using ReloadSet = SmallSet<RegSlotPair, 8>;
176 DenseMap<const MachineBasicBlock *, ReloadSet> Reloads;
177
178 public:
179 RegReloadCache() = default;
180
181 // Record reload of Reg from FI in block MBB
recordReload(Register Reg,int FI,const MachineBasicBlock * MBB)182 void recordReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
183 RegSlotPair RSP(Reg, FI);
184 auto Res = Reloads[MBB].insert(RSP);
185 (void)Res;
186 assert(Res.second && "reload already exists");
187 }
188
189 // Does basic block MBB contains reload of Reg from FI?
hasReload(Register Reg,int FI,const MachineBasicBlock * MBB)190 bool hasReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
191 RegSlotPair RSP(Reg, FI);
192 return Reloads.count(MBB) && Reloads[MBB].count(RSP);
193 }
194 };
195
196 // Cache used frame indexes during statepoint re-write to re-use them in
197 // processing next statepoint instruction.
198 // Two strategies. One is to preserve the size of spill slot while another one
199 // extends the size of spill slots to reduce the number of them, causing
200 // the less total frame size. But unspill will have "implicit" any extend.
201 class FrameIndexesCache {
202 private:
203 struct FrameIndexesPerSize {
204 // List of used frame indexes during processing previous statepoints.
205 SmallVector<int, 8> Slots;
206 // Current index of un-used yet frame index.
207 unsigned Index = 0;
208 };
209 MachineFrameInfo &MFI;
210 const TargetRegisterInfo &TRI;
211 // Map size to list of frame indexes of this size. If the mode is
212 // FixupSCSExtendSlotSize then the key 0 is used to keep all frame indexes.
213 // If the size of required spill slot is greater than in a cache then the
214 // size will be increased.
215 DenseMap<unsigned, FrameIndexesPerSize> Cache;
216
217 // Keeps track of slots reserved for the shared landing pad processing.
218 // Initialized from GlobalIndices for the current EHPad.
219 SmallSet<int, 8> ReservedSlots;
220
221 // Landing pad can be destination of several statepoints. Every register
222 // defined by such statepoints must be spilled to the same stack slot.
223 // This map keeps that information.
224 DenseMap<const MachineBasicBlock *, SmallVector<RegSlotPair, 8>>
225 GlobalIndices;
226
getCacheBucket(unsigned Size)227 FrameIndexesPerSize &getCacheBucket(unsigned Size) {
228 // In FixupSCSExtendSlotSize mode the bucket with 0 index is used
229 // for all sizes.
230 return Cache[FixupSCSExtendSlotSize ? 0 : Size];
231 }
232
233 public:
FrameIndexesCache(MachineFrameInfo & MFI,const TargetRegisterInfo & TRI)234 FrameIndexesCache(MachineFrameInfo &MFI, const TargetRegisterInfo &TRI)
235 : MFI(MFI), TRI(TRI) {}
236 // Reset the current state of used frame indexes. After invocation of
237 // this function all frame indexes are available for allocation with
238 // the exception of slots reserved for landing pad processing (if any).
reset(const MachineBasicBlock * EHPad)239 void reset(const MachineBasicBlock *EHPad) {
240 for (auto &It : Cache)
241 It.second.Index = 0;
242
243 ReservedSlots.clear();
244 if (EHPad && GlobalIndices.count(EHPad))
245 for (auto &RSP : GlobalIndices[EHPad])
246 ReservedSlots.insert(RSP.second);
247 }
248
249 // Get frame index to spill the register.
getFrameIndex(Register Reg,MachineBasicBlock * EHPad)250 int getFrameIndex(Register Reg, MachineBasicBlock *EHPad) {
251 // Check if slot for Reg is already reserved at EHPad.
252 auto It = GlobalIndices.find(EHPad);
253 if (It != GlobalIndices.end()) {
254 auto &Vec = It->second;
255 auto Idx = llvm::find_if(
256 Vec, [Reg](RegSlotPair &RSP) { return Reg == RSP.first; });
257 if (Idx != Vec.end()) {
258 int FI = Idx->second;
259 LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register "
260 << printReg(Reg, &TRI) << " at "
261 << printMBBReference(*EHPad) << "\n");
262 assert(ReservedSlots.count(FI) && "using unreserved slot");
263 return FI;
264 }
265 }
266
267 unsigned Size = getRegisterSize(TRI, Reg);
268 FrameIndexesPerSize &Line = getCacheBucket(Size);
269 while (Line.Index < Line.Slots.size()) {
270 int FI = Line.Slots[Line.Index++];
271 if (ReservedSlots.count(FI))
272 continue;
273 // If all sizes are kept together we probably need to extend the
274 // spill slot size.
275 if (MFI.getObjectSize(FI) < Size) {
276 MFI.setObjectSize(FI, Size);
277 MFI.setObjectAlignment(FI, Align(Size));
278 NumSpillSlotsExtended++;
279 }
280 return FI;
281 }
282 int FI = MFI.CreateSpillStackObject(Size, Align(Size));
283 NumSpillSlotsAllocated++;
284 Line.Slots.push_back(FI);
285 ++Line.Index;
286
287 // Remember assignment {Reg, FI} for EHPad
288 if (EHPad) {
289 GlobalIndices[EHPad].push_back(std::make_pair(Reg, FI));
290 LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling reg "
291 << printReg(Reg, &TRI) << " at landing pad "
292 << printMBBReference(*EHPad) << "\n");
293 }
294
295 return FI;
296 }
297
298 // Sort all registers to spill in descendent order. In the
299 // FixupSCSExtendSlotSize mode it will minimize the total frame size.
300 // In non FixupSCSExtendSlotSize mode we can skip this step.
sortRegisters(SmallVectorImpl<Register> & Regs)301 void sortRegisters(SmallVectorImpl<Register> &Regs) {
302 if (!FixupSCSExtendSlotSize)
303 return;
304 llvm::sort(Regs, [&](Register &A, Register &B) {
305 return getRegisterSize(TRI, A) > getRegisterSize(TRI, B);
306 });
307 }
308 };
309
310 // Describes the state of the current processing statepoint instruction.
311 class StatepointState {
312 private:
313 // statepoint instruction.
314 MachineInstr &MI;
315 MachineFunction &MF;
316 // If non-null then statepoint is invoke, and this points to the landing pad.
317 MachineBasicBlock *EHPad;
318 const TargetRegisterInfo &TRI;
319 const TargetInstrInfo &TII;
320 MachineFrameInfo &MFI;
321 // Mask with callee saved registers.
322 const uint32_t *Mask;
323 // Cache of frame indexes used on previous instruction processing.
324 FrameIndexesCache &CacheFI;
325 bool AllowGCPtrInCSR;
326 // Operands with physical registers requiring spilling.
327 SmallVector<unsigned, 8> OpsToSpill;
328 // Set of register to spill.
329 SmallVector<Register, 8> RegsToSpill;
330 // Set of registers to reload after statepoint.
331 SmallVector<Register, 8> RegsToReload;
332 // Map Register to Frame Slot index.
333 DenseMap<Register, int> RegToSlotIdx;
334
335 public:
StatepointState(MachineInstr & MI,const uint32_t * Mask,FrameIndexesCache & CacheFI,bool AllowGCPtrInCSR)336 StatepointState(MachineInstr &MI, const uint32_t *Mask,
337 FrameIndexesCache &CacheFI, bool AllowGCPtrInCSR)
338 : MI(MI), MF(*MI.getMF()), TRI(*MF.getSubtarget().getRegisterInfo()),
339 TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),
340 Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {
341
342 // Find statepoint's landing pad, if any.
343 EHPad = nullptr;
344 MachineBasicBlock *MBB = MI.getParent();
345 // Invoke statepoint must be last one in block.
346 bool Last = std::none_of(++MI.getIterator(), MBB->end().getInstrIterator(),
347 [](MachineInstr &I) {
348 return I.getOpcode() == TargetOpcode::STATEPOINT;
349 });
350
351 if (!Last)
352 return;
353
354 auto IsEHPad = [](MachineBasicBlock *B) { return B->isEHPad(); };
355
356 assert(llvm::count_if(MBB->successors(), IsEHPad) < 2 && "multiple EHPads");
357
358 auto It = llvm::find_if(MBB->successors(), IsEHPad);
359 if (It != MBB->succ_end())
360 EHPad = *It;
361 }
362
getEHPad() const363 MachineBasicBlock *getEHPad() const { return EHPad; }
364
365 // Return true if register is callee saved.
isCalleeSaved(Register Reg)366 bool isCalleeSaved(Register Reg) { return (Mask[Reg / 32] >> Reg % 32) & 1; }
367
368 // Iterates over statepoint meta args to find caller saver registers.
369 // Also cache the size of found registers.
370 // Returns true if caller save registers found.
findRegistersToSpill()371 bool findRegistersToSpill() {
372 SmallSet<Register, 8> GCRegs;
373 // All GC pointer operands assigned to registers produce new value.
374 // Since they're tied to their defs, it is enough to collect def registers.
375 for (const auto &Def : MI.defs())
376 GCRegs.insert(Def.getReg());
377
378 SmallSet<Register, 8> VisitedRegs;
379 for (unsigned Idx = StatepointOpers(&MI).getVarIdx(),
380 EndIdx = MI.getNumOperands();
381 Idx < EndIdx; ++Idx) {
382 MachineOperand &MO = MI.getOperand(Idx);
383 // Leave `undef` operands as is, StackMaps will rewrite them
384 // into a constant.
385 if (!MO.isReg() || MO.isImplicit() || MO.isUndef())
386 continue;
387 Register Reg = MO.getReg();
388 assert(Reg.isPhysical() && "Only physical regs are expected");
389
390 if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !is_contained(GCRegs, Reg)))
391 continue;
392
393 LLVM_DEBUG(dbgs() << "Will spill " << printReg(Reg, &TRI) << " at index "
394 << Idx << "\n");
395
396 if (VisitedRegs.insert(Reg).second)
397 RegsToSpill.push_back(Reg);
398 OpsToSpill.push_back(Idx);
399 }
400 CacheFI.sortRegisters(RegsToSpill);
401 return !RegsToSpill.empty();
402 }
403
404 // Spill all caller saved registers right before statepoint instruction.
405 // Remember frame index where register is spilled.
spillRegisters()406 void spillRegisters() {
407 for (Register Reg : RegsToSpill) {
408 int FI = CacheFI.getFrameIndex(Reg, EHPad);
409 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
410
411 NumSpilledRegisters++;
412 RegToSlotIdx[Reg] = FI;
413
414 LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, &TRI) << " to FI " << FI
415 << "\n");
416
417 // Perform trivial copy propagation
418 bool IsKill = true;
419 MachineBasicBlock::iterator InsertBefore(MI);
420 Reg = performCopyPropagation(Reg, InsertBefore, IsKill, TII, TRI);
421
422 LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore);
423 TII.storeRegToStackSlot(*MI.getParent(), InsertBefore, Reg, IsKill, FI,
424 RC, &TRI);
425 }
426 }
427
insertReloadBefore(unsigned Reg,MachineBasicBlock::iterator It,MachineBasicBlock * MBB)428 void insertReloadBefore(unsigned Reg, MachineBasicBlock::iterator It,
429 MachineBasicBlock *MBB) {
430 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
431 int FI = RegToSlotIdx[Reg];
432 if (It != MBB->end()) {
433 TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI);
434 return;
435 }
436
437 // To insert reload at the end of MBB, insert it before last instruction
438 // and then swap them.
439 assert(!MBB->empty() && "Empty block");
440 --It;
441 TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI);
442 MachineInstr *Reload = It->getPrevNode();
443 int Dummy = 0;
444 (void)Dummy;
445 assert(TII.isLoadFromStackSlot(*Reload, Dummy) == Reg);
446 assert(Dummy == FI);
447 MBB->remove(Reload);
448 MBB->insertAfter(It, Reload);
449 }
450
451 // Insert reloads of (relocated) registers spilled in statepoint.
insertReloads(MachineInstr * NewStatepoint,RegReloadCache & RC)452 void insertReloads(MachineInstr *NewStatepoint, RegReloadCache &RC) {
453 MachineBasicBlock *MBB = NewStatepoint->getParent();
454 auto InsertPoint = std::next(NewStatepoint->getIterator());
455
456 for (auto Reg : RegsToReload) {
457 insertReloadBefore(Reg, InsertPoint, MBB);
458 LLVM_DEBUG(dbgs() << "Reloading " << printReg(Reg, &TRI) << " from FI "
459 << RegToSlotIdx[Reg] << " after statepoint\n");
460
461 if (EHPad && !RC.hasReload(Reg, RegToSlotIdx[Reg], EHPad)) {
462 RC.recordReload(Reg, RegToSlotIdx[Reg], EHPad);
463 auto EHPadInsertPoint = EHPad->SkipPHIsLabelsAndDebug(EHPad->begin());
464 insertReloadBefore(Reg, EHPadInsertPoint, EHPad);
465 LLVM_DEBUG(dbgs() << "...also reload at EHPad "
466 << printMBBReference(*EHPad) << "\n");
467 }
468 }
469 }
470
471 // Re-write statepoint machine instruction to replace caller saved operands
472 // with indirect memory location (frame index).
rewriteStatepoint()473 MachineInstr *rewriteStatepoint() {
474 MachineInstr *NewMI =
475 MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true);
476 MachineInstrBuilder MIB(MF, NewMI);
477
478 unsigned NumOps = MI.getNumOperands();
479
480 // New indices for the remaining defs.
481 SmallVector<unsigned, 8> NewIndices;
482 unsigned NumDefs = MI.getNumDefs();
483 for (unsigned I = 0; I < NumDefs; ++I) {
484 MachineOperand &DefMO = MI.getOperand(I);
485 assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand");
486 Register Reg = DefMO.getReg();
487 if (!AllowGCPtrInCSR) {
488 assert(is_contained(RegsToSpill, Reg));
489 RegsToReload.push_back(Reg);
490 } else {
491 if (isCalleeSaved(Reg)) {
492 NewIndices.push_back(NewMI->getNumOperands());
493 MIB.addReg(Reg, RegState::Define);
494 } else {
495 NewIndices.push_back(NumOps);
496 RegsToReload.push_back(Reg);
497 }
498 }
499 }
500
501 // Add End marker.
502 OpsToSpill.push_back(MI.getNumOperands());
503 unsigned CurOpIdx = 0;
504
505 for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) {
506 MachineOperand &MO = MI.getOperand(I);
507 if (I == OpsToSpill[CurOpIdx]) {
508 int FI = RegToSlotIdx[MO.getReg()];
509 MIB.addImm(StackMaps::IndirectMemRefOp);
510 MIB.addImm(getRegisterSize(TRI, MO.getReg()));
511 assert(MO.isReg() && "Should be register");
512 assert(MO.getReg().isPhysical() && "Should be physical register");
513 MIB.addFrameIndex(FI);
514 MIB.addImm(0);
515 ++CurOpIdx;
516 } else {
517 MIB.add(MO);
518 unsigned OldDef;
519 if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(I, &OldDef)) {
520 assert(OldDef < NumDefs);
521 assert(NewIndices[OldDef] < NumOps);
522 MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1);
523 }
524 }
525 }
526 assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed");
527 // Add mem operands.
528 NewMI->setMemRefs(MF, MI.memoperands());
529 for (auto It : RegToSlotIdx) {
530 Register R = It.first;
531 int FrameIndex = It.second;
532 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
533 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;
534 if (is_contained(RegsToReload, R))
535 Flags |= MachineMemOperand::MOStore;
536 auto *MMO =
537 MF.getMachineMemOperand(PtrInfo, Flags, getRegisterSize(TRI, R),
538 MFI.getObjectAlign(FrameIndex));
539 NewMI->addMemOperand(MF, MMO);
540 }
541
542 // Insert new statepoint and erase old one.
543 MI.getParent()->insert(MI, NewMI);
544
545 LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n");
546 MI.eraseFromParent();
547 return NewMI;
548 }
549 };
550
551 class StatepointProcessor {
552 private:
553 MachineFunction &MF;
554 const TargetRegisterInfo &TRI;
555 FrameIndexesCache CacheFI;
556 RegReloadCache ReloadCache;
557
558 public:
StatepointProcessor(MachineFunction & MF)559 StatepointProcessor(MachineFunction &MF)
560 : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()),
561 CacheFI(MF.getFrameInfo(), TRI) {}
562
process(MachineInstr & MI,bool AllowGCPtrInCSR)563 bool process(MachineInstr &MI, bool AllowGCPtrInCSR) {
564 StatepointOpers SO(&MI);
565 uint64_t Flags = SO.getFlags();
566 // Do nothing for LiveIn, it supports all registers.
567 if (Flags & (uint64_t)StatepointFlags::DeoptLiveIn)
568 return false;
569 LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " "
570 << MI.getParent()->getName() << " : process statepoint "
571 << MI);
572 CallingConv::ID CC = SO.getCallingConv();
573 const uint32_t *Mask = TRI.getCallPreservedMask(MF, CC);
574 StatepointState SS(MI, Mask, CacheFI, AllowGCPtrInCSR);
575 CacheFI.reset(SS.getEHPad());
576
577 if (!SS.findRegistersToSpill())
578 return false;
579
580 SS.spillRegisters();
581 auto *NewStatepoint = SS.rewriteStatepoint();
582 SS.insertReloads(NewStatepoint, ReloadCache);
583 return true;
584 }
585 };
586 } // namespace
587
runOnMachineFunction(MachineFunction & MF)588 bool FixupStatepointCallerSaved::runOnMachineFunction(MachineFunction &MF) {
589 if (skipFunction(MF.getFunction()))
590 return false;
591
592 const Function &F = MF.getFunction();
593 if (!F.hasGC())
594 return false;
595
596 SmallVector<MachineInstr *, 16> Statepoints;
597 for (MachineBasicBlock &BB : MF)
598 for (MachineInstr &I : BB)
599 if (I.getOpcode() == TargetOpcode::STATEPOINT)
600 Statepoints.push_back(&I);
601
602 if (Statepoints.empty())
603 return false;
604
605 bool Changed = false;
606 StatepointProcessor SPP(MF);
607 unsigned NumStatepoints = 0;
608 bool AllowGCPtrInCSR = PassGCPtrInCSR;
609 for (MachineInstr *I : Statepoints) {
610 ++NumStatepoints;
611 if (MaxStatepointsWithRegs.getNumOccurrences() &&
612 NumStatepoints >= MaxStatepointsWithRegs)
613 AllowGCPtrInCSR = false;
614 Changed |= SPP.process(*I, AllowGCPtrInCSR);
615 }
616 return Changed;
617 }
618