1 //===- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function ---===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass is responsible for finalizing the functions frame layout, saving 10 // callee saved registers, and for emitting prolog & epilog code for the 11 // function. 12 // 13 // This pass must be run after register allocation. After this pass is 14 // executed, it is illegal to construct MO_FrameIndex operands. 15 // 16 //===----------------------------------------------------------------------===// 17 18 #include "llvm/ADT/ArrayRef.h" 19 #include "llvm/ADT/BitVector.h" 20 #include "llvm/ADT/DepthFirstIterator.h" 21 #include "llvm/ADT/STLExtras.h" 22 #include "llvm/ADT/SetVector.h" 23 #include "llvm/ADT/SmallPtrSet.h" 24 #include "llvm/ADT/SmallSet.h" 25 #include "llvm/ADT/SmallVector.h" 26 #include "llvm/ADT/Statistic.h" 27 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 28 #include "llvm/CodeGen/MachineBasicBlock.h" 29 #include "llvm/CodeGen/MachineDominators.h" 30 #include "llvm/CodeGen/MachineFrameInfo.h" 31 #include "llvm/CodeGen/MachineFunction.h" 32 #include "llvm/CodeGen/MachineFunctionPass.h" 33 #include "llvm/CodeGen/MachineInstr.h" 34 #include "llvm/CodeGen/MachineInstrBuilder.h" 35 #include "llvm/CodeGen/MachineLoopInfo.h" 36 #include "llvm/CodeGen/MachineModuleInfo.h" 37 #include "llvm/CodeGen/MachineOperand.h" 38 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" 39 #include "llvm/CodeGen/MachineRegisterInfo.h" 40 #include "llvm/CodeGen/RegisterScavenging.h" 41 #include "llvm/CodeGen/TargetFrameLowering.h" 42 #include "llvm/CodeGen/TargetInstrInfo.h" 43 #include "llvm/CodeGen/TargetOpcodes.h" 44 #include "llvm/CodeGen/TargetRegisterInfo.h" 45 #include "llvm/CodeGen/TargetSubtargetInfo.h" 46 #include "llvm/CodeGen/WinEHFuncInfo.h" 47 #include "llvm/IR/Attributes.h" 48 #include "llvm/IR/CallingConv.h" 49 #include "llvm/IR/DebugInfoMetadata.h" 50 #include "llvm/IR/DiagnosticInfo.h" 51 #include "llvm/IR/Function.h" 52 #include "llvm/IR/InlineAsm.h" 53 #include "llvm/IR/LLVMContext.h" 54 #include "llvm/InitializePasses.h" 55 #include "llvm/MC/MCRegisterInfo.h" 56 #include "llvm/Pass.h" 57 #include "llvm/Support/CodeGen.h" 58 #include "llvm/Support/Debug.h" 59 #include "llvm/Support/ErrorHandling.h" 60 #include "llvm/Support/FormatVariadic.h" 61 #include "llvm/Support/raw_ostream.h" 62 #include "llvm/Target/TargetMachine.h" 63 #include "llvm/Target/TargetOptions.h" 64 #include <algorithm> 65 #include <cassert> 66 #include <cstdint> 67 #include <functional> 68 #include <limits> 69 #include <utility> 70 #include <vector> 71 72 using namespace llvm; 73 74 #define DEBUG_TYPE "prologepilog" 75 76 using MBBVector = SmallVector<MachineBasicBlock *, 4>; 77 78 STATISTIC(NumLeafFuncWithSpills, "Number of leaf functions with CSRs"); 79 STATISTIC(NumFuncSeen, "Number of functions seen in PEI"); 80 81 82 namespace { 83 84 class PEI : public MachineFunctionPass { 85 public: 86 static char ID; 87 88 PEI() : MachineFunctionPass(ID) { 89 initializePEIPass(*PassRegistry::getPassRegistry()); 90 } 91 92 void getAnalysisUsage(AnalysisUsage &AU) const override; 93 94 /// runOnMachineFunction - Insert prolog/epilog code and replace abstract 95 /// frame indexes with appropriate references. 96 bool runOnMachineFunction(MachineFunction &MF) override; 97 98 private: 99 RegScavenger *RS; 100 101 // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved 102 // stack frame indexes. 103 unsigned MinCSFrameIndex = std::numeric_limits<unsigned>::max(); 104 unsigned MaxCSFrameIndex = 0; 105 106 // Save and Restore blocks of the current function. Typically there is a 107 // single save block, unless Windows EH funclets are involved. 108 MBBVector SaveBlocks; 109 MBBVector RestoreBlocks; 110 111 // Flag to control whether to use the register scavenger to resolve 112 // frame index materialization registers. Set according to 113 // TRI->requiresFrameIndexScavenging() for the current function. 114 bool FrameIndexVirtualScavenging; 115 116 // Flag to control whether the scavenger should be passed even though 117 // FrameIndexVirtualScavenging is used. 118 bool FrameIndexEliminationScavenging; 119 120 // Emit remarks. 121 MachineOptimizationRemarkEmitter *ORE = nullptr; 122 123 void calculateCallFrameInfo(MachineFunction &MF); 124 void calculateSaveRestoreBlocks(MachineFunction &MF); 125 void spillCalleeSavedRegs(MachineFunction &MF); 126 127 void calculateFrameObjectOffsets(MachineFunction &MF); 128 void replaceFrameIndices(MachineFunction &MF); 129 void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF, 130 int &SPAdj); 131 // Frame indices in debug values are encoded in a target independent 132 // way with simply the frame index and offset rather than any 133 // target-specific addressing mode. 134 bool replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI, 135 unsigned OpIdx, int SPAdj = 0); 136 // Does same as replaceFrameIndices but using the backward MIR walk and 137 // backward register scavenger walk. Does not yet support call sequence 138 // processing. 139 void replaceFrameIndicesBackward(MachineBasicBlock *BB, MachineFunction &MF, 140 int &SPAdj); 141 142 void insertPrologEpilogCode(MachineFunction &MF); 143 void insertZeroCallUsedRegs(MachineFunction &MF); 144 }; 145 146 } // end anonymous namespace 147 148 char PEI::ID = 0; 149 150 char &llvm::PrologEpilogCodeInserterID = PEI::ID; 151 152 INITIALIZE_PASS_BEGIN(PEI, DEBUG_TYPE, "Prologue/Epilogue Insertion", false, 153 false) 154 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) 155 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) 156 INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass) 157 INITIALIZE_PASS_END(PEI, DEBUG_TYPE, 158 "Prologue/Epilogue Insertion & Frame Finalization", false, 159 false) 160 161 MachineFunctionPass *llvm::createPrologEpilogInserterPass() { 162 return new PEI(); 163 } 164 165 STATISTIC(NumBytesStackSpace, 166 "Number of bytes used for stack in all functions"); 167 168 void PEI::getAnalysisUsage(AnalysisUsage &AU) const { 169 AU.setPreservesCFG(); 170 AU.addPreserved<MachineLoopInfo>(); 171 AU.addPreserved<MachineDominatorTree>(); 172 AU.addRequired<MachineOptimizationRemarkEmitterPass>(); 173 MachineFunctionPass::getAnalysisUsage(AU); 174 } 175 176 /// StackObjSet - A set of stack object indexes 177 using StackObjSet = SmallSetVector<int, 8>; 178 179 using SavedDbgValuesMap = 180 SmallDenseMap<MachineBasicBlock *, SmallVector<MachineInstr *, 4>, 4>; 181 182 /// Stash DBG_VALUEs that describe parameters and which are placed at the start 183 /// of the block. Later on, after the prologue code has been emitted, the 184 /// stashed DBG_VALUEs will be reinserted at the start of the block. 185 static void stashEntryDbgValues(MachineBasicBlock &MBB, 186 SavedDbgValuesMap &EntryDbgValues) { 187 SmallVector<const MachineInstr *, 4> FrameIndexValues; 188 189 for (auto &MI : MBB) { 190 if (!MI.isDebugInstr()) 191 break; 192 if (!MI.isDebugValue() || !MI.getDebugVariable()->isParameter()) 193 continue; 194 if (any_of(MI.debug_operands(), 195 [](const MachineOperand &MO) { return MO.isFI(); })) { 196 // We can only emit valid locations for frame indices after the frame 197 // setup, so do not stash away them. 198 FrameIndexValues.push_back(&MI); 199 continue; 200 } 201 const DILocalVariable *Var = MI.getDebugVariable(); 202 const DIExpression *Expr = MI.getDebugExpression(); 203 auto Overlaps = [Var, Expr](const MachineInstr *DV) { 204 return Var == DV->getDebugVariable() && 205 Expr->fragmentsOverlap(DV->getDebugExpression()); 206 }; 207 // See if the debug value overlaps with any preceding debug value that will 208 // not be stashed. If that is the case, then we can't stash this value, as 209 // we would then reorder the values at reinsertion. 210 if (llvm::none_of(FrameIndexValues, Overlaps)) 211 EntryDbgValues[&MBB].push_back(&MI); 212 } 213 214 // Remove stashed debug values from the block. 215 if (EntryDbgValues.count(&MBB)) 216 for (auto *MI : EntryDbgValues[&MBB]) 217 MI->removeFromParent(); 218 } 219 220 /// runOnMachineFunction - Insert prolog/epilog code and replace abstract 221 /// frame indexes with appropriate references. 222 bool PEI::runOnMachineFunction(MachineFunction &MF) { 223 NumFuncSeen++; 224 const Function &F = MF.getFunction(); 225 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 226 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); 227 228 RS = TRI->requiresRegisterScavenging(MF) ? new RegScavenger() : nullptr; 229 FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(MF); 230 ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE(); 231 232 // Calculate the MaxCallFrameSize and AdjustsStack variables for the 233 // function's frame information. Also eliminates call frame pseudo 234 // instructions. 235 calculateCallFrameInfo(MF); 236 237 // Determine placement of CSR spill/restore code and prolog/epilog code: 238 // place all spills in the entry block, all restores in return blocks. 239 calculateSaveRestoreBlocks(MF); 240 241 // Stash away DBG_VALUEs that should not be moved by insertion of prolog code. 242 SavedDbgValuesMap EntryDbgValues; 243 for (MachineBasicBlock *SaveBlock : SaveBlocks) 244 stashEntryDbgValues(*SaveBlock, EntryDbgValues); 245 246 // Handle CSR spilling and restoring, for targets that need it. 247 if (MF.getTarget().usesPhysRegsForValues()) 248 spillCalleeSavedRegs(MF); 249 250 // Allow the target machine to make final modifications to the function 251 // before the frame layout is finalized. 252 TFI->processFunctionBeforeFrameFinalized(MF, RS); 253 254 // Calculate actual frame offsets for all abstract stack objects... 255 calculateFrameObjectOffsets(MF); 256 257 // Add prolog and epilog code to the function. This function is required 258 // to align the stack frame as necessary for any stack variables or 259 // called functions. Because of this, calculateCalleeSavedRegisters() 260 // must be called before this function in order to set the AdjustsStack 261 // and MaxCallFrameSize variables. 262 if (!F.hasFnAttribute(Attribute::Naked)) 263 insertPrologEpilogCode(MF); 264 265 // Reinsert stashed debug values at the start of the entry blocks. 266 for (auto &I : EntryDbgValues) 267 I.first->insert(I.first->begin(), I.second.begin(), I.second.end()); 268 269 // Allow the target machine to make final modifications to the function 270 // before the frame layout is finalized. 271 TFI->processFunctionBeforeFrameIndicesReplaced(MF, RS); 272 273 // Replace all MO_FrameIndex operands with physical register references 274 // and actual offsets. 275 // 276 replaceFrameIndices(MF); 277 278 // If register scavenging is needed, as we've enabled doing it as a 279 // post-pass, scavenge the virtual registers that frame index elimination 280 // inserted. 281 if (TRI->requiresRegisterScavenging(MF) && FrameIndexVirtualScavenging) 282 scavengeFrameVirtualRegs(MF, *RS); 283 284 // Warn on stack size when we exceeds the given limit. 285 MachineFrameInfo &MFI = MF.getFrameInfo(); 286 uint64_t StackSize = MFI.getStackSize(); 287 288 unsigned Threshold = UINT_MAX; 289 if (MF.getFunction().hasFnAttribute("warn-stack-size")) { 290 bool Failed = MF.getFunction() 291 .getFnAttribute("warn-stack-size") 292 .getValueAsString() 293 .getAsInteger(10, Threshold); 294 // Verifier should have caught this. 295 assert(!Failed && "Invalid warn-stack-size fn attr value"); 296 (void)Failed; 297 } 298 uint64_t UnsafeStackSize = MFI.getUnsafeStackSize(); 299 if (MF.getFunction().hasFnAttribute(Attribute::SafeStack)) 300 StackSize += UnsafeStackSize; 301 302 if (StackSize > Threshold) { 303 DiagnosticInfoStackSize DiagStackSize(F, StackSize, Threshold, DS_Warning); 304 F.getContext().diagnose(DiagStackSize); 305 int64_t SpillSize = 0; 306 for (int Idx = MFI.getObjectIndexBegin(), End = MFI.getObjectIndexEnd(); 307 Idx != End; ++Idx) { 308 if (MFI.isSpillSlotObjectIndex(Idx)) 309 SpillSize += MFI.getObjectSize(Idx); 310 } 311 312 float SpillPct = 313 static_cast<float>(SpillSize) / static_cast<float>(StackSize); 314 float VarPct = 1.0f - SpillPct; 315 int64_t VariableSize = StackSize - SpillSize; 316 dbgs() << formatv("{0}/{1} ({3:P}) spills, {2}/{1} ({4:P}) variables", 317 SpillSize, StackSize, VariableSize, SpillPct, VarPct); 318 if (UnsafeStackSize != 0) { 319 float UnsafePct = 320 static_cast<float>(UnsafeStackSize) / static_cast<float>(StackSize); 321 dbgs() << formatv(", {0}/{2} ({1:P}) unsafe stack", UnsafeStackSize, 322 UnsafePct, StackSize); 323 } 324 dbgs() << "\n"; 325 } 326 327 ORE->emit([&]() { 328 return MachineOptimizationRemarkAnalysis(DEBUG_TYPE, "StackSize", 329 MF.getFunction().getSubprogram(), 330 &MF.front()) 331 << ore::NV("NumStackBytes", StackSize) << " stack bytes in function"; 332 }); 333 334 delete RS; 335 SaveBlocks.clear(); 336 RestoreBlocks.clear(); 337 MFI.setSavePoint(nullptr); 338 MFI.setRestorePoint(nullptr); 339 return true; 340 } 341 342 /// Calculate the MaxCallFrameSize and AdjustsStack 343 /// variables for the function's frame information and eliminate call frame 344 /// pseudo instructions. 345 void PEI::calculateCallFrameInfo(MachineFunction &MF) { 346 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); 347 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); 348 MachineFrameInfo &MFI = MF.getFrameInfo(); 349 350 unsigned MaxCallFrameSize = 0; 351 bool AdjustsStack = MFI.adjustsStack(); 352 353 // Get the function call frame set-up and tear-down instruction opcode 354 unsigned FrameSetupOpcode = TII.getCallFrameSetupOpcode(); 355 unsigned FrameDestroyOpcode = TII.getCallFrameDestroyOpcode(); 356 357 // Early exit for targets which have no call frame setup/destroy pseudo 358 // instructions. 359 if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u) 360 return; 361 362 std::vector<MachineBasicBlock::iterator> FrameSDOps; 363 for (MachineBasicBlock &BB : MF) 364 for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) 365 if (TII.isFrameInstr(*I)) { 366 unsigned Size = TII.getFrameSize(*I); 367 if (Size > MaxCallFrameSize) MaxCallFrameSize = Size; 368 AdjustsStack = true; 369 FrameSDOps.push_back(I); 370 } else if (I->isInlineAsm()) { 371 // Some inline asm's need a stack frame, as indicated by operand 1. 372 unsigned ExtraInfo = I->getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); 373 if (ExtraInfo & InlineAsm::Extra_IsAlignStack) 374 AdjustsStack = true; 375 } 376 377 assert(!MFI.isMaxCallFrameSizeComputed() || 378 (MFI.getMaxCallFrameSize() == MaxCallFrameSize && 379 MFI.adjustsStack() == AdjustsStack)); 380 MFI.setAdjustsStack(AdjustsStack); 381 MFI.setMaxCallFrameSize(MaxCallFrameSize); 382 383 for (MachineBasicBlock::iterator I : FrameSDOps) { 384 // If call frames are not being included as part of the stack frame, and 385 // the target doesn't indicate otherwise, remove the call frame pseudos 386 // here. The sub/add sp instruction pairs are still inserted, but we don't 387 // need to track the SP adjustment for frame index elimination. 388 if (TFI->canSimplifyCallFramePseudos(MF)) 389 TFI->eliminateCallFramePseudoInstr(MF, *I->getParent(), I); 390 } 391 } 392 393 /// Compute the sets of entry and return blocks for saving and restoring 394 /// callee-saved registers, and placing prolog and epilog code. 395 void PEI::calculateSaveRestoreBlocks(MachineFunction &MF) { 396 const MachineFrameInfo &MFI = MF.getFrameInfo(); 397 398 // Even when we do not change any CSR, we still want to insert the 399 // prologue and epilogue of the function. 400 // So set the save points for those. 401 402 // Use the points found by shrink-wrapping, if any. 403 if (MFI.getSavePoint()) { 404 SaveBlocks.push_back(MFI.getSavePoint()); 405 assert(MFI.getRestorePoint() && "Both restore and save must be set"); 406 MachineBasicBlock *RestoreBlock = MFI.getRestorePoint(); 407 // If RestoreBlock does not have any successor and is not a return block 408 // then the end point is unreachable and we do not need to insert any 409 // epilogue. 410 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) 411 RestoreBlocks.push_back(RestoreBlock); 412 return; 413 } 414 415 // Save refs to entry and return blocks. 416 SaveBlocks.push_back(&MF.front()); 417 for (MachineBasicBlock &MBB : MF) { 418 if (MBB.isEHFuncletEntry()) 419 SaveBlocks.push_back(&MBB); 420 if (MBB.isReturnBlock()) 421 RestoreBlocks.push_back(&MBB); 422 } 423 } 424 425 static void assignCalleeSavedSpillSlots(MachineFunction &F, 426 const BitVector &SavedRegs, 427 unsigned &MinCSFrameIndex, 428 unsigned &MaxCSFrameIndex) { 429 if (SavedRegs.empty()) 430 return; 431 432 const TargetRegisterInfo *RegInfo = F.getSubtarget().getRegisterInfo(); 433 const MCPhysReg *CSRegs = F.getRegInfo().getCalleeSavedRegs(); 434 BitVector CSMask(SavedRegs.size()); 435 436 for (unsigned i = 0; CSRegs[i]; ++i) 437 CSMask.set(CSRegs[i]); 438 439 std::vector<CalleeSavedInfo> CSI; 440 for (unsigned i = 0; CSRegs[i]; ++i) { 441 unsigned Reg = CSRegs[i]; 442 if (SavedRegs.test(Reg)) { 443 bool SavedSuper = false; 444 for (const MCPhysReg &SuperReg : RegInfo->superregs(Reg)) { 445 // Some backends set all aliases for some registers as saved, such as 446 // Mips's $fp, so they appear in SavedRegs but not CSRegs. 447 if (SavedRegs.test(SuperReg) && CSMask.test(SuperReg)) { 448 SavedSuper = true; 449 break; 450 } 451 } 452 453 if (!SavedSuper) 454 CSI.push_back(CalleeSavedInfo(Reg)); 455 } 456 } 457 458 const TargetFrameLowering *TFI = F.getSubtarget().getFrameLowering(); 459 MachineFrameInfo &MFI = F.getFrameInfo(); 460 if (!TFI->assignCalleeSavedSpillSlots(F, RegInfo, CSI, MinCSFrameIndex, 461 MaxCSFrameIndex)) { 462 // If target doesn't implement this, use generic code. 463 464 if (CSI.empty()) 465 return; // Early exit if no callee saved registers are modified! 466 467 unsigned NumFixedSpillSlots; 468 const TargetFrameLowering::SpillSlot *FixedSpillSlots = 469 TFI->getCalleeSavedSpillSlots(NumFixedSpillSlots); 470 471 // Now that we know which registers need to be saved and restored, allocate 472 // stack slots for them. 473 for (auto &CS : CSI) { 474 // If the target has spilled this register to another register, we don't 475 // need to allocate a stack slot. 476 if (CS.isSpilledToReg()) 477 continue; 478 479 unsigned Reg = CS.getReg(); 480 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg); 481 482 int FrameIdx; 483 if (RegInfo->hasReservedSpillSlot(F, Reg, FrameIdx)) { 484 CS.setFrameIdx(FrameIdx); 485 continue; 486 } 487 488 // Check to see if this physreg must be spilled to a particular stack slot 489 // on this target. 490 const TargetFrameLowering::SpillSlot *FixedSlot = FixedSpillSlots; 491 while (FixedSlot != FixedSpillSlots + NumFixedSpillSlots && 492 FixedSlot->Reg != Reg) 493 ++FixedSlot; 494 495 unsigned Size = RegInfo->getSpillSize(*RC); 496 if (FixedSlot == FixedSpillSlots + NumFixedSpillSlots) { 497 // Nope, just spill it anywhere convenient. 498 Align Alignment = RegInfo->getSpillAlign(*RC); 499 // We may not be able to satisfy the desired alignment specification of 500 // the TargetRegisterClass if the stack alignment is smaller. Use the 501 // min. 502 Alignment = std::min(Alignment, TFI->getStackAlign()); 503 FrameIdx = MFI.CreateStackObject(Size, Alignment, true); 504 if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx; 505 if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx; 506 } else { 507 // Spill it to the stack where we must. 508 FrameIdx = MFI.CreateFixedSpillStackObject(Size, FixedSlot->Offset); 509 } 510 511 CS.setFrameIdx(FrameIdx); 512 } 513 } 514 515 MFI.setCalleeSavedInfo(CSI); 516 } 517 518 /// Helper function to update the liveness information for the callee-saved 519 /// registers. 520 static void updateLiveness(MachineFunction &MF) { 521 MachineFrameInfo &MFI = MF.getFrameInfo(); 522 // Visited will contain all the basic blocks that are in the region 523 // where the callee saved registers are alive: 524 // - Anything that is not Save or Restore -> LiveThrough. 525 // - Save -> LiveIn. 526 // - Restore -> LiveOut. 527 // The live-out is not attached to the block, so no need to keep 528 // Restore in this set. 529 SmallPtrSet<MachineBasicBlock *, 8> Visited; 530 SmallVector<MachineBasicBlock *, 8> WorkList; 531 MachineBasicBlock *Entry = &MF.front(); 532 MachineBasicBlock *Save = MFI.getSavePoint(); 533 534 if (!Save) 535 Save = Entry; 536 537 if (Entry != Save) { 538 WorkList.push_back(Entry); 539 Visited.insert(Entry); 540 } 541 Visited.insert(Save); 542 543 MachineBasicBlock *Restore = MFI.getRestorePoint(); 544 if (Restore) 545 // By construction Restore cannot be visited, otherwise it 546 // means there exists a path to Restore that does not go 547 // through Save. 548 WorkList.push_back(Restore); 549 550 while (!WorkList.empty()) { 551 const MachineBasicBlock *CurBB = WorkList.pop_back_val(); 552 // By construction, the region that is after the save point is 553 // dominated by the Save and post-dominated by the Restore. 554 if (CurBB == Save && Save != Restore) 555 continue; 556 // Enqueue all the successors not already visited. 557 // Those are by construction either before Save or after Restore. 558 for (MachineBasicBlock *SuccBB : CurBB->successors()) 559 if (Visited.insert(SuccBB).second) 560 WorkList.push_back(SuccBB); 561 } 562 563 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); 564 565 MachineRegisterInfo &MRI = MF.getRegInfo(); 566 for (const CalleeSavedInfo &I : CSI) { 567 for (MachineBasicBlock *MBB : Visited) { 568 MCPhysReg Reg = I.getReg(); 569 // Add the callee-saved register as live-in. 570 // It's killed at the spill. 571 if (!MRI.isReserved(Reg) && !MBB->isLiveIn(Reg)) 572 MBB->addLiveIn(Reg); 573 } 574 // If callee-saved register is spilled to another register rather than 575 // spilling to stack, the destination register has to be marked as live for 576 // each MBB between the prologue and epilogue so that it is not clobbered 577 // before it is reloaded in the epilogue. The Visited set contains all 578 // blocks outside of the region delimited by prologue/epilogue. 579 if (I.isSpilledToReg()) { 580 for (MachineBasicBlock &MBB : MF) { 581 if (Visited.count(&MBB)) 582 continue; 583 MCPhysReg DstReg = I.getDstReg(); 584 if (!MBB.isLiveIn(DstReg)) 585 MBB.addLiveIn(DstReg); 586 } 587 } 588 } 589 } 590 591 /// Insert spill code for the callee-saved registers used in the function. 592 static void insertCSRSaves(MachineBasicBlock &SaveBlock, 593 ArrayRef<CalleeSavedInfo> CSI) { 594 MachineFunction &MF = *SaveBlock.getParent(); 595 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); 596 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); 597 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 598 599 MachineBasicBlock::iterator I = SaveBlock.begin(); 600 if (!TFI->spillCalleeSavedRegisters(SaveBlock, I, CSI, TRI)) { 601 for (const CalleeSavedInfo &CS : CSI) { 602 // Insert the spill to the stack frame. 603 unsigned Reg = CS.getReg(); 604 605 if (CS.isSpilledToReg()) { 606 BuildMI(SaveBlock, I, DebugLoc(), 607 TII.get(TargetOpcode::COPY), CS.getDstReg()) 608 .addReg(Reg, getKillRegState(true)); 609 } else { 610 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); 611 TII.storeRegToStackSlot(SaveBlock, I, Reg, true, CS.getFrameIdx(), RC, 612 TRI, Register()); 613 } 614 } 615 } 616 } 617 618 /// Insert restore code for the callee-saved registers used in the function. 619 static void insertCSRRestores(MachineBasicBlock &RestoreBlock, 620 std::vector<CalleeSavedInfo> &CSI) { 621 MachineFunction &MF = *RestoreBlock.getParent(); 622 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); 623 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); 624 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 625 626 // Restore all registers immediately before the return and any 627 // terminators that precede it. 628 MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator(); 629 630 if (!TFI->restoreCalleeSavedRegisters(RestoreBlock, I, CSI, TRI)) { 631 for (const CalleeSavedInfo &CI : reverse(CSI)) { 632 unsigned Reg = CI.getReg(); 633 if (CI.isSpilledToReg()) { 634 BuildMI(RestoreBlock, I, DebugLoc(), TII.get(TargetOpcode::COPY), Reg) 635 .addReg(CI.getDstReg(), getKillRegState(true)); 636 } else { 637 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); 638 TII.loadRegFromStackSlot(RestoreBlock, I, Reg, CI.getFrameIdx(), RC, 639 TRI, Register()); 640 assert(I != RestoreBlock.begin() && 641 "loadRegFromStackSlot didn't insert any code!"); 642 // Insert in reverse order. loadRegFromStackSlot can insert 643 // multiple instructions. 644 } 645 } 646 } 647 } 648 649 void PEI::spillCalleeSavedRegs(MachineFunction &MF) { 650 // We can't list this requirement in getRequiredProperties because some 651 // targets (WebAssembly) use virtual registers past this point, and the pass 652 // pipeline is set up without giving the passes a chance to look at the 653 // TargetMachine. 654 // FIXME: Find a way to express this in getRequiredProperties. 655 assert(MF.getProperties().hasProperty( 656 MachineFunctionProperties::Property::NoVRegs)); 657 658 const Function &F = MF.getFunction(); 659 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); 660 MachineFrameInfo &MFI = MF.getFrameInfo(); 661 MinCSFrameIndex = std::numeric_limits<unsigned>::max(); 662 MaxCSFrameIndex = 0; 663 664 // Determine which of the registers in the callee save list should be saved. 665 BitVector SavedRegs; 666 TFI->determineCalleeSaves(MF, SavedRegs, RS); 667 668 // Assign stack slots for any callee-saved registers that must be spilled. 669 assignCalleeSavedSpillSlots(MF, SavedRegs, MinCSFrameIndex, MaxCSFrameIndex); 670 671 // Add the code to save and restore the callee saved registers. 672 if (!F.hasFnAttribute(Attribute::Naked)) { 673 MFI.setCalleeSavedInfoValid(true); 674 675 std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); 676 if (!CSI.empty()) { 677 if (!MFI.hasCalls()) 678 NumLeafFuncWithSpills++; 679 680 for (MachineBasicBlock *SaveBlock : SaveBlocks) 681 insertCSRSaves(*SaveBlock, CSI); 682 683 // Update the live-in information of all the blocks up to the save point. 684 updateLiveness(MF); 685 686 for (MachineBasicBlock *RestoreBlock : RestoreBlocks) 687 insertCSRRestores(*RestoreBlock, CSI); 688 } 689 } 690 } 691 692 /// AdjustStackOffset - Helper function used to adjust the stack frame offset. 693 static inline void AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx, 694 bool StackGrowsDown, int64_t &Offset, 695 Align &MaxAlign, unsigned Skew) { 696 // If the stack grows down, add the object size to find the lowest address. 697 if (StackGrowsDown) 698 Offset += MFI.getObjectSize(FrameIdx); 699 700 Align Alignment = MFI.getObjectAlign(FrameIdx); 701 702 // If the alignment of this object is greater than that of the stack, then 703 // increase the stack alignment to match. 704 MaxAlign = std::max(MaxAlign, Alignment); 705 706 // Adjust to alignment boundary. 707 Offset = alignTo(Offset, Alignment, Skew); 708 709 if (StackGrowsDown) { 710 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << -Offset 711 << "]\n"); 712 MFI.setObjectOffset(FrameIdx, -Offset); // Set the computed offset 713 } else { 714 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << Offset 715 << "]\n"); 716 MFI.setObjectOffset(FrameIdx, Offset); 717 Offset += MFI.getObjectSize(FrameIdx); 718 } 719 } 720 721 /// Compute which bytes of fixed and callee-save stack area are unused and keep 722 /// track of them in StackBytesFree. 723 static inline void 724 computeFreeStackSlots(MachineFrameInfo &MFI, bool StackGrowsDown, 725 unsigned MinCSFrameIndex, unsigned MaxCSFrameIndex, 726 int64_t FixedCSEnd, BitVector &StackBytesFree) { 727 // Avoid undefined int64_t -> int conversion below in extreme case. 728 if (FixedCSEnd > std::numeric_limits<int>::max()) 729 return; 730 731 StackBytesFree.resize(FixedCSEnd, true); 732 733 SmallVector<int, 16> AllocatedFrameSlots; 734 // Add fixed objects. 735 for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) 736 // StackSlot scavenging is only implemented for the default stack. 737 if (MFI.getStackID(i) == TargetStackID::Default) 738 AllocatedFrameSlots.push_back(i); 739 // Add callee-save objects if there are any. 740 if (MinCSFrameIndex <= MaxCSFrameIndex) { 741 for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i) 742 if (MFI.getStackID(i) == TargetStackID::Default) 743 AllocatedFrameSlots.push_back(i); 744 } 745 746 for (int i : AllocatedFrameSlots) { 747 // These are converted from int64_t, but they should always fit in int 748 // because of the FixedCSEnd check above. 749 int ObjOffset = MFI.getObjectOffset(i); 750 int ObjSize = MFI.getObjectSize(i); 751 int ObjStart, ObjEnd; 752 if (StackGrowsDown) { 753 // ObjOffset is negative when StackGrowsDown is true. 754 ObjStart = -ObjOffset - ObjSize; 755 ObjEnd = -ObjOffset; 756 } else { 757 ObjStart = ObjOffset; 758 ObjEnd = ObjOffset + ObjSize; 759 } 760 // Ignore fixed holes that are in the previous stack frame. 761 if (ObjEnd > 0) 762 StackBytesFree.reset(ObjStart, ObjEnd); 763 } 764 } 765 766 /// Assign frame object to an unused portion of the stack in the fixed stack 767 /// object range. Return true if the allocation was successful. 768 static inline bool scavengeStackSlot(MachineFrameInfo &MFI, int FrameIdx, 769 bool StackGrowsDown, Align MaxAlign, 770 BitVector &StackBytesFree) { 771 if (MFI.isVariableSizedObjectIndex(FrameIdx)) 772 return false; 773 774 if (StackBytesFree.none()) { 775 // clear it to speed up later scavengeStackSlot calls to 776 // StackBytesFree.none() 777 StackBytesFree.clear(); 778 return false; 779 } 780 781 Align ObjAlign = MFI.getObjectAlign(FrameIdx); 782 if (ObjAlign > MaxAlign) 783 return false; 784 785 int64_t ObjSize = MFI.getObjectSize(FrameIdx); 786 int FreeStart; 787 for (FreeStart = StackBytesFree.find_first(); FreeStart != -1; 788 FreeStart = StackBytesFree.find_next(FreeStart)) { 789 790 // Check that free space has suitable alignment. 791 unsigned ObjStart = StackGrowsDown ? FreeStart + ObjSize : FreeStart; 792 if (alignTo(ObjStart, ObjAlign) != ObjStart) 793 continue; 794 795 if (FreeStart + ObjSize > StackBytesFree.size()) 796 return false; 797 798 bool AllBytesFree = true; 799 for (unsigned Byte = 0; Byte < ObjSize; ++Byte) 800 if (!StackBytesFree.test(FreeStart + Byte)) { 801 AllBytesFree = false; 802 break; 803 } 804 if (AllBytesFree) 805 break; 806 } 807 808 if (FreeStart == -1) 809 return false; 810 811 if (StackGrowsDown) { 812 int ObjStart = -(FreeStart + ObjSize); 813 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP[" 814 << ObjStart << "]\n"); 815 MFI.setObjectOffset(FrameIdx, ObjStart); 816 } else { 817 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP[" 818 << FreeStart << "]\n"); 819 MFI.setObjectOffset(FrameIdx, FreeStart); 820 } 821 822 StackBytesFree.reset(FreeStart, FreeStart + ObjSize); 823 return true; 824 } 825 826 /// AssignProtectedObjSet - Helper function to assign large stack objects (i.e., 827 /// those required to be close to the Stack Protector) to stack offsets. 828 static void AssignProtectedObjSet(const StackObjSet &UnassignedObjs, 829 SmallSet<int, 16> &ProtectedObjs, 830 MachineFrameInfo &MFI, bool StackGrowsDown, 831 int64_t &Offset, Align &MaxAlign, 832 unsigned Skew) { 833 834 for (int i : UnassignedObjs) { 835 AdjustStackOffset(MFI, i, StackGrowsDown, Offset, MaxAlign, Skew); 836 ProtectedObjs.insert(i); 837 } 838 } 839 840 /// calculateFrameObjectOffsets - Calculate actual frame offsets for all of the 841 /// abstract stack objects. 842 void PEI::calculateFrameObjectOffsets(MachineFunction &MF) { 843 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); 844 845 bool StackGrowsDown = 846 TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown; 847 848 // Loop over all of the stack objects, assigning sequential addresses... 849 MachineFrameInfo &MFI = MF.getFrameInfo(); 850 851 // Start at the beginning of the local area. 852 // The Offset is the distance from the stack top in the direction 853 // of stack growth -- so it's always nonnegative. 854 int LocalAreaOffset = TFI.getOffsetOfLocalArea(); 855 if (StackGrowsDown) 856 LocalAreaOffset = -LocalAreaOffset; 857 assert(LocalAreaOffset >= 0 858 && "Local area offset should be in direction of stack growth"); 859 int64_t Offset = LocalAreaOffset; 860 861 // Skew to be applied to alignment. 862 unsigned Skew = TFI.getStackAlignmentSkew(MF); 863 864 #ifdef EXPENSIVE_CHECKS 865 for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) 866 if (!MFI.isDeadObjectIndex(i) && 867 MFI.getStackID(i) == TargetStackID::Default) 868 assert(MFI.getObjectAlign(i) <= MFI.getMaxAlign() && 869 "MaxAlignment is invalid"); 870 #endif 871 872 // If there are fixed sized objects that are preallocated in the local area, 873 // non-fixed objects can't be allocated right at the start of local area. 874 // Adjust 'Offset' to point to the end of last fixed sized preallocated 875 // object. 876 for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) { 877 // Only allocate objects on the default stack. 878 if (MFI.getStackID(i) != TargetStackID::Default) 879 continue; 880 881 int64_t FixedOff; 882 if (StackGrowsDown) { 883 // The maximum distance from the stack pointer is at lower address of 884 // the object -- which is given by offset. For down growing stack 885 // the offset is negative, so we negate the offset to get the distance. 886 FixedOff = -MFI.getObjectOffset(i); 887 } else { 888 // The maximum distance from the start pointer is at the upper 889 // address of the object. 890 FixedOff = MFI.getObjectOffset(i) + MFI.getObjectSize(i); 891 } 892 if (FixedOff > Offset) Offset = FixedOff; 893 } 894 895 Align MaxAlign = MFI.getMaxAlign(); 896 // First assign frame offsets to stack objects that are used to spill 897 // callee saved registers. 898 if (MaxCSFrameIndex >= MinCSFrameIndex) { 899 for (unsigned i = 0; i <= MaxCSFrameIndex - MinCSFrameIndex; ++i) { 900 unsigned FrameIndex = 901 StackGrowsDown ? MinCSFrameIndex + i : MaxCSFrameIndex - i; 902 903 // Only allocate objects on the default stack. 904 if (MFI.getStackID(FrameIndex) != TargetStackID::Default) 905 continue; 906 907 // TODO: should this just be if (MFI.isDeadObjectIndex(FrameIndex)) 908 if (!StackGrowsDown && MFI.isDeadObjectIndex(FrameIndex)) 909 continue; 910 911 AdjustStackOffset(MFI, FrameIndex, StackGrowsDown, Offset, MaxAlign, 912 Skew); 913 } 914 } 915 916 assert(MaxAlign == MFI.getMaxAlign() && 917 "MFI.getMaxAlign should already account for all callee-saved " 918 "registers without a fixed stack slot"); 919 920 // FixedCSEnd is the stack offset to the end of the fixed and callee-save 921 // stack area. 922 int64_t FixedCSEnd = Offset; 923 924 // Make sure the special register scavenging spill slot is closest to the 925 // incoming stack pointer if a frame pointer is required and is closer 926 // to the incoming rather than the final stack pointer. 927 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); 928 bool EarlyScavengingSlots = TFI.allocateScavengingFrameIndexesNearIncomingSP(MF); 929 if (RS && EarlyScavengingSlots) { 930 SmallVector<int, 2> SFIs; 931 RS->getScavengingFrameIndices(SFIs); 932 for (int SFI : SFIs) 933 AdjustStackOffset(MFI, SFI, StackGrowsDown, Offset, MaxAlign, Skew); 934 } 935 936 // FIXME: Once this is working, then enable flag will change to a target 937 // check for whether the frame is large enough to want to use virtual 938 // frame index registers. Functions which don't want/need this optimization 939 // will continue to use the existing code path. 940 if (MFI.getUseLocalStackAllocationBlock()) { 941 Align Alignment = MFI.getLocalFrameMaxAlign(); 942 943 // Adjust to alignment boundary. 944 Offset = alignTo(Offset, Alignment, Skew); 945 946 LLVM_DEBUG(dbgs() << "Local frame base offset: " << Offset << "\n"); 947 948 // Resolve offsets for objects in the local block. 949 for (unsigned i = 0, e = MFI.getLocalFrameObjectCount(); i != e; ++i) { 950 std::pair<int, int64_t> Entry = MFI.getLocalFrameObjectMap(i); 951 int64_t FIOffset = (StackGrowsDown ? -Offset : Offset) + Entry.second; 952 LLVM_DEBUG(dbgs() << "alloc FI(" << Entry.first << ") at SP[" << FIOffset 953 << "]\n"); 954 MFI.setObjectOffset(Entry.first, FIOffset); 955 } 956 // Allocate the local block 957 Offset += MFI.getLocalFrameSize(); 958 959 MaxAlign = std::max(Alignment, MaxAlign); 960 } 961 962 // Retrieve the Exception Handler registration node. 963 int EHRegNodeFrameIndex = std::numeric_limits<int>::max(); 964 if (const WinEHFuncInfo *FuncInfo = MF.getWinEHFuncInfo()) 965 EHRegNodeFrameIndex = FuncInfo->EHRegNodeFrameIndex; 966 967 // Make sure that the stack protector comes before the local variables on the 968 // stack. 969 SmallSet<int, 16> ProtectedObjs; 970 if (MFI.hasStackProtectorIndex()) { 971 int StackProtectorFI = MFI.getStackProtectorIndex(); 972 StackObjSet LargeArrayObjs; 973 StackObjSet SmallArrayObjs; 974 StackObjSet AddrOfObjs; 975 976 // If we need a stack protector, we need to make sure that 977 // LocalStackSlotPass didn't already allocate a slot for it. 978 // If we are told to use the LocalStackAllocationBlock, the stack protector 979 // is expected to be already pre-allocated. 980 if (MFI.getStackID(StackProtectorFI) != TargetStackID::Default) { 981 // If the stack protector isn't on the default stack then it's up to the 982 // target to set the stack offset. 983 assert(MFI.getObjectOffset(StackProtectorFI) != 0 && 984 "Offset of stack protector on non-default stack expected to be " 985 "already set."); 986 assert(!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex()) && 987 "Stack protector on non-default stack expected to not be " 988 "pre-allocated by LocalStackSlotPass."); 989 } else if (!MFI.getUseLocalStackAllocationBlock()) { 990 AdjustStackOffset(MFI, StackProtectorFI, StackGrowsDown, Offset, MaxAlign, 991 Skew); 992 } else if (!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex())) { 993 llvm_unreachable( 994 "Stack protector not pre-allocated by LocalStackSlotPass."); 995 } 996 997 // Assign large stack objects first. 998 for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) { 999 if (MFI.isObjectPreAllocated(i) && MFI.getUseLocalStackAllocationBlock()) 1000 continue; 1001 if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) 1002 continue; 1003 if (RS && RS->isScavengingFrameIndex((int)i)) 1004 continue; 1005 if (MFI.isDeadObjectIndex(i)) 1006 continue; 1007 if (StackProtectorFI == (int)i || EHRegNodeFrameIndex == (int)i) 1008 continue; 1009 // Only allocate objects on the default stack. 1010 if (MFI.getStackID(i) != TargetStackID::Default) 1011 continue; 1012 1013 switch (MFI.getObjectSSPLayout(i)) { 1014 case MachineFrameInfo::SSPLK_None: 1015 continue; 1016 case MachineFrameInfo::SSPLK_SmallArray: 1017 SmallArrayObjs.insert(i); 1018 continue; 1019 case MachineFrameInfo::SSPLK_AddrOf: 1020 AddrOfObjs.insert(i); 1021 continue; 1022 case MachineFrameInfo::SSPLK_LargeArray: 1023 LargeArrayObjs.insert(i); 1024 continue; 1025 } 1026 llvm_unreachable("Unexpected SSPLayoutKind."); 1027 } 1028 1029 // We expect **all** the protected stack objects to be pre-allocated by 1030 // LocalStackSlotPass. If it turns out that PEI still has to allocate some 1031 // of them, we may end up messing up the expected order of the objects. 1032 if (MFI.getUseLocalStackAllocationBlock() && 1033 !(LargeArrayObjs.empty() && SmallArrayObjs.empty() && 1034 AddrOfObjs.empty())) 1035 llvm_unreachable("Found protected stack objects not pre-allocated by " 1036 "LocalStackSlotPass."); 1037 1038 AssignProtectedObjSet(LargeArrayObjs, ProtectedObjs, MFI, StackGrowsDown, 1039 Offset, MaxAlign, Skew); 1040 AssignProtectedObjSet(SmallArrayObjs, ProtectedObjs, MFI, StackGrowsDown, 1041 Offset, MaxAlign, Skew); 1042 AssignProtectedObjSet(AddrOfObjs, ProtectedObjs, MFI, StackGrowsDown, 1043 Offset, MaxAlign, Skew); 1044 } 1045 1046 SmallVector<int, 8> ObjectsToAllocate; 1047 1048 // Then prepare to assign frame offsets to stack objects that are not used to 1049 // spill callee saved registers. 1050 for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) { 1051 if (MFI.isObjectPreAllocated(i) && MFI.getUseLocalStackAllocationBlock()) 1052 continue; 1053 if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) 1054 continue; 1055 if (RS && RS->isScavengingFrameIndex((int)i)) 1056 continue; 1057 if (MFI.isDeadObjectIndex(i)) 1058 continue; 1059 if (MFI.getStackProtectorIndex() == (int)i || EHRegNodeFrameIndex == (int)i) 1060 continue; 1061 if (ProtectedObjs.count(i)) 1062 continue; 1063 // Only allocate objects on the default stack. 1064 if (MFI.getStackID(i) != TargetStackID::Default) 1065 continue; 1066 1067 // Add the objects that we need to allocate to our working set. 1068 ObjectsToAllocate.push_back(i); 1069 } 1070 1071 // Allocate the EH registration node first if one is present. 1072 if (EHRegNodeFrameIndex != std::numeric_limits<int>::max()) 1073 AdjustStackOffset(MFI, EHRegNodeFrameIndex, StackGrowsDown, Offset, 1074 MaxAlign, Skew); 1075 1076 // Give the targets a chance to order the objects the way they like it. 1077 if (MF.getTarget().getOptLevel() != CodeGenOpt::None && 1078 MF.getTarget().Options.StackSymbolOrdering) 1079 TFI.orderFrameObjects(MF, ObjectsToAllocate); 1080 1081 // Keep track of which bytes in the fixed and callee-save range are used so we 1082 // can use the holes when allocating later stack objects. Only do this if 1083 // stack protector isn't being used and the target requests it and we're 1084 // optimizing. 1085 BitVector StackBytesFree; 1086 if (!ObjectsToAllocate.empty() && 1087 MF.getTarget().getOptLevel() != CodeGenOpt::None && 1088 MFI.getStackProtectorIndex() < 0 && TFI.enableStackSlotScavenging(MF)) 1089 computeFreeStackSlots(MFI, StackGrowsDown, MinCSFrameIndex, MaxCSFrameIndex, 1090 FixedCSEnd, StackBytesFree); 1091 1092 // Now walk the objects and actually assign base offsets to them. 1093 for (auto &Object : ObjectsToAllocate) 1094 if (!scavengeStackSlot(MFI, Object, StackGrowsDown, MaxAlign, 1095 StackBytesFree)) 1096 AdjustStackOffset(MFI, Object, StackGrowsDown, Offset, MaxAlign, Skew); 1097 1098 // Make sure the special register scavenging spill slot is closest to the 1099 // stack pointer. 1100 if (RS && !EarlyScavengingSlots) { 1101 SmallVector<int, 2> SFIs; 1102 RS->getScavengingFrameIndices(SFIs); 1103 for (int SFI : SFIs) 1104 AdjustStackOffset(MFI, SFI, StackGrowsDown, Offset, MaxAlign, Skew); 1105 } 1106 1107 if (!TFI.targetHandlesStackFrameRounding()) { 1108 // If we have reserved argument space for call sites in the function 1109 // immediately on entry to the current function, count it as part of the 1110 // overall stack size. 1111 if (MFI.adjustsStack() && TFI.hasReservedCallFrame(MF)) 1112 Offset += MFI.getMaxCallFrameSize(); 1113 1114 // Round up the size to a multiple of the alignment. If the function has 1115 // any calls or alloca's, align to the target's StackAlignment value to 1116 // ensure that the callee's frame or the alloca data is suitably aligned; 1117 // otherwise, for leaf functions, align to the TransientStackAlignment 1118 // value. 1119 Align StackAlign; 1120 if (MFI.adjustsStack() || MFI.hasVarSizedObjects() || 1121 (RegInfo->hasStackRealignment(MF) && MFI.getObjectIndexEnd() != 0)) 1122 StackAlign = TFI.getStackAlign(); 1123 else 1124 StackAlign = TFI.getTransientStackAlign(); 1125 1126 // If the frame pointer is eliminated, all frame offsets will be relative to 1127 // SP not FP. Align to MaxAlign so this works. 1128 StackAlign = std::max(StackAlign, MaxAlign); 1129 int64_t OffsetBeforeAlignment = Offset; 1130 Offset = alignTo(Offset, StackAlign, Skew); 1131 1132 // If we have increased the offset to fulfill the alignment constrants, 1133 // then the scavenging spill slots may become harder to reach from the 1134 // stack pointer, float them so they stay close. 1135 if (StackGrowsDown && OffsetBeforeAlignment != Offset && RS && 1136 !EarlyScavengingSlots) { 1137 SmallVector<int, 2> SFIs; 1138 RS->getScavengingFrameIndices(SFIs); 1139 LLVM_DEBUG(if (!SFIs.empty()) llvm::dbgs() 1140 << "Adjusting emergency spill slots!\n";); 1141 int64_t Delta = Offset - OffsetBeforeAlignment; 1142 for (int SFI : SFIs) { 1143 LLVM_DEBUG(llvm::dbgs() 1144 << "Adjusting offset of emergency spill slot #" << SFI 1145 << " from " << MFI.getObjectOffset(SFI);); 1146 MFI.setObjectOffset(SFI, MFI.getObjectOffset(SFI) - Delta); 1147 LLVM_DEBUG(llvm::dbgs() << " to " << MFI.getObjectOffset(SFI) << "\n";); 1148 } 1149 } 1150 } 1151 1152 // Update frame info to pretend that this is part of the stack... 1153 int64_t StackSize = Offset - LocalAreaOffset; 1154 MFI.setStackSize(StackSize); 1155 NumBytesStackSpace += StackSize; 1156 } 1157 1158 /// insertPrologEpilogCode - Scan the function for modified callee saved 1159 /// registers, insert spill code for these callee saved registers, then add 1160 /// prolog and epilog code to the function. 1161 void PEI::insertPrologEpilogCode(MachineFunction &MF) { 1162 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); 1163 1164 // Add prologue to the function... 1165 for (MachineBasicBlock *SaveBlock : SaveBlocks) 1166 TFI.emitPrologue(MF, *SaveBlock); 1167 1168 // Add epilogue to restore the callee-save registers in each exiting block. 1169 for (MachineBasicBlock *RestoreBlock : RestoreBlocks) 1170 TFI.emitEpilogue(MF, *RestoreBlock); 1171 1172 // Zero call used registers before restoring callee-saved registers. 1173 insertZeroCallUsedRegs(MF); 1174 1175 for (MachineBasicBlock *SaveBlock : SaveBlocks) 1176 TFI.inlineStackProbe(MF, *SaveBlock); 1177 1178 // Emit additional code that is required to support segmented stacks, if 1179 // we've been asked for it. This, when linked with a runtime with support 1180 // for segmented stacks (libgcc is one), will result in allocating stack 1181 // space in small chunks instead of one large contiguous block. 1182 if (MF.shouldSplitStack()) { 1183 for (MachineBasicBlock *SaveBlock : SaveBlocks) 1184 TFI.adjustForSegmentedStacks(MF, *SaveBlock); 1185 } 1186 1187 // Emit additional code that is required to explicitly handle the stack in 1188 // HiPE native code (if needed) when loaded in the Erlang/OTP runtime. The 1189 // approach is rather similar to that of Segmented Stacks, but it uses a 1190 // different conditional check and another BIF for allocating more stack 1191 // space. 1192 if (MF.getFunction().getCallingConv() == CallingConv::HiPE) 1193 for (MachineBasicBlock *SaveBlock : SaveBlocks) 1194 TFI.adjustForHiPEPrologue(MF, *SaveBlock); 1195 } 1196 1197 /// insertZeroCallUsedRegs - Zero out call used registers. 1198 void PEI::insertZeroCallUsedRegs(MachineFunction &MF) { 1199 const Function &F = MF.getFunction(); 1200 1201 if (!F.hasFnAttribute("zero-call-used-regs")) 1202 return; 1203 1204 using namespace ZeroCallUsedRegs; 1205 1206 ZeroCallUsedRegsKind ZeroRegsKind = 1207 StringSwitch<ZeroCallUsedRegsKind>( 1208 F.getFnAttribute("zero-call-used-regs").getValueAsString()) 1209 .Case("skip", ZeroCallUsedRegsKind::Skip) 1210 .Case("used-gpr-arg", ZeroCallUsedRegsKind::UsedGPRArg) 1211 .Case("used-gpr", ZeroCallUsedRegsKind::UsedGPR) 1212 .Case("used-arg", ZeroCallUsedRegsKind::UsedArg) 1213 .Case("used", ZeroCallUsedRegsKind::Used) 1214 .Case("all-gpr-arg", ZeroCallUsedRegsKind::AllGPRArg) 1215 .Case("all-gpr", ZeroCallUsedRegsKind::AllGPR) 1216 .Case("all-arg", ZeroCallUsedRegsKind::AllArg) 1217 .Case("all", ZeroCallUsedRegsKind::All); 1218 1219 if (ZeroRegsKind == ZeroCallUsedRegsKind::Skip) 1220 return; 1221 1222 const bool OnlyGPR = static_cast<unsigned>(ZeroRegsKind) & ONLY_GPR; 1223 const bool OnlyUsed = static_cast<unsigned>(ZeroRegsKind) & ONLY_USED; 1224 const bool OnlyArg = static_cast<unsigned>(ZeroRegsKind) & ONLY_ARG; 1225 1226 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 1227 const BitVector AllocatableSet(TRI.getAllocatableSet(MF)); 1228 1229 // Mark all used registers. 1230 BitVector UsedRegs(TRI.getNumRegs()); 1231 if (OnlyUsed) 1232 for (const MachineBasicBlock &MBB : MF) 1233 for (const MachineInstr &MI : MBB) { 1234 // skip debug instructions 1235 if (MI.isDebugInstr()) 1236 continue; 1237 1238 for (const MachineOperand &MO : MI.operands()) { 1239 if (!MO.isReg()) 1240 continue; 1241 1242 MCRegister Reg = MO.getReg(); 1243 if (AllocatableSet[Reg] && !MO.isImplicit() && 1244 (MO.isDef() || MO.isUse())) 1245 UsedRegs.set(Reg); 1246 } 1247 } 1248 1249 // Get a list of registers that are used. 1250 BitVector LiveIns(TRI.getNumRegs()); 1251 for (const MachineBasicBlock::RegisterMaskPair &LI : MF.front().liveins()) 1252 LiveIns.set(LI.PhysReg); 1253 1254 BitVector RegsToZero(TRI.getNumRegs()); 1255 for (MCRegister Reg : AllocatableSet.set_bits()) { 1256 // Skip over fixed registers. 1257 if (TRI.isFixedRegister(MF, Reg)) 1258 continue; 1259 1260 // Want only general purpose registers. 1261 if (OnlyGPR && !TRI.isGeneralPurposeRegister(MF, Reg)) 1262 continue; 1263 1264 // Want only used registers. 1265 if (OnlyUsed && !UsedRegs[Reg]) 1266 continue; 1267 1268 // Want only registers used for arguments. 1269 if (OnlyArg) { 1270 if (OnlyUsed) { 1271 if (!LiveIns[Reg]) 1272 continue; 1273 } else if (!TRI.isArgumentRegister(MF, Reg)) { 1274 continue; 1275 } 1276 } 1277 1278 RegsToZero.set(Reg); 1279 } 1280 1281 // Don't clear registers that are live when leaving the function. 1282 for (const MachineBasicBlock &MBB : MF) 1283 for (const MachineInstr &MI : MBB.terminators()) { 1284 if (!MI.isReturn()) 1285 continue; 1286 1287 for (const auto &MO : MI.operands()) { 1288 if (!MO.isReg()) 1289 continue; 1290 1291 MCRegister Reg = MO.getReg(); 1292 1293 // This picks up sibling registers (e.q. %al -> %ah). 1294 for (MCRegUnitIterator Unit(Reg, &TRI); Unit.isValid(); ++Unit) 1295 RegsToZero.reset(*Unit); 1296 1297 for (MCPhysReg SReg : TRI.sub_and_superregs_inclusive(Reg)) 1298 RegsToZero.reset(SReg); 1299 } 1300 } 1301 1302 // Don't need to clear registers that are used/clobbered by terminating 1303 // instructions. 1304 for (const MachineBasicBlock &MBB : MF) { 1305 if (!MBB.isReturnBlock()) 1306 continue; 1307 1308 MachineBasicBlock::const_iterator MBBI = MBB.getFirstTerminator(); 1309 for (MachineBasicBlock::const_iterator I = MBBI, E = MBB.end(); I != E; 1310 ++I) { 1311 for (const MachineOperand &MO : I->operands()) { 1312 if (!MO.isReg()) 1313 continue; 1314 1315 for (const MCPhysReg &Reg : 1316 TRI.sub_and_superregs_inclusive(MO.getReg())) 1317 RegsToZero.reset(Reg); 1318 } 1319 } 1320 } 1321 1322 // Don't clear registers that must be preserved. 1323 for (const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(&MF); 1324 MCPhysReg CSReg = *CSRegs; ++CSRegs) 1325 for (MCRegister Reg : TRI.sub_and_superregs_inclusive(CSReg)) 1326 RegsToZero.reset(Reg); 1327 1328 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); 1329 for (MachineBasicBlock &MBB : MF) 1330 if (MBB.isReturnBlock()) 1331 TFI.emitZeroCallUsedRegs(RegsToZero, MBB); 1332 } 1333 1334 /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical 1335 /// register references and actual offsets. 1336 void PEI::replaceFrameIndices(MachineFunction &MF) { 1337 const auto &ST = MF.getSubtarget(); 1338 const TargetFrameLowering &TFI = *ST.getFrameLowering(); 1339 if (!TFI.needsFrameIndexResolution(MF)) 1340 return; 1341 1342 const TargetRegisterInfo *TRI = ST.getRegisterInfo(); 1343 1344 // Allow the target to determine this after knowing the frame size. 1345 FrameIndexEliminationScavenging = (RS && !FrameIndexVirtualScavenging) || 1346 TRI->requiresFrameIndexReplacementScavenging(MF); 1347 1348 // Store SPAdj at exit of a basic block. 1349 SmallVector<int, 8> SPState; 1350 SPState.resize(MF.getNumBlockIDs()); 1351 df_iterator_default_set<MachineBasicBlock*> Reachable; 1352 1353 // Iterate over the reachable blocks in DFS order. 1354 for (auto DFI = df_ext_begin(&MF, Reachable), DFE = df_ext_end(&MF, Reachable); 1355 DFI != DFE; ++DFI) { 1356 int SPAdj = 0; 1357 // Check the exit state of the DFS stack predecessor. 1358 if (DFI.getPathLength() >= 2) { 1359 MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2); 1360 assert(Reachable.count(StackPred) && 1361 "DFS stack predecessor is already visited.\n"); 1362 SPAdj = SPState[StackPred->getNumber()]; 1363 } 1364 MachineBasicBlock *BB = *DFI; 1365 replaceFrameIndices(BB, MF, SPAdj); 1366 SPState[BB->getNumber()] = SPAdj; 1367 } 1368 1369 // Handle the unreachable blocks. 1370 for (auto &BB : MF) { 1371 if (Reachable.count(&BB)) 1372 // Already handled in DFS traversal. 1373 continue; 1374 int SPAdj = 0; 1375 replaceFrameIndices(&BB, MF, SPAdj); 1376 } 1377 } 1378 1379 bool PEI::replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI, 1380 unsigned OpIdx, int SPAdj) { 1381 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); 1382 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 1383 if (MI.isDebugValue()) { 1384 1385 MachineOperand &Op = MI.getOperand(OpIdx); 1386 assert(MI.isDebugOperand(&Op) && 1387 "Frame indices can only appear as a debug operand in a DBG_VALUE*" 1388 " machine instruction"); 1389 Register Reg; 1390 unsigned FrameIdx = Op.getIndex(); 1391 unsigned Size = MF.getFrameInfo().getObjectSize(FrameIdx); 1392 1393 StackOffset Offset = TFI->getFrameIndexReference(MF, FrameIdx, Reg); 1394 Op.ChangeToRegister(Reg, false /*isDef*/); 1395 1396 const DIExpression *DIExpr = MI.getDebugExpression(); 1397 1398 // If we have a direct DBG_VALUE, and its location expression isn't 1399 // currently complex, then adding an offset will morph it into a 1400 // complex location that is interpreted as being a memory address. 1401 // This changes a pointer-valued variable to dereference that pointer, 1402 // which is incorrect. Fix by adding DW_OP_stack_value. 1403 1404 if (MI.isNonListDebugValue()) { 1405 unsigned PrependFlags = DIExpression::ApplyOffset; 1406 if (!MI.isIndirectDebugValue() && !DIExpr->isComplex()) 1407 PrependFlags |= DIExpression::StackValue; 1408 1409 // If we have DBG_VALUE that is indirect and has a Implicit location 1410 // expression need to insert a deref before prepending a Memory 1411 // location expression. Also after doing this we change the DBG_VALUE 1412 // to be direct. 1413 if (MI.isIndirectDebugValue() && DIExpr->isImplicit()) { 1414 SmallVector<uint64_t, 2> Ops = {dwarf::DW_OP_deref_size, Size}; 1415 bool WithStackValue = true; 1416 DIExpr = DIExpression::prependOpcodes(DIExpr, Ops, WithStackValue); 1417 // Make the DBG_VALUE direct. 1418 MI.getDebugOffset().ChangeToRegister(0, false); 1419 } 1420 DIExpr = TRI.prependOffsetExpression(DIExpr, PrependFlags, Offset); 1421 } else { 1422 // The debug operand at DebugOpIndex was a frame index at offset 1423 // `Offset`; now the operand has been replaced with the frame 1424 // register, we must add Offset with `register x, plus Offset`. 1425 unsigned DebugOpIndex = MI.getDebugOperandIndex(&Op); 1426 SmallVector<uint64_t, 3> Ops; 1427 TRI.getOffsetOpcodes(Offset, Ops); 1428 DIExpr = DIExpression::appendOpsToArg(DIExpr, Ops, DebugOpIndex); 1429 } 1430 MI.getDebugExpressionOp().setMetadata(DIExpr); 1431 return true; 1432 } 1433 1434 if (MI.isDebugPHI()) { 1435 // Allow stack ref to continue onwards. 1436 return true; 1437 } 1438 1439 // TODO: This code should be commoned with the code for 1440 // PATCHPOINT. There's no good reason for the difference in 1441 // implementation other than historical accident. The only 1442 // remaining difference is the unconditional use of the stack 1443 // pointer as the base register. 1444 if (MI.getOpcode() == TargetOpcode::STATEPOINT) { 1445 assert((!MI.isDebugValue() || OpIdx == 0) && 1446 "Frame indicies can only appear as the first operand of a " 1447 "DBG_VALUE machine instruction"); 1448 Register Reg; 1449 MachineOperand &Offset = MI.getOperand(OpIdx + 1); 1450 StackOffset refOffset = TFI->getFrameIndexReferencePreferSP( 1451 MF, MI.getOperand(OpIdx).getIndex(), Reg, /*IgnoreSPUpdates*/ false); 1452 assert(!refOffset.getScalable() && 1453 "Frame offsets with a scalable component are not supported"); 1454 Offset.setImm(Offset.getImm() + refOffset.getFixed() + SPAdj); 1455 MI.getOperand(OpIdx).ChangeToRegister(Reg, false /*isDef*/); 1456 return true; 1457 } 1458 return false; 1459 } 1460 1461 void PEI::replaceFrameIndicesBackward(MachineBasicBlock *BB, 1462 MachineFunction &MF, int &SPAdj) { 1463 assert(MF.getSubtarget().getRegisterInfo() && 1464 "getRegisterInfo() must be implemented!"); 1465 1466 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 1467 1468 RS->enterBasicBlockEnd(*BB); 1469 1470 for (MachineInstr &MI : make_early_inc_range(reverse(*BB))) { 1471 1472 // Register scavenger backward step 1473 MachineBasicBlock::iterator Step(MI); 1474 for (unsigned i = 0; i != MI.getNumOperands(); ++i) { 1475 if (!MI.getOperand(i).isFI()) 1476 continue; 1477 1478 if (replaceFrameIndexDebugInstr(MF, MI, i, SPAdj)) 1479 continue; 1480 1481 // If this instruction has a FrameIndex operand, we need to 1482 // use that target machine register info object to eliminate 1483 // it. 1484 1485 // TRI.eliminateFrameIndex may lower the frame index to a sequence of 1486 // instructions. It also can remove/change instructions passed by the 1487 // iterator and invalidate the iterator. We have to take care of this. For 1488 // that we support two iterators: *Step* - points to the position up to 1489 // which the scavenger should scan by the next iteration to have liveness 1490 // information up to date. *Curr* - keeps track of the correct RS->MBBI - 1491 // the scan start point. It points to the currently processed instruction 1492 // right before the frame lowering. 1493 // 1494 // ITERATORS WORK AS FOLLOWS: 1495 // *Step* is shifted one step back right before the frame lowering and 1496 // one step forward right after it. No matter how many instructions were 1497 // inserted, *Step* will be right after the position which is going to be 1498 // processed in the next iteration, thus, in the correct position for the 1499 // scavenger to go up to. 1500 // *Curr* is shifted one step forward right before calling 1501 // TRI.eliminateFrameIndex and one step backward after. Thus, we make sure 1502 // it points right to the position that is the correct starting point for 1503 // the scavenger to scan. 1504 MachineBasicBlock::iterator Curr = ++RS->getCurrentPosition(); 1505 1506 // Shift back 1507 --Step; 1508 1509 bool Removed = TRI.eliminateFrameIndex(MI, SPAdj, i, RS); 1510 // Restore to unify logic with a shift back that happens in the end of 1511 // the outer loop. 1512 ++Step; 1513 RS->skipTo(--Curr); 1514 if (Removed) 1515 break; 1516 } 1517 1518 // Shift it to make RS collect reg info up to the current instruction. 1519 if (Step != BB->begin()) 1520 Step--; 1521 1522 // Update register states. 1523 RS->backward(Step); 1524 } 1525 } 1526 1527 void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF, 1528 int &SPAdj) { 1529 assert(MF.getSubtarget().getRegisterInfo() && 1530 "getRegisterInfo() must be implemented!"); 1531 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); 1532 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 1533 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); 1534 1535 if (RS && TRI.supportsBackwardScavenger()) 1536 return replaceFrameIndicesBackward(BB, MF, SPAdj); 1537 1538 if (RS && FrameIndexEliminationScavenging) 1539 RS->enterBasicBlock(*BB); 1540 1541 bool InsideCallSequence = false; 1542 1543 for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) { 1544 if (TII.isFrameInstr(*I)) { 1545 InsideCallSequence = TII.isFrameSetup(*I); 1546 SPAdj += TII.getSPAdjust(*I); 1547 I = TFI->eliminateCallFramePseudoInstr(MF, *BB, I); 1548 continue; 1549 } 1550 1551 MachineInstr &MI = *I; 1552 bool DoIncr = true; 1553 bool DidFinishLoop = true; 1554 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 1555 if (!MI.getOperand(i).isFI()) 1556 continue; 1557 1558 if (replaceFrameIndexDebugInstr(MF, MI, i, SPAdj)) 1559 continue; 1560 1561 // Some instructions (e.g. inline asm instructions) can have 1562 // multiple frame indices and/or cause eliminateFrameIndex 1563 // to insert more than one instruction. We need the register 1564 // scavenger to go through all of these instructions so that 1565 // it can update its register information. We keep the 1566 // iterator at the point before insertion so that we can 1567 // revisit them in full. 1568 bool AtBeginning = (I == BB->begin()); 1569 if (!AtBeginning) --I; 1570 1571 // If this instruction has a FrameIndex operand, we need to 1572 // use that target machine register info object to eliminate 1573 // it. 1574 TRI.eliminateFrameIndex(MI, SPAdj, i, 1575 FrameIndexEliminationScavenging ? RS : nullptr); 1576 1577 // Reset the iterator if we were at the beginning of the BB. 1578 if (AtBeginning) { 1579 I = BB->begin(); 1580 DoIncr = false; 1581 } 1582 1583 DidFinishLoop = false; 1584 break; 1585 } 1586 1587 // If we are looking at a call sequence, we need to keep track of 1588 // the SP adjustment made by each instruction in the sequence. 1589 // This includes both the frame setup/destroy pseudos (handled above), 1590 // as well as other instructions that have side effects w.r.t the SP. 1591 // Note that this must come after eliminateFrameIndex, because 1592 // if I itself referred to a frame index, we shouldn't count its own 1593 // adjustment. 1594 if (DidFinishLoop && InsideCallSequence) 1595 SPAdj += TII.getSPAdjust(MI); 1596 1597 if (DoIncr && I != BB->end()) ++I; 1598 1599 // Update register states. 1600 if (RS && FrameIndexEliminationScavenging && DidFinishLoop) 1601 RS->forward(MI); 1602 } 1603 } 1604