1 //===----- X86AvoidTrailingCall.cpp - Insert int3 after trailing calls ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // The Windows x64 unwinder has trouble unwinding the stack when a return
10 // address points to the end of the function. This pass maintains the invariant
11 // that every return address is inside the bounds of its parent function or
12 // funclet by inserting int3 if the last instruction would otherwise be a call.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "X86.h"
17 #include "X86InstrInfo.h"
18 #include "X86Subtarget.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 
21 #define DEBUG_TYPE "x86-avoid-trailing-call"
22 
23 using namespace llvm;
24 
25 namespace {
26 
27 class X86AvoidTrailingCallPass : public MachineFunctionPass {
28 public:
29   X86AvoidTrailingCallPass() : MachineFunctionPass(ID) {}
30 
31   bool runOnMachineFunction(MachineFunction &MF) override;
32 
33 private:
34   StringRef getPassName() const override {
35     return "X86 avoid trailing call pass";
36   }
37   static char ID;
38 };
39 
40 char X86AvoidTrailingCallPass::ID = 0;
41 
42 } // end anonymous namespace
43 
44 FunctionPass *llvm::createX86AvoidTrailingCallPass() {
45   return new X86AvoidTrailingCallPass();
46 }
47 
48 // A real instruction is a non-meta, non-pseudo instruction.  Some pseudos
49 // expand to nothing, and some expand to code. This logic conservatively assumes
50 // they might expand to nothing.
51 static bool isRealInstruction(MachineInstr &MI) {
52   return !MI.isPseudo() && !MI.isMetaInstruction();
53 }
54 
55 // Return true if this is a call instruction, but not a tail call.
56 static bool isCallInstruction(const MachineInstr &MI) {
57   return MI.isCall() && !MI.isReturn();
58 }
59 
60 bool X86AvoidTrailingCallPass::runOnMachineFunction(MachineFunction &MF) {
61   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
62   const X86InstrInfo &TII = *STI.getInstrInfo();
63   assert(STI.isTargetWin64() && "pass only runs on Win64");
64 
65   // FIXME: Perhaps this pass should also replace SEH_Epilogue by inserting nops
66   // before epilogues.
67 
68   bool Changed = false;
69   for (MachineBasicBlock &MBB : MF) {
70     // Look for basic blocks that precede funclet entries or are at the end of
71     // the function.
72     MachineBasicBlock *NextMBB = MBB.getNextNode();
73     if (NextMBB && !NextMBB->isEHFuncletEntry())
74       continue;
75 
76     // Find the last real instruction in this block, or previous blocks if this
77     // block is empty.
78     MachineBasicBlock::reverse_iterator LastRealInstr;
79     for (MachineBasicBlock &RMBB :
80          make_range(MBB.getReverseIterator(), MF.rend())) {
81       LastRealInstr = llvm::find_if(reverse(RMBB), isRealInstruction);
82       if (LastRealInstr != RMBB.rend())
83         break;
84     }
85 
86     // Do nothing if this function or funclet has no instructions.
87     if (LastRealInstr == MF.begin()->rend())
88       continue;
89 
90     // If this is a call instruction, insert int3 right after it with the same
91     // DebugLoc. Convert back to a forward iterator and advance the insertion
92     // position once.
93     if (isCallInstruction(*LastRealInstr)) {
94       LLVM_DEBUG({
95         dbgs() << "inserting int3 after trailing call instruction:\n";
96         LastRealInstr->dump();
97         dbgs() << '\n';
98       });
99 
100       MachineBasicBlock::iterator MBBI = std::next(LastRealInstr.getReverse());
101       BuildMI(*LastRealInstr->getParent(), MBBI, LastRealInstr->getDebugLoc(),
102               TII.get(X86::INT3));
103       Changed = true;
104     }
105   }
106 
107   return Changed;
108 }
109