1 //===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the pass which inserts x86 AVX vzeroupper instructions
10 // before calls to SSE encoded functions. This avoids transition latency
11 // penalty when transferring control between AVX encoded instructions and old
12 // SSE encoding mode.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "X86.h"
17 #include "X86InstrInfo.h"
18 #include "X86Subtarget.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/Statistic.h"
21 #include "llvm/CodeGen/MachineBasicBlock.h"
22 #include "llvm/CodeGen/MachineFunction.h"
23 #include "llvm/CodeGen/MachineFunctionPass.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineOperand.h"
27 #include "llvm/CodeGen/MachineRegisterInfo.h"
28 #include "llvm/CodeGen/TargetInstrInfo.h"
29 #include "llvm/CodeGen/TargetRegisterInfo.h"
30 #include "llvm/IR/CallingConv.h"
31 #include "llvm/IR/DebugLoc.h"
32 #include "llvm/IR/Function.h"
33 #include "llvm/Support/Debug.h"
34 #include "llvm/Support/ErrorHandling.h"
35 #include "llvm/Support/raw_ostream.h"
36 #include <cassert>
37 
38 using namespace llvm;
39 
40 #define DEBUG_TYPE "x86-vzeroupper"
41 
42 static cl::opt<bool>
43 UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
44   cl::desc("Minimize AVX to SSE transition penalty"),
45   cl::init(true));
46 
47 STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
48 
49 namespace {
50 
51   class VZeroUpperInserter : public MachineFunctionPass {
52   public:
VZeroUpperInserter()53     VZeroUpperInserter() : MachineFunctionPass(ID) {}
54 
55     bool runOnMachineFunction(MachineFunction &MF) override;
56 
getRequiredProperties() const57     MachineFunctionProperties getRequiredProperties() const override {
58       return MachineFunctionProperties().set(
59           MachineFunctionProperties::Property::NoVRegs);
60     }
61 
getPassName() const62     StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
63 
64   private:
65     void processBasicBlock(MachineBasicBlock &MBB);
66     void insertVZeroUpper(MachineBasicBlock::iterator I,
67                           MachineBasicBlock &MBB);
68     void addDirtySuccessor(MachineBasicBlock &MBB);
69 
70     using BlockExitState = enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
71 
72     static const char* getBlockExitStateName(BlockExitState ST);
73 
74     // Core algorithm state:
75     // BlockState - Each block is either:
76     //   - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
77     //                   vzeroupper instructions in this block.
78     //   - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
79     //                  block that will ensure that YMM/ZMM is clean on exit.
80     //   - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
81     //                  subsequent vzeroupper in the block clears it.
82     //
83     // AddedToDirtySuccessors - This flag is raised when a block is added to the
84     //                          DirtySuccessors list to ensure that it's not
85     //                          added multiple times.
86     //
87     // FirstUnguardedCall - Records the location of the first unguarded call in
88     //                      each basic block that may need to be guarded by a
89     //                      vzeroupper. We won't know whether it actually needs
90     //                      to be guarded until we discover a predecessor that
91     //                      is DIRTY_OUT.
92     struct BlockState {
93       BlockExitState ExitState = PASS_THROUGH;
94       bool AddedToDirtySuccessors = false;
95       MachineBasicBlock::iterator FirstUnguardedCall;
96 
97       BlockState() = default;
98     };
99 
100     using BlockStateMap = SmallVector<BlockState, 8>;
101     using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
102 
103     BlockStateMap BlockStates;
104     DirtySuccessorsWorkList DirtySuccessors;
105     bool EverMadeChange;
106     bool IsX86INTR;
107     const TargetInstrInfo *TII;
108 
109     static char ID;
110   };
111 
112 } // end anonymous namespace
113 
114 char VZeroUpperInserter::ID = 0;
115 
createX86IssueVZeroUpperPass()116 FunctionPass *llvm::createX86IssueVZeroUpperPass() {
117   return new VZeroUpperInserter();
118 }
119 
120 #ifndef NDEBUG
getBlockExitStateName(BlockExitState ST)121 const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
122   switch (ST) {
123     case PASS_THROUGH: return "Pass-through";
124     case EXITS_DIRTY: return "Exits-dirty";
125     case EXITS_CLEAN: return "Exits-clean";
126   }
127   llvm_unreachable("Invalid block exit state.");
128 }
129 #endif
130 
131 /// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
132 /// Thus, there is no need to check for Y/ZMM16 and above.
isYmmOrZmmReg(unsigned Reg)133 static bool isYmmOrZmmReg(unsigned Reg) {
134   return (Reg >= X86::YMM0 && Reg <= X86::YMM15) ||
135          (Reg >= X86::ZMM0 && Reg <= X86::ZMM15);
136 }
137 
checkFnHasLiveInYmmOrZmm(MachineRegisterInfo & MRI)138 static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI) {
139   for (std::pair<unsigned, unsigned> LI : MRI.liveins())
140     if (isYmmOrZmmReg(LI.first))
141       return true;
142 
143   return false;
144 }
145 
clobbersAllYmmAndZmmRegs(const MachineOperand & MO)146 static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO) {
147   for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
148     if (!MO.clobbersPhysReg(reg))
149       return false;
150   }
151   for (unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
152     if (!MO.clobbersPhysReg(reg))
153       return false;
154   }
155   return true;
156 }
157 
hasYmmOrZmmReg(MachineInstr & MI)158 static bool hasYmmOrZmmReg(MachineInstr &MI) {
159   for (const MachineOperand &MO : MI.operands()) {
160     if (MI.isCall() && MO.isRegMask() && !clobbersAllYmmAndZmmRegs(MO))
161       return true;
162     if (!MO.isReg())
163       continue;
164     if (MO.isDebug())
165       continue;
166     if (isYmmOrZmmReg(MO.getReg()))
167       return true;
168   }
169   return false;
170 }
171 
172 /// Check if given call instruction has a RegMask operand.
callHasRegMask(MachineInstr & MI)173 static bool callHasRegMask(MachineInstr &MI) {
174   assert(MI.isCall() && "Can only be called on call instructions.");
175   for (const MachineOperand &MO : MI.operands()) {
176     if (MO.isRegMask())
177       return true;
178   }
179   return false;
180 }
181 
182 /// Insert a vzeroupper instruction before I.
insertVZeroUpper(MachineBasicBlock::iterator I,MachineBasicBlock & MBB)183 void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
184                                           MachineBasicBlock &MBB) {
185   BuildMI(MBB, I, I->getDebugLoc(), TII->get(X86::VZEROUPPER));
186   ++NumVZU;
187   EverMadeChange = true;
188 }
189 
190 /// Add MBB to the DirtySuccessors list if it hasn't already been added.
addDirtySuccessor(MachineBasicBlock & MBB)191 void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
192   if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
193     DirtySuccessors.push_back(&MBB);
194     BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
195   }
196 }
197 
198 /// Loop over all of the instructions in the basic block, inserting vzeroupper
199 /// instructions before function calls.
processBasicBlock(MachineBasicBlock & MBB)200 void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
201   // Start by assuming that the block is PASS_THROUGH which implies no unguarded
202   // calls.
203   BlockExitState CurState = PASS_THROUGH;
204   BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
205 
206   for (MachineInstr &MI : MBB) {
207     bool IsCall = MI.isCall();
208     bool IsReturn = MI.isReturn();
209     bool IsControlFlow = IsCall || IsReturn;
210 
211     // No need for vzeroupper before iret in interrupt handler function,
212     // epilogue will restore YMM/ZMM registers if needed.
213     if (IsX86INTR && IsReturn)
214       continue;
215 
216     // An existing VZERO* instruction resets the state.
217     if (MI.getOpcode() == X86::VZEROALL || MI.getOpcode() == X86::VZEROUPPER) {
218       CurState = EXITS_CLEAN;
219       continue;
220     }
221 
222     // Shortcut: don't need to check regular instructions in dirty state.
223     if (!IsControlFlow && CurState == EXITS_DIRTY)
224       continue;
225 
226     if (hasYmmOrZmmReg(MI)) {
227       // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
228       // instruction, or it could be control flow.
229       CurState = EXITS_DIRTY;
230       continue;
231     }
232 
233     // Check for control-flow out of the current function (which might
234     // indirectly execute SSE instructions).
235     if (!IsControlFlow)
236       continue;
237 
238     // If the call has no RegMask, skip it as well. It usually happens on
239     // helper function calls (such as '_chkstk', '_ftol2') where standard
240     // calling convention is not used (RegMask is not used to mark register
241     // clobbered and register usage (def/implicit-def/use) is well-defined and
242     // explicitly specified.
243     if (IsCall && !callHasRegMask(MI))
244       continue;
245 
246     // The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
247     // registers. In addition, the processor changes back to Clean state, after
248     // which execution of SSE instructions or AVX instructions has no transition
249     // penalty. Add the VZEROUPPER instruction before any function call/return
250     // that might execute SSE code.
251     // FIXME: In some cases, we may want to move the VZEROUPPER into a
252     // predecessor block.
253     if (CurState == EXITS_DIRTY) {
254       // After the inserted VZEROUPPER the state becomes clean again, but
255       // other YMM/ZMM may appear before other subsequent calls or even before
256       // the end of the BB.
257       insertVZeroUpper(MI, MBB);
258       CurState = EXITS_CLEAN;
259     } else if (CurState == PASS_THROUGH) {
260       // If this block is currently in pass-through state and we encounter a
261       // call then whether we need a vzeroupper or not depends on whether this
262       // block has successors that exit dirty. Record the location of the call,
263       // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
264       // It will be inserted later if necessary.
265       BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
266       CurState = EXITS_CLEAN;
267     }
268   }
269 
270   LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
271                     << getBlockExitStateName(CurState) << '\n');
272 
273   if (CurState == EXITS_DIRTY)
274     for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
275                                           SE = MBB.succ_end();
276          SI != SE; ++SI)
277       addDirtySuccessor(**SI);
278 
279   BlockStates[MBB.getNumber()].ExitState = CurState;
280 }
281 
282 /// Loop over all of the basic blocks, inserting vzeroupper instructions before
283 /// function calls.
runOnMachineFunction(MachineFunction & MF)284 bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
285   if (!UseVZeroUpper)
286     return false;
287 
288   const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
289   if (!ST.hasAVX() || !ST.insertVZEROUPPER())
290     return false;
291   TII = ST.getInstrInfo();
292   MachineRegisterInfo &MRI = MF.getRegInfo();
293   EverMadeChange = false;
294   IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
295 
296   bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
297 
298   // Fast check: if the function doesn't use any ymm/zmm registers, we don't
299   // need to insert any VZEROUPPER instructions.  This is constant-time, so it
300   // is cheap in the common case of no ymm/zmm use.
301   bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
302   for (auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
303     if (!YmmOrZmmUsed) {
304       for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end(); i != e;
305            i++) {
306         if (!MRI.reg_nodbg_empty(*i)) {
307           YmmOrZmmUsed = true;
308           break;
309         }
310       }
311     }
312   }
313   if (!YmmOrZmmUsed)
314     return false;
315 
316   assert(BlockStates.empty() && DirtySuccessors.empty() &&
317          "X86VZeroUpper state should be clear");
318   BlockStates.resize(MF.getNumBlockIDs());
319 
320   // Process all blocks. This will compute block exit states, record the first
321   // unguarded call in each block, and add successors of dirty blocks to the
322   // DirtySuccessors list.
323   for (MachineBasicBlock &MBB : MF)
324     processBasicBlock(MBB);
325 
326   // If any YMM/ZMM regs are live-in to this function, add the entry block to
327   // the DirtySuccessors list
328   if (FnHasLiveInYmmOrZmm)
329     addDirtySuccessor(MF.front());
330 
331   // Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
332   // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
333   // through PASS_THROUGH blocks.
334   while (!DirtySuccessors.empty()) {
335     MachineBasicBlock &MBB = *DirtySuccessors.back();
336     DirtySuccessors.pop_back();
337     BlockState &BBState = BlockStates[MBB.getNumber()];
338 
339     // MBB is a successor of a dirty block, so its first call needs to be
340     // guarded.
341     if (BBState.FirstUnguardedCall != MBB.end())
342       insertVZeroUpper(BBState.FirstUnguardedCall, MBB);
343 
344     // If this successor was a pass-through block, then it is now dirty. Its
345     // successors need to be added to the worklist (if they haven't been
346     // already).
347     if (BBState.ExitState == PASS_THROUGH) {
348       LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
349                         << " was Pass-through, is now Dirty-out.\n");
350       for (MachineBasicBlock *Succ : MBB.successors())
351         addDirtySuccessor(*Succ);
352     }
353   }
354 
355   BlockStates.clear();
356   return EverMadeChange;
357 }
358