1 //===------- X86ExpandPseudo.cpp - Expand pseudo instructions -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that expands pseudo instructions into target
10 // instructions to allow proper scheduling, if-conversion, other late
11 // optimizations, or simply the encoding of the instructions.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "X86.h"
16 #include "X86FrameLowering.h"
17 #include "X86InstrBuilder.h"
18 #include "X86InstrInfo.h"
19 #include "X86MachineFunctionInfo.h"
20 #include "X86Subtarget.h"
21 #include "llvm/CodeGen/LivePhysRegs.h"
22 #include "llvm/CodeGen/MachineFunctionPass.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/Passes.h" // For IDs of passes that are preserved.
25 #include "llvm/IR/EHPersonalities.h"
26 #include "llvm/IR/GlobalValue.h"
27 #include "llvm/Target/TargetMachine.h"
28 using namespace llvm;
29 
30 #define DEBUG_TYPE "x86-pseudo"
31 #define X86_EXPAND_PSEUDO_NAME "X86 pseudo instruction expansion pass"
32 
33 namespace {
34 class X86ExpandPseudo : public MachineFunctionPass {
35 public:
36   static char ID;
X86ExpandPseudo()37   X86ExpandPseudo() : MachineFunctionPass(ID) {}
38 
getAnalysisUsage(AnalysisUsage & AU) const39   void getAnalysisUsage(AnalysisUsage &AU) const override {
40     AU.setPreservesCFG();
41     AU.addPreservedID(MachineLoopInfoID);
42     AU.addPreservedID(MachineDominatorsID);
43     MachineFunctionPass::getAnalysisUsage(AU);
44   }
45 
46   const X86Subtarget *STI = nullptr;
47   const X86InstrInfo *TII = nullptr;
48   const X86RegisterInfo *TRI = nullptr;
49   const X86MachineFunctionInfo *X86FI = nullptr;
50   const X86FrameLowering *X86FL = nullptr;
51 
52   bool runOnMachineFunction(MachineFunction &MF) override;
53 
getRequiredProperties() const54   MachineFunctionProperties getRequiredProperties() const override {
55     return MachineFunctionProperties().set(
56         MachineFunctionProperties::Property::NoVRegs);
57   }
58 
getPassName() const59   StringRef getPassName() const override {
60     return "X86 pseudo instruction expansion pass";
61   }
62 
63 private:
64   void ExpandICallBranchFunnel(MachineBasicBlock *MBB,
65                                MachineBasicBlock::iterator MBBI);
66   void expandCALL_RVMARKER(MachineBasicBlock &MBB,
67                            MachineBasicBlock::iterator MBBI);
68   bool ExpandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
69   bool ExpandMBB(MachineBasicBlock &MBB);
70 
71   /// This function expands pseudos which affects control flow.
72   /// It is done in separate pass to simplify blocks navigation in main
73   /// pass(calling ExpandMBB).
74   bool ExpandPseudosWhichAffectControlFlow(MachineFunction &MF);
75 
76   /// Expand X86::VASTART_SAVE_XMM_REGS into set of xmm copying instructions,
77   /// placed into separate block guarded by check for al register(for SystemV
78   /// abi).
79   void ExpandVastartSaveXmmRegs(
80       MachineBasicBlock *EntryBlk,
81       MachineBasicBlock::iterator VAStartPseudoInstr) const;
82 };
83 char X86ExpandPseudo::ID = 0;
84 
85 } // End anonymous namespace.
86 
INITIALIZE_PASS(X86ExpandPseudo,DEBUG_TYPE,X86_EXPAND_PSEUDO_NAME,false,false)87 INITIALIZE_PASS(X86ExpandPseudo, DEBUG_TYPE, X86_EXPAND_PSEUDO_NAME, false,
88                 false)
89 
90 void X86ExpandPseudo::ExpandICallBranchFunnel(
91     MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) {
92   MachineBasicBlock *JTMBB = MBB;
93   MachineInstr *JTInst = &*MBBI;
94   MachineFunction *MF = MBB->getParent();
95   const BasicBlock *BB = MBB->getBasicBlock();
96   auto InsPt = MachineFunction::iterator(MBB);
97   ++InsPt;
98 
99   std::vector<std::pair<MachineBasicBlock *, unsigned>> TargetMBBs;
100   const DebugLoc &DL = JTInst->getDebugLoc();
101   MachineOperand Selector = JTInst->getOperand(0);
102   const GlobalValue *CombinedGlobal = JTInst->getOperand(1).getGlobal();
103 
104   auto CmpTarget = [&](unsigned Target) {
105     if (Selector.isReg())
106       MBB->addLiveIn(Selector.getReg());
107     BuildMI(*MBB, MBBI, DL, TII->get(X86::LEA64r), X86::R11)
108         .addReg(X86::RIP)
109         .addImm(1)
110         .addReg(0)
111         .addGlobalAddress(CombinedGlobal,
112                           JTInst->getOperand(2 + 2 * Target).getImm())
113         .addReg(0);
114     BuildMI(*MBB, MBBI, DL, TII->get(X86::CMP64rr))
115         .add(Selector)
116         .addReg(X86::R11);
117   };
118 
119   auto CreateMBB = [&]() {
120     auto *NewMBB = MF->CreateMachineBasicBlock(BB);
121     MBB->addSuccessor(NewMBB);
122     if (!MBB->isLiveIn(X86::EFLAGS))
123       MBB->addLiveIn(X86::EFLAGS);
124     return NewMBB;
125   };
126 
127   auto EmitCondJump = [&](unsigned CC, MachineBasicBlock *ThenMBB) {
128     BuildMI(*MBB, MBBI, DL, TII->get(X86::JCC_1)).addMBB(ThenMBB).addImm(CC);
129 
130     auto *ElseMBB = CreateMBB();
131     MF->insert(InsPt, ElseMBB);
132     MBB = ElseMBB;
133     MBBI = MBB->end();
134   };
135 
136   auto EmitCondJumpTarget = [&](unsigned CC, unsigned Target) {
137     auto *ThenMBB = CreateMBB();
138     TargetMBBs.push_back({ThenMBB, Target});
139     EmitCondJump(CC, ThenMBB);
140   };
141 
142   auto EmitTailCall = [&](unsigned Target) {
143     BuildMI(*MBB, MBBI, DL, TII->get(X86::TAILJMPd64))
144         .add(JTInst->getOperand(3 + 2 * Target));
145   };
146 
147   std::function<void(unsigned, unsigned)> EmitBranchFunnel =
148       [&](unsigned FirstTarget, unsigned NumTargets) {
149     if (NumTargets == 1) {
150       EmitTailCall(FirstTarget);
151       return;
152     }
153 
154     if (NumTargets == 2) {
155       CmpTarget(FirstTarget + 1);
156       EmitCondJumpTarget(X86::COND_B, FirstTarget);
157       EmitTailCall(FirstTarget + 1);
158       return;
159     }
160 
161     if (NumTargets < 6) {
162       CmpTarget(FirstTarget + 1);
163       EmitCondJumpTarget(X86::COND_B, FirstTarget);
164       EmitCondJumpTarget(X86::COND_E, FirstTarget + 1);
165       EmitBranchFunnel(FirstTarget + 2, NumTargets - 2);
166       return;
167     }
168 
169     auto *ThenMBB = CreateMBB();
170     CmpTarget(FirstTarget + (NumTargets / 2));
171     EmitCondJump(X86::COND_B, ThenMBB);
172     EmitCondJumpTarget(X86::COND_E, FirstTarget + (NumTargets / 2));
173     EmitBranchFunnel(FirstTarget + (NumTargets / 2) + 1,
174                   NumTargets - (NumTargets / 2) - 1);
175 
176     MF->insert(InsPt, ThenMBB);
177     MBB = ThenMBB;
178     MBBI = MBB->end();
179     EmitBranchFunnel(FirstTarget, NumTargets / 2);
180   };
181 
182   EmitBranchFunnel(0, (JTInst->getNumOperands() - 2) / 2);
183   for (auto P : TargetMBBs) {
184     MF->insert(InsPt, P.first);
185     BuildMI(P.first, DL, TII->get(X86::TAILJMPd64))
186         .add(JTInst->getOperand(3 + 2 * P.second));
187   }
188   JTMBB->erase(JTInst);
189 }
190 
expandCALL_RVMARKER(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI)191 void X86ExpandPseudo::expandCALL_RVMARKER(MachineBasicBlock &MBB,
192                                           MachineBasicBlock::iterator MBBI) {
193   // Expand CALL_RVMARKER pseudo to call instruction, followed by the special
194   //"movq %rax, %rdi" marker.
195   MachineInstr &MI = *MBBI;
196 
197   MachineInstr *OriginalCall;
198   assert((MI.getOperand(1).isGlobal() || MI.getOperand(1).isReg()) &&
199          "invalid operand for regular call");
200   unsigned Opc = -1;
201   if (MI.getOpcode() == X86::CALL64m_RVMARKER)
202     Opc = X86::CALL64m;
203   else if (MI.getOpcode() == X86::CALL64r_RVMARKER)
204     Opc = X86::CALL64r;
205   else if (MI.getOpcode() == X86::CALL64pcrel32_RVMARKER)
206     Opc = X86::CALL64pcrel32;
207   else
208     llvm_unreachable("unexpected opcode");
209 
210   OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr();
211   bool RAXImplicitDead = false;
212   for (MachineOperand &Op : llvm::drop_begin(MI.operands())) {
213     // RAX may be 'implicit dead', if there are no other users of the return
214     // value. We introduce a new use, so change it to 'implicit def'.
215     if (Op.isReg() && Op.isImplicit() && Op.isDead() &&
216         TRI->regsOverlap(Op.getReg(), X86::RAX)) {
217       Op.setIsDead(false);
218       Op.setIsDef(true);
219       RAXImplicitDead = true;
220     }
221     OriginalCall->addOperand(Op);
222   }
223 
224   // Emit marker "movq %rax, %rdi".  %rdi is not callee-saved, so it cannot be
225   // live across the earlier call. The call to the ObjC runtime function returns
226   // the first argument, so the value of %rax is unchanged after the ObjC
227   // runtime call. On Windows targets, the runtime call follows the regular
228   // x64 calling convention and expects the first argument in %rcx.
229   auto TargetReg = STI->getTargetTriple().isOSWindows() ? X86::RCX : X86::RDI;
230   auto *Marker = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::MOV64rr))
231                      .addReg(TargetReg, RegState::Define)
232                      .addReg(X86::RAX)
233                      .getInstr();
234   if (MI.shouldUpdateCallSiteInfo())
235     MBB.getParent()->moveCallSiteInfo(&MI, Marker);
236 
237   // Emit call to ObjC runtime.
238   const uint32_t *RegMask =
239       TRI->getCallPreservedMask(*MBB.getParent(), CallingConv::C);
240   MachineInstr *RtCall =
241       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32))
242           .addGlobalAddress(MI.getOperand(0).getGlobal(), 0, 0)
243           .addRegMask(RegMask)
244           .addReg(X86::RAX,
245                   RegState::Implicit |
246                       (RAXImplicitDead ? (RegState::Dead | RegState::Define)
247                                        : RegState::Define))
248           .getInstr();
249   MI.eraseFromParent();
250 
251   auto &TM = MBB.getParent()->getTarget();
252   // On Darwin platforms, wrap the expanded sequence in a bundle to prevent
253   // later optimizations from breaking up the sequence.
254   if (TM.getTargetTriple().isOSDarwin())
255     finalizeBundle(MBB, OriginalCall->getIterator(),
256                    std::next(RtCall->getIterator()));
257 }
258 
259 /// If \p MBBI is a pseudo instruction, this method expands
260 /// it to the corresponding (sequence of) actual instruction(s).
261 /// \returns true if \p MBBI has been expanded.
ExpandMI(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI)262 bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
263                                MachineBasicBlock::iterator MBBI) {
264   MachineInstr &MI = *MBBI;
265   unsigned Opcode = MI.getOpcode();
266   const DebugLoc &DL = MBBI->getDebugLoc();
267   bool HasEGPR = STI->hasEGPR();
268   switch (Opcode) {
269   default:
270     return false;
271   case X86::TCRETURNdi:
272   case X86::TCRETURNdicc:
273   case X86::TCRETURNri:
274   case X86::TCRETURNmi:
275   case X86::TCRETURNdi64:
276   case X86::TCRETURNdi64cc:
277   case X86::TCRETURNri64:
278   case X86::TCRETURNmi64: {
279     bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64;
280     MachineOperand &JumpTarget = MBBI->getOperand(0);
281     MachineOperand &StackAdjust = MBBI->getOperand(isMem ? X86::AddrNumOperands
282                                                          : 1);
283     assert(StackAdjust.isImm() && "Expecting immediate value.");
284 
285     // Adjust stack pointer.
286     int StackAdj = StackAdjust.getImm();
287     int MaxTCDelta = X86FI->getTCReturnAddrDelta();
288     int Offset = 0;
289     assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");
290 
291     // Incoporate the retaddr area.
292     Offset = StackAdj - MaxTCDelta;
293     assert(Offset >= 0 && "Offset should never be negative");
294 
295     if (Opcode == X86::TCRETURNdicc || Opcode == X86::TCRETURNdi64cc) {
296       assert(Offset == 0 && "Conditional tail call cannot adjust the stack.");
297     }
298 
299     if (Offset) {
300       // Check for possible merge with preceding ADD instruction.
301       Offset += X86FL->mergeSPUpdates(MBB, MBBI, true);
302       X86FL->emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue=*/true);
303     }
304 
305     // Jump to label or value in register.
306     bool IsWin64 = STI->isTargetWin64();
307     if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdicc ||
308         Opcode == X86::TCRETURNdi64 || Opcode == X86::TCRETURNdi64cc) {
309       unsigned Op;
310       switch (Opcode) {
311       case X86::TCRETURNdi:
312         Op = X86::TAILJMPd;
313         break;
314       case X86::TCRETURNdicc:
315         Op = X86::TAILJMPd_CC;
316         break;
317       case X86::TCRETURNdi64cc:
318         assert(!MBB.getParent()->hasWinCFI() &&
319                "Conditional tail calls confuse "
320                "the Win64 unwinder.");
321         Op = X86::TAILJMPd64_CC;
322         break;
323       default:
324         // Note: Win64 uses REX prefixes indirect jumps out of functions, but
325         // not direct ones.
326         Op = X86::TAILJMPd64;
327         break;
328       }
329       MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
330       if (JumpTarget.isGlobal()) {
331         MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
332                              JumpTarget.getTargetFlags());
333       } else {
334         assert(JumpTarget.isSymbol());
335         MIB.addExternalSymbol(JumpTarget.getSymbolName(),
336                               JumpTarget.getTargetFlags());
337       }
338       if (Op == X86::TAILJMPd_CC || Op == X86::TAILJMPd64_CC) {
339         MIB.addImm(MBBI->getOperand(2).getImm());
340       }
341 
342     } else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64) {
343       unsigned Op = (Opcode == X86::TCRETURNmi)
344                         ? X86::TAILJMPm
345                         : (IsWin64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
346       MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
347       for (unsigned i = 0; i != X86::AddrNumOperands; ++i)
348         MIB.add(MBBI->getOperand(i));
349     } else if (Opcode == X86::TCRETURNri64) {
350       JumpTarget.setIsKill();
351       BuildMI(MBB, MBBI, DL,
352               TII->get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
353           .add(JumpTarget);
354     } else {
355       JumpTarget.setIsKill();
356       BuildMI(MBB, MBBI, DL, TII->get(X86::TAILJMPr))
357           .add(JumpTarget);
358     }
359 
360     MachineInstr &NewMI = *std::prev(MBBI);
361     NewMI.copyImplicitOps(*MBBI->getParent()->getParent(), *MBBI);
362     NewMI.setCFIType(*MBB.getParent(), MI.getCFIType());
363 
364     // Update the call site info.
365     if (MBBI->isCandidateForCallSiteEntry())
366       MBB.getParent()->moveCallSiteInfo(&*MBBI, &NewMI);
367 
368     // Delete the pseudo instruction TCRETURN.
369     MBB.erase(MBBI);
370 
371     return true;
372   }
373   case X86::EH_RETURN:
374   case X86::EH_RETURN64: {
375     MachineOperand &DestAddr = MBBI->getOperand(0);
376     assert(DestAddr.isReg() && "Offset should be in register!");
377     const bool Uses64BitFramePtr =
378         STI->isTarget64BitLP64() || STI->isTargetNaCl64();
379     Register StackPtr = TRI->getStackRegister();
380     BuildMI(MBB, MBBI, DL,
381             TII->get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), StackPtr)
382         .addReg(DestAddr.getReg());
383     // The EH_RETURN pseudo is really removed during the MC Lowering.
384     return true;
385   }
386   case X86::IRET: {
387     // Adjust stack to erase error code
388     int64_t StackAdj = MBBI->getOperand(0).getImm();
389     X86FL->emitSPUpdate(MBB, MBBI, DL, StackAdj, true);
390     // Replace pseudo with machine iret
391     unsigned RetOp = STI->is64Bit() ? X86::IRET64 : X86::IRET32;
392     // Use UIRET if UINTR is present (except for building kernel)
393     if (STI->is64Bit() && STI->hasUINTR() &&
394         MBB.getParent()->getTarget().getCodeModel() != CodeModel::Kernel)
395       RetOp = X86::UIRET;
396     BuildMI(MBB, MBBI, DL, TII->get(RetOp));
397     MBB.erase(MBBI);
398     return true;
399   }
400   case X86::RET: {
401     // Adjust stack to erase error code
402     int64_t StackAdj = MBBI->getOperand(0).getImm();
403     MachineInstrBuilder MIB;
404     if (StackAdj == 0) {
405       MIB = BuildMI(MBB, MBBI, DL,
406                     TII->get(STI->is64Bit() ? X86::RET64 : X86::RET32));
407     } else if (isUInt<16>(StackAdj)) {
408       MIB = BuildMI(MBB, MBBI, DL,
409                     TII->get(STI->is64Bit() ? X86::RETI64 : X86::RETI32))
410                 .addImm(StackAdj);
411     } else {
412       assert(!STI->is64Bit() &&
413              "shouldn't need to do this for x86_64 targets!");
414       // A ret can only handle immediates as big as 2**16-1.  If we need to pop
415       // off bytes before the return address, we must do it manually.
416       BuildMI(MBB, MBBI, DL, TII->get(X86::POP32r)).addReg(X86::ECX, RegState::Define);
417       X86FL->emitSPUpdate(MBB, MBBI, DL, StackAdj, /*InEpilogue=*/true);
418       BuildMI(MBB, MBBI, DL, TII->get(X86::PUSH32r)).addReg(X86::ECX);
419       MIB = BuildMI(MBB, MBBI, DL, TII->get(X86::RET32));
420     }
421     for (unsigned I = 1, E = MBBI->getNumOperands(); I != E; ++I)
422       MIB.add(MBBI->getOperand(I));
423     MBB.erase(MBBI);
424     return true;
425   }
426   case X86::LCMPXCHG16B_SAVE_RBX: {
427     // Perform the following transformation.
428     // SaveRbx = pseudocmpxchg Addr, <4 opds for the address>, InArg, SaveRbx
429     // =>
430     // RBX = InArg
431     // actualcmpxchg Addr
432     // RBX = SaveRbx
433     const MachineOperand &InArg = MBBI->getOperand(6);
434     Register SaveRbx = MBBI->getOperand(7).getReg();
435 
436     // Copy the input argument of the pseudo into the argument of the
437     // actual instruction.
438     // NOTE: We don't copy the kill flag since the input might be the same reg
439     // as one of the other operands of LCMPXCHG16B.
440     TII->copyPhysReg(MBB, MBBI, DL, X86::RBX, InArg.getReg(), false);
441     // Create the actual instruction.
442     MachineInstr *NewInstr = BuildMI(MBB, MBBI, DL, TII->get(X86::LCMPXCHG16B));
443     // Copy the operands related to the address.
444     for (unsigned Idx = 1; Idx < 6; ++Idx)
445       NewInstr->addOperand(MBBI->getOperand(Idx));
446     // Finally, restore the value of RBX.
447     TII->copyPhysReg(MBB, MBBI, DL, X86::RBX, SaveRbx,
448                      /*SrcIsKill*/ true);
449 
450     // Delete the pseudo.
451     MBBI->eraseFromParent();
452     return true;
453   }
454   // Loading/storing mask pairs requires two kmov operations. The second one of
455   // these needs a 2 byte displacement relative to the specified address (with
456   // 32 bit spill size). The pairs of 1bit masks up to 16 bit masks all use the
457   // same spill size, they all are stored using MASKPAIR16STORE, loaded using
458   // MASKPAIR16LOAD.
459   //
460   // The displacement value might wrap around in theory, thus the asserts in
461   // both cases.
462   case X86::MASKPAIR16LOAD: {
463     int64_t Disp = MBBI->getOperand(1 + X86::AddrDisp).getImm();
464     assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement");
465     Register Reg = MBBI->getOperand(0).getReg();
466     bool DstIsDead = MBBI->getOperand(0).isDead();
467     Register Reg0 = TRI->getSubReg(Reg, X86::sub_mask_0);
468     Register Reg1 = TRI->getSubReg(Reg, X86::sub_mask_1);
469 
470     auto MIBLo =
471         BuildMI(MBB, MBBI, DL,
472                 TII->get(HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm))
473             .addReg(Reg0, RegState::Define | getDeadRegState(DstIsDead));
474     auto MIBHi =
475         BuildMI(MBB, MBBI, DL,
476                 TII->get(HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm))
477             .addReg(Reg1, RegState::Define | getDeadRegState(DstIsDead));
478 
479     for (int i = 0; i < X86::AddrNumOperands; ++i) {
480       MIBLo.add(MBBI->getOperand(1 + i));
481       if (i == X86::AddrDisp)
482         MIBHi.addImm(Disp + 2);
483       else
484         MIBHi.add(MBBI->getOperand(1 + i));
485     }
486 
487     // Split the memory operand, adjusting the offset and size for the halves.
488     MachineMemOperand *OldMMO = MBBI->memoperands().front();
489     MachineFunction *MF = MBB.getParent();
490     MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 2);
491     MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 2, 2);
492 
493     MIBLo.setMemRefs(MMOLo);
494     MIBHi.setMemRefs(MMOHi);
495 
496     // Delete the pseudo.
497     MBB.erase(MBBI);
498     return true;
499   }
500   case X86::MASKPAIR16STORE: {
501     int64_t Disp = MBBI->getOperand(X86::AddrDisp).getImm();
502     assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement");
503     Register Reg = MBBI->getOperand(X86::AddrNumOperands).getReg();
504     bool SrcIsKill = MBBI->getOperand(X86::AddrNumOperands).isKill();
505     Register Reg0 = TRI->getSubReg(Reg, X86::sub_mask_0);
506     Register Reg1 = TRI->getSubReg(Reg, X86::sub_mask_1);
507 
508     auto MIBLo = BuildMI(MBB, MBBI, DL,
509                          TII->get(HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk));
510     auto MIBHi = BuildMI(MBB, MBBI, DL,
511                          TII->get(HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk));
512 
513     for (int i = 0; i < X86::AddrNumOperands; ++i) {
514       MIBLo.add(MBBI->getOperand(i));
515       if (i == X86::AddrDisp)
516         MIBHi.addImm(Disp + 2);
517       else
518         MIBHi.add(MBBI->getOperand(i));
519     }
520     MIBLo.addReg(Reg0, getKillRegState(SrcIsKill));
521     MIBHi.addReg(Reg1, getKillRegState(SrcIsKill));
522 
523     // Split the memory operand, adjusting the offset and size for the halves.
524     MachineMemOperand *OldMMO = MBBI->memoperands().front();
525     MachineFunction *MF = MBB.getParent();
526     MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 2);
527     MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 2, 2);
528 
529     MIBLo.setMemRefs(MMOLo);
530     MIBHi.setMemRefs(MMOHi);
531 
532     // Delete the pseudo.
533     MBB.erase(MBBI);
534     return true;
535   }
536   case X86::MWAITX_SAVE_RBX: {
537     // Perform the following transformation.
538     // SaveRbx = pseudomwaitx InArg, SaveRbx
539     // =>
540     // [E|R]BX = InArg
541     // actualmwaitx
542     // [E|R]BX = SaveRbx
543     const MachineOperand &InArg = MBBI->getOperand(1);
544     // Copy the input argument of the pseudo into the argument of the
545     // actual instruction.
546     TII->copyPhysReg(MBB, MBBI, DL, X86::EBX, InArg.getReg(), InArg.isKill());
547     // Create the actual instruction.
548     BuildMI(MBB, MBBI, DL, TII->get(X86::MWAITXrrr));
549     // Finally, restore the value of RBX.
550     Register SaveRbx = MBBI->getOperand(2).getReg();
551     TII->copyPhysReg(MBB, MBBI, DL, X86::RBX, SaveRbx, /*SrcIsKill*/ true);
552     // Delete the pseudo.
553     MBBI->eraseFromParent();
554     return true;
555   }
556   case TargetOpcode::ICALL_BRANCH_FUNNEL:
557     ExpandICallBranchFunnel(&MBB, MBBI);
558     return true;
559 #define GET_EGPR_IF_ENABLED(OPC) (STI->hasEGPR() ? OPC##_EVEX : OPC)
560   case X86::PLDTILECFGV: {
561     MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::LDTILECFG)));
562     return true;
563   }
564   case X86::PTILELOADDV:
565   case X86::PTILELOADDT1V: {
566     for (unsigned i = 2; i > 0; --i)
567       MI.removeOperand(i);
568     unsigned Opc = Opcode == X86::PTILELOADDV
569                        ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
570                        : GET_EGPR_IF_ENABLED(X86::TILELOADDT1);
571     MI.setDesc(TII->get(Opc));
572     return true;
573   }
574   case X86::PTCMMIMFP16PSV:
575   case X86::PTCMMRLFP16PSV:
576   case X86::PTDPBSSDV:
577   case X86::PTDPBSUDV:
578   case X86::PTDPBUSDV:
579   case X86::PTDPBUUDV:
580   case X86::PTDPBF16PSV:
581   case X86::PTDPFP16PSV: {
582     MI.untieRegOperand(4);
583     for (unsigned i = 3; i > 0; --i)
584       MI.removeOperand(i);
585     unsigned Opc;
586     switch (Opcode) {
587     case X86::PTCMMIMFP16PSV:  Opc = X86::TCMMIMFP16PS; break;
588     case X86::PTCMMRLFP16PSV:  Opc = X86::TCMMRLFP16PS; break;
589     case X86::PTDPBSSDV:   Opc = X86::TDPBSSD; break;
590     case X86::PTDPBSUDV:   Opc = X86::TDPBSUD; break;
591     case X86::PTDPBUSDV:   Opc = X86::TDPBUSD; break;
592     case X86::PTDPBUUDV:   Opc = X86::TDPBUUD; break;
593     case X86::PTDPBF16PSV: Opc = X86::TDPBF16PS; break;
594     case X86::PTDPFP16PSV: Opc = X86::TDPFP16PS; break;
595     default: llvm_unreachable("Impossible Opcode!");
596     }
597     MI.setDesc(TII->get(Opc));
598     MI.tieOperands(0, 1);
599     return true;
600   }
601   case X86::PTILESTOREDV: {
602     for (int i = 1; i >= 0; --i)
603       MI.removeOperand(i);
604     MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::TILESTORED)));
605     return true;
606   }
607 #undef GET_EGPR_IF_ENABLED
608   case X86::PTILEZEROV: {
609     for (int i = 2; i > 0; --i) // Remove row, col
610       MI.removeOperand(i);
611     MI.setDesc(TII->get(X86::TILEZERO));
612     return true;
613   }
614   case X86::CALL64pcrel32_RVMARKER:
615   case X86::CALL64r_RVMARKER:
616   case X86::CALL64m_RVMARKER:
617     expandCALL_RVMARKER(MBB, MBBI);
618     return true;
619   }
620   llvm_unreachable("Previous switch has a fallthrough?");
621 }
622 
623 // This function creates additional block for storing varargs guarded
624 // registers. It adds check for %al into entry block, to skip
625 // GuardedRegsBlk if xmm registers should not be stored.
626 //
627 //     EntryBlk[VAStartPseudoInstr]     EntryBlk
628 //        |                              |     .
629 //        |                              |        .
630 //        |                              |   GuardedRegsBlk
631 //        |                      =>      |        .
632 //        |                              |     .
633 //        |                             TailBlk
634 //        |                              |
635 //        |                              |
636 //
ExpandVastartSaveXmmRegs(MachineBasicBlock * EntryBlk,MachineBasicBlock::iterator VAStartPseudoInstr) const637 void X86ExpandPseudo::ExpandVastartSaveXmmRegs(
638     MachineBasicBlock *EntryBlk,
639     MachineBasicBlock::iterator VAStartPseudoInstr) const {
640   assert(VAStartPseudoInstr->getOpcode() == X86::VASTART_SAVE_XMM_REGS);
641 
642   MachineFunction *Func = EntryBlk->getParent();
643   const TargetInstrInfo *TII = STI->getInstrInfo();
644   const DebugLoc &DL = VAStartPseudoInstr->getDebugLoc();
645   Register CountReg = VAStartPseudoInstr->getOperand(0).getReg();
646 
647   // Calculate liveins for newly created blocks.
648   LivePhysRegs LiveRegs(*STI->getRegisterInfo());
649   SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers;
650 
651   LiveRegs.addLiveIns(*EntryBlk);
652   for (MachineInstr &MI : EntryBlk->instrs()) {
653     if (MI.getOpcode() == VAStartPseudoInstr->getOpcode())
654       break;
655 
656     LiveRegs.stepForward(MI, Clobbers);
657   }
658 
659   // Create the new basic blocks. One block contains all the XMM stores,
660   // and another block is the final destination regardless of whether any
661   // stores were performed.
662   const BasicBlock *LLVMBlk = EntryBlk->getBasicBlock();
663   MachineFunction::iterator EntryBlkIter = ++EntryBlk->getIterator();
664   MachineBasicBlock *GuardedRegsBlk = Func->CreateMachineBasicBlock(LLVMBlk);
665   MachineBasicBlock *TailBlk = Func->CreateMachineBasicBlock(LLVMBlk);
666   Func->insert(EntryBlkIter, GuardedRegsBlk);
667   Func->insert(EntryBlkIter, TailBlk);
668 
669   // Transfer the remainder of EntryBlk and its successor edges to TailBlk.
670   TailBlk->splice(TailBlk->begin(), EntryBlk,
671                   std::next(MachineBasicBlock::iterator(VAStartPseudoInstr)),
672                   EntryBlk->end());
673   TailBlk->transferSuccessorsAndUpdatePHIs(EntryBlk);
674 
675   uint64_t FrameOffset = VAStartPseudoInstr->getOperand(4).getImm();
676   uint64_t VarArgsRegsOffset = VAStartPseudoInstr->getOperand(6).getImm();
677 
678   // TODO: add support for YMM and ZMM here.
679   unsigned MOVOpc = STI->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;
680 
681   // In the XMM save block, save all the XMM argument registers.
682   for (int64_t OpndIdx = 7, RegIdx = 0;
683        OpndIdx < VAStartPseudoInstr->getNumOperands() - 1;
684        OpndIdx++, RegIdx++) {
685     auto NewMI = BuildMI(GuardedRegsBlk, DL, TII->get(MOVOpc));
686     for (int i = 0; i < X86::AddrNumOperands; ++i) {
687       if (i == X86::AddrDisp)
688         NewMI.addImm(FrameOffset + VarArgsRegsOffset + RegIdx * 16);
689       else
690         NewMI.add(VAStartPseudoInstr->getOperand(i + 1));
691     }
692     NewMI.addReg(VAStartPseudoInstr->getOperand(OpndIdx).getReg());
693     assert(VAStartPseudoInstr->getOperand(OpndIdx).getReg().isPhysical());
694   }
695 
696   // The original block will now fall through to the GuardedRegsBlk.
697   EntryBlk->addSuccessor(GuardedRegsBlk);
698   // The GuardedRegsBlk will fall through to the TailBlk.
699   GuardedRegsBlk->addSuccessor(TailBlk);
700 
701   if (!STI->isCallingConvWin64(Func->getFunction().getCallingConv())) {
702     // If %al is 0, branch around the XMM save block.
703     BuildMI(EntryBlk, DL, TII->get(X86::TEST8rr))
704         .addReg(CountReg)
705         .addReg(CountReg);
706     BuildMI(EntryBlk, DL, TII->get(X86::JCC_1))
707         .addMBB(TailBlk)
708         .addImm(X86::COND_E);
709     EntryBlk->addSuccessor(TailBlk);
710   }
711 
712   // Add liveins to the created block.
713   addLiveIns(*GuardedRegsBlk, LiveRegs);
714   addLiveIns(*TailBlk, LiveRegs);
715 
716   // Delete the pseudo.
717   VAStartPseudoInstr->eraseFromParent();
718 }
719 
720 /// Expand all pseudo instructions contained in \p MBB.
721 /// \returns true if any expansion occurred for \p MBB.
ExpandMBB(MachineBasicBlock & MBB)722 bool X86ExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
723   bool Modified = false;
724 
725   // MBBI may be invalidated by the expansion.
726   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
727   while (MBBI != E) {
728     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
729     Modified |= ExpandMI(MBB, MBBI);
730     MBBI = NMBBI;
731   }
732 
733   return Modified;
734 }
735 
ExpandPseudosWhichAffectControlFlow(MachineFunction & MF)736 bool X86ExpandPseudo::ExpandPseudosWhichAffectControlFlow(MachineFunction &MF) {
737   // Currently pseudo which affects control flow is only
738   // X86::VASTART_SAVE_XMM_REGS which is located in Entry block.
739   // So we do not need to evaluate other blocks.
740   for (MachineInstr &Instr : MF.front().instrs()) {
741     if (Instr.getOpcode() == X86::VASTART_SAVE_XMM_REGS) {
742       ExpandVastartSaveXmmRegs(&(MF.front()), Instr);
743       return true;
744     }
745   }
746 
747   return false;
748 }
749 
runOnMachineFunction(MachineFunction & MF)750 bool X86ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
751   STI = &MF.getSubtarget<X86Subtarget>();
752   TII = STI->getInstrInfo();
753   TRI = STI->getRegisterInfo();
754   X86FI = MF.getInfo<X86MachineFunctionInfo>();
755   X86FL = STI->getFrameLowering();
756 
757   bool Modified = ExpandPseudosWhichAffectControlFlow(MF);
758 
759   for (MachineBasicBlock &MBB : MF)
760     Modified |= ExpandMBB(MBB);
761   return Modified;
762 }
763 
764 /// Returns an instance of the pseudo instruction expansion pass.
createX86ExpandPseudoPass()765 FunctionPass *llvm::createX86ExpandPseudoPass() {
766   return new X86ExpandPseudo();
767 }
768