1 //===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the Base ARM implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "ARMBaseInstrInfo.h"
14 #include "ARMBaseRegisterInfo.h"
15 #include "ARMConstantPoolValue.h"
16 #include "ARMFeatures.h"
17 #include "ARMHazardRecognizer.h"
18 #include "ARMMachineFunctionInfo.h"
19 #include "ARMSubtarget.h"
20 #include "MCTargetDesc/ARMAddressingModes.h"
21 #include "MCTargetDesc/ARMBaseInfo.h"
22 #include "MVETailPredUtils.h"
23 #include "llvm/ADT/DenseMap.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/SmallSet.h"
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/Triple.h"
28 #include "llvm/CodeGen/LiveVariables.h"
29 #include "llvm/CodeGen/MachineBasicBlock.h"
30 #include "llvm/CodeGen/MachineConstantPool.h"
31 #include "llvm/CodeGen/MachineFrameInfo.h"
32 #include "llvm/CodeGen/MachineFunction.h"
33 #include "llvm/CodeGen/MachineInstr.h"
34 #include "llvm/CodeGen/MachineInstrBuilder.h"
35 #include "llvm/CodeGen/MachineMemOperand.h"
36 #include "llvm/CodeGen/MachineModuleInfo.h"
37 #include "llvm/CodeGen/MachineOperand.h"
38 #include "llvm/CodeGen/MachineRegisterInfo.h"
39 #include "llvm/CodeGen/MachineScheduler.h"
40 #include "llvm/CodeGen/MultiHazardRecognizer.h"
41 #include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
42 #include "llvm/CodeGen/SelectionDAGNodes.h"
43 #include "llvm/CodeGen/TargetInstrInfo.h"
44 #include "llvm/CodeGen/TargetRegisterInfo.h"
45 #include "llvm/CodeGen/TargetSchedule.h"
46 #include "llvm/IR/Attributes.h"
47 #include "llvm/IR/Constants.h"
48 #include "llvm/IR/DebugLoc.h"
49 #include "llvm/IR/Function.h"
50 #include "llvm/IR/GlobalValue.h"
51 #include "llvm/MC/MCAsmInfo.h"
52 #include "llvm/MC/MCInstrDesc.h"
53 #include "llvm/MC/MCInstrItineraries.h"
54 #include "llvm/Support/BranchProbability.h"
55 #include "llvm/Support/Casting.h"
56 #include "llvm/Support/CommandLine.h"
57 #include "llvm/Support/Compiler.h"
58 #include "llvm/Support/Debug.h"
59 #include "llvm/Support/ErrorHandling.h"
60 #include "llvm/Support/raw_ostream.h"
61 #include "llvm/Target/TargetMachine.h"
62 #include <algorithm>
63 #include <cassert>
64 #include <cstdint>
65 #include <iterator>
66 #include <new>
67 #include <utility>
68 #include <vector>
69 
70 using namespace llvm;
71 
72 #define DEBUG_TYPE "arm-instrinfo"
73 
74 #define GET_INSTRINFO_CTOR_DTOR
75 #include "ARMGenInstrInfo.inc"
76 
77 static cl::opt<bool>
78 EnableARM3Addr("enable-arm-3-addr-conv", cl::Hidden,
79                cl::desc("Enable ARM 2-addr to 3-addr conv"));
80 
81 /// ARM_MLxEntry - Record information about MLA / MLS instructions.
82 struct ARM_MLxEntry {
83   uint16_t MLxOpc;     // MLA / MLS opcode
84   uint16_t MulOpc;     // Expanded multiplication opcode
85   uint16_t AddSubOpc;  // Expanded add / sub opcode
86   bool NegAcc;         // True if the acc is negated before the add / sub.
87   bool HasLane;        // True if instruction has an extra "lane" operand.
88 };
89 
90 static const ARM_MLxEntry ARM_MLxTable[] = {
91   // MLxOpc,          MulOpc,           AddSubOpc,       NegAcc, HasLane
92   // fp scalar ops
93   { ARM::VMLAS,       ARM::VMULS,       ARM::VADDS,      false,  false },
94   { ARM::VMLSS,       ARM::VMULS,       ARM::VSUBS,      false,  false },
95   { ARM::VMLAD,       ARM::VMULD,       ARM::VADDD,      false,  false },
96   { ARM::VMLSD,       ARM::VMULD,       ARM::VSUBD,      false,  false },
97   { ARM::VNMLAS,      ARM::VNMULS,      ARM::VSUBS,      true,   false },
98   { ARM::VNMLSS,      ARM::VMULS,       ARM::VSUBS,      true,   false },
99   { ARM::VNMLAD,      ARM::VNMULD,      ARM::VSUBD,      true,   false },
100   { ARM::VNMLSD,      ARM::VMULD,       ARM::VSUBD,      true,   false },
101 
102   // fp SIMD ops
103   { ARM::VMLAfd,      ARM::VMULfd,      ARM::VADDfd,     false,  false },
104   { ARM::VMLSfd,      ARM::VMULfd,      ARM::VSUBfd,     false,  false },
105   { ARM::VMLAfq,      ARM::VMULfq,      ARM::VADDfq,     false,  false },
106   { ARM::VMLSfq,      ARM::VMULfq,      ARM::VSUBfq,     false,  false },
107   { ARM::VMLAslfd,    ARM::VMULslfd,    ARM::VADDfd,     false,  true  },
108   { ARM::VMLSslfd,    ARM::VMULslfd,    ARM::VSUBfd,     false,  true  },
109   { ARM::VMLAslfq,    ARM::VMULslfq,    ARM::VADDfq,     false,  true  },
110   { ARM::VMLSslfq,    ARM::VMULslfq,    ARM::VSUBfq,     false,  true  },
111 };
112 
ARMBaseInstrInfo(const ARMSubtarget & STI)113 ARMBaseInstrInfo::ARMBaseInstrInfo(const ARMSubtarget& STI)
114   : ARMGenInstrInfo(ARM::ADJCALLSTACKDOWN, ARM::ADJCALLSTACKUP),
115     Subtarget(STI) {
116   for (unsigned i = 0, e = array_lengthof(ARM_MLxTable); i != e; ++i) {
117     if (!MLxEntryMap.insert(std::make_pair(ARM_MLxTable[i].MLxOpc, i)).second)
118       llvm_unreachable("Duplicated entries?");
119     MLxHazardOpcodes.insert(ARM_MLxTable[i].AddSubOpc);
120     MLxHazardOpcodes.insert(ARM_MLxTable[i].MulOpc);
121   }
122 }
123 
124 // Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl
125 // currently defaults to no prepass hazard recognizer.
126 ScheduleHazardRecognizer *
CreateTargetHazardRecognizer(const TargetSubtargetInfo * STI,const ScheduleDAG * DAG) const127 ARMBaseInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
128                                                const ScheduleDAG *DAG) const {
129   if (usePreRAHazardRecognizer()) {
130     const InstrItineraryData *II =
131         static_cast<const ARMSubtarget *>(STI)->getInstrItineraryData();
132     return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched");
133   }
134   return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
135 }
136 
137 // Called during:
138 // - pre-RA scheduling
139 // - post-RA scheduling when FeatureUseMISched is set
CreateTargetMIHazardRecognizer(const InstrItineraryData * II,const ScheduleDAGMI * DAG) const140 ScheduleHazardRecognizer *ARMBaseInstrInfo::CreateTargetMIHazardRecognizer(
141     const InstrItineraryData *II, const ScheduleDAGMI *DAG) const {
142   MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
143 
144   // We would like to restrict this hazard recognizer to only
145   // post-RA scheduling; we can tell that we're post-RA because we don't
146   // track VRegLiveness.
147   // Cortex-M7: TRM indicates that there is a single ITCM bank and two DTCM
148   //            banks banked on bit 2.  Assume that TCMs are in use.
149   if (Subtarget.isCortexM7() && !DAG->hasVRegLiveness())
150     MHR->AddHazardRecognizer(
151         std::make_unique<ARMBankConflictHazardRecognizer>(DAG, 0x4, true));
152 
153   // Not inserting ARMHazardRecognizerFPMLx because that would change
154   // legacy behavior
155 
156   auto BHR = TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
157   MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
158   return MHR;
159 }
160 
161 // Called during post-RA scheduling when FeatureUseMISched is not set
162 ScheduleHazardRecognizer *ARMBaseInstrInfo::
CreateTargetPostRAHazardRecognizer(const InstrItineraryData * II,const ScheduleDAG * DAG) const163 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
164                                    const ScheduleDAG *DAG) const {
165   MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
166 
167   if (Subtarget.isThumb2() || Subtarget.hasVFP2Base())
168     MHR->AddHazardRecognizer(std::make_unique<ARMHazardRecognizerFPMLx>());
169 
170   auto BHR = TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG);
171   if (BHR)
172     MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
173   return MHR;
174 }
175 
convertToThreeAddress(MachineFunction::iterator & MFI,MachineInstr & MI,LiveVariables * LV) const176 MachineInstr *ARMBaseInstrInfo::convertToThreeAddress(
177     MachineFunction::iterator &MFI, MachineInstr &MI, LiveVariables *LV) const {
178   // FIXME: Thumb2 support.
179 
180   if (!EnableARM3Addr)
181     return nullptr;
182 
183   MachineFunction &MF = *MI.getParent()->getParent();
184   uint64_t TSFlags = MI.getDesc().TSFlags;
185   bool isPre = false;
186   switch ((TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift) {
187   default: return nullptr;
188   case ARMII::IndexModePre:
189     isPre = true;
190     break;
191   case ARMII::IndexModePost:
192     break;
193   }
194 
195   // Try splitting an indexed load/store to an un-indexed one plus an add/sub
196   // operation.
197   unsigned MemOpc = getUnindexedOpcode(MI.getOpcode());
198   if (MemOpc == 0)
199     return nullptr;
200 
201   MachineInstr *UpdateMI = nullptr;
202   MachineInstr *MemMI = nullptr;
203   unsigned AddrMode = (TSFlags & ARMII::AddrModeMask);
204   const MCInstrDesc &MCID = MI.getDesc();
205   unsigned NumOps = MCID.getNumOperands();
206   bool isLoad = !MI.mayStore();
207   const MachineOperand &WB = isLoad ? MI.getOperand(1) : MI.getOperand(0);
208   const MachineOperand &Base = MI.getOperand(2);
209   const MachineOperand &Offset = MI.getOperand(NumOps - 3);
210   Register WBReg = WB.getReg();
211   Register BaseReg = Base.getReg();
212   Register OffReg = Offset.getReg();
213   unsigned OffImm = MI.getOperand(NumOps - 2).getImm();
214   ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI.getOperand(NumOps - 1).getImm();
215   switch (AddrMode) {
216   default: llvm_unreachable("Unknown indexed op!");
217   case ARMII::AddrMode2: {
218     bool isSub = ARM_AM::getAM2Op(OffImm) == ARM_AM::sub;
219     unsigned Amt = ARM_AM::getAM2Offset(OffImm);
220     if (OffReg == 0) {
221       if (ARM_AM::getSOImmVal(Amt) == -1)
222         // Can't encode it in a so_imm operand. This transformation will
223         // add more than 1 instruction. Abandon!
224         return nullptr;
225       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
226                          get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
227                      .addReg(BaseReg)
228                      .addImm(Amt)
229                      .add(predOps(Pred))
230                      .add(condCodeOp());
231     } else if (Amt != 0) {
232       ARM_AM::ShiftOpc ShOpc = ARM_AM::getAM2ShiftOpc(OffImm);
233       unsigned SOOpc = ARM_AM::getSORegOpc(ShOpc, Amt);
234       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
235                          get(isSub ? ARM::SUBrsi : ARM::ADDrsi), WBReg)
236                      .addReg(BaseReg)
237                      .addReg(OffReg)
238                      .addReg(0)
239                      .addImm(SOOpc)
240                      .add(predOps(Pred))
241                      .add(condCodeOp());
242     } else
243       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
244                          get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
245                      .addReg(BaseReg)
246                      .addReg(OffReg)
247                      .add(predOps(Pred))
248                      .add(condCodeOp());
249     break;
250   }
251   case ARMII::AddrMode3 : {
252     bool isSub = ARM_AM::getAM3Op(OffImm) == ARM_AM::sub;
253     unsigned Amt = ARM_AM::getAM3Offset(OffImm);
254     if (OffReg == 0)
255       // Immediate is 8-bits. It's guaranteed to fit in a so_imm operand.
256       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
257                          get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
258                      .addReg(BaseReg)
259                      .addImm(Amt)
260                      .add(predOps(Pred))
261                      .add(condCodeOp());
262     else
263       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
264                          get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
265                      .addReg(BaseReg)
266                      .addReg(OffReg)
267                      .add(predOps(Pred))
268                      .add(condCodeOp());
269     break;
270   }
271   }
272 
273   std::vector<MachineInstr*> NewMIs;
274   if (isPre) {
275     if (isLoad)
276       MemMI =
277           BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg())
278               .addReg(WBReg)
279               .addImm(0)
280               .addImm(Pred);
281     else
282       MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc))
283                   .addReg(MI.getOperand(1).getReg())
284                   .addReg(WBReg)
285                   .addReg(0)
286                   .addImm(0)
287                   .addImm(Pred);
288     NewMIs.push_back(MemMI);
289     NewMIs.push_back(UpdateMI);
290   } else {
291     if (isLoad)
292       MemMI =
293           BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg())
294               .addReg(BaseReg)
295               .addImm(0)
296               .addImm(Pred);
297     else
298       MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc))
299                   .addReg(MI.getOperand(1).getReg())
300                   .addReg(BaseReg)
301                   .addReg(0)
302                   .addImm(0)
303                   .addImm(Pred);
304     if (WB.isDead())
305       UpdateMI->getOperand(0).setIsDead();
306     NewMIs.push_back(UpdateMI);
307     NewMIs.push_back(MemMI);
308   }
309 
310   // Transfer LiveVariables states, kill / dead info.
311   if (LV) {
312     for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
313       MachineOperand &MO = MI.getOperand(i);
314       if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) {
315         Register Reg = MO.getReg();
316 
317         LiveVariables::VarInfo &VI = LV->getVarInfo(Reg);
318         if (MO.isDef()) {
319           MachineInstr *NewMI = (Reg == WBReg) ? UpdateMI : MemMI;
320           if (MO.isDead())
321             LV->addVirtualRegisterDead(Reg, *NewMI);
322         }
323         if (MO.isUse() && MO.isKill()) {
324           for (unsigned j = 0; j < 2; ++j) {
325             // Look at the two new MI's in reverse order.
326             MachineInstr *NewMI = NewMIs[j];
327             if (!NewMI->readsRegister(Reg))
328               continue;
329             LV->addVirtualRegisterKilled(Reg, *NewMI);
330             if (VI.removeKill(MI))
331               VI.Kills.push_back(NewMI);
332             break;
333           }
334         }
335       }
336     }
337   }
338 
339   MachineBasicBlock::iterator MBBI = MI.getIterator();
340   MFI->insert(MBBI, NewMIs[1]);
341   MFI->insert(MBBI, NewMIs[0]);
342   return NewMIs[0];
343 }
344 
345 // Branch analysis.
analyzeBranch(MachineBasicBlock & MBB,MachineBasicBlock * & TBB,MachineBasicBlock * & FBB,SmallVectorImpl<MachineOperand> & Cond,bool AllowModify) const346 bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
347                                      MachineBasicBlock *&TBB,
348                                      MachineBasicBlock *&FBB,
349                                      SmallVectorImpl<MachineOperand> &Cond,
350                                      bool AllowModify) const {
351   TBB = nullptr;
352   FBB = nullptr;
353 
354   MachineBasicBlock::instr_iterator I = MBB.instr_end();
355   if (I == MBB.instr_begin())
356     return false; // Empty blocks are easy.
357   --I;
358 
359   // Walk backwards from the end of the basic block until the branch is
360   // analyzed or we give up.
361   while (isPredicated(*I) || I->isTerminator() || I->isDebugValue()) {
362     // Flag to be raised on unanalyzeable instructions. This is useful in cases
363     // where we want to clean up on the end of the basic block before we bail
364     // out.
365     bool CantAnalyze = false;
366 
367     // Skip over DEBUG values, predicated nonterminators and speculation
368     // barrier terminators.
369     while (I->isDebugInstr() || !I->isTerminator() ||
370            isSpeculationBarrierEndBBOpcode(I->getOpcode()) ||
371            I->getOpcode() == ARM::t2DoLoopStartTP){
372       if (I == MBB.instr_begin())
373         return false;
374       --I;
375     }
376 
377     if (isIndirectBranchOpcode(I->getOpcode()) ||
378         isJumpTableBranchOpcode(I->getOpcode())) {
379       // Indirect branches and jump tables can't be analyzed, but we still want
380       // to clean up any instructions at the tail of the basic block.
381       CantAnalyze = true;
382     } else if (isUncondBranchOpcode(I->getOpcode())) {
383       TBB = I->getOperand(0).getMBB();
384     } else if (isCondBranchOpcode(I->getOpcode())) {
385       // Bail out if we encounter multiple conditional branches.
386       if (!Cond.empty())
387         return true;
388 
389       assert(!FBB && "FBB should have been null.");
390       FBB = TBB;
391       TBB = I->getOperand(0).getMBB();
392       Cond.push_back(I->getOperand(1));
393       Cond.push_back(I->getOperand(2));
394     } else if (I->isReturn()) {
395       // Returns can't be analyzed, but we should run cleanup.
396       CantAnalyze = true;
397     } else {
398       // We encountered other unrecognized terminator. Bail out immediately.
399       return true;
400     }
401 
402     // Cleanup code - to be run for unpredicated unconditional branches and
403     //                returns.
404     if (!isPredicated(*I) &&
405           (isUncondBranchOpcode(I->getOpcode()) ||
406            isIndirectBranchOpcode(I->getOpcode()) ||
407            isJumpTableBranchOpcode(I->getOpcode()) ||
408            I->isReturn())) {
409       // Forget any previous condition branch information - it no longer applies.
410       Cond.clear();
411       FBB = nullptr;
412 
413       // If we can modify the function, delete everything below this
414       // unconditional branch.
415       if (AllowModify) {
416         MachineBasicBlock::iterator DI = std::next(I);
417         while (DI != MBB.instr_end()) {
418           MachineInstr &InstToDelete = *DI;
419           ++DI;
420           // Speculation barriers must not be deleted.
421           if (isSpeculationBarrierEndBBOpcode(InstToDelete.getOpcode()))
422             continue;
423           InstToDelete.eraseFromParent();
424         }
425       }
426     }
427 
428     if (CantAnalyze) {
429       // We may not be able to analyze the block, but we could still have
430       // an unconditional branch as the last instruction in the block, which
431       // just branches to layout successor. If this is the case, then just
432       // remove it if we're allowed to make modifications.
433       if (AllowModify && !isPredicated(MBB.back()) &&
434           isUncondBranchOpcode(MBB.back().getOpcode()) &&
435           TBB && MBB.isLayoutSuccessor(TBB))
436         removeBranch(MBB);
437       return true;
438     }
439 
440     if (I == MBB.instr_begin())
441       return false;
442 
443     --I;
444   }
445 
446   // We made it past the terminators without bailing out - we must have
447   // analyzed this branch successfully.
448   return false;
449 }
450 
removeBranch(MachineBasicBlock & MBB,int * BytesRemoved) const451 unsigned ARMBaseInstrInfo::removeBranch(MachineBasicBlock &MBB,
452                                         int *BytesRemoved) const {
453   assert(!BytesRemoved && "code size not handled");
454 
455   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
456   if (I == MBB.end())
457     return 0;
458 
459   if (!isUncondBranchOpcode(I->getOpcode()) &&
460       !isCondBranchOpcode(I->getOpcode()))
461     return 0;
462 
463   // Remove the branch.
464   I->eraseFromParent();
465 
466   I = MBB.end();
467 
468   if (I == MBB.begin()) return 1;
469   --I;
470   if (!isCondBranchOpcode(I->getOpcode()))
471     return 1;
472 
473   // Remove the branch.
474   I->eraseFromParent();
475   return 2;
476 }
477 
insertBranch(MachineBasicBlock & MBB,MachineBasicBlock * TBB,MachineBasicBlock * FBB,ArrayRef<MachineOperand> Cond,const DebugLoc & DL,int * BytesAdded) const478 unsigned ARMBaseInstrInfo::insertBranch(MachineBasicBlock &MBB,
479                                         MachineBasicBlock *TBB,
480                                         MachineBasicBlock *FBB,
481                                         ArrayRef<MachineOperand> Cond,
482                                         const DebugLoc &DL,
483                                         int *BytesAdded) const {
484   assert(!BytesAdded && "code size not handled");
485   ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>();
486   int BOpc   = !AFI->isThumbFunction()
487     ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB);
488   int BccOpc = !AFI->isThumbFunction()
489     ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc);
490   bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function();
491 
492   // Shouldn't be a fall through.
493   assert(TBB && "insertBranch must not be told to insert a fallthrough");
494   assert((Cond.size() == 2 || Cond.size() == 0) &&
495          "ARM branch conditions have two components!");
496 
497   // For conditional branches, we use addOperand to preserve CPSR flags.
498 
499   if (!FBB) {
500     if (Cond.empty()) { // Unconditional branch?
501       if (isThumb)
502         BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB).add(predOps(ARMCC::AL));
503       else
504         BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
505     } else
506       BuildMI(&MBB, DL, get(BccOpc))
507           .addMBB(TBB)
508           .addImm(Cond[0].getImm())
509           .add(Cond[1]);
510     return 1;
511   }
512 
513   // Two-way conditional branch.
514   BuildMI(&MBB, DL, get(BccOpc))
515       .addMBB(TBB)
516       .addImm(Cond[0].getImm())
517       .add(Cond[1]);
518   if (isThumb)
519     BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB).add(predOps(ARMCC::AL));
520   else
521     BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB);
522   return 2;
523 }
524 
525 bool ARMBaseInstrInfo::
reverseBranchCondition(SmallVectorImpl<MachineOperand> & Cond) const526 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
527   ARMCC::CondCodes CC = (ARMCC::CondCodes)(int)Cond[0].getImm();
528   Cond[0].setImm(ARMCC::getOppositeCondition(CC));
529   return false;
530 }
531 
isPredicated(const MachineInstr & MI) const532 bool ARMBaseInstrInfo::isPredicated(const MachineInstr &MI) const {
533   if (MI.isBundle()) {
534     MachineBasicBlock::const_instr_iterator I = MI.getIterator();
535     MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
536     while (++I != E && I->isInsideBundle()) {
537       int PIdx = I->findFirstPredOperandIdx();
538       if (PIdx != -1 && I->getOperand(PIdx).getImm() != ARMCC::AL)
539         return true;
540     }
541     return false;
542   }
543 
544   int PIdx = MI.findFirstPredOperandIdx();
545   return PIdx != -1 && MI.getOperand(PIdx).getImm() != ARMCC::AL;
546 }
547 
createMIROperandComment(const MachineInstr & MI,const MachineOperand & Op,unsigned OpIdx,const TargetRegisterInfo * TRI) const548 std::string ARMBaseInstrInfo::createMIROperandComment(
549     const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
550     const TargetRegisterInfo *TRI) const {
551 
552   // First, let's see if there is a generic comment for this operand
553   std::string GenericComment =
554       TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
555   if (!GenericComment.empty())
556     return GenericComment;
557 
558   // If not, check if we have an immediate operand.
559   if (Op.getType() != MachineOperand::MO_Immediate)
560     return std::string();
561 
562   // And print its corresponding condition code if the immediate is a
563   // predicate.
564   int FirstPredOp = MI.findFirstPredOperandIdx();
565   if (FirstPredOp != (int) OpIdx)
566     return std::string();
567 
568   std::string CC = "CC::";
569   CC += ARMCondCodeToString((ARMCC::CondCodes)Op.getImm());
570   return CC;
571 }
572 
PredicateInstruction(MachineInstr & MI,ArrayRef<MachineOperand> Pred) const573 bool ARMBaseInstrInfo::PredicateInstruction(
574     MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
575   unsigned Opc = MI.getOpcode();
576   if (isUncondBranchOpcode(Opc)) {
577     MI.setDesc(get(getMatchingCondBranchOpcode(Opc)));
578     MachineInstrBuilder(*MI.getParent()->getParent(), MI)
579       .addImm(Pred[0].getImm())
580       .addReg(Pred[1].getReg());
581     return true;
582   }
583 
584   int PIdx = MI.findFirstPredOperandIdx();
585   if (PIdx != -1) {
586     MachineOperand &PMO = MI.getOperand(PIdx);
587     PMO.setImm(Pred[0].getImm());
588     MI.getOperand(PIdx+1).setReg(Pred[1].getReg());
589 
590     // Thumb 1 arithmetic instructions do not set CPSR when executed inside an
591     // IT block. This affects how they are printed.
592     const MCInstrDesc &MCID = MI.getDesc();
593     if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) {
594       assert(MCID.OpInfo[1].isOptionalDef() && "CPSR def isn't expected operand");
595       assert((MI.getOperand(1).isDead() ||
596               MI.getOperand(1).getReg() != ARM::CPSR) &&
597              "if conversion tried to stop defining used CPSR");
598       MI.getOperand(1).setReg(ARM::NoRegister);
599     }
600 
601     return true;
602   }
603   return false;
604 }
605 
SubsumesPredicate(ArrayRef<MachineOperand> Pred1,ArrayRef<MachineOperand> Pred2) const606 bool ARMBaseInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
607                                          ArrayRef<MachineOperand> Pred2) const {
608   if (Pred1.size() > 2 || Pred2.size() > 2)
609     return false;
610 
611   ARMCC::CondCodes CC1 = (ARMCC::CondCodes)Pred1[0].getImm();
612   ARMCC::CondCodes CC2 = (ARMCC::CondCodes)Pred2[0].getImm();
613   if (CC1 == CC2)
614     return true;
615 
616   switch (CC1) {
617   default:
618     return false;
619   case ARMCC::AL:
620     return true;
621   case ARMCC::HS:
622     return CC2 == ARMCC::HI;
623   case ARMCC::LS:
624     return CC2 == ARMCC::LO || CC2 == ARMCC::EQ;
625   case ARMCC::GE:
626     return CC2 == ARMCC::GT;
627   case ARMCC::LE:
628     return CC2 == ARMCC::LT;
629   }
630 }
631 
ClobbersPredicate(MachineInstr & MI,std::vector<MachineOperand> & Pred,bool SkipDead) const632 bool ARMBaseInstrInfo::ClobbersPredicate(MachineInstr &MI,
633                                          std::vector<MachineOperand> &Pred,
634                                          bool SkipDead) const {
635   bool Found = false;
636   for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
637     const MachineOperand &MO = MI.getOperand(i);
638     bool ClobbersCPSR = MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR);
639     bool IsCPSR = MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR;
640     if (ClobbersCPSR || IsCPSR) {
641 
642       // Filter out T1 instructions that have a dead CPSR,
643       // allowing IT blocks to be generated containing T1 instructions
644       const MCInstrDesc &MCID = MI.getDesc();
645       if (MCID.TSFlags & ARMII::ThumbArithFlagSetting && MO.isDead() &&
646           SkipDead)
647         continue;
648 
649       Pred.push_back(MO);
650       Found = true;
651     }
652   }
653 
654   return Found;
655 }
656 
isCPSRDefined(const MachineInstr & MI)657 bool ARMBaseInstrInfo::isCPSRDefined(const MachineInstr &MI) {
658   for (const auto &MO : MI.operands())
659     if (MO.isReg() && MO.getReg() == ARM::CPSR && MO.isDef() && !MO.isDead())
660       return true;
661   return false;
662 }
663 
isEligibleForITBlock(const MachineInstr * MI)664 static bool isEligibleForITBlock(const MachineInstr *MI) {
665   switch (MI->getOpcode()) {
666   default: return true;
667   case ARM::tADC:   // ADC (register) T1
668   case ARM::tADDi3: // ADD (immediate) T1
669   case ARM::tADDi8: // ADD (immediate) T2
670   case ARM::tADDrr: // ADD (register) T1
671   case ARM::tAND:   // AND (register) T1
672   case ARM::tASRri: // ASR (immediate) T1
673   case ARM::tASRrr: // ASR (register) T1
674   case ARM::tBIC:   // BIC (register) T1
675   case ARM::tEOR:   // EOR (register) T1
676   case ARM::tLSLri: // LSL (immediate) T1
677   case ARM::tLSLrr: // LSL (register) T1
678   case ARM::tLSRri: // LSR (immediate) T1
679   case ARM::tLSRrr: // LSR (register) T1
680   case ARM::tMUL:   // MUL T1
681   case ARM::tMVN:   // MVN (register) T1
682   case ARM::tORR:   // ORR (register) T1
683   case ARM::tROR:   // ROR (register) T1
684   case ARM::tRSB:   // RSB (immediate) T1
685   case ARM::tSBC:   // SBC (register) T1
686   case ARM::tSUBi3: // SUB (immediate) T1
687   case ARM::tSUBi8: // SUB (immediate) T2
688   case ARM::tSUBrr: // SUB (register) T1
689     return !ARMBaseInstrInfo::isCPSRDefined(*MI);
690   }
691 }
692 
693 /// isPredicable - Return true if the specified instruction can be predicated.
694 /// By default, this returns true for every instruction with a
695 /// PredicateOperand.
isPredicable(const MachineInstr & MI) const696 bool ARMBaseInstrInfo::isPredicable(const MachineInstr &MI) const {
697   if (!MI.isPredicable())
698     return false;
699 
700   if (MI.isBundle())
701     return false;
702 
703   if (!isEligibleForITBlock(&MI))
704     return false;
705 
706   const MachineFunction *MF = MI.getParent()->getParent();
707   const ARMFunctionInfo *AFI =
708       MF->getInfo<ARMFunctionInfo>();
709 
710   // Neon instructions in Thumb2 IT blocks are deprecated, see ARMARM.
711   // In their ARM encoding, they can't be encoded in a conditional form.
712   if ((MI.getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON)
713     return false;
714 
715   // Make indirect control flow changes unpredicable when SLS mitigation is
716   // enabled.
717   const ARMSubtarget &ST = MF->getSubtarget<ARMSubtarget>();
718   if (ST.hardenSlsRetBr() && isIndirectControlFlowNotComingBack(MI))
719     return false;
720   if (ST.hardenSlsBlr() && isIndirectCall(MI))
721     return false;
722 
723   if (AFI->isThumb2Function()) {
724     if (getSubtarget().restrictIT())
725       return isV8EligibleForIT(&MI);
726   }
727 
728   return true;
729 }
730 
731 namespace llvm {
732 
IsCPSRDead(const MachineInstr * MI)733 template <> bool IsCPSRDead<MachineInstr>(const MachineInstr *MI) {
734   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
735     const MachineOperand &MO = MI->getOperand(i);
736     if (!MO.isReg() || MO.isUndef() || MO.isUse())
737       continue;
738     if (MO.getReg() != ARM::CPSR)
739       continue;
740     if (!MO.isDead())
741       return false;
742   }
743   // all definitions of CPSR are dead
744   return true;
745 }
746 
747 } // end namespace llvm
748 
749 /// GetInstSize - Return the size of the specified MachineInstr.
750 ///
getInstSizeInBytes(const MachineInstr & MI) const751 unsigned ARMBaseInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
752   const MachineBasicBlock &MBB = *MI.getParent();
753   const MachineFunction *MF = MBB.getParent();
754   const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
755 
756   const MCInstrDesc &MCID = MI.getDesc();
757   if (MCID.getSize())
758     return MCID.getSize();
759 
760   switch (MI.getOpcode()) {
761   default:
762     // pseudo-instruction sizes are zero.
763     return 0;
764   case TargetOpcode::BUNDLE:
765     return getInstBundleLength(MI);
766   case ARM::MOVi16_ga_pcrel:
767   case ARM::MOVTi16_ga_pcrel:
768   case ARM::t2MOVi16_ga_pcrel:
769   case ARM::t2MOVTi16_ga_pcrel:
770     return 4;
771   case ARM::MOVi32imm:
772   case ARM::t2MOVi32imm:
773     return 8;
774   case ARM::CONSTPOOL_ENTRY:
775   case ARM::JUMPTABLE_INSTS:
776   case ARM::JUMPTABLE_ADDRS:
777   case ARM::JUMPTABLE_TBB:
778   case ARM::JUMPTABLE_TBH:
779     // If this machine instr is a constant pool entry, its size is recorded as
780     // operand #2.
781     return MI.getOperand(2).getImm();
782   case ARM::Int_eh_sjlj_longjmp:
783     return 16;
784   case ARM::tInt_eh_sjlj_longjmp:
785     return 10;
786   case ARM::tInt_WIN_eh_sjlj_longjmp:
787     return 12;
788   case ARM::Int_eh_sjlj_setjmp:
789   case ARM::Int_eh_sjlj_setjmp_nofp:
790     return 20;
791   case ARM::tInt_eh_sjlj_setjmp:
792   case ARM::t2Int_eh_sjlj_setjmp:
793   case ARM::t2Int_eh_sjlj_setjmp_nofp:
794     return 12;
795   case ARM::SPACE:
796     return MI.getOperand(1).getImm();
797   case ARM::INLINEASM:
798   case ARM::INLINEASM_BR: {
799     // If this machine instr is an inline asm, measure it.
800     unsigned Size = getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI);
801     if (!MF->getInfo<ARMFunctionInfo>()->isThumbFunction())
802       Size = alignTo(Size, 4);
803     return Size;
804   }
805   case ARM::SpeculationBarrierISBDSBEndBB:
806   case ARM::t2SpeculationBarrierISBDSBEndBB:
807     // This gets lowered to 2 4-byte instructions.
808     return 8;
809   case ARM::SpeculationBarrierSBEndBB:
810   case ARM::t2SpeculationBarrierSBEndBB:
811     // This gets lowered to 1 4-byte instructions.
812     return 4;
813   }
814 }
815 
getInstBundleLength(const MachineInstr & MI) const816 unsigned ARMBaseInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
817   unsigned Size = 0;
818   MachineBasicBlock::const_instr_iterator I = MI.getIterator();
819   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
820   while (++I != E && I->isInsideBundle()) {
821     assert(!I->isBundle() && "No nested bundle!");
822     Size += getInstSizeInBytes(*I);
823   }
824   return Size;
825 }
826 
copyFromCPSR(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,unsigned DestReg,bool KillSrc,const ARMSubtarget & Subtarget) const827 void ARMBaseInstrInfo::copyFromCPSR(MachineBasicBlock &MBB,
828                                     MachineBasicBlock::iterator I,
829                                     unsigned DestReg, bool KillSrc,
830                                     const ARMSubtarget &Subtarget) const {
831   unsigned Opc = Subtarget.isThumb()
832                      ? (Subtarget.isMClass() ? ARM::t2MRS_M : ARM::t2MRS_AR)
833                      : ARM::MRS;
834 
835   MachineInstrBuilder MIB =
836       BuildMI(MBB, I, I->getDebugLoc(), get(Opc), DestReg);
837 
838   // There is only 1 A/R class MRS instruction, and it always refers to
839   // APSR. However, there are lots of other possibilities on M-class cores.
840   if (Subtarget.isMClass())
841     MIB.addImm(0x800);
842 
843   MIB.add(predOps(ARMCC::AL))
844      .addReg(ARM::CPSR, RegState::Implicit | getKillRegState(KillSrc));
845 }
846 
copyToCPSR(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,unsigned SrcReg,bool KillSrc,const ARMSubtarget & Subtarget) const847 void ARMBaseInstrInfo::copyToCPSR(MachineBasicBlock &MBB,
848                                   MachineBasicBlock::iterator I,
849                                   unsigned SrcReg, bool KillSrc,
850                                   const ARMSubtarget &Subtarget) const {
851   unsigned Opc = Subtarget.isThumb()
852                      ? (Subtarget.isMClass() ? ARM::t2MSR_M : ARM::t2MSR_AR)
853                      : ARM::MSR;
854 
855   MachineInstrBuilder MIB = BuildMI(MBB, I, I->getDebugLoc(), get(Opc));
856 
857   if (Subtarget.isMClass())
858     MIB.addImm(0x800);
859   else
860     MIB.addImm(8);
861 
862   MIB.addReg(SrcReg, getKillRegState(KillSrc))
863      .add(predOps(ARMCC::AL))
864      .addReg(ARM::CPSR, RegState::Implicit | RegState::Define);
865 }
866 
addUnpredicatedMveVpredNOp(MachineInstrBuilder & MIB)867 void llvm::addUnpredicatedMveVpredNOp(MachineInstrBuilder &MIB) {
868   MIB.addImm(ARMVCC::None);
869   MIB.addReg(0);
870 }
871 
addUnpredicatedMveVpredROp(MachineInstrBuilder & MIB,Register DestReg)872 void llvm::addUnpredicatedMveVpredROp(MachineInstrBuilder &MIB,
873                                       Register DestReg) {
874   addUnpredicatedMveVpredNOp(MIB);
875   MIB.addReg(DestReg, RegState::Undef);
876 }
877 
addPredicatedMveVpredNOp(MachineInstrBuilder & MIB,unsigned Cond)878 void llvm::addPredicatedMveVpredNOp(MachineInstrBuilder &MIB, unsigned Cond) {
879   MIB.addImm(Cond);
880   MIB.addReg(ARM::VPR, RegState::Implicit);
881 }
882 
addPredicatedMveVpredROp(MachineInstrBuilder & MIB,unsigned Cond,unsigned Inactive)883 void llvm::addPredicatedMveVpredROp(MachineInstrBuilder &MIB,
884                                     unsigned Cond, unsigned Inactive) {
885   addPredicatedMveVpredNOp(MIB, Cond);
886   MIB.addReg(Inactive);
887 }
888 
copyPhysReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const DebugLoc & DL,MCRegister DestReg,MCRegister SrcReg,bool KillSrc) const889 void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
890                                    MachineBasicBlock::iterator I,
891                                    const DebugLoc &DL, MCRegister DestReg,
892                                    MCRegister SrcReg, bool KillSrc) const {
893   bool GPRDest = ARM::GPRRegClass.contains(DestReg);
894   bool GPRSrc = ARM::GPRRegClass.contains(SrcReg);
895 
896   if (GPRDest && GPRSrc) {
897     BuildMI(MBB, I, DL, get(ARM::MOVr), DestReg)
898         .addReg(SrcReg, getKillRegState(KillSrc))
899         .add(predOps(ARMCC::AL))
900         .add(condCodeOp());
901     return;
902   }
903 
904   bool SPRDest = ARM::SPRRegClass.contains(DestReg);
905   bool SPRSrc = ARM::SPRRegClass.contains(SrcReg);
906 
907   unsigned Opc = 0;
908   if (SPRDest && SPRSrc)
909     Opc = ARM::VMOVS;
910   else if (GPRDest && SPRSrc)
911     Opc = ARM::VMOVRS;
912   else if (SPRDest && GPRSrc)
913     Opc = ARM::VMOVSR;
914   else if (ARM::DPRRegClass.contains(DestReg, SrcReg) && Subtarget.hasFP64())
915     Opc = ARM::VMOVD;
916   else if (ARM::QPRRegClass.contains(DestReg, SrcReg))
917     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
918 
919   if (Opc) {
920     MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc), DestReg);
921     MIB.addReg(SrcReg, getKillRegState(KillSrc));
922     if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR)
923       MIB.addReg(SrcReg, getKillRegState(KillSrc));
924     if (Opc == ARM::MVE_VORR)
925       addUnpredicatedMveVpredROp(MIB, DestReg);
926     else
927       MIB.add(predOps(ARMCC::AL));
928     return;
929   }
930 
931   // Handle register classes that require multiple instructions.
932   unsigned BeginIdx = 0;
933   unsigned SubRegs = 0;
934   int Spacing = 1;
935 
936   // Use VORRq when possible.
937   if (ARM::QQPRRegClass.contains(DestReg, SrcReg)) {
938     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
939     BeginIdx = ARM::qsub_0;
940     SubRegs = 2;
941   } else if (ARM::QQQQPRRegClass.contains(DestReg, SrcReg)) {
942     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
943     BeginIdx = ARM::qsub_0;
944     SubRegs = 4;
945   // Fall back to VMOVD.
946   } else if (ARM::DPairRegClass.contains(DestReg, SrcReg)) {
947     Opc = ARM::VMOVD;
948     BeginIdx = ARM::dsub_0;
949     SubRegs = 2;
950   } else if (ARM::DTripleRegClass.contains(DestReg, SrcReg)) {
951     Opc = ARM::VMOVD;
952     BeginIdx = ARM::dsub_0;
953     SubRegs = 3;
954   } else if (ARM::DQuadRegClass.contains(DestReg, SrcReg)) {
955     Opc = ARM::VMOVD;
956     BeginIdx = ARM::dsub_0;
957     SubRegs = 4;
958   } else if (ARM::GPRPairRegClass.contains(DestReg, SrcReg)) {
959     Opc = Subtarget.isThumb2() ? ARM::tMOVr : ARM::MOVr;
960     BeginIdx = ARM::gsub_0;
961     SubRegs = 2;
962   } else if (ARM::DPairSpcRegClass.contains(DestReg, SrcReg)) {
963     Opc = ARM::VMOVD;
964     BeginIdx = ARM::dsub_0;
965     SubRegs = 2;
966     Spacing = 2;
967   } else if (ARM::DTripleSpcRegClass.contains(DestReg, SrcReg)) {
968     Opc = ARM::VMOVD;
969     BeginIdx = ARM::dsub_0;
970     SubRegs = 3;
971     Spacing = 2;
972   } else if (ARM::DQuadSpcRegClass.contains(DestReg, SrcReg)) {
973     Opc = ARM::VMOVD;
974     BeginIdx = ARM::dsub_0;
975     SubRegs = 4;
976     Spacing = 2;
977   } else if (ARM::DPRRegClass.contains(DestReg, SrcReg) &&
978              !Subtarget.hasFP64()) {
979     Opc = ARM::VMOVS;
980     BeginIdx = ARM::ssub_0;
981     SubRegs = 2;
982   } else if (SrcReg == ARM::CPSR) {
983     copyFromCPSR(MBB, I, DestReg, KillSrc, Subtarget);
984     return;
985   } else if (DestReg == ARM::CPSR) {
986     copyToCPSR(MBB, I, SrcReg, KillSrc, Subtarget);
987     return;
988   } else if (DestReg == ARM::VPR) {
989     assert(ARM::GPRRegClass.contains(SrcReg));
990     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_P0), DestReg)
991         .addReg(SrcReg, getKillRegState(KillSrc))
992         .add(predOps(ARMCC::AL));
993     return;
994   } else if (SrcReg == ARM::VPR) {
995     assert(ARM::GPRRegClass.contains(DestReg));
996     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_P0), DestReg)
997         .addReg(SrcReg, getKillRegState(KillSrc))
998         .add(predOps(ARMCC::AL));
999     return;
1000   } else if (DestReg == ARM::FPSCR_NZCV) {
1001     assert(ARM::GPRRegClass.contains(SrcReg));
1002     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_FPSCR_NZCVQC), DestReg)
1003         .addReg(SrcReg, getKillRegState(KillSrc))
1004         .add(predOps(ARMCC::AL));
1005     return;
1006   } else if (SrcReg == ARM::FPSCR_NZCV) {
1007     assert(ARM::GPRRegClass.contains(DestReg));
1008     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_FPSCR_NZCVQC), DestReg)
1009         .addReg(SrcReg, getKillRegState(KillSrc))
1010         .add(predOps(ARMCC::AL));
1011     return;
1012   }
1013 
1014   assert(Opc && "Impossible reg-to-reg copy");
1015 
1016   const TargetRegisterInfo *TRI = &getRegisterInfo();
1017   MachineInstrBuilder Mov;
1018 
1019   // Copy register tuples backward when the first Dest reg overlaps with SrcReg.
1020   if (TRI->regsOverlap(SrcReg, TRI->getSubReg(DestReg, BeginIdx))) {
1021     BeginIdx = BeginIdx + ((SubRegs - 1) * Spacing);
1022     Spacing = -Spacing;
1023   }
1024 #ifndef NDEBUG
1025   SmallSet<unsigned, 4> DstRegs;
1026 #endif
1027   for (unsigned i = 0; i != SubRegs; ++i) {
1028     Register Dst = TRI->getSubReg(DestReg, BeginIdx + i * Spacing);
1029     Register Src = TRI->getSubReg(SrcReg, BeginIdx + i * Spacing);
1030     assert(Dst && Src && "Bad sub-register");
1031 #ifndef NDEBUG
1032     assert(!DstRegs.count(Src) && "destructive vector copy");
1033     DstRegs.insert(Dst);
1034 #endif
1035     Mov = BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst).addReg(Src);
1036     // VORR (NEON or MVE) takes two source operands.
1037     if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR) {
1038       Mov.addReg(Src);
1039     }
1040     // MVE VORR takes predicate operands in place of an ordinary condition.
1041     if (Opc == ARM::MVE_VORR)
1042       addUnpredicatedMveVpredROp(Mov, Dst);
1043     else
1044       Mov = Mov.add(predOps(ARMCC::AL));
1045     // MOVr can set CC.
1046     if (Opc == ARM::MOVr)
1047       Mov = Mov.add(condCodeOp());
1048   }
1049   // Add implicit super-register defs and kills to the last instruction.
1050   Mov->addRegisterDefined(DestReg, TRI);
1051   if (KillSrc)
1052     Mov->addRegisterKilled(SrcReg, TRI);
1053 }
1054 
1055 Optional<DestSourcePair>
isCopyInstrImpl(const MachineInstr & MI) const1056 ARMBaseInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
1057   // VMOVRRD is also a copy instruction but it requires
1058   // special way of handling. It is more complex copy version
1059   // and since that we are not considering it. For recognition
1060   // of such instruction isExtractSubregLike MI interface fuction
1061   // could be used.
1062   // VORRq is considered as a move only if two inputs are
1063   // the same register.
1064   if (!MI.isMoveReg() ||
1065       (MI.getOpcode() == ARM::VORRq &&
1066        MI.getOperand(1).getReg() != MI.getOperand(2).getReg()))
1067     return None;
1068   return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1069 }
1070 
1071 Optional<ParamLoadedValue>
describeLoadedValue(const MachineInstr & MI,Register Reg) const1072 ARMBaseInstrInfo::describeLoadedValue(const MachineInstr &MI,
1073                                       Register Reg) const {
1074   if (auto DstSrcPair = isCopyInstrImpl(MI)) {
1075     Register DstReg = DstSrcPair->Destination->getReg();
1076 
1077     // TODO: We don't handle cases where the forwarding reg is narrower/wider
1078     // than the copy registers. Consider for example:
1079     //
1080     //   s16 = VMOVS s0
1081     //   s17 = VMOVS s1
1082     //   call @callee(d0)
1083     //
1084     // We'd like to describe the call site value of d0 as d8, but this requires
1085     // gathering and merging the descriptions for the two VMOVS instructions.
1086     //
1087     // We also don't handle the reverse situation, where the forwarding reg is
1088     // narrower than the copy destination:
1089     //
1090     //   d8 = VMOVD d0
1091     //   call @callee(s1)
1092     //
1093     // We need to produce a fragment description (the call site value of s1 is
1094     // /not/ just d8).
1095     if (DstReg != Reg)
1096       return None;
1097   }
1098   return TargetInstrInfo::describeLoadedValue(MI, Reg);
1099 }
1100 
1101 const MachineInstrBuilder &
AddDReg(MachineInstrBuilder & MIB,unsigned Reg,unsigned SubIdx,unsigned State,const TargetRegisterInfo * TRI) const1102 ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB, unsigned Reg,
1103                           unsigned SubIdx, unsigned State,
1104                           const TargetRegisterInfo *TRI) const {
1105   if (!SubIdx)
1106     return MIB.addReg(Reg, State);
1107 
1108   if (Register::isPhysicalRegister(Reg))
1109     return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
1110   return MIB.addReg(Reg, State, SubIdx);
1111 }
1112 
1113 void ARMBaseInstrInfo::
storeRegToStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,Register SrcReg,bool isKill,int FI,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1114 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
1115                     Register SrcReg, bool isKill, int FI,
1116                     const TargetRegisterClass *RC,
1117                     const TargetRegisterInfo *TRI) const {
1118   MachineFunction &MF = *MBB.getParent();
1119   MachineFrameInfo &MFI = MF.getFrameInfo();
1120   Align Alignment = MFI.getObjectAlign(FI);
1121 
1122   MachineMemOperand *MMO = MF.getMachineMemOperand(
1123       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore,
1124       MFI.getObjectSize(FI), Alignment);
1125 
1126   switch (TRI->getSpillSize(*RC)) {
1127     case 2:
1128       if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1129         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRH))
1130             .addReg(SrcReg, getKillRegState(isKill))
1131             .addFrameIndex(FI)
1132             .addImm(0)
1133             .addMemOperand(MMO)
1134             .add(predOps(ARMCC::AL));
1135       } else
1136         llvm_unreachable("Unknown reg class!");
1137       break;
1138     case 4:
1139       if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1140         BuildMI(MBB, I, DebugLoc(), get(ARM::STRi12))
1141             .addReg(SrcReg, getKillRegState(isKill))
1142             .addFrameIndex(FI)
1143             .addImm(0)
1144             .addMemOperand(MMO)
1145             .add(predOps(ARMCC::AL));
1146       } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1147         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRS))
1148             .addReg(SrcReg, getKillRegState(isKill))
1149             .addFrameIndex(FI)
1150             .addImm(0)
1151             .addMemOperand(MMO)
1152             .add(predOps(ARMCC::AL));
1153       } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1154         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTR_P0_off))
1155             .addReg(SrcReg, getKillRegState(isKill))
1156             .addFrameIndex(FI)
1157             .addImm(0)
1158             .addMemOperand(MMO)
1159             .add(predOps(ARMCC::AL));
1160       } else
1161         llvm_unreachable("Unknown reg class!");
1162       break;
1163     case 8:
1164       if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1165         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRD))
1166             .addReg(SrcReg, getKillRegState(isKill))
1167             .addFrameIndex(FI)
1168             .addImm(0)
1169             .addMemOperand(MMO)
1170             .add(predOps(ARMCC::AL));
1171       } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1172         if (Subtarget.hasV5TEOps()) {
1173           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STRD));
1174           AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
1175           AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
1176           MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO)
1177              .add(predOps(ARMCC::AL));
1178         } else {
1179           // Fallback to STM instruction, which has existed since the dawn of
1180           // time.
1181           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STMIA))
1182                                         .addFrameIndex(FI)
1183                                         .addMemOperand(MMO)
1184                                         .add(predOps(ARMCC::AL));
1185           AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
1186           AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
1187         }
1188       } else
1189         llvm_unreachable("Unknown reg class!");
1190       break;
1191     case 16:
1192       if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1193         // Use aligned spills if the stack can be realigned.
1194         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1195           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1q64))
1196               .addFrameIndex(FI)
1197               .addImm(16)
1198               .addReg(SrcReg, getKillRegState(isKill))
1199               .addMemOperand(MMO)
1200               .add(predOps(ARMCC::AL));
1201         } else {
1202           BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMQIA))
1203               .addReg(SrcReg, getKillRegState(isKill))
1204               .addFrameIndex(FI)
1205               .addMemOperand(MMO)
1206               .add(predOps(ARMCC::AL));
1207         }
1208       } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1209                  Subtarget.hasMVEIntegerOps()) {
1210         auto MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::MVE_VSTRWU32));
1211         MIB.addReg(SrcReg, getKillRegState(isKill))
1212           .addFrameIndex(FI)
1213           .addImm(0)
1214           .addMemOperand(MMO);
1215         addUnpredicatedMveVpredNOp(MIB);
1216       } else
1217         llvm_unreachable("Unknown reg class!");
1218       break;
1219     case 24:
1220       if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1221         // Use aligned spills if the stack can be realigned.
1222         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1223             Subtarget.hasNEON()) {
1224           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64TPseudo))
1225               .addFrameIndex(FI)
1226               .addImm(16)
1227               .addReg(SrcReg, getKillRegState(isKill))
1228               .addMemOperand(MMO)
1229               .add(predOps(ARMCC::AL));
1230         } else {
1231           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(),
1232                                             get(ARM::VSTMDIA))
1233                                         .addFrameIndex(FI)
1234                                         .add(predOps(ARMCC::AL))
1235                                         .addMemOperand(MMO);
1236           MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1237           MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1238           AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1239         }
1240       } else
1241         llvm_unreachable("Unknown reg class!");
1242       break;
1243     case 32:
1244       if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) {
1245         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1246             Subtarget.hasNEON()) {
1247           // FIXME: It's possible to only store part of the QQ register if the
1248           // spilled def has a sub-register index.
1249           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64QPseudo))
1250               .addFrameIndex(FI)
1251               .addImm(16)
1252               .addReg(SrcReg, getKillRegState(isKill))
1253               .addMemOperand(MMO)
1254               .add(predOps(ARMCC::AL));
1255         } else {
1256           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(),
1257                                             get(ARM::VSTMDIA))
1258                                         .addFrameIndex(FI)
1259                                         .add(predOps(ARMCC::AL))
1260                                         .addMemOperand(MMO);
1261           MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1262           MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1263           MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1264                 AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
1265         }
1266       } else
1267         llvm_unreachable("Unknown reg class!");
1268       break;
1269     case 64:
1270       if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1271         MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMDIA))
1272                                       .addFrameIndex(FI)
1273                                       .add(predOps(ARMCC::AL))
1274                                       .addMemOperand(MMO);
1275         MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1276         MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1277         MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1278         MIB = AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
1279         MIB = AddDReg(MIB, SrcReg, ARM::dsub_4, 0, TRI);
1280         MIB = AddDReg(MIB, SrcReg, ARM::dsub_5, 0, TRI);
1281         MIB = AddDReg(MIB, SrcReg, ARM::dsub_6, 0, TRI);
1282               AddDReg(MIB, SrcReg, ARM::dsub_7, 0, TRI);
1283       } else
1284         llvm_unreachable("Unknown reg class!");
1285       break;
1286     default:
1287       llvm_unreachable("Unknown reg class!");
1288   }
1289 }
1290 
isStoreToStackSlot(const MachineInstr & MI,int & FrameIndex) const1291 unsigned ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1292                                               int &FrameIndex) const {
1293   switch (MI.getOpcode()) {
1294   default: break;
1295   case ARM::STRrs:
1296   case ARM::t2STRs: // FIXME: don't use t2STRs to access frame.
1297     if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() &&
1298         MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 &&
1299         MI.getOperand(3).getImm() == 0) {
1300       FrameIndex = MI.getOperand(1).getIndex();
1301       return MI.getOperand(0).getReg();
1302     }
1303     break;
1304   case ARM::STRi12:
1305   case ARM::t2STRi12:
1306   case ARM::tSTRspi:
1307   case ARM::VSTRD:
1308   case ARM::VSTRS:
1309   case ARM::VSTR_P0_off:
1310   case ARM::MVE_VSTRWU32:
1311     if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
1312         MI.getOperand(2).getImm() == 0) {
1313       FrameIndex = MI.getOperand(1).getIndex();
1314       return MI.getOperand(0).getReg();
1315     }
1316     break;
1317   case ARM::VST1q64:
1318   case ARM::VST1d64TPseudo:
1319   case ARM::VST1d64QPseudo:
1320     if (MI.getOperand(0).isFI() && MI.getOperand(2).getSubReg() == 0) {
1321       FrameIndex = MI.getOperand(0).getIndex();
1322       return MI.getOperand(2).getReg();
1323     }
1324     break;
1325   case ARM::VSTMQIA:
1326     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1327       FrameIndex = MI.getOperand(1).getIndex();
1328       return MI.getOperand(0).getReg();
1329     }
1330     break;
1331   }
1332 
1333   return 0;
1334 }
1335 
isStoreToStackSlotPostFE(const MachineInstr & MI,int & FrameIndex) const1336 unsigned ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
1337                                                     int &FrameIndex) const {
1338   SmallVector<const MachineMemOperand *, 1> Accesses;
1339   if (MI.mayStore() && hasStoreToStackSlot(MI, Accesses) &&
1340       Accesses.size() == 1) {
1341     FrameIndex =
1342         cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
1343             ->getFrameIndex();
1344     return true;
1345   }
1346   return false;
1347 }
1348 
1349 void ARMBaseInstrInfo::
loadRegFromStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,Register DestReg,int FI,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI) const1350 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
1351                      Register DestReg, int FI,
1352                      const TargetRegisterClass *RC,
1353                      const TargetRegisterInfo *TRI) const {
1354   DebugLoc DL;
1355   if (I != MBB.end()) DL = I->getDebugLoc();
1356   MachineFunction &MF = *MBB.getParent();
1357   MachineFrameInfo &MFI = MF.getFrameInfo();
1358   const Align Alignment = MFI.getObjectAlign(FI);
1359   MachineMemOperand *MMO = MF.getMachineMemOperand(
1360       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad,
1361       MFI.getObjectSize(FI), Alignment);
1362 
1363   switch (TRI->getSpillSize(*RC)) {
1364   case 2:
1365     if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1366       BuildMI(MBB, I, DL, get(ARM::VLDRH), DestReg)
1367           .addFrameIndex(FI)
1368           .addImm(0)
1369           .addMemOperand(MMO)
1370           .add(predOps(ARMCC::AL));
1371     } else
1372       llvm_unreachable("Unknown reg class!");
1373     break;
1374   case 4:
1375     if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1376       BuildMI(MBB, I, DL, get(ARM::LDRi12), DestReg)
1377           .addFrameIndex(FI)
1378           .addImm(0)
1379           .addMemOperand(MMO)
1380           .add(predOps(ARMCC::AL));
1381     } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1382       BuildMI(MBB, I, DL, get(ARM::VLDRS), DestReg)
1383           .addFrameIndex(FI)
1384           .addImm(0)
1385           .addMemOperand(MMO)
1386           .add(predOps(ARMCC::AL));
1387     } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1388       BuildMI(MBB, I, DL, get(ARM::VLDR_P0_off), DestReg)
1389           .addFrameIndex(FI)
1390           .addImm(0)
1391           .addMemOperand(MMO)
1392           .add(predOps(ARMCC::AL));
1393     } else
1394       llvm_unreachable("Unknown reg class!");
1395     break;
1396   case 8:
1397     if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1398       BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg)
1399           .addFrameIndex(FI)
1400           .addImm(0)
1401           .addMemOperand(MMO)
1402           .add(predOps(ARMCC::AL));
1403     } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1404       MachineInstrBuilder MIB;
1405 
1406       if (Subtarget.hasV5TEOps()) {
1407         MIB = BuildMI(MBB, I, DL, get(ARM::LDRD));
1408         AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1409         AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1410         MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO)
1411            .add(predOps(ARMCC::AL));
1412       } else {
1413         // Fallback to LDM instruction, which has existed since the dawn of
1414         // time.
1415         MIB = BuildMI(MBB, I, DL, get(ARM::LDMIA))
1416                   .addFrameIndex(FI)
1417                   .addMemOperand(MMO)
1418                   .add(predOps(ARMCC::AL));
1419         MIB = AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1420         MIB = AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1421       }
1422 
1423       if (Register::isPhysicalRegister(DestReg))
1424         MIB.addReg(DestReg, RegState::ImplicitDefine);
1425     } else
1426       llvm_unreachable("Unknown reg class!");
1427     break;
1428   case 16:
1429     if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1430       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1431         BuildMI(MBB, I, DL, get(ARM::VLD1q64), DestReg)
1432             .addFrameIndex(FI)
1433             .addImm(16)
1434             .addMemOperand(MMO)
1435             .add(predOps(ARMCC::AL));
1436       } else {
1437         BuildMI(MBB, I, DL, get(ARM::VLDMQIA), DestReg)
1438             .addFrameIndex(FI)
1439             .addMemOperand(MMO)
1440             .add(predOps(ARMCC::AL));
1441       }
1442     } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1443                Subtarget.hasMVEIntegerOps()) {
1444       auto MIB = BuildMI(MBB, I, DL, get(ARM::MVE_VLDRWU32), DestReg);
1445       MIB.addFrameIndex(FI)
1446         .addImm(0)
1447         .addMemOperand(MMO);
1448       addUnpredicatedMveVpredNOp(MIB);
1449     } else
1450       llvm_unreachable("Unknown reg class!");
1451     break;
1452   case 24:
1453     if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1454       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1455           Subtarget.hasNEON()) {
1456         BuildMI(MBB, I, DL, get(ARM::VLD1d64TPseudo), DestReg)
1457             .addFrameIndex(FI)
1458             .addImm(16)
1459             .addMemOperand(MMO)
1460             .add(predOps(ARMCC::AL));
1461       } else {
1462         MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1463                                       .addFrameIndex(FI)
1464                                       .addMemOperand(MMO)
1465                                       .add(predOps(ARMCC::AL));
1466         MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1467         MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1468         MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1469         if (Register::isPhysicalRegister(DestReg))
1470           MIB.addReg(DestReg, RegState::ImplicitDefine);
1471       }
1472     } else
1473       llvm_unreachable("Unknown reg class!");
1474     break;
1475    case 32:
1476     if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) {
1477       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1478           Subtarget.hasNEON()) {
1479         BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg)
1480             .addFrameIndex(FI)
1481             .addImm(16)
1482             .addMemOperand(MMO)
1483             .add(predOps(ARMCC::AL));
1484       } else {
1485         MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1486                                       .addFrameIndex(FI)
1487                                       .add(predOps(ARMCC::AL))
1488                                       .addMemOperand(MMO);
1489         MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1490         MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1491         MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1492         MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1493         if (Register::isPhysicalRegister(DestReg))
1494           MIB.addReg(DestReg, RegState::ImplicitDefine);
1495       }
1496     } else
1497       llvm_unreachable("Unknown reg class!");
1498     break;
1499   case 64:
1500     if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1501       MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1502                                     .addFrameIndex(FI)
1503                                     .add(predOps(ARMCC::AL))
1504                                     .addMemOperand(MMO);
1505       MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1506       MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1507       MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1508       MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1509       MIB = AddDReg(MIB, DestReg, ARM::dsub_4, RegState::DefineNoRead, TRI);
1510       MIB = AddDReg(MIB, DestReg, ARM::dsub_5, RegState::DefineNoRead, TRI);
1511       MIB = AddDReg(MIB, DestReg, ARM::dsub_6, RegState::DefineNoRead, TRI);
1512       MIB = AddDReg(MIB, DestReg, ARM::dsub_7, RegState::DefineNoRead, TRI);
1513       if (Register::isPhysicalRegister(DestReg))
1514         MIB.addReg(DestReg, RegState::ImplicitDefine);
1515     } else
1516       llvm_unreachable("Unknown reg class!");
1517     break;
1518   default:
1519     llvm_unreachable("Unknown regclass!");
1520   }
1521 }
1522 
isLoadFromStackSlot(const MachineInstr & MI,int & FrameIndex) const1523 unsigned ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1524                                                int &FrameIndex) const {
1525   switch (MI.getOpcode()) {
1526   default: break;
1527   case ARM::LDRrs:
1528   case ARM::t2LDRs:  // FIXME: don't use t2LDRs to access frame.
1529     if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() &&
1530         MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 &&
1531         MI.getOperand(3).getImm() == 0) {
1532       FrameIndex = MI.getOperand(1).getIndex();
1533       return MI.getOperand(0).getReg();
1534     }
1535     break;
1536   case ARM::LDRi12:
1537   case ARM::t2LDRi12:
1538   case ARM::tLDRspi:
1539   case ARM::VLDRD:
1540   case ARM::VLDRS:
1541   case ARM::VLDR_P0_off:
1542   case ARM::MVE_VLDRWU32:
1543     if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
1544         MI.getOperand(2).getImm() == 0) {
1545       FrameIndex = MI.getOperand(1).getIndex();
1546       return MI.getOperand(0).getReg();
1547     }
1548     break;
1549   case ARM::VLD1q64:
1550   case ARM::VLD1d8TPseudo:
1551   case ARM::VLD1d16TPseudo:
1552   case ARM::VLD1d32TPseudo:
1553   case ARM::VLD1d64TPseudo:
1554   case ARM::VLD1d8QPseudo:
1555   case ARM::VLD1d16QPseudo:
1556   case ARM::VLD1d32QPseudo:
1557   case ARM::VLD1d64QPseudo:
1558     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1559       FrameIndex = MI.getOperand(1).getIndex();
1560       return MI.getOperand(0).getReg();
1561     }
1562     break;
1563   case ARM::VLDMQIA:
1564     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1565       FrameIndex = MI.getOperand(1).getIndex();
1566       return MI.getOperand(0).getReg();
1567     }
1568     break;
1569   }
1570 
1571   return 0;
1572 }
1573 
isLoadFromStackSlotPostFE(const MachineInstr & MI,int & FrameIndex) const1574 unsigned ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
1575                                                      int &FrameIndex) const {
1576   SmallVector<const MachineMemOperand *, 1> Accesses;
1577   if (MI.mayLoad() && hasLoadFromStackSlot(MI, Accesses) &&
1578       Accesses.size() == 1) {
1579     FrameIndex =
1580         cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
1581             ->getFrameIndex();
1582     return true;
1583   }
1584   return false;
1585 }
1586 
1587 /// Expands MEMCPY to either LDMIA/STMIA or LDMIA_UPD/STMID_UPD
1588 /// depending on whether the result is used.
expandMEMCPY(MachineBasicBlock::iterator MI) const1589 void ARMBaseInstrInfo::expandMEMCPY(MachineBasicBlock::iterator MI) const {
1590   bool isThumb1 = Subtarget.isThumb1Only();
1591   bool isThumb2 = Subtarget.isThumb2();
1592   const ARMBaseInstrInfo *TII = Subtarget.getInstrInfo();
1593 
1594   DebugLoc dl = MI->getDebugLoc();
1595   MachineBasicBlock *BB = MI->getParent();
1596 
1597   MachineInstrBuilder LDM, STM;
1598   if (isThumb1 || !MI->getOperand(1).isDead()) {
1599     MachineOperand LDWb(MI->getOperand(1));
1600     LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA_UPD
1601                                                  : isThumb1 ? ARM::tLDMIA_UPD
1602                                                             : ARM::LDMIA_UPD))
1603               .add(LDWb);
1604   } else {
1605     LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA : ARM::LDMIA));
1606   }
1607 
1608   if (isThumb1 || !MI->getOperand(0).isDead()) {
1609     MachineOperand STWb(MI->getOperand(0));
1610     STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA_UPD
1611                                                  : isThumb1 ? ARM::tSTMIA_UPD
1612                                                             : ARM::STMIA_UPD))
1613               .add(STWb);
1614   } else {
1615     STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA : ARM::STMIA));
1616   }
1617 
1618   MachineOperand LDBase(MI->getOperand(3));
1619   LDM.add(LDBase).add(predOps(ARMCC::AL));
1620 
1621   MachineOperand STBase(MI->getOperand(2));
1622   STM.add(STBase).add(predOps(ARMCC::AL));
1623 
1624   // Sort the scratch registers into ascending order.
1625   const TargetRegisterInfo &TRI = getRegisterInfo();
1626   SmallVector<unsigned, 6> ScratchRegs;
1627   for(unsigned I = 5; I < MI->getNumOperands(); ++I)
1628     ScratchRegs.push_back(MI->getOperand(I).getReg());
1629   llvm::sort(ScratchRegs,
1630              [&TRI](const unsigned &Reg1, const unsigned &Reg2) -> bool {
1631                return TRI.getEncodingValue(Reg1) <
1632                       TRI.getEncodingValue(Reg2);
1633              });
1634 
1635   for (const auto &Reg : ScratchRegs) {
1636     LDM.addReg(Reg, RegState::Define);
1637     STM.addReg(Reg, RegState::Kill);
1638   }
1639 
1640   BB->erase(MI);
1641 }
1642 
expandPostRAPseudo(MachineInstr & MI) const1643 bool ARMBaseInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1644   if (MI.getOpcode() == TargetOpcode::LOAD_STACK_GUARD) {
1645     assert(getSubtarget().getTargetTriple().isOSBinFormatMachO() &&
1646            "LOAD_STACK_GUARD currently supported only for MachO.");
1647     expandLoadStackGuard(MI);
1648     MI.getParent()->erase(MI);
1649     return true;
1650   }
1651 
1652   if (MI.getOpcode() == ARM::MEMCPY) {
1653     expandMEMCPY(MI);
1654     return true;
1655   }
1656 
1657   // This hook gets to expand COPY instructions before they become
1658   // copyPhysReg() calls.  Look for VMOVS instructions that can legally be
1659   // widened to VMOVD.  We prefer the VMOVD when possible because it may be
1660   // changed into a VORR that can go down the NEON pipeline.
1661   if (!MI.isCopy() || Subtarget.dontWidenVMOVS() || !Subtarget.hasFP64())
1662     return false;
1663 
1664   // Look for a copy between even S-registers.  That is where we keep floats
1665   // when using NEON v2f32 instructions for f32 arithmetic.
1666   Register DstRegS = MI.getOperand(0).getReg();
1667   Register SrcRegS = MI.getOperand(1).getReg();
1668   if (!ARM::SPRRegClass.contains(DstRegS, SrcRegS))
1669     return false;
1670 
1671   const TargetRegisterInfo *TRI = &getRegisterInfo();
1672   unsigned DstRegD = TRI->getMatchingSuperReg(DstRegS, ARM::ssub_0,
1673                                               &ARM::DPRRegClass);
1674   unsigned SrcRegD = TRI->getMatchingSuperReg(SrcRegS, ARM::ssub_0,
1675                                               &ARM::DPRRegClass);
1676   if (!DstRegD || !SrcRegD)
1677     return false;
1678 
1679   // We want to widen this into a DstRegD = VMOVD SrcRegD copy.  This is only
1680   // legal if the COPY already defines the full DstRegD, and it isn't a
1681   // sub-register insertion.
1682   if (!MI.definesRegister(DstRegD, TRI) || MI.readsRegister(DstRegD, TRI))
1683     return false;
1684 
1685   // A dead copy shouldn't show up here, but reject it just in case.
1686   if (MI.getOperand(0).isDead())
1687     return false;
1688 
1689   // All clear, widen the COPY.
1690   LLVM_DEBUG(dbgs() << "widening:    " << MI);
1691   MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
1692 
1693   // Get rid of the old implicit-def of DstRegD.  Leave it if it defines a Q-reg
1694   // or some other super-register.
1695   int ImpDefIdx = MI.findRegisterDefOperandIdx(DstRegD);
1696   if (ImpDefIdx != -1)
1697     MI.RemoveOperand(ImpDefIdx);
1698 
1699   // Change the opcode and operands.
1700   MI.setDesc(get(ARM::VMOVD));
1701   MI.getOperand(0).setReg(DstRegD);
1702   MI.getOperand(1).setReg(SrcRegD);
1703   MIB.add(predOps(ARMCC::AL));
1704 
1705   // We are now reading SrcRegD instead of SrcRegS.  This may upset the
1706   // register scavenger and machine verifier, so we need to indicate that we
1707   // are reading an undefined value from SrcRegD, but a proper value from
1708   // SrcRegS.
1709   MI.getOperand(1).setIsUndef();
1710   MIB.addReg(SrcRegS, RegState::Implicit);
1711 
1712   // SrcRegD may actually contain an unrelated value in the ssub_1
1713   // sub-register.  Don't kill it.  Only kill the ssub_0 sub-register.
1714   if (MI.getOperand(1).isKill()) {
1715     MI.getOperand(1).setIsKill(false);
1716     MI.addRegisterKilled(SrcRegS, TRI, true);
1717   }
1718 
1719   LLVM_DEBUG(dbgs() << "replaced by: " << MI);
1720   return true;
1721 }
1722 
1723 /// Create a copy of a const pool value. Update CPI to the new index and return
1724 /// the label UID.
duplicateCPV(MachineFunction & MF,unsigned & CPI)1725 static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) {
1726   MachineConstantPool *MCP = MF.getConstantPool();
1727   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1728 
1729   const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPI];
1730   assert(MCPE.isMachineConstantPoolEntry() &&
1731          "Expecting a machine constantpool entry!");
1732   ARMConstantPoolValue *ACPV =
1733     static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
1734 
1735   unsigned PCLabelId = AFI->createPICLabelUId();
1736   ARMConstantPoolValue *NewCPV = nullptr;
1737 
1738   // FIXME: The below assumes PIC relocation model and that the function
1739   // is Thumb mode (t1 or t2). PCAdjustment would be 8 for ARM mode PIC, and
1740   // zero for non-PIC in ARM or Thumb. The callers are all of thumb LDR
1741   // instructions, so that's probably OK, but is PIC always correct when
1742   // we get here?
1743   if (ACPV->isGlobalValue())
1744     NewCPV = ARMConstantPoolConstant::Create(
1745         cast<ARMConstantPoolConstant>(ACPV)->getGV(), PCLabelId, ARMCP::CPValue,
1746         4, ACPV->getModifier(), ACPV->mustAddCurrentAddress());
1747   else if (ACPV->isExtSymbol())
1748     NewCPV = ARMConstantPoolSymbol::
1749       Create(MF.getFunction().getContext(),
1750              cast<ARMConstantPoolSymbol>(ACPV)->getSymbol(), PCLabelId, 4);
1751   else if (ACPV->isBlockAddress())
1752     NewCPV = ARMConstantPoolConstant::
1753       Create(cast<ARMConstantPoolConstant>(ACPV)->getBlockAddress(), PCLabelId,
1754              ARMCP::CPBlockAddress, 4);
1755   else if (ACPV->isLSDA())
1756     NewCPV = ARMConstantPoolConstant::Create(&MF.getFunction(), PCLabelId,
1757                                              ARMCP::CPLSDA, 4);
1758   else if (ACPV->isMachineBasicBlock())
1759     NewCPV = ARMConstantPoolMBB::
1760       Create(MF.getFunction().getContext(),
1761              cast<ARMConstantPoolMBB>(ACPV)->getMBB(), PCLabelId, 4);
1762   else
1763     llvm_unreachable("Unexpected ARM constantpool value type!!");
1764   CPI = MCP->getConstantPoolIndex(NewCPV, MCPE.getAlign());
1765   return PCLabelId;
1766 }
1767 
reMaterialize(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,Register DestReg,unsigned SubIdx,const MachineInstr & Orig,const TargetRegisterInfo & TRI) const1768 void ARMBaseInstrInfo::reMaterialize(MachineBasicBlock &MBB,
1769                                      MachineBasicBlock::iterator I,
1770                                      Register DestReg, unsigned SubIdx,
1771                                      const MachineInstr &Orig,
1772                                      const TargetRegisterInfo &TRI) const {
1773   unsigned Opcode = Orig.getOpcode();
1774   switch (Opcode) {
1775   default: {
1776     MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
1777     MI->substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI);
1778     MBB.insert(I, MI);
1779     break;
1780   }
1781   case ARM::tLDRpci_pic:
1782   case ARM::t2LDRpci_pic: {
1783     MachineFunction &MF = *MBB.getParent();
1784     unsigned CPI = Orig.getOperand(1).getIndex();
1785     unsigned PCLabelId = duplicateCPV(MF, CPI);
1786     BuildMI(MBB, I, Orig.getDebugLoc(), get(Opcode), DestReg)
1787         .addConstantPoolIndex(CPI)
1788         .addImm(PCLabelId)
1789         .cloneMemRefs(Orig);
1790     break;
1791   }
1792   }
1793 }
1794 
1795 MachineInstr &
duplicate(MachineBasicBlock & MBB,MachineBasicBlock::iterator InsertBefore,const MachineInstr & Orig) const1796 ARMBaseInstrInfo::duplicate(MachineBasicBlock &MBB,
1797     MachineBasicBlock::iterator InsertBefore,
1798     const MachineInstr &Orig) const {
1799   MachineInstr &Cloned = TargetInstrInfo::duplicate(MBB, InsertBefore, Orig);
1800   MachineBasicBlock::instr_iterator I = Cloned.getIterator();
1801   for (;;) {
1802     switch (I->getOpcode()) {
1803     case ARM::tLDRpci_pic:
1804     case ARM::t2LDRpci_pic: {
1805       MachineFunction &MF = *MBB.getParent();
1806       unsigned CPI = I->getOperand(1).getIndex();
1807       unsigned PCLabelId = duplicateCPV(MF, CPI);
1808       I->getOperand(1).setIndex(CPI);
1809       I->getOperand(2).setImm(PCLabelId);
1810       break;
1811     }
1812     }
1813     if (!I->isBundledWithSucc())
1814       break;
1815     ++I;
1816   }
1817   return Cloned;
1818 }
1819 
produceSameValue(const MachineInstr & MI0,const MachineInstr & MI1,const MachineRegisterInfo * MRI) const1820 bool ARMBaseInstrInfo::produceSameValue(const MachineInstr &MI0,
1821                                         const MachineInstr &MI1,
1822                                         const MachineRegisterInfo *MRI) const {
1823   unsigned Opcode = MI0.getOpcode();
1824   if (Opcode == ARM::t2LDRpci ||
1825       Opcode == ARM::t2LDRpci_pic ||
1826       Opcode == ARM::tLDRpci ||
1827       Opcode == ARM::tLDRpci_pic ||
1828       Opcode == ARM::LDRLIT_ga_pcrel ||
1829       Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1830       Opcode == ARM::tLDRLIT_ga_pcrel ||
1831       Opcode == ARM::MOV_ga_pcrel ||
1832       Opcode == ARM::MOV_ga_pcrel_ldr ||
1833       Opcode == ARM::t2MOV_ga_pcrel) {
1834     if (MI1.getOpcode() != Opcode)
1835       return false;
1836     if (MI0.getNumOperands() != MI1.getNumOperands())
1837       return false;
1838 
1839     const MachineOperand &MO0 = MI0.getOperand(1);
1840     const MachineOperand &MO1 = MI1.getOperand(1);
1841     if (MO0.getOffset() != MO1.getOffset())
1842       return false;
1843 
1844     if (Opcode == ARM::LDRLIT_ga_pcrel ||
1845         Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1846         Opcode == ARM::tLDRLIT_ga_pcrel ||
1847         Opcode == ARM::MOV_ga_pcrel ||
1848         Opcode == ARM::MOV_ga_pcrel_ldr ||
1849         Opcode == ARM::t2MOV_ga_pcrel)
1850       // Ignore the PC labels.
1851       return MO0.getGlobal() == MO1.getGlobal();
1852 
1853     const MachineFunction *MF = MI0.getParent()->getParent();
1854     const MachineConstantPool *MCP = MF->getConstantPool();
1855     int CPI0 = MO0.getIndex();
1856     int CPI1 = MO1.getIndex();
1857     const MachineConstantPoolEntry &MCPE0 = MCP->getConstants()[CPI0];
1858     const MachineConstantPoolEntry &MCPE1 = MCP->getConstants()[CPI1];
1859     bool isARMCP0 = MCPE0.isMachineConstantPoolEntry();
1860     bool isARMCP1 = MCPE1.isMachineConstantPoolEntry();
1861     if (isARMCP0 && isARMCP1) {
1862       ARMConstantPoolValue *ACPV0 =
1863         static_cast<ARMConstantPoolValue*>(MCPE0.Val.MachineCPVal);
1864       ARMConstantPoolValue *ACPV1 =
1865         static_cast<ARMConstantPoolValue*>(MCPE1.Val.MachineCPVal);
1866       return ACPV0->hasSameValue(ACPV1);
1867     } else if (!isARMCP0 && !isARMCP1) {
1868       return MCPE0.Val.ConstVal == MCPE1.Val.ConstVal;
1869     }
1870     return false;
1871   } else if (Opcode == ARM::PICLDR) {
1872     if (MI1.getOpcode() != Opcode)
1873       return false;
1874     if (MI0.getNumOperands() != MI1.getNumOperands())
1875       return false;
1876 
1877     Register Addr0 = MI0.getOperand(1).getReg();
1878     Register Addr1 = MI1.getOperand(1).getReg();
1879     if (Addr0 != Addr1) {
1880       if (!MRI || !Register::isVirtualRegister(Addr0) ||
1881           !Register::isVirtualRegister(Addr1))
1882         return false;
1883 
1884       // This assumes SSA form.
1885       MachineInstr *Def0 = MRI->getVRegDef(Addr0);
1886       MachineInstr *Def1 = MRI->getVRegDef(Addr1);
1887       // Check if the loaded value, e.g. a constantpool of a global address, are
1888       // the same.
1889       if (!produceSameValue(*Def0, *Def1, MRI))
1890         return false;
1891     }
1892 
1893     for (unsigned i = 3, e = MI0.getNumOperands(); i != e; ++i) {
1894       // %12 = PICLDR %11, 0, 14, %noreg
1895       const MachineOperand &MO0 = MI0.getOperand(i);
1896       const MachineOperand &MO1 = MI1.getOperand(i);
1897       if (!MO0.isIdenticalTo(MO1))
1898         return false;
1899     }
1900     return true;
1901   }
1902 
1903   return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs);
1904 }
1905 
1906 /// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to
1907 /// determine if two loads are loading from the same base address. It should
1908 /// only return true if the base pointers are the same and the only differences
1909 /// between the two addresses is the offset. It also returns the offsets by
1910 /// reference.
1911 ///
1912 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1913 /// is permanently disabled.
areLoadsFromSameBasePtr(SDNode * Load1,SDNode * Load2,int64_t & Offset1,int64_t & Offset2) const1914 bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
1915                                                int64_t &Offset1,
1916                                                int64_t &Offset2) const {
1917   // Don't worry about Thumb: just ARM and Thumb2.
1918   if (Subtarget.isThumb1Only()) return false;
1919 
1920   if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
1921     return false;
1922 
1923   switch (Load1->getMachineOpcode()) {
1924   default:
1925     return false;
1926   case ARM::LDRi12:
1927   case ARM::LDRBi12:
1928   case ARM::LDRD:
1929   case ARM::LDRH:
1930   case ARM::LDRSB:
1931   case ARM::LDRSH:
1932   case ARM::VLDRD:
1933   case ARM::VLDRS:
1934   case ARM::t2LDRi8:
1935   case ARM::t2LDRBi8:
1936   case ARM::t2LDRDi8:
1937   case ARM::t2LDRSHi8:
1938   case ARM::t2LDRi12:
1939   case ARM::t2LDRBi12:
1940   case ARM::t2LDRSHi12:
1941     break;
1942   }
1943 
1944   switch (Load2->getMachineOpcode()) {
1945   default:
1946     return false;
1947   case ARM::LDRi12:
1948   case ARM::LDRBi12:
1949   case ARM::LDRD:
1950   case ARM::LDRH:
1951   case ARM::LDRSB:
1952   case ARM::LDRSH:
1953   case ARM::VLDRD:
1954   case ARM::VLDRS:
1955   case ARM::t2LDRi8:
1956   case ARM::t2LDRBi8:
1957   case ARM::t2LDRSHi8:
1958   case ARM::t2LDRi12:
1959   case ARM::t2LDRBi12:
1960   case ARM::t2LDRSHi12:
1961     break;
1962   }
1963 
1964   // Check if base addresses and chain operands match.
1965   if (Load1->getOperand(0) != Load2->getOperand(0) ||
1966       Load1->getOperand(4) != Load2->getOperand(4))
1967     return false;
1968 
1969   // Index should be Reg0.
1970   if (Load1->getOperand(3) != Load2->getOperand(3))
1971     return false;
1972 
1973   // Determine the offsets.
1974   if (isa<ConstantSDNode>(Load1->getOperand(1)) &&
1975       isa<ConstantSDNode>(Load2->getOperand(1))) {
1976     Offset1 = cast<ConstantSDNode>(Load1->getOperand(1))->getSExtValue();
1977     Offset2 = cast<ConstantSDNode>(Load2->getOperand(1))->getSExtValue();
1978     return true;
1979   }
1980 
1981   return false;
1982 }
1983 
1984 /// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
1985 /// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should
1986 /// be scheduled togther. On some targets if two loads are loading from
1987 /// addresses in the same cache line, it's better if they are scheduled
1988 /// together. This function takes two integers that represent the load offsets
1989 /// from the common base address. It returns true if it decides it's desirable
1990 /// to schedule the two loads together. "NumLoads" is the number of loads that
1991 /// have already been scheduled after Load1.
1992 ///
1993 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1994 /// is permanently disabled.
shouldScheduleLoadsNear(SDNode * Load1,SDNode * Load2,int64_t Offset1,int64_t Offset2,unsigned NumLoads) const1995 bool ARMBaseInstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
1996                                                int64_t Offset1, int64_t Offset2,
1997                                                unsigned NumLoads) const {
1998   // Don't worry about Thumb: just ARM and Thumb2.
1999   if (Subtarget.isThumb1Only()) return false;
2000 
2001   assert(Offset2 > Offset1);
2002 
2003   if ((Offset2 - Offset1) / 8 > 64)
2004     return false;
2005 
2006   // Check if the machine opcodes are different. If they are different
2007   // then we consider them to not be of the same base address,
2008   // EXCEPT in the case of Thumb2 byte loads where one is LDRBi8 and the other LDRBi12.
2009   // In this case, they are considered to be the same because they are different
2010   // encoding forms of the same basic instruction.
2011   if ((Load1->getMachineOpcode() != Load2->getMachineOpcode()) &&
2012       !((Load1->getMachineOpcode() == ARM::t2LDRBi8 &&
2013          Load2->getMachineOpcode() == ARM::t2LDRBi12) ||
2014         (Load1->getMachineOpcode() == ARM::t2LDRBi12 &&
2015          Load2->getMachineOpcode() == ARM::t2LDRBi8)))
2016     return false;  // FIXME: overly conservative?
2017 
2018   // Four loads in a row should be sufficient.
2019   if (NumLoads >= 3)
2020     return false;
2021 
2022   return true;
2023 }
2024 
isSchedulingBoundary(const MachineInstr & MI,const MachineBasicBlock * MBB,const MachineFunction & MF) const2025 bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2026                                             const MachineBasicBlock *MBB,
2027                                             const MachineFunction &MF) const {
2028   // Debug info is never a scheduling boundary. It's necessary to be explicit
2029   // due to the special treatment of IT instructions below, otherwise a
2030   // dbg_value followed by an IT will result in the IT instruction being
2031   // considered a scheduling hazard, which is wrong. It should be the actual
2032   // instruction preceding the dbg_value instruction(s), just like it is
2033   // when debug info is not present.
2034   if (MI.isDebugInstr())
2035     return false;
2036 
2037   // Terminators and labels can't be scheduled around.
2038   if (MI.isTerminator() || MI.isPosition())
2039     return true;
2040 
2041   // INLINEASM_BR can jump to another block
2042   if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
2043     return true;
2044 
2045   // Treat the start of the IT block as a scheduling boundary, but schedule
2046   // t2IT along with all instructions following it.
2047   // FIXME: This is a big hammer. But the alternative is to add all potential
2048   // true and anti dependencies to IT block instructions as implicit operands
2049   // to the t2IT instruction. The added compile time and complexity does not
2050   // seem worth it.
2051   MachineBasicBlock::const_iterator I = MI;
2052   // Make sure to skip any debug instructions
2053   while (++I != MBB->end() && I->isDebugInstr())
2054     ;
2055   if (I != MBB->end() && I->getOpcode() == ARM::t2IT)
2056     return true;
2057 
2058   // Don't attempt to schedule around any instruction that defines
2059   // a stack-oriented pointer, as it's unlikely to be profitable. This
2060   // saves compile time, because it doesn't require every single
2061   // stack slot reference to depend on the instruction that does the
2062   // modification.
2063   // Calls don't actually change the stack pointer, even if they have imp-defs.
2064   // No ARM calling conventions change the stack pointer. (X86 calling
2065   // conventions sometimes do).
2066   if (!MI.isCall() && MI.definesRegister(ARM::SP))
2067     return true;
2068 
2069   return false;
2070 }
2071 
2072 bool ARMBaseInstrInfo::
isProfitableToIfCvt(MachineBasicBlock & MBB,unsigned NumCycles,unsigned ExtraPredCycles,BranchProbability Probability) const2073 isProfitableToIfCvt(MachineBasicBlock &MBB,
2074                     unsigned NumCycles, unsigned ExtraPredCycles,
2075                     BranchProbability Probability) const {
2076   if (!NumCycles)
2077     return false;
2078 
2079   // If we are optimizing for size, see if the branch in the predecessor can be
2080   // lowered to cbn?z by the constant island lowering pass, and return false if
2081   // so. This results in a shorter instruction sequence.
2082   if (MBB.getParent()->getFunction().hasOptSize()) {
2083     MachineBasicBlock *Pred = *MBB.pred_begin();
2084     if (!Pred->empty()) {
2085       MachineInstr *LastMI = &*Pred->rbegin();
2086       if (LastMI->getOpcode() == ARM::t2Bcc) {
2087         const TargetRegisterInfo *TRI = &getRegisterInfo();
2088         MachineInstr *CmpMI = findCMPToFoldIntoCBZ(LastMI, TRI);
2089         if (CmpMI)
2090           return false;
2091       }
2092     }
2093   }
2094   return isProfitableToIfCvt(MBB, NumCycles, ExtraPredCycles,
2095                              MBB, 0, 0, Probability);
2096 }
2097 
2098 bool ARMBaseInstrInfo::
isProfitableToIfCvt(MachineBasicBlock & TBB,unsigned TCycles,unsigned TExtra,MachineBasicBlock & FBB,unsigned FCycles,unsigned FExtra,BranchProbability Probability) const2099 isProfitableToIfCvt(MachineBasicBlock &TBB,
2100                     unsigned TCycles, unsigned TExtra,
2101                     MachineBasicBlock &FBB,
2102                     unsigned FCycles, unsigned FExtra,
2103                     BranchProbability Probability) const {
2104   if (!TCycles)
2105     return false;
2106 
2107   // In thumb code we often end up trading one branch for a IT block, and
2108   // if we are cloning the instruction can increase code size. Prevent
2109   // blocks with multiple predecesors from being ifcvted to prevent this
2110   // cloning.
2111   if (Subtarget.isThumb2() && TBB.getParent()->getFunction().hasMinSize()) {
2112     if (TBB.pred_size() != 1 || FBB.pred_size() != 1)
2113       return false;
2114   }
2115 
2116   // Attempt to estimate the relative costs of predication versus branching.
2117   // Here we scale up each component of UnpredCost to avoid precision issue when
2118   // scaling TCycles/FCycles by Probability.
2119   const unsigned ScalingUpFactor = 1024;
2120 
2121   unsigned PredCost = (TCycles + FCycles + TExtra + FExtra) * ScalingUpFactor;
2122   unsigned UnpredCost;
2123   if (!Subtarget.hasBranchPredictor()) {
2124     // When we don't have a branch predictor it's always cheaper to not take a
2125     // branch than take it, so we have to take that into account.
2126     unsigned NotTakenBranchCost = 1;
2127     unsigned TakenBranchCost = Subtarget.getMispredictionPenalty();
2128     unsigned TUnpredCycles, FUnpredCycles;
2129     if (!FCycles) {
2130       // Triangle: TBB is the fallthrough
2131       TUnpredCycles = TCycles + NotTakenBranchCost;
2132       FUnpredCycles = TakenBranchCost;
2133     } else {
2134       // Diamond: TBB is the block that is branched to, FBB is the fallthrough
2135       TUnpredCycles = TCycles + TakenBranchCost;
2136       FUnpredCycles = FCycles + NotTakenBranchCost;
2137       // The branch at the end of FBB will disappear when it's predicated, so
2138       // discount it from PredCost.
2139       PredCost -= 1 * ScalingUpFactor;
2140     }
2141     // The total cost is the cost of each path scaled by their probabilites
2142     unsigned TUnpredCost = Probability.scale(TUnpredCycles * ScalingUpFactor);
2143     unsigned FUnpredCost = Probability.getCompl().scale(FUnpredCycles * ScalingUpFactor);
2144     UnpredCost = TUnpredCost + FUnpredCost;
2145     // When predicating assume that the first IT can be folded away but later
2146     // ones cost one cycle each
2147     if (Subtarget.isThumb2() && TCycles + FCycles > 4) {
2148       PredCost += ((TCycles + FCycles - 4) / 4) * ScalingUpFactor;
2149     }
2150   } else {
2151     unsigned TUnpredCost = Probability.scale(TCycles * ScalingUpFactor);
2152     unsigned FUnpredCost =
2153       Probability.getCompl().scale(FCycles * ScalingUpFactor);
2154     UnpredCost = TUnpredCost + FUnpredCost;
2155     UnpredCost += 1 * ScalingUpFactor; // The branch itself
2156     UnpredCost += Subtarget.getMispredictionPenalty() * ScalingUpFactor / 10;
2157   }
2158 
2159   return PredCost <= UnpredCost;
2160 }
2161 
2162 unsigned
extraSizeToPredicateInstructions(const MachineFunction & MF,unsigned NumInsts) const2163 ARMBaseInstrInfo::extraSizeToPredicateInstructions(const MachineFunction &MF,
2164                                                    unsigned NumInsts) const {
2165   // Thumb2 needs a 2-byte IT instruction to predicate up to 4 instructions.
2166   // ARM has a condition code field in every predicable instruction, using it
2167   // doesn't change code size.
2168   if (!Subtarget.isThumb2())
2169     return 0;
2170 
2171   // It's possible that the size of the IT is restricted to a single block.
2172   unsigned MaxInsts = Subtarget.restrictIT() ? 1 : 4;
2173   return divideCeil(NumInsts, MaxInsts) * 2;
2174 }
2175 
2176 unsigned
predictBranchSizeForIfCvt(MachineInstr & MI) const2177 ARMBaseInstrInfo::predictBranchSizeForIfCvt(MachineInstr &MI) const {
2178   // If this branch is likely to be folded into the comparison to form a
2179   // CB(N)Z, then removing it won't reduce code size at all, because that will
2180   // just replace the CB(N)Z with a CMP.
2181   if (MI.getOpcode() == ARM::t2Bcc &&
2182       findCMPToFoldIntoCBZ(&MI, &getRegisterInfo()))
2183     return 0;
2184 
2185   unsigned Size = getInstSizeInBytes(MI);
2186 
2187   // For Thumb2, all branches are 32-bit instructions during the if conversion
2188   // pass, but may be replaced with 16-bit instructions during size reduction.
2189   // Since the branches considered by if conversion tend to be forward branches
2190   // over small basic blocks, they are very likely to be in range for the
2191   // narrow instructions, so we assume the final code size will be half what it
2192   // currently is.
2193   if (Subtarget.isThumb2())
2194     Size /= 2;
2195 
2196   return Size;
2197 }
2198 
2199 bool
isProfitableToUnpredicate(MachineBasicBlock & TMBB,MachineBasicBlock & FMBB) const2200 ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
2201                                             MachineBasicBlock &FMBB) const {
2202   // Reduce false anti-dependencies to let the target's out-of-order execution
2203   // engine do its thing.
2204   return Subtarget.isProfitableToUnpredicate();
2205 }
2206 
2207 /// getInstrPredicate - If instruction is predicated, returns its predicate
2208 /// condition, otherwise returns AL. It also returns the condition code
2209 /// register by reference.
getInstrPredicate(const MachineInstr & MI,Register & PredReg)2210 ARMCC::CondCodes llvm::getInstrPredicate(const MachineInstr &MI,
2211                                          Register &PredReg) {
2212   int PIdx = MI.findFirstPredOperandIdx();
2213   if (PIdx == -1) {
2214     PredReg = 0;
2215     return ARMCC::AL;
2216   }
2217 
2218   PredReg = MI.getOperand(PIdx+1).getReg();
2219   return (ARMCC::CondCodes)MI.getOperand(PIdx).getImm();
2220 }
2221 
getMatchingCondBranchOpcode(unsigned Opc)2222 unsigned llvm::getMatchingCondBranchOpcode(unsigned Opc) {
2223   if (Opc == ARM::B)
2224     return ARM::Bcc;
2225   if (Opc == ARM::tB)
2226     return ARM::tBcc;
2227   if (Opc == ARM::t2B)
2228     return ARM::t2Bcc;
2229 
2230   llvm_unreachable("Unknown unconditional branch opcode!");
2231 }
2232 
commuteInstructionImpl(MachineInstr & MI,bool NewMI,unsigned OpIdx1,unsigned OpIdx2) const2233 MachineInstr *ARMBaseInstrInfo::commuteInstructionImpl(MachineInstr &MI,
2234                                                        bool NewMI,
2235                                                        unsigned OpIdx1,
2236                                                        unsigned OpIdx2) const {
2237   switch (MI.getOpcode()) {
2238   case ARM::MOVCCr:
2239   case ARM::t2MOVCCr: {
2240     // MOVCC can be commuted by inverting the condition.
2241     Register PredReg;
2242     ARMCC::CondCodes CC = getInstrPredicate(MI, PredReg);
2243     // MOVCC AL can't be inverted. Shouldn't happen.
2244     if (CC == ARMCC::AL || PredReg != ARM::CPSR)
2245       return nullptr;
2246     MachineInstr *CommutedMI =
2247         TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2248     if (!CommutedMI)
2249       return nullptr;
2250     // After swapping the MOVCC operands, also invert the condition.
2251     CommutedMI->getOperand(CommutedMI->findFirstPredOperandIdx())
2252         .setImm(ARMCC::getOppositeCondition(CC));
2253     return CommutedMI;
2254   }
2255   }
2256   return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2257 }
2258 
2259 /// Identify instructions that can be folded into a MOVCC instruction, and
2260 /// return the defining instruction.
2261 MachineInstr *
canFoldIntoMOVCC(Register Reg,const MachineRegisterInfo & MRI,const TargetInstrInfo * TII) const2262 ARMBaseInstrInfo::canFoldIntoMOVCC(Register Reg, const MachineRegisterInfo &MRI,
2263                                    const TargetInstrInfo *TII) const {
2264   if (!Reg.isVirtual())
2265     return nullptr;
2266   if (!MRI.hasOneNonDBGUse(Reg))
2267     return nullptr;
2268   MachineInstr *MI = MRI.getVRegDef(Reg);
2269   if (!MI)
2270     return nullptr;
2271   // Check if MI can be predicated and folded into the MOVCC.
2272   if (!isPredicable(*MI))
2273     return nullptr;
2274   // Check if MI has any non-dead defs or physreg uses. This also detects
2275   // predicated instructions which will be reading CPSR.
2276   for (unsigned i = 1, e = MI->getNumOperands(); i != e; ++i) {
2277     const MachineOperand &MO = MI->getOperand(i);
2278     // Reject frame index operands, PEI can't handle the predicated pseudos.
2279     if (MO.isFI() || MO.isCPI() || MO.isJTI())
2280       return nullptr;
2281     if (!MO.isReg())
2282       continue;
2283     // MI can't have any tied operands, that would conflict with predication.
2284     if (MO.isTied())
2285       return nullptr;
2286     if (Register::isPhysicalRegister(MO.getReg()))
2287       return nullptr;
2288     if (MO.isDef() && !MO.isDead())
2289       return nullptr;
2290   }
2291   bool DontMoveAcrossStores = true;
2292   if (!MI->isSafeToMove(/* AliasAnalysis = */ nullptr, DontMoveAcrossStores))
2293     return nullptr;
2294   return MI;
2295 }
2296 
analyzeSelect(const MachineInstr & MI,SmallVectorImpl<MachineOperand> & Cond,unsigned & TrueOp,unsigned & FalseOp,bool & Optimizable) const2297 bool ARMBaseInstrInfo::analyzeSelect(const MachineInstr &MI,
2298                                      SmallVectorImpl<MachineOperand> &Cond,
2299                                      unsigned &TrueOp, unsigned &FalseOp,
2300                                      bool &Optimizable) const {
2301   assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2302          "Unknown select instruction");
2303   // MOVCC operands:
2304   // 0: Def.
2305   // 1: True use.
2306   // 2: False use.
2307   // 3: Condition code.
2308   // 4: CPSR use.
2309   TrueOp = 1;
2310   FalseOp = 2;
2311   Cond.push_back(MI.getOperand(3));
2312   Cond.push_back(MI.getOperand(4));
2313   // We can always fold a def.
2314   Optimizable = true;
2315   return false;
2316 }
2317 
2318 MachineInstr *
optimizeSelect(MachineInstr & MI,SmallPtrSetImpl<MachineInstr * > & SeenMIs,bool PreferFalse) const2319 ARMBaseInstrInfo::optimizeSelect(MachineInstr &MI,
2320                                  SmallPtrSetImpl<MachineInstr *> &SeenMIs,
2321                                  bool PreferFalse) const {
2322   assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2323          "Unknown select instruction");
2324   MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2325   MachineInstr *DefMI = canFoldIntoMOVCC(MI.getOperand(2).getReg(), MRI, this);
2326   bool Invert = !DefMI;
2327   if (!DefMI)
2328     DefMI = canFoldIntoMOVCC(MI.getOperand(1).getReg(), MRI, this);
2329   if (!DefMI)
2330     return nullptr;
2331 
2332   // Find new register class to use.
2333   MachineOperand FalseReg = MI.getOperand(Invert ? 2 : 1);
2334   Register DestReg = MI.getOperand(0).getReg();
2335   const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());
2336   if (!MRI.constrainRegClass(DestReg, PreviousClass))
2337     return nullptr;
2338 
2339   // Create a new predicated version of DefMI.
2340   // Rfalse is the first use.
2341   MachineInstrBuilder NewMI =
2342       BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), DefMI->getDesc(), DestReg);
2343 
2344   // Copy all the DefMI operands, excluding its (null) predicate.
2345   const MCInstrDesc &DefDesc = DefMI->getDesc();
2346   for (unsigned i = 1, e = DefDesc.getNumOperands();
2347        i != e && !DefDesc.OpInfo[i].isPredicate(); ++i)
2348     NewMI.add(DefMI->getOperand(i));
2349 
2350   unsigned CondCode = MI.getOperand(3).getImm();
2351   if (Invert)
2352     NewMI.addImm(ARMCC::getOppositeCondition(ARMCC::CondCodes(CondCode)));
2353   else
2354     NewMI.addImm(CondCode);
2355   NewMI.add(MI.getOperand(4));
2356 
2357   // DefMI is not the -S version that sets CPSR, so add an optional %noreg.
2358   if (NewMI->hasOptionalDef())
2359     NewMI.add(condCodeOp());
2360 
2361   // The output register value when the predicate is false is an implicit
2362   // register operand tied to the first def.
2363   // The tie makes the register allocator ensure the FalseReg is allocated the
2364   // same register as operand 0.
2365   FalseReg.setImplicit();
2366   NewMI.add(FalseReg);
2367   NewMI->tieOperands(0, NewMI->getNumOperands() - 1);
2368 
2369   // Update SeenMIs set: register newly created MI and erase removed DefMI.
2370   SeenMIs.insert(NewMI);
2371   SeenMIs.erase(DefMI);
2372 
2373   // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
2374   // DefMI would be invalid when tranferred inside the loop.  Checking for a
2375   // loop is expensive, but at least remove kill flags if they are in different
2376   // BBs.
2377   if (DefMI->getParent() != MI.getParent())
2378     NewMI->clearKillInfo();
2379 
2380   // The caller will erase MI, but not DefMI.
2381   DefMI->eraseFromParent();
2382   return NewMI;
2383 }
2384 
2385 /// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the
2386 /// instruction is encoded with an 'S' bit is determined by the optional CPSR
2387 /// def operand.
2388 ///
2389 /// This will go away once we can teach tblgen how to set the optional CPSR def
2390 /// operand itself.
2391 struct AddSubFlagsOpcodePair {
2392   uint16_t PseudoOpc;
2393   uint16_t MachineOpc;
2394 };
2395 
2396 static const AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = {
2397   {ARM::ADDSri, ARM::ADDri},
2398   {ARM::ADDSrr, ARM::ADDrr},
2399   {ARM::ADDSrsi, ARM::ADDrsi},
2400   {ARM::ADDSrsr, ARM::ADDrsr},
2401 
2402   {ARM::SUBSri, ARM::SUBri},
2403   {ARM::SUBSrr, ARM::SUBrr},
2404   {ARM::SUBSrsi, ARM::SUBrsi},
2405   {ARM::SUBSrsr, ARM::SUBrsr},
2406 
2407   {ARM::RSBSri, ARM::RSBri},
2408   {ARM::RSBSrsi, ARM::RSBrsi},
2409   {ARM::RSBSrsr, ARM::RSBrsr},
2410 
2411   {ARM::tADDSi3, ARM::tADDi3},
2412   {ARM::tADDSi8, ARM::tADDi8},
2413   {ARM::tADDSrr, ARM::tADDrr},
2414   {ARM::tADCS, ARM::tADC},
2415 
2416   {ARM::tSUBSi3, ARM::tSUBi3},
2417   {ARM::tSUBSi8, ARM::tSUBi8},
2418   {ARM::tSUBSrr, ARM::tSUBrr},
2419   {ARM::tSBCS, ARM::tSBC},
2420   {ARM::tRSBS, ARM::tRSB},
2421   {ARM::tLSLSri, ARM::tLSLri},
2422 
2423   {ARM::t2ADDSri, ARM::t2ADDri},
2424   {ARM::t2ADDSrr, ARM::t2ADDrr},
2425   {ARM::t2ADDSrs, ARM::t2ADDrs},
2426 
2427   {ARM::t2SUBSri, ARM::t2SUBri},
2428   {ARM::t2SUBSrr, ARM::t2SUBrr},
2429   {ARM::t2SUBSrs, ARM::t2SUBrs},
2430 
2431   {ARM::t2RSBSri, ARM::t2RSBri},
2432   {ARM::t2RSBSrs, ARM::t2RSBrs},
2433 };
2434 
convertAddSubFlagsOpcode(unsigned OldOpc)2435 unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) {
2436   for (unsigned i = 0, e = array_lengthof(AddSubFlagsOpcodeMap); i != e; ++i)
2437     if (OldOpc == AddSubFlagsOpcodeMap[i].PseudoOpc)
2438       return AddSubFlagsOpcodeMap[i].MachineOpc;
2439   return 0;
2440 }
2441 
emitARMRegPlusImmediate(MachineBasicBlock & MBB,MachineBasicBlock::iterator & MBBI,const DebugLoc & dl,Register DestReg,Register BaseReg,int NumBytes,ARMCC::CondCodes Pred,Register PredReg,const ARMBaseInstrInfo & TII,unsigned MIFlags)2442 void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB,
2443                                    MachineBasicBlock::iterator &MBBI,
2444                                    const DebugLoc &dl, Register DestReg,
2445                                    Register BaseReg, int NumBytes,
2446                                    ARMCC::CondCodes Pred, Register PredReg,
2447                                    const ARMBaseInstrInfo &TII,
2448                                    unsigned MIFlags) {
2449   if (NumBytes == 0 && DestReg != BaseReg) {
2450     BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), DestReg)
2451         .addReg(BaseReg, RegState::Kill)
2452         .add(predOps(Pred, PredReg))
2453         .add(condCodeOp())
2454         .setMIFlags(MIFlags);
2455     return;
2456   }
2457 
2458   bool isSub = NumBytes < 0;
2459   if (isSub) NumBytes = -NumBytes;
2460 
2461   while (NumBytes) {
2462     unsigned RotAmt = ARM_AM::getSOImmValRotate(NumBytes);
2463     unsigned ThisVal = NumBytes & ARM_AM::rotr32(0xFF, RotAmt);
2464     assert(ThisVal && "Didn't extract field correctly");
2465 
2466     // We will handle these bits from offset, clear them.
2467     NumBytes &= ~ThisVal;
2468 
2469     assert(ARM_AM::getSOImmVal(ThisVal) != -1 && "Bit extraction didn't work?");
2470 
2471     // Build the new ADD / SUB.
2472     unsigned Opc = isSub ? ARM::SUBri : ARM::ADDri;
2473     BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg)
2474         .addReg(BaseReg, RegState::Kill)
2475         .addImm(ThisVal)
2476         .add(predOps(Pred, PredReg))
2477         .add(condCodeOp())
2478         .setMIFlags(MIFlags);
2479     BaseReg = DestReg;
2480   }
2481 }
2482 
tryFoldSPUpdateIntoPushPop(const ARMSubtarget & Subtarget,MachineFunction & MF,MachineInstr * MI,unsigned NumBytes)2483 bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
2484                                       MachineFunction &MF, MachineInstr *MI,
2485                                       unsigned NumBytes) {
2486   // This optimisation potentially adds lots of load and store
2487   // micro-operations, it's only really a great benefit to code-size.
2488   if (!Subtarget.hasMinSize())
2489     return false;
2490 
2491   // If only one register is pushed/popped, LLVM can use an LDR/STR
2492   // instead. We can't modify those so make sure we're dealing with an
2493   // instruction we understand.
2494   bool IsPop = isPopOpcode(MI->getOpcode());
2495   bool IsPush = isPushOpcode(MI->getOpcode());
2496   if (!IsPush && !IsPop)
2497     return false;
2498 
2499   bool IsVFPPushPop = MI->getOpcode() == ARM::VSTMDDB_UPD ||
2500                       MI->getOpcode() == ARM::VLDMDIA_UPD;
2501   bool IsT1PushPop = MI->getOpcode() == ARM::tPUSH ||
2502                      MI->getOpcode() == ARM::tPOP ||
2503                      MI->getOpcode() == ARM::tPOP_RET;
2504 
2505   assert((IsT1PushPop || (MI->getOperand(0).getReg() == ARM::SP &&
2506                           MI->getOperand(1).getReg() == ARM::SP)) &&
2507          "trying to fold sp update into non-sp-updating push/pop");
2508 
2509   // The VFP push & pop act on D-registers, so we can only fold an adjustment
2510   // by a multiple of 8 bytes in correctly. Similarly rN is 4-bytes. Don't try
2511   // if this is violated.
2512   if (NumBytes % (IsVFPPushPop ? 8 : 4) != 0)
2513     return false;
2514 
2515   // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
2516   // pred) so the list starts at 4. Thumb1 starts after the predicate.
2517   int RegListIdx = IsT1PushPop ? 2 : 4;
2518 
2519   // Calculate the space we'll need in terms of registers.
2520   unsigned RegsNeeded;
2521   const TargetRegisterClass *RegClass;
2522   if (IsVFPPushPop) {
2523     RegsNeeded = NumBytes / 8;
2524     RegClass = &ARM::DPRRegClass;
2525   } else {
2526     RegsNeeded = NumBytes / 4;
2527     RegClass = &ARM::GPRRegClass;
2528   }
2529 
2530   // We're going to have to strip all list operands off before
2531   // re-adding them since the order matters, so save the existing ones
2532   // for later.
2533   SmallVector<MachineOperand, 4> RegList;
2534 
2535   // We're also going to need the first register transferred by this
2536   // instruction, which won't necessarily be the first register in the list.
2537   unsigned FirstRegEnc = -1;
2538 
2539   const TargetRegisterInfo *TRI = MF.getRegInfo().getTargetRegisterInfo();
2540   for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i) {
2541     MachineOperand &MO = MI->getOperand(i);
2542     RegList.push_back(MO);
2543 
2544     if (MO.isReg() && !MO.isImplicit() &&
2545         TRI->getEncodingValue(MO.getReg()) < FirstRegEnc)
2546       FirstRegEnc = TRI->getEncodingValue(MO.getReg());
2547   }
2548 
2549   const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
2550 
2551   // Now try to find enough space in the reglist to allocate NumBytes.
2552   for (int CurRegEnc = FirstRegEnc - 1; CurRegEnc >= 0 && RegsNeeded;
2553        --CurRegEnc) {
2554     unsigned CurReg = RegClass->getRegister(CurRegEnc);
2555     if (IsT1PushPop && CurRegEnc > TRI->getEncodingValue(ARM::R7))
2556       continue;
2557     if (!IsPop) {
2558       // Pushing any register is completely harmless, mark the register involved
2559       // as undef since we don't care about its value and must not restore it
2560       // during stack unwinding.
2561       RegList.push_back(MachineOperand::CreateReg(CurReg, false, false,
2562                                                   false, false, true));
2563       --RegsNeeded;
2564       continue;
2565     }
2566 
2567     // However, we can only pop an extra register if it's not live. For
2568     // registers live within the function we might clobber a return value
2569     // register; the other way a register can be live here is if it's
2570     // callee-saved.
2571     if (isCalleeSavedRegister(CurReg, CSRegs) ||
2572         MI->getParent()->computeRegisterLiveness(TRI, CurReg, MI) !=
2573         MachineBasicBlock::LQR_Dead) {
2574       // VFP pops don't allow holes in the register list, so any skip is fatal
2575       // for our transformation. GPR pops do, so we should just keep looking.
2576       if (IsVFPPushPop)
2577         return false;
2578       else
2579         continue;
2580     }
2581 
2582     // Mark the unimportant registers as <def,dead> in the POP.
2583     RegList.push_back(MachineOperand::CreateReg(CurReg, true, false, false,
2584                                                 true));
2585     --RegsNeeded;
2586   }
2587 
2588   if (RegsNeeded > 0)
2589     return false;
2590 
2591   // Finally we know we can profitably perform the optimisation so go
2592   // ahead: strip all existing registers off and add them back again
2593   // in the right order.
2594   for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i)
2595     MI->RemoveOperand(i);
2596 
2597   // Add the complete list back in.
2598   MachineInstrBuilder MIB(MF, &*MI);
2599   for (int i = RegList.size() - 1; i >= 0; --i)
2600     MIB.add(RegList[i]);
2601 
2602   return true;
2603 }
2604 
rewriteARMFrameIndex(MachineInstr & MI,unsigned FrameRegIdx,Register FrameReg,int & Offset,const ARMBaseInstrInfo & TII)2605 bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2606                                 Register FrameReg, int &Offset,
2607                                 const ARMBaseInstrInfo &TII) {
2608   unsigned Opcode = MI.getOpcode();
2609   const MCInstrDesc &Desc = MI.getDesc();
2610   unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
2611   bool isSub = false;
2612 
2613   // Memory operands in inline assembly always use AddrMode2.
2614   if (Opcode == ARM::INLINEASM || Opcode == ARM::INLINEASM_BR)
2615     AddrMode = ARMII::AddrMode2;
2616 
2617   if (Opcode == ARM::ADDri) {
2618     Offset += MI.getOperand(FrameRegIdx+1).getImm();
2619     if (Offset == 0) {
2620       // Turn it into a move.
2621       MI.setDesc(TII.get(ARM::MOVr));
2622       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2623       MI.RemoveOperand(FrameRegIdx+1);
2624       Offset = 0;
2625       return true;
2626     } else if (Offset < 0) {
2627       Offset = -Offset;
2628       isSub = true;
2629       MI.setDesc(TII.get(ARM::SUBri));
2630     }
2631 
2632     // Common case: small offset, fits into instruction.
2633     if (ARM_AM::getSOImmVal(Offset) != -1) {
2634       // Replace the FrameIndex with sp / fp
2635       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2636       MI.getOperand(FrameRegIdx+1).ChangeToImmediate(Offset);
2637       Offset = 0;
2638       return true;
2639     }
2640 
2641     // Otherwise, pull as much of the immedidate into this ADDri/SUBri
2642     // as possible.
2643     unsigned RotAmt = ARM_AM::getSOImmValRotate(Offset);
2644     unsigned ThisImmVal = Offset & ARM_AM::rotr32(0xFF, RotAmt);
2645 
2646     // We will handle these bits from offset, clear them.
2647     Offset &= ~ThisImmVal;
2648 
2649     // Get the properly encoded SOImmVal field.
2650     assert(ARM_AM::getSOImmVal(ThisImmVal) != -1 &&
2651            "Bit extraction didn't work?");
2652     MI.getOperand(FrameRegIdx+1).ChangeToImmediate(ThisImmVal);
2653  } else {
2654     unsigned ImmIdx = 0;
2655     int InstrOffs = 0;
2656     unsigned NumBits = 0;
2657     unsigned Scale = 1;
2658     switch (AddrMode) {
2659     case ARMII::AddrMode_i12:
2660       ImmIdx = FrameRegIdx + 1;
2661       InstrOffs = MI.getOperand(ImmIdx).getImm();
2662       NumBits = 12;
2663       break;
2664     case ARMII::AddrMode2:
2665       ImmIdx = FrameRegIdx+2;
2666       InstrOffs = ARM_AM::getAM2Offset(MI.getOperand(ImmIdx).getImm());
2667       if (ARM_AM::getAM2Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2668         InstrOffs *= -1;
2669       NumBits = 12;
2670       break;
2671     case ARMII::AddrMode3:
2672       ImmIdx = FrameRegIdx+2;
2673       InstrOffs = ARM_AM::getAM3Offset(MI.getOperand(ImmIdx).getImm());
2674       if (ARM_AM::getAM3Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2675         InstrOffs *= -1;
2676       NumBits = 8;
2677       break;
2678     case ARMII::AddrMode4:
2679     case ARMII::AddrMode6:
2680       // Can't fold any offset even if it's zero.
2681       return false;
2682     case ARMII::AddrMode5:
2683       ImmIdx = FrameRegIdx+1;
2684       InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm());
2685       if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2686         InstrOffs *= -1;
2687       NumBits = 8;
2688       Scale = 4;
2689       break;
2690     case ARMII::AddrMode5FP16:
2691       ImmIdx = FrameRegIdx+1;
2692       InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm());
2693       if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2694         InstrOffs *= -1;
2695       NumBits = 8;
2696       Scale = 2;
2697       break;
2698     case ARMII::AddrModeT2_i7:
2699     case ARMII::AddrModeT2_i7s2:
2700     case ARMII::AddrModeT2_i7s4:
2701       ImmIdx = FrameRegIdx+1;
2702       InstrOffs = MI.getOperand(ImmIdx).getImm();
2703       NumBits = 7;
2704       Scale = (AddrMode == ARMII::AddrModeT2_i7s2 ? 2 :
2705                AddrMode == ARMII::AddrModeT2_i7s4 ? 4 : 1);
2706       break;
2707     default:
2708       llvm_unreachable("Unsupported addressing mode!");
2709     }
2710 
2711     Offset += InstrOffs * Scale;
2712     assert((Offset & (Scale-1)) == 0 && "Can't encode this offset!");
2713     if (Offset < 0) {
2714       Offset = -Offset;
2715       isSub = true;
2716     }
2717 
2718     // Attempt to fold address comp. if opcode has offset bits
2719     if (NumBits > 0) {
2720       // Common case: small offset, fits into instruction.
2721       MachineOperand &ImmOp = MI.getOperand(ImmIdx);
2722       int ImmedOffset = Offset / Scale;
2723       unsigned Mask = (1 << NumBits) - 1;
2724       if ((unsigned)Offset <= Mask * Scale) {
2725         // Replace the FrameIndex with sp
2726         MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2727         // FIXME: When addrmode2 goes away, this will simplify (like the
2728         // T2 version), as the LDR.i12 versions don't need the encoding
2729         // tricks for the offset value.
2730         if (isSub) {
2731           if (AddrMode == ARMII::AddrMode_i12)
2732             ImmedOffset = -ImmedOffset;
2733           else
2734             ImmedOffset |= 1 << NumBits;
2735         }
2736         ImmOp.ChangeToImmediate(ImmedOffset);
2737         Offset = 0;
2738         return true;
2739       }
2740 
2741       // Otherwise, it didn't fit. Pull in what we can to simplify the immed.
2742       ImmedOffset = ImmedOffset & Mask;
2743       if (isSub) {
2744         if (AddrMode == ARMII::AddrMode_i12)
2745           ImmedOffset = -ImmedOffset;
2746         else
2747           ImmedOffset |= 1 << NumBits;
2748       }
2749       ImmOp.ChangeToImmediate(ImmedOffset);
2750       Offset &= ~(Mask*Scale);
2751     }
2752   }
2753 
2754   Offset = (isSub) ? -Offset : Offset;
2755   return Offset == 0;
2756 }
2757 
2758 /// analyzeCompare - For a comparison instruction, return the source registers
2759 /// in SrcReg and SrcReg2 if having two register operands, and the value it
2760 /// compares against in CmpValue. Return true if the comparison instruction
2761 /// can be analyzed.
analyzeCompare(const MachineInstr & MI,Register & SrcReg,Register & SrcReg2,int & CmpMask,int & CmpValue) const2762 bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2763                                       Register &SrcReg2, int &CmpMask,
2764                                       int &CmpValue) const {
2765   switch (MI.getOpcode()) {
2766   default: break;
2767   case ARM::CMPri:
2768   case ARM::t2CMPri:
2769   case ARM::tCMPi8:
2770     SrcReg = MI.getOperand(0).getReg();
2771     SrcReg2 = 0;
2772     CmpMask = ~0;
2773     CmpValue = MI.getOperand(1).getImm();
2774     return true;
2775   case ARM::CMPrr:
2776   case ARM::t2CMPrr:
2777   case ARM::tCMPr:
2778     SrcReg = MI.getOperand(0).getReg();
2779     SrcReg2 = MI.getOperand(1).getReg();
2780     CmpMask = ~0;
2781     CmpValue = 0;
2782     return true;
2783   case ARM::TSTri:
2784   case ARM::t2TSTri:
2785     SrcReg = MI.getOperand(0).getReg();
2786     SrcReg2 = 0;
2787     CmpMask = MI.getOperand(1).getImm();
2788     CmpValue = 0;
2789     return true;
2790   }
2791 
2792   return false;
2793 }
2794 
2795 /// isSuitableForMask - Identify a suitable 'and' instruction that
2796 /// operates on the given source register and applies the same mask
2797 /// as a 'tst' instruction. Provide a limited look-through for copies.
2798 /// When successful, MI will hold the found instruction.
isSuitableForMask(MachineInstr * & MI,Register SrcReg,int CmpMask,bool CommonUse)2799 static bool isSuitableForMask(MachineInstr *&MI, Register SrcReg,
2800                               int CmpMask, bool CommonUse) {
2801   switch (MI->getOpcode()) {
2802     case ARM::ANDri:
2803     case ARM::t2ANDri:
2804       if (CmpMask != MI->getOperand(2).getImm())
2805         return false;
2806       if (SrcReg == MI->getOperand(CommonUse ? 1 : 0).getReg())
2807         return true;
2808       break;
2809   }
2810 
2811   return false;
2812 }
2813 
2814 /// getCmpToAddCondition - assume the flags are set by CMP(a,b), return
2815 /// the condition code if we modify the instructions such that flags are
2816 /// set by ADD(a,b,X).
getCmpToAddCondition(ARMCC::CondCodes CC)2817 inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) {
2818   switch (CC) {
2819   default: return ARMCC::AL;
2820   case ARMCC::HS: return ARMCC::LO;
2821   case ARMCC::LO: return ARMCC::HS;
2822   case ARMCC::VS: return ARMCC::VS;
2823   case ARMCC::VC: return ARMCC::VC;
2824   }
2825 }
2826 
2827 /// isRedundantFlagInstr - check whether the first instruction, whose only
2828 /// purpose is to update flags, can be made redundant.
2829 /// CMPrr can be made redundant by SUBrr if the operands are the same.
2830 /// CMPri can be made redundant by SUBri if the operands are the same.
2831 /// CMPrr(r0, r1) can be made redundant by ADDr[ri](r0, r1, X).
2832 /// This function can be extended later on.
isRedundantFlagInstr(const MachineInstr * CmpI,Register SrcReg,Register SrcReg2,int ImmValue,const MachineInstr * OI,bool & IsThumb1)2833 inline static bool isRedundantFlagInstr(const MachineInstr *CmpI,
2834                                         Register SrcReg, Register SrcReg2,
2835                                         int ImmValue, const MachineInstr *OI,
2836                                         bool &IsThumb1) {
2837   if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2838       (OI->getOpcode() == ARM::SUBrr || OI->getOpcode() == ARM::t2SUBrr) &&
2839       ((OI->getOperand(1).getReg() == SrcReg &&
2840         OI->getOperand(2).getReg() == SrcReg2) ||
2841        (OI->getOperand(1).getReg() == SrcReg2 &&
2842         OI->getOperand(2).getReg() == SrcReg))) {
2843     IsThumb1 = false;
2844     return true;
2845   }
2846 
2847   if (CmpI->getOpcode() == ARM::tCMPr && OI->getOpcode() == ARM::tSUBrr &&
2848       ((OI->getOperand(2).getReg() == SrcReg &&
2849         OI->getOperand(3).getReg() == SrcReg2) ||
2850        (OI->getOperand(2).getReg() == SrcReg2 &&
2851         OI->getOperand(3).getReg() == SrcReg))) {
2852     IsThumb1 = true;
2853     return true;
2854   }
2855 
2856   if ((CmpI->getOpcode() == ARM::CMPri || CmpI->getOpcode() == ARM::t2CMPri) &&
2857       (OI->getOpcode() == ARM::SUBri || OI->getOpcode() == ARM::t2SUBri) &&
2858       OI->getOperand(1).getReg() == SrcReg &&
2859       OI->getOperand(2).getImm() == ImmValue) {
2860     IsThumb1 = false;
2861     return true;
2862   }
2863 
2864   if (CmpI->getOpcode() == ARM::tCMPi8 &&
2865       (OI->getOpcode() == ARM::tSUBi8 || OI->getOpcode() == ARM::tSUBi3) &&
2866       OI->getOperand(2).getReg() == SrcReg &&
2867       OI->getOperand(3).getImm() == ImmValue) {
2868     IsThumb1 = true;
2869     return true;
2870   }
2871 
2872   if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2873       (OI->getOpcode() == ARM::ADDrr || OI->getOpcode() == ARM::t2ADDrr ||
2874        OI->getOpcode() == ARM::ADDri || OI->getOpcode() == ARM::t2ADDri) &&
2875       OI->getOperand(0).isReg() && OI->getOperand(1).isReg() &&
2876       OI->getOperand(0).getReg() == SrcReg &&
2877       OI->getOperand(1).getReg() == SrcReg2) {
2878     IsThumb1 = false;
2879     return true;
2880   }
2881 
2882   if (CmpI->getOpcode() == ARM::tCMPr &&
2883       (OI->getOpcode() == ARM::tADDi3 || OI->getOpcode() == ARM::tADDi8 ||
2884        OI->getOpcode() == ARM::tADDrr) &&
2885       OI->getOperand(0).getReg() == SrcReg &&
2886       OI->getOperand(2).getReg() == SrcReg2) {
2887     IsThumb1 = true;
2888     return true;
2889   }
2890 
2891   return false;
2892 }
2893 
isOptimizeCompareCandidate(MachineInstr * MI,bool & IsThumb1)2894 static bool isOptimizeCompareCandidate(MachineInstr *MI, bool &IsThumb1) {
2895   switch (MI->getOpcode()) {
2896   default: return false;
2897   case ARM::tLSLri:
2898   case ARM::tLSRri:
2899   case ARM::tLSLrr:
2900   case ARM::tLSRrr:
2901   case ARM::tSUBrr:
2902   case ARM::tADDrr:
2903   case ARM::tADDi3:
2904   case ARM::tADDi8:
2905   case ARM::tSUBi3:
2906   case ARM::tSUBi8:
2907   case ARM::tMUL:
2908   case ARM::tADC:
2909   case ARM::tSBC:
2910   case ARM::tRSB:
2911   case ARM::tAND:
2912   case ARM::tORR:
2913   case ARM::tEOR:
2914   case ARM::tBIC:
2915   case ARM::tMVN:
2916   case ARM::tASRri:
2917   case ARM::tASRrr:
2918   case ARM::tROR:
2919     IsThumb1 = true;
2920     LLVM_FALLTHROUGH;
2921   case ARM::RSBrr:
2922   case ARM::RSBri:
2923   case ARM::RSCrr:
2924   case ARM::RSCri:
2925   case ARM::ADDrr:
2926   case ARM::ADDri:
2927   case ARM::ADCrr:
2928   case ARM::ADCri:
2929   case ARM::SUBrr:
2930   case ARM::SUBri:
2931   case ARM::SBCrr:
2932   case ARM::SBCri:
2933   case ARM::t2RSBri:
2934   case ARM::t2ADDrr:
2935   case ARM::t2ADDri:
2936   case ARM::t2ADCrr:
2937   case ARM::t2ADCri:
2938   case ARM::t2SUBrr:
2939   case ARM::t2SUBri:
2940   case ARM::t2SBCrr:
2941   case ARM::t2SBCri:
2942   case ARM::ANDrr:
2943   case ARM::ANDri:
2944   case ARM::t2ANDrr:
2945   case ARM::t2ANDri:
2946   case ARM::ORRrr:
2947   case ARM::ORRri:
2948   case ARM::t2ORRrr:
2949   case ARM::t2ORRri:
2950   case ARM::EORrr:
2951   case ARM::EORri:
2952   case ARM::t2EORrr:
2953   case ARM::t2EORri:
2954   case ARM::t2LSRri:
2955   case ARM::t2LSRrr:
2956   case ARM::t2LSLri:
2957   case ARM::t2LSLrr:
2958     return true;
2959   }
2960 }
2961 
2962 /// optimizeCompareInstr - Convert the instruction supplying the argument to the
2963 /// comparison into one that sets the zero bit in the flags register;
2964 /// Remove a redundant Compare instruction if an earlier instruction can set the
2965 /// flags in the same way as Compare.
2966 /// E.g. SUBrr(r1,r2) and CMPrr(r1,r2). We also handle the case where two
2967 /// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the
2968 /// condition code of instructions which use the flags.
optimizeCompareInstr(MachineInstr & CmpInstr,Register SrcReg,Register SrcReg2,int CmpMask,int CmpValue,const MachineRegisterInfo * MRI) const2969 bool ARMBaseInstrInfo::optimizeCompareInstr(
2970     MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int CmpMask,
2971     int CmpValue, const MachineRegisterInfo *MRI) const {
2972   // Get the unique definition of SrcReg.
2973   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2974   if (!MI) return false;
2975 
2976   // Masked compares sometimes use the same register as the corresponding 'and'.
2977   if (CmpMask != ~0) {
2978     if (!isSuitableForMask(MI, SrcReg, CmpMask, false) || isPredicated(*MI)) {
2979       MI = nullptr;
2980       for (MachineRegisterInfo::use_instr_iterator
2981            UI = MRI->use_instr_begin(SrcReg), UE = MRI->use_instr_end();
2982            UI != UE; ++UI) {
2983         if (UI->getParent() != CmpInstr.getParent())
2984           continue;
2985         MachineInstr *PotentialAND = &*UI;
2986         if (!isSuitableForMask(PotentialAND, SrcReg, CmpMask, true) ||
2987             isPredicated(*PotentialAND))
2988           continue;
2989         MI = PotentialAND;
2990         break;
2991       }
2992       if (!MI) return false;
2993     }
2994   }
2995 
2996   // Get ready to iterate backward from CmpInstr.
2997   MachineBasicBlock::iterator I = CmpInstr, E = MI,
2998                               B = CmpInstr.getParent()->begin();
2999 
3000   // Early exit if CmpInstr is at the beginning of the BB.
3001   if (I == B) return false;
3002 
3003   // There are two possible candidates which can be changed to set CPSR:
3004   // One is MI, the other is a SUB or ADD instruction.
3005   // For CMPrr(r1,r2), we are looking for SUB(r1,r2), SUB(r2,r1), or
3006   // ADDr[ri](r1, r2, X).
3007   // For CMPri(r1, CmpValue), we are looking for SUBri(r1, CmpValue).
3008   MachineInstr *SubAdd = nullptr;
3009   if (SrcReg2 != 0)
3010     // MI is not a candidate for CMPrr.
3011     MI = nullptr;
3012   else if (MI->getParent() != CmpInstr.getParent() || CmpValue != 0) {
3013     // Conservatively refuse to convert an instruction which isn't in the same
3014     // BB as the comparison.
3015     // For CMPri w/ CmpValue != 0, a SubAdd may still be a candidate.
3016     // Thus we cannot return here.
3017     if (CmpInstr.getOpcode() == ARM::CMPri ||
3018         CmpInstr.getOpcode() == ARM::t2CMPri ||
3019         CmpInstr.getOpcode() == ARM::tCMPi8)
3020       MI = nullptr;
3021     else
3022       return false;
3023   }
3024 
3025   bool IsThumb1 = false;
3026   if (MI && !isOptimizeCompareCandidate(MI, IsThumb1))
3027     return false;
3028 
3029   // We also want to do this peephole for cases like this: if (a*b == 0),
3030   // and optimise away the CMP instruction from the generated code sequence:
3031   // MULS, MOVS, MOVS, CMP. Here the MOVS instructions load the boolean values
3032   // resulting from the select instruction, but these MOVS instructions for
3033   // Thumb1 (V6M) are flag setting and are thus preventing this optimisation.
3034   // However, if we only have MOVS instructions in between the CMP and the
3035   // other instruction (the MULS in this example), then the CPSR is dead so we
3036   // can safely reorder the sequence into: MOVS, MOVS, MULS, CMP. We do this
3037   // reordering and then continue the analysis hoping we can eliminate the
3038   // CMP. This peephole works on the vregs, so is still in SSA form. As a
3039   // consequence, the movs won't redefine/kill the MUL operands which would
3040   // make this reordering illegal.
3041   const TargetRegisterInfo *TRI = &getRegisterInfo();
3042   if (MI && IsThumb1) {
3043     --I;
3044     if (I != E && !MI->readsRegister(ARM::CPSR, TRI)) {
3045       bool CanReorder = true;
3046       for (; I != E; --I) {
3047         if (I->getOpcode() != ARM::tMOVi8) {
3048           CanReorder = false;
3049           break;
3050         }
3051       }
3052       if (CanReorder) {
3053         MI = MI->removeFromParent();
3054         E = CmpInstr;
3055         CmpInstr.getParent()->insert(E, MI);
3056       }
3057     }
3058     I = CmpInstr;
3059     E = MI;
3060   }
3061 
3062   // Check that CPSR isn't set between the comparison instruction and the one we
3063   // want to change. At the same time, search for SubAdd.
3064   bool SubAddIsThumb1 = false;
3065   do {
3066     const MachineInstr &Instr = *--I;
3067 
3068     // Check whether CmpInstr can be made redundant by the current instruction.
3069     if (isRedundantFlagInstr(&CmpInstr, SrcReg, SrcReg2, CmpValue, &Instr,
3070                              SubAddIsThumb1)) {
3071       SubAdd = &*I;
3072       break;
3073     }
3074 
3075     // Allow E (which was initially MI) to be SubAdd but do not search before E.
3076     if (I == E)
3077       break;
3078 
3079     if (Instr.modifiesRegister(ARM::CPSR, TRI) ||
3080         Instr.readsRegister(ARM::CPSR, TRI))
3081       // This instruction modifies or uses CPSR after the one we want to
3082       // change. We can't do this transformation.
3083       return false;
3084 
3085     if (I == B) {
3086       // In some cases, we scan the use-list of an instruction for an AND;
3087       // that AND is in the same BB, but may not be scheduled before the
3088       // corresponding TST.  In that case, bail out.
3089       //
3090       // FIXME: We could try to reschedule the AND.
3091       return false;
3092     }
3093   } while (true);
3094 
3095   // Return false if no candidates exist.
3096   if (!MI && !SubAdd)
3097     return false;
3098 
3099   // If we found a SubAdd, use it as it will be closer to the CMP
3100   if (SubAdd) {
3101     MI = SubAdd;
3102     IsThumb1 = SubAddIsThumb1;
3103   }
3104 
3105   // We can't use a predicated instruction - it doesn't always write the flags.
3106   if (isPredicated(*MI))
3107     return false;
3108 
3109   // Scan forward for the use of CPSR
3110   // When checking against MI: if it's a conditional code that requires
3111   // checking of the V bit or C bit, then this is not safe to do.
3112   // It is safe to remove CmpInstr if CPSR is redefined or killed.
3113   // If we are done with the basic block, we need to check whether CPSR is
3114   // live-out.
3115   SmallVector<std::pair<MachineOperand*, ARMCC::CondCodes>, 4>
3116       OperandsToUpdate;
3117   bool isSafe = false;
3118   I = CmpInstr;
3119   E = CmpInstr.getParent()->end();
3120   while (!isSafe && ++I != E) {
3121     const MachineInstr &Instr = *I;
3122     for (unsigned IO = 0, EO = Instr.getNumOperands();
3123          !isSafe && IO != EO; ++IO) {
3124       const MachineOperand &MO = Instr.getOperand(IO);
3125       if (MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) {
3126         isSafe = true;
3127         break;
3128       }
3129       if (!MO.isReg() || MO.getReg() != ARM::CPSR)
3130         continue;
3131       if (MO.isDef()) {
3132         isSafe = true;
3133         break;
3134       }
3135       // Condition code is after the operand before CPSR except for VSELs.
3136       ARMCC::CondCodes CC;
3137       bool IsInstrVSel = true;
3138       switch (Instr.getOpcode()) {
3139       default:
3140         IsInstrVSel = false;
3141         CC = (ARMCC::CondCodes)Instr.getOperand(IO - 1).getImm();
3142         break;
3143       case ARM::VSELEQD:
3144       case ARM::VSELEQS:
3145       case ARM::VSELEQH:
3146         CC = ARMCC::EQ;
3147         break;
3148       case ARM::VSELGTD:
3149       case ARM::VSELGTS:
3150       case ARM::VSELGTH:
3151         CC = ARMCC::GT;
3152         break;
3153       case ARM::VSELGED:
3154       case ARM::VSELGES:
3155       case ARM::VSELGEH:
3156         CC = ARMCC::GE;
3157         break;
3158       case ARM::VSELVSD:
3159       case ARM::VSELVSS:
3160       case ARM::VSELVSH:
3161         CC = ARMCC::VS;
3162         break;
3163       }
3164 
3165       if (SubAdd) {
3166         // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based
3167         // on CMP needs to be updated to be based on SUB.
3168         // If we have ADD(r1, r2, X) and CMP(r1, r2), the condition code also
3169         // needs to be modified.
3170         // Push the condition code operands to OperandsToUpdate.
3171         // If it is safe to remove CmpInstr, the condition code of these
3172         // operands will be modified.
3173         unsigned Opc = SubAdd->getOpcode();
3174         bool IsSub = Opc == ARM::SUBrr || Opc == ARM::t2SUBrr ||
3175                      Opc == ARM::SUBri || Opc == ARM::t2SUBri ||
3176                      Opc == ARM::tSUBrr || Opc == ARM::tSUBi3 ||
3177                      Opc == ARM::tSUBi8;
3178         unsigned OpI = Opc != ARM::tSUBrr ? 1 : 2;
3179         if (!IsSub ||
3180             (SrcReg2 != 0 && SubAdd->getOperand(OpI).getReg() == SrcReg2 &&
3181              SubAdd->getOperand(OpI + 1).getReg() == SrcReg)) {
3182           // VSel doesn't support condition code update.
3183           if (IsInstrVSel)
3184             return false;
3185           // Ensure we can swap the condition.
3186           ARMCC::CondCodes NewCC = (IsSub ? getSwappedCondition(CC) : getCmpToAddCondition(CC));
3187           if (NewCC == ARMCC::AL)
3188             return false;
3189           OperandsToUpdate.push_back(
3190               std::make_pair(&((*I).getOperand(IO - 1)), NewCC));
3191         }
3192       } else {
3193         // No SubAdd, so this is x = <op> y, z; cmp x, 0.
3194         switch (CC) {
3195         case ARMCC::EQ: // Z
3196         case ARMCC::NE: // Z
3197         case ARMCC::MI: // N
3198         case ARMCC::PL: // N
3199         case ARMCC::AL: // none
3200           // CPSR can be used multiple times, we should continue.
3201           break;
3202         case ARMCC::HS: // C
3203         case ARMCC::LO: // C
3204         case ARMCC::VS: // V
3205         case ARMCC::VC: // V
3206         case ARMCC::HI: // C Z
3207         case ARMCC::LS: // C Z
3208         case ARMCC::GE: // N V
3209         case ARMCC::LT: // N V
3210         case ARMCC::GT: // Z N V
3211         case ARMCC::LE: // Z N V
3212           // The instruction uses the V bit or C bit which is not safe.
3213           return false;
3214         }
3215       }
3216     }
3217   }
3218 
3219   // If CPSR is not killed nor re-defined, we should check whether it is
3220   // live-out. If it is live-out, do not optimize.
3221   if (!isSafe) {
3222     MachineBasicBlock *MBB = CmpInstr.getParent();
3223     for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
3224              SE = MBB->succ_end(); SI != SE; ++SI)
3225       if ((*SI)->isLiveIn(ARM::CPSR))
3226         return false;
3227   }
3228 
3229   // Toggle the optional operand to CPSR (if it exists - in Thumb1 we always
3230   // set CPSR so this is represented as an explicit output)
3231   if (!IsThumb1) {
3232     MI->getOperand(5).setReg(ARM::CPSR);
3233     MI->getOperand(5).setIsDef(true);
3234   }
3235   assert(!isPredicated(*MI) && "Can't use flags from predicated instruction");
3236   CmpInstr.eraseFromParent();
3237 
3238   // Modify the condition code of operands in OperandsToUpdate.
3239   // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
3240   // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
3241   for (unsigned i = 0, e = OperandsToUpdate.size(); i < e; i++)
3242     OperandsToUpdate[i].first->setImm(OperandsToUpdate[i].second);
3243 
3244   MI->clearRegisterDeads(ARM::CPSR);
3245 
3246   return true;
3247 }
3248 
shouldSink(const MachineInstr & MI) const3249 bool ARMBaseInstrInfo::shouldSink(const MachineInstr &MI) const {
3250   // Do not sink MI if it might be used to optimize a redundant compare.
3251   // We heuristically only look at the instruction immediately following MI to
3252   // avoid potentially searching the entire basic block.
3253   if (isPredicated(MI))
3254     return true;
3255   MachineBasicBlock::const_iterator Next = &MI;
3256   ++Next;
3257   Register SrcReg, SrcReg2;
3258   int CmpMask, CmpValue;
3259   bool IsThumb1;
3260   if (Next != MI.getParent()->end() &&
3261       analyzeCompare(*Next, SrcReg, SrcReg2, CmpMask, CmpValue) &&
3262       isRedundantFlagInstr(&*Next, SrcReg, SrcReg2, CmpValue, &MI, IsThumb1))
3263     return false;
3264   return true;
3265 }
3266 
FoldImmediate(MachineInstr & UseMI,MachineInstr & DefMI,Register Reg,MachineRegisterInfo * MRI) const3267 bool ARMBaseInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3268                                      Register Reg,
3269                                      MachineRegisterInfo *MRI) const {
3270   // Fold large immediates into add, sub, or, xor.
3271   unsigned DefOpc = DefMI.getOpcode();
3272   if (DefOpc != ARM::t2MOVi32imm && DefOpc != ARM::MOVi32imm)
3273     return false;
3274   if (!DefMI.getOperand(1).isImm())
3275     // Could be t2MOVi32imm @xx
3276     return false;
3277 
3278   if (!MRI->hasOneNonDBGUse(Reg))
3279     return false;
3280 
3281   const MCInstrDesc &DefMCID = DefMI.getDesc();
3282   if (DefMCID.hasOptionalDef()) {
3283     unsigned NumOps = DefMCID.getNumOperands();
3284     const MachineOperand &MO = DefMI.getOperand(NumOps - 1);
3285     if (MO.getReg() == ARM::CPSR && !MO.isDead())
3286       // If DefMI defines CPSR and it is not dead, it's obviously not safe
3287       // to delete DefMI.
3288       return false;
3289   }
3290 
3291   const MCInstrDesc &UseMCID = UseMI.getDesc();
3292   if (UseMCID.hasOptionalDef()) {
3293     unsigned NumOps = UseMCID.getNumOperands();
3294     if (UseMI.getOperand(NumOps - 1).getReg() == ARM::CPSR)
3295       // If the instruction sets the flag, do not attempt this optimization
3296       // since it may change the semantics of the code.
3297       return false;
3298   }
3299 
3300   unsigned UseOpc = UseMI.getOpcode();
3301   unsigned NewUseOpc = 0;
3302   uint32_t ImmVal = (uint32_t)DefMI.getOperand(1).getImm();
3303   uint32_t SOImmValV1 = 0, SOImmValV2 = 0;
3304   bool Commute = false;
3305   switch (UseOpc) {
3306   default: return false;
3307   case ARM::SUBrr:
3308   case ARM::ADDrr:
3309   case ARM::ORRrr:
3310   case ARM::EORrr:
3311   case ARM::t2SUBrr:
3312   case ARM::t2ADDrr:
3313   case ARM::t2ORRrr:
3314   case ARM::t2EORrr: {
3315     Commute = UseMI.getOperand(2).getReg() != Reg;
3316     switch (UseOpc) {
3317     default: break;
3318     case ARM::ADDrr:
3319     case ARM::SUBrr:
3320       if (UseOpc == ARM::SUBrr && Commute)
3321         return false;
3322 
3323       // ADD/SUB are special because they're essentially the same operation, so
3324       // we can handle a larger range of immediates.
3325       if (ARM_AM::isSOImmTwoPartVal(ImmVal))
3326         NewUseOpc = UseOpc == ARM::ADDrr ? ARM::ADDri : ARM::SUBri;
3327       else if (ARM_AM::isSOImmTwoPartVal(-ImmVal)) {
3328         ImmVal = -ImmVal;
3329         NewUseOpc = UseOpc == ARM::ADDrr ? ARM::SUBri : ARM::ADDri;
3330       } else
3331         return false;
3332       SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal);
3333       SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal);
3334       break;
3335     case ARM::ORRrr:
3336     case ARM::EORrr:
3337       if (!ARM_AM::isSOImmTwoPartVal(ImmVal))
3338         return false;
3339       SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal);
3340       SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal);
3341       switch (UseOpc) {
3342       default: break;
3343       case ARM::ORRrr: NewUseOpc = ARM::ORRri; break;
3344       case ARM::EORrr: NewUseOpc = ARM::EORri; break;
3345       }
3346       break;
3347     case ARM::t2ADDrr:
3348     case ARM::t2SUBrr: {
3349       if (UseOpc == ARM::t2SUBrr && Commute)
3350         return false;
3351 
3352       // ADD/SUB are special because they're essentially the same operation, so
3353       // we can handle a larger range of immediates.
3354       const bool ToSP = DefMI.getOperand(0).getReg() == ARM::SP;
3355       const unsigned t2ADD = ToSP ? ARM::t2ADDspImm : ARM::t2ADDri;
3356       const unsigned t2SUB = ToSP ? ARM::t2SUBspImm : ARM::t2SUBri;
3357       if (ARM_AM::isT2SOImmTwoPartVal(ImmVal))
3358         NewUseOpc = UseOpc == ARM::t2ADDrr ? t2ADD : t2SUB;
3359       else if (ARM_AM::isT2SOImmTwoPartVal(-ImmVal)) {
3360         ImmVal = -ImmVal;
3361         NewUseOpc = UseOpc == ARM::t2ADDrr ? t2SUB : t2ADD;
3362       } else
3363         return false;
3364       SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal);
3365       SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal);
3366       break;
3367     }
3368     case ARM::t2ORRrr:
3369     case ARM::t2EORrr:
3370       if (!ARM_AM::isT2SOImmTwoPartVal(ImmVal))
3371         return false;
3372       SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal);
3373       SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal);
3374       switch (UseOpc) {
3375       default: break;
3376       case ARM::t2ORRrr: NewUseOpc = ARM::t2ORRri; break;
3377       case ARM::t2EORrr: NewUseOpc = ARM::t2EORri; break;
3378       }
3379       break;
3380     }
3381   }
3382   }
3383 
3384   unsigned OpIdx = Commute ? 2 : 1;
3385   Register Reg1 = UseMI.getOperand(OpIdx).getReg();
3386   bool isKill = UseMI.getOperand(OpIdx).isKill();
3387   const TargetRegisterClass *TRC = MRI->getRegClass(Reg);
3388   Register NewReg = MRI->createVirtualRegister(TRC);
3389   BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), get(NewUseOpc),
3390           NewReg)
3391       .addReg(Reg1, getKillRegState(isKill))
3392       .addImm(SOImmValV1)
3393       .add(predOps(ARMCC::AL))
3394       .add(condCodeOp());
3395   UseMI.setDesc(get(NewUseOpc));
3396   UseMI.getOperand(1).setReg(NewReg);
3397   UseMI.getOperand(1).setIsKill();
3398   UseMI.getOperand(2).ChangeToImmediate(SOImmValV2);
3399   DefMI.eraseFromParent();
3400   // FIXME: t2ADDrr should be split, as different rulles apply when writing to SP.
3401   // Just as t2ADDri, that was split to [t2ADDri, t2ADDspImm].
3402   // Then the below code will not be needed, as the input/output register
3403   // classes will be rgpr or gprSP.
3404   // For now, we fix the UseMI operand explicitly here:
3405   switch(NewUseOpc){
3406     case ARM::t2ADDspImm:
3407     case ARM::t2SUBspImm:
3408     case ARM::t2ADDri:
3409     case ARM::t2SUBri:
3410       MRI->constrainRegClass(UseMI.getOperand(0).getReg(), TRC);
3411   }
3412   return true;
3413 }
3414 
getNumMicroOpsSwiftLdSt(const InstrItineraryData * ItinData,const MachineInstr & MI)3415 static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData,
3416                                         const MachineInstr &MI) {
3417   switch (MI.getOpcode()) {
3418   default: {
3419     const MCInstrDesc &Desc = MI.getDesc();
3420     int UOps = ItinData->getNumMicroOps(Desc.getSchedClass());
3421     assert(UOps >= 0 && "bad # UOps");
3422     return UOps;
3423   }
3424 
3425   case ARM::LDRrs:
3426   case ARM::LDRBrs:
3427   case ARM::STRrs:
3428   case ARM::STRBrs: {
3429     unsigned ShOpVal = MI.getOperand(3).getImm();
3430     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3431     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3432     if (!isSub &&
3433         (ShImm == 0 ||
3434          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3435           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3436       return 1;
3437     return 2;
3438   }
3439 
3440   case ARM::LDRH:
3441   case ARM::STRH: {
3442     if (!MI.getOperand(2).getReg())
3443       return 1;
3444 
3445     unsigned ShOpVal = MI.getOperand(3).getImm();
3446     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3447     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3448     if (!isSub &&
3449         (ShImm == 0 ||
3450          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3451           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3452       return 1;
3453     return 2;
3454   }
3455 
3456   case ARM::LDRSB:
3457   case ARM::LDRSH:
3458     return (ARM_AM::getAM3Op(MI.getOperand(3).getImm()) == ARM_AM::sub) ? 3 : 2;
3459 
3460   case ARM::LDRSB_POST:
3461   case ARM::LDRSH_POST: {
3462     Register Rt = MI.getOperand(0).getReg();
3463     Register Rm = MI.getOperand(3).getReg();
3464     return (Rt == Rm) ? 4 : 3;
3465   }
3466 
3467   case ARM::LDR_PRE_REG:
3468   case ARM::LDRB_PRE_REG: {
3469     Register Rt = MI.getOperand(0).getReg();
3470     Register Rm = MI.getOperand(3).getReg();
3471     if (Rt == Rm)
3472       return 3;
3473     unsigned ShOpVal = MI.getOperand(4).getImm();
3474     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3475     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3476     if (!isSub &&
3477         (ShImm == 0 ||
3478          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3479           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3480       return 2;
3481     return 3;
3482   }
3483 
3484   case ARM::STR_PRE_REG:
3485   case ARM::STRB_PRE_REG: {
3486     unsigned ShOpVal = MI.getOperand(4).getImm();
3487     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3488     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3489     if (!isSub &&
3490         (ShImm == 0 ||
3491          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3492           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3493       return 2;
3494     return 3;
3495   }
3496 
3497   case ARM::LDRH_PRE:
3498   case ARM::STRH_PRE: {
3499     Register Rt = MI.getOperand(0).getReg();
3500     Register Rm = MI.getOperand(3).getReg();
3501     if (!Rm)
3502       return 2;
3503     if (Rt == Rm)
3504       return 3;
3505     return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 3 : 2;
3506   }
3507 
3508   case ARM::LDR_POST_REG:
3509   case ARM::LDRB_POST_REG:
3510   case ARM::LDRH_POST: {
3511     Register Rt = MI.getOperand(0).getReg();
3512     Register Rm = MI.getOperand(3).getReg();
3513     return (Rt == Rm) ? 3 : 2;
3514   }
3515 
3516   case ARM::LDR_PRE_IMM:
3517   case ARM::LDRB_PRE_IMM:
3518   case ARM::LDR_POST_IMM:
3519   case ARM::LDRB_POST_IMM:
3520   case ARM::STRB_POST_IMM:
3521   case ARM::STRB_POST_REG:
3522   case ARM::STRB_PRE_IMM:
3523   case ARM::STRH_POST:
3524   case ARM::STR_POST_IMM:
3525   case ARM::STR_POST_REG:
3526   case ARM::STR_PRE_IMM:
3527     return 2;
3528 
3529   case ARM::LDRSB_PRE:
3530   case ARM::LDRSH_PRE: {
3531     Register Rm = MI.getOperand(3).getReg();
3532     if (Rm == 0)
3533       return 3;
3534     Register Rt = MI.getOperand(0).getReg();
3535     if (Rt == Rm)
3536       return 4;
3537     unsigned ShOpVal = MI.getOperand(4).getImm();
3538     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3539     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3540     if (!isSub &&
3541         (ShImm == 0 ||
3542          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3543           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3544       return 3;
3545     return 4;
3546   }
3547 
3548   case ARM::LDRD: {
3549     Register Rt = MI.getOperand(0).getReg();
3550     Register Rn = MI.getOperand(2).getReg();
3551     Register Rm = MI.getOperand(3).getReg();
3552     if (Rm)
3553       return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4
3554                                                                           : 3;
3555     return (Rt == Rn) ? 3 : 2;
3556   }
3557 
3558   case ARM::STRD: {
3559     Register Rm = MI.getOperand(3).getReg();
3560     if (Rm)
3561       return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4
3562                                                                           : 3;
3563     return 2;
3564   }
3565 
3566   case ARM::LDRD_POST:
3567   case ARM::t2LDRD_POST:
3568     return 3;
3569 
3570   case ARM::STRD_POST:
3571   case ARM::t2STRD_POST:
3572     return 4;
3573 
3574   case ARM::LDRD_PRE: {
3575     Register Rt = MI.getOperand(0).getReg();
3576     Register Rn = MI.getOperand(3).getReg();
3577     Register Rm = MI.getOperand(4).getReg();
3578     if (Rm)
3579       return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5
3580                                                                           : 4;
3581     return (Rt == Rn) ? 4 : 3;
3582   }
3583 
3584   case ARM::t2LDRD_PRE: {
3585     Register Rt = MI.getOperand(0).getReg();
3586     Register Rn = MI.getOperand(3).getReg();
3587     return (Rt == Rn) ? 4 : 3;
3588   }
3589 
3590   case ARM::STRD_PRE: {
3591     Register Rm = MI.getOperand(4).getReg();
3592     if (Rm)
3593       return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5
3594                                                                           : 4;
3595     return 3;
3596   }
3597 
3598   case ARM::t2STRD_PRE:
3599     return 3;
3600 
3601   case ARM::t2LDR_POST:
3602   case ARM::t2LDRB_POST:
3603   case ARM::t2LDRB_PRE:
3604   case ARM::t2LDRSBi12:
3605   case ARM::t2LDRSBi8:
3606   case ARM::t2LDRSBpci:
3607   case ARM::t2LDRSBs:
3608   case ARM::t2LDRH_POST:
3609   case ARM::t2LDRH_PRE:
3610   case ARM::t2LDRSBT:
3611   case ARM::t2LDRSB_POST:
3612   case ARM::t2LDRSB_PRE:
3613   case ARM::t2LDRSH_POST:
3614   case ARM::t2LDRSH_PRE:
3615   case ARM::t2LDRSHi12:
3616   case ARM::t2LDRSHi8:
3617   case ARM::t2LDRSHpci:
3618   case ARM::t2LDRSHs:
3619     return 2;
3620 
3621   case ARM::t2LDRDi8: {
3622     Register Rt = MI.getOperand(0).getReg();
3623     Register Rn = MI.getOperand(2).getReg();
3624     return (Rt == Rn) ? 3 : 2;
3625   }
3626 
3627   case ARM::t2STRB_POST:
3628   case ARM::t2STRB_PRE:
3629   case ARM::t2STRBs:
3630   case ARM::t2STRDi8:
3631   case ARM::t2STRH_POST:
3632   case ARM::t2STRH_PRE:
3633   case ARM::t2STRHs:
3634   case ARM::t2STR_POST:
3635   case ARM::t2STR_PRE:
3636   case ARM::t2STRs:
3637     return 2;
3638   }
3639 }
3640 
3641 // Return the number of 32-bit words loaded by LDM or stored by STM. If this
3642 // can't be easily determined return 0 (missing MachineMemOperand).
3643 //
3644 // FIXME: The current MachineInstr design does not support relying on machine
3645 // mem operands to determine the width of a memory access. Instead, we expect
3646 // the target to provide this information based on the instruction opcode and
3647 // operands. However, using MachineMemOperand is the best solution now for
3648 // two reasons:
3649 //
3650 // 1) getNumMicroOps tries to infer LDM memory width from the total number of MI
3651 // operands. This is much more dangerous than using the MachineMemOperand
3652 // sizes because CodeGen passes can insert/remove optional machine operands. In
3653 // fact, it's totally incorrect for preRA passes and appears to be wrong for
3654 // postRA passes as well.
3655 //
3656 // 2) getNumLDMAddresses is only used by the scheduling machine model and any
3657 // machine model that calls this should handle the unknown (zero size) case.
3658 //
3659 // Long term, we should require a target hook that verifies MachineMemOperand
3660 // sizes during MC lowering. That target hook should be local to MC lowering
3661 // because we can't ensure that it is aware of other MI forms. Doing this will
3662 // ensure that MachineMemOperands are correctly propagated through all passes.
getNumLDMAddresses(const MachineInstr & MI) const3663 unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr &MI) const {
3664   unsigned Size = 0;
3665   for (MachineInstr::mmo_iterator I = MI.memoperands_begin(),
3666                                   E = MI.memoperands_end();
3667        I != E; ++I) {
3668     Size += (*I)->getSize();
3669   }
3670   // FIXME: The scheduler currently can't handle values larger than 16. But
3671   // the values can actually go up to 32 for floating-point load/store
3672   // multiple (VLDMIA etc.). Also, the way this code is reasoning about memory
3673   // operations isn't right; we could end up with "extra" memory operands for
3674   // various reasons, like tail merge merging two memory operations.
3675   return std::min(Size / 4, 16U);
3676 }
3677 
getNumMicroOpsSingleIssuePlusExtras(unsigned Opc,unsigned NumRegs)3678 static unsigned getNumMicroOpsSingleIssuePlusExtras(unsigned Opc,
3679                                                     unsigned NumRegs) {
3680   unsigned UOps = 1 + NumRegs; // 1 for address computation.
3681   switch (Opc) {
3682   default:
3683     break;
3684   case ARM::VLDMDIA_UPD:
3685   case ARM::VLDMDDB_UPD:
3686   case ARM::VLDMSIA_UPD:
3687   case ARM::VLDMSDB_UPD:
3688   case ARM::VSTMDIA_UPD:
3689   case ARM::VSTMDDB_UPD:
3690   case ARM::VSTMSIA_UPD:
3691   case ARM::VSTMSDB_UPD:
3692   case ARM::LDMIA_UPD:
3693   case ARM::LDMDA_UPD:
3694   case ARM::LDMDB_UPD:
3695   case ARM::LDMIB_UPD:
3696   case ARM::STMIA_UPD:
3697   case ARM::STMDA_UPD:
3698   case ARM::STMDB_UPD:
3699   case ARM::STMIB_UPD:
3700   case ARM::tLDMIA_UPD:
3701   case ARM::tSTMIA_UPD:
3702   case ARM::t2LDMIA_UPD:
3703   case ARM::t2LDMDB_UPD:
3704   case ARM::t2STMIA_UPD:
3705   case ARM::t2STMDB_UPD:
3706     ++UOps; // One for base register writeback.
3707     break;
3708   case ARM::LDMIA_RET:
3709   case ARM::tPOP_RET:
3710   case ARM::t2LDMIA_RET:
3711     UOps += 2; // One for base reg wb, one for write to pc.
3712     break;
3713   }
3714   return UOps;
3715 }
3716 
getNumMicroOps(const InstrItineraryData * ItinData,const MachineInstr & MI) const3717 unsigned ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
3718                                           const MachineInstr &MI) const {
3719   if (!ItinData || ItinData->isEmpty())
3720     return 1;
3721 
3722   const MCInstrDesc &Desc = MI.getDesc();
3723   unsigned Class = Desc.getSchedClass();
3724   int ItinUOps = ItinData->getNumMicroOps(Class);
3725   if (ItinUOps >= 0) {
3726     if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore()))
3727       return getNumMicroOpsSwiftLdSt(ItinData, MI);
3728 
3729     return ItinUOps;
3730   }
3731 
3732   unsigned Opc = MI.getOpcode();
3733   switch (Opc) {
3734   default:
3735     llvm_unreachable("Unexpected multi-uops instruction!");
3736   case ARM::VLDMQIA:
3737   case ARM::VSTMQIA:
3738     return 2;
3739 
3740   // The number of uOps for load / store multiple are determined by the number
3741   // registers.
3742   //
3743   // On Cortex-A8, each pair of register loads / stores can be scheduled on the
3744   // same cycle. The scheduling for the first load / store must be done
3745   // separately by assuming the address is not 64-bit aligned.
3746   //
3747   // On Cortex-A9, the formula is simply (#reg / 2) + (#reg % 2). If the address
3748   // is not 64-bit aligned, then AGU would take an extra cycle.  For VFP / NEON
3749   // load / store multiple, the formula is (#reg / 2) + (#reg % 2) + 1.
3750   case ARM::VLDMDIA:
3751   case ARM::VLDMDIA_UPD:
3752   case ARM::VLDMDDB_UPD:
3753   case ARM::VLDMSIA:
3754   case ARM::VLDMSIA_UPD:
3755   case ARM::VLDMSDB_UPD:
3756   case ARM::VSTMDIA:
3757   case ARM::VSTMDIA_UPD:
3758   case ARM::VSTMDDB_UPD:
3759   case ARM::VSTMSIA:
3760   case ARM::VSTMSIA_UPD:
3761   case ARM::VSTMSDB_UPD: {
3762     unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands();
3763     return (NumRegs / 2) + (NumRegs % 2) + 1;
3764   }
3765 
3766   case ARM::LDMIA_RET:
3767   case ARM::LDMIA:
3768   case ARM::LDMDA:
3769   case ARM::LDMDB:
3770   case ARM::LDMIB:
3771   case ARM::LDMIA_UPD:
3772   case ARM::LDMDA_UPD:
3773   case ARM::LDMDB_UPD:
3774   case ARM::LDMIB_UPD:
3775   case ARM::STMIA:
3776   case ARM::STMDA:
3777   case ARM::STMDB:
3778   case ARM::STMIB:
3779   case ARM::STMIA_UPD:
3780   case ARM::STMDA_UPD:
3781   case ARM::STMDB_UPD:
3782   case ARM::STMIB_UPD:
3783   case ARM::tLDMIA:
3784   case ARM::tLDMIA_UPD:
3785   case ARM::tSTMIA_UPD:
3786   case ARM::tPOP_RET:
3787   case ARM::tPOP:
3788   case ARM::tPUSH:
3789   case ARM::t2LDMIA_RET:
3790   case ARM::t2LDMIA:
3791   case ARM::t2LDMDB:
3792   case ARM::t2LDMIA_UPD:
3793   case ARM::t2LDMDB_UPD:
3794   case ARM::t2STMIA:
3795   case ARM::t2STMDB:
3796   case ARM::t2STMIA_UPD:
3797   case ARM::t2STMDB_UPD: {
3798     unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands() + 1;
3799     switch (Subtarget.getLdStMultipleTiming()) {
3800     case ARMSubtarget::SingleIssuePlusExtras:
3801       return getNumMicroOpsSingleIssuePlusExtras(Opc, NumRegs);
3802     case ARMSubtarget::SingleIssue:
3803       // Assume the worst.
3804       return NumRegs;
3805     case ARMSubtarget::DoubleIssue: {
3806       if (NumRegs < 4)
3807         return 2;
3808       // 4 registers would be issued: 2, 2.
3809       // 5 registers would be issued: 2, 2, 1.
3810       unsigned UOps = (NumRegs / 2);
3811       if (NumRegs % 2)
3812         ++UOps;
3813       return UOps;
3814     }
3815     case ARMSubtarget::DoubleIssueCheckUnalignedAccess: {
3816       unsigned UOps = (NumRegs / 2);
3817       // If there are odd number of registers or if it's not 64-bit aligned,
3818       // then it takes an extra AGU (Address Generation Unit) cycle.
3819       if ((NumRegs % 2) || !MI.hasOneMemOperand() ||
3820           (*MI.memoperands_begin())->getAlign() < Align(8))
3821         ++UOps;
3822       return UOps;
3823       }
3824     }
3825   }
3826   }
3827   llvm_unreachable("Didn't find the number of microops");
3828 }
3829 
3830 int
getVLDMDefCycle(const InstrItineraryData * ItinData,const MCInstrDesc & DefMCID,unsigned DefClass,unsigned DefIdx,unsigned DefAlign) const3831 ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData,
3832                                   const MCInstrDesc &DefMCID,
3833                                   unsigned DefClass,
3834                                   unsigned DefIdx, unsigned DefAlign) const {
3835   int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3836   if (RegNo <= 0)
3837     // Def is the address writeback.
3838     return ItinData->getOperandCycle(DefClass, DefIdx);
3839 
3840   int DefCycle;
3841   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3842     // (regno / 2) + (regno % 2) + 1
3843     DefCycle = RegNo / 2 + 1;
3844     if (RegNo % 2)
3845       ++DefCycle;
3846   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3847     DefCycle = RegNo;
3848     bool isSLoad = false;
3849 
3850     switch (DefMCID.getOpcode()) {
3851     default: break;
3852     case ARM::VLDMSIA:
3853     case ARM::VLDMSIA_UPD:
3854     case ARM::VLDMSDB_UPD:
3855       isSLoad = true;
3856       break;
3857     }
3858 
3859     // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3860     // then it takes an extra cycle.
3861     if ((isSLoad && (RegNo % 2)) || DefAlign < 8)
3862       ++DefCycle;
3863   } else {
3864     // Assume the worst.
3865     DefCycle = RegNo + 2;
3866   }
3867 
3868   return DefCycle;
3869 }
3870 
3871 int
getLDMDefCycle(const InstrItineraryData * ItinData,const MCInstrDesc & DefMCID,unsigned DefClass,unsigned DefIdx,unsigned DefAlign) const3872 ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData,
3873                                  const MCInstrDesc &DefMCID,
3874                                  unsigned DefClass,
3875                                  unsigned DefIdx, unsigned DefAlign) const {
3876   int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3877   if (RegNo <= 0)
3878     // Def is the address writeback.
3879     return ItinData->getOperandCycle(DefClass, DefIdx);
3880 
3881   int DefCycle;
3882   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3883     // 4 registers would be issued: 1, 2, 1.
3884     // 5 registers would be issued: 1, 2, 2.
3885     DefCycle = RegNo / 2;
3886     if (DefCycle < 1)
3887       DefCycle = 1;
3888     // Result latency is issue cycle + 2: E2.
3889     DefCycle += 2;
3890   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3891     DefCycle = (RegNo / 2);
3892     // If there are odd number of registers or if it's not 64-bit aligned,
3893     // then it takes an extra AGU (Address Generation Unit) cycle.
3894     if ((RegNo % 2) || DefAlign < 8)
3895       ++DefCycle;
3896     // Result latency is AGU cycles + 2.
3897     DefCycle += 2;
3898   } else {
3899     // Assume the worst.
3900     DefCycle = RegNo + 2;
3901   }
3902 
3903   return DefCycle;
3904 }
3905 
3906 int
getVSTMUseCycle(const InstrItineraryData * ItinData,const MCInstrDesc & UseMCID,unsigned UseClass,unsigned UseIdx,unsigned UseAlign) const3907 ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData,
3908                                   const MCInstrDesc &UseMCID,
3909                                   unsigned UseClass,
3910                                   unsigned UseIdx, unsigned UseAlign) const {
3911   int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3912   if (RegNo <= 0)
3913     return ItinData->getOperandCycle(UseClass, UseIdx);
3914 
3915   int UseCycle;
3916   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3917     // (regno / 2) + (regno % 2) + 1
3918     UseCycle = RegNo / 2 + 1;
3919     if (RegNo % 2)
3920       ++UseCycle;
3921   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3922     UseCycle = RegNo;
3923     bool isSStore = false;
3924 
3925     switch (UseMCID.getOpcode()) {
3926     default: break;
3927     case ARM::VSTMSIA:
3928     case ARM::VSTMSIA_UPD:
3929     case ARM::VSTMSDB_UPD:
3930       isSStore = true;
3931       break;
3932     }
3933 
3934     // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3935     // then it takes an extra cycle.
3936     if ((isSStore && (RegNo % 2)) || UseAlign < 8)
3937       ++UseCycle;
3938   } else {
3939     // Assume the worst.
3940     UseCycle = RegNo + 2;
3941   }
3942 
3943   return UseCycle;
3944 }
3945 
3946 int
getSTMUseCycle(const InstrItineraryData * ItinData,const MCInstrDesc & UseMCID,unsigned UseClass,unsigned UseIdx,unsigned UseAlign) const3947 ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData,
3948                                  const MCInstrDesc &UseMCID,
3949                                  unsigned UseClass,
3950                                  unsigned UseIdx, unsigned UseAlign) const {
3951   int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3952   if (RegNo <= 0)
3953     return ItinData->getOperandCycle(UseClass, UseIdx);
3954 
3955   int UseCycle;
3956   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3957     UseCycle = RegNo / 2;
3958     if (UseCycle < 2)
3959       UseCycle = 2;
3960     // Read in E3.
3961     UseCycle += 2;
3962   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3963     UseCycle = (RegNo / 2);
3964     // If there are odd number of registers or if it's not 64-bit aligned,
3965     // then it takes an extra AGU (Address Generation Unit) cycle.
3966     if ((RegNo % 2) || UseAlign < 8)
3967       ++UseCycle;
3968   } else {
3969     // Assume the worst.
3970     UseCycle = 1;
3971   }
3972   return UseCycle;
3973 }
3974 
3975 int
getOperandLatency(const InstrItineraryData * ItinData,const MCInstrDesc & DefMCID,unsigned DefIdx,unsigned DefAlign,const MCInstrDesc & UseMCID,unsigned UseIdx,unsigned UseAlign) const3976 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
3977                                     const MCInstrDesc &DefMCID,
3978                                     unsigned DefIdx, unsigned DefAlign,
3979                                     const MCInstrDesc &UseMCID,
3980                                     unsigned UseIdx, unsigned UseAlign) const {
3981   unsigned DefClass = DefMCID.getSchedClass();
3982   unsigned UseClass = UseMCID.getSchedClass();
3983 
3984   if (DefIdx < DefMCID.getNumDefs() && UseIdx < UseMCID.getNumOperands())
3985     return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
3986 
3987   // This may be a def / use of a variable_ops instruction, the operand
3988   // latency might be determinable dynamically. Let the target try to
3989   // figure it out.
3990   int DefCycle = -1;
3991   bool LdmBypass = false;
3992   switch (DefMCID.getOpcode()) {
3993   default:
3994     DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
3995     break;
3996 
3997   case ARM::VLDMDIA:
3998   case ARM::VLDMDIA_UPD:
3999   case ARM::VLDMDDB_UPD:
4000   case ARM::VLDMSIA:
4001   case ARM::VLDMSIA_UPD:
4002   case ARM::VLDMSDB_UPD:
4003     DefCycle = getVLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
4004     break;
4005 
4006   case ARM::LDMIA_RET:
4007   case ARM::LDMIA:
4008   case ARM::LDMDA:
4009   case ARM::LDMDB:
4010   case ARM::LDMIB:
4011   case ARM::LDMIA_UPD:
4012   case ARM::LDMDA_UPD:
4013   case ARM::LDMDB_UPD:
4014   case ARM::LDMIB_UPD:
4015   case ARM::tLDMIA:
4016   case ARM::tLDMIA_UPD:
4017   case ARM::tPUSH:
4018   case ARM::t2LDMIA_RET:
4019   case ARM::t2LDMIA:
4020   case ARM::t2LDMDB:
4021   case ARM::t2LDMIA_UPD:
4022   case ARM::t2LDMDB_UPD:
4023     LdmBypass = true;
4024     DefCycle = getLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
4025     break;
4026   }
4027 
4028   if (DefCycle == -1)
4029     // We can't seem to determine the result latency of the def, assume it's 2.
4030     DefCycle = 2;
4031 
4032   int UseCycle = -1;
4033   switch (UseMCID.getOpcode()) {
4034   default:
4035     UseCycle = ItinData->getOperandCycle(UseClass, UseIdx);
4036     break;
4037 
4038   case ARM::VSTMDIA:
4039   case ARM::VSTMDIA_UPD:
4040   case ARM::VSTMDDB_UPD:
4041   case ARM::VSTMSIA:
4042   case ARM::VSTMSIA_UPD:
4043   case ARM::VSTMSDB_UPD:
4044     UseCycle = getVSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
4045     break;
4046 
4047   case ARM::STMIA:
4048   case ARM::STMDA:
4049   case ARM::STMDB:
4050   case ARM::STMIB:
4051   case ARM::STMIA_UPD:
4052   case ARM::STMDA_UPD:
4053   case ARM::STMDB_UPD:
4054   case ARM::STMIB_UPD:
4055   case ARM::tSTMIA_UPD:
4056   case ARM::tPOP_RET:
4057   case ARM::tPOP:
4058   case ARM::t2STMIA:
4059   case ARM::t2STMDB:
4060   case ARM::t2STMIA_UPD:
4061   case ARM::t2STMDB_UPD:
4062     UseCycle = getSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
4063     break;
4064   }
4065 
4066   if (UseCycle == -1)
4067     // Assume it's read in the first stage.
4068     UseCycle = 1;
4069 
4070   UseCycle = DefCycle - UseCycle + 1;
4071   if (UseCycle > 0) {
4072     if (LdmBypass) {
4073       // It's a variable_ops instruction so we can't use DefIdx here. Just use
4074       // first def operand.
4075       if (ItinData->hasPipelineForwarding(DefClass, DefMCID.getNumOperands()-1,
4076                                           UseClass, UseIdx))
4077         --UseCycle;
4078     } else if (ItinData->hasPipelineForwarding(DefClass, DefIdx,
4079                                                UseClass, UseIdx)) {
4080       --UseCycle;
4081     }
4082   }
4083 
4084   return UseCycle;
4085 }
4086 
getBundledDefMI(const TargetRegisterInfo * TRI,const MachineInstr * MI,unsigned Reg,unsigned & DefIdx,unsigned & Dist)4087 static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI,
4088                                            const MachineInstr *MI, unsigned Reg,
4089                                            unsigned &DefIdx, unsigned &Dist) {
4090   Dist = 0;
4091 
4092   MachineBasicBlock::const_iterator I = MI; ++I;
4093   MachineBasicBlock::const_instr_iterator II = std::prev(I.getInstrIterator());
4094   assert(II->isInsideBundle() && "Empty bundle?");
4095 
4096   int Idx = -1;
4097   while (II->isInsideBundle()) {
4098     Idx = II->findRegisterDefOperandIdx(Reg, false, true, TRI);
4099     if (Idx != -1)
4100       break;
4101     --II;
4102     ++Dist;
4103   }
4104 
4105   assert(Idx != -1 && "Cannot find bundled definition!");
4106   DefIdx = Idx;
4107   return &*II;
4108 }
4109 
getBundledUseMI(const TargetRegisterInfo * TRI,const MachineInstr & MI,unsigned Reg,unsigned & UseIdx,unsigned & Dist)4110 static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI,
4111                                            const MachineInstr &MI, unsigned Reg,
4112                                            unsigned &UseIdx, unsigned &Dist) {
4113   Dist = 0;
4114 
4115   MachineBasicBlock::const_instr_iterator II = ++MI.getIterator();
4116   assert(II->isInsideBundle() && "Empty bundle?");
4117   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4118 
4119   // FIXME: This doesn't properly handle multiple uses.
4120   int Idx = -1;
4121   while (II != E && II->isInsideBundle()) {
4122     Idx = II->findRegisterUseOperandIdx(Reg, false, TRI);
4123     if (Idx != -1)
4124       break;
4125     if (II->getOpcode() != ARM::t2IT)
4126       ++Dist;
4127     ++II;
4128   }
4129 
4130   if (Idx == -1) {
4131     Dist = 0;
4132     return nullptr;
4133   }
4134 
4135   UseIdx = Idx;
4136   return &*II;
4137 }
4138 
4139 /// Return the number of cycles to add to (or subtract from) the static
4140 /// itinerary based on the def opcode and alignment. The caller will ensure that
4141 /// adjusted latency is at least one cycle.
adjustDefLatency(const ARMSubtarget & Subtarget,const MachineInstr & DefMI,const MCInstrDesc & DefMCID,unsigned DefAlign)4142 static int adjustDefLatency(const ARMSubtarget &Subtarget,
4143                             const MachineInstr &DefMI,
4144                             const MCInstrDesc &DefMCID, unsigned DefAlign) {
4145   int Adjust = 0;
4146   if (Subtarget.isCortexA8() || Subtarget.isLikeA9() || Subtarget.isCortexA7()) {
4147     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4148     // variants are one cycle cheaper.
4149     switch (DefMCID.getOpcode()) {
4150     default: break;
4151     case ARM::LDRrs:
4152     case ARM::LDRBrs: {
4153       unsigned ShOpVal = DefMI.getOperand(3).getImm();
4154       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4155       if (ShImm == 0 ||
4156           (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4157         --Adjust;
4158       break;
4159     }
4160     case ARM::t2LDRs:
4161     case ARM::t2LDRBs:
4162     case ARM::t2LDRHs:
4163     case ARM::t2LDRSHs: {
4164       // Thumb2 mode: lsl only.
4165       unsigned ShAmt = DefMI.getOperand(3).getImm();
4166       if (ShAmt == 0 || ShAmt == 2)
4167         --Adjust;
4168       break;
4169     }
4170     }
4171   } else if (Subtarget.isSwift()) {
4172     // FIXME: Properly handle all of the latency adjustments for address
4173     // writeback.
4174     switch (DefMCID.getOpcode()) {
4175     default: break;
4176     case ARM::LDRrs:
4177     case ARM::LDRBrs: {
4178       unsigned ShOpVal = DefMI.getOperand(3).getImm();
4179       bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
4180       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4181       if (!isSub &&
4182           (ShImm == 0 ||
4183            ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4184             ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
4185         Adjust -= 2;
4186       else if (!isSub &&
4187                ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
4188         --Adjust;
4189       break;
4190     }
4191     case ARM::t2LDRs:
4192     case ARM::t2LDRBs:
4193     case ARM::t2LDRHs:
4194     case ARM::t2LDRSHs: {
4195       // Thumb2 mode: lsl only.
4196       unsigned ShAmt = DefMI.getOperand(3).getImm();
4197       if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3)
4198         Adjust -= 2;
4199       break;
4200     }
4201     }
4202   }
4203 
4204   if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment()) {
4205     switch (DefMCID.getOpcode()) {
4206     default: break;
4207     case ARM::VLD1q8:
4208     case ARM::VLD1q16:
4209     case ARM::VLD1q32:
4210     case ARM::VLD1q64:
4211     case ARM::VLD1q8wb_fixed:
4212     case ARM::VLD1q16wb_fixed:
4213     case ARM::VLD1q32wb_fixed:
4214     case ARM::VLD1q64wb_fixed:
4215     case ARM::VLD1q8wb_register:
4216     case ARM::VLD1q16wb_register:
4217     case ARM::VLD1q32wb_register:
4218     case ARM::VLD1q64wb_register:
4219     case ARM::VLD2d8:
4220     case ARM::VLD2d16:
4221     case ARM::VLD2d32:
4222     case ARM::VLD2q8:
4223     case ARM::VLD2q16:
4224     case ARM::VLD2q32:
4225     case ARM::VLD2d8wb_fixed:
4226     case ARM::VLD2d16wb_fixed:
4227     case ARM::VLD2d32wb_fixed:
4228     case ARM::VLD2q8wb_fixed:
4229     case ARM::VLD2q16wb_fixed:
4230     case ARM::VLD2q32wb_fixed:
4231     case ARM::VLD2d8wb_register:
4232     case ARM::VLD2d16wb_register:
4233     case ARM::VLD2d32wb_register:
4234     case ARM::VLD2q8wb_register:
4235     case ARM::VLD2q16wb_register:
4236     case ARM::VLD2q32wb_register:
4237     case ARM::VLD3d8:
4238     case ARM::VLD3d16:
4239     case ARM::VLD3d32:
4240     case ARM::VLD1d64T:
4241     case ARM::VLD3d8_UPD:
4242     case ARM::VLD3d16_UPD:
4243     case ARM::VLD3d32_UPD:
4244     case ARM::VLD1d64Twb_fixed:
4245     case ARM::VLD1d64Twb_register:
4246     case ARM::VLD3q8_UPD:
4247     case ARM::VLD3q16_UPD:
4248     case ARM::VLD3q32_UPD:
4249     case ARM::VLD4d8:
4250     case ARM::VLD4d16:
4251     case ARM::VLD4d32:
4252     case ARM::VLD1d64Q:
4253     case ARM::VLD4d8_UPD:
4254     case ARM::VLD4d16_UPD:
4255     case ARM::VLD4d32_UPD:
4256     case ARM::VLD1d64Qwb_fixed:
4257     case ARM::VLD1d64Qwb_register:
4258     case ARM::VLD4q8_UPD:
4259     case ARM::VLD4q16_UPD:
4260     case ARM::VLD4q32_UPD:
4261     case ARM::VLD1DUPq8:
4262     case ARM::VLD1DUPq16:
4263     case ARM::VLD1DUPq32:
4264     case ARM::VLD1DUPq8wb_fixed:
4265     case ARM::VLD1DUPq16wb_fixed:
4266     case ARM::VLD1DUPq32wb_fixed:
4267     case ARM::VLD1DUPq8wb_register:
4268     case ARM::VLD1DUPq16wb_register:
4269     case ARM::VLD1DUPq32wb_register:
4270     case ARM::VLD2DUPd8:
4271     case ARM::VLD2DUPd16:
4272     case ARM::VLD2DUPd32:
4273     case ARM::VLD2DUPd8wb_fixed:
4274     case ARM::VLD2DUPd16wb_fixed:
4275     case ARM::VLD2DUPd32wb_fixed:
4276     case ARM::VLD2DUPd8wb_register:
4277     case ARM::VLD2DUPd16wb_register:
4278     case ARM::VLD2DUPd32wb_register:
4279     case ARM::VLD4DUPd8:
4280     case ARM::VLD4DUPd16:
4281     case ARM::VLD4DUPd32:
4282     case ARM::VLD4DUPd8_UPD:
4283     case ARM::VLD4DUPd16_UPD:
4284     case ARM::VLD4DUPd32_UPD:
4285     case ARM::VLD1LNd8:
4286     case ARM::VLD1LNd16:
4287     case ARM::VLD1LNd32:
4288     case ARM::VLD1LNd8_UPD:
4289     case ARM::VLD1LNd16_UPD:
4290     case ARM::VLD1LNd32_UPD:
4291     case ARM::VLD2LNd8:
4292     case ARM::VLD2LNd16:
4293     case ARM::VLD2LNd32:
4294     case ARM::VLD2LNq16:
4295     case ARM::VLD2LNq32:
4296     case ARM::VLD2LNd8_UPD:
4297     case ARM::VLD2LNd16_UPD:
4298     case ARM::VLD2LNd32_UPD:
4299     case ARM::VLD2LNq16_UPD:
4300     case ARM::VLD2LNq32_UPD:
4301     case ARM::VLD4LNd8:
4302     case ARM::VLD4LNd16:
4303     case ARM::VLD4LNd32:
4304     case ARM::VLD4LNq16:
4305     case ARM::VLD4LNq32:
4306     case ARM::VLD4LNd8_UPD:
4307     case ARM::VLD4LNd16_UPD:
4308     case ARM::VLD4LNd32_UPD:
4309     case ARM::VLD4LNq16_UPD:
4310     case ARM::VLD4LNq32_UPD:
4311       // If the address is not 64-bit aligned, the latencies of these
4312       // instructions increases by one.
4313       ++Adjust;
4314       break;
4315     }
4316   }
4317   return Adjust;
4318 }
4319 
getOperandLatency(const InstrItineraryData * ItinData,const MachineInstr & DefMI,unsigned DefIdx,const MachineInstr & UseMI,unsigned UseIdx) const4320 int ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4321                                         const MachineInstr &DefMI,
4322                                         unsigned DefIdx,
4323                                         const MachineInstr &UseMI,
4324                                         unsigned UseIdx) const {
4325   // No operand latency. The caller may fall back to getInstrLatency.
4326   if (!ItinData || ItinData->isEmpty())
4327     return -1;
4328 
4329   const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
4330   Register Reg = DefMO.getReg();
4331 
4332   const MachineInstr *ResolvedDefMI = &DefMI;
4333   unsigned DefAdj = 0;
4334   if (DefMI.isBundle())
4335     ResolvedDefMI =
4336         getBundledDefMI(&getRegisterInfo(), &DefMI, Reg, DefIdx, DefAdj);
4337   if (ResolvedDefMI->isCopyLike() || ResolvedDefMI->isInsertSubreg() ||
4338       ResolvedDefMI->isRegSequence() || ResolvedDefMI->isImplicitDef()) {
4339     return 1;
4340   }
4341 
4342   const MachineInstr *ResolvedUseMI = &UseMI;
4343   unsigned UseAdj = 0;
4344   if (UseMI.isBundle()) {
4345     ResolvedUseMI =
4346         getBundledUseMI(&getRegisterInfo(), UseMI, Reg, UseIdx, UseAdj);
4347     if (!ResolvedUseMI)
4348       return -1;
4349   }
4350 
4351   return getOperandLatencyImpl(
4352       ItinData, *ResolvedDefMI, DefIdx, ResolvedDefMI->getDesc(), DefAdj, DefMO,
4353       Reg, *ResolvedUseMI, UseIdx, ResolvedUseMI->getDesc(), UseAdj);
4354 }
4355 
getOperandLatencyImpl(const InstrItineraryData * ItinData,const MachineInstr & DefMI,unsigned DefIdx,const MCInstrDesc & DefMCID,unsigned DefAdj,const MachineOperand & DefMO,unsigned Reg,const MachineInstr & UseMI,unsigned UseIdx,const MCInstrDesc & UseMCID,unsigned UseAdj) const4356 int ARMBaseInstrInfo::getOperandLatencyImpl(
4357     const InstrItineraryData *ItinData, const MachineInstr &DefMI,
4358     unsigned DefIdx, const MCInstrDesc &DefMCID, unsigned DefAdj,
4359     const MachineOperand &DefMO, unsigned Reg, const MachineInstr &UseMI,
4360     unsigned UseIdx, const MCInstrDesc &UseMCID, unsigned UseAdj) const {
4361   if (Reg == ARM::CPSR) {
4362     if (DefMI.getOpcode() == ARM::FMSTAT) {
4363       // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?)
4364       return Subtarget.isLikeA9() ? 1 : 20;
4365     }
4366 
4367     // CPSR set and branch can be paired in the same cycle.
4368     if (UseMI.isBranch())
4369       return 0;
4370 
4371     // Otherwise it takes the instruction latency (generally one).
4372     unsigned Latency = getInstrLatency(ItinData, DefMI);
4373 
4374     // For Thumb2 and -Os, prefer scheduling CPSR setting instruction close to
4375     // its uses. Instructions which are otherwise scheduled between them may
4376     // incur a code size penalty (not able to use the CPSR setting 16-bit
4377     // instructions).
4378     if (Latency > 0 && Subtarget.isThumb2()) {
4379       const MachineFunction *MF = DefMI.getParent()->getParent();
4380       // FIXME: Use Function::hasOptSize().
4381       if (MF->getFunction().hasFnAttribute(Attribute::OptimizeForSize))
4382         --Latency;
4383     }
4384     return Latency;
4385   }
4386 
4387   if (DefMO.isImplicit() || UseMI.getOperand(UseIdx).isImplicit())
4388     return -1;
4389 
4390   unsigned DefAlign = DefMI.hasOneMemOperand()
4391                           ? (*DefMI.memoperands_begin())->getAlign().value()
4392                           : 0;
4393   unsigned UseAlign = UseMI.hasOneMemOperand()
4394                           ? (*UseMI.memoperands_begin())->getAlign().value()
4395                           : 0;
4396 
4397   // Get the itinerary's latency if possible, and handle variable_ops.
4398   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign, UseMCID,
4399                                   UseIdx, UseAlign);
4400   // Unable to find operand latency. The caller may resort to getInstrLatency.
4401   if (Latency < 0)
4402     return Latency;
4403 
4404   // Adjust for IT block position.
4405   int Adj = DefAdj + UseAdj;
4406 
4407   // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4408   Adj += adjustDefLatency(Subtarget, DefMI, DefMCID, DefAlign);
4409   if (Adj >= 0 || (int)Latency > -Adj) {
4410     return Latency + Adj;
4411   }
4412   // Return the itinerary latency, which may be zero but not less than zero.
4413   return Latency;
4414 }
4415 
4416 int
getOperandLatency(const InstrItineraryData * ItinData,SDNode * DefNode,unsigned DefIdx,SDNode * UseNode,unsigned UseIdx) const4417 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4418                                     SDNode *DefNode, unsigned DefIdx,
4419                                     SDNode *UseNode, unsigned UseIdx) const {
4420   if (!DefNode->isMachineOpcode())
4421     return 1;
4422 
4423   const MCInstrDesc &DefMCID = get(DefNode->getMachineOpcode());
4424 
4425   if (isZeroCost(DefMCID.Opcode))
4426     return 0;
4427 
4428   if (!ItinData || ItinData->isEmpty())
4429     return DefMCID.mayLoad() ? 3 : 1;
4430 
4431   if (!UseNode->isMachineOpcode()) {
4432     int Latency = ItinData->getOperandCycle(DefMCID.getSchedClass(), DefIdx);
4433     int Adj = Subtarget.getPreISelOperandLatencyAdjustment();
4434     int Threshold = 1 + Adj;
4435     return Latency <= Threshold ? 1 : Latency - Adj;
4436   }
4437 
4438   const MCInstrDesc &UseMCID = get(UseNode->getMachineOpcode());
4439   auto *DefMN = cast<MachineSDNode>(DefNode);
4440   unsigned DefAlign = !DefMN->memoperands_empty()
4441                           ? (*DefMN->memoperands_begin())->getAlign().value()
4442                           : 0;
4443   auto *UseMN = cast<MachineSDNode>(UseNode);
4444   unsigned UseAlign = !UseMN->memoperands_empty()
4445                           ? (*UseMN->memoperands_begin())->getAlign().value()
4446                           : 0;
4447   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign,
4448                                   UseMCID, UseIdx, UseAlign);
4449 
4450   if (Latency > 1 &&
4451       (Subtarget.isCortexA8() || Subtarget.isLikeA9() ||
4452        Subtarget.isCortexA7())) {
4453     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4454     // variants are one cycle cheaper.
4455     switch (DefMCID.getOpcode()) {
4456     default: break;
4457     case ARM::LDRrs:
4458     case ARM::LDRBrs: {
4459       unsigned ShOpVal =
4460         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4461       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4462       if (ShImm == 0 ||
4463           (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4464         --Latency;
4465       break;
4466     }
4467     case ARM::t2LDRs:
4468     case ARM::t2LDRBs:
4469     case ARM::t2LDRHs:
4470     case ARM::t2LDRSHs: {
4471       // Thumb2 mode: lsl only.
4472       unsigned ShAmt =
4473         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4474       if (ShAmt == 0 || ShAmt == 2)
4475         --Latency;
4476       break;
4477     }
4478     }
4479   } else if (DefIdx == 0 && Latency > 2 && Subtarget.isSwift()) {
4480     // FIXME: Properly handle all of the latency adjustments for address
4481     // writeback.
4482     switch (DefMCID.getOpcode()) {
4483     default: break;
4484     case ARM::LDRrs:
4485     case ARM::LDRBrs: {
4486       unsigned ShOpVal =
4487         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4488       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4489       if (ShImm == 0 ||
4490           ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4491            ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4492         Latency -= 2;
4493       else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
4494         --Latency;
4495       break;
4496     }
4497     case ARM::t2LDRs:
4498     case ARM::t2LDRBs:
4499     case ARM::t2LDRHs:
4500     case ARM::t2LDRSHs:
4501       // Thumb2 mode: lsl 0-3 only.
4502       Latency -= 2;
4503       break;
4504     }
4505   }
4506 
4507   if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment())
4508     switch (DefMCID.getOpcode()) {
4509     default: break;
4510     case ARM::VLD1q8:
4511     case ARM::VLD1q16:
4512     case ARM::VLD1q32:
4513     case ARM::VLD1q64:
4514     case ARM::VLD1q8wb_register:
4515     case ARM::VLD1q16wb_register:
4516     case ARM::VLD1q32wb_register:
4517     case ARM::VLD1q64wb_register:
4518     case ARM::VLD1q8wb_fixed:
4519     case ARM::VLD1q16wb_fixed:
4520     case ARM::VLD1q32wb_fixed:
4521     case ARM::VLD1q64wb_fixed:
4522     case ARM::VLD2d8:
4523     case ARM::VLD2d16:
4524     case ARM::VLD2d32:
4525     case ARM::VLD2q8Pseudo:
4526     case ARM::VLD2q16Pseudo:
4527     case ARM::VLD2q32Pseudo:
4528     case ARM::VLD2d8wb_fixed:
4529     case ARM::VLD2d16wb_fixed:
4530     case ARM::VLD2d32wb_fixed:
4531     case ARM::VLD2q8PseudoWB_fixed:
4532     case ARM::VLD2q16PseudoWB_fixed:
4533     case ARM::VLD2q32PseudoWB_fixed:
4534     case ARM::VLD2d8wb_register:
4535     case ARM::VLD2d16wb_register:
4536     case ARM::VLD2d32wb_register:
4537     case ARM::VLD2q8PseudoWB_register:
4538     case ARM::VLD2q16PseudoWB_register:
4539     case ARM::VLD2q32PseudoWB_register:
4540     case ARM::VLD3d8Pseudo:
4541     case ARM::VLD3d16Pseudo:
4542     case ARM::VLD3d32Pseudo:
4543     case ARM::VLD1d8TPseudo:
4544     case ARM::VLD1d16TPseudo:
4545     case ARM::VLD1d32TPseudo:
4546     case ARM::VLD1d64TPseudo:
4547     case ARM::VLD1d64TPseudoWB_fixed:
4548     case ARM::VLD1d64TPseudoWB_register:
4549     case ARM::VLD3d8Pseudo_UPD:
4550     case ARM::VLD3d16Pseudo_UPD:
4551     case ARM::VLD3d32Pseudo_UPD:
4552     case ARM::VLD3q8Pseudo_UPD:
4553     case ARM::VLD3q16Pseudo_UPD:
4554     case ARM::VLD3q32Pseudo_UPD:
4555     case ARM::VLD3q8oddPseudo:
4556     case ARM::VLD3q16oddPseudo:
4557     case ARM::VLD3q32oddPseudo:
4558     case ARM::VLD3q8oddPseudo_UPD:
4559     case ARM::VLD3q16oddPseudo_UPD:
4560     case ARM::VLD3q32oddPseudo_UPD:
4561     case ARM::VLD4d8Pseudo:
4562     case ARM::VLD4d16Pseudo:
4563     case ARM::VLD4d32Pseudo:
4564     case ARM::VLD1d8QPseudo:
4565     case ARM::VLD1d16QPseudo:
4566     case ARM::VLD1d32QPseudo:
4567     case ARM::VLD1d64QPseudo:
4568     case ARM::VLD1d64QPseudoWB_fixed:
4569     case ARM::VLD1d64QPseudoWB_register:
4570     case ARM::VLD1q8HighQPseudo:
4571     case ARM::VLD1q8LowQPseudo_UPD:
4572     case ARM::VLD1q8HighTPseudo:
4573     case ARM::VLD1q8LowTPseudo_UPD:
4574     case ARM::VLD1q16HighQPseudo:
4575     case ARM::VLD1q16LowQPseudo_UPD:
4576     case ARM::VLD1q16HighTPseudo:
4577     case ARM::VLD1q16LowTPseudo_UPD:
4578     case ARM::VLD1q32HighQPseudo:
4579     case ARM::VLD1q32LowQPseudo_UPD:
4580     case ARM::VLD1q32HighTPseudo:
4581     case ARM::VLD1q32LowTPseudo_UPD:
4582     case ARM::VLD1q64HighQPseudo:
4583     case ARM::VLD1q64LowQPseudo_UPD:
4584     case ARM::VLD1q64HighTPseudo:
4585     case ARM::VLD1q64LowTPseudo_UPD:
4586     case ARM::VLD4d8Pseudo_UPD:
4587     case ARM::VLD4d16Pseudo_UPD:
4588     case ARM::VLD4d32Pseudo_UPD:
4589     case ARM::VLD4q8Pseudo_UPD:
4590     case ARM::VLD4q16Pseudo_UPD:
4591     case ARM::VLD4q32Pseudo_UPD:
4592     case ARM::VLD4q8oddPseudo:
4593     case ARM::VLD4q16oddPseudo:
4594     case ARM::VLD4q32oddPseudo:
4595     case ARM::VLD4q8oddPseudo_UPD:
4596     case ARM::VLD4q16oddPseudo_UPD:
4597     case ARM::VLD4q32oddPseudo_UPD:
4598     case ARM::VLD1DUPq8:
4599     case ARM::VLD1DUPq16:
4600     case ARM::VLD1DUPq32:
4601     case ARM::VLD1DUPq8wb_fixed:
4602     case ARM::VLD1DUPq16wb_fixed:
4603     case ARM::VLD1DUPq32wb_fixed:
4604     case ARM::VLD1DUPq8wb_register:
4605     case ARM::VLD1DUPq16wb_register:
4606     case ARM::VLD1DUPq32wb_register:
4607     case ARM::VLD2DUPd8:
4608     case ARM::VLD2DUPd16:
4609     case ARM::VLD2DUPd32:
4610     case ARM::VLD2DUPd8wb_fixed:
4611     case ARM::VLD2DUPd16wb_fixed:
4612     case ARM::VLD2DUPd32wb_fixed:
4613     case ARM::VLD2DUPd8wb_register:
4614     case ARM::VLD2DUPd16wb_register:
4615     case ARM::VLD2DUPd32wb_register:
4616     case ARM::VLD2DUPq8EvenPseudo:
4617     case ARM::VLD2DUPq8OddPseudo:
4618     case ARM::VLD2DUPq16EvenPseudo:
4619     case ARM::VLD2DUPq16OddPseudo:
4620     case ARM::VLD2DUPq32EvenPseudo:
4621     case ARM::VLD2DUPq32OddPseudo:
4622     case ARM::VLD3DUPq8EvenPseudo:
4623     case ARM::VLD3DUPq8OddPseudo:
4624     case ARM::VLD3DUPq16EvenPseudo:
4625     case ARM::VLD3DUPq16OddPseudo:
4626     case ARM::VLD3DUPq32EvenPseudo:
4627     case ARM::VLD3DUPq32OddPseudo:
4628     case ARM::VLD4DUPd8Pseudo:
4629     case ARM::VLD4DUPd16Pseudo:
4630     case ARM::VLD4DUPd32Pseudo:
4631     case ARM::VLD4DUPd8Pseudo_UPD:
4632     case ARM::VLD4DUPd16Pseudo_UPD:
4633     case ARM::VLD4DUPd32Pseudo_UPD:
4634     case ARM::VLD4DUPq8EvenPseudo:
4635     case ARM::VLD4DUPq8OddPseudo:
4636     case ARM::VLD4DUPq16EvenPseudo:
4637     case ARM::VLD4DUPq16OddPseudo:
4638     case ARM::VLD4DUPq32EvenPseudo:
4639     case ARM::VLD4DUPq32OddPseudo:
4640     case ARM::VLD1LNq8Pseudo:
4641     case ARM::VLD1LNq16Pseudo:
4642     case ARM::VLD1LNq32Pseudo:
4643     case ARM::VLD1LNq8Pseudo_UPD:
4644     case ARM::VLD1LNq16Pseudo_UPD:
4645     case ARM::VLD1LNq32Pseudo_UPD:
4646     case ARM::VLD2LNd8Pseudo:
4647     case ARM::VLD2LNd16Pseudo:
4648     case ARM::VLD2LNd32Pseudo:
4649     case ARM::VLD2LNq16Pseudo:
4650     case ARM::VLD2LNq32Pseudo:
4651     case ARM::VLD2LNd8Pseudo_UPD:
4652     case ARM::VLD2LNd16Pseudo_UPD:
4653     case ARM::VLD2LNd32Pseudo_UPD:
4654     case ARM::VLD2LNq16Pseudo_UPD:
4655     case ARM::VLD2LNq32Pseudo_UPD:
4656     case ARM::VLD4LNd8Pseudo:
4657     case ARM::VLD4LNd16Pseudo:
4658     case ARM::VLD4LNd32Pseudo:
4659     case ARM::VLD4LNq16Pseudo:
4660     case ARM::VLD4LNq32Pseudo:
4661     case ARM::VLD4LNd8Pseudo_UPD:
4662     case ARM::VLD4LNd16Pseudo_UPD:
4663     case ARM::VLD4LNd32Pseudo_UPD:
4664     case ARM::VLD4LNq16Pseudo_UPD:
4665     case ARM::VLD4LNq32Pseudo_UPD:
4666       // If the address is not 64-bit aligned, the latencies of these
4667       // instructions increases by one.
4668       ++Latency;
4669       break;
4670     }
4671 
4672   return Latency;
4673 }
4674 
getPredicationCost(const MachineInstr & MI) const4675 unsigned ARMBaseInstrInfo::getPredicationCost(const MachineInstr &MI) const {
4676   if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4677       MI.isImplicitDef())
4678     return 0;
4679 
4680   if (MI.isBundle())
4681     return 0;
4682 
4683   const MCInstrDesc &MCID = MI.getDesc();
4684 
4685   if (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) &&
4686                         !Subtarget.cheapPredicableCPSRDef())) {
4687     // When predicated, CPSR is an additional source operand for CPSR updating
4688     // instructions, this apparently increases their latencies.
4689     return 1;
4690   }
4691   return 0;
4692 }
4693 
getInstrLatency(const InstrItineraryData * ItinData,const MachineInstr & MI,unsigned * PredCost) const4694 unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4695                                            const MachineInstr &MI,
4696                                            unsigned *PredCost) const {
4697   if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4698       MI.isImplicitDef())
4699     return 1;
4700 
4701   // An instruction scheduler typically runs on unbundled instructions, however
4702   // other passes may query the latency of a bundled instruction.
4703   if (MI.isBundle()) {
4704     unsigned Latency = 0;
4705     MachineBasicBlock::const_instr_iterator I = MI.getIterator();
4706     MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4707     while (++I != E && I->isInsideBundle()) {
4708       if (I->getOpcode() != ARM::t2IT)
4709         Latency += getInstrLatency(ItinData, *I, PredCost);
4710     }
4711     return Latency;
4712   }
4713 
4714   const MCInstrDesc &MCID = MI.getDesc();
4715   if (PredCost && (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) &&
4716                                      !Subtarget.cheapPredicableCPSRDef()))) {
4717     // When predicated, CPSR is an additional source operand for CPSR updating
4718     // instructions, this apparently increases their latencies.
4719     *PredCost = 1;
4720   }
4721   // Be sure to call getStageLatency for an empty itinerary in case it has a
4722   // valid MinLatency property.
4723   if (!ItinData)
4724     return MI.mayLoad() ? 3 : 1;
4725 
4726   unsigned Class = MCID.getSchedClass();
4727 
4728   // For instructions with variable uops, use uops as latency.
4729   if (!ItinData->isEmpty() && ItinData->getNumMicroOps(Class) < 0)
4730     return getNumMicroOps(ItinData, MI);
4731 
4732   // For the common case, fall back on the itinerary's latency.
4733   unsigned Latency = ItinData->getStageLatency(Class);
4734 
4735   // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4736   unsigned DefAlign =
4737       MI.hasOneMemOperand() ? (*MI.memoperands_begin())->getAlign().value() : 0;
4738   int Adj = adjustDefLatency(Subtarget, MI, MCID, DefAlign);
4739   if (Adj >= 0 || (int)Latency > -Adj) {
4740     return Latency + Adj;
4741   }
4742   return Latency;
4743 }
4744 
getInstrLatency(const InstrItineraryData * ItinData,SDNode * Node) const4745 int ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4746                                       SDNode *Node) const {
4747   if (!Node->isMachineOpcode())
4748     return 1;
4749 
4750   if (!ItinData || ItinData->isEmpty())
4751     return 1;
4752 
4753   unsigned Opcode = Node->getMachineOpcode();
4754   switch (Opcode) {
4755   default:
4756     return ItinData->getStageLatency(get(Opcode).getSchedClass());
4757   case ARM::VLDMQIA:
4758   case ARM::VSTMQIA:
4759     return 2;
4760   }
4761 }
4762 
hasHighOperandLatency(const TargetSchedModel & SchedModel,const MachineRegisterInfo * MRI,const MachineInstr & DefMI,unsigned DefIdx,const MachineInstr & UseMI,unsigned UseIdx) const4763 bool ARMBaseInstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel,
4764                                              const MachineRegisterInfo *MRI,
4765                                              const MachineInstr &DefMI,
4766                                              unsigned DefIdx,
4767                                              const MachineInstr &UseMI,
4768                                              unsigned UseIdx) const {
4769   unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4770   unsigned UDomain = UseMI.getDesc().TSFlags & ARMII::DomainMask;
4771   if (Subtarget.nonpipelinedVFP() &&
4772       (DDomain == ARMII::DomainVFP || UDomain == ARMII::DomainVFP))
4773     return true;
4774 
4775   // Hoist VFP / NEON instructions with 4 or higher latency.
4776   unsigned Latency =
4777       SchedModel.computeOperandLatency(&DefMI, DefIdx, &UseMI, UseIdx);
4778   if (Latency <= 3)
4779     return false;
4780   return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON ||
4781          UDomain == ARMII::DomainVFP || UDomain == ARMII::DomainNEON;
4782 }
4783 
hasLowDefLatency(const TargetSchedModel & SchedModel,const MachineInstr & DefMI,unsigned DefIdx) const4784 bool ARMBaseInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
4785                                         const MachineInstr &DefMI,
4786                                         unsigned DefIdx) const {
4787   const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
4788   if (!ItinData || ItinData->isEmpty())
4789     return false;
4790 
4791   unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4792   if (DDomain == ARMII::DomainGeneral) {
4793     unsigned DefClass = DefMI.getDesc().getSchedClass();
4794     int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
4795     return (DefCycle != -1 && DefCycle <= 2);
4796   }
4797   return false;
4798 }
4799 
verifyInstruction(const MachineInstr & MI,StringRef & ErrInfo) const4800 bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr &MI,
4801                                          StringRef &ErrInfo) const {
4802   if (convertAddSubFlagsOpcode(MI.getOpcode())) {
4803     ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG";
4804     return false;
4805   }
4806   if (MI.getOpcode() == ARM::tMOVr && !Subtarget.hasV6Ops()) {
4807     // Make sure we don't generate a lo-lo mov that isn't supported.
4808     if (!ARM::hGPRRegClass.contains(MI.getOperand(0).getReg()) &&
4809         !ARM::hGPRRegClass.contains(MI.getOperand(1).getReg())) {
4810       ErrInfo = "Non-flag-setting Thumb1 mov is v6-only";
4811       return false;
4812     }
4813   }
4814   if (MI.getOpcode() == ARM::tPUSH ||
4815       MI.getOpcode() == ARM::tPOP ||
4816       MI.getOpcode() == ARM::tPOP_RET) {
4817     for (int i = 2, e = MI.getNumOperands(); i < e; ++i) {
4818       if (MI.getOperand(i).isImplicit() ||
4819           !MI.getOperand(i).isReg())
4820         continue;
4821       Register Reg = MI.getOperand(i).getReg();
4822       if (Reg < ARM::R0 || Reg > ARM::R7) {
4823         if (!(MI.getOpcode() == ARM::tPUSH && Reg == ARM::LR) &&
4824             !(MI.getOpcode() == ARM::tPOP_RET && Reg == ARM::PC)) {
4825           ErrInfo = "Unsupported register in Thumb1 push/pop";
4826           return false;
4827         }
4828       }
4829     }
4830   }
4831   if (MI.getOpcode() == ARM::MVE_VMOV_q_rr) {
4832     assert(MI.getOperand(4).isImm() && MI.getOperand(5).isImm());
4833     if ((MI.getOperand(4).getImm() != 2 && MI.getOperand(4).getImm() != 3) ||
4834         MI.getOperand(4).getImm() != MI.getOperand(5).getImm() + 2) {
4835       ErrInfo = "Incorrect array index for MVE_VMOV_q_rr";
4836       return false;
4837     }
4838   }
4839   return true;
4840 }
4841 
4842 // LoadStackGuard has so far only been implemented for MachO. Different code
4843 // sequence is needed for other targets.
expandLoadStackGuardBase(MachineBasicBlock::iterator MI,unsigned LoadImmOpc,unsigned LoadOpc) const4844 void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI,
4845                                                 unsigned LoadImmOpc,
4846                                                 unsigned LoadOpc) const {
4847   assert(!Subtarget.isROPI() && !Subtarget.isRWPI() &&
4848          "ROPI/RWPI not currently supported with stack guard");
4849 
4850   MachineBasicBlock &MBB = *MI->getParent();
4851   DebugLoc DL = MI->getDebugLoc();
4852   Register Reg = MI->getOperand(0).getReg();
4853   const GlobalValue *GV =
4854       cast<GlobalValue>((*MI->memoperands_begin())->getValue());
4855   MachineInstrBuilder MIB;
4856 
4857   BuildMI(MBB, MI, DL, get(LoadImmOpc), Reg)
4858       .addGlobalAddress(GV, 0, ARMII::MO_NONLAZY);
4859 
4860   if (Subtarget.isGVIndirectSymbol(GV)) {
4861     MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
4862     MIB.addReg(Reg, RegState::Kill).addImm(0);
4863     auto Flags = MachineMemOperand::MOLoad |
4864                  MachineMemOperand::MODereferenceable |
4865                  MachineMemOperand::MOInvariant;
4866     MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
4867         MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, Align(4));
4868     MIB.addMemOperand(MMO).add(predOps(ARMCC::AL));
4869   }
4870 
4871   MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
4872   MIB.addReg(Reg, RegState::Kill)
4873       .addImm(0)
4874       .cloneMemRefs(*MI)
4875       .add(predOps(ARMCC::AL));
4876 }
4877 
4878 bool
isFpMLxInstruction(unsigned Opcode,unsigned & MulOpc,unsigned & AddSubOpc,bool & NegAcc,bool & HasLane) const4879 ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
4880                                      unsigned &AddSubOpc,
4881                                      bool &NegAcc, bool &HasLane) const {
4882   DenseMap<unsigned, unsigned>::const_iterator I = MLxEntryMap.find(Opcode);
4883   if (I == MLxEntryMap.end())
4884     return false;
4885 
4886   const ARM_MLxEntry &Entry = ARM_MLxTable[I->second];
4887   MulOpc = Entry.MulOpc;
4888   AddSubOpc = Entry.AddSubOpc;
4889   NegAcc = Entry.NegAcc;
4890   HasLane = Entry.HasLane;
4891   return true;
4892 }
4893 
4894 //===----------------------------------------------------------------------===//
4895 // Execution domains.
4896 //===----------------------------------------------------------------------===//
4897 //
4898 // Some instructions go down the NEON pipeline, some go down the VFP pipeline,
4899 // and some can go down both.  The vmov instructions go down the VFP pipeline,
4900 // but they can be changed to vorr equivalents that are executed by the NEON
4901 // pipeline.
4902 //
4903 // We use the following execution domain numbering:
4904 //
4905 enum ARMExeDomain {
4906   ExeGeneric = 0,
4907   ExeVFP = 1,
4908   ExeNEON = 2
4909 };
4910 
4911 //
4912 // Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h
4913 //
4914 std::pair<uint16_t, uint16_t>
getExecutionDomain(const MachineInstr & MI) const4915 ARMBaseInstrInfo::getExecutionDomain(const MachineInstr &MI) const {
4916   // If we don't have access to NEON instructions then we won't be able
4917   // to swizzle anything to the NEON domain. Check to make sure.
4918   if (Subtarget.hasNEON()) {
4919     // VMOVD, VMOVRS and VMOVSR are VFP instructions, but can be changed to NEON
4920     // if they are not predicated.
4921     if (MI.getOpcode() == ARM::VMOVD && !isPredicated(MI))
4922       return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
4923 
4924     // CortexA9 is particularly picky about mixing the two and wants these
4925     // converted.
4926     if (Subtarget.useNEONForFPMovs() && !isPredicated(MI) &&
4927         (MI.getOpcode() == ARM::VMOVRS || MI.getOpcode() == ARM::VMOVSR ||
4928          MI.getOpcode() == ARM::VMOVS))
4929       return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
4930   }
4931   // No other instructions can be swizzled, so just determine their domain.
4932   unsigned Domain = MI.getDesc().TSFlags & ARMII::DomainMask;
4933 
4934   if (Domain & ARMII::DomainNEON)
4935     return std::make_pair(ExeNEON, 0);
4936 
4937   // Certain instructions can go either way on Cortex-A8.
4938   // Treat them as NEON instructions.
4939   if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8())
4940     return std::make_pair(ExeNEON, 0);
4941 
4942   if (Domain & ARMII::DomainVFP)
4943     return std::make_pair(ExeVFP, 0);
4944 
4945   return std::make_pair(ExeGeneric, 0);
4946 }
4947 
getCorrespondingDRegAndLane(const TargetRegisterInfo * TRI,unsigned SReg,unsigned & Lane)4948 static unsigned getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI,
4949                                             unsigned SReg, unsigned &Lane) {
4950   unsigned DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_0, &ARM::DPRRegClass);
4951   Lane = 0;
4952 
4953   if (DReg != ARM::NoRegister)
4954    return DReg;
4955 
4956   Lane = 1;
4957   DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_1, &ARM::DPRRegClass);
4958 
4959   assert(DReg && "S-register with no D super-register?");
4960   return DReg;
4961 }
4962 
4963 /// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane,
4964 /// set ImplicitSReg to a register number that must be marked as implicit-use or
4965 /// zero if no register needs to be defined as implicit-use.
4966 ///
4967 /// If the function cannot determine if an SPR should be marked implicit use or
4968 /// not, it returns false.
4969 ///
4970 /// This function handles cases where an instruction is being modified from taking
4971 /// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict
4972 /// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other
4973 /// lane of the DPR).
4974 ///
4975 /// If the other SPR is defined, an implicit-use of it should be added. Else,
4976 /// (including the case where the DPR itself is defined), it should not.
4977 ///
getImplicitSPRUseForDPRUse(const TargetRegisterInfo * TRI,MachineInstr & MI,unsigned DReg,unsigned Lane,unsigned & ImplicitSReg)4978 static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI,
4979                                        MachineInstr &MI, unsigned DReg,
4980                                        unsigned Lane, unsigned &ImplicitSReg) {
4981   // If the DPR is defined or used already, the other SPR lane will be chained
4982   // correctly, so there is nothing to be done.
4983   if (MI.definesRegister(DReg, TRI) || MI.readsRegister(DReg, TRI)) {
4984     ImplicitSReg = 0;
4985     return true;
4986   }
4987 
4988   // Otherwise we need to go searching to see if the SPR is set explicitly.
4989   ImplicitSReg = TRI->getSubReg(DReg,
4990                                 (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1);
4991   MachineBasicBlock::LivenessQueryResult LQR =
4992       MI.getParent()->computeRegisterLiveness(TRI, ImplicitSReg, MI);
4993 
4994   if (LQR == MachineBasicBlock::LQR_Live)
4995     return true;
4996   else if (LQR == MachineBasicBlock::LQR_Unknown)
4997     return false;
4998 
4999   // If the register is known not to be live, there is no need to add an
5000   // implicit-use.
5001   ImplicitSReg = 0;
5002   return true;
5003 }
5004 
setExecutionDomain(MachineInstr & MI,unsigned Domain) const5005 void ARMBaseInstrInfo::setExecutionDomain(MachineInstr &MI,
5006                                           unsigned Domain) const {
5007   unsigned DstReg, SrcReg, DReg;
5008   unsigned Lane;
5009   MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
5010   const TargetRegisterInfo *TRI = &getRegisterInfo();
5011   switch (MI.getOpcode()) {
5012   default:
5013     llvm_unreachable("cannot handle opcode!");
5014     break;
5015   case ARM::VMOVD:
5016     if (Domain != ExeNEON)
5017       break;
5018 
5019     // Zap the predicate operands.
5020     assert(!isPredicated(MI) && "Cannot predicate a VORRd");
5021 
5022     // Make sure we've got NEON instructions.
5023     assert(Subtarget.hasNEON() && "VORRd requires NEON");
5024 
5025     // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits)
5026     DstReg = MI.getOperand(0).getReg();
5027     SrcReg = MI.getOperand(1).getReg();
5028 
5029     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5030       MI.RemoveOperand(i - 1);
5031 
5032     // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits)
5033     MI.setDesc(get(ARM::VORRd));
5034     MIB.addReg(DstReg, RegState::Define)
5035         .addReg(SrcReg)
5036         .addReg(SrcReg)
5037         .add(predOps(ARMCC::AL));
5038     break;
5039   case ARM::VMOVRS:
5040     if (Domain != ExeNEON)
5041       break;
5042     assert(!isPredicated(MI) && "Cannot predicate a VGETLN");
5043 
5044     // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits)
5045     DstReg = MI.getOperand(0).getReg();
5046     SrcReg = MI.getOperand(1).getReg();
5047 
5048     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5049       MI.RemoveOperand(i - 1);
5050 
5051     DReg = getCorrespondingDRegAndLane(TRI, SrcReg, Lane);
5052 
5053     // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps)
5054     // Note that DSrc has been widened and the other lane may be undef, which
5055     // contaminates the entire register.
5056     MI.setDesc(get(ARM::VGETLNi32));
5057     MIB.addReg(DstReg, RegState::Define)
5058         .addReg(DReg, RegState::Undef)
5059         .addImm(Lane)
5060         .add(predOps(ARMCC::AL));
5061 
5062     // The old source should be an implicit use, otherwise we might think it
5063     // was dead before here.
5064     MIB.addReg(SrcReg, RegState::Implicit);
5065     break;
5066   case ARM::VMOVSR: {
5067     if (Domain != ExeNEON)
5068       break;
5069     assert(!isPredicated(MI) && "Cannot predicate a VSETLN");
5070 
5071     // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits)
5072     DstReg = MI.getOperand(0).getReg();
5073     SrcReg = MI.getOperand(1).getReg();
5074 
5075     DReg = getCorrespondingDRegAndLane(TRI, DstReg, Lane);
5076 
5077     unsigned ImplicitSReg;
5078     if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg))
5079       break;
5080 
5081     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5082       MI.RemoveOperand(i - 1);
5083 
5084     // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps)
5085     // Again DDst may be undefined at the beginning of this instruction.
5086     MI.setDesc(get(ARM::VSETLNi32));
5087     MIB.addReg(DReg, RegState::Define)
5088         .addReg(DReg, getUndefRegState(!MI.readsRegister(DReg, TRI)))
5089         .addReg(SrcReg)
5090         .addImm(Lane)
5091         .add(predOps(ARMCC::AL));
5092 
5093     // The narrower destination must be marked as set to keep previous chains
5094     // in place.
5095     MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
5096     if (ImplicitSReg != 0)
5097       MIB.addReg(ImplicitSReg, RegState::Implicit);
5098     break;
5099     }
5100     case ARM::VMOVS: {
5101       if (Domain != ExeNEON)
5102         break;
5103 
5104       // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits)
5105       DstReg = MI.getOperand(0).getReg();
5106       SrcReg = MI.getOperand(1).getReg();
5107 
5108       unsigned DstLane = 0, SrcLane = 0, DDst, DSrc;
5109       DDst = getCorrespondingDRegAndLane(TRI, DstReg, DstLane);
5110       DSrc = getCorrespondingDRegAndLane(TRI, SrcReg, SrcLane);
5111 
5112       unsigned ImplicitSReg;
5113       if (!getImplicitSPRUseForDPRUse(TRI, MI, DSrc, SrcLane, ImplicitSReg))
5114         break;
5115 
5116       for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5117         MI.RemoveOperand(i - 1);
5118 
5119       if (DSrc == DDst) {
5120         // Destination can be:
5121         //     %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits)
5122         MI.setDesc(get(ARM::VDUPLN32d));
5123         MIB.addReg(DDst, RegState::Define)
5124             .addReg(DDst, getUndefRegState(!MI.readsRegister(DDst, TRI)))
5125             .addImm(SrcLane)
5126             .add(predOps(ARMCC::AL));
5127 
5128         // Neither the source or the destination are naturally represented any
5129         // more, so add them in manually.
5130         MIB.addReg(DstReg, RegState::Implicit | RegState::Define);
5131         MIB.addReg(SrcReg, RegState::Implicit);
5132         if (ImplicitSReg != 0)
5133           MIB.addReg(ImplicitSReg, RegState::Implicit);
5134         break;
5135       }
5136 
5137       // In general there's no single instruction that can perform an S <-> S
5138       // move in NEON space, but a pair of VEXT instructions *can* do the
5139       // job. It turns out that the VEXTs needed will only use DSrc once, with
5140       // the position based purely on the combination of lane-0 and lane-1
5141       // involved. For example
5142       //     vmov s0, s2 -> vext.32 d0, d0, d1, #1  vext.32 d0, d0, d0, #1
5143       //     vmov s1, s3 -> vext.32 d0, d1, d0, #1  vext.32 d0, d0, d0, #1
5144       //     vmov s0, s3 -> vext.32 d0, d0, d0, #1  vext.32 d0, d1, d0, #1
5145       //     vmov s1, s2 -> vext.32 d0, d0, d0, #1  vext.32 d0, d0, d1, #1
5146       //
5147       // Pattern of the MachineInstrs is:
5148       //     %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits)
5149       MachineInstrBuilder NewMIB;
5150       NewMIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::VEXTd32),
5151                        DDst);
5152 
5153       // On the first instruction, both DSrc and DDst may be undef if present.
5154       // Specifically when the original instruction didn't have them as an
5155       // <imp-use>.
5156       unsigned CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst;
5157       bool CurUndef = !MI.readsRegister(CurReg, TRI);
5158       NewMIB.addReg(CurReg, getUndefRegState(CurUndef));
5159 
5160       CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst;
5161       CurUndef = !MI.readsRegister(CurReg, TRI);
5162       NewMIB.addReg(CurReg, getUndefRegState(CurUndef))
5163             .addImm(1)
5164             .add(predOps(ARMCC::AL));
5165 
5166       if (SrcLane == DstLane)
5167         NewMIB.addReg(SrcReg, RegState::Implicit);
5168 
5169       MI.setDesc(get(ARM::VEXTd32));
5170       MIB.addReg(DDst, RegState::Define);
5171 
5172       // On the second instruction, DDst has definitely been defined above, so
5173       // it is not undef. DSrc, if present, can be undef as above.
5174       CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst;
5175       CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI);
5176       MIB.addReg(CurReg, getUndefRegState(CurUndef));
5177 
5178       CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst;
5179       CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI);
5180       MIB.addReg(CurReg, getUndefRegState(CurUndef))
5181          .addImm(1)
5182          .add(predOps(ARMCC::AL));
5183 
5184       if (SrcLane != DstLane)
5185         MIB.addReg(SrcReg, RegState::Implicit);
5186 
5187       // As before, the original destination is no longer represented, add it
5188       // implicitly.
5189       MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
5190       if (ImplicitSReg != 0)
5191         MIB.addReg(ImplicitSReg, RegState::Implicit);
5192       break;
5193     }
5194   }
5195 }
5196 
5197 //===----------------------------------------------------------------------===//
5198 // Partial register updates
5199 //===----------------------------------------------------------------------===//
5200 //
5201 // Swift renames NEON registers with 64-bit granularity.  That means any
5202 // instruction writing an S-reg implicitly reads the containing D-reg.  The
5203 // problem is mostly avoided by translating f32 operations to v2f32 operations
5204 // on D-registers, but f32 loads are still a problem.
5205 //
5206 // These instructions can load an f32 into a NEON register:
5207 //
5208 // VLDRS - Only writes S, partial D update.
5209 // VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops.
5210 // VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops.
5211 //
5212 // FCONSTD can be used as a dependency-breaking instruction.
getPartialRegUpdateClearance(const MachineInstr & MI,unsigned OpNum,const TargetRegisterInfo * TRI) const5213 unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
5214     const MachineInstr &MI, unsigned OpNum,
5215     const TargetRegisterInfo *TRI) const {
5216   auto PartialUpdateClearance = Subtarget.getPartialUpdateClearance();
5217   if (!PartialUpdateClearance)
5218     return 0;
5219 
5220   assert(TRI && "Need TRI instance");
5221 
5222   const MachineOperand &MO = MI.getOperand(OpNum);
5223   if (MO.readsReg())
5224     return 0;
5225   Register Reg = MO.getReg();
5226   int UseOp = -1;
5227 
5228   switch (MI.getOpcode()) {
5229   // Normal instructions writing only an S-register.
5230   case ARM::VLDRS:
5231   case ARM::FCONSTS:
5232   case ARM::VMOVSR:
5233   case ARM::VMOVv8i8:
5234   case ARM::VMOVv4i16:
5235   case ARM::VMOVv2i32:
5236   case ARM::VMOVv2f32:
5237   case ARM::VMOVv1i64:
5238     UseOp = MI.findRegisterUseOperandIdx(Reg, false, TRI);
5239     break;
5240 
5241     // Explicitly reads the dependency.
5242   case ARM::VLD1LNd32:
5243     UseOp = 3;
5244     break;
5245   default:
5246     return 0;
5247   }
5248 
5249   // If this instruction actually reads a value from Reg, there is no unwanted
5250   // dependency.
5251   if (UseOp != -1 && MI.getOperand(UseOp).readsReg())
5252     return 0;
5253 
5254   // We must be able to clobber the whole D-reg.
5255   if (Register::isVirtualRegister(Reg)) {
5256     // Virtual register must be a def undef foo:ssub_0 operand.
5257     if (!MO.getSubReg() || MI.readsVirtualRegister(Reg))
5258       return 0;
5259   } else if (ARM::SPRRegClass.contains(Reg)) {
5260     // Physical register: MI must define the full D-reg.
5261     unsigned DReg = TRI->getMatchingSuperReg(Reg, ARM::ssub_0,
5262                                              &ARM::DPRRegClass);
5263     if (!DReg || !MI.definesRegister(DReg, TRI))
5264       return 0;
5265   }
5266 
5267   // MI has an unwanted D-register dependency.
5268   // Avoid defs in the previous N instructrions.
5269   return PartialUpdateClearance;
5270 }
5271 
5272 // Break a partial register dependency after getPartialRegUpdateClearance
5273 // returned non-zero.
breakPartialRegDependency(MachineInstr & MI,unsigned OpNum,const TargetRegisterInfo * TRI) const5274 void ARMBaseInstrInfo::breakPartialRegDependency(
5275     MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const {
5276   assert(OpNum < MI.getDesc().getNumDefs() && "OpNum is not a def");
5277   assert(TRI && "Need TRI instance");
5278 
5279   const MachineOperand &MO = MI.getOperand(OpNum);
5280   Register Reg = MO.getReg();
5281   assert(Register::isPhysicalRegister(Reg) &&
5282          "Can't break virtual register dependencies.");
5283   unsigned DReg = Reg;
5284 
5285   // If MI defines an S-reg, find the corresponding D super-register.
5286   if (ARM::SPRRegClass.contains(Reg)) {
5287     DReg = ARM::D0 + (Reg - ARM::S0) / 2;
5288     assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken");
5289   }
5290 
5291   assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps");
5292   assert(MI.definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg");
5293 
5294   // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines
5295   // the full D-register by loading the same value to both lanes.  The
5296   // instruction is micro-coded with 2 uops, so don't do this until we can
5297   // properly schedule micro-coded instructions.  The dispatcher stalls cause
5298   // too big regressions.
5299 
5300   // Insert the dependency-breaking FCONSTD before MI.
5301   // 96 is the encoding of 0.5, but the actual value doesn't matter here.
5302   BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::FCONSTD), DReg)
5303       .addImm(96)
5304       .add(predOps(ARMCC::AL));
5305   MI.addRegisterKilled(DReg, TRI, true);
5306 }
5307 
hasNOP() const5308 bool ARMBaseInstrInfo::hasNOP() const {
5309   return Subtarget.getFeatureBits()[ARM::HasV6KOps];
5310 }
5311 
isSwiftFastImmShift(const MachineInstr * MI) const5312 bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const {
5313   if (MI->getNumOperands() < 4)
5314     return true;
5315   unsigned ShOpVal = MI->getOperand(3).getImm();
5316   unsigned ShImm = ARM_AM::getSORegOffset(ShOpVal);
5317   // Swift supports faster shifts for: lsl 2, lsl 1, and lsr 1.
5318   if ((ShImm == 1 && ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsr) ||
5319       ((ShImm == 1 || ShImm == 2) &&
5320        ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsl))
5321     return true;
5322 
5323   return false;
5324 }
5325 
getRegSequenceLikeInputs(const MachineInstr & MI,unsigned DefIdx,SmallVectorImpl<RegSubRegPairAndIdx> & InputRegs) const5326 bool ARMBaseInstrInfo::getRegSequenceLikeInputs(
5327     const MachineInstr &MI, unsigned DefIdx,
5328     SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
5329   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5330   assert(MI.isRegSequenceLike() && "Invalid kind of instruction");
5331 
5332   switch (MI.getOpcode()) {
5333   case ARM::VMOVDRR:
5334     // dX = VMOVDRR rY, rZ
5335     // is the same as:
5336     // dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1
5337     // Populate the InputRegs accordingly.
5338     // rY
5339     const MachineOperand *MOReg = &MI.getOperand(1);
5340     if (!MOReg->isUndef())
5341       InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(),
5342                                               MOReg->getSubReg(), ARM::ssub_0));
5343     // rZ
5344     MOReg = &MI.getOperand(2);
5345     if (!MOReg->isUndef())
5346       InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(),
5347                                               MOReg->getSubReg(), ARM::ssub_1));
5348     return true;
5349   }
5350   llvm_unreachable("Target dependent opcode missing");
5351 }
5352 
getExtractSubregLikeInputs(const MachineInstr & MI,unsigned DefIdx,RegSubRegPairAndIdx & InputReg) const5353 bool ARMBaseInstrInfo::getExtractSubregLikeInputs(
5354     const MachineInstr &MI, unsigned DefIdx,
5355     RegSubRegPairAndIdx &InputReg) const {
5356   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5357   assert(MI.isExtractSubregLike() && "Invalid kind of instruction");
5358 
5359   switch (MI.getOpcode()) {
5360   case ARM::VMOVRRD:
5361     // rX, rY = VMOVRRD dZ
5362     // is the same as:
5363     // rX = EXTRACT_SUBREG dZ, ssub_0
5364     // rY = EXTRACT_SUBREG dZ, ssub_1
5365     const MachineOperand &MOReg = MI.getOperand(2);
5366     if (MOReg.isUndef())
5367       return false;
5368     InputReg.Reg = MOReg.getReg();
5369     InputReg.SubReg = MOReg.getSubReg();
5370     InputReg.SubIdx = DefIdx == 0 ? ARM::ssub_0 : ARM::ssub_1;
5371     return true;
5372   }
5373   llvm_unreachable("Target dependent opcode missing");
5374 }
5375 
getInsertSubregLikeInputs(const MachineInstr & MI,unsigned DefIdx,RegSubRegPair & BaseReg,RegSubRegPairAndIdx & InsertedReg) const5376 bool ARMBaseInstrInfo::getInsertSubregLikeInputs(
5377     const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg,
5378     RegSubRegPairAndIdx &InsertedReg) const {
5379   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5380   assert(MI.isInsertSubregLike() && "Invalid kind of instruction");
5381 
5382   switch (MI.getOpcode()) {
5383   case ARM::VSETLNi32:
5384   case ARM::MVE_VMOV_to_lane_32:
5385     // dX = VSETLNi32 dY, rZ, imm
5386     // qX = MVE_VMOV_to_lane_32 qY, rZ, imm
5387     const MachineOperand &MOBaseReg = MI.getOperand(1);
5388     const MachineOperand &MOInsertedReg = MI.getOperand(2);
5389     if (MOInsertedReg.isUndef())
5390       return false;
5391     const MachineOperand &MOIndex = MI.getOperand(3);
5392     BaseReg.Reg = MOBaseReg.getReg();
5393     BaseReg.SubReg = MOBaseReg.getSubReg();
5394 
5395     InsertedReg.Reg = MOInsertedReg.getReg();
5396     InsertedReg.SubReg = MOInsertedReg.getSubReg();
5397     InsertedReg.SubIdx = ARM::ssub_0 + MOIndex.getImm();
5398     return true;
5399   }
5400   llvm_unreachable("Target dependent opcode missing");
5401 }
5402 
5403 std::pair<unsigned, unsigned>
decomposeMachineOperandsTargetFlags(unsigned TF) const5404 ARMBaseInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5405   const unsigned Mask = ARMII::MO_OPTION_MASK;
5406   return std::make_pair(TF & Mask, TF & ~Mask);
5407 }
5408 
5409 ArrayRef<std::pair<unsigned, const char *>>
getSerializableDirectMachineOperandTargetFlags() const5410 ARMBaseInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5411   using namespace ARMII;
5412 
5413   static const std::pair<unsigned, const char *> TargetFlags[] = {
5414       {MO_LO16, "arm-lo16"}, {MO_HI16, "arm-hi16"}};
5415   return makeArrayRef(TargetFlags);
5416 }
5417 
5418 ArrayRef<std::pair<unsigned, const char *>>
getSerializableBitmaskMachineOperandTargetFlags() const5419 ARMBaseInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5420   using namespace ARMII;
5421 
5422   static const std::pair<unsigned, const char *> TargetFlags[] = {
5423       {MO_COFFSTUB, "arm-coffstub"},
5424       {MO_GOT, "arm-got"},
5425       {MO_SBREL, "arm-sbrel"},
5426       {MO_DLLIMPORT, "arm-dllimport"},
5427       {MO_SECREL, "arm-secrel"},
5428       {MO_NONLAZY, "arm-nonlazy"}};
5429   return makeArrayRef(TargetFlags);
5430 }
5431 
isAddImmediate(const MachineInstr & MI,Register Reg) const5432 Optional<RegImmPair> ARMBaseInstrInfo::isAddImmediate(const MachineInstr &MI,
5433                                                       Register Reg) const {
5434   int Sign = 1;
5435   unsigned Opcode = MI.getOpcode();
5436   int64_t Offset = 0;
5437 
5438   // TODO: Handle cases where Reg is a super- or sub-register of the
5439   // destination register.
5440   const MachineOperand &Op0 = MI.getOperand(0);
5441   if (!Op0.isReg() || Reg != Op0.getReg())
5442     return None;
5443 
5444   // We describe SUBri or ADDri instructions.
5445   if (Opcode == ARM::SUBri)
5446     Sign = -1;
5447   else if (Opcode != ARM::ADDri)
5448     return None;
5449 
5450   // TODO: Third operand can be global address (usually some string). Since
5451   //       strings can be relocated we cannot calculate their offsets for
5452   //       now.
5453   if (!MI.getOperand(1).isReg() || !MI.getOperand(2).isImm())
5454     return None;
5455 
5456   Offset = MI.getOperand(2).getImm() * Sign;
5457   return RegImmPair{MI.getOperand(1).getReg(), Offset};
5458 }
5459 
registerDefinedBetween(unsigned Reg,MachineBasicBlock::iterator From,MachineBasicBlock::iterator To,const TargetRegisterInfo * TRI)5460 bool llvm::registerDefinedBetween(unsigned Reg,
5461                                   MachineBasicBlock::iterator From,
5462                                   MachineBasicBlock::iterator To,
5463                                   const TargetRegisterInfo *TRI) {
5464   for (auto I = From; I != To; ++I)
5465     if (I->modifiesRegister(Reg, TRI))
5466       return true;
5467   return false;
5468 }
5469 
findCMPToFoldIntoCBZ(MachineInstr * Br,const TargetRegisterInfo * TRI)5470 MachineInstr *llvm::findCMPToFoldIntoCBZ(MachineInstr *Br,
5471                                          const TargetRegisterInfo *TRI) {
5472   // Search backwards to the instruction that defines CSPR. This may or not
5473   // be a CMP, we check that after this loop. If we find another instruction
5474   // that reads cpsr, we return nullptr.
5475   MachineBasicBlock::iterator CmpMI = Br;
5476   while (CmpMI != Br->getParent()->begin()) {
5477     --CmpMI;
5478     if (CmpMI->modifiesRegister(ARM::CPSR, TRI))
5479       break;
5480     if (CmpMI->readsRegister(ARM::CPSR, TRI))
5481       break;
5482   }
5483 
5484   // Check that this inst is a CMP r[0-7], #0 and that the register
5485   // is not redefined between the cmp and the br.
5486   if (CmpMI->getOpcode() != ARM::tCMPi8 && CmpMI->getOpcode() != ARM::t2CMPri)
5487     return nullptr;
5488   Register Reg = CmpMI->getOperand(0).getReg();
5489   Register PredReg;
5490   ARMCC::CondCodes Pred = getInstrPredicate(*CmpMI, PredReg);
5491   if (Pred != ARMCC::AL || CmpMI->getOperand(1).getImm() != 0)
5492     return nullptr;
5493   if (!isARMLowRegister(Reg))
5494     return nullptr;
5495   if (registerDefinedBetween(Reg, CmpMI->getNextNode(), Br, TRI))
5496     return nullptr;
5497 
5498   return &*CmpMI;
5499 }
5500 
ConstantMaterializationCost(unsigned Val,const ARMSubtarget * Subtarget,bool ForCodesize)5501 unsigned llvm::ConstantMaterializationCost(unsigned Val,
5502                                            const ARMSubtarget *Subtarget,
5503                                            bool ForCodesize) {
5504   if (Subtarget->isThumb()) {
5505     if (Val <= 255) // MOV
5506       return ForCodesize ? 2 : 1;
5507     if (Subtarget->hasV6T2Ops() && (Val <= 0xffff ||                    // MOV
5508                                     ARM_AM::getT2SOImmVal(Val) != -1 || // MOVW
5509                                     ARM_AM::getT2SOImmVal(~Val) != -1)) // MVN
5510       return ForCodesize ? 4 : 1;
5511     if (Val <= 510) // MOV + ADDi8
5512       return ForCodesize ? 4 : 2;
5513     if (~Val <= 255) // MOV + MVN
5514       return ForCodesize ? 4 : 2;
5515     if (ARM_AM::isThumbImmShiftedVal(Val)) // MOV + LSL
5516       return ForCodesize ? 4 : 2;
5517   } else {
5518     if (ARM_AM::getSOImmVal(Val) != -1) // MOV
5519       return ForCodesize ? 4 : 1;
5520     if (ARM_AM::getSOImmVal(~Val) != -1) // MVN
5521       return ForCodesize ? 4 : 1;
5522     if (Subtarget->hasV6T2Ops() && Val <= 0xffff) // MOVW
5523       return ForCodesize ? 4 : 1;
5524     if (ARM_AM::isSOImmTwoPartVal(Val)) // two instrs
5525       return ForCodesize ? 8 : 2;
5526     if (ARM_AM::isSOImmTwoPartValNeg(Val)) // two instrs
5527       return ForCodesize ? 8 : 2;
5528   }
5529   if (Subtarget->useMovt()) // MOVW + MOVT
5530     return ForCodesize ? 8 : 2;
5531   return ForCodesize ? 8 : 3; // Literal pool load
5532 }
5533 
HasLowerConstantMaterializationCost(unsigned Val1,unsigned Val2,const ARMSubtarget * Subtarget,bool ForCodesize)5534 bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2,
5535                                                const ARMSubtarget *Subtarget,
5536                                                bool ForCodesize) {
5537   // Check with ForCodesize
5538   unsigned Cost1 = ConstantMaterializationCost(Val1, Subtarget, ForCodesize);
5539   unsigned Cost2 = ConstantMaterializationCost(Val2, Subtarget, ForCodesize);
5540   if (Cost1 < Cost2)
5541     return true;
5542   if (Cost1 > Cost2)
5543     return false;
5544 
5545   // If they are equal, try with !ForCodesize
5546   return ConstantMaterializationCost(Val1, Subtarget, !ForCodesize) <
5547          ConstantMaterializationCost(Val2, Subtarget, !ForCodesize);
5548 }
5549 
5550 /// Constants defining how certain sequences should be outlined.
5551 /// This encompasses how an outlined function should be called, and what kind of
5552 /// frame should be emitted for that outlined function.
5553 ///
5554 /// \p MachineOutlinerTailCall implies that the function is being created from
5555 /// a sequence of instructions ending in a return.
5556 ///
5557 /// That is,
5558 ///
5559 /// I1                                OUTLINED_FUNCTION:
5560 /// I2    --> B OUTLINED_FUNCTION     I1
5561 /// BX LR                             I2
5562 ///                                   BX LR
5563 ///
5564 /// +-------------------------+--------+-----+
5565 /// |                         | Thumb2 | ARM |
5566 /// +-------------------------+--------+-----+
5567 /// | Call overhead in Bytes  |      4 |   4 |
5568 /// | Frame overhead in Bytes |      0 |   0 |
5569 /// | Stack fixup required    |     No |  No |
5570 /// +-------------------------+--------+-----+
5571 ///
5572 /// \p MachineOutlinerThunk implies that the function is being created from
5573 /// a sequence of instructions ending in a call. The outlined function is
5574 /// called with a BL instruction, and the outlined function tail-calls the
5575 /// original call destination.
5576 ///
5577 /// That is,
5578 ///
5579 /// I1                                OUTLINED_FUNCTION:
5580 /// I2   --> BL OUTLINED_FUNCTION     I1
5581 /// BL f                              I2
5582 ///                                   B f
5583 ///
5584 /// +-------------------------+--------+-----+
5585 /// |                         | Thumb2 | ARM |
5586 /// +-------------------------+--------+-----+
5587 /// | Call overhead in Bytes  |      4 |   4 |
5588 /// | Frame overhead in Bytes |      0 |   0 |
5589 /// | Stack fixup required    |     No |  No |
5590 /// +-------------------------+--------+-----+
5591 ///
5592 /// \p MachineOutlinerNoLRSave implies that the function should be called using
5593 /// a BL instruction, but doesn't require LR to be saved and restored. This
5594 /// happens when LR is known to be dead.
5595 ///
5596 /// That is,
5597 ///
5598 /// I1                                OUTLINED_FUNCTION:
5599 /// I2 --> BL OUTLINED_FUNCTION       I1
5600 /// I3                                I2
5601 ///                                   I3
5602 ///                                   BX LR
5603 ///
5604 /// +-------------------------+--------+-----+
5605 /// |                         | Thumb2 | ARM |
5606 /// +-------------------------+--------+-----+
5607 /// | Call overhead in Bytes  |      4 |   4 |
5608 /// | Frame overhead in Bytes |      4 |   4 |
5609 /// | Stack fixup required    |     No |  No |
5610 /// +-------------------------+--------+-----+
5611 ///
5612 /// \p MachineOutlinerRegSave implies that the function should be called with a
5613 /// save and restore of LR to an available register. This allows us to avoid
5614 /// stack fixups. Note that this outlining variant is compatible with the
5615 /// NoLRSave case.
5616 ///
5617 /// That is,
5618 ///
5619 /// I1     Save LR                    OUTLINED_FUNCTION:
5620 /// I2 --> BL OUTLINED_FUNCTION       I1
5621 /// I3     Restore LR                 I2
5622 ///                                   I3
5623 ///                                   BX LR
5624 ///
5625 /// +-------------------------+--------+-----+
5626 /// |                         | Thumb2 | ARM |
5627 /// +-------------------------+--------+-----+
5628 /// | Call overhead in Bytes  |      8 |  12 |
5629 /// | Frame overhead in Bytes |      2 |   4 |
5630 /// | Stack fixup required    |     No |  No |
5631 /// +-------------------------+--------+-----+
5632 ///
5633 /// \p MachineOutlinerDefault implies that the function should be called with
5634 /// a save and restore of LR to the stack.
5635 ///
5636 /// That is,
5637 ///
5638 /// I1     Save LR                    OUTLINED_FUNCTION:
5639 /// I2 --> BL OUTLINED_FUNCTION       I1
5640 /// I3     Restore LR                 I2
5641 ///                                   I3
5642 ///                                   BX LR
5643 ///
5644 /// +-------------------------+--------+-----+
5645 /// |                         | Thumb2 | ARM |
5646 /// +-------------------------+--------+-----+
5647 /// | Call overhead in Bytes  |      8 |  12 |
5648 /// | Frame overhead in Bytes |      2 |   4 |
5649 /// | Stack fixup required    |    Yes | Yes |
5650 /// +-------------------------+--------+-----+
5651 
5652 enum MachineOutlinerClass {
5653   MachineOutlinerTailCall,
5654   MachineOutlinerThunk,
5655   MachineOutlinerNoLRSave,
5656   MachineOutlinerRegSave,
5657   MachineOutlinerDefault
5658 };
5659 
5660 enum MachineOutlinerMBBFlags {
5661   LRUnavailableSomewhere = 0x2,
5662   HasCalls = 0x4,
5663   UnsafeRegsDead = 0x8
5664 };
5665 
5666 struct OutlinerCosts {
5667   const int CallTailCall;
5668   const int FrameTailCall;
5669   const int CallThunk;
5670   const int FrameThunk;
5671   const int CallNoLRSave;
5672   const int FrameNoLRSave;
5673   const int CallRegSave;
5674   const int FrameRegSave;
5675   const int CallDefault;
5676   const int FrameDefault;
5677   const int SaveRestoreLROnStack;
5678 
OutlinerCostsOutlinerCosts5679   OutlinerCosts(const ARMSubtarget &target)
5680       : CallTailCall(target.isThumb() ? 4 : 4),
5681         FrameTailCall(target.isThumb() ? 0 : 0),
5682         CallThunk(target.isThumb() ? 4 : 4),
5683         FrameThunk(target.isThumb() ? 0 : 0),
5684         CallNoLRSave(target.isThumb() ? 4 : 4),
5685         FrameNoLRSave(target.isThumb() ? 4 : 4),
5686         CallRegSave(target.isThumb() ? 8 : 12),
5687         FrameRegSave(target.isThumb() ? 2 : 4),
5688         CallDefault(target.isThumb() ? 8 : 12),
5689         FrameDefault(target.isThumb() ? 2 : 4),
5690         SaveRestoreLROnStack(target.isThumb() ? 8 : 8) {}
5691 };
5692 
5693 unsigned
findRegisterToSaveLRTo(const outliner::Candidate & C) const5694 ARMBaseInstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const {
5695   assert(C.LRUWasSet && "LRU wasn't set?");
5696   MachineFunction *MF = C.getMF();
5697   const ARMBaseRegisterInfo *ARI = static_cast<const ARMBaseRegisterInfo *>(
5698       MF->getSubtarget().getRegisterInfo());
5699 
5700   BitVector regsReserved = ARI->getReservedRegs(*MF);
5701   // Check if there is an available register across the sequence that we can
5702   // use.
5703   for (unsigned Reg : ARM::rGPRRegClass) {
5704     if (!(Reg < regsReserved.size() && regsReserved.test(Reg)) &&
5705         Reg != ARM::LR &&  // LR is not reserved, but don't use it.
5706         Reg != ARM::R12 && // R12 is not guaranteed to be preserved.
5707         C.LRU.available(Reg) && C.UsedInSequence.available(Reg))
5708       return Reg;
5709   }
5710 
5711   // No suitable register. Return 0.
5712   return 0u;
5713 }
5714 
5715 // Compute liveness of LR at the point after the interval [I, E), which
5716 // denotes a *backward* iteration through instructions. Used only for return
5717 // basic blocks, which do not end with a tail call.
isLRAvailable(const TargetRegisterInfo & TRI,MachineBasicBlock::reverse_iterator I,MachineBasicBlock::reverse_iterator E)5718 static bool isLRAvailable(const TargetRegisterInfo &TRI,
5719                           MachineBasicBlock::reverse_iterator I,
5720                           MachineBasicBlock::reverse_iterator E) {
5721   // At the end of the function LR dead.
5722   bool Live = false;
5723   for (; I != E; ++I) {
5724     const MachineInstr &MI = *I;
5725 
5726     // Check defs of LR.
5727     if (MI.modifiesRegister(ARM::LR, &TRI))
5728       Live = false;
5729 
5730     // Check uses of LR.
5731     unsigned Opcode = MI.getOpcode();
5732     if (Opcode == ARM::BX_RET || Opcode == ARM::MOVPCLR ||
5733         Opcode == ARM::SUBS_PC_LR || Opcode == ARM::tBX_RET ||
5734         Opcode == ARM::tBXNS_RET) {
5735       // These instructions use LR, but it's not an (explicit or implicit)
5736       // operand.
5737       Live = true;
5738       continue;
5739     }
5740     if (MI.readsRegister(ARM::LR, &TRI))
5741       Live = true;
5742   }
5743   return !Live;
5744 }
5745 
getOutliningCandidateInfo(std::vector<outliner::Candidate> & RepeatedSequenceLocs) const5746 outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo(
5747     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
5748   outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5749   unsigned SequenceSize =
5750       std::accumulate(FirstCand.front(), std::next(FirstCand.back()), 0,
5751                       [this](unsigned Sum, const MachineInstr &MI) {
5752                         return Sum + getInstSizeInBytes(MI);
5753                       });
5754 
5755   // Properties about candidate MBBs that hold for all of them.
5756   unsigned FlagsSetInAll = 0xF;
5757 
5758   // Compute liveness information for each candidate, and set FlagsSetInAll.
5759   const TargetRegisterInfo &TRI = getRegisterInfo();
5760   std::for_each(
5761       RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5762       [&FlagsSetInAll](outliner::Candidate &C) { FlagsSetInAll &= C.Flags; });
5763 
5764   // According to the ARM Procedure Call Standard, the following are
5765   // undefined on entry/exit from a function call:
5766   //
5767   // * Register R12(IP),
5768   // * Condition codes (and thus the CPSR register)
5769   //
5770   // Since we control the instructions which are part of the outlined regions
5771   // we don't need to be fully compliant with the AAPCS, but we have to
5772   // guarantee that if a veneer is inserted at link time the code is still
5773   // correct.  Because of this, we can't outline any sequence of instructions
5774   // where one of these registers is live into/across it. Thus, we need to
5775   // delete those candidates.
5776   auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
5777     // If the unsafe registers in this block are all dead, then we don't need
5778     // to compute liveness here.
5779     if (C.Flags & UnsafeRegsDead)
5780       return false;
5781     C.initLRU(TRI);
5782     LiveRegUnits LRU = C.LRU;
5783     return (!LRU.available(ARM::R12) || !LRU.available(ARM::CPSR));
5784   };
5785 
5786   // Are there any candidates where those registers are live?
5787   if (!(FlagsSetInAll & UnsafeRegsDead)) {
5788     // Erase every candidate that violates the restrictions above. (It could be
5789     // true that we have viable candidates, so it's not worth bailing out in
5790     // the case that, say, 1 out of 20 candidates violate the restructions.)
5791     llvm::erase_if(RepeatedSequenceLocs, CantGuaranteeValueAcrossCall);
5792 
5793     // If the sequence doesn't have enough candidates left, then we're done.
5794     if (RepeatedSequenceLocs.size() < 2)
5795       return outliner::OutlinedFunction();
5796   }
5797 
5798   // At this point, we have only "safe" candidates to outline. Figure out
5799   // frame + call instruction information.
5800 
5801   unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back()->getOpcode();
5802 
5803   // Helper lambda which sets call information for every candidate.
5804   auto SetCandidateCallInfo =
5805       [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
5806         for (outliner::Candidate &C : RepeatedSequenceLocs)
5807           C.setCallInfo(CallID, NumBytesForCall);
5808       };
5809 
5810   OutlinerCosts Costs(Subtarget);
5811   unsigned FrameID = MachineOutlinerDefault;
5812   unsigned NumBytesToCreateFrame = Costs.FrameDefault;
5813 
5814   // If the last instruction in any candidate is a terminator, then we should
5815   // tail call all of the candidates.
5816   if (RepeatedSequenceLocs[0].back()->isTerminator()) {
5817     FrameID = MachineOutlinerTailCall;
5818     NumBytesToCreateFrame = Costs.FrameTailCall;
5819     SetCandidateCallInfo(MachineOutlinerTailCall, Costs.CallTailCall);
5820   } else if (LastInstrOpcode == ARM::BL || LastInstrOpcode == ARM::BLX ||
5821              LastInstrOpcode == ARM::BLX_noip || LastInstrOpcode == ARM::tBL ||
5822              LastInstrOpcode == ARM::tBLXr ||
5823              LastInstrOpcode == ARM::tBLXr_noip ||
5824              LastInstrOpcode == ARM::tBLXi) {
5825     FrameID = MachineOutlinerThunk;
5826     NumBytesToCreateFrame = Costs.FrameThunk;
5827     SetCandidateCallInfo(MachineOutlinerThunk, Costs.CallThunk);
5828   } else {
5829     // We need to decide how to emit calls + frames. We can always emit the same
5830     // frame if we don't need to save to the stack. If we have to save to the
5831     // stack, then we need a different frame.
5832     unsigned NumBytesNoStackCalls = 0;
5833     std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
5834 
5835     for (outliner::Candidate &C : RepeatedSequenceLocs) {
5836       C.initLRU(TRI);
5837       // LR liveness is overestimated in return blocks, unless they end with a
5838       // tail call.
5839       const auto Last = C.getMBB()->rbegin();
5840       const bool LRIsAvailable =
5841           C.getMBB()->isReturnBlock() && !Last->isCall()
5842               ? isLRAvailable(TRI, Last,
5843                               (MachineBasicBlock::reverse_iterator)C.front())
5844               : C.LRU.available(ARM::LR);
5845       if (LRIsAvailable) {
5846         FrameID = MachineOutlinerNoLRSave;
5847         NumBytesNoStackCalls += Costs.CallNoLRSave;
5848         C.setCallInfo(MachineOutlinerNoLRSave, Costs.CallNoLRSave);
5849         CandidatesWithoutStackFixups.push_back(C);
5850       }
5851 
5852       // Is an unused register available? If so, we won't modify the stack, so
5853       // we can outline with the same frame type as those that don't save LR.
5854       else if (findRegisterToSaveLRTo(C)) {
5855         FrameID = MachineOutlinerRegSave;
5856         NumBytesNoStackCalls += Costs.CallRegSave;
5857         C.setCallInfo(MachineOutlinerRegSave, Costs.CallRegSave);
5858         CandidatesWithoutStackFixups.push_back(C);
5859       }
5860 
5861       // Is SP used in the sequence at all? If not, we don't have to modify
5862       // the stack, so we are guaranteed to get the same frame.
5863       else if (C.UsedInSequence.available(ARM::SP)) {
5864         NumBytesNoStackCalls += Costs.CallDefault;
5865         C.setCallInfo(MachineOutlinerDefault, Costs.CallDefault);
5866         CandidatesWithoutStackFixups.push_back(C);
5867       }
5868 
5869       // If we outline this, we need to modify the stack. Pretend we don't
5870       // outline this by saving all of its bytes.
5871       else
5872         NumBytesNoStackCalls += SequenceSize;
5873     }
5874 
5875     // If there are no places where we have to save LR, then note that we don't
5876     // have to update the stack. Otherwise, give every candidate the default
5877     // call type
5878     if (NumBytesNoStackCalls <=
5879         RepeatedSequenceLocs.size() * Costs.CallDefault) {
5880       RepeatedSequenceLocs = CandidatesWithoutStackFixups;
5881       FrameID = MachineOutlinerNoLRSave;
5882     } else
5883       SetCandidateCallInfo(MachineOutlinerDefault, Costs.CallDefault);
5884   }
5885 
5886   // Does every candidate's MBB contain a call?  If so, then we might have a
5887   // call in the range.
5888   if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
5889     // check if the range contains a call.  These require a save + restore of
5890     // the link register.
5891     if (std::any_of(FirstCand.front(), FirstCand.back(),
5892                     [](const MachineInstr &MI) { return MI.isCall(); }))
5893       NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
5894 
5895     // Handle the last instruction separately.  If it is tail call, then the
5896     // last instruction is a call, we don't want to save + restore in this
5897     // case.  However, it could be possible that the last instruction is a
5898     // call without it being valid to tail call this sequence.  We should
5899     // consider this as well.
5900     else if (FrameID != MachineOutlinerThunk &&
5901              FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall())
5902       NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
5903   }
5904 
5905   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
5906                                     NumBytesToCreateFrame, FrameID);
5907 }
5908 
checkAndUpdateStackOffset(MachineInstr * MI,int64_t Fixup,bool Updt) const5909 bool ARMBaseInstrInfo::checkAndUpdateStackOffset(MachineInstr *MI,
5910                                                  int64_t Fixup,
5911                                                  bool Updt) const {
5912   int SPIdx = MI->findRegisterUseOperandIdx(ARM::SP);
5913   unsigned AddrMode = (MI->getDesc().TSFlags & ARMII::AddrModeMask);
5914   if (SPIdx < 0)
5915     // No SP operand
5916     return true;
5917   else if (SPIdx != 1 && (AddrMode != ARMII::AddrModeT2_i8s4 || SPIdx != 2))
5918     // If SP is not the base register we can't do much
5919     return false;
5920 
5921   // Stack might be involved but addressing mode doesn't handle any offset.
5922   // Rq: AddrModeT1_[1|2|4] don't operate on SP
5923   if (AddrMode == ARMII::AddrMode1        // Arithmetic instructions
5924       || AddrMode == ARMII::AddrMode4     // Load/Store Multiple
5925       || AddrMode == ARMII::AddrMode6     // Neon Load/Store Multiple
5926       || AddrMode == ARMII::AddrModeT2_so // SP can't be used as based register
5927       || AddrMode == ARMII::AddrModeT2_pc // PCrel access
5928       || AddrMode == ARMII::AddrMode2     // Used by PRE and POST indexed LD/ST
5929       || AddrMode == ARMII::AddrModeT2_i7 // v8.1-M MVE
5930       || AddrMode == ARMII::AddrModeT2_i7s2 // v8.1-M MVE
5931       || AddrMode == ARMII::AddrModeT2_i7s4 // v8.1-M sys regs VLDR/VSTR
5932       || AddrMode == ARMII::AddrModeNone)
5933     return false;
5934 
5935   unsigned NumOps = MI->getDesc().getNumOperands();
5936   unsigned ImmIdx = NumOps - 3;
5937 
5938   const MachineOperand &Offset = MI->getOperand(ImmIdx);
5939   assert(Offset.isImm() && "Is not an immediate");
5940   int64_t OffVal = Offset.getImm();
5941 
5942   if (OffVal < 0)
5943     // Don't override data if the are below SP.
5944     return false;
5945 
5946   unsigned NumBits = 0;
5947   unsigned Scale = 1;
5948 
5949   switch (AddrMode) {
5950   case ARMII::AddrMode3:
5951     if (ARM_AM::getAM3Op(OffVal) == ARM_AM::sub)
5952       return false;
5953     OffVal = ARM_AM::getAM3Offset(OffVal);
5954     NumBits = 8;
5955     break;
5956   case ARMII::AddrMode5:
5957     if (ARM_AM::getAM5Op(OffVal) == ARM_AM::sub)
5958       return false;
5959     OffVal = ARM_AM::getAM5Offset(OffVal);
5960     NumBits = 8;
5961     Scale = 4;
5962     break;
5963   case ARMII::AddrMode5FP16:
5964     if (ARM_AM::getAM5FP16Op(OffVal) == ARM_AM::sub)
5965       return false;
5966     OffVal = ARM_AM::getAM5FP16Offset(OffVal);
5967     NumBits = 8;
5968     Scale = 2;
5969     break;
5970   case ARMII::AddrModeT2_i8:
5971     NumBits = 8;
5972     break;
5973   case ARMII::AddrModeT2_i8s4:
5974     // FIXME: Values are already scaled in this addressing mode.
5975     assert((Fixup & 3) == 0 && "Can't encode this offset!");
5976     NumBits = 10;
5977     break;
5978   case ARMII::AddrModeT2_ldrex:
5979     NumBits = 8;
5980     Scale = 4;
5981     break;
5982   case ARMII::AddrModeT2_i12:
5983   case ARMII::AddrMode_i12:
5984     NumBits = 12;
5985     break;
5986   case ARMII::AddrModeT1_s: // SP-relative LD/ST
5987     NumBits = 8;
5988     Scale = 4;
5989     break;
5990   default:
5991     llvm_unreachable("Unsupported addressing mode!");
5992   }
5993   // Make sure the offset is encodable for instructions that scale the
5994   // immediate.
5995   assert(((OffVal * Scale + Fixup) & (Scale - 1)) == 0 &&
5996          "Can't encode this offset!");
5997   OffVal += Fixup / Scale;
5998 
5999   unsigned Mask = (1 << NumBits) - 1;
6000 
6001   if (OffVal <= Mask) {
6002     if (Updt)
6003       MI->getOperand(ImmIdx).setImm(OffVal);
6004     return true;
6005   }
6006 
6007   return false;
6008 
6009 }
6010 
isFunctionSafeToOutlineFrom(MachineFunction & MF,bool OutlineFromLinkOnceODRs) const6011 bool ARMBaseInstrInfo::isFunctionSafeToOutlineFrom(
6012     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
6013   const Function &F = MF.getFunction();
6014 
6015   // Can F be deduplicated by the linker? If it can, don't outline from it.
6016   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
6017     return false;
6018 
6019   // Don't outline from functions with section markings; the program could
6020   // expect that all the code is in the named section.
6021   // FIXME: Allow outlining from multiple functions with the same section
6022   // marking.
6023   if (F.hasSection())
6024     return false;
6025 
6026   // FIXME: Thumb1 outlining is not handled
6027   if (MF.getInfo<ARMFunctionInfo>()->isThumb1OnlyFunction())
6028     return false;
6029 
6030   // It's safe to outline from MF.
6031   return true;
6032 }
6033 
isMBBSafeToOutlineFrom(MachineBasicBlock & MBB,unsigned & Flags) const6034 bool ARMBaseInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
6035                                               unsigned &Flags) const {
6036   // Check if LR is available through all of the MBB. If it's not, then set
6037   // a flag.
6038   assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
6039          "Suitable Machine Function for outlining must track liveness");
6040 
6041   LiveRegUnits LRU(getRegisterInfo());
6042 
6043   std::for_each(MBB.rbegin(), MBB.rend(),
6044                 [&LRU](MachineInstr &MI) { LRU.accumulate(MI); });
6045 
6046   // Check if each of the unsafe registers are available...
6047   bool R12AvailableInBlock = LRU.available(ARM::R12);
6048   bool CPSRAvailableInBlock = LRU.available(ARM::CPSR);
6049 
6050   // If all of these are dead (and not live out), we know we don't have to check
6051   // them later.
6052   if (R12AvailableInBlock && CPSRAvailableInBlock)
6053     Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
6054 
6055   // Now, add the live outs to the set.
6056   LRU.addLiveOuts(MBB);
6057 
6058   // If any of these registers is available in the MBB, but also a live out of
6059   // the block, then we know outlining is unsafe.
6060   if (R12AvailableInBlock && !LRU.available(ARM::R12))
6061     return false;
6062   if (CPSRAvailableInBlock && !LRU.available(ARM::CPSR))
6063     return false;
6064 
6065   // Check if there's a call inside this MachineBasicBlock.  If there is, then
6066   // set a flag.
6067   if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); }))
6068     Flags |= MachineOutlinerMBBFlags::HasCalls;
6069 
6070   // LR liveness is overestimated in return blocks.
6071 
6072   bool LRIsAvailable =
6073       MBB.isReturnBlock() && !MBB.back().isCall()
6074           ? isLRAvailable(getRegisterInfo(), MBB.rbegin(), MBB.rend())
6075           : LRU.available(ARM::LR);
6076   if (!LRIsAvailable)
6077     Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
6078 
6079   return true;
6080 }
6081 
6082 outliner::InstrType
getOutliningType(MachineBasicBlock::iterator & MIT,unsigned Flags) const6083 ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
6084                                    unsigned Flags) const {
6085   MachineInstr &MI = *MIT;
6086   const TargetRegisterInfo *TRI = &getRegisterInfo();
6087 
6088   // Be conservative with inline ASM
6089   if (MI.isInlineAsm())
6090     return outliner::InstrType::Illegal;
6091 
6092   // Don't allow debug values to impact outlining type.
6093   if (MI.isDebugInstr() || MI.isIndirectDebugValue())
6094     return outliner::InstrType::Invisible;
6095 
6096   // At this point, KILL or IMPLICIT_DEF instructions don't really tell us much
6097   // so we can go ahead and skip over them.
6098   if (MI.isKill() || MI.isImplicitDef())
6099     return outliner::InstrType::Invisible;
6100 
6101   // PIC instructions contain labels, outlining them would break offset
6102   // computing.  unsigned Opc = MI.getOpcode();
6103   unsigned Opc = MI.getOpcode();
6104   if (Opc == ARM::tPICADD || Opc == ARM::PICADD || Opc == ARM::PICSTR ||
6105       Opc == ARM::PICSTRB || Opc == ARM::PICSTRH || Opc == ARM::PICLDR ||
6106       Opc == ARM::PICLDRB || Opc == ARM::PICLDRH || Opc == ARM::PICLDRSB ||
6107       Opc == ARM::PICLDRSH || Opc == ARM::t2LDRpci_pic ||
6108       Opc == ARM::t2MOVi16_ga_pcrel || Opc == ARM::t2MOVTi16_ga_pcrel ||
6109       Opc == ARM::t2MOV_ga_pcrel)
6110     return outliner::InstrType::Illegal;
6111 
6112   // Be conservative with ARMv8.1 MVE instructions.
6113   if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart ||
6114       Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart ||
6115       Opc == ARM::t2WhileLoopStartLR || Opc == ARM::t2WhileLoopStartTP ||
6116       Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd ||
6117       Opc == ARM::t2LoopEndDec)
6118     return outliner::InstrType::Illegal;
6119 
6120   const MCInstrDesc &MCID = MI.getDesc();
6121   uint64_t MIFlags = MCID.TSFlags;
6122   if ((MIFlags & ARMII::DomainMask) == ARMII::DomainMVE)
6123     return outliner::InstrType::Illegal;
6124 
6125   // Is this a terminator for a basic block?
6126   if (MI.isTerminator()) {
6127     // Don't outline if the branch is not unconditional.
6128     if (isPredicated(MI))
6129       return outliner::InstrType::Illegal;
6130 
6131     // Is this the end of a function?
6132     if (MI.getParent()->succ_empty())
6133       return outliner::InstrType::Legal;
6134 
6135     // It's not, so don't outline it.
6136     return outliner::InstrType::Illegal;
6137   }
6138 
6139   // Make sure none of the operands are un-outlinable.
6140   for (const MachineOperand &MOP : MI.operands()) {
6141     if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() ||
6142         MOP.isTargetIndex())
6143       return outliner::InstrType::Illegal;
6144   }
6145 
6146   // Don't outline if link register or program counter value are used.
6147   if (MI.readsRegister(ARM::LR, TRI) || MI.readsRegister(ARM::PC, TRI))
6148     return outliner::InstrType::Illegal;
6149 
6150   if (MI.isCall()) {
6151     // Get the function associated with the call.  Look at each operand and find
6152     // the one that represents the calle and get its name.
6153     const Function *Callee = nullptr;
6154     for (const MachineOperand &MOP : MI.operands()) {
6155       if (MOP.isGlobal()) {
6156         Callee = dyn_cast<Function>(MOP.getGlobal());
6157         break;
6158       }
6159     }
6160 
6161     // Dont't outline calls to "mcount" like functions, in particular Linux
6162     // kernel function tracing relies on it.
6163     if (Callee &&
6164         (Callee->getName() == "\01__gnu_mcount_nc" ||
6165          Callee->getName() == "\01mcount" || Callee->getName() == "__mcount"))
6166       return outliner::InstrType::Illegal;
6167 
6168     // If we don't know anything about the callee, assume it depends on the
6169     // stack layout of the caller. In that case, it's only legal to outline
6170     // as a tail-call. Explicitly list the call instructions we know about so
6171     // we don't get unexpected results with call pseudo-instructions.
6172     auto UnknownCallOutlineType = outliner::InstrType::Illegal;
6173     if (Opc == ARM::BL || Opc == ARM::tBL || Opc == ARM::BLX ||
6174         Opc == ARM::BLX_noip || Opc == ARM::tBLXr || Opc == ARM::tBLXr_noip ||
6175         Opc == ARM::tBLXi)
6176       UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
6177 
6178     if (!Callee)
6179       return UnknownCallOutlineType;
6180 
6181     // We have a function we have information about.  Check if it's something we
6182     // can safely outline.
6183     MachineFunction *MF = MI.getParent()->getParent();
6184     MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee);
6185 
6186     // We don't know what's going on with the callee at all.  Don't touch it.
6187     if (!CalleeMF)
6188       return UnknownCallOutlineType;
6189 
6190     // Check if we know anything about the callee saves on the function. If we
6191     // don't, then don't touch it, since that implies that we haven't computed
6192     // anything about its stack frame yet.
6193     MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
6194     if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
6195         MFI.getNumObjects() > 0)
6196       return UnknownCallOutlineType;
6197 
6198     // At this point, we can say that CalleeMF ought to not pass anything on the
6199     // stack. Therefore, we can outline it.
6200     return outliner::InstrType::Legal;
6201   }
6202 
6203   // Since calls are handled, don't touch LR or PC
6204   if (MI.modifiesRegister(ARM::LR, TRI) || MI.modifiesRegister(ARM::PC, TRI))
6205     return outliner::InstrType::Illegal;
6206 
6207   // Does this use the stack?
6208   if (MI.modifiesRegister(ARM::SP, TRI) || MI.readsRegister(ARM::SP, TRI)) {
6209     // True if there is no chance that any outlined candidate from this range
6210     // could require stack fixups. That is, both
6211     // * LR is available in the range (No save/restore around call)
6212     // * The range doesn't include calls (No save/restore in outlined frame)
6213     // are true.
6214     // FIXME: This is very restrictive; the flags check the whole block,
6215     // not just the bit we will try to outline.
6216     bool MightNeedStackFixUp =
6217         (Flags & (MachineOutlinerMBBFlags::LRUnavailableSomewhere |
6218                   MachineOutlinerMBBFlags::HasCalls));
6219 
6220     if (!MightNeedStackFixUp)
6221       return outliner::InstrType::Legal;
6222 
6223     // Any modification of SP will break our code to save/restore LR.
6224     // FIXME: We could handle some instructions which add a constant offset to
6225     // SP, with a bit more work.
6226     if (MI.modifiesRegister(ARM::SP, TRI))
6227       return outliner::InstrType::Illegal;
6228 
6229     // At this point, we have a stack instruction that we might need to fix up.
6230     // up. We'll handle it if it's a load or store.
6231     if (checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(),
6232                                   false))
6233       return outliner::InstrType::Legal;
6234 
6235     // We can't fix it up, so don't outline it.
6236     return outliner::InstrType::Illegal;
6237   }
6238 
6239   // Be conservative with IT blocks.
6240   if (MI.readsRegister(ARM::ITSTATE, TRI) ||
6241       MI.modifiesRegister(ARM::ITSTATE, TRI))
6242     return outliner::InstrType::Illegal;
6243 
6244   // Don't outline positions.
6245   if (MI.isPosition())
6246     return outliner::InstrType::Illegal;
6247 
6248   return outliner::InstrType::Legal;
6249 }
6250 
fixupPostOutline(MachineBasicBlock & MBB) const6251 void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6252   for (MachineInstr &MI : MBB) {
6253     checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(), true);
6254   }
6255 }
6256 
saveLROnStack(MachineBasicBlock & MBB,MachineBasicBlock::iterator It) const6257 void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB,
6258                                      MachineBasicBlock::iterator It) const {
6259   unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM;
6260   int Align = -Subtarget.getStackAlignment().value();
6261   BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP)
6262     .addReg(ARM::LR, RegState::Kill)
6263     .addReg(ARM::SP)
6264     .addImm(Align)
6265     .add(predOps(ARMCC::AL));
6266 }
6267 
emitCFIForLRSaveOnStack(MachineBasicBlock & MBB,MachineBasicBlock::iterator It) const6268 void ARMBaseInstrInfo::emitCFIForLRSaveOnStack(
6269     MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
6270   MachineFunction &MF = *MBB.getParent();
6271   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6272   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6273   int Align = Subtarget.getStackAlignment().value();
6274   // Add a CFI saying the stack was moved down.
6275   int64_t StackPosEntry =
6276       MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align));
6277   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6278       .addCFIIndex(StackPosEntry)
6279       .setMIFlags(MachineInstr::FrameSetup);
6280 
6281   // Add a CFI saying that the LR that we want to find is now higher than
6282   // before.
6283   int64_t LRPosEntry =
6284       MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfLR, -Align));
6285   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6286       .addCFIIndex(LRPosEntry)
6287       .setMIFlags(MachineInstr::FrameSetup);
6288 }
6289 
emitCFIForLRSaveToReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator It,Register Reg) const6290 void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB,
6291                                              MachineBasicBlock::iterator It,
6292                                              Register Reg) const {
6293   MachineFunction &MF = *MBB.getParent();
6294   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6295   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6296   unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
6297 
6298   int64_t LRPosEntry = MF.addFrameInst(
6299       MCCFIInstruction::createRegister(nullptr, DwarfLR, DwarfReg));
6300   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6301       .addCFIIndex(LRPosEntry)
6302       .setMIFlags(MachineInstr::FrameSetup);
6303 }
6304 
restoreLRFromStack(MachineBasicBlock & MBB,MachineBasicBlock::iterator It) const6305 void ARMBaseInstrInfo::restoreLRFromStack(
6306     MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
6307   unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM;
6308   MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR)
6309     .addReg(ARM::SP, RegState::Define)
6310     .addReg(ARM::SP);
6311   if (!Subtarget.isThumb())
6312     MIB.addReg(0);
6313   MIB.addImm(Subtarget.getStackAlignment().value()).add(predOps(ARMCC::AL));
6314 }
6315 
emitCFIForLRRestoreFromStack(MachineBasicBlock & MBB,MachineBasicBlock::iterator It) const6316 void ARMBaseInstrInfo::emitCFIForLRRestoreFromStack(
6317     MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
6318   // Now stack has moved back up...
6319   MachineFunction &MF = *MBB.getParent();
6320   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6321   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6322   int64_t StackPosEntry =
6323       MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0));
6324   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6325       .addCFIIndex(StackPosEntry)
6326       .setMIFlags(MachineInstr::FrameDestroy);
6327 
6328   // ... and we have restored LR.
6329   int64_t LRPosEntry =
6330       MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
6331   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6332       .addCFIIndex(LRPosEntry)
6333       .setMIFlags(MachineInstr::FrameDestroy);
6334 }
6335 
emitCFIForLRRestoreFromReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator It) const6336 void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg(
6337     MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
6338   MachineFunction &MF = *MBB.getParent();
6339   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6340   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6341 
6342   int64_t LRPosEntry =
6343       MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
6344   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6345       .addCFIIndex(LRPosEntry)
6346       .setMIFlags(MachineInstr::FrameDestroy);
6347 }
6348 
buildOutlinedFrame(MachineBasicBlock & MBB,MachineFunction & MF,const outliner::OutlinedFunction & OF) const6349 void ARMBaseInstrInfo::buildOutlinedFrame(
6350     MachineBasicBlock &MBB, MachineFunction &MF,
6351     const outliner::OutlinedFunction &OF) const {
6352   // For thunk outlining, rewrite the last instruction from a call to a
6353   // tail-call.
6354   if (OF.FrameConstructionID == MachineOutlinerThunk) {
6355     MachineInstr *Call = &*--MBB.instr_end();
6356     bool isThumb = Subtarget.isThumb();
6357     unsigned FuncOp = isThumb ? 2 : 0;
6358     unsigned Opc = Call->getOperand(FuncOp).isReg()
6359                        ? isThumb ? ARM::tTAILJMPr : ARM::TAILJMPr
6360                        : isThumb ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd
6361                                                              : ARM::tTAILJMPdND
6362                                  : ARM::TAILJMPd;
6363     MachineInstrBuilder MIB = BuildMI(MBB, MBB.end(), DebugLoc(), get(Opc))
6364                                   .add(Call->getOperand(FuncOp));
6365     if (isThumb && !Call->getOperand(FuncOp).isReg())
6366       MIB.add(predOps(ARMCC::AL));
6367     Call->eraseFromParent();
6368   }
6369 
6370   // Is there a call in the outlined range?
6371   auto IsNonTailCall = [](MachineInstr &MI) {
6372     return MI.isCall() && !MI.isReturn();
6373   };
6374   if (llvm::any_of(MBB.instrs(), IsNonTailCall)) {
6375     MachineBasicBlock::iterator It = MBB.begin();
6376     MachineBasicBlock::iterator Et = MBB.end();
6377 
6378     if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6379         OF.FrameConstructionID == MachineOutlinerThunk)
6380       Et = std::prev(MBB.end());
6381 
6382     // We have to save and restore LR, we need to add it to the liveins if it
6383     // is not already part of the set.  This is suffient since outlined
6384     // functions only have one block.
6385     if (!MBB.isLiveIn(ARM::LR))
6386       MBB.addLiveIn(ARM::LR);
6387 
6388     // Insert a save before the outlined region
6389     saveLROnStack(MBB, It);
6390     emitCFIForLRSaveOnStack(MBB, It);
6391 
6392     // Fix up the instructions in the range, since we're going to modify the
6393     // stack.
6394     assert(OF.FrameConstructionID != MachineOutlinerDefault &&
6395            "Can only fix up stack references once");
6396     fixupPostOutline(MBB);
6397 
6398     // Insert a restore before the terminator for the function.  Restore LR.
6399     restoreLRFromStack(MBB, Et);
6400     emitCFIForLRRestoreFromStack(MBB, Et);
6401   }
6402 
6403   // If this is a tail call outlined function, then there's already a return.
6404   if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6405       OF.FrameConstructionID == MachineOutlinerThunk)
6406     return;
6407 
6408   // Here we have to insert the return ourselves.  Get the correct opcode from
6409   // current feature set.
6410   BuildMI(MBB, MBB.end(), DebugLoc(), get(Subtarget.getReturnOpcode()))
6411       .add(predOps(ARMCC::AL));
6412 
6413   // Did we have to modify the stack by saving the link register?
6414   if (OF.FrameConstructionID != MachineOutlinerDefault &&
6415       OF.Candidates[0].CallConstructionID != MachineOutlinerDefault)
6416     return;
6417 
6418   // We modified the stack.
6419   // Walk over the basic block and fix up all the stack accesses.
6420   fixupPostOutline(MBB);
6421 }
6422 
insertOutlinedCall(Module & M,MachineBasicBlock & MBB,MachineBasicBlock::iterator & It,MachineFunction & MF,const outliner::Candidate & C) const6423 MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall(
6424     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
6425     MachineFunction &MF, const outliner::Candidate &C) const {
6426   MachineInstrBuilder MIB;
6427   MachineBasicBlock::iterator CallPt;
6428   unsigned Opc;
6429   bool isThumb = Subtarget.isThumb();
6430 
6431   // Are we tail calling?
6432   if (C.CallConstructionID == MachineOutlinerTailCall) {
6433     // If yes, then we can just branch to the label.
6434     Opc = isThumb
6435               ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND
6436               : ARM::TAILJMPd;
6437     MIB = BuildMI(MF, DebugLoc(), get(Opc))
6438               .addGlobalAddress(M.getNamedValue(MF.getName()));
6439     if (isThumb)
6440       MIB.add(predOps(ARMCC::AL));
6441     It = MBB.insert(It, MIB);
6442     return It;
6443   }
6444 
6445   // Create the call instruction.
6446   Opc = isThumb ? ARM::tBL : ARM::BL;
6447   MachineInstrBuilder CallMIB = BuildMI(MF, DebugLoc(), get(Opc));
6448   if (isThumb)
6449     CallMIB.add(predOps(ARMCC::AL));
6450   CallMIB.addGlobalAddress(M.getNamedValue(MF.getName()));
6451 
6452   if (C.CallConstructionID == MachineOutlinerNoLRSave ||
6453       C.CallConstructionID == MachineOutlinerThunk) {
6454     // No, so just insert the call.
6455     It = MBB.insert(It, CallMIB);
6456     return It;
6457   }
6458 
6459   const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
6460   // Can we save to a register?
6461   if (C.CallConstructionID == MachineOutlinerRegSave) {
6462     unsigned Reg = findRegisterToSaveLRTo(C);
6463     assert(Reg != 0 && "No callee-saved register available?");
6464 
6465     // Save and restore LR from that register.
6466     copyPhysReg(MBB, It, DebugLoc(), Reg, ARM::LR, true);
6467     if (!AFI.isLRSpilled())
6468       emitCFIForLRSaveToReg(MBB, It, Reg);
6469     CallPt = MBB.insert(It, CallMIB);
6470     copyPhysReg(MBB, It, DebugLoc(), ARM::LR, Reg, true);
6471     if (!AFI.isLRSpilled())
6472       emitCFIForLRRestoreFromReg(MBB, It);
6473     It--;
6474     return CallPt;
6475   }
6476   // We have the default case. Save and restore from SP.
6477   if (!MBB.isLiveIn(ARM::LR))
6478     MBB.addLiveIn(ARM::LR);
6479   saveLROnStack(MBB, It);
6480   if (!AFI.isLRSpilled())
6481     emitCFIForLRSaveOnStack(MBB, It);
6482   CallPt = MBB.insert(It, CallMIB);
6483   restoreLRFromStack(MBB, It);
6484   if (!AFI.isLRSpilled())
6485     emitCFIForLRRestoreFromStack(MBB, It);
6486   It--;
6487   return CallPt;
6488 }
6489 
shouldOutlineFromFunctionByDefault(MachineFunction & MF) const6490 bool ARMBaseInstrInfo::shouldOutlineFromFunctionByDefault(
6491     MachineFunction &MF) const {
6492   return Subtarget.isMClass() && MF.getFunction().hasMinSize();
6493 }
6494 
isReallyTriviallyReMaterializable(const MachineInstr & MI,AAResults * AA) const6495 bool ARMBaseInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
6496                                                          AAResults *AA) const {
6497   // Try hard to rematerialize any VCTPs because if we spill P0, it will block
6498   // the tail predication conversion. This means that the element count
6499   // register has to be live for longer, but that has to be better than
6500   // spill/restore and VPT predication.
6501   return isVCTP(&MI) && !isPredicated(MI);
6502 }
6503 
getBLXOpcode(const MachineFunction & MF)6504 unsigned llvm::getBLXOpcode(const MachineFunction &MF) {
6505   return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_noip
6506                                                           : ARM::BLX;
6507 }
6508 
gettBLXrOpcode(const MachineFunction & MF)6509 unsigned llvm::gettBLXrOpcode(const MachineFunction &MF) {
6510   return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::tBLXr_noip
6511                                                           : ARM::tBLXr;
6512 }
6513 
getBLXpredOpcode(const MachineFunction & MF)6514 unsigned llvm::getBLXpredOpcode(const MachineFunction &MF) {
6515   return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_pred_noip
6516                                                           : ARM::BLX_pred;
6517 }
6518 
6519