1 //===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the Base ARM implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "ARMBaseInstrInfo.h"
14 #include "ARMBaseRegisterInfo.h"
15 #include "ARMConstantPoolValue.h"
16 #include "ARMFeatures.h"
17 #include "ARMHazardRecognizer.h"
18 #include "ARMMachineFunctionInfo.h"
19 #include "ARMSubtarget.h"
20 #include "MCTargetDesc/ARMAddressingModes.h"
21 #include "MCTargetDesc/ARMBaseInfo.h"
22 #include "MVETailPredUtils.h"
23 #include "llvm/ADT/DenseMap.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/SmallSet.h"
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/Triple.h"
28 #include "llvm/CodeGen/LiveVariables.h"
29 #include "llvm/CodeGen/MachineBasicBlock.h"
30 #include "llvm/CodeGen/MachineConstantPool.h"
31 #include "llvm/CodeGen/MachineFrameInfo.h"
32 #include "llvm/CodeGen/MachineFunction.h"
33 #include "llvm/CodeGen/MachineInstr.h"
34 #include "llvm/CodeGen/MachineInstrBuilder.h"
35 #include "llvm/CodeGen/MachineMemOperand.h"
36 #include "llvm/CodeGen/MachineModuleInfo.h"
37 #include "llvm/CodeGen/MachineOperand.h"
38 #include "llvm/CodeGen/MachineRegisterInfo.h"
39 #include "llvm/CodeGen/MachineScheduler.h"
40 #include "llvm/CodeGen/MultiHazardRecognizer.h"
41 #include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
42 #include "llvm/CodeGen/SelectionDAGNodes.h"
43 #include "llvm/CodeGen/TargetInstrInfo.h"
44 #include "llvm/CodeGen/TargetRegisterInfo.h"
45 #include "llvm/CodeGen/TargetSchedule.h"
46 #include "llvm/IR/Attributes.h"
47 #include "llvm/IR/Constants.h"
48 #include "llvm/IR/DebugLoc.h"
49 #include "llvm/IR/Function.h"
50 #include "llvm/IR/GlobalValue.h"
51 #include "llvm/MC/MCAsmInfo.h"
52 #include "llvm/MC/MCInstrDesc.h"
53 #include "llvm/MC/MCInstrItineraries.h"
54 #include "llvm/Support/BranchProbability.h"
55 #include "llvm/Support/Casting.h"
56 #include "llvm/Support/CommandLine.h"
57 #include "llvm/Support/Compiler.h"
58 #include "llvm/Support/Debug.h"
59 #include "llvm/Support/ErrorHandling.h"
60 #include "llvm/Support/raw_ostream.h"
61 #include "llvm/Target/TargetMachine.h"
62 #include <algorithm>
63 #include <cassert>
64 #include <cstdint>
65 #include <iterator>
66 #include <new>
67 #include <utility>
68 #include <vector>
69 
70 using namespace llvm;
71 
72 #define DEBUG_TYPE "arm-instrinfo"
73 
74 #define GET_INSTRINFO_CTOR_DTOR
75 #include "ARMGenInstrInfo.inc"
76 
77 static cl::opt<bool>
78 EnableARM3Addr("enable-arm-3-addr-conv", cl::Hidden,
79                cl::desc("Enable ARM 2-addr to 3-addr conv"));
80 
81 /// ARM_MLxEntry - Record information about MLA / MLS instructions.
82 struct ARM_MLxEntry {
83   uint16_t MLxOpc;     // MLA / MLS opcode
84   uint16_t MulOpc;     // Expanded multiplication opcode
85   uint16_t AddSubOpc;  // Expanded add / sub opcode
86   bool NegAcc;         // True if the acc is negated before the add / sub.
87   bool HasLane;        // True if instruction has an extra "lane" operand.
88 };
89 
90 static const ARM_MLxEntry ARM_MLxTable[] = {
91   // MLxOpc,          MulOpc,           AddSubOpc,       NegAcc, HasLane
92   // fp scalar ops
93   { ARM::VMLAS,       ARM::VMULS,       ARM::VADDS,      false,  false },
94   { ARM::VMLSS,       ARM::VMULS,       ARM::VSUBS,      false,  false },
95   { ARM::VMLAD,       ARM::VMULD,       ARM::VADDD,      false,  false },
96   { ARM::VMLSD,       ARM::VMULD,       ARM::VSUBD,      false,  false },
97   { ARM::VNMLAS,      ARM::VNMULS,      ARM::VSUBS,      true,   false },
98   { ARM::VNMLSS,      ARM::VMULS,       ARM::VSUBS,      true,   false },
99   { ARM::VNMLAD,      ARM::VNMULD,      ARM::VSUBD,      true,   false },
100   { ARM::VNMLSD,      ARM::VMULD,       ARM::VSUBD,      true,   false },
101 
102   // fp SIMD ops
103   { ARM::VMLAfd,      ARM::VMULfd,      ARM::VADDfd,     false,  false },
104   { ARM::VMLSfd,      ARM::VMULfd,      ARM::VSUBfd,     false,  false },
105   { ARM::VMLAfq,      ARM::VMULfq,      ARM::VADDfq,     false,  false },
106   { ARM::VMLSfq,      ARM::VMULfq,      ARM::VSUBfq,     false,  false },
107   { ARM::VMLAslfd,    ARM::VMULslfd,    ARM::VADDfd,     false,  true  },
108   { ARM::VMLSslfd,    ARM::VMULslfd,    ARM::VSUBfd,     false,  true  },
109   { ARM::VMLAslfq,    ARM::VMULslfq,    ARM::VADDfq,     false,  true  },
110   { ARM::VMLSslfq,    ARM::VMULslfq,    ARM::VSUBfq,     false,  true  },
111 };
112 
113 ARMBaseInstrInfo::ARMBaseInstrInfo(const ARMSubtarget& STI)
114   : ARMGenInstrInfo(ARM::ADJCALLSTACKDOWN, ARM::ADJCALLSTACKUP),
115     Subtarget(STI) {
116   for (unsigned i = 0, e = array_lengthof(ARM_MLxTable); i != e; ++i) {
117     if (!MLxEntryMap.insert(std::make_pair(ARM_MLxTable[i].MLxOpc, i)).second)
118       llvm_unreachable("Duplicated entries?");
119     MLxHazardOpcodes.insert(ARM_MLxTable[i].AddSubOpc);
120     MLxHazardOpcodes.insert(ARM_MLxTable[i].MulOpc);
121   }
122 }
123 
124 // Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl
125 // currently defaults to no prepass hazard recognizer.
126 ScheduleHazardRecognizer *
127 ARMBaseInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
128                                                const ScheduleDAG *DAG) const {
129   if (usePreRAHazardRecognizer()) {
130     const InstrItineraryData *II =
131         static_cast<const ARMSubtarget *>(STI)->getInstrItineraryData();
132     return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched");
133   }
134   return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
135 }
136 
137 // Called during:
138 // - pre-RA scheduling
139 // - post-RA scheduling when FeatureUseMISched is set
140 ScheduleHazardRecognizer *ARMBaseInstrInfo::CreateTargetMIHazardRecognizer(
141     const InstrItineraryData *II, const ScheduleDAGMI *DAG) const {
142   MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
143 
144   // We would like to restrict this hazard recognizer to only
145   // post-RA scheduling; we can tell that we're post-RA because we don't
146   // track VRegLiveness.
147   // Cortex-M7: TRM indicates that there is a single ITCM bank and two DTCM
148   //            banks banked on bit 2.  Assume that TCMs are in use.
149   if (Subtarget.isCortexM7() && !DAG->hasVRegLiveness())
150     MHR->AddHazardRecognizer(
151         std::make_unique<ARMBankConflictHazardRecognizer>(DAG, 0x4, true));
152 
153   // Not inserting ARMHazardRecognizerFPMLx because that would change
154   // legacy behavior
155 
156   auto BHR = TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
157   MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
158   return MHR;
159 }
160 
161 // Called during post-RA scheduling when FeatureUseMISched is not set
162 ScheduleHazardRecognizer *ARMBaseInstrInfo::
163 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
164                                    const ScheduleDAG *DAG) const {
165   MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
166 
167   if (Subtarget.isThumb2() || Subtarget.hasVFP2Base())
168     MHR->AddHazardRecognizer(std::make_unique<ARMHazardRecognizerFPMLx>());
169 
170   auto BHR = TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG);
171   if (BHR)
172     MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
173   return MHR;
174 }
175 
176 MachineInstr *
177 ARMBaseInstrInfo::convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
178                                         LiveIntervals *LIS) const {
179   // FIXME: Thumb2 support.
180 
181   if (!EnableARM3Addr)
182     return nullptr;
183 
184   MachineFunction &MF = *MI.getParent()->getParent();
185   uint64_t TSFlags = MI.getDesc().TSFlags;
186   bool isPre = false;
187   switch ((TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift) {
188   default: return nullptr;
189   case ARMII::IndexModePre:
190     isPre = true;
191     break;
192   case ARMII::IndexModePost:
193     break;
194   }
195 
196   // Try splitting an indexed load/store to an un-indexed one plus an add/sub
197   // operation.
198   unsigned MemOpc = getUnindexedOpcode(MI.getOpcode());
199   if (MemOpc == 0)
200     return nullptr;
201 
202   MachineInstr *UpdateMI = nullptr;
203   MachineInstr *MemMI = nullptr;
204   unsigned AddrMode = (TSFlags & ARMII::AddrModeMask);
205   const MCInstrDesc &MCID = MI.getDesc();
206   unsigned NumOps = MCID.getNumOperands();
207   bool isLoad = !MI.mayStore();
208   const MachineOperand &WB = isLoad ? MI.getOperand(1) : MI.getOperand(0);
209   const MachineOperand &Base = MI.getOperand(2);
210   const MachineOperand &Offset = MI.getOperand(NumOps - 3);
211   Register WBReg = WB.getReg();
212   Register BaseReg = Base.getReg();
213   Register OffReg = Offset.getReg();
214   unsigned OffImm = MI.getOperand(NumOps - 2).getImm();
215   ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI.getOperand(NumOps - 1).getImm();
216   switch (AddrMode) {
217   default: llvm_unreachable("Unknown indexed op!");
218   case ARMII::AddrMode2: {
219     bool isSub = ARM_AM::getAM2Op(OffImm) == ARM_AM::sub;
220     unsigned Amt = ARM_AM::getAM2Offset(OffImm);
221     if (OffReg == 0) {
222       if (ARM_AM::getSOImmVal(Amt) == -1)
223         // Can't encode it in a so_imm operand. This transformation will
224         // add more than 1 instruction. Abandon!
225         return nullptr;
226       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
227                          get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
228                      .addReg(BaseReg)
229                      .addImm(Amt)
230                      .add(predOps(Pred))
231                      .add(condCodeOp());
232     } else if (Amt != 0) {
233       ARM_AM::ShiftOpc ShOpc = ARM_AM::getAM2ShiftOpc(OffImm);
234       unsigned SOOpc = ARM_AM::getSORegOpc(ShOpc, Amt);
235       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
236                          get(isSub ? ARM::SUBrsi : ARM::ADDrsi), WBReg)
237                      .addReg(BaseReg)
238                      .addReg(OffReg)
239                      .addReg(0)
240                      .addImm(SOOpc)
241                      .add(predOps(Pred))
242                      .add(condCodeOp());
243     } else
244       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
245                          get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
246                      .addReg(BaseReg)
247                      .addReg(OffReg)
248                      .add(predOps(Pred))
249                      .add(condCodeOp());
250     break;
251   }
252   case ARMII::AddrMode3 : {
253     bool isSub = ARM_AM::getAM3Op(OffImm) == ARM_AM::sub;
254     unsigned Amt = ARM_AM::getAM3Offset(OffImm);
255     if (OffReg == 0)
256       // Immediate is 8-bits. It's guaranteed to fit in a so_imm operand.
257       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
258                          get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
259                      .addReg(BaseReg)
260                      .addImm(Amt)
261                      .add(predOps(Pred))
262                      .add(condCodeOp());
263     else
264       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
265                          get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
266                      .addReg(BaseReg)
267                      .addReg(OffReg)
268                      .add(predOps(Pred))
269                      .add(condCodeOp());
270     break;
271   }
272   }
273 
274   std::vector<MachineInstr*> NewMIs;
275   if (isPre) {
276     if (isLoad)
277       MemMI =
278           BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg())
279               .addReg(WBReg)
280               .addImm(0)
281               .addImm(Pred);
282     else
283       MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc))
284                   .addReg(MI.getOperand(1).getReg())
285                   .addReg(WBReg)
286                   .addReg(0)
287                   .addImm(0)
288                   .addImm(Pred);
289     NewMIs.push_back(MemMI);
290     NewMIs.push_back(UpdateMI);
291   } else {
292     if (isLoad)
293       MemMI =
294           BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg())
295               .addReg(BaseReg)
296               .addImm(0)
297               .addImm(Pred);
298     else
299       MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc))
300                   .addReg(MI.getOperand(1).getReg())
301                   .addReg(BaseReg)
302                   .addReg(0)
303                   .addImm(0)
304                   .addImm(Pred);
305     if (WB.isDead())
306       UpdateMI->getOperand(0).setIsDead();
307     NewMIs.push_back(UpdateMI);
308     NewMIs.push_back(MemMI);
309   }
310 
311   // Transfer LiveVariables states, kill / dead info.
312   if (LV) {
313     for (const MachineOperand &MO : MI.operands()) {
314       if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) {
315         Register Reg = MO.getReg();
316 
317         LiveVariables::VarInfo &VI = LV->getVarInfo(Reg);
318         if (MO.isDef()) {
319           MachineInstr *NewMI = (Reg == WBReg) ? UpdateMI : MemMI;
320           if (MO.isDead())
321             LV->addVirtualRegisterDead(Reg, *NewMI);
322         }
323         if (MO.isUse() && MO.isKill()) {
324           for (unsigned j = 0; j < 2; ++j) {
325             // Look at the two new MI's in reverse order.
326             MachineInstr *NewMI = NewMIs[j];
327             if (!NewMI->readsRegister(Reg))
328               continue;
329             LV->addVirtualRegisterKilled(Reg, *NewMI);
330             if (VI.removeKill(MI))
331               VI.Kills.push_back(NewMI);
332             break;
333           }
334         }
335       }
336     }
337   }
338 
339   MachineBasicBlock &MBB = *MI.getParent();
340   MBB.insert(MI, NewMIs[1]);
341   MBB.insert(MI, NewMIs[0]);
342   return NewMIs[0];
343 }
344 
345 // Branch analysis.
346 bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
347                                      MachineBasicBlock *&TBB,
348                                      MachineBasicBlock *&FBB,
349                                      SmallVectorImpl<MachineOperand> &Cond,
350                                      bool AllowModify) const {
351   TBB = nullptr;
352   FBB = nullptr;
353 
354   MachineBasicBlock::instr_iterator I = MBB.instr_end();
355   if (I == MBB.instr_begin())
356     return false; // Empty blocks are easy.
357   --I;
358 
359   // Walk backwards from the end of the basic block until the branch is
360   // analyzed or we give up.
361   while (isPredicated(*I) || I->isTerminator() || I->isDebugValue()) {
362     // Flag to be raised on unanalyzeable instructions. This is useful in cases
363     // where we want to clean up on the end of the basic block before we bail
364     // out.
365     bool CantAnalyze = false;
366 
367     // Skip over DEBUG values, predicated nonterminators and speculation
368     // barrier terminators.
369     while (I->isDebugInstr() || !I->isTerminator() ||
370            isSpeculationBarrierEndBBOpcode(I->getOpcode()) ||
371            I->getOpcode() == ARM::t2DoLoopStartTP){
372       if (I == MBB.instr_begin())
373         return false;
374       --I;
375     }
376 
377     if (isIndirectBranchOpcode(I->getOpcode()) ||
378         isJumpTableBranchOpcode(I->getOpcode())) {
379       // Indirect branches and jump tables can't be analyzed, but we still want
380       // to clean up any instructions at the tail of the basic block.
381       CantAnalyze = true;
382     } else if (isUncondBranchOpcode(I->getOpcode())) {
383       TBB = I->getOperand(0).getMBB();
384     } else if (isCondBranchOpcode(I->getOpcode())) {
385       // Bail out if we encounter multiple conditional branches.
386       if (!Cond.empty())
387         return true;
388 
389       assert(!FBB && "FBB should have been null.");
390       FBB = TBB;
391       TBB = I->getOperand(0).getMBB();
392       Cond.push_back(I->getOperand(1));
393       Cond.push_back(I->getOperand(2));
394     } else if (I->isReturn()) {
395       // Returns can't be analyzed, but we should run cleanup.
396       CantAnalyze = true;
397     } else {
398       // We encountered other unrecognized terminator. Bail out immediately.
399       return true;
400     }
401 
402     // Cleanup code - to be run for unpredicated unconditional branches and
403     //                returns.
404     if (!isPredicated(*I) &&
405           (isUncondBranchOpcode(I->getOpcode()) ||
406            isIndirectBranchOpcode(I->getOpcode()) ||
407            isJumpTableBranchOpcode(I->getOpcode()) ||
408            I->isReturn())) {
409       // Forget any previous condition branch information - it no longer applies.
410       Cond.clear();
411       FBB = nullptr;
412 
413       // If we can modify the function, delete everything below this
414       // unconditional branch.
415       if (AllowModify) {
416         MachineBasicBlock::iterator DI = std::next(I);
417         while (DI != MBB.instr_end()) {
418           MachineInstr &InstToDelete = *DI;
419           ++DI;
420           // Speculation barriers must not be deleted.
421           if (isSpeculationBarrierEndBBOpcode(InstToDelete.getOpcode()))
422             continue;
423           InstToDelete.eraseFromParent();
424         }
425       }
426     }
427 
428     if (CantAnalyze) {
429       // We may not be able to analyze the block, but we could still have
430       // an unconditional branch as the last instruction in the block, which
431       // just branches to layout successor. If this is the case, then just
432       // remove it if we're allowed to make modifications.
433       if (AllowModify && !isPredicated(MBB.back()) &&
434           isUncondBranchOpcode(MBB.back().getOpcode()) &&
435           TBB && MBB.isLayoutSuccessor(TBB))
436         removeBranch(MBB);
437       return true;
438     }
439 
440     if (I == MBB.instr_begin())
441       return false;
442 
443     --I;
444   }
445 
446   // We made it past the terminators without bailing out - we must have
447   // analyzed this branch successfully.
448   return false;
449 }
450 
451 unsigned ARMBaseInstrInfo::removeBranch(MachineBasicBlock &MBB,
452                                         int *BytesRemoved) const {
453   assert(!BytesRemoved && "code size not handled");
454 
455   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
456   if (I == MBB.end())
457     return 0;
458 
459   if (!isUncondBranchOpcode(I->getOpcode()) &&
460       !isCondBranchOpcode(I->getOpcode()))
461     return 0;
462 
463   // Remove the branch.
464   I->eraseFromParent();
465 
466   I = MBB.end();
467 
468   if (I == MBB.begin()) return 1;
469   --I;
470   if (!isCondBranchOpcode(I->getOpcode()))
471     return 1;
472 
473   // Remove the branch.
474   I->eraseFromParent();
475   return 2;
476 }
477 
478 unsigned ARMBaseInstrInfo::insertBranch(MachineBasicBlock &MBB,
479                                         MachineBasicBlock *TBB,
480                                         MachineBasicBlock *FBB,
481                                         ArrayRef<MachineOperand> Cond,
482                                         const DebugLoc &DL,
483                                         int *BytesAdded) const {
484   assert(!BytesAdded && "code size not handled");
485   ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>();
486   int BOpc   = !AFI->isThumbFunction()
487     ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB);
488   int BccOpc = !AFI->isThumbFunction()
489     ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc);
490   bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function();
491 
492   // Shouldn't be a fall through.
493   assert(TBB && "insertBranch must not be told to insert a fallthrough");
494   assert((Cond.size() == 2 || Cond.size() == 0) &&
495          "ARM branch conditions have two components!");
496 
497   // For conditional branches, we use addOperand to preserve CPSR flags.
498 
499   if (!FBB) {
500     if (Cond.empty()) { // Unconditional branch?
501       if (isThumb)
502         BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB).add(predOps(ARMCC::AL));
503       else
504         BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
505     } else
506       BuildMI(&MBB, DL, get(BccOpc))
507           .addMBB(TBB)
508           .addImm(Cond[0].getImm())
509           .add(Cond[1]);
510     return 1;
511   }
512 
513   // Two-way conditional branch.
514   BuildMI(&MBB, DL, get(BccOpc))
515       .addMBB(TBB)
516       .addImm(Cond[0].getImm())
517       .add(Cond[1]);
518   if (isThumb)
519     BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB).add(predOps(ARMCC::AL));
520   else
521     BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB);
522   return 2;
523 }
524 
525 bool ARMBaseInstrInfo::
526 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
527   ARMCC::CondCodes CC = (ARMCC::CondCodes)(int)Cond[0].getImm();
528   Cond[0].setImm(ARMCC::getOppositeCondition(CC));
529   return false;
530 }
531 
532 bool ARMBaseInstrInfo::isPredicated(const MachineInstr &MI) const {
533   if (MI.isBundle()) {
534     MachineBasicBlock::const_instr_iterator I = MI.getIterator();
535     MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
536     while (++I != E && I->isInsideBundle()) {
537       int PIdx = I->findFirstPredOperandIdx();
538       if (PIdx != -1 && I->getOperand(PIdx).getImm() != ARMCC::AL)
539         return true;
540     }
541     return false;
542   }
543 
544   int PIdx = MI.findFirstPredOperandIdx();
545   return PIdx != -1 && MI.getOperand(PIdx).getImm() != ARMCC::AL;
546 }
547 
548 std::string ARMBaseInstrInfo::createMIROperandComment(
549     const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
550     const TargetRegisterInfo *TRI) const {
551 
552   // First, let's see if there is a generic comment for this operand
553   std::string GenericComment =
554       TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
555   if (!GenericComment.empty())
556     return GenericComment;
557 
558   // If not, check if we have an immediate operand.
559   if (Op.getType() != MachineOperand::MO_Immediate)
560     return std::string();
561 
562   // And print its corresponding condition code if the immediate is a
563   // predicate.
564   int FirstPredOp = MI.findFirstPredOperandIdx();
565   if (FirstPredOp != (int) OpIdx)
566     return std::string();
567 
568   std::string CC = "CC::";
569   CC += ARMCondCodeToString((ARMCC::CondCodes)Op.getImm());
570   return CC;
571 }
572 
573 bool ARMBaseInstrInfo::PredicateInstruction(
574     MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
575   unsigned Opc = MI.getOpcode();
576   if (isUncondBranchOpcode(Opc)) {
577     MI.setDesc(get(getMatchingCondBranchOpcode(Opc)));
578     MachineInstrBuilder(*MI.getParent()->getParent(), MI)
579       .addImm(Pred[0].getImm())
580       .addReg(Pred[1].getReg());
581     return true;
582   }
583 
584   int PIdx = MI.findFirstPredOperandIdx();
585   if (PIdx != -1) {
586     MachineOperand &PMO = MI.getOperand(PIdx);
587     PMO.setImm(Pred[0].getImm());
588     MI.getOperand(PIdx+1).setReg(Pred[1].getReg());
589 
590     // Thumb 1 arithmetic instructions do not set CPSR when executed inside an
591     // IT block. This affects how they are printed.
592     const MCInstrDesc &MCID = MI.getDesc();
593     if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) {
594       assert(MCID.OpInfo[1].isOptionalDef() && "CPSR def isn't expected operand");
595       assert((MI.getOperand(1).isDead() ||
596               MI.getOperand(1).getReg() != ARM::CPSR) &&
597              "if conversion tried to stop defining used CPSR");
598       MI.getOperand(1).setReg(ARM::NoRegister);
599     }
600 
601     return true;
602   }
603   return false;
604 }
605 
606 bool ARMBaseInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
607                                          ArrayRef<MachineOperand> Pred2) const {
608   if (Pred1.size() > 2 || Pred2.size() > 2)
609     return false;
610 
611   ARMCC::CondCodes CC1 = (ARMCC::CondCodes)Pred1[0].getImm();
612   ARMCC::CondCodes CC2 = (ARMCC::CondCodes)Pred2[0].getImm();
613   if (CC1 == CC2)
614     return true;
615 
616   switch (CC1) {
617   default:
618     return false;
619   case ARMCC::AL:
620     return true;
621   case ARMCC::HS:
622     return CC2 == ARMCC::HI;
623   case ARMCC::LS:
624     return CC2 == ARMCC::LO || CC2 == ARMCC::EQ;
625   case ARMCC::GE:
626     return CC2 == ARMCC::GT;
627   case ARMCC::LE:
628     return CC2 == ARMCC::LT;
629   }
630 }
631 
632 bool ARMBaseInstrInfo::ClobbersPredicate(MachineInstr &MI,
633                                          std::vector<MachineOperand> &Pred,
634                                          bool SkipDead) const {
635   bool Found = false;
636   for (const MachineOperand &MO : MI.operands()) {
637     bool ClobbersCPSR = MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR);
638     bool IsCPSR = MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR;
639     if (ClobbersCPSR || IsCPSR) {
640 
641       // Filter out T1 instructions that have a dead CPSR,
642       // allowing IT blocks to be generated containing T1 instructions
643       const MCInstrDesc &MCID = MI.getDesc();
644       if (MCID.TSFlags & ARMII::ThumbArithFlagSetting && MO.isDead() &&
645           SkipDead)
646         continue;
647 
648       Pred.push_back(MO);
649       Found = true;
650     }
651   }
652 
653   return Found;
654 }
655 
656 bool ARMBaseInstrInfo::isCPSRDefined(const MachineInstr &MI) {
657   for (const auto &MO : MI.operands())
658     if (MO.isReg() && MO.getReg() == ARM::CPSR && MO.isDef() && !MO.isDead())
659       return true;
660   return false;
661 }
662 
663 static bool isEligibleForITBlock(const MachineInstr *MI) {
664   switch (MI->getOpcode()) {
665   default: return true;
666   case ARM::tADC:   // ADC (register) T1
667   case ARM::tADDi3: // ADD (immediate) T1
668   case ARM::tADDi8: // ADD (immediate) T2
669   case ARM::tADDrr: // ADD (register) T1
670   case ARM::tAND:   // AND (register) T1
671   case ARM::tASRri: // ASR (immediate) T1
672   case ARM::tASRrr: // ASR (register) T1
673   case ARM::tBIC:   // BIC (register) T1
674   case ARM::tEOR:   // EOR (register) T1
675   case ARM::tLSLri: // LSL (immediate) T1
676   case ARM::tLSLrr: // LSL (register) T1
677   case ARM::tLSRri: // LSR (immediate) T1
678   case ARM::tLSRrr: // LSR (register) T1
679   case ARM::tMUL:   // MUL T1
680   case ARM::tMVN:   // MVN (register) T1
681   case ARM::tORR:   // ORR (register) T1
682   case ARM::tROR:   // ROR (register) T1
683   case ARM::tRSB:   // RSB (immediate) T1
684   case ARM::tSBC:   // SBC (register) T1
685   case ARM::tSUBi3: // SUB (immediate) T1
686   case ARM::tSUBi8: // SUB (immediate) T2
687   case ARM::tSUBrr: // SUB (register) T1
688     return !ARMBaseInstrInfo::isCPSRDefined(*MI);
689   }
690 }
691 
692 /// isPredicable - Return true if the specified instruction can be predicated.
693 /// By default, this returns true for every instruction with a
694 /// PredicateOperand.
695 bool ARMBaseInstrInfo::isPredicable(const MachineInstr &MI) const {
696   if (!MI.isPredicable())
697     return false;
698 
699   if (MI.isBundle())
700     return false;
701 
702   if (!isEligibleForITBlock(&MI))
703     return false;
704 
705   const MachineFunction *MF = MI.getParent()->getParent();
706   const ARMFunctionInfo *AFI =
707       MF->getInfo<ARMFunctionInfo>();
708 
709   // Neon instructions in Thumb2 IT blocks are deprecated, see ARMARM.
710   // In their ARM encoding, they can't be encoded in a conditional form.
711   if ((MI.getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON)
712     return false;
713 
714   // Make indirect control flow changes unpredicable when SLS mitigation is
715   // enabled.
716   const ARMSubtarget &ST = MF->getSubtarget<ARMSubtarget>();
717   if (ST.hardenSlsRetBr() && isIndirectControlFlowNotComingBack(MI))
718     return false;
719   if (ST.hardenSlsBlr() && isIndirectCall(MI))
720     return false;
721 
722   if (AFI->isThumb2Function()) {
723     if (getSubtarget().restrictIT())
724       return isV8EligibleForIT(&MI);
725   }
726 
727   return true;
728 }
729 
730 namespace llvm {
731 
732 template <> bool IsCPSRDead<MachineInstr>(const MachineInstr *MI) {
733   for (const MachineOperand &MO : MI->operands()) {
734     if (!MO.isReg() || MO.isUndef() || MO.isUse())
735       continue;
736     if (MO.getReg() != ARM::CPSR)
737       continue;
738     if (!MO.isDead())
739       return false;
740   }
741   // all definitions of CPSR are dead
742   return true;
743 }
744 
745 } // end namespace llvm
746 
747 /// GetInstSize - Return the size of the specified MachineInstr.
748 ///
749 unsigned ARMBaseInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
750   const MachineBasicBlock &MBB = *MI.getParent();
751   const MachineFunction *MF = MBB.getParent();
752   const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
753 
754   const MCInstrDesc &MCID = MI.getDesc();
755 
756   switch (MI.getOpcode()) {
757   default:
758     // Return the size specified in .td file. If there's none, return 0, as we
759     // can't define a default size (Thumb1 instructions are 2 bytes, Thumb2
760     // instructions are 2-4 bytes, and ARM instructions are 4 bytes), in
761     // contrast to AArch64 instructions which have a default size of 4 bytes for
762     // example.
763     return MCID.getSize();
764   case TargetOpcode::BUNDLE:
765     return getInstBundleLength(MI);
766   case ARM::CONSTPOOL_ENTRY:
767   case ARM::JUMPTABLE_INSTS:
768   case ARM::JUMPTABLE_ADDRS:
769   case ARM::JUMPTABLE_TBB:
770   case ARM::JUMPTABLE_TBH:
771     // If this machine instr is a constant pool entry, its size is recorded as
772     // operand #2.
773     return MI.getOperand(2).getImm();
774   case ARM::SPACE:
775     return MI.getOperand(1).getImm();
776   case ARM::INLINEASM:
777   case ARM::INLINEASM_BR: {
778     // If this machine instr is an inline asm, measure it.
779     unsigned Size = getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI);
780     if (!MF->getInfo<ARMFunctionInfo>()->isThumbFunction())
781       Size = alignTo(Size, 4);
782     return Size;
783   }
784   }
785 }
786 
787 unsigned ARMBaseInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
788   unsigned Size = 0;
789   MachineBasicBlock::const_instr_iterator I = MI.getIterator();
790   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
791   while (++I != E && I->isInsideBundle()) {
792     assert(!I->isBundle() && "No nested bundle!");
793     Size += getInstSizeInBytes(*I);
794   }
795   return Size;
796 }
797 
798 void ARMBaseInstrInfo::copyFromCPSR(MachineBasicBlock &MBB,
799                                     MachineBasicBlock::iterator I,
800                                     unsigned DestReg, bool KillSrc,
801                                     const ARMSubtarget &Subtarget) const {
802   unsigned Opc = Subtarget.isThumb()
803                      ? (Subtarget.isMClass() ? ARM::t2MRS_M : ARM::t2MRS_AR)
804                      : ARM::MRS;
805 
806   MachineInstrBuilder MIB =
807       BuildMI(MBB, I, I->getDebugLoc(), get(Opc), DestReg);
808 
809   // There is only 1 A/R class MRS instruction, and it always refers to
810   // APSR. However, there are lots of other possibilities on M-class cores.
811   if (Subtarget.isMClass())
812     MIB.addImm(0x800);
813 
814   MIB.add(predOps(ARMCC::AL))
815      .addReg(ARM::CPSR, RegState::Implicit | getKillRegState(KillSrc));
816 }
817 
818 void ARMBaseInstrInfo::copyToCPSR(MachineBasicBlock &MBB,
819                                   MachineBasicBlock::iterator I,
820                                   unsigned SrcReg, bool KillSrc,
821                                   const ARMSubtarget &Subtarget) const {
822   unsigned Opc = Subtarget.isThumb()
823                      ? (Subtarget.isMClass() ? ARM::t2MSR_M : ARM::t2MSR_AR)
824                      : ARM::MSR;
825 
826   MachineInstrBuilder MIB = BuildMI(MBB, I, I->getDebugLoc(), get(Opc));
827 
828   if (Subtarget.isMClass())
829     MIB.addImm(0x800);
830   else
831     MIB.addImm(8);
832 
833   MIB.addReg(SrcReg, getKillRegState(KillSrc))
834      .add(predOps(ARMCC::AL))
835      .addReg(ARM::CPSR, RegState::Implicit | RegState::Define);
836 }
837 
838 void llvm::addUnpredicatedMveVpredNOp(MachineInstrBuilder &MIB) {
839   MIB.addImm(ARMVCC::None);
840   MIB.addReg(0);
841   MIB.addReg(0); // tp_reg
842 }
843 
844 void llvm::addUnpredicatedMveVpredROp(MachineInstrBuilder &MIB,
845                                       Register DestReg) {
846   addUnpredicatedMveVpredNOp(MIB);
847   MIB.addReg(DestReg, RegState::Undef);
848 }
849 
850 void llvm::addPredicatedMveVpredNOp(MachineInstrBuilder &MIB, unsigned Cond) {
851   MIB.addImm(Cond);
852   MIB.addReg(ARM::VPR, RegState::Implicit);
853   MIB.addReg(0); // tp_reg
854 }
855 
856 void llvm::addPredicatedMveVpredROp(MachineInstrBuilder &MIB,
857                                     unsigned Cond, unsigned Inactive) {
858   addPredicatedMveVpredNOp(MIB, Cond);
859   MIB.addReg(Inactive);
860 }
861 
862 void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
863                                    MachineBasicBlock::iterator I,
864                                    const DebugLoc &DL, MCRegister DestReg,
865                                    MCRegister SrcReg, bool KillSrc) const {
866   bool GPRDest = ARM::GPRRegClass.contains(DestReg);
867   bool GPRSrc = ARM::GPRRegClass.contains(SrcReg);
868 
869   if (GPRDest && GPRSrc) {
870     BuildMI(MBB, I, DL, get(ARM::MOVr), DestReg)
871         .addReg(SrcReg, getKillRegState(KillSrc))
872         .add(predOps(ARMCC::AL))
873         .add(condCodeOp());
874     return;
875   }
876 
877   bool SPRDest = ARM::SPRRegClass.contains(DestReg);
878   bool SPRSrc = ARM::SPRRegClass.contains(SrcReg);
879 
880   unsigned Opc = 0;
881   if (SPRDest && SPRSrc)
882     Opc = ARM::VMOVS;
883   else if (GPRDest && SPRSrc)
884     Opc = ARM::VMOVRS;
885   else if (SPRDest && GPRSrc)
886     Opc = ARM::VMOVSR;
887   else if (ARM::DPRRegClass.contains(DestReg, SrcReg) && Subtarget.hasFP64())
888     Opc = ARM::VMOVD;
889   else if (ARM::QPRRegClass.contains(DestReg, SrcReg))
890     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MQPRCopy;
891 
892   if (Opc) {
893     MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc), DestReg);
894     MIB.addReg(SrcReg, getKillRegState(KillSrc));
895     if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR)
896       MIB.addReg(SrcReg, getKillRegState(KillSrc));
897     if (Opc == ARM::MVE_VORR)
898       addUnpredicatedMveVpredROp(MIB, DestReg);
899     else if (Opc != ARM::MQPRCopy)
900       MIB.add(predOps(ARMCC::AL));
901     return;
902   }
903 
904   // Handle register classes that require multiple instructions.
905   unsigned BeginIdx = 0;
906   unsigned SubRegs = 0;
907   int Spacing = 1;
908 
909   // Use VORRq when possible.
910   if (ARM::QQPRRegClass.contains(DestReg, SrcReg)) {
911     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
912     BeginIdx = ARM::qsub_0;
913     SubRegs = 2;
914   } else if (ARM::QQQQPRRegClass.contains(DestReg, SrcReg)) {
915     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
916     BeginIdx = ARM::qsub_0;
917     SubRegs = 4;
918   // Fall back to VMOVD.
919   } else if (ARM::DPairRegClass.contains(DestReg, SrcReg)) {
920     Opc = ARM::VMOVD;
921     BeginIdx = ARM::dsub_0;
922     SubRegs = 2;
923   } else if (ARM::DTripleRegClass.contains(DestReg, SrcReg)) {
924     Opc = ARM::VMOVD;
925     BeginIdx = ARM::dsub_0;
926     SubRegs = 3;
927   } else if (ARM::DQuadRegClass.contains(DestReg, SrcReg)) {
928     Opc = ARM::VMOVD;
929     BeginIdx = ARM::dsub_0;
930     SubRegs = 4;
931   } else if (ARM::GPRPairRegClass.contains(DestReg, SrcReg)) {
932     Opc = Subtarget.isThumb2() ? ARM::tMOVr : ARM::MOVr;
933     BeginIdx = ARM::gsub_0;
934     SubRegs = 2;
935   } else if (ARM::DPairSpcRegClass.contains(DestReg, SrcReg)) {
936     Opc = ARM::VMOVD;
937     BeginIdx = ARM::dsub_0;
938     SubRegs = 2;
939     Spacing = 2;
940   } else if (ARM::DTripleSpcRegClass.contains(DestReg, SrcReg)) {
941     Opc = ARM::VMOVD;
942     BeginIdx = ARM::dsub_0;
943     SubRegs = 3;
944     Spacing = 2;
945   } else if (ARM::DQuadSpcRegClass.contains(DestReg, SrcReg)) {
946     Opc = ARM::VMOVD;
947     BeginIdx = ARM::dsub_0;
948     SubRegs = 4;
949     Spacing = 2;
950   } else if (ARM::DPRRegClass.contains(DestReg, SrcReg) &&
951              !Subtarget.hasFP64()) {
952     Opc = ARM::VMOVS;
953     BeginIdx = ARM::ssub_0;
954     SubRegs = 2;
955   } else if (SrcReg == ARM::CPSR) {
956     copyFromCPSR(MBB, I, DestReg, KillSrc, Subtarget);
957     return;
958   } else if (DestReg == ARM::CPSR) {
959     copyToCPSR(MBB, I, SrcReg, KillSrc, Subtarget);
960     return;
961   } else if (DestReg == ARM::VPR) {
962     assert(ARM::GPRRegClass.contains(SrcReg));
963     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_P0), DestReg)
964         .addReg(SrcReg, getKillRegState(KillSrc))
965         .add(predOps(ARMCC::AL));
966     return;
967   } else if (SrcReg == ARM::VPR) {
968     assert(ARM::GPRRegClass.contains(DestReg));
969     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_P0), DestReg)
970         .addReg(SrcReg, getKillRegState(KillSrc))
971         .add(predOps(ARMCC::AL));
972     return;
973   } else if (DestReg == ARM::FPSCR_NZCV) {
974     assert(ARM::GPRRegClass.contains(SrcReg));
975     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_FPSCR_NZCVQC), DestReg)
976         .addReg(SrcReg, getKillRegState(KillSrc))
977         .add(predOps(ARMCC::AL));
978     return;
979   } else if (SrcReg == ARM::FPSCR_NZCV) {
980     assert(ARM::GPRRegClass.contains(DestReg));
981     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_FPSCR_NZCVQC), DestReg)
982         .addReg(SrcReg, getKillRegState(KillSrc))
983         .add(predOps(ARMCC::AL));
984     return;
985   }
986 
987   assert(Opc && "Impossible reg-to-reg copy");
988 
989   const TargetRegisterInfo *TRI = &getRegisterInfo();
990   MachineInstrBuilder Mov;
991 
992   // Copy register tuples backward when the first Dest reg overlaps with SrcReg.
993   if (TRI->regsOverlap(SrcReg, TRI->getSubReg(DestReg, BeginIdx))) {
994     BeginIdx = BeginIdx + ((SubRegs - 1) * Spacing);
995     Spacing = -Spacing;
996   }
997 #ifndef NDEBUG
998   SmallSet<unsigned, 4> DstRegs;
999 #endif
1000   for (unsigned i = 0; i != SubRegs; ++i) {
1001     Register Dst = TRI->getSubReg(DestReg, BeginIdx + i * Spacing);
1002     Register Src = TRI->getSubReg(SrcReg, BeginIdx + i * Spacing);
1003     assert(Dst && Src && "Bad sub-register");
1004 #ifndef NDEBUG
1005     assert(!DstRegs.count(Src) && "destructive vector copy");
1006     DstRegs.insert(Dst);
1007 #endif
1008     Mov = BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst).addReg(Src);
1009     // VORR (NEON or MVE) takes two source operands.
1010     if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR) {
1011       Mov.addReg(Src);
1012     }
1013     // MVE VORR takes predicate operands in place of an ordinary condition.
1014     if (Opc == ARM::MVE_VORR)
1015       addUnpredicatedMveVpredROp(Mov, Dst);
1016     else
1017       Mov = Mov.add(predOps(ARMCC::AL));
1018     // MOVr can set CC.
1019     if (Opc == ARM::MOVr)
1020       Mov = Mov.add(condCodeOp());
1021   }
1022   // Add implicit super-register defs and kills to the last instruction.
1023   Mov->addRegisterDefined(DestReg, TRI);
1024   if (KillSrc)
1025     Mov->addRegisterKilled(SrcReg, TRI);
1026 }
1027 
1028 Optional<DestSourcePair>
1029 ARMBaseInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
1030   // VMOVRRD is also a copy instruction but it requires
1031   // special way of handling. It is more complex copy version
1032   // and since that we are not considering it. For recognition
1033   // of such instruction isExtractSubregLike MI interface fuction
1034   // could be used.
1035   // VORRq is considered as a move only if two inputs are
1036   // the same register.
1037   if (!MI.isMoveReg() ||
1038       (MI.getOpcode() == ARM::VORRq &&
1039        MI.getOperand(1).getReg() != MI.getOperand(2).getReg()))
1040     return None;
1041   return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1042 }
1043 
1044 Optional<ParamLoadedValue>
1045 ARMBaseInstrInfo::describeLoadedValue(const MachineInstr &MI,
1046                                       Register Reg) const {
1047   if (auto DstSrcPair = isCopyInstrImpl(MI)) {
1048     Register DstReg = DstSrcPair->Destination->getReg();
1049 
1050     // TODO: We don't handle cases where the forwarding reg is narrower/wider
1051     // than the copy registers. Consider for example:
1052     //
1053     //   s16 = VMOVS s0
1054     //   s17 = VMOVS s1
1055     //   call @callee(d0)
1056     //
1057     // We'd like to describe the call site value of d0 as d8, but this requires
1058     // gathering and merging the descriptions for the two VMOVS instructions.
1059     //
1060     // We also don't handle the reverse situation, where the forwarding reg is
1061     // narrower than the copy destination:
1062     //
1063     //   d8 = VMOVD d0
1064     //   call @callee(s1)
1065     //
1066     // We need to produce a fragment description (the call site value of s1 is
1067     // /not/ just d8).
1068     if (DstReg != Reg)
1069       return None;
1070   }
1071   return TargetInstrInfo::describeLoadedValue(MI, Reg);
1072 }
1073 
1074 const MachineInstrBuilder &
1075 ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB, unsigned Reg,
1076                           unsigned SubIdx, unsigned State,
1077                           const TargetRegisterInfo *TRI) const {
1078   if (!SubIdx)
1079     return MIB.addReg(Reg, State);
1080 
1081   if (Register::isPhysicalRegister(Reg))
1082     return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
1083   return MIB.addReg(Reg, State, SubIdx);
1084 }
1085 
1086 void ARMBaseInstrInfo::
1087 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
1088                     Register SrcReg, bool isKill, int FI,
1089                     const TargetRegisterClass *RC,
1090                     const TargetRegisterInfo *TRI) const {
1091   MachineFunction &MF = *MBB.getParent();
1092   MachineFrameInfo &MFI = MF.getFrameInfo();
1093   Align Alignment = MFI.getObjectAlign(FI);
1094 
1095   MachineMemOperand *MMO = MF.getMachineMemOperand(
1096       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore,
1097       MFI.getObjectSize(FI), Alignment);
1098 
1099   switch (TRI->getSpillSize(*RC)) {
1100     case 2:
1101       if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1102         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRH))
1103             .addReg(SrcReg, getKillRegState(isKill))
1104             .addFrameIndex(FI)
1105             .addImm(0)
1106             .addMemOperand(MMO)
1107             .add(predOps(ARMCC::AL));
1108       } else
1109         llvm_unreachable("Unknown reg class!");
1110       break;
1111     case 4:
1112       if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1113         BuildMI(MBB, I, DebugLoc(), get(ARM::STRi12))
1114             .addReg(SrcReg, getKillRegState(isKill))
1115             .addFrameIndex(FI)
1116             .addImm(0)
1117             .addMemOperand(MMO)
1118             .add(predOps(ARMCC::AL));
1119       } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1120         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRS))
1121             .addReg(SrcReg, getKillRegState(isKill))
1122             .addFrameIndex(FI)
1123             .addImm(0)
1124             .addMemOperand(MMO)
1125             .add(predOps(ARMCC::AL));
1126       } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1127         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTR_P0_off))
1128             .addReg(SrcReg, getKillRegState(isKill))
1129             .addFrameIndex(FI)
1130             .addImm(0)
1131             .addMemOperand(MMO)
1132             .add(predOps(ARMCC::AL));
1133       } else
1134         llvm_unreachable("Unknown reg class!");
1135       break;
1136     case 8:
1137       if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1138         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRD))
1139             .addReg(SrcReg, getKillRegState(isKill))
1140             .addFrameIndex(FI)
1141             .addImm(0)
1142             .addMemOperand(MMO)
1143             .add(predOps(ARMCC::AL));
1144       } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1145         if (Subtarget.hasV5TEOps()) {
1146           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STRD));
1147           AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
1148           AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
1149           MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO)
1150              .add(predOps(ARMCC::AL));
1151         } else {
1152           // Fallback to STM instruction, which has existed since the dawn of
1153           // time.
1154           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STMIA))
1155                                         .addFrameIndex(FI)
1156                                         .addMemOperand(MMO)
1157                                         .add(predOps(ARMCC::AL));
1158           AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
1159           AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
1160         }
1161       } else
1162         llvm_unreachable("Unknown reg class!");
1163       break;
1164     case 16:
1165       if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1166         // Use aligned spills if the stack can be realigned.
1167         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1168           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1q64))
1169               .addFrameIndex(FI)
1170               .addImm(16)
1171               .addReg(SrcReg, getKillRegState(isKill))
1172               .addMemOperand(MMO)
1173               .add(predOps(ARMCC::AL));
1174         } else {
1175           BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMQIA))
1176               .addReg(SrcReg, getKillRegState(isKill))
1177               .addFrameIndex(FI)
1178               .addMemOperand(MMO)
1179               .add(predOps(ARMCC::AL));
1180         }
1181       } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1182                  Subtarget.hasMVEIntegerOps()) {
1183         auto MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::MVE_VSTRWU32));
1184         MIB.addReg(SrcReg, getKillRegState(isKill))
1185           .addFrameIndex(FI)
1186           .addImm(0)
1187           .addMemOperand(MMO);
1188         addUnpredicatedMveVpredNOp(MIB);
1189       } else
1190         llvm_unreachable("Unknown reg class!");
1191       break;
1192     case 24:
1193       if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1194         // Use aligned spills if the stack can be realigned.
1195         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1196             Subtarget.hasNEON()) {
1197           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64TPseudo))
1198               .addFrameIndex(FI)
1199               .addImm(16)
1200               .addReg(SrcReg, getKillRegState(isKill))
1201               .addMemOperand(MMO)
1202               .add(predOps(ARMCC::AL));
1203         } else {
1204           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(),
1205                                             get(ARM::VSTMDIA))
1206                                         .addFrameIndex(FI)
1207                                         .add(predOps(ARMCC::AL))
1208                                         .addMemOperand(MMO);
1209           MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1210           MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1211           AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1212         }
1213       } else
1214         llvm_unreachable("Unknown reg class!");
1215       break;
1216     case 32:
1217       if (ARM::QQPRRegClass.hasSubClassEq(RC) ||
1218           ARM::MQQPRRegClass.hasSubClassEq(RC) ||
1219           ARM::DQuadRegClass.hasSubClassEq(RC)) {
1220         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1221             Subtarget.hasNEON()) {
1222           // FIXME: It's possible to only store part of the QQ register if the
1223           // spilled def has a sub-register index.
1224           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64QPseudo))
1225               .addFrameIndex(FI)
1226               .addImm(16)
1227               .addReg(SrcReg, getKillRegState(isKill))
1228               .addMemOperand(MMO)
1229               .add(predOps(ARMCC::AL));
1230         } else if (Subtarget.hasMVEIntegerOps()) {
1231           BuildMI(MBB, I, DebugLoc(), get(ARM::MQQPRStore))
1232               .addReg(SrcReg, getKillRegState(isKill))
1233               .addFrameIndex(FI)
1234               .addMemOperand(MMO);
1235         } else {
1236           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(),
1237                                             get(ARM::VSTMDIA))
1238                                         .addFrameIndex(FI)
1239                                         .add(predOps(ARMCC::AL))
1240                                         .addMemOperand(MMO);
1241           MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1242           MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1243           MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1244                 AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
1245         }
1246       } else
1247         llvm_unreachable("Unknown reg class!");
1248       break;
1249     case 64:
1250       if (ARM::MQQQQPRRegClass.hasSubClassEq(RC) &&
1251           Subtarget.hasMVEIntegerOps()) {
1252         BuildMI(MBB, I, DebugLoc(), get(ARM::MQQQQPRStore))
1253             .addReg(SrcReg, getKillRegState(isKill))
1254             .addFrameIndex(FI)
1255             .addMemOperand(MMO);
1256       } else if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1257         MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMDIA))
1258                                       .addFrameIndex(FI)
1259                                       .add(predOps(ARMCC::AL))
1260                                       .addMemOperand(MMO);
1261         MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1262         MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1263         MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1264         MIB = AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
1265         MIB = AddDReg(MIB, SrcReg, ARM::dsub_4, 0, TRI);
1266         MIB = AddDReg(MIB, SrcReg, ARM::dsub_5, 0, TRI);
1267         MIB = AddDReg(MIB, SrcReg, ARM::dsub_6, 0, TRI);
1268               AddDReg(MIB, SrcReg, ARM::dsub_7, 0, TRI);
1269       } else
1270         llvm_unreachable("Unknown reg class!");
1271       break;
1272     default:
1273       llvm_unreachable("Unknown reg class!");
1274   }
1275 }
1276 
1277 unsigned ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1278                                               int &FrameIndex) const {
1279   switch (MI.getOpcode()) {
1280   default: break;
1281   case ARM::STRrs:
1282   case ARM::t2STRs: // FIXME: don't use t2STRs to access frame.
1283     if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() &&
1284         MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 &&
1285         MI.getOperand(3).getImm() == 0) {
1286       FrameIndex = MI.getOperand(1).getIndex();
1287       return MI.getOperand(0).getReg();
1288     }
1289     break;
1290   case ARM::STRi12:
1291   case ARM::t2STRi12:
1292   case ARM::tSTRspi:
1293   case ARM::VSTRD:
1294   case ARM::VSTRS:
1295   case ARM::VSTR_P0_off:
1296   case ARM::MVE_VSTRWU32:
1297     if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
1298         MI.getOperand(2).getImm() == 0) {
1299       FrameIndex = MI.getOperand(1).getIndex();
1300       return MI.getOperand(0).getReg();
1301     }
1302     break;
1303   case ARM::VST1q64:
1304   case ARM::VST1d64TPseudo:
1305   case ARM::VST1d64QPseudo:
1306     if (MI.getOperand(0).isFI() && MI.getOperand(2).getSubReg() == 0) {
1307       FrameIndex = MI.getOperand(0).getIndex();
1308       return MI.getOperand(2).getReg();
1309     }
1310     break;
1311   case ARM::VSTMQIA:
1312     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1313       FrameIndex = MI.getOperand(1).getIndex();
1314       return MI.getOperand(0).getReg();
1315     }
1316     break;
1317   case ARM::MQQPRStore:
1318   case ARM::MQQQQPRStore:
1319     if (MI.getOperand(1).isFI()) {
1320       FrameIndex = MI.getOperand(1).getIndex();
1321       return MI.getOperand(0).getReg();
1322     }
1323     break;
1324   }
1325 
1326   return 0;
1327 }
1328 
1329 unsigned ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
1330                                                     int &FrameIndex) const {
1331   SmallVector<const MachineMemOperand *, 1> Accesses;
1332   if (MI.mayStore() && hasStoreToStackSlot(MI, Accesses) &&
1333       Accesses.size() == 1) {
1334     FrameIndex =
1335         cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
1336             ->getFrameIndex();
1337     return true;
1338   }
1339   return false;
1340 }
1341 
1342 void ARMBaseInstrInfo::
1343 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
1344                      Register DestReg, int FI,
1345                      const TargetRegisterClass *RC,
1346                      const TargetRegisterInfo *TRI) const {
1347   DebugLoc DL;
1348   if (I != MBB.end()) DL = I->getDebugLoc();
1349   MachineFunction &MF = *MBB.getParent();
1350   MachineFrameInfo &MFI = MF.getFrameInfo();
1351   const Align Alignment = MFI.getObjectAlign(FI);
1352   MachineMemOperand *MMO = MF.getMachineMemOperand(
1353       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad,
1354       MFI.getObjectSize(FI), Alignment);
1355 
1356   switch (TRI->getSpillSize(*RC)) {
1357   case 2:
1358     if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1359       BuildMI(MBB, I, DL, get(ARM::VLDRH), DestReg)
1360           .addFrameIndex(FI)
1361           .addImm(0)
1362           .addMemOperand(MMO)
1363           .add(predOps(ARMCC::AL));
1364     } else
1365       llvm_unreachable("Unknown reg class!");
1366     break;
1367   case 4:
1368     if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1369       BuildMI(MBB, I, DL, get(ARM::LDRi12), DestReg)
1370           .addFrameIndex(FI)
1371           .addImm(0)
1372           .addMemOperand(MMO)
1373           .add(predOps(ARMCC::AL));
1374     } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1375       BuildMI(MBB, I, DL, get(ARM::VLDRS), DestReg)
1376           .addFrameIndex(FI)
1377           .addImm(0)
1378           .addMemOperand(MMO)
1379           .add(predOps(ARMCC::AL));
1380     } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1381       BuildMI(MBB, I, DL, get(ARM::VLDR_P0_off), DestReg)
1382           .addFrameIndex(FI)
1383           .addImm(0)
1384           .addMemOperand(MMO)
1385           .add(predOps(ARMCC::AL));
1386     } else
1387       llvm_unreachable("Unknown reg class!");
1388     break;
1389   case 8:
1390     if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1391       BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg)
1392           .addFrameIndex(FI)
1393           .addImm(0)
1394           .addMemOperand(MMO)
1395           .add(predOps(ARMCC::AL));
1396     } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1397       MachineInstrBuilder MIB;
1398 
1399       if (Subtarget.hasV5TEOps()) {
1400         MIB = BuildMI(MBB, I, DL, get(ARM::LDRD));
1401         AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1402         AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1403         MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO)
1404            .add(predOps(ARMCC::AL));
1405       } else {
1406         // Fallback to LDM instruction, which has existed since the dawn of
1407         // time.
1408         MIB = BuildMI(MBB, I, DL, get(ARM::LDMIA))
1409                   .addFrameIndex(FI)
1410                   .addMemOperand(MMO)
1411                   .add(predOps(ARMCC::AL));
1412         MIB = AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1413         MIB = AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1414       }
1415 
1416       if (Register::isPhysicalRegister(DestReg))
1417         MIB.addReg(DestReg, RegState::ImplicitDefine);
1418     } else
1419       llvm_unreachable("Unknown reg class!");
1420     break;
1421   case 16:
1422     if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1423       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1424         BuildMI(MBB, I, DL, get(ARM::VLD1q64), DestReg)
1425             .addFrameIndex(FI)
1426             .addImm(16)
1427             .addMemOperand(MMO)
1428             .add(predOps(ARMCC::AL));
1429       } else {
1430         BuildMI(MBB, I, DL, get(ARM::VLDMQIA), DestReg)
1431             .addFrameIndex(FI)
1432             .addMemOperand(MMO)
1433             .add(predOps(ARMCC::AL));
1434       }
1435     } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1436                Subtarget.hasMVEIntegerOps()) {
1437       auto MIB = BuildMI(MBB, I, DL, get(ARM::MVE_VLDRWU32), DestReg);
1438       MIB.addFrameIndex(FI)
1439         .addImm(0)
1440         .addMemOperand(MMO);
1441       addUnpredicatedMveVpredNOp(MIB);
1442     } else
1443       llvm_unreachable("Unknown reg class!");
1444     break;
1445   case 24:
1446     if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1447       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1448           Subtarget.hasNEON()) {
1449         BuildMI(MBB, I, DL, get(ARM::VLD1d64TPseudo), DestReg)
1450             .addFrameIndex(FI)
1451             .addImm(16)
1452             .addMemOperand(MMO)
1453             .add(predOps(ARMCC::AL));
1454       } else {
1455         MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1456                                       .addFrameIndex(FI)
1457                                       .addMemOperand(MMO)
1458                                       .add(predOps(ARMCC::AL));
1459         MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1460         MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1461         MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1462         if (Register::isPhysicalRegister(DestReg))
1463           MIB.addReg(DestReg, RegState::ImplicitDefine);
1464       }
1465     } else
1466       llvm_unreachable("Unknown reg class!");
1467     break;
1468    case 32:
1469      if (ARM::QQPRRegClass.hasSubClassEq(RC) ||
1470          ARM::MQQPRRegClass.hasSubClassEq(RC) ||
1471          ARM::DQuadRegClass.hasSubClassEq(RC)) {
1472        if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1473            Subtarget.hasNEON()) {
1474          BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg)
1475              .addFrameIndex(FI)
1476              .addImm(16)
1477              .addMemOperand(MMO)
1478              .add(predOps(ARMCC::AL));
1479        } else if (Subtarget.hasMVEIntegerOps()) {
1480          BuildMI(MBB, I, DL, get(ARM::MQQPRLoad), DestReg)
1481              .addFrameIndex(FI)
1482              .addMemOperand(MMO);
1483        } else {
1484          MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1485                                        .addFrameIndex(FI)
1486                                        .add(predOps(ARMCC::AL))
1487                                        .addMemOperand(MMO);
1488          MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1489          MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1490          MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1491          MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1492          if (Register::isPhysicalRegister(DestReg))
1493            MIB.addReg(DestReg, RegState::ImplicitDefine);
1494        }
1495      } else
1496        llvm_unreachable("Unknown reg class!");
1497      break;
1498   case 64:
1499     if (ARM::MQQQQPRRegClass.hasSubClassEq(RC) &&
1500         Subtarget.hasMVEIntegerOps()) {
1501       BuildMI(MBB, I, DL, get(ARM::MQQQQPRLoad), DestReg)
1502           .addFrameIndex(FI)
1503           .addMemOperand(MMO);
1504     } else if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1505       MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1506                                     .addFrameIndex(FI)
1507                                     .add(predOps(ARMCC::AL))
1508                                     .addMemOperand(MMO);
1509       MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1510       MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1511       MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1512       MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1513       MIB = AddDReg(MIB, DestReg, ARM::dsub_4, RegState::DefineNoRead, TRI);
1514       MIB = AddDReg(MIB, DestReg, ARM::dsub_5, RegState::DefineNoRead, TRI);
1515       MIB = AddDReg(MIB, DestReg, ARM::dsub_6, RegState::DefineNoRead, TRI);
1516       MIB = AddDReg(MIB, DestReg, ARM::dsub_7, RegState::DefineNoRead, TRI);
1517       if (Register::isPhysicalRegister(DestReg))
1518         MIB.addReg(DestReg, RegState::ImplicitDefine);
1519     } else
1520       llvm_unreachable("Unknown reg class!");
1521     break;
1522   default:
1523     llvm_unreachable("Unknown regclass!");
1524   }
1525 }
1526 
1527 unsigned ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1528                                                int &FrameIndex) const {
1529   switch (MI.getOpcode()) {
1530   default: break;
1531   case ARM::LDRrs:
1532   case ARM::t2LDRs:  // FIXME: don't use t2LDRs to access frame.
1533     if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() &&
1534         MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 &&
1535         MI.getOperand(3).getImm() == 0) {
1536       FrameIndex = MI.getOperand(1).getIndex();
1537       return MI.getOperand(0).getReg();
1538     }
1539     break;
1540   case ARM::LDRi12:
1541   case ARM::t2LDRi12:
1542   case ARM::tLDRspi:
1543   case ARM::VLDRD:
1544   case ARM::VLDRS:
1545   case ARM::VLDR_P0_off:
1546   case ARM::MVE_VLDRWU32:
1547     if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
1548         MI.getOperand(2).getImm() == 0) {
1549       FrameIndex = MI.getOperand(1).getIndex();
1550       return MI.getOperand(0).getReg();
1551     }
1552     break;
1553   case ARM::VLD1q64:
1554   case ARM::VLD1d8TPseudo:
1555   case ARM::VLD1d16TPseudo:
1556   case ARM::VLD1d32TPseudo:
1557   case ARM::VLD1d64TPseudo:
1558   case ARM::VLD1d8QPseudo:
1559   case ARM::VLD1d16QPseudo:
1560   case ARM::VLD1d32QPseudo:
1561   case ARM::VLD1d64QPseudo:
1562     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1563       FrameIndex = MI.getOperand(1).getIndex();
1564       return MI.getOperand(0).getReg();
1565     }
1566     break;
1567   case ARM::VLDMQIA:
1568     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1569       FrameIndex = MI.getOperand(1).getIndex();
1570       return MI.getOperand(0).getReg();
1571     }
1572     break;
1573   case ARM::MQQPRLoad:
1574   case ARM::MQQQQPRLoad:
1575     if (MI.getOperand(1).isFI()) {
1576       FrameIndex = MI.getOperand(1).getIndex();
1577       return MI.getOperand(0).getReg();
1578     }
1579     break;
1580   }
1581 
1582   return 0;
1583 }
1584 
1585 unsigned ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
1586                                                      int &FrameIndex) const {
1587   SmallVector<const MachineMemOperand *, 1> Accesses;
1588   if (MI.mayLoad() && hasLoadFromStackSlot(MI, Accesses) &&
1589       Accesses.size() == 1) {
1590     FrameIndex =
1591         cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
1592             ->getFrameIndex();
1593     return true;
1594   }
1595   return false;
1596 }
1597 
1598 /// Expands MEMCPY to either LDMIA/STMIA or LDMIA_UPD/STMID_UPD
1599 /// depending on whether the result is used.
1600 void ARMBaseInstrInfo::expandMEMCPY(MachineBasicBlock::iterator MI) const {
1601   bool isThumb1 = Subtarget.isThumb1Only();
1602   bool isThumb2 = Subtarget.isThumb2();
1603   const ARMBaseInstrInfo *TII = Subtarget.getInstrInfo();
1604 
1605   DebugLoc dl = MI->getDebugLoc();
1606   MachineBasicBlock *BB = MI->getParent();
1607 
1608   MachineInstrBuilder LDM, STM;
1609   if (isThumb1 || !MI->getOperand(1).isDead()) {
1610     MachineOperand LDWb(MI->getOperand(1));
1611     LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA_UPD
1612                                                  : isThumb1 ? ARM::tLDMIA_UPD
1613                                                             : ARM::LDMIA_UPD))
1614               .add(LDWb);
1615   } else {
1616     LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA : ARM::LDMIA));
1617   }
1618 
1619   if (isThumb1 || !MI->getOperand(0).isDead()) {
1620     MachineOperand STWb(MI->getOperand(0));
1621     STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA_UPD
1622                                                  : isThumb1 ? ARM::tSTMIA_UPD
1623                                                             : ARM::STMIA_UPD))
1624               .add(STWb);
1625   } else {
1626     STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA : ARM::STMIA));
1627   }
1628 
1629   MachineOperand LDBase(MI->getOperand(3));
1630   LDM.add(LDBase).add(predOps(ARMCC::AL));
1631 
1632   MachineOperand STBase(MI->getOperand(2));
1633   STM.add(STBase).add(predOps(ARMCC::AL));
1634 
1635   // Sort the scratch registers into ascending order.
1636   const TargetRegisterInfo &TRI = getRegisterInfo();
1637   SmallVector<unsigned, 6> ScratchRegs;
1638   for(unsigned I = 5; I < MI->getNumOperands(); ++I)
1639     ScratchRegs.push_back(MI->getOperand(I).getReg());
1640   llvm::sort(ScratchRegs,
1641              [&TRI](const unsigned &Reg1, const unsigned &Reg2) -> bool {
1642                return TRI.getEncodingValue(Reg1) <
1643                       TRI.getEncodingValue(Reg2);
1644              });
1645 
1646   for (const auto &Reg : ScratchRegs) {
1647     LDM.addReg(Reg, RegState::Define);
1648     STM.addReg(Reg, RegState::Kill);
1649   }
1650 
1651   BB->erase(MI);
1652 }
1653 
1654 bool ARMBaseInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1655   if (MI.getOpcode() == TargetOpcode::LOAD_STACK_GUARD) {
1656     expandLoadStackGuard(MI);
1657     MI.getParent()->erase(MI);
1658     return true;
1659   }
1660 
1661   if (MI.getOpcode() == ARM::MEMCPY) {
1662     expandMEMCPY(MI);
1663     return true;
1664   }
1665 
1666   // This hook gets to expand COPY instructions before they become
1667   // copyPhysReg() calls.  Look for VMOVS instructions that can legally be
1668   // widened to VMOVD.  We prefer the VMOVD when possible because it may be
1669   // changed into a VORR that can go down the NEON pipeline.
1670   if (!MI.isCopy() || Subtarget.dontWidenVMOVS() || !Subtarget.hasFP64())
1671     return false;
1672 
1673   // Look for a copy between even S-registers.  That is where we keep floats
1674   // when using NEON v2f32 instructions for f32 arithmetic.
1675   Register DstRegS = MI.getOperand(0).getReg();
1676   Register SrcRegS = MI.getOperand(1).getReg();
1677   if (!ARM::SPRRegClass.contains(DstRegS, SrcRegS))
1678     return false;
1679 
1680   const TargetRegisterInfo *TRI = &getRegisterInfo();
1681   unsigned DstRegD = TRI->getMatchingSuperReg(DstRegS, ARM::ssub_0,
1682                                               &ARM::DPRRegClass);
1683   unsigned SrcRegD = TRI->getMatchingSuperReg(SrcRegS, ARM::ssub_0,
1684                                               &ARM::DPRRegClass);
1685   if (!DstRegD || !SrcRegD)
1686     return false;
1687 
1688   // We want to widen this into a DstRegD = VMOVD SrcRegD copy.  This is only
1689   // legal if the COPY already defines the full DstRegD, and it isn't a
1690   // sub-register insertion.
1691   if (!MI.definesRegister(DstRegD, TRI) || MI.readsRegister(DstRegD, TRI))
1692     return false;
1693 
1694   // A dead copy shouldn't show up here, but reject it just in case.
1695   if (MI.getOperand(0).isDead())
1696     return false;
1697 
1698   // All clear, widen the COPY.
1699   LLVM_DEBUG(dbgs() << "widening:    " << MI);
1700   MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
1701 
1702   // Get rid of the old implicit-def of DstRegD.  Leave it if it defines a Q-reg
1703   // or some other super-register.
1704   int ImpDefIdx = MI.findRegisterDefOperandIdx(DstRegD);
1705   if (ImpDefIdx != -1)
1706     MI.RemoveOperand(ImpDefIdx);
1707 
1708   // Change the opcode and operands.
1709   MI.setDesc(get(ARM::VMOVD));
1710   MI.getOperand(0).setReg(DstRegD);
1711   MI.getOperand(1).setReg(SrcRegD);
1712   MIB.add(predOps(ARMCC::AL));
1713 
1714   // We are now reading SrcRegD instead of SrcRegS.  This may upset the
1715   // register scavenger and machine verifier, so we need to indicate that we
1716   // are reading an undefined value from SrcRegD, but a proper value from
1717   // SrcRegS.
1718   MI.getOperand(1).setIsUndef();
1719   MIB.addReg(SrcRegS, RegState::Implicit);
1720 
1721   // SrcRegD may actually contain an unrelated value in the ssub_1
1722   // sub-register.  Don't kill it.  Only kill the ssub_0 sub-register.
1723   if (MI.getOperand(1).isKill()) {
1724     MI.getOperand(1).setIsKill(false);
1725     MI.addRegisterKilled(SrcRegS, TRI, true);
1726   }
1727 
1728   LLVM_DEBUG(dbgs() << "replaced by: " << MI);
1729   return true;
1730 }
1731 
1732 /// Create a copy of a const pool value. Update CPI to the new index and return
1733 /// the label UID.
1734 static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) {
1735   MachineConstantPool *MCP = MF.getConstantPool();
1736   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1737 
1738   const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPI];
1739   assert(MCPE.isMachineConstantPoolEntry() &&
1740          "Expecting a machine constantpool entry!");
1741   ARMConstantPoolValue *ACPV =
1742     static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
1743 
1744   unsigned PCLabelId = AFI->createPICLabelUId();
1745   ARMConstantPoolValue *NewCPV = nullptr;
1746 
1747   // FIXME: The below assumes PIC relocation model and that the function
1748   // is Thumb mode (t1 or t2). PCAdjustment would be 8 for ARM mode PIC, and
1749   // zero for non-PIC in ARM or Thumb. The callers are all of thumb LDR
1750   // instructions, so that's probably OK, but is PIC always correct when
1751   // we get here?
1752   if (ACPV->isGlobalValue())
1753     NewCPV = ARMConstantPoolConstant::Create(
1754         cast<ARMConstantPoolConstant>(ACPV)->getGV(), PCLabelId, ARMCP::CPValue,
1755         4, ACPV->getModifier(), ACPV->mustAddCurrentAddress());
1756   else if (ACPV->isExtSymbol())
1757     NewCPV = ARMConstantPoolSymbol::
1758       Create(MF.getFunction().getContext(),
1759              cast<ARMConstantPoolSymbol>(ACPV)->getSymbol(), PCLabelId, 4);
1760   else if (ACPV->isBlockAddress())
1761     NewCPV = ARMConstantPoolConstant::
1762       Create(cast<ARMConstantPoolConstant>(ACPV)->getBlockAddress(), PCLabelId,
1763              ARMCP::CPBlockAddress, 4);
1764   else if (ACPV->isLSDA())
1765     NewCPV = ARMConstantPoolConstant::Create(&MF.getFunction(), PCLabelId,
1766                                              ARMCP::CPLSDA, 4);
1767   else if (ACPV->isMachineBasicBlock())
1768     NewCPV = ARMConstantPoolMBB::
1769       Create(MF.getFunction().getContext(),
1770              cast<ARMConstantPoolMBB>(ACPV)->getMBB(), PCLabelId, 4);
1771   else
1772     llvm_unreachable("Unexpected ARM constantpool value type!!");
1773   CPI = MCP->getConstantPoolIndex(NewCPV, MCPE.getAlign());
1774   return PCLabelId;
1775 }
1776 
1777 void ARMBaseInstrInfo::reMaterialize(MachineBasicBlock &MBB,
1778                                      MachineBasicBlock::iterator I,
1779                                      Register DestReg, unsigned SubIdx,
1780                                      const MachineInstr &Orig,
1781                                      const TargetRegisterInfo &TRI) const {
1782   unsigned Opcode = Orig.getOpcode();
1783   switch (Opcode) {
1784   default: {
1785     MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
1786     MI->substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI);
1787     MBB.insert(I, MI);
1788     break;
1789   }
1790   case ARM::tLDRpci_pic:
1791   case ARM::t2LDRpci_pic: {
1792     MachineFunction &MF = *MBB.getParent();
1793     unsigned CPI = Orig.getOperand(1).getIndex();
1794     unsigned PCLabelId = duplicateCPV(MF, CPI);
1795     BuildMI(MBB, I, Orig.getDebugLoc(), get(Opcode), DestReg)
1796         .addConstantPoolIndex(CPI)
1797         .addImm(PCLabelId)
1798         .cloneMemRefs(Orig);
1799     break;
1800   }
1801   }
1802 }
1803 
1804 MachineInstr &
1805 ARMBaseInstrInfo::duplicate(MachineBasicBlock &MBB,
1806     MachineBasicBlock::iterator InsertBefore,
1807     const MachineInstr &Orig) const {
1808   MachineInstr &Cloned = TargetInstrInfo::duplicate(MBB, InsertBefore, Orig);
1809   MachineBasicBlock::instr_iterator I = Cloned.getIterator();
1810   for (;;) {
1811     switch (I->getOpcode()) {
1812     case ARM::tLDRpci_pic:
1813     case ARM::t2LDRpci_pic: {
1814       MachineFunction &MF = *MBB.getParent();
1815       unsigned CPI = I->getOperand(1).getIndex();
1816       unsigned PCLabelId = duplicateCPV(MF, CPI);
1817       I->getOperand(1).setIndex(CPI);
1818       I->getOperand(2).setImm(PCLabelId);
1819       break;
1820     }
1821     }
1822     if (!I->isBundledWithSucc())
1823       break;
1824     ++I;
1825   }
1826   return Cloned;
1827 }
1828 
1829 bool ARMBaseInstrInfo::produceSameValue(const MachineInstr &MI0,
1830                                         const MachineInstr &MI1,
1831                                         const MachineRegisterInfo *MRI) const {
1832   unsigned Opcode = MI0.getOpcode();
1833   if (Opcode == ARM::t2LDRpci || Opcode == ARM::t2LDRpci_pic ||
1834       Opcode == ARM::tLDRpci || Opcode == ARM::tLDRpci_pic ||
1835       Opcode == ARM::LDRLIT_ga_pcrel || Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1836       Opcode == ARM::tLDRLIT_ga_pcrel || Opcode == ARM::t2LDRLIT_ga_pcrel ||
1837       Opcode == ARM::MOV_ga_pcrel || Opcode == ARM::MOV_ga_pcrel_ldr ||
1838       Opcode == ARM::t2MOV_ga_pcrel) {
1839     if (MI1.getOpcode() != Opcode)
1840       return false;
1841     if (MI0.getNumOperands() != MI1.getNumOperands())
1842       return false;
1843 
1844     const MachineOperand &MO0 = MI0.getOperand(1);
1845     const MachineOperand &MO1 = MI1.getOperand(1);
1846     if (MO0.getOffset() != MO1.getOffset())
1847       return false;
1848 
1849     if (Opcode == ARM::LDRLIT_ga_pcrel || Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1850         Opcode == ARM::tLDRLIT_ga_pcrel || Opcode == ARM::t2LDRLIT_ga_pcrel ||
1851         Opcode == ARM::MOV_ga_pcrel || Opcode == ARM::MOV_ga_pcrel_ldr ||
1852         Opcode == ARM::t2MOV_ga_pcrel)
1853       // Ignore the PC labels.
1854       return MO0.getGlobal() == MO1.getGlobal();
1855 
1856     const MachineFunction *MF = MI0.getParent()->getParent();
1857     const MachineConstantPool *MCP = MF->getConstantPool();
1858     int CPI0 = MO0.getIndex();
1859     int CPI1 = MO1.getIndex();
1860     const MachineConstantPoolEntry &MCPE0 = MCP->getConstants()[CPI0];
1861     const MachineConstantPoolEntry &MCPE1 = MCP->getConstants()[CPI1];
1862     bool isARMCP0 = MCPE0.isMachineConstantPoolEntry();
1863     bool isARMCP1 = MCPE1.isMachineConstantPoolEntry();
1864     if (isARMCP0 && isARMCP1) {
1865       ARMConstantPoolValue *ACPV0 =
1866         static_cast<ARMConstantPoolValue*>(MCPE0.Val.MachineCPVal);
1867       ARMConstantPoolValue *ACPV1 =
1868         static_cast<ARMConstantPoolValue*>(MCPE1.Val.MachineCPVal);
1869       return ACPV0->hasSameValue(ACPV1);
1870     } else if (!isARMCP0 && !isARMCP1) {
1871       return MCPE0.Val.ConstVal == MCPE1.Val.ConstVal;
1872     }
1873     return false;
1874   } else if (Opcode == ARM::PICLDR) {
1875     if (MI1.getOpcode() != Opcode)
1876       return false;
1877     if (MI0.getNumOperands() != MI1.getNumOperands())
1878       return false;
1879 
1880     Register Addr0 = MI0.getOperand(1).getReg();
1881     Register Addr1 = MI1.getOperand(1).getReg();
1882     if (Addr0 != Addr1) {
1883       if (!MRI || !Register::isVirtualRegister(Addr0) ||
1884           !Register::isVirtualRegister(Addr1))
1885         return false;
1886 
1887       // This assumes SSA form.
1888       MachineInstr *Def0 = MRI->getVRegDef(Addr0);
1889       MachineInstr *Def1 = MRI->getVRegDef(Addr1);
1890       // Check if the loaded value, e.g. a constantpool of a global address, are
1891       // the same.
1892       if (!produceSameValue(*Def0, *Def1, MRI))
1893         return false;
1894     }
1895 
1896     for (unsigned i = 3, e = MI0.getNumOperands(); i != e; ++i) {
1897       // %12 = PICLDR %11, 0, 14, %noreg
1898       const MachineOperand &MO0 = MI0.getOperand(i);
1899       const MachineOperand &MO1 = MI1.getOperand(i);
1900       if (!MO0.isIdenticalTo(MO1))
1901         return false;
1902     }
1903     return true;
1904   }
1905 
1906   return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs);
1907 }
1908 
1909 /// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to
1910 /// determine if two loads are loading from the same base address. It should
1911 /// only return true if the base pointers are the same and the only differences
1912 /// between the two addresses is the offset. It also returns the offsets by
1913 /// reference.
1914 ///
1915 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1916 /// is permanently disabled.
1917 bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
1918                                                int64_t &Offset1,
1919                                                int64_t &Offset2) const {
1920   // Don't worry about Thumb: just ARM and Thumb2.
1921   if (Subtarget.isThumb1Only()) return false;
1922 
1923   if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
1924     return false;
1925 
1926   switch (Load1->getMachineOpcode()) {
1927   default:
1928     return false;
1929   case ARM::LDRi12:
1930   case ARM::LDRBi12:
1931   case ARM::LDRD:
1932   case ARM::LDRH:
1933   case ARM::LDRSB:
1934   case ARM::LDRSH:
1935   case ARM::VLDRD:
1936   case ARM::VLDRS:
1937   case ARM::t2LDRi8:
1938   case ARM::t2LDRBi8:
1939   case ARM::t2LDRDi8:
1940   case ARM::t2LDRSHi8:
1941   case ARM::t2LDRi12:
1942   case ARM::t2LDRBi12:
1943   case ARM::t2LDRSHi12:
1944     break;
1945   }
1946 
1947   switch (Load2->getMachineOpcode()) {
1948   default:
1949     return false;
1950   case ARM::LDRi12:
1951   case ARM::LDRBi12:
1952   case ARM::LDRD:
1953   case ARM::LDRH:
1954   case ARM::LDRSB:
1955   case ARM::LDRSH:
1956   case ARM::VLDRD:
1957   case ARM::VLDRS:
1958   case ARM::t2LDRi8:
1959   case ARM::t2LDRBi8:
1960   case ARM::t2LDRSHi8:
1961   case ARM::t2LDRi12:
1962   case ARM::t2LDRBi12:
1963   case ARM::t2LDRSHi12:
1964     break;
1965   }
1966 
1967   // Check if base addresses and chain operands match.
1968   if (Load1->getOperand(0) != Load2->getOperand(0) ||
1969       Load1->getOperand(4) != Load2->getOperand(4))
1970     return false;
1971 
1972   // Index should be Reg0.
1973   if (Load1->getOperand(3) != Load2->getOperand(3))
1974     return false;
1975 
1976   // Determine the offsets.
1977   if (isa<ConstantSDNode>(Load1->getOperand(1)) &&
1978       isa<ConstantSDNode>(Load2->getOperand(1))) {
1979     Offset1 = cast<ConstantSDNode>(Load1->getOperand(1))->getSExtValue();
1980     Offset2 = cast<ConstantSDNode>(Load2->getOperand(1))->getSExtValue();
1981     return true;
1982   }
1983 
1984   return false;
1985 }
1986 
1987 /// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
1988 /// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should
1989 /// be scheduled togther. On some targets if two loads are loading from
1990 /// addresses in the same cache line, it's better if they are scheduled
1991 /// together. This function takes two integers that represent the load offsets
1992 /// from the common base address. It returns true if it decides it's desirable
1993 /// to schedule the two loads together. "NumLoads" is the number of loads that
1994 /// have already been scheduled after Load1.
1995 ///
1996 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1997 /// is permanently disabled.
1998 bool ARMBaseInstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
1999                                                int64_t Offset1, int64_t Offset2,
2000                                                unsigned NumLoads) const {
2001   // Don't worry about Thumb: just ARM and Thumb2.
2002   if (Subtarget.isThumb1Only()) return false;
2003 
2004   assert(Offset2 > Offset1);
2005 
2006   if ((Offset2 - Offset1) / 8 > 64)
2007     return false;
2008 
2009   // Check if the machine opcodes are different. If they are different
2010   // then we consider them to not be of the same base address,
2011   // EXCEPT in the case of Thumb2 byte loads where one is LDRBi8 and the other LDRBi12.
2012   // In this case, they are considered to be the same because they are different
2013   // encoding forms of the same basic instruction.
2014   if ((Load1->getMachineOpcode() != Load2->getMachineOpcode()) &&
2015       !((Load1->getMachineOpcode() == ARM::t2LDRBi8 &&
2016          Load2->getMachineOpcode() == ARM::t2LDRBi12) ||
2017         (Load1->getMachineOpcode() == ARM::t2LDRBi12 &&
2018          Load2->getMachineOpcode() == ARM::t2LDRBi8)))
2019     return false;  // FIXME: overly conservative?
2020 
2021   // Four loads in a row should be sufficient.
2022   if (NumLoads >= 3)
2023     return false;
2024 
2025   return true;
2026 }
2027 
2028 bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2029                                             const MachineBasicBlock *MBB,
2030                                             const MachineFunction &MF) const {
2031   // Debug info is never a scheduling boundary. It's necessary to be explicit
2032   // due to the special treatment of IT instructions below, otherwise a
2033   // dbg_value followed by an IT will result in the IT instruction being
2034   // considered a scheduling hazard, which is wrong. It should be the actual
2035   // instruction preceding the dbg_value instruction(s), just like it is
2036   // when debug info is not present.
2037   if (MI.isDebugInstr())
2038     return false;
2039 
2040   // Terminators and labels can't be scheduled around.
2041   if (MI.isTerminator() || MI.isPosition())
2042     return true;
2043 
2044   // INLINEASM_BR can jump to another block
2045   if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
2046     return true;
2047 
2048   // Treat the start of the IT block as a scheduling boundary, but schedule
2049   // t2IT along with all instructions following it.
2050   // FIXME: This is a big hammer. But the alternative is to add all potential
2051   // true and anti dependencies to IT block instructions as implicit operands
2052   // to the t2IT instruction. The added compile time and complexity does not
2053   // seem worth it.
2054   MachineBasicBlock::const_iterator I = MI;
2055   // Make sure to skip any debug instructions
2056   while (++I != MBB->end() && I->isDebugInstr())
2057     ;
2058   if (I != MBB->end() && I->getOpcode() == ARM::t2IT)
2059     return true;
2060 
2061   // Don't attempt to schedule around any instruction that defines
2062   // a stack-oriented pointer, as it's unlikely to be profitable. This
2063   // saves compile time, because it doesn't require every single
2064   // stack slot reference to depend on the instruction that does the
2065   // modification.
2066   // Calls don't actually change the stack pointer, even if they have imp-defs.
2067   // No ARM calling conventions change the stack pointer. (X86 calling
2068   // conventions sometimes do).
2069   if (!MI.isCall() && MI.definesRegister(ARM::SP))
2070     return true;
2071 
2072   return false;
2073 }
2074 
2075 bool ARMBaseInstrInfo::
2076 isProfitableToIfCvt(MachineBasicBlock &MBB,
2077                     unsigned NumCycles, unsigned ExtraPredCycles,
2078                     BranchProbability Probability) const {
2079   if (!NumCycles)
2080     return false;
2081 
2082   // If we are optimizing for size, see if the branch in the predecessor can be
2083   // lowered to cbn?z by the constant island lowering pass, and return false if
2084   // so. This results in a shorter instruction sequence.
2085   if (MBB.getParent()->getFunction().hasOptSize()) {
2086     MachineBasicBlock *Pred = *MBB.pred_begin();
2087     if (!Pred->empty()) {
2088       MachineInstr *LastMI = &*Pred->rbegin();
2089       if (LastMI->getOpcode() == ARM::t2Bcc) {
2090         const TargetRegisterInfo *TRI = &getRegisterInfo();
2091         MachineInstr *CmpMI = findCMPToFoldIntoCBZ(LastMI, TRI);
2092         if (CmpMI)
2093           return false;
2094       }
2095     }
2096   }
2097   return isProfitableToIfCvt(MBB, NumCycles, ExtraPredCycles,
2098                              MBB, 0, 0, Probability);
2099 }
2100 
2101 bool ARMBaseInstrInfo::
2102 isProfitableToIfCvt(MachineBasicBlock &TBB,
2103                     unsigned TCycles, unsigned TExtra,
2104                     MachineBasicBlock &FBB,
2105                     unsigned FCycles, unsigned FExtra,
2106                     BranchProbability Probability) const {
2107   if (!TCycles)
2108     return false;
2109 
2110   // In thumb code we often end up trading one branch for a IT block, and
2111   // if we are cloning the instruction can increase code size. Prevent
2112   // blocks with multiple predecesors from being ifcvted to prevent this
2113   // cloning.
2114   if (Subtarget.isThumb2() && TBB.getParent()->getFunction().hasMinSize()) {
2115     if (TBB.pred_size() != 1 || FBB.pred_size() != 1)
2116       return false;
2117   }
2118 
2119   // Attempt to estimate the relative costs of predication versus branching.
2120   // Here we scale up each component of UnpredCost to avoid precision issue when
2121   // scaling TCycles/FCycles by Probability.
2122   const unsigned ScalingUpFactor = 1024;
2123 
2124   unsigned PredCost = (TCycles + FCycles + TExtra + FExtra) * ScalingUpFactor;
2125   unsigned UnpredCost;
2126   if (!Subtarget.hasBranchPredictor()) {
2127     // When we don't have a branch predictor it's always cheaper to not take a
2128     // branch than take it, so we have to take that into account.
2129     unsigned NotTakenBranchCost = 1;
2130     unsigned TakenBranchCost = Subtarget.getMispredictionPenalty();
2131     unsigned TUnpredCycles, FUnpredCycles;
2132     if (!FCycles) {
2133       // Triangle: TBB is the fallthrough
2134       TUnpredCycles = TCycles + NotTakenBranchCost;
2135       FUnpredCycles = TakenBranchCost;
2136     } else {
2137       // Diamond: TBB is the block that is branched to, FBB is the fallthrough
2138       TUnpredCycles = TCycles + TakenBranchCost;
2139       FUnpredCycles = FCycles + NotTakenBranchCost;
2140       // The branch at the end of FBB will disappear when it's predicated, so
2141       // discount it from PredCost.
2142       PredCost -= 1 * ScalingUpFactor;
2143     }
2144     // The total cost is the cost of each path scaled by their probabilites
2145     unsigned TUnpredCost = Probability.scale(TUnpredCycles * ScalingUpFactor);
2146     unsigned FUnpredCost = Probability.getCompl().scale(FUnpredCycles * ScalingUpFactor);
2147     UnpredCost = TUnpredCost + FUnpredCost;
2148     // When predicating assume that the first IT can be folded away but later
2149     // ones cost one cycle each
2150     if (Subtarget.isThumb2() && TCycles + FCycles > 4) {
2151       PredCost += ((TCycles + FCycles - 4) / 4) * ScalingUpFactor;
2152     }
2153   } else {
2154     unsigned TUnpredCost = Probability.scale(TCycles * ScalingUpFactor);
2155     unsigned FUnpredCost =
2156       Probability.getCompl().scale(FCycles * ScalingUpFactor);
2157     UnpredCost = TUnpredCost + FUnpredCost;
2158     UnpredCost += 1 * ScalingUpFactor; // The branch itself
2159     UnpredCost += Subtarget.getMispredictionPenalty() * ScalingUpFactor / 10;
2160   }
2161 
2162   return PredCost <= UnpredCost;
2163 }
2164 
2165 unsigned
2166 ARMBaseInstrInfo::extraSizeToPredicateInstructions(const MachineFunction &MF,
2167                                                    unsigned NumInsts) const {
2168   // Thumb2 needs a 2-byte IT instruction to predicate up to 4 instructions.
2169   // ARM has a condition code field in every predicable instruction, using it
2170   // doesn't change code size.
2171   if (!Subtarget.isThumb2())
2172     return 0;
2173 
2174   // It's possible that the size of the IT is restricted to a single block.
2175   unsigned MaxInsts = Subtarget.restrictIT() ? 1 : 4;
2176   return divideCeil(NumInsts, MaxInsts) * 2;
2177 }
2178 
2179 unsigned
2180 ARMBaseInstrInfo::predictBranchSizeForIfCvt(MachineInstr &MI) const {
2181   // If this branch is likely to be folded into the comparison to form a
2182   // CB(N)Z, then removing it won't reduce code size at all, because that will
2183   // just replace the CB(N)Z with a CMP.
2184   if (MI.getOpcode() == ARM::t2Bcc &&
2185       findCMPToFoldIntoCBZ(&MI, &getRegisterInfo()))
2186     return 0;
2187 
2188   unsigned Size = getInstSizeInBytes(MI);
2189 
2190   // For Thumb2, all branches are 32-bit instructions during the if conversion
2191   // pass, but may be replaced with 16-bit instructions during size reduction.
2192   // Since the branches considered by if conversion tend to be forward branches
2193   // over small basic blocks, they are very likely to be in range for the
2194   // narrow instructions, so we assume the final code size will be half what it
2195   // currently is.
2196   if (Subtarget.isThumb2())
2197     Size /= 2;
2198 
2199   return Size;
2200 }
2201 
2202 bool
2203 ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
2204                                             MachineBasicBlock &FMBB) const {
2205   // Reduce false anti-dependencies to let the target's out-of-order execution
2206   // engine do its thing.
2207   return Subtarget.isProfitableToUnpredicate();
2208 }
2209 
2210 /// getInstrPredicate - If instruction is predicated, returns its predicate
2211 /// condition, otherwise returns AL. It also returns the condition code
2212 /// register by reference.
2213 ARMCC::CondCodes llvm::getInstrPredicate(const MachineInstr &MI,
2214                                          Register &PredReg) {
2215   int PIdx = MI.findFirstPredOperandIdx();
2216   if (PIdx == -1) {
2217     PredReg = 0;
2218     return ARMCC::AL;
2219   }
2220 
2221   PredReg = MI.getOperand(PIdx+1).getReg();
2222   return (ARMCC::CondCodes)MI.getOperand(PIdx).getImm();
2223 }
2224 
2225 unsigned llvm::getMatchingCondBranchOpcode(unsigned Opc) {
2226   if (Opc == ARM::B)
2227     return ARM::Bcc;
2228   if (Opc == ARM::tB)
2229     return ARM::tBcc;
2230   if (Opc == ARM::t2B)
2231     return ARM::t2Bcc;
2232 
2233   llvm_unreachable("Unknown unconditional branch opcode!");
2234 }
2235 
2236 MachineInstr *ARMBaseInstrInfo::commuteInstructionImpl(MachineInstr &MI,
2237                                                        bool NewMI,
2238                                                        unsigned OpIdx1,
2239                                                        unsigned OpIdx2) const {
2240   switch (MI.getOpcode()) {
2241   case ARM::MOVCCr:
2242   case ARM::t2MOVCCr: {
2243     // MOVCC can be commuted by inverting the condition.
2244     Register PredReg;
2245     ARMCC::CondCodes CC = getInstrPredicate(MI, PredReg);
2246     // MOVCC AL can't be inverted. Shouldn't happen.
2247     if (CC == ARMCC::AL || PredReg != ARM::CPSR)
2248       return nullptr;
2249     MachineInstr *CommutedMI =
2250         TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2251     if (!CommutedMI)
2252       return nullptr;
2253     // After swapping the MOVCC operands, also invert the condition.
2254     CommutedMI->getOperand(CommutedMI->findFirstPredOperandIdx())
2255         .setImm(ARMCC::getOppositeCondition(CC));
2256     return CommutedMI;
2257   }
2258   }
2259   return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2260 }
2261 
2262 /// Identify instructions that can be folded into a MOVCC instruction, and
2263 /// return the defining instruction.
2264 MachineInstr *
2265 ARMBaseInstrInfo::canFoldIntoMOVCC(Register Reg, const MachineRegisterInfo &MRI,
2266                                    const TargetInstrInfo *TII) const {
2267   if (!Reg.isVirtual())
2268     return nullptr;
2269   if (!MRI.hasOneNonDBGUse(Reg))
2270     return nullptr;
2271   MachineInstr *MI = MRI.getVRegDef(Reg);
2272   if (!MI)
2273     return nullptr;
2274   // Check if MI can be predicated and folded into the MOVCC.
2275   if (!isPredicable(*MI))
2276     return nullptr;
2277   // Check if MI has any non-dead defs or physreg uses. This also detects
2278   // predicated instructions which will be reading CPSR.
2279   for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 1)) {
2280     // Reject frame index operands, PEI can't handle the predicated pseudos.
2281     if (MO.isFI() || MO.isCPI() || MO.isJTI())
2282       return nullptr;
2283     if (!MO.isReg())
2284       continue;
2285     // MI can't have any tied operands, that would conflict with predication.
2286     if (MO.isTied())
2287       return nullptr;
2288     if (Register::isPhysicalRegister(MO.getReg()))
2289       return nullptr;
2290     if (MO.isDef() && !MO.isDead())
2291       return nullptr;
2292   }
2293   bool DontMoveAcrossStores = true;
2294   if (!MI->isSafeToMove(/* AliasAnalysis = */ nullptr, DontMoveAcrossStores))
2295     return nullptr;
2296   return MI;
2297 }
2298 
2299 bool ARMBaseInstrInfo::analyzeSelect(const MachineInstr &MI,
2300                                      SmallVectorImpl<MachineOperand> &Cond,
2301                                      unsigned &TrueOp, unsigned &FalseOp,
2302                                      bool &Optimizable) const {
2303   assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2304          "Unknown select instruction");
2305   // MOVCC operands:
2306   // 0: Def.
2307   // 1: True use.
2308   // 2: False use.
2309   // 3: Condition code.
2310   // 4: CPSR use.
2311   TrueOp = 1;
2312   FalseOp = 2;
2313   Cond.push_back(MI.getOperand(3));
2314   Cond.push_back(MI.getOperand(4));
2315   // We can always fold a def.
2316   Optimizable = true;
2317   return false;
2318 }
2319 
2320 MachineInstr *
2321 ARMBaseInstrInfo::optimizeSelect(MachineInstr &MI,
2322                                  SmallPtrSetImpl<MachineInstr *> &SeenMIs,
2323                                  bool PreferFalse) const {
2324   assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2325          "Unknown select instruction");
2326   MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2327   MachineInstr *DefMI = canFoldIntoMOVCC(MI.getOperand(2).getReg(), MRI, this);
2328   bool Invert = !DefMI;
2329   if (!DefMI)
2330     DefMI = canFoldIntoMOVCC(MI.getOperand(1).getReg(), MRI, this);
2331   if (!DefMI)
2332     return nullptr;
2333 
2334   // Find new register class to use.
2335   MachineOperand FalseReg = MI.getOperand(Invert ? 2 : 1);
2336   MachineOperand TrueReg = MI.getOperand(Invert ? 1 : 2);
2337   Register DestReg = MI.getOperand(0).getReg();
2338   const TargetRegisterClass *FalseClass = MRI.getRegClass(FalseReg.getReg());
2339   const TargetRegisterClass *TrueClass = MRI.getRegClass(TrueReg.getReg());
2340   if (!MRI.constrainRegClass(DestReg, FalseClass))
2341     return nullptr;
2342   if (!MRI.constrainRegClass(DestReg, TrueClass))
2343     return nullptr;
2344 
2345   // Create a new predicated version of DefMI.
2346   // Rfalse is the first use.
2347   MachineInstrBuilder NewMI =
2348       BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), DefMI->getDesc(), DestReg);
2349 
2350   // Copy all the DefMI operands, excluding its (null) predicate.
2351   const MCInstrDesc &DefDesc = DefMI->getDesc();
2352   for (unsigned i = 1, e = DefDesc.getNumOperands();
2353        i != e && !DefDesc.OpInfo[i].isPredicate(); ++i)
2354     NewMI.add(DefMI->getOperand(i));
2355 
2356   unsigned CondCode = MI.getOperand(3).getImm();
2357   if (Invert)
2358     NewMI.addImm(ARMCC::getOppositeCondition(ARMCC::CondCodes(CondCode)));
2359   else
2360     NewMI.addImm(CondCode);
2361   NewMI.add(MI.getOperand(4));
2362 
2363   // DefMI is not the -S version that sets CPSR, so add an optional %noreg.
2364   if (NewMI->hasOptionalDef())
2365     NewMI.add(condCodeOp());
2366 
2367   // The output register value when the predicate is false is an implicit
2368   // register operand tied to the first def.
2369   // The tie makes the register allocator ensure the FalseReg is allocated the
2370   // same register as operand 0.
2371   FalseReg.setImplicit();
2372   NewMI.add(FalseReg);
2373   NewMI->tieOperands(0, NewMI->getNumOperands() - 1);
2374 
2375   // Update SeenMIs set: register newly created MI and erase removed DefMI.
2376   SeenMIs.insert(NewMI);
2377   SeenMIs.erase(DefMI);
2378 
2379   // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
2380   // DefMI would be invalid when tranferred inside the loop.  Checking for a
2381   // loop is expensive, but at least remove kill flags if they are in different
2382   // BBs.
2383   if (DefMI->getParent() != MI.getParent())
2384     NewMI->clearKillInfo();
2385 
2386   // The caller will erase MI, but not DefMI.
2387   DefMI->eraseFromParent();
2388   return NewMI;
2389 }
2390 
2391 /// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the
2392 /// instruction is encoded with an 'S' bit is determined by the optional CPSR
2393 /// def operand.
2394 ///
2395 /// This will go away once we can teach tblgen how to set the optional CPSR def
2396 /// operand itself.
2397 struct AddSubFlagsOpcodePair {
2398   uint16_t PseudoOpc;
2399   uint16_t MachineOpc;
2400 };
2401 
2402 static const AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = {
2403   {ARM::ADDSri, ARM::ADDri},
2404   {ARM::ADDSrr, ARM::ADDrr},
2405   {ARM::ADDSrsi, ARM::ADDrsi},
2406   {ARM::ADDSrsr, ARM::ADDrsr},
2407 
2408   {ARM::SUBSri, ARM::SUBri},
2409   {ARM::SUBSrr, ARM::SUBrr},
2410   {ARM::SUBSrsi, ARM::SUBrsi},
2411   {ARM::SUBSrsr, ARM::SUBrsr},
2412 
2413   {ARM::RSBSri, ARM::RSBri},
2414   {ARM::RSBSrsi, ARM::RSBrsi},
2415   {ARM::RSBSrsr, ARM::RSBrsr},
2416 
2417   {ARM::tADDSi3, ARM::tADDi3},
2418   {ARM::tADDSi8, ARM::tADDi8},
2419   {ARM::tADDSrr, ARM::tADDrr},
2420   {ARM::tADCS, ARM::tADC},
2421 
2422   {ARM::tSUBSi3, ARM::tSUBi3},
2423   {ARM::tSUBSi8, ARM::tSUBi8},
2424   {ARM::tSUBSrr, ARM::tSUBrr},
2425   {ARM::tSBCS, ARM::tSBC},
2426   {ARM::tRSBS, ARM::tRSB},
2427   {ARM::tLSLSri, ARM::tLSLri},
2428 
2429   {ARM::t2ADDSri, ARM::t2ADDri},
2430   {ARM::t2ADDSrr, ARM::t2ADDrr},
2431   {ARM::t2ADDSrs, ARM::t2ADDrs},
2432 
2433   {ARM::t2SUBSri, ARM::t2SUBri},
2434   {ARM::t2SUBSrr, ARM::t2SUBrr},
2435   {ARM::t2SUBSrs, ARM::t2SUBrs},
2436 
2437   {ARM::t2RSBSri, ARM::t2RSBri},
2438   {ARM::t2RSBSrs, ARM::t2RSBrs},
2439 };
2440 
2441 unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) {
2442   for (unsigned i = 0, e = array_lengthof(AddSubFlagsOpcodeMap); i != e; ++i)
2443     if (OldOpc == AddSubFlagsOpcodeMap[i].PseudoOpc)
2444       return AddSubFlagsOpcodeMap[i].MachineOpc;
2445   return 0;
2446 }
2447 
2448 void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB,
2449                                    MachineBasicBlock::iterator &MBBI,
2450                                    const DebugLoc &dl, Register DestReg,
2451                                    Register BaseReg, int NumBytes,
2452                                    ARMCC::CondCodes Pred, Register PredReg,
2453                                    const ARMBaseInstrInfo &TII,
2454                                    unsigned MIFlags) {
2455   if (NumBytes == 0 && DestReg != BaseReg) {
2456     BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), DestReg)
2457         .addReg(BaseReg, RegState::Kill)
2458         .add(predOps(Pred, PredReg))
2459         .add(condCodeOp())
2460         .setMIFlags(MIFlags);
2461     return;
2462   }
2463 
2464   bool isSub = NumBytes < 0;
2465   if (isSub) NumBytes = -NumBytes;
2466 
2467   while (NumBytes) {
2468     unsigned RotAmt = ARM_AM::getSOImmValRotate(NumBytes);
2469     unsigned ThisVal = NumBytes & ARM_AM::rotr32(0xFF, RotAmt);
2470     assert(ThisVal && "Didn't extract field correctly");
2471 
2472     // We will handle these bits from offset, clear them.
2473     NumBytes &= ~ThisVal;
2474 
2475     assert(ARM_AM::getSOImmVal(ThisVal) != -1 && "Bit extraction didn't work?");
2476 
2477     // Build the new ADD / SUB.
2478     unsigned Opc = isSub ? ARM::SUBri : ARM::ADDri;
2479     BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg)
2480         .addReg(BaseReg, RegState::Kill)
2481         .addImm(ThisVal)
2482         .add(predOps(Pred, PredReg))
2483         .add(condCodeOp())
2484         .setMIFlags(MIFlags);
2485     BaseReg = DestReg;
2486   }
2487 }
2488 
2489 bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
2490                                       MachineFunction &MF, MachineInstr *MI,
2491                                       unsigned NumBytes) {
2492   // This optimisation potentially adds lots of load and store
2493   // micro-operations, it's only really a great benefit to code-size.
2494   if (!Subtarget.hasMinSize())
2495     return false;
2496 
2497   // If only one register is pushed/popped, LLVM can use an LDR/STR
2498   // instead. We can't modify those so make sure we're dealing with an
2499   // instruction we understand.
2500   bool IsPop = isPopOpcode(MI->getOpcode());
2501   bool IsPush = isPushOpcode(MI->getOpcode());
2502   if (!IsPush && !IsPop)
2503     return false;
2504 
2505   bool IsVFPPushPop = MI->getOpcode() == ARM::VSTMDDB_UPD ||
2506                       MI->getOpcode() == ARM::VLDMDIA_UPD;
2507   bool IsT1PushPop = MI->getOpcode() == ARM::tPUSH ||
2508                      MI->getOpcode() == ARM::tPOP ||
2509                      MI->getOpcode() == ARM::tPOP_RET;
2510 
2511   assert((IsT1PushPop || (MI->getOperand(0).getReg() == ARM::SP &&
2512                           MI->getOperand(1).getReg() == ARM::SP)) &&
2513          "trying to fold sp update into non-sp-updating push/pop");
2514 
2515   // The VFP push & pop act on D-registers, so we can only fold an adjustment
2516   // by a multiple of 8 bytes in correctly. Similarly rN is 4-bytes. Don't try
2517   // if this is violated.
2518   if (NumBytes % (IsVFPPushPop ? 8 : 4) != 0)
2519     return false;
2520 
2521   // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
2522   // pred) so the list starts at 4. Thumb1 starts after the predicate.
2523   int RegListIdx = IsT1PushPop ? 2 : 4;
2524 
2525   // Calculate the space we'll need in terms of registers.
2526   unsigned RegsNeeded;
2527   const TargetRegisterClass *RegClass;
2528   if (IsVFPPushPop) {
2529     RegsNeeded = NumBytes / 8;
2530     RegClass = &ARM::DPRRegClass;
2531   } else {
2532     RegsNeeded = NumBytes / 4;
2533     RegClass = &ARM::GPRRegClass;
2534   }
2535 
2536   // We're going to have to strip all list operands off before
2537   // re-adding them since the order matters, so save the existing ones
2538   // for later.
2539   SmallVector<MachineOperand, 4> RegList;
2540 
2541   // We're also going to need the first register transferred by this
2542   // instruction, which won't necessarily be the first register in the list.
2543   unsigned FirstRegEnc = -1;
2544 
2545   const TargetRegisterInfo *TRI = MF.getRegInfo().getTargetRegisterInfo();
2546   for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i) {
2547     MachineOperand &MO = MI->getOperand(i);
2548     RegList.push_back(MO);
2549 
2550     if (MO.isReg() && !MO.isImplicit() &&
2551         TRI->getEncodingValue(MO.getReg()) < FirstRegEnc)
2552       FirstRegEnc = TRI->getEncodingValue(MO.getReg());
2553   }
2554 
2555   const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
2556 
2557   // Now try to find enough space in the reglist to allocate NumBytes.
2558   for (int CurRegEnc = FirstRegEnc - 1; CurRegEnc >= 0 && RegsNeeded;
2559        --CurRegEnc) {
2560     unsigned CurReg = RegClass->getRegister(CurRegEnc);
2561     if (IsT1PushPop && CurRegEnc > TRI->getEncodingValue(ARM::R7))
2562       continue;
2563     if (!IsPop) {
2564       // Pushing any register is completely harmless, mark the register involved
2565       // as undef since we don't care about its value and must not restore it
2566       // during stack unwinding.
2567       RegList.push_back(MachineOperand::CreateReg(CurReg, false, false,
2568                                                   false, false, true));
2569       --RegsNeeded;
2570       continue;
2571     }
2572 
2573     // However, we can only pop an extra register if it's not live. For
2574     // registers live within the function we might clobber a return value
2575     // register; the other way a register can be live here is if it's
2576     // callee-saved.
2577     if (isCalleeSavedRegister(CurReg, CSRegs) ||
2578         MI->getParent()->computeRegisterLiveness(TRI, CurReg, MI) !=
2579         MachineBasicBlock::LQR_Dead) {
2580       // VFP pops don't allow holes in the register list, so any skip is fatal
2581       // for our transformation. GPR pops do, so we should just keep looking.
2582       if (IsVFPPushPop)
2583         return false;
2584       else
2585         continue;
2586     }
2587 
2588     // Mark the unimportant registers as <def,dead> in the POP.
2589     RegList.push_back(MachineOperand::CreateReg(CurReg, true, false, false,
2590                                                 true));
2591     --RegsNeeded;
2592   }
2593 
2594   if (RegsNeeded > 0)
2595     return false;
2596 
2597   // Finally we know we can profitably perform the optimisation so go
2598   // ahead: strip all existing registers off and add them back again
2599   // in the right order.
2600   for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i)
2601     MI->RemoveOperand(i);
2602 
2603   // Add the complete list back in.
2604   MachineInstrBuilder MIB(MF, &*MI);
2605   for (const MachineOperand &MO : llvm::reverse(RegList))
2606     MIB.add(MO);
2607 
2608   return true;
2609 }
2610 
2611 bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2612                                 Register FrameReg, int &Offset,
2613                                 const ARMBaseInstrInfo &TII) {
2614   unsigned Opcode = MI.getOpcode();
2615   const MCInstrDesc &Desc = MI.getDesc();
2616   unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
2617   bool isSub = false;
2618 
2619   // Memory operands in inline assembly always use AddrMode2.
2620   if (Opcode == ARM::INLINEASM || Opcode == ARM::INLINEASM_BR)
2621     AddrMode = ARMII::AddrMode2;
2622 
2623   if (Opcode == ARM::ADDri) {
2624     Offset += MI.getOperand(FrameRegIdx+1).getImm();
2625     if (Offset == 0) {
2626       // Turn it into a move.
2627       MI.setDesc(TII.get(ARM::MOVr));
2628       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2629       MI.RemoveOperand(FrameRegIdx+1);
2630       Offset = 0;
2631       return true;
2632     } else if (Offset < 0) {
2633       Offset = -Offset;
2634       isSub = true;
2635       MI.setDesc(TII.get(ARM::SUBri));
2636     }
2637 
2638     // Common case: small offset, fits into instruction.
2639     if (ARM_AM::getSOImmVal(Offset) != -1) {
2640       // Replace the FrameIndex with sp / fp
2641       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2642       MI.getOperand(FrameRegIdx+1).ChangeToImmediate(Offset);
2643       Offset = 0;
2644       return true;
2645     }
2646 
2647     // Otherwise, pull as much of the immedidate into this ADDri/SUBri
2648     // as possible.
2649     unsigned RotAmt = ARM_AM::getSOImmValRotate(Offset);
2650     unsigned ThisImmVal = Offset & ARM_AM::rotr32(0xFF, RotAmt);
2651 
2652     // We will handle these bits from offset, clear them.
2653     Offset &= ~ThisImmVal;
2654 
2655     // Get the properly encoded SOImmVal field.
2656     assert(ARM_AM::getSOImmVal(ThisImmVal) != -1 &&
2657            "Bit extraction didn't work?");
2658     MI.getOperand(FrameRegIdx+1).ChangeToImmediate(ThisImmVal);
2659  } else {
2660     unsigned ImmIdx = 0;
2661     int InstrOffs = 0;
2662     unsigned NumBits = 0;
2663     unsigned Scale = 1;
2664     switch (AddrMode) {
2665     case ARMII::AddrMode_i12:
2666       ImmIdx = FrameRegIdx + 1;
2667       InstrOffs = MI.getOperand(ImmIdx).getImm();
2668       NumBits = 12;
2669       break;
2670     case ARMII::AddrMode2:
2671       ImmIdx = FrameRegIdx+2;
2672       InstrOffs = ARM_AM::getAM2Offset(MI.getOperand(ImmIdx).getImm());
2673       if (ARM_AM::getAM2Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2674         InstrOffs *= -1;
2675       NumBits = 12;
2676       break;
2677     case ARMII::AddrMode3:
2678       ImmIdx = FrameRegIdx+2;
2679       InstrOffs = ARM_AM::getAM3Offset(MI.getOperand(ImmIdx).getImm());
2680       if (ARM_AM::getAM3Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2681         InstrOffs *= -1;
2682       NumBits = 8;
2683       break;
2684     case ARMII::AddrMode4:
2685     case ARMII::AddrMode6:
2686       // Can't fold any offset even if it's zero.
2687       return false;
2688     case ARMII::AddrMode5:
2689       ImmIdx = FrameRegIdx+1;
2690       InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm());
2691       if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2692         InstrOffs *= -1;
2693       NumBits = 8;
2694       Scale = 4;
2695       break;
2696     case ARMII::AddrMode5FP16:
2697       ImmIdx = FrameRegIdx+1;
2698       InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm());
2699       if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2700         InstrOffs *= -1;
2701       NumBits = 8;
2702       Scale = 2;
2703       break;
2704     case ARMII::AddrModeT2_i7:
2705     case ARMII::AddrModeT2_i7s2:
2706     case ARMII::AddrModeT2_i7s4:
2707       ImmIdx = FrameRegIdx+1;
2708       InstrOffs = MI.getOperand(ImmIdx).getImm();
2709       NumBits = 7;
2710       Scale = (AddrMode == ARMII::AddrModeT2_i7s2 ? 2 :
2711                AddrMode == ARMII::AddrModeT2_i7s4 ? 4 : 1);
2712       break;
2713     default:
2714       llvm_unreachable("Unsupported addressing mode!");
2715     }
2716 
2717     Offset += InstrOffs * Scale;
2718     assert((Offset & (Scale-1)) == 0 && "Can't encode this offset!");
2719     if (Offset < 0) {
2720       Offset = -Offset;
2721       isSub = true;
2722     }
2723 
2724     // Attempt to fold address comp. if opcode has offset bits
2725     if (NumBits > 0) {
2726       // Common case: small offset, fits into instruction.
2727       MachineOperand &ImmOp = MI.getOperand(ImmIdx);
2728       int ImmedOffset = Offset / Scale;
2729       unsigned Mask = (1 << NumBits) - 1;
2730       if ((unsigned)Offset <= Mask * Scale) {
2731         // Replace the FrameIndex with sp
2732         MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2733         // FIXME: When addrmode2 goes away, this will simplify (like the
2734         // T2 version), as the LDR.i12 versions don't need the encoding
2735         // tricks for the offset value.
2736         if (isSub) {
2737           if (AddrMode == ARMII::AddrMode_i12)
2738             ImmedOffset = -ImmedOffset;
2739           else
2740             ImmedOffset |= 1 << NumBits;
2741         }
2742         ImmOp.ChangeToImmediate(ImmedOffset);
2743         Offset = 0;
2744         return true;
2745       }
2746 
2747       // Otherwise, it didn't fit. Pull in what we can to simplify the immed.
2748       ImmedOffset = ImmedOffset & Mask;
2749       if (isSub) {
2750         if (AddrMode == ARMII::AddrMode_i12)
2751           ImmedOffset = -ImmedOffset;
2752         else
2753           ImmedOffset |= 1 << NumBits;
2754       }
2755       ImmOp.ChangeToImmediate(ImmedOffset);
2756       Offset &= ~(Mask*Scale);
2757     }
2758   }
2759 
2760   Offset = (isSub) ? -Offset : Offset;
2761   return Offset == 0;
2762 }
2763 
2764 /// analyzeCompare - For a comparison instruction, return the source registers
2765 /// in SrcReg and SrcReg2 if having two register operands, and the value it
2766 /// compares against in CmpValue. Return true if the comparison instruction
2767 /// can be analyzed.
2768 bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2769                                       Register &SrcReg2, int64_t &CmpMask,
2770                                       int64_t &CmpValue) const {
2771   switch (MI.getOpcode()) {
2772   default: break;
2773   case ARM::CMPri:
2774   case ARM::t2CMPri:
2775   case ARM::tCMPi8:
2776     SrcReg = MI.getOperand(0).getReg();
2777     SrcReg2 = 0;
2778     CmpMask = ~0;
2779     CmpValue = MI.getOperand(1).getImm();
2780     return true;
2781   case ARM::CMPrr:
2782   case ARM::t2CMPrr:
2783   case ARM::tCMPr:
2784     SrcReg = MI.getOperand(0).getReg();
2785     SrcReg2 = MI.getOperand(1).getReg();
2786     CmpMask = ~0;
2787     CmpValue = 0;
2788     return true;
2789   case ARM::TSTri:
2790   case ARM::t2TSTri:
2791     SrcReg = MI.getOperand(0).getReg();
2792     SrcReg2 = 0;
2793     CmpMask = MI.getOperand(1).getImm();
2794     CmpValue = 0;
2795     return true;
2796   }
2797 
2798   return false;
2799 }
2800 
2801 /// isSuitableForMask - Identify a suitable 'and' instruction that
2802 /// operates on the given source register and applies the same mask
2803 /// as a 'tst' instruction. Provide a limited look-through for copies.
2804 /// When successful, MI will hold the found instruction.
2805 static bool isSuitableForMask(MachineInstr *&MI, Register SrcReg,
2806                               int CmpMask, bool CommonUse) {
2807   switch (MI->getOpcode()) {
2808     case ARM::ANDri:
2809     case ARM::t2ANDri:
2810       if (CmpMask != MI->getOperand(2).getImm())
2811         return false;
2812       if (SrcReg == MI->getOperand(CommonUse ? 1 : 0).getReg())
2813         return true;
2814       break;
2815   }
2816 
2817   return false;
2818 }
2819 
2820 /// getCmpToAddCondition - assume the flags are set by CMP(a,b), return
2821 /// the condition code if we modify the instructions such that flags are
2822 /// set by ADD(a,b,X).
2823 inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) {
2824   switch (CC) {
2825   default: return ARMCC::AL;
2826   case ARMCC::HS: return ARMCC::LO;
2827   case ARMCC::LO: return ARMCC::HS;
2828   case ARMCC::VS: return ARMCC::VS;
2829   case ARMCC::VC: return ARMCC::VC;
2830   }
2831 }
2832 
2833 /// isRedundantFlagInstr - check whether the first instruction, whose only
2834 /// purpose is to update flags, can be made redundant.
2835 /// CMPrr can be made redundant by SUBrr if the operands are the same.
2836 /// CMPri can be made redundant by SUBri if the operands are the same.
2837 /// CMPrr(r0, r1) can be made redundant by ADDr[ri](r0, r1, X).
2838 /// This function can be extended later on.
2839 inline static bool isRedundantFlagInstr(const MachineInstr *CmpI,
2840                                         Register SrcReg, Register SrcReg2,
2841                                         int64_t ImmValue,
2842                                         const MachineInstr *OI,
2843                                         bool &IsThumb1) {
2844   if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2845       (OI->getOpcode() == ARM::SUBrr || OI->getOpcode() == ARM::t2SUBrr) &&
2846       ((OI->getOperand(1).getReg() == SrcReg &&
2847         OI->getOperand(2).getReg() == SrcReg2) ||
2848        (OI->getOperand(1).getReg() == SrcReg2 &&
2849         OI->getOperand(2).getReg() == SrcReg))) {
2850     IsThumb1 = false;
2851     return true;
2852   }
2853 
2854   if (CmpI->getOpcode() == ARM::tCMPr && OI->getOpcode() == ARM::tSUBrr &&
2855       ((OI->getOperand(2).getReg() == SrcReg &&
2856         OI->getOperand(3).getReg() == SrcReg2) ||
2857        (OI->getOperand(2).getReg() == SrcReg2 &&
2858         OI->getOperand(3).getReg() == SrcReg))) {
2859     IsThumb1 = true;
2860     return true;
2861   }
2862 
2863   if ((CmpI->getOpcode() == ARM::CMPri || CmpI->getOpcode() == ARM::t2CMPri) &&
2864       (OI->getOpcode() == ARM::SUBri || OI->getOpcode() == ARM::t2SUBri) &&
2865       OI->getOperand(1).getReg() == SrcReg &&
2866       OI->getOperand(2).getImm() == ImmValue) {
2867     IsThumb1 = false;
2868     return true;
2869   }
2870 
2871   if (CmpI->getOpcode() == ARM::tCMPi8 &&
2872       (OI->getOpcode() == ARM::tSUBi8 || OI->getOpcode() == ARM::tSUBi3) &&
2873       OI->getOperand(2).getReg() == SrcReg &&
2874       OI->getOperand(3).getImm() == ImmValue) {
2875     IsThumb1 = true;
2876     return true;
2877   }
2878 
2879   if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2880       (OI->getOpcode() == ARM::ADDrr || OI->getOpcode() == ARM::t2ADDrr ||
2881        OI->getOpcode() == ARM::ADDri || OI->getOpcode() == ARM::t2ADDri) &&
2882       OI->getOperand(0).isReg() && OI->getOperand(1).isReg() &&
2883       OI->getOperand(0).getReg() == SrcReg &&
2884       OI->getOperand(1).getReg() == SrcReg2) {
2885     IsThumb1 = false;
2886     return true;
2887   }
2888 
2889   if (CmpI->getOpcode() == ARM::tCMPr &&
2890       (OI->getOpcode() == ARM::tADDi3 || OI->getOpcode() == ARM::tADDi8 ||
2891        OI->getOpcode() == ARM::tADDrr) &&
2892       OI->getOperand(0).getReg() == SrcReg &&
2893       OI->getOperand(2).getReg() == SrcReg2) {
2894     IsThumb1 = true;
2895     return true;
2896   }
2897 
2898   return false;
2899 }
2900 
2901 static bool isOptimizeCompareCandidate(MachineInstr *MI, bool &IsThumb1) {
2902   switch (MI->getOpcode()) {
2903   default: return false;
2904   case ARM::tLSLri:
2905   case ARM::tLSRri:
2906   case ARM::tLSLrr:
2907   case ARM::tLSRrr:
2908   case ARM::tSUBrr:
2909   case ARM::tADDrr:
2910   case ARM::tADDi3:
2911   case ARM::tADDi8:
2912   case ARM::tSUBi3:
2913   case ARM::tSUBi8:
2914   case ARM::tMUL:
2915   case ARM::tADC:
2916   case ARM::tSBC:
2917   case ARM::tRSB:
2918   case ARM::tAND:
2919   case ARM::tORR:
2920   case ARM::tEOR:
2921   case ARM::tBIC:
2922   case ARM::tMVN:
2923   case ARM::tASRri:
2924   case ARM::tASRrr:
2925   case ARM::tROR:
2926     IsThumb1 = true;
2927     LLVM_FALLTHROUGH;
2928   case ARM::RSBrr:
2929   case ARM::RSBri:
2930   case ARM::RSCrr:
2931   case ARM::RSCri:
2932   case ARM::ADDrr:
2933   case ARM::ADDri:
2934   case ARM::ADCrr:
2935   case ARM::ADCri:
2936   case ARM::SUBrr:
2937   case ARM::SUBri:
2938   case ARM::SBCrr:
2939   case ARM::SBCri:
2940   case ARM::t2RSBri:
2941   case ARM::t2ADDrr:
2942   case ARM::t2ADDri:
2943   case ARM::t2ADCrr:
2944   case ARM::t2ADCri:
2945   case ARM::t2SUBrr:
2946   case ARM::t2SUBri:
2947   case ARM::t2SBCrr:
2948   case ARM::t2SBCri:
2949   case ARM::ANDrr:
2950   case ARM::ANDri:
2951   case ARM::t2ANDrr:
2952   case ARM::t2ANDri:
2953   case ARM::ORRrr:
2954   case ARM::ORRri:
2955   case ARM::t2ORRrr:
2956   case ARM::t2ORRri:
2957   case ARM::EORrr:
2958   case ARM::EORri:
2959   case ARM::t2EORrr:
2960   case ARM::t2EORri:
2961   case ARM::t2LSRri:
2962   case ARM::t2LSRrr:
2963   case ARM::t2LSLri:
2964   case ARM::t2LSLrr:
2965     return true;
2966   }
2967 }
2968 
2969 /// optimizeCompareInstr - Convert the instruction supplying the argument to the
2970 /// comparison into one that sets the zero bit in the flags register;
2971 /// Remove a redundant Compare instruction if an earlier instruction can set the
2972 /// flags in the same way as Compare.
2973 /// E.g. SUBrr(r1,r2) and CMPrr(r1,r2). We also handle the case where two
2974 /// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the
2975 /// condition code of instructions which use the flags.
2976 bool ARMBaseInstrInfo::optimizeCompareInstr(
2977     MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
2978     int64_t CmpValue, const MachineRegisterInfo *MRI) const {
2979   // Get the unique definition of SrcReg.
2980   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2981   if (!MI) return false;
2982 
2983   // Masked compares sometimes use the same register as the corresponding 'and'.
2984   if (CmpMask != ~0) {
2985     if (!isSuitableForMask(MI, SrcReg, CmpMask, false) || isPredicated(*MI)) {
2986       MI = nullptr;
2987       for (MachineRegisterInfo::use_instr_iterator
2988            UI = MRI->use_instr_begin(SrcReg), UE = MRI->use_instr_end();
2989            UI != UE; ++UI) {
2990         if (UI->getParent() != CmpInstr.getParent())
2991           continue;
2992         MachineInstr *PotentialAND = &*UI;
2993         if (!isSuitableForMask(PotentialAND, SrcReg, CmpMask, true) ||
2994             isPredicated(*PotentialAND))
2995           continue;
2996         MI = PotentialAND;
2997         break;
2998       }
2999       if (!MI) return false;
3000     }
3001   }
3002 
3003   // Get ready to iterate backward from CmpInstr.
3004   MachineBasicBlock::iterator I = CmpInstr, E = MI,
3005                               B = CmpInstr.getParent()->begin();
3006 
3007   // Early exit if CmpInstr is at the beginning of the BB.
3008   if (I == B) return false;
3009 
3010   // There are two possible candidates which can be changed to set CPSR:
3011   // One is MI, the other is a SUB or ADD instruction.
3012   // For CMPrr(r1,r2), we are looking for SUB(r1,r2), SUB(r2,r1), or
3013   // ADDr[ri](r1, r2, X).
3014   // For CMPri(r1, CmpValue), we are looking for SUBri(r1, CmpValue).
3015   MachineInstr *SubAdd = nullptr;
3016   if (SrcReg2 != 0)
3017     // MI is not a candidate for CMPrr.
3018     MI = nullptr;
3019   else if (MI->getParent() != CmpInstr.getParent() || CmpValue != 0) {
3020     // Conservatively refuse to convert an instruction which isn't in the same
3021     // BB as the comparison.
3022     // For CMPri w/ CmpValue != 0, a SubAdd may still be a candidate.
3023     // Thus we cannot return here.
3024     if (CmpInstr.getOpcode() == ARM::CMPri ||
3025         CmpInstr.getOpcode() == ARM::t2CMPri ||
3026         CmpInstr.getOpcode() == ARM::tCMPi8)
3027       MI = nullptr;
3028     else
3029       return false;
3030   }
3031 
3032   bool IsThumb1 = false;
3033   if (MI && !isOptimizeCompareCandidate(MI, IsThumb1))
3034     return false;
3035 
3036   // We also want to do this peephole for cases like this: if (a*b == 0),
3037   // and optimise away the CMP instruction from the generated code sequence:
3038   // MULS, MOVS, MOVS, CMP. Here the MOVS instructions load the boolean values
3039   // resulting from the select instruction, but these MOVS instructions for
3040   // Thumb1 (V6M) are flag setting and are thus preventing this optimisation.
3041   // However, if we only have MOVS instructions in between the CMP and the
3042   // other instruction (the MULS in this example), then the CPSR is dead so we
3043   // can safely reorder the sequence into: MOVS, MOVS, MULS, CMP. We do this
3044   // reordering and then continue the analysis hoping we can eliminate the
3045   // CMP. This peephole works on the vregs, so is still in SSA form. As a
3046   // consequence, the movs won't redefine/kill the MUL operands which would
3047   // make this reordering illegal.
3048   const TargetRegisterInfo *TRI = &getRegisterInfo();
3049   if (MI && IsThumb1) {
3050     --I;
3051     if (I != E && !MI->readsRegister(ARM::CPSR, TRI)) {
3052       bool CanReorder = true;
3053       for (; I != E; --I) {
3054         if (I->getOpcode() != ARM::tMOVi8) {
3055           CanReorder = false;
3056           break;
3057         }
3058       }
3059       if (CanReorder) {
3060         MI = MI->removeFromParent();
3061         E = CmpInstr;
3062         CmpInstr.getParent()->insert(E, MI);
3063       }
3064     }
3065     I = CmpInstr;
3066     E = MI;
3067   }
3068 
3069   // Check that CPSR isn't set between the comparison instruction and the one we
3070   // want to change. At the same time, search for SubAdd.
3071   bool SubAddIsThumb1 = false;
3072   do {
3073     const MachineInstr &Instr = *--I;
3074 
3075     // Check whether CmpInstr can be made redundant by the current instruction.
3076     if (isRedundantFlagInstr(&CmpInstr, SrcReg, SrcReg2, CmpValue, &Instr,
3077                              SubAddIsThumb1)) {
3078       SubAdd = &*I;
3079       break;
3080     }
3081 
3082     // Allow E (which was initially MI) to be SubAdd but do not search before E.
3083     if (I == E)
3084       break;
3085 
3086     if (Instr.modifiesRegister(ARM::CPSR, TRI) ||
3087         Instr.readsRegister(ARM::CPSR, TRI))
3088       // This instruction modifies or uses CPSR after the one we want to
3089       // change. We can't do this transformation.
3090       return false;
3091 
3092     if (I == B) {
3093       // In some cases, we scan the use-list of an instruction for an AND;
3094       // that AND is in the same BB, but may not be scheduled before the
3095       // corresponding TST.  In that case, bail out.
3096       //
3097       // FIXME: We could try to reschedule the AND.
3098       return false;
3099     }
3100   } while (true);
3101 
3102   // Return false if no candidates exist.
3103   if (!MI && !SubAdd)
3104     return false;
3105 
3106   // If we found a SubAdd, use it as it will be closer to the CMP
3107   if (SubAdd) {
3108     MI = SubAdd;
3109     IsThumb1 = SubAddIsThumb1;
3110   }
3111 
3112   // We can't use a predicated instruction - it doesn't always write the flags.
3113   if (isPredicated(*MI))
3114     return false;
3115 
3116   // Scan forward for the use of CPSR
3117   // When checking against MI: if it's a conditional code that requires
3118   // checking of the V bit or C bit, then this is not safe to do.
3119   // It is safe to remove CmpInstr if CPSR is redefined or killed.
3120   // If we are done with the basic block, we need to check whether CPSR is
3121   // live-out.
3122   SmallVector<std::pair<MachineOperand*, ARMCC::CondCodes>, 4>
3123       OperandsToUpdate;
3124   bool isSafe = false;
3125   I = CmpInstr;
3126   E = CmpInstr.getParent()->end();
3127   while (!isSafe && ++I != E) {
3128     const MachineInstr &Instr = *I;
3129     for (unsigned IO = 0, EO = Instr.getNumOperands();
3130          !isSafe && IO != EO; ++IO) {
3131       const MachineOperand &MO = Instr.getOperand(IO);
3132       if (MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) {
3133         isSafe = true;
3134         break;
3135       }
3136       if (!MO.isReg() || MO.getReg() != ARM::CPSR)
3137         continue;
3138       if (MO.isDef()) {
3139         isSafe = true;
3140         break;
3141       }
3142       // Condition code is after the operand before CPSR except for VSELs.
3143       ARMCC::CondCodes CC;
3144       bool IsInstrVSel = true;
3145       switch (Instr.getOpcode()) {
3146       default:
3147         IsInstrVSel = false;
3148         CC = (ARMCC::CondCodes)Instr.getOperand(IO - 1).getImm();
3149         break;
3150       case ARM::VSELEQD:
3151       case ARM::VSELEQS:
3152       case ARM::VSELEQH:
3153         CC = ARMCC::EQ;
3154         break;
3155       case ARM::VSELGTD:
3156       case ARM::VSELGTS:
3157       case ARM::VSELGTH:
3158         CC = ARMCC::GT;
3159         break;
3160       case ARM::VSELGED:
3161       case ARM::VSELGES:
3162       case ARM::VSELGEH:
3163         CC = ARMCC::GE;
3164         break;
3165       case ARM::VSELVSD:
3166       case ARM::VSELVSS:
3167       case ARM::VSELVSH:
3168         CC = ARMCC::VS;
3169         break;
3170       }
3171 
3172       if (SubAdd) {
3173         // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based
3174         // on CMP needs to be updated to be based on SUB.
3175         // If we have ADD(r1, r2, X) and CMP(r1, r2), the condition code also
3176         // needs to be modified.
3177         // Push the condition code operands to OperandsToUpdate.
3178         // If it is safe to remove CmpInstr, the condition code of these
3179         // operands will be modified.
3180         unsigned Opc = SubAdd->getOpcode();
3181         bool IsSub = Opc == ARM::SUBrr || Opc == ARM::t2SUBrr ||
3182                      Opc == ARM::SUBri || Opc == ARM::t2SUBri ||
3183                      Opc == ARM::tSUBrr || Opc == ARM::tSUBi3 ||
3184                      Opc == ARM::tSUBi8;
3185         unsigned OpI = Opc != ARM::tSUBrr ? 1 : 2;
3186         if (!IsSub ||
3187             (SrcReg2 != 0 && SubAdd->getOperand(OpI).getReg() == SrcReg2 &&
3188              SubAdd->getOperand(OpI + 1).getReg() == SrcReg)) {
3189           // VSel doesn't support condition code update.
3190           if (IsInstrVSel)
3191             return false;
3192           // Ensure we can swap the condition.
3193           ARMCC::CondCodes NewCC = (IsSub ? getSwappedCondition(CC) : getCmpToAddCondition(CC));
3194           if (NewCC == ARMCC::AL)
3195             return false;
3196           OperandsToUpdate.push_back(
3197               std::make_pair(&((*I).getOperand(IO - 1)), NewCC));
3198         }
3199       } else {
3200         // No SubAdd, so this is x = <op> y, z; cmp x, 0.
3201         switch (CC) {
3202         case ARMCC::EQ: // Z
3203         case ARMCC::NE: // Z
3204         case ARMCC::MI: // N
3205         case ARMCC::PL: // N
3206         case ARMCC::AL: // none
3207           // CPSR can be used multiple times, we should continue.
3208           break;
3209         case ARMCC::HS: // C
3210         case ARMCC::LO: // C
3211         case ARMCC::VS: // V
3212         case ARMCC::VC: // V
3213         case ARMCC::HI: // C Z
3214         case ARMCC::LS: // C Z
3215         case ARMCC::GE: // N V
3216         case ARMCC::LT: // N V
3217         case ARMCC::GT: // Z N V
3218         case ARMCC::LE: // Z N V
3219           // The instruction uses the V bit or C bit which is not safe.
3220           return false;
3221         }
3222       }
3223     }
3224   }
3225 
3226   // If CPSR is not killed nor re-defined, we should check whether it is
3227   // live-out. If it is live-out, do not optimize.
3228   if (!isSafe) {
3229     MachineBasicBlock *MBB = CmpInstr.getParent();
3230     for (MachineBasicBlock *Succ : MBB->successors())
3231       if (Succ->isLiveIn(ARM::CPSR))
3232         return false;
3233   }
3234 
3235   // Toggle the optional operand to CPSR (if it exists - in Thumb1 we always
3236   // set CPSR so this is represented as an explicit output)
3237   if (!IsThumb1) {
3238     MI->getOperand(5).setReg(ARM::CPSR);
3239     MI->getOperand(5).setIsDef(true);
3240   }
3241   assert(!isPredicated(*MI) && "Can't use flags from predicated instruction");
3242   CmpInstr.eraseFromParent();
3243 
3244   // Modify the condition code of operands in OperandsToUpdate.
3245   // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
3246   // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
3247   for (unsigned i = 0, e = OperandsToUpdate.size(); i < e; i++)
3248     OperandsToUpdate[i].first->setImm(OperandsToUpdate[i].second);
3249 
3250   MI->clearRegisterDeads(ARM::CPSR);
3251 
3252   return true;
3253 }
3254 
3255 bool ARMBaseInstrInfo::shouldSink(const MachineInstr &MI) const {
3256   // Do not sink MI if it might be used to optimize a redundant compare.
3257   // We heuristically only look at the instruction immediately following MI to
3258   // avoid potentially searching the entire basic block.
3259   if (isPredicated(MI))
3260     return true;
3261   MachineBasicBlock::const_iterator Next = &MI;
3262   ++Next;
3263   Register SrcReg, SrcReg2;
3264   int64_t CmpMask, CmpValue;
3265   bool IsThumb1;
3266   if (Next != MI.getParent()->end() &&
3267       analyzeCompare(*Next, SrcReg, SrcReg2, CmpMask, CmpValue) &&
3268       isRedundantFlagInstr(&*Next, SrcReg, SrcReg2, CmpValue, &MI, IsThumb1))
3269     return false;
3270   return true;
3271 }
3272 
3273 bool ARMBaseInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3274                                      Register Reg,
3275                                      MachineRegisterInfo *MRI) const {
3276   // Fold large immediates into add, sub, or, xor.
3277   unsigned DefOpc = DefMI.getOpcode();
3278   if (DefOpc != ARM::t2MOVi32imm && DefOpc != ARM::MOVi32imm)
3279     return false;
3280   if (!DefMI.getOperand(1).isImm())
3281     // Could be t2MOVi32imm @xx
3282     return false;
3283 
3284   if (!MRI->hasOneNonDBGUse(Reg))
3285     return false;
3286 
3287   const MCInstrDesc &DefMCID = DefMI.getDesc();
3288   if (DefMCID.hasOptionalDef()) {
3289     unsigned NumOps = DefMCID.getNumOperands();
3290     const MachineOperand &MO = DefMI.getOperand(NumOps - 1);
3291     if (MO.getReg() == ARM::CPSR && !MO.isDead())
3292       // If DefMI defines CPSR and it is not dead, it's obviously not safe
3293       // to delete DefMI.
3294       return false;
3295   }
3296 
3297   const MCInstrDesc &UseMCID = UseMI.getDesc();
3298   if (UseMCID.hasOptionalDef()) {
3299     unsigned NumOps = UseMCID.getNumOperands();
3300     if (UseMI.getOperand(NumOps - 1).getReg() == ARM::CPSR)
3301       // If the instruction sets the flag, do not attempt this optimization
3302       // since it may change the semantics of the code.
3303       return false;
3304   }
3305 
3306   unsigned UseOpc = UseMI.getOpcode();
3307   unsigned NewUseOpc = 0;
3308   uint32_t ImmVal = (uint32_t)DefMI.getOperand(1).getImm();
3309   uint32_t SOImmValV1 = 0, SOImmValV2 = 0;
3310   bool Commute = false;
3311   switch (UseOpc) {
3312   default: return false;
3313   case ARM::SUBrr:
3314   case ARM::ADDrr:
3315   case ARM::ORRrr:
3316   case ARM::EORrr:
3317   case ARM::t2SUBrr:
3318   case ARM::t2ADDrr:
3319   case ARM::t2ORRrr:
3320   case ARM::t2EORrr: {
3321     Commute = UseMI.getOperand(2).getReg() != Reg;
3322     switch (UseOpc) {
3323     default: break;
3324     case ARM::ADDrr:
3325     case ARM::SUBrr:
3326       if (UseOpc == ARM::SUBrr && Commute)
3327         return false;
3328 
3329       // ADD/SUB are special because they're essentially the same operation, so
3330       // we can handle a larger range of immediates.
3331       if (ARM_AM::isSOImmTwoPartVal(ImmVal))
3332         NewUseOpc = UseOpc == ARM::ADDrr ? ARM::ADDri : ARM::SUBri;
3333       else if (ARM_AM::isSOImmTwoPartVal(-ImmVal)) {
3334         ImmVal = -ImmVal;
3335         NewUseOpc = UseOpc == ARM::ADDrr ? ARM::SUBri : ARM::ADDri;
3336       } else
3337         return false;
3338       SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal);
3339       SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal);
3340       break;
3341     case ARM::ORRrr:
3342     case ARM::EORrr:
3343       if (!ARM_AM::isSOImmTwoPartVal(ImmVal))
3344         return false;
3345       SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal);
3346       SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal);
3347       switch (UseOpc) {
3348       default: break;
3349       case ARM::ORRrr: NewUseOpc = ARM::ORRri; break;
3350       case ARM::EORrr: NewUseOpc = ARM::EORri; break;
3351       }
3352       break;
3353     case ARM::t2ADDrr:
3354     case ARM::t2SUBrr: {
3355       if (UseOpc == ARM::t2SUBrr && Commute)
3356         return false;
3357 
3358       // ADD/SUB are special because they're essentially the same operation, so
3359       // we can handle a larger range of immediates.
3360       const bool ToSP = DefMI.getOperand(0).getReg() == ARM::SP;
3361       const unsigned t2ADD = ToSP ? ARM::t2ADDspImm : ARM::t2ADDri;
3362       const unsigned t2SUB = ToSP ? ARM::t2SUBspImm : ARM::t2SUBri;
3363       if (ARM_AM::isT2SOImmTwoPartVal(ImmVal))
3364         NewUseOpc = UseOpc == ARM::t2ADDrr ? t2ADD : t2SUB;
3365       else if (ARM_AM::isT2SOImmTwoPartVal(-ImmVal)) {
3366         ImmVal = -ImmVal;
3367         NewUseOpc = UseOpc == ARM::t2ADDrr ? t2SUB : t2ADD;
3368       } else
3369         return false;
3370       SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal);
3371       SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal);
3372       break;
3373     }
3374     case ARM::t2ORRrr:
3375     case ARM::t2EORrr:
3376       if (!ARM_AM::isT2SOImmTwoPartVal(ImmVal))
3377         return false;
3378       SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal);
3379       SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal);
3380       switch (UseOpc) {
3381       default: break;
3382       case ARM::t2ORRrr: NewUseOpc = ARM::t2ORRri; break;
3383       case ARM::t2EORrr: NewUseOpc = ARM::t2EORri; break;
3384       }
3385       break;
3386     }
3387   }
3388   }
3389 
3390   unsigned OpIdx = Commute ? 2 : 1;
3391   Register Reg1 = UseMI.getOperand(OpIdx).getReg();
3392   bool isKill = UseMI.getOperand(OpIdx).isKill();
3393   const TargetRegisterClass *TRC = MRI->getRegClass(Reg);
3394   Register NewReg = MRI->createVirtualRegister(TRC);
3395   BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), get(NewUseOpc),
3396           NewReg)
3397       .addReg(Reg1, getKillRegState(isKill))
3398       .addImm(SOImmValV1)
3399       .add(predOps(ARMCC::AL))
3400       .add(condCodeOp());
3401   UseMI.setDesc(get(NewUseOpc));
3402   UseMI.getOperand(1).setReg(NewReg);
3403   UseMI.getOperand(1).setIsKill();
3404   UseMI.getOperand(2).ChangeToImmediate(SOImmValV2);
3405   DefMI.eraseFromParent();
3406   // FIXME: t2ADDrr should be split, as different rulles apply when writing to SP.
3407   // Just as t2ADDri, that was split to [t2ADDri, t2ADDspImm].
3408   // Then the below code will not be needed, as the input/output register
3409   // classes will be rgpr or gprSP.
3410   // For now, we fix the UseMI operand explicitly here:
3411   switch(NewUseOpc){
3412     case ARM::t2ADDspImm:
3413     case ARM::t2SUBspImm:
3414     case ARM::t2ADDri:
3415     case ARM::t2SUBri:
3416       MRI->constrainRegClass(UseMI.getOperand(0).getReg(), TRC);
3417   }
3418   return true;
3419 }
3420 
3421 static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData,
3422                                         const MachineInstr &MI) {
3423   switch (MI.getOpcode()) {
3424   default: {
3425     const MCInstrDesc &Desc = MI.getDesc();
3426     int UOps = ItinData->getNumMicroOps(Desc.getSchedClass());
3427     assert(UOps >= 0 && "bad # UOps");
3428     return UOps;
3429   }
3430 
3431   case ARM::LDRrs:
3432   case ARM::LDRBrs:
3433   case ARM::STRrs:
3434   case ARM::STRBrs: {
3435     unsigned ShOpVal = MI.getOperand(3).getImm();
3436     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3437     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3438     if (!isSub &&
3439         (ShImm == 0 ||
3440          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3441           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3442       return 1;
3443     return 2;
3444   }
3445 
3446   case ARM::LDRH:
3447   case ARM::STRH: {
3448     if (!MI.getOperand(2).getReg())
3449       return 1;
3450 
3451     unsigned ShOpVal = MI.getOperand(3).getImm();
3452     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3453     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3454     if (!isSub &&
3455         (ShImm == 0 ||
3456          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3457           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3458       return 1;
3459     return 2;
3460   }
3461 
3462   case ARM::LDRSB:
3463   case ARM::LDRSH:
3464     return (ARM_AM::getAM3Op(MI.getOperand(3).getImm()) == ARM_AM::sub) ? 3 : 2;
3465 
3466   case ARM::LDRSB_POST:
3467   case ARM::LDRSH_POST: {
3468     Register Rt = MI.getOperand(0).getReg();
3469     Register Rm = MI.getOperand(3).getReg();
3470     return (Rt == Rm) ? 4 : 3;
3471   }
3472 
3473   case ARM::LDR_PRE_REG:
3474   case ARM::LDRB_PRE_REG: {
3475     Register Rt = MI.getOperand(0).getReg();
3476     Register Rm = MI.getOperand(3).getReg();
3477     if (Rt == Rm)
3478       return 3;
3479     unsigned ShOpVal = MI.getOperand(4).getImm();
3480     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3481     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3482     if (!isSub &&
3483         (ShImm == 0 ||
3484          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3485           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3486       return 2;
3487     return 3;
3488   }
3489 
3490   case ARM::STR_PRE_REG:
3491   case ARM::STRB_PRE_REG: {
3492     unsigned ShOpVal = MI.getOperand(4).getImm();
3493     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3494     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3495     if (!isSub &&
3496         (ShImm == 0 ||
3497          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3498           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3499       return 2;
3500     return 3;
3501   }
3502 
3503   case ARM::LDRH_PRE:
3504   case ARM::STRH_PRE: {
3505     Register Rt = MI.getOperand(0).getReg();
3506     Register Rm = MI.getOperand(3).getReg();
3507     if (!Rm)
3508       return 2;
3509     if (Rt == Rm)
3510       return 3;
3511     return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 3 : 2;
3512   }
3513 
3514   case ARM::LDR_POST_REG:
3515   case ARM::LDRB_POST_REG:
3516   case ARM::LDRH_POST: {
3517     Register Rt = MI.getOperand(0).getReg();
3518     Register Rm = MI.getOperand(3).getReg();
3519     return (Rt == Rm) ? 3 : 2;
3520   }
3521 
3522   case ARM::LDR_PRE_IMM:
3523   case ARM::LDRB_PRE_IMM:
3524   case ARM::LDR_POST_IMM:
3525   case ARM::LDRB_POST_IMM:
3526   case ARM::STRB_POST_IMM:
3527   case ARM::STRB_POST_REG:
3528   case ARM::STRB_PRE_IMM:
3529   case ARM::STRH_POST:
3530   case ARM::STR_POST_IMM:
3531   case ARM::STR_POST_REG:
3532   case ARM::STR_PRE_IMM:
3533     return 2;
3534 
3535   case ARM::LDRSB_PRE:
3536   case ARM::LDRSH_PRE: {
3537     Register Rm = MI.getOperand(3).getReg();
3538     if (Rm == 0)
3539       return 3;
3540     Register Rt = MI.getOperand(0).getReg();
3541     if (Rt == Rm)
3542       return 4;
3543     unsigned ShOpVal = MI.getOperand(4).getImm();
3544     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3545     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3546     if (!isSub &&
3547         (ShImm == 0 ||
3548          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3549           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3550       return 3;
3551     return 4;
3552   }
3553 
3554   case ARM::LDRD: {
3555     Register Rt = MI.getOperand(0).getReg();
3556     Register Rn = MI.getOperand(2).getReg();
3557     Register Rm = MI.getOperand(3).getReg();
3558     if (Rm)
3559       return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4
3560                                                                           : 3;
3561     return (Rt == Rn) ? 3 : 2;
3562   }
3563 
3564   case ARM::STRD: {
3565     Register Rm = MI.getOperand(3).getReg();
3566     if (Rm)
3567       return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4
3568                                                                           : 3;
3569     return 2;
3570   }
3571 
3572   case ARM::LDRD_POST:
3573   case ARM::t2LDRD_POST:
3574     return 3;
3575 
3576   case ARM::STRD_POST:
3577   case ARM::t2STRD_POST:
3578     return 4;
3579 
3580   case ARM::LDRD_PRE: {
3581     Register Rt = MI.getOperand(0).getReg();
3582     Register Rn = MI.getOperand(3).getReg();
3583     Register Rm = MI.getOperand(4).getReg();
3584     if (Rm)
3585       return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5
3586                                                                           : 4;
3587     return (Rt == Rn) ? 4 : 3;
3588   }
3589 
3590   case ARM::t2LDRD_PRE: {
3591     Register Rt = MI.getOperand(0).getReg();
3592     Register Rn = MI.getOperand(3).getReg();
3593     return (Rt == Rn) ? 4 : 3;
3594   }
3595 
3596   case ARM::STRD_PRE: {
3597     Register Rm = MI.getOperand(4).getReg();
3598     if (Rm)
3599       return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5
3600                                                                           : 4;
3601     return 3;
3602   }
3603 
3604   case ARM::t2STRD_PRE:
3605     return 3;
3606 
3607   case ARM::t2LDR_POST:
3608   case ARM::t2LDRB_POST:
3609   case ARM::t2LDRB_PRE:
3610   case ARM::t2LDRSBi12:
3611   case ARM::t2LDRSBi8:
3612   case ARM::t2LDRSBpci:
3613   case ARM::t2LDRSBs:
3614   case ARM::t2LDRH_POST:
3615   case ARM::t2LDRH_PRE:
3616   case ARM::t2LDRSBT:
3617   case ARM::t2LDRSB_POST:
3618   case ARM::t2LDRSB_PRE:
3619   case ARM::t2LDRSH_POST:
3620   case ARM::t2LDRSH_PRE:
3621   case ARM::t2LDRSHi12:
3622   case ARM::t2LDRSHi8:
3623   case ARM::t2LDRSHpci:
3624   case ARM::t2LDRSHs:
3625     return 2;
3626 
3627   case ARM::t2LDRDi8: {
3628     Register Rt = MI.getOperand(0).getReg();
3629     Register Rn = MI.getOperand(2).getReg();
3630     return (Rt == Rn) ? 3 : 2;
3631   }
3632 
3633   case ARM::t2STRB_POST:
3634   case ARM::t2STRB_PRE:
3635   case ARM::t2STRBs:
3636   case ARM::t2STRDi8:
3637   case ARM::t2STRH_POST:
3638   case ARM::t2STRH_PRE:
3639   case ARM::t2STRHs:
3640   case ARM::t2STR_POST:
3641   case ARM::t2STR_PRE:
3642   case ARM::t2STRs:
3643     return 2;
3644   }
3645 }
3646 
3647 // Return the number of 32-bit words loaded by LDM or stored by STM. If this
3648 // can't be easily determined return 0 (missing MachineMemOperand).
3649 //
3650 // FIXME: The current MachineInstr design does not support relying on machine
3651 // mem operands to determine the width of a memory access. Instead, we expect
3652 // the target to provide this information based on the instruction opcode and
3653 // operands. However, using MachineMemOperand is the best solution now for
3654 // two reasons:
3655 //
3656 // 1) getNumMicroOps tries to infer LDM memory width from the total number of MI
3657 // operands. This is much more dangerous than using the MachineMemOperand
3658 // sizes because CodeGen passes can insert/remove optional machine operands. In
3659 // fact, it's totally incorrect for preRA passes and appears to be wrong for
3660 // postRA passes as well.
3661 //
3662 // 2) getNumLDMAddresses is only used by the scheduling machine model and any
3663 // machine model that calls this should handle the unknown (zero size) case.
3664 //
3665 // Long term, we should require a target hook that verifies MachineMemOperand
3666 // sizes during MC lowering. That target hook should be local to MC lowering
3667 // because we can't ensure that it is aware of other MI forms. Doing this will
3668 // ensure that MachineMemOperands are correctly propagated through all passes.
3669 unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr &MI) const {
3670   unsigned Size = 0;
3671   for (MachineInstr::mmo_iterator I = MI.memoperands_begin(),
3672                                   E = MI.memoperands_end();
3673        I != E; ++I) {
3674     Size += (*I)->getSize();
3675   }
3676   // FIXME: The scheduler currently can't handle values larger than 16. But
3677   // the values can actually go up to 32 for floating-point load/store
3678   // multiple (VLDMIA etc.). Also, the way this code is reasoning about memory
3679   // operations isn't right; we could end up with "extra" memory operands for
3680   // various reasons, like tail merge merging two memory operations.
3681   return std::min(Size / 4, 16U);
3682 }
3683 
3684 static unsigned getNumMicroOpsSingleIssuePlusExtras(unsigned Opc,
3685                                                     unsigned NumRegs) {
3686   unsigned UOps = 1 + NumRegs; // 1 for address computation.
3687   switch (Opc) {
3688   default:
3689     break;
3690   case ARM::VLDMDIA_UPD:
3691   case ARM::VLDMDDB_UPD:
3692   case ARM::VLDMSIA_UPD:
3693   case ARM::VLDMSDB_UPD:
3694   case ARM::VSTMDIA_UPD:
3695   case ARM::VSTMDDB_UPD:
3696   case ARM::VSTMSIA_UPD:
3697   case ARM::VSTMSDB_UPD:
3698   case ARM::LDMIA_UPD:
3699   case ARM::LDMDA_UPD:
3700   case ARM::LDMDB_UPD:
3701   case ARM::LDMIB_UPD:
3702   case ARM::STMIA_UPD:
3703   case ARM::STMDA_UPD:
3704   case ARM::STMDB_UPD:
3705   case ARM::STMIB_UPD:
3706   case ARM::tLDMIA_UPD:
3707   case ARM::tSTMIA_UPD:
3708   case ARM::t2LDMIA_UPD:
3709   case ARM::t2LDMDB_UPD:
3710   case ARM::t2STMIA_UPD:
3711   case ARM::t2STMDB_UPD:
3712     ++UOps; // One for base register writeback.
3713     break;
3714   case ARM::LDMIA_RET:
3715   case ARM::tPOP_RET:
3716   case ARM::t2LDMIA_RET:
3717     UOps += 2; // One for base reg wb, one for write to pc.
3718     break;
3719   }
3720   return UOps;
3721 }
3722 
3723 unsigned ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
3724                                           const MachineInstr &MI) const {
3725   if (!ItinData || ItinData->isEmpty())
3726     return 1;
3727 
3728   const MCInstrDesc &Desc = MI.getDesc();
3729   unsigned Class = Desc.getSchedClass();
3730   int ItinUOps = ItinData->getNumMicroOps(Class);
3731   if (ItinUOps >= 0) {
3732     if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore()))
3733       return getNumMicroOpsSwiftLdSt(ItinData, MI);
3734 
3735     return ItinUOps;
3736   }
3737 
3738   unsigned Opc = MI.getOpcode();
3739   switch (Opc) {
3740   default:
3741     llvm_unreachable("Unexpected multi-uops instruction!");
3742   case ARM::VLDMQIA:
3743   case ARM::VSTMQIA:
3744     return 2;
3745 
3746   // The number of uOps for load / store multiple are determined by the number
3747   // registers.
3748   //
3749   // On Cortex-A8, each pair of register loads / stores can be scheduled on the
3750   // same cycle. The scheduling for the first load / store must be done
3751   // separately by assuming the address is not 64-bit aligned.
3752   //
3753   // On Cortex-A9, the formula is simply (#reg / 2) + (#reg % 2). If the address
3754   // is not 64-bit aligned, then AGU would take an extra cycle.  For VFP / NEON
3755   // load / store multiple, the formula is (#reg / 2) + (#reg % 2) + 1.
3756   case ARM::VLDMDIA:
3757   case ARM::VLDMDIA_UPD:
3758   case ARM::VLDMDDB_UPD:
3759   case ARM::VLDMSIA:
3760   case ARM::VLDMSIA_UPD:
3761   case ARM::VLDMSDB_UPD:
3762   case ARM::VSTMDIA:
3763   case ARM::VSTMDIA_UPD:
3764   case ARM::VSTMDDB_UPD:
3765   case ARM::VSTMSIA:
3766   case ARM::VSTMSIA_UPD:
3767   case ARM::VSTMSDB_UPD: {
3768     unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands();
3769     return (NumRegs / 2) + (NumRegs % 2) + 1;
3770   }
3771 
3772   case ARM::LDMIA_RET:
3773   case ARM::LDMIA:
3774   case ARM::LDMDA:
3775   case ARM::LDMDB:
3776   case ARM::LDMIB:
3777   case ARM::LDMIA_UPD:
3778   case ARM::LDMDA_UPD:
3779   case ARM::LDMDB_UPD:
3780   case ARM::LDMIB_UPD:
3781   case ARM::STMIA:
3782   case ARM::STMDA:
3783   case ARM::STMDB:
3784   case ARM::STMIB:
3785   case ARM::STMIA_UPD:
3786   case ARM::STMDA_UPD:
3787   case ARM::STMDB_UPD:
3788   case ARM::STMIB_UPD:
3789   case ARM::tLDMIA:
3790   case ARM::tLDMIA_UPD:
3791   case ARM::tSTMIA_UPD:
3792   case ARM::tPOP_RET:
3793   case ARM::tPOP:
3794   case ARM::tPUSH:
3795   case ARM::t2LDMIA_RET:
3796   case ARM::t2LDMIA:
3797   case ARM::t2LDMDB:
3798   case ARM::t2LDMIA_UPD:
3799   case ARM::t2LDMDB_UPD:
3800   case ARM::t2STMIA:
3801   case ARM::t2STMDB:
3802   case ARM::t2STMIA_UPD:
3803   case ARM::t2STMDB_UPD: {
3804     unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands() + 1;
3805     switch (Subtarget.getLdStMultipleTiming()) {
3806     case ARMSubtarget::SingleIssuePlusExtras:
3807       return getNumMicroOpsSingleIssuePlusExtras(Opc, NumRegs);
3808     case ARMSubtarget::SingleIssue:
3809       // Assume the worst.
3810       return NumRegs;
3811     case ARMSubtarget::DoubleIssue: {
3812       if (NumRegs < 4)
3813         return 2;
3814       // 4 registers would be issued: 2, 2.
3815       // 5 registers would be issued: 2, 2, 1.
3816       unsigned UOps = (NumRegs / 2);
3817       if (NumRegs % 2)
3818         ++UOps;
3819       return UOps;
3820     }
3821     case ARMSubtarget::DoubleIssueCheckUnalignedAccess: {
3822       unsigned UOps = (NumRegs / 2);
3823       // If there are odd number of registers or if it's not 64-bit aligned,
3824       // then it takes an extra AGU (Address Generation Unit) cycle.
3825       if ((NumRegs % 2) || !MI.hasOneMemOperand() ||
3826           (*MI.memoperands_begin())->getAlign() < Align(8))
3827         ++UOps;
3828       return UOps;
3829       }
3830     }
3831   }
3832   }
3833   llvm_unreachable("Didn't find the number of microops");
3834 }
3835 
3836 int
3837 ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData,
3838                                   const MCInstrDesc &DefMCID,
3839                                   unsigned DefClass,
3840                                   unsigned DefIdx, unsigned DefAlign) const {
3841   int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3842   if (RegNo <= 0)
3843     // Def is the address writeback.
3844     return ItinData->getOperandCycle(DefClass, DefIdx);
3845 
3846   int DefCycle;
3847   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3848     // (regno / 2) + (regno % 2) + 1
3849     DefCycle = RegNo / 2 + 1;
3850     if (RegNo % 2)
3851       ++DefCycle;
3852   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3853     DefCycle = RegNo;
3854     bool isSLoad = false;
3855 
3856     switch (DefMCID.getOpcode()) {
3857     default: break;
3858     case ARM::VLDMSIA:
3859     case ARM::VLDMSIA_UPD:
3860     case ARM::VLDMSDB_UPD:
3861       isSLoad = true;
3862       break;
3863     }
3864 
3865     // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3866     // then it takes an extra cycle.
3867     if ((isSLoad && (RegNo % 2)) || DefAlign < 8)
3868       ++DefCycle;
3869   } else {
3870     // Assume the worst.
3871     DefCycle = RegNo + 2;
3872   }
3873 
3874   return DefCycle;
3875 }
3876 
3877 int
3878 ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData,
3879                                  const MCInstrDesc &DefMCID,
3880                                  unsigned DefClass,
3881                                  unsigned DefIdx, unsigned DefAlign) const {
3882   int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3883   if (RegNo <= 0)
3884     // Def is the address writeback.
3885     return ItinData->getOperandCycle(DefClass, DefIdx);
3886 
3887   int DefCycle;
3888   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3889     // 4 registers would be issued: 1, 2, 1.
3890     // 5 registers would be issued: 1, 2, 2.
3891     DefCycle = RegNo / 2;
3892     if (DefCycle < 1)
3893       DefCycle = 1;
3894     // Result latency is issue cycle + 2: E2.
3895     DefCycle += 2;
3896   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3897     DefCycle = (RegNo / 2);
3898     // If there are odd number of registers or if it's not 64-bit aligned,
3899     // then it takes an extra AGU (Address Generation Unit) cycle.
3900     if ((RegNo % 2) || DefAlign < 8)
3901       ++DefCycle;
3902     // Result latency is AGU cycles + 2.
3903     DefCycle += 2;
3904   } else {
3905     // Assume the worst.
3906     DefCycle = RegNo + 2;
3907   }
3908 
3909   return DefCycle;
3910 }
3911 
3912 int
3913 ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData,
3914                                   const MCInstrDesc &UseMCID,
3915                                   unsigned UseClass,
3916                                   unsigned UseIdx, unsigned UseAlign) const {
3917   int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3918   if (RegNo <= 0)
3919     return ItinData->getOperandCycle(UseClass, UseIdx);
3920 
3921   int UseCycle;
3922   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3923     // (regno / 2) + (regno % 2) + 1
3924     UseCycle = RegNo / 2 + 1;
3925     if (RegNo % 2)
3926       ++UseCycle;
3927   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3928     UseCycle = RegNo;
3929     bool isSStore = false;
3930 
3931     switch (UseMCID.getOpcode()) {
3932     default: break;
3933     case ARM::VSTMSIA:
3934     case ARM::VSTMSIA_UPD:
3935     case ARM::VSTMSDB_UPD:
3936       isSStore = true;
3937       break;
3938     }
3939 
3940     // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3941     // then it takes an extra cycle.
3942     if ((isSStore && (RegNo % 2)) || UseAlign < 8)
3943       ++UseCycle;
3944   } else {
3945     // Assume the worst.
3946     UseCycle = RegNo + 2;
3947   }
3948 
3949   return UseCycle;
3950 }
3951 
3952 int
3953 ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData,
3954                                  const MCInstrDesc &UseMCID,
3955                                  unsigned UseClass,
3956                                  unsigned UseIdx, unsigned UseAlign) const {
3957   int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3958   if (RegNo <= 0)
3959     return ItinData->getOperandCycle(UseClass, UseIdx);
3960 
3961   int UseCycle;
3962   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3963     UseCycle = RegNo / 2;
3964     if (UseCycle < 2)
3965       UseCycle = 2;
3966     // Read in E3.
3967     UseCycle += 2;
3968   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3969     UseCycle = (RegNo / 2);
3970     // If there are odd number of registers or if it's not 64-bit aligned,
3971     // then it takes an extra AGU (Address Generation Unit) cycle.
3972     if ((RegNo % 2) || UseAlign < 8)
3973       ++UseCycle;
3974   } else {
3975     // Assume the worst.
3976     UseCycle = 1;
3977   }
3978   return UseCycle;
3979 }
3980 
3981 int
3982 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
3983                                     const MCInstrDesc &DefMCID,
3984                                     unsigned DefIdx, unsigned DefAlign,
3985                                     const MCInstrDesc &UseMCID,
3986                                     unsigned UseIdx, unsigned UseAlign) const {
3987   unsigned DefClass = DefMCID.getSchedClass();
3988   unsigned UseClass = UseMCID.getSchedClass();
3989 
3990   if (DefIdx < DefMCID.getNumDefs() && UseIdx < UseMCID.getNumOperands())
3991     return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
3992 
3993   // This may be a def / use of a variable_ops instruction, the operand
3994   // latency might be determinable dynamically. Let the target try to
3995   // figure it out.
3996   int DefCycle = -1;
3997   bool LdmBypass = false;
3998   switch (DefMCID.getOpcode()) {
3999   default:
4000     DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
4001     break;
4002 
4003   case ARM::VLDMDIA:
4004   case ARM::VLDMDIA_UPD:
4005   case ARM::VLDMDDB_UPD:
4006   case ARM::VLDMSIA:
4007   case ARM::VLDMSIA_UPD:
4008   case ARM::VLDMSDB_UPD:
4009     DefCycle = getVLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
4010     break;
4011 
4012   case ARM::LDMIA_RET:
4013   case ARM::LDMIA:
4014   case ARM::LDMDA:
4015   case ARM::LDMDB:
4016   case ARM::LDMIB:
4017   case ARM::LDMIA_UPD:
4018   case ARM::LDMDA_UPD:
4019   case ARM::LDMDB_UPD:
4020   case ARM::LDMIB_UPD:
4021   case ARM::tLDMIA:
4022   case ARM::tLDMIA_UPD:
4023   case ARM::tPUSH:
4024   case ARM::t2LDMIA_RET:
4025   case ARM::t2LDMIA:
4026   case ARM::t2LDMDB:
4027   case ARM::t2LDMIA_UPD:
4028   case ARM::t2LDMDB_UPD:
4029     LdmBypass = true;
4030     DefCycle = getLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
4031     break;
4032   }
4033 
4034   if (DefCycle == -1)
4035     // We can't seem to determine the result latency of the def, assume it's 2.
4036     DefCycle = 2;
4037 
4038   int UseCycle = -1;
4039   switch (UseMCID.getOpcode()) {
4040   default:
4041     UseCycle = ItinData->getOperandCycle(UseClass, UseIdx);
4042     break;
4043 
4044   case ARM::VSTMDIA:
4045   case ARM::VSTMDIA_UPD:
4046   case ARM::VSTMDDB_UPD:
4047   case ARM::VSTMSIA:
4048   case ARM::VSTMSIA_UPD:
4049   case ARM::VSTMSDB_UPD:
4050     UseCycle = getVSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
4051     break;
4052 
4053   case ARM::STMIA:
4054   case ARM::STMDA:
4055   case ARM::STMDB:
4056   case ARM::STMIB:
4057   case ARM::STMIA_UPD:
4058   case ARM::STMDA_UPD:
4059   case ARM::STMDB_UPD:
4060   case ARM::STMIB_UPD:
4061   case ARM::tSTMIA_UPD:
4062   case ARM::tPOP_RET:
4063   case ARM::tPOP:
4064   case ARM::t2STMIA:
4065   case ARM::t2STMDB:
4066   case ARM::t2STMIA_UPD:
4067   case ARM::t2STMDB_UPD:
4068     UseCycle = getSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
4069     break;
4070   }
4071 
4072   if (UseCycle == -1)
4073     // Assume it's read in the first stage.
4074     UseCycle = 1;
4075 
4076   UseCycle = DefCycle - UseCycle + 1;
4077   if (UseCycle > 0) {
4078     if (LdmBypass) {
4079       // It's a variable_ops instruction so we can't use DefIdx here. Just use
4080       // first def operand.
4081       if (ItinData->hasPipelineForwarding(DefClass, DefMCID.getNumOperands()-1,
4082                                           UseClass, UseIdx))
4083         --UseCycle;
4084     } else if (ItinData->hasPipelineForwarding(DefClass, DefIdx,
4085                                                UseClass, UseIdx)) {
4086       --UseCycle;
4087     }
4088   }
4089 
4090   return UseCycle;
4091 }
4092 
4093 static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI,
4094                                            const MachineInstr *MI, unsigned Reg,
4095                                            unsigned &DefIdx, unsigned &Dist) {
4096   Dist = 0;
4097 
4098   MachineBasicBlock::const_iterator I = MI; ++I;
4099   MachineBasicBlock::const_instr_iterator II = std::prev(I.getInstrIterator());
4100   assert(II->isInsideBundle() && "Empty bundle?");
4101 
4102   int Idx = -1;
4103   while (II->isInsideBundle()) {
4104     Idx = II->findRegisterDefOperandIdx(Reg, false, true, TRI);
4105     if (Idx != -1)
4106       break;
4107     --II;
4108     ++Dist;
4109   }
4110 
4111   assert(Idx != -1 && "Cannot find bundled definition!");
4112   DefIdx = Idx;
4113   return &*II;
4114 }
4115 
4116 static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI,
4117                                            const MachineInstr &MI, unsigned Reg,
4118                                            unsigned &UseIdx, unsigned &Dist) {
4119   Dist = 0;
4120 
4121   MachineBasicBlock::const_instr_iterator II = ++MI.getIterator();
4122   assert(II->isInsideBundle() && "Empty bundle?");
4123   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4124 
4125   // FIXME: This doesn't properly handle multiple uses.
4126   int Idx = -1;
4127   while (II != E && II->isInsideBundle()) {
4128     Idx = II->findRegisterUseOperandIdx(Reg, false, TRI);
4129     if (Idx != -1)
4130       break;
4131     if (II->getOpcode() != ARM::t2IT)
4132       ++Dist;
4133     ++II;
4134   }
4135 
4136   if (Idx == -1) {
4137     Dist = 0;
4138     return nullptr;
4139   }
4140 
4141   UseIdx = Idx;
4142   return &*II;
4143 }
4144 
4145 /// Return the number of cycles to add to (or subtract from) the static
4146 /// itinerary based on the def opcode and alignment. The caller will ensure that
4147 /// adjusted latency is at least one cycle.
4148 static int adjustDefLatency(const ARMSubtarget &Subtarget,
4149                             const MachineInstr &DefMI,
4150                             const MCInstrDesc &DefMCID, unsigned DefAlign) {
4151   int Adjust = 0;
4152   if (Subtarget.isCortexA8() || Subtarget.isLikeA9() || Subtarget.isCortexA7()) {
4153     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4154     // variants are one cycle cheaper.
4155     switch (DefMCID.getOpcode()) {
4156     default: break;
4157     case ARM::LDRrs:
4158     case ARM::LDRBrs: {
4159       unsigned ShOpVal = DefMI.getOperand(3).getImm();
4160       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4161       if (ShImm == 0 ||
4162           (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4163         --Adjust;
4164       break;
4165     }
4166     case ARM::t2LDRs:
4167     case ARM::t2LDRBs:
4168     case ARM::t2LDRHs:
4169     case ARM::t2LDRSHs: {
4170       // Thumb2 mode: lsl only.
4171       unsigned ShAmt = DefMI.getOperand(3).getImm();
4172       if (ShAmt == 0 || ShAmt == 2)
4173         --Adjust;
4174       break;
4175     }
4176     }
4177   } else if (Subtarget.isSwift()) {
4178     // FIXME: Properly handle all of the latency adjustments for address
4179     // writeback.
4180     switch (DefMCID.getOpcode()) {
4181     default: break;
4182     case ARM::LDRrs:
4183     case ARM::LDRBrs: {
4184       unsigned ShOpVal = DefMI.getOperand(3).getImm();
4185       bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
4186       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4187       if (!isSub &&
4188           (ShImm == 0 ||
4189            ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4190             ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
4191         Adjust -= 2;
4192       else if (!isSub &&
4193                ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
4194         --Adjust;
4195       break;
4196     }
4197     case ARM::t2LDRs:
4198     case ARM::t2LDRBs:
4199     case ARM::t2LDRHs:
4200     case ARM::t2LDRSHs: {
4201       // Thumb2 mode: lsl only.
4202       unsigned ShAmt = DefMI.getOperand(3).getImm();
4203       if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3)
4204         Adjust -= 2;
4205       break;
4206     }
4207     }
4208   }
4209 
4210   if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment()) {
4211     switch (DefMCID.getOpcode()) {
4212     default: break;
4213     case ARM::VLD1q8:
4214     case ARM::VLD1q16:
4215     case ARM::VLD1q32:
4216     case ARM::VLD1q64:
4217     case ARM::VLD1q8wb_fixed:
4218     case ARM::VLD1q16wb_fixed:
4219     case ARM::VLD1q32wb_fixed:
4220     case ARM::VLD1q64wb_fixed:
4221     case ARM::VLD1q8wb_register:
4222     case ARM::VLD1q16wb_register:
4223     case ARM::VLD1q32wb_register:
4224     case ARM::VLD1q64wb_register:
4225     case ARM::VLD2d8:
4226     case ARM::VLD2d16:
4227     case ARM::VLD2d32:
4228     case ARM::VLD2q8:
4229     case ARM::VLD2q16:
4230     case ARM::VLD2q32:
4231     case ARM::VLD2d8wb_fixed:
4232     case ARM::VLD2d16wb_fixed:
4233     case ARM::VLD2d32wb_fixed:
4234     case ARM::VLD2q8wb_fixed:
4235     case ARM::VLD2q16wb_fixed:
4236     case ARM::VLD2q32wb_fixed:
4237     case ARM::VLD2d8wb_register:
4238     case ARM::VLD2d16wb_register:
4239     case ARM::VLD2d32wb_register:
4240     case ARM::VLD2q8wb_register:
4241     case ARM::VLD2q16wb_register:
4242     case ARM::VLD2q32wb_register:
4243     case ARM::VLD3d8:
4244     case ARM::VLD3d16:
4245     case ARM::VLD3d32:
4246     case ARM::VLD1d64T:
4247     case ARM::VLD3d8_UPD:
4248     case ARM::VLD3d16_UPD:
4249     case ARM::VLD3d32_UPD:
4250     case ARM::VLD1d64Twb_fixed:
4251     case ARM::VLD1d64Twb_register:
4252     case ARM::VLD3q8_UPD:
4253     case ARM::VLD3q16_UPD:
4254     case ARM::VLD3q32_UPD:
4255     case ARM::VLD4d8:
4256     case ARM::VLD4d16:
4257     case ARM::VLD4d32:
4258     case ARM::VLD1d64Q:
4259     case ARM::VLD4d8_UPD:
4260     case ARM::VLD4d16_UPD:
4261     case ARM::VLD4d32_UPD:
4262     case ARM::VLD1d64Qwb_fixed:
4263     case ARM::VLD1d64Qwb_register:
4264     case ARM::VLD4q8_UPD:
4265     case ARM::VLD4q16_UPD:
4266     case ARM::VLD4q32_UPD:
4267     case ARM::VLD1DUPq8:
4268     case ARM::VLD1DUPq16:
4269     case ARM::VLD1DUPq32:
4270     case ARM::VLD1DUPq8wb_fixed:
4271     case ARM::VLD1DUPq16wb_fixed:
4272     case ARM::VLD1DUPq32wb_fixed:
4273     case ARM::VLD1DUPq8wb_register:
4274     case ARM::VLD1DUPq16wb_register:
4275     case ARM::VLD1DUPq32wb_register:
4276     case ARM::VLD2DUPd8:
4277     case ARM::VLD2DUPd16:
4278     case ARM::VLD2DUPd32:
4279     case ARM::VLD2DUPd8wb_fixed:
4280     case ARM::VLD2DUPd16wb_fixed:
4281     case ARM::VLD2DUPd32wb_fixed:
4282     case ARM::VLD2DUPd8wb_register:
4283     case ARM::VLD2DUPd16wb_register:
4284     case ARM::VLD2DUPd32wb_register:
4285     case ARM::VLD4DUPd8:
4286     case ARM::VLD4DUPd16:
4287     case ARM::VLD4DUPd32:
4288     case ARM::VLD4DUPd8_UPD:
4289     case ARM::VLD4DUPd16_UPD:
4290     case ARM::VLD4DUPd32_UPD:
4291     case ARM::VLD1LNd8:
4292     case ARM::VLD1LNd16:
4293     case ARM::VLD1LNd32:
4294     case ARM::VLD1LNd8_UPD:
4295     case ARM::VLD1LNd16_UPD:
4296     case ARM::VLD1LNd32_UPD:
4297     case ARM::VLD2LNd8:
4298     case ARM::VLD2LNd16:
4299     case ARM::VLD2LNd32:
4300     case ARM::VLD2LNq16:
4301     case ARM::VLD2LNq32:
4302     case ARM::VLD2LNd8_UPD:
4303     case ARM::VLD2LNd16_UPD:
4304     case ARM::VLD2LNd32_UPD:
4305     case ARM::VLD2LNq16_UPD:
4306     case ARM::VLD2LNq32_UPD:
4307     case ARM::VLD4LNd8:
4308     case ARM::VLD4LNd16:
4309     case ARM::VLD4LNd32:
4310     case ARM::VLD4LNq16:
4311     case ARM::VLD4LNq32:
4312     case ARM::VLD4LNd8_UPD:
4313     case ARM::VLD4LNd16_UPD:
4314     case ARM::VLD4LNd32_UPD:
4315     case ARM::VLD4LNq16_UPD:
4316     case ARM::VLD4LNq32_UPD:
4317       // If the address is not 64-bit aligned, the latencies of these
4318       // instructions increases by one.
4319       ++Adjust;
4320       break;
4321     }
4322   }
4323   return Adjust;
4324 }
4325 
4326 int ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4327                                         const MachineInstr &DefMI,
4328                                         unsigned DefIdx,
4329                                         const MachineInstr &UseMI,
4330                                         unsigned UseIdx) const {
4331   // No operand latency. The caller may fall back to getInstrLatency.
4332   if (!ItinData || ItinData->isEmpty())
4333     return -1;
4334 
4335   const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
4336   Register Reg = DefMO.getReg();
4337 
4338   const MachineInstr *ResolvedDefMI = &DefMI;
4339   unsigned DefAdj = 0;
4340   if (DefMI.isBundle())
4341     ResolvedDefMI =
4342         getBundledDefMI(&getRegisterInfo(), &DefMI, Reg, DefIdx, DefAdj);
4343   if (ResolvedDefMI->isCopyLike() || ResolvedDefMI->isInsertSubreg() ||
4344       ResolvedDefMI->isRegSequence() || ResolvedDefMI->isImplicitDef()) {
4345     return 1;
4346   }
4347 
4348   const MachineInstr *ResolvedUseMI = &UseMI;
4349   unsigned UseAdj = 0;
4350   if (UseMI.isBundle()) {
4351     ResolvedUseMI =
4352         getBundledUseMI(&getRegisterInfo(), UseMI, Reg, UseIdx, UseAdj);
4353     if (!ResolvedUseMI)
4354       return -1;
4355   }
4356 
4357   return getOperandLatencyImpl(
4358       ItinData, *ResolvedDefMI, DefIdx, ResolvedDefMI->getDesc(), DefAdj, DefMO,
4359       Reg, *ResolvedUseMI, UseIdx, ResolvedUseMI->getDesc(), UseAdj);
4360 }
4361 
4362 int ARMBaseInstrInfo::getOperandLatencyImpl(
4363     const InstrItineraryData *ItinData, const MachineInstr &DefMI,
4364     unsigned DefIdx, const MCInstrDesc &DefMCID, unsigned DefAdj,
4365     const MachineOperand &DefMO, unsigned Reg, const MachineInstr &UseMI,
4366     unsigned UseIdx, const MCInstrDesc &UseMCID, unsigned UseAdj) const {
4367   if (Reg == ARM::CPSR) {
4368     if (DefMI.getOpcode() == ARM::FMSTAT) {
4369       // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?)
4370       return Subtarget.isLikeA9() ? 1 : 20;
4371     }
4372 
4373     // CPSR set and branch can be paired in the same cycle.
4374     if (UseMI.isBranch())
4375       return 0;
4376 
4377     // Otherwise it takes the instruction latency (generally one).
4378     unsigned Latency = getInstrLatency(ItinData, DefMI);
4379 
4380     // For Thumb2 and -Os, prefer scheduling CPSR setting instruction close to
4381     // its uses. Instructions which are otherwise scheduled between them may
4382     // incur a code size penalty (not able to use the CPSR setting 16-bit
4383     // instructions).
4384     if (Latency > 0 && Subtarget.isThumb2()) {
4385       const MachineFunction *MF = DefMI.getParent()->getParent();
4386       // FIXME: Use Function::hasOptSize().
4387       if (MF->getFunction().hasFnAttribute(Attribute::OptimizeForSize))
4388         --Latency;
4389     }
4390     return Latency;
4391   }
4392 
4393   if (DefMO.isImplicit() || UseMI.getOperand(UseIdx).isImplicit())
4394     return -1;
4395 
4396   unsigned DefAlign = DefMI.hasOneMemOperand()
4397                           ? (*DefMI.memoperands_begin())->getAlign().value()
4398                           : 0;
4399   unsigned UseAlign = UseMI.hasOneMemOperand()
4400                           ? (*UseMI.memoperands_begin())->getAlign().value()
4401                           : 0;
4402 
4403   // Get the itinerary's latency if possible, and handle variable_ops.
4404   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign, UseMCID,
4405                                   UseIdx, UseAlign);
4406   // Unable to find operand latency. The caller may resort to getInstrLatency.
4407   if (Latency < 0)
4408     return Latency;
4409 
4410   // Adjust for IT block position.
4411   int Adj = DefAdj + UseAdj;
4412 
4413   // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4414   Adj += adjustDefLatency(Subtarget, DefMI, DefMCID, DefAlign);
4415   if (Adj >= 0 || (int)Latency > -Adj) {
4416     return Latency + Adj;
4417   }
4418   // Return the itinerary latency, which may be zero but not less than zero.
4419   return Latency;
4420 }
4421 
4422 int
4423 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4424                                     SDNode *DefNode, unsigned DefIdx,
4425                                     SDNode *UseNode, unsigned UseIdx) const {
4426   if (!DefNode->isMachineOpcode())
4427     return 1;
4428 
4429   const MCInstrDesc &DefMCID = get(DefNode->getMachineOpcode());
4430 
4431   if (isZeroCost(DefMCID.Opcode))
4432     return 0;
4433 
4434   if (!ItinData || ItinData->isEmpty())
4435     return DefMCID.mayLoad() ? 3 : 1;
4436 
4437   if (!UseNode->isMachineOpcode()) {
4438     int Latency = ItinData->getOperandCycle(DefMCID.getSchedClass(), DefIdx);
4439     int Adj = Subtarget.getPreISelOperandLatencyAdjustment();
4440     int Threshold = 1 + Adj;
4441     return Latency <= Threshold ? 1 : Latency - Adj;
4442   }
4443 
4444   const MCInstrDesc &UseMCID = get(UseNode->getMachineOpcode());
4445   auto *DefMN = cast<MachineSDNode>(DefNode);
4446   unsigned DefAlign = !DefMN->memoperands_empty()
4447                           ? (*DefMN->memoperands_begin())->getAlign().value()
4448                           : 0;
4449   auto *UseMN = cast<MachineSDNode>(UseNode);
4450   unsigned UseAlign = !UseMN->memoperands_empty()
4451                           ? (*UseMN->memoperands_begin())->getAlign().value()
4452                           : 0;
4453   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign,
4454                                   UseMCID, UseIdx, UseAlign);
4455 
4456   if (Latency > 1 &&
4457       (Subtarget.isCortexA8() || Subtarget.isLikeA9() ||
4458        Subtarget.isCortexA7())) {
4459     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4460     // variants are one cycle cheaper.
4461     switch (DefMCID.getOpcode()) {
4462     default: break;
4463     case ARM::LDRrs:
4464     case ARM::LDRBrs: {
4465       unsigned ShOpVal =
4466         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4467       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4468       if (ShImm == 0 ||
4469           (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4470         --Latency;
4471       break;
4472     }
4473     case ARM::t2LDRs:
4474     case ARM::t2LDRBs:
4475     case ARM::t2LDRHs:
4476     case ARM::t2LDRSHs: {
4477       // Thumb2 mode: lsl only.
4478       unsigned ShAmt =
4479         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4480       if (ShAmt == 0 || ShAmt == 2)
4481         --Latency;
4482       break;
4483     }
4484     }
4485   } else if (DefIdx == 0 && Latency > 2 && Subtarget.isSwift()) {
4486     // FIXME: Properly handle all of the latency adjustments for address
4487     // writeback.
4488     switch (DefMCID.getOpcode()) {
4489     default: break;
4490     case ARM::LDRrs:
4491     case ARM::LDRBrs: {
4492       unsigned ShOpVal =
4493         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4494       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4495       if (ShImm == 0 ||
4496           ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4497            ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4498         Latency -= 2;
4499       else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
4500         --Latency;
4501       break;
4502     }
4503     case ARM::t2LDRs:
4504     case ARM::t2LDRBs:
4505     case ARM::t2LDRHs:
4506     case ARM::t2LDRSHs:
4507       // Thumb2 mode: lsl 0-3 only.
4508       Latency -= 2;
4509       break;
4510     }
4511   }
4512 
4513   if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment())
4514     switch (DefMCID.getOpcode()) {
4515     default: break;
4516     case ARM::VLD1q8:
4517     case ARM::VLD1q16:
4518     case ARM::VLD1q32:
4519     case ARM::VLD1q64:
4520     case ARM::VLD1q8wb_register:
4521     case ARM::VLD1q16wb_register:
4522     case ARM::VLD1q32wb_register:
4523     case ARM::VLD1q64wb_register:
4524     case ARM::VLD1q8wb_fixed:
4525     case ARM::VLD1q16wb_fixed:
4526     case ARM::VLD1q32wb_fixed:
4527     case ARM::VLD1q64wb_fixed:
4528     case ARM::VLD2d8:
4529     case ARM::VLD2d16:
4530     case ARM::VLD2d32:
4531     case ARM::VLD2q8Pseudo:
4532     case ARM::VLD2q16Pseudo:
4533     case ARM::VLD2q32Pseudo:
4534     case ARM::VLD2d8wb_fixed:
4535     case ARM::VLD2d16wb_fixed:
4536     case ARM::VLD2d32wb_fixed:
4537     case ARM::VLD2q8PseudoWB_fixed:
4538     case ARM::VLD2q16PseudoWB_fixed:
4539     case ARM::VLD2q32PseudoWB_fixed:
4540     case ARM::VLD2d8wb_register:
4541     case ARM::VLD2d16wb_register:
4542     case ARM::VLD2d32wb_register:
4543     case ARM::VLD2q8PseudoWB_register:
4544     case ARM::VLD2q16PseudoWB_register:
4545     case ARM::VLD2q32PseudoWB_register:
4546     case ARM::VLD3d8Pseudo:
4547     case ARM::VLD3d16Pseudo:
4548     case ARM::VLD3d32Pseudo:
4549     case ARM::VLD1d8TPseudo:
4550     case ARM::VLD1d16TPseudo:
4551     case ARM::VLD1d32TPseudo:
4552     case ARM::VLD1d64TPseudo:
4553     case ARM::VLD1d64TPseudoWB_fixed:
4554     case ARM::VLD1d64TPseudoWB_register:
4555     case ARM::VLD3d8Pseudo_UPD:
4556     case ARM::VLD3d16Pseudo_UPD:
4557     case ARM::VLD3d32Pseudo_UPD:
4558     case ARM::VLD3q8Pseudo_UPD:
4559     case ARM::VLD3q16Pseudo_UPD:
4560     case ARM::VLD3q32Pseudo_UPD:
4561     case ARM::VLD3q8oddPseudo:
4562     case ARM::VLD3q16oddPseudo:
4563     case ARM::VLD3q32oddPseudo:
4564     case ARM::VLD3q8oddPseudo_UPD:
4565     case ARM::VLD3q16oddPseudo_UPD:
4566     case ARM::VLD3q32oddPseudo_UPD:
4567     case ARM::VLD4d8Pseudo:
4568     case ARM::VLD4d16Pseudo:
4569     case ARM::VLD4d32Pseudo:
4570     case ARM::VLD1d8QPseudo:
4571     case ARM::VLD1d16QPseudo:
4572     case ARM::VLD1d32QPseudo:
4573     case ARM::VLD1d64QPseudo:
4574     case ARM::VLD1d64QPseudoWB_fixed:
4575     case ARM::VLD1d64QPseudoWB_register:
4576     case ARM::VLD1q8HighQPseudo:
4577     case ARM::VLD1q8LowQPseudo_UPD:
4578     case ARM::VLD1q8HighTPseudo:
4579     case ARM::VLD1q8LowTPseudo_UPD:
4580     case ARM::VLD1q16HighQPseudo:
4581     case ARM::VLD1q16LowQPseudo_UPD:
4582     case ARM::VLD1q16HighTPseudo:
4583     case ARM::VLD1q16LowTPseudo_UPD:
4584     case ARM::VLD1q32HighQPseudo:
4585     case ARM::VLD1q32LowQPseudo_UPD:
4586     case ARM::VLD1q32HighTPseudo:
4587     case ARM::VLD1q32LowTPseudo_UPD:
4588     case ARM::VLD1q64HighQPseudo:
4589     case ARM::VLD1q64LowQPseudo_UPD:
4590     case ARM::VLD1q64HighTPseudo:
4591     case ARM::VLD1q64LowTPseudo_UPD:
4592     case ARM::VLD4d8Pseudo_UPD:
4593     case ARM::VLD4d16Pseudo_UPD:
4594     case ARM::VLD4d32Pseudo_UPD:
4595     case ARM::VLD4q8Pseudo_UPD:
4596     case ARM::VLD4q16Pseudo_UPD:
4597     case ARM::VLD4q32Pseudo_UPD:
4598     case ARM::VLD4q8oddPseudo:
4599     case ARM::VLD4q16oddPseudo:
4600     case ARM::VLD4q32oddPseudo:
4601     case ARM::VLD4q8oddPseudo_UPD:
4602     case ARM::VLD4q16oddPseudo_UPD:
4603     case ARM::VLD4q32oddPseudo_UPD:
4604     case ARM::VLD1DUPq8:
4605     case ARM::VLD1DUPq16:
4606     case ARM::VLD1DUPq32:
4607     case ARM::VLD1DUPq8wb_fixed:
4608     case ARM::VLD1DUPq16wb_fixed:
4609     case ARM::VLD1DUPq32wb_fixed:
4610     case ARM::VLD1DUPq8wb_register:
4611     case ARM::VLD1DUPq16wb_register:
4612     case ARM::VLD1DUPq32wb_register:
4613     case ARM::VLD2DUPd8:
4614     case ARM::VLD2DUPd16:
4615     case ARM::VLD2DUPd32:
4616     case ARM::VLD2DUPd8wb_fixed:
4617     case ARM::VLD2DUPd16wb_fixed:
4618     case ARM::VLD2DUPd32wb_fixed:
4619     case ARM::VLD2DUPd8wb_register:
4620     case ARM::VLD2DUPd16wb_register:
4621     case ARM::VLD2DUPd32wb_register:
4622     case ARM::VLD2DUPq8EvenPseudo:
4623     case ARM::VLD2DUPq8OddPseudo:
4624     case ARM::VLD2DUPq16EvenPseudo:
4625     case ARM::VLD2DUPq16OddPseudo:
4626     case ARM::VLD2DUPq32EvenPseudo:
4627     case ARM::VLD2DUPq32OddPseudo:
4628     case ARM::VLD3DUPq8EvenPseudo:
4629     case ARM::VLD3DUPq8OddPseudo:
4630     case ARM::VLD3DUPq16EvenPseudo:
4631     case ARM::VLD3DUPq16OddPseudo:
4632     case ARM::VLD3DUPq32EvenPseudo:
4633     case ARM::VLD3DUPq32OddPseudo:
4634     case ARM::VLD4DUPd8Pseudo:
4635     case ARM::VLD4DUPd16Pseudo:
4636     case ARM::VLD4DUPd32Pseudo:
4637     case ARM::VLD4DUPd8Pseudo_UPD:
4638     case ARM::VLD4DUPd16Pseudo_UPD:
4639     case ARM::VLD4DUPd32Pseudo_UPD:
4640     case ARM::VLD4DUPq8EvenPseudo:
4641     case ARM::VLD4DUPq8OddPseudo:
4642     case ARM::VLD4DUPq16EvenPseudo:
4643     case ARM::VLD4DUPq16OddPseudo:
4644     case ARM::VLD4DUPq32EvenPseudo:
4645     case ARM::VLD4DUPq32OddPseudo:
4646     case ARM::VLD1LNq8Pseudo:
4647     case ARM::VLD1LNq16Pseudo:
4648     case ARM::VLD1LNq32Pseudo:
4649     case ARM::VLD1LNq8Pseudo_UPD:
4650     case ARM::VLD1LNq16Pseudo_UPD:
4651     case ARM::VLD1LNq32Pseudo_UPD:
4652     case ARM::VLD2LNd8Pseudo:
4653     case ARM::VLD2LNd16Pseudo:
4654     case ARM::VLD2LNd32Pseudo:
4655     case ARM::VLD2LNq16Pseudo:
4656     case ARM::VLD2LNq32Pseudo:
4657     case ARM::VLD2LNd8Pseudo_UPD:
4658     case ARM::VLD2LNd16Pseudo_UPD:
4659     case ARM::VLD2LNd32Pseudo_UPD:
4660     case ARM::VLD2LNq16Pseudo_UPD:
4661     case ARM::VLD2LNq32Pseudo_UPD:
4662     case ARM::VLD4LNd8Pseudo:
4663     case ARM::VLD4LNd16Pseudo:
4664     case ARM::VLD4LNd32Pseudo:
4665     case ARM::VLD4LNq16Pseudo:
4666     case ARM::VLD4LNq32Pseudo:
4667     case ARM::VLD4LNd8Pseudo_UPD:
4668     case ARM::VLD4LNd16Pseudo_UPD:
4669     case ARM::VLD4LNd32Pseudo_UPD:
4670     case ARM::VLD4LNq16Pseudo_UPD:
4671     case ARM::VLD4LNq32Pseudo_UPD:
4672       // If the address is not 64-bit aligned, the latencies of these
4673       // instructions increases by one.
4674       ++Latency;
4675       break;
4676     }
4677 
4678   return Latency;
4679 }
4680 
4681 unsigned ARMBaseInstrInfo::getPredicationCost(const MachineInstr &MI) const {
4682   if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4683       MI.isImplicitDef())
4684     return 0;
4685 
4686   if (MI.isBundle())
4687     return 0;
4688 
4689   const MCInstrDesc &MCID = MI.getDesc();
4690 
4691   if (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) &&
4692                         !Subtarget.cheapPredicableCPSRDef())) {
4693     // When predicated, CPSR is an additional source operand for CPSR updating
4694     // instructions, this apparently increases their latencies.
4695     return 1;
4696   }
4697   return 0;
4698 }
4699 
4700 unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4701                                            const MachineInstr &MI,
4702                                            unsigned *PredCost) const {
4703   if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4704       MI.isImplicitDef())
4705     return 1;
4706 
4707   // An instruction scheduler typically runs on unbundled instructions, however
4708   // other passes may query the latency of a bundled instruction.
4709   if (MI.isBundle()) {
4710     unsigned Latency = 0;
4711     MachineBasicBlock::const_instr_iterator I = MI.getIterator();
4712     MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4713     while (++I != E && I->isInsideBundle()) {
4714       if (I->getOpcode() != ARM::t2IT)
4715         Latency += getInstrLatency(ItinData, *I, PredCost);
4716     }
4717     return Latency;
4718   }
4719 
4720   const MCInstrDesc &MCID = MI.getDesc();
4721   if (PredCost && (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) &&
4722                                      !Subtarget.cheapPredicableCPSRDef()))) {
4723     // When predicated, CPSR is an additional source operand for CPSR updating
4724     // instructions, this apparently increases their latencies.
4725     *PredCost = 1;
4726   }
4727   // Be sure to call getStageLatency for an empty itinerary in case it has a
4728   // valid MinLatency property.
4729   if (!ItinData)
4730     return MI.mayLoad() ? 3 : 1;
4731 
4732   unsigned Class = MCID.getSchedClass();
4733 
4734   // For instructions with variable uops, use uops as latency.
4735   if (!ItinData->isEmpty() && ItinData->getNumMicroOps(Class) < 0)
4736     return getNumMicroOps(ItinData, MI);
4737 
4738   // For the common case, fall back on the itinerary's latency.
4739   unsigned Latency = ItinData->getStageLatency(Class);
4740 
4741   // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4742   unsigned DefAlign =
4743       MI.hasOneMemOperand() ? (*MI.memoperands_begin())->getAlign().value() : 0;
4744   int Adj = adjustDefLatency(Subtarget, MI, MCID, DefAlign);
4745   if (Adj >= 0 || (int)Latency > -Adj) {
4746     return Latency + Adj;
4747   }
4748   return Latency;
4749 }
4750 
4751 int ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4752                                       SDNode *Node) const {
4753   if (!Node->isMachineOpcode())
4754     return 1;
4755 
4756   if (!ItinData || ItinData->isEmpty())
4757     return 1;
4758 
4759   unsigned Opcode = Node->getMachineOpcode();
4760   switch (Opcode) {
4761   default:
4762     return ItinData->getStageLatency(get(Opcode).getSchedClass());
4763   case ARM::VLDMQIA:
4764   case ARM::VSTMQIA:
4765     return 2;
4766   }
4767 }
4768 
4769 bool ARMBaseInstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel,
4770                                              const MachineRegisterInfo *MRI,
4771                                              const MachineInstr &DefMI,
4772                                              unsigned DefIdx,
4773                                              const MachineInstr &UseMI,
4774                                              unsigned UseIdx) const {
4775   unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4776   unsigned UDomain = UseMI.getDesc().TSFlags & ARMII::DomainMask;
4777   if (Subtarget.nonpipelinedVFP() &&
4778       (DDomain == ARMII::DomainVFP || UDomain == ARMII::DomainVFP))
4779     return true;
4780 
4781   // Hoist VFP / NEON instructions with 4 or higher latency.
4782   unsigned Latency =
4783       SchedModel.computeOperandLatency(&DefMI, DefIdx, &UseMI, UseIdx);
4784   if (Latency <= 3)
4785     return false;
4786   return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON ||
4787          UDomain == ARMII::DomainVFP || UDomain == ARMII::DomainNEON;
4788 }
4789 
4790 bool ARMBaseInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
4791                                         const MachineInstr &DefMI,
4792                                         unsigned DefIdx) const {
4793   const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
4794   if (!ItinData || ItinData->isEmpty())
4795     return false;
4796 
4797   unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4798   if (DDomain == ARMII::DomainGeneral) {
4799     unsigned DefClass = DefMI.getDesc().getSchedClass();
4800     int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
4801     return (DefCycle != -1 && DefCycle <= 2);
4802   }
4803   return false;
4804 }
4805 
4806 bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr &MI,
4807                                          StringRef &ErrInfo) const {
4808   if (convertAddSubFlagsOpcode(MI.getOpcode())) {
4809     ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG";
4810     return false;
4811   }
4812   if (MI.getOpcode() == ARM::tMOVr && !Subtarget.hasV6Ops()) {
4813     // Make sure we don't generate a lo-lo mov that isn't supported.
4814     if (!ARM::hGPRRegClass.contains(MI.getOperand(0).getReg()) &&
4815         !ARM::hGPRRegClass.contains(MI.getOperand(1).getReg())) {
4816       ErrInfo = "Non-flag-setting Thumb1 mov is v6-only";
4817       return false;
4818     }
4819   }
4820   if (MI.getOpcode() == ARM::tPUSH ||
4821       MI.getOpcode() == ARM::tPOP ||
4822       MI.getOpcode() == ARM::tPOP_RET) {
4823     for (const MachineOperand &MO : llvm::drop_begin(MI.operands(), 2)) {
4824       if (MO.isImplicit() || !MO.isReg())
4825         continue;
4826       Register Reg = MO.getReg();
4827       if (Reg < ARM::R0 || Reg > ARM::R7) {
4828         if (!(MI.getOpcode() == ARM::tPUSH && Reg == ARM::LR) &&
4829             !(MI.getOpcode() == ARM::tPOP_RET && Reg == ARM::PC)) {
4830           ErrInfo = "Unsupported register in Thumb1 push/pop";
4831           return false;
4832         }
4833       }
4834     }
4835   }
4836   if (MI.getOpcode() == ARM::MVE_VMOV_q_rr) {
4837     assert(MI.getOperand(4).isImm() && MI.getOperand(5).isImm());
4838     if ((MI.getOperand(4).getImm() != 2 && MI.getOperand(4).getImm() != 3) ||
4839         MI.getOperand(4).getImm() != MI.getOperand(5).getImm() + 2) {
4840       ErrInfo = "Incorrect array index for MVE_VMOV_q_rr";
4841       return false;
4842     }
4843   }
4844 
4845   // Check the address model by taking the first Imm operand and checking it is
4846   // legal for that addressing mode.
4847   ARMII::AddrMode AddrMode =
4848       (ARMII::AddrMode)(MI.getDesc().TSFlags & ARMII::AddrModeMask);
4849   switch (AddrMode) {
4850   default:
4851     break;
4852   case ARMII::AddrModeT2_i7:
4853   case ARMII::AddrModeT2_i7s2:
4854   case ARMII::AddrModeT2_i7s4:
4855   case ARMII::AddrModeT2_i8:
4856   case ARMII::AddrModeT2_i8pos:
4857   case ARMII::AddrModeT2_i8neg:
4858   case ARMII::AddrModeT2_i8s4:
4859   case ARMII::AddrModeT2_i12: {
4860     uint32_t Imm = 0;
4861     for (auto Op : MI.operands()) {
4862       if (Op.isImm()) {
4863         Imm = Op.getImm();
4864         break;
4865       }
4866     }
4867     if (!isLegalAddressImm(MI.getOpcode(), Imm, this)) {
4868       ErrInfo = "Incorrect AddrMode Imm for instruction";
4869       return false;
4870     }
4871     break;
4872   }
4873   }
4874   return true;
4875 }
4876 
4877 void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI,
4878                                                 unsigned LoadImmOpc,
4879                                                 unsigned LoadOpc) const {
4880   assert(!Subtarget.isROPI() && !Subtarget.isRWPI() &&
4881          "ROPI/RWPI not currently supported with stack guard");
4882 
4883   MachineBasicBlock &MBB = *MI->getParent();
4884   DebugLoc DL = MI->getDebugLoc();
4885   Register Reg = MI->getOperand(0).getReg();
4886   MachineInstrBuilder MIB;
4887   unsigned int Offset = 0;
4888 
4889   if (LoadImmOpc == ARM::MRC || LoadImmOpc == ARM::t2MRC) {
4890     assert(Subtarget.isReadTPHard() &&
4891            "TLS stack protector requires hardware TLS register");
4892 
4893     BuildMI(MBB, MI, DL, get(LoadImmOpc), Reg)
4894         .addImm(15)
4895         .addImm(0)
4896         .addImm(13)
4897         .addImm(0)
4898         .addImm(3)
4899         .add(predOps(ARMCC::AL));
4900 
4901     Module &M = *MBB.getParent()->getFunction().getParent();
4902     Offset = M.getStackProtectorGuardOffset();
4903     if (Offset & ~0xfffU) {
4904       // The offset won't fit in the LDR's 12-bit immediate field, so emit an
4905       // extra ADD to cover the delta. This gives us a guaranteed 8 additional
4906       // bits, resulting in a range of 0 to +1 MiB for the guard offset.
4907       unsigned AddOpc = (LoadImmOpc == ARM::MRC) ? ARM::ADDri : ARM::t2ADDri;
4908       BuildMI(MBB, MI, DL, get(AddOpc), Reg)
4909           .addReg(Reg, RegState::Kill)
4910           .addImm(Offset & ~0xfffU)
4911           .add(predOps(ARMCC::AL))
4912           .addReg(0);
4913       Offset &= 0xfffU;
4914     }
4915   } else {
4916     const GlobalValue *GV =
4917         cast<GlobalValue>((*MI->memoperands_begin())->getValue());
4918     bool IsIndirect = Subtarget.isGVIndirectSymbol(GV);
4919 
4920     unsigned TargetFlags = ARMII::MO_NO_FLAG;
4921     if (Subtarget.isTargetMachO()) {
4922       TargetFlags |= ARMII::MO_NONLAZY;
4923     } else if (Subtarget.isTargetCOFF()) {
4924       if (GV->hasDLLImportStorageClass())
4925         TargetFlags |= ARMII::MO_DLLIMPORT;
4926       else if (IsIndirect)
4927         TargetFlags |= ARMII::MO_COFFSTUB;
4928     } else if (Subtarget.isGVInGOT(GV)) {
4929       TargetFlags |= ARMII::MO_GOT;
4930     }
4931 
4932     BuildMI(MBB, MI, DL, get(LoadImmOpc), Reg)
4933         .addGlobalAddress(GV, 0, TargetFlags);
4934 
4935     if (IsIndirect) {
4936       MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
4937       MIB.addReg(Reg, RegState::Kill).addImm(0);
4938       auto Flags = MachineMemOperand::MOLoad |
4939                    MachineMemOperand::MODereferenceable |
4940                    MachineMemOperand::MOInvariant;
4941       MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
4942           MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, Align(4));
4943       MIB.addMemOperand(MMO).add(predOps(ARMCC::AL));
4944     }
4945   }
4946 
4947   MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
4948   MIB.addReg(Reg, RegState::Kill)
4949       .addImm(Offset)
4950       .cloneMemRefs(*MI)
4951       .add(predOps(ARMCC::AL));
4952 }
4953 
4954 bool
4955 ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
4956                                      unsigned &AddSubOpc,
4957                                      bool &NegAcc, bool &HasLane) const {
4958   DenseMap<unsigned, unsigned>::const_iterator I = MLxEntryMap.find(Opcode);
4959   if (I == MLxEntryMap.end())
4960     return false;
4961 
4962   const ARM_MLxEntry &Entry = ARM_MLxTable[I->second];
4963   MulOpc = Entry.MulOpc;
4964   AddSubOpc = Entry.AddSubOpc;
4965   NegAcc = Entry.NegAcc;
4966   HasLane = Entry.HasLane;
4967   return true;
4968 }
4969 
4970 //===----------------------------------------------------------------------===//
4971 // Execution domains.
4972 //===----------------------------------------------------------------------===//
4973 //
4974 // Some instructions go down the NEON pipeline, some go down the VFP pipeline,
4975 // and some can go down both.  The vmov instructions go down the VFP pipeline,
4976 // but they can be changed to vorr equivalents that are executed by the NEON
4977 // pipeline.
4978 //
4979 // We use the following execution domain numbering:
4980 //
4981 enum ARMExeDomain {
4982   ExeGeneric = 0,
4983   ExeVFP = 1,
4984   ExeNEON = 2
4985 };
4986 
4987 //
4988 // Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h
4989 //
4990 std::pair<uint16_t, uint16_t>
4991 ARMBaseInstrInfo::getExecutionDomain(const MachineInstr &MI) const {
4992   // If we don't have access to NEON instructions then we won't be able
4993   // to swizzle anything to the NEON domain. Check to make sure.
4994   if (Subtarget.hasNEON()) {
4995     // VMOVD, VMOVRS and VMOVSR are VFP instructions, but can be changed to NEON
4996     // if they are not predicated.
4997     if (MI.getOpcode() == ARM::VMOVD && !isPredicated(MI))
4998       return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
4999 
5000     // CortexA9 is particularly picky about mixing the two and wants these
5001     // converted.
5002     if (Subtarget.useNEONForFPMovs() && !isPredicated(MI) &&
5003         (MI.getOpcode() == ARM::VMOVRS || MI.getOpcode() == ARM::VMOVSR ||
5004          MI.getOpcode() == ARM::VMOVS))
5005       return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
5006   }
5007   // No other instructions can be swizzled, so just determine their domain.
5008   unsigned Domain = MI.getDesc().TSFlags & ARMII::DomainMask;
5009 
5010   if (Domain & ARMII::DomainNEON)
5011     return std::make_pair(ExeNEON, 0);
5012 
5013   // Certain instructions can go either way on Cortex-A8.
5014   // Treat them as NEON instructions.
5015   if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8())
5016     return std::make_pair(ExeNEON, 0);
5017 
5018   if (Domain & ARMII::DomainVFP)
5019     return std::make_pair(ExeVFP, 0);
5020 
5021   return std::make_pair(ExeGeneric, 0);
5022 }
5023 
5024 static unsigned getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI,
5025                                             unsigned SReg, unsigned &Lane) {
5026   unsigned DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_0, &ARM::DPRRegClass);
5027   Lane = 0;
5028 
5029   if (DReg != ARM::NoRegister)
5030    return DReg;
5031 
5032   Lane = 1;
5033   DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_1, &ARM::DPRRegClass);
5034 
5035   assert(DReg && "S-register with no D super-register?");
5036   return DReg;
5037 }
5038 
5039 /// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane,
5040 /// set ImplicitSReg to a register number that must be marked as implicit-use or
5041 /// zero if no register needs to be defined as implicit-use.
5042 ///
5043 /// If the function cannot determine if an SPR should be marked implicit use or
5044 /// not, it returns false.
5045 ///
5046 /// This function handles cases where an instruction is being modified from taking
5047 /// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict
5048 /// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other
5049 /// lane of the DPR).
5050 ///
5051 /// If the other SPR is defined, an implicit-use of it should be added. Else,
5052 /// (including the case where the DPR itself is defined), it should not.
5053 ///
5054 static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI,
5055                                        MachineInstr &MI, unsigned DReg,
5056                                        unsigned Lane, unsigned &ImplicitSReg) {
5057   // If the DPR is defined or used already, the other SPR lane will be chained
5058   // correctly, so there is nothing to be done.
5059   if (MI.definesRegister(DReg, TRI) || MI.readsRegister(DReg, TRI)) {
5060     ImplicitSReg = 0;
5061     return true;
5062   }
5063 
5064   // Otherwise we need to go searching to see if the SPR is set explicitly.
5065   ImplicitSReg = TRI->getSubReg(DReg,
5066                                 (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1);
5067   MachineBasicBlock::LivenessQueryResult LQR =
5068       MI.getParent()->computeRegisterLiveness(TRI, ImplicitSReg, MI);
5069 
5070   if (LQR == MachineBasicBlock::LQR_Live)
5071     return true;
5072   else if (LQR == MachineBasicBlock::LQR_Unknown)
5073     return false;
5074 
5075   // If the register is known not to be live, there is no need to add an
5076   // implicit-use.
5077   ImplicitSReg = 0;
5078   return true;
5079 }
5080 
5081 void ARMBaseInstrInfo::setExecutionDomain(MachineInstr &MI,
5082                                           unsigned Domain) const {
5083   unsigned DstReg, SrcReg, DReg;
5084   unsigned Lane;
5085   MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
5086   const TargetRegisterInfo *TRI = &getRegisterInfo();
5087   switch (MI.getOpcode()) {
5088   default:
5089     llvm_unreachable("cannot handle opcode!");
5090     break;
5091   case ARM::VMOVD:
5092     if (Domain != ExeNEON)
5093       break;
5094 
5095     // Zap the predicate operands.
5096     assert(!isPredicated(MI) && "Cannot predicate a VORRd");
5097 
5098     // Make sure we've got NEON instructions.
5099     assert(Subtarget.hasNEON() && "VORRd requires NEON");
5100 
5101     // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits)
5102     DstReg = MI.getOperand(0).getReg();
5103     SrcReg = MI.getOperand(1).getReg();
5104 
5105     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5106       MI.RemoveOperand(i - 1);
5107 
5108     // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits)
5109     MI.setDesc(get(ARM::VORRd));
5110     MIB.addReg(DstReg, RegState::Define)
5111         .addReg(SrcReg)
5112         .addReg(SrcReg)
5113         .add(predOps(ARMCC::AL));
5114     break;
5115   case ARM::VMOVRS:
5116     if (Domain != ExeNEON)
5117       break;
5118     assert(!isPredicated(MI) && "Cannot predicate a VGETLN");
5119 
5120     // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits)
5121     DstReg = MI.getOperand(0).getReg();
5122     SrcReg = MI.getOperand(1).getReg();
5123 
5124     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5125       MI.RemoveOperand(i - 1);
5126 
5127     DReg = getCorrespondingDRegAndLane(TRI, SrcReg, Lane);
5128 
5129     // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps)
5130     // Note that DSrc has been widened and the other lane may be undef, which
5131     // contaminates the entire register.
5132     MI.setDesc(get(ARM::VGETLNi32));
5133     MIB.addReg(DstReg, RegState::Define)
5134         .addReg(DReg, RegState::Undef)
5135         .addImm(Lane)
5136         .add(predOps(ARMCC::AL));
5137 
5138     // The old source should be an implicit use, otherwise we might think it
5139     // was dead before here.
5140     MIB.addReg(SrcReg, RegState::Implicit);
5141     break;
5142   case ARM::VMOVSR: {
5143     if (Domain != ExeNEON)
5144       break;
5145     assert(!isPredicated(MI) && "Cannot predicate a VSETLN");
5146 
5147     // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits)
5148     DstReg = MI.getOperand(0).getReg();
5149     SrcReg = MI.getOperand(1).getReg();
5150 
5151     DReg = getCorrespondingDRegAndLane(TRI, DstReg, Lane);
5152 
5153     unsigned ImplicitSReg;
5154     if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg))
5155       break;
5156 
5157     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5158       MI.RemoveOperand(i - 1);
5159 
5160     // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps)
5161     // Again DDst may be undefined at the beginning of this instruction.
5162     MI.setDesc(get(ARM::VSETLNi32));
5163     MIB.addReg(DReg, RegState::Define)
5164         .addReg(DReg, getUndefRegState(!MI.readsRegister(DReg, TRI)))
5165         .addReg(SrcReg)
5166         .addImm(Lane)
5167         .add(predOps(ARMCC::AL));
5168 
5169     // The narrower destination must be marked as set to keep previous chains
5170     // in place.
5171     MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
5172     if (ImplicitSReg != 0)
5173       MIB.addReg(ImplicitSReg, RegState::Implicit);
5174     break;
5175     }
5176     case ARM::VMOVS: {
5177       if (Domain != ExeNEON)
5178         break;
5179 
5180       // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits)
5181       DstReg = MI.getOperand(0).getReg();
5182       SrcReg = MI.getOperand(1).getReg();
5183 
5184       unsigned DstLane = 0, SrcLane = 0, DDst, DSrc;
5185       DDst = getCorrespondingDRegAndLane(TRI, DstReg, DstLane);
5186       DSrc = getCorrespondingDRegAndLane(TRI, SrcReg, SrcLane);
5187 
5188       unsigned ImplicitSReg;
5189       if (!getImplicitSPRUseForDPRUse(TRI, MI, DSrc, SrcLane, ImplicitSReg))
5190         break;
5191 
5192       for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5193         MI.RemoveOperand(i - 1);
5194 
5195       if (DSrc == DDst) {
5196         // Destination can be:
5197         //     %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits)
5198         MI.setDesc(get(ARM::VDUPLN32d));
5199         MIB.addReg(DDst, RegState::Define)
5200             .addReg(DDst, getUndefRegState(!MI.readsRegister(DDst, TRI)))
5201             .addImm(SrcLane)
5202             .add(predOps(ARMCC::AL));
5203 
5204         // Neither the source or the destination are naturally represented any
5205         // more, so add them in manually.
5206         MIB.addReg(DstReg, RegState::Implicit | RegState::Define);
5207         MIB.addReg(SrcReg, RegState::Implicit);
5208         if (ImplicitSReg != 0)
5209           MIB.addReg(ImplicitSReg, RegState::Implicit);
5210         break;
5211       }
5212 
5213       // In general there's no single instruction that can perform an S <-> S
5214       // move in NEON space, but a pair of VEXT instructions *can* do the
5215       // job. It turns out that the VEXTs needed will only use DSrc once, with
5216       // the position based purely on the combination of lane-0 and lane-1
5217       // involved. For example
5218       //     vmov s0, s2 -> vext.32 d0, d0, d1, #1  vext.32 d0, d0, d0, #1
5219       //     vmov s1, s3 -> vext.32 d0, d1, d0, #1  vext.32 d0, d0, d0, #1
5220       //     vmov s0, s3 -> vext.32 d0, d0, d0, #1  vext.32 d0, d1, d0, #1
5221       //     vmov s1, s2 -> vext.32 d0, d0, d0, #1  vext.32 d0, d0, d1, #1
5222       //
5223       // Pattern of the MachineInstrs is:
5224       //     %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits)
5225       MachineInstrBuilder NewMIB;
5226       NewMIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::VEXTd32),
5227                        DDst);
5228 
5229       // On the first instruction, both DSrc and DDst may be undef if present.
5230       // Specifically when the original instruction didn't have them as an
5231       // <imp-use>.
5232       unsigned CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst;
5233       bool CurUndef = !MI.readsRegister(CurReg, TRI);
5234       NewMIB.addReg(CurReg, getUndefRegState(CurUndef));
5235 
5236       CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst;
5237       CurUndef = !MI.readsRegister(CurReg, TRI);
5238       NewMIB.addReg(CurReg, getUndefRegState(CurUndef))
5239             .addImm(1)
5240             .add(predOps(ARMCC::AL));
5241 
5242       if (SrcLane == DstLane)
5243         NewMIB.addReg(SrcReg, RegState::Implicit);
5244 
5245       MI.setDesc(get(ARM::VEXTd32));
5246       MIB.addReg(DDst, RegState::Define);
5247 
5248       // On the second instruction, DDst has definitely been defined above, so
5249       // it is not undef. DSrc, if present, can be undef as above.
5250       CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst;
5251       CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI);
5252       MIB.addReg(CurReg, getUndefRegState(CurUndef));
5253 
5254       CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst;
5255       CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI);
5256       MIB.addReg(CurReg, getUndefRegState(CurUndef))
5257          .addImm(1)
5258          .add(predOps(ARMCC::AL));
5259 
5260       if (SrcLane != DstLane)
5261         MIB.addReg(SrcReg, RegState::Implicit);
5262 
5263       // As before, the original destination is no longer represented, add it
5264       // implicitly.
5265       MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
5266       if (ImplicitSReg != 0)
5267         MIB.addReg(ImplicitSReg, RegState::Implicit);
5268       break;
5269     }
5270   }
5271 }
5272 
5273 //===----------------------------------------------------------------------===//
5274 // Partial register updates
5275 //===----------------------------------------------------------------------===//
5276 //
5277 // Swift renames NEON registers with 64-bit granularity.  That means any
5278 // instruction writing an S-reg implicitly reads the containing D-reg.  The
5279 // problem is mostly avoided by translating f32 operations to v2f32 operations
5280 // on D-registers, but f32 loads are still a problem.
5281 //
5282 // These instructions can load an f32 into a NEON register:
5283 //
5284 // VLDRS - Only writes S, partial D update.
5285 // VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops.
5286 // VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops.
5287 //
5288 // FCONSTD can be used as a dependency-breaking instruction.
5289 unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
5290     const MachineInstr &MI, unsigned OpNum,
5291     const TargetRegisterInfo *TRI) const {
5292   auto PartialUpdateClearance = Subtarget.getPartialUpdateClearance();
5293   if (!PartialUpdateClearance)
5294     return 0;
5295 
5296   assert(TRI && "Need TRI instance");
5297 
5298   const MachineOperand &MO = MI.getOperand(OpNum);
5299   if (MO.readsReg())
5300     return 0;
5301   Register Reg = MO.getReg();
5302   int UseOp = -1;
5303 
5304   switch (MI.getOpcode()) {
5305   // Normal instructions writing only an S-register.
5306   case ARM::VLDRS:
5307   case ARM::FCONSTS:
5308   case ARM::VMOVSR:
5309   case ARM::VMOVv8i8:
5310   case ARM::VMOVv4i16:
5311   case ARM::VMOVv2i32:
5312   case ARM::VMOVv2f32:
5313   case ARM::VMOVv1i64:
5314     UseOp = MI.findRegisterUseOperandIdx(Reg, false, TRI);
5315     break;
5316 
5317     // Explicitly reads the dependency.
5318   case ARM::VLD1LNd32:
5319     UseOp = 3;
5320     break;
5321   default:
5322     return 0;
5323   }
5324 
5325   // If this instruction actually reads a value from Reg, there is no unwanted
5326   // dependency.
5327   if (UseOp != -1 && MI.getOperand(UseOp).readsReg())
5328     return 0;
5329 
5330   // We must be able to clobber the whole D-reg.
5331   if (Register::isVirtualRegister(Reg)) {
5332     // Virtual register must be a def undef foo:ssub_0 operand.
5333     if (!MO.getSubReg() || MI.readsVirtualRegister(Reg))
5334       return 0;
5335   } else if (ARM::SPRRegClass.contains(Reg)) {
5336     // Physical register: MI must define the full D-reg.
5337     unsigned DReg = TRI->getMatchingSuperReg(Reg, ARM::ssub_0,
5338                                              &ARM::DPRRegClass);
5339     if (!DReg || !MI.definesRegister(DReg, TRI))
5340       return 0;
5341   }
5342 
5343   // MI has an unwanted D-register dependency.
5344   // Avoid defs in the previous N instructrions.
5345   return PartialUpdateClearance;
5346 }
5347 
5348 // Break a partial register dependency after getPartialRegUpdateClearance
5349 // returned non-zero.
5350 void ARMBaseInstrInfo::breakPartialRegDependency(
5351     MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const {
5352   assert(OpNum < MI.getDesc().getNumDefs() && "OpNum is not a def");
5353   assert(TRI && "Need TRI instance");
5354 
5355   const MachineOperand &MO = MI.getOperand(OpNum);
5356   Register Reg = MO.getReg();
5357   assert(Register::isPhysicalRegister(Reg) &&
5358          "Can't break virtual register dependencies.");
5359   unsigned DReg = Reg;
5360 
5361   // If MI defines an S-reg, find the corresponding D super-register.
5362   if (ARM::SPRRegClass.contains(Reg)) {
5363     DReg = ARM::D0 + (Reg - ARM::S0) / 2;
5364     assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken");
5365   }
5366 
5367   assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps");
5368   assert(MI.definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg");
5369 
5370   // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines
5371   // the full D-register by loading the same value to both lanes.  The
5372   // instruction is micro-coded with 2 uops, so don't do this until we can
5373   // properly schedule micro-coded instructions.  The dispatcher stalls cause
5374   // too big regressions.
5375 
5376   // Insert the dependency-breaking FCONSTD before MI.
5377   // 96 is the encoding of 0.5, but the actual value doesn't matter here.
5378   BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::FCONSTD), DReg)
5379       .addImm(96)
5380       .add(predOps(ARMCC::AL));
5381   MI.addRegisterKilled(DReg, TRI, true);
5382 }
5383 
5384 bool ARMBaseInstrInfo::hasNOP() const {
5385   return Subtarget.getFeatureBits()[ARM::HasV6KOps];
5386 }
5387 
5388 bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const {
5389   if (MI->getNumOperands() < 4)
5390     return true;
5391   unsigned ShOpVal = MI->getOperand(3).getImm();
5392   unsigned ShImm = ARM_AM::getSORegOffset(ShOpVal);
5393   // Swift supports faster shifts for: lsl 2, lsl 1, and lsr 1.
5394   if ((ShImm == 1 && ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsr) ||
5395       ((ShImm == 1 || ShImm == 2) &&
5396        ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsl))
5397     return true;
5398 
5399   return false;
5400 }
5401 
5402 bool ARMBaseInstrInfo::getRegSequenceLikeInputs(
5403     const MachineInstr &MI, unsigned DefIdx,
5404     SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
5405   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5406   assert(MI.isRegSequenceLike() && "Invalid kind of instruction");
5407 
5408   switch (MI.getOpcode()) {
5409   case ARM::VMOVDRR:
5410     // dX = VMOVDRR rY, rZ
5411     // is the same as:
5412     // dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1
5413     // Populate the InputRegs accordingly.
5414     // rY
5415     const MachineOperand *MOReg = &MI.getOperand(1);
5416     if (!MOReg->isUndef())
5417       InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(),
5418                                               MOReg->getSubReg(), ARM::ssub_0));
5419     // rZ
5420     MOReg = &MI.getOperand(2);
5421     if (!MOReg->isUndef())
5422       InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(),
5423                                               MOReg->getSubReg(), ARM::ssub_1));
5424     return true;
5425   }
5426   llvm_unreachable("Target dependent opcode missing");
5427 }
5428 
5429 bool ARMBaseInstrInfo::getExtractSubregLikeInputs(
5430     const MachineInstr &MI, unsigned DefIdx,
5431     RegSubRegPairAndIdx &InputReg) const {
5432   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5433   assert(MI.isExtractSubregLike() && "Invalid kind of instruction");
5434 
5435   switch (MI.getOpcode()) {
5436   case ARM::VMOVRRD:
5437     // rX, rY = VMOVRRD dZ
5438     // is the same as:
5439     // rX = EXTRACT_SUBREG dZ, ssub_0
5440     // rY = EXTRACT_SUBREG dZ, ssub_1
5441     const MachineOperand &MOReg = MI.getOperand(2);
5442     if (MOReg.isUndef())
5443       return false;
5444     InputReg.Reg = MOReg.getReg();
5445     InputReg.SubReg = MOReg.getSubReg();
5446     InputReg.SubIdx = DefIdx == 0 ? ARM::ssub_0 : ARM::ssub_1;
5447     return true;
5448   }
5449   llvm_unreachable("Target dependent opcode missing");
5450 }
5451 
5452 bool ARMBaseInstrInfo::getInsertSubregLikeInputs(
5453     const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg,
5454     RegSubRegPairAndIdx &InsertedReg) const {
5455   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5456   assert(MI.isInsertSubregLike() && "Invalid kind of instruction");
5457 
5458   switch (MI.getOpcode()) {
5459   case ARM::VSETLNi32:
5460   case ARM::MVE_VMOV_to_lane_32:
5461     // dX = VSETLNi32 dY, rZ, imm
5462     // qX = MVE_VMOV_to_lane_32 qY, rZ, imm
5463     const MachineOperand &MOBaseReg = MI.getOperand(1);
5464     const MachineOperand &MOInsertedReg = MI.getOperand(2);
5465     if (MOInsertedReg.isUndef())
5466       return false;
5467     const MachineOperand &MOIndex = MI.getOperand(3);
5468     BaseReg.Reg = MOBaseReg.getReg();
5469     BaseReg.SubReg = MOBaseReg.getSubReg();
5470 
5471     InsertedReg.Reg = MOInsertedReg.getReg();
5472     InsertedReg.SubReg = MOInsertedReg.getSubReg();
5473     InsertedReg.SubIdx = ARM::ssub_0 + MOIndex.getImm();
5474     return true;
5475   }
5476   llvm_unreachable("Target dependent opcode missing");
5477 }
5478 
5479 std::pair<unsigned, unsigned>
5480 ARMBaseInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5481   const unsigned Mask = ARMII::MO_OPTION_MASK;
5482   return std::make_pair(TF & Mask, TF & ~Mask);
5483 }
5484 
5485 ArrayRef<std::pair<unsigned, const char *>>
5486 ARMBaseInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5487   using namespace ARMII;
5488 
5489   static const std::pair<unsigned, const char *> TargetFlags[] = {
5490       {MO_LO16, "arm-lo16"}, {MO_HI16, "arm-hi16"}};
5491   return makeArrayRef(TargetFlags);
5492 }
5493 
5494 ArrayRef<std::pair<unsigned, const char *>>
5495 ARMBaseInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5496   using namespace ARMII;
5497 
5498   static const std::pair<unsigned, const char *> TargetFlags[] = {
5499       {MO_COFFSTUB, "arm-coffstub"},
5500       {MO_GOT, "arm-got"},
5501       {MO_SBREL, "arm-sbrel"},
5502       {MO_DLLIMPORT, "arm-dllimport"},
5503       {MO_SECREL, "arm-secrel"},
5504       {MO_NONLAZY, "arm-nonlazy"}};
5505   return makeArrayRef(TargetFlags);
5506 }
5507 
5508 Optional<RegImmPair> ARMBaseInstrInfo::isAddImmediate(const MachineInstr &MI,
5509                                                       Register Reg) const {
5510   int Sign = 1;
5511   unsigned Opcode = MI.getOpcode();
5512   int64_t Offset = 0;
5513 
5514   // TODO: Handle cases where Reg is a super- or sub-register of the
5515   // destination register.
5516   const MachineOperand &Op0 = MI.getOperand(0);
5517   if (!Op0.isReg() || Reg != Op0.getReg())
5518     return None;
5519 
5520   // We describe SUBri or ADDri instructions.
5521   if (Opcode == ARM::SUBri)
5522     Sign = -1;
5523   else if (Opcode != ARM::ADDri)
5524     return None;
5525 
5526   // TODO: Third operand can be global address (usually some string). Since
5527   //       strings can be relocated we cannot calculate their offsets for
5528   //       now.
5529   if (!MI.getOperand(1).isReg() || !MI.getOperand(2).isImm())
5530     return None;
5531 
5532   Offset = MI.getOperand(2).getImm() * Sign;
5533   return RegImmPair{MI.getOperand(1).getReg(), Offset};
5534 }
5535 
5536 bool llvm::registerDefinedBetween(unsigned Reg,
5537                                   MachineBasicBlock::iterator From,
5538                                   MachineBasicBlock::iterator To,
5539                                   const TargetRegisterInfo *TRI) {
5540   for (auto I = From; I != To; ++I)
5541     if (I->modifiesRegister(Reg, TRI))
5542       return true;
5543   return false;
5544 }
5545 
5546 MachineInstr *llvm::findCMPToFoldIntoCBZ(MachineInstr *Br,
5547                                          const TargetRegisterInfo *TRI) {
5548   // Search backwards to the instruction that defines CSPR. This may or not
5549   // be a CMP, we check that after this loop. If we find another instruction
5550   // that reads cpsr, we return nullptr.
5551   MachineBasicBlock::iterator CmpMI = Br;
5552   while (CmpMI != Br->getParent()->begin()) {
5553     --CmpMI;
5554     if (CmpMI->modifiesRegister(ARM::CPSR, TRI))
5555       break;
5556     if (CmpMI->readsRegister(ARM::CPSR, TRI))
5557       break;
5558   }
5559 
5560   // Check that this inst is a CMP r[0-7], #0 and that the register
5561   // is not redefined between the cmp and the br.
5562   if (CmpMI->getOpcode() != ARM::tCMPi8 && CmpMI->getOpcode() != ARM::t2CMPri)
5563     return nullptr;
5564   Register Reg = CmpMI->getOperand(0).getReg();
5565   Register PredReg;
5566   ARMCC::CondCodes Pred = getInstrPredicate(*CmpMI, PredReg);
5567   if (Pred != ARMCC::AL || CmpMI->getOperand(1).getImm() != 0)
5568     return nullptr;
5569   if (!isARMLowRegister(Reg))
5570     return nullptr;
5571   if (registerDefinedBetween(Reg, CmpMI->getNextNode(), Br, TRI))
5572     return nullptr;
5573 
5574   return &*CmpMI;
5575 }
5576 
5577 unsigned llvm::ConstantMaterializationCost(unsigned Val,
5578                                            const ARMSubtarget *Subtarget,
5579                                            bool ForCodesize) {
5580   if (Subtarget->isThumb()) {
5581     if (Val <= 255) // MOV
5582       return ForCodesize ? 2 : 1;
5583     if (Subtarget->hasV6T2Ops() && (Val <= 0xffff ||                    // MOV
5584                                     ARM_AM::getT2SOImmVal(Val) != -1 || // MOVW
5585                                     ARM_AM::getT2SOImmVal(~Val) != -1)) // MVN
5586       return ForCodesize ? 4 : 1;
5587     if (Val <= 510) // MOV + ADDi8
5588       return ForCodesize ? 4 : 2;
5589     if (~Val <= 255) // MOV + MVN
5590       return ForCodesize ? 4 : 2;
5591     if (ARM_AM::isThumbImmShiftedVal(Val)) // MOV + LSL
5592       return ForCodesize ? 4 : 2;
5593   } else {
5594     if (ARM_AM::getSOImmVal(Val) != -1) // MOV
5595       return ForCodesize ? 4 : 1;
5596     if (ARM_AM::getSOImmVal(~Val) != -1) // MVN
5597       return ForCodesize ? 4 : 1;
5598     if (Subtarget->hasV6T2Ops() && Val <= 0xffff) // MOVW
5599       return ForCodesize ? 4 : 1;
5600     if (ARM_AM::isSOImmTwoPartVal(Val)) // two instrs
5601       return ForCodesize ? 8 : 2;
5602     if (ARM_AM::isSOImmTwoPartValNeg(Val)) // two instrs
5603       return ForCodesize ? 8 : 2;
5604   }
5605   if (Subtarget->useMovt()) // MOVW + MOVT
5606     return ForCodesize ? 8 : 2;
5607   return ForCodesize ? 8 : 3; // Literal pool load
5608 }
5609 
5610 bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2,
5611                                                const ARMSubtarget *Subtarget,
5612                                                bool ForCodesize) {
5613   // Check with ForCodesize
5614   unsigned Cost1 = ConstantMaterializationCost(Val1, Subtarget, ForCodesize);
5615   unsigned Cost2 = ConstantMaterializationCost(Val2, Subtarget, ForCodesize);
5616   if (Cost1 < Cost2)
5617     return true;
5618   if (Cost1 > Cost2)
5619     return false;
5620 
5621   // If they are equal, try with !ForCodesize
5622   return ConstantMaterializationCost(Val1, Subtarget, !ForCodesize) <
5623          ConstantMaterializationCost(Val2, Subtarget, !ForCodesize);
5624 }
5625 
5626 /// Constants defining how certain sequences should be outlined.
5627 /// This encompasses how an outlined function should be called, and what kind of
5628 /// frame should be emitted for that outlined function.
5629 ///
5630 /// \p MachineOutlinerTailCall implies that the function is being created from
5631 /// a sequence of instructions ending in a return.
5632 ///
5633 /// That is,
5634 ///
5635 /// I1                                OUTLINED_FUNCTION:
5636 /// I2    --> B OUTLINED_FUNCTION     I1
5637 /// BX LR                             I2
5638 ///                                   BX LR
5639 ///
5640 /// +-------------------------+--------+-----+
5641 /// |                         | Thumb2 | ARM |
5642 /// +-------------------------+--------+-----+
5643 /// | Call overhead in Bytes  |      4 |   4 |
5644 /// | Frame overhead in Bytes |      0 |   0 |
5645 /// | Stack fixup required    |     No |  No |
5646 /// +-------------------------+--------+-----+
5647 ///
5648 /// \p MachineOutlinerThunk implies that the function is being created from
5649 /// a sequence of instructions ending in a call. The outlined function is
5650 /// called with a BL instruction, and the outlined function tail-calls the
5651 /// original call destination.
5652 ///
5653 /// That is,
5654 ///
5655 /// I1                                OUTLINED_FUNCTION:
5656 /// I2   --> BL OUTLINED_FUNCTION     I1
5657 /// BL f                              I2
5658 ///                                   B f
5659 ///
5660 /// +-------------------------+--------+-----+
5661 /// |                         | Thumb2 | ARM |
5662 /// +-------------------------+--------+-----+
5663 /// | Call overhead in Bytes  |      4 |   4 |
5664 /// | Frame overhead in Bytes |      0 |   0 |
5665 /// | Stack fixup required    |     No |  No |
5666 /// +-------------------------+--------+-----+
5667 ///
5668 /// \p MachineOutlinerNoLRSave implies that the function should be called using
5669 /// a BL instruction, but doesn't require LR to be saved and restored. This
5670 /// happens when LR is known to be dead.
5671 ///
5672 /// That is,
5673 ///
5674 /// I1                                OUTLINED_FUNCTION:
5675 /// I2 --> BL OUTLINED_FUNCTION       I1
5676 /// I3                                I2
5677 ///                                   I3
5678 ///                                   BX LR
5679 ///
5680 /// +-------------------------+--------+-----+
5681 /// |                         | Thumb2 | ARM |
5682 /// +-------------------------+--------+-----+
5683 /// | Call overhead in Bytes  |      4 |   4 |
5684 /// | Frame overhead in Bytes |      2 |   4 |
5685 /// | Stack fixup required    |     No |  No |
5686 /// +-------------------------+--------+-----+
5687 ///
5688 /// \p MachineOutlinerRegSave implies that the function should be called with a
5689 /// save and restore of LR to an available register. This allows us to avoid
5690 /// stack fixups. Note that this outlining variant is compatible with the
5691 /// NoLRSave case.
5692 ///
5693 /// That is,
5694 ///
5695 /// I1     Save LR                    OUTLINED_FUNCTION:
5696 /// I2 --> BL OUTLINED_FUNCTION       I1
5697 /// I3     Restore LR                 I2
5698 ///                                   I3
5699 ///                                   BX LR
5700 ///
5701 /// +-------------------------+--------+-----+
5702 /// |                         | Thumb2 | ARM |
5703 /// +-------------------------+--------+-----+
5704 /// | Call overhead in Bytes  |      8 |  12 |
5705 /// | Frame overhead in Bytes |      2 |   4 |
5706 /// | Stack fixup required    |     No |  No |
5707 /// +-------------------------+--------+-----+
5708 ///
5709 /// \p MachineOutlinerDefault implies that the function should be called with
5710 /// a save and restore of LR to the stack.
5711 ///
5712 /// That is,
5713 ///
5714 /// I1     Save LR                    OUTLINED_FUNCTION:
5715 /// I2 --> BL OUTLINED_FUNCTION       I1
5716 /// I3     Restore LR                 I2
5717 ///                                   I3
5718 ///                                   BX LR
5719 ///
5720 /// +-------------------------+--------+-----+
5721 /// |                         | Thumb2 | ARM |
5722 /// +-------------------------+--------+-----+
5723 /// | Call overhead in Bytes  |      8 |  12 |
5724 /// | Frame overhead in Bytes |      2 |   4 |
5725 /// | Stack fixup required    |    Yes | Yes |
5726 /// +-------------------------+--------+-----+
5727 
5728 enum MachineOutlinerClass {
5729   MachineOutlinerTailCall,
5730   MachineOutlinerThunk,
5731   MachineOutlinerNoLRSave,
5732   MachineOutlinerRegSave,
5733   MachineOutlinerDefault
5734 };
5735 
5736 enum MachineOutlinerMBBFlags {
5737   LRUnavailableSomewhere = 0x2,
5738   HasCalls = 0x4,
5739   UnsafeRegsDead = 0x8
5740 };
5741 
5742 struct OutlinerCosts {
5743   int CallTailCall;
5744   int FrameTailCall;
5745   int CallThunk;
5746   int FrameThunk;
5747   int CallNoLRSave;
5748   int FrameNoLRSave;
5749   int CallRegSave;
5750   int FrameRegSave;
5751   int CallDefault;
5752   int FrameDefault;
5753   int SaveRestoreLROnStack;
5754 
5755   OutlinerCosts(const ARMSubtarget &target)
5756       : CallTailCall(target.isThumb() ? 4 : 4),
5757         FrameTailCall(target.isThumb() ? 0 : 0),
5758         CallThunk(target.isThumb() ? 4 : 4),
5759         FrameThunk(target.isThumb() ? 0 : 0),
5760         CallNoLRSave(target.isThumb() ? 4 : 4),
5761         FrameNoLRSave(target.isThumb() ? 2 : 4),
5762         CallRegSave(target.isThumb() ? 8 : 12),
5763         FrameRegSave(target.isThumb() ? 2 : 4),
5764         CallDefault(target.isThumb() ? 8 : 12),
5765         FrameDefault(target.isThumb() ? 2 : 4),
5766         SaveRestoreLROnStack(target.isThumb() ? 8 : 8) {}
5767 };
5768 
5769 unsigned
5770 ARMBaseInstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const {
5771   assert(C.LRUWasSet && "LRU wasn't set?");
5772   MachineFunction *MF = C.getMF();
5773   const ARMBaseRegisterInfo *ARI = static_cast<const ARMBaseRegisterInfo *>(
5774       MF->getSubtarget().getRegisterInfo());
5775 
5776   BitVector regsReserved = ARI->getReservedRegs(*MF);
5777   // Check if there is an available register across the sequence that we can
5778   // use.
5779   for (unsigned Reg : ARM::rGPRRegClass) {
5780     if (!(Reg < regsReserved.size() && regsReserved.test(Reg)) &&
5781         Reg != ARM::LR &&  // LR is not reserved, but don't use it.
5782         Reg != ARM::R12 && // R12 is not guaranteed to be preserved.
5783         C.LRU.available(Reg) && C.UsedInSequence.available(Reg))
5784       return Reg;
5785   }
5786 
5787   // No suitable register. Return 0.
5788   return 0u;
5789 }
5790 
5791 // Compute liveness of LR at the point after the interval [I, E), which
5792 // denotes a *backward* iteration through instructions. Used only for return
5793 // basic blocks, which do not end with a tail call.
5794 static bool isLRAvailable(const TargetRegisterInfo &TRI,
5795                           MachineBasicBlock::reverse_iterator I,
5796                           MachineBasicBlock::reverse_iterator E) {
5797   // At the end of the function LR dead.
5798   bool Live = false;
5799   for (; I != E; ++I) {
5800     const MachineInstr &MI = *I;
5801 
5802     // Check defs of LR.
5803     if (MI.modifiesRegister(ARM::LR, &TRI))
5804       Live = false;
5805 
5806     // Check uses of LR.
5807     unsigned Opcode = MI.getOpcode();
5808     if (Opcode == ARM::BX_RET || Opcode == ARM::MOVPCLR ||
5809         Opcode == ARM::SUBS_PC_LR || Opcode == ARM::tBX_RET ||
5810         Opcode == ARM::tBXNS_RET) {
5811       // These instructions use LR, but it's not an (explicit or implicit)
5812       // operand.
5813       Live = true;
5814       continue;
5815     }
5816     if (MI.readsRegister(ARM::LR, &TRI))
5817       Live = true;
5818   }
5819   return !Live;
5820 }
5821 
5822 outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo(
5823     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
5824   outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5825   unsigned SequenceSize =
5826       std::accumulate(FirstCand.front(), std::next(FirstCand.back()), 0,
5827                       [this](unsigned Sum, const MachineInstr &MI) {
5828                         return Sum + getInstSizeInBytes(MI);
5829                       });
5830 
5831   // Properties about candidate MBBs that hold for all of them.
5832   unsigned FlagsSetInAll = 0xF;
5833 
5834   // Compute liveness information for each candidate, and set FlagsSetInAll.
5835   const TargetRegisterInfo &TRI = getRegisterInfo();
5836   std::for_each(
5837       RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
5838       [&FlagsSetInAll](outliner::Candidate &C) { FlagsSetInAll &= C.Flags; });
5839 
5840   // According to the ARM Procedure Call Standard, the following are
5841   // undefined on entry/exit from a function call:
5842   //
5843   // * Register R12(IP),
5844   // * Condition codes (and thus the CPSR register)
5845   //
5846   // Since we control the instructions which are part of the outlined regions
5847   // we don't need to be fully compliant with the AAPCS, but we have to
5848   // guarantee that if a veneer is inserted at link time the code is still
5849   // correct.  Because of this, we can't outline any sequence of instructions
5850   // where one of these registers is live into/across it. Thus, we need to
5851   // delete those candidates.
5852   auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
5853     // If the unsafe registers in this block are all dead, then we don't need
5854     // to compute liveness here.
5855     if (C.Flags & UnsafeRegsDead)
5856       return false;
5857     C.initLRU(TRI);
5858     LiveRegUnits LRU = C.LRU;
5859     return (!LRU.available(ARM::R12) || !LRU.available(ARM::CPSR));
5860   };
5861 
5862   // Are there any candidates where those registers are live?
5863   if (!(FlagsSetInAll & UnsafeRegsDead)) {
5864     // Erase every candidate that violates the restrictions above. (It could be
5865     // true that we have viable candidates, so it's not worth bailing out in
5866     // the case that, say, 1 out of 20 candidates violate the restructions.)
5867     llvm::erase_if(RepeatedSequenceLocs, CantGuaranteeValueAcrossCall);
5868 
5869     // If the sequence doesn't have enough candidates left, then we're done.
5870     if (RepeatedSequenceLocs.size() < 2)
5871       return outliner::OutlinedFunction();
5872   }
5873 
5874   // We expect the majority of the outlining candidates to be in consensus with
5875   // regard to return address sign and authentication, and branch target
5876   // enforcement, in other words, partitioning according to all the four
5877   // possible combinations of PAC-RET and BTI is going to yield one big subset
5878   // and three small (likely empty) subsets. That allows us to cull incompatible
5879   // candidates separately for PAC-RET and BTI.
5880 
5881   // Partition the candidates in two sets: one with BTI enabled and one with BTI
5882   // disabled. Remove the candidates from the smaller set. If they are the same
5883   // number prefer the non-BTI ones for outlining, since they have less
5884   // overhead.
5885   auto NoBTI =
5886       llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) {
5887         const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
5888         return AFI.branchTargetEnforcement();
5889       });
5890   if (std::distance(RepeatedSequenceLocs.begin(), NoBTI) >
5891       std::distance(NoBTI, RepeatedSequenceLocs.end()))
5892     RepeatedSequenceLocs.erase(NoBTI, RepeatedSequenceLocs.end());
5893   else
5894     RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoBTI);
5895 
5896   if (RepeatedSequenceLocs.size() < 2)
5897     return outliner::OutlinedFunction();
5898 
5899   // Likewise, partition the candidates according to PAC-RET enablement.
5900   auto NoPAC =
5901       llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) {
5902         const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
5903         // If the function happens to not spill the LR, do not disqualify it
5904         // from the outlining.
5905         return AFI.shouldSignReturnAddress(true);
5906       });
5907   if (std::distance(RepeatedSequenceLocs.begin(), NoPAC) >
5908       std::distance(NoPAC, RepeatedSequenceLocs.end()))
5909     RepeatedSequenceLocs.erase(NoPAC, RepeatedSequenceLocs.end());
5910   else
5911     RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoPAC);
5912 
5913   if (RepeatedSequenceLocs.size() < 2)
5914     return outliner::OutlinedFunction();
5915 
5916   // At this point, we have only "safe" candidates to outline. Figure out
5917   // frame + call instruction information.
5918 
5919   unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back()->getOpcode();
5920 
5921   // Helper lambda which sets call information for every candidate.
5922   auto SetCandidateCallInfo =
5923       [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
5924         for (outliner::Candidate &C : RepeatedSequenceLocs)
5925           C.setCallInfo(CallID, NumBytesForCall);
5926       };
5927 
5928   OutlinerCosts Costs(Subtarget);
5929 
5930   const auto &SomeMFI =
5931       *RepeatedSequenceLocs.front().getMF()->getInfo<ARMFunctionInfo>();
5932   // Adjust costs to account for the BTI instructions.
5933   if (SomeMFI.branchTargetEnforcement()) {
5934     Costs.FrameDefault += 4;
5935     Costs.FrameNoLRSave += 4;
5936     Costs.FrameRegSave += 4;
5937     Costs.FrameTailCall += 4;
5938     Costs.FrameThunk += 4;
5939   }
5940 
5941   // Adjust costs to account for sign and authentication instructions.
5942   if (SomeMFI.shouldSignReturnAddress(true)) {
5943     Costs.CallDefault += 8;          // +PAC instr, +AUT instr
5944     Costs.SaveRestoreLROnStack += 8; // +PAC instr, +AUT instr
5945   }
5946 
5947   unsigned FrameID = MachineOutlinerDefault;
5948   unsigned NumBytesToCreateFrame = Costs.FrameDefault;
5949 
5950   // If the last instruction in any candidate is a terminator, then we should
5951   // tail call all of the candidates.
5952   if (RepeatedSequenceLocs[0].back()->isTerminator()) {
5953     FrameID = MachineOutlinerTailCall;
5954     NumBytesToCreateFrame = Costs.FrameTailCall;
5955     SetCandidateCallInfo(MachineOutlinerTailCall, Costs.CallTailCall);
5956   } else if (LastInstrOpcode == ARM::BL || LastInstrOpcode == ARM::BLX ||
5957              LastInstrOpcode == ARM::BLX_noip || LastInstrOpcode == ARM::tBL ||
5958              LastInstrOpcode == ARM::tBLXr ||
5959              LastInstrOpcode == ARM::tBLXr_noip ||
5960              LastInstrOpcode == ARM::tBLXi) {
5961     FrameID = MachineOutlinerThunk;
5962     NumBytesToCreateFrame = Costs.FrameThunk;
5963     SetCandidateCallInfo(MachineOutlinerThunk, Costs.CallThunk);
5964   } else {
5965     // We need to decide how to emit calls + frames. We can always emit the same
5966     // frame if we don't need to save to the stack. If we have to save to the
5967     // stack, then we need a different frame.
5968     unsigned NumBytesNoStackCalls = 0;
5969     std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
5970 
5971     for (outliner::Candidate &C : RepeatedSequenceLocs) {
5972       C.initLRU(TRI);
5973       // LR liveness is overestimated in return blocks, unless they end with a
5974       // tail call.
5975       const auto Last = C.getMBB()->rbegin();
5976       const bool LRIsAvailable =
5977           C.getMBB()->isReturnBlock() && !Last->isCall()
5978               ? isLRAvailable(TRI, Last,
5979                               (MachineBasicBlock::reverse_iterator)C.front())
5980               : C.LRU.available(ARM::LR);
5981       if (LRIsAvailable) {
5982         FrameID = MachineOutlinerNoLRSave;
5983         NumBytesNoStackCalls += Costs.CallNoLRSave;
5984         C.setCallInfo(MachineOutlinerNoLRSave, Costs.CallNoLRSave);
5985         CandidatesWithoutStackFixups.push_back(C);
5986       }
5987 
5988       // Is an unused register available? If so, we won't modify the stack, so
5989       // we can outline with the same frame type as those that don't save LR.
5990       else if (findRegisterToSaveLRTo(C)) {
5991         FrameID = MachineOutlinerRegSave;
5992         NumBytesNoStackCalls += Costs.CallRegSave;
5993         C.setCallInfo(MachineOutlinerRegSave, Costs.CallRegSave);
5994         CandidatesWithoutStackFixups.push_back(C);
5995       }
5996 
5997       // Is SP used in the sequence at all? If not, we don't have to modify
5998       // the stack, so we are guaranteed to get the same frame.
5999       else if (C.UsedInSequence.available(ARM::SP)) {
6000         NumBytesNoStackCalls += Costs.CallDefault;
6001         C.setCallInfo(MachineOutlinerDefault, Costs.CallDefault);
6002         CandidatesWithoutStackFixups.push_back(C);
6003       }
6004 
6005       // If we outline this, we need to modify the stack. Pretend we don't
6006       // outline this by saving all of its bytes.
6007       else
6008         NumBytesNoStackCalls += SequenceSize;
6009     }
6010 
6011     // If there are no places where we have to save LR, then note that we don't
6012     // have to update the stack. Otherwise, give every candidate the default
6013     // call type
6014     if (NumBytesNoStackCalls <=
6015         RepeatedSequenceLocs.size() * Costs.CallDefault) {
6016       RepeatedSequenceLocs = CandidatesWithoutStackFixups;
6017       FrameID = MachineOutlinerNoLRSave;
6018     } else
6019       SetCandidateCallInfo(MachineOutlinerDefault, Costs.CallDefault);
6020   }
6021 
6022   // Does every candidate's MBB contain a call?  If so, then we might have a
6023   // call in the range.
6024   if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
6025     // check if the range contains a call.  These require a save + restore of
6026     // the link register.
6027     if (std::any_of(FirstCand.front(), FirstCand.back(),
6028                     [](const MachineInstr &MI) { return MI.isCall(); }))
6029       NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
6030 
6031     // Handle the last instruction separately.  If it is tail call, then the
6032     // last instruction is a call, we don't want to save + restore in this
6033     // case.  However, it could be possible that the last instruction is a
6034     // call without it being valid to tail call this sequence.  We should
6035     // consider this as well.
6036     else if (FrameID != MachineOutlinerThunk &&
6037              FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall())
6038       NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
6039   }
6040 
6041   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
6042                                     NumBytesToCreateFrame, FrameID);
6043 }
6044 
6045 bool ARMBaseInstrInfo::checkAndUpdateStackOffset(MachineInstr *MI,
6046                                                  int64_t Fixup,
6047                                                  bool Updt) const {
6048   int SPIdx = MI->findRegisterUseOperandIdx(ARM::SP);
6049   unsigned AddrMode = (MI->getDesc().TSFlags & ARMII::AddrModeMask);
6050   if (SPIdx < 0)
6051     // No SP operand
6052     return true;
6053   else if (SPIdx != 1 && (AddrMode != ARMII::AddrModeT2_i8s4 || SPIdx != 2))
6054     // If SP is not the base register we can't do much
6055     return false;
6056 
6057   // Stack might be involved but addressing mode doesn't handle any offset.
6058   // Rq: AddrModeT1_[1|2|4] don't operate on SP
6059   if (AddrMode == ARMII::AddrMode1 ||       // Arithmetic instructions
6060       AddrMode == ARMII::AddrMode4 ||       // Load/Store Multiple
6061       AddrMode == ARMII::AddrMode6 ||       // Neon Load/Store Multiple
6062       AddrMode == ARMII::AddrModeT2_so ||   // SP can't be used as based register
6063       AddrMode == ARMII::AddrModeT2_pc ||   // PCrel access
6064       AddrMode == ARMII::AddrMode2 ||       // Used by PRE and POST indexed LD/ST
6065       AddrMode == ARMII::AddrModeT2_i7 ||   // v8.1-M MVE
6066       AddrMode == ARMII::AddrModeT2_i7s2 || // v8.1-M MVE
6067       AddrMode == ARMII::AddrModeT2_i7s4 || // v8.1-M sys regs VLDR/VSTR
6068       AddrMode == ARMII::AddrModeNone ||
6069       AddrMode == ARMII::AddrModeT2_i8 ||   // Pre/Post inc instructions
6070       AddrMode == ARMII::AddrModeT2_i8neg)  // Always negative imm
6071     return false;
6072 
6073   unsigned NumOps = MI->getDesc().getNumOperands();
6074   unsigned ImmIdx = NumOps - 3;
6075 
6076   const MachineOperand &Offset = MI->getOperand(ImmIdx);
6077   assert(Offset.isImm() && "Is not an immediate");
6078   int64_t OffVal = Offset.getImm();
6079 
6080   if (OffVal < 0)
6081     // Don't override data if the are below SP.
6082     return false;
6083 
6084   unsigned NumBits = 0;
6085   unsigned Scale = 1;
6086 
6087   switch (AddrMode) {
6088   case ARMII::AddrMode3:
6089     if (ARM_AM::getAM3Op(OffVal) == ARM_AM::sub)
6090       return false;
6091     OffVal = ARM_AM::getAM3Offset(OffVal);
6092     NumBits = 8;
6093     break;
6094   case ARMII::AddrMode5:
6095     if (ARM_AM::getAM5Op(OffVal) == ARM_AM::sub)
6096       return false;
6097     OffVal = ARM_AM::getAM5Offset(OffVal);
6098     NumBits = 8;
6099     Scale = 4;
6100     break;
6101   case ARMII::AddrMode5FP16:
6102     if (ARM_AM::getAM5FP16Op(OffVal) == ARM_AM::sub)
6103       return false;
6104     OffVal = ARM_AM::getAM5FP16Offset(OffVal);
6105     NumBits = 8;
6106     Scale = 2;
6107     break;
6108   case ARMII::AddrModeT2_i8pos:
6109     NumBits = 8;
6110     break;
6111   case ARMII::AddrModeT2_i8s4:
6112     // FIXME: Values are already scaled in this addressing mode.
6113     assert((Fixup & 3) == 0 && "Can't encode this offset!");
6114     NumBits = 10;
6115     break;
6116   case ARMII::AddrModeT2_ldrex:
6117     NumBits = 8;
6118     Scale = 4;
6119     break;
6120   case ARMII::AddrModeT2_i12:
6121   case ARMII::AddrMode_i12:
6122     NumBits = 12;
6123     break;
6124   case ARMII::AddrModeT1_s: // SP-relative LD/ST
6125     NumBits = 8;
6126     Scale = 4;
6127     break;
6128   default:
6129     llvm_unreachable("Unsupported addressing mode!");
6130   }
6131   // Make sure the offset is encodable for instructions that scale the
6132   // immediate.
6133   assert(((OffVal * Scale + Fixup) & (Scale - 1)) == 0 &&
6134          "Can't encode this offset!");
6135   OffVal += Fixup / Scale;
6136 
6137   unsigned Mask = (1 << NumBits) - 1;
6138 
6139   if (OffVal <= Mask) {
6140     if (Updt)
6141       MI->getOperand(ImmIdx).setImm(OffVal);
6142     return true;
6143   }
6144 
6145   return false;
6146 }
6147 
6148 void ARMBaseInstrInfo::mergeOutliningCandidateAttributes(
6149     Function &F, std::vector<outliner::Candidate> &Candidates) const {
6150   outliner::Candidate &C = Candidates.front();
6151   // branch-target-enforcement is guaranteed to be consistent between all
6152   // candidates, so we only need to look at one.
6153   const Function &CFn = C.getMF()->getFunction();
6154   if (CFn.hasFnAttribute("branch-target-enforcement"))
6155     F.addFnAttr(CFn.getFnAttribute("branch-target-enforcement"));
6156 
6157   ARMGenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
6158 }
6159 
6160 bool ARMBaseInstrInfo::isFunctionSafeToOutlineFrom(
6161     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
6162   const Function &F = MF.getFunction();
6163 
6164   // Can F be deduplicated by the linker? If it can, don't outline from it.
6165   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
6166     return false;
6167 
6168   // Don't outline from functions with section markings; the program could
6169   // expect that all the code is in the named section.
6170   // FIXME: Allow outlining from multiple functions with the same section
6171   // marking.
6172   if (F.hasSection())
6173     return false;
6174 
6175   // FIXME: Thumb1 outlining is not handled
6176   if (MF.getInfo<ARMFunctionInfo>()->isThumb1OnlyFunction())
6177     return false;
6178 
6179   // It's safe to outline from MF.
6180   return true;
6181 }
6182 
6183 bool ARMBaseInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
6184                                               unsigned &Flags) const {
6185   // Check if LR is available through all of the MBB. If it's not, then set
6186   // a flag.
6187   assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
6188          "Suitable Machine Function for outlining must track liveness");
6189 
6190   LiveRegUnits LRU(getRegisterInfo());
6191 
6192   std::for_each(MBB.rbegin(), MBB.rend(),
6193                 [&LRU](MachineInstr &MI) { LRU.accumulate(MI); });
6194 
6195   // Check if each of the unsafe registers are available...
6196   bool R12AvailableInBlock = LRU.available(ARM::R12);
6197   bool CPSRAvailableInBlock = LRU.available(ARM::CPSR);
6198 
6199   // If all of these are dead (and not live out), we know we don't have to check
6200   // them later.
6201   if (R12AvailableInBlock && CPSRAvailableInBlock)
6202     Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
6203 
6204   // Now, add the live outs to the set.
6205   LRU.addLiveOuts(MBB);
6206 
6207   // If any of these registers is available in the MBB, but also a live out of
6208   // the block, then we know outlining is unsafe.
6209   if (R12AvailableInBlock && !LRU.available(ARM::R12))
6210     return false;
6211   if (CPSRAvailableInBlock && !LRU.available(ARM::CPSR))
6212     return false;
6213 
6214   // Check if there's a call inside this MachineBasicBlock.  If there is, then
6215   // set a flag.
6216   if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); }))
6217     Flags |= MachineOutlinerMBBFlags::HasCalls;
6218 
6219   // LR liveness is overestimated in return blocks.
6220 
6221   bool LRIsAvailable =
6222       MBB.isReturnBlock() && !MBB.back().isCall()
6223           ? isLRAvailable(getRegisterInfo(), MBB.rbegin(), MBB.rend())
6224           : LRU.available(ARM::LR);
6225   if (!LRIsAvailable)
6226     Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
6227 
6228   return true;
6229 }
6230 
6231 outliner::InstrType
6232 ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
6233                                    unsigned Flags) const {
6234   MachineInstr &MI = *MIT;
6235   const TargetRegisterInfo *TRI = &getRegisterInfo();
6236 
6237   // Be conservative with inline ASM
6238   if (MI.isInlineAsm())
6239     return outliner::InstrType::Illegal;
6240 
6241   // Don't allow debug values to impact outlining type.
6242   if (MI.isDebugInstr() || MI.isIndirectDebugValue())
6243     return outliner::InstrType::Invisible;
6244 
6245   // At this point, KILL or IMPLICIT_DEF instructions don't really tell us much
6246   // so we can go ahead and skip over them.
6247   if (MI.isKill() || MI.isImplicitDef())
6248     return outliner::InstrType::Invisible;
6249 
6250   // PIC instructions contain labels, outlining them would break offset
6251   // computing.  unsigned Opc = MI.getOpcode();
6252   unsigned Opc = MI.getOpcode();
6253   if (Opc == ARM::tPICADD || Opc == ARM::PICADD || Opc == ARM::PICSTR ||
6254       Opc == ARM::PICSTRB || Opc == ARM::PICSTRH || Opc == ARM::PICLDR ||
6255       Opc == ARM::PICLDRB || Opc == ARM::PICLDRH || Opc == ARM::PICLDRSB ||
6256       Opc == ARM::PICLDRSH || Opc == ARM::t2LDRpci_pic ||
6257       Opc == ARM::t2MOVi16_ga_pcrel || Opc == ARM::t2MOVTi16_ga_pcrel ||
6258       Opc == ARM::t2MOV_ga_pcrel)
6259     return outliner::InstrType::Illegal;
6260 
6261   // Be conservative with ARMv8.1 MVE instructions.
6262   if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart ||
6263       Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart ||
6264       Opc == ARM::t2WhileLoopStartLR || Opc == ARM::t2WhileLoopStartTP ||
6265       Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd ||
6266       Opc == ARM::t2LoopEndDec)
6267     return outliner::InstrType::Illegal;
6268 
6269   const MCInstrDesc &MCID = MI.getDesc();
6270   uint64_t MIFlags = MCID.TSFlags;
6271   if ((MIFlags & ARMII::DomainMask) == ARMII::DomainMVE)
6272     return outliner::InstrType::Illegal;
6273 
6274   // Is this a terminator for a basic block?
6275   if (MI.isTerminator()) {
6276     // Don't outline if the branch is not unconditional.
6277     if (isPredicated(MI))
6278       return outliner::InstrType::Illegal;
6279 
6280     // Is this the end of a function?
6281     if (MI.getParent()->succ_empty())
6282       return outliner::InstrType::Legal;
6283 
6284     // It's not, so don't outline it.
6285     return outliner::InstrType::Illegal;
6286   }
6287 
6288   // Make sure none of the operands are un-outlinable.
6289   for (const MachineOperand &MOP : MI.operands()) {
6290     if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() ||
6291         MOP.isTargetIndex())
6292       return outliner::InstrType::Illegal;
6293   }
6294 
6295   // Don't outline if link register or program counter value are used.
6296   if (MI.readsRegister(ARM::LR, TRI) || MI.readsRegister(ARM::PC, TRI))
6297     return outliner::InstrType::Illegal;
6298 
6299   if (MI.isCall()) {
6300     // Get the function associated with the call.  Look at each operand and find
6301     // the one that represents the calle and get its name.
6302     const Function *Callee = nullptr;
6303     for (const MachineOperand &MOP : MI.operands()) {
6304       if (MOP.isGlobal()) {
6305         Callee = dyn_cast<Function>(MOP.getGlobal());
6306         break;
6307       }
6308     }
6309 
6310     // Dont't outline calls to "mcount" like functions, in particular Linux
6311     // kernel function tracing relies on it.
6312     if (Callee &&
6313         (Callee->getName() == "\01__gnu_mcount_nc" ||
6314          Callee->getName() == "\01mcount" || Callee->getName() == "__mcount"))
6315       return outliner::InstrType::Illegal;
6316 
6317     // If we don't know anything about the callee, assume it depends on the
6318     // stack layout of the caller. In that case, it's only legal to outline
6319     // as a tail-call. Explicitly list the call instructions we know about so
6320     // we don't get unexpected results with call pseudo-instructions.
6321     auto UnknownCallOutlineType = outliner::InstrType::Illegal;
6322     if (Opc == ARM::BL || Opc == ARM::tBL || Opc == ARM::BLX ||
6323         Opc == ARM::BLX_noip || Opc == ARM::tBLXr || Opc == ARM::tBLXr_noip ||
6324         Opc == ARM::tBLXi)
6325       UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
6326 
6327     if (!Callee)
6328       return UnknownCallOutlineType;
6329 
6330     // We have a function we have information about.  Check if it's something we
6331     // can safely outline.
6332     MachineFunction *MF = MI.getParent()->getParent();
6333     MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee);
6334 
6335     // We don't know what's going on with the callee at all.  Don't touch it.
6336     if (!CalleeMF)
6337       return UnknownCallOutlineType;
6338 
6339     // Check if we know anything about the callee saves on the function. If we
6340     // don't, then don't touch it, since that implies that we haven't computed
6341     // anything about its stack frame yet.
6342     MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
6343     if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
6344         MFI.getNumObjects() > 0)
6345       return UnknownCallOutlineType;
6346 
6347     // At this point, we can say that CalleeMF ought to not pass anything on the
6348     // stack. Therefore, we can outline it.
6349     return outliner::InstrType::Legal;
6350   }
6351 
6352   // Since calls are handled, don't touch LR or PC
6353   if (MI.modifiesRegister(ARM::LR, TRI) || MI.modifiesRegister(ARM::PC, TRI))
6354     return outliner::InstrType::Illegal;
6355 
6356   // Does this use the stack?
6357   if (MI.modifiesRegister(ARM::SP, TRI) || MI.readsRegister(ARM::SP, TRI)) {
6358     // True if there is no chance that any outlined candidate from this range
6359     // could require stack fixups. That is, both
6360     // * LR is available in the range (No save/restore around call)
6361     // * The range doesn't include calls (No save/restore in outlined frame)
6362     // are true.
6363     // These conditions also ensure correctness of the return address
6364     // authentication - we insert sign and authentication instructions only if
6365     // we save/restore LR on stack, but then this condition ensures that the
6366     // outlined range does not modify the SP, therefore the SP value used for
6367     // signing is the same as the one used for authentication.
6368     // FIXME: This is very restrictive; the flags check the whole block,
6369     // not just the bit we will try to outline.
6370     bool MightNeedStackFixUp =
6371         (Flags & (MachineOutlinerMBBFlags::LRUnavailableSomewhere |
6372                   MachineOutlinerMBBFlags::HasCalls));
6373 
6374     if (!MightNeedStackFixUp)
6375       return outliner::InstrType::Legal;
6376 
6377     // Any modification of SP will break our code to save/restore LR.
6378     // FIXME: We could handle some instructions which add a constant offset to
6379     // SP, with a bit more work.
6380     if (MI.modifiesRegister(ARM::SP, TRI))
6381       return outliner::InstrType::Illegal;
6382 
6383     // At this point, we have a stack instruction that we might need to fix up.
6384     // up. We'll handle it if it's a load or store.
6385     if (checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(),
6386                                   false))
6387       return outliner::InstrType::Legal;
6388 
6389     // We can't fix it up, so don't outline it.
6390     return outliner::InstrType::Illegal;
6391   }
6392 
6393   // Be conservative with IT blocks.
6394   if (MI.readsRegister(ARM::ITSTATE, TRI) ||
6395       MI.modifiesRegister(ARM::ITSTATE, TRI))
6396     return outliner::InstrType::Illegal;
6397 
6398   // Don't outline positions.
6399   if (MI.isPosition())
6400     return outliner::InstrType::Illegal;
6401 
6402   return outliner::InstrType::Legal;
6403 }
6404 
6405 void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6406   for (MachineInstr &MI : MBB) {
6407     checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(), true);
6408   }
6409 }
6410 
6411 void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB,
6412                                      MachineBasicBlock::iterator It, bool CFI,
6413                                      bool Auth) const {
6414   int Align = std::max(Subtarget.getStackAlignment().value(), uint64_t(8));
6415   assert(Align >= 8 && Align <= 256);
6416   if (Auth) {
6417     assert(Subtarget.isThumb2());
6418     // Compute PAC in R12. Outlining ensures R12 is dead across the outlined
6419     // sequence.
6420     BuildMI(MBB, It, DebugLoc(), get(ARM::t2PAC))
6421         .setMIFlags(MachineInstr::FrameSetup);
6422     BuildMI(MBB, It, DebugLoc(), get(ARM::t2STRD_PRE), ARM::SP)
6423         .addReg(ARM::R12, RegState::Kill)
6424         .addReg(ARM::LR, RegState::Kill)
6425         .addReg(ARM::SP)
6426         .addImm(-Align)
6427         .add(predOps(ARMCC::AL))
6428         .setMIFlags(MachineInstr::FrameSetup);
6429   } else {
6430     unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM;
6431     BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP)
6432         .addReg(ARM::LR, RegState::Kill)
6433         .addReg(ARM::SP)
6434         .addImm(-Align)
6435         .add(predOps(ARMCC::AL))
6436         .setMIFlags(MachineInstr::FrameSetup);
6437   }
6438 
6439   if (!CFI)
6440     return;
6441 
6442   MachineFunction &MF = *MBB.getParent();
6443 
6444   // Add a CFI, saying CFA is offset by Align bytes from SP.
6445   int64_t StackPosEntry =
6446       MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align));
6447   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6448       .addCFIIndex(StackPosEntry)
6449       .setMIFlags(MachineInstr::FrameSetup);
6450 
6451   // Add a CFI saying that the LR that we want to find is now higher than
6452   // before.
6453   int LROffset = Auth ? Align - 4 : Align;
6454   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6455   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6456   int64_t LRPosEntry = MF.addFrameInst(
6457       MCCFIInstruction::createOffset(nullptr, DwarfLR, -LROffset));
6458   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6459       .addCFIIndex(LRPosEntry)
6460       .setMIFlags(MachineInstr::FrameSetup);
6461   if (Auth) {
6462     // Add a CFI for the location of the return adddress PAC.
6463     unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true);
6464     int64_t RACPosEntry = MF.addFrameInst(
6465         MCCFIInstruction::createOffset(nullptr, DwarfRAC, -Align));
6466     BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6467         .addCFIIndex(RACPosEntry)
6468         .setMIFlags(MachineInstr::FrameSetup);
6469   }
6470 }
6471 
6472 void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB,
6473                                              MachineBasicBlock::iterator It,
6474                                              Register Reg) const {
6475   MachineFunction &MF = *MBB.getParent();
6476   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6477   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6478   unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
6479 
6480   int64_t LRPosEntry = MF.addFrameInst(
6481       MCCFIInstruction::createRegister(nullptr, DwarfLR, DwarfReg));
6482   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6483       .addCFIIndex(LRPosEntry)
6484       .setMIFlags(MachineInstr::FrameSetup);
6485 }
6486 
6487 void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
6488                                           MachineBasicBlock::iterator It,
6489                                           bool CFI, bool Auth) const {
6490   int Align = Subtarget.getStackAlignment().value();
6491   if (Auth) {
6492     assert(Subtarget.isThumb2());
6493     // Restore return address PAC and LR.
6494     BuildMI(MBB, It, DebugLoc(), get(ARM::t2LDRD_POST))
6495         .addReg(ARM::R12, RegState::Define)
6496         .addReg(ARM::LR, RegState::Define)
6497         .addReg(ARM::SP, RegState::Define)
6498         .addReg(ARM::SP)
6499         .addImm(Align)
6500         .add(predOps(ARMCC::AL))
6501         .setMIFlags(MachineInstr::FrameDestroy);
6502     // LR authentication is after the CFI instructions, below.
6503   } else {
6504     unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM;
6505     MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR)
6506                                   .addReg(ARM::SP, RegState::Define)
6507                                   .addReg(ARM::SP);
6508     if (!Subtarget.isThumb())
6509       MIB.addReg(0);
6510     MIB.addImm(Subtarget.getStackAlignment().value())
6511         .add(predOps(ARMCC::AL))
6512         .setMIFlags(MachineInstr::FrameDestroy);
6513   }
6514 
6515   if (CFI) {
6516     // Now stack has moved back up...
6517     MachineFunction &MF = *MBB.getParent();
6518     const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6519     unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6520     int64_t StackPosEntry =
6521         MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0));
6522     BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6523         .addCFIIndex(StackPosEntry)
6524         .setMIFlags(MachineInstr::FrameDestroy);
6525 
6526     // ... and we have restored LR.
6527     int64_t LRPosEntry =
6528         MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
6529     BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6530         .addCFIIndex(LRPosEntry)
6531         .setMIFlags(MachineInstr::FrameDestroy);
6532 
6533     if (Auth) {
6534       unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true);
6535       int64_t Entry =
6536           MF.addFrameInst(MCCFIInstruction::createUndefined(nullptr, DwarfRAC));
6537       BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6538           .addCFIIndex(Entry)
6539           .setMIFlags(MachineInstr::FrameDestroy);
6540     }
6541   }
6542 
6543   if (Auth)
6544     BuildMI(MBB, It, DebugLoc(), get(ARM::t2AUT));
6545 }
6546 
6547 void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg(
6548     MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
6549   MachineFunction &MF = *MBB.getParent();
6550   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6551   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6552 
6553   int64_t LRPosEntry =
6554       MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
6555   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6556       .addCFIIndex(LRPosEntry)
6557       .setMIFlags(MachineInstr::FrameDestroy);
6558 }
6559 
6560 void ARMBaseInstrInfo::buildOutlinedFrame(
6561     MachineBasicBlock &MBB, MachineFunction &MF,
6562     const outliner::OutlinedFunction &OF) const {
6563   // For thunk outlining, rewrite the last instruction from a call to a
6564   // tail-call.
6565   if (OF.FrameConstructionID == MachineOutlinerThunk) {
6566     MachineInstr *Call = &*--MBB.instr_end();
6567     bool isThumb = Subtarget.isThumb();
6568     unsigned FuncOp = isThumb ? 2 : 0;
6569     unsigned Opc = Call->getOperand(FuncOp).isReg()
6570                        ? isThumb ? ARM::tTAILJMPr : ARM::TAILJMPr
6571                        : isThumb ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd
6572                                                              : ARM::tTAILJMPdND
6573                                  : ARM::TAILJMPd;
6574     MachineInstrBuilder MIB = BuildMI(MBB, MBB.end(), DebugLoc(), get(Opc))
6575                                   .add(Call->getOperand(FuncOp));
6576     if (isThumb && !Call->getOperand(FuncOp).isReg())
6577       MIB.add(predOps(ARMCC::AL));
6578     Call->eraseFromParent();
6579   }
6580 
6581   // Is there a call in the outlined range?
6582   auto IsNonTailCall = [](MachineInstr &MI) {
6583     return MI.isCall() && !MI.isReturn();
6584   };
6585   if (llvm::any_of(MBB.instrs(), IsNonTailCall)) {
6586     MachineBasicBlock::iterator It = MBB.begin();
6587     MachineBasicBlock::iterator Et = MBB.end();
6588 
6589     if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6590         OF.FrameConstructionID == MachineOutlinerThunk)
6591       Et = std::prev(MBB.end());
6592 
6593     // We have to save and restore LR, we need to add it to the liveins if it
6594     // is not already part of the set.  This is suffient since outlined
6595     // functions only have one block.
6596     if (!MBB.isLiveIn(ARM::LR))
6597       MBB.addLiveIn(ARM::LR);
6598 
6599     // Insert a save before the outlined region
6600     bool Auth = OF.Candidates.front()
6601                     .getMF()
6602                     ->getInfo<ARMFunctionInfo>()
6603                     ->shouldSignReturnAddress(true);
6604     saveLROnStack(MBB, It, true, Auth);
6605 
6606     // Fix up the instructions in the range, since we're going to modify the
6607     // stack.
6608     assert(OF.FrameConstructionID != MachineOutlinerDefault &&
6609            "Can only fix up stack references once");
6610     fixupPostOutline(MBB);
6611 
6612     // Insert a restore before the terminator for the function.  Restore LR.
6613     restoreLRFromStack(MBB, Et, true, Auth);
6614   }
6615 
6616   // If this is a tail call outlined function, then there's already a return.
6617   if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6618       OF.FrameConstructionID == MachineOutlinerThunk)
6619     return;
6620 
6621   // Here we have to insert the return ourselves.  Get the correct opcode from
6622   // current feature set.
6623   BuildMI(MBB, MBB.end(), DebugLoc(), get(Subtarget.getReturnOpcode()))
6624       .add(predOps(ARMCC::AL));
6625 
6626   // Did we have to modify the stack by saving the link register?
6627   if (OF.FrameConstructionID != MachineOutlinerDefault &&
6628       OF.Candidates[0].CallConstructionID != MachineOutlinerDefault)
6629     return;
6630 
6631   // We modified the stack.
6632   // Walk over the basic block and fix up all the stack accesses.
6633   fixupPostOutline(MBB);
6634 }
6635 
6636 MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall(
6637     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
6638     MachineFunction &MF, const outliner::Candidate &C) const {
6639   MachineInstrBuilder MIB;
6640   MachineBasicBlock::iterator CallPt;
6641   unsigned Opc;
6642   bool isThumb = Subtarget.isThumb();
6643 
6644   // Are we tail calling?
6645   if (C.CallConstructionID == MachineOutlinerTailCall) {
6646     // If yes, then we can just branch to the label.
6647     Opc = isThumb
6648               ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND
6649               : ARM::TAILJMPd;
6650     MIB = BuildMI(MF, DebugLoc(), get(Opc))
6651               .addGlobalAddress(M.getNamedValue(MF.getName()));
6652     if (isThumb)
6653       MIB.add(predOps(ARMCC::AL));
6654     It = MBB.insert(It, MIB);
6655     return It;
6656   }
6657 
6658   // Create the call instruction.
6659   Opc = isThumb ? ARM::tBL : ARM::BL;
6660   MachineInstrBuilder CallMIB = BuildMI(MF, DebugLoc(), get(Opc));
6661   if (isThumb)
6662     CallMIB.add(predOps(ARMCC::AL));
6663   CallMIB.addGlobalAddress(M.getNamedValue(MF.getName()));
6664 
6665   if (C.CallConstructionID == MachineOutlinerNoLRSave ||
6666       C.CallConstructionID == MachineOutlinerThunk) {
6667     // No, so just insert the call.
6668     It = MBB.insert(It, CallMIB);
6669     return It;
6670   }
6671 
6672   const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
6673   // Can we save to a register?
6674   if (C.CallConstructionID == MachineOutlinerRegSave) {
6675     unsigned Reg = findRegisterToSaveLRTo(C);
6676     assert(Reg != 0 && "No callee-saved register available?");
6677 
6678     // Save and restore LR from that register.
6679     copyPhysReg(MBB, It, DebugLoc(), Reg, ARM::LR, true);
6680     if (!AFI.isLRSpilled())
6681       emitCFIForLRSaveToReg(MBB, It, Reg);
6682     CallPt = MBB.insert(It, CallMIB);
6683     copyPhysReg(MBB, It, DebugLoc(), ARM::LR, Reg, true);
6684     if (!AFI.isLRSpilled())
6685       emitCFIForLRRestoreFromReg(MBB, It);
6686     It--;
6687     return CallPt;
6688   }
6689   // We have the default case. Save and restore from SP.
6690   if (!MBB.isLiveIn(ARM::LR))
6691     MBB.addLiveIn(ARM::LR);
6692   bool Auth = !AFI.isLRSpilled() && AFI.shouldSignReturnAddress(true);
6693   saveLROnStack(MBB, It, !AFI.isLRSpilled(), Auth);
6694   CallPt = MBB.insert(It, CallMIB);
6695   restoreLRFromStack(MBB, It, !AFI.isLRSpilled(), Auth);
6696   It--;
6697   return CallPt;
6698 }
6699 
6700 bool ARMBaseInstrInfo::shouldOutlineFromFunctionByDefault(
6701     MachineFunction &MF) const {
6702   return Subtarget.isMClass() && MF.getFunction().hasMinSize();
6703 }
6704 
6705 bool ARMBaseInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
6706                                                          AAResults *AA) const {
6707   // Try hard to rematerialize any VCTPs because if we spill P0, it will block
6708   // the tail predication conversion. This means that the element count
6709   // register has to be live for longer, but that has to be better than
6710   // spill/restore and VPT predication.
6711   return isVCTP(&MI) && !isPredicated(MI);
6712 }
6713 
6714 unsigned llvm::getBLXOpcode(const MachineFunction &MF) {
6715   return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_noip
6716                                                           : ARM::BLX;
6717 }
6718 
6719 unsigned llvm::gettBLXrOpcode(const MachineFunction &MF) {
6720   return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::tBLXr_noip
6721                                                           : ARM::tBLXr;
6722 }
6723 
6724 unsigned llvm::getBLXpredOpcode(const MachineFunction &MF) {
6725   return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_pred_noip
6726                                                           : ARM::BLX_pred;
6727 }
6728 
6729