1 //===-- AArch64A53Fix835769.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // This pass changes code to work around Cortex-A53 erratum 835769.
9 // It works around it by inserting a nop instruction in code sequences that
10 // in some circumstances may trigger the erratum.
11 // It inserts a nop instruction between a sequence of the following 2 classes
12 // of instructions:
13 // instr 1: mem-instr (including loads, stores and prefetches).
14 // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
15 //===----------------------------------------------------------------------===//
16 
17 #include "AArch64.h"
18 #include "AArch64Subtarget.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/CodeGen/MachineFunction.h"
21 #include "llvm/CodeGen/MachineFunctionPass.h"
22 #include "llvm/CodeGen/MachineInstr.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/CodeGen/TargetInstrInfo.h"
26 #include "llvm/Support/Debug.h"
27 #include "llvm/Support/raw_ostream.h"
28 
29 using namespace llvm;
30 
31 #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
32 
33 STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
34 
35 //===----------------------------------------------------------------------===//
36 // Helper functions
37 
38 // Is the instruction a match for the instruction that comes first in the
39 // sequence of instructions that can trigger the erratum?
40 static bool isFirstInstructionInSequence(MachineInstr *MI) {
41   // Must return true if this instruction is a load, a store or a prefetch.
42   switch (MI->getOpcode()) {
43   case AArch64::PRFMl:
44   case AArch64::PRFMroW:
45   case AArch64::PRFMroX:
46   case AArch64::PRFMui:
47   case AArch64::PRFUMi:
48     return true;
49   default:
50     return MI->mayLoadOrStore();
51   }
52 }
53 
54 // Is the instruction a match for the instruction that comes second in the
55 // sequence that can trigger the erratum?
56 static bool isSecondInstructionInSequence(MachineInstr *MI) {
57   // Must return true for non-SIMD integer multiply-accumulates, writing
58   // to a 64-bit register.
59   switch (MI->getOpcode()) {
60   // Erratum cannot be triggered when the destination register is 32 bits,
61   // therefore only include the following.
62   case AArch64::MSUBXrrr:
63   case AArch64::MADDXrrr:
64   case AArch64::SMADDLrrr:
65   case AArch64::SMSUBLrrr:
66   case AArch64::UMADDLrrr:
67   case AArch64::UMSUBLrrr:
68     // Erratum can only be triggered by multiply-adds, not by regular
69     // non-accumulating multiplies, i.e. when Ra=XZR='11111'
70     return MI->getOperand(3).getReg() != AArch64::XZR;
71   default:
72     return false;
73   }
74 }
75 
76 
77 //===----------------------------------------------------------------------===//
78 
79 namespace {
80 class AArch64A53Fix835769 : public MachineFunctionPass {
81   const TargetInstrInfo *TII;
82 
83 public:
84   static char ID;
85   explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {
86     initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry());
87   }
88 
89   bool runOnMachineFunction(MachineFunction &F) override;
90 
91   MachineFunctionProperties getRequiredProperties() const override {
92     return MachineFunctionProperties().set(
93         MachineFunctionProperties::Property::NoVRegs);
94   }
95 
96   StringRef getPassName() const override {
97     return "Workaround A53 erratum 835769 pass";
98   }
99 
100   void getAnalysisUsage(AnalysisUsage &AU) const override {
101     AU.setPreservesCFG();
102     MachineFunctionPass::getAnalysisUsage(AU);
103   }
104 
105 private:
106   bool runOnBasicBlock(MachineBasicBlock &MBB);
107 };
108 char AArch64A53Fix835769::ID = 0;
109 
110 } // end anonymous namespace
111 
112 INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",
113                 "AArch64 fix for A53 erratum 835769", false, false)
114 
115 //===----------------------------------------------------------------------===//
116 
117 bool
118 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
119   LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
120   auto &STI = F.getSubtarget<AArch64Subtarget>();
121   // Fix not requested, skip pass.
122   if (!STI.fixCortexA53_835769())
123     return false;
124 
125   bool Changed = false;
126   TII = STI.getInstrInfo();
127 
128   for (auto &MBB : F) {
129     Changed |= runOnBasicBlock(MBB);
130   }
131   return Changed;
132 }
133 
134 // Return the block that was fallen through to get to MBB, if any,
135 // otherwise nullptr.
136 static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
137                                              const TargetInstrInfo *TII) {
138   // Get the previous machine basic block in the function.
139   MachineFunction::iterator MBBI(MBB);
140 
141   // Can't go off top of function.
142   if (MBBI == MBB->getParent()->begin())
143     return nullptr;
144 
145   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
146   SmallVector<MachineOperand, 2> Cond;
147 
148   MachineBasicBlock *PrevBB = &*std::prev(MBBI);
149   for (MachineBasicBlock *S : MBB->predecessors())
150     if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&
151         !FBB)
152       return S;
153 
154   return nullptr;
155 }
156 
157 // Iterate through fallen through blocks trying to find a previous non-pseudo if
158 // there is one, otherwise return nullptr. Only look for instructions in
159 // previous blocks, not the current block, since we only use this to look at
160 // previous blocks.
161 static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
162                                       const TargetInstrInfo *TII) {
163   MachineBasicBlock *FMBB = &MBB;
164 
165   // If there is no non-pseudo in the current block, loop back around and try
166   // the previous block (if there is one).
167   while ((FMBB = getBBFallenThrough(FMBB, TII))) {
168     for (MachineInstr &I : llvm::reverse(*FMBB))
169       if (!I.isPseudo())
170         return &I;
171   }
172 
173   // There was no previous non-pseudo in the fallen through blocks
174   return nullptr;
175 }
176 
177 static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
178                                        const TargetInstrInfo *TII) {
179   // If we are the first instruction of the block, put the NOP at the end of
180   // the previous fallthrough block
181   if (MI == &MBB.front()) {
182     MachineInstr *I = getLastNonPseudo(MBB, TII);
183     assert(I && "Expected instruction");
184     DebugLoc DL = I->getDebugLoc();
185     BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
186   }
187   else {
188     DebugLoc DL = MI->getDebugLoc();
189     BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
190   }
191 
192   ++NumNopsAdded;
193 }
194 
195 bool
196 AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
197   bool Changed = false;
198   LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB
199                     << " - scanning instructions...\n");
200 
201   // First, scan the basic block, looking for a sequence of 2 instructions
202   // that match the conditions under which the erratum may trigger.
203 
204   // List of terminating instructions in matching sequences
205   std::vector<MachineInstr*> Sequences;
206   unsigned Idx = 0;
207   MachineInstr *PrevInstr = nullptr;
208 
209   // Try and find the last non-pseudo instruction in any fallen through blocks,
210   // if there isn't one, then we use nullptr to represent that.
211   PrevInstr = getLastNonPseudo(MBB, TII);
212 
213   for (auto &MI : MBB) {
214     MachineInstr *CurrInstr = &MI;
215     LLVM_DEBUG(dbgs() << "  Examining: " << MI);
216     if (PrevInstr) {
217       LLVM_DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
218                         << "    CurrInstr: " << *CurrInstr
219                         << "    isFirstInstructionInSequence(PrevInstr): "
220                         << isFirstInstructionInSequence(PrevInstr) << "\n"
221                         << "    isSecondInstructionInSequence(CurrInstr): "
222                         << isSecondInstructionInSequence(CurrInstr) << "\n");
223       if (isFirstInstructionInSequence(PrevInstr) &&
224           isSecondInstructionInSequence(CurrInstr)) {
225         LLVM_DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
226         (void) Idx;
227         Sequences.push_back(CurrInstr);
228       }
229     }
230     if (!CurrInstr->isPseudo())
231       PrevInstr = CurrInstr;
232     ++Idx;
233   }
234 
235   LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size()
236                     << " occurrences of pattern found.\n");
237 
238   // Then update the basic block, inserting nops between the detected sequences.
239   for (auto &MI : Sequences) {
240     Changed = true;
241     insertNopBeforeInstruction(MBB, MI, TII);
242   }
243 
244   return Changed;
245 }
246 
247 // Factory function used by AArch64TargetMachine to add the pass to
248 // the passmanager.
249 FunctionPass *llvm::createAArch64A53Fix835769() {
250   return new AArch64A53Fix835769();
251 }
252