1 //===-- AArch64A53Fix835769.cpp -------------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 // This pass changes code to work around Cortex-A53 erratum 835769.
10 // It works around it by inserting a nop instruction in code sequences that
11 // in some circumstances may trigger the erratum.
12 // It inserts a nop instruction between a sequence of the following 2 classes
13 // of instructions:
14 // instr 1: mem-instr (including loads, stores and prefetches).
15 // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
16 //===----------------------------------------------------------------------===//
17
18 #include "AArch64.h"
19 #include "AArch64InstrInfo.h"
20 #include "AArch64Subtarget.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/CodeGen/MachineFunction.h"
23 #include "llvm/CodeGen/MachineFunctionPass.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineRegisterInfo.h"
27 #include "llvm/Support/CommandLine.h"
28 #include "llvm/Support/Debug.h"
29
30 using namespace llvm;
31
32 #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
33
34 STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
35
36 //===----------------------------------------------------------------------===//
37 // Helper functions
38
39 // Is the instruction a match for the instruction that comes first in the
40 // sequence of instructions that can trigger the erratum?
isFirstInstructionInSequence(MachineInstr * MI)41 static bool isFirstInstructionInSequence(MachineInstr *MI) {
42 // Must return true if this instruction is a load, a store or a prefetch.
43 switch (MI->getOpcode()) {
44 case AArch64::PRFMl:
45 case AArch64::PRFMroW:
46 case AArch64::PRFMroX:
47 case AArch64::PRFMui:
48 case AArch64::PRFUMi:
49 return true;
50 default:
51 return (MI->mayLoad() || MI->mayStore());
52 }
53 }
54
55 // Is the instruction a match for the instruction that comes second in the
56 // sequence that can trigger the erratum?
isSecondInstructionInSequence(MachineInstr * MI)57 static bool isSecondInstructionInSequence(MachineInstr *MI) {
58 // Must return true for non-SIMD integer multiply-accumulates, writing
59 // to a 64-bit register.
60 switch (MI->getOpcode()) {
61 // Erratum cannot be triggered when the destination register is 32 bits,
62 // therefore only include the following.
63 case AArch64::MSUBXrrr:
64 case AArch64::MADDXrrr:
65 case AArch64::SMADDLrrr:
66 case AArch64::SMSUBLrrr:
67 case AArch64::UMADDLrrr:
68 case AArch64::UMSUBLrrr:
69 // Erratum can only be triggered by multiply-adds, not by regular
70 // non-accumulating multiplies, i.e. when Ra=XZR='11111'
71 return MI->getOperand(3).getReg() != AArch64::XZR;
72 default:
73 return false;
74 }
75 }
76
77
78 //===----------------------------------------------------------------------===//
79
80 namespace {
81 class AArch64A53Fix835769 : public MachineFunctionPass {
82 const AArch64InstrInfo *TII;
83
84 public:
85 static char ID;
AArch64A53Fix835769()86 explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {}
87
88 bool runOnMachineFunction(MachineFunction &F) override;
89
getPassName() const90 const char *getPassName() const override {
91 return "Workaround A53 erratum 835769 pass";
92 }
93
getAnalysisUsage(AnalysisUsage & AU) const94 void getAnalysisUsage(AnalysisUsage &AU) const override {
95 AU.setPreservesCFG();
96 MachineFunctionPass::getAnalysisUsage(AU);
97 }
98
99 private:
100 bool runOnBasicBlock(MachineBasicBlock &MBB);
101 };
102 char AArch64A53Fix835769::ID = 0;
103
104 } // end anonymous namespace
105
106 //===----------------------------------------------------------------------===//
107
108 bool
runOnMachineFunction(MachineFunction & F)109 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
110 const TargetMachine &TM = F.getTarget();
111
112 bool Changed = false;
113 DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
114
115 TII = TM.getSubtarget<AArch64Subtarget>().getInstrInfo();
116
117 for (auto &MBB : F) {
118 Changed |= runOnBasicBlock(MBB);
119 }
120
121 return Changed;
122 }
123
124 // Return the block that was fallen through to get to MBB, if any,
125 // otherwise nullptr.
getBBFallenThrough(MachineBasicBlock * MBB,const TargetInstrInfo * TII)126 static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
127 const TargetInstrInfo *TII) {
128 // Get the previous machine basic block in the function.
129 MachineFunction::iterator MBBI = *MBB;
130
131 // Can't go off top of function.
132 if (MBBI == MBB->getParent()->begin())
133 return nullptr;
134
135 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
136 SmallVector<MachineOperand, 2> Cond;
137
138 MachineBasicBlock *PrevBB = std::prev(MBBI);
139 for (MachineBasicBlock *S : MBB->predecessors())
140 if (S == PrevBB && !TII->AnalyzeBranch(*PrevBB, TBB, FBB, Cond) &&
141 !TBB && !FBB)
142 return S;
143
144 return nullptr;
145 }
146
147 // Iterate through fallen through blocks trying to find a previous non-pseudo if
148 // there is one, otherwise return nullptr. Only look for instructions in
149 // previous blocks, not the current block, since we only use this to look at
150 // previous blocks.
getLastNonPseudo(MachineBasicBlock & MBB,const TargetInstrInfo * TII)151 static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
152 const TargetInstrInfo *TII) {
153 MachineBasicBlock *FMBB = &MBB;
154
155 // If there is no non-pseudo in the current block, loop back around and try
156 // the previous block (if there is one).
157 while ((FMBB = getBBFallenThrough(FMBB, TII))) {
158 for (auto I = FMBB->rbegin(), E = FMBB->rend(); I != E; ++I) {
159 if (!I->isPseudo())
160 return &*I;
161 }
162 }
163
164 // There was no previous non-pseudo in the fallen through blocks
165 return nullptr;
166 }
167
insertNopBeforeInstruction(MachineBasicBlock & MBB,MachineInstr * MI,const TargetInstrInfo * TII)168 static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
169 const TargetInstrInfo *TII) {
170 // If we are the first instruction of the block, put the NOP at the end of
171 // the previous fallthrough block
172 if (MI == &MBB.front()) {
173 MachineInstr *I = getLastNonPseudo(MBB, TII);
174 assert(I && "Expected instruction");
175 DebugLoc DL = I->getDebugLoc();
176 BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
177 }
178 else {
179 DebugLoc DL = MI->getDebugLoc();
180 BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
181 }
182
183 ++NumNopsAdded;
184 }
185
186 bool
runOnBasicBlock(MachineBasicBlock & MBB)187 AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
188 bool Changed = false;
189 DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n");
190
191 // First, scan the basic block, looking for a sequence of 2 instructions
192 // that match the conditions under which the erratum may trigger.
193
194 // List of terminating instructions in matching sequences
195 std::vector<MachineInstr*> Sequences;
196 unsigned Idx = 0;
197 MachineInstr *PrevInstr = nullptr;
198
199 // Try and find the last non-pseudo instruction in any fallen through blocks,
200 // if there isn't one, then we use nullptr to represent that.
201 PrevInstr = getLastNonPseudo(MBB, TII);
202
203 for (auto &MI : MBB) {
204 MachineInstr *CurrInstr = &MI;
205 DEBUG(dbgs() << " Examining: " << MI);
206 if (PrevInstr) {
207 DEBUG(dbgs() << " PrevInstr: " << *PrevInstr
208 << " CurrInstr: " << *CurrInstr
209 << " isFirstInstructionInSequence(PrevInstr): "
210 << isFirstInstructionInSequence(PrevInstr) << "\n"
211 << " isSecondInstructionInSequence(CurrInstr): "
212 << isSecondInstructionInSequence(CurrInstr) << "\n");
213 if (isFirstInstructionInSequence(PrevInstr) &&
214 isSecondInstructionInSequence(CurrInstr)) {
215 DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n");
216 Sequences.push_back(CurrInstr);
217 }
218 }
219 if (!CurrInstr->isPseudo())
220 PrevInstr = CurrInstr;
221 ++Idx;
222 }
223
224 DEBUG(dbgs() << "Scan complete, "<< Sequences.size()
225 << " occurences of pattern found.\n");
226
227 // Then update the basic block, inserting nops between the detected sequences.
228 for (auto &MI : Sequences) {
229 Changed = true;
230 insertNopBeforeInstruction(MBB, MI, TII);
231 }
232
233 return Changed;
234 }
235
236 // Factory function used by AArch64TargetMachine to add the pass to
237 // the passmanager.
createAArch64A53Fix835769()238 FunctionPass *llvm::createAArch64A53Fix835769() {
239 return new AArch64A53Fix835769();
240 }
241