1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file contains the AArch64 implementation of the DAG scheduling
10 /// mutation to pair instructions back to back.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "AArch64Subtarget.h"
15 #include "llvm/CodeGen/MacroFusion.h"
16 #include "llvm/CodeGen/TargetInstrInfo.h"
17
18 using namespace llvm;
19
20 namespace {
21
22 /// CMN, CMP, TST followed by Bcc
isArithmeticBccPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI,bool CmpOnly)23 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
24 const MachineInstr &SecondMI, bool CmpOnly) {
25 if (SecondMI.getOpcode() != AArch64::Bcc)
26 return false;
27
28 // Assume the 1st instr to be a wildcard if it is unspecified.
29 if (FirstMI == nullptr)
30 return true;
31
32 // If we're in CmpOnly mode, we only fuse arithmetic instructions that
33 // discard their result.
34 if (CmpOnly && !(FirstMI->getOperand(0).getReg() == AArch64::XZR ||
35 FirstMI->getOperand(0).getReg() == AArch64::WZR)) {
36 return false;
37 }
38
39 switch (FirstMI->getOpcode()) {
40 case AArch64::ADDSWri:
41 case AArch64::ADDSWrr:
42 case AArch64::ADDSXri:
43 case AArch64::ADDSXrr:
44 case AArch64::ANDSWri:
45 case AArch64::ANDSWrr:
46 case AArch64::ANDSXri:
47 case AArch64::ANDSXrr:
48 case AArch64::SUBSWri:
49 case AArch64::SUBSWrr:
50 case AArch64::SUBSXri:
51 case AArch64::SUBSXrr:
52 case AArch64::BICSWrr:
53 case AArch64::BICSXrr:
54 return true;
55 case AArch64::ADDSWrs:
56 case AArch64::ADDSXrs:
57 case AArch64::ANDSWrs:
58 case AArch64::ANDSXrs:
59 case AArch64::SUBSWrs:
60 case AArch64::SUBSXrs:
61 case AArch64::BICSWrs:
62 case AArch64::BICSXrs:
63 // Shift value can be 0 making these behave like the "rr" variant...
64 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
65 }
66
67 return false;
68 }
69
70 /// ALU operations followed by CBZ/CBNZ.
isArithmeticCbzPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)71 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
72 const MachineInstr &SecondMI) {
73 if (SecondMI.getOpcode() != AArch64::CBZW &&
74 SecondMI.getOpcode() != AArch64::CBZX &&
75 SecondMI.getOpcode() != AArch64::CBNZW &&
76 SecondMI.getOpcode() != AArch64::CBNZX)
77 return false;
78
79 // Assume the 1st instr to be a wildcard if it is unspecified.
80 if (FirstMI == nullptr)
81 return true;
82
83 switch (FirstMI->getOpcode()) {
84 case AArch64::ADDWri:
85 case AArch64::ADDWrr:
86 case AArch64::ADDXri:
87 case AArch64::ADDXrr:
88 case AArch64::ANDWri:
89 case AArch64::ANDWrr:
90 case AArch64::ANDXri:
91 case AArch64::ANDXrr:
92 case AArch64::EORWri:
93 case AArch64::EORWrr:
94 case AArch64::EORXri:
95 case AArch64::EORXrr:
96 case AArch64::ORRWri:
97 case AArch64::ORRWrr:
98 case AArch64::ORRXri:
99 case AArch64::ORRXrr:
100 case AArch64::SUBWri:
101 case AArch64::SUBWrr:
102 case AArch64::SUBXri:
103 case AArch64::SUBXrr:
104 return true;
105 case AArch64::ADDWrs:
106 case AArch64::ADDXrs:
107 case AArch64::ANDWrs:
108 case AArch64::ANDXrs:
109 case AArch64::SUBWrs:
110 case AArch64::SUBXrs:
111 case AArch64::BICWrs:
112 case AArch64::BICXrs:
113 // Shift value can be 0 making these behave like the "rr" variant...
114 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
115 }
116
117 return false;
118 }
119
120 /// AES crypto encoding or decoding.
isAESPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)121 static bool isAESPair(const MachineInstr *FirstMI,
122 const MachineInstr &SecondMI) {
123 // Assume the 1st instr to be a wildcard if it is unspecified.
124 switch (SecondMI.getOpcode()) {
125 // AES encode.
126 case AArch64::AESMCrr:
127 case AArch64::AESMCrrTied:
128 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
129 // AES decode.
130 case AArch64::AESIMCrr:
131 case AArch64::AESIMCrrTied:
132 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
133 }
134
135 return false;
136 }
137
138 /// AESE/AESD/PMULL + EOR.
isCryptoEORPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)139 static bool isCryptoEORPair(const MachineInstr *FirstMI,
140 const MachineInstr &SecondMI) {
141 if (SecondMI.getOpcode() != AArch64::EORv16i8)
142 return false;
143
144 // Assume the 1st instr to be a wildcard if it is unspecified.
145 if (FirstMI == nullptr)
146 return true;
147
148 switch (FirstMI->getOpcode()) {
149 case AArch64::AESErr:
150 case AArch64::AESDrr:
151 case AArch64::PMULLv16i8:
152 case AArch64::PMULLv8i8:
153 case AArch64::PMULLv1i64:
154 case AArch64::PMULLv2i64:
155 return true;
156 }
157
158 return false;
159 }
160
161 /// Literal generation.
isLiteralsPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)162 static bool isLiteralsPair(const MachineInstr *FirstMI,
163 const MachineInstr &SecondMI) {
164 // Assume the 1st instr to be a wildcard if it is unspecified.
165
166 // PC relative address.
167 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
168 SecondMI.getOpcode() == AArch64::ADDXri)
169 return true;
170
171 // 32 bit immediate.
172 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
173 (SecondMI.getOpcode() == AArch64::MOVKWi &&
174 SecondMI.getOperand(3).getImm() == 16))
175 return true;
176
177 // Lower half of 64 bit immediate.
178 if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
179 (SecondMI.getOpcode() == AArch64::MOVKXi &&
180 SecondMI.getOperand(3).getImm() == 16))
181 return true;
182
183 // Upper half of 64 bit immediate.
184 if ((FirstMI == nullptr ||
185 (FirstMI->getOpcode() == AArch64::MOVKXi &&
186 FirstMI->getOperand(3).getImm() == 32)) &&
187 (SecondMI.getOpcode() == AArch64::MOVKXi &&
188 SecondMI.getOperand(3).getImm() == 48))
189 return true;
190
191 return false;
192 }
193
194 /// Fuse address generation and loads or stores.
isAddressLdStPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)195 static bool isAddressLdStPair(const MachineInstr *FirstMI,
196 const MachineInstr &SecondMI) {
197 switch (SecondMI.getOpcode()) {
198 case AArch64::STRBBui:
199 case AArch64::STRBui:
200 case AArch64::STRDui:
201 case AArch64::STRHHui:
202 case AArch64::STRHui:
203 case AArch64::STRQui:
204 case AArch64::STRSui:
205 case AArch64::STRWui:
206 case AArch64::STRXui:
207 case AArch64::LDRBBui:
208 case AArch64::LDRBui:
209 case AArch64::LDRDui:
210 case AArch64::LDRHHui:
211 case AArch64::LDRHui:
212 case AArch64::LDRQui:
213 case AArch64::LDRSui:
214 case AArch64::LDRWui:
215 case AArch64::LDRXui:
216 case AArch64::LDRSBWui:
217 case AArch64::LDRSBXui:
218 case AArch64::LDRSHWui:
219 case AArch64::LDRSHXui:
220 case AArch64::LDRSWui:
221 // Assume the 1st instr to be a wildcard if it is unspecified.
222 if (FirstMI == nullptr)
223 return true;
224
225 switch (FirstMI->getOpcode()) {
226 case AArch64::ADR:
227 return SecondMI.getOperand(2).getImm() == 0;
228 case AArch64::ADRP:
229 return true;
230 }
231 }
232
233 return false;
234 }
235
236 /// Compare and conditional select.
isCCSelectPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)237 static bool isCCSelectPair(const MachineInstr *FirstMI,
238 const MachineInstr &SecondMI) {
239 // 32 bits
240 if (SecondMI.getOpcode() == AArch64::CSELWr) {
241 // Assume the 1st instr to be a wildcard if it is unspecified.
242 if (FirstMI == nullptr)
243 return true;
244
245 if (FirstMI->definesRegister(AArch64::WZR))
246 switch (FirstMI->getOpcode()) {
247 case AArch64::SUBSWrs:
248 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
249 case AArch64::SUBSWrx:
250 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
251 case AArch64::SUBSWrr:
252 case AArch64::SUBSWri:
253 return true;
254 }
255 }
256
257 // 64 bits
258 if (SecondMI.getOpcode() == AArch64::CSELXr) {
259 // Assume the 1st instr to be a wildcard if it is unspecified.
260 if (FirstMI == nullptr)
261 return true;
262
263 if (FirstMI->definesRegister(AArch64::XZR))
264 switch (FirstMI->getOpcode()) {
265 case AArch64::SUBSXrs:
266 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
267 case AArch64::SUBSXrx:
268 case AArch64::SUBSXrx64:
269 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
270 case AArch64::SUBSXrr:
271 case AArch64::SUBSXri:
272 return true;
273 }
274 }
275
276 return false;
277 }
278
279 // Arithmetic and logic.
isArithmeticLogicPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)280 static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
281 const MachineInstr &SecondMI) {
282 if (AArch64InstrInfo::hasShiftedReg(SecondMI))
283 return false;
284
285 switch (SecondMI.getOpcode()) {
286 // Arithmetic
287 case AArch64::ADDWrr:
288 case AArch64::ADDXrr:
289 case AArch64::SUBWrr:
290 case AArch64::SUBXrr:
291 case AArch64::ADDWrs:
292 case AArch64::ADDXrs:
293 case AArch64::SUBWrs:
294 case AArch64::SUBXrs:
295 // Logic
296 case AArch64::ANDWrr:
297 case AArch64::ANDXrr:
298 case AArch64::BICWrr:
299 case AArch64::BICXrr:
300 case AArch64::EONWrr:
301 case AArch64::EONXrr:
302 case AArch64::EORWrr:
303 case AArch64::EORXrr:
304 case AArch64::ORNWrr:
305 case AArch64::ORNXrr:
306 case AArch64::ORRWrr:
307 case AArch64::ORRXrr:
308 case AArch64::ANDWrs:
309 case AArch64::ANDXrs:
310 case AArch64::BICWrs:
311 case AArch64::BICXrs:
312 case AArch64::EONWrs:
313 case AArch64::EONXrs:
314 case AArch64::EORWrs:
315 case AArch64::EORXrs:
316 case AArch64::ORNWrs:
317 case AArch64::ORNXrs:
318 case AArch64::ORRWrs:
319 case AArch64::ORRXrs:
320 // Assume the 1st instr to be a wildcard if it is unspecified.
321 if (FirstMI == nullptr)
322 return true;
323
324 // Arithmetic
325 switch (FirstMI->getOpcode()) {
326 case AArch64::ADDWrr:
327 case AArch64::ADDXrr:
328 case AArch64::ADDSWrr:
329 case AArch64::ADDSXrr:
330 case AArch64::SUBWrr:
331 case AArch64::SUBXrr:
332 case AArch64::SUBSWrr:
333 case AArch64::SUBSXrr:
334 return true;
335 case AArch64::ADDWrs:
336 case AArch64::ADDXrs:
337 case AArch64::ADDSWrs:
338 case AArch64::ADDSXrs:
339 case AArch64::SUBWrs:
340 case AArch64::SUBXrs:
341 case AArch64::SUBSWrs:
342 case AArch64::SUBSXrs:
343 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
344 }
345 break;
346
347 // Arithmetic, setting flags.
348 case AArch64::ADDSWrr:
349 case AArch64::ADDSXrr:
350 case AArch64::SUBSWrr:
351 case AArch64::SUBSXrr:
352 case AArch64::ADDSWrs:
353 case AArch64::ADDSXrs:
354 case AArch64::SUBSWrs:
355 case AArch64::SUBSXrs:
356 // Assume the 1st instr to be a wildcard if it is unspecified.
357 if (FirstMI == nullptr)
358 return true;
359
360 // Arithmetic, not setting flags.
361 switch (FirstMI->getOpcode()) {
362 case AArch64::ADDWrr:
363 case AArch64::ADDXrr:
364 case AArch64::SUBWrr:
365 case AArch64::SUBXrr:
366 return true;
367 case AArch64::ADDWrs:
368 case AArch64::ADDXrs:
369 case AArch64::SUBWrs:
370 case AArch64::SUBXrs:
371 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
372 }
373 break;
374 }
375
376 return false;
377 }
378
379 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
380 /// together. Given SecondMI, when FirstMI is unspecified, then check if
381 /// SecondMI may be part of a fused pair at all.
shouldScheduleAdjacent(const TargetInstrInfo & TII,const TargetSubtargetInfo & TSI,const MachineInstr * FirstMI,const MachineInstr & SecondMI)382 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
383 const TargetSubtargetInfo &TSI,
384 const MachineInstr *FirstMI,
385 const MachineInstr &SecondMI) {
386 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
387
388 // All checking functions assume that the 1st instr is a wildcard if it is
389 // unspecified.
390 if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
391 bool CmpOnly = !ST.hasArithmeticBccFusion();
392 if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly))
393 return true;
394 }
395 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
396 return true;
397 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
398 return true;
399 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
400 return true;
401 if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
402 return true;
403 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
404 return true;
405 if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
406 return true;
407 if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
408 return true;
409
410 return false;
411 }
412
413 } // end namespace
414
415
416 namespace llvm {
417
createAArch64MacroFusionDAGMutation()418 std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
419 return createMacroFusionDAGMutation(shouldScheduleAdjacent);
420 }
421
422 } // end namespace llvm
423