1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file contains the AArch64 implementation of the DAG scheduling
10 ///  mutation to pair instructions back to back.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64Subtarget.h"
15 #include "llvm/CodeGen/MacroFusion.h"
16 #include "llvm/CodeGen/TargetInstrInfo.h"
17 
18 using namespace llvm;
19 
20 namespace {
21 
22 /// CMN, CMP, TST followed by Bcc
isArithmeticBccPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)23 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
24                                 const MachineInstr &SecondMI) {
25   if (SecondMI.getOpcode() != AArch64::Bcc)
26     return false;
27 
28   // Assume the 1st instr to be a wildcard if it is unspecified.
29   if (FirstMI == nullptr)
30     return true;
31 
32   switch (FirstMI->getOpcode()) {
33   case AArch64::ADDSWri:
34   case AArch64::ADDSWrr:
35   case AArch64::ADDSXri:
36   case AArch64::ADDSXrr:
37   case AArch64::ANDSWri:
38   case AArch64::ANDSWrr:
39   case AArch64::ANDSXri:
40   case AArch64::ANDSXrr:
41   case AArch64::SUBSWri:
42   case AArch64::SUBSWrr:
43   case AArch64::SUBSXri:
44   case AArch64::SUBSXrr:
45   case AArch64::BICSWrr:
46   case AArch64::BICSXrr:
47     return true;
48   case AArch64::ADDSWrs:
49   case AArch64::ADDSXrs:
50   case AArch64::ANDSWrs:
51   case AArch64::ANDSXrs:
52   case AArch64::SUBSWrs:
53   case AArch64::SUBSXrs:
54   case AArch64::BICSWrs:
55   case AArch64::BICSXrs:
56     // Shift value can be 0 making these behave like the "rr" variant...
57     return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
58   }
59 
60   return false;
61 }
62 
63 /// ALU operations followed by CBZ/CBNZ.
isArithmeticCbzPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)64 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
65                                 const MachineInstr &SecondMI) {
66   if (SecondMI.getOpcode() != AArch64::CBZW &&
67       SecondMI.getOpcode() != AArch64::CBZX &&
68       SecondMI.getOpcode() != AArch64::CBNZW &&
69       SecondMI.getOpcode() != AArch64::CBNZX)
70     return false;
71 
72   // Assume the 1st instr to be a wildcard if it is unspecified.
73   if (FirstMI == nullptr)
74     return true;
75 
76   switch (FirstMI->getOpcode()) {
77   case AArch64::ADDWri:
78   case AArch64::ADDWrr:
79   case AArch64::ADDXri:
80   case AArch64::ADDXrr:
81   case AArch64::ANDWri:
82   case AArch64::ANDWrr:
83   case AArch64::ANDXri:
84   case AArch64::ANDXrr:
85   case AArch64::EORWri:
86   case AArch64::EORWrr:
87   case AArch64::EORXri:
88   case AArch64::EORXrr:
89   case AArch64::ORRWri:
90   case AArch64::ORRWrr:
91   case AArch64::ORRXri:
92   case AArch64::ORRXrr:
93   case AArch64::SUBWri:
94   case AArch64::SUBWrr:
95   case AArch64::SUBXri:
96   case AArch64::SUBXrr:
97     return true;
98   case AArch64::ADDWrs:
99   case AArch64::ADDXrs:
100   case AArch64::ANDWrs:
101   case AArch64::ANDXrs:
102   case AArch64::SUBWrs:
103   case AArch64::SUBXrs:
104   case AArch64::BICWrs:
105   case AArch64::BICXrs:
106     // Shift value can be 0 making these behave like the "rr" variant...
107     return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
108   }
109 
110   return false;
111 }
112 
113 /// AES crypto encoding or decoding.
isAESPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)114 static bool isAESPair(const MachineInstr *FirstMI,
115                       const MachineInstr &SecondMI) {
116   // Assume the 1st instr to be a wildcard if it is unspecified.
117   switch (SecondMI.getOpcode()) {
118   // AES encode.
119   case AArch64::AESMCrr:
120   case AArch64::AESMCrrTied:
121     return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
122   // AES decode.
123   case AArch64::AESIMCrr:
124   case AArch64::AESIMCrrTied:
125     return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
126   }
127 
128   return false;
129 }
130 
131 /// AESE/AESD/PMULL + EOR.
isCryptoEORPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)132 static bool isCryptoEORPair(const MachineInstr *FirstMI,
133                             const MachineInstr &SecondMI) {
134   if (SecondMI.getOpcode() != AArch64::EORv16i8)
135     return false;
136 
137   // Assume the 1st instr to be a wildcard if it is unspecified.
138   if (FirstMI == nullptr)
139     return true;
140 
141   switch (FirstMI->getOpcode()) {
142   case AArch64::AESErr:
143   case AArch64::AESDrr:
144   case AArch64::PMULLv16i8:
145   case AArch64::PMULLv8i8:
146   case AArch64::PMULLv1i64:
147   case AArch64::PMULLv2i64:
148     return true;
149   }
150 
151   return false;
152 }
153 
154 /// Literal generation.
isLiteralsPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)155 static bool isLiteralsPair(const MachineInstr *FirstMI,
156                            const MachineInstr &SecondMI) {
157   // Assume the 1st instr to be a wildcard if it is unspecified.
158 
159   // PC relative address.
160   if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
161       SecondMI.getOpcode() == AArch64::ADDXri)
162     return true;
163 
164   // 32 bit immediate.
165   if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
166       (SecondMI.getOpcode() == AArch64::MOVKWi &&
167        SecondMI.getOperand(3).getImm() == 16))
168     return true;
169 
170   // Lower half of 64 bit immediate.
171   if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
172      (SecondMI.getOpcode() == AArch64::MOVKXi &&
173       SecondMI.getOperand(3).getImm() == 16))
174     return true;
175 
176   // Upper half of 64 bit immediate.
177   if ((FirstMI == nullptr ||
178        (FirstMI->getOpcode() == AArch64::MOVKXi &&
179         FirstMI->getOperand(3).getImm() == 32)) &&
180       (SecondMI.getOpcode() == AArch64::MOVKXi &&
181        SecondMI.getOperand(3).getImm() == 48))
182     return true;
183 
184   return false;
185 }
186 
187 /// Fuse address generation and loads or stores.
isAddressLdStPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)188 static bool isAddressLdStPair(const MachineInstr *FirstMI,
189                               const MachineInstr &SecondMI) {
190   switch (SecondMI.getOpcode()) {
191   case AArch64::STRBBui:
192   case AArch64::STRBui:
193   case AArch64::STRDui:
194   case AArch64::STRHHui:
195   case AArch64::STRHui:
196   case AArch64::STRQui:
197   case AArch64::STRSui:
198   case AArch64::STRWui:
199   case AArch64::STRXui:
200   case AArch64::LDRBBui:
201   case AArch64::LDRBui:
202   case AArch64::LDRDui:
203   case AArch64::LDRHHui:
204   case AArch64::LDRHui:
205   case AArch64::LDRQui:
206   case AArch64::LDRSui:
207   case AArch64::LDRWui:
208   case AArch64::LDRXui:
209   case AArch64::LDRSBWui:
210   case AArch64::LDRSBXui:
211   case AArch64::LDRSHWui:
212   case AArch64::LDRSHXui:
213   case AArch64::LDRSWui:
214     // Assume the 1st instr to be a wildcard if it is unspecified.
215     if (FirstMI == nullptr)
216       return true;
217 
218    switch (FirstMI->getOpcode()) {
219     case AArch64::ADR:
220       return SecondMI.getOperand(2).getImm() == 0;
221     case AArch64::ADRP:
222       return true;
223     }
224   }
225 
226   return false;
227 }
228 
229 /// Compare and conditional select.
isCCSelectPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)230 static bool isCCSelectPair(const MachineInstr *FirstMI,
231                            const MachineInstr &SecondMI) {
232   // 32 bits
233   if (SecondMI.getOpcode() == AArch64::CSELWr) {
234     // Assume the 1st instr to be a wildcard if it is unspecified.
235     if (FirstMI == nullptr)
236       return true;
237 
238     if (FirstMI->definesRegister(AArch64::WZR))
239       switch (FirstMI->getOpcode()) {
240       case AArch64::SUBSWrs:
241         return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
242       case AArch64::SUBSWrx:
243         return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
244       case AArch64::SUBSWrr:
245       case AArch64::SUBSWri:
246         return true;
247       }
248   }
249 
250   // 64 bits
251   if (SecondMI.getOpcode() == AArch64::CSELXr) {
252     // Assume the 1st instr to be a wildcard if it is unspecified.
253     if (FirstMI == nullptr)
254       return true;
255 
256     if (FirstMI->definesRegister(AArch64::XZR))
257       switch (FirstMI->getOpcode()) {
258       case AArch64::SUBSXrs:
259         return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
260       case AArch64::SUBSXrx:
261       case AArch64::SUBSXrx64:
262         return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
263       case AArch64::SUBSXrr:
264       case AArch64::SUBSXri:
265         return true;
266       }
267   }
268 
269   return false;
270 }
271 
272 // Arithmetic and logic.
isArithmeticLogicPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)273 static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
274                                   const MachineInstr &SecondMI) {
275   if (AArch64InstrInfo::hasShiftedReg(SecondMI))
276     return false;
277 
278   switch (SecondMI.getOpcode()) {
279   // Arithmetic
280   case AArch64::ADDWrr:
281   case AArch64::ADDXrr:
282   case AArch64::SUBWrr:
283   case AArch64::SUBXrr:
284   case AArch64::ADDWrs:
285   case AArch64::ADDXrs:
286   case AArch64::SUBWrs:
287   case AArch64::SUBXrs:
288   // Logic
289   case AArch64::ANDWrr:
290   case AArch64::ANDXrr:
291   case AArch64::BICWrr:
292   case AArch64::BICXrr:
293   case AArch64::EONWrr:
294   case AArch64::EONXrr:
295   case AArch64::EORWrr:
296   case AArch64::EORXrr:
297   case AArch64::ORNWrr:
298   case AArch64::ORNXrr:
299   case AArch64::ORRWrr:
300   case AArch64::ORRXrr:
301   case AArch64::ANDWrs:
302   case AArch64::ANDXrs:
303   case AArch64::BICWrs:
304   case AArch64::BICXrs:
305   case AArch64::EONWrs:
306   case AArch64::EONXrs:
307   case AArch64::EORWrs:
308   case AArch64::EORXrs:
309   case AArch64::ORNWrs:
310   case AArch64::ORNXrs:
311   case AArch64::ORRWrs:
312   case AArch64::ORRXrs:
313     // Assume the 1st instr to be a wildcard if it is unspecified.
314     if (FirstMI == nullptr)
315       return true;
316 
317     // Arithmetic
318     switch (FirstMI->getOpcode()) {
319     case AArch64::ADDWrr:
320     case AArch64::ADDXrr:
321     case AArch64::ADDSWrr:
322     case AArch64::ADDSXrr:
323     case AArch64::SUBWrr:
324     case AArch64::SUBXrr:
325     case AArch64::SUBSWrr:
326     case AArch64::SUBSXrr:
327       return true;
328     case AArch64::ADDWrs:
329     case AArch64::ADDXrs:
330     case AArch64::ADDSWrs:
331     case AArch64::ADDSXrs:
332     case AArch64::SUBWrs:
333     case AArch64::SUBXrs:
334     case AArch64::SUBSWrs:
335     case AArch64::SUBSXrs:
336       return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
337     }
338     break;
339 
340   // Arithmetic, setting flags.
341   case AArch64::ADDSWrr:
342   case AArch64::ADDSXrr:
343   case AArch64::SUBSWrr:
344   case AArch64::SUBSXrr:
345   case AArch64::ADDSWrs:
346   case AArch64::ADDSXrs:
347   case AArch64::SUBSWrs:
348   case AArch64::SUBSXrs:
349     // Assume the 1st instr to be a wildcard if it is unspecified.
350     if (FirstMI == nullptr)
351       return true;
352 
353     // Arithmetic, not setting flags.
354     switch (FirstMI->getOpcode()) {
355     case AArch64::ADDWrr:
356     case AArch64::ADDXrr:
357     case AArch64::SUBWrr:
358     case AArch64::SUBXrr:
359       return true;
360     case AArch64::ADDWrs:
361     case AArch64::ADDXrs:
362     case AArch64::SUBWrs:
363     case AArch64::SUBXrs:
364       return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
365     }
366     break;
367   }
368 
369   return false;
370 }
371 
372 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
373 /// together. Given SecondMI, when FirstMI is unspecified, then check if
374 /// SecondMI may be part of a fused pair at all.
shouldScheduleAdjacent(const TargetInstrInfo & TII,const TargetSubtargetInfo & TSI,const MachineInstr * FirstMI,const MachineInstr & SecondMI)375 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
376                                    const TargetSubtargetInfo &TSI,
377                                    const MachineInstr *FirstMI,
378                                    const MachineInstr &SecondMI) {
379   const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
380 
381   // All checking functions assume that the 1st instr is a wildcard if it is
382   // unspecified.
383   if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI))
384     return true;
385   if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
386     return true;
387   if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
388     return true;
389   if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
390     return true;
391   if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
392     return true;
393   if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
394     return true;
395   if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
396     return true;
397   if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
398     return true;
399 
400   return false;
401 }
402 
403 } // end namespace
404 
405 
406 namespace llvm {
407 
createAArch64MacroFusionDAGMutation()408 std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
409   return createMacroFusionDAGMutation(shouldScheduleAdjacent);
410 }
411 
412 } // end namespace llvm
413