1 //===- AArch64RegisterBankInfo.cpp ----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the RegisterBankInfo class for
10 /// AArch64.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13
14 #include "AArch64RegisterBankInfo.h"
15 #include "AArch64InstrInfo.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
18 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
19 #include "llvm/CodeGen/LowLevelType.h"
20 #include "llvm/CodeGen/MachineFunction.h"
21 #include "llvm/CodeGen/MachineInstr.h"
22 #include "llvm/CodeGen/MachineOperand.h"
23 #include "llvm/CodeGen/MachineRegisterInfo.h"
24 #include "llvm/CodeGen/TargetOpcodes.h"
25 #include "llvm/CodeGen/TargetRegisterInfo.h"
26 #include "llvm/CodeGen/TargetSubtargetInfo.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include <algorithm>
29 #include <cassert>
30
31 #define GET_TARGET_REGBANK_IMPL
32 #include "AArch64GenRegisterBank.inc"
33
34 // This file will be TableGen'ed at some point.
35 #include "AArch64GenRegisterBankInfo.def"
36
37 using namespace llvm;
38
AArch64RegisterBankInfo(const TargetRegisterInfo & TRI)39 AArch64RegisterBankInfo::AArch64RegisterBankInfo(const TargetRegisterInfo &TRI)
40 : AArch64GenRegisterBankInfo() {
41 static llvm::once_flag InitializeRegisterBankFlag;
42
43 static auto InitializeRegisterBankOnce = [&]() {
44 // We have only one set of register banks, whatever the subtarget
45 // is. Therefore, the initialization of the RegBanks table should be
46 // done only once. Indeed the table of all register banks
47 // (AArch64::RegBanks) is unique in the compiler. At some point, it
48 // will get tablegen'ed and the whole constructor becomes empty.
49
50 const RegisterBank &RBGPR = getRegBank(AArch64::GPRRegBankID);
51 (void)RBGPR;
52 assert(&AArch64::GPRRegBank == &RBGPR &&
53 "The order in RegBanks is messed up");
54
55 const RegisterBank &RBFPR = getRegBank(AArch64::FPRRegBankID);
56 (void)RBFPR;
57 assert(&AArch64::FPRRegBank == &RBFPR &&
58 "The order in RegBanks is messed up");
59
60 const RegisterBank &RBCCR = getRegBank(AArch64::CCRegBankID);
61 (void)RBCCR;
62 assert(&AArch64::CCRegBank == &RBCCR &&
63 "The order in RegBanks is messed up");
64
65 // The GPR register bank is fully defined by all the registers in
66 // GR64all + its subclasses.
67 assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) &&
68 "Subclass not added?");
69 assert(RBGPR.getSize() == 64 && "GPRs should hold up to 64-bit");
70
71 // The FPR register bank is fully defined by all the registers in
72 // GR64all + its subclasses.
73 assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) &&
74 "Subclass not added?");
75 assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) &&
76 "Subclass not added?");
77 assert(RBFPR.getSize() == 512 &&
78 "FPRs should hold up to 512-bit via QQQQ sequence");
79
80 assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) &&
81 "Class not added?");
82 assert(RBCCR.getSize() == 32 && "CCR should hold up to 32-bit");
83
84 // Check that the TableGen'ed like file is in sync we our expectations.
85 // First, the Idx.
86 assert(checkPartialMappingIdx(PMI_FirstGPR, PMI_LastGPR,
87 {PMI_GPR32, PMI_GPR64}) &&
88 "PartialMappingIdx's are incorrectly ordered");
89 assert(checkPartialMappingIdx(PMI_FirstFPR, PMI_LastFPR,
90 {PMI_FPR16, PMI_FPR32, PMI_FPR64, PMI_FPR128,
91 PMI_FPR256, PMI_FPR512}) &&
92 "PartialMappingIdx's are incorrectly ordered");
93 // Now, the content.
94 // Check partial mapping.
95 #define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB) \
96 do { \
97 assert( \
98 checkPartialMap(PartialMappingIdx::Idx, ValStartIdx, ValLength, RB) && \
99 #Idx " is incorrectly initialized"); \
100 } while (false)
101
102 CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR);
103 CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR);
104 CHECK_PARTIALMAP(PMI_FPR16, 0, 16, RBFPR);
105 CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR);
106 CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR);
107 CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR);
108 CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR);
109 CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR);
110
111 // Check value mapping.
112 #define CHECK_VALUEMAP_IMPL(RBName, Size, Offset) \
113 do { \
114 assert(checkValueMapImpl(PartialMappingIdx::PMI_##RBName##Size, \
115 PartialMappingIdx::PMI_First##RBName, Size, \
116 Offset) && \
117 #RBName #Size " " #Offset " is incorrectly initialized"); \
118 } while (false)
119
120 #define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0)
121
122 CHECK_VALUEMAP(GPR, 32);
123 CHECK_VALUEMAP(GPR, 64);
124 CHECK_VALUEMAP(FPR, 16);
125 CHECK_VALUEMAP(FPR, 32);
126 CHECK_VALUEMAP(FPR, 64);
127 CHECK_VALUEMAP(FPR, 128);
128 CHECK_VALUEMAP(FPR, 256);
129 CHECK_VALUEMAP(FPR, 512);
130
131 // Check the value mapping for 3-operands instructions where all the operands
132 // map to the same value mapping.
133 #define CHECK_VALUEMAP_3OPS(RBName, Size) \
134 do { \
135 CHECK_VALUEMAP_IMPL(RBName, Size, 0); \
136 CHECK_VALUEMAP_IMPL(RBName, Size, 1); \
137 CHECK_VALUEMAP_IMPL(RBName, Size, 2); \
138 } while (false)
139
140 CHECK_VALUEMAP_3OPS(GPR, 32);
141 CHECK_VALUEMAP_3OPS(GPR, 64);
142 CHECK_VALUEMAP_3OPS(FPR, 32);
143 CHECK_VALUEMAP_3OPS(FPR, 64);
144 CHECK_VALUEMAP_3OPS(FPR, 128);
145 CHECK_VALUEMAP_3OPS(FPR, 256);
146 CHECK_VALUEMAP_3OPS(FPR, 512);
147
148 #define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size) \
149 do { \
150 unsigned PartialMapDstIdx = PMI_##RBNameDst##Size - PMI_Min; \
151 unsigned PartialMapSrcIdx = PMI_##RBNameSrc##Size - PMI_Min; \
152 (void)PartialMapDstIdx; \
153 (void)PartialMapSrcIdx; \
154 const ValueMapping *Map = getCopyMapping( \
155 AArch64::RBNameDst##RegBankID, AArch64::RBNameSrc##RegBankID, Size); \
156 (void)Map; \
157 assert(Map[0].BreakDown == \
158 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
159 Map[0].NumBreakDowns == 1 && #RBNameDst #Size \
160 " Dst is incorrectly initialized"); \
161 assert(Map[1].BreakDown == \
162 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
163 Map[1].NumBreakDowns == 1 && #RBNameSrc #Size \
164 " Src is incorrectly initialized"); \
165 \
166 } while (false)
167
168 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32);
169 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 32);
170 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64);
171 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 64);
172 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 32);
173 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 32);
174 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 64);
175 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 64);
176
177 #define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize) \
178 do { \
179 unsigned PartialMapDstIdx = PMI_FPR##DstSize - PMI_Min; \
180 unsigned PartialMapSrcIdx = PMI_FPR##SrcSize - PMI_Min; \
181 (void)PartialMapDstIdx; \
182 (void)PartialMapSrcIdx; \
183 const ValueMapping *Map = getFPExtMapping(DstSize, SrcSize); \
184 (void)Map; \
185 assert(Map[0].BreakDown == \
186 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
187 Map[0].NumBreakDowns == 1 && "FPR" #DstSize \
188 " Dst is incorrectly initialized"); \
189 assert(Map[1].BreakDown == \
190 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
191 Map[1].NumBreakDowns == 1 && "FPR" #SrcSize \
192 " Src is incorrectly initialized"); \
193 \
194 } while (false)
195
196 CHECK_VALUEMAP_FPEXT(32, 16);
197 CHECK_VALUEMAP_FPEXT(64, 16);
198 CHECK_VALUEMAP_FPEXT(64, 32);
199 CHECK_VALUEMAP_FPEXT(128, 64);
200
201 assert(verify(TRI) && "Invalid register bank information");
202 };
203
204 llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce);
205 }
206
copyCost(const RegisterBank & A,const RegisterBank & B,unsigned Size) const207 unsigned AArch64RegisterBankInfo::copyCost(const RegisterBank &A,
208 const RegisterBank &B,
209 unsigned Size) const {
210 // What do we do with different size?
211 // copy are same size.
212 // Will introduce other hooks for different size:
213 // * extract cost.
214 // * build_sequence cost.
215
216 // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV.
217 // FIXME: This should be deduced from the scheduling model.
218 if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank)
219 // FMOVXDr or FMOVWSr.
220 return 5;
221 if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank)
222 // FMOVDXr or FMOVSWr.
223 return 4;
224
225 return RegisterBankInfo::copyCost(A, B, Size);
226 }
227
228 const RegisterBank &
getRegBankFromRegClass(const TargetRegisterClass & RC,LLT) const229 AArch64RegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
230 LLT) const {
231 switch (RC.getID()) {
232 case AArch64::FPR8RegClassID:
233 case AArch64::FPR16RegClassID:
234 case AArch64::FPR16_loRegClassID:
235 case AArch64::FPR32_with_hsub_in_FPR16_loRegClassID:
236 case AArch64::FPR32RegClassID:
237 case AArch64::FPR64RegClassID:
238 case AArch64::FPR64_loRegClassID:
239 case AArch64::FPR128RegClassID:
240 case AArch64::FPR128_loRegClassID:
241 case AArch64::DDRegClassID:
242 case AArch64::DDDRegClassID:
243 case AArch64::DDDDRegClassID:
244 case AArch64::QQRegClassID:
245 case AArch64::QQQRegClassID:
246 case AArch64::QQQQRegClassID:
247 return getRegBank(AArch64::FPRRegBankID);
248 case AArch64::GPR32commonRegClassID:
249 case AArch64::GPR32RegClassID:
250 case AArch64::GPR32spRegClassID:
251 case AArch64::GPR32sponlyRegClassID:
252 case AArch64::GPR32argRegClassID:
253 case AArch64::GPR32allRegClassID:
254 case AArch64::GPR64commonRegClassID:
255 case AArch64::GPR64RegClassID:
256 case AArch64::GPR64spRegClassID:
257 case AArch64::GPR64sponlyRegClassID:
258 case AArch64::GPR64argRegClassID:
259 case AArch64::GPR64allRegClassID:
260 case AArch64::GPR64noipRegClassID:
261 case AArch64::GPR64common_and_GPR64noipRegClassID:
262 case AArch64::GPR64noip_and_tcGPR64RegClassID:
263 case AArch64::tcGPR64RegClassID:
264 case AArch64::rtcGPR64RegClassID:
265 case AArch64::WSeqPairsClassRegClassID:
266 case AArch64::XSeqPairsClassRegClassID:
267 return getRegBank(AArch64::GPRRegBankID);
268 case AArch64::CCRRegClassID:
269 return getRegBank(AArch64::CCRegBankID);
270 default:
271 llvm_unreachable("Register class not supported");
272 }
273 }
274
275 RegisterBankInfo::InstructionMappings
getInstrAlternativeMappings(const MachineInstr & MI) const276 AArch64RegisterBankInfo::getInstrAlternativeMappings(
277 const MachineInstr &MI) const {
278 const MachineFunction &MF = *MI.getParent()->getParent();
279 const TargetSubtargetInfo &STI = MF.getSubtarget();
280 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
281 const MachineRegisterInfo &MRI = MF.getRegInfo();
282
283 switch (MI.getOpcode()) {
284 case TargetOpcode::G_OR: {
285 // 32 and 64-bit or can be mapped on either FPR or
286 // GPR for the same cost.
287 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
288 if (Size != 32 && Size != 64)
289 break;
290
291 // If the instruction has any implicit-defs or uses,
292 // do not mess with it.
293 if (MI.getNumOperands() != 3)
294 break;
295 InstructionMappings AltMappings;
296 const InstructionMapping &GPRMapping = getInstructionMapping(
297 /*ID*/ 1, /*Cost*/ 1, getValueMapping(PMI_FirstGPR, Size),
298 /*NumOperands*/ 3);
299 const InstructionMapping &FPRMapping = getInstructionMapping(
300 /*ID*/ 2, /*Cost*/ 1, getValueMapping(PMI_FirstFPR, Size),
301 /*NumOperands*/ 3);
302
303 AltMappings.push_back(&GPRMapping);
304 AltMappings.push_back(&FPRMapping);
305 return AltMappings;
306 }
307 case TargetOpcode::G_BITCAST: {
308 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
309 if (Size != 32 && Size != 64)
310 break;
311
312 // If the instruction has any implicit-defs or uses,
313 // do not mess with it.
314 if (MI.getNumOperands() != 2)
315 break;
316
317 InstructionMappings AltMappings;
318 const InstructionMapping &GPRMapping = getInstructionMapping(
319 /*ID*/ 1, /*Cost*/ 1,
320 getCopyMapping(AArch64::GPRRegBankID, AArch64::GPRRegBankID, Size),
321 /*NumOperands*/ 2);
322 const InstructionMapping &FPRMapping = getInstructionMapping(
323 /*ID*/ 2, /*Cost*/ 1,
324 getCopyMapping(AArch64::FPRRegBankID, AArch64::FPRRegBankID, Size),
325 /*NumOperands*/ 2);
326 const InstructionMapping &GPRToFPRMapping = getInstructionMapping(
327 /*ID*/ 3,
328 /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size),
329 getCopyMapping(AArch64::FPRRegBankID, AArch64::GPRRegBankID, Size),
330 /*NumOperands*/ 2);
331 const InstructionMapping &FPRToGPRMapping = getInstructionMapping(
332 /*ID*/ 3,
333 /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size),
334 getCopyMapping(AArch64::GPRRegBankID, AArch64::FPRRegBankID, Size),
335 /*NumOperands*/ 2);
336
337 AltMappings.push_back(&GPRMapping);
338 AltMappings.push_back(&FPRMapping);
339 AltMappings.push_back(&GPRToFPRMapping);
340 AltMappings.push_back(&FPRToGPRMapping);
341 return AltMappings;
342 }
343 case TargetOpcode::G_LOAD: {
344 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
345 if (Size != 64)
346 break;
347
348 // If the instruction has any implicit-defs or uses,
349 // do not mess with it.
350 if (MI.getNumOperands() != 2)
351 break;
352
353 InstructionMappings AltMappings;
354 const InstructionMapping &GPRMapping = getInstructionMapping(
355 /*ID*/ 1, /*Cost*/ 1,
356 getOperandsMapping({getValueMapping(PMI_FirstGPR, Size),
357 // Addresses are GPR 64-bit.
358 getValueMapping(PMI_FirstGPR, 64)}),
359 /*NumOperands*/ 2);
360 const InstructionMapping &FPRMapping = getInstructionMapping(
361 /*ID*/ 2, /*Cost*/ 1,
362 getOperandsMapping({getValueMapping(PMI_FirstFPR, Size),
363 // Addresses are GPR 64-bit.
364 getValueMapping(PMI_FirstGPR, 64)}),
365 /*NumOperands*/ 2);
366
367 AltMappings.push_back(&GPRMapping);
368 AltMappings.push_back(&FPRMapping);
369 return AltMappings;
370 }
371 default:
372 break;
373 }
374 return RegisterBankInfo::getInstrAlternativeMappings(MI);
375 }
376
applyMappingImpl(const OperandsMapper & OpdMapper) const377 void AArch64RegisterBankInfo::applyMappingImpl(
378 const OperandsMapper &OpdMapper) const {
379 switch (OpdMapper.getMI().getOpcode()) {
380 case TargetOpcode::G_OR:
381 case TargetOpcode::G_BITCAST:
382 case TargetOpcode::G_LOAD:
383 // Those ID must match getInstrAlternativeMappings.
384 assert((OpdMapper.getInstrMapping().getID() >= 1 &&
385 OpdMapper.getInstrMapping().getID() <= 4) &&
386 "Don't know how to handle that ID");
387 return applyDefaultMapping(OpdMapper);
388 default:
389 llvm_unreachable("Don't know how to handle that operation");
390 }
391 }
392
393 /// Returns whether opcode \p Opc is a pre-isel generic floating-point opcode,
394 /// having only floating-point operands.
isPreISelGenericFloatingPointOpcode(unsigned Opc)395 static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) {
396 switch (Opc) {
397 case TargetOpcode::G_FADD:
398 case TargetOpcode::G_FSUB:
399 case TargetOpcode::G_FMUL:
400 case TargetOpcode::G_FMA:
401 case TargetOpcode::G_FDIV:
402 case TargetOpcode::G_FCONSTANT:
403 case TargetOpcode::G_FPEXT:
404 case TargetOpcode::G_FPTRUNC:
405 case TargetOpcode::G_FCEIL:
406 case TargetOpcode::G_FFLOOR:
407 case TargetOpcode::G_FNEARBYINT:
408 case TargetOpcode::G_FNEG:
409 case TargetOpcode::G_FCOS:
410 case TargetOpcode::G_FSIN:
411 case TargetOpcode::G_FLOG10:
412 case TargetOpcode::G_FLOG:
413 case TargetOpcode::G_FLOG2:
414 case TargetOpcode::G_FSQRT:
415 case TargetOpcode::G_FABS:
416 case TargetOpcode::G_FEXP:
417 case TargetOpcode::G_FRINT:
418 case TargetOpcode::G_INTRINSIC_TRUNC:
419 case TargetOpcode::G_INTRINSIC_ROUND:
420 return true;
421 }
422 return false;
423 }
424
425 const RegisterBankInfo::InstructionMapping &
getSameKindOfOperandsMapping(const MachineInstr & MI) const426 AArch64RegisterBankInfo::getSameKindOfOperandsMapping(
427 const MachineInstr &MI) const {
428 const unsigned Opc = MI.getOpcode();
429 const MachineFunction &MF = *MI.getParent()->getParent();
430 const MachineRegisterInfo &MRI = MF.getRegInfo();
431
432 unsigned NumOperands = MI.getNumOperands();
433 assert(NumOperands <= 3 &&
434 "This code is for instructions with 3 or less operands");
435
436 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
437 unsigned Size = Ty.getSizeInBits();
438 bool IsFPR = Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
439
440 PartialMappingIdx RBIdx = IsFPR ? PMI_FirstFPR : PMI_FirstGPR;
441
442 #ifndef NDEBUG
443 // Make sure all the operands are using similar size and type.
444 // Should probably be checked by the machine verifier.
445 // This code won't catch cases where the number of lanes is
446 // different between the operands.
447 // If we want to go to that level of details, it is probably
448 // best to check that the types are the same, period.
449 // Currently, we just check that the register banks are the same
450 // for each types.
451 for (unsigned Idx = 1; Idx != NumOperands; ++Idx) {
452 LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg());
453 assert(
454 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(
455 RBIdx, OpTy.getSizeInBits()) ==
456 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(RBIdx, Size) &&
457 "Operand has incompatible size");
458 bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
459 (void)OpIsFPR;
460 assert(IsFPR == OpIsFPR && "Operand has incompatible type");
461 }
462 #endif // End NDEBUG.
463
464 return getInstructionMapping(DefaultMappingID, 1,
465 getValueMapping(RBIdx, Size), NumOperands);
466 }
467
hasFPConstraints(const MachineInstr & MI,const MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI) const468 bool AArch64RegisterBankInfo::hasFPConstraints(
469 const MachineInstr &MI, const MachineRegisterInfo &MRI,
470 const TargetRegisterInfo &TRI) const {
471 unsigned Op = MI.getOpcode();
472
473 // Do we have an explicit floating point instruction?
474 if (isPreISelGenericFloatingPointOpcode(Op))
475 return true;
476
477 // No. Check if we have a copy-like instruction. If we do, then we could
478 // still be fed by floating point instructions.
479 if (Op != TargetOpcode::COPY && !MI.isPHI())
480 return false;
481
482 // MI is copy-like. Return true if it outputs an FPR.
483 return getRegBank(MI.getOperand(0).getReg(), MRI, TRI) ==
484 &AArch64::FPRRegBank;
485 }
486
onlyUsesFP(const MachineInstr & MI,const MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI) const487 bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
488 const MachineRegisterInfo &MRI,
489 const TargetRegisterInfo &TRI) const {
490 switch (MI.getOpcode()) {
491 case TargetOpcode::G_FPTOSI:
492 case TargetOpcode::G_FPTOUI:
493 case TargetOpcode::G_FCMP:
494 return true;
495 default:
496 break;
497 }
498 return hasFPConstraints(MI, MRI, TRI);
499 }
500
onlyDefinesFP(const MachineInstr & MI,const MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI) const501 bool AArch64RegisterBankInfo::onlyDefinesFP(
502 const MachineInstr &MI, const MachineRegisterInfo &MRI,
503 const TargetRegisterInfo &TRI) const {
504 switch (MI.getOpcode()) {
505 case AArch64::G_DUP:
506 case TargetOpcode::G_SITOFP:
507 case TargetOpcode::G_UITOFP:
508 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
509 case TargetOpcode::G_INSERT_VECTOR_ELT:
510 return true;
511 default:
512 break;
513 }
514 return hasFPConstraints(MI, MRI, TRI);
515 }
516
517 const RegisterBankInfo::InstructionMapping &
getInstrMapping(const MachineInstr & MI) const518 AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
519 const unsigned Opc = MI.getOpcode();
520
521 // Try the default logic for non-generic instructions that are either copies
522 // or already have some operands assigned to banks.
523 if ((Opc != TargetOpcode::COPY && !isPreISelGenericOpcode(Opc)) ||
524 Opc == TargetOpcode::G_PHI) {
525 const RegisterBankInfo::InstructionMapping &Mapping =
526 getInstrMappingImpl(MI);
527 if (Mapping.isValid())
528 return Mapping;
529 }
530
531 const MachineFunction &MF = *MI.getParent()->getParent();
532 const MachineRegisterInfo &MRI = MF.getRegInfo();
533 const TargetSubtargetInfo &STI = MF.getSubtarget();
534 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
535
536 switch (Opc) {
537 // G_{F|S|U}REM are not listed because they are not legal.
538 // Arithmetic ops.
539 case TargetOpcode::G_ADD:
540 case TargetOpcode::G_SUB:
541 case TargetOpcode::G_PTR_ADD:
542 case TargetOpcode::G_MUL:
543 case TargetOpcode::G_SDIV:
544 case TargetOpcode::G_UDIV:
545 // Bitwise ops.
546 case TargetOpcode::G_AND:
547 case TargetOpcode::G_OR:
548 case TargetOpcode::G_XOR:
549 // Floating point ops.
550 case TargetOpcode::G_FADD:
551 case TargetOpcode::G_FSUB:
552 case TargetOpcode::G_FMUL:
553 case TargetOpcode::G_FDIV:
554 return getSameKindOfOperandsMapping(MI);
555 case TargetOpcode::G_FPEXT: {
556 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
557 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
558 return getInstructionMapping(
559 DefaultMappingID, /*Cost*/ 1,
560 getFPExtMapping(DstTy.getSizeInBits(), SrcTy.getSizeInBits()),
561 /*NumOperands*/ 2);
562 }
563 // Shifts.
564 case TargetOpcode::G_SHL:
565 case TargetOpcode::G_LSHR:
566 case TargetOpcode::G_ASHR: {
567 LLT ShiftAmtTy = MRI.getType(MI.getOperand(2).getReg());
568 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
569 if (ShiftAmtTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() == 32)
570 return getInstructionMapping(DefaultMappingID, 1,
571 &ValMappings[Shift64Imm], 3);
572 return getSameKindOfOperandsMapping(MI);
573 }
574 case TargetOpcode::COPY: {
575 Register DstReg = MI.getOperand(0).getReg();
576 Register SrcReg = MI.getOperand(1).getReg();
577 // Check if one of the register is not a generic register.
578 if ((Register::isPhysicalRegister(DstReg) ||
579 !MRI.getType(DstReg).isValid()) ||
580 (Register::isPhysicalRegister(SrcReg) ||
581 !MRI.getType(SrcReg).isValid())) {
582 const RegisterBank *DstRB = getRegBank(DstReg, MRI, TRI);
583 const RegisterBank *SrcRB = getRegBank(SrcReg, MRI, TRI);
584 if (!DstRB)
585 DstRB = SrcRB;
586 else if (!SrcRB)
587 SrcRB = DstRB;
588 // If both RB are null that means both registers are generic.
589 // We shouldn't be here.
590 assert(DstRB && SrcRB && "Both RegBank were nullptr");
591 unsigned Size = getSizeInBits(DstReg, MRI, TRI);
592 return getInstructionMapping(
593 DefaultMappingID, copyCost(*DstRB, *SrcRB, Size),
594 getCopyMapping(DstRB->getID(), SrcRB->getID(), Size),
595 // We only care about the mapping of the destination.
596 /*NumOperands*/ 1);
597 }
598 // Both registers are generic, use G_BITCAST.
599 LLVM_FALLTHROUGH;
600 }
601 case TargetOpcode::G_BITCAST: {
602 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
603 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
604 unsigned Size = DstTy.getSizeInBits();
605 bool DstIsGPR = !DstTy.isVector() && DstTy.getSizeInBits() <= 64;
606 bool SrcIsGPR = !SrcTy.isVector() && SrcTy.getSizeInBits() <= 64;
607 const RegisterBank &DstRB =
608 DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
609 const RegisterBank &SrcRB =
610 SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
611 return getInstructionMapping(
612 DefaultMappingID, copyCost(DstRB, SrcRB, Size),
613 getCopyMapping(DstRB.getID(), SrcRB.getID(), Size),
614 // We only care about the mapping of the destination for COPY.
615 /*NumOperands*/ Opc == TargetOpcode::G_BITCAST ? 2 : 1);
616 }
617 default:
618 break;
619 }
620
621 unsigned NumOperands = MI.getNumOperands();
622
623 // Track the size and bank of each register. We don't do partial mappings.
624 SmallVector<unsigned, 4> OpSize(NumOperands);
625 SmallVector<PartialMappingIdx, 4> OpRegBankIdx(NumOperands);
626 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
627 auto &MO = MI.getOperand(Idx);
628 if (!MO.isReg() || !MO.getReg())
629 continue;
630
631 LLT Ty = MRI.getType(MO.getReg());
632 OpSize[Idx] = Ty.getSizeInBits();
633
634 // As a top-level guess, vectors go in FPRs, scalars and pointers in GPRs.
635 // For floating-point instructions, scalars go in FPRs.
636 if (Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc) ||
637 Ty.getSizeInBits() > 64)
638 OpRegBankIdx[Idx] = PMI_FirstFPR;
639 else
640 OpRegBankIdx[Idx] = PMI_FirstGPR;
641 }
642
643 unsigned Cost = 1;
644 // Some of the floating-point instructions have mixed GPR and FPR operands:
645 // fine-tune the computed mapping.
646 switch (Opc) {
647 case AArch64::G_DUP: {
648 Register ScalarReg = MI.getOperand(1).getReg();
649 auto ScalarDef = MRI.getVRegDef(ScalarReg);
650 if (getRegBank(ScalarReg, MRI, TRI) == &AArch64::FPRRegBank ||
651 onlyDefinesFP(*ScalarDef, MRI, TRI))
652 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
653 else
654 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
655 break;
656 }
657 case TargetOpcode::G_TRUNC: {
658 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
659 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128)
660 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
661 break;
662 }
663 case TargetOpcode::G_SITOFP:
664 case TargetOpcode::G_UITOFP:
665 if (MRI.getType(MI.getOperand(0).getReg()).isVector())
666 break;
667 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
668 break;
669 case TargetOpcode::G_FPTOSI:
670 case TargetOpcode::G_FPTOUI:
671 if (MRI.getType(MI.getOperand(0).getReg()).isVector())
672 break;
673 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
674 break;
675 case TargetOpcode::G_FCMP:
676 OpRegBankIdx = {PMI_FirstGPR,
677 /* Predicate */ PMI_None, PMI_FirstFPR, PMI_FirstFPR};
678 break;
679 case TargetOpcode::G_BITCAST:
680 // This is going to be a cross register bank copy and this is expensive.
681 if (OpRegBankIdx[0] != OpRegBankIdx[1])
682 Cost = copyCost(
683 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[0]].RegBank,
684 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[1]].RegBank,
685 OpSize[0]);
686 break;
687 case TargetOpcode::G_LOAD:
688 // Loading in vector unit is slightly more expensive.
689 // This is actually only true for the LD1R and co instructions,
690 // but anyway for the fast mode this number does not matter and
691 // for the greedy mode the cost of the cross bank copy will
692 // offset this number.
693 // FIXME: Should be derived from the scheduling model.
694 if (OpRegBankIdx[0] != PMI_FirstGPR)
695 Cost = 2;
696 else
697 // Check if that load feeds fp instructions.
698 // In that case, we want the default mapping to be on FPR
699 // instead of blind map every scalar to GPR.
700 for (const MachineInstr &UseMI :
701 MRI.use_nodbg_instructions(MI.getOperand(0).getReg())) {
702 // If we have at least one direct use in a FP instruction,
703 // assume this was a floating point load in the IR.
704 // If it was not, we would have had a bitcast before
705 // reaching that instruction.
706 if (onlyUsesFP(UseMI, MRI, TRI)) {
707 OpRegBankIdx[0] = PMI_FirstFPR;
708 break;
709 }
710 }
711 break;
712 case TargetOpcode::G_STORE:
713 // Check if that store is fed by fp instructions.
714 if (OpRegBankIdx[0] == PMI_FirstGPR) {
715 Register VReg = MI.getOperand(0).getReg();
716 if (!VReg)
717 break;
718 MachineInstr *DefMI = MRI.getVRegDef(VReg);
719 if (onlyDefinesFP(*DefMI, MRI, TRI))
720 OpRegBankIdx[0] = PMI_FirstFPR;
721 break;
722 }
723 break;
724 case TargetOpcode::G_SELECT: {
725 // If the destination is FPR, preserve that.
726 if (OpRegBankIdx[0] != PMI_FirstGPR)
727 break;
728
729 // If we're taking in vectors, we have no choice but to put everything on
730 // FPRs, except for the condition. The condition must always be on a GPR.
731 LLT SrcTy = MRI.getType(MI.getOperand(2).getReg());
732 if (SrcTy.isVector()) {
733 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
734 break;
735 }
736
737 // Try to minimize the number of copies. If we have more floating point
738 // constrained values than not, then we'll put everything on FPR. Otherwise,
739 // everything has to be on GPR.
740 unsigned NumFP = 0;
741
742 // Check if the uses of the result always produce floating point values.
743 //
744 // For example:
745 //
746 // %z = G_SELECT %cond %x %y
747 // fpr = G_FOO %z ...
748 if (any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
749 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); }))
750 ++NumFP;
751
752 // Check if the defs of the source values always produce floating point
753 // values.
754 //
755 // For example:
756 //
757 // %x = G_SOMETHING_ALWAYS_FLOAT %a ...
758 // %z = G_SELECT %cond %x %y
759 //
760 // Also check whether or not the sources have already been decided to be
761 // FPR. Keep track of this.
762 //
763 // This doesn't check the condition, since it's just whatever is in NZCV.
764 // This isn't passed explicitly in a register to fcsel/csel.
765 for (unsigned Idx = 2; Idx < 4; ++Idx) {
766 Register VReg = MI.getOperand(Idx).getReg();
767 MachineInstr *DefMI = MRI.getVRegDef(VReg);
768 if (getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank ||
769 onlyDefinesFP(*DefMI, MRI, TRI))
770 ++NumFP;
771 }
772
773 // If we have more FP constraints than not, then move everything over to
774 // FPR.
775 if (NumFP >= 2)
776 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
777
778 break;
779 }
780 case TargetOpcode::G_UNMERGE_VALUES: {
781 // If the first operand belongs to a FPR register bank, then make sure that
782 // we preserve that.
783 if (OpRegBankIdx[0] != PMI_FirstGPR)
784 break;
785
786 LLT SrcTy = MRI.getType(MI.getOperand(MI.getNumOperands()-1).getReg());
787 // UNMERGE into scalars from a vector should always use FPR.
788 // Likewise if any of the uses are FP instructions.
789 if (SrcTy.isVector() || SrcTy == LLT::scalar(128) ||
790 any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
791 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) {
792 // Set the register bank of every operand to FPR.
793 for (unsigned Idx = 0, NumOperands = MI.getNumOperands();
794 Idx < NumOperands; ++Idx)
795 OpRegBankIdx[Idx] = PMI_FirstFPR;
796 }
797 break;
798 }
799 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
800 // Destination and source need to be FPRs.
801 OpRegBankIdx[0] = PMI_FirstFPR;
802 OpRegBankIdx[1] = PMI_FirstFPR;
803
804 // Index needs to be a GPR.
805 OpRegBankIdx[2] = PMI_FirstGPR;
806 break;
807 case TargetOpcode::G_INSERT_VECTOR_ELT:
808 OpRegBankIdx[0] = PMI_FirstFPR;
809 OpRegBankIdx[1] = PMI_FirstFPR;
810
811 // The element may be either a GPR or FPR. Preserve that behaviour.
812 if (getRegBank(MI.getOperand(2).getReg(), MRI, TRI) == &AArch64::FPRRegBank)
813 OpRegBankIdx[2] = PMI_FirstFPR;
814 else
815 OpRegBankIdx[2] = PMI_FirstGPR;
816
817 // Index needs to be a GPR.
818 OpRegBankIdx[3] = PMI_FirstGPR;
819 break;
820 case TargetOpcode::G_EXTRACT: {
821 // For s128 sources we have to use fpr.
822 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
823 if (SrcTy.getSizeInBits() == 128) {
824 OpRegBankIdx[0] = PMI_FirstFPR;
825 OpRegBankIdx[1] = PMI_FirstFPR;
826 }
827 break;
828 }
829 case TargetOpcode::G_BUILD_VECTOR:
830 // If the first source operand belongs to a FPR register bank, then make
831 // sure that we preserve that.
832 if (OpRegBankIdx[1] != PMI_FirstGPR)
833 break;
834 Register VReg = MI.getOperand(1).getReg();
835 if (!VReg)
836 break;
837
838 // Get the instruction that defined the source operand reg, and check if
839 // it's a floating point operation. Or, if it's a type like s16 which
840 // doesn't have a exact size gpr register class.
841 MachineInstr *DefMI = MRI.getVRegDef(VReg);
842 unsigned DefOpc = DefMI->getOpcode();
843 const LLT SrcTy = MRI.getType(VReg);
844 if (isPreISelGenericFloatingPointOpcode(DefOpc) ||
845 SrcTy.getSizeInBits() < 32) {
846 // Have a floating point op.
847 // Make sure every operand gets mapped to a FPR register class.
848 unsigned NumOperands = MI.getNumOperands();
849 for (unsigned Idx = 0; Idx < NumOperands; ++Idx)
850 OpRegBankIdx[Idx] = PMI_FirstFPR;
851 }
852 break;
853 }
854
855 // Finally construct the computed mapping.
856 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOperands);
857 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
858 if (MI.getOperand(Idx).isReg() && MI.getOperand(Idx).getReg()) {
859 auto Mapping = getValueMapping(OpRegBankIdx[Idx], OpSize[Idx]);
860 if (!Mapping->isValid())
861 return getInvalidInstructionMapping();
862
863 OpdsMapping[Idx] = Mapping;
864 }
865 }
866
867 return getInstructionMapping(DefaultMappingID, Cost,
868 getOperandsMapping(OpdsMapping), NumOperands);
869 }
870