1 //===- AArch64RegisterBankInfo.cpp ----------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// This file implements the targeting of the RegisterBankInfo class for 10 /// AArch64. 11 /// \todo This should be generated by TableGen. 12 //===----------------------------------------------------------------------===// 13 14 #include "AArch64RegisterBankInfo.h" 15 #include "AArch64RegisterInfo.h" 16 #include "MCTargetDesc/AArch64MCTargetDesc.h" 17 #include "llvm/ADT/STLExtras.h" 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 20 #include "llvm/CodeGen/GlobalISel/Utils.h" 21 #include "llvm/CodeGen/LowLevelType.h" 22 #include "llvm/CodeGen/MachineFunction.h" 23 #include "llvm/CodeGen/MachineInstr.h" 24 #include "llvm/CodeGen/MachineOperand.h" 25 #include "llvm/CodeGen/MachineRegisterInfo.h" 26 #include "llvm/CodeGen/RegisterBank.h" 27 #include "llvm/CodeGen/RegisterBankInfo.h" 28 #include "llvm/CodeGen/TargetOpcodes.h" 29 #include "llvm/CodeGen/TargetRegisterInfo.h" 30 #include "llvm/CodeGen/TargetSubtargetInfo.h" 31 #include "llvm/IR/IntrinsicsAArch64.h" 32 #include "llvm/Support/ErrorHandling.h" 33 #include <algorithm> 34 #include <cassert> 35 36 #define GET_TARGET_REGBANK_IMPL 37 #include "AArch64GenRegisterBank.inc" 38 39 // This file will be TableGen'ed at some point. 40 #include "AArch64GenRegisterBankInfo.def" 41 42 using namespace llvm; 43 44 AArch64RegisterBankInfo::AArch64RegisterBankInfo( 45 const TargetRegisterInfo &TRI) { 46 static llvm::once_flag InitializeRegisterBankFlag; 47 48 static auto InitializeRegisterBankOnce = [&]() { 49 // We have only one set of register banks, whatever the subtarget 50 // is. Therefore, the initialization of the RegBanks table should be 51 // done only once. Indeed the table of all register banks 52 // (AArch64::RegBanks) is unique in the compiler. At some point, it 53 // will get tablegen'ed and the whole constructor becomes empty. 54 55 const RegisterBank &RBGPR = getRegBank(AArch64::GPRRegBankID); 56 (void)RBGPR; 57 assert(&AArch64::GPRRegBank == &RBGPR && 58 "The order in RegBanks is messed up"); 59 60 const RegisterBank &RBFPR = getRegBank(AArch64::FPRRegBankID); 61 (void)RBFPR; 62 assert(&AArch64::FPRRegBank == &RBFPR && 63 "The order in RegBanks is messed up"); 64 65 const RegisterBank &RBCCR = getRegBank(AArch64::CCRegBankID); 66 (void)RBCCR; 67 assert(&AArch64::CCRegBank == &RBCCR && 68 "The order in RegBanks is messed up"); 69 70 // The GPR register bank is fully defined by all the registers in 71 // GR64all + its subclasses. 72 assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) && 73 "Subclass not added?"); 74 assert(RBGPR.getSize() == 128 && "GPRs should hold up to 128-bit"); 75 76 // The FPR register bank is fully defined by all the registers in 77 // GR64all + its subclasses. 78 assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) && 79 "Subclass not added?"); 80 assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) && 81 "Subclass not added?"); 82 assert(RBFPR.getSize() == 512 && 83 "FPRs should hold up to 512-bit via QQQQ sequence"); 84 85 assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) && 86 "Class not added?"); 87 assert(RBCCR.getSize() == 32 && "CCR should hold up to 32-bit"); 88 89 // Check that the TableGen'ed like file is in sync we our expectations. 90 // First, the Idx. 91 assert(checkPartialMappingIdx(PMI_FirstGPR, PMI_LastGPR, 92 {PMI_GPR32, PMI_GPR64, PMI_GPR128}) && 93 "PartialMappingIdx's are incorrectly ordered"); 94 assert(checkPartialMappingIdx(PMI_FirstFPR, PMI_LastFPR, 95 {PMI_FPR16, PMI_FPR32, PMI_FPR64, PMI_FPR128, 96 PMI_FPR256, PMI_FPR512}) && 97 "PartialMappingIdx's are incorrectly ordered"); 98 // Now, the content. 99 // Check partial mapping. 100 #define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB) \ 101 do { \ 102 assert( \ 103 checkPartialMap(PartialMappingIdx::Idx, ValStartIdx, ValLength, RB) && \ 104 #Idx " is incorrectly initialized"); \ 105 } while (false) 106 107 CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR); 108 CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR); 109 CHECK_PARTIALMAP(PMI_GPR128, 0, 128, RBGPR); 110 CHECK_PARTIALMAP(PMI_FPR16, 0, 16, RBFPR); 111 CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR); 112 CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR); 113 CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR); 114 CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR); 115 CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR); 116 117 // Check value mapping. 118 #define CHECK_VALUEMAP_IMPL(RBName, Size, Offset) \ 119 do { \ 120 assert(checkValueMapImpl(PartialMappingIdx::PMI_##RBName##Size, \ 121 PartialMappingIdx::PMI_First##RBName, Size, \ 122 Offset) && \ 123 #RBName #Size " " #Offset " is incorrectly initialized"); \ 124 } while (false) 125 126 #define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0) 127 128 CHECK_VALUEMAP(GPR, 32); 129 CHECK_VALUEMAP(GPR, 64); 130 CHECK_VALUEMAP(GPR, 128); 131 CHECK_VALUEMAP(FPR, 16); 132 CHECK_VALUEMAP(FPR, 32); 133 CHECK_VALUEMAP(FPR, 64); 134 CHECK_VALUEMAP(FPR, 128); 135 CHECK_VALUEMAP(FPR, 256); 136 CHECK_VALUEMAP(FPR, 512); 137 138 // Check the value mapping for 3-operands instructions where all the operands 139 // map to the same value mapping. 140 #define CHECK_VALUEMAP_3OPS(RBName, Size) \ 141 do { \ 142 CHECK_VALUEMAP_IMPL(RBName, Size, 0); \ 143 CHECK_VALUEMAP_IMPL(RBName, Size, 1); \ 144 CHECK_VALUEMAP_IMPL(RBName, Size, 2); \ 145 } while (false) 146 147 CHECK_VALUEMAP_3OPS(GPR, 32); 148 CHECK_VALUEMAP_3OPS(GPR, 64); 149 CHECK_VALUEMAP_3OPS(GPR, 128); 150 CHECK_VALUEMAP_3OPS(FPR, 32); 151 CHECK_VALUEMAP_3OPS(FPR, 64); 152 CHECK_VALUEMAP_3OPS(FPR, 128); 153 CHECK_VALUEMAP_3OPS(FPR, 256); 154 CHECK_VALUEMAP_3OPS(FPR, 512); 155 156 #define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size) \ 157 do { \ 158 unsigned PartialMapDstIdx = PMI_##RBNameDst##Size - PMI_Min; \ 159 unsigned PartialMapSrcIdx = PMI_##RBNameSrc##Size - PMI_Min; \ 160 (void)PartialMapDstIdx; \ 161 (void)PartialMapSrcIdx; \ 162 const ValueMapping *Map = getCopyMapping( \ 163 AArch64::RBNameDst##RegBankID, AArch64::RBNameSrc##RegBankID, Size); \ 164 (void)Map; \ 165 assert(Map[0].BreakDown == \ 166 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \ 167 Map[0].NumBreakDowns == 1 && #RBNameDst #Size \ 168 " Dst is incorrectly initialized"); \ 169 assert(Map[1].BreakDown == \ 170 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \ 171 Map[1].NumBreakDowns == 1 && #RBNameSrc #Size \ 172 " Src is incorrectly initialized"); \ 173 \ 174 } while (false) 175 176 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32); 177 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 32); 178 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64); 179 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 64); 180 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 32); 181 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 32); 182 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 64); 183 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 64); 184 185 #define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize) \ 186 do { \ 187 unsigned PartialMapDstIdx = PMI_FPR##DstSize - PMI_Min; \ 188 unsigned PartialMapSrcIdx = PMI_FPR##SrcSize - PMI_Min; \ 189 (void)PartialMapDstIdx; \ 190 (void)PartialMapSrcIdx; \ 191 const ValueMapping *Map = getFPExtMapping(DstSize, SrcSize); \ 192 (void)Map; \ 193 assert(Map[0].BreakDown == \ 194 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \ 195 Map[0].NumBreakDowns == 1 && "FPR" #DstSize \ 196 " Dst is incorrectly initialized"); \ 197 assert(Map[1].BreakDown == \ 198 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \ 199 Map[1].NumBreakDowns == 1 && "FPR" #SrcSize \ 200 " Src is incorrectly initialized"); \ 201 \ 202 } while (false) 203 204 CHECK_VALUEMAP_FPEXT(32, 16); 205 CHECK_VALUEMAP_FPEXT(64, 16); 206 CHECK_VALUEMAP_FPEXT(64, 32); 207 CHECK_VALUEMAP_FPEXT(128, 64); 208 209 assert(verify(TRI) && "Invalid register bank information"); 210 }; 211 212 llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce); 213 } 214 215 unsigned AArch64RegisterBankInfo::copyCost(const RegisterBank &A, 216 const RegisterBank &B, 217 unsigned Size) const { 218 // What do we do with different size? 219 // copy are same size. 220 // Will introduce other hooks for different size: 221 // * extract cost. 222 // * build_sequence cost. 223 224 // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV. 225 // FIXME: This should be deduced from the scheduling model. 226 if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank) 227 // FMOVXDr or FMOVWSr. 228 return 5; 229 if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank) 230 // FMOVDXr or FMOVSWr. 231 return 4; 232 233 return RegisterBankInfo::copyCost(A, B, Size); 234 } 235 236 const RegisterBank & 237 AArch64RegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC, 238 LLT) const { 239 switch (RC.getID()) { 240 case AArch64::FPR8RegClassID: 241 case AArch64::FPR16RegClassID: 242 case AArch64::FPR16_loRegClassID: 243 case AArch64::FPR32_with_hsub_in_FPR16_loRegClassID: 244 case AArch64::FPR32RegClassID: 245 case AArch64::FPR64RegClassID: 246 case AArch64::FPR64_loRegClassID: 247 case AArch64::FPR128RegClassID: 248 case AArch64::FPR128_loRegClassID: 249 case AArch64::DDRegClassID: 250 case AArch64::DDDRegClassID: 251 case AArch64::DDDDRegClassID: 252 case AArch64::QQRegClassID: 253 case AArch64::QQQRegClassID: 254 case AArch64::QQQQRegClassID: 255 return getRegBank(AArch64::FPRRegBankID); 256 case AArch64::GPR32commonRegClassID: 257 case AArch64::GPR32RegClassID: 258 case AArch64::GPR32spRegClassID: 259 case AArch64::GPR32sponlyRegClassID: 260 case AArch64::GPR32argRegClassID: 261 case AArch64::GPR32allRegClassID: 262 case AArch64::GPR64commonRegClassID: 263 case AArch64::GPR64RegClassID: 264 case AArch64::GPR64spRegClassID: 265 case AArch64::GPR64sponlyRegClassID: 266 case AArch64::GPR64argRegClassID: 267 case AArch64::GPR64allRegClassID: 268 case AArch64::GPR64noipRegClassID: 269 case AArch64::GPR64common_and_GPR64noipRegClassID: 270 case AArch64::GPR64noip_and_tcGPR64RegClassID: 271 case AArch64::tcGPR64RegClassID: 272 case AArch64::rtcGPR64RegClassID: 273 case AArch64::WSeqPairsClassRegClassID: 274 case AArch64::XSeqPairsClassRegClassID: 275 case AArch64::MatrixIndexGPR32_12_15RegClassID: 276 case AArch64::GPR64_with_sub_32_in_MatrixIndexGPR32_12_15RegClassID: 277 return getRegBank(AArch64::GPRRegBankID); 278 case AArch64::CCRRegClassID: 279 return getRegBank(AArch64::CCRegBankID); 280 default: 281 llvm_unreachable("Register class not supported"); 282 } 283 } 284 285 RegisterBankInfo::InstructionMappings 286 AArch64RegisterBankInfo::getInstrAlternativeMappings( 287 const MachineInstr &MI) const { 288 const MachineFunction &MF = *MI.getParent()->getParent(); 289 const TargetSubtargetInfo &STI = MF.getSubtarget(); 290 const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); 291 const MachineRegisterInfo &MRI = MF.getRegInfo(); 292 293 switch (MI.getOpcode()) { 294 case TargetOpcode::G_OR: { 295 // 32 and 64-bit or can be mapped on either FPR or 296 // GPR for the same cost. 297 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI); 298 if (Size != 32 && Size != 64) 299 break; 300 301 // If the instruction has any implicit-defs or uses, 302 // do not mess with it. 303 if (MI.getNumOperands() != 3) 304 break; 305 InstructionMappings AltMappings; 306 const InstructionMapping &GPRMapping = getInstructionMapping( 307 /*ID*/ 1, /*Cost*/ 1, getValueMapping(PMI_FirstGPR, Size), 308 /*NumOperands*/ 3); 309 const InstructionMapping &FPRMapping = getInstructionMapping( 310 /*ID*/ 2, /*Cost*/ 1, getValueMapping(PMI_FirstFPR, Size), 311 /*NumOperands*/ 3); 312 313 AltMappings.push_back(&GPRMapping); 314 AltMappings.push_back(&FPRMapping); 315 return AltMappings; 316 } 317 case TargetOpcode::G_BITCAST: { 318 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI); 319 if (Size != 32 && Size != 64) 320 break; 321 322 // If the instruction has any implicit-defs or uses, 323 // do not mess with it. 324 if (MI.getNumOperands() != 2) 325 break; 326 327 InstructionMappings AltMappings; 328 const InstructionMapping &GPRMapping = getInstructionMapping( 329 /*ID*/ 1, /*Cost*/ 1, 330 getCopyMapping(AArch64::GPRRegBankID, AArch64::GPRRegBankID, Size), 331 /*NumOperands*/ 2); 332 const InstructionMapping &FPRMapping = getInstructionMapping( 333 /*ID*/ 2, /*Cost*/ 1, 334 getCopyMapping(AArch64::FPRRegBankID, AArch64::FPRRegBankID, Size), 335 /*NumOperands*/ 2); 336 const InstructionMapping &GPRToFPRMapping = getInstructionMapping( 337 /*ID*/ 3, 338 /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size), 339 getCopyMapping(AArch64::FPRRegBankID, AArch64::GPRRegBankID, Size), 340 /*NumOperands*/ 2); 341 const InstructionMapping &FPRToGPRMapping = getInstructionMapping( 342 /*ID*/ 3, 343 /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size), 344 getCopyMapping(AArch64::GPRRegBankID, AArch64::FPRRegBankID, Size), 345 /*NumOperands*/ 2); 346 347 AltMappings.push_back(&GPRMapping); 348 AltMappings.push_back(&FPRMapping); 349 AltMappings.push_back(&GPRToFPRMapping); 350 AltMappings.push_back(&FPRToGPRMapping); 351 return AltMappings; 352 } 353 case TargetOpcode::G_LOAD: { 354 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI); 355 if (Size != 64) 356 break; 357 358 // If the instruction has any implicit-defs or uses, 359 // do not mess with it. 360 if (MI.getNumOperands() != 2) 361 break; 362 363 InstructionMappings AltMappings; 364 const InstructionMapping &GPRMapping = getInstructionMapping( 365 /*ID*/ 1, /*Cost*/ 1, 366 getOperandsMapping({getValueMapping(PMI_FirstGPR, Size), 367 // Addresses are GPR 64-bit. 368 getValueMapping(PMI_FirstGPR, 64)}), 369 /*NumOperands*/ 2); 370 const InstructionMapping &FPRMapping = getInstructionMapping( 371 /*ID*/ 2, /*Cost*/ 1, 372 getOperandsMapping({getValueMapping(PMI_FirstFPR, Size), 373 // Addresses are GPR 64-bit. 374 getValueMapping(PMI_FirstGPR, 64)}), 375 /*NumOperands*/ 2); 376 377 AltMappings.push_back(&GPRMapping); 378 AltMappings.push_back(&FPRMapping); 379 return AltMappings; 380 } 381 default: 382 break; 383 } 384 return RegisterBankInfo::getInstrAlternativeMappings(MI); 385 } 386 387 void AArch64RegisterBankInfo::applyMappingImpl( 388 const OperandsMapper &OpdMapper) const { 389 switch (OpdMapper.getMI().getOpcode()) { 390 case TargetOpcode::G_OR: 391 case TargetOpcode::G_BITCAST: 392 case TargetOpcode::G_LOAD: 393 // Those ID must match getInstrAlternativeMappings. 394 assert((OpdMapper.getInstrMapping().getID() >= 1 && 395 OpdMapper.getInstrMapping().getID() <= 4) && 396 "Don't know how to handle that ID"); 397 return applyDefaultMapping(OpdMapper); 398 default: 399 llvm_unreachable("Don't know how to handle that operation"); 400 } 401 } 402 403 /// Returns whether opcode \p Opc is a pre-isel generic floating-point opcode, 404 /// having only floating-point operands. 405 static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) { 406 switch (Opc) { 407 case TargetOpcode::G_FADD: 408 case TargetOpcode::G_FSUB: 409 case TargetOpcode::G_FMUL: 410 case TargetOpcode::G_FMA: 411 case TargetOpcode::G_FDIV: 412 case TargetOpcode::G_FCONSTANT: 413 case TargetOpcode::G_FPEXT: 414 case TargetOpcode::G_FPTRUNC: 415 case TargetOpcode::G_FCEIL: 416 case TargetOpcode::G_FFLOOR: 417 case TargetOpcode::G_FNEARBYINT: 418 case TargetOpcode::G_FNEG: 419 case TargetOpcode::G_FCOS: 420 case TargetOpcode::G_FSIN: 421 case TargetOpcode::G_FLOG10: 422 case TargetOpcode::G_FLOG: 423 case TargetOpcode::G_FLOG2: 424 case TargetOpcode::G_FSQRT: 425 case TargetOpcode::G_FABS: 426 case TargetOpcode::G_FEXP: 427 case TargetOpcode::G_FRINT: 428 case TargetOpcode::G_INTRINSIC_TRUNC: 429 case TargetOpcode::G_INTRINSIC_ROUND: 430 case TargetOpcode::G_FMAXNUM: 431 case TargetOpcode::G_FMINNUM: 432 case TargetOpcode::G_FMAXIMUM: 433 case TargetOpcode::G_FMINIMUM: 434 return true; 435 } 436 return false; 437 } 438 439 const RegisterBankInfo::InstructionMapping & 440 AArch64RegisterBankInfo::getSameKindOfOperandsMapping( 441 const MachineInstr &MI) const { 442 const unsigned Opc = MI.getOpcode(); 443 const MachineFunction &MF = *MI.getParent()->getParent(); 444 const MachineRegisterInfo &MRI = MF.getRegInfo(); 445 446 unsigned NumOperands = MI.getNumOperands(); 447 assert(NumOperands <= 3 && 448 "This code is for instructions with 3 or less operands"); 449 450 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 451 unsigned Size = Ty.getSizeInBits(); 452 bool IsFPR = Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc); 453 454 PartialMappingIdx RBIdx = IsFPR ? PMI_FirstFPR : PMI_FirstGPR; 455 456 #ifndef NDEBUG 457 // Make sure all the operands are using similar size and type. 458 // Should probably be checked by the machine verifier. 459 // This code won't catch cases where the number of lanes is 460 // different between the operands. 461 // If we want to go to that level of details, it is probably 462 // best to check that the types are the same, period. 463 // Currently, we just check that the register banks are the same 464 // for each types. 465 for (unsigned Idx = 1; Idx != NumOperands; ++Idx) { 466 LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg()); 467 assert( 468 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset( 469 RBIdx, OpTy.getSizeInBits()) == 470 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(RBIdx, Size) && 471 "Operand has incompatible size"); 472 bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc); 473 (void)OpIsFPR; 474 assert(IsFPR == OpIsFPR && "Operand has incompatible type"); 475 } 476 #endif // End NDEBUG. 477 478 return getInstructionMapping(DefaultMappingID, 1, 479 getValueMapping(RBIdx, Size), NumOperands); 480 } 481 482 /// \returns true if a given intrinsic \p ID only uses and defines FPRs. 483 static bool isFPIntrinsic(unsigned ID) { 484 // TODO: Add more intrinsics. 485 switch (ID) { 486 default: 487 return false; 488 case Intrinsic::aarch64_neon_uaddlv: 489 return true; 490 } 491 } 492 493 bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI, 494 const MachineRegisterInfo &MRI, 495 const TargetRegisterInfo &TRI, 496 unsigned Depth) const { 497 unsigned Op = MI.getOpcode(); 498 if (Op == TargetOpcode::G_INTRINSIC && isFPIntrinsic(MI.getIntrinsicID())) 499 return true; 500 501 // Do we have an explicit floating point instruction? 502 if (isPreISelGenericFloatingPointOpcode(Op)) 503 return true; 504 505 // No. Check if we have a copy-like instruction. If we do, then we could 506 // still be fed by floating point instructions. 507 if (Op != TargetOpcode::COPY && !MI.isPHI() && 508 !isPreISelGenericOptimizationHint(Op)) 509 return false; 510 511 // Check if we already know the register bank. 512 auto *RB = getRegBank(MI.getOperand(0).getReg(), MRI, TRI); 513 if (RB == &AArch64::FPRRegBank) 514 return true; 515 if (RB == &AArch64::GPRRegBank) 516 return false; 517 518 // We don't know anything. 519 // 520 // If we have a phi, we may be able to infer that it will be assigned a FPR 521 // based off of its inputs. 522 if (!MI.isPHI() || Depth > MaxFPRSearchDepth) 523 return false; 524 525 return any_of(MI.explicit_uses(), [&](const MachineOperand &Op) { 526 return Op.isReg() && 527 onlyDefinesFP(*MRI.getVRegDef(Op.getReg()), MRI, TRI, Depth + 1); 528 }); 529 } 530 531 bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI, 532 const MachineRegisterInfo &MRI, 533 const TargetRegisterInfo &TRI, 534 unsigned Depth) const { 535 switch (MI.getOpcode()) { 536 case TargetOpcode::G_FPTOSI: 537 case TargetOpcode::G_FPTOUI: 538 case TargetOpcode::G_FCMP: 539 case TargetOpcode::G_LROUND: 540 case TargetOpcode::G_LLROUND: 541 return true; 542 default: 543 break; 544 } 545 return hasFPConstraints(MI, MRI, TRI, Depth); 546 } 547 548 bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI, 549 const MachineRegisterInfo &MRI, 550 const TargetRegisterInfo &TRI, 551 unsigned Depth) const { 552 switch (MI.getOpcode()) { 553 case AArch64::G_DUP: 554 case TargetOpcode::G_SITOFP: 555 case TargetOpcode::G_UITOFP: 556 case TargetOpcode::G_EXTRACT_VECTOR_ELT: 557 case TargetOpcode::G_INSERT_VECTOR_ELT: 558 case TargetOpcode::G_BUILD_VECTOR: 559 case TargetOpcode::G_BUILD_VECTOR_TRUNC: 560 return true; 561 default: 562 break; 563 } 564 return hasFPConstraints(MI, MRI, TRI, Depth); 565 } 566 567 const RegisterBankInfo::InstructionMapping & 568 AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { 569 const unsigned Opc = MI.getOpcode(); 570 571 // Try the default logic for non-generic instructions that are either copies 572 // or already have some operands assigned to banks. 573 if ((Opc != TargetOpcode::COPY && !isPreISelGenericOpcode(Opc)) || 574 Opc == TargetOpcode::G_PHI) { 575 const RegisterBankInfo::InstructionMapping &Mapping = 576 getInstrMappingImpl(MI); 577 if (Mapping.isValid()) 578 return Mapping; 579 } 580 581 const MachineFunction &MF = *MI.getParent()->getParent(); 582 const MachineRegisterInfo &MRI = MF.getRegInfo(); 583 const TargetSubtargetInfo &STI = MF.getSubtarget(); 584 const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); 585 586 switch (Opc) { 587 // G_{F|S|U}REM are not listed because they are not legal. 588 // Arithmetic ops. 589 case TargetOpcode::G_ADD: 590 case TargetOpcode::G_SUB: 591 case TargetOpcode::G_PTR_ADD: 592 case TargetOpcode::G_MUL: 593 case TargetOpcode::G_SDIV: 594 case TargetOpcode::G_UDIV: 595 // Bitwise ops. 596 case TargetOpcode::G_AND: 597 case TargetOpcode::G_OR: 598 case TargetOpcode::G_XOR: 599 // Floating point ops. 600 case TargetOpcode::G_FADD: 601 case TargetOpcode::G_FSUB: 602 case TargetOpcode::G_FMUL: 603 case TargetOpcode::G_FDIV: 604 case TargetOpcode::G_FMAXIMUM: 605 case TargetOpcode::G_FMINIMUM: 606 return getSameKindOfOperandsMapping(MI); 607 case TargetOpcode::G_FPEXT: { 608 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); 609 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); 610 return getInstructionMapping( 611 DefaultMappingID, /*Cost*/ 1, 612 getFPExtMapping(DstTy.getSizeInBits(), SrcTy.getSizeInBits()), 613 /*NumOperands*/ 2); 614 } 615 // Shifts. 616 case TargetOpcode::G_SHL: 617 case TargetOpcode::G_LSHR: 618 case TargetOpcode::G_ASHR: { 619 LLT ShiftAmtTy = MRI.getType(MI.getOperand(2).getReg()); 620 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); 621 if (ShiftAmtTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() == 32) 622 return getInstructionMapping(DefaultMappingID, 1, 623 &ValMappings[Shift64Imm], 3); 624 return getSameKindOfOperandsMapping(MI); 625 } 626 case TargetOpcode::COPY: { 627 Register DstReg = MI.getOperand(0).getReg(); 628 Register SrcReg = MI.getOperand(1).getReg(); 629 // Check if one of the register is not a generic register. 630 if ((Register::isPhysicalRegister(DstReg) || 631 !MRI.getType(DstReg).isValid()) || 632 (Register::isPhysicalRegister(SrcReg) || 633 !MRI.getType(SrcReg).isValid())) { 634 const RegisterBank *DstRB = getRegBank(DstReg, MRI, TRI); 635 const RegisterBank *SrcRB = getRegBank(SrcReg, MRI, TRI); 636 if (!DstRB) 637 DstRB = SrcRB; 638 else if (!SrcRB) 639 SrcRB = DstRB; 640 // If both RB are null that means both registers are generic. 641 // We shouldn't be here. 642 assert(DstRB && SrcRB && "Both RegBank were nullptr"); 643 unsigned Size = getSizeInBits(DstReg, MRI, TRI); 644 return getInstructionMapping( 645 DefaultMappingID, copyCost(*DstRB, *SrcRB, Size), 646 getCopyMapping(DstRB->getID(), SrcRB->getID(), Size), 647 // We only care about the mapping of the destination. 648 /*NumOperands*/ 1); 649 } 650 // Both registers are generic, use G_BITCAST. 651 LLVM_FALLTHROUGH; 652 } 653 case TargetOpcode::G_BITCAST: { 654 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); 655 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); 656 unsigned Size = DstTy.getSizeInBits(); 657 bool DstIsGPR = !DstTy.isVector() && DstTy.getSizeInBits() <= 64; 658 bool SrcIsGPR = !SrcTy.isVector() && SrcTy.getSizeInBits() <= 64; 659 const RegisterBank &DstRB = 660 DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank; 661 const RegisterBank &SrcRB = 662 SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank; 663 return getInstructionMapping( 664 DefaultMappingID, copyCost(DstRB, SrcRB, Size), 665 getCopyMapping(DstRB.getID(), SrcRB.getID(), Size), 666 // We only care about the mapping of the destination for COPY. 667 /*NumOperands*/ Opc == TargetOpcode::G_BITCAST ? 2 : 1); 668 } 669 default: 670 break; 671 } 672 673 unsigned NumOperands = MI.getNumOperands(); 674 675 // Track the size and bank of each register. We don't do partial mappings. 676 SmallVector<unsigned, 4> OpSize(NumOperands); 677 SmallVector<PartialMappingIdx, 4> OpRegBankIdx(NumOperands); 678 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) { 679 auto &MO = MI.getOperand(Idx); 680 if (!MO.isReg() || !MO.getReg()) 681 continue; 682 683 LLT Ty = MRI.getType(MO.getReg()); 684 OpSize[Idx] = Ty.getSizeInBits(); 685 686 // As a top-level guess, vectors go in FPRs, scalars and pointers in GPRs. 687 // For floating-point instructions, scalars go in FPRs. 688 if (Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc) || 689 Ty.getSizeInBits() > 64) 690 OpRegBankIdx[Idx] = PMI_FirstFPR; 691 else 692 OpRegBankIdx[Idx] = PMI_FirstGPR; 693 } 694 695 unsigned Cost = 1; 696 // Some of the floating-point instructions have mixed GPR and FPR operands: 697 // fine-tune the computed mapping. 698 switch (Opc) { 699 case AArch64::G_DUP: { 700 Register ScalarReg = MI.getOperand(1).getReg(); 701 LLT ScalarTy = MRI.getType(ScalarReg); 702 auto ScalarDef = MRI.getVRegDef(ScalarReg); 703 // s8 is an exception for G_DUP, which we always want on gpr. 704 if (ScalarTy.getSizeInBits() != 8 && 705 (getRegBank(ScalarReg, MRI, TRI) == &AArch64::FPRRegBank || 706 onlyDefinesFP(*ScalarDef, MRI, TRI))) 707 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR}; 708 else 709 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR}; 710 break; 711 } 712 case TargetOpcode::G_TRUNC: { 713 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); 714 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) 715 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR}; 716 break; 717 } 718 case TargetOpcode::G_SITOFP: 719 case TargetOpcode::G_UITOFP: { 720 if (MRI.getType(MI.getOperand(0).getReg()).isVector()) 721 break; 722 // Integer to FP conversions don't necessarily happen between GPR -> FPR 723 // regbanks. They can also be done within an FPR register. 724 Register SrcReg = MI.getOperand(1).getReg(); 725 if (getRegBank(SrcReg, MRI, TRI) == &AArch64::FPRRegBank) 726 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR}; 727 else 728 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR}; 729 break; 730 } 731 case TargetOpcode::G_FPTOSI: 732 case TargetOpcode::G_FPTOUI: 733 if (MRI.getType(MI.getOperand(0).getReg()).isVector()) 734 break; 735 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR}; 736 break; 737 case TargetOpcode::G_FCMP: { 738 // If the result is a vector, it must use a FPR. 739 AArch64GenRegisterBankInfo::PartialMappingIdx Idx0 = 740 MRI.getType(MI.getOperand(0).getReg()).isVector() ? PMI_FirstFPR 741 : PMI_FirstGPR; 742 OpRegBankIdx = {Idx0, 743 /* Predicate */ PMI_None, PMI_FirstFPR, PMI_FirstFPR}; 744 break; 745 } 746 case TargetOpcode::G_BITCAST: 747 // This is going to be a cross register bank copy and this is expensive. 748 if (OpRegBankIdx[0] != OpRegBankIdx[1]) 749 Cost = copyCost( 750 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[0]].RegBank, 751 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[1]].RegBank, 752 OpSize[0]); 753 break; 754 case TargetOpcode::G_LOAD: 755 // Loading in vector unit is slightly more expensive. 756 // This is actually only true for the LD1R and co instructions, 757 // but anyway for the fast mode this number does not matter and 758 // for the greedy mode the cost of the cross bank copy will 759 // offset this number. 760 // FIXME: Should be derived from the scheduling model. 761 if (OpRegBankIdx[0] != PMI_FirstGPR) { 762 Cost = 2; 763 break; 764 } 765 766 if (cast<GLoad>(MI).isAtomic()) { 767 // Atomics always use GPR destinations. Don't refine any further. 768 OpRegBankIdx[0] = PMI_FirstGPR; 769 break; 770 } 771 772 // Check if that load feeds fp instructions. 773 // In that case, we want the default mapping to be on FPR 774 // instead of blind map every scalar to GPR. 775 if (any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()), 776 [&](const MachineInstr &UseMI) { 777 // If we have at least one direct use in a FP instruction, 778 // assume this was a floating point load in the IR. If it was 779 // not, we would have had a bitcast before reaching that 780 // instruction. 781 // 782 // Int->FP conversion operations are also captured in 783 // onlyDefinesFP(). 784 return onlyUsesFP(UseMI, MRI, TRI) || 785 onlyDefinesFP(UseMI, MRI, TRI); 786 })) 787 OpRegBankIdx[0] = PMI_FirstFPR; 788 break; 789 case TargetOpcode::G_STORE: 790 // Check if that store is fed by fp instructions. 791 if (OpRegBankIdx[0] == PMI_FirstGPR) { 792 Register VReg = MI.getOperand(0).getReg(); 793 if (!VReg) 794 break; 795 MachineInstr *DefMI = MRI.getVRegDef(VReg); 796 if (onlyDefinesFP(*DefMI, MRI, TRI)) 797 OpRegBankIdx[0] = PMI_FirstFPR; 798 break; 799 } 800 break; 801 case TargetOpcode::G_SELECT: { 802 // If the destination is FPR, preserve that. 803 if (OpRegBankIdx[0] != PMI_FirstGPR) 804 break; 805 806 // If we're taking in vectors, we have no choice but to put everything on 807 // FPRs, except for the condition. The condition must always be on a GPR. 808 LLT SrcTy = MRI.getType(MI.getOperand(2).getReg()); 809 if (SrcTy.isVector()) { 810 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR}; 811 break; 812 } 813 814 // Try to minimize the number of copies. If we have more floating point 815 // constrained values than not, then we'll put everything on FPR. Otherwise, 816 // everything has to be on GPR. 817 unsigned NumFP = 0; 818 819 // Check if the uses of the result always produce floating point values. 820 // 821 // For example: 822 // 823 // %z = G_SELECT %cond %x %y 824 // fpr = G_FOO %z ... 825 if (any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()), 826 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) 827 ++NumFP; 828 829 // Check if the defs of the source values always produce floating point 830 // values. 831 // 832 // For example: 833 // 834 // %x = G_SOMETHING_ALWAYS_FLOAT %a ... 835 // %z = G_SELECT %cond %x %y 836 // 837 // Also check whether or not the sources have already been decided to be 838 // FPR. Keep track of this. 839 // 840 // This doesn't check the condition, since it's just whatever is in NZCV. 841 // This isn't passed explicitly in a register to fcsel/csel. 842 for (unsigned Idx = 2; Idx < 4; ++Idx) { 843 Register VReg = MI.getOperand(Idx).getReg(); 844 MachineInstr *DefMI = MRI.getVRegDef(VReg); 845 if (getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank || 846 onlyDefinesFP(*DefMI, MRI, TRI)) 847 ++NumFP; 848 } 849 850 // If we have more FP constraints than not, then move everything over to 851 // FPR. 852 if (NumFP >= 2) 853 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR}; 854 855 break; 856 } 857 case TargetOpcode::G_UNMERGE_VALUES: { 858 // If the first operand belongs to a FPR register bank, then make sure that 859 // we preserve that. 860 if (OpRegBankIdx[0] != PMI_FirstGPR) 861 break; 862 863 LLT SrcTy = MRI.getType(MI.getOperand(MI.getNumOperands()-1).getReg()); 864 // UNMERGE into scalars from a vector should always use FPR. 865 // Likewise if any of the uses are FP instructions. 866 if (SrcTy.isVector() || SrcTy == LLT::scalar(128) || 867 any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()), 868 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) { 869 // Set the register bank of every operand to FPR. 870 for (unsigned Idx = 0, NumOperands = MI.getNumOperands(); 871 Idx < NumOperands; ++Idx) 872 OpRegBankIdx[Idx] = PMI_FirstFPR; 873 } 874 break; 875 } 876 case TargetOpcode::G_EXTRACT_VECTOR_ELT: 877 // Destination and source need to be FPRs. 878 OpRegBankIdx[0] = PMI_FirstFPR; 879 OpRegBankIdx[1] = PMI_FirstFPR; 880 881 // Index needs to be a GPR. 882 OpRegBankIdx[2] = PMI_FirstGPR; 883 break; 884 case TargetOpcode::G_INSERT_VECTOR_ELT: 885 OpRegBankIdx[0] = PMI_FirstFPR; 886 OpRegBankIdx[1] = PMI_FirstFPR; 887 888 // The element may be either a GPR or FPR. Preserve that behaviour. 889 if (getRegBank(MI.getOperand(2).getReg(), MRI, TRI) == &AArch64::FPRRegBank) 890 OpRegBankIdx[2] = PMI_FirstFPR; 891 else 892 OpRegBankIdx[2] = PMI_FirstGPR; 893 894 // Index needs to be a GPR. 895 OpRegBankIdx[3] = PMI_FirstGPR; 896 break; 897 case TargetOpcode::G_EXTRACT: { 898 // For s128 sources we have to use fpr unless we know otherwise. 899 auto Src = MI.getOperand(1).getReg(); 900 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); 901 if (SrcTy.getSizeInBits() != 128) 902 break; 903 auto Idx = MRI.getRegClassOrNull(Src) == &AArch64::XSeqPairsClassRegClass 904 ? PMI_FirstGPR 905 : PMI_FirstFPR; 906 OpRegBankIdx[0] = Idx; 907 OpRegBankIdx[1] = Idx; 908 break; 909 } 910 case TargetOpcode::G_BUILD_VECTOR: { 911 // If the first source operand belongs to a FPR register bank, then make 912 // sure that we preserve that. 913 if (OpRegBankIdx[1] != PMI_FirstGPR) 914 break; 915 Register VReg = MI.getOperand(1).getReg(); 916 if (!VReg) 917 break; 918 919 // Get the instruction that defined the source operand reg, and check if 920 // it's a floating point operation. Or, if it's a type like s16 which 921 // doesn't have a exact size gpr register class. The exception is if the 922 // build_vector has all constant operands, which may be better to leave as 923 // gpr without copies, so it can be matched in imported patterns. 924 MachineInstr *DefMI = MRI.getVRegDef(VReg); 925 unsigned DefOpc = DefMI->getOpcode(); 926 const LLT SrcTy = MRI.getType(VReg); 927 if (all_of(MI.operands(), [&](const MachineOperand &Op) { 928 return Op.isDef() || MRI.getVRegDef(Op.getReg())->getOpcode() == 929 TargetOpcode::G_CONSTANT; 930 })) 931 break; 932 if (isPreISelGenericFloatingPointOpcode(DefOpc) || 933 SrcTy.getSizeInBits() < 32 || 934 getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank) { 935 // Have a floating point op. 936 // Make sure every operand gets mapped to a FPR register class. 937 unsigned NumOperands = MI.getNumOperands(); 938 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) 939 OpRegBankIdx[Idx] = PMI_FirstFPR; 940 } 941 break; 942 } 943 case TargetOpcode::G_VECREDUCE_FADD: 944 case TargetOpcode::G_VECREDUCE_FMUL: 945 case TargetOpcode::G_VECREDUCE_FMAX: 946 case TargetOpcode::G_VECREDUCE_FMIN: 947 case TargetOpcode::G_VECREDUCE_ADD: 948 case TargetOpcode::G_VECREDUCE_MUL: 949 case TargetOpcode::G_VECREDUCE_AND: 950 case TargetOpcode::G_VECREDUCE_OR: 951 case TargetOpcode::G_VECREDUCE_XOR: 952 case TargetOpcode::G_VECREDUCE_SMAX: 953 case TargetOpcode::G_VECREDUCE_SMIN: 954 case TargetOpcode::G_VECREDUCE_UMAX: 955 case TargetOpcode::G_VECREDUCE_UMIN: 956 // Reductions produce a scalar value from a vector, the scalar should be on 957 // FPR bank. 958 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR}; 959 break; 960 case TargetOpcode::G_VECREDUCE_SEQ_FADD: 961 case TargetOpcode::G_VECREDUCE_SEQ_FMUL: 962 // These reductions also take a scalar accumulator input. 963 // Assign them FPR for now. 964 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR, PMI_FirstFPR}; 965 break; 966 case TargetOpcode::G_INTRINSIC: { 967 // Check if we know that the intrinsic has any constraints on its register 968 // banks. If it does, then update the mapping accordingly. 969 unsigned ID = MI.getIntrinsicID(); 970 unsigned Idx = 0; 971 if (!isFPIntrinsic(ID)) 972 break; 973 for (const auto &Op : MI.explicit_operands()) { 974 if (Op.isReg()) 975 OpRegBankIdx[Idx] = PMI_FirstFPR; 976 ++Idx; 977 } 978 break; 979 } 980 case TargetOpcode::G_LROUND: 981 case TargetOpcode::G_LLROUND: { 982 // Source is always floating point and destination is always integer. 983 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR}; 984 break; 985 } 986 } 987 988 // Finally construct the computed mapping. 989 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOperands); 990 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) { 991 if (MI.getOperand(Idx).isReg() && MI.getOperand(Idx).getReg()) { 992 auto Mapping = getValueMapping(OpRegBankIdx[Idx], OpSize[Idx]); 993 if (!Mapping->isValid()) 994 return getInvalidInstructionMapping(); 995 996 OpdsMapping[Idx] = Mapping; 997 } 998 } 999 1000 return getInstructionMapping(DefaultMappingID, Cost, 1001 getOperandsMapping(OpdsMapping), NumOperands); 1002 } 1003