1 //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the X86 implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "X86InstrInfo.h" 14 #include "X86.h" 15 #include "X86InstrBuilder.h" 16 #include "X86InstrFoldTables.h" 17 #include "X86MachineFunctionInfo.h" 18 #include "X86Subtarget.h" 19 #include "X86TargetMachine.h" 20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/ADT/Sequence.h" 22 #include "llvm/CodeGen/LiveIntervals.h" 23 #include "llvm/CodeGen/LivePhysRegs.h" 24 #include "llvm/CodeGen/LiveVariables.h" 25 #include "llvm/CodeGen/MachineConstantPool.h" 26 #include "llvm/CodeGen/MachineDominators.h" 27 #include "llvm/CodeGen/MachineFrameInfo.h" 28 #include "llvm/CodeGen/MachineInstr.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfo.h" 31 #include "llvm/CodeGen/MachineOperand.h" 32 #include "llvm/CodeGen/MachineRegisterInfo.h" 33 #include "llvm/CodeGen/StackMaps.h" 34 #include "llvm/IR/DebugInfoMetadata.h" 35 #include "llvm/IR/DerivedTypes.h" 36 #include "llvm/IR/Function.h" 37 #include "llvm/IR/InstrTypes.h" 38 #include "llvm/MC/MCAsmInfo.h" 39 #include "llvm/MC/MCExpr.h" 40 #include "llvm/MC/MCInst.h" 41 #include "llvm/Support/CommandLine.h" 42 #include "llvm/Support/Debug.h" 43 #include "llvm/Support/ErrorHandling.h" 44 #include "llvm/Support/raw_ostream.h" 45 #include "llvm/Target/TargetOptions.h" 46 47 using namespace llvm; 48 49 #define DEBUG_TYPE "x86-instr-info" 50 51 #define GET_INSTRINFO_CTOR_DTOR 52 #include "X86GenInstrInfo.inc" 53 54 static cl::opt<bool> 55 NoFusing("disable-spill-fusing", 56 cl::desc("Disable fusing of spill code into instructions"), 57 cl::Hidden); 58 static cl::opt<bool> 59 PrintFailedFusing("print-failed-fuse-candidates", 60 cl::desc("Print instructions that the allocator wants to" 61 " fuse, but the X86 backend currently can't"), 62 cl::Hidden); 63 static cl::opt<bool> 64 ReMatPICStubLoad("remat-pic-stub-load", 65 cl::desc("Re-materialize load from stub in PIC mode"), 66 cl::init(false), cl::Hidden); 67 static cl::opt<unsigned> 68 PartialRegUpdateClearance("partial-reg-update-clearance", 69 cl::desc("Clearance between two register writes " 70 "for inserting XOR to avoid partial " 71 "register update"), 72 cl::init(64), cl::Hidden); 73 static cl::opt<unsigned> 74 UndefRegClearance("undef-reg-clearance", 75 cl::desc("How many idle instructions we would like before " 76 "certain undef register reads"), 77 cl::init(128), cl::Hidden); 78 79 80 // Pin the vtable to this file. 81 void X86InstrInfo::anchor() {} 82 83 X86InstrInfo::X86InstrInfo(X86Subtarget &STI) 84 : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64 85 : X86::ADJCALLSTACKDOWN32), 86 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64 87 : X86::ADJCALLSTACKUP32), 88 X86::CATCHRET, 89 (STI.is64Bit() ? X86::RET64 : X86::RET32)), 90 Subtarget(STI), RI(STI.getTargetTriple()) { 91 } 92 93 bool 94 X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI, 95 Register &SrcReg, Register &DstReg, 96 unsigned &SubIdx) const { 97 switch (MI.getOpcode()) { 98 default: break; 99 case X86::MOVSX16rr8: 100 case X86::MOVZX16rr8: 101 case X86::MOVSX32rr8: 102 case X86::MOVZX32rr8: 103 case X86::MOVSX64rr8: 104 if (!Subtarget.is64Bit()) 105 // It's not always legal to reference the low 8-bit of the larger 106 // register in 32-bit mode. 107 return false; 108 [[fallthrough]]; 109 case X86::MOVSX32rr16: 110 case X86::MOVZX32rr16: 111 case X86::MOVSX64rr16: 112 case X86::MOVSX64rr32: { 113 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) 114 // Be conservative. 115 return false; 116 SrcReg = MI.getOperand(1).getReg(); 117 DstReg = MI.getOperand(0).getReg(); 118 switch (MI.getOpcode()) { 119 default: llvm_unreachable("Unreachable!"); 120 case X86::MOVSX16rr8: 121 case X86::MOVZX16rr8: 122 case X86::MOVSX32rr8: 123 case X86::MOVZX32rr8: 124 case X86::MOVSX64rr8: 125 SubIdx = X86::sub_8bit; 126 break; 127 case X86::MOVSX32rr16: 128 case X86::MOVZX32rr16: 129 case X86::MOVSX64rr16: 130 SubIdx = X86::sub_16bit; 131 break; 132 case X86::MOVSX64rr32: 133 SubIdx = X86::sub_32bit; 134 break; 135 } 136 return true; 137 } 138 } 139 return false; 140 } 141 142 bool X86InstrInfo::isDataInvariant(MachineInstr &MI) { 143 if (MI.mayLoad() || MI.mayStore()) 144 return false; 145 146 // Some target-independent operations that trivially lower to data-invariant 147 // instructions. 148 if (MI.isCopyLike() || MI.isInsertSubreg()) 149 return true; 150 151 unsigned Opcode = MI.getOpcode(); 152 using namespace X86; 153 // On x86 it is believed that imul is constant time w.r.t. the loaded data. 154 // However, they set flags and are perhaps the most surprisingly constant 155 // time operations so we call them out here separately. 156 if (isIMUL(Opcode)) 157 return true; 158 // Bit scanning and counting instructions that are somewhat surprisingly 159 // constant time as they scan across bits and do other fairly complex 160 // operations like popcnt, but are believed to be constant time on x86. 161 // However, these set flags. 162 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) || 163 isTZCNT(Opcode)) 164 return true; 165 // Bit manipulation instructions are effectively combinations of basic 166 // arithmetic ops, and should still execute in constant time. These also 167 // set flags. 168 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) || 169 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) || 170 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) || 171 isTZMSK(Opcode)) 172 return true; 173 // Bit extracting and clearing instructions should execute in constant time, 174 // and set flags. 175 if (isBEXTR(Opcode) || isBZHI(Opcode)) 176 return true; 177 // Shift and rotate. 178 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) || 179 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode)) 180 return true; 181 // Basic arithmetic is constant time on the input but does set flags. 182 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) || 183 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode)) 184 return true; 185 // Arithmetic with just 32-bit and 64-bit variants and no immediates. 186 if (isADCX(Opcode) || isADOX(Opcode) || isANDN(Opcode)) 187 return true; 188 // Unary arithmetic operations. 189 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode)) 190 return true; 191 // Unlike other arithmetic, NOT doesn't set EFLAGS. 192 if (isNOT(Opcode)) 193 return true; 194 // Various move instructions used to zero or sign extend things. Note that we 195 // intentionally don't support the _NOREX variants as we can't handle that 196 // register constraint anyways. 197 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode)) 198 return true; 199 // Arithmetic instructions that are both constant time and don't set flags. 200 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode)) 201 return true; 202 // LEA doesn't actually access memory, and its arithmetic is constant time. 203 if (isLEA(Opcode)) 204 return true; 205 // By default, assume that the instruction is not data invariant. 206 return false; 207 } 208 209 bool X86InstrInfo::isDataInvariantLoad(MachineInstr &MI) { 210 switch (MI.getOpcode()) { 211 default: 212 // By default, assume that the load will immediately leak. 213 return false; 214 215 // On x86 it is believed that imul is constant time w.r.t. the loaded data. 216 // However, they set flags and are perhaps the most surprisingly constant 217 // time operations so we call them out here separately. 218 case X86::IMUL16rm: 219 case X86::IMUL16rmi8: 220 case X86::IMUL16rmi: 221 case X86::IMUL32rm: 222 case X86::IMUL32rmi8: 223 case X86::IMUL32rmi: 224 case X86::IMUL64rm: 225 case X86::IMUL64rmi32: 226 case X86::IMUL64rmi8: 227 228 // Bit scanning and counting instructions that are somewhat surprisingly 229 // constant time as they scan across bits and do other fairly complex 230 // operations like popcnt, but are believed to be constant time on x86. 231 // However, these set flags. 232 case X86::BSF16rm: 233 case X86::BSF32rm: 234 case X86::BSF64rm: 235 case X86::BSR16rm: 236 case X86::BSR32rm: 237 case X86::BSR64rm: 238 case X86::LZCNT16rm: 239 case X86::LZCNT32rm: 240 case X86::LZCNT64rm: 241 case X86::POPCNT16rm: 242 case X86::POPCNT32rm: 243 case X86::POPCNT64rm: 244 case X86::TZCNT16rm: 245 case X86::TZCNT32rm: 246 case X86::TZCNT64rm: 247 248 // Bit manipulation instructions are effectively combinations of basic 249 // arithmetic ops, and should still execute in constant time. These also 250 // set flags. 251 case X86::BLCFILL32rm: 252 case X86::BLCFILL64rm: 253 case X86::BLCI32rm: 254 case X86::BLCI64rm: 255 case X86::BLCIC32rm: 256 case X86::BLCIC64rm: 257 case X86::BLCMSK32rm: 258 case X86::BLCMSK64rm: 259 case X86::BLCS32rm: 260 case X86::BLCS64rm: 261 case X86::BLSFILL32rm: 262 case X86::BLSFILL64rm: 263 case X86::BLSI32rm: 264 case X86::BLSI64rm: 265 case X86::BLSIC32rm: 266 case X86::BLSIC64rm: 267 case X86::BLSMSK32rm: 268 case X86::BLSMSK64rm: 269 case X86::BLSR32rm: 270 case X86::BLSR64rm: 271 case X86::TZMSK32rm: 272 case X86::TZMSK64rm: 273 274 // Bit extracting and clearing instructions should execute in constant time, 275 // and set flags. 276 case X86::BEXTR32rm: 277 case X86::BEXTR64rm: 278 case X86::BEXTRI32mi: 279 case X86::BEXTRI64mi: 280 case X86::BZHI32rm: 281 case X86::BZHI64rm: 282 283 // Basic arithmetic is constant time on the input but does set flags. 284 case X86::ADC8rm: 285 case X86::ADC16rm: 286 case X86::ADC32rm: 287 case X86::ADC64rm: 288 case X86::ADCX32rm: 289 case X86::ADCX64rm: 290 case X86::ADD8rm: 291 case X86::ADD16rm: 292 case X86::ADD32rm: 293 case X86::ADD64rm: 294 case X86::ADOX32rm: 295 case X86::ADOX64rm: 296 case X86::AND8rm: 297 case X86::AND16rm: 298 case X86::AND32rm: 299 case X86::AND64rm: 300 case X86::ANDN32rm: 301 case X86::ANDN64rm: 302 case X86::OR8rm: 303 case X86::OR16rm: 304 case X86::OR32rm: 305 case X86::OR64rm: 306 case X86::SBB8rm: 307 case X86::SBB16rm: 308 case X86::SBB32rm: 309 case X86::SBB64rm: 310 case X86::SUB8rm: 311 case X86::SUB16rm: 312 case X86::SUB32rm: 313 case X86::SUB64rm: 314 case X86::XOR8rm: 315 case X86::XOR16rm: 316 case X86::XOR32rm: 317 case X86::XOR64rm: 318 319 // Integer multiply w/o affecting flags is still believed to be constant 320 // time on x86. Called out separately as this is among the most surprising 321 // instructions to exhibit that behavior. 322 case X86::MULX32rm: 323 case X86::MULX64rm: 324 325 // Arithmetic instructions that are both constant time and don't set flags. 326 case X86::RORX32mi: 327 case X86::RORX64mi: 328 case X86::SARX32rm: 329 case X86::SARX64rm: 330 case X86::SHLX32rm: 331 case X86::SHLX64rm: 332 case X86::SHRX32rm: 333 case X86::SHRX64rm: 334 335 // Conversions are believed to be constant time and don't set flags. 336 case X86::CVTTSD2SI64rm: 337 case X86::VCVTTSD2SI64rm: 338 case X86::VCVTTSD2SI64Zrm: 339 case X86::CVTTSD2SIrm: 340 case X86::VCVTTSD2SIrm: 341 case X86::VCVTTSD2SIZrm: 342 case X86::CVTTSS2SI64rm: 343 case X86::VCVTTSS2SI64rm: 344 case X86::VCVTTSS2SI64Zrm: 345 case X86::CVTTSS2SIrm: 346 case X86::VCVTTSS2SIrm: 347 case X86::VCVTTSS2SIZrm: 348 case X86::CVTSI2SDrm: 349 case X86::VCVTSI2SDrm: 350 case X86::VCVTSI2SDZrm: 351 case X86::CVTSI2SSrm: 352 case X86::VCVTSI2SSrm: 353 case X86::VCVTSI2SSZrm: 354 case X86::CVTSI642SDrm: 355 case X86::VCVTSI642SDrm: 356 case X86::VCVTSI642SDZrm: 357 case X86::CVTSI642SSrm: 358 case X86::VCVTSI642SSrm: 359 case X86::VCVTSI642SSZrm: 360 case X86::CVTSS2SDrm: 361 case X86::VCVTSS2SDrm: 362 case X86::VCVTSS2SDZrm: 363 case X86::CVTSD2SSrm: 364 case X86::VCVTSD2SSrm: 365 case X86::VCVTSD2SSZrm: 366 // AVX512 added unsigned integer conversions. 367 case X86::VCVTTSD2USI64Zrm: 368 case X86::VCVTTSD2USIZrm: 369 case X86::VCVTTSS2USI64Zrm: 370 case X86::VCVTTSS2USIZrm: 371 case X86::VCVTUSI2SDZrm: 372 case X86::VCVTUSI642SDZrm: 373 case X86::VCVTUSI2SSZrm: 374 case X86::VCVTUSI642SSZrm: 375 376 // Loads to register don't set flags. 377 case X86::MOV8rm: 378 case X86::MOV8rm_NOREX: 379 case X86::MOV16rm: 380 case X86::MOV32rm: 381 case X86::MOV64rm: 382 case X86::MOVSX16rm8: 383 case X86::MOVSX32rm16: 384 case X86::MOVSX32rm8: 385 case X86::MOVSX32rm8_NOREX: 386 case X86::MOVSX64rm16: 387 case X86::MOVSX64rm32: 388 case X86::MOVSX64rm8: 389 case X86::MOVZX16rm8: 390 case X86::MOVZX32rm16: 391 case X86::MOVZX32rm8: 392 case X86::MOVZX32rm8_NOREX: 393 case X86::MOVZX64rm16: 394 case X86::MOVZX64rm8: 395 return true; 396 } 397 } 398 399 int X86InstrInfo::getSPAdjust(const MachineInstr &MI) const { 400 const MachineFunction *MF = MI.getParent()->getParent(); 401 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); 402 403 if (isFrameInstr(MI)) { 404 int SPAdj = alignTo(getFrameSize(MI), TFI->getStackAlign()); 405 SPAdj -= getFrameAdjustment(MI); 406 if (!isFrameSetup(MI)) 407 SPAdj = -SPAdj; 408 return SPAdj; 409 } 410 411 // To know whether a call adjusts the stack, we need information 412 // that is bound to the following ADJCALLSTACKUP pseudo. 413 // Look for the next ADJCALLSTACKUP that follows the call. 414 if (MI.isCall()) { 415 const MachineBasicBlock *MBB = MI.getParent(); 416 auto I = ++MachineBasicBlock::const_iterator(MI); 417 for (auto E = MBB->end(); I != E; ++I) { 418 if (I->getOpcode() == getCallFrameDestroyOpcode() || 419 I->isCall()) 420 break; 421 } 422 423 // If we could not find a frame destroy opcode, then it has already 424 // been simplified, so we don't care. 425 if (I->getOpcode() != getCallFrameDestroyOpcode()) 426 return 0; 427 428 return -(I->getOperand(1).getImm()); 429 } 430 431 // Currently handle only PUSHes we can reasonably expect to see 432 // in call sequences 433 switch (MI.getOpcode()) { 434 default: 435 return 0; 436 case X86::PUSH32i8: 437 case X86::PUSH32r: 438 case X86::PUSH32rmm: 439 case X86::PUSH32rmr: 440 case X86::PUSHi32: 441 return 4; 442 case X86::PUSH64i8: 443 case X86::PUSH64r: 444 case X86::PUSH64rmm: 445 case X86::PUSH64rmr: 446 case X86::PUSH64i32: 447 return 8; 448 } 449 } 450 451 /// Return true and the FrameIndex if the specified 452 /// operand and follow operands form a reference to the stack frame. 453 bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op, 454 int &FrameIndex) const { 455 if (MI.getOperand(Op + X86::AddrBaseReg).isFI() && 456 MI.getOperand(Op + X86::AddrScaleAmt).isImm() && 457 MI.getOperand(Op + X86::AddrIndexReg).isReg() && 458 MI.getOperand(Op + X86::AddrDisp).isImm() && 459 MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 && 460 MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 && 461 MI.getOperand(Op + X86::AddrDisp).getImm() == 0) { 462 FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex(); 463 return true; 464 } 465 return false; 466 } 467 468 static bool isFrameLoadOpcode(int Opcode, unsigned &MemBytes) { 469 switch (Opcode) { 470 default: 471 return false; 472 case X86::MOV8rm: 473 case X86::KMOVBkm: 474 MemBytes = 1; 475 return true; 476 case X86::MOV16rm: 477 case X86::KMOVWkm: 478 case X86::VMOVSHZrm: 479 case X86::VMOVSHZrm_alt: 480 MemBytes = 2; 481 return true; 482 case X86::MOV32rm: 483 case X86::MOVSSrm: 484 case X86::MOVSSrm_alt: 485 case X86::VMOVSSrm: 486 case X86::VMOVSSrm_alt: 487 case X86::VMOVSSZrm: 488 case X86::VMOVSSZrm_alt: 489 case X86::KMOVDkm: 490 MemBytes = 4; 491 return true; 492 case X86::MOV64rm: 493 case X86::LD_Fp64m: 494 case X86::MOVSDrm: 495 case X86::MOVSDrm_alt: 496 case X86::VMOVSDrm: 497 case X86::VMOVSDrm_alt: 498 case X86::VMOVSDZrm: 499 case X86::VMOVSDZrm_alt: 500 case X86::MMX_MOVD64rm: 501 case X86::MMX_MOVQ64rm: 502 case X86::KMOVQkm: 503 MemBytes = 8; 504 return true; 505 case X86::MOVAPSrm: 506 case X86::MOVUPSrm: 507 case X86::MOVAPDrm: 508 case X86::MOVUPDrm: 509 case X86::MOVDQArm: 510 case X86::MOVDQUrm: 511 case X86::VMOVAPSrm: 512 case X86::VMOVUPSrm: 513 case X86::VMOVAPDrm: 514 case X86::VMOVUPDrm: 515 case X86::VMOVDQArm: 516 case X86::VMOVDQUrm: 517 case X86::VMOVAPSZ128rm: 518 case X86::VMOVUPSZ128rm: 519 case X86::VMOVAPSZ128rm_NOVLX: 520 case X86::VMOVUPSZ128rm_NOVLX: 521 case X86::VMOVAPDZ128rm: 522 case X86::VMOVUPDZ128rm: 523 case X86::VMOVDQU8Z128rm: 524 case X86::VMOVDQU16Z128rm: 525 case X86::VMOVDQA32Z128rm: 526 case X86::VMOVDQU32Z128rm: 527 case X86::VMOVDQA64Z128rm: 528 case X86::VMOVDQU64Z128rm: 529 MemBytes = 16; 530 return true; 531 case X86::VMOVAPSYrm: 532 case X86::VMOVUPSYrm: 533 case X86::VMOVAPDYrm: 534 case X86::VMOVUPDYrm: 535 case X86::VMOVDQAYrm: 536 case X86::VMOVDQUYrm: 537 case X86::VMOVAPSZ256rm: 538 case X86::VMOVUPSZ256rm: 539 case X86::VMOVAPSZ256rm_NOVLX: 540 case X86::VMOVUPSZ256rm_NOVLX: 541 case X86::VMOVAPDZ256rm: 542 case X86::VMOVUPDZ256rm: 543 case X86::VMOVDQU8Z256rm: 544 case X86::VMOVDQU16Z256rm: 545 case X86::VMOVDQA32Z256rm: 546 case X86::VMOVDQU32Z256rm: 547 case X86::VMOVDQA64Z256rm: 548 case X86::VMOVDQU64Z256rm: 549 MemBytes = 32; 550 return true; 551 case X86::VMOVAPSZrm: 552 case X86::VMOVUPSZrm: 553 case X86::VMOVAPDZrm: 554 case X86::VMOVUPDZrm: 555 case X86::VMOVDQU8Zrm: 556 case X86::VMOVDQU16Zrm: 557 case X86::VMOVDQA32Zrm: 558 case X86::VMOVDQU32Zrm: 559 case X86::VMOVDQA64Zrm: 560 case X86::VMOVDQU64Zrm: 561 MemBytes = 64; 562 return true; 563 } 564 } 565 566 static bool isFrameStoreOpcode(int Opcode, unsigned &MemBytes) { 567 switch (Opcode) { 568 default: 569 return false; 570 case X86::MOV8mr: 571 case X86::KMOVBmk: 572 MemBytes = 1; 573 return true; 574 case X86::MOV16mr: 575 case X86::KMOVWmk: 576 case X86::VMOVSHZmr: 577 MemBytes = 2; 578 return true; 579 case X86::MOV32mr: 580 case X86::MOVSSmr: 581 case X86::VMOVSSmr: 582 case X86::VMOVSSZmr: 583 case X86::KMOVDmk: 584 MemBytes = 4; 585 return true; 586 case X86::MOV64mr: 587 case X86::ST_FpP64m: 588 case X86::MOVSDmr: 589 case X86::VMOVSDmr: 590 case X86::VMOVSDZmr: 591 case X86::MMX_MOVD64mr: 592 case X86::MMX_MOVQ64mr: 593 case X86::MMX_MOVNTQmr: 594 case X86::KMOVQmk: 595 MemBytes = 8; 596 return true; 597 case X86::MOVAPSmr: 598 case X86::MOVUPSmr: 599 case X86::MOVAPDmr: 600 case X86::MOVUPDmr: 601 case X86::MOVDQAmr: 602 case X86::MOVDQUmr: 603 case X86::VMOVAPSmr: 604 case X86::VMOVUPSmr: 605 case X86::VMOVAPDmr: 606 case X86::VMOVUPDmr: 607 case X86::VMOVDQAmr: 608 case X86::VMOVDQUmr: 609 case X86::VMOVUPSZ128mr: 610 case X86::VMOVAPSZ128mr: 611 case X86::VMOVUPSZ128mr_NOVLX: 612 case X86::VMOVAPSZ128mr_NOVLX: 613 case X86::VMOVUPDZ128mr: 614 case X86::VMOVAPDZ128mr: 615 case X86::VMOVDQA32Z128mr: 616 case X86::VMOVDQU32Z128mr: 617 case X86::VMOVDQA64Z128mr: 618 case X86::VMOVDQU64Z128mr: 619 case X86::VMOVDQU8Z128mr: 620 case X86::VMOVDQU16Z128mr: 621 MemBytes = 16; 622 return true; 623 case X86::VMOVUPSYmr: 624 case X86::VMOVAPSYmr: 625 case X86::VMOVUPDYmr: 626 case X86::VMOVAPDYmr: 627 case X86::VMOVDQUYmr: 628 case X86::VMOVDQAYmr: 629 case X86::VMOVUPSZ256mr: 630 case X86::VMOVAPSZ256mr: 631 case X86::VMOVUPSZ256mr_NOVLX: 632 case X86::VMOVAPSZ256mr_NOVLX: 633 case X86::VMOVUPDZ256mr: 634 case X86::VMOVAPDZ256mr: 635 case X86::VMOVDQU8Z256mr: 636 case X86::VMOVDQU16Z256mr: 637 case X86::VMOVDQA32Z256mr: 638 case X86::VMOVDQU32Z256mr: 639 case X86::VMOVDQA64Z256mr: 640 case X86::VMOVDQU64Z256mr: 641 MemBytes = 32; 642 return true; 643 case X86::VMOVUPSZmr: 644 case X86::VMOVAPSZmr: 645 case X86::VMOVUPDZmr: 646 case X86::VMOVAPDZmr: 647 case X86::VMOVDQU8Zmr: 648 case X86::VMOVDQU16Zmr: 649 case X86::VMOVDQA32Zmr: 650 case X86::VMOVDQU32Zmr: 651 case X86::VMOVDQA64Zmr: 652 case X86::VMOVDQU64Zmr: 653 MemBytes = 64; 654 return true; 655 } 656 return false; 657 } 658 659 unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 660 int &FrameIndex) const { 661 unsigned Dummy; 662 return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy); 663 } 664 665 unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 666 int &FrameIndex, 667 unsigned &MemBytes) const { 668 if (isFrameLoadOpcode(MI.getOpcode(), MemBytes)) 669 if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex)) 670 return MI.getOperand(0).getReg(); 671 return 0; 672 } 673 674 unsigned X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI, 675 int &FrameIndex) const { 676 unsigned Dummy; 677 if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) { 678 unsigned Reg; 679 if ((Reg = isLoadFromStackSlot(MI, FrameIndex))) 680 return Reg; 681 // Check for post-frame index elimination operations 682 SmallVector<const MachineMemOperand *, 1> Accesses; 683 if (hasLoadFromStackSlot(MI, Accesses)) { 684 FrameIndex = 685 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) 686 ->getFrameIndex(); 687 return MI.getOperand(0).getReg(); 688 } 689 } 690 return 0; 691 } 692 693 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, 694 int &FrameIndex) const { 695 unsigned Dummy; 696 return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy); 697 } 698 699 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, 700 int &FrameIndex, 701 unsigned &MemBytes) const { 702 if (isFrameStoreOpcode(MI.getOpcode(), MemBytes)) 703 if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 && 704 isFrameOperand(MI, 0, FrameIndex)) 705 return MI.getOperand(X86::AddrNumOperands).getReg(); 706 return 0; 707 } 708 709 unsigned X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI, 710 int &FrameIndex) const { 711 unsigned Dummy; 712 if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) { 713 unsigned Reg; 714 if ((Reg = isStoreToStackSlot(MI, FrameIndex))) 715 return Reg; 716 // Check for post-frame index elimination operations 717 SmallVector<const MachineMemOperand *, 1> Accesses; 718 if (hasStoreToStackSlot(MI, Accesses)) { 719 FrameIndex = 720 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) 721 ->getFrameIndex(); 722 return MI.getOperand(X86::AddrNumOperands).getReg(); 723 } 724 } 725 return 0; 726 } 727 728 /// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r. 729 static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) { 730 // Don't waste compile time scanning use-def chains of physregs. 731 if (!BaseReg.isVirtual()) 732 return false; 733 bool isPICBase = false; 734 for (MachineRegisterInfo::def_instr_iterator I = MRI.def_instr_begin(BaseReg), 735 E = MRI.def_instr_end(); I != E; ++I) { 736 MachineInstr *DefMI = &*I; 737 if (DefMI->getOpcode() != X86::MOVPC32r) 738 return false; 739 assert(!isPICBase && "More than one PIC base?"); 740 isPICBase = true; 741 } 742 return isPICBase; 743 } 744 745 bool X86InstrInfo::isReallyTriviallyReMaterializable( 746 const MachineInstr &MI) const { 747 switch (MI.getOpcode()) { 748 default: 749 // This function should only be called for opcodes with the ReMaterializable 750 // flag set. 751 llvm_unreachable("Unknown rematerializable operation!"); 752 break; 753 754 case X86::LOAD_STACK_GUARD: 755 case X86::AVX1_SETALLONES: 756 case X86::AVX2_SETALLONES: 757 case X86::AVX512_128_SET0: 758 case X86::AVX512_256_SET0: 759 case X86::AVX512_512_SET0: 760 case X86::AVX512_512_SETALLONES: 761 case X86::AVX512_FsFLD0SD: 762 case X86::AVX512_FsFLD0SH: 763 case X86::AVX512_FsFLD0SS: 764 case X86::AVX512_FsFLD0F128: 765 case X86::AVX_SET0: 766 case X86::FsFLD0SD: 767 case X86::FsFLD0SS: 768 case X86::FsFLD0SH: 769 case X86::FsFLD0F128: 770 case X86::KSET0D: 771 case X86::KSET0Q: 772 case X86::KSET0W: 773 case X86::KSET1D: 774 case X86::KSET1Q: 775 case X86::KSET1W: 776 case X86::MMX_SET0: 777 case X86::MOV32ImmSExti8: 778 case X86::MOV32r0: 779 case X86::MOV32r1: 780 case X86::MOV32r_1: 781 case X86::MOV32ri64: 782 case X86::MOV64ImmSExti8: 783 case X86::V_SET0: 784 case X86::V_SETALLONES: 785 case X86::MOV16ri: 786 case X86::MOV32ri: 787 case X86::MOV64ri: 788 case X86::MOV64ri32: 789 case X86::MOV8ri: 790 case X86::PTILEZEROV: 791 return true; 792 793 case X86::MOV8rm: 794 case X86::MOV8rm_NOREX: 795 case X86::MOV16rm: 796 case X86::MOV32rm: 797 case X86::MOV64rm: 798 case X86::MOVSSrm: 799 case X86::MOVSSrm_alt: 800 case X86::MOVSDrm: 801 case X86::MOVSDrm_alt: 802 case X86::MOVAPSrm: 803 case X86::MOVUPSrm: 804 case X86::MOVAPDrm: 805 case X86::MOVUPDrm: 806 case X86::MOVDQArm: 807 case X86::MOVDQUrm: 808 case X86::VMOVSSrm: 809 case X86::VMOVSSrm_alt: 810 case X86::VMOVSDrm: 811 case X86::VMOVSDrm_alt: 812 case X86::VMOVAPSrm: 813 case X86::VMOVUPSrm: 814 case X86::VMOVAPDrm: 815 case X86::VMOVUPDrm: 816 case X86::VMOVDQArm: 817 case X86::VMOVDQUrm: 818 case X86::VMOVAPSYrm: 819 case X86::VMOVUPSYrm: 820 case X86::VMOVAPDYrm: 821 case X86::VMOVUPDYrm: 822 case X86::VMOVDQAYrm: 823 case X86::VMOVDQUYrm: 824 case X86::MMX_MOVD64rm: 825 case X86::MMX_MOVQ64rm: 826 // AVX-512 827 case X86::VMOVSSZrm: 828 case X86::VMOVSSZrm_alt: 829 case X86::VMOVSDZrm: 830 case X86::VMOVSDZrm_alt: 831 case X86::VMOVSHZrm: 832 case X86::VMOVSHZrm_alt: 833 case X86::VMOVAPDZ128rm: 834 case X86::VMOVAPDZ256rm: 835 case X86::VMOVAPDZrm: 836 case X86::VMOVAPSZ128rm: 837 case X86::VMOVAPSZ256rm: 838 case X86::VMOVAPSZ128rm_NOVLX: 839 case X86::VMOVAPSZ256rm_NOVLX: 840 case X86::VMOVAPSZrm: 841 case X86::VMOVDQA32Z128rm: 842 case X86::VMOVDQA32Z256rm: 843 case X86::VMOVDQA32Zrm: 844 case X86::VMOVDQA64Z128rm: 845 case X86::VMOVDQA64Z256rm: 846 case X86::VMOVDQA64Zrm: 847 case X86::VMOVDQU16Z128rm: 848 case X86::VMOVDQU16Z256rm: 849 case X86::VMOVDQU16Zrm: 850 case X86::VMOVDQU32Z128rm: 851 case X86::VMOVDQU32Z256rm: 852 case X86::VMOVDQU32Zrm: 853 case X86::VMOVDQU64Z128rm: 854 case X86::VMOVDQU64Z256rm: 855 case X86::VMOVDQU64Zrm: 856 case X86::VMOVDQU8Z128rm: 857 case X86::VMOVDQU8Z256rm: 858 case X86::VMOVDQU8Zrm: 859 case X86::VMOVUPDZ128rm: 860 case X86::VMOVUPDZ256rm: 861 case X86::VMOVUPDZrm: 862 case X86::VMOVUPSZ128rm: 863 case X86::VMOVUPSZ256rm: 864 case X86::VMOVUPSZ128rm_NOVLX: 865 case X86::VMOVUPSZ256rm_NOVLX: 866 case X86::VMOVUPSZrm: { 867 // Loads from constant pools are trivially rematerializable. 868 if (MI.getOperand(1 + X86::AddrBaseReg).isReg() && 869 MI.getOperand(1 + X86::AddrScaleAmt).isImm() && 870 MI.getOperand(1 + X86::AddrIndexReg).isReg() && 871 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && 872 MI.isDereferenceableInvariantLoad()) { 873 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); 874 if (BaseReg == 0 || BaseReg == X86::RIP) 875 return true; 876 // Allow re-materialization of PIC load. 877 if (!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal()) 878 return false; 879 const MachineFunction &MF = *MI.getParent()->getParent(); 880 const MachineRegisterInfo &MRI = MF.getRegInfo(); 881 return regIsPICBase(BaseReg, MRI); 882 } 883 return false; 884 } 885 886 case X86::LEA32r: 887 case X86::LEA64r: { 888 if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() && 889 MI.getOperand(1 + X86::AddrIndexReg).isReg() && 890 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && 891 !MI.getOperand(1 + X86::AddrDisp).isReg()) { 892 // lea fi#, lea GV, etc. are all rematerializable. 893 if (!MI.getOperand(1 + X86::AddrBaseReg).isReg()) 894 return true; 895 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); 896 if (BaseReg == 0) 897 return true; 898 // Allow re-materialization of lea PICBase + x. 899 const MachineFunction &MF = *MI.getParent()->getParent(); 900 const MachineRegisterInfo &MRI = MF.getRegInfo(); 901 return regIsPICBase(BaseReg, MRI); 902 } 903 return false; 904 } 905 } 906 } 907 908 void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB, 909 MachineBasicBlock::iterator I, 910 Register DestReg, unsigned SubIdx, 911 const MachineInstr &Orig, 912 const TargetRegisterInfo &TRI) const { 913 bool ClobbersEFLAGS = Orig.modifiesRegister(X86::EFLAGS, &TRI); 914 if (ClobbersEFLAGS && MBB.computeRegisterLiveness(&TRI, X86::EFLAGS, I) != 915 MachineBasicBlock::LQR_Dead) { 916 // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side 917 // effects. 918 int Value; 919 switch (Orig.getOpcode()) { 920 case X86::MOV32r0: Value = 0; break; 921 case X86::MOV32r1: Value = 1; break; 922 case X86::MOV32r_1: Value = -1; break; 923 default: 924 llvm_unreachable("Unexpected instruction!"); 925 } 926 927 const DebugLoc &DL = Orig.getDebugLoc(); 928 BuildMI(MBB, I, DL, get(X86::MOV32ri)) 929 .add(Orig.getOperand(0)) 930 .addImm(Value); 931 } else { 932 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); 933 MBB.insert(I, MI); 934 } 935 936 MachineInstr &NewMI = *std::prev(I); 937 NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI); 938 } 939 940 /// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead. 941 bool X86InstrInfo::hasLiveCondCodeDef(MachineInstr &MI) const { 942 for (const MachineOperand &MO : MI.operands()) { 943 if (MO.isReg() && MO.isDef() && 944 MO.getReg() == X86::EFLAGS && !MO.isDead()) { 945 return true; 946 } 947 } 948 return false; 949 } 950 951 /// Check whether the shift count for a machine operand is non-zero. 952 inline static unsigned getTruncatedShiftCount(const MachineInstr &MI, 953 unsigned ShiftAmtOperandIdx) { 954 // The shift count is six bits with the REX.W prefix and five bits without. 955 unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31; 956 unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm(); 957 return Imm & ShiftCountMask; 958 } 959 960 /// Check whether the given shift count is appropriate 961 /// can be represented by a LEA instruction. 962 inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) { 963 // Left shift instructions can be transformed into load-effective-address 964 // instructions if we can encode them appropriately. 965 // A LEA instruction utilizes a SIB byte to encode its scale factor. 966 // The SIB.scale field is two bits wide which means that we can encode any 967 // shift amount less than 4. 968 return ShAmt < 4 && ShAmt > 0; 969 } 970 971 static bool findRedundantFlagInstr(MachineInstr &CmpInstr, 972 MachineInstr &CmpValDefInstr, 973 const MachineRegisterInfo *MRI, 974 MachineInstr **AndInstr, 975 const TargetRegisterInfo *TRI, 976 bool &NoSignFlag, bool &ClearsOverflowFlag) { 977 if (CmpValDefInstr.getOpcode() != X86::SUBREG_TO_REG) 978 return false; 979 980 if (CmpInstr.getOpcode() != X86::TEST64rr) 981 return false; 982 983 // CmpInstr is a TEST64rr instruction, and `X86InstrInfo::analyzeCompare` 984 // guarantees that it's analyzable only if two registers are identical. 985 assert( 986 (CmpInstr.getOperand(0).getReg() == CmpInstr.getOperand(1).getReg()) && 987 "CmpInstr is an analyzable TEST64rr, and `X86InstrInfo::analyzeCompare` " 988 "requires two reg operands are the same."); 989 990 // Caller (`X86InstrInfo::optimizeCompareInstr`) guarantees that 991 // `CmpValDefInstr` defines the value that's used by `CmpInstr`; in this case 992 // if `CmpValDefInstr` sets the EFLAGS, it is likely that `CmpInstr` is 993 // redundant. 994 assert( 995 (MRI->getVRegDef(CmpInstr.getOperand(0).getReg()) == &CmpValDefInstr) && 996 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG."); 997 998 // As seen in X86 td files, CmpValDefInstr.getOperand(1).getImm() is typically 999 // 0. 1000 if (CmpValDefInstr.getOperand(1).getImm() != 0) 1001 return false; 1002 1003 // As seen in X86 td files, CmpValDefInstr.getOperand(3) is typically 1004 // sub_32bit or sub_xmm. 1005 if (CmpValDefInstr.getOperand(3).getImm() != X86::sub_32bit) 1006 return false; 1007 1008 MachineInstr *VregDefInstr = 1009 MRI->getVRegDef(CmpValDefInstr.getOperand(2).getReg()); 1010 1011 assert(VregDefInstr && "Must have a definition (SSA)"); 1012 1013 // Requires `CmpValDefInstr` and `VregDefInstr` are from the same MBB 1014 // to simplify the subsequent analysis. 1015 // 1016 // FIXME: If `VregDefInstr->getParent()` is the only predecessor of 1017 // `CmpValDefInstr.getParent()`, this could be handled. 1018 if (VregDefInstr->getParent() != CmpValDefInstr.getParent()) 1019 return false; 1020 1021 if (X86::isAND(VregDefInstr->getOpcode())) { 1022 // Get a sequence of instructions like 1023 // %reg = and* ... // Set EFLAGS 1024 // ... // EFLAGS not changed 1025 // %extended_reg = subreg_to_reg 0, %reg, %subreg.sub_32bit 1026 // test64rr %extended_reg, %extended_reg, implicit-def $eflags 1027 // 1028 // If subsequent readers use a subset of bits that don't change 1029 // after `and*` instructions, it's likely that the test64rr could 1030 // be optimized away. 1031 for (const MachineInstr &Instr : 1032 make_range(std::next(MachineBasicBlock::iterator(VregDefInstr)), 1033 MachineBasicBlock::iterator(CmpValDefInstr))) { 1034 // There are instructions between 'VregDefInstr' and 1035 // 'CmpValDefInstr' that modifies EFLAGS. 1036 if (Instr.modifiesRegister(X86::EFLAGS, TRI)) 1037 return false; 1038 } 1039 1040 *AndInstr = VregDefInstr; 1041 1042 // AND instruction will essentially update SF and clear OF, so 1043 // NoSignFlag should be false in the sense that SF is modified by `AND`. 1044 // 1045 // However, the implementation artifically sets `NoSignFlag` to true 1046 // to poison the SF bit; that is to say, if SF is looked at later, the 1047 // optimization (to erase TEST64rr) will be disabled. 1048 // 1049 // The reason to poison SF bit is that SF bit value could be different 1050 // in the `AND` and `TEST` operation; signed bit is not known for `AND`, 1051 // and is known to be 0 as a result of `TEST64rr`. 1052 // 1053 // FIXME: As opposed to poisoning the SF bit directly, consider peeking into 1054 // the AND instruction and using the static information to guide peephole 1055 // optimization if possible. For example, it's possible to fold a 1056 // conditional move into a copy if the relevant EFLAG bits could be deduced 1057 // from an immediate operand of and operation. 1058 // 1059 NoSignFlag = true; 1060 // ClearsOverflowFlag is true for AND operation (no surprise). 1061 ClearsOverflowFlag = true; 1062 return true; 1063 } 1064 return false; 1065 } 1066 1067 bool X86InstrInfo::classifyLEAReg(MachineInstr &MI, const MachineOperand &Src, 1068 unsigned Opc, bool AllowSP, Register &NewSrc, 1069 bool &isKill, MachineOperand &ImplicitOp, 1070 LiveVariables *LV, LiveIntervals *LIS) const { 1071 MachineFunction &MF = *MI.getParent()->getParent(); 1072 const TargetRegisterClass *RC; 1073 if (AllowSP) { 1074 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass; 1075 } else { 1076 RC = Opc != X86::LEA32r ? 1077 &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass; 1078 } 1079 Register SrcReg = Src.getReg(); 1080 isKill = MI.killsRegister(SrcReg); 1081 1082 // For both LEA64 and LEA32 the register already has essentially the right 1083 // type (32-bit or 64-bit) we may just need to forbid SP. 1084 if (Opc != X86::LEA64_32r) { 1085 NewSrc = SrcReg; 1086 assert(!Src.isUndef() && "Undef op doesn't need optimization"); 1087 1088 if (NewSrc.isVirtual() && !MF.getRegInfo().constrainRegClass(NewSrc, RC)) 1089 return false; 1090 1091 return true; 1092 } 1093 1094 // This is for an LEA64_32r and incoming registers are 32-bit. One way or 1095 // another we need to add 64-bit registers to the final MI. 1096 if (SrcReg.isPhysical()) { 1097 ImplicitOp = Src; 1098 ImplicitOp.setImplicit(); 1099 1100 NewSrc = getX86SubSuperRegister(SrcReg, 64); 1101 assert(!Src.isUndef() && "Undef op doesn't need optimization"); 1102 } else { 1103 // Virtual register of the wrong class, we have to create a temporary 64-bit 1104 // vreg to feed into the LEA. 1105 NewSrc = MF.getRegInfo().createVirtualRegister(RC); 1106 MachineInstr *Copy = 1107 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1108 .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit) 1109 .addReg(SrcReg, getKillRegState(isKill)); 1110 1111 // Which is obviously going to be dead after we're done with it. 1112 isKill = true; 1113 1114 if (LV) 1115 LV->replaceKillInstruction(SrcReg, MI, *Copy); 1116 1117 if (LIS) { 1118 SlotIndex CopyIdx = LIS->InsertMachineInstrInMaps(*Copy); 1119 SlotIndex Idx = LIS->getInstructionIndex(MI); 1120 LiveInterval &LI = LIS->getInterval(SrcReg); 1121 LiveRange::Segment *S = LI.getSegmentContaining(Idx); 1122 if (S->end.getBaseIndex() == Idx) 1123 S->end = CopyIdx.getRegSlot(); 1124 } 1125 } 1126 1127 // We've set all the parameters without issue. 1128 return true; 1129 } 1130 1131 MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc, 1132 MachineInstr &MI, 1133 LiveVariables *LV, 1134 LiveIntervals *LIS, 1135 bool Is8BitOp) const { 1136 // We handle 8-bit adds and various 16-bit opcodes in the switch below. 1137 MachineBasicBlock &MBB = *MI.getParent(); 1138 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo(); 1139 assert((Is8BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits( 1140 *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) && 1141 "Unexpected type for LEA transform"); 1142 1143 // TODO: For a 32-bit target, we need to adjust the LEA variables with 1144 // something like this: 1145 // Opcode = X86::LEA32r; 1146 // InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); 1147 // OutRegLEA = 1148 // Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass) 1149 // : RegInfo.createVirtualRegister(&X86::GR32RegClass); 1150 if (!Subtarget.is64Bit()) 1151 return nullptr; 1152 1153 unsigned Opcode = X86::LEA64_32r; 1154 Register InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 1155 Register OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass); 1156 Register InRegLEA2; 1157 1158 // Build and insert into an implicit UNDEF value. This is OK because 1159 // we will be shifting and then extracting the lower 8/16-bits. 1160 // This has the potential to cause partial register stall. e.g. 1161 // movw (%rbp,%rcx,2), %dx 1162 // leal -65(%rdx), %esi 1163 // But testing has shown this *does* help performance in 64-bit mode (at 1164 // least on modern x86 machines). 1165 MachineBasicBlock::iterator MBBI = MI.getIterator(); 1166 Register Dest = MI.getOperand(0).getReg(); 1167 Register Src = MI.getOperand(1).getReg(); 1168 Register Src2; 1169 bool IsDead = MI.getOperand(0).isDead(); 1170 bool IsKill = MI.getOperand(1).isKill(); 1171 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit; 1172 assert(!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization"); 1173 MachineInstr *ImpDef = 1174 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA); 1175 MachineInstr *InsMI = 1176 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1177 .addReg(InRegLEA, RegState::Define, SubReg) 1178 .addReg(Src, getKillRegState(IsKill)); 1179 MachineInstr *ImpDef2 = nullptr; 1180 MachineInstr *InsMI2 = nullptr; 1181 1182 MachineInstrBuilder MIB = 1183 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(Opcode), OutRegLEA); 1184 switch (MIOpc) { 1185 default: llvm_unreachable("Unreachable!"); 1186 case X86::SHL8ri: 1187 case X86::SHL16ri: { 1188 unsigned ShAmt = MI.getOperand(2).getImm(); 1189 MIB.addReg(0) 1190 .addImm(1LL << ShAmt) 1191 .addReg(InRegLEA, RegState::Kill) 1192 .addImm(0) 1193 .addReg(0); 1194 break; 1195 } 1196 case X86::INC8r: 1197 case X86::INC16r: 1198 addRegOffset(MIB, InRegLEA, true, 1); 1199 break; 1200 case X86::DEC8r: 1201 case X86::DEC16r: 1202 addRegOffset(MIB, InRegLEA, true, -1); 1203 break; 1204 case X86::ADD8ri: 1205 case X86::ADD8ri_DB: 1206 case X86::ADD16ri: 1207 case X86::ADD16ri8: 1208 case X86::ADD16ri_DB: 1209 case X86::ADD16ri8_DB: 1210 addRegOffset(MIB, InRegLEA, true, MI.getOperand(2).getImm()); 1211 break; 1212 case X86::ADD8rr: 1213 case X86::ADD8rr_DB: 1214 case X86::ADD16rr: 1215 case X86::ADD16rr_DB: { 1216 Src2 = MI.getOperand(2).getReg(); 1217 bool IsKill2 = MI.getOperand(2).isKill(); 1218 assert(!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization"); 1219 if (Src == Src2) { 1220 // ADD8rr/ADD16rr killed %reg1028, %reg1028 1221 // just a single insert_subreg. 1222 addRegReg(MIB, InRegLEA, true, InRegLEA, false); 1223 } else { 1224 if (Subtarget.is64Bit()) 1225 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 1226 else 1227 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); 1228 // Build and insert into an implicit UNDEF value. This is OK because 1229 // we will be shifting and then extracting the lower 8/16-bits. 1230 ImpDef2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), 1231 InRegLEA2); 1232 InsMI2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1233 .addReg(InRegLEA2, RegState::Define, SubReg) 1234 .addReg(Src2, getKillRegState(IsKill2)); 1235 addRegReg(MIB, InRegLEA, true, InRegLEA2, true); 1236 } 1237 if (LV && IsKill2 && InsMI2) 1238 LV->replaceKillInstruction(Src2, MI, *InsMI2); 1239 break; 1240 } 1241 } 1242 1243 MachineInstr *NewMI = MIB; 1244 MachineInstr *ExtMI = 1245 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1246 .addReg(Dest, RegState::Define | getDeadRegState(IsDead)) 1247 .addReg(OutRegLEA, RegState::Kill, SubReg); 1248 1249 if (LV) { 1250 // Update live variables. 1251 LV->getVarInfo(InRegLEA).Kills.push_back(NewMI); 1252 if (InRegLEA2) 1253 LV->getVarInfo(InRegLEA2).Kills.push_back(NewMI); 1254 LV->getVarInfo(OutRegLEA).Kills.push_back(ExtMI); 1255 if (IsKill) 1256 LV->replaceKillInstruction(Src, MI, *InsMI); 1257 if (IsDead) 1258 LV->replaceKillInstruction(Dest, MI, *ExtMI); 1259 } 1260 1261 if (LIS) { 1262 LIS->InsertMachineInstrInMaps(*ImpDef); 1263 SlotIndex InsIdx = LIS->InsertMachineInstrInMaps(*InsMI); 1264 if (ImpDef2) 1265 LIS->InsertMachineInstrInMaps(*ImpDef2); 1266 SlotIndex Ins2Idx; 1267 if (InsMI2) 1268 Ins2Idx = LIS->InsertMachineInstrInMaps(*InsMI2); 1269 SlotIndex NewIdx = LIS->ReplaceMachineInstrInMaps(MI, *NewMI); 1270 SlotIndex ExtIdx = LIS->InsertMachineInstrInMaps(*ExtMI); 1271 LIS->getInterval(InRegLEA); 1272 LIS->getInterval(OutRegLEA); 1273 if (InRegLEA2) 1274 LIS->getInterval(InRegLEA2); 1275 1276 // Move the use of Src up to InsMI. 1277 LiveInterval &SrcLI = LIS->getInterval(Src); 1278 LiveRange::Segment *SrcSeg = SrcLI.getSegmentContaining(NewIdx); 1279 if (SrcSeg->end == NewIdx.getRegSlot()) 1280 SrcSeg->end = InsIdx.getRegSlot(); 1281 1282 if (InsMI2) { 1283 // Move the use of Src2 up to InsMI2. 1284 LiveInterval &Src2LI = LIS->getInterval(Src2); 1285 LiveRange::Segment *Src2Seg = Src2LI.getSegmentContaining(NewIdx); 1286 if (Src2Seg->end == NewIdx.getRegSlot()) 1287 Src2Seg->end = Ins2Idx.getRegSlot(); 1288 } 1289 1290 // Move the definition of Dest down to ExtMI. 1291 LiveInterval &DestLI = LIS->getInterval(Dest); 1292 LiveRange::Segment *DestSeg = 1293 DestLI.getSegmentContaining(NewIdx.getRegSlot()); 1294 assert(DestSeg->start == NewIdx.getRegSlot() && 1295 DestSeg->valno->def == NewIdx.getRegSlot()); 1296 DestSeg->start = ExtIdx.getRegSlot(); 1297 DestSeg->valno->def = ExtIdx.getRegSlot(); 1298 } 1299 1300 return ExtMI; 1301 } 1302 1303 /// This method must be implemented by targets that 1304 /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target 1305 /// may be able to convert a two-address instruction into a true 1306 /// three-address instruction on demand. This allows the X86 target (for 1307 /// example) to convert ADD and SHL instructions into LEA instructions if they 1308 /// would require register copies due to two-addressness. 1309 /// 1310 /// This method returns a null pointer if the transformation cannot be 1311 /// performed, otherwise it returns the new instruction. 1312 /// 1313 MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI, 1314 LiveVariables *LV, 1315 LiveIntervals *LIS) const { 1316 // The following opcodes also sets the condition code register(s). Only 1317 // convert them to equivalent lea if the condition code register def's 1318 // are dead! 1319 if (hasLiveCondCodeDef(MI)) 1320 return nullptr; 1321 1322 MachineFunction &MF = *MI.getParent()->getParent(); 1323 // All instructions input are two-addr instructions. Get the known operands. 1324 const MachineOperand &Dest = MI.getOperand(0); 1325 const MachineOperand &Src = MI.getOperand(1); 1326 1327 // Ideally, operations with undef should be folded before we get here, but we 1328 // can't guarantee it. Bail out because optimizing undefs is a waste of time. 1329 // Without this, we have to forward undef state to new register operands to 1330 // avoid machine verifier errors. 1331 if (Src.isUndef()) 1332 return nullptr; 1333 if (MI.getNumOperands() > 2) 1334 if (MI.getOperand(2).isReg() && MI.getOperand(2).isUndef()) 1335 return nullptr; 1336 1337 MachineInstr *NewMI = nullptr; 1338 Register SrcReg, SrcReg2; 1339 bool Is64Bit = Subtarget.is64Bit(); 1340 1341 bool Is8BitOp = false; 1342 unsigned NumRegOperands = 2; 1343 unsigned MIOpc = MI.getOpcode(); 1344 switch (MIOpc) { 1345 default: llvm_unreachable("Unreachable!"); 1346 case X86::SHL64ri: { 1347 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 1348 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 1349 if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; 1350 1351 // LEA can't handle RSP. 1352 if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass( 1353 Src.getReg(), &X86::GR64_NOSPRegClass)) 1354 return nullptr; 1355 1356 NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)) 1357 .add(Dest) 1358 .addReg(0) 1359 .addImm(1LL << ShAmt) 1360 .add(Src) 1361 .addImm(0) 1362 .addReg(0); 1363 break; 1364 } 1365 case X86::SHL32ri: { 1366 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 1367 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 1368 if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; 1369 1370 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1371 1372 // LEA can't handle ESP. 1373 bool isKill; 1374 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1375 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill, 1376 ImplicitOp, LV, LIS)) 1377 return nullptr; 1378 1379 MachineInstrBuilder MIB = 1380 BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1381 .add(Dest) 1382 .addReg(0) 1383 .addImm(1LL << ShAmt) 1384 .addReg(SrcReg, getKillRegState(isKill)) 1385 .addImm(0) 1386 .addReg(0); 1387 if (ImplicitOp.getReg() != 0) 1388 MIB.add(ImplicitOp); 1389 NewMI = MIB; 1390 1391 // Add kills if classifyLEAReg created a new register. 1392 if (LV && SrcReg != Src.getReg()) 1393 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1394 break; 1395 } 1396 case X86::SHL8ri: 1397 Is8BitOp = true; 1398 [[fallthrough]]; 1399 case X86::SHL16ri: { 1400 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 1401 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 1402 if (!isTruncatedShiftCountForLEA(ShAmt)) 1403 return nullptr; 1404 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1405 } 1406 case X86::INC64r: 1407 case X86::INC32r: { 1408 assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!"); 1409 unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r : 1410 (Is64Bit ? X86::LEA64_32r : X86::LEA32r); 1411 bool isKill; 1412 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1413 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill, 1414 ImplicitOp, LV, LIS)) 1415 return nullptr; 1416 1417 MachineInstrBuilder MIB = 1418 BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1419 .add(Dest) 1420 .addReg(SrcReg, getKillRegState(isKill)); 1421 if (ImplicitOp.getReg() != 0) 1422 MIB.add(ImplicitOp); 1423 1424 NewMI = addOffset(MIB, 1); 1425 1426 // Add kills if classifyLEAReg created a new register. 1427 if (LV && SrcReg != Src.getReg()) 1428 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1429 break; 1430 } 1431 case X86::DEC64r: 1432 case X86::DEC32r: { 1433 assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!"); 1434 unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r 1435 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r); 1436 1437 bool isKill; 1438 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1439 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill, 1440 ImplicitOp, LV, LIS)) 1441 return nullptr; 1442 1443 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1444 .add(Dest) 1445 .addReg(SrcReg, getKillRegState(isKill)); 1446 if (ImplicitOp.getReg() != 0) 1447 MIB.add(ImplicitOp); 1448 1449 NewMI = addOffset(MIB, -1); 1450 1451 // Add kills if classifyLEAReg created a new register. 1452 if (LV && SrcReg != Src.getReg()) 1453 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1454 break; 1455 } 1456 case X86::DEC8r: 1457 case X86::INC8r: 1458 Is8BitOp = true; 1459 [[fallthrough]]; 1460 case X86::DEC16r: 1461 case X86::INC16r: 1462 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1463 case X86::ADD64rr: 1464 case X86::ADD64rr_DB: 1465 case X86::ADD32rr: 1466 case X86::ADD32rr_DB: { 1467 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1468 unsigned Opc; 1469 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB) 1470 Opc = X86::LEA64r; 1471 else 1472 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1473 1474 const MachineOperand &Src2 = MI.getOperand(2); 1475 bool isKill2; 1476 MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false); 1477 if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/false, SrcReg2, isKill2, 1478 ImplicitOp2, LV, LIS)) 1479 return nullptr; 1480 1481 bool isKill; 1482 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1483 if (Src.getReg() == Src2.getReg()) { 1484 // Don't call classify LEAReg a second time on the same register, in case 1485 // the first call inserted a COPY from Src2 and marked it as killed. 1486 isKill = isKill2; 1487 SrcReg = SrcReg2; 1488 } else { 1489 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill, 1490 ImplicitOp, LV, LIS)) 1491 return nullptr; 1492 } 1493 1494 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest); 1495 if (ImplicitOp.getReg() != 0) 1496 MIB.add(ImplicitOp); 1497 if (ImplicitOp2.getReg() != 0) 1498 MIB.add(ImplicitOp2); 1499 1500 NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2); 1501 1502 // Add kills if classifyLEAReg created a new register. 1503 if (LV) { 1504 if (SrcReg2 != Src2.getReg()) 1505 LV->getVarInfo(SrcReg2).Kills.push_back(NewMI); 1506 if (SrcReg != SrcReg2 && SrcReg != Src.getReg()) 1507 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1508 } 1509 NumRegOperands = 3; 1510 break; 1511 } 1512 case X86::ADD8rr: 1513 case X86::ADD8rr_DB: 1514 Is8BitOp = true; 1515 [[fallthrough]]; 1516 case X86::ADD16rr: 1517 case X86::ADD16rr_DB: 1518 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1519 case X86::ADD64ri32: 1520 case X86::ADD64ri8: 1521 case X86::ADD64ri32_DB: 1522 case X86::ADD64ri8_DB: 1523 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1524 NewMI = addOffset( 1525 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src), 1526 MI.getOperand(2)); 1527 break; 1528 case X86::ADD32ri: 1529 case X86::ADD32ri8: 1530 case X86::ADD32ri_DB: 1531 case X86::ADD32ri8_DB: { 1532 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1533 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1534 1535 bool isKill; 1536 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1537 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill, 1538 ImplicitOp, LV, LIS)) 1539 return nullptr; 1540 1541 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1542 .add(Dest) 1543 .addReg(SrcReg, getKillRegState(isKill)); 1544 if (ImplicitOp.getReg() != 0) 1545 MIB.add(ImplicitOp); 1546 1547 NewMI = addOffset(MIB, MI.getOperand(2)); 1548 1549 // Add kills if classifyLEAReg created a new register. 1550 if (LV && SrcReg != Src.getReg()) 1551 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1552 break; 1553 } 1554 case X86::ADD8ri: 1555 case X86::ADD8ri_DB: 1556 Is8BitOp = true; 1557 [[fallthrough]]; 1558 case X86::ADD16ri: 1559 case X86::ADD16ri8: 1560 case X86::ADD16ri_DB: 1561 case X86::ADD16ri8_DB: 1562 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1563 case X86::SUB8ri: 1564 case X86::SUB16ri8: 1565 case X86::SUB16ri: 1566 /// FIXME: Support these similar to ADD8ri/ADD16ri*. 1567 return nullptr; 1568 case X86::SUB32ri8: 1569 case X86::SUB32ri: { 1570 if (!MI.getOperand(2).isImm()) 1571 return nullptr; 1572 int64_t Imm = MI.getOperand(2).getImm(); 1573 if (!isInt<32>(-Imm)) 1574 return nullptr; 1575 1576 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1577 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1578 1579 bool isKill; 1580 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1581 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill, 1582 ImplicitOp, LV, LIS)) 1583 return nullptr; 1584 1585 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1586 .add(Dest) 1587 .addReg(SrcReg, getKillRegState(isKill)); 1588 if (ImplicitOp.getReg() != 0) 1589 MIB.add(ImplicitOp); 1590 1591 NewMI = addOffset(MIB, -Imm); 1592 1593 // Add kills if classifyLEAReg created a new register. 1594 if (LV && SrcReg != Src.getReg()) 1595 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1596 break; 1597 } 1598 1599 case X86::SUB64ri8: 1600 case X86::SUB64ri32: { 1601 if (!MI.getOperand(2).isImm()) 1602 return nullptr; 1603 int64_t Imm = MI.getOperand(2).getImm(); 1604 if (!isInt<32>(-Imm)) 1605 return nullptr; 1606 1607 assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!"); 1608 1609 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), 1610 get(X86::LEA64r)).add(Dest).add(Src); 1611 NewMI = addOffset(MIB, -Imm); 1612 break; 1613 } 1614 1615 case X86::VMOVDQU8Z128rmk: 1616 case X86::VMOVDQU8Z256rmk: 1617 case X86::VMOVDQU8Zrmk: 1618 case X86::VMOVDQU16Z128rmk: 1619 case X86::VMOVDQU16Z256rmk: 1620 case X86::VMOVDQU16Zrmk: 1621 case X86::VMOVDQU32Z128rmk: case X86::VMOVDQA32Z128rmk: 1622 case X86::VMOVDQU32Z256rmk: case X86::VMOVDQA32Z256rmk: 1623 case X86::VMOVDQU32Zrmk: case X86::VMOVDQA32Zrmk: 1624 case X86::VMOVDQU64Z128rmk: case X86::VMOVDQA64Z128rmk: 1625 case X86::VMOVDQU64Z256rmk: case X86::VMOVDQA64Z256rmk: 1626 case X86::VMOVDQU64Zrmk: case X86::VMOVDQA64Zrmk: 1627 case X86::VMOVUPDZ128rmk: case X86::VMOVAPDZ128rmk: 1628 case X86::VMOVUPDZ256rmk: case X86::VMOVAPDZ256rmk: 1629 case X86::VMOVUPDZrmk: case X86::VMOVAPDZrmk: 1630 case X86::VMOVUPSZ128rmk: case X86::VMOVAPSZ128rmk: 1631 case X86::VMOVUPSZ256rmk: case X86::VMOVAPSZ256rmk: 1632 case X86::VMOVUPSZrmk: case X86::VMOVAPSZrmk: 1633 case X86::VBROADCASTSDZ256rmk: 1634 case X86::VBROADCASTSDZrmk: 1635 case X86::VBROADCASTSSZ128rmk: 1636 case X86::VBROADCASTSSZ256rmk: 1637 case X86::VBROADCASTSSZrmk: 1638 case X86::VPBROADCASTDZ128rmk: 1639 case X86::VPBROADCASTDZ256rmk: 1640 case X86::VPBROADCASTDZrmk: 1641 case X86::VPBROADCASTQZ128rmk: 1642 case X86::VPBROADCASTQZ256rmk: 1643 case X86::VPBROADCASTQZrmk: { 1644 unsigned Opc; 1645 switch (MIOpc) { 1646 default: llvm_unreachable("Unreachable!"); 1647 case X86::VMOVDQU8Z128rmk: Opc = X86::VPBLENDMBZ128rmk; break; 1648 case X86::VMOVDQU8Z256rmk: Opc = X86::VPBLENDMBZ256rmk; break; 1649 case X86::VMOVDQU8Zrmk: Opc = X86::VPBLENDMBZrmk; break; 1650 case X86::VMOVDQU16Z128rmk: Opc = X86::VPBLENDMWZ128rmk; break; 1651 case X86::VMOVDQU16Z256rmk: Opc = X86::VPBLENDMWZ256rmk; break; 1652 case X86::VMOVDQU16Zrmk: Opc = X86::VPBLENDMWZrmk; break; 1653 case X86::VMOVDQU32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; 1654 case X86::VMOVDQU32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; 1655 case X86::VMOVDQU32Zrmk: Opc = X86::VPBLENDMDZrmk; break; 1656 case X86::VMOVDQU64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; 1657 case X86::VMOVDQU64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; 1658 case X86::VMOVDQU64Zrmk: Opc = X86::VPBLENDMQZrmk; break; 1659 case X86::VMOVUPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; 1660 case X86::VMOVUPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; 1661 case X86::VMOVUPDZrmk: Opc = X86::VBLENDMPDZrmk; break; 1662 case X86::VMOVUPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; 1663 case X86::VMOVUPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; 1664 case X86::VMOVUPSZrmk: Opc = X86::VBLENDMPSZrmk; break; 1665 case X86::VMOVDQA32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; 1666 case X86::VMOVDQA32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; 1667 case X86::VMOVDQA32Zrmk: Opc = X86::VPBLENDMDZrmk; break; 1668 case X86::VMOVDQA64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; 1669 case X86::VMOVDQA64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; 1670 case X86::VMOVDQA64Zrmk: Opc = X86::VPBLENDMQZrmk; break; 1671 case X86::VMOVAPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; 1672 case X86::VMOVAPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; 1673 case X86::VMOVAPDZrmk: Opc = X86::VBLENDMPDZrmk; break; 1674 case X86::VMOVAPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; 1675 case X86::VMOVAPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; 1676 case X86::VMOVAPSZrmk: Opc = X86::VBLENDMPSZrmk; break; 1677 case X86::VBROADCASTSDZ256rmk: Opc = X86::VBLENDMPDZ256rmbk; break; 1678 case X86::VBROADCASTSDZrmk: Opc = X86::VBLENDMPDZrmbk; break; 1679 case X86::VBROADCASTSSZ128rmk: Opc = X86::VBLENDMPSZ128rmbk; break; 1680 case X86::VBROADCASTSSZ256rmk: Opc = X86::VBLENDMPSZ256rmbk; break; 1681 case X86::VBROADCASTSSZrmk: Opc = X86::VBLENDMPSZrmbk; break; 1682 case X86::VPBROADCASTDZ128rmk: Opc = X86::VPBLENDMDZ128rmbk; break; 1683 case X86::VPBROADCASTDZ256rmk: Opc = X86::VPBLENDMDZ256rmbk; break; 1684 case X86::VPBROADCASTDZrmk: Opc = X86::VPBLENDMDZrmbk; break; 1685 case X86::VPBROADCASTQZ128rmk: Opc = X86::VPBLENDMQZ128rmbk; break; 1686 case X86::VPBROADCASTQZ256rmk: Opc = X86::VPBLENDMQZ256rmbk; break; 1687 case X86::VPBROADCASTQZrmk: Opc = X86::VPBLENDMQZrmbk; break; 1688 } 1689 1690 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1691 .add(Dest) 1692 .add(MI.getOperand(2)) 1693 .add(Src) 1694 .add(MI.getOperand(3)) 1695 .add(MI.getOperand(4)) 1696 .add(MI.getOperand(5)) 1697 .add(MI.getOperand(6)) 1698 .add(MI.getOperand(7)); 1699 NumRegOperands = 4; 1700 break; 1701 } 1702 1703 case X86::VMOVDQU8Z128rrk: 1704 case X86::VMOVDQU8Z256rrk: 1705 case X86::VMOVDQU8Zrrk: 1706 case X86::VMOVDQU16Z128rrk: 1707 case X86::VMOVDQU16Z256rrk: 1708 case X86::VMOVDQU16Zrrk: 1709 case X86::VMOVDQU32Z128rrk: case X86::VMOVDQA32Z128rrk: 1710 case X86::VMOVDQU32Z256rrk: case X86::VMOVDQA32Z256rrk: 1711 case X86::VMOVDQU32Zrrk: case X86::VMOVDQA32Zrrk: 1712 case X86::VMOVDQU64Z128rrk: case X86::VMOVDQA64Z128rrk: 1713 case X86::VMOVDQU64Z256rrk: case X86::VMOVDQA64Z256rrk: 1714 case X86::VMOVDQU64Zrrk: case X86::VMOVDQA64Zrrk: 1715 case X86::VMOVUPDZ128rrk: case X86::VMOVAPDZ128rrk: 1716 case X86::VMOVUPDZ256rrk: case X86::VMOVAPDZ256rrk: 1717 case X86::VMOVUPDZrrk: case X86::VMOVAPDZrrk: 1718 case X86::VMOVUPSZ128rrk: case X86::VMOVAPSZ128rrk: 1719 case X86::VMOVUPSZ256rrk: case X86::VMOVAPSZ256rrk: 1720 case X86::VMOVUPSZrrk: case X86::VMOVAPSZrrk: { 1721 unsigned Opc; 1722 switch (MIOpc) { 1723 default: llvm_unreachable("Unreachable!"); 1724 case X86::VMOVDQU8Z128rrk: Opc = X86::VPBLENDMBZ128rrk; break; 1725 case X86::VMOVDQU8Z256rrk: Opc = X86::VPBLENDMBZ256rrk; break; 1726 case X86::VMOVDQU8Zrrk: Opc = X86::VPBLENDMBZrrk; break; 1727 case X86::VMOVDQU16Z128rrk: Opc = X86::VPBLENDMWZ128rrk; break; 1728 case X86::VMOVDQU16Z256rrk: Opc = X86::VPBLENDMWZ256rrk; break; 1729 case X86::VMOVDQU16Zrrk: Opc = X86::VPBLENDMWZrrk; break; 1730 case X86::VMOVDQU32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; 1731 case X86::VMOVDQU32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; 1732 case X86::VMOVDQU32Zrrk: Opc = X86::VPBLENDMDZrrk; break; 1733 case X86::VMOVDQU64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; 1734 case X86::VMOVDQU64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; 1735 case X86::VMOVDQU64Zrrk: Opc = X86::VPBLENDMQZrrk; break; 1736 case X86::VMOVUPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; 1737 case X86::VMOVUPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; 1738 case X86::VMOVUPDZrrk: Opc = X86::VBLENDMPDZrrk; break; 1739 case X86::VMOVUPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; 1740 case X86::VMOVUPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; 1741 case X86::VMOVUPSZrrk: Opc = X86::VBLENDMPSZrrk; break; 1742 case X86::VMOVDQA32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; 1743 case X86::VMOVDQA32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; 1744 case X86::VMOVDQA32Zrrk: Opc = X86::VPBLENDMDZrrk; break; 1745 case X86::VMOVDQA64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; 1746 case X86::VMOVDQA64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; 1747 case X86::VMOVDQA64Zrrk: Opc = X86::VPBLENDMQZrrk; break; 1748 case X86::VMOVAPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; 1749 case X86::VMOVAPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; 1750 case X86::VMOVAPDZrrk: Opc = X86::VBLENDMPDZrrk; break; 1751 case X86::VMOVAPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; 1752 case X86::VMOVAPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; 1753 case X86::VMOVAPSZrrk: Opc = X86::VBLENDMPSZrrk; break; 1754 } 1755 1756 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1757 .add(Dest) 1758 .add(MI.getOperand(2)) 1759 .add(Src) 1760 .add(MI.getOperand(3)); 1761 NumRegOperands = 4; 1762 break; 1763 } 1764 } 1765 1766 if (!NewMI) return nullptr; 1767 1768 if (LV) { // Update live variables 1769 for (unsigned I = 0; I < NumRegOperands; ++I) { 1770 MachineOperand &Op = MI.getOperand(I); 1771 if (Op.isReg() && (Op.isDead() || Op.isKill())) 1772 LV->replaceKillInstruction(Op.getReg(), MI, *NewMI); 1773 } 1774 } 1775 1776 MachineBasicBlock &MBB = *MI.getParent(); 1777 MBB.insert(MI.getIterator(), NewMI); // Insert the new inst 1778 1779 if (LIS) { 1780 LIS->ReplaceMachineInstrInMaps(MI, *NewMI); 1781 if (SrcReg) 1782 LIS->getInterval(SrcReg); 1783 if (SrcReg2) 1784 LIS->getInterval(SrcReg2); 1785 } 1786 1787 return NewMI; 1788 } 1789 1790 /// This determines which of three possible cases of a three source commute 1791 /// the source indexes correspond to taking into account any mask operands. 1792 /// All prevents commuting a passthru operand. Returns -1 if the commute isn't 1793 /// possible. 1794 /// Case 0 - Possible to commute the first and second operands. 1795 /// Case 1 - Possible to commute the first and third operands. 1796 /// Case 2 - Possible to commute the second and third operands. 1797 static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1, 1798 unsigned SrcOpIdx2) { 1799 // Put the lowest index to SrcOpIdx1 to simplify the checks below. 1800 if (SrcOpIdx1 > SrcOpIdx2) 1801 std::swap(SrcOpIdx1, SrcOpIdx2); 1802 1803 unsigned Op1 = 1, Op2 = 2, Op3 = 3; 1804 if (X86II::isKMasked(TSFlags)) { 1805 Op2++; 1806 Op3++; 1807 } 1808 1809 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2) 1810 return 0; 1811 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3) 1812 return 1; 1813 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3) 1814 return 2; 1815 llvm_unreachable("Unknown three src commute case."); 1816 } 1817 1818 unsigned X86InstrInfo::getFMA3OpcodeToCommuteOperands( 1819 const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2, 1820 const X86InstrFMA3Group &FMA3Group) const { 1821 1822 unsigned Opc = MI.getOpcode(); 1823 1824 // TODO: Commuting the 1st operand of FMA*_Int requires some additional 1825 // analysis. The commute optimization is legal only if all users of FMA*_Int 1826 // use only the lowest element of the FMA*_Int instruction. Such analysis are 1827 // not implemented yet. So, just return 0 in that case. 1828 // When such analysis are available this place will be the right place for 1829 // calling it. 1830 assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && 1831 "Intrinsic instructions can't commute operand 1"); 1832 1833 // Determine which case this commute is or if it can't be done. 1834 unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, 1835 SrcOpIdx2); 1836 assert(Case < 3 && "Unexpected case number!"); 1837 1838 // Define the FMA forms mapping array that helps to map input FMA form 1839 // to output FMA form to preserve the operation semantics after 1840 // commuting the operands. 1841 const unsigned Form132Index = 0; 1842 const unsigned Form213Index = 1; 1843 const unsigned Form231Index = 2; 1844 static const unsigned FormMapping[][3] = { 1845 // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2; 1846 // FMA132 A, C, b; ==> FMA231 C, A, b; 1847 // FMA213 B, A, c; ==> FMA213 A, B, c; 1848 // FMA231 C, A, b; ==> FMA132 A, C, b; 1849 { Form231Index, Form213Index, Form132Index }, 1850 // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3; 1851 // FMA132 A, c, B; ==> FMA132 B, c, A; 1852 // FMA213 B, a, C; ==> FMA231 C, a, B; 1853 // FMA231 C, a, B; ==> FMA213 B, a, C; 1854 { Form132Index, Form231Index, Form213Index }, 1855 // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3; 1856 // FMA132 a, C, B; ==> FMA213 a, B, C; 1857 // FMA213 b, A, C; ==> FMA132 b, C, A; 1858 // FMA231 c, A, B; ==> FMA231 c, B, A; 1859 { Form213Index, Form132Index, Form231Index } 1860 }; 1861 1862 unsigned FMAForms[3]; 1863 FMAForms[0] = FMA3Group.get132Opcode(); 1864 FMAForms[1] = FMA3Group.get213Opcode(); 1865 FMAForms[2] = FMA3Group.get231Opcode(); 1866 1867 // Everything is ready, just adjust the FMA opcode and return it. 1868 for (unsigned FormIndex = 0; FormIndex < 3; FormIndex++) 1869 if (Opc == FMAForms[FormIndex]) 1870 return FMAForms[FormMapping[Case][FormIndex]]; 1871 1872 llvm_unreachable("Illegal FMA3 format"); 1873 } 1874 1875 static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1, 1876 unsigned SrcOpIdx2) { 1877 // Determine which case this commute is or if it can't be done. 1878 unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, 1879 SrcOpIdx2); 1880 assert(Case < 3 && "Unexpected case value!"); 1881 1882 // For each case we need to swap two pairs of bits in the final immediate. 1883 static const uint8_t SwapMasks[3][4] = { 1884 { 0x04, 0x10, 0x08, 0x20 }, // Swap bits 2/4 and 3/5. 1885 { 0x02, 0x10, 0x08, 0x40 }, // Swap bits 1/4 and 3/6. 1886 { 0x02, 0x04, 0x20, 0x40 }, // Swap bits 1/2 and 5/6. 1887 }; 1888 1889 uint8_t Imm = MI.getOperand(MI.getNumOperands()-1).getImm(); 1890 // Clear out the bits we are swapping. 1891 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] | 1892 SwapMasks[Case][2] | SwapMasks[Case][3]); 1893 // If the immediate had a bit of the pair set, then set the opposite bit. 1894 if (Imm & SwapMasks[Case][0]) NewImm |= SwapMasks[Case][1]; 1895 if (Imm & SwapMasks[Case][1]) NewImm |= SwapMasks[Case][0]; 1896 if (Imm & SwapMasks[Case][2]) NewImm |= SwapMasks[Case][3]; 1897 if (Imm & SwapMasks[Case][3]) NewImm |= SwapMasks[Case][2]; 1898 MI.getOperand(MI.getNumOperands()-1).setImm(NewImm); 1899 } 1900 1901 // Returns true if this is a VPERMI2 or VPERMT2 instruction that can be 1902 // commuted. 1903 static bool isCommutableVPERMV3Instruction(unsigned Opcode) { 1904 #define VPERM_CASES(Suffix) \ 1905 case X86::VPERMI2##Suffix##128rr: case X86::VPERMT2##Suffix##128rr: \ 1906 case X86::VPERMI2##Suffix##256rr: case X86::VPERMT2##Suffix##256rr: \ 1907 case X86::VPERMI2##Suffix##rr: case X86::VPERMT2##Suffix##rr: \ 1908 case X86::VPERMI2##Suffix##128rm: case X86::VPERMT2##Suffix##128rm: \ 1909 case X86::VPERMI2##Suffix##256rm: case X86::VPERMT2##Suffix##256rm: \ 1910 case X86::VPERMI2##Suffix##rm: case X86::VPERMT2##Suffix##rm: \ 1911 case X86::VPERMI2##Suffix##128rrkz: case X86::VPERMT2##Suffix##128rrkz: \ 1912 case X86::VPERMI2##Suffix##256rrkz: case X86::VPERMT2##Suffix##256rrkz: \ 1913 case X86::VPERMI2##Suffix##rrkz: case X86::VPERMT2##Suffix##rrkz: \ 1914 case X86::VPERMI2##Suffix##128rmkz: case X86::VPERMT2##Suffix##128rmkz: \ 1915 case X86::VPERMI2##Suffix##256rmkz: case X86::VPERMT2##Suffix##256rmkz: \ 1916 case X86::VPERMI2##Suffix##rmkz: case X86::VPERMT2##Suffix##rmkz: 1917 1918 #define VPERM_CASES_BROADCAST(Suffix) \ 1919 VPERM_CASES(Suffix) \ 1920 case X86::VPERMI2##Suffix##128rmb: case X86::VPERMT2##Suffix##128rmb: \ 1921 case X86::VPERMI2##Suffix##256rmb: case X86::VPERMT2##Suffix##256rmb: \ 1922 case X86::VPERMI2##Suffix##rmb: case X86::VPERMT2##Suffix##rmb: \ 1923 case X86::VPERMI2##Suffix##128rmbkz: case X86::VPERMT2##Suffix##128rmbkz: \ 1924 case X86::VPERMI2##Suffix##256rmbkz: case X86::VPERMT2##Suffix##256rmbkz: \ 1925 case X86::VPERMI2##Suffix##rmbkz: case X86::VPERMT2##Suffix##rmbkz: 1926 1927 switch (Opcode) { 1928 default: return false; 1929 VPERM_CASES(B) 1930 VPERM_CASES_BROADCAST(D) 1931 VPERM_CASES_BROADCAST(PD) 1932 VPERM_CASES_BROADCAST(PS) 1933 VPERM_CASES_BROADCAST(Q) 1934 VPERM_CASES(W) 1935 return true; 1936 } 1937 #undef VPERM_CASES_BROADCAST 1938 #undef VPERM_CASES 1939 } 1940 1941 // Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching 1942 // from the I opcode to the T opcode and vice versa. 1943 static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) { 1944 #define VPERM_CASES(Orig, New) \ 1945 case X86::Orig##128rr: return X86::New##128rr; \ 1946 case X86::Orig##128rrkz: return X86::New##128rrkz; \ 1947 case X86::Orig##128rm: return X86::New##128rm; \ 1948 case X86::Orig##128rmkz: return X86::New##128rmkz; \ 1949 case X86::Orig##256rr: return X86::New##256rr; \ 1950 case X86::Orig##256rrkz: return X86::New##256rrkz; \ 1951 case X86::Orig##256rm: return X86::New##256rm; \ 1952 case X86::Orig##256rmkz: return X86::New##256rmkz; \ 1953 case X86::Orig##rr: return X86::New##rr; \ 1954 case X86::Orig##rrkz: return X86::New##rrkz; \ 1955 case X86::Orig##rm: return X86::New##rm; \ 1956 case X86::Orig##rmkz: return X86::New##rmkz; 1957 1958 #define VPERM_CASES_BROADCAST(Orig, New) \ 1959 VPERM_CASES(Orig, New) \ 1960 case X86::Orig##128rmb: return X86::New##128rmb; \ 1961 case X86::Orig##128rmbkz: return X86::New##128rmbkz; \ 1962 case X86::Orig##256rmb: return X86::New##256rmb; \ 1963 case X86::Orig##256rmbkz: return X86::New##256rmbkz; \ 1964 case X86::Orig##rmb: return X86::New##rmb; \ 1965 case X86::Orig##rmbkz: return X86::New##rmbkz; 1966 1967 switch (Opcode) { 1968 VPERM_CASES(VPERMI2B, VPERMT2B) 1969 VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D) 1970 VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD) 1971 VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS) 1972 VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q) 1973 VPERM_CASES(VPERMI2W, VPERMT2W) 1974 VPERM_CASES(VPERMT2B, VPERMI2B) 1975 VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D) 1976 VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD) 1977 VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS) 1978 VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q) 1979 VPERM_CASES(VPERMT2W, VPERMI2W) 1980 } 1981 1982 llvm_unreachable("Unreachable!"); 1983 #undef VPERM_CASES_BROADCAST 1984 #undef VPERM_CASES 1985 } 1986 1987 MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 1988 unsigned OpIdx1, 1989 unsigned OpIdx2) const { 1990 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 1991 if (NewMI) 1992 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 1993 return MI; 1994 }; 1995 1996 switch (MI.getOpcode()) { 1997 case X86::SHRD16rri8: // A = SHRD16rri8 B, C, I -> A = SHLD16rri8 C, B, (16-I) 1998 case X86::SHLD16rri8: // A = SHLD16rri8 B, C, I -> A = SHRD16rri8 C, B, (16-I) 1999 case X86::SHRD32rri8: // A = SHRD32rri8 B, C, I -> A = SHLD32rri8 C, B, (32-I) 2000 case X86::SHLD32rri8: // A = SHLD32rri8 B, C, I -> A = SHRD32rri8 C, B, (32-I) 2001 case X86::SHRD64rri8: // A = SHRD64rri8 B, C, I -> A = SHLD64rri8 C, B, (64-I) 2002 case X86::SHLD64rri8:{// A = SHLD64rri8 B, C, I -> A = SHRD64rri8 C, B, (64-I) 2003 unsigned Opc; 2004 unsigned Size; 2005 switch (MI.getOpcode()) { 2006 default: llvm_unreachable("Unreachable!"); 2007 case X86::SHRD16rri8: Size = 16; Opc = X86::SHLD16rri8; break; 2008 case X86::SHLD16rri8: Size = 16; Opc = X86::SHRD16rri8; break; 2009 case X86::SHRD32rri8: Size = 32; Opc = X86::SHLD32rri8; break; 2010 case X86::SHLD32rri8: Size = 32; Opc = X86::SHRD32rri8; break; 2011 case X86::SHRD64rri8: Size = 64; Opc = X86::SHLD64rri8; break; 2012 case X86::SHLD64rri8: Size = 64; Opc = X86::SHRD64rri8; break; 2013 } 2014 unsigned Amt = MI.getOperand(3).getImm(); 2015 auto &WorkingMI = cloneIfNew(MI); 2016 WorkingMI.setDesc(get(Opc)); 2017 WorkingMI.getOperand(3).setImm(Size - Amt); 2018 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2019 OpIdx1, OpIdx2); 2020 } 2021 case X86::PFSUBrr: 2022 case X86::PFSUBRrr: { 2023 // PFSUB x, y: x = x - y 2024 // PFSUBR x, y: x = y - x 2025 unsigned Opc = 2026 (X86::PFSUBRrr == MI.getOpcode() ? X86::PFSUBrr : X86::PFSUBRrr); 2027 auto &WorkingMI = cloneIfNew(MI); 2028 WorkingMI.setDesc(get(Opc)); 2029 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2030 OpIdx1, OpIdx2); 2031 } 2032 case X86::BLENDPDrri: 2033 case X86::BLENDPSrri: 2034 case X86::VBLENDPDrri: 2035 case X86::VBLENDPSrri: 2036 // If we're optimizing for size, try to use MOVSD/MOVSS. 2037 if (MI.getParent()->getParent()->getFunction().hasOptSize()) { 2038 unsigned Mask, Opc; 2039 switch (MI.getOpcode()) { 2040 default: llvm_unreachable("Unreachable!"); 2041 case X86::BLENDPDrri: Opc = X86::MOVSDrr; Mask = 0x03; break; 2042 case X86::BLENDPSrri: Opc = X86::MOVSSrr; Mask = 0x0F; break; 2043 case X86::VBLENDPDrri: Opc = X86::VMOVSDrr; Mask = 0x03; break; 2044 case X86::VBLENDPSrri: Opc = X86::VMOVSSrr; Mask = 0x0F; break; 2045 } 2046 if ((MI.getOperand(3).getImm() ^ Mask) == 1) { 2047 auto &WorkingMI = cloneIfNew(MI); 2048 WorkingMI.setDesc(get(Opc)); 2049 WorkingMI.removeOperand(3); 2050 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, 2051 /*NewMI=*/false, 2052 OpIdx1, OpIdx2); 2053 } 2054 } 2055 [[fallthrough]]; 2056 case X86::PBLENDWrri: 2057 case X86::VBLENDPDYrri: 2058 case X86::VBLENDPSYrri: 2059 case X86::VPBLENDDrri: 2060 case X86::VPBLENDWrri: 2061 case X86::VPBLENDDYrri: 2062 case X86::VPBLENDWYrri:{ 2063 int8_t Mask; 2064 switch (MI.getOpcode()) { 2065 default: llvm_unreachable("Unreachable!"); 2066 case X86::BLENDPDrri: Mask = (int8_t)0x03; break; 2067 case X86::BLENDPSrri: Mask = (int8_t)0x0F; break; 2068 case X86::PBLENDWrri: Mask = (int8_t)0xFF; break; 2069 case X86::VBLENDPDrri: Mask = (int8_t)0x03; break; 2070 case X86::VBLENDPSrri: Mask = (int8_t)0x0F; break; 2071 case X86::VBLENDPDYrri: Mask = (int8_t)0x0F; break; 2072 case X86::VBLENDPSYrri: Mask = (int8_t)0xFF; break; 2073 case X86::VPBLENDDrri: Mask = (int8_t)0x0F; break; 2074 case X86::VPBLENDWrri: Mask = (int8_t)0xFF; break; 2075 case X86::VPBLENDDYrri: Mask = (int8_t)0xFF; break; 2076 case X86::VPBLENDWYrri: Mask = (int8_t)0xFF; break; 2077 } 2078 // Only the least significant bits of Imm are used. 2079 // Using int8_t to ensure it will be sign extended to the int64_t that 2080 // setImm takes in order to match isel behavior. 2081 int8_t Imm = MI.getOperand(3).getImm() & Mask; 2082 auto &WorkingMI = cloneIfNew(MI); 2083 WorkingMI.getOperand(3).setImm(Mask ^ Imm); 2084 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2085 OpIdx1, OpIdx2); 2086 } 2087 case X86::INSERTPSrr: 2088 case X86::VINSERTPSrr: 2089 case X86::VINSERTPSZrr: { 2090 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); 2091 unsigned ZMask = Imm & 15; 2092 unsigned DstIdx = (Imm >> 4) & 3; 2093 unsigned SrcIdx = (Imm >> 6) & 3; 2094 2095 // We can commute insertps if we zero 2 of the elements, the insertion is 2096 // "inline" and we don't override the insertion with a zero. 2097 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 && 2098 llvm::popcount(ZMask) == 2) { 2099 unsigned AltIdx = findFirstSet((ZMask | (1 << DstIdx)) ^ 15); 2100 assert(AltIdx < 4 && "Illegal insertion index"); 2101 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask; 2102 auto &WorkingMI = cloneIfNew(MI); 2103 WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(AltImm); 2104 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2105 OpIdx1, OpIdx2); 2106 } 2107 return nullptr; 2108 } 2109 case X86::MOVSDrr: 2110 case X86::MOVSSrr: 2111 case X86::VMOVSDrr: 2112 case X86::VMOVSSrr:{ 2113 // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD. 2114 if (Subtarget.hasSSE41()) { 2115 unsigned Mask, Opc; 2116 switch (MI.getOpcode()) { 2117 default: llvm_unreachable("Unreachable!"); 2118 case X86::MOVSDrr: Opc = X86::BLENDPDrri; Mask = 0x02; break; 2119 case X86::MOVSSrr: Opc = X86::BLENDPSrri; Mask = 0x0E; break; 2120 case X86::VMOVSDrr: Opc = X86::VBLENDPDrri; Mask = 0x02; break; 2121 case X86::VMOVSSrr: Opc = X86::VBLENDPSrri; Mask = 0x0E; break; 2122 } 2123 2124 auto &WorkingMI = cloneIfNew(MI); 2125 WorkingMI.setDesc(get(Opc)); 2126 WorkingMI.addOperand(MachineOperand::CreateImm(Mask)); 2127 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2128 OpIdx1, OpIdx2); 2129 } 2130 2131 // Convert to SHUFPD. 2132 assert(MI.getOpcode() == X86::MOVSDrr && 2133 "Can only commute MOVSDrr without SSE4.1"); 2134 2135 auto &WorkingMI = cloneIfNew(MI); 2136 WorkingMI.setDesc(get(X86::SHUFPDrri)); 2137 WorkingMI.addOperand(MachineOperand::CreateImm(0x02)); 2138 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2139 OpIdx1, OpIdx2); 2140 } 2141 case X86::SHUFPDrri: { 2142 // Commute to MOVSD. 2143 assert(MI.getOperand(3).getImm() == 0x02 && "Unexpected immediate!"); 2144 auto &WorkingMI = cloneIfNew(MI); 2145 WorkingMI.setDesc(get(X86::MOVSDrr)); 2146 WorkingMI.removeOperand(3); 2147 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2148 OpIdx1, OpIdx2); 2149 } 2150 case X86::PCLMULQDQrr: 2151 case X86::VPCLMULQDQrr: 2152 case X86::VPCLMULQDQYrr: 2153 case X86::VPCLMULQDQZrr: 2154 case X86::VPCLMULQDQZ128rr: 2155 case X86::VPCLMULQDQZ256rr: { 2156 // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0] 2157 // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0] 2158 unsigned Imm = MI.getOperand(3).getImm(); 2159 unsigned Src1Hi = Imm & 0x01; 2160 unsigned Src2Hi = Imm & 0x10; 2161 auto &WorkingMI = cloneIfNew(MI); 2162 WorkingMI.getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4)); 2163 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2164 OpIdx1, OpIdx2); 2165 } 2166 case X86::VPCMPBZ128rri: case X86::VPCMPUBZ128rri: 2167 case X86::VPCMPBZ256rri: case X86::VPCMPUBZ256rri: 2168 case X86::VPCMPBZrri: case X86::VPCMPUBZrri: 2169 case X86::VPCMPDZ128rri: case X86::VPCMPUDZ128rri: 2170 case X86::VPCMPDZ256rri: case X86::VPCMPUDZ256rri: 2171 case X86::VPCMPDZrri: case X86::VPCMPUDZrri: 2172 case X86::VPCMPQZ128rri: case X86::VPCMPUQZ128rri: 2173 case X86::VPCMPQZ256rri: case X86::VPCMPUQZ256rri: 2174 case X86::VPCMPQZrri: case X86::VPCMPUQZrri: 2175 case X86::VPCMPWZ128rri: case X86::VPCMPUWZ128rri: 2176 case X86::VPCMPWZ256rri: case X86::VPCMPUWZ256rri: 2177 case X86::VPCMPWZrri: case X86::VPCMPUWZrri: 2178 case X86::VPCMPBZ128rrik: case X86::VPCMPUBZ128rrik: 2179 case X86::VPCMPBZ256rrik: case X86::VPCMPUBZ256rrik: 2180 case X86::VPCMPBZrrik: case X86::VPCMPUBZrrik: 2181 case X86::VPCMPDZ128rrik: case X86::VPCMPUDZ128rrik: 2182 case X86::VPCMPDZ256rrik: case X86::VPCMPUDZ256rrik: 2183 case X86::VPCMPDZrrik: case X86::VPCMPUDZrrik: 2184 case X86::VPCMPQZ128rrik: case X86::VPCMPUQZ128rrik: 2185 case X86::VPCMPQZ256rrik: case X86::VPCMPUQZ256rrik: 2186 case X86::VPCMPQZrrik: case X86::VPCMPUQZrrik: 2187 case X86::VPCMPWZ128rrik: case X86::VPCMPUWZ128rrik: 2188 case X86::VPCMPWZ256rrik: case X86::VPCMPUWZ256rrik: 2189 case X86::VPCMPWZrrik: case X86::VPCMPUWZrrik: { 2190 // Flip comparison mode immediate (if necessary). 2191 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7; 2192 Imm = X86::getSwappedVPCMPImm(Imm); 2193 auto &WorkingMI = cloneIfNew(MI); 2194 WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(Imm); 2195 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2196 OpIdx1, OpIdx2); 2197 } 2198 case X86::VPCOMBri: case X86::VPCOMUBri: 2199 case X86::VPCOMDri: case X86::VPCOMUDri: 2200 case X86::VPCOMQri: case X86::VPCOMUQri: 2201 case X86::VPCOMWri: case X86::VPCOMUWri: { 2202 // Flip comparison mode immediate (if necessary). 2203 unsigned Imm = MI.getOperand(3).getImm() & 0x7; 2204 Imm = X86::getSwappedVPCOMImm(Imm); 2205 auto &WorkingMI = cloneIfNew(MI); 2206 WorkingMI.getOperand(3).setImm(Imm); 2207 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2208 OpIdx1, OpIdx2); 2209 } 2210 case X86::VCMPSDZrr: 2211 case X86::VCMPSSZrr: 2212 case X86::VCMPPDZrri: 2213 case X86::VCMPPSZrri: 2214 case X86::VCMPSHZrr: 2215 case X86::VCMPPHZrri: 2216 case X86::VCMPPHZ128rri: 2217 case X86::VCMPPHZ256rri: 2218 case X86::VCMPPDZ128rri: 2219 case X86::VCMPPSZ128rri: 2220 case X86::VCMPPDZ256rri: 2221 case X86::VCMPPSZ256rri: 2222 case X86::VCMPPDZrrik: 2223 case X86::VCMPPSZrrik: 2224 case X86::VCMPPDZ128rrik: 2225 case X86::VCMPPSZ128rrik: 2226 case X86::VCMPPDZ256rrik: 2227 case X86::VCMPPSZ256rrik: { 2228 unsigned Imm = 2229 MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 0x1f; 2230 Imm = X86::getSwappedVCMPImm(Imm); 2231 auto &WorkingMI = cloneIfNew(MI); 2232 WorkingMI.getOperand(MI.getNumExplicitOperands() - 1).setImm(Imm); 2233 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2234 OpIdx1, OpIdx2); 2235 } 2236 case X86::VPERM2F128rr: 2237 case X86::VPERM2I128rr: { 2238 // Flip permute source immediate. 2239 // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi. 2240 // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi. 2241 int8_t Imm = MI.getOperand(3).getImm() & 0xFF; 2242 auto &WorkingMI = cloneIfNew(MI); 2243 WorkingMI.getOperand(3).setImm(Imm ^ 0x22); 2244 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2245 OpIdx1, OpIdx2); 2246 } 2247 case X86::MOVHLPSrr: 2248 case X86::UNPCKHPDrr: 2249 case X86::VMOVHLPSrr: 2250 case X86::VUNPCKHPDrr: 2251 case X86::VMOVHLPSZrr: 2252 case X86::VUNPCKHPDZ128rr: { 2253 assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!"); 2254 2255 unsigned Opc = MI.getOpcode(); 2256 switch (Opc) { 2257 default: llvm_unreachable("Unreachable!"); 2258 case X86::MOVHLPSrr: Opc = X86::UNPCKHPDrr; break; 2259 case X86::UNPCKHPDrr: Opc = X86::MOVHLPSrr; break; 2260 case X86::VMOVHLPSrr: Opc = X86::VUNPCKHPDrr; break; 2261 case X86::VUNPCKHPDrr: Opc = X86::VMOVHLPSrr; break; 2262 case X86::VMOVHLPSZrr: Opc = X86::VUNPCKHPDZ128rr; break; 2263 case X86::VUNPCKHPDZ128rr: Opc = X86::VMOVHLPSZrr; break; 2264 } 2265 auto &WorkingMI = cloneIfNew(MI); 2266 WorkingMI.setDesc(get(Opc)); 2267 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2268 OpIdx1, OpIdx2); 2269 } 2270 case X86::CMOV16rr: case X86::CMOV32rr: case X86::CMOV64rr: { 2271 auto &WorkingMI = cloneIfNew(MI); 2272 unsigned OpNo = MI.getDesc().getNumOperands() - 1; 2273 X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm()); 2274 WorkingMI.getOperand(OpNo).setImm(X86::GetOppositeBranchCondition(CC)); 2275 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2276 OpIdx1, OpIdx2); 2277 } 2278 case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: 2279 case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: 2280 case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: 2281 case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: 2282 case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: 2283 case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: 2284 case X86::VPTERNLOGDZrrik: 2285 case X86::VPTERNLOGDZ128rrik: 2286 case X86::VPTERNLOGDZ256rrik: 2287 case X86::VPTERNLOGQZrrik: 2288 case X86::VPTERNLOGQZ128rrik: 2289 case X86::VPTERNLOGQZ256rrik: 2290 case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: 2291 case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: 2292 case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: 2293 case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: 2294 case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: 2295 case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: 2296 case X86::VPTERNLOGDZ128rmbi: 2297 case X86::VPTERNLOGDZ256rmbi: 2298 case X86::VPTERNLOGDZrmbi: 2299 case X86::VPTERNLOGQZ128rmbi: 2300 case X86::VPTERNLOGQZ256rmbi: 2301 case X86::VPTERNLOGQZrmbi: 2302 case X86::VPTERNLOGDZ128rmbikz: 2303 case X86::VPTERNLOGDZ256rmbikz: 2304 case X86::VPTERNLOGDZrmbikz: 2305 case X86::VPTERNLOGQZ128rmbikz: 2306 case X86::VPTERNLOGQZ256rmbikz: 2307 case X86::VPTERNLOGQZrmbikz: { 2308 auto &WorkingMI = cloneIfNew(MI); 2309 commuteVPTERNLOG(WorkingMI, OpIdx1, OpIdx2); 2310 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2311 OpIdx1, OpIdx2); 2312 } 2313 default: { 2314 if (isCommutableVPERMV3Instruction(MI.getOpcode())) { 2315 unsigned Opc = getCommutedVPERMV3Opcode(MI.getOpcode()); 2316 auto &WorkingMI = cloneIfNew(MI); 2317 WorkingMI.setDesc(get(Opc)); 2318 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2319 OpIdx1, OpIdx2); 2320 } 2321 2322 const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), 2323 MI.getDesc().TSFlags); 2324 if (FMA3Group) { 2325 unsigned Opc = 2326 getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group); 2327 auto &WorkingMI = cloneIfNew(MI); 2328 WorkingMI.setDesc(get(Opc)); 2329 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2330 OpIdx1, OpIdx2); 2331 } 2332 2333 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 2334 } 2335 } 2336 } 2337 2338 bool 2339 X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI, 2340 unsigned &SrcOpIdx1, 2341 unsigned &SrcOpIdx2, 2342 bool IsIntrinsic) const { 2343 uint64_t TSFlags = MI.getDesc().TSFlags; 2344 2345 unsigned FirstCommutableVecOp = 1; 2346 unsigned LastCommutableVecOp = 3; 2347 unsigned KMaskOp = -1U; 2348 if (X86II::isKMasked(TSFlags)) { 2349 // For k-zero-masked operations it is Ok to commute the first vector 2350 // operand. Unless this is an intrinsic instruction. 2351 // For regular k-masked operations a conservative choice is done as the 2352 // elements of the first vector operand, for which the corresponding bit 2353 // in the k-mask operand is set to 0, are copied to the result of the 2354 // instruction. 2355 // TODO/FIXME: The commute still may be legal if it is known that the 2356 // k-mask operand is set to either all ones or all zeroes. 2357 // It is also Ok to commute the 1st operand if all users of MI use only 2358 // the elements enabled by the k-mask operand. For example, 2359 // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i] 2360 // : v1[i]; 2361 // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 -> 2362 // // Ok, to commute v1 in FMADD213PSZrk. 2363 2364 // The k-mask operand has index = 2 for masked and zero-masked operations. 2365 KMaskOp = 2; 2366 2367 // The operand with index = 1 is used as a source for those elements for 2368 // which the corresponding bit in the k-mask is set to 0. 2369 if (X86II::isKMergeMasked(TSFlags) || IsIntrinsic) 2370 FirstCommutableVecOp = 3; 2371 2372 LastCommutableVecOp++; 2373 } else if (IsIntrinsic) { 2374 // Commuting the first operand of an intrinsic instruction isn't possible 2375 // unless we can prove that only the lowest element of the result is used. 2376 FirstCommutableVecOp = 2; 2377 } 2378 2379 if (isMem(MI, LastCommutableVecOp)) 2380 LastCommutableVecOp--; 2381 2382 // Only the first RegOpsNum operands are commutable. 2383 // Also, the value 'CommuteAnyOperandIndex' is valid here as it means 2384 // that the operand is not specified/fixed. 2385 if (SrcOpIdx1 != CommuteAnyOperandIndex && 2386 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp || 2387 SrcOpIdx1 == KMaskOp)) 2388 return false; 2389 if (SrcOpIdx2 != CommuteAnyOperandIndex && 2390 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp || 2391 SrcOpIdx2 == KMaskOp)) 2392 return false; 2393 2394 // Look for two different register operands assumed to be commutable 2395 // regardless of the FMA opcode. The FMA opcode is adjusted later. 2396 if (SrcOpIdx1 == CommuteAnyOperandIndex || 2397 SrcOpIdx2 == CommuteAnyOperandIndex) { 2398 unsigned CommutableOpIdx2 = SrcOpIdx2; 2399 2400 // At least one of operands to be commuted is not specified and 2401 // this method is free to choose appropriate commutable operands. 2402 if (SrcOpIdx1 == SrcOpIdx2) 2403 // Both of operands are not fixed. By default set one of commutable 2404 // operands to the last register operand of the instruction. 2405 CommutableOpIdx2 = LastCommutableVecOp; 2406 else if (SrcOpIdx2 == CommuteAnyOperandIndex) 2407 // Only one of operands is not fixed. 2408 CommutableOpIdx2 = SrcOpIdx1; 2409 2410 // CommutableOpIdx2 is well defined now. Let's choose another commutable 2411 // operand and assign its index to CommutableOpIdx1. 2412 Register Op2Reg = MI.getOperand(CommutableOpIdx2).getReg(); 2413 2414 unsigned CommutableOpIdx1; 2415 for (CommutableOpIdx1 = LastCommutableVecOp; 2416 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) { 2417 // Just ignore and skip the k-mask operand. 2418 if (CommutableOpIdx1 == KMaskOp) 2419 continue; 2420 2421 // The commuted operands must have different registers. 2422 // Otherwise, the commute transformation does not change anything and 2423 // is useless then. 2424 if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg()) 2425 break; 2426 } 2427 2428 // No appropriate commutable operands were found. 2429 if (CommutableOpIdx1 < FirstCommutableVecOp) 2430 return false; 2431 2432 // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2 2433 // to return those values. 2434 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2435 CommutableOpIdx1, CommutableOpIdx2)) 2436 return false; 2437 } 2438 2439 return true; 2440 } 2441 2442 bool X86InstrInfo::findCommutedOpIndices(const MachineInstr &MI, 2443 unsigned &SrcOpIdx1, 2444 unsigned &SrcOpIdx2) const { 2445 const MCInstrDesc &Desc = MI.getDesc(); 2446 if (!Desc.isCommutable()) 2447 return false; 2448 2449 switch (MI.getOpcode()) { 2450 case X86::CMPSDrr: 2451 case X86::CMPSSrr: 2452 case X86::CMPPDrri: 2453 case X86::CMPPSrri: 2454 case X86::VCMPSDrr: 2455 case X86::VCMPSSrr: 2456 case X86::VCMPPDrri: 2457 case X86::VCMPPSrri: 2458 case X86::VCMPPDYrri: 2459 case X86::VCMPPSYrri: 2460 case X86::VCMPSDZrr: 2461 case X86::VCMPSSZrr: 2462 case X86::VCMPPDZrri: 2463 case X86::VCMPPSZrri: 2464 case X86::VCMPSHZrr: 2465 case X86::VCMPPHZrri: 2466 case X86::VCMPPHZ128rri: 2467 case X86::VCMPPHZ256rri: 2468 case X86::VCMPPDZ128rri: 2469 case X86::VCMPPSZ128rri: 2470 case X86::VCMPPDZ256rri: 2471 case X86::VCMPPSZ256rri: 2472 case X86::VCMPPDZrrik: 2473 case X86::VCMPPSZrrik: 2474 case X86::VCMPPDZ128rrik: 2475 case X86::VCMPPSZ128rrik: 2476 case X86::VCMPPDZ256rrik: 2477 case X86::VCMPPSZ256rrik: { 2478 unsigned OpOffset = X86II::isKMasked(Desc.TSFlags) ? 1 : 0; 2479 2480 // Float comparison can be safely commuted for 2481 // Ordered/Unordered/Equal/NotEqual tests 2482 unsigned Imm = MI.getOperand(3 + OpOffset).getImm() & 0x7; 2483 switch (Imm) { 2484 default: 2485 // EVEX versions can be commuted. 2486 if ((Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX) 2487 break; 2488 return false; 2489 case 0x00: // EQUAL 2490 case 0x03: // UNORDERED 2491 case 0x04: // NOT EQUAL 2492 case 0x07: // ORDERED 2493 break; 2494 } 2495 2496 // The indices of the commutable operands are 1 and 2 (or 2 and 3 2497 // when masked). 2498 // Assign them to the returned operand indices here. 2499 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset, 2500 2 + OpOffset); 2501 } 2502 case X86::MOVSSrr: 2503 // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can 2504 // form sse4.1 blend. We assume VMOVSSrr/VMOVSDrr is always commutable since 2505 // AVX implies sse4.1. 2506 if (Subtarget.hasSSE41()) 2507 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2508 return false; 2509 case X86::SHUFPDrri: 2510 // We can commute this to MOVSD. 2511 if (MI.getOperand(3).getImm() == 0x02) 2512 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2513 return false; 2514 case X86::MOVHLPSrr: 2515 case X86::UNPCKHPDrr: 2516 case X86::VMOVHLPSrr: 2517 case X86::VUNPCKHPDrr: 2518 case X86::VMOVHLPSZrr: 2519 case X86::VUNPCKHPDZ128rr: 2520 if (Subtarget.hasSSE2()) 2521 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2522 return false; 2523 case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: 2524 case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: 2525 case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: 2526 case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: 2527 case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: 2528 case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: 2529 case X86::VPTERNLOGDZrrik: 2530 case X86::VPTERNLOGDZ128rrik: 2531 case X86::VPTERNLOGDZ256rrik: 2532 case X86::VPTERNLOGQZrrik: 2533 case X86::VPTERNLOGQZ128rrik: 2534 case X86::VPTERNLOGQZ256rrik: 2535 case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: 2536 case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: 2537 case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: 2538 case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: 2539 case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: 2540 case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: 2541 case X86::VPTERNLOGDZ128rmbi: 2542 case X86::VPTERNLOGDZ256rmbi: 2543 case X86::VPTERNLOGDZrmbi: 2544 case X86::VPTERNLOGQZ128rmbi: 2545 case X86::VPTERNLOGQZ256rmbi: 2546 case X86::VPTERNLOGQZrmbi: 2547 case X86::VPTERNLOGDZ128rmbikz: 2548 case X86::VPTERNLOGDZ256rmbikz: 2549 case X86::VPTERNLOGDZrmbikz: 2550 case X86::VPTERNLOGQZ128rmbikz: 2551 case X86::VPTERNLOGQZ256rmbikz: 2552 case X86::VPTERNLOGQZrmbikz: 2553 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2554 case X86::VPDPWSSDYrr: 2555 case X86::VPDPWSSDrr: 2556 case X86::VPDPWSSDSYrr: 2557 case X86::VPDPWSSDSrr: 2558 case X86::VPDPBSSDSrr: 2559 case X86::VPDPBSSDSYrr: 2560 case X86::VPDPBSSDrr: 2561 case X86::VPDPBSSDYrr: 2562 case X86::VPDPBUUDSrr: 2563 case X86::VPDPBUUDSYrr: 2564 case X86::VPDPBUUDrr: 2565 case X86::VPDPBUUDYrr: 2566 case X86::VPDPWSSDZ128r: 2567 case X86::VPDPWSSDZ128rk: 2568 case X86::VPDPWSSDZ128rkz: 2569 case X86::VPDPWSSDZ256r: 2570 case X86::VPDPWSSDZ256rk: 2571 case X86::VPDPWSSDZ256rkz: 2572 case X86::VPDPWSSDZr: 2573 case X86::VPDPWSSDZrk: 2574 case X86::VPDPWSSDZrkz: 2575 case X86::VPDPWSSDSZ128r: 2576 case X86::VPDPWSSDSZ128rk: 2577 case X86::VPDPWSSDSZ128rkz: 2578 case X86::VPDPWSSDSZ256r: 2579 case X86::VPDPWSSDSZ256rk: 2580 case X86::VPDPWSSDSZ256rkz: 2581 case X86::VPDPWSSDSZr: 2582 case X86::VPDPWSSDSZrk: 2583 case X86::VPDPWSSDSZrkz: 2584 case X86::VPMADD52HUQrr: 2585 case X86::VPMADD52HUQYrr: 2586 case X86::VPMADD52HUQZ128r: 2587 case X86::VPMADD52HUQZ128rk: 2588 case X86::VPMADD52HUQZ128rkz: 2589 case X86::VPMADD52HUQZ256r: 2590 case X86::VPMADD52HUQZ256rk: 2591 case X86::VPMADD52HUQZ256rkz: 2592 case X86::VPMADD52HUQZr: 2593 case X86::VPMADD52HUQZrk: 2594 case X86::VPMADD52HUQZrkz: 2595 case X86::VPMADD52LUQrr: 2596 case X86::VPMADD52LUQYrr: 2597 case X86::VPMADD52LUQZ128r: 2598 case X86::VPMADD52LUQZ128rk: 2599 case X86::VPMADD52LUQZ128rkz: 2600 case X86::VPMADD52LUQZ256r: 2601 case X86::VPMADD52LUQZ256rk: 2602 case X86::VPMADD52LUQZ256rkz: 2603 case X86::VPMADD52LUQZr: 2604 case X86::VPMADD52LUQZrk: 2605 case X86::VPMADD52LUQZrkz: 2606 case X86::VFMADDCPHZr: 2607 case X86::VFMADDCPHZrk: 2608 case X86::VFMADDCPHZrkz: 2609 case X86::VFMADDCPHZ128r: 2610 case X86::VFMADDCPHZ128rk: 2611 case X86::VFMADDCPHZ128rkz: 2612 case X86::VFMADDCPHZ256r: 2613 case X86::VFMADDCPHZ256rk: 2614 case X86::VFMADDCPHZ256rkz: 2615 case X86::VFMADDCSHZr: 2616 case X86::VFMADDCSHZrk: 2617 case X86::VFMADDCSHZrkz: { 2618 unsigned CommutableOpIdx1 = 2; 2619 unsigned CommutableOpIdx2 = 3; 2620 if (X86II::isKMasked(Desc.TSFlags)) { 2621 // Skip the mask register. 2622 ++CommutableOpIdx1; 2623 ++CommutableOpIdx2; 2624 } 2625 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2626 CommutableOpIdx1, CommutableOpIdx2)) 2627 return false; 2628 if (!MI.getOperand(SrcOpIdx1).isReg() || 2629 !MI.getOperand(SrcOpIdx2).isReg()) 2630 // No idea. 2631 return false; 2632 return true; 2633 } 2634 2635 default: 2636 const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), 2637 MI.getDesc().TSFlags); 2638 if (FMA3Group) 2639 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2, 2640 FMA3Group->isIntrinsic()); 2641 2642 // Handled masked instructions since we need to skip over the mask input 2643 // and the preserved input. 2644 if (X86II::isKMasked(Desc.TSFlags)) { 2645 // First assume that the first input is the mask operand and skip past it. 2646 unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1; 2647 unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2; 2648 // Check if the first input is tied. If there isn't one then we only 2649 // need to skip the mask operand which we did above. 2650 if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(), 2651 MCOI::TIED_TO) != -1)) { 2652 // If this is zero masking instruction with a tied operand, we need to 2653 // move the first index back to the first input since this must 2654 // be a 3 input instruction and we want the first two non-mask inputs. 2655 // Otherwise this is a 2 input instruction with a preserved input and 2656 // mask, so we need to move the indices to skip one more input. 2657 if (X86II::isKMergeMasked(Desc.TSFlags)) { 2658 ++CommutableOpIdx1; 2659 ++CommutableOpIdx2; 2660 } else { 2661 --CommutableOpIdx1; 2662 } 2663 } 2664 2665 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2666 CommutableOpIdx1, CommutableOpIdx2)) 2667 return false; 2668 2669 if (!MI.getOperand(SrcOpIdx1).isReg() || 2670 !MI.getOperand(SrcOpIdx2).isReg()) 2671 // No idea. 2672 return false; 2673 return true; 2674 } 2675 2676 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2677 } 2678 return false; 2679 } 2680 2681 static bool isConvertibleLEA(MachineInstr *MI) { 2682 unsigned Opcode = MI->getOpcode(); 2683 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r && 2684 Opcode != X86::LEA64_32r) 2685 return false; 2686 2687 const MachineOperand &Scale = MI->getOperand(1 + X86::AddrScaleAmt); 2688 const MachineOperand &Disp = MI->getOperand(1 + X86::AddrDisp); 2689 const MachineOperand &Segment = MI->getOperand(1 + X86::AddrSegmentReg); 2690 2691 if (Segment.getReg() != 0 || !Disp.isImm() || Disp.getImm() != 0 || 2692 Scale.getImm() > 1) 2693 return false; 2694 2695 return true; 2696 } 2697 2698 bool X86InstrInfo::hasCommutePreference(MachineInstr &MI, bool &Commute) const { 2699 // Currently we're interested in following sequence only. 2700 // r3 = lea r1, r2 2701 // r5 = add r3, r4 2702 // Both r3 and r4 are killed in add, we hope the add instruction has the 2703 // operand order 2704 // r5 = add r4, r3 2705 // So later in X86FixupLEAs the lea instruction can be rewritten as add. 2706 unsigned Opcode = MI.getOpcode(); 2707 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr) 2708 return false; 2709 2710 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 2711 Register Reg1 = MI.getOperand(1).getReg(); 2712 Register Reg2 = MI.getOperand(2).getReg(); 2713 2714 // Check if Reg1 comes from LEA in the same MBB. 2715 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg1)) { 2716 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) { 2717 Commute = true; 2718 return true; 2719 } 2720 } 2721 2722 // Check if Reg2 comes from LEA in the same MBB. 2723 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg2)) { 2724 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) { 2725 Commute = false; 2726 return true; 2727 } 2728 } 2729 2730 return false; 2731 } 2732 2733 int X86::getCondSrcNoFromDesc(const MCInstrDesc &MCID) { 2734 unsigned Opcode = MCID.getOpcode(); 2735 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isCMOVCC(Opcode))) 2736 return -1; 2737 // Assume that condition code is always the last use operand. 2738 unsigned NumUses = MCID.getNumOperands() - MCID.getNumDefs(); 2739 return NumUses - 1; 2740 } 2741 2742 X86::CondCode X86::getCondFromMI(const MachineInstr &MI) { 2743 const MCInstrDesc &MCID = MI.getDesc(); 2744 int CondNo = getCondSrcNoFromDesc(MCID); 2745 if (CondNo < 0) 2746 return X86::COND_INVALID; 2747 CondNo += MCID.getNumDefs(); 2748 return static_cast<X86::CondCode>(MI.getOperand(CondNo).getImm()); 2749 } 2750 2751 X86::CondCode X86::getCondFromBranch(const MachineInstr &MI) { 2752 return X86::isJCC(MI.getOpcode()) ? X86::getCondFromMI(MI) 2753 : X86::COND_INVALID; 2754 } 2755 2756 X86::CondCode X86::getCondFromSETCC(const MachineInstr &MI) { 2757 return X86::isSETCC(MI.getOpcode()) ? X86::getCondFromMI(MI) 2758 : X86::COND_INVALID; 2759 } 2760 2761 X86::CondCode X86::getCondFromCMov(const MachineInstr &MI) { 2762 return X86::isCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI) 2763 : X86::COND_INVALID; 2764 } 2765 2766 /// Return the inverse of the specified condition, 2767 /// e.g. turning COND_E to COND_NE. 2768 X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) { 2769 switch (CC) { 2770 default: llvm_unreachable("Illegal condition code!"); 2771 case X86::COND_E: return X86::COND_NE; 2772 case X86::COND_NE: return X86::COND_E; 2773 case X86::COND_L: return X86::COND_GE; 2774 case X86::COND_LE: return X86::COND_G; 2775 case X86::COND_G: return X86::COND_LE; 2776 case X86::COND_GE: return X86::COND_L; 2777 case X86::COND_B: return X86::COND_AE; 2778 case X86::COND_BE: return X86::COND_A; 2779 case X86::COND_A: return X86::COND_BE; 2780 case X86::COND_AE: return X86::COND_B; 2781 case X86::COND_S: return X86::COND_NS; 2782 case X86::COND_NS: return X86::COND_S; 2783 case X86::COND_P: return X86::COND_NP; 2784 case X86::COND_NP: return X86::COND_P; 2785 case X86::COND_O: return X86::COND_NO; 2786 case X86::COND_NO: return X86::COND_O; 2787 case X86::COND_NE_OR_P: return X86::COND_E_AND_NP; 2788 case X86::COND_E_AND_NP: return X86::COND_NE_OR_P; 2789 } 2790 } 2791 2792 /// Assuming the flags are set by MI(a,b), return the condition code if we 2793 /// modify the instructions such that flags are set by MI(b,a). 2794 static X86::CondCode getSwappedCondition(X86::CondCode CC) { 2795 switch (CC) { 2796 default: return X86::COND_INVALID; 2797 case X86::COND_E: return X86::COND_E; 2798 case X86::COND_NE: return X86::COND_NE; 2799 case X86::COND_L: return X86::COND_G; 2800 case X86::COND_LE: return X86::COND_GE; 2801 case X86::COND_G: return X86::COND_L; 2802 case X86::COND_GE: return X86::COND_LE; 2803 case X86::COND_B: return X86::COND_A; 2804 case X86::COND_BE: return X86::COND_AE; 2805 case X86::COND_A: return X86::COND_B; 2806 case X86::COND_AE: return X86::COND_BE; 2807 } 2808 } 2809 2810 std::pair<X86::CondCode, bool> 2811 X86::getX86ConditionCode(CmpInst::Predicate Predicate) { 2812 X86::CondCode CC = X86::COND_INVALID; 2813 bool NeedSwap = false; 2814 switch (Predicate) { 2815 default: break; 2816 // Floating-point Predicates 2817 case CmpInst::FCMP_UEQ: CC = X86::COND_E; break; 2818 case CmpInst::FCMP_OLT: NeedSwap = true; [[fallthrough]]; 2819 case CmpInst::FCMP_OGT: CC = X86::COND_A; break; 2820 case CmpInst::FCMP_OLE: NeedSwap = true; [[fallthrough]]; 2821 case CmpInst::FCMP_OGE: CC = X86::COND_AE; break; 2822 case CmpInst::FCMP_UGT: NeedSwap = true; [[fallthrough]]; 2823 case CmpInst::FCMP_ULT: CC = X86::COND_B; break; 2824 case CmpInst::FCMP_UGE: NeedSwap = true; [[fallthrough]]; 2825 case CmpInst::FCMP_ULE: CC = X86::COND_BE; break; 2826 case CmpInst::FCMP_ONE: CC = X86::COND_NE; break; 2827 case CmpInst::FCMP_UNO: CC = X86::COND_P; break; 2828 case CmpInst::FCMP_ORD: CC = X86::COND_NP; break; 2829 case CmpInst::FCMP_OEQ: [[fallthrough]]; 2830 case CmpInst::FCMP_UNE: CC = X86::COND_INVALID; break; 2831 2832 // Integer Predicates 2833 case CmpInst::ICMP_EQ: CC = X86::COND_E; break; 2834 case CmpInst::ICMP_NE: CC = X86::COND_NE; break; 2835 case CmpInst::ICMP_UGT: CC = X86::COND_A; break; 2836 case CmpInst::ICMP_UGE: CC = X86::COND_AE; break; 2837 case CmpInst::ICMP_ULT: CC = X86::COND_B; break; 2838 case CmpInst::ICMP_ULE: CC = X86::COND_BE; break; 2839 case CmpInst::ICMP_SGT: CC = X86::COND_G; break; 2840 case CmpInst::ICMP_SGE: CC = X86::COND_GE; break; 2841 case CmpInst::ICMP_SLT: CC = X86::COND_L; break; 2842 case CmpInst::ICMP_SLE: CC = X86::COND_LE; break; 2843 } 2844 2845 return std::make_pair(CC, NeedSwap); 2846 } 2847 2848 /// Return a cmov opcode for the given register size in bytes, and operand type. 2849 unsigned X86::getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand) { 2850 switch(RegBytes) { 2851 default: llvm_unreachable("Illegal register size!"); 2852 case 2: return HasMemoryOperand ? X86::CMOV16rm : X86::CMOV16rr; 2853 case 4: return HasMemoryOperand ? X86::CMOV32rm : X86::CMOV32rr; 2854 case 8: return HasMemoryOperand ? X86::CMOV64rm : X86::CMOV64rr; 2855 } 2856 } 2857 2858 /// Get the VPCMP immediate for the given condition. 2859 unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) { 2860 switch (CC) { 2861 default: llvm_unreachable("Unexpected SETCC condition"); 2862 case ISD::SETNE: return 4; 2863 case ISD::SETEQ: return 0; 2864 case ISD::SETULT: 2865 case ISD::SETLT: return 1; 2866 case ISD::SETUGT: 2867 case ISD::SETGT: return 6; 2868 case ISD::SETUGE: 2869 case ISD::SETGE: return 5; 2870 case ISD::SETULE: 2871 case ISD::SETLE: return 2; 2872 } 2873 } 2874 2875 /// Get the VPCMP immediate if the operands are swapped. 2876 unsigned X86::getSwappedVPCMPImm(unsigned Imm) { 2877 switch (Imm) { 2878 default: llvm_unreachable("Unreachable!"); 2879 case 0x01: Imm = 0x06; break; // LT -> NLE 2880 case 0x02: Imm = 0x05; break; // LE -> NLT 2881 case 0x05: Imm = 0x02; break; // NLT -> LE 2882 case 0x06: Imm = 0x01; break; // NLE -> LT 2883 case 0x00: // EQ 2884 case 0x03: // FALSE 2885 case 0x04: // NE 2886 case 0x07: // TRUE 2887 break; 2888 } 2889 2890 return Imm; 2891 } 2892 2893 /// Get the VPCOM immediate if the operands are swapped. 2894 unsigned X86::getSwappedVPCOMImm(unsigned Imm) { 2895 switch (Imm) { 2896 default: llvm_unreachable("Unreachable!"); 2897 case 0x00: Imm = 0x02; break; // LT -> GT 2898 case 0x01: Imm = 0x03; break; // LE -> GE 2899 case 0x02: Imm = 0x00; break; // GT -> LT 2900 case 0x03: Imm = 0x01; break; // GE -> LE 2901 case 0x04: // EQ 2902 case 0x05: // NE 2903 case 0x06: // FALSE 2904 case 0x07: // TRUE 2905 break; 2906 } 2907 2908 return Imm; 2909 } 2910 2911 /// Get the VCMP immediate if the operands are swapped. 2912 unsigned X86::getSwappedVCMPImm(unsigned Imm) { 2913 // Only need the lower 2 bits to distinquish. 2914 switch (Imm & 0x3) { 2915 default: llvm_unreachable("Unreachable!"); 2916 case 0x00: case 0x03: 2917 // EQ/NE/TRUE/FALSE/ORD/UNORD don't change immediate when commuted. 2918 break; 2919 case 0x01: case 0x02: 2920 // Need to toggle bits 3:0. Bit 4 stays the same. 2921 Imm ^= 0xf; 2922 break; 2923 } 2924 2925 return Imm; 2926 } 2927 2928 /// Return true if the Reg is X87 register. 2929 static bool isX87Reg(unsigned Reg) { 2930 return (Reg == X86::FPCW || Reg == X86::FPSW || 2931 (Reg >= X86::ST0 && Reg <= X86::ST7)); 2932 } 2933 2934 /// check if the instruction is X87 instruction 2935 bool X86::isX87Instruction(MachineInstr &MI) { 2936 for (const MachineOperand &MO : MI.operands()) { 2937 if (!MO.isReg()) 2938 continue; 2939 if (isX87Reg(MO.getReg())) 2940 return true; 2941 } 2942 return false; 2943 } 2944 2945 bool X86InstrInfo::isUnconditionalTailCall(const MachineInstr &MI) const { 2946 switch (MI.getOpcode()) { 2947 case X86::TCRETURNdi: 2948 case X86::TCRETURNri: 2949 case X86::TCRETURNmi: 2950 case X86::TCRETURNdi64: 2951 case X86::TCRETURNri64: 2952 case X86::TCRETURNmi64: 2953 return true; 2954 default: 2955 return false; 2956 } 2957 } 2958 2959 bool X86InstrInfo::canMakeTailCallConditional( 2960 SmallVectorImpl<MachineOperand> &BranchCond, 2961 const MachineInstr &TailCall) const { 2962 2963 const MachineFunction *MF = TailCall.getMF(); 2964 2965 if (MF->getTarget().getCodeModel() == CodeModel::Kernel) { 2966 // Kernel patches thunk calls in runtime, these should never be conditional. 2967 const MachineOperand &Target = TailCall.getOperand(0); 2968 if (Target.isSymbol()) { 2969 StringRef Symbol(Target.getSymbolName()); 2970 // this is currently only relevant to r11/kernel indirect thunk. 2971 if (Symbol.equals("__x86_indirect_thunk_r11")) 2972 return false; 2973 } 2974 } 2975 2976 if (TailCall.getOpcode() != X86::TCRETURNdi && 2977 TailCall.getOpcode() != X86::TCRETURNdi64) { 2978 // Only direct calls can be done with a conditional branch. 2979 return false; 2980 } 2981 2982 if (Subtarget.isTargetWin64() && MF->hasWinCFI()) { 2983 // Conditional tail calls confuse the Win64 unwinder. 2984 return false; 2985 } 2986 2987 assert(BranchCond.size() == 1); 2988 if (BranchCond[0].getImm() > X86::LAST_VALID_COND) { 2989 // Can't make a conditional tail call with this condition. 2990 return false; 2991 } 2992 2993 const X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); 2994 if (X86FI->getTCReturnAddrDelta() != 0 || 2995 TailCall.getOperand(1).getImm() != 0) { 2996 // A conditional tail call cannot do any stack adjustment. 2997 return false; 2998 } 2999 3000 return true; 3001 } 3002 3003 void X86InstrInfo::replaceBranchWithTailCall( 3004 MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &BranchCond, 3005 const MachineInstr &TailCall) const { 3006 assert(canMakeTailCallConditional(BranchCond, TailCall)); 3007 3008 MachineBasicBlock::iterator I = MBB.end(); 3009 while (I != MBB.begin()) { 3010 --I; 3011 if (I->isDebugInstr()) 3012 continue; 3013 if (!I->isBranch()) 3014 assert(0 && "Can't find the branch to replace!"); 3015 3016 X86::CondCode CC = X86::getCondFromBranch(*I); 3017 assert(BranchCond.size() == 1); 3018 if (CC != BranchCond[0].getImm()) 3019 continue; 3020 3021 break; 3022 } 3023 3024 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc 3025 : X86::TCRETURNdi64cc; 3026 3027 auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc)); 3028 MIB->addOperand(TailCall.getOperand(0)); // Destination. 3029 MIB.addImm(0); // Stack offset (not used). 3030 MIB->addOperand(BranchCond[0]); // Condition. 3031 MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters. 3032 3033 // Add implicit uses and defs of all live regs potentially clobbered by the 3034 // call. This way they still appear live across the call. 3035 LivePhysRegs LiveRegs(getRegisterInfo()); 3036 LiveRegs.addLiveOuts(MBB); 3037 SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers; 3038 LiveRegs.stepForward(*MIB, Clobbers); 3039 for (const auto &C : Clobbers) { 3040 MIB.addReg(C.first, RegState::Implicit); 3041 MIB.addReg(C.first, RegState::Implicit | RegState::Define); 3042 } 3043 3044 I->eraseFromParent(); 3045 } 3046 3047 // Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may 3048 // not be a fallthrough MBB now due to layout changes). Return nullptr if the 3049 // fallthrough MBB cannot be identified. 3050 static MachineBasicBlock *getFallThroughMBB(MachineBasicBlock *MBB, 3051 MachineBasicBlock *TBB) { 3052 // Look for non-EHPad successors other than TBB. If we find exactly one, it 3053 // is the fallthrough MBB. If we find zero, then TBB is both the target MBB 3054 // and fallthrough MBB. If we find more than one, we cannot identify the 3055 // fallthrough MBB and should return nullptr. 3056 MachineBasicBlock *FallthroughBB = nullptr; 3057 for (MachineBasicBlock *Succ : MBB->successors()) { 3058 if (Succ->isEHPad() || (Succ == TBB && FallthroughBB)) 3059 continue; 3060 // Return a nullptr if we found more than one fallthrough successor. 3061 if (FallthroughBB && FallthroughBB != TBB) 3062 return nullptr; 3063 FallthroughBB = Succ; 3064 } 3065 return FallthroughBB; 3066 } 3067 3068 bool X86InstrInfo::AnalyzeBranchImpl( 3069 MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, 3070 SmallVectorImpl<MachineOperand> &Cond, 3071 SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const { 3072 3073 // Start from the bottom of the block and work up, examining the 3074 // terminator instructions. 3075 MachineBasicBlock::iterator I = MBB.end(); 3076 MachineBasicBlock::iterator UnCondBrIter = MBB.end(); 3077 while (I != MBB.begin()) { 3078 --I; 3079 if (I->isDebugInstr()) 3080 continue; 3081 3082 // Working from the bottom, when we see a non-terminator instruction, we're 3083 // done. 3084 if (!isUnpredicatedTerminator(*I)) 3085 break; 3086 3087 // A terminator that isn't a branch can't easily be handled by this 3088 // analysis. 3089 if (!I->isBranch()) 3090 return true; 3091 3092 // Handle unconditional branches. 3093 if (I->getOpcode() == X86::JMP_1) { 3094 UnCondBrIter = I; 3095 3096 if (!AllowModify) { 3097 TBB = I->getOperand(0).getMBB(); 3098 continue; 3099 } 3100 3101 // If the block has any instructions after a JMP, delete them. 3102 MBB.erase(std::next(I), MBB.end()); 3103 3104 Cond.clear(); 3105 FBB = nullptr; 3106 3107 // Delete the JMP if it's equivalent to a fall-through. 3108 if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) { 3109 TBB = nullptr; 3110 I->eraseFromParent(); 3111 I = MBB.end(); 3112 UnCondBrIter = MBB.end(); 3113 continue; 3114 } 3115 3116 // TBB is used to indicate the unconditional destination. 3117 TBB = I->getOperand(0).getMBB(); 3118 continue; 3119 } 3120 3121 // Handle conditional branches. 3122 X86::CondCode BranchCode = X86::getCondFromBranch(*I); 3123 if (BranchCode == X86::COND_INVALID) 3124 return true; // Can't handle indirect branch. 3125 3126 // In practice we should never have an undef eflags operand, if we do 3127 // abort here as we are not prepared to preserve the flag. 3128 if (I->findRegisterUseOperand(X86::EFLAGS)->isUndef()) 3129 return true; 3130 3131 // Working from the bottom, handle the first conditional branch. 3132 if (Cond.empty()) { 3133 FBB = TBB; 3134 TBB = I->getOperand(0).getMBB(); 3135 Cond.push_back(MachineOperand::CreateImm(BranchCode)); 3136 CondBranches.push_back(&*I); 3137 continue; 3138 } 3139 3140 // Handle subsequent conditional branches. Only handle the case where all 3141 // conditional branches branch to the same destination and their condition 3142 // opcodes fit one of the special multi-branch idioms. 3143 assert(Cond.size() == 1); 3144 assert(TBB); 3145 3146 // If the conditions are the same, we can leave them alone. 3147 X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm(); 3148 auto NewTBB = I->getOperand(0).getMBB(); 3149 if (OldBranchCode == BranchCode && TBB == NewTBB) 3150 continue; 3151 3152 // If they differ, see if they fit one of the known patterns. Theoretically, 3153 // we could handle more patterns here, but we shouldn't expect to see them 3154 // if instruction selection has done a reasonable job. 3155 if (TBB == NewTBB && 3156 ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) || 3157 (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) { 3158 BranchCode = X86::COND_NE_OR_P; 3159 } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) || 3160 (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) { 3161 if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB))) 3162 return true; 3163 3164 // X86::COND_E_AND_NP usually has two different branch destinations. 3165 // 3166 // JP B1 3167 // JE B2 3168 // JMP B1 3169 // B1: 3170 // B2: 3171 // 3172 // Here this condition branches to B2 only if NP && E. It has another 3173 // equivalent form: 3174 // 3175 // JNE B1 3176 // JNP B2 3177 // JMP B1 3178 // B1: 3179 // B2: 3180 // 3181 // Similarly it branches to B2 only if E && NP. That is why this condition 3182 // is named with COND_E_AND_NP. 3183 BranchCode = X86::COND_E_AND_NP; 3184 } else 3185 return true; 3186 3187 // Update the MachineOperand. 3188 Cond[0].setImm(BranchCode); 3189 CondBranches.push_back(&*I); 3190 } 3191 3192 return false; 3193 } 3194 3195 bool X86InstrInfo::analyzeBranch(MachineBasicBlock &MBB, 3196 MachineBasicBlock *&TBB, 3197 MachineBasicBlock *&FBB, 3198 SmallVectorImpl<MachineOperand> &Cond, 3199 bool AllowModify) const { 3200 SmallVector<MachineInstr *, 4> CondBranches; 3201 return AnalyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify); 3202 } 3203 3204 bool X86InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB, 3205 MachineBranchPredicate &MBP, 3206 bool AllowModify) const { 3207 using namespace std::placeholders; 3208 3209 SmallVector<MachineOperand, 4> Cond; 3210 SmallVector<MachineInstr *, 4> CondBranches; 3211 if (AnalyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches, 3212 AllowModify)) 3213 return true; 3214 3215 if (Cond.size() != 1) 3216 return true; 3217 3218 assert(MBP.TrueDest && "expected!"); 3219 3220 if (!MBP.FalseDest) 3221 MBP.FalseDest = MBB.getNextNode(); 3222 3223 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3224 3225 MachineInstr *ConditionDef = nullptr; 3226 bool SingleUseCondition = true; 3227 3228 for (MachineInstr &MI : llvm::drop_begin(llvm::reverse(MBB))) { 3229 if (MI.modifiesRegister(X86::EFLAGS, TRI)) { 3230 ConditionDef = &MI; 3231 break; 3232 } 3233 3234 if (MI.readsRegister(X86::EFLAGS, TRI)) 3235 SingleUseCondition = false; 3236 } 3237 3238 if (!ConditionDef) 3239 return true; 3240 3241 if (SingleUseCondition) { 3242 for (auto *Succ : MBB.successors()) 3243 if (Succ->isLiveIn(X86::EFLAGS)) 3244 SingleUseCondition = false; 3245 } 3246 3247 MBP.ConditionDef = ConditionDef; 3248 MBP.SingleUseCondition = SingleUseCondition; 3249 3250 // Currently we only recognize the simple pattern: 3251 // 3252 // test %reg, %reg 3253 // je %label 3254 // 3255 const unsigned TestOpcode = 3256 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr; 3257 3258 if (ConditionDef->getOpcode() == TestOpcode && 3259 ConditionDef->getNumOperands() == 3 && 3260 ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) && 3261 (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) { 3262 MBP.LHS = ConditionDef->getOperand(0); 3263 MBP.RHS = MachineOperand::CreateImm(0); 3264 MBP.Predicate = Cond[0].getImm() == X86::COND_NE 3265 ? MachineBranchPredicate::PRED_NE 3266 : MachineBranchPredicate::PRED_EQ; 3267 return false; 3268 } 3269 3270 return true; 3271 } 3272 3273 unsigned X86InstrInfo::removeBranch(MachineBasicBlock &MBB, 3274 int *BytesRemoved) const { 3275 assert(!BytesRemoved && "code size not handled"); 3276 3277 MachineBasicBlock::iterator I = MBB.end(); 3278 unsigned Count = 0; 3279 3280 while (I != MBB.begin()) { 3281 --I; 3282 if (I->isDebugInstr()) 3283 continue; 3284 if (I->getOpcode() != X86::JMP_1 && 3285 X86::getCondFromBranch(*I) == X86::COND_INVALID) 3286 break; 3287 // Remove the branch. 3288 I->eraseFromParent(); 3289 I = MBB.end(); 3290 ++Count; 3291 } 3292 3293 return Count; 3294 } 3295 3296 unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB, 3297 MachineBasicBlock *TBB, 3298 MachineBasicBlock *FBB, 3299 ArrayRef<MachineOperand> Cond, 3300 const DebugLoc &DL, 3301 int *BytesAdded) const { 3302 // Shouldn't be a fall through. 3303 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 3304 assert((Cond.size() == 1 || Cond.size() == 0) && 3305 "X86 branch conditions have one component!"); 3306 assert(!BytesAdded && "code size not handled"); 3307 3308 if (Cond.empty()) { 3309 // Unconditional branch? 3310 assert(!FBB && "Unconditional branch with multiple successors!"); 3311 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB); 3312 return 1; 3313 } 3314 3315 // If FBB is null, it is implied to be a fall-through block. 3316 bool FallThru = FBB == nullptr; 3317 3318 // Conditional branch. 3319 unsigned Count = 0; 3320 X86::CondCode CC = (X86::CondCode)Cond[0].getImm(); 3321 switch (CC) { 3322 case X86::COND_NE_OR_P: 3323 // Synthesize NE_OR_P with two branches. 3324 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NE); 3325 ++Count; 3326 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_P); 3327 ++Count; 3328 break; 3329 case X86::COND_E_AND_NP: 3330 // Use the next block of MBB as FBB if it is null. 3331 if (FBB == nullptr) { 3332 FBB = getFallThroughMBB(&MBB, TBB); 3333 assert(FBB && "MBB cannot be the last block in function when the false " 3334 "body is a fall-through."); 3335 } 3336 // Synthesize COND_E_AND_NP with two branches. 3337 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(FBB).addImm(X86::COND_NE); 3338 ++Count; 3339 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NP); 3340 ++Count; 3341 break; 3342 default: { 3343 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(CC); 3344 ++Count; 3345 } 3346 } 3347 if (!FallThru) { 3348 // Two-way Conditional branch. Insert the second branch. 3349 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB); 3350 ++Count; 3351 } 3352 return Count; 3353 } 3354 3355 bool X86InstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 3356 ArrayRef<MachineOperand> Cond, 3357 Register DstReg, Register TrueReg, 3358 Register FalseReg, int &CondCycles, 3359 int &TrueCycles, int &FalseCycles) const { 3360 // Not all subtargets have cmov instructions. 3361 if (!Subtarget.canUseCMOV()) 3362 return false; 3363 if (Cond.size() != 1) 3364 return false; 3365 // We cannot do the composite conditions, at least not in SSA form. 3366 if ((X86::CondCode)Cond[0].getImm() > X86::LAST_VALID_COND) 3367 return false; 3368 3369 // Check register classes. 3370 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3371 const TargetRegisterClass *RC = 3372 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 3373 if (!RC) 3374 return false; 3375 3376 // We have cmov instructions for 16, 32, and 64 bit general purpose registers. 3377 if (X86::GR16RegClass.hasSubClassEq(RC) || 3378 X86::GR32RegClass.hasSubClassEq(RC) || 3379 X86::GR64RegClass.hasSubClassEq(RC)) { 3380 // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy 3381 // Bridge. Probably Ivy Bridge as well. 3382 CondCycles = 2; 3383 TrueCycles = 2; 3384 FalseCycles = 2; 3385 return true; 3386 } 3387 3388 // Can't do vectors. 3389 return false; 3390 } 3391 3392 void X86InstrInfo::insertSelect(MachineBasicBlock &MBB, 3393 MachineBasicBlock::iterator I, 3394 const DebugLoc &DL, Register DstReg, 3395 ArrayRef<MachineOperand> Cond, Register TrueReg, 3396 Register FalseReg) const { 3397 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3398 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); 3399 const TargetRegisterClass &RC = *MRI.getRegClass(DstReg); 3400 assert(Cond.size() == 1 && "Invalid Cond array"); 3401 unsigned Opc = X86::getCMovOpcode(TRI.getRegSizeInBits(RC) / 8, 3402 false /*HasMemoryOperand*/); 3403 BuildMI(MBB, I, DL, get(Opc), DstReg) 3404 .addReg(FalseReg) 3405 .addReg(TrueReg) 3406 .addImm(Cond[0].getImm()); 3407 } 3408 3409 /// Test if the given register is a physical h register. 3410 static bool isHReg(unsigned Reg) { 3411 return X86::GR8_ABCD_HRegClass.contains(Reg); 3412 } 3413 3414 // Try and copy between VR128/VR64 and GR64 registers. 3415 static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg, 3416 const X86Subtarget &Subtarget) { 3417 bool HasAVX = Subtarget.hasAVX(); 3418 bool HasAVX512 = Subtarget.hasAVX512(); 3419 3420 // SrcReg(MaskReg) -> DestReg(GR64) 3421 // SrcReg(MaskReg) -> DestReg(GR32) 3422 3423 // All KMASK RegClasses hold the same k registers, can be tested against anyone. 3424 if (X86::VK16RegClass.contains(SrcReg)) { 3425 if (X86::GR64RegClass.contains(DestReg)) { 3426 assert(Subtarget.hasBWI()); 3427 return X86::KMOVQrk; 3428 } 3429 if (X86::GR32RegClass.contains(DestReg)) 3430 return Subtarget.hasBWI() ? X86::KMOVDrk : X86::KMOVWrk; 3431 } 3432 3433 // SrcReg(GR64) -> DestReg(MaskReg) 3434 // SrcReg(GR32) -> DestReg(MaskReg) 3435 3436 // All KMASK RegClasses hold the same k registers, can be tested against anyone. 3437 if (X86::VK16RegClass.contains(DestReg)) { 3438 if (X86::GR64RegClass.contains(SrcReg)) { 3439 assert(Subtarget.hasBWI()); 3440 return X86::KMOVQkr; 3441 } 3442 if (X86::GR32RegClass.contains(SrcReg)) 3443 return Subtarget.hasBWI() ? X86::KMOVDkr : X86::KMOVWkr; 3444 } 3445 3446 3447 // SrcReg(VR128) -> DestReg(GR64) 3448 // SrcReg(VR64) -> DestReg(GR64) 3449 // SrcReg(GR64) -> DestReg(VR128) 3450 // SrcReg(GR64) -> DestReg(VR64) 3451 3452 if (X86::GR64RegClass.contains(DestReg)) { 3453 if (X86::VR128XRegClass.contains(SrcReg)) 3454 // Copy from a VR128 register to a GR64 register. 3455 return HasAVX512 ? X86::VMOVPQIto64Zrr : 3456 HasAVX ? X86::VMOVPQIto64rr : 3457 X86::MOVPQIto64rr; 3458 if (X86::VR64RegClass.contains(SrcReg)) 3459 // Copy from a VR64 register to a GR64 register. 3460 return X86::MMX_MOVD64from64rr; 3461 } else if (X86::GR64RegClass.contains(SrcReg)) { 3462 // Copy from a GR64 register to a VR128 register. 3463 if (X86::VR128XRegClass.contains(DestReg)) 3464 return HasAVX512 ? X86::VMOV64toPQIZrr : 3465 HasAVX ? X86::VMOV64toPQIrr : 3466 X86::MOV64toPQIrr; 3467 // Copy from a GR64 register to a VR64 register. 3468 if (X86::VR64RegClass.contains(DestReg)) 3469 return X86::MMX_MOVD64to64rr; 3470 } 3471 3472 // SrcReg(VR128) -> DestReg(GR32) 3473 // SrcReg(GR32) -> DestReg(VR128) 3474 3475 if (X86::GR32RegClass.contains(DestReg) && 3476 X86::VR128XRegClass.contains(SrcReg)) 3477 // Copy from a VR128 register to a GR32 register. 3478 return HasAVX512 ? X86::VMOVPDI2DIZrr : 3479 HasAVX ? X86::VMOVPDI2DIrr : 3480 X86::MOVPDI2DIrr; 3481 3482 if (X86::VR128XRegClass.contains(DestReg) && 3483 X86::GR32RegClass.contains(SrcReg)) 3484 // Copy from a VR128 register to a VR128 register. 3485 return HasAVX512 ? X86::VMOVDI2PDIZrr : 3486 HasAVX ? X86::VMOVDI2PDIrr : 3487 X86::MOVDI2PDIrr; 3488 return 0; 3489 } 3490 3491 void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB, 3492 MachineBasicBlock::iterator MI, 3493 const DebugLoc &DL, MCRegister DestReg, 3494 MCRegister SrcReg, bool KillSrc) const { 3495 // First deal with the normal symmetric copies. 3496 bool HasAVX = Subtarget.hasAVX(); 3497 bool HasVLX = Subtarget.hasVLX(); 3498 unsigned Opc = 0; 3499 if (X86::GR64RegClass.contains(DestReg, SrcReg)) 3500 Opc = X86::MOV64rr; 3501 else if (X86::GR32RegClass.contains(DestReg, SrcReg)) 3502 Opc = X86::MOV32rr; 3503 else if (X86::GR16RegClass.contains(DestReg, SrcReg)) 3504 Opc = X86::MOV16rr; 3505 else if (X86::GR8RegClass.contains(DestReg, SrcReg)) { 3506 // Copying to or from a physical H register on x86-64 requires a NOREX 3507 // move. Otherwise use a normal move. 3508 if ((isHReg(DestReg) || isHReg(SrcReg)) && 3509 Subtarget.is64Bit()) { 3510 Opc = X86::MOV8rr_NOREX; 3511 // Both operands must be encodable without an REX prefix. 3512 assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && 3513 "8-bit H register can not be copied outside GR8_NOREX"); 3514 } else 3515 Opc = X86::MOV8rr; 3516 } 3517 else if (X86::VR64RegClass.contains(DestReg, SrcReg)) 3518 Opc = X86::MMX_MOVQ64rr; 3519 else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) { 3520 if (HasVLX) 3521 Opc = X86::VMOVAPSZ128rr; 3522 else if (X86::VR128RegClass.contains(DestReg, SrcReg)) 3523 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr; 3524 else { 3525 // If this an extended register and we don't have VLX we need to use a 3526 // 512-bit move. 3527 Opc = X86::VMOVAPSZrr; 3528 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3529 DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, 3530 &X86::VR512RegClass); 3531 SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, 3532 &X86::VR512RegClass); 3533 } 3534 } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) { 3535 if (HasVLX) 3536 Opc = X86::VMOVAPSZ256rr; 3537 else if (X86::VR256RegClass.contains(DestReg, SrcReg)) 3538 Opc = X86::VMOVAPSYrr; 3539 else { 3540 // If this an extended register and we don't have VLX we need to use a 3541 // 512-bit move. 3542 Opc = X86::VMOVAPSZrr; 3543 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3544 DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, 3545 &X86::VR512RegClass); 3546 SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, 3547 &X86::VR512RegClass); 3548 } 3549 } else if (X86::VR512RegClass.contains(DestReg, SrcReg)) 3550 Opc = X86::VMOVAPSZrr; 3551 // All KMASK RegClasses hold the same k registers, can be tested against anyone. 3552 else if (X86::VK16RegClass.contains(DestReg, SrcReg)) 3553 Opc = Subtarget.hasBWI() ? X86::KMOVQkk : X86::KMOVWkk; 3554 if (!Opc) 3555 Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget); 3556 3557 if (Opc) { 3558 BuildMI(MBB, MI, DL, get(Opc), DestReg) 3559 .addReg(SrcReg, getKillRegState(KillSrc)); 3560 return; 3561 } 3562 3563 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) { 3564 // FIXME: We use a fatal error here because historically LLVM has tried 3565 // lower some of these physreg copies and we want to ensure we get 3566 // reasonable bug reports if someone encounters a case no other testing 3567 // found. This path should be removed after the LLVM 7 release. 3568 report_fatal_error("Unable to copy EFLAGS physical register!"); 3569 } 3570 3571 LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to " 3572 << RI.getName(DestReg) << '\n'); 3573 report_fatal_error("Cannot emit physreg copy instruction"); 3574 } 3575 3576 std::optional<DestSourcePair> 3577 X86InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { 3578 if (MI.isMoveReg()) 3579 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 3580 return std::nullopt; 3581 } 3582 3583 static unsigned getLoadStoreOpcodeForFP16(bool Load, const X86Subtarget &STI) { 3584 if (STI.hasFP16()) 3585 return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr; 3586 if (Load) 3587 return STI.hasAVX512() ? X86::VMOVSSZrm 3588 : STI.hasAVX() ? X86::VMOVSSrm 3589 : X86::MOVSSrm; 3590 else 3591 return STI.hasAVX512() ? X86::VMOVSSZmr 3592 : STI.hasAVX() ? X86::VMOVSSmr 3593 : X86::MOVSSmr; 3594 } 3595 3596 static unsigned getLoadStoreRegOpcode(Register Reg, 3597 const TargetRegisterClass *RC, 3598 bool IsStackAligned, 3599 const X86Subtarget &STI, bool Load) { 3600 bool HasAVX = STI.hasAVX(); 3601 bool HasAVX512 = STI.hasAVX512(); 3602 bool HasVLX = STI.hasVLX(); 3603 3604 switch (STI.getRegisterInfo()->getSpillSize(*RC)) { 3605 default: 3606 llvm_unreachable("Unknown spill size"); 3607 case 1: 3608 assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass"); 3609 if (STI.is64Bit()) 3610 // Copying to or from a physical H register on x86-64 requires a NOREX 3611 // move. Otherwise use a normal move. 3612 if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC)) 3613 return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX; 3614 return Load ? X86::MOV8rm : X86::MOV8mr; 3615 case 2: 3616 if (X86::VK16RegClass.hasSubClassEq(RC)) 3617 return Load ? X86::KMOVWkm : X86::KMOVWmk; 3618 assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass"); 3619 return Load ? X86::MOV16rm : X86::MOV16mr; 3620 case 4: 3621 if (X86::GR32RegClass.hasSubClassEq(RC)) 3622 return Load ? X86::MOV32rm : X86::MOV32mr; 3623 if (X86::FR32XRegClass.hasSubClassEq(RC)) 3624 return Load ? 3625 (HasAVX512 ? X86::VMOVSSZrm_alt : 3626 HasAVX ? X86::VMOVSSrm_alt : 3627 X86::MOVSSrm_alt) : 3628 (HasAVX512 ? X86::VMOVSSZmr : 3629 HasAVX ? X86::VMOVSSmr : 3630 X86::MOVSSmr); 3631 if (X86::RFP32RegClass.hasSubClassEq(RC)) 3632 return Load ? X86::LD_Fp32m : X86::ST_Fp32m; 3633 if (X86::VK32RegClass.hasSubClassEq(RC)) { 3634 assert(STI.hasBWI() && "KMOVD requires BWI"); 3635 return Load ? X86::KMOVDkm : X86::KMOVDmk; 3636 } 3637 // All of these mask pair classes have the same spill size, the same kind 3638 // of kmov instructions can be used with all of them. 3639 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) || 3640 X86::VK2PAIRRegClass.hasSubClassEq(RC) || 3641 X86::VK4PAIRRegClass.hasSubClassEq(RC) || 3642 X86::VK8PAIRRegClass.hasSubClassEq(RC) || 3643 X86::VK16PAIRRegClass.hasSubClassEq(RC)) 3644 return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE; 3645 if (X86::FR16RegClass.hasSubClassEq(RC) || 3646 X86::FR16XRegClass.hasSubClassEq(RC)) 3647 return getLoadStoreOpcodeForFP16(Load, STI); 3648 llvm_unreachable("Unknown 4-byte regclass"); 3649 case 8: 3650 if (X86::GR64RegClass.hasSubClassEq(RC)) 3651 return Load ? X86::MOV64rm : X86::MOV64mr; 3652 if (X86::FR64XRegClass.hasSubClassEq(RC)) 3653 return Load ? 3654 (HasAVX512 ? X86::VMOVSDZrm_alt : 3655 HasAVX ? X86::VMOVSDrm_alt : 3656 X86::MOVSDrm_alt) : 3657 (HasAVX512 ? X86::VMOVSDZmr : 3658 HasAVX ? X86::VMOVSDmr : 3659 X86::MOVSDmr); 3660 if (X86::VR64RegClass.hasSubClassEq(RC)) 3661 return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr; 3662 if (X86::RFP64RegClass.hasSubClassEq(RC)) 3663 return Load ? X86::LD_Fp64m : X86::ST_Fp64m; 3664 if (X86::VK64RegClass.hasSubClassEq(RC)) { 3665 assert(STI.hasBWI() && "KMOVQ requires BWI"); 3666 return Load ? X86::KMOVQkm : X86::KMOVQmk; 3667 } 3668 llvm_unreachable("Unknown 8-byte regclass"); 3669 case 10: 3670 assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass"); 3671 return Load ? X86::LD_Fp80m : X86::ST_FpP80m; 3672 case 16: { 3673 if (X86::VR128XRegClass.hasSubClassEq(RC)) { 3674 // If stack is realigned we can use aligned stores. 3675 if (IsStackAligned) 3676 return Load ? 3677 (HasVLX ? X86::VMOVAPSZ128rm : 3678 HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX : 3679 HasAVX ? X86::VMOVAPSrm : 3680 X86::MOVAPSrm): 3681 (HasVLX ? X86::VMOVAPSZ128mr : 3682 HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX : 3683 HasAVX ? X86::VMOVAPSmr : 3684 X86::MOVAPSmr); 3685 else 3686 return Load ? 3687 (HasVLX ? X86::VMOVUPSZ128rm : 3688 HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX : 3689 HasAVX ? X86::VMOVUPSrm : 3690 X86::MOVUPSrm): 3691 (HasVLX ? X86::VMOVUPSZ128mr : 3692 HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX : 3693 HasAVX ? X86::VMOVUPSmr : 3694 X86::MOVUPSmr); 3695 } 3696 llvm_unreachable("Unknown 16-byte regclass"); 3697 } 3698 case 32: 3699 assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass"); 3700 // If stack is realigned we can use aligned stores. 3701 if (IsStackAligned) 3702 return Load ? 3703 (HasVLX ? X86::VMOVAPSZ256rm : 3704 HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX : 3705 X86::VMOVAPSYrm) : 3706 (HasVLX ? X86::VMOVAPSZ256mr : 3707 HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX : 3708 X86::VMOVAPSYmr); 3709 else 3710 return Load ? 3711 (HasVLX ? X86::VMOVUPSZ256rm : 3712 HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX : 3713 X86::VMOVUPSYrm) : 3714 (HasVLX ? X86::VMOVUPSZ256mr : 3715 HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX : 3716 X86::VMOVUPSYmr); 3717 case 64: 3718 assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass"); 3719 assert(STI.hasAVX512() && "Using 512-bit register requires AVX512"); 3720 if (IsStackAligned) 3721 return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; 3722 else 3723 return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; 3724 case 1024: 3725 assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass"); 3726 assert(STI.hasAMXTILE() && "Using 8*1024-bit register requires AMX-TILE"); 3727 return Load ? X86::TILELOADD : X86::TILESTORED; 3728 } 3729 } 3730 3731 std::optional<ExtAddrMode> 3732 X86InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI, 3733 const TargetRegisterInfo *TRI) const { 3734 const MCInstrDesc &Desc = MemI.getDesc(); 3735 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); 3736 if (MemRefBegin < 0) 3737 return std::nullopt; 3738 3739 MemRefBegin += X86II::getOperandBias(Desc); 3740 3741 auto &BaseOp = MemI.getOperand(MemRefBegin + X86::AddrBaseReg); 3742 if (!BaseOp.isReg()) // Can be an MO_FrameIndex 3743 return std::nullopt; 3744 3745 const MachineOperand &DispMO = MemI.getOperand(MemRefBegin + X86::AddrDisp); 3746 // Displacement can be symbolic 3747 if (!DispMO.isImm()) 3748 return std::nullopt; 3749 3750 ExtAddrMode AM; 3751 AM.BaseReg = BaseOp.getReg(); 3752 AM.ScaledReg = MemI.getOperand(MemRefBegin + X86::AddrIndexReg).getReg(); 3753 AM.Scale = MemI.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm(); 3754 AM.Displacement = DispMO.getImm(); 3755 return AM; 3756 } 3757 3758 bool X86InstrInfo::verifyInstruction(const MachineInstr &MI, 3759 StringRef &ErrInfo) const { 3760 std::optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MI, nullptr); 3761 if (!AMOrNone) 3762 return true; 3763 3764 ExtAddrMode AM = *AMOrNone; 3765 3766 if (AM.ScaledReg != X86::NoRegister) { 3767 switch (AM.Scale) { 3768 case 1: 3769 case 2: 3770 case 4: 3771 case 8: 3772 break; 3773 default: 3774 ErrInfo = "Scale factor in address must be 1, 2, 4 or 8"; 3775 return false; 3776 } 3777 } 3778 if (!isInt<32>(AM.Displacement)) { 3779 ErrInfo = "Displacement in address must fit into 32-bit signed " 3780 "integer"; 3781 return false; 3782 } 3783 3784 return true; 3785 } 3786 3787 bool X86InstrInfo::getConstValDefinedInReg(const MachineInstr &MI, 3788 const Register Reg, 3789 int64_t &ImmVal) const { 3790 if (MI.getOpcode() != X86::MOV32ri && MI.getOpcode() != X86::MOV64ri) 3791 return false; 3792 // Mov Src can be a global address. 3793 if (!MI.getOperand(1).isImm() || MI.getOperand(0).getReg() != Reg) 3794 return false; 3795 ImmVal = MI.getOperand(1).getImm(); 3796 return true; 3797 } 3798 3799 bool X86InstrInfo::preservesZeroValueInReg( 3800 const MachineInstr *MI, const Register NullValueReg, 3801 const TargetRegisterInfo *TRI) const { 3802 if (!MI->modifiesRegister(NullValueReg, TRI)) 3803 return true; 3804 switch (MI->getOpcode()) { 3805 // Shift right/left of a null unto itself is still a null, i.e. rax = shl rax 3806 // X. 3807 case X86::SHR64ri: 3808 case X86::SHR32ri: 3809 case X86::SHL64ri: 3810 case X86::SHL32ri: 3811 assert(MI->getOperand(0).isDef() && MI->getOperand(1).isUse() && 3812 "expected for shift opcode!"); 3813 return MI->getOperand(0).getReg() == NullValueReg && 3814 MI->getOperand(1).getReg() == NullValueReg; 3815 // Zero extend of a sub-reg of NullValueReg into itself does not change the 3816 // null value. 3817 case X86::MOV32rr: 3818 return llvm::all_of(MI->operands(), [&](const MachineOperand &MO) { 3819 return TRI->isSubRegisterEq(NullValueReg, MO.getReg()); 3820 }); 3821 default: 3822 return false; 3823 } 3824 llvm_unreachable("Should be handled above!"); 3825 } 3826 3827 bool X86InstrInfo::getMemOperandsWithOffsetWidth( 3828 const MachineInstr &MemOp, SmallVectorImpl<const MachineOperand *> &BaseOps, 3829 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 3830 const TargetRegisterInfo *TRI) const { 3831 const MCInstrDesc &Desc = MemOp.getDesc(); 3832 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); 3833 if (MemRefBegin < 0) 3834 return false; 3835 3836 MemRefBegin += X86II::getOperandBias(Desc); 3837 3838 const MachineOperand *BaseOp = 3839 &MemOp.getOperand(MemRefBegin + X86::AddrBaseReg); 3840 if (!BaseOp->isReg()) // Can be an MO_FrameIndex 3841 return false; 3842 3843 if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1) 3844 return false; 3845 3846 if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() != 3847 X86::NoRegister) 3848 return false; 3849 3850 const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp); 3851 3852 // Displacement can be symbolic 3853 if (!DispMO.isImm()) 3854 return false; 3855 3856 Offset = DispMO.getImm(); 3857 3858 if (!BaseOp->isReg()) 3859 return false; 3860 3861 OffsetIsScalable = false; 3862 // FIXME: Relying on memoperands() may not be right thing to do here. Check 3863 // with X86 maintainers, and fix it accordingly. For now, it is ok, since 3864 // there is no use of `Width` for X86 back-end at the moment. 3865 Width = 3866 !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize() : 0; 3867 BaseOps.push_back(BaseOp); 3868 return true; 3869 } 3870 3871 static unsigned getStoreRegOpcode(Register SrcReg, 3872 const TargetRegisterClass *RC, 3873 bool IsStackAligned, 3874 const X86Subtarget &STI) { 3875 return getLoadStoreRegOpcode(SrcReg, RC, IsStackAligned, STI, false); 3876 } 3877 3878 static unsigned getLoadRegOpcode(Register DestReg, 3879 const TargetRegisterClass *RC, 3880 bool IsStackAligned, const X86Subtarget &STI) { 3881 return getLoadStoreRegOpcode(DestReg, RC, IsStackAligned, STI, true); 3882 } 3883 3884 static bool isAMXOpcode(unsigned Opc) { 3885 switch (Opc) { 3886 default: 3887 return false; 3888 case X86::TILELOADD: 3889 case X86::TILESTORED: 3890 return true; 3891 } 3892 } 3893 3894 void X86InstrInfo::loadStoreTileReg(MachineBasicBlock &MBB, 3895 MachineBasicBlock::iterator MI, 3896 unsigned Opc, Register Reg, int FrameIdx, 3897 bool isKill) const { 3898 switch (Opc) { 3899 default: 3900 llvm_unreachable("Unexpected special opcode!"); 3901 case X86::TILESTORED: { 3902 // tilestored %tmm, (%sp, %idx) 3903 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo(); 3904 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 3905 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64); 3906 MachineInstr *NewMI = 3907 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx) 3908 .addReg(Reg, getKillRegState(isKill)); 3909 MachineOperand &MO = NewMI->getOperand(X86::AddrIndexReg); 3910 MO.setReg(VirtReg); 3911 MO.setIsKill(true); 3912 break; 3913 } 3914 case X86::TILELOADD: { 3915 // tileloadd (%sp, %idx), %tmm 3916 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo(); 3917 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 3918 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64); 3919 MachineInstr *NewMI = addFrameReference( 3920 BuildMI(MBB, MI, DebugLoc(), get(Opc), Reg), FrameIdx); 3921 MachineOperand &MO = NewMI->getOperand(1 + X86::AddrIndexReg); 3922 MO.setReg(VirtReg); 3923 MO.setIsKill(true); 3924 break; 3925 } 3926 } 3927 } 3928 3929 void X86InstrInfo::storeRegToStackSlot( 3930 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, 3931 bool isKill, int FrameIdx, const TargetRegisterClass *RC, 3932 const TargetRegisterInfo *TRI, Register VReg) const { 3933 const MachineFunction &MF = *MBB.getParent(); 3934 const MachineFrameInfo &MFI = MF.getFrameInfo(); 3935 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && 3936 "Stack slot too small for store"); 3937 3938 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); 3939 bool isAligned = 3940 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) || 3941 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx)); 3942 3943 unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget); 3944 if (isAMXOpcode(Opc)) 3945 loadStoreTileReg(MBB, MI, Opc, SrcReg, FrameIdx, isKill); 3946 else 3947 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx) 3948 .addReg(SrcReg, getKillRegState(isKill)); 3949 } 3950 3951 void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 3952 MachineBasicBlock::iterator MI, 3953 Register DestReg, int FrameIdx, 3954 const TargetRegisterClass *RC, 3955 const TargetRegisterInfo *TRI, 3956 Register VReg) const { 3957 const MachineFunction &MF = *MBB.getParent(); 3958 const MachineFrameInfo &MFI = MF.getFrameInfo(); 3959 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && 3960 "Load size exceeds stack slot"); 3961 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); 3962 bool isAligned = 3963 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) || 3964 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx)); 3965 3966 unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget); 3967 if (isAMXOpcode(Opc)) 3968 loadStoreTileReg(MBB, MI, Opc, DestReg, FrameIdx); 3969 else 3970 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), 3971 FrameIdx); 3972 } 3973 3974 bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 3975 Register &SrcReg2, int64_t &CmpMask, 3976 int64_t &CmpValue) const { 3977 switch (MI.getOpcode()) { 3978 default: break; 3979 case X86::CMP64ri32: 3980 case X86::CMP64ri8: 3981 case X86::CMP32ri: 3982 case X86::CMP32ri8: 3983 case X86::CMP16ri: 3984 case X86::CMP16ri8: 3985 case X86::CMP8ri: 3986 SrcReg = MI.getOperand(0).getReg(); 3987 SrcReg2 = 0; 3988 if (MI.getOperand(1).isImm()) { 3989 CmpMask = ~0; 3990 CmpValue = MI.getOperand(1).getImm(); 3991 } else { 3992 CmpMask = CmpValue = 0; 3993 } 3994 return true; 3995 // A SUB can be used to perform comparison. 3996 case X86::SUB64rm: 3997 case X86::SUB32rm: 3998 case X86::SUB16rm: 3999 case X86::SUB8rm: 4000 SrcReg = MI.getOperand(1).getReg(); 4001 SrcReg2 = 0; 4002 CmpMask = 0; 4003 CmpValue = 0; 4004 return true; 4005 case X86::SUB64rr: 4006 case X86::SUB32rr: 4007 case X86::SUB16rr: 4008 case X86::SUB8rr: 4009 SrcReg = MI.getOperand(1).getReg(); 4010 SrcReg2 = MI.getOperand(2).getReg(); 4011 CmpMask = 0; 4012 CmpValue = 0; 4013 return true; 4014 case X86::SUB64ri32: 4015 case X86::SUB64ri8: 4016 case X86::SUB32ri: 4017 case X86::SUB32ri8: 4018 case X86::SUB16ri: 4019 case X86::SUB16ri8: 4020 case X86::SUB8ri: 4021 SrcReg = MI.getOperand(1).getReg(); 4022 SrcReg2 = 0; 4023 if (MI.getOperand(2).isImm()) { 4024 CmpMask = ~0; 4025 CmpValue = MI.getOperand(2).getImm(); 4026 } else { 4027 CmpMask = CmpValue = 0; 4028 } 4029 return true; 4030 case X86::CMP64rr: 4031 case X86::CMP32rr: 4032 case X86::CMP16rr: 4033 case X86::CMP8rr: 4034 SrcReg = MI.getOperand(0).getReg(); 4035 SrcReg2 = MI.getOperand(1).getReg(); 4036 CmpMask = 0; 4037 CmpValue = 0; 4038 return true; 4039 case X86::TEST8rr: 4040 case X86::TEST16rr: 4041 case X86::TEST32rr: 4042 case X86::TEST64rr: 4043 SrcReg = MI.getOperand(0).getReg(); 4044 if (MI.getOperand(1).getReg() != SrcReg) 4045 return false; 4046 // Compare against zero. 4047 SrcReg2 = 0; 4048 CmpMask = ~0; 4049 CmpValue = 0; 4050 return true; 4051 } 4052 return false; 4053 } 4054 4055 bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI, 4056 Register SrcReg, Register SrcReg2, 4057 int64_t ImmMask, int64_t ImmValue, 4058 const MachineInstr &OI, bool *IsSwapped, 4059 int64_t *ImmDelta) const { 4060 switch (OI.getOpcode()) { 4061 case X86::CMP64rr: 4062 case X86::CMP32rr: 4063 case X86::CMP16rr: 4064 case X86::CMP8rr: 4065 case X86::SUB64rr: 4066 case X86::SUB32rr: 4067 case X86::SUB16rr: 4068 case X86::SUB8rr: { 4069 Register OISrcReg; 4070 Register OISrcReg2; 4071 int64_t OIMask; 4072 int64_t OIValue; 4073 if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) || 4074 OIMask != ImmMask || OIValue != ImmValue) 4075 return false; 4076 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) { 4077 *IsSwapped = false; 4078 return true; 4079 } 4080 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) { 4081 *IsSwapped = true; 4082 return true; 4083 } 4084 return false; 4085 } 4086 case X86::CMP64ri32: 4087 case X86::CMP64ri8: 4088 case X86::CMP32ri: 4089 case X86::CMP32ri8: 4090 case X86::CMP16ri: 4091 case X86::CMP16ri8: 4092 case X86::CMP8ri: 4093 case X86::SUB64ri32: 4094 case X86::SUB64ri8: 4095 case X86::SUB32ri: 4096 case X86::SUB32ri8: 4097 case X86::SUB16ri: 4098 case X86::SUB16ri8: 4099 case X86::SUB8ri: 4100 case X86::TEST64rr: 4101 case X86::TEST32rr: 4102 case X86::TEST16rr: 4103 case X86::TEST8rr: { 4104 if (ImmMask != 0) { 4105 Register OISrcReg; 4106 Register OISrcReg2; 4107 int64_t OIMask; 4108 int64_t OIValue; 4109 if (analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) && 4110 SrcReg == OISrcReg && ImmMask == OIMask) { 4111 if (OIValue == ImmValue) { 4112 *ImmDelta = 0; 4113 return true; 4114 } else if (static_cast<uint64_t>(ImmValue) == 4115 static_cast<uint64_t>(OIValue) - 1) { 4116 *ImmDelta = -1; 4117 return true; 4118 } else if (static_cast<uint64_t>(ImmValue) == 4119 static_cast<uint64_t>(OIValue) + 1) { 4120 *ImmDelta = 1; 4121 return true; 4122 } else { 4123 return false; 4124 } 4125 } 4126 } 4127 return FlagI.isIdenticalTo(OI); 4128 } 4129 default: 4130 return false; 4131 } 4132 } 4133 4134 /// Check whether the definition can be converted 4135 /// to remove a comparison against zero. 4136 inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag, 4137 bool &ClearsOverflowFlag) { 4138 NoSignFlag = false; 4139 ClearsOverflowFlag = false; 4140 4141 // "ELF Handling for Thread-Local Storage" specifies that x86-64 GOTTPOFF, and 4142 // i386 GOTNTPOFF/INDNTPOFF relocations can convert an ADD to a LEA during 4143 // Initial Exec to Local Exec relaxation. In these cases, we must not depend 4144 // on the EFLAGS modification of ADD actually happening in the final binary. 4145 if (MI.getOpcode() == X86::ADD64rm || MI.getOpcode() == X86::ADD32rm) { 4146 unsigned Flags = MI.getOperand(5).getTargetFlags(); 4147 if (Flags == X86II::MO_GOTTPOFF || Flags == X86II::MO_INDNTPOFF || 4148 Flags == X86II::MO_GOTNTPOFF) 4149 return false; 4150 } 4151 4152 switch (MI.getOpcode()) { 4153 default: return false; 4154 4155 // The shift instructions only modify ZF if their shift count is non-zero. 4156 // N.B.: The processor truncates the shift count depending on the encoding. 4157 case X86::SAR8ri: case X86::SAR16ri: case X86::SAR32ri:case X86::SAR64ri: 4158 case X86::SHR8ri: case X86::SHR16ri: case X86::SHR32ri:case X86::SHR64ri: 4159 return getTruncatedShiftCount(MI, 2) != 0; 4160 4161 // Some left shift instructions can be turned into LEA instructions but only 4162 // if their flags aren't used. Avoid transforming such instructions. 4163 case X86::SHL8ri: case X86::SHL16ri: case X86::SHL32ri:case X86::SHL64ri:{ 4164 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 4165 if (isTruncatedShiftCountForLEA(ShAmt)) return false; 4166 return ShAmt != 0; 4167 } 4168 4169 case X86::SHRD16rri8:case X86::SHRD32rri8:case X86::SHRD64rri8: 4170 case X86::SHLD16rri8:case X86::SHLD32rri8:case X86::SHLD64rri8: 4171 return getTruncatedShiftCount(MI, 3) != 0; 4172 4173 case X86::SUB64ri32: case X86::SUB64ri8: case X86::SUB32ri: 4174 case X86::SUB32ri8: case X86::SUB16ri: case X86::SUB16ri8: 4175 case X86::SUB8ri: case X86::SUB64rr: case X86::SUB32rr: 4176 case X86::SUB16rr: case X86::SUB8rr: case X86::SUB64rm: 4177 case X86::SUB32rm: case X86::SUB16rm: case X86::SUB8rm: 4178 case X86::DEC64r: case X86::DEC32r: case X86::DEC16r: case X86::DEC8r: 4179 case X86::ADD64ri32: case X86::ADD64ri8: case X86::ADD32ri: 4180 case X86::ADD32ri8: case X86::ADD16ri: case X86::ADD16ri8: 4181 case X86::ADD8ri: case X86::ADD64rr: case X86::ADD32rr: 4182 case X86::ADD16rr: case X86::ADD8rr: case X86::ADD64rm: 4183 case X86::ADD32rm: case X86::ADD16rm: case X86::ADD8rm: 4184 case X86::INC64r: case X86::INC32r: case X86::INC16r: case X86::INC8r: 4185 case X86::ADC64ri32: case X86::ADC64ri8: case X86::ADC32ri: 4186 case X86::ADC32ri8: case X86::ADC16ri: case X86::ADC16ri8: 4187 case X86::ADC8ri: case X86::ADC64rr: case X86::ADC32rr: 4188 case X86::ADC16rr: case X86::ADC8rr: case X86::ADC64rm: 4189 case X86::ADC32rm: case X86::ADC16rm: case X86::ADC8rm: 4190 case X86::SBB64ri32: case X86::SBB64ri8: case X86::SBB32ri: 4191 case X86::SBB32ri8: case X86::SBB16ri: case X86::SBB16ri8: 4192 case X86::SBB8ri: case X86::SBB64rr: case X86::SBB32rr: 4193 case X86::SBB16rr: case X86::SBB8rr: case X86::SBB64rm: 4194 case X86::SBB32rm: case X86::SBB16rm: case X86::SBB8rm: 4195 case X86::NEG8r: case X86::NEG16r: case X86::NEG32r: case X86::NEG64r: 4196 case X86::SAR8r1: case X86::SAR16r1: case X86::SAR32r1:case X86::SAR64r1: 4197 case X86::SHR8r1: case X86::SHR16r1: case X86::SHR32r1:case X86::SHR64r1: 4198 case X86::SHL8r1: case X86::SHL16r1: case X86::SHL32r1:case X86::SHL64r1: 4199 case X86::LZCNT16rr: case X86::LZCNT16rm: 4200 case X86::LZCNT32rr: case X86::LZCNT32rm: 4201 case X86::LZCNT64rr: case X86::LZCNT64rm: 4202 case X86::POPCNT16rr:case X86::POPCNT16rm: 4203 case X86::POPCNT32rr:case X86::POPCNT32rm: 4204 case X86::POPCNT64rr:case X86::POPCNT64rm: 4205 case X86::TZCNT16rr: case X86::TZCNT16rm: 4206 case X86::TZCNT32rr: case X86::TZCNT32rm: 4207 case X86::TZCNT64rr: case X86::TZCNT64rm: 4208 return true; 4209 case X86::AND64ri32: case X86::AND64ri8: case X86::AND32ri: 4210 case X86::AND32ri8: case X86::AND16ri: case X86::AND16ri8: 4211 case X86::AND8ri: case X86::AND64rr: case X86::AND32rr: 4212 case X86::AND16rr: case X86::AND8rr: case X86::AND64rm: 4213 case X86::AND32rm: case X86::AND16rm: case X86::AND8rm: 4214 case X86::XOR64ri32: case X86::XOR64ri8: case X86::XOR32ri: 4215 case X86::XOR32ri8: case X86::XOR16ri: case X86::XOR16ri8: 4216 case X86::XOR8ri: case X86::XOR64rr: case X86::XOR32rr: 4217 case X86::XOR16rr: case X86::XOR8rr: case X86::XOR64rm: 4218 case X86::XOR32rm: case X86::XOR16rm: case X86::XOR8rm: 4219 case X86::OR64ri32: case X86::OR64ri8: case X86::OR32ri: 4220 case X86::OR32ri8: case X86::OR16ri: case X86::OR16ri8: 4221 case X86::OR8ri: case X86::OR64rr: case X86::OR32rr: 4222 case X86::OR16rr: case X86::OR8rr: case X86::OR64rm: 4223 case X86::OR32rm: case X86::OR16rm: case X86::OR8rm: 4224 case X86::ANDN32rr: case X86::ANDN32rm: 4225 case X86::ANDN64rr: case X86::ANDN64rm: 4226 case X86::BLSI32rr: case X86::BLSI32rm: 4227 case X86::BLSI64rr: case X86::BLSI64rm: 4228 case X86::BLSMSK32rr: case X86::BLSMSK32rm: 4229 case X86::BLSMSK64rr: case X86::BLSMSK64rm: 4230 case X86::BLSR32rr: case X86::BLSR32rm: 4231 case X86::BLSR64rr: case X86::BLSR64rm: 4232 case X86::BLCFILL32rr: case X86::BLCFILL32rm: 4233 case X86::BLCFILL64rr: case X86::BLCFILL64rm: 4234 case X86::BLCI32rr: case X86::BLCI32rm: 4235 case X86::BLCI64rr: case X86::BLCI64rm: 4236 case X86::BLCIC32rr: case X86::BLCIC32rm: 4237 case X86::BLCIC64rr: case X86::BLCIC64rm: 4238 case X86::BLCMSK32rr: case X86::BLCMSK32rm: 4239 case X86::BLCMSK64rr: case X86::BLCMSK64rm: 4240 case X86::BLCS32rr: case X86::BLCS32rm: 4241 case X86::BLCS64rr: case X86::BLCS64rm: 4242 case X86::BLSFILL32rr: case X86::BLSFILL32rm: 4243 case X86::BLSFILL64rr: case X86::BLSFILL64rm: 4244 case X86::BLSIC32rr: case X86::BLSIC32rm: 4245 case X86::BLSIC64rr: case X86::BLSIC64rm: 4246 case X86::BZHI32rr: case X86::BZHI32rm: 4247 case X86::BZHI64rr: case X86::BZHI64rm: 4248 case X86::T1MSKC32rr: case X86::T1MSKC32rm: 4249 case X86::T1MSKC64rr: case X86::T1MSKC64rm: 4250 case X86::TZMSK32rr: case X86::TZMSK32rm: 4251 case X86::TZMSK64rr: case X86::TZMSK64rm: 4252 // These instructions clear the overflow flag just like TEST. 4253 // FIXME: These are not the only instructions in this switch that clear the 4254 // overflow flag. 4255 ClearsOverflowFlag = true; 4256 return true; 4257 case X86::BEXTR32rr: case X86::BEXTR64rr: 4258 case X86::BEXTR32rm: case X86::BEXTR64rm: 4259 case X86::BEXTRI32ri: case X86::BEXTRI32mi: 4260 case X86::BEXTRI64ri: case X86::BEXTRI64mi: 4261 // BEXTR doesn't update the sign flag so we can't use it. It does clear 4262 // the overflow flag, but that's not useful without the sign flag. 4263 NoSignFlag = true; 4264 return true; 4265 } 4266 } 4267 4268 /// Check whether the use can be converted to remove a comparison against zero. 4269 static X86::CondCode isUseDefConvertible(const MachineInstr &MI) { 4270 switch (MI.getOpcode()) { 4271 default: return X86::COND_INVALID; 4272 case X86::NEG8r: 4273 case X86::NEG16r: 4274 case X86::NEG32r: 4275 case X86::NEG64r: 4276 return X86::COND_AE; 4277 case X86::LZCNT16rr: 4278 case X86::LZCNT32rr: 4279 case X86::LZCNT64rr: 4280 return X86::COND_B; 4281 case X86::POPCNT16rr: 4282 case X86::POPCNT32rr: 4283 case X86::POPCNT64rr: 4284 return X86::COND_E; 4285 case X86::TZCNT16rr: 4286 case X86::TZCNT32rr: 4287 case X86::TZCNT64rr: 4288 return X86::COND_B; 4289 case X86::BSF16rr: 4290 case X86::BSF32rr: 4291 case X86::BSF64rr: 4292 case X86::BSR16rr: 4293 case X86::BSR32rr: 4294 case X86::BSR64rr: 4295 return X86::COND_E; 4296 case X86::BLSI32rr: 4297 case X86::BLSI64rr: 4298 return X86::COND_AE; 4299 case X86::BLSR32rr: 4300 case X86::BLSR64rr: 4301 case X86::BLSMSK32rr: 4302 case X86::BLSMSK64rr: 4303 return X86::COND_B; 4304 // TODO: TBM instructions. 4305 } 4306 } 4307 4308 /// Check if there exists an earlier instruction that 4309 /// operates on the same source operands and sets flags in the same way as 4310 /// Compare; remove Compare if possible. 4311 bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, 4312 Register SrcReg2, int64_t CmpMask, 4313 int64_t CmpValue, 4314 const MachineRegisterInfo *MRI) const { 4315 // Check whether we can replace SUB with CMP. 4316 switch (CmpInstr.getOpcode()) { 4317 default: break; 4318 case X86::SUB64ri32: 4319 case X86::SUB64ri8: 4320 case X86::SUB32ri: 4321 case X86::SUB32ri8: 4322 case X86::SUB16ri: 4323 case X86::SUB16ri8: 4324 case X86::SUB8ri: 4325 case X86::SUB64rm: 4326 case X86::SUB32rm: 4327 case X86::SUB16rm: 4328 case X86::SUB8rm: 4329 case X86::SUB64rr: 4330 case X86::SUB32rr: 4331 case X86::SUB16rr: 4332 case X86::SUB8rr: { 4333 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg())) 4334 return false; 4335 // There is no use of the destination register, we can replace SUB with CMP. 4336 unsigned NewOpcode = 0; 4337 switch (CmpInstr.getOpcode()) { 4338 default: llvm_unreachable("Unreachable!"); 4339 case X86::SUB64rm: NewOpcode = X86::CMP64rm; break; 4340 case X86::SUB32rm: NewOpcode = X86::CMP32rm; break; 4341 case X86::SUB16rm: NewOpcode = X86::CMP16rm; break; 4342 case X86::SUB8rm: NewOpcode = X86::CMP8rm; break; 4343 case X86::SUB64rr: NewOpcode = X86::CMP64rr; break; 4344 case X86::SUB32rr: NewOpcode = X86::CMP32rr; break; 4345 case X86::SUB16rr: NewOpcode = X86::CMP16rr; break; 4346 case X86::SUB8rr: NewOpcode = X86::CMP8rr; break; 4347 case X86::SUB64ri32: NewOpcode = X86::CMP64ri32; break; 4348 case X86::SUB64ri8: NewOpcode = X86::CMP64ri8; break; 4349 case X86::SUB32ri: NewOpcode = X86::CMP32ri; break; 4350 case X86::SUB32ri8: NewOpcode = X86::CMP32ri8; break; 4351 case X86::SUB16ri: NewOpcode = X86::CMP16ri; break; 4352 case X86::SUB16ri8: NewOpcode = X86::CMP16ri8; break; 4353 case X86::SUB8ri: NewOpcode = X86::CMP8ri; break; 4354 } 4355 CmpInstr.setDesc(get(NewOpcode)); 4356 CmpInstr.removeOperand(0); 4357 // Mutating this instruction invalidates any debug data associated with it. 4358 CmpInstr.dropDebugNumber(); 4359 // Fall through to optimize Cmp if Cmp is CMPrr or CMPri. 4360 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm || 4361 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm) 4362 return false; 4363 } 4364 } 4365 4366 // The following code tries to remove the comparison by re-using EFLAGS 4367 // from earlier instructions. 4368 4369 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0); 4370 4371 // Transformation currently requires SSA values. 4372 if (SrcReg2.isPhysical()) 4373 return false; 4374 MachineInstr *SrcRegDef = MRI->getVRegDef(SrcReg); 4375 assert(SrcRegDef && "Must have a definition (SSA)"); 4376 4377 MachineInstr *MI = nullptr; 4378 MachineInstr *Sub = nullptr; 4379 MachineInstr *Movr0Inst = nullptr; 4380 bool NoSignFlag = false; 4381 bool ClearsOverflowFlag = false; 4382 bool ShouldUpdateCC = false; 4383 bool IsSwapped = false; 4384 X86::CondCode NewCC = X86::COND_INVALID; 4385 int64_t ImmDelta = 0; 4386 4387 // Search backward from CmpInstr for the next instruction defining EFLAGS. 4388 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4389 MachineBasicBlock &CmpMBB = *CmpInstr.getParent(); 4390 MachineBasicBlock::reverse_iterator From = 4391 std::next(MachineBasicBlock::reverse_iterator(CmpInstr)); 4392 for (MachineBasicBlock *MBB = &CmpMBB;;) { 4393 for (MachineInstr &Inst : make_range(From, MBB->rend())) { 4394 // Try to use EFLAGS from the instruction defining %SrcReg. Example: 4395 // %eax = addl ... 4396 // ... // EFLAGS not changed 4397 // testl %eax, %eax // <-- can be removed 4398 if (&Inst == SrcRegDef) { 4399 if (IsCmpZero && 4400 isDefConvertible(Inst, NoSignFlag, ClearsOverflowFlag)) { 4401 MI = &Inst; 4402 break; 4403 } 4404 4405 // Look back for the following pattern, in which case the test64rr 4406 // instruction could be erased. 4407 // 4408 // Example: 4409 // %reg = and32ri %in_reg, 5 4410 // ... // EFLAGS not changed. 4411 // %src_reg = subreg_to_reg 0, %reg, %subreg.sub_index 4412 // test64rr %src_reg, %src_reg, implicit-def $eflags 4413 MachineInstr *AndInstr = nullptr; 4414 if (IsCmpZero && 4415 findRedundantFlagInstr(CmpInstr, Inst, MRI, &AndInstr, TRI, 4416 NoSignFlag, ClearsOverflowFlag)) { 4417 assert(AndInstr != nullptr && X86::isAND(AndInstr->getOpcode())); 4418 MI = AndInstr; 4419 break; 4420 } 4421 // Cannot find other candidates before definition of SrcReg. 4422 return false; 4423 } 4424 4425 if (Inst.modifiesRegister(X86::EFLAGS, TRI)) { 4426 // Try to use EFLAGS produced by an instruction reading %SrcReg. 4427 // Example: 4428 // %eax = ... 4429 // ... 4430 // popcntl %eax 4431 // ... // EFLAGS not changed 4432 // testl %eax, %eax // <-- can be removed 4433 if (IsCmpZero) { 4434 NewCC = isUseDefConvertible(Inst); 4435 if (NewCC != X86::COND_INVALID && Inst.getOperand(1).isReg() && 4436 Inst.getOperand(1).getReg() == SrcReg) { 4437 ShouldUpdateCC = true; 4438 MI = &Inst; 4439 break; 4440 } 4441 } 4442 4443 // Try to use EFLAGS from an instruction with similar flag results. 4444 // Example: 4445 // sub x, y or cmp x, y 4446 // ... // EFLAGS not changed 4447 // cmp x, y // <-- can be removed 4448 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue, 4449 Inst, &IsSwapped, &ImmDelta)) { 4450 Sub = &Inst; 4451 break; 4452 } 4453 4454 // MOV32r0 is implemented with xor which clobbers condition code. It is 4455 // safe to move up, if the definition to EFLAGS is dead and earlier 4456 // instructions do not read or write EFLAGS. 4457 if (!Movr0Inst && Inst.getOpcode() == X86::MOV32r0 && 4458 Inst.registerDefIsDead(X86::EFLAGS, TRI)) { 4459 Movr0Inst = &Inst; 4460 continue; 4461 } 4462 4463 // Cannot do anything for any other EFLAG changes. 4464 return false; 4465 } 4466 } 4467 4468 if (MI || Sub) 4469 break; 4470 4471 // Reached begin of basic block. Continue in predecessor if there is 4472 // exactly one. 4473 if (MBB->pred_size() != 1) 4474 return false; 4475 MBB = *MBB->pred_begin(); 4476 From = MBB->rbegin(); 4477 } 4478 4479 // Scan forward from the instruction after CmpInstr for uses of EFLAGS. 4480 // It is safe to remove CmpInstr if EFLAGS is redefined or killed. 4481 // If we are done with the basic block, we need to check whether EFLAGS is 4482 // live-out. 4483 bool FlagsMayLiveOut = true; 4484 SmallVector<std::pair<MachineInstr*, X86::CondCode>, 4> OpsToUpdate; 4485 MachineBasicBlock::iterator AfterCmpInstr = 4486 std::next(MachineBasicBlock::iterator(CmpInstr)); 4487 for (MachineInstr &Instr : make_range(AfterCmpInstr, CmpMBB.end())) { 4488 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI); 4489 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI); 4490 // We should check the usage if this instruction uses and updates EFLAGS. 4491 if (!UseEFLAGS && ModifyEFLAGS) { 4492 // It is safe to remove CmpInstr if EFLAGS is updated again. 4493 FlagsMayLiveOut = false; 4494 break; 4495 } 4496 if (!UseEFLAGS && !ModifyEFLAGS) 4497 continue; 4498 4499 // EFLAGS is used by this instruction. 4500 X86::CondCode OldCC = X86::getCondFromMI(Instr); 4501 if ((MI || IsSwapped || ImmDelta != 0) && OldCC == X86::COND_INVALID) 4502 return false; 4503 4504 X86::CondCode ReplacementCC = X86::COND_INVALID; 4505 if (MI) { 4506 switch (OldCC) { 4507 default: break; 4508 case X86::COND_A: case X86::COND_AE: 4509 case X86::COND_B: case X86::COND_BE: 4510 // CF is used, we can't perform this optimization. 4511 return false; 4512 case X86::COND_G: case X86::COND_GE: 4513 case X86::COND_L: case X86::COND_LE: 4514 // If SF is used, but the instruction doesn't update the SF, then we 4515 // can't do the optimization. 4516 if (NoSignFlag) 4517 return false; 4518 [[fallthrough]]; 4519 case X86::COND_O: case X86::COND_NO: 4520 // If OF is used, the instruction needs to clear it like CmpZero does. 4521 if (!ClearsOverflowFlag) 4522 return false; 4523 break; 4524 case X86::COND_S: case X86::COND_NS: 4525 // If SF is used, but the instruction doesn't update the SF, then we 4526 // can't do the optimization. 4527 if (NoSignFlag) 4528 return false; 4529 break; 4530 } 4531 4532 // If we're updating the condition code check if we have to reverse the 4533 // condition. 4534 if (ShouldUpdateCC) 4535 switch (OldCC) { 4536 default: 4537 return false; 4538 case X86::COND_E: 4539 ReplacementCC = NewCC; 4540 break; 4541 case X86::COND_NE: 4542 ReplacementCC = GetOppositeBranchCondition(NewCC); 4543 break; 4544 } 4545 } else if (IsSwapped) { 4546 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs 4547 // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. 4548 // We swap the condition code and synthesize the new opcode. 4549 ReplacementCC = getSwappedCondition(OldCC); 4550 if (ReplacementCC == X86::COND_INVALID) 4551 return false; 4552 ShouldUpdateCC = true; 4553 } else if (ImmDelta != 0) { 4554 unsigned BitWidth = TRI->getRegSizeInBits(*MRI->getRegClass(SrcReg)); 4555 // Shift amount for min/max constants to adjust for 8/16/32 instruction 4556 // sizes. 4557 switch (OldCC) { 4558 case X86::COND_L: // x <s (C + 1) --> x <=s C 4559 if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue) 4560 return false; 4561 ReplacementCC = X86::COND_LE; 4562 break; 4563 case X86::COND_B: // x <u (C + 1) --> x <=u C 4564 if (ImmDelta != 1 || CmpValue == 0) 4565 return false; 4566 ReplacementCC = X86::COND_BE; 4567 break; 4568 case X86::COND_GE: // x >=s (C + 1) --> x >s C 4569 if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue) 4570 return false; 4571 ReplacementCC = X86::COND_G; 4572 break; 4573 case X86::COND_AE: // x >=u (C + 1) --> x >u C 4574 if (ImmDelta != 1 || CmpValue == 0) 4575 return false; 4576 ReplacementCC = X86::COND_A; 4577 break; 4578 case X86::COND_G: // x >s (C - 1) --> x >=s C 4579 if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue) 4580 return false; 4581 ReplacementCC = X86::COND_GE; 4582 break; 4583 case X86::COND_A: // x >u (C - 1) --> x >=u C 4584 if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue) 4585 return false; 4586 ReplacementCC = X86::COND_AE; 4587 break; 4588 case X86::COND_LE: // x <=s (C - 1) --> x <s C 4589 if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue) 4590 return false; 4591 ReplacementCC = X86::COND_L; 4592 break; 4593 case X86::COND_BE: // x <=u (C - 1) --> x <u C 4594 if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue) 4595 return false; 4596 ReplacementCC = X86::COND_B; 4597 break; 4598 default: 4599 return false; 4600 } 4601 ShouldUpdateCC = true; 4602 } 4603 4604 if (ShouldUpdateCC && ReplacementCC != OldCC) { 4605 // Push the MachineInstr to OpsToUpdate. 4606 // If it is safe to remove CmpInstr, the condition code of these 4607 // instructions will be modified. 4608 OpsToUpdate.push_back(std::make_pair(&Instr, ReplacementCC)); 4609 } 4610 if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) { 4611 // It is safe to remove CmpInstr if EFLAGS is updated again or killed. 4612 FlagsMayLiveOut = false; 4613 break; 4614 } 4615 } 4616 4617 // If we have to update users but EFLAGS is live-out abort, since we cannot 4618 // easily find all of the users. 4619 if ((MI != nullptr || ShouldUpdateCC) && FlagsMayLiveOut) { 4620 for (MachineBasicBlock *Successor : CmpMBB.successors()) 4621 if (Successor->isLiveIn(X86::EFLAGS)) 4622 return false; 4623 } 4624 4625 // The instruction to be updated is either Sub or MI. 4626 assert((MI == nullptr || Sub == nullptr) && "Should not have Sub and MI set"); 4627 Sub = MI != nullptr ? MI : Sub; 4628 MachineBasicBlock *SubBB = Sub->getParent(); 4629 // Move Movr0Inst to the appropriate place before Sub. 4630 if (Movr0Inst) { 4631 // Only move within the same block so we don't accidentally move to a 4632 // block with higher execution frequency. 4633 if (&CmpMBB != SubBB) 4634 return false; 4635 // Look backwards until we find a def that doesn't use the current EFLAGS. 4636 MachineBasicBlock::reverse_iterator InsertI = Sub, 4637 InsertE = Sub->getParent()->rend(); 4638 for (; InsertI != InsertE; ++InsertI) { 4639 MachineInstr *Instr = &*InsertI; 4640 if (!Instr->readsRegister(X86::EFLAGS, TRI) && 4641 Instr->modifiesRegister(X86::EFLAGS, TRI)) { 4642 Movr0Inst->getParent()->remove(Movr0Inst); 4643 Instr->getParent()->insert(MachineBasicBlock::iterator(Instr), 4644 Movr0Inst); 4645 break; 4646 } 4647 } 4648 if (InsertI == InsertE) 4649 return false; 4650 } 4651 4652 // Make sure Sub instruction defines EFLAGS and mark the def live. 4653 MachineOperand *FlagDef = Sub->findRegisterDefOperand(X86::EFLAGS); 4654 assert(FlagDef && "Unable to locate a def EFLAGS operand"); 4655 FlagDef->setIsDead(false); 4656 4657 CmpInstr.eraseFromParent(); 4658 4659 // Modify the condition code of instructions in OpsToUpdate. 4660 for (auto &Op : OpsToUpdate) { 4661 Op.first->getOperand(Op.first->getDesc().getNumOperands() - 1) 4662 .setImm(Op.second); 4663 } 4664 // Add EFLAGS to block live-ins between CmpBB and block of flags producer. 4665 for (MachineBasicBlock *MBB = &CmpMBB; MBB != SubBB; 4666 MBB = *MBB->pred_begin()) { 4667 assert(MBB->pred_size() == 1 && "Expected exactly one predecessor"); 4668 if (!MBB->isLiveIn(X86::EFLAGS)) 4669 MBB->addLiveIn(X86::EFLAGS); 4670 } 4671 return true; 4672 } 4673 4674 /// Try to remove the load by folding it to a register 4675 /// operand at the use. We fold the load instructions if load defines a virtual 4676 /// register, the virtual register is used once in the same BB, and the 4677 /// instructions in-between do not load or store, and have no side effects. 4678 MachineInstr *X86InstrInfo::optimizeLoadInstr(MachineInstr &MI, 4679 const MachineRegisterInfo *MRI, 4680 Register &FoldAsLoadDefReg, 4681 MachineInstr *&DefMI) const { 4682 // Check whether we can move DefMI here. 4683 DefMI = MRI->getVRegDef(FoldAsLoadDefReg); 4684 assert(DefMI); 4685 bool SawStore = false; 4686 if (!DefMI->isSafeToMove(nullptr, SawStore)) 4687 return nullptr; 4688 4689 // Collect information about virtual register operands of MI. 4690 SmallVector<unsigned, 1> SrcOperandIds; 4691 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 4692 MachineOperand &MO = MI.getOperand(i); 4693 if (!MO.isReg()) 4694 continue; 4695 Register Reg = MO.getReg(); 4696 if (Reg != FoldAsLoadDefReg) 4697 continue; 4698 // Do not fold if we have a subreg use or a def. 4699 if (MO.getSubReg() || MO.isDef()) 4700 return nullptr; 4701 SrcOperandIds.push_back(i); 4702 } 4703 if (SrcOperandIds.empty()) 4704 return nullptr; 4705 4706 // Check whether we can fold the def into SrcOperandId. 4707 if (MachineInstr *FoldMI = foldMemoryOperand(MI, SrcOperandIds, *DefMI)) { 4708 FoldAsLoadDefReg = 0; 4709 return FoldMI; 4710 } 4711 4712 return nullptr; 4713 } 4714 4715 /// Expand a single-def pseudo instruction to a two-addr 4716 /// instruction with two undef reads of the register being defined. 4717 /// This is used for mapping: 4718 /// %xmm4 = V_SET0 4719 /// to: 4720 /// %xmm4 = PXORrr undef %xmm4, undef %xmm4 4721 /// 4722 static bool Expand2AddrUndef(MachineInstrBuilder &MIB, 4723 const MCInstrDesc &Desc) { 4724 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction."); 4725 Register Reg = MIB.getReg(0); 4726 MIB->setDesc(Desc); 4727 4728 // MachineInstr::addOperand() will insert explicit operands before any 4729 // implicit operands. 4730 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); 4731 // But we don't trust that. 4732 assert(MIB.getReg(1) == Reg && 4733 MIB.getReg(2) == Reg && "Misplaced operand"); 4734 return true; 4735 } 4736 4737 /// Expand a single-def pseudo instruction to a two-addr 4738 /// instruction with two %k0 reads. 4739 /// This is used for mapping: 4740 /// %k4 = K_SET1 4741 /// to: 4742 /// %k4 = KXNORrr %k0, %k0 4743 static bool Expand2AddrKreg(MachineInstrBuilder &MIB, const MCInstrDesc &Desc, 4744 Register Reg) { 4745 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction."); 4746 MIB->setDesc(Desc); 4747 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); 4748 return true; 4749 } 4750 4751 static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, 4752 bool MinusOne) { 4753 MachineBasicBlock &MBB = *MIB->getParent(); 4754 const DebugLoc &DL = MIB->getDebugLoc(); 4755 Register Reg = MIB.getReg(0); 4756 4757 // Insert the XOR. 4758 BuildMI(MBB, MIB.getInstr(), DL, TII.get(X86::XOR32rr), Reg) 4759 .addReg(Reg, RegState::Undef) 4760 .addReg(Reg, RegState::Undef); 4761 4762 // Turn the pseudo into an INC or DEC. 4763 MIB->setDesc(TII.get(MinusOne ? X86::DEC32r : X86::INC32r)); 4764 MIB.addReg(Reg); 4765 4766 return true; 4767 } 4768 4769 static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB, 4770 const TargetInstrInfo &TII, 4771 const X86Subtarget &Subtarget) { 4772 MachineBasicBlock &MBB = *MIB->getParent(); 4773 const DebugLoc &DL = MIB->getDebugLoc(); 4774 int64_t Imm = MIB->getOperand(1).getImm(); 4775 assert(Imm != 0 && "Using push/pop for 0 is not efficient."); 4776 MachineBasicBlock::iterator I = MIB.getInstr(); 4777 4778 int StackAdjustment; 4779 4780 if (Subtarget.is64Bit()) { 4781 assert(MIB->getOpcode() == X86::MOV64ImmSExti8 || 4782 MIB->getOpcode() == X86::MOV32ImmSExti8); 4783 4784 // Can't use push/pop lowering if the function might write to the red zone. 4785 X86MachineFunctionInfo *X86FI = 4786 MBB.getParent()->getInfo<X86MachineFunctionInfo>(); 4787 if (X86FI->getUsesRedZone()) { 4788 MIB->setDesc(TII.get(MIB->getOpcode() == 4789 X86::MOV32ImmSExti8 ? X86::MOV32ri : X86::MOV64ri)); 4790 return true; 4791 } 4792 4793 // 64-bit mode doesn't have 32-bit push/pop, so use 64-bit operations and 4794 // widen the register if necessary. 4795 StackAdjustment = 8; 4796 BuildMI(MBB, I, DL, TII.get(X86::PUSH64i8)).addImm(Imm); 4797 MIB->setDesc(TII.get(X86::POP64r)); 4798 MIB->getOperand(0) 4799 .setReg(getX86SubSuperRegister(MIB.getReg(0), 64)); 4800 } else { 4801 assert(MIB->getOpcode() == X86::MOV32ImmSExti8); 4802 StackAdjustment = 4; 4803 BuildMI(MBB, I, DL, TII.get(X86::PUSH32i8)).addImm(Imm); 4804 MIB->setDesc(TII.get(X86::POP32r)); 4805 } 4806 MIB->removeOperand(1); 4807 MIB->addImplicitDefUseOperands(*MBB.getParent()); 4808 4809 // Build CFI if necessary. 4810 MachineFunction &MF = *MBB.getParent(); 4811 const X86FrameLowering *TFL = Subtarget.getFrameLowering(); 4812 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 4813 bool NeedsDwarfCFI = !IsWin64Prologue && MF.needsFrameMoves(); 4814 bool EmitCFI = !TFL->hasFP(MF) && NeedsDwarfCFI; 4815 if (EmitCFI) { 4816 TFL->BuildCFI(MBB, I, DL, 4817 MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment)); 4818 TFL->BuildCFI(MBB, std::next(I), DL, 4819 MCCFIInstruction::createAdjustCfaOffset(nullptr, -StackAdjustment)); 4820 } 4821 4822 return true; 4823 } 4824 4825 // LoadStackGuard has so far only been implemented for 64-bit MachO. Different 4826 // code sequence is needed for other targets. 4827 static void expandLoadStackGuard(MachineInstrBuilder &MIB, 4828 const TargetInstrInfo &TII) { 4829 MachineBasicBlock &MBB = *MIB->getParent(); 4830 const DebugLoc &DL = MIB->getDebugLoc(); 4831 Register Reg = MIB.getReg(0); 4832 const GlobalValue *GV = 4833 cast<GlobalValue>((*MIB->memoperands_begin())->getValue()); 4834 auto Flags = MachineMemOperand::MOLoad | 4835 MachineMemOperand::MODereferenceable | 4836 MachineMemOperand::MOInvariant; 4837 MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( 4838 MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, Align(8)); 4839 MachineBasicBlock::iterator I = MIB.getInstr(); 4840 4841 BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg).addReg(X86::RIP).addImm(1) 4842 .addReg(0).addGlobalAddress(GV, 0, X86II::MO_GOTPCREL).addReg(0) 4843 .addMemOperand(MMO); 4844 MIB->setDebugLoc(DL); 4845 MIB->setDesc(TII.get(X86::MOV64rm)); 4846 MIB.addReg(Reg, RegState::Kill).addImm(1).addReg(0).addImm(0).addReg(0); 4847 } 4848 4849 static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII) { 4850 MachineBasicBlock &MBB = *MIB->getParent(); 4851 MachineFunction &MF = *MBB.getParent(); 4852 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>(); 4853 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); 4854 unsigned XorOp = 4855 MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr; 4856 MIB->setDesc(TII.get(XorOp)); 4857 MIB.addReg(TRI->getFrameRegister(MF), RegState::Undef); 4858 return true; 4859 } 4860 4861 // This is used to handle spills for 128/256-bit registers when we have AVX512, 4862 // but not VLX. If it uses an extended register we need to use an instruction 4863 // that loads the lower 128/256-bit, but is available with only AVX512F. 4864 static bool expandNOVLXLoad(MachineInstrBuilder &MIB, 4865 const TargetRegisterInfo *TRI, 4866 const MCInstrDesc &LoadDesc, 4867 const MCInstrDesc &BroadcastDesc, 4868 unsigned SubIdx) { 4869 Register DestReg = MIB.getReg(0); 4870 // Check if DestReg is XMM16-31 or YMM16-31. 4871 if (TRI->getEncodingValue(DestReg) < 16) { 4872 // We can use a normal VEX encoded load. 4873 MIB->setDesc(LoadDesc); 4874 } else { 4875 // Use a 128/256-bit VBROADCAST instruction. 4876 MIB->setDesc(BroadcastDesc); 4877 // Change the destination to a 512-bit register. 4878 DestReg = TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass); 4879 MIB->getOperand(0).setReg(DestReg); 4880 } 4881 return true; 4882 } 4883 4884 // This is used to handle spills for 128/256-bit registers when we have AVX512, 4885 // but not VLX. If it uses an extended register we need to use an instruction 4886 // that stores the lower 128/256-bit, but is available with only AVX512F. 4887 static bool expandNOVLXStore(MachineInstrBuilder &MIB, 4888 const TargetRegisterInfo *TRI, 4889 const MCInstrDesc &StoreDesc, 4890 const MCInstrDesc &ExtractDesc, 4891 unsigned SubIdx) { 4892 Register SrcReg = MIB.getReg(X86::AddrNumOperands); 4893 // Check if DestReg is XMM16-31 or YMM16-31. 4894 if (TRI->getEncodingValue(SrcReg) < 16) { 4895 // We can use a normal VEX encoded store. 4896 MIB->setDesc(StoreDesc); 4897 } else { 4898 // Use a VEXTRACTF instruction. 4899 MIB->setDesc(ExtractDesc); 4900 // Change the destination to a 512-bit register. 4901 SrcReg = TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass); 4902 MIB->getOperand(X86::AddrNumOperands).setReg(SrcReg); 4903 MIB.addImm(0x0); // Append immediate to extract from the lower bits. 4904 } 4905 4906 return true; 4907 } 4908 4909 static bool expandSHXDROT(MachineInstrBuilder &MIB, const MCInstrDesc &Desc) { 4910 MIB->setDesc(Desc); 4911 int64_t ShiftAmt = MIB->getOperand(2).getImm(); 4912 // Temporarily remove the immediate so we can add another source register. 4913 MIB->removeOperand(2); 4914 // Add the register. Don't copy the kill flag if there is one. 4915 MIB.addReg(MIB.getReg(1), 4916 getUndefRegState(MIB->getOperand(1).isUndef())); 4917 // Add back the immediate. 4918 MIB.addImm(ShiftAmt); 4919 return true; 4920 } 4921 4922 bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 4923 bool HasAVX = Subtarget.hasAVX(); 4924 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI); 4925 switch (MI.getOpcode()) { 4926 case X86::MOV32r0: 4927 return Expand2AddrUndef(MIB, get(X86::XOR32rr)); 4928 case X86::MOV32r1: 4929 return expandMOV32r1(MIB, *this, /*MinusOne=*/ false); 4930 case X86::MOV32r_1: 4931 return expandMOV32r1(MIB, *this, /*MinusOne=*/ true); 4932 case X86::MOV32ImmSExti8: 4933 case X86::MOV64ImmSExti8: 4934 return ExpandMOVImmSExti8(MIB, *this, Subtarget); 4935 case X86::SETB_C32r: 4936 return Expand2AddrUndef(MIB, get(X86::SBB32rr)); 4937 case X86::SETB_C64r: 4938 return Expand2AddrUndef(MIB, get(X86::SBB64rr)); 4939 case X86::MMX_SET0: 4940 return Expand2AddrUndef(MIB, get(X86::MMX_PXORrr)); 4941 case X86::V_SET0: 4942 case X86::FsFLD0SS: 4943 case X86::FsFLD0SD: 4944 case X86::FsFLD0SH: 4945 case X86::FsFLD0F128: 4946 return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr)); 4947 case X86::AVX_SET0: { 4948 assert(HasAVX && "AVX not supported"); 4949 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4950 Register SrcReg = MIB.getReg(0); 4951 Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); 4952 MIB->getOperand(0).setReg(XReg); 4953 Expand2AddrUndef(MIB, get(X86::VXORPSrr)); 4954 MIB.addReg(SrcReg, RegState::ImplicitDefine); 4955 return true; 4956 } 4957 case X86::AVX512_128_SET0: 4958 case X86::AVX512_FsFLD0SH: 4959 case X86::AVX512_FsFLD0SS: 4960 case X86::AVX512_FsFLD0SD: 4961 case X86::AVX512_FsFLD0F128: { 4962 bool HasVLX = Subtarget.hasVLX(); 4963 Register SrcReg = MIB.getReg(0); 4964 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4965 if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) 4966 return Expand2AddrUndef(MIB, 4967 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); 4968 // Extended register without VLX. Use a larger XOR. 4969 SrcReg = 4970 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass); 4971 MIB->getOperand(0).setReg(SrcReg); 4972 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); 4973 } 4974 case X86::AVX512_256_SET0: 4975 case X86::AVX512_512_SET0: { 4976 bool HasVLX = Subtarget.hasVLX(); 4977 Register SrcReg = MIB.getReg(0); 4978 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4979 if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) { 4980 Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); 4981 MIB->getOperand(0).setReg(XReg); 4982 Expand2AddrUndef(MIB, 4983 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); 4984 MIB.addReg(SrcReg, RegState::ImplicitDefine); 4985 return true; 4986 } 4987 if (MI.getOpcode() == X86::AVX512_256_SET0) { 4988 // No VLX so we must reference a zmm. 4989 unsigned ZReg = 4990 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass); 4991 MIB->getOperand(0).setReg(ZReg); 4992 } 4993 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); 4994 } 4995 case X86::V_SETALLONES: 4996 return Expand2AddrUndef(MIB, get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr)); 4997 case X86::AVX2_SETALLONES: 4998 return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr)); 4999 case X86::AVX1_SETALLONES: { 5000 Register Reg = MIB.getReg(0); 5001 // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS. 5002 MIB->setDesc(get(X86::VCMPPSYrri)); 5003 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xf); 5004 return true; 5005 } 5006 case X86::AVX512_512_SETALLONES: { 5007 Register Reg = MIB.getReg(0); 5008 MIB->setDesc(get(X86::VPTERNLOGDZrri)); 5009 // VPTERNLOGD needs 3 register inputs and an immediate. 5010 // 0xff will return 1s for any input. 5011 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef) 5012 .addReg(Reg, RegState::Undef).addImm(0xff); 5013 return true; 5014 } 5015 case X86::AVX512_512_SEXT_MASK_32: 5016 case X86::AVX512_512_SEXT_MASK_64: { 5017 Register Reg = MIB.getReg(0); 5018 Register MaskReg = MIB.getReg(1); 5019 unsigned MaskState = getRegState(MIB->getOperand(1)); 5020 unsigned Opc = (MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64) ? 5021 X86::VPTERNLOGQZrrikz : X86::VPTERNLOGDZrrikz; 5022 MI.removeOperand(1); 5023 MIB->setDesc(get(Opc)); 5024 // VPTERNLOG needs 3 register inputs and an immediate. 5025 // 0xff will return 1s for any input. 5026 MIB.addReg(Reg, RegState::Undef).addReg(MaskReg, MaskState) 5027 .addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xff); 5028 return true; 5029 } 5030 case X86::VMOVAPSZ128rm_NOVLX: 5031 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSrm), 5032 get(X86::VBROADCASTF32X4rm), X86::sub_xmm); 5033 case X86::VMOVUPSZ128rm_NOVLX: 5034 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSrm), 5035 get(X86::VBROADCASTF32X4rm), X86::sub_xmm); 5036 case X86::VMOVAPSZ256rm_NOVLX: 5037 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSYrm), 5038 get(X86::VBROADCASTF64X4rm), X86::sub_ymm); 5039 case X86::VMOVUPSZ256rm_NOVLX: 5040 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSYrm), 5041 get(X86::VBROADCASTF64X4rm), X86::sub_ymm); 5042 case X86::VMOVAPSZ128mr_NOVLX: 5043 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSmr), 5044 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); 5045 case X86::VMOVUPSZ128mr_NOVLX: 5046 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSmr), 5047 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); 5048 case X86::VMOVAPSZ256mr_NOVLX: 5049 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSYmr), 5050 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); 5051 case X86::VMOVUPSZ256mr_NOVLX: 5052 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSYmr), 5053 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); 5054 case X86::MOV32ri64: { 5055 Register Reg = MIB.getReg(0); 5056 Register Reg32 = RI.getSubReg(Reg, X86::sub_32bit); 5057 MI.setDesc(get(X86::MOV32ri)); 5058 MIB->getOperand(0).setReg(Reg32); 5059 MIB.addReg(Reg, RegState::ImplicitDefine); 5060 return true; 5061 } 5062 5063 // KNL does not recognize dependency-breaking idioms for mask registers, 5064 // so kxnor %k1, %k1, %k2 has a RAW dependence on %k1. 5065 // Using %k0 as the undef input register is a performance heuristic based 5066 // on the assumption that %k0 is used less frequently than the other mask 5067 // registers, since it is not usable as a write mask. 5068 // FIXME: A more advanced approach would be to choose the best input mask 5069 // register based on context. 5070 case X86::KSET0W: return Expand2AddrKreg(MIB, get(X86::KXORWrr), X86::K0); 5071 case X86::KSET0D: return Expand2AddrKreg(MIB, get(X86::KXORDrr), X86::K0); 5072 case X86::KSET0Q: return Expand2AddrKreg(MIB, get(X86::KXORQrr), X86::K0); 5073 case X86::KSET1W: return Expand2AddrKreg(MIB, get(X86::KXNORWrr), X86::K0); 5074 case X86::KSET1D: return Expand2AddrKreg(MIB, get(X86::KXNORDrr), X86::K0); 5075 case X86::KSET1Q: return Expand2AddrKreg(MIB, get(X86::KXNORQrr), X86::K0); 5076 case TargetOpcode::LOAD_STACK_GUARD: 5077 expandLoadStackGuard(MIB, *this); 5078 return true; 5079 case X86::XOR64_FP: 5080 case X86::XOR32_FP: 5081 return expandXorFP(MIB, *this); 5082 case X86::SHLDROT32ri: return expandSHXDROT(MIB, get(X86::SHLD32rri8)); 5083 case X86::SHLDROT64ri: return expandSHXDROT(MIB, get(X86::SHLD64rri8)); 5084 case X86::SHRDROT32ri: return expandSHXDROT(MIB, get(X86::SHRD32rri8)); 5085 case X86::SHRDROT64ri: return expandSHXDROT(MIB, get(X86::SHRD64rri8)); 5086 case X86::ADD8rr_DB: MIB->setDesc(get(X86::OR8rr)); break; 5087 case X86::ADD16rr_DB: MIB->setDesc(get(X86::OR16rr)); break; 5088 case X86::ADD32rr_DB: MIB->setDesc(get(X86::OR32rr)); break; 5089 case X86::ADD64rr_DB: MIB->setDesc(get(X86::OR64rr)); break; 5090 case X86::ADD8ri_DB: MIB->setDesc(get(X86::OR8ri)); break; 5091 case X86::ADD16ri_DB: MIB->setDesc(get(X86::OR16ri)); break; 5092 case X86::ADD32ri_DB: MIB->setDesc(get(X86::OR32ri)); break; 5093 case X86::ADD64ri32_DB: MIB->setDesc(get(X86::OR64ri32)); break; 5094 case X86::ADD16ri8_DB: MIB->setDesc(get(X86::OR16ri8)); break; 5095 case X86::ADD32ri8_DB: MIB->setDesc(get(X86::OR32ri8)); break; 5096 case X86::ADD64ri8_DB: MIB->setDesc(get(X86::OR64ri8)); break; 5097 } 5098 return false; 5099 } 5100 5101 /// Return true for all instructions that only update 5102 /// the first 32 or 64-bits of the destination register and leave the rest 5103 /// unmodified. This can be used to avoid folding loads if the instructions 5104 /// only update part of the destination register, and the non-updated part is 5105 /// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these 5106 /// instructions breaks the partial register dependency and it can improve 5107 /// performance. e.g.: 5108 /// 5109 /// movss (%rdi), %xmm0 5110 /// cvtss2sd %xmm0, %xmm0 5111 /// 5112 /// Instead of 5113 /// cvtss2sd (%rdi), %xmm0 5114 /// 5115 /// FIXME: This should be turned into a TSFlags. 5116 /// 5117 static bool hasPartialRegUpdate(unsigned Opcode, 5118 const X86Subtarget &Subtarget, 5119 bool ForLoadFold = false) { 5120 switch (Opcode) { 5121 case X86::CVTSI2SSrr: 5122 case X86::CVTSI2SSrm: 5123 case X86::CVTSI642SSrr: 5124 case X86::CVTSI642SSrm: 5125 case X86::CVTSI2SDrr: 5126 case X86::CVTSI2SDrm: 5127 case X86::CVTSI642SDrr: 5128 case X86::CVTSI642SDrm: 5129 // Load folding won't effect the undef register update since the input is 5130 // a GPR. 5131 return !ForLoadFold; 5132 case X86::CVTSD2SSrr: 5133 case X86::CVTSD2SSrm: 5134 case X86::CVTSS2SDrr: 5135 case X86::CVTSS2SDrm: 5136 case X86::MOVHPDrm: 5137 case X86::MOVHPSrm: 5138 case X86::MOVLPDrm: 5139 case X86::MOVLPSrm: 5140 case X86::RCPSSr: 5141 case X86::RCPSSm: 5142 case X86::RCPSSr_Int: 5143 case X86::RCPSSm_Int: 5144 case X86::ROUNDSDr: 5145 case X86::ROUNDSDm: 5146 case X86::ROUNDSSr: 5147 case X86::ROUNDSSm: 5148 case X86::RSQRTSSr: 5149 case X86::RSQRTSSm: 5150 case X86::RSQRTSSr_Int: 5151 case X86::RSQRTSSm_Int: 5152 case X86::SQRTSSr: 5153 case X86::SQRTSSm: 5154 case X86::SQRTSSr_Int: 5155 case X86::SQRTSSm_Int: 5156 case X86::SQRTSDr: 5157 case X86::SQRTSDm: 5158 case X86::SQRTSDr_Int: 5159 case X86::SQRTSDm_Int: 5160 return true; 5161 case X86::VFCMULCPHZ128rm: 5162 case X86::VFCMULCPHZ128rmb: 5163 case X86::VFCMULCPHZ128rmbkz: 5164 case X86::VFCMULCPHZ128rmkz: 5165 case X86::VFCMULCPHZ128rr: 5166 case X86::VFCMULCPHZ128rrkz: 5167 case X86::VFCMULCPHZ256rm: 5168 case X86::VFCMULCPHZ256rmb: 5169 case X86::VFCMULCPHZ256rmbkz: 5170 case X86::VFCMULCPHZ256rmkz: 5171 case X86::VFCMULCPHZ256rr: 5172 case X86::VFCMULCPHZ256rrkz: 5173 case X86::VFCMULCPHZrm: 5174 case X86::VFCMULCPHZrmb: 5175 case X86::VFCMULCPHZrmbkz: 5176 case X86::VFCMULCPHZrmkz: 5177 case X86::VFCMULCPHZrr: 5178 case X86::VFCMULCPHZrrb: 5179 case X86::VFCMULCPHZrrbkz: 5180 case X86::VFCMULCPHZrrkz: 5181 case X86::VFMULCPHZ128rm: 5182 case X86::VFMULCPHZ128rmb: 5183 case X86::VFMULCPHZ128rmbkz: 5184 case X86::VFMULCPHZ128rmkz: 5185 case X86::VFMULCPHZ128rr: 5186 case X86::VFMULCPHZ128rrkz: 5187 case X86::VFMULCPHZ256rm: 5188 case X86::VFMULCPHZ256rmb: 5189 case X86::VFMULCPHZ256rmbkz: 5190 case X86::VFMULCPHZ256rmkz: 5191 case X86::VFMULCPHZ256rr: 5192 case X86::VFMULCPHZ256rrkz: 5193 case X86::VFMULCPHZrm: 5194 case X86::VFMULCPHZrmb: 5195 case X86::VFMULCPHZrmbkz: 5196 case X86::VFMULCPHZrmkz: 5197 case X86::VFMULCPHZrr: 5198 case X86::VFMULCPHZrrb: 5199 case X86::VFMULCPHZrrbkz: 5200 case X86::VFMULCPHZrrkz: 5201 case X86::VFCMULCSHZrm: 5202 case X86::VFCMULCSHZrmkz: 5203 case X86::VFCMULCSHZrr: 5204 case X86::VFCMULCSHZrrb: 5205 case X86::VFCMULCSHZrrbkz: 5206 case X86::VFCMULCSHZrrkz: 5207 case X86::VFMULCSHZrm: 5208 case X86::VFMULCSHZrmkz: 5209 case X86::VFMULCSHZrr: 5210 case X86::VFMULCSHZrrb: 5211 case X86::VFMULCSHZrrbkz: 5212 case X86::VFMULCSHZrrkz: 5213 return Subtarget.hasMULCFalseDeps(); 5214 case X86::VPERMDYrm: 5215 case X86::VPERMDYrr: 5216 case X86::VPERMQYmi: 5217 case X86::VPERMQYri: 5218 case X86::VPERMPSYrm: 5219 case X86::VPERMPSYrr: 5220 case X86::VPERMPDYmi: 5221 case X86::VPERMPDYri: 5222 case X86::VPERMDZ256rm: 5223 case X86::VPERMDZ256rmb: 5224 case X86::VPERMDZ256rmbkz: 5225 case X86::VPERMDZ256rmkz: 5226 case X86::VPERMDZ256rr: 5227 case X86::VPERMDZ256rrkz: 5228 case X86::VPERMDZrm: 5229 case X86::VPERMDZrmb: 5230 case X86::VPERMDZrmbkz: 5231 case X86::VPERMDZrmkz: 5232 case X86::VPERMDZrr: 5233 case X86::VPERMDZrrkz: 5234 case X86::VPERMQZ256mbi: 5235 case X86::VPERMQZ256mbikz: 5236 case X86::VPERMQZ256mi: 5237 case X86::VPERMQZ256mikz: 5238 case X86::VPERMQZ256ri: 5239 case X86::VPERMQZ256rikz: 5240 case X86::VPERMQZ256rm: 5241 case X86::VPERMQZ256rmb: 5242 case X86::VPERMQZ256rmbkz: 5243 case X86::VPERMQZ256rmkz: 5244 case X86::VPERMQZ256rr: 5245 case X86::VPERMQZ256rrkz: 5246 case X86::VPERMQZmbi: 5247 case X86::VPERMQZmbikz: 5248 case X86::VPERMQZmi: 5249 case X86::VPERMQZmikz: 5250 case X86::VPERMQZri: 5251 case X86::VPERMQZrikz: 5252 case X86::VPERMQZrm: 5253 case X86::VPERMQZrmb: 5254 case X86::VPERMQZrmbkz: 5255 case X86::VPERMQZrmkz: 5256 case X86::VPERMQZrr: 5257 case X86::VPERMQZrrkz: 5258 case X86::VPERMPSZ256rm: 5259 case X86::VPERMPSZ256rmb: 5260 case X86::VPERMPSZ256rmbkz: 5261 case X86::VPERMPSZ256rmkz: 5262 case X86::VPERMPSZ256rr: 5263 case X86::VPERMPSZ256rrkz: 5264 case X86::VPERMPSZrm: 5265 case X86::VPERMPSZrmb: 5266 case X86::VPERMPSZrmbkz: 5267 case X86::VPERMPSZrmkz: 5268 case X86::VPERMPSZrr: 5269 case X86::VPERMPSZrrkz: 5270 case X86::VPERMPDZ256mbi: 5271 case X86::VPERMPDZ256mbikz: 5272 case X86::VPERMPDZ256mi: 5273 case X86::VPERMPDZ256mikz: 5274 case X86::VPERMPDZ256ri: 5275 case X86::VPERMPDZ256rikz: 5276 case X86::VPERMPDZ256rm: 5277 case X86::VPERMPDZ256rmb: 5278 case X86::VPERMPDZ256rmbkz: 5279 case X86::VPERMPDZ256rmkz: 5280 case X86::VPERMPDZ256rr: 5281 case X86::VPERMPDZ256rrkz: 5282 case X86::VPERMPDZmbi: 5283 case X86::VPERMPDZmbikz: 5284 case X86::VPERMPDZmi: 5285 case X86::VPERMPDZmikz: 5286 case X86::VPERMPDZri: 5287 case X86::VPERMPDZrikz: 5288 case X86::VPERMPDZrm: 5289 case X86::VPERMPDZrmb: 5290 case X86::VPERMPDZrmbkz: 5291 case X86::VPERMPDZrmkz: 5292 case X86::VPERMPDZrr: 5293 case X86::VPERMPDZrrkz: 5294 return Subtarget.hasPERMFalseDeps(); 5295 case X86::VRANGEPDZ128rmbi: 5296 case X86::VRANGEPDZ128rmbikz: 5297 case X86::VRANGEPDZ128rmi: 5298 case X86::VRANGEPDZ128rmikz: 5299 case X86::VRANGEPDZ128rri: 5300 case X86::VRANGEPDZ128rrikz: 5301 case X86::VRANGEPDZ256rmbi: 5302 case X86::VRANGEPDZ256rmbikz: 5303 case X86::VRANGEPDZ256rmi: 5304 case X86::VRANGEPDZ256rmikz: 5305 case X86::VRANGEPDZ256rri: 5306 case X86::VRANGEPDZ256rrikz: 5307 case X86::VRANGEPDZrmbi: 5308 case X86::VRANGEPDZrmbikz: 5309 case X86::VRANGEPDZrmi: 5310 case X86::VRANGEPDZrmikz: 5311 case X86::VRANGEPDZrri: 5312 case X86::VRANGEPDZrrib: 5313 case X86::VRANGEPDZrribkz: 5314 case X86::VRANGEPDZrrikz: 5315 case X86::VRANGEPSZ128rmbi: 5316 case X86::VRANGEPSZ128rmbikz: 5317 case X86::VRANGEPSZ128rmi: 5318 case X86::VRANGEPSZ128rmikz: 5319 case X86::VRANGEPSZ128rri: 5320 case X86::VRANGEPSZ128rrikz: 5321 case X86::VRANGEPSZ256rmbi: 5322 case X86::VRANGEPSZ256rmbikz: 5323 case X86::VRANGEPSZ256rmi: 5324 case X86::VRANGEPSZ256rmikz: 5325 case X86::VRANGEPSZ256rri: 5326 case X86::VRANGEPSZ256rrikz: 5327 case X86::VRANGEPSZrmbi: 5328 case X86::VRANGEPSZrmbikz: 5329 case X86::VRANGEPSZrmi: 5330 case X86::VRANGEPSZrmikz: 5331 case X86::VRANGEPSZrri: 5332 case X86::VRANGEPSZrrib: 5333 case X86::VRANGEPSZrribkz: 5334 case X86::VRANGEPSZrrikz: 5335 case X86::VRANGESDZrmi: 5336 case X86::VRANGESDZrmikz: 5337 case X86::VRANGESDZrri: 5338 case X86::VRANGESDZrrib: 5339 case X86::VRANGESDZrribkz: 5340 case X86::VRANGESDZrrikz: 5341 case X86::VRANGESSZrmi: 5342 case X86::VRANGESSZrmikz: 5343 case X86::VRANGESSZrri: 5344 case X86::VRANGESSZrrib: 5345 case X86::VRANGESSZrribkz: 5346 case X86::VRANGESSZrrikz: 5347 return Subtarget.hasRANGEFalseDeps(); 5348 case X86::VGETMANTSSZrmi: 5349 case X86::VGETMANTSSZrmikz: 5350 case X86::VGETMANTSSZrri: 5351 case X86::VGETMANTSSZrrib: 5352 case X86::VGETMANTSSZrribkz: 5353 case X86::VGETMANTSSZrrikz: 5354 case X86::VGETMANTSDZrmi: 5355 case X86::VGETMANTSDZrmikz: 5356 case X86::VGETMANTSDZrri: 5357 case X86::VGETMANTSDZrrib: 5358 case X86::VGETMANTSDZrribkz: 5359 case X86::VGETMANTSDZrrikz: 5360 case X86::VGETMANTSHZrmi: 5361 case X86::VGETMANTSHZrmikz: 5362 case X86::VGETMANTSHZrri: 5363 case X86::VGETMANTSHZrrib: 5364 case X86::VGETMANTSHZrribkz: 5365 case X86::VGETMANTSHZrrikz: 5366 case X86::VGETMANTPSZ128rmbi: 5367 case X86::VGETMANTPSZ128rmbikz: 5368 case X86::VGETMANTPSZ128rmi: 5369 case X86::VGETMANTPSZ128rmikz: 5370 case X86::VGETMANTPSZ256rmbi: 5371 case X86::VGETMANTPSZ256rmbikz: 5372 case X86::VGETMANTPSZ256rmi: 5373 case X86::VGETMANTPSZ256rmikz: 5374 case X86::VGETMANTPSZrmbi: 5375 case X86::VGETMANTPSZrmbikz: 5376 case X86::VGETMANTPSZrmi: 5377 case X86::VGETMANTPSZrmikz: 5378 case X86::VGETMANTPDZ128rmbi: 5379 case X86::VGETMANTPDZ128rmbikz: 5380 case X86::VGETMANTPDZ128rmi: 5381 case X86::VGETMANTPDZ128rmikz: 5382 case X86::VGETMANTPDZ256rmbi: 5383 case X86::VGETMANTPDZ256rmbikz: 5384 case X86::VGETMANTPDZ256rmi: 5385 case X86::VGETMANTPDZ256rmikz: 5386 case X86::VGETMANTPDZrmbi: 5387 case X86::VGETMANTPDZrmbikz: 5388 case X86::VGETMANTPDZrmi: 5389 case X86::VGETMANTPDZrmikz: 5390 return Subtarget.hasGETMANTFalseDeps(); 5391 case X86::VPMULLQZ128rm: 5392 case X86::VPMULLQZ128rmb: 5393 case X86::VPMULLQZ128rmbkz: 5394 case X86::VPMULLQZ128rmkz: 5395 case X86::VPMULLQZ128rr: 5396 case X86::VPMULLQZ128rrkz: 5397 case X86::VPMULLQZ256rm: 5398 case X86::VPMULLQZ256rmb: 5399 case X86::VPMULLQZ256rmbkz: 5400 case X86::VPMULLQZ256rmkz: 5401 case X86::VPMULLQZ256rr: 5402 case X86::VPMULLQZ256rrkz: 5403 case X86::VPMULLQZrm: 5404 case X86::VPMULLQZrmb: 5405 case X86::VPMULLQZrmbkz: 5406 case X86::VPMULLQZrmkz: 5407 case X86::VPMULLQZrr: 5408 case X86::VPMULLQZrrkz: 5409 return Subtarget.hasMULLQFalseDeps(); 5410 // GPR 5411 case X86::POPCNT32rm: 5412 case X86::POPCNT32rr: 5413 case X86::POPCNT64rm: 5414 case X86::POPCNT64rr: 5415 return Subtarget.hasPOPCNTFalseDeps(); 5416 case X86::LZCNT32rm: 5417 case X86::LZCNT32rr: 5418 case X86::LZCNT64rm: 5419 case X86::LZCNT64rr: 5420 case X86::TZCNT32rm: 5421 case X86::TZCNT32rr: 5422 case X86::TZCNT64rm: 5423 case X86::TZCNT64rr: 5424 return Subtarget.hasLZCNTFalseDeps(); 5425 } 5426 5427 return false; 5428 } 5429 5430 /// Inform the BreakFalseDeps pass how many idle 5431 /// instructions we would like before a partial register update. 5432 unsigned X86InstrInfo::getPartialRegUpdateClearance( 5433 const MachineInstr &MI, unsigned OpNum, 5434 const TargetRegisterInfo *TRI) const { 5435 if (OpNum != 0 || !hasPartialRegUpdate(MI.getOpcode(), Subtarget)) 5436 return 0; 5437 5438 // If MI is marked as reading Reg, the partial register update is wanted. 5439 const MachineOperand &MO = MI.getOperand(0); 5440 Register Reg = MO.getReg(); 5441 if (Reg.isVirtual()) { 5442 if (MO.readsReg() || MI.readsVirtualRegister(Reg)) 5443 return 0; 5444 } else { 5445 if (MI.readsRegister(Reg, TRI)) 5446 return 0; 5447 } 5448 5449 // If any instructions in the clearance range are reading Reg, insert a 5450 // dependency breaking instruction, which is inexpensive and is likely to 5451 // be hidden in other instruction's cycles. 5452 return PartialRegUpdateClearance; 5453 } 5454 5455 // Return true for any instruction the copies the high bits of the first source 5456 // operand into the unused high bits of the destination operand. 5457 // Also returns true for instructions that have two inputs where one may 5458 // be undef and we want it to use the same register as the other input. 5459 static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum, 5460 bool ForLoadFold = false) { 5461 // Set the OpNum parameter to the first source operand. 5462 switch (Opcode) { 5463 case X86::MMX_PUNPCKHBWrr: 5464 case X86::MMX_PUNPCKHWDrr: 5465 case X86::MMX_PUNPCKHDQrr: 5466 case X86::MMX_PUNPCKLBWrr: 5467 case X86::MMX_PUNPCKLWDrr: 5468 case X86::MMX_PUNPCKLDQrr: 5469 case X86::MOVHLPSrr: 5470 case X86::PACKSSWBrr: 5471 case X86::PACKUSWBrr: 5472 case X86::PACKSSDWrr: 5473 case X86::PACKUSDWrr: 5474 case X86::PUNPCKHBWrr: 5475 case X86::PUNPCKLBWrr: 5476 case X86::PUNPCKHWDrr: 5477 case X86::PUNPCKLWDrr: 5478 case X86::PUNPCKHDQrr: 5479 case X86::PUNPCKLDQrr: 5480 case X86::PUNPCKHQDQrr: 5481 case X86::PUNPCKLQDQrr: 5482 case X86::SHUFPDrri: 5483 case X86::SHUFPSrri: 5484 // These instructions are sometimes used with an undef first or second 5485 // source. Return true here so BreakFalseDeps will assign this source to the 5486 // same register as the first source to avoid a false dependency. 5487 // Operand 1 of these instructions is tied so they're separate from their 5488 // VEX counterparts. 5489 return OpNum == 2 && !ForLoadFold; 5490 5491 case X86::VMOVLHPSrr: 5492 case X86::VMOVLHPSZrr: 5493 case X86::VPACKSSWBrr: 5494 case X86::VPACKUSWBrr: 5495 case X86::VPACKSSDWrr: 5496 case X86::VPACKUSDWrr: 5497 case X86::VPACKSSWBZ128rr: 5498 case X86::VPACKUSWBZ128rr: 5499 case X86::VPACKSSDWZ128rr: 5500 case X86::VPACKUSDWZ128rr: 5501 case X86::VPERM2F128rr: 5502 case X86::VPERM2I128rr: 5503 case X86::VSHUFF32X4Z256rri: 5504 case X86::VSHUFF32X4Zrri: 5505 case X86::VSHUFF64X2Z256rri: 5506 case X86::VSHUFF64X2Zrri: 5507 case X86::VSHUFI32X4Z256rri: 5508 case X86::VSHUFI32X4Zrri: 5509 case X86::VSHUFI64X2Z256rri: 5510 case X86::VSHUFI64X2Zrri: 5511 case X86::VPUNPCKHBWrr: 5512 case X86::VPUNPCKLBWrr: 5513 case X86::VPUNPCKHBWYrr: 5514 case X86::VPUNPCKLBWYrr: 5515 case X86::VPUNPCKHBWZ128rr: 5516 case X86::VPUNPCKLBWZ128rr: 5517 case X86::VPUNPCKHBWZ256rr: 5518 case X86::VPUNPCKLBWZ256rr: 5519 case X86::VPUNPCKHBWZrr: 5520 case X86::VPUNPCKLBWZrr: 5521 case X86::VPUNPCKHWDrr: 5522 case X86::VPUNPCKLWDrr: 5523 case X86::VPUNPCKHWDYrr: 5524 case X86::VPUNPCKLWDYrr: 5525 case X86::VPUNPCKHWDZ128rr: 5526 case X86::VPUNPCKLWDZ128rr: 5527 case X86::VPUNPCKHWDZ256rr: 5528 case X86::VPUNPCKLWDZ256rr: 5529 case X86::VPUNPCKHWDZrr: 5530 case X86::VPUNPCKLWDZrr: 5531 case X86::VPUNPCKHDQrr: 5532 case X86::VPUNPCKLDQrr: 5533 case X86::VPUNPCKHDQYrr: 5534 case X86::VPUNPCKLDQYrr: 5535 case X86::VPUNPCKHDQZ128rr: 5536 case X86::VPUNPCKLDQZ128rr: 5537 case X86::VPUNPCKHDQZ256rr: 5538 case X86::VPUNPCKLDQZ256rr: 5539 case X86::VPUNPCKHDQZrr: 5540 case X86::VPUNPCKLDQZrr: 5541 case X86::VPUNPCKHQDQrr: 5542 case X86::VPUNPCKLQDQrr: 5543 case X86::VPUNPCKHQDQYrr: 5544 case X86::VPUNPCKLQDQYrr: 5545 case X86::VPUNPCKHQDQZ128rr: 5546 case X86::VPUNPCKLQDQZ128rr: 5547 case X86::VPUNPCKHQDQZ256rr: 5548 case X86::VPUNPCKLQDQZ256rr: 5549 case X86::VPUNPCKHQDQZrr: 5550 case X86::VPUNPCKLQDQZrr: 5551 // These instructions are sometimes used with an undef first or second 5552 // source. Return true here so BreakFalseDeps will assign this source to the 5553 // same register as the first source to avoid a false dependency. 5554 return (OpNum == 1 || OpNum == 2) && !ForLoadFold; 5555 5556 case X86::VCVTSI2SSrr: 5557 case X86::VCVTSI2SSrm: 5558 case X86::VCVTSI2SSrr_Int: 5559 case X86::VCVTSI2SSrm_Int: 5560 case X86::VCVTSI642SSrr: 5561 case X86::VCVTSI642SSrm: 5562 case X86::VCVTSI642SSrr_Int: 5563 case X86::VCVTSI642SSrm_Int: 5564 case X86::VCVTSI2SDrr: 5565 case X86::VCVTSI2SDrm: 5566 case X86::VCVTSI2SDrr_Int: 5567 case X86::VCVTSI2SDrm_Int: 5568 case X86::VCVTSI642SDrr: 5569 case X86::VCVTSI642SDrm: 5570 case X86::VCVTSI642SDrr_Int: 5571 case X86::VCVTSI642SDrm_Int: 5572 // AVX-512 5573 case X86::VCVTSI2SSZrr: 5574 case X86::VCVTSI2SSZrm: 5575 case X86::VCVTSI2SSZrr_Int: 5576 case X86::VCVTSI2SSZrrb_Int: 5577 case X86::VCVTSI2SSZrm_Int: 5578 case X86::VCVTSI642SSZrr: 5579 case X86::VCVTSI642SSZrm: 5580 case X86::VCVTSI642SSZrr_Int: 5581 case X86::VCVTSI642SSZrrb_Int: 5582 case X86::VCVTSI642SSZrm_Int: 5583 case X86::VCVTSI2SDZrr: 5584 case X86::VCVTSI2SDZrm: 5585 case X86::VCVTSI2SDZrr_Int: 5586 case X86::VCVTSI2SDZrm_Int: 5587 case X86::VCVTSI642SDZrr: 5588 case X86::VCVTSI642SDZrm: 5589 case X86::VCVTSI642SDZrr_Int: 5590 case X86::VCVTSI642SDZrrb_Int: 5591 case X86::VCVTSI642SDZrm_Int: 5592 case X86::VCVTUSI2SSZrr: 5593 case X86::VCVTUSI2SSZrm: 5594 case X86::VCVTUSI2SSZrr_Int: 5595 case X86::VCVTUSI2SSZrrb_Int: 5596 case X86::VCVTUSI2SSZrm_Int: 5597 case X86::VCVTUSI642SSZrr: 5598 case X86::VCVTUSI642SSZrm: 5599 case X86::VCVTUSI642SSZrr_Int: 5600 case X86::VCVTUSI642SSZrrb_Int: 5601 case X86::VCVTUSI642SSZrm_Int: 5602 case X86::VCVTUSI2SDZrr: 5603 case X86::VCVTUSI2SDZrm: 5604 case X86::VCVTUSI2SDZrr_Int: 5605 case X86::VCVTUSI2SDZrm_Int: 5606 case X86::VCVTUSI642SDZrr: 5607 case X86::VCVTUSI642SDZrm: 5608 case X86::VCVTUSI642SDZrr_Int: 5609 case X86::VCVTUSI642SDZrrb_Int: 5610 case X86::VCVTUSI642SDZrm_Int: 5611 case X86::VCVTSI2SHZrr: 5612 case X86::VCVTSI2SHZrm: 5613 case X86::VCVTSI2SHZrr_Int: 5614 case X86::VCVTSI2SHZrrb_Int: 5615 case X86::VCVTSI2SHZrm_Int: 5616 case X86::VCVTSI642SHZrr: 5617 case X86::VCVTSI642SHZrm: 5618 case X86::VCVTSI642SHZrr_Int: 5619 case X86::VCVTSI642SHZrrb_Int: 5620 case X86::VCVTSI642SHZrm_Int: 5621 case X86::VCVTUSI2SHZrr: 5622 case X86::VCVTUSI2SHZrm: 5623 case X86::VCVTUSI2SHZrr_Int: 5624 case X86::VCVTUSI2SHZrrb_Int: 5625 case X86::VCVTUSI2SHZrm_Int: 5626 case X86::VCVTUSI642SHZrr: 5627 case X86::VCVTUSI642SHZrm: 5628 case X86::VCVTUSI642SHZrr_Int: 5629 case X86::VCVTUSI642SHZrrb_Int: 5630 case X86::VCVTUSI642SHZrm_Int: 5631 // Load folding won't effect the undef register update since the input is 5632 // a GPR. 5633 return OpNum == 1 && !ForLoadFold; 5634 case X86::VCVTSD2SSrr: 5635 case X86::VCVTSD2SSrm: 5636 case X86::VCVTSD2SSrr_Int: 5637 case X86::VCVTSD2SSrm_Int: 5638 case X86::VCVTSS2SDrr: 5639 case X86::VCVTSS2SDrm: 5640 case X86::VCVTSS2SDrr_Int: 5641 case X86::VCVTSS2SDrm_Int: 5642 case X86::VRCPSSr: 5643 case X86::VRCPSSr_Int: 5644 case X86::VRCPSSm: 5645 case X86::VRCPSSm_Int: 5646 case X86::VROUNDSDr: 5647 case X86::VROUNDSDm: 5648 case X86::VROUNDSDr_Int: 5649 case X86::VROUNDSDm_Int: 5650 case X86::VROUNDSSr: 5651 case X86::VROUNDSSm: 5652 case X86::VROUNDSSr_Int: 5653 case X86::VROUNDSSm_Int: 5654 case X86::VRSQRTSSr: 5655 case X86::VRSQRTSSr_Int: 5656 case X86::VRSQRTSSm: 5657 case X86::VRSQRTSSm_Int: 5658 case X86::VSQRTSSr: 5659 case X86::VSQRTSSr_Int: 5660 case X86::VSQRTSSm: 5661 case X86::VSQRTSSm_Int: 5662 case X86::VSQRTSDr: 5663 case X86::VSQRTSDr_Int: 5664 case X86::VSQRTSDm: 5665 case X86::VSQRTSDm_Int: 5666 // AVX-512 5667 case X86::VCVTSD2SSZrr: 5668 case X86::VCVTSD2SSZrr_Int: 5669 case X86::VCVTSD2SSZrrb_Int: 5670 case X86::VCVTSD2SSZrm: 5671 case X86::VCVTSD2SSZrm_Int: 5672 case X86::VCVTSS2SDZrr: 5673 case X86::VCVTSS2SDZrr_Int: 5674 case X86::VCVTSS2SDZrrb_Int: 5675 case X86::VCVTSS2SDZrm: 5676 case X86::VCVTSS2SDZrm_Int: 5677 case X86::VGETEXPSDZr: 5678 case X86::VGETEXPSDZrb: 5679 case X86::VGETEXPSDZm: 5680 case X86::VGETEXPSSZr: 5681 case X86::VGETEXPSSZrb: 5682 case X86::VGETEXPSSZm: 5683 case X86::VGETMANTSDZrri: 5684 case X86::VGETMANTSDZrrib: 5685 case X86::VGETMANTSDZrmi: 5686 case X86::VGETMANTSSZrri: 5687 case X86::VGETMANTSSZrrib: 5688 case X86::VGETMANTSSZrmi: 5689 case X86::VRNDSCALESDZr: 5690 case X86::VRNDSCALESDZr_Int: 5691 case X86::VRNDSCALESDZrb_Int: 5692 case X86::VRNDSCALESDZm: 5693 case X86::VRNDSCALESDZm_Int: 5694 case X86::VRNDSCALESSZr: 5695 case X86::VRNDSCALESSZr_Int: 5696 case X86::VRNDSCALESSZrb_Int: 5697 case X86::VRNDSCALESSZm: 5698 case X86::VRNDSCALESSZm_Int: 5699 case X86::VRCP14SDZrr: 5700 case X86::VRCP14SDZrm: 5701 case X86::VRCP14SSZrr: 5702 case X86::VRCP14SSZrm: 5703 case X86::VRCPSHZrr: 5704 case X86::VRCPSHZrm: 5705 case X86::VRSQRTSHZrr: 5706 case X86::VRSQRTSHZrm: 5707 case X86::VREDUCESHZrmi: 5708 case X86::VREDUCESHZrri: 5709 case X86::VREDUCESHZrrib: 5710 case X86::VGETEXPSHZr: 5711 case X86::VGETEXPSHZrb: 5712 case X86::VGETEXPSHZm: 5713 case X86::VGETMANTSHZrri: 5714 case X86::VGETMANTSHZrrib: 5715 case X86::VGETMANTSHZrmi: 5716 case X86::VRNDSCALESHZr: 5717 case X86::VRNDSCALESHZr_Int: 5718 case X86::VRNDSCALESHZrb_Int: 5719 case X86::VRNDSCALESHZm: 5720 case X86::VRNDSCALESHZm_Int: 5721 case X86::VSQRTSHZr: 5722 case X86::VSQRTSHZr_Int: 5723 case X86::VSQRTSHZrb_Int: 5724 case X86::VSQRTSHZm: 5725 case X86::VSQRTSHZm_Int: 5726 case X86::VRCP28SDZr: 5727 case X86::VRCP28SDZrb: 5728 case X86::VRCP28SDZm: 5729 case X86::VRCP28SSZr: 5730 case X86::VRCP28SSZrb: 5731 case X86::VRCP28SSZm: 5732 case X86::VREDUCESSZrmi: 5733 case X86::VREDUCESSZrri: 5734 case X86::VREDUCESSZrrib: 5735 case X86::VRSQRT14SDZrr: 5736 case X86::VRSQRT14SDZrm: 5737 case X86::VRSQRT14SSZrr: 5738 case X86::VRSQRT14SSZrm: 5739 case X86::VRSQRT28SDZr: 5740 case X86::VRSQRT28SDZrb: 5741 case X86::VRSQRT28SDZm: 5742 case X86::VRSQRT28SSZr: 5743 case X86::VRSQRT28SSZrb: 5744 case X86::VRSQRT28SSZm: 5745 case X86::VSQRTSSZr: 5746 case X86::VSQRTSSZr_Int: 5747 case X86::VSQRTSSZrb_Int: 5748 case X86::VSQRTSSZm: 5749 case X86::VSQRTSSZm_Int: 5750 case X86::VSQRTSDZr: 5751 case X86::VSQRTSDZr_Int: 5752 case X86::VSQRTSDZrb_Int: 5753 case X86::VSQRTSDZm: 5754 case X86::VSQRTSDZm_Int: 5755 case X86::VCVTSD2SHZrr: 5756 case X86::VCVTSD2SHZrr_Int: 5757 case X86::VCVTSD2SHZrrb_Int: 5758 case X86::VCVTSD2SHZrm: 5759 case X86::VCVTSD2SHZrm_Int: 5760 case X86::VCVTSS2SHZrr: 5761 case X86::VCVTSS2SHZrr_Int: 5762 case X86::VCVTSS2SHZrrb_Int: 5763 case X86::VCVTSS2SHZrm: 5764 case X86::VCVTSS2SHZrm_Int: 5765 case X86::VCVTSH2SDZrr: 5766 case X86::VCVTSH2SDZrr_Int: 5767 case X86::VCVTSH2SDZrrb_Int: 5768 case X86::VCVTSH2SDZrm: 5769 case X86::VCVTSH2SDZrm_Int: 5770 case X86::VCVTSH2SSZrr: 5771 case X86::VCVTSH2SSZrr_Int: 5772 case X86::VCVTSH2SSZrrb_Int: 5773 case X86::VCVTSH2SSZrm: 5774 case X86::VCVTSH2SSZrm_Int: 5775 return OpNum == 1; 5776 case X86::VMOVSSZrrk: 5777 case X86::VMOVSDZrrk: 5778 return OpNum == 3 && !ForLoadFold; 5779 case X86::VMOVSSZrrkz: 5780 case X86::VMOVSDZrrkz: 5781 return OpNum == 2 && !ForLoadFold; 5782 } 5783 5784 return false; 5785 } 5786 5787 /// Inform the BreakFalseDeps pass how many idle instructions we would like 5788 /// before certain undef register reads. 5789 /// 5790 /// This catches the VCVTSI2SD family of instructions: 5791 /// 5792 /// vcvtsi2sdq %rax, undef %xmm0, %xmm14 5793 /// 5794 /// We should to be careful *not* to catch VXOR idioms which are presumably 5795 /// handled specially in the pipeline: 5796 /// 5797 /// vxorps undef %xmm1, undef %xmm1, %xmm1 5798 /// 5799 /// Like getPartialRegUpdateClearance, this makes a strong assumption that the 5800 /// high bits that are passed-through are not live. 5801 unsigned 5802 X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned OpNum, 5803 const TargetRegisterInfo *TRI) const { 5804 const MachineOperand &MO = MI.getOperand(OpNum); 5805 if (MO.getReg().isPhysical() && hasUndefRegUpdate(MI.getOpcode(), OpNum)) 5806 return UndefRegClearance; 5807 5808 return 0; 5809 } 5810 5811 void X86InstrInfo::breakPartialRegDependency( 5812 MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const { 5813 Register Reg = MI.getOperand(OpNum).getReg(); 5814 // If MI kills this register, the false dependence is already broken. 5815 if (MI.killsRegister(Reg, TRI)) 5816 return; 5817 5818 if (X86::VR128RegClass.contains(Reg)) { 5819 // These instructions are all floating point domain, so xorps is the best 5820 // choice. 5821 unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr; 5822 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(Opc), Reg) 5823 .addReg(Reg, RegState::Undef) 5824 .addReg(Reg, RegState::Undef); 5825 MI.addRegisterKilled(Reg, TRI, true); 5826 } else if (X86::VR256RegClass.contains(Reg)) { 5827 // Use vxorps to clear the full ymm register. 5828 // It wants to read and write the xmm sub-register. 5829 Register XReg = TRI->getSubReg(Reg, X86::sub_xmm); 5830 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VXORPSrr), XReg) 5831 .addReg(XReg, RegState::Undef) 5832 .addReg(XReg, RegState::Undef) 5833 .addReg(Reg, RegState::ImplicitDefine); 5834 MI.addRegisterKilled(Reg, TRI, true); 5835 } else if (X86::VR128XRegClass.contains(Reg)) { 5836 // Only handle VLX targets. 5837 if (!Subtarget.hasVLX()) 5838 return; 5839 // Since vxorps requires AVX512DQ, vpxord should be the best choice. 5840 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), Reg) 5841 .addReg(Reg, RegState::Undef) 5842 .addReg(Reg, RegState::Undef); 5843 MI.addRegisterKilled(Reg, TRI, true); 5844 } else if (X86::VR256XRegClass.contains(Reg) || 5845 X86::VR512RegClass.contains(Reg)) { 5846 // Only handle VLX targets. 5847 if (!Subtarget.hasVLX()) 5848 return; 5849 // Use vpxord to clear the full ymm/zmm register. 5850 // It wants to read and write the xmm sub-register. 5851 Register XReg = TRI->getSubReg(Reg, X86::sub_xmm); 5852 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), XReg) 5853 .addReg(XReg, RegState::Undef) 5854 .addReg(XReg, RegState::Undef) 5855 .addReg(Reg, RegState::ImplicitDefine); 5856 MI.addRegisterKilled(Reg, TRI, true); 5857 } else if (X86::GR64RegClass.contains(Reg)) { 5858 // Using XOR32rr because it has shorter encoding and zeros up the upper bits 5859 // as well. 5860 Register XReg = TRI->getSubReg(Reg, X86::sub_32bit); 5861 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), XReg) 5862 .addReg(XReg, RegState::Undef) 5863 .addReg(XReg, RegState::Undef) 5864 .addReg(Reg, RegState::ImplicitDefine); 5865 MI.addRegisterKilled(Reg, TRI, true); 5866 } else if (X86::GR32RegClass.contains(Reg)) { 5867 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), Reg) 5868 .addReg(Reg, RegState::Undef) 5869 .addReg(Reg, RegState::Undef); 5870 MI.addRegisterKilled(Reg, TRI, true); 5871 } 5872 } 5873 5874 static void addOperands(MachineInstrBuilder &MIB, ArrayRef<MachineOperand> MOs, 5875 int PtrOffset = 0) { 5876 unsigned NumAddrOps = MOs.size(); 5877 5878 if (NumAddrOps < 4) { 5879 // FrameIndex only - add an immediate offset (whether its zero or not). 5880 for (unsigned i = 0; i != NumAddrOps; ++i) 5881 MIB.add(MOs[i]); 5882 addOffset(MIB, PtrOffset); 5883 } else { 5884 // General Memory Addressing - we need to add any offset to an existing 5885 // offset. 5886 assert(MOs.size() == 5 && "Unexpected memory operand list length"); 5887 for (unsigned i = 0; i != NumAddrOps; ++i) { 5888 const MachineOperand &MO = MOs[i]; 5889 if (i == 3 && PtrOffset != 0) { 5890 MIB.addDisp(MO, PtrOffset); 5891 } else { 5892 MIB.add(MO); 5893 } 5894 } 5895 } 5896 } 5897 5898 static void updateOperandRegConstraints(MachineFunction &MF, 5899 MachineInstr &NewMI, 5900 const TargetInstrInfo &TII) { 5901 MachineRegisterInfo &MRI = MF.getRegInfo(); 5902 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); 5903 5904 for (int Idx : llvm::seq<int>(0, NewMI.getNumOperands())) { 5905 MachineOperand &MO = NewMI.getOperand(Idx); 5906 // We only need to update constraints on virtual register operands. 5907 if (!MO.isReg()) 5908 continue; 5909 Register Reg = MO.getReg(); 5910 if (!Reg.isVirtual()) 5911 continue; 5912 5913 auto *NewRC = MRI.constrainRegClass( 5914 Reg, TII.getRegClass(NewMI.getDesc(), Idx, &TRI, MF)); 5915 if (!NewRC) { 5916 LLVM_DEBUG( 5917 dbgs() << "WARNING: Unable to update register constraint for operand " 5918 << Idx << " of instruction:\n"; 5919 NewMI.dump(); dbgs() << "\n"); 5920 } 5921 } 5922 } 5923 5924 static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode, 5925 ArrayRef<MachineOperand> MOs, 5926 MachineBasicBlock::iterator InsertPt, 5927 MachineInstr &MI, 5928 const TargetInstrInfo &TII) { 5929 // Create the base instruction with the memory operand as the first part. 5930 // Omit the implicit operands, something BuildMI can't do. 5931 MachineInstr *NewMI = 5932 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); 5933 MachineInstrBuilder MIB(MF, NewMI); 5934 addOperands(MIB, MOs); 5935 5936 // Loop over the rest of the ri operands, converting them over. 5937 unsigned NumOps = MI.getDesc().getNumOperands() - 2; 5938 for (unsigned i = 0; i != NumOps; ++i) { 5939 MachineOperand &MO = MI.getOperand(i + 2); 5940 MIB.add(MO); 5941 } 5942 for (const MachineOperand &MO : llvm::drop_begin(MI.operands(), NumOps + 2)) 5943 MIB.add(MO); 5944 5945 updateOperandRegConstraints(MF, *NewMI, TII); 5946 5947 MachineBasicBlock *MBB = InsertPt->getParent(); 5948 MBB->insert(InsertPt, NewMI); 5949 5950 return MIB; 5951 } 5952 5953 static MachineInstr *FuseInst(MachineFunction &MF, unsigned Opcode, 5954 unsigned OpNo, ArrayRef<MachineOperand> MOs, 5955 MachineBasicBlock::iterator InsertPt, 5956 MachineInstr &MI, const TargetInstrInfo &TII, 5957 int PtrOffset = 0) { 5958 // Omit the implicit operands, something BuildMI can't do. 5959 MachineInstr *NewMI = 5960 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); 5961 MachineInstrBuilder MIB(MF, NewMI); 5962 5963 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 5964 MachineOperand &MO = MI.getOperand(i); 5965 if (i == OpNo) { 5966 assert(MO.isReg() && "Expected to fold into reg operand!"); 5967 addOperands(MIB, MOs, PtrOffset); 5968 } else { 5969 MIB.add(MO); 5970 } 5971 } 5972 5973 updateOperandRegConstraints(MF, *NewMI, TII); 5974 5975 // Copy the NoFPExcept flag from the instruction we're fusing. 5976 if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept)) 5977 NewMI->setFlag(MachineInstr::MIFlag::NoFPExcept); 5978 5979 MachineBasicBlock *MBB = InsertPt->getParent(); 5980 MBB->insert(InsertPt, NewMI); 5981 5982 return MIB; 5983 } 5984 5985 static MachineInstr *MakeM0Inst(const TargetInstrInfo &TII, unsigned Opcode, 5986 ArrayRef<MachineOperand> MOs, 5987 MachineBasicBlock::iterator InsertPt, 5988 MachineInstr &MI) { 5989 MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, 5990 MI.getDebugLoc(), TII.get(Opcode)); 5991 addOperands(MIB, MOs); 5992 return MIB.addImm(0); 5993 } 5994 5995 MachineInstr *X86InstrInfo::foldMemoryOperandCustom( 5996 MachineFunction &MF, MachineInstr &MI, unsigned OpNum, 5997 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, 5998 unsigned Size, Align Alignment) const { 5999 switch (MI.getOpcode()) { 6000 case X86::INSERTPSrr: 6001 case X86::VINSERTPSrr: 6002 case X86::VINSERTPSZrr: 6003 // Attempt to convert the load of inserted vector into a fold load 6004 // of a single float. 6005 if (OpNum == 2) { 6006 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); 6007 unsigned ZMask = Imm & 15; 6008 unsigned DstIdx = (Imm >> 4) & 3; 6009 unsigned SrcIdx = (Imm >> 6) & 3; 6010 6011 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6012 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 6013 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 6014 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment >= Align(4)) { 6015 int PtrOffset = SrcIdx * 4; 6016 unsigned NewImm = (DstIdx << 4) | ZMask; 6017 unsigned NewOpCode = 6018 (MI.getOpcode() == X86::VINSERTPSZrr) ? X86::VINSERTPSZrm : 6019 (MI.getOpcode() == X86::VINSERTPSrr) ? X86::VINSERTPSrm : 6020 X86::INSERTPSrm; 6021 MachineInstr *NewMI = 6022 FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, PtrOffset); 6023 NewMI->getOperand(NewMI->getNumOperands() - 1).setImm(NewImm); 6024 return NewMI; 6025 } 6026 } 6027 break; 6028 case X86::MOVHLPSrr: 6029 case X86::VMOVHLPSrr: 6030 case X86::VMOVHLPSZrr: 6031 // Move the upper 64-bits of the second operand to the lower 64-bits. 6032 // To fold the load, adjust the pointer to the upper and use (V)MOVLPS. 6033 // TODO: In most cases AVX doesn't have a 8-byte alignment requirement. 6034 if (OpNum == 2) { 6035 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6036 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 6037 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 6038 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment >= Align(8)) { 6039 unsigned NewOpCode = 6040 (MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm : 6041 (MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm : 6042 X86::MOVLPSrm; 6043 MachineInstr *NewMI = 6044 FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, 8); 6045 return NewMI; 6046 } 6047 } 6048 break; 6049 case X86::UNPCKLPDrr: 6050 // If we won't be able to fold this to the memory form of UNPCKL, use 6051 // MOVHPD instead. Done as custom because we can't have this in the load 6052 // table twice. 6053 if (OpNum == 2) { 6054 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6055 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 6056 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 6057 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment < Align(16)) { 6058 MachineInstr *NewMI = 6059 FuseInst(MF, X86::MOVHPDrm, OpNum, MOs, InsertPt, MI, *this); 6060 return NewMI; 6061 } 6062 } 6063 break; 6064 } 6065 6066 return nullptr; 6067 } 6068 6069 static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF, 6070 MachineInstr &MI) { 6071 if (!hasUndefRegUpdate(MI.getOpcode(), 1, /*ForLoadFold*/true) || 6072 !MI.getOperand(1).isReg()) 6073 return false; 6074 6075 // The are two cases we need to handle depending on where in the pipeline 6076 // the folding attempt is being made. 6077 // -Register has the undef flag set. 6078 // -Register is produced by the IMPLICIT_DEF instruction. 6079 6080 if (MI.getOperand(1).isUndef()) 6081 return true; 6082 6083 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 6084 MachineInstr *VRegDef = RegInfo.getUniqueVRegDef(MI.getOperand(1).getReg()); 6085 return VRegDef && VRegDef->isImplicitDef(); 6086 } 6087 6088 MachineInstr *X86InstrInfo::foldMemoryOperandImpl( 6089 MachineFunction &MF, MachineInstr &MI, unsigned OpNum, 6090 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, 6091 unsigned Size, Align Alignment, bool AllowCommute) const { 6092 bool isSlowTwoMemOps = Subtarget.slowTwoMemOps(); 6093 bool isTwoAddrFold = false; 6094 6095 // For CPUs that favor the register form of a call or push, 6096 // do not fold loads into calls or pushes, unless optimizing for size 6097 // aggressively. 6098 if (isSlowTwoMemOps && !MF.getFunction().hasMinSize() && 6099 (MI.getOpcode() == X86::CALL32r || MI.getOpcode() == X86::CALL64r || 6100 MI.getOpcode() == X86::PUSH16r || MI.getOpcode() == X86::PUSH32r || 6101 MI.getOpcode() == X86::PUSH64r)) 6102 return nullptr; 6103 6104 // Avoid partial and undef register update stalls unless optimizing for size. 6105 if (!MF.getFunction().hasOptSize() && 6106 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/true) || 6107 shouldPreventUndefRegUpdateMemFold(MF, MI))) 6108 return nullptr; 6109 6110 unsigned NumOps = MI.getDesc().getNumOperands(); 6111 bool isTwoAddr = 6112 NumOps > 1 && MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1; 6113 6114 // FIXME: AsmPrinter doesn't know how to handle 6115 // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding. 6116 if (MI.getOpcode() == X86::ADD32ri && 6117 MI.getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS) 6118 return nullptr; 6119 6120 // GOTTPOFF relocation loads can only be folded into add instructions. 6121 // FIXME: Need to exclude other relocations that only support specific 6122 // instructions. 6123 if (MOs.size() == X86::AddrNumOperands && 6124 MOs[X86::AddrDisp].getTargetFlags() == X86II::MO_GOTTPOFF && 6125 MI.getOpcode() != X86::ADD64rr) 6126 return nullptr; 6127 6128 // Don't fold loads into indirect calls that need a KCFI check as we'll 6129 // have to unfold these in X86KCFIPass anyway. 6130 if (MI.isCall() && MI.getCFIType()) 6131 return nullptr; 6132 6133 MachineInstr *NewMI = nullptr; 6134 6135 // Attempt to fold any custom cases we have. 6136 if (MachineInstr *CustomMI = foldMemoryOperandCustom( 6137 MF, MI, OpNum, MOs, InsertPt, Size, Alignment)) 6138 return CustomMI; 6139 6140 const X86MemoryFoldTableEntry *I = nullptr; 6141 6142 // Folding a memory location into the two-address part of a two-address 6143 // instruction is different than folding it other places. It requires 6144 // replacing the *two* registers with the memory location. 6145 if (isTwoAddr && NumOps >= 2 && OpNum < 2 && MI.getOperand(0).isReg() && 6146 MI.getOperand(1).isReg() && 6147 MI.getOperand(0).getReg() == MI.getOperand(1).getReg()) { 6148 I = lookupTwoAddrFoldTable(MI.getOpcode()); 6149 isTwoAddrFold = true; 6150 } else { 6151 if (OpNum == 0) { 6152 if (MI.getOpcode() == X86::MOV32r0) { 6153 NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, InsertPt, MI); 6154 if (NewMI) 6155 return NewMI; 6156 } 6157 } 6158 6159 I = lookupFoldTable(MI.getOpcode(), OpNum); 6160 } 6161 6162 if (I != nullptr) { 6163 unsigned Opcode = I->DstOp; 6164 bool FoldedLoad = 6165 isTwoAddrFold || (OpNum == 0 && I->Flags & TB_FOLDED_LOAD) || OpNum > 0; 6166 bool FoldedStore = 6167 isTwoAddrFold || (OpNum == 0 && I->Flags & TB_FOLDED_STORE); 6168 MaybeAlign MinAlign = 6169 decodeMaybeAlign((I->Flags & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT); 6170 if (MinAlign && Alignment < *MinAlign) 6171 return nullptr; 6172 bool NarrowToMOV32rm = false; 6173 if (Size) { 6174 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6175 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, 6176 &RI, MF); 6177 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 6178 // Check if it's safe to fold the load. If the size of the object is 6179 // narrower than the load width, then it's not. 6180 // FIXME: Allow scalar intrinsic instructions like ADDSSrm_Int. 6181 if (FoldedLoad && Size < RCSize) { 6182 // If this is a 64-bit load, but the spill slot is 32, then we can do 6183 // a 32-bit load which is implicitly zero-extended. This likely is 6184 // due to live interval analysis remat'ing a load from stack slot. 6185 if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4) 6186 return nullptr; 6187 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) 6188 return nullptr; 6189 Opcode = X86::MOV32rm; 6190 NarrowToMOV32rm = true; 6191 } 6192 // For stores, make sure the size of the object is equal to the size of 6193 // the store. If the object is larger, the extra bits would be garbage. If 6194 // the object is smaller we might overwrite another object or fault. 6195 if (FoldedStore && Size != RCSize) 6196 return nullptr; 6197 } 6198 6199 if (isTwoAddrFold) 6200 NewMI = FuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, *this); 6201 else 6202 NewMI = FuseInst(MF, Opcode, OpNum, MOs, InsertPt, MI, *this); 6203 6204 if (NarrowToMOV32rm) { 6205 // If this is the special case where we use a MOV32rm to load a 32-bit 6206 // value and zero-extend the top bits. Change the destination register 6207 // to a 32-bit one. 6208 Register DstReg = NewMI->getOperand(0).getReg(); 6209 if (DstReg.isPhysical()) 6210 NewMI->getOperand(0).setReg(RI.getSubReg(DstReg, X86::sub_32bit)); 6211 else 6212 NewMI->getOperand(0).setSubReg(X86::sub_32bit); 6213 } 6214 return NewMI; 6215 } 6216 6217 // If the instruction and target operand are commutable, commute the 6218 // instruction and try again. 6219 if (AllowCommute) { 6220 unsigned CommuteOpIdx1 = OpNum, CommuteOpIdx2 = CommuteAnyOperandIndex; 6221 if (findCommutedOpIndices(MI, CommuteOpIdx1, CommuteOpIdx2)) { 6222 bool HasDef = MI.getDesc().getNumDefs(); 6223 Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register(); 6224 Register Reg1 = MI.getOperand(CommuteOpIdx1).getReg(); 6225 Register Reg2 = MI.getOperand(CommuteOpIdx2).getReg(); 6226 bool Tied1 = 6227 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx1, MCOI::TIED_TO); 6228 bool Tied2 = 6229 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx2, MCOI::TIED_TO); 6230 6231 // If either of the commutable operands are tied to the destination 6232 // then we can not commute + fold. 6233 if ((HasDef && Reg0 == Reg1 && Tied1) || 6234 (HasDef && Reg0 == Reg2 && Tied2)) 6235 return nullptr; 6236 6237 MachineInstr *CommutedMI = 6238 commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); 6239 if (!CommutedMI) { 6240 // Unable to commute. 6241 return nullptr; 6242 } 6243 if (CommutedMI != &MI) { 6244 // New instruction. We can't fold from this. 6245 CommutedMI->eraseFromParent(); 6246 return nullptr; 6247 } 6248 6249 // Attempt to fold with the commuted version of the instruction. 6250 NewMI = foldMemoryOperandImpl(MF, MI, CommuteOpIdx2, MOs, InsertPt, Size, 6251 Alignment, /*AllowCommute=*/false); 6252 if (NewMI) 6253 return NewMI; 6254 6255 // Folding failed again - undo the commute before returning. 6256 MachineInstr *UncommutedMI = 6257 commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); 6258 if (!UncommutedMI) { 6259 // Unable to commute. 6260 return nullptr; 6261 } 6262 if (UncommutedMI != &MI) { 6263 // New instruction. It doesn't need to be kept. 6264 UncommutedMI->eraseFromParent(); 6265 return nullptr; 6266 } 6267 6268 // Return here to prevent duplicate fuse failure report. 6269 return nullptr; 6270 } 6271 } 6272 6273 // No fusion 6274 if (PrintFailedFusing && !MI.isCopy()) 6275 dbgs() << "We failed to fuse operand " << OpNum << " in " << MI; 6276 return nullptr; 6277 } 6278 6279 MachineInstr * 6280 X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, 6281 ArrayRef<unsigned> Ops, 6282 MachineBasicBlock::iterator InsertPt, 6283 int FrameIndex, LiveIntervals *LIS, 6284 VirtRegMap *VRM) const { 6285 // Check switch flag 6286 if (NoFusing) 6287 return nullptr; 6288 6289 // Avoid partial and undef register update stalls unless optimizing for size. 6290 if (!MF.getFunction().hasOptSize() && 6291 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/true) || 6292 shouldPreventUndefRegUpdateMemFold(MF, MI))) 6293 return nullptr; 6294 6295 // Don't fold subreg spills, or reloads that use a high subreg. 6296 for (auto Op : Ops) { 6297 MachineOperand &MO = MI.getOperand(Op); 6298 auto SubReg = MO.getSubReg(); 6299 if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi)) 6300 return nullptr; 6301 } 6302 6303 const MachineFrameInfo &MFI = MF.getFrameInfo(); 6304 unsigned Size = MFI.getObjectSize(FrameIndex); 6305 Align Alignment = MFI.getObjectAlign(FrameIndex); 6306 // If the function stack isn't realigned we don't want to fold instructions 6307 // that need increased alignment. 6308 if (!RI.hasStackRealignment(MF)) 6309 Alignment = 6310 std::min(Alignment, Subtarget.getFrameLowering()->getStackAlign()); 6311 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 6312 unsigned NewOpc = 0; 6313 unsigned RCSize = 0; 6314 switch (MI.getOpcode()) { 6315 default: return nullptr; 6316 case X86::TEST8rr: NewOpc = X86::CMP8ri; RCSize = 1; break; 6317 case X86::TEST16rr: NewOpc = X86::CMP16ri8; RCSize = 2; break; 6318 case X86::TEST32rr: NewOpc = X86::CMP32ri8; RCSize = 4; break; 6319 case X86::TEST64rr: NewOpc = X86::CMP64ri8; RCSize = 8; break; 6320 } 6321 // Check if it's safe to fold the load. If the size of the object is 6322 // narrower than the load width, then it's not. 6323 if (Size < RCSize) 6324 return nullptr; 6325 // Change to CMPXXri r, 0 first. 6326 MI.setDesc(get(NewOpc)); 6327 MI.getOperand(1).ChangeToImmediate(0); 6328 } else if (Ops.size() != 1) 6329 return nullptr; 6330 6331 return foldMemoryOperandImpl(MF, MI, Ops[0], 6332 MachineOperand::CreateFI(FrameIndex), InsertPt, 6333 Size, Alignment, /*AllowCommute=*/true); 6334 } 6335 6336 /// Check if \p LoadMI is a partial register load that we can't fold into \p MI 6337 /// because the latter uses contents that wouldn't be defined in the folded 6338 /// version. For instance, this transformation isn't legal: 6339 /// movss (%rdi), %xmm0 6340 /// addps %xmm0, %xmm0 6341 /// -> 6342 /// addps (%rdi), %xmm0 6343 /// 6344 /// But this one is: 6345 /// movss (%rdi), %xmm0 6346 /// addss %xmm0, %xmm0 6347 /// -> 6348 /// addss (%rdi), %xmm0 6349 /// 6350 static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, 6351 const MachineInstr &UserMI, 6352 const MachineFunction &MF) { 6353 unsigned Opc = LoadMI.getOpcode(); 6354 unsigned UserOpc = UserMI.getOpcode(); 6355 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6356 const TargetRegisterClass *RC = 6357 MF.getRegInfo().getRegClass(LoadMI.getOperand(0).getReg()); 6358 unsigned RegSize = TRI.getRegSizeInBits(*RC); 6359 6360 if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm || Opc == X86::VMOVSSZrm || 6361 Opc == X86::MOVSSrm_alt || Opc == X86::VMOVSSrm_alt || 6362 Opc == X86::VMOVSSZrm_alt) && 6363 RegSize > 32) { 6364 // These instructions only load 32 bits, we can't fold them if the 6365 // destination register is wider than 32 bits (4 bytes), and its user 6366 // instruction isn't scalar (SS). 6367 switch (UserOpc) { 6368 case X86::CVTSS2SDrr_Int: 6369 case X86::VCVTSS2SDrr_Int: 6370 case X86::VCVTSS2SDZrr_Int: 6371 case X86::VCVTSS2SDZrr_Intk: 6372 case X86::VCVTSS2SDZrr_Intkz: 6373 case X86::CVTSS2SIrr_Int: case X86::CVTSS2SI64rr_Int: 6374 case X86::VCVTSS2SIrr_Int: case X86::VCVTSS2SI64rr_Int: 6375 case X86::VCVTSS2SIZrr_Int: case X86::VCVTSS2SI64Zrr_Int: 6376 case X86::CVTTSS2SIrr_Int: case X86::CVTTSS2SI64rr_Int: 6377 case X86::VCVTTSS2SIrr_Int: case X86::VCVTTSS2SI64rr_Int: 6378 case X86::VCVTTSS2SIZrr_Int: case X86::VCVTTSS2SI64Zrr_Int: 6379 case X86::VCVTSS2USIZrr_Int: case X86::VCVTSS2USI64Zrr_Int: 6380 case X86::VCVTTSS2USIZrr_Int: case X86::VCVTTSS2USI64Zrr_Int: 6381 case X86::RCPSSr_Int: case X86::VRCPSSr_Int: 6382 case X86::RSQRTSSr_Int: case X86::VRSQRTSSr_Int: 6383 case X86::ROUNDSSr_Int: case X86::VROUNDSSr_Int: 6384 case X86::COMISSrr_Int: case X86::VCOMISSrr_Int: case X86::VCOMISSZrr_Int: 6385 case X86::UCOMISSrr_Int:case X86::VUCOMISSrr_Int:case X86::VUCOMISSZrr_Int: 6386 case X86::ADDSSrr_Int: case X86::VADDSSrr_Int: case X86::VADDSSZrr_Int: 6387 case X86::CMPSSrr_Int: case X86::VCMPSSrr_Int: case X86::VCMPSSZrr_Int: 6388 case X86::DIVSSrr_Int: case X86::VDIVSSrr_Int: case X86::VDIVSSZrr_Int: 6389 case X86::MAXSSrr_Int: case X86::VMAXSSrr_Int: case X86::VMAXSSZrr_Int: 6390 case X86::MINSSrr_Int: case X86::VMINSSrr_Int: case X86::VMINSSZrr_Int: 6391 case X86::MULSSrr_Int: case X86::VMULSSrr_Int: case X86::VMULSSZrr_Int: 6392 case X86::SQRTSSr_Int: case X86::VSQRTSSr_Int: case X86::VSQRTSSZr_Int: 6393 case X86::SUBSSrr_Int: case X86::VSUBSSrr_Int: case X86::VSUBSSZrr_Int: 6394 case X86::VADDSSZrr_Intk: case X86::VADDSSZrr_Intkz: 6395 case X86::VCMPSSZrr_Intk: 6396 case X86::VDIVSSZrr_Intk: case X86::VDIVSSZrr_Intkz: 6397 case X86::VMAXSSZrr_Intk: case X86::VMAXSSZrr_Intkz: 6398 case X86::VMINSSZrr_Intk: case X86::VMINSSZrr_Intkz: 6399 case X86::VMULSSZrr_Intk: case X86::VMULSSZrr_Intkz: 6400 case X86::VSQRTSSZr_Intk: case X86::VSQRTSSZr_Intkz: 6401 case X86::VSUBSSZrr_Intk: case X86::VSUBSSZrr_Intkz: 6402 case X86::VFMADDSS4rr_Int: case X86::VFNMADDSS4rr_Int: 6403 case X86::VFMSUBSS4rr_Int: case X86::VFNMSUBSS4rr_Int: 6404 case X86::VFMADD132SSr_Int: case X86::VFNMADD132SSr_Int: 6405 case X86::VFMADD213SSr_Int: case X86::VFNMADD213SSr_Int: 6406 case X86::VFMADD231SSr_Int: case X86::VFNMADD231SSr_Int: 6407 case X86::VFMSUB132SSr_Int: case X86::VFNMSUB132SSr_Int: 6408 case X86::VFMSUB213SSr_Int: case X86::VFNMSUB213SSr_Int: 6409 case X86::VFMSUB231SSr_Int: case X86::VFNMSUB231SSr_Int: 6410 case X86::VFMADD132SSZr_Int: case X86::VFNMADD132SSZr_Int: 6411 case X86::VFMADD213SSZr_Int: case X86::VFNMADD213SSZr_Int: 6412 case X86::VFMADD231SSZr_Int: case X86::VFNMADD231SSZr_Int: 6413 case X86::VFMSUB132SSZr_Int: case X86::VFNMSUB132SSZr_Int: 6414 case X86::VFMSUB213SSZr_Int: case X86::VFNMSUB213SSZr_Int: 6415 case X86::VFMSUB231SSZr_Int: case X86::VFNMSUB231SSZr_Int: 6416 case X86::VFMADD132SSZr_Intk: case X86::VFNMADD132SSZr_Intk: 6417 case X86::VFMADD213SSZr_Intk: case X86::VFNMADD213SSZr_Intk: 6418 case X86::VFMADD231SSZr_Intk: case X86::VFNMADD231SSZr_Intk: 6419 case X86::VFMSUB132SSZr_Intk: case X86::VFNMSUB132SSZr_Intk: 6420 case X86::VFMSUB213SSZr_Intk: case X86::VFNMSUB213SSZr_Intk: 6421 case X86::VFMSUB231SSZr_Intk: case X86::VFNMSUB231SSZr_Intk: 6422 case X86::VFMADD132SSZr_Intkz: case X86::VFNMADD132SSZr_Intkz: 6423 case X86::VFMADD213SSZr_Intkz: case X86::VFNMADD213SSZr_Intkz: 6424 case X86::VFMADD231SSZr_Intkz: case X86::VFNMADD231SSZr_Intkz: 6425 case X86::VFMSUB132SSZr_Intkz: case X86::VFNMSUB132SSZr_Intkz: 6426 case X86::VFMSUB213SSZr_Intkz: case X86::VFNMSUB213SSZr_Intkz: 6427 case X86::VFMSUB231SSZr_Intkz: case X86::VFNMSUB231SSZr_Intkz: 6428 case X86::VFIXUPIMMSSZrri: 6429 case X86::VFIXUPIMMSSZrrik: 6430 case X86::VFIXUPIMMSSZrrikz: 6431 case X86::VFPCLASSSSZrr: 6432 case X86::VFPCLASSSSZrrk: 6433 case X86::VGETEXPSSZr: 6434 case X86::VGETEXPSSZrk: 6435 case X86::VGETEXPSSZrkz: 6436 case X86::VGETMANTSSZrri: 6437 case X86::VGETMANTSSZrrik: 6438 case X86::VGETMANTSSZrrikz: 6439 case X86::VRANGESSZrri: 6440 case X86::VRANGESSZrrik: 6441 case X86::VRANGESSZrrikz: 6442 case X86::VRCP14SSZrr: 6443 case X86::VRCP14SSZrrk: 6444 case X86::VRCP14SSZrrkz: 6445 case X86::VRCP28SSZr: 6446 case X86::VRCP28SSZrk: 6447 case X86::VRCP28SSZrkz: 6448 case X86::VREDUCESSZrri: 6449 case X86::VREDUCESSZrrik: 6450 case X86::VREDUCESSZrrikz: 6451 case X86::VRNDSCALESSZr_Int: 6452 case X86::VRNDSCALESSZr_Intk: 6453 case X86::VRNDSCALESSZr_Intkz: 6454 case X86::VRSQRT14SSZrr: 6455 case X86::VRSQRT14SSZrrk: 6456 case X86::VRSQRT14SSZrrkz: 6457 case X86::VRSQRT28SSZr: 6458 case X86::VRSQRT28SSZrk: 6459 case X86::VRSQRT28SSZrkz: 6460 case X86::VSCALEFSSZrr: 6461 case X86::VSCALEFSSZrrk: 6462 case X86::VSCALEFSSZrrkz: 6463 return false; 6464 default: 6465 return true; 6466 } 6467 } 6468 6469 if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm || Opc == X86::VMOVSDZrm || 6470 Opc == X86::MOVSDrm_alt || Opc == X86::VMOVSDrm_alt || 6471 Opc == X86::VMOVSDZrm_alt) && 6472 RegSize > 64) { 6473 // These instructions only load 64 bits, we can't fold them if the 6474 // destination register is wider than 64 bits (8 bytes), and its user 6475 // instruction isn't scalar (SD). 6476 switch (UserOpc) { 6477 case X86::CVTSD2SSrr_Int: 6478 case X86::VCVTSD2SSrr_Int: 6479 case X86::VCVTSD2SSZrr_Int: 6480 case X86::VCVTSD2SSZrr_Intk: 6481 case X86::VCVTSD2SSZrr_Intkz: 6482 case X86::CVTSD2SIrr_Int: case X86::CVTSD2SI64rr_Int: 6483 case X86::VCVTSD2SIrr_Int: case X86::VCVTSD2SI64rr_Int: 6484 case X86::VCVTSD2SIZrr_Int: case X86::VCVTSD2SI64Zrr_Int: 6485 case X86::CVTTSD2SIrr_Int: case X86::CVTTSD2SI64rr_Int: 6486 case X86::VCVTTSD2SIrr_Int: case X86::VCVTTSD2SI64rr_Int: 6487 case X86::VCVTTSD2SIZrr_Int: case X86::VCVTTSD2SI64Zrr_Int: 6488 case X86::VCVTSD2USIZrr_Int: case X86::VCVTSD2USI64Zrr_Int: 6489 case X86::VCVTTSD2USIZrr_Int: case X86::VCVTTSD2USI64Zrr_Int: 6490 case X86::ROUNDSDr_Int: case X86::VROUNDSDr_Int: 6491 case X86::COMISDrr_Int: case X86::VCOMISDrr_Int: case X86::VCOMISDZrr_Int: 6492 case X86::UCOMISDrr_Int:case X86::VUCOMISDrr_Int:case X86::VUCOMISDZrr_Int: 6493 case X86::ADDSDrr_Int: case X86::VADDSDrr_Int: case X86::VADDSDZrr_Int: 6494 case X86::CMPSDrr_Int: case X86::VCMPSDrr_Int: case X86::VCMPSDZrr_Int: 6495 case X86::DIVSDrr_Int: case X86::VDIVSDrr_Int: case X86::VDIVSDZrr_Int: 6496 case X86::MAXSDrr_Int: case X86::VMAXSDrr_Int: case X86::VMAXSDZrr_Int: 6497 case X86::MINSDrr_Int: case X86::VMINSDrr_Int: case X86::VMINSDZrr_Int: 6498 case X86::MULSDrr_Int: case X86::VMULSDrr_Int: case X86::VMULSDZrr_Int: 6499 case X86::SQRTSDr_Int: case X86::VSQRTSDr_Int: case X86::VSQRTSDZr_Int: 6500 case X86::SUBSDrr_Int: case X86::VSUBSDrr_Int: case X86::VSUBSDZrr_Int: 6501 case X86::VADDSDZrr_Intk: case X86::VADDSDZrr_Intkz: 6502 case X86::VCMPSDZrr_Intk: 6503 case X86::VDIVSDZrr_Intk: case X86::VDIVSDZrr_Intkz: 6504 case X86::VMAXSDZrr_Intk: case X86::VMAXSDZrr_Intkz: 6505 case X86::VMINSDZrr_Intk: case X86::VMINSDZrr_Intkz: 6506 case X86::VMULSDZrr_Intk: case X86::VMULSDZrr_Intkz: 6507 case X86::VSQRTSDZr_Intk: case X86::VSQRTSDZr_Intkz: 6508 case X86::VSUBSDZrr_Intk: case X86::VSUBSDZrr_Intkz: 6509 case X86::VFMADDSD4rr_Int: case X86::VFNMADDSD4rr_Int: 6510 case X86::VFMSUBSD4rr_Int: case X86::VFNMSUBSD4rr_Int: 6511 case X86::VFMADD132SDr_Int: case X86::VFNMADD132SDr_Int: 6512 case X86::VFMADD213SDr_Int: case X86::VFNMADD213SDr_Int: 6513 case X86::VFMADD231SDr_Int: case X86::VFNMADD231SDr_Int: 6514 case X86::VFMSUB132SDr_Int: case X86::VFNMSUB132SDr_Int: 6515 case X86::VFMSUB213SDr_Int: case X86::VFNMSUB213SDr_Int: 6516 case X86::VFMSUB231SDr_Int: case X86::VFNMSUB231SDr_Int: 6517 case X86::VFMADD132SDZr_Int: case X86::VFNMADD132SDZr_Int: 6518 case X86::VFMADD213SDZr_Int: case X86::VFNMADD213SDZr_Int: 6519 case X86::VFMADD231SDZr_Int: case X86::VFNMADD231SDZr_Int: 6520 case X86::VFMSUB132SDZr_Int: case X86::VFNMSUB132SDZr_Int: 6521 case X86::VFMSUB213SDZr_Int: case X86::VFNMSUB213SDZr_Int: 6522 case X86::VFMSUB231SDZr_Int: case X86::VFNMSUB231SDZr_Int: 6523 case X86::VFMADD132SDZr_Intk: case X86::VFNMADD132SDZr_Intk: 6524 case X86::VFMADD213SDZr_Intk: case X86::VFNMADD213SDZr_Intk: 6525 case X86::VFMADD231SDZr_Intk: case X86::VFNMADD231SDZr_Intk: 6526 case X86::VFMSUB132SDZr_Intk: case X86::VFNMSUB132SDZr_Intk: 6527 case X86::VFMSUB213SDZr_Intk: case X86::VFNMSUB213SDZr_Intk: 6528 case X86::VFMSUB231SDZr_Intk: case X86::VFNMSUB231SDZr_Intk: 6529 case X86::VFMADD132SDZr_Intkz: case X86::VFNMADD132SDZr_Intkz: 6530 case X86::VFMADD213SDZr_Intkz: case X86::VFNMADD213SDZr_Intkz: 6531 case X86::VFMADD231SDZr_Intkz: case X86::VFNMADD231SDZr_Intkz: 6532 case X86::VFMSUB132SDZr_Intkz: case X86::VFNMSUB132SDZr_Intkz: 6533 case X86::VFMSUB213SDZr_Intkz: case X86::VFNMSUB213SDZr_Intkz: 6534 case X86::VFMSUB231SDZr_Intkz: case X86::VFNMSUB231SDZr_Intkz: 6535 case X86::VFIXUPIMMSDZrri: 6536 case X86::VFIXUPIMMSDZrrik: 6537 case X86::VFIXUPIMMSDZrrikz: 6538 case X86::VFPCLASSSDZrr: 6539 case X86::VFPCLASSSDZrrk: 6540 case X86::VGETEXPSDZr: 6541 case X86::VGETEXPSDZrk: 6542 case X86::VGETEXPSDZrkz: 6543 case X86::VGETMANTSDZrri: 6544 case X86::VGETMANTSDZrrik: 6545 case X86::VGETMANTSDZrrikz: 6546 case X86::VRANGESDZrri: 6547 case X86::VRANGESDZrrik: 6548 case X86::VRANGESDZrrikz: 6549 case X86::VRCP14SDZrr: 6550 case X86::VRCP14SDZrrk: 6551 case X86::VRCP14SDZrrkz: 6552 case X86::VRCP28SDZr: 6553 case X86::VRCP28SDZrk: 6554 case X86::VRCP28SDZrkz: 6555 case X86::VREDUCESDZrri: 6556 case X86::VREDUCESDZrrik: 6557 case X86::VREDUCESDZrrikz: 6558 case X86::VRNDSCALESDZr_Int: 6559 case X86::VRNDSCALESDZr_Intk: 6560 case X86::VRNDSCALESDZr_Intkz: 6561 case X86::VRSQRT14SDZrr: 6562 case X86::VRSQRT14SDZrrk: 6563 case X86::VRSQRT14SDZrrkz: 6564 case X86::VRSQRT28SDZr: 6565 case X86::VRSQRT28SDZrk: 6566 case X86::VRSQRT28SDZrkz: 6567 case X86::VSCALEFSDZrr: 6568 case X86::VSCALEFSDZrrk: 6569 case X86::VSCALEFSDZrrkz: 6570 return false; 6571 default: 6572 return true; 6573 } 6574 } 6575 6576 if ((Opc == X86::VMOVSHZrm || Opc == X86::VMOVSHZrm_alt) && RegSize > 16) { 6577 // These instructions only load 16 bits, we can't fold them if the 6578 // destination register is wider than 16 bits (2 bytes), and its user 6579 // instruction isn't scalar (SH). 6580 switch (UserOpc) { 6581 case X86::VADDSHZrr_Int: 6582 case X86::VCMPSHZrr_Int: 6583 case X86::VDIVSHZrr_Int: 6584 case X86::VMAXSHZrr_Int: 6585 case X86::VMINSHZrr_Int: 6586 case X86::VMULSHZrr_Int: 6587 case X86::VSUBSHZrr_Int: 6588 case X86::VADDSHZrr_Intk: case X86::VADDSHZrr_Intkz: 6589 case X86::VCMPSHZrr_Intk: 6590 case X86::VDIVSHZrr_Intk: case X86::VDIVSHZrr_Intkz: 6591 case X86::VMAXSHZrr_Intk: case X86::VMAXSHZrr_Intkz: 6592 case X86::VMINSHZrr_Intk: case X86::VMINSHZrr_Intkz: 6593 case X86::VMULSHZrr_Intk: case X86::VMULSHZrr_Intkz: 6594 case X86::VSUBSHZrr_Intk: case X86::VSUBSHZrr_Intkz: 6595 case X86::VFMADD132SHZr_Int: case X86::VFNMADD132SHZr_Int: 6596 case X86::VFMADD213SHZr_Int: case X86::VFNMADD213SHZr_Int: 6597 case X86::VFMADD231SHZr_Int: case X86::VFNMADD231SHZr_Int: 6598 case X86::VFMSUB132SHZr_Int: case X86::VFNMSUB132SHZr_Int: 6599 case X86::VFMSUB213SHZr_Int: case X86::VFNMSUB213SHZr_Int: 6600 case X86::VFMSUB231SHZr_Int: case X86::VFNMSUB231SHZr_Int: 6601 case X86::VFMADD132SHZr_Intk: case X86::VFNMADD132SHZr_Intk: 6602 case X86::VFMADD213SHZr_Intk: case X86::VFNMADD213SHZr_Intk: 6603 case X86::VFMADD231SHZr_Intk: case X86::VFNMADD231SHZr_Intk: 6604 case X86::VFMSUB132SHZr_Intk: case X86::VFNMSUB132SHZr_Intk: 6605 case X86::VFMSUB213SHZr_Intk: case X86::VFNMSUB213SHZr_Intk: 6606 case X86::VFMSUB231SHZr_Intk: case X86::VFNMSUB231SHZr_Intk: 6607 case X86::VFMADD132SHZr_Intkz: case X86::VFNMADD132SHZr_Intkz: 6608 case X86::VFMADD213SHZr_Intkz: case X86::VFNMADD213SHZr_Intkz: 6609 case X86::VFMADD231SHZr_Intkz: case X86::VFNMADD231SHZr_Intkz: 6610 case X86::VFMSUB132SHZr_Intkz: case X86::VFNMSUB132SHZr_Intkz: 6611 case X86::VFMSUB213SHZr_Intkz: case X86::VFNMSUB213SHZr_Intkz: 6612 case X86::VFMSUB231SHZr_Intkz: case X86::VFNMSUB231SHZr_Intkz: 6613 return false; 6614 default: 6615 return true; 6616 } 6617 } 6618 6619 return false; 6620 } 6621 6622 MachineInstr *X86InstrInfo::foldMemoryOperandImpl( 6623 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 6624 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, 6625 LiveIntervals *LIS) const { 6626 6627 // TODO: Support the case where LoadMI loads a wide register, but MI 6628 // only uses a subreg. 6629 for (auto Op : Ops) { 6630 if (MI.getOperand(Op).getSubReg()) 6631 return nullptr; 6632 } 6633 6634 // If loading from a FrameIndex, fold directly from the FrameIndex. 6635 unsigned NumOps = LoadMI.getDesc().getNumOperands(); 6636 int FrameIndex; 6637 if (isLoadFromStackSlot(LoadMI, FrameIndex)) { 6638 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) 6639 return nullptr; 6640 return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex, LIS); 6641 } 6642 6643 // Check switch flag 6644 if (NoFusing) return nullptr; 6645 6646 // Avoid partial and undef register update stalls unless optimizing for size. 6647 if (!MF.getFunction().hasOptSize() && 6648 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/true) || 6649 shouldPreventUndefRegUpdateMemFold(MF, MI))) 6650 return nullptr; 6651 6652 // Determine the alignment of the load. 6653 Align Alignment; 6654 if (LoadMI.hasOneMemOperand()) 6655 Alignment = (*LoadMI.memoperands_begin())->getAlign(); 6656 else 6657 switch (LoadMI.getOpcode()) { 6658 case X86::AVX512_512_SET0: 6659 case X86::AVX512_512_SETALLONES: 6660 Alignment = Align(64); 6661 break; 6662 case X86::AVX2_SETALLONES: 6663 case X86::AVX1_SETALLONES: 6664 case X86::AVX_SET0: 6665 case X86::AVX512_256_SET0: 6666 Alignment = Align(32); 6667 break; 6668 case X86::V_SET0: 6669 case X86::V_SETALLONES: 6670 case X86::AVX512_128_SET0: 6671 case X86::FsFLD0F128: 6672 case X86::AVX512_FsFLD0F128: 6673 Alignment = Align(16); 6674 break; 6675 case X86::MMX_SET0: 6676 case X86::FsFLD0SD: 6677 case X86::AVX512_FsFLD0SD: 6678 Alignment = Align(8); 6679 break; 6680 case X86::FsFLD0SS: 6681 case X86::AVX512_FsFLD0SS: 6682 Alignment = Align(4); 6683 break; 6684 case X86::FsFLD0SH: 6685 case X86::AVX512_FsFLD0SH: 6686 Alignment = Align(2); 6687 break; 6688 default: 6689 return nullptr; 6690 } 6691 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 6692 unsigned NewOpc = 0; 6693 switch (MI.getOpcode()) { 6694 default: return nullptr; 6695 case X86::TEST8rr: NewOpc = X86::CMP8ri; break; 6696 case X86::TEST16rr: NewOpc = X86::CMP16ri8; break; 6697 case X86::TEST32rr: NewOpc = X86::CMP32ri8; break; 6698 case X86::TEST64rr: NewOpc = X86::CMP64ri8; break; 6699 } 6700 // Change to CMPXXri r, 0 first. 6701 MI.setDesc(get(NewOpc)); 6702 MI.getOperand(1).ChangeToImmediate(0); 6703 } else if (Ops.size() != 1) 6704 return nullptr; 6705 6706 // Make sure the subregisters match. 6707 // Otherwise we risk changing the size of the load. 6708 if (LoadMI.getOperand(0).getSubReg() != MI.getOperand(Ops[0]).getSubReg()) 6709 return nullptr; 6710 6711 SmallVector<MachineOperand,X86::AddrNumOperands> MOs; 6712 switch (LoadMI.getOpcode()) { 6713 case X86::MMX_SET0: 6714 case X86::V_SET0: 6715 case X86::V_SETALLONES: 6716 case X86::AVX2_SETALLONES: 6717 case X86::AVX1_SETALLONES: 6718 case X86::AVX_SET0: 6719 case X86::AVX512_128_SET0: 6720 case X86::AVX512_256_SET0: 6721 case X86::AVX512_512_SET0: 6722 case X86::AVX512_512_SETALLONES: 6723 case X86::FsFLD0SH: 6724 case X86::AVX512_FsFLD0SH: 6725 case X86::FsFLD0SD: 6726 case X86::AVX512_FsFLD0SD: 6727 case X86::FsFLD0SS: 6728 case X86::AVX512_FsFLD0SS: 6729 case X86::FsFLD0F128: 6730 case X86::AVX512_FsFLD0F128: { 6731 // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure. 6732 // Create a constant-pool entry and operands to load from it. 6733 6734 // Medium and large mode can't fold loads this way. 6735 if (MF.getTarget().getCodeModel() != CodeModel::Small && 6736 MF.getTarget().getCodeModel() != CodeModel::Kernel) 6737 return nullptr; 6738 6739 // x86-32 PIC requires a PIC base register for constant pools. 6740 unsigned PICBase = 0; 6741 // Since we're using Small or Kernel code model, we can always use 6742 // RIP-relative addressing for a smaller encoding. 6743 if (Subtarget.is64Bit()) { 6744 PICBase = X86::RIP; 6745 } else if (MF.getTarget().isPositionIndependent()) { 6746 // FIXME: PICBase = getGlobalBaseReg(&MF); 6747 // This doesn't work for several reasons. 6748 // 1. GlobalBaseReg may have been spilled. 6749 // 2. It may not be live at MI. 6750 return nullptr; 6751 } 6752 6753 // Create a constant-pool entry. 6754 MachineConstantPool &MCP = *MF.getConstantPool(); 6755 Type *Ty; 6756 unsigned Opc = LoadMI.getOpcode(); 6757 if (Opc == X86::FsFLD0SS || Opc == X86::AVX512_FsFLD0SS) 6758 Ty = Type::getFloatTy(MF.getFunction().getContext()); 6759 else if (Opc == X86::FsFLD0SD || Opc == X86::AVX512_FsFLD0SD) 6760 Ty = Type::getDoubleTy(MF.getFunction().getContext()); 6761 else if (Opc == X86::FsFLD0F128 || Opc == X86::AVX512_FsFLD0F128) 6762 Ty = Type::getFP128Ty(MF.getFunction().getContext()); 6763 else if (Opc == X86::FsFLD0SH || Opc == X86::AVX512_FsFLD0SH) 6764 Ty = Type::getHalfTy(MF.getFunction().getContext()); 6765 else if (Opc == X86::AVX512_512_SET0 || Opc == X86::AVX512_512_SETALLONES) 6766 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 6767 16); 6768 else if (Opc == X86::AVX2_SETALLONES || Opc == X86::AVX_SET0 || 6769 Opc == X86::AVX512_256_SET0 || Opc == X86::AVX1_SETALLONES) 6770 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 6771 8); 6772 else if (Opc == X86::MMX_SET0) 6773 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 6774 2); 6775 else 6776 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 6777 4); 6778 6779 bool IsAllOnes = (Opc == X86::V_SETALLONES || Opc == X86::AVX2_SETALLONES || 6780 Opc == X86::AVX512_512_SETALLONES || 6781 Opc == X86::AVX1_SETALLONES); 6782 const Constant *C = IsAllOnes ? Constant::getAllOnesValue(Ty) : 6783 Constant::getNullValue(Ty); 6784 unsigned CPI = MCP.getConstantPoolIndex(C, Alignment); 6785 6786 // Create operands to load from the constant pool entry. 6787 MOs.push_back(MachineOperand::CreateReg(PICBase, false)); 6788 MOs.push_back(MachineOperand::CreateImm(1)); 6789 MOs.push_back(MachineOperand::CreateReg(0, false)); 6790 MOs.push_back(MachineOperand::CreateCPI(CPI, 0)); 6791 MOs.push_back(MachineOperand::CreateReg(0, false)); 6792 break; 6793 } 6794 default: { 6795 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) 6796 return nullptr; 6797 6798 // Folding a normal load. Just copy the load's address operands. 6799 MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, 6800 LoadMI.operands_begin() + NumOps); 6801 break; 6802 } 6803 } 6804 return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, InsertPt, 6805 /*Size=*/0, Alignment, /*AllowCommute=*/true); 6806 } 6807 6808 static SmallVector<MachineMemOperand *, 2> 6809 extractLoadMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { 6810 SmallVector<MachineMemOperand *, 2> LoadMMOs; 6811 6812 for (MachineMemOperand *MMO : MMOs) { 6813 if (!MMO->isLoad()) 6814 continue; 6815 6816 if (!MMO->isStore()) { 6817 // Reuse the MMO. 6818 LoadMMOs.push_back(MMO); 6819 } else { 6820 // Clone the MMO and unset the store flag. 6821 LoadMMOs.push_back(MF.getMachineMemOperand( 6822 MMO, MMO->getFlags() & ~MachineMemOperand::MOStore)); 6823 } 6824 } 6825 6826 return LoadMMOs; 6827 } 6828 6829 static SmallVector<MachineMemOperand *, 2> 6830 extractStoreMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { 6831 SmallVector<MachineMemOperand *, 2> StoreMMOs; 6832 6833 for (MachineMemOperand *MMO : MMOs) { 6834 if (!MMO->isStore()) 6835 continue; 6836 6837 if (!MMO->isLoad()) { 6838 // Reuse the MMO. 6839 StoreMMOs.push_back(MMO); 6840 } else { 6841 // Clone the MMO and unset the load flag. 6842 StoreMMOs.push_back(MF.getMachineMemOperand( 6843 MMO, MMO->getFlags() & ~MachineMemOperand::MOLoad)); 6844 } 6845 } 6846 6847 return StoreMMOs; 6848 } 6849 6850 static unsigned getBroadcastOpcode(const X86MemoryFoldTableEntry *I, 6851 const TargetRegisterClass *RC, 6852 const X86Subtarget &STI) { 6853 assert(STI.hasAVX512() && "Expected at least AVX512!"); 6854 unsigned SpillSize = STI.getRegisterInfo()->getSpillSize(*RC); 6855 assert((SpillSize == 64 || STI.hasVLX()) && 6856 "Can't broadcast less than 64 bytes without AVX512VL!"); 6857 6858 switch (I->Flags & TB_BCAST_MASK) { 6859 default: llvm_unreachable("Unexpected broadcast type!"); 6860 case TB_BCAST_D: 6861 switch (SpillSize) { 6862 default: llvm_unreachable("Unknown spill size"); 6863 case 16: return X86::VPBROADCASTDZ128rm; 6864 case 32: return X86::VPBROADCASTDZ256rm; 6865 case 64: return X86::VPBROADCASTDZrm; 6866 } 6867 break; 6868 case TB_BCAST_Q: 6869 switch (SpillSize) { 6870 default: llvm_unreachable("Unknown spill size"); 6871 case 16: return X86::VPBROADCASTQZ128rm; 6872 case 32: return X86::VPBROADCASTQZ256rm; 6873 case 64: return X86::VPBROADCASTQZrm; 6874 } 6875 break; 6876 case TB_BCAST_SS: 6877 switch (SpillSize) { 6878 default: llvm_unreachable("Unknown spill size"); 6879 case 16: return X86::VBROADCASTSSZ128rm; 6880 case 32: return X86::VBROADCASTSSZ256rm; 6881 case 64: return X86::VBROADCASTSSZrm; 6882 } 6883 break; 6884 case TB_BCAST_SD: 6885 switch (SpillSize) { 6886 default: llvm_unreachable("Unknown spill size"); 6887 case 16: return X86::VMOVDDUPZ128rm; 6888 case 32: return X86::VBROADCASTSDZ256rm; 6889 case 64: return X86::VBROADCASTSDZrm; 6890 } 6891 break; 6892 } 6893 } 6894 6895 bool X86InstrInfo::unfoldMemoryOperand( 6896 MachineFunction &MF, MachineInstr &MI, unsigned Reg, bool UnfoldLoad, 6897 bool UnfoldStore, SmallVectorImpl<MachineInstr *> &NewMIs) const { 6898 const X86MemoryFoldTableEntry *I = lookupUnfoldTable(MI.getOpcode()); 6899 if (I == nullptr) 6900 return false; 6901 unsigned Opc = I->DstOp; 6902 unsigned Index = I->Flags & TB_INDEX_MASK; 6903 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 6904 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 6905 bool FoldedBCast = I->Flags & TB_FOLDED_BCAST; 6906 if (UnfoldLoad && !FoldedLoad) 6907 return false; 6908 UnfoldLoad &= FoldedLoad; 6909 if (UnfoldStore && !FoldedStore) 6910 return false; 6911 UnfoldStore &= FoldedStore; 6912 6913 const MCInstrDesc &MCID = get(Opc); 6914 6915 const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); 6916 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6917 // TODO: Check if 32-byte or greater accesses are slow too? 6918 if (!MI.hasOneMemOperand() && RC == &X86::VR128RegClass && 6919 Subtarget.isUnalignedMem16Slow()) 6920 // Without memoperands, loadRegFromAddr and storeRegToStackSlot will 6921 // conservatively assume the address is unaligned. That's bad for 6922 // performance. 6923 return false; 6924 SmallVector<MachineOperand, X86::AddrNumOperands> AddrOps; 6925 SmallVector<MachineOperand,2> BeforeOps; 6926 SmallVector<MachineOperand,2> AfterOps; 6927 SmallVector<MachineOperand,4> ImpOps; 6928 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 6929 MachineOperand &Op = MI.getOperand(i); 6930 if (i >= Index && i < Index + X86::AddrNumOperands) 6931 AddrOps.push_back(Op); 6932 else if (Op.isReg() && Op.isImplicit()) 6933 ImpOps.push_back(Op); 6934 else if (i < Index) 6935 BeforeOps.push_back(Op); 6936 else if (i > Index) 6937 AfterOps.push_back(Op); 6938 } 6939 6940 // Emit the load or broadcast instruction. 6941 if (UnfoldLoad) { 6942 auto MMOs = extractLoadMMOs(MI.memoperands(), MF); 6943 6944 unsigned Opc; 6945 if (FoldedBCast) { 6946 Opc = getBroadcastOpcode(I, RC, Subtarget); 6947 } else { 6948 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 6949 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 6950 Opc = getLoadRegOpcode(Reg, RC, isAligned, Subtarget); 6951 } 6952 6953 DebugLoc DL; 6954 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), Reg); 6955 for (unsigned i = 0, e = AddrOps.size(); i != e; ++i) 6956 MIB.add(AddrOps[i]); 6957 MIB.setMemRefs(MMOs); 6958 NewMIs.push_back(MIB); 6959 6960 if (UnfoldStore) { 6961 // Address operands cannot be marked isKill. 6962 for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) { 6963 MachineOperand &MO = NewMIs[0]->getOperand(i); 6964 if (MO.isReg()) 6965 MO.setIsKill(false); 6966 } 6967 } 6968 } 6969 6970 // Emit the data processing instruction. 6971 MachineInstr *DataMI = MF.CreateMachineInstr(MCID, MI.getDebugLoc(), true); 6972 MachineInstrBuilder MIB(MF, DataMI); 6973 6974 if (FoldedStore) 6975 MIB.addReg(Reg, RegState::Define); 6976 for (MachineOperand &BeforeOp : BeforeOps) 6977 MIB.add(BeforeOp); 6978 if (FoldedLoad) 6979 MIB.addReg(Reg); 6980 for (MachineOperand &AfterOp : AfterOps) 6981 MIB.add(AfterOp); 6982 for (MachineOperand &ImpOp : ImpOps) { 6983 MIB.addReg(ImpOp.getReg(), 6984 getDefRegState(ImpOp.isDef()) | 6985 RegState::Implicit | 6986 getKillRegState(ImpOp.isKill()) | 6987 getDeadRegState(ImpOp.isDead()) | 6988 getUndefRegState(ImpOp.isUndef())); 6989 } 6990 // Change CMP32ri r, 0 back to TEST32rr r, r, etc. 6991 switch (DataMI->getOpcode()) { 6992 default: break; 6993 case X86::CMP64ri32: 6994 case X86::CMP64ri8: 6995 case X86::CMP32ri: 6996 case X86::CMP32ri8: 6997 case X86::CMP16ri: 6998 case X86::CMP16ri8: 6999 case X86::CMP8ri: { 7000 MachineOperand &MO0 = DataMI->getOperand(0); 7001 MachineOperand &MO1 = DataMI->getOperand(1); 7002 if (MO1.isImm() && MO1.getImm() == 0) { 7003 unsigned NewOpc; 7004 switch (DataMI->getOpcode()) { 7005 default: llvm_unreachable("Unreachable!"); 7006 case X86::CMP64ri8: 7007 case X86::CMP64ri32: NewOpc = X86::TEST64rr; break; 7008 case X86::CMP32ri8: 7009 case X86::CMP32ri: NewOpc = X86::TEST32rr; break; 7010 case X86::CMP16ri8: 7011 case X86::CMP16ri: NewOpc = X86::TEST16rr; break; 7012 case X86::CMP8ri: NewOpc = X86::TEST8rr; break; 7013 } 7014 DataMI->setDesc(get(NewOpc)); 7015 MO1.ChangeToRegister(MO0.getReg(), false); 7016 } 7017 } 7018 } 7019 NewMIs.push_back(DataMI); 7020 7021 // Emit the store instruction. 7022 if (UnfoldStore) { 7023 const TargetRegisterClass *DstRC = getRegClass(MCID, 0, &RI, MF); 7024 auto MMOs = extractStoreMMOs(MI.memoperands(), MF); 7025 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*DstRC), 16); 7026 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 7027 unsigned Opc = getStoreRegOpcode(Reg, DstRC, isAligned, Subtarget); 7028 DebugLoc DL; 7029 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc)); 7030 for (unsigned i = 0, e = AddrOps.size(); i != e; ++i) 7031 MIB.add(AddrOps[i]); 7032 MIB.addReg(Reg, RegState::Kill); 7033 MIB.setMemRefs(MMOs); 7034 NewMIs.push_back(MIB); 7035 } 7036 7037 return true; 7038 } 7039 7040 bool 7041 X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, 7042 SmallVectorImpl<SDNode*> &NewNodes) const { 7043 if (!N->isMachineOpcode()) 7044 return false; 7045 7046 const X86MemoryFoldTableEntry *I = lookupUnfoldTable(N->getMachineOpcode()); 7047 if (I == nullptr) 7048 return false; 7049 unsigned Opc = I->DstOp; 7050 unsigned Index = I->Flags & TB_INDEX_MASK; 7051 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 7052 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 7053 bool FoldedBCast = I->Flags & TB_FOLDED_BCAST; 7054 const MCInstrDesc &MCID = get(Opc); 7055 MachineFunction &MF = DAG.getMachineFunction(); 7056 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 7057 const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); 7058 unsigned NumDefs = MCID.NumDefs; 7059 std::vector<SDValue> AddrOps; 7060 std::vector<SDValue> BeforeOps; 7061 std::vector<SDValue> AfterOps; 7062 SDLoc dl(N); 7063 unsigned NumOps = N->getNumOperands(); 7064 for (unsigned i = 0; i != NumOps-1; ++i) { 7065 SDValue Op = N->getOperand(i); 7066 if (i >= Index-NumDefs && i < Index-NumDefs + X86::AddrNumOperands) 7067 AddrOps.push_back(Op); 7068 else if (i < Index-NumDefs) 7069 BeforeOps.push_back(Op); 7070 else if (i > Index-NumDefs) 7071 AfterOps.push_back(Op); 7072 } 7073 SDValue Chain = N->getOperand(NumOps-1); 7074 AddrOps.push_back(Chain); 7075 7076 // Emit the load instruction. 7077 SDNode *Load = nullptr; 7078 if (FoldedLoad) { 7079 EVT VT = *TRI.legalclasstypes_begin(*RC); 7080 auto MMOs = extractLoadMMOs(cast<MachineSDNode>(N)->memoperands(), MF); 7081 if (MMOs.empty() && RC == &X86::VR128RegClass && 7082 Subtarget.isUnalignedMem16Slow()) 7083 // Do not introduce a slow unaligned load. 7084 return false; 7085 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte 7086 // memory access is slow above. 7087 7088 unsigned Opc; 7089 if (FoldedBCast) { 7090 Opc = getBroadcastOpcode(I, RC, Subtarget); 7091 } else { 7092 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 7093 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 7094 Opc = getLoadRegOpcode(0, RC, isAligned, Subtarget); 7095 } 7096 7097 Load = DAG.getMachineNode(Opc, dl, VT, MVT::Other, AddrOps); 7098 NewNodes.push_back(Load); 7099 7100 // Preserve memory reference information. 7101 DAG.setNodeMemRefs(cast<MachineSDNode>(Load), MMOs); 7102 } 7103 7104 // Emit the data processing instruction. 7105 std::vector<EVT> VTs; 7106 const TargetRegisterClass *DstRC = nullptr; 7107 if (MCID.getNumDefs() > 0) { 7108 DstRC = getRegClass(MCID, 0, &RI, MF); 7109 VTs.push_back(*TRI.legalclasstypes_begin(*DstRC)); 7110 } 7111 for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { 7112 EVT VT = N->getValueType(i); 7113 if (VT != MVT::Other && i >= (unsigned)MCID.getNumDefs()) 7114 VTs.push_back(VT); 7115 } 7116 if (Load) 7117 BeforeOps.push_back(SDValue(Load, 0)); 7118 llvm::append_range(BeforeOps, AfterOps); 7119 // Change CMP32ri r, 0 back to TEST32rr r, r, etc. 7120 switch (Opc) { 7121 default: break; 7122 case X86::CMP64ri32: 7123 case X86::CMP64ri8: 7124 case X86::CMP32ri: 7125 case X86::CMP32ri8: 7126 case X86::CMP16ri: 7127 case X86::CMP16ri8: 7128 case X86::CMP8ri: 7129 if (isNullConstant(BeforeOps[1])) { 7130 switch (Opc) { 7131 default: llvm_unreachable("Unreachable!"); 7132 case X86::CMP64ri8: 7133 case X86::CMP64ri32: Opc = X86::TEST64rr; break; 7134 case X86::CMP32ri8: 7135 case X86::CMP32ri: Opc = X86::TEST32rr; break; 7136 case X86::CMP16ri8: 7137 case X86::CMP16ri: Opc = X86::TEST16rr; break; 7138 case X86::CMP8ri: Opc = X86::TEST8rr; break; 7139 } 7140 BeforeOps[1] = BeforeOps[0]; 7141 } 7142 } 7143 SDNode *NewNode= DAG.getMachineNode(Opc, dl, VTs, BeforeOps); 7144 NewNodes.push_back(NewNode); 7145 7146 // Emit the store instruction. 7147 if (FoldedStore) { 7148 AddrOps.pop_back(); 7149 AddrOps.push_back(SDValue(NewNode, 0)); 7150 AddrOps.push_back(Chain); 7151 auto MMOs = extractStoreMMOs(cast<MachineSDNode>(N)->memoperands(), MF); 7152 if (MMOs.empty() && RC == &X86::VR128RegClass && 7153 Subtarget.isUnalignedMem16Slow()) 7154 // Do not introduce a slow unaligned store. 7155 return false; 7156 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte 7157 // memory access is slow above. 7158 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 7159 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 7160 SDNode *Store = 7161 DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget), 7162 dl, MVT::Other, AddrOps); 7163 NewNodes.push_back(Store); 7164 7165 // Preserve memory reference information. 7166 DAG.setNodeMemRefs(cast<MachineSDNode>(Store), MMOs); 7167 } 7168 7169 return true; 7170 } 7171 7172 unsigned X86InstrInfo::getOpcodeAfterMemoryUnfold(unsigned Opc, 7173 bool UnfoldLoad, bool UnfoldStore, 7174 unsigned *LoadRegIndex) const { 7175 const X86MemoryFoldTableEntry *I = lookupUnfoldTable(Opc); 7176 if (I == nullptr) 7177 return 0; 7178 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 7179 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 7180 if (UnfoldLoad && !FoldedLoad) 7181 return 0; 7182 if (UnfoldStore && !FoldedStore) 7183 return 0; 7184 if (LoadRegIndex) 7185 *LoadRegIndex = I->Flags & TB_INDEX_MASK; 7186 return I->DstOp; 7187 } 7188 7189 bool 7190 X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, 7191 int64_t &Offset1, int64_t &Offset2) const { 7192 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode()) 7193 return false; 7194 unsigned Opc1 = Load1->getMachineOpcode(); 7195 unsigned Opc2 = Load2->getMachineOpcode(); 7196 switch (Opc1) { 7197 default: return false; 7198 case X86::MOV8rm: 7199 case X86::MOV16rm: 7200 case X86::MOV32rm: 7201 case X86::MOV64rm: 7202 case X86::LD_Fp32m: 7203 case X86::LD_Fp64m: 7204 case X86::LD_Fp80m: 7205 case X86::MOVSSrm: 7206 case X86::MOVSSrm_alt: 7207 case X86::MOVSDrm: 7208 case X86::MOVSDrm_alt: 7209 case X86::MMX_MOVD64rm: 7210 case X86::MMX_MOVQ64rm: 7211 case X86::MOVAPSrm: 7212 case X86::MOVUPSrm: 7213 case X86::MOVAPDrm: 7214 case X86::MOVUPDrm: 7215 case X86::MOVDQArm: 7216 case X86::MOVDQUrm: 7217 // AVX load instructions 7218 case X86::VMOVSSrm: 7219 case X86::VMOVSSrm_alt: 7220 case X86::VMOVSDrm: 7221 case X86::VMOVSDrm_alt: 7222 case X86::VMOVAPSrm: 7223 case X86::VMOVUPSrm: 7224 case X86::VMOVAPDrm: 7225 case X86::VMOVUPDrm: 7226 case X86::VMOVDQArm: 7227 case X86::VMOVDQUrm: 7228 case X86::VMOVAPSYrm: 7229 case X86::VMOVUPSYrm: 7230 case X86::VMOVAPDYrm: 7231 case X86::VMOVUPDYrm: 7232 case X86::VMOVDQAYrm: 7233 case X86::VMOVDQUYrm: 7234 // AVX512 load instructions 7235 case X86::VMOVSSZrm: 7236 case X86::VMOVSSZrm_alt: 7237 case X86::VMOVSDZrm: 7238 case X86::VMOVSDZrm_alt: 7239 case X86::VMOVAPSZ128rm: 7240 case X86::VMOVUPSZ128rm: 7241 case X86::VMOVAPSZ128rm_NOVLX: 7242 case X86::VMOVUPSZ128rm_NOVLX: 7243 case X86::VMOVAPDZ128rm: 7244 case X86::VMOVUPDZ128rm: 7245 case X86::VMOVDQU8Z128rm: 7246 case X86::VMOVDQU16Z128rm: 7247 case X86::VMOVDQA32Z128rm: 7248 case X86::VMOVDQU32Z128rm: 7249 case X86::VMOVDQA64Z128rm: 7250 case X86::VMOVDQU64Z128rm: 7251 case X86::VMOVAPSZ256rm: 7252 case X86::VMOVUPSZ256rm: 7253 case X86::VMOVAPSZ256rm_NOVLX: 7254 case X86::VMOVUPSZ256rm_NOVLX: 7255 case X86::VMOVAPDZ256rm: 7256 case X86::VMOVUPDZ256rm: 7257 case X86::VMOVDQU8Z256rm: 7258 case X86::VMOVDQU16Z256rm: 7259 case X86::VMOVDQA32Z256rm: 7260 case X86::VMOVDQU32Z256rm: 7261 case X86::VMOVDQA64Z256rm: 7262 case X86::VMOVDQU64Z256rm: 7263 case X86::VMOVAPSZrm: 7264 case X86::VMOVUPSZrm: 7265 case X86::VMOVAPDZrm: 7266 case X86::VMOVUPDZrm: 7267 case X86::VMOVDQU8Zrm: 7268 case X86::VMOVDQU16Zrm: 7269 case X86::VMOVDQA32Zrm: 7270 case X86::VMOVDQU32Zrm: 7271 case X86::VMOVDQA64Zrm: 7272 case X86::VMOVDQU64Zrm: 7273 case X86::KMOVBkm: 7274 case X86::KMOVWkm: 7275 case X86::KMOVDkm: 7276 case X86::KMOVQkm: 7277 break; 7278 } 7279 switch (Opc2) { 7280 default: return false; 7281 case X86::MOV8rm: 7282 case X86::MOV16rm: 7283 case X86::MOV32rm: 7284 case X86::MOV64rm: 7285 case X86::LD_Fp32m: 7286 case X86::LD_Fp64m: 7287 case X86::LD_Fp80m: 7288 case X86::MOVSSrm: 7289 case X86::MOVSSrm_alt: 7290 case X86::MOVSDrm: 7291 case X86::MOVSDrm_alt: 7292 case X86::MMX_MOVD64rm: 7293 case X86::MMX_MOVQ64rm: 7294 case X86::MOVAPSrm: 7295 case X86::MOVUPSrm: 7296 case X86::MOVAPDrm: 7297 case X86::MOVUPDrm: 7298 case X86::MOVDQArm: 7299 case X86::MOVDQUrm: 7300 // AVX load instructions 7301 case X86::VMOVSSrm: 7302 case X86::VMOVSSrm_alt: 7303 case X86::VMOVSDrm: 7304 case X86::VMOVSDrm_alt: 7305 case X86::VMOVAPSrm: 7306 case X86::VMOVUPSrm: 7307 case X86::VMOVAPDrm: 7308 case X86::VMOVUPDrm: 7309 case X86::VMOVDQArm: 7310 case X86::VMOVDQUrm: 7311 case X86::VMOVAPSYrm: 7312 case X86::VMOVUPSYrm: 7313 case X86::VMOVAPDYrm: 7314 case X86::VMOVUPDYrm: 7315 case X86::VMOVDQAYrm: 7316 case X86::VMOVDQUYrm: 7317 // AVX512 load instructions 7318 case X86::VMOVSSZrm: 7319 case X86::VMOVSSZrm_alt: 7320 case X86::VMOVSDZrm: 7321 case X86::VMOVSDZrm_alt: 7322 case X86::VMOVAPSZ128rm: 7323 case X86::VMOVUPSZ128rm: 7324 case X86::VMOVAPSZ128rm_NOVLX: 7325 case X86::VMOVUPSZ128rm_NOVLX: 7326 case X86::VMOVAPDZ128rm: 7327 case X86::VMOVUPDZ128rm: 7328 case X86::VMOVDQU8Z128rm: 7329 case X86::VMOVDQU16Z128rm: 7330 case X86::VMOVDQA32Z128rm: 7331 case X86::VMOVDQU32Z128rm: 7332 case X86::VMOVDQA64Z128rm: 7333 case X86::VMOVDQU64Z128rm: 7334 case X86::VMOVAPSZ256rm: 7335 case X86::VMOVUPSZ256rm: 7336 case X86::VMOVAPSZ256rm_NOVLX: 7337 case X86::VMOVUPSZ256rm_NOVLX: 7338 case X86::VMOVAPDZ256rm: 7339 case X86::VMOVUPDZ256rm: 7340 case X86::VMOVDQU8Z256rm: 7341 case X86::VMOVDQU16Z256rm: 7342 case X86::VMOVDQA32Z256rm: 7343 case X86::VMOVDQU32Z256rm: 7344 case X86::VMOVDQA64Z256rm: 7345 case X86::VMOVDQU64Z256rm: 7346 case X86::VMOVAPSZrm: 7347 case X86::VMOVUPSZrm: 7348 case X86::VMOVAPDZrm: 7349 case X86::VMOVUPDZrm: 7350 case X86::VMOVDQU8Zrm: 7351 case X86::VMOVDQU16Zrm: 7352 case X86::VMOVDQA32Zrm: 7353 case X86::VMOVDQU32Zrm: 7354 case X86::VMOVDQA64Zrm: 7355 case X86::VMOVDQU64Zrm: 7356 case X86::KMOVBkm: 7357 case X86::KMOVWkm: 7358 case X86::KMOVDkm: 7359 case X86::KMOVQkm: 7360 break; 7361 } 7362 7363 // Lambda to check if both the loads have the same value for an operand index. 7364 auto HasSameOp = [&](int I) { 7365 return Load1->getOperand(I) == Load2->getOperand(I); 7366 }; 7367 7368 // All operands except the displacement should match. 7369 if (!HasSameOp(X86::AddrBaseReg) || !HasSameOp(X86::AddrScaleAmt) || 7370 !HasSameOp(X86::AddrIndexReg) || !HasSameOp(X86::AddrSegmentReg)) 7371 return false; 7372 7373 // Chain Operand must be the same. 7374 if (!HasSameOp(5)) 7375 return false; 7376 7377 // Now let's examine if the displacements are constants. 7378 auto Disp1 = dyn_cast<ConstantSDNode>(Load1->getOperand(X86::AddrDisp)); 7379 auto Disp2 = dyn_cast<ConstantSDNode>(Load2->getOperand(X86::AddrDisp)); 7380 if (!Disp1 || !Disp2) 7381 return false; 7382 7383 Offset1 = Disp1->getSExtValue(); 7384 Offset2 = Disp2->getSExtValue(); 7385 return true; 7386 } 7387 7388 bool X86InstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, 7389 int64_t Offset1, int64_t Offset2, 7390 unsigned NumLoads) const { 7391 assert(Offset2 > Offset1); 7392 if ((Offset2 - Offset1) / 8 > 64) 7393 return false; 7394 7395 unsigned Opc1 = Load1->getMachineOpcode(); 7396 unsigned Opc2 = Load2->getMachineOpcode(); 7397 if (Opc1 != Opc2) 7398 return false; // FIXME: overly conservative? 7399 7400 switch (Opc1) { 7401 default: break; 7402 case X86::LD_Fp32m: 7403 case X86::LD_Fp64m: 7404 case X86::LD_Fp80m: 7405 case X86::MMX_MOVD64rm: 7406 case X86::MMX_MOVQ64rm: 7407 return false; 7408 } 7409 7410 EVT VT = Load1->getValueType(0); 7411 switch (VT.getSimpleVT().SimpleTy) { 7412 default: 7413 // XMM registers. In 64-bit mode we can be a bit more aggressive since we 7414 // have 16 of them to play with. 7415 if (Subtarget.is64Bit()) { 7416 if (NumLoads >= 3) 7417 return false; 7418 } else if (NumLoads) { 7419 return false; 7420 } 7421 break; 7422 case MVT::i8: 7423 case MVT::i16: 7424 case MVT::i32: 7425 case MVT::i64: 7426 case MVT::f32: 7427 case MVT::f64: 7428 if (NumLoads) 7429 return false; 7430 break; 7431 } 7432 7433 return true; 7434 } 7435 7436 bool X86InstrInfo::isSchedulingBoundary(const MachineInstr &MI, 7437 const MachineBasicBlock *MBB, 7438 const MachineFunction &MF) const { 7439 7440 // ENDBR instructions should not be scheduled around. 7441 unsigned Opcode = MI.getOpcode(); 7442 if (Opcode == X86::ENDBR64 || Opcode == X86::ENDBR32 || 7443 Opcode == X86::PLDTILECFGV) 7444 return true; 7445 7446 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF); 7447 } 7448 7449 bool X86InstrInfo:: 7450 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 7451 assert(Cond.size() == 1 && "Invalid X86 branch condition!"); 7452 X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm()); 7453 Cond[0].setImm(GetOppositeBranchCondition(CC)); 7454 return false; 7455 } 7456 7457 bool X86InstrInfo:: 7458 isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const { 7459 // FIXME: Return false for x87 stack register classes for now. We can't 7460 // allow any loads of these registers before FpGet_ST0_80. 7461 return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass || 7462 RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass || 7463 RC == &X86::RFP80RegClass); 7464 } 7465 7466 /// Return a virtual register initialized with the 7467 /// the global base register value. Output instructions required to 7468 /// initialize the register in the function entry block, if necessary. 7469 /// 7470 /// TODO: Eliminate this and move the code to X86MachineFunctionInfo. 7471 /// 7472 unsigned X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const { 7473 assert((!Subtarget.is64Bit() || 7474 MF->getTarget().getCodeModel() == CodeModel::Medium || 7475 MF->getTarget().getCodeModel() == CodeModel::Large) && 7476 "X86-64 PIC uses RIP relative addressing"); 7477 7478 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); 7479 Register GlobalBaseReg = X86FI->getGlobalBaseReg(); 7480 if (GlobalBaseReg != 0) 7481 return GlobalBaseReg; 7482 7483 // Create the register. The code to initialize it is inserted 7484 // later, by the CGBR pass (below). 7485 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 7486 GlobalBaseReg = RegInfo.createVirtualRegister( 7487 Subtarget.is64Bit() ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass); 7488 X86FI->setGlobalBaseReg(GlobalBaseReg); 7489 return GlobalBaseReg; 7490 } 7491 7492 // These are the replaceable SSE instructions. Some of these have Int variants 7493 // that we don't include here. We don't want to replace instructions selected 7494 // by intrinsics. 7495 static const uint16_t ReplaceableInstrs[][3] = { 7496 //PackedSingle PackedDouble PackedInt 7497 { X86::MOVAPSmr, X86::MOVAPDmr, X86::MOVDQAmr }, 7498 { X86::MOVAPSrm, X86::MOVAPDrm, X86::MOVDQArm }, 7499 { X86::MOVAPSrr, X86::MOVAPDrr, X86::MOVDQArr }, 7500 { X86::MOVUPSmr, X86::MOVUPDmr, X86::MOVDQUmr }, 7501 { X86::MOVUPSrm, X86::MOVUPDrm, X86::MOVDQUrm }, 7502 { X86::MOVLPSmr, X86::MOVLPDmr, X86::MOVPQI2QImr }, 7503 { X86::MOVSDmr, X86::MOVSDmr, X86::MOVPQI2QImr }, 7504 { X86::MOVSSmr, X86::MOVSSmr, X86::MOVPDI2DImr }, 7505 { X86::MOVSDrm, X86::MOVSDrm, X86::MOVQI2PQIrm }, 7506 { X86::MOVSDrm_alt,X86::MOVSDrm_alt,X86::MOVQI2PQIrm }, 7507 { X86::MOVSSrm, X86::MOVSSrm, X86::MOVDI2PDIrm }, 7508 { X86::MOVSSrm_alt,X86::MOVSSrm_alt,X86::MOVDI2PDIrm }, 7509 { X86::MOVNTPSmr, X86::MOVNTPDmr, X86::MOVNTDQmr }, 7510 { X86::ANDNPSrm, X86::ANDNPDrm, X86::PANDNrm }, 7511 { X86::ANDNPSrr, X86::ANDNPDrr, X86::PANDNrr }, 7512 { X86::ANDPSrm, X86::ANDPDrm, X86::PANDrm }, 7513 { X86::ANDPSrr, X86::ANDPDrr, X86::PANDrr }, 7514 { X86::ORPSrm, X86::ORPDrm, X86::PORrm }, 7515 { X86::ORPSrr, X86::ORPDrr, X86::PORrr }, 7516 { X86::XORPSrm, X86::XORPDrm, X86::PXORrm }, 7517 { X86::XORPSrr, X86::XORPDrr, X86::PXORrr }, 7518 { X86::UNPCKLPDrm, X86::UNPCKLPDrm, X86::PUNPCKLQDQrm }, 7519 { X86::MOVLHPSrr, X86::UNPCKLPDrr, X86::PUNPCKLQDQrr }, 7520 { X86::UNPCKHPDrm, X86::UNPCKHPDrm, X86::PUNPCKHQDQrm }, 7521 { X86::UNPCKHPDrr, X86::UNPCKHPDrr, X86::PUNPCKHQDQrr }, 7522 { X86::UNPCKLPSrm, X86::UNPCKLPSrm, X86::PUNPCKLDQrm }, 7523 { X86::UNPCKLPSrr, X86::UNPCKLPSrr, X86::PUNPCKLDQrr }, 7524 { X86::UNPCKHPSrm, X86::UNPCKHPSrm, X86::PUNPCKHDQrm }, 7525 { X86::UNPCKHPSrr, X86::UNPCKHPSrr, X86::PUNPCKHDQrr }, 7526 { X86::EXTRACTPSmr, X86::EXTRACTPSmr, X86::PEXTRDmr }, 7527 { X86::EXTRACTPSrr, X86::EXTRACTPSrr, X86::PEXTRDrr }, 7528 // AVX 128-bit support 7529 { X86::VMOVAPSmr, X86::VMOVAPDmr, X86::VMOVDQAmr }, 7530 { X86::VMOVAPSrm, X86::VMOVAPDrm, X86::VMOVDQArm }, 7531 { X86::VMOVAPSrr, X86::VMOVAPDrr, X86::VMOVDQArr }, 7532 { X86::VMOVUPSmr, X86::VMOVUPDmr, X86::VMOVDQUmr }, 7533 { X86::VMOVUPSrm, X86::VMOVUPDrm, X86::VMOVDQUrm }, 7534 { X86::VMOVLPSmr, X86::VMOVLPDmr, X86::VMOVPQI2QImr }, 7535 { X86::VMOVSDmr, X86::VMOVSDmr, X86::VMOVPQI2QImr }, 7536 { X86::VMOVSSmr, X86::VMOVSSmr, X86::VMOVPDI2DImr }, 7537 { X86::VMOVSDrm, X86::VMOVSDrm, X86::VMOVQI2PQIrm }, 7538 { X86::VMOVSDrm_alt,X86::VMOVSDrm_alt,X86::VMOVQI2PQIrm }, 7539 { X86::VMOVSSrm, X86::VMOVSSrm, X86::VMOVDI2PDIrm }, 7540 { X86::VMOVSSrm_alt,X86::VMOVSSrm_alt,X86::VMOVDI2PDIrm }, 7541 { X86::VMOVNTPSmr, X86::VMOVNTPDmr, X86::VMOVNTDQmr }, 7542 { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNrm }, 7543 { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNrr }, 7544 { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDrm }, 7545 { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDrr }, 7546 { X86::VORPSrm, X86::VORPDrm, X86::VPORrm }, 7547 { X86::VORPSrr, X86::VORPDrr, X86::VPORrr }, 7548 { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORrm }, 7549 { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORrr }, 7550 { X86::VUNPCKLPDrm, X86::VUNPCKLPDrm, X86::VPUNPCKLQDQrm }, 7551 { X86::VMOVLHPSrr, X86::VUNPCKLPDrr, X86::VPUNPCKLQDQrr }, 7552 { X86::VUNPCKHPDrm, X86::VUNPCKHPDrm, X86::VPUNPCKHQDQrm }, 7553 { X86::VUNPCKHPDrr, X86::VUNPCKHPDrr, X86::VPUNPCKHQDQrr }, 7554 { X86::VUNPCKLPSrm, X86::VUNPCKLPSrm, X86::VPUNPCKLDQrm }, 7555 { X86::VUNPCKLPSrr, X86::VUNPCKLPSrr, X86::VPUNPCKLDQrr }, 7556 { X86::VUNPCKHPSrm, X86::VUNPCKHPSrm, X86::VPUNPCKHDQrm }, 7557 { X86::VUNPCKHPSrr, X86::VUNPCKHPSrr, X86::VPUNPCKHDQrr }, 7558 { X86::VEXTRACTPSmr, X86::VEXTRACTPSmr, X86::VPEXTRDmr }, 7559 { X86::VEXTRACTPSrr, X86::VEXTRACTPSrr, X86::VPEXTRDrr }, 7560 // AVX 256-bit support 7561 { X86::VMOVAPSYmr, X86::VMOVAPDYmr, X86::VMOVDQAYmr }, 7562 { X86::VMOVAPSYrm, X86::VMOVAPDYrm, X86::VMOVDQAYrm }, 7563 { X86::VMOVAPSYrr, X86::VMOVAPDYrr, X86::VMOVDQAYrr }, 7564 { X86::VMOVUPSYmr, X86::VMOVUPDYmr, X86::VMOVDQUYmr }, 7565 { X86::VMOVUPSYrm, X86::VMOVUPDYrm, X86::VMOVDQUYrm }, 7566 { X86::VMOVNTPSYmr, X86::VMOVNTPDYmr, X86::VMOVNTDQYmr }, 7567 { X86::VPERMPSYrm, X86::VPERMPSYrm, X86::VPERMDYrm }, 7568 { X86::VPERMPSYrr, X86::VPERMPSYrr, X86::VPERMDYrr }, 7569 { X86::VPERMPDYmi, X86::VPERMPDYmi, X86::VPERMQYmi }, 7570 { X86::VPERMPDYri, X86::VPERMPDYri, X86::VPERMQYri }, 7571 // AVX512 support 7572 { X86::VMOVLPSZ128mr, X86::VMOVLPDZ128mr, X86::VMOVPQI2QIZmr }, 7573 { X86::VMOVNTPSZ128mr, X86::VMOVNTPDZ128mr, X86::VMOVNTDQZ128mr }, 7574 { X86::VMOVNTPSZ256mr, X86::VMOVNTPDZ256mr, X86::VMOVNTDQZ256mr }, 7575 { X86::VMOVNTPSZmr, X86::VMOVNTPDZmr, X86::VMOVNTDQZmr }, 7576 { X86::VMOVSDZmr, X86::VMOVSDZmr, X86::VMOVPQI2QIZmr }, 7577 { X86::VMOVSSZmr, X86::VMOVSSZmr, X86::VMOVPDI2DIZmr }, 7578 { X86::VMOVSDZrm, X86::VMOVSDZrm, X86::VMOVQI2PQIZrm }, 7579 { X86::VMOVSDZrm_alt, X86::VMOVSDZrm_alt, X86::VMOVQI2PQIZrm }, 7580 { X86::VMOVSSZrm, X86::VMOVSSZrm, X86::VMOVDI2PDIZrm }, 7581 { X86::VMOVSSZrm_alt, X86::VMOVSSZrm_alt, X86::VMOVDI2PDIZrm }, 7582 { X86::VBROADCASTSSZ128rr,X86::VBROADCASTSSZ128rr,X86::VPBROADCASTDZ128rr }, 7583 { X86::VBROADCASTSSZ128rm,X86::VBROADCASTSSZ128rm,X86::VPBROADCASTDZ128rm }, 7584 { X86::VBROADCASTSSZ256rr,X86::VBROADCASTSSZ256rr,X86::VPBROADCASTDZ256rr }, 7585 { X86::VBROADCASTSSZ256rm,X86::VBROADCASTSSZ256rm,X86::VPBROADCASTDZ256rm }, 7586 { X86::VBROADCASTSSZrr, X86::VBROADCASTSSZrr, X86::VPBROADCASTDZrr }, 7587 { X86::VBROADCASTSSZrm, X86::VBROADCASTSSZrm, X86::VPBROADCASTDZrm }, 7588 { X86::VMOVDDUPZ128rr, X86::VMOVDDUPZ128rr, X86::VPBROADCASTQZ128rr }, 7589 { X86::VMOVDDUPZ128rm, X86::VMOVDDUPZ128rm, X86::VPBROADCASTQZ128rm }, 7590 { X86::VBROADCASTSDZ256rr,X86::VBROADCASTSDZ256rr,X86::VPBROADCASTQZ256rr }, 7591 { X86::VBROADCASTSDZ256rm,X86::VBROADCASTSDZ256rm,X86::VPBROADCASTQZ256rm }, 7592 { X86::VBROADCASTSDZrr, X86::VBROADCASTSDZrr, X86::VPBROADCASTQZrr }, 7593 { X86::VBROADCASTSDZrm, X86::VBROADCASTSDZrm, X86::VPBROADCASTQZrm }, 7594 { X86::VINSERTF32x4Zrr, X86::VINSERTF32x4Zrr, X86::VINSERTI32x4Zrr }, 7595 { X86::VINSERTF32x4Zrm, X86::VINSERTF32x4Zrm, X86::VINSERTI32x4Zrm }, 7596 { X86::VINSERTF32x8Zrr, X86::VINSERTF32x8Zrr, X86::VINSERTI32x8Zrr }, 7597 { X86::VINSERTF32x8Zrm, X86::VINSERTF32x8Zrm, X86::VINSERTI32x8Zrm }, 7598 { X86::VINSERTF64x2Zrr, X86::VINSERTF64x2Zrr, X86::VINSERTI64x2Zrr }, 7599 { X86::VINSERTF64x2Zrm, X86::VINSERTF64x2Zrm, X86::VINSERTI64x2Zrm }, 7600 { X86::VINSERTF64x4Zrr, X86::VINSERTF64x4Zrr, X86::VINSERTI64x4Zrr }, 7601 { X86::VINSERTF64x4Zrm, X86::VINSERTF64x4Zrm, X86::VINSERTI64x4Zrm }, 7602 { X86::VINSERTF32x4Z256rr,X86::VINSERTF32x4Z256rr,X86::VINSERTI32x4Z256rr }, 7603 { X86::VINSERTF32x4Z256rm,X86::VINSERTF32x4Z256rm,X86::VINSERTI32x4Z256rm }, 7604 { X86::VINSERTF64x2Z256rr,X86::VINSERTF64x2Z256rr,X86::VINSERTI64x2Z256rr }, 7605 { X86::VINSERTF64x2Z256rm,X86::VINSERTF64x2Z256rm,X86::VINSERTI64x2Z256rm }, 7606 { X86::VEXTRACTF32x4Zrr, X86::VEXTRACTF32x4Zrr, X86::VEXTRACTI32x4Zrr }, 7607 { X86::VEXTRACTF32x4Zmr, X86::VEXTRACTF32x4Zmr, X86::VEXTRACTI32x4Zmr }, 7608 { X86::VEXTRACTF32x8Zrr, X86::VEXTRACTF32x8Zrr, X86::VEXTRACTI32x8Zrr }, 7609 { X86::VEXTRACTF32x8Zmr, X86::VEXTRACTF32x8Zmr, X86::VEXTRACTI32x8Zmr }, 7610 { X86::VEXTRACTF64x2Zrr, X86::VEXTRACTF64x2Zrr, X86::VEXTRACTI64x2Zrr }, 7611 { X86::VEXTRACTF64x2Zmr, X86::VEXTRACTF64x2Zmr, X86::VEXTRACTI64x2Zmr }, 7612 { X86::VEXTRACTF64x4Zrr, X86::VEXTRACTF64x4Zrr, X86::VEXTRACTI64x4Zrr }, 7613 { X86::VEXTRACTF64x4Zmr, X86::VEXTRACTF64x4Zmr, X86::VEXTRACTI64x4Zmr }, 7614 { X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTI32x4Z256rr }, 7615 { X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTI32x4Z256mr }, 7616 { X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTI64x2Z256rr }, 7617 { X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTI64x2Z256mr }, 7618 { X86::VPERMILPSmi, X86::VPERMILPSmi, X86::VPSHUFDmi }, 7619 { X86::VPERMILPSri, X86::VPERMILPSri, X86::VPSHUFDri }, 7620 { X86::VPERMILPSZ128mi, X86::VPERMILPSZ128mi, X86::VPSHUFDZ128mi }, 7621 { X86::VPERMILPSZ128ri, X86::VPERMILPSZ128ri, X86::VPSHUFDZ128ri }, 7622 { X86::VPERMILPSZ256mi, X86::VPERMILPSZ256mi, X86::VPSHUFDZ256mi }, 7623 { X86::VPERMILPSZ256ri, X86::VPERMILPSZ256ri, X86::VPSHUFDZ256ri }, 7624 { X86::VPERMILPSZmi, X86::VPERMILPSZmi, X86::VPSHUFDZmi }, 7625 { X86::VPERMILPSZri, X86::VPERMILPSZri, X86::VPSHUFDZri }, 7626 { X86::VPERMPSZ256rm, X86::VPERMPSZ256rm, X86::VPERMDZ256rm }, 7627 { X86::VPERMPSZ256rr, X86::VPERMPSZ256rr, X86::VPERMDZ256rr }, 7628 { X86::VPERMPDZ256mi, X86::VPERMPDZ256mi, X86::VPERMQZ256mi }, 7629 { X86::VPERMPDZ256ri, X86::VPERMPDZ256ri, X86::VPERMQZ256ri }, 7630 { X86::VPERMPDZ256rm, X86::VPERMPDZ256rm, X86::VPERMQZ256rm }, 7631 { X86::VPERMPDZ256rr, X86::VPERMPDZ256rr, X86::VPERMQZ256rr }, 7632 { X86::VPERMPSZrm, X86::VPERMPSZrm, X86::VPERMDZrm }, 7633 { X86::VPERMPSZrr, X86::VPERMPSZrr, X86::VPERMDZrr }, 7634 { X86::VPERMPDZmi, X86::VPERMPDZmi, X86::VPERMQZmi }, 7635 { X86::VPERMPDZri, X86::VPERMPDZri, X86::VPERMQZri }, 7636 { X86::VPERMPDZrm, X86::VPERMPDZrm, X86::VPERMQZrm }, 7637 { X86::VPERMPDZrr, X86::VPERMPDZrr, X86::VPERMQZrr }, 7638 { X86::VUNPCKLPDZ256rm, X86::VUNPCKLPDZ256rm, X86::VPUNPCKLQDQZ256rm }, 7639 { X86::VUNPCKLPDZ256rr, X86::VUNPCKLPDZ256rr, X86::VPUNPCKLQDQZ256rr }, 7640 { X86::VUNPCKHPDZ256rm, X86::VUNPCKHPDZ256rm, X86::VPUNPCKHQDQZ256rm }, 7641 { X86::VUNPCKHPDZ256rr, X86::VUNPCKHPDZ256rr, X86::VPUNPCKHQDQZ256rr }, 7642 { X86::VUNPCKLPSZ256rm, X86::VUNPCKLPSZ256rm, X86::VPUNPCKLDQZ256rm }, 7643 { X86::VUNPCKLPSZ256rr, X86::VUNPCKLPSZ256rr, X86::VPUNPCKLDQZ256rr }, 7644 { X86::VUNPCKHPSZ256rm, X86::VUNPCKHPSZ256rm, X86::VPUNPCKHDQZ256rm }, 7645 { X86::VUNPCKHPSZ256rr, X86::VUNPCKHPSZ256rr, X86::VPUNPCKHDQZ256rr }, 7646 { X86::VUNPCKLPDZ128rm, X86::VUNPCKLPDZ128rm, X86::VPUNPCKLQDQZ128rm }, 7647 { X86::VMOVLHPSZrr, X86::VUNPCKLPDZ128rr, X86::VPUNPCKLQDQZ128rr }, 7648 { X86::VUNPCKHPDZ128rm, X86::VUNPCKHPDZ128rm, X86::VPUNPCKHQDQZ128rm }, 7649 { X86::VUNPCKHPDZ128rr, X86::VUNPCKHPDZ128rr, X86::VPUNPCKHQDQZ128rr }, 7650 { X86::VUNPCKLPSZ128rm, X86::VUNPCKLPSZ128rm, X86::VPUNPCKLDQZ128rm }, 7651 { X86::VUNPCKLPSZ128rr, X86::VUNPCKLPSZ128rr, X86::VPUNPCKLDQZ128rr }, 7652 { X86::VUNPCKHPSZ128rm, X86::VUNPCKHPSZ128rm, X86::VPUNPCKHDQZ128rm }, 7653 { X86::VUNPCKHPSZ128rr, X86::VUNPCKHPSZ128rr, X86::VPUNPCKHDQZ128rr }, 7654 { X86::VUNPCKLPDZrm, X86::VUNPCKLPDZrm, X86::VPUNPCKLQDQZrm }, 7655 { X86::VUNPCKLPDZrr, X86::VUNPCKLPDZrr, X86::VPUNPCKLQDQZrr }, 7656 { X86::VUNPCKHPDZrm, X86::VUNPCKHPDZrm, X86::VPUNPCKHQDQZrm }, 7657 { X86::VUNPCKHPDZrr, X86::VUNPCKHPDZrr, X86::VPUNPCKHQDQZrr }, 7658 { X86::VUNPCKLPSZrm, X86::VUNPCKLPSZrm, X86::VPUNPCKLDQZrm }, 7659 { X86::VUNPCKLPSZrr, X86::VUNPCKLPSZrr, X86::VPUNPCKLDQZrr }, 7660 { X86::VUNPCKHPSZrm, X86::VUNPCKHPSZrm, X86::VPUNPCKHDQZrm }, 7661 { X86::VUNPCKHPSZrr, X86::VUNPCKHPSZrr, X86::VPUNPCKHDQZrr }, 7662 { X86::VEXTRACTPSZmr, X86::VEXTRACTPSZmr, X86::VPEXTRDZmr }, 7663 { X86::VEXTRACTPSZrr, X86::VEXTRACTPSZrr, X86::VPEXTRDZrr }, 7664 }; 7665 7666 static const uint16_t ReplaceableInstrsAVX2[][3] = { 7667 //PackedSingle PackedDouble PackedInt 7668 { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNYrm }, 7669 { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNYrr }, 7670 { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDYrm }, 7671 { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDYrr }, 7672 { X86::VORPSYrm, X86::VORPDYrm, X86::VPORYrm }, 7673 { X86::VORPSYrr, X86::VORPDYrr, X86::VPORYrr }, 7674 { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORYrm }, 7675 { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORYrr }, 7676 { X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm }, 7677 { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }, 7678 { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm}, 7679 { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr}, 7680 { X86::VMOVDDUPrm, X86::VMOVDDUPrm, X86::VPBROADCASTQrm}, 7681 { X86::VMOVDDUPrr, X86::VMOVDDUPrr, X86::VPBROADCASTQrr}, 7682 { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr}, 7683 { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm}, 7684 { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr}, 7685 { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm}, 7686 { X86::VBROADCASTF128, X86::VBROADCASTF128, X86::VBROADCASTI128 }, 7687 { X86::VBLENDPSYrri, X86::VBLENDPSYrri, X86::VPBLENDDYrri }, 7688 { X86::VBLENDPSYrmi, X86::VBLENDPSYrmi, X86::VPBLENDDYrmi }, 7689 { X86::VPERMILPSYmi, X86::VPERMILPSYmi, X86::VPSHUFDYmi }, 7690 { X86::VPERMILPSYri, X86::VPERMILPSYri, X86::VPSHUFDYri }, 7691 { X86::VUNPCKLPDYrm, X86::VUNPCKLPDYrm, X86::VPUNPCKLQDQYrm }, 7692 { X86::VUNPCKLPDYrr, X86::VUNPCKLPDYrr, X86::VPUNPCKLQDQYrr }, 7693 { X86::VUNPCKHPDYrm, X86::VUNPCKHPDYrm, X86::VPUNPCKHQDQYrm }, 7694 { X86::VUNPCKHPDYrr, X86::VUNPCKHPDYrr, X86::VPUNPCKHQDQYrr }, 7695 { X86::VUNPCKLPSYrm, X86::VUNPCKLPSYrm, X86::VPUNPCKLDQYrm }, 7696 { X86::VUNPCKLPSYrr, X86::VUNPCKLPSYrr, X86::VPUNPCKLDQYrr }, 7697 { X86::VUNPCKHPSYrm, X86::VUNPCKHPSYrm, X86::VPUNPCKHDQYrm }, 7698 { X86::VUNPCKHPSYrr, X86::VUNPCKHPSYrr, X86::VPUNPCKHDQYrr }, 7699 }; 7700 7701 static const uint16_t ReplaceableInstrsFP[][3] = { 7702 //PackedSingle PackedDouble 7703 { X86::MOVLPSrm, X86::MOVLPDrm, X86::INSTRUCTION_LIST_END }, 7704 { X86::MOVHPSrm, X86::MOVHPDrm, X86::INSTRUCTION_LIST_END }, 7705 { X86::MOVHPSmr, X86::MOVHPDmr, X86::INSTRUCTION_LIST_END }, 7706 { X86::VMOVLPSrm, X86::VMOVLPDrm, X86::INSTRUCTION_LIST_END }, 7707 { X86::VMOVHPSrm, X86::VMOVHPDrm, X86::INSTRUCTION_LIST_END }, 7708 { X86::VMOVHPSmr, X86::VMOVHPDmr, X86::INSTRUCTION_LIST_END }, 7709 { X86::VMOVLPSZ128rm, X86::VMOVLPDZ128rm, X86::INSTRUCTION_LIST_END }, 7710 { X86::VMOVHPSZ128rm, X86::VMOVHPDZ128rm, X86::INSTRUCTION_LIST_END }, 7711 { X86::VMOVHPSZ128mr, X86::VMOVHPDZ128mr, X86::INSTRUCTION_LIST_END }, 7712 }; 7713 7714 static const uint16_t ReplaceableInstrsAVX2InsertExtract[][3] = { 7715 //PackedSingle PackedDouble PackedInt 7716 { X86::VEXTRACTF128mr, X86::VEXTRACTF128mr, X86::VEXTRACTI128mr }, 7717 { X86::VEXTRACTF128rr, X86::VEXTRACTF128rr, X86::VEXTRACTI128rr }, 7718 { X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm }, 7719 { X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr }, 7720 }; 7721 7722 static const uint16_t ReplaceableInstrsAVX512[][4] = { 7723 // Two integer columns for 64-bit and 32-bit elements. 7724 //PackedSingle PackedDouble PackedInt PackedInt 7725 { X86::VMOVAPSZ128mr, X86::VMOVAPDZ128mr, X86::VMOVDQA64Z128mr, X86::VMOVDQA32Z128mr }, 7726 { X86::VMOVAPSZ128rm, X86::VMOVAPDZ128rm, X86::VMOVDQA64Z128rm, X86::VMOVDQA32Z128rm }, 7727 { X86::VMOVAPSZ128rr, X86::VMOVAPDZ128rr, X86::VMOVDQA64Z128rr, X86::VMOVDQA32Z128rr }, 7728 { X86::VMOVUPSZ128mr, X86::VMOVUPDZ128mr, X86::VMOVDQU64Z128mr, X86::VMOVDQU32Z128mr }, 7729 { X86::VMOVUPSZ128rm, X86::VMOVUPDZ128rm, X86::VMOVDQU64Z128rm, X86::VMOVDQU32Z128rm }, 7730 { X86::VMOVAPSZ256mr, X86::VMOVAPDZ256mr, X86::VMOVDQA64Z256mr, X86::VMOVDQA32Z256mr }, 7731 { X86::VMOVAPSZ256rm, X86::VMOVAPDZ256rm, X86::VMOVDQA64Z256rm, X86::VMOVDQA32Z256rm }, 7732 { X86::VMOVAPSZ256rr, X86::VMOVAPDZ256rr, X86::VMOVDQA64Z256rr, X86::VMOVDQA32Z256rr }, 7733 { X86::VMOVUPSZ256mr, X86::VMOVUPDZ256mr, X86::VMOVDQU64Z256mr, X86::VMOVDQU32Z256mr }, 7734 { X86::VMOVUPSZ256rm, X86::VMOVUPDZ256rm, X86::VMOVDQU64Z256rm, X86::VMOVDQU32Z256rm }, 7735 { X86::VMOVAPSZmr, X86::VMOVAPDZmr, X86::VMOVDQA64Zmr, X86::VMOVDQA32Zmr }, 7736 { X86::VMOVAPSZrm, X86::VMOVAPDZrm, X86::VMOVDQA64Zrm, X86::VMOVDQA32Zrm }, 7737 { X86::VMOVAPSZrr, X86::VMOVAPDZrr, X86::VMOVDQA64Zrr, X86::VMOVDQA32Zrr }, 7738 { X86::VMOVUPSZmr, X86::VMOVUPDZmr, X86::VMOVDQU64Zmr, X86::VMOVDQU32Zmr }, 7739 { X86::VMOVUPSZrm, X86::VMOVUPDZrm, X86::VMOVDQU64Zrm, X86::VMOVDQU32Zrm }, 7740 }; 7741 7742 static const uint16_t ReplaceableInstrsAVX512DQ[][4] = { 7743 // Two integer columns for 64-bit and 32-bit elements. 7744 //PackedSingle PackedDouble PackedInt PackedInt 7745 { X86::VANDNPSZ128rm, X86::VANDNPDZ128rm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, 7746 { X86::VANDNPSZ128rr, X86::VANDNPDZ128rr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, 7747 { X86::VANDPSZ128rm, X86::VANDPDZ128rm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, 7748 { X86::VANDPSZ128rr, X86::VANDPDZ128rr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, 7749 { X86::VORPSZ128rm, X86::VORPDZ128rm, X86::VPORQZ128rm, X86::VPORDZ128rm }, 7750 { X86::VORPSZ128rr, X86::VORPDZ128rr, X86::VPORQZ128rr, X86::VPORDZ128rr }, 7751 { X86::VXORPSZ128rm, X86::VXORPDZ128rm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, 7752 { X86::VXORPSZ128rr, X86::VXORPDZ128rr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, 7753 { X86::VANDNPSZ256rm, X86::VANDNPDZ256rm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, 7754 { X86::VANDNPSZ256rr, X86::VANDNPDZ256rr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, 7755 { X86::VANDPSZ256rm, X86::VANDPDZ256rm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, 7756 { X86::VANDPSZ256rr, X86::VANDPDZ256rr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, 7757 { X86::VORPSZ256rm, X86::VORPDZ256rm, X86::VPORQZ256rm, X86::VPORDZ256rm }, 7758 { X86::VORPSZ256rr, X86::VORPDZ256rr, X86::VPORQZ256rr, X86::VPORDZ256rr }, 7759 { X86::VXORPSZ256rm, X86::VXORPDZ256rm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, 7760 { X86::VXORPSZ256rr, X86::VXORPDZ256rr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, 7761 { X86::VANDNPSZrm, X86::VANDNPDZrm, X86::VPANDNQZrm, X86::VPANDNDZrm }, 7762 { X86::VANDNPSZrr, X86::VANDNPDZrr, X86::VPANDNQZrr, X86::VPANDNDZrr }, 7763 { X86::VANDPSZrm, X86::VANDPDZrm, X86::VPANDQZrm, X86::VPANDDZrm }, 7764 { X86::VANDPSZrr, X86::VANDPDZrr, X86::VPANDQZrr, X86::VPANDDZrr }, 7765 { X86::VORPSZrm, X86::VORPDZrm, X86::VPORQZrm, X86::VPORDZrm }, 7766 { X86::VORPSZrr, X86::VORPDZrr, X86::VPORQZrr, X86::VPORDZrr }, 7767 { X86::VXORPSZrm, X86::VXORPDZrm, X86::VPXORQZrm, X86::VPXORDZrm }, 7768 { X86::VXORPSZrr, X86::VXORPDZrr, X86::VPXORQZrr, X86::VPXORDZrr }, 7769 }; 7770 7771 static const uint16_t ReplaceableInstrsAVX512DQMasked[][4] = { 7772 // Two integer columns for 64-bit and 32-bit elements. 7773 //PackedSingle PackedDouble 7774 //PackedInt PackedInt 7775 { X86::VANDNPSZ128rmk, X86::VANDNPDZ128rmk, 7776 X86::VPANDNQZ128rmk, X86::VPANDNDZ128rmk }, 7777 { X86::VANDNPSZ128rmkz, X86::VANDNPDZ128rmkz, 7778 X86::VPANDNQZ128rmkz, X86::VPANDNDZ128rmkz }, 7779 { X86::VANDNPSZ128rrk, X86::VANDNPDZ128rrk, 7780 X86::VPANDNQZ128rrk, X86::VPANDNDZ128rrk }, 7781 { X86::VANDNPSZ128rrkz, X86::VANDNPDZ128rrkz, 7782 X86::VPANDNQZ128rrkz, X86::VPANDNDZ128rrkz }, 7783 { X86::VANDPSZ128rmk, X86::VANDPDZ128rmk, 7784 X86::VPANDQZ128rmk, X86::VPANDDZ128rmk }, 7785 { X86::VANDPSZ128rmkz, X86::VANDPDZ128rmkz, 7786 X86::VPANDQZ128rmkz, X86::VPANDDZ128rmkz }, 7787 { X86::VANDPSZ128rrk, X86::VANDPDZ128rrk, 7788 X86::VPANDQZ128rrk, X86::VPANDDZ128rrk }, 7789 { X86::VANDPSZ128rrkz, X86::VANDPDZ128rrkz, 7790 X86::VPANDQZ128rrkz, X86::VPANDDZ128rrkz }, 7791 { X86::VORPSZ128rmk, X86::VORPDZ128rmk, 7792 X86::VPORQZ128rmk, X86::VPORDZ128rmk }, 7793 { X86::VORPSZ128rmkz, X86::VORPDZ128rmkz, 7794 X86::VPORQZ128rmkz, X86::VPORDZ128rmkz }, 7795 { X86::VORPSZ128rrk, X86::VORPDZ128rrk, 7796 X86::VPORQZ128rrk, X86::VPORDZ128rrk }, 7797 { X86::VORPSZ128rrkz, X86::VORPDZ128rrkz, 7798 X86::VPORQZ128rrkz, X86::VPORDZ128rrkz }, 7799 { X86::VXORPSZ128rmk, X86::VXORPDZ128rmk, 7800 X86::VPXORQZ128rmk, X86::VPXORDZ128rmk }, 7801 { X86::VXORPSZ128rmkz, X86::VXORPDZ128rmkz, 7802 X86::VPXORQZ128rmkz, X86::VPXORDZ128rmkz }, 7803 { X86::VXORPSZ128rrk, X86::VXORPDZ128rrk, 7804 X86::VPXORQZ128rrk, X86::VPXORDZ128rrk }, 7805 { X86::VXORPSZ128rrkz, X86::VXORPDZ128rrkz, 7806 X86::VPXORQZ128rrkz, X86::VPXORDZ128rrkz }, 7807 { X86::VANDNPSZ256rmk, X86::VANDNPDZ256rmk, 7808 X86::VPANDNQZ256rmk, X86::VPANDNDZ256rmk }, 7809 { X86::VANDNPSZ256rmkz, X86::VANDNPDZ256rmkz, 7810 X86::VPANDNQZ256rmkz, X86::VPANDNDZ256rmkz }, 7811 { X86::VANDNPSZ256rrk, X86::VANDNPDZ256rrk, 7812 X86::VPANDNQZ256rrk, X86::VPANDNDZ256rrk }, 7813 { X86::VANDNPSZ256rrkz, X86::VANDNPDZ256rrkz, 7814 X86::VPANDNQZ256rrkz, X86::VPANDNDZ256rrkz }, 7815 { X86::VANDPSZ256rmk, X86::VANDPDZ256rmk, 7816 X86::VPANDQZ256rmk, X86::VPANDDZ256rmk }, 7817 { X86::VANDPSZ256rmkz, X86::VANDPDZ256rmkz, 7818 X86::VPANDQZ256rmkz, X86::VPANDDZ256rmkz }, 7819 { X86::VANDPSZ256rrk, X86::VANDPDZ256rrk, 7820 X86::VPANDQZ256rrk, X86::VPANDDZ256rrk }, 7821 { X86::VANDPSZ256rrkz, X86::VANDPDZ256rrkz, 7822 X86::VPANDQZ256rrkz, X86::VPANDDZ256rrkz }, 7823 { X86::VORPSZ256rmk, X86::VORPDZ256rmk, 7824 X86::VPORQZ256rmk, X86::VPORDZ256rmk }, 7825 { X86::VORPSZ256rmkz, X86::VORPDZ256rmkz, 7826 X86::VPORQZ256rmkz, X86::VPORDZ256rmkz }, 7827 { X86::VORPSZ256rrk, X86::VORPDZ256rrk, 7828 X86::VPORQZ256rrk, X86::VPORDZ256rrk }, 7829 { X86::VORPSZ256rrkz, X86::VORPDZ256rrkz, 7830 X86::VPORQZ256rrkz, X86::VPORDZ256rrkz }, 7831 { X86::VXORPSZ256rmk, X86::VXORPDZ256rmk, 7832 X86::VPXORQZ256rmk, X86::VPXORDZ256rmk }, 7833 { X86::VXORPSZ256rmkz, X86::VXORPDZ256rmkz, 7834 X86::VPXORQZ256rmkz, X86::VPXORDZ256rmkz }, 7835 { X86::VXORPSZ256rrk, X86::VXORPDZ256rrk, 7836 X86::VPXORQZ256rrk, X86::VPXORDZ256rrk }, 7837 { X86::VXORPSZ256rrkz, X86::VXORPDZ256rrkz, 7838 X86::VPXORQZ256rrkz, X86::VPXORDZ256rrkz }, 7839 { X86::VANDNPSZrmk, X86::VANDNPDZrmk, 7840 X86::VPANDNQZrmk, X86::VPANDNDZrmk }, 7841 { X86::VANDNPSZrmkz, X86::VANDNPDZrmkz, 7842 X86::VPANDNQZrmkz, X86::VPANDNDZrmkz }, 7843 { X86::VANDNPSZrrk, X86::VANDNPDZrrk, 7844 X86::VPANDNQZrrk, X86::VPANDNDZrrk }, 7845 { X86::VANDNPSZrrkz, X86::VANDNPDZrrkz, 7846 X86::VPANDNQZrrkz, X86::VPANDNDZrrkz }, 7847 { X86::VANDPSZrmk, X86::VANDPDZrmk, 7848 X86::VPANDQZrmk, X86::VPANDDZrmk }, 7849 { X86::VANDPSZrmkz, X86::VANDPDZrmkz, 7850 X86::VPANDQZrmkz, X86::VPANDDZrmkz }, 7851 { X86::VANDPSZrrk, X86::VANDPDZrrk, 7852 X86::VPANDQZrrk, X86::VPANDDZrrk }, 7853 { X86::VANDPSZrrkz, X86::VANDPDZrrkz, 7854 X86::VPANDQZrrkz, X86::VPANDDZrrkz }, 7855 { X86::VORPSZrmk, X86::VORPDZrmk, 7856 X86::VPORQZrmk, X86::VPORDZrmk }, 7857 { X86::VORPSZrmkz, X86::VORPDZrmkz, 7858 X86::VPORQZrmkz, X86::VPORDZrmkz }, 7859 { X86::VORPSZrrk, X86::VORPDZrrk, 7860 X86::VPORQZrrk, X86::VPORDZrrk }, 7861 { X86::VORPSZrrkz, X86::VORPDZrrkz, 7862 X86::VPORQZrrkz, X86::VPORDZrrkz }, 7863 { X86::VXORPSZrmk, X86::VXORPDZrmk, 7864 X86::VPXORQZrmk, X86::VPXORDZrmk }, 7865 { X86::VXORPSZrmkz, X86::VXORPDZrmkz, 7866 X86::VPXORQZrmkz, X86::VPXORDZrmkz }, 7867 { X86::VXORPSZrrk, X86::VXORPDZrrk, 7868 X86::VPXORQZrrk, X86::VPXORDZrrk }, 7869 { X86::VXORPSZrrkz, X86::VXORPDZrrkz, 7870 X86::VPXORQZrrkz, X86::VPXORDZrrkz }, 7871 // Broadcast loads can be handled the same as masked operations to avoid 7872 // changing element size. 7873 { X86::VANDNPSZ128rmb, X86::VANDNPDZ128rmb, 7874 X86::VPANDNQZ128rmb, X86::VPANDNDZ128rmb }, 7875 { X86::VANDPSZ128rmb, X86::VANDPDZ128rmb, 7876 X86::VPANDQZ128rmb, X86::VPANDDZ128rmb }, 7877 { X86::VORPSZ128rmb, X86::VORPDZ128rmb, 7878 X86::VPORQZ128rmb, X86::VPORDZ128rmb }, 7879 { X86::VXORPSZ128rmb, X86::VXORPDZ128rmb, 7880 X86::VPXORQZ128rmb, X86::VPXORDZ128rmb }, 7881 { X86::VANDNPSZ256rmb, X86::VANDNPDZ256rmb, 7882 X86::VPANDNQZ256rmb, X86::VPANDNDZ256rmb }, 7883 { X86::VANDPSZ256rmb, X86::VANDPDZ256rmb, 7884 X86::VPANDQZ256rmb, X86::VPANDDZ256rmb }, 7885 { X86::VORPSZ256rmb, X86::VORPDZ256rmb, 7886 X86::VPORQZ256rmb, X86::VPORDZ256rmb }, 7887 { X86::VXORPSZ256rmb, X86::VXORPDZ256rmb, 7888 X86::VPXORQZ256rmb, X86::VPXORDZ256rmb }, 7889 { X86::VANDNPSZrmb, X86::VANDNPDZrmb, 7890 X86::VPANDNQZrmb, X86::VPANDNDZrmb }, 7891 { X86::VANDPSZrmb, X86::VANDPDZrmb, 7892 X86::VPANDQZrmb, X86::VPANDDZrmb }, 7893 { X86::VANDPSZrmb, X86::VANDPDZrmb, 7894 X86::VPANDQZrmb, X86::VPANDDZrmb }, 7895 { X86::VORPSZrmb, X86::VORPDZrmb, 7896 X86::VPORQZrmb, X86::VPORDZrmb }, 7897 { X86::VXORPSZrmb, X86::VXORPDZrmb, 7898 X86::VPXORQZrmb, X86::VPXORDZrmb }, 7899 { X86::VANDNPSZ128rmbk, X86::VANDNPDZ128rmbk, 7900 X86::VPANDNQZ128rmbk, X86::VPANDNDZ128rmbk }, 7901 { X86::VANDPSZ128rmbk, X86::VANDPDZ128rmbk, 7902 X86::VPANDQZ128rmbk, X86::VPANDDZ128rmbk }, 7903 { X86::VORPSZ128rmbk, X86::VORPDZ128rmbk, 7904 X86::VPORQZ128rmbk, X86::VPORDZ128rmbk }, 7905 { X86::VXORPSZ128rmbk, X86::VXORPDZ128rmbk, 7906 X86::VPXORQZ128rmbk, X86::VPXORDZ128rmbk }, 7907 { X86::VANDNPSZ256rmbk, X86::VANDNPDZ256rmbk, 7908 X86::VPANDNQZ256rmbk, X86::VPANDNDZ256rmbk }, 7909 { X86::VANDPSZ256rmbk, X86::VANDPDZ256rmbk, 7910 X86::VPANDQZ256rmbk, X86::VPANDDZ256rmbk }, 7911 { X86::VORPSZ256rmbk, X86::VORPDZ256rmbk, 7912 X86::VPORQZ256rmbk, X86::VPORDZ256rmbk }, 7913 { X86::VXORPSZ256rmbk, X86::VXORPDZ256rmbk, 7914 X86::VPXORQZ256rmbk, X86::VPXORDZ256rmbk }, 7915 { X86::VANDNPSZrmbk, X86::VANDNPDZrmbk, 7916 X86::VPANDNQZrmbk, X86::VPANDNDZrmbk }, 7917 { X86::VANDPSZrmbk, X86::VANDPDZrmbk, 7918 X86::VPANDQZrmbk, X86::VPANDDZrmbk }, 7919 { X86::VANDPSZrmbk, X86::VANDPDZrmbk, 7920 X86::VPANDQZrmbk, X86::VPANDDZrmbk }, 7921 { X86::VORPSZrmbk, X86::VORPDZrmbk, 7922 X86::VPORQZrmbk, X86::VPORDZrmbk }, 7923 { X86::VXORPSZrmbk, X86::VXORPDZrmbk, 7924 X86::VPXORQZrmbk, X86::VPXORDZrmbk }, 7925 { X86::VANDNPSZ128rmbkz,X86::VANDNPDZ128rmbkz, 7926 X86::VPANDNQZ128rmbkz,X86::VPANDNDZ128rmbkz}, 7927 { X86::VANDPSZ128rmbkz, X86::VANDPDZ128rmbkz, 7928 X86::VPANDQZ128rmbkz, X86::VPANDDZ128rmbkz }, 7929 { X86::VORPSZ128rmbkz, X86::VORPDZ128rmbkz, 7930 X86::VPORQZ128rmbkz, X86::VPORDZ128rmbkz }, 7931 { X86::VXORPSZ128rmbkz, X86::VXORPDZ128rmbkz, 7932 X86::VPXORQZ128rmbkz, X86::VPXORDZ128rmbkz }, 7933 { X86::VANDNPSZ256rmbkz,X86::VANDNPDZ256rmbkz, 7934 X86::VPANDNQZ256rmbkz,X86::VPANDNDZ256rmbkz}, 7935 { X86::VANDPSZ256rmbkz, X86::VANDPDZ256rmbkz, 7936 X86::VPANDQZ256rmbkz, X86::VPANDDZ256rmbkz }, 7937 { X86::VORPSZ256rmbkz, X86::VORPDZ256rmbkz, 7938 X86::VPORQZ256rmbkz, X86::VPORDZ256rmbkz }, 7939 { X86::VXORPSZ256rmbkz, X86::VXORPDZ256rmbkz, 7940 X86::VPXORQZ256rmbkz, X86::VPXORDZ256rmbkz }, 7941 { X86::VANDNPSZrmbkz, X86::VANDNPDZrmbkz, 7942 X86::VPANDNQZrmbkz, X86::VPANDNDZrmbkz }, 7943 { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, 7944 X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, 7945 { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, 7946 X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, 7947 { X86::VORPSZrmbkz, X86::VORPDZrmbkz, 7948 X86::VPORQZrmbkz, X86::VPORDZrmbkz }, 7949 { X86::VXORPSZrmbkz, X86::VXORPDZrmbkz, 7950 X86::VPXORQZrmbkz, X86::VPXORDZrmbkz }, 7951 }; 7952 7953 // NOTE: These should only be used by the custom domain methods. 7954 static const uint16_t ReplaceableBlendInstrs[][3] = { 7955 //PackedSingle PackedDouble PackedInt 7956 { X86::BLENDPSrmi, X86::BLENDPDrmi, X86::PBLENDWrmi }, 7957 { X86::BLENDPSrri, X86::BLENDPDrri, X86::PBLENDWrri }, 7958 { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDWrmi }, 7959 { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDWrri }, 7960 { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDWYrmi }, 7961 { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDWYrri }, 7962 }; 7963 static const uint16_t ReplaceableBlendAVX2Instrs[][3] = { 7964 //PackedSingle PackedDouble PackedInt 7965 { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDDrmi }, 7966 { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDDrri }, 7967 { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDDYrmi }, 7968 { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDDYrri }, 7969 }; 7970 7971 // Special table for changing EVEX logic instructions to VEX. 7972 // TODO: Should we run EVEX->VEX earlier? 7973 static const uint16_t ReplaceableCustomAVX512LogicInstrs[][4] = { 7974 // Two integer columns for 64-bit and 32-bit elements. 7975 //PackedSingle PackedDouble PackedInt PackedInt 7976 { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, 7977 { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, 7978 { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, 7979 { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, 7980 { X86::VORPSrm, X86::VORPDrm, X86::VPORQZ128rm, X86::VPORDZ128rm }, 7981 { X86::VORPSrr, X86::VORPDrr, X86::VPORQZ128rr, X86::VPORDZ128rr }, 7982 { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, 7983 { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, 7984 { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, 7985 { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, 7986 { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, 7987 { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, 7988 { X86::VORPSYrm, X86::VORPDYrm, X86::VPORQZ256rm, X86::VPORDZ256rm }, 7989 { X86::VORPSYrr, X86::VORPDYrr, X86::VPORQZ256rr, X86::VPORDZ256rr }, 7990 { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, 7991 { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, 7992 }; 7993 7994 // FIXME: Some shuffle and unpack instructions have equivalents in different 7995 // domains, but they require a bit more work than just switching opcodes. 7996 7997 static const uint16_t *lookup(unsigned opcode, unsigned domain, 7998 ArrayRef<uint16_t[3]> Table) { 7999 for (const uint16_t (&Row)[3] : Table) 8000 if (Row[domain-1] == opcode) 8001 return Row; 8002 return nullptr; 8003 } 8004 8005 static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain, 8006 ArrayRef<uint16_t[4]> Table) { 8007 // If this is the integer domain make sure to check both integer columns. 8008 for (const uint16_t (&Row)[4] : Table) 8009 if (Row[domain-1] == opcode || (domain == 3 && Row[3] == opcode)) 8010 return Row; 8011 return nullptr; 8012 } 8013 8014 // Helper to attempt to widen/narrow blend masks. 8015 static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth, 8016 unsigned NewWidth, unsigned *pNewMask = nullptr) { 8017 assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && 8018 "Illegal blend mask scale"); 8019 unsigned NewMask = 0; 8020 8021 if ((OldWidth % NewWidth) == 0) { 8022 unsigned Scale = OldWidth / NewWidth; 8023 unsigned SubMask = (1u << Scale) - 1; 8024 for (unsigned i = 0; i != NewWidth; ++i) { 8025 unsigned Sub = (OldMask >> (i * Scale)) & SubMask; 8026 if (Sub == SubMask) 8027 NewMask |= (1u << i); 8028 else if (Sub != 0x0) 8029 return false; 8030 } 8031 } else { 8032 unsigned Scale = NewWidth / OldWidth; 8033 unsigned SubMask = (1u << Scale) - 1; 8034 for (unsigned i = 0; i != OldWidth; ++i) { 8035 if (OldMask & (1 << i)) { 8036 NewMask |= (SubMask << (i * Scale)); 8037 } 8038 } 8039 } 8040 8041 if (pNewMask) 8042 *pNewMask = NewMask; 8043 return true; 8044 } 8045 8046 uint16_t X86InstrInfo::getExecutionDomainCustom(const MachineInstr &MI) const { 8047 unsigned Opcode = MI.getOpcode(); 8048 unsigned NumOperands = MI.getDesc().getNumOperands(); 8049 8050 auto GetBlendDomains = [&](unsigned ImmWidth, bool Is256) { 8051 uint16_t validDomains = 0; 8052 if (MI.getOperand(NumOperands - 1).isImm()) { 8053 unsigned Imm = MI.getOperand(NumOperands - 1).getImm(); 8054 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4)) 8055 validDomains |= 0x2; // PackedSingle 8056 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2)) 8057 validDomains |= 0x4; // PackedDouble 8058 if (!Is256 || Subtarget.hasAVX2()) 8059 validDomains |= 0x8; // PackedInt 8060 } 8061 return validDomains; 8062 }; 8063 8064 switch (Opcode) { 8065 case X86::BLENDPDrmi: 8066 case X86::BLENDPDrri: 8067 case X86::VBLENDPDrmi: 8068 case X86::VBLENDPDrri: 8069 return GetBlendDomains(2, false); 8070 case X86::VBLENDPDYrmi: 8071 case X86::VBLENDPDYrri: 8072 return GetBlendDomains(4, true); 8073 case X86::BLENDPSrmi: 8074 case X86::BLENDPSrri: 8075 case X86::VBLENDPSrmi: 8076 case X86::VBLENDPSrri: 8077 case X86::VPBLENDDrmi: 8078 case X86::VPBLENDDrri: 8079 return GetBlendDomains(4, false); 8080 case X86::VBLENDPSYrmi: 8081 case X86::VBLENDPSYrri: 8082 case X86::VPBLENDDYrmi: 8083 case X86::VPBLENDDYrri: 8084 return GetBlendDomains(8, true); 8085 case X86::PBLENDWrmi: 8086 case X86::PBLENDWrri: 8087 case X86::VPBLENDWrmi: 8088 case X86::VPBLENDWrri: 8089 // Treat VPBLENDWY as a 128-bit vector as it repeats the lo/hi masks. 8090 case X86::VPBLENDWYrmi: 8091 case X86::VPBLENDWYrri: 8092 return GetBlendDomains(8, false); 8093 case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: 8094 case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: 8095 case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: 8096 case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: 8097 case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: 8098 case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: 8099 case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: 8100 case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: 8101 case X86::VPORDZ128rr: case X86::VPORDZ128rm: 8102 case X86::VPORDZ256rr: case X86::VPORDZ256rm: 8103 case X86::VPORQZ128rr: case X86::VPORQZ128rm: 8104 case X86::VPORQZ256rr: case X86::VPORQZ256rm: 8105 case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: 8106 case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: 8107 case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: 8108 case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: 8109 // If we don't have DQI see if we can still switch from an EVEX integer 8110 // instruction to a VEX floating point instruction. 8111 if (Subtarget.hasDQI()) 8112 return 0; 8113 8114 if (RI.getEncodingValue(MI.getOperand(0).getReg()) >= 16) 8115 return 0; 8116 if (RI.getEncodingValue(MI.getOperand(1).getReg()) >= 16) 8117 return 0; 8118 // Register forms will have 3 operands. Memory form will have more. 8119 if (NumOperands == 3 && 8120 RI.getEncodingValue(MI.getOperand(2).getReg()) >= 16) 8121 return 0; 8122 8123 // All domains are valid. 8124 return 0xe; 8125 case X86::MOVHLPSrr: 8126 // We can swap domains when both inputs are the same register. 8127 // FIXME: This doesn't catch all the cases we would like. If the input 8128 // register isn't KILLed by the instruction, the two address instruction 8129 // pass puts a COPY on one input. The other input uses the original 8130 // register. This prevents the same physical register from being used by 8131 // both inputs. 8132 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && 8133 MI.getOperand(0).getSubReg() == 0 && 8134 MI.getOperand(1).getSubReg() == 0 && 8135 MI.getOperand(2).getSubReg() == 0) 8136 return 0x6; 8137 return 0; 8138 case X86::SHUFPDrri: 8139 return 0x6; 8140 } 8141 return 0; 8142 } 8143 8144 bool X86InstrInfo::setExecutionDomainCustom(MachineInstr &MI, 8145 unsigned Domain) const { 8146 assert(Domain > 0 && Domain < 4 && "Invalid execution domain"); 8147 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 8148 assert(dom && "Not an SSE instruction"); 8149 8150 unsigned Opcode = MI.getOpcode(); 8151 unsigned NumOperands = MI.getDesc().getNumOperands(); 8152 8153 auto SetBlendDomain = [&](unsigned ImmWidth, bool Is256) { 8154 if (MI.getOperand(NumOperands - 1).isImm()) { 8155 unsigned Imm = MI.getOperand(NumOperands - 1).getImm() & 255; 8156 Imm = (ImmWidth == 16 ? ((Imm << 8) | Imm) : Imm); 8157 unsigned NewImm = Imm; 8158 8159 const uint16_t *table = lookup(Opcode, dom, ReplaceableBlendInstrs); 8160 if (!table) 8161 table = lookup(Opcode, dom, ReplaceableBlendAVX2Instrs); 8162 8163 if (Domain == 1) { // PackedSingle 8164 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); 8165 } else if (Domain == 2) { // PackedDouble 8166 AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2, &NewImm); 8167 } else if (Domain == 3) { // PackedInt 8168 if (Subtarget.hasAVX2()) { 8169 // If we are already VPBLENDW use that, else use VPBLENDD. 8170 if ((ImmWidth / (Is256 ? 2 : 1)) != 8) { 8171 table = lookup(Opcode, dom, ReplaceableBlendAVX2Instrs); 8172 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); 8173 } 8174 } else { 8175 assert(!Is256 && "128-bit vector expected"); 8176 AdjustBlendMask(Imm, ImmWidth, 8, &NewImm); 8177 } 8178 } 8179 8180 assert(table && table[Domain - 1] && "Unknown domain op"); 8181 MI.setDesc(get(table[Domain - 1])); 8182 MI.getOperand(NumOperands - 1).setImm(NewImm & 255); 8183 } 8184 return true; 8185 }; 8186 8187 switch (Opcode) { 8188 case X86::BLENDPDrmi: 8189 case X86::BLENDPDrri: 8190 case X86::VBLENDPDrmi: 8191 case X86::VBLENDPDrri: 8192 return SetBlendDomain(2, false); 8193 case X86::VBLENDPDYrmi: 8194 case X86::VBLENDPDYrri: 8195 return SetBlendDomain(4, true); 8196 case X86::BLENDPSrmi: 8197 case X86::BLENDPSrri: 8198 case X86::VBLENDPSrmi: 8199 case X86::VBLENDPSrri: 8200 case X86::VPBLENDDrmi: 8201 case X86::VPBLENDDrri: 8202 return SetBlendDomain(4, false); 8203 case X86::VBLENDPSYrmi: 8204 case X86::VBLENDPSYrri: 8205 case X86::VPBLENDDYrmi: 8206 case X86::VPBLENDDYrri: 8207 return SetBlendDomain(8, true); 8208 case X86::PBLENDWrmi: 8209 case X86::PBLENDWrri: 8210 case X86::VPBLENDWrmi: 8211 case X86::VPBLENDWrri: 8212 return SetBlendDomain(8, false); 8213 case X86::VPBLENDWYrmi: 8214 case X86::VPBLENDWYrri: 8215 return SetBlendDomain(16, true); 8216 case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: 8217 case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: 8218 case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: 8219 case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: 8220 case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: 8221 case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: 8222 case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: 8223 case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: 8224 case X86::VPORDZ128rr: case X86::VPORDZ128rm: 8225 case X86::VPORDZ256rr: case X86::VPORDZ256rm: 8226 case X86::VPORQZ128rr: case X86::VPORQZ128rm: 8227 case X86::VPORQZ256rr: case X86::VPORQZ256rm: 8228 case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: 8229 case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: 8230 case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: 8231 case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: { 8232 // Without DQI, convert EVEX instructions to VEX instructions. 8233 if (Subtarget.hasDQI()) 8234 return false; 8235 8236 const uint16_t *table = lookupAVX512(MI.getOpcode(), dom, 8237 ReplaceableCustomAVX512LogicInstrs); 8238 assert(table && "Instruction not found in table?"); 8239 // Don't change integer Q instructions to D instructions and 8240 // use D intructions if we started with a PS instruction. 8241 if (Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 8242 Domain = 4; 8243 MI.setDesc(get(table[Domain - 1])); 8244 return true; 8245 } 8246 case X86::UNPCKHPDrr: 8247 case X86::MOVHLPSrr: 8248 // We just need to commute the instruction which will switch the domains. 8249 if (Domain != dom && Domain != 3 && 8250 MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && 8251 MI.getOperand(0).getSubReg() == 0 && 8252 MI.getOperand(1).getSubReg() == 0 && 8253 MI.getOperand(2).getSubReg() == 0) { 8254 commuteInstruction(MI, false); 8255 return true; 8256 } 8257 // We must always return true for MOVHLPSrr. 8258 if (Opcode == X86::MOVHLPSrr) 8259 return true; 8260 break; 8261 case X86::SHUFPDrri: { 8262 if (Domain == 1) { 8263 unsigned Imm = MI.getOperand(3).getImm(); 8264 unsigned NewImm = 0x44; 8265 if (Imm & 1) NewImm |= 0x0a; 8266 if (Imm & 2) NewImm |= 0xa0; 8267 MI.getOperand(3).setImm(NewImm); 8268 MI.setDesc(get(X86::SHUFPSrri)); 8269 } 8270 return true; 8271 } 8272 } 8273 return false; 8274 } 8275 8276 std::pair<uint16_t, uint16_t> 8277 X86InstrInfo::getExecutionDomain(const MachineInstr &MI) const { 8278 uint16_t domain = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 8279 unsigned opcode = MI.getOpcode(); 8280 uint16_t validDomains = 0; 8281 if (domain) { 8282 // Attempt to match for custom instructions. 8283 validDomains = getExecutionDomainCustom(MI); 8284 if (validDomains) 8285 return std::make_pair(domain, validDomains); 8286 8287 if (lookup(opcode, domain, ReplaceableInstrs)) { 8288 validDomains = 0xe; 8289 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2)) { 8290 validDomains = Subtarget.hasAVX2() ? 0xe : 0x6; 8291 } else if (lookup(opcode, domain, ReplaceableInstrsFP)) { 8292 validDomains = 0x6; 8293 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2InsertExtract)) { 8294 // Insert/extract instructions should only effect domain if AVX2 8295 // is enabled. 8296 if (!Subtarget.hasAVX2()) 8297 return std::make_pair(0, 0); 8298 validDomains = 0xe; 8299 } else if (lookupAVX512(opcode, domain, ReplaceableInstrsAVX512)) { 8300 validDomains = 0xe; 8301 } else if (Subtarget.hasDQI() && lookupAVX512(opcode, domain, 8302 ReplaceableInstrsAVX512DQ)) { 8303 validDomains = 0xe; 8304 } else if (Subtarget.hasDQI()) { 8305 if (const uint16_t *table = lookupAVX512(opcode, domain, 8306 ReplaceableInstrsAVX512DQMasked)) { 8307 if (domain == 1 || (domain == 3 && table[3] == opcode)) 8308 validDomains = 0xa; 8309 else 8310 validDomains = 0xc; 8311 } 8312 } 8313 } 8314 return std::make_pair(domain, validDomains); 8315 } 8316 8317 void X86InstrInfo::setExecutionDomain(MachineInstr &MI, unsigned Domain) const { 8318 assert(Domain>0 && Domain<4 && "Invalid execution domain"); 8319 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 8320 assert(dom && "Not an SSE instruction"); 8321 8322 // Attempt to match for custom instructions. 8323 if (setExecutionDomainCustom(MI, Domain)) 8324 return; 8325 8326 const uint16_t *table = lookup(MI.getOpcode(), dom, ReplaceableInstrs); 8327 if (!table) { // try the other table 8328 assert((Subtarget.hasAVX2() || Domain < 3) && 8329 "256-bit vector operations only available in AVX2"); 8330 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2); 8331 } 8332 if (!table) { // try the FP table 8333 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsFP); 8334 assert((!table || Domain < 3) && 8335 "Can only select PackedSingle or PackedDouble"); 8336 } 8337 if (!table) { // try the other table 8338 assert(Subtarget.hasAVX2() && 8339 "256-bit insert/extract only available in AVX2"); 8340 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2InsertExtract); 8341 } 8342 if (!table) { // try the AVX512 table 8343 assert(Subtarget.hasAVX512() && "Requires AVX-512"); 8344 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512); 8345 // Don't change integer Q instructions to D instructions. 8346 if (table && Domain == 3 && table[3] == MI.getOpcode()) 8347 Domain = 4; 8348 } 8349 if (!table) { // try the AVX512DQ table 8350 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ"); 8351 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQ); 8352 // Don't change integer Q instructions to D instructions and 8353 // use D instructions if we started with a PS instruction. 8354 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 8355 Domain = 4; 8356 } 8357 if (!table) { // try the AVX512DQMasked table 8358 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ"); 8359 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQMasked); 8360 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 8361 Domain = 4; 8362 } 8363 assert(table && "Cannot change domain"); 8364 MI.setDesc(get(table[Domain - 1])); 8365 } 8366 8367 /// Return the noop instruction to use for a noop. 8368 MCInst X86InstrInfo::getNop() const { 8369 MCInst Nop; 8370 Nop.setOpcode(X86::NOOP); 8371 return Nop; 8372 } 8373 8374 bool X86InstrInfo::isHighLatencyDef(int opc) const { 8375 switch (opc) { 8376 default: return false; 8377 case X86::DIVPDrm: 8378 case X86::DIVPDrr: 8379 case X86::DIVPSrm: 8380 case X86::DIVPSrr: 8381 case X86::DIVSDrm: 8382 case X86::DIVSDrm_Int: 8383 case X86::DIVSDrr: 8384 case X86::DIVSDrr_Int: 8385 case X86::DIVSSrm: 8386 case X86::DIVSSrm_Int: 8387 case X86::DIVSSrr: 8388 case X86::DIVSSrr_Int: 8389 case X86::SQRTPDm: 8390 case X86::SQRTPDr: 8391 case X86::SQRTPSm: 8392 case X86::SQRTPSr: 8393 case X86::SQRTSDm: 8394 case X86::SQRTSDm_Int: 8395 case X86::SQRTSDr: 8396 case X86::SQRTSDr_Int: 8397 case X86::SQRTSSm: 8398 case X86::SQRTSSm_Int: 8399 case X86::SQRTSSr: 8400 case X86::SQRTSSr_Int: 8401 // AVX instructions with high latency 8402 case X86::VDIVPDrm: 8403 case X86::VDIVPDrr: 8404 case X86::VDIVPDYrm: 8405 case X86::VDIVPDYrr: 8406 case X86::VDIVPSrm: 8407 case X86::VDIVPSrr: 8408 case X86::VDIVPSYrm: 8409 case X86::VDIVPSYrr: 8410 case X86::VDIVSDrm: 8411 case X86::VDIVSDrm_Int: 8412 case X86::VDIVSDrr: 8413 case X86::VDIVSDrr_Int: 8414 case X86::VDIVSSrm: 8415 case X86::VDIVSSrm_Int: 8416 case X86::VDIVSSrr: 8417 case X86::VDIVSSrr_Int: 8418 case X86::VSQRTPDm: 8419 case X86::VSQRTPDr: 8420 case X86::VSQRTPDYm: 8421 case X86::VSQRTPDYr: 8422 case X86::VSQRTPSm: 8423 case X86::VSQRTPSr: 8424 case X86::VSQRTPSYm: 8425 case X86::VSQRTPSYr: 8426 case X86::VSQRTSDm: 8427 case X86::VSQRTSDm_Int: 8428 case X86::VSQRTSDr: 8429 case X86::VSQRTSDr_Int: 8430 case X86::VSQRTSSm: 8431 case X86::VSQRTSSm_Int: 8432 case X86::VSQRTSSr: 8433 case X86::VSQRTSSr_Int: 8434 // AVX512 instructions with high latency 8435 case X86::VDIVPDZ128rm: 8436 case X86::VDIVPDZ128rmb: 8437 case X86::VDIVPDZ128rmbk: 8438 case X86::VDIVPDZ128rmbkz: 8439 case X86::VDIVPDZ128rmk: 8440 case X86::VDIVPDZ128rmkz: 8441 case X86::VDIVPDZ128rr: 8442 case X86::VDIVPDZ128rrk: 8443 case X86::VDIVPDZ128rrkz: 8444 case X86::VDIVPDZ256rm: 8445 case X86::VDIVPDZ256rmb: 8446 case X86::VDIVPDZ256rmbk: 8447 case X86::VDIVPDZ256rmbkz: 8448 case X86::VDIVPDZ256rmk: 8449 case X86::VDIVPDZ256rmkz: 8450 case X86::VDIVPDZ256rr: 8451 case X86::VDIVPDZ256rrk: 8452 case X86::VDIVPDZ256rrkz: 8453 case X86::VDIVPDZrrb: 8454 case X86::VDIVPDZrrbk: 8455 case X86::VDIVPDZrrbkz: 8456 case X86::VDIVPDZrm: 8457 case X86::VDIVPDZrmb: 8458 case X86::VDIVPDZrmbk: 8459 case X86::VDIVPDZrmbkz: 8460 case X86::VDIVPDZrmk: 8461 case X86::VDIVPDZrmkz: 8462 case X86::VDIVPDZrr: 8463 case X86::VDIVPDZrrk: 8464 case X86::VDIVPDZrrkz: 8465 case X86::VDIVPSZ128rm: 8466 case X86::VDIVPSZ128rmb: 8467 case X86::VDIVPSZ128rmbk: 8468 case X86::VDIVPSZ128rmbkz: 8469 case X86::VDIVPSZ128rmk: 8470 case X86::VDIVPSZ128rmkz: 8471 case X86::VDIVPSZ128rr: 8472 case X86::VDIVPSZ128rrk: 8473 case X86::VDIVPSZ128rrkz: 8474 case X86::VDIVPSZ256rm: 8475 case X86::VDIVPSZ256rmb: 8476 case X86::VDIVPSZ256rmbk: 8477 case X86::VDIVPSZ256rmbkz: 8478 case X86::VDIVPSZ256rmk: 8479 case X86::VDIVPSZ256rmkz: 8480 case X86::VDIVPSZ256rr: 8481 case X86::VDIVPSZ256rrk: 8482 case X86::VDIVPSZ256rrkz: 8483 case X86::VDIVPSZrrb: 8484 case X86::VDIVPSZrrbk: 8485 case X86::VDIVPSZrrbkz: 8486 case X86::VDIVPSZrm: 8487 case X86::VDIVPSZrmb: 8488 case X86::VDIVPSZrmbk: 8489 case X86::VDIVPSZrmbkz: 8490 case X86::VDIVPSZrmk: 8491 case X86::VDIVPSZrmkz: 8492 case X86::VDIVPSZrr: 8493 case X86::VDIVPSZrrk: 8494 case X86::VDIVPSZrrkz: 8495 case X86::VDIVSDZrm: 8496 case X86::VDIVSDZrr: 8497 case X86::VDIVSDZrm_Int: 8498 case X86::VDIVSDZrm_Intk: 8499 case X86::VDIVSDZrm_Intkz: 8500 case X86::VDIVSDZrr_Int: 8501 case X86::VDIVSDZrr_Intk: 8502 case X86::VDIVSDZrr_Intkz: 8503 case X86::VDIVSDZrrb_Int: 8504 case X86::VDIVSDZrrb_Intk: 8505 case X86::VDIVSDZrrb_Intkz: 8506 case X86::VDIVSSZrm: 8507 case X86::VDIVSSZrr: 8508 case X86::VDIVSSZrm_Int: 8509 case X86::VDIVSSZrm_Intk: 8510 case X86::VDIVSSZrm_Intkz: 8511 case X86::VDIVSSZrr_Int: 8512 case X86::VDIVSSZrr_Intk: 8513 case X86::VDIVSSZrr_Intkz: 8514 case X86::VDIVSSZrrb_Int: 8515 case X86::VDIVSSZrrb_Intk: 8516 case X86::VDIVSSZrrb_Intkz: 8517 case X86::VSQRTPDZ128m: 8518 case X86::VSQRTPDZ128mb: 8519 case X86::VSQRTPDZ128mbk: 8520 case X86::VSQRTPDZ128mbkz: 8521 case X86::VSQRTPDZ128mk: 8522 case X86::VSQRTPDZ128mkz: 8523 case X86::VSQRTPDZ128r: 8524 case X86::VSQRTPDZ128rk: 8525 case X86::VSQRTPDZ128rkz: 8526 case X86::VSQRTPDZ256m: 8527 case X86::VSQRTPDZ256mb: 8528 case X86::VSQRTPDZ256mbk: 8529 case X86::VSQRTPDZ256mbkz: 8530 case X86::VSQRTPDZ256mk: 8531 case X86::VSQRTPDZ256mkz: 8532 case X86::VSQRTPDZ256r: 8533 case X86::VSQRTPDZ256rk: 8534 case X86::VSQRTPDZ256rkz: 8535 case X86::VSQRTPDZm: 8536 case X86::VSQRTPDZmb: 8537 case X86::VSQRTPDZmbk: 8538 case X86::VSQRTPDZmbkz: 8539 case X86::VSQRTPDZmk: 8540 case X86::VSQRTPDZmkz: 8541 case X86::VSQRTPDZr: 8542 case X86::VSQRTPDZrb: 8543 case X86::VSQRTPDZrbk: 8544 case X86::VSQRTPDZrbkz: 8545 case X86::VSQRTPDZrk: 8546 case X86::VSQRTPDZrkz: 8547 case X86::VSQRTPSZ128m: 8548 case X86::VSQRTPSZ128mb: 8549 case X86::VSQRTPSZ128mbk: 8550 case X86::VSQRTPSZ128mbkz: 8551 case X86::VSQRTPSZ128mk: 8552 case X86::VSQRTPSZ128mkz: 8553 case X86::VSQRTPSZ128r: 8554 case X86::VSQRTPSZ128rk: 8555 case X86::VSQRTPSZ128rkz: 8556 case X86::VSQRTPSZ256m: 8557 case X86::VSQRTPSZ256mb: 8558 case X86::VSQRTPSZ256mbk: 8559 case X86::VSQRTPSZ256mbkz: 8560 case X86::VSQRTPSZ256mk: 8561 case X86::VSQRTPSZ256mkz: 8562 case X86::VSQRTPSZ256r: 8563 case X86::VSQRTPSZ256rk: 8564 case X86::VSQRTPSZ256rkz: 8565 case X86::VSQRTPSZm: 8566 case X86::VSQRTPSZmb: 8567 case X86::VSQRTPSZmbk: 8568 case X86::VSQRTPSZmbkz: 8569 case X86::VSQRTPSZmk: 8570 case X86::VSQRTPSZmkz: 8571 case X86::VSQRTPSZr: 8572 case X86::VSQRTPSZrb: 8573 case X86::VSQRTPSZrbk: 8574 case X86::VSQRTPSZrbkz: 8575 case X86::VSQRTPSZrk: 8576 case X86::VSQRTPSZrkz: 8577 case X86::VSQRTSDZm: 8578 case X86::VSQRTSDZm_Int: 8579 case X86::VSQRTSDZm_Intk: 8580 case X86::VSQRTSDZm_Intkz: 8581 case X86::VSQRTSDZr: 8582 case X86::VSQRTSDZr_Int: 8583 case X86::VSQRTSDZr_Intk: 8584 case X86::VSQRTSDZr_Intkz: 8585 case X86::VSQRTSDZrb_Int: 8586 case X86::VSQRTSDZrb_Intk: 8587 case X86::VSQRTSDZrb_Intkz: 8588 case X86::VSQRTSSZm: 8589 case X86::VSQRTSSZm_Int: 8590 case X86::VSQRTSSZm_Intk: 8591 case X86::VSQRTSSZm_Intkz: 8592 case X86::VSQRTSSZr: 8593 case X86::VSQRTSSZr_Int: 8594 case X86::VSQRTSSZr_Intk: 8595 case X86::VSQRTSSZr_Intkz: 8596 case X86::VSQRTSSZrb_Int: 8597 case X86::VSQRTSSZrb_Intk: 8598 case X86::VSQRTSSZrb_Intkz: 8599 8600 case X86::VGATHERDPDYrm: 8601 case X86::VGATHERDPDZ128rm: 8602 case X86::VGATHERDPDZ256rm: 8603 case X86::VGATHERDPDZrm: 8604 case X86::VGATHERDPDrm: 8605 case X86::VGATHERDPSYrm: 8606 case X86::VGATHERDPSZ128rm: 8607 case X86::VGATHERDPSZ256rm: 8608 case X86::VGATHERDPSZrm: 8609 case X86::VGATHERDPSrm: 8610 case X86::VGATHERPF0DPDm: 8611 case X86::VGATHERPF0DPSm: 8612 case X86::VGATHERPF0QPDm: 8613 case X86::VGATHERPF0QPSm: 8614 case X86::VGATHERPF1DPDm: 8615 case X86::VGATHERPF1DPSm: 8616 case X86::VGATHERPF1QPDm: 8617 case X86::VGATHERPF1QPSm: 8618 case X86::VGATHERQPDYrm: 8619 case X86::VGATHERQPDZ128rm: 8620 case X86::VGATHERQPDZ256rm: 8621 case X86::VGATHERQPDZrm: 8622 case X86::VGATHERQPDrm: 8623 case X86::VGATHERQPSYrm: 8624 case X86::VGATHERQPSZ128rm: 8625 case X86::VGATHERQPSZ256rm: 8626 case X86::VGATHERQPSZrm: 8627 case X86::VGATHERQPSrm: 8628 case X86::VPGATHERDDYrm: 8629 case X86::VPGATHERDDZ128rm: 8630 case X86::VPGATHERDDZ256rm: 8631 case X86::VPGATHERDDZrm: 8632 case X86::VPGATHERDDrm: 8633 case X86::VPGATHERDQYrm: 8634 case X86::VPGATHERDQZ128rm: 8635 case X86::VPGATHERDQZ256rm: 8636 case X86::VPGATHERDQZrm: 8637 case X86::VPGATHERDQrm: 8638 case X86::VPGATHERQDYrm: 8639 case X86::VPGATHERQDZ128rm: 8640 case X86::VPGATHERQDZ256rm: 8641 case X86::VPGATHERQDZrm: 8642 case X86::VPGATHERQDrm: 8643 case X86::VPGATHERQQYrm: 8644 case X86::VPGATHERQQZ128rm: 8645 case X86::VPGATHERQQZ256rm: 8646 case X86::VPGATHERQQZrm: 8647 case X86::VPGATHERQQrm: 8648 case X86::VSCATTERDPDZ128mr: 8649 case X86::VSCATTERDPDZ256mr: 8650 case X86::VSCATTERDPDZmr: 8651 case X86::VSCATTERDPSZ128mr: 8652 case X86::VSCATTERDPSZ256mr: 8653 case X86::VSCATTERDPSZmr: 8654 case X86::VSCATTERPF0DPDm: 8655 case X86::VSCATTERPF0DPSm: 8656 case X86::VSCATTERPF0QPDm: 8657 case X86::VSCATTERPF0QPSm: 8658 case X86::VSCATTERPF1DPDm: 8659 case X86::VSCATTERPF1DPSm: 8660 case X86::VSCATTERPF1QPDm: 8661 case X86::VSCATTERPF1QPSm: 8662 case X86::VSCATTERQPDZ128mr: 8663 case X86::VSCATTERQPDZ256mr: 8664 case X86::VSCATTERQPDZmr: 8665 case X86::VSCATTERQPSZ128mr: 8666 case X86::VSCATTERQPSZ256mr: 8667 case X86::VSCATTERQPSZmr: 8668 case X86::VPSCATTERDDZ128mr: 8669 case X86::VPSCATTERDDZ256mr: 8670 case X86::VPSCATTERDDZmr: 8671 case X86::VPSCATTERDQZ128mr: 8672 case X86::VPSCATTERDQZ256mr: 8673 case X86::VPSCATTERDQZmr: 8674 case X86::VPSCATTERQDZ128mr: 8675 case X86::VPSCATTERQDZ256mr: 8676 case X86::VPSCATTERQDZmr: 8677 case X86::VPSCATTERQQZ128mr: 8678 case X86::VPSCATTERQQZ256mr: 8679 case X86::VPSCATTERQQZmr: 8680 return true; 8681 } 8682 } 8683 8684 bool X86InstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel, 8685 const MachineRegisterInfo *MRI, 8686 const MachineInstr &DefMI, 8687 unsigned DefIdx, 8688 const MachineInstr &UseMI, 8689 unsigned UseIdx) const { 8690 return isHighLatencyDef(DefMI.getOpcode()); 8691 } 8692 8693 bool X86InstrInfo::hasReassociableOperands(const MachineInstr &Inst, 8694 const MachineBasicBlock *MBB) const { 8695 assert(Inst.getNumExplicitOperands() == 3 && Inst.getNumExplicitDefs() == 1 && 8696 Inst.getNumDefs() <= 2 && "Reassociation needs binary operators"); 8697 8698 // Integer binary math/logic instructions have a third source operand: 8699 // the EFLAGS register. That operand must be both defined here and never 8700 // used; ie, it must be dead. If the EFLAGS operand is live, then we can 8701 // not change anything because rearranging the operands could affect other 8702 // instructions that depend on the exact status flags (zero, sign, etc.) 8703 // that are set by using these particular operands with this operation. 8704 const MachineOperand *FlagDef = Inst.findRegisterDefOperand(X86::EFLAGS); 8705 assert((Inst.getNumDefs() == 1 || FlagDef) && 8706 "Implicit def isn't flags?"); 8707 if (FlagDef && !FlagDef->isDead()) 8708 return false; 8709 8710 return TargetInstrInfo::hasReassociableOperands(Inst, MBB); 8711 } 8712 8713 // TODO: There are many more machine instruction opcodes to match: 8714 // 1. Other data types (integer, vectors) 8715 // 2. Other math / logic operations (xor, or) 8716 // 3. Other forms of the same operation (intrinsics and other variants) 8717 bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst, 8718 bool Invert) const { 8719 if (Invert) 8720 return false; 8721 switch (Inst.getOpcode()) { 8722 case X86::ADD8rr: 8723 case X86::ADD16rr: 8724 case X86::ADD32rr: 8725 case X86::ADD64rr: 8726 case X86::AND8rr: 8727 case X86::AND16rr: 8728 case X86::AND32rr: 8729 case X86::AND64rr: 8730 case X86::OR8rr: 8731 case X86::OR16rr: 8732 case X86::OR32rr: 8733 case X86::OR64rr: 8734 case X86::XOR8rr: 8735 case X86::XOR16rr: 8736 case X86::XOR32rr: 8737 case X86::XOR64rr: 8738 case X86::IMUL16rr: 8739 case X86::IMUL32rr: 8740 case X86::IMUL64rr: 8741 case X86::PANDrr: 8742 case X86::PORrr: 8743 case X86::PXORrr: 8744 case X86::ANDPDrr: 8745 case X86::ANDPSrr: 8746 case X86::ORPDrr: 8747 case X86::ORPSrr: 8748 case X86::XORPDrr: 8749 case X86::XORPSrr: 8750 case X86::PADDBrr: 8751 case X86::PADDWrr: 8752 case X86::PADDDrr: 8753 case X86::PADDQrr: 8754 case X86::PMULLWrr: 8755 case X86::PMULLDrr: 8756 case X86::PMAXSBrr: 8757 case X86::PMAXSDrr: 8758 case X86::PMAXSWrr: 8759 case X86::PMAXUBrr: 8760 case X86::PMAXUDrr: 8761 case X86::PMAXUWrr: 8762 case X86::PMINSBrr: 8763 case X86::PMINSDrr: 8764 case X86::PMINSWrr: 8765 case X86::PMINUBrr: 8766 case X86::PMINUDrr: 8767 case X86::PMINUWrr: 8768 case X86::VPANDrr: 8769 case X86::VPANDYrr: 8770 case X86::VPANDDZ128rr: 8771 case X86::VPANDDZ256rr: 8772 case X86::VPANDDZrr: 8773 case X86::VPANDQZ128rr: 8774 case X86::VPANDQZ256rr: 8775 case X86::VPANDQZrr: 8776 case X86::VPORrr: 8777 case X86::VPORYrr: 8778 case X86::VPORDZ128rr: 8779 case X86::VPORDZ256rr: 8780 case X86::VPORDZrr: 8781 case X86::VPORQZ128rr: 8782 case X86::VPORQZ256rr: 8783 case X86::VPORQZrr: 8784 case X86::VPXORrr: 8785 case X86::VPXORYrr: 8786 case X86::VPXORDZ128rr: 8787 case X86::VPXORDZ256rr: 8788 case X86::VPXORDZrr: 8789 case X86::VPXORQZ128rr: 8790 case X86::VPXORQZ256rr: 8791 case X86::VPXORQZrr: 8792 case X86::VANDPDrr: 8793 case X86::VANDPSrr: 8794 case X86::VANDPDYrr: 8795 case X86::VANDPSYrr: 8796 case X86::VANDPDZ128rr: 8797 case X86::VANDPSZ128rr: 8798 case X86::VANDPDZ256rr: 8799 case X86::VANDPSZ256rr: 8800 case X86::VANDPDZrr: 8801 case X86::VANDPSZrr: 8802 case X86::VORPDrr: 8803 case X86::VORPSrr: 8804 case X86::VORPDYrr: 8805 case X86::VORPSYrr: 8806 case X86::VORPDZ128rr: 8807 case X86::VORPSZ128rr: 8808 case X86::VORPDZ256rr: 8809 case X86::VORPSZ256rr: 8810 case X86::VORPDZrr: 8811 case X86::VORPSZrr: 8812 case X86::VXORPDrr: 8813 case X86::VXORPSrr: 8814 case X86::VXORPDYrr: 8815 case X86::VXORPSYrr: 8816 case X86::VXORPDZ128rr: 8817 case X86::VXORPSZ128rr: 8818 case X86::VXORPDZ256rr: 8819 case X86::VXORPSZ256rr: 8820 case X86::VXORPDZrr: 8821 case X86::VXORPSZrr: 8822 case X86::KADDBrr: 8823 case X86::KADDWrr: 8824 case X86::KADDDrr: 8825 case X86::KADDQrr: 8826 case X86::KANDBrr: 8827 case X86::KANDWrr: 8828 case X86::KANDDrr: 8829 case X86::KANDQrr: 8830 case X86::KORBrr: 8831 case X86::KORWrr: 8832 case X86::KORDrr: 8833 case X86::KORQrr: 8834 case X86::KXORBrr: 8835 case X86::KXORWrr: 8836 case X86::KXORDrr: 8837 case X86::KXORQrr: 8838 case X86::VPADDBrr: 8839 case X86::VPADDWrr: 8840 case X86::VPADDDrr: 8841 case X86::VPADDQrr: 8842 case X86::VPADDBYrr: 8843 case X86::VPADDWYrr: 8844 case X86::VPADDDYrr: 8845 case X86::VPADDQYrr: 8846 case X86::VPADDBZ128rr: 8847 case X86::VPADDWZ128rr: 8848 case X86::VPADDDZ128rr: 8849 case X86::VPADDQZ128rr: 8850 case X86::VPADDBZ256rr: 8851 case X86::VPADDWZ256rr: 8852 case X86::VPADDDZ256rr: 8853 case X86::VPADDQZ256rr: 8854 case X86::VPADDBZrr: 8855 case X86::VPADDWZrr: 8856 case X86::VPADDDZrr: 8857 case X86::VPADDQZrr: 8858 case X86::VPMULLWrr: 8859 case X86::VPMULLWYrr: 8860 case X86::VPMULLWZ128rr: 8861 case X86::VPMULLWZ256rr: 8862 case X86::VPMULLWZrr: 8863 case X86::VPMULLDrr: 8864 case X86::VPMULLDYrr: 8865 case X86::VPMULLDZ128rr: 8866 case X86::VPMULLDZ256rr: 8867 case X86::VPMULLDZrr: 8868 case X86::VPMULLQZ128rr: 8869 case X86::VPMULLQZ256rr: 8870 case X86::VPMULLQZrr: 8871 case X86::VPMAXSBrr: 8872 case X86::VPMAXSBYrr: 8873 case X86::VPMAXSBZ128rr: 8874 case X86::VPMAXSBZ256rr: 8875 case X86::VPMAXSBZrr: 8876 case X86::VPMAXSDrr: 8877 case X86::VPMAXSDYrr: 8878 case X86::VPMAXSDZ128rr: 8879 case X86::VPMAXSDZ256rr: 8880 case X86::VPMAXSDZrr: 8881 case X86::VPMAXSQZ128rr: 8882 case X86::VPMAXSQZ256rr: 8883 case X86::VPMAXSQZrr: 8884 case X86::VPMAXSWrr: 8885 case X86::VPMAXSWYrr: 8886 case X86::VPMAXSWZ128rr: 8887 case X86::VPMAXSWZ256rr: 8888 case X86::VPMAXSWZrr: 8889 case X86::VPMAXUBrr: 8890 case X86::VPMAXUBYrr: 8891 case X86::VPMAXUBZ128rr: 8892 case X86::VPMAXUBZ256rr: 8893 case X86::VPMAXUBZrr: 8894 case X86::VPMAXUDrr: 8895 case X86::VPMAXUDYrr: 8896 case X86::VPMAXUDZ128rr: 8897 case X86::VPMAXUDZ256rr: 8898 case X86::VPMAXUDZrr: 8899 case X86::VPMAXUQZ128rr: 8900 case X86::VPMAXUQZ256rr: 8901 case X86::VPMAXUQZrr: 8902 case X86::VPMAXUWrr: 8903 case X86::VPMAXUWYrr: 8904 case X86::VPMAXUWZ128rr: 8905 case X86::VPMAXUWZ256rr: 8906 case X86::VPMAXUWZrr: 8907 case X86::VPMINSBrr: 8908 case X86::VPMINSBYrr: 8909 case X86::VPMINSBZ128rr: 8910 case X86::VPMINSBZ256rr: 8911 case X86::VPMINSBZrr: 8912 case X86::VPMINSDrr: 8913 case X86::VPMINSDYrr: 8914 case X86::VPMINSDZ128rr: 8915 case X86::VPMINSDZ256rr: 8916 case X86::VPMINSDZrr: 8917 case X86::VPMINSQZ128rr: 8918 case X86::VPMINSQZ256rr: 8919 case X86::VPMINSQZrr: 8920 case X86::VPMINSWrr: 8921 case X86::VPMINSWYrr: 8922 case X86::VPMINSWZ128rr: 8923 case X86::VPMINSWZ256rr: 8924 case X86::VPMINSWZrr: 8925 case X86::VPMINUBrr: 8926 case X86::VPMINUBYrr: 8927 case X86::VPMINUBZ128rr: 8928 case X86::VPMINUBZ256rr: 8929 case X86::VPMINUBZrr: 8930 case X86::VPMINUDrr: 8931 case X86::VPMINUDYrr: 8932 case X86::VPMINUDZ128rr: 8933 case X86::VPMINUDZ256rr: 8934 case X86::VPMINUDZrr: 8935 case X86::VPMINUQZ128rr: 8936 case X86::VPMINUQZ256rr: 8937 case X86::VPMINUQZrr: 8938 case X86::VPMINUWrr: 8939 case X86::VPMINUWYrr: 8940 case X86::VPMINUWZ128rr: 8941 case X86::VPMINUWZ256rr: 8942 case X86::VPMINUWZrr: 8943 // Normal min/max instructions are not commutative because of NaN and signed 8944 // zero semantics, but these are. Thus, there's no need to check for global 8945 // relaxed math; the instructions themselves have the properties we need. 8946 case X86::MAXCPDrr: 8947 case X86::MAXCPSrr: 8948 case X86::MAXCSDrr: 8949 case X86::MAXCSSrr: 8950 case X86::MINCPDrr: 8951 case X86::MINCPSrr: 8952 case X86::MINCSDrr: 8953 case X86::MINCSSrr: 8954 case X86::VMAXCPDrr: 8955 case X86::VMAXCPSrr: 8956 case X86::VMAXCPDYrr: 8957 case X86::VMAXCPSYrr: 8958 case X86::VMAXCPDZ128rr: 8959 case X86::VMAXCPSZ128rr: 8960 case X86::VMAXCPDZ256rr: 8961 case X86::VMAXCPSZ256rr: 8962 case X86::VMAXCPDZrr: 8963 case X86::VMAXCPSZrr: 8964 case X86::VMAXCSDrr: 8965 case X86::VMAXCSSrr: 8966 case X86::VMAXCSDZrr: 8967 case X86::VMAXCSSZrr: 8968 case X86::VMINCPDrr: 8969 case X86::VMINCPSrr: 8970 case X86::VMINCPDYrr: 8971 case X86::VMINCPSYrr: 8972 case X86::VMINCPDZ128rr: 8973 case X86::VMINCPSZ128rr: 8974 case X86::VMINCPDZ256rr: 8975 case X86::VMINCPSZ256rr: 8976 case X86::VMINCPDZrr: 8977 case X86::VMINCPSZrr: 8978 case X86::VMINCSDrr: 8979 case X86::VMINCSSrr: 8980 case X86::VMINCSDZrr: 8981 case X86::VMINCSSZrr: 8982 case X86::VMAXCPHZ128rr: 8983 case X86::VMAXCPHZ256rr: 8984 case X86::VMAXCPHZrr: 8985 case X86::VMAXCSHZrr: 8986 case X86::VMINCPHZ128rr: 8987 case X86::VMINCPHZ256rr: 8988 case X86::VMINCPHZrr: 8989 case X86::VMINCSHZrr: 8990 return true; 8991 case X86::ADDPDrr: 8992 case X86::ADDPSrr: 8993 case X86::ADDSDrr: 8994 case X86::ADDSSrr: 8995 case X86::MULPDrr: 8996 case X86::MULPSrr: 8997 case X86::MULSDrr: 8998 case X86::MULSSrr: 8999 case X86::VADDPDrr: 9000 case X86::VADDPSrr: 9001 case X86::VADDPDYrr: 9002 case X86::VADDPSYrr: 9003 case X86::VADDPDZ128rr: 9004 case X86::VADDPSZ128rr: 9005 case X86::VADDPDZ256rr: 9006 case X86::VADDPSZ256rr: 9007 case X86::VADDPDZrr: 9008 case X86::VADDPSZrr: 9009 case X86::VADDSDrr: 9010 case X86::VADDSSrr: 9011 case X86::VADDSDZrr: 9012 case X86::VADDSSZrr: 9013 case X86::VMULPDrr: 9014 case X86::VMULPSrr: 9015 case X86::VMULPDYrr: 9016 case X86::VMULPSYrr: 9017 case X86::VMULPDZ128rr: 9018 case X86::VMULPSZ128rr: 9019 case X86::VMULPDZ256rr: 9020 case X86::VMULPSZ256rr: 9021 case X86::VMULPDZrr: 9022 case X86::VMULPSZrr: 9023 case X86::VMULSDrr: 9024 case X86::VMULSSrr: 9025 case X86::VMULSDZrr: 9026 case X86::VMULSSZrr: 9027 case X86::VADDPHZ128rr: 9028 case X86::VADDPHZ256rr: 9029 case X86::VADDPHZrr: 9030 case X86::VADDSHZrr: 9031 case X86::VMULPHZ128rr: 9032 case X86::VMULPHZ256rr: 9033 case X86::VMULPHZrr: 9034 case X86::VMULSHZrr: 9035 return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) && 9036 Inst.getFlag(MachineInstr::MIFlag::FmNsz); 9037 default: 9038 return false; 9039 } 9040 } 9041 9042 /// If \p DescribedReg overlaps with the MOVrr instruction's destination 9043 /// register then, if possible, describe the value in terms of the source 9044 /// register. 9045 static std::optional<ParamLoadedValue> 9046 describeMOVrrLoadedValue(const MachineInstr &MI, Register DescribedReg, 9047 const TargetRegisterInfo *TRI) { 9048 Register DestReg = MI.getOperand(0).getReg(); 9049 Register SrcReg = MI.getOperand(1).getReg(); 9050 9051 auto Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {}); 9052 9053 // If the described register is the destination, just return the source. 9054 if (DestReg == DescribedReg) 9055 return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr); 9056 9057 // If the described register is a sub-register of the destination register, 9058 // then pick out the source register's corresponding sub-register. 9059 if (unsigned SubRegIdx = TRI->getSubRegIndex(DestReg, DescribedReg)) { 9060 Register SrcSubReg = TRI->getSubReg(SrcReg, SubRegIdx); 9061 return ParamLoadedValue(MachineOperand::CreateReg(SrcSubReg, false), Expr); 9062 } 9063 9064 // The remaining case to consider is when the described register is a 9065 // super-register of the destination register. MOV8rr and MOV16rr does not 9066 // write to any of the other bytes in the register, meaning that we'd have to 9067 // describe the value using a combination of the source register and the 9068 // non-overlapping bits in the described register, which is not currently 9069 // possible. 9070 if (MI.getOpcode() == X86::MOV8rr || MI.getOpcode() == X86::MOV16rr || 9071 !TRI->isSuperRegister(DestReg, DescribedReg)) 9072 return std::nullopt; 9073 9074 assert(MI.getOpcode() == X86::MOV32rr && "Unexpected super-register case"); 9075 return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr); 9076 } 9077 9078 std::optional<ParamLoadedValue> 9079 X86InstrInfo::describeLoadedValue(const MachineInstr &MI, Register Reg) const { 9080 const MachineOperand *Op = nullptr; 9081 DIExpression *Expr = nullptr; 9082 9083 const TargetRegisterInfo *TRI = &getRegisterInfo(); 9084 9085 switch (MI.getOpcode()) { 9086 case X86::LEA32r: 9087 case X86::LEA64r: 9088 case X86::LEA64_32r: { 9089 // We may need to describe a 64-bit parameter with a 32-bit LEA. 9090 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) 9091 return std::nullopt; 9092 9093 // Operand 4 could be global address. For now we do not support 9094 // such situation. 9095 if (!MI.getOperand(4).isImm() || !MI.getOperand(2).isImm()) 9096 return std::nullopt; 9097 9098 const MachineOperand &Op1 = MI.getOperand(1); 9099 const MachineOperand &Op2 = MI.getOperand(3); 9100 assert(Op2.isReg() && 9101 (Op2.getReg() == X86::NoRegister || Op2.getReg().isPhysical())); 9102 9103 // Omit situations like: 9104 // %rsi = lea %rsi, 4, ... 9105 if ((Op1.isReg() && Op1.getReg() == MI.getOperand(0).getReg()) || 9106 Op2.getReg() == MI.getOperand(0).getReg()) 9107 return std::nullopt; 9108 else if ((Op1.isReg() && Op1.getReg() != X86::NoRegister && 9109 TRI->regsOverlap(Op1.getReg(), MI.getOperand(0).getReg())) || 9110 (Op2.getReg() != X86::NoRegister && 9111 TRI->regsOverlap(Op2.getReg(), MI.getOperand(0).getReg()))) 9112 return std::nullopt; 9113 9114 int64_t Coef = MI.getOperand(2).getImm(); 9115 int64_t Offset = MI.getOperand(4).getImm(); 9116 SmallVector<uint64_t, 8> Ops; 9117 9118 if ((Op1.isReg() && Op1.getReg() != X86::NoRegister)) { 9119 Op = &Op1; 9120 } else if (Op1.isFI()) 9121 Op = &Op1; 9122 9123 if (Op && Op->isReg() && Op->getReg() == Op2.getReg() && Coef > 0) { 9124 Ops.push_back(dwarf::DW_OP_constu); 9125 Ops.push_back(Coef + 1); 9126 Ops.push_back(dwarf::DW_OP_mul); 9127 } else { 9128 if (Op && Op2.getReg() != X86::NoRegister) { 9129 int dwarfReg = TRI->getDwarfRegNum(Op2.getReg(), false); 9130 if (dwarfReg < 0) 9131 return std::nullopt; 9132 else if (dwarfReg < 32) { 9133 Ops.push_back(dwarf::DW_OP_breg0 + dwarfReg); 9134 Ops.push_back(0); 9135 } else { 9136 Ops.push_back(dwarf::DW_OP_bregx); 9137 Ops.push_back(dwarfReg); 9138 Ops.push_back(0); 9139 } 9140 } else if (!Op) { 9141 assert(Op2.getReg() != X86::NoRegister); 9142 Op = &Op2; 9143 } 9144 9145 if (Coef > 1) { 9146 assert(Op2.getReg() != X86::NoRegister); 9147 Ops.push_back(dwarf::DW_OP_constu); 9148 Ops.push_back(Coef); 9149 Ops.push_back(dwarf::DW_OP_mul); 9150 } 9151 9152 if (((Op1.isReg() && Op1.getReg() != X86::NoRegister) || Op1.isFI()) && 9153 Op2.getReg() != X86::NoRegister) { 9154 Ops.push_back(dwarf::DW_OP_plus); 9155 } 9156 } 9157 9158 DIExpression::appendOffset(Ops, Offset); 9159 Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), Ops); 9160 9161 return ParamLoadedValue(*Op, Expr);; 9162 } 9163 case X86::MOV8ri: 9164 case X86::MOV16ri: 9165 // TODO: Handle MOV8ri and MOV16ri. 9166 return std::nullopt; 9167 case X86::MOV32ri: 9168 case X86::MOV64ri: 9169 case X86::MOV64ri32: 9170 // MOV32ri may be used for producing zero-extended 32-bit immediates in 9171 // 64-bit parameters, so we need to consider super-registers. 9172 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) 9173 return std::nullopt; 9174 return ParamLoadedValue(MI.getOperand(1), Expr); 9175 case X86::MOV8rr: 9176 case X86::MOV16rr: 9177 case X86::MOV32rr: 9178 case X86::MOV64rr: 9179 return describeMOVrrLoadedValue(MI, Reg, TRI); 9180 case X86::XOR32rr: { 9181 // 64-bit parameters are zero-materialized using XOR32rr, so also consider 9182 // super-registers. 9183 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) 9184 return std::nullopt; 9185 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) 9186 return ParamLoadedValue(MachineOperand::CreateImm(0), Expr); 9187 return std::nullopt; 9188 } 9189 case X86::MOVSX64rr32: { 9190 // We may need to describe the lower 32 bits of the MOVSX; for example, in 9191 // cases like this: 9192 // 9193 // $ebx = [...] 9194 // $rdi = MOVSX64rr32 $ebx 9195 // $esi = MOV32rr $edi 9196 if (!TRI->isSubRegisterEq(MI.getOperand(0).getReg(), Reg)) 9197 return std::nullopt; 9198 9199 Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {}); 9200 9201 // If the described register is the destination register we need to 9202 // sign-extend the source register from 32 bits. The other case we handle 9203 // is when the described register is the 32-bit sub-register of the 9204 // destination register, in case we just need to return the source 9205 // register. 9206 if (Reg == MI.getOperand(0).getReg()) 9207 Expr = DIExpression::appendExt(Expr, 32, 64, true); 9208 else 9209 assert(X86MCRegisterClasses[X86::GR32RegClassID].contains(Reg) && 9210 "Unhandled sub-register case for MOVSX64rr32"); 9211 9212 return ParamLoadedValue(MI.getOperand(1), Expr); 9213 } 9214 default: 9215 assert(!MI.isMoveImmediate() && "Unexpected MoveImm instruction"); 9216 return TargetInstrInfo::describeLoadedValue(MI, Reg); 9217 } 9218 } 9219 9220 /// This is an architecture-specific helper function of reassociateOps. 9221 /// Set special operand attributes for new instructions after reassociation. 9222 void X86InstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1, 9223 MachineInstr &OldMI2, 9224 MachineInstr &NewMI1, 9225 MachineInstr &NewMI2) const { 9226 // Propagate FP flags from the original instructions. 9227 // But clear poison-generating flags because those may not be valid now. 9228 // TODO: There should be a helper function for copying only fast-math-flags. 9229 uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags(); 9230 NewMI1.setFlags(IntersectedFlags); 9231 NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap); 9232 NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap); 9233 NewMI1.clearFlag(MachineInstr::MIFlag::IsExact); 9234 9235 NewMI2.setFlags(IntersectedFlags); 9236 NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap); 9237 NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap); 9238 NewMI2.clearFlag(MachineInstr::MIFlag::IsExact); 9239 9240 // Integer instructions may define an implicit EFLAGS dest register operand. 9241 MachineOperand *OldFlagDef1 = OldMI1.findRegisterDefOperand(X86::EFLAGS); 9242 MachineOperand *OldFlagDef2 = OldMI2.findRegisterDefOperand(X86::EFLAGS); 9243 9244 assert(!OldFlagDef1 == !OldFlagDef2 && 9245 "Unexpected instruction type for reassociation"); 9246 9247 if (!OldFlagDef1 || !OldFlagDef2) 9248 return; 9249 9250 assert(OldFlagDef1->isDead() && OldFlagDef2->isDead() && 9251 "Must have dead EFLAGS operand in reassociable instruction"); 9252 9253 MachineOperand *NewFlagDef1 = NewMI1.findRegisterDefOperand(X86::EFLAGS); 9254 MachineOperand *NewFlagDef2 = NewMI2.findRegisterDefOperand(X86::EFLAGS); 9255 9256 assert(NewFlagDef1 && NewFlagDef2 && 9257 "Unexpected operand in reassociable instruction"); 9258 9259 // Mark the new EFLAGS operands as dead to be helpful to subsequent iterations 9260 // of this pass or other passes. The EFLAGS operands must be dead in these new 9261 // instructions because the EFLAGS operands in the original instructions must 9262 // be dead in order for reassociation to occur. 9263 NewFlagDef1->setIsDead(); 9264 NewFlagDef2->setIsDead(); 9265 } 9266 9267 std::pair<unsigned, unsigned> 9268 X86InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 9269 return std::make_pair(TF, 0u); 9270 } 9271 9272 ArrayRef<std::pair<unsigned, const char *>> 9273 X86InstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 9274 using namespace X86II; 9275 static const std::pair<unsigned, const char *> TargetFlags[] = { 9276 {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"}, 9277 {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"}, 9278 {MO_GOT, "x86-got"}, 9279 {MO_GOTOFF, "x86-gotoff"}, 9280 {MO_GOTPCREL, "x86-gotpcrel"}, 9281 {MO_GOTPCREL_NORELAX, "x86-gotpcrel-norelax"}, 9282 {MO_PLT, "x86-plt"}, 9283 {MO_TLSGD, "x86-tlsgd"}, 9284 {MO_TLSLD, "x86-tlsld"}, 9285 {MO_TLSLDM, "x86-tlsldm"}, 9286 {MO_GOTTPOFF, "x86-gottpoff"}, 9287 {MO_INDNTPOFF, "x86-indntpoff"}, 9288 {MO_TPOFF, "x86-tpoff"}, 9289 {MO_DTPOFF, "x86-dtpoff"}, 9290 {MO_NTPOFF, "x86-ntpoff"}, 9291 {MO_GOTNTPOFF, "x86-gotntpoff"}, 9292 {MO_DLLIMPORT, "x86-dllimport"}, 9293 {MO_DARWIN_NONLAZY, "x86-darwin-nonlazy"}, 9294 {MO_DARWIN_NONLAZY_PIC_BASE, "x86-darwin-nonlazy-pic-base"}, 9295 {MO_TLVP, "x86-tlvp"}, 9296 {MO_TLVP_PIC_BASE, "x86-tlvp-pic-base"}, 9297 {MO_SECREL, "x86-secrel"}, 9298 {MO_COFFSTUB, "x86-coffstub"}}; 9299 return ArrayRef(TargetFlags); 9300 } 9301 9302 namespace { 9303 /// Create Global Base Reg pass. This initializes the PIC 9304 /// global base register for x86-32. 9305 struct CGBR : public MachineFunctionPass { 9306 static char ID; 9307 CGBR() : MachineFunctionPass(ID) {} 9308 9309 bool runOnMachineFunction(MachineFunction &MF) override { 9310 const X86TargetMachine *TM = 9311 static_cast<const X86TargetMachine *>(&MF.getTarget()); 9312 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 9313 9314 // Don't do anything in the 64-bit small and kernel code models. They use 9315 // RIP-relative addressing for everything. 9316 if (STI.is64Bit() && (TM->getCodeModel() == CodeModel::Small || 9317 TM->getCodeModel() == CodeModel::Kernel)) 9318 return false; 9319 9320 // Only emit a global base reg in PIC mode. 9321 if (!TM->isPositionIndependent()) 9322 return false; 9323 9324 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 9325 Register GlobalBaseReg = X86FI->getGlobalBaseReg(); 9326 9327 // If we didn't need a GlobalBaseReg, don't insert code. 9328 if (GlobalBaseReg == 0) 9329 return false; 9330 9331 // Insert the set of GlobalBaseReg into the first MBB of the function 9332 MachineBasicBlock &FirstMBB = MF.front(); 9333 MachineBasicBlock::iterator MBBI = FirstMBB.begin(); 9334 DebugLoc DL = FirstMBB.findDebugLoc(MBBI); 9335 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 9336 const X86InstrInfo *TII = STI.getInstrInfo(); 9337 9338 Register PC; 9339 if (STI.isPICStyleGOT()) 9340 PC = RegInfo.createVirtualRegister(&X86::GR32RegClass); 9341 else 9342 PC = GlobalBaseReg; 9343 9344 if (STI.is64Bit()) { 9345 if (TM->getCodeModel() == CodeModel::Medium) { 9346 // In the medium code model, use a RIP-relative LEA to materialize the 9347 // GOT. 9348 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC) 9349 .addReg(X86::RIP) 9350 .addImm(0) 9351 .addReg(0) 9352 .addExternalSymbol("_GLOBAL_OFFSET_TABLE_") 9353 .addReg(0); 9354 } else if (TM->getCodeModel() == CodeModel::Large) { 9355 // In the large code model, we are aiming for this code, though the 9356 // register allocation may vary: 9357 // leaq .LN$pb(%rip), %rax 9358 // movq $_GLOBAL_OFFSET_TABLE_ - .LN$pb, %rcx 9359 // addq %rcx, %rax 9360 // RAX now holds address of _GLOBAL_OFFSET_TABLE_. 9361 Register PBReg = RegInfo.createVirtualRegister(&X86::GR64RegClass); 9362 Register GOTReg = RegInfo.createVirtualRegister(&X86::GR64RegClass); 9363 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PBReg) 9364 .addReg(X86::RIP) 9365 .addImm(0) 9366 .addReg(0) 9367 .addSym(MF.getPICBaseSymbol()) 9368 .addReg(0); 9369 std::prev(MBBI)->setPreInstrSymbol(MF, MF.getPICBaseSymbol()); 9370 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOV64ri), GOTReg) 9371 .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", 9372 X86II::MO_PIC_BASE_OFFSET); 9373 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD64rr), PC) 9374 .addReg(PBReg, RegState::Kill) 9375 .addReg(GOTReg, RegState::Kill); 9376 } else { 9377 llvm_unreachable("unexpected code model"); 9378 } 9379 } else { 9380 // Operand of MovePCtoStack is completely ignored by asm printer. It's 9381 // only used in JIT code emission as displacement to pc. 9382 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOVPC32r), PC).addImm(0); 9383 9384 // If we're using vanilla 'GOT' PIC style, we should use relative 9385 // addressing not to pc, but to _GLOBAL_OFFSET_TABLE_ external. 9386 if (STI.isPICStyleGOT()) { 9387 // Generate addl $__GLOBAL_OFFSET_TABLE_ + [.-piclabel], 9388 // %some_register 9389 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD32ri), GlobalBaseReg) 9390 .addReg(PC) 9391 .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", 9392 X86II::MO_GOT_ABSOLUTE_ADDRESS); 9393 } 9394 } 9395 9396 return true; 9397 } 9398 9399 StringRef getPassName() const override { 9400 return "X86 PIC Global Base Reg Initialization"; 9401 } 9402 9403 void getAnalysisUsage(AnalysisUsage &AU) const override { 9404 AU.setPreservesCFG(); 9405 MachineFunctionPass::getAnalysisUsage(AU); 9406 } 9407 }; 9408 } // namespace 9409 9410 char CGBR::ID = 0; 9411 FunctionPass* 9412 llvm::createX86GlobalBaseRegPass() { return new CGBR(); } 9413 9414 namespace { 9415 struct LDTLSCleanup : public MachineFunctionPass { 9416 static char ID; 9417 LDTLSCleanup() : MachineFunctionPass(ID) {} 9418 9419 bool runOnMachineFunction(MachineFunction &MF) override { 9420 if (skipFunction(MF.getFunction())) 9421 return false; 9422 9423 X86MachineFunctionInfo *MFI = MF.getInfo<X86MachineFunctionInfo>(); 9424 if (MFI->getNumLocalDynamicTLSAccesses() < 2) { 9425 // No point folding accesses if there isn't at least two. 9426 return false; 9427 } 9428 9429 MachineDominatorTree *DT = &getAnalysis<MachineDominatorTree>(); 9430 return VisitNode(DT->getRootNode(), 0); 9431 } 9432 9433 // Visit the dominator subtree rooted at Node in pre-order. 9434 // If TLSBaseAddrReg is non-null, then use that to replace any 9435 // TLS_base_addr instructions. Otherwise, create the register 9436 // when the first such instruction is seen, and then use it 9437 // as we encounter more instructions. 9438 bool VisitNode(MachineDomTreeNode *Node, unsigned TLSBaseAddrReg) { 9439 MachineBasicBlock *BB = Node->getBlock(); 9440 bool Changed = false; 9441 9442 // Traverse the current block. 9443 for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; 9444 ++I) { 9445 switch (I->getOpcode()) { 9446 case X86::TLS_base_addr32: 9447 case X86::TLS_base_addr64: 9448 if (TLSBaseAddrReg) 9449 I = ReplaceTLSBaseAddrCall(*I, TLSBaseAddrReg); 9450 else 9451 I = SetRegister(*I, &TLSBaseAddrReg); 9452 Changed = true; 9453 break; 9454 default: 9455 break; 9456 } 9457 } 9458 9459 // Visit the children of this block in the dominator tree. 9460 for (auto &I : *Node) { 9461 Changed |= VisitNode(I, TLSBaseAddrReg); 9462 } 9463 9464 return Changed; 9465 } 9466 9467 // Replace the TLS_base_addr instruction I with a copy from 9468 // TLSBaseAddrReg, returning the new instruction. 9469 MachineInstr *ReplaceTLSBaseAddrCall(MachineInstr &I, 9470 unsigned TLSBaseAddrReg) { 9471 MachineFunction *MF = I.getParent()->getParent(); 9472 const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); 9473 const bool is64Bit = STI.is64Bit(); 9474 const X86InstrInfo *TII = STI.getInstrInfo(); 9475 9476 // Insert a Copy from TLSBaseAddrReg to RAX/EAX. 9477 MachineInstr *Copy = 9478 BuildMI(*I.getParent(), I, I.getDebugLoc(), 9479 TII->get(TargetOpcode::COPY), is64Bit ? X86::RAX : X86::EAX) 9480 .addReg(TLSBaseAddrReg); 9481 9482 // Erase the TLS_base_addr instruction. 9483 I.eraseFromParent(); 9484 9485 return Copy; 9486 } 9487 9488 // Create a virtual register in *TLSBaseAddrReg, and populate it by 9489 // inserting a copy instruction after I. Returns the new instruction. 9490 MachineInstr *SetRegister(MachineInstr &I, unsigned *TLSBaseAddrReg) { 9491 MachineFunction *MF = I.getParent()->getParent(); 9492 const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); 9493 const bool is64Bit = STI.is64Bit(); 9494 const X86InstrInfo *TII = STI.getInstrInfo(); 9495 9496 // Create a virtual register for the TLS base address. 9497 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 9498 *TLSBaseAddrReg = RegInfo.createVirtualRegister(is64Bit 9499 ? &X86::GR64RegClass 9500 : &X86::GR32RegClass); 9501 9502 // Insert a copy from RAX/EAX to TLSBaseAddrReg. 9503 MachineInstr *Next = I.getNextNode(); 9504 MachineInstr *Copy = 9505 BuildMI(*I.getParent(), Next, I.getDebugLoc(), 9506 TII->get(TargetOpcode::COPY), *TLSBaseAddrReg) 9507 .addReg(is64Bit ? X86::RAX : X86::EAX); 9508 9509 return Copy; 9510 } 9511 9512 StringRef getPassName() const override { 9513 return "Local Dynamic TLS Access Clean-up"; 9514 } 9515 9516 void getAnalysisUsage(AnalysisUsage &AU) const override { 9517 AU.setPreservesCFG(); 9518 AU.addRequired<MachineDominatorTree>(); 9519 MachineFunctionPass::getAnalysisUsage(AU); 9520 } 9521 }; 9522 } 9523 9524 char LDTLSCleanup::ID = 0; 9525 FunctionPass* 9526 llvm::createCleanupLocalDynamicTLSPass() { return new LDTLSCleanup(); } 9527 9528 /// Constants defining how certain sequences should be outlined. 9529 /// 9530 /// \p MachineOutlinerDefault implies that the function is called with a call 9531 /// instruction, and a return must be emitted for the outlined function frame. 9532 /// 9533 /// That is, 9534 /// 9535 /// I1 OUTLINED_FUNCTION: 9536 /// I2 --> call OUTLINED_FUNCTION I1 9537 /// I3 I2 9538 /// I3 9539 /// ret 9540 /// 9541 /// * Call construction overhead: 1 (call instruction) 9542 /// * Frame construction overhead: 1 (return instruction) 9543 /// 9544 /// \p MachineOutlinerTailCall implies that the function is being tail called. 9545 /// A jump is emitted instead of a call, and the return is already present in 9546 /// the outlined sequence. That is, 9547 /// 9548 /// I1 OUTLINED_FUNCTION: 9549 /// I2 --> jmp OUTLINED_FUNCTION I1 9550 /// ret I2 9551 /// ret 9552 /// 9553 /// * Call construction overhead: 1 (jump instruction) 9554 /// * Frame construction overhead: 0 (don't need to return) 9555 /// 9556 enum MachineOutlinerClass { 9557 MachineOutlinerDefault, 9558 MachineOutlinerTailCall 9559 }; 9560 9561 outliner::OutlinedFunction X86InstrInfo::getOutliningCandidateInfo( 9562 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { 9563 unsigned SequenceSize = 9564 std::accumulate(RepeatedSequenceLocs[0].front(), 9565 std::next(RepeatedSequenceLocs[0].back()), 0, 9566 [](unsigned Sum, const MachineInstr &MI) { 9567 // FIXME: x86 doesn't implement getInstSizeInBytes, so 9568 // we can't tell the cost. Just assume each instruction 9569 // is one byte. 9570 if (MI.isDebugInstr() || MI.isKill()) 9571 return Sum; 9572 return Sum + 1; 9573 }); 9574 9575 // We check to see if CFI Instructions are present, and if they are 9576 // we find the number of CFI Instructions in the candidates. 9577 unsigned CFICount = 0; 9578 for (auto &I : make_range(RepeatedSequenceLocs[0].front(), 9579 std::next(RepeatedSequenceLocs[0].back()))) { 9580 if (I.isCFIInstruction()) 9581 CFICount++; 9582 } 9583 9584 // We compare the number of found CFI Instructions to the number of CFI 9585 // instructions in the parent function for each candidate. We must check this 9586 // since if we outline one of the CFI instructions in a function, we have to 9587 // outline them all for correctness. If we do not, the address offsets will be 9588 // incorrect between the two sections of the program. 9589 for (outliner::Candidate &C : RepeatedSequenceLocs) { 9590 std::vector<MCCFIInstruction> CFIInstructions = 9591 C.getMF()->getFrameInstructions(); 9592 9593 if (CFICount > 0 && CFICount != CFIInstructions.size()) 9594 return outliner::OutlinedFunction(); 9595 } 9596 9597 // FIXME: Use real size in bytes for call and ret instructions. 9598 if (RepeatedSequenceLocs[0].back()->isTerminator()) { 9599 for (outliner::Candidate &C : RepeatedSequenceLocs) 9600 C.setCallInfo(MachineOutlinerTailCall, 1); 9601 9602 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 9603 0, // Number of bytes to emit frame. 9604 MachineOutlinerTailCall // Type of frame. 9605 ); 9606 } 9607 9608 if (CFICount > 0) 9609 return outliner::OutlinedFunction(); 9610 9611 for (outliner::Candidate &C : RepeatedSequenceLocs) 9612 C.setCallInfo(MachineOutlinerDefault, 1); 9613 9614 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 1, 9615 MachineOutlinerDefault); 9616 } 9617 9618 bool X86InstrInfo::isFunctionSafeToOutlineFrom(MachineFunction &MF, 9619 bool OutlineFromLinkOnceODRs) const { 9620 const Function &F = MF.getFunction(); 9621 9622 // Does the function use a red zone? If it does, then we can't risk messing 9623 // with the stack. 9624 if (Subtarget.getFrameLowering()->has128ByteRedZone(MF)) { 9625 // It could have a red zone. If it does, then we don't want to touch it. 9626 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 9627 if (!X86FI || X86FI->getUsesRedZone()) 9628 return false; 9629 } 9630 9631 // If we *don't* want to outline from things that could potentially be deduped 9632 // then return false. 9633 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) 9634 return false; 9635 9636 // This function is viable for outlining, so return true. 9637 return true; 9638 } 9639 9640 outliner::InstrType 9641 X86InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, unsigned Flags) const { 9642 MachineInstr &MI = *MIT; 9643 // Don't allow debug values to impact outlining type. 9644 if (MI.isDebugInstr() || MI.isIndirectDebugValue()) 9645 return outliner::InstrType::Invisible; 9646 9647 // At this point, KILL instructions don't really tell us much so we can go 9648 // ahead and skip over them. 9649 if (MI.isKill()) 9650 return outliner::InstrType::Invisible; 9651 9652 // Is this a tail call? If yes, we can outline as a tail call. 9653 if (isTailCall(MI)) 9654 return outliner::InstrType::Legal; 9655 9656 // Is this the terminator of a basic block? 9657 if (MI.isTerminator() || MI.isReturn()) { 9658 9659 // Does its parent have any successors in its MachineFunction? 9660 if (MI.getParent()->succ_empty()) 9661 return outliner::InstrType::Legal; 9662 9663 // It does, so we can't tail call it. 9664 return outliner::InstrType::Illegal; 9665 } 9666 9667 // Don't outline anything that modifies or reads from the stack pointer. 9668 // 9669 // FIXME: There are instructions which are being manually built without 9670 // explicit uses/defs so we also have to check the MCInstrDesc. We should be 9671 // able to remove the extra checks once those are fixed up. For example, 9672 // sometimes we might get something like %rax = POP64r 1. This won't be 9673 // caught by modifiesRegister or readsRegister even though the instruction 9674 // really ought to be formed so that modifiesRegister/readsRegister would 9675 // catch it. 9676 if (MI.modifiesRegister(X86::RSP, &RI) || MI.readsRegister(X86::RSP, &RI) || 9677 MI.getDesc().hasImplicitUseOfPhysReg(X86::RSP) || 9678 MI.getDesc().hasImplicitDefOfPhysReg(X86::RSP)) 9679 return outliner::InstrType::Illegal; 9680 9681 // Outlined calls change the instruction pointer, so don't read from it. 9682 if (MI.readsRegister(X86::RIP, &RI) || 9683 MI.getDesc().hasImplicitUseOfPhysReg(X86::RIP) || 9684 MI.getDesc().hasImplicitDefOfPhysReg(X86::RIP)) 9685 return outliner::InstrType::Illegal; 9686 9687 // Positions can't safely be outlined. 9688 if (MI.isPosition()) 9689 return outliner::InstrType::Illegal; 9690 9691 // Make sure none of the operands of this instruction do anything tricky. 9692 for (const MachineOperand &MOP : MI.operands()) 9693 if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() || 9694 MOP.isTargetIndex()) 9695 return outliner::InstrType::Illegal; 9696 9697 return outliner::InstrType::Legal; 9698 } 9699 9700 void X86InstrInfo::buildOutlinedFrame(MachineBasicBlock &MBB, 9701 MachineFunction &MF, 9702 const outliner::OutlinedFunction &OF) 9703 const { 9704 // If we're a tail call, we already have a return, so don't do anything. 9705 if (OF.FrameConstructionID == MachineOutlinerTailCall) 9706 return; 9707 9708 // We're a normal call, so our sequence doesn't have a return instruction. 9709 // Add it in. 9710 MachineInstr *retq = BuildMI(MF, DebugLoc(), get(X86::RET64)); 9711 MBB.insert(MBB.end(), retq); 9712 } 9713 9714 MachineBasicBlock::iterator 9715 X86InstrInfo::insertOutlinedCall(Module &M, MachineBasicBlock &MBB, 9716 MachineBasicBlock::iterator &It, 9717 MachineFunction &MF, 9718 outliner::Candidate &C) const { 9719 // Is it a tail call? 9720 if (C.CallConstructionID == MachineOutlinerTailCall) { 9721 // Yes, just insert a JMP. 9722 It = MBB.insert(It, 9723 BuildMI(MF, DebugLoc(), get(X86::TAILJMPd64)) 9724 .addGlobalAddress(M.getNamedValue(MF.getName()))); 9725 } else { 9726 // No, insert a call. 9727 It = MBB.insert(It, 9728 BuildMI(MF, DebugLoc(), get(X86::CALL64pcrel32)) 9729 .addGlobalAddress(M.getNamedValue(MF.getName()))); 9730 } 9731 9732 return It; 9733 } 9734 9735 #define GET_INSTRINFO_HELPERS 9736 #include "X86GenInstrInfo.inc" 9737