1 //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the X86 implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "X86InstrInfo.h" 14 #include "X86.h" 15 #include "X86InstrBuilder.h" 16 #include "X86InstrFoldTables.h" 17 #include "X86MachineFunctionInfo.h" 18 #include "X86Subtarget.h" 19 #include "X86TargetMachine.h" 20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/ADT/Sequence.h" 22 #include "llvm/CodeGen/LiveIntervals.h" 23 #include "llvm/CodeGen/LivePhysRegs.h" 24 #include "llvm/CodeGen/LiveVariables.h" 25 #include "llvm/CodeGen/MachineConstantPool.h" 26 #include "llvm/CodeGen/MachineDominators.h" 27 #include "llvm/CodeGen/MachineFrameInfo.h" 28 #include "llvm/CodeGen/MachineInstr.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfo.h" 31 #include "llvm/CodeGen/MachineOperand.h" 32 #include "llvm/CodeGen/MachineRegisterInfo.h" 33 #include "llvm/CodeGen/StackMaps.h" 34 #include "llvm/IR/DebugInfoMetadata.h" 35 #include "llvm/IR/DerivedTypes.h" 36 #include "llvm/IR/Function.h" 37 #include "llvm/IR/InstrTypes.h" 38 #include "llvm/MC/MCAsmInfo.h" 39 #include "llvm/MC/MCExpr.h" 40 #include "llvm/MC/MCInst.h" 41 #include "llvm/Support/CommandLine.h" 42 #include "llvm/Support/Debug.h" 43 #include "llvm/Support/ErrorHandling.h" 44 #include "llvm/Support/raw_ostream.h" 45 #include "llvm/Target/TargetOptions.h" 46 47 using namespace llvm; 48 49 #define DEBUG_TYPE "x86-instr-info" 50 51 #define GET_INSTRINFO_CTOR_DTOR 52 #include "X86GenInstrInfo.inc" 53 54 static cl::opt<bool> 55 NoFusing("disable-spill-fusing", 56 cl::desc("Disable fusing of spill code into instructions"), 57 cl::Hidden); 58 static cl::opt<bool> 59 PrintFailedFusing("print-failed-fuse-candidates", 60 cl::desc("Print instructions that the allocator wants to" 61 " fuse, but the X86 backend currently can't"), 62 cl::Hidden); 63 static cl::opt<bool> 64 ReMatPICStubLoad("remat-pic-stub-load", 65 cl::desc("Re-materialize load from stub in PIC mode"), 66 cl::init(false), cl::Hidden); 67 static cl::opt<unsigned> 68 PartialRegUpdateClearance("partial-reg-update-clearance", 69 cl::desc("Clearance between two register writes " 70 "for inserting XOR to avoid partial " 71 "register update"), 72 cl::init(64), cl::Hidden); 73 static cl::opt<unsigned> 74 UndefRegClearance("undef-reg-clearance", 75 cl::desc("How many idle instructions we would like before " 76 "certain undef register reads"), 77 cl::init(128), cl::Hidden); 78 79 80 // Pin the vtable to this file. 81 void X86InstrInfo::anchor() {} 82 83 X86InstrInfo::X86InstrInfo(X86Subtarget &STI) 84 : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64 85 : X86::ADJCALLSTACKDOWN32), 86 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64 87 : X86::ADJCALLSTACKUP32), 88 X86::CATCHRET, 89 (STI.is64Bit() ? X86::RET64 : X86::RET32)), 90 Subtarget(STI), RI(STI.getTargetTriple()) { 91 } 92 93 bool 94 X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI, 95 Register &SrcReg, Register &DstReg, 96 unsigned &SubIdx) const { 97 switch (MI.getOpcode()) { 98 default: break; 99 case X86::MOVSX16rr8: 100 case X86::MOVZX16rr8: 101 case X86::MOVSX32rr8: 102 case X86::MOVZX32rr8: 103 case X86::MOVSX64rr8: 104 if (!Subtarget.is64Bit()) 105 // It's not always legal to reference the low 8-bit of the larger 106 // register in 32-bit mode. 107 return false; 108 LLVM_FALLTHROUGH; 109 case X86::MOVSX32rr16: 110 case X86::MOVZX32rr16: 111 case X86::MOVSX64rr16: 112 case X86::MOVSX64rr32: { 113 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) 114 // Be conservative. 115 return false; 116 SrcReg = MI.getOperand(1).getReg(); 117 DstReg = MI.getOperand(0).getReg(); 118 switch (MI.getOpcode()) { 119 default: llvm_unreachable("Unreachable!"); 120 case X86::MOVSX16rr8: 121 case X86::MOVZX16rr8: 122 case X86::MOVSX32rr8: 123 case X86::MOVZX32rr8: 124 case X86::MOVSX64rr8: 125 SubIdx = X86::sub_8bit; 126 break; 127 case X86::MOVSX32rr16: 128 case X86::MOVZX32rr16: 129 case X86::MOVSX64rr16: 130 SubIdx = X86::sub_16bit; 131 break; 132 case X86::MOVSX64rr32: 133 SubIdx = X86::sub_32bit; 134 break; 135 } 136 return true; 137 } 138 } 139 return false; 140 } 141 142 bool X86InstrInfo::isDataInvariant(MachineInstr &MI) { 143 if (MI.mayLoad() || MI.mayStore()) 144 return false; 145 146 // Some target-independent operations that trivially lower to data-invariant 147 // instructions. 148 if (MI.isCopyLike() || MI.isInsertSubreg()) 149 return true; 150 151 unsigned Opcode = MI.getOpcode(); 152 using namespace X86; 153 // On x86 it is believed that imul is constant time w.r.t. the loaded data. 154 // However, they set flags and are perhaps the most surprisingly constant 155 // time operations so we call them out here separately. 156 if (isIMUL(Opcode)) 157 return true; 158 // Bit scanning and counting instructions that are somewhat surprisingly 159 // constant time as they scan across bits and do other fairly complex 160 // operations like popcnt, but are believed to be constant time on x86. 161 // However, these set flags. 162 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) || 163 isTZCNT(Opcode)) 164 return true; 165 // Bit manipulation instructions are effectively combinations of basic 166 // arithmetic ops, and should still execute in constant time. These also 167 // set flags. 168 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) || 169 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) || 170 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) || 171 isTZMSK(Opcode)) 172 return true; 173 // Bit extracting and clearing instructions should execute in constant time, 174 // and set flags. 175 if (isBEXTR(Opcode) || isBZHI(Opcode)) 176 return true; 177 // Shift and rotate. 178 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) || 179 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode)) 180 return true; 181 // Basic arithmetic is constant time on the input but does set flags. 182 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) || 183 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode)) 184 return true; 185 // Arithmetic with just 32-bit and 64-bit variants and no immediates. 186 if (isADCX(Opcode) || isADOX(Opcode) || isANDN(Opcode)) 187 return true; 188 // Unary arithmetic operations. 189 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode)) 190 return true; 191 // Unlike other arithmetic, NOT doesn't set EFLAGS. 192 if (isNOT(Opcode)) 193 return true; 194 // Various move instructions used to zero or sign extend things. Note that we 195 // intentionally don't support the _NOREX variants as we can't handle that 196 // register constraint anyways. 197 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode)) 198 return true; 199 // Arithmetic instructions that are both constant time and don't set flags. 200 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode)) 201 return true; 202 // LEA doesn't actually access memory, and its arithmetic is constant time. 203 if (isLEA(Opcode)) 204 return true; 205 // By default, assume that the instruction is not data invariant. 206 return false; 207 } 208 209 bool X86InstrInfo::isDataInvariantLoad(MachineInstr &MI) { 210 switch (MI.getOpcode()) { 211 default: 212 // By default, assume that the load will immediately leak. 213 return false; 214 215 // On x86 it is believed that imul is constant time w.r.t. the loaded data. 216 // However, they set flags and are perhaps the most surprisingly constant 217 // time operations so we call them out here separately. 218 case X86::IMUL16rm: 219 case X86::IMUL16rmi8: 220 case X86::IMUL16rmi: 221 case X86::IMUL32rm: 222 case X86::IMUL32rmi8: 223 case X86::IMUL32rmi: 224 case X86::IMUL64rm: 225 case X86::IMUL64rmi32: 226 case X86::IMUL64rmi8: 227 228 // Bit scanning and counting instructions that are somewhat surprisingly 229 // constant time as they scan across bits and do other fairly complex 230 // operations like popcnt, but are believed to be constant time on x86. 231 // However, these set flags. 232 case X86::BSF16rm: 233 case X86::BSF32rm: 234 case X86::BSF64rm: 235 case X86::BSR16rm: 236 case X86::BSR32rm: 237 case X86::BSR64rm: 238 case X86::LZCNT16rm: 239 case X86::LZCNT32rm: 240 case X86::LZCNT64rm: 241 case X86::POPCNT16rm: 242 case X86::POPCNT32rm: 243 case X86::POPCNT64rm: 244 case X86::TZCNT16rm: 245 case X86::TZCNT32rm: 246 case X86::TZCNT64rm: 247 248 // Bit manipulation instructions are effectively combinations of basic 249 // arithmetic ops, and should still execute in constant time. These also 250 // set flags. 251 case X86::BLCFILL32rm: 252 case X86::BLCFILL64rm: 253 case X86::BLCI32rm: 254 case X86::BLCI64rm: 255 case X86::BLCIC32rm: 256 case X86::BLCIC64rm: 257 case X86::BLCMSK32rm: 258 case X86::BLCMSK64rm: 259 case X86::BLCS32rm: 260 case X86::BLCS64rm: 261 case X86::BLSFILL32rm: 262 case X86::BLSFILL64rm: 263 case X86::BLSI32rm: 264 case X86::BLSI64rm: 265 case X86::BLSIC32rm: 266 case X86::BLSIC64rm: 267 case X86::BLSMSK32rm: 268 case X86::BLSMSK64rm: 269 case X86::BLSR32rm: 270 case X86::BLSR64rm: 271 case X86::TZMSK32rm: 272 case X86::TZMSK64rm: 273 274 // Bit extracting and clearing instructions should execute in constant time, 275 // and set flags. 276 case X86::BEXTR32rm: 277 case X86::BEXTR64rm: 278 case X86::BEXTRI32mi: 279 case X86::BEXTRI64mi: 280 case X86::BZHI32rm: 281 case X86::BZHI64rm: 282 283 // Basic arithmetic is constant time on the input but does set flags. 284 case X86::ADC8rm: 285 case X86::ADC16rm: 286 case X86::ADC32rm: 287 case X86::ADC64rm: 288 case X86::ADCX32rm: 289 case X86::ADCX64rm: 290 case X86::ADD8rm: 291 case X86::ADD16rm: 292 case X86::ADD32rm: 293 case X86::ADD64rm: 294 case X86::ADOX32rm: 295 case X86::ADOX64rm: 296 case X86::AND8rm: 297 case X86::AND16rm: 298 case X86::AND32rm: 299 case X86::AND64rm: 300 case X86::ANDN32rm: 301 case X86::ANDN64rm: 302 case X86::OR8rm: 303 case X86::OR16rm: 304 case X86::OR32rm: 305 case X86::OR64rm: 306 case X86::SBB8rm: 307 case X86::SBB16rm: 308 case X86::SBB32rm: 309 case X86::SBB64rm: 310 case X86::SUB8rm: 311 case X86::SUB16rm: 312 case X86::SUB32rm: 313 case X86::SUB64rm: 314 case X86::XOR8rm: 315 case X86::XOR16rm: 316 case X86::XOR32rm: 317 case X86::XOR64rm: 318 319 // Integer multiply w/o affecting flags is still believed to be constant 320 // time on x86. Called out separately as this is among the most surprising 321 // instructions to exhibit that behavior. 322 case X86::MULX32rm: 323 case X86::MULX64rm: 324 325 // Arithmetic instructions that are both constant time and don't set flags. 326 case X86::RORX32mi: 327 case X86::RORX64mi: 328 case X86::SARX32rm: 329 case X86::SARX64rm: 330 case X86::SHLX32rm: 331 case X86::SHLX64rm: 332 case X86::SHRX32rm: 333 case X86::SHRX64rm: 334 335 // Conversions are believed to be constant time and don't set flags. 336 case X86::CVTTSD2SI64rm: 337 case X86::VCVTTSD2SI64rm: 338 case X86::VCVTTSD2SI64Zrm: 339 case X86::CVTTSD2SIrm: 340 case X86::VCVTTSD2SIrm: 341 case X86::VCVTTSD2SIZrm: 342 case X86::CVTTSS2SI64rm: 343 case X86::VCVTTSS2SI64rm: 344 case X86::VCVTTSS2SI64Zrm: 345 case X86::CVTTSS2SIrm: 346 case X86::VCVTTSS2SIrm: 347 case X86::VCVTTSS2SIZrm: 348 case X86::CVTSI2SDrm: 349 case X86::VCVTSI2SDrm: 350 case X86::VCVTSI2SDZrm: 351 case X86::CVTSI2SSrm: 352 case X86::VCVTSI2SSrm: 353 case X86::VCVTSI2SSZrm: 354 case X86::CVTSI642SDrm: 355 case X86::VCVTSI642SDrm: 356 case X86::VCVTSI642SDZrm: 357 case X86::CVTSI642SSrm: 358 case X86::VCVTSI642SSrm: 359 case X86::VCVTSI642SSZrm: 360 case X86::CVTSS2SDrm: 361 case X86::VCVTSS2SDrm: 362 case X86::VCVTSS2SDZrm: 363 case X86::CVTSD2SSrm: 364 case X86::VCVTSD2SSrm: 365 case X86::VCVTSD2SSZrm: 366 // AVX512 added unsigned integer conversions. 367 case X86::VCVTTSD2USI64Zrm: 368 case X86::VCVTTSD2USIZrm: 369 case X86::VCVTTSS2USI64Zrm: 370 case X86::VCVTTSS2USIZrm: 371 case X86::VCVTUSI2SDZrm: 372 case X86::VCVTUSI642SDZrm: 373 case X86::VCVTUSI2SSZrm: 374 case X86::VCVTUSI642SSZrm: 375 376 // Loads to register don't set flags. 377 case X86::MOV8rm: 378 case X86::MOV8rm_NOREX: 379 case X86::MOV16rm: 380 case X86::MOV32rm: 381 case X86::MOV64rm: 382 case X86::MOVSX16rm8: 383 case X86::MOVSX32rm16: 384 case X86::MOVSX32rm8: 385 case X86::MOVSX32rm8_NOREX: 386 case X86::MOVSX64rm16: 387 case X86::MOVSX64rm32: 388 case X86::MOVSX64rm8: 389 case X86::MOVZX16rm8: 390 case X86::MOVZX32rm16: 391 case X86::MOVZX32rm8: 392 case X86::MOVZX32rm8_NOREX: 393 case X86::MOVZX64rm16: 394 case X86::MOVZX64rm8: 395 return true; 396 } 397 } 398 399 int X86InstrInfo::getSPAdjust(const MachineInstr &MI) const { 400 const MachineFunction *MF = MI.getParent()->getParent(); 401 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); 402 403 if (isFrameInstr(MI)) { 404 int SPAdj = alignTo(getFrameSize(MI), TFI->getStackAlign()); 405 SPAdj -= getFrameAdjustment(MI); 406 if (!isFrameSetup(MI)) 407 SPAdj = -SPAdj; 408 return SPAdj; 409 } 410 411 // To know whether a call adjusts the stack, we need information 412 // that is bound to the following ADJCALLSTACKUP pseudo. 413 // Look for the next ADJCALLSTACKUP that follows the call. 414 if (MI.isCall()) { 415 const MachineBasicBlock *MBB = MI.getParent(); 416 auto I = ++MachineBasicBlock::const_iterator(MI); 417 for (auto E = MBB->end(); I != E; ++I) { 418 if (I->getOpcode() == getCallFrameDestroyOpcode() || 419 I->isCall()) 420 break; 421 } 422 423 // If we could not find a frame destroy opcode, then it has already 424 // been simplified, so we don't care. 425 if (I->getOpcode() != getCallFrameDestroyOpcode()) 426 return 0; 427 428 return -(I->getOperand(1).getImm()); 429 } 430 431 // Currently handle only PUSHes we can reasonably expect to see 432 // in call sequences 433 switch (MI.getOpcode()) { 434 default: 435 return 0; 436 case X86::PUSH32i8: 437 case X86::PUSH32r: 438 case X86::PUSH32rmm: 439 case X86::PUSH32rmr: 440 case X86::PUSHi32: 441 return 4; 442 case X86::PUSH64i8: 443 case X86::PUSH64r: 444 case X86::PUSH64rmm: 445 case X86::PUSH64rmr: 446 case X86::PUSH64i32: 447 return 8; 448 } 449 } 450 451 /// Return true and the FrameIndex if the specified 452 /// operand and follow operands form a reference to the stack frame. 453 bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op, 454 int &FrameIndex) const { 455 if (MI.getOperand(Op + X86::AddrBaseReg).isFI() && 456 MI.getOperand(Op + X86::AddrScaleAmt).isImm() && 457 MI.getOperand(Op + X86::AddrIndexReg).isReg() && 458 MI.getOperand(Op + X86::AddrDisp).isImm() && 459 MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 && 460 MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 && 461 MI.getOperand(Op + X86::AddrDisp).getImm() == 0) { 462 FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex(); 463 return true; 464 } 465 return false; 466 } 467 468 static bool isFrameLoadOpcode(int Opcode, unsigned &MemBytes) { 469 switch (Opcode) { 470 default: 471 return false; 472 case X86::MOV8rm: 473 case X86::KMOVBkm: 474 MemBytes = 1; 475 return true; 476 case X86::MOV16rm: 477 case X86::KMOVWkm: 478 case X86::VMOVSHZrm: 479 case X86::VMOVSHZrm_alt: 480 MemBytes = 2; 481 return true; 482 case X86::MOV32rm: 483 case X86::MOVSSrm: 484 case X86::MOVSSrm_alt: 485 case X86::VMOVSSrm: 486 case X86::VMOVSSrm_alt: 487 case X86::VMOVSSZrm: 488 case X86::VMOVSSZrm_alt: 489 case X86::KMOVDkm: 490 MemBytes = 4; 491 return true; 492 case X86::MOV64rm: 493 case X86::LD_Fp64m: 494 case X86::MOVSDrm: 495 case X86::MOVSDrm_alt: 496 case X86::VMOVSDrm: 497 case X86::VMOVSDrm_alt: 498 case X86::VMOVSDZrm: 499 case X86::VMOVSDZrm_alt: 500 case X86::MMX_MOVD64rm: 501 case X86::MMX_MOVQ64rm: 502 case X86::KMOVQkm: 503 MemBytes = 8; 504 return true; 505 case X86::MOVAPSrm: 506 case X86::MOVUPSrm: 507 case X86::MOVAPDrm: 508 case X86::MOVUPDrm: 509 case X86::MOVDQArm: 510 case X86::MOVDQUrm: 511 case X86::VMOVAPSrm: 512 case X86::VMOVUPSrm: 513 case X86::VMOVAPDrm: 514 case X86::VMOVUPDrm: 515 case X86::VMOVDQArm: 516 case X86::VMOVDQUrm: 517 case X86::VMOVAPSZ128rm: 518 case X86::VMOVUPSZ128rm: 519 case X86::VMOVAPSZ128rm_NOVLX: 520 case X86::VMOVUPSZ128rm_NOVLX: 521 case X86::VMOVAPDZ128rm: 522 case X86::VMOVUPDZ128rm: 523 case X86::VMOVDQU8Z128rm: 524 case X86::VMOVDQU16Z128rm: 525 case X86::VMOVDQA32Z128rm: 526 case X86::VMOVDQU32Z128rm: 527 case X86::VMOVDQA64Z128rm: 528 case X86::VMOVDQU64Z128rm: 529 MemBytes = 16; 530 return true; 531 case X86::VMOVAPSYrm: 532 case X86::VMOVUPSYrm: 533 case X86::VMOVAPDYrm: 534 case X86::VMOVUPDYrm: 535 case X86::VMOVDQAYrm: 536 case X86::VMOVDQUYrm: 537 case X86::VMOVAPSZ256rm: 538 case X86::VMOVUPSZ256rm: 539 case X86::VMOVAPSZ256rm_NOVLX: 540 case X86::VMOVUPSZ256rm_NOVLX: 541 case X86::VMOVAPDZ256rm: 542 case X86::VMOVUPDZ256rm: 543 case X86::VMOVDQU8Z256rm: 544 case X86::VMOVDQU16Z256rm: 545 case X86::VMOVDQA32Z256rm: 546 case X86::VMOVDQU32Z256rm: 547 case X86::VMOVDQA64Z256rm: 548 case X86::VMOVDQU64Z256rm: 549 MemBytes = 32; 550 return true; 551 case X86::VMOVAPSZrm: 552 case X86::VMOVUPSZrm: 553 case X86::VMOVAPDZrm: 554 case X86::VMOVUPDZrm: 555 case X86::VMOVDQU8Zrm: 556 case X86::VMOVDQU16Zrm: 557 case X86::VMOVDQA32Zrm: 558 case X86::VMOVDQU32Zrm: 559 case X86::VMOVDQA64Zrm: 560 case X86::VMOVDQU64Zrm: 561 MemBytes = 64; 562 return true; 563 } 564 } 565 566 static bool isFrameStoreOpcode(int Opcode, unsigned &MemBytes) { 567 switch (Opcode) { 568 default: 569 return false; 570 case X86::MOV8mr: 571 case X86::KMOVBmk: 572 MemBytes = 1; 573 return true; 574 case X86::MOV16mr: 575 case X86::KMOVWmk: 576 case X86::VMOVSHZmr: 577 MemBytes = 2; 578 return true; 579 case X86::MOV32mr: 580 case X86::MOVSSmr: 581 case X86::VMOVSSmr: 582 case X86::VMOVSSZmr: 583 case X86::KMOVDmk: 584 MemBytes = 4; 585 return true; 586 case X86::MOV64mr: 587 case X86::ST_FpP64m: 588 case X86::MOVSDmr: 589 case X86::VMOVSDmr: 590 case X86::VMOVSDZmr: 591 case X86::MMX_MOVD64mr: 592 case X86::MMX_MOVQ64mr: 593 case X86::MMX_MOVNTQmr: 594 case X86::KMOVQmk: 595 MemBytes = 8; 596 return true; 597 case X86::MOVAPSmr: 598 case X86::MOVUPSmr: 599 case X86::MOVAPDmr: 600 case X86::MOVUPDmr: 601 case X86::MOVDQAmr: 602 case X86::MOVDQUmr: 603 case X86::VMOVAPSmr: 604 case X86::VMOVUPSmr: 605 case X86::VMOVAPDmr: 606 case X86::VMOVUPDmr: 607 case X86::VMOVDQAmr: 608 case X86::VMOVDQUmr: 609 case X86::VMOVUPSZ128mr: 610 case X86::VMOVAPSZ128mr: 611 case X86::VMOVUPSZ128mr_NOVLX: 612 case X86::VMOVAPSZ128mr_NOVLX: 613 case X86::VMOVUPDZ128mr: 614 case X86::VMOVAPDZ128mr: 615 case X86::VMOVDQA32Z128mr: 616 case X86::VMOVDQU32Z128mr: 617 case X86::VMOVDQA64Z128mr: 618 case X86::VMOVDQU64Z128mr: 619 case X86::VMOVDQU8Z128mr: 620 case X86::VMOVDQU16Z128mr: 621 MemBytes = 16; 622 return true; 623 case X86::VMOVUPSYmr: 624 case X86::VMOVAPSYmr: 625 case X86::VMOVUPDYmr: 626 case X86::VMOVAPDYmr: 627 case X86::VMOVDQUYmr: 628 case X86::VMOVDQAYmr: 629 case X86::VMOVUPSZ256mr: 630 case X86::VMOVAPSZ256mr: 631 case X86::VMOVUPSZ256mr_NOVLX: 632 case X86::VMOVAPSZ256mr_NOVLX: 633 case X86::VMOVUPDZ256mr: 634 case X86::VMOVAPDZ256mr: 635 case X86::VMOVDQU8Z256mr: 636 case X86::VMOVDQU16Z256mr: 637 case X86::VMOVDQA32Z256mr: 638 case X86::VMOVDQU32Z256mr: 639 case X86::VMOVDQA64Z256mr: 640 case X86::VMOVDQU64Z256mr: 641 MemBytes = 32; 642 return true; 643 case X86::VMOVUPSZmr: 644 case X86::VMOVAPSZmr: 645 case X86::VMOVUPDZmr: 646 case X86::VMOVAPDZmr: 647 case X86::VMOVDQU8Zmr: 648 case X86::VMOVDQU16Zmr: 649 case X86::VMOVDQA32Zmr: 650 case X86::VMOVDQU32Zmr: 651 case X86::VMOVDQA64Zmr: 652 case X86::VMOVDQU64Zmr: 653 MemBytes = 64; 654 return true; 655 } 656 return false; 657 } 658 659 unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 660 int &FrameIndex) const { 661 unsigned Dummy; 662 return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy); 663 } 664 665 unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 666 int &FrameIndex, 667 unsigned &MemBytes) const { 668 if (isFrameLoadOpcode(MI.getOpcode(), MemBytes)) 669 if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex)) 670 return MI.getOperand(0).getReg(); 671 return 0; 672 } 673 674 unsigned X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI, 675 int &FrameIndex) const { 676 unsigned Dummy; 677 if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) { 678 unsigned Reg; 679 if ((Reg = isLoadFromStackSlot(MI, FrameIndex))) 680 return Reg; 681 // Check for post-frame index elimination operations 682 SmallVector<const MachineMemOperand *, 1> Accesses; 683 if (hasLoadFromStackSlot(MI, Accesses)) { 684 FrameIndex = 685 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) 686 ->getFrameIndex(); 687 return MI.getOperand(0).getReg(); 688 } 689 } 690 return 0; 691 } 692 693 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, 694 int &FrameIndex) const { 695 unsigned Dummy; 696 return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy); 697 } 698 699 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, 700 int &FrameIndex, 701 unsigned &MemBytes) const { 702 if (isFrameStoreOpcode(MI.getOpcode(), MemBytes)) 703 if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 && 704 isFrameOperand(MI, 0, FrameIndex)) 705 return MI.getOperand(X86::AddrNumOperands).getReg(); 706 return 0; 707 } 708 709 unsigned X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI, 710 int &FrameIndex) const { 711 unsigned Dummy; 712 if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) { 713 unsigned Reg; 714 if ((Reg = isStoreToStackSlot(MI, FrameIndex))) 715 return Reg; 716 // Check for post-frame index elimination operations 717 SmallVector<const MachineMemOperand *, 1> Accesses; 718 if (hasStoreToStackSlot(MI, Accesses)) { 719 FrameIndex = 720 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) 721 ->getFrameIndex(); 722 return MI.getOperand(X86::AddrNumOperands).getReg(); 723 } 724 } 725 return 0; 726 } 727 728 /// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r. 729 static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) { 730 // Don't waste compile time scanning use-def chains of physregs. 731 if (!BaseReg.isVirtual()) 732 return false; 733 bool isPICBase = false; 734 for (MachineRegisterInfo::def_instr_iterator I = MRI.def_instr_begin(BaseReg), 735 E = MRI.def_instr_end(); I != E; ++I) { 736 MachineInstr *DefMI = &*I; 737 if (DefMI->getOpcode() != X86::MOVPC32r) 738 return false; 739 assert(!isPICBase && "More than one PIC base?"); 740 isPICBase = true; 741 } 742 return isPICBase; 743 } 744 745 bool X86InstrInfo::isReallyTriviallyReMaterializable( 746 const MachineInstr &MI) const { 747 switch (MI.getOpcode()) { 748 default: 749 // This function should only be called for opcodes with the ReMaterializable 750 // flag set. 751 llvm_unreachable("Unknown rematerializable operation!"); 752 break; 753 754 case X86::LOAD_STACK_GUARD: 755 case X86::AVX1_SETALLONES: 756 case X86::AVX2_SETALLONES: 757 case X86::AVX512_128_SET0: 758 case X86::AVX512_256_SET0: 759 case X86::AVX512_512_SET0: 760 case X86::AVX512_512_SETALLONES: 761 case X86::AVX512_FsFLD0SD: 762 case X86::AVX512_FsFLD0SH: 763 case X86::AVX512_FsFLD0SS: 764 case X86::AVX512_FsFLD0F128: 765 case X86::AVX_SET0: 766 case X86::FsFLD0SD: 767 case X86::FsFLD0SS: 768 case X86::FsFLD0SH: 769 case X86::FsFLD0F128: 770 case X86::KSET0D: 771 case X86::KSET0Q: 772 case X86::KSET0W: 773 case X86::KSET1D: 774 case X86::KSET1Q: 775 case X86::KSET1W: 776 case X86::MMX_SET0: 777 case X86::MOV32ImmSExti8: 778 case X86::MOV32r0: 779 case X86::MOV32r1: 780 case X86::MOV32r_1: 781 case X86::MOV32ri64: 782 case X86::MOV64ImmSExti8: 783 case X86::V_SET0: 784 case X86::V_SETALLONES: 785 case X86::MOV16ri: 786 case X86::MOV32ri: 787 case X86::MOV64ri: 788 case X86::MOV64ri32: 789 case X86::MOV8ri: 790 case X86::PTILEZEROV: 791 return true; 792 793 case X86::MOV8rm: 794 case X86::MOV8rm_NOREX: 795 case X86::MOV16rm: 796 case X86::MOV32rm: 797 case X86::MOV64rm: 798 case X86::MOVSSrm: 799 case X86::MOVSSrm_alt: 800 case X86::MOVSDrm: 801 case X86::MOVSDrm_alt: 802 case X86::MOVAPSrm: 803 case X86::MOVUPSrm: 804 case X86::MOVAPDrm: 805 case X86::MOVUPDrm: 806 case X86::MOVDQArm: 807 case X86::MOVDQUrm: 808 case X86::VMOVSSrm: 809 case X86::VMOVSSrm_alt: 810 case X86::VMOVSDrm: 811 case X86::VMOVSDrm_alt: 812 case X86::VMOVAPSrm: 813 case X86::VMOVUPSrm: 814 case X86::VMOVAPDrm: 815 case X86::VMOVUPDrm: 816 case X86::VMOVDQArm: 817 case X86::VMOVDQUrm: 818 case X86::VMOVAPSYrm: 819 case X86::VMOVUPSYrm: 820 case X86::VMOVAPDYrm: 821 case X86::VMOVUPDYrm: 822 case X86::VMOVDQAYrm: 823 case X86::VMOVDQUYrm: 824 case X86::MMX_MOVD64rm: 825 case X86::MMX_MOVQ64rm: 826 // AVX-512 827 case X86::VMOVSSZrm: 828 case X86::VMOVSSZrm_alt: 829 case X86::VMOVSDZrm: 830 case X86::VMOVSDZrm_alt: 831 case X86::VMOVSHZrm: 832 case X86::VMOVSHZrm_alt: 833 case X86::VMOVAPDZ128rm: 834 case X86::VMOVAPDZ256rm: 835 case X86::VMOVAPDZrm: 836 case X86::VMOVAPSZ128rm: 837 case X86::VMOVAPSZ256rm: 838 case X86::VMOVAPSZ128rm_NOVLX: 839 case X86::VMOVAPSZ256rm_NOVLX: 840 case X86::VMOVAPSZrm: 841 case X86::VMOVDQA32Z128rm: 842 case X86::VMOVDQA32Z256rm: 843 case X86::VMOVDQA32Zrm: 844 case X86::VMOVDQA64Z128rm: 845 case X86::VMOVDQA64Z256rm: 846 case X86::VMOVDQA64Zrm: 847 case X86::VMOVDQU16Z128rm: 848 case X86::VMOVDQU16Z256rm: 849 case X86::VMOVDQU16Zrm: 850 case X86::VMOVDQU32Z128rm: 851 case X86::VMOVDQU32Z256rm: 852 case X86::VMOVDQU32Zrm: 853 case X86::VMOVDQU64Z128rm: 854 case X86::VMOVDQU64Z256rm: 855 case X86::VMOVDQU64Zrm: 856 case X86::VMOVDQU8Z128rm: 857 case X86::VMOVDQU8Z256rm: 858 case X86::VMOVDQU8Zrm: 859 case X86::VMOVUPDZ128rm: 860 case X86::VMOVUPDZ256rm: 861 case X86::VMOVUPDZrm: 862 case X86::VMOVUPSZ128rm: 863 case X86::VMOVUPSZ256rm: 864 case X86::VMOVUPSZ128rm_NOVLX: 865 case X86::VMOVUPSZ256rm_NOVLX: 866 case X86::VMOVUPSZrm: { 867 // Loads from constant pools are trivially rematerializable. 868 if (MI.getOperand(1 + X86::AddrBaseReg).isReg() && 869 MI.getOperand(1 + X86::AddrScaleAmt).isImm() && 870 MI.getOperand(1 + X86::AddrIndexReg).isReg() && 871 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && 872 MI.isDereferenceableInvariantLoad()) { 873 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); 874 if (BaseReg == 0 || BaseReg == X86::RIP) 875 return true; 876 // Allow re-materialization of PIC load. 877 if (!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal()) 878 return false; 879 const MachineFunction &MF = *MI.getParent()->getParent(); 880 const MachineRegisterInfo &MRI = MF.getRegInfo(); 881 return regIsPICBase(BaseReg, MRI); 882 } 883 return false; 884 } 885 886 case X86::LEA32r: 887 case X86::LEA64r: { 888 if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() && 889 MI.getOperand(1 + X86::AddrIndexReg).isReg() && 890 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && 891 !MI.getOperand(1 + X86::AddrDisp).isReg()) { 892 // lea fi#, lea GV, etc. are all rematerializable. 893 if (!MI.getOperand(1 + X86::AddrBaseReg).isReg()) 894 return true; 895 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); 896 if (BaseReg == 0) 897 return true; 898 // Allow re-materialization of lea PICBase + x. 899 const MachineFunction &MF = *MI.getParent()->getParent(); 900 const MachineRegisterInfo &MRI = MF.getRegInfo(); 901 return regIsPICBase(BaseReg, MRI); 902 } 903 return false; 904 } 905 } 906 } 907 908 void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB, 909 MachineBasicBlock::iterator I, 910 Register DestReg, unsigned SubIdx, 911 const MachineInstr &Orig, 912 const TargetRegisterInfo &TRI) const { 913 bool ClobbersEFLAGS = Orig.modifiesRegister(X86::EFLAGS, &TRI); 914 if (ClobbersEFLAGS && MBB.computeRegisterLiveness(&TRI, X86::EFLAGS, I) != 915 MachineBasicBlock::LQR_Dead) { 916 // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side 917 // effects. 918 int Value; 919 switch (Orig.getOpcode()) { 920 case X86::MOV32r0: Value = 0; break; 921 case X86::MOV32r1: Value = 1; break; 922 case X86::MOV32r_1: Value = -1; break; 923 default: 924 llvm_unreachable("Unexpected instruction!"); 925 } 926 927 const DebugLoc &DL = Orig.getDebugLoc(); 928 BuildMI(MBB, I, DL, get(X86::MOV32ri)) 929 .add(Orig.getOperand(0)) 930 .addImm(Value); 931 } else { 932 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); 933 MBB.insert(I, MI); 934 } 935 936 MachineInstr &NewMI = *std::prev(I); 937 NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI); 938 } 939 940 /// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead. 941 bool X86InstrInfo::hasLiveCondCodeDef(MachineInstr &MI) const { 942 for (const MachineOperand &MO : MI.operands()) { 943 if (MO.isReg() && MO.isDef() && 944 MO.getReg() == X86::EFLAGS && !MO.isDead()) { 945 return true; 946 } 947 } 948 return false; 949 } 950 951 /// Check whether the shift count for a machine operand is non-zero. 952 inline static unsigned getTruncatedShiftCount(const MachineInstr &MI, 953 unsigned ShiftAmtOperandIdx) { 954 // The shift count is six bits with the REX.W prefix and five bits without. 955 unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31; 956 unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm(); 957 return Imm & ShiftCountMask; 958 } 959 960 /// Check whether the given shift count is appropriate 961 /// can be represented by a LEA instruction. 962 inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) { 963 // Left shift instructions can be transformed into load-effective-address 964 // instructions if we can encode them appropriately. 965 // A LEA instruction utilizes a SIB byte to encode its scale factor. 966 // The SIB.scale field is two bits wide which means that we can encode any 967 // shift amount less than 4. 968 return ShAmt < 4 && ShAmt > 0; 969 } 970 971 static bool findRedundantFlagInstr(MachineInstr &CmpInstr, 972 MachineInstr &CmpValDefInstr, 973 const MachineRegisterInfo *MRI, 974 MachineInstr **AndInstr, 975 const TargetRegisterInfo *TRI, 976 bool &NoSignFlag, bool &ClearsOverflowFlag) { 977 if (CmpValDefInstr.getOpcode() != X86::SUBREG_TO_REG) 978 return false; 979 980 if (CmpInstr.getOpcode() != X86::TEST64rr) 981 return false; 982 983 // CmpInstr is a TEST64rr instruction, and `X86InstrInfo::analyzeCompare` 984 // guarantees that it's analyzable only if two registers are identical. 985 assert( 986 (CmpInstr.getOperand(0).getReg() == CmpInstr.getOperand(1).getReg()) && 987 "CmpInstr is an analyzable TEST64rr, and `X86InstrInfo::analyzeCompare` " 988 "requires two reg operands are the same."); 989 990 // Caller (`X86InstrInfo::optimizeCompareInstr`) guarantees that 991 // `CmpValDefInstr` defines the value that's used by `CmpInstr`; in this case 992 // if `CmpValDefInstr` sets the EFLAGS, it is likely that `CmpInstr` is 993 // redundant. 994 assert( 995 (MRI->getVRegDef(CmpInstr.getOperand(0).getReg()) == &CmpValDefInstr) && 996 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG."); 997 998 // As seen in X86 td files, CmpValDefInstr.getOperand(1).getImm() is typically 999 // 0. 1000 if (CmpValDefInstr.getOperand(1).getImm() != 0) 1001 return false; 1002 1003 // As seen in X86 td files, CmpValDefInstr.getOperand(3) is typically 1004 // sub_32bit or sub_xmm. 1005 if (CmpValDefInstr.getOperand(3).getImm() != X86::sub_32bit) 1006 return false; 1007 1008 MachineInstr *VregDefInstr = 1009 MRI->getVRegDef(CmpValDefInstr.getOperand(2).getReg()); 1010 1011 assert(VregDefInstr && "Must have a definition (SSA)"); 1012 1013 // Requires `CmpValDefInstr` and `VregDefInstr` are from the same MBB 1014 // to simplify the subsequent analysis. 1015 // 1016 // FIXME: If `VregDefInstr->getParent()` is the only predecessor of 1017 // `CmpValDefInstr.getParent()`, this could be handled. 1018 if (VregDefInstr->getParent() != CmpValDefInstr.getParent()) 1019 return false; 1020 1021 if (X86::isAND(VregDefInstr->getOpcode())) { 1022 // Get a sequence of instructions like 1023 // %reg = and* ... // Set EFLAGS 1024 // ... // EFLAGS not changed 1025 // %extended_reg = subreg_to_reg 0, %reg, %subreg.sub_32bit 1026 // test64rr %extended_reg, %extended_reg, implicit-def $eflags 1027 // 1028 // If subsequent readers use a subset of bits that don't change 1029 // after `and*` instructions, it's likely that the test64rr could 1030 // be optimized away. 1031 for (const MachineInstr &Instr : 1032 make_range(std::next(MachineBasicBlock::iterator(VregDefInstr)), 1033 MachineBasicBlock::iterator(CmpValDefInstr))) { 1034 // There are instructions between 'VregDefInstr' and 1035 // 'CmpValDefInstr' that modifies EFLAGS. 1036 if (Instr.modifiesRegister(X86::EFLAGS, TRI)) 1037 return false; 1038 } 1039 1040 *AndInstr = VregDefInstr; 1041 1042 // AND instruction will essentially update SF and clear OF, so 1043 // NoSignFlag should be false in the sense that SF is modified by `AND`. 1044 // 1045 // However, the implementation artifically sets `NoSignFlag` to true 1046 // to poison the SF bit; that is to say, if SF is looked at later, the 1047 // optimization (to erase TEST64rr) will be disabled. 1048 // 1049 // The reason to poison SF bit is that SF bit value could be different 1050 // in the `AND` and `TEST` operation; signed bit is not known for `AND`, 1051 // and is known to be 0 as a result of `TEST64rr`. 1052 // 1053 // FIXME: As opposed to poisoning the SF bit directly, consider peeking into 1054 // the AND instruction and using the static information to guide peephole 1055 // optimization if possible. For example, it's possible to fold a 1056 // conditional move into a copy if the relevant EFLAG bits could be deduced 1057 // from an immediate operand of and operation. 1058 // 1059 NoSignFlag = true; 1060 // ClearsOverflowFlag is true for AND operation (no surprise). 1061 ClearsOverflowFlag = true; 1062 return true; 1063 } 1064 return false; 1065 } 1066 1067 bool X86InstrInfo::classifyLEAReg(MachineInstr &MI, const MachineOperand &Src, 1068 unsigned Opc, bool AllowSP, Register &NewSrc, 1069 bool &isKill, MachineOperand &ImplicitOp, 1070 LiveVariables *LV, LiveIntervals *LIS) const { 1071 MachineFunction &MF = *MI.getParent()->getParent(); 1072 const TargetRegisterClass *RC; 1073 if (AllowSP) { 1074 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass; 1075 } else { 1076 RC = Opc != X86::LEA32r ? 1077 &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass; 1078 } 1079 Register SrcReg = Src.getReg(); 1080 isKill = MI.killsRegister(SrcReg); 1081 1082 // For both LEA64 and LEA32 the register already has essentially the right 1083 // type (32-bit or 64-bit) we may just need to forbid SP. 1084 if (Opc != X86::LEA64_32r) { 1085 NewSrc = SrcReg; 1086 assert(!Src.isUndef() && "Undef op doesn't need optimization"); 1087 1088 if (NewSrc.isVirtual() && !MF.getRegInfo().constrainRegClass(NewSrc, RC)) 1089 return false; 1090 1091 return true; 1092 } 1093 1094 // This is for an LEA64_32r and incoming registers are 32-bit. One way or 1095 // another we need to add 64-bit registers to the final MI. 1096 if (SrcReg.isPhysical()) { 1097 ImplicitOp = Src; 1098 ImplicitOp.setImplicit(); 1099 1100 NewSrc = getX86SubSuperRegister(SrcReg, 64); 1101 assert(!Src.isUndef() && "Undef op doesn't need optimization"); 1102 } else { 1103 // Virtual register of the wrong class, we have to create a temporary 64-bit 1104 // vreg to feed into the LEA. 1105 NewSrc = MF.getRegInfo().createVirtualRegister(RC); 1106 MachineInstr *Copy = 1107 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1108 .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit) 1109 .addReg(SrcReg, getKillRegState(isKill)); 1110 1111 // Which is obviously going to be dead after we're done with it. 1112 isKill = true; 1113 1114 if (LV) 1115 LV->replaceKillInstruction(SrcReg, MI, *Copy); 1116 1117 if (LIS) { 1118 SlotIndex CopyIdx = LIS->InsertMachineInstrInMaps(*Copy); 1119 SlotIndex Idx = LIS->getInstructionIndex(MI); 1120 LiveInterval &LI = LIS->getInterval(SrcReg); 1121 LiveRange::Segment *S = LI.getSegmentContaining(Idx); 1122 if (S->end.getBaseIndex() == Idx) 1123 S->end = CopyIdx.getRegSlot(); 1124 } 1125 } 1126 1127 // We've set all the parameters without issue. 1128 return true; 1129 } 1130 1131 MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc, 1132 MachineInstr &MI, 1133 LiveVariables *LV, 1134 LiveIntervals *LIS, 1135 bool Is8BitOp) const { 1136 // We handle 8-bit adds and various 16-bit opcodes in the switch below. 1137 MachineBasicBlock &MBB = *MI.getParent(); 1138 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo(); 1139 assert((Is8BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits( 1140 *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) && 1141 "Unexpected type for LEA transform"); 1142 1143 // TODO: For a 32-bit target, we need to adjust the LEA variables with 1144 // something like this: 1145 // Opcode = X86::LEA32r; 1146 // InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); 1147 // OutRegLEA = 1148 // Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass) 1149 // : RegInfo.createVirtualRegister(&X86::GR32RegClass); 1150 if (!Subtarget.is64Bit()) 1151 return nullptr; 1152 1153 unsigned Opcode = X86::LEA64_32r; 1154 Register InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 1155 Register OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass); 1156 Register InRegLEA2; 1157 1158 // Build and insert into an implicit UNDEF value. This is OK because 1159 // we will be shifting and then extracting the lower 8/16-bits. 1160 // This has the potential to cause partial register stall. e.g. 1161 // movw (%rbp,%rcx,2), %dx 1162 // leal -65(%rdx), %esi 1163 // But testing has shown this *does* help performance in 64-bit mode (at 1164 // least on modern x86 machines). 1165 MachineBasicBlock::iterator MBBI = MI.getIterator(); 1166 Register Dest = MI.getOperand(0).getReg(); 1167 Register Src = MI.getOperand(1).getReg(); 1168 Register Src2; 1169 bool IsDead = MI.getOperand(0).isDead(); 1170 bool IsKill = MI.getOperand(1).isKill(); 1171 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit; 1172 assert(!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization"); 1173 MachineInstr *ImpDef = 1174 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA); 1175 MachineInstr *InsMI = 1176 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1177 .addReg(InRegLEA, RegState::Define, SubReg) 1178 .addReg(Src, getKillRegState(IsKill)); 1179 MachineInstr *ImpDef2 = nullptr; 1180 MachineInstr *InsMI2 = nullptr; 1181 1182 MachineInstrBuilder MIB = 1183 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(Opcode), OutRegLEA); 1184 switch (MIOpc) { 1185 default: llvm_unreachable("Unreachable!"); 1186 case X86::SHL8ri: 1187 case X86::SHL16ri: { 1188 unsigned ShAmt = MI.getOperand(2).getImm(); 1189 MIB.addReg(0) 1190 .addImm(1LL << ShAmt) 1191 .addReg(InRegLEA, RegState::Kill) 1192 .addImm(0) 1193 .addReg(0); 1194 break; 1195 } 1196 case X86::INC8r: 1197 case X86::INC16r: 1198 addRegOffset(MIB, InRegLEA, true, 1); 1199 break; 1200 case X86::DEC8r: 1201 case X86::DEC16r: 1202 addRegOffset(MIB, InRegLEA, true, -1); 1203 break; 1204 case X86::ADD8ri: 1205 case X86::ADD8ri_DB: 1206 case X86::ADD16ri: 1207 case X86::ADD16ri8: 1208 case X86::ADD16ri_DB: 1209 case X86::ADD16ri8_DB: 1210 addRegOffset(MIB, InRegLEA, true, MI.getOperand(2).getImm()); 1211 break; 1212 case X86::ADD8rr: 1213 case X86::ADD8rr_DB: 1214 case X86::ADD16rr: 1215 case X86::ADD16rr_DB: { 1216 Src2 = MI.getOperand(2).getReg(); 1217 bool IsKill2 = MI.getOperand(2).isKill(); 1218 assert(!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization"); 1219 if (Src == Src2) { 1220 // ADD8rr/ADD16rr killed %reg1028, %reg1028 1221 // just a single insert_subreg. 1222 addRegReg(MIB, InRegLEA, true, InRegLEA, false); 1223 } else { 1224 if (Subtarget.is64Bit()) 1225 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 1226 else 1227 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); 1228 // Build and insert into an implicit UNDEF value. This is OK because 1229 // we will be shifting and then extracting the lower 8/16-bits. 1230 ImpDef2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), 1231 InRegLEA2); 1232 InsMI2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1233 .addReg(InRegLEA2, RegState::Define, SubReg) 1234 .addReg(Src2, getKillRegState(IsKill2)); 1235 addRegReg(MIB, InRegLEA, true, InRegLEA2, true); 1236 } 1237 if (LV && IsKill2 && InsMI2) 1238 LV->replaceKillInstruction(Src2, MI, *InsMI2); 1239 break; 1240 } 1241 } 1242 1243 MachineInstr *NewMI = MIB; 1244 MachineInstr *ExtMI = 1245 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1246 .addReg(Dest, RegState::Define | getDeadRegState(IsDead)) 1247 .addReg(OutRegLEA, RegState::Kill, SubReg); 1248 1249 if (LV) { 1250 // Update live variables. 1251 LV->getVarInfo(InRegLEA).Kills.push_back(NewMI); 1252 LV->getVarInfo(OutRegLEA).Kills.push_back(ExtMI); 1253 if (IsKill) 1254 LV->replaceKillInstruction(Src, MI, *InsMI); 1255 if (IsDead) 1256 LV->replaceKillInstruction(Dest, MI, *ExtMI); 1257 } 1258 1259 if (LIS) { 1260 LIS->InsertMachineInstrInMaps(*ImpDef); 1261 SlotIndex InsIdx = LIS->InsertMachineInstrInMaps(*InsMI); 1262 if (ImpDef2) 1263 LIS->InsertMachineInstrInMaps(*ImpDef2); 1264 SlotIndex Ins2Idx; 1265 if (InsMI2) 1266 Ins2Idx = LIS->InsertMachineInstrInMaps(*InsMI2); 1267 SlotIndex NewIdx = LIS->ReplaceMachineInstrInMaps(MI, *NewMI); 1268 SlotIndex ExtIdx = LIS->InsertMachineInstrInMaps(*ExtMI); 1269 LIS->getInterval(InRegLEA); 1270 LIS->getInterval(OutRegLEA); 1271 if (InRegLEA2) 1272 LIS->getInterval(InRegLEA2); 1273 1274 // Move the use of Src up to InsMI. 1275 LiveInterval &SrcLI = LIS->getInterval(Src); 1276 LiveRange::Segment *SrcSeg = SrcLI.getSegmentContaining(NewIdx); 1277 if (SrcSeg->end == NewIdx.getRegSlot()) 1278 SrcSeg->end = InsIdx.getRegSlot(); 1279 1280 if (InsMI2) { 1281 // Move the use of Src2 up to InsMI2. 1282 LiveInterval &Src2LI = LIS->getInterval(Src2); 1283 LiveRange::Segment *Src2Seg = Src2LI.getSegmentContaining(NewIdx); 1284 if (Src2Seg->end == NewIdx.getRegSlot()) 1285 Src2Seg->end = Ins2Idx.getRegSlot(); 1286 } 1287 1288 // Move the definition of Dest down to ExtMI. 1289 LiveInterval &DestLI = LIS->getInterval(Dest); 1290 LiveRange::Segment *DestSeg = 1291 DestLI.getSegmentContaining(NewIdx.getRegSlot()); 1292 assert(DestSeg->start == NewIdx.getRegSlot() && 1293 DestSeg->valno->def == NewIdx.getRegSlot()); 1294 DestSeg->start = ExtIdx.getRegSlot(); 1295 DestSeg->valno->def = ExtIdx.getRegSlot(); 1296 } 1297 1298 return ExtMI; 1299 } 1300 1301 /// This method must be implemented by targets that 1302 /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target 1303 /// may be able to convert a two-address instruction into a true 1304 /// three-address instruction on demand. This allows the X86 target (for 1305 /// example) to convert ADD and SHL instructions into LEA instructions if they 1306 /// would require register copies due to two-addressness. 1307 /// 1308 /// This method returns a null pointer if the transformation cannot be 1309 /// performed, otherwise it returns the new instruction. 1310 /// 1311 MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI, 1312 LiveVariables *LV, 1313 LiveIntervals *LIS) const { 1314 // The following opcodes also sets the condition code register(s). Only 1315 // convert them to equivalent lea if the condition code register def's 1316 // are dead! 1317 if (hasLiveCondCodeDef(MI)) 1318 return nullptr; 1319 1320 MachineFunction &MF = *MI.getParent()->getParent(); 1321 // All instructions input are two-addr instructions. Get the known operands. 1322 const MachineOperand &Dest = MI.getOperand(0); 1323 const MachineOperand &Src = MI.getOperand(1); 1324 1325 // Ideally, operations with undef should be folded before we get here, but we 1326 // can't guarantee it. Bail out because optimizing undefs is a waste of time. 1327 // Without this, we have to forward undef state to new register operands to 1328 // avoid machine verifier errors. 1329 if (Src.isUndef()) 1330 return nullptr; 1331 if (MI.getNumOperands() > 2) 1332 if (MI.getOperand(2).isReg() && MI.getOperand(2).isUndef()) 1333 return nullptr; 1334 1335 MachineInstr *NewMI = nullptr; 1336 Register SrcReg, SrcReg2; 1337 bool Is64Bit = Subtarget.is64Bit(); 1338 1339 bool Is8BitOp = false; 1340 unsigned MIOpc = MI.getOpcode(); 1341 switch (MIOpc) { 1342 default: llvm_unreachable("Unreachable!"); 1343 case X86::SHL64ri: { 1344 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 1345 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 1346 if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; 1347 1348 // LEA can't handle RSP. 1349 if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass( 1350 Src.getReg(), &X86::GR64_NOSPRegClass)) 1351 return nullptr; 1352 1353 NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)) 1354 .add(Dest) 1355 .addReg(0) 1356 .addImm(1LL << ShAmt) 1357 .add(Src) 1358 .addImm(0) 1359 .addReg(0); 1360 break; 1361 } 1362 case X86::SHL32ri: { 1363 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 1364 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 1365 if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; 1366 1367 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1368 1369 // LEA can't handle ESP. 1370 bool isKill; 1371 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1372 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill, 1373 ImplicitOp, LV, LIS)) 1374 return nullptr; 1375 1376 MachineInstrBuilder MIB = 1377 BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1378 .add(Dest) 1379 .addReg(0) 1380 .addImm(1LL << ShAmt) 1381 .addReg(SrcReg, getKillRegState(isKill)) 1382 .addImm(0) 1383 .addReg(0); 1384 if (ImplicitOp.getReg() != 0) 1385 MIB.add(ImplicitOp); 1386 NewMI = MIB; 1387 1388 break; 1389 } 1390 case X86::SHL8ri: 1391 Is8BitOp = true; 1392 LLVM_FALLTHROUGH; 1393 case X86::SHL16ri: { 1394 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 1395 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 1396 if (!isTruncatedShiftCountForLEA(ShAmt)) 1397 return nullptr; 1398 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1399 } 1400 case X86::INC64r: 1401 case X86::INC32r: { 1402 assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!"); 1403 unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r : 1404 (Is64Bit ? X86::LEA64_32r : X86::LEA32r); 1405 bool isKill; 1406 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1407 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill, 1408 ImplicitOp, LV, LIS)) 1409 return nullptr; 1410 1411 MachineInstrBuilder MIB = 1412 BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1413 .add(Dest) 1414 .addReg(SrcReg, getKillRegState(isKill)); 1415 if (ImplicitOp.getReg() != 0) 1416 MIB.add(ImplicitOp); 1417 1418 NewMI = addOffset(MIB, 1); 1419 break; 1420 } 1421 case X86::DEC64r: 1422 case X86::DEC32r: { 1423 assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!"); 1424 unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r 1425 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r); 1426 1427 bool isKill; 1428 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1429 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill, 1430 ImplicitOp, LV, LIS)) 1431 return nullptr; 1432 1433 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1434 .add(Dest) 1435 .addReg(SrcReg, getKillRegState(isKill)); 1436 if (ImplicitOp.getReg() != 0) 1437 MIB.add(ImplicitOp); 1438 1439 NewMI = addOffset(MIB, -1); 1440 1441 break; 1442 } 1443 case X86::DEC8r: 1444 case X86::INC8r: 1445 Is8BitOp = true; 1446 LLVM_FALLTHROUGH; 1447 case X86::DEC16r: 1448 case X86::INC16r: 1449 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1450 case X86::ADD64rr: 1451 case X86::ADD64rr_DB: 1452 case X86::ADD32rr: 1453 case X86::ADD32rr_DB: { 1454 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1455 unsigned Opc; 1456 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB) 1457 Opc = X86::LEA64r; 1458 else 1459 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1460 1461 const MachineOperand &Src2 = MI.getOperand(2); 1462 bool isKill2; 1463 MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false); 1464 if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/false, SrcReg2, isKill2, 1465 ImplicitOp2, LV, LIS)) 1466 return nullptr; 1467 1468 bool isKill; 1469 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1470 if (Src.getReg() == Src2.getReg()) { 1471 // Don't call classify LEAReg a second time on the same register, in case 1472 // the first call inserted a COPY from Src2 and marked it as killed. 1473 isKill = isKill2; 1474 SrcReg = SrcReg2; 1475 } else { 1476 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill, 1477 ImplicitOp, LV, LIS)) 1478 return nullptr; 1479 } 1480 1481 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest); 1482 if (ImplicitOp.getReg() != 0) 1483 MIB.add(ImplicitOp); 1484 if (ImplicitOp2.getReg() != 0) 1485 MIB.add(ImplicitOp2); 1486 1487 NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2); 1488 if (LV && Src2.isKill()) 1489 LV->replaceKillInstruction(SrcReg2, MI, *NewMI); 1490 break; 1491 } 1492 case X86::ADD8rr: 1493 case X86::ADD8rr_DB: 1494 Is8BitOp = true; 1495 LLVM_FALLTHROUGH; 1496 case X86::ADD16rr: 1497 case X86::ADD16rr_DB: 1498 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1499 case X86::ADD64ri32: 1500 case X86::ADD64ri8: 1501 case X86::ADD64ri32_DB: 1502 case X86::ADD64ri8_DB: 1503 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1504 NewMI = addOffset( 1505 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src), 1506 MI.getOperand(2)); 1507 break; 1508 case X86::ADD32ri: 1509 case X86::ADD32ri8: 1510 case X86::ADD32ri_DB: 1511 case X86::ADD32ri8_DB: { 1512 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1513 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1514 1515 bool isKill; 1516 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1517 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill, 1518 ImplicitOp, LV, LIS)) 1519 return nullptr; 1520 1521 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1522 .add(Dest) 1523 .addReg(SrcReg, getKillRegState(isKill)); 1524 if (ImplicitOp.getReg() != 0) 1525 MIB.add(ImplicitOp); 1526 1527 NewMI = addOffset(MIB, MI.getOperand(2)); 1528 break; 1529 } 1530 case X86::ADD8ri: 1531 case X86::ADD8ri_DB: 1532 Is8BitOp = true; 1533 LLVM_FALLTHROUGH; 1534 case X86::ADD16ri: 1535 case X86::ADD16ri8: 1536 case X86::ADD16ri_DB: 1537 case X86::ADD16ri8_DB: 1538 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1539 case X86::SUB8ri: 1540 case X86::SUB16ri8: 1541 case X86::SUB16ri: 1542 /// FIXME: Support these similar to ADD8ri/ADD16ri*. 1543 return nullptr; 1544 case X86::SUB32ri8: 1545 case X86::SUB32ri: { 1546 if (!MI.getOperand(2).isImm()) 1547 return nullptr; 1548 int64_t Imm = MI.getOperand(2).getImm(); 1549 if (!isInt<32>(-Imm)) 1550 return nullptr; 1551 1552 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1553 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1554 1555 bool isKill; 1556 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1557 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill, 1558 ImplicitOp, LV, LIS)) 1559 return nullptr; 1560 1561 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1562 .add(Dest) 1563 .addReg(SrcReg, getKillRegState(isKill)); 1564 if (ImplicitOp.getReg() != 0) 1565 MIB.add(ImplicitOp); 1566 1567 NewMI = addOffset(MIB, -Imm); 1568 break; 1569 } 1570 1571 case X86::SUB64ri8: 1572 case X86::SUB64ri32: { 1573 if (!MI.getOperand(2).isImm()) 1574 return nullptr; 1575 int64_t Imm = MI.getOperand(2).getImm(); 1576 if (!isInt<32>(-Imm)) 1577 return nullptr; 1578 1579 assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!"); 1580 1581 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), 1582 get(X86::LEA64r)).add(Dest).add(Src); 1583 NewMI = addOffset(MIB, -Imm); 1584 break; 1585 } 1586 1587 case X86::VMOVDQU8Z128rmk: 1588 case X86::VMOVDQU8Z256rmk: 1589 case X86::VMOVDQU8Zrmk: 1590 case X86::VMOVDQU16Z128rmk: 1591 case X86::VMOVDQU16Z256rmk: 1592 case X86::VMOVDQU16Zrmk: 1593 case X86::VMOVDQU32Z128rmk: case X86::VMOVDQA32Z128rmk: 1594 case X86::VMOVDQU32Z256rmk: case X86::VMOVDQA32Z256rmk: 1595 case X86::VMOVDQU32Zrmk: case X86::VMOVDQA32Zrmk: 1596 case X86::VMOVDQU64Z128rmk: case X86::VMOVDQA64Z128rmk: 1597 case X86::VMOVDQU64Z256rmk: case X86::VMOVDQA64Z256rmk: 1598 case X86::VMOVDQU64Zrmk: case X86::VMOVDQA64Zrmk: 1599 case X86::VMOVUPDZ128rmk: case X86::VMOVAPDZ128rmk: 1600 case X86::VMOVUPDZ256rmk: case X86::VMOVAPDZ256rmk: 1601 case X86::VMOVUPDZrmk: case X86::VMOVAPDZrmk: 1602 case X86::VMOVUPSZ128rmk: case X86::VMOVAPSZ128rmk: 1603 case X86::VMOVUPSZ256rmk: case X86::VMOVAPSZ256rmk: 1604 case X86::VMOVUPSZrmk: case X86::VMOVAPSZrmk: 1605 case X86::VBROADCASTSDZ256rmk: 1606 case X86::VBROADCASTSDZrmk: 1607 case X86::VBROADCASTSSZ128rmk: 1608 case X86::VBROADCASTSSZ256rmk: 1609 case X86::VBROADCASTSSZrmk: 1610 case X86::VPBROADCASTDZ128rmk: 1611 case X86::VPBROADCASTDZ256rmk: 1612 case X86::VPBROADCASTDZrmk: 1613 case X86::VPBROADCASTQZ128rmk: 1614 case X86::VPBROADCASTQZ256rmk: 1615 case X86::VPBROADCASTQZrmk: { 1616 unsigned Opc; 1617 switch (MIOpc) { 1618 default: llvm_unreachable("Unreachable!"); 1619 case X86::VMOVDQU8Z128rmk: Opc = X86::VPBLENDMBZ128rmk; break; 1620 case X86::VMOVDQU8Z256rmk: Opc = X86::VPBLENDMBZ256rmk; break; 1621 case X86::VMOVDQU8Zrmk: Opc = X86::VPBLENDMBZrmk; break; 1622 case X86::VMOVDQU16Z128rmk: Opc = X86::VPBLENDMWZ128rmk; break; 1623 case X86::VMOVDQU16Z256rmk: Opc = X86::VPBLENDMWZ256rmk; break; 1624 case X86::VMOVDQU16Zrmk: Opc = X86::VPBLENDMWZrmk; break; 1625 case X86::VMOVDQU32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; 1626 case X86::VMOVDQU32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; 1627 case X86::VMOVDQU32Zrmk: Opc = X86::VPBLENDMDZrmk; break; 1628 case X86::VMOVDQU64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; 1629 case X86::VMOVDQU64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; 1630 case X86::VMOVDQU64Zrmk: Opc = X86::VPBLENDMQZrmk; break; 1631 case X86::VMOVUPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; 1632 case X86::VMOVUPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; 1633 case X86::VMOVUPDZrmk: Opc = X86::VBLENDMPDZrmk; break; 1634 case X86::VMOVUPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; 1635 case X86::VMOVUPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; 1636 case X86::VMOVUPSZrmk: Opc = X86::VBLENDMPSZrmk; break; 1637 case X86::VMOVDQA32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; 1638 case X86::VMOVDQA32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; 1639 case X86::VMOVDQA32Zrmk: Opc = X86::VPBLENDMDZrmk; break; 1640 case X86::VMOVDQA64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; 1641 case X86::VMOVDQA64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; 1642 case X86::VMOVDQA64Zrmk: Opc = X86::VPBLENDMQZrmk; break; 1643 case X86::VMOVAPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; 1644 case X86::VMOVAPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; 1645 case X86::VMOVAPDZrmk: Opc = X86::VBLENDMPDZrmk; break; 1646 case X86::VMOVAPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; 1647 case X86::VMOVAPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; 1648 case X86::VMOVAPSZrmk: Opc = X86::VBLENDMPSZrmk; break; 1649 case X86::VBROADCASTSDZ256rmk: Opc = X86::VBLENDMPDZ256rmbk; break; 1650 case X86::VBROADCASTSDZrmk: Opc = X86::VBLENDMPDZrmbk; break; 1651 case X86::VBROADCASTSSZ128rmk: Opc = X86::VBLENDMPSZ128rmbk; break; 1652 case X86::VBROADCASTSSZ256rmk: Opc = X86::VBLENDMPSZ256rmbk; break; 1653 case X86::VBROADCASTSSZrmk: Opc = X86::VBLENDMPSZrmbk; break; 1654 case X86::VPBROADCASTDZ128rmk: Opc = X86::VPBLENDMDZ128rmbk; break; 1655 case X86::VPBROADCASTDZ256rmk: Opc = X86::VPBLENDMDZ256rmbk; break; 1656 case X86::VPBROADCASTDZrmk: Opc = X86::VPBLENDMDZrmbk; break; 1657 case X86::VPBROADCASTQZ128rmk: Opc = X86::VPBLENDMQZ128rmbk; break; 1658 case X86::VPBROADCASTQZ256rmk: Opc = X86::VPBLENDMQZ256rmbk; break; 1659 case X86::VPBROADCASTQZrmk: Opc = X86::VPBLENDMQZrmbk; break; 1660 } 1661 1662 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1663 .add(Dest) 1664 .add(MI.getOperand(2)) 1665 .add(Src) 1666 .add(MI.getOperand(3)) 1667 .add(MI.getOperand(4)) 1668 .add(MI.getOperand(5)) 1669 .add(MI.getOperand(6)) 1670 .add(MI.getOperand(7)); 1671 break; 1672 } 1673 1674 case X86::VMOVDQU8Z128rrk: 1675 case X86::VMOVDQU8Z256rrk: 1676 case X86::VMOVDQU8Zrrk: 1677 case X86::VMOVDQU16Z128rrk: 1678 case X86::VMOVDQU16Z256rrk: 1679 case X86::VMOVDQU16Zrrk: 1680 case X86::VMOVDQU32Z128rrk: case X86::VMOVDQA32Z128rrk: 1681 case X86::VMOVDQU32Z256rrk: case X86::VMOVDQA32Z256rrk: 1682 case X86::VMOVDQU32Zrrk: case X86::VMOVDQA32Zrrk: 1683 case X86::VMOVDQU64Z128rrk: case X86::VMOVDQA64Z128rrk: 1684 case X86::VMOVDQU64Z256rrk: case X86::VMOVDQA64Z256rrk: 1685 case X86::VMOVDQU64Zrrk: case X86::VMOVDQA64Zrrk: 1686 case X86::VMOVUPDZ128rrk: case X86::VMOVAPDZ128rrk: 1687 case X86::VMOVUPDZ256rrk: case X86::VMOVAPDZ256rrk: 1688 case X86::VMOVUPDZrrk: case X86::VMOVAPDZrrk: 1689 case X86::VMOVUPSZ128rrk: case X86::VMOVAPSZ128rrk: 1690 case X86::VMOVUPSZ256rrk: case X86::VMOVAPSZ256rrk: 1691 case X86::VMOVUPSZrrk: case X86::VMOVAPSZrrk: { 1692 unsigned Opc; 1693 switch (MIOpc) { 1694 default: llvm_unreachable("Unreachable!"); 1695 case X86::VMOVDQU8Z128rrk: Opc = X86::VPBLENDMBZ128rrk; break; 1696 case X86::VMOVDQU8Z256rrk: Opc = X86::VPBLENDMBZ256rrk; break; 1697 case X86::VMOVDQU8Zrrk: Opc = X86::VPBLENDMBZrrk; break; 1698 case X86::VMOVDQU16Z128rrk: Opc = X86::VPBLENDMWZ128rrk; break; 1699 case X86::VMOVDQU16Z256rrk: Opc = X86::VPBLENDMWZ256rrk; break; 1700 case X86::VMOVDQU16Zrrk: Opc = X86::VPBLENDMWZrrk; break; 1701 case X86::VMOVDQU32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; 1702 case X86::VMOVDQU32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; 1703 case X86::VMOVDQU32Zrrk: Opc = X86::VPBLENDMDZrrk; break; 1704 case X86::VMOVDQU64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; 1705 case X86::VMOVDQU64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; 1706 case X86::VMOVDQU64Zrrk: Opc = X86::VPBLENDMQZrrk; break; 1707 case X86::VMOVUPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; 1708 case X86::VMOVUPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; 1709 case X86::VMOVUPDZrrk: Opc = X86::VBLENDMPDZrrk; break; 1710 case X86::VMOVUPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; 1711 case X86::VMOVUPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; 1712 case X86::VMOVUPSZrrk: Opc = X86::VBLENDMPSZrrk; break; 1713 case X86::VMOVDQA32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; 1714 case X86::VMOVDQA32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; 1715 case X86::VMOVDQA32Zrrk: Opc = X86::VPBLENDMDZrrk; break; 1716 case X86::VMOVDQA64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; 1717 case X86::VMOVDQA64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; 1718 case X86::VMOVDQA64Zrrk: Opc = X86::VPBLENDMQZrrk; break; 1719 case X86::VMOVAPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; 1720 case X86::VMOVAPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; 1721 case X86::VMOVAPDZrrk: Opc = X86::VBLENDMPDZrrk; break; 1722 case X86::VMOVAPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; 1723 case X86::VMOVAPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; 1724 case X86::VMOVAPSZrrk: Opc = X86::VBLENDMPSZrrk; break; 1725 } 1726 1727 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1728 .add(Dest) 1729 .add(MI.getOperand(2)) 1730 .add(Src) 1731 .add(MI.getOperand(3)); 1732 break; 1733 } 1734 } 1735 1736 if (!NewMI) return nullptr; 1737 1738 if (LV) { // Update live variables 1739 if (Src.isKill()) 1740 LV->replaceKillInstruction(Src.getReg(), MI, *NewMI); 1741 if (Dest.isDead()) 1742 LV->replaceKillInstruction(Dest.getReg(), MI, *NewMI); 1743 } 1744 1745 MachineBasicBlock &MBB = *MI.getParent(); 1746 MBB.insert(MI.getIterator(), NewMI); // Insert the new inst 1747 1748 if (LIS) { 1749 LIS->ReplaceMachineInstrInMaps(MI, *NewMI); 1750 if (SrcReg) 1751 LIS->getInterval(SrcReg); 1752 if (SrcReg2) 1753 LIS->getInterval(SrcReg2); 1754 } 1755 1756 return NewMI; 1757 } 1758 1759 /// This determines which of three possible cases of a three source commute 1760 /// the source indexes correspond to taking into account any mask operands. 1761 /// All prevents commuting a passthru operand. Returns -1 if the commute isn't 1762 /// possible. 1763 /// Case 0 - Possible to commute the first and second operands. 1764 /// Case 1 - Possible to commute the first and third operands. 1765 /// Case 2 - Possible to commute the second and third operands. 1766 static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1, 1767 unsigned SrcOpIdx2) { 1768 // Put the lowest index to SrcOpIdx1 to simplify the checks below. 1769 if (SrcOpIdx1 > SrcOpIdx2) 1770 std::swap(SrcOpIdx1, SrcOpIdx2); 1771 1772 unsigned Op1 = 1, Op2 = 2, Op3 = 3; 1773 if (X86II::isKMasked(TSFlags)) { 1774 Op2++; 1775 Op3++; 1776 } 1777 1778 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2) 1779 return 0; 1780 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3) 1781 return 1; 1782 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3) 1783 return 2; 1784 llvm_unreachable("Unknown three src commute case."); 1785 } 1786 1787 unsigned X86InstrInfo::getFMA3OpcodeToCommuteOperands( 1788 const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2, 1789 const X86InstrFMA3Group &FMA3Group) const { 1790 1791 unsigned Opc = MI.getOpcode(); 1792 1793 // TODO: Commuting the 1st operand of FMA*_Int requires some additional 1794 // analysis. The commute optimization is legal only if all users of FMA*_Int 1795 // use only the lowest element of the FMA*_Int instruction. Such analysis are 1796 // not implemented yet. So, just return 0 in that case. 1797 // When such analysis are available this place will be the right place for 1798 // calling it. 1799 assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && 1800 "Intrinsic instructions can't commute operand 1"); 1801 1802 // Determine which case this commute is or if it can't be done. 1803 unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, 1804 SrcOpIdx2); 1805 assert(Case < 3 && "Unexpected case number!"); 1806 1807 // Define the FMA forms mapping array that helps to map input FMA form 1808 // to output FMA form to preserve the operation semantics after 1809 // commuting the operands. 1810 const unsigned Form132Index = 0; 1811 const unsigned Form213Index = 1; 1812 const unsigned Form231Index = 2; 1813 static const unsigned FormMapping[][3] = { 1814 // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2; 1815 // FMA132 A, C, b; ==> FMA231 C, A, b; 1816 // FMA213 B, A, c; ==> FMA213 A, B, c; 1817 // FMA231 C, A, b; ==> FMA132 A, C, b; 1818 { Form231Index, Form213Index, Form132Index }, 1819 // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3; 1820 // FMA132 A, c, B; ==> FMA132 B, c, A; 1821 // FMA213 B, a, C; ==> FMA231 C, a, B; 1822 // FMA231 C, a, B; ==> FMA213 B, a, C; 1823 { Form132Index, Form231Index, Form213Index }, 1824 // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3; 1825 // FMA132 a, C, B; ==> FMA213 a, B, C; 1826 // FMA213 b, A, C; ==> FMA132 b, C, A; 1827 // FMA231 c, A, B; ==> FMA231 c, B, A; 1828 { Form213Index, Form132Index, Form231Index } 1829 }; 1830 1831 unsigned FMAForms[3]; 1832 FMAForms[0] = FMA3Group.get132Opcode(); 1833 FMAForms[1] = FMA3Group.get213Opcode(); 1834 FMAForms[2] = FMA3Group.get231Opcode(); 1835 1836 // Everything is ready, just adjust the FMA opcode and return it. 1837 for (unsigned FormIndex = 0; FormIndex < 3; FormIndex++) 1838 if (Opc == FMAForms[FormIndex]) 1839 return FMAForms[FormMapping[Case][FormIndex]]; 1840 1841 llvm_unreachable("Illegal FMA3 format"); 1842 } 1843 1844 static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1, 1845 unsigned SrcOpIdx2) { 1846 // Determine which case this commute is or if it can't be done. 1847 unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, 1848 SrcOpIdx2); 1849 assert(Case < 3 && "Unexpected case value!"); 1850 1851 // For each case we need to swap two pairs of bits in the final immediate. 1852 static const uint8_t SwapMasks[3][4] = { 1853 { 0x04, 0x10, 0x08, 0x20 }, // Swap bits 2/4 and 3/5. 1854 { 0x02, 0x10, 0x08, 0x40 }, // Swap bits 1/4 and 3/6. 1855 { 0x02, 0x04, 0x20, 0x40 }, // Swap bits 1/2 and 5/6. 1856 }; 1857 1858 uint8_t Imm = MI.getOperand(MI.getNumOperands()-1).getImm(); 1859 // Clear out the bits we are swapping. 1860 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] | 1861 SwapMasks[Case][2] | SwapMasks[Case][3]); 1862 // If the immediate had a bit of the pair set, then set the opposite bit. 1863 if (Imm & SwapMasks[Case][0]) NewImm |= SwapMasks[Case][1]; 1864 if (Imm & SwapMasks[Case][1]) NewImm |= SwapMasks[Case][0]; 1865 if (Imm & SwapMasks[Case][2]) NewImm |= SwapMasks[Case][3]; 1866 if (Imm & SwapMasks[Case][3]) NewImm |= SwapMasks[Case][2]; 1867 MI.getOperand(MI.getNumOperands()-1).setImm(NewImm); 1868 } 1869 1870 // Returns true if this is a VPERMI2 or VPERMT2 instruction that can be 1871 // commuted. 1872 static bool isCommutableVPERMV3Instruction(unsigned Opcode) { 1873 #define VPERM_CASES(Suffix) \ 1874 case X86::VPERMI2##Suffix##128rr: case X86::VPERMT2##Suffix##128rr: \ 1875 case X86::VPERMI2##Suffix##256rr: case X86::VPERMT2##Suffix##256rr: \ 1876 case X86::VPERMI2##Suffix##rr: case X86::VPERMT2##Suffix##rr: \ 1877 case X86::VPERMI2##Suffix##128rm: case X86::VPERMT2##Suffix##128rm: \ 1878 case X86::VPERMI2##Suffix##256rm: case X86::VPERMT2##Suffix##256rm: \ 1879 case X86::VPERMI2##Suffix##rm: case X86::VPERMT2##Suffix##rm: \ 1880 case X86::VPERMI2##Suffix##128rrkz: case X86::VPERMT2##Suffix##128rrkz: \ 1881 case X86::VPERMI2##Suffix##256rrkz: case X86::VPERMT2##Suffix##256rrkz: \ 1882 case X86::VPERMI2##Suffix##rrkz: case X86::VPERMT2##Suffix##rrkz: \ 1883 case X86::VPERMI2##Suffix##128rmkz: case X86::VPERMT2##Suffix##128rmkz: \ 1884 case X86::VPERMI2##Suffix##256rmkz: case X86::VPERMT2##Suffix##256rmkz: \ 1885 case X86::VPERMI2##Suffix##rmkz: case X86::VPERMT2##Suffix##rmkz: 1886 1887 #define VPERM_CASES_BROADCAST(Suffix) \ 1888 VPERM_CASES(Suffix) \ 1889 case X86::VPERMI2##Suffix##128rmb: case X86::VPERMT2##Suffix##128rmb: \ 1890 case X86::VPERMI2##Suffix##256rmb: case X86::VPERMT2##Suffix##256rmb: \ 1891 case X86::VPERMI2##Suffix##rmb: case X86::VPERMT2##Suffix##rmb: \ 1892 case X86::VPERMI2##Suffix##128rmbkz: case X86::VPERMT2##Suffix##128rmbkz: \ 1893 case X86::VPERMI2##Suffix##256rmbkz: case X86::VPERMT2##Suffix##256rmbkz: \ 1894 case X86::VPERMI2##Suffix##rmbkz: case X86::VPERMT2##Suffix##rmbkz: 1895 1896 switch (Opcode) { 1897 default: return false; 1898 VPERM_CASES(B) 1899 VPERM_CASES_BROADCAST(D) 1900 VPERM_CASES_BROADCAST(PD) 1901 VPERM_CASES_BROADCAST(PS) 1902 VPERM_CASES_BROADCAST(Q) 1903 VPERM_CASES(W) 1904 return true; 1905 } 1906 #undef VPERM_CASES_BROADCAST 1907 #undef VPERM_CASES 1908 } 1909 1910 // Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching 1911 // from the I opcode to the T opcode and vice versa. 1912 static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) { 1913 #define VPERM_CASES(Orig, New) \ 1914 case X86::Orig##128rr: return X86::New##128rr; \ 1915 case X86::Orig##128rrkz: return X86::New##128rrkz; \ 1916 case X86::Orig##128rm: return X86::New##128rm; \ 1917 case X86::Orig##128rmkz: return X86::New##128rmkz; \ 1918 case X86::Orig##256rr: return X86::New##256rr; \ 1919 case X86::Orig##256rrkz: return X86::New##256rrkz; \ 1920 case X86::Orig##256rm: return X86::New##256rm; \ 1921 case X86::Orig##256rmkz: return X86::New##256rmkz; \ 1922 case X86::Orig##rr: return X86::New##rr; \ 1923 case X86::Orig##rrkz: return X86::New##rrkz; \ 1924 case X86::Orig##rm: return X86::New##rm; \ 1925 case X86::Orig##rmkz: return X86::New##rmkz; 1926 1927 #define VPERM_CASES_BROADCAST(Orig, New) \ 1928 VPERM_CASES(Orig, New) \ 1929 case X86::Orig##128rmb: return X86::New##128rmb; \ 1930 case X86::Orig##128rmbkz: return X86::New##128rmbkz; \ 1931 case X86::Orig##256rmb: return X86::New##256rmb; \ 1932 case X86::Orig##256rmbkz: return X86::New##256rmbkz; \ 1933 case X86::Orig##rmb: return X86::New##rmb; \ 1934 case X86::Orig##rmbkz: return X86::New##rmbkz; 1935 1936 switch (Opcode) { 1937 VPERM_CASES(VPERMI2B, VPERMT2B) 1938 VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D) 1939 VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD) 1940 VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS) 1941 VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q) 1942 VPERM_CASES(VPERMI2W, VPERMT2W) 1943 VPERM_CASES(VPERMT2B, VPERMI2B) 1944 VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D) 1945 VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD) 1946 VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS) 1947 VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q) 1948 VPERM_CASES(VPERMT2W, VPERMI2W) 1949 } 1950 1951 llvm_unreachable("Unreachable!"); 1952 #undef VPERM_CASES_BROADCAST 1953 #undef VPERM_CASES 1954 } 1955 1956 MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 1957 unsigned OpIdx1, 1958 unsigned OpIdx2) const { 1959 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 1960 if (NewMI) 1961 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 1962 return MI; 1963 }; 1964 1965 switch (MI.getOpcode()) { 1966 case X86::SHRD16rri8: // A = SHRD16rri8 B, C, I -> A = SHLD16rri8 C, B, (16-I) 1967 case X86::SHLD16rri8: // A = SHLD16rri8 B, C, I -> A = SHRD16rri8 C, B, (16-I) 1968 case X86::SHRD32rri8: // A = SHRD32rri8 B, C, I -> A = SHLD32rri8 C, B, (32-I) 1969 case X86::SHLD32rri8: // A = SHLD32rri8 B, C, I -> A = SHRD32rri8 C, B, (32-I) 1970 case X86::SHRD64rri8: // A = SHRD64rri8 B, C, I -> A = SHLD64rri8 C, B, (64-I) 1971 case X86::SHLD64rri8:{// A = SHLD64rri8 B, C, I -> A = SHRD64rri8 C, B, (64-I) 1972 unsigned Opc; 1973 unsigned Size; 1974 switch (MI.getOpcode()) { 1975 default: llvm_unreachable("Unreachable!"); 1976 case X86::SHRD16rri8: Size = 16; Opc = X86::SHLD16rri8; break; 1977 case X86::SHLD16rri8: Size = 16; Opc = X86::SHRD16rri8; break; 1978 case X86::SHRD32rri8: Size = 32; Opc = X86::SHLD32rri8; break; 1979 case X86::SHLD32rri8: Size = 32; Opc = X86::SHRD32rri8; break; 1980 case X86::SHRD64rri8: Size = 64; Opc = X86::SHLD64rri8; break; 1981 case X86::SHLD64rri8: Size = 64; Opc = X86::SHRD64rri8; break; 1982 } 1983 unsigned Amt = MI.getOperand(3).getImm(); 1984 auto &WorkingMI = cloneIfNew(MI); 1985 WorkingMI.setDesc(get(Opc)); 1986 WorkingMI.getOperand(3).setImm(Size - Amt); 1987 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1988 OpIdx1, OpIdx2); 1989 } 1990 case X86::PFSUBrr: 1991 case X86::PFSUBRrr: { 1992 // PFSUB x, y: x = x - y 1993 // PFSUBR x, y: x = y - x 1994 unsigned Opc = 1995 (X86::PFSUBRrr == MI.getOpcode() ? X86::PFSUBrr : X86::PFSUBRrr); 1996 auto &WorkingMI = cloneIfNew(MI); 1997 WorkingMI.setDesc(get(Opc)); 1998 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1999 OpIdx1, OpIdx2); 2000 } 2001 case X86::BLENDPDrri: 2002 case X86::BLENDPSrri: 2003 case X86::VBLENDPDrri: 2004 case X86::VBLENDPSrri: 2005 // If we're optimizing for size, try to use MOVSD/MOVSS. 2006 if (MI.getParent()->getParent()->getFunction().hasOptSize()) { 2007 unsigned Mask, Opc; 2008 switch (MI.getOpcode()) { 2009 default: llvm_unreachable("Unreachable!"); 2010 case X86::BLENDPDrri: Opc = X86::MOVSDrr; Mask = 0x03; break; 2011 case X86::BLENDPSrri: Opc = X86::MOVSSrr; Mask = 0x0F; break; 2012 case X86::VBLENDPDrri: Opc = X86::VMOVSDrr; Mask = 0x03; break; 2013 case X86::VBLENDPSrri: Opc = X86::VMOVSSrr; Mask = 0x0F; break; 2014 } 2015 if ((MI.getOperand(3).getImm() ^ Mask) == 1) { 2016 auto &WorkingMI = cloneIfNew(MI); 2017 WorkingMI.setDesc(get(Opc)); 2018 WorkingMI.removeOperand(3); 2019 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, 2020 /*NewMI=*/false, 2021 OpIdx1, OpIdx2); 2022 } 2023 } 2024 LLVM_FALLTHROUGH; 2025 case X86::PBLENDWrri: 2026 case X86::VBLENDPDYrri: 2027 case X86::VBLENDPSYrri: 2028 case X86::VPBLENDDrri: 2029 case X86::VPBLENDWrri: 2030 case X86::VPBLENDDYrri: 2031 case X86::VPBLENDWYrri:{ 2032 int8_t Mask; 2033 switch (MI.getOpcode()) { 2034 default: llvm_unreachable("Unreachable!"); 2035 case X86::BLENDPDrri: Mask = (int8_t)0x03; break; 2036 case X86::BLENDPSrri: Mask = (int8_t)0x0F; break; 2037 case X86::PBLENDWrri: Mask = (int8_t)0xFF; break; 2038 case X86::VBLENDPDrri: Mask = (int8_t)0x03; break; 2039 case X86::VBLENDPSrri: Mask = (int8_t)0x0F; break; 2040 case X86::VBLENDPDYrri: Mask = (int8_t)0x0F; break; 2041 case X86::VBLENDPSYrri: Mask = (int8_t)0xFF; break; 2042 case X86::VPBLENDDrri: Mask = (int8_t)0x0F; break; 2043 case X86::VPBLENDWrri: Mask = (int8_t)0xFF; break; 2044 case X86::VPBLENDDYrri: Mask = (int8_t)0xFF; break; 2045 case X86::VPBLENDWYrri: Mask = (int8_t)0xFF; break; 2046 } 2047 // Only the least significant bits of Imm are used. 2048 // Using int8_t to ensure it will be sign extended to the int64_t that 2049 // setImm takes in order to match isel behavior. 2050 int8_t Imm = MI.getOperand(3).getImm() & Mask; 2051 auto &WorkingMI = cloneIfNew(MI); 2052 WorkingMI.getOperand(3).setImm(Mask ^ Imm); 2053 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2054 OpIdx1, OpIdx2); 2055 } 2056 case X86::INSERTPSrr: 2057 case X86::VINSERTPSrr: 2058 case X86::VINSERTPSZrr: { 2059 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); 2060 unsigned ZMask = Imm & 15; 2061 unsigned DstIdx = (Imm >> 4) & 3; 2062 unsigned SrcIdx = (Imm >> 6) & 3; 2063 2064 // We can commute insertps if we zero 2 of the elements, the insertion is 2065 // "inline" and we don't override the insertion with a zero. 2066 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 && 2067 countPopulation(ZMask) == 2) { 2068 unsigned AltIdx = findFirstSet((ZMask | (1 << DstIdx)) ^ 15); 2069 assert(AltIdx < 4 && "Illegal insertion index"); 2070 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask; 2071 auto &WorkingMI = cloneIfNew(MI); 2072 WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(AltImm); 2073 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2074 OpIdx1, OpIdx2); 2075 } 2076 return nullptr; 2077 } 2078 case X86::MOVSDrr: 2079 case X86::MOVSSrr: 2080 case X86::VMOVSDrr: 2081 case X86::VMOVSSrr:{ 2082 // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD. 2083 if (Subtarget.hasSSE41()) { 2084 unsigned Mask, Opc; 2085 switch (MI.getOpcode()) { 2086 default: llvm_unreachable("Unreachable!"); 2087 case X86::MOVSDrr: Opc = X86::BLENDPDrri; Mask = 0x02; break; 2088 case X86::MOVSSrr: Opc = X86::BLENDPSrri; Mask = 0x0E; break; 2089 case X86::VMOVSDrr: Opc = X86::VBLENDPDrri; Mask = 0x02; break; 2090 case X86::VMOVSSrr: Opc = X86::VBLENDPSrri; Mask = 0x0E; break; 2091 } 2092 2093 auto &WorkingMI = cloneIfNew(MI); 2094 WorkingMI.setDesc(get(Opc)); 2095 WorkingMI.addOperand(MachineOperand::CreateImm(Mask)); 2096 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2097 OpIdx1, OpIdx2); 2098 } 2099 2100 // Convert to SHUFPD. 2101 assert(MI.getOpcode() == X86::MOVSDrr && 2102 "Can only commute MOVSDrr without SSE4.1"); 2103 2104 auto &WorkingMI = cloneIfNew(MI); 2105 WorkingMI.setDesc(get(X86::SHUFPDrri)); 2106 WorkingMI.addOperand(MachineOperand::CreateImm(0x02)); 2107 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2108 OpIdx1, OpIdx2); 2109 } 2110 case X86::SHUFPDrri: { 2111 // Commute to MOVSD. 2112 assert(MI.getOperand(3).getImm() == 0x02 && "Unexpected immediate!"); 2113 auto &WorkingMI = cloneIfNew(MI); 2114 WorkingMI.setDesc(get(X86::MOVSDrr)); 2115 WorkingMI.removeOperand(3); 2116 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2117 OpIdx1, OpIdx2); 2118 } 2119 case X86::PCLMULQDQrr: 2120 case X86::VPCLMULQDQrr: 2121 case X86::VPCLMULQDQYrr: 2122 case X86::VPCLMULQDQZrr: 2123 case X86::VPCLMULQDQZ128rr: 2124 case X86::VPCLMULQDQZ256rr: { 2125 // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0] 2126 // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0] 2127 unsigned Imm = MI.getOperand(3).getImm(); 2128 unsigned Src1Hi = Imm & 0x01; 2129 unsigned Src2Hi = Imm & 0x10; 2130 auto &WorkingMI = cloneIfNew(MI); 2131 WorkingMI.getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4)); 2132 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2133 OpIdx1, OpIdx2); 2134 } 2135 case X86::VPCMPBZ128rri: case X86::VPCMPUBZ128rri: 2136 case X86::VPCMPBZ256rri: case X86::VPCMPUBZ256rri: 2137 case X86::VPCMPBZrri: case X86::VPCMPUBZrri: 2138 case X86::VPCMPDZ128rri: case X86::VPCMPUDZ128rri: 2139 case X86::VPCMPDZ256rri: case X86::VPCMPUDZ256rri: 2140 case X86::VPCMPDZrri: case X86::VPCMPUDZrri: 2141 case X86::VPCMPQZ128rri: case X86::VPCMPUQZ128rri: 2142 case X86::VPCMPQZ256rri: case X86::VPCMPUQZ256rri: 2143 case X86::VPCMPQZrri: case X86::VPCMPUQZrri: 2144 case X86::VPCMPWZ128rri: case X86::VPCMPUWZ128rri: 2145 case X86::VPCMPWZ256rri: case X86::VPCMPUWZ256rri: 2146 case X86::VPCMPWZrri: case X86::VPCMPUWZrri: 2147 case X86::VPCMPBZ128rrik: case X86::VPCMPUBZ128rrik: 2148 case X86::VPCMPBZ256rrik: case X86::VPCMPUBZ256rrik: 2149 case X86::VPCMPBZrrik: case X86::VPCMPUBZrrik: 2150 case X86::VPCMPDZ128rrik: case X86::VPCMPUDZ128rrik: 2151 case X86::VPCMPDZ256rrik: case X86::VPCMPUDZ256rrik: 2152 case X86::VPCMPDZrrik: case X86::VPCMPUDZrrik: 2153 case X86::VPCMPQZ128rrik: case X86::VPCMPUQZ128rrik: 2154 case X86::VPCMPQZ256rrik: case X86::VPCMPUQZ256rrik: 2155 case X86::VPCMPQZrrik: case X86::VPCMPUQZrrik: 2156 case X86::VPCMPWZ128rrik: case X86::VPCMPUWZ128rrik: 2157 case X86::VPCMPWZ256rrik: case X86::VPCMPUWZ256rrik: 2158 case X86::VPCMPWZrrik: case X86::VPCMPUWZrrik: { 2159 // Flip comparison mode immediate (if necessary). 2160 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7; 2161 Imm = X86::getSwappedVPCMPImm(Imm); 2162 auto &WorkingMI = cloneIfNew(MI); 2163 WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(Imm); 2164 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2165 OpIdx1, OpIdx2); 2166 } 2167 case X86::VPCOMBri: case X86::VPCOMUBri: 2168 case X86::VPCOMDri: case X86::VPCOMUDri: 2169 case X86::VPCOMQri: case X86::VPCOMUQri: 2170 case X86::VPCOMWri: case X86::VPCOMUWri: { 2171 // Flip comparison mode immediate (if necessary). 2172 unsigned Imm = MI.getOperand(3).getImm() & 0x7; 2173 Imm = X86::getSwappedVPCOMImm(Imm); 2174 auto &WorkingMI = cloneIfNew(MI); 2175 WorkingMI.getOperand(3).setImm(Imm); 2176 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2177 OpIdx1, OpIdx2); 2178 } 2179 case X86::VCMPSDZrr: 2180 case X86::VCMPSSZrr: 2181 case X86::VCMPPDZrri: 2182 case X86::VCMPPSZrri: 2183 case X86::VCMPSHZrr: 2184 case X86::VCMPPHZrri: 2185 case X86::VCMPPHZ128rri: 2186 case X86::VCMPPHZ256rri: 2187 case X86::VCMPPDZ128rri: 2188 case X86::VCMPPSZ128rri: 2189 case X86::VCMPPDZ256rri: 2190 case X86::VCMPPSZ256rri: 2191 case X86::VCMPPDZrrik: 2192 case X86::VCMPPSZrrik: 2193 case X86::VCMPPDZ128rrik: 2194 case X86::VCMPPSZ128rrik: 2195 case X86::VCMPPDZ256rrik: 2196 case X86::VCMPPSZ256rrik: { 2197 unsigned Imm = 2198 MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 0x1f; 2199 Imm = X86::getSwappedVCMPImm(Imm); 2200 auto &WorkingMI = cloneIfNew(MI); 2201 WorkingMI.getOperand(MI.getNumExplicitOperands() - 1).setImm(Imm); 2202 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2203 OpIdx1, OpIdx2); 2204 } 2205 case X86::VPERM2F128rr: 2206 case X86::VPERM2I128rr: { 2207 // Flip permute source immediate. 2208 // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi. 2209 // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi. 2210 int8_t Imm = MI.getOperand(3).getImm() & 0xFF; 2211 auto &WorkingMI = cloneIfNew(MI); 2212 WorkingMI.getOperand(3).setImm(Imm ^ 0x22); 2213 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2214 OpIdx1, OpIdx2); 2215 } 2216 case X86::MOVHLPSrr: 2217 case X86::UNPCKHPDrr: 2218 case X86::VMOVHLPSrr: 2219 case X86::VUNPCKHPDrr: 2220 case X86::VMOVHLPSZrr: 2221 case X86::VUNPCKHPDZ128rr: { 2222 assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!"); 2223 2224 unsigned Opc = MI.getOpcode(); 2225 switch (Opc) { 2226 default: llvm_unreachable("Unreachable!"); 2227 case X86::MOVHLPSrr: Opc = X86::UNPCKHPDrr; break; 2228 case X86::UNPCKHPDrr: Opc = X86::MOVHLPSrr; break; 2229 case X86::VMOVHLPSrr: Opc = X86::VUNPCKHPDrr; break; 2230 case X86::VUNPCKHPDrr: Opc = X86::VMOVHLPSrr; break; 2231 case X86::VMOVHLPSZrr: Opc = X86::VUNPCKHPDZ128rr; break; 2232 case X86::VUNPCKHPDZ128rr: Opc = X86::VMOVHLPSZrr; break; 2233 } 2234 auto &WorkingMI = cloneIfNew(MI); 2235 WorkingMI.setDesc(get(Opc)); 2236 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2237 OpIdx1, OpIdx2); 2238 } 2239 case X86::CMOV16rr: case X86::CMOV32rr: case X86::CMOV64rr: { 2240 auto &WorkingMI = cloneIfNew(MI); 2241 unsigned OpNo = MI.getDesc().getNumOperands() - 1; 2242 X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm()); 2243 WorkingMI.getOperand(OpNo).setImm(X86::GetOppositeBranchCondition(CC)); 2244 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2245 OpIdx1, OpIdx2); 2246 } 2247 case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: 2248 case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: 2249 case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: 2250 case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: 2251 case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: 2252 case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: 2253 case X86::VPTERNLOGDZrrik: 2254 case X86::VPTERNLOGDZ128rrik: 2255 case X86::VPTERNLOGDZ256rrik: 2256 case X86::VPTERNLOGQZrrik: 2257 case X86::VPTERNLOGQZ128rrik: 2258 case X86::VPTERNLOGQZ256rrik: 2259 case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: 2260 case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: 2261 case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: 2262 case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: 2263 case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: 2264 case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: 2265 case X86::VPTERNLOGDZ128rmbi: 2266 case X86::VPTERNLOGDZ256rmbi: 2267 case X86::VPTERNLOGDZrmbi: 2268 case X86::VPTERNLOGQZ128rmbi: 2269 case X86::VPTERNLOGQZ256rmbi: 2270 case X86::VPTERNLOGQZrmbi: 2271 case X86::VPTERNLOGDZ128rmbikz: 2272 case X86::VPTERNLOGDZ256rmbikz: 2273 case X86::VPTERNLOGDZrmbikz: 2274 case X86::VPTERNLOGQZ128rmbikz: 2275 case X86::VPTERNLOGQZ256rmbikz: 2276 case X86::VPTERNLOGQZrmbikz: { 2277 auto &WorkingMI = cloneIfNew(MI); 2278 commuteVPTERNLOG(WorkingMI, OpIdx1, OpIdx2); 2279 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2280 OpIdx1, OpIdx2); 2281 } 2282 default: { 2283 if (isCommutableVPERMV3Instruction(MI.getOpcode())) { 2284 unsigned Opc = getCommutedVPERMV3Opcode(MI.getOpcode()); 2285 auto &WorkingMI = cloneIfNew(MI); 2286 WorkingMI.setDesc(get(Opc)); 2287 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2288 OpIdx1, OpIdx2); 2289 } 2290 2291 const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), 2292 MI.getDesc().TSFlags); 2293 if (FMA3Group) { 2294 unsigned Opc = 2295 getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group); 2296 auto &WorkingMI = cloneIfNew(MI); 2297 WorkingMI.setDesc(get(Opc)); 2298 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2299 OpIdx1, OpIdx2); 2300 } 2301 2302 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 2303 } 2304 } 2305 } 2306 2307 bool 2308 X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI, 2309 unsigned &SrcOpIdx1, 2310 unsigned &SrcOpIdx2, 2311 bool IsIntrinsic) const { 2312 uint64_t TSFlags = MI.getDesc().TSFlags; 2313 2314 unsigned FirstCommutableVecOp = 1; 2315 unsigned LastCommutableVecOp = 3; 2316 unsigned KMaskOp = -1U; 2317 if (X86II::isKMasked(TSFlags)) { 2318 // For k-zero-masked operations it is Ok to commute the first vector 2319 // operand. Unless this is an intrinsic instruction. 2320 // For regular k-masked operations a conservative choice is done as the 2321 // elements of the first vector operand, for which the corresponding bit 2322 // in the k-mask operand is set to 0, are copied to the result of the 2323 // instruction. 2324 // TODO/FIXME: The commute still may be legal if it is known that the 2325 // k-mask operand is set to either all ones or all zeroes. 2326 // It is also Ok to commute the 1st operand if all users of MI use only 2327 // the elements enabled by the k-mask operand. For example, 2328 // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i] 2329 // : v1[i]; 2330 // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 -> 2331 // // Ok, to commute v1 in FMADD213PSZrk. 2332 2333 // The k-mask operand has index = 2 for masked and zero-masked operations. 2334 KMaskOp = 2; 2335 2336 // The operand with index = 1 is used as a source for those elements for 2337 // which the corresponding bit in the k-mask is set to 0. 2338 if (X86II::isKMergeMasked(TSFlags) || IsIntrinsic) 2339 FirstCommutableVecOp = 3; 2340 2341 LastCommutableVecOp++; 2342 } else if (IsIntrinsic) { 2343 // Commuting the first operand of an intrinsic instruction isn't possible 2344 // unless we can prove that only the lowest element of the result is used. 2345 FirstCommutableVecOp = 2; 2346 } 2347 2348 if (isMem(MI, LastCommutableVecOp)) 2349 LastCommutableVecOp--; 2350 2351 // Only the first RegOpsNum operands are commutable. 2352 // Also, the value 'CommuteAnyOperandIndex' is valid here as it means 2353 // that the operand is not specified/fixed. 2354 if (SrcOpIdx1 != CommuteAnyOperandIndex && 2355 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp || 2356 SrcOpIdx1 == KMaskOp)) 2357 return false; 2358 if (SrcOpIdx2 != CommuteAnyOperandIndex && 2359 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp || 2360 SrcOpIdx2 == KMaskOp)) 2361 return false; 2362 2363 // Look for two different register operands assumed to be commutable 2364 // regardless of the FMA opcode. The FMA opcode is adjusted later. 2365 if (SrcOpIdx1 == CommuteAnyOperandIndex || 2366 SrcOpIdx2 == CommuteAnyOperandIndex) { 2367 unsigned CommutableOpIdx2 = SrcOpIdx2; 2368 2369 // At least one of operands to be commuted is not specified and 2370 // this method is free to choose appropriate commutable operands. 2371 if (SrcOpIdx1 == SrcOpIdx2) 2372 // Both of operands are not fixed. By default set one of commutable 2373 // operands to the last register operand of the instruction. 2374 CommutableOpIdx2 = LastCommutableVecOp; 2375 else if (SrcOpIdx2 == CommuteAnyOperandIndex) 2376 // Only one of operands is not fixed. 2377 CommutableOpIdx2 = SrcOpIdx1; 2378 2379 // CommutableOpIdx2 is well defined now. Let's choose another commutable 2380 // operand and assign its index to CommutableOpIdx1. 2381 Register Op2Reg = MI.getOperand(CommutableOpIdx2).getReg(); 2382 2383 unsigned CommutableOpIdx1; 2384 for (CommutableOpIdx1 = LastCommutableVecOp; 2385 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) { 2386 // Just ignore and skip the k-mask operand. 2387 if (CommutableOpIdx1 == KMaskOp) 2388 continue; 2389 2390 // The commuted operands must have different registers. 2391 // Otherwise, the commute transformation does not change anything and 2392 // is useless then. 2393 if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg()) 2394 break; 2395 } 2396 2397 // No appropriate commutable operands were found. 2398 if (CommutableOpIdx1 < FirstCommutableVecOp) 2399 return false; 2400 2401 // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2 2402 // to return those values. 2403 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2404 CommutableOpIdx1, CommutableOpIdx2)) 2405 return false; 2406 } 2407 2408 return true; 2409 } 2410 2411 bool X86InstrInfo::findCommutedOpIndices(const MachineInstr &MI, 2412 unsigned &SrcOpIdx1, 2413 unsigned &SrcOpIdx2) const { 2414 const MCInstrDesc &Desc = MI.getDesc(); 2415 if (!Desc.isCommutable()) 2416 return false; 2417 2418 switch (MI.getOpcode()) { 2419 case X86::CMPSDrr: 2420 case X86::CMPSSrr: 2421 case X86::CMPPDrri: 2422 case X86::CMPPSrri: 2423 case X86::VCMPSDrr: 2424 case X86::VCMPSSrr: 2425 case X86::VCMPPDrri: 2426 case X86::VCMPPSrri: 2427 case X86::VCMPPDYrri: 2428 case X86::VCMPPSYrri: 2429 case X86::VCMPSDZrr: 2430 case X86::VCMPSSZrr: 2431 case X86::VCMPPDZrri: 2432 case X86::VCMPPSZrri: 2433 case X86::VCMPSHZrr: 2434 case X86::VCMPPHZrri: 2435 case X86::VCMPPHZ128rri: 2436 case X86::VCMPPHZ256rri: 2437 case X86::VCMPPDZ128rri: 2438 case X86::VCMPPSZ128rri: 2439 case X86::VCMPPDZ256rri: 2440 case X86::VCMPPSZ256rri: 2441 case X86::VCMPPDZrrik: 2442 case X86::VCMPPSZrrik: 2443 case X86::VCMPPDZ128rrik: 2444 case X86::VCMPPSZ128rrik: 2445 case X86::VCMPPDZ256rrik: 2446 case X86::VCMPPSZ256rrik: { 2447 unsigned OpOffset = X86II::isKMasked(Desc.TSFlags) ? 1 : 0; 2448 2449 // Float comparison can be safely commuted for 2450 // Ordered/Unordered/Equal/NotEqual tests 2451 unsigned Imm = MI.getOperand(3 + OpOffset).getImm() & 0x7; 2452 switch (Imm) { 2453 default: 2454 // EVEX versions can be commuted. 2455 if ((Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX) 2456 break; 2457 return false; 2458 case 0x00: // EQUAL 2459 case 0x03: // UNORDERED 2460 case 0x04: // NOT EQUAL 2461 case 0x07: // ORDERED 2462 break; 2463 } 2464 2465 // The indices of the commutable operands are 1 and 2 (or 2 and 3 2466 // when masked). 2467 // Assign them to the returned operand indices here. 2468 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset, 2469 2 + OpOffset); 2470 } 2471 case X86::MOVSSrr: 2472 // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can 2473 // form sse4.1 blend. We assume VMOVSSrr/VMOVSDrr is always commutable since 2474 // AVX implies sse4.1. 2475 if (Subtarget.hasSSE41()) 2476 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2477 return false; 2478 case X86::SHUFPDrri: 2479 // We can commute this to MOVSD. 2480 if (MI.getOperand(3).getImm() == 0x02) 2481 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2482 return false; 2483 case X86::MOVHLPSrr: 2484 case X86::UNPCKHPDrr: 2485 case X86::VMOVHLPSrr: 2486 case X86::VUNPCKHPDrr: 2487 case X86::VMOVHLPSZrr: 2488 case X86::VUNPCKHPDZ128rr: 2489 if (Subtarget.hasSSE2()) 2490 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2491 return false; 2492 case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: 2493 case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: 2494 case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: 2495 case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: 2496 case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: 2497 case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: 2498 case X86::VPTERNLOGDZrrik: 2499 case X86::VPTERNLOGDZ128rrik: 2500 case X86::VPTERNLOGDZ256rrik: 2501 case X86::VPTERNLOGQZrrik: 2502 case X86::VPTERNLOGQZ128rrik: 2503 case X86::VPTERNLOGQZ256rrik: 2504 case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: 2505 case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: 2506 case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: 2507 case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: 2508 case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: 2509 case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: 2510 case X86::VPTERNLOGDZ128rmbi: 2511 case X86::VPTERNLOGDZ256rmbi: 2512 case X86::VPTERNLOGDZrmbi: 2513 case X86::VPTERNLOGQZ128rmbi: 2514 case X86::VPTERNLOGQZ256rmbi: 2515 case X86::VPTERNLOGQZrmbi: 2516 case X86::VPTERNLOGDZ128rmbikz: 2517 case X86::VPTERNLOGDZ256rmbikz: 2518 case X86::VPTERNLOGDZrmbikz: 2519 case X86::VPTERNLOGQZ128rmbikz: 2520 case X86::VPTERNLOGQZ256rmbikz: 2521 case X86::VPTERNLOGQZrmbikz: 2522 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2523 case X86::VPDPWSSDYrr: 2524 case X86::VPDPWSSDrr: 2525 case X86::VPDPWSSDSYrr: 2526 case X86::VPDPWSSDSrr: 2527 case X86::VPDPWSSDZ128r: 2528 case X86::VPDPWSSDZ128rk: 2529 case X86::VPDPWSSDZ128rkz: 2530 case X86::VPDPWSSDZ256r: 2531 case X86::VPDPWSSDZ256rk: 2532 case X86::VPDPWSSDZ256rkz: 2533 case X86::VPDPWSSDZr: 2534 case X86::VPDPWSSDZrk: 2535 case X86::VPDPWSSDZrkz: 2536 case X86::VPDPWSSDSZ128r: 2537 case X86::VPDPWSSDSZ128rk: 2538 case X86::VPDPWSSDSZ128rkz: 2539 case X86::VPDPWSSDSZ256r: 2540 case X86::VPDPWSSDSZ256rk: 2541 case X86::VPDPWSSDSZ256rkz: 2542 case X86::VPDPWSSDSZr: 2543 case X86::VPDPWSSDSZrk: 2544 case X86::VPDPWSSDSZrkz: 2545 case X86::VPMADD52HUQZ128r: 2546 case X86::VPMADD52HUQZ128rk: 2547 case X86::VPMADD52HUQZ128rkz: 2548 case X86::VPMADD52HUQZ256r: 2549 case X86::VPMADD52HUQZ256rk: 2550 case X86::VPMADD52HUQZ256rkz: 2551 case X86::VPMADD52HUQZr: 2552 case X86::VPMADD52HUQZrk: 2553 case X86::VPMADD52HUQZrkz: 2554 case X86::VPMADD52LUQZ128r: 2555 case X86::VPMADD52LUQZ128rk: 2556 case X86::VPMADD52LUQZ128rkz: 2557 case X86::VPMADD52LUQZ256r: 2558 case X86::VPMADD52LUQZ256rk: 2559 case X86::VPMADD52LUQZ256rkz: 2560 case X86::VPMADD52LUQZr: 2561 case X86::VPMADD52LUQZrk: 2562 case X86::VPMADD52LUQZrkz: 2563 case X86::VFMADDCPHZr: 2564 case X86::VFMADDCPHZrk: 2565 case X86::VFMADDCPHZrkz: 2566 case X86::VFMADDCPHZ128r: 2567 case X86::VFMADDCPHZ128rk: 2568 case X86::VFMADDCPHZ128rkz: 2569 case X86::VFMADDCPHZ256r: 2570 case X86::VFMADDCPHZ256rk: 2571 case X86::VFMADDCPHZ256rkz: 2572 case X86::VFMADDCSHZr: 2573 case X86::VFMADDCSHZrk: 2574 case X86::VFMADDCSHZrkz: { 2575 unsigned CommutableOpIdx1 = 2; 2576 unsigned CommutableOpIdx2 = 3; 2577 if (X86II::isKMasked(Desc.TSFlags)) { 2578 // Skip the mask register. 2579 ++CommutableOpIdx1; 2580 ++CommutableOpIdx2; 2581 } 2582 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2583 CommutableOpIdx1, CommutableOpIdx2)) 2584 return false; 2585 if (!MI.getOperand(SrcOpIdx1).isReg() || 2586 !MI.getOperand(SrcOpIdx2).isReg()) 2587 // No idea. 2588 return false; 2589 return true; 2590 } 2591 2592 default: 2593 const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), 2594 MI.getDesc().TSFlags); 2595 if (FMA3Group) 2596 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2, 2597 FMA3Group->isIntrinsic()); 2598 2599 // Handled masked instructions since we need to skip over the mask input 2600 // and the preserved input. 2601 if (X86II::isKMasked(Desc.TSFlags)) { 2602 // First assume that the first input is the mask operand and skip past it. 2603 unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1; 2604 unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2; 2605 // Check if the first input is tied. If there isn't one then we only 2606 // need to skip the mask operand which we did above. 2607 if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(), 2608 MCOI::TIED_TO) != -1)) { 2609 // If this is zero masking instruction with a tied operand, we need to 2610 // move the first index back to the first input since this must 2611 // be a 3 input instruction and we want the first two non-mask inputs. 2612 // Otherwise this is a 2 input instruction with a preserved input and 2613 // mask, so we need to move the indices to skip one more input. 2614 if (X86II::isKMergeMasked(Desc.TSFlags)) { 2615 ++CommutableOpIdx1; 2616 ++CommutableOpIdx2; 2617 } else { 2618 --CommutableOpIdx1; 2619 } 2620 } 2621 2622 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2623 CommutableOpIdx1, CommutableOpIdx2)) 2624 return false; 2625 2626 if (!MI.getOperand(SrcOpIdx1).isReg() || 2627 !MI.getOperand(SrcOpIdx2).isReg()) 2628 // No idea. 2629 return false; 2630 return true; 2631 } 2632 2633 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2634 } 2635 return false; 2636 } 2637 2638 static bool isConvertibleLEA(MachineInstr *MI) { 2639 unsigned Opcode = MI->getOpcode(); 2640 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r && 2641 Opcode != X86::LEA64_32r) 2642 return false; 2643 2644 const MachineOperand &Scale = MI->getOperand(1 + X86::AddrScaleAmt); 2645 const MachineOperand &Disp = MI->getOperand(1 + X86::AddrDisp); 2646 const MachineOperand &Segment = MI->getOperand(1 + X86::AddrSegmentReg); 2647 2648 if (Segment.getReg() != 0 || !Disp.isImm() || Disp.getImm() != 0 || 2649 Scale.getImm() > 1) 2650 return false; 2651 2652 return true; 2653 } 2654 2655 bool X86InstrInfo::hasCommutePreference(MachineInstr &MI, bool &Commute) const { 2656 // Currently we're interested in following sequence only. 2657 // r3 = lea r1, r2 2658 // r5 = add r3, r4 2659 // Both r3 and r4 are killed in add, we hope the add instruction has the 2660 // operand order 2661 // r5 = add r4, r3 2662 // So later in X86FixupLEAs the lea instruction can be rewritten as add. 2663 unsigned Opcode = MI.getOpcode(); 2664 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr) 2665 return false; 2666 2667 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 2668 Register Reg1 = MI.getOperand(1).getReg(); 2669 Register Reg2 = MI.getOperand(2).getReg(); 2670 2671 // Check if Reg1 comes from LEA in the same MBB. 2672 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg1)) { 2673 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) { 2674 Commute = true; 2675 return true; 2676 } 2677 } 2678 2679 // Check if Reg2 comes from LEA in the same MBB. 2680 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg2)) { 2681 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) { 2682 Commute = false; 2683 return true; 2684 } 2685 } 2686 2687 return false; 2688 } 2689 2690 int X86::getCondSrcNoFromDesc(const MCInstrDesc &MCID) { 2691 unsigned Opcode = MCID.getOpcode(); 2692 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isCMOVCC(Opcode))) 2693 return -1; 2694 // Assume that condition code is always the last use operand. 2695 unsigned NumUses = MCID.getNumOperands() - MCID.getNumDefs(); 2696 return NumUses - 1; 2697 } 2698 2699 X86::CondCode X86::getCondFromMI(const MachineInstr &MI) { 2700 const MCInstrDesc &MCID = MI.getDesc(); 2701 int CondNo = getCondSrcNoFromDesc(MCID); 2702 if (CondNo < 0) 2703 return X86::COND_INVALID; 2704 CondNo += MCID.getNumDefs(); 2705 return static_cast<X86::CondCode>(MI.getOperand(CondNo).getImm()); 2706 } 2707 2708 X86::CondCode X86::getCondFromBranch(const MachineInstr &MI) { 2709 return X86::isJCC(MI.getOpcode()) ? X86::getCondFromMI(MI) 2710 : X86::COND_INVALID; 2711 } 2712 2713 X86::CondCode X86::getCondFromSETCC(const MachineInstr &MI) { 2714 return X86::isSETCC(MI.getOpcode()) ? X86::getCondFromMI(MI) 2715 : X86::COND_INVALID; 2716 } 2717 2718 X86::CondCode X86::getCondFromCMov(const MachineInstr &MI) { 2719 return X86::isCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI) 2720 : X86::COND_INVALID; 2721 } 2722 2723 /// Return the inverse of the specified condition, 2724 /// e.g. turning COND_E to COND_NE. 2725 X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) { 2726 switch (CC) { 2727 default: llvm_unreachable("Illegal condition code!"); 2728 case X86::COND_E: return X86::COND_NE; 2729 case X86::COND_NE: return X86::COND_E; 2730 case X86::COND_L: return X86::COND_GE; 2731 case X86::COND_LE: return X86::COND_G; 2732 case X86::COND_G: return X86::COND_LE; 2733 case X86::COND_GE: return X86::COND_L; 2734 case X86::COND_B: return X86::COND_AE; 2735 case X86::COND_BE: return X86::COND_A; 2736 case X86::COND_A: return X86::COND_BE; 2737 case X86::COND_AE: return X86::COND_B; 2738 case X86::COND_S: return X86::COND_NS; 2739 case X86::COND_NS: return X86::COND_S; 2740 case X86::COND_P: return X86::COND_NP; 2741 case X86::COND_NP: return X86::COND_P; 2742 case X86::COND_O: return X86::COND_NO; 2743 case X86::COND_NO: return X86::COND_O; 2744 case X86::COND_NE_OR_P: return X86::COND_E_AND_NP; 2745 case X86::COND_E_AND_NP: return X86::COND_NE_OR_P; 2746 } 2747 } 2748 2749 /// Assuming the flags are set by MI(a,b), return the condition code if we 2750 /// modify the instructions such that flags are set by MI(b,a). 2751 static X86::CondCode getSwappedCondition(X86::CondCode CC) { 2752 switch (CC) { 2753 default: return X86::COND_INVALID; 2754 case X86::COND_E: return X86::COND_E; 2755 case X86::COND_NE: return X86::COND_NE; 2756 case X86::COND_L: return X86::COND_G; 2757 case X86::COND_LE: return X86::COND_GE; 2758 case X86::COND_G: return X86::COND_L; 2759 case X86::COND_GE: return X86::COND_LE; 2760 case X86::COND_B: return X86::COND_A; 2761 case X86::COND_BE: return X86::COND_AE; 2762 case X86::COND_A: return X86::COND_B; 2763 case X86::COND_AE: return X86::COND_BE; 2764 } 2765 } 2766 2767 std::pair<X86::CondCode, bool> 2768 X86::getX86ConditionCode(CmpInst::Predicate Predicate) { 2769 X86::CondCode CC = X86::COND_INVALID; 2770 bool NeedSwap = false; 2771 switch (Predicate) { 2772 default: break; 2773 // Floating-point Predicates 2774 case CmpInst::FCMP_UEQ: CC = X86::COND_E; break; 2775 case CmpInst::FCMP_OLT: NeedSwap = true; LLVM_FALLTHROUGH; 2776 case CmpInst::FCMP_OGT: CC = X86::COND_A; break; 2777 case CmpInst::FCMP_OLE: NeedSwap = true; LLVM_FALLTHROUGH; 2778 case CmpInst::FCMP_OGE: CC = X86::COND_AE; break; 2779 case CmpInst::FCMP_UGT: NeedSwap = true; LLVM_FALLTHROUGH; 2780 case CmpInst::FCMP_ULT: CC = X86::COND_B; break; 2781 case CmpInst::FCMP_UGE: NeedSwap = true; LLVM_FALLTHROUGH; 2782 case CmpInst::FCMP_ULE: CC = X86::COND_BE; break; 2783 case CmpInst::FCMP_ONE: CC = X86::COND_NE; break; 2784 case CmpInst::FCMP_UNO: CC = X86::COND_P; break; 2785 case CmpInst::FCMP_ORD: CC = X86::COND_NP; break; 2786 case CmpInst::FCMP_OEQ: LLVM_FALLTHROUGH; 2787 case CmpInst::FCMP_UNE: CC = X86::COND_INVALID; break; 2788 2789 // Integer Predicates 2790 case CmpInst::ICMP_EQ: CC = X86::COND_E; break; 2791 case CmpInst::ICMP_NE: CC = X86::COND_NE; break; 2792 case CmpInst::ICMP_UGT: CC = X86::COND_A; break; 2793 case CmpInst::ICMP_UGE: CC = X86::COND_AE; break; 2794 case CmpInst::ICMP_ULT: CC = X86::COND_B; break; 2795 case CmpInst::ICMP_ULE: CC = X86::COND_BE; break; 2796 case CmpInst::ICMP_SGT: CC = X86::COND_G; break; 2797 case CmpInst::ICMP_SGE: CC = X86::COND_GE; break; 2798 case CmpInst::ICMP_SLT: CC = X86::COND_L; break; 2799 case CmpInst::ICMP_SLE: CC = X86::COND_LE; break; 2800 } 2801 2802 return std::make_pair(CC, NeedSwap); 2803 } 2804 2805 /// Return a cmov opcode for the given register size in bytes, and operand type. 2806 unsigned X86::getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand) { 2807 switch(RegBytes) { 2808 default: llvm_unreachable("Illegal register size!"); 2809 case 2: return HasMemoryOperand ? X86::CMOV16rm : X86::CMOV16rr; 2810 case 4: return HasMemoryOperand ? X86::CMOV32rm : X86::CMOV32rr; 2811 case 8: return HasMemoryOperand ? X86::CMOV64rm : X86::CMOV64rr; 2812 } 2813 } 2814 2815 /// Get the VPCMP immediate for the given condition. 2816 unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) { 2817 switch (CC) { 2818 default: llvm_unreachable("Unexpected SETCC condition"); 2819 case ISD::SETNE: return 4; 2820 case ISD::SETEQ: return 0; 2821 case ISD::SETULT: 2822 case ISD::SETLT: return 1; 2823 case ISD::SETUGT: 2824 case ISD::SETGT: return 6; 2825 case ISD::SETUGE: 2826 case ISD::SETGE: return 5; 2827 case ISD::SETULE: 2828 case ISD::SETLE: return 2; 2829 } 2830 } 2831 2832 /// Get the VPCMP immediate if the operands are swapped. 2833 unsigned X86::getSwappedVPCMPImm(unsigned Imm) { 2834 switch (Imm) { 2835 default: llvm_unreachable("Unreachable!"); 2836 case 0x01: Imm = 0x06; break; // LT -> NLE 2837 case 0x02: Imm = 0x05; break; // LE -> NLT 2838 case 0x05: Imm = 0x02; break; // NLT -> LE 2839 case 0x06: Imm = 0x01; break; // NLE -> LT 2840 case 0x00: // EQ 2841 case 0x03: // FALSE 2842 case 0x04: // NE 2843 case 0x07: // TRUE 2844 break; 2845 } 2846 2847 return Imm; 2848 } 2849 2850 /// Get the VPCOM immediate if the operands are swapped. 2851 unsigned X86::getSwappedVPCOMImm(unsigned Imm) { 2852 switch (Imm) { 2853 default: llvm_unreachable("Unreachable!"); 2854 case 0x00: Imm = 0x02; break; // LT -> GT 2855 case 0x01: Imm = 0x03; break; // LE -> GE 2856 case 0x02: Imm = 0x00; break; // GT -> LT 2857 case 0x03: Imm = 0x01; break; // GE -> LE 2858 case 0x04: // EQ 2859 case 0x05: // NE 2860 case 0x06: // FALSE 2861 case 0x07: // TRUE 2862 break; 2863 } 2864 2865 return Imm; 2866 } 2867 2868 /// Get the VCMP immediate if the operands are swapped. 2869 unsigned X86::getSwappedVCMPImm(unsigned Imm) { 2870 // Only need the lower 2 bits to distinquish. 2871 switch (Imm & 0x3) { 2872 default: llvm_unreachable("Unreachable!"); 2873 case 0x00: case 0x03: 2874 // EQ/NE/TRUE/FALSE/ORD/UNORD don't change immediate when commuted. 2875 break; 2876 case 0x01: case 0x02: 2877 // Need to toggle bits 3:0. Bit 4 stays the same. 2878 Imm ^= 0xf; 2879 break; 2880 } 2881 2882 return Imm; 2883 } 2884 2885 /// Return true if the Reg is X87 register. 2886 static bool isX87Reg(unsigned Reg) { 2887 return (Reg == X86::FPCW || Reg == X86::FPSW || 2888 (Reg >= X86::ST0 && Reg <= X86::ST7)); 2889 } 2890 2891 /// check if the instruction is X87 instruction 2892 bool X86::isX87Instruction(MachineInstr &MI) { 2893 for (const MachineOperand &MO : MI.operands()) { 2894 if (!MO.isReg()) 2895 continue; 2896 if (isX87Reg(MO.getReg())) 2897 return true; 2898 } 2899 return false; 2900 } 2901 2902 bool X86InstrInfo::isUnconditionalTailCall(const MachineInstr &MI) const { 2903 switch (MI.getOpcode()) { 2904 case X86::TCRETURNdi: 2905 case X86::TCRETURNri: 2906 case X86::TCRETURNmi: 2907 case X86::TCRETURNdi64: 2908 case X86::TCRETURNri64: 2909 case X86::TCRETURNmi64: 2910 return true; 2911 default: 2912 return false; 2913 } 2914 } 2915 2916 bool X86InstrInfo::canMakeTailCallConditional( 2917 SmallVectorImpl<MachineOperand> &BranchCond, 2918 const MachineInstr &TailCall) const { 2919 if (TailCall.getOpcode() != X86::TCRETURNdi && 2920 TailCall.getOpcode() != X86::TCRETURNdi64) { 2921 // Only direct calls can be done with a conditional branch. 2922 return false; 2923 } 2924 2925 const MachineFunction *MF = TailCall.getParent()->getParent(); 2926 if (Subtarget.isTargetWin64() && MF->hasWinCFI()) { 2927 // Conditional tail calls confuse the Win64 unwinder. 2928 return false; 2929 } 2930 2931 assert(BranchCond.size() == 1); 2932 if (BranchCond[0].getImm() > X86::LAST_VALID_COND) { 2933 // Can't make a conditional tail call with this condition. 2934 return false; 2935 } 2936 2937 const X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); 2938 if (X86FI->getTCReturnAddrDelta() != 0 || 2939 TailCall.getOperand(1).getImm() != 0) { 2940 // A conditional tail call cannot do any stack adjustment. 2941 return false; 2942 } 2943 2944 return true; 2945 } 2946 2947 void X86InstrInfo::replaceBranchWithTailCall( 2948 MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &BranchCond, 2949 const MachineInstr &TailCall) const { 2950 assert(canMakeTailCallConditional(BranchCond, TailCall)); 2951 2952 MachineBasicBlock::iterator I = MBB.end(); 2953 while (I != MBB.begin()) { 2954 --I; 2955 if (I->isDebugInstr()) 2956 continue; 2957 if (!I->isBranch()) 2958 assert(0 && "Can't find the branch to replace!"); 2959 2960 X86::CondCode CC = X86::getCondFromBranch(*I); 2961 assert(BranchCond.size() == 1); 2962 if (CC != BranchCond[0].getImm()) 2963 continue; 2964 2965 break; 2966 } 2967 2968 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc 2969 : X86::TCRETURNdi64cc; 2970 2971 auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc)); 2972 MIB->addOperand(TailCall.getOperand(0)); // Destination. 2973 MIB.addImm(0); // Stack offset (not used). 2974 MIB->addOperand(BranchCond[0]); // Condition. 2975 MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters. 2976 2977 // Add implicit uses and defs of all live regs potentially clobbered by the 2978 // call. This way they still appear live across the call. 2979 LivePhysRegs LiveRegs(getRegisterInfo()); 2980 LiveRegs.addLiveOuts(MBB); 2981 SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers; 2982 LiveRegs.stepForward(*MIB, Clobbers); 2983 for (const auto &C : Clobbers) { 2984 MIB.addReg(C.first, RegState::Implicit); 2985 MIB.addReg(C.first, RegState::Implicit | RegState::Define); 2986 } 2987 2988 I->eraseFromParent(); 2989 } 2990 2991 // Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may 2992 // not be a fallthrough MBB now due to layout changes). Return nullptr if the 2993 // fallthrough MBB cannot be identified. 2994 static MachineBasicBlock *getFallThroughMBB(MachineBasicBlock *MBB, 2995 MachineBasicBlock *TBB) { 2996 // Look for non-EHPad successors other than TBB. If we find exactly one, it 2997 // is the fallthrough MBB. If we find zero, then TBB is both the target MBB 2998 // and fallthrough MBB. If we find more than one, we cannot identify the 2999 // fallthrough MBB and should return nullptr. 3000 MachineBasicBlock *FallthroughBB = nullptr; 3001 for (MachineBasicBlock *Succ : MBB->successors()) { 3002 if (Succ->isEHPad() || (Succ == TBB && FallthroughBB)) 3003 continue; 3004 // Return a nullptr if we found more than one fallthrough successor. 3005 if (FallthroughBB && FallthroughBB != TBB) 3006 return nullptr; 3007 FallthroughBB = Succ; 3008 } 3009 return FallthroughBB; 3010 } 3011 3012 bool X86InstrInfo::AnalyzeBranchImpl( 3013 MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, 3014 SmallVectorImpl<MachineOperand> &Cond, 3015 SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const { 3016 3017 // Start from the bottom of the block and work up, examining the 3018 // terminator instructions. 3019 MachineBasicBlock::iterator I = MBB.end(); 3020 MachineBasicBlock::iterator UnCondBrIter = MBB.end(); 3021 while (I != MBB.begin()) { 3022 --I; 3023 if (I->isDebugInstr()) 3024 continue; 3025 3026 // Working from the bottom, when we see a non-terminator instruction, we're 3027 // done. 3028 if (!isUnpredicatedTerminator(*I)) 3029 break; 3030 3031 // A terminator that isn't a branch can't easily be handled by this 3032 // analysis. 3033 if (!I->isBranch()) 3034 return true; 3035 3036 // Handle unconditional branches. 3037 if (I->getOpcode() == X86::JMP_1) { 3038 UnCondBrIter = I; 3039 3040 if (!AllowModify) { 3041 TBB = I->getOperand(0).getMBB(); 3042 continue; 3043 } 3044 3045 // If the block has any instructions after a JMP, delete them. 3046 MBB.erase(std::next(I), MBB.end()); 3047 3048 Cond.clear(); 3049 FBB = nullptr; 3050 3051 // Delete the JMP if it's equivalent to a fall-through. 3052 if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) { 3053 TBB = nullptr; 3054 I->eraseFromParent(); 3055 I = MBB.end(); 3056 UnCondBrIter = MBB.end(); 3057 continue; 3058 } 3059 3060 // TBB is used to indicate the unconditional destination. 3061 TBB = I->getOperand(0).getMBB(); 3062 continue; 3063 } 3064 3065 // Handle conditional branches. 3066 X86::CondCode BranchCode = X86::getCondFromBranch(*I); 3067 if (BranchCode == X86::COND_INVALID) 3068 return true; // Can't handle indirect branch. 3069 3070 // In practice we should never have an undef eflags operand, if we do 3071 // abort here as we are not prepared to preserve the flag. 3072 if (I->findRegisterUseOperand(X86::EFLAGS)->isUndef()) 3073 return true; 3074 3075 // Working from the bottom, handle the first conditional branch. 3076 if (Cond.empty()) { 3077 MachineBasicBlock *TargetBB = I->getOperand(0).getMBB(); 3078 if (AllowModify && UnCondBrIter != MBB.end() && 3079 MBB.isLayoutSuccessor(TargetBB)) { 3080 // If we can modify the code and it ends in something like: 3081 // 3082 // jCC L1 3083 // jmp L2 3084 // L1: 3085 // ... 3086 // L2: 3087 // 3088 // Then we can change this to: 3089 // 3090 // jnCC L2 3091 // L1: 3092 // ... 3093 // L2: 3094 // 3095 // Which is a bit more efficient. 3096 // We conditionally jump to the fall-through block. 3097 BranchCode = GetOppositeBranchCondition(BranchCode); 3098 MachineBasicBlock::iterator OldInst = I; 3099 3100 BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(X86::JCC_1)) 3101 .addMBB(UnCondBrIter->getOperand(0).getMBB()) 3102 .addImm(BranchCode); 3103 BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(X86::JMP_1)) 3104 .addMBB(TargetBB); 3105 3106 OldInst->eraseFromParent(); 3107 UnCondBrIter->eraseFromParent(); 3108 3109 // Restart the analysis. 3110 UnCondBrIter = MBB.end(); 3111 I = MBB.end(); 3112 continue; 3113 } 3114 3115 FBB = TBB; 3116 TBB = I->getOperand(0).getMBB(); 3117 Cond.push_back(MachineOperand::CreateImm(BranchCode)); 3118 CondBranches.push_back(&*I); 3119 continue; 3120 } 3121 3122 // Handle subsequent conditional branches. Only handle the case where all 3123 // conditional branches branch to the same destination and their condition 3124 // opcodes fit one of the special multi-branch idioms. 3125 assert(Cond.size() == 1); 3126 assert(TBB); 3127 3128 // If the conditions are the same, we can leave them alone. 3129 X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm(); 3130 auto NewTBB = I->getOperand(0).getMBB(); 3131 if (OldBranchCode == BranchCode && TBB == NewTBB) 3132 continue; 3133 3134 // If they differ, see if they fit one of the known patterns. Theoretically, 3135 // we could handle more patterns here, but we shouldn't expect to see them 3136 // if instruction selection has done a reasonable job. 3137 if (TBB == NewTBB && 3138 ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) || 3139 (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) { 3140 BranchCode = X86::COND_NE_OR_P; 3141 } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) || 3142 (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) { 3143 if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB))) 3144 return true; 3145 3146 // X86::COND_E_AND_NP usually has two different branch destinations. 3147 // 3148 // JP B1 3149 // JE B2 3150 // JMP B1 3151 // B1: 3152 // B2: 3153 // 3154 // Here this condition branches to B2 only if NP && E. It has another 3155 // equivalent form: 3156 // 3157 // JNE B1 3158 // JNP B2 3159 // JMP B1 3160 // B1: 3161 // B2: 3162 // 3163 // Similarly it branches to B2 only if E && NP. That is why this condition 3164 // is named with COND_E_AND_NP. 3165 BranchCode = X86::COND_E_AND_NP; 3166 } else 3167 return true; 3168 3169 // Update the MachineOperand. 3170 Cond[0].setImm(BranchCode); 3171 CondBranches.push_back(&*I); 3172 } 3173 3174 return false; 3175 } 3176 3177 bool X86InstrInfo::analyzeBranch(MachineBasicBlock &MBB, 3178 MachineBasicBlock *&TBB, 3179 MachineBasicBlock *&FBB, 3180 SmallVectorImpl<MachineOperand> &Cond, 3181 bool AllowModify) const { 3182 SmallVector<MachineInstr *, 4> CondBranches; 3183 return AnalyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify); 3184 } 3185 3186 bool X86InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB, 3187 MachineBranchPredicate &MBP, 3188 bool AllowModify) const { 3189 using namespace std::placeholders; 3190 3191 SmallVector<MachineOperand, 4> Cond; 3192 SmallVector<MachineInstr *, 4> CondBranches; 3193 if (AnalyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches, 3194 AllowModify)) 3195 return true; 3196 3197 if (Cond.size() != 1) 3198 return true; 3199 3200 assert(MBP.TrueDest && "expected!"); 3201 3202 if (!MBP.FalseDest) 3203 MBP.FalseDest = MBB.getNextNode(); 3204 3205 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3206 3207 MachineInstr *ConditionDef = nullptr; 3208 bool SingleUseCondition = true; 3209 3210 for (MachineInstr &MI : llvm::drop_begin(llvm::reverse(MBB))) { 3211 if (MI.modifiesRegister(X86::EFLAGS, TRI)) { 3212 ConditionDef = &MI; 3213 break; 3214 } 3215 3216 if (MI.readsRegister(X86::EFLAGS, TRI)) 3217 SingleUseCondition = false; 3218 } 3219 3220 if (!ConditionDef) 3221 return true; 3222 3223 if (SingleUseCondition) { 3224 for (auto *Succ : MBB.successors()) 3225 if (Succ->isLiveIn(X86::EFLAGS)) 3226 SingleUseCondition = false; 3227 } 3228 3229 MBP.ConditionDef = ConditionDef; 3230 MBP.SingleUseCondition = SingleUseCondition; 3231 3232 // Currently we only recognize the simple pattern: 3233 // 3234 // test %reg, %reg 3235 // je %label 3236 // 3237 const unsigned TestOpcode = 3238 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr; 3239 3240 if (ConditionDef->getOpcode() == TestOpcode && 3241 ConditionDef->getNumOperands() == 3 && 3242 ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) && 3243 (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) { 3244 MBP.LHS = ConditionDef->getOperand(0); 3245 MBP.RHS = MachineOperand::CreateImm(0); 3246 MBP.Predicate = Cond[0].getImm() == X86::COND_NE 3247 ? MachineBranchPredicate::PRED_NE 3248 : MachineBranchPredicate::PRED_EQ; 3249 return false; 3250 } 3251 3252 return true; 3253 } 3254 3255 unsigned X86InstrInfo::removeBranch(MachineBasicBlock &MBB, 3256 int *BytesRemoved) const { 3257 assert(!BytesRemoved && "code size not handled"); 3258 3259 MachineBasicBlock::iterator I = MBB.end(); 3260 unsigned Count = 0; 3261 3262 while (I != MBB.begin()) { 3263 --I; 3264 if (I->isDebugInstr()) 3265 continue; 3266 if (I->getOpcode() != X86::JMP_1 && 3267 X86::getCondFromBranch(*I) == X86::COND_INVALID) 3268 break; 3269 // Remove the branch. 3270 I->eraseFromParent(); 3271 I = MBB.end(); 3272 ++Count; 3273 } 3274 3275 return Count; 3276 } 3277 3278 unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB, 3279 MachineBasicBlock *TBB, 3280 MachineBasicBlock *FBB, 3281 ArrayRef<MachineOperand> Cond, 3282 const DebugLoc &DL, 3283 int *BytesAdded) const { 3284 // Shouldn't be a fall through. 3285 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 3286 assert((Cond.size() == 1 || Cond.size() == 0) && 3287 "X86 branch conditions have one component!"); 3288 assert(!BytesAdded && "code size not handled"); 3289 3290 if (Cond.empty()) { 3291 // Unconditional branch? 3292 assert(!FBB && "Unconditional branch with multiple successors!"); 3293 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB); 3294 return 1; 3295 } 3296 3297 // If FBB is null, it is implied to be a fall-through block. 3298 bool FallThru = FBB == nullptr; 3299 3300 // Conditional branch. 3301 unsigned Count = 0; 3302 X86::CondCode CC = (X86::CondCode)Cond[0].getImm(); 3303 switch (CC) { 3304 case X86::COND_NE_OR_P: 3305 // Synthesize NE_OR_P with two branches. 3306 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NE); 3307 ++Count; 3308 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_P); 3309 ++Count; 3310 break; 3311 case X86::COND_E_AND_NP: 3312 // Use the next block of MBB as FBB if it is null. 3313 if (FBB == nullptr) { 3314 FBB = getFallThroughMBB(&MBB, TBB); 3315 assert(FBB && "MBB cannot be the last block in function when the false " 3316 "body is a fall-through."); 3317 } 3318 // Synthesize COND_E_AND_NP with two branches. 3319 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(FBB).addImm(X86::COND_NE); 3320 ++Count; 3321 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NP); 3322 ++Count; 3323 break; 3324 default: { 3325 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(CC); 3326 ++Count; 3327 } 3328 } 3329 if (!FallThru) { 3330 // Two-way Conditional branch. Insert the second branch. 3331 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB); 3332 ++Count; 3333 } 3334 return Count; 3335 } 3336 3337 bool X86InstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 3338 ArrayRef<MachineOperand> Cond, 3339 Register DstReg, Register TrueReg, 3340 Register FalseReg, int &CondCycles, 3341 int &TrueCycles, int &FalseCycles) const { 3342 // Not all subtargets have cmov instructions. 3343 if (!Subtarget.canUseCMOV()) 3344 return false; 3345 if (Cond.size() != 1) 3346 return false; 3347 // We cannot do the composite conditions, at least not in SSA form. 3348 if ((X86::CondCode)Cond[0].getImm() > X86::LAST_VALID_COND) 3349 return false; 3350 3351 // Check register classes. 3352 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3353 const TargetRegisterClass *RC = 3354 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 3355 if (!RC) 3356 return false; 3357 3358 // We have cmov instructions for 16, 32, and 64 bit general purpose registers. 3359 if (X86::GR16RegClass.hasSubClassEq(RC) || 3360 X86::GR32RegClass.hasSubClassEq(RC) || 3361 X86::GR64RegClass.hasSubClassEq(RC)) { 3362 // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy 3363 // Bridge. Probably Ivy Bridge as well. 3364 CondCycles = 2; 3365 TrueCycles = 2; 3366 FalseCycles = 2; 3367 return true; 3368 } 3369 3370 // Can't do vectors. 3371 return false; 3372 } 3373 3374 void X86InstrInfo::insertSelect(MachineBasicBlock &MBB, 3375 MachineBasicBlock::iterator I, 3376 const DebugLoc &DL, Register DstReg, 3377 ArrayRef<MachineOperand> Cond, Register TrueReg, 3378 Register FalseReg) const { 3379 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3380 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); 3381 const TargetRegisterClass &RC = *MRI.getRegClass(DstReg); 3382 assert(Cond.size() == 1 && "Invalid Cond array"); 3383 unsigned Opc = X86::getCMovOpcode(TRI.getRegSizeInBits(RC) / 8, 3384 false /*HasMemoryOperand*/); 3385 BuildMI(MBB, I, DL, get(Opc), DstReg) 3386 .addReg(FalseReg) 3387 .addReg(TrueReg) 3388 .addImm(Cond[0].getImm()); 3389 } 3390 3391 /// Test if the given register is a physical h register. 3392 static bool isHReg(unsigned Reg) { 3393 return X86::GR8_ABCD_HRegClass.contains(Reg); 3394 } 3395 3396 // Try and copy between VR128/VR64 and GR64 registers. 3397 static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg, 3398 const X86Subtarget &Subtarget) { 3399 bool HasAVX = Subtarget.hasAVX(); 3400 bool HasAVX512 = Subtarget.hasAVX512(); 3401 3402 // SrcReg(MaskReg) -> DestReg(GR64) 3403 // SrcReg(MaskReg) -> DestReg(GR32) 3404 3405 // All KMASK RegClasses hold the same k registers, can be tested against anyone. 3406 if (X86::VK16RegClass.contains(SrcReg)) { 3407 if (X86::GR64RegClass.contains(DestReg)) { 3408 assert(Subtarget.hasBWI()); 3409 return X86::KMOVQrk; 3410 } 3411 if (X86::GR32RegClass.contains(DestReg)) 3412 return Subtarget.hasBWI() ? X86::KMOVDrk : X86::KMOVWrk; 3413 } 3414 3415 // SrcReg(GR64) -> DestReg(MaskReg) 3416 // SrcReg(GR32) -> DestReg(MaskReg) 3417 3418 // All KMASK RegClasses hold the same k registers, can be tested against anyone. 3419 if (X86::VK16RegClass.contains(DestReg)) { 3420 if (X86::GR64RegClass.contains(SrcReg)) { 3421 assert(Subtarget.hasBWI()); 3422 return X86::KMOVQkr; 3423 } 3424 if (X86::GR32RegClass.contains(SrcReg)) 3425 return Subtarget.hasBWI() ? X86::KMOVDkr : X86::KMOVWkr; 3426 } 3427 3428 3429 // SrcReg(VR128) -> DestReg(GR64) 3430 // SrcReg(VR64) -> DestReg(GR64) 3431 // SrcReg(GR64) -> DestReg(VR128) 3432 // SrcReg(GR64) -> DestReg(VR64) 3433 3434 if (X86::GR64RegClass.contains(DestReg)) { 3435 if (X86::VR128XRegClass.contains(SrcReg)) 3436 // Copy from a VR128 register to a GR64 register. 3437 return HasAVX512 ? X86::VMOVPQIto64Zrr : 3438 HasAVX ? X86::VMOVPQIto64rr : 3439 X86::MOVPQIto64rr; 3440 if (X86::VR64RegClass.contains(SrcReg)) 3441 // Copy from a VR64 register to a GR64 register. 3442 return X86::MMX_MOVD64from64rr; 3443 } else if (X86::GR64RegClass.contains(SrcReg)) { 3444 // Copy from a GR64 register to a VR128 register. 3445 if (X86::VR128XRegClass.contains(DestReg)) 3446 return HasAVX512 ? X86::VMOV64toPQIZrr : 3447 HasAVX ? X86::VMOV64toPQIrr : 3448 X86::MOV64toPQIrr; 3449 // Copy from a GR64 register to a VR64 register. 3450 if (X86::VR64RegClass.contains(DestReg)) 3451 return X86::MMX_MOVD64to64rr; 3452 } 3453 3454 // SrcReg(VR128) -> DestReg(GR32) 3455 // SrcReg(GR32) -> DestReg(VR128) 3456 3457 if (X86::GR32RegClass.contains(DestReg) && 3458 X86::VR128XRegClass.contains(SrcReg)) 3459 // Copy from a VR128 register to a GR32 register. 3460 return HasAVX512 ? X86::VMOVPDI2DIZrr : 3461 HasAVX ? X86::VMOVPDI2DIrr : 3462 X86::MOVPDI2DIrr; 3463 3464 if (X86::VR128XRegClass.contains(DestReg) && 3465 X86::GR32RegClass.contains(SrcReg)) 3466 // Copy from a VR128 register to a VR128 register. 3467 return HasAVX512 ? X86::VMOVDI2PDIZrr : 3468 HasAVX ? X86::VMOVDI2PDIrr : 3469 X86::MOVDI2PDIrr; 3470 return 0; 3471 } 3472 3473 void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB, 3474 MachineBasicBlock::iterator MI, 3475 const DebugLoc &DL, MCRegister DestReg, 3476 MCRegister SrcReg, bool KillSrc) const { 3477 // First deal with the normal symmetric copies. 3478 bool HasAVX = Subtarget.hasAVX(); 3479 bool HasVLX = Subtarget.hasVLX(); 3480 unsigned Opc = 0; 3481 if (X86::GR64RegClass.contains(DestReg, SrcReg)) 3482 Opc = X86::MOV64rr; 3483 else if (X86::GR32RegClass.contains(DestReg, SrcReg)) 3484 Opc = X86::MOV32rr; 3485 else if (X86::GR16RegClass.contains(DestReg, SrcReg)) 3486 Opc = X86::MOV16rr; 3487 else if (X86::GR8RegClass.contains(DestReg, SrcReg)) { 3488 // Copying to or from a physical H register on x86-64 requires a NOREX 3489 // move. Otherwise use a normal move. 3490 if ((isHReg(DestReg) || isHReg(SrcReg)) && 3491 Subtarget.is64Bit()) { 3492 Opc = X86::MOV8rr_NOREX; 3493 // Both operands must be encodable without an REX prefix. 3494 assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && 3495 "8-bit H register can not be copied outside GR8_NOREX"); 3496 } else 3497 Opc = X86::MOV8rr; 3498 } 3499 else if (X86::VR64RegClass.contains(DestReg, SrcReg)) 3500 Opc = X86::MMX_MOVQ64rr; 3501 else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) { 3502 if (HasVLX) 3503 Opc = X86::VMOVAPSZ128rr; 3504 else if (X86::VR128RegClass.contains(DestReg, SrcReg)) 3505 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr; 3506 else { 3507 // If this an extended register and we don't have VLX we need to use a 3508 // 512-bit move. 3509 Opc = X86::VMOVAPSZrr; 3510 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3511 DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, 3512 &X86::VR512RegClass); 3513 SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, 3514 &X86::VR512RegClass); 3515 } 3516 } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) { 3517 if (HasVLX) 3518 Opc = X86::VMOVAPSZ256rr; 3519 else if (X86::VR256RegClass.contains(DestReg, SrcReg)) 3520 Opc = X86::VMOVAPSYrr; 3521 else { 3522 // If this an extended register and we don't have VLX we need to use a 3523 // 512-bit move. 3524 Opc = X86::VMOVAPSZrr; 3525 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3526 DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, 3527 &X86::VR512RegClass); 3528 SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, 3529 &X86::VR512RegClass); 3530 } 3531 } else if (X86::VR512RegClass.contains(DestReg, SrcReg)) 3532 Opc = X86::VMOVAPSZrr; 3533 // All KMASK RegClasses hold the same k registers, can be tested against anyone. 3534 else if (X86::VK16RegClass.contains(DestReg, SrcReg)) 3535 Opc = Subtarget.hasBWI() ? X86::KMOVQkk : X86::KMOVWkk; 3536 if (!Opc) 3537 Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget); 3538 3539 if (Opc) { 3540 BuildMI(MBB, MI, DL, get(Opc), DestReg) 3541 .addReg(SrcReg, getKillRegState(KillSrc)); 3542 return; 3543 } 3544 3545 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) { 3546 // FIXME: We use a fatal error here because historically LLVM has tried 3547 // lower some of these physreg copies and we want to ensure we get 3548 // reasonable bug reports if someone encounters a case no other testing 3549 // found. This path should be removed after the LLVM 7 release. 3550 report_fatal_error("Unable to copy EFLAGS physical register!"); 3551 } 3552 3553 LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to " 3554 << RI.getName(DestReg) << '\n'); 3555 report_fatal_error("Cannot emit physreg copy instruction"); 3556 } 3557 3558 Optional<DestSourcePair> 3559 X86InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { 3560 if (MI.isMoveReg()) 3561 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 3562 return None; 3563 } 3564 3565 static unsigned getLoadStoreRegOpcode(Register Reg, 3566 const TargetRegisterClass *RC, 3567 bool IsStackAligned, 3568 const X86Subtarget &STI, bool load) { 3569 bool HasAVX = STI.hasAVX(); 3570 bool HasAVX512 = STI.hasAVX512(); 3571 bool HasVLX = STI.hasVLX(); 3572 3573 switch (STI.getRegisterInfo()->getSpillSize(*RC)) { 3574 default: 3575 llvm_unreachable("Unknown spill size"); 3576 case 1: 3577 assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass"); 3578 if (STI.is64Bit()) 3579 // Copying to or from a physical H register on x86-64 requires a NOREX 3580 // move. Otherwise use a normal move. 3581 if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC)) 3582 return load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX; 3583 return load ? X86::MOV8rm : X86::MOV8mr; 3584 case 2: 3585 if (X86::VK16RegClass.hasSubClassEq(RC)) 3586 return load ? X86::KMOVWkm : X86::KMOVWmk; 3587 assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass"); 3588 return load ? X86::MOV16rm : X86::MOV16mr; 3589 case 4: 3590 if (X86::GR32RegClass.hasSubClassEq(RC)) 3591 return load ? X86::MOV32rm : X86::MOV32mr; 3592 if (X86::FR32XRegClass.hasSubClassEq(RC)) 3593 return load ? 3594 (HasAVX512 ? X86::VMOVSSZrm_alt : 3595 HasAVX ? X86::VMOVSSrm_alt : 3596 X86::MOVSSrm_alt) : 3597 (HasAVX512 ? X86::VMOVSSZmr : 3598 HasAVX ? X86::VMOVSSmr : 3599 X86::MOVSSmr); 3600 if (X86::RFP32RegClass.hasSubClassEq(RC)) 3601 return load ? X86::LD_Fp32m : X86::ST_Fp32m; 3602 if (X86::VK32RegClass.hasSubClassEq(RC)) { 3603 assert(STI.hasBWI() && "KMOVD requires BWI"); 3604 return load ? X86::KMOVDkm : X86::KMOVDmk; 3605 } 3606 // All of these mask pair classes have the same spill size, the same kind 3607 // of kmov instructions can be used with all of them. 3608 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) || 3609 X86::VK2PAIRRegClass.hasSubClassEq(RC) || 3610 X86::VK4PAIRRegClass.hasSubClassEq(RC) || 3611 X86::VK8PAIRRegClass.hasSubClassEq(RC) || 3612 X86::VK16PAIRRegClass.hasSubClassEq(RC)) 3613 return load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE; 3614 if ((X86::FR16RegClass.hasSubClassEq(RC) || 3615 X86::FR16XRegClass.hasSubClassEq(RC)) && 3616 STI.hasFP16()) 3617 return load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr; 3618 llvm_unreachable("Unknown 4-byte regclass"); 3619 case 8: 3620 if (X86::GR64RegClass.hasSubClassEq(RC)) 3621 return load ? X86::MOV64rm : X86::MOV64mr; 3622 if (X86::FR64XRegClass.hasSubClassEq(RC)) 3623 return load ? 3624 (HasAVX512 ? X86::VMOVSDZrm_alt : 3625 HasAVX ? X86::VMOVSDrm_alt : 3626 X86::MOVSDrm_alt) : 3627 (HasAVX512 ? X86::VMOVSDZmr : 3628 HasAVX ? X86::VMOVSDmr : 3629 X86::MOVSDmr); 3630 if (X86::VR64RegClass.hasSubClassEq(RC)) 3631 return load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr; 3632 if (X86::RFP64RegClass.hasSubClassEq(RC)) 3633 return load ? X86::LD_Fp64m : X86::ST_Fp64m; 3634 if (X86::VK64RegClass.hasSubClassEq(RC)) { 3635 assert(STI.hasBWI() && "KMOVQ requires BWI"); 3636 return load ? X86::KMOVQkm : X86::KMOVQmk; 3637 } 3638 llvm_unreachable("Unknown 8-byte regclass"); 3639 case 10: 3640 assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass"); 3641 return load ? X86::LD_Fp80m : X86::ST_FpP80m; 3642 case 16: { 3643 if (X86::VR128XRegClass.hasSubClassEq(RC)) { 3644 // If stack is realigned we can use aligned stores. 3645 if (IsStackAligned) 3646 return load ? 3647 (HasVLX ? X86::VMOVAPSZ128rm : 3648 HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX : 3649 HasAVX ? X86::VMOVAPSrm : 3650 X86::MOVAPSrm): 3651 (HasVLX ? X86::VMOVAPSZ128mr : 3652 HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX : 3653 HasAVX ? X86::VMOVAPSmr : 3654 X86::MOVAPSmr); 3655 else 3656 return load ? 3657 (HasVLX ? X86::VMOVUPSZ128rm : 3658 HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX : 3659 HasAVX ? X86::VMOVUPSrm : 3660 X86::MOVUPSrm): 3661 (HasVLX ? X86::VMOVUPSZ128mr : 3662 HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX : 3663 HasAVX ? X86::VMOVUPSmr : 3664 X86::MOVUPSmr); 3665 } 3666 llvm_unreachable("Unknown 16-byte regclass"); 3667 } 3668 case 32: 3669 assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass"); 3670 // If stack is realigned we can use aligned stores. 3671 if (IsStackAligned) 3672 return load ? 3673 (HasVLX ? X86::VMOVAPSZ256rm : 3674 HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX : 3675 X86::VMOVAPSYrm) : 3676 (HasVLX ? X86::VMOVAPSZ256mr : 3677 HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX : 3678 X86::VMOVAPSYmr); 3679 else 3680 return load ? 3681 (HasVLX ? X86::VMOVUPSZ256rm : 3682 HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX : 3683 X86::VMOVUPSYrm) : 3684 (HasVLX ? X86::VMOVUPSZ256mr : 3685 HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX : 3686 X86::VMOVUPSYmr); 3687 case 64: 3688 assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass"); 3689 assert(STI.hasAVX512() && "Using 512-bit register requires AVX512"); 3690 if (IsStackAligned) 3691 return load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; 3692 else 3693 return load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; 3694 } 3695 } 3696 3697 Optional<ExtAddrMode> 3698 X86InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI, 3699 const TargetRegisterInfo *TRI) const { 3700 const MCInstrDesc &Desc = MemI.getDesc(); 3701 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); 3702 if (MemRefBegin < 0) 3703 return None; 3704 3705 MemRefBegin += X86II::getOperandBias(Desc); 3706 3707 auto &BaseOp = MemI.getOperand(MemRefBegin + X86::AddrBaseReg); 3708 if (!BaseOp.isReg()) // Can be an MO_FrameIndex 3709 return None; 3710 3711 const MachineOperand &DispMO = MemI.getOperand(MemRefBegin + X86::AddrDisp); 3712 // Displacement can be symbolic 3713 if (!DispMO.isImm()) 3714 return None; 3715 3716 ExtAddrMode AM; 3717 AM.BaseReg = BaseOp.getReg(); 3718 AM.ScaledReg = MemI.getOperand(MemRefBegin + X86::AddrIndexReg).getReg(); 3719 AM.Scale = MemI.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm(); 3720 AM.Displacement = DispMO.getImm(); 3721 return AM; 3722 } 3723 3724 bool X86InstrInfo::verifyInstruction(const MachineInstr &MI, 3725 StringRef &ErrInfo) const { 3726 Optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MI, nullptr); 3727 if (!AMOrNone) 3728 return true; 3729 3730 ExtAddrMode AM = *AMOrNone; 3731 3732 if (AM.ScaledReg != X86::NoRegister) { 3733 switch (AM.Scale) { 3734 case 1: 3735 case 2: 3736 case 4: 3737 case 8: 3738 break; 3739 default: 3740 ErrInfo = "Scale factor in address must be 1, 2, 4 or 8"; 3741 return false; 3742 } 3743 } 3744 if (!isInt<32>(AM.Displacement)) { 3745 ErrInfo = "Displacement in address must fit into 32-bit signed " 3746 "integer"; 3747 return false; 3748 } 3749 3750 return true; 3751 } 3752 3753 bool X86InstrInfo::getConstValDefinedInReg(const MachineInstr &MI, 3754 const Register Reg, 3755 int64_t &ImmVal) const { 3756 if (MI.getOpcode() != X86::MOV32ri && MI.getOpcode() != X86::MOV64ri) 3757 return false; 3758 // Mov Src can be a global address. 3759 if (!MI.getOperand(1).isImm() || MI.getOperand(0).getReg() != Reg) 3760 return false; 3761 ImmVal = MI.getOperand(1).getImm(); 3762 return true; 3763 } 3764 3765 bool X86InstrInfo::preservesZeroValueInReg( 3766 const MachineInstr *MI, const Register NullValueReg, 3767 const TargetRegisterInfo *TRI) const { 3768 if (!MI->modifiesRegister(NullValueReg, TRI)) 3769 return true; 3770 switch (MI->getOpcode()) { 3771 // Shift right/left of a null unto itself is still a null, i.e. rax = shl rax 3772 // X. 3773 case X86::SHR64ri: 3774 case X86::SHR32ri: 3775 case X86::SHL64ri: 3776 case X86::SHL32ri: 3777 assert(MI->getOperand(0).isDef() && MI->getOperand(1).isUse() && 3778 "expected for shift opcode!"); 3779 return MI->getOperand(0).getReg() == NullValueReg && 3780 MI->getOperand(1).getReg() == NullValueReg; 3781 // Zero extend of a sub-reg of NullValueReg into itself does not change the 3782 // null value. 3783 case X86::MOV32rr: 3784 return llvm::all_of(MI->operands(), [&](const MachineOperand &MO) { 3785 return TRI->isSubRegisterEq(NullValueReg, MO.getReg()); 3786 }); 3787 default: 3788 return false; 3789 } 3790 llvm_unreachable("Should be handled above!"); 3791 } 3792 3793 bool X86InstrInfo::getMemOperandsWithOffsetWidth( 3794 const MachineInstr &MemOp, SmallVectorImpl<const MachineOperand *> &BaseOps, 3795 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 3796 const TargetRegisterInfo *TRI) const { 3797 const MCInstrDesc &Desc = MemOp.getDesc(); 3798 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); 3799 if (MemRefBegin < 0) 3800 return false; 3801 3802 MemRefBegin += X86II::getOperandBias(Desc); 3803 3804 const MachineOperand *BaseOp = 3805 &MemOp.getOperand(MemRefBegin + X86::AddrBaseReg); 3806 if (!BaseOp->isReg()) // Can be an MO_FrameIndex 3807 return false; 3808 3809 if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1) 3810 return false; 3811 3812 if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() != 3813 X86::NoRegister) 3814 return false; 3815 3816 const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp); 3817 3818 // Displacement can be symbolic 3819 if (!DispMO.isImm()) 3820 return false; 3821 3822 Offset = DispMO.getImm(); 3823 3824 if (!BaseOp->isReg()) 3825 return false; 3826 3827 OffsetIsScalable = false; 3828 // FIXME: Relying on memoperands() may not be right thing to do here. Check 3829 // with X86 maintainers, and fix it accordingly. For now, it is ok, since 3830 // there is no use of `Width` for X86 back-end at the moment. 3831 Width = 3832 !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize() : 0; 3833 BaseOps.push_back(BaseOp); 3834 return true; 3835 } 3836 3837 static unsigned getStoreRegOpcode(Register SrcReg, 3838 const TargetRegisterClass *RC, 3839 bool IsStackAligned, 3840 const X86Subtarget &STI) { 3841 return getLoadStoreRegOpcode(SrcReg, RC, IsStackAligned, STI, false); 3842 } 3843 3844 static unsigned getLoadRegOpcode(Register DestReg, 3845 const TargetRegisterClass *RC, 3846 bool IsStackAligned, const X86Subtarget &STI) { 3847 return getLoadStoreRegOpcode(DestReg, RC, IsStackAligned, STI, true); 3848 } 3849 3850 void X86InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 3851 MachineBasicBlock::iterator MI, 3852 Register SrcReg, bool isKill, int FrameIdx, 3853 const TargetRegisterClass *RC, 3854 const TargetRegisterInfo *TRI) const { 3855 const MachineFunction &MF = *MBB.getParent(); 3856 const MachineFrameInfo &MFI = MF.getFrameInfo(); 3857 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo(); 3858 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && 3859 "Stack slot too small for store"); 3860 if (RC->getID() == X86::TILERegClassID) { 3861 unsigned Opc = X86::TILESTORED; 3862 // tilestored %tmm, (%sp, %idx) 3863 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 3864 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64); 3865 MachineInstr *NewMI = 3866 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx) 3867 .addReg(SrcReg, getKillRegState(isKill)); 3868 MachineOperand &MO = NewMI->getOperand(2); 3869 MO.setReg(VirtReg); 3870 MO.setIsKill(true); 3871 } else if ((RC->getID() == X86::FR16RegClassID || 3872 RC->getID() == X86::FR16XRegClassID) && 3873 !Subtarget.hasFP16()) { 3874 unsigned Opc = Subtarget.hasAVX512() ? X86::VMOVSSZmr 3875 : Subtarget.hasAVX() ? X86::VMOVSSmr 3876 : X86::MOVSSmr; 3877 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx) 3878 .addReg(SrcReg, getKillRegState(isKill)); 3879 } else { 3880 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); 3881 bool isAligned = 3882 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) || 3883 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx)); 3884 unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget); 3885 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx) 3886 .addReg(SrcReg, getKillRegState(isKill)); 3887 } 3888 } 3889 3890 void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 3891 MachineBasicBlock::iterator MI, 3892 Register DestReg, int FrameIdx, 3893 const TargetRegisterClass *RC, 3894 const TargetRegisterInfo *TRI) const { 3895 const MachineFunction &MF = *MBB.getParent(); 3896 const MachineFrameInfo &MFI = MF.getFrameInfo(); 3897 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && 3898 "Load size exceeds stack slot"); 3899 if (RC->getID() == X86::TILERegClassID) { 3900 unsigned Opc = X86::TILELOADD; 3901 // tileloadd (%sp, %idx), %tmm 3902 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo(); 3903 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 3904 MachineInstr *NewMI = 3905 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64); 3906 NewMI = addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), 3907 FrameIdx); 3908 MachineOperand &MO = NewMI->getOperand(3); 3909 MO.setReg(VirtReg); 3910 MO.setIsKill(true); 3911 } else if ((RC->getID() == X86::FR16RegClassID || 3912 RC->getID() == X86::FR16XRegClassID) && 3913 !Subtarget.hasFP16()) { 3914 unsigned Opc = Subtarget.hasAVX512() ? X86::VMOVSSZrm 3915 : Subtarget.hasAVX() ? X86::VMOVSSrm 3916 : X86::MOVSSrm; 3917 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), 3918 FrameIdx); 3919 } else { 3920 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); 3921 bool isAligned = 3922 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) || 3923 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx)); 3924 unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget); 3925 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), 3926 FrameIdx); 3927 } 3928 } 3929 3930 bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 3931 Register &SrcReg2, int64_t &CmpMask, 3932 int64_t &CmpValue) const { 3933 switch (MI.getOpcode()) { 3934 default: break; 3935 case X86::CMP64ri32: 3936 case X86::CMP64ri8: 3937 case X86::CMP32ri: 3938 case X86::CMP32ri8: 3939 case X86::CMP16ri: 3940 case X86::CMP16ri8: 3941 case X86::CMP8ri: 3942 SrcReg = MI.getOperand(0).getReg(); 3943 SrcReg2 = 0; 3944 if (MI.getOperand(1).isImm()) { 3945 CmpMask = ~0; 3946 CmpValue = MI.getOperand(1).getImm(); 3947 } else { 3948 CmpMask = CmpValue = 0; 3949 } 3950 return true; 3951 // A SUB can be used to perform comparison. 3952 case X86::SUB64rm: 3953 case X86::SUB32rm: 3954 case X86::SUB16rm: 3955 case X86::SUB8rm: 3956 SrcReg = MI.getOperand(1).getReg(); 3957 SrcReg2 = 0; 3958 CmpMask = 0; 3959 CmpValue = 0; 3960 return true; 3961 case X86::SUB64rr: 3962 case X86::SUB32rr: 3963 case X86::SUB16rr: 3964 case X86::SUB8rr: 3965 SrcReg = MI.getOperand(1).getReg(); 3966 SrcReg2 = MI.getOperand(2).getReg(); 3967 CmpMask = 0; 3968 CmpValue = 0; 3969 return true; 3970 case X86::SUB64ri32: 3971 case X86::SUB64ri8: 3972 case X86::SUB32ri: 3973 case X86::SUB32ri8: 3974 case X86::SUB16ri: 3975 case X86::SUB16ri8: 3976 case X86::SUB8ri: 3977 SrcReg = MI.getOperand(1).getReg(); 3978 SrcReg2 = 0; 3979 if (MI.getOperand(2).isImm()) { 3980 CmpMask = ~0; 3981 CmpValue = MI.getOperand(2).getImm(); 3982 } else { 3983 CmpMask = CmpValue = 0; 3984 } 3985 return true; 3986 case X86::CMP64rr: 3987 case X86::CMP32rr: 3988 case X86::CMP16rr: 3989 case X86::CMP8rr: 3990 SrcReg = MI.getOperand(0).getReg(); 3991 SrcReg2 = MI.getOperand(1).getReg(); 3992 CmpMask = 0; 3993 CmpValue = 0; 3994 return true; 3995 case X86::TEST8rr: 3996 case X86::TEST16rr: 3997 case X86::TEST32rr: 3998 case X86::TEST64rr: 3999 SrcReg = MI.getOperand(0).getReg(); 4000 if (MI.getOperand(1).getReg() != SrcReg) 4001 return false; 4002 // Compare against zero. 4003 SrcReg2 = 0; 4004 CmpMask = ~0; 4005 CmpValue = 0; 4006 return true; 4007 } 4008 return false; 4009 } 4010 4011 bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI, 4012 Register SrcReg, Register SrcReg2, 4013 int64_t ImmMask, int64_t ImmValue, 4014 const MachineInstr &OI, bool *IsSwapped, 4015 int64_t *ImmDelta) const { 4016 switch (OI.getOpcode()) { 4017 case X86::CMP64rr: 4018 case X86::CMP32rr: 4019 case X86::CMP16rr: 4020 case X86::CMP8rr: 4021 case X86::SUB64rr: 4022 case X86::SUB32rr: 4023 case X86::SUB16rr: 4024 case X86::SUB8rr: { 4025 Register OISrcReg; 4026 Register OISrcReg2; 4027 int64_t OIMask; 4028 int64_t OIValue; 4029 if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) || 4030 OIMask != ImmMask || OIValue != ImmValue) 4031 return false; 4032 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) { 4033 *IsSwapped = false; 4034 return true; 4035 } 4036 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) { 4037 *IsSwapped = true; 4038 return true; 4039 } 4040 return false; 4041 } 4042 case X86::CMP64ri32: 4043 case X86::CMP64ri8: 4044 case X86::CMP32ri: 4045 case X86::CMP32ri8: 4046 case X86::CMP16ri: 4047 case X86::CMP16ri8: 4048 case X86::CMP8ri: 4049 case X86::SUB64ri32: 4050 case X86::SUB64ri8: 4051 case X86::SUB32ri: 4052 case X86::SUB32ri8: 4053 case X86::SUB16ri: 4054 case X86::SUB16ri8: 4055 case X86::SUB8ri: 4056 case X86::TEST64rr: 4057 case X86::TEST32rr: 4058 case X86::TEST16rr: 4059 case X86::TEST8rr: { 4060 if (ImmMask != 0) { 4061 Register OISrcReg; 4062 Register OISrcReg2; 4063 int64_t OIMask; 4064 int64_t OIValue; 4065 if (analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) && 4066 SrcReg == OISrcReg && ImmMask == OIMask) { 4067 if (OIValue == ImmValue) { 4068 *ImmDelta = 0; 4069 return true; 4070 } else if (static_cast<uint64_t>(ImmValue) == 4071 static_cast<uint64_t>(OIValue) - 1) { 4072 *ImmDelta = -1; 4073 return true; 4074 } else if (static_cast<uint64_t>(ImmValue) == 4075 static_cast<uint64_t>(OIValue) + 1) { 4076 *ImmDelta = 1; 4077 return true; 4078 } else { 4079 return false; 4080 } 4081 } 4082 } 4083 return FlagI.isIdenticalTo(OI); 4084 } 4085 default: 4086 return false; 4087 } 4088 } 4089 4090 /// Check whether the definition can be converted 4091 /// to remove a comparison against zero. 4092 inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag, 4093 bool &ClearsOverflowFlag) { 4094 NoSignFlag = false; 4095 ClearsOverflowFlag = false; 4096 4097 switch (MI.getOpcode()) { 4098 default: return false; 4099 4100 // The shift instructions only modify ZF if their shift count is non-zero. 4101 // N.B.: The processor truncates the shift count depending on the encoding. 4102 case X86::SAR8ri: case X86::SAR16ri: case X86::SAR32ri:case X86::SAR64ri: 4103 case X86::SHR8ri: case X86::SHR16ri: case X86::SHR32ri:case X86::SHR64ri: 4104 return getTruncatedShiftCount(MI, 2) != 0; 4105 4106 // Some left shift instructions can be turned into LEA instructions but only 4107 // if their flags aren't used. Avoid transforming such instructions. 4108 case X86::SHL8ri: case X86::SHL16ri: case X86::SHL32ri:case X86::SHL64ri:{ 4109 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 4110 if (isTruncatedShiftCountForLEA(ShAmt)) return false; 4111 return ShAmt != 0; 4112 } 4113 4114 case X86::SHRD16rri8:case X86::SHRD32rri8:case X86::SHRD64rri8: 4115 case X86::SHLD16rri8:case X86::SHLD32rri8:case X86::SHLD64rri8: 4116 return getTruncatedShiftCount(MI, 3) != 0; 4117 4118 case X86::SUB64ri32: case X86::SUB64ri8: case X86::SUB32ri: 4119 case X86::SUB32ri8: case X86::SUB16ri: case X86::SUB16ri8: 4120 case X86::SUB8ri: case X86::SUB64rr: case X86::SUB32rr: 4121 case X86::SUB16rr: case X86::SUB8rr: case X86::SUB64rm: 4122 case X86::SUB32rm: case X86::SUB16rm: case X86::SUB8rm: 4123 case X86::DEC64r: case X86::DEC32r: case X86::DEC16r: case X86::DEC8r: 4124 case X86::ADD64ri32: case X86::ADD64ri8: case X86::ADD32ri: 4125 case X86::ADD32ri8: case X86::ADD16ri: case X86::ADD16ri8: 4126 case X86::ADD8ri: case X86::ADD64rr: case X86::ADD32rr: 4127 case X86::ADD16rr: case X86::ADD8rr: case X86::ADD64rm: 4128 case X86::ADD32rm: case X86::ADD16rm: case X86::ADD8rm: 4129 case X86::INC64r: case X86::INC32r: case X86::INC16r: case X86::INC8r: 4130 case X86::ADC64ri32: case X86::ADC64ri8: case X86::ADC32ri: 4131 case X86::ADC32ri8: case X86::ADC16ri: case X86::ADC16ri8: 4132 case X86::ADC8ri: case X86::ADC64rr: case X86::ADC32rr: 4133 case X86::ADC16rr: case X86::ADC8rr: case X86::ADC64rm: 4134 case X86::ADC32rm: case X86::ADC16rm: case X86::ADC8rm: 4135 case X86::SBB64ri32: case X86::SBB64ri8: case X86::SBB32ri: 4136 case X86::SBB32ri8: case X86::SBB16ri: case X86::SBB16ri8: 4137 case X86::SBB8ri: case X86::SBB64rr: case X86::SBB32rr: 4138 case X86::SBB16rr: case X86::SBB8rr: case X86::SBB64rm: 4139 case X86::SBB32rm: case X86::SBB16rm: case X86::SBB8rm: 4140 case X86::NEG8r: case X86::NEG16r: case X86::NEG32r: case X86::NEG64r: 4141 case X86::SAR8r1: case X86::SAR16r1: case X86::SAR32r1:case X86::SAR64r1: 4142 case X86::SHR8r1: case X86::SHR16r1: case X86::SHR32r1:case X86::SHR64r1: 4143 case X86::SHL8r1: case X86::SHL16r1: case X86::SHL32r1:case X86::SHL64r1: 4144 case X86::LZCNT16rr: case X86::LZCNT16rm: 4145 case X86::LZCNT32rr: case X86::LZCNT32rm: 4146 case X86::LZCNT64rr: case X86::LZCNT64rm: 4147 case X86::POPCNT16rr:case X86::POPCNT16rm: 4148 case X86::POPCNT32rr:case X86::POPCNT32rm: 4149 case X86::POPCNT64rr:case X86::POPCNT64rm: 4150 case X86::TZCNT16rr: case X86::TZCNT16rm: 4151 case X86::TZCNT32rr: case X86::TZCNT32rm: 4152 case X86::TZCNT64rr: case X86::TZCNT64rm: 4153 return true; 4154 case X86::AND64ri32: case X86::AND64ri8: case X86::AND32ri: 4155 case X86::AND32ri8: case X86::AND16ri: case X86::AND16ri8: 4156 case X86::AND8ri: case X86::AND64rr: case X86::AND32rr: 4157 case X86::AND16rr: case X86::AND8rr: case X86::AND64rm: 4158 case X86::AND32rm: case X86::AND16rm: case X86::AND8rm: 4159 case X86::XOR64ri32: case X86::XOR64ri8: case X86::XOR32ri: 4160 case X86::XOR32ri8: case X86::XOR16ri: case X86::XOR16ri8: 4161 case X86::XOR8ri: case X86::XOR64rr: case X86::XOR32rr: 4162 case X86::XOR16rr: case X86::XOR8rr: case X86::XOR64rm: 4163 case X86::XOR32rm: case X86::XOR16rm: case X86::XOR8rm: 4164 case X86::OR64ri32: case X86::OR64ri8: case X86::OR32ri: 4165 case X86::OR32ri8: case X86::OR16ri: case X86::OR16ri8: 4166 case X86::OR8ri: case X86::OR64rr: case X86::OR32rr: 4167 case X86::OR16rr: case X86::OR8rr: case X86::OR64rm: 4168 case X86::OR32rm: case X86::OR16rm: case X86::OR8rm: 4169 case X86::ANDN32rr: case X86::ANDN32rm: 4170 case X86::ANDN64rr: case X86::ANDN64rm: 4171 case X86::BLSI32rr: case X86::BLSI32rm: 4172 case X86::BLSI64rr: case X86::BLSI64rm: 4173 case X86::BLSMSK32rr: case X86::BLSMSK32rm: 4174 case X86::BLSMSK64rr: case X86::BLSMSK64rm: 4175 case X86::BLSR32rr: case X86::BLSR32rm: 4176 case X86::BLSR64rr: case X86::BLSR64rm: 4177 case X86::BLCFILL32rr: case X86::BLCFILL32rm: 4178 case X86::BLCFILL64rr: case X86::BLCFILL64rm: 4179 case X86::BLCI32rr: case X86::BLCI32rm: 4180 case X86::BLCI64rr: case X86::BLCI64rm: 4181 case X86::BLCIC32rr: case X86::BLCIC32rm: 4182 case X86::BLCIC64rr: case X86::BLCIC64rm: 4183 case X86::BLCMSK32rr: case X86::BLCMSK32rm: 4184 case X86::BLCMSK64rr: case X86::BLCMSK64rm: 4185 case X86::BLCS32rr: case X86::BLCS32rm: 4186 case X86::BLCS64rr: case X86::BLCS64rm: 4187 case X86::BLSFILL32rr: case X86::BLSFILL32rm: 4188 case X86::BLSFILL64rr: case X86::BLSFILL64rm: 4189 case X86::BLSIC32rr: case X86::BLSIC32rm: 4190 case X86::BLSIC64rr: case X86::BLSIC64rm: 4191 case X86::BZHI32rr: case X86::BZHI32rm: 4192 case X86::BZHI64rr: case X86::BZHI64rm: 4193 case X86::T1MSKC32rr: case X86::T1MSKC32rm: 4194 case X86::T1MSKC64rr: case X86::T1MSKC64rm: 4195 case X86::TZMSK32rr: case X86::TZMSK32rm: 4196 case X86::TZMSK64rr: case X86::TZMSK64rm: 4197 // These instructions clear the overflow flag just like TEST. 4198 // FIXME: These are not the only instructions in this switch that clear the 4199 // overflow flag. 4200 ClearsOverflowFlag = true; 4201 return true; 4202 case X86::BEXTR32rr: case X86::BEXTR64rr: 4203 case X86::BEXTR32rm: case X86::BEXTR64rm: 4204 case X86::BEXTRI32ri: case X86::BEXTRI32mi: 4205 case X86::BEXTRI64ri: case X86::BEXTRI64mi: 4206 // BEXTR doesn't update the sign flag so we can't use it. It does clear 4207 // the overflow flag, but that's not useful without the sign flag. 4208 NoSignFlag = true; 4209 return true; 4210 } 4211 } 4212 4213 /// Check whether the use can be converted to remove a comparison against zero. 4214 static X86::CondCode isUseDefConvertible(const MachineInstr &MI) { 4215 switch (MI.getOpcode()) { 4216 default: return X86::COND_INVALID; 4217 case X86::NEG8r: 4218 case X86::NEG16r: 4219 case X86::NEG32r: 4220 case X86::NEG64r: 4221 return X86::COND_AE; 4222 case X86::LZCNT16rr: 4223 case X86::LZCNT32rr: 4224 case X86::LZCNT64rr: 4225 return X86::COND_B; 4226 case X86::POPCNT16rr: 4227 case X86::POPCNT32rr: 4228 case X86::POPCNT64rr: 4229 return X86::COND_E; 4230 case X86::TZCNT16rr: 4231 case X86::TZCNT32rr: 4232 case X86::TZCNT64rr: 4233 return X86::COND_B; 4234 case X86::BSF16rr: 4235 case X86::BSF32rr: 4236 case X86::BSF64rr: 4237 case X86::BSR16rr: 4238 case X86::BSR32rr: 4239 case X86::BSR64rr: 4240 return X86::COND_E; 4241 case X86::BLSI32rr: 4242 case X86::BLSI64rr: 4243 return X86::COND_AE; 4244 case X86::BLSR32rr: 4245 case X86::BLSR64rr: 4246 case X86::BLSMSK32rr: 4247 case X86::BLSMSK64rr: 4248 return X86::COND_B; 4249 // TODO: TBM instructions. 4250 } 4251 } 4252 4253 /// Check if there exists an earlier instruction that 4254 /// operates on the same source operands and sets flags in the same way as 4255 /// Compare; remove Compare if possible. 4256 bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, 4257 Register SrcReg2, int64_t CmpMask, 4258 int64_t CmpValue, 4259 const MachineRegisterInfo *MRI) const { 4260 // Check whether we can replace SUB with CMP. 4261 switch (CmpInstr.getOpcode()) { 4262 default: break; 4263 case X86::SUB64ri32: 4264 case X86::SUB64ri8: 4265 case X86::SUB32ri: 4266 case X86::SUB32ri8: 4267 case X86::SUB16ri: 4268 case X86::SUB16ri8: 4269 case X86::SUB8ri: 4270 case X86::SUB64rm: 4271 case X86::SUB32rm: 4272 case X86::SUB16rm: 4273 case X86::SUB8rm: 4274 case X86::SUB64rr: 4275 case X86::SUB32rr: 4276 case X86::SUB16rr: 4277 case X86::SUB8rr: { 4278 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg())) 4279 return false; 4280 // There is no use of the destination register, we can replace SUB with CMP. 4281 unsigned NewOpcode = 0; 4282 switch (CmpInstr.getOpcode()) { 4283 default: llvm_unreachable("Unreachable!"); 4284 case X86::SUB64rm: NewOpcode = X86::CMP64rm; break; 4285 case X86::SUB32rm: NewOpcode = X86::CMP32rm; break; 4286 case X86::SUB16rm: NewOpcode = X86::CMP16rm; break; 4287 case X86::SUB8rm: NewOpcode = X86::CMP8rm; break; 4288 case X86::SUB64rr: NewOpcode = X86::CMP64rr; break; 4289 case X86::SUB32rr: NewOpcode = X86::CMP32rr; break; 4290 case X86::SUB16rr: NewOpcode = X86::CMP16rr; break; 4291 case X86::SUB8rr: NewOpcode = X86::CMP8rr; break; 4292 case X86::SUB64ri32: NewOpcode = X86::CMP64ri32; break; 4293 case X86::SUB64ri8: NewOpcode = X86::CMP64ri8; break; 4294 case X86::SUB32ri: NewOpcode = X86::CMP32ri; break; 4295 case X86::SUB32ri8: NewOpcode = X86::CMP32ri8; break; 4296 case X86::SUB16ri: NewOpcode = X86::CMP16ri; break; 4297 case X86::SUB16ri8: NewOpcode = X86::CMP16ri8; break; 4298 case X86::SUB8ri: NewOpcode = X86::CMP8ri; break; 4299 } 4300 CmpInstr.setDesc(get(NewOpcode)); 4301 CmpInstr.removeOperand(0); 4302 // Mutating this instruction invalidates any debug data associated with it. 4303 CmpInstr.dropDebugNumber(); 4304 // Fall through to optimize Cmp if Cmp is CMPrr or CMPri. 4305 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm || 4306 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm) 4307 return false; 4308 } 4309 } 4310 4311 // The following code tries to remove the comparison by re-using EFLAGS 4312 // from earlier instructions. 4313 4314 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0); 4315 4316 // Transformation currently requires SSA values. 4317 if (SrcReg2.isPhysical()) 4318 return false; 4319 MachineInstr *SrcRegDef = MRI->getVRegDef(SrcReg); 4320 assert(SrcRegDef && "Must have a definition (SSA)"); 4321 4322 MachineInstr *MI = nullptr; 4323 MachineInstr *Sub = nullptr; 4324 MachineInstr *Movr0Inst = nullptr; 4325 bool NoSignFlag = false; 4326 bool ClearsOverflowFlag = false; 4327 bool ShouldUpdateCC = false; 4328 bool IsSwapped = false; 4329 X86::CondCode NewCC = X86::COND_INVALID; 4330 int64_t ImmDelta = 0; 4331 4332 // Search backward from CmpInstr for the next instruction defining EFLAGS. 4333 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4334 MachineBasicBlock &CmpMBB = *CmpInstr.getParent(); 4335 MachineBasicBlock::reverse_iterator From = 4336 std::next(MachineBasicBlock::reverse_iterator(CmpInstr)); 4337 for (MachineBasicBlock *MBB = &CmpMBB;;) { 4338 for (MachineInstr &Inst : make_range(From, MBB->rend())) { 4339 // Try to use EFLAGS from the instruction defining %SrcReg. Example: 4340 // %eax = addl ... 4341 // ... // EFLAGS not changed 4342 // testl %eax, %eax // <-- can be removed 4343 if (&Inst == SrcRegDef) { 4344 if (IsCmpZero && 4345 isDefConvertible(Inst, NoSignFlag, ClearsOverflowFlag)) { 4346 MI = &Inst; 4347 break; 4348 } 4349 4350 // Look back for the following pattern, in which case the test64rr 4351 // instruction could be erased. 4352 // 4353 // Example: 4354 // %reg = and32ri %in_reg, 5 4355 // ... // EFLAGS not changed. 4356 // %src_reg = subreg_to_reg 0, %reg, %subreg.sub_index 4357 // test64rr %src_reg, %src_reg, implicit-def $eflags 4358 MachineInstr *AndInstr = nullptr; 4359 if (IsCmpZero && 4360 findRedundantFlagInstr(CmpInstr, Inst, MRI, &AndInstr, TRI, 4361 NoSignFlag, ClearsOverflowFlag)) { 4362 assert(AndInstr != nullptr && X86::isAND(AndInstr->getOpcode())); 4363 MI = AndInstr; 4364 break; 4365 } 4366 // Cannot find other candidates before definition of SrcReg. 4367 return false; 4368 } 4369 4370 if (Inst.modifiesRegister(X86::EFLAGS, TRI)) { 4371 // Try to use EFLAGS produced by an instruction reading %SrcReg. 4372 // Example: 4373 // %eax = ... 4374 // ... 4375 // popcntl %eax 4376 // ... // EFLAGS not changed 4377 // testl %eax, %eax // <-- can be removed 4378 if (IsCmpZero) { 4379 NewCC = isUseDefConvertible(Inst); 4380 if (NewCC != X86::COND_INVALID && Inst.getOperand(1).isReg() && 4381 Inst.getOperand(1).getReg() == SrcReg) { 4382 ShouldUpdateCC = true; 4383 MI = &Inst; 4384 break; 4385 } 4386 } 4387 4388 // Try to use EFLAGS from an instruction with similar flag results. 4389 // Example: 4390 // sub x, y or cmp x, y 4391 // ... // EFLAGS not changed 4392 // cmp x, y // <-- can be removed 4393 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue, 4394 Inst, &IsSwapped, &ImmDelta)) { 4395 Sub = &Inst; 4396 break; 4397 } 4398 4399 // MOV32r0 is implemented with xor which clobbers condition code. It is 4400 // safe to move up, if the definition to EFLAGS is dead and earlier 4401 // instructions do not read or write EFLAGS. 4402 if (!Movr0Inst && Inst.getOpcode() == X86::MOV32r0 && 4403 Inst.registerDefIsDead(X86::EFLAGS, TRI)) { 4404 Movr0Inst = &Inst; 4405 continue; 4406 } 4407 4408 // Cannot do anything for any other EFLAG changes. 4409 return false; 4410 } 4411 } 4412 4413 if (MI || Sub) 4414 break; 4415 4416 // Reached begin of basic block. Continue in predecessor if there is 4417 // exactly one. 4418 if (MBB->pred_size() != 1) 4419 return false; 4420 MBB = *MBB->pred_begin(); 4421 From = MBB->rbegin(); 4422 } 4423 4424 // Scan forward from the instruction after CmpInstr for uses of EFLAGS. 4425 // It is safe to remove CmpInstr if EFLAGS is redefined or killed. 4426 // If we are done with the basic block, we need to check whether EFLAGS is 4427 // live-out. 4428 bool FlagsMayLiveOut = true; 4429 SmallVector<std::pair<MachineInstr*, X86::CondCode>, 4> OpsToUpdate; 4430 MachineBasicBlock::iterator AfterCmpInstr = 4431 std::next(MachineBasicBlock::iterator(CmpInstr)); 4432 for (MachineInstr &Instr : make_range(AfterCmpInstr, CmpMBB.end())) { 4433 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI); 4434 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI); 4435 // We should check the usage if this instruction uses and updates EFLAGS. 4436 if (!UseEFLAGS && ModifyEFLAGS) { 4437 // It is safe to remove CmpInstr if EFLAGS is updated again. 4438 FlagsMayLiveOut = false; 4439 break; 4440 } 4441 if (!UseEFLAGS && !ModifyEFLAGS) 4442 continue; 4443 4444 // EFLAGS is used by this instruction. 4445 X86::CondCode OldCC = X86::COND_INVALID; 4446 if (MI || IsSwapped || ImmDelta != 0) { 4447 // We decode the condition code from opcode. 4448 if (Instr.isBranch()) 4449 OldCC = X86::getCondFromBranch(Instr); 4450 else { 4451 OldCC = X86::getCondFromSETCC(Instr); 4452 if (OldCC == X86::COND_INVALID) 4453 OldCC = X86::getCondFromCMov(Instr); 4454 } 4455 if (OldCC == X86::COND_INVALID) return false; 4456 } 4457 X86::CondCode ReplacementCC = X86::COND_INVALID; 4458 if (MI) { 4459 switch (OldCC) { 4460 default: break; 4461 case X86::COND_A: case X86::COND_AE: 4462 case X86::COND_B: case X86::COND_BE: 4463 // CF is used, we can't perform this optimization. 4464 return false; 4465 case X86::COND_G: case X86::COND_GE: 4466 case X86::COND_L: case X86::COND_LE: 4467 // If SF is used, but the instruction doesn't update the SF, then we 4468 // can't do the optimization. 4469 if (NoSignFlag) 4470 return false; 4471 LLVM_FALLTHROUGH; 4472 case X86::COND_O: case X86::COND_NO: 4473 // If OF is used, the instruction needs to clear it like CmpZero does. 4474 if (!ClearsOverflowFlag) 4475 return false; 4476 break; 4477 case X86::COND_S: case X86::COND_NS: 4478 // If SF is used, but the instruction doesn't update the SF, then we 4479 // can't do the optimization. 4480 if (NoSignFlag) 4481 return false; 4482 break; 4483 } 4484 4485 // If we're updating the condition code check if we have to reverse the 4486 // condition. 4487 if (ShouldUpdateCC) 4488 switch (OldCC) { 4489 default: 4490 return false; 4491 case X86::COND_E: 4492 ReplacementCC = NewCC; 4493 break; 4494 case X86::COND_NE: 4495 ReplacementCC = GetOppositeBranchCondition(NewCC); 4496 break; 4497 } 4498 } else if (IsSwapped) { 4499 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs 4500 // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. 4501 // We swap the condition code and synthesize the new opcode. 4502 ReplacementCC = getSwappedCondition(OldCC); 4503 if (ReplacementCC == X86::COND_INVALID) 4504 return false; 4505 ShouldUpdateCC = true; 4506 } else if (ImmDelta != 0) { 4507 unsigned BitWidth = TRI->getRegSizeInBits(*MRI->getRegClass(SrcReg)); 4508 // Shift amount for min/max constants to adjust for 8/16/32 instruction 4509 // sizes. 4510 switch (OldCC) { 4511 case X86::COND_L: // x <s (C + 1) --> x <=s C 4512 if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue) 4513 return false; 4514 ReplacementCC = X86::COND_LE; 4515 break; 4516 case X86::COND_B: // x <u (C + 1) --> x <=u C 4517 if (ImmDelta != 1 || CmpValue == 0) 4518 return false; 4519 ReplacementCC = X86::COND_BE; 4520 break; 4521 case X86::COND_GE: // x >=s (C + 1) --> x >s C 4522 if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue) 4523 return false; 4524 ReplacementCC = X86::COND_G; 4525 break; 4526 case X86::COND_AE: // x >=u (C + 1) --> x >u C 4527 if (ImmDelta != 1 || CmpValue == 0) 4528 return false; 4529 ReplacementCC = X86::COND_A; 4530 break; 4531 case X86::COND_G: // x >s (C - 1) --> x >=s C 4532 if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue) 4533 return false; 4534 ReplacementCC = X86::COND_GE; 4535 break; 4536 case X86::COND_A: // x >u (C - 1) --> x >=u C 4537 if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue) 4538 return false; 4539 ReplacementCC = X86::COND_AE; 4540 break; 4541 case X86::COND_LE: // x <=s (C - 1) --> x <s C 4542 if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue) 4543 return false; 4544 ReplacementCC = X86::COND_L; 4545 break; 4546 case X86::COND_BE: // x <=u (C - 1) --> x <u C 4547 if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue) 4548 return false; 4549 ReplacementCC = X86::COND_B; 4550 break; 4551 default: 4552 return false; 4553 } 4554 ShouldUpdateCC = true; 4555 } 4556 4557 if (ShouldUpdateCC && ReplacementCC != OldCC) { 4558 // Push the MachineInstr to OpsToUpdate. 4559 // If it is safe to remove CmpInstr, the condition code of these 4560 // instructions will be modified. 4561 OpsToUpdate.push_back(std::make_pair(&Instr, ReplacementCC)); 4562 } 4563 if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) { 4564 // It is safe to remove CmpInstr if EFLAGS is updated again or killed. 4565 FlagsMayLiveOut = false; 4566 break; 4567 } 4568 } 4569 4570 // If we have to update users but EFLAGS is live-out abort, since we cannot 4571 // easily find all of the users. 4572 if ((MI != nullptr || ShouldUpdateCC) && FlagsMayLiveOut) { 4573 for (MachineBasicBlock *Successor : CmpMBB.successors()) 4574 if (Successor->isLiveIn(X86::EFLAGS)) 4575 return false; 4576 } 4577 4578 // The instruction to be updated is either Sub or MI. 4579 assert((MI == nullptr || Sub == nullptr) && "Should not have Sub and MI set"); 4580 Sub = MI != nullptr ? MI : Sub; 4581 MachineBasicBlock *SubBB = Sub->getParent(); 4582 // Move Movr0Inst to the appropriate place before Sub. 4583 if (Movr0Inst) { 4584 // Only move within the same block so we don't accidentally move to a 4585 // block with higher execution frequency. 4586 if (&CmpMBB != SubBB) 4587 return false; 4588 // Look backwards until we find a def that doesn't use the current EFLAGS. 4589 MachineBasicBlock::reverse_iterator InsertI = Sub, 4590 InsertE = Sub->getParent()->rend(); 4591 for (; InsertI != InsertE; ++InsertI) { 4592 MachineInstr *Instr = &*InsertI; 4593 if (!Instr->readsRegister(X86::EFLAGS, TRI) && 4594 Instr->modifiesRegister(X86::EFLAGS, TRI)) { 4595 Movr0Inst->getParent()->remove(Movr0Inst); 4596 Instr->getParent()->insert(MachineBasicBlock::iterator(Instr), 4597 Movr0Inst); 4598 break; 4599 } 4600 } 4601 if (InsertI == InsertE) 4602 return false; 4603 } 4604 4605 // Make sure Sub instruction defines EFLAGS and mark the def live. 4606 MachineOperand *FlagDef = Sub->findRegisterDefOperand(X86::EFLAGS); 4607 assert(FlagDef && "Unable to locate a def EFLAGS operand"); 4608 FlagDef->setIsDead(false); 4609 4610 CmpInstr.eraseFromParent(); 4611 4612 // Modify the condition code of instructions in OpsToUpdate. 4613 for (auto &Op : OpsToUpdate) { 4614 Op.first->getOperand(Op.first->getDesc().getNumOperands() - 1) 4615 .setImm(Op.second); 4616 } 4617 // Add EFLAGS to block live-ins between CmpBB and block of flags producer. 4618 for (MachineBasicBlock *MBB = &CmpMBB; MBB != SubBB; 4619 MBB = *MBB->pred_begin()) { 4620 assert(MBB->pred_size() == 1 && "Expected exactly one predecessor"); 4621 if (!MBB->isLiveIn(X86::EFLAGS)) 4622 MBB->addLiveIn(X86::EFLAGS); 4623 } 4624 return true; 4625 } 4626 4627 /// Try to remove the load by folding it to a register 4628 /// operand at the use. We fold the load instructions if load defines a virtual 4629 /// register, the virtual register is used once in the same BB, and the 4630 /// instructions in-between do not load or store, and have no side effects. 4631 MachineInstr *X86InstrInfo::optimizeLoadInstr(MachineInstr &MI, 4632 const MachineRegisterInfo *MRI, 4633 Register &FoldAsLoadDefReg, 4634 MachineInstr *&DefMI) const { 4635 // Check whether we can move DefMI here. 4636 DefMI = MRI->getVRegDef(FoldAsLoadDefReg); 4637 assert(DefMI); 4638 bool SawStore = false; 4639 if (!DefMI->isSafeToMove(nullptr, SawStore)) 4640 return nullptr; 4641 4642 // Collect information about virtual register operands of MI. 4643 SmallVector<unsigned, 1> SrcOperandIds; 4644 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 4645 MachineOperand &MO = MI.getOperand(i); 4646 if (!MO.isReg()) 4647 continue; 4648 Register Reg = MO.getReg(); 4649 if (Reg != FoldAsLoadDefReg) 4650 continue; 4651 // Do not fold if we have a subreg use or a def. 4652 if (MO.getSubReg() || MO.isDef()) 4653 return nullptr; 4654 SrcOperandIds.push_back(i); 4655 } 4656 if (SrcOperandIds.empty()) 4657 return nullptr; 4658 4659 // Check whether we can fold the def into SrcOperandId. 4660 if (MachineInstr *FoldMI = foldMemoryOperand(MI, SrcOperandIds, *DefMI)) { 4661 FoldAsLoadDefReg = 0; 4662 return FoldMI; 4663 } 4664 4665 return nullptr; 4666 } 4667 4668 /// Expand a single-def pseudo instruction to a two-addr 4669 /// instruction with two undef reads of the register being defined. 4670 /// This is used for mapping: 4671 /// %xmm4 = V_SET0 4672 /// to: 4673 /// %xmm4 = PXORrr undef %xmm4, undef %xmm4 4674 /// 4675 static bool Expand2AddrUndef(MachineInstrBuilder &MIB, 4676 const MCInstrDesc &Desc) { 4677 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction."); 4678 Register Reg = MIB.getReg(0); 4679 MIB->setDesc(Desc); 4680 4681 // MachineInstr::addOperand() will insert explicit operands before any 4682 // implicit operands. 4683 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); 4684 // But we don't trust that. 4685 assert(MIB.getReg(1) == Reg && 4686 MIB.getReg(2) == Reg && "Misplaced operand"); 4687 return true; 4688 } 4689 4690 /// Expand a single-def pseudo instruction to a two-addr 4691 /// instruction with two %k0 reads. 4692 /// This is used for mapping: 4693 /// %k4 = K_SET1 4694 /// to: 4695 /// %k4 = KXNORrr %k0, %k0 4696 static bool Expand2AddrKreg(MachineInstrBuilder &MIB, const MCInstrDesc &Desc, 4697 Register Reg) { 4698 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction."); 4699 MIB->setDesc(Desc); 4700 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); 4701 return true; 4702 } 4703 4704 static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, 4705 bool MinusOne) { 4706 MachineBasicBlock &MBB = *MIB->getParent(); 4707 const DebugLoc &DL = MIB->getDebugLoc(); 4708 Register Reg = MIB.getReg(0); 4709 4710 // Insert the XOR. 4711 BuildMI(MBB, MIB.getInstr(), DL, TII.get(X86::XOR32rr), Reg) 4712 .addReg(Reg, RegState::Undef) 4713 .addReg(Reg, RegState::Undef); 4714 4715 // Turn the pseudo into an INC or DEC. 4716 MIB->setDesc(TII.get(MinusOne ? X86::DEC32r : X86::INC32r)); 4717 MIB.addReg(Reg); 4718 4719 return true; 4720 } 4721 4722 static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB, 4723 const TargetInstrInfo &TII, 4724 const X86Subtarget &Subtarget) { 4725 MachineBasicBlock &MBB = *MIB->getParent(); 4726 const DebugLoc &DL = MIB->getDebugLoc(); 4727 int64_t Imm = MIB->getOperand(1).getImm(); 4728 assert(Imm != 0 && "Using push/pop for 0 is not efficient."); 4729 MachineBasicBlock::iterator I = MIB.getInstr(); 4730 4731 int StackAdjustment; 4732 4733 if (Subtarget.is64Bit()) { 4734 assert(MIB->getOpcode() == X86::MOV64ImmSExti8 || 4735 MIB->getOpcode() == X86::MOV32ImmSExti8); 4736 4737 // Can't use push/pop lowering if the function might write to the red zone. 4738 X86MachineFunctionInfo *X86FI = 4739 MBB.getParent()->getInfo<X86MachineFunctionInfo>(); 4740 if (X86FI->getUsesRedZone()) { 4741 MIB->setDesc(TII.get(MIB->getOpcode() == 4742 X86::MOV32ImmSExti8 ? X86::MOV32ri : X86::MOV64ri)); 4743 return true; 4744 } 4745 4746 // 64-bit mode doesn't have 32-bit push/pop, so use 64-bit operations and 4747 // widen the register if necessary. 4748 StackAdjustment = 8; 4749 BuildMI(MBB, I, DL, TII.get(X86::PUSH64i8)).addImm(Imm); 4750 MIB->setDesc(TII.get(X86::POP64r)); 4751 MIB->getOperand(0) 4752 .setReg(getX86SubSuperRegister(MIB.getReg(0), 64)); 4753 } else { 4754 assert(MIB->getOpcode() == X86::MOV32ImmSExti8); 4755 StackAdjustment = 4; 4756 BuildMI(MBB, I, DL, TII.get(X86::PUSH32i8)).addImm(Imm); 4757 MIB->setDesc(TII.get(X86::POP32r)); 4758 } 4759 MIB->removeOperand(1); 4760 MIB->addImplicitDefUseOperands(*MBB.getParent()); 4761 4762 // Build CFI if necessary. 4763 MachineFunction &MF = *MBB.getParent(); 4764 const X86FrameLowering *TFL = Subtarget.getFrameLowering(); 4765 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 4766 bool NeedsDwarfCFI = !IsWin64Prologue && MF.needsFrameMoves(); 4767 bool EmitCFI = !TFL->hasFP(MF) && NeedsDwarfCFI; 4768 if (EmitCFI) { 4769 TFL->BuildCFI(MBB, I, DL, 4770 MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment)); 4771 TFL->BuildCFI(MBB, std::next(I), DL, 4772 MCCFIInstruction::createAdjustCfaOffset(nullptr, -StackAdjustment)); 4773 } 4774 4775 return true; 4776 } 4777 4778 // LoadStackGuard has so far only been implemented for 64-bit MachO. Different 4779 // code sequence is needed for other targets. 4780 static void expandLoadStackGuard(MachineInstrBuilder &MIB, 4781 const TargetInstrInfo &TII) { 4782 MachineBasicBlock &MBB = *MIB->getParent(); 4783 const DebugLoc &DL = MIB->getDebugLoc(); 4784 Register Reg = MIB.getReg(0); 4785 const GlobalValue *GV = 4786 cast<GlobalValue>((*MIB->memoperands_begin())->getValue()); 4787 auto Flags = MachineMemOperand::MOLoad | 4788 MachineMemOperand::MODereferenceable | 4789 MachineMemOperand::MOInvariant; 4790 MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( 4791 MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, Align(8)); 4792 MachineBasicBlock::iterator I = MIB.getInstr(); 4793 4794 BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg).addReg(X86::RIP).addImm(1) 4795 .addReg(0).addGlobalAddress(GV, 0, X86II::MO_GOTPCREL).addReg(0) 4796 .addMemOperand(MMO); 4797 MIB->setDebugLoc(DL); 4798 MIB->setDesc(TII.get(X86::MOV64rm)); 4799 MIB.addReg(Reg, RegState::Kill).addImm(1).addReg(0).addImm(0).addReg(0); 4800 } 4801 4802 static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII) { 4803 MachineBasicBlock &MBB = *MIB->getParent(); 4804 MachineFunction &MF = *MBB.getParent(); 4805 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>(); 4806 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); 4807 unsigned XorOp = 4808 MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr; 4809 MIB->setDesc(TII.get(XorOp)); 4810 MIB.addReg(TRI->getFrameRegister(MF), RegState::Undef); 4811 return true; 4812 } 4813 4814 // This is used to handle spills for 128/256-bit registers when we have AVX512, 4815 // but not VLX. If it uses an extended register we need to use an instruction 4816 // that loads the lower 128/256-bit, but is available with only AVX512F. 4817 static bool expandNOVLXLoad(MachineInstrBuilder &MIB, 4818 const TargetRegisterInfo *TRI, 4819 const MCInstrDesc &LoadDesc, 4820 const MCInstrDesc &BroadcastDesc, 4821 unsigned SubIdx) { 4822 Register DestReg = MIB.getReg(0); 4823 // Check if DestReg is XMM16-31 or YMM16-31. 4824 if (TRI->getEncodingValue(DestReg) < 16) { 4825 // We can use a normal VEX encoded load. 4826 MIB->setDesc(LoadDesc); 4827 } else { 4828 // Use a 128/256-bit VBROADCAST instruction. 4829 MIB->setDesc(BroadcastDesc); 4830 // Change the destination to a 512-bit register. 4831 DestReg = TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass); 4832 MIB->getOperand(0).setReg(DestReg); 4833 } 4834 return true; 4835 } 4836 4837 // This is used to handle spills for 128/256-bit registers when we have AVX512, 4838 // but not VLX. If it uses an extended register we need to use an instruction 4839 // that stores the lower 128/256-bit, but is available with only AVX512F. 4840 static bool expandNOVLXStore(MachineInstrBuilder &MIB, 4841 const TargetRegisterInfo *TRI, 4842 const MCInstrDesc &StoreDesc, 4843 const MCInstrDesc &ExtractDesc, 4844 unsigned SubIdx) { 4845 Register SrcReg = MIB.getReg(X86::AddrNumOperands); 4846 // Check if DestReg is XMM16-31 or YMM16-31. 4847 if (TRI->getEncodingValue(SrcReg) < 16) { 4848 // We can use a normal VEX encoded store. 4849 MIB->setDesc(StoreDesc); 4850 } else { 4851 // Use a VEXTRACTF instruction. 4852 MIB->setDesc(ExtractDesc); 4853 // Change the destination to a 512-bit register. 4854 SrcReg = TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass); 4855 MIB->getOperand(X86::AddrNumOperands).setReg(SrcReg); 4856 MIB.addImm(0x0); // Append immediate to extract from the lower bits. 4857 } 4858 4859 return true; 4860 } 4861 4862 static bool expandSHXDROT(MachineInstrBuilder &MIB, const MCInstrDesc &Desc) { 4863 MIB->setDesc(Desc); 4864 int64_t ShiftAmt = MIB->getOperand(2).getImm(); 4865 // Temporarily remove the immediate so we can add another source register. 4866 MIB->removeOperand(2); 4867 // Add the register. Don't copy the kill flag if there is one. 4868 MIB.addReg(MIB.getReg(1), 4869 getUndefRegState(MIB->getOperand(1).isUndef())); 4870 // Add back the immediate. 4871 MIB.addImm(ShiftAmt); 4872 return true; 4873 } 4874 4875 bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 4876 bool HasAVX = Subtarget.hasAVX(); 4877 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI); 4878 switch (MI.getOpcode()) { 4879 case X86::MOV32r0: 4880 return Expand2AddrUndef(MIB, get(X86::XOR32rr)); 4881 case X86::MOV32r1: 4882 return expandMOV32r1(MIB, *this, /*MinusOne=*/ false); 4883 case X86::MOV32r_1: 4884 return expandMOV32r1(MIB, *this, /*MinusOne=*/ true); 4885 case X86::MOV32ImmSExti8: 4886 case X86::MOV64ImmSExti8: 4887 return ExpandMOVImmSExti8(MIB, *this, Subtarget); 4888 case X86::SETB_C32r: 4889 return Expand2AddrUndef(MIB, get(X86::SBB32rr)); 4890 case X86::SETB_C64r: 4891 return Expand2AddrUndef(MIB, get(X86::SBB64rr)); 4892 case X86::MMX_SET0: 4893 return Expand2AddrUndef(MIB, get(X86::MMX_PXORrr)); 4894 case X86::V_SET0: 4895 case X86::FsFLD0SS: 4896 case X86::FsFLD0SD: 4897 case X86::FsFLD0SH: 4898 case X86::FsFLD0F128: 4899 return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr)); 4900 case X86::AVX_SET0: { 4901 assert(HasAVX && "AVX not supported"); 4902 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4903 Register SrcReg = MIB.getReg(0); 4904 Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); 4905 MIB->getOperand(0).setReg(XReg); 4906 Expand2AddrUndef(MIB, get(X86::VXORPSrr)); 4907 MIB.addReg(SrcReg, RegState::ImplicitDefine); 4908 return true; 4909 } 4910 case X86::AVX512_128_SET0: 4911 case X86::AVX512_FsFLD0SH: 4912 case X86::AVX512_FsFLD0SS: 4913 case X86::AVX512_FsFLD0SD: 4914 case X86::AVX512_FsFLD0F128: { 4915 bool HasVLX = Subtarget.hasVLX(); 4916 Register SrcReg = MIB.getReg(0); 4917 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4918 if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) 4919 return Expand2AddrUndef(MIB, 4920 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); 4921 // Extended register without VLX. Use a larger XOR. 4922 SrcReg = 4923 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass); 4924 MIB->getOperand(0).setReg(SrcReg); 4925 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); 4926 } 4927 case X86::AVX512_256_SET0: 4928 case X86::AVX512_512_SET0: { 4929 bool HasVLX = Subtarget.hasVLX(); 4930 Register SrcReg = MIB.getReg(0); 4931 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4932 if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) { 4933 Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); 4934 MIB->getOperand(0).setReg(XReg); 4935 Expand2AddrUndef(MIB, 4936 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); 4937 MIB.addReg(SrcReg, RegState::ImplicitDefine); 4938 return true; 4939 } 4940 if (MI.getOpcode() == X86::AVX512_256_SET0) { 4941 // No VLX so we must reference a zmm. 4942 unsigned ZReg = 4943 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass); 4944 MIB->getOperand(0).setReg(ZReg); 4945 } 4946 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); 4947 } 4948 case X86::V_SETALLONES: 4949 return Expand2AddrUndef(MIB, get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr)); 4950 case X86::AVX2_SETALLONES: 4951 return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr)); 4952 case X86::AVX1_SETALLONES: { 4953 Register Reg = MIB.getReg(0); 4954 // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS. 4955 MIB->setDesc(get(X86::VCMPPSYrri)); 4956 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xf); 4957 return true; 4958 } 4959 case X86::AVX512_512_SETALLONES: { 4960 Register Reg = MIB.getReg(0); 4961 MIB->setDesc(get(X86::VPTERNLOGDZrri)); 4962 // VPTERNLOGD needs 3 register inputs and an immediate. 4963 // 0xff will return 1s for any input. 4964 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef) 4965 .addReg(Reg, RegState::Undef).addImm(0xff); 4966 return true; 4967 } 4968 case X86::AVX512_512_SEXT_MASK_32: 4969 case X86::AVX512_512_SEXT_MASK_64: { 4970 Register Reg = MIB.getReg(0); 4971 Register MaskReg = MIB.getReg(1); 4972 unsigned MaskState = getRegState(MIB->getOperand(1)); 4973 unsigned Opc = (MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64) ? 4974 X86::VPTERNLOGQZrrikz : X86::VPTERNLOGDZrrikz; 4975 MI.removeOperand(1); 4976 MIB->setDesc(get(Opc)); 4977 // VPTERNLOG needs 3 register inputs and an immediate. 4978 // 0xff will return 1s for any input. 4979 MIB.addReg(Reg, RegState::Undef).addReg(MaskReg, MaskState) 4980 .addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xff); 4981 return true; 4982 } 4983 case X86::VMOVAPSZ128rm_NOVLX: 4984 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSrm), 4985 get(X86::VBROADCASTF32X4rm), X86::sub_xmm); 4986 case X86::VMOVUPSZ128rm_NOVLX: 4987 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSrm), 4988 get(X86::VBROADCASTF32X4rm), X86::sub_xmm); 4989 case X86::VMOVAPSZ256rm_NOVLX: 4990 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSYrm), 4991 get(X86::VBROADCASTF64X4rm), X86::sub_ymm); 4992 case X86::VMOVUPSZ256rm_NOVLX: 4993 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSYrm), 4994 get(X86::VBROADCASTF64X4rm), X86::sub_ymm); 4995 case X86::VMOVAPSZ128mr_NOVLX: 4996 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSmr), 4997 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); 4998 case X86::VMOVUPSZ128mr_NOVLX: 4999 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSmr), 5000 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); 5001 case X86::VMOVAPSZ256mr_NOVLX: 5002 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSYmr), 5003 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); 5004 case X86::VMOVUPSZ256mr_NOVLX: 5005 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSYmr), 5006 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); 5007 case X86::MOV32ri64: { 5008 Register Reg = MIB.getReg(0); 5009 Register Reg32 = RI.getSubReg(Reg, X86::sub_32bit); 5010 MI.setDesc(get(X86::MOV32ri)); 5011 MIB->getOperand(0).setReg(Reg32); 5012 MIB.addReg(Reg, RegState::ImplicitDefine); 5013 return true; 5014 } 5015 5016 // KNL does not recognize dependency-breaking idioms for mask registers, 5017 // so kxnor %k1, %k1, %k2 has a RAW dependence on %k1. 5018 // Using %k0 as the undef input register is a performance heuristic based 5019 // on the assumption that %k0 is used less frequently than the other mask 5020 // registers, since it is not usable as a write mask. 5021 // FIXME: A more advanced approach would be to choose the best input mask 5022 // register based on context. 5023 case X86::KSET0W: return Expand2AddrKreg(MIB, get(X86::KXORWrr), X86::K0); 5024 case X86::KSET0D: return Expand2AddrKreg(MIB, get(X86::KXORDrr), X86::K0); 5025 case X86::KSET0Q: return Expand2AddrKreg(MIB, get(X86::KXORQrr), X86::K0); 5026 case X86::KSET1W: return Expand2AddrKreg(MIB, get(X86::KXNORWrr), X86::K0); 5027 case X86::KSET1D: return Expand2AddrKreg(MIB, get(X86::KXNORDrr), X86::K0); 5028 case X86::KSET1Q: return Expand2AddrKreg(MIB, get(X86::KXNORQrr), X86::K0); 5029 case TargetOpcode::LOAD_STACK_GUARD: 5030 expandLoadStackGuard(MIB, *this); 5031 return true; 5032 case X86::XOR64_FP: 5033 case X86::XOR32_FP: 5034 return expandXorFP(MIB, *this); 5035 case X86::SHLDROT32ri: return expandSHXDROT(MIB, get(X86::SHLD32rri8)); 5036 case X86::SHLDROT64ri: return expandSHXDROT(MIB, get(X86::SHLD64rri8)); 5037 case X86::SHRDROT32ri: return expandSHXDROT(MIB, get(X86::SHRD32rri8)); 5038 case X86::SHRDROT64ri: return expandSHXDROT(MIB, get(X86::SHRD64rri8)); 5039 case X86::ADD8rr_DB: MIB->setDesc(get(X86::OR8rr)); break; 5040 case X86::ADD16rr_DB: MIB->setDesc(get(X86::OR16rr)); break; 5041 case X86::ADD32rr_DB: MIB->setDesc(get(X86::OR32rr)); break; 5042 case X86::ADD64rr_DB: MIB->setDesc(get(X86::OR64rr)); break; 5043 case X86::ADD8ri_DB: MIB->setDesc(get(X86::OR8ri)); break; 5044 case X86::ADD16ri_DB: MIB->setDesc(get(X86::OR16ri)); break; 5045 case X86::ADD32ri_DB: MIB->setDesc(get(X86::OR32ri)); break; 5046 case X86::ADD64ri32_DB: MIB->setDesc(get(X86::OR64ri32)); break; 5047 case X86::ADD16ri8_DB: MIB->setDesc(get(X86::OR16ri8)); break; 5048 case X86::ADD32ri8_DB: MIB->setDesc(get(X86::OR32ri8)); break; 5049 case X86::ADD64ri8_DB: MIB->setDesc(get(X86::OR64ri8)); break; 5050 } 5051 return false; 5052 } 5053 5054 /// Return true for all instructions that only update 5055 /// the first 32 or 64-bits of the destination register and leave the rest 5056 /// unmodified. This can be used to avoid folding loads if the instructions 5057 /// only update part of the destination register, and the non-updated part is 5058 /// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these 5059 /// instructions breaks the partial register dependency and it can improve 5060 /// performance. e.g.: 5061 /// 5062 /// movss (%rdi), %xmm0 5063 /// cvtss2sd %xmm0, %xmm0 5064 /// 5065 /// Instead of 5066 /// cvtss2sd (%rdi), %xmm0 5067 /// 5068 /// FIXME: This should be turned into a TSFlags. 5069 /// 5070 static bool hasPartialRegUpdate(unsigned Opcode, 5071 const X86Subtarget &Subtarget, 5072 bool ForLoadFold = false) { 5073 switch (Opcode) { 5074 case X86::CVTSI2SSrr: 5075 case X86::CVTSI2SSrm: 5076 case X86::CVTSI642SSrr: 5077 case X86::CVTSI642SSrm: 5078 case X86::CVTSI2SDrr: 5079 case X86::CVTSI2SDrm: 5080 case X86::CVTSI642SDrr: 5081 case X86::CVTSI642SDrm: 5082 // Load folding won't effect the undef register update since the input is 5083 // a GPR. 5084 return !ForLoadFold; 5085 case X86::CVTSD2SSrr: 5086 case X86::CVTSD2SSrm: 5087 case X86::CVTSS2SDrr: 5088 case X86::CVTSS2SDrm: 5089 case X86::MOVHPDrm: 5090 case X86::MOVHPSrm: 5091 case X86::MOVLPDrm: 5092 case X86::MOVLPSrm: 5093 case X86::RCPSSr: 5094 case X86::RCPSSm: 5095 case X86::RCPSSr_Int: 5096 case X86::RCPSSm_Int: 5097 case X86::ROUNDSDr: 5098 case X86::ROUNDSDm: 5099 case X86::ROUNDSSr: 5100 case X86::ROUNDSSm: 5101 case X86::RSQRTSSr: 5102 case X86::RSQRTSSm: 5103 case X86::RSQRTSSr_Int: 5104 case X86::RSQRTSSm_Int: 5105 case X86::SQRTSSr: 5106 case X86::SQRTSSm: 5107 case X86::SQRTSSr_Int: 5108 case X86::SQRTSSm_Int: 5109 case X86::SQRTSDr: 5110 case X86::SQRTSDm: 5111 case X86::SQRTSDr_Int: 5112 case X86::SQRTSDm_Int: 5113 return true; 5114 case X86::VFCMULCPHZ128rm: 5115 case X86::VFCMULCPHZ128rmb: 5116 case X86::VFCMULCPHZ128rmbkz: 5117 case X86::VFCMULCPHZ128rmkz: 5118 case X86::VFCMULCPHZ128rr: 5119 case X86::VFCMULCPHZ128rrkz: 5120 case X86::VFCMULCPHZ256rm: 5121 case X86::VFCMULCPHZ256rmb: 5122 case X86::VFCMULCPHZ256rmbkz: 5123 case X86::VFCMULCPHZ256rmkz: 5124 case X86::VFCMULCPHZ256rr: 5125 case X86::VFCMULCPHZ256rrkz: 5126 case X86::VFCMULCPHZrm: 5127 case X86::VFCMULCPHZrmb: 5128 case X86::VFCMULCPHZrmbkz: 5129 case X86::VFCMULCPHZrmkz: 5130 case X86::VFCMULCPHZrr: 5131 case X86::VFCMULCPHZrrb: 5132 case X86::VFCMULCPHZrrbkz: 5133 case X86::VFCMULCPHZrrkz: 5134 case X86::VFMULCPHZ128rm: 5135 case X86::VFMULCPHZ128rmb: 5136 case X86::VFMULCPHZ128rmbkz: 5137 case X86::VFMULCPHZ128rmkz: 5138 case X86::VFMULCPHZ128rr: 5139 case X86::VFMULCPHZ128rrkz: 5140 case X86::VFMULCPHZ256rm: 5141 case X86::VFMULCPHZ256rmb: 5142 case X86::VFMULCPHZ256rmbkz: 5143 case X86::VFMULCPHZ256rmkz: 5144 case X86::VFMULCPHZ256rr: 5145 case X86::VFMULCPHZ256rrkz: 5146 case X86::VFMULCPHZrm: 5147 case X86::VFMULCPHZrmb: 5148 case X86::VFMULCPHZrmbkz: 5149 case X86::VFMULCPHZrmkz: 5150 case X86::VFMULCPHZrr: 5151 case X86::VFMULCPHZrrb: 5152 case X86::VFMULCPHZrrbkz: 5153 case X86::VFMULCPHZrrkz: 5154 case X86::VFCMULCSHZrm: 5155 case X86::VFCMULCSHZrmkz: 5156 case X86::VFCMULCSHZrr: 5157 case X86::VFCMULCSHZrrb: 5158 case X86::VFCMULCSHZrrbkz: 5159 case X86::VFCMULCSHZrrkz: 5160 case X86::VFMULCSHZrm: 5161 case X86::VFMULCSHZrmkz: 5162 case X86::VFMULCSHZrr: 5163 case X86::VFMULCSHZrrb: 5164 case X86::VFMULCSHZrrbkz: 5165 case X86::VFMULCSHZrrkz: 5166 return Subtarget.hasMULCFalseDeps(); 5167 case X86::VPERMDYrm: 5168 case X86::VPERMDYrr: 5169 case X86::VPERMQYmi: 5170 case X86::VPERMQYri: 5171 case X86::VPERMPSYrm: 5172 case X86::VPERMPSYrr: 5173 case X86::VPERMPDYmi: 5174 case X86::VPERMPDYri: 5175 case X86::VPERMDZ256rm: 5176 case X86::VPERMDZ256rmb: 5177 case X86::VPERMDZ256rmbkz: 5178 case X86::VPERMDZ256rmkz: 5179 case X86::VPERMDZ256rr: 5180 case X86::VPERMDZ256rrkz: 5181 case X86::VPERMDZrm: 5182 case X86::VPERMDZrmb: 5183 case X86::VPERMDZrmbkz: 5184 case X86::VPERMDZrmkz: 5185 case X86::VPERMDZrr: 5186 case X86::VPERMDZrrkz: 5187 case X86::VPERMQZ256mbi: 5188 case X86::VPERMQZ256mbikz: 5189 case X86::VPERMQZ256mi: 5190 case X86::VPERMQZ256mikz: 5191 case X86::VPERMQZ256ri: 5192 case X86::VPERMQZ256rikz: 5193 case X86::VPERMQZ256rm: 5194 case X86::VPERMQZ256rmb: 5195 case X86::VPERMQZ256rmbkz: 5196 case X86::VPERMQZ256rmkz: 5197 case X86::VPERMQZ256rr: 5198 case X86::VPERMQZ256rrkz: 5199 case X86::VPERMQZmbi: 5200 case X86::VPERMQZmbikz: 5201 case X86::VPERMQZmi: 5202 case X86::VPERMQZmikz: 5203 case X86::VPERMQZri: 5204 case X86::VPERMQZrikz: 5205 case X86::VPERMQZrm: 5206 case X86::VPERMQZrmb: 5207 case X86::VPERMQZrmbkz: 5208 case X86::VPERMQZrmkz: 5209 case X86::VPERMQZrr: 5210 case X86::VPERMQZrrkz: 5211 case X86::VPERMPSZ256rm: 5212 case X86::VPERMPSZ256rmb: 5213 case X86::VPERMPSZ256rmbkz: 5214 case X86::VPERMPSZ256rmkz: 5215 case X86::VPERMPSZ256rr: 5216 case X86::VPERMPSZ256rrkz: 5217 case X86::VPERMPSZrm: 5218 case X86::VPERMPSZrmb: 5219 case X86::VPERMPSZrmbkz: 5220 case X86::VPERMPSZrmkz: 5221 case X86::VPERMPSZrr: 5222 case X86::VPERMPSZrrkz: 5223 case X86::VPERMPDZ256mbi: 5224 case X86::VPERMPDZ256mbikz: 5225 case X86::VPERMPDZ256mi: 5226 case X86::VPERMPDZ256mikz: 5227 case X86::VPERMPDZ256ri: 5228 case X86::VPERMPDZ256rikz: 5229 case X86::VPERMPDZ256rm: 5230 case X86::VPERMPDZ256rmb: 5231 case X86::VPERMPDZ256rmbkz: 5232 case X86::VPERMPDZ256rmkz: 5233 case X86::VPERMPDZ256rr: 5234 case X86::VPERMPDZ256rrkz: 5235 case X86::VPERMPDZmbi: 5236 case X86::VPERMPDZmbikz: 5237 case X86::VPERMPDZmi: 5238 case X86::VPERMPDZmikz: 5239 case X86::VPERMPDZri: 5240 case X86::VPERMPDZrikz: 5241 case X86::VPERMPDZrm: 5242 case X86::VPERMPDZrmb: 5243 case X86::VPERMPDZrmbkz: 5244 case X86::VPERMPDZrmkz: 5245 case X86::VPERMPDZrr: 5246 case X86::VPERMPDZrrkz: 5247 return Subtarget.hasPERMFalseDeps(); 5248 case X86::VRANGEPDZ128rmbi: 5249 case X86::VRANGEPDZ128rmbikz: 5250 case X86::VRANGEPDZ128rmi: 5251 case X86::VRANGEPDZ128rmikz: 5252 case X86::VRANGEPDZ128rri: 5253 case X86::VRANGEPDZ128rrikz: 5254 case X86::VRANGEPDZ256rmbi: 5255 case X86::VRANGEPDZ256rmbikz: 5256 case X86::VRANGEPDZ256rmi: 5257 case X86::VRANGEPDZ256rmikz: 5258 case X86::VRANGEPDZ256rri: 5259 case X86::VRANGEPDZ256rrikz: 5260 case X86::VRANGEPDZrmbi: 5261 case X86::VRANGEPDZrmbikz: 5262 case X86::VRANGEPDZrmi: 5263 case X86::VRANGEPDZrmikz: 5264 case X86::VRANGEPDZrri: 5265 case X86::VRANGEPDZrrib: 5266 case X86::VRANGEPDZrribkz: 5267 case X86::VRANGEPDZrrikz: 5268 case X86::VRANGEPSZ128rmbi: 5269 case X86::VRANGEPSZ128rmbikz: 5270 case X86::VRANGEPSZ128rmi: 5271 case X86::VRANGEPSZ128rmikz: 5272 case X86::VRANGEPSZ128rri: 5273 case X86::VRANGEPSZ128rrikz: 5274 case X86::VRANGEPSZ256rmbi: 5275 case X86::VRANGEPSZ256rmbikz: 5276 case X86::VRANGEPSZ256rmi: 5277 case X86::VRANGEPSZ256rmikz: 5278 case X86::VRANGEPSZ256rri: 5279 case X86::VRANGEPSZ256rrikz: 5280 case X86::VRANGEPSZrmbi: 5281 case X86::VRANGEPSZrmbikz: 5282 case X86::VRANGEPSZrmi: 5283 case X86::VRANGEPSZrmikz: 5284 case X86::VRANGEPSZrri: 5285 case X86::VRANGEPSZrrib: 5286 case X86::VRANGEPSZrribkz: 5287 case X86::VRANGEPSZrrikz: 5288 case X86::VRANGESDZrmi: 5289 case X86::VRANGESDZrmikz: 5290 case X86::VRANGESDZrri: 5291 case X86::VRANGESDZrrib: 5292 case X86::VRANGESDZrribkz: 5293 case X86::VRANGESDZrrikz: 5294 case X86::VRANGESSZrmi: 5295 case X86::VRANGESSZrmikz: 5296 case X86::VRANGESSZrri: 5297 case X86::VRANGESSZrrib: 5298 case X86::VRANGESSZrribkz: 5299 case X86::VRANGESSZrrikz: 5300 return Subtarget.hasRANGEFalseDeps(); 5301 case X86::VGETMANTSSZrmi: 5302 case X86::VGETMANTSSZrmikz: 5303 case X86::VGETMANTSSZrri: 5304 case X86::VGETMANTSSZrrib: 5305 case X86::VGETMANTSSZrribkz: 5306 case X86::VGETMANTSSZrrikz: 5307 case X86::VGETMANTSDZrmi: 5308 case X86::VGETMANTSDZrmikz: 5309 case X86::VGETMANTSDZrri: 5310 case X86::VGETMANTSDZrrib: 5311 case X86::VGETMANTSDZrribkz: 5312 case X86::VGETMANTSDZrrikz: 5313 case X86::VGETMANTSHZrmi: 5314 case X86::VGETMANTSHZrmikz: 5315 case X86::VGETMANTSHZrri: 5316 case X86::VGETMANTSHZrrib: 5317 case X86::VGETMANTSHZrribkz: 5318 case X86::VGETMANTSHZrrikz: 5319 case X86::VGETMANTPSZ128rmbi: 5320 case X86::VGETMANTPSZ128rmbikz: 5321 case X86::VGETMANTPSZ128rmi: 5322 case X86::VGETMANTPSZ128rmikz: 5323 case X86::VGETMANTPSZ256rmbi: 5324 case X86::VGETMANTPSZ256rmbikz: 5325 case X86::VGETMANTPSZ256rmi: 5326 case X86::VGETMANTPSZ256rmikz: 5327 case X86::VGETMANTPSZrmbi: 5328 case X86::VGETMANTPSZrmbikz: 5329 case X86::VGETMANTPSZrmi: 5330 case X86::VGETMANTPSZrmikz: 5331 case X86::VGETMANTPDZ128rmbi: 5332 case X86::VGETMANTPDZ128rmbikz: 5333 case X86::VGETMANTPDZ128rmi: 5334 case X86::VGETMANTPDZ128rmikz: 5335 case X86::VGETMANTPDZ256rmbi: 5336 case X86::VGETMANTPDZ256rmbikz: 5337 case X86::VGETMANTPDZ256rmi: 5338 case X86::VGETMANTPDZ256rmikz: 5339 case X86::VGETMANTPDZrmbi: 5340 case X86::VGETMANTPDZrmbikz: 5341 case X86::VGETMANTPDZrmi: 5342 case X86::VGETMANTPDZrmikz: 5343 return Subtarget.hasGETMANTFalseDeps(); 5344 case X86::VPMULLQZ128rm: 5345 case X86::VPMULLQZ128rmb: 5346 case X86::VPMULLQZ128rmbkz: 5347 case X86::VPMULLQZ128rmkz: 5348 case X86::VPMULLQZ128rr: 5349 case X86::VPMULLQZ128rrkz: 5350 case X86::VPMULLQZ256rm: 5351 case X86::VPMULLQZ256rmb: 5352 case X86::VPMULLQZ256rmbkz: 5353 case X86::VPMULLQZ256rmkz: 5354 case X86::VPMULLQZ256rr: 5355 case X86::VPMULLQZ256rrkz: 5356 case X86::VPMULLQZrm: 5357 case X86::VPMULLQZrmb: 5358 case X86::VPMULLQZrmbkz: 5359 case X86::VPMULLQZrmkz: 5360 case X86::VPMULLQZrr: 5361 case X86::VPMULLQZrrkz: 5362 return Subtarget.hasMULLQFalseDeps(); 5363 // GPR 5364 case X86::POPCNT32rm: 5365 case X86::POPCNT32rr: 5366 case X86::POPCNT64rm: 5367 case X86::POPCNT64rr: 5368 return Subtarget.hasPOPCNTFalseDeps(); 5369 case X86::LZCNT32rm: 5370 case X86::LZCNT32rr: 5371 case X86::LZCNT64rm: 5372 case X86::LZCNT64rr: 5373 case X86::TZCNT32rm: 5374 case X86::TZCNT32rr: 5375 case X86::TZCNT64rm: 5376 case X86::TZCNT64rr: 5377 return Subtarget.hasLZCNTFalseDeps(); 5378 } 5379 5380 return false; 5381 } 5382 5383 /// Inform the BreakFalseDeps pass how many idle 5384 /// instructions we would like before a partial register update. 5385 unsigned X86InstrInfo::getPartialRegUpdateClearance( 5386 const MachineInstr &MI, unsigned OpNum, 5387 const TargetRegisterInfo *TRI) const { 5388 if (OpNum != 0 || !hasPartialRegUpdate(MI.getOpcode(), Subtarget)) 5389 return 0; 5390 5391 // If MI is marked as reading Reg, the partial register update is wanted. 5392 const MachineOperand &MO = MI.getOperand(0); 5393 Register Reg = MO.getReg(); 5394 if (Reg.isVirtual()) { 5395 if (MO.readsReg() || MI.readsVirtualRegister(Reg)) 5396 return 0; 5397 } else { 5398 if (MI.readsRegister(Reg, TRI)) 5399 return 0; 5400 } 5401 5402 // If any instructions in the clearance range are reading Reg, insert a 5403 // dependency breaking instruction, which is inexpensive and is likely to 5404 // be hidden in other instruction's cycles. 5405 return PartialRegUpdateClearance; 5406 } 5407 5408 // Return true for any instruction the copies the high bits of the first source 5409 // operand into the unused high bits of the destination operand. 5410 // Also returns true for instructions that have two inputs where one may 5411 // be undef and we want it to use the same register as the other input. 5412 static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum, 5413 bool ForLoadFold = false) { 5414 // Set the OpNum parameter to the first source operand. 5415 switch (Opcode) { 5416 case X86::MMX_PUNPCKHBWrr: 5417 case X86::MMX_PUNPCKHWDrr: 5418 case X86::MMX_PUNPCKHDQrr: 5419 case X86::MMX_PUNPCKLBWrr: 5420 case X86::MMX_PUNPCKLWDrr: 5421 case X86::MMX_PUNPCKLDQrr: 5422 case X86::MOVHLPSrr: 5423 case X86::PACKSSWBrr: 5424 case X86::PACKUSWBrr: 5425 case X86::PACKSSDWrr: 5426 case X86::PACKUSDWrr: 5427 case X86::PUNPCKHBWrr: 5428 case X86::PUNPCKLBWrr: 5429 case X86::PUNPCKHWDrr: 5430 case X86::PUNPCKLWDrr: 5431 case X86::PUNPCKHDQrr: 5432 case X86::PUNPCKLDQrr: 5433 case X86::PUNPCKHQDQrr: 5434 case X86::PUNPCKLQDQrr: 5435 case X86::SHUFPDrri: 5436 case X86::SHUFPSrri: 5437 // These instructions are sometimes used with an undef first or second 5438 // source. Return true here so BreakFalseDeps will assign this source to the 5439 // same register as the first source to avoid a false dependency. 5440 // Operand 1 of these instructions is tied so they're separate from their 5441 // VEX counterparts. 5442 return OpNum == 2 && !ForLoadFold; 5443 5444 case X86::VMOVLHPSrr: 5445 case X86::VMOVLHPSZrr: 5446 case X86::VPACKSSWBrr: 5447 case X86::VPACKUSWBrr: 5448 case X86::VPACKSSDWrr: 5449 case X86::VPACKUSDWrr: 5450 case X86::VPACKSSWBZ128rr: 5451 case X86::VPACKUSWBZ128rr: 5452 case X86::VPACKSSDWZ128rr: 5453 case X86::VPACKUSDWZ128rr: 5454 case X86::VPERM2F128rr: 5455 case X86::VPERM2I128rr: 5456 case X86::VSHUFF32X4Z256rri: 5457 case X86::VSHUFF32X4Zrri: 5458 case X86::VSHUFF64X2Z256rri: 5459 case X86::VSHUFF64X2Zrri: 5460 case X86::VSHUFI32X4Z256rri: 5461 case X86::VSHUFI32X4Zrri: 5462 case X86::VSHUFI64X2Z256rri: 5463 case X86::VSHUFI64X2Zrri: 5464 case X86::VPUNPCKHBWrr: 5465 case X86::VPUNPCKLBWrr: 5466 case X86::VPUNPCKHBWYrr: 5467 case X86::VPUNPCKLBWYrr: 5468 case X86::VPUNPCKHBWZ128rr: 5469 case X86::VPUNPCKLBWZ128rr: 5470 case X86::VPUNPCKHBWZ256rr: 5471 case X86::VPUNPCKLBWZ256rr: 5472 case X86::VPUNPCKHBWZrr: 5473 case X86::VPUNPCKLBWZrr: 5474 case X86::VPUNPCKHWDrr: 5475 case X86::VPUNPCKLWDrr: 5476 case X86::VPUNPCKHWDYrr: 5477 case X86::VPUNPCKLWDYrr: 5478 case X86::VPUNPCKHWDZ128rr: 5479 case X86::VPUNPCKLWDZ128rr: 5480 case X86::VPUNPCKHWDZ256rr: 5481 case X86::VPUNPCKLWDZ256rr: 5482 case X86::VPUNPCKHWDZrr: 5483 case X86::VPUNPCKLWDZrr: 5484 case X86::VPUNPCKHDQrr: 5485 case X86::VPUNPCKLDQrr: 5486 case X86::VPUNPCKHDQYrr: 5487 case X86::VPUNPCKLDQYrr: 5488 case X86::VPUNPCKHDQZ128rr: 5489 case X86::VPUNPCKLDQZ128rr: 5490 case X86::VPUNPCKHDQZ256rr: 5491 case X86::VPUNPCKLDQZ256rr: 5492 case X86::VPUNPCKHDQZrr: 5493 case X86::VPUNPCKLDQZrr: 5494 case X86::VPUNPCKHQDQrr: 5495 case X86::VPUNPCKLQDQrr: 5496 case X86::VPUNPCKHQDQYrr: 5497 case X86::VPUNPCKLQDQYrr: 5498 case X86::VPUNPCKHQDQZ128rr: 5499 case X86::VPUNPCKLQDQZ128rr: 5500 case X86::VPUNPCKHQDQZ256rr: 5501 case X86::VPUNPCKLQDQZ256rr: 5502 case X86::VPUNPCKHQDQZrr: 5503 case X86::VPUNPCKLQDQZrr: 5504 // These instructions are sometimes used with an undef first or second 5505 // source. Return true here so BreakFalseDeps will assign this source to the 5506 // same register as the first source to avoid a false dependency. 5507 return (OpNum == 1 || OpNum == 2) && !ForLoadFold; 5508 5509 case X86::VCVTSI2SSrr: 5510 case X86::VCVTSI2SSrm: 5511 case X86::VCVTSI2SSrr_Int: 5512 case X86::VCVTSI2SSrm_Int: 5513 case X86::VCVTSI642SSrr: 5514 case X86::VCVTSI642SSrm: 5515 case X86::VCVTSI642SSrr_Int: 5516 case X86::VCVTSI642SSrm_Int: 5517 case X86::VCVTSI2SDrr: 5518 case X86::VCVTSI2SDrm: 5519 case X86::VCVTSI2SDrr_Int: 5520 case X86::VCVTSI2SDrm_Int: 5521 case X86::VCVTSI642SDrr: 5522 case X86::VCVTSI642SDrm: 5523 case X86::VCVTSI642SDrr_Int: 5524 case X86::VCVTSI642SDrm_Int: 5525 // AVX-512 5526 case X86::VCVTSI2SSZrr: 5527 case X86::VCVTSI2SSZrm: 5528 case X86::VCVTSI2SSZrr_Int: 5529 case X86::VCVTSI2SSZrrb_Int: 5530 case X86::VCVTSI2SSZrm_Int: 5531 case X86::VCVTSI642SSZrr: 5532 case X86::VCVTSI642SSZrm: 5533 case X86::VCVTSI642SSZrr_Int: 5534 case X86::VCVTSI642SSZrrb_Int: 5535 case X86::VCVTSI642SSZrm_Int: 5536 case X86::VCVTSI2SDZrr: 5537 case X86::VCVTSI2SDZrm: 5538 case X86::VCVTSI2SDZrr_Int: 5539 case X86::VCVTSI2SDZrm_Int: 5540 case X86::VCVTSI642SDZrr: 5541 case X86::VCVTSI642SDZrm: 5542 case X86::VCVTSI642SDZrr_Int: 5543 case X86::VCVTSI642SDZrrb_Int: 5544 case X86::VCVTSI642SDZrm_Int: 5545 case X86::VCVTUSI2SSZrr: 5546 case X86::VCVTUSI2SSZrm: 5547 case X86::VCVTUSI2SSZrr_Int: 5548 case X86::VCVTUSI2SSZrrb_Int: 5549 case X86::VCVTUSI2SSZrm_Int: 5550 case X86::VCVTUSI642SSZrr: 5551 case X86::VCVTUSI642SSZrm: 5552 case X86::VCVTUSI642SSZrr_Int: 5553 case X86::VCVTUSI642SSZrrb_Int: 5554 case X86::VCVTUSI642SSZrm_Int: 5555 case X86::VCVTUSI2SDZrr: 5556 case X86::VCVTUSI2SDZrm: 5557 case X86::VCVTUSI2SDZrr_Int: 5558 case X86::VCVTUSI2SDZrm_Int: 5559 case X86::VCVTUSI642SDZrr: 5560 case X86::VCVTUSI642SDZrm: 5561 case X86::VCVTUSI642SDZrr_Int: 5562 case X86::VCVTUSI642SDZrrb_Int: 5563 case X86::VCVTUSI642SDZrm_Int: 5564 case X86::VCVTSI2SHZrr: 5565 case X86::VCVTSI2SHZrm: 5566 case X86::VCVTSI2SHZrr_Int: 5567 case X86::VCVTSI2SHZrrb_Int: 5568 case X86::VCVTSI2SHZrm_Int: 5569 case X86::VCVTSI642SHZrr: 5570 case X86::VCVTSI642SHZrm: 5571 case X86::VCVTSI642SHZrr_Int: 5572 case X86::VCVTSI642SHZrrb_Int: 5573 case X86::VCVTSI642SHZrm_Int: 5574 case X86::VCVTUSI2SHZrr: 5575 case X86::VCVTUSI2SHZrm: 5576 case X86::VCVTUSI2SHZrr_Int: 5577 case X86::VCVTUSI2SHZrrb_Int: 5578 case X86::VCVTUSI2SHZrm_Int: 5579 case X86::VCVTUSI642SHZrr: 5580 case X86::VCVTUSI642SHZrm: 5581 case X86::VCVTUSI642SHZrr_Int: 5582 case X86::VCVTUSI642SHZrrb_Int: 5583 case X86::VCVTUSI642SHZrm_Int: 5584 // Load folding won't effect the undef register update since the input is 5585 // a GPR. 5586 return OpNum == 1 && !ForLoadFold; 5587 case X86::VCVTSD2SSrr: 5588 case X86::VCVTSD2SSrm: 5589 case X86::VCVTSD2SSrr_Int: 5590 case X86::VCVTSD2SSrm_Int: 5591 case X86::VCVTSS2SDrr: 5592 case X86::VCVTSS2SDrm: 5593 case X86::VCVTSS2SDrr_Int: 5594 case X86::VCVTSS2SDrm_Int: 5595 case X86::VRCPSSr: 5596 case X86::VRCPSSr_Int: 5597 case X86::VRCPSSm: 5598 case X86::VRCPSSm_Int: 5599 case X86::VROUNDSDr: 5600 case X86::VROUNDSDm: 5601 case X86::VROUNDSDr_Int: 5602 case X86::VROUNDSDm_Int: 5603 case X86::VROUNDSSr: 5604 case X86::VROUNDSSm: 5605 case X86::VROUNDSSr_Int: 5606 case X86::VROUNDSSm_Int: 5607 case X86::VRSQRTSSr: 5608 case X86::VRSQRTSSr_Int: 5609 case X86::VRSQRTSSm: 5610 case X86::VRSQRTSSm_Int: 5611 case X86::VSQRTSSr: 5612 case X86::VSQRTSSr_Int: 5613 case X86::VSQRTSSm: 5614 case X86::VSQRTSSm_Int: 5615 case X86::VSQRTSDr: 5616 case X86::VSQRTSDr_Int: 5617 case X86::VSQRTSDm: 5618 case X86::VSQRTSDm_Int: 5619 // AVX-512 5620 case X86::VCVTSD2SSZrr: 5621 case X86::VCVTSD2SSZrr_Int: 5622 case X86::VCVTSD2SSZrrb_Int: 5623 case X86::VCVTSD2SSZrm: 5624 case X86::VCVTSD2SSZrm_Int: 5625 case X86::VCVTSS2SDZrr: 5626 case X86::VCVTSS2SDZrr_Int: 5627 case X86::VCVTSS2SDZrrb_Int: 5628 case X86::VCVTSS2SDZrm: 5629 case X86::VCVTSS2SDZrm_Int: 5630 case X86::VGETEXPSDZr: 5631 case X86::VGETEXPSDZrb: 5632 case X86::VGETEXPSDZm: 5633 case X86::VGETEXPSSZr: 5634 case X86::VGETEXPSSZrb: 5635 case X86::VGETEXPSSZm: 5636 case X86::VGETMANTSDZrri: 5637 case X86::VGETMANTSDZrrib: 5638 case X86::VGETMANTSDZrmi: 5639 case X86::VGETMANTSSZrri: 5640 case X86::VGETMANTSSZrrib: 5641 case X86::VGETMANTSSZrmi: 5642 case X86::VRNDSCALESDZr: 5643 case X86::VRNDSCALESDZr_Int: 5644 case X86::VRNDSCALESDZrb_Int: 5645 case X86::VRNDSCALESDZm: 5646 case X86::VRNDSCALESDZm_Int: 5647 case X86::VRNDSCALESSZr: 5648 case X86::VRNDSCALESSZr_Int: 5649 case X86::VRNDSCALESSZrb_Int: 5650 case X86::VRNDSCALESSZm: 5651 case X86::VRNDSCALESSZm_Int: 5652 case X86::VRCP14SDZrr: 5653 case X86::VRCP14SDZrm: 5654 case X86::VRCP14SSZrr: 5655 case X86::VRCP14SSZrm: 5656 case X86::VRCPSHZrr: 5657 case X86::VRCPSHZrm: 5658 case X86::VRSQRTSHZrr: 5659 case X86::VRSQRTSHZrm: 5660 case X86::VREDUCESHZrmi: 5661 case X86::VREDUCESHZrri: 5662 case X86::VREDUCESHZrrib: 5663 case X86::VGETEXPSHZr: 5664 case X86::VGETEXPSHZrb: 5665 case X86::VGETEXPSHZm: 5666 case X86::VGETMANTSHZrri: 5667 case X86::VGETMANTSHZrrib: 5668 case X86::VGETMANTSHZrmi: 5669 case X86::VRNDSCALESHZr: 5670 case X86::VRNDSCALESHZr_Int: 5671 case X86::VRNDSCALESHZrb_Int: 5672 case X86::VRNDSCALESHZm: 5673 case X86::VRNDSCALESHZm_Int: 5674 case X86::VSQRTSHZr: 5675 case X86::VSQRTSHZr_Int: 5676 case X86::VSQRTSHZrb_Int: 5677 case X86::VSQRTSHZm: 5678 case X86::VSQRTSHZm_Int: 5679 case X86::VRCP28SDZr: 5680 case X86::VRCP28SDZrb: 5681 case X86::VRCP28SDZm: 5682 case X86::VRCP28SSZr: 5683 case X86::VRCP28SSZrb: 5684 case X86::VRCP28SSZm: 5685 case X86::VREDUCESSZrmi: 5686 case X86::VREDUCESSZrri: 5687 case X86::VREDUCESSZrrib: 5688 case X86::VRSQRT14SDZrr: 5689 case X86::VRSQRT14SDZrm: 5690 case X86::VRSQRT14SSZrr: 5691 case X86::VRSQRT14SSZrm: 5692 case X86::VRSQRT28SDZr: 5693 case X86::VRSQRT28SDZrb: 5694 case X86::VRSQRT28SDZm: 5695 case X86::VRSQRT28SSZr: 5696 case X86::VRSQRT28SSZrb: 5697 case X86::VRSQRT28SSZm: 5698 case X86::VSQRTSSZr: 5699 case X86::VSQRTSSZr_Int: 5700 case X86::VSQRTSSZrb_Int: 5701 case X86::VSQRTSSZm: 5702 case X86::VSQRTSSZm_Int: 5703 case X86::VSQRTSDZr: 5704 case X86::VSQRTSDZr_Int: 5705 case X86::VSQRTSDZrb_Int: 5706 case X86::VSQRTSDZm: 5707 case X86::VSQRTSDZm_Int: 5708 case X86::VCVTSD2SHZrr: 5709 case X86::VCVTSD2SHZrr_Int: 5710 case X86::VCVTSD2SHZrrb_Int: 5711 case X86::VCVTSD2SHZrm: 5712 case X86::VCVTSD2SHZrm_Int: 5713 case X86::VCVTSS2SHZrr: 5714 case X86::VCVTSS2SHZrr_Int: 5715 case X86::VCVTSS2SHZrrb_Int: 5716 case X86::VCVTSS2SHZrm: 5717 case X86::VCVTSS2SHZrm_Int: 5718 case X86::VCVTSH2SDZrr: 5719 case X86::VCVTSH2SDZrr_Int: 5720 case X86::VCVTSH2SDZrrb_Int: 5721 case X86::VCVTSH2SDZrm: 5722 case X86::VCVTSH2SDZrm_Int: 5723 case X86::VCVTSH2SSZrr: 5724 case X86::VCVTSH2SSZrr_Int: 5725 case X86::VCVTSH2SSZrrb_Int: 5726 case X86::VCVTSH2SSZrm: 5727 case X86::VCVTSH2SSZrm_Int: 5728 return OpNum == 1; 5729 case X86::VMOVSSZrrk: 5730 case X86::VMOVSDZrrk: 5731 return OpNum == 3 && !ForLoadFold; 5732 case X86::VMOVSSZrrkz: 5733 case X86::VMOVSDZrrkz: 5734 return OpNum == 2 && !ForLoadFold; 5735 } 5736 5737 return false; 5738 } 5739 5740 /// Inform the BreakFalseDeps pass how many idle instructions we would like 5741 /// before certain undef register reads. 5742 /// 5743 /// This catches the VCVTSI2SD family of instructions: 5744 /// 5745 /// vcvtsi2sdq %rax, undef %xmm0, %xmm14 5746 /// 5747 /// We should to be careful *not* to catch VXOR idioms which are presumably 5748 /// handled specially in the pipeline: 5749 /// 5750 /// vxorps undef %xmm1, undef %xmm1, %xmm1 5751 /// 5752 /// Like getPartialRegUpdateClearance, this makes a strong assumption that the 5753 /// high bits that are passed-through are not live. 5754 unsigned 5755 X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned OpNum, 5756 const TargetRegisterInfo *TRI) const { 5757 const MachineOperand &MO = MI.getOperand(OpNum); 5758 if (Register::isPhysicalRegister(MO.getReg()) && 5759 hasUndefRegUpdate(MI.getOpcode(), OpNum)) 5760 return UndefRegClearance; 5761 5762 return 0; 5763 } 5764 5765 void X86InstrInfo::breakPartialRegDependency( 5766 MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const { 5767 Register Reg = MI.getOperand(OpNum).getReg(); 5768 // If MI kills this register, the false dependence is already broken. 5769 if (MI.killsRegister(Reg, TRI)) 5770 return; 5771 5772 if (X86::VR128RegClass.contains(Reg)) { 5773 // These instructions are all floating point domain, so xorps is the best 5774 // choice. 5775 unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr; 5776 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(Opc), Reg) 5777 .addReg(Reg, RegState::Undef) 5778 .addReg(Reg, RegState::Undef); 5779 MI.addRegisterKilled(Reg, TRI, true); 5780 } else if (X86::VR256RegClass.contains(Reg)) { 5781 // Use vxorps to clear the full ymm register. 5782 // It wants to read and write the xmm sub-register. 5783 Register XReg = TRI->getSubReg(Reg, X86::sub_xmm); 5784 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VXORPSrr), XReg) 5785 .addReg(XReg, RegState::Undef) 5786 .addReg(XReg, RegState::Undef) 5787 .addReg(Reg, RegState::ImplicitDefine); 5788 MI.addRegisterKilled(Reg, TRI, true); 5789 } else if (X86::VR128XRegClass.contains(Reg)) { 5790 // Only handle VLX targets. 5791 if (!Subtarget.hasVLX()) 5792 return; 5793 // Since vxorps requires AVX512DQ, vpxord should be the best choice. 5794 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), Reg) 5795 .addReg(Reg, RegState::Undef) 5796 .addReg(Reg, RegState::Undef); 5797 MI.addRegisterKilled(Reg, TRI, true); 5798 } else if (X86::VR256XRegClass.contains(Reg) || 5799 X86::VR512RegClass.contains(Reg)) { 5800 // Only handle VLX targets. 5801 if (!Subtarget.hasVLX()) 5802 return; 5803 // Use vpxord to clear the full ymm/zmm register. 5804 // It wants to read and write the xmm sub-register. 5805 Register XReg = TRI->getSubReg(Reg, X86::sub_xmm); 5806 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), XReg) 5807 .addReg(XReg, RegState::Undef) 5808 .addReg(XReg, RegState::Undef) 5809 .addReg(Reg, RegState::ImplicitDefine); 5810 MI.addRegisterKilled(Reg, TRI, true); 5811 } else if (X86::GR64RegClass.contains(Reg)) { 5812 // Using XOR32rr because it has shorter encoding and zeros up the upper bits 5813 // as well. 5814 Register XReg = TRI->getSubReg(Reg, X86::sub_32bit); 5815 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), XReg) 5816 .addReg(XReg, RegState::Undef) 5817 .addReg(XReg, RegState::Undef) 5818 .addReg(Reg, RegState::ImplicitDefine); 5819 MI.addRegisterKilled(Reg, TRI, true); 5820 } else if (X86::GR32RegClass.contains(Reg)) { 5821 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), Reg) 5822 .addReg(Reg, RegState::Undef) 5823 .addReg(Reg, RegState::Undef); 5824 MI.addRegisterKilled(Reg, TRI, true); 5825 } 5826 } 5827 5828 static void addOperands(MachineInstrBuilder &MIB, ArrayRef<MachineOperand> MOs, 5829 int PtrOffset = 0) { 5830 unsigned NumAddrOps = MOs.size(); 5831 5832 if (NumAddrOps < 4) { 5833 // FrameIndex only - add an immediate offset (whether its zero or not). 5834 for (unsigned i = 0; i != NumAddrOps; ++i) 5835 MIB.add(MOs[i]); 5836 addOffset(MIB, PtrOffset); 5837 } else { 5838 // General Memory Addressing - we need to add any offset to an existing 5839 // offset. 5840 assert(MOs.size() == 5 && "Unexpected memory operand list length"); 5841 for (unsigned i = 0; i != NumAddrOps; ++i) { 5842 const MachineOperand &MO = MOs[i]; 5843 if (i == 3 && PtrOffset != 0) { 5844 MIB.addDisp(MO, PtrOffset); 5845 } else { 5846 MIB.add(MO); 5847 } 5848 } 5849 } 5850 } 5851 5852 static void updateOperandRegConstraints(MachineFunction &MF, 5853 MachineInstr &NewMI, 5854 const TargetInstrInfo &TII) { 5855 MachineRegisterInfo &MRI = MF.getRegInfo(); 5856 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); 5857 5858 for (int Idx : llvm::seq<int>(0, NewMI.getNumOperands())) { 5859 MachineOperand &MO = NewMI.getOperand(Idx); 5860 // We only need to update constraints on virtual register operands. 5861 if (!MO.isReg()) 5862 continue; 5863 Register Reg = MO.getReg(); 5864 if (!Reg.isVirtual()) 5865 continue; 5866 5867 auto *NewRC = MRI.constrainRegClass( 5868 Reg, TII.getRegClass(NewMI.getDesc(), Idx, &TRI, MF)); 5869 if (!NewRC) { 5870 LLVM_DEBUG( 5871 dbgs() << "WARNING: Unable to update register constraint for operand " 5872 << Idx << " of instruction:\n"; 5873 NewMI.dump(); dbgs() << "\n"); 5874 } 5875 } 5876 } 5877 5878 static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode, 5879 ArrayRef<MachineOperand> MOs, 5880 MachineBasicBlock::iterator InsertPt, 5881 MachineInstr &MI, 5882 const TargetInstrInfo &TII) { 5883 // Create the base instruction with the memory operand as the first part. 5884 // Omit the implicit operands, something BuildMI can't do. 5885 MachineInstr *NewMI = 5886 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); 5887 MachineInstrBuilder MIB(MF, NewMI); 5888 addOperands(MIB, MOs); 5889 5890 // Loop over the rest of the ri operands, converting them over. 5891 unsigned NumOps = MI.getDesc().getNumOperands() - 2; 5892 for (unsigned i = 0; i != NumOps; ++i) { 5893 MachineOperand &MO = MI.getOperand(i + 2); 5894 MIB.add(MO); 5895 } 5896 for (const MachineOperand &MO : llvm::drop_begin(MI.operands(), NumOps + 2)) 5897 MIB.add(MO); 5898 5899 updateOperandRegConstraints(MF, *NewMI, TII); 5900 5901 MachineBasicBlock *MBB = InsertPt->getParent(); 5902 MBB->insert(InsertPt, NewMI); 5903 5904 return MIB; 5905 } 5906 5907 static MachineInstr *FuseInst(MachineFunction &MF, unsigned Opcode, 5908 unsigned OpNo, ArrayRef<MachineOperand> MOs, 5909 MachineBasicBlock::iterator InsertPt, 5910 MachineInstr &MI, const TargetInstrInfo &TII, 5911 int PtrOffset = 0) { 5912 // Omit the implicit operands, something BuildMI can't do. 5913 MachineInstr *NewMI = 5914 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); 5915 MachineInstrBuilder MIB(MF, NewMI); 5916 5917 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 5918 MachineOperand &MO = MI.getOperand(i); 5919 if (i == OpNo) { 5920 assert(MO.isReg() && "Expected to fold into reg operand!"); 5921 addOperands(MIB, MOs, PtrOffset); 5922 } else { 5923 MIB.add(MO); 5924 } 5925 } 5926 5927 updateOperandRegConstraints(MF, *NewMI, TII); 5928 5929 // Copy the NoFPExcept flag from the instruction we're fusing. 5930 if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept)) 5931 NewMI->setFlag(MachineInstr::MIFlag::NoFPExcept); 5932 5933 MachineBasicBlock *MBB = InsertPt->getParent(); 5934 MBB->insert(InsertPt, NewMI); 5935 5936 return MIB; 5937 } 5938 5939 static MachineInstr *MakeM0Inst(const TargetInstrInfo &TII, unsigned Opcode, 5940 ArrayRef<MachineOperand> MOs, 5941 MachineBasicBlock::iterator InsertPt, 5942 MachineInstr &MI) { 5943 MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, 5944 MI.getDebugLoc(), TII.get(Opcode)); 5945 addOperands(MIB, MOs); 5946 return MIB.addImm(0); 5947 } 5948 5949 MachineInstr *X86InstrInfo::foldMemoryOperandCustom( 5950 MachineFunction &MF, MachineInstr &MI, unsigned OpNum, 5951 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, 5952 unsigned Size, Align Alignment) const { 5953 switch (MI.getOpcode()) { 5954 case X86::INSERTPSrr: 5955 case X86::VINSERTPSrr: 5956 case X86::VINSERTPSZrr: 5957 // Attempt to convert the load of inserted vector into a fold load 5958 // of a single float. 5959 if (OpNum == 2) { 5960 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); 5961 unsigned ZMask = Imm & 15; 5962 unsigned DstIdx = (Imm >> 4) & 3; 5963 unsigned SrcIdx = (Imm >> 6) & 3; 5964 5965 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 5966 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 5967 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 5968 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment >= Align(4)) { 5969 int PtrOffset = SrcIdx * 4; 5970 unsigned NewImm = (DstIdx << 4) | ZMask; 5971 unsigned NewOpCode = 5972 (MI.getOpcode() == X86::VINSERTPSZrr) ? X86::VINSERTPSZrm : 5973 (MI.getOpcode() == X86::VINSERTPSrr) ? X86::VINSERTPSrm : 5974 X86::INSERTPSrm; 5975 MachineInstr *NewMI = 5976 FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, PtrOffset); 5977 NewMI->getOperand(NewMI->getNumOperands() - 1).setImm(NewImm); 5978 return NewMI; 5979 } 5980 } 5981 break; 5982 case X86::MOVHLPSrr: 5983 case X86::VMOVHLPSrr: 5984 case X86::VMOVHLPSZrr: 5985 // Move the upper 64-bits of the second operand to the lower 64-bits. 5986 // To fold the load, adjust the pointer to the upper and use (V)MOVLPS. 5987 // TODO: In most cases AVX doesn't have a 8-byte alignment requirement. 5988 if (OpNum == 2) { 5989 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 5990 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 5991 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 5992 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment >= Align(8)) { 5993 unsigned NewOpCode = 5994 (MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm : 5995 (MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm : 5996 X86::MOVLPSrm; 5997 MachineInstr *NewMI = 5998 FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, 8); 5999 return NewMI; 6000 } 6001 } 6002 break; 6003 case X86::UNPCKLPDrr: 6004 // If we won't be able to fold this to the memory form of UNPCKL, use 6005 // MOVHPD instead. Done as custom because we can't have this in the load 6006 // table twice. 6007 if (OpNum == 2) { 6008 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6009 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 6010 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 6011 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment < Align(16)) { 6012 MachineInstr *NewMI = 6013 FuseInst(MF, X86::MOVHPDrm, OpNum, MOs, InsertPt, MI, *this); 6014 return NewMI; 6015 } 6016 } 6017 break; 6018 } 6019 6020 return nullptr; 6021 } 6022 6023 static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF, 6024 MachineInstr &MI) { 6025 if (!hasUndefRegUpdate(MI.getOpcode(), 1, /*ForLoadFold*/true) || 6026 !MI.getOperand(1).isReg()) 6027 return false; 6028 6029 // The are two cases we need to handle depending on where in the pipeline 6030 // the folding attempt is being made. 6031 // -Register has the undef flag set. 6032 // -Register is produced by the IMPLICIT_DEF instruction. 6033 6034 if (MI.getOperand(1).isUndef()) 6035 return true; 6036 6037 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 6038 MachineInstr *VRegDef = RegInfo.getUniqueVRegDef(MI.getOperand(1).getReg()); 6039 return VRegDef && VRegDef->isImplicitDef(); 6040 } 6041 6042 MachineInstr *X86InstrInfo::foldMemoryOperandImpl( 6043 MachineFunction &MF, MachineInstr &MI, unsigned OpNum, 6044 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, 6045 unsigned Size, Align Alignment, bool AllowCommute) const { 6046 bool isSlowTwoMemOps = Subtarget.slowTwoMemOps(); 6047 bool isTwoAddrFold = false; 6048 6049 // For CPUs that favor the register form of a call or push, 6050 // do not fold loads into calls or pushes, unless optimizing for size 6051 // aggressively. 6052 if (isSlowTwoMemOps && !MF.getFunction().hasMinSize() && 6053 (MI.getOpcode() == X86::CALL32r || MI.getOpcode() == X86::CALL64r || 6054 MI.getOpcode() == X86::PUSH16r || MI.getOpcode() == X86::PUSH32r || 6055 MI.getOpcode() == X86::PUSH64r)) 6056 return nullptr; 6057 6058 // Avoid partial and undef register update stalls unless optimizing for size. 6059 if (!MF.getFunction().hasOptSize() && 6060 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/true) || 6061 shouldPreventUndefRegUpdateMemFold(MF, MI))) 6062 return nullptr; 6063 6064 unsigned NumOps = MI.getDesc().getNumOperands(); 6065 bool isTwoAddr = 6066 NumOps > 1 && MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1; 6067 6068 // FIXME: AsmPrinter doesn't know how to handle 6069 // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding. 6070 if (MI.getOpcode() == X86::ADD32ri && 6071 MI.getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS) 6072 return nullptr; 6073 6074 // GOTTPOFF relocation loads can only be folded into add instructions. 6075 // FIXME: Need to exclude other relocations that only support specific 6076 // instructions. 6077 if (MOs.size() == X86::AddrNumOperands && 6078 MOs[X86::AddrDisp].getTargetFlags() == X86II::MO_GOTTPOFF && 6079 MI.getOpcode() != X86::ADD64rr) 6080 return nullptr; 6081 6082 MachineInstr *NewMI = nullptr; 6083 6084 // Attempt to fold any custom cases we have. 6085 if (MachineInstr *CustomMI = foldMemoryOperandCustom( 6086 MF, MI, OpNum, MOs, InsertPt, Size, Alignment)) 6087 return CustomMI; 6088 6089 const X86MemoryFoldTableEntry *I = nullptr; 6090 6091 // Folding a memory location into the two-address part of a two-address 6092 // instruction is different than folding it other places. It requires 6093 // replacing the *two* registers with the memory location. 6094 if (isTwoAddr && NumOps >= 2 && OpNum < 2 && MI.getOperand(0).isReg() && 6095 MI.getOperand(1).isReg() && 6096 MI.getOperand(0).getReg() == MI.getOperand(1).getReg()) { 6097 I = lookupTwoAddrFoldTable(MI.getOpcode()); 6098 isTwoAddrFold = true; 6099 } else { 6100 if (OpNum == 0) { 6101 if (MI.getOpcode() == X86::MOV32r0) { 6102 NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, InsertPt, MI); 6103 if (NewMI) 6104 return NewMI; 6105 } 6106 } 6107 6108 I = lookupFoldTable(MI.getOpcode(), OpNum); 6109 } 6110 6111 if (I != nullptr) { 6112 unsigned Opcode = I->DstOp; 6113 bool FoldedLoad = 6114 isTwoAddrFold || (OpNum == 0 && I->Flags & TB_FOLDED_LOAD) || OpNum > 0; 6115 bool FoldedStore = 6116 isTwoAddrFold || (OpNum == 0 && I->Flags & TB_FOLDED_STORE); 6117 MaybeAlign MinAlign = 6118 decodeMaybeAlign((I->Flags & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT); 6119 if (MinAlign && Alignment < *MinAlign) 6120 return nullptr; 6121 bool NarrowToMOV32rm = false; 6122 if (Size) { 6123 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6124 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, 6125 &RI, MF); 6126 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 6127 // Check if it's safe to fold the load. If the size of the object is 6128 // narrower than the load width, then it's not. 6129 // FIXME: Allow scalar intrinsic instructions like ADDSSrm_Int. 6130 if (FoldedLoad && Size < RCSize) { 6131 // If this is a 64-bit load, but the spill slot is 32, then we can do 6132 // a 32-bit load which is implicitly zero-extended. This likely is 6133 // due to live interval analysis remat'ing a load from stack slot. 6134 if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4) 6135 return nullptr; 6136 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) 6137 return nullptr; 6138 Opcode = X86::MOV32rm; 6139 NarrowToMOV32rm = true; 6140 } 6141 // For stores, make sure the size of the object is equal to the size of 6142 // the store. If the object is larger, the extra bits would be garbage. If 6143 // the object is smaller we might overwrite another object or fault. 6144 if (FoldedStore && Size != RCSize) 6145 return nullptr; 6146 } 6147 6148 if (isTwoAddrFold) 6149 NewMI = FuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, *this); 6150 else 6151 NewMI = FuseInst(MF, Opcode, OpNum, MOs, InsertPt, MI, *this); 6152 6153 if (NarrowToMOV32rm) { 6154 // If this is the special case where we use a MOV32rm to load a 32-bit 6155 // value and zero-extend the top bits. Change the destination register 6156 // to a 32-bit one. 6157 Register DstReg = NewMI->getOperand(0).getReg(); 6158 if (DstReg.isPhysical()) 6159 NewMI->getOperand(0).setReg(RI.getSubReg(DstReg, X86::sub_32bit)); 6160 else 6161 NewMI->getOperand(0).setSubReg(X86::sub_32bit); 6162 } 6163 return NewMI; 6164 } 6165 6166 // If the instruction and target operand are commutable, commute the 6167 // instruction and try again. 6168 if (AllowCommute) { 6169 unsigned CommuteOpIdx1 = OpNum, CommuteOpIdx2 = CommuteAnyOperandIndex; 6170 if (findCommutedOpIndices(MI, CommuteOpIdx1, CommuteOpIdx2)) { 6171 bool HasDef = MI.getDesc().getNumDefs(); 6172 Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register(); 6173 Register Reg1 = MI.getOperand(CommuteOpIdx1).getReg(); 6174 Register Reg2 = MI.getOperand(CommuteOpIdx2).getReg(); 6175 bool Tied1 = 6176 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx1, MCOI::TIED_TO); 6177 bool Tied2 = 6178 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx2, MCOI::TIED_TO); 6179 6180 // If either of the commutable operands are tied to the destination 6181 // then we can not commute + fold. 6182 if ((HasDef && Reg0 == Reg1 && Tied1) || 6183 (HasDef && Reg0 == Reg2 && Tied2)) 6184 return nullptr; 6185 6186 MachineInstr *CommutedMI = 6187 commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); 6188 if (!CommutedMI) { 6189 // Unable to commute. 6190 return nullptr; 6191 } 6192 if (CommutedMI != &MI) { 6193 // New instruction. We can't fold from this. 6194 CommutedMI->eraseFromParent(); 6195 return nullptr; 6196 } 6197 6198 // Attempt to fold with the commuted version of the instruction. 6199 NewMI = foldMemoryOperandImpl(MF, MI, CommuteOpIdx2, MOs, InsertPt, Size, 6200 Alignment, /*AllowCommute=*/false); 6201 if (NewMI) 6202 return NewMI; 6203 6204 // Folding failed again - undo the commute before returning. 6205 MachineInstr *UncommutedMI = 6206 commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); 6207 if (!UncommutedMI) { 6208 // Unable to commute. 6209 return nullptr; 6210 } 6211 if (UncommutedMI != &MI) { 6212 // New instruction. It doesn't need to be kept. 6213 UncommutedMI->eraseFromParent(); 6214 return nullptr; 6215 } 6216 6217 // Return here to prevent duplicate fuse failure report. 6218 return nullptr; 6219 } 6220 } 6221 6222 // No fusion 6223 if (PrintFailedFusing && !MI.isCopy()) 6224 dbgs() << "We failed to fuse operand " << OpNum << " in " << MI; 6225 return nullptr; 6226 } 6227 6228 MachineInstr * 6229 X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, 6230 ArrayRef<unsigned> Ops, 6231 MachineBasicBlock::iterator InsertPt, 6232 int FrameIndex, LiveIntervals *LIS, 6233 VirtRegMap *VRM) const { 6234 // Check switch flag 6235 if (NoFusing) 6236 return nullptr; 6237 6238 // Avoid partial and undef register update stalls unless optimizing for size. 6239 if (!MF.getFunction().hasOptSize() && 6240 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/true) || 6241 shouldPreventUndefRegUpdateMemFold(MF, MI))) 6242 return nullptr; 6243 6244 // Don't fold subreg spills, or reloads that use a high subreg. 6245 for (auto Op : Ops) { 6246 MachineOperand &MO = MI.getOperand(Op); 6247 auto SubReg = MO.getSubReg(); 6248 if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi)) 6249 return nullptr; 6250 } 6251 6252 const MachineFrameInfo &MFI = MF.getFrameInfo(); 6253 unsigned Size = MFI.getObjectSize(FrameIndex); 6254 Align Alignment = MFI.getObjectAlign(FrameIndex); 6255 // If the function stack isn't realigned we don't want to fold instructions 6256 // that need increased alignment. 6257 if (!RI.hasStackRealignment(MF)) 6258 Alignment = 6259 std::min(Alignment, Subtarget.getFrameLowering()->getStackAlign()); 6260 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 6261 unsigned NewOpc = 0; 6262 unsigned RCSize = 0; 6263 switch (MI.getOpcode()) { 6264 default: return nullptr; 6265 case X86::TEST8rr: NewOpc = X86::CMP8ri; RCSize = 1; break; 6266 case X86::TEST16rr: NewOpc = X86::CMP16ri8; RCSize = 2; break; 6267 case X86::TEST32rr: NewOpc = X86::CMP32ri8; RCSize = 4; break; 6268 case X86::TEST64rr: NewOpc = X86::CMP64ri8; RCSize = 8; break; 6269 } 6270 // Check if it's safe to fold the load. If the size of the object is 6271 // narrower than the load width, then it's not. 6272 if (Size < RCSize) 6273 return nullptr; 6274 // Change to CMPXXri r, 0 first. 6275 MI.setDesc(get(NewOpc)); 6276 MI.getOperand(1).ChangeToImmediate(0); 6277 } else if (Ops.size() != 1) 6278 return nullptr; 6279 6280 return foldMemoryOperandImpl(MF, MI, Ops[0], 6281 MachineOperand::CreateFI(FrameIndex), InsertPt, 6282 Size, Alignment, /*AllowCommute=*/true); 6283 } 6284 6285 /// Check if \p LoadMI is a partial register load that we can't fold into \p MI 6286 /// because the latter uses contents that wouldn't be defined in the folded 6287 /// version. For instance, this transformation isn't legal: 6288 /// movss (%rdi), %xmm0 6289 /// addps %xmm0, %xmm0 6290 /// -> 6291 /// addps (%rdi), %xmm0 6292 /// 6293 /// But this one is: 6294 /// movss (%rdi), %xmm0 6295 /// addss %xmm0, %xmm0 6296 /// -> 6297 /// addss (%rdi), %xmm0 6298 /// 6299 static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, 6300 const MachineInstr &UserMI, 6301 const MachineFunction &MF) { 6302 unsigned Opc = LoadMI.getOpcode(); 6303 unsigned UserOpc = UserMI.getOpcode(); 6304 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6305 const TargetRegisterClass *RC = 6306 MF.getRegInfo().getRegClass(LoadMI.getOperand(0).getReg()); 6307 unsigned RegSize = TRI.getRegSizeInBits(*RC); 6308 6309 if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm || Opc == X86::VMOVSSZrm || 6310 Opc == X86::MOVSSrm_alt || Opc == X86::VMOVSSrm_alt || 6311 Opc == X86::VMOVSSZrm_alt) && 6312 RegSize > 32) { 6313 // These instructions only load 32 bits, we can't fold them if the 6314 // destination register is wider than 32 bits (4 bytes), and its user 6315 // instruction isn't scalar (SS). 6316 switch (UserOpc) { 6317 case X86::CVTSS2SDrr_Int: 6318 case X86::VCVTSS2SDrr_Int: 6319 case X86::VCVTSS2SDZrr_Int: 6320 case X86::VCVTSS2SDZrr_Intk: 6321 case X86::VCVTSS2SDZrr_Intkz: 6322 case X86::CVTSS2SIrr_Int: case X86::CVTSS2SI64rr_Int: 6323 case X86::VCVTSS2SIrr_Int: case X86::VCVTSS2SI64rr_Int: 6324 case X86::VCVTSS2SIZrr_Int: case X86::VCVTSS2SI64Zrr_Int: 6325 case X86::CVTTSS2SIrr_Int: case X86::CVTTSS2SI64rr_Int: 6326 case X86::VCVTTSS2SIrr_Int: case X86::VCVTTSS2SI64rr_Int: 6327 case X86::VCVTTSS2SIZrr_Int: case X86::VCVTTSS2SI64Zrr_Int: 6328 case X86::VCVTSS2USIZrr_Int: case X86::VCVTSS2USI64Zrr_Int: 6329 case X86::VCVTTSS2USIZrr_Int: case X86::VCVTTSS2USI64Zrr_Int: 6330 case X86::RCPSSr_Int: case X86::VRCPSSr_Int: 6331 case X86::RSQRTSSr_Int: case X86::VRSQRTSSr_Int: 6332 case X86::ROUNDSSr_Int: case X86::VROUNDSSr_Int: 6333 case X86::COMISSrr_Int: case X86::VCOMISSrr_Int: case X86::VCOMISSZrr_Int: 6334 case X86::UCOMISSrr_Int:case X86::VUCOMISSrr_Int:case X86::VUCOMISSZrr_Int: 6335 case X86::ADDSSrr_Int: case X86::VADDSSrr_Int: case X86::VADDSSZrr_Int: 6336 case X86::CMPSSrr_Int: case X86::VCMPSSrr_Int: case X86::VCMPSSZrr_Int: 6337 case X86::DIVSSrr_Int: case X86::VDIVSSrr_Int: case X86::VDIVSSZrr_Int: 6338 case X86::MAXSSrr_Int: case X86::VMAXSSrr_Int: case X86::VMAXSSZrr_Int: 6339 case X86::MINSSrr_Int: case X86::VMINSSrr_Int: case X86::VMINSSZrr_Int: 6340 case X86::MULSSrr_Int: case X86::VMULSSrr_Int: case X86::VMULSSZrr_Int: 6341 case X86::SQRTSSr_Int: case X86::VSQRTSSr_Int: case X86::VSQRTSSZr_Int: 6342 case X86::SUBSSrr_Int: case X86::VSUBSSrr_Int: case X86::VSUBSSZrr_Int: 6343 case X86::VADDSSZrr_Intk: case X86::VADDSSZrr_Intkz: 6344 case X86::VCMPSSZrr_Intk: 6345 case X86::VDIVSSZrr_Intk: case X86::VDIVSSZrr_Intkz: 6346 case X86::VMAXSSZrr_Intk: case X86::VMAXSSZrr_Intkz: 6347 case X86::VMINSSZrr_Intk: case X86::VMINSSZrr_Intkz: 6348 case X86::VMULSSZrr_Intk: case X86::VMULSSZrr_Intkz: 6349 case X86::VSQRTSSZr_Intk: case X86::VSQRTSSZr_Intkz: 6350 case X86::VSUBSSZrr_Intk: case X86::VSUBSSZrr_Intkz: 6351 case X86::VFMADDSS4rr_Int: case X86::VFNMADDSS4rr_Int: 6352 case X86::VFMSUBSS4rr_Int: case X86::VFNMSUBSS4rr_Int: 6353 case X86::VFMADD132SSr_Int: case X86::VFNMADD132SSr_Int: 6354 case X86::VFMADD213SSr_Int: case X86::VFNMADD213SSr_Int: 6355 case X86::VFMADD231SSr_Int: case X86::VFNMADD231SSr_Int: 6356 case X86::VFMSUB132SSr_Int: case X86::VFNMSUB132SSr_Int: 6357 case X86::VFMSUB213SSr_Int: case X86::VFNMSUB213SSr_Int: 6358 case X86::VFMSUB231SSr_Int: case X86::VFNMSUB231SSr_Int: 6359 case X86::VFMADD132SSZr_Int: case X86::VFNMADD132SSZr_Int: 6360 case X86::VFMADD213SSZr_Int: case X86::VFNMADD213SSZr_Int: 6361 case X86::VFMADD231SSZr_Int: case X86::VFNMADD231SSZr_Int: 6362 case X86::VFMSUB132SSZr_Int: case X86::VFNMSUB132SSZr_Int: 6363 case X86::VFMSUB213SSZr_Int: case X86::VFNMSUB213SSZr_Int: 6364 case X86::VFMSUB231SSZr_Int: case X86::VFNMSUB231SSZr_Int: 6365 case X86::VFMADD132SSZr_Intk: case X86::VFNMADD132SSZr_Intk: 6366 case X86::VFMADD213SSZr_Intk: case X86::VFNMADD213SSZr_Intk: 6367 case X86::VFMADD231SSZr_Intk: case X86::VFNMADD231SSZr_Intk: 6368 case X86::VFMSUB132SSZr_Intk: case X86::VFNMSUB132SSZr_Intk: 6369 case X86::VFMSUB213SSZr_Intk: case X86::VFNMSUB213SSZr_Intk: 6370 case X86::VFMSUB231SSZr_Intk: case X86::VFNMSUB231SSZr_Intk: 6371 case X86::VFMADD132SSZr_Intkz: case X86::VFNMADD132SSZr_Intkz: 6372 case X86::VFMADD213SSZr_Intkz: case X86::VFNMADD213SSZr_Intkz: 6373 case X86::VFMADD231SSZr_Intkz: case X86::VFNMADD231SSZr_Intkz: 6374 case X86::VFMSUB132SSZr_Intkz: case X86::VFNMSUB132SSZr_Intkz: 6375 case X86::VFMSUB213SSZr_Intkz: case X86::VFNMSUB213SSZr_Intkz: 6376 case X86::VFMSUB231SSZr_Intkz: case X86::VFNMSUB231SSZr_Intkz: 6377 case X86::VFIXUPIMMSSZrri: 6378 case X86::VFIXUPIMMSSZrrik: 6379 case X86::VFIXUPIMMSSZrrikz: 6380 case X86::VFPCLASSSSZrr: 6381 case X86::VFPCLASSSSZrrk: 6382 case X86::VGETEXPSSZr: 6383 case X86::VGETEXPSSZrk: 6384 case X86::VGETEXPSSZrkz: 6385 case X86::VGETMANTSSZrri: 6386 case X86::VGETMANTSSZrrik: 6387 case X86::VGETMANTSSZrrikz: 6388 case X86::VRANGESSZrri: 6389 case X86::VRANGESSZrrik: 6390 case X86::VRANGESSZrrikz: 6391 case X86::VRCP14SSZrr: 6392 case X86::VRCP14SSZrrk: 6393 case X86::VRCP14SSZrrkz: 6394 case X86::VRCP28SSZr: 6395 case X86::VRCP28SSZrk: 6396 case X86::VRCP28SSZrkz: 6397 case X86::VREDUCESSZrri: 6398 case X86::VREDUCESSZrrik: 6399 case X86::VREDUCESSZrrikz: 6400 case X86::VRNDSCALESSZr_Int: 6401 case X86::VRNDSCALESSZr_Intk: 6402 case X86::VRNDSCALESSZr_Intkz: 6403 case X86::VRSQRT14SSZrr: 6404 case X86::VRSQRT14SSZrrk: 6405 case X86::VRSQRT14SSZrrkz: 6406 case X86::VRSQRT28SSZr: 6407 case X86::VRSQRT28SSZrk: 6408 case X86::VRSQRT28SSZrkz: 6409 case X86::VSCALEFSSZrr: 6410 case X86::VSCALEFSSZrrk: 6411 case X86::VSCALEFSSZrrkz: 6412 return false; 6413 default: 6414 return true; 6415 } 6416 } 6417 6418 if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm || Opc == X86::VMOVSDZrm || 6419 Opc == X86::MOVSDrm_alt || Opc == X86::VMOVSDrm_alt || 6420 Opc == X86::VMOVSDZrm_alt) && 6421 RegSize > 64) { 6422 // These instructions only load 64 bits, we can't fold them if the 6423 // destination register is wider than 64 bits (8 bytes), and its user 6424 // instruction isn't scalar (SD). 6425 switch (UserOpc) { 6426 case X86::CVTSD2SSrr_Int: 6427 case X86::VCVTSD2SSrr_Int: 6428 case X86::VCVTSD2SSZrr_Int: 6429 case X86::VCVTSD2SSZrr_Intk: 6430 case X86::VCVTSD2SSZrr_Intkz: 6431 case X86::CVTSD2SIrr_Int: case X86::CVTSD2SI64rr_Int: 6432 case X86::VCVTSD2SIrr_Int: case X86::VCVTSD2SI64rr_Int: 6433 case X86::VCVTSD2SIZrr_Int: case X86::VCVTSD2SI64Zrr_Int: 6434 case X86::CVTTSD2SIrr_Int: case X86::CVTTSD2SI64rr_Int: 6435 case X86::VCVTTSD2SIrr_Int: case X86::VCVTTSD2SI64rr_Int: 6436 case X86::VCVTTSD2SIZrr_Int: case X86::VCVTTSD2SI64Zrr_Int: 6437 case X86::VCVTSD2USIZrr_Int: case X86::VCVTSD2USI64Zrr_Int: 6438 case X86::VCVTTSD2USIZrr_Int: case X86::VCVTTSD2USI64Zrr_Int: 6439 case X86::ROUNDSDr_Int: case X86::VROUNDSDr_Int: 6440 case X86::COMISDrr_Int: case X86::VCOMISDrr_Int: case X86::VCOMISDZrr_Int: 6441 case X86::UCOMISDrr_Int:case X86::VUCOMISDrr_Int:case X86::VUCOMISDZrr_Int: 6442 case X86::ADDSDrr_Int: case X86::VADDSDrr_Int: case X86::VADDSDZrr_Int: 6443 case X86::CMPSDrr_Int: case X86::VCMPSDrr_Int: case X86::VCMPSDZrr_Int: 6444 case X86::DIVSDrr_Int: case X86::VDIVSDrr_Int: case X86::VDIVSDZrr_Int: 6445 case X86::MAXSDrr_Int: case X86::VMAXSDrr_Int: case X86::VMAXSDZrr_Int: 6446 case X86::MINSDrr_Int: case X86::VMINSDrr_Int: case X86::VMINSDZrr_Int: 6447 case X86::MULSDrr_Int: case X86::VMULSDrr_Int: case X86::VMULSDZrr_Int: 6448 case X86::SQRTSDr_Int: case X86::VSQRTSDr_Int: case X86::VSQRTSDZr_Int: 6449 case X86::SUBSDrr_Int: case X86::VSUBSDrr_Int: case X86::VSUBSDZrr_Int: 6450 case X86::VADDSDZrr_Intk: case X86::VADDSDZrr_Intkz: 6451 case X86::VCMPSDZrr_Intk: 6452 case X86::VDIVSDZrr_Intk: case X86::VDIVSDZrr_Intkz: 6453 case X86::VMAXSDZrr_Intk: case X86::VMAXSDZrr_Intkz: 6454 case X86::VMINSDZrr_Intk: case X86::VMINSDZrr_Intkz: 6455 case X86::VMULSDZrr_Intk: case X86::VMULSDZrr_Intkz: 6456 case X86::VSQRTSDZr_Intk: case X86::VSQRTSDZr_Intkz: 6457 case X86::VSUBSDZrr_Intk: case X86::VSUBSDZrr_Intkz: 6458 case X86::VFMADDSD4rr_Int: case X86::VFNMADDSD4rr_Int: 6459 case X86::VFMSUBSD4rr_Int: case X86::VFNMSUBSD4rr_Int: 6460 case X86::VFMADD132SDr_Int: case X86::VFNMADD132SDr_Int: 6461 case X86::VFMADD213SDr_Int: case X86::VFNMADD213SDr_Int: 6462 case X86::VFMADD231SDr_Int: case X86::VFNMADD231SDr_Int: 6463 case X86::VFMSUB132SDr_Int: case X86::VFNMSUB132SDr_Int: 6464 case X86::VFMSUB213SDr_Int: case X86::VFNMSUB213SDr_Int: 6465 case X86::VFMSUB231SDr_Int: case X86::VFNMSUB231SDr_Int: 6466 case X86::VFMADD132SDZr_Int: case X86::VFNMADD132SDZr_Int: 6467 case X86::VFMADD213SDZr_Int: case X86::VFNMADD213SDZr_Int: 6468 case X86::VFMADD231SDZr_Int: case X86::VFNMADD231SDZr_Int: 6469 case X86::VFMSUB132SDZr_Int: case X86::VFNMSUB132SDZr_Int: 6470 case X86::VFMSUB213SDZr_Int: case X86::VFNMSUB213SDZr_Int: 6471 case X86::VFMSUB231SDZr_Int: case X86::VFNMSUB231SDZr_Int: 6472 case X86::VFMADD132SDZr_Intk: case X86::VFNMADD132SDZr_Intk: 6473 case X86::VFMADD213SDZr_Intk: case X86::VFNMADD213SDZr_Intk: 6474 case X86::VFMADD231SDZr_Intk: case X86::VFNMADD231SDZr_Intk: 6475 case X86::VFMSUB132SDZr_Intk: case X86::VFNMSUB132SDZr_Intk: 6476 case X86::VFMSUB213SDZr_Intk: case X86::VFNMSUB213SDZr_Intk: 6477 case X86::VFMSUB231SDZr_Intk: case X86::VFNMSUB231SDZr_Intk: 6478 case X86::VFMADD132SDZr_Intkz: case X86::VFNMADD132SDZr_Intkz: 6479 case X86::VFMADD213SDZr_Intkz: case X86::VFNMADD213SDZr_Intkz: 6480 case X86::VFMADD231SDZr_Intkz: case X86::VFNMADD231SDZr_Intkz: 6481 case X86::VFMSUB132SDZr_Intkz: case X86::VFNMSUB132SDZr_Intkz: 6482 case X86::VFMSUB213SDZr_Intkz: case X86::VFNMSUB213SDZr_Intkz: 6483 case X86::VFMSUB231SDZr_Intkz: case X86::VFNMSUB231SDZr_Intkz: 6484 case X86::VFIXUPIMMSDZrri: 6485 case X86::VFIXUPIMMSDZrrik: 6486 case X86::VFIXUPIMMSDZrrikz: 6487 case X86::VFPCLASSSDZrr: 6488 case X86::VFPCLASSSDZrrk: 6489 case X86::VGETEXPSDZr: 6490 case X86::VGETEXPSDZrk: 6491 case X86::VGETEXPSDZrkz: 6492 case X86::VGETMANTSDZrri: 6493 case X86::VGETMANTSDZrrik: 6494 case X86::VGETMANTSDZrrikz: 6495 case X86::VRANGESDZrri: 6496 case X86::VRANGESDZrrik: 6497 case X86::VRANGESDZrrikz: 6498 case X86::VRCP14SDZrr: 6499 case X86::VRCP14SDZrrk: 6500 case X86::VRCP14SDZrrkz: 6501 case X86::VRCP28SDZr: 6502 case X86::VRCP28SDZrk: 6503 case X86::VRCP28SDZrkz: 6504 case X86::VREDUCESDZrri: 6505 case X86::VREDUCESDZrrik: 6506 case X86::VREDUCESDZrrikz: 6507 case X86::VRNDSCALESDZr_Int: 6508 case X86::VRNDSCALESDZr_Intk: 6509 case X86::VRNDSCALESDZr_Intkz: 6510 case X86::VRSQRT14SDZrr: 6511 case X86::VRSQRT14SDZrrk: 6512 case X86::VRSQRT14SDZrrkz: 6513 case X86::VRSQRT28SDZr: 6514 case X86::VRSQRT28SDZrk: 6515 case X86::VRSQRT28SDZrkz: 6516 case X86::VSCALEFSDZrr: 6517 case X86::VSCALEFSDZrrk: 6518 case X86::VSCALEFSDZrrkz: 6519 return false; 6520 default: 6521 return true; 6522 } 6523 } 6524 6525 if ((Opc == X86::VMOVSHZrm || Opc == X86::VMOVSHZrm_alt) && RegSize > 16) { 6526 // These instructions only load 16 bits, we can't fold them if the 6527 // destination register is wider than 16 bits (2 bytes), and its user 6528 // instruction isn't scalar (SH). 6529 switch (UserOpc) { 6530 case X86::VADDSHZrr_Int: 6531 case X86::VCMPSHZrr_Int: 6532 case X86::VDIVSHZrr_Int: 6533 case X86::VMAXSHZrr_Int: 6534 case X86::VMINSHZrr_Int: 6535 case X86::VMULSHZrr_Int: 6536 case X86::VSUBSHZrr_Int: 6537 case X86::VADDSHZrr_Intk: case X86::VADDSHZrr_Intkz: 6538 case X86::VCMPSHZrr_Intk: 6539 case X86::VDIVSHZrr_Intk: case X86::VDIVSHZrr_Intkz: 6540 case X86::VMAXSHZrr_Intk: case X86::VMAXSHZrr_Intkz: 6541 case X86::VMINSHZrr_Intk: case X86::VMINSHZrr_Intkz: 6542 case X86::VMULSHZrr_Intk: case X86::VMULSHZrr_Intkz: 6543 case X86::VSUBSHZrr_Intk: case X86::VSUBSHZrr_Intkz: 6544 case X86::VFMADD132SHZr_Int: case X86::VFNMADD132SHZr_Int: 6545 case X86::VFMADD213SHZr_Int: case X86::VFNMADD213SHZr_Int: 6546 case X86::VFMADD231SHZr_Int: case X86::VFNMADD231SHZr_Int: 6547 case X86::VFMSUB132SHZr_Int: case X86::VFNMSUB132SHZr_Int: 6548 case X86::VFMSUB213SHZr_Int: case X86::VFNMSUB213SHZr_Int: 6549 case X86::VFMSUB231SHZr_Int: case X86::VFNMSUB231SHZr_Int: 6550 case X86::VFMADD132SHZr_Intk: case X86::VFNMADD132SHZr_Intk: 6551 case X86::VFMADD213SHZr_Intk: case X86::VFNMADD213SHZr_Intk: 6552 case X86::VFMADD231SHZr_Intk: case X86::VFNMADD231SHZr_Intk: 6553 case X86::VFMSUB132SHZr_Intk: case X86::VFNMSUB132SHZr_Intk: 6554 case X86::VFMSUB213SHZr_Intk: case X86::VFNMSUB213SHZr_Intk: 6555 case X86::VFMSUB231SHZr_Intk: case X86::VFNMSUB231SHZr_Intk: 6556 case X86::VFMADD132SHZr_Intkz: case X86::VFNMADD132SHZr_Intkz: 6557 case X86::VFMADD213SHZr_Intkz: case X86::VFNMADD213SHZr_Intkz: 6558 case X86::VFMADD231SHZr_Intkz: case X86::VFNMADD231SHZr_Intkz: 6559 case X86::VFMSUB132SHZr_Intkz: case X86::VFNMSUB132SHZr_Intkz: 6560 case X86::VFMSUB213SHZr_Intkz: case X86::VFNMSUB213SHZr_Intkz: 6561 case X86::VFMSUB231SHZr_Intkz: case X86::VFNMSUB231SHZr_Intkz: 6562 return false; 6563 default: 6564 return true; 6565 } 6566 } 6567 6568 return false; 6569 } 6570 6571 MachineInstr *X86InstrInfo::foldMemoryOperandImpl( 6572 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 6573 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, 6574 LiveIntervals *LIS) const { 6575 6576 // TODO: Support the case where LoadMI loads a wide register, but MI 6577 // only uses a subreg. 6578 for (auto Op : Ops) { 6579 if (MI.getOperand(Op).getSubReg()) 6580 return nullptr; 6581 } 6582 6583 // If loading from a FrameIndex, fold directly from the FrameIndex. 6584 unsigned NumOps = LoadMI.getDesc().getNumOperands(); 6585 int FrameIndex; 6586 if (isLoadFromStackSlot(LoadMI, FrameIndex)) { 6587 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) 6588 return nullptr; 6589 return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex, LIS); 6590 } 6591 6592 // Check switch flag 6593 if (NoFusing) return nullptr; 6594 6595 // Avoid partial and undef register update stalls unless optimizing for size. 6596 if (!MF.getFunction().hasOptSize() && 6597 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/true) || 6598 shouldPreventUndefRegUpdateMemFold(MF, MI))) 6599 return nullptr; 6600 6601 // Determine the alignment of the load. 6602 Align Alignment; 6603 if (LoadMI.hasOneMemOperand()) 6604 Alignment = (*LoadMI.memoperands_begin())->getAlign(); 6605 else 6606 switch (LoadMI.getOpcode()) { 6607 case X86::AVX512_512_SET0: 6608 case X86::AVX512_512_SETALLONES: 6609 Alignment = Align(64); 6610 break; 6611 case X86::AVX2_SETALLONES: 6612 case X86::AVX1_SETALLONES: 6613 case X86::AVX_SET0: 6614 case X86::AVX512_256_SET0: 6615 Alignment = Align(32); 6616 break; 6617 case X86::V_SET0: 6618 case X86::V_SETALLONES: 6619 case X86::AVX512_128_SET0: 6620 case X86::FsFLD0F128: 6621 case X86::AVX512_FsFLD0F128: 6622 Alignment = Align(16); 6623 break; 6624 case X86::MMX_SET0: 6625 case X86::FsFLD0SD: 6626 case X86::AVX512_FsFLD0SD: 6627 Alignment = Align(8); 6628 break; 6629 case X86::FsFLD0SS: 6630 case X86::AVX512_FsFLD0SS: 6631 Alignment = Align(4); 6632 break; 6633 case X86::FsFLD0SH: 6634 case X86::AVX512_FsFLD0SH: 6635 Alignment = Align(2); 6636 break; 6637 default: 6638 return nullptr; 6639 } 6640 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 6641 unsigned NewOpc = 0; 6642 switch (MI.getOpcode()) { 6643 default: return nullptr; 6644 case X86::TEST8rr: NewOpc = X86::CMP8ri; break; 6645 case X86::TEST16rr: NewOpc = X86::CMP16ri8; break; 6646 case X86::TEST32rr: NewOpc = X86::CMP32ri8; break; 6647 case X86::TEST64rr: NewOpc = X86::CMP64ri8; break; 6648 } 6649 // Change to CMPXXri r, 0 first. 6650 MI.setDesc(get(NewOpc)); 6651 MI.getOperand(1).ChangeToImmediate(0); 6652 } else if (Ops.size() != 1) 6653 return nullptr; 6654 6655 // Make sure the subregisters match. 6656 // Otherwise we risk changing the size of the load. 6657 if (LoadMI.getOperand(0).getSubReg() != MI.getOperand(Ops[0]).getSubReg()) 6658 return nullptr; 6659 6660 SmallVector<MachineOperand,X86::AddrNumOperands> MOs; 6661 switch (LoadMI.getOpcode()) { 6662 case X86::MMX_SET0: 6663 case X86::V_SET0: 6664 case X86::V_SETALLONES: 6665 case X86::AVX2_SETALLONES: 6666 case X86::AVX1_SETALLONES: 6667 case X86::AVX_SET0: 6668 case X86::AVX512_128_SET0: 6669 case X86::AVX512_256_SET0: 6670 case X86::AVX512_512_SET0: 6671 case X86::AVX512_512_SETALLONES: 6672 case X86::FsFLD0SH: 6673 case X86::AVX512_FsFLD0SH: 6674 case X86::FsFLD0SD: 6675 case X86::AVX512_FsFLD0SD: 6676 case X86::FsFLD0SS: 6677 case X86::AVX512_FsFLD0SS: 6678 case X86::FsFLD0F128: 6679 case X86::AVX512_FsFLD0F128: { 6680 // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure. 6681 // Create a constant-pool entry and operands to load from it. 6682 6683 // Medium and large mode can't fold loads this way. 6684 if (MF.getTarget().getCodeModel() != CodeModel::Small && 6685 MF.getTarget().getCodeModel() != CodeModel::Kernel) 6686 return nullptr; 6687 6688 // x86-32 PIC requires a PIC base register for constant pools. 6689 unsigned PICBase = 0; 6690 // Since we're using Small or Kernel code model, we can always use 6691 // RIP-relative addressing for a smaller encoding. 6692 if (Subtarget.is64Bit()) { 6693 PICBase = X86::RIP; 6694 } else if (MF.getTarget().isPositionIndependent()) { 6695 // FIXME: PICBase = getGlobalBaseReg(&MF); 6696 // This doesn't work for several reasons. 6697 // 1. GlobalBaseReg may have been spilled. 6698 // 2. It may not be live at MI. 6699 return nullptr; 6700 } 6701 6702 // Create a constant-pool entry. 6703 MachineConstantPool &MCP = *MF.getConstantPool(); 6704 Type *Ty; 6705 unsigned Opc = LoadMI.getOpcode(); 6706 if (Opc == X86::FsFLD0SS || Opc == X86::AVX512_FsFLD0SS) 6707 Ty = Type::getFloatTy(MF.getFunction().getContext()); 6708 else if (Opc == X86::FsFLD0SD || Opc == X86::AVX512_FsFLD0SD) 6709 Ty = Type::getDoubleTy(MF.getFunction().getContext()); 6710 else if (Opc == X86::FsFLD0F128 || Opc == X86::AVX512_FsFLD0F128) 6711 Ty = Type::getFP128Ty(MF.getFunction().getContext()); 6712 else if (Opc == X86::FsFLD0SH || Opc == X86::AVX512_FsFLD0SH) 6713 Ty = Type::getHalfTy(MF.getFunction().getContext()); 6714 else if (Opc == X86::AVX512_512_SET0 || Opc == X86::AVX512_512_SETALLONES) 6715 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 6716 16); 6717 else if (Opc == X86::AVX2_SETALLONES || Opc == X86::AVX_SET0 || 6718 Opc == X86::AVX512_256_SET0 || Opc == X86::AVX1_SETALLONES) 6719 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 6720 8); 6721 else if (Opc == X86::MMX_SET0) 6722 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 6723 2); 6724 else 6725 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 6726 4); 6727 6728 bool IsAllOnes = (Opc == X86::V_SETALLONES || Opc == X86::AVX2_SETALLONES || 6729 Opc == X86::AVX512_512_SETALLONES || 6730 Opc == X86::AVX1_SETALLONES); 6731 const Constant *C = IsAllOnes ? Constant::getAllOnesValue(Ty) : 6732 Constant::getNullValue(Ty); 6733 unsigned CPI = MCP.getConstantPoolIndex(C, Alignment); 6734 6735 // Create operands to load from the constant pool entry. 6736 MOs.push_back(MachineOperand::CreateReg(PICBase, false)); 6737 MOs.push_back(MachineOperand::CreateImm(1)); 6738 MOs.push_back(MachineOperand::CreateReg(0, false)); 6739 MOs.push_back(MachineOperand::CreateCPI(CPI, 0)); 6740 MOs.push_back(MachineOperand::CreateReg(0, false)); 6741 break; 6742 } 6743 default: { 6744 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) 6745 return nullptr; 6746 6747 // Folding a normal load. Just copy the load's address operands. 6748 MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, 6749 LoadMI.operands_begin() + NumOps); 6750 break; 6751 } 6752 } 6753 return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, InsertPt, 6754 /*Size=*/0, Alignment, /*AllowCommute=*/true); 6755 } 6756 6757 static SmallVector<MachineMemOperand *, 2> 6758 extractLoadMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { 6759 SmallVector<MachineMemOperand *, 2> LoadMMOs; 6760 6761 for (MachineMemOperand *MMO : MMOs) { 6762 if (!MMO->isLoad()) 6763 continue; 6764 6765 if (!MMO->isStore()) { 6766 // Reuse the MMO. 6767 LoadMMOs.push_back(MMO); 6768 } else { 6769 // Clone the MMO and unset the store flag. 6770 LoadMMOs.push_back(MF.getMachineMemOperand( 6771 MMO, MMO->getFlags() & ~MachineMemOperand::MOStore)); 6772 } 6773 } 6774 6775 return LoadMMOs; 6776 } 6777 6778 static SmallVector<MachineMemOperand *, 2> 6779 extractStoreMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { 6780 SmallVector<MachineMemOperand *, 2> StoreMMOs; 6781 6782 for (MachineMemOperand *MMO : MMOs) { 6783 if (!MMO->isStore()) 6784 continue; 6785 6786 if (!MMO->isLoad()) { 6787 // Reuse the MMO. 6788 StoreMMOs.push_back(MMO); 6789 } else { 6790 // Clone the MMO and unset the load flag. 6791 StoreMMOs.push_back(MF.getMachineMemOperand( 6792 MMO, MMO->getFlags() & ~MachineMemOperand::MOLoad)); 6793 } 6794 } 6795 6796 return StoreMMOs; 6797 } 6798 6799 static unsigned getBroadcastOpcode(const X86MemoryFoldTableEntry *I, 6800 const TargetRegisterClass *RC, 6801 const X86Subtarget &STI) { 6802 assert(STI.hasAVX512() && "Expected at least AVX512!"); 6803 unsigned SpillSize = STI.getRegisterInfo()->getSpillSize(*RC); 6804 assert((SpillSize == 64 || STI.hasVLX()) && 6805 "Can't broadcast less than 64 bytes without AVX512VL!"); 6806 6807 switch (I->Flags & TB_BCAST_MASK) { 6808 default: llvm_unreachable("Unexpected broadcast type!"); 6809 case TB_BCAST_D: 6810 switch (SpillSize) { 6811 default: llvm_unreachable("Unknown spill size"); 6812 case 16: return X86::VPBROADCASTDZ128rm; 6813 case 32: return X86::VPBROADCASTDZ256rm; 6814 case 64: return X86::VPBROADCASTDZrm; 6815 } 6816 break; 6817 case TB_BCAST_Q: 6818 switch (SpillSize) { 6819 default: llvm_unreachable("Unknown spill size"); 6820 case 16: return X86::VPBROADCASTQZ128rm; 6821 case 32: return X86::VPBROADCASTQZ256rm; 6822 case 64: return X86::VPBROADCASTQZrm; 6823 } 6824 break; 6825 case TB_BCAST_SS: 6826 switch (SpillSize) { 6827 default: llvm_unreachable("Unknown spill size"); 6828 case 16: return X86::VBROADCASTSSZ128rm; 6829 case 32: return X86::VBROADCASTSSZ256rm; 6830 case 64: return X86::VBROADCASTSSZrm; 6831 } 6832 break; 6833 case TB_BCAST_SD: 6834 switch (SpillSize) { 6835 default: llvm_unreachable("Unknown spill size"); 6836 case 16: return X86::VMOVDDUPZ128rm; 6837 case 32: return X86::VBROADCASTSDZ256rm; 6838 case 64: return X86::VBROADCASTSDZrm; 6839 } 6840 break; 6841 } 6842 } 6843 6844 bool X86InstrInfo::unfoldMemoryOperand( 6845 MachineFunction &MF, MachineInstr &MI, unsigned Reg, bool UnfoldLoad, 6846 bool UnfoldStore, SmallVectorImpl<MachineInstr *> &NewMIs) const { 6847 const X86MemoryFoldTableEntry *I = lookupUnfoldTable(MI.getOpcode()); 6848 if (I == nullptr) 6849 return false; 6850 unsigned Opc = I->DstOp; 6851 unsigned Index = I->Flags & TB_INDEX_MASK; 6852 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 6853 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 6854 bool FoldedBCast = I->Flags & TB_FOLDED_BCAST; 6855 if (UnfoldLoad && !FoldedLoad) 6856 return false; 6857 UnfoldLoad &= FoldedLoad; 6858 if (UnfoldStore && !FoldedStore) 6859 return false; 6860 UnfoldStore &= FoldedStore; 6861 6862 const MCInstrDesc &MCID = get(Opc); 6863 6864 const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); 6865 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 6866 // TODO: Check if 32-byte or greater accesses are slow too? 6867 if (!MI.hasOneMemOperand() && RC == &X86::VR128RegClass && 6868 Subtarget.isUnalignedMem16Slow()) 6869 // Without memoperands, loadRegFromAddr and storeRegToStackSlot will 6870 // conservatively assume the address is unaligned. That's bad for 6871 // performance. 6872 return false; 6873 SmallVector<MachineOperand, X86::AddrNumOperands> AddrOps; 6874 SmallVector<MachineOperand,2> BeforeOps; 6875 SmallVector<MachineOperand,2> AfterOps; 6876 SmallVector<MachineOperand,4> ImpOps; 6877 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 6878 MachineOperand &Op = MI.getOperand(i); 6879 if (i >= Index && i < Index + X86::AddrNumOperands) 6880 AddrOps.push_back(Op); 6881 else if (Op.isReg() && Op.isImplicit()) 6882 ImpOps.push_back(Op); 6883 else if (i < Index) 6884 BeforeOps.push_back(Op); 6885 else if (i > Index) 6886 AfterOps.push_back(Op); 6887 } 6888 6889 // Emit the load or broadcast instruction. 6890 if (UnfoldLoad) { 6891 auto MMOs = extractLoadMMOs(MI.memoperands(), MF); 6892 6893 unsigned Opc; 6894 if (FoldedBCast) { 6895 Opc = getBroadcastOpcode(I, RC, Subtarget); 6896 } else { 6897 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 6898 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 6899 Opc = getLoadRegOpcode(Reg, RC, isAligned, Subtarget); 6900 } 6901 6902 DebugLoc DL; 6903 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), Reg); 6904 for (unsigned i = 0, e = AddrOps.size(); i != e; ++i) 6905 MIB.add(AddrOps[i]); 6906 MIB.setMemRefs(MMOs); 6907 NewMIs.push_back(MIB); 6908 6909 if (UnfoldStore) { 6910 // Address operands cannot be marked isKill. 6911 for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) { 6912 MachineOperand &MO = NewMIs[0]->getOperand(i); 6913 if (MO.isReg()) 6914 MO.setIsKill(false); 6915 } 6916 } 6917 } 6918 6919 // Emit the data processing instruction. 6920 MachineInstr *DataMI = MF.CreateMachineInstr(MCID, MI.getDebugLoc(), true); 6921 MachineInstrBuilder MIB(MF, DataMI); 6922 6923 if (FoldedStore) 6924 MIB.addReg(Reg, RegState::Define); 6925 for (MachineOperand &BeforeOp : BeforeOps) 6926 MIB.add(BeforeOp); 6927 if (FoldedLoad) 6928 MIB.addReg(Reg); 6929 for (MachineOperand &AfterOp : AfterOps) 6930 MIB.add(AfterOp); 6931 for (MachineOperand &ImpOp : ImpOps) { 6932 MIB.addReg(ImpOp.getReg(), 6933 getDefRegState(ImpOp.isDef()) | 6934 RegState::Implicit | 6935 getKillRegState(ImpOp.isKill()) | 6936 getDeadRegState(ImpOp.isDead()) | 6937 getUndefRegState(ImpOp.isUndef())); 6938 } 6939 // Change CMP32ri r, 0 back to TEST32rr r, r, etc. 6940 switch (DataMI->getOpcode()) { 6941 default: break; 6942 case X86::CMP64ri32: 6943 case X86::CMP64ri8: 6944 case X86::CMP32ri: 6945 case X86::CMP32ri8: 6946 case X86::CMP16ri: 6947 case X86::CMP16ri8: 6948 case X86::CMP8ri: { 6949 MachineOperand &MO0 = DataMI->getOperand(0); 6950 MachineOperand &MO1 = DataMI->getOperand(1); 6951 if (MO1.isImm() && MO1.getImm() == 0) { 6952 unsigned NewOpc; 6953 switch (DataMI->getOpcode()) { 6954 default: llvm_unreachable("Unreachable!"); 6955 case X86::CMP64ri8: 6956 case X86::CMP64ri32: NewOpc = X86::TEST64rr; break; 6957 case X86::CMP32ri8: 6958 case X86::CMP32ri: NewOpc = X86::TEST32rr; break; 6959 case X86::CMP16ri8: 6960 case X86::CMP16ri: NewOpc = X86::TEST16rr; break; 6961 case X86::CMP8ri: NewOpc = X86::TEST8rr; break; 6962 } 6963 DataMI->setDesc(get(NewOpc)); 6964 MO1.ChangeToRegister(MO0.getReg(), false); 6965 } 6966 } 6967 } 6968 NewMIs.push_back(DataMI); 6969 6970 // Emit the store instruction. 6971 if (UnfoldStore) { 6972 const TargetRegisterClass *DstRC = getRegClass(MCID, 0, &RI, MF); 6973 auto MMOs = extractStoreMMOs(MI.memoperands(), MF); 6974 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*DstRC), 16); 6975 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 6976 unsigned Opc = getStoreRegOpcode(Reg, DstRC, isAligned, Subtarget); 6977 DebugLoc DL; 6978 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc)); 6979 for (unsigned i = 0, e = AddrOps.size(); i != e; ++i) 6980 MIB.add(AddrOps[i]); 6981 MIB.addReg(Reg, RegState::Kill); 6982 MIB.setMemRefs(MMOs); 6983 NewMIs.push_back(MIB); 6984 } 6985 6986 return true; 6987 } 6988 6989 bool 6990 X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, 6991 SmallVectorImpl<SDNode*> &NewNodes) const { 6992 if (!N->isMachineOpcode()) 6993 return false; 6994 6995 const X86MemoryFoldTableEntry *I = lookupUnfoldTable(N->getMachineOpcode()); 6996 if (I == nullptr) 6997 return false; 6998 unsigned Opc = I->DstOp; 6999 unsigned Index = I->Flags & TB_INDEX_MASK; 7000 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 7001 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 7002 bool FoldedBCast = I->Flags & TB_FOLDED_BCAST; 7003 const MCInstrDesc &MCID = get(Opc); 7004 MachineFunction &MF = DAG.getMachineFunction(); 7005 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 7006 const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); 7007 unsigned NumDefs = MCID.NumDefs; 7008 std::vector<SDValue> AddrOps; 7009 std::vector<SDValue> BeforeOps; 7010 std::vector<SDValue> AfterOps; 7011 SDLoc dl(N); 7012 unsigned NumOps = N->getNumOperands(); 7013 for (unsigned i = 0; i != NumOps-1; ++i) { 7014 SDValue Op = N->getOperand(i); 7015 if (i >= Index-NumDefs && i < Index-NumDefs + X86::AddrNumOperands) 7016 AddrOps.push_back(Op); 7017 else if (i < Index-NumDefs) 7018 BeforeOps.push_back(Op); 7019 else if (i > Index-NumDefs) 7020 AfterOps.push_back(Op); 7021 } 7022 SDValue Chain = N->getOperand(NumOps-1); 7023 AddrOps.push_back(Chain); 7024 7025 // Emit the load instruction. 7026 SDNode *Load = nullptr; 7027 if (FoldedLoad) { 7028 EVT VT = *TRI.legalclasstypes_begin(*RC); 7029 auto MMOs = extractLoadMMOs(cast<MachineSDNode>(N)->memoperands(), MF); 7030 if (MMOs.empty() && RC == &X86::VR128RegClass && 7031 Subtarget.isUnalignedMem16Slow()) 7032 // Do not introduce a slow unaligned load. 7033 return false; 7034 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte 7035 // memory access is slow above. 7036 7037 unsigned Opc; 7038 if (FoldedBCast) { 7039 Opc = getBroadcastOpcode(I, RC, Subtarget); 7040 } else { 7041 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 7042 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 7043 Opc = getLoadRegOpcode(0, RC, isAligned, Subtarget); 7044 } 7045 7046 Load = DAG.getMachineNode(Opc, dl, VT, MVT::Other, AddrOps); 7047 NewNodes.push_back(Load); 7048 7049 // Preserve memory reference information. 7050 DAG.setNodeMemRefs(cast<MachineSDNode>(Load), MMOs); 7051 } 7052 7053 // Emit the data processing instruction. 7054 std::vector<EVT> VTs; 7055 const TargetRegisterClass *DstRC = nullptr; 7056 if (MCID.getNumDefs() > 0) { 7057 DstRC = getRegClass(MCID, 0, &RI, MF); 7058 VTs.push_back(*TRI.legalclasstypes_begin(*DstRC)); 7059 } 7060 for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { 7061 EVT VT = N->getValueType(i); 7062 if (VT != MVT::Other && i >= (unsigned)MCID.getNumDefs()) 7063 VTs.push_back(VT); 7064 } 7065 if (Load) 7066 BeforeOps.push_back(SDValue(Load, 0)); 7067 llvm::append_range(BeforeOps, AfterOps); 7068 // Change CMP32ri r, 0 back to TEST32rr r, r, etc. 7069 switch (Opc) { 7070 default: break; 7071 case X86::CMP64ri32: 7072 case X86::CMP64ri8: 7073 case X86::CMP32ri: 7074 case X86::CMP32ri8: 7075 case X86::CMP16ri: 7076 case X86::CMP16ri8: 7077 case X86::CMP8ri: 7078 if (isNullConstant(BeforeOps[1])) { 7079 switch (Opc) { 7080 default: llvm_unreachable("Unreachable!"); 7081 case X86::CMP64ri8: 7082 case X86::CMP64ri32: Opc = X86::TEST64rr; break; 7083 case X86::CMP32ri8: 7084 case X86::CMP32ri: Opc = X86::TEST32rr; break; 7085 case X86::CMP16ri8: 7086 case X86::CMP16ri: Opc = X86::TEST16rr; break; 7087 case X86::CMP8ri: Opc = X86::TEST8rr; break; 7088 } 7089 BeforeOps[1] = BeforeOps[0]; 7090 } 7091 } 7092 SDNode *NewNode= DAG.getMachineNode(Opc, dl, VTs, BeforeOps); 7093 NewNodes.push_back(NewNode); 7094 7095 // Emit the store instruction. 7096 if (FoldedStore) { 7097 AddrOps.pop_back(); 7098 AddrOps.push_back(SDValue(NewNode, 0)); 7099 AddrOps.push_back(Chain); 7100 auto MMOs = extractStoreMMOs(cast<MachineSDNode>(N)->memoperands(), MF); 7101 if (MMOs.empty() && RC == &X86::VR128RegClass && 7102 Subtarget.isUnalignedMem16Slow()) 7103 // Do not introduce a slow unaligned store. 7104 return false; 7105 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte 7106 // memory access is slow above. 7107 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 7108 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 7109 SDNode *Store = 7110 DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget), 7111 dl, MVT::Other, AddrOps); 7112 NewNodes.push_back(Store); 7113 7114 // Preserve memory reference information. 7115 DAG.setNodeMemRefs(cast<MachineSDNode>(Store), MMOs); 7116 } 7117 7118 return true; 7119 } 7120 7121 unsigned X86InstrInfo::getOpcodeAfterMemoryUnfold(unsigned Opc, 7122 bool UnfoldLoad, bool UnfoldStore, 7123 unsigned *LoadRegIndex) const { 7124 const X86MemoryFoldTableEntry *I = lookupUnfoldTable(Opc); 7125 if (I == nullptr) 7126 return 0; 7127 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 7128 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 7129 if (UnfoldLoad && !FoldedLoad) 7130 return 0; 7131 if (UnfoldStore && !FoldedStore) 7132 return 0; 7133 if (LoadRegIndex) 7134 *LoadRegIndex = I->Flags & TB_INDEX_MASK; 7135 return I->DstOp; 7136 } 7137 7138 bool 7139 X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, 7140 int64_t &Offset1, int64_t &Offset2) const { 7141 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode()) 7142 return false; 7143 unsigned Opc1 = Load1->getMachineOpcode(); 7144 unsigned Opc2 = Load2->getMachineOpcode(); 7145 switch (Opc1) { 7146 default: return false; 7147 case X86::MOV8rm: 7148 case X86::MOV16rm: 7149 case X86::MOV32rm: 7150 case X86::MOV64rm: 7151 case X86::LD_Fp32m: 7152 case X86::LD_Fp64m: 7153 case X86::LD_Fp80m: 7154 case X86::MOVSSrm: 7155 case X86::MOVSSrm_alt: 7156 case X86::MOVSDrm: 7157 case X86::MOVSDrm_alt: 7158 case X86::MMX_MOVD64rm: 7159 case X86::MMX_MOVQ64rm: 7160 case X86::MOVAPSrm: 7161 case X86::MOVUPSrm: 7162 case X86::MOVAPDrm: 7163 case X86::MOVUPDrm: 7164 case X86::MOVDQArm: 7165 case X86::MOVDQUrm: 7166 // AVX load instructions 7167 case X86::VMOVSSrm: 7168 case X86::VMOVSSrm_alt: 7169 case X86::VMOVSDrm: 7170 case X86::VMOVSDrm_alt: 7171 case X86::VMOVAPSrm: 7172 case X86::VMOVUPSrm: 7173 case X86::VMOVAPDrm: 7174 case X86::VMOVUPDrm: 7175 case X86::VMOVDQArm: 7176 case X86::VMOVDQUrm: 7177 case X86::VMOVAPSYrm: 7178 case X86::VMOVUPSYrm: 7179 case X86::VMOVAPDYrm: 7180 case X86::VMOVUPDYrm: 7181 case X86::VMOVDQAYrm: 7182 case X86::VMOVDQUYrm: 7183 // AVX512 load instructions 7184 case X86::VMOVSSZrm: 7185 case X86::VMOVSSZrm_alt: 7186 case X86::VMOVSDZrm: 7187 case X86::VMOVSDZrm_alt: 7188 case X86::VMOVAPSZ128rm: 7189 case X86::VMOVUPSZ128rm: 7190 case X86::VMOVAPSZ128rm_NOVLX: 7191 case X86::VMOVUPSZ128rm_NOVLX: 7192 case X86::VMOVAPDZ128rm: 7193 case X86::VMOVUPDZ128rm: 7194 case X86::VMOVDQU8Z128rm: 7195 case X86::VMOVDQU16Z128rm: 7196 case X86::VMOVDQA32Z128rm: 7197 case X86::VMOVDQU32Z128rm: 7198 case X86::VMOVDQA64Z128rm: 7199 case X86::VMOVDQU64Z128rm: 7200 case X86::VMOVAPSZ256rm: 7201 case X86::VMOVUPSZ256rm: 7202 case X86::VMOVAPSZ256rm_NOVLX: 7203 case X86::VMOVUPSZ256rm_NOVLX: 7204 case X86::VMOVAPDZ256rm: 7205 case X86::VMOVUPDZ256rm: 7206 case X86::VMOVDQU8Z256rm: 7207 case X86::VMOVDQU16Z256rm: 7208 case X86::VMOVDQA32Z256rm: 7209 case X86::VMOVDQU32Z256rm: 7210 case X86::VMOVDQA64Z256rm: 7211 case X86::VMOVDQU64Z256rm: 7212 case X86::VMOVAPSZrm: 7213 case X86::VMOVUPSZrm: 7214 case X86::VMOVAPDZrm: 7215 case X86::VMOVUPDZrm: 7216 case X86::VMOVDQU8Zrm: 7217 case X86::VMOVDQU16Zrm: 7218 case X86::VMOVDQA32Zrm: 7219 case X86::VMOVDQU32Zrm: 7220 case X86::VMOVDQA64Zrm: 7221 case X86::VMOVDQU64Zrm: 7222 case X86::KMOVBkm: 7223 case X86::KMOVWkm: 7224 case X86::KMOVDkm: 7225 case X86::KMOVQkm: 7226 break; 7227 } 7228 switch (Opc2) { 7229 default: return false; 7230 case X86::MOV8rm: 7231 case X86::MOV16rm: 7232 case X86::MOV32rm: 7233 case X86::MOV64rm: 7234 case X86::LD_Fp32m: 7235 case X86::LD_Fp64m: 7236 case X86::LD_Fp80m: 7237 case X86::MOVSSrm: 7238 case X86::MOVSSrm_alt: 7239 case X86::MOVSDrm: 7240 case X86::MOVSDrm_alt: 7241 case X86::MMX_MOVD64rm: 7242 case X86::MMX_MOVQ64rm: 7243 case X86::MOVAPSrm: 7244 case X86::MOVUPSrm: 7245 case X86::MOVAPDrm: 7246 case X86::MOVUPDrm: 7247 case X86::MOVDQArm: 7248 case X86::MOVDQUrm: 7249 // AVX load instructions 7250 case X86::VMOVSSrm: 7251 case X86::VMOVSSrm_alt: 7252 case X86::VMOVSDrm: 7253 case X86::VMOVSDrm_alt: 7254 case X86::VMOVAPSrm: 7255 case X86::VMOVUPSrm: 7256 case X86::VMOVAPDrm: 7257 case X86::VMOVUPDrm: 7258 case X86::VMOVDQArm: 7259 case X86::VMOVDQUrm: 7260 case X86::VMOVAPSYrm: 7261 case X86::VMOVUPSYrm: 7262 case X86::VMOVAPDYrm: 7263 case X86::VMOVUPDYrm: 7264 case X86::VMOVDQAYrm: 7265 case X86::VMOVDQUYrm: 7266 // AVX512 load instructions 7267 case X86::VMOVSSZrm: 7268 case X86::VMOVSSZrm_alt: 7269 case X86::VMOVSDZrm: 7270 case X86::VMOVSDZrm_alt: 7271 case X86::VMOVAPSZ128rm: 7272 case X86::VMOVUPSZ128rm: 7273 case X86::VMOVAPSZ128rm_NOVLX: 7274 case X86::VMOVUPSZ128rm_NOVLX: 7275 case X86::VMOVAPDZ128rm: 7276 case X86::VMOVUPDZ128rm: 7277 case X86::VMOVDQU8Z128rm: 7278 case X86::VMOVDQU16Z128rm: 7279 case X86::VMOVDQA32Z128rm: 7280 case X86::VMOVDQU32Z128rm: 7281 case X86::VMOVDQA64Z128rm: 7282 case X86::VMOVDQU64Z128rm: 7283 case X86::VMOVAPSZ256rm: 7284 case X86::VMOVUPSZ256rm: 7285 case X86::VMOVAPSZ256rm_NOVLX: 7286 case X86::VMOVUPSZ256rm_NOVLX: 7287 case X86::VMOVAPDZ256rm: 7288 case X86::VMOVUPDZ256rm: 7289 case X86::VMOVDQU8Z256rm: 7290 case X86::VMOVDQU16Z256rm: 7291 case X86::VMOVDQA32Z256rm: 7292 case X86::VMOVDQU32Z256rm: 7293 case X86::VMOVDQA64Z256rm: 7294 case X86::VMOVDQU64Z256rm: 7295 case X86::VMOVAPSZrm: 7296 case X86::VMOVUPSZrm: 7297 case X86::VMOVAPDZrm: 7298 case X86::VMOVUPDZrm: 7299 case X86::VMOVDQU8Zrm: 7300 case X86::VMOVDQU16Zrm: 7301 case X86::VMOVDQA32Zrm: 7302 case X86::VMOVDQU32Zrm: 7303 case X86::VMOVDQA64Zrm: 7304 case X86::VMOVDQU64Zrm: 7305 case X86::KMOVBkm: 7306 case X86::KMOVWkm: 7307 case X86::KMOVDkm: 7308 case X86::KMOVQkm: 7309 break; 7310 } 7311 7312 // Lambda to check if both the loads have the same value for an operand index. 7313 auto HasSameOp = [&](int I) { 7314 return Load1->getOperand(I) == Load2->getOperand(I); 7315 }; 7316 7317 // All operands except the displacement should match. 7318 if (!HasSameOp(X86::AddrBaseReg) || !HasSameOp(X86::AddrScaleAmt) || 7319 !HasSameOp(X86::AddrIndexReg) || !HasSameOp(X86::AddrSegmentReg)) 7320 return false; 7321 7322 // Chain Operand must be the same. 7323 if (!HasSameOp(5)) 7324 return false; 7325 7326 // Now let's examine if the displacements are constants. 7327 auto Disp1 = dyn_cast<ConstantSDNode>(Load1->getOperand(X86::AddrDisp)); 7328 auto Disp2 = dyn_cast<ConstantSDNode>(Load2->getOperand(X86::AddrDisp)); 7329 if (!Disp1 || !Disp2) 7330 return false; 7331 7332 Offset1 = Disp1->getSExtValue(); 7333 Offset2 = Disp2->getSExtValue(); 7334 return true; 7335 } 7336 7337 bool X86InstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, 7338 int64_t Offset1, int64_t Offset2, 7339 unsigned NumLoads) const { 7340 assert(Offset2 > Offset1); 7341 if ((Offset2 - Offset1) / 8 > 64) 7342 return false; 7343 7344 unsigned Opc1 = Load1->getMachineOpcode(); 7345 unsigned Opc2 = Load2->getMachineOpcode(); 7346 if (Opc1 != Opc2) 7347 return false; // FIXME: overly conservative? 7348 7349 switch (Opc1) { 7350 default: break; 7351 case X86::LD_Fp32m: 7352 case X86::LD_Fp64m: 7353 case X86::LD_Fp80m: 7354 case X86::MMX_MOVD64rm: 7355 case X86::MMX_MOVQ64rm: 7356 return false; 7357 } 7358 7359 EVT VT = Load1->getValueType(0); 7360 switch (VT.getSimpleVT().SimpleTy) { 7361 default: 7362 // XMM registers. In 64-bit mode we can be a bit more aggressive since we 7363 // have 16 of them to play with. 7364 if (Subtarget.is64Bit()) { 7365 if (NumLoads >= 3) 7366 return false; 7367 } else if (NumLoads) { 7368 return false; 7369 } 7370 break; 7371 case MVT::i8: 7372 case MVT::i16: 7373 case MVT::i32: 7374 case MVT::i64: 7375 case MVT::f32: 7376 case MVT::f64: 7377 if (NumLoads) 7378 return false; 7379 break; 7380 } 7381 7382 return true; 7383 } 7384 7385 bool X86InstrInfo::isSchedulingBoundary(const MachineInstr &MI, 7386 const MachineBasicBlock *MBB, 7387 const MachineFunction &MF) const { 7388 7389 // ENDBR instructions should not be scheduled around. 7390 unsigned Opcode = MI.getOpcode(); 7391 if (Opcode == X86::ENDBR64 || Opcode == X86::ENDBR32 || 7392 Opcode == X86::PLDTILECFGV) 7393 return true; 7394 7395 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF); 7396 } 7397 7398 bool X86InstrInfo:: 7399 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 7400 assert(Cond.size() == 1 && "Invalid X86 branch condition!"); 7401 X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm()); 7402 Cond[0].setImm(GetOppositeBranchCondition(CC)); 7403 return false; 7404 } 7405 7406 bool X86InstrInfo:: 7407 isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const { 7408 // FIXME: Return false for x87 stack register classes for now. We can't 7409 // allow any loads of these registers before FpGet_ST0_80. 7410 return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass || 7411 RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass || 7412 RC == &X86::RFP80RegClass); 7413 } 7414 7415 /// Return a virtual register initialized with the 7416 /// the global base register value. Output instructions required to 7417 /// initialize the register in the function entry block, if necessary. 7418 /// 7419 /// TODO: Eliminate this and move the code to X86MachineFunctionInfo. 7420 /// 7421 unsigned X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const { 7422 assert((!Subtarget.is64Bit() || 7423 MF->getTarget().getCodeModel() == CodeModel::Medium || 7424 MF->getTarget().getCodeModel() == CodeModel::Large) && 7425 "X86-64 PIC uses RIP relative addressing"); 7426 7427 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); 7428 Register GlobalBaseReg = X86FI->getGlobalBaseReg(); 7429 if (GlobalBaseReg != 0) 7430 return GlobalBaseReg; 7431 7432 // Create the register. The code to initialize it is inserted 7433 // later, by the CGBR pass (below). 7434 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 7435 GlobalBaseReg = RegInfo.createVirtualRegister( 7436 Subtarget.is64Bit() ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass); 7437 X86FI->setGlobalBaseReg(GlobalBaseReg); 7438 return GlobalBaseReg; 7439 } 7440 7441 // These are the replaceable SSE instructions. Some of these have Int variants 7442 // that we don't include here. We don't want to replace instructions selected 7443 // by intrinsics. 7444 static const uint16_t ReplaceableInstrs[][3] = { 7445 //PackedSingle PackedDouble PackedInt 7446 { X86::MOVAPSmr, X86::MOVAPDmr, X86::MOVDQAmr }, 7447 { X86::MOVAPSrm, X86::MOVAPDrm, X86::MOVDQArm }, 7448 { X86::MOVAPSrr, X86::MOVAPDrr, X86::MOVDQArr }, 7449 { X86::MOVUPSmr, X86::MOVUPDmr, X86::MOVDQUmr }, 7450 { X86::MOVUPSrm, X86::MOVUPDrm, X86::MOVDQUrm }, 7451 { X86::MOVLPSmr, X86::MOVLPDmr, X86::MOVPQI2QImr }, 7452 { X86::MOVSDmr, X86::MOVSDmr, X86::MOVPQI2QImr }, 7453 { X86::MOVSSmr, X86::MOVSSmr, X86::MOVPDI2DImr }, 7454 { X86::MOVSDrm, X86::MOVSDrm, X86::MOVQI2PQIrm }, 7455 { X86::MOVSDrm_alt,X86::MOVSDrm_alt,X86::MOVQI2PQIrm }, 7456 { X86::MOVSSrm, X86::MOVSSrm, X86::MOVDI2PDIrm }, 7457 { X86::MOVSSrm_alt,X86::MOVSSrm_alt,X86::MOVDI2PDIrm }, 7458 { X86::MOVNTPSmr, X86::MOVNTPDmr, X86::MOVNTDQmr }, 7459 { X86::ANDNPSrm, X86::ANDNPDrm, X86::PANDNrm }, 7460 { X86::ANDNPSrr, X86::ANDNPDrr, X86::PANDNrr }, 7461 { X86::ANDPSrm, X86::ANDPDrm, X86::PANDrm }, 7462 { X86::ANDPSrr, X86::ANDPDrr, X86::PANDrr }, 7463 { X86::ORPSrm, X86::ORPDrm, X86::PORrm }, 7464 { X86::ORPSrr, X86::ORPDrr, X86::PORrr }, 7465 { X86::XORPSrm, X86::XORPDrm, X86::PXORrm }, 7466 { X86::XORPSrr, X86::XORPDrr, X86::PXORrr }, 7467 { X86::UNPCKLPDrm, X86::UNPCKLPDrm, X86::PUNPCKLQDQrm }, 7468 { X86::MOVLHPSrr, X86::UNPCKLPDrr, X86::PUNPCKLQDQrr }, 7469 { X86::UNPCKHPDrm, X86::UNPCKHPDrm, X86::PUNPCKHQDQrm }, 7470 { X86::UNPCKHPDrr, X86::UNPCKHPDrr, X86::PUNPCKHQDQrr }, 7471 { X86::UNPCKLPSrm, X86::UNPCKLPSrm, X86::PUNPCKLDQrm }, 7472 { X86::UNPCKLPSrr, X86::UNPCKLPSrr, X86::PUNPCKLDQrr }, 7473 { X86::UNPCKHPSrm, X86::UNPCKHPSrm, X86::PUNPCKHDQrm }, 7474 { X86::UNPCKHPSrr, X86::UNPCKHPSrr, X86::PUNPCKHDQrr }, 7475 { X86::EXTRACTPSmr, X86::EXTRACTPSmr, X86::PEXTRDmr }, 7476 { X86::EXTRACTPSrr, X86::EXTRACTPSrr, X86::PEXTRDrr }, 7477 // AVX 128-bit support 7478 { X86::VMOVAPSmr, X86::VMOVAPDmr, X86::VMOVDQAmr }, 7479 { X86::VMOVAPSrm, X86::VMOVAPDrm, X86::VMOVDQArm }, 7480 { X86::VMOVAPSrr, X86::VMOVAPDrr, X86::VMOVDQArr }, 7481 { X86::VMOVUPSmr, X86::VMOVUPDmr, X86::VMOVDQUmr }, 7482 { X86::VMOVUPSrm, X86::VMOVUPDrm, X86::VMOVDQUrm }, 7483 { X86::VMOVLPSmr, X86::VMOVLPDmr, X86::VMOVPQI2QImr }, 7484 { X86::VMOVSDmr, X86::VMOVSDmr, X86::VMOVPQI2QImr }, 7485 { X86::VMOVSSmr, X86::VMOVSSmr, X86::VMOVPDI2DImr }, 7486 { X86::VMOVSDrm, X86::VMOVSDrm, X86::VMOVQI2PQIrm }, 7487 { X86::VMOVSDrm_alt,X86::VMOVSDrm_alt,X86::VMOVQI2PQIrm }, 7488 { X86::VMOVSSrm, X86::VMOVSSrm, X86::VMOVDI2PDIrm }, 7489 { X86::VMOVSSrm_alt,X86::VMOVSSrm_alt,X86::VMOVDI2PDIrm }, 7490 { X86::VMOVNTPSmr, X86::VMOVNTPDmr, X86::VMOVNTDQmr }, 7491 { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNrm }, 7492 { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNrr }, 7493 { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDrm }, 7494 { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDrr }, 7495 { X86::VORPSrm, X86::VORPDrm, X86::VPORrm }, 7496 { X86::VORPSrr, X86::VORPDrr, X86::VPORrr }, 7497 { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORrm }, 7498 { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORrr }, 7499 { X86::VUNPCKLPDrm, X86::VUNPCKLPDrm, X86::VPUNPCKLQDQrm }, 7500 { X86::VMOVLHPSrr, X86::VUNPCKLPDrr, X86::VPUNPCKLQDQrr }, 7501 { X86::VUNPCKHPDrm, X86::VUNPCKHPDrm, X86::VPUNPCKHQDQrm }, 7502 { X86::VUNPCKHPDrr, X86::VUNPCKHPDrr, X86::VPUNPCKHQDQrr }, 7503 { X86::VUNPCKLPSrm, X86::VUNPCKLPSrm, X86::VPUNPCKLDQrm }, 7504 { X86::VUNPCKLPSrr, X86::VUNPCKLPSrr, X86::VPUNPCKLDQrr }, 7505 { X86::VUNPCKHPSrm, X86::VUNPCKHPSrm, X86::VPUNPCKHDQrm }, 7506 { X86::VUNPCKHPSrr, X86::VUNPCKHPSrr, X86::VPUNPCKHDQrr }, 7507 { X86::VEXTRACTPSmr, X86::VEXTRACTPSmr, X86::VPEXTRDmr }, 7508 { X86::VEXTRACTPSrr, X86::VEXTRACTPSrr, X86::VPEXTRDrr }, 7509 // AVX 256-bit support 7510 { X86::VMOVAPSYmr, X86::VMOVAPDYmr, X86::VMOVDQAYmr }, 7511 { X86::VMOVAPSYrm, X86::VMOVAPDYrm, X86::VMOVDQAYrm }, 7512 { X86::VMOVAPSYrr, X86::VMOVAPDYrr, X86::VMOVDQAYrr }, 7513 { X86::VMOVUPSYmr, X86::VMOVUPDYmr, X86::VMOVDQUYmr }, 7514 { X86::VMOVUPSYrm, X86::VMOVUPDYrm, X86::VMOVDQUYrm }, 7515 { X86::VMOVNTPSYmr, X86::VMOVNTPDYmr, X86::VMOVNTDQYmr }, 7516 { X86::VPERMPSYrm, X86::VPERMPSYrm, X86::VPERMDYrm }, 7517 { X86::VPERMPSYrr, X86::VPERMPSYrr, X86::VPERMDYrr }, 7518 { X86::VPERMPDYmi, X86::VPERMPDYmi, X86::VPERMQYmi }, 7519 { X86::VPERMPDYri, X86::VPERMPDYri, X86::VPERMQYri }, 7520 // AVX512 support 7521 { X86::VMOVLPSZ128mr, X86::VMOVLPDZ128mr, X86::VMOVPQI2QIZmr }, 7522 { X86::VMOVNTPSZ128mr, X86::VMOVNTPDZ128mr, X86::VMOVNTDQZ128mr }, 7523 { X86::VMOVNTPSZ256mr, X86::VMOVNTPDZ256mr, X86::VMOVNTDQZ256mr }, 7524 { X86::VMOVNTPSZmr, X86::VMOVNTPDZmr, X86::VMOVNTDQZmr }, 7525 { X86::VMOVSDZmr, X86::VMOVSDZmr, X86::VMOVPQI2QIZmr }, 7526 { X86::VMOVSSZmr, X86::VMOVSSZmr, X86::VMOVPDI2DIZmr }, 7527 { X86::VMOVSDZrm, X86::VMOVSDZrm, X86::VMOVQI2PQIZrm }, 7528 { X86::VMOVSDZrm_alt, X86::VMOVSDZrm_alt, X86::VMOVQI2PQIZrm }, 7529 { X86::VMOVSSZrm, X86::VMOVSSZrm, X86::VMOVDI2PDIZrm }, 7530 { X86::VMOVSSZrm_alt, X86::VMOVSSZrm_alt, X86::VMOVDI2PDIZrm }, 7531 { X86::VBROADCASTSSZ128rr,X86::VBROADCASTSSZ128rr,X86::VPBROADCASTDZ128rr }, 7532 { X86::VBROADCASTSSZ128rm,X86::VBROADCASTSSZ128rm,X86::VPBROADCASTDZ128rm }, 7533 { X86::VBROADCASTSSZ256rr,X86::VBROADCASTSSZ256rr,X86::VPBROADCASTDZ256rr }, 7534 { X86::VBROADCASTSSZ256rm,X86::VBROADCASTSSZ256rm,X86::VPBROADCASTDZ256rm }, 7535 { X86::VBROADCASTSSZrr, X86::VBROADCASTSSZrr, X86::VPBROADCASTDZrr }, 7536 { X86::VBROADCASTSSZrm, X86::VBROADCASTSSZrm, X86::VPBROADCASTDZrm }, 7537 { X86::VMOVDDUPZ128rr, X86::VMOVDDUPZ128rr, X86::VPBROADCASTQZ128rr }, 7538 { X86::VMOVDDUPZ128rm, X86::VMOVDDUPZ128rm, X86::VPBROADCASTQZ128rm }, 7539 { X86::VBROADCASTSDZ256rr,X86::VBROADCASTSDZ256rr,X86::VPBROADCASTQZ256rr }, 7540 { X86::VBROADCASTSDZ256rm,X86::VBROADCASTSDZ256rm,X86::VPBROADCASTQZ256rm }, 7541 { X86::VBROADCASTSDZrr, X86::VBROADCASTSDZrr, X86::VPBROADCASTQZrr }, 7542 { X86::VBROADCASTSDZrm, X86::VBROADCASTSDZrm, X86::VPBROADCASTQZrm }, 7543 { X86::VINSERTF32x4Zrr, X86::VINSERTF32x4Zrr, X86::VINSERTI32x4Zrr }, 7544 { X86::VINSERTF32x4Zrm, X86::VINSERTF32x4Zrm, X86::VINSERTI32x4Zrm }, 7545 { X86::VINSERTF32x8Zrr, X86::VINSERTF32x8Zrr, X86::VINSERTI32x8Zrr }, 7546 { X86::VINSERTF32x8Zrm, X86::VINSERTF32x8Zrm, X86::VINSERTI32x8Zrm }, 7547 { X86::VINSERTF64x2Zrr, X86::VINSERTF64x2Zrr, X86::VINSERTI64x2Zrr }, 7548 { X86::VINSERTF64x2Zrm, X86::VINSERTF64x2Zrm, X86::VINSERTI64x2Zrm }, 7549 { X86::VINSERTF64x4Zrr, X86::VINSERTF64x4Zrr, X86::VINSERTI64x4Zrr }, 7550 { X86::VINSERTF64x4Zrm, X86::VINSERTF64x4Zrm, X86::VINSERTI64x4Zrm }, 7551 { X86::VINSERTF32x4Z256rr,X86::VINSERTF32x4Z256rr,X86::VINSERTI32x4Z256rr }, 7552 { X86::VINSERTF32x4Z256rm,X86::VINSERTF32x4Z256rm,X86::VINSERTI32x4Z256rm }, 7553 { X86::VINSERTF64x2Z256rr,X86::VINSERTF64x2Z256rr,X86::VINSERTI64x2Z256rr }, 7554 { X86::VINSERTF64x2Z256rm,X86::VINSERTF64x2Z256rm,X86::VINSERTI64x2Z256rm }, 7555 { X86::VEXTRACTF32x4Zrr, X86::VEXTRACTF32x4Zrr, X86::VEXTRACTI32x4Zrr }, 7556 { X86::VEXTRACTF32x4Zmr, X86::VEXTRACTF32x4Zmr, X86::VEXTRACTI32x4Zmr }, 7557 { X86::VEXTRACTF32x8Zrr, X86::VEXTRACTF32x8Zrr, X86::VEXTRACTI32x8Zrr }, 7558 { X86::VEXTRACTF32x8Zmr, X86::VEXTRACTF32x8Zmr, X86::VEXTRACTI32x8Zmr }, 7559 { X86::VEXTRACTF64x2Zrr, X86::VEXTRACTF64x2Zrr, X86::VEXTRACTI64x2Zrr }, 7560 { X86::VEXTRACTF64x2Zmr, X86::VEXTRACTF64x2Zmr, X86::VEXTRACTI64x2Zmr }, 7561 { X86::VEXTRACTF64x4Zrr, X86::VEXTRACTF64x4Zrr, X86::VEXTRACTI64x4Zrr }, 7562 { X86::VEXTRACTF64x4Zmr, X86::VEXTRACTF64x4Zmr, X86::VEXTRACTI64x4Zmr }, 7563 { X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTI32x4Z256rr }, 7564 { X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTI32x4Z256mr }, 7565 { X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTI64x2Z256rr }, 7566 { X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTI64x2Z256mr }, 7567 { X86::VPERMILPSmi, X86::VPERMILPSmi, X86::VPSHUFDmi }, 7568 { X86::VPERMILPSri, X86::VPERMILPSri, X86::VPSHUFDri }, 7569 { X86::VPERMILPSZ128mi, X86::VPERMILPSZ128mi, X86::VPSHUFDZ128mi }, 7570 { X86::VPERMILPSZ128ri, X86::VPERMILPSZ128ri, X86::VPSHUFDZ128ri }, 7571 { X86::VPERMILPSZ256mi, X86::VPERMILPSZ256mi, X86::VPSHUFDZ256mi }, 7572 { X86::VPERMILPSZ256ri, X86::VPERMILPSZ256ri, X86::VPSHUFDZ256ri }, 7573 { X86::VPERMILPSZmi, X86::VPERMILPSZmi, X86::VPSHUFDZmi }, 7574 { X86::VPERMILPSZri, X86::VPERMILPSZri, X86::VPSHUFDZri }, 7575 { X86::VPERMPSZ256rm, X86::VPERMPSZ256rm, X86::VPERMDZ256rm }, 7576 { X86::VPERMPSZ256rr, X86::VPERMPSZ256rr, X86::VPERMDZ256rr }, 7577 { X86::VPERMPDZ256mi, X86::VPERMPDZ256mi, X86::VPERMQZ256mi }, 7578 { X86::VPERMPDZ256ri, X86::VPERMPDZ256ri, X86::VPERMQZ256ri }, 7579 { X86::VPERMPDZ256rm, X86::VPERMPDZ256rm, X86::VPERMQZ256rm }, 7580 { X86::VPERMPDZ256rr, X86::VPERMPDZ256rr, X86::VPERMQZ256rr }, 7581 { X86::VPERMPSZrm, X86::VPERMPSZrm, X86::VPERMDZrm }, 7582 { X86::VPERMPSZrr, X86::VPERMPSZrr, X86::VPERMDZrr }, 7583 { X86::VPERMPDZmi, X86::VPERMPDZmi, X86::VPERMQZmi }, 7584 { X86::VPERMPDZri, X86::VPERMPDZri, X86::VPERMQZri }, 7585 { X86::VPERMPDZrm, X86::VPERMPDZrm, X86::VPERMQZrm }, 7586 { X86::VPERMPDZrr, X86::VPERMPDZrr, X86::VPERMQZrr }, 7587 { X86::VUNPCKLPDZ256rm, X86::VUNPCKLPDZ256rm, X86::VPUNPCKLQDQZ256rm }, 7588 { X86::VUNPCKLPDZ256rr, X86::VUNPCKLPDZ256rr, X86::VPUNPCKLQDQZ256rr }, 7589 { X86::VUNPCKHPDZ256rm, X86::VUNPCKHPDZ256rm, X86::VPUNPCKHQDQZ256rm }, 7590 { X86::VUNPCKHPDZ256rr, X86::VUNPCKHPDZ256rr, X86::VPUNPCKHQDQZ256rr }, 7591 { X86::VUNPCKLPSZ256rm, X86::VUNPCKLPSZ256rm, X86::VPUNPCKLDQZ256rm }, 7592 { X86::VUNPCKLPSZ256rr, X86::VUNPCKLPSZ256rr, X86::VPUNPCKLDQZ256rr }, 7593 { X86::VUNPCKHPSZ256rm, X86::VUNPCKHPSZ256rm, X86::VPUNPCKHDQZ256rm }, 7594 { X86::VUNPCKHPSZ256rr, X86::VUNPCKHPSZ256rr, X86::VPUNPCKHDQZ256rr }, 7595 { X86::VUNPCKLPDZ128rm, X86::VUNPCKLPDZ128rm, X86::VPUNPCKLQDQZ128rm }, 7596 { X86::VMOVLHPSZrr, X86::VUNPCKLPDZ128rr, X86::VPUNPCKLQDQZ128rr }, 7597 { X86::VUNPCKHPDZ128rm, X86::VUNPCKHPDZ128rm, X86::VPUNPCKHQDQZ128rm }, 7598 { X86::VUNPCKHPDZ128rr, X86::VUNPCKHPDZ128rr, X86::VPUNPCKHQDQZ128rr }, 7599 { X86::VUNPCKLPSZ128rm, X86::VUNPCKLPSZ128rm, X86::VPUNPCKLDQZ128rm }, 7600 { X86::VUNPCKLPSZ128rr, X86::VUNPCKLPSZ128rr, X86::VPUNPCKLDQZ128rr }, 7601 { X86::VUNPCKHPSZ128rm, X86::VUNPCKHPSZ128rm, X86::VPUNPCKHDQZ128rm }, 7602 { X86::VUNPCKHPSZ128rr, X86::VUNPCKHPSZ128rr, X86::VPUNPCKHDQZ128rr }, 7603 { X86::VUNPCKLPDZrm, X86::VUNPCKLPDZrm, X86::VPUNPCKLQDQZrm }, 7604 { X86::VUNPCKLPDZrr, X86::VUNPCKLPDZrr, X86::VPUNPCKLQDQZrr }, 7605 { X86::VUNPCKHPDZrm, X86::VUNPCKHPDZrm, X86::VPUNPCKHQDQZrm }, 7606 { X86::VUNPCKHPDZrr, X86::VUNPCKHPDZrr, X86::VPUNPCKHQDQZrr }, 7607 { X86::VUNPCKLPSZrm, X86::VUNPCKLPSZrm, X86::VPUNPCKLDQZrm }, 7608 { X86::VUNPCKLPSZrr, X86::VUNPCKLPSZrr, X86::VPUNPCKLDQZrr }, 7609 { X86::VUNPCKHPSZrm, X86::VUNPCKHPSZrm, X86::VPUNPCKHDQZrm }, 7610 { X86::VUNPCKHPSZrr, X86::VUNPCKHPSZrr, X86::VPUNPCKHDQZrr }, 7611 { X86::VEXTRACTPSZmr, X86::VEXTRACTPSZmr, X86::VPEXTRDZmr }, 7612 { X86::VEXTRACTPSZrr, X86::VEXTRACTPSZrr, X86::VPEXTRDZrr }, 7613 }; 7614 7615 static const uint16_t ReplaceableInstrsAVX2[][3] = { 7616 //PackedSingle PackedDouble PackedInt 7617 { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNYrm }, 7618 { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNYrr }, 7619 { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDYrm }, 7620 { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDYrr }, 7621 { X86::VORPSYrm, X86::VORPDYrm, X86::VPORYrm }, 7622 { X86::VORPSYrr, X86::VORPDYrr, X86::VPORYrr }, 7623 { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORYrm }, 7624 { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORYrr }, 7625 { X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm }, 7626 { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }, 7627 { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm}, 7628 { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr}, 7629 { X86::VMOVDDUPrm, X86::VMOVDDUPrm, X86::VPBROADCASTQrm}, 7630 { X86::VMOVDDUPrr, X86::VMOVDDUPrr, X86::VPBROADCASTQrr}, 7631 { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr}, 7632 { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm}, 7633 { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr}, 7634 { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm}, 7635 { X86::VBROADCASTF128, X86::VBROADCASTF128, X86::VBROADCASTI128 }, 7636 { X86::VBLENDPSYrri, X86::VBLENDPSYrri, X86::VPBLENDDYrri }, 7637 { X86::VBLENDPSYrmi, X86::VBLENDPSYrmi, X86::VPBLENDDYrmi }, 7638 { X86::VPERMILPSYmi, X86::VPERMILPSYmi, X86::VPSHUFDYmi }, 7639 { X86::VPERMILPSYri, X86::VPERMILPSYri, X86::VPSHUFDYri }, 7640 { X86::VUNPCKLPDYrm, X86::VUNPCKLPDYrm, X86::VPUNPCKLQDQYrm }, 7641 { X86::VUNPCKLPDYrr, X86::VUNPCKLPDYrr, X86::VPUNPCKLQDQYrr }, 7642 { X86::VUNPCKHPDYrm, X86::VUNPCKHPDYrm, X86::VPUNPCKHQDQYrm }, 7643 { X86::VUNPCKHPDYrr, X86::VUNPCKHPDYrr, X86::VPUNPCKHQDQYrr }, 7644 { X86::VUNPCKLPSYrm, X86::VUNPCKLPSYrm, X86::VPUNPCKLDQYrm }, 7645 { X86::VUNPCKLPSYrr, X86::VUNPCKLPSYrr, X86::VPUNPCKLDQYrr }, 7646 { X86::VUNPCKHPSYrm, X86::VUNPCKHPSYrm, X86::VPUNPCKHDQYrm }, 7647 { X86::VUNPCKHPSYrr, X86::VUNPCKHPSYrr, X86::VPUNPCKHDQYrr }, 7648 }; 7649 7650 static const uint16_t ReplaceableInstrsFP[][3] = { 7651 //PackedSingle PackedDouble 7652 { X86::MOVLPSrm, X86::MOVLPDrm, X86::INSTRUCTION_LIST_END }, 7653 { X86::MOVHPSrm, X86::MOVHPDrm, X86::INSTRUCTION_LIST_END }, 7654 { X86::MOVHPSmr, X86::MOVHPDmr, X86::INSTRUCTION_LIST_END }, 7655 { X86::VMOVLPSrm, X86::VMOVLPDrm, X86::INSTRUCTION_LIST_END }, 7656 { X86::VMOVHPSrm, X86::VMOVHPDrm, X86::INSTRUCTION_LIST_END }, 7657 { X86::VMOVHPSmr, X86::VMOVHPDmr, X86::INSTRUCTION_LIST_END }, 7658 { X86::VMOVLPSZ128rm, X86::VMOVLPDZ128rm, X86::INSTRUCTION_LIST_END }, 7659 { X86::VMOVHPSZ128rm, X86::VMOVHPDZ128rm, X86::INSTRUCTION_LIST_END }, 7660 { X86::VMOVHPSZ128mr, X86::VMOVHPDZ128mr, X86::INSTRUCTION_LIST_END }, 7661 }; 7662 7663 static const uint16_t ReplaceableInstrsAVX2InsertExtract[][3] = { 7664 //PackedSingle PackedDouble PackedInt 7665 { X86::VEXTRACTF128mr, X86::VEXTRACTF128mr, X86::VEXTRACTI128mr }, 7666 { X86::VEXTRACTF128rr, X86::VEXTRACTF128rr, X86::VEXTRACTI128rr }, 7667 { X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm }, 7668 { X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr }, 7669 }; 7670 7671 static const uint16_t ReplaceableInstrsAVX512[][4] = { 7672 // Two integer columns for 64-bit and 32-bit elements. 7673 //PackedSingle PackedDouble PackedInt PackedInt 7674 { X86::VMOVAPSZ128mr, X86::VMOVAPDZ128mr, X86::VMOVDQA64Z128mr, X86::VMOVDQA32Z128mr }, 7675 { X86::VMOVAPSZ128rm, X86::VMOVAPDZ128rm, X86::VMOVDQA64Z128rm, X86::VMOVDQA32Z128rm }, 7676 { X86::VMOVAPSZ128rr, X86::VMOVAPDZ128rr, X86::VMOVDQA64Z128rr, X86::VMOVDQA32Z128rr }, 7677 { X86::VMOVUPSZ128mr, X86::VMOVUPDZ128mr, X86::VMOVDQU64Z128mr, X86::VMOVDQU32Z128mr }, 7678 { X86::VMOVUPSZ128rm, X86::VMOVUPDZ128rm, X86::VMOVDQU64Z128rm, X86::VMOVDQU32Z128rm }, 7679 { X86::VMOVAPSZ256mr, X86::VMOVAPDZ256mr, X86::VMOVDQA64Z256mr, X86::VMOVDQA32Z256mr }, 7680 { X86::VMOVAPSZ256rm, X86::VMOVAPDZ256rm, X86::VMOVDQA64Z256rm, X86::VMOVDQA32Z256rm }, 7681 { X86::VMOVAPSZ256rr, X86::VMOVAPDZ256rr, X86::VMOVDQA64Z256rr, X86::VMOVDQA32Z256rr }, 7682 { X86::VMOVUPSZ256mr, X86::VMOVUPDZ256mr, X86::VMOVDQU64Z256mr, X86::VMOVDQU32Z256mr }, 7683 { X86::VMOVUPSZ256rm, X86::VMOVUPDZ256rm, X86::VMOVDQU64Z256rm, X86::VMOVDQU32Z256rm }, 7684 { X86::VMOVAPSZmr, X86::VMOVAPDZmr, X86::VMOVDQA64Zmr, X86::VMOVDQA32Zmr }, 7685 { X86::VMOVAPSZrm, X86::VMOVAPDZrm, X86::VMOVDQA64Zrm, X86::VMOVDQA32Zrm }, 7686 { X86::VMOVAPSZrr, X86::VMOVAPDZrr, X86::VMOVDQA64Zrr, X86::VMOVDQA32Zrr }, 7687 { X86::VMOVUPSZmr, X86::VMOVUPDZmr, X86::VMOVDQU64Zmr, X86::VMOVDQU32Zmr }, 7688 { X86::VMOVUPSZrm, X86::VMOVUPDZrm, X86::VMOVDQU64Zrm, X86::VMOVDQU32Zrm }, 7689 }; 7690 7691 static const uint16_t ReplaceableInstrsAVX512DQ[][4] = { 7692 // Two integer columns for 64-bit and 32-bit elements. 7693 //PackedSingle PackedDouble PackedInt PackedInt 7694 { X86::VANDNPSZ128rm, X86::VANDNPDZ128rm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, 7695 { X86::VANDNPSZ128rr, X86::VANDNPDZ128rr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, 7696 { X86::VANDPSZ128rm, X86::VANDPDZ128rm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, 7697 { X86::VANDPSZ128rr, X86::VANDPDZ128rr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, 7698 { X86::VORPSZ128rm, X86::VORPDZ128rm, X86::VPORQZ128rm, X86::VPORDZ128rm }, 7699 { X86::VORPSZ128rr, X86::VORPDZ128rr, X86::VPORQZ128rr, X86::VPORDZ128rr }, 7700 { X86::VXORPSZ128rm, X86::VXORPDZ128rm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, 7701 { X86::VXORPSZ128rr, X86::VXORPDZ128rr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, 7702 { X86::VANDNPSZ256rm, X86::VANDNPDZ256rm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, 7703 { X86::VANDNPSZ256rr, X86::VANDNPDZ256rr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, 7704 { X86::VANDPSZ256rm, X86::VANDPDZ256rm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, 7705 { X86::VANDPSZ256rr, X86::VANDPDZ256rr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, 7706 { X86::VORPSZ256rm, X86::VORPDZ256rm, X86::VPORQZ256rm, X86::VPORDZ256rm }, 7707 { X86::VORPSZ256rr, X86::VORPDZ256rr, X86::VPORQZ256rr, X86::VPORDZ256rr }, 7708 { X86::VXORPSZ256rm, X86::VXORPDZ256rm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, 7709 { X86::VXORPSZ256rr, X86::VXORPDZ256rr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, 7710 { X86::VANDNPSZrm, X86::VANDNPDZrm, X86::VPANDNQZrm, X86::VPANDNDZrm }, 7711 { X86::VANDNPSZrr, X86::VANDNPDZrr, X86::VPANDNQZrr, X86::VPANDNDZrr }, 7712 { X86::VANDPSZrm, X86::VANDPDZrm, X86::VPANDQZrm, X86::VPANDDZrm }, 7713 { X86::VANDPSZrr, X86::VANDPDZrr, X86::VPANDQZrr, X86::VPANDDZrr }, 7714 { X86::VORPSZrm, X86::VORPDZrm, X86::VPORQZrm, X86::VPORDZrm }, 7715 { X86::VORPSZrr, X86::VORPDZrr, X86::VPORQZrr, X86::VPORDZrr }, 7716 { X86::VXORPSZrm, X86::VXORPDZrm, X86::VPXORQZrm, X86::VPXORDZrm }, 7717 { X86::VXORPSZrr, X86::VXORPDZrr, X86::VPXORQZrr, X86::VPXORDZrr }, 7718 }; 7719 7720 static const uint16_t ReplaceableInstrsAVX512DQMasked[][4] = { 7721 // Two integer columns for 64-bit and 32-bit elements. 7722 //PackedSingle PackedDouble 7723 //PackedInt PackedInt 7724 { X86::VANDNPSZ128rmk, X86::VANDNPDZ128rmk, 7725 X86::VPANDNQZ128rmk, X86::VPANDNDZ128rmk }, 7726 { X86::VANDNPSZ128rmkz, X86::VANDNPDZ128rmkz, 7727 X86::VPANDNQZ128rmkz, X86::VPANDNDZ128rmkz }, 7728 { X86::VANDNPSZ128rrk, X86::VANDNPDZ128rrk, 7729 X86::VPANDNQZ128rrk, X86::VPANDNDZ128rrk }, 7730 { X86::VANDNPSZ128rrkz, X86::VANDNPDZ128rrkz, 7731 X86::VPANDNQZ128rrkz, X86::VPANDNDZ128rrkz }, 7732 { X86::VANDPSZ128rmk, X86::VANDPDZ128rmk, 7733 X86::VPANDQZ128rmk, X86::VPANDDZ128rmk }, 7734 { X86::VANDPSZ128rmkz, X86::VANDPDZ128rmkz, 7735 X86::VPANDQZ128rmkz, X86::VPANDDZ128rmkz }, 7736 { X86::VANDPSZ128rrk, X86::VANDPDZ128rrk, 7737 X86::VPANDQZ128rrk, X86::VPANDDZ128rrk }, 7738 { X86::VANDPSZ128rrkz, X86::VANDPDZ128rrkz, 7739 X86::VPANDQZ128rrkz, X86::VPANDDZ128rrkz }, 7740 { X86::VORPSZ128rmk, X86::VORPDZ128rmk, 7741 X86::VPORQZ128rmk, X86::VPORDZ128rmk }, 7742 { X86::VORPSZ128rmkz, X86::VORPDZ128rmkz, 7743 X86::VPORQZ128rmkz, X86::VPORDZ128rmkz }, 7744 { X86::VORPSZ128rrk, X86::VORPDZ128rrk, 7745 X86::VPORQZ128rrk, X86::VPORDZ128rrk }, 7746 { X86::VORPSZ128rrkz, X86::VORPDZ128rrkz, 7747 X86::VPORQZ128rrkz, X86::VPORDZ128rrkz }, 7748 { X86::VXORPSZ128rmk, X86::VXORPDZ128rmk, 7749 X86::VPXORQZ128rmk, X86::VPXORDZ128rmk }, 7750 { X86::VXORPSZ128rmkz, X86::VXORPDZ128rmkz, 7751 X86::VPXORQZ128rmkz, X86::VPXORDZ128rmkz }, 7752 { X86::VXORPSZ128rrk, X86::VXORPDZ128rrk, 7753 X86::VPXORQZ128rrk, X86::VPXORDZ128rrk }, 7754 { X86::VXORPSZ128rrkz, X86::VXORPDZ128rrkz, 7755 X86::VPXORQZ128rrkz, X86::VPXORDZ128rrkz }, 7756 { X86::VANDNPSZ256rmk, X86::VANDNPDZ256rmk, 7757 X86::VPANDNQZ256rmk, X86::VPANDNDZ256rmk }, 7758 { X86::VANDNPSZ256rmkz, X86::VANDNPDZ256rmkz, 7759 X86::VPANDNQZ256rmkz, X86::VPANDNDZ256rmkz }, 7760 { X86::VANDNPSZ256rrk, X86::VANDNPDZ256rrk, 7761 X86::VPANDNQZ256rrk, X86::VPANDNDZ256rrk }, 7762 { X86::VANDNPSZ256rrkz, X86::VANDNPDZ256rrkz, 7763 X86::VPANDNQZ256rrkz, X86::VPANDNDZ256rrkz }, 7764 { X86::VANDPSZ256rmk, X86::VANDPDZ256rmk, 7765 X86::VPANDQZ256rmk, X86::VPANDDZ256rmk }, 7766 { X86::VANDPSZ256rmkz, X86::VANDPDZ256rmkz, 7767 X86::VPANDQZ256rmkz, X86::VPANDDZ256rmkz }, 7768 { X86::VANDPSZ256rrk, X86::VANDPDZ256rrk, 7769 X86::VPANDQZ256rrk, X86::VPANDDZ256rrk }, 7770 { X86::VANDPSZ256rrkz, X86::VANDPDZ256rrkz, 7771 X86::VPANDQZ256rrkz, X86::VPANDDZ256rrkz }, 7772 { X86::VORPSZ256rmk, X86::VORPDZ256rmk, 7773 X86::VPORQZ256rmk, X86::VPORDZ256rmk }, 7774 { X86::VORPSZ256rmkz, X86::VORPDZ256rmkz, 7775 X86::VPORQZ256rmkz, X86::VPORDZ256rmkz }, 7776 { X86::VORPSZ256rrk, X86::VORPDZ256rrk, 7777 X86::VPORQZ256rrk, X86::VPORDZ256rrk }, 7778 { X86::VORPSZ256rrkz, X86::VORPDZ256rrkz, 7779 X86::VPORQZ256rrkz, X86::VPORDZ256rrkz }, 7780 { X86::VXORPSZ256rmk, X86::VXORPDZ256rmk, 7781 X86::VPXORQZ256rmk, X86::VPXORDZ256rmk }, 7782 { X86::VXORPSZ256rmkz, X86::VXORPDZ256rmkz, 7783 X86::VPXORQZ256rmkz, X86::VPXORDZ256rmkz }, 7784 { X86::VXORPSZ256rrk, X86::VXORPDZ256rrk, 7785 X86::VPXORQZ256rrk, X86::VPXORDZ256rrk }, 7786 { X86::VXORPSZ256rrkz, X86::VXORPDZ256rrkz, 7787 X86::VPXORQZ256rrkz, X86::VPXORDZ256rrkz }, 7788 { X86::VANDNPSZrmk, X86::VANDNPDZrmk, 7789 X86::VPANDNQZrmk, X86::VPANDNDZrmk }, 7790 { X86::VANDNPSZrmkz, X86::VANDNPDZrmkz, 7791 X86::VPANDNQZrmkz, X86::VPANDNDZrmkz }, 7792 { X86::VANDNPSZrrk, X86::VANDNPDZrrk, 7793 X86::VPANDNQZrrk, X86::VPANDNDZrrk }, 7794 { X86::VANDNPSZrrkz, X86::VANDNPDZrrkz, 7795 X86::VPANDNQZrrkz, X86::VPANDNDZrrkz }, 7796 { X86::VANDPSZrmk, X86::VANDPDZrmk, 7797 X86::VPANDQZrmk, X86::VPANDDZrmk }, 7798 { X86::VANDPSZrmkz, X86::VANDPDZrmkz, 7799 X86::VPANDQZrmkz, X86::VPANDDZrmkz }, 7800 { X86::VANDPSZrrk, X86::VANDPDZrrk, 7801 X86::VPANDQZrrk, X86::VPANDDZrrk }, 7802 { X86::VANDPSZrrkz, X86::VANDPDZrrkz, 7803 X86::VPANDQZrrkz, X86::VPANDDZrrkz }, 7804 { X86::VORPSZrmk, X86::VORPDZrmk, 7805 X86::VPORQZrmk, X86::VPORDZrmk }, 7806 { X86::VORPSZrmkz, X86::VORPDZrmkz, 7807 X86::VPORQZrmkz, X86::VPORDZrmkz }, 7808 { X86::VORPSZrrk, X86::VORPDZrrk, 7809 X86::VPORQZrrk, X86::VPORDZrrk }, 7810 { X86::VORPSZrrkz, X86::VORPDZrrkz, 7811 X86::VPORQZrrkz, X86::VPORDZrrkz }, 7812 { X86::VXORPSZrmk, X86::VXORPDZrmk, 7813 X86::VPXORQZrmk, X86::VPXORDZrmk }, 7814 { X86::VXORPSZrmkz, X86::VXORPDZrmkz, 7815 X86::VPXORQZrmkz, X86::VPXORDZrmkz }, 7816 { X86::VXORPSZrrk, X86::VXORPDZrrk, 7817 X86::VPXORQZrrk, X86::VPXORDZrrk }, 7818 { X86::VXORPSZrrkz, X86::VXORPDZrrkz, 7819 X86::VPXORQZrrkz, X86::VPXORDZrrkz }, 7820 // Broadcast loads can be handled the same as masked operations to avoid 7821 // changing element size. 7822 { X86::VANDNPSZ128rmb, X86::VANDNPDZ128rmb, 7823 X86::VPANDNQZ128rmb, X86::VPANDNDZ128rmb }, 7824 { X86::VANDPSZ128rmb, X86::VANDPDZ128rmb, 7825 X86::VPANDQZ128rmb, X86::VPANDDZ128rmb }, 7826 { X86::VORPSZ128rmb, X86::VORPDZ128rmb, 7827 X86::VPORQZ128rmb, X86::VPORDZ128rmb }, 7828 { X86::VXORPSZ128rmb, X86::VXORPDZ128rmb, 7829 X86::VPXORQZ128rmb, X86::VPXORDZ128rmb }, 7830 { X86::VANDNPSZ256rmb, X86::VANDNPDZ256rmb, 7831 X86::VPANDNQZ256rmb, X86::VPANDNDZ256rmb }, 7832 { X86::VANDPSZ256rmb, X86::VANDPDZ256rmb, 7833 X86::VPANDQZ256rmb, X86::VPANDDZ256rmb }, 7834 { X86::VORPSZ256rmb, X86::VORPDZ256rmb, 7835 X86::VPORQZ256rmb, X86::VPORDZ256rmb }, 7836 { X86::VXORPSZ256rmb, X86::VXORPDZ256rmb, 7837 X86::VPXORQZ256rmb, X86::VPXORDZ256rmb }, 7838 { X86::VANDNPSZrmb, X86::VANDNPDZrmb, 7839 X86::VPANDNQZrmb, X86::VPANDNDZrmb }, 7840 { X86::VANDPSZrmb, X86::VANDPDZrmb, 7841 X86::VPANDQZrmb, X86::VPANDDZrmb }, 7842 { X86::VANDPSZrmb, X86::VANDPDZrmb, 7843 X86::VPANDQZrmb, X86::VPANDDZrmb }, 7844 { X86::VORPSZrmb, X86::VORPDZrmb, 7845 X86::VPORQZrmb, X86::VPORDZrmb }, 7846 { X86::VXORPSZrmb, X86::VXORPDZrmb, 7847 X86::VPXORQZrmb, X86::VPXORDZrmb }, 7848 { X86::VANDNPSZ128rmbk, X86::VANDNPDZ128rmbk, 7849 X86::VPANDNQZ128rmbk, X86::VPANDNDZ128rmbk }, 7850 { X86::VANDPSZ128rmbk, X86::VANDPDZ128rmbk, 7851 X86::VPANDQZ128rmbk, X86::VPANDDZ128rmbk }, 7852 { X86::VORPSZ128rmbk, X86::VORPDZ128rmbk, 7853 X86::VPORQZ128rmbk, X86::VPORDZ128rmbk }, 7854 { X86::VXORPSZ128rmbk, X86::VXORPDZ128rmbk, 7855 X86::VPXORQZ128rmbk, X86::VPXORDZ128rmbk }, 7856 { X86::VANDNPSZ256rmbk, X86::VANDNPDZ256rmbk, 7857 X86::VPANDNQZ256rmbk, X86::VPANDNDZ256rmbk }, 7858 { X86::VANDPSZ256rmbk, X86::VANDPDZ256rmbk, 7859 X86::VPANDQZ256rmbk, X86::VPANDDZ256rmbk }, 7860 { X86::VORPSZ256rmbk, X86::VORPDZ256rmbk, 7861 X86::VPORQZ256rmbk, X86::VPORDZ256rmbk }, 7862 { X86::VXORPSZ256rmbk, X86::VXORPDZ256rmbk, 7863 X86::VPXORQZ256rmbk, X86::VPXORDZ256rmbk }, 7864 { X86::VANDNPSZrmbk, X86::VANDNPDZrmbk, 7865 X86::VPANDNQZrmbk, X86::VPANDNDZrmbk }, 7866 { X86::VANDPSZrmbk, X86::VANDPDZrmbk, 7867 X86::VPANDQZrmbk, X86::VPANDDZrmbk }, 7868 { X86::VANDPSZrmbk, X86::VANDPDZrmbk, 7869 X86::VPANDQZrmbk, X86::VPANDDZrmbk }, 7870 { X86::VORPSZrmbk, X86::VORPDZrmbk, 7871 X86::VPORQZrmbk, X86::VPORDZrmbk }, 7872 { X86::VXORPSZrmbk, X86::VXORPDZrmbk, 7873 X86::VPXORQZrmbk, X86::VPXORDZrmbk }, 7874 { X86::VANDNPSZ128rmbkz,X86::VANDNPDZ128rmbkz, 7875 X86::VPANDNQZ128rmbkz,X86::VPANDNDZ128rmbkz}, 7876 { X86::VANDPSZ128rmbkz, X86::VANDPDZ128rmbkz, 7877 X86::VPANDQZ128rmbkz, X86::VPANDDZ128rmbkz }, 7878 { X86::VORPSZ128rmbkz, X86::VORPDZ128rmbkz, 7879 X86::VPORQZ128rmbkz, X86::VPORDZ128rmbkz }, 7880 { X86::VXORPSZ128rmbkz, X86::VXORPDZ128rmbkz, 7881 X86::VPXORQZ128rmbkz, X86::VPXORDZ128rmbkz }, 7882 { X86::VANDNPSZ256rmbkz,X86::VANDNPDZ256rmbkz, 7883 X86::VPANDNQZ256rmbkz,X86::VPANDNDZ256rmbkz}, 7884 { X86::VANDPSZ256rmbkz, X86::VANDPDZ256rmbkz, 7885 X86::VPANDQZ256rmbkz, X86::VPANDDZ256rmbkz }, 7886 { X86::VORPSZ256rmbkz, X86::VORPDZ256rmbkz, 7887 X86::VPORQZ256rmbkz, X86::VPORDZ256rmbkz }, 7888 { X86::VXORPSZ256rmbkz, X86::VXORPDZ256rmbkz, 7889 X86::VPXORQZ256rmbkz, X86::VPXORDZ256rmbkz }, 7890 { X86::VANDNPSZrmbkz, X86::VANDNPDZrmbkz, 7891 X86::VPANDNQZrmbkz, X86::VPANDNDZrmbkz }, 7892 { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, 7893 X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, 7894 { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, 7895 X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, 7896 { X86::VORPSZrmbkz, X86::VORPDZrmbkz, 7897 X86::VPORQZrmbkz, X86::VPORDZrmbkz }, 7898 { X86::VXORPSZrmbkz, X86::VXORPDZrmbkz, 7899 X86::VPXORQZrmbkz, X86::VPXORDZrmbkz }, 7900 }; 7901 7902 // NOTE: These should only be used by the custom domain methods. 7903 static const uint16_t ReplaceableBlendInstrs[][3] = { 7904 //PackedSingle PackedDouble PackedInt 7905 { X86::BLENDPSrmi, X86::BLENDPDrmi, X86::PBLENDWrmi }, 7906 { X86::BLENDPSrri, X86::BLENDPDrri, X86::PBLENDWrri }, 7907 { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDWrmi }, 7908 { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDWrri }, 7909 { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDWYrmi }, 7910 { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDWYrri }, 7911 }; 7912 static const uint16_t ReplaceableBlendAVX2Instrs[][3] = { 7913 //PackedSingle PackedDouble PackedInt 7914 { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDDrmi }, 7915 { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDDrri }, 7916 { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDDYrmi }, 7917 { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDDYrri }, 7918 }; 7919 7920 // Special table for changing EVEX logic instructions to VEX. 7921 // TODO: Should we run EVEX->VEX earlier? 7922 static const uint16_t ReplaceableCustomAVX512LogicInstrs[][4] = { 7923 // Two integer columns for 64-bit and 32-bit elements. 7924 //PackedSingle PackedDouble PackedInt PackedInt 7925 { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, 7926 { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, 7927 { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, 7928 { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, 7929 { X86::VORPSrm, X86::VORPDrm, X86::VPORQZ128rm, X86::VPORDZ128rm }, 7930 { X86::VORPSrr, X86::VORPDrr, X86::VPORQZ128rr, X86::VPORDZ128rr }, 7931 { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, 7932 { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, 7933 { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, 7934 { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, 7935 { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, 7936 { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, 7937 { X86::VORPSYrm, X86::VORPDYrm, X86::VPORQZ256rm, X86::VPORDZ256rm }, 7938 { X86::VORPSYrr, X86::VORPDYrr, X86::VPORQZ256rr, X86::VPORDZ256rr }, 7939 { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, 7940 { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, 7941 }; 7942 7943 // FIXME: Some shuffle and unpack instructions have equivalents in different 7944 // domains, but they require a bit more work than just switching opcodes. 7945 7946 static const uint16_t *lookup(unsigned opcode, unsigned domain, 7947 ArrayRef<uint16_t[3]> Table) { 7948 for (const uint16_t (&Row)[3] : Table) 7949 if (Row[domain-1] == opcode) 7950 return Row; 7951 return nullptr; 7952 } 7953 7954 static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain, 7955 ArrayRef<uint16_t[4]> Table) { 7956 // If this is the integer domain make sure to check both integer columns. 7957 for (const uint16_t (&Row)[4] : Table) 7958 if (Row[domain-1] == opcode || (domain == 3 && Row[3] == opcode)) 7959 return Row; 7960 return nullptr; 7961 } 7962 7963 // Helper to attempt to widen/narrow blend masks. 7964 static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth, 7965 unsigned NewWidth, unsigned *pNewMask = nullptr) { 7966 assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && 7967 "Illegal blend mask scale"); 7968 unsigned NewMask = 0; 7969 7970 if ((OldWidth % NewWidth) == 0) { 7971 unsigned Scale = OldWidth / NewWidth; 7972 unsigned SubMask = (1u << Scale) - 1; 7973 for (unsigned i = 0; i != NewWidth; ++i) { 7974 unsigned Sub = (OldMask >> (i * Scale)) & SubMask; 7975 if (Sub == SubMask) 7976 NewMask |= (1u << i); 7977 else if (Sub != 0x0) 7978 return false; 7979 } 7980 } else { 7981 unsigned Scale = NewWidth / OldWidth; 7982 unsigned SubMask = (1u << Scale) - 1; 7983 for (unsigned i = 0; i != OldWidth; ++i) { 7984 if (OldMask & (1 << i)) { 7985 NewMask |= (SubMask << (i * Scale)); 7986 } 7987 } 7988 } 7989 7990 if (pNewMask) 7991 *pNewMask = NewMask; 7992 return true; 7993 } 7994 7995 uint16_t X86InstrInfo::getExecutionDomainCustom(const MachineInstr &MI) const { 7996 unsigned Opcode = MI.getOpcode(); 7997 unsigned NumOperands = MI.getDesc().getNumOperands(); 7998 7999 auto GetBlendDomains = [&](unsigned ImmWidth, bool Is256) { 8000 uint16_t validDomains = 0; 8001 if (MI.getOperand(NumOperands - 1).isImm()) { 8002 unsigned Imm = MI.getOperand(NumOperands - 1).getImm(); 8003 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4)) 8004 validDomains |= 0x2; // PackedSingle 8005 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2)) 8006 validDomains |= 0x4; // PackedDouble 8007 if (!Is256 || Subtarget.hasAVX2()) 8008 validDomains |= 0x8; // PackedInt 8009 } 8010 return validDomains; 8011 }; 8012 8013 switch (Opcode) { 8014 case X86::BLENDPDrmi: 8015 case X86::BLENDPDrri: 8016 case X86::VBLENDPDrmi: 8017 case X86::VBLENDPDrri: 8018 return GetBlendDomains(2, false); 8019 case X86::VBLENDPDYrmi: 8020 case X86::VBLENDPDYrri: 8021 return GetBlendDomains(4, true); 8022 case X86::BLENDPSrmi: 8023 case X86::BLENDPSrri: 8024 case X86::VBLENDPSrmi: 8025 case X86::VBLENDPSrri: 8026 case X86::VPBLENDDrmi: 8027 case X86::VPBLENDDrri: 8028 return GetBlendDomains(4, false); 8029 case X86::VBLENDPSYrmi: 8030 case X86::VBLENDPSYrri: 8031 case X86::VPBLENDDYrmi: 8032 case X86::VPBLENDDYrri: 8033 return GetBlendDomains(8, true); 8034 case X86::PBLENDWrmi: 8035 case X86::PBLENDWrri: 8036 case X86::VPBLENDWrmi: 8037 case X86::VPBLENDWrri: 8038 // Treat VPBLENDWY as a 128-bit vector as it repeats the lo/hi masks. 8039 case X86::VPBLENDWYrmi: 8040 case X86::VPBLENDWYrri: 8041 return GetBlendDomains(8, false); 8042 case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: 8043 case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: 8044 case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: 8045 case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: 8046 case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: 8047 case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: 8048 case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: 8049 case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: 8050 case X86::VPORDZ128rr: case X86::VPORDZ128rm: 8051 case X86::VPORDZ256rr: case X86::VPORDZ256rm: 8052 case X86::VPORQZ128rr: case X86::VPORQZ128rm: 8053 case X86::VPORQZ256rr: case X86::VPORQZ256rm: 8054 case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: 8055 case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: 8056 case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: 8057 case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: 8058 // If we don't have DQI see if we can still switch from an EVEX integer 8059 // instruction to a VEX floating point instruction. 8060 if (Subtarget.hasDQI()) 8061 return 0; 8062 8063 if (RI.getEncodingValue(MI.getOperand(0).getReg()) >= 16) 8064 return 0; 8065 if (RI.getEncodingValue(MI.getOperand(1).getReg()) >= 16) 8066 return 0; 8067 // Register forms will have 3 operands. Memory form will have more. 8068 if (NumOperands == 3 && 8069 RI.getEncodingValue(MI.getOperand(2).getReg()) >= 16) 8070 return 0; 8071 8072 // All domains are valid. 8073 return 0xe; 8074 case X86::MOVHLPSrr: 8075 // We can swap domains when both inputs are the same register. 8076 // FIXME: This doesn't catch all the cases we would like. If the input 8077 // register isn't KILLed by the instruction, the two address instruction 8078 // pass puts a COPY on one input. The other input uses the original 8079 // register. This prevents the same physical register from being used by 8080 // both inputs. 8081 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && 8082 MI.getOperand(0).getSubReg() == 0 && 8083 MI.getOperand(1).getSubReg() == 0 && 8084 MI.getOperand(2).getSubReg() == 0) 8085 return 0x6; 8086 return 0; 8087 case X86::SHUFPDrri: 8088 return 0x6; 8089 } 8090 return 0; 8091 } 8092 8093 bool X86InstrInfo::setExecutionDomainCustom(MachineInstr &MI, 8094 unsigned Domain) const { 8095 assert(Domain > 0 && Domain < 4 && "Invalid execution domain"); 8096 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 8097 assert(dom && "Not an SSE instruction"); 8098 8099 unsigned Opcode = MI.getOpcode(); 8100 unsigned NumOperands = MI.getDesc().getNumOperands(); 8101 8102 auto SetBlendDomain = [&](unsigned ImmWidth, bool Is256) { 8103 if (MI.getOperand(NumOperands - 1).isImm()) { 8104 unsigned Imm = MI.getOperand(NumOperands - 1).getImm() & 255; 8105 Imm = (ImmWidth == 16 ? ((Imm << 8) | Imm) : Imm); 8106 unsigned NewImm = Imm; 8107 8108 const uint16_t *table = lookup(Opcode, dom, ReplaceableBlendInstrs); 8109 if (!table) 8110 table = lookup(Opcode, dom, ReplaceableBlendAVX2Instrs); 8111 8112 if (Domain == 1) { // PackedSingle 8113 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); 8114 } else if (Domain == 2) { // PackedDouble 8115 AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2, &NewImm); 8116 } else if (Domain == 3) { // PackedInt 8117 if (Subtarget.hasAVX2()) { 8118 // If we are already VPBLENDW use that, else use VPBLENDD. 8119 if ((ImmWidth / (Is256 ? 2 : 1)) != 8) { 8120 table = lookup(Opcode, dom, ReplaceableBlendAVX2Instrs); 8121 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); 8122 } 8123 } else { 8124 assert(!Is256 && "128-bit vector expected"); 8125 AdjustBlendMask(Imm, ImmWidth, 8, &NewImm); 8126 } 8127 } 8128 8129 assert(table && table[Domain - 1] && "Unknown domain op"); 8130 MI.setDesc(get(table[Domain - 1])); 8131 MI.getOperand(NumOperands - 1).setImm(NewImm & 255); 8132 } 8133 return true; 8134 }; 8135 8136 switch (Opcode) { 8137 case X86::BLENDPDrmi: 8138 case X86::BLENDPDrri: 8139 case X86::VBLENDPDrmi: 8140 case X86::VBLENDPDrri: 8141 return SetBlendDomain(2, false); 8142 case X86::VBLENDPDYrmi: 8143 case X86::VBLENDPDYrri: 8144 return SetBlendDomain(4, true); 8145 case X86::BLENDPSrmi: 8146 case X86::BLENDPSrri: 8147 case X86::VBLENDPSrmi: 8148 case X86::VBLENDPSrri: 8149 case X86::VPBLENDDrmi: 8150 case X86::VPBLENDDrri: 8151 return SetBlendDomain(4, false); 8152 case X86::VBLENDPSYrmi: 8153 case X86::VBLENDPSYrri: 8154 case X86::VPBLENDDYrmi: 8155 case X86::VPBLENDDYrri: 8156 return SetBlendDomain(8, true); 8157 case X86::PBLENDWrmi: 8158 case X86::PBLENDWrri: 8159 case X86::VPBLENDWrmi: 8160 case X86::VPBLENDWrri: 8161 return SetBlendDomain(8, false); 8162 case X86::VPBLENDWYrmi: 8163 case X86::VPBLENDWYrri: 8164 return SetBlendDomain(16, true); 8165 case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: 8166 case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: 8167 case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: 8168 case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: 8169 case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: 8170 case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: 8171 case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: 8172 case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: 8173 case X86::VPORDZ128rr: case X86::VPORDZ128rm: 8174 case X86::VPORDZ256rr: case X86::VPORDZ256rm: 8175 case X86::VPORQZ128rr: case X86::VPORQZ128rm: 8176 case X86::VPORQZ256rr: case X86::VPORQZ256rm: 8177 case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: 8178 case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: 8179 case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: 8180 case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: { 8181 // Without DQI, convert EVEX instructions to VEX instructions. 8182 if (Subtarget.hasDQI()) 8183 return false; 8184 8185 const uint16_t *table = lookupAVX512(MI.getOpcode(), dom, 8186 ReplaceableCustomAVX512LogicInstrs); 8187 assert(table && "Instruction not found in table?"); 8188 // Don't change integer Q instructions to D instructions and 8189 // use D intructions if we started with a PS instruction. 8190 if (Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 8191 Domain = 4; 8192 MI.setDesc(get(table[Domain - 1])); 8193 return true; 8194 } 8195 case X86::UNPCKHPDrr: 8196 case X86::MOVHLPSrr: 8197 // We just need to commute the instruction which will switch the domains. 8198 if (Domain != dom && Domain != 3 && 8199 MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && 8200 MI.getOperand(0).getSubReg() == 0 && 8201 MI.getOperand(1).getSubReg() == 0 && 8202 MI.getOperand(2).getSubReg() == 0) { 8203 commuteInstruction(MI, false); 8204 return true; 8205 } 8206 // We must always return true for MOVHLPSrr. 8207 if (Opcode == X86::MOVHLPSrr) 8208 return true; 8209 break; 8210 case X86::SHUFPDrri: { 8211 if (Domain == 1) { 8212 unsigned Imm = MI.getOperand(3).getImm(); 8213 unsigned NewImm = 0x44; 8214 if (Imm & 1) NewImm |= 0x0a; 8215 if (Imm & 2) NewImm |= 0xa0; 8216 MI.getOperand(3).setImm(NewImm); 8217 MI.setDesc(get(X86::SHUFPSrri)); 8218 } 8219 return true; 8220 } 8221 } 8222 return false; 8223 } 8224 8225 std::pair<uint16_t, uint16_t> 8226 X86InstrInfo::getExecutionDomain(const MachineInstr &MI) const { 8227 uint16_t domain = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 8228 unsigned opcode = MI.getOpcode(); 8229 uint16_t validDomains = 0; 8230 if (domain) { 8231 // Attempt to match for custom instructions. 8232 validDomains = getExecutionDomainCustom(MI); 8233 if (validDomains) 8234 return std::make_pair(domain, validDomains); 8235 8236 if (lookup(opcode, domain, ReplaceableInstrs)) { 8237 validDomains = 0xe; 8238 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2)) { 8239 validDomains = Subtarget.hasAVX2() ? 0xe : 0x6; 8240 } else if (lookup(opcode, domain, ReplaceableInstrsFP)) { 8241 validDomains = 0x6; 8242 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2InsertExtract)) { 8243 // Insert/extract instructions should only effect domain if AVX2 8244 // is enabled. 8245 if (!Subtarget.hasAVX2()) 8246 return std::make_pair(0, 0); 8247 validDomains = 0xe; 8248 } else if (lookupAVX512(opcode, domain, ReplaceableInstrsAVX512)) { 8249 validDomains = 0xe; 8250 } else if (Subtarget.hasDQI() && lookupAVX512(opcode, domain, 8251 ReplaceableInstrsAVX512DQ)) { 8252 validDomains = 0xe; 8253 } else if (Subtarget.hasDQI()) { 8254 if (const uint16_t *table = lookupAVX512(opcode, domain, 8255 ReplaceableInstrsAVX512DQMasked)) { 8256 if (domain == 1 || (domain == 3 && table[3] == opcode)) 8257 validDomains = 0xa; 8258 else 8259 validDomains = 0xc; 8260 } 8261 } 8262 } 8263 return std::make_pair(domain, validDomains); 8264 } 8265 8266 void X86InstrInfo::setExecutionDomain(MachineInstr &MI, unsigned Domain) const { 8267 assert(Domain>0 && Domain<4 && "Invalid execution domain"); 8268 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 8269 assert(dom && "Not an SSE instruction"); 8270 8271 // Attempt to match for custom instructions. 8272 if (setExecutionDomainCustom(MI, Domain)) 8273 return; 8274 8275 const uint16_t *table = lookup(MI.getOpcode(), dom, ReplaceableInstrs); 8276 if (!table) { // try the other table 8277 assert((Subtarget.hasAVX2() || Domain < 3) && 8278 "256-bit vector operations only available in AVX2"); 8279 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2); 8280 } 8281 if (!table) { // try the FP table 8282 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsFP); 8283 assert((!table || Domain < 3) && 8284 "Can only select PackedSingle or PackedDouble"); 8285 } 8286 if (!table) { // try the other table 8287 assert(Subtarget.hasAVX2() && 8288 "256-bit insert/extract only available in AVX2"); 8289 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2InsertExtract); 8290 } 8291 if (!table) { // try the AVX512 table 8292 assert(Subtarget.hasAVX512() && "Requires AVX-512"); 8293 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512); 8294 // Don't change integer Q instructions to D instructions. 8295 if (table && Domain == 3 && table[3] == MI.getOpcode()) 8296 Domain = 4; 8297 } 8298 if (!table) { // try the AVX512DQ table 8299 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ"); 8300 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQ); 8301 // Don't change integer Q instructions to D instructions and 8302 // use D instructions if we started with a PS instruction. 8303 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 8304 Domain = 4; 8305 } 8306 if (!table) { // try the AVX512DQMasked table 8307 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ"); 8308 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQMasked); 8309 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 8310 Domain = 4; 8311 } 8312 assert(table && "Cannot change domain"); 8313 MI.setDesc(get(table[Domain - 1])); 8314 } 8315 8316 /// Return the noop instruction to use for a noop. 8317 MCInst X86InstrInfo::getNop() const { 8318 MCInst Nop; 8319 Nop.setOpcode(X86::NOOP); 8320 return Nop; 8321 } 8322 8323 bool X86InstrInfo::isHighLatencyDef(int opc) const { 8324 switch (opc) { 8325 default: return false; 8326 case X86::DIVPDrm: 8327 case X86::DIVPDrr: 8328 case X86::DIVPSrm: 8329 case X86::DIVPSrr: 8330 case X86::DIVSDrm: 8331 case X86::DIVSDrm_Int: 8332 case X86::DIVSDrr: 8333 case X86::DIVSDrr_Int: 8334 case X86::DIVSSrm: 8335 case X86::DIVSSrm_Int: 8336 case X86::DIVSSrr: 8337 case X86::DIVSSrr_Int: 8338 case X86::SQRTPDm: 8339 case X86::SQRTPDr: 8340 case X86::SQRTPSm: 8341 case X86::SQRTPSr: 8342 case X86::SQRTSDm: 8343 case X86::SQRTSDm_Int: 8344 case X86::SQRTSDr: 8345 case X86::SQRTSDr_Int: 8346 case X86::SQRTSSm: 8347 case X86::SQRTSSm_Int: 8348 case X86::SQRTSSr: 8349 case X86::SQRTSSr_Int: 8350 // AVX instructions with high latency 8351 case X86::VDIVPDrm: 8352 case X86::VDIVPDrr: 8353 case X86::VDIVPDYrm: 8354 case X86::VDIVPDYrr: 8355 case X86::VDIVPSrm: 8356 case X86::VDIVPSrr: 8357 case X86::VDIVPSYrm: 8358 case X86::VDIVPSYrr: 8359 case X86::VDIVSDrm: 8360 case X86::VDIVSDrm_Int: 8361 case X86::VDIVSDrr: 8362 case X86::VDIVSDrr_Int: 8363 case X86::VDIVSSrm: 8364 case X86::VDIVSSrm_Int: 8365 case X86::VDIVSSrr: 8366 case X86::VDIVSSrr_Int: 8367 case X86::VSQRTPDm: 8368 case X86::VSQRTPDr: 8369 case X86::VSQRTPDYm: 8370 case X86::VSQRTPDYr: 8371 case X86::VSQRTPSm: 8372 case X86::VSQRTPSr: 8373 case X86::VSQRTPSYm: 8374 case X86::VSQRTPSYr: 8375 case X86::VSQRTSDm: 8376 case X86::VSQRTSDm_Int: 8377 case X86::VSQRTSDr: 8378 case X86::VSQRTSDr_Int: 8379 case X86::VSQRTSSm: 8380 case X86::VSQRTSSm_Int: 8381 case X86::VSQRTSSr: 8382 case X86::VSQRTSSr_Int: 8383 // AVX512 instructions with high latency 8384 case X86::VDIVPDZ128rm: 8385 case X86::VDIVPDZ128rmb: 8386 case X86::VDIVPDZ128rmbk: 8387 case X86::VDIVPDZ128rmbkz: 8388 case X86::VDIVPDZ128rmk: 8389 case X86::VDIVPDZ128rmkz: 8390 case X86::VDIVPDZ128rr: 8391 case X86::VDIVPDZ128rrk: 8392 case X86::VDIVPDZ128rrkz: 8393 case X86::VDIVPDZ256rm: 8394 case X86::VDIVPDZ256rmb: 8395 case X86::VDIVPDZ256rmbk: 8396 case X86::VDIVPDZ256rmbkz: 8397 case X86::VDIVPDZ256rmk: 8398 case X86::VDIVPDZ256rmkz: 8399 case X86::VDIVPDZ256rr: 8400 case X86::VDIVPDZ256rrk: 8401 case X86::VDIVPDZ256rrkz: 8402 case X86::VDIVPDZrrb: 8403 case X86::VDIVPDZrrbk: 8404 case X86::VDIVPDZrrbkz: 8405 case X86::VDIVPDZrm: 8406 case X86::VDIVPDZrmb: 8407 case X86::VDIVPDZrmbk: 8408 case X86::VDIVPDZrmbkz: 8409 case X86::VDIVPDZrmk: 8410 case X86::VDIVPDZrmkz: 8411 case X86::VDIVPDZrr: 8412 case X86::VDIVPDZrrk: 8413 case X86::VDIVPDZrrkz: 8414 case X86::VDIVPSZ128rm: 8415 case X86::VDIVPSZ128rmb: 8416 case X86::VDIVPSZ128rmbk: 8417 case X86::VDIVPSZ128rmbkz: 8418 case X86::VDIVPSZ128rmk: 8419 case X86::VDIVPSZ128rmkz: 8420 case X86::VDIVPSZ128rr: 8421 case X86::VDIVPSZ128rrk: 8422 case X86::VDIVPSZ128rrkz: 8423 case X86::VDIVPSZ256rm: 8424 case X86::VDIVPSZ256rmb: 8425 case X86::VDIVPSZ256rmbk: 8426 case X86::VDIVPSZ256rmbkz: 8427 case X86::VDIVPSZ256rmk: 8428 case X86::VDIVPSZ256rmkz: 8429 case X86::VDIVPSZ256rr: 8430 case X86::VDIVPSZ256rrk: 8431 case X86::VDIVPSZ256rrkz: 8432 case X86::VDIVPSZrrb: 8433 case X86::VDIVPSZrrbk: 8434 case X86::VDIVPSZrrbkz: 8435 case X86::VDIVPSZrm: 8436 case X86::VDIVPSZrmb: 8437 case X86::VDIVPSZrmbk: 8438 case X86::VDIVPSZrmbkz: 8439 case X86::VDIVPSZrmk: 8440 case X86::VDIVPSZrmkz: 8441 case X86::VDIVPSZrr: 8442 case X86::VDIVPSZrrk: 8443 case X86::VDIVPSZrrkz: 8444 case X86::VDIVSDZrm: 8445 case X86::VDIVSDZrr: 8446 case X86::VDIVSDZrm_Int: 8447 case X86::VDIVSDZrm_Intk: 8448 case X86::VDIVSDZrm_Intkz: 8449 case X86::VDIVSDZrr_Int: 8450 case X86::VDIVSDZrr_Intk: 8451 case X86::VDIVSDZrr_Intkz: 8452 case X86::VDIVSDZrrb_Int: 8453 case X86::VDIVSDZrrb_Intk: 8454 case X86::VDIVSDZrrb_Intkz: 8455 case X86::VDIVSSZrm: 8456 case X86::VDIVSSZrr: 8457 case X86::VDIVSSZrm_Int: 8458 case X86::VDIVSSZrm_Intk: 8459 case X86::VDIVSSZrm_Intkz: 8460 case X86::VDIVSSZrr_Int: 8461 case X86::VDIVSSZrr_Intk: 8462 case X86::VDIVSSZrr_Intkz: 8463 case X86::VDIVSSZrrb_Int: 8464 case X86::VDIVSSZrrb_Intk: 8465 case X86::VDIVSSZrrb_Intkz: 8466 case X86::VSQRTPDZ128m: 8467 case X86::VSQRTPDZ128mb: 8468 case X86::VSQRTPDZ128mbk: 8469 case X86::VSQRTPDZ128mbkz: 8470 case X86::VSQRTPDZ128mk: 8471 case X86::VSQRTPDZ128mkz: 8472 case X86::VSQRTPDZ128r: 8473 case X86::VSQRTPDZ128rk: 8474 case X86::VSQRTPDZ128rkz: 8475 case X86::VSQRTPDZ256m: 8476 case X86::VSQRTPDZ256mb: 8477 case X86::VSQRTPDZ256mbk: 8478 case X86::VSQRTPDZ256mbkz: 8479 case X86::VSQRTPDZ256mk: 8480 case X86::VSQRTPDZ256mkz: 8481 case X86::VSQRTPDZ256r: 8482 case X86::VSQRTPDZ256rk: 8483 case X86::VSQRTPDZ256rkz: 8484 case X86::VSQRTPDZm: 8485 case X86::VSQRTPDZmb: 8486 case X86::VSQRTPDZmbk: 8487 case X86::VSQRTPDZmbkz: 8488 case X86::VSQRTPDZmk: 8489 case X86::VSQRTPDZmkz: 8490 case X86::VSQRTPDZr: 8491 case X86::VSQRTPDZrb: 8492 case X86::VSQRTPDZrbk: 8493 case X86::VSQRTPDZrbkz: 8494 case X86::VSQRTPDZrk: 8495 case X86::VSQRTPDZrkz: 8496 case X86::VSQRTPSZ128m: 8497 case X86::VSQRTPSZ128mb: 8498 case X86::VSQRTPSZ128mbk: 8499 case X86::VSQRTPSZ128mbkz: 8500 case X86::VSQRTPSZ128mk: 8501 case X86::VSQRTPSZ128mkz: 8502 case X86::VSQRTPSZ128r: 8503 case X86::VSQRTPSZ128rk: 8504 case X86::VSQRTPSZ128rkz: 8505 case X86::VSQRTPSZ256m: 8506 case X86::VSQRTPSZ256mb: 8507 case X86::VSQRTPSZ256mbk: 8508 case X86::VSQRTPSZ256mbkz: 8509 case X86::VSQRTPSZ256mk: 8510 case X86::VSQRTPSZ256mkz: 8511 case X86::VSQRTPSZ256r: 8512 case X86::VSQRTPSZ256rk: 8513 case X86::VSQRTPSZ256rkz: 8514 case X86::VSQRTPSZm: 8515 case X86::VSQRTPSZmb: 8516 case X86::VSQRTPSZmbk: 8517 case X86::VSQRTPSZmbkz: 8518 case X86::VSQRTPSZmk: 8519 case X86::VSQRTPSZmkz: 8520 case X86::VSQRTPSZr: 8521 case X86::VSQRTPSZrb: 8522 case X86::VSQRTPSZrbk: 8523 case X86::VSQRTPSZrbkz: 8524 case X86::VSQRTPSZrk: 8525 case X86::VSQRTPSZrkz: 8526 case X86::VSQRTSDZm: 8527 case X86::VSQRTSDZm_Int: 8528 case X86::VSQRTSDZm_Intk: 8529 case X86::VSQRTSDZm_Intkz: 8530 case X86::VSQRTSDZr: 8531 case X86::VSQRTSDZr_Int: 8532 case X86::VSQRTSDZr_Intk: 8533 case X86::VSQRTSDZr_Intkz: 8534 case X86::VSQRTSDZrb_Int: 8535 case X86::VSQRTSDZrb_Intk: 8536 case X86::VSQRTSDZrb_Intkz: 8537 case X86::VSQRTSSZm: 8538 case X86::VSQRTSSZm_Int: 8539 case X86::VSQRTSSZm_Intk: 8540 case X86::VSQRTSSZm_Intkz: 8541 case X86::VSQRTSSZr: 8542 case X86::VSQRTSSZr_Int: 8543 case X86::VSQRTSSZr_Intk: 8544 case X86::VSQRTSSZr_Intkz: 8545 case X86::VSQRTSSZrb_Int: 8546 case X86::VSQRTSSZrb_Intk: 8547 case X86::VSQRTSSZrb_Intkz: 8548 8549 case X86::VGATHERDPDYrm: 8550 case X86::VGATHERDPDZ128rm: 8551 case X86::VGATHERDPDZ256rm: 8552 case X86::VGATHERDPDZrm: 8553 case X86::VGATHERDPDrm: 8554 case X86::VGATHERDPSYrm: 8555 case X86::VGATHERDPSZ128rm: 8556 case X86::VGATHERDPSZ256rm: 8557 case X86::VGATHERDPSZrm: 8558 case X86::VGATHERDPSrm: 8559 case X86::VGATHERPF0DPDm: 8560 case X86::VGATHERPF0DPSm: 8561 case X86::VGATHERPF0QPDm: 8562 case X86::VGATHERPF0QPSm: 8563 case X86::VGATHERPF1DPDm: 8564 case X86::VGATHERPF1DPSm: 8565 case X86::VGATHERPF1QPDm: 8566 case X86::VGATHERPF1QPSm: 8567 case X86::VGATHERQPDYrm: 8568 case X86::VGATHERQPDZ128rm: 8569 case X86::VGATHERQPDZ256rm: 8570 case X86::VGATHERQPDZrm: 8571 case X86::VGATHERQPDrm: 8572 case X86::VGATHERQPSYrm: 8573 case X86::VGATHERQPSZ128rm: 8574 case X86::VGATHERQPSZ256rm: 8575 case X86::VGATHERQPSZrm: 8576 case X86::VGATHERQPSrm: 8577 case X86::VPGATHERDDYrm: 8578 case X86::VPGATHERDDZ128rm: 8579 case X86::VPGATHERDDZ256rm: 8580 case X86::VPGATHERDDZrm: 8581 case X86::VPGATHERDDrm: 8582 case X86::VPGATHERDQYrm: 8583 case X86::VPGATHERDQZ128rm: 8584 case X86::VPGATHERDQZ256rm: 8585 case X86::VPGATHERDQZrm: 8586 case X86::VPGATHERDQrm: 8587 case X86::VPGATHERQDYrm: 8588 case X86::VPGATHERQDZ128rm: 8589 case X86::VPGATHERQDZ256rm: 8590 case X86::VPGATHERQDZrm: 8591 case X86::VPGATHERQDrm: 8592 case X86::VPGATHERQQYrm: 8593 case X86::VPGATHERQQZ128rm: 8594 case X86::VPGATHERQQZ256rm: 8595 case X86::VPGATHERQQZrm: 8596 case X86::VPGATHERQQrm: 8597 case X86::VSCATTERDPDZ128mr: 8598 case X86::VSCATTERDPDZ256mr: 8599 case X86::VSCATTERDPDZmr: 8600 case X86::VSCATTERDPSZ128mr: 8601 case X86::VSCATTERDPSZ256mr: 8602 case X86::VSCATTERDPSZmr: 8603 case X86::VSCATTERPF0DPDm: 8604 case X86::VSCATTERPF0DPSm: 8605 case X86::VSCATTERPF0QPDm: 8606 case X86::VSCATTERPF0QPSm: 8607 case X86::VSCATTERPF1DPDm: 8608 case X86::VSCATTERPF1DPSm: 8609 case X86::VSCATTERPF1QPDm: 8610 case X86::VSCATTERPF1QPSm: 8611 case X86::VSCATTERQPDZ128mr: 8612 case X86::VSCATTERQPDZ256mr: 8613 case X86::VSCATTERQPDZmr: 8614 case X86::VSCATTERQPSZ128mr: 8615 case X86::VSCATTERQPSZ256mr: 8616 case X86::VSCATTERQPSZmr: 8617 case X86::VPSCATTERDDZ128mr: 8618 case X86::VPSCATTERDDZ256mr: 8619 case X86::VPSCATTERDDZmr: 8620 case X86::VPSCATTERDQZ128mr: 8621 case X86::VPSCATTERDQZ256mr: 8622 case X86::VPSCATTERDQZmr: 8623 case X86::VPSCATTERQDZ128mr: 8624 case X86::VPSCATTERQDZ256mr: 8625 case X86::VPSCATTERQDZmr: 8626 case X86::VPSCATTERQQZ128mr: 8627 case X86::VPSCATTERQQZ256mr: 8628 case X86::VPSCATTERQQZmr: 8629 return true; 8630 } 8631 } 8632 8633 bool X86InstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel, 8634 const MachineRegisterInfo *MRI, 8635 const MachineInstr &DefMI, 8636 unsigned DefIdx, 8637 const MachineInstr &UseMI, 8638 unsigned UseIdx) const { 8639 return isHighLatencyDef(DefMI.getOpcode()); 8640 } 8641 8642 bool X86InstrInfo::hasReassociableOperands(const MachineInstr &Inst, 8643 const MachineBasicBlock *MBB) const { 8644 assert(Inst.getNumExplicitOperands() == 3 && Inst.getNumExplicitDefs() == 1 && 8645 Inst.getNumDefs() <= 2 && "Reassociation needs binary operators"); 8646 8647 // Integer binary math/logic instructions have a third source operand: 8648 // the EFLAGS register. That operand must be both defined here and never 8649 // used; ie, it must be dead. If the EFLAGS operand is live, then we can 8650 // not change anything because rearranging the operands could affect other 8651 // instructions that depend on the exact status flags (zero, sign, etc.) 8652 // that are set by using these particular operands with this operation. 8653 const MachineOperand *FlagDef = Inst.findRegisterDefOperand(X86::EFLAGS); 8654 assert((Inst.getNumDefs() == 1 || FlagDef) && 8655 "Implicit def isn't flags?"); 8656 if (FlagDef && !FlagDef->isDead()) 8657 return false; 8658 8659 return TargetInstrInfo::hasReassociableOperands(Inst, MBB); 8660 } 8661 8662 // TODO: There are many more machine instruction opcodes to match: 8663 // 1. Other data types (integer, vectors) 8664 // 2. Other math / logic operations (xor, or) 8665 // 3. Other forms of the same operation (intrinsics and other variants) 8666 bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const { 8667 switch (Inst.getOpcode()) { 8668 case X86::AND8rr: 8669 case X86::AND16rr: 8670 case X86::AND32rr: 8671 case X86::AND64rr: 8672 case X86::OR8rr: 8673 case X86::OR16rr: 8674 case X86::OR32rr: 8675 case X86::OR64rr: 8676 case X86::XOR8rr: 8677 case X86::XOR16rr: 8678 case X86::XOR32rr: 8679 case X86::XOR64rr: 8680 case X86::IMUL16rr: 8681 case X86::IMUL32rr: 8682 case X86::IMUL64rr: 8683 case X86::PANDrr: 8684 case X86::PORrr: 8685 case X86::PXORrr: 8686 case X86::ANDPDrr: 8687 case X86::ANDPSrr: 8688 case X86::ORPDrr: 8689 case X86::ORPSrr: 8690 case X86::XORPDrr: 8691 case X86::XORPSrr: 8692 case X86::PADDBrr: 8693 case X86::PADDWrr: 8694 case X86::PADDDrr: 8695 case X86::PADDQrr: 8696 case X86::PMULLWrr: 8697 case X86::PMULLDrr: 8698 case X86::PMAXSBrr: 8699 case X86::PMAXSDrr: 8700 case X86::PMAXSWrr: 8701 case X86::PMAXUBrr: 8702 case X86::PMAXUDrr: 8703 case X86::PMAXUWrr: 8704 case X86::PMINSBrr: 8705 case X86::PMINSDrr: 8706 case X86::PMINSWrr: 8707 case X86::PMINUBrr: 8708 case X86::PMINUDrr: 8709 case X86::PMINUWrr: 8710 case X86::VPANDrr: 8711 case X86::VPANDYrr: 8712 case X86::VPANDDZ128rr: 8713 case X86::VPANDDZ256rr: 8714 case X86::VPANDDZrr: 8715 case X86::VPANDQZ128rr: 8716 case X86::VPANDQZ256rr: 8717 case X86::VPANDQZrr: 8718 case X86::VPORrr: 8719 case X86::VPORYrr: 8720 case X86::VPORDZ128rr: 8721 case X86::VPORDZ256rr: 8722 case X86::VPORDZrr: 8723 case X86::VPORQZ128rr: 8724 case X86::VPORQZ256rr: 8725 case X86::VPORQZrr: 8726 case X86::VPXORrr: 8727 case X86::VPXORYrr: 8728 case X86::VPXORDZ128rr: 8729 case X86::VPXORDZ256rr: 8730 case X86::VPXORDZrr: 8731 case X86::VPXORQZ128rr: 8732 case X86::VPXORQZ256rr: 8733 case X86::VPXORQZrr: 8734 case X86::VANDPDrr: 8735 case X86::VANDPSrr: 8736 case X86::VANDPDYrr: 8737 case X86::VANDPSYrr: 8738 case X86::VANDPDZ128rr: 8739 case X86::VANDPSZ128rr: 8740 case X86::VANDPDZ256rr: 8741 case X86::VANDPSZ256rr: 8742 case X86::VANDPDZrr: 8743 case X86::VANDPSZrr: 8744 case X86::VORPDrr: 8745 case X86::VORPSrr: 8746 case X86::VORPDYrr: 8747 case X86::VORPSYrr: 8748 case X86::VORPDZ128rr: 8749 case X86::VORPSZ128rr: 8750 case X86::VORPDZ256rr: 8751 case X86::VORPSZ256rr: 8752 case X86::VORPDZrr: 8753 case X86::VORPSZrr: 8754 case X86::VXORPDrr: 8755 case X86::VXORPSrr: 8756 case X86::VXORPDYrr: 8757 case X86::VXORPSYrr: 8758 case X86::VXORPDZ128rr: 8759 case X86::VXORPSZ128rr: 8760 case X86::VXORPDZ256rr: 8761 case X86::VXORPSZ256rr: 8762 case X86::VXORPDZrr: 8763 case X86::VXORPSZrr: 8764 case X86::KADDBrr: 8765 case X86::KADDWrr: 8766 case X86::KADDDrr: 8767 case X86::KADDQrr: 8768 case X86::KANDBrr: 8769 case X86::KANDWrr: 8770 case X86::KANDDrr: 8771 case X86::KANDQrr: 8772 case X86::KORBrr: 8773 case X86::KORWrr: 8774 case X86::KORDrr: 8775 case X86::KORQrr: 8776 case X86::KXORBrr: 8777 case X86::KXORWrr: 8778 case X86::KXORDrr: 8779 case X86::KXORQrr: 8780 case X86::VPADDBrr: 8781 case X86::VPADDWrr: 8782 case X86::VPADDDrr: 8783 case X86::VPADDQrr: 8784 case X86::VPADDBYrr: 8785 case X86::VPADDWYrr: 8786 case X86::VPADDDYrr: 8787 case X86::VPADDQYrr: 8788 case X86::VPADDBZ128rr: 8789 case X86::VPADDWZ128rr: 8790 case X86::VPADDDZ128rr: 8791 case X86::VPADDQZ128rr: 8792 case X86::VPADDBZ256rr: 8793 case X86::VPADDWZ256rr: 8794 case X86::VPADDDZ256rr: 8795 case X86::VPADDQZ256rr: 8796 case X86::VPADDBZrr: 8797 case X86::VPADDWZrr: 8798 case X86::VPADDDZrr: 8799 case X86::VPADDQZrr: 8800 case X86::VPMULLWrr: 8801 case X86::VPMULLWYrr: 8802 case X86::VPMULLWZ128rr: 8803 case X86::VPMULLWZ256rr: 8804 case X86::VPMULLWZrr: 8805 case X86::VPMULLDrr: 8806 case X86::VPMULLDYrr: 8807 case X86::VPMULLDZ128rr: 8808 case X86::VPMULLDZ256rr: 8809 case X86::VPMULLDZrr: 8810 case X86::VPMULLQZ128rr: 8811 case X86::VPMULLQZ256rr: 8812 case X86::VPMULLQZrr: 8813 case X86::VPMAXSBrr: 8814 case X86::VPMAXSBYrr: 8815 case X86::VPMAXSBZ128rr: 8816 case X86::VPMAXSBZ256rr: 8817 case X86::VPMAXSBZrr: 8818 case X86::VPMAXSDrr: 8819 case X86::VPMAXSDYrr: 8820 case X86::VPMAXSDZ128rr: 8821 case X86::VPMAXSDZ256rr: 8822 case X86::VPMAXSDZrr: 8823 case X86::VPMAXSQZ128rr: 8824 case X86::VPMAXSQZ256rr: 8825 case X86::VPMAXSQZrr: 8826 case X86::VPMAXSWrr: 8827 case X86::VPMAXSWYrr: 8828 case X86::VPMAXSWZ128rr: 8829 case X86::VPMAXSWZ256rr: 8830 case X86::VPMAXSWZrr: 8831 case X86::VPMAXUBrr: 8832 case X86::VPMAXUBYrr: 8833 case X86::VPMAXUBZ128rr: 8834 case X86::VPMAXUBZ256rr: 8835 case X86::VPMAXUBZrr: 8836 case X86::VPMAXUDrr: 8837 case X86::VPMAXUDYrr: 8838 case X86::VPMAXUDZ128rr: 8839 case X86::VPMAXUDZ256rr: 8840 case X86::VPMAXUDZrr: 8841 case X86::VPMAXUQZ128rr: 8842 case X86::VPMAXUQZ256rr: 8843 case X86::VPMAXUQZrr: 8844 case X86::VPMAXUWrr: 8845 case X86::VPMAXUWYrr: 8846 case X86::VPMAXUWZ128rr: 8847 case X86::VPMAXUWZ256rr: 8848 case X86::VPMAXUWZrr: 8849 case X86::VPMINSBrr: 8850 case X86::VPMINSBYrr: 8851 case X86::VPMINSBZ128rr: 8852 case X86::VPMINSBZ256rr: 8853 case X86::VPMINSBZrr: 8854 case X86::VPMINSDrr: 8855 case X86::VPMINSDYrr: 8856 case X86::VPMINSDZ128rr: 8857 case X86::VPMINSDZ256rr: 8858 case X86::VPMINSDZrr: 8859 case X86::VPMINSQZ128rr: 8860 case X86::VPMINSQZ256rr: 8861 case X86::VPMINSQZrr: 8862 case X86::VPMINSWrr: 8863 case X86::VPMINSWYrr: 8864 case X86::VPMINSWZ128rr: 8865 case X86::VPMINSWZ256rr: 8866 case X86::VPMINSWZrr: 8867 case X86::VPMINUBrr: 8868 case X86::VPMINUBYrr: 8869 case X86::VPMINUBZ128rr: 8870 case X86::VPMINUBZ256rr: 8871 case X86::VPMINUBZrr: 8872 case X86::VPMINUDrr: 8873 case X86::VPMINUDYrr: 8874 case X86::VPMINUDZ128rr: 8875 case X86::VPMINUDZ256rr: 8876 case X86::VPMINUDZrr: 8877 case X86::VPMINUQZ128rr: 8878 case X86::VPMINUQZ256rr: 8879 case X86::VPMINUQZrr: 8880 case X86::VPMINUWrr: 8881 case X86::VPMINUWYrr: 8882 case X86::VPMINUWZ128rr: 8883 case X86::VPMINUWZ256rr: 8884 case X86::VPMINUWZrr: 8885 // Normal min/max instructions are not commutative because of NaN and signed 8886 // zero semantics, but these are. Thus, there's no need to check for global 8887 // relaxed math; the instructions themselves have the properties we need. 8888 case X86::MAXCPDrr: 8889 case X86::MAXCPSrr: 8890 case X86::MAXCSDrr: 8891 case X86::MAXCSSrr: 8892 case X86::MINCPDrr: 8893 case X86::MINCPSrr: 8894 case X86::MINCSDrr: 8895 case X86::MINCSSrr: 8896 case X86::VMAXCPDrr: 8897 case X86::VMAXCPSrr: 8898 case X86::VMAXCPDYrr: 8899 case X86::VMAXCPSYrr: 8900 case X86::VMAXCPDZ128rr: 8901 case X86::VMAXCPSZ128rr: 8902 case X86::VMAXCPDZ256rr: 8903 case X86::VMAXCPSZ256rr: 8904 case X86::VMAXCPDZrr: 8905 case X86::VMAXCPSZrr: 8906 case X86::VMAXCSDrr: 8907 case X86::VMAXCSSrr: 8908 case X86::VMAXCSDZrr: 8909 case X86::VMAXCSSZrr: 8910 case X86::VMINCPDrr: 8911 case X86::VMINCPSrr: 8912 case X86::VMINCPDYrr: 8913 case X86::VMINCPSYrr: 8914 case X86::VMINCPDZ128rr: 8915 case X86::VMINCPSZ128rr: 8916 case X86::VMINCPDZ256rr: 8917 case X86::VMINCPSZ256rr: 8918 case X86::VMINCPDZrr: 8919 case X86::VMINCPSZrr: 8920 case X86::VMINCSDrr: 8921 case X86::VMINCSSrr: 8922 case X86::VMINCSDZrr: 8923 case X86::VMINCSSZrr: 8924 case X86::VMAXCPHZ128rr: 8925 case X86::VMAXCPHZ256rr: 8926 case X86::VMAXCPHZrr: 8927 case X86::VMAXCSHZrr: 8928 case X86::VMINCPHZ128rr: 8929 case X86::VMINCPHZ256rr: 8930 case X86::VMINCPHZrr: 8931 case X86::VMINCSHZrr: 8932 return true; 8933 case X86::ADDPDrr: 8934 case X86::ADDPSrr: 8935 case X86::ADDSDrr: 8936 case X86::ADDSSrr: 8937 case X86::MULPDrr: 8938 case X86::MULPSrr: 8939 case X86::MULSDrr: 8940 case X86::MULSSrr: 8941 case X86::VADDPDrr: 8942 case X86::VADDPSrr: 8943 case X86::VADDPDYrr: 8944 case X86::VADDPSYrr: 8945 case X86::VADDPDZ128rr: 8946 case X86::VADDPSZ128rr: 8947 case X86::VADDPDZ256rr: 8948 case X86::VADDPSZ256rr: 8949 case X86::VADDPDZrr: 8950 case X86::VADDPSZrr: 8951 case X86::VADDSDrr: 8952 case X86::VADDSSrr: 8953 case X86::VADDSDZrr: 8954 case X86::VADDSSZrr: 8955 case X86::VMULPDrr: 8956 case X86::VMULPSrr: 8957 case X86::VMULPDYrr: 8958 case X86::VMULPSYrr: 8959 case X86::VMULPDZ128rr: 8960 case X86::VMULPSZ128rr: 8961 case X86::VMULPDZ256rr: 8962 case X86::VMULPSZ256rr: 8963 case X86::VMULPDZrr: 8964 case X86::VMULPSZrr: 8965 case X86::VMULSDrr: 8966 case X86::VMULSSrr: 8967 case X86::VMULSDZrr: 8968 case X86::VMULSSZrr: 8969 case X86::VADDPHZ128rr: 8970 case X86::VADDPHZ256rr: 8971 case X86::VADDPHZrr: 8972 case X86::VADDSHZrr: 8973 case X86::VMULPHZ128rr: 8974 case X86::VMULPHZ256rr: 8975 case X86::VMULPHZrr: 8976 case X86::VMULSHZrr: 8977 return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) && 8978 Inst.getFlag(MachineInstr::MIFlag::FmNsz); 8979 default: 8980 return false; 8981 } 8982 } 8983 8984 /// If \p DescribedReg overlaps with the MOVrr instruction's destination 8985 /// register then, if possible, describe the value in terms of the source 8986 /// register. 8987 static Optional<ParamLoadedValue> 8988 describeMOVrrLoadedValue(const MachineInstr &MI, Register DescribedReg, 8989 const TargetRegisterInfo *TRI) { 8990 Register DestReg = MI.getOperand(0).getReg(); 8991 Register SrcReg = MI.getOperand(1).getReg(); 8992 8993 auto Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {}); 8994 8995 // If the described register is the destination, just return the source. 8996 if (DestReg == DescribedReg) 8997 return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr); 8998 8999 // If the described register is a sub-register of the destination register, 9000 // then pick out the source register's corresponding sub-register. 9001 if (unsigned SubRegIdx = TRI->getSubRegIndex(DestReg, DescribedReg)) { 9002 Register SrcSubReg = TRI->getSubReg(SrcReg, SubRegIdx); 9003 return ParamLoadedValue(MachineOperand::CreateReg(SrcSubReg, false), Expr); 9004 } 9005 9006 // The remaining case to consider is when the described register is a 9007 // super-register of the destination register. MOV8rr and MOV16rr does not 9008 // write to any of the other bytes in the register, meaning that we'd have to 9009 // describe the value using a combination of the source register and the 9010 // non-overlapping bits in the described register, which is not currently 9011 // possible. 9012 if (MI.getOpcode() == X86::MOV8rr || MI.getOpcode() == X86::MOV16rr || 9013 !TRI->isSuperRegister(DestReg, DescribedReg)) 9014 return None; 9015 9016 assert(MI.getOpcode() == X86::MOV32rr && "Unexpected super-register case"); 9017 return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr); 9018 } 9019 9020 Optional<ParamLoadedValue> 9021 X86InstrInfo::describeLoadedValue(const MachineInstr &MI, Register Reg) const { 9022 const MachineOperand *Op = nullptr; 9023 DIExpression *Expr = nullptr; 9024 9025 const TargetRegisterInfo *TRI = &getRegisterInfo(); 9026 9027 switch (MI.getOpcode()) { 9028 case X86::LEA32r: 9029 case X86::LEA64r: 9030 case X86::LEA64_32r: { 9031 // We may need to describe a 64-bit parameter with a 32-bit LEA. 9032 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) 9033 return None; 9034 9035 // Operand 4 could be global address. For now we do not support 9036 // such situation. 9037 if (!MI.getOperand(4).isImm() || !MI.getOperand(2).isImm()) 9038 return None; 9039 9040 const MachineOperand &Op1 = MI.getOperand(1); 9041 const MachineOperand &Op2 = MI.getOperand(3); 9042 assert(Op2.isReg() && (Op2.getReg() == X86::NoRegister || 9043 Register::isPhysicalRegister(Op2.getReg()))); 9044 9045 // Omit situations like: 9046 // %rsi = lea %rsi, 4, ... 9047 if ((Op1.isReg() && Op1.getReg() == MI.getOperand(0).getReg()) || 9048 Op2.getReg() == MI.getOperand(0).getReg()) 9049 return None; 9050 else if ((Op1.isReg() && Op1.getReg() != X86::NoRegister && 9051 TRI->regsOverlap(Op1.getReg(), MI.getOperand(0).getReg())) || 9052 (Op2.getReg() != X86::NoRegister && 9053 TRI->regsOverlap(Op2.getReg(), MI.getOperand(0).getReg()))) 9054 return None; 9055 9056 int64_t Coef = MI.getOperand(2).getImm(); 9057 int64_t Offset = MI.getOperand(4).getImm(); 9058 SmallVector<uint64_t, 8> Ops; 9059 9060 if ((Op1.isReg() && Op1.getReg() != X86::NoRegister)) { 9061 Op = &Op1; 9062 } else if (Op1.isFI()) 9063 Op = &Op1; 9064 9065 if (Op && Op->isReg() && Op->getReg() == Op2.getReg() && Coef > 0) { 9066 Ops.push_back(dwarf::DW_OP_constu); 9067 Ops.push_back(Coef + 1); 9068 Ops.push_back(dwarf::DW_OP_mul); 9069 } else { 9070 if (Op && Op2.getReg() != X86::NoRegister) { 9071 int dwarfReg = TRI->getDwarfRegNum(Op2.getReg(), false); 9072 if (dwarfReg < 0) 9073 return None; 9074 else if (dwarfReg < 32) { 9075 Ops.push_back(dwarf::DW_OP_breg0 + dwarfReg); 9076 Ops.push_back(0); 9077 } else { 9078 Ops.push_back(dwarf::DW_OP_bregx); 9079 Ops.push_back(dwarfReg); 9080 Ops.push_back(0); 9081 } 9082 } else if (!Op) { 9083 assert(Op2.getReg() != X86::NoRegister); 9084 Op = &Op2; 9085 } 9086 9087 if (Coef > 1) { 9088 assert(Op2.getReg() != X86::NoRegister); 9089 Ops.push_back(dwarf::DW_OP_constu); 9090 Ops.push_back(Coef); 9091 Ops.push_back(dwarf::DW_OP_mul); 9092 } 9093 9094 if (((Op1.isReg() && Op1.getReg() != X86::NoRegister) || Op1.isFI()) && 9095 Op2.getReg() != X86::NoRegister) { 9096 Ops.push_back(dwarf::DW_OP_plus); 9097 } 9098 } 9099 9100 DIExpression::appendOffset(Ops, Offset); 9101 Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), Ops); 9102 9103 return ParamLoadedValue(*Op, Expr);; 9104 } 9105 case X86::MOV8ri: 9106 case X86::MOV16ri: 9107 // TODO: Handle MOV8ri and MOV16ri. 9108 return None; 9109 case X86::MOV32ri: 9110 case X86::MOV64ri: 9111 case X86::MOV64ri32: 9112 // MOV32ri may be used for producing zero-extended 32-bit immediates in 9113 // 64-bit parameters, so we need to consider super-registers. 9114 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) 9115 return None; 9116 return ParamLoadedValue(MI.getOperand(1), Expr); 9117 case X86::MOV8rr: 9118 case X86::MOV16rr: 9119 case X86::MOV32rr: 9120 case X86::MOV64rr: 9121 return describeMOVrrLoadedValue(MI, Reg, TRI); 9122 case X86::XOR32rr: { 9123 // 64-bit parameters are zero-materialized using XOR32rr, so also consider 9124 // super-registers. 9125 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) 9126 return None; 9127 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) 9128 return ParamLoadedValue(MachineOperand::CreateImm(0), Expr); 9129 return None; 9130 } 9131 case X86::MOVSX64rr32: { 9132 // We may need to describe the lower 32 bits of the MOVSX; for example, in 9133 // cases like this: 9134 // 9135 // $ebx = [...] 9136 // $rdi = MOVSX64rr32 $ebx 9137 // $esi = MOV32rr $edi 9138 if (!TRI->isSubRegisterEq(MI.getOperand(0).getReg(), Reg)) 9139 return None; 9140 9141 Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {}); 9142 9143 // If the described register is the destination register we need to 9144 // sign-extend the source register from 32 bits. The other case we handle 9145 // is when the described register is the 32-bit sub-register of the 9146 // destination register, in case we just need to return the source 9147 // register. 9148 if (Reg == MI.getOperand(0).getReg()) 9149 Expr = DIExpression::appendExt(Expr, 32, 64, true); 9150 else 9151 assert(X86MCRegisterClasses[X86::GR32RegClassID].contains(Reg) && 9152 "Unhandled sub-register case for MOVSX64rr32"); 9153 9154 return ParamLoadedValue(MI.getOperand(1), Expr); 9155 } 9156 default: 9157 assert(!MI.isMoveImmediate() && "Unexpected MoveImm instruction"); 9158 return TargetInstrInfo::describeLoadedValue(MI, Reg); 9159 } 9160 } 9161 9162 /// This is an architecture-specific helper function of reassociateOps. 9163 /// Set special operand attributes for new instructions after reassociation. 9164 void X86InstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1, 9165 MachineInstr &OldMI2, 9166 MachineInstr &NewMI1, 9167 MachineInstr &NewMI2) const { 9168 // Propagate FP flags from the original instructions. 9169 // But clear poison-generating flags because those may not be valid now. 9170 // TODO: There should be a helper function for copying only fast-math-flags. 9171 uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags(); 9172 NewMI1.setFlags(IntersectedFlags); 9173 NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap); 9174 NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap); 9175 NewMI1.clearFlag(MachineInstr::MIFlag::IsExact); 9176 9177 NewMI2.setFlags(IntersectedFlags); 9178 NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap); 9179 NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap); 9180 NewMI2.clearFlag(MachineInstr::MIFlag::IsExact); 9181 9182 // Integer instructions may define an implicit EFLAGS dest register operand. 9183 MachineOperand *OldFlagDef1 = OldMI1.findRegisterDefOperand(X86::EFLAGS); 9184 MachineOperand *OldFlagDef2 = OldMI2.findRegisterDefOperand(X86::EFLAGS); 9185 9186 assert(!OldFlagDef1 == !OldFlagDef2 && 9187 "Unexpected instruction type for reassociation"); 9188 9189 if (!OldFlagDef1 || !OldFlagDef2) 9190 return; 9191 9192 assert(OldFlagDef1->isDead() && OldFlagDef2->isDead() && 9193 "Must have dead EFLAGS operand in reassociable instruction"); 9194 9195 MachineOperand *NewFlagDef1 = NewMI1.findRegisterDefOperand(X86::EFLAGS); 9196 MachineOperand *NewFlagDef2 = NewMI2.findRegisterDefOperand(X86::EFLAGS); 9197 9198 assert(NewFlagDef1 && NewFlagDef2 && 9199 "Unexpected operand in reassociable instruction"); 9200 9201 // Mark the new EFLAGS operands as dead to be helpful to subsequent iterations 9202 // of this pass or other passes. The EFLAGS operands must be dead in these new 9203 // instructions because the EFLAGS operands in the original instructions must 9204 // be dead in order for reassociation to occur. 9205 NewFlagDef1->setIsDead(); 9206 NewFlagDef2->setIsDead(); 9207 } 9208 9209 std::pair<unsigned, unsigned> 9210 X86InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 9211 return std::make_pair(TF, 0u); 9212 } 9213 9214 ArrayRef<std::pair<unsigned, const char *>> 9215 X86InstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 9216 using namespace X86II; 9217 static const std::pair<unsigned, const char *> TargetFlags[] = { 9218 {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"}, 9219 {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"}, 9220 {MO_GOT, "x86-got"}, 9221 {MO_GOTOFF, "x86-gotoff"}, 9222 {MO_GOTPCREL, "x86-gotpcrel"}, 9223 {MO_GOTPCREL_NORELAX, "x86-gotpcrel-norelax"}, 9224 {MO_PLT, "x86-plt"}, 9225 {MO_TLSGD, "x86-tlsgd"}, 9226 {MO_TLSLD, "x86-tlsld"}, 9227 {MO_TLSLDM, "x86-tlsldm"}, 9228 {MO_GOTTPOFF, "x86-gottpoff"}, 9229 {MO_INDNTPOFF, "x86-indntpoff"}, 9230 {MO_TPOFF, "x86-tpoff"}, 9231 {MO_DTPOFF, "x86-dtpoff"}, 9232 {MO_NTPOFF, "x86-ntpoff"}, 9233 {MO_GOTNTPOFF, "x86-gotntpoff"}, 9234 {MO_DLLIMPORT, "x86-dllimport"}, 9235 {MO_DARWIN_NONLAZY, "x86-darwin-nonlazy"}, 9236 {MO_DARWIN_NONLAZY_PIC_BASE, "x86-darwin-nonlazy-pic-base"}, 9237 {MO_TLVP, "x86-tlvp"}, 9238 {MO_TLVP_PIC_BASE, "x86-tlvp-pic-base"}, 9239 {MO_SECREL, "x86-secrel"}, 9240 {MO_COFFSTUB, "x86-coffstub"}}; 9241 return makeArrayRef(TargetFlags); 9242 } 9243 9244 namespace { 9245 /// Create Global Base Reg pass. This initializes the PIC 9246 /// global base register for x86-32. 9247 struct CGBR : public MachineFunctionPass { 9248 static char ID; 9249 CGBR() : MachineFunctionPass(ID) {} 9250 9251 bool runOnMachineFunction(MachineFunction &MF) override { 9252 const X86TargetMachine *TM = 9253 static_cast<const X86TargetMachine *>(&MF.getTarget()); 9254 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 9255 9256 // Don't do anything in the 64-bit small and kernel code models. They use 9257 // RIP-relative addressing for everything. 9258 if (STI.is64Bit() && (TM->getCodeModel() == CodeModel::Small || 9259 TM->getCodeModel() == CodeModel::Kernel)) 9260 return false; 9261 9262 // Only emit a global base reg in PIC mode. 9263 if (!TM->isPositionIndependent()) 9264 return false; 9265 9266 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 9267 Register GlobalBaseReg = X86FI->getGlobalBaseReg(); 9268 9269 // If we didn't need a GlobalBaseReg, don't insert code. 9270 if (GlobalBaseReg == 0) 9271 return false; 9272 9273 // Insert the set of GlobalBaseReg into the first MBB of the function 9274 MachineBasicBlock &FirstMBB = MF.front(); 9275 MachineBasicBlock::iterator MBBI = FirstMBB.begin(); 9276 DebugLoc DL = FirstMBB.findDebugLoc(MBBI); 9277 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 9278 const X86InstrInfo *TII = STI.getInstrInfo(); 9279 9280 Register PC; 9281 if (STI.isPICStyleGOT()) 9282 PC = RegInfo.createVirtualRegister(&X86::GR32RegClass); 9283 else 9284 PC = GlobalBaseReg; 9285 9286 if (STI.is64Bit()) { 9287 if (TM->getCodeModel() == CodeModel::Medium) { 9288 // In the medium code model, use a RIP-relative LEA to materialize the 9289 // GOT. 9290 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC) 9291 .addReg(X86::RIP) 9292 .addImm(0) 9293 .addReg(0) 9294 .addExternalSymbol("_GLOBAL_OFFSET_TABLE_") 9295 .addReg(0); 9296 } else if (TM->getCodeModel() == CodeModel::Large) { 9297 // In the large code model, we are aiming for this code, though the 9298 // register allocation may vary: 9299 // leaq .LN$pb(%rip), %rax 9300 // movq $_GLOBAL_OFFSET_TABLE_ - .LN$pb, %rcx 9301 // addq %rcx, %rax 9302 // RAX now holds address of _GLOBAL_OFFSET_TABLE_. 9303 Register PBReg = RegInfo.createVirtualRegister(&X86::GR64RegClass); 9304 Register GOTReg = RegInfo.createVirtualRegister(&X86::GR64RegClass); 9305 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PBReg) 9306 .addReg(X86::RIP) 9307 .addImm(0) 9308 .addReg(0) 9309 .addSym(MF.getPICBaseSymbol()) 9310 .addReg(0); 9311 std::prev(MBBI)->setPreInstrSymbol(MF, MF.getPICBaseSymbol()); 9312 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOV64ri), GOTReg) 9313 .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", 9314 X86II::MO_PIC_BASE_OFFSET); 9315 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD64rr), PC) 9316 .addReg(PBReg, RegState::Kill) 9317 .addReg(GOTReg, RegState::Kill); 9318 } else { 9319 llvm_unreachable("unexpected code model"); 9320 } 9321 } else { 9322 // Operand of MovePCtoStack is completely ignored by asm printer. It's 9323 // only used in JIT code emission as displacement to pc. 9324 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOVPC32r), PC).addImm(0); 9325 9326 // If we're using vanilla 'GOT' PIC style, we should use relative 9327 // addressing not to pc, but to _GLOBAL_OFFSET_TABLE_ external. 9328 if (STI.isPICStyleGOT()) { 9329 // Generate addl $__GLOBAL_OFFSET_TABLE_ + [.-piclabel], 9330 // %some_register 9331 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD32ri), GlobalBaseReg) 9332 .addReg(PC) 9333 .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", 9334 X86II::MO_GOT_ABSOLUTE_ADDRESS); 9335 } 9336 } 9337 9338 return true; 9339 } 9340 9341 StringRef getPassName() const override { 9342 return "X86 PIC Global Base Reg Initialization"; 9343 } 9344 9345 void getAnalysisUsage(AnalysisUsage &AU) const override { 9346 AU.setPreservesCFG(); 9347 MachineFunctionPass::getAnalysisUsage(AU); 9348 } 9349 }; 9350 } // namespace 9351 9352 char CGBR::ID = 0; 9353 FunctionPass* 9354 llvm::createX86GlobalBaseRegPass() { return new CGBR(); } 9355 9356 namespace { 9357 struct LDTLSCleanup : public MachineFunctionPass { 9358 static char ID; 9359 LDTLSCleanup() : MachineFunctionPass(ID) {} 9360 9361 bool runOnMachineFunction(MachineFunction &MF) override { 9362 if (skipFunction(MF.getFunction())) 9363 return false; 9364 9365 X86MachineFunctionInfo *MFI = MF.getInfo<X86MachineFunctionInfo>(); 9366 if (MFI->getNumLocalDynamicTLSAccesses() < 2) { 9367 // No point folding accesses if there isn't at least two. 9368 return false; 9369 } 9370 9371 MachineDominatorTree *DT = &getAnalysis<MachineDominatorTree>(); 9372 return VisitNode(DT->getRootNode(), 0); 9373 } 9374 9375 // Visit the dominator subtree rooted at Node in pre-order. 9376 // If TLSBaseAddrReg is non-null, then use that to replace any 9377 // TLS_base_addr instructions. Otherwise, create the register 9378 // when the first such instruction is seen, and then use it 9379 // as we encounter more instructions. 9380 bool VisitNode(MachineDomTreeNode *Node, unsigned TLSBaseAddrReg) { 9381 MachineBasicBlock *BB = Node->getBlock(); 9382 bool Changed = false; 9383 9384 // Traverse the current block. 9385 for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; 9386 ++I) { 9387 switch (I->getOpcode()) { 9388 case X86::TLS_base_addr32: 9389 case X86::TLS_base_addr64: 9390 if (TLSBaseAddrReg) 9391 I = ReplaceTLSBaseAddrCall(*I, TLSBaseAddrReg); 9392 else 9393 I = SetRegister(*I, &TLSBaseAddrReg); 9394 Changed = true; 9395 break; 9396 default: 9397 break; 9398 } 9399 } 9400 9401 // Visit the children of this block in the dominator tree. 9402 for (auto I = Node->begin(), E = Node->end(); I != E; ++I) { 9403 Changed |= VisitNode(*I, TLSBaseAddrReg); 9404 } 9405 9406 return Changed; 9407 } 9408 9409 // Replace the TLS_base_addr instruction I with a copy from 9410 // TLSBaseAddrReg, returning the new instruction. 9411 MachineInstr *ReplaceTLSBaseAddrCall(MachineInstr &I, 9412 unsigned TLSBaseAddrReg) { 9413 MachineFunction *MF = I.getParent()->getParent(); 9414 const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); 9415 const bool is64Bit = STI.is64Bit(); 9416 const X86InstrInfo *TII = STI.getInstrInfo(); 9417 9418 // Insert a Copy from TLSBaseAddrReg to RAX/EAX. 9419 MachineInstr *Copy = 9420 BuildMI(*I.getParent(), I, I.getDebugLoc(), 9421 TII->get(TargetOpcode::COPY), is64Bit ? X86::RAX : X86::EAX) 9422 .addReg(TLSBaseAddrReg); 9423 9424 // Erase the TLS_base_addr instruction. 9425 I.eraseFromParent(); 9426 9427 return Copy; 9428 } 9429 9430 // Create a virtual register in *TLSBaseAddrReg, and populate it by 9431 // inserting a copy instruction after I. Returns the new instruction. 9432 MachineInstr *SetRegister(MachineInstr &I, unsigned *TLSBaseAddrReg) { 9433 MachineFunction *MF = I.getParent()->getParent(); 9434 const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); 9435 const bool is64Bit = STI.is64Bit(); 9436 const X86InstrInfo *TII = STI.getInstrInfo(); 9437 9438 // Create a virtual register for the TLS base address. 9439 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 9440 *TLSBaseAddrReg = RegInfo.createVirtualRegister(is64Bit 9441 ? &X86::GR64RegClass 9442 : &X86::GR32RegClass); 9443 9444 // Insert a copy from RAX/EAX to TLSBaseAddrReg. 9445 MachineInstr *Next = I.getNextNode(); 9446 MachineInstr *Copy = 9447 BuildMI(*I.getParent(), Next, I.getDebugLoc(), 9448 TII->get(TargetOpcode::COPY), *TLSBaseAddrReg) 9449 .addReg(is64Bit ? X86::RAX : X86::EAX); 9450 9451 return Copy; 9452 } 9453 9454 StringRef getPassName() const override { 9455 return "Local Dynamic TLS Access Clean-up"; 9456 } 9457 9458 void getAnalysisUsage(AnalysisUsage &AU) const override { 9459 AU.setPreservesCFG(); 9460 AU.addRequired<MachineDominatorTree>(); 9461 MachineFunctionPass::getAnalysisUsage(AU); 9462 } 9463 }; 9464 } 9465 9466 char LDTLSCleanup::ID = 0; 9467 FunctionPass* 9468 llvm::createCleanupLocalDynamicTLSPass() { return new LDTLSCleanup(); } 9469 9470 /// Constants defining how certain sequences should be outlined. 9471 /// 9472 /// \p MachineOutlinerDefault implies that the function is called with a call 9473 /// instruction, and a return must be emitted for the outlined function frame. 9474 /// 9475 /// That is, 9476 /// 9477 /// I1 OUTLINED_FUNCTION: 9478 /// I2 --> call OUTLINED_FUNCTION I1 9479 /// I3 I2 9480 /// I3 9481 /// ret 9482 /// 9483 /// * Call construction overhead: 1 (call instruction) 9484 /// * Frame construction overhead: 1 (return instruction) 9485 /// 9486 /// \p MachineOutlinerTailCall implies that the function is being tail called. 9487 /// A jump is emitted instead of a call, and the return is already present in 9488 /// the outlined sequence. That is, 9489 /// 9490 /// I1 OUTLINED_FUNCTION: 9491 /// I2 --> jmp OUTLINED_FUNCTION I1 9492 /// ret I2 9493 /// ret 9494 /// 9495 /// * Call construction overhead: 1 (jump instruction) 9496 /// * Frame construction overhead: 0 (don't need to return) 9497 /// 9498 enum MachineOutlinerClass { 9499 MachineOutlinerDefault, 9500 MachineOutlinerTailCall 9501 }; 9502 9503 outliner::OutlinedFunction X86InstrInfo::getOutliningCandidateInfo( 9504 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { 9505 unsigned SequenceSize = 9506 std::accumulate(RepeatedSequenceLocs[0].front(), 9507 std::next(RepeatedSequenceLocs[0].back()), 0, 9508 [](unsigned Sum, const MachineInstr &MI) { 9509 // FIXME: x86 doesn't implement getInstSizeInBytes, so 9510 // we can't tell the cost. Just assume each instruction 9511 // is one byte. 9512 if (MI.isDebugInstr() || MI.isKill()) 9513 return Sum; 9514 return Sum + 1; 9515 }); 9516 9517 // We check to see if CFI Instructions are present, and if they are 9518 // we find the number of CFI Instructions in the candidates. 9519 unsigned CFICount = 0; 9520 for (auto &I : make_range(RepeatedSequenceLocs[0].front(), 9521 std::next(RepeatedSequenceLocs[0].back()))) { 9522 if (I.isCFIInstruction()) 9523 CFICount++; 9524 } 9525 9526 // We compare the number of found CFI Instructions to the number of CFI 9527 // instructions in the parent function for each candidate. We must check this 9528 // since if we outline one of the CFI instructions in a function, we have to 9529 // outline them all for correctness. If we do not, the address offsets will be 9530 // incorrect between the two sections of the program. 9531 for (outliner::Candidate &C : RepeatedSequenceLocs) { 9532 std::vector<MCCFIInstruction> CFIInstructions = 9533 C.getMF()->getFrameInstructions(); 9534 9535 if (CFICount > 0 && CFICount != CFIInstructions.size()) 9536 return outliner::OutlinedFunction(); 9537 } 9538 9539 // FIXME: Use real size in bytes for call and ret instructions. 9540 if (RepeatedSequenceLocs[0].back()->isTerminator()) { 9541 for (outliner::Candidate &C : RepeatedSequenceLocs) 9542 C.setCallInfo(MachineOutlinerTailCall, 1); 9543 9544 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 9545 0, // Number of bytes to emit frame. 9546 MachineOutlinerTailCall // Type of frame. 9547 ); 9548 } 9549 9550 if (CFICount > 0) 9551 return outliner::OutlinedFunction(); 9552 9553 for (outliner::Candidate &C : RepeatedSequenceLocs) 9554 C.setCallInfo(MachineOutlinerDefault, 1); 9555 9556 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 1, 9557 MachineOutlinerDefault); 9558 } 9559 9560 bool X86InstrInfo::isFunctionSafeToOutlineFrom(MachineFunction &MF, 9561 bool OutlineFromLinkOnceODRs) const { 9562 const Function &F = MF.getFunction(); 9563 9564 // Does the function use a red zone? If it does, then we can't risk messing 9565 // with the stack. 9566 if (Subtarget.getFrameLowering()->has128ByteRedZone(MF)) { 9567 // It could have a red zone. If it does, then we don't want to touch it. 9568 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 9569 if (!X86FI || X86FI->getUsesRedZone()) 9570 return false; 9571 } 9572 9573 // If we *don't* want to outline from things that could potentially be deduped 9574 // then return false. 9575 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) 9576 return false; 9577 9578 // This function is viable for outlining, so return true. 9579 return true; 9580 } 9581 9582 outliner::InstrType 9583 X86InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, unsigned Flags) const { 9584 MachineInstr &MI = *MIT; 9585 // Don't allow debug values to impact outlining type. 9586 if (MI.isDebugInstr() || MI.isIndirectDebugValue()) 9587 return outliner::InstrType::Invisible; 9588 9589 // At this point, KILL instructions don't really tell us much so we can go 9590 // ahead and skip over them. 9591 if (MI.isKill()) 9592 return outliner::InstrType::Invisible; 9593 9594 // Is this a tail call? If yes, we can outline as a tail call. 9595 if (isTailCall(MI)) 9596 return outliner::InstrType::Legal; 9597 9598 // Is this the terminator of a basic block? 9599 if (MI.isTerminator() || MI.isReturn()) { 9600 9601 // Does its parent have any successors in its MachineFunction? 9602 if (MI.getParent()->succ_empty()) 9603 return outliner::InstrType::Legal; 9604 9605 // It does, so we can't tail call it. 9606 return outliner::InstrType::Illegal; 9607 } 9608 9609 // Don't outline anything that modifies or reads from the stack pointer. 9610 // 9611 // FIXME: There are instructions which are being manually built without 9612 // explicit uses/defs so we also have to check the MCInstrDesc. We should be 9613 // able to remove the extra checks once those are fixed up. For example, 9614 // sometimes we might get something like %rax = POP64r 1. This won't be 9615 // caught by modifiesRegister or readsRegister even though the instruction 9616 // really ought to be formed so that modifiesRegister/readsRegister would 9617 // catch it. 9618 if (MI.modifiesRegister(X86::RSP, &RI) || MI.readsRegister(X86::RSP, &RI) || 9619 MI.getDesc().hasImplicitUseOfPhysReg(X86::RSP) || 9620 MI.getDesc().hasImplicitDefOfPhysReg(X86::RSP)) 9621 return outliner::InstrType::Illegal; 9622 9623 // Outlined calls change the instruction pointer, so don't read from it. 9624 if (MI.readsRegister(X86::RIP, &RI) || 9625 MI.getDesc().hasImplicitUseOfPhysReg(X86::RIP) || 9626 MI.getDesc().hasImplicitDefOfPhysReg(X86::RIP)) 9627 return outliner::InstrType::Illegal; 9628 9629 // Positions can't safely be outlined. 9630 if (MI.isPosition()) 9631 return outliner::InstrType::Illegal; 9632 9633 // Make sure none of the operands of this instruction do anything tricky. 9634 for (const MachineOperand &MOP : MI.operands()) 9635 if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() || 9636 MOP.isTargetIndex()) 9637 return outliner::InstrType::Illegal; 9638 9639 return outliner::InstrType::Legal; 9640 } 9641 9642 void X86InstrInfo::buildOutlinedFrame(MachineBasicBlock &MBB, 9643 MachineFunction &MF, 9644 const outliner::OutlinedFunction &OF) 9645 const { 9646 // If we're a tail call, we already have a return, so don't do anything. 9647 if (OF.FrameConstructionID == MachineOutlinerTailCall) 9648 return; 9649 9650 // We're a normal call, so our sequence doesn't have a return instruction. 9651 // Add it in. 9652 MachineInstr *retq = BuildMI(MF, DebugLoc(), get(X86::RET64)); 9653 MBB.insert(MBB.end(), retq); 9654 } 9655 9656 MachineBasicBlock::iterator 9657 X86InstrInfo::insertOutlinedCall(Module &M, MachineBasicBlock &MBB, 9658 MachineBasicBlock::iterator &It, 9659 MachineFunction &MF, 9660 outliner::Candidate &C) const { 9661 // Is it a tail call? 9662 if (C.CallConstructionID == MachineOutlinerTailCall) { 9663 // Yes, just insert a JMP. 9664 It = MBB.insert(It, 9665 BuildMI(MF, DebugLoc(), get(X86::TAILJMPd64)) 9666 .addGlobalAddress(M.getNamedValue(MF.getName()))); 9667 } else { 9668 // No, insert a call. 9669 It = MBB.insert(It, 9670 BuildMI(MF, DebugLoc(), get(X86::CALL64pcrel32)) 9671 .addGlobalAddress(M.getNamedValue(MF.getName()))); 9672 } 9673 9674 return It; 9675 } 9676 9677 #define GET_INSTRINFO_HELPERS 9678 #include "X86GenInstrInfo.inc" 9679