1 //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the X86 implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "X86InstrInfo.h" 14 #include "X86.h" 15 #include "X86InstrBuilder.h" 16 #include "X86InstrFoldTables.h" 17 #include "X86MachineFunctionInfo.h" 18 #include "X86Subtarget.h" 19 #include "X86TargetMachine.h" 20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/ADT/Sequence.h" 22 #include "llvm/CodeGen/LiveIntervals.h" 23 #include "llvm/CodeGen/LivePhysRegs.h" 24 #include "llvm/CodeGen/LiveVariables.h" 25 #include "llvm/CodeGen/MachineCombinerPattern.h" 26 #include "llvm/CodeGen/MachineConstantPool.h" 27 #include "llvm/CodeGen/MachineDominators.h" 28 #include "llvm/CodeGen/MachineFrameInfo.h" 29 #include "llvm/CodeGen/MachineInstr.h" 30 #include "llvm/CodeGen/MachineInstrBuilder.h" 31 #include "llvm/CodeGen/MachineModuleInfo.h" 32 #include "llvm/CodeGen/MachineOperand.h" 33 #include "llvm/CodeGen/MachineRegisterInfo.h" 34 #include "llvm/CodeGen/StackMaps.h" 35 #include "llvm/IR/DebugInfoMetadata.h" 36 #include "llvm/IR/DerivedTypes.h" 37 #include "llvm/IR/Function.h" 38 #include "llvm/IR/InstrTypes.h" 39 #include "llvm/MC/MCAsmInfo.h" 40 #include "llvm/MC/MCExpr.h" 41 #include "llvm/MC/MCInst.h" 42 #include "llvm/Support/CommandLine.h" 43 #include "llvm/Support/Debug.h" 44 #include "llvm/Support/ErrorHandling.h" 45 #include "llvm/Support/raw_ostream.h" 46 #include "llvm/Target/TargetOptions.h" 47 #include <optional> 48 49 using namespace llvm; 50 51 #define DEBUG_TYPE "x86-instr-info" 52 53 #define GET_INSTRINFO_CTOR_DTOR 54 #include "X86GenInstrInfo.inc" 55 56 static cl::opt<bool> 57 NoFusing("disable-spill-fusing", 58 cl::desc("Disable fusing of spill code into instructions"), 59 cl::Hidden); 60 static cl::opt<bool> 61 PrintFailedFusing("print-failed-fuse-candidates", 62 cl::desc("Print instructions that the allocator wants to" 63 " fuse, but the X86 backend currently can't"), 64 cl::Hidden); 65 static cl::opt<bool> 66 ReMatPICStubLoad("remat-pic-stub-load", 67 cl::desc("Re-materialize load from stub in PIC mode"), 68 cl::init(false), cl::Hidden); 69 static cl::opt<unsigned> 70 PartialRegUpdateClearance("partial-reg-update-clearance", 71 cl::desc("Clearance between two register writes " 72 "for inserting XOR to avoid partial " 73 "register update"), 74 cl::init(64), cl::Hidden); 75 static cl::opt<unsigned> UndefRegClearance( 76 "undef-reg-clearance", 77 cl::desc("How many idle instructions we would like before " 78 "certain undef register reads"), 79 cl::init(128), cl::Hidden); 80 81 // Pin the vtable to this file. 82 void X86InstrInfo::anchor() {} 83 84 X86InstrInfo::X86InstrInfo(X86Subtarget &STI) 85 : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64 86 : X86::ADJCALLSTACKDOWN32), 87 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64 88 : X86::ADJCALLSTACKUP32), 89 X86::CATCHRET, (STI.is64Bit() ? X86::RET64 : X86::RET32)), 90 Subtarget(STI), RI(STI.getTargetTriple()) {} 91 92 const TargetRegisterClass * 93 X86InstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum, 94 const TargetRegisterInfo *TRI, 95 const MachineFunction &MF) const { 96 auto *RC = TargetInstrInfo::getRegClass(MCID, OpNum, TRI, MF); 97 // If the target does not have egpr, then r16-r31 will be resereved for all 98 // instructions. 99 if (!RC || !Subtarget.hasEGPR()) 100 return RC; 101 102 if (X86II::canUseApxExtendedReg(MCID)) 103 return RC; 104 105 switch (RC->getID()) { 106 default: 107 return RC; 108 case X86::GR8RegClassID: 109 return &X86::GR8_NOREX2RegClass; 110 case X86::GR16RegClassID: 111 return &X86::GR16_NOREX2RegClass; 112 case X86::GR32RegClassID: 113 return &X86::GR32_NOREX2RegClass; 114 case X86::GR64RegClassID: 115 return &X86::GR64_NOREX2RegClass; 116 case X86::GR32_NOSPRegClassID: 117 return &X86::GR32_NOREX2_NOSPRegClass; 118 case X86::GR64_NOSPRegClassID: 119 return &X86::GR64_NOREX2_NOSPRegClass; 120 } 121 } 122 123 bool X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI, 124 Register &SrcReg, Register &DstReg, 125 unsigned &SubIdx) const { 126 switch (MI.getOpcode()) { 127 default: 128 break; 129 case X86::MOVSX16rr8: 130 case X86::MOVZX16rr8: 131 case X86::MOVSX32rr8: 132 case X86::MOVZX32rr8: 133 case X86::MOVSX64rr8: 134 if (!Subtarget.is64Bit()) 135 // It's not always legal to reference the low 8-bit of the larger 136 // register in 32-bit mode. 137 return false; 138 [[fallthrough]]; 139 case X86::MOVSX32rr16: 140 case X86::MOVZX32rr16: 141 case X86::MOVSX64rr16: 142 case X86::MOVSX64rr32: { 143 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) 144 // Be conservative. 145 return false; 146 SrcReg = MI.getOperand(1).getReg(); 147 DstReg = MI.getOperand(0).getReg(); 148 switch (MI.getOpcode()) { 149 default: 150 llvm_unreachable("Unreachable!"); 151 case X86::MOVSX16rr8: 152 case X86::MOVZX16rr8: 153 case X86::MOVSX32rr8: 154 case X86::MOVZX32rr8: 155 case X86::MOVSX64rr8: 156 SubIdx = X86::sub_8bit; 157 break; 158 case X86::MOVSX32rr16: 159 case X86::MOVZX32rr16: 160 case X86::MOVSX64rr16: 161 SubIdx = X86::sub_16bit; 162 break; 163 case X86::MOVSX64rr32: 164 SubIdx = X86::sub_32bit; 165 break; 166 } 167 return true; 168 } 169 } 170 return false; 171 } 172 173 bool X86InstrInfo::isDataInvariant(MachineInstr &MI) { 174 if (MI.mayLoad() || MI.mayStore()) 175 return false; 176 177 // Some target-independent operations that trivially lower to data-invariant 178 // instructions. 179 if (MI.isCopyLike() || MI.isInsertSubreg()) 180 return true; 181 182 unsigned Opcode = MI.getOpcode(); 183 using namespace X86; 184 // On x86 it is believed that imul is constant time w.r.t. the loaded data. 185 // However, they set flags and are perhaps the most surprisingly constant 186 // time operations so we call them out here separately. 187 if (isIMUL(Opcode)) 188 return true; 189 // Bit scanning and counting instructions that are somewhat surprisingly 190 // constant time as they scan across bits and do other fairly complex 191 // operations like popcnt, but are believed to be constant time on x86. 192 // However, these set flags. 193 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) || 194 isTZCNT(Opcode)) 195 return true; 196 // Bit manipulation instructions are effectively combinations of basic 197 // arithmetic ops, and should still execute in constant time. These also 198 // set flags. 199 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) || 200 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) || 201 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) || 202 isTZMSK(Opcode)) 203 return true; 204 // Bit extracting and clearing instructions should execute in constant time, 205 // and set flags. 206 if (isBEXTR(Opcode) || isBZHI(Opcode)) 207 return true; 208 // Shift and rotate. 209 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) || 210 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode)) 211 return true; 212 // Basic arithmetic is constant time on the input but does set flags. 213 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) || 214 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode)) 215 return true; 216 // Arithmetic with just 32-bit and 64-bit variants and no immediates. 217 if (isANDN(Opcode)) 218 return true; 219 // Unary arithmetic operations. 220 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode)) 221 return true; 222 // Unlike other arithmetic, NOT doesn't set EFLAGS. 223 if (isNOT(Opcode)) 224 return true; 225 // Various move instructions used to zero or sign extend things. Note that we 226 // intentionally don't support the _NOREX variants as we can't handle that 227 // register constraint anyways. 228 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode)) 229 return true; 230 // Arithmetic instructions that are both constant time and don't set flags. 231 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode)) 232 return true; 233 // LEA doesn't actually access memory, and its arithmetic is constant time. 234 if (isLEA(Opcode)) 235 return true; 236 // By default, assume that the instruction is not data invariant. 237 return false; 238 } 239 240 bool X86InstrInfo::isDataInvariantLoad(MachineInstr &MI) { 241 switch (MI.getOpcode()) { 242 default: 243 // By default, assume that the load will immediately leak. 244 return false; 245 246 // On x86 it is believed that imul is constant time w.r.t. the loaded data. 247 // However, they set flags and are perhaps the most surprisingly constant 248 // time operations so we call them out here separately. 249 case X86::IMUL16rm: 250 case X86::IMUL16rmi: 251 case X86::IMUL32rm: 252 case X86::IMUL32rmi: 253 case X86::IMUL64rm: 254 case X86::IMUL64rmi32: 255 256 // Bit scanning and counting instructions that are somewhat surprisingly 257 // constant time as they scan across bits and do other fairly complex 258 // operations like popcnt, but are believed to be constant time on x86. 259 // However, these set flags. 260 case X86::BSF16rm: 261 case X86::BSF32rm: 262 case X86::BSF64rm: 263 case X86::BSR16rm: 264 case X86::BSR32rm: 265 case X86::BSR64rm: 266 case X86::LZCNT16rm: 267 case X86::LZCNT32rm: 268 case X86::LZCNT64rm: 269 case X86::POPCNT16rm: 270 case X86::POPCNT32rm: 271 case X86::POPCNT64rm: 272 case X86::TZCNT16rm: 273 case X86::TZCNT32rm: 274 case X86::TZCNT64rm: 275 276 // Bit manipulation instructions are effectively combinations of basic 277 // arithmetic ops, and should still execute in constant time. These also 278 // set flags. 279 case X86::BLCFILL32rm: 280 case X86::BLCFILL64rm: 281 case X86::BLCI32rm: 282 case X86::BLCI64rm: 283 case X86::BLCIC32rm: 284 case X86::BLCIC64rm: 285 case X86::BLCMSK32rm: 286 case X86::BLCMSK64rm: 287 case X86::BLCS32rm: 288 case X86::BLCS64rm: 289 case X86::BLSFILL32rm: 290 case X86::BLSFILL64rm: 291 case X86::BLSI32rm: 292 case X86::BLSI64rm: 293 case X86::BLSIC32rm: 294 case X86::BLSIC64rm: 295 case X86::BLSMSK32rm: 296 case X86::BLSMSK64rm: 297 case X86::BLSR32rm: 298 case X86::BLSR64rm: 299 case X86::TZMSK32rm: 300 case X86::TZMSK64rm: 301 302 // Bit extracting and clearing instructions should execute in constant time, 303 // and set flags. 304 case X86::BEXTR32rm: 305 case X86::BEXTR64rm: 306 case X86::BEXTRI32mi: 307 case X86::BEXTRI64mi: 308 case X86::BZHI32rm: 309 case X86::BZHI64rm: 310 311 // Basic arithmetic is constant time on the input but does set flags. 312 case X86::ADC8rm: 313 case X86::ADC16rm: 314 case X86::ADC32rm: 315 case X86::ADC64rm: 316 case X86::ADD8rm: 317 case X86::ADD16rm: 318 case X86::ADD32rm: 319 case X86::ADD64rm: 320 case X86::AND8rm: 321 case X86::AND16rm: 322 case X86::AND32rm: 323 case X86::AND64rm: 324 case X86::ANDN32rm: 325 case X86::ANDN64rm: 326 case X86::OR8rm: 327 case X86::OR16rm: 328 case X86::OR32rm: 329 case X86::OR64rm: 330 case X86::SBB8rm: 331 case X86::SBB16rm: 332 case X86::SBB32rm: 333 case X86::SBB64rm: 334 case X86::SUB8rm: 335 case X86::SUB16rm: 336 case X86::SUB32rm: 337 case X86::SUB64rm: 338 case X86::XOR8rm: 339 case X86::XOR16rm: 340 case X86::XOR32rm: 341 case X86::XOR64rm: 342 343 // Integer multiply w/o affecting flags is still believed to be constant 344 // time on x86. Called out separately as this is among the most surprising 345 // instructions to exhibit that behavior. 346 case X86::MULX32rm: 347 case X86::MULX64rm: 348 349 // Arithmetic instructions that are both constant time and don't set flags. 350 case X86::RORX32mi: 351 case X86::RORX64mi: 352 case X86::SARX32rm: 353 case X86::SARX64rm: 354 case X86::SHLX32rm: 355 case X86::SHLX64rm: 356 case X86::SHRX32rm: 357 case X86::SHRX64rm: 358 359 // Conversions are believed to be constant time and don't set flags. 360 case X86::CVTTSD2SI64rm: 361 case X86::VCVTTSD2SI64rm: 362 case X86::VCVTTSD2SI64Zrm: 363 case X86::CVTTSD2SIrm: 364 case X86::VCVTTSD2SIrm: 365 case X86::VCVTTSD2SIZrm: 366 case X86::CVTTSS2SI64rm: 367 case X86::VCVTTSS2SI64rm: 368 case X86::VCVTTSS2SI64Zrm: 369 case X86::CVTTSS2SIrm: 370 case X86::VCVTTSS2SIrm: 371 case X86::VCVTTSS2SIZrm: 372 case X86::CVTSI2SDrm: 373 case X86::VCVTSI2SDrm: 374 case X86::VCVTSI2SDZrm: 375 case X86::CVTSI2SSrm: 376 case X86::VCVTSI2SSrm: 377 case X86::VCVTSI2SSZrm: 378 case X86::CVTSI642SDrm: 379 case X86::VCVTSI642SDrm: 380 case X86::VCVTSI642SDZrm: 381 case X86::CVTSI642SSrm: 382 case X86::VCVTSI642SSrm: 383 case X86::VCVTSI642SSZrm: 384 case X86::CVTSS2SDrm: 385 case X86::VCVTSS2SDrm: 386 case X86::VCVTSS2SDZrm: 387 case X86::CVTSD2SSrm: 388 case X86::VCVTSD2SSrm: 389 case X86::VCVTSD2SSZrm: 390 // AVX512 added unsigned integer conversions. 391 case X86::VCVTTSD2USI64Zrm: 392 case X86::VCVTTSD2USIZrm: 393 case X86::VCVTTSS2USI64Zrm: 394 case X86::VCVTTSS2USIZrm: 395 case X86::VCVTUSI2SDZrm: 396 case X86::VCVTUSI642SDZrm: 397 case X86::VCVTUSI2SSZrm: 398 case X86::VCVTUSI642SSZrm: 399 400 // Loads to register don't set flags. 401 case X86::MOV8rm: 402 case X86::MOV8rm_NOREX: 403 case X86::MOV16rm: 404 case X86::MOV32rm: 405 case X86::MOV64rm: 406 case X86::MOVSX16rm8: 407 case X86::MOVSX32rm16: 408 case X86::MOVSX32rm8: 409 case X86::MOVSX32rm8_NOREX: 410 case X86::MOVSX64rm16: 411 case X86::MOVSX64rm32: 412 case X86::MOVSX64rm8: 413 case X86::MOVZX16rm8: 414 case X86::MOVZX32rm16: 415 case X86::MOVZX32rm8: 416 case X86::MOVZX32rm8_NOREX: 417 case X86::MOVZX64rm16: 418 case X86::MOVZX64rm8: 419 return true; 420 } 421 } 422 423 int X86InstrInfo::getSPAdjust(const MachineInstr &MI) const { 424 const MachineFunction *MF = MI.getParent()->getParent(); 425 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); 426 427 if (isFrameInstr(MI)) { 428 int SPAdj = alignTo(getFrameSize(MI), TFI->getStackAlign()); 429 SPAdj -= getFrameAdjustment(MI); 430 if (!isFrameSetup(MI)) 431 SPAdj = -SPAdj; 432 return SPAdj; 433 } 434 435 // To know whether a call adjusts the stack, we need information 436 // that is bound to the following ADJCALLSTACKUP pseudo. 437 // Look for the next ADJCALLSTACKUP that follows the call. 438 if (MI.isCall()) { 439 const MachineBasicBlock *MBB = MI.getParent(); 440 auto I = ++MachineBasicBlock::const_iterator(MI); 441 for (auto E = MBB->end(); I != E; ++I) { 442 if (I->getOpcode() == getCallFrameDestroyOpcode() || I->isCall()) 443 break; 444 } 445 446 // If we could not find a frame destroy opcode, then it has already 447 // been simplified, so we don't care. 448 if (I->getOpcode() != getCallFrameDestroyOpcode()) 449 return 0; 450 451 return -(I->getOperand(1).getImm()); 452 } 453 454 // Currently handle only PUSHes we can reasonably expect to see 455 // in call sequences 456 switch (MI.getOpcode()) { 457 default: 458 return 0; 459 case X86::PUSH32r: 460 case X86::PUSH32rmm: 461 case X86::PUSH32rmr: 462 case X86::PUSH32i: 463 return 4; 464 case X86::PUSH64r: 465 case X86::PUSH64rmm: 466 case X86::PUSH64rmr: 467 case X86::PUSH64i32: 468 return 8; 469 } 470 } 471 472 /// Return true and the FrameIndex if the specified 473 /// operand and follow operands form a reference to the stack frame. 474 bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op, 475 int &FrameIndex) const { 476 if (MI.getOperand(Op + X86::AddrBaseReg).isFI() && 477 MI.getOperand(Op + X86::AddrScaleAmt).isImm() && 478 MI.getOperand(Op + X86::AddrIndexReg).isReg() && 479 MI.getOperand(Op + X86::AddrDisp).isImm() && 480 MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 && 481 MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 && 482 MI.getOperand(Op + X86::AddrDisp).getImm() == 0) { 483 FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex(); 484 return true; 485 } 486 return false; 487 } 488 489 static bool isFrameLoadOpcode(int Opcode, unsigned &MemBytes) { 490 switch (Opcode) { 491 default: 492 return false; 493 case X86::MOV8rm: 494 case X86::KMOVBkm: 495 case X86::KMOVBkm_EVEX: 496 MemBytes = 1; 497 return true; 498 case X86::MOV16rm: 499 case X86::KMOVWkm: 500 case X86::KMOVWkm_EVEX: 501 case X86::VMOVSHZrm: 502 case X86::VMOVSHZrm_alt: 503 MemBytes = 2; 504 return true; 505 case X86::MOV32rm: 506 case X86::MOVSSrm: 507 case X86::MOVSSrm_alt: 508 case X86::VMOVSSrm: 509 case X86::VMOVSSrm_alt: 510 case X86::VMOVSSZrm: 511 case X86::VMOVSSZrm_alt: 512 case X86::KMOVDkm: 513 case X86::KMOVDkm_EVEX: 514 MemBytes = 4; 515 return true; 516 case X86::MOV64rm: 517 case X86::LD_Fp64m: 518 case X86::MOVSDrm: 519 case X86::MOVSDrm_alt: 520 case X86::VMOVSDrm: 521 case X86::VMOVSDrm_alt: 522 case X86::VMOVSDZrm: 523 case X86::VMOVSDZrm_alt: 524 case X86::MMX_MOVD64rm: 525 case X86::MMX_MOVQ64rm: 526 case X86::KMOVQkm: 527 case X86::KMOVQkm_EVEX: 528 MemBytes = 8; 529 return true; 530 case X86::MOVAPSrm: 531 case X86::MOVUPSrm: 532 case X86::MOVAPDrm: 533 case X86::MOVUPDrm: 534 case X86::MOVDQArm: 535 case X86::MOVDQUrm: 536 case X86::VMOVAPSrm: 537 case X86::VMOVUPSrm: 538 case X86::VMOVAPDrm: 539 case X86::VMOVUPDrm: 540 case X86::VMOVDQArm: 541 case X86::VMOVDQUrm: 542 case X86::VMOVAPSZ128rm: 543 case X86::VMOVUPSZ128rm: 544 case X86::VMOVAPSZ128rm_NOVLX: 545 case X86::VMOVUPSZ128rm_NOVLX: 546 case X86::VMOVAPDZ128rm: 547 case X86::VMOVUPDZ128rm: 548 case X86::VMOVDQU8Z128rm: 549 case X86::VMOVDQU16Z128rm: 550 case X86::VMOVDQA32Z128rm: 551 case X86::VMOVDQU32Z128rm: 552 case X86::VMOVDQA64Z128rm: 553 case X86::VMOVDQU64Z128rm: 554 MemBytes = 16; 555 return true; 556 case X86::VMOVAPSYrm: 557 case X86::VMOVUPSYrm: 558 case X86::VMOVAPDYrm: 559 case X86::VMOVUPDYrm: 560 case X86::VMOVDQAYrm: 561 case X86::VMOVDQUYrm: 562 case X86::VMOVAPSZ256rm: 563 case X86::VMOVUPSZ256rm: 564 case X86::VMOVAPSZ256rm_NOVLX: 565 case X86::VMOVUPSZ256rm_NOVLX: 566 case X86::VMOVAPDZ256rm: 567 case X86::VMOVUPDZ256rm: 568 case X86::VMOVDQU8Z256rm: 569 case X86::VMOVDQU16Z256rm: 570 case X86::VMOVDQA32Z256rm: 571 case X86::VMOVDQU32Z256rm: 572 case X86::VMOVDQA64Z256rm: 573 case X86::VMOVDQU64Z256rm: 574 MemBytes = 32; 575 return true; 576 case X86::VMOVAPSZrm: 577 case X86::VMOVUPSZrm: 578 case X86::VMOVAPDZrm: 579 case X86::VMOVUPDZrm: 580 case X86::VMOVDQU8Zrm: 581 case X86::VMOVDQU16Zrm: 582 case X86::VMOVDQA32Zrm: 583 case X86::VMOVDQU32Zrm: 584 case X86::VMOVDQA64Zrm: 585 case X86::VMOVDQU64Zrm: 586 MemBytes = 64; 587 return true; 588 } 589 } 590 591 static bool isFrameStoreOpcode(int Opcode, unsigned &MemBytes) { 592 switch (Opcode) { 593 default: 594 return false; 595 case X86::MOV8mr: 596 case X86::KMOVBmk: 597 case X86::KMOVBmk_EVEX: 598 MemBytes = 1; 599 return true; 600 case X86::MOV16mr: 601 case X86::KMOVWmk: 602 case X86::KMOVWmk_EVEX: 603 case X86::VMOVSHZmr: 604 MemBytes = 2; 605 return true; 606 case X86::MOV32mr: 607 case X86::MOVSSmr: 608 case X86::VMOVSSmr: 609 case X86::VMOVSSZmr: 610 case X86::KMOVDmk: 611 case X86::KMOVDmk_EVEX: 612 MemBytes = 4; 613 return true; 614 case X86::MOV64mr: 615 case X86::ST_FpP64m: 616 case X86::MOVSDmr: 617 case X86::VMOVSDmr: 618 case X86::VMOVSDZmr: 619 case X86::MMX_MOVD64mr: 620 case X86::MMX_MOVQ64mr: 621 case X86::MMX_MOVNTQmr: 622 case X86::KMOVQmk: 623 case X86::KMOVQmk_EVEX: 624 MemBytes = 8; 625 return true; 626 case X86::MOVAPSmr: 627 case X86::MOVUPSmr: 628 case X86::MOVAPDmr: 629 case X86::MOVUPDmr: 630 case X86::MOVDQAmr: 631 case X86::MOVDQUmr: 632 case X86::VMOVAPSmr: 633 case X86::VMOVUPSmr: 634 case X86::VMOVAPDmr: 635 case X86::VMOVUPDmr: 636 case X86::VMOVDQAmr: 637 case X86::VMOVDQUmr: 638 case X86::VMOVUPSZ128mr: 639 case X86::VMOVAPSZ128mr: 640 case X86::VMOVUPSZ128mr_NOVLX: 641 case X86::VMOVAPSZ128mr_NOVLX: 642 case X86::VMOVUPDZ128mr: 643 case X86::VMOVAPDZ128mr: 644 case X86::VMOVDQA32Z128mr: 645 case X86::VMOVDQU32Z128mr: 646 case X86::VMOVDQA64Z128mr: 647 case X86::VMOVDQU64Z128mr: 648 case X86::VMOVDQU8Z128mr: 649 case X86::VMOVDQU16Z128mr: 650 MemBytes = 16; 651 return true; 652 case X86::VMOVUPSYmr: 653 case X86::VMOVAPSYmr: 654 case X86::VMOVUPDYmr: 655 case X86::VMOVAPDYmr: 656 case X86::VMOVDQUYmr: 657 case X86::VMOVDQAYmr: 658 case X86::VMOVUPSZ256mr: 659 case X86::VMOVAPSZ256mr: 660 case X86::VMOVUPSZ256mr_NOVLX: 661 case X86::VMOVAPSZ256mr_NOVLX: 662 case X86::VMOVUPDZ256mr: 663 case X86::VMOVAPDZ256mr: 664 case X86::VMOVDQU8Z256mr: 665 case X86::VMOVDQU16Z256mr: 666 case X86::VMOVDQA32Z256mr: 667 case X86::VMOVDQU32Z256mr: 668 case X86::VMOVDQA64Z256mr: 669 case X86::VMOVDQU64Z256mr: 670 MemBytes = 32; 671 return true; 672 case X86::VMOVUPSZmr: 673 case X86::VMOVAPSZmr: 674 case X86::VMOVUPDZmr: 675 case X86::VMOVAPDZmr: 676 case X86::VMOVDQU8Zmr: 677 case X86::VMOVDQU16Zmr: 678 case X86::VMOVDQA32Zmr: 679 case X86::VMOVDQU32Zmr: 680 case X86::VMOVDQA64Zmr: 681 case X86::VMOVDQU64Zmr: 682 MemBytes = 64; 683 return true; 684 } 685 return false; 686 } 687 688 unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 689 int &FrameIndex) const { 690 unsigned Dummy; 691 return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy); 692 } 693 694 unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 695 int &FrameIndex, 696 unsigned &MemBytes) const { 697 if (isFrameLoadOpcode(MI.getOpcode(), MemBytes)) 698 if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex)) 699 return MI.getOperand(0).getReg(); 700 return 0; 701 } 702 703 unsigned X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI, 704 int &FrameIndex) const { 705 unsigned Dummy; 706 if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) { 707 unsigned Reg; 708 if ((Reg = isLoadFromStackSlot(MI, FrameIndex))) 709 return Reg; 710 // Check for post-frame index elimination operations 711 SmallVector<const MachineMemOperand *, 1> Accesses; 712 if (hasLoadFromStackSlot(MI, Accesses)) { 713 FrameIndex = 714 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) 715 ->getFrameIndex(); 716 return MI.getOperand(0).getReg(); 717 } 718 } 719 return 0; 720 } 721 722 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, 723 int &FrameIndex) const { 724 unsigned Dummy; 725 return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy); 726 } 727 728 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, 729 int &FrameIndex, 730 unsigned &MemBytes) const { 731 if (isFrameStoreOpcode(MI.getOpcode(), MemBytes)) 732 if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 && 733 isFrameOperand(MI, 0, FrameIndex)) 734 return MI.getOperand(X86::AddrNumOperands).getReg(); 735 return 0; 736 } 737 738 unsigned X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI, 739 int &FrameIndex) const { 740 unsigned Dummy; 741 if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) { 742 unsigned Reg; 743 if ((Reg = isStoreToStackSlot(MI, FrameIndex))) 744 return Reg; 745 // Check for post-frame index elimination operations 746 SmallVector<const MachineMemOperand *, 1> Accesses; 747 if (hasStoreToStackSlot(MI, Accesses)) { 748 FrameIndex = 749 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) 750 ->getFrameIndex(); 751 return MI.getOperand(X86::AddrNumOperands).getReg(); 752 } 753 } 754 return 0; 755 } 756 757 /// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r. 758 static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) { 759 // Don't waste compile time scanning use-def chains of physregs. 760 if (!BaseReg.isVirtual()) 761 return false; 762 bool isPICBase = false; 763 for (MachineRegisterInfo::def_instr_iterator I = MRI.def_instr_begin(BaseReg), 764 E = MRI.def_instr_end(); 765 I != E; ++I) { 766 MachineInstr *DefMI = &*I; 767 if (DefMI->getOpcode() != X86::MOVPC32r) 768 return false; 769 assert(!isPICBase && "More than one PIC base?"); 770 isPICBase = true; 771 } 772 return isPICBase; 773 } 774 775 bool X86InstrInfo::isReallyTriviallyReMaterializable( 776 const MachineInstr &MI) const { 777 switch (MI.getOpcode()) { 778 default: 779 // This function should only be called for opcodes with the ReMaterializable 780 // flag set. 781 llvm_unreachable("Unknown rematerializable operation!"); 782 break; 783 case X86::IMPLICIT_DEF: 784 // Defer to generic logic. 785 break; 786 case X86::LOAD_STACK_GUARD: 787 case X86::LD_Fp032: 788 case X86::LD_Fp064: 789 case X86::LD_Fp080: 790 case X86::LD_Fp132: 791 case X86::LD_Fp164: 792 case X86::LD_Fp180: 793 case X86::AVX1_SETALLONES: 794 case X86::AVX2_SETALLONES: 795 case X86::AVX512_128_SET0: 796 case X86::AVX512_256_SET0: 797 case X86::AVX512_512_SET0: 798 case X86::AVX512_512_SETALLONES: 799 case X86::AVX512_FsFLD0SD: 800 case X86::AVX512_FsFLD0SH: 801 case X86::AVX512_FsFLD0SS: 802 case X86::AVX512_FsFLD0F128: 803 case X86::AVX_SET0: 804 case X86::FsFLD0SD: 805 case X86::FsFLD0SS: 806 case X86::FsFLD0SH: 807 case X86::FsFLD0F128: 808 case X86::KSET0D: 809 case X86::KSET0Q: 810 case X86::KSET0W: 811 case X86::KSET1D: 812 case X86::KSET1Q: 813 case X86::KSET1W: 814 case X86::MMX_SET0: 815 case X86::MOV32ImmSExti8: 816 case X86::MOV32r0: 817 case X86::MOV32r1: 818 case X86::MOV32r_1: 819 case X86::MOV32ri64: 820 case X86::MOV64ImmSExti8: 821 case X86::V_SET0: 822 case X86::V_SETALLONES: 823 case X86::MOV16ri: 824 case X86::MOV32ri: 825 case X86::MOV64ri: 826 case X86::MOV64ri32: 827 case X86::MOV8ri: 828 case X86::PTILEZEROV: 829 return true; 830 831 case X86::MOV8rm: 832 case X86::MOV8rm_NOREX: 833 case X86::MOV16rm: 834 case X86::MOV32rm: 835 case X86::MOV64rm: 836 case X86::MOVSSrm: 837 case X86::MOVSSrm_alt: 838 case X86::MOVSDrm: 839 case X86::MOVSDrm_alt: 840 case X86::MOVAPSrm: 841 case X86::MOVUPSrm: 842 case X86::MOVAPDrm: 843 case X86::MOVUPDrm: 844 case X86::MOVDQArm: 845 case X86::MOVDQUrm: 846 case X86::VMOVSSrm: 847 case X86::VMOVSSrm_alt: 848 case X86::VMOVSDrm: 849 case X86::VMOVSDrm_alt: 850 case X86::VMOVAPSrm: 851 case X86::VMOVUPSrm: 852 case X86::VMOVAPDrm: 853 case X86::VMOVUPDrm: 854 case X86::VMOVDQArm: 855 case X86::VMOVDQUrm: 856 case X86::VMOVAPSYrm: 857 case X86::VMOVUPSYrm: 858 case X86::VMOVAPDYrm: 859 case X86::VMOVUPDYrm: 860 case X86::VMOVDQAYrm: 861 case X86::VMOVDQUYrm: 862 case X86::MMX_MOVD64rm: 863 case X86::MMX_MOVQ64rm: 864 // AVX-512 865 case X86::VMOVSSZrm: 866 case X86::VMOVSSZrm_alt: 867 case X86::VMOVSDZrm: 868 case X86::VMOVSDZrm_alt: 869 case X86::VMOVSHZrm: 870 case X86::VMOVSHZrm_alt: 871 case X86::VMOVAPDZ128rm: 872 case X86::VMOVAPDZ256rm: 873 case X86::VMOVAPDZrm: 874 case X86::VMOVAPSZ128rm: 875 case X86::VMOVAPSZ256rm: 876 case X86::VMOVAPSZ128rm_NOVLX: 877 case X86::VMOVAPSZ256rm_NOVLX: 878 case X86::VMOVAPSZrm: 879 case X86::VMOVDQA32Z128rm: 880 case X86::VMOVDQA32Z256rm: 881 case X86::VMOVDQA32Zrm: 882 case X86::VMOVDQA64Z128rm: 883 case X86::VMOVDQA64Z256rm: 884 case X86::VMOVDQA64Zrm: 885 case X86::VMOVDQU16Z128rm: 886 case X86::VMOVDQU16Z256rm: 887 case X86::VMOVDQU16Zrm: 888 case X86::VMOVDQU32Z128rm: 889 case X86::VMOVDQU32Z256rm: 890 case X86::VMOVDQU32Zrm: 891 case X86::VMOVDQU64Z128rm: 892 case X86::VMOVDQU64Z256rm: 893 case X86::VMOVDQU64Zrm: 894 case X86::VMOVDQU8Z128rm: 895 case X86::VMOVDQU8Z256rm: 896 case X86::VMOVDQU8Zrm: 897 case X86::VMOVUPDZ128rm: 898 case X86::VMOVUPDZ256rm: 899 case X86::VMOVUPDZrm: 900 case X86::VMOVUPSZ128rm: 901 case X86::VMOVUPSZ256rm: 902 case X86::VMOVUPSZ128rm_NOVLX: 903 case X86::VMOVUPSZ256rm_NOVLX: 904 case X86::VMOVUPSZrm: { 905 // Loads from constant pools are trivially rematerializable. 906 if (MI.getOperand(1 + X86::AddrBaseReg).isReg() && 907 MI.getOperand(1 + X86::AddrScaleAmt).isImm() && 908 MI.getOperand(1 + X86::AddrIndexReg).isReg() && 909 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && 910 MI.isDereferenceableInvariantLoad()) { 911 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); 912 if (BaseReg == 0 || BaseReg == X86::RIP) 913 return true; 914 // Allow re-materialization of PIC load. 915 if (!(!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal())) { 916 const MachineFunction &MF = *MI.getParent()->getParent(); 917 const MachineRegisterInfo &MRI = MF.getRegInfo(); 918 if (regIsPICBase(BaseReg, MRI)) 919 return true; 920 } 921 } 922 break; 923 } 924 925 case X86::LEA32r: 926 case X86::LEA64r: { 927 if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() && 928 MI.getOperand(1 + X86::AddrIndexReg).isReg() && 929 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && 930 !MI.getOperand(1 + X86::AddrDisp).isReg()) { 931 // lea fi#, lea GV, etc. are all rematerializable. 932 if (!MI.getOperand(1 + X86::AddrBaseReg).isReg()) 933 return true; 934 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); 935 if (BaseReg == 0) 936 return true; 937 // Allow re-materialization of lea PICBase + x. 938 const MachineFunction &MF = *MI.getParent()->getParent(); 939 const MachineRegisterInfo &MRI = MF.getRegInfo(); 940 if (regIsPICBase(BaseReg, MRI)) 941 return true; 942 } 943 break; 944 } 945 } 946 return TargetInstrInfo::isReallyTriviallyReMaterializable(MI); 947 } 948 949 void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB, 950 MachineBasicBlock::iterator I, 951 Register DestReg, unsigned SubIdx, 952 const MachineInstr &Orig, 953 const TargetRegisterInfo &TRI) const { 954 bool ClobbersEFLAGS = Orig.modifiesRegister(X86::EFLAGS, &TRI); 955 if (ClobbersEFLAGS && MBB.computeRegisterLiveness(&TRI, X86::EFLAGS, I) != 956 MachineBasicBlock::LQR_Dead) { 957 // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side 958 // effects. 959 int Value; 960 switch (Orig.getOpcode()) { 961 case X86::MOV32r0: 962 Value = 0; 963 break; 964 case X86::MOV32r1: 965 Value = 1; 966 break; 967 case X86::MOV32r_1: 968 Value = -1; 969 break; 970 default: 971 llvm_unreachable("Unexpected instruction!"); 972 } 973 974 const DebugLoc &DL = Orig.getDebugLoc(); 975 BuildMI(MBB, I, DL, get(X86::MOV32ri)) 976 .add(Orig.getOperand(0)) 977 .addImm(Value); 978 } else { 979 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); 980 MBB.insert(I, MI); 981 } 982 983 MachineInstr &NewMI = *std::prev(I); 984 NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI); 985 } 986 987 /// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead. 988 bool X86InstrInfo::hasLiveCondCodeDef(MachineInstr &MI) const { 989 for (const MachineOperand &MO : MI.operands()) { 990 if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS && 991 !MO.isDead()) { 992 return true; 993 } 994 } 995 return false; 996 } 997 998 /// Check whether the shift count for a machine operand is non-zero. 999 inline static unsigned getTruncatedShiftCount(const MachineInstr &MI, 1000 unsigned ShiftAmtOperandIdx) { 1001 // The shift count is six bits with the REX.W prefix and five bits without. 1002 unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31; 1003 unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm(); 1004 return Imm & ShiftCountMask; 1005 } 1006 1007 /// Check whether the given shift count is appropriate 1008 /// can be represented by a LEA instruction. 1009 inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) { 1010 // Left shift instructions can be transformed into load-effective-address 1011 // instructions if we can encode them appropriately. 1012 // A LEA instruction utilizes a SIB byte to encode its scale factor. 1013 // The SIB.scale field is two bits wide which means that we can encode any 1014 // shift amount less than 4. 1015 return ShAmt < 4 && ShAmt > 0; 1016 } 1017 1018 static bool findRedundantFlagInstr(MachineInstr &CmpInstr, 1019 MachineInstr &CmpValDefInstr, 1020 const MachineRegisterInfo *MRI, 1021 MachineInstr **AndInstr, 1022 const TargetRegisterInfo *TRI, 1023 bool &NoSignFlag, bool &ClearsOverflowFlag) { 1024 if (!(CmpValDefInstr.getOpcode() == X86::SUBREG_TO_REG && 1025 CmpInstr.getOpcode() == X86::TEST64rr) && 1026 !(CmpValDefInstr.getOpcode() == X86::COPY && 1027 CmpInstr.getOpcode() == X86::TEST16rr)) 1028 return false; 1029 1030 // CmpInstr is a TEST16rr/TEST64rr instruction, and 1031 // `X86InstrInfo::analyzeCompare` guarantees that it's analyzable only if two 1032 // registers are identical. 1033 assert((CmpInstr.getOperand(0).getReg() == CmpInstr.getOperand(1).getReg()) && 1034 "CmpInstr is an analyzable TEST16rr/TEST64rr, and " 1035 "`X86InstrInfo::analyzeCompare` requires two reg operands are the" 1036 "same."); 1037 1038 // Caller (`X86InstrInfo::optimizeCompareInstr`) guarantees that 1039 // `CmpValDefInstr` defines the value that's used by `CmpInstr`; in this case 1040 // if `CmpValDefInstr` sets the EFLAGS, it is likely that `CmpInstr` is 1041 // redundant. 1042 assert( 1043 (MRI->getVRegDef(CmpInstr.getOperand(0).getReg()) == &CmpValDefInstr) && 1044 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG or TEST16rr " 1045 "is a user of COPY sub16bit."); 1046 MachineInstr *VregDefInstr = nullptr; 1047 if (CmpInstr.getOpcode() == X86::TEST16rr) { 1048 if (!CmpValDefInstr.getOperand(1).getReg().isVirtual()) 1049 return false; 1050 VregDefInstr = MRI->getVRegDef(CmpValDefInstr.getOperand(1).getReg()); 1051 if (!VregDefInstr) 1052 return false; 1053 // We can only remove test when AND32ri or AND64ri32 whose imm can fit 16bit 1054 // size, others 32/64 bit ops would test higher bits which test16rr don't 1055 // want to. 1056 if (!((VregDefInstr->getOpcode() == X86::AND32ri || 1057 VregDefInstr->getOpcode() == X86::AND64ri32) && 1058 isUInt<16>(VregDefInstr->getOperand(2).getImm()))) 1059 return false; 1060 } 1061 1062 if (CmpInstr.getOpcode() == X86::TEST64rr) { 1063 // As seen in X86 td files, CmpValDefInstr.getOperand(1).getImm() is 1064 // typically 0. 1065 if (CmpValDefInstr.getOperand(1).getImm() != 0) 1066 return false; 1067 1068 // As seen in X86 td files, CmpValDefInstr.getOperand(3) is typically 1069 // sub_32bit or sub_xmm. 1070 if (CmpValDefInstr.getOperand(3).getImm() != X86::sub_32bit) 1071 return false; 1072 1073 VregDefInstr = MRI->getVRegDef(CmpValDefInstr.getOperand(2).getReg()); 1074 } 1075 1076 assert(VregDefInstr && "Must have a definition (SSA)"); 1077 1078 // Requires `CmpValDefInstr` and `VregDefInstr` are from the same MBB 1079 // to simplify the subsequent analysis. 1080 // 1081 // FIXME: If `VregDefInstr->getParent()` is the only predecessor of 1082 // `CmpValDefInstr.getParent()`, this could be handled. 1083 if (VregDefInstr->getParent() != CmpValDefInstr.getParent()) 1084 return false; 1085 1086 if (X86::isAND(VregDefInstr->getOpcode())) { 1087 // Get a sequence of instructions like 1088 // %reg = and* ... // Set EFLAGS 1089 // ... // EFLAGS not changed 1090 // %extended_reg = subreg_to_reg 0, %reg, %subreg.sub_32bit 1091 // test64rr %extended_reg, %extended_reg, implicit-def $eflags 1092 // or 1093 // %reg = and32* ... 1094 // ... // EFLAGS not changed. 1095 // %src_reg = copy %reg.sub_16bit:gr32 1096 // test16rr %src_reg, %src_reg, implicit-def $eflags 1097 // 1098 // If subsequent readers use a subset of bits that don't change 1099 // after `and*` instructions, it's likely that the test64rr could 1100 // be optimized away. 1101 for (const MachineInstr &Instr : 1102 make_range(std::next(MachineBasicBlock::iterator(VregDefInstr)), 1103 MachineBasicBlock::iterator(CmpValDefInstr))) { 1104 // There are instructions between 'VregDefInstr' and 1105 // 'CmpValDefInstr' that modifies EFLAGS. 1106 if (Instr.modifiesRegister(X86::EFLAGS, TRI)) 1107 return false; 1108 } 1109 1110 *AndInstr = VregDefInstr; 1111 1112 // AND instruction will essentially update SF and clear OF, so 1113 // NoSignFlag should be false in the sense that SF is modified by `AND`. 1114 // 1115 // However, the implementation artifically sets `NoSignFlag` to true 1116 // to poison the SF bit; that is to say, if SF is looked at later, the 1117 // optimization (to erase TEST64rr) will be disabled. 1118 // 1119 // The reason to poison SF bit is that SF bit value could be different 1120 // in the `AND` and `TEST` operation; signed bit is not known for `AND`, 1121 // and is known to be 0 as a result of `TEST64rr`. 1122 // 1123 // FIXME: As opposed to poisoning the SF bit directly, consider peeking into 1124 // the AND instruction and using the static information to guide peephole 1125 // optimization if possible. For example, it's possible to fold a 1126 // conditional move into a copy if the relevant EFLAG bits could be deduced 1127 // from an immediate operand of and operation. 1128 // 1129 NoSignFlag = true; 1130 // ClearsOverflowFlag is true for AND operation (no surprise). 1131 ClearsOverflowFlag = true; 1132 return true; 1133 } 1134 return false; 1135 } 1136 1137 bool X86InstrInfo::classifyLEAReg(MachineInstr &MI, const MachineOperand &Src, 1138 unsigned Opc, bool AllowSP, Register &NewSrc, 1139 bool &isKill, MachineOperand &ImplicitOp, 1140 LiveVariables *LV, LiveIntervals *LIS) const { 1141 MachineFunction &MF = *MI.getParent()->getParent(); 1142 const TargetRegisterClass *RC; 1143 if (AllowSP) { 1144 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass; 1145 } else { 1146 RC = Opc != X86::LEA32r ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass; 1147 } 1148 Register SrcReg = Src.getReg(); 1149 isKill = MI.killsRegister(SrcReg); 1150 1151 // For both LEA64 and LEA32 the register already has essentially the right 1152 // type (32-bit or 64-bit) we may just need to forbid SP. 1153 if (Opc != X86::LEA64_32r) { 1154 NewSrc = SrcReg; 1155 assert(!Src.isUndef() && "Undef op doesn't need optimization"); 1156 1157 if (NewSrc.isVirtual() && !MF.getRegInfo().constrainRegClass(NewSrc, RC)) 1158 return false; 1159 1160 return true; 1161 } 1162 1163 // This is for an LEA64_32r and incoming registers are 32-bit. One way or 1164 // another we need to add 64-bit registers to the final MI. 1165 if (SrcReg.isPhysical()) { 1166 ImplicitOp = Src; 1167 ImplicitOp.setImplicit(); 1168 1169 NewSrc = getX86SubSuperRegister(SrcReg, 64); 1170 assert(NewSrc.isValid() && "Invalid Operand"); 1171 assert(!Src.isUndef() && "Undef op doesn't need optimization"); 1172 } else { 1173 // Virtual register of the wrong class, we have to create a temporary 64-bit 1174 // vreg to feed into the LEA. 1175 NewSrc = MF.getRegInfo().createVirtualRegister(RC); 1176 MachineInstr *Copy = 1177 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1178 .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit) 1179 .addReg(SrcReg, getKillRegState(isKill)); 1180 1181 // Which is obviously going to be dead after we're done with it. 1182 isKill = true; 1183 1184 if (LV) 1185 LV->replaceKillInstruction(SrcReg, MI, *Copy); 1186 1187 if (LIS) { 1188 SlotIndex CopyIdx = LIS->InsertMachineInstrInMaps(*Copy); 1189 SlotIndex Idx = LIS->getInstructionIndex(MI); 1190 LiveInterval &LI = LIS->getInterval(SrcReg); 1191 LiveRange::Segment *S = LI.getSegmentContaining(Idx); 1192 if (S->end.getBaseIndex() == Idx) 1193 S->end = CopyIdx.getRegSlot(); 1194 } 1195 } 1196 1197 // We've set all the parameters without issue. 1198 return true; 1199 } 1200 1201 MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc, 1202 MachineInstr &MI, 1203 LiveVariables *LV, 1204 LiveIntervals *LIS, 1205 bool Is8BitOp) const { 1206 // We handle 8-bit adds and various 16-bit opcodes in the switch below. 1207 MachineBasicBlock &MBB = *MI.getParent(); 1208 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo(); 1209 assert((Is8BitOp || 1210 RegInfo.getTargetRegisterInfo()->getRegSizeInBits( 1211 *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) && 1212 "Unexpected type for LEA transform"); 1213 1214 // TODO: For a 32-bit target, we need to adjust the LEA variables with 1215 // something like this: 1216 // Opcode = X86::LEA32r; 1217 // InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); 1218 // OutRegLEA = 1219 // Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass) 1220 // : RegInfo.createVirtualRegister(&X86::GR32RegClass); 1221 if (!Subtarget.is64Bit()) 1222 return nullptr; 1223 1224 unsigned Opcode = X86::LEA64_32r; 1225 Register InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 1226 Register OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass); 1227 Register InRegLEA2; 1228 1229 // Build and insert into an implicit UNDEF value. This is OK because 1230 // we will be shifting and then extracting the lower 8/16-bits. 1231 // This has the potential to cause partial register stall. e.g. 1232 // movw (%rbp,%rcx,2), %dx 1233 // leal -65(%rdx), %esi 1234 // But testing has shown this *does* help performance in 64-bit mode (at 1235 // least on modern x86 machines). 1236 MachineBasicBlock::iterator MBBI = MI.getIterator(); 1237 Register Dest = MI.getOperand(0).getReg(); 1238 Register Src = MI.getOperand(1).getReg(); 1239 Register Src2; 1240 bool IsDead = MI.getOperand(0).isDead(); 1241 bool IsKill = MI.getOperand(1).isKill(); 1242 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit; 1243 assert(!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization"); 1244 MachineInstr *ImpDef = 1245 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA); 1246 MachineInstr *InsMI = 1247 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1248 .addReg(InRegLEA, RegState::Define, SubReg) 1249 .addReg(Src, getKillRegState(IsKill)); 1250 MachineInstr *ImpDef2 = nullptr; 1251 MachineInstr *InsMI2 = nullptr; 1252 1253 MachineInstrBuilder MIB = 1254 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(Opcode), OutRegLEA); 1255 switch (MIOpc) { 1256 default: 1257 llvm_unreachable("Unreachable!"); 1258 case X86::SHL8ri: 1259 case X86::SHL16ri: { 1260 unsigned ShAmt = MI.getOperand(2).getImm(); 1261 MIB.addReg(0) 1262 .addImm(1LL << ShAmt) 1263 .addReg(InRegLEA, RegState::Kill) 1264 .addImm(0) 1265 .addReg(0); 1266 break; 1267 } 1268 case X86::INC8r: 1269 case X86::INC16r: 1270 addRegOffset(MIB, InRegLEA, true, 1); 1271 break; 1272 case X86::DEC8r: 1273 case X86::DEC16r: 1274 addRegOffset(MIB, InRegLEA, true, -1); 1275 break; 1276 case X86::ADD8ri: 1277 case X86::ADD8ri_DB: 1278 case X86::ADD16ri: 1279 case X86::ADD16ri_DB: 1280 addRegOffset(MIB, InRegLEA, true, MI.getOperand(2).getImm()); 1281 break; 1282 case X86::ADD8rr: 1283 case X86::ADD8rr_DB: 1284 case X86::ADD16rr: 1285 case X86::ADD16rr_DB: { 1286 Src2 = MI.getOperand(2).getReg(); 1287 bool IsKill2 = MI.getOperand(2).isKill(); 1288 assert(!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization"); 1289 if (Src == Src2) { 1290 // ADD8rr/ADD16rr killed %reg1028, %reg1028 1291 // just a single insert_subreg. 1292 addRegReg(MIB, InRegLEA, true, InRegLEA, false); 1293 } else { 1294 if (Subtarget.is64Bit()) 1295 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 1296 else 1297 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); 1298 // Build and insert into an implicit UNDEF value. This is OK because 1299 // we will be shifting and then extracting the lower 8/16-bits. 1300 ImpDef2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), 1301 InRegLEA2); 1302 InsMI2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1303 .addReg(InRegLEA2, RegState::Define, SubReg) 1304 .addReg(Src2, getKillRegState(IsKill2)); 1305 addRegReg(MIB, InRegLEA, true, InRegLEA2, true); 1306 } 1307 if (LV && IsKill2 && InsMI2) 1308 LV->replaceKillInstruction(Src2, MI, *InsMI2); 1309 break; 1310 } 1311 } 1312 1313 MachineInstr *NewMI = MIB; 1314 MachineInstr *ExtMI = 1315 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 1316 .addReg(Dest, RegState::Define | getDeadRegState(IsDead)) 1317 .addReg(OutRegLEA, RegState::Kill, SubReg); 1318 1319 if (LV) { 1320 // Update live variables. 1321 LV->getVarInfo(InRegLEA).Kills.push_back(NewMI); 1322 if (InRegLEA2) 1323 LV->getVarInfo(InRegLEA2).Kills.push_back(NewMI); 1324 LV->getVarInfo(OutRegLEA).Kills.push_back(ExtMI); 1325 if (IsKill) 1326 LV->replaceKillInstruction(Src, MI, *InsMI); 1327 if (IsDead) 1328 LV->replaceKillInstruction(Dest, MI, *ExtMI); 1329 } 1330 1331 if (LIS) { 1332 LIS->InsertMachineInstrInMaps(*ImpDef); 1333 SlotIndex InsIdx = LIS->InsertMachineInstrInMaps(*InsMI); 1334 if (ImpDef2) 1335 LIS->InsertMachineInstrInMaps(*ImpDef2); 1336 SlotIndex Ins2Idx; 1337 if (InsMI2) 1338 Ins2Idx = LIS->InsertMachineInstrInMaps(*InsMI2); 1339 SlotIndex NewIdx = LIS->ReplaceMachineInstrInMaps(MI, *NewMI); 1340 SlotIndex ExtIdx = LIS->InsertMachineInstrInMaps(*ExtMI); 1341 LIS->getInterval(InRegLEA); 1342 LIS->getInterval(OutRegLEA); 1343 if (InRegLEA2) 1344 LIS->getInterval(InRegLEA2); 1345 1346 // Move the use of Src up to InsMI. 1347 LiveInterval &SrcLI = LIS->getInterval(Src); 1348 LiveRange::Segment *SrcSeg = SrcLI.getSegmentContaining(NewIdx); 1349 if (SrcSeg->end == NewIdx.getRegSlot()) 1350 SrcSeg->end = InsIdx.getRegSlot(); 1351 1352 if (InsMI2) { 1353 // Move the use of Src2 up to InsMI2. 1354 LiveInterval &Src2LI = LIS->getInterval(Src2); 1355 LiveRange::Segment *Src2Seg = Src2LI.getSegmentContaining(NewIdx); 1356 if (Src2Seg->end == NewIdx.getRegSlot()) 1357 Src2Seg->end = Ins2Idx.getRegSlot(); 1358 } 1359 1360 // Move the definition of Dest down to ExtMI. 1361 LiveInterval &DestLI = LIS->getInterval(Dest); 1362 LiveRange::Segment *DestSeg = 1363 DestLI.getSegmentContaining(NewIdx.getRegSlot()); 1364 assert(DestSeg->start == NewIdx.getRegSlot() && 1365 DestSeg->valno->def == NewIdx.getRegSlot()); 1366 DestSeg->start = ExtIdx.getRegSlot(); 1367 DestSeg->valno->def = ExtIdx.getRegSlot(); 1368 } 1369 1370 return ExtMI; 1371 } 1372 1373 /// This method must be implemented by targets that 1374 /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target 1375 /// may be able to convert a two-address instruction into a true 1376 /// three-address instruction on demand. This allows the X86 target (for 1377 /// example) to convert ADD and SHL instructions into LEA instructions if they 1378 /// would require register copies due to two-addressness. 1379 /// 1380 /// This method returns a null pointer if the transformation cannot be 1381 /// performed, otherwise it returns the new instruction. 1382 /// 1383 MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI, 1384 LiveVariables *LV, 1385 LiveIntervals *LIS) const { 1386 // The following opcodes also sets the condition code register(s). Only 1387 // convert them to equivalent lea if the condition code register def's 1388 // are dead! 1389 if (hasLiveCondCodeDef(MI)) 1390 return nullptr; 1391 1392 MachineFunction &MF = *MI.getParent()->getParent(); 1393 // All instructions input are two-addr instructions. Get the known operands. 1394 const MachineOperand &Dest = MI.getOperand(0); 1395 const MachineOperand &Src = MI.getOperand(1); 1396 1397 // Ideally, operations with undef should be folded before we get here, but we 1398 // can't guarantee it. Bail out because optimizing undefs is a waste of time. 1399 // Without this, we have to forward undef state to new register operands to 1400 // avoid machine verifier errors. 1401 if (Src.isUndef()) 1402 return nullptr; 1403 if (MI.getNumOperands() > 2) 1404 if (MI.getOperand(2).isReg() && MI.getOperand(2).isUndef()) 1405 return nullptr; 1406 1407 MachineInstr *NewMI = nullptr; 1408 Register SrcReg, SrcReg2; 1409 bool Is64Bit = Subtarget.is64Bit(); 1410 1411 bool Is8BitOp = false; 1412 unsigned NumRegOperands = 2; 1413 unsigned MIOpc = MI.getOpcode(); 1414 switch (MIOpc) { 1415 default: 1416 llvm_unreachable("Unreachable!"); 1417 case X86::SHL64ri: { 1418 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 1419 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 1420 if (!isTruncatedShiftCountForLEA(ShAmt)) 1421 return nullptr; 1422 1423 // LEA can't handle RSP. 1424 if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass( 1425 Src.getReg(), &X86::GR64_NOSPRegClass)) 1426 return nullptr; 1427 1428 NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)) 1429 .add(Dest) 1430 .addReg(0) 1431 .addImm(1LL << ShAmt) 1432 .add(Src) 1433 .addImm(0) 1434 .addReg(0); 1435 break; 1436 } 1437 case X86::SHL32ri: { 1438 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 1439 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 1440 if (!isTruncatedShiftCountForLEA(ShAmt)) 1441 return nullptr; 1442 1443 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1444 1445 // LEA can't handle ESP. 1446 bool isKill; 1447 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1448 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill, 1449 ImplicitOp, LV, LIS)) 1450 return nullptr; 1451 1452 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1453 .add(Dest) 1454 .addReg(0) 1455 .addImm(1LL << ShAmt) 1456 .addReg(SrcReg, getKillRegState(isKill)) 1457 .addImm(0) 1458 .addReg(0); 1459 if (ImplicitOp.getReg() != 0) 1460 MIB.add(ImplicitOp); 1461 NewMI = MIB; 1462 1463 // Add kills if classifyLEAReg created a new register. 1464 if (LV && SrcReg != Src.getReg()) 1465 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1466 break; 1467 } 1468 case X86::SHL8ri: 1469 Is8BitOp = true; 1470 [[fallthrough]]; 1471 case X86::SHL16ri: { 1472 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 1473 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 1474 if (!isTruncatedShiftCountForLEA(ShAmt)) 1475 return nullptr; 1476 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1477 } 1478 case X86::INC64r: 1479 case X86::INC32r: { 1480 assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!"); 1481 unsigned Opc = MIOpc == X86::INC64r 1482 ? X86::LEA64r 1483 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r); 1484 bool isKill; 1485 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1486 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill, 1487 ImplicitOp, LV, LIS)) 1488 return nullptr; 1489 1490 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1491 .add(Dest) 1492 .addReg(SrcReg, getKillRegState(isKill)); 1493 if (ImplicitOp.getReg() != 0) 1494 MIB.add(ImplicitOp); 1495 1496 NewMI = addOffset(MIB, 1); 1497 1498 // Add kills if classifyLEAReg created a new register. 1499 if (LV && SrcReg != Src.getReg()) 1500 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1501 break; 1502 } 1503 case X86::DEC64r: 1504 case X86::DEC32r: { 1505 assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!"); 1506 unsigned Opc = MIOpc == X86::DEC64r 1507 ? X86::LEA64r 1508 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r); 1509 1510 bool isKill; 1511 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1512 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill, 1513 ImplicitOp, LV, LIS)) 1514 return nullptr; 1515 1516 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1517 .add(Dest) 1518 .addReg(SrcReg, getKillRegState(isKill)); 1519 if (ImplicitOp.getReg() != 0) 1520 MIB.add(ImplicitOp); 1521 1522 NewMI = addOffset(MIB, -1); 1523 1524 // Add kills if classifyLEAReg created a new register. 1525 if (LV && SrcReg != Src.getReg()) 1526 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1527 break; 1528 } 1529 case X86::DEC8r: 1530 case X86::INC8r: 1531 Is8BitOp = true; 1532 [[fallthrough]]; 1533 case X86::DEC16r: 1534 case X86::INC16r: 1535 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1536 case X86::ADD64rr: 1537 case X86::ADD64rr_DB: 1538 case X86::ADD32rr: 1539 case X86::ADD32rr_DB: { 1540 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1541 unsigned Opc; 1542 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB) 1543 Opc = X86::LEA64r; 1544 else 1545 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1546 1547 const MachineOperand &Src2 = MI.getOperand(2); 1548 bool isKill2; 1549 MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false); 1550 if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/false, SrcReg2, isKill2, 1551 ImplicitOp2, LV, LIS)) 1552 return nullptr; 1553 1554 bool isKill; 1555 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1556 if (Src.getReg() == Src2.getReg()) { 1557 // Don't call classify LEAReg a second time on the same register, in case 1558 // the first call inserted a COPY from Src2 and marked it as killed. 1559 isKill = isKill2; 1560 SrcReg = SrcReg2; 1561 } else { 1562 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill, 1563 ImplicitOp, LV, LIS)) 1564 return nullptr; 1565 } 1566 1567 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest); 1568 if (ImplicitOp.getReg() != 0) 1569 MIB.add(ImplicitOp); 1570 if (ImplicitOp2.getReg() != 0) 1571 MIB.add(ImplicitOp2); 1572 1573 NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2); 1574 1575 // Add kills if classifyLEAReg created a new register. 1576 if (LV) { 1577 if (SrcReg2 != Src2.getReg()) 1578 LV->getVarInfo(SrcReg2).Kills.push_back(NewMI); 1579 if (SrcReg != SrcReg2 && SrcReg != Src.getReg()) 1580 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1581 } 1582 NumRegOperands = 3; 1583 break; 1584 } 1585 case X86::ADD8rr: 1586 case X86::ADD8rr_DB: 1587 Is8BitOp = true; 1588 [[fallthrough]]; 1589 case X86::ADD16rr: 1590 case X86::ADD16rr_DB: 1591 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1592 case X86::ADD64ri32: 1593 case X86::ADD64ri32_DB: 1594 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1595 NewMI = addOffset( 1596 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src), 1597 MI.getOperand(2)); 1598 break; 1599 case X86::ADD32ri: 1600 case X86::ADD32ri_DB: { 1601 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1602 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1603 1604 bool isKill; 1605 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1606 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill, 1607 ImplicitOp, LV, LIS)) 1608 return nullptr; 1609 1610 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1611 .add(Dest) 1612 .addReg(SrcReg, getKillRegState(isKill)); 1613 if (ImplicitOp.getReg() != 0) 1614 MIB.add(ImplicitOp); 1615 1616 NewMI = addOffset(MIB, MI.getOperand(2)); 1617 1618 // Add kills if classifyLEAReg created a new register. 1619 if (LV && SrcReg != Src.getReg()) 1620 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1621 break; 1622 } 1623 case X86::ADD8ri: 1624 case X86::ADD8ri_DB: 1625 Is8BitOp = true; 1626 [[fallthrough]]; 1627 case X86::ADD16ri: 1628 case X86::ADD16ri_DB: 1629 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp); 1630 case X86::SUB8ri: 1631 case X86::SUB16ri: 1632 /// FIXME: Support these similar to ADD8ri/ADD16ri*. 1633 return nullptr; 1634 case X86::SUB32ri: { 1635 if (!MI.getOperand(2).isImm()) 1636 return nullptr; 1637 int64_t Imm = MI.getOperand(2).getImm(); 1638 if (!isInt<32>(-Imm)) 1639 return nullptr; 1640 1641 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1642 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; 1643 1644 bool isKill; 1645 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1646 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill, 1647 ImplicitOp, LV, LIS)) 1648 return nullptr; 1649 1650 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1651 .add(Dest) 1652 .addReg(SrcReg, getKillRegState(isKill)); 1653 if (ImplicitOp.getReg() != 0) 1654 MIB.add(ImplicitOp); 1655 1656 NewMI = addOffset(MIB, -Imm); 1657 1658 // Add kills if classifyLEAReg created a new register. 1659 if (LV && SrcReg != Src.getReg()) 1660 LV->getVarInfo(SrcReg).Kills.push_back(NewMI); 1661 break; 1662 } 1663 1664 case X86::SUB64ri32: { 1665 if (!MI.getOperand(2).isImm()) 1666 return nullptr; 1667 int64_t Imm = MI.getOperand(2).getImm(); 1668 if (!isInt<32>(-Imm)) 1669 return nullptr; 1670 1671 assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!"); 1672 1673 MachineInstrBuilder MIB = 1674 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src); 1675 NewMI = addOffset(MIB, -Imm); 1676 break; 1677 } 1678 1679 case X86::VMOVDQU8Z128rmk: 1680 case X86::VMOVDQU8Z256rmk: 1681 case X86::VMOVDQU8Zrmk: 1682 case X86::VMOVDQU16Z128rmk: 1683 case X86::VMOVDQU16Z256rmk: 1684 case X86::VMOVDQU16Zrmk: 1685 case X86::VMOVDQU32Z128rmk: 1686 case X86::VMOVDQA32Z128rmk: 1687 case X86::VMOVDQU32Z256rmk: 1688 case X86::VMOVDQA32Z256rmk: 1689 case X86::VMOVDQU32Zrmk: 1690 case X86::VMOVDQA32Zrmk: 1691 case X86::VMOVDQU64Z128rmk: 1692 case X86::VMOVDQA64Z128rmk: 1693 case X86::VMOVDQU64Z256rmk: 1694 case X86::VMOVDQA64Z256rmk: 1695 case X86::VMOVDQU64Zrmk: 1696 case X86::VMOVDQA64Zrmk: 1697 case X86::VMOVUPDZ128rmk: 1698 case X86::VMOVAPDZ128rmk: 1699 case X86::VMOVUPDZ256rmk: 1700 case X86::VMOVAPDZ256rmk: 1701 case X86::VMOVUPDZrmk: 1702 case X86::VMOVAPDZrmk: 1703 case X86::VMOVUPSZ128rmk: 1704 case X86::VMOVAPSZ128rmk: 1705 case X86::VMOVUPSZ256rmk: 1706 case X86::VMOVAPSZ256rmk: 1707 case X86::VMOVUPSZrmk: 1708 case X86::VMOVAPSZrmk: 1709 case X86::VBROADCASTSDZ256rmk: 1710 case X86::VBROADCASTSDZrmk: 1711 case X86::VBROADCASTSSZ128rmk: 1712 case X86::VBROADCASTSSZ256rmk: 1713 case X86::VBROADCASTSSZrmk: 1714 case X86::VPBROADCASTDZ128rmk: 1715 case X86::VPBROADCASTDZ256rmk: 1716 case X86::VPBROADCASTDZrmk: 1717 case X86::VPBROADCASTQZ128rmk: 1718 case X86::VPBROADCASTQZ256rmk: 1719 case X86::VPBROADCASTQZrmk: { 1720 unsigned Opc; 1721 switch (MIOpc) { 1722 default: 1723 llvm_unreachable("Unreachable!"); 1724 case X86::VMOVDQU8Z128rmk: 1725 Opc = X86::VPBLENDMBZ128rmk; 1726 break; 1727 case X86::VMOVDQU8Z256rmk: 1728 Opc = X86::VPBLENDMBZ256rmk; 1729 break; 1730 case X86::VMOVDQU8Zrmk: 1731 Opc = X86::VPBLENDMBZrmk; 1732 break; 1733 case X86::VMOVDQU16Z128rmk: 1734 Opc = X86::VPBLENDMWZ128rmk; 1735 break; 1736 case X86::VMOVDQU16Z256rmk: 1737 Opc = X86::VPBLENDMWZ256rmk; 1738 break; 1739 case X86::VMOVDQU16Zrmk: 1740 Opc = X86::VPBLENDMWZrmk; 1741 break; 1742 case X86::VMOVDQU32Z128rmk: 1743 Opc = X86::VPBLENDMDZ128rmk; 1744 break; 1745 case X86::VMOVDQU32Z256rmk: 1746 Opc = X86::VPBLENDMDZ256rmk; 1747 break; 1748 case X86::VMOVDQU32Zrmk: 1749 Opc = X86::VPBLENDMDZrmk; 1750 break; 1751 case X86::VMOVDQU64Z128rmk: 1752 Opc = X86::VPBLENDMQZ128rmk; 1753 break; 1754 case X86::VMOVDQU64Z256rmk: 1755 Opc = X86::VPBLENDMQZ256rmk; 1756 break; 1757 case X86::VMOVDQU64Zrmk: 1758 Opc = X86::VPBLENDMQZrmk; 1759 break; 1760 case X86::VMOVUPDZ128rmk: 1761 Opc = X86::VBLENDMPDZ128rmk; 1762 break; 1763 case X86::VMOVUPDZ256rmk: 1764 Opc = X86::VBLENDMPDZ256rmk; 1765 break; 1766 case X86::VMOVUPDZrmk: 1767 Opc = X86::VBLENDMPDZrmk; 1768 break; 1769 case X86::VMOVUPSZ128rmk: 1770 Opc = X86::VBLENDMPSZ128rmk; 1771 break; 1772 case X86::VMOVUPSZ256rmk: 1773 Opc = X86::VBLENDMPSZ256rmk; 1774 break; 1775 case X86::VMOVUPSZrmk: 1776 Opc = X86::VBLENDMPSZrmk; 1777 break; 1778 case X86::VMOVDQA32Z128rmk: 1779 Opc = X86::VPBLENDMDZ128rmk; 1780 break; 1781 case X86::VMOVDQA32Z256rmk: 1782 Opc = X86::VPBLENDMDZ256rmk; 1783 break; 1784 case X86::VMOVDQA32Zrmk: 1785 Opc = X86::VPBLENDMDZrmk; 1786 break; 1787 case X86::VMOVDQA64Z128rmk: 1788 Opc = X86::VPBLENDMQZ128rmk; 1789 break; 1790 case X86::VMOVDQA64Z256rmk: 1791 Opc = X86::VPBLENDMQZ256rmk; 1792 break; 1793 case X86::VMOVDQA64Zrmk: 1794 Opc = X86::VPBLENDMQZrmk; 1795 break; 1796 case X86::VMOVAPDZ128rmk: 1797 Opc = X86::VBLENDMPDZ128rmk; 1798 break; 1799 case X86::VMOVAPDZ256rmk: 1800 Opc = X86::VBLENDMPDZ256rmk; 1801 break; 1802 case X86::VMOVAPDZrmk: 1803 Opc = X86::VBLENDMPDZrmk; 1804 break; 1805 case X86::VMOVAPSZ128rmk: 1806 Opc = X86::VBLENDMPSZ128rmk; 1807 break; 1808 case X86::VMOVAPSZ256rmk: 1809 Opc = X86::VBLENDMPSZ256rmk; 1810 break; 1811 case X86::VMOVAPSZrmk: 1812 Opc = X86::VBLENDMPSZrmk; 1813 break; 1814 case X86::VBROADCASTSDZ256rmk: 1815 Opc = X86::VBLENDMPDZ256rmbk; 1816 break; 1817 case X86::VBROADCASTSDZrmk: 1818 Opc = X86::VBLENDMPDZrmbk; 1819 break; 1820 case X86::VBROADCASTSSZ128rmk: 1821 Opc = X86::VBLENDMPSZ128rmbk; 1822 break; 1823 case X86::VBROADCASTSSZ256rmk: 1824 Opc = X86::VBLENDMPSZ256rmbk; 1825 break; 1826 case X86::VBROADCASTSSZrmk: 1827 Opc = X86::VBLENDMPSZrmbk; 1828 break; 1829 case X86::VPBROADCASTDZ128rmk: 1830 Opc = X86::VPBLENDMDZ128rmbk; 1831 break; 1832 case X86::VPBROADCASTDZ256rmk: 1833 Opc = X86::VPBLENDMDZ256rmbk; 1834 break; 1835 case X86::VPBROADCASTDZrmk: 1836 Opc = X86::VPBLENDMDZrmbk; 1837 break; 1838 case X86::VPBROADCASTQZ128rmk: 1839 Opc = X86::VPBLENDMQZ128rmbk; 1840 break; 1841 case X86::VPBROADCASTQZ256rmk: 1842 Opc = X86::VPBLENDMQZ256rmbk; 1843 break; 1844 case X86::VPBROADCASTQZrmk: 1845 Opc = X86::VPBLENDMQZrmbk; 1846 break; 1847 } 1848 1849 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1850 .add(Dest) 1851 .add(MI.getOperand(2)) 1852 .add(Src) 1853 .add(MI.getOperand(3)) 1854 .add(MI.getOperand(4)) 1855 .add(MI.getOperand(5)) 1856 .add(MI.getOperand(6)) 1857 .add(MI.getOperand(7)); 1858 NumRegOperands = 4; 1859 break; 1860 } 1861 1862 case X86::VMOVDQU8Z128rrk: 1863 case X86::VMOVDQU8Z256rrk: 1864 case X86::VMOVDQU8Zrrk: 1865 case X86::VMOVDQU16Z128rrk: 1866 case X86::VMOVDQU16Z256rrk: 1867 case X86::VMOVDQU16Zrrk: 1868 case X86::VMOVDQU32Z128rrk: 1869 case X86::VMOVDQA32Z128rrk: 1870 case X86::VMOVDQU32Z256rrk: 1871 case X86::VMOVDQA32Z256rrk: 1872 case X86::VMOVDQU32Zrrk: 1873 case X86::VMOVDQA32Zrrk: 1874 case X86::VMOVDQU64Z128rrk: 1875 case X86::VMOVDQA64Z128rrk: 1876 case X86::VMOVDQU64Z256rrk: 1877 case X86::VMOVDQA64Z256rrk: 1878 case X86::VMOVDQU64Zrrk: 1879 case X86::VMOVDQA64Zrrk: 1880 case X86::VMOVUPDZ128rrk: 1881 case X86::VMOVAPDZ128rrk: 1882 case X86::VMOVUPDZ256rrk: 1883 case X86::VMOVAPDZ256rrk: 1884 case X86::VMOVUPDZrrk: 1885 case X86::VMOVAPDZrrk: 1886 case X86::VMOVUPSZ128rrk: 1887 case X86::VMOVAPSZ128rrk: 1888 case X86::VMOVUPSZ256rrk: 1889 case X86::VMOVAPSZ256rrk: 1890 case X86::VMOVUPSZrrk: 1891 case X86::VMOVAPSZrrk: { 1892 unsigned Opc; 1893 switch (MIOpc) { 1894 default: 1895 llvm_unreachable("Unreachable!"); 1896 case X86::VMOVDQU8Z128rrk: 1897 Opc = X86::VPBLENDMBZ128rrk; 1898 break; 1899 case X86::VMOVDQU8Z256rrk: 1900 Opc = X86::VPBLENDMBZ256rrk; 1901 break; 1902 case X86::VMOVDQU8Zrrk: 1903 Opc = X86::VPBLENDMBZrrk; 1904 break; 1905 case X86::VMOVDQU16Z128rrk: 1906 Opc = X86::VPBLENDMWZ128rrk; 1907 break; 1908 case X86::VMOVDQU16Z256rrk: 1909 Opc = X86::VPBLENDMWZ256rrk; 1910 break; 1911 case X86::VMOVDQU16Zrrk: 1912 Opc = X86::VPBLENDMWZrrk; 1913 break; 1914 case X86::VMOVDQU32Z128rrk: 1915 Opc = X86::VPBLENDMDZ128rrk; 1916 break; 1917 case X86::VMOVDQU32Z256rrk: 1918 Opc = X86::VPBLENDMDZ256rrk; 1919 break; 1920 case X86::VMOVDQU32Zrrk: 1921 Opc = X86::VPBLENDMDZrrk; 1922 break; 1923 case X86::VMOVDQU64Z128rrk: 1924 Opc = X86::VPBLENDMQZ128rrk; 1925 break; 1926 case X86::VMOVDQU64Z256rrk: 1927 Opc = X86::VPBLENDMQZ256rrk; 1928 break; 1929 case X86::VMOVDQU64Zrrk: 1930 Opc = X86::VPBLENDMQZrrk; 1931 break; 1932 case X86::VMOVUPDZ128rrk: 1933 Opc = X86::VBLENDMPDZ128rrk; 1934 break; 1935 case X86::VMOVUPDZ256rrk: 1936 Opc = X86::VBLENDMPDZ256rrk; 1937 break; 1938 case X86::VMOVUPDZrrk: 1939 Opc = X86::VBLENDMPDZrrk; 1940 break; 1941 case X86::VMOVUPSZ128rrk: 1942 Opc = X86::VBLENDMPSZ128rrk; 1943 break; 1944 case X86::VMOVUPSZ256rrk: 1945 Opc = X86::VBLENDMPSZ256rrk; 1946 break; 1947 case X86::VMOVUPSZrrk: 1948 Opc = X86::VBLENDMPSZrrk; 1949 break; 1950 case X86::VMOVDQA32Z128rrk: 1951 Opc = X86::VPBLENDMDZ128rrk; 1952 break; 1953 case X86::VMOVDQA32Z256rrk: 1954 Opc = X86::VPBLENDMDZ256rrk; 1955 break; 1956 case X86::VMOVDQA32Zrrk: 1957 Opc = X86::VPBLENDMDZrrk; 1958 break; 1959 case X86::VMOVDQA64Z128rrk: 1960 Opc = X86::VPBLENDMQZ128rrk; 1961 break; 1962 case X86::VMOVDQA64Z256rrk: 1963 Opc = X86::VPBLENDMQZ256rrk; 1964 break; 1965 case X86::VMOVDQA64Zrrk: 1966 Opc = X86::VPBLENDMQZrrk; 1967 break; 1968 case X86::VMOVAPDZ128rrk: 1969 Opc = X86::VBLENDMPDZ128rrk; 1970 break; 1971 case X86::VMOVAPDZ256rrk: 1972 Opc = X86::VBLENDMPDZ256rrk; 1973 break; 1974 case X86::VMOVAPDZrrk: 1975 Opc = X86::VBLENDMPDZrrk; 1976 break; 1977 case X86::VMOVAPSZ128rrk: 1978 Opc = X86::VBLENDMPSZ128rrk; 1979 break; 1980 case X86::VMOVAPSZ256rrk: 1981 Opc = X86::VBLENDMPSZ256rrk; 1982 break; 1983 case X86::VMOVAPSZrrk: 1984 Opc = X86::VBLENDMPSZrrk; 1985 break; 1986 } 1987 1988 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1989 .add(Dest) 1990 .add(MI.getOperand(2)) 1991 .add(Src) 1992 .add(MI.getOperand(3)); 1993 NumRegOperands = 4; 1994 break; 1995 } 1996 } 1997 1998 if (!NewMI) 1999 return nullptr; 2000 2001 if (LV) { // Update live variables 2002 for (unsigned I = 0; I < NumRegOperands; ++I) { 2003 MachineOperand &Op = MI.getOperand(I); 2004 if (Op.isReg() && (Op.isDead() || Op.isKill())) 2005 LV->replaceKillInstruction(Op.getReg(), MI, *NewMI); 2006 } 2007 } 2008 2009 MachineBasicBlock &MBB = *MI.getParent(); 2010 MBB.insert(MI.getIterator(), NewMI); // Insert the new inst 2011 2012 if (LIS) { 2013 LIS->ReplaceMachineInstrInMaps(MI, *NewMI); 2014 if (SrcReg) 2015 LIS->getInterval(SrcReg); 2016 if (SrcReg2) 2017 LIS->getInterval(SrcReg2); 2018 } 2019 2020 return NewMI; 2021 } 2022 2023 /// This determines which of three possible cases of a three source commute 2024 /// the source indexes correspond to taking into account any mask operands. 2025 /// All prevents commuting a passthru operand. Returns -1 if the commute isn't 2026 /// possible. 2027 /// Case 0 - Possible to commute the first and second operands. 2028 /// Case 1 - Possible to commute the first and third operands. 2029 /// Case 2 - Possible to commute the second and third operands. 2030 static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1, 2031 unsigned SrcOpIdx2) { 2032 // Put the lowest index to SrcOpIdx1 to simplify the checks below. 2033 if (SrcOpIdx1 > SrcOpIdx2) 2034 std::swap(SrcOpIdx1, SrcOpIdx2); 2035 2036 unsigned Op1 = 1, Op2 = 2, Op3 = 3; 2037 if (X86II::isKMasked(TSFlags)) { 2038 Op2++; 2039 Op3++; 2040 } 2041 2042 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2) 2043 return 0; 2044 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3) 2045 return 1; 2046 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3) 2047 return 2; 2048 llvm_unreachable("Unknown three src commute case."); 2049 } 2050 2051 unsigned X86InstrInfo::getFMA3OpcodeToCommuteOperands( 2052 const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2, 2053 const X86InstrFMA3Group &FMA3Group) const { 2054 2055 unsigned Opc = MI.getOpcode(); 2056 2057 // TODO: Commuting the 1st operand of FMA*_Int requires some additional 2058 // analysis. The commute optimization is legal only if all users of FMA*_Int 2059 // use only the lowest element of the FMA*_Int instruction. Such analysis are 2060 // not implemented yet. So, just return 0 in that case. 2061 // When such analysis are available this place will be the right place for 2062 // calling it. 2063 assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && 2064 "Intrinsic instructions can't commute operand 1"); 2065 2066 // Determine which case this commute is or if it can't be done. 2067 unsigned Case = 2068 getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2); 2069 assert(Case < 3 && "Unexpected case number!"); 2070 2071 // Define the FMA forms mapping array that helps to map input FMA form 2072 // to output FMA form to preserve the operation semantics after 2073 // commuting the operands. 2074 const unsigned Form132Index = 0; 2075 const unsigned Form213Index = 1; 2076 const unsigned Form231Index = 2; 2077 static const unsigned FormMapping[][3] = { 2078 // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2; 2079 // FMA132 A, C, b; ==> FMA231 C, A, b; 2080 // FMA213 B, A, c; ==> FMA213 A, B, c; 2081 // FMA231 C, A, b; ==> FMA132 A, C, b; 2082 {Form231Index, Form213Index, Form132Index}, 2083 // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3; 2084 // FMA132 A, c, B; ==> FMA132 B, c, A; 2085 // FMA213 B, a, C; ==> FMA231 C, a, B; 2086 // FMA231 C, a, B; ==> FMA213 B, a, C; 2087 {Form132Index, Form231Index, Form213Index}, 2088 // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3; 2089 // FMA132 a, C, B; ==> FMA213 a, B, C; 2090 // FMA213 b, A, C; ==> FMA132 b, C, A; 2091 // FMA231 c, A, B; ==> FMA231 c, B, A; 2092 {Form213Index, Form132Index, Form231Index}}; 2093 2094 unsigned FMAForms[3]; 2095 FMAForms[0] = FMA3Group.get132Opcode(); 2096 FMAForms[1] = FMA3Group.get213Opcode(); 2097 FMAForms[2] = FMA3Group.get231Opcode(); 2098 2099 // Everything is ready, just adjust the FMA opcode and return it. 2100 for (unsigned FormIndex = 0; FormIndex < 3; FormIndex++) 2101 if (Opc == FMAForms[FormIndex]) 2102 return FMAForms[FormMapping[Case][FormIndex]]; 2103 2104 llvm_unreachable("Illegal FMA3 format"); 2105 } 2106 2107 static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1, 2108 unsigned SrcOpIdx2) { 2109 // Determine which case this commute is or if it can't be done. 2110 unsigned Case = 2111 getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2); 2112 assert(Case < 3 && "Unexpected case value!"); 2113 2114 // For each case we need to swap two pairs of bits in the final immediate. 2115 static const uint8_t SwapMasks[3][4] = { 2116 {0x04, 0x10, 0x08, 0x20}, // Swap bits 2/4 and 3/5. 2117 {0x02, 0x10, 0x08, 0x40}, // Swap bits 1/4 and 3/6. 2118 {0x02, 0x04, 0x20, 0x40}, // Swap bits 1/2 and 5/6. 2119 }; 2120 2121 uint8_t Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); 2122 // Clear out the bits we are swapping. 2123 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] | 2124 SwapMasks[Case][2] | SwapMasks[Case][3]); 2125 // If the immediate had a bit of the pair set, then set the opposite bit. 2126 if (Imm & SwapMasks[Case][0]) 2127 NewImm |= SwapMasks[Case][1]; 2128 if (Imm & SwapMasks[Case][1]) 2129 NewImm |= SwapMasks[Case][0]; 2130 if (Imm & SwapMasks[Case][2]) 2131 NewImm |= SwapMasks[Case][3]; 2132 if (Imm & SwapMasks[Case][3]) 2133 NewImm |= SwapMasks[Case][2]; 2134 MI.getOperand(MI.getNumOperands() - 1).setImm(NewImm); 2135 } 2136 2137 // Returns true if this is a VPERMI2 or VPERMT2 instruction that can be 2138 // commuted. 2139 static bool isCommutableVPERMV3Instruction(unsigned Opcode) { 2140 #define VPERM_CASES(Suffix) \ 2141 case X86::VPERMI2##Suffix##Z128rr: \ 2142 case X86::VPERMT2##Suffix##Z128rr: \ 2143 case X86::VPERMI2##Suffix##Z256rr: \ 2144 case X86::VPERMT2##Suffix##Z256rr: \ 2145 case X86::VPERMI2##Suffix##Zrr: \ 2146 case X86::VPERMT2##Suffix##Zrr: \ 2147 case X86::VPERMI2##Suffix##Z128rm: \ 2148 case X86::VPERMT2##Suffix##Z128rm: \ 2149 case X86::VPERMI2##Suffix##Z256rm: \ 2150 case X86::VPERMT2##Suffix##Z256rm: \ 2151 case X86::VPERMI2##Suffix##Zrm: \ 2152 case X86::VPERMT2##Suffix##Zrm: \ 2153 case X86::VPERMI2##Suffix##Z128rrkz: \ 2154 case X86::VPERMT2##Suffix##Z128rrkz: \ 2155 case X86::VPERMI2##Suffix##Z256rrkz: \ 2156 case X86::VPERMT2##Suffix##Z256rrkz: \ 2157 case X86::VPERMI2##Suffix##Zrrkz: \ 2158 case X86::VPERMT2##Suffix##Zrrkz: \ 2159 case X86::VPERMI2##Suffix##Z128rmkz: \ 2160 case X86::VPERMT2##Suffix##Z128rmkz: \ 2161 case X86::VPERMI2##Suffix##Z256rmkz: \ 2162 case X86::VPERMT2##Suffix##Z256rmkz: \ 2163 case X86::VPERMI2##Suffix##Zrmkz: \ 2164 case X86::VPERMT2##Suffix##Zrmkz: 2165 2166 #define VPERM_CASES_BROADCAST(Suffix) \ 2167 VPERM_CASES(Suffix) \ 2168 case X86::VPERMI2##Suffix##Z128rmb: \ 2169 case X86::VPERMT2##Suffix##Z128rmb: \ 2170 case X86::VPERMI2##Suffix##Z256rmb: \ 2171 case X86::VPERMT2##Suffix##Z256rmb: \ 2172 case X86::VPERMI2##Suffix##Zrmb: \ 2173 case X86::VPERMT2##Suffix##Zrmb: \ 2174 case X86::VPERMI2##Suffix##Z128rmbkz: \ 2175 case X86::VPERMT2##Suffix##Z128rmbkz: \ 2176 case X86::VPERMI2##Suffix##Z256rmbkz: \ 2177 case X86::VPERMT2##Suffix##Z256rmbkz: \ 2178 case X86::VPERMI2##Suffix##Zrmbkz: \ 2179 case X86::VPERMT2##Suffix##Zrmbkz: 2180 2181 switch (Opcode) { 2182 default: 2183 return false; 2184 VPERM_CASES(B) 2185 VPERM_CASES_BROADCAST(D) 2186 VPERM_CASES_BROADCAST(PD) 2187 VPERM_CASES_BROADCAST(PS) 2188 VPERM_CASES_BROADCAST(Q) 2189 VPERM_CASES(W) 2190 return true; 2191 } 2192 #undef VPERM_CASES_BROADCAST 2193 #undef VPERM_CASES 2194 } 2195 2196 // Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching 2197 // from the I opcode to the T opcode and vice versa. 2198 static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) { 2199 #define VPERM_CASES(Orig, New) \ 2200 case X86::Orig##Z128rr: \ 2201 return X86::New##Z128rr; \ 2202 case X86::Orig##Z128rrkz: \ 2203 return X86::New##Z128rrkz; \ 2204 case X86::Orig##Z128rm: \ 2205 return X86::New##Z128rm; \ 2206 case X86::Orig##Z128rmkz: \ 2207 return X86::New##Z128rmkz; \ 2208 case X86::Orig##Z256rr: \ 2209 return X86::New##Z256rr; \ 2210 case X86::Orig##Z256rrkz: \ 2211 return X86::New##Z256rrkz; \ 2212 case X86::Orig##Z256rm: \ 2213 return X86::New##Z256rm; \ 2214 case X86::Orig##Z256rmkz: \ 2215 return X86::New##Z256rmkz; \ 2216 case X86::Orig##Zrr: \ 2217 return X86::New##Zrr; \ 2218 case X86::Orig##Zrrkz: \ 2219 return X86::New##Zrrkz; \ 2220 case X86::Orig##Zrm: \ 2221 return X86::New##Zrm; \ 2222 case X86::Orig##Zrmkz: \ 2223 return X86::New##Zrmkz; 2224 2225 #define VPERM_CASES_BROADCAST(Orig, New) \ 2226 VPERM_CASES(Orig, New) \ 2227 case X86::Orig##Z128rmb: \ 2228 return X86::New##Z128rmb; \ 2229 case X86::Orig##Z128rmbkz: \ 2230 return X86::New##Z128rmbkz; \ 2231 case X86::Orig##Z256rmb: \ 2232 return X86::New##Z256rmb; \ 2233 case X86::Orig##Z256rmbkz: \ 2234 return X86::New##Z256rmbkz; \ 2235 case X86::Orig##Zrmb: \ 2236 return X86::New##Zrmb; \ 2237 case X86::Orig##Zrmbkz: \ 2238 return X86::New##Zrmbkz; 2239 2240 switch (Opcode) { 2241 VPERM_CASES(VPERMI2B, VPERMT2B) 2242 VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D) 2243 VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD) 2244 VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS) 2245 VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q) 2246 VPERM_CASES(VPERMI2W, VPERMT2W) 2247 VPERM_CASES(VPERMT2B, VPERMI2B) 2248 VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D) 2249 VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD) 2250 VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS) 2251 VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q) 2252 VPERM_CASES(VPERMT2W, VPERMI2W) 2253 } 2254 2255 llvm_unreachable("Unreachable!"); 2256 #undef VPERM_CASES_BROADCAST 2257 #undef VPERM_CASES 2258 } 2259 2260 MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 2261 unsigned OpIdx1, 2262 unsigned OpIdx2) const { 2263 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 2264 if (NewMI) 2265 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 2266 return MI; 2267 }; 2268 2269 switch (MI.getOpcode()) { 2270 case X86::SHRD16rri8: // A = SHRD16rri8 B, C, I -> A = SHLD16rri8 C, B, (16-I) 2271 case X86::SHLD16rri8: // A = SHLD16rri8 B, C, I -> A = SHRD16rri8 C, B, (16-I) 2272 case X86::SHRD32rri8: // A = SHRD32rri8 B, C, I -> A = SHLD32rri8 C, B, (32-I) 2273 case X86::SHLD32rri8: // A = SHLD32rri8 B, C, I -> A = SHRD32rri8 C, B, (32-I) 2274 case X86::SHRD64rri8: // A = SHRD64rri8 B, C, I -> A = SHLD64rri8 C, B, (64-I) 2275 case X86::SHLD64rri8: { // A = SHLD64rri8 B, C, I -> A = SHRD64rri8 C, B, 2276 // (64-I) 2277 unsigned Opc; 2278 unsigned Size; 2279 switch (MI.getOpcode()) { 2280 default: 2281 llvm_unreachable("Unreachable!"); 2282 case X86::SHRD16rri8: 2283 Size = 16; 2284 Opc = X86::SHLD16rri8; 2285 break; 2286 case X86::SHLD16rri8: 2287 Size = 16; 2288 Opc = X86::SHRD16rri8; 2289 break; 2290 case X86::SHRD32rri8: 2291 Size = 32; 2292 Opc = X86::SHLD32rri8; 2293 break; 2294 case X86::SHLD32rri8: 2295 Size = 32; 2296 Opc = X86::SHRD32rri8; 2297 break; 2298 case X86::SHRD64rri8: 2299 Size = 64; 2300 Opc = X86::SHLD64rri8; 2301 break; 2302 case X86::SHLD64rri8: 2303 Size = 64; 2304 Opc = X86::SHRD64rri8; 2305 break; 2306 } 2307 unsigned Amt = MI.getOperand(3).getImm(); 2308 auto &WorkingMI = cloneIfNew(MI); 2309 WorkingMI.setDesc(get(Opc)); 2310 WorkingMI.getOperand(3).setImm(Size - Amt); 2311 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2312 OpIdx1, OpIdx2); 2313 } 2314 case X86::PFSUBrr: 2315 case X86::PFSUBRrr: { 2316 // PFSUB x, y: x = x - y 2317 // PFSUBR x, y: x = y - x 2318 unsigned Opc = 2319 (X86::PFSUBRrr == MI.getOpcode() ? X86::PFSUBrr : X86::PFSUBRrr); 2320 auto &WorkingMI = cloneIfNew(MI); 2321 WorkingMI.setDesc(get(Opc)); 2322 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2323 OpIdx1, OpIdx2); 2324 } 2325 case X86::BLENDPDrri: 2326 case X86::BLENDPSrri: 2327 case X86::VBLENDPDrri: 2328 case X86::VBLENDPSrri: 2329 // If we're optimizing for size, try to use MOVSD/MOVSS. 2330 if (MI.getParent()->getParent()->getFunction().hasOptSize()) { 2331 unsigned Mask, Opc; 2332 switch (MI.getOpcode()) { 2333 default: 2334 llvm_unreachable("Unreachable!"); 2335 case X86::BLENDPDrri: 2336 Opc = X86::MOVSDrr; 2337 Mask = 0x03; 2338 break; 2339 case X86::BLENDPSrri: 2340 Opc = X86::MOVSSrr; 2341 Mask = 0x0F; 2342 break; 2343 case X86::VBLENDPDrri: 2344 Opc = X86::VMOVSDrr; 2345 Mask = 0x03; 2346 break; 2347 case X86::VBLENDPSrri: 2348 Opc = X86::VMOVSSrr; 2349 Mask = 0x0F; 2350 break; 2351 } 2352 if ((MI.getOperand(3).getImm() ^ Mask) == 1) { 2353 auto &WorkingMI = cloneIfNew(MI); 2354 WorkingMI.setDesc(get(Opc)); 2355 WorkingMI.removeOperand(3); 2356 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, 2357 /*NewMI=*/false, OpIdx1, 2358 OpIdx2); 2359 } 2360 } 2361 [[fallthrough]]; 2362 case X86::PBLENDWrri: 2363 case X86::VBLENDPDYrri: 2364 case X86::VBLENDPSYrri: 2365 case X86::VPBLENDDrri: 2366 case X86::VPBLENDWrri: 2367 case X86::VPBLENDDYrri: 2368 case X86::VPBLENDWYrri: { 2369 int8_t Mask; 2370 switch (MI.getOpcode()) { 2371 default: 2372 llvm_unreachable("Unreachable!"); 2373 case X86::BLENDPDrri: 2374 Mask = (int8_t)0x03; 2375 break; 2376 case X86::BLENDPSrri: 2377 Mask = (int8_t)0x0F; 2378 break; 2379 case X86::PBLENDWrri: 2380 Mask = (int8_t)0xFF; 2381 break; 2382 case X86::VBLENDPDrri: 2383 Mask = (int8_t)0x03; 2384 break; 2385 case X86::VBLENDPSrri: 2386 Mask = (int8_t)0x0F; 2387 break; 2388 case X86::VBLENDPDYrri: 2389 Mask = (int8_t)0x0F; 2390 break; 2391 case X86::VBLENDPSYrri: 2392 Mask = (int8_t)0xFF; 2393 break; 2394 case X86::VPBLENDDrri: 2395 Mask = (int8_t)0x0F; 2396 break; 2397 case X86::VPBLENDWrri: 2398 Mask = (int8_t)0xFF; 2399 break; 2400 case X86::VPBLENDDYrri: 2401 Mask = (int8_t)0xFF; 2402 break; 2403 case X86::VPBLENDWYrri: 2404 Mask = (int8_t)0xFF; 2405 break; 2406 } 2407 // Only the least significant bits of Imm are used. 2408 // Using int8_t to ensure it will be sign extended to the int64_t that 2409 // setImm takes in order to match isel behavior. 2410 int8_t Imm = MI.getOperand(3).getImm() & Mask; 2411 auto &WorkingMI = cloneIfNew(MI); 2412 WorkingMI.getOperand(3).setImm(Mask ^ Imm); 2413 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2414 OpIdx1, OpIdx2); 2415 } 2416 case X86::INSERTPSrr: 2417 case X86::VINSERTPSrr: 2418 case X86::VINSERTPSZrr: { 2419 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); 2420 unsigned ZMask = Imm & 15; 2421 unsigned DstIdx = (Imm >> 4) & 3; 2422 unsigned SrcIdx = (Imm >> 6) & 3; 2423 2424 // We can commute insertps if we zero 2 of the elements, the insertion is 2425 // "inline" and we don't override the insertion with a zero. 2426 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 && 2427 llvm::popcount(ZMask) == 2) { 2428 unsigned AltIdx = llvm::countr_zero((ZMask | (1 << DstIdx)) ^ 15); 2429 assert(AltIdx < 4 && "Illegal insertion index"); 2430 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask; 2431 auto &WorkingMI = cloneIfNew(MI); 2432 WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(AltImm); 2433 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2434 OpIdx1, OpIdx2); 2435 } 2436 return nullptr; 2437 } 2438 case X86::MOVSDrr: 2439 case X86::MOVSSrr: 2440 case X86::VMOVSDrr: 2441 case X86::VMOVSSrr: { 2442 // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD. 2443 if (Subtarget.hasSSE41()) { 2444 unsigned Mask, Opc; 2445 switch (MI.getOpcode()) { 2446 default: 2447 llvm_unreachable("Unreachable!"); 2448 case X86::MOVSDrr: 2449 Opc = X86::BLENDPDrri; 2450 Mask = 0x02; 2451 break; 2452 case X86::MOVSSrr: 2453 Opc = X86::BLENDPSrri; 2454 Mask = 0x0E; 2455 break; 2456 case X86::VMOVSDrr: 2457 Opc = X86::VBLENDPDrri; 2458 Mask = 0x02; 2459 break; 2460 case X86::VMOVSSrr: 2461 Opc = X86::VBLENDPSrri; 2462 Mask = 0x0E; 2463 break; 2464 } 2465 2466 auto &WorkingMI = cloneIfNew(MI); 2467 WorkingMI.setDesc(get(Opc)); 2468 WorkingMI.addOperand(MachineOperand::CreateImm(Mask)); 2469 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2470 OpIdx1, OpIdx2); 2471 } 2472 2473 // Convert to SHUFPD. 2474 assert(MI.getOpcode() == X86::MOVSDrr && 2475 "Can only commute MOVSDrr without SSE4.1"); 2476 2477 auto &WorkingMI = cloneIfNew(MI); 2478 WorkingMI.setDesc(get(X86::SHUFPDrri)); 2479 WorkingMI.addOperand(MachineOperand::CreateImm(0x02)); 2480 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2481 OpIdx1, OpIdx2); 2482 } 2483 case X86::SHUFPDrri: { 2484 // Commute to MOVSD. 2485 assert(MI.getOperand(3).getImm() == 0x02 && "Unexpected immediate!"); 2486 auto &WorkingMI = cloneIfNew(MI); 2487 WorkingMI.setDesc(get(X86::MOVSDrr)); 2488 WorkingMI.removeOperand(3); 2489 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2490 OpIdx1, OpIdx2); 2491 } 2492 case X86::PCLMULQDQrr: 2493 case X86::VPCLMULQDQrr: 2494 case X86::VPCLMULQDQYrr: 2495 case X86::VPCLMULQDQZrr: 2496 case X86::VPCLMULQDQZ128rr: 2497 case X86::VPCLMULQDQZ256rr: { 2498 // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0] 2499 // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0] 2500 unsigned Imm = MI.getOperand(3).getImm(); 2501 unsigned Src1Hi = Imm & 0x01; 2502 unsigned Src2Hi = Imm & 0x10; 2503 auto &WorkingMI = cloneIfNew(MI); 2504 WorkingMI.getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4)); 2505 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2506 OpIdx1, OpIdx2); 2507 } 2508 case X86::VPCMPBZ128rri: 2509 case X86::VPCMPUBZ128rri: 2510 case X86::VPCMPBZ256rri: 2511 case X86::VPCMPUBZ256rri: 2512 case X86::VPCMPBZrri: 2513 case X86::VPCMPUBZrri: 2514 case X86::VPCMPDZ128rri: 2515 case X86::VPCMPUDZ128rri: 2516 case X86::VPCMPDZ256rri: 2517 case X86::VPCMPUDZ256rri: 2518 case X86::VPCMPDZrri: 2519 case X86::VPCMPUDZrri: 2520 case X86::VPCMPQZ128rri: 2521 case X86::VPCMPUQZ128rri: 2522 case X86::VPCMPQZ256rri: 2523 case X86::VPCMPUQZ256rri: 2524 case X86::VPCMPQZrri: 2525 case X86::VPCMPUQZrri: 2526 case X86::VPCMPWZ128rri: 2527 case X86::VPCMPUWZ128rri: 2528 case X86::VPCMPWZ256rri: 2529 case X86::VPCMPUWZ256rri: 2530 case X86::VPCMPWZrri: 2531 case X86::VPCMPUWZrri: 2532 case X86::VPCMPBZ128rrik: 2533 case X86::VPCMPUBZ128rrik: 2534 case X86::VPCMPBZ256rrik: 2535 case X86::VPCMPUBZ256rrik: 2536 case X86::VPCMPBZrrik: 2537 case X86::VPCMPUBZrrik: 2538 case X86::VPCMPDZ128rrik: 2539 case X86::VPCMPUDZ128rrik: 2540 case X86::VPCMPDZ256rrik: 2541 case X86::VPCMPUDZ256rrik: 2542 case X86::VPCMPDZrrik: 2543 case X86::VPCMPUDZrrik: 2544 case X86::VPCMPQZ128rrik: 2545 case X86::VPCMPUQZ128rrik: 2546 case X86::VPCMPQZ256rrik: 2547 case X86::VPCMPUQZ256rrik: 2548 case X86::VPCMPQZrrik: 2549 case X86::VPCMPUQZrrik: 2550 case X86::VPCMPWZ128rrik: 2551 case X86::VPCMPUWZ128rrik: 2552 case X86::VPCMPWZ256rrik: 2553 case X86::VPCMPUWZ256rrik: 2554 case X86::VPCMPWZrrik: 2555 case X86::VPCMPUWZrrik: { 2556 // Flip comparison mode immediate (if necessary). 2557 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7; 2558 Imm = X86::getSwappedVPCMPImm(Imm); 2559 auto &WorkingMI = cloneIfNew(MI); 2560 WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(Imm); 2561 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2562 OpIdx1, OpIdx2); 2563 } 2564 case X86::VPCOMBri: 2565 case X86::VPCOMUBri: 2566 case X86::VPCOMDri: 2567 case X86::VPCOMUDri: 2568 case X86::VPCOMQri: 2569 case X86::VPCOMUQri: 2570 case X86::VPCOMWri: 2571 case X86::VPCOMUWri: { 2572 // Flip comparison mode immediate (if necessary). 2573 unsigned Imm = MI.getOperand(3).getImm() & 0x7; 2574 Imm = X86::getSwappedVPCOMImm(Imm); 2575 auto &WorkingMI = cloneIfNew(MI); 2576 WorkingMI.getOperand(3).setImm(Imm); 2577 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2578 OpIdx1, OpIdx2); 2579 } 2580 case X86::VCMPSDZrr: 2581 case X86::VCMPSSZrr: 2582 case X86::VCMPPDZrri: 2583 case X86::VCMPPSZrri: 2584 case X86::VCMPSHZrr: 2585 case X86::VCMPPHZrri: 2586 case X86::VCMPPHZ128rri: 2587 case X86::VCMPPHZ256rri: 2588 case X86::VCMPPDZ128rri: 2589 case X86::VCMPPSZ128rri: 2590 case X86::VCMPPDZ256rri: 2591 case X86::VCMPPSZ256rri: 2592 case X86::VCMPPDZrrik: 2593 case X86::VCMPPSZrrik: 2594 case X86::VCMPPDZ128rrik: 2595 case X86::VCMPPSZ128rrik: 2596 case X86::VCMPPDZ256rrik: 2597 case X86::VCMPPSZ256rrik: { 2598 unsigned Imm = 2599 MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 0x1f; 2600 Imm = X86::getSwappedVCMPImm(Imm); 2601 auto &WorkingMI = cloneIfNew(MI); 2602 WorkingMI.getOperand(MI.getNumExplicitOperands() - 1).setImm(Imm); 2603 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2604 OpIdx1, OpIdx2); 2605 } 2606 case X86::VPERM2F128rr: 2607 case X86::VPERM2I128rr: { 2608 // Flip permute source immediate. 2609 // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi. 2610 // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi. 2611 int8_t Imm = MI.getOperand(3).getImm() & 0xFF; 2612 auto &WorkingMI = cloneIfNew(MI); 2613 WorkingMI.getOperand(3).setImm(Imm ^ 0x22); 2614 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2615 OpIdx1, OpIdx2); 2616 } 2617 case X86::MOVHLPSrr: 2618 case X86::UNPCKHPDrr: 2619 case X86::VMOVHLPSrr: 2620 case X86::VUNPCKHPDrr: 2621 case X86::VMOVHLPSZrr: 2622 case X86::VUNPCKHPDZ128rr: { 2623 assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!"); 2624 2625 unsigned Opc = MI.getOpcode(); 2626 switch (Opc) { 2627 default: 2628 llvm_unreachable("Unreachable!"); 2629 case X86::MOVHLPSrr: 2630 Opc = X86::UNPCKHPDrr; 2631 break; 2632 case X86::UNPCKHPDrr: 2633 Opc = X86::MOVHLPSrr; 2634 break; 2635 case X86::VMOVHLPSrr: 2636 Opc = X86::VUNPCKHPDrr; 2637 break; 2638 case X86::VUNPCKHPDrr: 2639 Opc = X86::VMOVHLPSrr; 2640 break; 2641 case X86::VMOVHLPSZrr: 2642 Opc = X86::VUNPCKHPDZ128rr; 2643 break; 2644 case X86::VUNPCKHPDZ128rr: 2645 Opc = X86::VMOVHLPSZrr; 2646 break; 2647 } 2648 auto &WorkingMI = cloneIfNew(MI); 2649 WorkingMI.setDesc(get(Opc)); 2650 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2651 OpIdx1, OpIdx2); 2652 } 2653 case X86::CMOV16rr: 2654 case X86::CMOV32rr: 2655 case X86::CMOV64rr: { 2656 auto &WorkingMI = cloneIfNew(MI); 2657 unsigned OpNo = MI.getDesc().getNumOperands() - 1; 2658 X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm()); 2659 WorkingMI.getOperand(OpNo).setImm(X86::GetOppositeBranchCondition(CC)); 2660 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2661 OpIdx1, OpIdx2); 2662 } 2663 case X86::VPTERNLOGDZrri: 2664 case X86::VPTERNLOGDZrmi: 2665 case X86::VPTERNLOGDZ128rri: 2666 case X86::VPTERNLOGDZ128rmi: 2667 case X86::VPTERNLOGDZ256rri: 2668 case X86::VPTERNLOGDZ256rmi: 2669 case X86::VPTERNLOGQZrri: 2670 case X86::VPTERNLOGQZrmi: 2671 case X86::VPTERNLOGQZ128rri: 2672 case X86::VPTERNLOGQZ128rmi: 2673 case X86::VPTERNLOGQZ256rri: 2674 case X86::VPTERNLOGQZ256rmi: 2675 case X86::VPTERNLOGDZrrik: 2676 case X86::VPTERNLOGDZ128rrik: 2677 case X86::VPTERNLOGDZ256rrik: 2678 case X86::VPTERNLOGQZrrik: 2679 case X86::VPTERNLOGQZ128rrik: 2680 case X86::VPTERNLOGQZ256rrik: 2681 case X86::VPTERNLOGDZrrikz: 2682 case X86::VPTERNLOGDZrmikz: 2683 case X86::VPTERNLOGDZ128rrikz: 2684 case X86::VPTERNLOGDZ128rmikz: 2685 case X86::VPTERNLOGDZ256rrikz: 2686 case X86::VPTERNLOGDZ256rmikz: 2687 case X86::VPTERNLOGQZrrikz: 2688 case X86::VPTERNLOGQZrmikz: 2689 case X86::VPTERNLOGQZ128rrikz: 2690 case X86::VPTERNLOGQZ128rmikz: 2691 case X86::VPTERNLOGQZ256rrikz: 2692 case X86::VPTERNLOGQZ256rmikz: 2693 case X86::VPTERNLOGDZ128rmbi: 2694 case X86::VPTERNLOGDZ256rmbi: 2695 case X86::VPTERNLOGDZrmbi: 2696 case X86::VPTERNLOGQZ128rmbi: 2697 case X86::VPTERNLOGQZ256rmbi: 2698 case X86::VPTERNLOGQZrmbi: 2699 case X86::VPTERNLOGDZ128rmbikz: 2700 case X86::VPTERNLOGDZ256rmbikz: 2701 case X86::VPTERNLOGDZrmbikz: 2702 case X86::VPTERNLOGQZ128rmbikz: 2703 case X86::VPTERNLOGQZ256rmbikz: 2704 case X86::VPTERNLOGQZrmbikz: { 2705 auto &WorkingMI = cloneIfNew(MI); 2706 commuteVPTERNLOG(WorkingMI, OpIdx1, OpIdx2); 2707 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2708 OpIdx1, OpIdx2); 2709 } 2710 default: { 2711 if (isCommutableVPERMV3Instruction(MI.getOpcode())) { 2712 unsigned Opc = getCommutedVPERMV3Opcode(MI.getOpcode()); 2713 auto &WorkingMI = cloneIfNew(MI); 2714 WorkingMI.setDesc(get(Opc)); 2715 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2716 OpIdx1, OpIdx2); 2717 } 2718 2719 const X86InstrFMA3Group *FMA3Group = 2720 getFMA3Group(MI.getOpcode(), MI.getDesc().TSFlags); 2721 if (FMA3Group) { 2722 unsigned Opc = 2723 getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group); 2724 auto &WorkingMI = cloneIfNew(MI); 2725 WorkingMI.setDesc(get(Opc)); 2726 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 2727 OpIdx1, OpIdx2); 2728 } 2729 2730 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 2731 } 2732 } 2733 } 2734 2735 bool X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI, 2736 unsigned &SrcOpIdx1, 2737 unsigned &SrcOpIdx2, 2738 bool IsIntrinsic) const { 2739 uint64_t TSFlags = MI.getDesc().TSFlags; 2740 2741 unsigned FirstCommutableVecOp = 1; 2742 unsigned LastCommutableVecOp = 3; 2743 unsigned KMaskOp = -1U; 2744 if (X86II::isKMasked(TSFlags)) { 2745 // For k-zero-masked operations it is Ok to commute the first vector 2746 // operand. Unless this is an intrinsic instruction. 2747 // For regular k-masked operations a conservative choice is done as the 2748 // elements of the first vector operand, for which the corresponding bit 2749 // in the k-mask operand is set to 0, are copied to the result of the 2750 // instruction. 2751 // TODO/FIXME: The commute still may be legal if it is known that the 2752 // k-mask operand is set to either all ones or all zeroes. 2753 // It is also Ok to commute the 1st operand if all users of MI use only 2754 // the elements enabled by the k-mask operand. For example, 2755 // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i] 2756 // : v1[i]; 2757 // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 -> 2758 // // Ok, to commute v1 in FMADD213PSZrk. 2759 2760 // The k-mask operand has index = 2 for masked and zero-masked operations. 2761 KMaskOp = 2; 2762 2763 // The operand with index = 1 is used as a source for those elements for 2764 // which the corresponding bit in the k-mask is set to 0. 2765 if (X86II::isKMergeMasked(TSFlags) || IsIntrinsic) 2766 FirstCommutableVecOp = 3; 2767 2768 LastCommutableVecOp++; 2769 } else if (IsIntrinsic) { 2770 // Commuting the first operand of an intrinsic instruction isn't possible 2771 // unless we can prove that only the lowest element of the result is used. 2772 FirstCommutableVecOp = 2; 2773 } 2774 2775 if (isMem(MI, LastCommutableVecOp)) 2776 LastCommutableVecOp--; 2777 2778 // Only the first RegOpsNum operands are commutable. 2779 // Also, the value 'CommuteAnyOperandIndex' is valid here as it means 2780 // that the operand is not specified/fixed. 2781 if (SrcOpIdx1 != CommuteAnyOperandIndex && 2782 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp || 2783 SrcOpIdx1 == KMaskOp)) 2784 return false; 2785 if (SrcOpIdx2 != CommuteAnyOperandIndex && 2786 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp || 2787 SrcOpIdx2 == KMaskOp)) 2788 return false; 2789 2790 // Look for two different register operands assumed to be commutable 2791 // regardless of the FMA opcode. The FMA opcode is adjusted later. 2792 if (SrcOpIdx1 == CommuteAnyOperandIndex || 2793 SrcOpIdx2 == CommuteAnyOperandIndex) { 2794 unsigned CommutableOpIdx2 = SrcOpIdx2; 2795 2796 // At least one of operands to be commuted is not specified and 2797 // this method is free to choose appropriate commutable operands. 2798 if (SrcOpIdx1 == SrcOpIdx2) 2799 // Both of operands are not fixed. By default set one of commutable 2800 // operands to the last register operand of the instruction. 2801 CommutableOpIdx2 = LastCommutableVecOp; 2802 else if (SrcOpIdx2 == CommuteAnyOperandIndex) 2803 // Only one of operands is not fixed. 2804 CommutableOpIdx2 = SrcOpIdx1; 2805 2806 // CommutableOpIdx2 is well defined now. Let's choose another commutable 2807 // operand and assign its index to CommutableOpIdx1. 2808 Register Op2Reg = MI.getOperand(CommutableOpIdx2).getReg(); 2809 2810 unsigned CommutableOpIdx1; 2811 for (CommutableOpIdx1 = LastCommutableVecOp; 2812 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) { 2813 // Just ignore and skip the k-mask operand. 2814 if (CommutableOpIdx1 == KMaskOp) 2815 continue; 2816 2817 // The commuted operands must have different registers. 2818 // Otherwise, the commute transformation does not change anything and 2819 // is useless then. 2820 if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg()) 2821 break; 2822 } 2823 2824 // No appropriate commutable operands were found. 2825 if (CommutableOpIdx1 < FirstCommutableVecOp) 2826 return false; 2827 2828 // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2 2829 // to return those values. 2830 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1, 2831 CommutableOpIdx2)) 2832 return false; 2833 } 2834 2835 return true; 2836 } 2837 2838 bool X86InstrInfo::findCommutedOpIndices(const MachineInstr &MI, 2839 unsigned &SrcOpIdx1, 2840 unsigned &SrcOpIdx2) const { 2841 const MCInstrDesc &Desc = MI.getDesc(); 2842 if (!Desc.isCommutable()) 2843 return false; 2844 2845 switch (MI.getOpcode()) { 2846 case X86::CMPSDrr: 2847 case X86::CMPSSrr: 2848 case X86::CMPPDrri: 2849 case X86::CMPPSrri: 2850 case X86::VCMPSDrr: 2851 case X86::VCMPSSrr: 2852 case X86::VCMPPDrri: 2853 case X86::VCMPPSrri: 2854 case X86::VCMPPDYrri: 2855 case X86::VCMPPSYrri: 2856 case X86::VCMPSDZrr: 2857 case X86::VCMPSSZrr: 2858 case X86::VCMPPDZrri: 2859 case X86::VCMPPSZrri: 2860 case X86::VCMPSHZrr: 2861 case X86::VCMPPHZrri: 2862 case X86::VCMPPHZ128rri: 2863 case X86::VCMPPHZ256rri: 2864 case X86::VCMPPDZ128rri: 2865 case X86::VCMPPSZ128rri: 2866 case X86::VCMPPDZ256rri: 2867 case X86::VCMPPSZ256rri: 2868 case X86::VCMPPDZrrik: 2869 case X86::VCMPPSZrrik: 2870 case X86::VCMPPDZ128rrik: 2871 case X86::VCMPPSZ128rrik: 2872 case X86::VCMPPDZ256rrik: 2873 case X86::VCMPPSZ256rrik: { 2874 unsigned OpOffset = X86II::isKMasked(Desc.TSFlags) ? 1 : 0; 2875 2876 // Float comparison can be safely commuted for 2877 // Ordered/Unordered/Equal/NotEqual tests 2878 unsigned Imm = MI.getOperand(3 + OpOffset).getImm() & 0x7; 2879 switch (Imm) { 2880 default: 2881 // EVEX versions can be commuted. 2882 if ((Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX) 2883 break; 2884 return false; 2885 case 0x00: // EQUAL 2886 case 0x03: // UNORDERED 2887 case 0x04: // NOT EQUAL 2888 case 0x07: // ORDERED 2889 break; 2890 } 2891 2892 // The indices of the commutable operands are 1 and 2 (or 2 and 3 2893 // when masked). 2894 // Assign them to the returned operand indices here. 2895 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset, 2896 2 + OpOffset); 2897 } 2898 case X86::MOVSSrr: 2899 // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can 2900 // form sse4.1 blend. We assume VMOVSSrr/VMOVSDrr is always commutable since 2901 // AVX implies sse4.1. 2902 if (Subtarget.hasSSE41()) 2903 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2904 return false; 2905 case X86::SHUFPDrri: 2906 // We can commute this to MOVSD. 2907 if (MI.getOperand(3).getImm() == 0x02) 2908 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2909 return false; 2910 case X86::MOVHLPSrr: 2911 case X86::UNPCKHPDrr: 2912 case X86::VMOVHLPSrr: 2913 case X86::VUNPCKHPDrr: 2914 case X86::VMOVHLPSZrr: 2915 case X86::VUNPCKHPDZ128rr: 2916 if (Subtarget.hasSSE2()) 2917 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2918 return false; 2919 case X86::VPTERNLOGDZrri: 2920 case X86::VPTERNLOGDZrmi: 2921 case X86::VPTERNLOGDZ128rri: 2922 case X86::VPTERNLOGDZ128rmi: 2923 case X86::VPTERNLOGDZ256rri: 2924 case X86::VPTERNLOGDZ256rmi: 2925 case X86::VPTERNLOGQZrri: 2926 case X86::VPTERNLOGQZrmi: 2927 case X86::VPTERNLOGQZ128rri: 2928 case X86::VPTERNLOGQZ128rmi: 2929 case X86::VPTERNLOGQZ256rri: 2930 case X86::VPTERNLOGQZ256rmi: 2931 case X86::VPTERNLOGDZrrik: 2932 case X86::VPTERNLOGDZ128rrik: 2933 case X86::VPTERNLOGDZ256rrik: 2934 case X86::VPTERNLOGQZrrik: 2935 case X86::VPTERNLOGQZ128rrik: 2936 case X86::VPTERNLOGQZ256rrik: 2937 case X86::VPTERNLOGDZrrikz: 2938 case X86::VPTERNLOGDZrmikz: 2939 case X86::VPTERNLOGDZ128rrikz: 2940 case X86::VPTERNLOGDZ128rmikz: 2941 case X86::VPTERNLOGDZ256rrikz: 2942 case X86::VPTERNLOGDZ256rmikz: 2943 case X86::VPTERNLOGQZrrikz: 2944 case X86::VPTERNLOGQZrmikz: 2945 case X86::VPTERNLOGQZ128rrikz: 2946 case X86::VPTERNLOGQZ128rmikz: 2947 case X86::VPTERNLOGQZ256rrikz: 2948 case X86::VPTERNLOGQZ256rmikz: 2949 case X86::VPTERNLOGDZ128rmbi: 2950 case X86::VPTERNLOGDZ256rmbi: 2951 case X86::VPTERNLOGDZrmbi: 2952 case X86::VPTERNLOGQZ128rmbi: 2953 case X86::VPTERNLOGQZ256rmbi: 2954 case X86::VPTERNLOGQZrmbi: 2955 case X86::VPTERNLOGDZ128rmbikz: 2956 case X86::VPTERNLOGDZ256rmbikz: 2957 case X86::VPTERNLOGDZrmbikz: 2958 case X86::VPTERNLOGQZ128rmbikz: 2959 case X86::VPTERNLOGQZ256rmbikz: 2960 case X86::VPTERNLOGQZrmbikz: 2961 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2962 case X86::VPDPWSSDYrr: 2963 case X86::VPDPWSSDrr: 2964 case X86::VPDPWSSDSYrr: 2965 case X86::VPDPWSSDSrr: 2966 case X86::VPDPWUUDrr: 2967 case X86::VPDPWUUDYrr: 2968 case X86::VPDPWUUDSrr: 2969 case X86::VPDPWUUDSYrr: 2970 case X86::VPDPBSSDSrr: 2971 case X86::VPDPBSSDSYrr: 2972 case X86::VPDPBSSDrr: 2973 case X86::VPDPBSSDYrr: 2974 case X86::VPDPBUUDSrr: 2975 case X86::VPDPBUUDSYrr: 2976 case X86::VPDPBUUDrr: 2977 case X86::VPDPBUUDYrr: 2978 case X86::VPDPWSSDZ128r: 2979 case X86::VPDPWSSDZ128rk: 2980 case X86::VPDPWSSDZ128rkz: 2981 case X86::VPDPWSSDZ256r: 2982 case X86::VPDPWSSDZ256rk: 2983 case X86::VPDPWSSDZ256rkz: 2984 case X86::VPDPWSSDZr: 2985 case X86::VPDPWSSDZrk: 2986 case X86::VPDPWSSDZrkz: 2987 case X86::VPDPWSSDSZ128r: 2988 case X86::VPDPWSSDSZ128rk: 2989 case X86::VPDPWSSDSZ128rkz: 2990 case X86::VPDPWSSDSZ256r: 2991 case X86::VPDPWSSDSZ256rk: 2992 case X86::VPDPWSSDSZ256rkz: 2993 case X86::VPDPWSSDSZr: 2994 case X86::VPDPWSSDSZrk: 2995 case X86::VPDPWSSDSZrkz: 2996 case X86::VPMADD52HUQrr: 2997 case X86::VPMADD52HUQYrr: 2998 case X86::VPMADD52HUQZ128r: 2999 case X86::VPMADD52HUQZ128rk: 3000 case X86::VPMADD52HUQZ128rkz: 3001 case X86::VPMADD52HUQZ256r: 3002 case X86::VPMADD52HUQZ256rk: 3003 case X86::VPMADD52HUQZ256rkz: 3004 case X86::VPMADD52HUQZr: 3005 case X86::VPMADD52HUQZrk: 3006 case X86::VPMADD52HUQZrkz: 3007 case X86::VPMADD52LUQrr: 3008 case X86::VPMADD52LUQYrr: 3009 case X86::VPMADD52LUQZ128r: 3010 case X86::VPMADD52LUQZ128rk: 3011 case X86::VPMADD52LUQZ128rkz: 3012 case X86::VPMADD52LUQZ256r: 3013 case X86::VPMADD52LUQZ256rk: 3014 case X86::VPMADD52LUQZ256rkz: 3015 case X86::VPMADD52LUQZr: 3016 case X86::VPMADD52LUQZrk: 3017 case X86::VPMADD52LUQZrkz: 3018 case X86::VFMADDCPHZr: 3019 case X86::VFMADDCPHZrk: 3020 case X86::VFMADDCPHZrkz: 3021 case X86::VFMADDCPHZ128r: 3022 case X86::VFMADDCPHZ128rk: 3023 case X86::VFMADDCPHZ128rkz: 3024 case X86::VFMADDCPHZ256r: 3025 case X86::VFMADDCPHZ256rk: 3026 case X86::VFMADDCPHZ256rkz: 3027 case X86::VFMADDCSHZr: 3028 case X86::VFMADDCSHZrk: 3029 case X86::VFMADDCSHZrkz: { 3030 unsigned CommutableOpIdx1 = 2; 3031 unsigned CommutableOpIdx2 = 3; 3032 if (X86II::isKMasked(Desc.TSFlags)) { 3033 // Skip the mask register. 3034 ++CommutableOpIdx1; 3035 ++CommutableOpIdx2; 3036 } 3037 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1, 3038 CommutableOpIdx2)) 3039 return false; 3040 if (!MI.getOperand(SrcOpIdx1).isReg() || !MI.getOperand(SrcOpIdx2).isReg()) 3041 // No idea. 3042 return false; 3043 return true; 3044 } 3045 3046 default: 3047 const X86InstrFMA3Group *FMA3Group = 3048 getFMA3Group(MI.getOpcode(), MI.getDesc().TSFlags); 3049 if (FMA3Group) 3050 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2, 3051 FMA3Group->isIntrinsic()); 3052 3053 // Handled masked instructions since we need to skip over the mask input 3054 // and the preserved input. 3055 if (X86II::isKMasked(Desc.TSFlags)) { 3056 // First assume that the first input is the mask operand and skip past it. 3057 unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1; 3058 unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2; 3059 // Check if the first input is tied. If there isn't one then we only 3060 // need to skip the mask operand which we did above. 3061 if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(), 3062 MCOI::TIED_TO) != -1)) { 3063 // If this is zero masking instruction with a tied operand, we need to 3064 // move the first index back to the first input since this must 3065 // be a 3 input instruction and we want the first two non-mask inputs. 3066 // Otherwise this is a 2 input instruction with a preserved input and 3067 // mask, so we need to move the indices to skip one more input. 3068 if (X86II::isKMergeMasked(Desc.TSFlags)) { 3069 ++CommutableOpIdx1; 3070 ++CommutableOpIdx2; 3071 } else { 3072 --CommutableOpIdx1; 3073 } 3074 } 3075 3076 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1, 3077 CommutableOpIdx2)) 3078 return false; 3079 3080 if (!MI.getOperand(SrcOpIdx1).isReg() || 3081 !MI.getOperand(SrcOpIdx2).isReg()) 3082 // No idea. 3083 return false; 3084 return true; 3085 } 3086 3087 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 3088 } 3089 return false; 3090 } 3091 3092 static bool isConvertibleLEA(MachineInstr *MI) { 3093 unsigned Opcode = MI->getOpcode(); 3094 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r && 3095 Opcode != X86::LEA64_32r) 3096 return false; 3097 3098 const MachineOperand &Scale = MI->getOperand(1 + X86::AddrScaleAmt); 3099 const MachineOperand &Disp = MI->getOperand(1 + X86::AddrDisp); 3100 const MachineOperand &Segment = MI->getOperand(1 + X86::AddrSegmentReg); 3101 3102 if (Segment.getReg() != 0 || !Disp.isImm() || Disp.getImm() != 0 || 3103 Scale.getImm() > 1) 3104 return false; 3105 3106 return true; 3107 } 3108 3109 bool X86InstrInfo::hasCommutePreference(MachineInstr &MI, bool &Commute) const { 3110 // Currently we're interested in following sequence only. 3111 // r3 = lea r1, r2 3112 // r5 = add r3, r4 3113 // Both r3 and r4 are killed in add, we hope the add instruction has the 3114 // operand order 3115 // r5 = add r4, r3 3116 // So later in X86FixupLEAs the lea instruction can be rewritten as add. 3117 unsigned Opcode = MI.getOpcode(); 3118 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr) 3119 return false; 3120 3121 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 3122 Register Reg1 = MI.getOperand(1).getReg(); 3123 Register Reg2 = MI.getOperand(2).getReg(); 3124 3125 // Check if Reg1 comes from LEA in the same MBB. 3126 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg1)) { 3127 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) { 3128 Commute = true; 3129 return true; 3130 } 3131 } 3132 3133 // Check if Reg2 comes from LEA in the same MBB. 3134 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg2)) { 3135 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) { 3136 Commute = false; 3137 return true; 3138 } 3139 } 3140 3141 return false; 3142 } 3143 3144 int X86::getCondSrcNoFromDesc(const MCInstrDesc &MCID) { 3145 unsigned Opcode = MCID.getOpcode(); 3146 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isCMOVCC(Opcode))) 3147 return -1; 3148 // Assume that condition code is always the last use operand. 3149 unsigned NumUses = MCID.getNumOperands() - MCID.getNumDefs(); 3150 return NumUses - 1; 3151 } 3152 3153 X86::CondCode X86::getCondFromMI(const MachineInstr &MI) { 3154 const MCInstrDesc &MCID = MI.getDesc(); 3155 int CondNo = getCondSrcNoFromDesc(MCID); 3156 if (CondNo < 0) 3157 return X86::COND_INVALID; 3158 CondNo += MCID.getNumDefs(); 3159 return static_cast<X86::CondCode>(MI.getOperand(CondNo).getImm()); 3160 } 3161 3162 X86::CondCode X86::getCondFromBranch(const MachineInstr &MI) { 3163 return X86::isJCC(MI.getOpcode()) ? X86::getCondFromMI(MI) 3164 : X86::COND_INVALID; 3165 } 3166 3167 X86::CondCode X86::getCondFromSETCC(const MachineInstr &MI) { 3168 return X86::isSETCC(MI.getOpcode()) ? X86::getCondFromMI(MI) 3169 : X86::COND_INVALID; 3170 } 3171 3172 X86::CondCode X86::getCondFromCMov(const MachineInstr &MI) { 3173 return X86::isCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI) 3174 : X86::COND_INVALID; 3175 } 3176 3177 /// Return the inverse of the specified condition, 3178 /// e.g. turning COND_E to COND_NE. 3179 X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) { 3180 switch (CC) { 3181 default: 3182 llvm_unreachable("Illegal condition code!"); 3183 case X86::COND_E: 3184 return X86::COND_NE; 3185 case X86::COND_NE: 3186 return X86::COND_E; 3187 case X86::COND_L: 3188 return X86::COND_GE; 3189 case X86::COND_LE: 3190 return X86::COND_G; 3191 case X86::COND_G: 3192 return X86::COND_LE; 3193 case X86::COND_GE: 3194 return X86::COND_L; 3195 case X86::COND_B: 3196 return X86::COND_AE; 3197 case X86::COND_BE: 3198 return X86::COND_A; 3199 case X86::COND_A: 3200 return X86::COND_BE; 3201 case X86::COND_AE: 3202 return X86::COND_B; 3203 case X86::COND_S: 3204 return X86::COND_NS; 3205 case X86::COND_NS: 3206 return X86::COND_S; 3207 case X86::COND_P: 3208 return X86::COND_NP; 3209 case X86::COND_NP: 3210 return X86::COND_P; 3211 case X86::COND_O: 3212 return X86::COND_NO; 3213 case X86::COND_NO: 3214 return X86::COND_O; 3215 case X86::COND_NE_OR_P: 3216 return X86::COND_E_AND_NP; 3217 case X86::COND_E_AND_NP: 3218 return X86::COND_NE_OR_P; 3219 } 3220 } 3221 3222 /// Assuming the flags are set by MI(a,b), return the condition code if we 3223 /// modify the instructions such that flags are set by MI(b,a). 3224 static X86::CondCode getSwappedCondition(X86::CondCode CC) { 3225 switch (CC) { 3226 default: 3227 return X86::COND_INVALID; 3228 case X86::COND_E: 3229 return X86::COND_E; 3230 case X86::COND_NE: 3231 return X86::COND_NE; 3232 case X86::COND_L: 3233 return X86::COND_G; 3234 case X86::COND_LE: 3235 return X86::COND_GE; 3236 case X86::COND_G: 3237 return X86::COND_L; 3238 case X86::COND_GE: 3239 return X86::COND_LE; 3240 case X86::COND_B: 3241 return X86::COND_A; 3242 case X86::COND_BE: 3243 return X86::COND_AE; 3244 case X86::COND_A: 3245 return X86::COND_B; 3246 case X86::COND_AE: 3247 return X86::COND_BE; 3248 } 3249 } 3250 3251 std::pair<X86::CondCode, bool> 3252 X86::getX86ConditionCode(CmpInst::Predicate Predicate) { 3253 X86::CondCode CC = X86::COND_INVALID; 3254 bool NeedSwap = false; 3255 switch (Predicate) { 3256 default: 3257 break; 3258 // Floating-point Predicates 3259 case CmpInst::FCMP_UEQ: 3260 CC = X86::COND_E; 3261 break; 3262 case CmpInst::FCMP_OLT: 3263 NeedSwap = true; 3264 [[fallthrough]]; 3265 case CmpInst::FCMP_OGT: 3266 CC = X86::COND_A; 3267 break; 3268 case CmpInst::FCMP_OLE: 3269 NeedSwap = true; 3270 [[fallthrough]]; 3271 case CmpInst::FCMP_OGE: 3272 CC = X86::COND_AE; 3273 break; 3274 case CmpInst::FCMP_UGT: 3275 NeedSwap = true; 3276 [[fallthrough]]; 3277 case CmpInst::FCMP_ULT: 3278 CC = X86::COND_B; 3279 break; 3280 case CmpInst::FCMP_UGE: 3281 NeedSwap = true; 3282 [[fallthrough]]; 3283 case CmpInst::FCMP_ULE: 3284 CC = X86::COND_BE; 3285 break; 3286 case CmpInst::FCMP_ONE: 3287 CC = X86::COND_NE; 3288 break; 3289 case CmpInst::FCMP_UNO: 3290 CC = X86::COND_P; 3291 break; 3292 case CmpInst::FCMP_ORD: 3293 CC = X86::COND_NP; 3294 break; 3295 case CmpInst::FCMP_OEQ: 3296 [[fallthrough]]; 3297 case CmpInst::FCMP_UNE: 3298 CC = X86::COND_INVALID; 3299 break; 3300 3301 // Integer Predicates 3302 case CmpInst::ICMP_EQ: 3303 CC = X86::COND_E; 3304 break; 3305 case CmpInst::ICMP_NE: 3306 CC = X86::COND_NE; 3307 break; 3308 case CmpInst::ICMP_UGT: 3309 CC = X86::COND_A; 3310 break; 3311 case CmpInst::ICMP_UGE: 3312 CC = X86::COND_AE; 3313 break; 3314 case CmpInst::ICMP_ULT: 3315 CC = X86::COND_B; 3316 break; 3317 case CmpInst::ICMP_ULE: 3318 CC = X86::COND_BE; 3319 break; 3320 case CmpInst::ICMP_SGT: 3321 CC = X86::COND_G; 3322 break; 3323 case CmpInst::ICMP_SGE: 3324 CC = X86::COND_GE; 3325 break; 3326 case CmpInst::ICMP_SLT: 3327 CC = X86::COND_L; 3328 break; 3329 case CmpInst::ICMP_SLE: 3330 CC = X86::COND_LE; 3331 break; 3332 } 3333 3334 return std::make_pair(CC, NeedSwap); 3335 } 3336 3337 /// Return a cmov opcode for the given register size in bytes, and operand type. 3338 unsigned X86::getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand) { 3339 switch (RegBytes) { 3340 default: 3341 llvm_unreachable("Illegal register size!"); 3342 case 2: 3343 return HasMemoryOperand ? X86::CMOV16rm : X86::CMOV16rr; 3344 case 4: 3345 return HasMemoryOperand ? X86::CMOV32rm : X86::CMOV32rr; 3346 case 8: 3347 return HasMemoryOperand ? X86::CMOV64rm : X86::CMOV64rr; 3348 } 3349 } 3350 3351 /// Get the VPCMP immediate for the given condition. 3352 unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) { 3353 switch (CC) { 3354 default: 3355 llvm_unreachable("Unexpected SETCC condition"); 3356 case ISD::SETNE: 3357 return 4; 3358 case ISD::SETEQ: 3359 return 0; 3360 case ISD::SETULT: 3361 case ISD::SETLT: 3362 return 1; 3363 case ISD::SETUGT: 3364 case ISD::SETGT: 3365 return 6; 3366 case ISD::SETUGE: 3367 case ISD::SETGE: 3368 return 5; 3369 case ISD::SETULE: 3370 case ISD::SETLE: 3371 return 2; 3372 } 3373 } 3374 3375 /// Get the VPCMP immediate if the operands are swapped. 3376 unsigned X86::getSwappedVPCMPImm(unsigned Imm) { 3377 switch (Imm) { 3378 default: 3379 llvm_unreachable("Unreachable!"); 3380 case 0x01: 3381 Imm = 0x06; 3382 break; // LT -> NLE 3383 case 0x02: 3384 Imm = 0x05; 3385 break; // LE -> NLT 3386 case 0x05: 3387 Imm = 0x02; 3388 break; // NLT -> LE 3389 case 0x06: 3390 Imm = 0x01; 3391 break; // NLE -> LT 3392 case 0x00: // EQ 3393 case 0x03: // FALSE 3394 case 0x04: // NE 3395 case 0x07: // TRUE 3396 break; 3397 } 3398 3399 return Imm; 3400 } 3401 3402 /// Get the VPCOM immediate if the operands are swapped. 3403 unsigned X86::getSwappedVPCOMImm(unsigned Imm) { 3404 switch (Imm) { 3405 default: 3406 llvm_unreachable("Unreachable!"); 3407 case 0x00: 3408 Imm = 0x02; 3409 break; // LT -> GT 3410 case 0x01: 3411 Imm = 0x03; 3412 break; // LE -> GE 3413 case 0x02: 3414 Imm = 0x00; 3415 break; // GT -> LT 3416 case 0x03: 3417 Imm = 0x01; 3418 break; // GE -> LE 3419 case 0x04: // EQ 3420 case 0x05: // NE 3421 case 0x06: // FALSE 3422 case 0x07: // TRUE 3423 break; 3424 } 3425 3426 return Imm; 3427 } 3428 3429 /// Get the VCMP immediate if the operands are swapped. 3430 unsigned X86::getSwappedVCMPImm(unsigned Imm) { 3431 // Only need the lower 2 bits to distinquish. 3432 switch (Imm & 0x3) { 3433 default: 3434 llvm_unreachable("Unreachable!"); 3435 case 0x00: 3436 case 0x03: 3437 // EQ/NE/TRUE/FALSE/ORD/UNORD don't change immediate when commuted. 3438 break; 3439 case 0x01: 3440 case 0x02: 3441 // Need to toggle bits 3:0. Bit 4 stays the same. 3442 Imm ^= 0xf; 3443 break; 3444 } 3445 3446 return Imm; 3447 } 3448 3449 /// Return true if the Reg is X87 register. 3450 static bool isX87Reg(unsigned Reg) { 3451 return (Reg == X86::FPCW || Reg == X86::FPSW || 3452 (Reg >= X86::ST0 && Reg <= X86::ST7)); 3453 } 3454 3455 /// check if the instruction is X87 instruction 3456 bool X86::isX87Instruction(MachineInstr &MI) { 3457 for (const MachineOperand &MO : MI.operands()) { 3458 if (!MO.isReg()) 3459 continue; 3460 if (isX87Reg(MO.getReg())) 3461 return true; 3462 } 3463 return false; 3464 } 3465 3466 bool X86InstrInfo::isUnconditionalTailCall(const MachineInstr &MI) const { 3467 switch (MI.getOpcode()) { 3468 case X86::TCRETURNdi: 3469 case X86::TCRETURNri: 3470 case X86::TCRETURNmi: 3471 case X86::TCRETURNdi64: 3472 case X86::TCRETURNri64: 3473 case X86::TCRETURNmi64: 3474 return true; 3475 default: 3476 return false; 3477 } 3478 } 3479 3480 bool X86InstrInfo::canMakeTailCallConditional( 3481 SmallVectorImpl<MachineOperand> &BranchCond, 3482 const MachineInstr &TailCall) const { 3483 3484 const MachineFunction *MF = TailCall.getMF(); 3485 3486 if (MF->getTarget().getCodeModel() == CodeModel::Kernel) { 3487 // Kernel patches thunk calls in runtime, these should never be conditional. 3488 const MachineOperand &Target = TailCall.getOperand(0); 3489 if (Target.isSymbol()) { 3490 StringRef Symbol(Target.getSymbolName()); 3491 // this is currently only relevant to r11/kernel indirect thunk. 3492 if (Symbol.equals("__x86_indirect_thunk_r11")) 3493 return false; 3494 } 3495 } 3496 3497 if (TailCall.getOpcode() != X86::TCRETURNdi && 3498 TailCall.getOpcode() != X86::TCRETURNdi64) { 3499 // Only direct calls can be done with a conditional branch. 3500 return false; 3501 } 3502 3503 if (Subtarget.isTargetWin64() && MF->hasWinCFI()) { 3504 // Conditional tail calls confuse the Win64 unwinder. 3505 return false; 3506 } 3507 3508 assert(BranchCond.size() == 1); 3509 if (BranchCond[0].getImm() > X86::LAST_VALID_COND) { 3510 // Can't make a conditional tail call with this condition. 3511 return false; 3512 } 3513 3514 const X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); 3515 if (X86FI->getTCReturnAddrDelta() != 0 || 3516 TailCall.getOperand(1).getImm() != 0) { 3517 // A conditional tail call cannot do any stack adjustment. 3518 return false; 3519 } 3520 3521 return true; 3522 } 3523 3524 void X86InstrInfo::replaceBranchWithTailCall( 3525 MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &BranchCond, 3526 const MachineInstr &TailCall) const { 3527 assert(canMakeTailCallConditional(BranchCond, TailCall)); 3528 3529 MachineBasicBlock::iterator I = MBB.end(); 3530 while (I != MBB.begin()) { 3531 --I; 3532 if (I->isDebugInstr()) 3533 continue; 3534 if (!I->isBranch()) 3535 assert(0 && "Can't find the branch to replace!"); 3536 3537 X86::CondCode CC = X86::getCondFromBranch(*I); 3538 assert(BranchCond.size() == 1); 3539 if (CC != BranchCond[0].getImm()) 3540 continue; 3541 3542 break; 3543 } 3544 3545 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc 3546 : X86::TCRETURNdi64cc; 3547 3548 auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc)); 3549 MIB->addOperand(TailCall.getOperand(0)); // Destination. 3550 MIB.addImm(0); // Stack offset (not used). 3551 MIB->addOperand(BranchCond[0]); // Condition. 3552 MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters. 3553 3554 // Add implicit uses and defs of all live regs potentially clobbered by the 3555 // call. This way they still appear live across the call. 3556 LivePhysRegs LiveRegs(getRegisterInfo()); 3557 LiveRegs.addLiveOuts(MBB); 3558 SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers; 3559 LiveRegs.stepForward(*MIB, Clobbers); 3560 for (const auto &C : Clobbers) { 3561 MIB.addReg(C.first, RegState::Implicit); 3562 MIB.addReg(C.first, RegState::Implicit | RegState::Define); 3563 } 3564 3565 I->eraseFromParent(); 3566 } 3567 3568 // Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may 3569 // not be a fallthrough MBB now due to layout changes). Return nullptr if the 3570 // fallthrough MBB cannot be identified. 3571 static MachineBasicBlock *getFallThroughMBB(MachineBasicBlock *MBB, 3572 MachineBasicBlock *TBB) { 3573 // Look for non-EHPad successors other than TBB. If we find exactly one, it 3574 // is the fallthrough MBB. If we find zero, then TBB is both the target MBB 3575 // and fallthrough MBB. If we find more than one, we cannot identify the 3576 // fallthrough MBB and should return nullptr. 3577 MachineBasicBlock *FallthroughBB = nullptr; 3578 for (MachineBasicBlock *Succ : MBB->successors()) { 3579 if (Succ->isEHPad() || (Succ == TBB && FallthroughBB)) 3580 continue; 3581 // Return a nullptr if we found more than one fallthrough successor. 3582 if (FallthroughBB && FallthroughBB != TBB) 3583 return nullptr; 3584 FallthroughBB = Succ; 3585 } 3586 return FallthroughBB; 3587 } 3588 3589 bool X86InstrInfo::AnalyzeBranchImpl( 3590 MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, 3591 SmallVectorImpl<MachineOperand> &Cond, 3592 SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const { 3593 3594 // Start from the bottom of the block and work up, examining the 3595 // terminator instructions. 3596 MachineBasicBlock::iterator I = MBB.end(); 3597 MachineBasicBlock::iterator UnCondBrIter = MBB.end(); 3598 while (I != MBB.begin()) { 3599 --I; 3600 if (I->isDebugInstr()) 3601 continue; 3602 3603 // Working from the bottom, when we see a non-terminator instruction, we're 3604 // done. 3605 if (!isUnpredicatedTerminator(*I)) 3606 break; 3607 3608 // A terminator that isn't a branch can't easily be handled by this 3609 // analysis. 3610 if (!I->isBranch()) 3611 return true; 3612 3613 // Handle unconditional branches. 3614 if (I->getOpcode() == X86::JMP_1) { 3615 UnCondBrIter = I; 3616 3617 if (!AllowModify) { 3618 TBB = I->getOperand(0).getMBB(); 3619 continue; 3620 } 3621 3622 // If the block has any instructions after a JMP, delete them. 3623 MBB.erase(std::next(I), MBB.end()); 3624 3625 Cond.clear(); 3626 FBB = nullptr; 3627 3628 // Delete the JMP if it's equivalent to a fall-through. 3629 if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) { 3630 TBB = nullptr; 3631 I->eraseFromParent(); 3632 I = MBB.end(); 3633 UnCondBrIter = MBB.end(); 3634 continue; 3635 } 3636 3637 // TBB is used to indicate the unconditional destination. 3638 TBB = I->getOperand(0).getMBB(); 3639 continue; 3640 } 3641 3642 // Handle conditional branches. 3643 X86::CondCode BranchCode = X86::getCondFromBranch(*I); 3644 if (BranchCode == X86::COND_INVALID) 3645 return true; // Can't handle indirect branch. 3646 3647 // In practice we should never have an undef eflags operand, if we do 3648 // abort here as we are not prepared to preserve the flag. 3649 if (I->findRegisterUseOperand(X86::EFLAGS)->isUndef()) 3650 return true; 3651 3652 // Working from the bottom, handle the first conditional branch. 3653 if (Cond.empty()) { 3654 FBB = TBB; 3655 TBB = I->getOperand(0).getMBB(); 3656 Cond.push_back(MachineOperand::CreateImm(BranchCode)); 3657 CondBranches.push_back(&*I); 3658 continue; 3659 } 3660 3661 // Handle subsequent conditional branches. Only handle the case where all 3662 // conditional branches branch to the same destination and their condition 3663 // opcodes fit one of the special multi-branch idioms. 3664 assert(Cond.size() == 1); 3665 assert(TBB); 3666 3667 // If the conditions are the same, we can leave them alone. 3668 X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm(); 3669 auto NewTBB = I->getOperand(0).getMBB(); 3670 if (OldBranchCode == BranchCode && TBB == NewTBB) 3671 continue; 3672 3673 // If they differ, see if they fit one of the known patterns. Theoretically, 3674 // we could handle more patterns here, but we shouldn't expect to see them 3675 // if instruction selection has done a reasonable job. 3676 if (TBB == NewTBB && 3677 ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) || 3678 (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) { 3679 BranchCode = X86::COND_NE_OR_P; 3680 } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) || 3681 (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) { 3682 if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB))) 3683 return true; 3684 3685 // X86::COND_E_AND_NP usually has two different branch destinations. 3686 // 3687 // JP B1 3688 // JE B2 3689 // JMP B1 3690 // B1: 3691 // B2: 3692 // 3693 // Here this condition branches to B2 only if NP && E. It has another 3694 // equivalent form: 3695 // 3696 // JNE B1 3697 // JNP B2 3698 // JMP B1 3699 // B1: 3700 // B2: 3701 // 3702 // Similarly it branches to B2 only if E && NP. That is why this condition 3703 // is named with COND_E_AND_NP. 3704 BranchCode = X86::COND_E_AND_NP; 3705 } else 3706 return true; 3707 3708 // Update the MachineOperand. 3709 Cond[0].setImm(BranchCode); 3710 CondBranches.push_back(&*I); 3711 } 3712 3713 return false; 3714 } 3715 3716 bool X86InstrInfo::analyzeBranch(MachineBasicBlock &MBB, 3717 MachineBasicBlock *&TBB, 3718 MachineBasicBlock *&FBB, 3719 SmallVectorImpl<MachineOperand> &Cond, 3720 bool AllowModify) const { 3721 SmallVector<MachineInstr *, 4> CondBranches; 3722 return AnalyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify); 3723 } 3724 3725 static int getJumpTableIndexFromAddr(const MachineInstr &MI) { 3726 const MCInstrDesc &Desc = MI.getDesc(); 3727 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); 3728 assert(MemRefBegin >= 0 && "instr should have memory operand"); 3729 MemRefBegin += X86II::getOperandBias(Desc); 3730 3731 const MachineOperand &MO = MI.getOperand(MemRefBegin + X86::AddrDisp); 3732 if (!MO.isJTI()) 3733 return -1; 3734 3735 return MO.getIndex(); 3736 } 3737 3738 static int getJumpTableIndexFromReg(const MachineRegisterInfo &MRI, 3739 Register Reg) { 3740 if (!Reg.isVirtual()) 3741 return -1; 3742 MachineInstr *MI = MRI.getUniqueVRegDef(Reg); 3743 if (MI == nullptr) 3744 return -1; 3745 unsigned Opcode = MI->getOpcode(); 3746 if (Opcode != X86::LEA64r && Opcode != X86::LEA32r) 3747 return -1; 3748 return getJumpTableIndexFromAddr(*MI); 3749 } 3750 3751 int X86InstrInfo::getJumpTableIndex(const MachineInstr &MI) const { 3752 unsigned Opcode = MI.getOpcode(); 3753 // Switch-jump pattern for non-PIC code looks like: 3754 // JMP64m $noreg, 8, %X, %jump-table.X, $noreg 3755 if (Opcode == X86::JMP64m || Opcode == X86::JMP32m) { 3756 return getJumpTableIndexFromAddr(MI); 3757 } 3758 // The pattern for PIC code looks like: 3759 // %0 = LEA64r $rip, 1, $noreg, %jump-table.X 3760 // %1 = MOVSX64rm32 %0, 4, XX, 0, $noreg 3761 // %2 = ADD64rr %1, %0 3762 // JMP64r %2 3763 if (Opcode == X86::JMP64r || Opcode == X86::JMP32r) { 3764 Register Reg = MI.getOperand(0).getReg(); 3765 if (!Reg.isVirtual()) 3766 return -1; 3767 const MachineFunction &MF = *MI.getParent()->getParent(); 3768 const MachineRegisterInfo &MRI = MF.getRegInfo(); 3769 MachineInstr *Add = MRI.getUniqueVRegDef(Reg); 3770 if (Add == nullptr) 3771 return -1; 3772 if (Add->getOpcode() != X86::ADD64rr && Add->getOpcode() != X86::ADD32rr) 3773 return -1; 3774 int JTI1 = getJumpTableIndexFromReg(MRI, Add->getOperand(1).getReg()); 3775 if (JTI1 >= 0) 3776 return JTI1; 3777 int JTI2 = getJumpTableIndexFromReg(MRI, Add->getOperand(2).getReg()); 3778 if (JTI2 >= 0) 3779 return JTI2; 3780 } 3781 return -1; 3782 } 3783 3784 bool X86InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB, 3785 MachineBranchPredicate &MBP, 3786 bool AllowModify) const { 3787 using namespace std::placeholders; 3788 3789 SmallVector<MachineOperand, 4> Cond; 3790 SmallVector<MachineInstr *, 4> CondBranches; 3791 if (AnalyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches, 3792 AllowModify)) 3793 return true; 3794 3795 if (Cond.size() != 1) 3796 return true; 3797 3798 assert(MBP.TrueDest && "expected!"); 3799 3800 if (!MBP.FalseDest) 3801 MBP.FalseDest = MBB.getNextNode(); 3802 3803 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3804 3805 MachineInstr *ConditionDef = nullptr; 3806 bool SingleUseCondition = true; 3807 3808 for (MachineInstr &MI : llvm::drop_begin(llvm::reverse(MBB))) { 3809 if (MI.modifiesRegister(X86::EFLAGS, TRI)) { 3810 ConditionDef = &MI; 3811 break; 3812 } 3813 3814 if (MI.readsRegister(X86::EFLAGS, TRI)) 3815 SingleUseCondition = false; 3816 } 3817 3818 if (!ConditionDef) 3819 return true; 3820 3821 if (SingleUseCondition) { 3822 for (auto *Succ : MBB.successors()) 3823 if (Succ->isLiveIn(X86::EFLAGS)) 3824 SingleUseCondition = false; 3825 } 3826 3827 MBP.ConditionDef = ConditionDef; 3828 MBP.SingleUseCondition = SingleUseCondition; 3829 3830 // Currently we only recognize the simple pattern: 3831 // 3832 // test %reg, %reg 3833 // je %label 3834 // 3835 const unsigned TestOpcode = 3836 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr; 3837 3838 if (ConditionDef->getOpcode() == TestOpcode && 3839 ConditionDef->getNumOperands() == 3 && 3840 ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) && 3841 (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) { 3842 MBP.LHS = ConditionDef->getOperand(0); 3843 MBP.RHS = MachineOperand::CreateImm(0); 3844 MBP.Predicate = Cond[0].getImm() == X86::COND_NE 3845 ? MachineBranchPredicate::PRED_NE 3846 : MachineBranchPredicate::PRED_EQ; 3847 return false; 3848 } 3849 3850 return true; 3851 } 3852 3853 unsigned X86InstrInfo::removeBranch(MachineBasicBlock &MBB, 3854 int *BytesRemoved) const { 3855 assert(!BytesRemoved && "code size not handled"); 3856 3857 MachineBasicBlock::iterator I = MBB.end(); 3858 unsigned Count = 0; 3859 3860 while (I != MBB.begin()) { 3861 --I; 3862 if (I->isDebugInstr()) 3863 continue; 3864 if (I->getOpcode() != X86::JMP_1 && 3865 X86::getCondFromBranch(*I) == X86::COND_INVALID) 3866 break; 3867 // Remove the branch. 3868 I->eraseFromParent(); 3869 I = MBB.end(); 3870 ++Count; 3871 } 3872 3873 return Count; 3874 } 3875 3876 unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB, 3877 MachineBasicBlock *TBB, 3878 MachineBasicBlock *FBB, 3879 ArrayRef<MachineOperand> Cond, 3880 const DebugLoc &DL, int *BytesAdded) const { 3881 // Shouldn't be a fall through. 3882 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 3883 assert((Cond.size() == 1 || Cond.size() == 0) && 3884 "X86 branch conditions have one component!"); 3885 assert(!BytesAdded && "code size not handled"); 3886 3887 if (Cond.empty()) { 3888 // Unconditional branch? 3889 assert(!FBB && "Unconditional branch with multiple successors!"); 3890 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB); 3891 return 1; 3892 } 3893 3894 // If FBB is null, it is implied to be a fall-through block. 3895 bool FallThru = FBB == nullptr; 3896 3897 // Conditional branch. 3898 unsigned Count = 0; 3899 X86::CondCode CC = (X86::CondCode)Cond[0].getImm(); 3900 switch (CC) { 3901 case X86::COND_NE_OR_P: 3902 // Synthesize NE_OR_P with two branches. 3903 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NE); 3904 ++Count; 3905 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_P); 3906 ++Count; 3907 break; 3908 case X86::COND_E_AND_NP: 3909 // Use the next block of MBB as FBB if it is null. 3910 if (FBB == nullptr) { 3911 FBB = getFallThroughMBB(&MBB, TBB); 3912 assert(FBB && "MBB cannot be the last block in function when the false " 3913 "body is a fall-through."); 3914 } 3915 // Synthesize COND_E_AND_NP with two branches. 3916 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(FBB).addImm(X86::COND_NE); 3917 ++Count; 3918 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NP); 3919 ++Count; 3920 break; 3921 default: { 3922 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(CC); 3923 ++Count; 3924 } 3925 } 3926 if (!FallThru) { 3927 // Two-way Conditional branch. Insert the second branch. 3928 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB); 3929 ++Count; 3930 } 3931 return Count; 3932 } 3933 3934 bool X86InstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 3935 ArrayRef<MachineOperand> Cond, 3936 Register DstReg, Register TrueReg, 3937 Register FalseReg, int &CondCycles, 3938 int &TrueCycles, int &FalseCycles) const { 3939 // Not all subtargets have cmov instructions. 3940 if (!Subtarget.canUseCMOV()) 3941 return false; 3942 if (Cond.size() != 1) 3943 return false; 3944 // We cannot do the composite conditions, at least not in SSA form. 3945 if ((X86::CondCode)Cond[0].getImm() > X86::LAST_VALID_COND) 3946 return false; 3947 3948 // Check register classes. 3949 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3950 const TargetRegisterClass *RC = 3951 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 3952 if (!RC) 3953 return false; 3954 3955 // We have cmov instructions for 16, 32, and 64 bit general purpose registers. 3956 if (X86::GR16RegClass.hasSubClassEq(RC) || 3957 X86::GR32RegClass.hasSubClassEq(RC) || 3958 X86::GR64RegClass.hasSubClassEq(RC)) { 3959 // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy 3960 // Bridge. Probably Ivy Bridge as well. 3961 CondCycles = 2; 3962 TrueCycles = 2; 3963 FalseCycles = 2; 3964 return true; 3965 } 3966 3967 // Can't do vectors. 3968 return false; 3969 } 3970 3971 void X86InstrInfo::insertSelect(MachineBasicBlock &MBB, 3972 MachineBasicBlock::iterator I, 3973 const DebugLoc &DL, Register DstReg, 3974 ArrayRef<MachineOperand> Cond, Register TrueReg, 3975 Register FalseReg) const { 3976 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3977 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); 3978 const TargetRegisterClass &RC = *MRI.getRegClass(DstReg); 3979 assert(Cond.size() == 1 && "Invalid Cond array"); 3980 unsigned Opc = X86::getCMovOpcode(TRI.getRegSizeInBits(RC) / 8, 3981 false /*HasMemoryOperand*/); 3982 BuildMI(MBB, I, DL, get(Opc), DstReg) 3983 .addReg(FalseReg) 3984 .addReg(TrueReg) 3985 .addImm(Cond[0].getImm()); 3986 } 3987 3988 /// Test if the given register is a physical h register. 3989 static bool isHReg(unsigned Reg) { 3990 return X86::GR8_ABCD_HRegClass.contains(Reg); 3991 } 3992 3993 // Try and copy between VR128/VR64 and GR64 registers. 3994 static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg, 3995 const X86Subtarget &Subtarget) { 3996 bool HasAVX = Subtarget.hasAVX(); 3997 bool HasAVX512 = Subtarget.hasAVX512(); 3998 bool HasEGPR = Subtarget.hasEGPR(); 3999 4000 // SrcReg(MaskReg) -> DestReg(GR64) 4001 // SrcReg(MaskReg) -> DestReg(GR32) 4002 4003 // All KMASK RegClasses hold the same k registers, can be tested against 4004 // anyone. 4005 if (X86::VK16RegClass.contains(SrcReg)) { 4006 if (X86::GR64RegClass.contains(DestReg)) { 4007 assert(Subtarget.hasBWI()); 4008 return HasEGPR ? X86::KMOVQrk_EVEX : X86::KMOVQrk; 4009 } 4010 if (X86::GR32RegClass.contains(DestReg)) 4011 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDrk_EVEX : X86::KMOVDrk) 4012 : (HasEGPR ? X86::KMOVWrk_EVEX : X86::KMOVWrk); 4013 } 4014 4015 // SrcReg(GR64) -> DestReg(MaskReg) 4016 // SrcReg(GR32) -> DestReg(MaskReg) 4017 4018 // All KMASK RegClasses hold the same k registers, can be tested against 4019 // anyone. 4020 if (X86::VK16RegClass.contains(DestReg)) { 4021 if (X86::GR64RegClass.contains(SrcReg)) { 4022 assert(Subtarget.hasBWI()); 4023 return HasEGPR ? X86::KMOVQkr_EVEX : X86::KMOVQkr; 4024 } 4025 if (X86::GR32RegClass.contains(SrcReg)) 4026 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDkr_EVEX : X86::KMOVDkr) 4027 : (HasEGPR ? X86::KMOVWkr_EVEX : X86::KMOVWkr); 4028 } 4029 4030 // SrcReg(VR128) -> DestReg(GR64) 4031 // SrcReg(VR64) -> DestReg(GR64) 4032 // SrcReg(GR64) -> DestReg(VR128) 4033 // SrcReg(GR64) -> DestReg(VR64) 4034 4035 if (X86::GR64RegClass.contains(DestReg)) { 4036 if (X86::VR128XRegClass.contains(SrcReg)) 4037 // Copy from a VR128 register to a GR64 register. 4038 return HasAVX512 ? X86::VMOVPQIto64Zrr 4039 : HasAVX ? X86::VMOVPQIto64rr 4040 : X86::MOVPQIto64rr; 4041 if (X86::VR64RegClass.contains(SrcReg)) 4042 // Copy from a VR64 register to a GR64 register. 4043 return X86::MMX_MOVD64from64rr; 4044 } else if (X86::GR64RegClass.contains(SrcReg)) { 4045 // Copy from a GR64 register to a VR128 register. 4046 if (X86::VR128XRegClass.contains(DestReg)) 4047 return HasAVX512 ? X86::VMOV64toPQIZrr 4048 : HasAVX ? X86::VMOV64toPQIrr 4049 : X86::MOV64toPQIrr; 4050 // Copy from a GR64 register to a VR64 register. 4051 if (X86::VR64RegClass.contains(DestReg)) 4052 return X86::MMX_MOVD64to64rr; 4053 } 4054 4055 // SrcReg(VR128) -> DestReg(GR32) 4056 // SrcReg(GR32) -> DestReg(VR128) 4057 4058 if (X86::GR32RegClass.contains(DestReg) && 4059 X86::VR128XRegClass.contains(SrcReg)) 4060 // Copy from a VR128 register to a GR32 register. 4061 return HasAVX512 ? X86::VMOVPDI2DIZrr 4062 : HasAVX ? X86::VMOVPDI2DIrr 4063 : X86::MOVPDI2DIrr; 4064 4065 if (X86::VR128XRegClass.contains(DestReg) && 4066 X86::GR32RegClass.contains(SrcReg)) 4067 // Copy from a VR128 register to a VR128 register. 4068 return HasAVX512 ? X86::VMOVDI2PDIZrr 4069 : HasAVX ? X86::VMOVDI2PDIrr 4070 : X86::MOVDI2PDIrr; 4071 return 0; 4072 } 4073 4074 void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB, 4075 MachineBasicBlock::iterator MI, 4076 const DebugLoc &DL, MCRegister DestReg, 4077 MCRegister SrcReg, bool KillSrc) const { 4078 // First deal with the normal symmetric copies. 4079 bool HasAVX = Subtarget.hasAVX(); 4080 bool HasVLX = Subtarget.hasVLX(); 4081 bool HasEGPR = Subtarget.hasEGPR(); 4082 unsigned Opc = 0; 4083 if (X86::GR64RegClass.contains(DestReg, SrcReg)) 4084 Opc = X86::MOV64rr; 4085 else if (X86::GR32RegClass.contains(DestReg, SrcReg)) 4086 Opc = X86::MOV32rr; 4087 else if (X86::GR16RegClass.contains(DestReg, SrcReg)) 4088 Opc = X86::MOV16rr; 4089 else if (X86::GR8RegClass.contains(DestReg, SrcReg)) { 4090 // Copying to or from a physical H register on x86-64 requires a NOREX 4091 // move. Otherwise use a normal move. 4092 if ((isHReg(DestReg) || isHReg(SrcReg)) && Subtarget.is64Bit()) { 4093 Opc = X86::MOV8rr_NOREX; 4094 // Both operands must be encodable without an REX prefix. 4095 assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && 4096 "8-bit H register can not be copied outside GR8_NOREX"); 4097 } else 4098 Opc = X86::MOV8rr; 4099 } else if (X86::VR64RegClass.contains(DestReg, SrcReg)) 4100 Opc = X86::MMX_MOVQ64rr; 4101 else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) { 4102 if (HasVLX) 4103 Opc = X86::VMOVAPSZ128rr; 4104 else if (X86::VR128RegClass.contains(DestReg, SrcReg)) 4105 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr; 4106 else { 4107 // If this an extended register and we don't have VLX we need to use a 4108 // 512-bit move. 4109 Opc = X86::VMOVAPSZrr; 4110 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4111 DestReg = 4112 TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, &X86::VR512RegClass); 4113 SrcReg = 4114 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass); 4115 } 4116 } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) { 4117 if (HasVLX) 4118 Opc = X86::VMOVAPSZ256rr; 4119 else if (X86::VR256RegClass.contains(DestReg, SrcReg)) 4120 Opc = X86::VMOVAPSYrr; 4121 else { 4122 // If this an extended register and we don't have VLX we need to use a 4123 // 512-bit move. 4124 Opc = X86::VMOVAPSZrr; 4125 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4126 DestReg = 4127 TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, &X86::VR512RegClass); 4128 SrcReg = 4129 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass); 4130 } 4131 } else if (X86::VR512RegClass.contains(DestReg, SrcReg)) 4132 Opc = X86::VMOVAPSZrr; 4133 // All KMASK RegClasses hold the same k registers, can be tested against 4134 // anyone. 4135 else if (X86::VK16RegClass.contains(DestReg, SrcReg)) 4136 Opc = Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVQkk) 4137 : (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVWkk); 4138 if (!Opc) 4139 Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget); 4140 4141 if (Opc) { 4142 BuildMI(MBB, MI, DL, get(Opc), DestReg) 4143 .addReg(SrcReg, getKillRegState(KillSrc)); 4144 return; 4145 } 4146 4147 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) { 4148 // FIXME: We use a fatal error here because historically LLVM has tried 4149 // lower some of these physreg copies and we want to ensure we get 4150 // reasonable bug reports if someone encounters a case no other testing 4151 // found. This path should be removed after the LLVM 7 release. 4152 report_fatal_error("Unable to copy EFLAGS physical register!"); 4153 } 4154 4155 LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to " 4156 << RI.getName(DestReg) << '\n'); 4157 report_fatal_error("Cannot emit physreg copy instruction"); 4158 } 4159 4160 std::optional<DestSourcePair> 4161 X86InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { 4162 if (MI.isMoveReg()) { 4163 // FIXME: Dirty hack for apparent invariant that doesn't hold when 4164 // subreg_to_reg is coalesced with ordinary copies, such that the bits that 4165 // were asserted as 0 are now undef. 4166 if (MI.getOperand(0).isUndef() && MI.getOperand(0).getSubReg()) 4167 return std::nullopt; 4168 4169 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 4170 } 4171 return std::nullopt; 4172 } 4173 4174 static unsigned getLoadStoreOpcodeForFP16(bool Load, const X86Subtarget &STI) { 4175 if (STI.hasFP16()) 4176 return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr; 4177 if (Load) 4178 return STI.hasAVX512() ? X86::VMOVSSZrm 4179 : STI.hasAVX() ? X86::VMOVSSrm 4180 : X86::MOVSSrm; 4181 else 4182 return STI.hasAVX512() ? X86::VMOVSSZmr 4183 : STI.hasAVX() ? X86::VMOVSSmr 4184 : X86::MOVSSmr; 4185 } 4186 4187 static unsigned getLoadStoreRegOpcode(Register Reg, 4188 const TargetRegisterClass *RC, 4189 bool IsStackAligned, 4190 const X86Subtarget &STI, bool Load) { 4191 bool HasAVX = STI.hasAVX(); 4192 bool HasAVX512 = STI.hasAVX512(); 4193 bool HasVLX = STI.hasVLX(); 4194 bool HasEGPR = STI.hasEGPR(); 4195 4196 assert(RC != nullptr && "Invalid target register class"); 4197 switch (STI.getRegisterInfo()->getSpillSize(*RC)) { 4198 default: 4199 llvm_unreachable("Unknown spill size"); 4200 case 1: 4201 assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass"); 4202 if (STI.is64Bit()) 4203 // Copying to or from a physical H register on x86-64 requires a NOREX 4204 // move. Otherwise use a normal move. 4205 if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC)) 4206 return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX; 4207 return Load ? X86::MOV8rm : X86::MOV8mr; 4208 case 2: 4209 if (X86::VK16RegClass.hasSubClassEq(RC)) 4210 return Load ? (HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm) 4211 : (HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk); 4212 assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass"); 4213 return Load ? X86::MOV16rm : X86::MOV16mr; 4214 case 4: 4215 if (X86::GR32RegClass.hasSubClassEq(RC)) 4216 return Load ? X86::MOV32rm : X86::MOV32mr; 4217 if (X86::FR32XRegClass.hasSubClassEq(RC)) 4218 return Load ? (HasAVX512 ? X86::VMOVSSZrm_alt 4219 : HasAVX ? X86::VMOVSSrm_alt 4220 : X86::MOVSSrm_alt) 4221 : (HasAVX512 ? X86::VMOVSSZmr 4222 : HasAVX ? X86::VMOVSSmr 4223 : X86::MOVSSmr); 4224 if (X86::RFP32RegClass.hasSubClassEq(RC)) 4225 return Load ? X86::LD_Fp32m : X86::ST_Fp32m; 4226 if (X86::VK32RegClass.hasSubClassEq(RC)) { 4227 assert(STI.hasBWI() && "KMOVD requires BWI"); 4228 return Load ? (HasEGPR ? X86::KMOVDkm_EVEX : X86::KMOVDkm) 4229 : (HasEGPR ? X86::KMOVDmk_EVEX : X86::KMOVDmk); 4230 } 4231 // All of these mask pair classes have the same spill size, the same kind 4232 // of kmov instructions can be used with all of them. 4233 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) || 4234 X86::VK2PAIRRegClass.hasSubClassEq(RC) || 4235 X86::VK4PAIRRegClass.hasSubClassEq(RC) || 4236 X86::VK8PAIRRegClass.hasSubClassEq(RC) || 4237 X86::VK16PAIRRegClass.hasSubClassEq(RC)) 4238 return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE; 4239 if (X86::FR16RegClass.hasSubClassEq(RC) || 4240 X86::FR16XRegClass.hasSubClassEq(RC)) 4241 return getLoadStoreOpcodeForFP16(Load, STI); 4242 llvm_unreachable("Unknown 4-byte regclass"); 4243 case 8: 4244 if (X86::GR64RegClass.hasSubClassEq(RC)) 4245 return Load ? X86::MOV64rm : X86::MOV64mr; 4246 if (X86::FR64XRegClass.hasSubClassEq(RC)) 4247 return Load ? (HasAVX512 ? X86::VMOVSDZrm_alt 4248 : HasAVX ? X86::VMOVSDrm_alt 4249 : X86::MOVSDrm_alt) 4250 : (HasAVX512 ? X86::VMOVSDZmr 4251 : HasAVX ? X86::VMOVSDmr 4252 : X86::MOVSDmr); 4253 if (X86::VR64RegClass.hasSubClassEq(RC)) 4254 return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr; 4255 if (X86::RFP64RegClass.hasSubClassEq(RC)) 4256 return Load ? X86::LD_Fp64m : X86::ST_Fp64m; 4257 if (X86::VK64RegClass.hasSubClassEq(RC)) { 4258 assert(STI.hasBWI() && "KMOVQ requires BWI"); 4259 return Load ? (HasEGPR ? X86::KMOVQkm_EVEX : X86::KMOVQkm) 4260 : (HasEGPR ? X86::KMOVQmk_EVEX : X86::KMOVQmk); 4261 } 4262 llvm_unreachable("Unknown 8-byte regclass"); 4263 case 10: 4264 assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass"); 4265 return Load ? X86::LD_Fp80m : X86::ST_FpP80m; 4266 case 16: { 4267 if (X86::VR128XRegClass.hasSubClassEq(RC)) { 4268 // If stack is realigned we can use aligned stores. 4269 if (IsStackAligned) 4270 return Load ? (HasVLX ? X86::VMOVAPSZ128rm 4271 : HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX 4272 : HasAVX ? X86::VMOVAPSrm 4273 : X86::MOVAPSrm) 4274 : (HasVLX ? X86::VMOVAPSZ128mr 4275 : HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX 4276 : HasAVX ? X86::VMOVAPSmr 4277 : X86::MOVAPSmr); 4278 else 4279 return Load ? (HasVLX ? X86::VMOVUPSZ128rm 4280 : HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX 4281 : HasAVX ? X86::VMOVUPSrm 4282 : X86::MOVUPSrm) 4283 : (HasVLX ? X86::VMOVUPSZ128mr 4284 : HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX 4285 : HasAVX ? X86::VMOVUPSmr 4286 : X86::MOVUPSmr); 4287 } 4288 llvm_unreachable("Unknown 16-byte regclass"); 4289 } 4290 case 32: 4291 assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass"); 4292 // If stack is realigned we can use aligned stores. 4293 if (IsStackAligned) 4294 return Load ? (HasVLX ? X86::VMOVAPSZ256rm 4295 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX 4296 : X86::VMOVAPSYrm) 4297 : (HasVLX ? X86::VMOVAPSZ256mr 4298 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX 4299 : X86::VMOVAPSYmr); 4300 else 4301 return Load ? (HasVLX ? X86::VMOVUPSZ256rm 4302 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX 4303 : X86::VMOVUPSYrm) 4304 : (HasVLX ? X86::VMOVUPSZ256mr 4305 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX 4306 : X86::VMOVUPSYmr); 4307 case 64: 4308 assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass"); 4309 assert(STI.hasAVX512() && "Using 512-bit register requires AVX512"); 4310 if (IsStackAligned) 4311 return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; 4312 else 4313 return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; 4314 case 1024: 4315 assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass"); 4316 assert(STI.hasAMXTILE() && "Using 8*1024-bit register requires AMX-TILE"); 4317 return Load ? X86::TILELOADD : X86::TILESTORED; 4318 } 4319 } 4320 4321 std::optional<ExtAddrMode> 4322 X86InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI, 4323 const TargetRegisterInfo *TRI) const { 4324 const MCInstrDesc &Desc = MemI.getDesc(); 4325 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); 4326 if (MemRefBegin < 0) 4327 return std::nullopt; 4328 4329 MemRefBegin += X86II::getOperandBias(Desc); 4330 4331 auto &BaseOp = MemI.getOperand(MemRefBegin + X86::AddrBaseReg); 4332 if (!BaseOp.isReg()) // Can be an MO_FrameIndex 4333 return std::nullopt; 4334 4335 const MachineOperand &DispMO = MemI.getOperand(MemRefBegin + X86::AddrDisp); 4336 // Displacement can be symbolic 4337 if (!DispMO.isImm()) 4338 return std::nullopt; 4339 4340 ExtAddrMode AM; 4341 AM.BaseReg = BaseOp.getReg(); 4342 AM.ScaledReg = MemI.getOperand(MemRefBegin + X86::AddrIndexReg).getReg(); 4343 AM.Scale = MemI.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm(); 4344 AM.Displacement = DispMO.getImm(); 4345 return AM; 4346 } 4347 4348 bool X86InstrInfo::verifyInstruction(const MachineInstr &MI, 4349 StringRef &ErrInfo) const { 4350 std::optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MI, nullptr); 4351 if (!AMOrNone) 4352 return true; 4353 4354 ExtAddrMode AM = *AMOrNone; 4355 assert(AM.Form == ExtAddrMode::Formula::Basic); 4356 if (AM.ScaledReg != X86::NoRegister) { 4357 switch (AM.Scale) { 4358 case 1: 4359 case 2: 4360 case 4: 4361 case 8: 4362 break; 4363 default: 4364 ErrInfo = "Scale factor in address must be 1, 2, 4 or 8"; 4365 return false; 4366 } 4367 } 4368 if (!isInt<32>(AM.Displacement)) { 4369 ErrInfo = "Displacement in address must fit into 32-bit signed " 4370 "integer"; 4371 return false; 4372 } 4373 4374 return true; 4375 } 4376 4377 bool X86InstrInfo::getConstValDefinedInReg(const MachineInstr &MI, 4378 const Register Reg, 4379 int64_t &ImmVal) const { 4380 Register MovReg = Reg; 4381 const MachineInstr *MovMI = &MI; 4382 4383 // Follow use-def for SUBREG_TO_REG to find the real move immediate 4384 // instruction. It is quite common for x86-64. 4385 if (MI.isSubregToReg()) { 4386 // We use following pattern to setup 64b immediate. 4387 // %8:gr32 = MOV32r0 implicit-def dead $eflags 4388 // %6:gr64 = SUBREG_TO_REG 0, killed %8:gr32, %subreg.sub_32bit 4389 if (!MI.getOperand(1).isImm()) 4390 return false; 4391 unsigned FillBits = MI.getOperand(1).getImm(); 4392 unsigned SubIdx = MI.getOperand(3).getImm(); 4393 MovReg = MI.getOperand(2).getReg(); 4394 if (SubIdx != X86::sub_32bit || FillBits != 0) 4395 return false; 4396 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4397 MovMI = MRI.getUniqueVRegDef(MovReg); 4398 if (!MovMI) 4399 return false; 4400 } 4401 4402 if (MovMI->getOpcode() == X86::MOV32r0 && 4403 MovMI->getOperand(0).getReg() == MovReg) { 4404 ImmVal = 0; 4405 return true; 4406 } 4407 4408 if (MovMI->getOpcode() != X86::MOV32ri && 4409 MovMI->getOpcode() != X86::MOV64ri && 4410 MovMI->getOpcode() != X86::MOV32ri64 && MovMI->getOpcode() != X86::MOV8ri) 4411 return false; 4412 // Mov Src can be a global address. 4413 if (!MovMI->getOperand(1).isImm() || MovMI->getOperand(0).getReg() != MovReg) 4414 return false; 4415 ImmVal = MovMI->getOperand(1).getImm(); 4416 return true; 4417 } 4418 4419 bool X86InstrInfo::preservesZeroValueInReg( 4420 const MachineInstr *MI, const Register NullValueReg, 4421 const TargetRegisterInfo *TRI) const { 4422 if (!MI->modifiesRegister(NullValueReg, TRI)) 4423 return true; 4424 switch (MI->getOpcode()) { 4425 // Shift right/left of a null unto itself is still a null, i.e. rax = shl rax 4426 // X. 4427 case X86::SHR64ri: 4428 case X86::SHR32ri: 4429 case X86::SHL64ri: 4430 case X86::SHL32ri: 4431 assert(MI->getOperand(0).isDef() && MI->getOperand(1).isUse() && 4432 "expected for shift opcode!"); 4433 return MI->getOperand(0).getReg() == NullValueReg && 4434 MI->getOperand(1).getReg() == NullValueReg; 4435 // Zero extend of a sub-reg of NullValueReg into itself does not change the 4436 // null value. 4437 case X86::MOV32rr: 4438 return llvm::all_of(MI->operands(), [&](const MachineOperand &MO) { 4439 return TRI->isSubRegisterEq(NullValueReg, MO.getReg()); 4440 }); 4441 default: 4442 return false; 4443 } 4444 llvm_unreachable("Should be handled above!"); 4445 } 4446 4447 bool X86InstrInfo::getMemOperandsWithOffsetWidth( 4448 const MachineInstr &MemOp, SmallVectorImpl<const MachineOperand *> &BaseOps, 4449 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 4450 const TargetRegisterInfo *TRI) const { 4451 const MCInstrDesc &Desc = MemOp.getDesc(); 4452 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); 4453 if (MemRefBegin < 0) 4454 return false; 4455 4456 MemRefBegin += X86II::getOperandBias(Desc); 4457 4458 const MachineOperand *BaseOp = 4459 &MemOp.getOperand(MemRefBegin + X86::AddrBaseReg); 4460 if (!BaseOp->isReg()) // Can be an MO_FrameIndex 4461 return false; 4462 4463 if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1) 4464 return false; 4465 4466 if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() != 4467 X86::NoRegister) 4468 return false; 4469 4470 const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp); 4471 4472 // Displacement can be symbolic 4473 if (!DispMO.isImm()) 4474 return false; 4475 4476 Offset = DispMO.getImm(); 4477 4478 if (!BaseOp->isReg()) 4479 return false; 4480 4481 OffsetIsScalable = false; 4482 // FIXME: Relying on memoperands() may not be right thing to do here. Check 4483 // with X86 maintainers, and fix it accordingly. For now, it is ok, since 4484 // there is no use of `Width` for X86 back-end at the moment. 4485 Width = 4486 !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize() : 0; 4487 BaseOps.push_back(BaseOp); 4488 return true; 4489 } 4490 4491 static unsigned getStoreRegOpcode(Register SrcReg, 4492 const TargetRegisterClass *RC, 4493 bool IsStackAligned, 4494 const X86Subtarget &STI) { 4495 return getLoadStoreRegOpcode(SrcReg, RC, IsStackAligned, STI, false); 4496 } 4497 4498 static unsigned getLoadRegOpcode(Register DestReg, 4499 const TargetRegisterClass *RC, 4500 bool IsStackAligned, const X86Subtarget &STI) { 4501 return getLoadStoreRegOpcode(DestReg, RC, IsStackAligned, STI, true); 4502 } 4503 4504 static bool isAMXOpcode(unsigned Opc) { 4505 switch (Opc) { 4506 default: 4507 return false; 4508 case X86::TILELOADD: 4509 case X86::TILESTORED: 4510 return true; 4511 } 4512 } 4513 4514 void X86InstrInfo::loadStoreTileReg(MachineBasicBlock &MBB, 4515 MachineBasicBlock::iterator MI, 4516 unsigned Opc, Register Reg, int FrameIdx, 4517 bool isKill) const { 4518 switch (Opc) { 4519 default: 4520 llvm_unreachable("Unexpected special opcode!"); 4521 case X86::TILESTORED: { 4522 // tilestored %tmm, (%sp, %idx) 4523 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo(); 4524 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 4525 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64); 4526 MachineInstr *NewMI = 4527 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx) 4528 .addReg(Reg, getKillRegState(isKill)); 4529 MachineOperand &MO = NewMI->getOperand(X86::AddrIndexReg); 4530 MO.setReg(VirtReg); 4531 MO.setIsKill(true); 4532 break; 4533 } 4534 case X86::TILELOADD: { 4535 // tileloadd (%sp, %idx), %tmm 4536 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo(); 4537 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 4538 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64); 4539 MachineInstr *NewMI = addFrameReference( 4540 BuildMI(MBB, MI, DebugLoc(), get(Opc), Reg), FrameIdx); 4541 MachineOperand &MO = NewMI->getOperand(1 + X86::AddrIndexReg); 4542 MO.setReg(VirtReg); 4543 MO.setIsKill(true); 4544 break; 4545 } 4546 } 4547 } 4548 4549 void X86InstrInfo::storeRegToStackSlot( 4550 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, 4551 bool isKill, int FrameIdx, const TargetRegisterClass *RC, 4552 const TargetRegisterInfo *TRI, Register VReg) const { 4553 const MachineFunction &MF = *MBB.getParent(); 4554 const MachineFrameInfo &MFI = MF.getFrameInfo(); 4555 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && 4556 "Stack slot too small for store"); 4557 4558 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); 4559 bool isAligned = 4560 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) || 4561 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx)); 4562 4563 unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget); 4564 if (isAMXOpcode(Opc)) 4565 loadStoreTileReg(MBB, MI, Opc, SrcReg, FrameIdx, isKill); 4566 else 4567 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx) 4568 .addReg(SrcReg, getKillRegState(isKill)); 4569 } 4570 4571 void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 4572 MachineBasicBlock::iterator MI, 4573 Register DestReg, int FrameIdx, 4574 const TargetRegisterClass *RC, 4575 const TargetRegisterInfo *TRI, 4576 Register VReg) const { 4577 const MachineFunction &MF = *MBB.getParent(); 4578 const MachineFrameInfo &MFI = MF.getFrameInfo(); 4579 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && 4580 "Load size exceeds stack slot"); 4581 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); 4582 bool isAligned = 4583 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) || 4584 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx)); 4585 4586 unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget); 4587 if (isAMXOpcode(Opc)) 4588 loadStoreTileReg(MBB, MI, Opc, DestReg, FrameIdx); 4589 else 4590 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), 4591 FrameIdx); 4592 } 4593 4594 bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 4595 Register &SrcReg2, int64_t &CmpMask, 4596 int64_t &CmpValue) const { 4597 switch (MI.getOpcode()) { 4598 default: 4599 break; 4600 case X86::CMP64ri32: 4601 case X86::CMP32ri: 4602 case X86::CMP16ri: 4603 case X86::CMP8ri: 4604 SrcReg = MI.getOperand(0).getReg(); 4605 SrcReg2 = 0; 4606 if (MI.getOperand(1).isImm()) { 4607 CmpMask = ~0; 4608 CmpValue = MI.getOperand(1).getImm(); 4609 } else { 4610 CmpMask = CmpValue = 0; 4611 } 4612 return true; 4613 // A SUB can be used to perform comparison. 4614 case X86::SUB64rm: 4615 case X86::SUB32rm: 4616 case X86::SUB16rm: 4617 case X86::SUB8rm: 4618 SrcReg = MI.getOperand(1).getReg(); 4619 SrcReg2 = 0; 4620 CmpMask = 0; 4621 CmpValue = 0; 4622 return true; 4623 case X86::SUB64rr: 4624 case X86::SUB32rr: 4625 case X86::SUB16rr: 4626 case X86::SUB8rr: 4627 SrcReg = MI.getOperand(1).getReg(); 4628 SrcReg2 = MI.getOperand(2).getReg(); 4629 CmpMask = 0; 4630 CmpValue = 0; 4631 return true; 4632 case X86::SUB64ri32: 4633 case X86::SUB32ri: 4634 case X86::SUB16ri: 4635 case X86::SUB8ri: 4636 SrcReg = MI.getOperand(1).getReg(); 4637 SrcReg2 = 0; 4638 if (MI.getOperand(2).isImm()) { 4639 CmpMask = ~0; 4640 CmpValue = MI.getOperand(2).getImm(); 4641 } else { 4642 CmpMask = CmpValue = 0; 4643 } 4644 return true; 4645 case X86::CMP64rr: 4646 case X86::CMP32rr: 4647 case X86::CMP16rr: 4648 case X86::CMP8rr: 4649 SrcReg = MI.getOperand(0).getReg(); 4650 SrcReg2 = MI.getOperand(1).getReg(); 4651 CmpMask = 0; 4652 CmpValue = 0; 4653 return true; 4654 case X86::TEST8rr: 4655 case X86::TEST16rr: 4656 case X86::TEST32rr: 4657 case X86::TEST64rr: 4658 SrcReg = MI.getOperand(0).getReg(); 4659 if (MI.getOperand(1).getReg() != SrcReg) 4660 return false; 4661 // Compare against zero. 4662 SrcReg2 = 0; 4663 CmpMask = ~0; 4664 CmpValue = 0; 4665 return true; 4666 } 4667 return false; 4668 } 4669 4670 bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI, 4671 Register SrcReg, Register SrcReg2, 4672 int64_t ImmMask, int64_t ImmValue, 4673 const MachineInstr &OI, bool *IsSwapped, 4674 int64_t *ImmDelta) const { 4675 switch (OI.getOpcode()) { 4676 case X86::CMP64rr: 4677 case X86::CMP32rr: 4678 case X86::CMP16rr: 4679 case X86::CMP8rr: 4680 case X86::SUB64rr: 4681 case X86::SUB32rr: 4682 case X86::SUB16rr: 4683 case X86::SUB8rr: { 4684 Register OISrcReg; 4685 Register OISrcReg2; 4686 int64_t OIMask; 4687 int64_t OIValue; 4688 if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) || 4689 OIMask != ImmMask || OIValue != ImmValue) 4690 return false; 4691 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) { 4692 *IsSwapped = false; 4693 return true; 4694 } 4695 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) { 4696 *IsSwapped = true; 4697 return true; 4698 } 4699 return false; 4700 } 4701 case X86::CMP64ri32: 4702 case X86::CMP32ri: 4703 case X86::CMP16ri: 4704 case X86::CMP8ri: 4705 case X86::SUB64ri32: 4706 case X86::SUB32ri: 4707 case X86::SUB16ri: 4708 case X86::SUB8ri: 4709 case X86::TEST64rr: 4710 case X86::TEST32rr: 4711 case X86::TEST16rr: 4712 case X86::TEST8rr: { 4713 if (ImmMask != 0) { 4714 Register OISrcReg; 4715 Register OISrcReg2; 4716 int64_t OIMask; 4717 int64_t OIValue; 4718 if (analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) && 4719 SrcReg == OISrcReg && ImmMask == OIMask) { 4720 if (OIValue == ImmValue) { 4721 *ImmDelta = 0; 4722 return true; 4723 } else if (static_cast<uint64_t>(ImmValue) == 4724 static_cast<uint64_t>(OIValue) - 1) { 4725 *ImmDelta = -1; 4726 return true; 4727 } else if (static_cast<uint64_t>(ImmValue) == 4728 static_cast<uint64_t>(OIValue) + 1) { 4729 *ImmDelta = 1; 4730 return true; 4731 } else { 4732 return false; 4733 } 4734 } 4735 } 4736 return FlagI.isIdenticalTo(OI); 4737 } 4738 default: 4739 return false; 4740 } 4741 } 4742 4743 /// Check whether the definition can be converted 4744 /// to remove a comparison against zero. 4745 inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag, 4746 bool &ClearsOverflowFlag) { 4747 NoSignFlag = false; 4748 ClearsOverflowFlag = false; 4749 4750 // "ELF Handling for Thread-Local Storage" specifies that x86-64 GOTTPOFF, and 4751 // i386 GOTNTPOFF/INDNTPOFF relocations can convert an ADD to a LEA during 4752 // Initial Exec to Local Exec relaxation. In these cases, we must not depend 4753 // on the EFLAGS modification of ADD actually happening in the final binary. 4754 if (MI.getOpcode() == X86::ADD64rm || MI.getOpcode() == X86::ADD32rm) { 4755 unsigned Flags = MI.getOperand(5).getTargetFlags(); 4756 if (Flags == X86II::MO_GOTTPOFF || Flags == X86II::MO_INDNTPOFF || 4757 Flags == X86II::MO_GOTNTPOFF) 4758 return false; 4759 } 4760 4761 switch (MI.getOpcode()) { 4762 default: 4763 return false; 4764 4765 // The shift instructions only modify ZF if their shift count is non-zero. 4766 // N.B.: The processor truncates the shift count depending on the encoding. 4767 case X86::SAR8ri: 4768 case X86::SAR16ri: 4769 case X86::SAR32ri: 4770 case X86::SAR64ri: 4771 case X86::SHR8ri: 4772 case X86::SHR16ri: 4773 case X86::SHR32ri: 4774 case X86::SHR64ri: 4775 return getTruncatedShiftCount(MI, 2) != 0; 4776 4777 // Some left shift instructions can be turned into LEA instructions but only 4778 // if their flags aren't used. Avoid transforming such instructions. 4779 case X86::SHL8ri: 4780 case X86::SHL16ri: 4781 case X86::SHL32ri: 4782 case X86::SHL64ri: { 4783 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 4784 if (isTruncatedShiftCountForLEA(ShAmt)) 4785 return false; 4786 return ShAmt != 0; 4787 } 4788 4789 case X86::SHRD16rri8: 4790 case X86::SHRD32rri8: 4791 case X86::SHRD64rri8: 4792 case X86::SHLD16rri8: 4793 case X86::SHLD32rri8: 4794 case X86::SHLD64rri8: 4795 return getTruncatedShiftCount(MI, 3) != 0; 4796 4797 case X86::SUB64ri32: 4798 case X86::SUB32ri: 4799 case X86::SUB16ri: 4800 case X86::SUB8ri: 4801 case X86::SUB64rr: 4802 case X86::SUB32rr: 4803 case X86::SUB16rr: 4804 case X86::SUB8rr: 4805 case X86::SUB64rm: 4806 case X86::SUB32rm: 4807 case X86::SUB16rm: 4808 case X86::SUB8rm: 4809 case X86::DEC64r: 4810 case X86::DEC32r: 4811 case X86::DEC16r: 4812 case X86::DEC8r: 4813 case X86::ADD64ri32: 4814 case X86::ADD32ri: 4815 case X86::ADD16ri: 4816 case X86::ADD8ri: 4817 case X86::ADD64rr: 4818 case X86::ADD32rr: 4819 case X86::ADD16rr: 4820 case X86::ADD8rr: 4821 case X86::ADD64rm: 4822 case X86::ADD32rm: 4823 case X86::ADD16rm: 4824 case X86::ADD8rm: 4825 case X86::INC64r: 4826 case X86::INC32r: 4827 case X86::INC16r: 4828 case X86::INC8r: 4829 case X86::ADC64ri32: 4830 case X86::ADC32ri: 4831 case X86::ADC16ri: 4832 case X86::ADC8ri: 4833 case X86::ADC64rr: 4834 case X86::ADC32rr: 4835 case X86::ADC16rr: 4836 case X86::ADC8rr: 4837 case X86::ADC64rm: 4838 case X86::ADC32rm: 4839 case X86::ADC16rm: 4840 case X86::ADC8rm: 4841 case X86::SBB64ri32: 4842 case X86::SBB32ri: 4843 case X86::SBB16ri: 4844 case X86::SBB8ri: 4845 case X86::SBB64rr: 4846 case X86::SBB32rr: 4847 case X86::SBB16rr: 4848 case X86::SBB8rr: 4849 case X86::SBB64rm: 4850 case X86::SBB32rm: 4851 case X86::SBB16rm: 4852 case X86::SBB8rm: 4853 case X86::NEG8r: 4854 case X86::NEG16r: 4855 case X86::NEG32r: 4856 case X86::NEG64r: 4857 case X86::LZCNT16rr: 4858 case X86::LZCNT16rm: 4859 case X86::LZCNT32rr: 4860 case X86::LZCNT32rm: 4861 case X86::LZCNT64rr: 4862 case X86::LZCNT64rm: 4863 case X86::POPCNT16rr: 4864 case X86::POPCNT16rm: 4865 case X86::POPCNT32rr: 4866 case X86::POPCNT32rm: 4867 case X86::POPCNT64rr: 4868 case X86::POPCNT64rm: 4869 case X86::TZCNT16rr: 4870 case X86::TZCNT16rm: 4871 case X86::TZCNT32rr: 4872 case X86::TZCNT32rm: 4873 case X86::TZCNT64rr: 4874 case X86::TZCNT64rm: 4875 return true; 4876 case X86::AND64ri32: 4877 case X86::AND32ri: 4878 case X86::AND16ri: 4879 case X86::AND8ri: 4880 case X86::AND64rr: 4881 case X86::AND32rr: 4882 case X86::AND16rr: 4883 case X86::AND8rr: 4884 case X86::AND64rm: 4885 case X86::AND32rm: 4886 case X86::AND16rm: 4887 case X86::AND8rm: 4888 case X86::XOR64ri32: 4889 case X86::XOR32ri: 4890 case X86::XOR16ri: 4891 case X86::XOR8ri: 4892 case X86::XOR64rr: 4893 case X86::XOR32rr: 4894 case X86::XOR16rr: 4895 case X86::XOR8rr: 4896 case X86::XOR64rm: 4897 case X86::XOR32rm: 4898 case X86::XOR16rm: 4899 case X86::XOR8rm: 4900 case X86::OR64ri32: 4901 case X86::OR32ri: 4902 case X86::OR16ri: 4903 case X86::OR8ri: 4904 case X86::OR64rr: 4905 case X86::OR32rr: 4906 case X86::OR16rr: 4907 case X86::OR8rr: 4908 case X86::OR64rm: 4909 case X86::OR32rm: 4910 case X86::OR16rm: 4911 case X86::OR8rm: 4912 case X86::ANDN32rr: 4913 case X86::ANDN32rm: 4914 case X86::ANDN64rr: 4915 case X86::ANDN64rm: 4916 case X86::BLSI32rr: 4917 case X86::BLSI32rm: 4918 case X86::BLSI64rr: 4919 case X86::BLSI64rm: 4920 case X86::BLSMSK32rr: 4921 case X86::BLSMSK32rm: 4922 case X86::BLSMSK64rr: 4923 case X86::BLSMSK64rm: 4924 case X86::BLSR32rr: 4925 case X86::BLSR32rm: 4926 case X86::BLSR64rr: 4927 case X86::BLSR64rm: 4928 case X86::BLCFILL32rr: 4929 case X86::BLCFILL32rm: 4930 case X86::BLCFILL64rr: 4931 case X86::BLCFILL64rm: 4932 case X86::BLCI32rr: 4933 case X86::BLCI32rm: 4934 case X86::BLCI64rr: 4935 case X86::BLCI64rm: 4936 case X86::BLCIC32rr: 4937 case X86::BLCIC32rm: 4938 case X86::BLCIC64rr: 4939 case X86::BLCIC64rm: 4940 case X86::BLCMSK32rr: 4941 case X86::BLCMSK32rm: 4942 case X86::BLCMSK64rr: 4943 case X86::BLCMSK64rm: 4944 case X86::BLCS32rr: 4945 case X86::BLCS32rm: 4946 case X86::BLCS64rr: 4947 case X86::BLCS64rm: 4948 case X86::BLSFILL32rr: 4949 case X86::BLSFILL32rm: 4950 case X86::BLSFILL64rr: 4951 case X86::BLSFILL64rm: 4952 case X86::BLSIC32rr: 4953 case X86::BLSIC32rm: 4954 case X86::BLSIC64rr: 4955 case X86::BLSIC64rm: 4956 case X86::BZHI32rr: 4957 case X86::BZHI32rm: 4958 case X86::BZHI64rr: 4959 case X86::BZHI64rm: 4960 case X86::T1MSKC32rr: 4961 case X86::T1MSKC32rm: 4962 case X86::T1MSKC64rr: 4963 case X86::T1MSKC64rm: 4964 case X86::TZMSK32rr: 4965 case X86::TZMSK32rm: 4966 case X86::TZMSK64rr: 4967 case X86::TZMSK64rm: 4968 // These instructions clear the overflow flag just like TEST. 4969 // FIXME: These are not the only instructions in this switch that clear the 4970 // overflow flag. 4971 ClearsOverflowFlag = true; 4972 return true; 4973 case X86::BEXTR32rr: 4974 case X86::BEXTR64rr: 4975 case X86::BEXTR32rm: 4976 case X86::BEXTR64rm: 4977 case X86::BEXTRI32ri: 4978 case X86::BEXTRI32mi: 4979 case X86::BEXTRI64ri: 4980 case X86::BEXTRI64mi: 4981 // BEXTR doesn't update the sign flag so we can't use it. It does clear 4982 // the overflow flag, but that's not useful without the sign flag. 4983 NoSignFlag = true; 4984 return true; 4985 } 4986 } 4987 4988 /// Check whether the use can be converted to remove a comparison against zero. 4989 static X86::CondCode isUseDefConvertible(const MachineInstr &MI) { 4990 switch (MI.getOpcode()) { 4991 default: 4992 return X86::COND_INVALID; 4993 case X86::NEG8r: 4994 case X86::NEG16r: 4995 case X86::NEG32r: 4996 case X86::NEG64r: 4997 return X86::COND_AE; 4998 case X86::LZCNT16rr: 4999 case X86::LZCNT32rr: 5000 case X86::LZCNT64rr: 5001 return X86::COND_B; 5002 case X86::POPCNT16rr: 5003 case X86::POPCNT32rr: 5004 case X86::POPCNT64rr: 5005 return X86::COND_E; 5006 case X86::TZCNT16rr: 5007 case X86::TZCNT32rr: 5008 case X86::TZCNT64rr: 5009 return X86::COND_B; 5010 case X86::BSF16rr: 5011 case X86::BSF32rr: 5012 case X86::BSF64rr: 5013 case X86::BSR16rr: 5014 case X86::BSR32rr: 5015 case X86::BSR64rr: 5016 return X86::COND_E; 5017 case X86::BLSI32rr: 5018 case X86::BLSI64rr: 5019 return X86::COND_AE; 5020 case X86::BLSR32rr: 5021 case X86::BLSR64rr: 5022 case X86::BLSMSK32rr: 5023 case X86::BLSMSK64rr: 5024 return X86::COND_B; 5025 // TODO: TBM instructions. 5026 } 5027 } 5028 5029 /// Check if there exists an earlier instruction that 5030 /// operates on the same source operands and sets flags in the same way as 5031 /// Compare; remove Compare if possible. 5032 bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, 5033 Register SrcReg2, int64_t CmpMask, 5034 int64_t CmpValue, 5035 const MachineRegisterInfo *MRI) const { 5036 // Check whether we can replace SUB with CMP. 5037 switch (CmpInstr.getOpcode()) { 5038 default: 5039 break; 5040 case X86::SUB64ri32: 5041 case X86::SUB32ri: 5042 case X86::SUB16ri: 5043 case X86::SUB8ri: 5044 case X86::SUB64rm: 5045 case X86::SUB32rm: 5046 case X86::SUB16rm: 5047 case X86::SUB8rm: 5048 case X86::SUB64rr: 5049 case X86::SUB32rr: 5050 case X86::SUB16rr: 5051 case X86::SUB8rr: { 5052 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg())) 5053 return false; 5054 // There is no use of the destination register, we can replace SUB with CMP. 5055 unsigned NewOpcode = 0; 5056 switch (CmpInstr.getOpcode()) { 5057 default: 5058 llvm_unreachable("Unreachable!"); 5059 case X86::SUB64rm: 5060 NewOpcode = X86::CMP64rm; 5061 break; 5062 case X86::SUB32rm: 5063 NewOpcode = X86::CMP32rm; 5064 break; 5065 case X86::SUB16rm: 5066 NewOpcode = X86::CMP16rm; 5067 break; 5068 case X86::SUB8rm: 5069 NewOpcode = X86::CMP8rm; 5070 break; 5071 case X86::SUB64rr: 5072 NewOpcode = X86::CMP64rr; 5073 break; 5074 case X86::SUB32rr: 5075 NewOpcode = X86::CMP32rr; 5076 break; 5077 case X86::SUB16rr: 5078 NewOpcode = X86::CMP16rr; 5079 break; 5080 case X86::SUB8rr: 5081 NewOpcode = X86::CMP8rr; 5082 break; 5083 case X86::SUB64ri32: 5084 NewOpcode = X86::CMP64ri32; 5085 break; 5086 case X86::SUB32ri: 5087 NewOpcode = X86::CMP32ri; 5088 break; 5089 case X86::SUB16ri: 5090 NewOpcode = X86::CMP16ri; 5091 break; 5092 case X86::SUB8ri: 5093 NewOpcode = X86::CMP8ri; 5094 break; 5095 } 5096 CmpInstr.setDesc(get(NewOpcode)); 5097 CmpInstr.removeOperand(0); 5098 // Mutating this instruction invalidates any debug data associated with it. 5099 CmpInstr.dropDebugNumber(); 5100 // Fall through to optimize Cmp if Cmp is CMPrr or CMPri. 5101 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm || 5102 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm) 5103 return false; 5104 } 5105 } 5106 5107 // The following code tries to remove the comparison by re-using EFLAGS 5108 // from earlier instructions. 5109 5110 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0); 5111 5112 // Transformation currently requires SSA values. 5113 if (SrcReg2.isPhysical()) 5114 return false; 5115 MachineInstr *SrcRegDef = MRI->getVRegDef(SrcReg); 5116 assert(SrcRegDef && "Must have a definition (SSA)"); 5117 5118 MachineInstr *MI = nullptr; 5119 MachineInstr *Sub = nullptr; 5120 MachineInstr *Movr0Inst = nullptr; 5121 bool NoSignFlag = false; 5122 bool ClearsOverflowFlag = false; 5123 bool ShouldUpdateCC = false; 5124 bool IsSwapped = false; 5125 X86::CondCode NewCC = X86::COND_INVALID; 5126 int64_t ImmDelta = 0; 5127 5128 // Search backward from CmpInstr for the next instruction defining EFLAGS. 5129 const TargetRegisterInfo *TRI = &getRegisterInfo(); 5130 MachineBasicBlock &CmpMBB = *CmpInstr.getParent(); 5131 MachineBasicBlock::reverse_iterator From = 5132 std::next(MachineBasicBlock::reverse_iterator(CmpInstr)); 5133 for (MachineBasicBlock *MBB = &CmpMBB;;) { 5134 for (MachineInstr &Inst : make_range(From, MBB->rend())) { 5135 // Try to use EFLAGS from the instruction defining %SrcReg. Example: 5136 // %eax = addl ... 5137 // ... // EFLAGS not changed 5138 // testl %eax, %eax // <-- can be removed 5139 if (&Inst == SrcRegDef) { 5140 if (IsCmpZero && 5141 isDefConvertible(Inst, NoSignFlag, ClearsOverflowFlag)) { 5142 MI = &Inst; 5143 break; 5144 } 5145 5146 // Look back for the following pattern, in which case the 5147 // test16rr/test64rr instruction could be erased. 5148 // 5149 // Example for test16rr: 5150 // %reg = and32ri %in_reg, 5 5151 // ... // EFLAGS not changed. 5152 // %src_reg = copy %reg.sub_16bit:gr32 5153 // test16rr %src_reg, %src_reg, implicit-def $eflags 5154 // Example for test64rr: 5155 // %reg = and32ri %in_reg, 5 5156 // ... // EFLAGS not changed. 5157 // %src_reg = subreg_to_reg 0, %reg, %subreg.sub_index 5158 // test64rr %src_reg, %src_reg, implicit-def $eflags 5159 MachineInstr *AndInstr = nullptr; 5160 if (IsCmpZero && 5161 findRedundantFlagInstr(CmpInstr, Inst, MRI, &AndInstr, TRI, 5162 NoSignFlag, ClearsOverflowFlag)) { 5163 assert(AndInstr != nullptr && X86::isAND(AndInstr->getOpcode())); 5164 MI = AndInstr; 5165 break; 5166 } 5167 // Cannot find other candidates before definition of SrcReg. 5168 return false; 5169 } 5170 5171 if (Inst.modifiesRegister(X86::EFLAGS, TRI)) { 5172 // Try to use EFLAGS produced by an instruction reading %SrcReg. 5173 // Example: 5174 // %eax = ... 5175 // ... 5176 // popcntl %eax 5177 // ... // EFLAGS not changed 5178 // testl %eax, %eax // <-- can be removed 5179 if (IsCmpZero) { 5180 NewCC = isUseDefConvertible(Inst); 5181 if (NewCC != X86::COND_INVALID && Inst.getOperand(1).isReg() && 5182 Inst.getOperand(1).getReg() == SrcReg) { 5183 ShouldUpdateCC = true; 5184 MI = &Inst; 5185 break; 5186 } 5187 } 5188 5189 // Try to use EFLAGS from an instruction with similar flag results. 5190 // Example: 5191 // sub x, y or cmp x, y 5192 // ... // EFLAGS not changed 5193 // cmp x, y // <-- can be removed 5194 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue, 5195 Inst, &IsSwapped, &ImmDelta)) { 5196 Sub = &Inst; 5197 break; 5198 } 5199 5200 // MOV32r0 is implemented with xor which clobbers condition code. It is 5201 // safe to move up, if the definition to EFLAGS is dead and earlier 5202 // instructions do not read or write EFLAGS. 5203 if (!Movr0Inst && Inst.getOpcode() == X86::MOV32r0 && 5204 Inst.registerDefIsDead(X86::EFLAGS, TRI)) { 5205 Movr0Inst = &Inst; 5206 continue; 5207 } 5208 5209 // Cannot do anything for any other EFLAG changes. 5210 return false; 5211 } 5212 } 5213 5214 if (MI || Sub) 5215 break; 5216 5217 // Reached begin of basic block. Continue in predecessor if there is 5218 // exactly one. 5219 if (MBB->pred_size() != 1) 5220 return false; 5221 MBB = *MBB->pred_begin(); 5222 From = MBB->rbegin(); 5223 } 5224 5225 // Scan forward from the instruction after CmpInstr for uses of EFLAGS. 5226 // It is safe to remove CmpInstr if EFLAGS is redefined or killed. 5227 // If we are done with the basic block, we need to check whether EFLAGS is 5228 // live-out. 5229 bool FlagsMayLiveOut = true; 5230 SmallVector<std::pair<MachineInstr *, X86::CondCode>, 4> OpsToUpdate; 5231 MachineBasicBlock::iterator AfterCmpInstr = 5232 std::next(MachineBasicBlock::iterator(CmpInstr)); 5233 for (MachineInstr &Instr : make_range(AfterCmpInstr, CmpMBB.end())) { 5234 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI); 5235 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI); 5236 // We should check the usage if this instruction uses and updates EFLAGS. 5237 if (!UseEFLAGS && ModifyEFLAGS) { 5238 // It is safe to remove CmpInstr if EFLAGS is updated again. 5239 FlagsMayLiveOut = false; 5240 break; 5241 } 5242 if (!UseEFLAGS && !ModifyEFLAGS) 5243 continue; 5244 5245 // EFLAGS is used by this instruction. 5246 X86::CondCode OldCC = X86::getCondFromMI(Instr); 5247 if ((MI || IsSwapped || ImmDelta != 0) && OldCC == X86::COND_INVALID) 5248 return false; 5249 5250 X86::CondCode ReplacementCC = X86::COND_INVALID; 5251 if (MI) { 5252 switch (OldCC) { 5253 default: 5254 break; 5255 case X86::COND_A: 5256 case X86::COND_AE: 5257 case X86::COND_B: 5258 case X86::COND_BE: 5259 // CF is used, we can't perform this optimization. 5260 return false; 5261 case X86::COND_G: 5262 case X86::COND_GE: 5263 case X86::COND_L: 5264 case X86::COND_LE: 5265 // If SF is used, but the instruction doesn't update the SF, then we 5266 // can't do the optimization. 5267 if (NoSignFlag) 5268 return false; 5269 [[fallthrough]]; 5270 case X86::COND_O: 5271 case X86::COND_NO: 5272 // If OF is used, the instruction needs to clear it like CmpZero does. 5273 if (!ClearsOverflowFlag) 5274 return false; 5275 break; 5276 case X86::COND_S: 5277 case X86::COND_NS: 5278 // If SF is used, but the instruction doesn't update the SF, then we 5279 // can't do the optimization. 5280 if (NoSignFlag) 5281 return false; 5282 break; 5283 } 5284 5285 // If we're updating the condition code check if we have to reverse the 5286 // condition. 5287 if (ShouldUpdateCC) 5288 switch (OldCC) { 5289 default: 5290 return false; 5291 case X86::COND_E: 5292 ReplacementCC = NewCC; 5293 break; 5294 case X86::COND_NE: 5295 ReplacementCC = GetOppositeBranchCondition(NewCC); 5296 break; 5297 } 5298 } else if (IsSwapped) { 5299 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs 5300 // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. 5301 // We swap the condition code and synthesize the new opcode. 5302 ReplacementCC = getSwappedCondition(OldCC); 5303 if (ReplacementCC == X86::COND_INVALID) 5304 return false; 5305 ShouldUpdateCC = true; 5306 } else if (ImmDelta != 0) { 5307 unsigned BitWidth = TRI->getRegSizeInBits(*MRI->getRegClass(SrcReg)); 5308 // Shift amount for min/max constants to adjust for 8/16/32 instruction 5309 // sizes. 5310 switch (OldCC) { 5311 case X86::COND_L: // x <s (C + 1) --> x <=s C 5312 if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue) 5313 return false; 5314 ReplacementCC = X86::COND_LE; 5315 break; 5316 case X86::COND_B: // x <u (C + 1) --> x <=u C 5317 if (ImmDelta != 1 || CmpValue == 0) 5318 return false; 5319 ReplacementCC = X86::COND_BE; 5320 break; 5321 case X86::COND_GE: // x >=s (C + 1) --> x >s C 5322 if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue) 5323 return false; 5324 ReplacementCC = X86::COND_G; 5325 break; 5326 case X86::COND_AE: // x >=u (C + 1) --> x >u C 5327 if (ImmDelta != 1 || CmpValue == 0) 5328 return false; 5329 ReplacementCC = X86::COND_A; 5330 break; 5331 case X86::COND_G: // x >s (C - 1) --> x >=s C 5332 if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue) 5333 return false; 5334 ReplacementCC = X86::COND_GE; 5335 break; 5336 case X86::COND_A: // x >u (C - 1) --> x >=u C 5337 if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue) 5338 return false; 5339 ReplacementCC = X86::COND_AE; 5340 break; 5341 case X86::COND_LE: // x <=s (C - 1) --> x <s C 5342 if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue) 5343 return false; 5344 ReplacementCC = X86::COND_L; 5345 break; 5346 case X86::COND_BE: // x <=u (C - 1) --> x <u C 5347 if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue) 5348 return false; 5349 ReplacementCC = X86::COND_B; 5350 break; 5351 default: 5352 return false; 5353 } 5354 ShouldUpdateCC = true; 5355 } 5356 5357 if (ShouldUpdateCC && ReplacementCC != OldCC) { 5358 // Push the MachineInstr to OpsToUpdate. 5359 // If it is safe to remove CmpInstr, the condition code of these 5360 // instructions will be modified. 5361 OpsToUpdate.push_back(std::make_pair(&Instr, ReplacementCC)); 5362 } 5363 if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) { 5364 // It is safe to remove CmpInstr if EFLAGS is updated again or killed. 5365 FlagsMayLiveOut = false; 5366 break; 5367 } 5368 } 5369 5370 // If we have to update users but EFLAGS is live-out abort, since we cannot 5371 // easily find all of the users. 5372 if ((MI != nullptr || ShouldUpdateCC) && FlagsMayLiveOut) { 5373 for (MachineBasicBlock *Successor : CmpMBB.successors()) 5374 if (Successor->isLiveIn(X86::EFLAGS)) 5375 return false; 5376 } 5377 5378 // The instruction to be updated is either Sub or MI. 5379 assert((MI == nullptr || Sub == nullptr) && "Should not have Sub and MI set"); 5380 Sub = MI != nullptr ? MI : Sub; 5381 MachineBasicBlock *SubBB = Sub->getParent(); 5382 // Move Movr0Inst to the appropriate place before Sub. 5383 if (Movr0Inst) { 5384 // Only move within the same block so we don't accidentally move to a 5385 // block with higher execution frequency. 5386 if (&CmpMBB != SubBB) 5387 return false; 5388 // Look backwards until we find a def that doesn't use the current EFLAGS. 5389 MachineBasicBlock::reverse_iterator InsertI = Sub, 5390 InsertE = Sub->getParent()->rend(); 5391 for (; InsertI != InsertE; ++InsertI) { 5392 MachineInstr *Instr = &*InsertI; 5393 if (!Instr->readsRegister(X86::EFLAGS, TRI) && 5394 Instr->modifiesRegister(X86::EFLAGS, TRI)) { 5395 Movr0Inst->getParent()->remove(Movr0Inst); 5396 Instr->getParent()->insert(MachineBasicBlock::iterator(Instr), 5397 Movr0Inst); 5398 break; 5399 } 5400 } 5401 if (InsertI == InsertE) 5402 return false; 5403 } 5404 5405 // Make sure Sub instruction defines EFLAGS and mark the def live. 5406 MachineOperand *FlagDef = Sub->findRegisterDefOperand(X86::EFLAGS); 5407 assert(FlagDef && "Unable to locate a def EFLAGS operand"); 5408 FlagDef->setIsDead(false); 5409 5410 CmpInstr.eraseFromParent(); 5411 5412 // Modify the condition code of instructions in OpsToUpdate. 5413 for (auto &Op : OpsToUpdate) { 5414 Op.first->getOperand(Op.first->getDesc().getNumOperands() - 1) 5415 .setImm(Op.second); 5416 } 5417 // Add EFLAGS to block live-ins between CmpBB and block of flags producer. 5418 for (MachineBasicBlock *MBB = &CmpMBB; MBB != SubBB; 5419 MBB = *MBB->pred_begin()) { 5420 assert(MBB->pred_size() == 1 && "Expected exactly one predecessor"); 5421 if (!MBB->isLiveIn(X86::EFLAGS)) 5422 MBB->addLiveIn(X86::EFLAGS); 5423 } 5424 return true; 5425 } 5426 5427 /// Try to remove the load by folding it to a register 5428 /// operand at the use. We fold the load instructions if load defines a virtual 5429 /// register, the virtual register is used once in the same BB, and the 5430 /// instructions in-between do not load or store, and have no side effects. 5431 MachineInstr *X86InstrInfo::optimizeLoadInstr(MachineInstr &MI, 5432 const MachineRegisterInfo *MRI, 5433 Register &FoldAsLoadDefReg, 5434 MachineInstr *&DefMI) const { 5435 // Check whether we can move DefMI here. 5436 DefMI = MRI->getVRegDef(FoldAsLoadDefReg); 5437 assert(DefMI); 5438 bool SawStore = false; 5439 if (!DefMI->isSafeToMove(nullptr, SawStore)) 5440 return nullptr; 5441 5442 // Collect information about virtual register operands of MI. 5443 SmallVector<unsigned, 1> SrcOperandIds; 5444 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 5445 MachineOperand &MO = MI.getOperand(i); 5446 if (!MO.isReg()) 5447 continue; 5448 Register Reg = MO.getReg(); 5449 if (Reg != FoldAsLoadDefReg) 5450 continue; 5451 // Do not fold if we have a subreg use or a def. 5452 if (MO.getSubReg() || MO.isDef()) 5453 return nullptr; 5454 SrcOperandIds.push_back(i); 5455 } 5456 if (SrcOperandIds.empty()) 5457 return nullptr; 5458 5459 // Check whether we can fold the def into SrcOperandId. 5460 if (MachineInstr *FoldMI = foldMemoryOperand(MI, SrcOperandIds, *DefMI)) { 5461 FoldAsLoadDefReg = 0; 5462 return FoldMI; 5463 } 5464 5465 return nullptr; 5466 } 5467 5468 /// Convert an ALUrr opcode to corresponding ALUri opcode. Such as 5469 /// ADD32rr ==> ADD32ri 5470 /// ShiftRotate will be set to true if the Opcode is shift or rotate. 5471 /// If the ALUri can be further changed to COPY when the immediate is 0, set 5472 /// CanConvert2Copy to true. 5473 static unsigned ConvertALUrr2ALUri(unsigned Opcode, bool &CanConvert2Copy, 5474 bool &ShiftRotate) { 5475 CanConvert2Copy = false; 5476 ShiftRotate = false; 5477 unsigned NewOpcode = 0; 5478 switch (Opcode) { 5479 case X86::ADD64rr: 5480 NewOpcode = X86::ADD64ri32; 5481 CanConvert2Copy = true; 5482 break; 5483 case X86::ADC64rr: 5484 NewOpcode = X86::ADC64ri32; 5485 break; 5486 case X86::SUB64rr: 5487 NewOpcode = X86::SUB64ri32; 5488 CanConvert2Copy = true; 5489 break; 5490 case X86::SBB64rr: 5491 NewOpcode = X86::SBB64ri32; 5492 break; 5493 case X86::AND64rr: 5494 NewOpcode = X86::AND64ri32; 5495 break; 5496 case X86::OR64rr: 5497 NewOpcode = X86::OR64ri32; 5498 CanConvert2Copy = true; 5499 break; 5500 case X86::XOR64rr: 5501 NewOpcode = X86::XOR64ri32; 5502 CanConvert2Copy = true; 5503 break; 5504 case X86::TEST64rr: 5505 NewOpcode = X86::TEST64ri32; 5506 break; 5507 case X86::CMP64rr: 5508 NewOpcode = X86::CMP64ri32; 5509 break; 5510 case X86::SHR64rCL: 5511 NewOpcode = X86::SHR64ri; 5512 ShiftRotate = true; 5513 break; 5514 case X86::SHL64rCL: 5515 NewOpcode = X86::SHL64ri; 5516 ShiftRotate = true; 5517 break; 5518 case X86::SAR64rCL: 5519 NewOpcode = X86::SAR64ri; 5520 ShiftRotate = true; 5521 break; 5522 case X86::ROL64rCL: 5523 NewOpcode = X86::ROL64ri; 5524 ShiftRotate = true; 5525 break; 5526 case X86::ROR64rCL: 5527 NewOpcode = X86::ROR64ri; 5528 ShiftRotate = true; 5529 break; 5530 case X86::RCL64rCL: 5531 NewOpcode = X86::RCL64ri; 5532 ShiftRotate = true; 5533 break; 5534 case X86::RCR64rCL: 5535 NewOpcode = X86::RCR64ri; 5536 ShiftRotate = true; 5537 break; 5538 case X86::ADD32rr: 5539 NewOpcode = X86::ADD32ri; 5540 CanConvert2Copy = true; 5541 break; 5542 case X86::ADC32rr: 5543 NewOpcode = X86::ADC32ri; 5544 break; 5545 case X86::SUB32rr: 5546 NewOpcode = X86::SUB32ri; 5547 CanConvert2Copy = true; 5548 break; 5549 case X86::SBB32rr: 5550 NewOpcode = X86::SBB32ri; 5551 break; 5552 case X86::AND32rr: 5553 NewOpcode = X86::AND32ri; 5554 break; 5555 case X86::OR32rr: 5556 NewOpcode = X86::OR32ri; 5557 CanConvert2Copy = true; 5558 break; 5559 case X86::XOR32rr: 5560 NewOpcode = X86::XOR32ri; 5561 CanConvert2Copy = true; 5562 break; 5563 case X86::TEST32rr: 5564 NewOpcode = X86::TEST32ri; 5565 break; 5566 case X86::CMP32rr: 5567 NewOpcode = X86::CMP32ri; 5568 break; 5569 case X86::SHR32rCL: 5570 NewOpcode = X86::SHR32ri; 5571 ShiftRotate = true; 5572 break; 5573 case X86::SHL32rCL: 5574 NewOpcode = X86::SHL32ri; 5575 ShiftRotate = true; 5576 break; 5577 case X86::SAR32rCL: 5578 NewOpcode = X86::SAR32ri; 5579 ShiftRotate = true; 5580 break; 5581 case X86::ROL32rCL: 5582 NewOpcode = X86::ROL32ri; 5583 ShiftRotate = true; 5584 break; 5585 case X86::ROR32rCL: 5586 NewOpcode = X86::ROR32ri; 5587 ShiftRotate = true; 5588 break; 5589 case X86::RCL32rCL: 5590 NewOpcode = X86::RCL32ri; 5591 ShiftRotate = true; 5592 break; 5593 case X86::RCR32rCL: 5594 NewOpcode = X86::RCR32ri; 5595 ShiftRotate = true; 5596 break; 5597 } 5598 return NewOpcode; 5599 } 5600 5601 /// Real implementation of FoldImmediate. 5602 /// Reg is assigned ImmVal in DefMI, and is used in UseMI. 5603 /// If MakeChange is true, this function tries to replace Reg by ImmVal in 5604 /// UseMI. If MakeChange is false, just check if folding is possible. 5605 /// Return true if folding is successful or possible. 5606 bool X86InstrInfo::FoldImmediateImpl(MachineInstr &UseMI, MachineInstr *DefMI, 5607 Register Reg, int64_t ImmVal, 5608 MachineRegisterInfo *MRI, 5609 bool MakeChange) const { 5610 bool Modified = false; 5611 bool ShiftRotate = false; 5612 // When ImmVal is 0, some instructions can be changed to COPY. 5613 bool CanChangeToCopy = false; 5614 unsigned Opc = UseMI.getOpcode(); 5615 5616 // 64 bit operations accept sign extended 32 bit immediates. 5617 // 32 bit operations accept all 32 bit immediates, so we don't need to check 5618 // them. 5619 const TargetRegisterClass *RC = nullptr; 5620 if (Reg.isVirtual()) 5621 RC = MRI->getRegClass(Reg); 5622 if ((Reg.isPhysical() && X86::GR64RegClass.contains(Reg)) || 5623 (Reg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC))) { 5624 if (!isInt<32>(ImmVal)) 5625 return false; 5626 } 5627 5628 if (UseMI.findRegisterUseOperand(Reg)->getSubReg()) 5629 return false; 5630 // Immediate has larger code size than register. So avoid folding the 5631 // immediate if it has more than 1 use and we are optimizing for size. 5632 if (UseMI.getMF()->getFunction().hasOptSize() && Reg.isVirtual() && 5633 !MRI->hasOneNonDBGUse(Reg)) 5634 return false; 5635 5636 unsigned NewOpc; 5637 if (Opc == TargetOpcode::COPY) { 5638 Register ToReg = UseMI.getOperand(0).getReg(); 5639 const TargetRegisterClass *RC = nullptr; 5640 if (ToReg.isVirtual()) 5641 RC = MRI->getRegClass(ToReg); 5642 bool GR32Reg = (ToReg.isVirtual() && X86::GR32RegClass.hasSubClassEq(RC)) || 5643 (ToReg.isPhysical() && X86::GR32RegClass.contains(ToReg)); 5644 bool GR64Reg = (ToReg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC)) || 5645 (ToReg.isPhysical() && X86::GR64RegClass.contains(ToReg)); 5646 bool GR8Reg = (ToReg.isVirtual() && X86::GR8RegClass.hasSubClassEq(RC)) || 5647 (ToReg.isPhysical() && X86::GR8RegClass.contains(ToReg)); 5648 5649 if (ImmVal == 0) { 5650 // We have MOV32r0 only. 5651 if (!GR32Reg) 5652 return false; 5653 } 5654 5655 if (GR64Reg) { 5656 if (isUInt<32>(ImmVal)) 5657 NewOpc = X86::MOV32ri64; 5658 else 5659 NewOpc = X86::MOV64ri; 5660 } else if (GR32Reg) { 5661 NewOpc = X86::MOV32ri; 5662 if (ImmVal == 0) { 5663 // MOV32r0 clobbers EFLAGS. 5664 const TargetRegisterInfo *TRI = &getRegisterInfo(); 5665 if (UseMI.getParent()->computeRegisterLiveness( 5666 TRI, X86::EFLAGS, UseMI) != MachineBasicBlock::LQR_Dead) 5667 return false; 5668 5669 // MOV32r0 is different than other cases because it doesn't encode the 5670 // immediate in the instruction. So we directly modify it here. 5671 if (!MakeChange) 5672 return true; 5673 UseMI.setDesc(get(X86::MOV32r0)); 5674 UseMI.removeOperand(UseMI.findRegisterUseOperandIdx(Reg)); 5675 UseMI.addOperand(MachineOperand::CreateReg(X86::EFLAGS, /*isDef=*/true, 5676 /*isImp=*/true, 5677 /*isKill=*/false, 5678 /*isDead=*/true)); 5679 Modified = true; 5680 } 5681 } else if (GR8Reg) 5682 NewOpc = X86::MOV8ri; 5683 else 5684 return false; 5685 } else 5686 NewOpc = ConvertALUrr2ALUri(Opc, CanChangeToCopy, ShiftRotate); 5687 5688 if (!NewOpc) 5689 return false; 5690 5691 // For SUB instructions the immediate can only be the second source operand. 5692 if ((NewOpc == X86::SUB64ri32 || NewOpc == X86::SUB32ri || 5693 NewOpc == X86::SBB64ri32 || NewOpc == X86::SBB32ri) && 5694 UseMI.findRegisterUseOperandIdx(Reg) != 2) 5695 return false; 5696 // For CMP instructions the immediate can only be at index 1. 5697 if ((NewOpc == X86::CMP64ri32 || NewOpc == X86::CMP32ri) && 5698 UseMI.findRegisterUseOperandIdx(Reg) != 1) 5699 return false; 5700 5701 if (ShiftRotate) { 5702 unsigned RegIdx = UseMI.findRegisterUseOperandIdx(Reg); 5703 if (RegIdx < 2) 5704 return false; 5705 if (!isInt<8>(ImmVal)) 5706 return false; 5707 assert(Reg == X86::CL); 5708 5709 if (!MakeChange) 5710 return true; 5711 UseMI.setDesc(get(NewOpc)); 5712 UseMI.removeOperand(RegIdx); 5713 UseMI.addOperand(MachineOperand::CreateImm(ImmVal)); 5714 // Reg is physical register $cl, so we don't know if DefMI is dead through 5715 // MRI. Let the caller handle it, or pass dead-mi-elimination can delete 5716 // the dead physical register define instruction. 5717 return true; 5718 } 5719 5720 if (!MakeChange) 5721 return true; 5722 5723 if (!Modified) { 5724 // Modify the instruction. 5725 if (ImmVal == 0 && CanChangeToCopy && 5726 UseMI.registerDefIsDead(X86::EFLAGS)) { 5727 // %100 = add %101, 0 5728 // ==> 5729 // %100 = COPY %101 5730 UseMI.setDesc(get(TargetOpcode::COPY)); 5731 UseMI.removeOperand(UseMI.findRegisterUseOperandIdx(Reg)); 5732 UseMI.removeOperand(UseMI.findRegisterDefOperandIdx(X86::EFLAGS)); 5733 UseMI.untieRegOperand(0); 5734 UseMI.clearFlag(MachineInstr::MIFlag::NoSWrap); 5735 UseMI.clearFlag(MachineInstr::MIFlag::NoUWrap); 5736 } else { 5737 unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex; 5738 unsigned ImmOpNum = 2; 5739 if (!UseMI.getOperand(0).isDef()) { 5740 Op1 = 0; // TEST, CMP 5741 ImmOpNum = 1; 5742 } 5743 if (Opc == TargetOpcode::COPY) 5744 ImmOpNum = 1; 5745 if (findCommutedOpIndices(UseMI, Op1, Op2) && 5746 UseMI.getOperand(Op1).getReg() == Reg) 5747 commuteInstruction(UseMI); 5748 5749 assert(UseMI.getOperand(ImmOpNum).getReg() == Reg); 5750 UseMI.setDesc(get(NewOpc)); 5751 UseMI.getOperand(ImmOpNum).ChangeToImmediate(ImmVal); 5752 } 5753 } 5754 5755 if (Reg.isVirtual() && MRI->use_nodbg_empty(Reg)) 5756 DefMI->eraseFromBundle(); 5757 5758 return true; 5759 } 5760 5761 /// FoldImmediate - 'Reg' is known to be defined by a move immediate 5762 /// instruction, try to fold the immediate into the use instruction. 5763 bool X86InstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 5764 Register Reg, MachineRegisterInfo *MRI) const { 5765 int64_t ImmVal; 5766 if (!getConstValDefinedInReg(DefMI, Reg, ImmVal)) 5767 return false; 5768 5769 return FoldImmediateImpl(UseMI, &DefMI, Reg, ImmVal, MRI, true); 5770 } 5771 5772 /// Expand a single-def pseudo instruction to a two-addr 5773 /// instruction with two undef reads of the register being defined. 5774 /// This is used for mapping: 5775 /// %xmm4 = V_SET0 5776 /// to: 5777 /// %xmm4 = PXORrr undef %xmm4, undef %xmm4 5778 /// 5779 static bool Expand2AddrUndef(MachineInstrBuilder &MIB, 5780 const MCInstrDesc &Desc) { 5781 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction."); 5782 Register Reg = MIB.getReg(0); 5783 MIB->setDesc(Desc); 5784 5785 // MachineInstr::addOperand() will insert explicit operands before any 5786 // implicit operands. 5787 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); 5788 // But we don't trust that. 5789 assert(MIB.getReg(1) == Reg && MIB.getReg(2) == Reg && "Misplaced operand"); 5790 return true; 5791 } 5792 5793 /// Expand a single-def pseudo instruction to a two-addr 5794 /// instruction with two %k0 reads. 5795 /// This is used for mapping: 5796 /// %k4 = K_SET1 5797 /// to: 5798 /// %k4 = KXNORrr %k0, %k0 5799 static bool Expand2AddrKreg(MachineInstrBuilder &MIB, const MCInstrDesc &Desc, 5800 Register Reg) { 5801 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction."); 5802 MIB->setDesc(Desc); 5803 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); 5804 return true; 5805 } 5806 5807 static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, 5808 bool MinusOne) { 5809 MachineBasicBlock &MBB = *MIB->getParent(); 5810 const DebugLoc &DL = MIB->getDebugLoc(); 5811 Register Reg = MIB.getReg(0); 5812 5813 // Insert the XOR. 5814 BuildMI(MBB, MIB.getInstr(), DL, TII.get(X86::XOR32rr), Reg) 5815 .addReg(Reg, RegState::Undef) 5816 .addReg(Reg, RegState::Undef); 5817 5818 // Turn the pseudo into an INC or DEC. 5819 MIB->setDesc(TII.get(MinusOne ? X86::DEC32r : X86::INC32r)); 5820 MIB.addReg(Reg); 5821 5822 return true; 5823 } 5824 5825 static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB, 5826 const TargetInstrInfo &TII, 5827 const X86Subtarget &Subtarget) { 5828 MachineBasicBlock &MBB = *MIB->getParent(); 5829 const DebugLoc &DL = MIB->getDebugLoc(); 5830 int64_t Imm = MIB->getOperand(1).getImm(); 5831 assert(Imm != 0 && "Using push/pop for 0 is not efficient."); 5832 MachineBasicBlock::iterator I = MIB.getInstr(); 5833 5834 int StackAdjustment; 5835 5836 if (Subtarget.is64Bit()) { 5837 assert(MIB->getOpcode() == X86::MOV64ImmSExti8 || 5838 MIB->getOpcode() == X86::MOV32ImmSExti8); 5839 5840 // Can't use push/pop lowering if the function might write to the red zone. 5841 X86MachineFunctionInfo *X86FI = 5842 MBB.getParent()->getInfo<X86MachineFunctionInfo>(); 5843 if (X86FI->getUsesRedZone()) { 5844 MIB->setDesc(TII.get(MIB->getOpcode() == X86::MOV32ImmSExti8 5845 ? X86::MOV32ri 5846 : X86::MOV64ri)); 5847 return true; 5848 } 5849 5850 // 64-bit mode doesn't have 32-bit push/pop, so use 64-bit operations and 5851 // widen the register if necessary. 5852 StackAdjustment = 8; 5853 BuildMI(MBB, I, DL, TII.get(X86::PUSH64i32)).addImm(Imm); 5854 MIB->setDesc(TII.get(X86::POP64r)); 5855 MIB->getOperand(0).setReg(getX86SubSuperRegister(MIB.getReg(0), 64)); 5856 } else { 5857 assert(MIB->getOpcode() == X86::MOV32ImmSExti8); 5858 StackAdjustment = 4; 5859 BuildMI(MBB, I, DL, TII.get(X86::PUSH32i)).addImm(Imm); 5860 MIB->setDesc(TII.get(X86::POP32r)); 5861 } 5862 MIB->removeOperand(1); 5863 MIB->addImplicitDefUseOperands(*MBB.getParent()); 5864 5865 // Build CFI if necessary. 5866 MachineFunction &MF = *MBB.getParent(); 5867 const X86FrameLowering *TFL = Subtarget.getFrameLowering(); 5868 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 5869 bool NeedsDwarfCFI = !IsWin64Prologue && MF.needsFrameMoves(); 5870 bool EmitCFI = !TFL->hasFP(MF) && NeedsDwarfCFI; 5871 if (EmitCFI) { 5872 TFL->BuildCFI( 5873 MBB, I, DL, 5874 MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment)); 5875 TFL->BuildCFI( 5876 MBB, std::next(I), DL, 5877 MCCFIInstruction::createAdjustCfaOffset(nullptr, -StackAdjustment)); 5878 } 5879 5880 return true; 5881 } 5882 5883 // LoadStackGuard has so far only been implemented for 64-bit MachO. Different 5884 // code sequence is needed for other targets. 5885 static void expandLoadStackGuard(MachineInstrBuilder &MIB, 5886 const TargetInstrInfo &TII) { 5887 MachineBasicBlock &MBB = *MIB->getParent(); 5888 const DebugLoc &DL = MIB->getDebugLoc(); 5889 Register Reg = MIB.getReg(0); 5890 const GlobalValue *GV = 5891 cast<GlobalValue>((*MIB->memoperands_begin())->getValue()); 5892 auto Flags = MachineMemOperand::MOLoad | 5893 MachineMemOperand::MODereferenceable | 5894 MachineMemOperand::MOInvariant; 5895 MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( 5896 MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, Align(8)); 5897 MachineBasicBlock::iterator I = MIB.getInstr(); 5898 5899 BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg) 5900 .addReg(X86::RIP) 5901 .addImm(1) 5902 .addReg(0) 5903 .addGlobalAddress(GV, 0, X86II::MO_GOTPCREL) 5904 .addReg(0) 5905 .addMemOperand(MMO); 5906 MIB->setDebugLoc(DL); 5907 MIB->setDesc(TII.get(X86::MOV64rm)); 5908 MIB.addReg(Reg, RegState::Kill).addImm(1).addReg(0).addImm(0).addReg(0); 5909 } 5910 5911 static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII) { 5912 MachineBasicBlock &MBB = *MIB->getParent(); 5913 MachineFunction &MF = *MBB.getParent(); 5914 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>(); 5915 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); 5916 unsigned XorOp = 5917 MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr; 5918 MIB->setDesc(TII.get(XorOp)); 5919 MIB.addReg(TRI->getFrameRegister(MF), RegState::Undef); 5920 return true; 5921 } 5922 5923 // This is used to handle spills for 128/256-bit registers when we have AVX512, 5924 // but not VLX. If it uses an extended register we need to use an instruction 5925 // that loads the lower 128/256-bit, but is available with only AVX512F. 5926 static bool expandNOVLXLoad(MachineInstrBuilder &MIB, 5927 const TargetRegisterInfo *TRI, 5928 const MCInstrDesc &LoadDesc, 5929 const MCInstrDesc &BroadcastDesc, unsigned SubIdx) { 5930 Register DestReg = MIB.getReg(0); 5931 // Check if DestReg is XMM16-31 or YMM16-31. 5932 if (TRI->getEncodingValue(DestReg) < 16) { 5933 // We can use a normal VEX encoded load. 5934 MIB->setDesc(LoadDesc); 5935 } else { 5936 // Use a 128/256-bit VBROADCAST instruction. 5937 MIB->setDesc(BroadcastDesc); 5938 // Change the destination to a 512-bit register. 5939 DestReg = TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass); 5940 MIB->getOperand(0).setReg(DestReg); 5941 } 5942 return true; 5943 } 5944 5945 // This is used to handle spills for 128/256-bit registers when we have AVX512, 5946 // but not VLX. If it uses an extended register we need to use an instruction 5947 // that stores the lower 128/256-bit, but is available with only AVX512F. 5948 static bool expandNOVLXStore(MachineInstrBuilder &MIB, 5949 const TargetRegisterInfo *TRI, 5950 const MCInstrDesc &StoreDesc, 5951 const MCInstrDesc &ExtractDesc, unsigned SubIdx) { 5952 Register SrcReg = MIB.getReg(X86::AddrNumOperands); 5953 // Check if DestReg is XMM16-31 or YMM16-31. 5954 if (TRI->getEncodingValue(SrcReg) < 16) { 5955 // We can use a normal VEX encoded store. 5956 MIB->setDesc(StoreDesc); 5957 } else { 5958 // Use a VEXTRACTF instruction. 5959 MIB->setDesc(ExtractDesc); 5960 // Change the destination to a 512-bit register. 5961 SrcReg = TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass); 5962 MIB->getOperand(X86::AddrNumOperands).setReg(SrcReg); 5963 MIB.addImm(0x0); // Append immediate to extract from the lower bits. 5964 } 5965 5966 return true; 5967 } 5968 5969 static bool expandSHXDROT(MachineInstrBuilder &MIB, const MCInstrDesc &Desc) { 5970 MIB->setDesc(Desc); 5971 int64_t ShiftAmt = MIB->getOperand(2).getImm(); 5972 // Temporarily remove the immediate so we can add another source register. 5973 MIB->removeOperand(2); 5974 // Add the register. Don't copy the kill flag if there is one. 5975 MIB.addReg(MIB.getReg(1), getUndefRegState(MIB->getOperand(1).isUndef())); 5976 // Add back the immediate. 5977 MIB.addImm(ShiftAmt); 5978 return true; 5979 } 5980 5981 bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 5982 bool HasAVX = Subtarget.hasAVX(); 5983 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI); 5984 switch (MI.getOpcode()) { 5985 case X86::MOV32r0: 5986 return Expand2AddrUndef(MIB, get(X86::XOR32rr)); 5987 case X86::MOV32r1: 5988 return expandMOV32r1(MIB, *this, /*MinusOne=*/false); 5989 case X86::MOV32r_1: 5990 return expandMOV32r1(MIB, *this, /*MinusOne=*/true); 5991 case X86::MOV32ImmSExti8: 5992 case X86::MOV64ImmSExti8: 5993 return ExpandMOVImmSExti8(MIB, *this, Subtarget); 5994 case X86::SETB_C32r: 5995 return Expand2AddrUndef(MIB, get(X86::SBB32rr)); 5996 case X86::SETB_C64r: 5997 return Expand2AddrUndef(MIB, get(X86::SBB64rr)); 5998 case X86::MMX_SET0: 5999 return Expand2AddrUndef(MIB, get(X86::MMX_PXORrr)); 6000 case X86::V_SET0: 6001 case X86::FsFLD0SS: 6002 case X86::FsFLD0SD: 6003 case X86::FsFLD0SH: 6004 case X86::FsFLD0F128: 6005 return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr)); 6006 case X86::AVX_SET0: { 6007 assert(HasAVX && "AVX not supported"); 6008 const TargetRegisterInfo *TRI = &getRegisterInfo(); 6009 Register SrcReg = MIB.getReg(0); 6010 Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); 6011 MIB->getOperand(0).setReg(XReg); 6012 Expand2AddrUndef(MIB, get(X86::VXORPSrr)); 6013 MIB.addReg(SrcReg, RegState::ImplicitDefine); 6014 return true; 6015 } 6016 case X86::AVX512_128_SET0: 6017 case X86::AVX512_FsFLD0SH: 6018 case X86::AVX512_FsFLD0SS: 6019 case X86::AVX512_FsFLD0SD: 6020 case X86::AVX512_FsFLD0F128: { 6021 bool HasVLX = Subtarget.hasVLX(); 6022 Register SrcReg = MIB.getReg(0); 6023 const TargetRegisterInfo *TRI = &getRegisterInfo(); 6024 if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) 6025 return Expand2AddrUndef(MIB, 6026 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); 6027 // Extended register without VLX. Use a larger XOR. 6028 SrcReg = 6029 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass); 6030 MIB->getOperand(0).setReg(SrcReg); 6031 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); 6032 } 6033 case X86::AVX512_256_SET0: 6034 case X86::AVX512_512_SET0: { 6035 bool HasVLX = Subtarget.hasVLX(); 6036 Register SrcReg = MIB.getReg(0); 6037 const TargetRegisterInfo *TRI = &getRegisterInfo(); 6038 if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) { 6039 Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); 6040 MIB->getOperand(0).setReg(XReg); 6041 Expand2AddrUndef(MIB, get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); 6042 MIB.addReg(SrcReg, RegState::ImplicitDefine); 6043 return true; 6044 } 6045 if (MI.getOpcode() == X86::AVX512_256_SET0) { 6046 // No VLX so we must reference a zmm. 6047 unsigned ZReg = 6048 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass); 6049 MIB->getOperand(0).setReg(ZReg); 6050 } 6051 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); 6052 } 6053 case X86::V_SETALLONES: 6054 return Expand2AddrUndef(MIB, 6055 get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr)); 6056 case X86::AVX2_SETALLONES: 6057 return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr)); 6058 case X86::AVX1_SETALLONES: { 6059 Register Reg = MIB.getReg(0); 6060 // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS. 6061 MIB->setDesc(get(X86::VCMPPSYrri)); 6062 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xf); 6063 return true; 6064 } 6065 case X86::AVX512_512_SETALLONES: { 6066 Register Reg = MIB.getReg(0); 6067 MIB->setDesc(get(X86::VPTERNLOGDZrri)); 6068 // VPTERNLOGD needs 3 register inputs and an immediate. 6069 // 0xff will return 1s for any input. 6070 MIB.addReg(Reg, RegState::Undef) 6071 .addReg(Reg, RegState::Undef) 6072 .addReg(Reg, RegState::Undef) 6073 .addImm(0xff); 6074 return true; 6075 } 6076 case X86::AVX512_512_SEXT_MASK_32: 6077 case X86::AVX512_512_SEXT_MASK_64: { 6078 Register Reg = MIB.getReg(0); 6079 Register MaskReg = MIB.getReg(1); 6080 unsigned MaskState = getRegState(MIB->getOperand(1)); 6081 unsigned Opc = (MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64) 6082 ? X86::VPTERNLOGQZrrikz 6083 : X86::VPTERNLOGDZrrikz; 6084 MI.removeOperand(1); 6085 MIB->setDesc(get(Opc)); 6086 // VPTERNLOG needs 3 register inputs and an immediate. 6087 // 0xff will return 1s for any input. 6088 MIB.addReg(Reg, RegState::Undef) 6089 .addReg(MaskReg, MaskState) 6090 .addReg(Reg, RegState::Undef) 6091 .addReg(Reg, RegState::Undef) 6092 .addImm(0xff); 6093 return true; 6094 } 6095 case X86::VMOVAPSZ128rm_NOVLX: 6096 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSrm), 6097 get(X86::VBROADCASTF32X4rm), X86::sub_xmm); 6098 case X86::VMOVUPSZ128rm_NOVLX: 6099 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSrm), 6100 get(X86::VBROADCASTF32X4rm), X86::sub_xmm); 6101 case X86::VMOVAPSZ256rm_NOVLX: 6102 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSYrm), 6103 get(X86::VBROADCASTF64X4rm), X86::sub_ymm); 6104 case X86::VMOVUPSZ256rm_NOVLX: 6105 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSYrm), 6106 get(X86::VBROADCASTF64X4rm), X86::sub_ymm); 6107 case X86::VMOVAPSZ128mr_NOVLX: 6108 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSmr), 6109 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); 6110 case X86::VMOVUPSZ128mr_NOVLX: 6111 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSmr), 6112 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); 6113 case X86::VMOVAPSZ256mr_NOVLX: 6114 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSYmr), 6115 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); 6116 case X86::VMOVUPSZ256mr_NOVLX: 6117 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSYmr), 6118 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); 6119 case X86::MOV32ri64: { 6120 Register Reg = MIB.getReg(0); 6121 Register Reg32 = RI.getSubReg(Reg, X86::sub_32bit); 6122 MI.setDesc(get(X86::MOV32ri)); 6123 MIB->getOperand(0).setReg(Reg32); 6124 MIB.addReg(Reg, RegState::ImplicitDefine); 6125 return true; 6126 } 6127 6128 case X86::RDFLAGS32: 6129 case X86::RDFLAGS64: { 6130 unsigned Is64Bit = MI.getOpcode() == X86::RDFLAGS64; 6131 MachineBasicBlock &MBB = *MIB->getParent(); 6132 6133 MachineInstr *NewMI = BuildMI(MBB, MI, MIB->getDebugLoc(), 6134 get(Is64Bit ? X86::PUSHF64 : X86::PUSHF32)) 6135 .getInstr(); 6136 6137 // Permit reads of the EFLAGS and DF registers without them being defined. 6138 // This intrinsic exists to read external processor state in flags, such as 6139 // the trap flag, interrupt flag, and direction flag, none of which are 6140 // modeled by the backend. 6141 assert(NewMI->getOperand(2).getReg() == X86::EFLAGS && 6142 "Unexpected register in operand! Should be EFLAGS."); 6143 NewMI->getOperand(2).setIsUndef(); 6144 assert(NewMI->getOperand(3).getReg() == X86::DF && 6145 "Unexpected register in operand! Should be DF."); 6146 NewMI->getOperand(3).setIsUndef(); 6147 6148 MIB->setDesc(get(Is64Bit ? X86::POP64r : X86::POP32r)); 6149 return true; 6150 } 6151 6152 case X86::WRFLAGS32: 6153 case X86::WRFLAGS64: { 6154 unsigned Is64Bit = MI.getOpcode() == X86::WRFLAGS64; 6155 MachineBasicBlock &MBB = *MIB->getParent(); 6156 6157 BuildMI(MBB, MI, MIB->getDebugLoc(), 6158 get(Is64Bit ? X86::PUSH64r : X86::PUSH32r)) 6159 .addReg(MI.getOperand(0).getReg()); 6160 BuildMI(MBB, MI, MIB->getDebugLoc(), 6161 get(Is64Bit ? X86::POPF64 : X86::POPF32)); 6162 MI.eraseFromParent(); 6163 return true; 6164 } 6165 6166 // KNL does not recognize dependency-breaking idioms for mask registers, 6167 // so kxnor %k1, %k1, %k2 has a RAW dependence on %k1. 6168 // Using %k0 as the undef input register is a performance heuristic based 6169 // on the assumption that %k0 is used less frequently than the other mask 6170 // registers, since it is not usable as a write mask. 6171 // FIXME: A more advanced approach would be to choose the best input mask 6172 // register based on context. 6173 case X86::KSET0W: 6174 return Expand2AddrKreg(MIB, get(X86::KXORWrr), X86::K0); 6175 case X86::KSET0D: 6176 return Expand2AddrKreg(MIB, get(X86::KXORDrr), X86::K0); 6177 case X86::KSET0Q: 6178 return Expand2AddrKreg(MIB, get(X86::KXORQrr), X86::K0); 6179 case X86::KSET1W: 6180 return Expand2AddrKreg(MIB, get(X86::KXNORWrr), X86::K0); 6181 case X86::KSET1D: 6182 return Expand2AddrKreg(MIB, get(X86::KXNORDrr), X86::K0); 6183 case X86::KSET1Q: 6184 return Expand2AddrKreg(MIB, get(X86::KXNORQrr), X86::K0); 6185 case TargetOpcode::LOAD_STACK_GUARD: 6186 expandLoadStackGuard(MIB, *this); 6187 return true; 6188 case X86::XOR64_FP: 6189 case X86::XOR32_FP: 6190 return expandXorFP(MIB, *this); 6191 case X86::SHLDROT32ri: 6192 return expandSHXDROT(MIB, get(X86::SHLD32rri8)); 6193 case X86::SHLDROT64ri: 6194 return expandSHXDROT(MIB, get(X86::SHLD64rri8)); 6195 case X86::SHRDROT32ri: 6196 return expandSHXDROT(MIB, get(X86::SHRD32rri8)); 6197 case X86::SHRDROT64ri: 6198 return expandSHXDROT(MIB, get(X86::SHRD64rri8)); 6199 case X86::ADD8rr_DB: 6200 MIB->setDesc(get(X86::OR8rr)); 6201 break; 6202 case X86::ADD16rr_DB: 6203 MIB->setDesc(get(X86::OR16rr)); 6204 break; 6205 case X86::ADD32rr_DB: 6206 MIB->setDesc(get(X86::OR32rr)); 6207 break; 6208 case X86::ADD64rr_DB: 6209 MIB->setDesc(get(X86::OR64rr)); 6210 break; 6211 case X86::ADD8ri_DB: 6212 MIB->setDesc(get(X86::OR8ri)); 6213 break; 6214 case X86::ADD16ri_DB: 6215 MIB->setDesc(get(X86::OR16ri)); 6216 break; 6217 case X86::ADD32ri_DB: 6218 MIB->setDesc(get(X86::OR32ri)); 6219 break; 6220 case X86::ADD64ri32_DB: 6221 MIB->setDesc(get(X86::OR64ri32)); 6222 break; 6223 } 6224 return false; 6225 } 6226 6227 /// Return true for all instructions that only update 6228 /// the first 32 or 64-bits of the destination register and leave the rest 6229 /// unmodified. This can be used to avoid folding loads if the instructions 6230 /// only update part of the destination register, and the non-updated part is 6231 /// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these 6232 /// instructions breaks the partial register dependency and it can improve 6233 /// performance. e.g.: 6234 /// 6235 /// movss (%rdi), %xmm0 6236 /// cvtss2sd %xmm0, %xmm0 6237 /// 6238 /// Instead of 6239 /// cvtss2sd (%rdi), %xmm0 6240 /// 6241 /// FIXME: This should be turned into a TSFlags. 6242 /// 6243 static bool hasPartialRegUpdate(unsigned Opcode, const X86Subtarget &Subtarget, 6244 bool ForLoadFold = false) { 6245 switch (Opcode) { 6246 case X86::CVTSI2SSrr: 6247 case X86::CVTSI2SSrm: 6248 case X86::CVTSI642SSrr: 6249 case X86::CVTSI642SSrm: 6250 case X86::CVTSI2SDrr: 6251 case X86::CVTSI2SDrm: 6252 case X86::CVTSI642SDrr: 6253 case X86::CVTSI642SDrm: 6254 // Load folding won't effect the undef register update since the input is 6255 // a GPR. 6256 return !ForLoadFold; 6257 case X86::CVTSD2SSrr: 6258 case X86::CVTSD2SSrm: 6259 case X86::CVTSS2SDrr: 6260 case X86::CVTSS2SDrm: 6261 case X86::MOVHPDrm: 6262 case X86::MOVHPSrm: 6263 case X86::MOVLPDrm: 6264 case X86::MOVLPSrm: 6265 case X86::RCPSSr: 6266 case X86::RCPSSm: 6267 case X86::RCPSSr_Int: 6268 case X86::RCPSSm_Int: 6269 case X86::ROUNDSDr: 6270 case X86::ROUNDSDm: 6271 case X86::ROUNDSSr: 6272 case X86::ROUNDSSm: 6273 case X86::RSQRTSSr: 6274 case X86::RSQRTSSm: 6275 case X86::RSQRTSSr_Int: 6276 case X86::RSQRTSSm_Int: 6277 case X86::SQRTSSr: 6278 case X86::SQRTSSm: 6279 case X86::SQRTSSr_Int: 6280 case X86::SQRTSSm_Int: 6281 case X86::SQRTSDr: 6282 case X86::SQRTSDm: 6283 case X86::SQRTSDr_Int: 6284 case X86::SQRTSDm_Int: 6285 return true; 6286 case X86::VFCMULCPHZ128rm: 6287 case X86::VFCMULCPHZ128rmb: 6288 case X86::VFCMULCPHZ128rmbkz: 6289 case X86::VFCMULCPHZ128rmkz: 6290 case X86::VFCMULCPHZ128rr: 6291 case X86::VFCMULCPHZ128rrkz: 6292 case X86::VFCMULCPHZ256rm: 6293 case X86::VFCMULCPHZ256rmb: 6294 case X86::VFCMULCPHZ256rmbkz: 6295 case X86::VFCMULCPHZ256rmkz: 6296 case X86::VFCMULCPHZ256rr: 6297 case X86::VFCMULCPHZ256rrkz: 6298 case X86::VFCMULCPHZrm: 6299 case X86::VFCMULCPHZrmb: 6300 case X86::VFCMULCPHZrmbkz: 6301 case X86::VFCMULCPHZrmkz: 6302 case X86::VFCMULCPHZrr: 6303 case X86::VFCMULCPHZrrb: 6304 case X86::VFCMULCPHZrrbkz: 6305 case X86::VFCMULCPHZrrkz: 6306 case X86::VFMULCPHZ128rm: 6307 case X86::VFMULCPHZ128rmb: 6308 case X86::VFMULCPHZ128rmbkz: 6309 case X86::VFMULCPHZ128rmkz: 6310 case X86::VFMULCPHZ128rr: 6311 case X86::VFMULCPHZ128rrkz: 6312 case X86::VFMULCPHZ256rm: 6313 case X86::VFMULCPHZ256rmb: 6314 case X86::VFMULCPHZ256rmbkz: 6315 case X86::VFMULCPHZ256rmkz: 6316 case X86::VFMULCPHZ256rr: 6317 case X86::VFMULCPHZ256rrkz: 6318 case X86::VFMULCPHZrm: 6319 case X86::VFMULCPHZrmb: 6320 case X86::VFMULCPHZrmbkz: 6321 case X86::VFMULCPHZrmkz: 6322 case X86::VFMULCPHZrr: 6323 case X86::VFMULCPHZrrb: 6324 case X86::VFMULCPHZrrbkz: 6325 case X86::VFMULCPHZrrkz: 6326 case X86::VFCMULCSHZrm: 6327 case X86::VFCMULCSHZrmkz: 6328 case X86::VFCMULCSHZrr: 6329 case X86::VFCMULCSHZrrb: 6330 case X86::VFCMULCSHZrrbkz: 6331 case X86::VFCMULCSHZrrkz: 6332 case X86::VFMULCSHZrm: 6333 case X86::VFMULCSHZrmkz: 6334 case X86::VFMULCSHZrr: 6335 case X86::VFMULCSHZrrb: 6336 case X86::VFMULCSHZrrbkz: 6337 case X86::VFMULCSHZrrkz: 6338 return Subtarget.hasMULCFalseDeps(); 6339 case X86::VPERMDYrm: 6340 case X86::VPERMDYrr: 6341 case X86::VPERMQYmi: 6342 case X86::VPERMQYri: 6343 case X86::VPERMPSYrm: 6344 case X86::VPERMPSYrr: 6345 case X86::VPERMPDYmi: 6346 case X86::VPERMPDYri: 6347 case X86::VPERMDZ256rm: 6348 case X86::VPERMDZ256rmb: 6349 case X86::VPERMDZ256rmbkz: 6350 case X86::VPERMDZ256rmkz: 6351 case X86::VPERMDZ256rr: 6352 case X86::VPERMDZ256rrkz: 6353 case X86::VPERMDZrm: 6354 case X86::VPERMDZrmb: 6355 case X86::VPERMDZrmbkz: 6356 case X86::VPERMDZrmkz: 6357 case X86::VPERMDZrr: 6358 case X86::VPERMDZrrkz: 6359 case X86::VPERMQZ256mbi: 6360 case X86::VPERMQZ256mbikz: 6361 case X86::VPERMQZ256mi: 6362 case X86::VPERMQZ256mikz: 6363 case X86::VPERMQZ256ri: 6364 case X86::VPERMQZ256rikz: 6365 case X86::VPERMQZ256rm: 6366 case X86::VPERMQZ256rmb: 6367 case X86::VPERMQZ256rmbkz: 6368 case X86::VPERMQZ256rmkz: 6369 case X86::VPERMQZ256rr: 6370 case X86::VPERMQZ256rrkz: 6371 case X86::VPERMQZmbi: 6372 case X86::VPERMQZmbikz: 6373 case X86::VPERMQZmi: 6374 case X86::VPERMQZmikz: 6375 case X86::VPERMQZri: 6376 case X86::VPERMQZrikz: 6377 case X86::VPERMQZrm: 6378 case X86::VPERMQZrmb: 6379 case X86::VPERMQZrmbkz: 6380 case X86::VPERMQZrmkz: 6381 case X86::VPERMQZrr: 6382 case X86::VPERMQZrrkz: 6383 case X86::VPERMPSZ256rm: 6384 case X86::VPERMPSZ256rmb: 6385 case X86::VPERMPSZ256rmbkz: 6386 case X86::VPERMPSZ256rmkz: 6387 case X86::VPERMPSZ256rr: 6388 case X86::VPERMPSZ256rrkz: 6389 case X86::VPERMPSZrm: 6390 case X86::VPERMPSZrmb: 6391 case X86::VPERMPSZrmbkz: 6392 case X86::VPERMPSZrmkz: 6393 case X86::VPERMPSZrr: 6394 case X86::VPERMPSZrrkz: 6395 case X86::VPERMPDZ256mbi: 6396 case X86::VPERMPDZ256mbikz: 6397 case X86::VPERMPDZ256mi: 6398 case X86::VPERMPDZ256mikz: 6399 case X86::VPERMPDZ256ri: 6400 case X86::VPERMPDZ256rikz: 6401 case X86::VPERMPDZ256rm: 6402 case X86::VPERMPDZ256rmb: 6403 case X86::VPERMPDZ256rmbkz: 6404 case X86::VPERMPDZ256rmkz: 6405 case X86::VPERMPDZ256rr: 6406 case X86::VPERMPDZ256rrkz: 6407 case X86::VPERMPDZmbi: 6408 case X86::VPERMPDZmbikz: 6409 case X86::VPERMPDZmi: 6410 case X86::VPERMPDZmikz: 6411 case X86::VPERMPDZri: 6412 case X86::VPERMPDZrikz: 6413 case X86::VPERMPDZrm: 6414 case X86::VPERMPDZrmb: 6415 case X86::VPERMPDZrmbkz: 6416 case X86::VPERMPDZrmkz: 6417 case X86::VPERMPDZrr: 6418 case X86::VPERMPDZrrkz: 6419 return Subtarget.hasPERMFalseDeps(); 6420 case X86::VRANGEPDZ128rmbi: 6421 case X86::VRANGEPDZ128rmbikz: 6422 case X86::VRANGEPDZ128rmi: 6423 case X86::VRANGEPDZ128rmikz: 6424 case X86::VRANGEPDZ128rri: 6425 case X86::VRANGEPDZ128rrikz: 6426 case X86::VRANGEPDZ256rmbi: 6427 case X86::VRANGEPDZ256rmbikz: 6428 case X86::VRANGEPDZ256rmi: 6429 case X86::VRANGEPDZ256rmikz: 6430 case X86::VRANGEPDZ256rri: 6431 case X86::VRANGEPDZ256rrikz: 6432 case X86::VRANGEPDZrmbi: 6433 case X86::VRANGEPDZrmbikz: 6434 case X86::VRANGEPDZrmi: 6435 case X86::VRANGEPDZrmikz: 6436 case X86::VRANGEPDZrri: 6437 case X86::VRANGEPDZrrib: 6438 case X86::VRANGEPDZrribkz: 6439 case X86::VRANGEPDZrrikz: 6440 case X86::VRANGEPSZ128rmbi: 6441 case X86::VRANGEPSZ128rmbikz: 6442 case X86::VRANGEPSZ128rmi: 6443 case X86::VRANGEPSZ128rmikz: 6444 case X86::VRANGEPSZ128rri: 6445 case X86::VRANGEPSZ128rrikz: 6446 case X86::VRANGEPSZ256rmbi: 6447 case X86::VRANGEPSZ256rmbikz: 6448 case X86::VRANGEPSZ256rmi: 6449 case X86::VRANGEPSZ256rmikz: 6450 case X86::VRANGEPSZ256rri: 6451 case X86::VRANGEPSZ256rrikz: 6452 case X86::VRANGEPSZrmbi: 6453 case X86::VRANGEPSZrmbikz: 6454 case X86::VRANGEPSZrmi: 6455 case X86::VRANGEPSZrmikz: 6456 case X86::VRANGEPSZrri: 6457 case X86::VRANGEPSZrrib: 6458 case X86::VRANGEPSZrribkz: 6459 case X86::VRANGEPSZrrikz: 6460 case X86::VRANGESDZrmi: 6461 case X86::VRANGESDZrmikz: 6462 case X86::VRANGESDZrri: 6463 case X86::VRANGESDZrrib: 6464 case X86::VRANGESDZrribkz: 6465 case X86::VRANGESDZrrikz: 6466 case X86::VRANGESSZrmi: 6467 case X86::VRANGESSZrmikz: 6468 case X86::VRANGESSZrri: 6469 case X86::VRANGESSZrrib: 6470 case X86::VRANGESSZrribkz: 6471 case X86::VRANGESSZrrikz: 6472 return Subtarget.hasRANGEFalseDeps(); 6473 case X86::VGETMANTSSZrmi: 6474 case X86::VGETMANTSSZrmikz: 6475 case X86::VGETMANTSSZrri: 6476 case X86::VGETMANTSSZrrib: 6477 case X86::VGETMANTSSZrribkz: 6478 case X86::VGETMANTSSZrrikz: 6479 case X86::VGETMANTSDZrmi: 6480 case X86::VGETMANTSDZrmikz: 6481 case X86::VGETMANTSDZrri: 6482 case X86::VGETMANTSDZrrib: 6483 case X86::VGETMANTSDZrribkz: 6484 case X86::VGETMANTSDZrrikz: 6485 case X86::VGETMANTSHZrmi: 6486 case X86::VGETMANTSHZrmikz: 6487 case X86::VGETMANTSHZrri: 6488 case X86::VGETMANTSHZrrib: 6489 case X86::VGETMANTSHZrribkz: 6490 case X86::VGETMANTSHZrrikz: 6491 case X86::VGETMANTPSZ128rmbi: 6492 case X86::VGETMANTPSZ128rmbikz: 6493 case X86::VGETMANTPSZ128rmi: 6494 case X86::VGETMANTPSZ128rmikz: 6495 case X86::VGETMANTPSZ256rmbi: 6496 case X86::VGETMANTPSZ256rmbikz: 6497 case X86::VGETMANTPSZ256rmi: 6498 case X86::VGETMANTPSZ256rmikz: 6499 case X86::VGETMANTPSZrmbi: 6500 case X86::VGETMANTPSZrmbikz: 6501 case X86::VGETMANTPSZrmi: 6502 case X86::VGETMANTPSZrmikz: 6503 case X86::VGETMANTPDZ128rmbi: 6504 case X86::VGETMANTPDZ128rmbikz: 6505 case X86::VGETMANTPDZ128rmi: 6506 case X86::VGETMANTPDZ128rmikz: 6507 case X86::VGETMANTPDZ256rmbi: 6508 case X86::VGETMANTPDZ256rmbikz: 6509 case X86::VGETMANTPDZ256rmi: 6510 case X86::VGETMANTPDZ256rmikz: 6511 case X86::VGETMANTPDZrmbi: 6512 case X86::VGETMANTPDZrmbikz: 6513 case X86::VGETMANTPDZrmi: 6514 case X86::VGETMANTPDZrmikz: 6515 return Subtarget.hasGETMANTFalseDeps(); 6516 case X86::VPMULLQZ128rm: 6517 case X86::VPMULLQZ128rmb: 6518 case X86::VPMULLQZ128rmbkz: 6519 case X86::VPMULLQZ128rmkz: 6520 case X86::VPMULLQZ128rr: 6521 case X86::VPMULLQZ128rrkz: 6522 case X86::VPMULLQZ256rm: 6523 case X86::VPMULLQZ256rmb: 6524 case X86::VPMULLQZ256rmbkz: 6525 case X86::VPMULLQZ256rmkz: 6526 case X86::VPMULLQZ256rr: 6527 case X86::VPMULLQZ256rrkz: 6528 case X86::VPMULLQZrm: 6529 case X86::VPMULLQZrmb: 6530 case X86::VPMULLQZrmbkz: 6531 case X86::VPMULLQZrmkz: 6532 case X86::VPMULLQZrr: 6533 case X86::VPMULLQZrrkz: 6534 return Subtarget.hasMULLQFalseDeps(); 6535 // GPR 6536 case X86::POPCNT32rm: 6537 case X86::POPCNT32rr: 6538 case X86::POPCNT64rm: 6539 case X86::POPCNT64rr: 6540 return Subtarget.hasPOPCNTFalseDeps(); 6541 case X86::LZCNT32rm: 6542 case X86::LZCNT32rr: 6543 case X86::LZCNT64rm: 6544 case X86::LZCNT64rr: 6545 case X86::TZCNT32rm: 6546 case X86::TZCNT32rr: 6547 case X86::TZCNT64rm: 6548 case X86::TZCNT64rr: 6549 return Subtarget.hasLZCNTFalseDeps(); 6550 } 6551 6552 return false; 6553 } 6554 6555 /// Inform the BreakFalseDeps pass how many idle 6556 /// instructions we would like before a partial register update. 6557 unsigned X86InstrInfo::getPartialRegUpdateClearance( 6558 const MachineInstr &MI, unsigned OpNum, 6559 const TargetRegisterInfo *TRI) const { 6560 if (OpNum != 0 || !hasPartialRegUpdate(MI.getOpcode(), Subtarget)) 6561 return 0; 6562 6563 // If MI is marked as reading Reg, the partial register update is wanted. 6564 const MachineOperand &MO = MI.getOperand(0); 6565 Register Reg = MO.getReg(); 6566 if (Reg.isVirtual()) { 6567 if (MO.readsReg() || MI.readsVirtualRegister(Reg)) 6568 return 0; 6569 } else { 6570 if (MI.readsRegister(Reg, TRI)) 6571 return 0; 6572 } 6573 6574 // If any instructions in the clearance range are reading Reg, insert a 6575 // dependency breaking instruction, which is inexpensive and is likely to 6576 // be hidden in other instruction's cycles. 6577 return PartialRegUpdateClearance; 6578 } 6579 6580 // Return true for any instruction the copies the high bits of the first source 6581 // operand into the unused high bits of the destination operand. 6582 // Also returns true for instructions that have two inputs where one may 6583 // be undef and we want it to use the same register as the other input. 6584 static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum, 6585 bool ForLoadFold = false) { 6586 // Set the OpNum parameter to the first source operand. 6587 switch (Opcode) { 6588 case X86::MMX_PUNPCKHBWrr: 6589 case X86::MMX_PUNPCKHWDrr: 6590 case X86::MMX_PUNPCKHDQrr: 6591 case X86::MMX_PUNPCKLBWrr: 6592 case X86::MMX_PUNPCKLWDrr: 6593 case X86::MMX_PUNPCKLDQrr: 6594 case X86::MOVHLPSrr: 6595 case X86::PACKSSWBrr: 6596 case X86::PACKUSWBrr: 6597 case X86::PACKSSDWrr: 6598 case X86::PACKUSDWrr: 6599 case X86::PUNPCKHBWrr: 6600 case X86::PUNPCKLBWrr: 6601 case X86::PUNPCKHWDrr: 6602 case X86::PUNPCKLWDrr: 6603 case X86::PUNPCKHDQrr: 6604 case X86::PUNPCKLDQrr: 6605 case X86::PUNPCKHQDQrr: 6606 case X86::PUNPCKLQDQrr: 6607 case X86::SHUFPDrri: 6608 case X86::SHUFPSrri: 6609 // These instructions are sometimes used with an undef first or second 6610 // source. Return true here so BreakFalseDeps will assign this source to the 6611 // same register as the first source to avoid a false dependency. 6612 // Operand 1 of these instructions is tied so they're separate from their 6613 // VEX counterparts. 6614 return OpNum == 2 && !ForLoadFold; 6615 6616 case X86::VMOVLHPSrr: 6617 case X86::VMOVLHPSZrr: 6618 case X86::VPACKSSWBrr: 6619 case X86::VPACKUSWBrr: 6620 case X86::VPACKSSDWrr: 6621 case X86::VPACKUSDWrr: 6622 case X86::VPACKSSWBZ128rr: 6623 case X86::VPACKUSWBZ128rr: 6624 case X86::VPACKSSDWZ128rr: 6625 case X86::VPACKUSDWZ128rr: 6626 case X86::VPERM2F128rr: 6627 case X86::VPERM2I128rr: 6628 case X86::VSHUFF32X4Z256rri: 6629 case X86::VSHUFF32X4Zrri: 6630 case X86::VSHUFF64X2Z256rri: 6631 case X86::VSHUFF64X2Zrri: 6632 case X86::VSHUFI32X4Z256rri: 6633 case X86::VSHUFI32X4Zrri: 6634 case X86::VSHUFI64X2Z256rri: 6635 case X86::VSHUFI64X2Zrri: 6636 case X86::VPUNPCKHBWrr: 6637 case X86::VPUNPCKLBWrr: 6638 case X86::VPUNPCKHBWYrr: 6639 case X86::VPUNPCKLBWYrr: 6640 case X86::VPUNPCKHBWZ128rr: 6641 case X86::VPUNPCKLBWZ128rr: 6642 case X86::VPUNPCKHBWZ256rr: 6643 case X86::VPUNPCKLBWZ256rr: 6644 case X86::VPUNPCKHBWZrr: 6645 case X86::VPUNPCKLBWZrr: 6646 case X86::VPUNPCKHWDrr: 6647 case X86::VPUNPCKLWDrr: 6648 case X86::VPUNPCKHWDYrr: 6649 case X86::VPUNPCKLWDYrr: 6650 case X86::VPUNPCKHWDZ128rr: 6651 case X86::VPUNPCKLWDZ128rr: 6652 case X86::VPUNPCKHWDZ256rr: 6653 case X86::VPUNPCKLWDZ256rr: 6654 case X86::VPUNPCKHWDZrr: 6655 case X86::VPUNPCKLWDZrr: 6656 case X86::VPUNPCKHDQrr: 6657 case X86::VPUNPCKLDQrr: 6658 case X86::VPUNPCKHDQYrr: 6659 case X86::VPUNPCKLDQYrr: 6660 case X86::VPUNPCKHDQZ128rr: 6661 case X86::VPUNPCKLDQZ128rr: 6662 case X86::VPUNPCKHDQZ256rr: 6663 case X86::VPUNPCKLDQZ256rr: 6664 case X86::VPUNPCKHDQZrr: 6665 case X86::VPUNPCKLDQZrr: 6666 case X86::VPUNPCKHQDQrr: 6667 case X86::VPUNPCKLQDQrr: 6668 case X86::VPUNPCKHQDQYrr: 6669 case X86::VPUNPCKLQDQYrr: 6670 case X86::VPUNPCKHQDQZ128rr: 6671 case X86::VPUNPCKLQDQZ128rr: 6672 case X86::VPUNPCKHQDQZ256rr: 6673 case X86::VPUNPCKLQDQZ256rr: 6674 case X86::VPUNPCKHQDQZrr: 6675 case X86::VPUNPCKLQDQZrr: 6676 // These instructions are sometimes used with an undef first or second 6677 // source. Return true here so BreakFalseDeps will assign this source to the 6678 // same register as the first source to avoid a false dependency. 6679 return (OpNum == 1 || OpNum == 2) && !ForLoadFold; 6680 6681 case X86::VCVTSI2SSrr: 6682 case X86::VCVTSI2SSrm: 6683 case X86::VCVTSI2SSrr_Int: 6684 case X86::VCVTSI2SSrm_Int: 6685 case X86::VCVTSI642SSrr: 6686 case X86::VCVTSI642SSrm: 6687 case X86::VCVTSI642SSrr_Int: 6688 case X86::VCVTSI642SSrm_Int: 6689 case X86::VCVTSI2SDrr: 6690 case X86::VCVTSI2SDrm: 6691 case X86::VCVTSI2SDrr_Int: 6692 case X86::VCVTSI2SDrm_Int: 6693 case X86::VCVTSI642SDrr: 6694 case X86::VCVTSI642SDrm: 6695 case X86::VCVTSI642SDrr_Int: 6696 case X86::VCVTSI642SDrm_Int: 6697 // AVX-512 6698 case X86::VCVTSI2SSZrr: 6699 case X86::VCVTSI2SSZrm: 6700 case X86::VCVTSI2SSZrr_Int: 6701 case X86::VCVTSI2SSZrrb_Int: 6702 case X86::VCVTSI2SSZrm_Int: 6703 case X86::VCVTSI642SSZrr: 6704 case X86::VCVTSI642SSZrm: 6705 case X86::VCVTSI642SSZrr_Int: 6706 case X86::VCVTSI642SSZrrb_Int: 6707 case X86::VCVTSI642SSZrm_Int: 6708 case X86::VCVTSI2SDZrr: 6709 case X86::VCVTSI2SDZrm: 6710 case X86::VCVTSI2SDZrr_Int: 6711 case X86::VCVTSI2SDZrm_Int: 6712 case X86::VCVTSI642SDZrr: 6713 case X86::VCVTSI642SDZrm: 6714 case X86::VCVTSI642SDZrr_Int: 6715 case X86::VCVTSI642SDZrrb_Int: 6716 case X86::VCVTSI642SDZrm_Int: 6717 case X86::VCVTUSI2SSZrr: 6718 case X86::VCVTUSI2SSZrm: 6719 case X86::VCVTUSI2SSZrr_Int: 6720 case X86::VCVTUSI2SSZrrb_Int: 6721 case X86::VCVTUSI2SSZrm_Int: 6722 case X86::VCVTUSI642SSZrr: 6723 case X86::VCVTUSI642SSZrm: 6724 case X86::VCVTUSI642SSZrr_Int: 6725 case X86::VCVTUSI642SSZrrb_Int: 6726 case X86::VCVTUSI642SSZrm_Int: 6727 case X86::VCVTUSI2SDZrr: 6728 case X86::VCVTUSI2SDZrm: 6729 case X86::VCVTUSI2SDZrr_Int: 6730 case X86::VCVTUSI2SDZrm_Int: 6731 case X86::VCVTUSI642SDZrr: 6732 case X86::VCVTUSI642SDZrm: 6733 case X86::VCVTUSI642SDZrr_Int: 6734 case X86::VCVTUSI642SDZrrb_Int: 6735 case X86::VCVTUSI642SDZrm_Int: 6736 case X86::VCVTSI2SHZrr: 6737 case X86::VCVTSI2SHZrm: 6738 case X86::VCVTSI2SHZrr_Int: 6739 case X86::VCVTSI2SHZrrb_Int: 6740 case X86::VCVTSI2SHZrm_Int: 6741 case X86::VCVTSI642SHZrr: 6742 case X86::VCVTSI642SHZrm: 6743 case X86::VCVTSI642SHZrr_Int: 6744 case X86::VCVTSI642SHZrrb_Int: 6745 case X86::VCVTSI642SHZrm_Int: 6746 case X86::VCVTUSI2SHZrr: 6747 case X86::VCVTUSI2SHZrm: 6748 case X86::VCVTUSI2SHZrr_Int: 6749 case X86::VCVTUSI2SHZrrb_Int: 6750 case X86::VCVTUSI2SHZrm_Int: 6751 case X86::VCVTUSI642SHZrr: 6752 case X86::VCVTUSI642SHZrm: 6753 case X86::VCVTUSI642SHZrr_Int: 6754 case X86::VCVTUSI642SHZrrb_Int: 6755 case X86::VCVTUSI642SHZrm_Int: 6756 // Load folding won't effect the undef register update since the input is 6757 // a GPR. 6758 return OpNum == 1 && !ForLoadFold; 6759 case X86::VCVTSD2SSrr: 6760 case X86::VCVTSD2SSrm: 6761 case X86::VCVTSD2SSrr_Int: 6762 case X86::VCVTSD2SSrm_Int: 6763 case X86::VCVTSS2SDrr: 6764 case X86::VCVTSS2SDrm: 6765 case X86::VCVTSS2SDrr_Int: 6766 case X86::VCVTSS2SDrm_Int: 6767 case X86::VRCPSSr: 6768 case X86::VRCPSSr_Int: 6769 case X86::VRCPSSm: 6770 case X86::VRCPSSm_Int: 6771 case X86::VROUNDSDr: 6772 case X86::VROUNDSDm: 6773 case X86::VROUNDSDr_Int: 6774 case X86::VROUNDSDm_Int: 6775 case X86::VROUNDSSr: 6776 case X86::VROUNDSSm: 6777 case X86::VROUNDSSr_Int: 6778 case X86::VROUNDSSm_Int: 6779 case X86::VRSQRTSSr: 6780 case X86::VRSQRTSSr_Int: 6781 case X86::VRSQRTSSm: 6782 case X86::VRSQRTSSm_Int: 6783 case X86::VSQRTSSr: 6784 case X86::VSQRTSSr_Int: 6785 case X86::VSQRTSSm: 6786 case X86::VSQRTSSm_Int: 6787 case X86::VSQRTSDr: 6788 case X86::VSQRTSDr_Int: 6789 case X86::VSQRTSDm: 6790 case X86::VSQRTSDm_Int: 6791 // AVX-512 6792 case X86::VCVTSD2SSZrr: 6793 case X86::VCVTSD2SSZrr_Int: 6794 case X86::VCVTSD2SSZrrb_Int: 6795 case X86::VCVTSD2SSZrm: 6796 case X86::VCVTSD2SSZrm_Int: 6797 case X86::VCVTSS2SDZrr: 6798 case X86::VCVTSS2SDZrr_Int: 6799 case X86::VCVTSS2SDZrrb_Int: 6800 case X86::VCVTSS2SDZrm: 6801 case X86::VCVTSS2SDZrm_Int: 6802 case X86::VGETEXPSDZr: 6803 case X86::VGETEXPSDZrb: 6804 case X86::VGETEXPSDZm: 6805 case X86::VGETEXPSSZr: 6806 case X86::VGETEXPSSZrb: 6807 case X86::VGETEXPSSZm: 6808 case X86::VGETMANTSDZrri: 6809 case X86::VGETMANTSDZrrib: 6810 case X86::VGETMANTSDZrmi: 6811 case X86::VGETMANTSSZrri: 6812 case X86::VGETMANTSSZrrib: 6813 case X86::VGETMANTSSZrmi: 6814 case X86::VRNDSCALESDZr: 6815 case X86::VRNDSCALESDZr_Int: 6816 case X86::VRNDSCALESDZrb_Int: 6817 case X86::VRNDSCALESDZm: 6818 case X86::VRNDSCALESDZm_Int: 6819 case X86::VRNDSCALESSZr: 6820 case X86::VRNDSCALESSZr_Int: 6821 case X86::VRNDSCALESSZrb_Int: 6822 case X86::VRNDSCALESSZm: 6823 case X86::VRNDSCALESSZm_Int: 6824 case X86::VRCP14SDZrr: 6825 case X86::VRCP14SDZrm: 6826 case X86::VRCP14SSZrr: 6827 case X86::VRCP14SSZrm: 6828 case X86::VRCPSHZrr: 6829 case X86::VRCPSHZrm: 6830 case X86::VRSQRTSHZrr: 6831 case X86::VRSQRTSHZrm: 6832 case X86::VREDUCESHZrmi: 6833 case X86::VREDUCESHZrri: 6834 case X86::VREDUCESHZrrib: 6835 case X86::VGETEXPSHZr: 6836 case X86::VGETEXPSHZrb: 6837 case X86::VGETEXPSHZm: 6838 case X86::VGETMANTSHZrri: 6839 case X86::VGETMANTSHZrrib: 6840 case X86::VGETMANTSHZrmi: 6841 case X86::VRNDSCALESHZr: 6842 case X86::VRNDSCALESHZr_Int: 6843 case X86::VRNDSCALESHZrb_Int: 6844 case X86::VRNDSCALESHZm: 6845 case X86::VRNDSCALESHZm_Int: 6846 case X86::VSQRTSHZr: 6847 case X86::VSQRTSHZr_Int: 6848 case X86::VSQRTSHZrb_Int: 6849 case X86::VSQRTSHZm: 6850 case X86::VSQRTSHZm_Int: 6851 case X86::VRCP28SDZr: 6852 case X86::VRCP28SDZrb: 6853 case X86::VRCP28SDZm: 6854 case X86::VRCP28SSZr: 6855 case X86::VRCP28SSZrb: 6856 case X86::VRCP28SSZm: 6857 case X86::VREDUCESSZrmi: 6858 case X86::VREDUCESSZrri: 6859 case X86::VREDUCESSZrrib: 6860 case X86::VRSQRT14SDZrr: 6861 case X86::VRSQRT14SDZrm: 6862 case X86::VRSQRT14SSZrr: 6863 case X86::VRSQRT14SSZrm: 6864 case X86::VRSQRT28SDZr: 6865 case X86::VRSQRT28SDZrb: 6866 case X86::VRSQRT28SDZm: 6867 case X86::VRSQRT28SSZr: 6868 case X86::VRSQRT28SSZrb: 6869 case X86::VRSQRT28SSZm: 6870 case X86::VSQRTSSZr: 6871 case X86::VSQRTSSZr_Int: 6872 case X86::VSQRTSSZrb_Int: 6873 case X86::VSQRTSSZm: 6874 case X86::VSQRTSSZm_Int: 6875 case X86::VSQRTSDZr: 6876 case X86::VSQRTSDZr_Int: 6877 case X86::VSQRTSDZrb_Int: 6878 case X86::VSQRTSDZm: 6879 case X86::VSQRTSDZm_Int: 6880 case X86::VCVTSD2SHZrr: 6881 case X86::VCVTSD2SHZrr_Int: 6882 case X86::VCVTSD2SHZrrb_Int: 6883 case X86::VCVTSD2SHZrm: 6884 case X86::VCVTSD2SHZrm_Int: 6885 case X86::VCVTSS2SHZrr: 6886 case X86::VCVTSS2SHZrr_Int: 6887 case X86::VCVTSS2SHZrrb_Int: 6888 case X86::VCVTSS2SHZrm: 6889 case X86::VCVTSS2SHZrm_Int: 6890 case X86::VCVTSH2SDZrr: 6891 case X86::VCVTSH2SDZrr_Int: 6892 case X86::VCVTSH2SDZrrb_Int: 6893 case X86::VCVTSH2SDZrm: 6894 case X86::VCVTSH2SDZrm_Int: 6895 case X86::VCVTSH2SSZrr: 6896 case X86::VCVTSH2SSZrr_Int: 6897 case X86::VCVTSH2SSZrrb_Int: 6898 case X86::VCVTSH2SSZrm: 6899 case X86::VCVTSH2SSZrm_Int: 6900 return OpNum == 1; 6901 case X86::VMOVSSZrrk: 6902 case X86::VMOVSDZrrk: 6903 return OpNum == 3 && !ForLoadFold; 6904 case X86::VMOVSSZrrkz: 6905 case X86::VMOVSDZrrkz: 6906 return OpNum == 2 && !ForLoadFold; 6907 } 6908 6909 return false; 6910 } 6911 6912 /// Inform the BreakFalseDeps pass how many idle instructions we would like 6913 /// before certain undef register reads. 6914 /// 6915 /// This catches the VCVTSI2SD family of instructions: 6916 /// 6917 /// vcvtsi2sdq %rax, undef %xmm0, %xmm14 6918 /// 6919 /// We should to be careful *not* to catch VXOR idioms which are presumably 6920 /// handled specially in the pipeline: 6921 /// 6922 /// vxorps undef %xmm1, undef %xmm1, %xmm1 6923 /// 6924 /// Like getPartialRegUpdateClearance, this makes a strong assumption that the 6925 /// high bits that are passed-through are not live. 6926 unsigned 6927 X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned OpNum, 6928 const TargetRegisterInfo *TRI) const { 6929 const MachineOperand &MO = MI.getOperand(OpNum); 6930 if (MO.getReg().isPhysical() && hasUndefRegUpdate(MI.getOpcode(), OpNum)) 6931 return UndefRegClearance; 6932 6933 return 0; 6934 } 6935 6936 void X86InstrInfo::breakPartialRegDependency( 6937 MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const { 6938 Register Reg = MI.getOperand(OpNum).getReg(); 6939 // If MI kills this register, the false dependence is already broken. 6940 if (MI.killsRegister(Reg, TRI)) 6941 return; 6942 6943 if (X86::VR128RegClass.contains(Reg)) { 6944 // These instructions are all floating point domain, so xorps is the best 6945 // choice. 6946 unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr; 6947 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(Opc), Reg) 6948 .addReg(Reg, RegState::Undef) 6949 .addReg(Reg, RegState::Undef); 6950 MI.addRegisterKilled(Reg, TRI, true); 6951 } else if (X86::VR256RegClass.contains(Reg)) { 6952 // Use vxorps to clear the full ymm register. 6953 // It wants to read and write the xmm sub-register. 6954 Register XReg = TRI->getSubReg(Reg, X86::sub_xmm); 6955 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VXORPSrr), XReg) 6956 .addReg(XReg, RegState::Undef) 6957 .addReg(XReg, RegState::Undef) 6958 .addReg(Reg, RegState::ImplicitDefine); 6959 MI.addRegisterKilled(Reg, TRI, true); 6960 } else if (X86::VR128XRegClass.contains(Reg)) { 6961 // Only handle VLX targets. 6962 if (!Subtarget.hasVLX()) 6963 return; 6964 // Since vxorps requires AVX512DQ, vpxord should be the best choice. 6965 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), Reg) 6966 .addReg(Reg, RegState::Undef) 6967 .addReg(Reg, RegState::Undef); 6968 MI.addRegisterKilled(Reg, TRI, true); 6969 } else if (X86::VR256XRegClass.contains(Reg) || 6970 X86::VR512RegClass.contains(Reg)) { 6971 // Only handle VLX targets. 6972 if (!Subtarget.hasVLX()) 6973 return; 6974 // Use vpxord to clear the full ymm/zmm register. 6975 // It wants to read and write the xmm sub-register. 6976 Register XReg = TRI->getSubReg(Reg, X86::sub_xmm); 6977 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), XReg) 6978 .addReg(XReg, RegState::Undef) 6979 .addReg(XReg, RegState::Undef) 6980 .addReg(Reg, RegState::ImplicitDefine); 6981 MI.addRegisterKilled(Reg, TRI, true); 6982 } else if (X86::GR64RegClass.contains(Reg)) { 6983 // Using XOR32rr because it has shorter encoding and zeros up the upper bits 6984 // as well. 6985 Register XReg = TRI->getSubReg(Reg, X86::sub_32bit); 6986 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), XReg) 6987 .addReg(XReg, RegState::Undef) 6988 .addReg(XReg, RegState::Undef) 6989 .addReg(Reg, RegState::ImplicitDefine); 6990 MI.addRegisterKilled(Reg, TRI, true); 6991 } else if (X86::GR32RegClass.contains(Reg)) { 6992 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), Reg) 6993 .addReg(Reg, RegState::Undef) 6994 .addReg(Reg, RegState::Undef); 6995 MI.addRegisterKilled(Reg, TRI, true); 6996 } 6997 } 6998 6999 static void addOperands(MachineInstrBuilder &MIB, ArrayRef<MachineOperand> MOs, 7000 int PtrOffset = 0) { 7001 unsigned NumAddrOps = MOs.size(); 7002 7003 if (NumAddrOps < 4) { 7004 // FrameIndex only - add an immediate offset (whether its zero or not). 7005 for (unsigned i = 0; i != NumAddrOps; ++i) 7006 MIB.add(MOs[i]); 7007 addOffset(MIB, PtrOffset); 7008 } else { 7009 // General Memory Addressing - we need to add any offset to an existing 7010 // offset. 7011 assert(MOs.size() == 5 && "Unexpected memory operand list length"); 7012 for (unsigned i = 0; i != NumAddrOps; ++i) { 7013 const MachineOperand &MO = MOs[i]; 7014 if (i == 3 && PtrOffset != 0) { 7015 MIB.addDisp(MO, PtrOffset); 7016 } else { 7017 MIB.add(MO); 7018 } 7019 } 7020 } 7021 } 7022 7023 static void updateOperandRegConstraints(MachineFunction &MF, 7024 MachineInstr &NewMI, 7025 const TargetInstrInfo &TII) { 7026 MachineRegisterInfo &MRI = MF.getRegInfo(); 7027 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); 7028 7029 for (int Idx : llvm::seq<int>(0, NewMI.getNumOperands())) { 7030 MachineOperand &MO = NewMI.getOperand(Idx); 7031 // We only need to update constraints on virtual register operands. 7032 if (!MO.isReg()) 7033 continue; 7034 Register Reg = MO.getReg(); 7035 if (!Reg.isVirtual()) 7036 continue; 7037 7038 auto *NewRC = MRI.constrainRegClass( 7039 Reg, TII.getRegClass(NewMI.getDesc(), Idx, &TRI, MF)); 7040 if (!NewRC) { 7041 LLVM_DEBUG( 7042 dbgs() << "WARNING: Unable to update register constraint for operand " 7043 << Idx << " of instruction:\n"; 7044 NewMI.dump(); dbgs() << "\n"); 7045 } 7046 } 7047 } 7048 7049 static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode, 7050 ArrayRef<MachineOperand> MOs, 7051 MachineBasicBlock::iterator InsertPt, 7052 MachineInstr &MI, 7053 const TargetInstrInfo &TII) { 7054 // Create the base instruction with the memory operand as the first part. 7055 // Omit the implicit operands, something BuildMI can't do. 7056 MachineInstr *NewMI = 7057 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); 7058 MachineInstrBuilder MIB(MF, NewMI); 7059 addOperands(MIB, MOs); 7060 7061 // Loop over the rest of the ri operands, converting them over. 7062 unsigned NumOps = MI.getDesc().getNumOperands() - 2; 7063 for (unsigned i = 0; i != NumOps; ++i) { 7064 MachineOperand &MO = MI.getOperand(i + 2); 7065 MIB.add(MO); 7066 } 7067 for (const MachineOperand &MO : llvm::drop_begin(MI.operands(), NumOps + 2)) 7068 MIB.add(MO); 7069 7070 updateOperandRegConstraints(MF, *NewMI, TII); 7071 7072 MachineBasicBlock *MBB = InsertPt->getParent(); 7073 MBB->insert(InsertPt, NewMI); 7074 7075 return MIB; 7076 } 7077 7078 static MachineInstr *FuseInst(MachineFunction &MF, unsigned Opcode, 7079 unsigned OpNo, ArrayRef<MachineOperand> MOs, 7080 MachineBasicBlock::iterator InsertPt, 7081 MachineInstr &MI, const TargetInstrInfo &TII, 7082 int PtrOffset = 0) { 7083 // Omit the implicit operands, something BuildMI can't do. 7084 MachineInstr *NewMI = 7085 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); 7086 MachineInstrBuilder MIB(MF, NewMI); 7087 7088 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 7089 MachineOperand &MO = MI.getOperand(i); 7090 if (i == OpNo) { 7091 assert(MO.isReg() && "Expected to fold into reg operand!"); 7092 addOperands(MIB, MOs, PtrOffset); 7093 } else { 7094 MIB.add(MO); 7095 } 7096 } 7097 7098 updateOperandRegConstraints(MF, *NewMI, TII); 7099 7100 // Copy the NoFPExcept flag from the instruction we're fusing. 7101 if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept)) 7102 NewMI->setFlag(MachineInstr::MIFlag::NoFPExcept); 7103 7104 MachineBasicBlock *MBB = InsertPt->getParent(); 7105 MBB->insert(InsertPt, NewMI); 7106 7107 return MIB; 7108 } 7109 7110 static MachineInstr *MakeM0Inst(const TargetInstrInfo &TII, unsigned Opcode, 7111 ArrayRef<MachineOperand> MOs, 7112 MachineBasicBlock::iterator InsertPt, 7113 MachineInstr &MI) { 7114 MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, 7115 MI.getDebugLoc(), TII.get(Opcode)); 7116 addOperands(MIB, MOs); 7117 return MIB.addImm(0); 7118 } 7119 7120 MachineInstr *X86InstrInfo::foldMemoryOperandCustom( 7121 MachineFunction &MF, MachineInstr &MI, unsigned OpNum, 7122 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, 7123 unsigned Size, Align Alignment) const { 7124 switch (MI.getOpcode()) { 7125 case X86::INSERTPSrr: 7126 case X86::VINSERTPSrr: 7127 case X86::VINSERTPSZrr: 7128 // Attempt to convert the load of inserted vector into a fold load 7129 // of a single float. 7130 if (OpNum == 2) { 7131 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); 7132 unsigned ZMask = Imm & 15; 7133 unsigned DstIdx = (Imm >> 4) & 3; 7134 unsigned SrcIdx = (Imm >> 6) & 3; 7135 7136 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 7137 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 7138 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 7139 if ((Size == 0 || Size >= 16) && RCSize >= 16 && 7140 (MI.getOpcode() != X86::INSERTPSrr || Alignment >= Align(4))) { 7141 int PtrOffset = SrcIdx * 4; 7142 unsigned NewImm = (DstIdx << 4) | ZMask; 7143 unsigned NewOpCode = 7144 (MI.getOpcode() == X86::VINSERTPSZrr) ? X86::VINSERTPSZrm 7145 : (MI.getOpcode() == X86::VINSERTPSrr) ? X86::VINSERTPSrm 7146 : X86::INSERTPSrm; 7147 MachineInstr *NewMI = 7148 FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, PtrOffset); 7149 NewMI->getOperand(NewMI->getNumOperands() - 1).setImm(NewImm); 7150 return NewMI; 7151 } 7152 } 7153 break; 7154 case X86::MOVHLPSrr: 7155 case X86::VMOVHLPSrr: 7156 case X86::VMOVHLPSZrr: 7157 // Move the upper 64-bits of the second operand to the lower 64-bits. 7158 // To fold the load, adjust the pointer to the upper and use (V)MOVLPS. 7159 // TODO: In most cases AVX doesn't have a 8-byte alignment requirement. 7160 if (OpNum == 2) { 7161 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 7162 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 7163 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 7164 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment >= Align(8)) { 7165 unsigned NewOpCode = 7166 (MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm 7167 : (MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm 7168 : X86::MOVLPSrm; 7169 MachineInstr *NewMI = 7170 FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, 8); 7171 return NewMI; 7172 } 7173 } 7174 break; 7175 case X86::UNPCKLPDrr: 7176 // If we won't be able to fold this to the memory form of UNPCKL, use 7177 // MOVHPD instead. Done as custom because we can't have this in the load 7178 // table twice. 7179 if (OpNum == 2) { 7180 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 7181 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 7182 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 7183 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment < Align(16)) { 7184 MachineInstr *NewMI = 7185 FuseInst(MF, X86::MOVHPDrm, OpNum, MOs, InsertPt, MI, *this); 7186 return NewMI; 7187 } 7188 } 7189 break; 7190 } 7191 7192 return nullptr; 7193 } 7194 7195 static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF, 7196 MachineInstr &MI) { 7197 if (!hasUndefRegUpdate(MI.getOpcode(), 1, /*ForLoadFold*/ true) || 7198 !MI.getOperand(1).isReg()) 7199 return false; 7200 7201 // The are two cases we need to handle depending on where in the pipeline 7202 // the folding attempt is being made. 7203 // -Register has the undef flag set. 7204 // -Register is produced by the IMPLICIT_DEF instruction. 7205 7206 if (MI.getOperand(1).isUndef()) 7207 return true; 7208 7209 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 7210 MachineInstr *VRegDef = RegInfo.getUniqueVRegDef(MI.getOperand(1).getReg()); 7211 return VRegDef && VRegDef->isImplicitDef(); 7212 } 7213 7214 MachineInstr *X86InstrInfo::foldMemoryOperandImpl( 7215 MachineFunction &MF, MachineInstr &MI, unsigned OpNum, 7216 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, 7217 unsigned Size, Align Alignment, bool AllowCommute) const { 7218 bool isSlowTwoMemOps = Subtarget.slowTwoMemOps(); 7219 bool isTwoAddrFold = false; 7220 7221 // For CPUs that favor the register form of a call or push, 7222 // do not fold loads into calls or pushes, unless optimizing for size 7223 // aggressively. 7224 if (isSlowTwoMemOps && !MF.getFunction().hasMinSize() && 7225 (MI.getOpcode() == X86::CALL32r || MI.getOpcode() == X86::CALL64r || 7226 MI.getOpcode() == X86::PUSH16r || MI.getOpcode() == X86::PUSH32r || 7227 MI.getOpcode() == X86::PUSH64r)) 7228 return nullptr; 7229 7230 // Avoid partial and undef register update stalls unless optimizing for size. 7231 if (!MF.getFunction().hasOptSize() && 7232 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/ true) || 7233 shouldPreventUndefRegUpdateMemFold(MF, MI))) 7234 return nullptr; 7235 7236 unsigned NumOps = MI.getDesc().getNumOperands(); 7237 bool isTwoAddr = 7238 NumOps > 1 && MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1; 7239 7240 // FIXME: AsmPrinter doesn't know how to handle 7241 // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding. 7242 if (MI.getOpcode() == X86::ADD32ri && 7243 MI.getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS) 7244 return nullptr; 7245 7246 // GOTTPOFF relocation loads can only be folded into add instructions. 7247 // FIXME: Need to exclude other relocations that only support specific 7248 // instructions. 7249 if (MOs.size() == X86::AddrNumOperands && 7250 MOs[X86::AddrDisp].getTargetFlags() == X86II::MO_GOTTPOFF && 7251 MI.getOpcode() != X86::ADD64rr) 7252 return nullptr; 7253 7254 // Don't fold loads into indirect calls that need a KCFI check as we'll 7255 // have to unfold these in X86TargetLowering::EmitKCFICheck anyway. 7256 if (MI.isCall() && MI.getCFIType()) 7257 return nullptr; 7258 7259 MachineInstr *NewMI = nullptr; 7260 7261 // Attempt to fold any custom cases we have. 7262 if (MachineInstr *CustomMI = foldMemoryOperandCustom( 7263 MF, MI, OpNum, MOs, InsertPt, Size, Alignment)) 7264 return CustomMI; 7265 7266 const X86FoldTableEntry *I = nullptr; 7267 7268 // Folding a memory location into the two-address part of a two-address 7269 // instruction is different than folding it other places. It requires 7270 // replacing the *two* registers with the memory location. 7271 if (isTwoAddr && NumOps >= 2 && OpNum < 2 && MI.getOperand(0).isReg() && 7272 MI.getOperand(1).isReg() && 7273 MI.getOperand(0).getReg() == MI.getOperand(1).getReg()) { 7274 I = lookupTwoAddrFoldTable(MI.getOpcode()); 7275 isTwoAddrFold = true; 7276 } else { 7277 if (OpNum == 0) { 7278 if (MI.getOpcode() == X86::MOV32r0) { 7279 NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, InsertPt, MI); 7280 if (NewMI) 7281 return NewMI; 7282 } 7283 } 7284 7285 I = lookupFoldTable(MI.getOpcode(), OpNum); 7286 } 7287 7288 if (I != nullptr) { 7289 unsigned Opcode = I->DstOp; 7290 bool FoldedLoad = 7291 isTwoAddrFold || (OpNum == 0 && I->Flags & TB_FOLDED_LOAD) || OpNum > 0; 7292 bool FoldedStore = 7293 isTwoAddrFold || (OpNum == 0 && I->Flags & TB_FOLDED_STORE); 7294 if (Alignment < 7295 Align(1ULL << ((I->Flags & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT))) 7296 return nullptr; 7297 bool NarrowToMOV32rm = false; 7298 if (Size) { 7299 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 7300 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 7301 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 7302 // Check if it's safe to fold the load. If the size of the object is 7303 // narrower than the load width, then it's not. 7304 // FIXME: Allow scalar intrinsic instructions like ADDSSrm_Int. 7305 if (FoldedLoad && Size < RCSize) { 7306 // If this is a 64-bit load, but the spill slot is 32, then we can do 7307 // a 32-bit load which is implicitly zero-extended. This likely is 7308 // due to live interval analysis remat'ing a load from stack slot. 7309 if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4) 7310 return nullptr; 7311 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) 7312 return nullptr; 7313 Opcode = X86::MOV32rm; 7314 NarrowToMOV32rm = true; 7315 } 7316 // For stores, make sure the size of the object is equal to the size of 7317 // the store. If the object is larger, the extra bits would be garbage. If 7318 // the object is smaller we might overwrite another object or fault. 7319 if (FoldedStore && Size != RCSize) 7320 return nullptr; 7321 } 7322 7323 if (isTwoAddrFold) 7324 NewMI = FuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, *this); 7325 else 7326 NewMI = FuseInst(MF, Opcode, OpNum, MOs, InsertPt, MI, *this); 7327 7328 if (NarrowToMOV32rm) { 7329 // If this is the special case where we use a MOV32rm to load a 32-bit 7330 // value and zero-extend the top bits. Change the destination register 7331 // to a 32-bit one. 7332 Register DstReg = NewMI->getOperand(0).getReg(); 7333 if (DstReg.isPhysical()) 7334 NewMI->getOperand(0).setReg(RI.getSubReg(DstReg, X86::sub_32bit)); 7335 else 7336 NewMI->getOperand(0).setSubReg(X86::sub_32bit); 7337 } 7338 return NewMI; 7339 } 7340 7341 // If the instruction and target operand are commutable, commute the 7342 // instruction and try again. 7343 if (AllowCommute) { 7344 unsigned CommuteOpIdx1 = OpNum, CommuteOpIdx2 = CommuteAnyOperandIndex; 7345 if (findCommutedOpIndices(MI, CommuteOpIdx1, CommuteOpIdx2)) { 7346 bool HasDef = MI.getDesc().getNumDefs(); 7347 Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register(); 7348 Register Reg1 = MI.getOperand(CommuteOpIdx1).getReg(); 7349 Register Reg2 = MI.getOperand(CommuteOpIdx2).getReg(); 7350 bool Tied1 = 7351 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx1, MCOI::TIED_TO); 7352 bool Tied2 = 7353 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx2, MCOI::TIED_TO); 7354 7355 // If either of the commutable operands are tied to the destination 7356 // then we can not commute + fold. 7357 if ((HasDef && Reg0 == Reg1 && Tied1) || 7358 (HasDef && Reg0 == Reg2 && Tied2)) 7359 return nullptr; 7360 7361 MachineInstr *CommutedMI = 7362 commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); 7363 if (!CommutedMI) { 7364 // Unable to commute. 7365 return nullptr; 7366 } 7367 if (CommutedMI != &MI) { 7368 // New instruction. We can't fold from this. 7369 CommutedMI->eraseFromParent(); 7370 return nullptr; 7371 } 7372 7373 // Attempt to fold with the commuted version of the instruction. 7374 NewMI = foldMemoryOperandImpl(MF, MI, CommuteOpIdx2, MOs, InsertPt, Size, 7375 Alignment, /*AllowCommute=*/false); 7376 if (NewMI) 7377 return NewMI; 7378 7379 // Folding failed again - undo the commute before returning. 7380 MachineInstr *UncommutedMI = 7381 commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); 7382 if (!UncommutedMI) { 7383 // Unable to commute. 7384 return nullptr; 7385 } 7386 if (UncommutedMI != &MI) { 7387 // New instruction. It doesn't need to be kept. 7388 UncommutedMI->eraseFromParent(); 7389 return nullptr; 7390 } 7391 7392 // Return here to prevent duplicate fuse failure report. 7393 return nullptr; 7394 } 7395 } 7396 7397 // No fusion 7398 if (PrintFailedFusing && !MI.isCopy()) 7399 dbgs() << "We failed to fuse operand " << OpNum << " in " << MI; 7400 return nullptr; 7401 } 7402 7403 MachineInstr *X86InstrInfo::foldMemoryOperandImpl( 7404 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 7405 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS, 7406 VirtRegMap *VRM) const { 7407 // Check switch flag 7408 if (NoFusing) 7409 return nullptr; 7410 7411 // Avoid partial and undef register update stalls unless optimizing for size. 7412 if (!MF.getFunction().hasOptSize() && 7413 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/ true) || 7414 shouldPreventUndefRegUpdateMemFold(MF, MI))) 7415 return nullptr; 7416 7417 // Don't fold subreg spills, or reloads that use a high subreg. 7418 for (auto Op : Ops) { 7419 MachineOperand &MO = MI.getOperand(Op); 7420 auto SubReg = MO.getSubReg(); 7421 if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi)) 7422 return nullptr; 7423 } 7424 7425 const MachineFrameInfo &MFI = MF.getFrameInfo(); 7426 unsigned Size = MFI.getObjectSize(FrameIndex); 7427 Align Alignment = MFI.getObjectAlign(FrameIndex); 7428 // If the function stack isn't realigned we don't want to fold instructions 7429 // that need increased alignment. 7430 if (!RI.hasStackRealignment(MF)) 7431 Alignment = 7432 std::min(Alignment, Subtarget.getFrameLowering()->getStackAlign()); 7433 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 7434 unsigned NewOpc = 0; 7435 unsigned RCSize = 0; 7436 switch (MI.getOpcode()) { 7437 default: 7438 return nullptr; 7439 case X86::TEST8rr: 7440 NewOpc = X86::CMP8ri; 7441 RCSize = 1; 7442 break; 7443 case X86::TEST16rr: 7444 NewOpc = X86::CMP16ri; 7445 RCSize = 2; 7446 break; 7447 case X86::TEST32rr: 7448 NewOpc = X86::CMP32ri; 7449 RCSize = 4; 7450 break; 7451 case X86::TEST64rr: 7452 NewOpc = X86::CMP64ri32; 7453 RCSize = 8; 7454 break; 7455 } 7456 // Check if it's safe to fold the load. If the size of the object is 7457 // narrower than the load width, then it's not. 7458 if (Size < RCSize) 7459 return nullptr; 7460 // Change to CMPXXri r, 0 first. 7461 MI.setDesc(get(NewOpc)); 7462 MI.getOperand(1).ChangeToImmediate(0); 7463 } else if (Ops.size() != 1) 7464 return nullptr; 7465 7466 return foldMemoryOperandImpl(MF, MI, Ops[0], 7467 MachineOperand::CreateFI(FrameIndex), InsertPt, 7468 Size, Alignment, /*AllowCommute=*/true); 7469 } 7470 7471 /// Check if \p LoadMI is a partial register load that we can't fold into \p MI 7472 /// because the latter uses contents that wouldn't be defined in the folded 7473 /// version. For instance, this transformation isn't legal: 7474 /// movss (%rdi), %xmm0 7475 /// addps %xmm0, %xmm0 7476 /// -> 7477 /// addps (%rdi), %xmm0 7478 /// 7479 /// But this one is: 7480 /// movss (%rdi), %xmm0 7481 /// addss %xmm0, %xmm0 7482 /// -> 7483 /// addss (%rdi), %xmm0 7484 /// 7485 static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, 7486 const MachineInstr &UserMI, 7487 const MachineFunction &MF) { 7488 unsigned Opc = LoadMI.getOpcode(); 7489 unsigned UserOpc = UserMI.getOpcode(); 7490 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 7491 const TargetRegisterClass *RC = 7492 MF.getRegInfo().getRegClass(LoadMI.getOperand(0).getReg()); 7493 unsigned RegSize = TRI.getRegSizeInBits(*RC); 7494 7495 if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm || Opc == X86::VMOVSSZrm || 7496 Opc == X86::MOVSSrm_alt || Opc == X86::VMOVSSrm_alt || 7497 Opc == X86::VMOVSSZrm_alt) && 7498 RegSize > 32) { 7499 // These instructions only load 32 bits, we can't fold them if the 7500 // destination register is wider than 32 bits (4 bytes), and its user 7501 // instruction isn't scalar (SS). 7502 switch (UserOpc) { 7503 case X86::CVTSS2SDrr_Int: 7504 case X86::VCVTSS2SDrr_Int: 7505 case X86::VCVTSS2SDZrr_Int: 7506 case X86::VCVTSS2SDZrr_Intk: 7507 case X86::VCVTSS2SDZrr_Intkz: 7508 case X86::CVTSS2SIrr_Int: 7509 case X86::CVTSS2SI64rr_Int: 7510 case X86::VCVTSS2SIrr_Int: 7511 case X86::VCVTSS2SI64rr_Int: 7512 case X86::VCVTSS2SIZrr_Int: 7513 case X86::VCVTSS2SI64Zrr_Int: 7514 case X86::CVTTSS2SIrr_Int: 7515 case X86::CVTTSS2SI64rr_Int: 7516 case X86::VCVTTSS2SIrr_Int: 7517 case X86::VCVTTSS2SI64rr_Int: 7518 case X86::VCVTTSS2SIZrr_Int: 7519 case X86::VCVTTSS2SI64Zrr_Int: 7520 case X86::VCVTSS2USIZrr_Int: 7521 case X86::VCVTSS2USI64Zrr_Int: 7522 case X86::VCVTTSS2USIZrr_Int: 7523 case X86::VCVTTSS2USI64Zrr_Int: 7524 case X86::RCPSSr_Int: 7525 case X86::VRCPSSr_Int: 7526 case X86::RSQRTSSr_Int: 7527 case X86::VRSQRTSSr_Int: 7528 case X86::ROUNDSSr_Int: 7529 case X86::VROUNDSSr_Int: 7530 case X86::COMISSrr_Int: 7531 case X86::VCOMISSrr_Int: 7532 case X86::VCOMISSZrr_Int: 7533 case X86::UCOMISSrr_Int: 7534 case X86::VUCOMISSrr_Int: 7535 case X86::VUCOMISSZrr_Int: 7536 case X86::ADDSSrr_Int: 7537 case X86::VADDSSrr_Int: 7538 case X86::VADDSSZrr_Int: 7539 case X86::CMPSSrr_Int: 7540 case X86::VCMPSSrr_Int: 7541 case X86::VCMPSSZrr_Int: 7542 case X86::DIVSSrr_Int: 7543 case X86::VDIVSSrr_Int: 7544 case X86::VDIVSSZrr_Int: 7545 case X86::MAXSSrr_Int: 7546 case X86::VMAXSSrr_Int: 7547 case X86::VMAXSSZrr_Int: 7548 case X86::MINSSrr_Int: 7549 case X86::VMINSSrr_Int: 7550 case X86::VMINSSZrr_Int: 7551 case X86::MULSSrr_Int: 7552 case X86::VMULSSrr_Int: 7553 case X86::VMULSSZrr_Int: 7554 case X86::SQRTSSr_Int: 7555 case X86::VSQRTSSr_Int: 7556 case X86::VSQRTSSZr_Int: 7557 case X86::SUBSSrr_Int: 7558 case X86::VSUBSSrr_Int: 7559 case X86::VSUBSSZrr_Int: 7560 case X86::VADDSSZrr_Intk: 7561 case X86::VADDSSZrr_Intkz: 7562 case X86::VCMPSSZrr_Intk: 7563 case X86::VDIVSSZrr_Intk: 7564 case X86::VDIVSSZrr_Intkz: 7565 case X86::VMAXSSZrr_Intk: 7566 case X86::VMAXSSZrr_Intkz: 7567 case X86::VMINSSZrr_Intk: 7568 case X86::VMINSSZrr_Intkz: 7569 case X86::VMULSSZrr_Intk: 7570 case X86::VMULSSZrr_Intkz: 7571 case X86::VSQRTSSZr_Intk: 7572 case X86::VSQRTSSZr_Intkz: 7573 case X86::VSUBSSZrr_Intk: 7574 case X86::VSUBSSZrr_Intkz: 7575 case X86::VFMADDSS4rr_Int: 7576 case X86::VFNMADDSS4rr_Int: 7577 case X86::VFMSUBSS4rr_Int: 7578 case X86::VFNMSUBSS4rr_Int: 7579 case X86::VFMADD132SSr_Int: 7580 case X86::VFNMADD132SSr_Int: 7581 case X86::VFMADD213SSr_Int: 7582 case X86::VFNMADD213SSr_Int: 7583 case X86::VFMADD231SSr_Int: 7584 case X86::VFNMADD231SSr_Int: 7585 case X86::VFMSUB132SSr_Int: 7586 case X86::VFNMSUB132SSr_Int: 7587 case X86::VFMSUB213SSr_Int: 7588 case X86::VFNMSUB213SSr_Int: 7589 case X86::VFMSUB231SSr_Int: 7590 case X86::VFNMSUB231SSr_Int: 7591 case X86::VFMADD132SSZr_Int: 7592 case X86::VFNMADD132SSZr_Int: 7593 case X86::VFMADD213SSZr_Int: 7594 case X86::VFNMADD213SSZr_Int: 7595 case X86::VFMADD231SSZr_Int: 7596 case X86::VFNMADD231SSZr_Int: 7597 case X86::VFMSUB132SSZr_Int: 7598 case X86::VFNMSUB132SSZr_Int: 7599 case X86::VFMSUB213SSZr_Int: 7600 case X86::VFNMSUB213SSZr_Int: 7601 case X86::VFMSUB231SSZr_Int: 7602 case X86::VFNMSUB231SSZr_Int: 7603 case X86::VFMADD132SSZr_Intk: 7604 case X86::VFNMADD132SSZr_Intk: 7605 case X86::VFMADD213SSZr_Intk: 7606 case X86::VFNMADD213SSZr_Intk: 7607 case X86::VFMADD231SSZr_Intk: 7608 case X86::VFNMADD231SSZr_Intk: 7609 case X86::VFMSUB132SSZr_Intk: 7610 case X86::VFNMSUB132SSZr_Intk: 7611 case X86::VFMSUB213SSZr_Intk: 7612 case X86::VFNMSUB213SSZr_Intk: 7613 case X86::VFMSUB231SSZr_Intk: 7614 case X86::VFNMSUB231SSZr_Intk: 7615 case X86::VFMADD132SSZr_Intkz: 7616 case X86::VFNMADD132SSZr_Intkz: 7617 case X86::VFMADD213SSZr_Intkz: 7618 case X86::VFNMADD213SSZr_Intkz: 7619 case X86::VFMADD231SSZr_Intkz: 7620 case X86::VFNMADD231SSZr_Intkz: 7621 case X86::VFMSUB132SSZr_Intkz: 7622 case X86::VFNMSUB132SSZr_Intkz: 7623 case X86::VFMSUB213SSZr_Intkz: 7624 case X86::VFNMSUB213SSZr_Intkz: 7625 case X86::VFMSUB231SSZr_Intkz: 7626 case X86::VFNMSUB231SSZr_Intkz: 7627 case X86::VFIXUPIMMSSZrri: 7628 case X86::VFIXUPIMMSSZrrik: 7629 case X86::VFIXUPIMMSSZrrikz: 7630 case X86::VFPCLASSSSZrr: 7631 case X86::VFPCLASSSSZrrk: 7632 case X86::VGETEXPSSZr: 7633 case X86::VGETEXPSSZrk: 7634 case X86::VGETEXPSSZrkz: 7635 case X86::VGETMANTSSZrri: 7636 case X86::VGETMANTSSZrrik: 7637 case X86::VGETMANTSSZrrikz: 7638 case X86::VRANGESSZrri: 7639 case X86::VRANGESSZrrik: 7640 case X86::VRANGESSZrrikz: 7641 case X86::VRCP14SSZrr: 7642 case X86::VRCP14SSZrrk: 7643 case X86::VRCP14SSZrrkz: 7644 case X86::VRCP28SSZr: 7645 case X86::VRCP28SSZrk: 7646 case X86::VRCP28SSZrkz: 7647 case X86::VREDUCESSZrri: 7648 case X86::VREDUCESSZrrik: 7649 case X86::VREDUCESSZrrikz: 7650 case X86::VRNDSCALESSZr_Int: 7651 case X86::VRNDSCALESSZr_Intk: 7652 case X86::VRNDSCALESSZr_Intkz: 7653 case X86::VRSQRT14SSZrr: 7654 case X86::VRSQRT14SSZrrk: 7655 case X86::VRSQRT14SSZrrkz: 7656 case X86::VRSQRT28SSZr: 7657 case X86::VRSQRT28SSZrk: 7658 case X86::VRSQRT28SSZrkz: 7659 case X86::VSCALEFSSZrr: 7660 case X86::VSCALEFSSZrrk: 7661 case X86::VSCALEFSSZrrkz: 7662 return false; 7663 default: 7664 return true; 7665 } 7666 } 7667 7668 if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm || Opc == X86::VMOVSDZrm || 7669 Opc == X86::MOVSDrm_alt || Opc == X86::VMOVSDrm_alt || 7670 Opc == X86::VMOVSDZrm_alt) && 7671 RegSize > 64) { 7672 // These instructions only load 64 bits, we can't fold them if the 7673 // destination register is wider than 64 bits (8 bytes), and its user 7674 // instruction isn't scalar (SD). 7675 switch (UserOpc) { 7676 case X86::CVTSD2SSrr_Int: 7677 case X86::VCVTSD2SSrr_Int: 7678 case X86::VCVTSD2SSZrr_Int: 7679 case X86::VCVTSD2SSZrr_Intk: 7680 case X86::VCVTSD2SSZrr_Intkz: 7681 case X86::CVTSD2SIrr_Int: 7682 case X86::CVTSD2SI64rr_Int: 7683 case X86::VCVTSD2SIrr_Int: 7684 case X86::VCVTSD2SI64rr_Int: 7685 case X86::VCVTSD2SIZrr_Int: 7686 case X86::VCVTSD2SI64Zrr_Int: 7687 case X86::CVTTSD2SIrr_Int: 7688 case X86::CVTTSD2SI64rr_Int: 7689 case X86::VCVTTSD2SIrr_Int: 7690 case X86::VCVTTSD2SI64rr_Int: 7691 case X86::VCVTTSD2SIZrr_Int: 7692 case X86::VCVTTSD2SI64Zrr_Int: 7693 case X86::VCVTSD2USIZrr_Int: 7694 case X86::VCVTSD2USI64Zrr_Int: 7695 case X86::VCVTTSD2USIZrr_Int: 7696 case X86::VCVTTSD2USI64Zrr_Int: 7697 case X86::ROUNDSDr_Int: 7698 case X86::VROUNDSDr_Int: 7699 case X86::COMISDrr_Int: 7700 case X86::VCOMISDrr_Int: 7701 case X86::VCOMISDZrr_Int: 7702 case X86::UCOMISDrr_Int: 7703 case X86::VUCOMISDrr_Int: 7704 case X86::VUCOMISDZrr_Int: 7705 case X86::ADDSDrr_Int: 7706 case X86::VADDSDrr_Int: 7707 case X86::VADDSDZrr_Int: 7708 case X86::CMPSDrr_Int: 7709 case X86::VCMPSDrr_Int: 7710 case X86::VCMPSDZrr_Int: 7711 case X86::DIVSDrr_Int: 7712 case X86::VDIVSDrr_Int: 7713 case X86::VDIVSDZrr_Int: 7714 case X86::MAXSDrr_Int: 7715 case X86::VMAXSDrr_Int: 7716 case X86::VMAXSDZrr_Int: 7717 case X86::MINSDrr_Int: 7718 case X86::VMINSDrr_Int: 7719 case X86::VMINSDZrr_Int: 7720 case X86::MULSDrr_Int: 7721 case X86::VMULSDrr_Int: 7722 case X86::VMULSDZrr_Int: 7723 case X86::SQRTSDr_Int: 7724 case X86::VSQRTSDr_Int: 7725 case X86::VSQRTSDZr_Int: 7726 case X86::SUBSDrr_Int: 7727 case X86::VSUBSDrr_Int: 7728 case X86::VSUBSDZrr_Int: 7729 case X86::VADDSDZrr_Intk: 7730 case X86::VADDSDZrr_Intkz: 7731 case X86::VCMPSDZrr_Intk: 7732 case X86::VDIVSDZrr_Intk: 7733 case X86::VDIVSDZrr_Intkz: 7734 case X86::VMAXSDZrr_Intk: 7735 case X86::VMAXSDZrr_Intkz: 7736 case X86::VMINSDZrr_Intk: 7737 case X86::VMINSDZrr_Intkz: 7738 case X86::VMULSDZrr_Intk: 7739 case X86::VMULSDZrr_Intkz: 7740 case X86::VSQRTSDZr_Intk: 7741 case X86::VSQRTSDZr_Intkz: 7742 case X86::VSUBSDZrr_Intk: 7743 case X86::VSUBSDZrr_Intkz: 7744 case X86::VFMADDSD4rr_Int: 7745 case X86::VFNMADDSD4rr_Int: 7746 case X86::VFMSUBSD4rr_Int: 7747 case X86::VFNMSUBSD4rr_Int: 7748 case X86::VFMADD132SDr_Int: 7749 case X86::VFNMADD132SDr_Int: 7750 case X86::VFMADD213SDr_Int: 7751 case X86::VFNMADD213SDr_Int: 7752 case X86::VFMADD231SDr_Int: 7753 case X86::VFNMADD231SDr_Int: 7754 case X86::VFMSUB132SDr_Int: 7755 case X86::VFNMSUB132SDr_Int: 7756 case X86::VFMSUB213SDr_Int: 7757 case X86::VFNMSUB213SDr_Int: 7758 case X86::VFMSUB231SDr_Int: 7759 case X86::VFNMSUB231SDr_Int: 7760 case X86::VFMADD132SDZr_Int: 7761 case X86::VFNMADD132SDZr_Int: 7762 case X86::VFMADD213SDZr_Int: 7763 case X86::VFNMADD213SDZr_Int: 7764 case X86::VFMADD231SDZr_Int: 7765 case X86::VFNMADD231SDZr_Int: 7766 case X86::VFMSUB132SDZr_Int: 7767 case X86::VFNMSUB132SDZr_Int: 7768 case X86::VFMSUB213SDZr_Int: 7769 case X86::VFNMSUB213SDZr_Int: 7770 case X86::VFMSUB231SDZr_Int: 7771 case X86::VFNMSUB231SDZr_Int: 7772 case X86::VFMADD132SDZr_Intk: 7773 case X86::VFNMADD132SDZr_Intk: 7774 case X86::VFMADD213SDZr_Intk: 7775 case X86::VFNMADD213SDZr_Intk: 7776 case X86::VFMADD231SDZr_Intk: 7777 case X86::VFNMADD231SDZr_Intk: 7778 case X86::VFMSUB132SDZr_Intk: 7779 case X86::VFNMSUB132SDZr_Intk: 7780 case X86::VFMSUB213SDZr_Intk: 7781 case X86::VFNMSUB213SDZr_Intk: 7782 case X86::VFMSUB231SDZr_Intk: 7783 case X86::VFNMSUB231SDZr_Intk: 7784 case X86::VFMADD132SDZr_Intkz: 7785 case X86::VFNMADD132SDZr_Intkz: 7786 case X86::VFMADD213SDZr_Intkz: 7787 case X86::VFNMADD213SDZr_Intkz: 7788 case X86::VFMADD231SDZr_Intkz: 7789 case X86::VFNMADD231SDZr_Intkz: 7790 case X86::VFMSUB132SDZr_Intkz: 7791 case X86::VFNMSUB132SDZr_Intkz: 7792 case X86::VFMSUB213SDZr_Intkz: 7793 case X86::VFNMSUB213SDZr_Intkz: 7794 case X86::VFMSUB231SDZr_Intkz: 7795 case X86::VFNMSUB231SDZr_Intkz: 7796 case X86::VFIXUPIMMSDZrri: 7797 case X86::VFIXUPIMMSDZrrik: 7798 case X86::VFIXUPIMMSDZrrikz: 7799 case X86::VFPCLASSSDZrr: 7800 case X86::VFPCLASSSDZrrk: 7801 case X86::VGETEXPSDZr: 7802 case X86::VGETEXPSDZrk: 7803 case X86::VGETEXPSDZrkz: 7804 case X86::VGETMANTSDZrri: 7805 case X86::VGETMANTSDZrrik: 7806 case X86::VGETMANTSDZrrikz: 7807 case X86::VRANGESDZrri: 7808 case X86::VRANGESDZrrik: 7809 case X86::VRANGESDZrrikz: 7810 case X86::VRCP14SDZrr: 7811 case X86::VRCP14SDZrrk: 7812 case X86::VRCP14SDZrrkz: 7813 case X86::VRCP28SDZr: 7814 case X86::VRCP28SDZrk: 7815 case X86::VRCP28SDZrkz: 7816 case X86::VREDUCESDZrri: 7817 case X86::VREDUCESDZrrik: 7818 case X86::VREDUCESDZrrikz: 7819 case X86::VRNDSCALESDZr_Int: 7820 case X86::VRNDSCALESDZr_Intk: 7821 case X86::VRNDSCALESDZr_Intkz: 7822 case X86::VRSQRT14SDZrr: 7823 case X86::VRSQRT14SDZrrk: 7824 case X86::VRSQRT14SDZrrkz: 7825 case X86::VRSQRT28SDZr: 7826 case X86::VRSQRT28SDZrk: 7827 case X86::VRSQRT28SDZrkz: 7828 case X86::VSCALEFSDZrr: 7829 case X86::VSCALEFSDZrrk: 7830 case X86::VSCALEFSDZrrkz: 7831 return false; 7832 default: 7833 return true; 7834 } 7835 } 7836 7837 if ((Opc == X86::VMOVSHZrm || Opc == X86::VMOVSHZrm_alt) && RegSize > 16) { 7838 // These instructions only load 16 bits, we can't fold them if the 7839 // destination register is wider than 16 bits (2 bytes), and its user 7840 // instruction isn't scalar (SH). 7841 switch (UserOpc) { 7842 case X86::VADDSHZrr_Int: 7843 case X86::VCMPSHZrr_Int: 7844 case X86::VDIVSHZrr_Int: 7845 case X86::VMAXSHZrr_Int: 7846 case X86::VMINSHZrr_Int: 7847 case X86::VMULSHZrr_Int: 7848 case X86::VSUBSHZrr_Int: 7849 case X86::VADDSHZrr_Intk: 7850 case X86::VADDSHZrr_Intkz: 7851 case X86::VCMPSHZrr_Intk: 7852 case X86::VDIVSHZrr_Intk: 7853 case X86::VDIVSHZrr_Intkz: 7854 case X86::VMAXSHZrr_Intk: 7855 case X86::VMAXSHZrr_Intkz: 7856 case X86::VMINSHZrr_Intk: 7857 case X86::VMINSHZrr_Intkz: 7858 case X86::VMULSHZrr_Intk: 7859 case X86::VMULSHZrr_Intkz: 7860 case X86::VSUBSHZrr_Intk: 7861 case X86::VSUBSHZrr_Intkz: 7862 case X86::VFMADD132SHZr_Int: 7863 case X86::VFNMADD132SHZr_Int: 7864 case X86::VFMADD213SHZr_Int: 7865 case X86::VFNMADD213SHZr_Int: 7866 case X86::VFMADD231SHZr_Int: 7867 case X86::VFNMADD231SHZr_Int: 7868 case X86::VFMSUB132SHZr_Int: 7869 case X86::VFNMSUB132SHZr_Int: 7870 case X86::VFMSUB213SHZr_Int: 7871 case X86::VFNMSUB213SHZr_Int: 7872 case X86::VFMSUB231SHZr_Int: 7873 case X86::VFNMSUB231SHZr_Int: 7874 case X86::VFMADD132SHZr_Intk: 7875 case X86::VFNMADD132SHZr_Intk: 7876 case X86::VFMADD213SHZr_Intk: 7877 case X86::VFNMADD213SHZr_Intk: 7878 case X86::VFMADD231SHZr_Intk: 7879 case X86::VFNMADD231SHZr_Intk: 7880 case X86::VFMSUB132SHZr_Intk: 7881 case X86::VFNMSUB132SHZr_Intk: 7882 case X86::VFMSUB213SHZr_Intk: 7883 case X86::VFNMSUB213SHZr_Intk: 7884 case X86::VFMSUB231SHZr_Intk: 7885 case X86::VFNMSUB231SHZr_Intk: 7886 case X86::VFMADD132SHZr_Intkz: 7887 case X86::VFNMADD132SHZr_Intkz: 7888 case X86::VFMADD213SHZr_Intkz: 7889 case X86::VFNMADD213SHZr_Intkz: 7890 case X86::VFMADD231SHZr_Intkz: 7891 case X86::VFNMADD231SHZr_Intkz: 7892 case X86::VFMSUB132SHZr_Intkz: 7893 case X86::VFNMSUB132SHZr_Intkz: 7894 case X86::VFMSUB213SHZr_Intkz: 7895 case X86::VFNMSUB213SHZr_Intkz: 7896 case X86::VFMSUB231SHZr_Intkz: 7897 case X86::VFNMSUB231SHZr_Intkz: 7898 return false; 7899 default: 7900 return true; 7901 } 7902 } 7903 7904 return false; 7905 } 7906 7907 MachineInstr *X86InstrInfo::foldMemoryOperandImpl( 7908 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 7909 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, 7910 LiveIntervals *LIS) const { 7911 7912 // TODO: Support the case where LoadMI loads a wide register, but MI 7913 // only uses a subreg. 7914 for (auto Op : Ops) { 7915 if (MI.getOperand(Op).getSubReg()) 7916 return nullptr; 7917 } 7918 7919 // If loading from a FrameIndex, fold directly from the FrameIndex. 7920 unsigned NumOps = LoadMI.getDesc().getNumOperands(); 7921 int FrameIndex; 7922 if (isLoadFromStackSlot(LoadMI, FrameIndex)) { 7923 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) 7924 return nullptr; 7925 return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex, LIS); 7926 } 7927 7928 // Check switch flag 7929 if (NoFusing) 7930 return nullptr; 7931 7932 // Avoid partial and undef register update stalls unless optimizing for size. 7933 if (!MF.getFunction().hasOptSize() && 7934 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/ true) || 7935 shouldPreventUndefRegUpdateMemFold(MF, MI))) 7936 return nullptr; 7937 7938 // Determine the alignment of the load. 7939 Align Alignment; 7940 if (LoadMI.hasOneMemOperand()) 7941 Alignment = (*LoadMI.memoperands_begin())->getAlign(); 7942 else 7943 switch (LoadMI.getOpcode()) { 7944 case X86::AVX512_512_SET0: 7945 case X86::AVX512_512_SETALLONES: 7946 Alignment = Align(64); 7947 break; 7948 case X86::AVX2_SETALLONES: 7949 case X86::AVX1_SETALLONES: 7950 case X86::AVX_SET0: 7951 case X86::AVX512_256_SET0: 7952 Alignment = Align(32); 7953 break; 7954 case X86::V_SET0: 7955 case X86::V_SETALLONES: 7956 case X86::AVX512_128_SET0: 7957 case X86::FsFLD0F128: 7958 case X86::AVX512_FsFLD0F128: 7959 Alignment = Align(16); 7960 break; 7961 case X86::MMX_SET0: 7962 case X86::FsFLD0SD: 7963 case X86::AVX512_FsFLD0SD: 7964 Alignment = Align(8); 7965 break; 7966 case X86::FsFLD0SS: 7967 case X86::AVX512_FsFLD0SS: 7968 Alignment = Align(4); 7969 break; 7970 case X86::FsFLD0SH: 7971 case X86::AVX512_FsFLD0SH: 7972 Alignment = Align(2); 7973 break; 7974 default: 7975 return nullptr; 7976 } 7977 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 7978 unsigned NewOpc = 0; 7979 switch (MI.getOpcode()) { 7980 default: 7981 return nullptr; 7982 case X86::TEST8rr: 7983 NewOpc = X86::CMP8ri; 7984 break; 7985 case X86::TEST16rr: 7986 NewOpc = X86::CMP16ri; 7987 break; 7988 case X86::TEST32rr: 7989 NewOpc = X86::CMP32ri; 7990 break; 7991 case X86::TEST64rr: 7992 NewOpc = X86::CMP64ri32; 7993 break; 7994 } 7995 // Change to CMPXXri r, 0 first. 7996 MI.setDesc(get(NewOpc)); 7997 MI.getOperand(1).ChangeToImmediate(0); 7998 } else if (Ops.size() != 1) 7999 return nullptr; 8000 8001 // Make sure the subregisters match. 8002 // Otherwise we risk changing the size of the load. 8003 if (LoadMI.getOperand(0).getSubReg() != MI.getOperand(Ops[0]).getSubReg()) 8004 return nullptr; 8005 8006 SmallVector<MachineOperand, X86::AddrNumOperands> MOs; 8007 switch (LoadMI.getOpcode()) { 8008 case X86::MMX_SET0: 8009 case X86::V_SET0: 8010 case X86::V_SETALLONES: 8011 case X86::AVX2_SETALLONES: 8012 case X86::AVX1_SETALLONES: 8013 case X86::AVX_SET0: 8014 case X86::AVX512_128_SET0: 8015 case X86::AVX512_256_SET0: 8016 case X86::AVX512_512_SET0: 8017 case X86::AVX512_512_SETALLONES: 8018 case X86::FsFLD0SH: 8019 case X86::AVX512_FsFLD0SH: 8020 case X86::FsFLD0SD: 8021 case X86::AVX512_FsFLD0SD: 8022 case X86::FsFLD0SS: 8023 case X86::AVX512_FsFLD0SS: 8024 case X86::FsFLD0F128: 8025 case X86::AVX512_FsFLD0F128: { 8026 // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure. 8027 // Create a constant-pool entry and operands to load from it. 8028 8029 // Large code model can't fold loads this way. 8030 if (MF.getTarget().getCodeModel() == CodeModel::Large) 8031 return nullptr; 8032 8033 // x86-32 PIC requires a PIC base register for constant pools. 8034 unsigned PICBase = 0; 8035 // Since we're using Small or Kernel code model, we can always use 8036 // RIP-relative addressing for a smaller encoding. 8037 if (Subtarget.is64Bit()) { 8038 PICBase = X86::RIP; 8039 } else if (MF.getTarget().isPositionIndependent()) { 8040 // FIXME: PICBase = getGlobalBaseReg(&MF); 8041 // This doesn't work for several reasons. 8042 // 1. GlobalBaseReg may have been spilled. 8043 // 2. It may not be live at MI. 8044 return nullptr; 8045 } 8046 8047 // Create a constant-pool entry. 8048 MachineConstantPool &MCP = *MF.getConstantPool(); 8049 Type *Ty; 8050 unsigned Opc = LoadMI.getOpcode(); 8051 if (Opc == X86::FsFLD0SS || Opc == X86::AVX512_FsFLD0SS) 8052 Ty = Type::getFloatTy(MF.getFunction().getContext()); 8053 else if (Opc == X86::FsFLD0SD || Opc == X86::AVX512_FsFLD0SD) 8054 Ty = Type::getDoubleTy(MF.getFunction().getContext()); 8055 else if (Opc == X86::FsFLD0F128 || Opc == X86::AVX512_FsFLD0F128) 8056 Ty = Type::getFP128Ty(MF.getFunction().getContext()); 8057 else if (Opc == X86::FsFLD0SH || Opc == X86::AVX512_FsFLD0SH) 8058 Ty = Type::getHalfTy(MF.getFunction().getContext()); 8059 else if (Opc == X86::AVX512_512_SET0 || Opc == X86::AVX512_512_SETALLONES) 8060 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 8061 16); 8062 else if (Opc == X86::AVX2_SETALLONES || Opc == X86::AVX_SET0 || 8063 Opc == X86::AVX512_256_SET0 || Opc == X86::AVX1_SETALLONES) 8064 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 8065 8); 8066 else if (Opc == X86::MMX_SET0) 8067 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 8068 2); 8069 else 8070 Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 8071 4); 8072 8073 bool IsAllOnes = 8074 (Opc == X86::V_SETALLONES || Opc == X86::AVX2_SETALLONES || 8075 Opc == X86::AVX512_512_SETALLONES || Opc == X86::AVX1_SETALLONES); 8076 const Constant *C = 8077 IsAllOnes ? Constant::getAllOnesValue(Ty) : Constant::getNullValue(Ty); 8078 unsigned CPI = MCP.getConstantPoolIndex(C, Alignment); 8079 8080 // Create operands to load from the constant pool entry. 8081 MOs.push_back(MachineOperand::CreateReg(PICBase, false)); 8082 MOs.push_back(MachineOperand::CreateImm(1)); 8083 MOs.push_back(MachineOperand::CreateReg(0, false)); 8084 MOs.push_back(MachineOperand::CreateCPI(CPI, 0)); 8085 MOs.push_back(MachineOperand::CreateReg(0, false)); 8086 break; 8087 } 8088 default: { 8089 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) 8090 return nullptr; 8091 8092 // Folding a normal load. Just copy the load's address operands. 8093 MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, 8094 LoadMI.operands_begin() + NumOps); 8095 break; 8096 } 8097 } 8098 return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, InsertPt, 8099 /*Size=*/0, Alignment, /*AllowCommute=*/true); 8100 } 8101 8102 static SmallVector<MachineMemOperand *, 2> 8103 extractLoadMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { 8104 SmallVector<MachineMemOperand *, 2> LoadMMOs; 8105 8106 for (MachineMemOperand *MMO : MMOs) { 8107 if (!MMO->isLoad()) 8108 continue; 8109 8110 if (!MMO->isStore()) { 8111 // Reuse the MMO. 8112 LoadMMOs.push_back(MMO); 8113 } else { 8114 // Clone the MMO and unset the store flag. 8115 LoadMMOs.push_back(MF.getMachineMemOperand( 8116 MMO, MMO->getFlags() & ~MachineMemOperand::MOStore)); 8117 } 8118 } 8119 8120 return LoadMMOs; 8121 } 8122 8123 static SmallVector<MachineMemOperand *, 2> 8124 extractStoreMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { 8125 SmallVector<MachineMemOperand *, 2> StoreMMOs; 8126 8127 for (MachineMemOperand *MMO : MMOs) { 8128 if (!MMO->isStore()) 8129 continue; 8130 8131 if (!MMO->isLoad()) { 8132 // Reuse the MMO. 8133 StoreMMOs.push_back(MMO); 8134 } else { 8135 // Clone the MMO and unset the load flag. 8136 StoreMMOs.push_back(MF.getMachineMemOperand( 8137 MMO, MMO->getFlags() & ~MachineMemOperand::MOLoad)); 8138 } 8139 } 8140 8141 return StoreMMOs; 8142 } 8143 8144 static unsigned getBroadcastOpcode(const X86FoldTableEntry *I, 8145 const TargetRegisterClass *RC, 8146 const X86Subtarget &STI) { 8147 assert(STI.hasAVX512() && "Expected at least AVX512!"); 8148 unsigned SpillSize = STI.getRegisterInfo()->getSpillSize(*RC); 8149 assert((SpillSize == 64 || STI.hasVLX()) && 8150 "Can't broadcast less than 64 bytes without AVX512VL!"); 8151 8152 switch (I->Flags & TB_BCAST_MASK) { 8153 default: 8154 llvm_unreachable("Unexpected broadcast type!"); 8155 case TB_BCAST_D: 8156 switch (SpillSize) { 8157 default: 8158 llvm_unreachable("Unknown spill size"); 8159 case 16: 8160 return X86::VPBROADCASTDZ128rm; 8161 case 32: 8162 return X86::VPBROADCASTDZ256rm; 8163 case 64: 8164 return X86::VPBROADCASTDZrm; 8165 } 8166 break; 8167 case TB_BCAST_Q: 8168 switch (SpillSize) { 8169 default: 8170 llvm_unreachable("Unknown spill size"); 8171 case 16: 8172 return X86::VPBROADCASTQZ128rm; 8173 case 32: 8174 return X86::VPBROADCASTQZ256rm; 8175 case 64: 8176 return X86::VPBROADCASTQZrm; 8177 } 8178 break; 8179 case TB_BCAST_SS: 8180 switch (SpillSize) { 8181 default: 8182 llvm_unreachable("Unknown spill size"); 8183 case 16: 8184 return X86::VBROADCASTSSZ128rm; 8185 case 32: 8186 return X86::VBROADCASTSSZ256rm; 8187 case 64: 8188 return X86::VBROADCASTSSZrm; 8189 } 8190 break; 8191 case TB_BCAST_SD: 8192 switch (SpillSize) { 8193 default: 8194 llvm_unreachable("Unknown spill size"); 8195 case 16: 8196 return X86::VMOVDDUPZ128rm; 8197 case 32: 8198 return X86::VBROADCASTSDZ256rm; 8199 case 64: 8200 return X86::VBROADCASTSDZrm; 8201 } 8202 break; 8203 } 8204 } 8205 8206 bool X86InstrInfo::unfoldMemoryOperand( 8207 MachineFunction &MF, MachineInstr &MI, unsigned Reg, bool UnfoldLoad, 8208 bool UnfoldStore, SmallVectorImpl<MachineInstr *> &NewMIs) const { 8209 const X86FoldTableEntry *I = lookupUnfoldTable(MI.getOpcode()); 8210 if (I == nullptr) 8211 return false; 8212 unsigned Opc = I->DstOp; 8213 unsigned Index = I->Flags & TB_INDEX_MASK; 8214 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 8215 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 8216 bool FoldedBCast = I->Flags & TB_FOLDED_BCAST; 8217 if (UnfoldLoad && !FoldedLoad) 8218 return false; 8219 UnfoldLoad &= FoldedLoad; 8220 if (UnfoldStore && !FoldedStore) 8221 return false; 8222 UnfoldStore &= FoldedStore; 8223 8224 const MCInstrDesc &MCID = get(Opc); 8225 8226 const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); 8227 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 8228 // TODO: Check if 32-byte or greater accesses are slow too? 8229 if (!MI.hasOneMemOperand() && RC == &X86::VR128RegClass && 8230 Subtarget.isUnalignedMem16Slow()) 8231 // Without memoperands, loadRegFromAddr and storeRegToStackSlot will 8232 // conservatively assume the address is unaligned. That's bad for 8233 // performance. 8234 return false; 8235 SmallVector<MachineOperand, X86::AddrNumOperands> AddrOps; 8236 SmallVector<MachineOperand, 2> BeforeOps; 8237 SmallVector<MachineOperand, 2> AfterOps; 8238 SmallVector<MachineOperand, 4> ImpOps; 8239 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 8240 MachineOperand &Op = MI.getOperand(i); 8241 if (i >= Index && i < Index + X86::AddrNumOperands) 8242 AddrOps.push_back(Op); 8243 else if (Op.isReg() && Op.isImplicit()) 8244 ImpOps.push_back(Op); 8245 else if (i < Index) 8246 BeforeOps.push_back(Op); 8247 else if (i > Index) 8248 AfterOps.push_back(Op); 8249 } 8250 8251 // Emit the load or broadcast instruction. 8252 if (UnfoldLoad) { 8253 auto MMOs = extractLoadMMOs(MI.memoperands(), MF); 8254 8255 unsigned Opc; 8256 if (FoldedBCast) { 8257 Opc = getBroadcastOpcode(I, RC, Subtarget); 8258 } else { 8259 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 8260 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 8261 Opc = getLoadRegOpcode(Reg, RC, isAligned, Subtarget); 8262 } 8263 8264 DebugLoc DL; 8265 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), Reg); 8266 for (const MachineOperand &AddrOp : AddrOps) 8267 MIB.add(AddrOp); 8268 MIB.setMemRefs(MMOs); 8269 NewMIs.push_back(MIB); 8270 8271 if (UnfoldStore) { 8272 // Address operands cannot be marked isKill. 8273 for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) { 8274 MachineOperand &MO = NewMIs[0]->getOperand(i); 8275 if (MO.isReg()) 8276 MO.setIsKill(false); 8277 } 8278 } 8279 } 8280 8281 // Emit the data processing instruction. 8282 MachineInstr *DataMI = MF.CreateMachineInstr(MCID, MI.getDebugLoc(), true); 8283 MachineInstrBuilder MIB(MF, DataMI); 8284 8285 if (FoldedStore) 8286 MIB.addReg(Reg, RegState::Define); 8287 for (MachineOperand &BeforeOp : BeforeOps) 8288 MIB.add(BeforeOp); 8289 if (FoldedLoad) 8290 MIB.addReg(Reg); 8291 for (MachineOperand &AfterOp : AfterOps) 8292 MIB.add(AfterOp); 8293 for (MachineOperand &ImpOp : ImpOps) { 8294 MIB.addReg(ImpOp.getReg(), getDefRegState(ImpOp.isDef()) | 8295 RegState::Implicit | 8296 getKillRegState(ImpOp.isKill()) | 8297 getDeadRegState(ImpOp.isDead()) | 8298 getUndefRegState(ImpOp.isUndef())); 8299 } 8300 // Change CMP32ri r, 0 back to TEST32rr r, r, etc. 8301 switch (DataMI->getOpcode()) { 8302 default: 8303 break; 8304 case X86::CMP64ri32: 8305 case X86::CMP32ri: 8306 case X86::CMP16ri: 8307 case X86::CMP8ri: { 8308 MachineOperand &MO0 = DataMI->getOperand(0); 8309 MachineOperand &MO1 = DataMI->getOperand(1); 8310 if (MO1.isImm() && MO1.getImm() == 0) { 8311 unsigned NewOpc; 8312 switch (DataMI->getOpcode()) { 8313 default: 8314 llvm_unreachable("Unreachable!"); 8315 case X86::CMP64ri32: 8316 NewOpc = X86::TEST64rr; 8317 break; 8318 case X86::CMP32ri: 8319 NewOpc = X86::TEST32rr; 8320 break; 8321 case X86::CMP16ri: 8322 NewOpc = X86::TEST16rr; 8323 break; 8324 case X86::CMP8ri: 8325 NewOpc = X86::TEST8rr; 8326 break; 8327 } 8328 DataMI->setDesc(get(NewOpc)); 8329 MO1.ChangeToRegister(MO0.getReg(), false); 8330 } 8331 } 8332 } 8333 NewMIs.push_back(DataMI); 8334 8335 // Emit the store instruction. 8336 if (UnfoldStore) { 8337 const TargetRegisterClass *DstRC = getRegClass(MCID, 0, &RI, MF); 8338 auto MMOs = extractStoreMMOs(MI.memoperands(), MF); 8339 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*DstRC), 16); 8340 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 8341 unsigned Opc = getStoreRegOpcode(Reg, DstRC, isAligned, Subtarget); 8342 DebugLoc DL; 8343 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc)); 8344 for (const MachineOperand &AddrOp : AddrOps) 8345 MIB.add(AddrOp); 8346 MIB.addReg(Reg, RegState::Kill); 8347 MIB.setMemRefs(MMOs); 8348 NewMIs.push_back(MIB); 8349 } 8350 8351 return true; 8352 } 8353 8354 bool X86InstrInfo::unfoldMemoryOperand( 8355 SelectionDAG &DAG, SDNode *N, SmallVectorImpl<SDNode *> &NewNodes) const { 8356 if (!N->isMachineOpcode()) 8357 return false; 8358 8359 const X86FoldTableEntry *I = lookupUnfoldTable(N->getMachineOpcode()); 8360 if (I == nullptr) 8361 return false; 8362 unsigned Opc = I->DstOp; 8363 unsigned Index = I->Flags & TB_INDEX_MASK; 8364 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 8365 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 8366 bool FoldedBCast = I->Flags & TB_FOLDED_BCAST; 8367 const MCInstrDesc &MCID = get(Opc); 8368 MachineFunction &MF = DAG.getMachineFunction(); 8369 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 8370 const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); 8371 unsigned NumDefs = MCID.NumDefs; 8372 std::vector<SDValue> AddrOps; 8373 std::vector<SDValue> BeforeOps; 8374 std::vector<SDValue> AfterOps; 8375 SDLoc dl(N); 8376 unsigned NumOps = N->getNumOperands(); 8377 for (unsigned i = 0; i != NumOps - 1; ++i) { 8378 SDValue Op = N->getOperand(i); 8379 if (i >= Index - NumDefs && i < Index - NumDefs + X86::AddrNumOperands) 8380 AddrOps.push_back(Op); 8381 else if (i < Index - NumDefs) 8382 BeforeOps.push_back(Op); 8383 else if (i > Index - NumDefs) 8384 AfterOps.push_back(Op); 8385 } 8386 SDValue Chain = N->getOperand(NumOps - 1); 8387 AddrOps.push_back(Chain); 8388 8389 // Emit the load instruction. 8390 SDNode *Load = nullptr; 8391 if (FoldedLoad) { 8392 EVT VT = *TRI.legalclasstypes_begin(*RC); 8393 auto MMOs = extractLoadMMOs(cast<MachineSDNode>(N)->memoperands(), MF); 8394 if (MMOs.empty() && RC == &X86::VR128RegClass && 8395 Subtarget.isUnalignedMem16Slow()) 8396 // Do not introduce a slow unaligned load. 8397 return false; 8398 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte 8399 // memory access is slow above. 8400 8401 unsigned Opc; 8402 if (FoldedBCast) { 8403 Opc = getBroadcastOpcode(I, RC, Subtarget); 8404 } else { 8405 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 8406 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 8407 Opc = getLoadRegOpcode(0, RC, isAligned, Subtarget); 8408 } 8409 8410 Load = DAG.getMachineNode(Opc, dl, VT, MVT::Other, AddrOps); 8411 NewNodes.push_back(Load); 8412 8413 // Preserve memory reference information. 8414 DAG.setNodeMemRefs(cast<MachineSDNode>(Load), MMOs); 8415 } 8416 8417 // Emit the data processing instruction. 8418 std::vector<EVT> VTs; 8419 const TargetRegisterClass *DstRC = nullptr; 8420 if (MCID.getNumDefs() > 0) { 8421 DstRC = getRegClass(MCID, 0, &RI, MF); 8422 VTs.push_back(*TRI.legalclasstypes_begin(*DstRC)); 8423 } 8424 for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { 8425 EVT VT = N->getValueType(i); 8426 if (VT != MVT::Other && i >= (unsigned)MCID.getNumDefs()) 8427 VTs.push_back(VT); 8428 } 8429 if (Load) 8430 BeforeOps.push_back(SDValue(Load, 0)); 8431 llvm::append_range(BeforeOps, AfterOps); 8432 // Change CMP32ri r, 0 back to TEST32rr r, r, etc. 8433 switch (Opc) { 8434 default: 8435 break; 8436 case X86::CMP64ri32: 8437 case X86::CMP32ri: 8438 case X86::CMP16ri: 8439 case X86::CMP8ri: 8440 if (isNullConstant(BeforeOps[1])) { 8441 switch (Opc) { 8442 default: 8443 llvm_unreachable("Unreachable!"); 8444 case X86::CMP64ri32: 8445 Opc = X86::TEST64rr; 8446 break; 8447 case X86::CMP32ri: 8448 Opc = X86::TEST32rr; 8449 break; 8450 case X86::CMP16ri: 8451 Opc = X86::TEST16rr; 8452 break; 8453 case X86::CMP8ri: 8454 Opc = X86::TEST8rr; 8455 break; 8456 } 8457 BeforeOps[1] = BeforeOps[0]; 8458 } 8459 } 8460 SDNode *NewNode = DAG.getMachineNode(Opc, dl, VTs, BeforeOps); 8461 NewNodes.push_back(NewNode); 8462 8463 // Emit the store instruction. 8464 if (FoldedStore) { 8465 AddrOps.pop_back(); 8466 AddrOps.push_back(SDValue(NewNode, 0)); 8467 AddrOps.push_back(Chain); 8468 auto MMOs = extractStoreMMOs(cast<MachineSDNode>(N)->memoperands(), MF); 8469 if (MMOs.empty() && RC == &X86::VR128RegClass && 8470 Subtarget.isUnalignedMem16Slow()) 8471 // Do not introduce a slow unaligned store. 8472 return false; 8473 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte 8474 // memory access is slow above. 8475 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 8476 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment; 8477 SDNode *Store = 8478 DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget), 8479 dl, MVT::Other, AddrOps); 8480 NewNodes.push_back(Store); 8481 8482 // Preserve memory reference information. 8483 DAG.setNodeMemRefs(cast<MachineSDNode>(Store), MMOs); 8484 } 8485 8486 return true; 8487 } 8488 8489 unsigned 8490 X86InstrInfo::getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, 8491 bool UnfoldStore, 8492 unsigned *LoadRegIndex) const { 8493 const X86FoldTableEntry *I = lookupUnfoldTable(Opc); 8494 if (I == nullptr) 8495 return 0; 8496 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 8497 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 8498 if (UnfoldLoad && !FoldedLoad) 8499 return 0; 8500 if (UnfoldStore && !FoldedStore) 8501 return 0; 8502 if (LoadRegIndex) 8503 *LoadRegIndex = I->Flags & TB_INDEX_MASK; 8504 return I->DstOp; 8505 } 8506 8507 bool X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, 8508 int64_t &Offset1, 8509 int64_t &Offset2) const { 8510 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode()) 8511 return false; 8512 8513 auto IsLoadOpcode = [&](unsigned Opcode) { 8514 switch (Opcode) { 8515 default: 8516 return false; 8517 case X86::MOV8rm: 8518 case X86::MOV16rm: 8519 case X86::MOV32rm: 8520 case X86::MOV64rm: 8521 case X86::LD_Fp32m: 8522 case X86::LD_Fp64m: 8523 case X86::LD_Fp80m: 8524 case X86::MOVSSrm: 8525 case X86::MOVSSrm_alt: 8526 case X86::MOVSDrm: 8527 case X86::MOVSDrm_alt: 8528 case X86::MMX_MOVD64rm: 8529 case X86::MMX_MOVQ64rm: 8530 case X86::MOVAPSrm: 8531 case X86::MOVUPSrm: 8532 case X86::MOVAPDrm: 8533 case X86::MOVUPDrm: 8534 case X86::MOVDQArm: 8535 case X86::MOVDQUrm: 8536 // AVX load instructions 8537 case X86::VMOVSSrm: 8538 case X86::VMOVSSrm_alt: 8539 case X86::VMOVSDrm: 8540 case X86::VMOVSDrm_alt: 8541 case X86::VMOVAPSrm: 8542 case X86::VMOVUPSrm: 8543 case X86::VMOVAPDrm: 8544 case X86::VMOVUPDrm: 8545 case X86::VMOVDQArm: 8546 case X86::VMOVDQUrm: 8547 case X86::VMOVAPSYrm: 8548 case X86::VMOVUPSYrm: 8549 case X86::VMOVAPDYrm: 8550 case X86::VMOVUPDYrm: 8551 case X86::VMOVDQAYrm: 8552 case X86::VMOVDQUYrm: 8553 // AVX512 load instructions 8554 case X86::VMOVSSZrm: 8555 case X86::VMOVSSZrm_alt: 8556 case X86::VMOVSDZrm: 8557 case X86::VMOVSDZrm_alt: 8558 case X86::VMOVAPSZ128rm: 8559 case X86::VMOVUPSZ128rm: 8560 case X86::VMOVAPSZ128rm_NOVLX: 8561 case X86::VMOVUPSZ128rm_NOVLX: 8562 case X86::VMOVAPDZ128rm: 8563 case X86::VMOVUPDZ128rm: 8564 case X86::VMOVDQU8Z128rm: 8565 case X86::VMOVDQU16Z128rm: 8566 case X86::VMOVDQA32Z128rm: 8567 case X86::VMOVDQU32Z128rm: 8568 case X86::VMOVDQA64Z128rm: 8569 case X86::VMOVDQU64Z128rm: 8570 case X86::VMOVAPSZ256rm: 8571 case X86::VMOVUPSZ256rm: 8572 case X86::VMOVAPSZ256rm_NOVLX: 8573 case X86::VMOVUPSZ256rm_NOVLX: 8574 case X86::VMOVAPDZ256rm: 8575 case X86::VMOVUPDZ256rm: 8576 case X86::VMOVDQU8Z256rm: 8577 case X86::VMOVDQU16Z256rm: 8578 case X86::VMOVDQA32Z256rm: 8579 case X86::VMOVDQU32Z256rm: 8580 case X86::VMOVDQA64Z256rm: 8581 case X86::VMOVDQU64Z256rm: 8582 case X86::VMOVAPSZrm: 8583 case X86::VMOVUPSZrm: 8584 case X86::VMOVAPDZrm: 8585 case X86::VMOVUPDZrm: 8586 case X86::VMOVDQU8Zrm: 8587 case X86::VMOVDQU16Zrm: 8588 case X86::VMOVDQA32Zrm: 8589 case X86::VMOVDQU32Zrm: 8590 case X86::VMOVDQA64Zrm: 8591 case X86::VMOVDQU64Zrm: 8592 case X86::KMOVBkm: 8593 case X86::KMOVBkm_EVEX: 8594 case X86::KMOVWkm: 8595 case X86::KMOVWkm_EVEX: 8596 case X86::KMOVDkm: 8597 case X86::KMOVDkm_EVEX: 8598 case X86::KMOVQkm: 8599 case X86::KMOVQkm_EVEX: 8600 return true; 8601 } 8602 }; 8603 8604 if (!IsLoadOpcode(Load1->getMachineOpcode()) || 8605 !IsLoadOpcode(Load2->getMachineOpcode())) 8606 return false; 8607 8608 // Lambda to check if both the loads have the same value for an operand index. 8609 auto HasSameOp = [&](int I) { 8610 return Load1->getOperand(I) == Load2->getOperand(I); 8611 }; 8612 8613 // All operands except the displacement should match. 8614 if (!HasSameOp(X86::AddrBaseReg) || !HasSameOp(X86::AddrScaleAmt) || 8615 !HasSameOp(X86::AddrIndexReg) || !HasSameOp(X86::AddrSegmentReg)) 8616 return false; 8617 8618 // Chain Operand must be the same. 8619 if (!HasSameOp(5)) 8620 return false; 8621 8622 // Now let's examine if the displacements are constants. 8623 auto Disp1 = dyn_cast<ConstantSDNode>(Load1->getOperand(X86::AddrDisp)); 8624 auto Disp2 = dyn_cast<ConstantSDNode>(Load2->getOperand(X86::AddrDisp)); 8625 if (!Disp1 || !Disp2) 8626 return false; 8627 8628 Offset1 = Disp1->getSExtValue(); 8629 Offset2 = Disp2->getSExtValue(); 8630 return true; 8631 } 8632 8633 bool X86InstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, 8634 int64_t Offset1, int64_t Offset2, 8635 unsigned NumLoads) const { 8636 assert(Offset2 > Offset1); 8637 if ((Offset2 - Offset1) / 8 > 64) 8638 return false; 8639 8640 unsigned Opc1 = Load1->getMachineOpcode(); 8641 unsigned Opc2 = Load2->getMachineOpcode(); 8642 if (Opc1 != Opc2) 8643 return false; // FIXME: overly conservative? 8644 8645 switch (Opc1) { 8646 default: 8647 break; 8648 case X86::LD_Fp32m: 8649 case X86::LD_Fp64m: 8650 case X86::LD_Fp80m: 8651 case X86::MMX_MOVD64rm: 8652 case X86::MMX_MOVQ64rm: 8653 return false; 8654 } 8655 8656 EVT VT = Load1->getValueType(0); 8657 switch (VT.getSimpleVT().SimpleTy) { 8658 default: 8659 // XMM registers. In 64-bit mode we can be a bit more aggressive since we 8660 // have 16 of them to play with. 8661 if (Subtarget.is64Bit()) { 8662 if (NumLoads >= 3) 8663 return false; 8664 } else if (NumLoads) { 8665 return false; 8666 } 8667 break; 8668 case MVT::i8: 8669 case MVT::i16: 8670 case MVT::i32: 8671 case MVT::i64: 8672 case MVT::f32: 8673 case MVT::f64: 8674 if (NumLoads) 8675 return false; 8676 break; 8677 } 8678 8679 return true; 8680 } 8681 8682 bool X86InstrInfo::isSchedulingBoundary(const MachineInstr &MI, 8683 const MachineBasicBlock *MBB, 8684 const MachineFunction &MF) const { 8685 8686 // ENDBR instructions should not be scheduled around. 8687 unsigned Opcode = MI.getOpcode(); 8688 if (Opcode == X86::ENDBR64 || Opcode == X86::ENDBR32 || 8689 Opcode == X86::PLDTILECFGV) 8690 return true; 8691 8692 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF); 8693 } 8694 8695 bool X86InstrInfo::reverseBranchCondition( 8696 SmallVectorImpl<MachineOperand> &Cond) const { 8697 assert(Cond.size() == 1 && "Invalid X86 branch condition!"); 8698 X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm()); 8699 Cond[0].setImm(GetOppositeBranchCondition(CC)); 8700 return false; 8701 } 8702 8703 bool X86InstrInfo::isSafeToMoveRegClassDefs( 8704 const TargetRegisterClass *RC) const { 8705 // FIXME: Return false for x87 stack register classes for now. We can't 8706 // allow any loads of these registers before FpGet_ST0_80. 8707 return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass || 8708 RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass || 8709 RC == &X86::RFP80RegClass); 8710 } 8711 8712 /// Return a virtual register initialized with the 8713 /// the global base register value. Output instructions required to 8714 /// initialize the register in the function entry block, if necessary. 8715 /// 8716 /// TODO: Eliminate this and move the code to X86MachineFunctionInfo. 8717 /// 8718 unsigned X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const { 8719 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); 8720 Register GlobalBaseReg = X86FI->getGlobalBaseReg(); 8721 if (GlobalBaseReg != 0) 8722 return GlobalBaseReg; 8723 8724 // Create the register. The code to initialize it is inserted 8725 // later, by the CGBR pass (below). 8726 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 8727 GlobalBaseReg = RegInfo.createVirtualRegister( 8728 Subtarget.is64Bit() ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass); 8729 X86FI->setGlobalBaseReg(GlobalBaseReg); 8730 return GlobalBaseReg; 8731 } 8732 8733 // FIXME: Some shuffle and unpack instructions have equivalents in different 8734 // domains, but they require a bit more work than just switching opcodes. 8735 8736 static const uint16_t *lookup(unsigned opcode, unsigned domain, 8737 ArrayRef<uint16_t[3]> Table) { 8738 for (const uint16_t(&Row)[3] : Table) 8739 if (Row[domain - 1] == opcode) 8740 return Row; 8741 return nullptr; 8742 } 8743 8744 static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain, 8745 ArrayRef<uint16_t[4]> Table) { 8746 // If this is the integer domain make sure to check both integer columns. 8747 for (const uint16_t(&Row)[4] : Table) 8748 if (Row[domain - 1] == opcode || (domain == 3 && Row[3] == opcode)) 8749 return Row; 8750 return nullptr; 8751 } 8752 8753 // Helper to attempt to widen/narrow blend masks. 8754 static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth, 8755 unsigned NewWidth, unsigned *pNewMask = nullptr) { 8756 assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && 8757 "Illegal blend mask scale"); 8758 unsigned NewMask = 0; 8759 8760 if ((OldWidth % NewWidth) == 0) { 8761 unsigned Scale = OldWidth / NewWidth; 8762 unsigned SubMask = (1u << Scale) - 1; 8763 for (unsigned i = 0; i != NewWidth; ++i) { 8764 unsigned Sub = (OldMask >> (i * Scale)) & SubMask; 8765 if (Sub == SubMask) 8766 NewMask |= (1u << i); 8767 else if (Sub != 0x0) 8768 return false; 8769 } 8770 } else { 8771 unsigned Scale = NewWidth / OldWidth; 8772 unsigned SubMask = (1u << Scale) - 1; 8773 for (unsigned i = 0; i != OldWidth; ++i) { 8774 if (OldMask & (1 << i)) { 8775 NewMask |= (SubMask << (i * Scale)); 8776 } 8777 } 8778 } 8779 8780 if (pNewMask) 8781 *pNewMask = NewMask; 8782 return true; 8783 } 8784 8785 uint16_t X86InstrInfo::getExecutionDomainCustom(const MachineInstr &MI) const { 8786 unsigned Opcode = MI.getOpcode(); 8787 unsigned NumOperands = MI.getDesc().getNumOperands(); 8788 8789 auto GetBlendDomains = [&](unsigned ImmWidth, bool Is256) { 8790 uint16_t validDomains = 0; 8791 if (MI.getOperand(NumOperands - 1).isImm()) { 8792 unsigned Imm = MI.getOperand(NumOperands - 1).getImm(); 8793 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4)) 8794 validDomains |= 0x2; // PackedSingle 8795 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2)) 8796 validDomains |= 0x4; // PackedDouble 8797 if (!Is256 || Subtarget.hasAVX2()) 8798 validDomains |= 0x8; // PackedInt 8799 } 8800 return validDomains; 8801 }; 8802 8803 switch (Opcode) { 8804 case X86::BLENDPDrmi: 8805 case X86::BLENDPDrri: 8806 case X86::VBLENDPDrmi: 8807 case X86::VBLENDPDrri: 8808 return GetBlendDomains(2, false); 8809 case X86::VBLENDPDYrmi: 8810 case X86::VBLENDPDYrri: 8811 return GetBlendDomains(4, true); 8812 case X86::BLENDPSrmi: 8813 case X86::BLENDPSrri: 8814 case X86::VBLENDPSrmi: 8815 case X86::VBLENDPSrri: 8816 case X86::VPBLENDDrmi: 8817 case X86::VPBLENDDrri: 8818 return GetBlendDomains(4, false); 8819 case X86::VBLENDPSYrmi: 8820 case X86::VBLENDPSYrri: 8821 case X86::VPBLENDDYrmi: 8822 case X86::VPBLENDDYrri: 8823 return GetBlendDomains(8, true); 8824 case X86::PBLENDWrmi: 8825 case X86::PBLENDWrri: 8826 case X86::VPBLENDWrmi: 8827 case X86::VPBLENDWrri: 8828 // Treat VPBLENDWY as a 128-bit vector as it repeats the lo/hi masks. 8829 case X86::VPBLENDWYrmi: 8830 case X86::VPBLENDWYrri: 8831 return GetBlendDomains(8, false); 8832 case X86::VPANDDZ128rr: 8833 case X86::VPANDDZ128rm: 8834 case X86::VPANDDZ256rr: 8835 case X86::VPANDDZ256rm: 8836 case X86::VPANDQZ128rr: 8837 case X86::VPANDQZ128rm: 8838 case X86::VPANDQZ256rr: 8839 case X86::VPANDQZ256rm: 8840 case X86::VPANDNDZ128rr: 8841 case X86::VPANDNDZ128rm: 8842 case X86::VPANDNDZ256rr: 8843 case X86::VPANDNDZ256rm: 8844 case X86::VPANDNQZ128rr: 8845 case X86::VPANDNQZ128rm: 8846 case X86::VPANDNQZ256rr: 8847 case X86::VPANDNQZ256rm: 8848 case X86::VPORDZ128rr: 8849 case X86::VPORDZ128rm: 8850 case X86::VPORDZ256rr: 8851 case X86::VPORDZ256rm: 8852 case X86::VPORQZ128rr: 8853 case X86::VPORQZ128rm: 8854 case X86::VPORQZ256rr: 8855 case X86::VPORQZ256rm: 8856 case X86::VPXORDZ128rr: 8857 case X86::VPXORDZ128rm: 8858 case X86::VPXORDZ256rr: 8859 case X86::VPXORDZ256rm: 8860 case X86::VPXORQZ128rr: 8861 case X86::VPXORQZ128rm: 8862 case X86::VPXORQZ256rr: 8863 case X86::VPXORQZ256rm: 8864 // If we don't have DQI see if we can still switch from an EVEX integer 8865 // instruction to a VEX floating point instruction. 8866 if (Subtarget.hasDQI()) 8867 return 0; 8868 8869 if (RI.getEncodingValue(MI.getOperand(0).getReg()) >= 16) 8870 return 0; 8871 if (RI.getEncodingValue(MI.getOperand(1).getReg()) >= 16) 8872 return 0; 8873 // Register forms will have 3 operands. Memory form will have more. 8874 if (NumOperands == 3 && 8875 RI.getEncodingValue(MI.getOperand(2).getReg()) >= 16) 8876 return 0; 8877 8878 // All domains are valid. 8879 return 0xe; 8880 case X86::MOVHLPSrr: 8881 // We can swap domains when both inputs are the same register. 8882 // FIXME: This doesn't catch all the cases we would like. If the input 8883 // register isn't KILLed by the instruction, the two address instruction 8884 // pass puts a COPY on one input. The other input uses the original 8885 // register. This prevents the same physical register from being used by 8886 // both inputs. 8887 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && 8888 MI.getOperand(0).getSubReg() == 0 && 8889 MI.getOperand(1).getSubReg() == 0 && MI.getOperand(2).getSubReg() == 0) 8890 return 0x6; 8891 return 0; 8892 case X86::SHUFPDrri: 8893 return 0x6; 8894 } 8895 return 0; 8896 } 8897 8898 #include "X86ReplaceableInstrs.def" 8899 8900 bool X86InstrInfo::setExecutionDomainCustom(MachineInstr &MI, 8901 unsigned Domain) const { 8902 assert(Domain > 0 && Domain < 4 && "Invalid execution domain"); 8903 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 8904 assert(dom && "Not an SSE instruction"); 8905 8906 unsigned Opcode = MI.getOpcode(); 8907 unsigned NumOperands = MI.getDesc().getNumOperands(); 8908 8909 auto SetBlendDomain = [&](unsigned ImmWidth, bool Is256) { 8910 if (MI.getOperand(NumOperands - 1).isImm()) { 8911 unsigned Imm = MI.getOperand(NumOperands - 1).getImm() & 255; 8912 Imm = (ImmWidth == 16 ? ((Imm << 8) | Imm) : Imm); 8913 unsigned NewImm = Imm; 8914 8915 const uint16_t *table = lookup(Opcode, dom, ReplaceableBlendInstrs); 8916 if (!table) 8917 table = lookup(Opcode, dom, ReplaceableBlendAVX2Instrs); 8918 8919 if (Domain == 1) { // PackedSingle 8920 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); 8921 } else if (Domain == 2) { // PackedDouble 8922 AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2, &NewImm); 8923 } else if (Domain == 3) { // PackedInt 8924 if (Subtarget.hasAVX2()) { 8925 // If we are already VPBLENDW use that, else use VPBLENDD. 8926 if ((ImmWidth / (Is256 ? 2 : 1)) != 8) { 8927 table = lookup(Opcode, dom, ReplaceableBlendAVX2Instrs); 8928 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); 8929 } 8930 } else { 8931 assert(!Is256 && "128-bit vector expected"); 8932 AdjustBlendMask(Imm, ImmWidth, 8, &NewImm); 8933 } 8934 } 8935 8936 assert(table && table[Domain - 1] && "Unknown domain op"); 8937 MI.setDesc(get(table[Domain - 1])); 8938 MI.getOperand(NumOperands - 1).setImm(NewImm & 255); 8939 } 8940 return true; 8941 }; 8942 8943 switch (Opcode) { 8944 case X86::BLENDPDrmi: 8945 case X86::BLENDPDrri: 8946 case X86::VBLENDPDrmi: 8947 case X86::VBLENDPDrri: 8948 return SetBlendDomain(2, false); 8949 case X86::VBLENDPDYrmi: 8950 case X86::VBLENDPDYrri: 8951 return SetBlendDomain(4, true); 8952 case X86::BLENDPSrmi: 8953 case X86::BLENDPSrri: 8954 case X86::VBLENDPSrmi: 8955 case X86::VBLENDPSrri: 8956 case X86::VPBLENDDrmi: 8957 case X86::VPBLENDDrri: 8958 return SetBlendDomain(4, false); 8959 case X86::VBLENDPSYrmi: 8960 case X86::VBLENDPSYrri: 8961 case X86::VPBLENDDYrmi: 8962 case X86::VPBLENDDYrri: 8963 return SetBlendDomain(8, true); 8964 case X86::PBLENDWrmi: 8965 case X86::PBLENDWrri: 8966 case X86::VPBLENDWrmi: 8967 case X86::VPBLENDWrri: 8968 return SetBlendDomain(8, false); 8969 case X86::VPBLENDWYrmi: 8970 case X86::VPBLENDWYrri: 8971 return SetBlendDomain(16, true); 8972 case X86::VPANDDZ128rr: 8973 case X86::VPANDDZ128rm: 8974 case X86::VPANDDZ256rr: 8975 case X86::VPANDDZ256rm: 8976 case X86::VPANDQZ128rr: 8977 case X86::VPANDQZ128rm: 8978 case X86::VPANDQZ256rr: 8979 case X86::VPANDQZ256rm: 8980 case X86::VPANDNDZ128rr: 8981 case X86::VPANDNDZ128rm: 8982 case X86::VPANDNDZ256rr: 8983 case X86::VPANDNDZ256rm: 8984 case X86::VPANDNQZ128rr: 8985 case X86::VPANDNQZ128rm: 8986 case X86::VPANDNQZ256rr: 8987 case X86::VPANDNQZ256rm: 8988 case X86::VPORDZ128rr: 8989 case X86::VPORDZ128rm: 8990 case X86::VPORDZ256rr: 8991 case X86::VPORDZ256rm: 8992 case X86::VPORQZ128rr: 8993 case X86::VPORQZ128rm: 8994 case X86::VPORQZ256rr: 8995 case X86::VPORQZ256rm: 8996 case X86::VPXORDZ128rr: 8997 case X86::VPXORDZ128rm: 8998 case X86::VPXORDZ256rr: 8999 case X86::VPXORDZ256rm: 9000 case X86::VPXORQZ128rr: 9001 case X86::VPXORQZ128rm: 9002 case X86::VPXORQZ256rr: 9003 case X86::VPXORQZ256rm: { 9004 // Without DQI, convert EVEX instructions to VEX instructions. 9005 if (Subtarget.hasDQI()) 9006 return false; 9007 9008 const uint16_t *table = 9009 lookupAVX512(MI.getOpcode(), dom, ReplaceableCustomAVX512LogicInstrs); 9010 assert(table && "Instruction not found in table?"); 9011 // Don't change integer Q instructions to D instructions and 9012 // use D intructions if we started with a PS instruction. 9013 if (Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 9014 Domain = 4; 9015 MI.setDesc(get(table[Domain - 1])); 9016 return true; 9017 } 9018 case X86::UNPCKHPDrr: 9019 case X86::MOVHLPSrr: 9020 // We just need to commute the instruction which will switch the domains. 9021 if (Domain != dom && Domain != 3 && 9022 MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && 9023 MI.getOperand(0).getSubReg() == 0 && 9024 MI.getOperand(1).getSubReg() == 0 && 9025 MI.getOperand(2).getSubReg() == 0) { 9026 commuteInstruction(MI, false); 9027 return true; 9028 } 9029 // We must always return true for MOVHLPSrr. 9030 if (Opcode == X86::MOVHLPSrr) 9031 return true; 9032 break; 9033 case X86::SHUFPDrri: { 9034 if (Domain == 1) { 9035 unsigned Imm = MI.getOperand(3).getImm(); 9036 unsigned NewImm = 0x44; 9037 if (Imm & 1) 9038 NewImm |= 0x0a; 9039 if (Imm & 2) 9040 NewImm |= 0xa0; 9041 MI.getOperand(3).setImm(NewImm); 9042 MI.setDesc(get(X86::SHUFPSrri)); 9043 } 9044 return true; 9045 } 9046 } 9047 return false; 9048 } 9049 9050 std::pair<uint16_t, uint16_t> 9051 X86InstrInfo::getExecutionDomain(const MachineInstr &MI) const { 9052 uint16_t domain = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 9053 unsigned opcode = MI.getOpcode(); 9054 uint16_t validDomains = 0; 9055 if (domain) { 9056 // Attempt to match for custom instructions. 9057 validDomains = getExecutionDomainCustom(MI); 9058 if (validDomains) 9059 return std::make_pair(domain, validDomains); 9060 9061 if (lookup(opcode, domain, ReplaceableInstrs)) { 9062 validDomains = 0xe; 9063 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2)) { 9064 validDomains = Subtarget.hasAVX2() ? 0xe : 0x6; 9065 } else if (lookup(opcode, domain, ReplaceableInstrsFP)) { 9066 validDomains = 0x6; 9067 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2InsertExtract)) { 9068 // Insert/extract instructions should only effect domain if AVX2 9069 // is enabled. 9070 if (!Subtarget.hasAVX2()) 9071 return std::make_pair(0, 0); 9072 validDomains = 0xe; 9073 } else if (lookupAVX512(opcode, domain, ReplaceableInstrsAVX512)) { 9074 validDomains = 0xe; 9075 } else if (Subtarget.hasDQI() && 9076 lookupAVX512(opcode, domain, ReplaceableInstrsAVX512DQ)) { 9077 validDomains = 0xe; 9078 } else if (Subtarget.hasDQI()) { 9079 if (const uint16_t *table = 9080 lookupAVX512(opcode, domain, ReplaceableInstrsAVX512DQMasked)) { 9081 if (domain == 1 || (domain == 3 && table[3] == opcode)) 9082 validDomains = 0xa; 9083 else 9084 validDomains = 0xc; 9085 } 9086 } 9087 } 9088 return std::make_pair(domain, validDomains); 9089 } 9090 9091 void X86InstrInfo::setExecutionDomain(MachineInstr &MI, unsigned Domain) const { 9092 assert(Domain > 0 && Domain < 4 && "Invalid execution domain"); 9093 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 9094 assert(dom && "Not an SSE instruction"); 9095 9096 // Attempt to match for custom instructions. 9097 if (setExecutionDomainCustom(MI, Domain)) 9098 return; 9099 9100 const uint16_t *table = lookup(MI.getOpcode(), dom, ReplaceableInstrs); 9101 if (!table) { // try the other table 9102 assert((Subtarget.hasAVX2() || Domain < 3) && 9103 "256-bit vector operations only available in AVX2"); 9104 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2); 9105 } 9106 if (!table) { // try the FP table 9107 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsFP); 9108 assert((!table || Domain < 3) && 9109 "Can only select PackedSingle or PackedDouble"); 9110 } 9111 if (!table) { // try the other table 9112 assert(Subtarget.hasAVX2() && 9113 "256-bit insert/extract only available in AVX2"); 9114 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2InsertExtract); 9115 } 9116 if (!table) { // try the AVX512 table 9117 assert(Subtarget.hasAVX512() && "Requires AVX-512"); 9118 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512); 9119 // Don't change integer Q instructions to D instructions. 9120 if (table && Domain == 3 && table[3] == MI.getOpcode()) 9121 Domain = 4; 9122 } 9123 if (!table) { // try the AVX512DQ table 9124 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ"); 9125 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQ); 9126 // Don't change integer Q instructions to D instructions and 9127 // use D instructions if we started with a PS instruction. 9128 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 9129 Domain = 4; 9130 } 9131 if (!table) { // try the AVX512DQMasked table 9132 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ"); 9133 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQMasked); 9134 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 9135 Domain = 4; 9136 } 9137 assert(table && "Cannot change domain"); 9138 MI.setDesc(get(table[Domain - 1])); 9139 } 9140 9141 void X86InstrInfo::insertNoop(MachineBasicBlock &MBB, 9142 MachineBasicBlock::iterator MI) const { 9143 DebugLoc DL; 9144 BuildMI(MBB, MI, DL, get(X86::NOOP)); 9145 } 9146 9147 /// Return the noop instruction to use for a noop. 9148 MCInst X86InstrInfo::getNop() const { 9149 MCInst Nop; 9150 Nop.setOpcode(X86::NOOP); 9151 return Nop; 9152 } 9153 9154 bool X86InstrInfo::isHighLatencyDef(int opc) const { 9155 switch (opc) { 9156 default: 9157 return false; 9158 case X86::DIVPDrm: 9159 case X86::DIVPDrr: 9160 case X86::DIVPSrm: 9161 case X86::DIVPSrr: 9162 case X86::DIVSDrm: 9163 case X86::DIVSDrm_Int: 9164 case X86::DIVSDrr: 9165 case X86::DIVSDrr_Int: 9166 case X86::DIVSSrm: 9167 case X86::DIVSSrm_Int: 9168 case X86::DIVSSrr: 9169 case X86::DIVSSrr_Int: 9170 case X86::SQRTPDm: 9171 case X86::SQRTPDr: 9172 case X86::SQRTPSm: 9173 case X86::SQRTPSr: 9174 case X86::SQRTSDm: 9175 case X86::SQRTSDm_Int: 9176 case X86::SQRTSDr: 9177 case X86::SQRTSDr_Int: 9178 case X86::SQRTSSm: 9179 case X86::SQRTSSm_Int: 9180 case X86::SQRTSSr: 9181 case X86::SQRTSSr_Int: 9182 // AVX instructions with high latency 9183 case X86::VDIVPDrm: 9184 case X86::VDIVPDrr: 9185 case X86::VDIVPDYrm: 9186 case X86::VDIVPDYrr: 9187 case X86::VDIVPSrm: 9188 case X86::VDIVPSrr: 9189 case X86::VDIVPSYrm: 9190 case X86::VDIVPSYrr: 9191 case X86::VDIVSDrm: 9192 case X86::VDIVSDrm_Int: 9193 case X86::VDIVSDrr: 9194 case X86::VDIVSDrr_Int: 9195 case X86::VDIVSSrm: 9196 case X86::VDIVSSrm_Int: 9197 case X86::VDIVSSrr: 9198 case X86::VDIVSSrr_Int: 9199 case X86::VSQRTPDm: 9200 case X86::VSQRTPDr: 9201 case X86::VSQRTPDYm: 9202 case X86::VSQRTPDYr: 9203 case X86::VSQRTPSm: 9204 case X86::VSQRTPSr: 9205 case X86::VSQRTPSYm: 9206 case X86::VSQRTPSYr: 9207 case X86::VSQRTSDm: 9208 case X86::VSQRTSDm_Int: 9209 case X86::VSQRTSDr: 9210 case X86::VSQRTSDr_Int: 9211 case X86::VSQRTSSm: 9212 case X86::VSQRTSSm_Int: 9213 case X86::VSQRTSSr: 9214 case X86::VSQRTSSr_Int: 9215 // AVX512 instructions with high latency 9216 case X86::VDIVPDZ128rm: 9217 case X86::VDIVPDZ128rmb: 9218 case X86::VDIVPDZ128rmbk: 9219 case X86::VDIVPDZ128rmbkz: 9220 case X86::VDIVPDZ128rmk: 9221 case X86::VDIVPDZ128rmkz: 9222 case X86::VDIVPDZ128rr: 9223 case X86::VDIVPDZ128rrk: 9224 case X86::VDIVPDZ128rrkz: 9225 case X86::VDIVPDZ256rm: 9226 case X86::VDIVPDZ256rmb: 9227 case X86::VDIVPDZ256rmbk: 9228 case X86::VDIVPDZ256rmbkz: 9229 case X86::VDIVPDZ256rmk: 9230 case X86::VDIVPDZ256rmkz: 9231 case X86::VDIVPDZ256rr: 9232 case X86::VDIVPDZ256rrk: 9233 case X86::VDIVPDZ256rrkz: 9234 case X86::VDIVPDZrrb: 9235 case X86::VDIVPDZrrbk: 9236 case X86::VDIVPDZrrbkz: 9237 case X86::VDIVPDZrm: 9238 case X86::VDIVPDZrmb: 9239 case X86::VDIVPDZrmbk: 9240 case X86::VDIVPDZrmbkz: 9241 case X86::VDIVPDZrmk: 9242 case X86::VDIVPDZrmkz: 9243 case X86::VDIVPDZrr: 9244 case X86::VDIVPDZrrk: 9245 case X86::VDIVPDZrrkz: 9246 case X86::VDIVPSZ128rm: 9247 case X86::VDIVPSZ128rmb: 9248 case X86::VDIVPSZ128rmbk: 9249 case X86::VDIVPSZ128rmbkz: 9250 case X86::VDIVPSZ128rmk: 9251 case X86::VDIVPSZ128rmkz: 9252 case X86::VDIVPSZ128rr: 9253 case X86::VDIVPSZ128rrk: 9254 case X86::VDIVPSZ128rrkz: 9255 case X86::VDIVPSZ256rm: 9256 case X86::VDIVPSZ256rmb: 9257 case X86::VDIVPSZ256rmbk: 9258 case X86::VDIVPSZ256rmbkz: 9259 case X86::VDIVPSZ256rmk: 9260 case X86::VDIVPSZ256rmkz: 9261 case X86::VDIVPSZ256rr: 9262 case X86::VDIVPSZ256rrk: 9263 case X86::VDIVPSZ256rrkz: 9264 case X86::VDIVPSZrrb: 9265 case X86::VDIVPSZrrbk: 9266 case X86::VDIVPSZrrbkz: 9267 case X86::VDIVPSZrm: 9268 case X86::VDIVPSZrmb: 9269 case X86::VDIVPSZrmbk: 9270 case X86::VDIVPSZrmbkz: 9271 case X86::VDIVPSZrmk: 9272 case X86::VDIVPSZrmkz: 9273 case X86::VDIVPSZrr: 9274 case X86::VDIVPSZrrk: 9275 case X86::VDIVPSZrrkz: 9276 case X86::VDIVSDZrm: 9277 case X86::VDIVSDZrr: 9278 case X86::VDIVSDZrm_Int: 9279 case X86::VDIVSDZrm_Intk: 9280 case X86::VDIVSDZrm_Intkz: 9281 case X86::VDIVSDZrr_Int: 9282 case X86::VDIVSDZrr_Intk: 9283 case X86::VDIVSDZrr_Intkz: 9284 case X86::VDIVSDZrrb_Int: 9285 case X86::VDIVSDZrrb_Intk: 9286 case X86::VDIVSDZrrb_Intkz: 9287 case X86::VDIVSSZrm: 9288 case X86::VDIVSSZrr: 9289 case X86::VDIVSSZrm_Int: 9290 case X86::VDIVSSZrm_Intk: 9291 case X86::VDIVSSZrm_Intkz: 9292 case X86::VDIVSSZrr_Int: 9293 case X86::VDIVSSZrr_Intk: 9294 case X86::VDIVSSZrr_Intkz: 9295 case X86::VDIVSSZrrb_Int: 9296 case X86::VDIVSSZrrb_Intk: 9297 case X86::VDIVSSZrrb_Intkz: 9298 case X86::VSQRTPDZ128m: 9299 case X86::VSQRTPDZ128mb: 9300 case X86::VSQRTPDZ128mbk: 9301 case X86::VSQRTPDZ128mbkz: 9302 case X86::VSQRTPDZ128mk: 9303 case X86::VSQRTPDZ128mkz: 9304 case X86::VSQRTPDZ128r: 9305 case X86::VSQRTPDZ128rk: 9306 case X86::VSQRTPDZ128rkz: 9307 case X86::VSQRTPDZ256m: 9308 case X86::VSQRTPDZ256mb: 9309 case X86::VSQRTPDZ256mbk: 9310 case X86::VSQRTPDZ256mbkz: 9311 case X86::VSQRTPDZ256mk: 9312 case X86::VSQRTPDZ256mkz: 9313 case X86::VSQRTPDZ256r: 9314 case X86::VSQRTPDZ256rk: 9315 case X86::VSQRTPDZ256rkz: 9316 case X86::VSQRTPDZm: 9317 case X86::VSQRTPDZmb: 9318 case X86::VSQRTPDZmbk: 9319 case X86::VSQRTPDZmbkz: 9320 case X86::VSQRTPDZmk: 9321 case X86::VSQRTPDZmkz: 9322 case X86::VSQRTPDZr: 9323 case X86::VSQRTPDZrb: 9324 case X86::VSQRTPDZrbk: 9325 case X86::VSQRTPDZrbkz: 9326 case X86::VSQRTPDZrk: 9327 case X86::VSQRTPDZrkz: 9328 case X86::VSQRTPSZ128m: 9329 case X86::VSQRTPSZ128mb: 9330 case X86::VSQRTPSZ128mbk: 9331 case X86::VSQRTPSZ128mbkz: 9332 case X86::VSQRTPSZ128mk: 9333 case X86::VSQRTPSZ128mkz: 9334 case X86::VSQRTPSZ128r: 9335 case X86::VSQRTPSZ128rk: 9336 case X86::VSQRTPSZ128rkz: 9337 case X86::VSQRTPSZ256m: 9338 case X86::VSQRTPSZ256mb: 9339 case X86::VSQRTPSZ256mbk: 9340 case X86::VSQRTPSZ256mbkz: 9341 case X86::VSQRTPSZ256mk: 9342 case X86::VSQRTPSZ256mkz: 9343 case X86::VSQRTPSZ256r: 9344 case X86::VSQRTPSZ256rk: 9345 case X86::VSQRTPSZ256rkz: 9346 case X86::VSQRTPSZm: 9347 case X86::VSQRTPSZmb: 9348 case X86::VSQRTPSZmbk: 9349 case X86::VSQRTPSZmbkz: 9350 case X86::VSQRTPSZmk: 9351 case X86::VSQRTPSZmkz: 9352 case X86::VSQRTPSZr: 9353 case X86::VSQRTPSZrb: 9354 case X86::VSQRTPSZrbk: 9355 case X86::VSQRTPSZrbkz: 9356 case X86::VSQRTPSZrk: 9357 case X86::VSQRTPSZrkz: 9358 case X86::VSQRTSDZm: 9359 case X86::VSQRTSDZm_Int: 9360 case X86::VSQRTSDZm_Intk: 9361 case X86::VSQRTSDZm_Intkz: 9362 case X86::VSQRTSDZr: 9363 case X86::VSQRTSDZr_Int: 9364 case X86::VSQRTSDZr_Intk: 9365 case X86::VSQRTSDZr_Intkz: 9366 case X86::VSQRTSDZrb_Int: 9367 case X86::VSQRTSDZrb_Intk: 9368 case X86::VSQRTSDZrb_Intkz: 9369 case X86::VSQRTSSZm: 9370 case X86::VSQRTSSZm_Int: 9371 case X86::VSQRTSSZm_Intk: 9372 case X86::VSQRTSSZm_Intkz: 9373 case X86::VSQRTSSZr: 9374 case X86::VSQRTSSZr_Int: 9375 case X86::VSQRTSSZr_Intk: 9376 case X86::VSQRTSSZr_Intkz: 9377 case X86::VSQRTSSZrb_Int: 9378 case X86::VSQRTSSZrb_Intk: 9379 case X86::VSQRTSSZrb_Intkz: 9380 9381 case X86::VGATHERDPDYrm: 9382 case X86::VGATHERDPDZ128rm: 9383 case X86::VGATHERDPDZ256rm: 9384 case X86::VGATHERDPDZrm: 9385 case X86::VGATHERDPDrm: 9386 case X86::VGATHERDPSYrm: 9387 case X86::VGATHERDPSZ128rm: 9388 case X86::VGATHERDPSZ256rm: 9389 case X86::VGATHERDPSZrm: 9390 case X86::VGATHERDPSrm: 9391 case X86::VGATHERPF0DPDm: 9392 case X86::VGATHERPF0DPSm: 9393 case X86::VGATHERPF0QPDm: 9394 case X86::VGATHERPF0QPSm: 9395 case X86::VGATHERPF1DPDm: 9396 case X86::VGATHERPF1DPSm: 9397 case X86::VGATHERPF1QPDm: 9398 case X86::VGATHERPF1QPSm: 9399 case X86::VGATHERQPDYrm: 9400 case X86::VGATHERQPDZ128rm: 9401 case X86::VGATHERQPDZ256rm: 9402 case X86::VGATHERQPDZrm: 9403 case X86::VGATHERQPDrm: 9404 case X86::VGATHERQPSYrm: 9405 case X86::VGATHERQPSZ128rm: 9406 case X86::VGATHERQPSZ256rm: 9407 case X86::VGATHERQPSZrm: 9408 case X86::VGATHERQPSrm: 9409 case X86::VPGATHERDDYrm: 9410 case X86::VPGATHERDDZ128rm: 9411 case X86::VPGATHERDDZ256rm: 9412 case X86::VPGATHERDDZrm: 9413 case X86::VPGATHERDDrm: 9414 case X86::VPGATHERDQYrm: 9415 case X86::VPGATHERDQZ128rm: 9416 case X86::VPGATHERDQZ256rm: 9417 case X86::VPGATHERDQZrm: 9418 case X86::VPGATHERDQrm: 9419 case X86::VPGATHERQDYrm: 9420 case X86::VPGATHERQDZ128rm: 9421 case X86::VPGATHERQDZ256rm: 9422 case X86::VPGATHERQDZrm: 9423 case X86::VPGATHERQDrm: 9424 case X86::VPGATHERQQYrm: 9425 case X86::VPGATHERQQZ128rm: 9426 case X86::VPGATHERQQZ256rm: 9427 case X86::VPGATHERQQZrm: 9428 case X86::VPGATHERQQrm: 9429 case X86::VSCATTERDPDZ128mr: 9430 case X86::VSCATTERDPDZ256mr: 9431 case X86::VSCATTERDPDZmr: 9432 case X86::VSCATTERDPSZ128mr: 9433 case X86::VSCATTERDPSZ256mr: 9434 case X86::VSCATTERDPSZmr: 9435 case X86::VSCATTERPF0DPDm: 9436 case X86::VSCATTERPF0DPSm: 9437 case X86::VSCATTERPF0QPDm: 9438 case X86::VSCATTERPF0QPSm: 9439 case X86::VSCATTERPF1DPDm: 9440 case X86::VSCATTERPF1DPSm: 9441 case X86::VSCATTERPF1QPDm: 9442 case X86::VSCATTERPF1QPSm: 9443 case X86::VSCATTERQPDZ128mr: 9444 case X86::VSCATTERQPDZ256mr: 9445 case X86::VSCATTERQPDZmr: 9446 case X86::VSCATTERQPSZ128mr: 9447 case X86::VSCATTERQPSZ256mr: 9448 case X86::VSCATTERQPSZmr: 9449 case X86::VPSCATTERDDZ128mr: 9450 case X86::VPSCATTERDDZ256mr: 9451 case X86::VPSCATTERDDZmr: 9452 case X86::VPSCATTERDQZ128mr: 9453 case X86::VPSCATTERDQZ256mr: 9454 case X86::VPSCATTERDQZmr: 9455 case X86::VPSCATTERQDZ128mr: 9456 case X86::VPSCATTERQDZ256mr: 9457 case X86::VPSCATTERQDZmr: 9458 case X86::VPSCATTERQQZ128mr: 9459 case X86::VPSCATTERQQZ256mr: 9460 case X86::VPSCATTERQQZmr: 9461 return true; 9462 } 9463 } 9464 9465 bool X86InstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel, 9466 const MachineRegisterInfo *MRI, 9467 const MachineInstr &DefMI, 9468 unsigned DefIdx, 9469 const MachineInstr &UseMI, 9470 unsigned UseIdx) const { 9471 return isHighLatencyDef(DefMI.getOpcode()); 9472 } 9473 9474 bool X86InstrInfo::hasReassociableOperands(const MachineInstr &Inst, 9475 const MachineBasicBlock *MBB) const { 9476 assert(Inst.getNumExplicitOperands() == 3 && Inst.getNumExplicitDefs() == 1 && 9477 Inst.getNumDefs() <= 2 && "Reassociation needs binary operators"); 9478 9479 // Integer binary math/logic instructions have a third source operand: 9480 // the EFLAGS register. That operand must be both defined here and never 9481 // used; ie, it must be dead. If the EFLAGS operand is live, then we can 9482 // not change anything because rearranging the operands could affect other 9483 // instructions that depend on the exact status flags (zero, sign, etc.) 9484 // that are set by using these particular operands with this operation. 9485 const MachineOperand *FlagDef = Inst.findRegisterDefOperand(X86::EFLAGS); 9486 assert((Inst.getNumDefs() == 1 || FlagDef) && "Implicit def isn't flags?"); 9487 if (FlagDef && !FlagDef->isDead()) 9488 return false; 9489 9490 return TargetInstrInfo::hasReassociableOperands(Inst, MBB); 9491 } 9492 9493 // TODO: There are many more machine instruction opcodes to match: 9494 // 1. Other data types (integer, vectors) 9495 // 2. Other math / logic operations (xor, or) 9496 // 3. Other forms of the same operation (intrinsics and other variants) 9497 bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst, 9498 bool Invert) const { 9499 if (Invert) 9500 return false; 9501 switch (Inst.getOpcode()) { 9502 case X86::ADD8rr: 9503 case X86::ADD16rr: 9504 case X86::ADD32rr: 9505 case X86::ADD64rr: 9506 case X86::AND8rr: 9507 case X86::AND16rr: 9508 case X86::AND32rr: 9509 case X86::AND64rr: 9510 case X86::OR8rr: 9511 case X86::OR16rr: 9512 case X86::OR32rr: 9513 case X86::OR64rr: 9514 case X86::XOR8rr: 9515 case X86::XOR16rr: 9516 case X86::XOR32rr: 9517 case X86::XOR64rr: 9518 case X86::IMUL16rr: 9519 case X86::IMUL32rr: 9520 case X86::IMUL64rr: 9521 case X86::PANDrr: 9522 case X86::PORrr: 9523 case X86::PXORrr: 9524 case X86::ANDPDrr: 9525 case X86::ANDPSrr: 9526 case X86::ORPDrr: 9527 case X86::ORPSrr: 9528 case X86::XORPDrr: 9529 case X86::XORPSrr: 9530 case X86::PADDBrr: 9531 case X86::PADDWrr: 9532 case X86::PADDDrr: 9533 case X86::PADDQrr: 9534 case X86::PMULLWrr: 9535 case X86::PMULLDrr: 9536 case X86::PMAXSBrr: 9537 case X86::PMAXSDrr: 9538 case X86::PMAXSWrr: 9539 case X86::PMAXUBrr: 9540 case X86::PMAXUDrr: 9541 case X86::PMAXUWrr: 9542 case X86::PMINSBrr: 9543 case X86::PMINSDrr: 9544 case X86::PMINSWrr: 9545 case X86::PMINUBrr: 9546 case X86::PMINUDrr: 9547 case X86::PMINUWrr: 9548 case X86::VPANDrr: 9549 case X86::VPANDYrr: 9550 case X86::VPANDDZ128rr: 9551 case X86::VPANDDZ256rr: 9552 case X86::VPANDDZrr: 9553 case X86::VPANDQZ128rr: 9554 case X86::VPANDQZ256rr: 9555 case X86::VPANDQZrr: 9556 case X86::VPORrr: 9557 case X86::VPORYrr: 9558 case X86::VPORDZ128rr: 9559 case X86::VPORDZ256rr: 9560 case X86::VPORDZrr: 9561 case X86::VPORQZ128rr: 9562 case X86::VPORQZ256rr: 9563 case X86::VPORQZrr: 9564 case X86::VPXORrr: 9565 case X86::VPXORYrr: 9566 case X86::VPXORDZ128rr: 9567 case X86::VPXORDZ256rr: 9568 case X86::VPXORDZrr: 9569 case X86::VPXORQZ128rr: 9570 case X86::VPXORQZ256rr: 9571 case X86::VPXORQZrr: 9572 case X86::VANDPDrr: 9573 case X86::VANDPSrr: 9574 case X86::VANDPDYrr: 9575 case X86::VANDPSYrr: 9576 case X86::VANDPDZ128rr: 9577 case X86::VANDPSZ128rr: 9578 case X86::VANDPDZ256rr: 9579 case X86::VANDPSZ256rr: 9580 case X86::VANDPDZrr: 9581 case X86::VANDPSZrr: 9582 case X86::VORPDrr: 9583 case X86::VORPSrr: 9584 case X86::VORPDYrr: 9585 case X86::VORPSYrr: 9586 case X86::VORPDZ128rr: 9587 case X86::VORPSZ128rr: 9588 case X86::VORPDZ256rr: 9589 case X86::VORPSZ256rr: 9590 case X86::VORPDZrr: 9591 case X86::VORPSZrr: 9592 case X86::VXORPDrr: 9593 case X86::VXORPSrr: 9594 case X86::VXORPDYrr: 9595 case X86::VXORPSYrr: 9596 case X86::VXORPDZ128rr: 9597 case X86::VXORPSZ128rr: 9598 case X86::VXORPDZ256rr: 9599 case X86::VXORPSZ256rr: 9600 case X86::VXORPDZrr: 9601 case X86::VXORPSZrr: 9602 case X86::KADDBrr: 9603 case X86::KADDWrr: 9604 case X86::KADDDrr: 9605 case X86::KADDQrr: 9606 case X86::KANDBrr: 9607 case X86::KANDWrr: 9608 case X86::KANDDrr: 9609 case X86::KANDQrr: 9610 case X86::KORBrr: 9611 case X86::KORWrr: 9612 case X86::KORDrr: 9613 case X86::KORQrr: 9614 case X86::KXORBrr: 9615 case X86::KXORWrr: 9616 case X86::KXORDrr: 9617 case X86::KXORQrr: 9618 case X86::VPADDBrr: 9619 case X86::VPADDWrr: 9620 case X86::VPADDDrr: 9621 case X86::VPADDQrr: 9622 case X86::VPADDBYrr: 9623 case X86::VPADDWYrr: 9624 case X86::VPADDDYrr: 9625 case X86::VPADDQYrr: 9626 case X86::VPADDBZ128rr: 9627 case X86::VPADDWZ128rr: 9628 case X86::VPADDDZ128rr: 9629 case X86::VPADDQZ128rr: 9630 case X86::VPADDBZ256rr: 9631 case X86::VPADDWZ256rr: 9632 case X86::VPADDDZ256rr: 9633 case X86::VPADDQZ256rr: 9634 case X86::VPADDBZrr: 9635 case X86::VPADDWZrr: 9636 case X86::VPADDDZrr: 9637 case X86::VPADDQZrr: 9638 case X86::VPMULLWrr: 9639 case X86::VPMULLWYrr: 9640 case X86::VPMULLWZ128rr: 9641 case X86::VPMULLWZ256rr: 9642 case X86::VPMULLWZrr: 9643 case X86::VPMULLDrr: 9644 case X86::VPMULLDYrr: 9645 case X86::VPMULLDZ128rr: 9646 case X86::VPMULLDZ256rr: 9647 case X86::VPMULLDZrr: 9648 case X86::VPMULLQZ128rr: 9649 case X86::VPMULLQZ256rr: 9650 case X86::VPMULLQZrr: 9651 case X86::VPMAXSBrr: 9652 case X86::VPMAXSBYrr: 9653 case X86::VPMAXSBZ128rr: 9654 case X86::VPMAXSBZ256rr: 9655 case X86::VPMAXSBZrr: 9656 case X86::VPMAXSDrr: 9657 case X86::VPMAXSDYrr: 9658 case X86::VPMAXSDZ128rr: 9659 case X86::VPMAXSDZ256rr: 9660 case X86::VPMAXSDZrr: 9661 case X86::VPMAXSQZ128rr: 9662 case X86::VPMAXSQZ256rr: 9663 case X86::VPMAXSQZrr: 9664 case X86::VPMAXSWrr: 9665 case X86::VPMAXSWYrr: 9666 case X86::VPMAXSWZ128rr: 9667 case X86::VPMAXSWZ256rr: 9668 case X86::VPMAXSWZrr: 9669 case X86::VPMAXUBrr: 9670 case X86::VPMAXUBYrr: 9671 case X86::VPMAXUBZ128rr: 9672 case X86::VPMAXUBZ256rr: 9673 case X86::VPMAXUBZrr: 9674 case X86::VPMAXUDrr: 9675 case X86::VPMAXUDYrr: 9676 case X86::VPMAXUDZ128rr: 9677 case X86::VPMAXUDZ256rr: 9678 case X86::VPMAXUDZrr: 9679 case X86::VPMAXUQZ128rr: 9680 case X86::VPMAXUQZ256rr: 9681 case X86::VPMAXUQZrr: 9682 case X86::VPMAXUWrr: 9683 case X86::VPMAXUWYrr: 9684 case X86::VPMAXUWZ128rr: 9685 case X86::VPMAXUWZ256rr: 9686 case X86::VPMAXUWZrr: 9687 case X86::VPMINSBrr: 9688 case X86::VPMINSBYrr: 9689 case X86::VPMINSBZ128rr: 9690 case X86::VPMINSBZ256rr: 9691 case X86::VPMINSBZrr: 9692 case X86::VPMINSDrr: 9693 case X86::VPMINSDYrr: 9694 case X86::VPMINSDZ128rr: 9695 case X86::VPMINSDZ256rr: 9696 case X86::VPMINSDZrr: 9697 case X86::VPMINSQZ128rr: 9698 case X86::VPMINSQZ256rr: 9699 case X86::VPMINSQZrr: 9700 case X86::VPMINSWrr: 9701 case X86::VPMINSWYrr: 9702 case X86::VPMINSWZ128rr: 9703 case X86::VPMINSWZ256rr: 9704 case X86::VPMINSWZrr: 9705 case X86::VPMINUBrr: 9706 case X86::VPMINUBYrr: 9707 case X86::VPMINUBZ128rr: 9708 case X86::VPMINUBZ256rr: 9709 case X86::VPMINUBZrr: 9710 case X86::VPMINUDrr: 9711 case X86::VPMINUDYrr: 9712 case X86::VPMINUDZ128rr: 9713 case X86::VPMINUDZ256rr: 9714 case X86::VPMINUDZrr: 9715 case X86::VPMINUQZ128rr: 9716 case X86::VPMINUQZ256rr: 9717 case X86::VPMINUQZrr: 9718 case X86::VPMINUWrr: 9719 case X86::VPMINUWYrr: 9720 case X86::VPMINUWZ128rr: 9721 case X86::VPMINUWZ256rr: 9722 case X86::VPMINUWZrr: 9723 // Normal min/max instructions are not commutative because of NaN and signed 9724 // zero semantics, but these are. Thus, there's no need to check for global 9725 // relaxed math; the instructions themselves have the properties we need. 9726 case X86::MAXCPDrr: 9727 case X86::MAXCPSrr: 9728 case X86::MAXCSDrr: 9729 case X86::MAXCSSrr: 9730 case X86::MINCPDrr: 9731 case X86::MINCPSrr: 9732 case X86::MINCSDrr: 9733 case X86::MINCSSrr: 9734 case X86::VMAXCPDrr: 9735 case X86::VMAXCPSrr: 9736 case X86::VMAXCPDYrr: 9737 case X86::VMAXCPSYrr: 9738 case X86::VMAXCPDZ128rr: 9739 case X86::VMAXCPSZ128rr: 9740 case X86::VMAXCPDZ256rr: 9741 case X86::VMAXCPSZ256rr: 9742 case X86::VMAXCPDZrr: 9743 case X86::VMAXCPSZrr: 9744 case X86::VMAXCSDrr: 9745 case X86::VMAXCSSrr: 9746 case X86::VMAXCSDZrr: 9747 case X86::VMAXCSSZrr: 9748 case X86::VMINCPDrr: 9749 case X86::VMINCPSrr: 9750 case X86::VMINCPDYrr: 9751 case X86::VMINCPSYrr: 9752 case X86::VMINCPDZ128rr: 9753 case X86::VMINCPSZ128rr: 9754 case X86::VMINCPDZ256rr: 9755 case X86::VMINCPSZ256rr: 9756 case X86::VMINCPDZrr: 9757 case X86::VMINCPSZrr: 9758 case X86::VMINCSDrr: 9759 case X86::VMINCSSrr: 9760 case X86::VMINCSDZrr: 9761 case X86::VMINCSSZrr: 9762 case X86::VMAXCPHZ128rr: 9763 case X86::VMAXCPHZ256rr: 9764 case X86::VMAXCPHZrr: 9765 case X86::VMAXCSHZrr: 9766 case X86::VMINCPHZ128rr: 9767 case X86::VMINCPHZ256rr: 9768 case X86::VMINCPHZrr: 9769 case X86::VMINCSHZrr: 9770 return true; 9771 case X86::ADDPDrr: 9772 case X86::ADDPSrr: 9773 case X86::ADDSDrr: 9774 case X86::ADDSSrr: 9775 case X86::MULPDrr: 9776 case X86::MULPSrr: 9777 case X86::MULSDrr: 9778 case X86::MULSSrr: 9779 case X86::VADDPDrr: 9780 case X86::VADDPSrr: 9781 case X86::VADDPDYrr: 9782 case X86::VADDPSYrr: 9783 case X86::VADDPDZ128rr: 9784 case X86::VADDPSZ128rr: 9785 case X86::VADDPDZ256rr: 9786 case X86::VADDPSZ256rr: 9787 case X86::VADDPDZrr: 9788 case X86::VADDPSZrr: 9789 case X86::VADDSDrr: 9790 case X86::VADDSSrr: 9791 case X86::VADDSDZrr: 9792 case X86::VADDSSZrr: 9793 case X86::VMULPDrr: 9794 case X86::VMULPSrr: 9795 case X86::VMULPDYrr: 9796 case X86::VMULPSYrr: 9797 case X86::VMULPDZ128rr: 9798 case X86::VMULPSZ128rr: 9799 case X86::VMULPDZ256rr: 9800 case X86::VMULPSZ256rr: 9801 case X86::VMULPDZrr: 9802 case X86::VMULPSZrr: 9803 case X86::VMULSDrr: 9804 case X86::VMULSSrr: 9805 case X86::VMULSDZrr: 9806 case X86::VMULSSZrr: 9807 case X86::VADDPHZ128rr: 9808 case X86::VADDPHZ256rr: 9809 case X86::VADDPHZrr: 9810 case X86::VADDSHZrr: 9811 case X86::VMULPHZ128rr: 9812 case X86::VMULPHZ256rr: 9813 case X86::VMULPHZrr: 9814 case X86::VMULSHZrr: 9815 return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) && 9816 Inst.getFlag(MachineInstr::MIFlag::FmNsz); 9817 default: 9818 return false; 9819 } 9820 } 9821 9822 /// If \p DescribedReg overlaps with the MOVrr instruction's destination 9823 /// register then, if possible, describe the value in terms of the source 9824 /// register. 9825 static std::optional<ParamLoadedValue> 9826 describeMOVrrLoadedValue(const MachineInstr &MI, Register DescribedReg, 9827 const TargetRegisterInfo *TRI) { 9828 Register DestReg = MI.getOperand(0).getReg(); 9829 Register SrcReg = MI.getOperand(1).getReg(); 9830 9831 auto Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {}); 9832 9833 // If the described register is the destination, just return the source. 9834 if (DestReg == DescribedReg) 9835 return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr); 9836 9837 // If the described register is a sub-register of the destination register, 9838 // then pick out the source register's corresponding sub-register. 9839 if (unsigned SubRegIdx = TRI->getSubRegIndex(DestReg, DescribedReg)) { 9840 Register SrcSubReg = TRI->getSubReg(SrcReg, SubRegIdx); 9841 return ParamLoadedValue(MachineOperand::CreateReg(SrcSubReg, false), Expr); 9842 } 9843 9844 // The remaining case to consider is when the described register is a 9845 // super-register of the destination register. MOV8rr and MOV16rr does not 9846 // write to any of the other bytes in the register, meaning that we'd have to 9847 // describe the value using a combination of the source register and the 9848 // non-overlapping bits in the described register, which is not currently 9849 // possible. 9850 if (MI.getOpcode() == X86::MOV8rr || MI.getOpcode() == X86::MOV16rr || 9851 !TRI->isSuperRegister(DestReg, DescribedReg)) 9852 return std::nullopt; 9853 9854 assert(MI.getOpcode() == X86::MOV32rr && "Unexpected super-register case"); 9855 return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr); 9856 } 9857 9858 std::optional<ParamLoadedValue> 9859 X86InstrInfo::describeLoadedValue(const MachineInstr &MI, Register Reg) const { 9860 const MachineOperand *Op = nullptr; 9861 DIExpression *Expr = nullptr; 9862 9863 const TargetRegisterInfo *TRI = &getRegisterInfo(); 9864 9865 switch (MI.getOpcode()) { 9866 case X86::LEA32r: 9867 case X86::LEA64r: 9868 case X86::LEA64_32r: { 9869 // We may need to describe a 64-bit parameter with a 32-bit LEA. 9870 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) 9871 return std::nullopt; 9872 9873 // Operand 4 could be global address. For now we do not support 9874 // such situation. 9875 if (!MI.getOperand(4).isImm() || !MI.getOperand(2).isImm()) 9876 return std::nullopt; 9877 9878 const MachineOperand &Op1 = MI.getOperand(1); 9879 const MachineOperand &Op2 = MI.getOperand(3); 9880 assert(Op2.isReg() && 9881 (Op2.getReg() == X86::NoRegister || Op2.getReg().isPhysical())); 9882 9883 // Omit situations like: 9884 // %rsi = lea %rsi, 4, ... 9885 if ((Op1.isReg() && Op1.getReg() == MI.getOperand(0).getReg()) || 9886 Op2.getReg() == MI.getOperand(0).getReg()) 9887 return std::nullopt; 9888 else if ((Op1.isReg() && Op1.getReg() != X86::NoRegister && 9889 TRI->regsOverlap(Op1.getReg(), MI.getOperand(0).getReg())) || 9890 (Op2.getReg() != X86::NoRegister && 9891 TRI->regsOverlap(Op2.getReg(), MI.getOperand(0).getReg()))) 9892 return std::nullopt; 9893 9894 int64_t Coef = MI.getOperand(2).getImm(); 9895 int64_t Offset = MI.getOperand(4).getImm(); 9896 SmallVector<uint64_t, 8> Ops; 9897 9898 if ((Op1.isReg() && Op1.getReg() != X86::NoRegister)) { 9899 Op = &Op1; 9900 } else if (Op1.isFI()) 9901 Op = &Op1; 9902 9903 if (Op && Op->isReg() && Op->getReg() == Op2.getReg() && Coef > 0) { 9904 Ops.push_back(dwarf::DW_OP_constu); 9905 Ops.push_back(Coef + 1); 9906 Ops.push_back(dwarf::DW_OP_mul); 9907 } else { 9908 if (Op && Op2.getReg() != X86::NoRegister) { 9909 int dwarfReg = TRI->getDwarfRegNum(Op2.getReg(), false); 9910 if (dwarfReg < 0) 9911 return std::nullopt; 9912 else if (dwarfReg < 32) { 9913 Ops.push_back(dwarf::DW_OP_breg0 + dwarfReg); 9914 Ops.push_back(0); 9915 } else { 9916 Ops.push_back(dwarf::DW_OP_bregx); 9917 Ops.push_back(dwarfReg); 9918 Ops.push_back(0); 9919 } 9920 } else if (!Op) { 9921 assert(Op2.getReg() != X86::NoRegister); 9922 Op = &Op2; 9923 } 9924 9925 if (Coef > 1) { 9926 assert(Op2.getReg() != X86::NoRegister); 9927 Ops.push_back(dwarf::DW_OP_constu); 9928 Ops.push_back(Coef); 9929 Ops.push_back(dwarf::DW_OP_mul); 9930 } 9931 9932 if (((Op1.isReg() && Op1.getReg() != X86::NoRegister) || Op1.isFI()) && 9933 Op2.getReg() != X86::NoRegister) { 9934 Ops.push_back(dwarf::DW_OP_plus); 9935 } 9936 } 9937 9938 DIExpression::appendOffset(Ops, Offset); 9939 Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), Ops); 9940 9941 return ParamLoadedValue(*Op, Expr); 9942 } 9943 case X86::MOV8ri: 9944 case X86::MOV16ri: 9945 // TODO: Handle MOV8ri and MOV16ri. 9946 return std::nullopt; 9947 case X86::MOV32ri: 9948 case X86::MOV64ri: 9949 case X86::MOV64ri32: 9950 // MOV32ri may be used for producing zero-extended 32-bit immediates in 9951 // 64-bit parameters, so we need to consider super-registers. 9952 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) 9953 return std::nullopt; 9954 return ParamLoadedValue(MI.getOperand(1), Expr); 9955 case X86::MOV8rr: 9956 case X86::MOV16rr: 9957 case X86::MOV32rr: 9958 case X86::MOV64rr: 9959 return describeMOVrrLoadedValue(MI, Reg, TRI); 9960 case X86::XOR32rr: { 9961 // 64-bit parameters are zero-materialized using XOR32rr, so also consider 9962 // super-registers. 9963 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) 9964 return std::nullopt; 9965 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) 9966 return ParamLoadedValue(MachineOperand::CreateImm(0), Expr); 9967 return std::nullopt; 9968 } 9969 case X86::MOVSX64rr32: { 9970 // We may need to describe the lower 32 bits of the MOVSX; for example, in 9971 // cases like this: 9972 // 9973 // $ebx = [...] 9974 // $rdi = MOVSX64rr32 $ebx 9975 // $esi = MOV32rr $edi 9976 if (!TRI->isSubRegisterEq(MI.getOperand(0).getReg(), Reg)) 9977 return std::nullopt; 9978 9979 Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {}); 9980 9981 // If the described register is the destination register we need to 9982 // sign-extend the source register from 32 bits. The other case we handle 9983 // is when the described register is the 32-bit sub-register of the 9984 // destination register, in case we just need to return the source 9985 // register. 9986 if (Reg == MI.getOperand(0).getReg()) 9987 Expr = DIExpression::appendExt(Expr, 32, 64, true); 9988 else 9989 assert(X86MCRegisterClasses[X86::GR32RegClassID].contains(Reg) && 9990 "Unhandled sub-register case for MOVSX64rr32"); 9991 9992 return ParamLoadedValue(MI.getOperand(1), Expr); 9993 } 9994 default: 9995 assert(!MI.isMoveImmediate() && "Unexpected MoveImm instruction"); 9996 return TargetInstrInfo::describeLoadedValue(MI, Reg); 9997 } 9998 } 9999 10000 /// This is an architecture-specific helper function of reassociateOps. 10001 /// Set special operand attributes for new instructions after reassociation. 10002 void X86InstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1, 10003 MachineInstr &OldMI2, 10004 MachineInstr &NewMI1, 10005 MachineInstr &NewMI2) const { 10006 // Integer instructions may define an implicit EFLAGS dest register operand. 10007 MachineOperand *OldFlagDef1 = OldMI1.findRegisterDefOperand(X86::EFLAGS); 10008 MachineOperand *OldFlagDef2 = OldMI2.findRegisterDefOperand(X86::EFLAGS); 10009 10010 assert(!OldFlagDef1 == !OldFlagDef2 && 10011 "Unexpected instruction type for reassociation"); 10012 10013 if (!OldFlagDef1 || !OldFlagDef2) 10014 return; 10015 10016 assert(OldFlagDef1->isDead() && OldFlagDef2->isDead() && 10017 "Must have dead EFLAGS operand in reassociable instruction"); 10018 10019 MachineOperand *NewFlagDef1 = NewMI1.findRegisterDefOperand(X86::EFLAGS); 10020 MachineOperand *NewFlagDef2 = NewMI2.findRegisterDefOperand(X86::EFLAGS); 10021 10022 assert(NewFlagDef1 && NewFlagDef2 && 10023 "Unexpected operand in reassociable instruction"); 10024 10025 // Mark the new EFLAGS operands as dead to be helpful to subsequent iterations 10026 // of this pass or other passes. The EFLAGS operands must be dead in these new 10027 // instructions because the EFLAGS operands in the original instructions must 10028 // be dead in order for reassociation to occur. 10029 NewFlagDef1->setIsDead(); 10030 NewFlagDef2->setIsDead(); 10031 } 10032 10033 std::pair<unsigned, unsigned> 10034 X86InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 10035 return std::make_pair(TF, 0u); 10036 } 10037 10038 ArrayRef<std::pair<unsigned, const char *>> 10039 X86InstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 10040 using namespace X86II; 10041 static const std::pair<unsigned, const char *> TargetFlags[] = { 10042 {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"}, 10043 {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"}, 10044 {MO_GOT, "x86-got"}, 10045 {MO_GOTOFF, "x86-gotoff"}, 10046 {MO_GOTPCREL, "x86-gotpcrel"}, 10047 {MO_GOTPCREL_NORELAX, "x86-gotpcrel-norelax"}, 10048 {MO_PLT, "x86-plt"}, 10049 {MO_TLSGD, "x86-tlsgd"}, 10050 {MO_TLSLD, "x86-tlsld"}, 10051 {MO_TLSLDM, "x86-tlsldm"}, 10052 {MO_GOTTPOFF, "x86-gottpoff"}, 10053 {MO_INDNTPOFF, "x86-indntpoff"}, 10054 {MO_TPOFF, "x86-tpoff"}, 10055 {MO_DTPOFF, "x86-dtpoff"}, 10056 {MO_NTPOFF, "x86-ntpoff"}, 10057 {MO_GOTNTPOFF, "x86-gotntpoff"}, 10058 {MO_DLLIMPORT, "x86-dllimport"}, 10059 {MO_DARWIN_NONLAZY, "x86-darwin-nonlazy"}, 10060 {MO_DARWIN_NONLAZY_PIC_BASE, "x86-darwin-nonlazy-pic-base"}, 10061 {MO_TLVP, "x86-tlvp"}, 10062 {MO_TLVP_PIC_BASE, "x86-tlvp-pic-base"}, 10063 {MO_SECREL, "x86-secrel"}, 10064 {MO_COFFSTUB, "x86-coffstub"}}; 10065 return ArrayRef(TargetFlags); 10066 } 10067 10068 namespace { 10069 /// Create Global Base Reg pass. This initializes the PIC 10070 /// global base register for x86-32. 10071 struct CGBR : public MachineFunctionPass { 10072 static char ID; 10073 CGBR() : MachineFunctionPass(ID) {} 10074 10075 bool runOnMachineFunction(MachineFunction &MF) override { 10076 const X86TargetMachine *TM = 10077 static_cast<const X86TargetMachine *>(&MF.getTarget()); 10078 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 10079 10080 // Only emit a global base reg in PIC mode. 10081 if (!TM->isPositionIndependent()) 10082 return false; 10083 10084 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 10085 Register GlobalBaseReg = X86FI->getGlobalBaseReg(); 10086 10087 // If we didn't need a GlobalBaseReg, don't insert code. 10088 if (GlobalBaseReg == 0) 10089 return false; 10090 10091 // Insert the set of GlobalBaseReg into the first MBB of the function 10092 MachineBasicBlock &FirstMBB = MF.front(); 10093 MachineBasicBlock::iterator MBBI = FirstMBB.begin(); 10094 DebugLoc DL = FirstMBB.findDebugLoc(MBBI); 10095 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 10096 const X86InstrInfo *TII = STI.getInstrInfo(); 10097 10098 Register PC; 10099 if (STI.isPICStyleGOT()) 10100 PC = RegInfo.createVirtualRegister(&X86::GR32RegClass); 10101 else 10102 PC = GlobalBaseReg; 10103 10104 if (STI.is64Bit()) { 10105 if (TM->getCodeModel() == CodeModel::Large) { 10106 // In the large code model, we are aiming for this code, though the 10107 // register allocation may vary: 10108 // leaq .LN$pb(%rip), %rax 10109 // movq $_GLOBAL_OFFSET_TABLE_ - .LN$pb, %rcx 10110 // addq %rcx, %rax 10111 // RAX now holds address of _GLOBAL_OFFSET_TABLE_. 10112 Register PBReg = RegInfo.createVirtualRegister(&X86::GR64RegClass); 10113 Register GOTReg = RegInfo.createVirtualRegister(&X86::GR64RegClass); 10114 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PBReg) 10115 .addReg(X86::RIP) 10116 .addImm(0) 10117 .addReg(0) 10118 .addSym(MF.getPICBaseSymbol()) 10119 .addReg(0); 10120 std::prev(MBBI)->setPreInstrSymbol(MF, MF.getPICBaseSymbol()); 10121 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOV64ri), GOTReg) 10122 .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", 10123 X86II::MO_PIC_BASE_OFFSET); 10124 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD64rr), PC) 10125 .addReg(PBReg, RegState::Kill) 10126 .addReg(GOTReg, RegState::Kill); 10127 } else { 10128 // In other code models, use a RIP-relative LEA to materialize the 10129 // GOT. 10130 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC) 10131 .addReg(X86::RIP) 10132 .addImm(0) 10133 .addReg(0) 10134 .addExternalSymbol("_GLOBAL_OFFSET_TABLE_") 10135 .addReg(0); 10136 } 10137 } else { 10138 // Operand of MovePCtoStack is completely ignored by asm printer. It's 10139 // only used in JIT code emission as displacement to pc. 10140 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOVPC32r), PC).addImm(0); 10141 10142 // If we're using vanilla 'GOT' PIC style, we should use relative 10143 // addressing not to pc, but to _GLOBAL_OFFSET_TABLE_ external. 10144 if (STI.isPICStyleGOT()) { 10145 // Generate addl $__GLOBAL_OFFSET_TABLE_ + [.-piclabel], 10146 // %some_register 10147 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD32ri), GlobalBaseReg) 10148 .addReg(PC) 10149 .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", 10150 X86II::MO_GOT_ABSOLUTE_ADDRESS); 10151 } 10152 } 10153 10154 return true; 10155 } 10156 10157 StringRef getPassName() const override { 10158 return "X86 PIC Global Base Reg Initialization"; 10159 } 10160 10161 void getAnalysisUsage(AnalysisUsage &AU) const override { 10162 AU.setPreservesCFG(); 10163 MachineFunctionPass::getAnalysisUsage(AU); 10164 } 10165 }; 10166 } // namespace 10167 10168 char CGBR::ID = 0; 10169 FunctionPass *llvm::createX86GlobalBaseRegPass() { return new CGBR(); } 10170 10171 namespace { 10172 struct LDTLSCleanup : public MachineFunctionPass { 10173 static char ID; 10174 LDTLSCleanup() : MachineFunctionPass(ID) {} 10175 10176 bool runOnMachineFunction(MachineFunction &MF) override { 10177 if (skipFunction(MF.getFunction())) 10178 return false; 10179 10180 X86MachineFunctionInfo *MFI = MF.getInfo<X86MachineFunctionInfo>(); 10181 if (MFI->getNumLocalDynamicTLSAccesses() < 2) { 10182 // No point folding accesses if there isn't at least two. 10183 return false; 10184 } 10185 10186 MachineDominatorTree *DT = &getAnalysis<MachineDominatorTree>(); 10187 return VisitNode(DT->getRootNode(), 0); 10188 } 10189 10190 // Visit the dominator subtree rooted at Node in pre-order. 10191 // If TLSBaseAddrReg is non-null, then use that to replace any 10192 // TLS_base_addr instructions. Otherwise, create the register 10193 // when the first such instruction is seen, and then use it 10194 // as we encounter more instructions. 10195 bool VisitNode(MachineDomTreeNode *Node, unsigned TLSBaseAddrReg) { 10196 MachineBasicBlock *BB = Node->getBlock(); 10197 bool Changed = false; 10198 10199 // Traverse the current block. 10200 for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; 10201 ++I) { 10202 switch (I->getOpcode()) { 10203 case X86::TLS_base_addr32: 10204 case X86::TLS_base_addr64: 10205 if (TLSBaseAddrReg) 10206 I = ReplaceTLSBaseAddrCall(*I, TLSBaseAddrReg); 10207 else 10208 I = SetRegister(*I, &TLSBaseAddrReg); 10209 Changed = true; 10210 break; 10211 default: 10212 break; 10213 } 10214 } 10215 10216 // Visit the children of this block in the dominator tree. 10217 for (auto &I : *Node) { 10218 Changed |= VisitNode(I, TLSBaseAddrReg); 10219 } 10220 10221 return Changed; 10222 } 10223 10224 // Replace the TLS_base_addr instruction I with a copy from 10225 // TLSBaseAddrReg, returning the new instruction. 10226 MachineInstr *ReplaceTLSBaseAddrCall(MachineInstr &I, 10227 unsigned TLSBaseAddrReg) { 10228 MachineFunction *MF = I.getParent()->getParent(); 10229 const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); 10230 const bool is64Bit = STI.is64Bit(); 10231 const X86InstrInfo *TII = STI.getInstrInfo(); 10232 10233 // Insert a Copy from TLSBaseAddrReg to RAX/EAX. 10234 MachineInstr *Copy = 10235 BuildMI(*I.getParent(), I, I.getDebugLoc(), 10236 TII->get(TargetOpcode::COPY), is64Bit ? X86::RAX : X86::EAX) 10237 .addReg(TLSBaseAddrReg); 10238 10239 // Erase the TLS_base_addr instruction. 10240 I.eraseFromParent(); 10241 10242 return Copy; 10243 } 10244 10245 // Create a virtual register in *TLSBaseAddrReg, and populate it by 10246 // inserting a copy instruction after I. Returns the new instruction. 10247 MachineInstr *SetRegister(MachineInstr &I, unsigned *TLSBaseAddrReg) { 10248 MachineFunction *MF = I.getParent()->getParent(); 10249 const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); 10250 const bool is64Bit = STI.is64Bit(); 10251 const X86InstrInfo *TII = STI.getInstrInfo(); 10252 10253 // Create a virtual register for the TLS base address. 10254 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 10255 *TLSBaseAddrReg = RegInfo.createVirtualRegister( 10256 is64Bit ? &X86::GR64RegClass : &X86::GR32RegClass); 10257 10258 // Insert a copy from RAX/EAX to TLSBaseAddrReg. 10259 MachineInstr *Next = I.getNextNode(); 10260 MachineInstr *Copy = BuildMI(*I.getParent(), Next, I.getDebugLoc(), 10261 TII->get(TargetOpcode::COPY), *TLSBaseAddrReg) 10262 .addReg(is64Bit ? X86::RAX : X86::EAX); 10263 10264 return Copy; 10265 } 10266 10267 StringRef getPassName() const override { 10268 return "Local Dynamic TLS Access Clean-up"; 10269 } 10270 10271 void getAnalysisUsage(AnalysisUsage &AU) const override { 10272 AU.setPreservesCFG(); 10273 AU.addRequired<MachineDominatorTree>(); 10274 MachineFunctionPass::getAnalysisUsage(AU); 10275 } 10276 }; 10277 } // namespace 10278 10279 char LDTLSCleanup::ID = 0; 10280 FunctionPass *llvm::createCleanupLocalDynamicTLSPass() { 10281 return new LDTLSCleanup(); 10282 } 10283 10284 /// Constants defining how certain sequences should be outlined. 10285 /// 10286 /// \p MachineOutlinerDefault implies that the function is called with a call 10287 /// instruction, and a return must be emitted for the outlined function frame. 10288 /// 10289 /// That is, 10290 /// 10291 /// I1 OUTLINED_FUNCTION: 10292 /// I2 --> call OUTLINED_FUNCTION I1 10293 /// I3 I2 10294 /// I3 10295 /// ret 10296 /// 10297 /// * Call construction overhead: 1 (call instruction) 10298 /// * Frame construction overhead: 1 (return instruction) 10299 /// 10300 /// \p MachineOutlinerTailCall implies that the function is being tail called. 10301 /// A jump is emitted instead of a call, and the return is already present in 10302 /// the outlined sequence. That is, 10303 /// 10304 /// I1 OUTLINED_FUNCTION: 10305 /// I2 --> jmp OUTLINED_FUNCTION I1 10306 /// ret I2 10307 /// ret 10308 /// 10309 /// * Call construction overhead: 1 (jump instruction) 10310 /// * Frame construction overhead: 0 (don't need to return) 10311 /// 10312 enum MachineOutlinerClass { MachineOutlinerDefault, MachineOutlinerTailCall }; 10313 10314 std::optional<outliner::OutlinedFunction> 10315 X86InstrInfo::getOutliningCandidateInfo( 10316 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { 10317 unsigned SequenceSize = 10318 std::accumulate(RepeatedSequenceLocs[0].front(), 10319 std::next(RepeatedSequenceLocs[0].back()), 0, 10320 [](unsigned Sum, const MachineInstr &MI) { 10321 // FIXME: x86 doesn't implement getInstSizeInBytes, so 10322 // we can't tell the cost. Just assume each instruction 10323 // is one byte. 10324 if (MI.isDebugInstr() || MI.isKill()) 10325 return Sum; 10326 return Sum + 1; 10327 }); 10328 10329 // We check to see if CFI Instructions are present, and if they are 10330 // we find the number of CFI Instructions in the candidates. 10331 unsigned CFICount = 0; 10332 for (auto &I : make_range(RepeatedSequenceLocs[0].front(), 10333 std::next(RepeatedSequenceLocs[0].back()))) { 10334 if (I.isCFIInstruction()) 10335 CFICount++; 10336 } 10337 10338 // We compare the number of found CFI Instructions to the number of CFI 10339 // instructions in the parent function for each candidate. We must check this 10340 // since if we outline one of the CFI instructions in a function, we have to 10341 // outline them all for correctness. If we do not, the address offsets will be 10342 // incorrect between the two sections of the program. 10343 for (outliner::Candidate &C : RepeatedSequenceLocs) { 10344 std::vector<MCCFIInstruction> CFIInstructions = 10345 C.getMF()->getFrameInstructions(); 10346 10347 if (CFICount > 0 && CFICount != CFIInstructions.size()) 10348 return std::nullopt; 10349 } 10350 10351 // FIXME: Use real size in bytes for call and ret instructions. 10352 if (RepeatedSequenceLocs[0].back()->isTerminator()) { 10353 for (outliner::Candidate &C : RepeatedSequenceLocs) 10354 C.setCallInfo(MachineOutlinerTailCall, 1); 10355 10356 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 10357 0, // Number of bytes to emit frame. 10358 MachineOutlinerTailCall // Type of frame. 10359 ); 10360 } 10361 10362 if (CFICount > 0) 10363 return std::nullopt; 10364 10365 for (outliner::Candidate &C : RepeatedSequenceLocs) 10366 C.setCallInfo(MachineOutlinerDefault, 1); 10367 10368 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 1, 10369 MachineOutlinerDefault); 10370 } 10371 10372 bool X86InstrInfo::isFunctionSafeToOutlineFrom( 10373 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const { 10374 const Function &F = MF.getFunction(); 10375 10376 // Does the function use a red zone? If it does, then we can't risk messing 10377 // with the stack. 10378 if (Subtarget.getFrameLowering()->has128ByteRedZone(MF)) { 10379 // It could have a red zone. If it does, then we don't want to touch it. 10380 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 10381 if (!X86FI || X86FI->getUsesRedZone()) 10382 return false; 10383 } 10384 10385 // If we *don't* want to outline from things that could potentially be deduped 10386 // then return false. 10387 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) 10388 return false; 10389 10390 // This function is viable for outlining, so return true. 10391 return true; 10392 } 10393 10394 outliner::InstrType 10395 X86InstrInfo::getOutliningTypeImpl(MachineBasicBlock::iterator &MIT, 10396 unsigned Flags) const { 10397 MachineInstr &MI = *MIT; 10398 10399 // Is this a terminator for a basic block? 10400 if (MI.isTerminator()) 10401 // TargetInstrInfo::getOutliningType has already filtered out anything 10402 // that would break this, so we can allow it here. 10403 return outliner::InstrType::Legal; 10404 10405 // Don't outline anything that modifies or reads from the stack pointer. 10406 // 10407 // FIXME: There are instructions which are being manually built without 10408 // explicit uses/defs so we also have to check the MCInstrDesc. We should be 10409 // able to remove the extra checks once those are fixed up. For example, 10410 // sometimes we might get something like %rax = POP64r 1. This won't be 10411 // caught by modifiesRegister or readsRegister even though the instruction 10412 // really ought to be formed so that modifiesRegister/readsRegister would 10413 // catch it. 10414 if (MI.modifiesRegister(X86::RSP, &RI) || MI.readsRegister(X86::RSP, &RI) || 10415 MI.getDesc().hasImplicitUseOfPhysReg(X86::RSP) || 10416 MI.getDesc().hasImplicitDefOfPhysReg(X86::RSP)) 10417 return outliner::InstrType::Illegal; 10418 10419 // Outlined calls change the instruction pointer, so don't read from it. 10420 if (MI.readsRegister(X86::RIP, &RI) || 10421 MI.getDesc().hasImplicitUseOfPhysReg(X86::RIP) || 10422 MI.getDesc().hasImplicitDefOfPhysReg(X86::RIP)) 10423 return outliner::InstrType::Illegal; 10424 10425 // Don't outline CFI instructions. 10426 if (MI.isCFIInstruction()) 10427 return outliner::InstrType::Illegal; 10428 10429 return outliner::InstrType::Legal; 10430 } 10431 10432 void X86InstrInfo::buildOutlinedFrame( 10433 MachineBasicBlock &MBB, MachineFunction &MF, 10434 const outliner::OutlinedFunction &OF) const { 10435 // If we're a tail call, we already have a return, so don't do anything. 10436 if (OF.FrameConstructionID == MachineOutlinerTailCall) 10437 return; 10438 10439 // We're a normal call, so our sequence doesn't have a return instruction. 10440 // Add it in. 10441 MachineInstr *retq = BuildMI(MF, DebugLoc(), get(X86::RET64)); 10442 MBB.insert(MBB.end(), retq); 10443 } 10444 10445 MachineBasicBlock::iterator X86InstrInfo::insertOutlinedCall( 10446 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, 10447 MachineFunction &MF, outliner::Candidate &C) const { 10448 // Is it a tail call? 10449 if (C.CallConstructionID == MachineOutlinerTailCall) { 10450 // Yes, just insert a JMP. 10451 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(X86::TAILJMPd64)) 10452 .addGlobalAddress(M.getNamedValue(MF.getName()))); 10453 } else { 10454 // No, insert a call. 10455 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(X86::CALL64pcrel32)) 10456 .addGlobalAddress(M.getNamedValue(MF.getName()))); 10457 } 10458 10459 return It; 10460 } 10461 10462 void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB, 10463 MachineBasicBlock::iterator Iter, 10464 DebugLoc &DL, 10465 bool AllowSideEffects) const { 10466 const MachineFunction &MF = *MBB.getParent(); 10467 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>(); 10468 const TargetRegisterInfo &TRI = getRegisterInfo(); 10469 10470 if (ST.hasMMX() && X86::VR64RegClass.contains(Reg)) 10471 // FIXME: Should we ignore MMX registers? 10472 return; 10473 10474 if (TRI.isGeneralPurposeRegister(MF, Reg)) { 10475 // Convert register to the 32-bit version. Both 'movl' and 'xorl' clear the 10476 // upper bits of a 64-bit register automagically. 10477 Reg = getX86SubSuperRegister(Reg, 32); 10478 10479 if (!AllowSideEffects) 10480 // XOR affects flags, so use a MOV instead. 10481 BuildMI(MBB, Iter, DL, get(X86::MOV32ri), Reg).addImm(0); 10482 else 10483 BuildMI(MBB, Iter, DL, get(X86::XOR32rr), Reg) 10484 .addReg(Reg, RegState::Undef) 10485 .addReg(Reg, RegState::Undef); 10486 } else if (X86::VR128RegClass.contains(Reg)) { 10487 // XMM# 10488 if (!ST.hasSSE1()) 10489 return; 10490 10491 // PXOR is safe to use because it doesn't affect flags. 10492 BuildMI(MBB, Iter, DL, get(X86::PXORrr), Reg) 10493 .addReg(Reg, RegState::Undef) 10494 .addReg(Reg, RegState::Undef); 10495 } else if (X86::VR256RegClass.contains(Reg)) { 10496 // YMM# 10497 if (!ST.hasAVX()) 10498 return; 10499 10500 // VPXOR is safe to use because it doesn't affect flags. 10501 BuildMI(MBB, Iter, DL, get(X86::VPXORrr), Reg) 10502 .addReg(Reg, RegState::Undef) 10503 .addReg(Reg, RegState::Undef); 10504 } else if (X86::VR512RegClass.contains(Reg)) { 10505 // ZMM# 10506 if (!ST.hasAVX512()) 10507 return; 10508 10509 // VPXORY is safe to use because it doesn't affect flags. 10510 BuildMI(MBB, Iter, DL, get(X86::VPXORYrr), Reg) 10511 .addReg(Reg, RegState::Undef) 10512 .addReg(Reg, RegState::Undef); 10513 } else if (X86::VK1RegClass.contains(Reg) || X86::VK2RegClass.contains(Reg) || 10514 X86::VK4RegClass.contains(Reg) || X86::VK8RegClass.contains(Reg) || 10515 X86::VK16RegClass.contains(Reg)) { 10516 if (!ST.hasVLX()) 10517 return; 10518 10519 // KXOR is safe to use because it doesn't affect flags. 10520 unsigned Op = ST.hasBWI() ? X86::KXORQrr : X86::KXORWrr; 10521 BuildMI(MBB, Iter, DL, get(Op), Reg) 10522 .addReg(Reg, RegState::Undef) 10523 .addReg(Reg, RegState::Undef); 10524 } 10525 } 10526 10527 bool X86InstrInfo::getMachineCombinerPatterns( 10528 MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns, 10529 bool DoRegPressureReduce) const { 10530 unsigned Opc = Root.getOpcode(); 10531 switch (Opc) { 10532 default: 10533 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns, 10534 DoRegPressureReduce); 10535 case X86::VPDPWSSDrr: 10536 case X86::VPDPWSSDrm: 10537 case X86::VPDPWSSDYrr: 10538 case X86::VPDPWSSDYrm: { 10539 Patterns.push_back(MachineCombinerPattern::DPWSSD); 10540 return true; 10541 } 10542 case X86::VPDPWSSDZ128r: 10543 case X86::VPDPWSSDZ128m: 10544 case X86::VPDPWSSDZ256r: 10545 case X86::VPDPWSSDZ256m: 10546 case X86::VPDPWSSDZr: 10547 case X86::VPDPWSSDZm: { 10548 if (Subtarget.hasBWI()) 10549 Patterns.push_back(MachineCombinerPattern::DPWSSD); 10550 return true; 10551 } 10552 } 10553 } 10554 10555 static void 10556 genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII, 10557 SmallVectorImpl<MachineInstr *> &InsInstrs, 10558 SmallVectorImpl<MachineInstr *> &DelInstrs, 10559 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) { 10560 MachineFunction *MF = Root.getMF(); 10561 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 10562 10563 unsigned Opc = Root.getOpcode(); 10564 unsigned AddOpc = 0; 10565 unsigned MaddOpc = 0; 10566 switch (Opc) { 10567 default: 10568 assert(false && "It should not reach here"); 10569 break; 10570 // vpdpwssd xmm2,xmm3,xmm1 10571 // --> 10572 // vpmaddwd xmm3,xmm3,xmm1 10573 // vpaddd xmm2,xmm2,xmm3 10574 case X86::VPDPWSSDrr: 10575 MaddOpc = X86::VPMADDWDrr; 10576 AddOpc = X86::VPADDDrr; 10577 break; 10578 case X86::VPDPWSSDrm: 10579 MaddOpc = X86::VPMADDWDrm; 10580 AddOpc = X86::VPADDDrr; 10581 break; 10582 case X86::VPDPWSSDZ128r: 10583 MaddOpc = X86::VPMADDWDZ128rr; 10584 AddOpc = X86::VPADDDZ128rr; 10585 break; 10586 case X86::VPDPWSSDZ128m: 10587 MaddOpc = X86::VPMADDWDZ128rm; 10588 AddOpc = X86::VPADDDZ128rr; 10589 break; 10590 // vpdpwssd ymm2,ymm3,ymm1 10591 // --> 10592 // vpmaddwd ymm3,ymm3,ymm1 10593 // vpaddd ymm2,ymm2,ymm3 10594 case X86::VPDPWSSDYrr: 10595 MaddOpc = X86::VPMADDWDYrr; 10596 AddOpc = X86::VPADDDYrr; 10597 break; 10598 case X86::VPDPWSSDYrm: 10599 MaddOpc = X86::VPMADDWDYrm; 10600 AddOpc = X86::VPADDDYrr; 10601 break; 10602 case X86::VPDPWSSDZ256r: 10603 MaddOpc = X86::VPMADDWDZ256rr; 10604 AddOpc = X86::VPADDDZ256rr; 10605 break; 10606 case X86::VPDPWSSDZ256m: 10607 MaddOpc = X86::VPMADDWDZ256rm; 10608 AddOpc = X86::VPADDDZ256rr; 10609 break; 10610 // vpdpwssd zmm2,zmm3,zmm1 10611 // --> 10612 // vpmaddwd zmm3,zmm3,zmm1 10613 // vpaddd zmm2,zmm2,zmm3 10614 case X86::VPDPWSSDZr: 10615 MaddOpc = X86::VPMADDWDZrr; 10616 AddOpc = X86::VPADDDZrr; 10617 break; 10618 case X86::VPDPWSSDZm: 10619 MaddOpc = X86::VPMADDWDZrm; 10620 AddOpc = X86::VPADDDZrr; 10621 break; 10622 } 10623 // Create vpmaddwd. 10624 const TargetRegisterClass *RC = 10625 RegInfo.getRegClass(Root.getOperand(0).getReg()); 10626 Register NewReg = RegInfo.createVirtualRegister(RC); 10627 MachineInstr *Madd = Root.getMF()->CloneMachineInstr(&Root); 10628 Madd->setDesc(TII.get(MaddOpc)); 10629 Madd->untieRegOperand(1); 10630 Madd->removeOperand(1); 10631 Madd->getOperand(0).setReg(NewReg); 10632 InstrIdxForVirtReg.insert(std::make_pair(NewReg, 0)); 10633 // Create vpaddd. 10634 Register DstReg = Root.getOperand(0).getReg(); 10635 bool IsKill = Root.getOperand(1).isKill(); 10636 MachineInstr *Add = 10637 BuildMI(*MF, MIMetadata(Root), TII.get(AddOpc), DstReg) 10638 .addReg(Root.getOperand(1).getReg(), getKillRegState(IsKill)) 10639 .addReg(Madd->getOperand(0).getReg(), getKillRegState(true)); 10640 InsInstrs.push_back(Madd); 10641 InsInstrs.push_back(Add); 10642 DelInstrs.push_back(&Root); 10643 } 10644 10645 void X86InstrInfo::genAlternativeCodeSequence( 10646 MachineInstr &Root, MachineCombinerPattern Pattern, 10647 SmallVectorImpl<MachineInstr *> &InsInstrs, 10648 SmallVectorImpl<MachineInstr *> &DelInstrs, 10649 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const { 10650 switch (Pattern) { 10651 default: 10652 // Reassociate instructions. 10653 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs, 10654 DelInstrs, InstrIdxForVirtReg); 10655 return; 10656 case MachineCombinerPattern::DPWSSD: 10657 genAlternativeDpCodeSequence(Root, *this, InsInstrs, DelInstrs, 10658 InstrIdxForVirtReg); 10659 return; 10660 } 10661 } 10662 10663 // See also: X86DAGToDAGISel::SelectInlineAsmMemoryOperand(). 10664 void X86InstrInfo::getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops, 10665 int FI) const { 10666 X86AddressMode M; 10667 M.BaseType = X86AddressMode::FrameIndexBase; 10668 M.Base.FrameIndex = FI; 10669 M.getFullAddress(Ops); 10670 } 10671 10672 #define GET_INSTRINFO_HELPERS 10673 #include "X86GenInstrInfo.inc" 10674