1 //===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines a DAG pattern matching instruction selector for X86, 10 // converting from a legalized dag to a X86 dag. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "X86.h" 15 #include "X86MachineFunctionInfo.h" 16 #include "X86RegisterInfo.h" 17 #include "X86Subtarget.h" 18 #include "X86TargetMachine.h" 19 #include "llvm/ADT/Statistic.h" 20 #include "llvm/CodeGen/MachineModuleInfo.h" 21 #include "llvm/CodeGen/SelectionDAGISel.h" 22 #include "llvm/Config/llvm-config.h" 23 #include "llvm/IR/ConstantRange.h" 24 #include "llvm/IR/Function.h" 25 #include "llvm/IR/Instructions.h" 26 #include "llvm/IR/Intrinsics.h" 27 #include "llvm/IR/IntrinsicsX86.h" 28 #include "llvm/IR/Type.h" 29 #include "llvm/Support/Debug.h" 30 #include "llvm/Support/ErrorHandling.h" 31 #include "llvm/Support/KnownBits.h" 32 #include "llvm/Support/MathExtras.h" 33 #include <cstdint> 34 35 using namespace llvm; 36 37 #define DEBUG_TYPE "x86-isel" 38 #define PASS_NAME "X86 DAG->DAG Instruction Selection" 39 40 STATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor"); 41 42 static cl::opt<bool> AndImmShrink("x86-and-imm-shrink", cl::init(true), 43 cl::desc("Enable setting constant bits to reduce size of mask immediates"), 44 cl::Hidden); 45 46 static cl::opt<bool> EnablePromoteAnyextLoad( 47 "x86-promote-anyext-load", cl::init(true), 48 cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden); 49 50 extern cl::opt<bool> IndirectBranchTracking; 51 52 //===----------------------------------------------------------------------===// 53 // Pattern Matcher Implementation 54 //===----------------------------------------------------------------------===// 55 56 namespace { 57 /// This corresponds to X86AddressMode, but uses SDValue's instead of register 58 /// numbers for the leaves of the matched tree. 59 struct X86ISelAddressMode { 60 enum { 61 RegBase, 62 FrameIndexBase 63 } BaseType = RegBase; 64 65 // This is really a union, discriminated by BaseType! 66 SDValue Base_Reg; 67 int Base_FrameIndex = 0; 68 69 unsigned Scale = 1; 70 SDValue IndexReg; 71 int32_t Disp = 0; 72 SDValue Segment; 73 const GlobalValue *GV = nullptr; 74 const Constant *CP = nullptr; 75 const BlockAddress *BlockAddr = nullptr; 76 const char *ES = nullptr; 77 MCSymbol *MCSym = nullptr; 78 int JT = -1; 79 Align Alignment; // CP alignment. 80 unsigned char SymbolFlags = X86II::MO_NO_FLAG; // X86II::MO_* 81 bool NegateIndex = false; 82 83 X86ISelAddressMode() = default; 84 85 bool hasSymbolicDisplacement() const { 86 return GV != nullptr || CP != nullptr || ES != nullptr || 87 MCSym != nullptr || JT != -1 || BlockAddr != nullptr; 88 } 89 90 bool hasBaseOrIndexReg() const { 91 return BaseType == FrameIndexBase || 92 IndexReg.getNode() != nullptr || Base_Reg.getNode() != nullptr; 93 } 94 95 /// Return true if this addressing mode is already RIP-relative. 96 bool isRIPRelative() const { 97 if (BaseType != RegBase) return false; 98 if (RegisterSDNode *RegNode = 99 dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode())) 100 return RegNode->getReg() == X86::RIP; 101 return false; 102 } 103 104 void setBaseReg(SDValue Reg) { 105 BaseType = RegBase; 106 Base_Reg = Reg; 107 } 108 109 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 110 void dump(SelectionDAG *DAG = nullptr) { 111 dbgs() << "X86ISelAddressMode " << this << '\n'; 112 dbgs() << "Base_Reg "; 113 if (Base_Reg.getNode()) 114 Base_Reg.getNode()->dump(DAG); 115 else 116 dbgs() << "nul\n"; 117 if (BaseType == FrameIndexBase) 118 dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n'; 119 dbgs() << " Scale " << Scale << '\n' 120 << "IndexReg "; 121 if (NegateIndex) 122 dbgs() << "negate "; 123 if (IndexReg.getNode()) 124 IndexReg.getNode()->dump(DAG); 125 else 126 dbgs() << "nul\n"; 127 dbgs() << " Disp " << Disp << '\n' 128 << "GV "; 129 if (GV) 130 GV->dump(); 131 else 132 dbgs() << "nul"; 133 dbgs() << " CP "; 134 if (CP) 135 CP->dump(); 136 else 137 dbgs() << "nul"; 138 dbgs() << '\n' 139 << "ES "; 140 if (ES) 141 dbgs() << ES; 142 else 143 dbgs() << "nul"; 144 dbgs() << " MCSym "; 145 if (MCSym) 146 dbgs() << MCSym; 147 else 148 dbgs() << "nul"; 149 dbgs() << " JT" << JT << " Align" << Alignment.value() << '\n'; 150 } 151 #endif 152 }; 153 } 154 155 namespace { 156 //===--------------------------------------------------------------------===// 157 /// ISel - X86-specific code to select X86 machine instructions for 158 /// SelectionDAG operations. 159 /// 160 class X86DAGToDAGISel final : public SelectionDAGISel { 161 /// Keep a pointer to the X86Subtarget around so that we can 162 /// make the right decision when generating code for different targets. 163 const X86Subtarget *Subtarget; 164 165 /// If true, selector should try to optimize for minimum code size. 166 bool OptForMinSize; 167 168 /// Disable direct TLS access through segment registers. 169 bool IndirectTlsSegRefs; 170 171 public: 172 static char ID; 173 174 X86DAGToDAGISel() = delete; 175 176 explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOptLevel OptLevel) 177 : SelectionDAGISel(ID, tm, OptLevel), Subtarget(nullptr), 178 OptForMinSize(false), IndirectTlsSegRefs(false) {} 179 180 bool runOnMachineFunction(MachineFunction &MF) override { 181 // Reset the subtarget each time through. 182 Subtarget = &MF.getSubtarget<X86Subtarget>(); 183 IndirectTlsSegRefs = MF.getFunction().hasFnAttribute( 184 "indirect-tls-seg-refs"); 185 186 // OptFor[Min]Size are used in pattern predicates that isel is matching. 187 OptForMinSize = MF.getFunction().hasMinSize(); 188 assert((!OptForMinSize || MF.getFunction().hasOptSize()) && 189 "OptForMinSize implies OptForSize"); 190 191 SelectionDAGISel::runOnMachineFunction(MF); 192 return true; 193 } 194 195 void emitFunctionEntryCode() override; 196 197 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override; 198 199 void PreprocessISelDAG() override; 200 void PostprocessISelDAG() override; 201 202 // Include the pieces autogenerated from the target description. 203 #include "X86GenDAGISel.inc" 204 205 private: 206 void Select(SDNode *N) override; 207 208 bool foldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM); 209 bool matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM, 210 bool AllowSegmentRegForX32 = false); 211 bool matchWrapper(SDValue N, X86ISelAddressMode &AM); 212 bool matchAddress(SDValue N, X86ISelAddressMode &AM); 213 bool matchVectorAddress(SDValue N, X86ISelAddressMode &AM); 214 bool matchAdd(SDValue &N, X86ISelAddressMode &AM, unsigned Depth); 215 SDValue matchIndexRecursively(SDValue N, X86ISelAddressMode &AM, 216 unsigned Depth); 217 bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 218 unsigned Depth); 219 bool matchVectorAddressRecursively(SDValue N, X86ISelAddressMode &AM, 220 unsigned Depth); 221 bool matchAddressBase(SDValue N, X86ISelAddressMode &AM); 222 bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base, 223 SDValue &Scale, SDValue &Index, SDValue &Disp, 224 SDValue &Segment); 225 bool selectVectorAddr(MemSDNode *Parent, SDValue BasePtr, SDValue IndexOp, 226 SDValue ScaleOp, SDValue &Base, SDValue &Scale, 227 SDValue &Index, SDValue &Disp, SDValue &Segment); 228 bool selectMOV64Imm32(SDValue N, SDValue &Imm); 229 bool selectLEAAddr(SDValue N, SDValue &Base, 230 SDValue &Scale, SDValue &Index, SDValue &Disp, 231 SDValue &Segment); 232 bool selectLEA64_32Addr(SDValue N, SDValue &Base, 233 SDValue &Scale, SDValue &Index, SDValue &Disp, 234 SDValue &Segment); 235 bool selectTLSADDRAddr(SDValue N, SDValue &Base, 236 SDValue &Scale, SDValue &Index, SDValue &Disp, 237 SDValue &Segment); 238 bool selectRelocImm(SDValue N, SDValue &Op); 239 240 bool tryFoldLoad(SDNode *Root, SDNode *P, SDValue N, 241 SDValue &Base, SDValue &Scale, 242 SDValue &Index, SDValue &Disp, 243 SDValue &Segment); 244 245 // Convenience method where P is also root. 246 bool tryFoldLoad(SDNode *P, SDValue N, 247 SDValue &Base, SDValue &Scale, 248 SDValue &Index, SDValue &Disp, 249 SDValue &Segment) { 250 return tryFoldLoad(P, P, N, Base, Scale, Index, Disp, Segment); 251 } 252 253 bool tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N, 254 SDValue &Base, SDValue &Scale, 255 SDValue &Index, SDValue &Disp, 256 SDValue &Segment); 257 258 bool isProfitableToFormMaskedOp(SDNode *N) const; 259 260 /// Implement addressing mode selection for inline asm expressions. 261 bool SelectInlineAsmMemoryOperand(const SDValue &Op, 262 InlineAsm::ConstraintCode ConstraintID, 263 std::vector<SDValue> &OutOps) override; 264 265 void emitSpecialCodeForMain(); 266 267 inline void getAddressOperands(X86ISelAddressMode &AM, const SDLoc &DL, 268 MVT VT, SDValue &Base, SDValue &Scale, 269 SDValue &Index, SDValue &Disp, 270 SDValue &Segment) { 271 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 272 Base = CurDAG->getTargetFrameIndex( 273 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout())); 274 else if (AM.Base_Reg.getNode()) 275 Base = AM.Base_Reg; 276 else 277 Base = CurDAG->getRegister(0, VT); 278 279 Scale = getI8Imm(AM.Scale, DL); 280 281 // Negate the index if needed. 282 if (AM.NegateIndex) { 283 unsigned NegOpc = VT == MVT::i64 ? X86::NEG64r : X86::NEG32r; 284 SDValue Neg = SDValue(CurDAG->getMachineNode(NegOpc, DL, VT, MVT::i32, 285 AM.IndexReg), 0); 286 AM.IndexReg = Neg; 287 } 288 289 if (AM.IndexReg.getNode()) 290 Index = AM.IndexReg; 291 else 292 Index = CurDAG->getRegister(0, VT); 293 294 // These are 32-bit even in 64-bit mode since RIP-relative offset 295 // is 32-bit. 296 if (AM.GV) 297 Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(), 298 MVT::i32, AM.Disp, 299 AM.SymbolFlags); 300 else if (AM.CP) 301 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment, 302 AM.Disp, AM.SymbolFlags); 303 else if (AM.ES) { 304 assert(!AM.Disp && "Non-zero displacement is ignored with ES."); 305 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); 306 } else if (AM.MCSym) { 307 assert(!AM.Disp && "Non-zero displacement is ignored with MCSym."); 308 assert(AM.SymbolFlags == 0 && "oo"); 309 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32); 310 } else if (AM.JT != -1) { 311 assert(!AM.Disp && "Non-zero displacement is ignored with JT."); 312 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); 313 } else if (AM.BlockAddr) 314 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp, 315 AM.SymbolFlags); 316 else 317 Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32); 318 319 if (AM.Segment.getNode()) 320 Segment = AM.Segment; 321 else 322 Segment = CurDAG->getRegister(0, MVT::i16); 323 } 324 325 // Utility function to determine whether we should avoid selecting 326 // immediate forms of instructions for better code size or not. 327 // At a high level, we'd like to avoid such instructions when 328 // we have similar constants used within the same basic block 329 // that can be kept in a register. 330 // 331 bool shouldAvoidImmediateInstFormsForSize(SDNode *N) const { 332 uint32_t UseCount = 0; 333 334 // Do not want to hoist if we're not optimizing for size. 335 // TODO: We'd like to remove this restriction. 336 // See the comment in X86InstrInfo.td for more info. 337 if (!CurDAG->shouldOptForSize()) 338 return false; 339 340 // Walk all the users of the immediate. 341 for (const SDNode *User : N->uses()) { 342 if (UseCount >= 2) 343 break; 344 345 // This user is already selected. Count it as a legitimate use and 346 // move on. 347 if (User->isMachineOpcode()) { 348 UseCount++; 349 continue; 350 } 351 352 // We want to count stores of immediates as real uses. 353 if (User->getOpcode() == ISD::STORE && 354 User->getOperand(1).getNode() == N) { 355 UseCount++; 356 continue; 357 } 358 359 // We don't currently match users that have > 2 operands (except 360 // for stores, which are handled above) 361 // Those instruction won't match in ISEL, for now, and would 362 // be counted incorrectly. 363 // This may change in the future as we add additional instruction 364 // types. 365 if (User->getNumOperands() != 2) 366 continue; 367 368 // If this is a sign-extended 8-bit integer immediate used in an ALU 369 // instruction, there is probably an opcode encoding to save space. 370 auto *C = dyn_cast<ConstantSDNode>(N); 371 if (C && isInt<8>(C->getSExtValue())) 372 continue; 373 374 // Immediates that are used for offsets as part of stack 375 // manipulation should be left alone. These are typically 376 // used to indicate SP offsets for argument passing and 377 // will get pulled into stores/pushes (implicitly). 378 if (User->getOpcode() == X86ISD::ADD || 379 User->getOpcode() == ISD::ADD || 380 User->getOpcode() == X86ISD::SUB || 381 User->getOpcode() == ISD::SUB) { 382 383 // Find the other operand of the add/sub. 384 SDValue OtherOp = User->getOperand(0); 385 if (OtherOp.getNode() == N) 386 OtherOp = User->getOperand(1); 387 388 // Don't count if the other operand is SP. 389 RegisterSDNode *RegNode; 390 if (OtherOp->getOpcode() == ISD::CopyFromReg && 391 (RegNode = dyn_cast_or_null<RegisterSDNode>( 392 OtherOp->getOperand(1).getNode()))) 393 if ((RegNode->getReg() == X86::ESP) || 394 (RegNode->getReg() == X86::RSP)) 395 continue; 396 } 397 398 // ... otherwise, count this and move on. 399 UseCount++; 400 } 401 402 // If we have more than 1 use, then recommend for hoisting. 403 return (UseCount > 1); 404 } 405 406 /// Return a target constant with the specified value of type i8. 407 inline SDValue getI8Imm(unsigned Imm, const SDLoc &DL) { 408 return CurDAG->getTargetConstant(Imm, DL, MVT::i8); 409 } 410 411 /// Return a target constant with the specified value, of type i32. 412 inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) { 413 return CurDAG->getTargetConstant(Imm, DL, MVT::i32); 414 } 415 416 /// Return a target constant with the specified value, of type i64. 417 inline SDValue getI64Imm(uint64_t Imm, const SDLoc &DL) { 418 return CurDAG->getTargetConstant(Imm, DL, MVT::i64); 419 } 420 421 SDValue getExtractVEXTRACTImmediate(SDNode *N, unsigned VecWidth, 422 const SDLoc &DL) { 423 assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width"); 424 uint64_t Index = N->getConstantOperandVal(1); 425 MVT VecVT = N->getOperand(0).getSimpleValueType(); 426 return getI8Imm((Index * VecVT.getScalarSizeInBits()) / VecWidth, DL); 427 } 428 429 SDValue getInsertVINSERTImmediate(SDNode *N, unsigned VecWidth, 430 const SDLoc &DL) { 431 assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width"); 432 uint64_t Index = N->getConstantOperandVal(2); 433 MVT VecVT = N->getSimpleValueType(0); 434 return getI8Imm((Index * VecVT.getScalarSizeInBits()) / VecWidth, DL); 435 } 436 437 SDValue getPermuteVINSERTCommutedImmediate(SDNode *N, unsigned VecWidth, 438 const SDLoc &DL) { 439 assert(VecWidth == 128 && "Unexpected vector width"); 440 uint64_t Index = N->getConstantOperandVal(2); 441 MVT VecVT = N->getSimpleValueType(0); 442 uint64_t InsertIdx = (Index * VecVT.getScalarSizeInBits()) / VecWidth; 443 assert((InsertIdx == 0 || InsertIdx == 1) && "Bad insertf128 index"); 444 // vinsert(0,sub,vec) -> [sub0][vec1] -> vperm2x128(0x30,vec,sub) 445 // vinsert(1,sub,vec) -> [vec0][sub0] -> vperm2x128(0x02,vec,sub) 446 return getI8Imm(InsertIdx ? 0x02 : 0x30, DL); 447 } 448 449 SDValue getSBBZero(SDNode *N) { 450 SDLoc dl(N); 451 MVT VT = N->getSimpleValueType(0); 452 453 // Create zero. 454 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32); 455 SDValue Zero = SDValue( 456 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0); 457 if (VT == MVT::i64) { 458 Zero = SDValue( 459 CurDAG->getMachineNode( 460 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64, 461 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero, 462 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)), 463 0); 464 } 465 466 // Copy flags to the EFLAGS register and glue it to next node. 467 unsigned Opcode = N->getOpcode(); 468 assert((Opcode == X86ISD::SBB || Opcode == X86ISD::SETCC_CARRY) && 469 "Unexpected opcode for SBB materialization"); 470 unsigned FlagOpIndex = Opcode == X86ISD::SBB ? 2 : 1; 471 SDValue EFLAGS = 472 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS, 473 N->getOperand(FlagOpIndex), SDValue()); 474 475 // Create a 64-bit instruction if the result is 64-bits otherwise use the 476 // 32-bit version. 477 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr; 478 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32; 479 VTs = CurDAG->getVTList(SBBVT, MVT::i32); 480 return SDValue( 481 CurDAG->getMachineNode(Opc, dl, VTs, 482 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}), 483 0); 484 } 485 486 // Helper to detect unneeded and instructions on shift amounts. Called 487 // from PatFrags in tablegen. 488 bool isUnneededShiftMask(SDNode *N, unsigned Width) const { 489 assert(N->getOpcode() == ISD::AND && "Unexpected opcode"); 490 const APInt &Val = cast<ConstantSDNode>(N->getOperand(1))->getAPIntValue(); 491 492 if (Val.countr_one() >= Width) 493 return true; 494 495 APInt Mask = Val | CurDAG->computeKnownBits(N->getOperand(0)).Zero; 496 return Mask.countr_one() >= Width; 497 } 498 499 /// Return an SDNode that returns the value of the global base register. 500 /// Output instructions required to initialize the global base register, 501 /// if necessary. 502 SDNode *getGlobalBaseReg(); 503 504 /// Return a reference to the TargetMachine, casted to the target-specific 505 /// type. 506 const X86TargetMachine &getTargetMachine() const { 507 return static_cast<const X86TargetMachine &>(TM); 508 } 509 510 /// Return a reference to the TargetInstrInfo, casted to the target-specific 511 /// type. 512 const X86InstrInfo *getInstrInfo() const { 513 return Subtarget->getInstrInfo(); 514 } 515 516 /// Return a condition code of the given SDNode 517 X86::CondCode getCondFromNode(SDNode *N) const; 518 519 /// Address-mode matching performs shift-of-and to and-of-shift 520 /// reassociation in order to expose more scaled addressing 521 /// opportunities. 522 bool ComplexPatternFuncMutatesDAG() const override { 523 return true; 524 } 525 526 bool isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const; 527 528 // Indicates we should prefer to use a non-temporal load for this load. 529 bool useNonTemporalLoad(LoadSDNode *N) const { 530 if (!N->isNonTemporal()) 531 return false; 532 533 unsigned StoreSize = N->getMemoryVT().getStoreSize(); 534 535 if (N->getAlign().value() < StoreSize) 536 return false; 537 538 switch (StoreSize) { 539 default: llvm_unreachable("Unsupported store size"); 540 case 4: 541 case 8: 542 return false; 543 case 16: 544 return Subtarget->hasSSE41(); 545 case 32: 546 return Subtarget->hasAVX2(); 547 case 64: 548 return Subtarget->hasAVX512(); 549 } 550 } 551 552 bool foldLoadStoreIntoMemOperand(SDNode *Node); 553 MachineSDNode *matchBEXTRFromAndImm(SDNode *Node); 554 bool matchBitExtract(SDNode *Node); 555 bool shrinkAndImmediate(SDNode *N); 556 bool isMaskZeroExtended(SDNode *N) const; 557 bool tryShiftAmountMod(SDNode *N); 558 bool tryShrinkShlLogicImm(SDNode *N); 559 bool tryVPTERNLOG(SDNode *N); 560 bool matchVPTERNLOG(SDNode *Root, SDNode *ParentA, SDNode *ParentB, 561 SDNode *ParentC, SDValue A, SDValue B, SDValue C, 562 uint8_t Imm); 563 bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask); 564 bool tryMatchBitSelect(SDNode *N); 565 566 MachineSDNode *emitPCMPISTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad, 567 const SDLoc &dl, MVT VT, SDNode *Node); 568 MachineSDNode *emitPCMPESTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad, 569 const SDLoc &dl, MVT VT, SDNode *Node, 570 SDValue &InGlue); 571 572 bool tryOptimizeRem8Extend(SDNode *N); 573 574 bool onlyUsesZeroFlag(SDValue Flags) const; 575 bool hasNoSignFlagUses(SDValue Flags) const; 576 bool hasNoCarryFlagUses(SDValue Flags) const; 577 }; 578 } 579 580 char X86DAGToDAGISel::ID = 0; 581 582 INITIALIZE_PASS(X86DAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false) 583 584 // Returns true if this masked compare can be implemented legally with this 585 // type. 586 static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget) { 587 unsigned Opcode = N->getOpcode(); 588 if (Opcode == X86ISD::CMPM || Opcode == X86ISD::CMPMM || 589 Opcode == X86ISD::STRICT_CMPM || Opcode == ISD::SETCC || 590 Opcode == X86ISD::CMPMM_SAE || Opcode == X86ISD::VFPCLASS) { 591 // We can get 256-bit 8 element types here without VLX being enabled. When 592 // this happens we will use 512-bit operations and the mask will not be 593 // zero extended. 594 EVT OpVT = N->getOperand(0).getValueType(); 595 // The first operand of X86ISD::STRICT_CMPM is chain, so we need to get the 596 // second operand. 597 if (Opcode == X86ISD::STRICT_CMPM) 598 OpVT = N->getOperand(1).getValueType(); 599 if (OpVT.is256BitVector() || OpVT.is128BitVector()) 600 return Subtarget->hasVLX(); 601 602 return true; 603 } 604 // Scalar opcodes use 128 bit registers, but aren't subject to the VLX check. 605 if (Opcode == X86ISD::VFPCLASSS || Opcode == X86ISD::FSETCCM || 606 Opcode == X86ISD::FSETCCM_SAE) 607 return true; 608 609 return false; 610 } 611 612 // Returns true if we can assume the writer of the mask has zero extended it 613 // for us. 614 bool X86DAGToDAGISel::isMaskZeroExtended(SDNode *N) const { 615 // If this is an AND, check if we have a compare on either side. As long as 616 // one side guarantees the mask is zero extended, the AND will preserve those 617 // zeros. 618 if (N->getOpcode() == ISD::AND) 619 return isLegalMaskCompare(N->getOperand(0).getNode(), Subtarget) || 620 isLegalMaskCompare(N->getOperand(1).getNode(), Subtarget); 621 622 return isLegalMaskCompare(N, Subtarget); 623 } 624 625 bool 626 X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const { 627 if (OptLevel == CodeGenOptLevel::None) 628 return false; 629 630 if (!N.hasOneUse()) 631 return false; 632 633 if (N.getOpcode() != ISD::LOAD) 634 return true; 635 636 // Don't fold non-temporal loads if we have an instruction for them. 637 if (useNonTemporalLoad(cast<LoadSDNode>(N))) 638 return false; 639 640 // If N is a load, do additional profitability checks. 641 if (U == Root) { 642 switch (U->getOpcode()) { 643 default: break; 644 case X86ISD::ADD: 645 case X86ISD::ADC: 646 case X86ISD::SUB: 647 case X86ISD::SBB: 648 case X86ISD::AND: 649 case X86ISD::XOR: 650 case X86ISD::OR: 651 case ISD::ADD: 652 case ISD::UADDO_CARRY: 653 case ISD::AND: 654 case ISD::OR: 655 case ISD::XOR: { 656 SDValue Op1 = U->getOperand(1); 657 658 // If the other operand is a 8-bit immediate we should fold the immediate 659 // instead. This reduces code size. 660 // e.g. 661 // movl 4(%esp), %eax 662 // addl $4, %eax 663 // vs. 664 // movl $4, %eax 665 // addl 4(%esp), %eax 666 // The former is 2 bytes shorter. In case where the increment is 1, then 667 // the saving can be 4 bytes (by using incl %eax). 668 if (auto *Imm = dyn_cast<ConstantSDNode>(Op1)) { 669 if (Imm->getAPIntValue().isSignedIntN(8)) 670 return false; 671 672 // If this is a 64-bit AND with an immediate that fits in 32-bits, 673 // prefer using the smaller and over folding the load. This is needed to 674 // make sure immediates created by shrinkAndImmediate are always folded. 675 // Ideally we would narrow the load during DAG combine and get the 676 // best of both worlds. 677 if (U->getOpcode() == ISD::AND && 678 Imm->getAPIntValue().getBitWidth() == 64 && 679 Imm->getAPIntValue().isIntN(32)) 680 return false; 681 682 // If this really a zext_inreg that can be represented with a movzx 683 // instruction, prefer that. 684 // TODO: We could shrink the load and fold if it is non-volatile. 685 if (U->getOpcode() == ISD::AND && 686 (Imm->getAPIntValue() == UINT8_MAX || 687 Imm->getAPIntValue() == UINT16_MAX || 688 Imm->getAPIntValue() == UINT32_MAX)) 689 return false; 690 691 // ADD/SUB with can negate the immediate and use the opposite operation 692 // to fit 128 into a sign extended 8 bit immediate. 693 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB) && 694 (-Imm->getAPIntValue()).isSignedIntN(8)) 695 return false; 696 697 if ((U->getOpcode() == X86ISD::ADD || U->getOpcode() == X86ISD::SUB) && 698 (-Imm->getAPIntValue()).isSignedIntN(8) && 699 hasNoCarryFlagUses(SDValue(U, 1))) 700 return false; 701 } 702 703 // If the other operand is a TLS address, we should fold it instead. 704 // This produces 705 // movl %gs:0, %eax 706 // leal i@NTPOFF(%eax), %eax 707 // instead of 708 // movl $i@NTPOFF, %eax 709 // addl %gs:0, %eax 710 // if the block also has an access to a second TLS address this will save 711 // a load. 712 // FIXME: This is probably also true for non-TLS addresses. 713 if (Op1.getOpcode() == X86ISD::Wrapper) { 714 SDValue Val = Op1.getOperand(0); 715 if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) 716 return false; 717 } 718 719 // Don't fold load if this matches the BTS/BTR/BTC patterns. 720 // BTS: (or X, (shl 1, n)) 721 // BTR: (and X, (rotl -2, n)) 722 // BTC: (xor X, (shl 1, n)) 723 if (U->getOpcode() == ISD::OR || U->getOpcode() == ISD::XOR) { 724 if (U->getOperand(0).getOpcode() == ISD::SHL && 725 isOneConstant(U->getOperand(0).getOperand(0))) 726 return false; 727 728 if (U->getOperand(1).getOpcode() == ISD::SHL && 729 isOneConstant(U->getOperand(1).getOperand(0))) 730 return false; 731 } 732 if (U->getOpcode() == ISD::AND) { 733 SDValue U0 = U->getOperand(0); 734 SDValue U1 = U->getOperand(1); 735 if (U0.getOpcode() == ISD::ROTL) { 736 auto *C = dyn_cast<ConstantSDNode>(U0.getOperand(0)); 737 if (C && C->getSExtValue() == -2) 738 return false; 739 } 740 741 if (U1.getOpcode() == ISD::ROTL) { 742 auto *C = dyn_cast<ConstantSDNode>(U1.getOperand(0)); 743 if (C && C->getSExtValue() == -2) 744 return false; 745 } 746 } 747 748 break; 749 } 750 case ISD::SHL: 751 case ISD::SRA: 752 case ISD::SRL: 753 // Don't fold a load into a shift by immediate. The BMI2 instructions 754 // support folding a load, but not an immediate. The legacy instructions 755 // support folding an immediate, but can't fold a load. Folding an 756 // immediate is preferable to folding a load. 757 if (isa<ConstantSDNode>(U->getOperand(1))) 758 return false; 759 760 break; 761 } 762 } 763 764 // Prevent folding a load if this can implemented with an insert_subreg or 765 // a move that implicitly zeroes. 766 if (Root->getOpcode() == ISD::INSERT_SUBVECTOR && 767 isNullConstant(Root->getOperand(2)) && 768 (Root->getOperand(0).isUndef() || 769 ISD::isBuildVectorAllZeros(Root->getOperand(0).getNode()))) 770 return false; 771 772 return true; 773 } 774 775 // Indicates it is profitable to form an AVX512 masked operation. Returning 776 // false will favor a masked register-register masked move or vblendm and the 777 // operation will be selected separately. 778 bool X86DAGToDAGISel::isProfitableToFormMaskedOp(SDNode *N) const { 779 assert( 780 (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::SELECTS) && 781 "Unexpected opcode!"); 782 783 // If the operation has additional users, the operation will be duplicated. 784 // Check the use count to prevent that. 785 // FIXME: Are there cheap opcodes we might want to duplicate? 786 return N->getOperand(1).hasOneUse(); 787 } 788 789 /// Replace the original chain operand of the call with 790 /// load's chain operand and move load below the call's chain operand. 791 static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, 792 SDValue Call, SDValue OrigChain) { 793 SmallVector<SDValue, 8> Ops; 794 SDValue Chain = OrigChain.getOperand(0); 795 if (Chain.getNode() == Load.getNode()) 796 Ops.push_back(Load.getOperand(0)); 797 else { 798 assert(Chain.getOpcode() == ISD::TokenFactor && 799 "Unexpected chain operand"); 800 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) 801 if (Chain.getOperand(i).getNode() == Load.getNode()) 802 Ops.push_back(Load.getOperand(0)); 803 else 804 Ops.push_back(Chain.getOperand(i)); 805 SDValue NewChain = 806 CurDAG->getNode(ISD::TokenFactor, SDLoc(Load), MVT::Other, Ops); 807 Ops.clear(); 808 Ops.push_back(NewChain); 809 } 810 Ops.append(OrigChain->op_begin() + 1, OrigChain->op_end()); 811 CurDAG->UpdateNodeOperands(OrigChain.getNode(), Ops); 812 CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0), 813 Load.getOperand(1), Load.getOperand(2)); 814 815 Ops.clear(); 816 Ops.push_back(SDValue(Load.getNode(), 1)); 817 Ops.append(Call->op_begin() + 1, Call->op_end()); 818 CurDAG->UpdateNodeOperands(Call.getNode(), Ops); 819 } 820 821 /// Return true if call address is a load and it can be 822 /// moved below CALLSEQ_START and the chains leading up to the call. 823 /// Return the CALLSEQ_START by reference as a second output. 824 /// In the case of a tail call, there isn't a callseq node between the call 825 /// chain and the load. 826 static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) { 827 // The transformation is somewhat dangerous if the call's chain was glued to 828 // the call. After MoveBelowOrigChain the load is moved between the call and 829 // the chain, this can create a cycle if the load is not folded. So it is 830 // *really* important that we are sure the load will be folded. 831 if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse()) 832 return false; 833 auto *LD = dyn_cast<LoadSDNode>(Callee.getNode()); 834 if (!LD || 835 !LD->isSimple() || 836 LD->getAddressingMode() != ISD::UNINDEXED || 837 LD->getExtensionType() != ISD::NON_EXTLOAD) 838 return false; 839 840 // Now let's find the callseq_start. 841 while (HasCallSeq && Chain.getOpcode() != ISD::CALLSEQ_START) { 842 if (!Chain.hasOneUse()) 843 return false; 844 Chain = Chain.getOperand(0); 845 } 846 847 if (!Chain.getNumOperands()) 848 return false; 849 // Since we are not checking for AA here, conservatively abort if the chain 850 // writes to memory. It's not safe to move the callee (a load) across a store. 851 if (isa<MemSDNode>(Chain.getNode()) && 852 cast<MemSDNode>(Chain.getNode())->writeMem()) 853 return false; 854 if (Chain.getOperand(0).getNode() == Callee.getNode()) 855 return true; 856 if (Chain.getOperand(0).getOpcode() == ISD::TokenFactor && 857 Callee.getValue(1).isOperandOf(Chain.getOperand(0).getNode()) && 858 Callee.getValue(1).hasOneUse()) 859 return true; 860 return false; 861 } 862 863 static bool isEndbrImm64(uint64_t Imm) { 864 // There may be some other prefix bytes between 0xF3 and 0x0F1EFA. 865 // i.g: 0xF3660F1EFA, 0xF3670F1EFA 866 if ((Imm & 0x00FFFFFF) != 0x0F1EFA) 867 return false; 868 869 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64, 870 0x65, 0x66, 0x67, 0xf0, 0xf2}; 871 int i = 24; // 24bit 0x0F1EFA has matched 872 while (i < 64) { 873 uint8_t Byte = (Imm >> i) & 0xFF; 874 if (Byte == 0xF3) 875 return true; 876 if (!llvm::is_contained(OptionalPrefixBytes, Byte)) 877 return false; 878 i += 8; 879 } 880 881 return false; 882 } 883 884 static bool needBWI(MVT VT) { 885 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8); 886 } 887 888 void X86DAGToDAGISel::PreprocessISelDAG() { 889 bool MadeChange = false; 890 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 891 E = CurDAG->allnodes_end(); I != E; ) { 892 SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues. 893 894 // This is for CET enhancement. 895 // 896 // ENDBR32 and ENDBR64 have specific opcodes: 897 // ENDBR32: F3 0F 1E FB 898 // ENDBR64: F3 0F 1E FA 899 // And we want that attackers won’t find unintended ENDBR32/64 900 // opcode matches in the binary 901 // Here’s an example: 902 // If the compiler had to generate asm for the following code: 903 // a = 0xF30F1EFA 904 // it could, for example, generate: 905 // mov 0xF30F1EFA, dword ptr[a] 906 // In such a case, the binary would include a gadget that starts 907 // with a fake ENDBR64 opcode. Therefore, we split such generation 908 // into multiple operations, let it not shows in the binary 909 if (N->getOpcode() == ISD::Constant) { 910 MVT VT = N->getSimpleValueType(0); 911 int64_t Imm = cast<ConstantSDNode>(N)->getSExtValue(); 912 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB; 913 if (Imm == EndbrImm || isEndbrImm64(Imm)) { 914 // Check that the cf-protection-branch is enabled. 915 Metadata *CFProtectionBranch = 916 MF->getMMI().getModule()->getModuleFlag("cf-protection-branch"); 917 if (CFProtectionBranch || IndirectBranchTracking) { 918 SDLoc dl(N); 919 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT, false, true); 920 Complement = CurDAG->getNOT(dl, Complement, VT); 921 --I; 922 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Complement); 923 ++I; 924 MadeChange = true; 925 continue; 926 } 927 } 928 } 929 930 // If this is a target specific AND node with no flag usages, turn it back 931 // into ISD::AND to enable test instruction matching. 932 if (N->getOpcode() == X86ISD::AND && !N->hasAnyUseOfValue(1)) { 933 SDValue Res = CurDAG->getNode(ISD::AND, SDLoc(N), N->getValueType(0), 934 N->getOperand(0), N->getOperand(1)); 935 --I; 936 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); 937 ++I; 938 MadeChange = true; 939 continue; 940 } 941 942 // Convert vector increment or decrement to sub/add with an all-ones 943 // constant: 944 // add X, <1, 1...> --> sub X, <-1, -1...> 945 // sub X, <1, 1...> --> add X, <-1, -1...> 946 // The all-ones vector constant can be materialized using a pcmpeq 947 // instruction that is commonly recognized as an idiom (has no register 948 // dependency), so that's better/smaller than loading a splat 1 constant. 949 // 950 // But don't do this if it would inhibit a potentially profitable load 951 // folding opportunity for the other operand. That only occurs with the 952 // intersection of: 953 // (1) The other operand (op0) is load foldable. 954 // (2) The op is an add (otherwise, we are *creating* an add and can still 955 // load fold the other op). 956 // (3) The target has AVX (otherwise, we have a destructive add and can't 957 // load fold the other op without killing the constant op). 958 // (4) The constant 1 vector has multiple uses (so it is profitable to load 959 // into a register anyway). 960 auto mayPreventLoadFold = [&]() { 961 return X86::mayFoldLoad(N->getOperand(0), *Subtarget) && 962 N->getOpcode() == ISD::ADD && Subtarget->hasAVX() && 963 !N->getOperand(1).hasOneUse(); 964 }; 965 if ((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && 966 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) { 967 APInt SplatVal; 968 if (X86::isConstantSplat(N->getOperand(1), SplatVal) && 969 SplatVal.isOne()) { 970 SDLoc DL(N); 971 972 MVT VT = N->getSimpleValueType(0); 973 unsigned NumElts = VT.getSizeInBits() / 32; 974 SDValue AllOnes = 975 CurDAG->getAllOnesConstant(DL, MVT::getVectorVT(MVT::i32, NumElts)); 976 AllOnes = CurDAG->getBitcast(VT, AllOnes); 977 978 unsigned NewOpcode = N->getOpcode() == ISD::ADD ? ISD::SUB : ISD::ADD; 979 SDValue Res = 980 CurDAG->getNode(NewOpcode, DL, VT, N->getOperand(0), AllOnes); 981 --I; 982 CurDAG->ReplaceAllUsesWith(N, Res.getNode()); 983 ++I; 984 MadeChange = true; 985 continue; 986 } 987 } 988 989 switch (N->getOpcode()) { 990 case X86ISD::VBROADCAST: { 991 MVT VT = N->getSimpleValueType(0); 992 // Emulate v32i16/v64i8 broadcast without BWI. 993 if (!Subtarget->hasBWI() && needBWI(VT)) { 994 MVT NarrowVT = VT.getHalfNumVectorElementsVT(); 995 SDLoc dl(N); 996 SDValue NarrowBCast = 997 CurDAG->getNode(X86ISD::VBROADCAST, dl, NarrowVT, N->getOperand(0)); 998 SDValue Res = 999 CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, CurDAG->getUNDEF(VT), 1000 NarrowBCast, CurDAG->getIntPtrConstant(0, dl)); 1001 unsigned Index = NarrowVT.getVectorMinNumElements(); 1002 Res = CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, NarrowBCast, 1003 CurDAG->getIntPtrConstant(Index, dl)); 1004 1005 --I; 1006 CurDAG->ReplaceAllUsesWith(N, Res.getNode()); 1007 ++I; 1008 MadeChange = true; 1009 continue; 1010 } 1011 1012 break; 1013 } 1014 case X86ISD::VBROADCAST_LOAD: { 1015 MVT VT = N->getSimpleValueType(0); 1016 // Emulate v32i16/v64i8 broadcast without BWI. 1017 if (!Subtarget->hasBWI() && needBWI(VT)) { 1018 MVT NarrowVT = VT.getHalfNumVectorElementsVT(); 1019 auto *MemNode = cast<MemSDNode>(N); 1020 SDLoc dl(N); 1021 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other); 1022 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()}; 1023 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode( 1024 X86ISD::VBROADCAST_LOAD, dl, VTs, Ops, MemNode->getMemoryVT(), 1025 MemNode->getMemOperand()); 1026 SDValue Res = 1027 CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, CurDAG->getUNDEF(VT), 1028 NarrowBCast, CurDAG->getIntPtrConstant(0, dl)); 1029 unsigned Index = NarrowVT.getVectorMinNumElements(); 1030 Res = CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, NarrowBCast, 1031 CurDAG->getIntPtrConstant(Index, dl)); 1032 1033 --I; 1034 SDValue To[] = {Res, NarrowBCast.getValue(1)}; 1035 CurDAG->ReplaceAllUsesWith(N, To); 1036 ++I; 1037 MadeChange = true; 1038 continue; 1039 } 1040 1041 break; 1042 } 1043 case ISD::LOAD: { 1044 // If this is a XMM/YMM load of the same lower bits as another YMM/ZMM 1045 // load, then just extract the lower subvector and avoid the second load. 1046 auto *Ld = cast<LoadSDNode>(N); 1047 MVT VT = N->getSimpleValueType(0); 1048 if (!ISD::isNormalLoad(Ld) || !Ld->isSimple() || 1049 !(VT.is128BitVector() || VT.is256BitVector())) 1050 break; 1051 1052 MVT MaxVT = VT; 1053 SDNode *MaxLd = nullptr; 1054 SDValue Ptr = Ld->getBasePtr(); 1055 SDValue Chain = Ld->getChain(); 1056 for (SDNode *User : Ptr->uses()) { 1057 auto *UserLd = dyn_cast<LoadSDNode>(User); 1058 MVT UserVT = User->getSimpleValueType(0); 1059 if (User != N && UserLd && ISD::isNormalLoad(User) && 1060 UserLd->getBasePtr() == Ptr && UserLd->getChain() == Chain && 1061 !User->hasAnyUseOfValue(1) && 1062 (UserVT.is256BitVector() || UserVT.is512BitVector()) && 1063 UserVT.getSizeInBits() > VT.getSizeInBits() && 1064 (!MaxLd || UserVT.getSizeInBits() > MaxVT.getSizeInBits())) { 1065 MaxLd = User; 1066 MaxVT = UserVT; 1067 } 1068 } 1069 if (MaxLd) { 1070 SDLoc dl(N); 1071 unsigned NumSubElts = VT.getSizeInBits() / MaxVT.getScalarSizeInBits(); 1072 MVT SubVT = MVT::getVectorVT(MaxVT.getScalarType(), NumSubElts); 1073 SDValue Extract = CurDAG->getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, 1074 SDValue(MaxLd, 0), 1075 CurDAG->getIntPtrConstant(0, dl)); 1076 SDValue Res = CurDAG->getBitcast(VT, Extract); 1077 1078 --I; 1079 SDValue To[] = {Res, SDValue(MaxLd, 1)}; 1080 CurDAG->ReplaceAllUsesWith(N, To); 1081 ++I; 1082 MadeChange = true; 1083 continue; 1084 } 1085 break; 1086 } 1087 case ISD::VSELECT: { 1088 // Replace VSELECT with non-mask conditions with with BLENDV/VPTERNLOG. 1089 EVT EleVT = N->getOperand(0).getValueType().getVectorElementType(); 1090 if (EleVT == MVT::i1) 1091 break; 1092 1093 assert(Subtarget->hasSSE41() && "Expected SSE4.1 support!"); 1094 assert(N->getValueType(0).getVectorElementType() != MVT::i16 && 1095 "We can't replace VSELECT with BLENDV in vXi16!"); 1096 SDValue R; 1097 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(N->getOperand(0)) == 1098 EleVT.getSizeInBits()) { 1099 R = CurDAG->getNode(X86ISD::VPTERNLOG, SDLoc(N), N->getValueType(0), 1100 N->getOperand(0), N->getOperand(1), N->getOperand(2), 1101 CurDAG->getTargetConstant(0xCA, SDLoc(N), MVT::i8)); 1102 } else { 1103 R = CurDAG->getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0), 1104 N->getOperand(0), N->getOperand(1), 1105 N->getOperand(2)); 1106 } 1107 --I; 1108 CurDAG->ReplaceAllUsesWith(N, R.getNode()); 1109 ++I; 1110 MadeChange = true; 1111 continue; 1112 } 1113 case ISD::FP_ROUND: 1114 case ISD::STRICT_FP_ROUND: 1115 case ISD::FP_TO_SINT: 1116 case ISD::FP_TO_UINT: 1117 case ISD::STRICT_FP_TO_SINT: 1118 case ISD::STRICT_FP_TO_UINT: { 1119 // Replace vector fp_to_s/uint with their X86 specific equivalent so we 1120 // don't need 2 sets of patterns. 1121 if (!N->getSimpleValueType(0).isVector()) 1122 break; 1123 1124 unsigned NewOpc; 1125 switch (N->getOpcode()) { 1126 default: llvm_unreachable("Unexpected opcode!"); 1127 case ISD::FP_ROUND: NewOpc = X86ISD::VFPROUND; break; 1128 case ISD::STRICT_FP_ROUND: NewOpc = X86ISD::STRICT_VFPROUND; break; 1129 case ISD::STRICT_FP_TO_SINT: NewOpc = X86ISD::STRICT_CVTTP2SI; break; 1130 case ISD::FP_TO_SINT: NewOpc = X86ISD::CVTTP2SI; break; 1131 case ISD::STRICT_FP_TO_UINT: NewOpc = X86ISD::STRICT_CVTTP2UI; break; 1132 case ISD::FP_TO_UINT: NewOpc = X86ISD::CVTTP2UI; break; 1133 } 1134 SDValue Res; 1135 if (N->isStrictFPOpcode()) 1136 Res = 1137 CurDAG->getNode(NewOpc, SDLoc(N), {N->getValueType(0), MVT::Other}, 1138 {N->getOperand(0), N->getOperand(1)}); 1139 else 1140 Res = 1141 CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0), 1142 N->getOperand(0)); 1143 --I; 1144 CurDAG->ReplaceAllUsesWith(N, Res.getNode()); 1145 ++I; 1146 MadeChange = true; 1147 continue; 1148 } 1149 case ISD::SHL: 1150 case ISD::SRA: 1151 case ISD::SRL: { 1152 // Replace vector shifts with their X86 specific equivalent so we don't 1153 // need 2 sets of patterns. 1154 if (!N->getValueType(0).isVector()) 1155 break; 1156 1157 unsigned NewOpc; 1158 switch (N->getOpcode()) { 1159 default: llvm_unreachable("Unexpected opcode!"); 1160 case ISD::SHL: NewOpc = X86ISD::VSHLV; break; 1161 case ISD::SRA: NewOpc = X86ISD::VSRAV; break; 1162 case ISD::SRL: NewOpc = X86ISD::VSRLV; break; 1163 } 1164 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0), 1165 N->getOperand(0), N->getOperand(1)); 1166 --I; 1167 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); 1168 ++I; 1169 MadeChange = true; 1170 continue; 1171 } 1172 case ISD::ANY_EXTEND: 1173 case ISD::ANY_EXTEND_VECTOR_INREG: { 1174 // Replace vector any extend with the zero extend equivalents so we don't 1175 // need 2 sets of patterns. Ignore vXi1 extensions. 1176 if (!N->getValueType(0).isVector()) 1177 break; 1178 1179 unsigned NewOpc; 1180 if (N->getOperand(0).getScalarValueSizeInBits() == 1) { 1181 assert(N->getOpcode() == ISD::ANY_EXTEND && 1182 "Unexpected opcode for mask vector!"); 1183 NewOpc = ISD::SIGN_EXTEND; 1184 } else { 1185 NewOpc = N->getOpcode() == ISD::ANY_EXTEND 1186 ? ISD::ZERO_EXTEND 1187 : ISD::ZERO_EXTEND_VECTOR_INREG; 1188 } 1189 1190 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0), 1191 N->getOperand(0)); 1192 --I; 1193 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); 1194 ++I; 1195 MadeChange = true; 1196 continue; 1197 } 1198 case ISD::FCEIL: 1199 case ISD::STRICT_FCEIL: 1200 case ISD::FFLOOR: 1201 case ISD::STRICT_FFLOOR: 1202 case ISD::FTRUNC: 1203 case ISD::STRICT_FTRUNC: 1204 case ISD::FROUNDEVEN: 1205 case ISD::STRICT_FROUNDEVEN: 1206 case ISD::FNEARBYINT: 1207 case ISD::STRICT_FNEARBYINT: 1208 case ISD::FRINT: 1209 case ISD::STRICT_FRINT: { 1210 // Replace fp rounding with their X86 specific equivalent so we don't 1211 // need 2 sets of patterns. 1212 unsigned Imm; 1213 switch (N->getOpcode()) { 1214 default: llvm_unreachable("Unexpected opcode!"); 1215 case ISD::STRICT_FCEIL: 1216 case ISD::FCEIL: Imm = 0xA; break; 1217 case ISD::STRICT_FFLOOR: 1218 case ISD::FFLOOR: Imm = 0x9; break; 1219 case ISD::STRICT_FTRUNC: 1220 case ISD::FTRUNC: Imm = 0xB; break; 1221 case ISD::STRICT_FROUNDEVEN: 1222 case ISD::FROUNDEVEN: Imm = 0x8; break; 1223 case ISD::STRICT_FNEARBYINT: 1224 case ISD::FNEARBYINT: Imm = 0xC; break; 1225 case ISD::STRICT_FRINT: 1226 case ISD::FRINT: Imm = 0x4; break; 1227 } 1228 SDLoc dl(N); 1229 bool IsStrict = N->isStrictFPOpcode(); 1230 SDValue Res; 1231 if (IsStrict) 1232 Res = CurDAG->getNode(X86ISD::STRICT_VRNDSCALE, dl, 1233 {N->getValueType(0), MVT::Other}, 1234 {N->getOperand(0), N->getOperand(1), 1235 CurDAG->getTargetConstant(Imm, dl, MVT::i32)}); 1236 else 1237 Res = CurDAG->getNode(X86ISD::VRNDSCALE, dl, N->getValueType(0), 1238 N->getOperand(0), 1239 CurDAG->getTargetConstant(Imm, dl, MVT::i32)); 1240 --I; 1241 CurDAG->ReplaceAllUsesWith(N, Res.getNode()); 1242 ++I; 1243 MadeChange = true; 1244 continue; 1245 } 1246 case X86ISD::FANDN: 1247 case X86ISD::FAND: 1248 case X86ISD::FOR: 1249 case X86ISD::FXOR: { 1250 // Widen scalar fp logic ops to vector to reduce isel patterns. 1251 // FIXME: Can we do this during lowering/combine. 1252 MVT VT = N->getSimpleValueType(0); 1253 if (VT.isVector() || VT == MVT::f128) 1254 break; 1255 1256 MVT VecVT = VT == MVT::f64 ? MVT::v2f64 1257 : VT == MVT::f32 ? MVT::v4f32 1258 : MVT::v8f16; 1259 1260 SDLoc dl(N); 1261 SDValue Op0 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, 1262 N->getOperand(0)); 1263 SDValue Op1 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, 1264 N->getOperand(1)); 1265 1266 SDValue Res; 1267 if (Subtarget->hasSSE2()) { 1268 EVT IntVT = EVT(VecVT).changeVectorElementTypeToInteger(); 1269 Op0 = CurDAG->getNode(ISD::BITCAST, dl, IntVT, Op0); 1270 Op1 = CurDAG->getNode(ISD::BITCAST, dl, IntVT, Op1); 1271 unsigned Opc; 1272 switch (N->getOpcode()) { 1273 default: llvm_unreachable("Unexpected opcode!"); 1274 case X86ISD::FANDN: Opc = X86ISD::ANDNP; break; 1275 case X86ISD::FAND: Opc = ISD::AND; break; 1276 case X86ISD::FOR: Opc = ISD::OR; break; 1277 case X86ISD::FXOR: Opc = ISD::XOR; break; 1278 } 1279 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1); 1280 Res = CurDAG->getNode(ISD::BITCAST, dl, VecVT, Res); 1281 } else { 1282 Res = CurDAG->getNode(N->getOpcode(), dl, VecVT, Op0, Op1); 1283 } 1284 Res = CurDAG->getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, 1285 CurDAG->getIntPtrConstant(0, dl)); 1286 --I; 1287 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); 1288 ++I; 1289 MadeChange = true; 1290 continue; 1291 } 1292 } 1293 1294 if (OptLevel != CodeGenOptLevel::None && 1295 // Only do this when the target can fold the load into the call or 1296 // jmp. 1297 !Subtarget->useIndirectThunkCalls() && 1298 ((N->getOpcode() == X86ISD::CALL && !Subtarget->slowTwoMemOps()) || 1299 (N->getOpcode() == X86ISD::TC_RETURN && 1300 (Subtarget->is64Bit() || 1301 !getTargetMachine().isPositionIndependent())))) { 1302 /// Also try moving call address load from outside callseq_start to just 1303 /// before the call to allow it to be folded. 1304 /// 1305 /// [Load chain] 1306 /// ^ 1307 /// | 1308 /// [Load] 1309 /// ^ ^ 1310 /// | | 1311 /// / \-- 1312 /// / | 1313 ///[CALLSEQ_START] | 1314 /// ^ | 1315 /// | | 1316 /// [LOAD/C2Reg] | 1317 /// | | 1318 /// \ / 1319 /// \ / 1320 /// [CALL] 1321 bool HasCallSeq = N->getOpcode() == X86ISD::CALL; 1322 SDValue Chain = N->getOperand(0); 1323 SDValue Load = N->getOperand(1); 1324 if (!isCalleeLoad(Load, Chain, HasCallSeq)) 1325 continue; 1326 moveBelowOrigChain(CurDAG, Load, SDValue(N, 0), Chain); 1327 ++NumLoadMoved; 1328 MadeChange = true; 1329 continue; 1330 } 1331 1332 // Lower fpround and fpextend nodes that target the FP stack to be store and 1333 // load to the stack. This is a gross hack. We would like to simply mark 1334 // these as being illegal, but when we do that, legalize produces these when 1335 // it expands calls, then expands these in the same legalize pass. We would 1336 // like dag combine to be able to hack on these between the call expansion 1337 // and the node legalization. As such this pass basically does "really 1338 // late" legalization of these inline with the X86 isel pass. 1339 // FIXME: This should only happen when not compiled with -O0. 1340 switch (N->getOpcode()) { 1341 default: continue; 1342 case ISD::FP_ROUND: 1343 case ISD::FP_EXTEND: 1344 { 1345 MVT SrcVT = N->getOperand(0).getSimpleValueType(); 1346 MVT DstVT = N->getSimpleValueType(0); 1347 1348 // If any of the sources are vectors, no fp stack involved. 1349 if (SrcVT.isVector() || DstVT.isVector()) 1350 continue; 1351 1352 // If the source and destination are SSE registers, then this is a legal 1353 // conversion that should not be lowered. 1354 const X86TargetLowering *X86Lowering = 1355 static_cast<const X86TargetLowering *>(TLI); 1356 bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(SrcVT); 1357 bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(DstVT); 1358 if (SrcIsSSE && DstIsSSE) 1359 continue; 1360 1361 if (!SrcIsSSE && !DstIsSSE) { 1362 // If this is an FPStack extension, it is a noop. 1363 if (N->getOpcode() == ISD::FP_EXTEND) 1364 continue; 1365 // If this is a value-preserving FPStack truncation, it is a noop. 1366 if (N->getConstantOperandVal(1)) 1367 continue; 1368 } 1369 1370 // Here we could have an FP stack truncation or an FPStack <-> SSE convert. 1371 // FPStack has extload and truncstore. SSE can fold direct loads into other 1372 // operations. Based on this, decide what we want to do. 1373 MVT MemVT = (N->getOpcode() == ISD::FP_ROUND) ? DstVT : SrcVT; 1374 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); 1375 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex(); 1376 MachinePointerInfo MPI = 1377 MachinePointerInfo::getFixedStack(CurDAG->getMachineFunction(), SPFI); 1378 SDLoc dl(N); 1379 1380 // FIXME: optimize the case where the src/dest is a load or store? 1381 1382 SDValue Store = CurDAG->getTruncStore( 1383 CurDAG->getEntryNode(), dl, N->getOperand(0), MemTmp, MPI, MemVT); 1384 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, 1385 MemTmp, MPI, MemVT); 1386 1387 // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the 1388 // extload we created. This will cause general havok on the dag because 1389 // anything below the conversion could be folded into other existing nodes. 1390 // To avoid invalidating 'I', back it up to the convert node. 1391 --I; 1392 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 1393 break; 1394 } 1395 1396 //The sequence of events for lowering STRICT_FP versions of these nodes requires 1397 //dealing with the chain differently, as there is already a preexisting chain. 1398 case ISD::STRICT_FP_ROUND: 1399 case ISD::STRICT_FP_EXTEND: 1400 { 1401 MVT SrcVT = N->getOperand(1).getSimpleValueType(); 1402 MVT DstVT = N->getSimpleValueType(0); 1403 1404 // If any of the sources are vectors, no fp stack involved. 1405 if (SrcVT.isVector() || DstVT.isVector()) 1406 continue; 1407 1408 // If the source and destination are SSE registers, then this is a legal 1409 // conversion that should not be lowered. 1410 const X86TargetLowering *X86Lowering = 1411 static_cast<const X86TargetLowering *>(TLI); 1412 bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(SrcVT); 1413 bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(DstVT); 1414 if (SrcIsSSE && DstIsSSE) 1415 continue; 1416 1417 if (!SrcIsSSE && !DstIsSSE) { 1418 // If this is an FPStack extension, it is a noop. 1419 if (N->getOpcode() == ISD::STRICT_FP_EXTEND) 1420 continue; 1421 // If this is a value-preserving FPStack truncation, it is a noop. 1422 if (N->getConstantOperandVal(2)) 1423 continue; 1424 } 1425 1426 // Here we could have an FP stack truncation or an FPStack <-> SSE convert. 1427 // FPStack has extload and truncstore. SSE can fold direct loads into other 1428 // operations. Based on this, decide what we want to do. 1429 MVT MemVT = (N->getOpcode() == ISD::STRICT_FP_ROUND) ? DstVT : SrcVT; 1430 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); 1431 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex(); 1432 MachinePointerInfo MPI = 1433 MachinePointerInfo::getFixedStack(CurDAG->getMachineFunction(), SPFI); 1434 SDLoc dl(N); 1435 1436 // FIXME: optimize the case where the src/dest is a load or store? 1437 1438 //Since the operation is StrictFP, use the preexisting chain. 1439 SDValue Store, Result; 1440 if (!SrcIsSSE) { 1441 SDVTList VTs = CurDAG->getVTList(MVT::Other); 1442 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), MemTmp}; 1443 Store = CurDAG->getMemIntrinsicNode(X86ISD::FST, dl, VTs, Ops, MemVT, 1444 MPI, /*Align*/ std::nullopt, 1445 MachineMemOperand::MOStore); 1446 if (N->getFlags().hasNoFPExcept()) { 1447 SDNodeFlags Flags = Store->getFlags(); 1448 Flags.setNoFPExcept(true); 1449 Store->setFlags(Flags); 1450 } 1451 } else { 1452 assert(SrcVT == MemVT && "Unexpected VT!"); 1453 Store = CurDAG->getStore(N->getOperand(0), dl, N->getOperand(1), MemTmp, 1454 MPI); 1455 } 1456 1457 if (!DstIsSSE) { 1458 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other); 1459 SDValue Ops[] = {Store, MemTmp}; 1460 Result = CurDAG->getMemIntrinsicNode( 1461 X86ISD::FLD, dl, VTs, Ops, MemVT, MPI, 1462 /*Align*/ std::nullopt, MachineMemOperand::MOLoad); 1463 if (N->getFlags().hasNoFPExcept()) { 1464 SDNodeFlags Flags = Result->getFlags(); 1465 Flags.setNoFPExcept(true); 1466 Result->setFlags(Flags); 1467 } 1468 } else { 1469 assert(DstVT == MemVT && "Unexpected VT!"); 1470 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI); 1471 } 1472 1473 // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the 1474 // extload we created. This will cause general havok on the dag because 1475 // anything below the conversion could be folded into other existing nodes. 1476 // To avoid invalidating 'I', back it up to the convert node. 1477 --I; 1478 CurDAG->ReplaceAllUsesWith(N, Result.getNode()); 1479 break; 1480 } 1481 } 1482 1483 1484 // Now that we did that, the node is dead. Increment the iterator to the 1485 // next node to process, then delete N. 1486 ++I; 1487 MadeChange = true; 1488 } 1489 1490 // Remove any dead nodes that may have been left behind. 1491 if (MadeChange) 1492 CurDAG->RemoveDeadNodes(); 1493 } 1494 1495 // Look for a redundant movzx/movsx that can occur after an 8-bit divrem. 1496 bool X86DAGToDAGISel::tryOptimizeRem8Extend(SDNode *N) { 1497 unsigned Opc = N->getMachineOpcode(); 1498 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 && 1499 Opc != X86::MOVSX64rr8) 1500 return false; 1501 1502 SDValue N0 = N->getOperand(0); 1503 1504 // We need to be extracting the lower bit of an extend. 1505 if (!N0.isMachineOpcode() || 1506 N0.getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG || 1507 N0.getConstantOperandVal(1) != X86::sub_8bit) 1508 return false; 1509 1510 // We're looking for either a movsx or movzx to match the original opcode. 1511 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX 1512 : X86::MOVSX32rr8_NOREX; 1513 SDValue N00 = N0.getOperand(0); 1514 if (!N00.isMachineOpcode() || N00.getMachineOpcode() != ExpectedOpc) 1515 return false; 1516 1517 if (Opc == X86::MOVSX64rr8) { 1518 // If we had a sign extend from 8 to 64 bits. We still need to go from 32 1519 // to 64. 1520 MachineSDNode *Extend = CurDAG->getMachineNode(X86::MOVSX64rr32, SDLoc(N), 1521 MVT::i64, N00); 1522 ReplaceUses(N, Extend); 1523 } else { 1524 // Ok we can drop this extend and just use the original extend. 1525 ReplaceUses(N, N00.getNode()); 1526 } 1527 1528 return true; 1529 } 1530 1531 void X86DAGToDAGISel::PostprocessISelDAG() { 1532 // Skip peepholes at -O0. 1533 if (TM.getOptLevel() == CodeGenOptLevel::None) 1534 return; 1535 1536 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 1537 1538 bool MadeChange = false; 1539 while (Position != CurDAG->allnodes_begin()) { 1540 SDNode *N = &*--Position; 1541 // Skip dead nodes and any non-machine opcodes. 1542 if (N->use_empty() || !N->isMachineOpcode()) 1543 continue; 1544 1545 if (tryOptimizeRem8Extend(N)) { 1546 MadeChange = true; 1547 continue; 1548 } 1549 1550 // Look for a TESTrr+ANDrr pattern where both operands of the test are 1551 // the same. Rewrite to remove the AND. 1552 unsigned Opc = N->getMachineOpcode(); 1553 if ((Opc == X86::TEST8rr || Opc == X86::TEST16rr || 1554 Opc == X86::TEST32rr || Opc == X86::TEST64rr) && 1555 N->getOperand(0) == N->getOperand(1) && 1556 N->getOperand(0)->hasNUsesOfValue(2, N->getOperand(0).getResNo()) && 1557 N->getOperand(0).isMachineOpcode()) { 1558 SDValue And = N->getOperand(0); 1559 unsigned N0Opc = And.getMachineOpcode(); 1560 if ((N0Opc == X86::AND8rr || N0Opc == X86::AND16rr || 1561 N0Opc == X86::AND32rr || N0Opc == X86::AND64rr) && 1562 !And->hasAnyUseOfValue(1)) { 1563 MachineSDNode *Test = CurDAG->getMachineNode(Opc, SDLoc(N), 1564 MVT::i32, 1565 And.getOperand(0), 1566 And.getOperand(1)); 1567 ReplaceUses(N, Test); 1568 MadeChange = true; 1569 continue; 1570 } 1571 if ((N0Opc == X86::AND8rm || N0Opc == X86::AND16rm || 1572 N0Opc == X86::AND32rm || N0Opc == X86::AND64rm) && 1573 !And->hasAnyUseOfValue(1)) { 1574 unsigned NewOpc; 1575 switch (N0Opc) { 1576 case X86::AND8rm: NewOpc = X86::TEST8mr; break; 1577 case X86::AND16rm: NewOpc = X86::TEST16mr; break; 1578 case X86::AND32rm: NewOpc = X86::TEST32mr; break; 1579 case X86::AND64rm: NewOpc = X86::TEST64mr; break; 1580 } 1581 1582 // Need to swap the memory and register operand. 1583 SDValue Ops[] = { And.getOperand(1), 1584 And.getOperand(2), 1585 And.getOperand(3), 1586 And.getOperand(4), 1587 And.getOperand(5), 1588 And.getOperand(0), 1589 And.getOperand(6) /* Chain */ }; 1590 MachineSDNode *Test = CurDAG->getMachineNode(NewOpc, SDLoc(N), 1591 MVT::i32, MVT::Other, Ops); 1592 CurDAG->setNodeMemRefs( 1593 Test, cast<MachineSDNode>(And.getNode())->memoperands()); 1594 ReplaceUses(And.getValue(2), SDValue(Test, 1)); 1595 ReplaceUses(SDValue(N, 0), SDValue(Test, 0)); 1596 MadeChange = true; 1597 continue; 1598 } 1599 } 1600 1601 // Look for a KAND+KORTEST and turn it into KTEST if only the zero flag is 1602 // used. We're doing this late so we can prefer to fold the AND into masked 1603 // comparisons. Doing that can be better for the live range of the mask 1604 // register. 1605 if ((Opc == X86::KORTESTBrr || Opc == X86::KORTESTWrr || 1606 Opc == X86::KORTESTDrr || Opc == X86::KORTESTQrr) && 1607 N->getOperand(0) == N->getOperand(1) && 1608 N->isOnlyUserOf(N->getOperand(0).getNode()) && 1609 N->getOperand(0).isMachineOpcode() && 1610 onlyUsesZeroFlag(SDValue(N, 0))) { 1611 SDValue And = N->getOperand(0); 1612 unsigned N0Opc = And.getMachineOpcode(); 1613 // KANDW is legal with AVX512F, but KTESTW requires AVX512DQ. The other 1614 // KAND instructions and KTEST use the same ISA feature. 1615 if (N0Opc == X86::KANDBrr || 1616 (N0Opc == X86::KANDWrr && Subtarget->hasDQI()) || 1617 N0Opc == X86::KANDDrr || N0Opc == X86::KANDQrr) { 1618 unsigned NewOpc; 1619 switch (Opc) { 1620 default: llvm_unreachable("Unexpected opcode!"); 1621 case X86::KORTESTBrr: NewOpc = X86::KTESTBrr; break; 1622 case X86::KORTESTWrr: NewOpc = X86::KTESTWrr; break; 1623 case X86::KORTESTDrr: NewOpc = X86::KTESTDrr; break; 1624 case X86::KORTESTQrr: NewOpc = X86::KTESTQrr; break; 1625 } 1626 MachineSDNode *KTest = CurDAG->getMachineNode(NewOpc, SDLoc(N), 1627 MVT::i32, 1628 And.getOperand(0), 1629 And.getOperand(1)); 1630 ReplaceUses(N, KTest); 1631 MadeChange = true; 1632 continue; 1633 } 1634 } 1635 1636 // Attempt to remove vectors moves that were inserted to zero upper bits. 1637 if (Opc != TargetOpcode::SUBREG_TO_REG) 1638 continue; 1639 1640 unsigned SubRegIdx = N->getConstantOperandVal(2); 1641 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm) 1642 continue; 1643 1644 SDValue Move = N->getOperand(1); 1645 if (!Move.isMachineOpcode()) 1646 continue; 1647 1648 // Make sure its one of the move opcodes we recognize. 1649 switch (Move.getMachineOpcode()) { 1650 default: 1651 continue; 1652 case X86::VMOVAPDrr: case X86::VMOVUPDrr: 1653 case X86::VMOVAPSrr: case X86::VMOVUPSrr: 1654 case X86::VMOVDQArr: case X86::VMOVDQUrr: 1655 case X86::VMOVAPDYrr: case X86::VMOVUPDYrr: 1656 case X86::VMOVAPSYrr: case X86::VMOVUPSYrr: 1657 case X86::VMOVDQAYrr: case X86::VMOVDQUYrr: 1658 case X86::VMOVAPDZ128rr: case X86::VMOVUPDZ128rr: 1659 case X86::VMOVAPSZ128rr: case X86::VMOVUPSZ128rr: 1660 case X86::VMOVDQA32Z128rr: case X86::VMOVDQU32Z128rr: 1661 case X86::VMOVDQA64Z128rr: case X86::VMOVDQU64Z128rr: 1662 case X86::VMOVAPDZ256rr: case X86::VMOVUPDZ256rr: 1663 case X86::VMOVAPSZ256rr: case X86::VMOVUPSZ256rr: 1664 case X86::VMOVDQA32Z256rr: case X86::VMOVDQU32Z256rr: 1665 case X86::VMOVDQA64Z256rr: case X86::VMOVDQU64Z256rr: 1666 break; 1667 } 1668 1669 SDValue In = Move.getOperand(0); 1670 if (!In.isMachineOpcode() || 1671 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END) 1672 continue; 1673 1674 // Make sure the instruction has a VEX, XOP, or EVEX prefix. This covers 1675 // the SHA instructions which use a legacy encoding. 1676 uint64_t TSFlags = getInstrInfo()->get(In.getMachineOpcode()).TSFlags; 1677 if ((TSFlags & X86II::EncodingMask) != X86II::VEX && 1678 (TSFlags & X86II::EncodingMask) != X86II::EVEX && 1679 (TSFlags & X86II::EncodingMask) != X86II::XOP) 1680 continue; 1681 1682 // Producing instruction is another vector instruction. We can drop the 1683 // move. 1684 CurDAG->UpdateNodeOperands(N, N->getOperand(0), In, N->getOperand(2)); 1685 MadeChange = true; 1686 } 1687 1688 if (MadeChange) 1689 CurDAG->RemoveDeadNodes(); 1690 } 1691 1692 1693 /// Emit any code that needs to be executed only in the main function. 1694 void X86DAGToDAGISel::emitSpecialCodeForMain() { 1695 if (Subtarget->isTargetCygMing()) { 1696 TargetLowering::ArgListTy Args; 1697 auto &DL = CurDAG->getDataLayout(); 1698 1699 TargetLowering::CallLoweringInfo CLI(*CurDAG); 1700 CLI.setChain(CurDAG->getRoot()) 1701 .setCallee(CallingConv::C, Type::getVoidTy(*CurDAG->getContext()), 1702 CurDAG->getExternalSymbol("__main", TLI->getPointerTy(DL)), 1703 std::move(Args)); 1704 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); 1705 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); 1706 CurDAG->setRoot(Result.second); 1707 } 1708 } 1709 1710 void X86DAGToDAGISel::emitFunctionEntryCode() { 1711 // If this is main, emit special code for main. 1712 const Function &F = MF->getFunction(); 1713 if (F.hasExternalLinkage() && F.getName() == "main") 1714 emitSpecialCodeForMain(); 1715 } 1716 1717 static bool isDispSafeForFrameIndex(int64_t Val) { 1718 // On 64-bit platforms, we can run into an issue where a frame index 1719 // includes a displacement that, when added to the explicit displacement, 1720 // will overflow the displacement field. Assuming that the frame index 1721 // displacement fits into a 31-bit integer (which is only slightly more 1722 // aggressive than the current fundamental assumption that it fits into 1723 // a 32-bit integer), a 31-bit disp should always be safe. 1724 return isInt<31>(Val); 1725 } 1726 1727 bool X86DAGToDAGISel::foldOffsetIntoAddress(uint64_t Offset, 1728 X86ISelAddressMode &AM) { 1729 // We may have already matched a displacement and the caller just added the 1730 // symbolic displacement. So we still need to do the checks even if Offset 1731 // is zero. 1732 1733 int64_t Val = AM.Disp + Offset; 1734 1735 // Cannot combine ExternalSymbol displacements with integer offsets. 1736 if (Val != 0 && (AM.ES || AM.MCSym)) 1737 return true; 1738 1739 CodeModel::Model M = TM.getCodeModel(); 1740 if (Subtarget->is64Bit()) { 1741 if (Val != 0 && 1742 !X86::isOffsetSuitableForCodeModel(Val, M, 1743 AM.hasSymbolicDisplacement())) 1744 return true; 1745 // In addition to the checks required for a register base, check that 1746 // we do not try to use an unsafe Disp with a frame index. 1747 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase && 1748 !isDispSafeForFrameIndex(Val)) 1749 return true; 1750 // In ILP32 (x32) mode, pointers are 32 bits and need to be zero-extended to 1751 // 64 bits. Instructions with 32-bit register addresses perform this zero 1752 // extension for us and we can safely ignore the high bits of Offset. 1753 // Instructions with only a 32-bit immediate address do not, though: they 1754 // sign extend instead. This means only address the low 2GB of address space 1755 // is directly addressable, we need indirect addressing for the high 2GB of 1756 // address space. 1757 // TODO: Some of the earlier checks may be relaxed for ILP32 mode as the 1758 // implicit zero extension of instructions would cover up any problem. 1759 // However, we have asserts elsewhere that get triggered if we do, so keep 1760 // the checks for now. 1761 // TODO: We would actually be able to accept these, as well as the same 1762 // addresses in LP64 mode, by adding the EIZ pseudo-register as an operand 1763 // to get an address size override to be emitted. However, this 1764 // pseudo-register is not part of any register class and therefore causes 1765 // MIR verification to fail. 1766 if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) && 1767 !AM.hasBaseOrIndexReg()) 1768 return true; 1769 } 1770 AM.Disp = Val; 1771 return false; 1772 } 1773 1774 bool X86DAGToDAGISel::matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM, 1775 bool AllowSegmentRegForX32) { 1776 SDValue Address = N->getOperand(1); 1777 1778 // load gs:0 -> GS segment register. 1779 // load fs:0 -> FS segment register. 1780 // 1781 // This optimization is generally valid because the GNU TLS model defines that 1782 // gs:0 (or fs:0 on X86-64) contains its own address. However, for X86-64 mode 1783 // with 32-bit registers, as we get in ILP32 mode, those registers are first 1784 // zero-extended to 64 bits and then added it to the base address, which gives 1785 // unwanted results when the register holds a negative value. 1786 // For more information see http://people.redhat.com/drepper/tls.pdf 1787 if (isNullConstant(Address) && AM.Segment.getNode() == nullptr && 1788 !IndirectTlsSegRefs && 1789 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() || 1790 Subtarget->isTargetFuchsia())) { 1791 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32) 1792 return true; 1793 switch (N->getPointerInfo().getAddrSpace()) { 1794 case X86AS::GS: 1795 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 1796 return false; 1797 case X86AS::FS: 1798 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 1799 return false; 1800 // Address space X86AS::SS is not handled here, because it is not used to 1801 // address TLS areas. 1802 } 1803 } 1804 1805 return true; 1806 } 1807 1808 /// Try to match X86ISD::Wrapper and X86ISD::WrapperRIP nodes into an addressing 1809 /// mode. These wrap things that will resolve down into a symbol reference. 1810 /// If no match is possible, this returns true, otherwise it returns false. 1811 bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) { 1812 // If the addressing mode already has a symbol as the displacement, we can 1813 // never match another symbol. 1814 if (AM.hasSymbolicDisplacement()) 1815 return true; 1816 1817 bool IsRIPRelTLS = false; 1818 bool IsRIPRel = N.getOpcode() == X86ISD::WrapperRIP; 1819 if (IsRIPRel) { 1820 SDValue Val = N.getOperand(0); 1821 if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) 1822 IsRIPRelTLS = true; 1823 } 1824 1825 // We can't use an addressing mode in the 64-bit large code model. 1826 // Global TLS addressing is an exception. In the medium code model, 1827 // we use can use a mode when RIP wrappers are present. 1828 // That signifies access to globals that are known to be "near", 1829 // such as the GOT itself. 1830 CodeModel::Model M = TM.getCodeModel(); 1831 if (Subtarget->is64Bit() && 1832 ((M == CodeModel::Large && !IsRIPRelTLS) || 1833 (M == CodeModel::Medium && !IsRIPRel))) 1834 return true; 1835 1836 // Base and index reg must be 0 in order to use %rip as base. 1837 if (IsRIPRel && AM.hasBaseOrIndexReg()) 1838 return true; 1839 1840 // Make a local copy in case we can't do this fold. 1841 X86ISelAddressMode Backup = AM; 1842 1843 int64_t Offset = 0; 1844 SDValue N0 = N.getOperand(0); 1845 if (auto *G = dyn_cast<GlobalAddressSDNode>(N0)) { 1846 AM.GV = G->getGlobal(); 1847 AM.SymbolFlags = G->getTargetFlags(); 1848 Offset = G->getOffset(); 1849 } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 1850 AM.CP = CP->getConstVal(); 1851 AM.Alignment = CP->getAlign(); 1852 AM.SymbolFlags = CP->getTargetFlags(); 1853 Offset = CP->getOffset(); 1854 } else if (auto *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 1855 AM.ES = S->getSymbol(); 1856 AM.SymbolFlags = S->getTargetFlags(); 1857 } else if (auto *S = dyn_cast<MCSymbolSDNode>(N0)) { 1858 AM.MCSym = S->getMCSymbol(); 1859 } else if (auto *J = dyn_cast<JumpTableSDNode>(N0)) { 1860 AM.JT = J->getIndex(); 1861 AM.SymbolFlags = J->getTargetFlags(); 1862 } else if (auto *BA = dyn_cast<BlockAddressSDNode>(N0)) { 1863 AM.BlockAddr = BA->getBlockAddress(); 1864 AM.SymbolFlags = BA->getTargetFlags(); 1865 Offset = BA->getOffset(); 1866 } else 1867 llvm_unreachable("Unhandled symbol reference node."); 1868 1869 if (foldOffsetIntoAddress(Offset, AM)) { 1870 AM = Backup; 1871 return true; 1872 } 1873 1874 if (IsRIPRel) 1875 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); 1876 1877 // Commit the changes now that we know this fold is safe. 1878 return false; 1879 } 1880 1881 /// Add the specified node to the specified addressing mode, returning true if 1882 /// it cannot be done. This just pattern matches for the addressing mode. 1883 bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) { 1884 if (matchAddressRecursively(N, AM, 0)) 1885 return true; 1886 1887 // Post-processing: Make a second attempt to fold a load, if we now know 1888 // that there will not be any other register. This is only performed for 1889 // 64-bit ILP32 mode since 32-bit mode and 64-bit LP64 mode will have folded 1890 // any foldable load the first time. 1891 if (Subtarget->isTarget64BitILP32() && 1892 AM.BaseType == X86ISelAddressMode::RegBase && 1893 AM.Base_Reg.getNode() != nullptr && AM.IndexReg.getNode() == nullptr) { 1894 SDValue Save_Base_Reg = AM.Base_Reg; 1895 if (auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) { 1896 AM.Base_Reg = SDValue(); 1897 if (matchLoadInAddress(LoadN, AM, /*AllowSegmentRegForX32=*/true)) 1898 AM.Base_Reg = Save_Base_Reg; 1899 } 1900 } 1901 1902 // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has 1903 // a smaller encoding and avoids a scaled-index. 1904 if (AM.Scale == 2 && 1905 AM.BaseType == X86ISelAddressMode::RegBase && 1906 AM.Base_Reg.getNode() == nullptr) { 1907 AM.Base_Reg = AM.IndexReg; 1908 AM.Scale = 1; 1909 } 1910 1911 // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, 1912 // because it has a smaller encoding. 1913 // TODO: Which other code models can use this? 1914 switch (TM.getCodeModel()) { 1915 default: break; 1916 case CodeModel::Small: 1917 case CodeModel::Kernel: 1918 if (Subtarget->is64Bit() && 1919 AM.Scale == 1 && 1920 AM.BaseType == X86ISelAddressMode::RegBase && 1921 AM.Base_Reg.getNode() == nullptr && 1922 AM.IndexReg.getNode() == nullptr && 1923 AM.SymbolFlags == X86II::MO_NO_FLAG && 1924 AM.hasSymbolicDisplacement()) 1925 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); 1926 break; 1927 } 1928 1929 return false; 1930 } 1931 1932 bool X86DAGToDAGISel::matchAdd(SDValue &N, X86ISelAddressMode &AM, 1933 unsigned Depth) { 1934 // Add an artificial use to this node so that we can keep track of 1935 // it if it gets CSE'd with a different node. 1936 HandleSDNode Handle(N); 1937 1938 X86ISelAddressMode Backup = AM; 1939 if (!matchAddressRecursively(N.getOperand(0), AM, Depth+1) && 1940 !matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)) 1941 return false; 1942 AM = Backup; 1943 1944 // Try again after commutating the operands. 1945 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM, 1946 Depth + 1) && 1947 !matchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth + 1)) 1948 return false; 1949 AM = Backup; 1950 1951 // If we couldn't fold both operands into the address at the same time, 1952 // see if we can just put each operand into a register and fold at least 1953 // the add. 1954 if (AM.BaseType == X86ISelAddressMode::RegBase && 1955 !AM.Base_Reg.getNode() && 1956 !AM.IndexReg.getNode()) { 1957 N = Handle.getValue(); 1958 AM.Base_Reg = N.getOperand(0); 1959 AM.IndexReg = N.getOperand(1); 1960 AM.Scale = 1; 1961 return false; 1962 } 1963 N = Handle.getValue(); 1964 return true; 1965 } 1966 1967 // Insert a node into the DAG at least before the Pos node's position. This 1968 // will reposition the node as needed, and will assign it a node ID that is <= 1969 // the Pos node's ID. Note that this does *not* preserve the uniqueness of node 1970 // IDs! The selection DAG must no longer depend on their uniqueness when this 1971 // is used. 1972 static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N) { 1973 if (N->getNodeId() == -1 || 1974 (SelectionDAGISel::getUninvalidatedNodeId(N.getNode()) > 1975 SelectionDAGISel::getUninvalidatedNodeId(Pos.getNode()))) { 1976 DAG.RepositionNode(Pos->getIterator(), N.getNode()); 1977 // Mark Node as invalid for pruning as after this it may be a successor to a 1978 // selected node but otherwise be in the same position of Pos. 1979 // Conservatively mark it with the same -abs(Id) to assure node id 1980 // invariant is preserved. 1981 N->setNodeId(Pos->getNodeId()); 1982 SelectionDAGISel::InvalidateNodeId(N.getNode()); 1983 } 1984 } 1985 1986 // Transform "(X >> (8-C1)) & (0xff << C1)" to "((X >> 8) & 0xff) << C1" if 1987 // safe. This allows us to convert the shift and and into an h-register 1988 // extract and a scaled index. Returns false if the simplification is 1989 // performed. 1990 static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, 1991 uint64_t Mask, 1992 SDValue Shift, SDValue X, 1993 X86ISelAddressMode &AM) { 1994 if (Shift.getOpcode() != ISD::SRL || 1995 !isa<ConstantSDNode>(Shift.getOperand(1)) || 1996 !Shift.hasOneUse()) 1997 return true; 1998 1999 int ScaleLog = 8 - Shift.getConstantOperandVal(1); 2000 if (ScaleLog <= 0 || ScaleLog >= 4 || 2001 Mask != (0xffu << ScaleLog)) 2002 return true; 2003 2004 MVT XVT = X.getSimpleValueType(); 2005 MVT VT = N.getSimpleValueType(); 2006 SDLoc DL(N); 2007 SDValue Eight = DAG.getConstant(8, DL, MVT::i8); 2008 SDValue NewMask = DAG.getConstant(0xff, DL, XVT); 2009 SDValue Srl = DAG.getNode(ISD::SRL, DL, XVT, X, Eight); 2010 SDValue And = DAG.getNode(ISD::AND, DL, XVT, Srl, NewMask); 2011 SDValue Ext = DAG.getZExtOrTrunc(And, DL, VT); 2012 SDValue ShlCount = DAG.getConstant(ScaleLog, DL, MVT::i8); 2013 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Ext, ShlCount); 2014 2015 // Insert the new nodes into the topological ordering. We must do this in 2016 // a valid topological ordering as nothing is going to go back and re-sort 2017 // these nodes. We continually insert before 'N' in sequence as this is 2018 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 2019 // hierarchy left to express. 2020 insertDAGNode(DAG, N, Eight); 2021 insertDAGNode(DAG, N, NewMask); 2022 insertDAGNode(DAG, N, Srl); 2023 insertDAGNode(DAG, N, And); 2024 insertDAGNode(DAG, N, Ext); 2025 insertDAGNode(DAG, N, ShlCount); 2026 insertDAGNode(DAG, N, Shl); 2027 DAG.ReplaceAllUsesWith(N, Shl); 2028 DAG.RemoveDeadNode(N.getNode()); 2029 AM.IndexReg = Ext; 2030 AM.Scale = (1 << ScaleLog); 2031 return false; 2032 } 2033 2034 // Transforms "(X << C1) & C2" to "(X & (C2>>C1)) << C1" if safe and if this 2035 // allows us to fold the shift into this addressing mode. Returns false if the 2036 // transform succeeded. 2037 static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, 2038 X86ISelAddressMode &AM) { 2039 SDValue Shift = N.getOperand(0); 2040 2041 // Use a signed mask so that shifting right will insert sign bits. These 2042 // bits will be removed when we shift the result left so it doesn't matter 2043 // what we use. This might allow a smaller immediate encoding. 2044 int64_t Mask = cast<ConstantSDNode>(N->getOperand(1))->getSExtValue(); 2045 2046 // If we have an any_extend feeding the AND, look through it to see if there 2047 // is a shift behind it. But only if the AND doesn't use the extended bits. 2048 // FIXME: Generalize this to other ANY_EXTEND than i32 to i64? 2049 bool FoundAnyExtend = false; 2050 if (Shift.getOpcode() == ISD::ANY_EXTEND && Shift.hasOneUse() && 2051 Shift.getOperand(0).getSimpleValueType() == MVT::i32 && 2052 isUInt<32>(Mask)) { 2053 FoundAnyExtend = true; 2054 Shift = Shift.getOperand(0); 2055 } 2056 2057 if (Shift.getOpcode() != ISD::SHL || 2058 !isa<ConstantSDNode>(Shift.getOperand(1))) 2059 return true; 2060 2061 SDValue X = Shift.getOperand(0); 2062 2063 // Not likely to be profitable if either the AND or SHIFT node has more 2064 // than one use (unless all uses are for address computation). Besides, 2065 // isel mechanism requires their node ids to be reused. 2066 if (!N.hasOneUse() || !Shift.hasOneUse()) 2067 return true; 2068 2069 // Verify that the shift amount is something we can fold. 2070 unsigned ShiftAmt = Shift.getConstantOperandVal(1); 2071 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3) 2072 return true; 2073 2074 MVT VT = N.getSimpleValueType(); 2075 SDLoc DL(N); 2076 if (FoundAnyExtend) { 2077 SDValue NewX = DAG.getNode(ISD::ANY_EXTEND, DL, VT, X); 2078 insertDAGNode(DAG, N, NewX); 2079 X = NewX; 2080 } 2081 2082 SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT); 2083 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask); 2084 SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1)); 2085 2086 // Insert the new nodes into the topological ordering. We must do this in 2087 // a valid topological ordering as nothing is going to go back and re-sort 2088 // these nodes. We continually insert before 'N' in sequence as this is 2089 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 2090 // hierarchy left to express. 2091 insertDAGNode(DAG, N, NewMask); 2092 insertDAGNode(DAG, N, NewAnd); 2093 insertDAGNode(DAG, N, NewShift); 2094 DAG.ReplaceAllUsesWith(N, NewShift); 2095 DAG.RemoveDeadNode(N.getNode()); 2096 2097 AM.Scale = 1 << ShiftAmt; 2098 AM.IndexReg = NewAnd; 2099 return false; 2100 } 2101 2102 // Implement some heroics to detect shifts of masked values where the mask can 2103 // be replaced by extending the shift and undoing that in the addressing mode 2104 // scale. Patterns such as (shl (srl x, c1), c2) are canonicalized into (and 2105 // (srl x, SHIFT), MASK) by DAGCombines that don't know the shl can be done in 2106 // the addressing mode. This results in code such as: 2107 // 2108 // int f(short *y, int *lookup_table) { 2109 // ... 2110 // return *y + lookup_table[*y >> 11]; 2111 // } 2112 // 2113 // Turning into: 2114 // movzwl (%rdi), %eax 2115 // movl %eax, %ecx 2116 // shrl $11, %ecx 2117 // addl (%rsi,%rcx,4), %eax 2118 // 2119 // Instead of: 2120 // movzwl (%rdi), %eax 2121 // movl %eax, %ecx 2122 // shrl $9, %ecx 2123 // andl $124, %rcx 2124 // addl (%rsi,%rcx), %eax 2125 // 2126 // Note that this function assumes the mask is provided as a mask *after* the 2127 // value is shifted. The input chain may or may not match that, but computing 2128 // such a mask is trivial. 2129 static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, 2130 uint64_t Mask, 2131 SDValue Shift, SDValue X, 2132 X86ISelAddressMode &AM) { 2133 if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse() || 2134 !isa<ConstantSDNode>(Shift.getOperand(1))) 2135 return true; 2136 2137 // We need to ensure that mask is a continuous run of bits. 2138 unsigned MaskIdx, MaskLen; 2139 if (!isShiftedMask_64(Mask, MaskIdx, MaskLen)) 2140 return true; 2141 unsigned MaskLZ = 64 - (MaskIdx + MaskLen); 2142 2143 unsigned ShiftAmt = Shift.getConstantOperandVal(1); 2144 2145 // The amount of shift we're trying to fit into the addressing mode is taken 2146 // from the shifted mask index (number of trailing zeros of the mask). 2147 unsigned AMShiftAmt = MaskIdx; 2148 2149 // There is nothing we can do here unless the mask is removing some bits. 2150 // Also, the addressing mode can only represent shifts of 1, 2, or 3 bits. 2151 if (AMShiftAmt == 0 || AMShiftAmt > 3) return true; 2152 2153 // Scale the leading zero count down based on the actual size of the value. 2154 // Also scale it down based on the size of the shift. 2155 unsigned ScaleDown = (64 - X.getSimpleValueType().getSizeInBits()) + ShiftAmt; 2156 if (MaskLZ < ScaleDown) 2157 return true; 2158 MaskLZ -= ScaleDown; 2159 2160 // The final check is to ensure that any masked out high bits of X are 2161 // already known to be zero. Otherwise, the mask has a semantic impact 2162 // other than masking out a couple of low bits. Unfortunately, because of 2163 // the mask, zero extensions will be removed from operands in some cases. 2164 // This code works extra hard to look through extensions because we can 2165 // replace them with zero extensions cheaply if necessary. 2166 bool ReplacingAnyExtend = false; 2167 if (X.getOpcode() == ISD::ANY_EXTEND) { 2168 unsigned ExtendBits = X.getSimpleValueType().getSizeInBits() - 2169 X.getOperand(0).getSimpleValueType().getSizeInBits(); 2170 // Assume that we'll replace the any-extend with a zero-extend, and 2171 // narrow the search to the extended value. 2172 X = X.getOperand(0); 2173 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits; 2174 ReplacingAnyExtend = true; 2175 } 2176 APInt MaskedHighBits = 2177 APInt::getHighBitsSet(X.getSimpleValueType().getSizeInBits(), MaskLZ); 2178 if (!DAG.MaskedValueIsZero(X, MaskedHighBits)) 2179 return true; 2180 2181 // We've identified a pattern that can be transformed into a single shift 2182 // and an addressing mode. Make it so. 2183 MVT VT = N.getSimpleValueType(); 2184 if (ReplacingAnyExtend) { 2185 assert(X.getValueType() != VT); 2186 // We looked through an ANY_EXTEND node, insert a ZERO_EXTEND. 2187 SDValue NewX = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(X), VT, X); 2188 insertDAGNode(DAG, N, NewX); 2189 X = NewX; 2190 } 2191 2192 MVT XVT = X.getSimpleValueType(); 2193 SDLoc DL(N); 2194 SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8); 2195 SDValue NewSRL = DAG.getNode(ISD::SRL, DL, XVT, X, NewSRLAmt); 2196 SDValue NewExt = DAG.getZExtOrTrunc(NewSRL, DL, VT); 2197 SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8); 2198 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewExt, NewSHLAmt); 2199 2200 // Insert the new nodes into the topological ordering. We must do this in 2201 // a valid topological ordering as nothing is going to go back and re-sort 2202 // these nodes. We continually insert before 'N' in sequence as this is 2203 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 2204 // hierarchy left to express. 2205 insertDAGNode(DAG, N, NewSRLAmt); 2206 insertDAGNode(DAG, N, NewSRL); 2207 insertDAGNode(DAG, N, NewExt); 2208 insertDAGNode(DAG, N, NewSHLAmt); 2209 insertDAGNode(DAG, N, NewSHL); 2210 DAG.ReplaceAllUsesWith(N, NewSHL); 2211 DAG.RemoveDeadNode(N.getNode()); 2212 2213 AM.Scale = 1 << AMShiftAmt; 2214 AM.IndexReg = NewExt; 2215 return false; 2216 } 2217 2218 // Transform "(X >> SHIFT) & (MASK << C1)" to 2219 // "((X >> (SHIFT + C1)) & (MASK)) << C1". Everything before the SHL will be 2220 // matched to a BEXTR later. Returns false if the simplification is performed. 2221 static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, 2222 uint64_t Mask, 2223 SDValue Shift, SDValue X, 2224 X86ISelAddressMode &AM, 2225 const X86Subtarget &Subtarget) { 2226 if (Shift.getOpcode() != ISD::SRL || 2227 !isa<ConstantSDNode>(Shift.getOperand(1)) || 2228 !Shift.hasOneUse() || !N.hasOneUse()) 2229 return true; 2230 2231 // Only do this if BEXTR will be matched by matchBEXTRFromAndImm. 2232 if (!Subtarget.hasTBM() && 2233 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR())) 2234 return true; 2235 2236 // We need to ensure that mask is a continuous run of bits. 2237 unsigned MaskIdx, MaskLen; 2238 if (!isShiftedMask_64(Mask, MaskIdx, MaskLen)) 2239 return true; 2240 2241 unsigned ShiftAmt = Shift.getConstantOperandVal(1); 2242 2243 // The amount of shift we're trying to fit into the addressing mode is taken 2244 // from the shifted mask index (number of trailing zeros of the mask). 2245 unsigned AMShiftAmt = MaskIdx; 2246 2247 // There is nothing we can do here unless the mask is removing some bits. 2248 // Also, the addressing mode can only represent shifts of 1, 2, or 3 bits. 2249 if (AMShiftAmt == 0 || AMShiftAmt > 3) return true; 2250 2251 MVT XVT = X.getSimpleValueType(); 2252 MVT VT = N.getSimpleValueType(); 2253 SDLoc DL(N); 2254 SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8); 2255 SDValue NewSRL = DAG.getNode(ISD::SRL, DL, XVT, X, NewSRLAmt); 2256 SDValue NewMask = DAG.getConstant(Mask >> AMShiftAmt, DL, XVT); 2257 SDValue NewAnd = DAG.getNode(ISD::AND, DL, XVT, NewSRL, NewMask); 2258 SDValue NewExt = DAG.getZExtOrTrunc(NewAnd, DL, VT); 2259 SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8); 2260 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewExt, NewSHLAmt); 2261 2262 // Insert the new nodes into the topological ordering. We must do this in 2263 // a valid topological ordering as nothing is going to go back and re-sort 2264 // these nodes. We continually insert before 'N' in sequence as this is 2265 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 2266 // hierarchy left to express. 2267 insertDAGNode(DAG, N, NewSRLAmt); 2268 insertDAGNode(DAG, N, NewSRL); 2269 insertDAGNode(DAG, N, NewMask); 2270 insertDAGNode(DAG, N, NewAnd); 2271 insertDAGNode(DAG, N, NewExt); 2272 insertDAGNode(DAG, N, NewSHLAmt); 2273 insertDAGNode(DAG, N, NewSHL); 2274 DAG.ReplaceAllUsesWith(N, NewSHL); 2275 DAG.RemoveDeadNode(N.getNode()); 2276 2277 AM.Scale = 1 << AMShiftAmt; 2278 AM.IndexReg = NewExt; 2279 return false; 2280 } 2281 2282 // Attempt to peek further into a scaled index register, collecting additional 2283 // extensions / offsets / etc. Returns /p N if we can't peek any further. 2284 SDValue X86DAGToDAGISel::matchIndexRecursively(SDValue N, 2285 X86ISelAddressMode &AM, 2286 unsigned Depth) { 2287 assert(AM.IndexReg.getNode() == nullptr && "IndexReg already matched"); 2288 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) && 2289 "Illegal index scale"); 2290 2291 // Limit recursion. 2292 if (Depth >= SelectionDAG::MaxRecursionDepth) 2293 return N; 2294 2295 EVT VT = N.getValueType(); 2296 unsigned Opc = N.getOpcode(); 2297 2298 // index: add(x,c) -> index: x, disp + c 2299 if (CurDAG->isBaseWithConstantOffset(N)) { 2300 auto *AddVal = cast<ConstantSDNode>(N.getOperand(1)); 2301 uint64_t Offset = (uint64_t)AddVal->getSExtValue() * AM.Scale; 2302 if (!foldOffsetIntoAddress(Offset, AM)) 2303 return matchIndexRecursively(N.getOperand(0), AM, Depth + 1); 2304 } 2305 2306 // index: add(x,x) -> index: x, scale * 2 2307 if (Opc == ISD::ADD && N.getOperand(0) == N.getOperand(1)) { 2308 if (AM.Scale <= 4) { 2309 AM.Scale *= 2; 2310 return matchIndexRecursively(N.getOperand(0), AM, Depth + 1); 2311 } 2312 } 2313 2314 // index: shl(x,i) -> index: x, scale * (1 << i) 2315 if (Opc == X86ISD::VSHLI) { 2316 uint64_t ShiftAmt = N.getConstantOperandVal(1); 2317 uint64_t ScaleAmt = 1ULL << ShiftAmt; 2318 if ((AM.Scale * ScaleAmt) <= 8) { 2319 AM.Scale *= ScaleAmt; 2320 return matchIndexRecursively(N.getOperand(0), AM, Depth + 1); 2321 } 2322 } 2323 2324 // index: sext(add_nsw(x,c)) -> index: sext(x), disp + sext(c) 2325 // TODO: call matchIndexRecursively(AddSrc) if we won't corrupt sext? 2326 if (Opc == ISD::SIGN_EXTEND && !VT.isVector() && N.hasOneUse()) { 2327 SDValue Src = N.getOperand(0); 2328 if (Src.getOpcode() == ISD::ADD && Src->getFlags().hasNoSignedWrap() && 2329 Src.hasOneUse()) { 2330 if (CurDAG->isBaseWithConstantOffset(Src)) { 2331 SDValue AddSrc = Src.getOperand(0); 2332 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1)); 2333 uint64_t Offset = (uint64_t)AddVal->getSExtValue(); 2334 if (!foldOffsetIntoAddress(Offset * AM.Scale, AM)) { 2335 SDLoc DL(N); 2336 SDValue ExtSrc = CurDAG->getNode(Opc, DL, VT, AddSrc); 2337 SDValue ExtVal = CurDAG->getConstant(Offset, DL, VT); 2338 SDValue ExtAdd = CurDAG->getNode(ISD::ADD, DL, VT, ExtSrc, ExtVal); 2339 insertDAGNode(*CurDAG, N, ExtSrc); 2340 insertDAGNode(*CurDAG, N, ExtVal); 2341 insertDAGNode(*CurDAG, N, ExtAdd); 2342 CurDAG->ReplaceAllUsesWith(N, ExtAdd); 2343 CurDAG->RemoveDeadNode(N.getNode()); 2344 return ExtSrc; 2345 } 2346 } 2347 } 2348 } 2349 2350 // index: zext(add_nuw(x,c)) -> index: zext(x), disp + zext(c) 2351 // index: zext(addlike(x,c)) -> index: zext(x), disp + zext(c) 2352 // TODO: call matchIndexRecursively(AddSrc) if we won't corrupt sext? 2353 if (Opc == ISD::ZERO_EXTEND && !VT.isVector() && N.hasOneUse()) { 2354 SDValue Src = N.getOperand(0); 2355 unsigned SrcOpc = Src.getOpcode(); 2356 if (((SrcOpc == ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) || 2357 CurDAG->isADDLike(Src)) && 2358 Src.hasOneUse()) { 2359 if (CurDAG->isBaseWithConstantOffset(Src)) { 2360 SDValue AddSrc = Src.getOperand(0); 2361 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1)); 2362 uint64_t Offset = (uint64_t)AddVal->getZExtValue(); 2363 if (!foldOffsetIntoAddress(Offset * AM.Scale, AM)) { 2364 SDLoc DL(N); 2365 SDValue Res; 2366 // If we're also scaling, see if we can use that as well. 2367 if (AddSrc.getOpcode() == ISD::SHL && 2368 isa<ConstantSDNode>(AddSrc.getOperand(1))) { 2369 SDValue ShVal = AddSrc.getOperand(0); 2370 uint64_t ShAmt = AddSrc.getConstantOperandVal(1); 2371 APInt HiBits = 2372 APInt::getHighBitsSet(AddSrc.getScalarValueSizeInBits(), ShAmt); 2373 uint64_t ScaleAmt = 1ULL << ShAmt; 2374 if ((AM.Scale * ScaleAmt) <= 8 && 2375 (AddSrc->getFlags().hasNoUnsignedWrap() || 2376 CurDAG->MaskedValueIsZero(ShVal, HiBits))) { 2377 AM.Scale *= ScaleAmt; 2378 SDValue ExtShVal = CurDAG->getNode(Opc, DL, VT, ShVal); 2379 SDValue ExtShift = CurDAG->getNode(ISD::SHL, DL, VT, ExtShVal, 2380 AddSrc.getOperand(1)); 2381 insertDAGNode(*CurDAG, N, ExtShVal); 2382 insertDAGNode(*CurDAG, N, ExtShift); 2383 AddSrc = ExtShift; 2384 Res = ExtShVal; 2385 } 2386 } 2387 SDValue ExtSrc = CurDAG->getNode(Opc, DL, VT, AddSrc); 2388 SDValue ExtVal = CurDAG->getConstant(Offset, DL, VT); 2389 SDValue ExtAdd = CurDAG->getNode(SrcOpc, DL, VT, ExtSrc, ExtVal); 2390 insertDAGNode(*CurDAG, N, ExtSrc); 2391 insertDAGNode(*CurDAG, N, ExtVal); 2392 insertDAGNode(*CurDAG, N, ExtAdd); 2393 CurDAG->ReplaceAllUsesWith(N, ExtAdd); 2394 CurDAG->RemoveDeadNode(N.getNode()); 2395 return Res ? Res : ExtSrc; 2396 } 2397 } 2398 } 2399 } 2400 2401 // TODO: Handle extensions, shifted masks etc. 2402 return N; 2403 } 2404 2405 bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 2406 unsigned Depth) { 2407 SDLoc dl(N); 2408 LLVM_DEBUG({ 2409 dbgs() << "MatchAddress: "; 2410 AM.dump(CurDAG); 2411 }); 2412 // Limit recursion. 2413 if (Depth >= SelectionDAG::MaxRecursionDepth) 2414 return matchAddressBase(N, AM); 2415 2416 // If this is already a %rip relative address, we can only merge immediates 2417 // into it. Instead of handling this in every case, we handle it here. 2418 // RIP relative addressing: %rip + 32-bit displacement! 2419 if (AM.isRIPRelative()) { 2420 // FIXME: JumpTable and ExternalSymbol address currently don't like 2421 // displacements. It isn't very important, but this should be fixed for 2422 // consistency. 2423 if (!(AM.ES || AM.MCSym) && AM.JT != -1) 2424 return true; 2425 2426 if (auto *Cst = dyn_cast<ConstantSDNode>(N)) 2427 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM)) 2428 return false; 2429 return true; 2430 } 2431 2432 switch (N.getOpcode()) { 2433 default: break; 2434 case ISD::LOCAL_RECOVER: { 2435 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0) 2436 if (const auto *ESNode = dyn_cast<MCSymbolSDNode>(N.getOperand(0))) { 2437 // Use the symbol and don't prefix it. 2438 AM.MCSym = ESNode->getMCSymbol(); 2439 return false; 2440 } 2441 break; 2442 } 2443 case ISD::Constant: { 2444 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 2445 if (!foldOffsetIntoAddress(Val, AM)) 2446 return false; 2447 break; 2448 } 2449 2450 case X86ISD::Wrapper: 2451 case X86ISD::WrapperRIP: 2452 if (!matchWrapper(N, AM)) 2453 return false; 2454 break; 2455 2456 case ISD::LOAD: 2457 if (!matchLoadInAddress(cast<LoadSDNode>(N), AM)) 2458 return false; 2459 break; 2460 2461 case ISD::FrameIndex: 2462 if (AM.BaseType == X86ISelAddressMode::RegBase && 2463 AM.Base_Reg.getNode() == nullptr && 2464 (!Subtarget->is64Bit() || isDispSafeForFrameIndex(AM.Disp))) { 2465 AM.BaseType = X86ISelAddressMode::FrameIndexBase; 2466 AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); 2467 return false; 2468 } 2469 break; 2470 2471 case ISD::SHL: 2472 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) 2473 break; 2474 2475 if (auto *CN = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 2476 unsigned Val = CN->getZExtValue(); 2477 // Note that we handle x<<1 as (,x,2) rather than (x,x) here so 2478 // that the base operand remains free for further matching. If 2479 // the base doesn't end up getting used, a post-processing step 2480 // in MatchAddress turns (,x,2) into (x,x), which is cheaper. 2481 if (Val == 1 || Val == 2 || Val == 3) { 2482 SDValue ShVal = N.getOperand(0); 2483 AM.Scale = 1 << Val; 2484 AM.IndexReg = matchIndexRecursively(ShVal, AM, Depth + 1); 2485 return false; 2486 } 2487 } 2488 break; 2489 2490 case ISD::SRL: { 2491 // Scale must not be used already. 2492 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; 2493 2494 // We only handle up to 64-bit values here as those are what matter for 2495 // addressing mode optimizations. 2496 assert(N.getSimpleValueType().getSizeInBits() <= 64 && 2497 "Unexpected value size!"); 2498 2499 SDValue And = N.getOperand(0); 2500 if (And.getOpcode() != ISD::AND) break; 2501 SDValue X = And.getOperand(0); 2502 2503 // The mask used for the transform is expected to be post-shift, but we 2504 // found the shift first so just apply the shift to the mask before passing 2505 // it down. 2506 if (!isa<ConstantSDNode>(N.getOperand(1)) || 2507 !isa<ConstantSDNode>(And.getOperand(1))) 2508 break; 2509 uint64_t Mask = And.getConstantOperandVal(1) >> N.getConstantOperandVal(1); 2510 2511 // Try to fold the mask and shift into the scale, and return false if we 2512 // succeed. 2513 if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, N, X, AM)) 2514 return false; 2515 break; 2516 } 2517 2518 case ISD::SMUL_LOHI: 2519 case ISD::UMUL_LOHI: 2520 // A mul_lohi where we need the low part can be folded as a plain multiply. 2521 if (N.getResNo() != 0) break; 2522 [[fallthrough]]; 2523 case ISD::MUL: 2524 case X86ISD::MUL_IMM: 2525 // X*[3,5,9] -> X+X*[2,4,8] 2526 if (AM.BaseType == X86ISelAddressMode::RegBase && 2527 AM.Base_Reg.getNode() == nullptr && 2528 AM.IndexReg.getNode() == nullptr) { 2529 if (auto *CN = dyn_cast<ConstantSDNode>(N.getOperand(1))) 2530 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || 2531 CN->getZExtValue() == 9) { 2532 AM.Scale = unsigned(CN->getZExtValue())-1; 2533 2534 SDValue MulVal = N.getOperand(0); 2535 SDValue Reg; 2536 2537 // Okay, we know that we have a scale by now. However, if the scaled 2538 // value is an add of something and a constant, we can fold the 2539 // constant into the disp field here. 2540 if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && 2541 isa<ConstantSDNode>(MulVal.getOperand(1))) { 2542 Reg = MulVal.getOperand(0); 2543 auto *AddVal = cast<ConstantSDNode>(MulVal.getOperand(1)); 2544 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue(); 2545 if (foldOffsetIntoAddress(Disp, AM)) 2546 Reg = N.getOperand(0); 2547 } else { 2548 Reg = N.getOperand(0); 2549 } 2550 2551 AM.IndexReg = AM.Base_Reg = Reg; 2552 return false; 2553 } 2554 } 2555 break; 2556 2557 case ISD::SUB: { 2558 // Given A-B, if A can be completely folded into the address and 2559 // the index field with the index field unused, use -B as the index. 2560 // This is a win if a has multiple parts that can be folded into 2561 // the address. Also, this saves a mov if the base register has 2562 // other uses, since it avoids a two-address sub instruction, however 2563 // it costs an additional mov if the index register has other uses. 2564 2565 // Add an artificial use to this node so that we can keep track of 2566 // it if it gets CSE'd with a different node. 2567 HandleSDNode Handle(N); 2568 2569 // Test if the LHS of the sub can be folded. 2570 X86ISelAddressMode Backup = AM; 2571 if (matchAddressRecursively(N.getOperand(0), AM, Depth+1)) { 2572 N = Handle.getValue(); 2573 AM = Backup; 2574 break; 2575 } 2576 N = Handle.getValue(); 2577 // Test if the index field is free for use. 2578 if (AM.IndexReg.getNode() || AM.isRIPRelative()) { 2579 AM = Backup; 2580 break; 2581 } 2582 2583 int Cost = 0; 2584 SDValue RHS = N.getOperand(1); 2585 // If the RHS involves a register with multiple uses, this 2586 // transformation incurs an extra mov, due to the neg instruction 2587 // clobbering its operand. 2588 if (!RHS.getNode()->hasOneUse() || 2589 RHS.getNode()->getOpcode() == ISD::CopyFromReg || 2590 RHS.getNode()->getOpcode() == ISD::TRUNCATE || 2591 RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || 2592 (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && 2593 RHS.getOperand(0).getValueType() == MVT::i32)) 2594 ++Cost; 2595 // If the base is a register with multiple uses, this 2596 // transformation may save a mov. 2597 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() && 2598 !AM.Base_Reg.getNode()->hasOneUse()) || 2599 AM.BaseType == X86ISelAddressMode::FrameIndexBase) 2600 --Cost; 2601 // If the folded LHS was interesting, this transformation saves 2602 // address arithmetic. 2603 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) + 2604 ((AM.Disp != 0) && (Backup.Disp == 0)) + 2605 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2) 2606 --Cost; 2607 // If it doesn't look like it may be an overall win, don't do it. 2608 if (Cost >= 0) { 2609 AM = Backup; 2610 break; 2611 } 2612 2613 // Ok, the transformation is legal and appears profitable. Go for it. 2614 // Negation will be emitted later to avoid creating dangling nodes if this 2615 // was an unprofitable LEA. 2616 AM.IndexReg = RHS; 2617 AM.NegateIndex = true; 2618 AM.Scale = 1; 2619 return false; 2620 } 2621 2622 case ISD::OR: 2623 case ISD::XOR: 2624 // See if we can treat the OR/XOR node as an ADD node. 2625 if (!CurDAG->isADDLike(N)) 2626 break; 2627 [[fallthrough]]; 2628 case ISD::ADD: 2629 if (!matchAdd(N, AM, Depth)) 2630 return false; 2631 break; 2632 2633 case ISD::AND: { 2634 // Perform some heroic transforms on an and of a constant-count shift 2635 // with a constant to enable use of the scaled offset field. 2636 2637 // Scale must not be used already. 2638 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; 2639 2640 // We only handle up to 64-bit values here as those are what matter for 2641 // addressing mode optimizations. 2642 assert(N.getSimpleValueType().getSizeInBits() <= 64 && 2643 "Unexpected value size!"); 2644 2645 if (!isa<ConstantSDNode>(N.getOperand(1))) 2646 break; 2647 2648 if (N.getOperand(0).getOpcode() == ISD::SRL) { 2649 SDValue Shift = N.getOperand(0); 2650 SDValue X = Shift.getOperand(0); 2651 2652 uint64_t Mask = N.getConstantOperandVal(1); 2653 2654 // Try to fold the mask and shift into an extract and scale. 2655 if (!foldMaskAndShiftToExtract(*CurDAG, N, Mask, Shift, X, AM)) 2656 return false; 2657 2658 // Try to fold the mask and shift directly into the scale. 2659 if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, Shift, X, AM)) 2660 return false; 2661 2662 // Try to fold the mask and shift into BEXTR and scale. 2663 if (!foldMaskedShiftToBEXTR(*CurDAG, N, Mask, Shift, X, AM, *Subtarget)) 2664 return false; 2665 } 2666 2667 // Try to swap the mask and shift to place shifts which can be done as 2668 // a scale on the outside of the mask. 2669 if (!foldMaskedShiftToScaledMask(*CurDAG, N, AM)) 2670 return false; 2671 2672 break; 2673 } 2674 case ISD::ZERO_EXTEND: { 2675 // Try to widen a zexted shift left to the same size as its use, so we can 2676 // match the shift as a scale factor. 2677 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) 2678 break; 2679 2680 SDValue Src = N.getOperand(0); 2681 2682 // See if we can match a zext(addlike(x,c)). 2683 // TODO: Move more ZERO_EXTEND patterns into matchIndexRecursively. 2684 if (Src.getOpcode() == ISD::ADD || Src.getOpcode() == ISD::OR) 2685 if (SDValue Index = matchIndexRecursively(N, AM, Depth + 1)) 2686 if (Index != N) { 2687 AM.IndexReg = Index; 2688 return false; 2689 } 2690 2691 // Peek through mask: zext(and(shl(x,c1),c2)) 2692 APInt Mask = APInt::getAllOnes(Src.getScalarValueSizeInBits()); 2693 if (Src.getOpcode() == ISD::AND && Src.hasOneUse()) 2694 if (auto *MaskC = dyn_cast<ConstantSDNode>(Src.getOperand(1))) { 2695 Mask = MaskC->getAPIntValue(); 2696 Src = Src.getOperand(0); 2697 } 2698 2699 if (Src.getOpcode() == ISD::SHL && Src.hasOneUse()) { 2700 // Give up if the shift is not a valid scale factor [1,2,3]. 2701 SDValue ShlSrc = Src.getOperand(0); 2702 SDValue ShlAmt = Src.getOperand(1); 2703 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShlAmt); 2704 if (!ShAmtC) 2705 break; 2706 unsigned ShAmtV = ShAmtC->getZExtValue(); 2707 if (ShAmtV > 3) 2708 break; 2709 2710 // The narrow shift must only shift out zero bits (it must be 'nuw'). 2711 // That makes it safe to widen to the destination type. 2712 APInt HighZeros = 2713 APInt::getHighBitsSet(ShlSrc.getValueSizeInBits(), ShAmtV); 2714 if (!Src->getFlags().hasNoUnsignedWrap() && 2715 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask)) 2716 break; 2717 2718 // zext (shl nuw i8 %x, C1) to i32 2719 // --> shl (zext i8 %x to i32), (zext C1) 2720 // zext (and (shl nuw i8 %x, C1), C2) to i32 2721 // --> shl (zext i8 (and %x, C2 >> C1) to i32), (zext C1) 2722 MVT SrcVT = ShlSrc.getSimpleValueType(); 2723 MVT VT = N.getSimpleValueType(); 2724 SDLoc DL(N); 2725 2726 SDValue Res = ShlSrc; 2727 if (!Mask.isAllOnes()) { 2728 Res = CurDAG->getConstant(Mask.lshr(ShAmtV), DL, SrcVT); 2729 insertDAGNode(*CurDAG, N, Res); 2730 Res = CurDAG->getNode(ISD::AND, DL, SrcVT, ShlSrc, Res); 2731 insertDAGNode(*CurDAG, N, Res); 2732 } 2733 SDValue Zext = CurDAG->getNode(ISD::ZERO_EXTEND, DL, VT, Res); 2734 insertDAGNode(*CurDAG, N, Zext); 2735 SDValue NewShl = CurDAG->getNode(ISD::SHL, DL, VT, Zext, ShlAmt); 2736 insertDAGNode(*CurDAG, N, NewShl); 2737 2738 // Convert the shift to scale factor. 2739 AM.Scale = 1 << ShAmtV; 2740 AM.IndexReg = Zext; 2741 2742 CurDAG->ReplaceAllUsesWith(N, NewShl); 2743 CurDAG->RemoveDeadNode(N.getNode()); 2744 return false; 2745 } 2746 2747 if (Src.getOpcode() == ISD::SRL && !Mask.isAllOnes()) { 2748 // Try to fold the mask and shift into an extract and scale. 2749 if (!foldMaskAndShiftToExtract(*CurDAG, N, Mask.getZExtValue(), Src, 2750 Src.getOperand(0), AM)) 2751 return false; 2752 2753 // Try to fold the mask and shift directly into the scale. 2754 if (!foldMaskAndShiftToScale(*CurDAG, N, Mask.getZExtValue(), Src, 2755 Src.getOperand(0), AM)) 2756 return false; 2757 2758 // Try to fold the mask and shift into BEXTR and scale. 2759 if (!foldMaskedShiftToBEXTR(*CurDAG, N, Mask.getZExtValue(), Src, 2760 Src.getOperand(0), AM, *Subtarget)) 2761 return false; 2762 } 2763 2764 break; 2765 } 2766 } 2767 2768 return matchAddressBase(N, AM); 2769 } 2770 2771 /// Helper for MatchAddress. Add the specified node to the 2772 /// specified addressing mode without any further recursion. 2773 bool X86DAGToDAGISel::matchAddressBase(SDValue N, X86ISelAddressMode &AM) { 2774 // Is the base register already occupied? 2775 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) { 2776 // If so, check to see if the scale index register is set. 2777 if (!AM.IndexReg.getNode()) { 2778 AM.IndexReg = N; 2779 AM.Scale = 1; 2780 return false; 2781 } 2782 2783 // Otherwise, we cannot select it. 2784 return true; 2785 } 2786 2787 // Default, generate it as a register. 2788 AM.BaseType = X86ISelAddressMode::RegBase; 2789 AM.Base_Reg = N; 2790 return false; 2791 } 2792 2793 bool X86DAGToDAGISel::matchVectorAddressRecursively(SDValue N, 2794 X86ISelAddressMode &AM, 2795 unsigned Depth) { 2796 SDLoc dl(N); 2797 LLVM_DEBUG({ 2798 dbgs() << "MatchVectorAddress: "; 2799 AM.dump(CurDAG); 2800 }); 2801 // Limit recursion. 2802 if (Depth >= SelectionDAG::MaxRecursionDepth) 2803 return matchAddressBase(N, AM); 2804 2805 // TODO: Support other operations. 2806 switch (N.getOpcode()) { 2807 case ISD::Constant: { 2808 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 2809 if (!foldOffsetIntoAddress(Val, AM)) 2810 return false; 2811 break; 2812 } 2813 case X86ISD::Wrapper: 2814 if (!matchWrapper(N, AM)) 2815 return false; 2816 break; 2817 case ISD::ADD: { 2818 // Add an artificial use to this node so that we can keep track of 2819 // it if it gets CSE'd with a different node. 2820 HandleSDNode Handle(N); 2821 2822 X86ISelAddressMode Backup = AM; 2823 if (!matchVectorAddressRecursively(N.getOperand(0), AM, Depth + 1) && 2824 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM, 2825 Depth + 1)) 2826 return false; 2827 AM = Backup; 2828 2829 // Try again after commuting the operands. 2830 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM, 2831 Depth + 1) && 2832 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM, 2833 Depth + 1)) 2834 return false; 2835 AM = Backup; 2836 2837 N = Handle.getValue(); 2838 break; 2839 } 2840 } 2841 2842 return matchAddressBase(N, AM); 2843 } 2844 2845 /// Helper for selectVectorAddr. Handles things that can be folded into a 2846 /// gather/scatter address. The index register and scale should have already 2847 /// been handled. 2848 bool X86DAGToDAGISel::matchVectorAddress(SDValue N, X86ISelAddressMode &AM) { 2849 return matchVectorAddressRecursively(N, AM, 0); 2850 } 2851 2852 bool X86DAGToDAGISel::selectVectorAddr(MemSDNode *Parent, SDValue BasePtr, 2853 SDValue IndexOp, SDValue ScaleOp, 2854 SDValue &Base, SDValue &Scale, 2855 SDValue &Index, SDValue &Disp, 2856 SDValue &Segment) { 2857 X86ISelAddressMode AM; 2858 AM.Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue(); 2859 2860 // Attempt to match index patterns, as long as we're not relying on implicit 2861 // sign-extension, which is performed BEFORE scale. 2862 if (IndexOp.getScalarValueSizeInBits() == BasePtr.getScalarValueSizeInBits()) 2863 AM.IndexReg = matchIndexRecursively(IndexOp, AM, 0); 2864 else 2865 AM.IndexReg = IndexOp; 2866 2867 unsigned AddrSpace = Parent->getPointerInfo().getAddrSpace(); 2868 if (AddrSpace == X86AS::GS) 2869 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 2870 if (AddrSpace == X86AS::FS) 2871 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 2872 if (AddrSpace == X86AS::SS) 2873 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16); 2874 2875 SDLoc DL(BasePtr); 2876 MVT VT = BasePtr.getSimpleValueType(); 2877 2878 // Try to match into the base and displacement fields. 2879 if (matchVectorAddress(BasePtr, AM)) 2880 return false; 2881 2882 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); 2883 return true; 2884 } 2885 2886 /// Returns true if it is able to pattern match an addressing mode. 2887 /// It returns the operands which make up the maximal addressing mode it can 2888 /// match by reference. 2889 /// 2890 /// Parent is the parent node of the addr operand that is being matched. It 2891 /// is always a load, store, atomic node, or null. It is only null when 2892 /// checking memory operands for inline asm nodes. 2893 bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base, 2894 SDValue &Scale, SDValue &Index, 2895 SDValue &Disp, SDValue &Segment) { 2896 X86ISelAddressMode AM; 2897 2898 if (Parent && 2899 // This list of opcodes are all the nodes that have an "addr:$ptr" operand 2900 // that are not a MemSDNode, and thus don't have proper addrspace info. 2901 Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme 2902 Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores 2903 Parent->getOpcode() != X86ISD::TLSCALL && // Fixme 2904 Parent->getOpcode() != X86ISD::ENQCMD && // Fixme 2905 Parent->getOpcode() != X86ISD::ENQCMDS && // Fixme 2906 Parent->getOpcode() != X86ISD::EH_SJLJ_SETJMP && // setjmp 2907 Parent->getOpcode() != X86ISD::EH_SJLJ_LONGJMP) { // longjmp 2908 unsigned AddrSpace = 2909 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace(); 2910 if (AddrSpace == X86AS::GS) 2911 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 2912 if (AddrSpace == X86AS::FS) 2913 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 2914 if (AddrSpace == X86AS::SS) 2915 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16); 2916 } 2917 2918 // Save the DL and VT before calling matchAddress, it can invalidate N. 2919 SDLoc DL(N); 2920 MVT VT = N.getSimpleValueType(); 2921 2922 if (matchAddress(N, AM)) 2923 return false; 2924 2925 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); 2926 return true; 2927 } 2928 2929 bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) { 2930 // Cannot use 32 bit constants to reference objects in kernel code model. 2931 // Cannot use 32 bit constants to reference objects in large PIC mode since 2932 // GOTOFF is 64 bits. 2933 if (TM.getCodeModel() == CodeModel::Kernel || 2934 (TM.getCodeModel() == CodeModel::Large && TM.isPositionIndependent())) 2935 return false; 2936 2937 // In static codegen with small code model, we can get the address of a label 2938 // into a register with 'movl' 2939 if (N->getOpcode() != X86ISD::Wrapper) 2940 return false; 2941 2942 N = N.getOperand(0); 2943 2944 // At least GNU as does not accept 'movl' for TPOFF relocations. 2945 // FIXME: We could use 'movl' when we know we are targeting MC. 2946 if (N->getOpcode() == ISD::TargetGlobalTLSAddress) 2947 return false; 2948 2949 Imm = N; 2950 // Small/medium code model can reference non-TargetGlobalAddress objects with 2951 // 32 bit constants. 2952 if (N->getOpcode() != ISD::TargetGlobalAddress) { 2953 return TM.getCodeModel() == CodeModel::Small || 2954 TM.getCodeModel() == CodeModel::Medium; 2955 } 2956 2957 const GlobalValue *GV = cast<GlobalAddressSDNode>(N)->getGlobal(); 2958 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange()) 2959 return CR->getUnsignedMax().ult(1ull << 32); 2960 2961 return !TM.isLargeGlobalValue(GV); 2962 } 2963 2964 bool X86DAGToDAGISel::selectLEA64_32Addr(SDValue N, SDValue &Base, 2965 SDValue &Scale, SDValue &Index, 2966 SDValue &Disp, SDValue &Segment) { 2967 // Save the debug loc before calling selectLEAAddr, in case it invalidates N. 2968 SDLoc DL(N); 2969 2970 if (!selectLEAAddr(N, Base, Scale, Index, Disp, Segment)) 2971 return false; 2972 2973 auto *RN = dyn_cast<RegisterSDNode>(Base); 2974 if (RN && RN->getReg() == 0) 2975 Base = CurDAG->getRegister(0, MVT::i64); 2976 else if (Base.getValueType() == MVT::i32 && !isa<FrameIndexSDNode>(Base)) { 2977 // Base could already be %rip, particularly in the x32 ABI. 2978 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL, 2979 MVT::i64), 0); 2980 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit, DL, MVT::i64, ImplDef, 2981 Base); 2982 } 2983 2984 RN = dyn_cast<RegisterSDNode>(Index); 2985 if (RN && RN->getReg() == 0) 2986 Index = CurDAG->getRegister(0, MVT::i64); 2987 else { 2988 assert(Index.getValueType() == MVT::i32 && 2989 "Expect to be extending 32-bit registers for use in LEA"); 2990 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL, 2991 MVT::i64), 0); 2992 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit, DL, MVT::i64, ImplDef, 2993 Index); 2994 } 2995 2996 return true; 2997 } 2998 2999 /// Calls SelectAddr and determines if the maximal addressing 3000 /// mode it matches can be cost effectively emitted as an LEA instruction. 3001 bool X86DAGToDAGISel::selectLEAAddr(SDValue N, 3002 SDValue &Base, SDValue &Scale, 3003 SDValue &Index, SDValue &Disp, 3004 SDValue &Segment) { 3005 X86ISelAddressMode AM; 3006 3007 // Save the DL and VT before calling matchAddress, it can invalidate N. 3008 SDLoc DL(N); 3009 MVT VT = N.getSimpleValueType(); 3010 3011 // Set AM.Segment to prevent MatchAddress from using one. LEA doesn't support 3012 // segments. 3013 SDValue Copy = AM.Segment; 3014 SDValue T = CurDAG->getRegister(0, MVT::i32); 3015 AM.Segment = T; 3016 if (matchAddress(N, AM)) 3017 return false; 3018 assert (T == AM.Segment); 3019 AM.Segment = Copy; 3020 3021 unsigned Complexity = 0; 3022 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode()) 3023 Complexity = 1; 3024 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 3025 Complexity = 4; 3026 3027 if (AM.IndexReg.getNode()) 3028 Complexity++; 3029 3030 // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with 3031 // a simple shift. 3032 if (AM.Scale > 1) 3033 Complexity++; 3034 3035 // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA 3036 // to a LEA. This is determined with some experimentation but is by no means 3037 // optimal (especially for code size consideration). LEA is nice because of 3038 // its three-address nature. Tweak the cost function again when we can run 3039 // convertToThreeAddress() at register allocation time. 3040 if (AM.hasSymbolicDisplacement()) { 3041 // For X86-64, always use LEA to materialize RIP-relative addresses. 3042 if (Subtarget->is64Bit()) 3043 Complexity = 4; 3044 else 3045 Complexity += 2; 3046 } 3047 3048 // Heuristic: try harder to form an LEA from ADD if the operands set flags. 3049 // Unlike ADD, LEA does not affect flags, so we will be less likely to require 3050 // duplicating flag-producing instructions later in the pipeline. 3051 if (N.getOpcode() == ISD::ADD) { 3052 auto isMathWithFlags = [](SDValue V) { 3053 switch (V.getOpcode()) { 3054 case X86ISD::ADD: 3055 case X86ISD::SUB: 3056 case X86ISD::ADC: 3057 case X86ISD::SBB: 3058 case X86ISD::SMUL: 3059 case X86ISD::UMUL: 3060 /* TODO: These opcodes can be added safely, but we may want to justify 3061 their inclusion for different reasons (better for reg-alloc). 3062 case X86ISD::OR: 3063 case X86ISD::XOR: 3064 case X86ISD::AND: 3065 */ 3066 // Value 1 is the flag output of the node - verify it's not dead. 3067 return !SDValue(V.getNode(), 1).use_empty(); 3068 default: 3069 return false; 3070 } 3071 }; 3072 // TODO: We might want to factor in whether there's a load folding 3073 // opportunity for the math op that disappears with LEA. 3074 if (isMathWithFlags(N.getOperand(0)) || isMathWithFlags(N.getOperand(1))) 3075 Complexity++; 3076 } 3077 3078 if (AM.Disp) 3079 Complexity++; 3080 3081 // If it isn't worth using an LEA, reject it. 3082 if (Complexity <= 2) 3083 return false; 3084 3085 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); 3086 return true; 3087 } 3088 3089 /// This is only run on TargetGlobalTLSAddress nodes. 3090 bool X86DAGToDAGISel::selectTLSADDRAddr(SDValue N, SDValue &Base, 3091 SDValue &Scale, SDValue &Index, 3092 SDValue &Disp, SDValue &Segment) { 3093 assert(N.getOpcode() == ISD::TargetGlobalTLSAddress); 3094 auto *GA = cast<GlobalAddressSDNode>(N); 3095 3096 X86ISelAddressMode AM; 3097 AM.GV = GA->getGlobal(); 3098 AM.Disp += GA->getOffset(); 3099 AM.SymbolFlags = GA->getTargetFlags(); 3100 3101 if (Subtarget->is32Bit()) { 3102 AM.Scale = 1; 3103 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); 3104 } 3105 3106 MVT VT = N.getSimpleValueType(); 3107 getAddressOperands(AM, SDLoc(N), VT, Base, Scale, Index, Disp, Segment); 3108 return true; 3109 } 3110 3111 bool X86DAGToDAGISel::selectRelocImm(SDValue N, SDValue &Op) { 3112 // Keep track of the original value type and whether this value was 3113 // truncated. If we see a truncation from pointer type to VT that truncates 3114 // bits that are known to be zero, we can use a narrow reference. 3115 EVT VT = N.getValueType(); 3116 bool WasTruncated = false; 3117 if (N.getOpcode() == ISD::TRUNCATE) { 3118 WasTruncated = true; 3119 N = N.getOperand(0); 3120 } 3121 3122 if (N.getOpcode() != X86ISD::Wrapper) 3123 return false; 3124 3125 // We can only use non-GlobalValues as immediates if they were not truncated, 3126 // as we do not have any range information. If we have a GlobalValue and the 3127 // address was not truncated, we can select it as an operand directly. 3128 unsigned Opc = N.getOperand(0)->getOpcode(); 3129 if (Opc != ISD::TargetGlobalAddress || !WasTruncated) { 3130 Op = N.getOperand(0); 3131 // We can only select the operand directly if we didn't have to look past a 3132 // truncate. 3133 return !WasTruncated; 3134 } 3135 3136 // Check that the global's range fits into VT. 3137 auto *GA = cast<GlobalAddressSDNode>(N.getOperand(0)); 3138 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange(); 3139 if (!CR || CR->getUnsignedMax().uge(1ull << VT.getSizeInBits())) 3140 return false; 3141 3142 // Okay, we can use a narrow reference. 3143 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(N), VT, 3144 GA->getOffset(), GA->getTargetFlags()); 3145 return true; 3146 } 3147 3148 bool X86DAGToDAGISel::tryFoldLoad(SDNode *Root, SDNode *P, SDValue N, 3149 SDValue &Base, SDValue &Scale, 3150 SDValue &Index, SDValue &Disp, 3151 SDValue &Segment) { 3152 assert(Root && P && "Unknown root/parent nodes"); 3153 if (!ISD::isNON_EXTLoad(N.getNode()) || 3154 !IsProfitableToFold(N, P, Root) || 3155 !IsLegalToFold(N, P, Root, OptLevel)) 3156 return false; 3157 3158 return selectAddr(N.getNode(), 3159 N.getOperand(1), Base, Scale, Index, Disp, Segment); 3160 } 3161 3162 bool X86DAGToDAGISel::tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N, 3163 SDValue &Base, SDValue &Scale, 3164 SDValue &Index, SDValue &Disp, 3165 SDValue &Segment) { 3166 assert(Root && P && "Unknown root/parent nodes"); 3167 if (N->getOpcode() != X86ISD::VBROADCAST_LOAD || 3168 !IsProfitableToFold(N, P, Root) || 3169 !IsLegalToFold(N, P, Root, OptLevel)) 3170 return false; 3171 3172 return selectAddr(N.getNode(), 3173 N.getOperand(1), Base, Scale, Index, Disp, Segment); 3174 } 3175 3176 /// Return an SDNode that returns the value of the global base register. 3177 /// Output instructions required to initialize the global base register, 3178 /// if necessary. 3179 SDNode *X86DAGToDAGISel::getGlobalBaseReg() { 3180 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF); 3181 auto &DL = MF->getDataLayout(); 3182 return CurDAG->getRegister(GlobalBaseReg, TLI->getPointerTy(DL)).getNode(); 3183 } 3184 3185 bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const { 3186 if (N->getOpcode() == ISD::TRUNCATE) 3187 N = N->getOperand(0).getNode(); 3188 if (N->getOpcode() != X86ISD::Wrapper) 3189 return false; 3190 3191 auto *GA = dyn_cast<GlobalAddressSDNode>(N->getOperand(0)); 3192 if (!GA) 3193 return false; 3194 3195 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange(); 3196 if (!CR) 3197 return Width == 32 && TM.getCodeModel() == CodeModel::Small; 3198 3199 return CR->getSignedMin().sge(-1ull << Width) && 3200 CR->getSignedMax().slt(1ull << Width); 3201 } 3202 3203 X86::CondCode X86DAGToDAGISel::getCondFromNode(SDNode *N) const { 3204 assert(N->isMachineOpcode() && "Unexpected node"); 3205 unsigned Opc = N->getMachineOpcode(); 3206 const MCInstrDesc &MCID = getInstrInfo()->get(Opc); 3207 int CondNo = X86::getCondSrcNoFromDesc(MCID); 3208 if (CondNo < 0) 3209 return X86::COND_INVALID; 3210 3211 return static_cast<X86::CondCode>(N->getConstantOperandVal(CondNo)); 3212 } 3213 3214 /// Test whether the given X86ISD::CMP node has any users that use a flag 3215 /// other than ZF. 3216 bool X86DAGToDAGISel::onlyUsesZeroFlag(SDValue Flags) const { 3217 // Examine each user of the node. 3218 for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end(); 3219 UI != UE; ++UI) { 3220 // Only check things that use the flags. 3221 if (UI.getUse().getResNo() != Flags.getResNo()) 3222 continue; 3223 // Only examine CopyToReg uses that copy to EFLAGS. 3224 if (UI->getOpcode() != ISD::CopyToReg || 3225 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS) 3226 return false; 3227 // Examine each user of the CopyToReg use. 3228 for (SDNode::use_iterator FlagUI = UI->use_begin(), 3229 FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { 3230 // Only examine the Flag result. 3231 if (FlagUI.getUse().getResNo() != 1) continue; 3232 // Anything unusual: assume conservatively. 3233 if (!FlagUI->isMachineOpcode()) return false; 3234 // Examine the condition code of the user. 3235 X86::CondCode CC = getCondFromNode(*FlagUI); 3236 3237 switch (CC) { 3238 // Comparisons which only use the zero flag. 3239 case X86::COND_E: case X86::COND_NE: 3240 continue; 3241 // Anything else: assume conservatively. 3242 default: 3243 return false; 3244 } 3245 } 3246 } 3247 return true; 3248 } 3249 3250 /// Test whether the given X86ISD::CMP node has any uses which require the SF 3251 /// flag to be accurate. 3252 bool X86DAGToDAGISel::hasNoSignFlagUses(SDValue Flags) const { 3253 // Examine each user of the node. 3254 for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end(); 3255 UI != UE; ++UI) { 3256 // Only check things that use the flags. 3257 if (UI.getUse().getResNo() != Flags.getResNo()) 3258 continue; 3259 // Only examine CopyToReg uses that copy to EFLAGS. 3260 if (UI->getOpcode() != ISD::CopyToReg || 3261 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS) 3262 return false; 3263 // Examine each user of the CopyToReg use. 3264 for (SDNode::use_iterator FlagUI = UI->use_begin(), 3265 FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { 3266 // Only examine the Flag result. 3267 if (FlagUI.getUse().getResNo() != 1) continue; 3268 // Anything unusual: assume conservatively. 3269 if (!FlagUI->isMachineOpcode()) return false; 3270 // Examine the condition code of the user. 3271 X86::CondCode CC = getCondFromNode(*FlagUI); 3272 3273 switch (CC) { 3274 // Comparisons which don't examine the SF flag. 3275 case X86::COND_A: case X86::COND_AE: 3276 case X86::COND_B: case X86::COND_BE: 3277 case X86::COND_E: case X86::COND_NE: 3278 case X86::COND_O: case X86::COND_NO: 3279 case X86::COND_P: case X86::COND_NP: 3280 continue; 3281 // Anything else: assume conservatively. 3282 default: 3283 return false; 3284 } 3285 } 3286 } 3287 return true; 3288 } 3289 3290 static bool mayUseCarryFlag(X86::CondCode CC) { 3291 switch (CC) { 3292 // Comparisons which don't examine the CF flag. 3293 case X86::COND_O: case X86::COND_NO: 3294 case X86::COND_E: case X86::COND_NE: 3295 case X86::COND_S: case X86::COND_NS: 3296 case X86::COND_P: case X86::COND_NP: 3297 case X86::COND_L: case X86::COND_GE: 3298 case X86::COND_G: case X86::COND_LE: 3299 return false; 3300 // Anything else: assume conservatively. 3301 default: 3302 return true; 3303 } 3304 } 3305 3306 /// Test whether the given node which sets flags has any uses which require the 3307 /// CF flag to be accurate. 3308 bool X86DAGToDAGISel::hasNoCarryFlagUses(SDValue Flags) const { 3309 // Examine each user of the node. 3310 for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end(); 3311 UI != UE; ++UI) { 3312 // Only check things that use the flags. 3313 if (UI.getUse().getResNo() != Flags.getResNo()) 3314 continue; 3315 3316 unsigned UIOpc = UI->getOpcode(); 3317 3318 if (UIOpc == ISD::CopyToReg) { 3319 // Only examine CopyToReg uses that copy to EFLAGS. 3320 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS) 3321 return false; 3322 // Examine each user of the CopyToReg use. 3323 for (SDNode::use_iterator FlagUI = UI->use_begin(), FlagUE = UI->use_end(); 3324 FlagUI != FlagUE; ++FlagUI) { 3325 // Only examine the Flag result. 3326 if (FlagUI.getUse().getResNo() != 1) 3327 continue; 3328 // Anything unusual: assume conservatively. 3329 if (!FlagUI->isMachineOpcode()) 3330 return false; 3331 // Examine the condition code of the user. 3332 X86::CondCode CC = getCondFromNode(*FlagUI); 3333 3334 if (mayUseCarryFlag(CC)) 3335 return false; 3336 } 3337 3338 // This CopyToReg is ok. Move on to the next user. 3339 continue; 3340 } 3341 3342 // This might be an unselected node. So look for the pre-isel opcodes that 3343 // use flags. 3344 unsigned CCOpNo; 3345 switch (UIOpc) { 3346 default: 3347 // Something unusual. Be conservative. 3348 return false; 3349 case X86ISD::SETCC: CCOpNo = 0; break; 3350 case X86ISD::SETCC_CARRY: CCOpNo = 0; break; 3351 case X86ISD::CMOV: CCOpNo = 2; break; 3352 case X86ISD::BRCOND: CCOpNo = 2; break; 3353 } 3354 3355 X86::CondCode CC = (X86::CondCode)UI->getConstantOperandVal(CCOpNo); 3356 if (mayUseCarryFlag(CC)) 3357 return false; 3358 } 3359 return true; 3360 } 3361 3362 /// Check whether or not the chain ending in StoreNode is suitable for doing 3363 /// the {load; op; store} to modify transformation. 3364 static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, 3365 SDValue StoredVal, SelectionDAG *CurDAG, 3366 unsigned LoadOpNo, 3367 LoadSDNode *&LoadNode, 3368 SDValue &InputChain) { 3369 // Is the stored value result 0 of the operation? 3370 if (StoredVal.getResNo() != 0) return false; 3371 3372 // Are there other uses of the operation other than the store? 3373 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) return false; 3374 3375 // Is the store non-extending and non-indexed? 3376 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal()) 3377 return false; 3378 3379 SDValue Load = StoredVal->getOperand(LoadOpNo); 3380 // Is the stored value a non-extending and non-indexed load? 3381 if (!ISD::isNormalLoad(Load.getNode())) return false; 3382 3383 // Return LoadNode by reference. 3384 LoadNode = cast<LoadSDNode>(Load); 3385 3386 // Is store the only read of the loaded value? 3387 if (!Load.hasOneUse()) 3388 return false; 3389 3390 // Is the address of the store the same as the load? 3391 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() || 3392 LoadNode->getOffset() != StoreNode->getOffset()) 3393 return false; 3394 3395 bool FoundLoad = false; 3396 SmallVector<SDValue, 4> ChainOps; 3397 SmallVector<const SDNode *, 4> LoopWorklist; 3398 SmallPtrSet<const SDNode *, 16> Visited; 3399 const unsigned int Max = 1024; 3400 3401 // Visualization of Load-Op-Store fusion: 3402 // ------------------------- 3403 // Legend: 3404 // *-lines = Chain operand dependencies. 3405 // |-lines = Normal operand dependencies. 3406 // Dependencies flow down and right. n-suffix references multiple nodes. 3407 // 3408 // C Xn C 3409 // * * * 3410 // * * * 3411 // Xn A-LD Yn TF Yn 3412 // * * \ | * | 3413 // * * \ | * | 3414 // * * \ | => A--LD_OP_ST 3415 // * * \| \ 3416 // TF OP \ 3417 // * | \ Zn 3418 // * | \ 3419 // A-ST Zn 3420 // 3421 3422 // This merge induced dependences from: #1: Xn -> LD, OP, Zn 3423 // #2: Yn -> LD 3424 // #3: ST -> Zn 3425 3426 // Ensure the transform is safe by checking for the dual 3427 // dependencies to make sure we do not induce a loop. 3428 3429 // As LD is a predecessor to both OP and ST we can do this by checking: 3430 // a). if LD is a predecessor to a member of Xn or Yn. 3431 // b). if a Zn is a predecessor to ST. 3432 3433 // However, (b) can only occur through being a chain predecessor to 3434 // ST, which is the same as Zn being a member or predecessor of Xn, 3435 // which is a subset of LD being a predecessor of Xn. So it's 3436 // subsumed by check (a). 3437 3438 SDValue Chain = StoreNode->getChain(); 3439 3440 // Gather X elements in ChainOps. 3441 if (Chain == Load.getValue(1)) { 3442 FoundLoad = true; 3443 ChainOps.push_back(Load.getOperand(0)); 3444 } else if (Chain.getOpcode() == ISD::TokenFactor) { 3445 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) { 3446 SDValue Op = Chain.getOperand(i); 3447 if (Op == Load.getValue(1)) { 3448 FoundLoad = true; 3449 // Drop Load, but keep its chain. No cycle check necessary. 3450 ChainOps.push_back(Load.getOperand(0)); 3451 continue; 3452 } 3453 LoopWorklist.push_back(Op.getNode()); 3454 ChainOps.push_back(Op); 3455 } 3456 } 3457 3458 if (!FoundLoad) 3459 return false; 3460 3461 // Worklist is currently Xn. Add Yn to worklist. 3462 for (SDValue Op : StoredVal->ops()) 3463 if (Op.getNode() != LoadNode) 3464 LoopWorklist.push_back(Op.getNode()); 3465 3466 // Check (a) if Load is a predecessor to Xn + Yn 3467 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max, 3468 true)) 3469 return false; 3470 3471 InputChain = 3472 CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ChainOps); 3473 return true; 3474 } 3475 3476 // Change a chain of {load; op; store} of the same value into a simple op 3477 // through memory of that value, if the uses of the modified value and its 3478 // address are suitable. 3479 // 3480 // The tablegen pattern memory operand pattern is currently not able to match 3481 // the case where the EFLAGS on the original operation are used. 3482 // 3483 // To move this to tablegen, we'll need to improve tablegen to allow flags to 3484 // be transferred from a node in the pattern to the result node, probably with 3485 // a new keyword. For example, we have this 3486 // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst", 3487 // [(store (add (loadi64 addr:$dst), -1), addr:$dst), 3488 // (implicit EFLAGS)]>; 3489 // but maybe need something like this 3490 // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst", 3491 // [(store (add (loadi64 addr:$dst), -1), addr:$dst), 3492 // (transferrable EFLAGS)]>; 3493 // 3494 // Until then, we manually fold these and instruction select the operation 3495 // here. 3496 bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) { 3497 auto *StoreNode = cast<StoreSDNode>(Node); 3498 SDValue StoredVal = StoreNode->getOperand(1); 3499 unsigned Opc = StoredVal->getOpcode(); 3500 3501 // Before we try to select anything, make sure this is memory operand size 3502 // and opcode we can handle. Note that this must match the code below that 3503 // actually lowers the opcodes. 3504 EVT MemVT = StoreNode->getMemoryVT(); 3505 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 && 3506 MemVT != MVT::i8) 3507 return false; 3508 3509 bool IsCommutable = false; 3510 bool IsNegate = false; 3511 switch (Opc) { 3512 default: 3513 return false; 3514 case X86ISD::SUB: 3515 IsNegate = isNullConstant(StoredVal.getOperand(0)); 3516 break; 3517 case X86ISD::SBB: 3518 break; 3519 case X86ISD::ADD: 3520 case X86ISD::ADC: 3521 case X86ISD::AND: 3522 case X86ISD::OR: 3523 case X86ISD::XOR: 3524 IsCommutable = true; 3525 break; 3526 } 3527 3528 unsigned LoadOpNo = IsNegate ? 1 : 0; 3529 LoadSDNode *LoadNode = nullptr; 3530 SDValue InputChain; 3531 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadOpNo, 3532 LoadNode, InputChain)) { 3533 if (!IsCommutable) 3534 return false; 3535 3536 // This operation is commutable, try the other operand. 3537 LoadOpNo = 1; 3538 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadOpNo, 3539 LoadNode, InputChain)) 3540 return false; 3541 } 3542 3543 SDValue Base, Scale, Index, Disp, Segment; 3544 if (!selectAddr(LoadNode, LoadNode->getBasePtr(), Base, Scale, Index, Disp, 3545 Segment)) 3546 return false; 3547 3548 auto SelectOpcode = [&](unsigned Opc64, unsigned Opc32, unsigned Opc16, 3549 unsigned Opc8) { 3550 switch (MemVT.getSimpleVT().SimpleTy) { 3551 case MVT::i64: 3552 return Opc64; 3553 case MVT::i32: 3554 return Opc32; 3555 case MVT::i16: 3556 return Opc16; 3557 case MVT::i8: 3558 return Opc8; 3559 default: 3560 llvm_unreachable("Invalid size!"); 3561 } 3562 }; 3563 3564 MachineSDNode *Result; 3565 switch (Opc) { 3566 case X86ISD::SUB: 3567 // Handle negate. 3568 if (IsNegate) { 3569 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m, 3570 X86::NEG8m); 3571 const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain}; 3572 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, 3573 MVT::Other, Ops); 3574 break; 3575 } 3576 [[fallthrough]]; 3577 case X86ISD::ADD: 3578 // Try to match inc/dec. 3579 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) { 3580 bool IsOne = isOneConstant(StoredVal.getOperand(1)); 3581 bool IsNegOne = isAllOnesConstant(StoredVal.getOperand(1)); 3582 // ADD/SUB with 1/-1 and carry flag isn't used can use inc/dec. 3583 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.getValue(1))) { 3584 unsigned NewOpc = 3585 ((Opc == X86ISD::ADD) == IsOne) 3586 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m) 3587 : SelectOpcode(X86::DEC64m, X86::DEC32m, X86::DEC16m, X86::DEC8m); 3588 const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain}; 3589 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, 3590 MVT::Other, Ops); 3591 break; 3592 } 3593 } 3594 [[fallthrough]]; 3595 case X86ISD::ADC: 3596 case X86ISD::SBB: 3597 case X86ISD::AND: 3598 case X86ISD::OR: 3599 case X86ISD::XOR: { 3600 auto SelectRegOpcode = [SelectOpcode](unsigned Opc) { 3601 switch (Opc) { 3602 case X86ISD::ADD: 3603 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr, 3604 X86::ADD8mr); 3605 case X86ISD::ADC: 3606 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr, 3607 X86::ADC8mr); 3608 case X86ISD::SUB: 3609 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr, 3610 X86::SUB8mr); 3611 case X86ISD::SBB: 3612 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr, 3613 X86::SBB8mr); 3614 case X86ISD::AND: 3615 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr, 3616 X86::AND8mr); 3617 case X86ISD::OR: 3618 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr); 3619 case X86ISD::XOR: 3620 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr, 3621 X86::XOR8mr); 3622 default: 3623 llvm_unreachable("Invalid opcode!"); 3624 } 3625 }; 3626 auto SelectImmOpcode = [SelectOpcode](unsigned Opc) { 3627 switch (Opc) { 3628 case X86ISD::ADD: 3629 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi, 3630 X86::ADD8mi); 3631 case X86ISD::ADC: 3632 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi, 3633 X86::ADC8mi); 3634 case X86ISD::SUB: 3635 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi, 3636 X86::SUB8mi); 3637 case X86ISD::SBB: 3638 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi, 3639 X86::SBB8mi); 3640 case X86ISD::AND: 3641 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi, 3642 X86::AND8mi); 3643 case X86ISD::OR: 3644 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi, 3645 X86::OR8mi); 3646 case X86ISD::XOR: 3647 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi, 3648 X86::XOR8mi); 3649 default: 3650 llvm_unreachable("Invalid opcode!"); 3651 } 3652 }; 3653 3654 unsigned NewOpc = SelectRegOpcode(Opc); 3655 SDValue Operand = StoredVal->getOperand(1-LoadOpNo); 3656 3657 // See if the operand is a constant that we can fold into an immediate 3658 // operand. 3659 if (auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) { 3660 int64_t OperandV = OperandC->getSExtValue(); 3661 3662 // Check if we can shrink the operand enough to fit in an immediate (or 3663 // fit into a smaller immediate) by negating it and switching the 3664 // operation. 3665 if ((Opc == X86ISD::ADD || Opc == X86ISD::SUB) && 3666 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) || 3667 (MemVT == MVT::i64 && !isInt<32>(OperandV) && 3668 isInt<32>(-OperandV))) && 3669 hasNoCarryFlagUses(StoredVal.getValue(1))) { 3670 OperandV = -OperandV; 3671 Opc = Opc == X86ISD::ADD ? X86ISD::SUB : X86ISD::ADD; 3672 } 3673 3674 if (MemVT != MVT::i64 || isInt<32>(OperandV)) { 3675 Operand = CurDAG->getTargetConstant(OperandV, SDLoc(Node), MemVT); 3676 NewOpc = SelectImmOpcode(Opc); 3677 } 3678 } 3679 3680 if (Opc == X86ISD::ADC || Opc == X86ISD::SBB) { 3681 SDValue CopyTo = 3682 CurDAG->getCopyToReg(InputChain, SDLoc(Node), X86::EFLAGS, 3683 StoredVal.getOperand(2), SDValue()); 3684 3685 const SDValue Ops[] = {Base, Scale, Index, Disp, 3686 Segment, Operand, CopyTo, CopyTo.getValue(1)}; 3687 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, MVT::Other, 3688 Ops); 3689 } else { 3690 const SDValue Ops[] = {Base, Scale, Index, Disp, 3691 Segment, Operand, InputChain}; 3692 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, MVT::Other, 3693 Ops); 3694 } 3695 break; 3696 } 3697 default: 3698 llvm_unreachable("Invalid opcode!"); 3699 } 3700 3701 MachineMemOperand *MemOps[] = {StoreNode->getMemOperand(), 3702 LoadNode->getMemOperand()}; 3703 CurDAG->setNodeMemRefs(Result, MemOps); 3704 3705 // Update Load Chain uses as well. 3706 ReplaceUses(SDValue(LoadNode, 1), SDValue(Result, 1)); 3707 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1)); 3708 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0)); 3709 CurDAG->RemoveDeadNode(Node); 3710 return true; 3711 } 3712 3713 // See if this is an X & Mask that we can match to BEXTR/BZHI. 3714 // Where Mask is one of the following patterns: 3715 // a) x & (1 << nbits) - 1 3716 // b) x & ~(-1 << nbits) 3717 // c) x & (-1 >> (32 - y)) 3718 // d) x << (32 - y) >> (32 - y) 3719 // e) (1 << nbits) - 1 3720 bool X86DAGToDAGISel::matchBitExtract(SDNode *Node) { 3721 assert( 3722 (Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::AND || 3723 Node->getOpcode() == ISD::SRL) && 3724 "Should be either an and-mask, or right-shift after clearing high bits."); 3725 3726 // BEXTR is BMI instruction, BZHI is BMI2 instruction. We need at least one. 3727 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2()) 3728 return false; 3729 3730 MVT NVT = Node->getSimpleValueType(0); 3731 3732 // Only supported for 32 and 64 bits. 3733 if (NVT != MVT::i32 && NVT != MVT::i64) 3734 return false; 3735 3736 SDValue NBits; 3737 bool NegateNBits; 3738 3739 // If we have BMI2's BZHI, we are ok with muti-use patterns. 3740 // Else, if we only have BMI1's BEXTR, we require one-use. 3741 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2(); 3742 auto checkUses = [AllowExtraUsesByDefault]( 3743 SDValue Op, unsigned NUses, 3744 std::optional<bool> AllowExtraUses) { 3745 return AllowExtraUses.value_or(AllowExtraUsesByDefault) || 3746 Op.getNode()->hasNUsesOfValue(NUses, Op.getResNo()); 3747 }; 3748 auto checkOneUse = [checkUses](SDValue Op, 3749 std::optional<bool> AllowExtraUses = 3750 std::nullopt) { 3751 return checkUses(Op, 1, AllowExtraUses); 3752 }; 3753 auto checkTwoUse = [checkUses](SDValue Op, 3754 std::optional<bool> AllowExtraUses = 3755 std::nullopt) { 3756 return checkUses(Op, 2, AllowExtraUses); 3757 }; 3758 3759 auto peekThroughOneUseTruncation = [checkOneUse](SDValue V) { 3760 if (V->getOpcode() == ISD::TRUNCATE && checkOneUse(V)) { 3761 assert(V.getSimpleValueType() == MVT::i32 && 3762 V.getOperand(0).getSimpleValueType() == MVT::i64 && 3763 "Expected i64 -> i32 truncation"); 3764 V = V.getOperand(0); 3765 } 3766 return V; 3767 }; 3768 3769 // a) x & ((1 << nbits) + (-1)) 3770 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits, 3771 &NegateNBits](SDValue Mask) -> bool { 3772 // Match `add`. Must only have one use! 3773 if (Mask->getOpcode() != ISD::ADD || !checkOneUse(Mask)) 3774 return false; 3775 // We should be adding all-ones constant (i.e. subtracting one.) 3776 if (!isAllOnesConstant(Mask->getOperand(1))) 3777 return false; 3778 // Match `1 << nbits`. Might be truncated. Must only have one use! 3779 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0)); 3780 if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0)) 3781 return false; 3782 if (!isOneConstant(M0->getOperand(0))) 3783 return false; 3784 NBits = M0->getOperand(1); 3785 NegateNBits = false; 3786 return true; 3787 }; 3788 3789 auto isAllOnes = [this, peekThroughOneUseTruncation, NVT](SDValue V) { 3790 V = peekThroughOneUseTruncation(V); 3791 return CurDAG->MaskedValueIsAllOnes( 3792 V, APInt::getLowBitsSet(V.getSimpleValueType().getSizeInBits(), 3793 NVT.getSizeInBits())); 3794 }; 3795 3796 // b) x & ~(-1 << nbits) 3797 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation, 3798 &NBits, &NegateNBits](SDValue Mask) -> bool { 3799 // Match `~()`. Must only have one use! 3800 if (Mask.getOpcode() != ISD::XOR || !checkOneUse(Mask)) 3801 return false; 3802 // The -1 only has to be all-ones for the final Node's NVT. 3803 if (!isAllOnes(Mask->getOperand(1))) 3804 return false; 3805 // Match `-1 << nbits`. Might be truncated. Must only have one use! 3806 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0)); 3807 if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0)) 3808 return false; 3809 // The -1 only has to be all-ones for the final Node's NVT. 3810 if (!isAllOnes(M0->getOperand(0))) 3811 return false; 3812 NBits = M0->getOperand(1); 3813 NegateNBits = false; 3814 return true; 3815 }; 3816 3817 // Try to match potentially-truncated shift amount as `(bitwidth - y)`, 3818 // or leave the shift amount as-is, but then we'll have to negate it. 3819 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](SDValue ShiftAmt, 3820 unsigned Bitwidth) { 3821 NBits = ShiftAmt; 3822 NegateNBits = true; 3823 // Skip over a truncate of the shift amount, if any. 3824 if (NBits.getOpcode() == ISD::TRUNCATE) 3825 NBits = NBits.getOperand(0); 3826 // Try to match the shift amount as (bitwidth - y). It should go away, too. 3827 // If it doesn't match, that's fine, we'll just negate it ourselves. 3828 if (NBits.getOpcode() != ISD::SUB) 3829 return; 3830 auto *V0 = dyn_cast<ConstantSDNode>(NBits.getOperand(0)); 3831 if (!V0 || V0->getZExtValue() != Bitwidth) 3832 return; 3833 NBits = NBits.getOperand(1); 3834 NegateNBits = false; 3835 }; 3836 3837 // c) x & (-1 >> z) but then we'll have to subtract z from bitwidth 3838 // or 3839 // c) x & (-1 >> (32 - y)) 3840 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits, 3841 canonicalizeShiftAmt](SDValue Mask) -> bool { 3842 // The mask itself may be truncated. 3843 Mask = peekThroughOneUseTruncation(Mask); 3844 unsigned Bitwidth = Mask.getSimpleValueType().getSizeInBits(); 3845 // Match `l>>`. Must only have one use! 3846 if (Mask.getOpcode() != ISD::SRL || !checkOneUse(Mask)) 3847 return false; 3848 // We should be shifting truly all-ones constant. 3849 if (!isAllOnesConstant(Mask.getOperand(0))) 3850 return false; 3851 SDValue M1 = Mask.getOperand(1); 3852 // The shift amount should not be used externally. 3853 if (!checkOneUse(M1)) 3854 return false; 3855 canonicalizeShiftAmt(M1, Bitwidth); 3856 // Pattern c. is non-canonical, and is expanded into pattern d. iff there 3857 // is no extra use of the mask. Clearly, there was one since we are here. 3858 // But at the same time, if we need to negate the shift amount, 3859 // then we don't want the mask to stick around, else it's unprofitable. 3860 return !NegateNBits; 3861 }; 3862 3863 SDValue X; 3864 3865 // d) x << z >> z but then we'll have to subtract z from bitwidth 3866 // or 3867 // d) x << (32 - y) >> (32 - y) 3868 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt, 3869 AllowExtraUsesByDefault, &NegateNBits, 3870 &X](SDNode *Node) -> bool { 3871 if (Node->getOpcode() != ISD::SRL) 3872 return false; 3873 SDValue N0 = Node->getOperand(0); 3874 if (N0->getOpcode() != ISD::SHL) 3875 return false; 3876 unsigned Bitwidth = N0.getSimpleValueType().getSizeInBits(); 3877 SDValue N1 = Node->getOperand(1); 3878 SDValue N01 = N0->getOperand(1); 3879 // Both of the shifts must be by the exact same value. 3880 if (N1 != N01) 3881 return false; 3882 canonicalizeShiftAmt(N1, Bitwidth); 3883 // There should not be any external uses of the inner shift / shift amount. 3884 // Note that while we are generally okay with external uses given BMI2, 3885 // iff we need to negate the shift amount, we are not okay with extra uses. 3886 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits; 3887 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses)) 3888 return false; 3889 X = N0->getOperand(0); 3890 return true; 3891 }; 3892 3893 auto matchLowBitMask = [matchPatternA, matchPatternB, 3894 matchPatternC](SDValue Mask) -> bool { 3895 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask); 3896 }; 3897 3898 if (Node->getOpcode() == ISD::AND) { 3899 X = Node->getOperand(0); 3900 SDValue Mask = Node->getOperand(1); 3901 3902 if (matchLowBitMask(Mask)) { 3903 // Great. 3904 } else { 3905 std::swap(X, Mask); 3906 if (!matchLowBitMask(Mask)) 3907 return false; 3908 } 3909 } else if (matchLowBitMask(SDValue(Node, 0))) { 3910 X = CurDAG->getAllOnesConstant(SDLoc(Node), NVT); 3911 } else if (!matchPatternD(Node)) 3912 return false; 3913 3914 // If we need to negate the shift amount, require BMI2 BZHI support. 3915 // It's just too unprofitable for BMI1 BEXTR. 3916 if (NegateNBits && !Subtarget->hasBMI2()) 3917 return false; 3918 3919 SDLoc DL(Node); 3920 3921 // Truncate the shift amount. 3922 NBits = CurDAG->getNode(ISD::TRUNCATE, DL, MVT::i8, NBits); 3923 insertDAGNode(*CurDAG, SDValue(Node, 0), NBits); 3924 3925 // Insert 8-bit NBits into lowest 8 bits of 32-bit register. 3926 // All the other bits are undefined, we do not care about them. 3927 SDValue ImplDef = SDValue( 3928 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::i32), 0); 3929 insertDAGNode(*CurDAG, SDValue(Node, 0), ImplDef); 3930 3931 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit, DL, MVT::i32); 3932 insertDAGNode(*CurDAG, SDValue(Node, 0), SRIdxVal); 3933 NBits = SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 3934 MVT::i32, ImplDef, NBits, SRIdxVal), 3935 0); 3936 insertDAGNode(*CurDAG, SDValue(Node, 0), NBits); 3937 3938 // We might have matched the amount of high bits to be cleared, 3939 // but we want the amount of low bits to be kept, so negate it then. 3940 if (NegateNBits) { 3941 SDValue BitWidthC = CurDAG->getConstant(NVT.getSizeInBits(), DL, MVT::i32); 3942 insertDAGNode(*CurDAG, SDValue(Node, 0), BitWidthC); 3943 3944 NBits = CurDAG->getNode(ISD::SUB, DL, MVT::i32, BitWidthC, NBits); 3945 insertDAGNode(*CurDAG, SDValue(Node, 0), NBits); 3946 } 3947 3948 if (Subtarget->hasBMI2()) { 3949 // Great, just emit the BZHI.. 3950 if (NVT != MVT::i32) { 3951 // But have to place the bit count into the wide-enough register first. 3952 NBits = CurDAG->getNode(ISD::ANY_EXTEND, DL, NVT, NBits); 3953 insertDAGNode(*CurDAG, SDValue(Node, 0), NBits); 3954 } 3955 3956 SDValue Extract = CurDAG->getNode(X86ISD::BZHI, DL, NVT, X, NBits); 3957 ReplaceNode(Node, Extract.getNode()); 3958 SelectCode(Extract.getNode()); 3959 return true; 3960 } 3961 3962 // Else, if we do *NOT* have BMI2, let's find out if the if the 'X' is 3963 // *logically* shifted (potentially with one-use trunc inbetween), 3964 // and the truncation was the only use of the shift, 3965 // and if so look past one-use truncation. 3966 { 3967 SDValue RealX = peekThroughOneUseTruncation(X); 3968 // FIXME: only if the shift is one-use? 3969 if (RealX != X && RealX.getOpcode() == ISD::SRL) 3970 X = RealX; 3971 } 3972 3973 MVT XVT = X.getSimpleValueType(); 3974 3975 // Else, emitting BEXTR requires one more step. 3976 // The 'control' of BEXTR has the pattern of: 3977 // [15...8 bit][ 7...0 bit] location 3978 // [ bit count][ shift] name 3979 // I.e. 0b000000011'00000001 means (x >> 0b1) & 0b11 3980 3981 // Shift NBits left by 8 bits, thus producing 'control'. 3982 // This makes the low 8 bits to be zero. 3983 SDValue C8 = CurDAG->getConstant(8, DL, MVT::i8); 3984 insertDAGNode(*CurDAG, SDValue(Node, 0), C8); 3985 SDValue Control = CurDAG->getNode(ISD::SHL, DL, MVT::i32, NBits, C8); 3986 insertDAGNode(*CurDAG, SDValue(Node, 0), Control); 3987 3988 // If the 'X' is *logically* shifted, we can fold that shift into 'control'. 3989 // FIXME: only if the shift is one-use? 3990 if (X.getOpcode() == ISD::SRL) { 3991 SDValue ShiftAmt = X.getOperand(1); 3992 X = X.getOperand(0); 3993 3994 assert(ShiftAmt.getValueType() == MVT::i8 && 3995 "Expected shift amount to be i8"); 3996 3997 // Now, *zero*-extend the shift amount. The bits 8...15 *must* be zero! 3998 // We could zext to i16 in some form, but we intentionally don't do that. 3999 SDValue OrigShiftAmt = ShiftAmt; 4000 ShiftAmt = CurDAG->getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShiftAmt); 4001 insertDAGNode(*CurDAG, OrigShiftAmt, ShiftAmt); 4002 4003 // And now 'or' these low 8 bits of shift amount into the 'control'. 4004 Control = CurDAG->getNode(ISD::OR, DL, MVT::i32, Control, ShiftAmt); 4005 insertDAGNode(*CurDAG, SDValue(Node, 0), Control); 4006 } 4007 4008 // But have to place the 'control' into the wide-enough register first. 4009 if (XVT != MVT::i32) { 4010 Control = CurDAG->getNode(ISD::ANY_EXTEND, DL, XVT, Control); 4011 insertDAGNode(*CurDAG, SDValue(Node, 0), Control); 4012 } 4013 4014 // And finally, form the BEXTR itself. 4015 SDValue Extract = CurDAG->getNode(X86ISD::BEXTR, DL, XVT, X, Control); 4016 4017 // The 'X' was originally truncated. Do that now. 4018 if (XVT != NVT) { 4019 insertDAGNode(*CurDAG, SDValue(Node, 0), Extract); 4020 Extract = CurDAG->getNode(ISD::TRUNCATE, DL, NVT, Extract); 4021 } 4022 4023 ReplaceNode(Node, Extract.getNode()); 4024 SelectCode(Extract.getNode()); 4025 4026 return true; 4027 } 4028 4029 // See if this is an (X >> C1) & C2 that we can match to BEXTR/BEXTRI. 4030 MachineSDNode *X86DAGToDAGISel::matchBEXTRFromAndImm(SDNode *Node) { 4031 MVT NVT = Node->getSimpleValueType(0); 4032 SDLoc dl(Node); 4033 4034 SDValue N0 = Node->getOperand(0); 4035 SDValue N1 = Node->getOperand(1); 4036 4037 // If we have TBM we can use an immediate for the control. If we have BMI 4038 // we should only do this if the BEXTR instruction is implemented well. 4039 // Otherwise moving the control into a register makes this more costly. 4040 // TODO: Maybe load folding, greater than 32-bit masks, or a guarantee of LICM 4041 // hoisting the move immediate would make it worthwhile with a less optimal 4042 // BEXTR? 4043 bool PreferBEXTR = 4044 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR()); 4045 if (!PreferBEXTR && !Subtarget->hasBMI2()) 4046 return nullptr; 4047 4048 // Must have a shift right. 4049 if (N0->getOpcode() != ISD::SRL && N0->getOpcode() != ISD::SRA) 4050 return nullptr; 4051 4052 // Shift can't have additional users. 4053 if (!N0->hasOneUse()) 4054 return nullptr; 4055 4056 // Only supported for 32 and 64 bits. 4057 if (NVT != MVT::i32 && NVT != MVT::i64) 4058 return nullptr; 4059 4060 // Shift amount and RHS of and must be constant. 4061 auto *MaskCst = dyn_cast<ConstantSDNode>(N1); 4062 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 4063 if (!MaskCst || !ShiftCst) 4064 return nullptr; 4065 4066 // And RHS must be a mask. 4067 uint64_t Mask = MaskCst->getZExtValue(); 4068 if (!isMask_64(Mask)) 4069 return nullptr; 4070 4071 uint64_t Shift = ShiftCst->getZExtValue(); 4072 uint64_t MaskSize = llvm::popcount(Mask); 4073 4074 // Don't interfere with something that can be handled by extracting AH. 4075 // TODO: If we are able to fold a load, BEXTR might still be better than AH. 4076 if (Shift == 8 && MaskSize == 8) 4077 return nullptr; 4078 4079 // Make sure we are only using bits that were in the original value, not 4080 // shifted in. 4081 if (Shift + MaskSize > NVT.getSizeInBits()) 4082 return nullptr; 4083 4084 // BZHI, if available, is always fast, unlike BEXTR. But even if we decide 4085 // that we can't use BEXTR, it is only worthwhile using BZHI if the mask 4086 // does not fit into 32 bits. Load folding is not a sufficient reason. 4087 if (!PreferBEXTR && MaskSize <= 32) 4088 return nullptr; 4089 4090 SDValue Control; 4091 unsigned ROpc, MOpc; 4092 4093 if (!PreferBEXTR) { 4094 assert(Subtarget->hasBMI2() && "We must have BMI2's BZHI then."); 4095 // If we can't make use of BEXTR then we can't fuse shift+mask stages. 4096 // Let's perform the mask first, and apply shift later. Note that we need to 4097 // widen the mask to account for the fact that we'll apply shift afterwards! 4098 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT); 4099 ROpc = NVT == MVT::i64 ? X86::BZHI64rr : X86::BZHI32rr; 4100 MOpc = NVT == MVT::i64 ? X86::BZHI64rm : X86::BZHI32rm; 4101 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri; 4102 Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0); 4103 } else { 4104 // The 'control' of BEXTR has the pattern of: 4105 // [15...8 bit][ 7...0 bit] location 4106 // [ bit count][ shift] name 4107 // I.e. 0b000000011'00000001 means (x >> 0b1) & 0b11 4108 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT); 4109 if (Subtarget->hasTBM()) { 4110 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri; 4111 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi; 4112 } else { 4113 assert(Subtarget->hasBMI() && "We must have BMI1's BEXTR then."); 4114 // BMI requires the immediate to placed in a register. 4115 ROpc = NVT == MVT::i64 ? X86::BEXTR64rr : X86::BEXTR32rr; 4116 MOpc = NVT == MVT::i64 ? X86::BEXTR64rm : X86::BEXTR32rm; 4117 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri; 4118 Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0); 4119 } 4120 } 4121 4122 MachineSDNode *NewNode; 4123 SDValue Input = N0->getOperand(0); 4124 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 4125 if (tryFoldLoad(Node, N0.getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 4126 SDValue Ops[] = { 4127 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.getOperand(0)}; 4128 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other); 4129 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 4130 // Update the chain. 4131 ReplaceUses(Input.getValue(1), SDValue(NewNode, 2)); 4132 // Record the mem-refs 4133 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()}); 4134 } else { 4135 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control); 4136 } 4137 4138 if (!PreferBEXTR) { 4139 // We still need to apply the shift. 4140 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT); 4141 unsigned NewOpc = NVT == MVT::i64 ? X86::SHR64ri : X86::SHR32ri; 4142 NewNode = 4143 CurDAG->getMachineNode(NewOpc, dl, NVT, SDValue(NewNode, 0), ShAmt); 4144 } 4145 4146 return NewNode; 4147 } 4148 4149 // Emit a PCMISTR(I/M) instruction. 4150 MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(unsigned ROpc, unsigned MOpc, 4151 bool MayFoldLoad, const SDLoc &dl, 4152 MVT VT, SDNode *Node) { 4153 SDValue N0 = Node->getOperand(0); 4154 SDValue N1 = Node->getOperand(1); 4155 SDValue Imm = Node->getOperand(2); 4156 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue(); 4157 Imm = CurDAG->getTargetConstant(*Val, SDLoc(Node), Imm.getValueType()); 4158 4159 // Try to fold a load. No need to check alignment. 4160 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 4161 if (MayFoldLoad && tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 4162 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm, 4163 N1.getOperand(0) }; 4164 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other); 4165 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 4166 // Update the chain. 4167 ReplaceUses(N1.getValue(1), SDValue(CNode, 2)); 4168 // Record the mem-refs 4169 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); 4170 return CNode; 4171 } 4172 4173 SDValue Ops[] = { N0, N1, Imm }; 4174 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32); 4175 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops); 4176 return CNode; 4177 } 4178 4179 // Emit a PCMESTR(I/M) instruction. Also return the Glue result in case we need 4180 // to emit a second instruction after this one. This is needed since we have two 4181 // copyToReg nodes glued before this and we need to continue that glue through. 4182 MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(unsigned ROpc, unsigned MOpc, 4183 bool MayFoldLoad, const SDLoc &dl, 4184 MVT VT, SDNode *Node, 4185 SDValue &InGlue) { 4186 SDValue N0 = Node->getOperand(0); 4187 SDValue N2 = Node->getOperand(2); 4188 SDValue Imm = Node->getOperand(4); 4189 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue(); 4190 Imm = CurDAG->getTargetConstant(*Val, SDLoc(Node), Imm.getValueType()); 4191 4192 // Try to fold a load. No need to check alignment. 4193 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 4194 if (MayFoldLoad && tryFoldLoad(Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 4195 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm, 4196 N2.getOperand(0), InGlue }; 4197 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue); 4198 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 4199 InGlue = SDValue(CNode, 3); 4200 // Update the chain. 4201 ReplaceUses(N2.getValue(1), SDValue(CNode, 2)); 4202 // Record the mem-refs 4203 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()}); 4204 return CNode; 4205 } 4206 4207 SDValue Ops[] = { N0, N2, Imm, InGlue }; 4208 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue); 4209 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops); 4210 InGlue = SDValue(CNode, 2); 4211 return CNode; 4212 } 4213 4214 bool X86DAGToDAGISel::tryShiftAmountMod(SDNode *N) { 4215 EVT VT = N->getValueType(0); 4216 4217 // Only handle scalar shifts. 4218 if (VT.isVector()) 4219 return false; 4220 4221 // Narrower shifts only mask to 5 bits in hardware. 4222 unsigned Size = VT == MVT::i64 ? 64 : 32; 4223 4224 SDValue OrigShiftAmt = N->getOperand(1); 4225 SDValue ShiftAmt = OrigShiftAmt; 4226 SDLoc DL(N); 4227 4228 // Skip over a truncate of the shift amount. 4229 if (ShiftAmt->getOpcode() == ISD::TRUNCATE) 4230 ShiftAmt = ShiftAmt->getOperand(0); 4231 4232 // This function is called after X86DAGToDAGISel::matchBitExtract(), 4233 // so we are not afraid that we might mess up BZHI/BEXTR pattern. 4234 4235 SDValue NewShiftAmt; 4236 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB || 4237 ShiftAmt->getOpcode() == ISD::XOR) { 4238 SDValue Add0 = ShiftAmt->getOperand(0); 4239 SDValue Add1 = ShiftAmt->getOperand(1); 4240 auto *Add0C = dyn_cast<ConstantSDNode>(Add0); 4241 auto *Add1C = dyn_cast<ConstantSDNode>(Add1); 4242 // If we are shifting by X+/-/^N where N == 0 mod Size, then just shift by X 4243 // to avoid the ADD/SUB/XOR. 4244 if (Add1C && Add1C->getAPIntValue().urem(Size) == 0) { 4245 NewShiftAmt = Add0; 4246 4247 } else if (ShiftAmt->getOpcode() != ISD::ADD && ShiftAmt.hasOneUse() && 4248 ((Add0C && Add0C->getAPIntValue().urem(Size) == Size - 1) || 4249 (Add1C && Add1C->getAPIntValue().urem(Size) == Size - 1))) { 4250 // If we are doing a NOT on just the lower bits with (Size*N-1) -/^ X 4251 // we can replace it with a NOT. In the XOR case it may save some code 4252 // size, in the SUB case it also may save a move. 4253 assert(Add0C == nullptr || Add1C == nullptr); 4254 4255 // We can only do N-X, not X-N 4256 if (ShiftAmt->getOpcode() == ISD::SUB && Add0C == nullptr) 4257 return false; 4258 4259 EVT OpVT = ShiftAmt.getValueType(); 4260 4261 SDValue AllOnes = CurDAG->getAllOnesConstant(DL, OpVT); 4262 NewShiftAmt = CurDAG->getNode(ISD::XOR, DL, OpVT, 4263 Add0C == nullptr ? Add0 : Add1, AllOnes); 4264 insertDAGNode(*CurDAG, OrigShiftAmt, AllOnes); 4265 insertDAGNode(*CurDAG, OrigShiftAmt, NewShiftAmt); 4266 // If we are shifting by N-X where N == 0 mod Size, then just shift by 4267 // -X to generate a NEG instead of a SUB of a constant. 4268 } else if (ShiftAmt->getOpcode() == ISD::SUB && Add0C && 4269 Add0C->getZExtValue() != 0) { 4270 EVT SubVT = ShiftAmt.getValueType(); 4271 SDValue X; 4272 if (Add0C->getZExtValue() % Size == 0) 4273 X = Add1; 4274 else if (ShiftAmt.hasOneUse() && Size == 64 && 4275 Add0C->getZExtValue() % 32 == 0) { 4276 // We have a 64-bit shift by (n*32-x), turn it into -(x+n*32). 4277 // This is mainly beneficial if we already compute (x+n*32). 4278 if (Add1.getOpcode() == ISD::TRUNCATE) { 4279 Add1 = Add1.getOperand(0); 4280 SubVT = Add1.getValueType(); 4281 } 4282 if (Add0.getValueType() != SubVT) { 4283 Add0 = CurDAG->getZExtOrTrunc(Add0, DL, SubVT); 4284 insertDAGNode(*CurDAG, OrigShiftAmt, Add0); 4285 } 4286 4287 X = CurDAG->getNode(ISD::ADD, DL, SubVT, Add1, Add0); 4288 insertDAGNode(*CurDAG, OrigShiftAmt, X); 4289 } else 4290 return false; 4291 // Insert a negate op. 4292 // TODO: This isn't guaranteed to replace the sub if there is a logic cone 4293 // that uses it that's not a shift. 4294 SDValue Zero = CurDAG->getConstant(0, DL, SubVT); 4295 SDValue Neg = CurDAG->getNode(ISD::SUB, DL, SubVT, Zero, X); 4296 NewShiftAmt = Neg; 4297 4298 // Insert these operands into a valid topological order so they can 4299 // get selected independently. 4300 insertDAGNode(*CurDAG, OrigShiftAmt, Zero); 4301 insertDAGNode(*CurDAG, OrigShiftAmt, Neg); 4302 } else 4303 return false; 4304 } else 4305 return false; 4306 4307 if (NewShiftAmt.getValueType() != MVT::i8) { 4308 // Need to truncate the shift amount. 4309 NewShiftAmt = CurDAG->getNode(ISD::TRUNCATE, DL, MVT::i8, NewShiftAmt); 4310 // Add to a correct topological ordering. 4311 insertDAGNode(*CurDAG, OrigShiftAmt, NewShiftAmt); 4312 } 4313 4314 // Insert a new mask to keep the shift amount legal. This should be removed 4315 // by isel patterns. 4316 NewShiftAmt = CurDAG->getNode(ISD::AND, DL, MVT::i8, NewShiftAmt, 4317 CurDAG->getConstant(Size - 1, DL, MVT::i8)); 4318 // Place in a correct topological ordering. 4319 insertDAGNode(*CurDAG, OrigShiftAmt, NewShiftAmt); 4320 4321 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(N, N->getOperand(0), 4322 NewShiftAmt); 4323 if (UpdatedNode != N) { 4324 // If we found an existing node, we should replace ourselves with that node 4325 // and wait for it to be selected after its other users. 4326 ReplaceNode(N, UpdatedNode); 4327 return true; 4328 } 4329 4330 // If the original shift amount is now dead, delete it so that we don't run 4331 // it through isel. 4332 if (OrigShiftAmt.getNode()->use_empty()) 4333 CurDAG->RemoveDeadNode(OrigShiftAmt.getNode()); 4334 4335 // Now that we've optimized the shift amount, defer to normal isel to get 4336 // load folding and legacy vs BMI2 selection without repeating it here. 4337 SelectCode(N); 4338 return true; 4339 } 4340 4341 bool X86DAGToDAGISel::tryShrinkShlLogicImm(SDNode *N) { 4342 MVT NVT = N->getSimpleValueType(0); 4343 unsigned Opcode = N->getOpcode(); 4344 SDLoc dl(N); 4345 4346 // For operations of the form (x << C1) op C2, check if we can use a smaller 4347 // encoding for C2 by transforming it into (x op (C2>>C1)) << C1. 4348 SDValue Shift = N->getOperand(0); 4349 SDValue N1 = N->getOperand(1); 4350 4351 auto *Cst = dyn_cast<ConstantSDNode>(N1); 4352 if (!Cst) 4353 return false; 4354 4355 int64_t Val = Cst->getSExtValue(); 4356 4357 // If we have an any_extend feeding the AND, look through it to see if there 4358 // is a shift behind it. But only if the AND doesn't use the extended bits. 4359 // FIXME: Generalize this to other ANY_EXTEND than i32 to i64? 4360 bool FoundAnyExtend = false; 4361 if (Shift.getOpcode() == ISD::ANY_EXTEND && Shift.hasOneUse() && 4362 Shift.getOperand(0).getSimpleValueType() == MVT::i32 && 4363 isUInt<32>(Val)) { 4364 FoundAnyExtend = true; 4365 Shift = Shift.getOperand(0); 4366 } 4367 4368 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse()) 4369 return false; 4370 4371 // i8 is unshrinkable, i16 should be promoted to i32. 4372 if (NVT != MVT::i32 && NVT != MVT::i64) 4373 return false; 4374 4375 auto *ShlCst = dyn_cast<ConstantSDNode>(Shift.getOperand(1)); 4376 if (!ShlCst) 4377 return false; 4378 4379 uint64_t ShAmt = ShlCst->getZExtValue(); 4380 4381 // Make sure that we don't change the operation by removing bits. 4382 // This only matters for OR and XOR, AND is unaffected. 4383 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1; 4384 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0) 4385 return false; 4386 4387 // Check the minimum bitwidth for the new constant. 4388 // TODO: Using 16 and 8 bit operations is also possible for or32 & xor32. 4389 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) { 4390 if (Opcode == ISD::AND) { 4391 // AND32ri is the same as AND64ri32 with zext imm. 4392 // Try this before sign extended immediates below. 4393 ShiftedVal = (uint64_t)Val >> ShAmt; 4394 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal)) 4395 return true; 4396 // Also swap order when the AND can become MOVZX. 4397 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX) 4398 return true; 4399 } 4400 ShiftedVal = Val >> ShAmt; 4401 if ((!isInt<8>(Val) && isInt<8>(ShiftedVal)) || 4402 (!isInt<32>(Val) && isInt<32>(ShiftedVal))) 4403 return true; 4404 if (Opcode != ISD::AND) { 4405 // MOV32ri+OR64r/XOR64r is cheaper than MOV64ri64+OR64rr/XOR64rr 4406 ShiftedVal = (uint64_t)Val >> ShAmt; 4407 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal)) 4408 return true; 4409 } 4410 return false; 4411 }; 4412 4413 int64_t ShiftedVal; 4414 if (!CanShrinkImmediate(ShiftedVal)) 4415 return false; 4416 4417 // Ok, we can reorder to get a smaller immediate. 4418 4419 // But, its possible the original immediate allowed an AND to become MOVZX. 4420 // Doing this late due to avoid the MakedValueIsZero call as late as 4421 // possible. 4422 if (Opcode == ISD::AND) { 4423 // Find the smallest zext this could possibly be. 4424 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits(); 4425 ZExtWidth = llvm::bit_ceil(std::max(ZExtWidth, 8U)); 4426 4427 // Figure out which bits need to be zero to achieve that mask. 4428 APInt NeededMask = APInt::getLowBitsSet(NVT.getSizeInBits(), 4429 ZExtWidth); 4430 NeededMask &= ~Cst->getAPIntValue(); 4431 4432 if (CurDAG->MaskedValueIsZero(N->getOperand(0), NeededMask)) 4433 return false; 4434 } 4435 4436 SDValue X = Shift.getOperand(0); 4437 if (FoundAnyExtend) { 4438 SDValue NewX = CurDAG->getNode(ISD::ANY_EXTEND, dl, NVT, X); 4439 insertDAGNode(*CurDAG, SDValue(N, 0), NewX); 4440 X = NewX; 4441 } 4442 4443 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT); 4444 insertDAGNode(*CurDAG, SDValue(N, 0), NewCst); 4445 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT, X, NewCst); 4446 insertDAGNode(*CurDAG, SDValue(N, 0), NewBinOp); 4447 SDValue NewSHL = CurDAG->getNode(ISD::SHL, dl, NVT, NewBinOp, 4448 Shift.getOperand(1)); 4449 ReplaceNode(N, NewSHL.getNode()); 4450 SelectCode(NewSHL.getNode()); 4451 return true; 4452 } 4453 4454 bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA, 4455 SDNode *ParentB, SDNode *ParentC, 4456 SDValue A, SDValue B, SDValue C, 4457 uint8_t Imm) { 4458 assert(A.isOperandOf(ParentA) && B.isOperandOf(ParentB) && 4459 C.isOperandOf(ParentC) && "Incorrect parent node"); 4460 4461 auto tryFoldLoadOrBCast = 4462 [this](SDNode *Root, SDNode *P, SDValue &L, SDValue &Base, SDValue &Scale, 4463 SDValue &Index, SDValue &Disp, SDValue &Segment) { 4464 if (tryFoldLoad(Root, P, L, Base, Scale, Index, Disp, Segment)) 4465 return true; 4466 4467 // Not a load, check for broadcast which may be behind a bitcast. 4468 if (L.getOpcode() == ISD::BITCAST && L.hasOneUse()) { 4469 P = L.getNode(); 4470 L = L.getOperand(0); 4471 } 4472 4473 if (L.getOpcode() != X86ISD::VBROADCAST_LOAD) 4474 return false; 4475 4476 // Only 32 and 64 bit broadcasts are supported. 4477 auto *MemIntr = cast<MemIntrinsicSDNode>(L); 4478 unsigned Size = MemIntr->getMemoryVT().getSizeInBits(); 4479 if (Size != 32 && Size != 64) 4480 return false; 4481 4482 return tryFoldBroadcast(Root, P, L, Base, Scale, Index, Disp, Segment); 4483 }; 4484 4485 bool FoldedLoad = false; 4486 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 4487 if (tryFoldLoadOrBCast(Root, ParentC, C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 4488 FoldedLoad = true; 4489 } else if (tryFoldLoadOrBCast(Root, ParentA, A, Tmp0, Tmp1, Tmp2, Tmp3, 4490 Tmp4)) { 4491 FoldedLoad = true; 4492 std::swap(A, C); 4493 // Swap bits 1/4 and 3/6. 4494 uint8_t OldImm = Imm; 4495 Imm = OldImm & 0xa5; 4496 if (OldImm & 0x02) Imm |= 0x10; 4497 if (OldImm & 0x10) Imm |= 0x02; 4498 if (OldImm & 0x08) Imm |= 0x40; 4499 if (OldImm & 0x40) Imm |= 0x08; 4500 } else if (tryFoldLoadOrBCast(Root, ParentB, B, Tmp0, Tmp1, Tmp2, Tmp3, 4501 Tmp4)) { 4502 FoldedLoad = true; 4503 std::swap(B, C); 4504 // Swap bits 1/2 and 5/6. 4505 uint8_t OldImm = Imm; 4506 Imm = OldImm & 0x99; 4507 if (OldImm & 0x02) Imm |= 0x04; 4508 if (OldImm & 0x04) Imm |= 0x02; 4509 if (OldImm & 0x20) Imm |= 0x40; 4510 if (OldImm & 0x40) Imm |= 0x20; 4511 } 4512 4513 SDLoc DL(Root); 4514 4515 SDValue TImm = CurDAG->getTargetConstant(Imm, DL, MVT::i8); 4516 4517 MVT NVT = Root->getSimpleValueType(0); 4518 4519 MachineSDNode *MNode; 4520 if (FoldedLoad) { 4521 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other); 4522 4523 unsigned Opc; 4524 if (C.getOpcode() == X86ISD::VBROADCAST_LOAD) { 4525 auto *MemIntr = cast<MemIntrinsicSDNode>(C); 4526 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits(); 4527 assert((EltSize == 32 || EltSize == 64) && "Unexpected broadcast size!"); 4528 4529 bool UseD = EltSize == 32; 4530 if (NVT.is128BitVector()) 4531 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi; 4532 else if (NVT.is256BitVector()) 4533 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi; 4534 else if (NVT.is512BitVector()) 4535 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi; 4536 else 4537 llvm_unreachable("Unexpected vector size!"); 4538 } else { 4539 bool UseD = NVT.getVectorElementType() == MVT::i32; 4540 if (NVT.is128BitVector()) 4541 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi; 4542 else if (NVT.is256BitVector()) 4543 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi; 4544 else if (NVT.is512BitVector()) 4545 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi; 4546 else 4547 llvm_unreachable("Unexpected vector size!"); 4548 } 4549 4550 SDValue Ops[] = {A, B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm, C.getOperand(0)}; 4551 MNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops); 4552 4553 // Update the chain. 4554 ReplaceUses(C.getValue(1), SDValue(MNode, 1)); 4555 // Record the mem-refs 4556 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(C)->getMemOperand()}); 4557 } else { 4558 bool UseD = NVT.getVectorElementType() == MVT::i32; 4559 unsigned Opc; 4560 if (NVT.is128BitVector()) 4561 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri; 4562 else if (NVT.is256BitVector()) 4563 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri; 4564 else if (NVT.is512BitVector()) 4565 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri; 4566 else 4567 llvm_unreachable("Unexpected vector size!"); 4568 4569 MNode = CurDAG->getMachineNode(Opc, DL, NVT, {A, B, C, TImm}); 4570 } 4571 4572 ReplaceUses(SDValue(Root, 0), SDValue(MNode, 0)); 4573 CurDAG->RemoveDeadNode(Root); 4574 return true; 4575 } 4576 4577 // Try to match two logic ops to a VPTERNLOG. 4578 // FIXME: Handle more complex patterns that use an operand more than once? 4579 bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) { 4580 MVT NVT = N->getSimpleValueType(0); 4581 4582 // Make sure we support VPTERNLOG. 4583 if (!NVT.isVector() || !Subtarget->hasAVX512() || 4584 NVT.getVectorElementType() == MVT::i1) 4585 return false; 4586 4587 // We need VLX for 128/256-bit. 4588 if (!(Subtarget->hasVLX() || NVT.is512BitVector())) 4589 return false; 4590 4591 SDValue N0 = N->getOperand(0); 4592 SDValue N1 = N->getOperand(1); 4593 4594 auto getFoldableLogicOp = [](SDValue Op) { 4595 // Peek through single use bitcast. 4596 if (Op.getOpcode() == ISD::BITCAST && Op.hasOneUse()) 4597 Op = Op.getOperand(0); 4598 4599 if (!Op.hasOneUse()) 4600 return SDValue(); 4601 4602 unsigned Opc = Op.getOpcode(); 4603 if (Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR || 4604 Opc == X86ISD::ANDNP) 4605 return Op; 4606 4607 return SDValue(); 4608 }; 4609 4610 SDValue A, FoldableOp; 4611 if ((FoldableOp = getFoldableLogicOp(N1))) { 4612 A = N0; 4613 } else if ((FoldableOp = getFoldableLogicOp(N0))) { 4614 A = N1; 4615 } else 4616 return false; 4617 4618 SDValue B = FoldableOp.getOperand(0); 4619 SDValue C = FoldableOp.getOperand(1); 4620 SDNode *ParentA = N; 4621 SDNode *ParentB = FoldableOp.getNode(); 4622 SDNode *ParentC = FoldableOp.getNode(); 4623 4624 // We can build the appropriate control immediate by performing the logic 4625 // operation we're matching using these constants for A, B, and C. 4626 uint8_t TernlogMagicA = 0xf0; 4627 uint8_t TernlogMagicB = 0xcc; 4628 uint8_t TernlogMagicC = 0xaa; 4629 4630 // Some of the inputs may be inverted, peek through them and invert the 4631 // magic values accordingly. 4632 // TODO: There may be a bitcast before the xor that we should peek through. 4633 auto PeekThroughNot = [](SDValue &Op, SDNode *&Parent, uint8_t &Magic) { 4634 if (Op.getOpcode() == ISD::XOR && Op.hasOneUse() && 4635 ISD::isBuildVectorAllOnes(Op.getOperand(1).getNode())) { 4636 Magic = ~Magic; 4637 Parent = Op.getNode(); 4638 Op = Op.getOperand(0); 4639 } 4640 }; 4641 4642 PeekThroughNot(A, ParentA, TernlogMagicA); 4643 PeekThroughNot(B, ParentB, TernlogMagicB); 4644 PeekThroughNot(C, ParentC, TernlogMagicC); 4645 4646 uint8_t Imm; 4647 switch (FoldableOp.getOpcode()) { 4648 default: llvm_unreachable("Unexpected opcode!"); 4649 case ISD::AND: Imm = TernlogMagicB & TernlogMagicC; break; 4650 case ISD::OR: Imm = TernlogMagicB | TernlogMagicC; break; 4651 case ISD::XOR: Imm = TernlogMagicB ^ TernlogMagicC; break; 4652 case X86ISD::ANDNP: Imm = ~(TernlogMagicB) & TernlogMagicC; break; 4653 } 4654 4655 switch (N->getOpcode()) { 4656 default: llvm_unreachable("Unexpected opcode!"); 4657 case X86ISD::ANDNP: 4658 if (A == N0) 4659 Imm &= ~TernlogMagicA; 4660 else 4661 Imm = ~(Imm) & TernlogMagicA; 4662 break; 4663 case ISD::AND: Imm &= TernlogMagicA; break; 4664 case ISD::OR: Imm |= TernlogMagicA; break; 4665 case ISD::XOR: Imm ^= TernlogMagicA; break; 4666 } 4667 4668 return matchVPTERNLOG(N, ParentA, ParentB, ParentC, A, B, C, Imm); 4669 } 4670 4671 /// If the high bits of an 'and' operand are known zero, try setting the 4672 /// high bits of an 'and' constant operand to produce a smaller encoding by 4673 /// creating a small, sign-extended negative immediate rather than a large 4674 /// positive one. This reverses a transform in SimplifyDemandedBits that 4675 /// shrinks mask constants by clearing bits. There is also a possibility that 4676 /// the 'and' mask can be made -1, so the 'and' itself is unnecessary. In that 4677 /// case, just replace the 'and'. Return 'true' if the node is replaced. 4678 bool X86DAGToDAGISel::shrinkAndImmediate(SDNode *And) { 4679 // i8 is unshrinkable, i16 should be promoted to i32, and vector ops don't 4680 // have immediate operands. 4681 MVT VT = And->getSimpleValueType(0); 4682 if (VT != MVT::i32 && VT != MVT::i64) 4683 return false; 4684 4685 auto *And1C = dyn_cast<ConstantSDNode>(And->getOperand(1)); 4686 if (!And1C) 4687 return false; 4688 4689 // Bail out if the mask constant is already negative. It's can't shrink more. 4690 // If the upper 32 bits of a 64 bit mask are all zeros, we have special isel 4691 // patterns to use a 32-bit and instead of a 64-bit and by relying on the 4692 // implicit zeroing of 32 bit ops. So we should check if the lower 32 bits 4693 // are negative too. 4694 APInt MaskVal = And1C->getAPIntValue(); 4695 unsigned MaskLZ = MaskVal.countl_zero(); 4696 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32)) 4697 return false; 4698 4699 // Don't extend into the upper 32 bits of a 64 bit mask. 4700 if (VT == MVT::i64 && MaskLZ >= 32) { 4701 MaskLZ -= 32; 4702 MaskVal = MaskVal.trunc(32); 4703 } 4704 4705 SDValue And0 = And->getOperand(0); 4706 APInt HighZeros = APInt::getHighBitsSet(MaskVal.getBitWidth(), MaskLZ); 4707 APInt NegMaskVal = MaskVal | HighZeros; 4708 4709 // If a negative constant would not allow a smaller encoding, there's no need 4710 // to continue. Only change the constant when we know it's a win. 4711 unsigned MinWidth = NegMaskVal.getSignificantBits(); 4712 if (MinWidth > 32 || (MinWidth > 8 && MaskVal.getSignificantBits() <= 32)) 4713 return false; 4714 4715 // Extend masks if we truncated above. 4716 if (VT == MVT::i64 && MaskVal.getBitWidth() < 64) { 4717 NegMaskVal = NegMaskVal.zext(64); 4718 HighZeros = HighZeros.zext(64); 4719 } 4720 4721 // The variable operand must be all zeros in the top bits to allow using the 4722 // new, negative constant as the mask. 4723 if (!CurDAG->MaskedValueIsZero(And0, HighZeros)) 4724 return false; 4725 4726 // Check if the mask is -1. In that case, this is an unnecessary instruction 4727 // that escaped earlier analysis. 4728 if (NegMaskVal.isAllOnes()) { 4729 ReplaceNode(And, And0.getNode()); 4730 return true; 4731 } 4732 4733 // A negative mask allows a smaller encoding. Create a new 'and' node. 4734 SDValue NewMask = CurDAG->getConstant(NegMaskVal, SDLoc(And), VT); 4735 insertDAGNode(*CurDAG, SDValue(And, 0), NewMask); 4736 SDValue NewAnd = CurDAG->getNode(ISD::AND, SDLoc(And), VT, And0, NewMask); 4737 ReplaceNode(And, NewAnd.getNode()); 4738 SelectCode(NewAnd.getNode()); 4739 return true; 4740 } 4741 4742 static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad, 4743 bool FoldedBCast, bool Masked) { 4744 #define VPTESTM_CASE(VT, SUFFIX) \ 4745 case MVT::VT: \ 4746 if (Masked) \ 4747 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \ 4748 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX; 4749 4750 4751 #define VPTESTM_BROADCAST_CASES(SUFFIX) \ 4752 default: llvm_unreachable("Unexpected VT!"); \ 4753 VPTESTM_CASE(v4i32, DZ128##SUFFIX) \ 4754 VPTESTM_CASE(v2i64, QZ128##SUFFIX) \ 4755 VPTESTM_CASE(v8i32, DZ256##SUFFIX) \ 4756 VPTESTM_CASE(v4i64, QZ256##SUFFIX) \ 4757 VPTESTM_CASE(v16i32, DZ##SUFFIX) \ 4758 VPTESTM_CASE(v8i64, QZ##SUFFIX) 4759 4760 #define VPTESTM_FULL_CASES(SUFFIX) \ 4761 VPTESTM_BROADCAST_CASES(SUFFIX) \ 4762 VPTESTM_CASE(v16i8, BZ128##SUFFIX) \ 4763 VPTESTM_CASE(v8i16, WZ128##SUFFIX) \ 4764 VPTESTM_CASE(v32i8, BZ256##SUFFIX) \ 4765 VPTESTM_CASE(v16i16, WZ256##SUFFIX) \ 4766 VPTESTM_CASE(v64i8, BZ##SUFFIX) \ 4767 VPTESTM_CASE(v32i16, WZ##SUFFIX) 4768 4769 if (FoldedBCast) { 4770 switch (TestVT.SimpleTy) { 4771 VPTESTM_BROADCAST_CASES(rmb) 4772 } 4773 } 4774 4775 if (FoldedLoad) { 4776 switch (TestVT.SimpleTy) { 4777 VPTESTM_FULL_CASES(rm) 4778 } 4779 } 4780 4781 switch (TestVT.SimpleTy) { 4782 VPTESTM_FULL_CASES(rr) 4783 } 4784 4785 #undef VPTESTM_FULL_CASES 4786 #undef VPTESTM_BROADCAST_CASES 4787 #undef VPTESTM_CASE 4788 } 4789 4790 // Try to create VPTESTM instruction. If InMask is not null, it will be used 4791 // to form a masked operation. 4792 bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc, 4793 SDValue InMask) { 4794 assert(Subtarget->hasAVX512() && "Expected AVX512!"); 4795 assert(Setcc.getSimpleValueType().getVectorElementType() == MVT::i1 && 4796 "Unexpected VT!"); 4797 4798 // Look for equal and not equal compares. 4799 ISD::CondCode CC = cast<CondCodeSDNode>(Setcc.getOperand(2))->get(); 4800 if (CC != ISD::SETEQ && CC != ISD::SETNE) 4801 return false; 4802 4803 SDValue SetccOp0 = Setcc.getOperand(0); 4804 SDValue SetccOp1 = Setcc.getOperand(1); 4805 4806 // Canonicalize the all zero vector to the RHS. 4807 if (ISD::isBuildVectorAllZeros(SetccOp0.getNode())) 4808 std::swap(SetccOp0, SetccOp1); 4809 4810 // See if we're comparing against zero. 4811 if (!ISD::isBuildVectorAllZeros(SetccOp1.getNode())) 4812 return false; 4813 4814 SDValue N0 = SetccOp0; 4815 4816 MVT CmpVT = N0.getSimpleValueType(); 4817 MVT CmpSVT = CmpVT.getVectorElementType(); 4818 4819 // Start with both operands the same. We'll try to refine this. 4820 SDValue Src0 = N0; 4821 SDValue Src1 = N0; 4822 4823 { 4824 // Look through single use bitcasts. 4825 SDValue N0Temp = N0; 4826 if (N0Temp.getOpcode() == ISD::BITCAST && N0Temp.hasOneUse()) 4827 N0Temp = N0.getOperand(0); 4828 4829 // Look for single use AND. 4830 if (N0Temp.getOpcode() == ISD::AND && N0Temp.hasOneUse()) { 4831 Src0 = N0Temp.getOperand(0); 4832 Src1 = N0Temp.getOperand(1); 4833 } 4834 } 4835 4836 // Without VLX we need to widen the operation. 4837 bool Widen = !Subtarget->hasVLX() && !CmpVT.is512BitVector(); 4838 4839 auto tryFoldLoadOrBCast = [&](SDNode *Root, SDNode *P, SDValue &L, 4840 SDValue &Base, SDValue &Scale, SDValue &Index, 4841 SDValue &Disp, SDValue &Segment) { 4842 // If we need to widen, we can't fold the load. 4843 if (!Widen) 4844 if (tryFoldLoad(Root, P, L, Base, Scale, Index, Disp, Segment)) 4845 return true; 4846 4847 // If we didn't fold a load, try to match broadcast. No widening limitation 4848 // for this. But only 32 and 64 bit types are supported. 4849 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64) 4850 return false; 4851 4852 // Look through single use bitcasts. 4853 if (L.getOpcode() == ISD::BITCAST && L.hasOneUse()) { 4854 P = L.getNode(); 4855 L = L.getOperand(0); 4856 } 4857 4858 if (L.getOpcode() != X86ISD::VBROADCAST_LOAD) 4859 return false; 4860 4861 auto *MemIntr = cast<MemIntrinsicSDNode>(L); 4862 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.getSizeInBits()) 4863 return false; 4864 4865 return tryFoldBroadcast(Root, P, L, Base, Scale, Index, Disp, Segment); 4866 }; 4867 4868 // We can only fold loads if the sources are unique. 4869 bool CanFoldLoads = Src0 != Src1; 4870 4871 bool FoldedLoad = false; 4872 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 4873 if (CanFoldLoads) { 4874 FoldedLoad = tryFoldLoadOrBCast(Root, N0.getNode(), Src1, Tmp0, Tmp1, Tmp2, 4875 Tmp3, Tmp4); 4876 if (!FoldedLoad) { 4877 // And is commutative. 4878 FoldedLoad = tryFoldLoadOrBCast(Root, N0.getNode(), Src0, Tmp0, Tmp1, 4879 Tmp2, Tmp3, Tmp4); 4880 if (FoldedLoad) 4881 std::swap(Src0, Src1); 4882 } 4883 } 4884 4885 bool FoldedBCast = FoldedLoad && Src1.getOpcode() == X86ISD::VBROADCAST_LOAD; 4886 4887 bool IsMasked = InMask.getNode() != nullptr; 4888 4889 SDLoc dl(Root); 4890 4891 MVT ResVT = Setcc.getSimpleValueType(); 4892 MVT MaskVT = ResVT; 4893 if (Widen) { 4894 // Widen the inputs using insert_subreg or copy_to_regclass. 4895 unsigned Scale = CmpVT.is128BitVector() ? 4 : 2; 4896 unsigned SubReg = CmpVT.is128BitVector() ? X86::sub_xmm : X86::sub_ymm; 4897 unsigned NumElts = CmpVT.getVectorNumElements() * Scale; 4898 CmpVT = MVT::getVectorVT(CmpSVT, NumElts); 4899 MaskVT = MVT::getVectorVT(MVT::i1, NumElts); 4900 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl, 4901 CmpVT), 0); 4902 Src0 = CurDAG->getTargetInsertSubreg(SubReg, dl, CmpVT, ImplDef, Src0); 4903 4904 if (!FoldedBCast) 4905 Src1 = CurDAG->getTargetInsertSubreg(SubReg, dl, CmpVT, ImplDef, Src1); 4906 4907 if (IsMasked) { 4908 // Widen the mask. 4909 unsigned RegClass = TLI->getRegClassFor(MaskVT)->getID(); 4910 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32); 4911 InMask = SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 4912 dl, MaskVT, InMask, RC), 0); 4913 } 4914 } 4915 4916 bool IsTestN = CC == ISD::SETEQ; 4917 unsigned Opc = getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast, 4918 IsMasked); 4919 4920 MachineSDNode *CNode; 4921 if (FoldedLoad) { 4922 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other); 4923 4924 if (IsMasked) { 4925 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, 4926 Src1.getOperand(0) }; 4927 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 4928 } else { 4929 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, 4930 Src1.getOperand(0) }; 4931 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 4932 } 4933 4934 // Update the chain. 4935 ReplaceUses(Src1.getValue(1), SDValue(CNode, 1)); 4936 // Record the mem-refs 4937 CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()}); 4938 } else { 4939 if (IsMasked) 4940 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1); 4941 else 4942 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1); 4943 } 4944 4945 // If we widened, we need to shrink the mask VT. 4946 if (Widen) { 4947 unsigned RegClass = TLI->getRegClassFor(ResVT)->getID(); 4948 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32); 4949 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 4950 dl, ResVT, SDValue(CNode, 0), RC); 4951 } 4952 4953 ReplaceUses(SDValue(Root, 0), SDValue(CNode, 0)); 4954 CurDAG->RemoveDeadNode(Root); 4955 return true; 4956 } 4957 4958 // Try to match the bitselect pattern (or (and A, B), (andn A, C)). Turn it 4959 // into vpternlog. 4960 bool X86DAGToDAGISel::tryMatchBitSelect(SDNode *N) { 4961 assert(N->getOpcode() == ISD::OR && "Unexpected opcode!"); 4962 4963 MVT NVT = N->getSimpleValueType(0); 4964 4965 // Make sure we support VPTERNLOG. 4966 if (!NVT.isVector() || !Subtarget->hasAVX512()) 4967 return false; 4968 4969 // We need VLX for 128/256-bit. 4970 if (!(Subtarget->hasVLX() || NVT.is512BitVector())) 4971 return false; 4972 4973 SDValue N0 = N->getOperand(0); 4974 SDValue N1 = N->getOperand(1); 4975 4976 // Canonicalize AND to LHS. 4977 if (N1.getOpcode() == ISD::AND) 4978 std::swap(N0, N1); 4979 4980 if (N0.getOpcode() != ISD::AND || 4981 N1.getOpcode() != X86ISD::ANDNP || 4982 !N0.hasOneUse() || !N1.hasOneUse()) 4983 return false; 4984 4985 // ANDN is not commutable, use it to pick down A and C. 4986 SDValue A = N1.getOperand(0); 4987 SDValue C = N1.getOperand(1); 4988 4989 // AND is commutable, if one operand matches A, the other operand is B. 4990 // Otherwise this isn't a match. 4991 SDValue B; 4992 if (N0.getOperand(0) == A) 4993 B = N0.getOperand(1); 4994 else if (N0.getOperand(1) == A) 4995 B = N0.getOperand(0); 4996 else 4997 return false; 4998 4999 SDLoc dl(N); 5000 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8); 5001 SDValue Ternlog = CurDAG->getNode(X86ISD::VPTERNLOG, dl, NVT, A, B, C, Imm); 5002 ReplaceNode(N, Ternlog.getNode()); 5003 5004 return matchVPTERNLOG(Ternlog.getNode(), Ternlog.getNode(), Ternlog.getNode(), 5005 Ternlog.getNode(), A, B, C, 0xCA); 5006 } 5007 5008 void X86DAGToDAGISel::Select(SDNode *Node) { 5009 MVT NVT = Node->getSimpleValueType(0); 5010 unsigned Opcode = Node->getOpcode(); 5011 SDLoc dl(Node); 5012 5013 if (Node->isMachineOpcode()) { 5014 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n'); 5015 Node->setNodeId(-1); 5016 return; // Already selected. 5017 } 5018 5019 switch (Opcode) { 5020 default: break; 5021 case ISD::INTRINSIC_W_CHAIN: { 5022 unsigned IntNo = Node->getConstantOperandVal(1); 5023 switch (IntNo) { 5024 default: break; 5025 case Intrinsic::x86_encodekey128: 5026 case Intrinsic::x86_encodekey256: { 5027 if (!Subtarget->hasKL()) 5028 break; 5029 5030 unsigned Opcode; 5031 switch (IntNo) { 5032 default: llvm_unreachable("Impossible intrinsic"); 5033 case Intrinsic::x86_encodekey128: Opcode = X86::ENCODEKEY128; break; 5034 case Intrinsic::x86_encodekey256: Opcode = X86::ENCODEKEY256; break; 5035 } 5036 5037 SDValue Chain = Node->getOperand(0); 5038 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0, Node->getOperand(3), 5039 SDValue()); 5040 if (Opcode == X86::ENCODEKEY256) 5041 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1, Node->getOperand(4), 5042 Chain.getValue(1)); 5043 5044 MachineSDNode *Res = CurDAG->getMachineNode( 5045 Opcode, dl, Node->getVTList(), 5046 {Node->getOperand(2), Chain, Chain.getValue(1)}); 5047 ReplaceNode(Node, Res); 5048 return; 5049 } 5050 case Intrinsic::x86_tileloadd64_internal: 5051 case Intrinsic::x86_tileloaddt164_internal: { 5052 if (!Subtarget->hasAMXTILE()) 5053 break; 5054 unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal 5055 ? X86::PTILELOADDV 5056 : X86::PTILELOADDT1V; 5057 // _tile_loadd_internal(row, col, buf, STRIDE) 5058 SDValue Base = Node->getOperand(4); 5059 SDValue Scale = getI8Imm(1, dl); 5060 SDValue Index = Node->getOperand(5); 5061 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32); 5062 SDValue Segment = CurDAG->getRegister(0, MVT::i16); 5063 SDValue Chain = Node->getOperand(0); 5064 MachineSDNode *CNode; 5065 SDValue Ops[] = {Node->getOperand(2), 5066 Node->getOperand(3), 5067 Base, 5068 Scale, 5069 Index, 5070 Disp, 5071 Segment, 5072 Chain}; 5073 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops); 5074 ReplaceNode(Node, CNode); 5075 return; 5076 } 5077 } 5078 break; 5079 } 5080 case ISD::INTRINSIC_VOID: { 5081 unsigned IntNo = Node->getConstantOperandVal(1); 5082 switch (IntNo) { 5083 default: break; 5084 case Intrinsic::x86_sse3_monitor: 5085 case Intrinsic::x86_monitorx: 5086 case Intrinsic::x86_clzero: { 5087 bool Use64BitPtr = Node->getOperand(2).getValueType() == MVT::i64; 5088 5089 unsigned Opc = 0; 5090 switch (IntNo) { 5091 default: llvm_unreachable("Unexpected intrinsic!"); 5092 case Intrinsic::x86_sse3_monitor: 5093 if (!Subtarget->hasSSE3()) 5094 break; 5095 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr; 5096 break; 5097 case Intrinsic::x86_monitorx: 5098 if (!Subtarget->hasMWAITX()) 5099 break; 5100 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr; 5101 break; 5102 case Intrinsic::x86_clzero: 5103 if (!Subtarget->hasCLZERO()) 5104 break; 5105 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r; 5106 break; 5107 } 5108 5109 if (Opc) { 5110 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX; 5111 SDValue Chain = CurDAG->getCopyToReg(Node->getOperand(0), dl, PtrReg, 5112 Node->getOperand(2), SDValue()); 5113 SDValue InGlue = Chain.getValue(1); 5114 5115 if (IntNo == Intrinsic::x86_sse3_monitor || 5116 IntNo == Intrinsic::x86_monitorx) { 5117 // Copy the other two operands to ECX and EDX. 5118 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX, Node->getOperand(3), 5119 InGlue); 5120 InGlue = Chain.getValue(1); 5121 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX, Node->getOperand(4), 5122 InGlue); 5123 InGlue = Chain.getValue(1); 5124 } 5125 5126 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, 5127 { Chain, InGlue}); 5128 ReplaceNode(Node, CNode); 5129 return; 5130 } 5131 5132 break; 5133 } 5134 case Intrinsic::x86_tilestored64_internal: { 5135 unsigned Opc = X86::PTILESTOREDV; 5136 // _tile_stored_internal(row, col, buf, STRIDE, c) 5137 SDValue Base = Node->getOperand(4); 5138 SDValue Scale = getI8Imm(1, dl); 5139 SDValue Index = Node->getOperand(5); 5140 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32); 5141 SDValue Segment = CurDAG->getRegister(0, MVT::i16); 5142 SDValue Chain = Node->getOperand(0); 5143 MachineSDNode *CNode; 5144 SDValue Ops[] = {Node->getOperand(2), 5145 Node->getOperand(3), 5146 Base, 5147 Scale, 5148 Index, 5149 Disp, 5150 Segment, 5151 Node->getOperand(6), 5152 Chain}; 5153 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); 5154 ReplaceNode(Node, CNode); 5155 return; 5156 } 5157 case Intrinsic::x86_tileloadd64: 5158 case Intrinsic::x86_tileloaddt164: 5159 case Intrinsic::x86_tilestored64: { 5160 if (!Subtarget->hasAMXTILE()) 5161 break; 5162 unsigned Opc; 5163 switch (IntNo) { 5164 default: llvm_unreachable("Unexpected intrinsic!"); 5165 case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD; break; 5166 case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1; break; 5167 case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED; break; 5168 } 5169 // FIXME: Match displacement and scale. 5170 unsigned TIndex = Node->getConstantOperandVal(2); 5171 SDValue TReg = getI8Imm(TIndex, dl); 5172 SDValue Base = Node->getOperand(3); 5173 SDValue Scale = getI8Imm(1, dl); 5174 SDValue Index = Node->getOperand(4); 5175 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32); 5176 SDValue Segment = CurDAG->getRegister(0, MVT::i16); 5177 SDValue Chain = Node->getOperand(0); 5178 MachineSDNode *CNode; 5179 if (Opc == X86::PTILESTORED) { 5180 SDValue Ops[] = { Base, Scale, Index, Disp, Segment, TReg, Chain }; 5181 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); 5182 } else { 5183 SDValue Ops[] = { TReg, Base, Scale, Index, Disp, Segment, Chain }; 5184 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); 5185 } 5186 ReplaceNode(Node, CNode); 5187 return; 5188 } 5189 } 5190 break; 5191 } 5192 case ISD::BRIND: 5193 case X86ISD::NT_BRIND: { 5194 if (Subtarget->isTargetNaCl()) 5195 // NaCl has its own pass where jmp %r32 are converted to jmp %r64. We 5196 // leave the instruction alone. 5197 break; 5198 if (Subtarget->isTarget64BitILP32()) { 5199 // Converts a 32-bit register to a 64-bit, zero-extended version of 5200 // it. This is needed because x86-64 can do many things, but jmp %r32 5201 // ain't one of them. 5202 SDValue Target = Node->getOperand(1); 5203 assert(Target.getValueType() == MVT::i32 && "Unexpected VT!"); 5204 SDValue ZextTarget = CurDAG->getZExtOrTrunc(Target, dl, MVT::i64); 5205 SDValue Brind = CurDAG->getNode(Opcode, dl, MVT::Other, 5206 Node->getOperand(0), ZextTarget); 5207 ReplaceNode(Node, Brind.getNode()); 5208 SelectCode(ZextTarget.getNode()); 5209 SelectCode(Brind.getNode()); 5210 return; 5211 } 5212 break; 5213 } 5214 case X86ISD::GlobalBaseReg: 5215 ReplaceNode(Node, getGlobalBaseReg()); 5216 return; 5217 5218 case ISD::BITCAST: 5219 // Just drop all 128/256/512-bit bitcasts. 5220 if (NVT.is512BitVector() || NVT.is256BitVector() || NVT.is128BitVector() || 5221 NVT == MVT::f128) { 5222 ReplaceUses(SDValue(Node, 0), Node->getOperand(0)); 5223 CurDAG->RemoveDeadNode(Node); 5224 return; 5225 } 5226 break; 5227 5228 case ISD::SRL: 5229 if (matchBitExtract(Node)) 5230 return; 5231 [[fallthrough]]; 5232 case ISD::SRA: 5233 case ISD::SHL: 5234 if (tryShiftAmountMod(Node)) 5235 return; 5236 break; 5237 5238 case X86ISD::VPTERNLOG: { 5239 uint8_t Imm = cast<ConstantSDNode>(Node->getOperand(3))->getZExtValue(); 5240 if (matchVPTERNLOG(Node, Node, Node, Node, Node->getOperand(0), 5241 Node->getOperand(1), Node->getOperand(2), Imm)) 5242 return; 5243 break; 5244 } 5245 5246 case X86ISD::ANDNP: 5247 if (tryVPTERNLOG(Node)) 5248 return; 5249 break; 5250 5251 case ISD::AND: 5252 if (NVT.isVector() && NVT.getVectorElementType() == MVT::i1) { 5253 // Try to form a masked VPTESTM. Operands can be in either order. 5254 SDValue N0 = Node->getOperand(0); 5255 SDValue N1 = Node->getOperand(1); 5256 if (N0.getOpcode() == ISD::SETCC && N0.hasOneUse() && 5257 tryVPTESTM(Node, N0, N1)) 5258 return; 5259 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse() && 5260 tryVPTESTM(Node, N1, N0)) 5261 return; 5262 } 5263 5264 if (MachineSDNode *NewNode = matchBEXTRFromAndImm(Node)) { 5265 ReplaceUses(SDValue(Node, 0), SDValue(NewNode, 0)); 5266 CurDAG->RemoveDeadNode(Node); 5267 return; 5268 } 5269 if (matchBitExtract(Node)) 5270 return; 5271 if (AndImmShrink && shrinkAndImmediate(Node)) 5272 return; 5273 5274 [[fallthrough]]; 5275 case ISD::OR: 5276 case ISD::XOR: 5277 if (tryShrinkShlLogicImm(Node)) 5278 return; 5279 if (Opcode == ISD::OR && tryMatchBitSelect(Node)) 5280 return; 5281 if (tryVPTERNLOG(Node)) 5282 return; 5283 5284 [[fallthrough]]; 5285 case ISD::ADD: 5286 if (Opcode == ISD::ADD && matchBitExtract(Node)) 5287 return; 5288 [[fallthrough]]; 5289 case ISD::SUB: { 5290 // Try to avoid folding immediates with multiple uses for optsize. 5291 // This code tries to select to register form directly to avoid going 5292 // through the isel table which might fold the immediate. We can't change 5293 // the patterns on the add/sub/and/or/xor with immediate paterns in the 5294 // tablegen files to check immediate use count without making the patterns 5295 // unavailable to the fast-isel table. 5296 if (!CurDAG->shouldOptForSize()) 5297 break; 5298 5299 // Only handle i8/i16/i32/i64. 5300 if (NVT != MVT::i8 && NVT != MVT::i16 && NVT != MVT::i32 && NVT != MVT::i64) 5301 break; 5302 5303 SDValue N0 = Node->getOperand(0); 5304 SDValue N1 = Node->getOperand(1); 5305 5306 auto *Cst = dyn_cast<ConstantSDNode>(N1); 5307 if (!Cst) 5308 break; 5309 5310 int64_t Val = Cst->getSExtValue(); 5311 5312 // Make sure its an immediate that is considered foldable. 5313 // FIXME: Handle unsigned 32 bit immediates for 64-bit AND. 5314 if (!isInt<8>(Val) && !isInt<32>(Val)) 5315 break; 5316 5317 // If this can match to INC/DEC, let it go. 5318 if (Opcode == ISD::ADD && (Val == 1 || Val == -1)) 5319 break; 5320 5321 // Check if we should avoid folding this immediate. 5322 if (!shouldAvoidImmediateInstFormsForSize(N1.getNode())) 5323 break; 5324 5325 // We should not fold the immediate. So we need a register form instead. 5326 unsigned ROpc, MOpc; 5327 switch (NVT.SimpleTy) { 5328 default: llvm_unreachable("Unexpected VT!"); 5329 case MVT::i8: 5330 switch (Opcode) { 5331 default: llvm_unreachable("Unexpected opcode!"); 5332 case ISD::ADD: ROpc = X86::ADD8rr; MOpc = X86::ADD8rm; break; 5333 case ISD::SUB: ROpc = X86::SUB8rr; MOpc = X86::SUB8rm; break; 5334 case ISD::AND: ROpc = X86::AND8rr; MOpc = X86::AND8rm; break; 5335 case ISD::OR: ROpc = X86::OR8rr; MOpc = X86::OR8rm; break; 5336 case ISD::XOR: ROpc = X86::XOR8rr; MOpc = X86::XOR8rm; break; 5337 } 5338 break; 5339 case MVT::i16: 5340 switch (Opcode) { 5341 default: llvm_unreachable("Unexpected opcode!"); 5342 case ISD::ADD: ROpc = X86::ADD16rr; MOpc = X86::ADD16rm; break; 5343 case ISD::SUB: ROpc = X86::SUB16rr; MOpc = X86::SUB16rm; break; 5344 case ISD::AND: ROpc = X86::AND16rr; MOpc = X86::AND16rm; break; 5345 case ISD::OR: ROpc = X86::OR16rr; MOpc = X86::OR16rm; break; 5346 case ISD::XOR: ROpc = X86::XOR16rr; MOpc = X86::XOR16rm; break; 5347 } 5348 break; 5349 case MVT::i32: 5350 switch (Opcode) { 5351 default: llvm_unreachable("Unexpected opcode!"); 5352 case ISD::ADD: ROpc = X86::ADD32rr; MOpc = X86::ADD32rm; break; 5353 case ISD::SUB: ROpc = X86::SUB32rr; MOpc = X86::SUB32rm; break; 5354 case ISD::AND: ROpc = X86::AND32rr; MOpc = X86::AND32rm; break; 5355 case ISD::OR: ROpc = X86::OR32rr; MOpc = X86::OR32rm; break; 5356 case ISD::XOR: ROpc = X86::XOR32rr; MOpc = X86::XOR32rm; break; 5357 } 5358 break; 5359 case MVT::i64: 5360 switch (Opcode) { 5361 default: llvm_unreachable("Unexpected opcode!"); 5362 case ISD::ADD: ROpc = X86::ADD64rr; MOpc = X86::ADD64rm; break; 5363 case ISD::SUB: ROpc = X86::SUB64rr; MOpc = X86::SUB64rm; break; 5364 case ISD::AND: ROpc = X86::AND64rr; MOpc = X86::AND64rm; break; 5365 case ISD::OR: ROpc = X86::OR64rr; MOpc = X86::OR64rm; break; 5366 case ISD::XOR: ROpc = X86::XOR64rr; MOpc = X86::XOR64rm; break; 5367 } 5368 break; 5369 } 5370 5371 // Ok this is a AND/OR/XOR/ADD/SUB with constant. 5372 5373 // If this is a not a subtract, we can still try to fold a load. 5374 if (Opcode != ISD::SUB) { 5375 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 5376 if (tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 5377 SDValue Ops[] = { N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) }; 5378 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other); 5379 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 5380 // Update the chain. 5381 ReplaceUses(N0.getValue(1), SDValue(CNode, 2)); 5382 // Record the mem-refs 5383 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N0)->getMemOperand()}); 5384 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 5385 CurDAG->RemoveDeadNode(Node); 5386 return; 5387 } 5388 } 5389 5390 CurDAG->SelectNodeTo(Node, ROpc, NVT, MVT::i32, N0, N1); 5391 return; 5392 } 5393 5394 case X86ISD::SMUL: 5395 // i16/i32/i64 are handled with isel patterns. 5396 if (NVT != MVT::i8) 5397 break; 5398 [[fallthrough]]; 5399 case X86ISD::UMUL: { 5400 SDValue N0 = Node->getOperand(0); 5401 SDValue N1 = Node->getOperand(1); 5402 5403 unsigned LoReg, ROpc, MOpc; 5404 switch (NVT.SimpleTy) { 5405 default: llvm_unreachable("Unsupported VT!"); 5406 case MVT::i8: 5407 LoReg = X86::AL; 5408 ROpc = Opcode == X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r; 5409 MOpc = Opcode == X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m; 5410 break; 5411 case MVT::i16: 5412 LoReg = X86::AX; 5413 ROpc = X86::MUL16r; 5414 MOpc = X86::MUL16m; 5415 break; 5416 case MVT::i32: 5417 LoReg = X86::EAX; 5418 ROpc = X86::MUL32r; 5419 MOpc = X86::MUL32m; 5420 break; 5421 case MVT::i64: 5422 LoReg = X86::RAX; 5423 ROpc = X86::MUL64r; 5424 MOpc = X86::MUL64m; 5425 break; 5426 } 5427 5428 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 5429 bool FoldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 5430 // Multiply is commutative. 5431 if (!FoldedLoad) { 5432 FoldedLoad = tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 5433 if (FoldedLoad) 5434 std::swap(N0, N1); 5435 } 5436 5437 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 5438 N0, SDValue()).getValue(1); 5439 5440 MachineSDNode *CNode; 5441 if (FoldedLoad) { 5442 // i16/i32/i64 use an instruction that produces a low and high result even 5443 // though only the low result is used. 5444 SDVTList VTs; 5445 if (NVT == MVT::i8) 5446 VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other); 5447 else 5448 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other); 5449 5450 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 5451 InGlue }; 5452 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 5453 5454 // Update the chain. 5455 ReplaceUses(N1.getValue(1), SDValue(CNode, NVT == MVT::i8 ? 2 : 3)); 5456 // Record the mem-refs 5457 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); 5458 } else { 5459 // i16/i32/i64 use an instruction that produces a low and high result even 5460 // though only the low result is used. 5461 SDVTList VTs; 5462 if (NVT == MVT::i8) 5463 VTs = CurDAG->getVTList(NVT, MVT::i32); 5464 else 5465 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32); 5466 5467 CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InGlue}); 5468 } 5469 5470 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 5471 ReplaceUses(SDValue(Node, 1), SDValue(CNode, NVT == MVT::i8 ? 1 : 2)); 5472 CurDAG->RemoveDeadNode(Node); 5473 return; 5474 } 5475 5476 case ISD::SMUL_LOHI: 5477 case ISD::UMUL_LOHI: { 5478 SDValue N0 = Node->getOperand(0); 5479 SDValue N1 = Node->getOperand(1); 5480 5481 unsigned Opc, MOpc; 5482 unsigned LoReg, HiReg; 5483 bool IsSigned = Opcode == ISD::SMUL_LOHI; 5484 bool UseMULX = !IsSigned && Subtarget->hasBMI2(); 5485 bool UseMULXHi = UseMULX && SDValue(Node, 0).use_empty(); 5486 switch (NVT.SimpleTy) { 5487 default: llvm_unreachable("Unsupported VT!"); 5488 case MVT::i32: 5489 Opc = UseMULXHi ? X86::MULX32Hrr : 5490 UseMULX ? X86::MULX32rr : 5491 IsSigned ? X86::IMUL32r : X86::MUL32r; 5492 MOpc = UseMULXHi ? X86::MULX32Hrm : 5493 UseMULX ? X86::MULX32rm : 5494 IsSigned ? X86::IMUL32m : X86::MUL32m; 5495 LoReg = UseMULX ? X86::EDX : X86::EAX; 5496 HiReg = X86::EDX; 5497 break; 5498 case MVT::i64: 5499 Opc = UseMULXHi ? X86::MULX64Hrr : 5500 UseMULX ? X86::MULX64rr : 5501 IsSigned ? X86::IMUL64r : X86::MUL64r; 5502 MOpc = UseMULXHi ? X86::MULX64Hrm : 5503 UseMULX ? X86::MULX64rm : 5504 IsSigned ? X86::IMUL64m : X86::MUL64m; 5505 LoReg = UseMULX ? X86::RDX : X86::RAX; 5506 HiReg = X86::RDX; 5507 break; 5508 } 5509 5510 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 5511 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 5512 // Multiply is commutative. 5513 if (!foldedLoad) { 5514 foldedLoad = tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 5515 if (foldedLoad) 5516 std::swap(N0, N1); 5517 } 5518 5519 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 5520 N0, SDValue()).getValue(1); 5521 SDValue ResHi, ResLo; 5522 if (foldedLoad) { 5523 SDValue Chain; 5524 MachineSDNode *CNode = nullptr; 5525 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 5526 InGlue }; 5527 if (UseMULXHi) { 5528 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other); 5529 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 5530 ResHi = SDValue(CNode, 0); 5531 Chain = SDValue(CNode, 1); 5532 } else if (UseMULX) { 5533 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other); 5534 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 5535 ResHi = SDValue(CNode, 0); 5536 ResLo = SDValue(CNode, 1); 5537 Chain = SDValue(CNode, 2); 5538 } else { 5539 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue); 5540 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 5541 Chain = SDValue(CNode, 0); 5542 InGlue = SDValue(CNode, 1); 5543 } 5544 5545 // Update the chain. 5546 ReplaceUses(N1.getValue(1), Chain); 5547 // Record the mem-refs 5548 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); 5549 } else { 5550 SDValue Ops[] = { N1, InGlue }; 5551 if (UseMULXHi) { 5552 SDVTList VTs = CurDAG->getVTList(NVT); 5553 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 5554 ResHi = SDValue(CNode, 0); 5555 } else if (UseMULX) { 5556 SDVTList VTs = CurDAG->getVTList(NVT, NVT); 5557 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 5558 ResHi = SDValue(CNode, 0); 5559 ResLo = SDValue(CNode, 1); 5560 } else { 5561 SDVTList VTs = CurDAG->getVTList(MVT::Glue); 5562 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 5563 InGlue = SDValue(CNode, 0); 5564 } 5565 } 5566 5567 // Copy the low half of the result, if it is needed. 5568 if (!SDValue(Node, 0).use_empty()) { 5569 if (!ResLo) { 5570 assert(LoReg && "Register for low half is not defined!"); 5571 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg, 5572 NVT, InGlue); 5573 InGlue = ResLo.getValue(2); 5574 } 5575 ReplaceUses(SDValue(Node, 0), ResLo); 5576 LLVM_DEBUG(dbgs() << "=> "; ResLo.getNode()->dump(CurDAG); 5577 dbgs() << '\n'); 5578 } 5579 // Copy the high half of the result, if it is needed. 5580 if (!SDValue(Node, 1).use_empty()) { 5581 if (!ResHi) { 5582 assert(HiReg && "Register for high half is not defined!"); 5583 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg, 5584 NVT, InGlue); 5585 InGlue = ResHi.getValue(2); 5586 } 5587 ReplaceUses(SDValue(Node, 1), ResHi); 5588 LLVM_DEBUG(dbgs() << "=> "; ResHi.getNode()->dump(CurDAG); 5589 dbgs() << '\n'); 5590 } 5591 5592 CurDAG->RemoveDeadNode(Node); 5593 return; 5594 } 5595 5596 case ISD::SDIVREM: 5597 case ISD::UDIVREM: { 5598 SDValue N0 = Node->getOperand(0); 5599 SDValue N1 = Node->getOperand(1); 5600 5601 unsigned ROpc, MOpc; 5602 bool isSigned = Opcode == ISD::SDIVREM; 5603 if (!isSigned) { 5604 switch (NVT.SimpleTy) { 5605 default: llvm_unreachable("Unsupported VT!"); 5606 case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m; break; 5607 case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m; break; 5608 case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m; break; 5609 case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m; break; 5610 } 5611 } else { 5612 switch (NVT.SimpleTy) { 5613 default: llvm_unreachable("Unsupported VT!"); 5614 case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m; break; 5615 case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m; break; 5616 case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m; break; 5617 case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m; break; 5618 } 5619 } 5620 5621 unsigned LoReg, HiReg, ClrReg; 5622 unsigned SExtOpcode; 5623 switch (NVT.SimpleTy) { 5624 default: llvm_unreachable("Unsupported VT!"); 5625 case MVT::i8: 5626 LoReg = X86::AL; ClrReg = HiReg = X86::AH; 5627 SExtOpcode = 0; // Not used. 5628 break; 5629 case MVT::i16: 5630 LoReg = X86::AX; HiReg = X86::DX; 5631 ClrReg = X86::DX; 5632 SExtOpcode = X86::CWD; 5633 break; 5634 case MVT::i32: 5635 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX; 5636 SExtOpcode = X86::CDQ; 5637 break; 5638 case MVT::i64: 5639 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX; 5640 SExtOpcode = X86::CQO; 5641 break; 5642 } 5643 5644 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 5645 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 5646 bool signBitIsZero = CurDAG->SignBitIsZero(N0); 5647 5648 SDValue InGlue; 5649 if (NVT == MVT::i8) { 5650 // Special case for div8, just use a move with zero extension to AX to 5651 // clear the upper 8 bits (AH). 5652 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain; 5653 MachineSDNode *Move; 5654 if (tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 5655 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) }; 5656 unsigned Opc = (isSigned && !signBitIsZero) ? X86::MOVSX16rm8 5657 : X86::MOVZX16rm8; 5658 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops); 5659 Chain = SDValue(Move, 1); 5660 ReplaceUses(N0.getValue(1), Chain); 5661 // Record the mem-refs 5662 CurDAG->setNodeMemRefs(Move, {cast<LoadSDNode>(N0)->getMemOperand()}); 5663 } else { 5664 unsigned Opc = (isSigned && !signBitIsZero) ? X86::MOVSX16rr8 5665 : X86::MOVZX16rr8; 5666 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0); 5667 Chain = CurDAG->getEntryNode(); 5668 } 5669 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX, SDValue(Move, 0), 5670 SDValue()); 5671 InGlue = Chain.getValue(1); 5672 } else { 5673 InGlue = 5674 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, 5675 LoReg, N0, SDValue()).getValue(1); 5676 if (isSigned && !signBitIsZero) { 5677 // Sign extend the low part into the high part. 5678 InGlue = 5679 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0); 5680 } else { 5681 // Zero out the high part, effectively zero extending the input. 5682 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32); 5683 SDValue ClrNode = SDValue( 5684 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0); 5685 switch (NVT.SimpleTy) { 5686 case MVT::i16: 5687 ClrNode = 5688 SDValue(CurDAG->getMachineNode( 5689 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode, 5690 CurDAG->getTargetConstant(X86::sub_16bit, dl, 5691 MVT::i32)), 5692 0); 5693 break; 5694 case MVT::i32: 5695 break; 5696 case MVT::i64: 5697 ClrNode = 5698 SDValue(CurDAG->getMachineNode( 5699 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64, 5700 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode, 5701 CurDAG->getTargetConstant(X86::sub_32bit, dl, 5702 MVT::i32)), 5703 0); 5704 break; 5705 default: 5706 llvm_unreachable("Unexpected division source"); 5707 } 5708 5709 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg, 5710 ClrNode, InGlue).getValue(1); 5711 } 5712 } 5713 5714 if (foldedLoad) { 5715 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 5716 InGlue }; 5717 MachineSDNode *CNode = 5718 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops); 5719 InGlue = SDValue(CNode, 1); 5720 // Update the chain. 5721 ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 5722 // Record the mem-refs 5723 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); 5724 } else { 5725 InGlue = 5726 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0); 5727 } 5728 5729 // Prevent use of AH in a REX instruction by explicitly copying it to 5730 // an ABCD_L register. 5731 // 5732 // The current assumption of the register allocator is that isel 5733 // won't generate explicit references to the GR8_ABCD_H registers. If 5734 // the allocator and/or the backend get enhanced to be more robust in 5735 // that regard, this can be, and should be, removed. 5736 if (HiReg == X86::AH && !SDValue(Node, 1).use_empty()) { 5737 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8); 5738 unsigned AHExtOpcode = 5739 isSigned ? X86::MOVSX32rr8_NOREX : X86::MOVZX32rr8_NOREX; 5740 5741 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32, 5742 MVT::Glue, AHCopy, InGlue); 5743 SDValue Result(RNode, 0); 5744 InGlue = SDValue(RNode, 1); 5745 5746 Result = 5747 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result); 5748 5749 ReplaceUses(SDValue(Node, 1), Result); 5750 LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); 5751 dbgs() << '\n'); 5752 } 5753 // Copy the division (low) result, if it is needed. 5754 if (!SDValue(Node, 0).use_empty()) { 5755 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 5756 LoReg, NVT, InGlue); 5757 InGlue = Result.getValue(2); 5758 ReplaceUses(SDValue(Node, 0), Result); 5759 LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); 5760 dbgs() << '\n'); 5761 } 5762 // Copy the remainder (high) result, if it is needed. 5763 if (!SDValue(Node, 1).use_empty()) { 5764 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 5765 HiReg, NVT, InGlue); 5766 InGlue = Result.getValue(2); 5767 ReplaceUses(SDValue(Node, 1), Result); 5768 LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); 5769 dbgs() << '\n'); 5770 } 5771 CurDAG->RemoveDeadNode(Node); 5772 return; 5773 } 5774 5775 case X86ISD::FCMP: 5776 case X86ISD::STRICT_FCMP: 5777 case X86ISD::STRICT_FCMPS: { 5778 bool IsStrictCmp = Node->getOpcode() == X86ISD::STRICT_FCMP || 5779 Node->getOpcode() == X86ISD::STRICT_FCMPS; 5780 SDValue N0 = Node->getOperand(IsStrictCmp ? 1 : 0); 5781 SDValue N1 = Node->getOperand(IsStrictCmp ? 2 : 1); 5782 5783 // Save the original VT of the compare. 5784 MVT CmpVT = N0.getSimpleValueType(); 5785 5786 // Floating point needs special handling if we don't have FCOMI. 5787 if (Subtarget->canUseCMOV()) 5788 break; 5789 5790 bool IsSignaling = Node->getOpcode() == X86ISD::STRICT_FCMPS; 5791 5792 unsigned Opc; 5793 switch (CmpVT.SimpleTy) { 5794 default: llvm_unreachable("Unexpected type!"); 5795 case MVT::f32: 5796 Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32; 5797 break; 5798 case MVT::f64: 5799 Opc = IsSignaling ? X86::COM_Fpr64 : X86::UCOM_Fpr64; 5800 break; 5801 case MVT::f80: 5802 Opc = IsSignaling ? X86::COM_Fpr80 : X86::UCOM_Fpr80; 5803 break; 5804 } 5805 5806 SDValue Chain = 5807 IsStrictCmp ? Node->getOperand(0) : CurDAG->getEntryNode(); 5808 SDValue Glue; 5809 if (IsStrictCmp) { 5810 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue); 5811 Chain = SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0); 5812 Glue = Chain.getValue(1); 5813 } else { 5814 Glue = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N0, N1), 0); 5815 } 5816 5817 // Move FPSW to AX. 5818 SDValue FNSTSW = 5819 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, Glue), 0); 5820 5821 // Extract upper 8-bits of AX. 5822 SDValue Extract = 5823 CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, MVT::i8, FNSTSW); 5824 5825 // Move AH into flags. 5826 // Some 64-bit targets lack SAHF support, but they do support FCOMI. 5827 assert(Subtarget->canUseLAHFSAHF() && 5828 "Target doesn't support SAHF or FCOMI?"); 5829 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract, SDValue()); 5830 Chain = AH; 5831 SDValue SAHF = SDValue( 5832 CurDAG->getMachineNode(X86::SAHF, dl, MVT::i32, AH.getValue(1)), 0); 5833 5834 if (IsStrictCmp) 5835 ReplaceUses(SDValue(Node, 1), Chain); 5836 5837 ReplaceUses(SDValue(Node, 0), SAHF); 5838 CurDAG->RemoveDeadNode(Node); 5839 return; 5840 } 5841 5842 case X86ISD::CMP: { 5843 SDValue N0 = Node->getOperand(0); 5844 SDValue N1 = Node->getOperand(1); 5845 5846 // Optimizations for TEST compares. 5847 if (!isNullConstant(N1)) 5848 break; 5849 5850 // Save the original VT of the compare. 5851 MVT CmpVT = N0.getSimpleValueType(); 5852 5853 // If we are comparing (and (shr X, C, Mask) with 0, emit a BEXTR followed 5854 // by a test instruction. The test should be removed later by 5855 // analyzeCompare if we are using only the zero flag. 5856 // TODO: Should we check the users and use the BEXTR flags directly? 5857 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) { 5858 if (MachineSDNode *NewNode = matchBEXTRFromAndImm(N0.getNode())) { 5859 unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr 5860 : X86::TEST32rr; 5861 SDValue BEXTR = SDValue(NewNode, 0); 5862 NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR); 5863 ReplaceUses(SDValue(Node, 0), SDValue(NewNode, 0)); 5864 CurDAG->RemoveDeadNode(Node); 5865 return; 5866 } 5867 } 5868 5869 // We can peek through truncates, but we need to be careful below. 5870 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse()) 5871 N0 = N0.getOperand(0); 5872 5873 // Look for (X86cmp (and $op, $imm), 0) and see if we can convert it to 5874 // use a smaller encoding. 5875 // Look past the truncate if CMP is the only use of it. 5876 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && 5877 N0.getValueType() != MVT::i8) { 5878 auto *MaskC = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 5879 if (!MaskC) 5880 break; 5881 5882 // We may have looked through a truncate so mask off any bits that 5883 // shouldn't be part of the compare. 5884 uint64_t Mask = MaskC->getZExtValue(); 5885 Mask &= maskTrailingOnes<uint64_t>(CmpVT.getScalarSizeInBits()); 5886 5887 // Check if we can replace AND+IMM{32,64} with a shift. This is possible 5888 // for masks like 0xFF000000 or 0x00FFFFFF and if we care only about the 5889 // zero flag. 5890 if (CmpVT == MVT::i64 && !isInt<8>(Mask) && isShiftedMask_64(Mask) && 5891 onlyUsesZeroFlag(SDValue(Node, 0))) { 5892 unsigned ShiftOpcode = ISD::DELETED_NODE; 5893 unsigned ShiftAmt; 5894 unsigned SubRegIdx; 5895 MVT SubRegVT; 5896 unsigned TestOpcode; 5897 unsigned LeadingZeros = llvm::countl_zero(Mask); 5898 unsigned TrailingZeros = llvm::countr_zero(Mask); 5899 5900 // With leading/trailing zeros, the transform is profitable if we can 5901 // eliminate a movabsq or shrink a 32-bit immediate to 8-bit without 5902 // incurring any extra register moves. 5903 bool SavesBytes = !isInt<32>(Mask) || N0.getOperand(0).hasOneUse(); 5904 if (LeadingZeros == 0 && SavesBytes) { 5905 // If the mask covers the most significant bit, then we can replace 5906 // TEST+AND with a SHR and check eflags. 5907 // This emits a redundant TEST which is subsequently eliminated. 5908 ShiftOpcode = X86::SHR64ri; 5909 ShiftAmt = TrailingZeros; 5910 SubRegIdx = 0; 5911 TestOpcode = X86::TEST64rr; 5912 } else if (TrailingZeros == 0 && SavesBytes) { 5913 // If the mask covers the least significant bit, then we can replace 5914 // TEST+AND with a SHL and check eflags. 5915 // This emits a redundant TEST which is subsequently eliminated. 5916 ShiftOpcode = X86::SHL64ri; 5917 ShiftAmt = LeadingZeros; 5918 SubRegIdx = 0; 5919 TestOpcode = X86::TEST64rr; 5920 } else if (MaskC->hasOneUse() && !isInt<32>(Mask)) { 5921 // If the shifted mask extends into the high half and is 8/16/32 bits 5922 // wide, then replace it with a SHR and a TEST8rr/TEST16rr/TEST32rr. 5923 unsigned PopCount = 64 - LeadingZeros - TrailingZeros; 5924 if (PopCount == 8) { 5925 ShiftOpcode = X86::SHR64ri; 5926 ShiftAmt = TrailingZeros; 5927 SubRegIdx = X86::sub_8bit; 5928 SubRegVT = MVT::i8; 5929 TestOpcode = X86::TEST8rr; 5930 } else if (PopCount == 16) { 5931 ShiftOpcode = X86::SHR64ri; 5932 ShiftAmt = TrailingZeros; 5933 SubRegIdx = X86::sub_16bit; 5934 SubRegVT = MVT::i16; 5935 TestOpcode = X86::TEST16rr; 5936 } else if (PopCount == 32) { 5937 ShiftOpcode = X86::SHR64ri; 5938 ShiftAmt = TrailingZeros; 5939 SubRegIdx = X86::sub_32bit; 5940 SubRegVT = MVT::i32; 5941 TestOpcode = X86::TEST32rr; 5942 } 5943 } 5944 if (ShiftOpcode != ISD::DELETED_NODE) { 5945 SDValue ShiftC = CurDAG->getTargetConstant(ShiftAmt, dl, MVT::i64); 5946 SDValue Shift = SDValue( 5947 CurDAG->getMachineNode(ShiftOpcode, dl, MVT::i64, MVT::i32, 5948 N0.getOperand(0), ShiftC), 5949 0); 5950 if (SubRegIdx != 0) { 5951 Shift = 5952 CurDAG->getTargetExtractSubreg(SubRegIdx, dl, SubRegVT, Shift); 5953 } 5954 MachineSDNode *Test = 5955 CurDAG->getMachineNode(TestOpcode, dl, MVT::i32, Shift, Shift); 5956 ReplaceNode(Node, Test); 5957 return; 5958 } 5959 } 5960 5961 MVT VT; 5962 int SubRegOp; 5963 unsigned ROpc, MOpc; 5964 5965 // For each of these checks we need to be careful if the sign flag is 5966 // being used. It is only safe to use the sign flag in two conditions, 5967 // either the sign bit in the shrunken mask is zero or the final test 5968 // size is equal to the original compare size. 5969 5970 if (isUInt<8>(Mask) && 5971 (!(Mask & 0x80) || CmpVT == MVT::i8 || 5972 hasNoSignFlagUses(SDValue(Node, 0)))) { 5973 // For example, convert "testl %eax, $8" to "testb %al, $8" 5974 VT = MVT::i8; 5975 SubRegOp = X86::sub_8bit; 5976 ROpc = X86::TEST8ri; 5977 MOpc = X86::TEST8mi; 5978 } else if (OptForMinSize && isUInt<16>(Mask) && 5979 (!(Mask & 0x8000) || CmpVT == MVT::i16 || 5980 hasNoSignFlagUses(SDValue(Node, 0)))) { 5981 // For example, "testl %eax, $32776" to "testw %ax, $32776". 5982 // NOTE: We only want to form TESTW instructions if optimizing for 5983 // min size. Otherwise we only save one byte and possibly get a length 5984 // changing prefix penalty in the decoders. 5985 VT = MVT::i16; 5986 SubRegOp = X86::sub_16bit; 5987 ROpc = X86::TEST16ri; 5988 MOpc = X86::TEST16mi; 5989 } else if (isUInt<32>(Mask) && N0.getValueType() != MVT::i16 && 5990 ((!(Mask & 0x80000000) && 5991 // Without minsize 16-bit Cmps can get here so we need to 5992 // be sure we calculate the correct sign flag if needed. 5993 (CmpVT != MVT::i16 || !(Mask & 0x8000))) || 5994 CmpVT == MVT::i32 || 5995 hasNoSignFlagUses(SDValue(Node, 0)))) { 5996 // For example, "testq %rax, $268468232" to "testl %eax, $268468232". 5997 // NOTE: We only want to run that transform if N0 is 32 or 64 bits. 5998 // Otherwize, we find ourselves in a position where we have to do 5999 // promotion. If previous passes did not promote the and, we assume 6000 // they had a good reason not to and do not promote here. 6001 VT = MVT::i32; 6002 SubRegOp = X86::sub_32bit; 6003 ROpc = X86::TEST32ri; 6004 MOpc = X86::TEST32mi; 6005 } else { 6006 // No eligible transformation was found. 6007 break; 6008 } 6009 6010 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT); 6011 SDValue Reg = N0.getOperand(0); 6012 6013 // Emit a testl or testw. 6014 MachineSDNode *NewNode; 6015 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 6016 if (tryFoldLoad(Node, N0.getNode(), Reg, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 6017 if (auto *LoadN = dyn_cast<LoadSDNode>(N0.getOperand(0).getNode())) { 6018 if (!LoadN->isSimple()) { 6019 unsigned NumVolBits = LoadN->getValueType(0).getSizeInBits(); 6020 if ((MOpc == X86::TEST8mi && NumVolBits != 8) || 6021 (MOpc == X86::TEST16mi && NumVolBits != 16) || 6022 (MOpc == X86::TEST32mi && NumVolBits != 32)) 6023 break; 6024 } 6025 } 6026 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm, 6027 Reg.getOperand(0) }; 6028 NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops); 6029 // Update the chain. 6030 ReplaceUses(Reg.getValue(1), SDValue(NewNode, 1)); 6031 // Record the mem-refs 6032 CurDAG->setNodeMemRefs(NewNode, 6033 {cast<LoadSDNode>(Reg)->getMemOperand()}); 6034 } else { 6035 // Extract the subregister if necessary. 6036 if (N0.getValueType() != VT) 6037 Reg = CurDAG->getTargetExtractSubreg(SubRegOp, dl, VT, Reg); 6038 6039 NewNode = CurDAG->getMachineNode(ROpc, dl, MVT::i32, Reg, Imm); 6040 } 6041 // Replace CMP with TEST. 6042 ReplaceNode(Node, NewNode); 6043 return; 6044 } 6045 break; 6046 } 6047 case X86ISD::PCMPISTR: { 6048 if (!Subtarget->hasSSE42()) 6049 break; 6050 6051 bool NeedIndex = !SDValue(Node, 0).use_empty(); 6052 bool NeedMask = !SDValue(Node, 1).use_empty(); 6053 // We can't fold a load if we are going to make two instructions. 6054 bool MayFoldLoad = !NeedIndex || !NeedMask; 6055 6056 MachineSDNode *CNode; 6057 if (NeedMask) { 6058 unsigned ROpc = Subtarget->hasAVX() ? X86::VPCMPISTRMrr : X86::PCMPISTRMrr; 6059 unsigned MOpc = Subtarget->hasAVX() ? X86::VPCMPISTRMrm : X86::PCMPISTRMrm; 6060 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8, Node); 6061 ReplaceUses(SDValue(Node, 1), SDValue(CNode, 0)); 6062 } 6063 if (NeedIndex || !NeedMask) { 6064 unsigned ROpc = Subtarget->hasAVX() ? X86::VPCMPISTRIrr : X86::PCMPISTRIrr; 6065 unsigned MOpc = Subtarget->hasAVX() ? X86::VPCMPISTRIrm : X86::PCMPISTRIrm; 6066 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32, Node); 6067 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 6068 } 6069 6070 // Connect the flag usage to the last instruction created. 6071 ReplaceUses(SDValue(Node, 2), SDValue(CNode, 1)); 6072 CurDAG->RemoveDeadNode(Node); 6073 return; 6074 } 6075 case X86ISD::PCMPESTR: { 6076 if (!Subtarget->hasSSE42()) 6077 break; 6078 6079 // Copy the two implicit register inputs. 6080 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX, 6081 Node->getOperand(1), 6082 SDValue()).getValue(1); 6083 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX, 6084 Node->getOperand(3), InGlue).getValue(1); 6085 6086 bool NeedIndex = !SDValue(Node, 0).use_empty(); 6087 bool NeedMask = !SDValue(Node, 1).use_empty(); 6088 // We can't fold a load if we are going to make two instructions. 6089 bool MayFoldLoad = !NeedIndex || !NeedMask; 6090 6091 MachineSDNode *CNode; 6092 if (NeedMask) { 6093 unsigned ROpc = Subtarget->hasAVX() ? X86::VPCMPESTRMrr : X86::PCMPESTRMrr; 6094 unsigned MOpc = Subtarget->hasAVX() ? X86::VPCMPESTRMrm : X86::PCMPESTRMrm; 6095 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8, Node, 6096 InGlue); 6097 ReplaceUses(SDValue(Node, 1), SDValue(CNode, 0)); 6098 } 6099 if (NeedIndex || !NeedMask) { 6100 unsigned ROpc = Subtarget->hasAVX() ? X86::VPCMPESTRIrr : X86::PCMPESTRIrr; 6101 unsigned MOpc = Subtarget->hasAVX() ? X86::VPCMPESTRIrm : X86::PCMPESTRIrm; 6102 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32, Node, InGlue); 6103 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 6104 } 6105 // Connect the flag usage to the last instruction created. 6106 ReplaceUses(SDValue(Node, 2), SDValue(CNode, 1)); 6107 CurDAG->RemoveDeadNode(Node); 6108 return; 6109 } 6110 6111 case ISD::SETCC: { 6112 if (NVT.isVector() && tryVPTESTM(Node, SDValue(Node, 0), SDValue())) 6113 return; 6114 6115 break; 6116 } 6117 6118 case ISD::STORE: 6119 if (foldLoadStoreIntoMemOperand(Node)) 6120 return; 6121 break; 6122 6123 case X86ISD::SETCC_CARRY: { 6124 MVT VT = Node->getSimpleValueType(0); 6125 SDValue Result; 6126 if (Subtarget->hasSBBDepBreaking()) { 6127 // We have to do this manually because tblgen will put the eflags copy in 6128 // the wrong place if we use an extract_subreg in the pattern. 6129 // Copy flags to the EFLAGS register and glue it to next node. 6130 SDValue EFLAGS = 6131 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS, 6132 Node->getOperand(1), SDValue()); 6133 6134 // Create a 64-bit instruction if the result is 64-bits otherwise use the 6135 // 32-bit version. 6136 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r; 6137 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32; 6138 Result = SDValue( 6139 CurDAG->getMachineNode(Opc, dl, SetVT, EFLAGS, EFLAGS.getValue(1)), 6140 0); 6141 } else { 6142 // The target does not recognize sbb with the same reg operand as a 6143 // no-source idiom, so we explicitly zero the input values. 6144 Result = getSBBZero(Node); 6145 } 6146 6147 // For less than 32-bits we need to extract from the 32-bit node. 6148 if (VT == MVT::i8 || VT == MVT::i16) { 6149 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit; 6150 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result); 6151 } 6152 6153 ReplaceUses(SDValue(Node, 0), Result); 6154 CurDAG->RemoveDeadNode(Node); 6155 return; 6156 } 6157 case X86ISD::SBB: { 6158 if (isNullConstant(Node->getOperand(0)) && 6159 isNullConstant(Node->getOperand(1))) { 6160 SDValue Result = getSBBZero(Node); 6161 6162 // Replace the flag use. 6163 ReplaceUses(SDValue(Node, 1), Result.getValue(1)); 6164 6165 // Replace the result use. 6166 if (!SDValue(Node, 0).use_empty()) { 6167 // For less than 32-bits we need to extract from the 32-bit node. 6168 MVT VT = Node->getSimpleValueType(0); 6169 if (VT == MVT::i8 || VT == MVT::i16) { 6170 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit; 6171 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result); 6172 } 6173 ReplaceUses(SDValue(Node, 0), Result); 6174 } 6175 6176 CurDAG->RemoveDeadNode(Node); 6177 return; 6178 } 6179 break; 6180 } 6181 case X86ISD::MGATHER: { 6182 auto *Mgt = cast<X86MaskedGatherSDNode>(Node); 6183 SDValue IndexOp = Mgt->getIndex(); 6184 SDValue Mask = Mgt->getMask(); 6185 MVT IndexVT = IndexOp.getSimpleValueType(); 6186 MVT ValueVT = Node->getSimpleValueType(0); 6187 MVT MaskVT = Mask.getSimpleValueType(); 6188 6189 // This is just to prevent crashes if the nodes are malformed somehow. We're 6190 // otherwise only doing loose type checking in here based on type what 6191 // a type constraint would say just like table based isel. 6192 if (!ValueVT.isVector() || !MaskVT.isVector()) 6193 break; 6194 6195 unsigned NumElts = ValueVT.getVectorNumElements(); 6196 MVT ValueSVT = ValueVT.getVectorElementType(); 6197 6198 bool IsFP = ValueSVT.isFloatingPoint(); 6199 unsigned EltSize = ValueSVT.getSizeInBits(); 6200 6201 unsigned Opc = 0; 6202 bool AVX512Gather = MaskVT.getVectorElementType() == MVT::i1; 6203 if (AVX512Gather) { 6204 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32) 6205 Opc = IsFP ? X86::VGATHERDPSZ128rm : X86::VPGATHERDDZ128rm; 6206 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32) 6207 Opc = IsFP ? X86::VGATHERDPSZ256rm : X86::VPGATHERDDZ256rm; 6208 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32) 6209 Opc = IsFP ? X86::VGATHERDPSZrm : X86::VPGATHERDDZrm; 6210 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64) 6211 Opc = IsFP ? X86::VGATHERDPDZ128rm : X86::VPGATHERDQZ128rm; 6212 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64) 6213 Opc = IsFP ? X86::VGATHERDPDZ256rm : X86::VPGATHERDQZ256rm; 6214 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64) 6215 Opc = IsFP ? X86::VGATHERDPDZrm : X86::VPGATHERDQZrm; 6216 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32) 6217 Opc = IsFP ? X86::VGATHERQPSZ128rm : X86::VPGATHERQDZ128rm; 6218 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32) 6219 Opc = IsFP ? X86::VGATHERQPSZ256rm : X86::VPGATHERQDZ256rm; 6220 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32) 6221 Opc = IsFP ? X86::VGATHERQPSZrm : X86::VPGATHERQDZrm; 6222 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64) 6223 Opc = IsFP ? X86::VGATHERQPDZ128rm : X86::VPGATHERQQZ128rm; 6224 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64) 6225 Opc = IsFP ? X86::VGATHERQPDZ256rm : X86::VPGATHERQQZ256rm; 6226 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64) 6227 Opc = IsFP ? X86::VGATHERQPDZrm : X86::VPGATHERQQZrm; 6228 } else { 6229 assert(EVT(MaskVT) == EVT(ValueVT).changeVectorElementTypeToInteger() && 6230 "Unexpected mask VT!"); 6231 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32) 6232 Opc = IsFP ? X86::VGATHERDPSrm : X86::VPGATHERDDrm; 6233 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32) 6234 Opc = IsFP ? X86::VGATHERDPSYrm : X86::VPGATHERDDYrm; 6235 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64) 6236 Opc = IsFP ? X86::VGATHERDPDrm : X86::VPGATHERDQrm; 6237 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64) 6238 Opc = IsFP ? X86::VGATHERDPDYrm : X86::VPGATHERDQYrm; 6239 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32) 6240 Opc = IsFP ? X86::VGATHERQPSrm : X86::VPGATHERQDrm; 6241 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32) 6242 Opc = IsFP ? X86::VGATHERQPSYrm : X86::VPGATHERQDYrm; 6243 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64) 6244 Opc = IsFP ? X86::VGATHERQPDrm : X86::VPGATHERQQrm; 6245 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64) 6246 Opc = IsFP ? X86::VGATHERQPDYrm : X86::VPGATHERQQYrm; 6247 } 6248 6249 if (!Opc) 6250 break; 6251 6252 SDValue Base, Scale, Index, Disp, Segment; 6253 if (!selectVectorAddr(Mgt, Mgt->getBasePtr(), IndexOp, Mgt->getScale(), 6254 Base, Scale, Index, Disp, Segment)) 6255 break; 6256 6257 SDValue PassThru = Mgt->getPassThru(); 6258 SDValue Chain = Mgt->getChain(); 6259 // Gather instructions have a mask output not in the ISD node. 6260 SDVTList VTs = CurDAG->getVTList(ValueVT, MaskVT, MVT::Other); 6261 6262 MachineSDNode *NewNode; 6263 if (AVX512Gather) { 6264 SDValue Ops[] = {PassThru, Mask, Base, Scale, 6265 Index, Disp, Segment, Chain}; 6266 NewNode = CurDAG->getMachineNode(Opc, SDLoc(dl), VTs, Ops); 6267 } else { 6268 SDValue Ops[] = {PassThru, Base, Scale, Index, 6269 Disp, Segment, Mask, Chain}; 6270 NewNode = CurDAG->getMachineNode(Opc, SDLoc(dl), VTs, Ops); 6271 } 6272 CurDAG->setNodeMemRefs(NewNode, {Mgt->getMemOperand()}); 6273 ReplaceUses(SDValue(Node, 0), SDValue(NewNode, 0)); 6274 ReplaceUses(SDValue(Node, 1), SDValue(NewNode, 2)); 6275 CurDAG->RemoveDeadNode(Node); 6276 return; 6277 } 6278 case X86ISD::MSCATTER: { 6279 auto *Sc = cast<X86MaskedScatterSDNode>(Node); 6280 SDValue Value = Sc->getValue(); 6281 SDValue IndexOp = Sc->getIndex(); 6282 MVT IndexVT = IndexOp.getSimpleValueType(); 6283 MVT ValueVT = Value.getSimpleValueType(); 6284 6285 // This is just to prevent crashes if the nodes are malformed somehow. We're 6286 // otherwise only doing loose type checking in here based on type what 6287 // a type constraint would say just like table based isel. 6288 if (!ValueVT.isVector()) 6289 break; 6290 6291 unsigned NumElts = ValueVT.getVectorNumElements(); 6292 MVT ValueSVT = ValueVT.getVectorElementType(); 6293 6294 bool IsFP = ValueSVT.isFloatingPoint(); 6295 unsigned EltSize = ValueSVT.getSizeInBits(); 6296 6297 unsigned Opc; 6298 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32) 6299 Opc = IsFP ? X86::VSCATTERDPSZ128mr : X86::VPSCATTERDDZ128mr; 6300 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32) 6301 Opc = IsFP ? X86::VSCATTERDPSZ256mr : X86::VPSCATTERDDZ256mr; 6302 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32) 6303 Opc = IsFP ? X86::VSCATTERDPSZmr : X86::VPSCATTERDDZmr; 6304 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64) 6305 Opc = IsFP ? X86::VSCATTERDPDZ128mr : X86::VPSCATTERDQZ128mr; 6306 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64) 6307 Opc = IsFP ? X86::VSCATTERDPDZ256mr : X86::VPSCATTERDQZ256mr; 6308 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64) 6309 Opc = IsFP ? X86::VSCATTERDPDZmr : X86::VPSCATTERDQZmr; 6310 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32) 6311 Opc = IsFP ? X86::VSCATTERQPSZ128mr : X86::VPSCATTERQDZ128mr; 6312 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32) 6313 Opc = IsFP ? X86::VSCATTERQPSZ256mr : X86::VPSCATTERQDZ256mr; 6314 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32) 6315 Opc = IsFP ? X86::VSCATTERQPSZmr : X86::VPSCATTERQDZmr; 6316 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64) 6317 Opc = IsFP ? X86::VSCATTERQPDZ128mr : X86::VPSCATTERQQZ128mr; 6318 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64) 6319 Opc = IsFP ? X86::VSCATTERQPDZ256mr : X86::VPSCATTERQQZ256mr; 6320 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64) 6321 Opc = IsFP ? X86::VSCATTERQPDZmr : X86::VPSCATTERQQZmr; 6322 else 6323 break; 6324 6325 SDValue Base, Scale, Index, Disp, Segment; 6326 if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(), 6327 Base, Scale, Index, Disp, Segment)) 6328 break; 6329 6330 SDValue Mask = Sc->getMask(); 6331 SDValue Chain = Sc->getChain(); 6332 // Scatter instructions have a mask output not in the ISD node. 6333 SDVTList VTs = CurDAG->getVTList(Mask.getValueType(), MVT::Other); 6334 SDValue Ops[] = {Base, Scale, Index, Disp, Segment, Mask, Value, Chain}; 6335 6336 MachineSDNode *NewNode = CurDAG->getMachineNode(Opc, SDLoc(dl), VTs, Ops); 6337 CurDAG->setNodeMemRefs(NewNode, {Sc->getMemOperand()}); 6338 ReplaceUses(SDValue(Node, 0), SDValue(NewNode, 1)); 6339 CurDAG->RemoveDeadNode(Node); 6340 return; 6341 } 6342 case ISD::PREALLOCATED_SETUP: { 6343 auto *MFI = CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>(); 6344 auto CallId = MFI->getPreallocatedIdForCallSite( 6345 cast<SrcValueSDNode>(Node->getOperand(1))->getValue()); 6346 SDValue Chain = Node->getOperand(0); 6347 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32); 6348 MachineSDNode *New = CurDAG->getMachineNode( 6349 TargetOpcode::PREALLOCATED_SETUP, dl, MVT::Other, CallIdValue, Chain); 6350 ReplaceUses(SDValue(Node, 0), SDValue(New, 0)); // Chain 6351 CurDAG->RemoveDeadNode(Node); 6352 return; 6353 } 6354 case ISD::PREALLOCATED_ARG: { 6355 auto *MFI = CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>(); 6356 auto CallId = MFI->getPreallocatedIdForCallSite( 6357 cast<SrcValueSDNode>(Node->getOperand(1))->getValue()); 6358 SDValue Chain = Node->getOperand(0); 6359 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32); 6360 SDValue ArgIndex = Node->getOperand(2); 6361 SDValue Ops[3]; 6362 Ops[0] = CallIdValue; 6363 Ops[1] = ArgIndex; 6364 Ops[2] = Chain; 6365 MachineSDNode *New = CurDAG->getMachineNode( 6366 TargetOpcode::PREALLOCATED_ARG, dl, 6367 CurDAG->getVTList(TLI->getPointerTy(CurDAG->getDataLayout()), 6368 MVT::Other), 6369 Ops); 6370 ReplaceUses(SDValue(Node, 0), SDValue(New, 0)); // Arg pointer 6371 ReplaceUses(SDValue(Node, 1), SDValue(New, 1)); // Chain 6372 CurDAG->RemoveDeadNode(Node); 6373 return; 6374 } 6375 case X86ISD::AESENCWIDE128KL: 6376 case X86ISD::AESDECWIDE128KL: 6377 case X86ISD::AESENCWIDE256KL: 6378 case X86ISD::AESDECWIDE256KL: { 6379 if (!Subtarget->hasWIDEKL()) 6380 break; 6381 6382 unsigned Opcode; 6383 switch (Node->getOpcode()) { 6384 default: 6385 llvm_unreachable("Unexpected opcode!"); 6386 case X86ISD::AESENCWIDE128KL: 6387 Opcode = X86::AESENCWIDE128KL; 6388 break; 6389 case X86ISD::AESDECWIDE128KL: 6390 Opcode = X86::AESDECWIDE128KL; 6391 break; 6392 case X86ISD::AESENCWIDE256KL: 6393 Opcode = X86::AESENCWIDE256KL; 6394 break; 6395 case X86ISD::AESDECWIDE256KL: 6396 Opcode = X86::AESDECWIDE256KL; 6397 break; 6398 } 6399 6400 SDValue Chain = Node->getOperand(0); 6401 SDValue Addr = Node->getOperand(1); 6402 6403 SDValue Base, Scale, Index, Disp, Segment; 6404 if (!selectAddr(Node, Addr, Base, Scale, Index, Disp, Segment)) 6405 break; 6406 6407 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0, Node->getOperand(2), 6408 SDValue()); 6409 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1, Node->getOperand(3), 6410 Chain.getValue(1)); 6411 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM2, Node->getOperand(4), 6412 Chain.getValue(1)); 6413 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM3, Node->getOperand(5), 6414 Chain.getValue(1)); 6415 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM4, Node->getOperand(6), 6416 Chain.getValue(1)); 6417 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM5, Node->getOperand(7), 6418 Chain.getValue(1)); 6419 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM6, Node->getOperand(8), 6420 Chain.getValue(1)); 6421 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM7, Node->getOperand(9), 6422 Chain.getValue(1)); 6423 6424 MachineSDNode *Res = CurDAG->getMachineNode( 6425 Opcode, dl, Node->getVTList(), 6426 {Base, Scale, Index, Disp, Segment, Chain, Chain.getValue(1)}); 6427 CurDAG->setNodeMemRefs(Res, cast<MemSDNode>(Node)->getMemOperand()); 6428 ReplaceNode(Node, Res); 6429 return; 6430 } 6431 } 6432 6433 SelectCode(Node); 6434 } 6435 6436 bool X86DAGToDAGISel::SelectInlineAsmMemoryOperand( 6437 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, 6438 std::vector<SDValue> &OutOps) { 6439 SDValue Op0, Op1, Op2, Op3, Op4; 6440 switch (ConstraintID) { 6441 default: 6442 llvm_unreachable("Unexpected asm memory constraint"); 6443 case InlineAsm::ConstraintCode::o: // offsetable ?? 6444 case InlineAsm::ConstraintCode::v: // not offsetable ?? 6445 case InlineAsm::ConstraintCode::m: // memory 6446 case InlineAsm::ConstraintCode::X: 6447 case InlineAsm::ConstraintCode::p: // address 6448 if (!selectAddr(nullptr, Op, Op0, Op1, Op2, Op3, Op4)) 6449 return true; 6450 break; 6451 } 6452 6453 OutOps.push_back(Op0); 6454 OutOps.push_back(Op1); 6455 OutOps.push_back(Op2); 6456 OutOps.push_back(Op3); 6457 OutOps.push_back(Op4); 6458 return false; 6459 } 6460 6461 /// This pass converts a legalized DAG into a X86-specific DAG, 6462 /// ready for instruction scheduling. 6463 FunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM, 6464 CodeGenOptLevel OptLevel) { 6465 return new X86DAGToDAGISel(TM, OptLevel); 6466 } 6467