1 //===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file declares the SelectionDAG class, and transitively defines the 10 // SDNode class and subclasses. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_CODEGEN_SELECTIONDAG_H 15 #define LLVM_CODEGEN_SELECTIONDAG_H 16 17 #include "llvm/ADT/APFloat.h" 18 #include "llvm/ADT/APInt.h" 19 #include "llvm/ADT/ArrayRef.h" 20 #include "llvm/ADT/DenseMap.h" 21 #include "llvm/ADT/DenseSet.h" 22 #include "llvm/ADT/FoldingSet.h" 23 #include "llvm/ADT/SmallVector.h" 24 #include "llvm/ADT/StringMap.h" 25 #include "llvm/ADT/ilist.h" 26 #include "llvm/ADT/iterator.h" 27 #include "llvm/ADT/iterator_range.h" 28 #include "llvm/CodeGen/DAGCombine.h" 29 #include "llvm/CodeGen/ISDOpcodes.h" 30 #include "llvm/CodeGen/MachineFunction.h" 31 #include "llvm/CodeGen/MachineMemOperand.h" 32 #include "llvm/CodeGen/MachineValueType.h" 33 #include "llvm/CodeGen/SelectionDAGNodes.h" 34 #include "llvm/CodeGen/ValueTypes.h" 35 #include "llvm/IR/DebugLoc.h" 36 #include "llvm/IR/Metadata.h" 37 #include "llvm/Support/Allocator.h" 38 #include "llvm/Support/ArrayRecycler.h" 39 #include "llvm/Support/CodeGen.h" 40 #include "llvm/Support/ErrorHandling.h" 41 #include "llvm/Support/RecyclingAllocator.h" 42 #include <cassert> 43 #include <cstdint> 44 #include <functional> 45 #include <map> 46 #include <string> 47 #include <tuple> 48 #include <utility> 49 #include <vector> 50 51 namespace llvm { 52 53 class DIExpression; 54 class DILabel; 55 class DIVariable; 56 class Function; 57 class Pass; 58 class Type; 59 template <class GraphType> struct GraphTraits; 60 template <typename T, unsigned int N> class SmallSetVector; 61 template <typename T, typename Enable> struct FoldingSetTrait; 62 class AAResults; 63 class BlockAddress; 64 class BlockFrequencyInfo; 65 class Constant; 66 class ConstantFP; 67 class ConstantInt; 68 class DataLayout; 69 struct fltSemantics; 70 class FunctionLoweringInfo; 71 class FunctionVarLocs; 72 class GlobalValue; 73 struct KnownBits; 74 class LLVMContext; 75 class MachineBasicBlock; 76 class MachineConstantPoolValue; 77 class MCSymbol; 78 class OptimizationRemarkEmitter; 79 class ProfileSummaryInfo; 80 class SDDbgValue; 81 class SDDbgOperand; 82 class SDDbgLabel; 83 class SelectionDAG; 84 class SelectionDAGTargetInfo; 85 class TargetLibraryInfo; 86 class TargetLowering; 87 class TargetMachine; 88 class TargetSubtargetInfo; 89 class Value; 90 91 template <typename T> class GenericSSAContext; 92 using SSAContext = GenericSSAContext<Function>; 93 template <typename T> class GenericUniformityInfo; 94 using UniformityInfo = GenericUniformityInfo<SSAContext>; 95 96 class SDVTListNode : public FoldingSetNode { 97 friend struct FoldingSetTrait<SDVTListNode>; 98 99 /// A reference to an Interned FoldingSetNodeID for this node. 100 /// The Allocator in SelectionDAG holds the data. 101 /// SDVTList contains all types which are frequently accessed in SelectionDAG. 102 /// The size of this list is not expected to be big so it won't introduce 103 /// a memory penalty. 104 FoldingSetNodeIDRef FastID; 105 const EVT *VTs; 106 unsigned int NumVTs; 107 /// The hash value for SDVTList is fixed, so cache it to avoid 108 /// hash calculation. 109 unsigned HashValue; 110 111 public: 112 SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) : 113 FastID(ID), VTs(VT), NumVTs(Num) { 114 HashValue = ID.ComputeHash(); 115 } 116 117 SDVTList getSDVTList() { 118 SDVTList result = {VTs, NumVTs}; 119 return result; 120 } 121 }; 122 123 /// Specialize FoldingSetTrait for SDVTListNode 124 /// to avoid computing temp FoldingSetNodeID and hash value. 125 template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> { 126 static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) { 127 ID = X.FastID; 128 } 129 130 static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID, 131 unsigned IDHash, FoldingSetNodeID &TempID) { 132 if (X.HashValue != IDHash) 133 return false; 134 return ID == X.FastID; 135 } 136 137 static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) { 138 return X.HashValue; 139 } 140 }; 141 142 template <> struct ilist_alloc_traits<SDNode> { 143 static void deleteNode(SDNode *) { 144 llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!"); 145 } 146 }; 147 148 /// Keeps track of dbg_value information through SDISel. We do 149 /// not build SDNodes for these so as not to perturb the generated code; 150 /// instead the info is kept off to the side in this structure. Each SDNode may 151 /// have one or more associated dbg_value entries. This information is kept in 152 /// DbgValMap. 153 /// Byval parameters are handled separately because they don't use alloca's, 154 /// which busts the normal mechanism. There is good reason for handling all 155 /// parameters separately: they may not have code generated for them, they 156 /// should always go at the beginning of the function regardless of other code 157 /// motion, and debug info for them is potentially useful even if the parameter 158 /// is unused. Right now only byval parameters are handled separately. 159 class SDDbgInfo { 160 BumpPtrAllocator Alloc; 161 SmallVector<SDDbgValue*, 32> DbgValues; 162 SmallVector<SDDbgValue*, 32> ByvalParmDbgValues; 163 SmallVector<SDDbgLabel*, 4> DbgLabels; 164 using DbgValMapType = DenseMap<const SDNode *, SmallVector<SDDbgValue *, 2>>; 165 DbgValMapType DbgValMap; 166 167 public: 168 SDDbgInfo() = default; 169 SDDbgInfo(const SDDbgInfo &) = delete; 170 SDDbgInfo &operator=(const SDDbgInfo &) = delete; 171 172 void add(SDDbgValue *V, bool isParameter); 173 174 void add(SDDbgLabel *L) { DbgLabels.push_back(L); } 175 176 /// Invalidate all DbgValues attached to the node and remove 177 /// it from the Node-to-DbgValues map. 178 void erase(const SDNode *Node); 179 180 void clear() { 181 DbgValMap.clear(); 182 DbgValues.clear(); 183 ByvalParmDbgValues.clear(); 184 DbgLabels.clear(); 185 Alloc.Reset(); 186 } 187 188 BumpPtrAllocator &getAlloc() { return Alloc; } 189 190 bool empty() const { 191 return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty(); 192 } 193 194 ArrayRef<SDDbgValue*> getSDDbgValues(const SDNode *Node) const { 195 auto I = DbgValMap.find(Node); 196 if (I != DbgValMap.end()) 197 return I->second; 198 return ArrayRef<SDDbgValue*>(); 199 } 200 201 using DbgIterator = SmallVectorImpl<SDDbgValue*>::iterator; 202 using DbgLabelIterator = SmallVectorImpl<SDDbgLabel*>::iterator; 203 204 DbgIterator DbgBegin() { return DbgValues.begin(); } 205 DbgIterator DbgEnd() { return DbgValues.end(); } 206 DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); } 207 DbgIterator ByvalParmDbgEnd() { return ByvalParmDbgValues.end(); } 208 DbgLabelIterator DbgLabelBegin() { return DbgLabels.begin(); } 209 DbgLabelIterator DbgLabelEnd() { return DbgLabels.end(); } 210 }; 211 212 void checkForCycles(const SelectionDAG *DAG, bool force = false); 213 214 /// This is used to represent a portion of an LLVM function in a low-level 215 /// Data Dependence DAG representation suitable for instruction selection. 216 /// This DAG is constructed as the first step of instruction selection in order 217 /// to allow implementation of machine specific optimizations 218 /// and code simplifications. 219 /// 220 /// The representation used by the SelectionDAG is a target-independent 221 /// representation, which has some similarities to the GCC RTL representation, 222 /// but is significantly more simple, powerful, and is a graph form instead of a 223 /// linear form. 224 /// 225 class SelectionDAG { 226 const TargetMachine &TM; 227 const SelectionDAGTargetInfo *TSI = nullptr; 228 const TargetLowering *TLI = nullptr; 229 const TargetLibraryInfo *LibInfo = nullptr; 230 const FunctionVarLocs *FnVarLocs = nullptr; 231 MachineFunction *MF; 232 Pass *SDAGISelPass = nullptr; 233 LLVMContext *Context; 234 CodeGenOpt::Level OptLevel; 235 236 UniformityInfo *UA = nullptr; 237 FunctionLoweringInfo * FLI = nullptr; 238 239 /// The function-level optimization remark emitter. Used to emit remarks 240 /// whenever manipulating the DAG. 241 OptimizationRemarkEmitter *ORE; 242 243 ProfileSummaryInfo *PSI = nullptr; 244 BlockFrequencyInfo *BFI = nullptr; 245 246 /// List of non-single value types. 247 FoldingSet<SDVTListNode> VTListMap; 248 249 /// Pool allocation for misc. objects that are created once per SelectionDAG. 250 BumpPtrAllocator Allocator; 251 252 /// The starting token. 253 SDNode EntryNode; 254 255 /// The root of the entire DAG. 256 SDValue Root; 257 258 /// A linked list of nodes in the current DAG. 259 ilist<SDNode> AllNodes; 260 261 /// The AllocatorType for allocating SDNodes. We use 262 /// pool allocation with recycling. 263 using NodeAllocatorType = RecyclingAllocator<BumpPtrAllocator, SDNode, 264 sizeof(LargestSDNode), 265 alignof(MostAlignedSDNode)>; 266 267 /// Pool allocation for nodes. 268 NodeAllocatorType NodeAllocator; 269 270 /// This structure is used to memoize nodes, automatically performing 271 /// CSE with existing nodes when a duplicate is requested. 272 FoldingSet<SDNode> CSEMap; 273 274 /// Pool allocation for machine-opcode SDNode operands. 275 BumpPtrAllocator OperandAllocator; 276 ArrayRecycler<SDUse> OperandRecycler; 277 278 /// Tracks dbg_value and dbg_label information through SDISel. 279 SDDbgInfo *DbgInfo; 280 281 using CallSiteInfo = MachineFunction::CallSiteInfo; 282 using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl; 283 284 struct NodeExtraInfo { 285 CallSiteInfo CSInfo; 286 MDNode *HeapAllocSite = nullptr; 287 MDNode *PCSections = nullptr; 288 bool NoMerge = false; 289 }; 290 /// Out-of-line extra information for SDNodes. 291 DenseMap<const SDNode *, NodeExtraInfo> SDEI; 292 293 /// PersistentId counter to be used when inserting the next 294 /// SDNode to this SelectionDAG. We do not place that under 295 /// `#if LLVM_ENABLE_ABI_BREAKING_CHECKS` intentionally because 296 /// it adds unneeded complexity without noticeable 297 /// benefits (see discussion with @thakis in D120714). 298 uint16_t NextPersistentId = 0; 299 300 public: 301 /// Clients of various APIs that cause global effects on 302 /// the DAG can optionally implement this interface. This allows the clients 303 /// to handle the various sorts of updates that happen. 304 /// 305 /// A DAGUpdateListener automatically registers itself with DAG when it is 306 /// constructed, and removes itself when destroyed in RAII fashion. 307 struct DAGUpdateListener { 308 DAGUpdateListener *const Next; 309 SelectionDAG &DAG; 310 311 explicit DAGUpdateListener(SelectionDAG &D) 312 : Next(D.UpdateListeners), DAG(D) { 313 DAG.UpdateListeners = this; 314 } 315 316 virtual ~DAGUpdateListener() { 317 assert(DAG.UpdateListeners == this && 318 "DAGUpdateListeners must be destroyed in LIFO order"); 319 DAG.UpdateListeners = Next; 320 } 321 322 /// The node N that was deleted and, if E is not null, an 323 /// equivalent node E that replaced it. 324 virtual void NodeDeleted(SDNode *N, SDNode *E); 325 326 /// The node N that was updated. 327 virtual void NodeUpdated(SDNode *N); 328 329 /// The node N that was inserted. 330 virtual void NodeInserted(SDNode *N); 331 }; 332 333 struct DAGNodeDeletedListener : public DAGUpdateListener { 334 std::function<void(SDNode *, SDNode *)> Callback; 335 336 DAGNodeDeletedListener(SelectionDAG &DAG, 337 std::function<void(SDNode *, SDNode *)> Callback) 338 : DAGUpdateListener(DAG), Callback(std::move(Callback)) {} 339 340 void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); } 341 342 private: 343 virtual void anchor(); 344 }; 345 346 struct DAGNodeInsertedListener : public DAGUpdateListener { 347 std::function<void(SDNode *)> Callback; 348 349 DAGNodeInsertedListener(SelectionDAG &DAG, 350 std::function<void(SDNode *)> Callback) 351 : DAGUpdateListener(DAG), Callback(std::move(Callback)) {} 352 353 void NodeInserted(SDNode *N) override { Callback(N); } 354 355 private: 356 virtual void anchor(); 357 }; 358 359 /// Help to insert SDNodeFlags automatically in transforming. Use 360 /// RAII to save and resume flags in current scope. 361 class FlagInserter { 362 SelectionDAG &DAG; 363 SDNodeFlags Flags; 364 FlagInserter *LastInserter; 365 366 public: 367 FlagInserter(SelectionDAG &SDAG, SDNodeFlags Flags) 368 : DAG(SDAG), Flags(Flags), 369 LastInserter(SDAG.getFlagInserter()) { 370 SDAG.setFlagInserter(this); 371 } 372 FlagInserter(SelectionDAG &SDAG, SDNode *N) 373 : FlagInserter(SDAG, N->getFlags()) {} 374 375 FlagInserter(const FlagInserter &) = delete; 376 FlagInserter &operator=(const FlagInserter &) = delete; 377 ~FlagInserter() { DAG.setFlagInserter(LastInserter); } 378 379 SDNodeFlags getFlags() const { return Flags; } 380 }; 381 382 /// When true, additional steps are taken to 383 /// ensure that getConstant() and similar functions return DAG nodes that 384 /// have legal types. This is important after type legalization since 385 /// any illegally typed nodes generated after this point will not experience 386 /// type legalization. 387 bool NewNodesMustHaveLegalTypes = false; 388 389 private: 390 /// DAGUpdateListener is a friend so it can manipulate the listener stack. 391 friend struct DAGUpdateListener; 392 393 /// Linked list of registered DAGUpdateListener instances. 394 /// This stack is maintained by DAGUpdateListener RAII. 395 DAGUpdateListener *UpdateListeners = nullptr; 396 397 /// Implementation of setSubgraphColor. 398 /// Return whether we had to truncate the search. 399 bool setSubgraphColorHelper(SDNode *N, const char *Color, 400 DenseSet<SDNode *> &visited, 401 int level, bool &printed); 402 403 template <typename SDNodeT, typename... ArgTypes> 404 SDNodeT *newSDNode(ArgTypes &&... Args) { 405 return new (NodeAllocator.template Allocate<SDNodeT>()) 406 SDNodeT(std::forward<ArgTypes>(Args)...); 407 } 408 409 /// Build a synthetic SDNodeT with the given args and extract its subclass 410 /// data as an integer (e.g. for use in a folding set). 411 /// 412 /// The args to this function are the same as the args to SDNodeT's 413 /// constructor, except the second arg (assumed to be a const DebugLoc&) is 414 /// omitted. 415 template <typename SDNodeT, typename... ArgTypes> 416 static uint16_t getSyntheticNodeSubclassData(unsigned IROrder, 417 ArgTypes &&... Args) { 418 // The compiler can reduce this expression to a constant iff we pass an 419 // empty DebugLoc. Thankfully, the debug location doesn't have any bearing 420 // on the subclass data. 421 return SDNodeT(IROrder, DebugLoc(), std::forward<ArgTypes>(Args)...) 422 .getRawSubclassData(); 423 } 424 425 template <typename SDNodeTy> 426 static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order, 427 SDVTList VTs, EVT MemoryVT, 428 MachineMemOperand *MMO) { 429 return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO) 430 .getRawSubclassData(); 431 } 432 433 void createOperands(SDNode *Node, ArrayRef<SDValue> Vals); 434 435 void removeOperands(SDNode *Node) { 436 if (!Node->OperandList) 437 return; 438 OperandRecycler.deallocate( 439 ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands), 440 Node->OperandList); 441 Node->NumOperands = 0; 442 Node->OperandList = nullptr; 443 } 444 void CreateTopologicalOrder(std::vector<SDNode*>& Order); 445 446 public: 447 // Maximum depth for recursive analysis such as computeKnownBits, etc. 448 static constexpr unsigned MaxRecursionDepth = 6; 449 450 explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level); 451 SelectionDAG(const SelectionDAG &) = delete; 452 SelectionDAG &operator=(const SelectionDAG &) = delete; 453 ~SelectionDAG(); 454 455 /// Prepare this SelectionDAG to process code in the given MachineFunction. 456 void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, 457 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, 458 UniformityInfo *UA, ProfileSummaryInfo *PSIin, 459 BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs); 460 461 void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) { 462 FLI = FuncInfo; 463 } 464 465 /// Clear state and free memory necessary to make this 466 /// SelectionDAG ready to process a new block. 467 void clear(); 468 469 MachineFunction &getMachineFunction() const { return *MF; } 470 const Pass *getPass() const { return SDAGISelPass; } 471 472 const DataLayout &getDataLayout() const { return MF->getDataLayout(); } 473 const TargetMachine &getTarget() const { return TM; } 474 const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); } 475 template <typename STC> const STC &getSubtarget() const { 476 return MF->getSubtarget<STC>(); 477 } 478 const TargetLowering &getTargetLoweringInfo() const { return *TLI; } 479 const TargetLibraryInfo &getLibInfo() const { return *LibInfo; } 480 const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; } 481 const UniformityInfo *getUniformityInfo() const { return UA; } 482 /// Returns the result of the AssignmentTrackingAnalysis pass if it's 483 /// available, otherwise return nullptr. 484 const FunctionVarLocs *getFunctionVarLocs() const { return FnVarLocs; } 485 LLVMContext *getContext() const { return Context; } 486 OptimizationRemarkEmitter &getORE() const { return *ORE; } 487 ProfileSummaryInfo *getPSI() const { return PSI; } 488 BlockFrequencyInfo *getBFI() const { return BFI; } 489 490 FlagInserter *getFlagInserter() { return Inserter; } 491 void setFlagInserter(FlagInserter *FI) { Inserter = FI; } 492 493 /// Just dump dot graph to a user-provided path and title. 494 /// This doesn't open the dot viewer program and 495 /// helps visualization when outside debugging session. 496 /// FileName expects absolute path. If provided 497 /// without any path separators then the file 498 /// will be created in the current directory. 499 /// Error will be emitted if the path is insane. 500 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 501 LLVM_DUMP_METHOD void dumpDotGraph(const Twine &FileName, const Twine &Title); 502 #endif 503 504 /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'. 505 void viewGraph(const std::string &Title); 506 void viewGraph(); 507 508 #if LLVM_ENABLE_ABI_BREAKING_CHECKS 509 std::map<const SDNode *, std::string> NodeGraphAttrs; 510 #endif 511 512 /// Clear all previously defined node graph attributes. 513 /// Intended to be used from a debugging tool (eg. gdb). 514 void clearGraphAttrs(); 515 516 /// Set graph attributes for a node. (eg. "color=red".) 517 void setGraphAttrs(const SDNode *N, const char *Attrs); 518 519 /// Get graph attributes for a node. (eg. "color=red".) 520 /// Used from getNodeAttributes. 521 std::string getGraphAttrs(const SDNode *N) const; 522 523 /// Convenience for setting node color attribute. 524 void setGraphColor(const SDNode *N, const char *Color); 525 526 /// Convenience for setting subgraph color attribute. 527 void setSubgraphColor(SDNode *N, const char *Color); 528 529 using allnodes_const_iterator = ilist<SDNode>::const_iterator; 530 531 allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); } 532 allnodes_const_iterator allnodes_end() const { return AllNodes.end(); } 533 534 using allnodes_iterator = ilist<SDNode>::iterator; 535 536 allnodes_iterator allnodes_begin() { return AllNodes.begin(); } 537 allnodes_iterator allnodes_end() { return AllNodes.end(); } 538 539 ilist<SDNode>::size_type allnodes_size() const { 540 return AllNodes.size(); 541 } 542 543 iterator_range<allnodes_iterator> allnodes() { 544 return make_range(allnodes_begin(), allnodes_end()); 545 } 546 iterator_range<allnodes_const_iterator> allnodes() const { 547 return make_range(allnodes_begin(), allnodes_end()); 548 } 549 550 /// Return the root tag of the SelectionDAG. 551 const SDValue &getRoot() const { return Root; } 552 553 /// Return the token chain corresponding to the entry of the function. 554 SDValue getEntryNode() const { 555 return SDValue(const_cast<SDNode *>(&EntryNode), 0); 556 } 557 558 /// Set the current root tag of the SelectionDAG. 559 /// 560 const SDValue &setRoot(SDValue N) { 561 assert((!N.getNode() || N.getValueType() == MVT::Other) && 562 "DAG root value is not a chain!"); 563 if (N.getNode()) 564 checkForCycles(N.getNode(), this); 565 Root = N; 566 if (N.getNode()) 567 checkForCycles(this); 568 return Root; 569 } 570 571 #ifndef NDEBUG 572 void VerifyDAGDivergence(); 573 #endif 574 575 /// This iterates over the nodes in the SelectionDAG, folding 576 /// certain types of nodes together, or eliminating superfluous nodes. The 577 /// Level argument controls whether Combine is allowed to produce nodes and 578 /// types that are illegal on the target. 579 void Combine(CombineLevel Level, AAResults *AA, 580 CodeGenOpt::Level OptLevel); 581 582 /// This transforms the SelectionDAG into a SelectionDAG that 583 /// only uses types natively supported by the target. 584 /// Returns "true" if it made any changes. 585 /// 586 /// Note that this is an involved process that may invalidate pointers into 587 /// the graph. 588 bool LegalizeTypes(); 589 590 /// This transforms the SelectionDAG into a SelectionDAG that is 591 /// compatible with the target instruction selector, as indicated by the 592 /// TargetLowering object. 593 /// 594 /// Note that this is an involved process that may invalidate pointers into 595 /// the graph. 596 void Legalize(); 597 598 /// Transforms a SelectionDAG node and any operands to it into a node 599 /// that is compatible with the target instruction selector, as indicated by 600 /// the TargetLowering object. 601 /// 602 /// \returns true if \c N is a valid, legal node after calling this. 603 /// 604 /// This essentially runs a single recursive walk of the \c Legalize process 605 /// over the given node (and its operands). This can be used to incrementally 606 /// legalize the DAG. All of the nodes which are directly replaced, 607 /// potentially including N, are added to the output parameter \c 608 /// UpdatedNodes so that the delta to the DAG can be understood by the 609 /// caller. 610 /// 611 /// When this returns false, N has been legalized in a way that make the 612 /// pointer passed in no longer valid. It may have even been deleted from the 613 /// DAG, and so it shouldn't be used further. When this returns true, the 614 /// N passed in is a legal node, and can be immediately processed as such. 615 /// This may still have done some work on the DAG, and will still populate 616 /// UpdatedNodes with any new nodes replacing those originally in the DAG. 617 bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes); 618 619 /// This transforms the SelectionDAG into a SelectionDAG 620 /// that only uses vector math operations supported by the target. This is 621 /// necessary as a separate step from Legalize because unrolling a vector 622 /// operation can introduce illegal types, which requires running 623 /// LegalizeTypes again. 624 /// 625 /// This returns true if it made any changes; in that case, LegalizeTypes 626 /// is called again before Legalize. 627 /// 628 /// Note that this is an involved process that may invalidate pointers into 629 /// the graph. 630 bool LegalizeVectors(); 631 632 /// This method deletes all unreachable nodes in the SelectionDAG. 633 void RemoveDeadNodes(); 634 635 /// Remove the specified node from the system. This node must 636 /// have no referrers. 637 void DeleteNode(SDNode *N); 638 639 /// Return an SDVTList that represents the list of values specified. 640 SDVTList getVTList(EVT VT); 641 SDVTList getVTList(EVT VT1, EVT VT2); 642 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3); 643 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4); 644 SDVTList getVTList(ArrayRef<EVT> VTs); 645 646 //===--------------------------------------------------------------------===// 647 // Node creation methods. 648 649 /// Create a ConstantSDNode wrapping a constant value. 650 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR. 651 /// 652 /// If only legal types can be produced, this does the necessary 653 /// transformations (e.g., if the vector element type is illegal). 654 /// @{ 655 SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, 656 bool isTarget = false, bool isOpaque = false); 657 SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT, 658 bool isTarget = false, bool isOpaque = false); 659 660 SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false, 661 bool IsOpaque = false) { 662 return getConstant(APInt::getAllOnes(VT.getScalarSizeInBits()), DL, VT, 663 IsTarget, IsOpaque); 664 } 665 666 SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT, 667 bool isTarget = false, bool isOpaque = false); 668 SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, 669 bool isTarget = false); 670 SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, 671 bool LegalTypes = true); 672 SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, 673 bool isTarget = false); 674 675 SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, 676 bool isOpaque = false) { 677 return getConstant(Val, DL, VT, true, isOpaque); 678 } 679 SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT, 680 bool isOpaque = false) { 681 return getConstant(Val, DL, VT, true, isOpaque); 682 } 683 SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT, 684 bool isOpaque = false) { 685 return getConstant(Val, DL, VT, true, isOpaque); 686 } 687 688 /// Create a true or false constant of type \p VT using the target's 689 /// BooleanContent for type \p OpVT. 690 SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT); 691 /// @} 692 693 /// Create a ConstantFPSDNode wrapping a constant value. 694 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR. 695 /// 696 /// If only legal types can be produced, this does the necessary 697 /// transformations (e.g., if the vector element type is illegal). 698 /// The forms that take a double should only be used for simple constants 699 /// that can be exactly represented in VT. No checks are made. 700 /// @{ 701 SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, 702 bool isTarget = false); 703 SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT, 704 bool isTarget = false); 705 SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT, 706 bool isTarget = false); 707 SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) { 708 return getConstantFP(Val, DL, VT, true); 709 } 710 SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) { 711 return getConstantFP(Val, DL, VT, true); 712 } 713 SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT) { 714 return getConstantFP(Val, DL, VT, true); 715 } 716 /// @} 717 718 SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, 719 int64_t offset = 0, bool isTargetGA = false, 720 unsigned TargetFlags = 0); 721 SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, 722 int64_t offset = 0, unsigned TargetFlags = 0) { 723 return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags); 724 } 725 SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false); 726 SDValue getTargetFrameIndex(int FI, EVT VT) { 727 return getFrameIndex(FI, VT, true); 728 } 729 SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false, 730 unsigned TargetFlags = 0); 731 SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) { 732 return getJumpTable(JTI, VT, true, TargetFlags); 733 } 734 SDValue getConstantPool(const Constant *C, EVT VT, 735 MaybeAlign Align = std::nullopt, int Offs = 0, 736 bool isT = false, unsigned TargetFlags = 0); 737 SDValue getTargetConstantPool(const Constant *C, EVT VT, 738 MaybeAlign Align = std::nullopt, int Offset = 0, 739 unsigned TargetFlags = 0) { 740 return getConstantPool(C, VT, Align, Offset, true, TargetFlags); 741 } 742 SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT, 743 MaybeAlign Align = std::nullopt, int Offs = 0, 744 bool isT = false, unsigned TargetFlags = 0); 745 SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT, 746 MaybeAlign Align = std::nullopt, int Offset = 0, 747 unsigned TargetFlags = 0) { 748 return getConstantPool(C, VT, Align, Offset, true, TargetFlags); 749 } 750 SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0, 751 unsigned TargetFlags = 0); 752 // When generating a branch to a BB, we don't in general know enough 753 // to provide debug info for the BB at that time, so keep this one around. 754 SDValue getBasicBlock(MachineBasicBlock *MBB); 755 SDValue getExternalSymbol(const char *Sym, EVT VT); 756 SDValue getTargetExternalSymbol(const char *Sym, EVT VT, 757 unsigned TargetFlags = 0); 758 SDValue getMCSymbol(MCSymbol *Sym, EVT VT); 759 760 SDValue getValueType(EVT); 761 SDValue getRegister(unsigned Reg, EVT VT); 762 SDValue getRegisterMask(const uint32_t *RegMask); 763 SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label); 764 SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, 765 MCSymbol *Label); 766 SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0, 767 bool isTarget = false, unsigned TargetFlags = 0); 768 SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, 769 int64_t Offset = 0, unsigned TargetFlags = 0) { 770 return getBlockAddress(BA, VT, Offset, true, TargetFlags); 771 } 772 773 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, 774 SDValue N) { 775 return getNode(ISD::CopyToReg, dl, MVT::Other, Chain, 776 getRegister(Reg, N.getValueType()), N); 777 } 778 779 // This version of the getCopyToReg method takes an extra operand, which 780 // indicates that there is potentially an incoming glue value (if Glue is not 781 // null) and that there should be a glue result. 782 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N, 783 SDValue Glue) { 784 SDVTList VTs = getVTList(MVT::Other, MVT::Glue); 785 SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue }; 786 return getNode(ISD::CopyToReg, dl, VTs, 787 ArrayRef(Ops, Glue.getNode() ? 4 : 3)); 788 } 789 790 // Similar to last getCopyToReg() except parameter Reg is a SDValue 791 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N, 792 SDValue Glue) { 793 SDVTList VTs = getVTList(MVT::Other, MVT::Glue); 794 SDValue Ops[] = { Chain, Reg, N, Glue }; 795 return getNode(ISD::CopyToReg, dl, VTs, 796 ArrayRef(Ops, Glue.getNode() ? 4 : 3)); 797 } 798 799 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) { 800 SDVTList VTs = getVTList(VT, MVT::Other); 801 SDValue Ops[] = { Chain, getRegister(Reg, VT) }; 802 return getNode(ISD::CopyFromReg, dl, VTs, Ops); 803 } 804 805 // This version of the getCopyFromReg method takes an extra operand, which 806 // indicates that there is potentially an incoming glue value (if Glue is not 807 // null) and that there should be a glue result. 808 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT, 809 SDValue Glue) { 810 SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue); 811 SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue }; 812 return getNode(ISD::CopyFromReg, dl, VTs, 813 ArrayRef(Ops, Glue.getNode() ? 3 : 2)); 814 } 815 816 SDValue getCondCode(ISD::CondCode Cond); 817 818 /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT, 819 /// which must be a vector type, must match the number of mask elements 820 /// NumElts. An integer mask element equal to -1 is treated as undefined. 821 SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, 822 ArrayRef<int> Mask); 823 824 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT, 825 /// which must be a vector type, must match the number of operands in Ops. 826 /// The operands must have the same type as (or, for integers, a type wider 827 /// than) VT's element type. 828 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) { 829 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. 830 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); 831 } 832 833 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT, 834 /// which must be a vector type, must match the number of operands in Ops. 835 /// The operands must have the same type as (or, for integers, a type wider 836 /// than) VT's element type. 837 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDUse> Ops) { 838 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. 839 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); 840 } 841 842 /// Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all 843 /// elements. VT must be a vector type. Op's type must be the same as (or, 844 /// for integers, a type wider than) VT's element type. 845 SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) { 846 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. 847 if (Op.getOpcode() == ISD::UNDEF) { 848 assert((VT.getVectorElementType() == Op.getValueType() || 849 (VT.isInteger() && 850 VT.getVectorElementType().bitsLE(Op.getValueType()))) && 851 "A splatted value must have a width equal or (for integers) " 852 "greater than the vector element type!"); 853 return getNode(ISD::UNDEF, SDLoc(), VT); 854 } 855 856 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Op); 857 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); 858 } 859 860 // Return a splat ISD::SPLAT_VECTOR node, consisting of Op splatted to all 861 // elements. 862 SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) { 863 if (Op.getOpcode() == ISD::UNDEF) { 864 assert((VT.getVectorElementType() == Op.getValueType() || 865 (VT.isInteger() && 866 VT.getVectorElementType().bitsLE(Op.getValueType()))) && 867 "A splatted value must have a width equal or (for integers) " 868 "greater than the vector element type!"); 869 return getNode(ISD::UNDEF, SDLoc(), VT); 870 } 871 return getNode(ISD::SPLAT_VECTOR, DL, VT, Op); 872 } 873 874 /// Returns a node representing a splat of one value into all lanes 875 /// of the provided vector type. This is a utility which returns 876 /// either a BUILD_VECTOR or SPLAT_VECTOR depending on the 877 /// scalability of the desired vector type. 878 SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op) { 879 assert(VT.isVector() && "Can't splat to non-vector type"); 880 return VT.isScalableVector() ? 881 getSplatVector(VT, DL, Op) : getSplatBuildVector(VT, DL, Op); 882 } 883 884 /// Returns a vector of type ResVT whose elements contain the linear sequence 885 /// <0, Step, Step * 2, Step * 3, ...> 886 SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal); 887 888 /// Returns a vector of type ResVT whose elements contain the linear sequence 889 /// <0, 1, 2, 3, ...> 890 SDValue getStepVector(const SDLoc &DL, EVT ResVT); 891 892 /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to 893 /// the shuffle node in input but with swapped operands. 894 /// 895 /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3> 896 SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV); 897 898 /// Convert Op, which must be of float type, to the 899 /// float type VT, by either extending or rounding (by truncation). 900 SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT); 901 902 /// Convert Op, which must be a STRICT operation of float type, to the 903 /// float type VT, by either extending or rounding (by truncation). 904 std::pair<SDValue, SDValue> 905 getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT); 906 907 /// Convert *_EXTEND_VECTOR_INREG to *_EXTEND opcode. 908 static unsigned getOpcode_EXTEND(unsigned Opcode) { 909 switch (Opcode) { 910 case ISD::ANY_EXTEND: 911 case ISD::ANY_EXTEND_VECTOR_INREG: 912 return ISD::ANY_EXTEND; 913 case ISD::ZERO_EXTEND: 914 case ISD::ZERO_EXTEND_VECTOR_INREG: 915 return ISD::ZERO_EXTEND; 916 case ISD::SIGN_EXTEND: 917 case ISD::SIGN_EXTEND_VECTOR_INREG: 918 return ISD::SIGN_EXTEND; 919 } 920 llvm_unreachable("Unknown opcode"); 921 } 922 923 /// Convert *_EXTEND to *_EXTEND_VECTOR_INREG opcode. 924 static unsigned getOpcode_EXTEND_VECTOR_INREG(unsigned Opcode) { 925 switch (Opcode) { 926 case ISD::ANY_EXTEND: 927 case ISD::ANY_EXTEND_VECTOR_INREG: 928 return ISD::ANY_EXTEND_VECTOR_INREG; 929 case ISD::ZERO_EXTEND: 930 case ISD::ZERO_EXTEND_VECTOR_INREG: 931 return ISD::ZERO_EXTEND_VECTOR_INREG; 932 case ISD::SIGN_EXTEND: 933 case ISD::SIGN_EXTEND_VECTOR_INREG: 934 return ISD::SIGN_EXTEND_VECTOR_INREG; 935 } 936 llvm_unreachable("Unknown opcode"); 937 } 938 939 /// Convert Op, which must be of integer type, to the 940 /// integer type VT, by either any-extending or truncating it. 941 SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 942 943 /// Convert Op, which must be of integer type, to the 944 /// integer type VT, by either sign-extending or truncating it. 945 SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 946 947 /// Convert Op, which must be of integer type, to the 948 /// integer type VT, by either zero-extending or truncating it. 949 SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 950 951 /// Convert Op, which must be of integer type, to the 952 /// integer type VT, by either sign/zero-extending (depending on IsSigned) or 953 /// truncating it. 954 SDValue getExtOrTrunc(bool IsSigned, SDValue Op, const SDLoc &DL, EVT VT) { 955 return IsSigned ? getSExtOrTrunc(Op, DL, VT) : getZExtOrTrunc(Op, DL, VT); 956 } 957 958 /// Return the expression required to zero extend the Op 959 /// value assuming it was the smaller SrcTy value. 960 SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); 961 962 /// Convert Op, which must be of integer type, to the integer type VT, by 963 /// either truncating it or performing either zero or sign extension as 964 /// appropriate extension for the pointer's semantics. 965 SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); 966 967 /// Return the expression required to extend the Op as a pointer value 968 /// assuming it was the smaller SrcTy value. This may be either a zero extend 969 /// or a sign extend. 970 SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); 971 972 /// Convert Op, which must be of integer type, to the integer type VT, 973 /// by using an extension appropriate for the target's 974 /// BooleanContent for type OpVT or truncating it. 975 SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT); 976 977 /// Create negative operation as (SUB 0, Val). 978 SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT); 979 980 /// Create a bitwise NOT operation as (XOR Val, -1). 981 SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT); 982 983 /// Create a logical NOT operation as (XOR Val, BooleanOne). 984 SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT); 985 986 /// Create a vector-predicated logical NOT operation as (VP_XOR Val, 987 /// BooleanOne, Mask, EVL). 988 SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, 989 SDValue EVL, EVT VT); 990 991 /// Convert a vector-predicated Op, which must be an integer vector, to the 992 /// vector-type VT, by performing either vector-predicated zext or truncating 993 /// it. The Op will be returned as-is if Op and VT are vectors containing 994 /// integer with same width. 995 SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, 996 SDValue EVL); 997 998 /// Convert a vector-predicated Op, which must be of integer type, to the 999 /// vector-type integer type VT, by either truncating it or performing either 1000 /// vector-predicated zero or sign extension as appropriate extension for the 1001 /// pointer's semantics. This function just redirects to getVPZExtOrTrunc 1002 /// right now. 1003 SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, 1004 SDValue EVL); 1005 1006 /// Returns sum of the base pointer and offset. 1007 /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default. 1008 SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, 1009 const SDNodeFlags Flags = SDNodeFlags()); 1010 SDValue getMemBasePlusOffset(SDValue Base, SDValue Offset, const SDLoc &DL, 1011 const SDNodeFlags Flags = SDNodeFlags()); 1012 1013 /// Create an add instruction with appropriate flags when used for 1014 /// addressing some offset of an object. i.e. if a load is split into multiple 1015 /// components, create an add nuw from the base pointer to the offset. 1016 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset) { 1017 SDNodeFlags Flags; 1018 Flags.setNoUnsignedWrap(true); 1019 return getMemBasePlusOffset(Ptr, Offset, SL, Flags); 1020 } 1021 1022 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) { 1023 // The object itself can't wrap around the address space, so it shouldn't be 1024 // possible for the adds of the offsets to the split parts to overflow. 1025 SDNodeFlags Flags; 1026 Flags.setNoUnsignedWrap(true); 1027 return getMemBasePlusOffset(Ptr, Offset, SL, Flags); 1028 } 1029 1030 /// Return a new CALLSEQ_START node, that starts new call frame, in which 1031 /// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and 1032 /// OutSize specifies part of the frame set up prior to the sequence. 1033 SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, 1034 const SDLoc &DL) { 1035 SDVTList VTs = getVTList(MVT::Other, MVT::Glue); 1036 SDValue Ops[] = { Chain, 1037 getIntPtrConstant(InSize, DL, true), 1038 getIntPtrConstant(OutSize, DL, true) }; 1039 return getNode(ISD::CALLSEQ_START, DL, VTs, Ops); 1040 } 1041 1042 /// Return a new CALLSEQ_END node, which always must have a 1043 /// glue result (to ensure it's not CSE'd). 1044 /// CALLSEQ_END does not have a useful SDLoc. 1045 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, 1046 SDValue InGlue, const SDLoc &DL) { 1047 SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue); 1048 SmallVector<SDValue, 4> Ops; 1049 Ops.push_back(Chain); 1050 Ops.push_back(Op1); 1051 Ops.push_back(Op2); 1052 if (InGlue.getNode()) 1053 Ops.push_back(InGlue); 1054 return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops); 1055 } 1056 1057 SDValue getCALLSEQ_END(SDValue Chain, uint64_t Size1, uint64_t Size2, 1058 SDValue Glue, const SDLoc &DL) { 1059 return getCALLSEQ_END( 1060 Chain, getIntPtrConstant(Size1, DL, /*isTarget=*/true), 1061 getIntPtrConstant(Size2, DL, /*isTarget=*/true), Glue, DL); 1062 } 1063 1064 /// Return true if the result of this operation is always undefined. 1065 bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops); 1066 1067 /// Return an UNDEF node. UNDEF does not have a useful SDLoc. 1068 SDValue getUNDEF(EVT VT) { 1069 return getNode(ISD::UNDEF, SDLoc(), VT); 1070 } 1071 1072 /// Return a node that represents the runtime scaling 'MulImm * RuntimeVL'. 1073 SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, 1074 bool ConstantFold = true); 1075 1076 SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, 1077 bool ConstantFold = true); 1078 1079 /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc. 1080 SDValue getGLOBAL_OFFSET_TABLE(EVT VT) { 1081 return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT); 1082 } 1083 1084 /// Gets or creates the specified node. 1085 /// 1086 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 1087 ArrayRef<SDUse> Ops); 1088 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 1089 ArrayRef<SDValue> Ops, const SDNodeFlags Flags); 1090 SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys, 1091 ArrayRef<SDValue> Ops); 1092 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 1093 ArrayRef<SDValue> Ops, const SDNodeFlags Flags); 1094 1095 // Use flags from current flag inserter. 1096 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 1097 ArrayRef<SDValue> Ops); 1098 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 1099 ArrayRef<SDValue> Ops); 1100 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand); 1101 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1102 SDValue N2); 1103 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1104 SDValue N2, SDValue N3); 1105 1106 // Specialize based on number of operands. 1107 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT); 1108 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand, 1109 const SDNodeFlags Flags); 1110 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1111 SDValue N2, const SDNodeFlags Flags); 1112 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1113 SDValue N2, SDValue N3, const SDNodeFlags Flags); 1114 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1115 SDValue N2, SDValue N3, SDValue N4); 1116 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, 1117 SDValue N2, SDValue N3, SDValue N4, SDValue N5); 1118 1119 // Specialize again based on number of operands for nodes with a VTList 1120 // rather than a single VT. 1121 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList); 1122 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N); 1123 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1124 SDValue N2); 1125 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1126 SDValue N2, SDValue N3); 1127 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1128 SDValue N2, SDValue N3, SDValue N4); 1129 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, 1130 SDValue N2, SDValue N3, SDValue N4, SDValue N5); 1131 1132 /// Compute a TokenFactor to force all the incoming stack arguments to be 1133 /// loaded from the stack. This is used in tail call lowering to protect 1134 /// stack arguments from being clobbered. 1135 SDValue getStackArgumentTokenFactor(SDValue Chain); 1136 1137 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, 1138 SDValue Size, Align Alignment, bool isVol, 1139 bool AlwaysInline, bool isTailCall, 1140 MachinePointerInfo DstPtrInfo, 1141 MachinePointerInfo SrcPtrInfo, 1142 const AAMDNodes &AAInfo = AAMDNodes(), 1143 AAResults *AA = nullptr); 1144 1145 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, 1146 SDValue Size, Align Alignment, bool isVol, bool isTailCall, 1147 MachinePointerInfo DstPtrInfo, 1148 MachinePointerInfo SrcPtrInfo, 1149 const AAMDNodes &AAInfo = AAMDNodes(), 1150 AAResults *AA = nullptr); 1151 1152 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, 1153 SDValue Size, Align Alignment, bool isVol, 1154 bool AlwaysInline, bool isTailCall, 1155 MachinePointerInfo DstPtrInfo, 1156 const AAMDNodes &AAInfo = AAMDNodes()); 1157 1158 SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, 1159 SDValue Src, SDValue Size, Type *SizeTy, 1160 unsigned ElemSz, bool isTailCall, 1161 MachinePointerInfo DstPtrInfo, 1162 MachinePointerInfo SrcPtrInfo); 1163 1164 SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, 1165 SDValue Src, SDValue Size, Type *SizeTy, 1166 unsigned ElemSz, bool isTailCall, 1167 MachinePointerInfo DstPtrInfo, 1168 MachinePointerInfo SrcPtrInfo); 1169 1170 SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, 1171 SDValue Value, SDValue Size, Type *SizeTy, 1172 unsigned ElemSz, bool isTailCall, 1173 MachinePointerInfo DstPtrInfo); 1174 1175 /// Helper function to make it easier to build SetCC's if you just have an 1176 /// ISD::CondCode instead of an SDValue. 1177 SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, 1178 ISD::CondCode Cond, SDValue Chain = SDValue(), 1179 bool IsSignaling = false) { 1180 assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() && 1181 "Vector/scalar operand type mismatch for setcc"); 1182 assert(LHS.getValueType().isVector() == VT.isVector() && 1183 "Vector/scalar result type mismatch for setcc"); 1184 assert(Cond != ISD::SETCC_INVALID && 1185 "Cannot create a setCC of an invalid node."); 1186 if (Chain) 1187 return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL, 1188 {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)}); 1189 return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond)); 1190 } 1191 1192 /// Helper function to make it easier to build VP_SETCCs if you just have an 1193 /// ISD::CondCode instead of an SDValue. 1194 SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, 1195 ISD::CondCode Cond, SDValue Mask, SDValue EVL) { 1196 assert(LHS.getValueType().isVector() && RHS.getValueType().isVector() && 1197 "Cannot compare scalars"); 1198 assert(Cond != ISD::SETCC_INVALID && 1199 "Cannot create a setCC of an invalid node."); 1200 return getNode(ISD::VP_SETCC, DL, VT, LHS, RHS, getCondCode(Cond), Mask, 1201 EVL); 1202 } 1203 1204 /// Helper function to make it easier to build Select's if you just have 1205 /// operands and don't want to check for vector. 1206 SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, 1207 SDValue RHS) { 1208 assert(LHS.getValueType() == VT && RHS.getValueType() == VT && 1209 "Cannot use select on differing types"); 1210 auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; 1211 return getNode(Opcode, DL, VT, Cond, LHS, RHS); 1212 } 1213 1214 /// Helper function to make it easier to build SelectCC's if you just have an 1215 /// ISD::CondCode instead of an SDValue. 1216 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, 1217 SDValue False, ISD::CondCode Cond) { 1218 return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True, 1219 False, getCondCode(Cond)); 1220 } 1221 1222 /// Try to simplify a select/vselect into 1 of its operands or a constant. 1223 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal); 1224 1225 /// Try to simplify a shift into 1 of its operands or a constant. 1226 SDValue simplifyShift(SDValue X, SDValue Y); 1227 1228 /// Try to simplify a floating-point binary operation into 1 of its operands 1229 /// or a constant. 1230 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, 1231 SDNodeFlags Flags); 1232 1233 /// VAArg produces a result and token chain, and takes a pointer 1234 /// and a source value as input. 1235 SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1236 SDValue SV, unsigned Align); 1237 1238 /// Gets a node for an atomic cmpxchg op. There are two 1239 /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a 1240 /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded, 1241 /// a success flag (initially i1), and a chain. 1242 SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, 1243 SDVTList VTs, SDValue Chain, SDValue Ptr, 1244 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO); 1245 1246 /// Gets a node for an atomic op, produces result (if relevant) 1247 /// and chain and takes 2 operands. 1248 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, 1249 SDValue Ptr, SDValue Val, MachineMemOperand *MMO); 1250 1251 /// Gets a node for an atomic op, produces result and chain and 1252 /// takes 1 operand. 1253 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT, 1254 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO); 1255 1256 /// Gets a node for an atomic op, produces result and chain and takes N 1257 /// operands. 1258 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 1259 SDVTList VTList, ArrayRef<SDValue> Ops, 1260 MachineMemOperand *MMO); 1261 1262 /// Creates a MemIntrinsicNode that may produce a 1263 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, 1264 /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not 1265 /// less than FIRST_TARGET_MEMORY_OPCODE. 1266 SDValue getMemIntrinsicNode( 1267 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, 1268 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, 1269 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | 1270 MachineMemOperand::MOStore, 1271 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()); 1272 1273 inline SDValue getMemIntrinsicNode( 1274 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, 1275 EVT MemVT, MachinePointerInfo PtrInfo, 1276 MaybeAlign Alignment = std::nullopt, 1277 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | 1278 MachineMemOperand::MOStore, 1279 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) { 1280 // Ensure that codegen never sees alignment 0 1281 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo, 1282 Alignment.value_or(getEVTAlign(MemVT)), Flags, 1283 Size, AAInfo); 1284 } 1285 1286 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, 1287 ArrayRef<SDValue> Ops, EVT MemVT, 1288 MachineMemOperand *MMO); 1289 1290 /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends 1291 /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between 1292 /// offsets `Offset` and `Offset + Size`. 1293 SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, 1294 int FrameIndex, int64_t Size, int64_t Offset = -1); 1295 1296 /// Creates a PseudoProbeSDNode with function GUID `Guid` and 1297 /// the index of the block `Index` it is probing, as well as the attributes 1298 /// `attr` of the probe. 1299 SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, 1300 uint64_t Index, uint32_t Attr); 1301 1302 /// Create a MERGE_VALUES node from the given operands. 1303 SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl); 1304 1305 /// Loads are not normal binary operators: their result type is not 1306 /// determined by their operands, and they produce a value AND a token chain. 1307 /// 1308 /// This function will set the MOLoad flag on MMOFlags, but you can set it if 1309 /// you want. The MOStore flag must not be set. 1310 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1311 MachinePointerInfo PtrInfo, 1312 MaybeAlign Alignment = MaybeAlign(), 1313 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1314 const AAMDNodes &AAInfo = AAMDNodes(), 1315 const MDNode *Ranges = nullptr); 1316 /// FIXME: Remove once transition to Align is over. 1317 LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead", 1318 "") 1319 inline SDValue 1320 getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1321 MachinePointerInfo PtrInfo, unsigned Alignment, 1322 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1323 const AAMDNodes &AAInfo = AAMDNodes(), 1324 const MDNode *Ranges = nullptr) { 1325 return getLoad(VT, dl, Chain, Ptr, PtrInfo, MaybeAlign(Alignment), MMOFlags, 1326 AAInfo, Ranges); 1327 } 1328 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1329 MachineMemOperand *MMO); 1330 SDValue 1331 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, 1332 SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, 1333 MaybeAlign Alignment = MaybeAlign(), 1334 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1335 const AAMDNodes &AAInfo = AAMDNodes()); 1336 SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, 1337 SDValue Chain, SDValue Ptr, EVT MemVT, 1338 MachineMemOperand *MMO); 1339 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, 1340 SDValue Offset, ISD::MemIndexedMode AM); 1341 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1342 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1343 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, 1344 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1345 const AAMDNodes &AAInfo = AAMDNodes(), 1346 const MDNode *Ranges = nullptr); 1347 inline SDValue getLoad( 1348 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, 1349 SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, 1350 EVT MemVT, MaybeAlign Alignment = MaybeAlign(), 1351 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1352 const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr) { 1353 // Ensures that codegen never sees a None Alignment. 1354 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT, 1355 Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, AAInfo, 1356 Ranges); 1357 } 1358 /// FIXME: Remove once transition to Align is over. 1359 LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead", 1360 "") 1361 inline SDValue 1362 getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1363 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1364 MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment, 1365 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1366 const AAMDNodes &AAInfo = AAMDNodes(), 1367 const MDNode *Ranges = nullptr) { 1368 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT, 1369 MaybeAlign(Alignment), MMOFlags, AAInfo, Ranges); 1370 } 1371 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1372 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1373 EVT MemVT, MachineMemOperand *MMO); 1374 1375 /// Helper function to build ISD::STORE nodes. 1376 /// 1377 /// This function will set the MOStore flag on MMOFlags, but you can set it if 1378 /// you want. The MOLoad and MOInvariant flags must not be set. 1379 1380 SDValue 1381 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1382 MachinePointerInfo PtrInfo, Align Alignment, 1383 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1384 const AAMDNodes &AAInfo = AAMDNodes()); 1385 inline SDValue 1386 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1387 MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(), 1388 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1389 const AAMDNodes &AAInfo = AAMDNodes()) { 1390 return getStore(Chain, dl, Val, Ptr, PtrInfo, 1391 Alignment.value_or(getEVTAlign(Val.getValueType())), 1392 MMOFlags, AAInfo); 1393 } 1394 /// FIXME: Remove once transition to Align is over. 1395 LLVM_DEPRECATED("Use the version that takes a MaybeAlign instead", "") 1396 inline SDValue 1397 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1398 MachinePointerInfo PtrInfo, unsigned Alignment, 1399 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1400 const AAMDNodes &AAInfo = AAMDNodes()) { 1401 return getStore(Chain, dl, Val, Ptr, PtrInfo, MaybeAlign(Alignment), 1402 MMOFlags, AAInfo); 1403 } 1404 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1405 MachineMemOperand *MMO); 1406 SDValue 1407 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1408 MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, 1409 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1410 const AAMDNodes &AAInfo = AAMDNodes()); 1411 inline SDValue 1412 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1413 MachinePointerInfo PtrInfo, EVT SVT, 1414 MaybeAlign Alignment = MaybeAlign(), 1415 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1416 const AAMDNodes &AAInfo = AAMDNodes()) { 1417 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT, 1418 Alignment.value_or(getEVTAlign(SVT)), MMOFlags, 1419 AAInfo); 1420 } 1421 /// FIXME: Remove once transition to Align is over. 1422 LLVM_DEPRECATED("Use the version that takes a MaybeAlign instead", "") 1423 inline SDValue 1424 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1425 MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment, 1426 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1427 const AAMDNodes &AAInfo = AAMDNodes()) { 1428 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT, 1429 MaybeAlign(Alignment), MMOFlags, AAInfo); 1430 } 1431 SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, 1432 SDValue Ptr, EVT SVT, MachineMemOperand *MMO); 1433 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, 1434 SDValue Offset, ISD::MemIndexedMode AM); 1435 1436 SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1437 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1438 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, 1439 EVT MemVT, Align Alignment, 1440 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, 1441 const MDNode *Ranges = nullptr, bool IsExpanding = false); 1442 inline SDValue 1443 getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1444 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1445 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, 1446 MaybeAlign Alignment = MaybeAlign(), 1447 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1448 const AAMDNodes &AAInfo = AAMDNodes(), 1449 const MDNode *Ranges = nullptr, bool IsExpanding = false) { 1450 // Ensures that codegen never sees a None Alignment. 1451 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, 1452 PtrInfo, MemVT, Alignment.value_or(getEVTAlign(MemVT)), 1453 MMOFlags, AAInfo, Ranges, IsExpanding); 1454 } 1455 SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, 1456 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, 1457 SDValue Mask, SDValue EVL, EVT MemVT, 1458 MachineMemOperand *MMO, bool IsExpanding = false); 1459 SDValue getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1460 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, 1461 MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, 1462 const AAMDNodes &AAInfo, const MDNode *Ranges = nullptr, 1463 bool IsExpanding = false); 1464 SDValue getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, 1465 SDValue Mask, SDValue EVL, MachineMemOperand *MMO, 1466 bool IsExpanding = false); 1467 SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, 1468 SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, 1469 MachinePointerInfo PtrInfo, EVT MemVT, 1470 MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, 1471 const AAMDNodes &AAInfo, bool IsExpanding = false); 1472 SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, 1473 SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, 1474 EVT MemVT, MachineMemOperand *MMO, 1475 bool IsExpanding = false); 1476 SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base, 1477 SDValue Offset, ISD::MemIndexedMode AM); 1478 SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, 1479 SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, 1480 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1481 bool IsTruncating = false, bool IsCompressing = false); 1482 SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, 1483 SDValue Ptr, SDValue Mask, SDValue EVL, 1484 MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, 1485 MachineMemOperand::Flags MMOFlags, 1486 const AAMDNodes &AAInfo, bool IsCompressing = false); 1487 SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, 1488 SDValue Ptr, SDValue Mask, SDValue EVL, EVT SVT, 1489 MachineMemOperand *MMO, bool IsCompressing = false); 1490 SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base, 1491 SDValue Offset, ISD::MemIndexedMode AM); 1492 1493 SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 1494 EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, 1495 SDValue Offset, SDValue Stride, SDValue Mask, 1496 SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, 1497 Align Alignment, MachineMemOperand::Flags MMOFlags, 1498 const AAMDNodes &AAInfo, 1499 const MDNode *Ranges = nullptr, 1500 bool IsExpanding = false); 1501 inline SDValue getStridedLoadVP( 1502 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, 1503 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, 1504 SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, 1505 MaybeAlign Alignment = MaybeAlign(), 1506 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, 1507 const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr, 1508 bool IsExpanding = false) { 1509 // Ensures that codegen never sees a None Alignment. 1510 return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, 1511 Mask, EVL, PtrInfo, MemVT, 1512 Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, 1513 AAInfo, Ranges, IsExpanding); 1514 } 1515 SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 1516 EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, 1517 SDValue Offset, SDValue Stride, SDValue Mask, 1518 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, 1519 bool IsExpanding = false); 1520 SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, 1521 SDValue Stride, SDValue Mask, SDValue EVL, 1522 MachinePointerInfo PtrInfo, MaybeAlign Alignment, 1523 MachineMemOperand::Flags MMOFlags, 1524 const AAMDNodes &AAInfo, 1525 const MDNode *Ranges = nullptr, 1526 bool IsExpanding = false); 1527 SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, 1528 SDValue Stride, SDValue Mask, SDValue EVL, 1529 MachineMemOperand *MMO, bool IsExpanding = false); 1530 SDValue 1531 getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, 1532 SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, 1533 SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, 1534 MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, 1535 const AAMDNodes &AAInfo, bool IsExpanding = false); 1536 SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, 1537 SDValue Chain, SDValue Ptr, SDValue Stride, 1538 SDValue Mask, SDValue EVL, EVT MemVT, 1539 MachineMemOperand *MMO, bool IsExpanding = false); 1540 SDValue getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL, 1541 SDValue Base, SDValue Offset, 1542 ISD::MemIndexedMode AM); 1543 SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, 1544 SDValue Ptr, SDValue Offset, SDValue Stride, 1545 SDValue Mask, SDValue EVL, EVT MemVT, 1546 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1547 bool IsTruncating = false, 1548 bool IsCompressing = false); 1549 SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, 1550 SDValue Ptr, SDValue Stride, SDValue Mask, 1551 SDValue EVL, MachinePointerInfo PtrInfo, 1552 EVT SVT, Align Alignment, 1553 MachineMemOperand::Flags MMOFlags, 1554 const AAMDNodes &AAInfo, 1555 bool IsCompressing = false); 1556 SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, 1557 SDValue Ptr, SDValue Stride, SDValue Mask, 1558 SDValue EVL, EVT SVT, MachineMemOperand *MMO, 1559 bool IsCompressing = false); 1560 SDValue getIndexedStridedStoreVP(SDValue OrigStore, const SDLoc &DL, 1561 SDValue Base, SDValue Offset, 1562 ISD::MemIndexedMode AM); 1563 1564 SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, 1565 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1566 ISD::MemIndexType IndexType); 1567 SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, 1568 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1569 ISD::MemIndexType IndexType); 1570 1571 SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, 1572 SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, 1573 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1574 ISD::LoadExtType, bool IsExpanding = false); 1575 SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, 1576 SDValue Offset, ISD::MemIndexedMode AM); 1577 SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, 1578 SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, 1579 MachineMemOperand *MMO, ISD::MemIndexedMode AM, 1580 bool IsTruncating = false, bool IsCompressing = false); 1581 SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, 1582 SDValue Base, SDValue Offset, 1583 ISD::MemIndexedMode AM); 1584 SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, 1585 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1586 ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy); 1587 SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, 1588 ArrayRef<SDValue> Ops, MachineMemOperand *MMO, 1589 ISD::MemIndexType IndexType, 1590 bool IsTruncating = false); 1591 1592 SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, 1593 MachineMemOperand *MMO); 1594 SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, 1595 MachineMemOperand *MMO); 1596 1597 /// Construct a node to track a Value* through the backend. 1598 SDValue getSrcValue(const Value *v); 1599 1600 /// Return an MDNodeSDNode which holds an MDNode. 1601 SDValue getMDNode(const MDNode *MD); 1602 1603 /// Return a bitcast using the SDLoc of the value operand, and casting to the 1604 /// provided type. Use getNode to set a custom SDLoc. 1605 SDValue getBitcast(EVT VT, SDValue V); 1606 1607 /// Return an AddrSpaceCastSDNode. 1608 SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, 1609 unsigned DestAS); 1610 1611 /// Return a freeze using the SDLoc of the value operand. 1612 SDValue getFreeze(SDValue V); 1613 1614 /// Return an AssertAlignSDNode. 1615 SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A); 1616 1617 /// Swap N1 and N2 if Opcode is a commutative binary opcode 1618 /// and the canonical form expects the opposite order. 1619 void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, 1620 SDValue &N2) const; 1621 1622 /// Return the specified value casted to 1623 /// the target's desired shift amount type. 1624 SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op); 1625 1626 /// Expand the specified \c ISD::VAARG node as the Legalize pass would. 1627 SDValue expandVAArg(SDNode *Node); 1628 1629 /// Expand the specified \c ISD::VACOPY node as the Legalize pass would. 1630 SDValue expandVACopy(SDNode *Node); 1631 1632 /// Returs an GlobalAddress of the function from the current module with 1633 /// name matching the given ExternalSymbol. Additionally can provide the 1634 /// matched function. 1635 /// Panics the function doesn't exists. 1636 SDValue getSymbolFunctionGlobalAddress(SDValue Op, 1637 Function **TargetFunction = nullptr); 1638 1639 /// *Mutate* the specified node in-place to have the 1640 /// specified operands. If the resultant node already exists in the DAG, 1641 /// this does not modify the specified node, instead it returns the node that 1642 /// already exists. If the resultant node does not exist in the DAG, the 1643 /// input node is returned. As a degenerate case, if you specify the same 1644 /// input operands as the node already has, the input node is returned. 1645 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op); 1646 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2); 1647 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 1648 SDValue Op3); 1649 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 1650 SDValue Op3, SDValue Op4); 1651 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 1652 SDValue Op3, SDValue Op4, SDValue Op5); 1653 SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops); 1654 1655 /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k 1656 /// values or more, move values into new TokenFactors in 64k-1 blocks, until 1657 /// the final TokenFactor has less than 64k operands. 1658 SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals); 1659 1660 /// *Mutate* the specified machine node's memory references to the provided 1661 /// list. 1662 void setNodeMemRefs(MachineSDNode *N, 1663 ArrayRef<MachineMemOperand *> NewMemRefs); 1664 1665 // Calculate divergence of node \p N based on its operands. 1666 bool calculateDivergence(SDNode *N); 1667 1668 // Propagates the change in divergence to users 1669 void updateDivergence(SDNode * N); 1670 1671 /// These are used for target selectors to *mutate* the 1672 /// specified node to have the specified return type, Target opcode, and 1673 /// operands. Note that target opcodes are stored as 1674 /// ~TargetOpcode in the node opcode field. The resultant node is returned. 1675 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT); 1676 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1); 1677 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, 1678 SDValue Op1, SDValue Op2); 1679 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, 1680 SDValue Op1, SDValue Op2, SDValue Op3); 1681 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, 1682 ArrayRef<SDValue> Ops); 1683 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, EVT VT2); 1684 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, 1685 EVT VT2, ArrayRef<SDValue> Ops); 1686 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, 1687 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops); 1688 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, 1689 EVT VT2, SDValue Op1, SDValue Op2); 1690 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs, 1691 ArrayRef<SDValue> Ops); 1692 1693 /// This *mutates* the specified node to have the specified 1694 /// return type, opcode, and operands. 1695 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, 1696 ArrayRef<SDValue> Ops); 1697 1698 /// Mutate the specified strict FP node to its non-strict equivalent, 1699 /// unlinking the node from its chain and dropping the metadata arguments. 1700 /// The node must be a strict FP node. 1701 SDNode *mutateStrictFPToFP(SDNode *Node); 1702 1703 /// These are used for target selectors to create a new node 1704 /// with specified return type(s), MachineInstr opcode, and operands. 1705 /// 1706 /// Note that getMachineNode returns the resultant node. If there is already 1707 /// a node of the specified opcode and operands, it returns that node instead 1708 /// of the current one. 1709 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT); 1710 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1711 SDValue Op1); 1712 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1713 SDValue Op1, SDValue Op2); 1714 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1715 SDValue Op1, SDValue Op2, SDValue Op3); 1716 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, 1717 ArrayRef<SDValue> Ops); 1718 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1719 EVT VT2, SDValue Op1, SDValue Op2); 1720 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1721 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3); 1722 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1723 EVT VT2, ArrayRef<SDValue> Ops); 1724 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1725 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2); 1726 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1727 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2, 1728 SDValue Op3); 1729 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, 1730 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops); 1731 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, 1732 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops); 1733 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, SDVTList VTs, 1734 ArrayRef<SDValue> Ops); 1735 1736 /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes. 1737 SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, 1738 SDValue Operand); 1739 1740 /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes. 1741 SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, 1742 SDValue Operand, SDValue Subreg); 1743 1744 /// Get the specified node if it's already available, or else return NULL. 1745 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, 1746 ArrayRef<SDValue> Ops, const SDNodeFlags Flags); 1747 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, 1748 ArrayRef<SDValue> Ops); 1749 1750 /// Check if a node exists without modifying its flags. 1751 bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops); 1752 1753 /// Creates a SDDbgValue node. 1754 SDDbgValue *getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, 1755 unsigned R, bool IsIndirect, const DebugLoc &DL, 1756 unsigned O); 1757 1758 /// Creates a constant SDDbgValue node. 1759 SDDbgValue *getConstantDbgValue(DIVariable *Var, DIExpression *Expr, 1760 const Value *C, const DebugLoc &DL, 1761 unsigned O); 1762 1763 /// Creates a FrameIndex SDDbgValue node. 1764 SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, 1765 unsigned FI, bool IsIndirect, 1766 const DebugLoc &DL, unsigned O); 1767 1768 /// Creates a FrameIndex SDDbgValue node. 1769 SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, 1770 unsigned FI, 1771 ArrayRef<SDNode *> Dependencies, 1772 bool IsIndirect, const DebugLoc &DL, 1773 unsigned O); 1774 1775 /// Creates a VReg SDDbgValue node. 1776 SDDbgValue *getVRegDbgValue(DIVariable *Var, DIExpression *Expr, 1777 unsigned VReg, bool IsIndirect, 1778 const DebugLoc &DL, unsigned O); 1779 1780 /// Creates a SDDbgValue node from a list of locations. 1781 SDDbgValue *getDbgValueList(DIVariable *Var, DIExpression *Expr, 1782 ArrayRef<SDDbgOperand> Locs, 1783 ArrayRef<SDNode *> Dependencies, bool IsIndirect, 1784 const DebugLoc &DL, unsigned O, bool IsVariadic); 1785 1786 /// Creates a SDDbgLabel node. 1787 SDDbgLabel *getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O); 1788 1789 /// Transfer debug values from one node to another, while optionally 1790 /// generating fragment expressions for split-up values. If \p InvalidateDbg 1791 /// is set, debug values are invalidated after they are transferred. 1792 void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0, 1793 unsigned SizeInBits = 0, bool InvalidateDbg = true); 1794 1795 /// Remove the specified node from the system. If any of its 1796 /// operands then becomes dead, remove them as well. Inform UpdateListener 1797 /// for each node deleted. 1798 void RemoveDeadNode(SDNode *N); 1799 1800 /// This method deletes the unreachable nodes in the 1801 /// given list, and any nodes that become unreachable as a result. 1802 void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes); 1803 1804 /// Modify anything using 'From' to use 'To' instead. 1805 /// This can cause recursive merging of nodes in the DAG. Use the first 1806 /// version if 'From' is known to have a single result, use the second 1807 /// if you have two nodes with identical results (or if 'To' has a superset 1808 /// of the results of 'From'), use the third otherwise. 1809 /// 1810 /// These methods all take an optional UpdateListener, which (if not null) is 1811 /// informed about nodes that are deleted and modified due to recursive 1812 /// changes in the dag. 1813 /// 1814 /// These functions only replace all existing uses. It's possible that as 1815 /// these replacements are being performed, CSE may cause the From node 1816 /// to be given new uses. These new uses of From are left in place, and 1817 /// not automatically transferred to To. 1818 /// 1819 void ReplaceAllUsesWith(SDValue From, SDValue To); 1820 void ReplaceAllUsesWith(SDNode *From, SDNode *To); 1821 void ReplaceAllUsesWith(SDNode *From, const SDValue *To); 1822 1823 /// Replace any uses of From with To, leaving 1824 /// uses of other values produced by From.getNode() alone. 1825 void ReplaceAllUsesOfValueWith(SDValue From, SDValue To); 1826 1827 /// Like ReplaceAllUsesOfValueWith, but for multiple values at once. 1828 /// This correctly handles the case where 1829 /// there is an overlap between the From values and the To values. 1830 void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, 1831 unsigned Num); 1832 1833 /// If an existing load has uses of its chain, create a token factor node with 1834 /// that chain and the new memory node's chain and update users of the old 1835 /// chain to the token factor. This ensures that the new memory node will have 1836 /// the same relative memory dependency position as the old load. Returns the 1837 /// new merged load chain. 1838 SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain); 1839 1840 /// If an existing load has uses of its chain, create a token factor node with 1841 /// that chain and the new memory node's chain and update users of the old 1842 /// chain to the token factor. This ensures that the new memory node will have 1843 /// the same relative memory dependency position as the old load. Returns the 1844 /// new merged load chain. 1845 SDValue makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, SDValue NewMemOp); 1846 1847 /// Topological-sort the AllNodes list and a 1848 /// assign a unique node id for each node in the DAG based on their 1849 /// topological order. Returns the number of nodes. 1850 unsigned AssignTopologicalOrder(); 1851 1852 /// Move node N in the AllNodes list to be immediately 1853 /// before the given iterator Position. This may be used to update the 1854 /// topological ordering when the list of nodes is modified. 1855 void RepositionNode(allnodes_iterator Position, SDNode *N) { 1856 AllNodes.insert(Position, AllNodes.remove(N)); 1857 } 1858 1859 /// Returns an APFloat semantics tag appropriate for the given type. If VT is 1860 /// a vector type, the element semantics are returned. 1861 static const fltSemantics &EVTToAPFloatSemantics(EVT VT) { 1862 switch (VT.getScalarType().getSimpleVT().SimpleTy) { 1863 default: llvm_unreachable("Unknown FP format"); 1864 case MVT::f16: return APFloat::IEEEhalf(); 1865 case MVT::bf16: return APFloat::BFloat(); 1866 case MVT::f32: return APFloat::IEEEsingle(); 1867 case MVT::f64: return APFloat::IEEEdouble(); 1868 case MVT::f80: return APFloat::x87DoubleExtended(); 1869 case MVT::f128: return APFloat::IEEEquad(); 1870 case MVT::ppcf128: return APFloat::PPCDoubleDouble(); 1871 } 1872 } 1873 1874 /// Add a dbg_value SDNode. If SD is non-null that means the 1875 /// value is produced by SD. 1876 void AddDbgValue(SDDbgValue *DB, bool isParameter); 1877 1878 /// Add a dbg_label SDNode. 1879 void AddDbgLabel(SDDbgLabel *DB); 1880 1881 /// Get the debug values which reference the given SDNode. 1882 ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) const { 1883 return DbgInfo->getSDDbgValues(SD); 1884 } 1885 1886 public: 1887 /// Return true if there are any SDDbgValue nodes associated 1888 /// with this SelectionDAG. 1889 bool hasDebugValues() const { return !DbgInfo->empty(); } 1890 1891 SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); } 1892 SDDbgInfo::DbgIterator DbgEnd() const { return DbgInfo->DbgEnd(); } 1893 1894 SDDbgInfo::DbgIterator ByvalParmDbgBegin() const { 1895 return DbgInfo->ByvalParmDbgBegin(); 1896 } 1897 SDDbgInfo::DbgIterator ByvalParmDbgEnd() const { 1898 return DbgInfo->ByvalParmDbgEnd(); 1899 } 1900 1901 SDDbgInfo::DbgLabelIterator DbgLabelBegin() const { 1902 return DbgInfo->DbgLabelBegin(); 1903 } 1904 SDDbgInfo::DbgLabelIterator DbgLabelEnd() const { 1905 return DbgInfo->DbgLabelEnd(); 1906 } 1907 1908 /// To be invoked on an SDNode that is slated to be erased. This 1909 /// function mirrors \c llvm::salvageDebugInfo. 1910 void salvageDebugInfo(SDNode &N); 1911 1912 void dump() const; 1913 1914 /// In most cases this function returns the ABI alignment for a given type, 1915 /// except for illegal vector types where the alignment exceeds that of the 1916 /// stack. In such cases we attempt to break the vector down to a legal type 1917 /// and return the ABI alignment for that instead. 1918 Align getReducedAlign(EVT VT, bool UseABI); 1919 1920 /// Create a stack temporary based on the size in bytes and the alignment 1921 SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment); 1922 1923 /// Create a stack temporary, suitable for holding the specified value type. 1924 /// If minAlign is specified, the slot size will have at least that alignment. 1925 SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1); 1926 1927 /// Create a stack temporary suitable for holding either of the specified 1928 /// value types. 1929 SDValue CreateStackTemporary(EVT VT1, EVT VT2); 1930 1931 SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, 1932 const GlobalAddressSDNode *GA, 1933 const SDNode *N2); 1934 1935 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, 1936 ArrayRef<SDValue> Ops); 1937 1938 /// Fold floating-point operations with 2 operands when both operands are 1939 /// constants and/or undefined. 1940 SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, 1941 SDValue N1, SDValue N2); 1942 1943 /// Constant fold a setcc to true or false. 1944 SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, 1945 const SDLoc &dl); 1946 1947 /// Return true if the sign bit of Op is known to be zero. 1948 /// We use this predicate to simplify operations downstream. 1949 bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const; 1950 1951 /// Return true if 'Op & Mask' is known to be zero. We 1952 /// use this predicate to simplify operations downstream. Op and Mask are 1953 /// known to be the same type. 1954 bool MaskedValueIsZero(SDValue Op, const APInt &Mask, 1955 unsigned Depth = 0) const; 1956 1957 /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We 1958 /// use this predicate to simplify operations downstream. Op and Mask are 1959 /// known to be the same type. 1960 bool MaskedValueIsZero(SDValue Op, const APInt &Mask, 1961 const APInt &DemandedElts, unsigned Depth = 0) const; 1962 1963 /// Return true if 'Op' is known to be zero in DemandedElts. We 1964 /// use this predicate to simplify operations downstream. 1965 bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, 1966 unsigned Depth = 0) const; 1967 1968 /// Return true if '(Op & Mask) == Mask'. 1969 /// Op and Mask are known to be the same type. 1970 bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, 1971 unsigned Depth = 0) const; 1972 1973 /// For each demanded element of a vector, see if it is known to be zero. 1974 APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, 1975 unsigned Depth = 0) const; 1976 1977 /// Determine which bits of Op are known to be either zero or one and return 1978 /// them in Known. For vectors, the known bits are those that are shared by 1979 /// every vector element. 1980 /// Targets can implement the computeKnownBitsForTargetNode method in the 1981 /// TargetLowering class to allow target nodes to be understood. 1982 KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const; 1983 1984 /// Determine which bits of Op are known to be either zero or one and return 1985 /// them in Known. The DemandedElts argument allows us to only collect the 1986 /// known bits that are shared by the requested vector elements. 1987 /// Targets can implement the computeKnownBitsForTargetNode method in the 1988 /// TargetLowering class to allow target nodes to be understood. 1989 KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts, 1990 unsigned Depth = 0) const; 1991 1992 /// Used to represent the possible overflow behavior of an operation. 1993 /// Never: the operation cannot overflow. 1994 /// Always: the operation will always overflow. 1995 /// Sometime: the operation may or may not overflow. 1996 enum OverflowKind { 1997 OFK_Never, 1998 OFK_Sometime, 1999 OFK_Always, 2000 }; 2001 2002 /// Determine if the result of the signed addition of 2 nodes can overflow. 2003 OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const; 2004 2005 /// Determine if the result of the unsigned addition of 2 nodes can overflow. 2006 OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const; 2007 2008 /// Determine if the result of the addition of 2 nodes can overflow. 2009 OverflowKind computeOverflowForAdd(bool IsSigned, SDValue N0, 2010 SDValue N1) const { 2011 return IsSigned ? computeOverflowForSignedAdd(N0, N1) 2012 : computeOverflowForUnsignedAdd(N0, N1); 2013 } 2014 2015 /// Determine if the result of the signed sub of 2 nodes can overflow. 2016 OverflowKind computeOverflowForSignedSub(SDValue N0, SDValue N1) const; 2017 2018 /// Determine if the result of the unsigned sub of 2 nodes can overflow. 2019 OverflowKind computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const; 2020 2021 /// Determine if the result of the sub of 2 nodes can overflow. 2022 OverflowKind computeOverflowForSub(bool IsSigned, SDValue N0, 2023 SDValue N1) const { 2024 return IsSigned ? computeOverflowForSignedSub(N0, N1) 2025 : computeOverflowForUnsignedSub(N0, N1); 2026 } 2027 2028 /// Test if the given value is known to have exactly one bit set. This differs 2029 /// from computeKnownBits in that it doesn't necessarily determine which bit 2030 /// is set. 2031 bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth = 0) const; 2032 2033 /// Return the number of times the sign bit of the register is replicated into 2034 /// the other bits. We know that at least 1 bit is always equal to the sign 2035 /// bit (itself), but other cases can give us information. For example, 2036 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal 2037 /// to each other, so we return 3. Targets can implement the 2038 /// ComputeNumSignBitsForTarget method in the TargetLowering class to allow 2039 /// target nodes to be understood. 2040 unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const; 2041 2042 /// Return the number of times the sign bit of the register is replicated into 2043 /// the other bits. We know that at least 1 bit is always equal to the sign 2044 /// bit (itself), but other cases can give us information. For example, 2045 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal 2046 /// to each other, so we return 3. The DemandedElts argument allows 2047 /// us to only collect the minimum sign bits of the requested vector elements. 2048 /// Targets can implement the ComputeNumSignBitsForTarget method in the 2049 /// TargetLowering class to allow target nodes to be understood. 2050 unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, 2051 unsigned Depth = 0) const; 2052 2053 /// Get the upper bound on bit size for this Value \p Op as a signed integer. 2054 /// i.e. x == sext(trunc(x to MaxSignedBits) to bitwidth(x)). 2055 /// Similar to the APInt::getSignificantBits function. 2056 /// Helper wrapper to ComputeNumSignBits. 2057 unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth = 0) const; 2058 2059 /// Get the upper bound on bit size for this Value \p Op as a signed integer. 2060 /// i.e. x == sext(trunc(x to MaxSignedBits) to bitwidth(x)). 2061 /// Similar to the APInt::getSignificantBits function. 2062 /// Helper wrapper to ComputeNumSignBits. 2063 unsigned ComputeMaxSignificantBits(SDValue Op, const APInt &DemandedElts, 2064 unsigned Depth = 0) const; 2065 2066 /// Return true if this function can prove that \p Op is never poison 2067 /// and, if \p PoisonOnly is false, does not have undef bits. 2068 bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly = false, 2069 unsigned Depth = 0) const; 2070 2071 /// Return true if this function can prove that \p Op is never poison 2072 /// and, if \p PoisonOnly is false, does not have undef bits. The DemandedElts 2073 /// argument limits the check to the requested vector elements. 2074 bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, const APInt &DemandedElts, 2075 bool PoisonOnly = false, 2076 unsigned Depth = 0) const; 2077 2078 /// Return true if this function can prove that \p Op is never poison. 2079 bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth = 0) const { 2080 return isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ true, Depth); 2081 } 2082 2083 /// Return true if this function can prove that \p Op is never poison. The 2084 /// DemandedElts argument limits the check to the requested vector elements. 2085 bool isGuaranteedNotToBePoison(SDValue Op, const APInt &DemandedElts, 2086 unsigned Depth = 0) const { 2087 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, 2088 /*PoisonOnly*/ true, Depth); 2089 } 2090 2091 /// Return true if Op can create undef or poison from non-undef & non-poison 2092 /// operands. The DemandedElts argument limits the check to the requested 2093 /// vector elements. 2094 /// 2095 /// \p ConsiderFlags controls whether poison producing flags on the 2096 /// instruction are considered. This can be used to see if the instruction 2097 /// could still introduce undef or poison even without poison generating flags 2098 /// which might be on the instruction. (i.e. could the result of 2099 /// Op->dropPoisonGeneratingFlags() still create poison or undef) 2100 bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, 2101 bool PoisonOnly = false, 2102 bool ConsiderFlags = true, 2103 unsigned Depth = 0) const; 2104 2105 /// Return true if Op can create undef or poison from non-undef & non-poison 2106 /// operands. 2107 /// 2108 /// \p ConsiderFlags controls whether poison producing flags on the 2109 /// instruction are considered. This can be used to see if the instruction 2110 /// could still introduce undef or poison even without poison generating flags 2111 /// which might be on the instruction. (i.e. could the result of 2112 /// Op->dropPoisonGeneratingFlags() still create poison or undef) 2113 bool canCreateUndefOrPoison(SDValue Op, bool PoisonOnly = false, 2114 bool ConsiderFlags = true, 2115 unsigned Depth = 0) const; 2116 2117 /// Return true if the specified operand is an ISD::ADD with a ConstantSDNode 2118 /// on the right-hand side, or if it is an ISD::OR with a ConstantSDNode that 2119 /// is guaranteed to have the same semantics as an ADD. This handles the 2120 /// equivalence: 2121 /// X|Cst == X+Cst iff X&Cst = 0. 2122 bool isBaseWithConstantOffset(SDValue Op) const; 2123 2124 /// Test whether the given SDValue (or all elements of it, if it is a 2125 /// vector) is known to never be NaN. If \p SNaN is true, returns if \p Op is 2126 /// known to never be a signaling NaN (it may still be a qNaN). 2127 bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const; 2128 2129 /// \returns true if \p Op is known to never be a signaling NaN. 2130 bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const { 2131 return isKnownNeverNaN(Op, true, Depth); 2132 } 2133 2134 /// Test whether the given floating point SDValue is known to never be 2135 /// positive or negative zero. 2136 bool isKnownNeverZeroFloat(SDValue Op) const; 2137 2138 /// Test whether the given SDValue is known to contain non-zero value(s). 2139 bool isKnownNeverZero(SDValue Op, unsigned Depth = 0) const; 2140 2141 /// Test whether two SDValues are known to compare equal. This 2142 /// is true if they are the same value, or if one is negative zero and the 2143 /// other positive zero. 2144 bool isEqualTo(SDValue A, SDValue B) const; 2145 2146 /// Return true if A and B have no common bits set. As an example, this can 2147 /// allow an 'add' to be transformed into an 'or'. 2148 bool haveNoCommonBitsSet(SDValue A, SDValue B) const; 2149 2150 /// Test whether \p V has a splatted value for all the demanded elements. 2151 /// 2152 /// On success \p UndefElts will indicate the elements that have UNDEF 2153 /// values instead of the splat value, this is only guaranteed to be correct 2154 /// for \p DemandedElts. 2155 /// 2156 /// NOTE: The function will return true for a demanded splat of UNDEF values. 2157 bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, 2158 unsigned Depth = 0) const; 2159 2160 /// Test whether \p V has a splatted value. 2161 bool isSplatValue(SDValue V, bool AllowUndefs = false) const; 2162 2163 /// If V is a splatted value, return the source vector and its splat index. 2164 SDValue getSplatSourceVector(SDValue V, int &SplatIndex); 2165 2166 /// If V is a splat vector, return its scalar source operand by extracting 2167 /// that element from the source vector. If LegalTypes is true, this method 2168 /// may only return a legally-typed splat value. If it cannot legalize the 2169 /// splatted value it will return SDValue(). 2170 SDValue getSplatValue(SDValue V, bool LegalTypes = false); 2171 2172 /// If a SHL/SRA/SRL node \p V has a constant or splat constant shift amount 2173 /// that is less than the element bit-width of the shift node, return it. 2174 const APInt *getValidShiftAmountConstant(SDValue V, 2175 const APInt &DemandedElts) const; 2176 2177 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less 2178 /// than the element bit-width of the shift node, return the minimum value. 2179 const APInt * 2180 getValidMinimumShiftAmountConstant(SDValue V, 2181 const APInt &DemandedElts) const; 2182 2183 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less 2184 /// than the element bit-width of the shift node, return the maximum value. 2185 const APInt * 2186 getValidMaximumShiftAmountConstant(SDValue V, 2187 const APInt &DemandedElts) const; 2188 2189 /// Match a binop + shuffle pyramid that represents a horizontal reduction 2190 /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p 2191 /// Extract. The reduction must use one of the opcodes listed in /p 2192 /// CandidateBinOps and on success /p BinOp will contain the matching opcode. 2193 /// Returns the vector that is being reduced on, or SDValue() if a reduction 2194 /// was not matched. If \p AllowPartials is set then in the case of a 2195 /// reduction pattern that only matches the first few stages, the extracted 2196 /// subvector of the start of the reduction is returned. 2197 SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, 2198 ArrayRef<ISD::NodeType> CandidateBinOps, 2199 bool AllowPartials = false); 2200 2201 /// Utility function used by legalize and lowering to 2202 /// "unroll" a vector operation by splitting out the scalars and operating 2203 /// on each element individually. If the ResNE is 0, fully unroll the vector 2204 /// op. If ResNE is less than the width of the vector op, unroll up to ResNE. 2205 /// If the ResNE is greater than the width of the vector op, unroll the 2206 /// vector op and fill the end of the resulting vector with UNDEFS. 2207 SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0); 2208 2209 /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes. 2210 /// This is a separate function because those opcodes have two results. 2211 std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N, 2212 unsigned ResNE = 0); 2213 2214 /// Return true if loads are next to each other and can be 2215 /// merged. Check that both are nonvolatile and if LD is loading 2216 /// 'Bytes' bytes from a location that is 'Dist' units away from the 2217 /// location that the 'Base' load is loading from. 2218 bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, 2219 unsigned Bytes, int Dist) const; 2220 2221 /// Infer alignment of a load / store address. Return std::nullopt if it 2222 /// cannot be inferred. 2223 MaybeAlign InferPtrAlign(SDValue Ptr) const; 2224 2225 /// Split the scalar node with EXTRACT_ELEMENT using the provided VTs and 2226 /// return the low/high part. 2227 std::pair<SDValue, SDValue> SplitScalar(const SDValue &N, const SDLoc &DL, 2228 const EVT &LoVT, const EVT &HiVT); 2229 2230 /// Compute the VTs needed for the low/hi parts of a type 2231 /// which is split (or expanded) into two not necessarily identical pieces. 2232 std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const; 2233 2234 /// Compute the VTs needed for the low/hi parts of a type, dependent on an 2235 /// enveloping VT that has been split into two identical pieces. Sets the 2236 /// HisIsEmpty flag when hi type has zero storage size. 2237 std::pair<EVT, EVT> GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, 2238 bool *HiIsEmpty) const; 2239 2240 /// Split the vector with EXTRACT_SUBVECTOR using the provides 2241 /// VTs and return the low/high part. 2242 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL, 2243 const EVT &LoVT, const EVT &HiVT); 2244 2245 /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part. 2246 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) { 2247 EVT LoVT, HiVT; 2248 std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType()); 2249 return SplitVector(N, DL, LoVT, HiVT); 2250 } 2251 2252 /// Split the explicit vector length parameter of a VP operation. 2253 std::pair<SDValue, SDValue> SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL); 2254 2255 /// Split the node's operand with EXTRACT_SUBVECTOR and 2256 /// return the low/high part. 2257 std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo) 2258 { 2259 return SplitVector(N->getOperand(OpNo), SDLoc(N)); 2260 } 2261 2262 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR. 2263 SDValue WidenVector(const SDValue &N, const SDLoc &DL); 2264 2265 /// Append the extracted elements from Start to Count out of the vector Op in 2266 /// Args. If Count is 0, all of the elements will be extracted. The extracted 2267 /// elements will have type EVT if it is provided, and otherwise their type 2268 /// will be Op's element type. 2269 void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args, 2270 unsigned Start = 0, unsigned Count = 0, 2271 EVT EltVT = EVT()); 2272 2273 /// Compute the default alignment value for the given type. 2274 Align getEVTAlign(EVT MemoryVT) const; 2275 2276 /// Test whether the given value is a constant int or similar node. 2277 SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) const; 2278 2279 /// Test whether the given value is a constant FP or similar node. 2280 SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) const ; 2281 2282 /// \returns true if \p N is any kind of constant or build_vector of 2283 /// constants, int or float. If a vector, it may not necessarily be a splat. 2284 inline bool isConstantValueOfAnyType(SDValue N) const { 2285 return isConstantIntBuildVectorOrConstantInt(N) || 2286 isConstantFPBuildVectorOrConstantFP(N); 2287 } 2288 2289 /// Set CallSiteInfo to be associated with Node. 2290 void addCallSiteInfo(const SDNode *Node, CallSiteInfoImpl &&CallInfo) { 2291 SDEI[Node].CSInfo = std::move(CallInfo); 2292 } 2293 /// Return CallSiteInfo associated with Node, or a default if none exists. 2294 CallSiteInfo getCallSiteInfo(const SDNode *Node) { 2295 auto I = SDEI.find(Node); 2296 return I != SDEI.end() ? std::move(I->second).CSInfo : CallSiteInfo(); 2297 } 2298 /// Set HeapAllocSite to be associated with Node. 2299 void addHeapAllocSite(const SDNode *Node, MDNode *MD) { 2300 SDEI[Node].HeapAllocSite = MD; 2301 } 2302 /// Return HeapAllocSite associated with Node, or nullptr if none exists. 2303 MDNode *getHeapAllocSite(const SDNode *Node) const { 2304 auto I = SDEI.find(Node); 2305 return I != SDEI.end() ? I->second.HeapAllocSite : nullptr; 2306 } 2307 /// Set PCSections to be associated with Node. 2308 void addPCSections(const SDNode *Node, MDNode *MD) { 2309 SDEI[Node].PCSections = MD; 2310 } 2311 /// Return PCSections associated with Node, or nullptr if none exists. 2312 MDNode *getPCSections(const SDNode *Node) const { 2313 auto It = SDEI.find(Node); 2314 return It != SDEI.end() ? It->second.PCSections : nullptr; 2315 } 2316 /// Set NoMergeSiteInfo to be associated with Node if NoMerge is true. 2317 void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) { 2318 if (NoMerge) 2319 SDEI[Node].NoMerge = NoMerge; 2320 } 2321 /// Return NoMerge info associated with Node. 2322 bool getNoMergeSiteInfo(const SDNode *Node) const { 2323 auto I = SDEI.find(Node); 2324 return I != SDEI.end() ? I->second.NoMerge : false; 2325 } 2326 2327 /// Copy extra info associated with one node to another. 2328 void copyExtraInfo(SDNode *From, SDNode *To); 2329 2330 /// Return the current function's default denormal handling kind for the given 2331 /// floating point type. 2332 DenormalMode getDenormalMode(EVT VT) const { 2333 return MF->getDenormalMode(EVTToAPFloatSemantics(VT)); 2334 } 2335 2336 bool shouldOptForSize() const; 2337 2338 /// Get the (commutative) neutral element for the given opcode, if it exists. 2339 SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, 2340 SDNodeFlags Flags); 2341 2342 /// Some opcodes may create immediate undefined behavior when used with some 2343 /// values (integer division-by-zero for example). Therefore, these operations 2344 /// are not generally safe to move around or change. 2345 bool isSafeToSpeculativelyExecute(unsigned Opcode) const { 2346 switch (Opcode) { 2347 case ISD::SDIV: 2348 case ISD::SREM: 2349 case ISD::SDIVREM: 2350 case ISD::UDIV: 2351 case ISD::UREM: 2352 case ISD::UDIVREM: 2353 return false; 2354 default: 2355 return true; 2356 } 2357 } 2358 2359 /// Check if the provided node is save to speculatively executed given its 2360 /// current arguments. So, while `udiv` the opcode is not safe to 2361 /// speculatively execute, a given `udiv` node may be if the denominator is 2362 /// known nonzero. 2363 bool isSafeToSpeculativelyExecuteNode(const SDNode *N) const { 2364 switch (N->getOpcode()) { 2365 case ISD::UDIV: 2366 return isKnownNeverZero(N->getOperand(1)); 2367 default: 2368 return isSafeToSpeculativelyExecute(N->getOpcode()); 2369 } 2370 } 2371 2372 SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, 2373 const SDLoc &DLoc); 2374 2375 private: 2376 void InsertNode(SDNode *N); 2377 bool RemoveNodeFromCSEMaps(SDNode *N); 2378 void AddModifiedNodeToCSEMaps(SDNode *N); 2379 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos); 2380 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2, 2381 void *&InsertPos); 2382 SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops, 2383 void *&InsertPos); 2384 SDNode *UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &loc); 2385 2386 void DeleteNodeNotInCSEMaps(SDNode *N); 2387 void DeallocateNode(SDNode *N); 2388 2389 void allnodes_clear(); 2390 2391 /// Look up the node specified by ID in CSEMap. If it exists, return it. If 2392 /// not, return the insertion token that will make insertion faster. This 2393 /// overload is for nodes other than Constant or ConstantFP, use the other one 2394 /// for those. 2395 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos); 2396 2397 /// Look up the node specified by ID in CSEMap. If it exists, return it. If 2398 /// not, return the insertion token that will make insertion faster. Performs 2399 /// additional processing for constant nodes. 2400 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, const SDLoc &DL, 2401 void *&InsertPos); 2402 2403 /// Maps to auto-CSE operations. 2404 std::vector<CondCodeSDNode*> CondCodeNodes; 2405 2406 std::vector<SDNode*> ValueTypeNodes; 2407 std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes; 2408 StringMap<SDNode*> ExternalSymbols; 2409 2410 std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols; 2411 DenseMap<MCSymbol *, SDNode *> MCSymbols; 2412 2413 FlagInserter *Inserter = nullptr; 2414 }; 2415 2416 template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> { 2417 using nodes_iterator = pointer_iterator<SelectionDAG::allnodes_iterator>; 2418 2419 static nodes_iterator nodes_begin(SelectionDAG *G) { 2420 return nodes_iterator(G->allnodes_begin()); 2421 } 2422 2423 static nodes_iterator nodes_end(SelectionDAG *G) { 2424 return nodes_iterator(G->allnodes_end()); 2425 } 2426 }; 2427 2428 } // end namespace llvm 2429 2430 #endif // LLVM_CODEGEN_SELECTIONDAG_H 2431